From bbe5e46b5c8524f5ffa43dd1e4e6752e179d37b2 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 5 Dec 2017 20:56:17 +0200 Subject: [PATCH] dvc: remove CommandFile As we're moving to more user friendly single *.dvc file approach, separate command files become uneeded. Signed-off-by: Ruslan Kuprieiev --- dvc/command/add.py | 7 ++-- dvc/command/common/command_file.py | 54 ------------------------------ dvc/command/merge.py | 14 ++++---- dvc/command/repro.py | 11 ++---- dvc/command/run.py | 19 ++++++----- dvc/state_file.py | 34 ++++++++++++++----- 6 files changed, 50 insertions(+), 89 deletions(-) delete mode 100644 dvc/command/common/command_file.py diff --git a/dvc/command/add.py b/dvc/command/add.py index 1003e7924..5e47d0ee8 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -3,7 +3,6 @@ import os from dvc.command.common.base import CmdBase from dvc.logger import Logger from dvc.state_file import StateFile -from dvc.command.common.command_file import CommandFile class CmdAdd(CmdBase): @@ -40,10 +39,12 @@ class CmdAdd(CmdBase): for data_item in targets: Logger.debug('Creating state file for {}'.format(data_item.data.relative)) - command = CommandFile(None, [data_item.data.dvc], [], [], True, None) state_file = StateFile(data_item=data_item, - command=command.dict, + cmd=None, + out=[data_item.data.dvc], + out_git=[], deps=[], + locked=True, md5=os.path.basename(data_item.cache.relative)) state_file.save() Logger.debug('State file "{}" was created'.format(data_item.state.relative)) diff --git a/dvc/command/common/command_file.py b/dvc/command/common/command_file.py deleted file mode 100644 index 4008f2b85..000000000 --- a/dvc/command/common/command_file.py +++ /dev/null @@ -1,54 +0,0 @@ -import yaml - -from dvc.state_file import StateFileBase - - -class CommandFile(StateFileBase): - MAGIC = 'DVC-Command-State' - VERSION = '0.1' - - PARAM_CMD = 'cmd' - PARAM_OUT = 'out' - PARAM_OUT_GIT = 'out-git' - PARAM_DEPS = 'deps' - PARAM_LOCK = 'locked' - - def __init__(self, cmd, out, out_git, deps, locked, fname): - self.cmd = cmd - self.out = out - self.out_git = out_git - self.deps = deps - self.locked = locked - self.fname = fname - - @property - def dict(self): - data = { - self.PARAM_CMD: self.cmd, - self.PARAM_OUT: self.out, - self.PARAM_OUT_GIT: self.out_git, - self.PARAM_DEPS: self.deps, - self.PARAM_LOCK: self.locked - } - - return data - - def dumps(self): - return yaml.dump(self.dict) - - def dump(self, fname): - with open(fname, 'w+') as fd: - fd.write(self.dumps()) - - @staticmethod - def loadd(data, fname=None): - return CommandFile(data.get(CommandFile.PARAM_CMD, None), - data.get(CommandFile.PARAM_OUT, None), - data.get(CommandFile.PARAM_OUT_GIT, None), - data.get(CommandFile.PARAM_DEPS, None), - data.get(CommandFile.PARAM_LOCK, None), - fname) - - @staticmethod - def load(fname): - return CommandFile._load(fname, CommandFile, fname) diff --git a/dvc/command/merge.py b/dvc/command/merge.py index c47cc3223..eaeab2f28 100644 --- a/dvc/command/merge.py +++ b/dvc/command/merge.py @@ -7,7 +7,6 @@ from dvc.state_file import StateFile from dvc.path.data_item import DataItem from dvc.system import System from dvc.command.checkout import CmdCheckout -from dvc.command.run import CommandFile class CmdMerge(CmdBase): @@ -34,11 +33,7 @@ class CmdMerge(CmdBase): items = self.settings.path_factory.to_data_items(flist)[0] for item in items: - state = StateFile.load(item) - if isinstance(state.command, str): - command = CommandFile.load(state.command) - else: - command = CommandFile.loadd(state.command) + command = StateFile.load(item) if not command.cmd and command.locked: targets.append(item) @@ -52,8 +47,11 @@ class CmdMerge(CmdBase): curr_state = StateFile.load(item) state = StateFile(data_item=item, - command=curr_state.command, - deps=curr_state.command, + cmd=curr_state.cmd, + out=curr_state.out, + out_git=curr_state.out_git, + locked=curr_state.locked, + deps=curr_state.deps, md5=prev_state.md5) state.save() diff --git a/dvc/command/repro.py b/dvc/command/repro.py index a3f2b8e25..412d9719f 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -1,7 +1,7 @@ import os import copy -from dvc.command.run import CmdRun, CommandFile +from dvc.command.run import CmdRun from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.state_file import StateFile @@ -88,12 +88,7 @@ class ReproChange(object): self._recursive = recursive self._force = force - self.state = StateFile.load(data_item) - - if isinstance(self.state.command, str): - self.command = CommandFile.load(self.state.command) - else: - self.command = CommandFile.loadd(self.state.command) + self.command = StateFile.load(data_item) if not self.command.cmd and not self.command.locked: msg = 'Error: data item "{}" is not locked, but has no command for reproduction' @@ -174,7 +169,7 @@ class ReproChange(object): def reproduce_deps(self, data_item_dvc, recursive): result = False - for dep in self.state.deps: + for dep in self.command.deps: path = dep[StateFile.PARAM_PATH] md5 = dep[StateFile.PARAM_MD5] diff --git a/dvc/command/run.py b/dvc/command/run.py index b400a5db6..35c23b2b6 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,7 +1,6 @@ import os from dvc.command.common.base import CmdBase -from dvc.command.common.command_file import CommandFile from dvc.exceptions import DvcException from dvc.logger import Logger from dvc.state_file import StateFile @@ -19,12 +18,13 @@ class CmdRun(CmdBase): def run(self): cmd = ' '.join(self.parsed_args.command) - try: - command = CommandFile.load(cmd) - except Exception as exc: - Logger.debug("Failed to load {}: {}".format(cmd, str(exc))) - command = CommandFile(cmd, self.parsed_args.out, self.parsed_args.out_git, - self.parsed_args.deps, self.parsed_args.lock, None) + command = StateFile(data_item=None, + cmd=cmd, + out=self.parsed_args.out, + out_git=self.parsed_args.out_git, + deps=self.parsed_args.deps, + locked=self.parsed_args.lock, + md5=None) self.run_command(self.settings, command) return self.commit_if_needed('DVC run: {}'.format(command.cmd)) @@ -51,7 +51,10 @@ class CmdRun(CmdBase): def _create_state_file(data_item, command, settings): Logger.debug('Create state file "{}"'.format(data_item.state.relative)) state_file = StateFile(data_item=data_item, - command=command.fname if command.fname else command.dict, + cmd=command.cmd, + out=command.out, + out_git=command.out_git, + locked=command.locked, deps=StateFile.parse_deps_state(settings, command.deps), md5=os.path.basename(data_item.cache.relative)) state_file.save() diff --git a/dvc/state_file.py b/dvc/state_file.py index e6ce2cd79..d84a0ce56 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -31,28 +31,40 @@ class StateFile(StateFileBase): MAGIC = 'DVC-State' VERSION = '0.1' - PARAM_COMMAND = 'command' + PARAM_CMD = 'cmd' PARAM_DEPS = 'deps' PARAM_PATH = 'path' PARAM_MD5 = 'md5' + PARAM_LOCKED = 'locked' + PARAM_OUT = 'out' + PARAM_OUT_GIT = 'out-git' def __init__(self, data_item=None, - command=None, + cmd=None, + out=None, + out_git=None, deps=None, + locked=None, md5=None): super(StateFile, self).__init__() self.data_item = data_item - self.command = command + self.cmd = cmd + self.out = out + self.out_git = out_git self.deps = deps + self.locked = locked self.md5 = md5 @staticmethod def parse_deps_state(settings, deps): state = [] for dep in deps: + if isinstance(dep, dict): + dep = dep[StateFile.PARAM_PATH] + if settings.path_factory.is_data_item(dep): item = settings.path_factory.data_item(dep) state.append({StateFile.PARAM_PATH: item.data.dvc, @@ -69,9 +81,12 @@ class StateFile(StateFileBase): @staticmethod def loadd(data): return StateFile(data_item=None, - command=data.get(StateFile.PARAM_COMMAND, None), + cmd=data.get(StateFile.PARAM_CMD, None), + out=data.get(StateFile.PARAM_OUT, None), + out_git=data.get(StateFile.PARAM_OUT_GIT, None), deps=data.get(StateFile.PARAM_DEPS, []), - md5=data.get(StateFile.PARAM_MD5, [])) + md5=data.get(StateFile.PARAM_MD5, []), + locked=data.get(StateFile.PARAM_LOCKED, None)) @staticmethod def load(data_item): @@ -84,9 +99,12 @@ class StateFile(StateFileBase): def save(self): res = { - self.PARAM_COMMAND: self.command, - self.PARAM_DEPS: self.deps, - self.PARAM_MD5: self.md5 + self.PARAM_CMD: self.cmd, + self.PARAM_OUT: self.out, + self.PARAM_OUT_GIT: self.out_git, + self.PARAM_DEPS: self.deps, + self.PARAM_LOCKED: self.locked, + self.PARAM_MD5: self.md5 } file_dir = os.path.dirname(self.file)