From 0b639a9c0964145aed08bf2b409bf9427ae0d36e Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 7 Dec 2017 01:58:13 +0200 Subject: [PATCH] state: rework state format Dependencies are now specified in a more natural way: deps: path/to/file1: path/to/file2: And outputs also have md5 sum, that allows us to track them by it and not by separate md5 field in state file, which will allow us to remove strict data <-> data.state relation in the future. Signed-off-by: Ruslan Kuprieiev --- dvc/command/add.py | 5 ++--- dvc/command/merge.py | 10 ++-------- dvc/command/repro.py | 5 +---- dvc/command/run.py | 23 ++++++++--------------- dvc/path/data_item.py | 3 +-- dvc/state_file.py | 21 +++++---------------- 6 files changed, 19 insertions(+), 48 deletions(-) diff --git a/dvc/command/add.py b/dvc/command/add.py index 5e47d0ee8..60c1b8951 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -41,11 +41,10 @@ class CmdAdd(CmdBase): state_file = StateFile(data_item=data_item, cmd=None, - out=[data_item.data.dvc], + out=StateFile.parse_deps_state(self.settings, [data_item.data.dvc]), out_git=[], deps=[], - locked=True, - md5=os.path.basename(data_item.cache.relative)) + locked=True) state_file.save() Logger.debug('State file "{}" was created'.format(data_item.state.relative)) diff --git a/dvc/command/merge.py b/dvc/command/merge.py index eaeab2f28..2a9566344 100644 --- a/dvc/command/merge.py +++ b/dvc/command/merge.py @@ -46,14 +46,8 @@ class CmdMerge(CmdBase): prev_state = StateFile.loads(self.git.get_file_content_before_last_merge(item.state.relative)) curr_state = StateFile.load(item) - state = StateFile(data_item=item, - cmd=curr_state.cmd, - out=curr_state.out, - out_git=curr_state.out_git, - locked=curr_state.locked, - deps=curr_state.deps, - md5=prev_state.md5) - state.save() + curr_state.out[item.data.dvc] = prev_state.out[item.data.dvc] + curr_state.save() CmdCheckout.checkout([item]) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 412d9719f..2544664e7 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -169,10 +169,7 @@ class ReproChange(object): def reproduce_deps(self, data_item_dvc, recursive): result = False - for dep in self.command.deps: - path = dep[StateFile.PARAM_PATH] - md5 = dep[StateFile.PARAM_MD5] - + for path,md5 in self.command.deps.items(): if self.reproduce_dep(path, md5, recursive): result = True diff --git a/dvc/command/run.py b/dvc/command/run.py index 25b9bae62..a1a53b59e 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -24,8 +24,7 @@ class CmdRun(CmdBase): out=self.parsed_args.out, out_git=self.parsed_args.out_git, deps=self.parsed_args.deps, - locked=self.parsed_args.lock, - md5=None) + locked=self.parsed_args.lock) self.run_command(self.settings, state) return self.commit_if_needed('DVC run: {}'.format(state.cmd)) @@ -47,23 +46,17 @@ class CmdRun(CmdBase): Logger.debug('Move output file "{}" to cache dir "{}" and create a hardlink'.format( data_item.data.relative, data_item.cache_dir_abs)) data_item.move_data_to_cache() - return CmdRun._create_state_file(data_item, state, settings, cache_file_exists = True) + return CmdRun._create_state_file(data_item, state, settings) @staticmethod - def _create_state_file(data_item, state, settings, cache_file_exists=False): + def _create_state_file(data_item, state, settings): Logger.debug('Create state file "{}"'.format(data_item.state.relative)) - if cache_file_exists: - md5 = os.path.basename(data_item.cache.relative) - else: - md5 = file_md5(data_item.data.relative)[0] - - state_file = StateFile(data_item, - state.cmd, - state.out, - state.out_git, + state_file = StateFile(data_item=data_item, + cmd=state.cmd, + out=StateFile.parse_deps_state(settings, state.out), + out_git=StateFile.parse_deps_state(settings, state.out_git), locked=state.locked, - deps=StateFile.parse_deps_state(settings, state.deps), - md5=md5) + deps=StateFile.parse_deps_state(settings, state.deps)) state_file.save() return state_file diff --git a/dvc/path/data_item.py b/dvc/path/data_item.py index fe539f84a..277c7b212 100644 --- a/dvc/path/data_item.py +++ b/dvc/path/data_item.py @@ -99,7 +99,7 @@ class DataItem(object): if self._cache_file: file_name = os.path.relpath(os.path.realpath(self._cache_file), cache_dir) else: - file_name = StateFile.load(self).md5 + file_name = StateFile.load(self).out[self.data.dvc] cache_file = os.path.join(cache_dir, file_name) return Path(cache_file, self._git) @@ -112,5 +112,4 @@ class DataItem(object): System.hardlink(self.data.relative, self.cache.relative) os.chmod(self.data.relative, stat.S_IREAD) - StateFile(self, md5=md5).save() LocalStateFile(self).save() diff --git a/dvc/state_file.py b/dvc/state_file.py index 7f439af05..395f623af 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -29,8 +29,6 @@ class StateFile(StateFileBase): PARAM_CMD = 'cmd' PARAM_DEPS = 'deps' - PARAM_PATH = 'path' - PARAM_MD5 = 'md5' PARAM_LOCKED = 'locked' PARAM_OUT = 'out' PARAM_OUT_GIT = 'out-git' @@ -41,8 +39,7 @@ class StateFile(StateFileBase): out=None, out_git=None, deps=None, - locked=None, - md5=None): + locked=None): super(StateFile, self).__init__() self.data_item = data_item @@ -51,22 +48,16 @@ class StateFile(StateFileBase): self.out_git = out_git self.deps = deps self.locked = locked - self.md5 = md5 @staticmethod def parse_deps_state(settings, deps): - state = [] + state = {} for dep in deps: - if isinstance(dep, dict): - dep = dep[StateFile.PARAM_PATH] - if settings.path_factory.is_data_item(dep): item = settings.path_factory.data_item(dep) - state.append({StateFile.PARAM_PATH: item.data.dvc, - StateFile.PARAM_MD5: StateFile.load(item).md5}) + state[item.data.dvc] = StateFile.load(item).out[item.data.dvc] else: - state.append({StateFile.PARAM_PATH: dep, - StateFile.PARAM_MD5: file_md5(os.path.join(settings.git.git_dir_abs, dep))[0]}) + state[dep] = file_md5(os.path.join(settings.git.git_dir_abs, dep))[0] return state @property @@ -80,7 +71,6 @@ class StateFile(StateFileBase): out=data.get(StateFile.PARAM_OUT, None), out_git=data.get(StateFile.PARAM_OUT_GIT, None), deps=data.get(StateFile.PARAM_DEPS, []), - md5=data.get(StateFile.PARAM_MD5, []), locked=data.get(StateFile.PARAM_LOCKED, None)) @staticmethod @@ -98,8 +88,7 @@ class StateFile(StateFileBase): self.PARAM_OUT: self.out, self.PARAM_OUT_GIT: self.out_git, self.PARAM_DEPS: self.deps, - self.PARAM_LOCKED: self.locked, - self.PARAM_MD5: self.md5 + self.PARAM_LOCKED: self.locked } file_dir = os.path.dirname(self.file)