Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-13 15:28:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

state: rework state format

Browse Source 
Dependencies are now specified in a more natural way:

deps:
  path/to/file1: <md5>
  path/to/file2: <md5>

And outputs also have md5 sum, that allows us to track
them by it and not by separate md5 field in state file,
which will allow us to remove strict data <-> data.state
relation in the future.

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2017-12-07 01:58:13 +02:00
parent 2ee71f8687
commit 0b639a9c09
6 changed files with 19 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/add.py
+2 -3
View File
@@ -41,11 +41,10 @@ class CmdAdd(CmdBase):
state_file = StateFile(data_item=data_item,
cmd=None,
out=[data_item.data.dvc],
out=StateFile.parse_deps_state(self.settings, [data_item.data.dvc]),
out_git=[],
deps=[],
locked=True,
md5=os.path.basename(data_item.cache.relative))
locked=True)
state_file.save()
Logger.debug('State file "{}" was created'.format(data_item.state.relative))
dvc/command/merge.py
+2 -8
View File
@@ -46,14 +46,8 @@ class CmdMerge(CmdBase):
prev_state = StateFile.loads(self.git.get_file_content_before_last_merge(item.state.relative))
curr_state = StateFile.load(item)
state = StateFile(data_item=item,
cmd=curr_state.cmd,
out=curr_state.out,
out_git=curr_state.out_git,
locked=curr_state.locked,
deps=curr_state.deps,
md5=prev_state.md5)
state.save()
curr_state.out[item.data.dvc] = prev_state.out[item.data.dvc]
curr_state.save()
CmdCheckout.checkout([item])
dvc/command/repro.py
+1 -4
View File
@@ -169,10 +169,7 @@ class ReproChange(object):
def reproduce_deps(self, data_item_dvc, recursive):
result = False
for dep in self.command.deps:
path = dep[StateFile.PARAM_PATH]
md5 = dep[StateFile.PARAM_MD5]
for path,md5 in self.command.deps.items():
if self.reproduce_dep(path, md5, recursive):
result = True
dvc/command/run.py
+8 -15
View File
@@ -24,8 +24,7 @@ class CmdRun(CmdBase):
out=self.parsed_args.out,
out_git=self.parsed_args.out_git,
deps=self.parsed_args.deps,
locked=self.parsed_args.lock,
md5=None)
locked=self.parsed_args.lock)
self.run_command(self.settings, state)
return self.commit_if_needed('DVC run: {}'.format(state.cmd))
@@ -47,23 +46,17 @@ class CmdRun(CmdBase):
Logger.debug('Move output file "{}" to cache dir "{}" and create a hardlink'.format(
data_item.data.relative, data_item.cache_dir_abs))
data_item.move_data_to_cache()
return CmdRun._create_state_file(data_item, state, settings, cache_file_exists = True)
return CmdRun._create_state_file(data_item, state, settings)
@staticmethod
def _create_state_file(data_item, state, settings, cache_file_exists=False):
def _create_state_file(data_item, state, settings):
Logger.debug('Create state file "{}"'.format(data_item.state.relative))
if cache_file_exists:
md5 = os.path.basename(data_item.cache.relative)
else:
md5 = file_md5(data_item.data.relative)[0]
state_file = StateFile(data_item,
state.cmd,
state.out,
state.out_git,
state_file = StateFile(data_item=data_item,
cmd=state.cmd,
out=StateFile.parse_deps_state(settings, state.out),
out_git=StateFile.parse_deps_state(settings, state.out_git),
locked=state.locked,
deps=StateFile.parse_deps_state(settings, state.deps),
md5=md5)
deps=StateFile.parse_deps_state(settings, state.deps))
state_file.save()
return state_file
dvc/path/data_item.py
+1 -2
View File
@@ -99,7 +99,7 @@ class DataItem(object):
if self._cache_file:
file_name = os.path.relpath(os.path.realpath(self._cache_file), cache_dir)
else:
file_name = StateFile.load(self).md5
file_name = StateFile.load(self).out[self.data.dvc]
cache_file = os.path.join(cache_dir, file_name)
return Path(cache_file, self._git)
@@ -112,5 +112,4 @@ class DataItem(object):
System.hardlink(self.data.relative, self.cache.relative)
os.chmod(self.data.relative, stat.S_IREAD)
StateFile(self, md5=md5).save()
LocalStateFile(self).save()
dvc/state_file.py
+5 -16
View File
@@ -29,8 +29,6 @@ class StateFile(StateFileBase):
PARAM_CMD = 'cmd'
PARAM_DEPS = 'deps'
PARAM_PATH = 'path'
PARAM_MD5 = 'md5'
PARAM_LOCKED = 'locked'
PARAM_OUT = 'out'
PARAM_OUT_GIT = 'out-git'
@@ -41,8 +39,7 @@ class StateFile(StateFileBase):
out=None,
out_git=None,
deps=None,
locked=None,
md5=None):
locked=None):
super(StateFile, self).__init__()
self.data_item = data_item
@@ -51,22 +48,16 @@ class StateFile(StateFileBase):
self.out_git = out_git
self.deps = deps
self.locked = locked
self.md5 = md5
@staticmethod
def parse_deps_state(settings, deps):
state = []
state = {}
for dep in deps:
if isinstance(dep, dict):
dep = dep[StateFile.PARAM_PATH]
if settings.path_factory.is_data_item(dep):
item = settings.path_factory.data_item(dep)
state.append({StateFile.PARAM_PATH: item.data.dvc,
StateFile.PARAM_MD5: StateFile.load(item).md5})
state[item.data.dvc] = StateFile.load(item).out[item.data.dvc]
else:
state.append({StateFile.PARAM_PATH: dep,
StateFile.PARAM_MD5: file_md5(os.path.join(settings.git.git_dir_abs, dep))[0]})
state[dep] = file_md5(os.path.join(settings.git.git_dir_abs, dep))[0]
return state
@property
@@ -80,7 +71,6 @@ class StateFile(StateFileBase):
out=data.get(StateFile.PARAM_OUT, None),
out_git=data.get(StateFile.PARAM_OUT_GIT, None),
deps=data.get(StateFile.PARAM_DEPS, []),
md5=data.get(StateFile.PARAM_MD5, []),
locked=data.get(StateFile.PARAM_LOCKED, None))
@staticmethod
@@ -98,8 +88,7 @@ class StateFile(StateFileBase):
self.PARAM_OUT: self.out,
self.PARAM_OUT_GIT: self.out_git,
self.PARAM_DEPS: self.deps,
self.PARAM_LOCKED: self.locked,
self.PARAM_MD5: self.md5
self.PARAM_LOCKED: self.locked
}
file_dir = os.path.dirname(self.file)