Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-10 07:18:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvc: get rid of state dir and cache_state in favor of single .dvc

Browse Source 
From now on everything is stored in filename + .dvc path, except for local_state.
This allows us to be more explicit and help users see everything they need in relation
to version control of their own data files.

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2017-12-05 20:33:21 +02:00
parent 3322638b1b
commit e853fc76aa
15 changed files with 73 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/add.py
+4 -4
View File
@@ -41,10 +41,10 @@ class CmdAdd(CmdBase):
Logger.debug('Creating state file for {}'.format(data_item.data.relative))
command = CommandFile(None, [data_item.data.dvc], [], [], True, None)
state_file = StateFile(data_item,
self.settings,
command.dict,
[])
state_file = StateFile(data_item=data_item,
command=command.dict,
deps=[],
md5=os.path.basename(data_item.cache.relative))
state_file.save()
Logger.debug('State file "{}" was created'.format(data_item.state.relative))
dvc/command/init.py
+4 -7
View File
@@ -91,17 +91,14 @@ ProjectName =
config_dir_path = self.get_not_existing_path(Config.CONFIG_DIR)
cache_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.CACHE_DIR_NAME)
state_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.STATE_DIR_NAME)
conf_file_name = self.get_not_existing_conf_file_name()
config_dir_path.mkdir()
cache_dir_path.mkdir()
state_dir_path.mkdir()
Logger.info('Directories {}/, {}/, {}/ were created'.format(
Logger.info('Directories {}/, {}/ were created'.format(
config_dir_path.name,
os.path.join(config_dir_path.name, cache_dir_path.name),
os.path.join(config_dir_path.name, state_dir_path.name)))
os.path.join(config_dir_path.name, cache_dir_path.name)))
conf_file = open(conf_file_name, 'wt')
conf_file.write(self.CONFIG_TEMPLATE)
@@ -109,7 +106,7 @@ ProjectName =
self.git.modify_gitignore([os.path.join(config_dir_path.name, cache_dir_path.name),
os.path.join(config_dir_path.name, os.path.basename(self.git.lock_file)),
os.path.join(config_dir_path.name, state_dir_path.name, '*' + DataItem.LOCAL_STATE_FILE_SUFFIX)])
'*' + DataItem.LOCAL_STATE_FILE_SUFFIX])
message = 'DVC init. cache dir {}, state dir {}, '.format(cache_dir_path.name, state_dir_path.name)
message = 'DVC init. cache dir {}'.format(cache_dir_path.name)
return self.commit_if_needed(message)
dvc/command/merge.py
+11 -4
View File
@@ -23,8 +23,8 @@ class CmdMerge(CmdBase):
dlist = []
flist = self.git.get_last_merge_changed_files()
for fname in flist:
if fname.startswith(self.settings.config.state_dir) and fname.endswith(DataItem.CACHE_STATE_FILE_SUFFIX):
data = os.path.relpath(fname, self.settings.config.state_dir)[:-len(DataItem.CACHE_STATE_FILE_SUFFIX)]
if fname.endswith(DataItem.STATE_FILE_SUFFIX):
data = fname[:-len(DataItem.STATE_FILE_SUFFIX)]
dlist.append(data)
return dlist
@@ -34,7 +34,7 @@ class CmdMerge(CmdBase):
items = self.settings.path_factory.to_data_items(flist)[0]
for item in items:
state = StateFile.load(item, self.settings)
state = StateFile.load(item)
if isinstance(state.command, str):
command = CommandFile.load(state.command)
else:
@@ -48,7 +48,14 @@ class CmdMerge(CmdBase):
def checkout_targets(self, targets):
data = []
for item in targets:
self.git.checkout_file_before_last_merge(item.cache_state.relative)
prev_state = StateFile.loads(self.git.get_file_content_before_last_merge(item.state.relative))
curr_state = StateFile.load(item)
state = StateFile(data_item=item,
command=curr_state.command,
deps=curr_state.command,
md5=prev_state.md5)
state.save()
CmdCheckout.checkout([item])
dvc/command/remove.py
+1 -1
View File
@@ -21,8 +21,8 @@ class CmdRemove(Traverse):
data_item = self._get_data_item(target)
self._remove_cache_file(data_item)
self._remove_state_file(data_item)
self._remove_cloud_cache(data_item)
self._remove_state_file(data_item)
os.remove(data_item.data.relative)
Logger.debug(u'[Cmd-Remove] Remove data item {}. Success.'.format(data_item.data.relative))
dvc/command/repro.py
+1 -1
View File
@@ -88,7 +88,7 @@ class ReproChange(object):
self._recursive = recursive
self._force = force
self.state = StateFile.load(data_item, self.settings)
self.state = StateFile.load(data_item)
if isinstance(self.state.command, str):
self.command = CommandFile.load(self.state.command)
dvc/command/run.py
+6 -4
View File
@@ -1,3 +1,5 @@
import os
from dvc.command.common.base import CmdBase
from dvc.command.common.command_file import CommandFile
from dvc.exceptions import DvcException
@@ -48,9 +50,9 @@ class CmdRun(CmdBase):
@staticmethod
def _create_state_file(data_item, command, settings):
Logger.debug('Create state file "{}"'.format(data_item.state.relative))
state_file = StateFile(data_item,
settings,
command.fname if command.fname else command.dict,
StateFile.parse_deps_state(settings, command.deps))
state_file = StateFile(data_item=data_item,
command=command.fname if command.fname else command.dict,
deps=StateFile.parse_deps_state(settings, command.deps),
md5=os.path.basename(data_item.cache.relative))
state_file.save()
return state_file
dvc/config.py
-7
View File
@@ -19,11 +19,9 @@ class ConfigI(object):
CONFIG_DIR = '.dvc'
TARGET_FILE_DEFAULT = 'target'
CONFIG = 'config'
STATE_DIR_NAME = 'state'
CACHE_DIR_NAME = 'cache'
CACHE_DIR = os.path.join(CONFIG_DIR, CACHE_DIR_NAME)
STATE_DIR = os.path.join(CONFIG_DIR, STATE_DIR_NAME)
def __init__(self, cloud=None, conf_parser=None):
self._conf_parser = None
@@ -40,11 +38,6 @@ class ConfigI(object):
""" Directory with cached data files """
return self.CACHE_DIR
@property
def state_dir(self):
""" Directory with state files """
return self.STATE_DIR
@property
def cloud(self):
""" Cloud config """
dvc/git_wrapper.py
+5 -1
View File
@@ -317,6 +317,10 @@ class GitWrapper(GitWrapperI):
Logger.debug('[dvc-git] Checkout file {} before last merge. Command: {}'.format(fname, cmd))
Executor.exec_cmd_only_success(cmd.split())
@staticmethod
def get_file_content_before_last_merge(fname):
return GitWrapper.get_file_content(fname, '@{1}')
@staticmethod
def checkout(commit_or_branch, create_new=False):
prefix = '-b ' if create_new else ''
@@ -427,7 +431,7 @@ class GitWrapper(GitWrapperI):
git_cmd = u'git ls-tree --name-only -r {}'.format(commit)
lines = Executor.exec_cmd_only_success(git_cmd.split()).split('\n')
return list(filter(lambda s: s.startswith(ConfigI.STATE_DIR), lines))
return list(filter(lambda s: s.endswith(DataItem.STATE_FILE_SUFFIX), lines))
@staticmethod
def branches(regexp):
dvc/path/data_item.py
+13 -24
View File
@@ -7,7 +7,7 @@ from dvc.exceptions import DvcException
from dvc.system import System
from dvc.utils import cached_property
from dvc.data_cloud import file_md5
from dvc.state_file import CacheStateFile, LocalStateFile
from dvc.state_file import StateFile, LocalStateFile
class DataItemError(DvcException):
@@ -33,9 +33,9 @@ class NotInGitDirError(DataDirError):
class DataItem(object):
STATE_FILE_SUFFIX = '.state'
LOCAL_STATE_FILE_SUFFIX = '.local_state'
CACHE_STATE_FILE_SUFFIX = '.cache_state'
STATE_FILE_SUFFIX = '.dvc'
LOCAL_STATE_FILE_PREFIX = '.'
LOCAL_STATE_FILE_SUFFIX = '.dvc_local_state'
CACHE_FILE_SEP = '_'
def __init__(self, data_file, git, config, cache_file=None):
@@ -70,17 +70,15 @@ class DataItem(object):
def data(self):
return self._data
@cached_property
def state_dir(self):
return os.path.join(self._git.git_dir_abs, self._config.state_dir)
def _state(self, suffix):
state_file = os.path.join(self.state_dir, self.data.dvc + suffix)
def _state(self, prefix, suffix):
state_file = os.path.join(self._git.git_dir_abs,
os.path.dirname(self.data.dvc),
prefix + os.path.basename(self.data.dvc) + suffix)
return Path(state_file, self._git)
@cached_property
def state(self):
return self._state(self.STATE_FILE_SUFFIX)
return self._state('', self.STATE_FILE_SUFFIX)
@cached_property
def cache_dir_abs(self):
@@ -88,11 +86,7 @@ class DataItem(object):
@cached_property
def local_state(self):
return self._state(self.LOCAL_STATE_FILE_SUFFIX)
@cached_property
def cache_state(self):
return self._state(self.CACHE_STATE_FILE_SUFFIX)
return self._state(self.LOCAL_STATE_FILE_PREFIX, self.LOCAL_STATE_FILE_SUFFIX)
@cached_property
def cache_dir(self):
@@ -105,15 +99,11 @@ class DataItem(object):
if self._cache_file:
file_name = os.path.relpath(os.path.realpath(self._cache_file), cache_dir)
else:
file_name = CacheStateFile.load(self).md5
file_name = StateFile.load(self).md5
cache_file = os.path.join(cache_dir, file_name)
return Path(cache_file, self._git)
@cached_property
def state_dir_abs(self):
return os.path.join(self._git.git_dir_abs, ConfigI.STATE_DIR)
def move_data_to_cache(self):
md5 = file_md5(self.data.relative)[0]
self._cache_file = os.path.join(self.cache_dir_abs, md5)
@@ -122,6 +112,5 @@ class DataItem(object):
System.hardlink(self.data.relative, self.cache.relative)
os.chmod(self.data.relative, stat.S_IREAD)
cache_state = CacheStateFile(self).save()
local_state = LocalStateFile(self).save()
StateFile(self, md5=md5).save()
LocalStateFile(self).save()
dvc/path/factory.py
+4 -7
View File
@@ -31,9 +31,7 @@ class PathFactory(object):
return self.data_item(path.relative)
def is_data_item(self, fname):
data = os.path.relpath(os.path.realpath(fname), self._git.git_dir_abs)
state = os.path.join(ConfigI.STATE_DIR, data + DataItem.STATE_FILE_SUFFIX)
return os.path.isfile(state)
return os.path.isfile(fname + DataItem.STATE_FILE_SUFFIX)
def existing_data_item(self, fname):
if not self.is_data_item(fname):
@@ -57,7 +55,7 @@ class PathFactory(object):
def all_existing_data_items(self, subdir='.', cache_exists=True):
states = []
for root, dirs, files in os.walk(os.path.join(ConfigI.STATE_DIR, subdir)):
for root, dirs, files in os.walk(os.path.join(self._git.git_dir_abs, subdir)):
for fname in files:
path = os.path.join(root, fname)
@@ -68,7 +66,7 @@ class PathFactory(object):
data_items = self.data_items_from_states(states)
if cache_exists:
data_items = filter(lambda i: os.path.exists(i.cache_state.relative), data_items)
data_items = filter(lambda i: os.path.exists(i.state.relative), data_items)
return data_items
@@ -78,5 +76,4 @@ class PathFactory(object):
@staticmethod
def state_path_to_dvc_path(state):
filename = os.path.relpath(state, ConfigI.STATE_DIR)
return re.sub(DataItem.STATE_FILE_SUFFIX + '$', '', filename)
return re.sub(DataItem.STATE_FILE_SUFFIX + '$', '', state)
dvc/state_file.py
+19 -108
View File
@@ -31,43 +31,23 @@ class StateFile(StateFileBase):
MAGIC = 'DVC-State'
VERSION = '0.1'
DVC_PYTHON_FILE_NAME = 'dvc.py'
DVC_COMMAND = 'dvc'
PARAM_TYPE = 'Type'
PARAM_VERSION = 'Version'
PARAM_ARGV = 'Argv'
PARAM_CWD = 'Cwd'
PARAM_INPUT_FILES = 'InputFiles'
PARAM_OUTPUT_FILES = 'OutputFiles'
PARAM_CODE_DEPENDENCIES = 'CodeDependencies'
PARAM_LOCKED = 'Locked'
PARAM_STDOUT = "Stdout"
PARAM_STDERR = "Stderr"
PARAM_SHELL = "Shell"
PARAM_TARGET_METRICS = 'TargetMetrics'
TARGET_METRICS_SINGLE_METRIC = 'SingleMetric'
PARAM_COMMAND = 'command'
PARAM_DEPS = 'deps'
PARAM_PATH = 'path'
PARAM_MD5 = 'md5'
def __init__(self,
data_item,
settings,
command,
deps,
target_metrics={}):
data_item=None,
command=None,
deps=None,
md5=None):
super(StateFile, self).__init__()
self.data_item = data_item
self.settings = settings
self.command = command
self.deps = deps
self.target_metrics = target_metrics
self.md5 = md5
@staticmethod
def parse_deps_state(settings, deps):
@@ -76,7 +56,7 @@ class StateFile(StateFileBase):
if settings.path_factory.is_data_item(dep):
item = settings.path_factory.data_item(dep)
state.append({StateFile.PARAM_PATH: item.data.dvc,
StateFile.PARAM_MD5: CacheStateFile.load(item).md5})
StateFile.PARAM_MD5: StateFile.load(item).md5})
else:
state.append({StateFile.PARAM_PATH: dep,
StateFile.PARAM_MD5: file_md5(os.path.join(settings.git.git_dir_abs, dep))[0]})
@@ -86,60 +66,27 @@ class StateFile(StateFileBase):
def file(self):
return self.data_item.state.relative
@property
def single_target_metric(self):
res = self.target_metrics.get(self.TARGET_METRICS_SINGLE_METRIC)
if res is None:
return None
return float(res)
def update_target_metrics(self):
if not self.data_item:
raise StateFileError('Unable to get target metric: data item is not defined')
data_file = self.data_item.data.relative
if not os.path.exists(data_file):
raise StateFileError('Unable to get target metric: data item does not exist')
target_metric = StateFile.try_parse_target_metric(data_file)
if target_metric:
self.target_metrics[StateFile.TARGET_METRICS_SINGLE_METRIC] = target_metric
@staticmethod
def loadd(data):
return StateFile(data_item=None,
command=data.get(StateFile.PARAM_COMMAND, None),
deps=data.get(StateFile.PARAM_DEPS, []),
md5=data.get(StateFile.PARAM_MD5, []))
@staticmethod
def try_parse_target_metric(file_name):
try:
metric = utils.parse_target_metric_file(file_name)
if not metric:
raise StateFileError('Unable to parse metrics from the first line of file {}'.format(file_name))
return metric
except StateFileError:
return None
def load(data_item):
return StateFile._load(data_item.state.relative, StateFile)
@staticmethod
def loadd(data, settings):
return StateFile(None,
settings,
data.get(StateFile.PARAM_COMMAND, None),
data.get(StateFile.PARAM_DEPS, []),
data.get(StateFile.PARAM_TARGET_METRICS, {}))
@staticmethod
def load(data_item, settings):
return StateFile._load(data_item.state.relative, StateFile, settings)
@staticmethod
def loads(content, settings):
data = yaml.loads(content)
return StateFile.loadd(data, settings)
def save(self, is_update_target_metrics=True):
if is_update_target_metrics:
self.update_target_metrics()
def loads(content):
data = yaml.load(content)
return StateFile.loadd(data)
def save(self):
res = {
self.PARAM_COMMAND: self.command,
self.PARAM_DEPS: self.deps,
self.PARAM_TARGET_METRICS: self.target_metrics
self.PARAM_MD5: self.md5
}
file_dir = os.path.dirname(self.file)
@@ -149,42 +96,6 @@ class StateFile(StateFileBase):
self._save(self.file, res)
class CacheStateFile(StateFileBase):
MAGIC = 'DVC-Cache-State'
VERSION = '0.1'
PARAM_MD5 = StateFile.PARAM_MD5
def __init__(self, data_item, md5=None):
super(CacheStateFile, self).__init__()
self.data_item = data_item
self.md5 = md5
if not md5:
self.md5 = file_md5(data_item.data.relative)[0]
@staticmethod
def loadd(data):
return CacheStateFile(None,
data.get(CacheStateFile.PARAM_MD5, None))
@staticmethod
def load(data_item):
return CacheStateFile._load(data_item.cache_state.relative, CacheStateFile)
def save(self):
res = {
self.PARAM_MD5 : self.md5,
}
file_dir = os.path.dirname(self.data_item.cache_state.relative)
if not os.path.isdir(file_dir):
os.makedirs(file_dir)
self._save(self.data_item.cache_state.relative, res)
class LocalStateFile(StateFileBase):
MAGIC = 'DVC-Local-State'
VERSION = '0.1'
functest/test-add.sh
+1 -1
View File
@@ -9,5 +9,5 @@ dvc_create_repo
dvc_info "Add file"
cp $DATA_CACHE/foo data/local
dvc add data/local
dvc_check_files "data/local .dvc/state/data/local.state .dvc/state/data/local.cache_state"
dvc_check_files "data/local data/local.dvc data/.local.dvc_local_state"
dvc_pass
functest/test-checkout.sh
+1 -1
View File
@@ -8,5 +8,5 @@ dvc_create_repo
rm -f data/foo
dvc checkout
dvc_check_files data/foo
dvc_check_files data/foo data/foo.dvc
dvc_pass
functest/test-merge.sh
+1 -1
View File
@@ -52,6 +52,6 @@ if [ "$(dvc_md5 data/data)" != "$ORIG_DATA" ]; then
dvc_fail
fi
grep $(echo "$ORIG_DATA" | cut -d " " -f1) .dvc/state/data/data.cache_state || dvc_fail
grep $(echo "$ORIG_DATA" | cut -d " " -f1) data/data.dvc || dvc_fail
dvc_pass
functest/test-repro-code.sh
+2 -2
View File
@@ -18,7 +18,7 @@ function test_generic() {
dvc_info 'Reproduce foo1'
dvc repro data/foo1
dvc_check_files data/foo1
dvc_check_files data/foo1 data/foo1.dvc
if [ "$(cat data/foo1)" != "foo" ]; then
dvc_fail
fi
@@ -34,7 +34,7 @@ function test_generic() {
dvc_info 'Reproduce foo1 as default target'
dvc repro
dvc_check_files data/foo1
dvc_check_files data/foo1 data/foo1.dvc
if [ "$(cat data/foo1)" != "bar" ]; then
dvc_fail
fi