diff --git a/dvc/command/add.py b/dvc/command/add.py index 917e64499..1003e7924 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -41,10 +41,10 @@ class CmdAdd(CmdBase): Logger.debug('Creating state file for {}'.format(data_item.data.relative)) command = CommandFile(None, [data_item.data.dvc], [], [], True, None) - state_file = StateFile(data_item, - self.settings, - command.dict, - []) + state_file = StateFile(data_item=data_item, + command=command.dict, + deps=[], + md5=os.path.basename(data_item.cache.relative)) state_file.save() Logger.debug('State file "{}" was created'.format(data_item.state.relative)) diff --git a/dvc/command/init.py b/dvc/command/init.py index 4bbeac7e0..aeaa2f74b 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -91,17 +91,14 @@ ProjectName = config_dir_path = self.get_not_existing_path(Config.CONFIG_DIR) cache_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.CACHE_DIR_NAME) - state_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.STATE_DIR_NAME) conf_file_name = self.get_not_existing_conf_file_name() config_dir_path.mkdir() cache_dir_path.mkdir() - state_dir_path.mkdir() - Logger.info('Directories {}/, {}/, {}/ were created'.format( + Logger.info('Directories {}/, {}/ were created'.format( config_dir_path.name, - os.path.join(config_dir_path.name, cache_dir_path.name), - os.path.join(config_dir_path.name, state_dir_path.name))) + os.path.join(config_dir_path.name, cache_dir_path.name))) conf_file = open(conf_file_name, 'wt') conf_file.write(self.CONFIG_TEMPLATE) @@ -109,7 +106,7 @@ ProjectName = self.git.modify_gitignore([os.path.join(config_dir_path.name, cache_dir_path.name), os.path.join(config_dir_path.name, os.path.basename(self.git.lock_file)), - os.path.join(config_dir_path.name, state_dir_path.name, '*' + DataItem.LOCAL_STATE_FILE_SUFFIX)]) + '*' + DataItem.LOCAL_STATE_FILE_SUFFIX]) - message = 'DVC init. cache dir {}, state dir {}, '.format(cache_dir_path.name, state_dir_path.name) + message = 'DVC init. cache dir {}'.format(cache_dir_path.name) return self.commit_if_needed(message) diff --git a/dvc/command/merge.py b/dvc/command/merge.py index e01f6ca36..c47cc3223 100644 --- a/dvc/command/merge.py +++ b/dvc/command/merge.py @@ -23,8 +23,8 @@ class CmdMerge(CmdBase): dlist = [] flist = self.git.get_last_merge_changed_files() for fname in flist: - if fname.startswith(self.settings.config.state_dir) and fname.endswith(DataItem.CACHE_STATE_FILE_SUFFIX): - data = os.path.relpath(fname, self.settings.config.state_dir)[:-len(DataItem.CACHE_STATE_FILE_SUFFIX)] + if fname.endswith(DataItem.STATE_FILE_SUFFIX): + data = fname[:-len(DataItem.STATE_FILE_SUFFIX)] dlist.append(data) return dlist @@ -34,7 +34,7 @@ class CmdMerge(CmdBase): items = self.settings.path_factory.to_data_items(flist)[0] for item in items: - state = StateFile.load(item, self.settings) + state = StateFile.load(item) if isinstance(state.command, str): command = CommandFile.load(state.command) else: @@ -48,7 +48,14 @@ class CmdMerge(CmdBase): def checkout_targets(self, targets): data = [] for item in targets: - self.git.checkout_file_before_last_merge(item.cache_state.relative) + prev_state = StateFile.loads(self.git.get_file_content_before_last_merge(item.state.relative)) + curr_state = StateFile.load(item) + + state = StateFile(data_item=item, + command=curr_state.command, + deps=curr_state.command, + md5=prev_state.md5) + state.save() CmdCheckout.checkout([item]) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index a9f96027c..1293a1e2c 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -21,8 +21,8 @@ class CmdRemove(Traverse): data_item = self._get_data_item(target) self._remove_cache_file(data_item) - self._remove_state_file(data_item) self._remove_cloud_cache(data_item) + self._remove_state_file(data_item) os.remove(data_item.data.relative) Logger.debug(u'[Cmd-Remove] Remove data item {}. Success.'.format(data_item.data.relative)) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index bdeba08ee..a3f2b8e25 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -88,7 +88,7 @@ class ReproChange(object): self._recursive = recursive self._force = force - self.state = StateFile.load(data_item, self.settings) + self.state = StateFile.load(data_item) if isinstance(self.state.command, str): self.command = CommandFile.load(self.state.command) diff --git a/dvc/command/run.py b/dvc/command/run.py index 193bd0cde..b400a5db6 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,3 +1,5 @@ +import os + from dvc.command.common.base import CmdBase from dvc.command.common.command_file import CommandFile from dvc.exceptions import DvcException @@ -48,9 +50,9 @@ class CmdRun(CmdBase): @staticmethod def _create_state_file(data_item, command, settings): Logger.debug('Create state file "{}"'.format(data_item.state.relative)) - state_file = StateFile(data_item, - settings, - command.fname if command.fname else command.dict, - StateFile.parse_deps_state(settings, command.deps)) + state_file = StateFile(data_item=data_item, + command=command.fname if command.fname else command.dict, + deps=StateFile.parse_deps_state(settings, command.deps), + md5=os.path.basename(data_item.cache.relative)) state_file.save() return state_file diff --git a/dvc/config.py b/dvc/config.py index ff5de8435..f06d31394 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -19,11 +19,9 @@ class ConfigI(object): CONFIG_DIR = '.dvc' TARGET_FILE_DEFAULT = 'target' CONFIG = 'config' - STATE_DIR_NAME = 'state' CACHE_DIR_NAME = 'cache' CACHE_DIR = os.path.join(CONFIG_DIR, CACHE_DIR_NAME) - STATE_DIR = os.path.join(CONFIG_DIR, STATE_DIR_NAME) def __init__(self, cloud=None, conf_parser=None): self._conf_parser = None @@ -40,11 +38,6 @@ class ConfigI(object): """ Directory with cached data files """ return self.CACHE_DIR - @property - def state_dir(self): - """ Directory with state files """ - return self.STATE_DIR - @property def cloud(self): """ Cloud config """ diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py index f9800b04d..d02e4fbe3 100644 --- a/dvc/git_wrapper.py +++ b/dvc/git_wrapper.py @@ -317,6 +317,10 @@ class GitWrapper(GitWrapperI): Logger.debug('[dvc-git] Checkout file {} before last merge. Command: {}'.format(fname, cmd)) Executor.exec_cmd_only_success(cmd.split()) + @staticmethod + def get_file_content_before_last_merge(fname): + return GitWrapper.get_file_content(fname, '@{1}') + @staticmethod def checkout(commit_or_branch, create_new=False): prefix = '-b ' if create_new else '' @@ -427,7 +431,7 @@ class GitWrapper(GitWrapperI): git_cmd = u'git ls-tree --name-only -r {}'.format(commit) lines = Executor.exec_cmd_only_success(git_cmd.split()).split('\n') - return list(filter(lambda s: s.startswith(ConfigI.STATE_DIR), lines)) + return list(filter(lambda s: s.endswith(DataItem.STATE_FILE_SUFFIX), lines)) @staticmethod def branches(regexp): diff --git a/dvc/path/data_item.py b/dvc/path/data_item.py index 46664e6b9..9a35d5293 100644 --- a/dvc/path/data_item.py +++ b/dvc/path/data_item.py @@ -7,7 +7,7 @@ from dvc.exceptions import DvcException from dvc.system import System from dvc.utils import cached_property from dvc.data_cloud import file_md5 -from dvc.state_file import CacheStateFile, LocalStateFile +from dvc.state_file import StateFile, LocalStateFile class DataItemError(DvcException): @@ -33,9 +33,9 @@ class NotInGitDirError(DataDirError): class DataItem(object): - STATE_FILE_SUFFIX = '.state' - LOCAL_STATE_FILE_SUFFIX = '.local_state' - CACHE_STATE_FILE_SUFFIX = '.cache_state' + STATE_FILE_SUFFIX = '.dvc' + LOCAL_STATE_FILE_PREFIX = '.' + LOCAL_STATE_FILE_SUFFIX = '.dvc_local_state' CACHE_FILE_SEP = '_' def __init__(self, data_file, git, config, cache_file=None): @@ -70,17 +70,15 @@ class DataItem(object): def data(self): return self._data - @cached_property - def state_dir(self): - return os.path.join(self._git.git_dir_abs, self._config.state_dir) - - def _state(self, suffix): - state_file = os.path.join(self.state_dir, self.data.dvc + suffix) + def _state(self, prefix, suffix): + state_file = os.path.join(self._git.git_dir_abs, + os.path.dirname(self.data.dvc), + prefix + os.path.basename(self.data.dvc) + suffix) return Path(state_file, self._git) @cached_property def state(self): - return self._state(self.STATE_FILE_SUFFIX) + return self._state('', self.STATE_FILE_SUFFIX) @cached_property def cache_dir_abs(self): @@ -88,11 +86,7 @@ class DataItem(object): @cached_property def local_state(self): - return self._state(self.LOCAL_STATE_FILE_SUFFIX) - - @cached_property - def cache_state(self): - return self._state(self.CACHE_STATE_FILE_SUFFIX) + return self._state(self.LOCAL_STATE_FILE_PREFIX, self.LOCAL_STATE_FILE_SUFFIX) @cached_property def cache_dir(self): @@ -105,15 +99,11 @@ class DataItem(object): if self._cache_file: file_name = os.path.relpath(os.path.realpath(self._cache_file), cache_dir) else: - file_name = CacheStateFile.load(self).md5 + file_name = StateFile.load(self).md5 cache_file = os.path.join(cache_dir, file_name) return Path(cache_file, self._git) - @cached_property - def state_dir_abs(self): - return os.path.join(self._git.git_dir_abs, ConfigI.STATE_DIR) - def move_data_to_cache(self): md5 = file_md5(self.data.relative)[0] self._cache_file = os.path.join(self.cache_dir_abs, md5) @@ -122,6 +112,5 @@ class DataItem(object): System.hardlink(self.data.relative, self.cache.relative) os.chmod(self.data.relative, stat.S_IREAD) - cache_state = CacheStateFile(self).save() - - local_state = LocalStateFile(self).save() + StateFile(self, md5=md5).save() + LocalStateFile(self).save() diff --git a/dvc/path/factory.py b/dvc/path/factory.py index 85da563ab..0793994cc 100644 --- a/dvc/path/factory.py +++ b/dvc/path/factory.py @@ -31,9 +31,7 @@ class PathFactory(object): return self.data_item(path.relative) def is_data_item(self, fname): - data = os.path.relpath(os.path.realpath(fname), self._git.git_dir_abs) - state = os.path.join(ConfigI.STATE_DIR, data + DataItem.STATE_FILE_SUFFIX) - return os.path.isfile(state) + return os.path.isfile(fname + DataItem.STATE_FILE_SUFFIX) def existing_data_item(self, fname): if not self.is_data_item(fname): @@ -57,7 +55,7 @@ class PathFactory(object): def all_existing_data_items(self, subdir='.', cache_exists=True): states = [] - for root, dirs, files in os.walk(os.path.join(ConfigI.STATE_DIR, subdir)): + for root, dirs, files in os.walk(os.path.join(self._git.git_dir_abs, subdir)): for fname in files: path = os.path.join(root, fname) @@ -68,7 +66,7 @@ class PathFactory(object): data_items = self.data_items_from_states(states) if cache_exists: - data_items = filter(lambda i: os.path.exists(i.cache_state.relative), data_items) + data_items = filter(lambda i: os.path.exists(i.state.relative), data_items) return data_items @@ -78,5 +76,4 @@ class PathFactory(object): @staticmethod def state_path_to_dvc_path(state): - filename = os.path.relpath(state, ConfigI.STATE_DIR) - return re.sub(DataItem.STATE_FILE_SUFFIX + '$', '', filename) + return re.sub(DataItem.STATE_FILE_SUFFIX + '$', '', state) diff --git a/dvc/state_file.py b/dvc/state_file.py index 1bb208b50..e6ce2cd79 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -31,43 +31,23 @@ class StateFile(StateFileBase): MAGIC = 'DVC-State' VERSION = '0.1' - DVC_PYTHON_FILE_NAME = 'dvc.py' - DVC_COMMAND = 'dvc' - - PARAM_TYPE = 'Type' - PARAM_VERSION = 'Version' - PARAM_ARGV = 'Argv' - PARAM_CWD = 'Cwd' - PARAM_INPUT_FILES = 'InputFiles' - PARAM_OUTPUT_FILES = 'OutputFiles' - PARAM_CODE_DEPENDENCIES = 'CodeDependencies' - PARAM_LOCKED = 'Locked' - PARAM_STDOUT = "Stdout" - PARAM_STDERR = "Stderr" - PARAM_SHELL = "Shell" - PARAM_TARGET_METRICS = 'TargetMetrics' - TARGET_METRICS_SINGLE_METRIC = 'SingleMetric' - PARAM_COMMAND = 'command' PARAM_DEPS = 'deps' PARAM_PATH = 'path' PARAM_MD5 = 'md5' def __init__(self, - data_item, - settings, - command, - deps, - target_metrics={}): + data_item=None, + command=None, + deps=None, + md5=None): super(StateFile, self).__init__() self.data_item = data_item - self.settings = settings self.command = command self.deps = deps - - self.target_metrics = target_metrics + self.md5 = md5 @staticmethod def parse_deps_state(settings, deps): @@ -76,7 +56,7 @@ class StateFile(StateFileBase): if settings.path_factory.is_data_item(dep): item = settings.path_factory.data_item(dep) state.append({StateFile.PARAM_PATH: item.data.dvc, - StateFile.PARAM_MD5: CacheStateFile.load(item).md5}) + StateFile.PARAM_MD5: StateFile.load(item).md5}) else: state.append({StateFile.PARAM_PATH: dep, StateFile.PARAM_MD5: file_md5(os.path.join(settings.git.git_dir_abs, dep))[0]}) @@ -86,60 +66,27 @@ class StateFile(StateFileBase): def file(self): return self.data_item.state.relative - @property - def single_target_metric(self): - res = self.target_metrics.get(self.TARGET_METRICS_SINGLE_METRIC) - if res is None: - return None - return float(res) - - def update_target_metrics(self): - if not self.data_item: - raise StateFileError('Unable to get target metric: data item is not defined') - - data_file = self.data_item.data.relative - if not os.path.exists(data_file): - raise StateFileError('Unable to get target metric: data item does not exist') - - target_metric = StateFile.try_parse_target_metric(data_file) - if target_metric: - self.target_metrics[StateFile.TARGET_METRICS_SINGLE_METRIC] = target_metric + @staticmethod + def loadd(data): + return StateFile(data_item=None, + command=data.get(StateFile.PARAM_COMMAND, None), + deps=data.get(StateFile.PARAM_DEPS, []), + md5=data.get(StateFile.PARAM_MD5, [])) @staticmethod - def try_parse_target_metric(file_name): - try: - metric = utils.parse_target_metric_file(file_name) - if not metric: - raise StateFileError('Unable to parse metrics from the first line of file {}'.format(file_name)) - return metric - except StateFileError: - return None + def load(data_item): + return StateFile._load(data_item.state.relative, StateFile) @staticmethod - def loadd(data, settings): - return StateFile(None, - settings, - data.get(StateFile.PARAM_COMMAND, None), - data.get(StateFile.PARAM_DEPS, []), - data.get(StateFile.PARAM_TARGET_METRICS, {})) - - @staticmethod - def load(data_item, settings): - return StateFile._load(data_item.state.relative, StateFile, settings) - - @staticmethod - def loads(content, settings): - data = yaml.loads(content) - return StateFile.loadd(data, settings) - - def save(self, is_update_target_metrics=True): - if is_update_target_metrics: - self.update_target_metrics() + def loads(content): + data = yaml.load(content) + return StateFile.loadd(data) + def save(self): res = { self.PARAM_COMMAND: self.command, self.PARAM_DEPS: self.deps, - self.PARAM_TARGET_METRICS: self.target_metrics + self.PARAM_MD5: self.md5 } file_dir = os.path.dirname(self.file) @@ -149,42 +96,6 @@ class StateFile(StateFileBase): self._save(self.file, res) -class CacheStateFile(StateFileBase): - MAGIC = 'DVC-Cache-State' - VERSION = '0.1' - - PARAM_MD5 = StateFile.PARAM_MD5 - - def __init__(self, data_item, md5=None): - super(CacheStateFile, self).__init__() - - self.data_item = data_item - self.md5 = md5 - - if not md5: - self.md5 = file_md5(data_item.data.relative)[0] - - @staticmethod - def loadd(data): - return CacheStateFile(None, - data.get(CacheStateFile.PARAM_MD5, None)) - - @staticmethod - def load(data_item): - return CacheStateFile._load(data_item.cache_state.relative, CacheStateFile) - - def save(self): - res = { - self.PARAM_MD5 : self.md5, - } - - file_dir = os.path.dirname(self.data_item.cache_state.relative) - if not os.path.isdir(file_dir): - os.makedirs(file_dir) - - self._save(self.data_item.cache_state.relative, res) - - class LocalStateFile(StateFileBase): MAGIC = 'DVC-Local-State' VERSION = '0.1' diff --git a/functest/test-add.sh b/functest/test-add.sh index 3ab215cc2..4b0655db8 100755 --- a/functest/test-add.sh +++ b/functest/test-add.sh @@ -9,5 +9,5 @@ dvc_create_repo dvc_info "Add file" cp $DATA_CACHE/foo data/local dvc add data/local -dvc_check_files "data/local .dvc/state/data/local.state .dvc/state/data/local.cache_state" +dvc_check_files "data/local data/local.dvc data/.local.dvc_local_state" dvc_pass diff --git a/functest/test-checkout.sh b/functest/test-checkout.sh index a6efddea9..b58e7c062 100755 --- a/functest/test-checkout.sh +++ b/functest/test-checkout.sh @@ -8,5 +8,5 @@ dvc_create_repo rm -f data/foo dvc checkout -dvc_check_files data/foo +dvc_check_files data/foo data/foo.dvc dvc_pass diff --git a/functest/test-merge.sh b/functest/test-merge.sh index e626d686f..d872c13f0 100755 --- a/functest/test-merge.sh +++ b/functest/test-merge.sh @@ -52,6 +52,6 @@ if [ "$(dvc_md5 data/data)" != "$ORIG_DATA" ]; then dvc_fail fi -grep $(echo "$ORIG_DATA" | cut -d " " -f1) .dvc/state/data/data.cache_state || dvc_fail +grep $(echo "$ORIG_DATA" | cut -d " " -f1) data/data.dvc || dvc_fail dvc_pass diff --git a/functest/test-repro-code.sh b/functest/test-repro-code.sh index ffa814aea..d8e592186 100755 --- a/functest/test-repro-code.sh +++ b/functest/test-repro-code.sh @@ -18,7 +18,7 @@ function test_generic() { dvc_info 'Reproduce foo1' dvc repro data/foo1 - dvc_check_files data/foo1 + dvc_check_files data/foo1 data/foo1.dvc if [ "$(cat data/foo1)" != "foo" ]; then dvc_fail fi @@ -34,7 +34,7 @@ function test_generic() { dvc_info 'Reproduce foo1 as default target' dvc repro - dvc_check_files data/foo1 + dvc_check_files data/foo1 data/foo1.dvc if [ "$(cat data/foo1)" != "bar" ]; then dvc_fail fi