diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index 5e9fa3ab9..eb51a824b 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -35,16 +35,12 @@ class CmdCheckout(CmdBase): def run(self): with DvcLock(self.is_locker, self.git): - curr_commit = self.git.curr_branch_or_commit - prev_items = [] try: - self.git.checkout_previous() - prev_items = self.settings.path_factory.all_existing_data_items() + states = self.git.state_files_for_previous_commit() + prev_items = self.settings.path_factory.data_items_from_states(states, existing=False) except Exception as ex: - Logger.error(u'Unable to get '.format(ex)) + Logger.error(u'Unable to get data files from previous commit: {}'.format(ex)) return 1 - finally: - self.git.checkout(curr_commit) curr_items = self.settings.path_factory.all_existing_data_items() self.checkout(curr_items) @@ -55,10 +51,15 @@ class CmdCheckout(CmdBase): @staticmethod def remove_files(removed_items_set): for item in removed_items_set: - Logger.info(u'Remove \'{}\''.format(item.data.relative)) - os.remove(item.data.relative) - dir = os.path.dirname(item.data.relative) - if not os.listdir(dir): - Logger.info(u'Remove directory \'{}\''.format(dir)) - os.removedirs(dir) + path = item.data.relative + if not os.path.exists(path): + print(u'Remove \'{}\'.'.format(path)) + Logger.warn(u'File \'{}\' does not exist'.format(path)) + else: + Logger.info(u'Remove \'{}\''.format(path)) + os.remove(path) + dir = os.path.dirname(path) + if not os.listdir(dir): + Logger.info(u'Remove directory \'{}\''.format(dir)) + os.removedirs(dir) pass diff --git a/dvc/command/gc.py b/dvc/command/gc.py index 88142e264..ed39afc6a 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -4,6 +4,8 @@ from dvc.command.base import DvcLock from dvc.command.traverse import Traverse from dvc.logger import Logger from dvc.path.data_item import DataItemError +from dvc.config import ConfigI + class CmdGC(Traverse): def __init__(self, settings): @@ -16,8 +18,8 @@ class CmdGC(Traverse): Logger.error('Failed to collect used cache') return 1 - for cache in os.listdir(self.settings.config.cache_dir): - fname = os.path.join(self.settings.config.cache_dir, cache) + for cache in os.listdir(ConfigI.CACHE_DIR): + fname = os.path.join(ConfigI.CACHE_DIR, cache) if fname in self.clist: continue os.remove(fname) diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 9bcb3d304..ff9e53126 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -107,8 +107,8 @@ class CmdImportFile(CmdBase): raise ImportFileError('Output file directory "{}" does not exists'.format( os.path.dirname(data_item.data.relative))) - if not os.path.exists(data_item.cache_dir): - os.makedirs(data_item.cache_dir) + if not os.path.exists(data_item.cache_dir_abs): + os.makedirs(data_item.cache_dir_abs) data_targets.append((input, data_item)) diff --git a/dvc/command/init.py b/dvc/command/init.py index 0af442928..b5e3b5e40 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -85,8 +85,8 @@ ProjectName = return 1 config_dir_path = self.get_not_existing_path(Config.CONFIG_DIR) - cache_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.CACHE_DIR) - state_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.STATE_DIR) + cache_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.CACHE_DIR_NAME) + state_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.STATE_DIR_NAME) conf_file_name = self.get_not_existing_conf_file_name() diff --git a/dvc/command/merge.py b/dvc/command/merge.py index 35795bd0d..ed30906e9 100644 --- a/dvc/command/merge.py +++ b/dvc/command/merge.py @@ -2,6 +2,7 @@ import os import json from dvc.command.base import CmdBase, DvcLock +from dvc.config import ConfigI from dvc.logger import Logger from dvc.state_file import StateFile from dvc.path.data_item import DataItem @@ -20,8 +21,8 @@ class CmdMerge(CmdBase): dlist = [] flist = self.git.get_last_merge_changed_files() for fname in flist: - if fname.startswith(self.settings.config.state_dir) and fname.endswith(DataItem.STATE_FILE_SUFFIX): - data = os.path.relpath(fname, self.settings.config.state_dir)[:-len(DataItem.STATE_FILE_SUFFIX)] + if fname.startswith(ConfigI.STATE_DIR) and fname.endswith(DataItem.STATE_FILE_SUFFIX): + data = os.path.relpath(fname, ConfigI.STATE_DIR)[:-len(DataItem.STATE_FILE_SUFFIX)] dlist.append(data) return dlist diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 990cbb953..ff64210b3 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -156,7 +156,7 @@ class ReproChange(object): Logger.debug('Removing output file {} before reproduction.'.format(output_dvc)) try: - data_item = self.cmd_obj.settings.path_factory.existing_data_item_from_dvc_path(output_dvc) + data_item = self.cmd_obj.settings.path_factory.data_item_from_dvc_path(output_dvc) os.remove(data_item.data.relative) except Exception as ex: msg = 'Data item {} cannot be removed before reproduction: {}' diff --git a/dvc/command/run.py b/dvc/command/run.py index 498d1b2a5..30c081277 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -117,7 +117,7 @@ class CmdRun(CmdBase): result = [] for data_item in repo_change.changed_data_items: Logger.debug('Move output file "{}" to cache dir "{}" and create a hardlink'.format( - data_item.data.relative, data_item.cache_dir)) + data_item.data.relative, data_item.cache_dir_abs)) data_item.move_data_to_cache() Logger.debug('Create state file "{}"'.format(data_item.state.relative)) diff --git a/dvc/config.py b/dvc/config.py index cff3b41e0..af9b97ca7 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -19,8 +19,11 @@ class ConfigI(object): CONFIG_DIR = '.dvc' TARGET_FILE_DEFAULT = 'target' CONFIG = 'config' - STATE_DIR = 'state' - CACHE_DIR = 'cache' + STATE_DIR_NAME = 'state' + CACHE_DIR_NAME = 'cache' + + CACHE_DIR = os.path.join(CONFIG_DIR, CACHE_DIR_NAME) + STATE_DIR = os.path.join(CONFIG_DIR, STATE_DIR_NAME) def __init__(self, cloud=None, conf_parser=None): self._conf_parser = None @@ -32,15 +35,15 @@ class ConfigI(object): self._cloud = cloud self._conf_parser = conf_parser - @property - def cache_dir(self): - """ Directory with cached data files """ - return os.path.join(self.CONFIG_DIR, self.CACHE_DIR) - - @property - def state_dir(self): - """ Directory with state files """ - return os.path.join(self.CONFIG_DIR, self.STATE_DIR) + # @classmethod + # def CACHE_DIR(self): + # """ Directory with cached data files """ + # return os.path.join(self.CONFIG_DIR, self.CACHE_DIR_NAME) + # + # @classmethod + # def STATE_DIR(self): + # """ Directory with state files """ + # return os.path.join(self.CONFIG_DIR, self.STATE_DIR_NAME) @property def cloud(self): diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py index 29e590dc3..05ac13b9f 100644 --- a/dvc/git_wrapper.py +++ b/dvc/git_wrapper.py @@ -4,7 +4,7 @@ from collections import defaultdict from dvc.exceptions import DvcException from dvc.logger import Logger -from dvc.config import Config +from dvc.config import Config, ConfigI from dvc.executor import Executor, ExecutorError from dvc.path.data_item import DataItemError from dvc.state_file import StateFile @@ -423,3 +423,24 @@ class GitWrapper(GitWrapperI): for (hash, branch) in items: result[hash].append(branch) return result + + @staticmethod + def git_prev_commit(): + cmd = 'git check-ref-format --branch @{-1}' + code, out, err = Executor.exec_cmd(cmd.split()) + if code != 0: + return '' + return out + + @staticmethod + def state_files_for_previous_commit(): + commit = GitWrapper.git_prev_commit() + Logger.debug(u'[dvc-git] Previous commit "{}"'.format(commit)) + + if not commit: + Logger.debug(u'[dvc-git] Previous commit does not exist') + return [] + + git_cmd = u'git ls-tree --name-only -r {}'.format(commit) + lines = Executor.exec_cmd_only_success(git_cmd.split()).split('\n') + return filter(lambda s: s.startswith(ConfigI.STATE_DIR), lines) diff --git a/dvc/path/data_item.py b/dvc/path/data_item.py index ccfba0c2c..e1e9df32d 100644 --- a/dvc/path/data_item.py +++ b/dvc/path/data_item.py @@ -1,7 +1,7 @@ import os import stat -import shutil +from dvc.config import ConfigI from dvc.path.path import Path from dvc.exceptions import DvcException from dvc.system import System @@ -71,17 +71,17 @@ class DataItem(object): @cached_property def state(self): - state_dir = os.path.join(self._git.git_dir_abs, self._config.state_dir) + state_dir = os.path.join(self._git.git_dir_abs, ConfigI.STATE_DIR) state_file = os.path.join(state_dir, self.data.dvc + self.STATE_FILE_SUFFIX) return Path(state_file, self._git) @cached_property - def cache_dir(self): - return os.path.join(self._git.git_dir_abs, self._config.cache_dir) + def cache_dir_abs(self): + return os.path.join(self._git.git_dir_abs, ConfigI.CACHE_DIR) @cached_property def cache(self): - cache_dir = self.cache_dir + cache_dir = self.cache_dir_abs if self._cache_file: file_name = os.path.relpath(os.path.realpath(self._cache_file), cache_dir) @@ -94,11 +94,11 @@ class DataItem(object): @cached_property def state_dir_abs(self): - return os.path.join(self._git.git_dir_abs, self._config.state_dir) + return os.path.join(self._git.git_dir_abs, ConfigI.STATE_DIR) def move_data_to_cache(self): md5 = file_md5(self.data.relative)[0] - self._cache_file = os.path.join(self.cache_dir, md5) + self._cache_file = os.path.join(self.cache_dir_abs, md5) self._git.modify_gitignore([self.data.relative]) if not os.path.isfile(self.cache.relative): System.hardlink(self.data.relative, self.cache.relative) diff --git a/dvc/path/factory.py b/dvc/path/factory.py index f79ddd0f6..4f7d8a591 100644 --- a/dvc/path/factory.py +++ b/dvc/path/factory.py @@ -1,5 +1,6 @@ import os +from dvc.config import ConfigI from dvc.path.data_item import DataItem, DataItemError, DataDirError from dvc.path.path import Path from dvc.path.stated_data_item import StatedDataItem @@ -21,13 +22,16 @@ class PathFactory(object): def stated_data_item(self, state, data_file): return StatedDataItem(state, data_file, self._git, self._config) - def existing_data_item_from_dvc_path(self, dvc_path): + def data_item_from_dvc_path(self, dvc_path, existing=True): path = Path.from_dvc_path(dvc_path, self._git) - return self.existing_data_item(path.relative) + if existing: + return self.existing_data_item(path.relative) + else: + return self.data_item(path.relative) def is_data_item(self, fname): data = os.path.relpath(os.path.realpath(fname), self._git.git_dir_abs) - state = os.path.join(self._config.state_dir, data + DataItem.STATE_FILE_SUFFIX) + state = os.path.join(ConfigI.STATE_DIR, data + DataItem.STATE_FILE_SUFFIX) return os.path.isfile(state) def existing_data_item(self, fname): @@ -50,10 +54,9 @@ class PathFactory(object): return result, externally_created_files def all_existing_data_items(self, subdir='.'): - items = [] states = [] - for root, dirs, files in os.walk(os.path.join(self._config.state_dir, subdir)): + for root, dirs, files in os.walk(os.path.join(ConfigI.STATE_DIR, subdir)): for fname in files: path = os.path.join(root, fname) @@ -63,8 +66,12 @@ class PathFactory(object): states.append(path) - for state in states: - data = os.path.relpath(state, self._config.state_dir)[:-len(DataItem.STATE_FILE_SUFFIX)] - items.append(self.existing_data_item_from_dvc_path(data)) + return self.data_items_from_states(states) - return items + def data_items_from_states(self, states, existing=True): + return map(lambda s: self.data_item_from_dvc_path(self.state_path_to_dvc_path(s), existing), + states) + + @staticmethod + def state_path_to_dvc_path(state): + return os.path.relpath(state, ConfigI.STATE_DIR)[:-len(DataItem.STATE_FILE_SUFFIX)] diff --git a/tests/basic_env.py b/tests/basic_env.py index ebfdab9fd..1d01f1c0d 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -34,8 +34,8 @@ class BasicEnvironment(TestCase): os.makedirs(self._test_git_dir) data_dir = os.path.join(self._test_git_dir, 'data') - cache_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR) - state_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data') + cache_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME) + state_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data') os.makedirs(data_dir) os.makedirs(cache_dir) @@ -103,13 +103,13 @@ class DirHierarchyEnvironment(BasicEnvironment): def crate_data_item(self, data_file, cache_file=True, content='random text'): file_result = os.path.join('data', data_file) - state_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, file_result + DataItem.STATE_FILE_SUFFIX) + state_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, file_result + DataItem.STATE_FILE_SUFFIX) dummy_md5 = os.path.basename(data_file) + DataItem.CACHE_FILE_SEP + self._commit self.create_content_file(state_result, '{"Md5" : "' + dummy_md5 + '", "Command" : "run", "Cwd" : "dir"}') if cache_file: - cache_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dummy_md5) + cache_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, dummy_md5) self.create_content_file(cache_result, content) System.hardlink(cache_result, file_result) @@ -121,4 +121,4 @@ class DirHierarchyEnvironment(BasicEnvironment): def create_dirs(self, dir): os.makedirs(os.path.join('data', dir)) - os.makedirs(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data', dir)) + os.makedirs(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', dir)) diff --git a/tests/test_cmd_run.py b/tests/test_cmd_run.py index 8c2772390..cda6a5a9f 100644 --- a/tests/test_cmd_run.py +++ b/tests/test_cmd_run.py @@ -23,8 +23,8 @@ class RunBasicTest(TestCase): os.chdir(self.test_dir) os.mkdir('data') os.mkdir(ConfigI.CONFIG_DIR) - os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR)) - os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR)) + os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME)) + os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME)) self.init_git_repo() self.git = GitWrapper() @@ -37,7 +37,7 @@ class RunBasicTest(TestCase): def init_git_repo(self): Executor.exec_cmd_only_success(['git', 'init']) - msg = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR) + msg = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME) msg += '\n' msg += os.path.join(ConfigI.CONFIG_DIR, '.' + ConfigI.CONFIG + '.lock') self.create_file('.gitignore', msg) @@ -72,8 +72,8 @@ class RunTwoFilesBase(RunBasicTest): stdout=self.file_name1, stderr=self.file_name2) - self.state_file_name1 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, self.file_name1 + DataItem.STATE_FILE_SUFFIX) - self.state_file_name2 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, self.file_name2 + DataItem.STATE_FILE_SUFFIX) + self.state_file_name1 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, self.file_name1 + DataItem.STATE_FILE_SUFFIX) + self.state_file_name2 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, self.file_name2 + DataItem.STATE_FILE_SUFFIX) self.state_file1 = None self.state_file2 = None diff --git a/tests/test_data_file_obj.py b/tests/test_data_file_obj.py index cc7ea821a..1b8c748b6 100644 --- a/tests/test_data_file_obj.py +++ b/tests/test_data_file_obj.py @@ -16,7 +16,7 @@ class TestDataFileObjBasic(BasicEnvironment): config = ConfigI() file = os.path.join('data', 'file.txt') - self.cache_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'file.txt_ad45ba8') + self.cache_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file.txt_ad45ba8') self._data_path = DataItem(file, git, config, cache_file=self.cache_file_full_path) pass @@ -28,7 +28,7 @@ class TestDataFileObjBasic(BasicEnvironment): self.assertEqual(self._data_path.cache.abs, self.cache_file_full_path) def test_state_file(self): - state_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data', 'file.txt.state') + state_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file.txt.state') self.assertEqual(self._data_path.state.abs, state_file_full_path) @@ -40,7 +40,7 @@ class TestDataItemWithGivenCache(BasicEnvironment): self._config = ConfigI('data') self._file = os.path.join('data', 'file.txt') - self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'file.txt_abcd') + self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file.txt_abcd') self._data_item = DataItem(self._file, self._git, self._config, self._cache) pass @@ -58,7 +58,7 @@ class TestDeepDataItemWithGivenCache(BasicEnvironment): self._config = ConfigI('data') self._file = os.path.join('data', 'dir1', 'file.txt') - self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'dir1', 'file.txt_abcd') + self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'dir1', 'file.txt_abcd') self._data_item = DataItem(self._file, self._git, self._config, self._cache) pass @@ -78,8 +78,8 @@ class TestPathFactory(BasicEnvironment): self.data_file = os.path.join('data', 'file.txt') dummy_md5 = 'fsymlinc.txt' - self.cache_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dummy_md5) - self.state_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data', 'file.txt' + DataItem.STATE_FILE_SUFFIX) + self.cache_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, dummy_md5) + self.state_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file.txt' + DataItem.STATE_FILE_SUFFIX) with open(self.cache_file, 'w+') as fd: fd.write('some text') @@ -134,7 +134,7 @@ class TestDataPathInDataDir(BasicEnvironment): self.path_factory = PathFactory(git, config) deep_path = os.path.join(self.data_dir, 'dir1', 'd2', 'file.txt') - self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'dir1', 'd2', 'file.txt_eeeff8f') + self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'dir1', 'd2', 'file.txt_eeeff8f') self.data_path = self.path_factory.data_item(deep_path, cache_file=self.cache_file) pass @@ -146,7 +146,7 @@ class TestDataPathInDataDir(BasicEnvironment): self.assertEqual(self.data_path.cache.abs, self.cache_file) def test_state_file(self): - target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'da', 'dir1', 'd2', 'file.txt.state') + target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'da', 'dir1', 'd2', 'file.txt.state') self.assertEqual(self.data_path.state.abs, target) @@ -162,7 +162,7 @@ class TestDataFileObjLongPath(BasicEnvironment): self._config = ConfigI('data') self.path_factory = PathFactory(self._git, self._config) - self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'file1.txt_123ed8') + self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file1.txt_123ed8') self.data_path = self.path_factory.data_item(os.path.join('..', '..', 'data', 'file1.txt'), cache_file=self.cache_file) pass @@ -174,7 +174,7 @@ class TestDataFileObjLongPath(BasicEnvironment): self.assertEqual(self.data_path.cache.abs, self.cache_file) def test_state_file(self): - target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data', 'file1.txt.state') + target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file1.txt.state') self.assertEqual(self.data_path.state.abs, target)