From 4ffac507df3366a1e8d8f5cf5b9b758fa33d97cb Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 25 Oct 2017 14:27:56 +0300 Subject: [PATCH] dvc: remove data dir From now on, we're no longer tied to fixed data dir, so we can track individual data items without forcing user to use separate dir for them. This way we add much more flexibility for dvc, so users can use their own project structure. Fixes #68 Signed-off-by: Ruslan Kuprieiev --- dvc/command/init.py | 13 +++---------- dvc/command/merge.py | 1 - dvc/command/repro.py | 8 ++++---- dvc/command/run.py | 2 +- dvc/command/show_pipeline.py | 6 +++--- dvc/command/traverse.py | 4 ---- dvc/config.py | 16 ++++------------ dvc/path/data_item.py | 36 ++++++++++++------------------------ dvc/path/factory.py | 6 +++--- dvc/path/stated_data_item.py | 2 +- dvc/repository_change.py | 6 +++--- dvc/state_file.py | 4 ++-- 12 files changed, 36 insertions(+), 68 deletions(-) diff --git a/dvc/command/init.py b/dvc/command/init.py index d0ddeb2cf..73aee7603 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -16,8 +16,6 @@ class InitError(DvcException): class CmdInit(CmdBase): CONFIG_TEMPLATE = '''[Global] -DataDir = {} - # Default target Target = @@ -92,28 +90,23 @@ ProjectName = return 1 config_dir_path = self.get_not_existing_path(Config.CONFIG_DIR) - data_dir_path = self.get_not_existing_path(self.parsed_args.data_dir) cache_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.CACHE_DIR) state_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.STATE_DIR) - self.settings.config.set(self.parsed_args.data_dir) - conf_file_name = self.get_not_existing_conf_file_name() config_dir_path.mkdir() - data_dir_path.mkdir() cache_dir_path.mkdir() state_dir_path.mkdir() - Logger.info('Directories {}/, {}/, {}/, {}/ were created'.format( + Logger.info('Directories {}/, {}/, {}/ were created'.format( config_dir_path.name, - data_dir_path.name, os.path.join(config_dir_path.name, cache_dir_path.name), os.path.join(config_dir_path.name, state_dir_path.name))) self.create_empty_file() conf_file = open(conf_file_name, 'wt') - conf_file.write(self.CONFIG_TEMPLATE.format(data_dir_path.name)) + conf_file.write(self.CONFIG_TEMPLATE) conf_file.close() self.git.modify_gitignore([os.path.join(config_dir_path.name, cache_dir_path.name), @@ -123,7 +116,7 @@ ProjectName = return self.commit_if_needed(message) def create_empty_file(self): - empty_data_path = os.path.join(self.parsed_args.data_dir, self.EMPTY_FILE_NAME) + empty_data_path = os.path.join(self.settings.git.git_dir_abs, self.EMPTY_FILE_NAME) data_item = self.settings.path_factory.data_item(empty_data_path) open(empty_data_path, 'w').close() diff --git a/dvc/command/merge.py b/dvc/command/merge.py index 13afab86b..f730aa784 100644 --- a/dvc/command/merge.py +++ b/dvc/command/merge.py @@ -3,7 +3,6 @@ import json from dvc.command.base import CmdBase, DvcLock from dvc.logger import Logger from dvc.state_file import StateFile -from dvc.path.data_item import NotInDataDirError class CmdMerge(CmdBase): def __init__(self, settings): diff --git a/dvc/command/repro.py b/dvc/command/repro.py index ba1cbdb1d..16b7f1763 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -6,7 +6,7 @@ from dvc.command.import_file import CmdImportFile from dvc.command.run import CmdRun from dvc.logger import Logger from dvc.exceptions import DvcException -from dvc.path.data_item import NotInDataDirError +from dvc.path.data_item import DataDirError from dvc.state_file import StateFile from dvc.system import System @@ -64,8 +64,8 @@ class CmdRepro(CmdRun): data_item_list, external_files_names = self.settings.path_factory.to_data_items(target) if external_files_names: - Logger.error('Files from outside of the data directory "{}" could not be reproduced: {}'. - format(self.config.data_dir, ' '.join(external_files_names))) + Logger.error('Files from outside of the repo could not be reproduced: {}'. + format(' '.join(external_files_names))) return 1 if self.repro_data_items(data_item_list, recursive, force): @@ -303,7 +303,7 @@ class ReproChange(object): for input_file in self.state.input_files: try: data_item = self._settings.path_factory.data_item(input_file) - except NotInDataDirError: + except DataDirError: raise ReproError(u'The dependency file "{}" is not a data item'.format(input_file)) except Exception as ex: raise ReproError(u'Unable to reproduced the dependency file "{}": {}'.format( diff --git a/dvc/command/run.py b/dvc/command/run.py index 51759977b..498d1b2a5 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -5,7 +5,7 @@ from dvc.command.base import CmdBase, DvcLock from dvc.exceptions import DvcException from dvc.git_wrapper import GitWrapper from dvc.logger import Logger -from dvc.path.data_item import NotInDataDirError, NotInGitDirError +from dvc.path.data_item import DataDirError, NotInGitDirError from dvc.repository_change import RepositoryChange from dvc.state_file import StateFile from dvc.utils import cached_property diff --git a/dvc/command/show_pipeline.py b/dvc/command/show_pipeline.py index 55f0c6e03..6f7d54b48 100644 --- a/dvc/command/show_pipeline.py +++ b/dvc/command/show_pipeline.py @@ -34,7 +34,7 @@ class CmdShowPipeline(Traverse): def draw_targets(self, target): if not target: - target = self.settings.config.data_dir + target = '.' return self.draw(self.g, target, target) for t in self.parsed_args.target: @@ -49,7 +49,7 @@ class CmdShowPipeline(Traverse): def run(self): saved_targets = self.settings.parsed_args.target - self.settings.parsed_args.target = [self.settings.config.data_dir] + self.settings.parsed_args.target = ['.'] ret = super(CmdShowPipeline, self).run() @@ -70,7 +70,7 @@ class CmdShowPipeline(Traverse): name = data_item.data.relative state = StateFile.load(data_item, self.git) - if name == os.path.join(self.settings.config.data_dir, CmdInit.EMPTY_FILE_NAME): + if name == os.path.join('.', CmdInit.EMPTY_FILE_NAME): return self.g.add_node(name) diff --git a/dvc/command/traverse.py b/dvc/command/traverse.py index 6958d8241..2a29ab796 100644 --- a/dvc/command/traverse.py +++ b/dvc/command/traverse.py @@ -44,10 +44,6 @@ class Traverse(CmdBase): msg = '[TraverseFileTree] Directory "%s" cannot be traversed. Use --recurcive flag.' raise TraverseError(msg % target) - if self._do_not_start_from_root and self.settings.path_factory.data_item(target).data_dvc_short == '': - # The entire data directory - raise TraverseError('[Traverse] Root data directory "%s" cannot be traversed' % target) - self._traverse_dir(target) else: self.process_file(target) diff --git a/dvc/config.py b/dvc/config.py index d2b890d93..cff3b41e0 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -22,23 +22,16 @@ class ConfigI(object): STATE_DIR = 'state' CACHE_DIR = 'cache' - def __init__(self, data_dir=None, cloud=None, conf_parser=None): - self._data_dir = None + def __init__(self, cloud=None, conf_parser=None): self._conf_parser = None self._cloud = None - self.set(data_dir, cloud, conf_parser) + self.set(cloud, conf_parser) - def set(self, data_dir, cloud=None, conf_parser=None): + def set(self, cloud=None, conf_parser=None): """ Set config params """ - self._data_dir = data_dir self._cloud = cloud self._conf_parser = conf_parser - @property - def data_dir(self): - """ Directory with data files """ - return self._data_dir - @property def cache_dir(self): """ Directory with cached data files """ @@ -82,8 +75,7 @@ class Config(ConfigI): level = self._config['Global']['LogLevel'] Logger.set_level(level) - super(Config, self).__init__(self._config['Global']['DataDir'], - self._config['Global']['Cloud'], + super(Config, self).__init__(self._config['Global']['Cloud'], self._config) @property diff --git a/dvc/path/data_item.py b/dvc/path/data_item.py index e5478f7a5..1620868ed 100644 --- a/dvc/path/data_item.py +++ b/dvc/path/data_item.py @@ -13,22 +13,21 @@ class DataItemError(DvcException): super(DataItemError, self).__init__('Data item error: {}'.format(msg)) -class NotInDataDirError(DvcException): - def __init__(self, file, data_dir, pattern='the file "{}" is not in the data directory "{}"'): - msg = pattern.format(file, data_dir) - super(NotInDataDirError, self).__init__(msg) +class DataDirError(DvcException): + def __init__(self, msg): + super(DataDirError, self).__init__(msg) -class DataItemInStatusDirError(NotInDataDirError): - def __init__(self, file, data_dir): - pattern = 'the file "{}" is in state directory, not in the data directory "{}"' - super(DataItemInStatusDirError, self).__init__(file, data_dir, pattern) +class DataItemInStatusDirError(DataDirError): + def __init__(self, file): + msg = 'File "{}" is in state directory'.format(file) + super(DataItemInStatusDirError, self).__init__(msg) -class NotInGitDirError(NotInDataDirError): +class NotInGitDirError(DataDirError): def __init__(self, file, git_dir): - pattern = 'the file "{}" is not in git directory "{}"' - super(NotInGitDirError, self).__init__(file, git_dir, pattern) + msg = 'File "{}" is not in git directory "{}"'.format(file, git_dir) + super(NotInGitDirError, self).__init__(msg) class DataItem(object): @@ -46,11 +45,8 @@ class DataItem(object): raise NotInGitDirError(data_file, self._git.git_dir_abs) if self._data.abs.startswith(self.state_dir_abs): - raise DataItemInStatusDirError(data_file, self._config.data_dir) + raise DataItemInStatusDirError(data_file) - if not self._data.abs.startswith(self.data_dir_abs) or \ - not self._data.dvc.startswith(self._config.data_dir): - raise NotInDataDirError(data_file, self._config.data_dir) pass def copy(self, cache_file=None): @@ -72,14 +68,10 @@ class DataItem(object): def data(self): return self._data - @property - def data_dvc_short(self): - return self._data.dvc[len(self._config.data_dir)+1:] - @cached_property def state(self): state_dir = os.path.join(self._git.git_dir_abs, self._config.state_dir) - state_file = os.path.join(state_dir, self.data_dvc_short + self.STATE_FILE_SUFFIX) + state_file = os.path.join(state_dir, self.data.dvc + self.STATE_FILE_SUFFIX) return Path(state_file, self._git) @cached_property @@ -117,10 +109,6 @@ class DataItem(object): resolved_cache = os.path.realpath(self._data.relative) return Path(resolved_cache, self._git) - @cached_property - def data_dir_abs(self): - return os.path.join(self._git.git_dir_abs, self._config.data_dir) - @cached_property def state_dir_abs(self): return os.path.join(self._git.git_dir_abs, self._config.state_dir) diff --git a/dvc/path/factory.py b/dvc/path/factory.py index 68078f74a..1ca938cc5 100644 --- a/dvc/path/factory.py +++ b/dvc/path/factory.py @@ -1,6 +1,6 @@ import os -from dvc.path.data_item import DataItem, DataItemError, NotInDataDirError +from dvc.path.data_item import DataItem, DataItemError, DataDirError from dvc.path.path import Path from dvc.path.stated_data_item import StatedDataItem from dvc.system import System @@ -26,7 +26,7 @@ class PathFactory(object): return self.existing_data_item(path.relative) def is_data_item(self, fname): - data = os.path.relpath(os.path.realpath(fname), self._config.data_dir) + data = os.path.relpath(os.path.realpath(fname), self._git.git_dir_abs) state = os.path.join(self._config.state_dir, data + DataItem.STATE_FILE_SUFFIX) return os.path.isfile(state) @@ -44,7 +44,7 @@ class PathFactory(object): try: data_item = DataItem(file, self._git, self._config) result.append(data_item) - except NotInDataDirError: + except DataDirError: externally_created_files.append(file) return result, externally_created_files diff --git a/dvc/path/stated_data_item.py b/dvc/path/stated_data_item.py index 01fef1532..15feb3ba6 100644 --- a/dvc/path/stated_data_item.py +++ b/dvc/path/stated_data_item.py @@ -36,4 +36,4 @@ class StatedDataItem(DataItem): return self.is_new or self.is_modified or self.is_removed def __repr__(self): - return u'({}, {})'.format(self.state, self.data_dvc_short) + return u'({}, {})'.format(self.state, self.data.dvc) diff --git a/dvc/repository_change.py b/dvc/repository_change.py index 10aad3d25..c34916a92 100644 --- a/dvc/repository_change.py +++ b/dvc/repository_change.py @@ -4,7 +4,7 @@ from dvc.exceptions import DvcException from dvc.git_wrapper import GitWrapper from dvc.executor import Executor from dvc.logger import Logger -from dvc.path.data_item import NotInDataDirError, DataItemInStatusDirError +from dvc.path.data_item import DataDirError, DataItemInStatusDirError from dvc.path.stated_data_item import StatedDataItem from dvc.utils import cached_property @@ -76,7 +76,7 @@ class RepositoryChange(object): item.data.dvc)) except DataItemInStatusDirError: self._created_status_files.append(file) - except NotInDataDirError: + except DataDirError: self._externally_created_files.append(file) pass @@ -116,7 +116,7 @@ class RepositoryChange(object): def data_file_timesteps(self): res = set() - for root, dirs, files in os.walk(self._settings.config.data_dir): + for root, dirs, files in os.walk('.'): for file in files: filename = os.path.join(root, file) if os.path.exists(filename): diff --git a/dvc/state_file.py b/dvc/state_file.py index 22004fb52..2e1bed6b5 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -5,7 +5,7 @@ import json import re from dvc.exceptions import DvcException -from dvc.path.data_item import NotInDataDirError +from dvc.path.data_item import DataDirError from dvc.system import System @@ -248,7 +248,7 @@ class StateFile(object): try: data_item = self.settings.path_factory.data_item(arg) result.append(data_item.data.dvc) - except NotInDataDirError: + except DataDirError: result.append(arg) return result