Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-06 23:09:55 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvc: remove data dir

Browse Source 
From now on, we're no longer tied to fixed data dir, so we can
track individual data items without forcing user to use separate
dir for them. This way we add much more flexibility for dvc, so
users can use their own project structure.

Fixes #68

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2017-10-30 05:05:28 +02:00
parent a813ebaeb7
commit 4ffac507df
12 changed files with 36 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/init.py
+3 -10
View File
@@ -16,8 +16,6 @@ class InitError(DvcException):
class CmdInit(CmdBase):
CONFIG_TEMPLATE = '''[Global]
DataDir = {}
# Default target
Target =
@@ -92,28 +90,23 @@ ProjectName =
return 1
config_dir_path = self.get_not_existing_path(Config.CONFIG_DIR)
data_dir_path = self.get_not_existing_path(self.parsed_args.data_dir)
cache_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.CACHE_DIR)
state_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.STATE_DIR)
self.settings.config.set(self.parsed_args.data_dir)
conf_file_name = self.get_not_existing_conf_file_name()
config_dir_path.mkdir()
data_dir_path.mkdir()
cache_dir_path.mkdir()
state_dir_path.mkdir()
Logger.info('Directories {}/, {}/, {}/, {}/ were created'.format(
Logger.info('Directories {}/, {}/, {}/ were created'.format(
config_dir_path.name,
data_dir_path.name,
os.path.join(config_dir_path.name, cache_dir_path.name),
os.path.join(config_dir_path.name, state_dir_path.name)))
self.create_empty_file()
conf_file = open(conf_file_name, 'wt')
conf_file.write(self.CONFIG_TEMPLATE.format(data_dir_path.name))
conf_file.write(self.CONFIG_TEMPLATE)
conf_file.close()
self.git.modify_gitignore([os.path.join(config_dir_path.name, cache_dir_path.name),
@@ -123,7 +116,7 @@ ProjectName =
return self.commit_if_needed(message)
def create_empty_file(self):
empty_data_path = os.path.join(self.parsed_args.data_dir, self.EMPTY_FILE_NAME)
empty_data_path = os.path.join(self.settings.git.git_dir_abs, self.EMPTY_FILE_NAME)
data_item = self.settings.path_factory.data_item(empty_data_path)
open(empty_data_path, 'w').close()
dvc/command/merge.py
-1
View File
@@ -3,7 +3,6 @@ import json
from dvc.command.base import CmdBase, DvcLock
from dvc.logger import Logger
from dvc.state_file import StateFile
from dvc.path.data_item import NotInDataDirError
class CmdMerge(CmdBase):
def __init__(self, settings):
dvc/command/repro.py
+4 -4
View File
@@ -6,7 +6,7 @@ from dvc.command.import_file import CmdImportFile
from dvc.command.run import CmdRun
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.path.data_item import NotInDataDirError
from dvc.path.data_item import DataDirError
from dvc.state_file import StateFile
from dvc.system import System
@@ -64,8 +64,8 @@ class CmdRepro(CmdRun):
data_item_list, external_files_names = self.settings.path_factory.to_data_items(target)
if external_files_names:
Logger.error('Files from outside of the data directory "{}" could not be reproduced: {}'.
format(self.config.data_dir, ' '.join(external_files_names)))
Logger.error('Files from outside of the repo could not be reproduced: {}'.
format(' '.join(external_files_names)))
return 1
if self.repro_data_items(data_item_list, recursive, force):
@@ -303,7 +303,7 @@ class ReproChange(object):
for input_file in self.state.input_files:
try:
data_item = self._settings.path_factory.data_item(input_file)
except NotInDataDirError:
except DataDirError:
raise ReproError(u'The dependency file "{}" is not a data item'.format(input_file))
except Exception as ex:
raise ReproError(u'Unable to reproduced the dependency file "{}": {}'.format(
dvc/command/run.py
+1 -1
View File
@@ -5,7 +5,7 @@ from dvc.command.base import CmdBase, DvcLock
from dvc.exceptions import DvcException
from dvc.git_wrapper import GitWrapper
from dvc.logger import Logger
from dvc.path.data_item import NotInDataDirError, NotInGitDirError
from dvc.path.data_item import DataDirError, NotInGitDirError
from dvc.repository_change import RepositoryChange
from dvc.state_file import StateFile
from dvc.utils import cached_property
dvc/command/show_pipeline.py
+3 -3
View File
@@ -34,7 +34,7 @@ class CmdShowPipeline(Traverse):
def draw_targets(self, target):
if not target:
target = self.settings.config.data_dir
target = '.'
return self.draw(self.g, target, target)
for t in self.parsed_args.target:
@@ -49,7 +49,7 @@ class CmdShowPipeline(Traverse):
def run(self):
saved_targets = self.settings.parsed_args.target
self.settings.parsed_args.target = [self.settings.config.data_dir]
self.settings.parsed_args.target = ['.']
ret = super(CmdShowPipeline, self).run()
@@ -70,7 +70,7 @@ class CmdShowPipeline(Traverse):
name = data_item.data.relative
state = StateFile.load(data_item, self.git)
if name == os.path.join(self.settings.config.data_dir, CmdInit.EMPTY_FILE_NAME):
if name == os.path.join('.', CmdInit.EMPTY_FILE_NAME):
return
self.g.add_node(name)
dvc/command/traverse.py
-4
View File
@@ -44,10 +44,6 @@ class Traverse(CmdBase):
msg = '[TraverseFileTree] Directory "%s" cannot be traversed. Use --recurcive flag.'
raise TraverseError(msg % target)
if self._do_not_start_from_root and self.settings.path_factory.data_item(target).data_dvc_short == '':
# The entire data directory
raise TraverseError('[Traverse] Root data directory "%s" cannot be traversed' % target)
self._traverse_dir(target)
else:
self.process_file(target)
dvc/config.py
+4 -12
View File
@@ -22,23 +22,16 @@ class ConfigI(object):
STATE_DIR = 'state'
CACHE_DIR = 'cache'
def __init__(self, data_dir=None, cloud=None, conf_parser=None):
self._data_dir = None
def __init__(self, cloud=None, conf_parser=None):
self._conf_parser = None
self._cloud = None
self.set(data_dir, cloud, conf_parser)
self.set(cloud, conf_parser)
def set(self, data_dir, cloud=None, conf_parser=None):
def set(self, cloud=None, conf_parser=None):
""" Set config params """
self._data_dir = data_dir
self._cloud = cloud
self._conf_parser = conf_parser
@property
def data_dir(self):
""" Directory with data files """
return self._data_dir
@property
def cache_dir(self):
""" Directory with cached data files """
@@ -82,8 +75,7 @@ class Config(ConfigI):
level = self._config['Global']['LogLevel']
Logger.set_level(level)
super(Config, self).__init__(self._config['Global']['DataDir'],
self._config['Global']['Cloud'],
super(Config, self).__init__(self._config['Global']['Cloud'],
self._config)
@property
dvc/path/data_item.py
+12 -24
View File
@@ -13,22 +13,21 @@ class DataItemError(DvcException):
super(DataItemError, self).__init__('Data item error: {}'.format(msg))
class NotInDataDirError(DvcException):
def __init__(self, file, data_dir, pattern='the file "{}" is not in the data directory "{}"'):
msg = pattern.format(file, data_dir)
super(NotInDataDirError, self).__init__(msg)
class DataDirError(DvcException):
def __init__(self, msg):
super(DataDirError, self).__init__(msg)
class DataItemInStatusDirError(NotInDataDirError):
def __init__(self, file, data_dir):
pattern = 'the file "{}" is in state directory, not in the data directory "{}"'
super(DataItemInStatusDirError, self).__init__(file, data_dir, pattern)
class DataItemInStatusDirError(DataDirError):
def __init__(self, file):
msg = 'File "{}" is in state directory'.format(file)
super(DataItemInStatusDirError, self).__init__(msg)
class NotInGitDirError(NotInDataDirError):
class NotInGitDirError(DataDirError):
def __init__(self, file, git_dir):
pattern = 'the file "{}" is not in git directory "{}"'
super(NotInGitDirError, self).__init__(file, git_dir, pattern)
msg = 'File "{}" is not in git directory "{}"'.format(file, git_dir)
super(NotInGitDirError, self).__init__(msg)
class DataItem(object):
@@ -46,11 +45,8 @@ class DataItem(object):
raise NotInGitDirError(data_file, self._git.git_dir_abs)
if self._data.abs.startswith(self.state_dir_abs):
raise DataItemInStatusDirError(data_file, self._config.data_dir)
raise DataItemInStatusDirError(data_file)
if not self._data.abs.startswith(self.data_dir_abs) or \
not self._data.dvc.startswith(self._config.data_dir):
raise NotInDataDirError(data_file, self._config.data_dir)
pass
def copy(self, cache_file=None):
@@ -72,14 +68,10 @@ class DataItem(object):
def data(self):
return self._data
@property
def data_dvc_short(self):
return self._data.dvc[len(self._config.data_dir)+1:]
@cached_property
def state(self):
state_dir = os.path.join(self._git.git_dir_abs, self._config.state_dir)
state_file = os.path.join(state_dir, self.data_dvc_short + self.STATE_FILE_SUFFIX)
state_file = os.path.join(state_dir, self.data.dvc + self.STATE_FILE_SUFFIX)
return Path(state_file, self._git)
@cached_property
@@ -117,10 +109,6 @@ class DataItem(object):
resolved_cache = os.path.realpath(self._data.relative)
return Path(resolved_cache, self._git)
@cached_property
def data_dir_abs(self):
return os.path.join(self._git.git_dir_abs, self._config.data_dir)
@cached_property
def state_dir_abs(self):
return os.path.join(self._git.git_dir_abs, self._config.state_dir)
dvc/path/factory.py
+3 -3
View File
@@ -1,6 +1,6 @@
import os
from dvc.path.data_item import DataItem, DataItemError, NotInDataDirError
from dvc.path.data_item import DataItem, DataItemError, DataDirError
from dvc.path.path import Path
from dvc.path.stated_data_item import StatedDataItem
from dvc.system import System
@@ -26,7 +26,7 @@ class PathFactory(object):
return self.existing_data_item(path.relative)
def is_data_item(self, fname):
data = os.path.relpath(os.path.realpath(fname), self._config.data_dir)
data = os.path.relpath(os.path.realpath(fname), self._git.git_dir_abs)
state = os.path.join(self._config.state_dir, data + DataItem.STATE_FILE_SUFFIX)
return os.path.isfile(state)
@@ -44,7 +44,7 @@ class PathFactory(object):
try:
data_item = DataItem(file, self._git, self._config)
result.append(data_item)
except NotInDataDirError:
except DataDirError:
externally_created_files.append(file)
return result, externally_created_files
dvc/path/stated_data_item.py
+1 -1
View File
@@ -36,4 +36,4 @@ class StatedDataItem(DataItem):
return self.is_new or self.is_modified or self.is_removed
def __repr__(self):
return u'({}, {})'.format(self.state, self.data_dvc_short)
return u'({}, {})'.format(self.state, self.data.dvc)
dvc/repository_change.py
+3 -3
View File
@@ -4,7 +4,7 @@ from dvc.exceptions import DvcException
from dvc.git_wrapper import GitWrapper
from dvc.executor import Executor
from dvc.logger import Logger
from dvc.path.data_item import NotInDataDirError, DataItemInStatusDirError
from dvc.path.data_item import DataDirError, DataItemInStatusDirError
from dvc.path.stated_data_item import StatedDataItem
from dvc.utils import cached_property
@@ -76,7 +76,7 @@ class RepositoryChange(object):
item.data.dvc))
except DataItemInStatusDirError:
self._created_status_files.append(file)
except NotInDataDirError:
except DataDirError:
self._externally_created_files.append(file)
pass
@@ -116,7 +116,7 @@ class RepositoryChange(object):
def data_file_timesteps(self):
res = set()
for root, dirs, files in os.walk(self._settings.config.data_dir):
for root, dirs, files in os.walk('.'):
for file in files:
filename = os.path.join(root, file)
if os.path.exists(filename):
dvc/state_file.py
+2 -2
View File
@@ -5,7 +5,7 @@ import json
import re
from dvc.exceptions import DvcException
from dvc.path.data_item import NotInDataDirError
from dvc.path.data_item import DataDirError
from dvc.system import System
@@ -248,7 +248,7 @@ class StateFile(object):
try:
data_item = self.settings.path_factory.data_item(arg)
result.append(data_item.data.dvc)
except NotInDataDirError:
except DataDirError:
result.append(arg)
return result