diff --git a/dvc/cmd_base.py b/dvc/cmd_base.py index 603783caa..9bfa3e636 100644 --- a/dvc/cmd_base.py +++ b/dvc/cmd_base.py @@ -4,6 +4,8 @@ import os from dvc.git_wrapper import GitWrapper from dvc.config import Config, ConfigError from dvc.logger import Logger +from dvc.path_factory import PathFactory +from dvc.utils import cached_property class CmdBase(object): @@ -30,10 +32,17 @@ class CmdBase(object): raise ConfigError("DVC_HOME directory doesn't exists") pass + @cached_property + def path_factory(self): + return PathFactory(self.git, self.config) + @property def config(self): return self._config + def cache_file_aws_key(self, file): + return '{}/{}'.format(self._config.aws_storage_prefix, file).strip('/') + @property def lnx_home(self): return self._dvc_home @@ -46,7 +55,7 @@ class CmdBase(object): def git(self): return self._git - def define_args(self): + def define_args(self, parser): pass def set_skip_git_actions(self, parser): diff --git a/dvc/cmd_data_import.py b/dvc/cmd_data_import.py index f57c0de23..b97dfedc9 100644 --- a/dvc/cmd_data_import.py +++ b/dvc/cmd_data_import.py @@ -8,7 +8,6 @@ import requests from dvc.cmd_base import CmdBase from dvc.cmd_data_sync import sizeof_fmt from dvc.logger import Logger -from dvc.data_file_obj import DataFileObj from dvc.exceptions import NeatLynxException from dvc.state_file import StateFile @@ -72,40 +71,40 @@ class CmdDataImport(CmdBase): if os.path.isdir(output): output = os.path.join(output, os.path.basename(input)) - dobj = DataFileObj(output, self.git, self.config) + data_path = self.path_factory.data_path(output) - if os.path.exists(dobj.data_file_relative): - raise DataImportError('Output file "{}" already exists'.format(dobj.data_file_relative)) - if not os.path.isdir(os.path.dirname(dobj.data_file_abs)): + if os.path.exists(data_path.data.relative): + raise DataImportError('Output file "{}" already exists'.format(data_path.data.relative)) + if not os.path.isdir(os.path.dirname(data_path.data.relative)): raise DataImportError('Output file directory "{}" does not exists'.format( - os.path.dirname(dobj.data_file_relative))) + os.path.dirname(data_path.data.relative))) - cache_dir = os.path.dirname(dobj.cache_file_relative) + cache_dir = os.path.dirname(data_path.cache.relative) if not os.path.exists(cache_dir): os.makedirs(cache_dir) if CmdDataImport.is_url(input): Logger.debug('Downloading file {} ...'.format(input)) - self.download_file(input, dobj.cache_file_relative) + self.download_file(input, data_path.cache.relative) Logger.debug('Input file "{}" was downloaded to cache "{}"'.format( - input, dobj.cache_file_relative)) + input, data_path.cache.relative)) else: - copyfile(input, dobj.cache_file_relative) + copyfile(input, data_path.cache.relative) Logger.debug('Input file "{}" was copied to cache "{}"'.format( - input, dobj.cache_file_relative)) + input, data_path.cache.relative)) - dobj.create_symlink() + data_path.create_symlink() Logger.debug('Symlink from data file "{}" to the cache file "{}" was created'. - format(dobj.data_file_relative, dobj.cache_file_relative)) + format(data_path.data.relative, data_path.cache.relative)) - state_file = StateFile(dobj.state_file_relative, + state_file = StateFile(data_path.state.relative, self.git, [input], [output], [], is_reproducible) state_file.save() - Logger.debug('State file "{}" was created'.format(dobj.state_file_relative)) + Logger.debug('State file "{}" was created'.format(data_path.state.relative)) pass URL_REGEX = re.compile( diff --git a/dvc/cmd_data_remove.py b/dvc/cmd_data_remove.py index a9b3009ea..df90ca614 100644 --- a/dvc/cmd_data_remove.py +++ b/dvc/cmd_data_remove.py @@ -5,7 +5,6 @@ import fasteners from dvc.cmd_base import CmdBase from dvc.logger import Logger from dvc.exceptions import NeatLynxException -from dvc.data_file_obj import DataFileObjExisting class DataRemoveError(NeatLynxException): @@ -59,27 +58,32 @@ class CmdDataRemove(CmdBase): raise DataRemoveError('data directory cannot be removed') return self.remove_dir(target) - dobj = DataFileObjExisting(target, self.git, self.config) - if os.path.islink(dobj.data_file_relative): - return self.remove_symlink(dobj.data_file_relative) + + data_path = self.path_factory.existing_data_path(target) + if os.path.islink(data_path.data.relative): + return self.remove_symlink(data_path) raise DataRemoveError('Cannot remove a regular file "{}"'.format(target)) - def remove_symlink(self, file): - dobj = DataFileObjExisting(file, self.git, self.config) + @staticmethod + def remove_dir_if_empty(file): + dir = os.path.dirname(file) + if dir != '' and not os.listdir(dir): + os.rmdir(dir) - if os.path.isfile(dobj.cache_file_relative): - os.remove(dobj.cache_file_relative) - dobj.remove_cache_dir_if_empty() + def remove_symlink(self, data_path): + if os.path.isfile(data_path.cache.relative): + os.remove(data_path.cache.relative) + self.remove_dir_if_empty(data_path.cache.relavive) - if os.path.isfile(dobj.state_file_relative): - os.remove(dobj.state_file_relative) - dobj.remove_state_dir_if_empty() + if os.path.isfile(data_path.state.relative): + os.remove(data_path.state.relative) + self.remove_dir_if_empty(data_path.state.relative) if self.args.remove_from_cloud: - self.remove_from_cloud(dobj.cache_file_aws_key) + self.remove_from_cloud(data_path.cache_file_aws_key) - os.remove(file) + os.remove(data_path.data.relative) pass def remove_from_cloud(self, aws_file_name): diff --git a/dvc/cmd_data_sync.py b/dvc/cmd_data_sync.py index a70cfd936..e5fbfe815 100644 --- a/dvc/cmd_data_sync.py +++ b/dvc/cmd_data_sync.py @@ -5,7 +5,6 @@ from boto.s3.connection import S3Connection from dvc.cmd_base import CmdBase from dvc.logger import Logger -from dvc.data_file_obj import DataFileObjExisting from dvc.exceptions import NeatLynxException @@ -53,8 +52,8 @@ class CmdDataSync(CmdBase): def run(self): if os.path.islink(self.args.target): - dobj = DataFileObjExisting(self.args.target, self.git, self.config) - return self.sync_symlink(dobj) + data_path = self.path_factory.existing_data_path(self.args.target) + return self.sync_symlink(data_path) if os.path.isdir(self.args.target): return self.sync_dir(self.args.target) @@ -67,46 +66,48 @@ class CmdDataSync(CmdBase): if os.path.isdir(fname): self.sync_dir(fname) elif os.path.islink(fname): - self.sync_symlink(DataFileObjExisting(fname, self.git, self.config)) + self.sync_symlink(self.path_factory.existing_data_path(fname)) else: raise DataSyncError('Unsupported file type "{}"'.format(fname)) pass - def sync_symlink(self, dobj): - if os.path.isfile(dobj.cache_file_relative): - self.sync_to_cloud(dobj) + def sync_symlink(self, data_path): + if os.path.isfile(data_path.cache.relative): + self.sync_to_cloud(data_path) else: - self.sync_from_cloud(dobj) + self.sync_from_cloud(data_path) pass - def sync_from_cloud(self, dobj): - key = self._bucket.get_key(dobj.cache_file_aws_key) + def sync_from_cloud(self, data_path): + aws_key = self.cache_file_aws_key(data_path.cache.dvc) + key = self._bucket.get_key(aws_key) if not key: - raise DataSyncError('File "{}" does not exist in the cloud'.format(dobj.cache_file_aws_key)) + raise DataSyncError('File "{}" does not exist in the cloud'.format(aws_key)) Logger.printing('Downloading cache file from S3 "{}/{}"'.format(self._bucket.name, - dobj.cache_file_aws_key)) - key.get_contents_to_filename(dobj.cache_file_relative, cb=percent_cb) + aws_key)) + key.get_contents_to_filename(data_path.cache.relative, cb=percent_cb) Logger.printing('Downloading completed') pass - def sync_to_cloud(self, dobj): - key = self._bucket.get_key(dobj.cache_file_aws_key) + def sync_to_cloud(self, data_path): + aws_key = self.cache_file_aws_key(data_path.cache.dvc) + key = self._bucket.get_key(aws_key) if key: Logger.debug('File already uploaded to the cloud. Checksum validation...') md5_cloud = key.etag[1:-1] - md5_local = file_md5(dobj.cache_file_relative) + md5_local = file_md5(data_path.cache.relative) if md5_cloud == md5_local: Logger.debug('File checksum matches. No uploading is needed.') return Logger.printing('Checksum miss-match. Re-uploading is required.') - Logger.printing('Uploading cache file "{}" to S3 "{}"'.format(dobj.cache_file_relative, - dobj.cache_file_aws_key)) - key = self._bucket.new_key(dobj.cache_file_aws_key) - key.set_contents_from_filename(dobj.cache_file_relative, cb=percent_cb) + Logger.printing('Uploading cache file "{}" to S3 "{}"'.format(data_path.cache.relative, + aws_key)) + key = self._bucket.new_key(aws_key) + key.set_contents_from_filename(data_path.cache.relative, cb=percent_cb) Logger.printing('Uploading completed') pass diff --git a/dvc/cmd_repro.py b/dvc/cmd_repro.py index acda7723e..ecfa7a483 100644 --- a/dvc/cmd_repro.py +++ b/dvc/cmd_repro.py @@ -4,10 +4,9 @@ import fasteners from dvc.cmd_base import CmdBase from dvc.cmd_run import CmdRun -from dvc.git_wrapper import GitWrapper from dvc.logger import Logger from dvc.exceptions import NeatLynxException -from dvc.data_file_obj import DataFileObj, NotInDataDirError +from dvc.data_file_obj import NotInDataDirError from dvc.state_file import StateFile @@ -55,29 +54,29 @@ class CmdRepro(CmdRun): if not self.skip_git_actions and not self.git.is_ready_to_go(): return 1 - dobjs, externally_created_files = DataFileObj.files_to_dobjs(self.args.target, self.git, self.config) - if externally_created_files: + data_path_list, external_files = self.path_factory.to_data_path(self.args.target) + if external_files: Logger.error('Files from outside of the data directory "{}" could not be reproduced: {}'. - format(self.config.data_dir, ' '.join(externally_created_files))) + format(self.config.data_dir, ' '.join(external_files))) return 1 error = False changed = False - for dobj in dobjs: + for data_path in data_path_list: try: - repro_change = ReproChange(dobj, self.git, self) + repro_change = ReproChange(data_path, self) if repro_change.reproduce(): changed = True Logger.info(u'Data file "{}" was reproduced.'.format( - dobj.data_file_relative + data_path.data.relative )) else: Logger.info(u'Reproduction is not required for data file "{}".'.format( - dobj.data_file_relative + data_path.data.relative )) except ReproError as err: Logger.error('Error in reproducing data file {}: {}'.format( - dobj.data_file_relative, str(err) + data_path.data.relative, str(err) )) error = True break @@ -100,11 +99,11 @@ class CmdRepro(CmdRun): class ReproChange(object): - def __init__(self, dobj, git, cmd_obj): - self._dobj = dobj - self._state = StateFile.load(dobj.state_file_relative, git) - self.git = git + def __init__(self, data_path, cmd_obj): + self._data_path = data_path + self.git = cmd_obj.git self._cmd_obj = cmd_obj + self._state = StateFile.load(data_path.state.relative, self.git) cmd_obj._code = self._state.code_sources # HACK!!! @@ -112,7 +111,7 @@ class ReproChange(object): if not argv: raise ReproError('Error: parameter {} is nor defined in state file "{}"'. - format(StateFile.PARAM_NORM_ARGV, dobj.state_file_relative)) + format(StateFile.PARAM_NORM_ARGV, data_path.state.relative)) if len(argv) < 2: raise ReproError('Error: reproducible command in state file "{}" is too short'. format(self._state.file)) @@ -129,39 +128,39 @@ class ReproChange(object): def reproduce_data_file(self): Logger.debug('Reproducing data file "{}". Removing the file...'.format( - self._dobj.data_file_relative)) - os.remove(self._dobj.data_file_relative) + self._data_path.data.relative)) + os.remove(self._data_path.data.relative) Logger.debug('Reproducing data file "{}". Re-runs command: {}'.format( - self._dobj.data_file_relative, ' '.join(self._repro_argv))) + self._data_path.data.relative, ' '.join(self._repro_argv))) return self._cmd_obj.run_command(self._repro_argv) def reproduce(self, force=False): were_input_files_changed = False if not self._state.is_reproducible: - Logger.debug('Data file "{}" is not reproducible'.format(self._dobj.data_file_relative)) + Logger.debug('Data file "{}" is not reproducible'.format(self._data_path.data.relative)) return False for input_file in self._state.input_files: try: - dobj = DataFileObj(input_file, self.git, self._cmd_obj.config) + data_path = self._cmd_obj.path_factory.data_path(input_file) except NotInDataDirError: raise ReproError(u'The dependency files "{}" is not a data file'.format(input_file)) except Exception as ex: raise ReproError(u'The dependency files "{}" can not be reproduced: {}'.format( input_file, ex)) - change = ReproChange(dobj, self.git, self._cmd_obj) + change = ReproChange(data_path, self._cmd_obj) if change.reproduce(force): were_input_files_changed = True - was_source_code_changed = self.git.were_files_changed(self._dobj.data_file_relative, + was_source_code_changed = self.git.were_files_changed(self._data_path.data.relative, self._state.code_sources) if not force and not was_source_code_changed and not were_input_files_changed: Logger.debug('Data file "{}" is up to date'.format( - self._dobj.data_file_relative)) + self._data_path.data.relative)) return False return self.reproduce_data_file() diff --git a/dvc/cmd_run.py b/dvc/cmd_run.py index 53e925251..74ba566c8 100644 --- a/dvc/cmd_run.py +++ b/dvc/cmd_run.py @@ -2,7 +2,7 @@ import os import shutil import fasteners -from dvc.data_file_obj import DataFileObj, NotInDataDirError +from dvc.data_file_obj import NotInDataDirError from dvc.git_wrapper import GitWrapper from dvc.cmd_base import CmdBase from dvc.logger import Logger @@ -77,7 +77,7 @@ class CmdRun(CmdBase): return 0 def run_command(self, argv, stdout=None, stderr=None): - repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config) + repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config, self.path_factory) # print('===== new_files={}'.format(repo_change.new_files)) # print('===== modified_files={}'.format(repo_change.modified_files)) @@ -96,21 +96,21 @@ class CmdRun(CmdBase): input_files_from_args = list(set(args_files_nlx) - set(changed_files_nlx)) input_files = self.git.abs_paths_to_nlx(input_files_from_args + self.declaration_input_files) - for dobj in repo_change.dobj_for_changed_files: - dirname = os.path.dirname(dobj.cache_file_relative) + for data_path in repo_change.data_paths_for_changed_files: + dirname = os.path.dirname(data_path.cache.relative) if not os.path.isdir(dirname): os.makedirs(dirname) Logger.debug('Move output file "{}" to cache dir "{}" and create a symlink'.format( - dobj.data_file_relative, dobj.cache_file_relative)) - shutil.move(dobj.data_file_relative, dobj.cache_file_relative) + data_path.data.relative, data_path.cache.relative)) + shutil.move(data_path.data.relative, data_path.cache.relative) - dobj.create_symlink() + data_path.create_symlink() nlx_code_sources = map(lambda x: self.git.abs_paths_to_nlx([x])[0], self.code) - Logger.debug('Create state file "{}"'.format(dobj.state_file_relative)) - state_file = StateFile(dobj.state_file_relative, self.git, + Logger.debug('Create state file "{}"'.format(data_path.state.relative)) + state_file = StateFile(data_path.state.relative, self.git, input_files, output_files, nlx_code_sources, @@ -165,7 +165,7 @@ class CmdRun(CmdBase): for arg in argv: try: if os.path.isfile(arg): - DataFileObj(arg, self.git, self.config) + self.path_factory.data_path(arg) result.append(arg) except NotInDataDirError: pass diff --git a/dvc/data_file_obj.py b/dvc/data_file_obj.py index f0df6b256..1922a7936 100644 --- a/dvc/data_file_obj.py +++ b/dvc/data_file_obj.py @@ -1,6 +1,8 @@ import os +from dvc.dvc_path import DvcPath from dvc.exceptions import NeatLynxException +from dvc.utils import cached_property class DataFilePathError(NeatLynxException): @@ -15,135 +17,55 @@ class NotInDataDirError(NeatLynxException): format(file, data_dir)) -class DataFileObj(object): +class DataPath(object): STATE_FILE_SUFFIX = '.state' CACHE_FILE_SEP = '_' - def __init__(self, data_file, git, config): - ''' - nlx file name - system file name without data/cache/state prefix - ''' + def __init__(self, data_file, git, config, cache_file=None): self._git = git self._config = config - self._curr_dir_abs = os.path.realpath(os.curdir) + self._cache_file = cache_file - self._data_file = data_file + self._data = DvcPath(data_file, git) + if not self._data.abs.startswith(self.data_dir_abs): + raise NotInDataDirError(data_file, self._config.data_dir) - if not self.data_file_abs.startswith(self.data_dir_abs): - raise NotInDataDirError(self._data_file, self._config.data_dir) + if not self._data.dvc.startswith(self._config.data_dir): + raise NotInDataDirError(data_file, self._config.data_dir) pass - def create_symlink(self): - data_dir = os.path.dirname(self.data_file_relative) - cache_relative_to_data_dir = os.path.relpath(self.cache_file_relative, data_dir) - os.symlink(cache_relative_to_data_dir, self.data_file_relative) + @property + def data(self): + return self._data @property + def data_dvc_short(self): + return self._data.dvc[len(self._config.data_dir)+1:] + + @cached_property + def state(self): + state_dir = os.path.join(self._git.git_dir_abs, self._config.state_dir) + state_file = os.path.join(state_dir, self.data_dvc_short + self.STATE_FILE_SUFFIX) + return DvcPath(state_file, self._git) + + @cached_property + def cache(self): + if self._cache_file: + return DvcPath(self._cache_file, self._git) + else: + cache_dir = os.path.join(self._git.git_dir_abs, self._config.cache_dir) + cache_file_suffix = self.CACHE_FILE_SEP + self._git.curr_commit + cache_file = os.path.join(cache_dir, self.data_dvc_short + cache_file_suffix) + return DvcPath(cache_file, self._git) + + @cached_property def data_dir_abs(self): return os.path.join(self._git.git_dir_abs, self._config.data_dir) - # Data file properties @property - def data_file_name(self): - return os.path.basename(self._data_file) + def _symlink_file(self): + data_file_dir = os.path.dirname(self.data.relative) + return os.path.relpath(self.cache.relative, data_file_dir) - @property - def data_file_nlx(self): - return os.path.relpath(self.data_file_abs, self.data_dir_abs) - - # @property - # def data_file_abs(self): - # return os.path.realpath(self.data_file_relative) - - @property - def data_file_abs(self): - '''Do not fully resolve file name since it is a link''' - return os.path.abspath(self.data_file_relative) - - @property - def data_file_relative(self): - return self._data_file - - # Cache file properties - @property - def cache_file_name(self): - if not self._git.curr_commit: - return None - return self.data_file_name + self.CACHE_FILE_SEP + self._git.curr_commit - - @property - def cache_file_nlx(self): - if not self._git.curr_commit: - return None - return self.data_file_nlx + self.CACHE_FILE_SEP + self._git.curr_commit - - @property - def cache_file_aws_key(self): - return '{}/{}'.format(self._config.aws_storage_prefix, self.cache_file_nlx).strip('/') - - @property - def cache_file_abs(self): - return os.path.join(self._git.git_dir_abs, self._config.cache_dir, self.cache_file_nlx) - - @property - def cache_file_relative(self): - return os.path.relpath(self.cache_file_abs, self._curr_dir_abs) - - # State file properties - @property - def state_file_name(self): - return self.data_file_name + self.STATE_FILE_SUFFIX - - @property - def state_file_nlx(self): - return self.data_file_nlx + self.STATE_FILE_SUFFIX - - @property - def state_file_abs(self): - return os.path.join(self._git.git_dir_abs, self._config.state_dir, self.state_file_nlx) - - @property - def state_file_relative(self): - return os.path.relpath(self.state_file_abs, self._curr_dir_abs) - - @staticmethod - def files_to_dobjs(files, git, config): - result = [] - externally_created_files = [] - - for file in files: - try: - result.append(DataFileObj(file, git, config)) - except NotInDataDirError: - externally_created_files.append(file) - - return result, externally_created_files - - -class DataFileObjExisting(DataFileObj): - def __init__(self, data_file, git, config): - DataFileObj.__init__(self, data_file, git, config) - - if not os.path.islink(data_file): - raise DataFilePathError('Data file must be a symbolic link') - pass - - @property - def cache_file_name(self): - return os.path.basename(self.cache_file_abs) - - @property - def cache_file_abs(self): - return os.path.realpath(self.data_file_relative) - - @staticmethod - def remove_dir_if_empty(dir): - cache_file_dir = os.path.dirname(dir) - if cache_file_dir != '' and not os.listdir(cache_file_dir): - os.rmdir(cache_file_dir) - - def remove_state_dir_if_empty(self): - self.remove_dir_if_empty(self.state_file_relative) - - def remove_cache_dir_if_empty(self): - self.remove_dir_if_empty(self.cache_file_relative) + def create_symlink(self): + os.symlink(self._symlink_file, self.data.relative) diff --git a/dvc/dvc_path.py b/dvc/dvc_path.py index ad1e188a5..9f297577d 100644 --- a/dvc/dvc_path.py +++ b/dvc/dvc_path.py @@ -2,10 +2,10 @@ import os class DvcPath(object): - def __init__(self, relative_raw, git_dir_abs, curr_dir_abs): - self._abs = os.path.realpath(relative_raw) - self._dvc = os.path.relpath(self.abs, git_dir_abs) - self._relative = os.path.relpath(self._abs, curr_dir_abs) + def __init__(self, relative_raw, git): + self._abs = os.path.abspath(relative_raw) + self._dvc = os.path.relpath(self.abs, git.git_dir_abs) + self._relative = os.path.relpath(self._abs, git.curr_dir_abs) self._filename = os.path.basename(self._abs) @property diff --git a/dvc/path_factory.py b/dvc/path_factory.py new file mode 100644 index 000000000..e9854debf --- /dev/null +++ b/dvc/path_factory.py @@ -0,0 +1,36 @@ +import os + +from dvc.data_file_obj import DataPath, DataFilePathError, NotInDataDirError +from dvc.dvc_path import DvcPath + + +class PathFactory(object): + def __init__(self, git, config): + self._git = git + self._config = config + self._curr_dir_abs = os.path.realpath(os.curdir) + + def path(self, relative_raw): + return DvcPath(relative_raw, self._git) + + def data_path(self, data_file): + return DataPath(data_file, self._git, self._config) + + def existing_data_path(self, data_file): + if not os.path.islink(data_file): + raise DataFilePathError(u'Data file "%s" must be a symbolic link' % data_file) + resolved_symlink = os.path.realpath(data_file) + return DataPath(data_file, self._git, self._config, resolved_symlink) + + def to_data_path(self, files): + result = [] + externally_created_files = [] + + for file in files: + try: + data_path = DataPath(file, self._git, self._config) + result.append(data_path) + except NotInDataDirError: + externally_created_files.append(file) + + return result, externally_created_files diff --git a/dvc/repository_change.py b/dvc/repository_change.py index 1c3c5603b..add7c1edc 100644 --- a/dvc/repository_change.py +++ b/dvc/repository_change.py @@ -1,6 +1,5 @@ import os -from dvc.data_file_obj import DataFileObj, NotInDataDirError from dvc.exceptions import NeatLynxException from dvc.git_wrapper import GitWrapper @@ -44,18 +43,18 @@ class FileModificationState(object): class RepositoryChange(object): """Pre-condition: git repository has no changes""" - def __init__(self, args, stdout, stderr, git, config): + def __init__(self, args, stdout, stderr, git, config, path_factory): self.git = git self.config = config + self.path_factory = path_factory GitWrapper.exec_cmd_only_success(args, stdout, stderr) self._file_states = self._get_file_states() - self._changed_dobj, self._externally_created_files = DataFileObj.files_to_dobjs( - self.changed_files, self.git, self.config) + self._changed_data_paths, self._externally_created_files = path_factory.to_data_path(self.changed_files) @staticmethod - def exec_cmd(args, stdout, stderr, git): + def exec_cmd(args, stdout, stderr, git, path_factory): return RepositoryChange(args, stdout, stderr, git) @staticmethod @@ -113,8 +112,8 @@ class RepositoryChange(object): pass @property - def dobj_for_changed_files(self): - return self._changed_dobj + def data_paths_for_changed_files(self): + return self._changed_data_paths @property def externally_created_files(self): diff --git a/dvc/state_file.py b/dvc/state_file.py index 4e5677b83..441586226 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -4,8 +4,6 @@ import json import time from dvc.exceptions import NeatLynxException -from dvc.git_wrapper import GitWrapper -from dvc.logger import Logger class StateFileError(NeatLynxException): diff --git a/dvc/utils.py b/dvc/utils.py new file mode 100644 index 000000000..d90a50cdc --- /dev/null +++ b/dvc/utils.py @@ -0,0 +1,14 @@ + +def cached_property(f): + def get(self): + try: + return self._property_cache[f] + except AttributeError: + self._property_cache = {} + x = self._property_cache[f] = f(self) + return x + except KeyError: + x = self._property_cache[f] = f(self) + return x + + return property(get) \ No newline at end of file diff --git a/tests/test_data_file_obj.py b/tests/test_data_file_obj.py index f4be881f2..5d67fe54f 100644 --- a/tests/test_data_file_obj.py +++ b/tests/test_data_file_obj.py @@ -3,14 +3,16 @@ import os import shutil from dvc.config import ConfigI -from dvc.data_file_obj import DataFileObj, NotInDataDirError, DataFilePathError +from dvc.data_file_obj import NotInDataDirError, DataFilePathError, DataPath from dvc.git_wrapper import GitWrapperI +from dvc.path_factory import PathFactory class BasicDataDirTest(TestCase): def setUp(self, test_dir, curr_dir=None): self._test_dir = os.path.realpath(test_dir) - self._test_git_dir = os.path.join(self._test_dir, 'proj') + self._proj_dir = 'proj' + self._test_git_dir = os.path.join(self._test_dir, self._proj_dir) self._old_curr_dir_abs = os.path.realpath(os.curdir) shutil.rmtree(self._test_dir, ignore_errors=True) @@ -26,6 +28,10 @@ class BasicDataDirTest(TestCase): if not os.path.isdir(self._test_git_dir): os.makedirs(self._test_git_dir) + data_dir = os.path.join(self._test_git_dir, 'data') + if not os.path.isdir(data_dir): + os.makedirs(data_dir) + os.chdir(self._curr_dir) def tearDown(self): @@ -40,56 +46,98 @@ class TestDataFileObjBasic(BasicDataDirTest): git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8') config = ConfigI('data', 'cache', 'state') - self._dobj = DataFileObj(os.path.join('data', 'file.txt'), git, config) + file = os.path.join('data', 'file.txt') + self._data_path = DataPath(file, git, config) pass def test_data_file(self): - self.assertEqual(self._dobj.data_file_abs, os.path.join(self._test_git_dir, 'data', 'file.txt')) + data_file_full_path = os.path.join(self._test_git_dir, 'data', 'file.txt') + self.assertEqual(self._data_path.data.abs, data_file_full_path) def test_cache_file(self): - self.assertEqual(self._dobj.cache_file_abs, os.path.join(self._test_git_dir, 'cache', 'file.txt_ad45ba8')) + cache_file_full_path = os.path.join(self._test_git_dir, 'cache', 'file.txt_ad45ba8') + self.assertEqual(self._data_path.cache.abs, cache_file_full_path) def test_state_file(self): - self.assertEqual(self._dobj.state_file_abs, os.path.join(self._test_git_dir, 'state', 'file.txt.state')) + state_file_full_path = os.path.join(self._test_git_dir, 'state', 'file.txt.state') + self.assertEqual(self._data_path.state.abs, state_file_full_path) + + def test_data_dvs_short(self): + self.assertEqual(self._data_path.data_dvc_short, 'file.txt') + pass -class TestNlxNames(BasicDataDirTest): +class TestPathFactory(BasicDataDirTest): def setUp(self): BasicDataDirTest.setUp(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test')) git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8') config = ConfigI('data', 'cache', 'state') + self.path_factory = PathFactory(git, config) - self._dobj = DataFileObj(os.path.join('data', 'file.txt'), git, config) + self.file = os.path.join('data', 'file.txt') + + fd = open(self.file, 'w+') + fd.write('some text') + fd.close() + + self.file_symlink = os.path.join('data', 'fsymlinc.txt') + os.chdir('data') + os.symlink('file.txt', 'fsymlinc.txt') + os.chdir('..') pass - def test_data_nlx(self): - print('==============================') - self.assertEqual(self._dobj.data_file_nlx, 'file.txt') + def test_data_file_factory(self): + data_path = self.path_factory.data_path(self.file) + self.assertEqual(data_path.data.dvc, self.file) + + indirect_file_path = os.path.join('..', self._proj_dir, self.file) + data_path_indirect = self.path_factory.data_path(indirect_file_path) + self.assertEqual(data_path_indirect.data.dvc, self.file) + pass + + def test_data_symlink_factory(self): + data_path = self.path_factory.existing_data_path(self.file_symlink) + self.assertEqual(data_path.data.dvc, self.file_symlink) + pass + + def test_data_symlink_factory_cache(self): + data_path = self.path_factory.existing_data_path(self.file_symlink) + self.assertEqual(data_path.cache.dvc, self.file) + pass + + def test_data_symlink_factory_exception(self): + with self.assertRaises(DataFilePathError): + self.path_factory.existing_data_path(self.file) pass -class TestDataFileObjInDataDir(BasicDataDirTest): +class TestDataPathInDataDir(BasicDataDirTest): def setUp(self): BasicDataDirTest.setUp(self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test')) git = GitWrapperI(git_dir=self._test_git_dir, commit='eeeff8f') config = ConfigI('da', 'ca', 'st') + path_factory = PathFactory(git, config) - self._dobj = DataFileObj(os.path.join('da', 'dir1', 'd2', 'file.txt'), git, config) + deep_path = os.path.join('da', 'dir1', 'd2', 'file.txt') + self.data_path = path_factory.data_path(deep_path) pass def test_data_file(self): target = os.path.join(self._test_git_dir, 'da', 'dir1', 'd2', 'file.txt') - self.assertEqual(self._dobj.data_file_abs, target) + self.assertEqual(self.data_path.data.abs, target) def test_cache_file(self): target = os.path.join(self._test_git_dir, 'ca', 'dir1', 'd2', 'file.txt_eeeff8f') - self.assertEqual(self._dobj.cache_file_abs, target) + self.assertEqual(self.data_path.cache.abs, target) def test_state_file(self): target = os.path.join(self._test_git_dir, 'st', 'dir1', 'd2', 'file.txt.state') - self.assertEqual(self._dobj.state_file_abs, target) + self.assertEqual(self.data_path.state.abs, target) + + def test_symlink(self): + self.assertEqual(self.data_path._symlink_file, '../../../ca/dir1/d2/file.txt_eeeff8f') class TestDataFileObjLongPath(BasicDataDirTest): @@ -102,36 +150,38 @@ class TestDataFileObjLongPath(BasicDataDirTest): self._git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8') self._config = ConfigI('data', 'cache', 'state') + self.path_factory = PathFactory(self._git, self._config) - self._dobj = DataFileObj(os.path.join('..', '..', 'data', 'file1.txt'), - self._git, self._config) + # self._dobj = DataFileObj(os.path.join('..', '..', 'data', 'file1.txt'), + # self._git, self._config) + self.data_path = self.path_factory.data_path(os.path.join('..', '..', 'data', 'file1.txt')) pass def test_data_file(self): target = os.path.join(self._test_git_dir, 'data', 'file1.txt') - self.assertEqual(self._dobj.data_file_abs, target) + self.assertEqual(self.data_path.data.abs, target) def test_cache_file(self): target = os.path.join(self._test_git_dir, 'cache', 'file1.txt_123ed8') - self.assertEqual(self._dobj.cache_file_abs, target) + self.assertEqual(self.data_path.cache.abs, target) def test_state_file(self): target = os.path.join(self._test_git_dir, 'state', 'file1.txt.state') - self.assertEqual(self._dobj.state_file_abs, target) + self.assertEqual(self.data_path.state.abs, target) def test_file_name_only(self): with self.assertRaises(NotInDataDirError): - DataFileObj('file.txt', self._git, self._config) + self.path_factory.data_path('file.txt') pass def test_relative_git_dir(self): with self.assertRaises(NotInDataDirError): - DataFileObj('data/file.txt', self._git, self._config) + self.path_factory.data_path('data/file.txt') pass def test_relative_path_error(self): with self.assertRaises(NotInDataDirError): - DataFileObj('../data/file.txt', self._git, self._config) + self.path_factory.data_path('../data/file.txt') pass @@ -146,7 +196,8 @@ class RunOutsideGitRepoTest(BasicDataDirTest): def test_outside_git_dir(self): git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8') config = ConfigI('data', 'cache', 'state') + path_factory = PathFactory(git, config) with self.assertRaises(NotInDataDirError): - DataFileObj('file.txt', git, config) + path_factory.data_path('file.txt') pass diff --git a/tests/test_dvc_path.py b/tests/test_dvc_path.py index 83cc8269d..402d17dae 100644 --- a/tests/test_dvc_path.py +++ b/tests/test_dvc_path.py @@ -2,6 +2,7 @@ import os import shutil from unittest import TestCase +from dvc.dvc_path import DvcPath from dvc.git_wrapper import GitWrapperI @@ -13,7 +14,7 @@ class TestDvcPathTest(TestCase): os.makedirs(os.path.join(self.test_dir, 'code', 'lib')) os.makedirs(os.path.join(self.test_dir, 'd1', 'd2', 'dir3', 'd4', 'dir5')) - self.git = GitWrapperI(self.test_dir) + self._git = GitWrapperI(self.test_dir) def _validate_dvc_path(self, path, dvc_file_name, relative_file_name): self.assertEqual(path.dvc, dvc_file_name) @@ -23,11 +24,10 @@ class TestDvcPathTest(TestCase): self.assertTrue(path.abs.endswith(dvc_file_name)) def basic_test(self): - print('----------------------------------------------') os.chdir(self.test_dir) file = os.path.join('data', 'file.txt') - path = self.git.build_dvc_path(file) + path = DvcPath(file, self._git) self._validate_dvc_path(path, file, file) pass @@ -37,9 +37,8 @@ class TestDvcPathTest(TestCase): file_dvc_path = os.path.join('data', 'file1.txt') file_relative_path = os.path.join('..', file_dvc_path) - path = self.git.build_dvc_path(file_relative_path) + path = DvcPath(file_relative_path, self._git) self._validate_dvc_path(path, file_dvc_path, file_relative_path) - print('+++++++++++++++++++++++++++++') pass def from_deep_dirs_test(self): @@ -49,7 +48,7 @@ class TestDvcPathTest(TestCase): file_dvc = os.path.join('code', 'lib', 'context_switcher_structs.asm') file_relative = os.path.join('..', '..', '..', file_dvc) - path = self.git.build_dvc_path(file_relative) + path = DvcPath(file_relative, self._git) self._validate_dvc_path(path, file_dvc, file_relative) pass @@ -60,6 +59,6 @@ class TestDvcPathTest(TestCase): file_relative_path = os.path.join(deep_dir, 'd4', 'dir5', 'rawdata.tsv') file_dvc_path = os.path.join(deep_dir, file_relative_path) - path = self.git.build_dvc_path(file_relative_path) + path = DvcPath(file_relative_path, self._git) self._validate_dvc_path(path, file_dvc_path, file_relative_path) pass