Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Refactoring: DataFileObj to DvcPath and DataPath

Browse Source
This commit is contained in:
Dmitry Petrov
2017-03-22 01:03:09 -07:00
parent 874073bb01
commit 4b4bf15a8f
14 changed files with 275 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cmd_base.py
+10 -1
View File
@@ -4,6 +4,8 @@ import os
from dvc.git_wrapper import GitWrapper
from dvc.config import Config, ConfigError
from dvc.logger import Logger
from dvc.path_factory import PathFactory
from dvc.utils import cached_property
class CmdBase(object):
@@ -30,10 +32,17 @@ class CmdBase(object):
raise ConfigError("DVC_HOME directory doesn't exists")
pass
@cached_property
def path_factory(self):
return PathFactory(self.git, self.config)
@property
def config(self):
return self._config
def cache_file_aws_key(self, file):
return '{}/{}'.format(self._config.aws_storage_prefix, file).strip('/')
@property
def lnx_home(self):
return self._dvc_home
@@ -46,7 +55,7 @@ class CmdBase(object):
def git(self):
return self._git
def define_args(self):
def define_args(self, parser):
pass
def set_skip_git_actions(self, parser):
dvc/cmd_data_import.py
+14 -15
View File
@@ -8,7 +8,6 @@ import requests
from dvc.cmd_base import CmdBase
from dvc.cmd_data_sync import sizeof_fmt
from dvc.logger import Logger
from dvc.data_file_obj import DataFileObj
from dvc.exceptions import NeatLynxException
from dvc.state_file import StateFile
@@ -72,40 +71,40 @@ class CmdDataImport(CmdBase):
if os.path.isdir(output):
output = os.path.join(output, os.path.basename(input))
dobj = DataFileObj(output, self.git, self.config)
data_path = self.path_factory.data_path(output)
if os.path.exists(dobj.data_file_relative):
raise DataImportError('Output file "{}" already exists'.format(dobj.data_file_relative))
if not os.path.isdir(os.path.dirname(dobj.data_file_abs)):
if os.path.exists(data_path.data.relative):
raise DataImportError('Output file "{}" already exists'.format(data_path.data.relative))
if not os.path.isdir(os.path.dirname(data_path.data.relative)):
raise DataImportError('Output file directory "{}" does not exists'.format(
os.path.dirname(dobj.data_file_relative)))
os.path.dirname(data_path.data.relative)))
cache_dir = os.path.dirname(dobj.cache_file_relative)
cache_dir = os.path.dirname(data_path.cache.relative)
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
if CmdDataImport.is_url(input):
Logger.debug('Downloading file {} ...'.format(input))
self.download_file(input, dobj.cache_file_relative)
self.download_file(input, data_path.cache.relative)
Logger.debug('Input file "{}" was downloaded to cache "{}"'.format(
input, dobj.cache_file_relative))
input, data_path.cache.relative))
else:
copyfile(input, dobj.cache_file_relative)
copyfile(input, data_path.cache.relative)
Logger.debug('Input file "{}" was copied to cache "{}"'.format(
input, dobj.cache_file_relative))
input, data_path.cache.relative))
dobj.create_symlink()
data_path.create_symlink()
Logger.debug('Symlink from data file "{}" to the cache file "{}" was created'.
format(dobj.data_file_relative, dobj.cache_file_relative))
format(data_path.data.relative, data_path.cache.relative))
state_file = StateFile(dobj.state_file_relative,
state_file = StateFile(data_path.state.relative,
self.git,
[input],
[output],
[],
is_reproducible)
state_file.save()
Logger.debug('State file "{}" was created'.format(dobj.state_file_relative))
Logger.debug('State file "{}" was created'.format(data_path.state.relative))
pass
URL_REGEX = re.compile(
dvc/cmd_data_remove.py
+18 -14
View File
@@ -5,7 +5,6 @@ import fasteners
from dvc.cmd_base import CmdBase
from dvc.logger import Logger
from dvc.exceptions import NeatLynxException
from dvc.data_file_obj import DataFileObjExisting
class DataRemoveError(NeatLynxException):
@@ -59,27 +58,32 @@ class CmdDataRemove(CmdBase):
raise DataRemoveError('data directory cannot be removed')
return self.remove_dir(target)
dobj = DataFileObjExisting(target, self.git, self.config)
if os.path.islink(dobj.data_file_relative):
return self.remove_symlink(dobj.data_file_relative)
data_path = self.path_factory.existing_data_path(target)
if os.path.islink(data_path.data.relative):
return self.remove_symlink(data_path)
raise DataRemoveError('Cannot remove a regular file "{}"'.format(target))
def remove_symlink(self, file):
dobj = DataFileObjExisting(file, self.git, self.config)
@staticmethod
def remove_dir_if_empty(file):
dir = os.path.dirname(file)
if dir != '' and not os.listdir(dir):
os.rmdir(dir)
if os.path.isfile(dobj.cache_file_relative):
os.remove(dobj.cache_file_relative)
dobj.remove_cache_dir_if_empty()
def remove_symlink(self, data_path):
if os.path.isfile(data_path.cache.relative):
os.remove(data_path.cache.relative)
self.remove_dir_if_empty(data_path.cache.relavive)
if os.path.isfile(dobj.state_file_relative):
os.remove(dobj.state_file_relative)
dobj.remove_state_dir_if_empty()
if os.path.isfile(data_path.state.relative):
os.remove(data_path.state.relative)
self.remove_dir_if_empty(data_path.state.relative)
if self.args.remove_from_cloud:
self.remove_from_cloud(dobj.cache_file_aws_key)
self.remove_from_cloud(data_path.cache_file_aws_key)
os.remove(file)
os.remove(data_path.data.relative)
pass
def remove_from_cloud(self, aws_file_name):
dvc/cmd_data_sync.py
+21 -20
View File
@@ -5,7 +5,6 @@ from boto.s3.connection import S3Connection
from dvc.cmd_base import CmdBase
from dvc.logger import Logger
from dvc.data_file_obj import DataFileObjExisting
from dvc.exceptions import NeatLynxException
@@ -53,8 +52,8 @@ class CmdDataSync(CmdBase):
def run(self):
if os.path.islink(self.args.target):
dobj = DataFileObjExisting(self.args.target, self.git, self.config)
return self.sync_symlink(dobj)
data_path = self.path_factory.existing_data_path(self.args.target)
return self.sync_symlink(data_path)
if os.path.isdir(self.args.target):
return self.sync_dir(self.args.target)
@@ -67,46 +66,48 @@ class CmdDataSync(CmdBase):
if os.path.isdir(fname):
self.sync_dir(fname)
elif os.path.islink(fname):
self.sync_symlink(DataFileObjExisting(fname, self.git, self.config))
self.sync_symlink(self.path_factory.existing_data_path(fname))
else:
raise DataSyncError('Unsupported file type "{}"'.format(fname))
pass
def sync_symlink(self, dobj):
if os.path.isfile(dobj.cache_file_relative):
self.sync_to_cloud(dobj)
def sync_symlink(self, data_path):
if os.path.isfile(data_path.cache.relative):
self.sync_to_cloud(data_path)
else:
self.sync_from_cloud(dobj)
self.sync_from_cloud(data_path)
pass
def sync_from_cloud(self, dobj):
key = self._bucket.get_key(dobj.cache_file_aws_key)
def sync_from_cloud(self, data_path):
aws_key = self.cache_file_aws_key(data_path.cache.dvc)
key = self._bucket.get_key(aws_key)
if not key:
raise DataSyncError('File "{}" does not exist in the cloud'.format(dobj.cache_file_aws_key))
raise DataSyncError('File "{}" does not exist in the cloud'.format(aws_key))
Logger.printing('Downloading cache file from S3 "{}/{}"'.format(self._bucket.name,
dobj.cache_file_aws_key))
key.get_contents_to_filename(dobj.cache_file_relative, cb=percent_cb)
aws_key))
key.get_contents_to_filename(data_path.cache.relative, cb=percent_cb)
Logger.printing('Downloading completed')
pass
def sync_to_cloud(self, dobj):
key = self._bucket.get_key(dobj.cache_file_aws_key)
def sync_to_cloud(self, data_path):
aws_key = self.cache_file_aws_key(data_path.cache.dvc)
key = self._bucket.get_key(aws_key)
if key:
Logger.debug('File already uploaded to the cloud. Checksum validation...')
md5_cloud = key.etag[1:-1]
md5_local = file_md5(dobj.cache_file_relative)
md5_local = file_md5(data_path.cache.relative)
if md5_cloud == md5_local:
Logger.debug('File checksum matches. No uploading is needed.')
return
Logger.printing('Checksum miss-match. Re-uploading is required.')
Logger.printing('Uploading cache file "{}" to S3 "{}"'.format(dobj.cache_file_relative,
dobj.cache_file_aws_key))
key = self._bucket.new_key(dobj.cache_file_aws_key)
key.set_contents_from_filename(dobj.cache_file_relative, cb=percent_cb)
Logger.printing('Uploading cache file "{}" to S3 "{}"'.format(data_path.cache.relative,
aws_key))
key = self._bucket.new_key(aws_key)
key.set_contents_from_filename(data_path.cache.relative, cb=percent_cb)
Logger.printing('Uploading completed')
pass
dvc/cmd_repro.py
+22 -23
View File
@@ -4,10 +4,9 @@ import fasteners
from dvc.cmd_base import CmdBase
from dvc.cmd_run import CmdRun
from dvc.git_wrapper import GitWrapper
from dvc.logger import Logger
from dvc.exceptions import NeatLynxException
from dvc.data_file_obj import DataFileObj, NotInDataDirError
from dvc.data_file_obj import NotInDataDirError
from dvc.state_file import StateFile
@@ -55,29 +54,29 @@ class CmdRepro(CmdRun):
if not self.skip_git_actions and not self.git.is_ready_to_go():
return 1
dobjs, externally_created_files = DataFileObj.files_to_dobjs(self.args.target, self.git, self.config)
if externally_created_files:
data_path_list, external_files = self.path_factory.to_data_path(self.args.target)
if external_files:
Logger.error('Files from outside of the data directory "{}" could not be reproduced: {}'.
format(self.config.data_dir, ' '.join(externally_created_files)))
format(self.config.data_dir, ' '.join(external_files)))
return 1
error = False
changed = False
for dobj in dobjs:
for data_path in data_path_list:
try:
repro_change = ReproChange(dobj, self.git, self)
repro_change = ReproChange(data_path, self)
if repro_change.reproduce():
changed = True
Logger.info(u'Data file "{}" was reproduced.'.format(
dobj.data_file_relative
data_path.data.relative
))
else:
Logger.info(u'Reproduction is not required for data file "{}".'.format(
dobj.data_file_relative
data_path.data.relative
))
except ReproError as err:
Logger.error('Error in reproducing data file {}: {}'.format(
dobj.data_file_relative, str(err)
data_path.data.relative, str(err)
))
error = True
break
@@ -100,11 +99,11 @@ class CmdRepro(CmdRun):
class ReproChange(object):
def __init__(self, dobj, git, cmd_obj):
self._dobj = dobj
self._state = StateFile.load(dobj.state_file_relative, git)
self.git = git
def __init__(self, data_path, cmd_obj):
self._data_path = data_path
self.git = cmd_obj.git
self._cmd_obj = cmd_obj
self._state = StateFile.load(data_path.state.relative, self.git)
cmd_obj._code = self._state.code_sources # HACK!!!
@@ -112,7 +111,7 @@ class ReproChange(object):
if not argv:
raise ReproError('Error: parameter {} is nor defined in state file "{}"'.
format(StateFile.PARAM_NORM_ARGV, dobj.state_file_relative))
format(StateFile.PARAM_NORM_ARGV, data_path.state.relative))
if len(argv) < 2:
raise ReproError('Error: reproducible command in state file "{}" is too short'.
format(self._state.file))
@@ -129,39 +128,39 @@ class ReproChange(object):
def reproduce_data_file(self):
Logger.debug('Reproducing data file "{}". Removing the file...'.format(
self._dobj.data_file_relative))
os.remove(self._dobj.data_file_relative)
self._data_path.data.relative))
os.remove(self._data_path.data.relative)
Logger.debug('Reproducing data file "{}". Re-runs command: {}'.format(
self._dobj.data_file_relative, ' '.join(self._repro_argv)))
self._data_path.data.relative, ' '.join(self._repro_argv)))
return self._cmd_obj.run_command(self._repro_argv)
def reproduce(self, force=False):
were_input_files_changed = False
if not self._state.is_reproducible:
Logger.debug('Data file "{}" is not reproducible'.format(self._dobj.data_file_relative))
Logger.debug('Data file "{}" is not reproducible'.format(self._data_path.data.relative))
return False
for input_file in self._state.input_files:
try:
dobj = DataFileObj(input_file, self.git, self._cmd_obj.config)
data_path = self._cmd_obj.path_factory.data_path(input_file)
except NotInDataDirError:
raise ReproError(u'The dependency files "{}" is not a data file'.format(input_file))
except Exception as ex:
raise ReproError(u'The dependency files "{}" can not be reproduced: {}'.format(
input_file, ex))
change = ReproChange(dobj, self.git, self._cmd_obj)
change = ReproChange(data_path, self._cmd_obj)
if change.reproduce(force):
were_input_files_changed = True
was_source_code_changed = self.git.were_files_changed(self._dobj.data_file_relative,
was_source_code_changed = self.git.were_files_changed(self._data_path.data.relative,
self._state.code_sources)
if not force and not was_source_code_changed and not were_input_files_changed:
Logger.debug('Data file "{}" is up to date'.format(
self._dobj.data_file_relative))
self._data_path.data.relative))
return False
return self.reproduce_data_file()
dvc/cmd_run.py
+10 -10
View File
@@ -2,7 +2,7 @@ import os
import shutil
import fasteners
from dvc.data_file_obj import DataFileObj, NotInDataDirError
from dvc.data_file_obj import NotInDataDirError
from dvc.git_wrapper import GitWrapper
from dvc.cmd_base import CmdBase
from dvc.logger import Logger
@@ -77,7 +77,7 @@ class CmdRun(CmdBase):
return 0
def run_command(self, argv, stdout=None, stderr=None):
repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config)
repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config, self.path_factory)
# print('===== new_files={}'.format(repo_change.new_files))
# print('===== modified_files={}'.format(repo_change.modified_files))
@@ -96,21 +96,21 @@ class CmdRun(CmdBase):
input_files_from_args = list(set(args_files_nlx) - set(changed_files_nlx))
input_files = self.git.abs_paths_to_nlx(input_files_from_args + self.declaration_input_files)
for dobj in repo_change.dobj_for_changed_files:
dirname = os.path.dirname(dobj.cache_file_relative)
for data_path in repo_change.data_paths_for_changed_files:
dirname = os.path.dirname(data_path.cache.relative)
if not os.path.isdir(dirname):
os.makedirs(dirname)
Logger.debug('Move output file "{}" to cache dir "{}" and create a symlink'.format(
dobj.data_file_relative, dobj.cache_file_relative))
shutil.move(dobj.data_file_relative, dobj.cache_file_relative)
data_path.data.relative, data_path.cache.relative))
shutil.move(data_path.data.relative, data_path.cache.relative)
dobj.create_symlink()
data_path.create_symlink()
nlx_code_sources = map(lambda x: self.git.abs_paths_to_nlx([x])[0], self.code)
Logger.debug('Create state file "{}"'.format(dobj.state_file_relative))
state_file = StateFile(dobj.state_file_relative, self.git,
Logger.debug('Create state file "{}"'.format(data_path.state.relative))
state_file = StateFile(data_path.state.relative, self.git,
input_files,
output_files,
nlx_code_sources,
@@ -165,7 +165,7 @@ class CmdRun(CmdBase):
for arg in argv:
try:
if os.path.isfile(arg):
DataFileObj(arg, self.git, self.config)
self.path_factory.data_path(arg)
result.append(arg)
except NotInDataDirError:
pass
dvc/data_file_obj.py
+38 -116
View File
@@ -1,6 +1,8 @@
import os
from dvc.dvc_path import DvcPath
from dvc.exceptions import NeatLynxException
from dvc.utils import cached_property
class DataFilePathError(NeatLynxException):
@@ -15,135 +17,55 @@ class NotInDataDirError(NeatLynxException):
format(file, data_dir))
class DataFileObj(object):
class DataPath(object):
STATE_FILE_SUFFIX = '.state'
CACHE_FILE_SEP = '_'
def __init__(self, data_file, git, config):
'''
nlx file name - system file name without data/cache/state prefix
'''
def __init__(self, data_file, git, config, cache_file=None):
self._git = git
self._config = config
self._curr_dir_abs = os.path.realpath(os.curdir)
self._cache_file = cache_file
self._data_file = data_file
self._data = DvcPath(data_file, git)
if not self._data.abs.startswith(self.data_dir_abs):
raise NotInDataDirError(data_file, self._config.data_dir)
if not self.data_file_abs.startswith(self.data_dir_abs):
raise NotInDataDirError(self._data_file, self._config.data_dir)
if not self._data.dvc.startswith(self._config.data_dir):
raise NotInDataDirError(data_file, self._config.data_dir)
pass
def create_symlink(self):
data_dir = os.path.dirname(self.data_file_relative)
cache_relative_to_data_dir = os.path.relpath(self.cache_file_relative, data_dir)
os.symlink(cache_relative_to_data_dir, self.data_file_relative)
@property
def data(self):
return self._data
@property
def data_dvc_short(self):
return self._data.dvc[len(self._config.data_dir)+1:]
@cached_property
def state(self):
state_dir = os.path.join(self._git.git_dir_abs, self._config.state_dir)
state_file = os.path.join(state_dir, self.data_dvc_short + self.STATE_FILE_SUFFIX)
return DvcPath(state_file, self._git)
@cached_property
def cache(self):
if self._cache_file:
return DvcPath(self._cache_file, self._git)
else:
cache_dir = os.path.join(self._git.git_dir_abs, self._config.cache_dir)
cache_file_suffix = self.CACHE_FILE_SEP + self._git.curr_commit
cache_file = os.path.join(cache_dir, self.data_dvc_short + cache_file_suffix)
return DvcPath(cache_file, self._git)
@cached_property
def data_dir_abs(self):
return os.path.join(self._git.git_dir_abs, self._config.data_dir)
# Data file properties
@property
def data_file_name(self):
return os.path.basename(self._data_file)
def _symlink_file(self):
data_file_dir = os.path.dirname(self.data.relative)
return os.path.relpath(self.cache.relative, data_file_dir)
@property
def data_file_nlx(self):
return os.path.relpath(self.data_file_abs, self.data_dir_abs)
# @property
# def data_file_abs(self):
# return os.path.realpath(self.data_file_relative)
@property
def data_file_abs(self):
'''Do not fully resolve file name since it is a link'''
return os.path.abspath(self.data_file_relative)
@property
def data_file_relative(self):
return self._data_file
# Cache file properties
@property
def cache_file_name(self):
if not self._git.curr_commit:
return None
return self.data_file_name + self.CACHE_FILE_SEP + self._git.curr_commit
@property
def cache_file_nlx(self):
if not self._git.curr_commit:
return None
return self.data_file_nlx + self.CACHE_FILE_SEP + self._git.curr_commit
@property
def cache_file_aws_key(self):
return '{}/{}'.format(self._config.aws_storage_prefix, self.cache_file_nlx).strip('/')
@property
def cache_file_abs(self):
return os.path.join(self._git.git_dir_abs, self._config.cache_dir, self.cache_file_nlx)
@property
def cache_file_relative(self):
return os.path.relpath(self.cache_file_abs, self._curr_dir_abs)
# State file properties
@property
def state_file_name(self):
return self.data_file_name + self.STATE_FILE_SUFFIX
@property
def state_file_nlx(self):
return self.data_file_nlx + self.STATE_FILE_SUFFIX
@property
def state_file_abs(self):
return os.path.join(self._git.git_dir_abs, self._config.state_dir, self.state_file_nlx)
@property
def state_file_relative(self):
return os.path.relpath(self.state_file_abs, self._curr_dir_abs)
@staticmethod
def files_to_dobjs(files, git, config):
result = []
externally_created_files = []
for file in files:
try:
result.append(DataFileObj(file, git, config))
except NotInDataDirError:
externally_created_files.append(file)
return result, externally_created_files
class DataFileObjExisting(DataFileObj):
def __init__(self, data_file, git, config):
DataFileObj.__init__(self, data_file, git, config)
if not os.path.islink(data_file):
raise DataFilePathError('Data file must be a symbolic link')
pass
@property
def cache_file_name(self):
return os.path.basename(self.cache_file_abs)
@property
def cache_file_abs(self):
return os.path.realpath(self.data_file_relative)
@staticmethod
def remove_dir_if_empty(dir):
cache_file_dir = os.path.dirname(dir)
if cache_file_dir != '' and not os.listdir(cache_file_dir):
os.rmdir(cache_file_dir)
def remove_state_dir_if_empty(self):
self.remove_dir_if_empty(self.state_file_relative)
def remove_cache_dir_if_empty(self):
self.remove_dir_if_empty(self.cache_file_relative)
def create_symlink(self):
os.symlink(self._symlink_file, self.data.relative)
dvc/dvc_path.py
+4 -4
View File
@@ -2,10 +2,10 @@ import os
class DvcPath(object):
def __init__(self, relative_raw, git_dir_abs, curr_dir_abs):
self._abs = os.path.realpath(relative_raw)
self._dvc = os.path.relpath(self.abs, git_dir_abs)
self._relative = os.path.relpath(self._abs, curr_dir_abs)
def __init__(self, relative_raw, git):
self._abs = os.path.abspath(relative_raw)
self._dvc = os.path.relpath(self.abs, git.git_dir_abs)
self._relative = os.path.relpath(self._abs, git.curr_dir_abs)
self._filename = os.path.basename(self._abs)
@property
dvc/path_factory.py
+36
View File
@@ -0,0 +1,36 @@
import os
from dvc.data_file_obj import DataPath, DataFilePathError, NotInDataDirError
from dvc.dvc_path import DvcPath
class PathFactory(object):
def __init__(self, git, config):
self._git = git
self._config = config
self._curr_dir_abs = os.path.realpath(os.curdir)
def path(self, relative_raw):
return DvcPath(relative_raw, self._git)
def data_path(self, data_file):
return DataPath(data_file, self._git, self._config)
def existing_data_path(self, data_file):
if not os.path.islink(data_file):
raise DataFilePathError(u'Data file "%s" must be a symbolic link' % data_file)
resolved_symlink = os.path.realpath(data_file)
return DataPath(data_file, self._git, self._config, resolved_symlink)
def to_data_path(self, files):
result = []
externally_created_files = []
for file in files:
try:
data_path = DataPath(file, self._git, self._config)
result.append(data_path)
except NotInDataDirError:
externally_created_files.append(file)
return result, externally_created_files
dvc/repository_change.py
+6 -7
View File
@@ -1,6 +1,5 @@
import os
from dvc.data_file_obj import DataFileObj, NotInDataDirError
from dvc.exceptions import NeatLynxException
from dvc.git_wrapper import GitWrapper
@@ -44,18 +43,18 @@ class FileModificationState(object):
class RepositoryChange(object):
"""Pre-condition: git repository has no changes"""
def __init__(self, args, stdout, stderr, git, config):
def __init__(self, args, stdout, stderr, git, config, path_factory):
self.git = git
self.config = config
self.path_factory = path_factory
GitWrapper.exec_cmd_only_success(args, stdout, stderr)
self._file_states = self._get_file_states()
self._changed_dobj, self._externally_created_files = DataFileObj.files_to_dobjs(
self.changed_files, self.git, self.config)
self._changed_data_paths, self._externally_created_files = path_factory.to_data_path(self.changed_files)
@staticmethod
def exec_cmd(args, stdout, stderr, git):
def exec_cmd(args, stdout, stderr, git, path_factory):
return RepositoryChange(args, stdout, stderr, git)
@staticmethod
@@ -113,8 +112,8 @@ class RepositoryChange(object):
pass
@property
def dobj_for_changed_files(self):
return self._changed_dobj
def data_paths_for_changed_files(self):
return self._changed_data_paths
@property
def externally_created_files(self):
dvc/state_file.py
-2
View File
@@ -4,8 +4,6 @@ import json
import time
from dvc.exceptions import NeatLynxException
from dvc.git_wrapper import GitWrapper
from dvc.logger import Logger
class StateFileError(NeatLynxException):
dvc/utils.py
+14
View File
@@ -0,0 +1,14 @@
def cached_property(f):
def get(self):
try:
return self._property_cache[f]
except AttributeError:
self._property_cache = {}
x = self._property_cache[f] = f(self)
return x
except KeyError:
x = self._property_cache[f] = f(self)
return x
return property(get)
tests/test_data_file_obj.py
+76 -25
View File
@@ -3,14 +3,16 @@ import os
import shutil
from dvc.config import ConfigI
from dvc.data_file_obj import DataFileObj, NotInDataDirError, DataFilePathError
from dvc.data_file_obj import NotInDataDirError, DataFilePathError, DataPath
from dvc.git_wrapper import GitWrapperI
from dvc.path_factory import PathFactory
class BasicDataDirTest(TestCase):
def setUp(self, test_dir, curr_dir=None):
self._test_dir = os.path.realpath(test_dir)
self._test_git_dir = os.path.join(self._test_dir, 'proj')
self._proj_dir = 'proj'
self._test_git_dir = os.path.join(self._test_dir, self._proj_dir)
self._old_curr_dir_abs = os.path.realpath(os.curdir)
shutil.rmtree(self._test_dir, ignore_errors=True)
@@ -26,6 +28,10 @@ class BasicDataDirTest(TestCase):
if not os.path.isdir(self._test_git_dir):
os.makedirs(self._test_git_dir)
data_dir = os.path.join(self._test_git_dir, 'data')
if not os.path.isdir(data_dir):
os.makedirs(data_dir)
os.chdir(self._curr_dir)
def tearDown(self):
@@ -40,56 +46,98 @@ class TestDataFileObjBasic(BasicDataDirTest):
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
config = ConfigI('data', 'cache', 'state')
self._dobj = DataFileObj(os.path.join('data', 'file.txt'), git, config)
file = os.path.join('data', 'file.txt')
self._data_path = DataPath(file, git, config)
pass
def test_data_file(self):
self.assertEqual(self._dobj.data_file_abs, os.path.join(self._test_git_dir, 'data', 'file.txt'))
data_file_full_path = os.path.join(self._test_git_dir, 'data', 'file.txt')
self.assertEqual(self._data_path.data.abs, data_file_full_path)
def test_cache_file(self):
self.assertEqual(self._dobj.cache_file_abs, os.path.join(self._test_git_dir, 'cache', 'file.txt_ad45ba8'))
cache_file_full_path = os.path.join(self._test_git_dir, 'cache', 'file.txt_ad45ba8')
self.assertEqual(self._data_path.cache.abs, cache_file_full_path)
def test_state_file(self):
self.assertEqual(self._dobj.state_file_abs, os.path.join(self._test_git_dir, 'state', 'file.txt.state'))
state_file_full_path = os.path.join(self._test_git_dir, 'state', 'file.txt.state')
self.assertEqual(self._data_path.state.abs, state_file_full_path)
def test_data_dvs_short(self):
self.assertEqual(self._data_path.data_dvc_short, 'file.txt')
pass
class TestNlxNames(BasicDataDirTest):
class TestPathFactory(BasicDataDirTest):
def setUp(self):
BasicDataDirTest.setUp(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(git, config)
self._dobj = DataFileObj(os.path.join('data', 'file.txt'), git, config)
self.file = os.path.join('data', 'file.txt')
fd = open(self.file, 'w+')
fd.write('some text')
fd.close()
self.file_symlink = os.path.join('data', 'fsymlinc.txt')
os.chdir('data')
os.symlink('file.txt', 'fsymlinc.txt')
os.chdir('..')
pass
def test_data_nlx(self):
print('==============================')
self.assertEqual(self._dobj.data_file_nlx, 'file.txt')
def test_data_file_factory(self):
data_path = self.path_factory.data_path(self.file)
self.assertEqual(data_path.data.dvc, self.file)
indirect_file_path = os.path.join('..', self._proj_dir, self.file)
data_path_indirect = self.path_factory.data_path(indirect_file_path)
self.assertEqual(data_path_indirect.data.dvc, self.file)
pass
def test_data_symlink_factory(self):
data_path = self.path_factory.existing_data_path(self.file_symlink)
self.assertEqual(data_path.data.dvc, self.file_symlink)
pass
def test_data_symlink_factory_cache(self):
data_path = self.path_factory.existing_data_path(self.file_symlink)
self.assertEqual(data_path.cache.dvc, self.file)
pass
def test_data_symlink_factory_exception(self):
with self.assertRaises(DataFilePathError):
self.path_factory.existing_data_path(self.file)
pass
class TestDataFileObjInDataDir(BasicDataDirTest):
class TestDataPathInDataDir(BasicDataDirTest):
def setUp(self):
BasicDataDirTest.setUp(self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
git = GitWrapperI(git_dir=self._test_git_dir, commit='eeeff8f')
config = ConfigI('da', 'ca', 'st')
path_factory = PathFactory(git, config)
self._dobj = DataFileObj(os.path.join('da', 'dir1', 'd2', 'file.txt'), git, config)
deep_path = os.path.join('da', 'dir1', 'd2', 'file.txt')
self.data_path = path_factory.data_path(deep_path)
pass
def test_data_file(self):
target = os.path.join(self._test_git_dir, 'da', 'dir1', 'd2', 'file.txt')
self.assertEqual(self._dobj.data_file_abs, target)
self.assertEqual(self.data_path.data.abs, target)
def test_cache_file(self):
target = os.path.join(self._test_git_dir, 'ca', 'dir1', 'd2', 'file.txt_eeeff8f')
self.assertEqual(self._dobj.cache_file_abs, target)
self.assertEqual(self.data_path.cache.abs, target)
def test_state_file(self):
target = os.path.join(self._test_git_dir, 'st', 'dir1', 'd2', 'file.txt.state')
self.assertEqual(self._dobj.state_file_abs, target)
self.assertEqual(self.data_path.state.abs, target)
def test_symlink(self):
self.assertEqual(self.data_path._symlink_file, '../../../ca/dir1/d2/file.txt_eeeff8f')
class TestDataFileObjLongPath(BasicDataDirTest):
@@ -102,36 +150,38 @@ class TestDataFileObjLongPath(BasicDataDirTest):
self._git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
self._config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(self._git, self._config)
self._dobj = DataFileObj(os.path.join('..', '..', 'data', 'file1.txt'),
self._git, self._config)
# self._dobj = DataFileObj(os.path.join('..', '..', 'data', 'file1.txt'),
# self._git, self._config)
self.data_path = self.path_factory.data_path(os.path.join('..', '..', 'data', 'file1.txt'))
pass
def test_data_file(self):
target = os.path.join(self._test_git_dir, 'data', 'file1.txt')
self.assertEqual(self._dobj.data_file_abs, target)
self.assertEqual(self.data_path.data.abs, target)
def test_cache_file(self):
target = os.path.join(self._test_git_dir, 'cache', 'file1.txt_123ed8')
self.assertEqual(self._dobj.cache_file_abs, target)
self.assertEqual(self.data_path.cache.abs, target)
def test_state_file(self):
target = os.path.join(self._test_git_dir, 'state', 'file1.txt.state')
self.assertEqual(self._dobj.state_file_abs, target)
self.assertEqual(self.data_path.state.abs, target)
def test_file_name_only(self):
with self.assertRaises(NotInDataDirError):
DataFileObj('file.txt', self._git, self._config)
self.path_factory.data_path('file.txt')
pass
def test_relative_git_dir(self):
with self.assertRaises(NotInDataDirError):
DataFileObj('data/file.txt', self._git, self._config)
self.path_factory.data_path('data/file.txt')
pass
def test_relative_path_error(self):
with self.assertRaises(NotInDataDirError):
DataFileObj('../data/file.txt', self._git, self._config)
self.path_factory.data_path('../data/file.txt')
pass
@@ -146,7 +196,8 @@ class RunOutsideGitRepoTest(BasicDataDirTest):
def test_outside_git_dir(self):
git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
config = ConfigI('data', 'cache', 'state')
path_factory = PathFactory(git, config)
with self.assertRaises(NotInDataDirError):
DataFileObj('file.txt', git, config)
path_factory.data_path('file.txt')
pass
tests/test_dvc_path.py
+6 -7
View File
@@ -2,6 +2,7 @@ import os
import shutil
from unittest import TestCase
from dvc.dvc_path import DvcPath
from dvc.git_wrapper import GitWrapperI
@@ -13,7 +14,7 @@ class TestDvcPathTest(TestCase):
os.makedirs(os.path.join(self.test_dir, 'code', 'lib'))
os.makedirs(os.path.join(self.test_dir, 'd1', 'd2', 'dir3', 'd4', 'dir5'))
self.git = GitWrapperI(self.test_dir)
self._git = GitWrapperI(self.test_dir)
def _validate_dvc_path(self, path, dvc_file_name, relative_file_name):
self.assertEqual(path.dvc, dvc_file_name)
@@ -23,11 +24,10 @@ class TestDvcPathTest(TestCase):
self.assertTrue(path.abs.endswith(dvc_file_name))
def basic_test(self):
print('----------------------------------------------')
os.chdir(self.test_dir)
file = os.path.join('data', 'file.txt')
path = self.git.build_dvc_path(file)
path = DvcPath(file, self._git)
self._validate_dvc_path(path, file, file)
pass
@@ -37,9 +37,8 @@ class TestDvcPathTest(TestCase):
file_dvc_path = os.path.join('data', 'file1.txt')
file_relative_path = os.path.join('..', file_dvc_path)
path = self.git.build_dvc_path(file_relative_path)
path = DvcPath(file_relative_path, self._git)
self._validate_dvc_path(path, file_dvc_path, file_relative_path)
print('+++++++++++++++++++++++++++++')
pass
def from_deep_dirs_test(self):
@@ -49,7 +48,7 @@ class TestDvcPathTest(TestCase):
file_dvc = os.path.join('code', 'lib', 'context_switcher_structs.asm')
file_relative = os.path.join('..', '..', '..', file_dvc)
path = self.git.build_dvc_path(file_relative)
path = DvcPath(file_relative, self._git)
self._validate_dvc_path(path, file_dvc, file_relative)
pass
@@ -60,6 +59,6 @@ class TestDvcPathTest(TestCase):
file_relative_path = os.path.join(deep_dir, 'd4', 'dir5', 'rawdata.tsv')
file_dvc_path = os.path.join(deep_dir, file_relative_path)
path = self.git.build_dvc_path(file_relative_path)
path = DvcPath(file_relative_path, self._git)
self._validate_dvc_path(path, file_dvc_path, file_relative_path)
pass