mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
Refactoring: extract Settings from BaseCmd; move path_factory to Settings
This commit is contained in:
+14
-43
@@ -1,53 +1,28 @@
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
from dvc.git_wrapper import GitWrapper
|
||||
from dvc.config import Config, ConfigError
|
||||
from dvc.config import ConfigError
|
||||
from dvc.logger import Logger
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.utils import cached_property
|
||||
|
||||
|
||||
class CmdBase(object):
|
||||
CONFIG = 'dvc.conf'
|
||||
|
||||
def __init__(self, args=None, parse_config=True, git_obj=None, config_obj=None):
|
||||
if git_obj:
|
||||
self._git = git_obj
|
||||
else:
|
||||
self._git = GitWrapper()
|
||||
|
||||
if config_obj:
|
||||
self._config = config_obj
|
||||
else:
|
||||
self._config = None
|
||||
if parse_config:
|
||||
self._config = Config(os.path.realpath(os.path.join(self.git.git_dir, self.CONFIG)))
|
||||
|
||||
if args is None:
|
||||
self._args = sys.argv[1:]
|
||||
else:
|
||||
self._args = args
|
||||
|
||||
self._parsed_args = None
|
||||
self._dvc_home = None
|
||||
def __init__(self, settings):
|
||||
self._settings = settings
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
self.define_args(parser)
|
||||
self._parsed_args, self._command_args = parser.parse_known_args(args=self._args)
|
||||
|
||||
self._dvc_home = os.environ.get('DVC_HOME')
|
||||
|
||||
if not self.lnx_home:
|
||||
raise ConfigError('DVC_HOME environment variable is not defined')
|
||||
if not os.path.exists(self._dvc_home):
|
||||
raise ConfigError("DVC_HOME directory doesn't exists")
|
||||
self._parsed_args, self._command_args = parser.parse_known_args(args=self.args)
|
||||
pass
|
||||
|
||||
@property
|
||||
def settings(self):
|
||||
return self._settings
|
||||
|
||||
@property
|
||||
def args(self):
|
||||
return self._args
|
||||
return self._settings.args
|
||||
|
||||
@property
|
||||
def parsed_args(self):
|
||||
@@ -57,24 +32,20 @@ class CmdBase(object):
|
||||
def command_args(self):
|
||||
return self._command_args
|
||||
|
||||
@cached_property
|
||||
def path_factory(self):
|
||||
return PathFactory(self.git, self.config)
|
||||
|
||||
@property
|
||||
def config(self):
|
||||
return self._config
|
||||
return self._settings.config
|
||||
|
||||
def cache_file_aws_key(self, file):
|
||||
return '{}/{}'.format(self._config.aws_storage_prefix, file).strip('/')
|
||||
return '{}/{}'.format(self.config.aws_storage_prefix, file).strip('/')
|
||||
|
||||
@property
|
||||
def lnx_home(self):
|
||||
return self._dvc_home
|
||||
def dvc_home(self):
|
||||
return self._settings.dvc_home
|
||||
|
||||
@property
|
||||
def git(self):
|
||||
return self._git
|
||||
return self._settings.git
|
||||
|
||||
def define_args(self, parser):
|
||||
pass
|
||||
|
||||
@@ -8,8 +8,8 @@ from dvc.command.base import CmdBase
|
||||
from dvc.command.data_sync import sizeof_fmt
|
||||
from dvc.logger import Logger
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.runtime import Runtime
|
||||
from dvc.state_file import StateFile
|
||||
from dvc.utils import run
|
||||
|
||||
|
||||
class DataImportError(DvcException):
|
||||
@@ -18,9 +18,8 @@ class DataImportError(DvcException):
|
||||
|
||||
|
||||
class CmdDataImport(CmdBase):
|
||||
def __init__(self):
|
||||
CmdBase.__init__(self)
|
||||
pass
|
||||
def __init__(self, settings):
|
||||
super(CmdDataImport, self).__init__(settings)
|
||||
|
||||
def define_args(self, parser):
|
||||
self.set_skip_git_actions(parser)
|
||||
@@ -67,7 +66,7 @@ class CmdDataImport(CmdBase):
|
||||
if os.path.isdir(output):
|
||||
output = os.path.join(output, os.path.basename(input))
|
||||
|
||||
data_item = self.path_factory.data_item(output)
|
||||
data_item = self.settings.path_factory.data_item(output)
|
||||
|
||||
if os.path.exists(data_item.data.relative):
|
||||
raise DataImportError('Output file "{}" already exists'.format(data_item.data.relative))
|
||||
@@ -135,4 +134,4 @@ class CmdDataImport(CmdBase):
|
||||
return
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(CmdDataImport())
|
||||
Runtime.run(CmdDataImport)
|
||||
|
||||
@@ -6,7 +6,7 @@ from boto.s3.connection import S3Connection
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.logger import Logger
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import run
|
||||
from dvc.runtime import Runtime
|
||||
|
||||
|
||||
class DataSyncError(DvcException):
|
||||
@@ -35,8 +35,8 @@ def file_md5(fname):
|
||||
|
||||
|
||||
class CmdDataSync(CmdBase):
|
||||
def __init__(self):
|
||||
CmdBase.__init__(self)
|
||||
def __init__(self, settings):
|
||||
super(CmdDataSync, self).__init__(settings)
|
||||
|
||||
conn = S3Connection(self.config.aws_access_key_id, self.config.aws_secret_access_key)
|
||||
|
||||
@@ -114,4 +114,4 @@ class CmdDataSync(CmdBase):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(CmdDataSync())
|
||||
Runtime.run(CmdDataSync)
|
||||
|
||||
+4
-5
@@ -5,7 +5,7 @@ from dvc.command.base import CmdBase
|
||||
from dvc.logger import Logger
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import run
|
||||
from dvc.runtime import Runtime
|
||||
|
||||
|
||||
class InitError(DvcException):
|
||||
@@ -44,9 +44,8 @@ KeyName = dvc-key
|
||||
|
||||
SecurityGroup = dvc-group'''
|
||||
|
||||
def __init__(self):
|
||||
CmdBase.__init__(self, parse_config=False)
|
||||
pass
|
||||
def __init__(self, settings):
|
||||
super(CmdInit, self).__init__(settings)
|
||||
|
||||
def define_args(self, parser):
|
||||
self.set_skip_git_actions(parser)
|
||||
@@ -110,4 +109,4 @@ SecurityGroup = dvc-group'''
|
||||
Logger.info('Directory {} was added to .gitignore file'.format(cache_dir_name))
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(CmdInit())
|
||||
Runtime.run(CmdInit, False)
|
||||
|
||||
@@ -6,7 +6,7 @@ from boto.s3.connection import S3Connection
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import run
|
||||
from dvc.runtime import Runtime
|
||||
|
||||
|
||||
class DataRemoveError(DvcException):
|
||||
@@ -15,9 +15,8 @@ class DataRemoveError(DvcException):
|
||||
|
||||
|
||||
class CmdDataRemove(CmdBase):
|
||||
def __init__(self, args=None, parse_config=True, git_obj=None, config_obj=None):
|
||||
super(CmdDataRemove, self).__init__(args, parse_config, git_obj, config_obj)
|
||||
pass
|
||||
def __init__(self, settings):
|
||||
super(CmdDataRemove, self).__init__(settings)
|
||||
|
||||
def define_args(self, parser):
|
||||
self.set_skip_git_actions(parser)
|
||||
@@ -71,7 +70,7 @@ class CmdDataRemove(CmdBase):
|
||||
if not self.parsed_args.recursive:
|
||||
raise DataRemoveError('[Cmd-Remove] Directory "%s" cannot be removed. Use --recurcive flag.' % target)
|
||||
|
||||
data_item = self.path_factory.data_item(target)
|
||||
data_item = self.settings.path_factory.data_item(target)
|
||||
if data_item.data_dvc_short == '':
|
||||
raise DataRemoveError('[Cmd-Remove] Data directory "%s" cannot be removed' % target)
|
||||
|
||||
@@ -89,7 +88,7 @@ class CmdDataRemove(CmdBase):
|
||||
# it raises exception if not a symlink is provided
|
||||
Logger.debug(u'[Cmd-Remove] Remove file {}.'.format(target))
|
||||
|
||||
data_item = self.path_factory.existing_data_item(target)
|
||||
data_item = self.settings.path_factory.existing_data_item(target)
|
||||
|
||||
self._remove_cache_file(data_item)
|
||||
self._remove_state_file(data_item)
|
||||
@@ -154,4 +153,4 @@ class CmdDataRemove(CmdBase):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(CmdDataRemove())
|
||||
Runtime.run(CmdDataRemove)
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
import os
|
||||
import fasteners
|
||||
|
||||
from dvc.command.base import CmdBase
|
||||
from dvc.command.run import CmdRun
|
||||
from dvc.logger import Logger
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path.data_item import NotInDataDirError
|
||||
from dvc.runtime import Runtime
|
||||
from dvc.state_file import StateFile
|
||||
from dvc.utils import run
|
||||
|
||||
|
||||
class ReproError(DvcException):
|
||||
@@ -16,8 +15,8 @@ class ReproError(DvcException):
|
||||
|
||||
|
||||
class CmdRepro(CmdRun):
|
||||
def __init__(self, args=None,parse_config=True, git_obj=None, config_obj=None):
|
||||
CmdBase.__init__(self, args, parse_config, git_obj, config_obj)
|
||||
def __init__(self, settings):
|
||||
super(CmdRun, self).__init__(settings)
|
||||
|
||||
self._code = []
|
||||
pass
|
||||
@@ -56,7 +55,7 @@ class CmdRepro(CmdRun):
|
||||
if not self.skip_git_actions and not self.git.is_ready_to_go():
|
||||
return 1
|
||||
|
||||
data_item_list, external_files_names = self.path_factory.to_data_items(self.parsed_args.target)
|
||||
data_item_list, external_files_names = self.settings.path_factory.to_data_items(self.parsed_args.target)
|
||||
if external_files_names:
|
||||
Logger.error('Files from outside of the data directory "{}" could not be reproduced: {}'.
|
||||
format(self.config.data_dir, ' '.join(external_files_names)))
|
||||
@@ -160,7 +159,7 @@ class ReproChange(object):
|
||||
|
||||
was_source_code_changed = self.git.were_files_changed(self._data_item.data.relative,
|
||||
self.state.code_dependencies + self.state.input_files,
|
||||
self.cmd_obj.path_factory)
|
||||
self.cmd_obj.settings.path_factory)
|
||||
if was_source_code_changed:
|
||||
Logger.debug('Source code was changed for "{}"'.format(self._data_item.data.dvc))
|
||||
|
||||
@@ -176,7 +175,7 @@ class ReproChange(object):
|
||||
dependency_data_items = []
|
||||
for input_file in self.state.input_files:
|
||||
try:
|
||||
data_item = self._cmd_obj.path_factory.data_item(input_file)
|
||||
data_item = self._cmd_obj.settings.path_factory.data_item(input_file)
|
||||
except NotInDataDirError:
|
||||
raise ReproError(u'The dependency file "{}" is not a data item'.format(input_file))
|
||||
except Exception as ex:
|
||||
@@ -189,4 +188,4 @@ class ReproChange(object):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(CmdRepro())
|
||||
Runtime.run(CmdRepro)
|
||||
|
||||
+12
-13
@@ -7,9 +7,9 @@ from dvc.git_wrapper import GitWrapper
|
||||
from dvc.logger import Logger
|
||||
from dvc.path.data_item import NotInDataDirError
|
||||
from dvc.repository_change import RepositoryChange
|
||||
from dvc.runtime import Runtime
|
||||
from dvc.state_file import StateFile
|
||||
from dvc.utils import cached_property
|
||||
from dvc.utils import run
|
||||
|
||||
|
||||
class RunError(DvcException):
|
||||
@@ -18,9 +18,8 @@ class RunError(DvcException):
|
||||
|
||||
|
||||
class CmdRun(CmdBase):
|
||||
def __init__(self, args=None, parse_config=True, git_obj=None, config_obj=None):
|
||||
super(CmdRun, self).__init__(args, parse_config, git_obj, config_obj)
|
||||
pass
|
||||
def __init__(self, settings):
|
||||
super(CmdRun, self).__init__(settings)
|
||||
|
||||
def define_args(self, parser):
|
||||
self.set_skip_git_actions(parser)
|
||||
@@ -76,14 +75,14 @@ class CmdRun(CmdBase):
|
||||
|
||||
return 1
|
||||
|
||||
def run_command(self, argv, data_items_from_args, stdout=None, stderr=None):
|
||||
Logger.debug('Run command: {}. Data items from args: {}. stdout={}, stderr={}'.format(
|
||||
' '.join(argv),
|
||||
def run_command(self, cmd_args, data_items_from_args, stdout=None, stderr=None):
|
||||
Logger.debug('Run command with args: {}. Data items from args: {}. stdout={}, stderr={}'.format(
|
||||
' '.join(cmd_args),
|
||||
', '.join([x.data.dvc for x in data_items_from_args]),
|
||||
stdout,
|
||||
stderr))
|
||||
|
||||
repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config, self.path_factory)
|
||||
repo_change = RepositoryChange(cmd_args, self.settings, stdout, stderr)
|
||||
|
||||
if not self.skip_git_actions and not self._validate_file_states(repo_change):
|
||||
self.remove_new_files(repo_change)
|
||||
@@ -109,7 +108,7 @@ class CmdRun(CmdBase):
|
||||
input_files_dvc,
|
||||
output_files_dvc,
|
||||
code_dependencies_dvc,
|
||||
argv=argv,
|
||||
argv=cmd_args,
|
||||
is_reproducible=self.is_reproducible,
|
||||
stdout=self._stdout_to_dvc(stdout),
|
||||
stderr=self._stdout_to_dvc(stderr))
|
||||
@@ -121,7 +120,7 @@ class CmdRun(CmdBase):
|
||||
def _stdout_to_dvc(self, stdout):
|
||||
if stdout in {None, '-'}:
|
||||
return stdout
|
||||
return self.path_factory.data_item(stdout).data.dvc
|
||||
return self.settings.path_factory.data_item(stdout).data.dvc
|
||||
|
||||
@staticmethod
|
||||
def remove_new_files(repo_change):
|
||||
@@ -149,7 +148,7 @@ class CmdRun(CmdBase):
|
||||
for arg in argv:
|
||||
try:
|
||||
if os.path.isfile(arg):
|
||||
data_item = self.path_factory.data_item(arg)
|
||||
data_item = self.settings.path_factory.data_item(arg)
|
||||
result.append(data_item)
|
||||
except NotInDataDirError:
|
||||
pass
|
||||
@@ -160,7 +159,7 @@ class CmdRun(CmdBase):
|
||||
if not files:
|
||||
return []
|
||||
|
||||
data_items, external = self.path_factory.to_data_items(files)
|
||||
data_items, external = self.settings.path_factory.to_data_items(files)
|
||||
if external:
|
||||
raise RunError('{} should point to data items from data dir: {}'.format(
|
||||
param_text, ', '.join(external))
|
||||
@@ -169,4 +168,4 @@ class CmdRun(CmdBase):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(CmdRun())
|
||||
Runtime.run(CmdRun)
|
||||
|
||||
+4
-21
@@ -10,7 +10,7 @@ class ConfigError(DvcException):
|
||||
|
||||
|
||||
class ConfigI(object):
|
||||
def __init__(self, data_dir, cache_dir, state_dir):
|
||||
def __init__(self, data_dir=None, cache_dir=None, state_dir=None):
|
||||
self._data_dir = data_dir
|
||||
self._cache_dir = cache_dir
|
||||
self._state_dir = state_dir
|
||||
@@ -44,32 +44,15 @@ class Config(ConfigI):
|
||||
raise ConfigError('Config file "{}" does not exist'.format(conf_file))
|
||||
self._config.read(conf_file)
|
||||
|
||||
# self._global = self._config['Global']
|
||||
#
|
||||
# self._data_dir = self._global['DataDir']
|
||||
# self._cache_dir = self._global['CacheDir']
|
||||
# self._state_dir = self._global['StateDir']
|
||||
super(Config, self).__init__(self._config['Global']['DataDir'],
|
||||
self._config['Global']['CacheDir'],
|
||||
self._config['Global']['StateDir'])
|
||||
pass
|
||||
|
||||
# def get_data_file_obj(self):
|
||||
# return DataFileObj(self.config)
|
||||
|
||||
@property
|
||||
def file(self):
|
||||
return self._conf_file
|
||||
|
||||
@property
|
||||
def data_dir(self):
|
||||
return self._config['Global']['DataDir']
|
||||
|
||||
@property
|
||||
def cache_dir(self):
|
||||
return self._config['Global']['CacheDir']
|
||||
|
||||
@property
|
||||
def state_dir(self):
|
||||
return self._config['Global']['StateDir']
|
||||
|
||||
@property
|
||||
def aws_access_key_id(self):
|
||||
return self._config['AWS']['AccessKeyId']
|
||||
|
||||
@@ -17,25 +17,19 @@ class RepositoryChangeError(DvcException):
|
||||
class RepositoryChange(object):
|
||||
"""Pre-condition: git repository has no changes"""
|
||||
|
||||
def __init__(self, args, stdout, stderr, git, config, path_factory):
|
||||
self.git = git
|
||||
self.config = config
|
||||
self.path_factory = path_factory
|
||||
def __init__(self, cmd_args, settings, stdout, stderr):
|
||||
self._settings = settings
|
||||
|
||||
Logger.debug(u'[Repository change] Exec command: {}. stdout={}, stderr={}'.format(
|
||||
u' '.join(args),
|
||||
u' '.join(cmd_args),
|
||||
stdout,
|
||||
stderr))
|
||||
Executor.exec_cmd_only_success(args, stdout, stderr)
|
||||
Executor.exec_cmd_only_success(cmd_args, stdout, stderr)
|
||||
|
||||
self._stated_data_items = []
|
||||
self._externally_created_files = []
|
||||
self._init_file_states()
|
||||
|
||||
@staticmethod
|
||||
def exec_cmd(args, stdout, stderr, git, path_factory):
|
||||
return RepositoryChange(args, stdout, stderr, git)
|
||||
|
||||
@cached_property
|
||||
def removed_data_items(self):
|
||||
return [x for x in self._stated_data_items if x.is_removed]
|
||||
@@ -58,7 +52,7 @@ class RepositoryChange(object):
|
||||
|
||||
def _add_stated_data_item(self, state, file):
|
||||
try:
|
||||
item = self.path_factory.stated_data_item(state, file)
|
||||
item = self._settings.path_factory.stated_data_item(state, file)
|
||||
self._stated_data_items.append(item)
|
||||
Logger.debug('[Repository change] Add status: {} {}'.format(
|
||||
item.status,
|
||||
@@ -71,7 +65,7 @@ class RepositoryChange(object):
|
||||
statuses = GitWrapper.git_file_statuses()
|
||||
|
||||
for status, file in statuses:
|
||||
file_path = os.path.join(self.git.git_dir_abs, file)
|
||||
file_path = os.path.join(self._settings.git.git_dir_abs, file)
|
||||
|
||||
if not os.path.isdir(file_path):
|
||||
self._add_stated_data_item(status, file_path)
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
from dvc.config import Config, ConfigI
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.git_wrapper import GitWrapper
|
||||
from dvc.logger import Logger
|
||||
from dvc.settings import Settings
|
||||
|
||||
|
||||
class Runtime(object):
|
||||
CONFIG = 'dvc.conf'
|
||||
|
||||
@staticmethod
|
||||
def conf_file_path(git_dir):
|
||||
return os.path.realpath(os.path.join(git_dir, Runtime.CONFIG))
|
||||
|
||||
@staticmethod
|
||||
def run(cmd_class, parse_config=True):
|
||||
try:
|
||||
runtime_git = GitWrapper()
|
||||
|
||||
if parse_config:
|
||||
runtime_config = Config(Runtime.conf_file_path(runtime_git.git_dir))
|
||||
else:
|
||||
runtime_config = ConfigI()
|
||||
|
||||
args = sys.argv[1:]
|
||||
|
||||
instance = cmd_class(Settings(args, runtime_git, runtime_config))
|
||||
sys.exit(instance.run())
|
||||
except DvcException as e:
|
||||
Logger.error(e)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,44 @@
|
||||
import os
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path.factory import PathFactory
|
||||
|
||||
|
||||
class SettingsError(DvcException):
|
||||
def __init__(self, msg):
|
||||
DvcException.__init__(self, msg)
|
||||
|
||||
|
||||
class Settings(object):
|
||||
def __init__(self, args, git, config):
|
||||
self._args = args
|
||||
self._git = git
|
||||
self._config = config
|
||||
self._path_factory = PathFactory(git, config)
|
||||
|
||||
self._dvc_home = os.environ.get('DVC_HOME')
|
||||
if not self._dvc_home:
|
||||
raise SettingsError('DVC_HOME environment variable is not defined')
|
||||
if not os.path.exists(self._dvc_home):
|
||||
raise SettingsError("DVC_HOME directory doesn't exists")
|
||||
pass
|
||||
|
||||
@property
|
||||
def args(self):
|
||||
return self._args
|
||||
|
||||
@property
|
||||
def git(self):
|
||||
return self._git
|
||||
|
||||
@property
|
||||
def config(self):
|
||||
return self._config
|
||||
|
||||
@property
|
||||
def dvc_home(self):
|
||||
return self._dvc_home
|
||||
|
||||
@property
|
||||
def path_factory(self):
|
||||
return self._path_factory
|
||||
@@ -1,6 +1,3 @@
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
def cached_property(f):
|
||||
def get(self):
|
||||
@@ -15,12 +12,3 @@ def cached_property(f):
|
||||
return x
|
||||
|
||||
return property(get)
|
||||
|
||||
|
||||
def run(cmd):
|
||||
import sys
|
||||
try:
|
||||
sys.exit(cmd.run())
|
||||
except DvcException as e:
|
||||
Logger.error(e)
|
||||
sys.exit(1)
|
||||
+3
-5
@@ -3,9 +3,10 @@ import shutil
|
||||
from unittest import TestCase
|
||||
|
||||
from dvc.config import ConfigI
|
||||
from dvc.git_wrapper import GitWrapperI, GitWrapper
|
||||
from dvc.git_wrapper import GitWrapperI
|
||||
from dvc.path.data_item import DataItem
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.settings import Settings
|
||||
|
||||
|
||||
class BasicEnvironment(TestCase):
|
||||
@@ -33,8 +34,6 @@ class BasicEnvironment(TestCase):
|
||||
os.makedirs(data_dir)
|
||||
|
||||
os.chdir(self._curr_dir)
|
||||
# GitWrapper.exec_cmd(['git', 'init'])
|
||||
# self._commit = GitWrapper().curr_commit
|
||||
|
||||
def tearDown(self):
|
||||
if self._old_curr_dir_abs:
|
||||
@@ -69,6 +68,7 @@ class DirHierarchyEnvironment(BasicEnvironment):
|
||||
self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit)
|
||||
self._config = ConfigI('data', 'cache', 'state')
|
||||
self.path_factory = PathFactory(self._git, self._config)
|
||||
self.settings = Settings([], self._git, self._config)
|
||||
|
||||
self.dir1 = os.path.join('dir1')
|
||||
self.dir11 = os.path.join('dir1/dir11')
|
||||
@@ -91,7 +91,6 @@ class DirHierarchyEnvironment(BasicEnvironment):
|
||||
|
||||
@staticmethod
|
||||
def create_content_file(file, content='some test'):
|
||||
# print(file)
|
||||
fd = open(file, 'w+')
|
||||
fd.write(content)
|
||||
fd.close()
|
||||
@@ -108,7 +107,6 @@ class DirHierarchyEnvironment(BasicEnvironment):
|
||||
|
||||
relevant_dir = self.relevant_dir(data_file)
|
||||
os.symlink(os.path.join(relevant_dir, cache_result), file_result)
|
||||
# print(file_result)
|
||||
else:
|
||||
cache_result = None
|
||||
self.create_content_file(file_result, content)
|
||||
|
||||
@@ -10,7 +10,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test_file_by_file_removal(self):
|
||||
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
|
||||
cmd = CmdDataRemove(self.settings)
|
||||
cmd.parsed_args.keep_in_cloud = True
|
||||
|
||||
dir1_dvc_name = os.path.join('data', self.dir1)
|
||||
@@ -20,7 +20,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment):
|
||||
self.assertFalse(os.path.exists(dir1_dvc_name))
|
||||
|
||||
def test_remove_deep_dir(self):
|
||||
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
|
||||
cmd = CmdDataRemove(self.settings)
|
||||
cmd.parsed_args.keep_in_cloud = True
|
||||
cmd.parsed_args.recursive = True
|
||||
|
||||
@@ -31,7 +31,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment):
|
||||
self.assertFalse(os.path.exists(dir1_dvc_name))
|
||||
|
||||
def test_not_recursive_removal(self):
|
||||
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
|
||||
cmd = CmdDataRemove(self.settings)
|
||||
cmd.parsed_args.keep_in_cloud = True
|
||||
cmd.parsed_args.recursive = False
|
||||
|
||||
@@ -43,7 +43,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment):
|
||||
pass
|
||||
|
||||
def test_data_dir_removal(self):
|
||||
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
|
||||
cmd = CmdDataRemove(self.settings)
|
||||
cmd.parsed_args.keep_in_cloud = True
|
||||
cmd.parsed_args.recursive = True
|
||||
|
||||
@@ -60,7 +60,7 @@ class TestRemoveDataItem(DirHierarchyEnvironment):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test_remove_data_instance(self):
|
||||
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
|
||||
cmd = CmdDataRemove(self.settings)
|
||||
cmd.parsed_args.keep_in_cloud = True
|
||||
|
||||
self.assertTrue(os.path.isfile(self.file1))
|
||||
@@ -73,7 +73,7 @@ class TestRemoveDataItem(DirHierarchyEnvironment):
|
||||
self.assertFalse(os.path.exists(self.state1))
|
||||
|
||||
def test_keep_cache(self):
|
||||
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
|
||||
cmd = CmdDataRemove(self.settings)
|
||||
cmd.parsed_args.keep_in_cloud = True
|
||||
cmd.parsed_args.keep_in_cache = True
|
||||
|
||||
@@ -87,7 +87,7 @@ class TestRemoveDataItem(DirHierarchyEnvironment):
|
||||
self.assertFalse(os.path.exists(self.state1))
|
||||
|
||||
def test_remove_data_instance_without_cache(self):
|
||||
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
|
||||
cmd = CmdDataRemove(self.settings)
|
||||
cmd.parsed_args.keep_in_cloud = True
|
||||
|
||||
self.assertTrue(os.path.isfile(self.file6))
|
||||
@@ -103,7 +103,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test_end_to_end_with_an_error(self):
|
||||
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
|
||||
cmd = CmdDataRemove(self.settings)
|
||||
cmd.parsed_args.keep_in_cloud = True
|
||||
|
||||
dir11_full = os.path.join('data', self.dir11)
|
||||
@@ -130,7 +130,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment):
|
||||
self.assertTrue(os.path.exists(self.file6))
|
||||
|
||||
def test_end_to_end_with_no_error(self):
|
||||
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
|
||||
cmd = CmdDataRemove(self.settings)
|
||||
cmd.parsed_args.keep_in_cloud = True
|
||||
|
||||
dir11_full = os.path.join('data', self.dir11)
|
||||
@@ -154,7 +154,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment):
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
|
||||
def test_run(self):
|
||||
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
|
||||
cmd = CmdDataRemove(self.settings)
|
||||
cmd.parsed_args.keep_in_cloud = True
|
||||
|
||||
dir11_full = os.path.join('data', self.dir11)
|
||||
|
||||
@@ -9,6 +9,7 @@ from dvc.executor import Executor
|
||||
from dvc.path.data_item import DataItem
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.git_wrapper import GitWrapper
|
||||
from dvc.settings import Settings
|
||||
|
||||
|
||||
class RunBasicTest(TestCase):
|
||||
@@ -28,6 +29,8 @@ class RunBasicTest(TestCase):
|
||||
|
||||
self.config = ConfigI('data', 'cache', 'state')
|
||||
self.path_factory = PathFactory(self.git, self.config)
|
||||
|
||||
self.settings = Settings([], self.git, self.config)
|
||||
pass
|
||||
|
||||
def init_git_repo(self):
|
||||
@@ -49,7 +52,8 @@ class RunBasicTest(TestCase):
|
||||
|
||||
class TestRunOutsideData(RunBasicTest):
|
||||
def test(self):
|
||||
cmd_run = CmdRun(config_obj=self.config)
|
||||
self.settings._args = []
|
||||
cmd_run = CmdRun(self.settings)
|
||||
with self.assertRaises(RunError):
|
||||
cmd_run.run_command(['touch', 'file1', 'file2'], [])
|
||||
pass
|
||||
@@ -59,7 +63,8 @@ class RunTwoFilesBase(RunBasicTest):
|
||||
def setUp(self):
|
||||
super(RunTwoFilesBase, self).setUp()
|
||||
|
||||
cmd_run = CmdRun(config_obj=self.config, git_obj=self.git)
|
||||
self.settings._args = []
|
||||
cmd_run = CmdRun(self.settings)
|
||||
self.input_param_file = 'data/extra_input'
|
||||
cmd_run.parsed_args.input = [self.input_param_file]
|
||||
|
||||
|
||||
@@ -152,8 +152,6 @@ class TestDataFileObjLongPath(BasicEnvironment):
|
||||
self._config = ConfigI('data', 'cache', 'state')
|
||||
self.path_factory = PathFactory(self._git, self._config)
|
||||
|
||||
# self._dobj = DataFileObj(os.path.join('..', '..', 'data', 'file1.txt'),
|
||||
# self._git, self._config)
|
||||
self.data_path = self.path_factory.data_item(os.path.join('..', '..', 'data', 'file1.txt'))
|
||||
pass
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from dvc.config import ConfigI
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.git_wrapper import GitWrapperI
|
||||
from dvc.repository_change import RepositoryChange
|
||||
from dvc.settings import Settings
|
||||
|
||||
|
||||
class BasicTestRepositoryChange(TestCase):
|
||||
@@ -26,6 +27,7 @@ class BasicTestRepositoryChange(TestCase):
|
||||
self.git = GitWrapperI(self.test_dir)
|
||||
self.config = ConfigI('data', 'cache', 'state')
|
||||
self.path_factory = PathFactory(self.git, self.config)
|
||||
self.settings = Settings([], self.git, self.config)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir, ignore_errors=True)
|
||||
@@ -56,11 +58,9 @@ class TestRepositoryChange(BasicTestRepositoryChange):
|
||||
os.remove(file_to_remove)
|
||||
|
||||
change = RepositoryChange(['ls', '-la'],
|
||||
self.settings,
|
||||
file_created_by_run,
|
||||
file_to_modify,
|
||||
self.git,
|
||||
self.config,
|
||||
self.path_factory)
|
||||
file_to_modify)
|
||||
|
||||
expected_new = [file_created_by_run, file_created_before_run]
|
||||
new_file_abs = [x.data.dvc for x in change.new_data_items]
|
||||
@@ -86,11 +86,9 @@ class TestRepositoryChangeDeepInDirectory(BasicTestRepositoryChange):
|
||||
self.create_file(deep_file)
|
||||
|
||||
change = RepositoryChange(['ls', '-la'],
|
||||
self.settings,
|
||||
deep_file,
|
||||
None,
|
||||
self.git,
|
||||
self.config,
|
||||
self.path_factory)
|
||||
None)
|
||||
|
||||
self.assertEqual([x.data.dvc for x in change.new_data_items], [deep_file])
|
||||
|
||||
@@ -101,11 +99,9 @@ class TestRepositoryChangeExternallyCreated(BasicTestRepositoryChange):
|
||||
self.create_file(not_in_data_dir_file)
|
||||
|
||||
change = RepositoryChange(['ls', '-la'],
|
||||
self.settings,
|
||||
not_in_data_dir_file,
|
||||
None,
|
||||
self.git,
|
||||
self.config,
|
||||
self.path_factory)
|
||||
None)
|
||||
|
||||
self.assertEqual([os.path.realpath(not_in_data_dir_file)],
|
||||
change.externally_created_files)
|
||||
|
||||
+22
-54
@@ -2,6 +2,7 @@ from dvc.command.remove import CmdDataRemove
|
||||
from dvc.command.repro import ReproChange, CmdRepro
|
||||
from dvc.command.run import CmdRun, RunError
|
||||
from dvc.executor import Executor
|
||||
from dvc.settings import Settings
|
||||
from tests.test_cmd_run import RunBasicTest
|
||||
|
||||
|
||||
@@ -12,50 +13,26 @@ class ReproBasicEnv(RunBasicTest):
|
||||
self.file_name1 = 'data/file1'
|
||||
self.file1_code_file = 'file1.py'
|
||||
self.create_file_and_commit(self.file1_code_file, 'An awesome code...')
|
||||
cmd_file1 = CmdRun(args=[
|
||||
'printf',
|
||||
'Hello\nMary',
|
||||
'--not-repro',
|
||||
'--stdout',
|
||||
self.file_name1,
|
||||
'--code',
|
||||
self.file1_code_file
|
||||
],
|
||||
parse_config=False,
|
||||
config_obj=self.config,
|
||||
git_obj=self.git)
|
||||
self.settings._args = ['printf', 'Hello\nMary', '--not-repro',
|
||||
'--stdout', self.file_name1, '--code', self.file1_code_file]
|
||||
cmd_file1 = CmdRun(self.settings)
|
||||
self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file])
|
||||
cmd_file1.run()
|
||||
|
||||
self.file_name11 = 'data/file11'
|
||||
CmdRun(args=['head', '-n', '1', self.file_name1, '--stdout', self.file_name11],
|
||||
parse_config=False,
|
||||
config_obj=self.config,
|
||||
git_obj=self.git
|
||||
).run()
|
||||
self.settings._args = ['head', '-n', '1', self.file_name1, '--stdout', self.file_name11]
|
||||
CmdRun(self.settings).run()
|
||||
|
||||
self.file_name2 = 'data/file2'
|
||||
CmdRun(args=['printf', 'Bobby', '--not-repro', '--stdout', self.file_name2],
|
||||
parse_config=False,
|
||||
config_obj=self.config,
|
||||
git_obj=self.git
|
||||
).run()
|
||||
self.settings._args = ['printf', 'Bobby', '--not-repro', '--stdout', self.file_name2]
|
||||
CmdRun(self.settings).run()
|
||||
|
||||
self.file_res_code_file = 'code_res.py'
|
||||
self.create_file_and_commit(self.file_res_code_file, 'Another piece of code')
|
||||
self.file_name_res = 'data/file_res'
|
||||
cmd_res = CmdRun(args=[
|
||||
'cat',
|
||||
self.file_name11,
|
||||
self.file_name2,
|
||||
'--stdout',
|
||||
self.file_name_res,
|
||||
'--code',
|
||||
self.file_res_code_file
|
||||
],
|
||||
parse_config=False,
|
||||
config_obj=self.config,
|
||||
git_obj=self.git)
|
||||
self.settings._args = ['cat', self.file_name11, self.file_name2, '--stdout', self.file_name_res,
|
||||
'--code', self.file_res_code_file]
|
||||
cmd_res = CmdRun(self.settings)
|
||||
self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file])
|
||||
cmd_res.run()
|
||||
|
||||
@@ -81,10 +58,8 @@ class ReproCodeDependencyTest(ReproBasicEnv):
|
||||
def test(self):
|
||||
self.modify_file_and_commit(self.file_res_code_file)
|
||||
|
||||
CmdRepro(args=[self.file_name_res],
|
||||
config_obj=self.config,
|
||||
git_obj=self.git
|
||||
).run()
|
||||
self.settings._args = [self.file_name_res]
|
||||
CmdRepro(self.settings).run()
|
||||
|
||||
self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby')
|
||||
|
||||
@@ -93,31 +68,24 @@ class ReproChangedDependency(ReproBasicEnv):
|
||||
def test(self):
|
||||
self.recreate_file1()
|
||||
|
||||
CmdRepro(args=[self.file_name11],
|
||||
config_obj=self.config,
|
||||
git_obj=self.git
|
||||
).run()
|
||||
self.settings._args = [self.file_name11]
|
||||
CmdRepro(self.settings).run()
|
||||
|
||||
self.assertEqual(open(self.file_name11).read(), 'Goodbye\n')
|
||||
|
||||
def recreate_file1(self):
|
||||
CmdDataRemove(args=[self.file_name1, '--keep-in-cloud'], # ???
|
||||
config_obj=self.config,
|
||||
git_obj=self.git
|
||||
).run()
|
||||
CmdRun(args=['printf', 'Goodbye\nJack', '--stdout', self.file_name1],
|
||||
config_obj=self.config,
|
||||
git_obj=self.git
|
||||
).run()
|
||||
self.settings._args = [self.file_name1, '--keep-in-cloud']
|
||||
CmdDataRemove(self.settings).run()
|
||||
|
||||
self.settings._args = ['printf', 'Goodbye\nJack', '--stdout', self.file_name1]
|
||||
CmdRun(self.settings).run()
|
||||
|
||||
|
||||
class ReproChangedDeepDependency(ReproChangedDependency):
|
||||
def test(self):
|
||||
self.recreate_file1()
|
||||
|
||||
CmdRepro(args=[self.file_name_res],
|
||||
config_obj=self.config,
|
||||
git_obj=self.git
|
||||
).run()
|
||||
self.settings._args = [self.file_name_res]
|
||||
CmdRepro(self.settings).run()
|
||||
|
||||
self.assertEqual(open(self.file_name_res).read(), 'Goodbye\nBobby')
|
||||
|
||||
Reference in New Issue
Block a user