mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-04 07:18:27 +01:00
dvc: initial unittests
Initial unittests for new design. Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
+1
-2
@@ -44,8 +44,7 @@ install:
|
||||
- printf "[default]\naws_access_key_id = $AWS_ACCESS_KEY_ID\naws_secret_access_key = $AWS_SECRET_ACCESS_KEY\n" > ~/.aws/credentials
|
||||
- printf "[default]\n" > ~/.aws/config
|
||||
script:
|
||||
# Temporarily disabled untill dvc is stable
|
||||
# - ./unittests.sh
|
||||
- ./unittests.sh
|
||||
- ./functests.sh
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./build_macos.sh; else ./build_linux.sh; fi
|
||||
- ./build_package.sh
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ class Cache(object):
|
||||
CACHE_DIR = 'cache'
|
||||
|
||||
def __init__(self, dvc_dir):
|
||||
self.cache_dir = os.path.join(dvc_dir, self.CACHE_DIR)
|
||||
self.cache_dir = os.path.abspath(os.path.realpath(os.path.join(dvc_dir, self.CACHE_DIR)))
|
||||
|
||||
@staticmethod
|
||||
def init(dvc_dir):
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
import os
|
||||
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.logger import Logger
|
||||
from dvc.path.data_item import DataItemError
|
||||
|
||||
|
||||
class TraverseError(DvcException):
|
||||
def __init__(self, msg):
|
||||
DvcException.__init__(self, 'Traverse file tree error: {}'.format(msg))
|
||||
|
||||
|
||||
class Traverse(CmdBase):
|
||||
def __init__(self, settings, cmd_name, do_not_start_from_root=True):
|
||||
super(Traverse, self).__init__(settings)
|
||||
self._cmd_name = cmd_name
|
||||
self._do_not_start_from_root = do_not_start_from_root
|
||||
|
||||
def run(self):
|
||||
if not self._traverse_all():
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
def _traverse_all(self):
|
||||
if not self.no_git_actions and not self.git.is_ready_to_go():
|
||||
return False
|
||||
|
||||
error = False
|
||||
for target in self.parsed_args.target:
|
||||
if not self._traverse(target):
|
||||
error = True
|
||||
|
||||
message = 'DVC {}: {}'.format(self._cmd_name, ' '.join(self.parsed_args.target))
|
||||
self.commit_if_needed(message, error)
|
||||
|
||||
return error == 0
|
||||
|
||||
def _traverse(self, target):
|
||||
try:
|
||||
if os.path.isdir(target):
|
||||
if not self.is_recursive:
|
||||
msg = '[TraverseFileTree] Directory "%s" cannot be traversed. Use --recurcive flag.'
|
||||
raise TraverseError(msg % target)
|
||||
|
||||
self._traverse_dir(target)
|
||||
else:
|
||||
self.process_file(target)
|
||||
return True
|
||||
except DvcException as ex:
|
||||
Logger.error('[TraverseFileTree] Unable to {} data item "{}": {}'.format(self._cmd_name, target, ex))
|
||||
return False
|
||||
|
||||
def _traverse_dir(self, target):
|
||||
if target == self.settings.config.CONFIG_DIR:
|
||||
return
|
||||
|
||||
for f in os.listdir(target):
|
||||
file = os.path.join(target, f)
|
||||
if os.path.isdir(file):
|
||||
self._traverse_dir(file)
|
||||
else:
|
||||
self.process_file(file)
|
||||
|
||||
self.traverse_dir_finalize(target)
|
||||
pass
|
||||
|
||||
# Common helpers
|
||||
def _get_data_item(self, target):
|
||||
try:
|
||||
data_item = self.settings.path_factory.existing_data_item(target)
|
||||
except DataItemError:
|
||||
Logger.warn(u'[TraverseFileTree] Data file {} does not have existing state file'.format(target))
|
||||
data_item = self.settings.path_factory.data_item(target)
|
||||
|
||||
return data_item
|
||||
|
||||
def _remove_cloud_cache(self, data_item):
|
||||
if not self.parsed_args.keep_in_cloud:
|
||||
self.settings.cloud.remove(data_item)
|
||||
|
||||
## API:
|
||||
def is_recursive(self):
|
||||
return True
|
||||
|
||||
def process_file(self, target):
|
||||
pass
|
||||
|
||||
def traverse_dir_finalize(self, target):
|
||||
pass
|
||||
+1
-1
@@ -57,7 +57,7 @@ ProjectName =
|
||||
'''
|
||||
|
||||
def __init__(self, dvc_dir):
|
||||
self.dvc_dir = dvc_dir
|
||||
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
|
||||
self.config_file = os.path.join(dvc_dir, self.CONFIG)
|
||||
|
||||
self._config = configparser.SafeConfigParser()
|
||||
|
||||
+43
-6
@@ -60,7 +60,7 @@ class Project(object):
|
||||
DVC_DIR = '.dvc'
|
||||
|
||||
def __init__(self, root_dir):
|
||||
self.root_dir = os.path.abspath(root_dir)
|
||||
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
|
||||
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
|
||||
|
||||
self.scm = SCM(self.root_dir)
|
||||
@@ -73,6 +73,18 @@ class Project(object):
|
||||
|
||||
@staticmethod
|
||||
def init(root_dir):
|
||||
"""
|
||||
Initiate dvc project in directory.
|
||||
|
||||
Args:
|
||||
root_dir: Path to project's root directory.
|
||||
|
||||
Returns:
|
||||
Project instance.
|
||||
|
||||
Raises:
|
||||
KeyError: Raises an exception.
|
||||
"""
|
||||
root_dir = os.path.abspath(root_dir)
|
||||
dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
|
||||
os.mkdir(dvc_dir)
|
||||
@@ -106,12 +118,23 @@ class Project(object):
|
||||
locked=True)
|
||||
stage.save()
|
||||
stage.dump()
|
||||
return stage
|
||||
|
||||
def remove(self, fname):
|
||||
path = os.path.abspath(fname)
|
||||
stages = []
|
||||
output = Output.loads(self, fname)
|
||||
for out in self.outs():
|
||||
if out.path == path:
|
||||
out.stage().remove()
|
||||
if out.path == output.path:
|
||||
stage = out.stage()
|
||||
stages.append(stage)
|
||||
|
||||
if len(stages) == 0:
|
||||
raise StageNotFoundError(fname)
|
||||
|
||||
for stage in stages:
|
||||
stage.remove()
|
||||
|
||||
return stages
|
||||
|
||||
def _add_orphans(self, deps):
|
||||
outs = [out.path for out in self.outs()]
|
||||
@@ -120,7 +143,15 @@ class Project(object):
|
||||
continue
|
||||
self.add(dep.path)
|
||||
|
||||
def run(self, cmd, deps, deps_no_cache, outs, outs_no_cache, locked, fname, cwd):
|
||||
def run(self,
|
||||
cmd=None,
|
||||
deps=[],
|
||||
deps_no_cache=[],
|
||||
outs=[],
|
||||
outs_no_cache=[],
|
||||
locked=False,
|
||||
fname=Stage.STAGE_FILE,
|
||||
cwd=os.curdir):
|
||||
cwd = os.path.abspath(cwd)
|
||||
path = os.path.join(cwd, fname)
|
||||
outputs = Output.loads_from(self, outs, use_cache=True, cwd=cwd)
|
||||
@@ -140,8 +171,10 @@ class Project(object):
|
||||
locked=locked)
|
||||
stage.run()
|
||||
stage.dump()
|
||||
return stage
|
||||
|
||||
def reproduce(self, targets, recursive=False, force=False):
|
||||
def reproduce(self, targets, recursive=True, force=False):
|
||||
reproduced = []
|
||||
stages = nx.get_node_attributes(self.graph(), 'stage')
|
||||
for target in targets:
|
||||
node = os.path.relpath(os.path.abspath(target), self.root_dir)
|
||||
@@ -152,9 +185,13 @@ class Project(object):
|
||||
for n in nx.dfs_postorder_nodes(self.graph(), node):
|
||||
stages[n].reproduce(force=force)
|
||||
stages[n].dump()
|
||||
reproduced.append(stages[n])
|
||||
|
||||
stages[node].reproduce(force=force)
|
||||
stages[node].dump()
|
||||
reproduced.append(stages[node])
|
||||
|
||||
return reproduced
|
||||
|
||||
def checkout(self):
|
||||
for stage in self.stages():
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
import os
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.git_wrapper import GitWrapper
|
||||
from dvc.executor import Executor
|
||||
from dvc.logger import Logger
|
||||
from dvc.path.data_item import DataDirError, DataItemInConfigDirError
|
||||
from dvc.path.stated_data_item import StatedDataItem
|
||||
from dvc.utils import cached_property
|
||||
from dvc.config import ConfigI
|
||||
|
||||
class RepositoryChangeError(DvcException):
|
||||
def __init__(self, msg):
|
||||
DvcException.__init__(self, 'Repository change error: {}'.format(msg))
|
||||
|
||||
|
||||
class RepositoryChange(object):
|
||||
"""Pre-condition: git repository has no changes"""
|
||||
|
||||
def __init__(self, command, settings, stdout, stderr, shell=False):
|
||||
self._settings = settings
|
||||
|
||||
stemps_before = self.data_file_timesteps()
|
||||
|
||||
Logger.debug(u'[Repository change] Exec command: {}. stdout={}, stderr={}, shell={}'.format(
|
||||
command,
|
||||
stdout,
|
||||
stderr,
|
||||
shell))
|
||||
Executor.exec_cmd_only_success(command.split(' '), stdout, stderr, shell=shell)
|
||||
|
||||
stemps_after = self.data_file_timesteps()
|
||||
|
||||
sym_diff = stemps_after ^ stemps_before
|
||||
self._modified_content_filenames = set([filename for filename, timestemp in sym_diff])
|
||||
|
||||
Logger.debug(u'[Repository change] Identified modifications: {}'.format(
|
||||
u', '.join(self._modified_content_filenames)))
|
||||
|
||||
self._stated_data_items = []
|
||||
self._externally_created_files = []
|
||||
self._created_status_files = []
|
||||
self._init_file_states()
|
||||
|
||||
@property
|
||||
def modified_content_data_items(self):
|
||||
return [self._settings.path_factory.data_item(file) for file in self._modified_content_filenames]
|
||||
|
||||
@cached_property
|
||||
def removed_data_items(self):
|
||||
return [x for x in self._stated_data_items if x.is_removed]
|
||||
|
||||
@cached_property
|
||||
def modified_data_items(self):
|
||||
return [x for x in self._stated_data_items if x.is_modified]
|
||||
|
||||
@cached_property
|
||||
def new_data_items(self):
|
||||
return [x for x in self._stated_data_items if x.is_new]
|
||||
|
||||
@cached_property
|
||||
def unusual_data_items(self):
|
||||
return [x for x in self._stated_data_items if x.is_unusual]
|
||||
|
||||
@property
|
||||
def changed_data_items(self):
|
||||
res = set(self.new_data_items + self.modified_data_items + self.modified_content_data_items)
|
||||
return list(res)
|
||||
|
||||
def _add_stated_data_item(self, state, file):
|
||||
try:
|
||||
item = self._settings.path_factory.stated_data_item(state, file)
|
||||
self._stated_data_items.append(item)
|
||||
Logger.debug('[Repository change] Add status: {} {}'.format(
|
||||
item.status,
|
||||
item.data.dvc))
|
||||
except DataItemInConfigDirError:
|
||||
self._created_status_files.append(file)
|
||||
except DataDirError:
|
||||
self._externally_created_files.append(file)
|
||||
pass
|
||||
|
||||
def _init_file_states(self):
|
||||
statuses = GitWrapper.git_file_statuses()
|
||||
|
||||
for status, file in statuses:
|
||||
file_path = os.path.join(self._settings.git.git_dir_abs, file)
|
||||
|
||||
if not os.path.isdir(file_path):
|
||||
self._add_stated_data_item(status, file_path)
|
||||
else:
|
||||
files = []
|
||||
self.get_all_files_from_dir(file_path, files)
|
||||
state = StatedDataItem.STATUS_UNTRACKED + StatedDataItem.STATUS_UNTRACKED
|
||||
for f in files:
|
||||
self._add_stated_data_item(state, f)
|
||||
pass
|
||||
|
||||
def get_all_files_from_dir(self, dir, result):
|
||||
files = os.listdir(dir)
|
||||
for f in files:
|
||||
path = os.path.join(dir, f)
|
||||
if os.path.isfile(path):
|
||||
result.append(path)
|
||||
else:
|
||||
self.get_all_files_from_dir(path, result)
|
||||
pass
|
||||
|
||||
@property
|
||||
def externally_created_files(self):
|
||||
return self._externally_created_files
|
||||
|
||||
@property
|
||||
def created_status_files(self):
|
||||
return self._created_status_files
|
||||
|
||||
def data_file_timesteps(self):
|
||||
res = set()
|
||||
for root, dirs, files in os.walk(self._settings.git.git_dir_abs):
|
||||
if root.startswith(os.path.join(self._settings.git.git_dir_abs, ConfigI.CONFIG_DIR)):
|
||||
continue
|
||||
|
||||
for file in files:
|
||||
filename = os.path.join(root, file)
|
||||
if os.path.exists(filename):
|
||||
timestemp = os.path.getmtime(filename)
|
||||
res.add((filename, timestemp))
|
||||
|
||||
return res
|
||||
+21
-8
@@ -5,11 +5,18 @@ from dvc.exceptions import DvcException
|
||||
from dvc.command.common.branch_changer import BranchChanger
|
||||
|
||||
|
||||
class SCMError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class FileNotInRepoError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class Base(object):
|
||||
def __init__(self, root_dir=os.curdir):
|
||||
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
|
||||
|
||||
@staticmethod
|
||||
def is_repo(root_dir):
|
||||
return True
|
||||
@@ -23,6 +30,9 @@ class Base(object):
|
||||
def ignore_list(self, p_list):
|
||||
return [self.ignore(path) for path in p_list]
|
||||
|
||||
def ignore_file(self):
|
||||
return self.GITIGNORE
|
||||
|
||||
def add(self, paths):
|
||||
pass
|
||||
|
||||
@@ -46,9 +56,13 @@ class Git(Base):
|
||||
GITIGNORE = '.gitignore'
|
||||
GIT_DIR = '.git'
|
||||
|
||||
def __init__(self, root_dir):
|
||||
self.root_dir = root_dir
|
||||
self.repo = git.Repo(root_dir)
|
||||
def __init__(self, root_dir=os.curdir):
|
||||
super(Git, self).__init__(root_dir)
|
||||
|
||||
try:
|
||||
self.repo = git.Repo(root_dir)
|
||||
except InvalidGitRepositoryError:
|
||||
raise SCMError('{} is not a git repository'.format(root_dir))
|
||||
|
||||
@staticmethod
|
||||
def is_repo(root_dir):
|
||||
@@ -57,7 +71,9 @@ class Git(Base):
|
||||
|
||||
def ignore(self, path):
|
||||
entry = os.path.basename(path)
|
||||
gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE)
|
||||
gitignore = os.path.join(os.path.abspath(os.path.dirname(path)),
|
||||
self.GITIGNORE)
|
||||
gitignore = os.path.abspath(os.path.realpath(gitignore))
|
||||
|
||||
if not gitignore.startswith(self.root_dir):
|
||||
raise FileNotInRepoError()
|
||||
@@ -74,9 +90,6 @@ class Git(Base):
|
||||
|
||||
open(gitignore, 'a').write(content)
|
||||
|
||||
def ignore_file(self):
|
||||
return self.GITIGNORE
|
||||
|
||||
def add(self, paths):
|
||||
self.repo.index.add(paths)
|
||||
|
||||
@@ -96,7 +109,7 @@ class Git(Base):
|
||||
return self.repo.untracked_files
|
||||
|
||||
|
||||
def SCM(root_dir):
|
||||
def SCM(root_dir=os.curdir):
|
||||
if Git.is_repo(root_dir):
|
||||
return Git(root_dir)
|
||||
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
import os
|
||||
|
||||
from dvc.git_wrapper import GitWrapperI, GitWrapper
|
||||
from dvc.config import ConfigI, Config
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.cli import parse_args
|
||||
from dvc.data_cloud import DataCloud
|
||||
|
||||
class SettingsError(DvcException):
|
||||
def __init__(self, msg):
|
||||
DvcException.__init__(self, msg)
|
||||
|
||||
|
||||
class Settings(object):
|
||||
def __init__(self, argv=None, git=None, config=None):
|
||||
self._args = None
|
||||
args = None
|
||||
|
||||
if argv is not None and len(argv) != 0:
|
||||
args = parse_args(argv)
|
||||
self._args = argv[2:]
|
||||
|
||||
if git is None:
|
||||
git = GitWrapper()
|
||||
|
||||
if config is None:
|
||||
if args is not None and args.cmd != 'init':
|
||||
config = Config()
|
||||
else:
|
||||
config = ConfigI()
|
||||
|
||||
self._git = git
|
||||
self._config = config
|
||||
self._path_factory = PathFactory(self)
|
||||
self._parsed_args = args
|
||||
self._cloud = DataCloud(self)
|
||||
|
||||
@property
|
||||
def args(self):
|
||||
return self._args
|
||||
|
||||
def set_args(self, args):
|
||||
self._args = args
|
||||
|
||||
@property
|
||||
def parsed_args(self):
|
||||
return self._parsed_args
|
||||
|
||||
def parse_args(self, args):
|
||||
self._parsed_args = parse_args(args)
|
||||
|
||||
@property
|
||||
def git(self):
|
||||
return self._git
|
||||
|
||||
@property
|
||||
def config(self):
|
||||
return self._config
|
||||
|
||||
@property
|
||||
def path_factory(self):
|
||||
return self._path_factory
|
||||
|
||||
@property
|
||||
def cloud(self):
|
||||
return self._cloud
|
||||
+34
-6
@@ -8,9 +8,29 @@ from dvc.data_cloud import file_md5
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.executor import Executor
|
||||
|
||||
class OutputError(DvcException):
|
||||
def __init__(self, path, msg):
|
||||
super(OutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg))
|
||||
|
||||
class OutputNoCacheError(DvcException):
|
||||
pass
|
||||
|
||||
class OutputNoCacheError(OutputError):
|
||||
def __init__(self, path):
|
||||
super(OutputNoCacheError, self).__init__(path, 'no cache')
|
||||
|
||||
|
||||
class OutputOutsideOfRepoError(OutputError):
|
||||
def __init__(self, path):
|
||||
super(OutputOutsideOfRepoError, self).__init__(path, 'outside of repository')
|
||||
|
||||
|
||||
class OutputDoesNotExistError(OutputError):
|
||||
def __init__(self, path):
|
||||
super(OutputDoesNotExistError, self).__init__(path, 'does not exist')
|
||||
|
||||
|
||||
class OutputIsNotFileError(OutputError):
|
||||
def __init__(self, path):
|
||||
super(OutputIsNotFileError, self).__init__(path, 'not a file')
|
||||
|
||||
|
||||
class Output(object):
|
||||
@@ -20,7 +40,11 @@ class Output(object):
|
||||
|
||||
def __init__(self, project, path, md5=None, use_cache=False):
|
||||
self.project = project
|
||||
self.path = path
|
||||
self.path = os.path.abspath(os.path.realpath(path))
|
||||
|
||||
if not self.path.startswith(self.project.root_dir):
|
||||
raise OutputOutsideOfRepoError(self.path)
|
||||
|
||||
self.md5 = md5
|
||||
self.use_cache = use_cache
|
||||
|
||||
@@ -46,10 +70,10 @@ class Output(object):
|
||||
|
||||
def link(self, checkout=False):
|
||||
if not self.use_cache:
|
||||
raise OutputNoCacheError()
|
||||
raise OutputNoCacheError(self.path)
|
||||
|
||||
if not os.path.exists(self.path) and not os.path.exists(self.cache):
|
||||
raise OutputNoCacheError()
|
||||
raise OutputNoCacheError(self.path)
|
||||
|
||||
if os.path.exists(self.path) and os.path.exists(self.cache) and os.path.samefile(self.path, self.cache):
|
||||
return
|
||||
@@ -66,7 +90,7 @@ class Output(object):
|
||||
src = self.path
|
||||
link = self.cache
|
||||
else:
|
||||
raise OutputNoCacheError()
|
||||
raise OutputNoCacheError(self.path)
|
||||
|
||||
System.hardlink(src, link)
|
||||
|
||||
@@ -81,6 +105,10 @@ class Output(object):
|
||||
def update(self, md5=None):
|
||||
self.md5 = md5
|
||||
if not self.md5:
|
||||
if not os.path.exists(self.path):
|
||||
raise OutputDoesNotExistError(self.path)
|
||||
if not os.path.isfile(self.path):
|
||||
raise OutputIsNotFileError(self.path)
|
||||
self.md5 = file_md5(self.path)[0]
|
||||
self.project.state.update(self.path, self.md5, self.mtime())
|
||||
|
||||
|
||||
@@ -32,15 +32,6 @@ def cached_property(func):
|
||||
return property(get)
|
||||
|
||||
|
||||
def rmtree(directory):
|
||||
'''Cross platform rmtree()'''
|
||||
if os.name == 'nt':
|
||||
if os.path.exists(directory) and not os.access(directory, os.W_OK):
|
||||
os.chmod(directory, stat.S_IWUSR)
|
||||
|
||||
shutil.rmtree(directory, ignore_errors=True)
|
||||
|
||||
|
||||
def copyfile(src, dest):
|
||||
'''Copy file with progress bar'''
|
||||
copied = 0
|
||||
|
||||
+38
-107
@@ -1,124 +1,55 @@
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from git import Repo
|
||||
from unittest import TestCase
|
||||
|
||||
from dvc.config import ConfigI
|
||||
from dvc.git_wrapper import GitWrapperI
|
||||
from dvc.path.data_item import DataItem
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.settings import Settings
|
||||
from dvc.system import System
|
||||
from dvc.utils import rmtree
|
||||
from dvc.project import Project
|
||||
|
||||
class BasicEnvironment(TestCase):
|
||||
def init_environment(self,
|
||||
test_dir=System.get_long_path(tempfile.mkdtemp()),
|
||||
curr_dir=None):
|
||||
self._test_dir = System.realpath(test_dir)
|
||||
self._proj_dir = 'proj'
|
||||
self._test_git_dir = os.path.join(self._test_dir, self._proj_dir)
|
||||
self._old_curr_dir_abs = System.realpath(os.curdir)
|
||||
|
||||
if os.path.exists(self._test_dir):
|
||||
rmtree(self._test_dir)
|
||||
class TestDir(TestCase):
|
||||
FOO = 'foo'
|
||||
FOO_CONTENTS = FOO
|
||||
BAR = 'bar'
|
||||
BAR_CONTENTS = BAR
|
||||
CODE = 'code.py'
|
||||
CODE_CONTENTS = 'import sys\nimport shutil\nshutil.copyfile(sys.argv[1], sys.argv[2])'
|
||||
|
||||
if curr_dir:
|
||||
self._curr_dir = System.realpath(curr_dir)
|
||||
else:
|
||||
self._curr_dir = self._test_git_dir
|
||||
def _pushd(self, d):
|
||||
self._saved_dir = System.realpath(os.curdir)
|
||||
os.chdir(d)
|
||||
|
||||
if not os.path.exists(self._curr_dir):
|
||||
os.makedirs(self._curr_dir)
|
||||
def _popd(self):
|
||||
os.chdir(self._saved_dir)
|
||||
self._saved_dir = None
|
||||
|
||||
if not os.path.isdir(self._test_git_dir):
|
||||
os.makedirs(self._test_git_dir)
|
||||
def create(self, name, contents):
|
||||
with open(name, 'a') as f:
|
||||
f.write(contents)
|
||||
|
||||
data_dir = os.path.join(self._test_git_dir, 'data')
|
||||
cache_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME)
|
||||
state_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data')
|
||||
|
||||
os.makedirs(data_dir)
|
||||
os.makedirs(cache_dir)
|
||||
os.makedirs(state_dir)
|
||||
|
||||
os.chdir(self._curr_dir)
|
||||
def setUp(self):
|
||||
self._root_dir = System.get_long_path(tempfile.mkdtemp())
|
||||
self._pushd(self._root_dir)
|
||||
self.create(self.FOO, self.FOO_CONTENTS)
|
||||
self.create(self.BAR, self.BAR_CONTENTS)
|
||||
self.create(self.CODE, self.CODE_CONTENTS)
|
||||
|
||||
def tearDown(self):
|
||||
if self._old_curr_dir_abs:
|
||||
os.chdir(self._old_curr_dir_abs)
|
||||
if self._test_git_dir:
|
||||
rmtree(self._test_dir)
|
||||
pass
|
||||
self._popd()
|
||||
shutil.rmtree(self._root_dir)
|
||||
|
||||
|
||||
class DirHierarchyEnvironment(BasicEnvironment):
|
||||
def init_environment(self,
|
||||
test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
|
||||
curr_dir=None,
|
||||
commit='abc12345'):
|
||||
''' Creates data environment with data, cache and state dirs.
|
||||
data/
|
||||
file1.txt --> ../cache/file1.txt_abc123
|
||||
dir1/
|
||||
file2.txt --> ../../cache/dir1/file2.txt_abc123
|
||||
file3.txt --> ../../cache/dir1/file3.txt_abc123
|
||||
dir11/
|
||||
file4.txt --> ../../../cache/dir1/dir11/file4.txt_abc123
|
||||
dir2/
|
||||
file5.txt --> ../../cache/dir2/file5.txt_abc123
|
||||
file6.txt --> an actual file
|
||||
'''
|
||||
class TestGit(TestDir):
|
||||
def setUp(self):
|
||||
super(TestGit, self).setUp()
|
||||
self.git = Repo.init()
|
||||
self.git.index.add([self.CODE])
|
||||
self.git.index.commit('add code')
|
||||
|
||||
BasicEnvironment.init_environment(self, test_dir, curr_dir)
|
||||
|
||||
self._commit = commit
|
||||
|
||||
self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit)
|
||||
self._config = ConfigI('data')
|
||||
self.path_factory = PathFactory(self._git, self._config)
|
||||
self.settings = Settings([], self._git, self._config)
|
||||
|
||||
self.dir1 = 'dir1'
|
||||
self.dir11 = os.path.join('dir1', 'dir11')
|
||||
self.dir2 = 'dir2'
|
||||
|
||||
self.create_dirs(self.dir1)
|
||||
self.create_dirs(self.dir11)
|
||||
self.create_dirs(self.dir2)
|
||||
|
||||
self.file1, self.cache1, self.state1 = self.crate_data_item('file1.txt')
|
||||
self.file2, self.cache2, self.state2 = self.crate_data_item(os.path.join(self.dir1, 'file2.txt'))
|
||||
self.file3, self.cache3, self.state3 = self.crate_data_item(os.path.join(self.dir1, 'file3.txt'))
|
||||
self.file4, self.cache4, self.state4 = self.crate_data_item(os.path.join(self.dir11, 'file4.txt'))
|
||||
self.file5, self.cache5, self.state5 = self.crate_data_item(os.path.join(self.dir2, 'file5.txt'))
|
||||
self.file6, self.cache6, self.state6 = self.crate_data_item(os.path.join(self.dir2, 'file6.txt'),
|
||||
cache_file=False)
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def create_content_file(file, content='some test'):
|
||||
fd = open(file, 'w+')
|
||||
fd.write(content)
|
||||
fd.close()
|
||||
|
||||
def crate_data_item(self, data_file, cache_file=True, content='random text'):
|
||||
file_result = os.path.join('data', data_file)
|
||||
state_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, file_result + DataItem.STATE_FILE_SUFFIX)
|
||||
|
||||
dummy_md5 = os.path.basename(data_file) + DataItem.CACHE_FILE_SEP + self._commit
|
||||
self.create_content_file(state_result, '{"Md5" : "' + dummy_md5 + '", "Command" : "run", "Cwd" : "dir"}')
|
||||
|
||||
if cache_file:
|
||||
cache_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, dummy_md5)
|
||||
self.create_content_file(cache_result, content)
|
||||
|
||||
System.hardlink(cache_result, file_result)
|
||||
else:
|
||||
cache_result = None
|
||||
self.create_content_file(file_result, content)
|
||||
|
||||
return file_result, cache_result, state_result
|
||||
|
||||
def create_dirs(self, dir):
|
||||
os.makedirs(os.path.join('data', dir))
|
||||
os.makedirs(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', dir))
|
||||
class TestDvc(TestGit):
|
||||
def setUp(self):
|
||||
super(TestDvc, self).setUp()
|
||||
self.dvc = Project.init(self._root_dir)
|
||||
self.dvc.logger.be_verbose()
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import os
|
||||
import shutil
|
||||
import filecmp
|
||||
|
||||
from dvc.data_cloud import file_md5
|
||||
from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError
|
||||
from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError
|
||||
from dvc.project import StageNotFoundError
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestAdd(TestDvc):
|
||||
def test(self):
|
||||
md5 = file_md5(self.FOO)[0]
|
||||
|
||||
stage = self.dvc.add(self.FOO)
|
||||
|
||||
self.assertIsInstance(stage, Stage)
|
||||
self.assertTrue(os.path.isfile(stage.path))
|
||||
self.assertEqual(len(stage.outs), 1)
|
||||
self.assertEqual(len(stage.deps), 0)
|
||||
self.assertTrue(stage.locked)
|
||||
self.assertEqual(stage.cmd, None)
|
||||
self.assertEqual(stage.outs[0].md5, md5)
|
||||
|
||||
|
||||
class TestAddNonExistentFile(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(OutputDoesNotExistError) as cx:
|
||||
self.dvc.add('non_existent_file')
|
||||
|
||||
|
||||
class TestAddFileOutsideOfRepo(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(OutputOutsideOfRepoError) as cx:
|
||||
self.dvc.add(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO))
|
||||
|
||||
|
||||
class TestAddDirectory(TestDvc):
|
||||
def test(self):
|
||||
dname = 'directory'
|
||||
os.mkdir(dname)
|
||||
with self.assertRaises(OutputIsNotFileError) as cx:
|
||||
self.dvc.add(dname)
|
||||
@@ -0,0 +1,24 @@
|
||||
import os
|
||||
|
||||
from dvc.cache import Cache
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestCache(TestDvc):
|
||||
def setUp(self):
|
||||
super(TestCache, self).setUp()
|
||||
self.cache1 = os.path.join(self.dvc.cache.cache_dir, '1')
|
||||
self.cache2 = os.path.join(self.dvc.cache.cache_dir, '2')
|
||||
self.create(self.cache1, '1')
|
||||
self.create(self.cache2, '2')
|
||||
|
||||
def test_all(self):
|
||||
flist = Cache(self.dvc.dvc_dir).all()
|
||||
self.assertEquals(len(flist), 2)
|
||||
self.assertTrue(self.cache1 in flist)
|
||||
self.assertTrue(self.cache2 in flist)
|
||||
|
||||
def test_get(self):
|
||||
cache = Cache(self.dvc.dvc_dir).get(os.path.basename(self.cache1))
|
||||
self.assertEquals(cache, self.cache1)
|
||||
@@ -0,0 +1,30 @@
|
||||
import os
|
||||
import shutil
|
||||
import filecmp
|
||||
|
||||
from dvc.data_cloud import file_md5
|
||||
from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError
|
||||
from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError
|
||||
from dvc.project import StageNotFoundError
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestCheckout(TestDvc):
|
||||
def test_checkout(self):
|
||||
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
|
||||
deps = [self.FOO]
|
||||
deps_no_cache = [self.CODE]
|
||||
outs = [os.path.join(self.dvc.root_dir, 'out')]
|
||||
outs_no_cache = []
|
||||
locked = False
|
||||
fname = os.path.join(self.dvc.root_dir, 'out.dvc')
|
||||
cwd = os.curdir
|
||||
|
||||
stage = self.dvc.run(cmd, deps, deps_no_cache, outs, outs_no_cache, locked, fname, cwd)
|
||||
|
||||
shutil.copy(self.FOO, 'orig')
|
||||
os.unlink(self.FOO)
|
||||
self.dvc.checkout()
|
||||
self.assertTrue(os.path.isfile(self.FOO))
|
||||
self.assertTrue(filecmp.cmp(self.FOO, 'orig'))
|
||||
@@ -1,159 +0,0 @@
|
||||
import os
|
||||
|
||||
from dvc.command.remove import CmdRemove
|
||||
from tests.basic_env import DirHierarchyEnvironment
|
||||
|
||||
class TestCmdRemove(DirHierarchyEnvironment):
|
||||
def setUp(self):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test_file_by_file_removal(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
dir1_dvc_name = os.path.join('data', self.dir1)
|
||||
self.assertTrue(os.path.exists(dir1_dvc_name))
|
||||
|
||||
cmd.remove_dir(dir1_dvc_name)
|
||||
self.assertFalse(os.path.exists(dir1_dvc_name))
|
||||
|
||||
def test_remove_deep_dir(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud --recursive')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
dir1_dvc_name = os.path.join('data', self.dir1)
|
||||
self.assertTrue(os.path.exists(dir1_dvc_name))
|
||||
|
||||
cmd.remove_dir(dir1_dvc_name)
|
||||
self.assertFalse(os.path.exists(dir1_dvc_name))
|
||||
|
||||
def test_not_recursive_removal(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
dir1_dvc_name = os.path.join('data', self.dir1)
|
||||
self.assertTrue(os.path.exists(dir1_dvc_name))
|
||||
|
||||
cmd.remove_dir(dir1_dvc_name)
|
||||
pass
|
||||
|
||||
def test_data_dir_removal(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud --recursive')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
data_dir = 'data'
|
||||
self.assertTrue(os.path.exists(data_dir))
|
||||
|
||||
cmd.remove_dir(data_dir)
|
||||
pass
|
||||
|
||||
|
||||
class TestRemoveDataItem(DirHierarchyEnvironment):
|
||||
def setUp(self):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test_remove_data_instance(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
self.assertTrue(os.path.isfile(self.file1))
|
||||
self.assertTrue(os.path.isfile(self.cache1))
|
||||
self.assertTrue(os.path.isfile(self.state1))
|
||||
|
||||
cmd.remove_file(self.file1)
|
||||
self.assertFalse(os.path.exists(self.file1))
|
||||
self.assertFalse(os.path.exists(self.cache1))
|
||||
self.assertFalse(os.path.exists(self.state1))
|
||||
|
||||
def test_keep_cache(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud --keep-in-cache')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
self.assertTrue(os.path.isfile(self.file1))
|
||||
self.assertTrue(os.path.isfile(self.cache1))
|
||||
self.assertTrue(os.path.isfile(self.state1))
|
||||
|
||||
cmd.remove_file(self.file1)
|
||||
self.assertFalse(os.path.exists(self.file1))
|
||||
self.assertTrue(os.path.exists(self.cache1))
|
||||
self.assertFalse(os.path.exists(self.state1))
|
||||
|
||||
def test_remove_data_instance_without_cache(self):
|
||||
self.settings.parse_args('remove --keep-in-cloud')
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
self.assertTrue(os.path.isfile(self.file6))
|
||||
self.assertIsNone(self.cache6)
|
||||
self.assertTrue(os.path.exists(self.state6))
|
||||
self.assertTrue(os.path.isfile(self.state6))
|
||||
|
||||
cmd.remove_file(self.file6)
|
||||
self.assertFalse(os.path.exists(self.file6))
|
||||
|
||||
|
||||
class TestRemoveEndToEnd(DirHierarchyEnvironment):
|
||||
def setUp(self):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test_end_to_end_with_no_error(self):
|
||||
dir11_full = os.path.join('data', self.dir11)
|
||||
dir2_full = os.path.join('data', self.dir2)
|
||||
|
||||
self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5))
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
self.assertTrue(os.path.exists(dir11_full))
|
||||
self.assertTrue(os.path.exists(self.file5))
|
||||
self.assertTrue(os.path.exists(dir2_full))
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
|
||||
self.assertTrue(cmd.remove_all())
|
||||
|
||||
self.assertFalse(os.path.exists(dir11_full))
|
||||
self.assertFalse(os.path.exists(self.file5))
|
||||
self.assertTrue(os.path.exists(dir2_full))
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
|
||||
def test_run(self):
|
||||
dir11_full = os.path.join('data', self.dir11)
|
||||
dir2_full = os.path.join('data', self.dir2)
|
||||
|
||||
self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5))
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
self.assertTrue(os.path.exists(dir11_full))
|
||||
self.assertTrue(os.path.exists(self.file5))
|
||||
self.assertTrue(os.path.exists(dir2_full))
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
|
||||
self.assertEqual(cmd.run(), 0)
|
||||
|
||||
self.assertFalse(os.path.exists(dir11_full))
|
||||
self.assertFalse(os.path.exists(self.file5))
|
||||
self.assertTrue(os.path.exists(dir2_full))
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
|
||||
|
||||
class TestRemoveEndToEndWithAnError(DirHierarchyEnvironment):
|
||||
def setUp(self):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test(self):
|
||||
dir11_full = os.path.join('data', self.dir11)
|
||||
dir2_full = os.path.join('data', self.dir2)
|
||||
|
||||
self.settings.parse_args('remove --no-git-actions --keep-in-cloud --recursive {} {}'.format(dir11_full, self.file5))
|
||||
cmd = CmdRemove(self.settings)
|
||||
|
||||
self.assertTrue(os.path.exists(dir11_full))
|
||||
self.assertTrue(os.path.exists(self.file5))
|
||||
self.assertTrue(os.path.exists(dir2_full))
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
self.assertTrue(os.path.exists(self.file6))
|
||||
|
||||
self.assertTrue(cmd.remove_all())
|
||||
|
||||
self.assertFalse(os.path.exists(dir11_full))
|
||||
self.assertFalse(os.path.exists(self.file5))
|
||||
self.assertTrue(os.path.exists(dir2_full))
|
||||
self.assertTrue(os.path.exists(self.file1))
|
||||
@@ -1,100 +0,0 @@
|
||||
import os
|
||||
import tempfile
|
||||
from unittest import TestCase
|
||||
|
||||
from dvc.command.run import CmdRun, RunError
|
||||
from dvc.config import ConfigI
|
||||
from dvc.executor import Executor
|
||||
from dvc.path.data_item import DataItem
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.git_wrapper import GitWrapper
|
||||
from dvc.settings import Settings
|
||||
from dvc.system import System
|
||||
from dvc.utils import rmtree
|
||||
|
||||
|
||||
class RunBasicTest(TestCase):
|
||||
def setUp(self):
|
||||
self.test_dir = System.get_long_path(tempfile.mkdtemp())
|
||||
self._old_curr_dir_abs = System.realpath(os.curdir)
|
||||
|
||||
self.tearDown()
|
||||
os.mkdir(self.test_dir)
|
||||
os.chdir(self.test_dir)
|
||||
os.mkdir('data')
|
||||
os.mkdir(ConfigI.CONFIG_DIR)
|
||||
os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME))
|
||||
os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME))
|
||||
|
||||
self.init_git_repo()
|
||||
self.git = GitWrapper()
|
||||
|
||||
self.config = ConfigI('data')
|
||||
self.path_factory = PathFactory(self.git, self.config)
|
||||
|
||||
self.settings = Settings('run cmd', self.git, self.config)
|
||||
pass
|
||||
|
||||
def init_git_repo(self):
|
||||
Executor.exec_cmd_only_success(['git', 'init'])
|
||||
msg = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME)
|
||||
msg += '\n'
|
||||
msg += os.path.join(ConfigI.CONFIG_DIR, '.' + ConfigI.CONFIG + '.lock')
|
||||
self.create_file('.gitignore', msg)
|
||||
Executor.exec_cmd_only_success(['git', 'add', '.gitignore'])
|
||||
Executor.exec_cmd_only_success(['git', 'commit', '-m', '"Init test repo"'])
|
||||
|
||||
@staticmethod
|
||||
def create_file(file2, content='random text'):
|
||||
fd = open(file2, 'w+')
|
||||
fd.write(content)
|
||||
fd.close()
|
||||
|
||||
def tearDown(self):
|
||||
rmtree(self.test_dir)
|
||||
os.chdir(self._old_curr_dir_abs)
|
||||
|
||||
|
||||
class RunTwoFilesBase(RunBasicTest):
|
||||
def setUp(self):
|
||||
super(RunTwoFilesBase, self).setUp()
|
||||
|
||||
self.input_param_file = os.path.join('data', 'extra_input')
|
||||
self.extra_output_file = os.path.join('data', 'extra_output')
|
||||
|
||||
self.settings.parse_args('run --input {} --output {} cmd'.format(self.input_param_file, self.extra_output_file))
|
||||
self.settings._args = []
|
||||
cmd_run = CmdRun(self.settings)
|
||||
|
||||
self.file_name1 = os.path.join('data', 'file1')
|
||||
self.file_name2 = os.path.join('data', 'file2')
|
||||
self.state_objs = cmd_run.run_command(['echo', 'test'], [], [], shell=True,
|
||||
stdout=self.file_name1,
|
||||
stderr=self.file_name2)
|
||||
|
||||
self.state_file_name1 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, self.file_name1 + DataItem.STATE_FILE_SUFFIX)
|
||||
self.state_file_name2 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, self.file_name2 + DataItem.STATE_FILE_SUFFIX)
|
||||
|
||||
self.state_file1 = None
|
||||
self.state_file2 = None
|
||||
for s in self.state_objs:
|
||||
if s.file == self.state_file_name1:
|
||||
self.state_file1 = s
|
||||
elif s.file == self.state_file_name2:
|
||||
self.state_file2 = s
|
||||
pass
|
||||
|
||||
|
||||
class TestRunStateFiles(RunTwoFilesBase):
|
||||
def test(self):
|
||||
names = [x.file for x in self.state_objs]
|
||||
self.assertEqual({self.state_file_name1, self.state_file_name2}, set(names))
|
||||
|
||||
|
||||
class TestRunExtraInExtraOutFiles(RunTwoFilesBase):
|
||||
def test(self):
|
||||
self.assertIsNotNone(self.state_file1)
|
||||
self.assertEqual(self.state_file1.input_files, [self.input_param_file])
|
||||
|
||||
output_set = {self.file_name1, self.file_name2, self.extra_output_file}
|
||||
self.assertEqual(set(self.state_file1.output_files), output_set)
|
||||
+14
-11
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import configparser
|
||||
from unittest import TestCase
|
||||
import mock
|
||||
|
||||
@@ -15,15 +16,18 @@ except ImportError:
|
||||
# Python 3
|
||||
builtin_module_name = 'builtins'
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.data_cloud import DataCloud
|
||||
from dvc.settings import Settings
|
||||
|
||||
|
||||
class TestConfigTest(TestCase):
|
||||
def setUp(self):
|
||||
pass
|
||||
|
||||
def _conf(self, s):
|
||||
c = configparser.SafeConfigParser()
|
||||
c.readfp(s)
|
||||
return c
|
||||
|
||||
def storage_path_hierarchy_test(self):
|
||||
""" StoragePath should be read first from global and then from cloud section """
|
||||
|
||||
@@ -41,9 +45,8 @@ class TestConfigTest(TestCase):
|
||||
"StoragePath = googlesb/google_storage_path"
|
||||
)
|
||||
s = StringIO('\n'.join(c))
|
||||
conf = Config(s, conf_pseudo_file = s)
|
||||
settings = Settings(None, None, conf)
|
||||
cloud = DataCloud(settings)
|
||||
conf = self._conf(s)
|
||||
cloud = DataCloud(conf)
|
||||
self.assertEqual(cloud.typ, 'AWS')
|
||||
self.assertEqual(cloud._cloud.storage_bucket, 'globalsb')
|
||||
self.assertEqual(cloud._cloud.storage_prefix, 'global_storage_path')
|
||||
@@ -61,8 +64,8 @@ class TestConfigTest(TestCase):
|
||||
"StoragePath = googlesb/google_storage_path"
|
||||
)
|
||||
s = StringIO('\n'.join(c))
|
||||
conf = Config(s, conf_pseudo_file=s)
|
||||
cloud = DataCloud(Settings(None, None, conf))
|
||||
conf = self._conf(s)
|
||||
cloud = DataCloud(conf)
|
||||
self.assertEqual(cloud.typ, 'AWS')
|
||||
self.assertEqual(cloud._cloud.storage_bucket, 'awssb')
|
||||
self.assertEqual(cloud._cloud.storage_prefix, 'aws_storage_path')
|
||||
@@ -100,8 +103,8 @@ class TestConfigTest(TestCase):
|
||||
patcher = mock.patch(builtin_module_name + '.open', side_effect=self.mocked_open_aws_default_credentials)
|
||||
|
||||
# patcher.start()
|
||||
conf = Config(s, conf_pseudo_file=s)
|
||||
cloud = DataCloud(Settings(None, None, conf))
|
||||
conf = self._conf(s)
|
||||
cloud = DataCloud(conf)
|
||||
aws_creds = cloud._cloud._get_credentials()
|
||||
patcher.stop()
|
||||
|
||||
@@ -131,8 +134,8 @@ class TestConfigTest(TestCase):
|
||||
patcher = mock.patch(builtin_module_name + '.open', side_effect=self.mocked_open_aws_default_credentials)
|
||||
|
||||
patcher.start()
|
||||
conf = Config(s, conf_pseudo_file=s)
|
||||
cloud = DataCloud(Settings(None, None, conf))
|
||||
conf = self._conf(s)
|
||||
cloud = DataCloud(conf)
|
||||
aws_creds = cloud._cloud._get_credentials()
|
||||
patcher.stop()
|
||||
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
import os
|
||||
|
||||
from dvc.config import ConfigI
|
||||
from dvc.git_wrapper import GitWrapperI
|
||||
from dvc.path.data_item import DataDirError, DataItemError, DataItem
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.system import System
|
||||
from tests.basic_env import BasicEnvironment
|
||||
|
||||
|
||||
class TestDataFileObjBasic(BasicEnvironment):
|
||||
def setUp(self):
|
||||
BasicEnvironment.init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
|
||||
|
||||
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
|
||||
config = ConfigI()
|
||||
|
||||
file = os.path.join('data', 'file.txt')
|
||||
self.cache_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file.txt_ad45ba8')
|
||||
self._data_path = DataItem(file, git, config, cache_file=self.cache_file_full_path)
|
||||
pass
|
||||
|
||||
def test_data_file(self):
|
||||
data_file_full_path = os.path.join(self._test_git_dir, 'data', 'file.txt')
|
||||
self.assertEqual(self._data_path.data.abs, data_file_full_path)
|
||||
|
||||
def test_cache_file(self):
|
||||
self.assertEqual(self._data_path.cache.abs, self.cache_file_full_path)
|
||||
|
||||
def test_state_file(self):
|
||||
state_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file.txt.state')
|
||||
self.assertEqual(self._data_path.state.abs, state_file_full_path)
|
||||
|
||||
|
||||
class TestDataItemWithGivenCache(BasicEnvironment):
|
||||
def setUp(self):
|
||||
BasicEnvironment.init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
|
||||
|
||||
self._git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
|
||||
self._config = ConfigI('data')
|
||||
|
||||
self._file = os.path.join('data', 'file.txt')
|
||||
self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file.txt_abcd')
|
||||
|
||||
self._data_item = DataItem(self._file, self._git, self._config, self._cache)
|
||||
pass
|
||||
|
||||
def test_basic(self):
|
||||
self.assertEqual(self._data_item.data.relative, self._file)
|
||||
self.assertEqual(self._data_item.cache.relative, self._cache)
|
||||
|
||||
|
||||
class TestDeepDataItemWithGivenCache(BasicEnvironment):
|
||||
def setUp(self):
|
||||
BasicEnvironment.init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
|
||||
|
||||
self._git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
|
||||
self._config = ConfigI('data')
|
||||
|
||||
self._file = os.path.join('data', 'dir1', 'file.txt')
|
||||
self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'dir1', 'file.txt_abcd')
|
||||
|
||||
self._data_item = DataItem(self._file, self._git, self._config, self._cache)
|
||||
pass
|
||||
|
||||
def test_basic(self):
|
||||
self.assertEqual(self._data_item.data.relative, self._file)
|
||||
self.assertEqual(self._data_item.cache.relative, self._cache)
|
||||
|
||||
|
||||
class TestPathFactory(BasicEnvironment):
|
||||
def setUp(self):
|
||||
BasicEnvironment.init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
|
||||
|
||||
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
|
||||
config = ConfigI()
|
||||
self.path_factory = PathFactory(git, config)
|
||||
|
||||
self.data_file = os.path.join('data', 'file.txt')
|
||||
dummy_md5 = 'fsymlinc.txt'
|
||||
self.cache_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, dummy_md5)
|
||||
self.state_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file.txt' + DataItem.STATE_FILE_SUFFIX)
|
||||
|
||||
with open(self.cache_file, 'w+') as fd:
|
||||
fd.write('some text')
|
||||
|
||||
with open(self.state_file, 'w+') as fd:
|
||||
fd.write('{"Md5" : "' + dummy_md5 + '", "Command" : "run", "Cwd" : "dir"}')
|
||||
|
||||
os.chdir('data')
|
||||
System.hardlink(os.path.join('..', self.cache_file), 'file.txt')
|
||||
os.chdir('..')
|
||||
pass
|
||||
|
||||
def test_data_file_factory(self):
|
||||
data_path = self.path_factory.data_item(self.data_file)
|
||||
self.assertEqual(data_path.data.dvc, self.data_file)
|
||||
|
||||
indirect_file_path = os.path.join('..', self._proj_dir, self.data_file)
|
||||
data_path_indirect = self.path_factory.data_item(indirect_file_path)
|
||||
self.assertEqual(data_path_indirect.data.dvc, self.data_file)
|
||||
pass
|
||||
|
||||
def test_data_hardlink_factory(self):
|
||||
data_path = self.path_factory.existing_data_item(self.data_file)
|
||||
self.assertEqual(data_path.cache.dvc, self.cache_file)
|
||||
pass
|
||||
|
||||
def test_data_hardlink_factory_cache(self):
|
||||
data_path = self.path_factory.existing_data_item(self.data_file)
|
||||
self.assertEqual(data_path.data.dvc, self.data_file)
|
||||
pass
|
||||
|
||||
def test_data_hardlink_factory_exception(self):
|
||||
with self.assertRaises(DataItemError):
|
||||
self.path_factory.existing_data_item(self.cache_file)
|
||||
pass
|
||||
|
||||
def test_data_path(self):
|
||||
file = os.path.join('data', 'file1')
|
||||
path = self.path_factory.path(file)
|
||||
self.assertEqual(path.dvc, file)
|
||||
self.assertEqual(path.relative, file)
|
||||
self.assertTrue(path.abs.endswith(file))
|
||||
|
||||
|
||||
class TestDataPathInDataDir(BasicEnvironment):
|
||||
def setUp(self):
|
||||
BasicEnvironment.init_environment(self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
|
||||
|
||||
git = GitWrapperI(git_dir=self._test_git_dir, commit='eeeff8f')
|
||||
self.data_dir = 'da'
|
||||
config = ConfigI()
|
||||
self.path_factory = PathFactory(git, config)
|
||||
|
||||
deep_path = os.path.join(self.data_dir, 'dir1', 'd2', 'file.txt')
|
||||
self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'dir1', 'd2', 'file.txt_eeeff8f')
|
||||
self.data_path = self.path_factory.data_item(deep_path, cache_file=self.cache_file)
|
||||
pass
|
||||
|
||||
def test_data_file(self):
|
||||
target = os.path.join(self._test_git_dir, 'da', 'dir1', 'd2', 'file.txt')
|
||||
self.assertEqual(self.data_path.data.abs, target)
|
||||
|
||||
def test_cache_file(self):
|
||||
self.assertEqual(self.data_path.cache.abs, self.cache_file)
|
||||
|
||||
def test_state_file(self):
|
||||
target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'da', 'dir1', 'd2', 'file.txt.state')
|
||||
self.assertEqual(self.data_path.state.abs, target)
|
||||
|
||||
|
||||
class TestDataFileObjLongPath(BasicEnvironment):
|
||||
def setUp(self):
|
||||
curr_dir = os.path.join(os.path.sep, 'tmp', 'ntx_unit_test', 'proj', 'mydir', 'dd3')
|
||||
|
||||
BasicEnvironment.init_environment(self,
|
||||
os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
|
||||
curr_dir)
|
||||
|
||||
self._git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
|
||||
self._config = ConfigI('data')
|
||||
self.path_factory = PathFactory(self._git, self._config)
|
||||
|
||||
self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file1.txt_123ed8')
|
||||
self.data_path = self.path_factory.data_item(os.path.join('..', '..', 'data', 'file1.txt'), cache_file=self.cache_file)
|
||||
pass
|
||||
|
||||
def test_data_file(self):
|
||||
target = os.path.join(self._test_git_dir, 'data', 'file1.txt')
|
||||
self.assertEqual(self.data_path.data.abs, target)
|
||||
|
||||
def test_cache_file(self):
|
||||
self.assertEqual(self.data_path.cache.abs, self.cache_file)
|
||||
|
||||
def test_state_file(self):
|
||||
target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file1.txt.state')
|
||||
self.assertEqual(self.data_path.state.abs, target)
|
||||
|
||||
|
||||
class RunOutsideGitRepoTest(BasicEnvironment):
|
||||
def setUp(self):
|
||||
curr_dir = os.path.join(os.path.sep, 'tmp')
|
||||
|
||||
BasicEnvironment.init_environment(self,
|
||||
os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
|
||||
curr_dir)
|
||||
|
||||
def test_outside_git_dir(self):
|
||||
git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
|
||||
config = ConfigI('data')
|
||||
path_factory = PathFactory(git, config)
|
||||
|
||||
with self.assertRaises(DataDirError):
|
||||
path_factory.data_item('file.txt')
|
||||
pass
|
||||
@@ -1,70 +0,0 @@
|
||||
import os
|
||||
import shutil
|
||||
from unittest import TestCase
|
||||
|
||||
from dvc.path.path import Path
|
||||
from dvc.git_wrapper import GitWrapperI
|
||||
from dvc.system import System
|
||||
|
||||
|
||||
class TestDvcPathTest(TestCase):
|
||||
def setUp(self):
|
||||
self.curr_dir = System.realpath('.')
|
||||
self.test_dir = System.realpath('/tmp/ntx_unit_test/dvc_path')
|
||||
self.tearDown()
|
||||
os.makedirs(os.path.join(self.test_dir, 'data'))
|
||||
os.makedirs(os.path.join(self.test_dir, 'code', 'lib'))
|
||||
os.makedirs(os.path.join(self.test_dir, 'd1', 'd2', 'dir3', 'd4', 'dir5'))
|
||||
|
||||
self._git = GitWrapperI(self.test_dir)
|
||||
|
||||
def tearDown(self):
|
||||
os.chdir(self.curr_dir)
|
||||
shutil.rmtree(self.test_dir, ignore_errors=True)
|
||||
|
||||
def _validate_dvc_path(self, path, dvc_file_name, relative_file_name):
|
||||
self.assertEqual(path.dvc, dvc_file_name)
|
||||
self.assertEqual(path.filename, os.path.basename(dvc_file_name))
|
||||
self.assertEqual(path.relative, relative_file_name)
|
||||
self.assertTrue(os.path.isabs(path.abs))
|
||||
self.assertTrue(path.abs.endswith(dvc_file_name))
|
||||
|
||||
def basic_test(self):
|
||||
os.chdir(self.test_dir)
|
||||
|
||||
file = os.path.join('data', 'file.txt')
|
||||
path = Path(file, self._git)
|
||||
self._validate_dvc_path(path, file, file)
|
||||
pass
|
||||
|
||||
def from_dir_test(self):
|
||||
os.chdir(os.path.join(self.test_dir, 'code'))
|
||||
|
||||
file_dvc_path = os.path.join('data', 'file1.txt')
|
||||
file_relative_path = os.path.join('..', file_dvc_path)
|
||||
|
||||
path = Path(file_relative_path, self._git)
|
||||
self._validate_dvc_path(path, file_dvc_path, file_relative_path)
|
||||
pass
|
||||
|
||||
def from_deep_dirs_test(self):
|
||||
deep_dir = os.path.join('d1', 'd2', 'dir3')
|
||||
os.chdir(os.path.join(self.test_dir, deep_dir))
|
||||
|
||||
file_dvc = os.path.join('code', 'lib', 'context_switcher_structs.asm')
|
||||
file_relative = os.path.join('..', '..', '..', file_dvc)
|
||||
|
||||
path = Path(file_relative, self._git)
|
||||
self._validate_dvc_path(path, file_dvc, file_relative)
|
||||
pass
|
||||
|
||||
def go_deeper_test(self):
|
||||
deep_dir = os.path.join('d1', 'd2', 'dir3')
|
||||
os.chdir(os.path.join(self.test_dir, deep_dir))
|
||||
|
||||
file_relative_path = os.path.join(deep_dir, 'd4', 'dir5', 'rawdata.tsv')
|
||||
file_dvc_path = os.path.join(deep_dir, file_relative_path)
|
||||
|
||||
path = Path(file_relative_path, self._git)
|
||||
self._validate_dvc_path(path, file_dvc_path, file_relative_path)
|
||||
pass
|
||||
@@ -0,0 +1,32 @@
|
||||
import os
|
||||
import shutil
|
||||
import filecmp
|
||||
|
||||
from dvc.data_cloud import file_md5
|
||||
from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError
|
||||
from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError
|
||||
from dvc.project import StageNotFoundError
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestGC(TestDvc):
|
||||
def test_gc(self):
|
||||
stage = self.dvc.add(self.FOO)
|
||||
good_cache = self.dvc.cache.all()
|
||||
|
||||
bad_cache = []
|
||||
for i in ['1', '2', '3']:
|
||||
path = os.path.join(self.dvc.cache.cache_dir, i)
|
||||
self.create(path, i)
|
||||
bad_cache.append(path)
|
||||
|
||||
self.dvc.gc()
|
||||
|
||||
self.assertTrue(os.path.isdir(self.dvc.cache.cache_dir))
|
||||
for c in bad_cache:
|
||||
self.assertFalse(os.path.exists(c))
|
||||
|
||||
for c in good_cache:
|
||||
self.assertTrue(os.path.exists(c))
|
||||
self.assertTrue(os.path.isfile(c))
|
||||
@@ -1,119 +0,0 @@
|
||||
import os
|
||||
import tempfile
|
||||
import filecmp
|
||||
from copy import copy
|
||||
|
||||
from dvc.command.import_file import CmdImportFile, ImportFileError
|
||||
from tests.basic_env import DirHierarchyEnvironment
|
||||
|
||||
|
||||
class TestCmdRemove(DirHierarchyEnvironment):
|
||||
def setUp(self):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def test_import_file_to_dir(self):
|
||||
with tempfile.NamedTemporaryFile(mode='w') as temp:
|
||||
content = 'Some data'
|
||||
temp.write(content)
|
||||
temp.flush()
|
||||
|
||||
dir = os.path.join('data', self.dir1)
|
||||
settings = copy(self.settings)
|
||||
settings.parse_args('import {} {}'.format(temp.name, dir))
|
||||
cmd = CmdImportFile(settings)
|
||||
cmd.run()
|
||||
|
||||
dvc_name = os.path.join(dir, os.path.basename(temp.name))
|
||||
data_item = self.path_factory.existing_data_item(dvc_name)
|
||||
|
||||
self.assertTrue(os.path.exists(data_item.data.relative))
|
||||
self.assertTrue(filecmp.cmp(data_item.data.relative, data_item.cache.relative))
|
||||
self.assertEqual(open(data_item.data.relative).read(), content)
|
||||
|
||||
self.assertTrue(os.path.exists(data_item.cache.relative))
|
||||
self.assertEqual(open(data_item.cache.relative).read(), content)
|
||||
|
||||
self.assertTrue(os.path.exists(data_item.state.relative))
|
||||
pass
|
||||
|
||||
def test_import_file_to_file(self):
|
||||
with tempfile.NamedTemporaryFile(mode='w') as temp:
|
||||
content = 'Some data'
|
||||
temp.write(content)
|
||||
temp.flush()
|
||||
|
||||
target_file_name = 'targ.txt'
|
||||
target_file = os.path.join('data', self.dir1, target_file_name)
|
||||
settings = copy(self.settings)
|
||||
settings.parse_args('import {} {}'.format(temp.name, target_file))
|
||||
cmd = CmdImportFile(settings)
|
||||
cmd.run()
|
||||
|
||||
self.assertTrue(os.path.exists(target_file))
|
||||
self.assertTrue(open(target_file).read(), content)
|
||||
pass
|
||||
|
||||
def test_import_to_existing_dir(self):
|
||||
with tempfile.NamedTemporaryFile(mode='w') as temp:
|
||||
content = 'Some data'
|
||||
temp.write(content)
|
||||
temp.flush()
|
||||
|
||||
dir = os.path.join('data', self.dir11)
|
||||
settings = copy(self.settings)
|
||||
settings.parse_args('import {} {}'.format(temp.name, dir))
|
||||
cmd = CmdImportFile(settings)
|
||||
cmd.run()
|
||||
|
||||
dvc_name = os.path.join(dir, os.path.basename(temp.name))
|
||||
|
||||
self.assertTrue(os.path.exists(dvc_name))
|
||||
self.assertTrue(open(dvc_name).read(), content)
|
||||
pass
|
||||
|
||||
def test_import_to_not_existing_dir(self):
|
||||
with tempfile.NamedTemporaryFile(mode='w') as temp:
|
||||
content = 'Some data'
|
||||
temp.write(content)
|
||||
temp.flush()
|
||||
|
||||
dir = os.path.join('data', self.dir1, 'not_existing_dir/')
|
||||
settings = copy(self.settings)
|
||||
settings.parse_args('import {} {}'.format(temp.name, dir))
|
||||
cmd = CmdImportFile(settings)
|
||||
|
||||
with self.assertRaises(ImportFileError):
|
||||
cmd.run()
|
||||
|
||||
dvc_name = os.path.join(dir, os.path.basename(temp.name))
|
||||
|
||||
self.assertFalse(os.path.exists(dir))
|
||||
self.assertFalse(os.path.exists(dvc_name))
|
||||
pass
|
||||
|
||||
def test_multiple_import(self):
|
||||
with tempfile.NamedTemporaryFile(mode='w') as temp1,\
|
||||
tempfile.NamedTemporaryFile(mode='w') as temp2:
|
||||
content = 'Some data'
|
||||
|
||||
temp1.write(content)
|
||||
temp1.flush()
|
||||
|
||||
temp2.write(content)
|
||||
temp2.flush()
|
||||
|
||||
dir = os.path.join('data', self.dir11)
|
||||
settings = copy(self.settings)
|
||||
settings.parse_args('import {} {} {}'.format(temp1.name, temp2.name, dir))
|
||||
cmd = CmdImportFile(settings)
|
||||
cmd.run()
|
||||
|
||||
dvc_name1 = os.path.join(dir, os.path.basename(temp1.name))
|
||||
dvc_name2 = os.path.join(dir, os.path.basename(temp2.name))
|
||||
|
||||
self.assertTrue(os.path.exists(dvc_name1))
|
||||
self.assertTrue(open(dvc_name1).read(), content)
|
||||
|
||||
self.assertTrue(os.path.exists(dvc_name2))
|
||||
self.assertTrue(open(dvc_name2).read(), content)
|
||||
pass
|
||||
@@ -0,0 +1,19 @@
|
||||
from dvc.lock import Lock
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestLock(TestDvc):
|
||||
def test_lock(self):
|
||||
lock = Lock(self.dvc.dvc_dir)
|
||||
lock.lock()
|
||||
self.assertTrue(lock._lock.exists())
|
||||
self.assertTrue(lock._lock.acquired)
|
||||
lock.unlock()
|
||||
self.assertFalse(lock._lock.acquired)
|
||||
|
||||
def test_with(self):
|
||||
lock = Lock(self.dvc.dvc_dir)
|
||||
with lock:
|
||||
self.assertTrue(lock._lock.acquired)
|
||||
self.assertFalse(lock._lock.acquired)
|
||||
@@ -0,0 +1,28 @@
|
||||
import os
|
||||
|
||||
from dvc.stage import Stage, OutputOutsideOfRepoError
|
||||
from dvc.project import StageNotFoundError
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestRemove(TestDvc):
|
||||
def test(self):
|
||||
stage = self.dvc.add(self.FOO)
|
||||
stages_removed = self.dvc.remove(self.FOO)
|
||||
|
||||
self.assertEqual(len(stages_removed), 1)
|
||||
self.assertIsInstance(stages_removed[0], Stage)
|
||||
self.assertEqual(stage.path, stages_removed[0].path)
|
||||
|
||||
|
||||
class TestRemoveNonExistentFile(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(StageNotFoundError) as cx:
|
||||
self.dvc.remove('non_existent_file')
|
||||
|
||||
|
||||
class TestRemoveFileOutsideOfRepo(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(OutputOutsideOfRepoError) as cx:
|
||||
self.dvc.remove(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO))
|
||||
@@ -1,98 +0,0 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from unittest import TestCase
|
||||
|
||||
from dvc.config import ConfigI
|
||||
from dvc.path.factory import PathFactory
|
||||
from dvc.git_wrapper import GitWrapperI
|
||||
from dvc.repository_change import RepositoryChange
|
||||
from dvc.settings import Settings
|
||||
from dvc.system import System
|
||||
|
||||
|
||||
class BasicTestRepositoryChange(TestCase):
|
||||
def setUp(self):
|
||||
self.test_dir = System.get_long_path(tempfile.mkdtemp())
|
||||
self._old_curr_dir_abs = System.realpath(os.curdir)
|
||||
|
||||
self.tearDown()
|
||||
os.mkdir(self.test_dir)
|
||||
os.chdir(self.test_dir)
|
||||
os.mkdir('data')
|
||||
|
||||
self._devnull = open(os.devnull, 'w')
|
||||
subprocess.Popen(['git', 'init'], stdout=self._devnull, stderr=None).wait()
|
||||
|
||||
self.git = GitWrapperI(self.test_dir)
|
||||
self.config = ConfigI()
|
||||
self.path_factory = PathFactory(self.git, self.config)
|
||||
self.settings = Settings([], self.git, self.config)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir, ignore_errors=True)
|
||||
os.chdir(self._old_curr_dir_abs)
|
||||
|
||||
@staticmethod
|
||||
def create_file(file2):
|
||||
fd = open(file2, 'w+')
|
||||
fd.write('random text')
|
||||
fd.close()
|
||||
|
||||
|
||||
class TestRepositoryChange(BasicTestRepositoryChange):
|
||||
def test_all(self):
|
||||
file_to_modify = os.path.join('data', 'file1.txt')
|
||||
file_created_before_run = os.path.join('data', 'file2.txt')
|
||||
file_created_by_run = os.path.join('data', 'out.txt')
|
||||
file_to_remove = os.path.join('data', 'file3.txt')
|
||||
|
||||
self.create_file(file_to_modify)
|
||||
self.create_file(file_created_before_run)
|
||||
self.create_file(file_to_remove)
|
||||
|
||||
subprocess.Popen(['git', 'add', file_to_modify, file_to_remove],
|
||||
stdout=self._devnull, stderr=None).wait()
|
||||
subprocess.Popen(['git', 'commit', '-m', '"Adding one file3"'],
|
||||
stdout=self._devnull, stderr=None).wait()
|
||||
os.remove(file_to_remove)
|
||||
|
||||
change = RepositoryChange(['echo', 'test'],
|
||||
self.settings,
|
||||
file_created_by_run,
|
||||
file_to_modify,
|
||||
shell=True)
|
||||
|
||||
expected_new = [file_created_by_run, file_created_before_run]
|
||||
new_file_abs = [x.data.dvc for x in change.new_data_items]
|
||||
self.assertEqual(set(new_file_abs), set(expected_new))
|
||||
|
||||
self.assertEqual(len(change.removed_data_items), 1)
|
||||
self.assertEqual(change.removed_data_items[0].data.dvc, file_to_remove)
|
||||
self.assertEqual(len(change.modified_data_items), 1)
|
||||
self.assertEqual(change.modified_data_items[0].data.dvc, file_to_modify)
|
||||
|
||||
expected_to_change_set = set(expected_new + [file_to_modify])
|
||||
changed_file_names = [x.data.dvc for x in change.changed_data_items]
|
||||
self.assertEqual(set(changed_file_names), expected_to_change_set)
|
||||
|
||||
self.assertEqual([], change.externally_created_files)
|
||||
pass
|
||||
|
||||
|
||||
class TestRepositoryChangeDeepInDirectory(BasicTestRepositoryChange):
|
||||
def test(self):
|
||||
os.mkdir(os.path.join('data', 'dir32'))
|
||||
deep_file = os.path.join('data', 'dir32', 'file1.txt')
|
||||
self.create_file(deep_file)
|
||||
|
||||
change = RepositoryChange(['echo', 'text'],
|
||||
self.settings,
|
||||
deep_file,
|
||||
None,
|
||||
shell=True)
|
||||
|
||||
self.assertEqual([x.data.dvc for x in change.new_data_items], [deep_file])
|
||||
|
||||
|
||||
+50
-72
@@ -1,90 +1,73 @@
|
||||
import os
|
||||
|
||||
from dvc.command.remove import CmdRemove
|
||||
from dvc.command.repro import CmdRepro
|
||||
from dvc.command.run import CmdRun
|
||||
from dvc.executor import Executor
|
||||
from tests.test_cmd_run import RunBasicTest
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class ReproBasicEnv(RunBasicTest):
|
||||
class ReproBasicEnv(TestDvc):
|
||||
def setUp(self):
|
||||
super(ReproBasicEnv, self).setUp()
|
||||
|
||||
os.mkdir('data')
|
||||
|
||||
self.file_name1 = os.path.join('data', 'file1')
|
||||
self.file1_code_file = 'file1.py'
|
||||
self.create_file_and_commit(self.file1_code_file, 'print("Hello")' + os.linesep + 'print("Mary")')
|
||||
self.settings.parse_args(['run',
|
||||
'--stdout', self.file_name1,
|
||||
'--code', self.file1_code_file,
|
||||
'python', self.file1_code_file, '--not-repro'])
|
||||
cmd_file1 = CmdRun(self.settings)
|
||||
self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file])
|
||||
cmd_file1.run()
|
||||
self.create(self.file1_code_file, 'print("Hello")' + os.linesep + 'print("Mary")')
|
||||
stage = self.dvc.run(fname='file1.dvc',
|
||||
outs=[self.file_name1],
|
||||
deps_no_cache=[self.file1_code_file],
|
||||
cmd='python {} --not-repro > {}'.format(self.file1_code_file,
|
||||
self.file_name1))
|
||||
|
||||
self.file_name11 = os.path.join('data', 'file11')
|
||||
self.file11_code_file = 'file11.py'
|
||||
self.create_file_and_commit(self.file11_code_file,
|
||||
'import sys' + os.linesep + 'print(open(sys.argv[1]).readline().strip())')
|
||||
self.create(self.file11_code_file,
|
||||
'import sys' + os.linesep + 'print(open(sys.argv[1]).readline().strip())')
|
||||
|
||||
self.settings.parse_args(['run',
|
||||
'--stdout', self.file_name11,
|
||||
'--code', self.file11_code_file,
|
||||
'python', self.file11_code_file, self.file_name1])
|
||||
CmdRun(self.settings).run()
|
||||
stage = self.dvc.run(fname='file11.dvc',
|
||||
outs=[self.file_name11],
|
||||
deps=[self.file_name1],
|
||||
deps_no_cache=[self.file11_code_file],
|
||||
cmd='python {} {} > {}'.format(self.file11_code_file,
|
||||
self.file_name1,
|
||||
self.file_name11))
|
||||
self.file_name11_stage_name = stage.path
|
||||
|
||||
self.file_name2 = os.path.join('data', 'file2')
|
||||
self.file2_code_file = 'file2.py'
|
||||
self.create_file_and_commit(self.file2_code_file,
|
||||
'print("Bobby")')
|
||||
self.settings.parse_args(['run',
|
||||
'--stdout', self.file_name2,
|
||||
'python', self.file2_code_file, '--not-repro'])
|
||||
CmdRun(self.settings).run()
|
||||
self.create(self.file2_code_file, 'print("Bobby")')
|
||||
self.dvc.run(fname='file2.dvc',
|
||||
outs=[self.file_name2],
|
||||
cmd='python {} --not-repro > {}'.format(self.file2_code_file,
|
||||
self.file_name2))
|
||||
|
||||
self.file_res_code_file = 'code_res.py'
|
||||
self.create_file_and_commit(self.file_res_code_file,
|
||||
'import sys' + os.linesep +
|
||||
'text1 = open(sys.argv[1]).read()' + os.linesep +
|
||||
'text2 = open(sys.argv[2]).read()' + os.linesep +
|
||||
'print(text1 + text2)')
|
||||
self.create(self.file_res_code_file,
|
||||
'import sys' + os.linesep +
|
||||
'text1 = open(sys.argv[1]).read()' + os.linesep +
|
||||
'text2 = open(sys.argv[2]).read()' + os.linesep +
|
||||
'print(text1 + text2)')
|
||||
self.file_name_res = os.path.join('data', 'file_res')
|
||||
self.settings.parse_args(['run',
|
||||
'--stdout', self.file_name_res,
|
||||
'--code', self.file_res_code_file,
|
||||
'python', self.file_res_code_file,
|
||||
self.file_name11,
|
||||
self.file_name2])
|
||||
stage = self.dvc.run(fname='file_res.dvc',
|
||||
outs=[self.file_name_res],
|
||||
deps_no_cache=[self.file_res_code_file],
|
||||
deps=[self.file_name11, self.file_name2],
|
||||
cmd='python {} {} {} > {}'.format(self.file_res_code_file,
|
||||
self.file_name11,
|
||||
self.file_name2,
|
||||
self.file_name_res))
|
||||
self.file_name_res_stage_name = stage.path
|
||||
|
||||
cmd_res = CmdRun(self.settings)
|
||||
self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file])
|
||||
cmd_res.run()
|
||||
|
||||
lines = list(filter(None, map(str.strip, open(self.file_name_res).readlines())))
|
||||
self.assertEqual(lines, ['Hello', 'Bobby'])
|
||||
|
||||
def create_file_and_commit(self, file_name, content='Any', message='Just a commit'):
|
||||
self.create_file(file_name, content)
|
||||
self.commit_file(file_name, message)
|
||||
|
||||
@staticmethod
|
||||
def commit_file(file_name, message='Just a commit'):
|
||||
Executor.exec_cmd_only_success(['git', 'add', file_name])
|
||||
Executor.exec_cmd_only_success(['git', 'commit', '-m', message])
|
||||
|
||||
def modify_file_and_commit(self, filename, content_to_add=' '):
|
||||
def modify_file(self, filename, content_to_add=' '):
|
||||
fd = open(filename, 'a')
|
||||
fd.write(content_to_add)
|
||||
fd.close()
|
||||
self.commit_file(filename)
|
||||
|
||||
|
||||
class ReproCodeDependencyTest(ReproBasicEnv):
|
||||
def test(self):
|
||||
self.modify_file_and_commit(self.file_res_code_file)
|
||||
self.modify_file(self.file_res_code_file)
|
||||
|
||||
self.settings.parse_args('repro {}'.format(self.file_name_res))
|
||||
CmdRepro(self.settings).run()
|
||||
self.dvc.reproduce([self.file_name_res_stage_name])
|
||||
|
||||
self.assertEqual(open(self.file_name_res).read().strip(), 'Hello\nBobby')
|
||||
|
||||
@@ -93,30 +76,25 @@ class ReproChangedDependency(ReproBasicEnv):
|
||||
def test(self):
|
||||
self.recreate_file1()
|
||||
|
||||
self.settings.parse_args('repro {}'.format(self.file_name11))
|
||||
CmdRepro(self.settings).run()
|
||||
self.dvc.reproduce([self.file_name11_stage_name])
|
||||
|
||||
self.assertEqual(open(self.file_name11).read(), 'Goodbye\n')
|
||||
|
||||
def recreate_file1(self):
|
||||
self.settings.parse_args('remove {} --keep-in-cloud'.format(self.file_name1))
|
||||
CmdRemove(self.settings).run()
|
||||
self.dvc.remove(self.file_name1)
|
||||
|
||||
file1_code_file = 'file1_2.py'
|
||||
self.create_file_and_commit(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")')
|
||||
self.settings.parse_args(['run',
|
||||
'--stdout', self.file_name1,
|
||||
'--code', file1_code_file,
|
||||
'python', file1_code_file, '--not-repro'])
|
||||
|
||||
CmdRun(self.settings).run()
|
||||
self.create(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")')
|
||||
self.dvc.run(fname='file1.dvc',
|
||||
outs=[self.file_name1],
|
||||
deps_no_cache=[file1_code_file],
|
||||
cmd='python {} --not-repro > {}'.format(file1_code_file, self.file_name1))
|
||||
|
||||
|
||||
class ReproChangedDeepDependency(ReproChangedDependency):
|
||||
def test(self):
|
||||
self.recreate_file1()
|
||||
|
||||
self.settings.parse_args('repro {}'.format(self.file_name_res))
|
||||
CmdRepro(self.settings).run()
|
||||
self.dvc.reproduce([self.file_name_res_stage_name])
|
||||
|
||||
self.assertEqual(open(self.file_name_res).read().strip(), 'Goodbye\nBobby')
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import os
|
||||
import filecmp
|
||||
|
||||
from dvc.data_cloud import file_md5
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestRun(TestDvc):
|
||||
def test(self):
|
||||
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
|
||||
deps = [self.FOO]
|
||||
deps_no_cache = [self.CODE]
|
||||
outs = [os.path.join(self.dvc.root_dir, 'out')]
|
||||
outs_no_cache = []
|
||||
locked = False
|
||||
fname = os.path.join(self.dvc.root_dir, 'out.dvc')
|
||||
cwd = os.curdir
|
||||
|
||||
stage = self.dvc.run(cmd, deps, deps_no_cache, outs, outs_no_cache, locked, fname, cwd)
|
||||
|
||||
self.assertTrue(filecmp.cmp(self.FOO, 'out'))
|
||||
self.assertTrue(os.path.isfile(stage.path))
|
||||
self.assertEqual(stage.cmd, cmd)
|
||||
self.assertEqual(len(stage.deps), len(deps + deps_no_cache))
|
||||
self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
|
||||
self.assertEqual(stage.outs[0].path, outs[0])
|
||||
self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])
|
||||
self.assertEqual(stage.locked, locked)
|
||||
self.assertTrue(stage.path, fname)
|
||||
|
||||
|
||||
class TestRunEmpty(TestDvc):
|
||||
def test(self):
|
||||
self.dvc.run('', [], [], [], [], False, 'empty.dvc', os.curdir)
|
||||
@@ -0,0 +1,32 @@
|
||||
import os
|
||||
|
||||
from git import Repo
|
||||
|
||||
from dvc.scm import SCM, Base, Git
|
||||
|
||||
from tests.basic_env import TestDir, TestGit
|
||||
|
||||
|
||||
class TestSCM(TestDir):
|
||||
def test_none(self):
|
||||
self.assertIsInstance(SCM(self._root_dir), Base)
|
||||
|
||||
def test_git(self):
|
||||
Repo.init(os.curdir)
|
||||
self.assertIsInstance(SCM(self._root_dir), Git)
|
||||
|
||||
|
||||
class TestSCMGit(TestGit):
|
||||
def test_is_repo(self):
|
||||
self.assertTrue(Git.is_repo(os.curdir))
|
||||
|
||||
def test_ignore(self):
|
||||
Git(self._root_dir).ignore('foo')
|
||||
self.assertTrue(os.path.isfile(Git.GITIGNORE))
|
||||
self.assertTrue('foo' in open(Git.GITIGNORE, 'r').readlines())
|
||||
|
||||
def test_commit(self):
|
||||
G = Git(self._root_dir)
|
||||
G.add(['foo'])
|
||||
G.commit('add')
|
||||
self.assertTrue('foo' in self.git.git.ls_files())
|
||||
@@ -0,0 +1,31 @@
|
||||
import os
|
||||
|
||||
from dvc.state import State, StateEntry
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestState(TestDvc):
|
||||
def test_update(self):
|
||||
path = os.path.join(self.dvc.root_dir, '1')
|
||||
md5_1 = '1'
|
||||
md5_2 = '2'
|
||||
mtime_1 = 1
|
||||
mtime_2 = 2
|
||||
|
||||
state = State(self.dvc.root_dir, self.dvc.dvc_dir)
|
||||
self.assertIsNone(state.get(path))
|
||||
|
||||
state.add(path, md5_1, mtime_1)
|
||||
entry = state.get(path)
|
||||
self.assertIsInstance(entry, StateEntry)
|
||||
self.assertEqual(entry.path, path)
|
||||
self.assertEqual(entry.md5, md5_1)
|
||||
self.assertEqual(entry.mtime, mtime_1)
|
||||
|
||||
state.update(path, md5_2, mtime_2)
|
||||
entry = state.get(path)
|
||||
self.assertIsInstance(entry, StateEntry)
|
||||
self.assertEqual(entry.path, path)
|
||||
self.assertEqual(entry.md5, md5_2)
|
||||
self.assertEqual(entry.mtime, mtime_2)
|
||||
+2
-19
@@ -3,27 +3,10 @@ import shutil
|
||||
import filecmp
|
||||
|
||||
from dvc import utils
|
||||
from tests.basic_env import DirHierarchyEnvironment
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
class TestExecutor(DirHierarchyEnvironment):
|
||||
def setUp(self):
|
||||
DirHierarchyEnvironment.init_environment(self)
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
|
||||
def test_rmtree(self):
|
||||
root = 'testdir'
|
||||
|
||||
os.makedirs(root + '/subdir')
|
||||
with open(root + '/file1', 'w+') as f:
|
||||
f.write('file1contents')
|
||||
with open(root + '/subdir/file2', 'w+') as f:
|
||||
f.write('file2contents')
|
||||
|
||||
utils.rmtree(root)
|
||||
self.assertFalse(os.path.exists(root))
|
||||
|
||||
class TestUtils(TestDvc):
|
||||
def test_copyfile(self):
|
||||
src = 'file1'
|
||||
dest = 'file2'
|
||||
|
||||
Reference in New Issue
Block a user