diff --git a/dvc/cli.py b/dvc/cli.py index 3a0379f2a..63d674c75 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -9,6 +9,7 @@ import argparse import dvc.logger as logger from dvc.command.base import fix_subparsers import dvc.command.init as init +import dvc.command.pkg as pkg import dvc.command.destroy as destroy import dvc.command.remove as remove import dvc.command.move as move @@ -36,6 +37,7 @@ from dvc import VERSION COMMANDS = [ init, + pkg, destroy, add, remove, diff --git a/dvc/command/pkg.py b/dvc/command/pkg.py new file mode 100644 index 000000000..e4496c7ca --- /dev/null +++ b/dvc/command/pkg.py @@ -0,0 +1,81 @@ +from __future__ import unicode_literals + +import dvc.logger as logger +from dvc.exceptions import DvcException +from dvc.command.base import CmdBase, fix_subparsers +from dvc.repo.pkg import PackageParams + + +class CmdPkg(CmdBase): + def run(self, unlock=False): + try: + pkg_params = PackageParams( + self.args.address, + self.args.target_dir, + self.args.select, + self.args.file, + ) + return self.repo.install_pkg(pkg_params) + except DvcException: + logger.error( + "failed to install package '{}'".format(self.args.address) + ) + return 1 + pass + + +def add_parser(subparsers, parent_parser): + from dvc.command.config import parent_config_parser + + PKG_HELP = "Manage packages and modules" + pkg_parser = subparsers.add_parser( + "pkg", parents=[parent_parser], description=PKG_HELP, add_help=False + ) + + pkg_subparsers = pkg_parser.add_subparsers( + dest="cmd", help="Use dvc pkg CMD --help for command-specific help." + ) + + fix_subparsers(pkg_subparsers) + + PKG_INSTALL_HELP = "Install package." + pkg_install_parser = pkg_subparsers.add_parser( + "install", + parents=[parent_config_parser, parent_parser], + description=PKG_INSTALL_HELP, + help=PKG_INSTALL_HELP, + ) + pkg_install_parser.add_argument( + "address", + nargs="?", + default="", + help="Package address: git:// or https://github.com/...", + ) + pkg_install_parser.add_argument( + "target_dir", + metavar="target", + nargs="?", + default=".", + help="Target directory to deploy package outputs. " + "Default value is the current dir.", + ) + pkg_install_parser.add_argument( + "-s", + "--select", + metavar="OUT", + action="append", + default=[], + help="Select and persist only specified outputs from a package. " + "The parameter can be used multiple times. " + "All outputs will be selected by default.", + ) + pkg_install_parser.add_argument( + "-f", + "--file", + help="Specify name of the stage file. It should be " + "either 'Dvcfile' or have a '.dvc' suffix (e.g. " + "'prepare.dvc', 'clean.dvc', etc). " + "By default the file has 'mod_' prefix and imported package name " + "followed by .dvc", + ) + pkg_install_parser.set_defaults(func=CmdPkg) diff --git a/dvc/output/base.py b/dvc/output/base.py index 14ad6de4e..67d0917a7 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -84,6 +84,11 @@ class OutputBase(object): "is local is not supported for {}".format(self.scheme) ) + def assign_to_stage_file(self, target_repo): + raise DvcException( + "change repo is not supported for {}".format(self.scheme) + ) + @classmethod def match(cls, url): return re.match(cls.REMOTE.REGEX, url) diff --git a/dvc/output/local.py b/dvc/output/local.py index cb501be69..de320554d 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -42,6 +42,17 @@ class OutputLOCAL(OutputBase): self.url ) + def assign_to_stage_file(self, stage): + from dvc.repo import Repo + + fullpath = os.path.abspath(stage.wdir) + self.path_info["path"] = os.path.join(fullpath, self.stage_path) + + self.repo = Repo(self.path) + + self.stage = stage + return self + @property def sep(self): return os.sep @@ -50,6 +61,10 @@ class OutputLOCAL(OutputBase): def rel_path(self): return os.path.relpath(self.path) + @property + def stage_path(self): + return os.path.relpath(self.path, self.stage.wdir) + @property def cache(self): return self.repo.cache.local.get(self.checksum) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index c667f9f0f..2205eb234 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -28,6 +28,7 @@ class Repo(object): from dvc.repo.status import status from dvc.repo.gc import gc from dvc.repo.commit import commit + from dvc.repo.pkg import install_pkg def __init__(self, root_dir=None): from dvc.config import Config diff --git a/dvc/repo/pkg.py b/dvc/repo/pkg.py new file mode 100644 index 000000000..9b2ffd9e0 --- /dev/null +++ b/dvc/repo/pkg.py @@ -0,0 +1,201 @@ +from __future__ import unicode_literals + +import os +import shutil +from git.cmd import Git + +import dvc.logger as logger +from dvc.exceptions import DvcException +from dvc.stage import Stage +from dvc.temp_git_repo import TempGitRepo + + +class PackageParams(object): + def __init__(self, address, target_dir, select=[], file=None): + self.address = address + self.target_dir = target_dir + self.select = select + self.file = file + + @property + def all_addresses(self): + if self.address: + return [self.address] + return PackageManager.read_packages() + + +class PackageManager(object): + PACKAGE_FILE = "package.yaml" + + @staticmethod + def read_packages(): + return [] + + @staticmethod + def get_package(addr): + for pkg_class in [GitPackage]: + try: + return pkg_class() + except Exception: + pass + return None + + def __init__(self, addr): + self._addr = addr + + +class Package(object): + MODULES_DIR = "dvc_mod" + + def install_or_update(self, parent_repo, pkg_param): + raise NotImplementedError( + "A method of abstract Package class was called" + ) + + def is_in_root(self): + return True + + +class GitPackage(Package): + DEF_DVC_FILE_PREFIX = "mod_" + + def install_or_update(self, parent_repo, pkg_params): + if not self.is_in_root(): + raise DvcException( + "This command can be run only from a repository root" + ) + + if not os.path.exists(self.MODULES_DIR): + logger.debug("Creating modules dir {}".format(self.MODULES_DIR)) + os.makedirs(self.MODULES_DIR) + parent_repo.scm.ignore(os.path.abspath(self.MODULES_DIR)) + + module_name = ( + Git.polish_url(pkg_params.address).strip("/").split("/")[-1] + ) + if not module_name: + raise DvcException( + "Package address error: unable to extract package name" + ) + + with TempGitRepo( + pkg_params.address, module_name, Package.MODULES_DIR + ) as tmp_repo: + outputs_to_copy = tmp_repo.outs + if pkg_params.select: + outputs_to_copy = list( + filter( + lambda out: out.dvc_path in pkg_params.select, + outputs_to_copy, + ) + ) + + fetched_stage_files = set( + map(lambda o: o.stage.path, outputs_to_copy) + ) + tmp_repo.fetch(fetched_stage_files) + + module_dir = self.create_module_dir(module_name) + tmp_repo.persist_to(module_dir, parent_repo) + + dvc_file = self.get_dvc_file_name( + pkg_params.file, pkg_params.target_dir, module_name + ) + try: + self.persist_stage_and_scm_state( + parent_repo, + outputs_to_copy, + pkg_params.target_dir, + dvc_file, + ) + except Exception as ex: + raise DvcException( + "Package '{}' was installed " + "but stage file '{}' " + "was not created properly: {}".format( + pkg_params.address, dvc_file, ex + ) + ) + + parent_repo.checkout(dvc_file) + + @staticmethod + def persist_stage_and_scm_state( + parent_repo, outputs_to_copy, target_dir, dvc_file + ): + stage = Stage.create( + repo=parent_repo, + fname=dvc_file, + validate_state=False, + wdir=target_dir, + ) + stage.outs = list( + map(lambda o: o.assign_to_stage_file(stage), outputs_to_copy) + ) + + for out in stage.outs: + parent_repo.scm.ignore(out.path, in_curr_dir=True) + + stage.dump() + + @staticmethod + def create_module_dir(module_name): + module_dir = os.path.join(GitPackage.MODULES_DIR, module_name) + if os.path.exists(module_dir): + logger.info("Updating package {}".format(module_name)) + shutil.rmtree(module_dir) + else: + logger.info("Adding package {}".format(module_name)) + return module_dir + + def get_dvc_file_name(self, stage_file, target_dir, module_name): + if stage_file: + dvc_file_path = stage_file + else: + dvc_file_name = self.DEF_DVC_FILE_PREFIX + module_name + ".dvc" + dvc_file_path = os.path.join(target_dir, dvc_file_name) + return dvc_file_path + + +def install_pkg(self, pkg_params): + """ + Install package. + + The command can be run only from DVC project root. + + E.g. + Having: DVC package in https://github.com/dmpetrov/tag_classifier + + $ dvc pkg install https://github.com/dmpetrov/tag_classifier + + Result: tag_classifier package in dvc_mod/ directory + """ + + if not os.path.isdir(pkg_params.target_dir): + logger.error( + "Unable to install package: " + "target directory '{}' does not exist".format( + pkg_params.target_dir + ) + ) + return 1 + + curr_dir = os.path.realpath(os.curdir) + if not os.path.realpath(pkg_params.target_dir).startswith(curr_dir): + logger.error( + "Unable to install package: the current dir should be" + " a subdirectory of the target dir {}".format( + pkg_params.target_dir + ) + ) + return 1 + + for addr in pkg_params.all_addresses: + try: + mgr = PackageManager.get_package(addr) + mgr.install_or_update(self, pkg_params) + except Exception as ex: + logger.error("Unable to install package: ".format(ex)) + return 1 + + return 0 diff --git a/dvc/scm/base.py b/dvc/scm/base.py index 48031ec3e..eb957f030 100644 --- a/dvc/scm/base.py +++ b/dvc/scm/base.py @@ -17,12 +17,17 @@ class FileNotInRepoError(DvcException): """ +class FileNotInTargetSubdirError(DvcException): + """Thrown when trying to place .gitignore for a file that not in + the file subdirectory.""" + + class Base(object): """Base class for source control management driver implementations.""" def __init__(self, root_dir=os.curdir, repo=None): self.repo = repo - self.root_dir = root_dir + self.root_dir = os.path.realpath(root_dir) def __repr__(self): return "{class_name}: '{directory}'".format( diff --git a/dvc/scm/git.py b/dvc/scm/git.py index f60c2cfb9..4000f92bd 100644 --- a/dvc/scm/git.py +++ b/dvc/scm/git.py @@ -12,7 +12,12 @@ import os import dvc.logger as logger from dvc.utils import fix_env -from dvc.scm.base import Base, SCMError, FileNotInRepoError +from dvc.scm.base import ( + Base, + SCMError, + FileNotInRepoError, + FileNotInTargetSubdirError, +) class Git(Base): @@ -62,28 +67,47 @@ class Git(Base): def ignore_file(self): return self.GITIGNORE - def _get_gitignore(self, path): - assert os.path.isabs(path) - # NOTE: using '/' prefix to make path unambiguous - entry = "/" + os.path.basename(path) - gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE) + def _get_gitignore(self, path, ignore_file_dir=None): + if not ignore_file_dir: + ignore_file_dir = os.path.dirname(os.path.realpath(path)) - if not gitignore.startswith(self.root_dir): + assert os.path.isabs(path) + assert os.path.isabs(ignore_file_dir) + + if not path.startswith(ignore_file_dir): + msg = ( + "{} file has to be located in one of '{}' subdirectories" + ", not outside '{}'" + ) + raise FileNotInTargetSubdirError( + msg.format(self.GITIGNORE, path, ignore_file_dir) + ) + + entry = os.path.relpath(path, ignore_file_dir).replace(os.sep, "/") + # NOTE: using '/' prefix to make path unambiguous + if len(entry) > 0 and entry[0] != "/": + entry = "/" + entry + + gitignore = os.path.join(ignore_file_dir, self.GITIGNORE) + + if not gitignore.startswith(os.path.realpath(self.root_dir)): raise FileNotInRepoError(path) return entry, gitignore - def ignore(self, path): - entry, gitignore = self._get_gitignore(path) + def ignore(self, path, in_curr_dir=False): + base_dir = ( + os.path.realpath(os.curdir) + if in_curr_dir + else os.path.dirname(path) + ) + entry, gitignore = self._get_gitignore(path, base_dir) ignore_list = [] if os.path.exists(gitignore): with open(gitignore, "r") as f: ignore_list = f.readlines() - filtered = list( - filter(lambda x: x.strip() == entry.strip(), ignore_list) - ) - if filtered: + if any(filter(lambda x: x.strip() == entry.strip(), ignore_list)): return msg = "Adding '{}' to '{}'.".format( @@ -91,17 +115,20 @@ class Git(Base): ) logger.info(msg) - content = entry - if ignore_list: - content = "\n" + content - - with open(gitignore, "a") as fobj: - fobj.write(content) + self._add_entry_to_gitignore(entry, gitignore, ignore_list) self.track_file(os.path.relpath(gitignore)) self.ignored_paths.append(path) + @staticmethod + def _add_entry_to_gitignore(entry, gitignore, ignore_list): + content = entry + if ignore_list: + content = "\n" + content + with open(gitignore, "a") as fobj: + fobj.write(content) + def ignore_remove(self, path): entry, gitignore = self._get_gitignore(path) diff --git a/dvc/stage.py b/dvc/stage.py index 83e46dc8b..3d4760058 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -396,6 +396,7 @@ class Stage(object): overwrite=True, ignore_build_cache=False, remove_outs=False, + validate_state=True, ): if outs is None: outs = [] @@ -458,20 +459,21 @@ class Stage(object): else: stage.unprotect_outs() - if os.path.exists(path): - if not ignore_build_cache and stage.is_cached: - logger.info("Stage is cached, skipping.") - return None + if validate_state: + if os.path.exists(path): + if not ignore_build_cache and stage.is_cached: + logger.info("Stage is cached, skipping.") + return None - msg = ( - "'{}' already exists. Do you wish to run the command and " - "overwrite it?".format(stage.relpath) - ) + msg = ( + "'{}' already exists. Do you wish to run the command and " + "overwrite it?".format(stage.relpath) + ) - if not overwrite and not prompt.confirm(msg): - raise StageFileAlreadyExistsError(stage.relpath) + if not overwrite and not prompt.confirm(msg): + raise StageFileAlreadyExistsError(stage.relpath) - os.unlink(path) + os.unlink(path) return stage diff --git a/dvc/temp_git_repo.py b/dvc/temp_git_repo.py new file mode 100644 index 000000000..6f2c90875 --- /dev/null +++ b/dvc/temp_git_repo.py @@ -0,0 +1,145 @@ +import os +import shutil +import tempfile + +import git + +from dvc import logger +from dvc.exceptions import DvcException + + +class TempRepoException(DvcException): + """Raise when temporary package repository has not properly created""" + + def __init__(self, temp_repo, msg, cause=None): + m = "temp repository '{}' error: {}".format(temp_repo.addr, msg) + super(TempRepoException, self).__init__(m, cause) + + +class TempGitRepo(object): + GIT_DIR_TO_REMOVE = ".git" + GIT_FILES_TO_REMOVE = [".gitignore", ".gitmodules"] + SUFFIX = "tmp_DVC_mod" + + def __init__(self, addr, module_name, modules_dir): + self.addr = addr + self.modules_dir = modules_dir + self._tmp_mod_prefix = module_name + "_" + self.SUFFIX + "_" + self._reset_state() + + def _reset_state(self): + self._set_state(None, []) + + def _set_state(self, cloned_tmp_dir, outs): + from dvc.repo import Repo + + self.repo = Repo(cloned_tmp_dir) if cloned_tmp_dir else None + + self._cloned_tmp_dir = cloned_tmp_dir + self.outs = outs + self._moved_files = [] + + def fetch(self, targets): + for target in targets: + try: + self.repo.fetch(target) + except Exception as ex: + msg = "error in fetching data from {}: {}".format( + os.path.basename(target), ex + ) + raise TempRepoException(self, msg) + + @property + def is_state_set(self): + return self._cloned_tmp_dir is not None + + def __enter__(self): + if self.is_state_set: + raise TempRepoException(self, "Git repo cloning duplication") + + module_temp_dir = None + try: + module_temp_dir = self._clone_to_temp_dir() + self._remove_git_files_and_dirs(module_temp_dir) + + outputs = self._read_outputs(module_temp_dir) + self._set_state(module_temp_dir, outputs) + finally: + self._clean_mod_temp_dir(module_temp_dir) + + return self + + def _clone_to_temp_dir(self): + result = tempfile.mktemp( + prefix=self._tmp_mod_prefix, dir=self.modules_dir + ) + logger.debug( + "Cloning git repository {} to temp dir {}".format( + self.addr, result + ) + ) + git.Repo.clone_from(self.addr, result, depth=1) + return result + + def _remove_git_files_and_dirs(self, module_temp_dir): + logger.debug( + "Removing git meta files from {}: {} dif and {}".format( + module_temp_dir, + TempGitRepo.GIT_DIR_TO_REMOVE, + ", ".join(TempGitRepo.GIT_FILES_TO_REMOVE), + ) + ) + shutil.rmtree( + os.path.join(module_temp_dir, TempGitRepo.GIT_DIR_TO_REMOVE) + ) + for item in TempGitRepo.GIT_FILES_TO_REMOVE: + fname = os.path.join(module_temp_dir, item) + if os.path.exists(fname): + os.remove(fname) + + def _clean_mod_temp_dir(self, module_temp_dir): + if not self.is_state_set: + if module_temp_dir and os.path.exists(module_temp_dir): + shutil.rmtree(module_temp_dir, ignore_errors=True) + + def _read_outputs(self, module_temp_dir): + from dvc.repo import Repo + + pkg_repo = Repo(module_temp_dir) + stages = pkg_repo.stages() + return [out for s in stages for out in s.outs] + + def persist_to(self, module_dir, parent_repo): + if not self.is_state_set: + raise TempRepoException(self, "cannot persist") + + tmp_repo_cache = self.repo.cache.local.url + + for prefix in os.listdir(tmp_repo_cache): + if len(prefix) != 2: + logger.warning( + "wrong dir format in cache {}: dir {}".format( + tmp_repo_cache, prefix + ) + ) + self._move_all_cache_files(parent_repo, prefix, tmp_repo_cache) + + shutil.move(self._cloned_tmp_dir, module_dir) + self._reset_state() + + @staticmethod + def _move_all_cache_files(parent_repo, prefix, tmp_repo_cache): + obj_name = os.path.join(tmp_repo_cache, prefix) + for suffix in os.listdir(obj_name): + src_path = os.path.join(tmp_repo_cache, prefix, suffix) + pre = os.path.join(parent_repo.cache.local.url, prefix) + if not os.path.exists(pre): + os.mkdir(pre) + dest = os.path.join(pre, suffix) + shutil.move(src_path, dest) + + def __exit__(self, exc_type, exc_value, traceback): + if self.is_state_set: + shutil.rmtree(self._cloned_tmp_dir) + self._reset_state() + pass diff --git a/tests/basic_env.py b/tests/basic_env.py index c38e9bb74..05416178d 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -1,4 +1,5 @@ import os +import shutil import uuid import tempfile @@ -7,10 +8,11 @@ from git.exc import GitCommandNotFound from unittest import TestCase import dvc.logger as logger +from dvc.command.remote import CmdRemoteAdd from dvc.repo import Repo as DvcRepo -class TestDir(TestCase): +class TestDirFixture(object): GCP_CREDS_FILE = os.path.abspath( os.path.join("scripts", "ci", "gcp-creds.json") ) @@ -39,6 +41,13 @@ class TestDir(TestCase): "shutil.copyfile(sys.argv[1], sys.argv[2])" ) + def __init__(self, root_dir=None): + if root_dir: + os.mkdir(root_dir) + else: + root_dir = self.mkdtemp() + self._root_dir = os.path.realpath(root_dir) + def _pushd(self, d): self._saved_dir = os.path.realpath(os.curdir) os.chdir(d) @@ -62,8 +71,6 @@ class TestDir(TestCase): return tempfile.mkdtemp(prefix=prefix, suffix=suffix) def setUp(self): - self._root_dir = TestDir.mkdtemp() - self._pushd(self._root_dir) self.create(self.FOO, self.FOO_CONTENTS) self.create(self.BAR, self.BAR_CONTENTS) @@ -77,11 +84,11 @@ class TestDir(TestCase): self._popd() -class TestGit(TestDir): +class TestGitFixture(TestDirFixture): N_RETRIES = 5 def setUp(self): - super(TestGit, self).setUp() + super(TestGitFixture, self).setUp() # NOTE: handles EAGAIN error on BSD systems (osx in our case). # Otherwise when running tests you might get this exception: # @@ -100,17 +107,117 @@ class TestGit(TestDir): self.git.index.commit("add code") -class TestGitSubmodule(TestGit): +class TestGitSubmoduleFixture(TestGitFixture): + def __init__(self, root_dir=None): + super(TestGitSubmoduleFixture, self).__init__(root_dir) + def setUp(self): - super(TestGitSubmodule, self).setUp() + super(TestGitSubmoduleFixture, self).setUp() subrepo = Repo.init() subrepo_path = "subrepo" self.git.create_submodule(subrepo_path, subrepo_path, subrepo.git_dir) self._pushd(subrepo_path) -class TestDvc(TestGit): +class TestDvcFixture(TestGitFixture): + def __init__(self, root_dir=None): + super(TestDvcFixture, self).__init__(root_dir) + def setUp(self): - super(TestDvc, self).setUp() + super(TestDvcFixture, self).setUp() self.dvc = DvcRepo.init(self._root_dir) logger.be_verbose() + + +class TestDvcGitInitializedFixture(TestDvcFixture): + def __init__(self, root_dir=None): + super(TestDvcGitInitializedFixture, self).__init__(root_dir) + + def setUp(self): + super(TestDvcGitInitializedFixture, self).setUp() + self.git.init() + + +class TestDvcDataFileFixture(TestDvcGitInitializedFixture): + DATA_DVC_FILE = "data.dvc" + DATA_DIR_DVC_FILE = "data_sub_dir.dvc" + REMOTE = "myremote2" + + def __init__(self, root_dir=None, cache_dir=None): + super(TestDvcDataFileFixture, self).__init__(root_dir) + self.cache_dir = cache_dir + + def setUp(self): + super(TestDvcDataFileFixture, self).setUp() + + self.dvc.add(self.DATA) + self.dvc.add(self.DATA_SUB_DIR) + + if self.cache_dir: + shutil.copytree(self.dvc.cache.local.cache_dir, self.cache_dir) + + class MockConfig: + system = None + glob = None + local = None + default = True + name = self.REMOTE + url = self.cache_dir + + cmd = CmdRemoteAdd(MockConfig()) + cmd.run() + + self.git.index.add( + [ + os.path.join(self.DATA_DIR, self.DATA_DVC_FILE), + os.path.join(self.DATA_DIR, self.DATA_DIR_DVC_FILE), + os.path.join(self.DATA_DIR, ".gitignore"), + ".dvc/config", + self.FOO, + self.BAR, + ] + ) + self.git.index.commit("Hello world commit") + + +# NOTE: Inheritance order in the classes below is important. + + +class TestDir(TestDirFixture, TestCase): + def __init__(self, methodName, root_dir=None): + TestDirFixture.__init__(self, root_dir) + TestCase.__init__(self, methodName) + + +class TestGit(TestGitFixture, TestCase): + def __init__(self, methodName, root_dir=None): + TestGitFixture.__init__(self, root_dir) + TestCase.__init__(self, methodName) + + +class TestGitSubmodule(TestGitSubmoduleFixture, TestCase): + def __init__(self, methodName, root_dir=None): + TestGitSubmoduleFixture.__init__(self, root_dir) + TestCase.__init__(self, methodName) + + +class TestDvc(TestDvcFixture, TestCase): + def __init__(self, methodName, root_dir=None): + TestDvcFixture.__init__(self, root_dir) + TestCase.__init__(self, methodName) + + +class TestDvcPkg(TestDvcFixture, TestCase): + GIT_PKG = "git_pkg" + CACHE_DIR = "mycache" + + def __init__(self, methodName, root_dir=None): + TestDvcFixture.__init__(self, root_dir) + TestCase.__init__(self, methodName) + + self.pkg_dir = os.path.join(self._root_dir, self.GIT_PKG) + cache_dir = os.path.join(self._root_dir, self.CACHE_DIR) + self.pkg_fixture = TestDvcDataFileFixture( + root_dir=self.pkg_dir, cache_dir=cache_dir + ) + self.pkg_fixture.setUp() diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 200250d00..4a739fbec 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -675,7 +675,7 @@ class TestRecursiveSyncOperations(TestDataCloudBase): ) self.dvc.add(self.DATA) - self.dvc.add(self.DATA_SUB_CONTENTS) + self.dvc.add(self.DATA_SUB) def _remove_local_data_files(self): os.remove(self.DATA) diff --git a/tests/test_move.py b/tests/test_move.py index c135cf16a..7b5398a51 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -173,13 +173,16 @@ class TestMoveFileBetweenDirectories(TestDvc): class TestMoveFileInsideDirectory(TestDvc): def test(self): - ret = main(["add", "data_dir/data"]) + ret = main(["add", self.DATA]) self.assertEqual(ret, 0) - with cd("data_dir"): - ret = main(["move", "data", "data.txt"]) + with cd(self.DATA_DIR): + ret = main(["move", os.path.basename(self.DATA), "data.txt"]) self.assertEqual(ret, 0) - self.assertFalse(os.path.exists("data_dir/data")) - self.assertTrue(os.path.exists("data_dir/data.txt")) - self.assertTrue(os.path.exists("data_dir/data.txt.dvc")) + self.assertFalse(os.path.exists(self.DATA)) + + data_fullpath = os.path.join(self.DATA_DIR, "data.txt") + dvc_fullpath = os.path.join(self.DATA_DIR, "data.txt.dvc") + self.assertTrue(os.path.exists(data_fullpath)) + self.assertTrue(os.path.exists(dvc_fullpath)) diff --git a/tests/test_pkg.py b/tests/test_pkg.py new file mode 100644 index 000000000..5a29b6c49 --- /dev/null +++ b/tests/test_pkg.py @@ -0,0 +1,37 @@ +import os + +from dvc.repo.pkg import Package, PackageParams +from tests.basic_env import TestDvcPkg + + +class TestPkgBasic(TestDvcPkg): + NEW_PKG_DIR = "new_pkg" + + def setUp(self): + super(TestPkgBasic, self).setUp() + + self.to_dir = os.path.join(self._root_dir, self.NEW_PKG_DIR) + os.mkdir(self.to_dir) + self.dvc.install_pkg( + PackageParams(self.pkg_fixture._root_dir, self.to_dir) + ) + + def test_installed_dirs_and_files(self): + self.assertTrue(os.path.isdir(self.pkg_dir)) + + mod_dir_fullpath = os.path.join(self._root_dir, Package.MODULES_DIR) + self.assertTrue(os.path.isdir(mod_dir_fullpath)) + + installed_pkg_dir = os.path.join(mod_dir_fullpath, self.GIT_PKG) + for file in [self.FOO, self.BAR, self.CODE]: + installed_file = os.path.join(installed_pkg_dir, file) + self.assertTrue(os.path.isfile(installed_file)) + + def test_target_dir_checkout(self): + pkg_data_file = os.path.join(self.to_dir, self.DATA) + self.assertTrue(os.path.exists(pkg_data_file)) + self.assertTrue(os.path.isfile(pkg_data_file)) + + pkg_data_dir = os.path.join(self.to_dir, self.DATA_SUB_DIR) + self.assertTrue(os.path.exists(pkg_data_dir)) + self.assertTrue(os.path.isdir(pkg_data_dir)) diff --git a/tests/test_scm.py b/tests/test_scm.py index fd2a2a622..7bf7e3726 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -1,8 +1,11 @@ -import os +from __future__ import unicode_literals +import os from git import Repo +from dvc.utils.compat import str # noqa: F401 from dvc.scm import SCM, Base, Git +from dvc.scm.base import FileNotInTargetSubdirError from tests.basic_env import TestDir, TestGit, TestGitSubmodule from tests.utils import get_gitignore_content @@ -48,7 +51,7 @@ class TestIgnore(TestGit): return len(list(filter(lambda x: x.strip() == "/" + self.FOO, lines))) - def test(self): + def test_ignore(self): git = Git(self._root_dir) foo = os.path.join(self._root_dir, self.FOO) @@ -61,3 +64,62 @@ class TestIgnore(TestGit): git.ignore_remove(foo) self.assertEqual(self._count_gitignore(), 0) + + def test_get_gitignore(self): + data_dir = os.path.join(self._root_dir, "file1") + entry, gitignore = Git(self._root_dir)._get_gitignore(data_dir) + self.assertEqual(entry, "/file1") + self.assertEqual( + gitignore, os.path.join(self._root_dir, Git.GITIGNORE) + ) + + data_dir = os.path.join(self._root_dir, "dir") + entry, gitignore = Git(self._root_dir)._get_gitignore(data_dir) + self.assertEqual(entry, "/dir") + self.assertEqual( + gitignore, os.path.join(self._root_dir, Git.GITIGNORE) + ) + + def test_get_gitignore_subdir(self): + data_dir = os.path.join(self._root_dir, os.path.join("dir1", "file1")) + entry, gitignore = Git(self._root_dir)._get_gitignore(data_dir) + self.assertEqual(entry, "/file1") + self.assertEqual( + gitignore, os.path.join(self._root_dir, "dir1", Git.GITIGNORE) + ) + + data_dir = os.path.join(self._root_dir, os.path.join("dir1", "dir2")) + entry, gitignore = Git(self._root_dir)._get_gitignore(data_dir) + self.assertEqual(entry, "/dir2") + self.assertEqual( + gitignore, os.path.join(self._root_dir, "dir1", Git.GITIGNORE) + ) + + def test_get_gitignore_ignorefile_dir(self): + git = Git(self._root_dir) + + file_double_dir = os.path.join("dir1", "dir2", "file1") + data_dir1 = os.path.join(self._root_dir, file_double_dir) + dir1_real1 = os.path.realpath("dir1") + entry, gitignore = git._get_gitignore(data_dir1, dir1_real1) + self.assertEqual(entry, "/dir2/file1") + gitignore1 = os.path.join(self._root_dir, "dir1", Git.GITIGNORE) + self.assertEqual(gitignore, gitignore1) + + triple_dir = os.path.join("dir1", "dir2", "dir3") + data_dir2 = os.path.join(self._root_dir, triple_dir) + dir1_real2 = os.path.realpath("dir1") + entry, gitignore = git._get_gitignore(data_dir2, dir1_real2) + self.assertEqual(entry, "/dir2/dir3") + gitignore2 = os.path.join(self._root_dir, "dir1", Git.GITIGNORE) + self.assertEqual(gitignore, gitignore2) + + def test_get_gitignore_ignorefile_dir_upper_level(self): + git = Git(self._root_dir) + + file_double_dir = os.path.join("dir1", "dir2", "file1") + data_dir1 = os.path.join(self._root_dir, file_double_dir) + ignore_file_dir = os.path.realpath(os.path.join("aa", "bb")) + + with self.assertRaises(FileNotInTargetSubdirError): + git._get_gitignore(data_dir1, ignore_file_dir)