[WIP] Pkg install (#1742)

* Download Git package

* Extract tmp git repo & fetch data

* Handle fetch\cloud errors and rename select option

* Modify gitignore files when deploy a package

* More tests for git ignore file. Fix an old issue.

* Raise and exception when .gitignor not in a file subdir

* Existing tests fixes

* Basic test for packages

* Comment out pkg tests

* Codeclimate improvements

* Reformatting

* Enable pkg tests

* Fix tests no Windows

* Another old bug fix

* Fix encoding issue on Windows

* Another windows fix: apply compat.str magic to file write

* Back to posix path in gitignore and apply compat.str magic to tests

* gitignorepath: OS specifix path to posix

* Forgotten assignment

* misc: cleanup small issues

Signed-off-by: Ruslan Kuprieiev <[email protected]>

* cli: pkg: hide from help

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/cli.py

@@ -9,6 +9,7 @@ import argparse
 import dvc.logger as logger
 from dvc.command.base import fix_subparsers
 import dvc.command.init as init
+import dvc.command.pkg as pkg
 import dvc.command.destroy as destroy
 import dvc.command.remove as remove
 import dvc.command.move as move
@@ -36,6 +37,7 @@ from dvc import VERSION
 
 COMMANDS = [
     init,
+    pkg,
     destroy,
     add,
     remove,

dvc/command/pkg.py

@@ -0,0 +1,81 @@
+from __future__ import unicode_literals
+
+import dvc.logger as logger
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, fix_subparsers
+from dvc.repo.pkg import PackageParams
+
+
+class CmdPkg(CmdBase):
+    def run(self, unlock=False):
+        try:
+            pkg_params = PackageParams(
+                self.args.address,
+                self.args.target_dir,
+                self.args.select,
+                self.args.file,
+            )
+            return self.repo.install_pkg(pkg_params)
+        except DvcException:
+            logger.error(
+                "failed to install package '{}'".format(self.args.address)
+            )
+            return 1
+        pass
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    PKG_HELP = "Manage packages and modules"
+    pkg_parser = subparsers.add_parser(
+        "pkg", parents=[parent_parser], description=PKG_HELP, add_help=False
+    )
+
+    pkg_subparsers = pkg_parser.add_subparsers(
+        dest="cmd", help="Use dvc pkg CMD --help for command-specific help."
+    )
+
+    fix_subparsers(pkg_subparsers)
+
+    PKG_INSTALL_HELP = "Install package."
+    pkg_install_parser = pkg_subparsers.add_parser(
+        "install",
+        parents=[parent_config_parser, parent_parser],
+        description=PKG_INSTALL_HELP,
+        help=PKG_INSTALL_HELP,
+    )
+    pkg_install_parser.add_argument(
+        "address",
+        nargs="?",
+        default="",
+        help="Package address: git://<url> or https://github.com/...",
+    )
+    pkg_install_parser.add_argument(
+        "target_dir",
+        metavar="target",
+        nargs="?",
+        default=".",
+        help="Target directory to deploy package outputs. "
+        "Default value is the current dir.",
+    )
+    pkg_install_parser.add_argument(
+        "-s",
+        "--select",
+        metavar="OUT",
+        action="append",
+        default=[],
+        help="Select and persist only specified outputs from a package. "
+        "The parameter can be used multiple times. "
+        "All outputs will be selected by default.",
+    )
+    pkg_install_parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the stage file. It should be "
+        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+        "'prepare.dvc', 'clean.dvc', etc). "
+        "By default the file has 'mod_' prefix and imported package name "
+        "followed by .dvc",
+    )
+    pkg_install_parser.set_defaults(func=CmdPkg)

dvc/output/base.py

@@ -84,6 +84,11 @@ class OutputBase(object):
             "is local is not supported for {}".format(self.scheme)
         )
 
+    def assign_to_stage_file(self, target_repo):
+        raise DvcException(
+            "change repo is not supported for {}".format(self.scheme)
+        )
+
     @classmethod
     def match(cls, url):
         return re.match(cls.REMOTE.REGEX, url)

dvc/output/local.py

@@ -42,6 +42,17 @@ class OutputLOCAL(OutputBase):
             self.url
         )
 
+    def assign_to_stage_file(self, stage):
+        from dvc.repo import Repo
+
+        fullpath = os.path.abspath(stage.wdir)
+        self.path_info["path"] = os.path.join(fullpath, self.stage_path)
+
+        self.repo = Repo(self.path)
+
+        self.stage = stage
+        return self
+
     @property
     def sep(self):
         return os.sep
@@ -50,6 +61,10 @@ class OutputLOCAL(OutputBase):
     def rel_path(self):
         return os.path.relpath(self.path)
 
+    @property
+    def stage_path(self):
+        return os.path.relpath(self.path, self.stage.wdir)
+
     @property
     def cache(self):
         return self.repo.cache.local.get(self.checksum)

dvc/repo/__init__.py

@@ -28,6 +28,7 @@ class Repo(object):
     from dvc.repo.status import status
     from dvc.repo.gc import gc
     from dvc.repo.commit import commit
+    from dvc.repo.pkg import install_pkg
 
     def __init__(self, root_dir=None):
         from dvc.config import Config

dvc/repo/pkg.py

@@ -0,0 +1,201 @@
+from __future__ import unicode_literals
+
+import os
+import shutil
+from git.cmd import Git
+
+import dvc.logger as logger
+from dvc.exceptions import DvcException
+from dvc.stage import Stage
+from dvc.temp_git_repo import TempGitRepo
+
+
+class PackageParams(object):
+    def __init__(self, address, target_dir, select=[], file=None):
+        self.address = address
+        self.target_dir = target_dir
+        self.select = select
+        self.file = file
+
+    @property
+    def all_addresses(self):
+        if self.address:
+            return [self.address]
+        return PackageManager.read_packages()
+
+
+class PackageManager(object):
+    PACKAGE_FILE = "package.yaml"
+
+    @staticmethod
+    def read_packages():
+        return []
+
+    @staticmethod
+    def get_package(addr):
+        for pkg_class in [GitPackage]:
+            try:
+                return pkg_class()
+            except Exception:
+                pass
+        return None
+
+    def __init__(self, addr):
+        self._addr = addr
+
+
+class Package(object):
+    MODULES_DIR = "dvc_mod"
+
+    def install_or_update(self, parent_repo, pkg_param):
+        raise NotImplementedError(
+            "A method of abstract Package class was called"
+        )
+
+    def is_in_root(self):
+        return True
+
+
+class GitPackage(Package):
+    DEF_DVC_FILE_PREFIX = "mod_"
+
+    def install_or_update(self, parent_repo, pkg_params):
+        if not self.is_in_root():
+            raise DvcException(
+                "This command can be run only from a repository root"
+            )
+
+        if not os.path.exists(self.MODULES_DIR):
+            logger.debug("Creating modules dir {}".format(self.MODULES_DIR))
+            os.makedirs(self.MODULES_DIR)
+            parent_repo.scm.ignore(os.path.abspath(self.MODULES_DIR))
+
+        module_name = (
+            Git.polish_url(pkg_params.address).strip("/").split("/")[-1]
+        )
+        if not module_name:
+            raise DvcException(
+                "Package address error: unable to extract package name"
+            )
+
+        with TempGitRepo(
+            pkg_params.address, module_name, Package.MODULES_DIR
+        ) as tmp_repo:
+            outputs_to_copy = tmp_repo.outs
+            if pkg_params.select:
+                outputs_to_copy = list(
+                    filter(
+                        lambda out: out.dvc_path in pkg_params.select,
+                        outputs_to_copy,
+                    )
+                )
+
+            fetched_stage_files = set(
+                map(lambda o: o.stage.path, outputs_to_copy)
+            )
+            tmp_repo.fetch(fetched_stage_files)
+
+            module_dir = self.create_module_dir(module_name)
+            tmp_repo.persist_to(module_dir, parent_repo)
+
+            dvc_file = self.get_dvc_file_name(
+                pkg_params.file, pkg_params.target_dir, module_name
+            )
+            try:
+                self.persist_stage_and_scm_state(
+                    parent_repo,
+                    outputs_to_copy,
+                    pkg_params.target_dir,
+                    dvc_file,
+                )
+            except Exception as ex:
+                raise DvcException(
+                    "Package '{}' was installed "
+                    "but stage file '{}' "
+                    "was not created properly: {}".format(
+                        pkg_params.address, dvc_file, ex
+                    )
+                )
+
+        parent_repo.checkout(dvc_file)
+
+    @staticmethod
+    def persist_stage_and_scm_state(
+        parent_repo, outputs_to_copy, target_dir, dvc_file
+    ):
+        stage = Stage.create(
+            repo=parent_repo,
+            fname=dvc_file,
+            validate_state=False,
+            wdir=target_dir,
+        )
+        stage.outs = list(
+            map(lambda o: o.assign_to_stage_file(stage), outputs_to_copy)
+        )
+
+        for out in stage.outs:
+            parent_repo.scm.ignore(out.path, in_curr_dir=True)
+
+        stage.dump()
+
+    @staticmethod
+    def create_module_dir(module_name):
+        module_dir = os.path.join(GitPackage.MODULES_DIR, module_name)
+        if os.path.exists(module_dir):
+            logger.info("Updating package {}".format(module_name))
+            shutil.rmtree(module_dir)
+        else:
+            logger.info("Adding package {}".format(module_name))
+        return module_dir
+
+    def get_dvc_file_name(self, stage_file, target_dir, module_name):
+        if stage_file:
+            dvc_file_path = stage_file
+        else:
+            dvc_file_name = self.DEF_DVC_FILE_PREFIX + module_name + ".dvc"
+            dvc_file_path = os.path.join(target_dir, dvc_file_name)
+        return dvc_file_path
+
+
+def install_pkg(self, pkg_params):
+    """
+    Install package.
+
+    The command can be run only from DVC project root.
+
+    E.g.
+          Having: DVC package in https://github.com/dmpetrov/tag_classifier
+
+          $ dvc pkg install https://github.com/dmpetrov/tag_classifier
+
+          Result: tag_classifier package in dvc_mod/ directory
+    """
+
+    if not os.path.isdir(pkg_params.target_dir):
+        logger.error(
+            "Unable to install package: "
+            "target directory '{}' does not exist".format(
+                pkg_params.target_dir
+            )
+        )
+        return 1
+
+    curr_dir = os.path.realpath(os.curdir)
+    if not os.path.realpath(pkg_params.target_dir).startswith(curr_dir):
+        logger.error(
+            "Unable to install package: the current dir should be"
+            " a subdirectory of the target dir {}".format(
+                pkg_params.target_dir
+            )
+        )
+        return 1
+
+    for addr in pkg_params.all_addresses:
+        try:
+            mgr = PackageManager.get_package(addr)
+            mgr.install_or_update(self, pkg_params)
+        except Exception as ex:
+            logger.error("Unable to install package: ".format(ex))
+            return 1
+
+    return 0

dvc/scm/base.py

@@ -17,12 +17,17 @@ class FileNotInRepoError(DvcException):
     """
 
 
+class FileNotInTargetSubdirError(DvcException):
+    """Thrown when trying to place .gitignore for a file that not in
+    the file subdirectory."""
+
+
 class Base(object):
     """Base class for source control management driver implementations."""
 
     def __init__(self, root_dir=os.curdir, repo=None):
         self.repo = repo
-        self.root_dir = root_dir
+        self.root_dir = os.path.realpath(root_dir)
 
     def __repr__(self):
         return "{class_name}: '{directory}'".format(

dvc/scm/git.py

@@ -12,7 +12,12 @@ import os
 
 import dvc.logger as logger
 from dvc.utils import fix_env
-from dvc.scm.base import Base, SCMError, FileNotInRepoError
+from dvc.scm.base import (
+    Base,
+    SCMError,
+    FileNotInRepoError,
+    FileNotInTargetSubdirError,
+)
 
 
 class Git(Base):
@@ -62,28 +67,47 @@ class Git(Base):
     def ignore_file(self):
         return self.GITIGNORE
 
-    def _get_gitignore(self, path):
-        assert os.path.isabs(path)
-        # NOTE: using '/' prefix to make path unambiguous
-        entry = "/" + os.path.basename(path)
-        gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE)
+    def _get_gitignore(self, path, ignore_file_dir=None):
+        if not ignore_file_dir:
+            ignore_file_dir = os.path.dirname(os.path.realpath(path))
 
-        if not gitignore.startswith(self.root_dir):
+        assert os.path.isabs(path)
+        assert os.path.isabs(ignore_file_dir)
+
+        if not path.startswith(ignore_file_dir):
+            msg = (
+                "{} file has to be located in one of '{}' subdirectories"
+                ", not outside '{}'"
+            )
+            raise FileNotInTargetSubdirError(
+                msg.format(self.GITIGNORE, path, ignore_file_dir)
+            )
+
+        entry = os.path.relpath(path, ignore_file_dir).replace(os.sep, "/")
+        # NOTE: using '/' prefix to make path unambiguous
+        if len(entry) > 0 and entry[0] != "/":
+            entry = "/" + entry
+
+        gitignore = os.path.join(ignore_file_dir, self.GITIGNORE)
+
+        if not gitignore.startswith(os.path.realpath(self.root_dir)):
             raise FileNotInRepoError(path)
 
         return entry, gitignore
 
-    def ignore(self, path):
-        entry, gitignore = self._get_gitignore(path)
+    def ignore(self, path, in_curr_dir=False):
+        base_dir = (
+            os.path.realpath(os.curdir)
+            if in_curr_dir
+            else os.path.dirname(path)
+        )
+        entry, gitignore = self._get_gitignore(path, base_dir)
 
         ignore_list = []
         if os.path.exists(gitignore):
             with open(gitignore, "r") as f:
                 ignore_list = f.readlines()
-            filtered = list(
-                filter(lambda x: x.strip() == entry.strip(), ignore_list)
-            )
-            if filtered:
+            if any(filter(lambda x: x.strip() == entry.strip(), ignore_list)):
                 return
 
         msg = "Adding '{}' to '{}'.".format(
@@ -91,17 +115,20 @@ class Git(Base):
         )
         logger.info(msg)
 
-        content = entry
-        if ignore_list:
-            content = "\n" + content
-
-        with open(gitignore, "a") as fobj:
-            fobj.write(content)
+        self._add_entry_to_gitignore(entry, gitignore, ignore_list)
 
         self.track_file(os.path.relpath(gitignore))
 
         self.ignored_paths.append(path)
 
+    @staticmethod
+    def _add_entry_to_gitignore(entry, gitignore, ignore_list):
+        content = entry
+        if ignore_list:
+            content = "\n" + content
+        with open(gitignore, "a") as fobj:
+            fobj.write(content)
+
     def ignore_remove(self, path):
         entry, gitignore = self._get_gitignore(path)
 

dvc/stage.py

@@ -396,6 +396,7 @@ class Stage(object):
         overwrite=True,
         ignore_build_cache=False,
         remove_outs=False,
+        validate_state=True,
     ):
         if outs is None:
             outs = []
@@ -458,20 +459,21 @@ class Stage(object):
         else:
             stage.unprotect_outs()
 
-        if os.path.exists(path):
-            if not ignore_build_cache and stage.is_cached:
-                logger.info("Stage is cached, skipping.")
-                return None
+        if validate_state:
+            if os.path.exists(path):
+                if not ignore_build_cache and stage.is_cached:
+                    logger.info("Stage is cached, skipping.")
+                    return None
 
-            msg = (
-                "'{}' already exists. Do you wish to run the command and "
-                "overwrite it?".format(stage.relpath)
-            )
+                msg = (
+                    "'{}' already exists. Do you wish to run the command and "
+                    "overwrite it?".format(stage.relpath)
+                )
 
-            if not overwrite and not prompt.confirm(msg):
-                raise StageFileAlreadyExistsError(stage.relpath)
+                if not overwrite and not prompt.confirm(msg):
+                    raise StageFileAlreadyExistsError(stage.relpath)
 
-            os.unlink(path)
+                os.unlink(path)
 
         return stage
 

dvc/temp_git_repo.py

@@ -0,0 +1,145 @@
+import os
+import shutil
+import tempfile
+
+import git
+
+from dvc import logger
+from dvc.exceptions import DvcException
+
+
+class TempRepoException(DvcException):
+    """Raise when temporary package repository has not properly created"""
+
+    def __init__(self, temp_repo, msg, cause=None):
+        m = "temp repository '{}' error: {}".format(temp_repo.addr, msg)
+        super(TempRepoException, self).__init__(m, cause)
+
+
+class TempGitRepo(object):
+    GIT_DIR_TO_REMOVE = ".git"
+    GIT_FILES_TO_REMOVE = [".gitignore", ".gitmodules"]
+    SUFFIX = "tmp_DVC_mod"
+
+    def __init__(self, addr, module_name, modules_dir):
+        self.addr = addr
+        self.modules_dir = modules_dir
+        self._tmp_mod_prefix = module_name + "_" + self.SUFFIX + "_"
+        self._reset_state()
+
+    def _reset_state(self):
+        self._set_state(None, [])
+
+    def _set_state(self, cloned_tmp_dir, outs):
+        from dvc.repo import Repo
+
+        self.repo = Repo(cloned_tmp_dir) if cloned_tmp_dir else None
+
+        self._cloned_tmp_dir = cloned_tmp_dir
+        self.outs = outs
+        self._moved_files = []
+
+    def fetch(self, targets):
+        for target in targets:
+            try:
+                self.repo.fetch(target)
+            except Exception as ex:
+                msg = "error in fetching data from {}: {}".format(
+                    os.path.basename(target), ex
+                )
+                raise TempRepoException(self, msg)
+
+    @property
+    def is_state_set(self):
+        return self._cloned_tmp_dir is not None
+
+    def __enter__(self):
+        if self.is_state_set:
+            raise TempRepoException(self, "Git repo cloning duplication")
+
+        module_temp_dir = None
+        try:
+            module_temp_dir = self._clone_to_temp_dir()
+            self._remove_git_files_and_dirs(module_temp_dir)
+
+            outputs = self._read_outputs(module_temp_dir)
+            self._set_state(module_temp_dir, outputs)
+        finally:
+            self._clean_mod_temp_dir(module_temp_dir)
+
+        return self
+
+    def _clone_to_temp_dir(self):
+        result = tempfile.mktemp(
+            prefix=self._tmp_mod_prefix, dir=self.modules_dir
+        )
+        logger.debug(
+            "Cloning git repository {} to temp dir {}".format(
+                self.addr, result
+            )
+        )
+        git.Repo.clone_from(self.addr, result, depth=1)
+        return result
+
+    def _remove_git_files_and_dirs(self, module_temp_dir):
+        logger.debug(
+            "Removing git meta files from {}: {} dif and {}".format(
+                module_temp_dir,
+                TempGitRepo.GIT_DIR_TO_REMOVE,
+                ", ".join(TempGitRepo.GIT_FILES_TO_REMOVE),
+            )
+        )
+        shutil.rmtree(
+            os.path.join(module_temp_dir, TempGitRepo.GIT_DIR_TO_REMOVE)
+        )
+        for item in TempGitRepo.GIT_FILES_TO_REMOVE:
+            fname = os.path.join(module_temp_dir, item)
+            if os.path.exists(fname):
+                os.remove(fname)
+
+    def _clean_mod_temp_dir(self, module_temp_dir):
+        if not self.is_state_set:
+            if module_temp_dir and os.path.exists(module_temp_dir):
+                shutil.rmtree(module_temp_dir, ignore_errors=True)
+
+    def _read_outputs(self, module_temp_dir):
+        from dvc.repo import Repo
+
+        pkg_repo = Repo(module_temp_dir)
+        stages = pkg_repo.stages()
+        return [out for s in stages for out in s.outs]
+
+    def persist_to(self, module_dir, parent_repo):
+        if not self.is_state_set:
+            raise TempRepoException(self, "cannot persist")
+
+        tmp_repo_cache = self.repo.cache.local.url
+
+        for prefix in os.listdir(tmp_repo_cache):
+            if len(prefix) != 2:
+                logger.warning(
+                    "wrong dir format in cache {}: dir {}".format(
+                        tmp_repo_cache, prefix
+                    )
+                )
+            self._move_all_cache_files(parent_repo, prefix, tmp_repo_cache)
+
+        shutil.move(self._cloned_tmp_dir, module_dir)
+        self._reset_state()
+
+    @staticmethod
+    def _move_all_cache_files(parent_repo, prefix, tmp_repo_cache):
+        obj_name = os.path.join(tmp_repo_cache, prefix)
+        for suffix in os.listdir(obj_name):
+            src_path = os.path.join(tmp_repo_cache, prefix, suffix)
+            pre = os.path.join(parent_repo.cache.local.url, prefix)
+            if not os.path.exists(pre):
+                os.mkdir(pre)
+            dest = os.path.join(pre, suffix)
+            shutil.move(src_path, dest)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.is_state_set:
+            shutil.rmtree(self._cloned_tmp_dir)
+            self._reset_state()
+        pass

tests/basic_env.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import uuid
 import tempfile
 
@@ -7,10 +8,11 @@ from git.exc import GitCommandNotFound
 from unittest import TestCase
 
 import dvc.logger as logger
+from dvc.command.remote import CmdRemoteAdd
 from dvc.repo import Repo as DvcRepo
 
 
-class TestDir(TestCase):
+class TestDirFixture(object):
     GCP_CREDS_FILE = os.path.abspath(
         os.path.join("scripts", "ci", "gcp-creds.json")
     )
@@ -39,6 +41,13 @@ class TestDir(TestCase):
         "shutil.copyfile(sys.argv[1], sys.argv[2])"
     )
 
+    def __init__(self, root_dir=None):
+        if root_dir:
+            os.mkdir(root_dir)
+        else:
+            root_dir = self.mkdtemp()
+        self._root_dir = os.path.realpath(root_dir)
+
     def _pushd(self, d):
         self._saved_dir = os.path.realpath(os.curdir)
         os.chdir(d)
@@ -62,8 +71,6 @@ class TestDir(TestCase):
         return tempfile.mkdtemp(prefix=prefix, suffix=suffix)
 
     def setUp(self):
-        self._root_dir = TestDir.mkdtemp()
-
         self._pushd(self._root_dir)
         self.create(self.FOO, self.FOO_CONTENTS)
         self.create(self.BAR, self.BAR_CONTENTS)
@@ -77,11 +84,11 @@ class TestDir(TestCase):
         self._popd()
 
 
-class TestGit(TestDir):
+class TestGitFixture(TestDirFixture):
     N_RETRIES = 5
 
     def setUp(self):
-        super(TestGit, self).setUp()
+        super(TestGitFixture, self).setUp()
         # NOTE: handles EAGAIN error on BSD systems (osx in our case).
         # Otherwise when running tests you might get this exception:
         #
@@ -100,17 +107,117 @@ class TestGit(TestDir):
         self.git.index.commit("add code")
 
 
-class TestGitSubmodule(TestGit):
+class TestGitSubmoduleFixture(TestGitFixture):
+    def __init__(self, root_dir=None):
+        super(TestGitSubmoduleFixture, self).__init__(root_dir)
+
     def setUp(self):
-        super(TestGitSubmodule, self).setUp()
+        super(TestGitSubmoduleFixture, self).setUp()
         subrepo = Repo.init()
         subrepo_path = "subrepo"
         self.git.create_submodule(subrepo_path, subrepo_path, subrepo.git_dir)
         self._pushd(subrepo_path)
 
 
-class TestDvc(TestGit):
+class TestDvcFixture(TestGitFixture):
+    def __init__(self, root_dir=None):
+        super(TestDvcFixture, self).__init__(root_dir)
+
     def setUp(self):
-        super(TestDvc, self).setUp()
+        super(TestDvcFixture, self).setUp()
         self.dvc = DvcRepo.init(self._root_dir)
         logger.be_verbose()
+
+
+class TestDvcGitInitializedFixture(TestDvcFixture):
+    def __init__(self, root_dir=None):
+        super(TestDvcGitInitializedFixture, self).__init__(root_dir)
+
+    def setUp(self):
+        super(TestDvcGitInitializedFixture, self).setUp()
+        self.git.init()
+
+
+class TestDvcDataFileFixture(TestDvcGitInitializedFixture):
+    DATA_DVC_FILE = "data.dvc"
+    DATA_DIR_DVC_FILE = "data_sub_dir.dvc"
+    REMOTE = "myremote2"
+
+    def __init__(self, root_dir=None, cache_dir=None):
+        super(TestDvcDataFileFixture, self).__init__(root_dir)
+        self.cache_dir = cache_dir
+
+    def setUp(self):
+        super(TestDvcDataFileFixture, self).setUp()
+
+        self.dvc.add(self.DATA)
+        self.dvc.add(self.DATA_SUB_DIR)
+
+        if self.cache_dir:
+            shutil.copytree(self.dvc.cache.local.cache_dir, self.cache_dir)
+
+            class MockConfig:
+                system = None
+                glob = None
+                local = None
+                default = True
+                name = self.REMOTE
+                url = self.cache_dir
+
+            cmd = CmdRemoteAdd(MockConfig())
+            cmd.run()
+
+        self.git.index.add(
+            [
+                os.path.join(self.DATA_DIR, self.DATA_DVC_FILE),
+                os.path.join(self.DATA_DIR, self.DATA_DIR_DVC_FILE),
+                os.path.join(self.DATA_DIR, ".gitignore"),
+                ".dvc/config",
+                self.FOO,
+                self.BAR,
+            ]
+        )
+        self.git.index.commit("Hello world commit")
+
+
+# NOTE: Inheritance order in the classes below is important.
+
+
+class TestDir(TestDirFixture, TestCase):
+    def __init__(self, methodName, root_dir=None):
+        TestDirFixture.__init__(self, root_dir)
+        TestCase.__init__(self, methodName)
+
+
+class TestGit(TestGitFixture, TestCase):
+    def __init__(self, methodName, root_dir=None):
+        TestGitFixture.__init__(self, root_dir)
+        TestCase.__init__(self, methodName)
+
+
+class TestGitSubmodule(TestGitSubmoduleFixture, TestCase):
+    def __init__(self, methodName, root_dir=None):
+        TestGitSubmoduleFixture.__init__(self, root_dir)
+        TestCase.__init__(self, methodName)
+
+
+class TestDvc(TestDvcFixture, TestCase):
+    def __init__(self, methodName, root_dir=None):
+        TestDvcFixture.__init__(self, root_dir)
+        TestCase.__init__(self, methodName)
+
+
+class TestDvcPkg(TestDvcFixture, TestCase):
+    GIT_PKG = "git_pkg"
+    CACHE_DIR = "mycache"
+
+    def __init__(self, methodName, root_dir=None):
+        TestDvcFixture.__init__(self, root_dir)
+        TestCase.__init__(self, methodName)
+
+        self.pkg_dir = os.path.join(self._root_dir, self.GIT_PKG)
+        cache_dir = os.path.join(self._root_dir, self.CACHE_DIR)
+        self.pkg_fixture = TestDvcDataFileFixture(
+            root_dir=self.pkg_dir, cache_dir=cache_dir
+        )
+        self.pkg_fixture.setUp()

tests/test_data_cloud.py

@@ -675,7 +675,7 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
         )
 
         self.dvc.add(self.DATA)
-        self.dvc.add(self.DATA_SUB_CONTENTS)
+        self.dvc.add(self.DATA_SUB)
 
     def _remove_local_data_files(self):
         os.remove(self.DATA)

tests/test_move.py

@@ -173,13 +173,16 @@ class TestMoveFileBetweenDirectories(TestDvc):
 
 class TestMoveFileInsideDirectory(TestDvc):
     def test(self):
-        ret = main(["add", "data_dir/data"])
+        ret = main(["add", self.DATA])
         self.assertEqual(ret, 0)
 
-        with cd("data_dir"):
-            ret = main(["move", "data", "data.txt"])
+        with cd(self.DATA_DIR):
+            ret = main(["move", os.path.basename(self.DATA), "data.txt"])
             self.assertEqual(ret, 0)
 
-        self.assertFalse(os.path.exists("data_dir/data"))
-        self.assertTrue(os.path.exists("data_dir/data.txt"))
-        self.assertTrue(os.path.exists("data_dir/data.txt.dvc"))
+        self.assertFalse(os.path.exists(self.DATA))
+
+        data_fullpath = os.path.join(self.DATA_DIR, "data.txt")
+        dvc_fullpath = os.path.join(self.DATA_DIR, "data.txt.dvc")
+        self.assertTrue(os.path.exists(data_fullpath))
+        self.assertTrue(os.path.exists(dvc_fullpath))

tests/test_pkg.py

@@ -0,0 +1,37 @@
+import os
+
+from dvc.repo.pkg import Package, PackageParams
+from tests.basic_env import TestDvcPkg
+
+
+class TestPkgBasic(TestDvcPkg):
+    NEW_PKG_DIR = "new_pkg"
+
+    def setUp(self):
+        super(TestPkgBasic, self).setUp()
+
+        self.to_dir = os.path.join(self._root_dir, self.NEW_PKG_DIR)
+        os.mkdir(self.to_dir)
+        self.dvc.install_pkg(
+            PackageParams(self.pkg_fixture._root_dir, self.to_dir)
+        )
+
+    def test_installed_dirs_and_files(self):
+        self.assertTrue(os.path.isdir(self.pkg_dir))
+
+        mod_dir_fullpath = os.path.join(self._root_dir, Package.MODULES_DIR)
+        self.assertTrue(os.path.isdir(mod_dir_fullpath))
+
+        installed_pkg_dir = os.path.join(mod_dir_fullpath, self.GIT_PKG)
+        for file in [self.FOO, self.BAR, self.CODE]:
+            installed_file = os.path.join(installed_pkg_dir, file)
+            self.assertTrue(os.path.isfile(installed_file))
+
+    def test_target_dir_checkout(self):
+        pkg_data_file = os.path.join(self.to_dir, self.DATA)
+        self.assertTrue(os.path.exists(pkg_data_file))
+        self.assertTrue(os.path.isfile(pkg_data_file))
+
+        pkg_data_dir = os.path.join(self.to_dir, self.DATA_SUB_DIR)
+        self.assertTrue(os.path.exists(pkg_data_dir))
+        self.assertTrue(os.path.isdir(pkg_data_dir))

tests/test_scm.py

@@ -1,8 +1,11 @@
-import os
+from __future__ import unicode_literals
 
+import os
 from git import Repo
 
+from dvc.utils.compat import str  # noqa: F401
 from dvc.scm import SCM, Base, Git
+from dvc.scm.base import FileNotInTargetSubdirError
 
 from tests.basic_env import TestDir, TestGit, TestGitSubmodule
 from tests.utils import get_gitignore_content
@@ -48,7 +51,7 @@ class TestIgnore(TestGit):
 
         return len(list(filter(lambda x: x.strip() == "/" + self.FOO, lines)))
 
-    def test(self):
+    def test_ignore(self):
         git = Git(self._root_dir)
         foo = os.path.join(self._root_dir, self.FOO)
 
@@ -61,3 +64,62 @@ class TestIgnore(TestGit):
 
         git.ignore_remove(foo)
         self.assertEqual(self._count_gitignore(), 0)
+
+    def test_get_gitignore(self):
+        data_dir = os.path.join(self._root_dir, "file1")
+        entry, gitignore = Git(self._root_dir)._get_gitignore(data_dir)
+        self.assertEqual(entry, "/file1")
+        self.assertEqual(
+            gitignore, os.path.join(self._root_dir, Git.GITIGNORE)
+        )
+
+        data_dir = os.path.join(self._root_dir, "dir")
+        entry, gitignore = Git(self._root_dir)._get_gitignore(data_dir)
+        self.assertEqual(entry, "/dir")
+        self.assertEqual(
+            gitignore, os.path.join(self._root_dir, Git.GITIGNORE)
+        )
+
+    def test_get_gitignore_subdir(self):
+        data_dir = os.path.join(self._root_dir, os.path.join("dir1", "file1"))
+        entry, gitignore = Git(self._root_dir)._get_gitignore(data_dir)
+        self.assertEqual(entry, "/file1")
+        self.assertEqual(
+            gitignore, os.path.join(self._root_dir, "dir1", Git.GITIGNORE)
+        )
+
+        data_dir = os.path.join(self._root_dir, os.path.join("dir1", "dir2"))
+        entry, gitignore = Git(self._root_dir)._get_gitignore(data_dir)
+        self.assertEqual(entry, "/dir2")
+        self.assertEqual(
+            gitignore, os.path.join(self._root_dir, "dir1", Git.GITIGNORE)
+        )
+
+    def test_get_gitignore_ignorefile_dir(self):
+        git = Git(self._root_dir)
+
+        file_double_dir = os.path.join("dir1", "dir2", "file1")
+        data_dir1 = os.path.join(self._root_dir, file_double_dir)
+        dir1_real1 = os.path.realpath("dir1")
+        entry, gitignore = git._get_gitignore(data_dir1, dir1_real1)
+        self.assertEqual(entry, "/dir2/file1")
+        gitignore1 = os.path.join(self._root_dir, "dir1", Git.GITIGNORE)
+        self.assertEqual(gitignore, gitignore1)
+
+        triple_dir = os.path.join("dir1", "dir2", "dir3")
+        data_dir2 = os.path.join(self._root_dir, triple_dir)
+        dir1_real2 = os.path.realpath("dir1")
+        entry, gitignore = git._get_gitignore(data_dir2, dir1_real2)
+        self.assertEqual(entry, "/dir2/dir3")
+        gitignore2 = os.path.join(self._root_dir, "dir1", Git.GITIGNORE)
+        self.assertEqual(gitignore, gitignore2)
+
+    def test_get_gitignore_ignorefile_dir_upper_level(self):
+        git = Git(self._root_dir)
+
+        file_double_dir = os.path.join("dir1", "dir2", "file1")
+        data_dir1 = os.path.join(self._root_dir, file_double_dir)
+        ignore_file_dir = os.path.realpath(os.path.join("aa", "bb"))
+
+        with self.assertRaises(FileNotInTargetSubdirError):
+            git._get_gitignore(data_dir1, ignore_file_dir)