dvc: bump to 0.63.1

https://opendatascience.slack.com/archives/CGGLZJ119/p1571253702008100?thread_ts=1571240388.006500&cid=CGGLZJ119

.dvc/.gitignore

@@ -8,3 +8,4 @@
 /cache
 /pkg
 /repos
+/tmp

.pylintrc

@@ -139,7 +139,9 @@ disable=print-statement,
         deprecated-sys-function,
         exception-escape,
         comprehension-escape,
-        useless-object-inheritance
+        useless-object-inheritance,
+        missing-docstring,
+        bad-continuation
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -314,7 +316,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=79
 
 # Maximum number of lines in a module.
 max-module-lines=1000

.travis.yml

@@ -19,7 +19,7 @@ stages:
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
-# Note that we specify language as ruby-2.4.0 to help us satisfy
+# Note that we specify language as ruby-2.4.3 to help us satisfy
 # fpm dependencies, as travis runs el capitan with an old ruby
 # version, that causes fpm to crash.
 jobs:
@@ -78,7 +78,7 @@ jobs:
     os: osx
     osx_image: xcode9.4
     language: ruby
-    rvm: 2.4.0
+    rvm: 2.4.3
     install:
     script: ./scripts/build_posix.sh
   - name: "Windows pkg"

MANIFEST.in

@@ -0,0 +1,2 @@
+include fastentrypoints.py
+include LICENSE

README.rst

@@ -46,7 +46,7 @@ learning projects. Key features:
 
 #. it helps manage experiments with Git tags or branches and **metrics** tracking;
 
-It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
+**DVC** aims to replace tools like Excel and Google Docs that are being commonly used as a knowledge repo and
 a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
 data file suffixes and prefixes.
 

dvc/analytics.py

@@ -66,7 +66,7 @@ class Analytics(object):
                 raise
 
         self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
-        self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + ".lock")
+        self.user_id_file_lock = Lock(self.user_id_file + ".lock")
 
     @staticmethod
     def load(path):
@@ -113,7 +113,7 @@ class Analytics(object):
                 return user_id
         except LockError:
             msg = "Failed to acquire '{}'"
-            logger.debug(msg.format(self.user_id_file_lock.lock_file))
+            logger.debug(msg.format(self.user_id_file_lock.lockfile))
 
     def _collect_windows(self):
         import sys

dvc/cache.py

@@ -3,7 +3,9 @@
 from __future__ import unicode_literals
 
 import os
+from funcy import cached_property
 
+from dvc.utils.compat import builtin_str
 from dvc.config import Config
 
 
@@ -21,6 +23,42 @@ class CacheConfig(object):
         )
 
 
+def _make_remote_property(name):
+    """
+    The config file is stored in a way that allows you to have a
+    cache for each remote.
+
+    This is needed when specifying external outputs
+    (as they require you to have an external cache location).
+
+    Imagine a config file like the following:
+
+            ['remote "dvc-storage"']
+            url = ssh://localhost/tmp
+            ask_password = true
+
+            [cache]
+            ssh = dvc-storage
+
+    This method creates a cached property, containing cache named `name`:
+
+        self.config == {'ssh': 'dvc-storage'}
+        self.ssh  # a RemoteSSH instance
+    """
+
+    def getter(self):
+        from dvc.remote import Remote
+
+        remote = self.config.get(name)
+        if not remote:
+            return None
+
+        return Remote(self.repo, name=remote)
+
+    getter.__name__ = builtin_str(name)
+    return cached_property(getter)
+
+
 class Cache(object):
     """Class that manages cache locations of a dvc repo.
 
@@ -35,7 +73,7 @@ class Cache(object):
 
         self.repo = repo
 
-        config = repo.config.config[Config.SECTION_CACHE]
+        self.config = config = repo.config.config[Config.SECTION_CACHE]
         local = config.get(Config.SECTION_CACHE_LOCAL)
 
         if local:
@@ -59,48 +97,9 @@ class Cache(object):
             }
 
         self.local = Remote(repo, **settings)
-        self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
-        self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
-        self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
-        self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
-        self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
 
-    def _get_remote(self, config, name):
-        """
-        The config file is stored in a way that allows you to have a
-        cache for each remote.
-
-        This is needed when specifying external outputs
-        (as they require you to have an external cache location).
-
-        Imagine a config file like the following:
-
-                ['remote "dvc-storage"']
-                url = ssh://localhost/tmp
-                ask_password = true
-
-                [cache]
-                ssh = dvc-storage
-
-        This method resolves the name under the cache section into the
-        correct Remote instance.
-
-        Args:
-            config (dict): The cache section on the config file
-            name (str): Name of the section we are interested in to retrieve
-
-        Returns:
-            remote (dvc.Remote): Remote instance that the section is referring.
-                None when there's no remote with that name.
-
-        Example:
-            >>> _get_remote(config={'ssh': 'dvc-storage'}, name='ssh')
-        """
-        from dvc.remote import Remote
-
-        remote = config.get(name)
-
-        if not remote:
-            return None
-
-        return Remote(self.repo, name=remote)
+    s3 = _make_remote_property(Config.SECTION_CACHE_S3)
+    gs = _make_remote_property(Config.SECTION_CACHE_GS)
+    ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
+    hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
+    azure = _make_remote_property(Config.SECTION_CACHE_AZURE)

dvc/cli.py

@@ -144,6 +144,19 @@ def parse_args(argv=None):
         description=desc,
         parents=[parent_parser],
         formatter_class=argparse.RawTextHelpFormatter,
+        add_help=False,
+    )
+
+    # NOTE: We are doing this to capitalize help message.
+    # Unfortunately, there is no easier and clearer way to do it,
+    # as adding this argument in get_parent_parser() either in
+    # log_level_group or on parent_parser itself will cause unexpected error.
+    parser.add_argument(
+        "-h",
+        "--help",
+        action="help",
+        default=argparse.SUPPRESS,
+        help="Show this help message and exit.",
     )
 
     # NOTE: On some python versions action='version' prints to stderr

dvc/command/base.py

@@ -55,16 +55,6 @@ class CmdBase(object):
         logger.warning(msg)
         return [Stage.STAGE_FILE]
 
-    def run_cmd(self):
-        from dvc.lock import LockError
-
-        try:
-            with self.repo.lock:
-                return self.run()
-        except LockError:
-            logger.exception("failed to lock before running a command")
-            return 1
-
     # Abstract methods that have to be implemented by any inheritance class
     def run(self):
         pass
@@ -73,6 +63,3 @@ class CmdBase(object):
 class CmdBaseNoRepo(CmdBase):
     def __init__(self, args):
         self.args = args
-
-    def run_cmd(self):
-        return self.run()

dvc/command/destroy.py

@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
 
 
 class CmdDestroy(CmdBase):
-    def run_cmd(self):
+    def run(self):
         try:
             statement = (
                 "This will destroy all information about your pipelines,"

dvc/command/diff.py

@@ -9,7 +9,6 @@ from funcy import compact
 
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase, append_doc_link
-import dvc.repo.diff as diff
 
 
 logger = logging.getLogger(__name__)
@@ -37,6 +36,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_dir_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         engine = inflect.engine()
         changes_msg = (
             "{} {} untouched, {} {} modified, {} {} added, "
@@ -57,6 +58,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_file_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         if (
             dct.get(diff.DIFF_OLD_FILE)
             and dct.get(diff.DIFF_NEW_FILE)
@@ -79,6 +82,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _get_royal_changes(cls, dct):
+        import dvc.repo.diff as diff
+
         if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
             if dct.get("is_dir"):
                 return cls._get_dir_changes(dct)
@@ -88,6 +93,8 @@ class CmdDiff(CmdBase):
 
     @classmethod
     def _show(cls, diff_dct):
+        import dvc.repo.diff as diff
+
         msg = "dvc diff from {} to {}".format(
             diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
         )

dvc/command/get.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
 from dvc.exceptions import DvcException
 from .base import CmdBaseNoRepo, append_doc_link
 
@@ -13,6 +12,8 @@ logger = logging.getLogger(__name__)
 
 class CmdGet(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         try:
             Repo.get(
                 self.args.url,

dvc/command/get_url.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
 from dvc.exceptions import DvcException
 from .base import CmdBaseNoRepo, append_doc_link
 
@@ -13,6 +12,8 @@ logger = logging.getLogger(__name__)
 
 class CmdGetUrl(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         try:
             Repo.get_url(self.args.url, out=self.args.out)
             return 0

dvc/command/install.py

@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
 
 
 class CmdInstall(CmdBase):
-    def run_cmd(self):
+    def run(self):
         try:
             self.repo.install()
         except Exception:

dvc/command/pipeline.py

@@ -19,7 +19,7 @@ class CmdPipelineShow(CmdBase):
         from dvc.stage import Stage
 
         stage = Stage.load(self.repo, target)
-        G = self.repo.graph()[0]
+        G = self.repo.graph
         stages = networkx.get_node_attributes(G, "stage")
         node = relpath(stage.path, self.repo.root_dir)
         nodes = networkx.dfs_postorder_nodes(G, node)
@@ -41,20 +41,16 @@ class CmdPipelineShow(CmdBase):
     def __build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
+        from dvc.repo.graph import get_pipeline
 
         stage = Stage.load(self.repo, target)
         node = relpath(stage.path, self.repo.root_dir)
 
-        pipelines = list(
-            filter(lambda g: node in g.nodes(), self.repo.pipelines())
-        )
-
-        assert len(pipelines) == 1
-        G = pipelines[0]
+        G = get_pipeline(self.repo.pipelines, node)
         stages = networkx.get_node_attributes(G, "stage")
 
         nodes = []
-        for n in G.nodes():
+        for n in G:
             stage = stages[n]
             if commands:
                 if stage.cmd is None:
@@ -168,7 +164,7 @@ class CmdPipelineList(CmdBase):
     def run(self):
         import networkx
 
-        pipelines = self.repo.pipelines()
+        pipelines = self.repo.pipelines
         for p in pipelines:
             stages = networkx.get_node_attributes(p, "stage")
             for stage in stages:

dvc/command/root.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.repo import Repo
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.utils import relpath
 
@@ -12,6 +11,8 @@ logger = logging.getLogger(__name__)
 
 class CmdRoot(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         logger.info(relpath(Repo.find_root()))
         return 0
 

dvc/command/version.py

@@ -5,6 +5,7 @@ import platform
 import argparse
 import logging
 import uuid
+import itertools
 
 try:
     import psutil
@@ -13,7 +14,6 @@ except ImportError:
 
 from dvc.utils import is_binary
 from dvc.utils.compat import pathlib
-from dvc.repo import Repo
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.version import __version__
 from dvc.exceptions import DvcException, NotDvcRepoError
@@ -25,6 +25,8 @@ logger = logging.getLogger(__name__)
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
+        from dvc.repo import Repo
+
         dvc_version = __version__
         python_version = platform.python_version()
         platform_type = platform.platform()
@@ -46,9 +48,21 @@ class CmdVersion(CmdBaseNoRepo):
             repo = Repo()
             root_directory = repo.root_dir
 
-            info += "Cache: {cache}\n".format(
-                cache=self.get_linktype_support_info(repo)
-            )
+            # cache_dir might not exist yet (e.g. after `dvc init`), and we
+            # can't auto-create it, as it might cause issues if the user
+            # later decides to enable shared cache mode with
+            # `dvc config cache.shared group`.
+            if os.path.exists(repo.cache.local.cache_dir):
+                info += "Cache: {cache}\n".format(
+                    cache=self.get_linktype_support_info(repo)
+                )
+            else:
+                logger.warning(
+                    "Unable to detect supported link types, as cache "
+                    "directory '{}' doesn't exist. It is usually auto-created "
+                    "by commands such as `dvc add/fetch/pull/run/import`, but "
+                    "you could create it manually to enable this check."
+                )
 
             if psutil:
                 info += (
@@ -69,9 +83,12 @@ class CmdVersion(CmdBaseNoRepo):
     def get_fs_type(path):
         partition = {
             pathlib.Path(part.mountpoint): (part.fstype, part.device)
-            for part in psutil.disk_partitions()
+            for part in psutil.disk_partitions(all=True)
         }
-        for parent in pathlib.Path(path).parents:
+
+        path = pathlib.Path(path)
+
+        for parent in itertools.chain([path], path.parents):
             if parent in partition:
                 return partition[parent]
         return ("unkown", "none")

dvc/config.py

@@ -31,6 +31,18 @@ class ConfigError(DvcException):
         )
 
 
+class NoRemoteError(ConfigError):
+    def __init__(self, command, cause=None):
+        msg = (
+            "no remote specified. Setup default remote with\n"
+            "    dvc config core.remote <name>\n"
+            "or use:\n"
+            "    dvc {} -r <name>\n".format(command)
+        )
+
+        super(NoRemoteError, self).__init__(msg, cause=cause)
+
+
 def supported_cache_type(types):
     """Checks if link type config option has a valid value.
 

dvc/daemon.py

@@ -94,7 +94,7 @@ def daemon(args):
 
     cmd = [sys.executable]
     if not is_binary():
-        cmd += ["-m", "dvc"]
+        cmd += [sys.argv[0]]
     cmd += ["daemon", "-q"] + args
 
     env = fix_env()

dvc/data_cloud.py

@@ -4,16 +4,8 @@ from __future__ import unicode_literals
 
 import logging
 
-from dvc.config import Config, ConfigError
+from dvc.config import Config, NoRemoteError
 from dvc.remote import Remote
-from dvc.remote.s3 import RemoteS3
-from dvc.remote.gs import RemoteGS
-from dvc.remote.azure import RemoteAZURE
-from dvc.remote.oss import RemoteOSS
-from dvc.remote.ssh import RemoteSSH
-from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.http import RemoteHTTP
 
 
 logger = logging.getLogger(__name__)
@@ -30,18 +22,6 @@ class DataCloud(object):
         config.ConfigError: thrown when config has invalid format.
     """
 
-    CLOUD_MAP = {
-        "aws": RemoteS3,
-        "gcp": RemoteGS,
-        "azure": RemoteAZURE,
-        "oss": RemoteOSS,
-        "ssh": RemoteSSH,
-        "hdfs": RemoteHDFS,
-        "local": RemoteLOCAL,
-        "http": RemoteHTTP,
-        "https": RemoteHTTP,
-    }
-
     def __init__(self, repo):
         self.repo = repo
 
@@ -60,12 +40,7 @@ class DataCloud(object):
         if remote:
             return self._init_remote(remote)
 
-        raise ConfigError(
-            "No remote repository specified. Setup default repository with\n"
-            "    dvc config core.remote <name>\n"
-            "or use:\n"
-            "    dvc {} -r <name>\n".format(command)
-        )
+        raise NoRemoteError(command)
 
     def _init_remote(self, remote):
         return Remote(self.repo, name=remote)

dvc/dependency/base.py

@@ -15,11 +15,19 @@ class DependencyIsNotFileOrDirError(DvcException):
         super(DependencyIsNotFileOrDirError, self).__init__(msg)
 
 
+class DependencyIsStageFileError(DvcException):
+    def __init__(self, path):
+        super(DependencyIsStageFileError, self).__init__(
+            "Stage file '{}' cannot be a dependency.".format(path)
+        )
+
+
 class DependencyBase(object):
     IS_DEPENDENCY = True
 
     DoesNotExistError = DependencyDoesNotExistError
     IsNotFileOrDirError = DependencyIsNotFileOrDirError
+    IsStageFileError = DependencyIsStageFileError
 
     def update(self):
         pass

dvc/dependency/repo.py

@@ -61,7 +61,7 @@ class DependencyREPO(DependencyLOCAL):
     def dumpd(self):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def download(self, to, resume=False):
+    def fetch(self):
         with self._make_repo(
             cache_dir=self.repo.cache.local.cache_dir
         ) as repo:
@@ -70,8 +70,13 @@ class DependencyREPO(DependencyLOCAL):
             out = repo.find_out_by_relpath(self.def_path)
             with repo.state:
                 repo.cloud.pull(out.get_used_cache())
-            to.info = copy.copy(out.info)
-            to.checkout()
+
+        return out
+
+    def download(self, to):
+        out = self.fetch()
+        to.info = copy.copy(out.info)
+        to.checkout()
 
     def update(self):
         with self._make_repo(rev_lock=None) as repo:

dvc/exceptions.py

@@ -47,7 +47,7 @@ class OutputDuplicationError(DvcException):
             for stage in stages
         )
         msg = (
-            "file/directory '{}' is specified as an output in more than one"
+            "file/directory '{}' is specified as an output in more than one "
             "stage: {}"
         ).format(output, "\n    ".join(stages))
         super(OutputDuplicationError, self).__init__(msg)
@@ -82,8 +82,8 @@ class StagePathAsOutputError(DvcException):
         assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
         assert isinstance(fname, str) or isinstance(fname, builtin_str)
         msg = (
-            "current working directory '{cwd}' is specified as an output in"
-            " '{fname}'. Use another CWD to prevent any data removal.".format(
+            "current working directory '{cwd}' is specified as an output in "
+            "'{fname}'. Use another CWD to prevent any data removal.".format(
                 cwd=wdir, fname=fname
             )
         )
@@ -167,8 +167,8 @@ class CyclicGraphError(DvcException):
         assert isinstance(stages, list)
         stages = "\n".join("\t- {}".format(stage) for stage in stages)
         msg = (
-            "you've introduced a cycle in your pipeline that involves"
-            " the following stages:"
+            "you've introduced a cycle in your pipeline that involves "
+            "the following stages:"
             "\n"
             "{stages}".format(stages=stages)
         )
@@ -244,13 +244,6 @@ class OverlappingOutputPathsError(DvcException):
         )
 
 
-class TargetNotDirectoryError(DvcException):
-    def __init__(self, path):
-        super(TargetNotDirectoryError, self).__init__(
-            "Target: {} is not a directory".format(path)
-        )
-
-
 class CheckoutErrorSuggestGit(DvcException):
     def __init__(self, target, cause):
         super(CheckoutErrorSuggestGit, self).__init__(
@@ -266,10 +259,11 @@ class ETagMismatchError(DvcException):
         )
 
 
-class OutputFileMissingError(DvcException):
-    def __init__(self, path):
-        super(OutputFileMissingError, self).__init__(
-            "Can't find {} neither locally nor on remote".format(path)
+class FileMissingError(DvcException):
+    def __init__(self, path, cause=None):
+        super(FileMissingError, self).__init__(
+            "Can't find '{}' neither locally nor on remote".format(path),
+            cause=cause,
         )
 
 
@@ -294,3 +288,40 @@ class GetDVCFileError(DvcException):
             "the given path is a DVC-file, you must specify a data file "
             "or a directory"
         )
+
+
+class GitHookAlreadyExistsError(DvcException):
+    def __init__(self, hook_name):
+        super(GitHookAlreadyExistsError, self).__init__(
+            "Hook '{}' already exists. Please refer to "
+            "https://man.dvc.org/install "
+            "for more info.".format(hook_name)
+        )
+
+
+class DownloadError(DvcException):
+    def __init__(self, amount):
+        self.amount = amount
+
+        super(DownloadError, self).__init__(
+            "{amount} files failed to download".format(amount=amount)
+        )
+
+
+class UploadError(DvcException):
+    def __init__(self, amount):
+        self.amount = amount
+
+        super(UploadError, self).__init__(
+            "{amount} files failed to upload".format(amount=amount)
+        )
+
+
+class CheckoutError(DvcException):
+    def __init__(self, target_infos):
+        targets = [str(t) for t in target_infos]
+        m = (
+            "Checkout failed for following targets:\n {}\nDid you "
+            "forget to fetch?".format("\n".join(targets))
+        )
+        super(CheckoutError, self).__init__(m)

dvc/external_repo.py

@@ -1,50 +1,91 @@
 from __future__ import unicode_literals
 
 import os
-import logging
 import tempfile
+from distutils.dir_util import copy_tree
 
 from funcy import retry
 from contextlib import contextmanager
 
-from dvc.config import Config
-from dvc.cache import CacheConfig
 from dvc.utils import remove
 
 
-logger = logging.getLogger(__name__)
-
-
-def _clone(cache_dir=None, url=None, rev=None, rev_lock=None):
-    from dvc.repo import Repo
-
-    _path = tempfile.mkdtemp("dvc-repo")
-
-    repo = Repo.clone(url, _path, rev=(rev_lock or rev))
-
-    try:
-        if cache_dir:
-            cache_config = CacheConfig(repo.config)
-            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
-    finally:
-        repo.close()
-
-    return Repo(_path)
-
-
-def _remove(repo):
-    repo.close()
-
-    if os.name == "nt":
-        # git.exe may hang for a while not permitting to remove temp dir
-        os_retry = retry(5, errors=OSError, timeout=0.1)
-        os_retry(remove)(repo.root_dir)
-    else:
-        remove(repo.root_dir)
+REPO_CACHE = {}
 
 
 @contextmanager
-def external_repo(**kwargs):
-    repo = _clone(**kwargs)
+def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None):
+    from dvc.repo import Repo
+
+    path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir)
+    repo = Repo(path)
     yield repo
-    _remove(repo)
+    repo.close()
+
+
+def _external_repo(url=None, rev=None, cache_dir=None):
+    from dvc.config import Config
+    from dvc.cache import CacheConfig
+    from dvc.repo import Repo
+
+    key = (url, rev, cache_dir)
+    if key in REPO_CACHE:
+        return REPO_CACHE[key]
+
+    new_path = tempfile.mkdtemp("dvc-erepo")
+
+    # Copy and adjust existing clone
+    if (url, None, None) in REPO_CACHE:
+        old_path = REPO_CACHE[url, None, None]
+
+        # This one unlike shutil.copytree() works with an existing dir
+        copy_tree(old_path, new_path)
+    else:
+        # Create a new clone
+        _clone_repo(url, new_path)
+
+        # Save clean clone dir so that we will have access to a default branch
+        clean_clone_path = tempfile.mkdtemp("dvc-erepo")
+        copy_tree(new_path, clean_clone_path)
+        REPO_CACHE[url, None, None] = clean_clone_path
+
+    # Adjust new clone/copy to fit rev and cache_dir
+    repo = Repo(new_path)
+    try:
+        if rev is not None:
+            repo.scm.checkout(rev)
+
+        if cache_dir is not None:
+            cache_config = CacheConfig(repo.config)
+            cache_config.set_dir(cache_dir, level=Config.LEVEL_LOCAL)
+    finally:
+        # Need to close/reopen repo to force config reread
+        repo.close()
+
+    REPO_CACHE[key] = new_path
+    return new_path
+
+
+def clean_repos():
+    # Outside code should not see cache while we are removing
+    repo_paths = list(REPO_CACHE.values())
+    REPO_CACHE.clear()
+
+    for path in repo_paths:
+        _remove(path)
+
+
+def _remove(path):
+    if os.name == "nt":
+        # git.exe may hang for a while not permitting to remove temp dir
+        os_retry = retry(5, errors=OSError, timeout=0.1)
+        os_retry(remove)(path)
+    else:
+        remove(path)
+
+
+def _clone_repo(url, path):
+    from dvc.scm.git import Git
+
+    git = Git.clone(url, path)
+    git.close()

dvc/lock.py

@@ -3,58 +3,113 @@
 from __future__ import unicode_literals
 
 import os
+import hashlib
 import time
-import zc.lockfile
+from datetime import timedelta
+
+from funcy.py3 import lkeep
 
 from dvc.exceptions import DvcException
+from dvc.utils import makedirs
+from dvc.utils.compat import is_py3
+
+
+DEFAULT_TIMEOUT = 5
 
 
 class LockError(DvcException):
     """Thrown when unable to acquire the lock for dvc repo."""
 
 
-class Lock(object):
-    """Class for dvc repo lock.
+if is_py3:
+    import flufl.lock
 
-    Args:
-        dvc_dir (str): path to the directory that the lock should be created
-            in.
-        name (str): name of the lock file.
-    """
+    class Lock(flufl.lock.Lock):
+        """Class for dvc repo lock.
 
-    LOCK_FILE = "lock"
-    TIMEOUT = 5
+        Args:
+            lockfile (str): the lock filename
+                in.
+            tmp_dir (str): a directory to store claim files.
+        """
 
-    def __init__(self, dvc_dir, name=LOCK_FILE):
-        self.lock_file = os.path.join(dvc_dir, name)
-        self._lock = None
+        def __init__(self, lockfile, tmp_dir=None):
+            lifetime = timedelta(days=365)  # Lock for good by default
+            self._tmp_dir = tmp_dir
+            if self._tmp_dir is not None:
+                makedirs(self._tmp_dir, exist_ok=True)
 
-    def _do_lock(self):
-        try:
-            self._lock = zc.lockfile.LockFile(self.lock_file)
-        except zc.lockfile.LockError:
-            raise LockError(
-                "cannot perform the cmd since DVC is busy and "
-                "locked. Please retry the cmd later."
-            )
+            super(Lock, self).__init__(lockfile, lifetime=lifetime)
+
+        @property
+        def lockfile(self):
+            return self._lockfile
+
+        @property
+        def files(self):
+            return lkeep([self._lockfile, self._tmp_dir])
+
+        def lock(self):
+            try:
+                super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
+            except flufl.lock.TimeOutError:
+                raise LockError(
+                    "cannot perform the cmd since DVC is busy and "
+                    "locked. Please retry the cmd later."
+                )
+
+        def _set_claimfile(self, pid=None):
+            super(Lock, self)._set_claimfile(pid)
+
+            if self._tmp_dir is not None:
+                # Under Windows file path length is limited so we hash it
+                filename = hashlib.md5(self._claimfile.encode()).hexdigest()
+                self._claimfile = os.path.join(
+                    self._tmp_dir, filename + ".lock"
+                )
+
+
+else:
+    import zc.lockfile
+
+    class Lock(object):
+        """Class for dvc repo lock.
+
+        Uses zc.lockfile as backend.
+        """
+
+        def __init__(self, lockfile, tmp_dir=None):
+            self.lockfile = lockfile
+            self._lock = None
+
+        @property
+        def files(self):
+            return [self.lockfile]
+
+        def _do_lock(self):
+            try:
+                self._lock = zc.lockfile.LockFile(self.lockfile)
+            except zc.lockfile.LockError:
+                raise LockError(
+                    "cannot perform the cmd since DVC is busy and "
+                    "locked. Please retry the cmd later."
+                )
+
+        def lock(self):
+            try:
+                self._do_lock()
+                return
+            except LockError:
+                time.sleep(DEFAULT_TIMEOUT)
 
-    def lock(self):
-        """Acquire lock for dvc repo."""
-        try:
             self._do_lock()
-            return
-        except LockError:
-            time.sleep(self.TIMEOUT)
 
-        self._do_lock()
+        def unlock(self):
+            self._lock.close()
+            self._lock = None
 
-    def unlock(self):
-        """Release lock for dvc repo."""
-        self._lock.close()
-        self._lock = None
+        def __enter__(self):
+            self.lock()
 
-    def __enter__(self):
-        self.lock()
-
-    def __exit__(self, typ, value, tbck):
-        self.unlock()
+        def __exit__(self, typ, value, tbck):
+            self.unlock()

dvc/logger.py

@@ -2,7 +2,7 @@
 
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, StringIO
+from dvc.utils.compat import str, StringIO, RecursionError
 from dvc.progress import Tqdm
 
 import logging
@@ -19,7 +19,7 @@ class LoggingException(Exception):
 
 class ExcludeErrorsFilter(logging.Filter):
     def filter(self, record):
-        return record.levelno < logging.ERROR
+        return record.levelno < logging.WARNING
 
 
 class ColorFormatter(logging.Formatter):
@@ -44,7 +44,7 @@ class ColorFormatter(logging.Formatter):
     }
 
     footer = (
-        "{yellow}Having any troubles?{nc}."
+        "{yellow}Having any troubles?{nc}"
         " Hit us up at {blue}https://dvc.org/support{nc},"
         " we are always happy to help!"
     ).format(
@@ -182,7 +182,7 @@ def setup(level=logging.INFO):
                 },
                 "console_errors": {
                     "class": "dvc.logger.LoggerHandler",
-                    "level": "ERROR",
+                    "level": "WARNING",
                     "formatter": "color",
                     "stream": "ext://sys.stderr",
                 },
@@ -196,6 +196,10 @@ def setup(level=logging.INFO):
                     "level": logging.CRITICAL,
                     "handlers": ["console", "console_errors"],
                 },
+                "flufl.lock": {
+                    "level": logging.CRITICAL,
+                    "handlers": ["console", "console_errors"],
+                },
             },
         }
     )

dvc/main.py

@@ -5,10 +5,13 @@ from __future__ import unicode_literals
 import logging
 
 from dvc.cli import parse_args
+from dvc.lock import LockError
 from dvc.config import ConfigError
 from dvc.analytics import Analytics
 from dvc.exceptions import NotDvcRepoError, DvcParserError
 from dvc.remote.pool import close_pools
+from dvc.external_repo import clean_repos
+from dvc.utils.compat import is_py2
 
 
 logger = logging.getLogger("dvc")
@@ -37,7 +40,10 @@ def main(argv=None):
             logger.setLevel(logging.DEBUG)
 
         cmd = args.func(args)
-        ret = cmd.run_cmd()
+        ret = cmd.run()
+    except LockError:
+        logger.exception("failed to lock before running a command")
+        ret = 250
     except ConfigError:
         logger.exception("configuration error")
         ret = 251
@@ -49,15 +55,26 @@ def main(argv=None):
         ret = 253
     except DvcParserError:
         ret = 254
-    except Exception:  # pylint: disable=broad-except
-        logger.exception("unexpected error")
+    except Exception as exc:  # pylint: disable=broad-except
+        if isinstance(exc, UnicodeError) and is_py2:
+            logger.exception(
+                "unicode is not supported in DVC for Python 2 "
+                "(end-of-life January 1, 2020), please upgrade to Python 3"
+            )
+        else:
+            logger.exception("unexpected error")
         ret = 255
     finally:
         logger.setLevel(outerLogLevel)
+
         # Python 2 fails to close these clean occasionally and users see
         # weird error messages, so we do it manually
         close_pools()
 
+        # Remove cached repos in the end of the call, these are anonymous
+        # so won't be reused by any other subsequent run anyway.
+        clean_repos()
+
     Analytics().send_cmd(cmd, args, ret)
 
     return ret

dvc/output/base.py

@@ -32,6 +32,13 @@ class OutputAlreadyTrackedError(DvcException):
         super(OutputAlreadyTrackedError, self).__init__(msg)
 
 
+class OutputIsStageFileError(DvcException):
+    def __init__(self, path):
+        super(OutputIsStageFileError, self).__init__(
+            "Stage file '{}' cannot be an output.".format(path)
+        )
+
+
 class OutputBase(object):
     IS_DEPENDENCY = False
 
@@ -57,6 +64,7 @@ class OutputBase(object):
 
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
+    IsStageFileError = OutputIsStageFileError
 
     sep = "/"
 
@@ -71,6 +79,7 @@ class OutputBase(object):
         persist=False,
         tags=None,
     ):
+        self._validate_output_path(path)
         # This output (and dependency) objects have too many paths/urls
         # here is a list and comments:
         #
@@ -278,14 +287,14 @@ class OutputBase(object):
     def checkout(self, force=False, progress_callback=None, tag=None):
         if not self.use_cache:
             progress_callback(str(self.path_info), self.get_files_number())
-            return
+            return None
 
         if tag:
             info = self.tags[tag]
         else:
             info = self.info
 
-        self.cache.checkout(
+        return self.cache.checkout(
             self.path_info,
             info,
             force=force,
@@ -393,9 +402,6 @@ class OutputBase(object):
         include the `info` of its files.
         """
 
-        if self.stage.is_repo_import:
-            return []
-
         if not self.use_cache:
             return []
 
@@ -415,3 +421,10 @@ class OutputBase(object):
         ret.extend(self._collect_used_dir_cache(**kwargs))
 
         return ret
+
+    @classmethod
+    def _validate_output_path(cls, path):
+        from dvc.stage import Stage
+
+        if Stage.is_valid_filename(path):
+            raise cls.IsStageFileError(path)

dvc/path_info.py

@@ -36,6 +36,15 @@ class PathInfo(pathlib.PurePath):
     def from_posix(cls, s):
         return cls(PosixPathInfo(s))
 
+    def as_posix(self):
+        f = self._flavour
+        # Unlike original implementation [1] that uses `str()` we actually need
+        # to use `fspath`, because we've overriden `__str__` method to return
+        # relative paths, which will break original `as_posix`.
+        #
+        # [1] https://github.com/python/cpython/blob/v3.7.0/Lib/pathlib.py#L692
+        return self.fspath.replace(f.sep, "/")
+
     def __str__(self):
         path = self.__fspath__()
         return relpath(path)
@@ -94,10 +103,6 @@ class PathInfo(pathlib.PurePath):
         def with_name(self, name):
             return pathlib.PurePath.with_name(self, name.encode(fs_encoding))
 
-        def as_posix(self):
-            f = self._flavour
-            return str(self).replace(f.sep, "/")
-
 
 class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
     pass

dvc/remote/azure.py

@@ -10,12 +10,6 @@ from funcy import cached_property
 
 from dvc.scheme import Schemes
 
-try:
-    from azure.storage.blob import BlockBlobService, BlobPermissions
-    from azure.common import AzureMissingResourceHttpError
-except ImportError:
-    BlockBlobService = None
-
 from dvc.utils.compat import urlparse
 from dvc.progress import Tqdm
 from dvc.config import Config
@@ -35,7 +29,7 @@ class RemoteAZURE(RemoteBASE):
         r"(ContainerName=(?P<container_name>[^;]+);?)?"
         r"(?P<connection_string>.+)?)?)$"
     )
-    REQUIRES = {"azure-storage-blob": BlockBlobService}
+    REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
 
@@ -72,6 +66,9 @@ class RemoteAZURE(RemoteBASE):
 
     @cached_property
     def blob_service(self):
+        from azure.storage.blob import BlockBlobService
+        from azure.common import AzureMissingResourceHttpError
+
         logger.debug("URL {}".format(self.path_info))
         logger.debug("Connection string {}".format(self.connection_string))
         blob_service = BlockBlobService(
@@ -139,6 +136,8 @@ class RemoteAZURE(RemoteBASE):
         return any(path_info.path == path for path in paths)
 
     def _generate_download_url(self, path_info, expires=3600):
+        from azure.storage.blob import BlobPermissions
+
         expires_at = datetime.utcnow() + timedelta(seconds=expires)
 
         sas_token = self.blob_service.generate_blob_shared_access_signature(

dvc/remote/base.py

@@ -1,9 +1,8 @@
 from __future__ import unicode_literals
 
 from dvc.ignore import DvcIgnore
-from dvc.utils.compat import str, basestring, urlparse
+from dvc.utils.compat import str, basestring, urlparse, FileNotFoundError
 
-import os
 import json
 import logging
 import tempfile
@@ -12,6 +11,8 @@ from operator import itemgetter
 from multiprocessing import cpu_count
 from functools import partial
 from concurrent.futures import ThreadPoolExecutor
+from copy import copy
+from dvc.remote.slow_link_detection import slow_link_guard
 
 import dvc.prompt as prompt
 from dvc.config import Config
@@ -20,7 +21,7 @@ from dvc.exceptions import (
     ConfirmRemoveError,
     DvcIgnoreInCollectedDirError,
 )
-from dvc.progress import Tqdm, TqdmThreadPoolExecutor
+from dvc.progress import Tqdm
 from dvc.utils import LARGE_DIR_SIZE, tmp_fname, move, relpath, makedirs
 from dvc.state import StateNoop
 from dvc.path_info import PathInfo, URLInfo
@@ -70,6 +71,14 @@ class RemoteMissingDepsError(DvcException):
     pass
 
 
+class DirCacheError(DvcException):
+    def __init__(self, checksum, cause=None):
+        super(DirCacheError, self).__init__(
+            "Failed to load dir cache for checksum: '{}'.".format(checksum),
+            cause=cause,
+        )
+
+
 class RemoteBASE(object):
     scheme = "base"
     path_cls = URLInfo
@@ -79,36 +88,14 @@ class RemoteBASE(object):
     PARAM_RELPATH = "relpath"
     CHECKSUM_DIR_SUFFIX = ".dir"
     CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
+    DEFAULT_CACHE_TYPES = ["copy"]
 
     state = StateNoop()
 
     def __init__(self, repo, config):
         self.repo = repo
-        deps_ok = all(self.REQUIRES.values())
-        if not deps_ok:
-            missing = [k for k, v in self.REQUIRES.items() if v is None]
-            url = config.get(
-                Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
-            )
-            msg = (
-                "URL '{}' is supported but requires these missing "
-                "dependencies: {}. If you have installed dvc using pip, "
-                "choose one of these options to proceed: \n"
-                "\n"
-                "    1) Install specific missing dependencies:\n"
-                "        pip install {}\n"
-                "    2) Install dvc package that includes those missing "
-                "dependencies: \n"
-                "        pip install 'dvc[{}]'\n"
-                "    3) Install dvc package with all possible "
-                "dependencies included: \n"
-                "        pip install 'dvc[all]'\n"
-                "\n"
-                "If you have installed dvc from a binary package and you "
-                "are still seeing this message, please report it to us "
-                "using https://github.com/iterative/dvc/issues. Thank you!"
-            ).format(url, missing, " ".join(missing), self.scheme)
-            raise RemoteMissingDepsError(msg)
+
+        self._check_requires(config)
 
         core = config.get(Config.SECTION_CORE, {})
         self.checksum_jobs = core.get(
@@ -119,6 +106,52 @@ class RemoteBASE(object):
         self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE, True)
         self._dir_info = {}
 
+        types = config.get(Config.SECTION_CACHE_TYPE, None)
+        if types:
+            if isinstance(types, str):
+                types = [t.strip() for t in types.split(",")]
+            self.cache_types = types
+        else:
+            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
+        self.cache_type_confirmed = False
+
+    def _check_requires(self, config):
+        import importlib
+
+        missing = []
+
+        for package, module in self.REQUIRES.items():
+            try:
+                importlib.import_module(module)
+            except ImportError:
+                missing.append(package)
+
+        if not missing:
+            return
+
+        url = config.get(
+            Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
+        )
+        msg = (
+            "URL '{}' is supported but requires these missing "
+            "dependencies: {}. If you have installed dvc using pip, "
+            "choose one of these options to proceed: \n"
+            "\n"
+            "    1) Install specific missing dependencies:\n"
+            "        pip install {}\n"
+            "    2) Install dvc package that includes those missing "
+            "dependencies: \n"
+            "        pip install 'dvc[{}]'\n"
+            "    3) Install dvc package with all possible "
+            "dependencies included: \n"
+            "        pip install 'dvc[all]'\n"
+            "\n"
+            "If you have installed dvc from a binary package and you "
+            "are still seeing this message, please report it to us "
+            "using https://github.com/iterative/dvc/issues. Thank you!"
+        ).format(url, missing, " ".join(missing), self.scheme)
+        raise RemoteMissingDepsError(msg)
+
     def __repr__(self):
         return "{class_name}: '{path_info}'".format(
             class_name=type(self).__name__,
@@ -145,9 +178,7 @@ class RemoteBASE(object):
 
     def _calculate_checksums(self, file_infos):
         file_infos = list(file_infos)
-        with TqdmThreadPoolExecutor(
-            max_workers=self.checksum_jobs
-        ) as executor:
+        with ThreadPoolExecutor(max_workers=self.checksum_jobs) as executor:
             tasks = executor.map(self.get_file_checksum, file_infos)
 
             if len(file_infos) > LARGE_DIR_SIZE:
@@ -231,30 +262,26 @@ class RemoteBASE(object):
         if dir_info:
             return dir_info
 
-        dir_info = self.load_dir_cache(checksum)
+        try:
+            dir_info = self.load_dir_cache(checksum)
+        except DirCacheError:
+            dir_info = []
+
         self._dir_info[checksum] = dir_info
         return dir_info
 
     def load_dir_cache(self, checksum):
         path_info = self.checksum_to_path_info(checksum)
 
-        fobj = tempfile.NamedTemporaryFile(delete=False)
-        path = fobj.name
-        to_info = PathInfo(path)
-        self.cache.download(path_info, to_info, no_progress_bar=True)
-
         try:
-            with open(path, "r") as fobj:
+            with self.cache.open(path_info, "r") as fobj:
                 d = json.load(fobj)
-        except ValueError:
-            logger.exception("Failed to load dir cache '{}'".format(path_info))
-            return []
-        finally:
-            os.unlink(path)
+        except (ValueError, FileNotFoundError) as exc:
+            raise DirCacheError(checksum, cause=exc)
 
         if not isinstance(d, list):
             msg = "dir cache file format error '{}' [skipping the file]"
-            logger.error(msg.format(relpath(path)))
+            logger.error(msg.format(relpath(path_info)))
             return []
 
         for info in d:
@@ -352,7 +379,47 @@ class RemoteBASE(object):
         return False
 
     def link(self, from_info, to_info):
-        self.copy(from_info, to_info)
+        self._link(from_info, to_info, self.cache_types)
+
+    def _link(self, from_info, to_info, link_types):
+        assert self.isfile(from_info)
+
+        self.makedirs(to_info.parent)
+
+        self._try_links(from_info, to_info, link_types)
+
+    @slow_link_guard
+    def _try_links(self, from_info, to_info, link_types):
+        while link_types:
+            link_method = getattr(self, link_types[0])
+            try:
+                self._do_link(from_info, to_info, link_method)
+                self.cache_type_confirmed = True
+                return
+
+            except DvcException as exc:
+                msg = "Cache type '{}' is not supported: {}"
+                logger.debug(msg.format(link_types[0], str(exc)))
+                del link_types[0]
+
+        raise DvcException("no possible cache types left to try out.")
+
+    def _do_link(self, from_info, to_info, link_method):
+        if self.exists(to_info):
+            raise DvcException("Link '{}' already exists!".format(to_info))
+
+        link_method(from_info, to_info)
+
+        if self.protected:
+            self.protect(to_info)
+
+        msg = "Created {}'{}': {} -> {}".format(
+            "protected " if self.protected else "",
+            self.cache_types[0],
+            from_info,
+            to_info,
+        )
+        logger.debug(msg)
 
     def _save_file(self, path_info, checksum, save_link=True):
         assert checksum
@@ -391,9 +458,15 @@ class RemoteBASE(object):
         return False
 
     def isfile(self, path_info):
-        raise NotImplementedError
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return True
 
     def isdir(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
         return False
 
     def walk(self, path_info):
@@ -419,7 +492,7 @@ class RemoteBASE(object):
 
     def _save(self, path_info, checksum):
         to_info = self.checksum_to_path_info(checksum)
-        logger.info("Saving '{}' to '{}'.".format(path_info, to_info))
+        logger.debug("Saving '{}' to '{}'.".format(path_info, to_info))
         if self.isdir(path_info):
             self._save_dir(path_info, checksum)
             return
@@ -512,6 +585,15 @@ class RemoteBASE(object):
     def copy(self, from_info, to_info):
         raise RemoteActionNotImplemented("copy", self.scheme)
 
+    def symlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("symlink", self.scheme)
+
+    def hardlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("hardlink", self.scheme)
+
+    def reflink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("reflink", self.scheme)
+
     def exists(self, path_info):
         raise NotImplementedError
 
@@ -720,7 +802,10 @@ class RemoteBASE(object):
         return True
 
     def makedirs(self, path_info):
-        raise NotImplementedError
+        """Optional: Implement only if the remote needs to create
+        directories before copying/linking/moving data
+        """
+        pass
 
     def _checkout_dir(
         self, path_info, checksum, force, progress_callback=None
@@ -772,37 +857,33 @@ class RemoteBASE(object):
             raise NotImplementedError
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
-        skip = False
+        failed = None
         if not checksum:
             logger.warning(
                 "No checksum info found for '{}'. "
                 "It won't be created.".format(str(path_info))
             )
             self.safe_remove(path_info, force=force)
-            skip = True
-
-        elif not self.changed(path_info, checksum_info):
-            msg = "Data '{}' didn't change."
-            logger.debug(msg.format(str(path_info)))
-            skip = True
+            failed = path_info
 
         elif self.changed_cache(checksum):
             msg = "Cache '{}' not found. File '{}' won't be created."
             logger.warning(msg.format(checksum, str(path_info)))
             self.safe_remove(path_info, force=force)
-            skip = True
+            failed = path_info
 
-        if skip:
+        if failed:
             if progress_callback:
                 progress_callback(
                     str(path_info), self.get_files_number(checksum)
                 )
-            return
+            return failed
 
         msg = "Checking out '{}' with cache '{}'."
         logger.debug(msg.format(str(path_info), checksum))
 
         self._checkout(path_info, checksum, force, progress_callback)
+        return None
 
     def _checkout(
         self, path_info, checksum, force=False, progress_callback=None

dvc/remote/gs.py

@@ -5,11 +5,6 @@ from datetime import timedelta
 from funcy import cached_property
 from dvc.utils.compat import FileNotFoundError
 
-try:
-    from google.cloud import storage
-except ImportError:
-    storage = None
-
 from dvc.remote.base import RemoteBASE
 from dvc.config import Config
 from dvc.exceptions import DvcException
@@ -22,7 +17,7 @@ logger = logging.getLogger(__name__)
 class RemoteGS(RemoteBASE):
     scheme = Schemes.GS
     path_cls = CloudURLInfo
-    REQUIRES = {"google.cloud.storage": storage}
+    REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
     PARAM_CHECKSUM = "md5"
 
     def __init__(self, repo, config):
@@ -37,10 +32,12 @@ class RemoteGS(RemoteBASE):
 
     @cached_property
     def gs(self):
+        from google.cloud.storage import Client
+
         return (
-            storage.Client.from_service_account_json(self.credentialpath)
+            Client.from_service_account_json(self.credentialpath)
             if self.credentialpath
-            else storage.Client(self.projectname)
+            else Client(self.projectname)
         )
 
     def get_file_checksum(self, path_info):

dvc/remote/hdfs.py

@@ -3,17 +3,11 @@ from __future__ import unicode_literals
 import os
 import re
 import io
-import getpass
 import posixpath
 import logging
 from subprocess import Popen, PIPE
 from contextlib import contextmanager, closing
 
-try:
-    import pyarrow
-except ImportError:
-    pyarrow = None
-
 from dvc.config import Config
 from dvc.scheme import Schemes
 
@@ -30,7 +24,7 @@ class RemoteHDFS(RemoteBASE):
     scheme = Schemes.HDFS
     REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
     PARAM_CHECKSUM = "checksum"
-    REQUIRES = {"pyarrow": pyarrow}
+    REQUIRES = {"pyarrow": "pyarrow"}
 
     def __init__(self, repo, config):
         super(RemoteHDFS, self).__init__(repo, config)
@@ -41,11 +35,7 @@ class RemoteHDFS(RemoteBASE):
 
         parsed = urlparse(url)
 
-        user = (
-            parsed.username
-            or config.get(Config.SECTION_REMOTE_USER)
-            or getpass.getuser()
-        )
+        user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
 
         self.path_info = self.path_cls.from_parts(
             scheme=self.scheme,
@@ -55,8 +45,9 @@ class RemoteHDFS(RemoteBASE):
             path=parsed.path,
         )
 
-    @staticmethod
-    def hdfs(path_info):
+    def hdfs(self, path_info):
+        import pyarrow
+
         return get_connection(
             pyarrow.hdfs.connect,
             path_info.host,

dvc/remote/http.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 from dvc.scheme import Schemes
 from dvc.utils.compat import open
 
-import requests
 import logging
 
 from dvc.progress import Tqdm
@@ -79,6 +78,8 @@ class RemoteHTTP(RemoteBASE):
         return etag
 
     def _request(self, method, url, **kwargs):
+        import requests
+
         kwargs.setdefault("allow_redirects", True)
         kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 

dvc/remote/local.py

@@ -1,16 +1,12 @@
 from __future__ import unicode_literals
 
-from copy import copy
-
 from dvc.scheme import Schemes
-from dvc.remote.local.slow_link_detection import slow_link_guard
 from dvc.utils.compat import str, fspath_py35, open
 
 import os
 import stat
 import errno
 from shortuuid import uuid
-import shutil
 import logging
 from functools import partial
 
@@ -34,7 +30,7 @@ from dvc.utils import (
     makedirs,
 )
 from dvc.config import Config
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, DownloadError, UploadError
 from dvc.progress import Tqdm, TqdmThreadPoolExecutor
 
 from dvc.path_info import PathInfo
@@ -51,12 +47,6 @@ class RemoteLOCAL(RemoteBASE):
     UNPACKED_DIR_SUFFIX = ".unpacked"
 
     DEFAULT_CACHE_TYPES = ["reflink", "copy"]
-    CACHE_TYPE_MAP = {
-        "copy": shutil.copyfile,
-        "symlink": System.symlink,
-        "hardlink": System.hardlink,
-        "reflink": System.reflink,
-    }
 
     SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
 
@@ -71,15 +61,6 @@ class RemoteLOCAL(RemoteBASE):
             # cache files are set to be read-only for everyone
             self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
 
-        types = config.get(Config.SECTION_CACHE_TYPE, None)
-        if types:
-            if isinstance(types, str):
-                types = [t.strip() for t in types.split(",")]
-            self.cache_types = types
-        else:
-            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
-        self.cache_type_confirmed = False
-
         # A clunky way to detect cache dir
         storagepath = config.get(Config.SECTION_LOCAL_STORAGEPATH, None)
         cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
@@ -88,9 +69,6 @@ class RemoteLOCAL(RemoteBASE):
             cwd = config[Config.PRIVATE_CWD]
             cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
 
-        if cache_dir is not None and not os.path.exists(cache_dir):
-            os.mkdir(cache_dir)
-
         self.path_info = PathInfo(cache_dir) if cache_dir else None
         self._dir_info = {}
 
@@ -126,80 +104,14 @@ class RemoteLOCAL(RemoteBASE):
 
         return self.checksum_to_path_info(md5).url
 
-    def exists(self, path_info):
+    @staticmethod
+    def exists(path_info):
         assert path_info.scheme == "local"
         return os.path.lexists(fspath_py35(path_info))
 
     def makedirs(self, path_info):
         makedirs(path_info, exist_ok=True, mode=self._dir_mode)
 
-    def link(self, from_info, to_info):
-        self._link(from_info, to_info, self.cache_types)
-
-    def _link(self, from_info, to_info, link_types):
-        from_path = from_info.fspath
-        to_path = to_info.fspath
-
-        assert os.path.isfile(from_path)
-
-        dname = os.path.dirname(to_path)
-        if not os.path.exists(dname):
-            os.makedirs(dname)
-
-        # NOTE: just create an empty file for an empty cache
-        if os.path.getsize(from_path) == 0:
-            open(to_path, "w+").close()
-
-            msg = "Created empty file: {} -> {}".format(from_path, to_path)
-            logger.debug(msg)
-            return
-
-        self._try_links(from_info, to_info, link_types)
-
-    @classmethod
-    def _get_link_method(cls, link_type):
-        try:
-            return cls.CACHE_TYPE_MAP[link_type]
-        except KeyError:
-            raise DvcException(
-                "Cache type: '{}' not supported!".format(link_type)
-            )
-
-    def _do_link(self, from_info, to_info, link_method):
-        if self.exists(to_info):
-            raise DvcException("Link '{}' already exists!".format(to_info))
-        else:
-            link_method(from_info.fspath, to_info.fspath)
-
-        if self.protected:
-            self.protect(to_info)
-
-        msg = "Created {}'{}': {} -> {}".format(
-            "protected " if self.protected else "",
-            self.cache_types[0],
-            from_info,
-            to_info,
-        )
-        logger.debug(msg)
-
-    @slow_link_guard
-    def _try_links(self, from_info, to_info, link_types):
-        i = len(link_types)
-        while i > 0:
-            link_method = self._get_link_method(link_types[0])
-            try:
-                self._do_link(from_info, to_info, link_method)
-                self.cache_type_confirmed = True
-                return
-
-            except DvcException as exc:
-                msg = "Cache type '{}' is not supported: {}"
-                logger.debug(msg.format(link_types[0], str(exc)))
-                del link_types[0]
-                i -= 1
-
-        raise DvcException("no possible cache types left to try out.")
-
     def already_cached(self, path_info):
         assert path_info.scheme in ["", "local"]
 
@@ -221,12 +133,18 @@ class RemoteLOCAL(RemoteBASE):
 
         return False
 
-    def isfile(self, path_info):
+    @staticmethod
+    def isfile(path_info):
         return os.path.isfile(fspath_py35(path_info))
 
-    def isdir(self, path_info):
+    @staticmethod
+    def isdir(path_info):
         return os.path.isdir(fspath_py35(path_info))
 
+    @staticmethod
+    def getsize(path_info):
+        return os.path.getsize(fspath_py35(path_info))
+
     def walk(self, path_info):
         return dvc_walk(path_info, self.repo.dvcignore)
 
@@ -253,6 +171,44 @@ class RemoteLOCAL(RemoteBASE):
 
         move(from_info, to_info, mode=mode)
 
+    @staticmethod
+    def copy(from_info, to_info):
+        System.copy(from_info, to_info)
+
+    @staticmethod
+    def symlink(from_info, to_info):
+        System.symlink(from_info, to_info)
+
+    def hardlink(self, from_info, to_info):
+        # If there are a lot of empty files (which happens a lot in datasets),
+        # and the cache type is `hardlink`, we might reach link limits and
+        # will get something like: `too many links error`
+        #
+        # This is because all those empty files will have the same checksum
+        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therfore, they will be
+        # linked to the same file in the cache.
+        #
+        # From https://en.wikipedia.org/wiki/Hard_link
+        #   * ext4 limits the number of hard links on a file to 65,000
+        #   * Windows with NTFS has a limit of 1024 hard links on a file
+        #
+        # That's why we simply create an empty file rather than a link.
+        if self.getsize(from_info) == 0:
+            self.open(to_info, "w").close()
+
+            logger.debug(
+                "Created empty file: {src} -> {dest}".format(
+                    src=str(from_info), dest=str(to_info)
+                )
+            )
+            return
+
+        System.hardlink(from_info, to_info)
+
+    @staticmethod
+    def reflink(from_info, to_info):
+        System.reflink(from_info, to_info)
+
     def cache_exists(self, checksums, jobs=None, name=None):
         return [
             checksum
@@ -286,8 +242,8 @@ class RemoteLOCAL(RemoteBASE):
             name=name,
         )
 
-    def open(self, path_info, mode="r", encoding=None):
-        assert mode in {"r", "rt", "rb"}
+    @staticmethod
+    def open(path_info, mode="r", encoding=None):
         return open(fspath_py35(path_info), mode=mode, encoding=encoding)
 
     def _group(self, checksum_infos, show_checksums=False):
@@ -429,10 +385,9 @@ class RemoteLOCAL(RemoteBASE):
             fails = sum(map(func, *plans))
 
         if fails:
-            msg = "{} files failed to {}"
-            raise DvcException(
-                msg.format(fails, "download" if download else "upload")
-            )
+            if download:
+                raise DownloadError(fails)
+            raise UploadError(fails)
 
         return len(plans[0])
 

dvc/remote/local/slow_link_detection.py

@@ -1,62 +0,0 @@
-import logging
-import time
-
-import colorama
-from dvc.config import Config
-
-logger = logging.getLogger(__name__)
-
-
-class SlowLinkDetectorDecorator(object):
-    LINKING_TIMEOUT_SECONDS = 10.0
-    was_displayed = False
-
-    @classmethod
-    def should_display(cls):
-        if not cls.was_displayed:
-            cls.was_displayed = True
-            return True
-        return False
-
-    def __init__(self, method):
-        self.method = method
-
-    def __call__(self, *args, **kwargs):
-        start = time.time()
-        result = self.method(*args, **kwargs)
-        execution_time_seconds = time.time() - start
-
-        if (
-            execution_time_seconds >= self.LINKING_TIMEOUT_SECONDS
-            and self.should_display()
-        ):
-            msg = (
-                "You can cut execution time considerably by using a different"
-                " `cache.type` configuration.\nSee "
-                "{blue}https://dvc.org/doc/commands-reference/config#cache{"
-                "reset} for more information.\n"
-                "To disable this message, run:\n"
-                "'dvc config cache.slow_link_warning false'".format(
-                    blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
-                )
-            )
-            logger.warning(msg)
-
-        return result
-
-
-def slow_link_guard(method):
-    def call(remote_local, *args, **kwargs):
-        cache_config = remote_local.repo.config.config.get(
-            Config.SECTION_CACHE, {}
-        )
-        should_warn = cache_config.get(
-            Config.SECTION_CACHE_SLOW_LINK_WARNING, True
-        ) and not cache_config.get(Config.SECTION_CACHE_TYPE, None)
-
-        if should_warn:
-            decorated = SlowLinkDetectorDecorator(method)
-            return decorated(remote_local, *args, **kwargs)
-        return method(remote_local, *args, **kwargs)
-
-    return call

dvc/remote/oss.py

@@ -6,11 +6,6 @@ import logging
 
 from dvc.scheme import Schemes
 
-try:
-    import oss2
-except ImportError:
-    oss2 = None
-
 from dvc.config import Config
 from dvc.remote.base import RemoteBASE
 from dvc.progress import Tqdm
@@ -41,7 +36,7 @@ class RemoteOSS(RemoteBASE):
 
     scheme = Schemes.OSS
     path_cls = CloudURLInfo
-    REQUIRES = {"oss2": oss2}
+    REQUIRES = {"oss2": "oss2"}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
 
@@ -71,6 +66,8 @@ class RemoteOSS(RemoteBASE):
 
     @property
     def oss_service(self):
+        import oss2
+
         if self._bucket is None:
             logger.debug("URL {}".format(self.path_info))
             logger.debug("key id {}".format(self.key_id))
@@ -98,6 +95,8 @@ class RemoteOSS(RemoteBASE):
         self.oss_service.delete_object(path_info.path)
 
     def _list_paths(self, prefix):
+        import oss2
+
         for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
             yield blob.key
 

dvc/remote/s3.py

@@ -4,11 +4,6 @@ import os
 import logging
 from funcy import cached_property
 
-try:
-    import boto3
-except ImportError:
-    boto3 = None
-
 from dvc.progress import Tqdm
 from dvc.config import Config
 from dvc.remote.base import RemoteBASE
@@ -22,7 +17,7 @@ logger = logging.getLogger(__name__)
 class RemoteS3(RemoteBASE):
     scheme = Schemes.S3
     path_cls = CloudURLInfo
-    REQUIRES = {"boto3": boto3}
+    REQUIRES = {"boto3": "boto3"}
     PARAM_CHECKSUM = "etag"
 
     def __init__(self, repo, config):
@@ -61,6 +56,8 @@ class RemoteS3(RemoteBASE):
 
     @cached_property
     def s3(self):
+        import boto3
+
         session = boto3.session.Session(
             profile_name=self.profile, region_name=self.region
         )

dvc/remote/slow_link_detection.py

@@ -0,0 +1,50 @@
+import logging
+import time
+import sys
+from functools import wraps
+
+import colorama
+from dvc.config import Config
+
+
+logger = logging.getLogger(__name__)
+this = sys.modules[__name__]
+
+this.timeout_seconds = 10.0
+this.already_displayed = False
+this.message = (
+    "You can cut execution time considerably by using a different "
+    "`cache.type` configuration.\n"
+    "See {blue}https://dvc.org/doc/commands-reference/config#cache{reset} "
+    "for more information.\n"
+    "To disable this message, run:\n"
+    "'dvc config cache.slow_link_warning false'".format(
+        blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
+    )
+)
+
+
+def slow_link_guard(f):
+    @wraps(f)
+    def wrapper(remote, *args, **kwargs):
+        if this.already_displayed:
+            return f(remote, *args, **kwargs)
+
+        config = remote.repo.config.config.get(Config.SECTION_CACHE, {})
+        cache_type = config.get(Config.SECTION_CACHE_TYPE)
+        should_warn = config.get(Config.SECTION_CACHE_SLOW_LINK_WARNING, True)
+
+        if not should_warn or cache_type:
+            return f(remote, *args, **kwargs)
+
+        start = time.time()
+        result = f(remote, *args, **kwargs)
+        delta = time.time() - start
+
+        if delta >= this.timeout_seconds:
+            logger.warning(this.message)
+            this.already_displayed = True
+
+        return result
+
+    return wrapper

dvc/remote/ssh/__init__.py

@@ -10,11 +10,6 @@ import threading
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager, closing
 
-try:
-    import paramiko
-except ImportError:
-    paramiko = None
-
 import dvc.prompt as prompt
 from dvc.config import Config
 from dvc.utils import to_chunks
@@ -24,9 +19,6 @@ from dvc.scheme import Schemes
 from dvc.remote.pool import get_connection
 from dvc.progress import Tqdm
 
-from .connection import SSHConnection
-
-
 logger = logging.getLogger(__name__)
 
 
@@ -36,7 +28,7 @@ saved_passwords_lock = threading.Lock()
 
 class RemoteSSH(RemoteBASE):
     scheme = Schemes.SSH
-    REQUIRES = {"paramiko": paramiko}
+    REQUIRES = {"paramiko": "paramiko"}
 
     JOBS = 4
     PARAM_CHECKSUM = "md5"
@@ -47,9 +39,10 @@ class RemoteSSH(RemoteBASE):
     # We use conservative setting of 4 instead to not exhaust max sessions.
     CHECKSUM_JOBS = 4
 
+    DEFAULT_CACHE_TYPES = ["copy"]
+
     def __init__(self, repo, config):
         super(RemoteSSH, self).__init__(repo, config)
-
         url = config.get(Config.SECTION_REMOTE_URL)
         if url:
             parsed = urlparse(url)
@@ -95,6 +88,8 @@ class RemoteSSH(RemoteBASE):
 
     @staticmethod
     def _load_user_ssh_config(hostname):
+        import paramiko
+
         user_config_file = RemoteSSH.ssh_config_filename()
         user_ssh_config = {}
         if hostname and os.path.exists(user_config_file):
@@ -138,6 +133,8 @@ class RemoteSSH(RemoteBASE):
                     )
                 self.password = password
 
+        from .connection import SSHConnection
+
         return get_connection(
             SSHConnection,
             host,
@@ -164,12 +161,54 @@ class RemoteSSH(RemoteBASE):
         with self.ssh(path_info) as ssh:
             return ssh.isdir(path_info.path)
 
+    def isfile(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.isfile(path_info.path)
+
+    def getsize(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.getsize(path_info.path)
+
     def copy(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.copy(from_info.path, to_info.path)
+
+    def symlink(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.symlink(from_info.path, to_info.path)
+
+    def hardlink(self, from_info, to_info):
+        if not from_info.scheme == to_info.scheme == self.scheme:
+            raise NotImplementedError
+
+        # See dvc/remote/local/__init__.py - hardlink()
+        if self.getsize(from_info) == 0:
+
+            with self.ssh(to_info) as ssh:
+                ssh.sftp.open(to_info.path, "w").close()
+
+            logger.debug(
+                "Created empty file: {src} -> {dest}".format(
+                    src=str(from_info), dest=str(to_info)
+                )
+            )
+            return
+
+        with self.ssh(from_info) as ssh:
+            ssh.hardlink(from_info.path, to_info.path)
+
+    def reflink(self, from_info, to_info):
         if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
         with self.ssh(from_info) as ssh:
-            ssh.cp(from_info.path, to_info.path)
+            ssh.reflink(from_info.path, to_info.path)
 
     def remove(self, path_info):
         if path_info.scheme != self.scheme:
@@ -207,12 +246,12 @@ class RemoteSSH(RemoteBASE):
 
     @contextmanager
     def open(self, path_info, mode="r", encoding=None):
-        assert mode in {"r", "rt", "rb"}
+        assert mode in {"r", "rt", "rb", "wb"}
 
         with self.ssh(path_info) as ssh, closing(
-            ssh.sftp.file(path_info.path, mode="r")
+            ssh.sftp.open(path_info.path, mode)
         ) as fd:
-            if mode == "rb":
+            if "b" in mode:
                 yield fd
             else:
                 yield io.TextIOWrapper(fd, encoding=encoding)

dvc/remote/ssh/connection.py

@@ -60,7 +60,13 @@ class SSHConnection:
 
     def st_mode(self, path):
         with ignore_file_not_found():
-            return self.sftp.stat(path).st_mode
+            return self.sftp.lstat(path).st_mode
+
+        return 0
+
+    def getsize(self, path):
+        with ignore_file_not_found():
+            return self.sftp.lstat(path).st_size
 
         return 0
 
@@ -100,7 +106,10 @@ class SSHConnection:
                 # Since paramiko errors are very vague we need to recheck
                 # whether it's because path already exists or something else
                 if e.errno == errno.EACCES or not self.exists(path):
-                    raise
+                    raise DvcException(
+                        "unable to create remote directory '{}'".format(path),
+                        cause=e,
+                    )
 
     def walk(self, directory, topdown=True):
         # NOTE: original os.walk() implementation [1] with default options was
@@ -267,8 +276,7 @@ class SSHConnection:
         assert len(md5) == 32
         return md5
 
-    def cp(self, src, dest):
-        self.makedirs(posixpath.dirname(dest))
+    def copy(self, src, dest):
         self.execute("cp {} {}".format(src, dest))
 
     def open_max_sftp_channels(self):
@@ -282,3 +290,23 @@ class SSHConnection:
                         raise
                     break
         return self._sftp_channels
+
+    def open(self, *args, **kwargs):
+        return self.sftp.open(*args, **kwargs)
+
+    def symlink(self, src, dest):
+        self.sftp.symlink(src, dest)
+
+    def reflink(self, src, dest):
+        if self.uname == "Linux":
+            return self.execute("cp --reflink {} {}".format(src, dest))
+
+        if self.uname == "Darwin":
+            return self.execute("cp -c {} {}".format(src, dest))
+
+        raise DvcException(
+            "'{}' is not supported as a SSH remote".format(self.uname)
+        )
+
+    def hardlink(self, src, dest):
+        self.execute("ln {} {}".format(src, dest))

dvc/repo/__init__.py

@@ -5,14 +5,14 @@ import logging
 from contextlib import contextmanager
 from itertools import chain
 
+from functools import wraps
 from funcy import cached_property
 
 from dvc.config import Config
 from dvc.exceptions import (
     NotDvcRepoError,
     OutputNotFoundError,
-    TargetNotDirectoryError,
-    OutputFileMissingError,
+    FileMissingError,
 )
 from dvc.ignore import DvcIgnoreFilter
 from dvc.path_info import PathInfo
@@ -20,9 +20,24 @@ from dvc.remote.base import RemoteActionNotImplemented
 from dvc.utils.compat import open as _open, fspath_py35, FileNotFoundError
 from dvc.utils import relpath
 
+from .graph import check_acyclic, get_pipeline, get_pipelines, get_stages
+
+
 logger = logging.getLogger(__name__)
 
 
+def locked(f):
+    @wraps(f)
+    def wrapper(repo, *args, **kwargs):
+        with repo.lock, repo.state:
+            ret = f(repo, *args, **kwargs)
+            # Our graph cache is no longer valid after we release the repo.lock
+            repo._reset()
+            return ret
+
+    return wrapper
+
+
 class Repo(object):
     DVC_DIR = ".dvc"
 
@@ -36,9 +51,9 @@ class Repo(object):
     from dvc.repo.imp import imp
     from dvc.repo.imp_url import imp_url
     from dvc.repo.reproduce import reproduce
-    from dvc.repo.checkout import checkout
+    from dvc.repo.checkout import _checkout
     from dvc.repo.push import push
-    from dvc.repo.fetch import fetch
+    from dvc.repo.fetch import _fetch
     from dvc.repo.pull import pull
     from dvc.repo.status import status
     from dvc.repo.gc import gc
@@ -70,7 +85,10 @@ class Repo(object):
 
         self.tree = WorkingTree(self.root_dir)
 
-        self.lock = Lock(self.dvc_dir)
+        self.lock = Lock(
+            os.path.join(self.dvc_dir, "lock"),
+            tmp_dir=os.path.join(self.dvc_dir, "tmp"),
+        )
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
         self.state = State(self, self.config.config)
@@ -89,6 +107,17 @@ class Repo(object):
 
         self._ignore()
 
+    @property
+    def tree(self):
+        return self._tree
+
+    @tree.setter
+    def tree(self, tree):
+        self._tree = tree
+        # Our graph cache is no longer valid, as it was based on the previous
+        # tree.
+        self._reset()
+
     def __repr__(self):
         return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
 
@@ -128,56 +157,55 @@ class Repo(object):
 
         updater = Updater(self.dvc_dir)
 
-        flist = [
-            self.lock.lock_file,
-            self.config.config_local_file,
-            updater.updater_file,
-            updater.lock.lock_file,
-        ] + self.state.files
+        flist = (
+            [self.config.config_local_file, updater.updater_file]
+            + self.state.files
+            + self.lock.files
+            + updater.lock.files
+        )
 
-        if self.cache.local.cache_dir.startswith(self.root_dir):
+        if self.cache.local.cache_dir.startswith(self.root_dir + os.sep):
             flist += [self.cache.local.cache_dir]
 
         self.scm.ignore_list(flist)
 
-    def check_dag(self, stages):
+    def check_modified_graph(self, new_stages):
         """Generate graph including the new stage to check for errors"""
-        self.graph(stages=stages)
+        self._collect_graph(self.stages + new_stages)
 
-    @staticmethod
-    def _check_cyclic_graph(graph):
-        import networkx as nx
-        from dvc.exceptions import CyclicGraphError
-
-        cycles = list(nx.simple_cycles(graph))
-
-        if cycles:
-            raise CyclicGraphError(cycles[0])
-
-    def _get_pipeline(self, node):
-        pipelines = [i for i in self.pipelines() if i.has_node(node)]
-        assert len(pipelines) == 1
-        return pipelines[0]
-
-    def collect(self, target, with_deps=False, recursive=False):
+    def collect(self, target, with_deps=False, recursive=False, graph=None):
         import networkx as nx
         from dvc.stage import Stage
 
-        if not target or (recursive and os.path.isdir(target)):
-            return self.stages(target)
+        G = graph or self.graph
+
+        if not target:
+            return get_stages(G)
+
+        target = os.path.abspath(target)
+
+        if recursive and os.path.isdir(target):
+            attrs = nx.get_node_attributes(G, "stage")
+            nodes = [node for node in nx.dfs_postorder_nodes(G)]
+
+            ret = []
+            for node in nodes:
+                stage = attrs[node]
+                if stage.path.startswith(target + os.sep):
+                    ret.append(stage)
+            return ret
 
         stage = Stage.load(self, target)
         if not with_deps:
             return [stage]
 
         node = relpath(stage.path, self.root_dir)
-        G = self._get_pipeline(node)
+        pipeline = get_pipeline(get_pipelines(G), node)
 
-        ret = []
-        for n in nx.dfs_postorder_nodes(G, node):
-            ret.append(G.node[n]["stage"])
-
-        return ret
+        return [
+            pipeline.node[n]["stage"]
+            for n in nx.dfs_postorder_nodes(pipeline, node)
+        ]
 
     def used_cache(
         self,
@@ -211,6 +239,7 @@ class Repo(object):
         cache["hdfs"] = []
         cache["ssh"] = []
         cache["azure"] = []
+        cache["repo"] = []
 
         for branch in self.brancher(
             all_branches=all_branches, all_tags=all_tags
@@ -218,16 +247,18 @@ class Repo(object):
             if targets:
                 stages = []
                 for target in targets:
-                    if recursive and os.path.isdir(target):
-                        stages.extend(self.stages(target))
-                    else:
-                        stages.extend(
-                            self.collect(target, with_deps=with_deps)
-                        )
+                    collected = self.collect(
+                        target, recursive=recursive, with_deps=with_deps
+                    )
+                    stages.extend(collected)
             else:
-                stages = self.stages()
+                stages = self.stages
 
             for stage in stages:
+                if stage.is_repo_import:
+                    cache["repo"] += stage.deps
+                    continue
+
                 for out in stage.outs:
                     scheme = out.path_info.scheme
                     used_cache = out.get_used_cache(
@@ -240,7 +271,7 @@ class Repo(object):
 
         return cache
 
-    def graph(self, stages=None, from_directory=None):
+    def _collect_graph(self, stages=None):
         """Generate a graph by using the given stages on the given directory
 
         The nodes of the graph are the stage's path relative to the root.
@@ -270,11 +301,8 @@ class Repo(object):
               (weakly connected components)
 
         Args:
-            stages (list): used to build a graph, if None given, use the ones
-                on the `from_directory`.
-
-            from_directory (str): directory where to look at for stages, if
-                None is given, use the current working directory
+            stages (list): used to build a graph, if None given, collect stages
+                in the repository.
 
         Raises:
             OutputDuplicationError: two outputs with the same path
@@ -290,8 +318,7 @@ class Repo(object):
         )
 
         G = nx.DiGraph()
-        G_active = nx.DiGraph()
-        stages = stages or self.stages(from_directory, check_dag=False)
+        stages = stages or self.collect_stages()
         stages = [stage for stage in stages if stage]
         outs = {}
 
@@ -318,7 +345,6 @@ class Repo(object):
             node = relpath(stage.path, self.root_dir)
 
             G.add_node(node, stage=stage)
-            G_active.add_node(node, stage=stage)
 
             for dep in stage.deps:
                 if dep.path_info is None:
@@ -334,22 +360,18 @@ class Repo(object):
                         dep_node = relpath(dep_stage.path, self.root_dir)
                         G.add_node(dep_node, stage=dep_stage)
                         G.add_edge(node, dep_node)
-                        if not stage.locked:
-                            G_active.add_node(dep_node, stage=dep_stage)
-                            G_active.add_edge(node, dep_node)
 
-        self._check_cyclic_graph(G)
+        check_acyclic(G)
 
-        return G, G_active
+        return G
 
-    def pipelines(self, from_directory=None):
-        import networkx as nx
+    @cached_property
+    def graph(self):
+        return self._collect_graph()
 
-        G, G_active = self.graph(from_directory=from_directory)
-
-        return [
-            G.subgraph(c).copy() for c in nx.weakly_connected_components(G)
-        ]
+    @cached_property
+    def pipelines(self):
+        return get_pipelines(self.graph)
 
     @staticmethod
     def _filter_out_dirs(dirs, outs, root_dir):
@@ -362,7 +384,7 @@ class Repo(object):
 
         return list(filter(filter_dirs, dirs))
 
-    def stages(self, from_directory=None, check_dag=True):
+    def collect_stages(self):
         """
         Walks down the root directory looking for Dvcfiles,
         skipping the directories that are related with
@@ -374,16 +396,11 @@ class Repo(object):
         """
         from dvc.stage import Stage
 
-        if not from_directory:
-            from_directory = self.root_dir
-        elif not os.path.isdir(from_directory):
-            raise TargetNotDirectoryError(from_directory)
-
         stages = []
         outs = []
 
         for root, dirs, files in self.tree.walk(
-            from_directory, dvcignore=self.dvcignore
+            self.root_dir, dvcignore=self.dvcignore
         ):
             for fname in files:
                 path = os.path.join(root, fname)
@@ -397,18 +414,15 @@ class Repo(object):
 
             dirs[:] = self._filter_out_dirs(dirs, outs, root)
 
-        if check_dag:
-            self.check_dag(stages)
-
         return stages
 
+    @cached_property
+    def stages(self):
+        return get_stages(self.graph)
+
     def find_outs_by_path(self, path, outs=None, recursive=False):
         if not outs:
-            # there is no `from_directory=path` argument because some data
-            # files might be generated to an upper level, and so it is
-            # needed to look at all the files (project root_dir)
-            stages = self.stages()
-            outs = [out for stage in stages for out in stage.outs]
+            outs = [out for stage in self.stages for out in stage.outs]
 
         abs_path = os.path.abspath(path)
         is_dir = self.tree.isdir(abs_path)
@@ -440,14 +454,29 @@ class Repo(object):
     @contextmanager
     def open(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
+        cause = None
         try:
-            with self._open(path, remote, mode, encoding) as fd:
-                yield fd
-        except FileNotFoundError:
-            raise OutputFileMissingError(relpath(path, self.root_dir))
+            out, = self.find_outs_by_path(path)
+        except OutputNotFoundError as e:
+            out = None
+            cause = e
 
-    def _open(self, path, remote=None, mode="r", encoding=None):
-        out, = self.find_outs_by_path(path)
+        if out and out.use_cache:
+            try:
+                with self._open_cached(out, remote, mode, encoding) as fd:
+                    yield fd
+                return
+            except FileNotFoundError as e:
+                raise FileMissingError(relpath(path, self.root_dir), cause=e)
+
+        if self.tree.exists(path):
+            with self.tree.open(path, mode, encoding) as fd:
+                yield fd
+            return
+
+        raise FileMissingError(relpath(path, self.root_dir), cause=cause)
+
+    def _open_cached(self, out, remote=None, mode="r", encoding=None):
         if out.isdir():
             raise ValueError("Can't open a dir")
 
@@ -475,11 +504,16 @@ class Repo(object):
     def close(self):
         self.scm.close()
 
-    @staticmethod
-    def clone(url, to_path, rev=None):
-        from dvc.scm.git import Git
+    @locked
+    def checkout(self, *args, **kwargs):
+        return self._checkout(*args, **kwargs)
 
-        git = Git.clone(url, to_path, rev=rev)
-        git.close()
+    @locked
+    def fetch(self, *args, **kwargs):
+        return self._fetch(*args, **kwargs)
 
-        return Repo(to_path)
+    def _reset(self):
+        self.__dict__.pop("graph", None)
+        self.__dict__.pop("stages", None)
+        self.__dict__.pop("pipelines", None)
+        self.__dict__.pop("dvcignore", None)

dvc/repo/add.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import logging
 import colorama
@@ -7,10 +9,13 @@ from dvc.stage import Stage
 from dvc.utils import walk_files, LARGE_DIR_SIZE
 from dvc.exceptions import RecursiveAddingWhileUsingFilename
 
+from . import locked
+
 
 logger = logging.getLogger(__name__)
 
 
+@locked
 @scm_context
 def add(repo, target, recursive=False, no_commit=False, fname=None):
     if recursive and fname:
@@ -31,12 +36,18 @@ def add(repo, target, recursive=False, no_commit=False, fname=None):
             )
         )
 
-    stages = _create_stages(repo, targets, fname, no_commit)
+    stages = _create_stages(repo, targets, fname)
 
-    repo.check_dag(repo.stages() + stages)
+    repo.check_modified_graph(stages)
 
     for stage in stages:
+        stage.save()
+
+        if not no_commit:
+            stage.commit()
+
         stage.dump()
+
     return stages
 
 
@@ -53,21 +64,15 @@ def _find_all_targets(repo, target, recursive):
     return [target]
 
 
-def _create_stages(repo, targets, fname, no_commit):
+def _create_stages(repo, targets, fname):
     stages = []
 
-    with repo.state:
-        for out in targets:
-            stage = Stage.create(repo, outs=[out], add=True, fname=fname)
+    for out in targets:
+        stage = Stage.create(repo, outs=[out], add=True, fname=fname)
 
-            if not stage:
-                continue
+        if not stage:
+            continue
 
-            stage.save()
-
-            if not no_commit:
-                stage.commit()
-
-            stages.append(stage)
+        stages.append(stage)
 
     return stages

dvc/repo/checkout.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import logging
 
-from dvc.exceptions import CheckoutErrorSuggestGit
+from dvc.exceptions import CheckoutErrorSuggestGit, CheckoutError
 from dvc.progress import Tqdm
 
 
@@ -23,11 +23,11 @@ def get_all_files_numbers(stages):
     return sum(stage.get_all_files_number() for stage in stages)
 
 
-def checkout(self, target=None, with_deps=False, force=False, recursive=False):
+def _checkout(
+    self, target=None, with_deps=False, force=False, recursive=False
+):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    all_stages = self.stages()
-
     try:
         stages = self.collect(target, with_deps=with_deps, recursive=recursive)
     except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
@@ -35,21 +35,23 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False):
             raise
         raise CheckoutErrorSuggestGit(target, exc)
 
-    with self.state:
-        _cleanup_unused_links(self, all_stages)
-        total = get_all_files_numbers(stages)
-        if total == 0:
-            logger.info("Nothing to do")
-        with Tqdm(
-            total=total, unit="file", desc="Checkout", disable=total == 0
-        ) as pbar:
-            for stage in stages:
-                if stage.locked:
-                    logger.warning(
-                        "DVC-file '{path}' is locked. Its dependencies are"
-                        " not going to be checked out.".format(
-                            path=stage.relpath
-                        )
-                    )
+    _cleanup_unused_links(self, self.stages)
+    total = get_all_files_numbers(stages)
+    if total == 0:
+        logger.info("Nothing to do")
+    failed = []
+    with Tqdm(
+        total=total, unit="file", desc="Checkout", disable=total == 0
+    ) as pbar:
+        for stage in stages:
+            if stage.locked:
+                logger.warning(
+                    "DVC-file '{path}' is locked. Its dependencies are"
+                    " not going to be checked out.".format(path=stage.relpath)
+                )
 
+            failed.extend(
                 stage.checkout(force=force, progress_callback=pbar.update_desc)
+            )
+    if failed:
+        raise CheckoutError(failed)

dvc/repo/commit.py

@@ -1,7 +1,10 @@
+from . import locked
+
+
+@locked
 def commit(self, target, with_deps=False, recursive=False, force=False):
     stages = self.collect(target, with_deps=with_deps, recursive=recursive)
-    with self.state:
-        for stage in stages:
-            stage.check_can_commit(force=force)
-            stage.commit()
-            stage.dump()
+    for stage in stages:
+        stage.check_can_commit(force=force)
+        stage.commit()
+        stage.dump()

dvc/repo/destroy.py

@@ -2,7 +2,7 @@ from dvc.utils import remove
 
 
 def destroy(self):
-    for stage in self.stages():
+    for stage in self.stages:
         stage.remove(remove_outs=False)
 
     remove(self.dvc_dir)

dvc/repo/diff.py

@@ -14,6 +14,8 @@ from dvc.scm.git import (
     DIFF_EQUAL,
 )
 
+from . import locked
+
 
 DIFF_TARGET = "target"
 DIFF_IS_DIR = "is_dir"
@@ -140,9 +142,9 @@ def _is_dir(path, a_outs, b_outs):
 
 def _get_diff_outs(self, diff_dct):
     self.tree = diff_dct[DIFF_A_TREE]
-    a_outs = {str(out): out for st in self.stages() for out in st.outs}
+    a_outs = {str(out): out for st in self.stages for out in st.outs}
     self.tree = diff_dct[DIFF_B_TREE]
-    b_outs = {str(out): out for st in self.stages() for out in st.outs}
+    b_outs = {str(out): out for st in self.stages for out in st.outs}
     outs_paths = set(a_outs.keys())
     outs_paths.update(b_outs.keys())
     results = {}
@@ -220,6 +222,7 @@ def _diff_royal(self, target, diff_dct):
     return _diff_file(self, target, diff_dct)
 
 
+@locked
 def diff(self, a_ref, target=None, b_ref=None):
     """Gerenates diff message string output
 

dvc/repo/fetch.py

@@ -1,7 +1,16 @@
 from __future__ import unicode_literals
 
+import logging
 
-def fetch(
+from dvc.config import NoRemoteError
+from dvc.exceptions import DownloadError, OutputNotFoundError
+from dvc.scm.base import CloneError
+
+
+logger = logging.getLogger(__name__)
+
+
+def _fetch(
     self,
     targets=None,
     jobs=None,
@@ -12,17 +21,56 @@ def fetch(
     all_tags=False,
     recursive=False,
 ):
-    with self.state:
-        used = self.used_cache(
-            targets,
-            all_branches=all_branches,
-            all_tags=all_tags,
-            with_deps=with_deps,
-            force=True,
-            remote=remote,
-            jobs=jobs,
-            recursive=recursive,
-        )["local"]
-        return self.cloud.pull(
-            used, jobs, remote=remote, show_checksums=show_checksums
+    """Download data items from a cloud and imported repositories
+
+    Returns:
+        int: number of succesfully downloaded files
+
+    Raises:
+        DownloadError: thrown when there are failed downloads, either
+            during `cloud.pull` or trying to fetch imported files
+
+        config.NoRemoteError: thrown when downloading only local files and no
+            remote is configured
+    """
+    used = self.used_cache(
+        targets,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        with_deps=with_deps,
+        force=True,
+        remote=remote,
+        jobs=jobs,
+        recursive=recursive,
+    )
+
+    downloaded = 0
+    failed = 0
+
+    try:
+        downloaded += self.cloud.pull(
+            used["local"], jobs, remote=remote, show_checksums=show_checksums
         )
+    except NoRemoteError:
+        if not used["repo"] and used["local"]:
+            raise
+
+    except DownloadError as exc:
+        failed += exc.amount
+
+    for dep in used["repo"]:
+        try:
+            out = dep.fetch()
+            downloaded += out.get_files_number()
+        except DownloadError as exc:
+            failed += exc.amount
+        except (CloneError, OutputNotFoundError):
+            failed += 1
+            logger.exception(
+                "failed to fetch data for '{}'".format(dep.stage.outs[0])
+            )
+
+    if failed:
+        raise DownloadError(failed)
+
+    return downloaded

dvc/repo/gc.py

@@ -3,6 +3,8 @@ from __future__ import unicode_literals
 import collections
 import logging
 
+from . import locked
+
 
 logger = logging.getLogger(__name__)
 
@@ -31,18 +33,17 @@ def _load_all_used_cache(
     clists = []
 
     for repo in repos:
-        with repo.state:
-            repo_clist = repo.used_cache(
-                targets=None,
-                all_branches=all_branches,
-                with_deps=with_deps,
-                all_tags=all_tags,
-                remote=remote,
-                force=force,
-                jobs=jobs,
-            )
+        repo_clist = repo.used_cache(
+            targets=None,
+            all_branches=all_branches,
+            with_deps=with_deps,
+            all_tags=all_tags,
+            remote=remote,
+            force=force,
+            jobs=jobs,
+        )
 
-            clists.append(repo_clist)
+        clists.append(repo_clist)
 
     return clists
 
@@ -53,6 +54,7 @@ def _do_gc(typ, func, clist):
         logger.info("No unused {} cache to remove.".format(typ))
 
 
+@locked
 def gc(
     self,
     all_branches=False,
@@ -64,45 +66,50 @@ def gc(
     jobs=None,
     repos=None,
 ):
+    from dvc.utils.compat import ExitStack
     from dvc.repo import Repo
 
-    all_repos = [self]
+    all_repos = []
 
     if repos:
-        all_repos.extend(Repo(path) for path in repos)
+        all_repos = [Repo(path) for path in repos]
 
-    all_clists = _load_all_used_cache(
-        all_repos,
-        all_branches=all_branches,
-        with_deps=with_deps,
-        all_tags=all_tags,
-        remote=remote,
-        force=force,
-        jobs=jobs,
-    )
+    with ExitStack() as stack:
+        for repo in all_repos:
+            stack.enter_context(repo.lock)
+            stack.enter_context(repo.state)
+
+        all_clists = _load_all_used_cache(
+            all_repos + [self],
+            all_branches=all_branches,
+            with_deps=with_deps,
+            all_tags=all_tags,
+            remote=remote,
+            force=force,
+            jobs=jobs,
+        )
 
     if len(all_clists) > 1:
         clist = _merge_cache_lists(all_clists)
     else:
         clist = all_clists[0]
 
-    with self.state:
-        _do_gc("local", self.cache.local.gc, clist)
+    _do_gc("local", self.cache.local.gc, clist)
 
-        if self.cache.s3:
-            _do_gc("s3", self.cache.s3.gc, clist)
+    if self.cache.s3:
+        _do_gc("s3", self.cache.s3.gc, clist)
 
-        if self.cache.gs:
-            _do_gc("gs", self.cache.gs.gc, clist)
+    if self.cache.gs:
+        _do_gc("gs", self.cache.gs.gc, clist)
 
-        if self.cache.ssh:
-            _do_gc("ssh", self.cache.ssh.gc, clist)
+    if self.cache.ssh:
+        _do_gc("ssh", self.cache.ssh.gc, clist)
 
-        if self.cache.hdfs:
-            _do_gc("hdfs", self.cache.hdfs.gc, clist)
+    if self.cache.hdfs:
+        _do_gc("hdfs", self.cache.hdfs.gc, clist)
 
-        if self.cache.azure:
-            _do_gc("azure", self.cache.azure.gc, clist)
+    if self.cache.azure:
+        _do_gc("azure", self.cache.azure.gc, clist)
 
-        if cloud:
-            _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, clist)
+    if cloud:
+        _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, clist)

dvc/repo/get.py

@@ -13,15 +13,14 @@ from dvc.exceptions import (
 from dvc.path_info import PathInfo
 from dvc.external_repo import external_repo
 from dvc.state import StateNoop
-from dvc.utils import remove
-from dvc.utils.compat import urlparse
+from dvc.utils import remove, resolve_output
 
 logger = logging.getLogger(__name__)
 
 
 @staticmethod
 def get(url, path, out=None, rev=None):
-    out = out or os.path.basename(urlparse(path).path)
+    out = resolve_output(path, out)
 
     if Stage.is_valid_filename(out):
         raise GetDVCFileError()

dvc/repo/get_url.py

@@ -3,12 +3,12 @@ import os
 import dvc.output as output
 import dvc.dependency as dependency
 
-from dvc.utils.compat import urlparse
+from dvc.utils import resolve_output
 
 
 @staticmethod
 def get_url(url, out=None):
-    out = out or os.path.basename(urlparse(url).path)
+    out = resolve_output(url, out)
 
     if os.path.exists(url):
         url = os.path.abspath(url)

dvc/repo/graph.py

@@ -0,0 +1,29 @@
+from __future__ import unicode_literals
+
+
+def check_acyclic(graph):
+    import networkx as nx
+    from dvc.exceptions import CyclicGraphError
+
+    cycle = next(nx.simple_cycles(graph), None)
+
+    if cycle:
+        raise CyclicGraphError(cycle)
+
+
+def get_pipeline(pipelines, node):
+    found = [i for i in pipelines if i.has_node(node)]
+    assert len(found) == 1
+    return found[0]
+
+
+def get_pipelines(G):
+    import networkx as nx
+
+    return [G.subgraph(c).copy() for c in nx.weakly_connected_components(G)]
+
+
+def get_stages(G):
+    import networkx
+
+    return list(networkx.get_node_attributes(G, "stage").values())

dvc/repo/imp_url.py

@@ -1,25 +1,26 @@
-from dvc.utils.compat import pathlib
+from dvc.utils import resolve_output
 from dvc.repo.scm_context import scm_context
 
+from . import locked as locked_repo
 
+
+@locked_repo
 @scm_context
 def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
     from dvc.stage import Stage
 
-    out = out or pathlib.PurePath(url).name
+    out = resolve_output(url, out)
 
-    with self.state:
-        stage = Stage.create(
-            self, cmd=None, deps=[url], outs=[out], fname=fname, erepo=erepo
-        )
+    stage = Stage.create(
+        self, cmd=None, deps=[url], outs=[out], fname=fname, erepo=erepo
+    )
 
     if stage is None:
         return None
 
-    self.check_dag(self.stages() + [stage])
+    self.check_modified_graph([stage])
 
-    with self.state:
-        stage.run()
+    stage.run()
 
     stage.locked = locked
 

dvc/repo/lock.py

@@ -1,3 +1,7 @@
+from . import locked
+
+
+@locked
 def lock(self, target, unlock=False):
     from dvc.stage import Stage
 

dvc/repo/metrics/modify.py

@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
+from dvc.repo import locked
 from dvc.exceptions import DvcException
 
 
+@locked
 def modify(repo, path, typ=None, xpath=None, delete=False):
     supported_types = ["raw", "json", "csv", "tsv", "hcsv", "htsv"]
     outs = repo.find_outs_by_path(path)

dvc/repo/metrics/show.py

@@ -8,6 +8,7 @@ import logging
 
 from jsonpath_ng.ext import parse
 
+from dvc.repo import locked
 from dvc.exceptions import OutputNotFoundError, NoMetricsError
 from dvc.utils.compat import builtin_str, open, StringIO, csv_reader
 
@@ -170,7 +171,7 @@ def _collect_metrics(repo, path, recursive, typ, xpath, branch):
             - typ:
             - xpath:
     """
-    outs = [out for stage in repo.stages() for out in stage.outs]
+    outs = [out for stage in repo.stages for out in stage.outs]
 
     if path:
         try:
@@ -257,6 +258,7 @@ def _read_metrics(repo, metrics, branch):
     return res
 
 
+@locked
 def show(
     repo,
     targets=None,

dvc/repo/move.py

@@ -3,6 +3,8 @@ import os
 from dvc.exceptions import MoveNotDataSourceError
 from dvc.repo.scm_context import scm_context
 
+from . import locked
+
 
 def _expand_target_path(from_path, to_path):
     if os.path.isdir(to_path):
@@ -10,6 +12,7 @@ def _expand_target_path(from_path, to_path):
     return to_path
 
 
+@locked
 @scm_context
 def move(self, from_path, to_path):
     """
@@ -59,12 +62,11 @@ def move(self, from_path, to_path):
             os.path.join(os.curdir, os.path.dirname(to_path))
         )
 
-    to_out = Output.loads_from(
-        stage, [os.path.basename(to_path)], out.use_cache, out.metric
-    )[0]
+    to_path = os.path.relpath(to_path, stage.wdir)
 
-    with self.state:
-        out.move(to_out)
-        stage.save()
+    to_out = Output.loads_from(stage, [to_path], out.use_cache, out.metric)[0]
+
+    out.move(to_out)
+    stage.save()
 
     stage.dump()

dvc/repo/pull.py

@@ -1,6 +1,9 @@
 from __future__ import unicode_literals
 
+from . import locked
 
+
+@locked
 def pull(
     self,
     targets=None,
@@ -12,7 +15,7 @@ def pull(
     force=False,
     recursive=False,
 ):
-    processed_files_count = self.fetch(
+    processed_files_count = self._fetch(
         targets,
         jobs,
         remote=remote,
@@ -22,7 +25,7 @@ def pull(
         recursive=recursive,
     )
     for target in targets or [None]:
-        self.checkout(
+        self._checkout(
             target=target,
             with_deps=with_deps,
             force=force,

dvc/repo/push.py

@@ -1,6 +1,9 @@
 from __future__ import unicode_literals
 
+from . import locked
 
+
+@locked
 def push(
     self,
     targets=None,
@@ -11,15 +14,14 @@ def push(
     all_tags=False,
     recursive=False,
 ):
-    with self.state:
-        used = self.used_cache(
-            targets,
-            all_branches=all_branches,
-            all_tags=all_tags,
-            with_deps=with_deps,
-            force=True,
-            remote=remote,
-            jobs=jobs,
-            recursive=recursive,
-        )["local"]
-        return self.cloud.push(used, jobs, remote=remote)
+    used = self.used_cache(
+        targets,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        with_deps=with_deps,
+        force=True,
+        remote=remote,
+        jobs=jobs,
+        recursive=recursive,
+    )["local"]
+    return self.cloud.push(used, jobs, remote=remote)

dvc/repo/remove.py

@@ -1,3 +1,7 @@
+from . import locked
+
+
+@locked
 def remove(self, target, outs_only=False):
     from dvc.stage import Stage
 

dvc/repo/reproduce.py

@@ -1,12 +1,15 @@
 from __future__ import unicode_literals
 
-import os
 import logging
 
 from dvc.exceptions import ReproductionError
 from dvc.repo.scm_context import scm_context
 from dvc.utils import relpath
 
+from . import locked
+from .graph import get_pipeline, get_pipelines
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -29,6 +32,24 @@ def _reproduce_stage(stages, node, **kwargs):
     return [stage]
 
 
+def _get_active_graph(G):
+    import networkx as nx
+
+    active = G.copy()
+    stages = nx.get_node_attributes(G, "stage")
+    for node in G:
+        stage = stages[node]
+        if not stage.locked:
+            continue
+        for n in nx.dfs_postorder_nodes(G, node):
+            if n == node:
+                continue
+            if n in active:
+                active.remove_node(n)
+    return active
+
+
+@locked
 @scm_context
 def reproduce(
     self,
@@ -52,43 +73,37 @@ def reproduce(
             config.SECTION_CORE_INTERACTIVE, False
         )
 
-    targets = []
-    if recursive and os.path.isdir(target):
-        G = self.graph(from_directory=target)[1]
-        dir_targets = [
-            os.path.join(self.root_dir, n) for n in nx.dfs_postorder_nodes(G)
-        ]
-        targets.extend(dir_targets)
-    elif pipeline or all_pipelines:
-        if pipeline:
+    active_graph = _get_active_graph(self.graph)
+    active_pipelines = get_pipelines(active_graph)
+
+    if pipeline or all_pipelines:
+        if all_pipelines:
+            pipelines = active_pipelines
+        else:
             stage = Stage.load(self, target)
             node = relpath(stage.path, self.root_dir)
-            pipelines = [self._get_pipeline(node)]
-        else:
-            pipelines = self.pipelines()
+            pipelines = [get_pipeline(active_pipelines, node)]
 
+        targets = []
         for G in pipelines:
-            for node in G.nodes():
+            attrs = nx.get_node_attributes(G, "stage")
+            for node in G:
                 if G.in_degree(node) == 0:
-                    targets.append(os.path.join(self.root_dir, node))
+                    targets.append(attrs[node])
     else:
-        targets.append(target)
+        targets = self.collect(target, recursive=recursive, graph=active_graph)
 
     ret = []
-    with self.state:
-        for target in targets:
-            stages = _reproduce(self, target, **kwargs)
-            ret.extend(stages)
+    for target in targets:
+        stages = _reproduce(self, active_graph, target, **kwargs)
+        ret.extend(stages)
 
     return ret
 
 
-def _reproduce(self, target, **kwargs):
+def _reproduce(self, G, stage, **kwargs):
     import networkx as nx
-    from dvc.stage import Stage
 
-    stage = Stage.load(self, target)
-    G = self.graph()[1]
     stages = nx.get_node_attributes(G, "stage")
     node = relpath(stage.path, self.root_dir)
 

dvc/repo/run.py

@@ -1,23 +1,23 @@
 from __future__ import unicode_literals
 
-from dvc.repo.scm_context import scm_context
+from . import locked
+from .scm_context import scm_context
 
 
+@locked
 @scm_context
 def run(self, no_exec=False, **kwargs):
     from dvc.stage import Stage
 
-    with self.state:
-        stage = Stage.create(self, **kwargs)
+    stage = Stage.create(self, **kwargs)
 
     if stage is None:
         return None
 
-    self.check_dag(self.stages() + [stage])
+    self.check_modified_graph([stage])
 
-    with self.state:
-        if not no_exec:
-            stage.run(no_commit=kwargs.get("no_commit", False))
+    if not no_exec:
+        stage.run(no_commit=kwargs.get("no_commit", False))
 
     stage.dump()
 

dvc/repo/status.py

@@ -3,6 +3,8 @@ from __future__ import unicode_literals
 import logging
 from funcy.py3 import cat
 
+from . import locked
+
 
 logger = logging.getLogger(__name__)
 
@@ -91,6 +93,7 @@ def _cloud_status(
     return ret
 
 
+@locked
 def status(
     self,
     targets=None,
@@ -101,15 +104,14 @@ def status(
     with_deps=False,
     all_tags=False,
 ):
-    with self.state:
-        if cloud or remote:
-            return _cloud_status(
-                self,
-                targets,
-                jobs,
-                all_branches=all_branches,
-                with_deps=with_deps,
-                remote=remote,
-                all_tags=all_tags,
-            )
-        return _local_status(self, targets, with_deps=with_deps)
+    if cloud or remote:
+        return _cloud_status(
+            self,
+            targets,
+            jobs,
+            all_branches=all_branches,
+            with_deps=with_deps,
+            remote=remote,
+            all_tags=all_tags,
+        )
+    return _local_status(self, targets, with_deps=with_deps)

dvc/repo/tag/add.py

@@ -1,9 +1,13 @@
 import logging
 from copy import copy
 
+from dvc.repo import locked
+
+
 logger = logging.getLogger(__name__)
 
 
+@locked
 def add(self, tag, target=None, with_deps=False, recursive=False):
     stages = self.collect(target, with_deps=with_deps, recursive=recursive)
     for stage in stages:

dvc/repo/tag/list.py

@@ -1,3 +1,7 @@
+from dvc.repo import locked
+
+
+@locked
 def list(self, target=None, with_deps=False, recursive=False):
     ret = {}
     stages = self.collect(target, with_deps=with_deps, recursive=recursive)

dvc/repo/tag/remove.py

@@ -1,9 +1,12 @@
 import logging
 
+from dvc.repo import locked
+
 
 logger = logging.getLogger(__name__)
 
 
+@locked
 def remove(self, tag, target=None, with_deps=False, recursive=False):
     stages = self.collect(target, with_deps=with_deps, recursive=recursive)
     for stage in stages:

dvc/repo/update.py

@@ -1,8 +1,11 @@
+from . import locked
+
+
+@locked
 def update(self, target):
     from dvc.stage import Stage
 
     stage = Stage.load(self, target)
-    with self.state:
-        stage.update()
+    stage.update()
 
     stage.dump()

dvc/scm/git/__init__.py

@@ -5,8 +5,12 @@ from __future__ import unicode_literals
 import os
 import logging
 
-from dvc.utils.compat import str, open
-from dvc.utils import fix_env, relpath
+from funcy import cached_property
+from pathspec.patterns import GitWildMatchPattern
+
+from dvc.exceptions import GitHookAlreadyExistsError
+from dvc.utils.compat import str, open, cast_bytes_py2
+from dvc.utils import fix_env, relpath, is_binary
 from dvc.scm.base import (
     Base,
     SCMError,
@@ -61,11 +65,24 @@ class Git(Base):
     def clone(url, to_path, rev=None):
         import git
 
+        ld_key = "LD_LIBRARY_PATH"
+
+        env = fix_env(None)
+        if is_binary() and ld_key not in env.keys():
+            # In fix_env, we delete LD_LIBRARY_PATH key if it was empty before
+            # PyInstaller modified it. GitPython, in git.Repo.clone_from, uses
+            # env to update its own internal state. When there is no key in
+            # env, this value is not updated and GitPython re-uses
+            # LD_LIBRARY_PATH that has been set by PyInstaller.
+            # See [1] for more info.
+            # [1] https://github.com/gitpython-developers/GitPython/issues/924
+            env[cast_bytes_py2(ld_key)] = ""
+
         try:
             tmp_repo = git.Repo.clone_from(
                 url,
                 to_path,
-                env=fix_env(None),  # needed before we can fix it in __init__
+                env=env,  # needed before we can fix it in __init__
                 no_single_branch=True,
             )
             tmp_repo.close()
@@ -138,7 +155,7 @@ class Git(Base):
             return
 
         msg = "Adding '{}' to '{}'.".format(relpath(path), relpath(gitignore))
-        logger.info(msg)
+        logger.debug(msg)
 
         self._add_entry_to_gitignore(entry, gitignore)
 
@@ -146,8 +163,9 @@ class Git(Base):
 
         self.ignored_paths.append(path)
 
-    @staticmethod
-    def _add_entry_to_gitignore(entry, gitignore):
+    def _add_entry_to_gitignore(self, entry, gitignore):
+        entry = GitWildMatchPattern.escape(entry)
+
         with open(gitignore, "a+", encoding="utf-8") as fobj:
             fobj.seek(0, os.SEEK_END)
             if fobj.tell() == 0:
@@ -234,7 +252,7 @@ class Git(Base):
             " || exec dvc {}".format(cmd)
         )
 
-        hook = os.path.join(self.root_dir, self.GIT_DIR, "hooks", name)
+        hook = self._hook_path(name)
 
         if os.path.isfile(hook):
             with open(hook, "r+") as fobj:
@@ -247,6 +265,8 @@ class Git(Base):
         os.chmod(hook, 0o777)
 
     def install(self):
+        self._verify_dvc_hooks()
+
         self._install_hook("post-checkout", "checkout")
         self._install_hook("pre-commit", "status")
         self._install_hook("pre-push", "push")
@@ -342,3 +362,19 @@ class Git(Base):
 
     def close(self):
         self.repo.close()
+
+    @cached_property
+    def _hooks_home(self):
+        return os.path.join(self.root_dir, self.GIT_DIR, "hooks")
+
+    def _hook_path(self, name):
+        return os.path.join(self._hooks_home, name)
+
+    def _verify_hook(self, name):
+        if os.path.exists(self._hook_path(name)):
+            raise GitHookAlreadyExistsError(name)
+
+    def _verify_dvc_hooks(self):
+        self._verify_hook("post-checkout")
+        self._verify_hook("pre-commit")
+        self._verify_hook("pre-push")

dvc/scm/git/tree.py

@@ -45,7 +45,8 @@ class GitTree(BaseTree):
     def tree_root(self):
         return self.git.working_dir
 
-    def open(self, path, binary=False):
+    def open(self, path, mode="r", encoding="utf-8"):
+        assert mode in {"r", "rb"}
 
         relative_path = relpath(path, self.git.working_dir)
 
@@ -61,9 +62,9 @@ class GitTree(BaseTree):
         # the `open()` behavior (since data_stream.read() returns bytes,
         # and `open` with default "r" mode returns str)
         data = obj.data_stream.read()
-        if binary:
+        if mode == "rb":
             return BytesIO(data)
-        return StringIO(data.decode("utf-8"))
+        return StringIO(data.decode(encoding))
 
     def exists(self, path):
         return self.git_object_by_path(path) is not None

dvc/scm/tree.py

@@ -11,7 +11,7 @@ class BaseTree(object):
     def tree_root(self):
         pass
 
-    def open(self, path, binary=False):
+    def open(self, path, mode="r", encoding="utf-8"):
         """Open file and return a stream."""
 
     def exists(self, path):
@@ -42,11 +42,9 @@ class WorkingTree(BaseTree):
     def tree_root(self):
         return self.repo_root
 
-    def open(self, path, binary=False):
+    def open(self, path, mode="r", encoding="utf-8"):
         """Open file and return a stream."""
-        if binary:
-            return open(path, "rb")
-        return open(path, encoding="utf-8")
+        return open(path, mode=mode, encoding=encoding)
 
     def exists(self, path):
         """Test whether a path exists."""

dvc/stage.py

@@ -293,7 +293,7 @@ class Stage(object):
         if ret:
             logger.warning("Stage '{}' changed.".format(self.relpath))
         else:
-            logger.info("Stage '{}' didn't change.".format(self.relpath))
+            logger.debug("Stage '{}' didn't change.".format(self.relpath))
 
         return ret
 
@@ -731,9 +731,8 @@ class Stage(object):
             msg += " and " if (changed_deps and changed_outs) else ""
             msg += "outputs {}".format(changed_outs) if changed_outs else ""
             msg += "md5" if not (changed_deps or changed_outs) else ""
-            msg += " of '{}' changed. Are you sure you commit it?".format(
-                self.relpath
-            )
+            msg += " of '{}' changed.".format(self.relpath)
+            msg += "Are you sure you want to commit it?"
             if not force and not prompt.confirm(msg):
                 raise StageCommitError(
                     "unable to commit changed '{}'. Use `-f|--force` to "
@@ -789,8 +788,34 @@ class Stage(object):
 
     def _run(self):
         self._check_missing_deps()
-        executable = os.getenv("SHELL") if os.name != "nt" else None
-        self._warn_if_fish(executable)
+
+        kwargs = {"cwd": self.wdir, "env": fix_env(None), "close_fds": True}
+
+        if os.name == "nt":
+            kwargs["shell"] = True
+            cmd = self.cmd
+        else:
+            # NOTE: when you specify `shell=True`, `Popen` [1] will default to
+            # `/bin/sh` on *nix and will add ["/bin/sh", "-c"] to your command.
+            # But we actually want to run the same shell that we are running
+            # from right now, which is usually determined by the `SHELL` env
+            # var. So instead, we compose our command on our own, making sure
+            # to include special flags to prevent shell from reading any
+            # configs and modifying env, which may change the behavior or the
+            # command we are running. See [2] for more info.
+            #
+            # [1] https://github.com/python/cpython/blob/3.7/Lib/subprocess.py
+            #                                                            #L1426
+            # [2] https://github.com/iterative/dvc/issues/2506
+            #                                           #issuecomment-535396799
+            kwargs["shell"] = False
+            executable = os.getenv("SHELL") or "/bin/sh"
+
+            self._warn_if_fish(executable)
+
+            opts = {"zsh": ["--no-rcs"], "bash": ["--noprofile", "--norc"]}
+            name = os.path.basename(executable).lower()
+            cmd = [executable] + opts.get(name, []) + ["-c", self.cmd]
 
         main_thread = isinstance(
             threading.current_thread(), threading._MainThread
@@ -799,14 +824,7 @@ class Stage(object):
         p = None
 
         try:
-            p = subprocess.Popen(
-                self.cmd,
-                cwd=self.wdir,
-                shell=True,
-                env=fix_env(os.environ),
-                executable=executable,
-                close_fds=True,
-            )
+            p = subprocess.Popen(cmd, **kwargs)
             if main_thread:
                 old_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
             p.communicate()
@@ -872,10 +890,14 @@ class Stage(object):
             raise MissingDataSource(paths)
 
     def checkout(self, force=False, progress_callback=None):
+        failed_checkouts = []
         for out in self.outs:
-            out.checkout(
+            failed = out.checkout(
                 force=force, tag=self.tag, progress_callback=progress_callback
             )
+            if failed:
+                failed_checkouts.append(failed)
+        return failed_checkouts
 
     @staticmethod
     def _status(entries):

dvc/state.py

@@ -3,6 +3,7 @@
 from __future__ import unicode_literals
 
 import os
+import re
 import sqlite3
 import logging
 
@@ -10,7 +11,8 @@ from dvc.config import Config
 from dvc.utils import remove, current_timestamp, relpath, to_chunks
 from dvc.exceptions import DvcException
 from dvc.utils.fs import get_mtime_and_size, get_inode
-from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import fspath_py35, urlencode, urlunparse, is_py2
+
 
 SQLITE_MAX_VARIABLES_NUMBER = 999
 
@@ -213,7 +215,11 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             assert self.cursor is None
             assert self.inserts == 0
             empty = not os.path.exists(self.state_file)
-            self.database = sqlite3.connect(self.state_file)
+            # NOTE: we use nolock option because fcntl() lock sqlite uses
+            # doesn't work on some older NFS/CIFS filesystems.
+            # This opens a possibility of data corruption by concurrent writes,
+            # which is prevented by repo lock.
+            self.database = _connect_sqlite(self.state_file, {"nolock": 1})
             self.cursor = self.database.cursor()
 
             # Try loading once to check that the file is indeed a database
@@ -473,3 +479,25 @@ class State(object):  # pylint: disable=too-many-instance-attributes
                 self.LINK_STATE_TABLE, ",".join(["?"] * len(chunk_unused))
             )
             self._execute(cmd, tuple(chunk_unused))
+
+
+def _connect_sqlite(filename, options):
+    # Connect by URI was added in Python 3.4 and sqlite 3.7.7,
+    # we ignore options, which should be fine unless repo is on old NFS/CIFS
+    if is_py2 or sqlite3.sqlite_version_info < (3, 7, 7):
+        return sqlite3.connect(filename)
+
+    uri = _build_sqlite_uri(filename, options)
+    return sqlite3.connect(uri, uri=True)
+
+
+def _build_sqlite_uri(filename, options):
+    # Convert filename to uri according to https://www.sqlite.org/uri.html, 3.1
+    uri_path = filename.replace("?", "%3f").replace("#", "%23")
+    if os.name == "nt":
+        uri_path = uri_path.replace("\\", "/")
+        uri_path = re.sub(r"^([a-z]:)", "/\\1", uri_path, flags=re.I)
+    uri_path = re.sub(r"/+", "/", uri_path)
+
+    # Empty netloc, params and fragment
+    return urlunparse(("file", "", uri_path, "", urlencode(options), ""))

dvc/system.py

@@ -4,6 +4,7 @@ from dvc.utils.compat import str, open, fspath
 
 import os
 import errno
+import shutil
 
 
 class System(object):
@@ -11,6 +12,11 @@ class System(object):
     def is_unix():
         return os.name != "nt"
 
+    @staticmethod
+    def copy(src, dest):
+        src, dest = fspath(src), fspath(dest)
+        return shutil.copyfile(src, dest)
+
     @staticmethod
     def hardlink(source, link_name):
         import ctypes

dvc/updater.py

@@ -4,9 +4,8 @@ import sys
 import os
 import time
 import logging
-import requests
 import colorama
-from pkg_resources import parse_version
+from packaging import version
 
 from dvc import __version__
 from dvc.lock import Lock, LockError
@@ -25,8 +24,10 @@ class Updater(object):  # pragma: no cover
     def __init__(self, dvc_dir):
         self.dvc_dir = dvc_dir
         self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
-        self.lock = Lock(dvc_dir, self.updater_file + ".lock")
-        self.current = parse_version(__version__).base_version
+        self.lock = Lock(
+            self.updater_file + ".lock", tmp_dir=os.path.join(dvc_dir, "tmp")
+        )
+        self.current = version.parse(__version__).base_version
 
     def _is_outdated_file(self):
         ctime = os.path.getmtime(self.updater_file)
@@ -41,7 +42,7 @@ class Updater(object):  # pragma: no cover
                 func()
         except LockError:
             msg = "Failed to acquire '{}' before {} updates"
-            logger.debug(msg.format(self.lock.lock_file, action))
+            logger.debug(msg.format(self.lock.lockfile, action))
 
     def check(self):
         if os.getenv("CI") or os.getenv("DVC_TEST"):
@@ -80,6 +81,7 @@ class Updater(object):  # pragma: no cover
 
     def _get_latest_version(self):
         import json
+        import requests
 
         try:
             r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
@@ -93,7 +95,7 @@ class Updater(object):  # pragma: no cover
             json.dump(info, fobj)
 
     def _is_outdated(self):
-        return parse_version(self.current) < parse_version(self.latest)
+        return version.parse(self.current) < version.parse(self.latest)
 
     def _notify(self):
         if not sys.stdout.isatty():

dvc/utils/__init__.py

@@ -245,9 +245,11 @@ def is_binary():
     return getattr(sys, "frozen", False)
 
 
-# NOTE: Fix env variables modified by PyInstaller
-# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
 def fix_env(env=None):
+    """Fix env variables modified by PyInstaller [1] and pyenv [2].
+    [1] http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
+    [2] https://github.com/pyenv/pyenv/issues/985
+    """
     if env is None:
         env = os.environ.copy()
     else:
@@ -262,6 +264,55 @@ def fix_env(env=None):
         else:
             env.pop(lp_key, None)
 
+    # Unlike PyInstaller, pyenv doesn't leave backups of original env vars
+    # when it modifies them. If we look into the shim, pyenv and pyenv-exec,
+    # we can figure out that the PATH is modified like this:
+    #
+    #     PATH=$PYENV_BIN_PATH:${bin_path}:${plugin_bin}:$PATH
+    #
+    # where
+    #
+    #     PYENV_BIN_PATH - might not start with $PYENV_ROOT if we are running
+    #         `system` version of the command, see pyenv-exec source code.
+    #     bin_path - might not start with $PYENV_ROOT as it runs realpath on
+    #         it, but always has `libexec` part in it, see pyenv source code.
+    #     plugin_bin - might contain more than 1 entry, which start with
+    #         $PYENV_ROOT, see pyenv source code.
+    #
+    # Also, we know that whenever pyenv is running, it exports these env vars:
+    #
+    #     PYENV_DIR
+    #     PYENV_HOOK_PATH
+    #     PYENV_VERSION
+    #     PYENV_ROOT
+    #
+    # So having this, we can make a rightful assumption about what parts of the
+    # PATH we need to remove in order to get the original PATH.
+    path = env.get("PATH", "")
+    parts = path.split(":")
+    bin_path = parts[1] if len(parts) > 2 else ""
+    pyenv_dir = env.get("PYENV_DIR")
+    pyenv_hook_path = env.get("PYENV_HOOK_PATH")
+    pyenv_version = env.get("PYENV_VERSION")
+    pyenv_root = env.get("PYENV_ROOT")
+
+    env_matches = all([pyenv_dir, pyenv_hook_path, pyenv_version, pyenv_root])
+
+    bin_path_matches = os.path.basename(bin_path) == "libexec"
+
+    # NOTE: we don't support pyenv-win
+    if os.name != "nt" and env_matches and bin_path_matches:
+        # removing PYENV_BIN_PATH and bin_path
+        parts = parts[2:]
+
+        if parts:
+            # removing plugin_bin from the left
+            plugin_bin = os.path.join(pyenv_root, "plugins")
+            while parts[0].startswith(plugin_bin):
+                del parts[0]
+
+        env["PATH"] = ":".join(parts)
+
     return env
 
 
@@ -409,3 +460,14 @@ def relpath(path, start=os.curdir):
     ):
         return path
     return os.path.relpath(path, start)
+
+
+def resolve_output(inp, out):
+    from dvc.utils.compat import urlparse
+
+    name = os.path.basename(urlparse(inp).path)
+    if not out:
+        return name
+    if os.path.isdir(out):
+        return os.path.join(out, name)
+    return out

dvc/utils/compat.py

@@ -18,6 +18,11 @@ is_py3 = _ver[0] == 3
 # simplified version of ipython_genutils/encoding.py
 DEFAULT_ENCODING = sys.getdefaultencoding()
 
+if _ver[:2] < (3, 5):
+    RecursionError = RuntimeError
+else:
+    RecursionError = RecursionError
+
 
 def no_code(x, encoding=None):
     return x
@@ -101,13 +106,13 @@ def ignore_file_not_found():
 
 
 if is_py2:
-    from urlparse import urlparse, urljoin, urlsplit, urlunsplit  # noqa: F401
-    from BaseHTTPServer import HTTPServer  # noqa: F401
-    from SimpleHTTPServer import SimpleHTTPRequestHandler  # noqa: F401
+    from urlparse import urlparse, urlunparse, urljoin  # noqa: F401
+    from urllib import urlencode  # noqa: F401
     import ConfigParser  # noqa: F401
     from io import open  # noqa: F401
     import pathlib2 as pathlib  # noqa: F401
     from collections import Mapping  # noqa: F401
+    from contextlib2 import ExitStack  # noqa: F401
 
     builtin_str = str  # noqa: F821
     bytes = str  # noqa: F821
@@ -144,17 +149,14 @@ elif is_py3:
     from os import makedirs  # noqa: F401
     from urllib.parse import (  # noqa: F401
         urlparse,  # noqa: F401
+        urlunparse,  # noqa: F401
+        urlencode,  # noqa: F401
         urljoin,  # noqa: F401
-        urlsplit,  # noqa: F401
-        urlunsplit,  # noqa: F401
     )
     from io import StringIO, BytesIO  # noqa: F401
-    from http.server import (  # noqa: F401
-        HTTPServer,  # noqa: F401
-        SimpleHTTPRequestHandler,  # noqa: F401
-    )  # noqa: F401
     import configparser as ConfigParser  # noqa: F401
     from collections.abc import Mapping  # noqa: F401
+    from contextlib import ExitStack  # noqa: F401
 
     builtin_str = str  # noqa: F821
     str = str  # noqa: F821

dvc/utils/http.py

@@ -1,8 +1,6 @@
 from contextlib import contextmanager
 import io
 
-import requests
-
 from dvc.utils.compat import FileNotFoundError
 
 
@@ -28,6 +26,7 @@ def open_url(url, mode="r", encoding=None):
 @contextmanager
 def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE):
     """Iterate over chunks requested from url."""
+    import requests
 
     def request(headers=None):
         the_url = url() if callable(url) else url

dvc/version.py

@@ -7,12 +7,12 @@ import os
 import subprocess
 
 
-_BASE_VERSION = "0.59.2"
+_BASE_VERSION = "0.63.1"
 
 
 def _generate_version(base_version):
     """Generate a version with information about the git repository"""
-    pkg_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+    pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
     if not _is_git_repo(pkg_dir) or not _have_git():
         return base_version

fastentrypoints.py

@@ -0,0 +1,122 @@
+# noqa: D300,D400
+# Copyright (c) 2016, Aaron Christianson
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+Monkey patch setuptools to write faster console_scripts with this format:
+
+    import sys
+    from mymodule import entry_function
+    sys.exit(entry_function())
+
+This is better.
+
+(c) 2016, Aaron Christianson
+http://github.com/ninjaaron/fast-entry_points
+"""
+from setuptools.command import easy_install
+import re
+
+TEMPLATE = r"""
+# -*- coding: utf-8 -*-
+# EASY-INSTALL-ENTRY-SCRIPT: '{3}','{4}','{5}'
+__requires__ = '{3}'
+import re
+import sys
+
+from {0} import {1}
+
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
+    sys.exit({2}())
+""".lstrip()
+
+
+@classmethod
+def get_args(cls, dist, header=None):  # noqa: D205,D400
+    """
+    Yield write_script() argument tuples for a distribution's
+    console_scripts and gui_scripts entry points.
+    """
+    if header is None:
+        # pylint: disable=E1101
+        header = cls.get_header()
+    spec = str(dist.as_requirement())
+    for type_ in "console", "gui":
+        group = type_ + "_scripts"
+        for name, ep in dist.get_entry_map(group).items():
+            # ensure_safe_name
+            if re.search(r"[\\/]", name):
+                raise ValueError("Path separators not allowed in script names")
+            script_text = TEMPLATE.format(
+                ep.module_name,
+                ep.attrs[0],
+                ".".join(ep.attrs),
+                spec,
+                group,
+                name,
+            )
+            # pylint: disable=E1101
+            args = cls._get_script_args(type_, name, header, script_text)
+            for res in args:
+                yield res
+
+
+# pylint: disable=E1101
+easy_install.ScriptWriter.get_args = get_args
+
+
+def main():
+    import os
+    import re
+    import shutil
+    import sys
+
+    dests = sys.argv[1:] or ["."]
+    filename = re.sub(r"\.pyc$", ".py", __file__)
+
+    for dst in dests:
+        shutil.copy(filename, dst)
+        manifest_path = os.path.join(dst, "MANIFEST.in")
+        setup_path = os.path.join(dst, "setup.py")
+
+        # Insert the include statement to MANIFEST.in if not present
+        with open(manifest_path, "a+") as manifest:
+            manifest.seek(0)
+            manifest_content = manifest.read()
+            if "include fastentrypoints.py" not in manifest_content:
+                manifest.write(
+                    ("\n" if manifest_content else "")
+                    + "include fastentrypoints.py"
+                )
+
+        # Insert the import statement to setup.py if not present
+        with open(setup_path, "a+") as setup:
+            setup.seek(0)
+            setup_content = setup.read()
+            if "import fastentrypoints" not in setup_content:
+                setup.seek(0)
+                setup.truncate()
+                setup.write("import fastentrypoints\n" + setup_content)

scripts/build_posix.sh

@@ -18,6 +18,15 @@ BIN_DIR=$BUILD_DIR/$INSTALL_DIR/bin
 DESC='Data Version Control - datasets, models, and experiments versioning for ML or data science projects'
 LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib
 
+FPM_PACKAGE_DIRS="usr"
+ZSH_CMPLT_DIR=usr/share/zsh/site-functions/_dvc
+if [[ "$(uname)" == 'Linux' ]]; then
+	BASH_CMPLT_DIR=etc/bash_completion.d
+	FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc"
+else
+	BASH_CMPLT_DIR=usr/local/etc/bash_completion.d
+fi
+
 print_error()
 {
 	echo -e "\e[31m$1\e[0m"
@@ -51,7 +60,8 @@ fpm_build()
 	    $FPM_FLAGS \
 	    -n dvc \
 	    -v $VERSION \
-	    -C $BUILD_DIR usr
+	    -C $BUILD_DIR \
+	    $FPM_PACKAGE_DIRS
 }
 
 cleanup()
@@ -78,8 +88,11 @@ install_dependencies()
 
 	gem install --no-document fpm
 
+	print_info "Uprgading pip..."
+	pip install --upgrade pip
+
 	print_info "Installing requirements..."
-	pip install .[all]
+	pip install .[all] psutil
 
 	print_info "Installing pyinstaller..."
 	pip install pyinstaller
@@ -106,6 +119,16 @@ build_dvc()
 		popd
 		$BIN_DIR/dvc --help
 	fi
+
+        # NOTE: temporarily not adding scripts to mac package. See [1]
+        # [1] https://github.com/iterative/dvc/issues/2585
+        if [[ "$(uname)" == 'Linux' ]]; then
+            mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
+            cp scripts/completion/dvc.bash $BUILD_DIR/$BASH_CMPLT_DIR/dvc
+
+            mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
+            cp scripts/completion/dvc.zsh $BUILD_DIR/$ZSH_CMPLT_DIR
+        fi
 }
 
 cleanup

scripts/ci/before_install.sh

@@ -48,7 +48,7 @@ if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
         choco install python2
         echo 'PATH="/c/Python27:/c/Python27/Scripts:$PATH"' >> env.sh
     else
-        choco install python
+        choco install python --version 3.7.4
         echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >> env.sh
     fi
 elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then

scripts/completion/dvc.bash

@@ -72,6 +72,10 @@ _dvc_version=''
 #       ${!x} ->  ${hello} ->  "world"
 #
 _dvc () {
+  replace_hyphen () {
+    echo $(echo $1 | sed 's/-/_/g')
+  }
+
   local word="${COMP_WORDS[COMP_CWORD]}"
 
   COMPREPLY=()
@@ -82,11 +86,11 @@ _dvc () {
       *)  COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
     esac
   elif [ "${COMP_CWORD}" -eq 2 ]; then
-    local options_list="_dvc_${COMP_WORDS[1]}"
+    local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})"
 
     COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
   elif [ "${COMP_CWORD}" -eq 3 ]; then
-    local options_list="_dvc_${COMP_WORDS[1]}_${COMP_WORDS[2]}"
+    local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})_$(replace_hyphen ${COMP_WORDS[2]})"
 
     COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
   fi

setup.py

@@ -3,15 +3,27 @@ from setuptools.command.build_py import build_py as _build_py
 import os
 import sys
 
+# Prevents pkg_resources import in entry point script,
+# see https://github.com/ninjaaron/fast-entry_points.
+# This saves about 200 ms on startup time for non-wheel installs.
+import fastentrypoints  # noqa: F401
+
 
 # https://packaging.python.org/guides/single-sourcing-package-version/
-pkg_dir = os.path.dirname(__file__)
+pkg_dir = os.path.dirname(os.path.abspath(__file__))
+version_path = os.path.join(pkg_dir, "dvc", "version.py")
+if sys.version_info[0] == 2:
+    import imp
 
-# This will define __version__ implicitly
-with open(os.path.join(pkg_dir, "dvc", "version.py")) as fobj:
-    exec(fobj.read())
+    dvc_version = imp.load_source("dvc.version", version_path)
+else:
+    import importlib.util
 
-version = __version__  # noqa: F821
+    spec = importlib.util.spec_from_file_location("dvc.version", version_path)
+    dvc_version = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(dvc_version)
+
+version = dvc_version.__version__  # noqa: F821
 
 
 # To achieve consistency between the build version and the one provided
@@ -39,11 +51,11 @@ class build_py(_build_py):
 install_requires = [
     "ply>=3.9",  # See https://github.com/pyinstaller/pyinstaller/issues/1945
     "configparser>=3.5.0",
-    "zc.lockfile>=1.2.1",
     "future>=0.16.0",
     "colorama>=0.3.9",
     "configobj>=5.0.6",
     "gitpython>=2.1.8",
+    "gitdb2>=2.0.6",  # See https://github.com/iterative/dvc/issues/1880
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
@@ -59,24 +71,21 @@ install_requires = [
     "humanize>=0.5.1",
     "ruamel.yaml>=0.16.1",
     "funcy>=1.12",
-    "pathspec>=0.5.9",
+    "pathspec>=0.6.0",
     "shortuuid>=0.5.0",
     "tqdm>=4.35.0",
+    "packaging>=19.0",
     "win-unicode-console>=0.5; sys_platform == 'win32'",
 ]
 
 if sys.version_info[0] == 2:
     install_requires.append("networkx>=2.1,<2.3")
 else:
-    install_requires.append("networkx>=2.1")
+    install_requires.append("networkx>=2.1,<2.4")
 
 # Extra dependencies for remote integrations
 
-# Restricting google-resumable-media version is due to the fact that
-# since 0.3.3 our binaries built by pyinstaller started to fail on Linux and
-# macOS. This restriction will be removed after
-# https://github.com/iterative/dvc/issues/2441 is resolved.
-gs = ["google-resumable-media==0.3.2", "google-cloud-storage==1.13.0"]
+gs = ["google-cloud-storage==1.19.0"]
 s3 = ["boto3==1.9.115"]
 azure = ["azure-storage-blob==2.1.0"]
 oss = ["oss2==2.6.1"]
@@ -95,7 +104,7 @@ if os.name != "nt" or sys.version_info[0] != 2:
 
 # Extra dependecies to run tests
 tests_requirements = [
-    "PyInstaller==3.4",
+    "PyInstaller==3.5",
     "wheel>=0.31.1",
     "pydot>=1.2.4",
     # Test requirements:
@@ -108,15 +117,16 @@ tests_requirements = [
     "mock>=3.0.0",
     "xmltodict>=0.11.0",
     "awscli>=1.16.125",
-    "google-compute-engine",
+    "google-compute-engine==2.8.13",
     "pywin32; sys_platform == 'win32'",
     "Pygments",  # required by collective.checkdocs,
     "collective.checkdocs",
     "flake8",
+    "psutil",
     "flake8-docstrings",
     "pydocstyle<4.0",
     "jaraco.windows==3.9.2",
-    "mock-ssh-server>=0.5.0",
+    "mock-ssh-server>=0.6.0",
     "moto>=1.3.14.dev55",
     "rangehttpserver==1.2.0",
 ]
@@ -144,7 +154,12 @@ setup(
         "ssh_gssapi": ssh_gssapi,
         "hdfs": hdfs,
         # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1
-        ":python_version=='2.7'": ["futures", "pathlib2"],
+        ":python_version=='2.7'": [
+            "futures",
+            "pathlib2",
+            "zc.lockfile>=1.2.1",
+        ],
+        ":python_version>='3.0'": ["flufl.lock>=3.2"],
         "tests": tests_requirements,
     },
     keywords="data science, data version control, machine learning",

tests/func/test_add.py

@@ -325,7 +325,7 @@ class TestAddCommit(TestDvc):
         ret = main(["add", self.FOO, "--no-commit"])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.FOO))
-        self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 0)
+        self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir))
 
         ret = main(["commit", self.FOO + ".dvc"])
         self.assertEqual(ret, 0)
@@ -565,12 +565,12 @@ def test_should_not_checkout_when_adding_cached_copy(repo_dir, dvc_repo):
 
     shutil.copy(repo_dir.BAR, repo_dir.FOO)
 
-    copy_spy = spy(shutil.copyfile)
+    copy_spy = spy(dvc_repo.cache.local.copy)
 
-    RemoteLOCAL.CACHE_TYPE_MAP["copy"] = copy_spy
-    dvc_repo.add(repo_dir.FOO)
+    with patch.object(dvc_repo.cache.local, "copy", copy_spy):
+        dvc_repo.add(repo_dir.FOO)
 
-    assert copy_spy.mock.call_count == 0
+        assert copy_spy.mock.call_count == 0
 
 
 @pytest.mark.parametrize(
@@ -585,13 +585,13 @@ def test_should_not_checkout_when_adding_cached_copy(repo_dir, dvc_repo):
 def test_should_relink_on_repeated_add(
     link, new_link, link_test_func, repo_dir, dvc_repo
 ):
-    dvc_repo.cache.local.cache_types = [link]
+    dvc_repo.config.set("cache", "type", link)
 
     dvc_repo.add(repo_dir.FOO)
     dvc_repo.add(repo_dir.BAR)
 
     os.remove(repo_dir.FOO)
-    RemoteLOCAL.CACHE_TYPE_MAP[link](repo_dir.BAR, repo_dir.FOO)
+    getattr(dvc_repo.cache.local, link)(repo_dir.BAR, repo_dir.FOO)
 
     dvc_repo.cache.local.cache_types = [new_link]
 
@@ -615,10 +615,11 @@ def test_should_relink_single_file_in_dir(link, link_func, dvc_repo, repo_dir):
     dvc_repo.unprotect(repo_dir.DATA_SUB)
 
     link_spy = spy(link_func)
-    RemoteLOCAL.CACHE_TYPE_MAP[link] = link_spy
-    dvc_repo.add(repo_dir.DATA_DIR)
 
-    assert link_spy.mock.call_count == 1
+    with patch.object(dvc_repo.cache.local, link, link_spy):
+        dvc_repo.add(repo_dir.DATA_DIR)
+
+        assert link_spy.mock.call_count == 1
 
 
 @pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
@@ -633,3 +634,20 @@ def test_should_protect_on_repeated_add(link, dvc_repo, repo_dir):
     dvc_repo.add(repo_dir.FOO)
 
     assert not os.access(repo_dir.FOO, os.W_OK)
+
+
+def test_escape_gitignore_entries(git, dvc_repo, repo_dir):
+    fname = "file!with*weird#naming_[1].t?t"
+    ignored_fname = r"/file\!with\*weird\#naming_\[1\].t\?t"
+
+    if os.name == "nt":
+        # Some characters are not supported by Windows in the filename
+        # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+        fname = "file!with_weird#naming_[1].txt"
+        ignored_fname = r"/file\!with_weird\#naming_\[1\].txt"
+
+    os.rename(repo_dir.FOO, fname)
+
+    dvc_repo.add(fname)
+
+    assert ignored_fname in get_gitignore_content()

tests/func/test_api.py

@@ -1,8 +1,10 @@
+import os
+
 import pytest
 import shutil
 
 from dvc import api
-from dvc.exceptions import OutputFileMissingError
+from dvc.exceptions import FileMissingError
 from dvc.main import main
 from dvc.path_info import URLInfo
 from dvc.remote.config import RemoteConfig
@@ -134,7 +136,7 @@ def test_missing(repo_dir, dvc_repo, remote_url):
     # Remove cache to make foo missing
     shutil.rmtree(dvc_repo.cache.local.cache_dir)
 
-    with pytest.raises(OutputFileMissingError):
+    with pytest.raises(FileMissingError):
         api.read(repo_dir.FOO)
 
 
@@ -143,3 +145,30 @@ def _set_remote_url_and_commit(repo, remote_url):
     rconfig.modify("upstream", "url", remote_url)
     repo.scm.add([repo.config.config_file])
     repo.scm.commit("modify remote")
+
+
+def test_open_scm_controlled(dvc_repo, repo_dir):
+    stage, = dvc_repo.add(repo_dir.FOO)
+
+    stage_content = open(stage.path, "r").read()
+    with api.open(stage.path) as fd:
+        assert fd.read() == stage_content
+
+
+def test_open_not_cached(dvc_repo):
+    metric_file = "metric.txt"
+    metric_content = "0.6"
+    metric_code = "open('{}', 'w').write('{}')".format(
+        metric_file, metric_content
+    )
+    dvc_repo.run(
+        metrics_no_cache=[metric_file],
+        cmd=('python -c "{}"'.format(metric_code)),
+    )
+
+    with api.open(metric_file) as fd:
+        assert fd.read() == metric_content
+
+    os.remove(metric_file)
+    with pytest.raises(FileMissingError):
+        api.read(metric_file)

tests/func/test_cache.py

@@ -1,11 +1,11 @@
 import os
 import stat
 import pytest
-import shutil
 import configobj
 
 from dvc.cache import Cache
 from dvc.main import main
+from dvc.remote.base import DirCacheError
 from dvc.utils import relpath
 
 from tests.basic_env import TestDvc, TestDir
@@ -46,13 +46,16 @@ class TestCacheLoadBadDirCache(TestDvc):
         self.assertEqual(len(ret), 0)
 
     def test(self):
-        fname = self.dvc.cache.local.get("123.dir")
+        checksum = "123.dir"
+        fname = self.dvc.cache.local.get(checksum)
         self.create(fname, "<clearly>not,json")
-        self._do_test(self.dvc.cache.local.load_dir_cache(fname))
+        with pytest.raises(DirCacheError):
+            self.dvc.cache.local.load_dir_cache(checksum)
 
-        fname = self.dvc.cache.local.get("234.dir")
+        checksum = "234.dir"
+        fname = self.dvc.cache.local.get(checksum)
         self.create(fname, '{"a": "b"}')
-        self._do_test(self.dvc.cache.local.load_dir_cache(fname))
+        self._do_test(self.dvc.cache.local.load_dir_cache(checksum))
 
 
 class TestExternalCacheDir(TestDvc):
@@ -62,7 +65,7 @@ class TestExternalCacheDir(TestDvc):
         ret = main(["config", "cache.dir", cache_dir])
         self.assertEqual(ret, 0)
 
-        shutil.rmtree(".dvc/cache")
+        self.assertFalse(os.path.exists(self.dvc.cache.local.cache_dir))
 
         ret = main(["add", self.FOO])
         self.assertEqual(ret, 0)
@@ -97,7 +100,7 @@ class TestSharedCacheDir(TestDir):
             ret = main(["config", "cache.dir", cache_dir])
             self.assertEqual(ret, 0)
 
-            shutil.rmtree(os.path.join(".dvc", "cache"))
+            self.assertFalse(os.path.exists(os.path.join(".dvc", "cache")))
 
             with open("common", "w+") as fd:
                 fd.write("common")

tests/func/test_checkout.py

@@ -4,6 +4,8 @@ import filecmp
 import collections
 import logging
 
+import pytest
+
 from dvc.main import main
 from dvc.repo import Repo as DvcRepo
 from dvc.system import System
@@ -17,12 +19,13 @@ from dvc.remote.local import RemoteLOCAL
 from dvc.exceptions import (
     DvcException,
     ConfirmRemoveError,
-    TargetNotDirectoryError,
     CheckoutErrorSuggestGit,
+    CheckoutError,
 )
 
 from mock import patch
 
+from tests.utils import spy
 
 logger = logging.getLogger("dvc")
 
@@ -71,7 +74,8 @@ class TestCheckoutCorruptedCacheFile(TestRepro):
         with open(cache, "a") as fd:
             fd.write("1")
 
-        self.dvc.checkout(force=True)
+        with pytest.raises(CheckoutError):
+            self.dvc.checkout(force=True)
 
         self.assertFalse(os.path.isfile(self.FOO))
         self.assertFalse(os.path.isfile(cache))
@@ -98,7 +102,8 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
         with open(cache, "w+") as fobj:
             fobj.write("1")
 
-        self.dvc.checkout(force=True)
+        with pytest.raises(CheckoutError):
+            self.dvc.checkout(force=True)
 
         self.assertFalse(os.path.exists(cache))
 
@@ -299,7 +304,8 @@ class TestCheckoutMissingMd5InStageFile(TestRepro):
         del d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM]
         dump_stage_file(self.file1_stage, d)
 
-        self.dvc.checkout(force=True)
+        with pytest.raises(CheckoutError):
+            self.dvc.checkout(force=True)
 
 
 class TestCheckoutEmptyDir(TestDvc):
@@ -416,10 +422,7 @@ class TestCheckoutRecursiveNotDirectory(TestDvc):
         ret = main(["add", self.FOO])
         self.assertEqual(0, ret)
 
-        try:
-            self.dvc.checkout(target=self.FOO + ".dvc", recursive=True)
-        except TargetNotDirectoryError:
-            self.fail("should not raise TargetNotDirectoryError")
+        self.dvc.checkout(target=self.FOO + ".dvc", recursive=True)
 
 
 class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
@@ -473,5 +476,59 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
 
 def test_checkout_no_checksum(repo_dir, dvc_repo):
     stage = dvc_repo.run(outs=[repo_dir.FOO], no_exec=True, cmd="somecmd")
-    dvc_repo.checkout(stage.path, force=True)
+    with pytest.raises(CheckoutError):
+        dvc_repo.checkout(stage.path, force=True)
     assert not os.path.exists(repo_dir.FOO)
+
+
+@pytest.mark.parametrize(
+    "link, link_test_func",
+    [("hardlink", System.is_hardlink), ("symlink", System.is_symlink)],
+)
+def test_should_relink_on_checkout(link, link_test_func, repo_dir, dvc_repo):
+    dvc_repo.cache.local.cache_types = [link]
+
+    dvc_repo.add(repo_dir.DATA_DIR)
+    dvc_repo.unprotect(repo_dir.DATA_SUB)
+
+    dvc_repo.checkout(repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
+
+    assert link_test_func(repo_dir.DATA_SUB)
+
+
+@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
+def test_should_protect_on_checkout(link, dvc_repo, repo_dir):
+    dvc_repo.cache.local.cache_types = [link]
+    dvc_repo.cache.local.protected = True
+
+    dvc_repo.add(repo_dir.FOO)
+    dvc_repo.unprotect(repo_dir.FOO)
+
+    dvc_repo.checkout(repo_dir.FOO + Stage.STAGE_FILE_SUFFIX)
+
+    assert not os.access(repo_dir.FOO, os.W_OK)
+
+
+def test_should_relink_only_one_file_in_dir(dvc_repo, repo_dir):
+    dvc_repo.cache.local.cache_types = ["symlink"]
+
+    dvc_repo.add(repo_dir.DATA_DIR)
+    dvc_repo.unprotect(repo_dir.DATA_SUB)
+
+    link_spy = spy(System.symlink)
+    with patch.object(dvc_repo.cache.local, "symlink", link_spy):
+        dvc_repo.checkout(repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
+
+    assert link_spy.mock.call_count == 1
+
+
+@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
+def test_should_not_relink_on_unchanged_dependency(link, dvc_repo, repo_dir):
+    dvc_repo.cache.local.cache_types = [link]
+
+    dvc_repo.add(repo_dir.DATA_DIR)
+
+    with patch.object(dvc_repo.cache.local, "link") as mock_link:
+        dvc_repo.checkout(repo_dir.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
+
+    assert not mock_link.called

tests/func/test_data_cloud.py

@@ -15,8 +15,8 @@ from mock import patch
 from dvc.utils.compat import str
 from dvc.main import main
 from dvc.config import Config
-from dvc.data_cloud import (
-    DataCloud,
+from dvc.data_cloud import DataCloud
+from dvc.remote import (
     RemoteS3,
     RemoteGS,
     RemoteAZURE,

tests/func/test_diff.py

@@ -8,22 +8,15 @@ import dvc.repo.diff as diff
 from tests.basic_env import TestDvcGit
 
 
-def _get_checksum(repo, file_name):
-    outs = [out for s in repo.stages() for out in s.outs]
-    for out in outs:
-        if out.def_path == file_name:
-            return out.checksum
-
-
 class TestDiff(TestDvcGit):
     def setUp(self):
         super(TestDiff, self).setUp()
 
         self.new_file = "new_test_file"
         self.create(self.new_file, self.new_file)
-        self.dvc.add(self.new_file)
+        stage = self.dvc.add(self.new_file)[0]
         self.a_ref = self.git.git.rev_parse(self.git.head.commit, short=True)
-        self.new_checksum = _get_checksum(self.dvc, self.new_file)
+        self.new_checksum = stage.outs[0].checksum
         self.git.index.add([self.new_file + ".dvc"])
         self.git.index.commit("adds new_file")
         self.test_dct = {
@@ -67,19 +60,19 @@ class TestDiffDir(TestDvcGit):
     def setUp(self):
         super(TestDiffDir, self).setUp()
 
-        self.dvc.add(self.DATA_DIR)
+        stage = self.dvc.add(self.DATA_DIR)[0]
         self.git.index.add([self.DATA_DIR + ".dvc"])
         self.git.index.commit("adds data_dir")
         self.a_ref = self.git.git.rev_parse(
             self.dvc.scm.repo.head.commit, short=True
         )
-        self.old_checksum = _get_checksum(self.dvc, self.DATA_DIR)
+        self.old_checksum = stage.outs[0].checksum
         self.new_file = os.path.join(self.DATA_SUB_DIR, diff.DIFF_NEW_FILE)
         self.create(self.new_file, self.new_file)
-        self.dvc.add(self.DATA_DIR)
+        stage = self.dvc.add(self.DATA_DIR)[0]
         self.git.index.add([self.DATA_DIR + ".dvc"])
         self.git.index.commit(message="adds data_dir with new_file")
-        self.new_checksum = _get_checksum(self.dvc, self.DATA_DIR)
+        self.new_checksum = stage.outs[0].checksum
 
     def test(self):
         out = self.dvc.scm.get_diff_trees(self.a_ref)
@@ -128,9 +121,10 @@ class TestDiffDirRepoDeletedFile(TestDiffDir):
         super(TestDiffDirRepoDeletedFile, self).setUp()
 
         self.b_ref = self.a_ref
+        tmp = self.new_checksum
         self.new_checksum = self.old_checksum
         self.a_ref = str(self.dvc.scm.repo.head.commit)
-        self.old_checksum = _get_checksum(self.dvc, self.DATA_DIR)
+        self.old_checksum = tmp
 
     def test(self):
         result = self.dvc.diff(
@@ -175,14 +169,14 @@ class TestDiffModifiedFile(TestDiff):
     def setUp(self):
         super(TestDiffModifiedFile, self).setUp()
 
-        self.old_checksum = _get_checksum(self.dvc, self.new_file)
+        self.old_checksum = self.new_checksum
         self.new_file_content = "new_test_file_bigger_content_123456789"
         self.diff_len = len(self.new_file) + len(self.new_file_content)
         self.create(self.new_file, self.new_file_content)
-        self.dvc.add(self.new_file)
+        stage = self.dvc.add(self.new_file)[0]
         self.git.index.add([self.new_file + ".dvc"])
         self.git.index.commit("change new_file content to be bigger")
-        self.new_checksum = _get_checksum(self.dvc, self.new_file)
+        self.new_checksum = stage.outs[0].checksum
         self.b_ref = self.git.git.rev_parse(self.git.head.commit, short=True)
 
     def test(self):
@@ -211,14 +205,14 @@ class TestDiffDirWithFile(TestDiffDir):
         super(TestDiffDirWithFile, self).setUp()
 
         self.a_ref = self.git.git.rev_parse(self.git.head.commit, short=True)
-        self.old_checksum = _get_checksum(self.dvc, self.DATA_DIR)
+        self.old_checksum = self.new_checksum
         self.new_file_content = "new_test_file_bigger_content_123456789"
         self.diff_len = len(self.new_file_content)
         self.create(self.new_file, self.new_file_content)
-        self.dvc.add(self.DATA_DIR)
+        stage = self.dvc.add(self.DATA_DIR)[0]
         self.git.index.add([self.DATA_DIR + ".dvc"])
         self.git.index.commit(message="modify file in the data dir")
-        self.new_checksum = _get_checksum(self.dvc, self.DATA_DIR)
+        self.new_checksum = stage.outs[0].checksum
         self.b_ref = self.git.git.rev_parse(self.git.head.commit, short=True)
 
     def test(self):

tests/func/test_external_repo.py

@@ -0,0 +1,28 @@
+import os
+from mock import patch
+
+from dvc.external_repo import external_repo
+from dvc.scm.git import Git
+
+
+def test_external_repo(erepo):
+    url = erepo.root_dir
+    # We will share cache dir, to fetch version file
+    cache_dir = erepo.dvc.cache.local.cache_dir
+
+    with patch.object(Git, "clone", wraps=Git.clone) as mock:
+        with external_repo(url, cache_dir=cache_dir) as repo:
+            with repo.open(os.path.join(repo.root_dir, "version")) as fd:
+                assert fd.read() == "master"
+
+        with external_repo(url, rev="branch", cache_dir=cache_dir) as repo:
+            with repo.open(os.path.join(repo.root_dir, "version")) as fd:
+                assert fd.read() == "branch"
+
+        # Check cache_dir is unset
+        with external_repo(url) as repo:
+            assert repo.cache.local.cache_dir.startswith(
+                repo.root_dir + os.sep
+            )
+
+        assert mock.call_count == 1

tests/func/test_get.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import filecmp
 
@@ -5,6 +7,7 @@ import pytest
 
 from dvc.exceptions import UrlNotDvcRepoError, GetDVCFileError
 from dvc.repo import Repo
+from dvc.utils import makedirs
 
 from tests.utils import trees_equal
 
@@ -51,3 +54,17 @@ def test_get_from_non_dvc_repo(git_erepo):
 def test_get_a_dvc_file(repo_dir, erepo):
     with pytest.raises(GetDVCFileError):
         Repo.get(erepo.root_dir, "some_file.dvc")
+
+
+@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
+def test_get_to_dir(dname, repo_dir, erepo):
+    src = erepo.FOO
+
+    makedirs(dname, exist_ok=True)
+
+    Repo.get(erepo.root_dir, src, dname)
+
+    dst = os.path.join(dname, os.path.basename(src))
+
+    assert os.path.isdir(dname)
+    assert filecmp.cmp(repo_dir.FOO, dst, shallow=False)

tests/func/test_get_url.py

@@ -1,7 +1,12 @@
+from __future__ import unicode_literals
+
 import os
 import filecmp
 
+import pytest
+
 from dvc.repo import Repo
+from dvc.utils import makedirs
 
 
 def test_get_file(repo_dir):
@@ -13,3 +18,17 @@ def test_get_file(repo_dir):
     assert os.path.exists(dst)
     assert os.path.isfile(dst)
     assert filecmp.cmp(repo_dir.FOO, dst, shallow=False)
+
+
+@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
+def test_get_url_to_dir(dname, repo_dir):
+    src = repo_dir.DATA
+
+    makedirs(dname, exist_ok=True)
+
+    Repo.get_url(src, dname)
+
+    dst = os.path.join(dname, os.path.basename(src))
+
+    assert os.path.isdir(dname)
+    assert filecmp.cmp(repo_dir.DATA, dst, shallow=False)

tests/func/test_import.py

@@ -1,8 +1,16 @@
+from __future__ import unicode_literals
+
 import os
 import filecmp
 
+import pytest
+from mock import patch
 from tests.utils import trees_equal
 
+from dvc.stage import Stage
+from dvc.utils import makedirs
+from dvc.exceptions import DownloadError
+
 
 def test_import(repo_dir, git, dvc_repo, erepo):
     src = erepo.FOO
@@ -39,3 +47,54 @@ def test_import_rev(repo_dir, git, dvc_repo, erepo):
     with open(dst, "r+") as fobj:
         assert fobj.read() == "branch"
     assert git.git.check_ignore(dst)
+
+
+def test_pull_imported_stage(dvc_repo, erepo):
+    src = erepo.FOO
+    dst = erepo.FOO + "_imported"
+
+    dvc_repo.imp(erepo.root_dir, src, dst)
+
+    dst_stage = Stage.load(dvc_repo, "foo_imported.dvc")
+    dst_cache = dst_stage.outs[0].cache_path
+
+    os.remove(dst)
+    os.remove(dst_cache)
+
+    dvc_repo.pull(["foo_imported.dvc"])
+
+    assert os.path.isfile(dst)
+    assert os.path.isfile(dst_cache)
+
+
+def test_download_error_pulling_imported_stage(dvc_repo, erepo):
+    src = erepo.FOO
+    dst = erepo.FOO + "_imported"
+
+    dvc_repo.imp(erepo.root_dir, src, dst)
+
+    dst_stage = Stage.load(dvc_repo, "foo_imported.dvc")
+    dst_cache = dst_stage.outs[0].cache_path
+
+    os.remove(dst)
+    os.remove(dst_cache)
+
+    with patch(
+        "dvc.remote.RemoteLOCAL._download", side_effect=Exception
+    ), pytest.raises(DownloadError):
+        dvc_repo.pull(["foo_imported.dvc"])
+
+
+@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
+def test_import_to_dir(dname, repo_dir, dvc_repo, erepo):
+    src = erepo.FOO
+
+    makedirs(dname, exist_ok=True)
+
+    stage = dvc_repo.imp(erepo.root_dir, src, dname)
+
+    dst = os.path.join(dname, os.path.basename(src))
+
+    assert stage.outs[0].fspath == os.path.abspath(dst)
+    assert os.path.isdir(dname)
+    assert filecmp.cmp(repo_dir.FOO, dst, shallow=False)

tests/func/test_import_url.py

@@ -1,12 +1,17 @@
-import dvc
-
-from dvc.utils.compat import str
+from __future__ import unicode_literals
 
 import os
 from uuid import uuid4
 
-from dvc.main import main
+import pytest
+import filecmp
 from mock import patch
+
+import dvc
+from dvc.main import main
+from dvc.utils import makedirs
+from dvc.utils.compat import str
+
 from tests.basic_env import TestDvc
 from tests.utils import spy
 
@@ -86,3 +91,18 @@ class TestImportFilename(TestDvc):
         ret = main(["import-url", "--file", path, self.external_source])
         self.assertEqual(0, ret)
         self.assertTrue(os.path.exists(path))
+
+
+@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
+def test_import_url_to_dir(dname, repo_dir, dvc_repo):
+    src = repo_dir.DATA
+
+    makedirs(dname, exist_ok=True)
+
+    stage = dvc_repo.imp_url(src, dname)
+
+    dst = os.path.join(dname, os.path.basename(src))
+
+    assert stage.outs[0].fspath == os.path.abspath(dst)
+    assert os.path.isdir(dname)
+    assert filecmp.cmp(repo_dir.DATA, dst, shallow=False)

tests/func/test_install.py

@@ -2,6 +2,8 @@ import os
 import sys
 
 import pytest
+
+from dvc.exceptions import GitHookAlreadyExistsError
 from dvc.utils import file_md5
 
 from dvc.main import main
@@ -30,36 +32,12 @@ class TestInstall(object):
             with open(self._hook(fname), "r") as fobj:
                 assert command in fobj.read()
 
-    def test_should_append_hooks_if_file_already_exists(self, git, dvc_repo):
+    def test_should_fail_if_file_already_exists(self, git, dvc_repo):
         with open(self._hook("post-checkout"), "w") as fobj:
-            fobj.write("#!/bin/sh\n" "echo hello\n")
+            fobj.write("hook content")
 
-        assert main(["install"]) == 0
-
-        expected_script = (
-            "#!/bin/sh\n"
-            "echo hello\n"
-            '[ "$3" = "0" ]'
-            ' || [ -z "$(git ls-files .dvc)" ]'
-            " || exec dvc checkout\n"
-        )
-
-        with open(self._hook("post-checkout"), "r") as fobj:
-            assert fobj.read() == expected_script
-
-    def test_should_be_idempotent(self, git, dvc_repo):
-        assert main(["install"]) == 0
-        assert main(["install"]) == 0
-
-        expected_script = (
-            "#!/bin/sh\n"
-            '[ "$3" = "0" ]'
-            ' || [ -z "$(git ls-files .dvc)" ]'
-            " || exec dvc checkout\n"
-        )
-
-        with open(self._hook("post-checkout"), "r") as fobj:
-            assert fobj.read() == expected_script
+        with pytest.raises(GitHookAlreadyExistsError):
+            dvc_repo.scm.install()
 
     def test_should_post_checkout_hook_checkout(self, repo_dir, git, dvc_repo):
         assert main(["install"]) == 0

tests/func/test_lock.py

@@ -1,3 +1,5 @@
+import os
+
 from dvc.lock import LockError
 from dvc.main import main
 from dvc.lock import Lock
@@ -7,15 +9,17 @@ from tests.basic_env import TestDvc
 
 class TestLock(TestDvc):
     def test_with(self):
-        lock = Lock(self.dvc.dvc_dir)
+        lockfile = os.path.join(self.dvc.dvc_dir, "lock")
+        lock = Lock(lockfile)
         with lock:
             with self.assertRaises(LockError):
-                lock2 = Lock(self.dvc.dvc_dir)
+                lock2 = Lock(lockfile)
                 with lock2:
                     self.assertTrue(False)
 
     def test_cli(self):
-        lock = Lock(self.dvc.dvc_dir)
+        lockfile = os.path.join(self.dvc.dvc_dir, "lock")
+        lock = Lock(lockfile)
         with lock:
             ret = main(["add", self.FOO])
             self.assertEqual(ret, 1)

tests/func/test_move.py

@@ -200,3 +200,13 @@ def test_move_should_save_stage_info(dvc_repo, repo_dir):
     dvc_repo.move(repo_dir.DATA_DIR, "new_name")
 
     assert dvc_repo.status() == {}
+
+
+def test_should_move_to_dir_on_non_default_stage_file(dvc_repo, repo_dir):
+    stage_file_name = "stage.dvc"
+
+    dvc_repo.add(repo_dir.FOO, fname=stage_file_name)
+
+    dvc_repo.move(repo_dir.FOO, repo_dir.DATA_DIR)
+
+    assert os.path.exists(os.path.join(repo_dir.DATA_DIR, repo_dir.FOO))

tests/func/test_pipeline.py

@@ -186,14 +186,14 @@ class TestPipelineListSingle(TestPipelineShowDeep):
 
 class TestDvcRepoPipeline(TestDvc):
     def test_no_stages(self):
-        pipelines = self.dvc.pipelines()
+        pipelines = self.dvc.pipelines
         self.assertEqual(len(pipelines), 0)
 
     def one_pipeline(self):
         self.dvc.add("foo")
         self.dvc.run(deps=["foo"], outs=["bar"], cmd="")
         self.dvc.run(deps=["bar"], outs=["baz"], cmd="echo baz > baz")
-        pipelines = self.dvc.pipelines()
+        pipelines = self.dvc.pipelines
 
         self.assertEqual(len(pipelines), 1)
         self.assertEqual(pipelines[0].nodes, 3)
@@ -206,7 +206,7 @@ class TestDvcRepoPipeline(TestDvc):
 
         self.dvc.add("code.py")
 
-        pipelines = self.dvc.pipelines()
+        pipelines = self.dvc.pipelines
 
         self.assertEqual(len(pipelines), 2)
         self.assertEqual(pipelines[0].nodes, 3)
@@ -218,5 +218,5 @@ class TestDvcRepoPipeline(TestDvc):
         self.dvc.add("foo")
         self.dvc.lock_stage("foo.dvc")
 
-        pipelines = self.dvc.pipelines()
+        pipelines = self.dvc.pipelines
         self.assertEqual(len(pipelines), 0)

tests/func/test_repo.py

@@ -56,13 +56,13 @@ class TestIgnore(TestDvcGit):
         ret = main(["add", self.FOO, self.BAR, self.DATA, self.DATA_SUB])
         self.assertEqual(0, ret)
 
-        stages = self.dvc.stages()
+        stages = self.dvc.stages
         self.assertEqual(4, len(stages))
 
         self.create(DvcIgnore.DVCIGNORE_FILE, self.DATA_DIR)
 
         self.dvc = Repo(self.dvc.root_dir)
-        stages = self.dvc.stages()
+        stages = self.dvc.stages
         self.assertEqual(2, len(stages))
 
         stagenames = [s.relpath for s in stages]