dvc: bump to 0.41.2

remote: support remote:// notation for push/pull/gc

.appveyor.yml

@@ -26,16 +26,11 @@ environment:
 
   matrix:
 
-    - PYTHON: "C:\\Python27"
-      PYTHON_VERSION: "2.7.x" # currently 2.7.9
+    - PYTHON: "C:\\Python37"
+      PYTHON_VERSION: "3.7.x"
       PYTHON_ARCH: "32"
       RUBY: C:\Ruby25\bin
 
-# Disabled as unnecessary
-#    - PYTHON: "C:\\Python27-x64"
-#      PYTHON_VERSION: "2.7.x" # currently 2.7.9
-#      PYTHON_ARCH: "64"
-
 install:
   - SET PATH=%RUBY%;%PATH%
   - git clone -q https://github.com/nanaya-tachibana/oss-emulator.git
@@ -52,7 +47,7 @@ install:
   - cinst gsutil
   - cinst openssl.light --version 1.1.1
   - python -m pip install -U pip setuptools wheel
-  - pip install -r requirements.txt
+  - pip install -e .[all]
   - python setup.py install
 
 build: false
@@ -62,7 +57,7 @@ before_test:
   - aws configure set aws_secret_access_key "%aws_secret_access_key%"
   - aws configure set region us-east-2
   - openssl enc -d -aes-256-cbc -md md5 -k "%GCP_CREDS%" -in scripts\ci\gcp-creds.json.enc -out scripts\ci\gcp-creds.json & exit 0
-  - pip install -r tests/requirements.txt
+  - pip install -e .[tests]
 
 test_script:
   - python -mtests
@@ -86,4 +81,4 @@ deploy:
   auth_token:
     secure: 25c1pNoXWbtTwUkKLcThwcl2A2xG8oXHsPuD73HVbQ7SRy9X6HzkCc2sXzhpfRtF
   on:
-    appveyor_repo_tag: true
+    APPVEYOR_REPO_TAG: true

.gitignore

@@ -28,3 +28,9 @@ innosetup/config.ini
 
 pip-wheel-metadata/
 .vscode/
+
+azurite
+env.sh
+tests/remotes_env
+scripts/ci/gcp-creds.json
+.gcp-creds.json

.travis.yml

@@ -48,7 +48,7 @@ matrix:
     language: ruby
     rvm: 2.4.0
     env:
-    - PYTHON_VER=3.6.2
+    - PYTHON_VER=3.7.3
     - PYTHON_CONFIGURE_OPTS="--enable-framework"
   - if: type = cron OR tag is present
     os: osx
@@ -84,17 +84,34 @@ install:
   - bash ./scripts/ci/install.sh
 script:
   - bash ./scripts/ci/script.sh
+before_deploy:
+  - bash ./scripts/ci/before_deploy.sh
+  - pip install twine
 deploy:
-  condition: "[[ $(./scripts/ci/deploy_condition.sh) == 'true' ]]"
-  provider: releases
-  api_key:
-    secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
-  file_glob: true
-  file:
-    - dvc*.rpm
-    - dvc*.deb
-    - dvc*.pkg
-  skip_cleanup: true
-  on:
-    tags: true
-    repo: iterative/dvc
+  - provider: releases
+    condition: "[[ $(./scripts/ci/deploy_condition.sh) == 'true' ]]"
+    api_key:
+      secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
+    file_glob: true
+    file:
+      - dvc*.rpm
+      - dvc*.deb
+      - dvc*.pkg
+    skip_cleanup: true
+    on:
+      tags: true
+      repo: iterative/dvc
+  - provider: pypi
+    user:
+      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
+    password:
+      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
+    skip_existing: true
+    file_glob: true
+    file:
+      - dist/dvc*.whl
+      - dist/dvc*.tar.gz
+    skip_cleanup: true
+    on:
+      tags: true
+      repo: iterative/dvc

dvc/api.py

@@ -0,0 +1,15 @@
+from dvc.repo import Repo
+
+
+def get_url(path, repo_dir=None, remote=None):
+    """Returns an url of `path` in default or specified remote"""
+    repo = Repo(repo_dir)
+    out, = repo.find_outs_by_path(path)
+    remote_obj = repo.cloud.get_remote(remote)
+    return str(remote_obj.checksum_to_path_info(out.checksum))
+
+
+def open(path, repo_dir=None, remote=None, mode="r", encoding=None):
+    """Opens a specified resource as a file descriptor"""
+    repo = Repo(repo_dir)
+    return repo.open(path, remote=remote, mode=mode, encoding=encoding)

dvc/command/pipeline.py

@@ -115,7 +115,8 @@ class CmdPipelineShow(CmdBase):
             observe_list.pop(0)
         tree.show()
 
-    def __write_dot(self, target, commands, outs, filename):
+    def __write_dot(self, target, commands, outs):
+        from dvc.utils.compat import StringIO
         import networkx
         from networkx.drawing.nx_pydot import write_dot
 
@@ -124,7 +125,10 @@ class CmdPipelineShow(CmdBase):
 
         simple_g = networkx.DiGraph()
         simple_g.add_edges_from(edges)
-        write_dot(simple_g, filename)
+
+        dot_file = StringIO()
+        write_dot(simple_g, dot_file)
+        logger.info(dot_file.getvalue())
 
     def run(self):
         if not self.args.targets:
@@ -138,10 +142,7 @@ class CmdPipelineShow(CmdBase):
                     )
                 elif self.args.dot:
                     self.__write_dot(
-                        target,
-                        self.args.commands,
-                        self.args.outs,
-                        self.args.dot,
+                        target, self.args.commands, self.args.outs
                     )
                 elif self.args.tree:
                     self._show_dependencies_tree(
@@ -231,7 +232,10 @@ def add_parser(subparsers, parent_parser):
         help="Output DAG as ASCII.",
     )
     pipeline_show_parser.add_argument(
-        "--dot", help="Write DAG in .dot format."
+        "--dot",
+        action="store_true",
+        default=False,
+        help="Print DAG with .dot format.",
     )
     pipeline_show_parser.add_argument(
         "--tree",

dvc/command/repro.py

@@ -15,7 +15,6 @@ logger = logging.getLogger(__name__)
 
 class CmdRepro(CmdBase):
     def run(self):
-        recursive = not self.args.single_item
         saved_dir = os.path.realpath(os.curdir)
         if self.args.cwd:
             os.chdir(self.args.cwd)
@@ -31,7 +30,7 @@ class CmdRepro(CmdBase):
             try:
                 stages = self.repo.reproduce(
                     target,
-                    recursive=recursive,
+                    single_item=self.args.single_item,
                     force=self.args.force,
                     dry=self.args.dry,
                     interactive=self.args.interactive,
@@ -40,6 +39,7 @@ class CmdRepro(CmdBase):
                     ignore_build_cache=self.args.ignore_build_cache,
                     no_commit=self.args.no_commit,
                     downstream=self.args.downstream,
+                    recursive=self.args.recursive,
                 )
 
                 if len(stages) == 0:
@@ -128,6 +128,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Reproduce all pipelines in the repo.",
     )
+    repro_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Reproduce all stages in the specified directory.",
+    )
     repro_parser.add_argument(
         "--ignore-build-cache",
         action="store_true",

dvc/config.py

@@ -498,7 +498,15 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
             >>> get_remote_settings("server")
             {'url': 'ssh://localhost/'}
         """
-        settings = self.config[self.SECTION_REMOTE_FMT.format(name)]
+        settings = self.config.get(
+            self.SECTION_REMOTE_FMT.format(name.lower())
+        )
+
+        if settings is None:
+            raise ConfigError(
+                "unable to find remote section '{}'".format(name)
+            )
+
         parsed = urlparse(settings["url"])
 
         # Support for cross referenced remotes.

dvc/data_cloud.py

@@ -43,40 +43,44 @@ class DataCloud(object):
         "https": RemoteHTTP,
     }
 
-    def __init__(self, repo, config=None):
+    def __init__(self, repo):
         self.repo = repo
-        self._config = config
-        self._core = self._config[Config.SECTION_CORE]
 
     @property
-    def _cloud(self):
-        """Returns a Remote instance using the `core.remote` config"""
-        remote = self._core.get(Config.SECTION_CORE_REMOTE, "")
+    def _config(self):
+        return self.repo.config.config
+
+    @property
+    def _core(self):
+        return self._config.get(Config.SECTION_CORE, {})
+
+    def get_remote(self, remote=None, command="<command>"):
+        if not remote:
+            remote = self._core.get(Config.SECTION_CORE_REMOTE)
 
         if remote:
             return self._init_remote(remote)
 
-        if self._core.get(Config.SECTION_CORE_CLOUD, None):
-            # backward compatibility
+        # Old config format support for backward compatibility
+        if Config.SECTION_CORE_CLOUD in self._core:
             msg = "using obsoleted config format. Consider updating."
             logger.warning(msg)
             return self._init_compat()
 
-        return None
+        raise ConfigError(
+            "No remote repository specified. Setup default repository with\n"
+            "    dvc config core.remote <name>\n"
+            "or use:\n"
+            "    dvc {} -r <name>\n".format(command)
+        )
 
     def _init_remote(self, remote):
-        section = Config.SECTION_REMOTE_FMT.format(remote).lower()
-        cloud_config = self._config.get(section, None)
-        if not cloud_config:
-            msg = "can't find remote section '{}' in config"
-            raise ConfigError(msg.format(section))
-
-        return Remote(self.repo, cloud_config)
+        config = self.repo.config.get_remote_settings(remote)
+        return Remote(self.repo, config)
 
     def _init_compat(self):
         name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower()
         if name == "":
-            self._cloud = None
             return None
 
         cloud_type = self.CLOUD_MAP.get(name, None)
@@ -104,20 +108,6 @@ class DataCloud(object):
         cloud = cloud_type(self.repo, cloud_config)
         return cloud
 
-    def _get_cloud(self, remote, cmd):
-        if remote:
-            return self._init_remote(remote)
-
-        if self._cloud:
-            return self._cloud
-
-        raise ConfigError(
-            "No remote repository specified. Setup default repository with\n"
-            "    dvc config core.remote <name>\n"
-            "or use:\n"
-            "    dvc {} -r <name>\n".format(cmd)
-        )
-
     def push(self, targets, jobs=None, remote=None, show_checksums=False):
         """Push data items in a cloud-agnostic way.
 
@@ -132,7 +122,7 @@ class DataCloud(object):
         return self.repo.cache.local.push(
             targets,
             jobs=jobs,
-            remote=self._get_cloud(remote, "push"),
+            remote=self.get_remote(remote, "push"),
             show_checksums=show_checksums,
         )
 
@@ -150,7 +140,7 @@ class DataCloud(object):
         return self.repo.cache.local.pull(
             targets,
             jobs=jobs,
-            remote=self._get_cloud(remote, "pull"),
+            remote=self.get_remote(remote, "pull"),
             show_checksums=show_checksums,
         )
 
@@ -166,7 +156,7 @@ class DataCloud(object):
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
-        cloud = self._get_cloud(remote, "status")
+        remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            targets, jobs=jobs, remote=cloud, show_checksums=show_checksums
+            targets, jobs=jobs, remote=remote, show_checksums=show_checksums
         )

dvc/path/hdfs.py

@@ -1,4 +1,3 @@
-from dvc.utils.compat import urlunsplit
 from dvc.scheme import Schemes
 from .base import PathBASE
 
@@ -11,6 +10,4 @@ class PathHDFS(PathBASE):
         self.user = user
 
     def __str__(self):
-        if not self.url:
-            return urlunsplit((self.scheme, self.user, self.path, "", ""))
-        return self.url
+        return self.path

dvc/path/local.py

@@ -8,4 +8,10 @@ class PathLOCAL(PathBASE):
     scheme = Schemes.LOCAL
 
     def __str__(self):
+        if os.name == "nt" and not os.path.commonprefix(
+            [os.getcwd(), self.path]
+        ):
+            # In case of windows, when cache is on different drive than
+            # workspace, we will get ValueError when trying to get relpath
+            return self.path
         return os.path.relpath(self.path)

dvc/remote/base.py

@@ -503,6 +503,17 @@ class RemoteBASE(object):
         return removed
 
     def changed_cache_file(self, checksum):
+        """Compare the given checksum with the (corresponding) actual one.
+
+        - Use `State` as a cache for computed checksums
+            + The entries are invalidated by taking into account the following:
+                * mtime
+                * inode
+                * size
+                * checksum
+
+        - Remove the file from cache if it doesn't match the actual checksum
+        """
         cache_info = self.checksum_to_path_info(checksum)
         actual = self.get_checksum(cache_info)
 
@@ -524,14 +535,7 @@ class RemoteBASE(object):
 
         return True
 
-    def _changed_cache_dir(self):
-        # NOTE: only implemented for RemoteLOCAL
-        return True
-
     def _changed_dir_cache(self, checksum):
-        if not self._changed_cache_dir():
-            return False
-
         if self.changed_cache_file(checksum):
             return True
 

dvc/remote/local/__init__.py

@@ -38,8 +38,6 @@ from dvc.exceptions import DvcException
 from dvc.progress import progress
 from concurrent.futures import ThreadPoolExecutor
 
-from dvc.utils.fs import get_mtime_and_size, get_inode
-
 
 logger = logging.getLogger(__name__)
 
@@ -495,19 +493,6 @@ class RemoteLOCAL(RemoteBASE):
             download=True,
         )
 
-    def _changed_cache_dir(self):
-        mtime, size = get_mtime_and_size(self.cache_dir)
-        inode = get_inode(self.cache_dir)
-
-        existing_record = self.state.get_state_record_for_inode(inode)
-        if existing_record:
-            cached_mtime, cached_size, _, _ = existing_record
-            changed = not (mtime == cached_mtime and size == cached_size)
-        else:
-            changed = True
-
-        return changed
-
     def _log_missing_caches(self, checksum_info_dict):
         missing_caches = [
             (md5, info)

dvc/remote/local/slow_link_detection.py

@@ -31,13 +31,12 @@ class SlowLinkDetectorDecorator(object):
             and self.should_display()
         ):
             msg = (
-                "You can cut execution time considerably. Check:\n"
+                "You can cut execution time considerably by using a different"
+                " `cache.type` configuration.\nSee "
                 "{blue}https://dvc.org/doc/commands-reference/config#cache{"
-                "reset}"
-                "\nfor "
-                "more information.\nTo disable this message, run:\n'dvc "
-                "config "
-                "cache.slow_link_warning False'".format(
+                "reset} for more information.\n"
+                "To disable this message, run:\n"
+                "'dvc config cache.slow_link_warning false'".format(
                     blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
                 )
             )
@@ -49,7 +48,7 @@ class SlowLinkDetectorDecorator(object):
 def slow_link_guard(method):
     def call(remote_local, *args, **kwargs):
         cache_config = remote_local.repo.config.config.get(
-            Config.SECTION_CACHE
+            Config.SECTION_CACHE, {}
         )
         should_warn = cache_config.get(
             Config.SECTION_CACHE_SLOW_LINK_WARNING, True

dvc/remote/ssh/__init__.py

@@ -234,13 +234,10 @@ class RemoteSSH(RemoteBASE):
                     )
                 except Exception:
                     logger.exception(
-                        "failed to upload '{host}/{path}' to '{dest}'".format(
-                            host=from_info.host,
-                            path=from_info.path,
-                            dest=to_info.path,
+                        "failed to upload '{}' to '{}'".format(
+                            from_info, to_info
                         )
                     )
-                    pass
 
     def list_cache_paths(self):
         with self.ssh(self.path_info) as ssh:

dvc/repo/__init__.py

@@ -13,6 +13,7 @@ from dvc.exceptions import (
 )
 from dvc.ignore import DvcIgnoreFileHandler
 from dvc.path.local import PathLOCAL
+from dvc.utils.compat import open as _open
 
 logger = logging.getLogger(__name__)
 
@@ -74,7 +75,7 @@ class Repo(object):
             logger.setLevel(level.upper())
 
         self.cache = Cache(self)
-        self.cloud = DataCloud(self, config=self.config.config)
+        self.cloud = DataCloud(self)
         self.updater = Updater(self.dvc_dir)
 
         self.metrics = Metrics(self)
@@ -157,7 +158,7 @@ class Repo(object):
         import networkx as nx
         from dvc.stage import Stage
 
-        if not target or recursive:
+        if not target or (recursive and os.path.isdir(target)):
             return self.active_stages(target)
 
         stage = Stage.load(self, target)
@@ -176,6 +177,26 @@ class Repo(object):
     def _collect_dir_cache(
         self, out, branch=None, remote=None, force=False, jobs=None
     ):
+        """Get a list of `info`s retaled to the given directory.
+
+        - Pull the directory entry from the remote cache if it was changed.
+
+        Example:
+
+            Given the following commands:
+
+            $ echo "foo" > directory/foo
+            $ echo "bar" > directory/bar
+            $ dvc add directory
+
+            It will return something similar to the following list:
+
+            [
+                { 'path': 'directory',     'md5': '168fd6761b9c.dir', ... },
+                { 'path': 'directory/foo', 'md5': 'c157a79031e1', ... },
+                { 'path': 'directory/bar', 'md5': 'd3b07384d113', ... },
+            ]
+        """
         info = out.dumpd()
         ret = [info]
         r = out.remote
@@ -216,6 +237,13 @@ class Repo(object):
     def _collect_used_cache(
         self, out, branch=None, remote=None, force=False, jobs=None
     ):
+        """Get a dumpd of the given `out`, with an entry including the branch.
+
+        The `used_cache` of an output is no more than its `info`.
+
+        In case that the given output is a directory, it will also
+        include the `info` of its files.
+        """
         if not out.use_cache or not out.info:
             if not out.info:
                 logger.warning(
@@ -252,6 +280,20 @@ class Repo(object):
         jobs=None,
         recursive=False,
     ):
+        """Get the stages related to the given target and collect
+        the `info` of its outputs.
+
+        This is useful to know what files from the cache are _in use_
+        (namely, a file described as an output on a stage).
+
+        The scope is, by default, the working directory, but you can use
+        `all_branches` or `all_tags` to expand scope.
+
+        Returns:
+            A dictionary with Schemes (representing output's location) as keys,
+            and a list with the outputs' `dumpd` as values.
+        """
+
         cache = {}
         cache["local"] = []
         cache["s3"] = []
@@ -264,7 +306,7 @@ class Repo(object):
             all_branches=all_branches, all_tags=all_tags
         ):
             if target:
-                if recursive:
+                if recursive and os.path.isdir(target):
                     stages = self.stages(target)
                 else:
                     stages = self.collect(target, with_deps=with_deps)
@@ -497,3 +539,16 @@ class Repo(object):
     def is_dvc_internal(self, path):
         path_parts = os.path.normpath(path).split(os.path.sep)
         return self.DVC_DIR in path_parts
+
+    def open(self, path, remote=None, mode="r", encoding=None):
+        """Opens a specified resource as a file descriptor"""
+        out, = self.find_outs_by_path(path)
+        if out.isdir():
+            raise ValueError("Can't open a dir")
+
+        with self.state:
+            cache_info = self._collect_used_cache(out, remote=remote)
+            self.cloud.pull(cache_info, remote=remote)
+
+        cache_filename = self.cache.local.checksum_to_path(out.checksum)
+        return _open(cache_filename, mode=mode, encoding=encoding)

dvc/repo/gc.py

@@ -110,4 +110,4 @@ def gc(
             _do_gc("azure", self.cache.azure.gc, clist)
 
         if cloud:
-            _do_gc("remote", self.cloud._get_cloud(remote, "gc -c").gc, clist)
+            _do_gc("remote", self.cloud.get_remote(remote, "gc -c").gc, clist)

dvc/repo/reproduce.py

@@ -35,7 +35,7 @@ def _reproduce_stage(stages, node, force, dry, interactive, no_commit):
 def reproduce(
     self,
     target=None,
-    recursive=True,
+    single_item=False,
     force=False,
     dry=False,
     interactive=False,
@@ -44,7 +44,9 @@ def reproduce(
     ignore_build_cache=False,
     no_commit=False,
     downstream=False,
+    recursive=False,
 ):
+    import networkx as nx
     from dvc.stage import Stage
 
     if not target and not all_pipelines:
@@ -56,7 +58,13 @@ def reproduce(
         interactive = core.get(config.SECTION_CORE_INTERACTIVE, False)
 
     targets = []
-    if pipeline or all_pipelines:
+    if recursive and os.path.isdir(target):
+        G = self.graph(from_directory=target)[1]
+        dir_targets = [
+            os.path.join(self.root_dir, n) for n in nx.dfs_postorder_nodes(G)
+        ]
+        targets.extend(dir_targets)
+    elif pipeline or all_pipelines:
         if pipeline:
             stage = Stage.load(self, target)
             node = os.path.relpath(stage.path, self.root_dir)
@@ -77,7 +85,7 @@ def reproduce(
             stages = _reproduce(
                 self,
                 target,
-                recursive=recursive,
+                single_item=single_item,
                 force=force,
                 dry=dry,
                 interactive=interactive,
@@ -93,7 +101,7 @@ def reproduce(
 def _reproduce(
     self,
     target,
-    recursive=True,
+    single_item=False,
     force=False,
     dry=False,
     interactive=False,
@@ -109,7 +117,11 @@ def _reproduce(
     stages = nx.get_node_attributes(G, "stage")
     node = os.path.relpath(stage.path, self.root_dir)
 
-    if recursive:
+    if single_item:
+        ret = _reproduce_stage(
+            stages, node, force, dry, interactive, no_commit
+        )
+    else:
         ret = _reproduce_stages(
             G,
             stages,
@@ -121,10 +133,6 @@ def _reproduce(
             no_commit,
             downstream,
         )
-    else:
-        ret = _reproduce_stage(
-            stages, node, force, dry, interactive, no_commit
-        )
 
     return ret
 

dvc/repo/status.py

@@ -35,6 +35,32 @@ def _cloud_status(
     with_deps=False,
     all_tags=False,
 ):
+    """Returns a dictionary with the files that are new or deleted.
+
+    - new: Remote doesn't have the file
+    - deleted: File is no longer in the local cache
+
+    Example:
+            Given the following commands:
+
+            $ echo "foo" > foo
+            $ echo "bar" > bar
+            $ dvc add foo bar
+            $ dvc status -c
+
+            It will return something like:
+
+            { "foo": "new", "bar": "new" }
+
+            Now, after pushing and removing "bar" from the local cache:
+
+            $ dvc push
+            $ rm .dvc/cache/c1/57a79031e1c40f85931829bc5fc552
+
+            The result would be:
+
+            { "bar": "deleted" }
+    """
     import dvc.remote.base as cloud
 
     used = self.used_cache(

dvc/scm/git/__init__.py

@@ -260,15 +260,14 @@ class Git(Base):
 
     def _get_diff_trees(self, a_ref, b_ref):
         """Private method for getting the trees and commit hashes of 2 git
-        references.
-        Requires `gitdb` module (from gitpython package).
+        references. Requires `gitdb` module (from gitpython package).
 
         Args:
-            a_ref(str) - git reference
-            b_ref(str) - second git reference. If None, uses HEAD
+            a_ref (str): git reference
+            b_ref (str): second git reference. If None, uses HEAD
 
         Returns:
-            tuple - tuple with elements: (trees, commits)
+            tuple: tuple with elements: (trees, commits)
         """
         from gitdb.exc import BadObject, BadName
 
@@ -294,11 +293,12 @@ class Git(Base):
         Returns the dvc hash names of changed file/directory
 
         Args:
-            a_ref(str) - git reference
-            b_ref(str) - optional second git reference, default None
+            a_ref (str): git reference
+            b_ref (str): optional second git reference, default None
 
         Returns:
-            dict - dictionary with keys: (a_tree, b_tree, a_ref, b_ref, equal)
+            dict: dictionary with keys: (a_tree, b_tree, a_ref, b_ref,
+            equal)
         """
         diff_dct = {DIFF_EQUAL: False}
         trees, commits = self._get_diff_trees(a_ref, b_ref)

dvc/state.py

@@ -116,26 +116,6 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
     def __exit__(self, typ, value, tbck):
         self.dump()
 
-    def changed(self, path, md5):
-        """Check if file/directory has the expected md5.
-
-        Args:
-            path (str): path to the file/directory to check.
-            md5 (str): expected md5.
-
-        Returns:
-            bool: True if path has the expected md5, False otherwise.
-        """
-        actual = self.update(path)
-
-        msg = "File '{}', md5 '{}', actual '{}'"
-        logger.debug(msg.format(path, md5, actual))
-
-        if not md5 or not actual:
-            return True
-
-        return actual.split(".")[0] != md5.split(".")[0]
-
     def _execute(self, cmd):
         logger.debug(cmd)
         return self.cursor.execute(cmd)
@@ -289,8 +269,6 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
             )
         )
 
-        self._update_cache_directory_state()
-
         self.database.commit()
         self.cursor.close()
         self.database.close()
@@ -313,7 +291,7 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
         )
 
     def _update_state_for_path_changed(
-        self, path, actual_inode, actual_mtime, actual_size, checksum
+        self, actual_inode, actual_mtime, actual_size, checksum
     ):
         cmd = (
             "UPDATE {} SET "
@@ -333,7 +311,7 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
         )
 
     def _insert_new_state_record(
-        self, path, actual_inode, actual_mtime, actual_size, checksum
+        self, actual_inode, actual_mtime, actual_size, checksum
     ):
         assert checksum is not None
 
@@ -383,12 +361,12 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
         existing_record = self.get_state_record_for_inode(actual_inode)
         if not existing_record:
             self._insert_new_state_record(
-                path, actual_inode, actual_mtime, actual_size, checksum
+                actual_inode, actual_mtime, actual_size, checksum
             )
             return
 
         self._update_state_for_path_changed(
-            path, actual_inode, actual_mtime, actual_size, checksum
+            actual_inode, actual_mtime, actual_size, checksum
         )
 
     def get(self, path_info):
@@ -478,20 +456,3 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
         for relpath in unused:
             cmd = 'DELETE FROM {} WHERE path = "{}"'
             self._execute(cmd.format(self.LINK_STATE_TABLE, relpath))
-
-    def _update_cache_directory_state(self):
-        cache_path = self.repo.cache.local.cache_dir
-        mtime, size = get_mtime_and_size(cache_path)
-        inode = get_inode(cache_path)
-
-        cmd = (
-            "INSERT OR REPLACE INTO {}(inode, size, mtime, timestamp, md5) "
-            'VALUES ({}, "{}", "{}", "{}", "")'.format(
-                self.STATE_TABLE,
-                self._to_sqlite(inode),
-                size,
-                mtime,
-                current_timestamp(),
-            )
-        )
-        self._execute(cmd)

dvc/system.py

@@ -138,15 +138,16 @@ class System(object):
         import ctypes
         from ctypes import c_void_p, c_wchar_p, Structure, WinError, POINTER
         from ctypes.wintypes import DWORD, HANDLE, BOOL
+        from win32file import (
+            FILE_FLAG_BACKUP_SEMANTICS,
+            FILE_FLAG_OPEN_REPARSE_POINT,
+            FILE_SHARE_READ,
+            OPEN_EXISTING,
+        )
 
-        # NOTE: use this flag to open symlink itself and not the target
-        # See https://docs.microsoft.com/en-us/windows/desktop/api/
+        # NOTE: use FILE_FLAG_OPEN_REPARSE_POINT to open symlink itself and not
+        # the target See https://docs.microsoft.com/en-us/windows/desktop/api/
         # fileapi/nf-fileapi-createfilew#symbolic-link-behavior
-        FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
-
-        FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
-        FILE_SHARE_READ = 0x00000001
-        OPEN_EXISTING = 3
 
         class FILETIME(Structure):
             _fields_ = [("dwLowDateTime", DWORD), ("dwHighDateTime", DWORD)]
@@ -237,8 +238,7 @@ class System(object):
         from ctypes.wintypes import DWORD, HANDLE
 
         # https://docs.microsoft.com/en-us/windows/desktop/api/synchapi/nf-synchapi-waitforsingleobject
-        WAIT_OBJECT_0 = 0
-        WAIT_TIMEOUT = 0x00000102
+        from win32event import WAIT_OBJECT_0, WAIT_TIMEOUT
 
         func = ctypes.windll.kernel32.WaitForSingleObject
         func.argtypes = [HANDLE, DWORD]
@@ -272,7 +272,7 @@ class System(object):
 
         # https://docs.microsoft.com/en-us/windows/desktop/fileio/
         # file-attribute-constants
-        FILE_ATTRIBUTE_REPARSE_POINT = 0x400
+        from winnt import FILE_ATTRIBUTE_REPARSE_POINT
 
         if os.path.lexists(path):
             info = System.getdirinfo(path)

dvc/utils/__init__.py

@@ -113,7 +113,9 @@ def dict_md5(d, exclude=[]):
 
 def copyfile(src, dest, no_progress_bar=False, name=None):
     """Copy file with progress bar"""
+    from dvc.exceptions import DvcException
     from dvc.progress import progress
+    from dvc.system import System
 
     copied = 0
     name = name if name else os.path.basename(dest)
@@ -122,15 +124,18 @@ def copyfile(src, dest, no_progress_bar=False, name=None):
     if os.path.isdir(dest):
         dest = os.path.join(dest, os.path.basename(src))
 
-    with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
-        while True:
-            buf = fsrc.read(LOCAL_CHUNK_SIZE)
-            if not buf:
-                break
-            fdest.write(buf)
-            copied += len(buf)
-            if not no_progress_bar:
-                progress.update_target(name, copied, total)
+    try:
+        System.reflink(src, dest)
+    except DvcException:
+        with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
+            while True:
+                buf = fsrc.read(LOCAL_CHUNK_SIZE)
+                if not buf:
+                    break
+                fdest.write(buf)
+                copied += len(buf)
+                if not no_progress_bar:
+                    progress.update_target(name, copied, total)
 
     if not no_progress_bar:
         progress.finish_target(name)

dvc/version.py

@@ -7,7 +7,7 @@ import os
 import subprocess
 
 
-_BASE_VERSION = "0.40.2"
+_BASE_VERSION = "0.41.2"
 
 
 def _generate_version(base_version):

requirements.txt

@@ -1,33 +0,0 @@
-azure-storage-blob==1.3.0
-boto3==1.9.115
-ply>=3.9 # See https://github.com/pyinstaller/pyinstaller/issues/1945
-configparser>=3.5.0
-zc.lockfile>=1.2.1
-future>=0.16.0
-google-cloud-storage==1.13.0
-PyInstaller==3.4
-colorama>=0.3.9
-configobj>=5.0.6
-networkx>=2.1
-gitpython>=2.1.8
-setuptools>=34.0.0
-nanotime>=0.5.2
-pyasn1>=0.4.1
-schema>=0.6.7
-paramiko>=2.4.1
-jsonpath-ng>=1.4.3
-requests>=2.18.4
-wheel>=0.31.1
-futures>=3.2.0; python_version == "2.7"
-grandalf==0.6
-pydot>=1.2.4
-asciimatics>=1.10.0
-distro>=1.3.0
-appdirs>=1.4.3
-treelib>=1.5.5
-inflect>=2.1.0
-humanize>=0.5.1
-dulwich>=0.19.11
-ruamel.yaml>=0.15.91
-pathlib2==2.3.3; python_version == "2.7"
-oss2==2.6.1

scripts/build_posix.sh

@@ -78,7 +78,7 @@ install_dependencies()
 	gem install --no-document fpm
 
 	print_info "Installing requirements..."
-	pip install -r requirements.txt
+	pip install -e .[all]
 
 	print_info "Installing pyinstaller..."
 	pip install pyinstaller

scripts/build_windows.cmd

@@ -15,7 +15,7 @@ if %errorlevel% neq 0 (echo Error: pip not found && goto :error)
 if not exist "C:\Program Files (x86)\Inno Setup 5\ISCC.exe" (echo Error: Couldn't find Inno Setup compiler. Please go to jrsoftware.org/isinfo.php and install Inno Setup 5 && goto :error)
 
 echo ====== Installing requirements... ======
-call pip install -r requirements.txt || goto :error
+call pip install -e .[all] || goto :error
 call pip install pyinstaller || goto :error
 
 echo ====== Building dvc binary... ======

scripts/ci/before_deploy.sh

@@ -3,8 +3,38 @@
 set -x
 set -e
 
-if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]]; then
-	git config --local user.name 'Nightly Travis Builds'
-	git config --local user.email '<>'
-	git tag "$(date +'%Y%m%d%H%M%S')-$(git log --format=%h -1)"
+if [[ -z "$TRAVIS_TAG" && "$TRAVIS_EVENT_TYPE" != "cron" ]]; then
+    echo "Skipping building package."
+    exit 0
 fi
+
+./scripts/build_package.sh
+
+if [[ -n "$TRAVIS_TAG" ]]; then
+    # Test version
+    pip uninstall -y dvc
+    if [ -x "$(command -v dvc)" ]; then
+        echo "ERROR: dvc command already exists!!!"
+        exit 1
+    fi
+
+    pip install dist/dvc-*.whl
+    if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
+        echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'"
+        exit 1
+    fi
+    pip uninstall -y dvc
+fi
+
+PY_VER=$(python -c 'import sys; print(sys.version_info[0:2])')
+if [[ "$PY_VER" != '(3, 7)' ]]; then
+    echo "Skipping building binary package because of a wrong python version."
+    exit 0
+fi
+
+if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_OSX_IMAGE" != "xcode7.3" ]]; then
+    echo "Skipping building binary package because of a wrong xcode version."
+    exit 0
+fi
+
+./scripts/build_posix.sh

scripts/ci/before_install.sh

@@ -19,7 +19,7 @@ if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
     python --version
 fi
 
-pip install -r tests/requirements.txt
+pip install -e .[tests]
 
 # stop the build if there are any readme formatting errors
 python setup.py checkdocs

scripts/ci/install.sh

@@ -15,8 +15,8 @@ function retry {
 # networking issues. Thus, let's retry a few times to see if it will eventially
 # work or not.
 retry pip install --upgrade pip setuptools wheel
-retry pip install -r requirements.txt
-retry pip install -r tests/requirements.txt
+retry pip install -e .[all]
+retry pip install -e .[tests]
 
 git config --global user.email "[email protected]"
 git config --global user.name "DVC Tester"

scripts/ci/install_hadoop.sh

@@ -6,7 +6,7 @@ set -x
 set -e
 
 sudo apt-get update -y
-sudo apt-get install default-jdk
+sudo apt-get install openjdk-8-jdk
 java -version
 
 pushd /usr/local
@@ -23,7 +23,7 @@ echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >> env.sh
 echo "export YARN_HOME=/usr/local/hadoop" >> env.sh
 echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >> env.sh
 echo "export JAVA_HOME=/usr/" >> env.sh
-echo "export PATH=$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> env.sh
+echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> env.sh
 
 source env.sh
 

scripts/ci/remove_hadoop.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+stop-dfs.sh
+sudo rm -rf /usr/local/hadoop
+sudo rm -rf /usr/local/hadoop-2.6.5
+sudo rm -rf /usr/local/hadoop-2.6.5.tar.gz
+rm env.sh

scripts/ci/script.sh

@@ -11,39 +11,3 @@ if [[ "$TRAVIS_PULL_REQUEST" == "false" && \
 	pip install codecov
 	codecov
 fi
-
-if [[ -z "$TRAVIS_TAG" && "$TRAVIS_EVENT_TYPE" != "cron" ]]; then
-    echo "Skipping building package."
-    exit 0
-fi
-
-./scripts/build_package.sh
-
-if [[ -n "$TRAVIS_TAG" ]]; then
-    # Test version
-    pip uninstall -y dvc
-    if [ -x "$(command -v dvc)" ]; then
-        echo "ERROR: dvc command already exists!!!"
-        exit 1
-    fi
-
-    pip install dist/dvc-*.whl
-    if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
-        echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'"
-        exit 1
-    fi
-    pip uninstall -y dvc
-fi
-
-PY_VER=$(python -c 'import sys; print(sys.version_info[0:2])')
-if [[ "$PY_VER" != '(2, 7)' ]]; then
-    echo "Skipping building binary package because of a wrong python version."
-    exit 0
-fi
-
-if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_OSX_IMAGE" != "xcode7.3" ]]; then
-    echo "Skipping building binary package because of a wrong xcode version."
-    exit 0
-fi
-
-./scripts/build_posix.sh

scripts/completion/dvc.zsh

@@ -12,30 +12,33 @@
 
 _dvc_commands() {
   local _commands=(
-    "init:Initialize dvc over a directory (should already be a git dir)."
-    "destroy:Destroy dvc. Will remove all project's information, data files and cache."
-    "add:Add files/directories to dvc."
-    "import:Import files from URL."
+    "add:Take data files or directories under DVC control."
+    "cache:Manage cache settings."
     "checkout:Checkout data files from cache."
-    "run:Generate a stage file from a given command and execute the command."
-    "pull:Pull data files from the cloud."
-    "push:Push data files to the cloud."
-    "fetch:Fetch data files from the cloud."
-    "status:Show the project status."
-    "repro:Reproduce DVC file. Default file name - 'Dvcfile'."
-    "remove:Remove outputs of DVC file."
-    "move:Move output of DVC file."
-    "unprotect:Unprotect data file/directory."
-    "gc:Collect garbage."
-    "config:Get or set config options."
-    "remote:Manage set of tracked repositories."
-    "metrics:Get metrics from all branches."
-    "install:Install dvc hooks into the repository."
-    "root:Relative path to project's directory."
+    "commit:Save changed data to cache and update DVC files."
+    "config:Get or set config settings."
+    "destroy:Remove DVC files, local DVC config and data cache."
+    "diff:Show a diff of a DVC controlled data file or a directory."
+    "fetch:Fetch data files from a DVC remote storage."
+    "gc:Collect unused data from DVC cache or a remote storage."
+    "import:Download or copy files from URL and take under DVC control."
+    "init:Initialize DVC in the current directory."
+    "install:Install DVC git hooks into the repository."
     "lock:Lock DVC file."
+    "metrics:Commands to add, manage, collect and display metrics."
+    "move:Rename or move a DVC controlled data file or a directory."
+    "pipeline:Manage pipelines."
+    "pull:Pull data files from a DVC remote storage."
+    "push:Push data files to a DVC remote storage."
+    "remote:Manage remote storage configuration."
+    "remove:Remove outputs of DVC file."
+    "repro:Check for changes and reproduce DVC file and dependencies."
+    "root:Relative path to project's directory."
+    "run:Generate a stage file from a command and execute the command."
+    "status:Show changed stages, compare local cache and a remote storage."
     "unlock:Unlock DVC file."
-    "pipeline:Manage pipeline."
-    "commit:Record changes to the repository."
+    "unprotect:Unprotect data file/directory."
+    "version:Show DVC version and system/environment informaion."
   )
 
   _describe 'dvc commands' _commands
@@ -52,15 +55,6 @@ _dvc_options=(
   "(-)"{-v,--version}"[Show program's version]"
 )
 
-_dvc_init=(
-  "--no-scm[Initiate dvc in directory that is not tracket by any scm tool]"
-  {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
-)
-
-_dvc_destroy=(
-  {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
-)
-
 _dvc_add=(
   {-R,--recursive}"[Recursively add each file under the directory.]"
   {-f,--file}"[Specify name of the stage file.]:File:_files"
@@ -68,10 +62,8 @@ _dvc_add=(
   "1:File:_files"
 )
 
-_dvc_import=(
-  "--resume[Resume previously started download.]"
-  "1:URL:"
-  "2:Output:"
+_dvc_cache=(
+  "1:Sub command:(dir)"
 )
 
 _dvc_checkout=(
@@ -80,24 +72,77 @@ _dvc_checkout=(
   "1:Stages:_files -g '(*.dvc|Dvcfile)'"
 )
 
-_dvc_unprotect=(
-  "*:Data files:_files"
+_dvc_commit=(
+  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
+  {-d,--with-deps}"[Commit all dependencies of the specified target.]"
+  {-f,--force}"[Commit even if checksums for dependencies/outputs changed.]"
+  {-R,--recursive}"[Commit cache for subdirectories of the specified directory.]"
 )
 
-_dvc_run=(
-  "--no-exec[Only create stage file without actually running it.]"
-  "--overwrite-dvcfile[Overwrite existing dvc file without asking for confirmation.]"
-  "--ignore-build-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
-  "--no-commit[Don't put files/directories into cache.]"
-  {-f,--file}"[Specify name of the stage file.]:File:_files"
-  {-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/"
-  {-w,--wdir}"[Directory to run your command and place state file in.]:WDIR:_files -/"
-  "*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files"
-  "*"{-o,--outs}"[Declare output data file or data directory.]:Output data:_files"
-  "*"{-O,--outs-no-cache}"[Declare output regular file or directory.]:Output regular:_files"
-  "*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache)]:Metrics (no cache):_files"
-  "*"{-m,--metrics}"[Declare output metric file or directory]:Metrics:_files"
-  {-y,--yes}"[Automatic 'yes' answer to all prompts.]"
+_dvc_config=(
+  {-u,--unset}"[Unset option.]"
+  "--local[Unset option.]"
+  "--system[Use system config.]"
+  "--global[Use global config.]"
+)
+
+_dvc_destroy=(
+  {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
+)
+
+_dvc_diff=(
+  {-t,--target}"[Source path to a data file or directory.]:Target file(s):"
+)
+
+_dvc_fetch=(
+  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+  {-R,--recursive}"[Fetch cache for subdirectories of the specified directory.]"
+)
+
+_dvc_gc=(
+  {-a,--all-branches}"[Collect garbage for all branches]"
+  {-T,--all-tags}"[Collect garbage for all tags]"
+  {-c,--cloud}"[Collect garbage in remote repository]"
+  {-r,--remote}"[Remote repository to collect garbage in]:Remote repository:"
+  {-f,--force}"[Force garbage collection - automatically agree to all prompts.]"
+
+  {-p,--projects}"[Keep data files required by these projects in addition to the current one. Useful if you share a single cache across repos.]:Repos:_files"
+)
+
+_dvc_import=(
+  "--resume[Resume previously started download.]"
+  "1:URL:"
+  "2:Output:"
+)
+
+_dvc_init=(
+  "--no-scm[Initiate dvc in directory that is not tracket by any scm tool]"
+  {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
+)
+
+_dvc_install=()
+
+_dvc_lock=(
+  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_metrics=(
+  "1:Sub command:(show add modify remove)"
+)
+
+_dvc_move=(
+  "1:Source:_files"
+  "2:Destination:"
+)
+
+_dvc_pipeline=(
+  "1:Sub command:(show,list)"
 )
 
 _dvc_pull=(
@@ -123,27 +168,16 @@ _dvc_push=(
   {-R,--recursive}"[Push cache for subdirectories of the specified directory.]"
 )
 
-_dvc_fetch=(
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-  "--show-checksums[Show checksums instead of file names]"
-  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
-  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
-  {-a,--all-branches}"[Fetch cache for all branches]"
-  {-T,--all-tags}"[Fetch cache for all tags]"
-  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
-  {-R,--recursive}"[Fetch cache for subdirectories of the specified directory.]"
+_dvc_remote=(
+  "1:Sub command:(add default remove modify list)"
 )
 
-_dvc_status=(
+_dvc_remove=(
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-  "--show-checksums[Show checksums instead of file names]"
-  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
-  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
-  {-a,--all-branches}"[Fetch cache for all branches]"
-  {-T,--all-tags}"[Fetch cache for all tags]"
-  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
-  {-c,--cloud}"[Show status of a local cache compared to a remote repository]"
-  {-q,--quiet}"[Suppresses all output. Exit with 0 if pipeline is up]"
+  "--dry[Only print the commands that would be executed without actually executing]"
+  {-o,--outs}"[Only remove DVC file outputs.(default)]"
+  {-p,--purge}"[Remove DVC file and all its outputs]"
+  {-f,--force}"[Force purge.]"
 )
 
 _dvc_repro=(
@@ -160,69 +194,42 @@ _dvc_repro=(
   {-d,--downstream}"[Reproduce the pipeline starting from the specified stage.]"
 )
 
-_dvc_remove=(
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-  "--dry[Only print the commands that would be executed without actually executing]"
-  {-o,--outs}"[Only remove DVC file outputs.(default)]"
-  {-p,--purge}"[Remove DVC file and all its outputs]"
-  {-f,--force}"[Force purge.]"
-)
-
-_dvc_move=(
-  "1:Source:_files"
-  "2:Destination:"
-)
-
-_dvc_gc=(
-  {-a,--all-branches}"[Collect garbage for all branches]"
-  {-T,--all-tags}"[Collect garbage for all tags]"
-  {-c,--cloud}"[Collect garbage in remote repository]"
-  {-r,--remote}"[Remote repository to collect garbage in]:Remote repository:"
-  {-f,--force}"[Force garbage collection - automatically agree to all prompts.]"
-
-  {-p,--projects}"[Keep data files required by these projects in addition to the current one. Useful if you share a single cache across repos.]:Repos:_files"
-)
-
-_dvc_config=(
-  {-u,--unset}"[Unset option.]"
-  "--local[Unset option.]"
-  "--system[Use system config.]"
-  "--global[Use global config.]"
-)
-
-_dvc_remote=(
-  "1:Sub command:(add default remove modify list)"
-)
-
-_dvc_cache=(
-  "1:Sub command:(dir)"
-)
-
-_dvc_metrics=(
-  "1:Sub command:(show add modify remove)"
-)
-
-_dvc_install=()
-
 _dvc_root=()
 
-_dvc_lock=(
+_dvc_run=(
+  "--no-exec[Only create stage file without actually running it.]"
+  "--overwrite-dvcfile[Overwrite existing dvc file without asking for confirmation.]"
+  "--ignore-build-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
+  "--no-commit[Don't put files/directories into cache.]"
+  {-f,--file}"[Specify name of the stage file.]:File:_files"
+  {-c,--cwd}"[Deprecated, use -w and -f instead.]:CWD:_files -/"
+  {-w,--wdir}"[Directory to run your command and place state file in.]:WDIR:_files -/"
+  "*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files"
+  "*"{-o,--outs}"[Declare output data file or data directory.]:Output data:_files"
+  "*"{-O,--outs-no-cache}"[Declare output regular file or directory.]:Output regular:_files"
+  "*"{-M,--metrics-no-cache}"[Declare output metric file or directory (do not put into DVC cache)]:Metrics (no cache):_files"
+  "*"{-m,--metrics}"[Declare output metric file or directory]:Metrics:_files"
+  {-y,--yes}"[Automatic 'yes' answer to all prompts.]"
+)
+
+_dvc_status=(
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+  {-c,--cloud}"[Show status of a local cache compared to a remote repository]"
+  {-q,--quiet}"[Suppresses all output. Exit with 0 if pipeline is up]"
 )
 
 _dvc_unlock=(
   "*:Stages:_files -g '(*.dvc|Dvcfile)'"
 )
 
-_dvc_pipeline=(
-  "1:Sub command:(show,list)"
-)
-
-_dvc_commit=(
-  "*:Stages:_files -g '(*.dvc|Dvcfile)'"
-  {-d,--with-deps}"[Commit all dependencies of the specified target.]"
-  {-f,--force}"[Commit even if checksums for dependencies/outputs changed.]"
-  {-R,--recursive}"[Commit cache for subdirectories of the specified directory.]"
+_dvc_unprotect=(
+  "*:Data files:_files"
 )
 
 typeset -A opt_args
@@ -234,28 +241,30 @@ _arguments \
   '*::args:->args'
 
 case $words[1] in
-  init)      _arguments $_dvc_global_options   $_dvc_init      ;;
-  destroy)   _arguments $_dvc_global_options   $_dvc_destroy   ;;
   add)       _arguments $_dvc_global_options   $_dvc_add       ;;
-  import)    _arguments $_dvc_global_options   $_dvc_import    ;;
+  cache)     _arguments $_dvc_global_options   $_dvc_cache     ;;
   checkout)  _arguments $_dvc_global_options   $_dvc_checkout  ;;
-  run)       _arguments $_dvc_global_options   $_dvc_run       ;;
+  commit)    _arguments $_dvc_global_options   $_dvc_commit    ;;
+  config)    _arguments $_dvc_global_options   $_dvc_config    ;;
+  destroy)   _arguments $_dvc_global_options   $_dvc_destroy   ;;
+  diff)      _arguments $_dvc_global_options   $_dvc_diff      ;;
+  fetch)     _arguments $_dvc_global_options   $_dvc_fetch     ;;
+  gc)        _arguments $_dvc_global_options   $_dvc_gc        ;;
+  import)    _arguments $_dvc_global_options   $_dvc_import    ;;
+  init)      _arguments $_dvc_global_options   $_dvc_init      ;;
+  install)   _arguments $_dvc_global_options   $_dvc_install   ;;
+  lock)      _arguments $_dvc_global_options   $_dvc_lock      ;;
+  metrics)   _arguments $_dvc_global_options   $_dvc_metrics   ;;
+  move)      _arguments $_dvc_global_options   $_dvc_move      ;;
+  pipeline)  _arguments $_dvc_global_options   $_dvc_pipeline  ;;
   pull)      _arguments $_dvc_global_options   $_dvc_pull      ;;
   push)      _arguments $_dvc_global_options   $_dvc_push      ;;
-  fetch)     _arguments $_dvc_global_options   $_dvc_fetch     ;;
-  status)    _arguments $_dvc_global_options   $_dvc_status    ;;
-  repro)     _arguments $_dvc_global_options   $_dvc_repro     ;;
-  remove)    _arguments $_dvc_global_options   $_dvc_remove    ;;
-  move)      _arguments $_dvc_global_options   $_dvc_move      ;;
-  gc)        _arguments $_dvc_global_options   $_dvc_gc        ;;
-  config)    _arguments $_dvc_global_options   $_dvc_config    ;;
   remote)    _arguments $_dvc_global_options   $_dvc_remote    ;;
-  metrics)   _arguments $_dvc_global_options   $_dvc_metrics   ;;
-  install)   _arguments $_dvc_global_options   $_dvc_install   ;;
+  remove)    _arguments $_dvc_global_options   $_dvc_remove    ;;
+  repro)     _arguments $_dvc_global_options   $_dvc_repro     ;;
   root)      _arguments $_dvc_global_options   $_dvc_root      ;;
-  lock)      _arguments $_dvc_global_options   $_dvc_lock      ;;
+  run)       _arguments $_dvc_global_options   $_dvc_run       ;;
+  status)    _arguments $_dvc_global_options   $_dvc_status    ;;
   unlock)    _arguments $_dvc_global_options   $_dvc_unlock    ;;
-  pipeline)  _arguments $_dvc_global_options   $_dvc_pipeline  ;;
-  commit)    _arguments $_dvc_global_options   $_dvc_commit    ;;
   unprotect) _arguments $_dvc_global_options   $_dvc_unprotect ;;
 esac

setup.py

@@ -1,6 +1,7 @@
 from setuptools import setup, find_packages
 from setuptools.command.build_py import build_py as _build_py
 import os
+import sys
 
 
 # https://packaging.python.org/guides/single-sourcing-package-version/
@@ -49,7 +50,7 @@ install_requires = [
     "pyasn1>=0.4.1",
     "schema>=0.6.7",
     "jsonpath-ng>=1.4.3",
-    "requests>=2.18.4",
+    "requests>=2.22.0",
     "grandalf==0.6",
     "asciimatics>=1.10.0",
     "distro>=1.3.0",
@@ -69,6 +70,34 @@ oss = ["oss2==2.6.1"]
 ssh = ["paramiko>=2.4.1"]
 all_remotes = gs + s3 + azure + ssh + oss
 
+# Extra dependecies to run tests
+tests_requirements = [
+    "PyInstaller==3.4",
+    "wheel>=0.31.1",
+    "pydot>=1.2.4",
+    # Test requirements:
+    "pytest>=4.4.0",
+    "pytest-timeout>=1.3.3",
+    "pytest-cov>=2.6.1",
+    "pytest-xdist>=1.26.1",
+    "pytest-mock>=1.10.4",
+    "flaky>=3.5.3",
+    "mock>=3.0.0",
+    "xmltodict>=0.11.0",
+    "awscli>=1.16.125",
+    "google-compute-engine",
+    "pywin32; sys_platform == 'win32'",
+    "Pygments",  # required by collective.checkdocs,
+    "collective.checkdocs",
+    "flake8",
+    "flake8-docstrings",
+    "jaraco.windows==3.9.2",
+    "mock-ssh-server>=0.5.0",
+]
+
+if (sys.version_info) >= (3, 6):
+    tests_requirements.append("black==19.3b0")
+
 setup(
     name="dvc",
     version=version,
@@ -87,7 +116,8 @@ setup(
         "oss": oss,
         "ssh": ssh,
         # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1
-        ':python_version=="2.7"': ["futures", "pathlib2"],
+        ":python_version=='2.7'": ["futures", "pathlib2"],
+        "tests": tests_requirements,
     },
     keywords="data science, data version control, machine learning",
     python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",

tests/basic_env.py

@@ -23,9 +23,6 @@ logger = logging.getLogger("dvc")
 
 
 class TestDirFixture(object):
-    GCP_CREDS_FILE = os.path.abspath(
-        os.path.join("scripts", "ci", "gcp-creds.json")
-    )
     DATA_DIR = "data_dir"
     DATA_SUB_DIR = os.path.join(DATA_DIR, "data_sub_dir")
     DATA = os.path.join(DATA_DIR, "data")

tests/conftest.py

@@ -67,7 +67,7 @@ def git(repo_dir):
 
 
 @pytest.fixture
-def dvc(repo_dir, git):
+def dvc_repo(repo_dir, git):
     try:
         dvc = DvcRepo.init(repo_dir._root_dir)
         dvc.scm.commit("init dvc")
@@ -97,3 +97,36 @@ def ssh(ssh_server):
         port=ssh_server.port,
         key_filename=key_path,
     )
+
+
+@pytest.fixture
+def temporary_windows_drive(repo_dir):
+    import string
+    import win32api
+    from ctypes import windll
+    from win32con import DDD_REMOVE_DEFINITION
+
+    drives = [
+        s[0].upper()
+        for s in win32api.GetLogicalDriveStrings().split("\000")
+        if len(s) > 0
+    ]
+
+    new_drive_name = [
+        letter for letter in string.ascii_uppercase if letter not in drives
+    ][0]
+    new_drive = "{}:".format(new_drive_name)
+
+    target_path = repo_dir.mkdtemp()
+
+    set_up_result = windll.kernel32.DefineDosDeviceW(0, new_drive, target_path)
+    if set_up_result == 0:
+        raise RuntimeError("Failed to mount windows drive!")
+
+    yield new_drive
+
+    tear_down_result = windll.kernel32.DefineDosDeviceW(
+        DDD_REMOVE_DEFINITION, new_drive, target_path
+    )
+    if tear_down_result == 0:
+        raise RuntimeError("Could not unmount windows drive!")

tests/func/test_add.py

@@ -2,6 +2,8 @@ from __future__ import unicode_literals
 
 import os
 
+import pytest
+
 import dvc
 import time
 import shutil
@@ -323,29 +325,6 @@ class TestAddCommit(TestDvc):
         self.assertEqual(len(os.listdir(self.dvc.cache.local.cache_dir)), 1)
 
 
-class TestShouldNotCheckCacheForDirIfCacheMetadataDidNotChange(TestDvc):
-    def test(self):
-        remote_local_loader_spy = spy(
-            dvc.remote.local.RemoteLOCAL.load_dir_cache
-        )
-        with patch.object(
-            dvc.remote.local.RemoteLOCAL,
-            "load_dir_cache",
-            remote_local_loader_spy,
-        ):
-
-            ret = main(["config", "cache.type", "copy"])
-            self.assertEqual(ret, 0)
-
-            ret = main(["add", self.DATA_DIR])
-            self.assertEqual(ret, 0)
-            self.assertEqual(1, remote_local_loader_spy.mock.call_count)
-
-            ret = main(["status", "{}.dvc".format(self.DATA_DIR)])
-            self.assertEqual(ret, 0)
-            self.assertEqual(1, remote_local_loader_spy.mock.call_count)
-
-
 class TestShouldCollectDirCacheOnlyOnce(TestDvc):
     def test(self):
         from dvc.remote.local import RemoteLOCAL
@@ -541,3 +520,14 @@ class TestAddUnprotected(TestDvc):
 
         self.assertFalse(os.access(self.FOO, os.W_OK))
         self.assertTrue(System.is_hardlink(self.FOO))
+
+
+@pytest.mark.skipif(os.name != "nt", reason="Windows specific")
+def test_windows_should_add_when_cache_on_different_drive(
+    dvc_repo, repo_dir, temporary_windows_drive
+):
+    ret = main(["config", "cache.dir", temporary_windows_drive])
+    assert ret == 0
+
+    ret = main(["add", repo_dir.DATA])
+    assert ret == 0

tests/func/test_api.py

@@ -0,0 +1,93 @@
+import pytest
+import shutil
+
+from dvc import api
+from dvc.main import main
+from .test_data_cloud import (
+    _should_test_aws,
+    get_aws_url,
+    _should_test_gcp,
+    get_gcp_url,
+    _should_test_azure,
+    get_azure_url,
+    _should_test_oss,
+    get_oss_url,
+    _should_test_ssh,
+    get_ssh_url,
+    _should_test_hdfs,
+    get_hdfs_url,
+)
+
+
+# NOTE: staticmethod is only needed in Python 2
+class S3:
+    should_test = staticmethod(_should_test_aws)
+    get_url = staticmethod(get_aws_url)
+
+
+class GCP:
+    should_test = staticmethod(_should_test_gcp)
+    get_url = staticmethod(get_gcp_url)
+
+
+class Azure:
+    should_test = staticmethod(_should_test_azure)
+    get_url = staticmethod(get_azure_url)
+
+
+class OSS:
+    should_test = staticmethod(_should_test_oss)
+    get_url = staticmethod(get_oss_url)
+
+
+class SSH:
+    should_test = staticmethod(_should_test_ssh)
+    get_url = staticmethod(get_ssh_url)
+
+
+class HDFS:
+    should_test = staticmethod(_should_test_hdfs)
+    get_url = staticmethod(get_hdfs_url)
+
+
+remote_params = [S3, GCP, Azure, OSS, SSH, HDFS]
+
+
+@pytest.fixture
+def remote(request):
+    if not request.param.should_test():
+        raise pytest.skip()
+    return request.param
+
+
+def pytest_generate_tests(metafunc):
+    if "remote" in metafunc.fixturenames:
+        metafunc.parametrize("remote", remote_params, indirect=True)
+
+
+def run_dvc(*argv):
+    assert main(argv) == 0
+
+
+def test_get_url(repo_dir, dvc_repo, remote):
+    remote_url = remote.get_url()
+
+    run_dvc("remote", "add", "-d", "upstream", remote_url)
+    dvc_repo.add(repo_dir.FOO)
+
+    assert api.get_url(repo_dir.FOO) == "%s/%s" % (
+        remote_url,
+        "ac/bd18db4cc2f85cedef654fccc4a4d8",
+    )
+
+
+def test_open(repo_dir, dvc_repo, remote):
+    run_dvc("remote", "add", "-d", "upstream", remote.get_url())
+    dvc_repo.add(repo_dir.FOO)
+    run_dvc("push")
+
+    # Remove cache to force download
+    shutil.rmtree(dvc_repo.cache.local.cache_dir)
+
+    with api.open(repo_dir.FOO) as fd:
+        assert fd.read() == repo_dir.FOO_CONTENTS

tests/func/test_checkout.py

@@ -428,10 +428,10 @@ class TestCheckoutShouldHaveSelfClearingProgressBar(TestDvc):
         finish_bar = progress_bars[-1]
 
         self.assertEqual(4, len(update_bars))
-        self.assertRegexpMatches(progress_bars[0], ".*\\[#{7} {23}\\] 25%.*")
-        self.assertRegexpMatches(progress_bars[1], ".*\\[#{15} {15}\\] 50%.*")
-        self.assertRegexpMatches(progress_bars[2], ".*\\[#{22} {8}\\] 75%.*")
-        self.assertRegexpMatches(progress_bars[3], ".*\\[#{30}\\] 100%.*")
+        assert re.search(".*\\[#{7} {23}\\] 25%.*", progress_bars[0])
+        assert re.search(".*\\[#{15} {15}\\] 50%.*", progress_bars[1])
+        assert re.search(".*\\[#{22} {8}\\] 75%.*", progress_bars[2])
+        assert re.search(".*\\[#{30}\\] 100%.*", progress_bars[3])
 
         self.assertCaretReturnFollowsEach(update_bars)
         self.assertNewLineFollows(finish_bar)
@@ -517,8 +517,10 @@ class TestCheckoutRecursiveNotDirectory(TestDvc):
         ret = main(["add", self.FOO])
         self.assertEqual(0, ret)
 
-        with self.assertRaises(TargetNotDirectoryError):
-            self.dvc.checkout(target=self.FOO, recursive=True)
+        try:
+            self.dvc.checkout(target=self.FOO + ".dvc", recursive=True)
+        except TargetNotDirectoryError:
+            self.fail("should not raise TargetNotDirectoryError")
 
 
 class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):

tests/func/test_commit.py

@@ -4,51 +4,51 @@ from dvc.utils.stage import load_stage_file, dump_stage_file
 from dvc.stage import StageCommitError
 
 
-def test_commit_recursive(dvc, repo_dir):
-    stages = dvc.add(repo_dir.DATA_DIR, recursive=True, no_commit=True)
+def test_commit_recursive(dvc_repo, repo_dir):
+    stages = dvc_repo.add(repo_dir.DATA_DIR, recursive=True, no_commit=True)
     assert len(stages) == 2
 
-    assert dvc.status() != {}
+    assert dvc_repo.status() != {}
 
-    dvc.commit(repo_dir.DATA_DIR, recursive=True)
+    dvc_repo.commit(repo_dir.DATA_DIR, recursive=True)
 
-    assert dvc.status() == {}
+    assert dvc_repo.status() == {}
 
 
-def test_commit_force(dvc, repo_dir):
-    stages = dvc.add(repo_dir.FOO, no_commit=True)
+def test_commit_force(dvc_repo, repo_dir):
+    stages = dvc_repo.add(repo_dir.FOO, no_commit=True)
     assert len(stages) == 1
     stage = stages[0]
 
-    with dvc.state:
+    with dvc_repo.state:
         assert stage.outs[0].changed_cache()
 
     with open(repo_dir.FOO, "a") as fobj:
         fobj.write(repo_dir.FOO_CONTENTS)
 
-    with dvc.state:
+    with dvc_repo.state:
         assert stage.outs[0].changed_cache()
 
     with pytest.raises(StageCommitError):
-        dvc.commit(stage.path)
+        dvc_repo.commit(stage.path)
 
-    with dvc.state:
+    with dvc_repo.state:
         assert stage.outs[0].changed_cache()
 
-    dvc.commit(stage.path, force=True)
+    dvc_repo.commit(stage.path, force=True)
 
-    assert dvc.status(stage.path) == {}
+    assert dvc_repo.status(stage.path) == {}
 
 
-def test_commit_with_deps(dvc, repo_dir):
-    stages = dvc.add(repo_dir.FOO, no_commit=True)
+def test_commit_with_deps(dvc_repo, repo_dir):
+    stages = dvc_repo.add(repo_dir.FOO, no_commit=True)
     assert len(stages) == 1
     foo_stage = stages[0]
     assert foo_stage is not None
     assert len(foo_stage.outs) == 1
 
     fname = "file"
-    stage = dvc.run(
+    stage = dvc_repo.run(
         cmd="python {} {} {}".format(repo_dir.CODE, repo_dir.FOO, fname),
         outs=[fname],
         deps=[repo_dir.FOO, repo_dir.CODE],
@@ -57,18 +57,18 @@ def test_commit_with_deps(dvc, repo_dir):
     assert stage is not None
     assert len(stage.outs) == 1
 
-    with dvc.state:
+    with dvc_repo.state:
         assert foo_stage.outs[0].changed_cache()
         assert stage.outs[0].changed_cache()
 
-    dvc.commit(stage.path, with_deps=True)
-    with dvc.state:
+    dvc_repo.commit(stage.path, with_deps=True)
+    with dvc_repo.state:
         assert not foo_stage.outs[0].changed_cache()
         assert not stage.outs[0].changed_cache()
 
 
-def test_commit_changed_md5(dvc, repo_dir):
-    stages = dvc.add(repo_dir.FOO, no_commit=True)
+def test_commit_changed_md5(dvc_repo, repo_dir):
+    stages = dvc_repo.add(repo_dir.FOO, no_commit=True)
     assert len(stages) == 1
     stage = stages[0]
 
@@ -77,6 +77,6 @@ def test_commit_changed_md5(dvc, repo_dir):
     dump_stage_file(stage.path, stage_file_content)
 
     with pytest.raises(StageCommitError):
-        dvc.commit(stage.path)
+        dvc_repo.commit(stage.path)
 
-    dvc.commit(stage.path, force=True)
+    dvc_repo.commit(stage.path, force=True)

tests/func/test_data_cloud.py

@@ -43,10 +43,19 @@ TEST_CONFIG = {
     TEST_SECTION: {Config.SECTION_REMOTE_URL: ""},
 }
 
-TEST_AWS_REPO_BUCKET = "dvc-test"
-TEST_GCP_REPO_BUCKET = "dvc-test"
+TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-test")
+TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test")
 TEST_OSS_REPO_BUCKET = "dvc-test"
 
+TEST_GCP_CREDS_FILE = os.path.abspath(
+    os.environ.get(
+        "GOOGLE_APPLICATION_CREDENTIALS",
+        os.path.join("scripts", "ci", "gcp-creds.json"),
+    )
+)
+# Ensure that absolute path is used
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE
+
 
 def _should_test_aws():
     dvc_test_aws = os.getenv("DVC_TEST_AWS")
@@ -65,29 +74,22 @@ def _should_test_gcp():
     if os.getenv("DVC_TEST_GCP") == "true":
         return True
 
-    if not os.path.exists(TestDvc.GCP_CREDS_FILE):
+    if not os.path.exists(TEST_GCP_CREDS_FILE):
         return False
 
-    creds = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
-    if creds and os.getenv("GCP_CREDS"):
-        if os.path.exists(creds):
-            os.unlink(creds)
-        shutil.copyfile(TestDvc.GCP_CREDS_FILE, creds)
-        try:
-            check_output(
-                [
-                    "gcloud",
-                    "auth",
-                    "activate-service-account",
-                    "--key-file",
-                    creds,
-                ]
-            )
-        except (CalledProcessError, OSError):
-            return False
-        return True
-
-    return False
+    try:
+        check_output(
+            [
+                "gcloud",
+                "auth",
+                "activate-service-account",
+                "--key-file",
+                TEST_GCP_CREDS_FILE,
+            ]
+        )
+    except (CalledProcessError, OSError):
+        return False
+    return True
 
 
 def _should_test_azure():
@@ -162,7 +164,7 @@ def get_local_url():
 
 
 def get_ssh_url():
-    return "ssh://{}@127.0.0.1:{}".format(
+    return "ssh://{}@127.0.0.1:22{}".format(
         getpass.getuser(), get_local_storagepath()
     )
 
@@ -233,8 +235,9 @@ def get_oss_url():
 
 class TestDataCloud(TestDvc):
     def _test_cloud(self, config, cl):
-        cloud = DataCloud(self.dvc, config=config)
-        self.assertIsInstance(cloud._cloud, cl)
+        self.dvc.config.config = config
+        cloud = DataCloud(self.dvc)
+        self.assertIsInstance(cloud.get_remote(), cl)
 
     def test(self):
         config = copy.deepcopy(TEST_CONFIG)
@@ -283,9 +286,10 @@ class TestDataCloudBase(TestDvc):
         config = copy.deepcopy(TEST_CONFIG)
         config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
         config[TEST_SECTION][Config.SECTION_REMOTE_KEY_FILE] = keyfile
-        self.cloud = DataCloud(self.dvc, config)
+        self.dvc.config.config = config
+        self.cloud = DataCloud(self.dvc)
 
-        self.assertIsInstance(self.cloud._cloud, self._get_cloud_class())
+        self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
 
     def _test_cloud(self):
         self._setup_cloud()
@@ -391,10 +395,11 @@ class TestRemoteGS(TestDataCloudBase):
         config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
         config[TEST_SECTION][
             Config.SECTION_GCP_CREDENTIALPATH
-        ] = TestDvc.GCP_CREDS_FILE
-        self.cloud = DataCloud(self.dvc, config)
+        ] = TEST_GCP_CREDS_FILE
+        self.dvc.config.config = config
+        self.cloud = DataCloud(self.dvc)
 
-        self.assertIsInstance(self.cloud._cloud, self._get_cloud_class())
+        self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
 
     def _get_url(self):
         return get_gcp_url()
@@ -666,7 +671,7 @@ class TestRemoteGSCLI(TestDataCloudCLIBase):
                 "modify",
                 TEST_REMOTE,
                 "credentialpath",
-                TestDvc.GCP_CREDS_FILE,
+                TEST_GCP_CREDS_FILE,
             ]
         )
 
@@ -758,9 +763,8 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
         return RemoteLOCAL
 
     def _prepare_repo(self):
-        self.main(
-            ["remote", "add", "-d", TEST_REMOTE, self.cloud._cloud.cache_dir]
-        )
+        remote = self.cloud.get_remote()
+        self.main(["remote", "add", "-d", TEST_REMOTE, remote.cache_dir])
 
         self.dvc.add(self.DATA)
         self.dvc.add(self.DATA_SUB)
@@ -799,8 +803,9 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
         self.assertTrue(os.path.exists(local_cache_data_sub_path))
 
     def _test_recursive_push(self, data_md5, data_sub_md5):
-        cloud_data_path = self.cloud._cloud.get(data_md5)
-        cloud_data_sub_path = self.cloud._cloud.get(data_sub_md5)
+        remote = self.cloud.get_remote()
+        cloud_data_path = remote.get(data_md5)
+        cloud_data_sub_path = remote.get(data_sub_md5)
 
         self.assertFalse(os.path.exists(cloud_data_path))
         self.assertFalse(os.path.exists(cloud_data_sub_path))

tests/func/test_dependency.py

@@ -17,5 +17,5 @@ def _get_dep(dvc, path):
 
 
 @pytest.mark.parametrize("url,scheme", TESTS)
-def test_scheme(dvc, url, scheme):
-    assert type(_get_dep(dvc, url)) == DEP_MAP[scheme]
+def test_scheme(dvc_repo, url, scheme):
+    assert type(_get_dep(dvc_repo, url)) == DEP_MAP[scheme]

tests/func/test_install.py

@@ -15,7 +15,7 @@ class TestInstall(object):
     def _hook(self, name):
         return os.path.join(".git", "hooks", name)
 
-    def test_should_create_hooks(self, dvc):
+    def test_should_create_hooks(self, dvc_repo):
         assert main(["install"]) == 0
 
         hooks_with_commands = [
@@ -30,7 +30,7 @@ class TestInstall(object):
             with open(self._hook(fname), "r") as fobj:
                 assert command in fobj.read()
 
-    def test_should_append_hooks_if_file_already_exists(self, dvc):
+    def test_should_append_hooks_if_file_already_exists(self, dvc_repo):
         with open(self._hook("post-checkout"), "w") as fobj:
             fobj.write("#!/bin/sh\n" "echo hello\n")
 
@@ -41,7 +41,7 @@ class TestInstall(object):
         with open(self._hook("post-checkout"), "r") as fobj:
             assert fobj.read() == expected_script
 
-    def test_should_be_idempotent(self, dvc):
+    def test_should_be_idempotent(self, dvc_repo):
         assert main(["install"]) == 0
         assert main(["install"]) == 0
 
@@ -50,21 +50,21 @@ class TestInstall(object):
         with open(self._hook("post-checkout"), "r") as fobj:
             assert fobj.read() == expected_script
 
-    def test_should_post_checkout_hook_checkout(self, repo_dir, dvc):
+    def test_should_post_checkout_hook_checkout(self, repo_dir, dvc_repo):
         assert main(["install"]) == 0
 
         stage_file = repo_dir.FOO + Stage.STAGE_FILE_SUFFIX
 
-        dvc.add(repo_dir.FOO)
-        dvc.scm.add([".gitignore", stage_file])
-        dvc.scm.commit("add")
+        dvc_repo.add(repo_dir.FOO)
+        dvc_repo.scm.add([".gitignore", stage_file])
+        dvc_repo.scm.commit("add")
 
         os.unlink(repo_dir.FOO)
-        dvc.scm.checkout("new_branc", create_new=True)
+        dvc_repo.scm.checkout("new_branc", create_new=True)
 
         assert os.path.isfile(repo_dir.FOO)
 
-    def test_should_pre_push_hook_push(self, repo_dir, dvc):
+    def test_should_pre_push_hook_push(self, repo_dir, dvc_repo):
         assert main(["install"]) == 0
 
         temp = repo_dir.mkdtemp()
@@ -72,7 +72,7 @@ class TestInstall(object):
         storage_path = os.path.join(temp, "dvc_storage")
 
         foo_checksum = file_md5(repo_dir.FOO)[0]
-        expected_cache_path = dvc.cache.local.get(foo_checksum)
+        expected_cache_path = dvc_repo.cache.local.get(foo_checksum)
 
         ret = main(["remote", "add", "-d", "store", storage_path])
         assert ret == 0
@@ -81,12 +81,12 @@ class TestInstall(object):
         assert ret == 0
 
         stage_file = repo_dir.FOO + Stage.STAGE_FILE_SUFFIX
-        dvc.scm.git.index.add([stage_file, ".gitignore"])
-        dvc.scm.git.index.commit("commit message")
+        dvc_repo.scm.git.index.add([stage_file, ".gitignore"])
+        dvc_repo.scm.git.index.commit("commit message")
 
-        dvc.scm.git.clone(git_remote)
-        dvc.scm.git.create_remote("origin", git_remote)
+        dvc_repo.scm.git.clone(git_remote)
+        dvc_repo.scm.git.create_remote("origin", git_remote)
 
-        dvc.scm.git.git.push("origin", "master")
+        dvc_repo.scm.git.git.push("origin", "master")
 
         assert os.path.isfile(expected_cache_path)

tests/func/test_metrics.py

@@ -807,16 +807,16 @@ class TestShouldDisplayMetricsEvenIfMetricIsMissing(object):
 
         self._commit_metric(dvc, "master")
 
-    def test(self, dvc, caplog):
-        self.setUp(dvc)
+    def test(self, dvc_repo, caplog):
+        self.setUp(dvc_repo)
 
-        dvc.scm.checkout(self.BRANCH_MISSING_METRIC)
+        dvc_repo.scm.checkout(self.BRANCH_MISSING_METRIC)
 
         self._write_metric()
         ret = main(["run", "-M", self.METRIC_FILE])
         assert 0 == ret
 
-        self._commit_metric(dvc, self.BRANCH_MISSING_METRIC)
+        self._commit_metric(dvc_repo, self.BRANCH_MISSING_METRIC)
         os.remove(self.METRIC_FILE)
 
         ret = main(["metrics", "show", "-a"])

tests/func/test_output.py

@@ -25,5 +25,5 @@ def _get_out(dvc, path):
 
 
 @pytest.mark.parametrize("url,scheme", TESTS)
-def test_scheme(dvc, url, scheme):
-    assert type(_get_out(dvc, url)) == OUTS_MAP[scheme]
+def test_scheme(dvc_repo, url, scheme):
+    assert type(_get_out(dvc_repo, url)) == OUTS_MAP[scheme]

tests/func/test_pipeline.py

@@ -4,14 +4,12 @@ from dvc.main import main
 
 from tests.basic_env import TestDvc
 from tests.func.test_repro import TestRepro, TestReproChangedDeepData
-import os
 
 
 class TestPipelineShowSingle(TestDvc):
     def setUp(self):
         super(TestPipelineShowSingle, self).setUp()
         self.stage = "foo.dvc"
-        self.dotFile = "graph.dot"
         ret = main(["add", self.FOO])
         self.assertEqual(ret, 0)
 
@@ -32,9 +30,8 @@ class TestPipelineShowSingle(TestDvc):
         self.assertEqual(ret, 0)
 
     def test_dot(self):
-        ret = main(["pipeline", "show", "--dot", self.dotFile, self.stage])
+        ret = main(["pipeline", "show", "--dot", self.stage])
         self.assertEqual(ret, 0)
-        self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_tree(self):
         ret = main(["pipeline", "show", "--tree", self.stage])
@@ -49,25 +46,12 @@ class TestPipelineShowSingle(TestDvc):
         self.assertEqual(ret, 0)
 
     def test_dot_commands(self):
-        ret = main(
-            [
-                "pipeline",
-                "show",
-                "--dot",
-                self.dotFile,
-                self.stage,
-                "--commands",
-            ]
-        )
+        ret = main(["pipeline", "show", "--dot", self.stage, "--commands"])
         self.assertEqual(ret, 0)
-        self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_dot_outs(self):
-        ret = main(
-            ["pipeline", "show", "--dot", self.dotFile, self.stage, "--outs"]
-        )
+        ret = main(["pipeline", "show", "--dot", self.stage, "--outs"])
         self.assertEqual(ret, 0)
-        self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_not_dvc_file(self):
         ret = main(["pipeline", "show", self.FOO])
@@ -81,7 +65,6 @@ class TestPipelineShowSingle(TestDvc):
 class TestPipelineShow(TestRepro):
     def setUp(self):
         super(TestPipelineShow, self).setUp()
-        self.dotFile = "graph.dot"
 
     def test(self):
         ret = main(["pipeline", "show", self.file1_stage])
@@ -100,11 +83,8 @@ class TestPipelineShow(TestRepro):
         self.assertEqual(ret, 0)
 
     def test_dot(self):
-        ret = main(
-            ["pipeline", "show", "--dot", self.dotFile, self.file1_stage]
-        )
+        ret = main(["pipeline", "show", "--dot", self.file1_stage])
         self.assertEqual(ret, 0)
-        self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_ascii_commands(self):
         ret = main(
@@ -118,31 +98,13 @@ class TestPipelineShow(TestRepro):
 
     def test_dot_commands(self):
         ret = main(
-            [
-                "pipeline",
-                "show",
-                "--dot",
-                self.dotFile,
-                self.file1_stage,
-                "--commands",
-            ]
+            ["pipeline", "show", "--dot", self.file1_stage, "--commands"]
         )
         self.assertEqual(ret, 0)
-        self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_dot_outs(self):
-        ret = main(
-            [
-                "pipeline",
-                "show",
-                "--dot",
-                self.dotFile,
-                self.file1_stage,
-                "--outs",
-            ]
-        )
+        ret = main(["pipeline", "show", "--dot", self.file1_stage, "--outs"])
         self.assertEqual(ret, 0)
-        self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_not_dvc_file(self):
         ret = main(["pipeline", "show", self.file1])
@@ -169,7 +131,6 @@ class TestPipelineShow(TestRepro):
 class TestPipelineShowDeep(TestReproChangedDeepData):
     def setUp(self):
         super(TestPipelineShowDeep, self).setUp()
-        self.dotFile = "graph.dot"
 
     def test(self):
         ret = main(["pipeline", "show", self.file1_stage])
@@ -188,11 +149,8 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
         self.assertEqual(ret, 0)
 
     def test_dot(self):
-        ret = main(
-            ["pipeline", "show", "--dot", self.dotFile, self.file1_stage]
-        )
+        ret = main(["pipeline", "show", "--dot", self.file1_stage])
         self.assertEqual(ret, 0)
-        self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_ascii_commands(self):
         ret = main(
@@ -206,31 +164,13 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
 
     def test_dot_commands(self):
         ret = main(
-            [
-                "pipeline",
-                "show",
-                "--dot",
-                self.dotFile,
-                self.file1_stage,
-                "--commands",
-            ]
+            ["pipeline", "show", "--dot", self.file1_stage, "--commands"]
         )
         self.assertEqual(ret, 0)
-        self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_dot_outs(self):
-        ret = main(
-            [
-                "pipeline",
-                "show",
-                "--dot",
-                self.dotFile,
-                self.file1_stage,
-                "--outs",
-            ]
-        )
+        ret = main(["pipeline", "show", "--dot", self.file1_stage, "--outs"])
         self.assertEqual(ret, 0)
-        self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_not_dvc_file(self):
         ret = main(["pipeline", "show", self.file1])

tests/func/test_remote.py

@@ -174,7 +174,7 @@ class TestRemoteShouldHandleUppercaseRemoteName(TestDvc):
         self.assertEqual(ret, 0)
 
 
-def test_large_dir_progress(repo_dir, dvc):
+def test_large_dir_progress(repo_dir, dvc_repo):
     from dvc.utils import LARGE_DIR_SIZE
     from dvc.progress import progress
 
@@ -183,12 +183,12 @@ def test_large_dir_progress(repo_dir, dvc):
         repo_dir.create(os.path.join("gen", "{}.txt".format(i)), str(i))
 
     with patch.object(progress, "update_target") as update_target:
-        dvc.add("gen")
+        dvc_repo.add("gen")
         assert update_target.called
 
 
-def test_dir_checksum_should_be_key_order_agnostic(dvc):
-    data_dir = os.path.join(dvc.root_dir, "data")
+def test_dir_checksum_should_be_key_order_agnostic(dvc_repo):
+    data_dir = os.path.join(dvc_repo.root_dir, "data")
     file1 = os.path.join(data_dir, "1")
     file2 = os.path.join(data_dir, "2")
 
@@ -200,7 +200,7 @@ def test_dir_checksum_should_be_key_order_agnostic(dvc):
         fobj.write("2")
 
     path_info = PathLOCAL(path=data_dir)
-    with dvc.state:
+    with dvc_repo.state:
         with patch.object(
             RemoteBASE,
             "_collect_dir",
@@ -209,7 +209,7 @@ def test_dir_checksum_should_be_key_order_agnostic(dvc):
                 {"relpath": "2", "md5": "2"},
             ],
         ):
-            checksum1 = dvc.cache.local.get_dir_checksum(path_info)
+            checksum1 = dvc_repo.cache.local.get_dir_checksum(path_info)
 
         with patch.object(
             RemoteBASE,
@@ -219,6 +219,6 @@ def test_dir_checksum_should_be_key_order_agnostic(dvc):
                 {"md5": "2", "relpath": "2"},
             ],
         ):
-            checksum2 = dvc.cache.local.get_dir_checksum(path_info)
+            checksum2 = dvc_repo.cache.local.get_dir_checksum(path_info)
 
     assert checksum1 == checksum2

tests/func/test_repro.py

@@ -1371,15 +1371,186 @@ class TestShouldDisplayMetricsOnReproWithMetricsOption(TestDvc):
 
 
 @pytest.fixture
-def foo_copy(repo_dir, dvc):
-    stages = dvc.add(repo_dir.FOO)
+def repro_dir(dvc_repo, repo_dir):
+    repo_dir.dname = "dir"
+    os.mkdir(repo_dir.dname)
+    repo_dir.emptydname = "emptydir"
+    os.mkdir(repo_dir.emptydname)
+    subdname = os.path.join(repo_dir.dname, "subdir")
+    os.mkdir(subdname)
+
+    repo_dir.source = "source"
+    repo_dir.source_stage = repo_dir.source + ".dvc"
+    stage = dvc_repo.run(
+        fname=repo_dir.source_stage,
+        outs=[repo_dir.source],
+        deps=[repo_dir.FOO],
+    )
+    assert stage is not None
+    assert filecmp.cmp(repo_dir.source, repo_dir.FOO, shallow=False)
+
+    repo_dir.unrelated1 = "unrelated1"
+    repo_dir.unrelated1_stage = repo_dir.unrelated1 + ".dvc"
+    stage = dvc_repo.run(
+        fname=repo_dir.unrelated1_stage,
+        outs=[repo_dir.unrelated1],
+        deps=[repo_dir.source],
+    )
+    assert stage is not None
+
+    repo_dir.unrelated2 = "unrelated2"
+    repo_dir.unrelated2_stage = repo_dir.unrelated2 + ".dvc"
+    stage = dvc_repo.run(
+        fname=repo_dir.unrelated2_stage,
+        outs=[repo_dir.unrelated2],
+        deps=[repo_dir.DATA],
+    )
+    assert stage is not None
+
+    repo_dir.first = os.path.join(repo_dir.dname, "first")
+    repo_dir.first_stage = repo_dir.first + ".dvc"
+
+    stage = dvc_repo.run(
+        fname=repo_dir.first_stage,
+        deps=[repo_dir.source],
+        outs=[repo_dir.first],
+        cmd="python {} {} {}".format(
+            repo_dir.CODE, repo_dir.source, repo_dir.first
+        ),
+    )
+    assert stage is not None
+    assert filecmp.cmp(repo_dir.first, repo_dir.FOO, shallow=False)
+
+    repo_dir.second = os.path.join(subdname, "second")
+    repo_dir.second_stage = repo_dir.second + ".dvc"
+    stage = dvc_repo.run(
+        fname=repo_dir.second_stage,
+        outs=[repo_dir.second],
+        deps=[repo_dir.DATA],
+    )
+    assert stage is not None
+    assert filecmp.cmp(repo_dir.second, repo_dir.DATA, shallow=False)
+
+    repo_dir.third_stage = os.path.join(repo_dir.dname, "Dvcfile")
+    stage = dvc_repo.run(
+        fname=repo_dir.third_stage, deps=[repo_dir.first, repo_dir.second]
+    )
+    assert stage is not None
+
+    yield repo_dir
+
+
+def test_recursive_repro_default(dvc_repo, repro_dir):
+    """
+    Test recursive repro on dir after a dep outside this dir has changed.
+    """
+    os.unlink(repro_dir.FOO)
+    shutil.copyfile(repro_dir.BAR, repro_dir.FOO)
+
+    stages = dvc_repo.reproduce(repro_dir.dname, recursive=True)
+    # Check that the dependency ("source") and the dependent stages
+    # inside the folder have been reproduced ("first", "third")
+    assert len(stages) == 3
+    names = [stage.relpath for stage in stages]
+    assert repro_dir.source_stage in names
+    assert repro_dir.first_stage in names
+    assert repro_dir.third_stage in names
+    assert filecmp.cmp(repro_dir.source, repro_dir.BAR, shallow=False)
+    assert filecmp.cmp(repro_dir.first, repro_dir.BAR, shallow=False)
+
+
+def test_recursive_repro_single(dvc_repo, repro_dir):
+    """
+    Test recursive single-item repro on dir
+    after a dep outside this dir has changed.
+    """
+    os.unlink(repro_dir.FOO)
+    shutil.copyfile(repro_dir.BAR, repro_dir.FOO)
+
+    os.unlink(repro_dir.DATA)
+    shutil.copyfile(repro_dir.BAR, repro_dir.DATA)
+
+    stages = dvc_repo.reproduce(
+        repro_dir.dname, recursive=True, single_item=True
+    )
+    # Check that just stages inside given dir
+    # with changed direct deps have been reproduced.
+    # This means that "first" stage should not be reproduced
+    # since it depends on "source".
+    # Also check that "second" stage was reproduced before "third" stage
+    assert len(stages) == 2
+    assert repro_dir.second_stage == stages[0].relpath
+    assert repro_dir.third_stage == stages[1].relpath
+    assert filecmp.cmp(repro_dir.second, repro_dir.BAR, shallow=False)
+
+
+def test_recursive_repro_single_force(dvc_repo, repro_dir):
+    """
+    Test recursive single-item force repro on dir
+    without any dependencies changing.
+    """
+    stages = dvc_repo.reproduce(
+        repro_dir.dname, recursive=True, single_item=True, force=True
+    )
+    assert len(stages) == 3
+    names = [stage.relpath for stage in stages]
+    # Check that all stages inside given dir have been reproduced
+    # Also check that "second" stage was reproduced before "third" stage
+    # and that "first" stage was reproduced before "third" stage
+    assert repro_dir.first_stage in names
+    assert repro_dir.second_stage in names
+    assert repro_dir.third_stage in names
+    assert names.index(repro_dir.first_stage) < names.index(
+        repro_dir.third_stage
+    )
+    assert names.index(repro_dir.second_stage) < names.index(
+        repro_dir.third_stage
+    )
+
+
+def test_recursive_repro_empty_dir(dvc_repo, repro_dir):
+    """
+    Test recursive repro on an empty directory
+    """
+    stages = dvc_repo.reproduce(
+        repro_dir.emptydname, recursive=True, force=True
+    )
+    assert len(stages) == 0
+
+
+def test_recursive_repro_recursive_missing_file(dvc_repo):
+    """
+    Test recursive repro on a missing file
+    """
+    with pytest.raises(StageFileDoesNotExistError):
+        dvc_repo.reproduce("notExistingStage.dvc", recursive=True)
+    with pytest.raises(StageFileDoesNotExistError):
+        dvc_repo.reproduce("notExistingDir/", recursive=True)
+
+
+def test_recursive_repro_on_stage_file(dvc_repo, repro_dir):
+    """
+    Test recursive repro on a stage file instead of directory
+    """
+    stages = dvc_repo.reproduce(
+        repro_dir.first_stage, recursive=True, force=True
+    )
+    assert len(stages) == 2
+    names = [stage.relpath for stage in stages]
+    assert repro_dir.source_stage in names
+    assert repro_dir.first_stage in names
+
+
+@pytest.fixture
+def foo_copy(repo_dir, dvc_repo):
+    stages = dvc_repo.add(repo_dir.FOO)
     assert len(stages) == 1
     foo_stage = stages[0]
     assert foo_stage is not None
 
     fname = "foo_copy"
     stage_fname = fname + ".dvc"
-    dvc.run(
+    dvc_repo.run(
         fname=stage_fname,
         outs=[fname],
         deps=[repo_dir.FOO, repo_dir.CODE],
@@ -1388,8 +1559,8 @@ def foo_copy(repo_dir, dvc):
     return {"fname": fname, "stage_fname": stage_fname}
 
 
-def test_dvc_formatting_retained(dvc, foo_copy):
-    root = Path(dvc.root_dir)
+def test_dvc_formatting_retained(dvc_repo, foo_copy):
+    root = Path(dvc_repo.root_dir)
     stage_file = root / foo_copy["stage_fname"]
 
     # Add comments and custom formatting to stage file
@@ -1400,7 +1571,7 @@ def test_dvc_formatting_retained(dvc, foo_copy):
 
     # Rewrite data source and repro
     (root / "foo").write_text("new_foo")
-    dvc.reproduce(foo_copy["stage_fname"])
+    dvc_repo.reproduce(foo_copy["stage_fname"])
 
     # All differences should be only about md5
     assert _hide_md5(stage_text) == _hide_md5(stage_file.read_text())

tests/func/test_stage.py

@@ -159,18 +159,18 @@ class TestExternalRemoteResolution(TestDvc):
         assert os.path.exists("movie.txt")
 
 
-def test_md5_ignores_comments(repo_dir, dvc):
-    stage, = dvc.add("foo")
+def test_md5_ignores_comments(repo_dir, dvc_repo):
+    stage, = dvc_repo.add("foo")
 
     with open(stage.path, "a") as f:
         f.write("# End comment\n")
 
-    new_stage = Stage.load(dvc, stage.path)
+    new_stage = Stage.load(dvc_repo, stage.path)
     assert not new_stage.changed_md5()
 
 
-def test_meta_is_preserved(dvc):
-    stage, = dvc.add("foo")
+def test_meta_is_preserved(dvc_repo):
+    stage, = dvc_repo.add("foo")
 
     # Add meta to stage file
     data = load_stage_file(stage.path)
@@ -178,7 +178,7 @@ def test_meta_is_preserved(dvc):
     dump_stage_file(stage.path, data)
 
     # Loading and dumping to test that it works and meta is retained
-    new_stage = Stage.load(dvc, stage.path)
+    new_stage = Stage.load(dvc_repo, stage.path)
     new_stage.dump()
 
     new_data = load_stage_file(stage.path)

tests/remotes_env.sample

@@ -0,0 +1,11 @@
+# Uncomment and fill in to test against S3
+# export AWS_ACCESS_KEY_ID="...SET-ME..."
+# export AWS_SECRET_ACCESS_KEY="...SET-ME..."
+# export DVC_TEST_AWS_REPO_BUCKET="...SET-ME..."
+
+# Uncomment and set a bucket name to test against Google Storage
+# export GOOGLE_APPLICATION_CREDENTIALS="scripts/ci/gcp-creds.json"
+
+# Uncomment to test against Microsoft Azure via Azurite
+# export AZURE_STORAGE_CONTAINER_NAME="dvc-test"
+# export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"

tests/requirements.txt

@@ -1,18 +0,0 @@
-pytest>=4.4.0
-pytest-timeout>=1.3.3
-pytest-cov>=2.6.1
-pytest-xdist>=1.26.1
-pytest-mock>=1.10.4
-flaky>=3.5.3
-mock>=3.0.0
-xmltodict>=0.11.0
-awscli>=1.16.125
-google-compute-engine
-pywin32; sys_platform == 'win32'
-Pygments  # required by collective.checkdocs
-collective.checkdocs
-black==19.3b0 ; python_version >= '3.6'
-flake8
-flake8-docstrings
-jaraco.windows==3.9.2
-mock-ssh-server >= 0.5.0

tests/unit/command/test_repro.py

@@ -10,18 +10,19 @@ default_arguments = {
     "interactive": False,
     "no_commit": False,
     "pipeline": False,
-    "recursive": True,
+    "single_item": False,
+    "recursive": False,
 }
 
 
-def test_default_arguments(dvc, mocker):
+def test_default_arguments(dvc_repo, mocker):
     cmd = CmdRepro(parse_args(["repro"]))
     mocker.patch.object(cmd.repo, "reproduce")
     cmd.run()
     cmd.repo.reproduce.assert_called_with("Dvcfile", **default_arguments)
 
 
-def test_downstream(dvc, mocker):
+def test_downstream(dvc_repo, mocker):
     cmd = CmdRepro(parse_args(["repro", "--downstream"]))
     mocker.patch.object(cmd.repo, "reproduce")
     cmd.run()

tests/unit/command/test_version.py

@@ -4,7 +4,7 @@ from dvc.command.version import CmdVersion
 from dvc.cli import parse_args
 
 
-def test_run_in_repo(repo_dir, dvc):
+def test_run_in_repo(repo_dir, dvc_repo):
     cmd = CmdVersion(parse_args(["version"]))
     ret = cmd.run_cmd()
     assert ret == 0

tests/unit/remote/test_slow_link_detection.py

@@ -89,13 +89,12 @@ class TestSlowLinkDetectorDecorator(object):
             assert self.method_result_mock == result
 
         msg = (
-            "You can cut execution time considerably. Check:\n"
+            "You can cut execution time considerably by using a different"
+            " `cache.type` configuration.\nSee "
             "{blue}https://dvc.org/doc/commands-reference/config#cache{"
-            "reset}"
-            "\nfor "
-            "more information.\nTo disable this message, run:\n'dvc "
-            "config "
-            "cache.slow_link_warning False'".format(
+            "reset} for more information.\n"
+            "To disable this message, run:\n"
+            "'dvc config cache.slow_link_warning false'".format(
                 blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
             )
         )

tests/unit/test_logger.py

@@ -150,10 +150,10 @@ class TestColorFormatter:
 
             assert expected == formatter.format(caplog.records[0])
 
-    def test_progress_awareness(self, mock, capsys, caplog):
+    def test_progress_awareness(self, mocker, capsys, caplog):
         from dvc.progress import progress
 
-        with mock.patch("sys.stdout.isatty", return_value=True):
+        with mocker.patch("sys.stdout.isatty", return_value=True):
             progress.set_n_total(100)
             progress.update_target("progress", 1, 10)