dvc: bump to 1.1.10

* pull: target message

* update fetch/status/etc help

Co-authored-by: Ruslan Kuprieiev <[email protected]>

dvc/cli.py

@@ -80,34 +80,36 @@ COMMANDS = [
 ]
 
 
+def _find_parser(parser, cmd_cls):
+    defaults = parser._defaults  # pylint: disable=protected-access
+    if not cmd_cls or cmd_cls == defaults.get("func"):
+        parser.print_help()
+        raise DvcParserError()
+
+    actions = parser._actions  # pylint: disable=protected-access
+    for action in actions:
+        if not isinstance(action.choices, dict):
+            # NOTE: we are only interested in subparsers
+            continue
+        for subparser in action.choices.values():
+            _find_parser(subparser, cmd_cls)
+
+
 class DvcParser(argparse.ArgumentParser):
     """Custom parser class for dvc CLI."""
 
-    def error(self, message, command=None):  # pylint: disable=arguments-differ
-        """Custom error method.
-        Args:
-            message (str): error message.
-            command (str): subcommand name for help message
-        Raises:
-            dvc.exceptions.DvcParser: dvc parser exception.
-
-        """
+    def error(self, message, cmd_cls=None):  # pylint: disable=arguments-differ
         logger.error(message)
-        if command is not None:
-            for action in self._actions:
-                if action.dest == "cmd" and command in action.choices:
-                    subparser = action.choices[command]
-                    subparser.print_help()
-                    raise DvcParserError()
-        self.print_help()
-        raise DvcParserError()
+        _find_parser(self, cmd_cls)
 
-    # override this to send subcommand name to error method
     def parse_args(self, args=None, namespace=None):
+        # NOTE: overriding to provide a more granular help message.
+        # E.g. `dvc plots diff --bad-flag` would result in a `dvc plots diff`
+        # help message instead of generic `dvc` usage.
         args, argv = self.parse_known_args(args, namespace)
         if argv:
             msg = "unrecognized arguments: %s"
-            self.error(msg % " ".join(argv), args.cmd)
+            self.error(msg % " ".join(argv), getattr(args, "func", None))
         return args
 
 

dvc/command/add.py

@@ -63,10 +63,6 @@ def add_parser(subparsers, parent_parser):
         metavar="<filename>",
     )
     parser.add_argument(
-        "targets",
-        nargs="+",
-        help="Input files/directories to add.",
-        metavar="targets",
-        choices=completion.Required.FILE,
-    )
+        "targets", nargs="+", help="Input files/directories to add.",
+    ).complete = completion.FILE
     parser.set_defaults(func=CmdAdd)

dvc/command/cache.py

@@ -56,7 +56,5 @@ def add_parser(subparsers, parent_parser):
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
         "config file location.",
-        metavar="value",
-        choices=completion.Required.DIR,
-    )
+    ).complete = completion.DIR
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/checkout.py

@@ -112,9 +112,9 @@ def add_parser(subparsers, parent_parser):
     checkout_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-files to checkout. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
-        metavar="targets",
-        choices=completion.Optional.DVC_FILE,
-    )
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVC_FILE
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -67,7 +67,5 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="DVC-files to commit. Optional. "
         "(Finds all DVC-files in the workspace by default.)",
-        metavar="targets",
-        choices=completion.Optional.DVC_FILE,
-    )
+    ).complete = completion.DVC_FILE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -6,9 +6,11 @@ import shtab
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
-CHOICE_FUNCTIONS = {
-    "bash": {"DVCFile": "_dvc_compgen_DVCFiles"},
-    "zsh": {"DVCFile": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'"},
+FILE = shtab.FILE
+DIR = shtab.DIRECTORY
+DVC_FILE = {
+    "bash": "_dvc_compgen_DVCFiles",
+    "zsh": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'",
 }
 PREAMBLE = {
     "bash": """
@@ -24,26 +26,13 @@ _dvc_compgen_DVCFiles() {
 }
 
 
-class Optional(shtab.Optional):
-    DVC_FILE = [shtab.Choice("DVCFile", required=False)]
-
-
-class Required(shtab.Required):
-    DVC_FILE = [shtab.Choice("DVCFile", required=True)]
-
-
 class CmdCompletion(CmdBaseNoRepo):
     def run(self):
         from dvc.cli import get_main_parser
 
         parser = get_main_parser()
         shell = self.args.shell
-        script = shtab.complete(
-            parser,
-            shell=shell,
-            preamble=PREAMBLE[shell],
-            choice_functions=CHOICE_FUNCTIONS[shell],
-        )
+        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
         print(script)
         return 0
 

dvc/command/daemon.py

@@ -66,9 +66,6 @@ def add_parser(subparsers, parent_parser):
         help=DAEMON_ANALYTICS_HELP,
     )
     daemon_analytics_parser.add_argument(
-        "target",
-        help="Analytics file.",
-        metavar="target",
-        choices=completion.Required.FILE,
-    )
+        "target", help="Analytics file.",
+    ).complete = completion.FILE
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/data_sync.py

@@ -103,11 +103,11 @@ def shared_parent_parser():
     parent_parser.add_argument(
         "targets",
         nargs="*",
-        help="Limit command scope to these DVC-files. "
-        "Using -R, directories to search DVC-files in can also be given.",
-        metavar="targets",
-        choices=completion.Optional.DVC_FILE,
-    )
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVC_FILE
 
     return parent_parser
 

dvc/command/freeze.py

@@ -40,12 +40,8 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     freeze_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="Stages or .dvc files to freeze",
-        metavar="targets",
-        choices=completion.Required.DVC_FILE,
-    )
+        "targets", nargs="+", help="Stages or .dvc files to freeze",
+    ).complete = completion.DVC_FILE
     freeze_parser.set_defaults(func=CmdFreeze)
 
     UNFREEZE_HELP = "Unfreeze stages or .dvc files."
@@ -57,10 +53,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unfreeze_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="Stages or .dvc files to unfreeze",
-        metavar="targets",
-        choices=completion.Required.DVC_FILE,
-    )
+        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
+    ).complete = completion.DVC_FILE
     unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/get.py

@@ -63,19 +63,15 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the repository",
-        metavar="path",
-        choices=completion.Required.FILE,
-    )
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
     get_parser.add_argument(
         "-o",
         "--out",
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     get_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/get_url.py

@@ -34,10 +34,6 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out",
-        nargs="?",
-        help="Destination path to put data to.",
-        metavar="out",
-        choices=completion.Optional.DIR,
-    )
+        "out", nargs="?", help="Destination path to put data to.",
+    ).complete = completion.DIR
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/imp.py

@@ -44,19 +44,15 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the repository",
-        metavar="path",
-        choices=completion.Required.FILE,
-    )
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
     import_parser.add_argument(
         "-o",
         "--out",
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     import_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/imp_url.py

@@ -58,18 +58,13 @@ def add_parser(subparsers, parent_parser):
         "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out",
-        nargs="?",
-        help="Destination path to put files to.",
-        metavar="out",
-        choices=completion.Optional.DIR,
-    )
+        "out", nargs="?", help="Destination path to put files to.",
+    ).complete = completion.DIR
     import_parser.add_argument(
         "--file",
         help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     import_parser.add_argument(
         "--no-exec",
         action="store_true",

dvc/command/ls/__init__.py

@@ -75,7 +75,5 @@ def add_parser(subparsers, parent_parser):
         "path",
         nargs="?",
         help="Path to directory within the repository to list outputs for",
-        metavar="path",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     list_parser.set_defaults(func=CmdList)

dvc/command/metrics.py

@@ -170,9 +170,7 @@ def add_parser(subparsers, parent_parser):
             "Limit command scope to these metric files. Using -R, "
             "directories to search metric files in can also be given."
         ),
-        metavar="targets",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     metrics_show_parser.add_argument(
         "-a",
         "--all-branches",
@@ -238,8 +236,7 @@ def add_parser(subparsers, parent_parser):
             "directories to search metric files in can also be given."
         ),
         metavar="<paths>",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     metrics_diff_parser.add_argument(
         "-R",
         "--recursive",

dvc/command/move.py

@@ -37,15 +37,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src",
-        help="Source path to a data file or directory.",
-        metavar="src",
-        choices=completion.Required.FILE,
-    )
+        "src", help="Source path to a data file or directory.",
+    ).complete = completion.FILE
     move_parser.add_argument(
-        "dst",
-        help="Destination path.",
-        metavar="dst",
-        choices=completion.Required.FILE,
-    )
+        "dst", help="Destination path.",
+    ).complete = completion.FILE
     move_parser.set_defaults(func=CmdMove)

dvc/command/plots.py

@@ -129,9 +129,7 @@ def add_parser(subparsers, parent_parser):
         "targets",
         nargs="*",
         help="Plots files to visualize. Shows all plots by default.",
-        metavar="targets",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     _add_props_arguments(plots_show_parser)
     _add_output_arguments(plots_show_parser)
     plots_show_parser.set_defaults(func=CmdPlotsShow)
@@ -152,8 +150,7 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="Plots file to visualize. Shows all plots by default.",
         metavar="<path>",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     plots_diff_parser.add_argument(
         "revisions", nargs="*", default=None, help="Git commits to plot from",
     )
@@ -170,11 +167,8 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_modify_parser.add_argument(
-        "target",
-        help="Metric file to set properties to",
-        metavar="target",
-        choices=completion.Required.FILE,
-    )
+        "target", help="Metric file to set properties to",
+    ).complete = completion.FILE
     _add_props_arguments(plots_modify_parser)
     plots_modify_parser.add_argument(
         "--unset",
@@ -198,8 +192,7 @@ def _add_props_arguments(parser):
             )
         ),
         metavar="<path>",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     parser.add_argument(
         "-x", default=None, help="Field name for X axis.", metavar="<field>"
     )
@@ -231,8 +224,7 @@ def _add_output_arguments(parser):
         default=None,
         help="Destination path to save plots to",
         metavar="<path>",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     parser.add_argument(
         "--show-vega",
         action="store_true",

dvc/command/remove.py

@@ -37,10 +37,6 @@ def add_parser(subparsers, parent_parser):
         help="Remove outputs as well.",
     )
     remove_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="DVC-files to remove.",
-        metavar="targets",
-        choices=completion.Required.DVC_FILE,
-    )
+        "targets", nargs="+", help="DVC-files to remove.",
+    ).complete = completion.DVC_FILE
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -72,9 +72,7 @@ def add_parser(subparsers, parent_parser):
         "targets",
         nargs="*",
         help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
-        metavar="targets",
-        choices=completion.Optional.DVC_FILE,
-    )
+    ).complete = completion.DVC_FILE
     repro_parser.add_argument(
         "-f",
         "--force",

dvc/command/unprotect.py

@@ -33,10 +33,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="Data files/directories to unprotect.",
-        metavar="targets",
-        choices=completion.Required.FILE,
-    )
+        "targets", nargs="+", help="Data files/directories to unprotect.",
+    ).complete = completion.FILE
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -33,12 +33,8 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="DVC-files to update.",
-        metavar="targets",
-        choices=completion.Required.DVC_FILE,
-    )
+        "targets", nargs="+", help="DVC-files to update.",
+    ).complete = completion.DVC_FILE
     update_parser.add_argument(
         "--rev",
         nargs="?",

dvc/config.py

@@ -232,8 +232,10 @@ class Config(dict):
     CONFIG_LOCAL = "config.local"
 
     def __init__(
-        self, dvc_dir=None, validate=True
+        self, dvc_dir=None, validate=True, tree=None,
     ):  # pylint: disable=super-init-not-called
+        from dvc.scm.tree import WorkingTree
+
         self.dvc_dir = dvc_dir
 
         if not dvc_dir:
@@ -246,6 +248,9 @@ class Config(dict):
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
+        self.wtree = WorkingTree(self.dvc_dir)
+        self.tree = tree.tree if tree else self.wtree
+
         self.load(validate=validate)
 
     @classmethod
@@ -304,8 +309,32 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
+    def _load_config(self, level):
+        filename = self.files[level]
+        tree = self.tree if level == "repo" else self.wtree
+
+        if tree.exists(filename):
+            with tree.open(filename) as fobj:
+                conf_obj = configobj.ConfigObj(fobj)
+        else:
+            conf_obj = configobj.ConfigObj()
+        return _parse_remotes(_lower_keys(conf_obj.dict()))
+
+    def _save_config(self, level, conf_dict):
+        filename = self.files[level]
+        tree = self.tree if level == "repo" else self.wtree
+
+        logger.debug(f"Writing '{filename}'.")
+
+        tree.makedirs(os.path.dirname(filename), exist_ok=True)
+
+        config = configobj.ConfigObj(_pack_remotes(conf_dict))
+        with tree.open(filename, "wb") as fobj:
+            config.write(fobj)
+        config.filename = filename
+
     def load_one(self, level):
-        conf = _load_config(self.files[level])
+        conf = self._load_config(level)
         conf = self._load_paths(conf, self.files[level])
 
         # Auto-verify sections
@@ -375,7 +404,7 @@ class Config(dict):
         _merge(merged_conf, conf)
         self.validate(merged_conf)
 
-        _save_config(self.files[level], conf)
+        self._save_config(level, conf)
         self.load()
 
     @staticmethod
@@ -386,20 +415,6 @@ class Config(dict):
             raise ConfigError(str(exc)) from None
 
 
-def _load_config(filename):
-    conf_obj = configobj.ConfigObj(filename)
-    return _parse_remotes(_lower_keys(conf_obj.dict()))
-
-
-def _save_config(filename, conf_dict):
-    logger.debug(f"Writing '{filename}'.")
-    os.makedirs(os.path.dirname(filename), exist_ok=True)
-
-    config = configobj.ConfigObj(_pack_remotes(conf_dict))
-    config.filename = filename
-    config.write()
-
-
 def _parse_remotes(conf):
     result = {"remote": {}}
 

dvc/dependency/param.py

@@ -45,9 +45,8 @@ class ParamsDependency(LocalDependency):
         if not values:
             return
         for param in self.params:
-            value = values.get(param)
-            if value:
-                self.info[param] = value
+            if param in values:
+                self.info[param] = values[param]
 
     def save(self):
         super().save()

dvc/dependency/repo.py

@@ -64,7 +64,7 @@ class RepoDependency(LocalDependency):
 
                 # We are polluting our repo cache with some dir listing here
                 if tree.isdir(path):
-                    return self.repo.cache.local.get_hash(path, tree)
+                    return self.repo.cache.local.get_hash(path, tree=tree)
                 return tree.get_file_hash(path)
 
     def status(self):

dvc/ignore.py

@@ -44,11 +44,11 @@ class DvcIgnorePatterns(DvcIgnore):
 
     def __call__(self, root, dirs, files):
         files = [f for f in files if not self.matches(root, f)]
-        dirs = [d for d in dirs if not self.matches(root, d)]
+        dirs = [d for d in dirs if not self.matches(root, d, True)]
 
         return dirs, files
 
-    def matches(self, dirname, basename):
+    def matches(self, dirname, basename, is_dir=False):
         # NOTE: `relpath` is too slow, so we have to assume that both
         # `dirname` and `self.dirname` are relative or absolute together.
         prefix = self.dirname + os.sep
@@ -63,13 +63,19 @@ class DvcIgnorePatterns(DvcIgnore):
 
         if not System.is_unix():
             path = normalize_file(path)
-        return self.ignore(path)
+        return self.ignore(path, is_dir)
 
-    def ignore(self, path):
+    def ignore(self, path, is_dir):
         result = False
-        for ignore, pattern in self.ignore_spec:
-            if pattern.match(path):
-                result = ignore
+        if is_dir:
+            path_dir = f"{path}/"
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path) or pattern.match(path_dir):
+                    result = ignore
+        else:
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path):
+                    result = ignore
         return result
 
     def __hash__(self):
@@ -248,3 +254,6 @@ class CleanTree(BaseTree):
     @property
     def hash_jobs(self):
         return self.tree.hash_jobs
+
+    def makedirs(self, path, mode=0o777, exist_ok=True):
+        self.tree.makedirs(path, mode=mode, exist_ok=exist_ok)

dvc/remote/azure.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import threading
 from datetime import datetime, timedelta
 
@@ -17,38 +18,45 @@ class AzureRemoteTree(BaseRemoteTree):
     PATH_CLS = CloudURLInfo
     REQUIRES = {
         "azure-storage-blob": "azure.storage.blob",
-        "azure-cli-core": "azure.cli.core",
+        "knack": "knack",
     }
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
     LIST_OBJECT_PAGE_SIZE = 5000
 
     def __init__(self, repo, config):
-        from azure.cli.core import get_default_cli
-
         super().__init__(repo, config)
 
-        # NOTE: az_config takes care of env vars
-        az_config = get_default_cli().config
-
         url = config.get("url", "azure://")
         self.path_info = self.PATH_CLS(url)
 
         if not self.path_info.bucket:
-            container = az_config.get("storage", "container_name", None)
+            container = self._az_config.get("storage", "container_name", None)
             self.path_info = self.PATH_CLS(f"azure://{container}")
 
         self._conn_kwargs = {
-            opt: config.get(opt) or az_config.get("storage", opt, None)
+            opt: config.get(opt) or self._az_config.get("storage", opt, None)
             for opt in ["connection_string", "sas_token"]
         }
-        self._conn_kwargs["account_name"] = az_config.get(
+        self._conn_kwargs["account_name"] = self._az_config.get(
             "storage", "account", None
         )
-        self._conn_kwargs["account_key"] = az_config.get(
+        self._conn_kwargs["account_key"] = self._az_config.get(
             "storage", "key", None
         )
 
+    @cached_property
+    def _az_config(self):
+        # NOTE: ideally we would've used get_default_cli().config from
+        # azure.cli.core, but azure-cli-core has a lot of conflicts with other
+        # dependencies. So instead we are just use knack directly
+        from knack.config import CLIConfig
+
+        config_dir = os.getenv(
+            "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
+        )
+        return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
+
     @wrap_prop(threading.Lock())
     @cached_property
     def blob_service(self):
@@ -113,6 +121,8 @@ class AzureRemoteTree(BaseRemoteTree):
             next_marker = blobs.next_marker
 
     def walk_files(self, path_info, **kwargs):
+        if not kwargs.pop("prefix", False):
+            path_info = path_info / ""
         for fname in self._list_paths(
             path_info.bucket, path_info.path, **kwargs
         ):

dvc/remote/base.py

@@ -213,7 +213,12 @@ class BaseRemoteTree:
         return False  # We can't be sure by default
 
     def walk_files(self, path_info, **kwargs):
-        """Return a generator with `PathInfo`s to all the files"""
+        """Return a generator with `PathInfo`s to all the files.
+
+        Optional kwargs:
+            prefix (bool): If true `path_info` will be treated as a prefix
+                rather than directory path.
+        """
         raise NotImplementedError
 
     def is_empty(self, path_info):
@@ -522,14 +527,16 @@ class BaseRemoteTree:
                 path_info = self.path_info / prefix[:2] / prefix[2:]
             else:
                 path_info = self.path_info / prefix[:2]
+            prefix = True
         else:
             path_info = self.path_info
+            prefix = False
         if progress_callback:
-            for file_info in self.walk_files(path_info):
+            for file_info in self.walk_files(path_info, prefix=prefix):
                 progress_callback()
                 yield file_info.path
         else:
-            yield from self.walk_files(path_info)
+            yield from self.walk_files(path_info, prefix=prefix)
 
     def list_hashes(self, prefix=None, progress_callback=None):
         """Iterate over hashes in this tree.

dvc/remote/gdrive.py

@@ -553,7 +553,8 @@ class GDriveRemoteTree(BaseRemoteTree):
             )
 
     def walk_files(self, path_info, **kwargs):
-        if path_info == self.path_info:
+        use_prefix = kwargs.pop("prefix", False)
+        if path_info == self.path_info or not use_prefix:
             prefix = None
         else:
             prefix = path_info.path

dvc/remote/gs.py

@@ -144,7 +144,9 @@ class GSRemoteTree(BaseRemoteTree):
             yield blob.name
 
     def walk_files(self, path_info, **kwargs):
-        for fname in self._list_paths(path_info / "", **kwargs):
+        if not kwargs.pop("prefix", False):
+            path_info = path_info / ""
+        for fname in self._list_paths(path_info, **kwargs):
             # skip nested empty directories
             if fname.endswith("/"):
                 continue

dvc/remote/hdfs.py

@@ -165,7 +165,7 @@ class HDFSRemoteTree(BaseRemoteTree):
         # NOTE: pyarrow doesn't support checksum, so we need to use hadoop
         regex = r".*\t.*\t(?P<checksum>.*)"
         stdout = self.hadoop_fs(
-            f"checksum {path_info.path}", user=path_info.user
+            f"checksum {path_info.url}", user=path_info.user
         )
         return self._group(regex, stdout, "checksum")
 

dvc/remote/oss.py

@@ -100,6 +100,8 @@ class OSSRemoteTree(BaseRemoteTree):  # pylint:disable=abstract-method
             yield blob.key
 
     def walk_files(self, path_info, **kwargs):
+        if not kwargs.pop("prefix", False):
+            path_info = path_info / ""
         for fname in self._list_paths(path_info):
             if fname.endswith("/"):
                 continue

dvc/remote/s3.py

@@ -187,7 +187,9 @@ class S3RemoteTree(BaseRemoteTree):
         )
 
     def walk_files(self, path_info, **kwargs):
-        for fname in self._list_paths(path_info / "", **kwargs):
+        if not kwargs.pop("prefix", False):
+            path_info = path_info / ""
+        for fname in self._list_paths(path_info, **kwargs):
             if fname.endswith("/"):
                 continue
 

dvc/repo/__init__.py

@@ -91,14 +91,14 @@ class Repo:
         else:
             root_dir = self.find_root(root_dir)
             self.root_dir = os.path.abspath(os.path.realpath(root_dir))
+            self.tree = WorkingTree(self.root_dir)
 
         self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
-        self.config = Config(self.dvc_dir)
+        self.config = Config(self.dvc_dir, tree=self.tree)
 
         if not scm:
             no_scm = self.config["core"].get("no_scm", False)
             self.scm = SCM(self.root_dir, no_scm=no_scm)
-            self.tree = WorkingTree(self.root_dir)
 
         self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
         self.index_dir = os.path.join(self.tmp_dir, "index")

dvc/repo/tree.py

@@ -12,7 +12,7 @@ from dvc.utils.fs import copy_fobj_to_file, makedirs
 logger = logging.getLogger(__name__)
 
 
-class DvcTree(BaseTree):
+class DvcTree(BaseTree):  # pylint:disable=abstract-method
     """DVC repo tree.
 
     Args:
@@ -236,7 +236,7 @@ class DvcTree(BaseTree):
         return out.checksum
 
 
-class RepoTree(BaseTree):
+class RepoTree(BaseTree):  # pylint:disable=abstract-method
     """DVC + git-tracked files tree.
 
     Args:

dvc/scm/git/tree.py

@@ -19,7 +19,7 @@ def _item_basename(item):
     return os.path.basename(item.path)
 
 
-class GitTree(BaseTree):
+class GitTree(BaseTree):  # pylint:disable=abstract-method
     """Proxies the repo file access methods to Git objects"""
 
     def __init__(self, git, rev):

dvc/scm/tree.py

@@ -37,6 +37,9 @@ class BaseTree:
                 # NOTE: os.path.join is ~5.5 times slower
                 yield f"{root}{os.sep}{file}"
 
+    def makedirs(self, path, mode=0o777, exist_ok=True):
+        raise NotImplementedError
+
 
 class WorkingTree(BaseTree):
     """Proxies the repo file access methods to working tree files"""
@@ -90,6 +93,9 @@ class WorkingTree(BaseTree):
     def hash_jobs(self):
         return max(1, min(4, cpu_count() // 2))
 
+    def makedirs(self, path, mode=0o777, exist_ok=True):
+        os.makedirs(path, mode=mode, exist_ok=exist_ok)
+
 
 def is_working_tree(tree):
     return isinstance(tree, WorkingTree) or isinstance(

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "1.1.2"
+_BASE_VERSION = "1.1.10"
 
 
 def _generate_version(base_version):

scripts/ci/before_install.sh

@@ -31,12 +31,6 @@ if [[ "$TRAVIS_BUILD_STAGE_NAME" == "test" ]]; then
     ssh 0.0.0.0 ls &>/dev/null
   fi
 
-  if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-    bash "$scriptdir/install_azurite.sh"
-    bash "$scriptdir/install_oss.sh"
-    bash "$scriptdir/install_hadoop.sh"
-  fi
-
   if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
     brew install openssl
     $scriptdir/retry.sh brew cask install google-cloud-sdk

scripts/ci/core-site.xml

@@ -1,6 +0,0 @@
-<configuration>
-<property>
-<name>fs.defaultFS</name>
-<value>hdfs://localhost:8020</value>
-</property>
-</configuration>

scripts/ci/hdfs-site.xml

@@ -1,6 +0,0 @@
-<configuration>
-<property>
-<name>dfs.replication</name>
-<value>1</value>
-</property>
-</configuration>

scripts/ci/install_azurite.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -x
-
-# run azurite
-sudo docker run -d --restart always -e executable=blob -p 10000:10000 \
-  --tmpfs /opt/azurite/folder \
-  mcr.microsoft.com/azure-storage/azurite:3.3.0-preview \
-  azurite -l /data --blobHost 0.0.0.0
-
-# save secrets
-echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >>env.sh
-echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >>env.sh

scripts/ci/install_hadoop.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-# NOTE: based on http://sharafjaffri.blogspot.com/2015/04/installing-single-node-hadoop-26-using.html
-
-set -x
-set -e
-
-sudo add-apt-repository -y ppa:openjdk-r/ppa
-sudo apt-get update -y
-sudo apt-get install openjdk-7-jdk
-java -version
-
-pushd /usr/local
-sudo wget https://s3-us-east-2.amazonaws.com/dvc-public/travis-test/hadoop-2.6.5.tar.gz
-sudo tar xzf hadoop-2.6.5.tar.gz
-sudo mkdir hadoop
-sudo mv hadoop-2.6.5/* hadoop/
-popd
-
-echo "export HADOOP_HOME=/usr/local/hadoop" >>env.sh
-echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >>env.sh
-echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >>env.sh
-echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >>env.sh
-echo "export YARN_HOME=/usr/local/hadoop" >>env.sh
-echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >>env.sh
-
-# PyArrow==0.16.0 regression https://issues.apache.org/jira/browse/ARROW-7841
-# retrieves native library from $HADOOP_HOME directory instead of
-# `$HADOOP_HOME/lib/native`.
-# Fix: force search for `libhdfs.so` inside `$HADOOP_HOME/lib/native`.
-# Note: not needed for PyArrow==0.17.0.
-echo "export ARROW_LIBHDFS_DIR=/usr/local/hadoop/lib/native" >> env.sh
-
-echo "export JAVA_HOME=/usr/" >>env.sh
-echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >>env.sh
-
-source env.sh
-
-sudo bash -c 'echo "export JAVA_HOME=/usr/" >> /usr/local/hadoop/etc/hadoop/hadoop-env.sh'
-
-sudo cp scripts/ci/core-site.xml /usr/local/hadoop/etc/hadoop
-sudo cp scripts/ci/hdfs-site.xml /usr/local/hadoop/etc/hadoop
-
-hadoop fs -help
-pushd ~
-sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/
-sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/
-sudo chmod 777 -R /usr/local/hadoop
-hdfs namenode -format
-start-dfs.sh
-jps
-hadoop fs -ls /
-hadoop fs -ls hdfs://127.0.0.1/
-hadoop fs -mkdir hdfs://127.0.0.1/ololo
-popd

scripts/ci/install_oss.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-git clone https://github.com/iterative/oss-emulator.git
-sudo docker image build -t oss:1.0 oss-emulator
-sudo docker run --detach --restart always -p 8880:8880 --name oss-emulator oss:1.0
-echo "export OSS_ENDPOINT='localhost:8880'" >>env.sh
-echo "export OSS_ACCESS_KEY_ID='AccessKeyID'" >>env.sh
-echo "export OSS_ACCESS_KEY_SECRET='AccessKeySecret'" >>env.sh

scripts/ci/remove_hadoop.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-stop-dfs.sh
-sudo rm -rf /usr/local/hadoop
-sudo rm -rf /usr/local/hadoop-2.6.5
-sudo rm -rf /usr/local/hadoop-2.6.5.tar.gz
-rm env.sh

setup.cfg

@@ -17,7 +17,7 @@ count=true
 [isort]
 include_trailing_comma=true
 known_first_party=dvc,tests
-known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,moto,nanotime,networkx,packaging,paramiko,pathspec,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
+known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
 line_length=79
 force_grid_wrap=0
 use_parentheses=True

setup.py

@@ -68,7 +68,7 @@ install_requires = [
     "tqdm>=4.45.0,<5",
     "packaging>=19.0",
     "zc.lockfile>=1.2.1",
-    "flufl.lock>=3.2",
+    "flufl.lock>=3.2,<4",
     "win-unicode-console>=0.5; sys_platform == 'win32'",
     "pywin32>=225; sys_platform == 'win32'",
     "networkx>=2.1,<2.4",
@@ -78,7 +78,7 @@ install_requires = [
     "tabulate>=0.8.7",
     "pygtrie==2.3.2",
     "dpath>=2.0.1,<3",
-    "shtab>=1.0.3",
+    "shtab>=1.1.0,<2",
 ]
 
 
@@ -87,7 +87,7 @@ install_requires = [
 gs = ["google-cloud-storage==1.19.0"]
 gdrive = ["pydrive2>=1.4.14"]
 s3 = ["boto3>=1.9.201"]
-azure = ["azure-storage-blob==2.1.0", "azure-cli-core>=2.0.70"]
+azure = ["azure-storage-blob==2.1.0", "knack"]
 oss = ["oss2==2.6.1"]
 ssh = ["paramiko>=2.5.0"]
 hdfs = ["pyarrow>=0.17.0"]
@@ -103,6 +103,7 @@ tests_requirements = [
     "wheel>=0.31.1",
     # Test requirements:
     "pytest>=4.6.0",
+    "pytest-docker>=0.7.2",
     "pytest-timeout>=1.3.3",
     "pytest-cov>=2.6.1",
     "pytest-xdist>=1.26.1",
@@ -130,6 +131,8 @@ tests_requirements = [
     "pylint",
     "pylint-pytest",
     "pylint-plugin-utils",
+    "wget",
+    "filelock",
 ]
 
 if (sys.version_info) >= (3, 6):

tests/dir_helpers.py

@@ -221,6 +221,17 @@ class TmpDir(pathlib.Path):
         finally:
             self.scm.checkout(old)
 
+    def read_text(self, *args, **kwargs):  # pylint: disable=signature-differs
+        # NOTE: on windows we'll get PermissionError instead of
+        # IsADirectoryError when we try to `open` a directory, so we can't
+        # rely on exception flow control
+        if self.is_dir():
+            return {
+                path.name: path.read_text(*args, **kwargs)
+                for path in self.iterdir()
+            }
+        return super().read_text(*args, **kwargs)
+
 
 def _coerce_filenames(filenames):
     if isinstance(filenames, (str, bytes, pathlib.PurePath)):

tests/docker-compose.yml

@@ -0,0 +1,19 @@
+---
+version: '3.2'
+services:
+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite:3.3.0-preview
+    ports:
+      - "10000"
+  oss:
+    image: rkuprieiev/oss-emulator
+    ports:
+      - "8880"
+  hdfs:
+    image: rkuprieiev/docker-hdfs
+    ports:
+      - "8020"
+      # NOTE: having this port as dynamic one will require modifying
+      # `dfs.datanode.address` in `hdfs-site.xml` and probably something
+      # else, so using default one for now.
+      - "50010:50010"

tests/func/test_add.py

@@ -185,27 +185,97 @@ def test_add_file_in_dir(tmp_dir, dvc):
     assert stage.outs[0].def_path == "subdata"
 
 
-class TestAddExternalLocalFile(TestDvc):
-    def test(self):
-        from dvc.stage.exceptions import StageExternalOutputsError
+@pytest.mark.parametrize(
+    "workspace, hash_name, hash_value",
+    [
+        (
+            pytest.lazy_fixture("local_cloud"),
+            "md5",
+            "8c7dd922ad47494fc02c388e12c00eac",
+        ),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            "md5",
+            "8c7dd922ad47494fc02c388e12c00eac",
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+        (
+            pytest.lazy_fixture("s3"),
+            "etag",
+            "8c7dd922ad47494fc02c388e12c00eac",
+        ),
+        (pytest.lazy_fixture("gs"), "md5", "8c7dd922ad47494fc02c388e12c00eac"),
+        (
+            pytest.lazy_fixture("hdfs"),
+            "checksum",
+            "000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e",
+        ),
+    ],
+    indirect=["workspace"],
+)
+def test_add_external_file(tmp_dir, dvc, workspace, hash_name, hash_value):
+    from dvc.stage.exceptions import StageExternalOutputsError
 
-        dname = TestDvc.mkdtemp()
-        fname = os.path.join(dname, "foo")
-        shutil.copyfile(self.FOO, fname)
+    workspace.gen("file", "file")
 
-        with self.assertRaises(StageExternalOutputsError):
-            self.dvc.add(fname)
+    with pytest.raises(StageExternalOutputsError):
+        dvc.add(workspace.url)
 
-        stages = self.dvc.add(fname, external=True)
-        self.assertEqual(len(stages), 1)
-        stage = stages[0]
-        self.assertNotEqual(stage, None)
-        self.assertEqual(len(stage.deps), 0)
-        self.assertEqual(len(stage.outs), 1)
-        self.assertEqual(stage.relpath, "foo.dvc")
-        self.assertEqual(len(os.listdir(dname)), 1)
-        self.assertTrue(os.path.isfile(fname))
-        self.assertTrue(filecmp.cmp(fname, "foo", shallow=False))
+    dvc.add("remote://workspace/file")
+    assert (tmp_dir / "file.dvc").read_text() == (
+        "outs:\n"
+        f"- {hash_name}: {hash_value}\n"
+        "  path: remote://workspace/file\n"
+    )
+    assert (workspace / "file").read_text() == "file"
+    assert (
+        workspace / "cache" / hash_value[:2] / hash_value[2:]
+    ).read_text() == "file"
+
+    assert dvc.status() == {}
+
+
+@pytest.mark.parametrize(
+    "workspace, hash_name, hash_value",
+    [
+        (
+            pytest.lazy_fixture("local_cloud"),
+            "md5",
+            "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
+        ),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            "md5",
+            "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+        (
+            pytest.lazy_fixture("s3"),
+            "etag",
+            "ec602a6ba97b2dd07bd6d2cd89674a60.dir",
+        ),
+        (
+            pytest.lazy_fixture("gs"),
+            "md5",
+            "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
+        ),
+    ],
+    indirect=["workspace"],
+)
+def test_add_external_dir(tmp_dir, dvc, workspace, hash_name, hash_value):
+    workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})
+
+    dvc.add("remote://workspace/dir")
+    assert (tmp_dir / "dir.dvc").read_text() == (
+        "outs:\n"
+        f"- {hash_name}: {hash_value}\n"
+        "  path: remote://workspace/dir\n"
+    )
+    assert (workspace / "cache" / hash_value[:2] / hash_value[2:]).is_file()
 
 
 class TestAddLocalRemoteFile(TestDvc):

tests/func/test_api.py

@@ -19,14 +19,12 @@ cloud_names = [
     "http",
 ]
 clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names]
-all_clouds = [pytest.lazy_fixture("local")] + clouds
-remotes = [pytest.lazy_fixture(f"{cloud}_remote") for cloud in cloud_names]
-all_remotes = [pytest.lazy_fixture("local_remote")] + remotes
+all_clouds = [pytest.lazy_fixture("local_cloud")] + clouds
 
 # `lazy_fixture` is confusing pylint, pylint: disable=unused-argument
 
 
-@pytest.mark.parametrize("remote", remotes)
+@pytest.mark.parametrize("remote", clouds, indirect=True)
 def test_get_url(tmp_dir, dvc, remote):
     tmp_dir.dvc_gen("foo", "foo")
 
@@ -56,7 +54,7 @@ def test_get_url_requires_dvc(tmp_dir, scm):
         api.get_url("foo", repo=f"file://{tmp_dir}")
 
 
-@pytest.mark.parametrize("remote", all_remotes)
+@pytest.mark.parametrize("remote", all_clouds, indirect=True)
 def test_open(tmp_dir, dvc, remote):
     tmp_dir.dvc_gen("foo", "foo-text")
     dvc.push()
@@ -68,7 +66,22 @@ def test_open(tmp_dir, dvc, remote):
         assert fd.read() == "foo-text"
 
 
-@pytest.mark.parametrize("cloud", clouds)
+@pytest.mark.parametrize(
+    "cloud",
+    [
+        pytest.lazy_fixture(cloud)
+        for cloud in [
+            "real_s3",  # NOTE: moto's s3 fails in some tests
+            "gs",
+            "azure",
+            "gdrive",
+            "oss",
+            "ssh",
+            "hdfs",
+            "http",
+        ]
+    ],
+)
 def test_open_external(erepo_dir, cloud):
     erepo_dir.add_remote(config=cloud.config)
 
@@ -92,7 +105,7 @@ def test_open_external(erepo_dir, cloud):
     assert api.read("version", repo=repo_url, rev="branch") == "branchver"
 
 
-@pytest.mark.parametrize("remote", all_remotes)
+@pytest.mark.parametrize("remote", all_clouds, indirect=True)
 def test_open_granular(tmp_dir, dvc, remote):
     tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}})
     dvc.push()
@@ -104,7 +117,23 @@ def test_open_granular(tmp_dir, dvc, remote):
         assert fd.read() == "foo-text"
 
 
-@pytest.mark.parametrize("remote", all_remotes)
+@pytest.mark.parametrize(
+    "remote",
+    [
+        pytest.lazy_fixture(cloud)
+        for cloud in [
+            "real_s3",  # NOTE: moto's s3 fails in some tests
+            "gs",
+            "azure",
+            "gdrive",
+            "oss",
+            "ssh",
+            "hdfs",
+            "http",
+        ]
+    ],
+    indirect=True,
+)
 def test_missing(tmp_dir, dvc, remote):
     tmp_dir.dvc_gen("foo", "foo")
 

tests/func/test_data_cloud.py

@@ -14,10 +14,10 @@ from dvc.stage.exceptions import StageNotFound
 from dvc.utils.fs import remove
 from dvc.utils.yaml import dump_yaml, load_yaml
 
-from .test_api import all_remotes
+from .test_api import all_clouds
 
 
-@pytest.mark.parametrize("remote", all_remotes)
+@pytest.mark.parametrize("remote", all_clouds, indirect=True)
 def test_cloud(tmp_dir, dvc, remote):  # pylint:disable=unused-argument
     (stage,) = tmp_dir.dvc_gen("foo", "foo")
     out = stage.outs[0]
@@ -95,7 +95,7 @@ def test_cloud(tmp_dir, dvc, remote):  # pylint:disable=unused-argument
         assert status_dir == expected
 
 
-@pytest.mark.parametrize("remote", all_remotes)
+@pytest.mark.parametrize("remote", all_clouds, indirect=True)
 def test_cloud_cli(tmp_dir, dvc, remote):
     args = ["-v", "-j", "2"]
 

tests/func/test_gc.py

@@ -348,3 +348,45 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
     Dvcfile(dvc, PIPELINE_FILE).remove(force=True)
     dvc.gc(workspace=True, force=True)
     assert _count_files(dvc.cache.local.cache_dir) == 0
+
+
+@pytest.mark.parametrize(
+    "workspace",
+    [
+        pytest.lazy_fixture("local_cloud"),
+        pytest.lazy_fixture("s3"),
+        pytest.lazy_fixture("gs"),
+        pytest.lazy_fixture("hdfs"),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+    ],
+    indirect=True,
+)
+def test_gc_external_output(tmp_dir, dvc, workspace):
+    workspace.gen({"foo": "foo", "bar": "bar"})
+
+    (foo_stage,) = dvc.add("remote://workspace/foo")
+    (bar_stage,) = dvc.add("remote://workspace/bar")
+
+    foo_hash = foo_stage.outs[0].checksum
+    bar_hash = bar_stage.outs[0].checksum
+
+    assert (
+        workspace / "cache" / foo_hash[:2] / foo_hash[2:]
+    ).read_text() == "foo"
+    assert (
+        workspace / "cache" / bar_hash[:2] / bar_hash[2:]
+    ).read_text() == "bar"
+
+    (tmp_dir / "foo.dvc").unlink()
+
+    dvc.gc(workspace=True)
+
+    assert not (workspace / "cache" / foo_hash[:2] / foo_hash[2:]).exists()
+    assert (
+        workspace / "cache" / bar_hash[:2] / bar_hash[2:]
+    ).read_text() == "bar"

tests/func/test_get.py

@@ -10,7 +10,6 @@ from dvc.repo import Repo
 from dvc.repo.get import GetDVCFileError
 from dvc.system import System
 from dvc.utils.fs import makedirs
-from tests.utils import trees_equal
 
 
 def test_get_repo_file(tmp_dir, erepo_dir):
@@ -29,8 +28,7 @@ def test_get_repo_dir(tmp_dir, erepo_dir):
 
     Repo.get(os.fspath(erepo_dir), "dir", "dir_imported")
 
-    assert os.path.isdir("dir_imported")
-    trees_equal(erepo_dir / "dir", "dir_imported")
+    assert (tmp_dir / "dir_imported").read_text() == {"file": "contents"}
 
 
 @pytest.mark.parametrize(
@@ -44,7 +42,6 @@ def test_get_git_file(tmp_dir, erepo):
 
     Repo.get(os.fspath(erepo), src, dst)
 
-    assert (tmp_dir / dst).is_file()
     assert (tmp_dir / dst).read_text() == "hello"
 
 
@@ -61,8 +58,7 @@ def test_get_git_dir(tmp_dir, erepo):
 
     Repo.get(os.fspath(erepo), src, dst)
 
-    assert (tmp_dir / dst).is_dir()
-    trees_equal(erepo / src, tmp_dir / dst)
+    assert (tmp_dir / dst).read_text() == {"dir": {"file.txt": "hello"}}
 
 
 def test_cache_type_is_properly_overridden(tmp_dir, erepo_dir):

tests/func/test_ignore.py

@@ -173,3 +173,66 @@ def test_ignore_blank_line(tmp_dir, dvc):
     tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "foo\n\ndir/ignored")
 
     assert _files_set("dir", dvc.tree) == {"dir/other"}
+
+
+# It is not possible to re-include a file if a parent directory of
+# that file is excluded.
+# Git doesn’t list excluded directories for performance reasons,
+# so any patterns on contained files have no effect,
+# no matter where they are defined.
+@pytest.mark.parametrize(
+    "data_struct, pattern_list, result_set",
+    [
+        (
+            {"dir": {"subdir": {"not_ignore": "121"}}},
+            ["subdir/*", "!not_ignore"],
+            {"dir/subdir/not_ignore"},
+        ),
+        (
+            {"dir": {"subdir": {"should_ignore": "121"}}},
+            ["subdir", "!should_ignore"],
+            set(),
+        ),
+        (
+            {"dir": {"subdir": {"should_ignore": "121"}}},
+            ["subdir/", "!should_ignore"],
+            set(),
+        ),
+    ],
+)
+def test_ignore_file_in_parent_path(
+    tmp_dir, dvc, data_struct, pattern_list, result_set
+):
+    tmp_dir.gen(data_struct)
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "\n".join(pattern_list))
+    assert _files_set("dir", dvc.tree) == result_set
+
+
+# If there is a separator at the end of the pattern then the pattern
+# will only match directories,
+# otherwise the pattern can match both files and directories.
+# For example, a pattern doc/frotz/ matches doc/frotz directory,
+# but not a/doc/frotz directory;
+def test_ignore_sub_directory(tmp_dir, dvc):
+    tmp_dir.gen(
+        {
+            "dir": {
+                "doc": {"fortz": {"b": "b"}},
+                "a": {"doc": {"fortz": {"a": "a"}}},
+            }
+        }
+    )
+    tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "doc/fortz"}})
+    assert _files_set("dir", dvc.tree) == {
+        "dir/a/doc/fortz/a",
+        "dir/{}".format(DvcIgnore.DVCIGNORE_FILE),
+    }
+
+
+# however frotz/ matches frotz and a/frotz that is a directory
+def test_ignore_directory(tmp_dir, dvc):
+    tmp_dir.gen({"dir": {"fortz": {}, "a": {"fortz": {}}}})
+    tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "fortz"}})
+    assert _files_set("dir", dvc.tree) == {
+        "dir/{}".format(DvcIgnore.DVCIGNORE_FILE),
+    }

tests/func/test_import.py

@@ -11,7 +11,6 @@ from dvc.dvcfile import Dvcfile
 from dvc.exceptions import DownloadError, PathMissingError
 from dvc.system import System
 from dvc.utils.fs import makedirs, remove
-from tests.utils import trees_equal
 
 
 def test_import(tmp_dir, scm, dvc, erepo_dir):
@@ -73,8 +72,7 @@ def test_import_git_dir(tmp_dir, scm, dvc, git_dir, src_is_dvc):
 
     stage = dvc.imp(os.fspath(git_dir), "src", "dst")
 
-    assert (tmp_dir / "dst").is_dir()
-    trees_equal(git_dir / "src", tmp_dir / "dst")
+    assert (tmp_dir / "dst").read_text() == {"file.txt": "hello"}
     assert tmp_dir.scm.repo.git.check_ignore(os.fspath(tmp_dir / "dst"))
     assert stage.deps[0].def_repo == {
         "url": os.fspath(git_dir),
@@ -88,8 +86,7 @@ def test_import_dir(tmp_dir, scm, dvc, erepo_dir):
 
     stage = dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported")
 
-    assert os.path.isdir("dir_imported")
-    trees_equal(erepo_dir / "dir", "dir_imported")
+    assert (tmp_dir / "dir_imported").read_text() == {"foo": "foo content"}
     assert scm.repo.git.check_ignore("dir_imported")
     assert stage.deps[0].def_repo == {
         "url": os.fspath(erepo_dir),
@@ -222,8 +219,7 @@ def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir):
 
     dvc.pull(["dir_imported.dvc"])
 
-    assert os.path.isdir("dir_imported")
-    trees_equal(erepo_dir / "dir", "dir_imported")
+    assert (tmp_dir / "dir_imported").read_text() == {"foo": "foo content"}
 
 
 def test_download_error_pulling_imported_stage(tmp_dir, dvc, erepo_dir):

tests/func/test_import_url.py

@@ -112,3 +112,56 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
     dvc.imp_url(src, ".", no_exec=True)
     dst = tmp_dir / "file"
     assert not dst.exists()
+
+
+@pytest.mark.parametrize(
+    "workspace",
+    [
+        pytest.lazy_fixture("local_cloud"),
+        pytest.lazy_fixture("s3"),
+        pytest.lazy_fixture("gs"),
+        pytest.lazy_fixture("hdfs"),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+        pytest.lazy_fixture("http"),
+    ],
+    indirect=True,
+)
+def test_import_url(tmp_dir, dvc, workspace):
+    workspace.gen("file", "file")
+    assert not (tmp_dir / "file").exists()  # sanity check
+    dvc.imp_url("remote://workspace/file")
+    assert (tmp_dir / "file").read_text() == "file"
+
+    assert dvc.status() == {}
+
+
+@pytest.mark.parametrize(
+    "workspace",
+    [
+        pytest.lazy_fixture("local_cloud"),
+        pytest.lazy_fixture("s3"),
+        pytest.lazy_fixture("gs"),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+    ],
+    indirect=True,
+)
+def test_import_url_dir(tmp_dir, dvc, workspace):
+    workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})
+    assert not (tmp_dir / "dir").exists()  # sanity check
+    dvc.imp_url("remote://workspace/dir")
+    assert set(os.listdir(tmp_dir / "dir")) == {"file", "subdir"}
+    assert (tmp_dir / "dir" / "file").read_text() == "file"
+    assert list(os.listdir(tmp_dir / "dir" / "subdir")) == ["subfile"]
+    assert (tmp_dir / "dir" / "subdir" / "subfile").read_text() == "subfile"
+
+    assert dvc.status() == {}

tests/func/test_repro.py

@@ -1,20 +1,10 @@
 import filecmp
-import getpass
 import os
-import posixpath
 import re
 import shutil
-import uuid
 from pathlib import Path
-from subprocess import PIPE, Popen
-from unittest import SkipTest
-from urllib.parse import urljoin
 
-import boto3
-import paramiko
 import pytest
-from flaky.flaky_decorator import flaky
-from google.cloud import storage as gc
 from mock import patch
 
 from dvc.dvcfile import DVC_FILE, Dvcfile
@@ -25,9 +15,7 @@ from dvc.exceptions import (
 )
 from dvc.main import main
 from dvc.output.base import BaseOutput
-from dvc.path_info import URLInfo
 from dvc.remote.local import LocalRemoteTree
-from dvc.repo import Repo as DvcRepo
 from dvc.stage import Stage
 from dvc.stage.exceptions import StageFileDoesNotExistError
 from dvc.system import System
@@ -35,17 +23,6 @@ from dvc.utils import file_md5, relpath
 from dvc.utils.fs import remove
 from dvc.utils.yaml import dump_yaml, load_yaml
 from tests.basic_env import TestDvc
-from tests.remotes import (
-    GCP,
-    HDFS,
-    S3,
-    SSH,
-    TEST_AWS_REPO_BUCKET,
-    TEST_GCP_REPO_BUCKET,
-    Local,
-    SSHMocked,
-)
-from tests.utils.httpd import ContentMD5Handler, StaticFileServer
 
 
 class SingleStageRun:
@@ -841,430 +818,6 @@ class TestCmdReproChdir(TestDvc):
         self.assertTrue(filecmp.cmp(foo, bar, shallow=False))
 
 
-class ReproExternalTestMixin(SingleStageRun, TestDvc):
-    cache_type = None
-
-    @staticmethod
-    def should_test():
-        return False
-
-    @property
-    def cache_scheme(self):
-        return self.scheme
-
-    @property
-    def scheme(self):
-        return None
-
-    @property
-    def scheme_sep(self):
-        return "://"
-
-    @property
-    def sep(self):
-        return "/"
-
-    def check_already_cached(self, stage):
-        stage.outs[0].remove()
-
-        patch_download = patch.object(
-            stage.deps[0], "download", wraps=stage.deps[0].download
-        )
-
-        patch_checkout = patch.object(
-            stage.outs[0], "checkout", wraps=stage.outs[0].checkout
-        )
-
-        from dvc.stage.run import cmd_run
-
-        patch_run = patch("dvc.stage.run.cmd_run", wraps=cmd_run)
-
-        with self.dvc.lock, self.dvc.state:
-            with patch_download as mock_download:
-                with patch_checkout as mock_checkout:
-                    with patch_run as mock_run:
-                        stage.frozen = False
-                        stage.run()
-                        stage.frozen = True
-
-                        mock_run.assert_not_called()
-                        mock_download.assert_not_called()
-                        mock_checkout.assert_called_once()
-
-    @patch("dvc.prompt.confirm", return_value=True)
-    def test(self, _mock_prompt):
-        if not self.should_test():
-            raise SkipTest(f"Test {self.__class__.__name__} is disabled")
-
-        cache = (
-            self.scheme
-            + self.scheme_sep
-            + self.bucket
-            + self.sep
-            + str(uuid.uuid4())
-        )
-
-        ret = main(["config", "cache." + self.cache_scheme, "myrepo"])
-        self.assertEqual(ret, 0)
-        ret = main(["remote", "add", "myrepo", cache])
-        self.assertEqual(ret, 0)
-        if self.cache_type:
-            ret = main(["remote", "modify", "myrepo", "type", self.cache_type])
-            self.assertEqual(ret, 0)
-
-        remote_name = "myremote"
-        remote_key = str(uuid.uuid4())
-        remote = (
-            self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key
-        )
-
-        ret = main(["remote", "add", remote_name, remote])
-        self.assertEqual(ret, 0)
-        if self.cache_type:
-            ret = main(
-                ["remote", "modify", remote_name, "type", self.cache_type]
-            )
-            self.assertEqual(ret, 0)
-
-        self.dvc = DvcRepo(".")
-
-        foo_key = remote_key + self.sep + self.FOO
-        bar_key = remote_key + self.sep + self.BAR
-
-        foo_path = (
-            self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key
-        )
-        bar_path = (
-            self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key
-        )
-
-        # Using both plain and remote notation
-        out_foo_path = "remote://" + remote_name + "/" + self.FOO
-        out_bar_path = bar_path
-
-        self.write(self.bucket, foo_key, self.FOO_CONTENTS)
-
-        import_stage = self.dvc.imp_url(out_foo_path, "import")
-
-        self.assertTrue(os.path.exists("import"))
-        self.assertTrue(filecmp.cmp("import", self.FOO, shallow=False))
-        self.assertEqual(self.dvc.status([import_stage.path]), {})
-        self.check_already_cached(import_stage)
-
-        import_remote_stage = self.dvc.imp_url(
-            out_foo_path, out_foo_path + "_imported"
-        )
-        self.assertEqual(self.dvc.status([import_remote_stage.path]), {})
-
-        cmd_stage = self._run(
-            outs=[out_bar_path],
-            deps=[out_foo_path],
-            cmd=self.cmd(foo_path, bar_path),
-            name="external-base",
-            external=True,
-        )
-
-        self.assertEqual(self.dvc.status([cmd_stage.addressing]), {})
-        self.assertEqual(self.dvc.status(), {})
-        self.check_already_cached(cmd_stage)
-
-        self.write(self.bucket, foo_key, self.BAR_CONTENTS)
-
-        self.assertNotEqual(self.dvc.status(), {})
-
-        self.dvc.update([import_stage.path])
-        self.assertTrue(os.path.exists("import"))
-        self.assertTrue(filecmp.cmp("import", self.BAR, shallow=False))
-        self.assertEqual(self.dvc.status([import_stage.path]), {})
-
-        self.dvc.update([import_remote_stage.path])
-        self.assertEqual(self.dvc.status([import_remote_stage.path]), {})
-
-        stages = self.dvc.reproduce(cmd_stage.addressing)
-        self.assertEqual(len(stages), 1)
-        self.assertEqual(self.dvc.status([cmd_stage.addressing]), {})
-
-        self.assertEqual(self.dvc.status(), {})
-        self.dvc.gc(workspace=True)
-        self.assertEqual(self.dvc.status(), {})
-
-        with self.dvc.lock:
-            cmd_stage.remove_outs(force=True)
-        self.assertNotEqual(self.dvc.status([cmd_stage.addressing]), {})
-
-        self.dvc.checkout([cmd_stage.path], force=True)
-        self.assertEqual(self.dvc.status([cmd_stage.addressing]), {})
-
-
-@pytest.mark.skipif(os.name == "nt", reason="temporarily disabled on windows")
-@flaky(max_runs=3, min_passes=1)
-class TestReproExternalS3(S3, ReproExternalTestMixin):
-    @property
-    def scheme(self):
-        return "s3"
-
-    @property
-    def bucket(self):
-        return TEST_AWS_REPO_BUCKET
-
-    def cmd(self, i, o):
-        return f"aws s3 cp {i} {o}"
-
-    def write(self, bucket, key, body):
-        s3 = boto3.client("s3")
-        s3.put_object(Bucket=bucket, Key=key, Body=body)
-
-
-class TestReproExternalGS(GCP, ReproExternalTestMixin):
-    @property
-    def scheme(self):
-        return "gs"
-
-    @property
-    def bucket(self):
-        return TEST_GCP_REPO_BUCKET
-
-    def cmd(self, i, o):
-        return f"gsutil cp {i} {o}"
-
-    def write(self, bucket, key, body):
-        client = gc.Client()
-        bucket = client.bucket(bucket)
-        bucket.blob(key).upload_from_string(body)
-
-
-class TestReproExternalHDFS(HDFS, ReproExternalTestMixin):
-    @property
-    def scheme(self):
-        return "hdfs"
-
-    @property
-    def bucket(self):
-        return f"{getpass.getuser()}@127.0.0.1"
-
-    def cmd(self, i, o):
-        return f"hadoop fs -cp {i} {o}"
-
-    def write(self, bucket, key, body):
-        url = self.scheme + "://" + bucket + "/" + key
-        p = Popen(
-            f"hadoop fs -rm -f {url}",
-            shell=True,
-            executable=os.getenv("SHELL"),
-            stdin=PIPE,
-            stdout=PIPE,
-            stderr=PIPE,
-        )
-        p.communicate()
-
-        p = Popen(
-            "hadoop fs -mkdir -p {}".format(posixpath.dirname(url)),
-            shell=True,
-            executable=os.getenv("SHELL"),
-            stdin=PIPE,
-            stdout=PIPE,
-            stderr=PIPE,
-        )
-        out, err = p.communicate()
-        if p.returncode != 0:
-            print(out)
-            print(err)
-        self.assertEqual(p.returncode, 0)
-
-        with open("tmp", "w+") as fd:
-            fd.write(body)
-
-        p = Popen(
-            "hadoop fs -copyFromLocal {} {}".format("tmp", url),
-            shell=True,
-            executable=os.getenv("SHELL"),
-            stdin=PIPE,
-            stdout=PIPE,
-            stderr=PIPE,
-        )
-        out, err = p.communicate()
-        if p.returncode != 0:
-            print(out)
-            print(err)
-        self.assertEqual(p.returncode, 0)
-
-
-@flaky(max_runs=5, min_passes=1)
-class TestReproExternalSSH(SSH, ReproExternalTestMixin):
-    _dir = None
-    cache_type = "copy"
-
-    @property
-    def scheme(self):
-        return "ssh"
-
-    @property
-    def bucket(self):
-        if not self._dir:
-            self._dir = self.mkdtemp()
-        return f"{getpass.getuser()}@127.0.0.1:{self._dir}"
-
-    def cmd(self, i, o):
-        prefix = "ssh://"
-        assert i.startswith(prefix) and o.startswith(prefix)
-        i = i[len(prefix) :]
-        o = o[len(prefix) :]
-        return f"scp {i} {o}"
-
-    def write(self, _, key, body):
-        path = posixpath.join(self._dir, key)
-
-        ssh = None
-        sftp = None
-        try:
-            ssh = paramiko.SSHClient()
-            ssh.load_system_host_keys()
-            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-            ssh.connect("127.0.0.1")
-
-            sftp = ssh.open_sftp()
-            try:
-                sftp.stat(path)
-                sftp.remove(path)
-            except OSError:
-                pass
-
-            _, stdout, _ = ssh.exec_command(f"mkdir -p $(dirname {path})")
-            self.assertEqual(stdout.channel.recv_exit_status(), 0)
-
-            with sftp.open(path, "w+") as fobj:
-                fobj.write(body)
-        finally:
-            if sftp:
-                sftp.close()
-            if ssh:
-                ssh.close()
-
-
-class TestReproExternalLOCAL(Local, ReproExternalTestMixin):
-    cache_type = "hardlink"
-
-    def setUp(self):
-        super().setUp()
-        self.tmpdir = self.mkdtemp()
-        ret = main(["config", "cache.type", "hardlink"])
-        self.assertEqual(ret, 0)
-        self.dvc = DvcRepo(".")
-
-    @property
-    def cache_scheme(self):
-        return "local"
-
-    @property
-    def scheme(self):
-        return ""
-
-    @property
-    def scheme_sep(self):
-        return ""
-
-    @property
-    def sep(self):
-        return os.sep
-
-    @property
-    def bucket(self):
-        return self.tmpdir
-
-    def cmd(self, i, o):
-        if os.name == "nt":
-            return f"copy {i} {o}"
-        return f"cp {i} {o}"
-
-    def write(self, bucket, key, body):
-        path = os.path.join(bucket, key)
-        dname = os.path.dirname(path)
-
-        if not os.path.exists(dname):
-            os.makedirs(dname)
-
-        with open(path, "w+") as fd:
-            fd.write(body)
-
-
-class TestReproExternalHTTP(ReproExternalTestMixin):
-    _external_cache_id = None
-
-    @staticmethod
-    def get_remote(port):
-        return f"http://localhost:{port}/"
-
-    @property
-    def local_cache(self):
-        return os.path.join(self.dvc.dvc_dir, "cache")
-
-    def test(self):  # pylint: disable=arguments-differ
-        # Import
-        with StaticFileServer() as httpd:
-            import_url = urljoin(self.get_remote(httpd.server_port), self.FOO)
-            import_output = "imported_file"
-            import_stage = self.dvc.imp_url(import_url, import_output)
-
-        self.assertTrue(os.path.exists(import_output))
-        self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
-
-        self.dvc.remove("imported_file.dvc", outs=True)
-
-        with StaticFileServer(handler_class=ContentMD5Handler) as httpd:
-            import_url = urljoin(self.get_remote(httpd.server_port), self.FOO)
-            import_output = "imported_file"
-            import_stage = self.dvc.imp_url(import_url, import_output)
-            assert import_stage.repo == self.dvc
-
-        self.assertTrue(os.path.exists(import_output))
-        self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
-
-        # Run --deps
-        with StaticFileServer() as httpd:
-            remote = self.get_remote(httpd.server_port)
-
-            cache_id = str(uuid.uuid4())
-            cache = urljoin(remote, cache_id)
-
-            ret1 = main(["remote", "add", "mycache", cache])
-            ret2 = main(["remote", "add", "myremote", remote])
-            self.assertEqual(ret1, 0)
-            self.assertEqual(ret2, 0)
-
-            self.dvc = import_stage.repo = DvcRepo(".")
-
-            run_dependency = urljoin(remote, self.BAR)
-            run_output = "remote_file"
-            cmd = f'open("{run_output}", "w+")'
-
-            with open("create-output.py", "w") as fd:
-                fd.write(cmd)
-
-            run_stage = self._run(
-                deps=[run_dependency],
-                outs=[run_output],
-                cmd="python create-output.py",
-                name="http_run",
-            )
-            self.assertTrue(run_stage is not None)
-
-            self.assertTrue(os.path.exists(run_output))
-
-            # Pull
-            with self.dvc.lock:
-                self.assertEqual(import_stage.repo.lock.is_locked, True)
-                self.assertEqual(self.dvc.lock.is_locked, True)
-                import_stage.remove_outs(force=True)
-            self.assertFalse(os.path.exists(import_output))
-
-            shutil.move(self.local_cache, cache_id)
-            self.assertFalse(os.path.exists(self.local_cache))
-
-            self.dvc.pull([import_stage.path], remote="mycache")
-
-            self.assertTrue(os.path.exists(import_output))
-
-
 class TestReproShell(TestDvc):
     def test(self):
         if os.name == "nt":
@@ -1728,56 +1281,6 @@ def test_downstream(dvc):
     assert evaluation[4].relpath == "E.dvc"
 
 
-@pytest.mark.skipif(
-    os.name == "nt",
-    reason="external output scenario is not supported on Windows",
-)
-def test_ssh_dir_out(tmp_dir, dvc, ssh_server):
-    from tests.remotes.ssh import TEST_SSH_USER, TEST_SSH_KEY_PATH
-
-    tmp_dir.gen({"foo": "foo content"})
-
-    # Set up remote and cache
-    user = TEST_SSH_USER
-    port = ssh_server.port
-    keyfile = TEST_SSH_KEY_PATH
-
-    remote_url = SSHMocked.get_url(user, port)
-    assert main(["remote", "add", "upstream", remote_url]) == 0
-    assert main(["remote", "modify", "upstream", "keyfile", keyfile]) == 0
-
-    cache_url = SSHMocked.get_url(user, port)
-    assert main(["remote", "add", "sshcache", cache_url]) == 0
-    assert main(["config", "cache.ssh", "sshcache"]) == 0
-    assert main(["remote", "modify", "sshcache", "keyfile", keyfile]) == 0
-
-    # Recreating to reread configs
-    repo = DvcRepo(dvc.root_dir)
-
-    # To avoid "WARNING: UNPROTECTED PRIVATE KEY FILE" from ssh
-    os.chmod(keyfile, 0o600)
-
-    (tmp_dir / "script.py").write_text(
-        "import sys, pathlib\n"
-        "path = pathlib.Path(sys.argv[1])\n"
-        "dir_out = path / 'dir-out'\n"
-        "dir_out.mkdir()\n"
-        "(dir_out / '1.txt').write_text('1')\n"
-        "(dir_out / '2.txt').write_text('2')\n"
-    )
-
-    url_info = URLInfo(remote_url)
-    repo.run(
-        cmd="python {} {}".format(tmp_dir / "script.py", url_info.path),
-        single_stage=True,
-        outs=["remote://upstream/dir-out"],
-        deps=["foo"],  # add a fake dep to not consider this a callback
-    )
-
-    repo.reproduce("dir-out.dvc")
-    repo.reproduce("dir-out.dvc", force=True)
-
-
 def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker):
     from dvc.dvcfile import SingleStageFile
 

tests/func/test_repro_multistage.py

@@ -159,42 +159,6 @@ class TestReproChangedDirDataMultiStage(
     pass
 
 
-class TestReproExternalS3MultiStage(
-    MultiStageRun, test_repro.TestReproExternalS3
-):
-    pass
-
-
-class TestReproExternalGSMultiStage(
-    MultiStageRun, test_repro.TestReproExternalGS
-):
-    pass
-
-
-class TestReproExternalHDFSMultiStage(
-    MultiStageRun, test_repro.TestReproExternalHDFS
-):
-    pass
-
-
-class TestReproExternalHTTPMultiStage(
-    MultiStageRun, test_repro.TestReproExternalHTTP
-):
-    pass
-
-
-class TestReproExternalLOCALMultiStage(
-    MultiStageRun, test_repro.TestReproExternalLOCAL
-):
-    pass
-
-
-class TestReproExternalSSHMultiStage(
-    MultiStageRun, test_repro.TestReproExternalSSH
-):
-    pass
-
-
 def test_non_existing_stage_name(tmp_dir, dvc, run_copy):
     from dvc.exceptions import DvcException
 

tests/func/test_run_multistage.py

@@ -244,7 +244,7 @@ def test_run_params_default(tmp_dir, dvc):
         params=["nested.nested1.nested2"],
         cmd="cat params.yaml",
     )
-    isinstance(stage.deps[0], ParamsDependency)
+    assert isinstance(stage.deps[0], ParamsDependency)
     assert stage.deps[0].params == ["nested.nested1.nested2"]
 
     lockfile = stage.dvcfile._lockfile
@@ -361,3 +361,89 @@ def test_run_overwrite_preserves_meta_and_comment(tmp_dir, dvc, run_copy):
     assert (tmp_dir / PIPELINE_FILE).read_text() == text.format(
         src="foo1", dest="bar1"
     )
+
+
+@pytest.mark.parametrize(
+    "workspace, hash_name, foo_hash, bar_hash",
+    [
+        (
+            pytest.lazy_fixture("local_cloud"),
+            "md5",
+            "acbd18db4cc2f85cedef654fccc4a4d8",
+            "37b51d194a7513e45b56f6524f2d51f2",
+        ),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            "md5",
+            "acbd18db4cc2f85cedef654fccc4a4d8",
+            "37b51d194a7513e45b56f6524f2d51f2",
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+        (
+            pytest.lazy_fixture("s3"),
+            "etag",
+            "acbd18db4cc2f85cedef654fccc4a4d8",
+            "37b51d194a7513e45b56f6524f2d51f2",
+        ),
+        (
+            pytest.lazy_fixture("gs"),
+            "md5",
+            "acbd18db4cc2f85cedef654fccc4a4d8",
+            "37b51d194a7513e45b56f6524f2d51f2",
+        ),
+        (
+            pytest.lazy_fixture("hdfs"),
+            "checksum",
+            "0000020000000000000000003dba826b9be9c6a8e2f8310a770555c4",
+            "00000200000000000000000075433c81259d3c38e364b348af52e84d",
+        ),
+    ],
+    indirect=["workspace"],
+)
+def test_run_external_outputs(
+    tmp_dir, dvc, workspace, hash_name, foo_hash, bar_hash
+):
+    workspace.gen("foo", "foo")
+    dvc.run(
+        name="mystage",
+        cmd="mycmd",
+        deps=["remote://workspace/foo"],
+        outs=["remote://workspace/bar"],
+        no_exec=True,
+    )
+
+    dvc_yaml = (
+        "stages:\n"
+        "  mystage:\n"
+        "    cmd: mycmd\n"
+        "    deps:\n"
+        "    - remote://workspace/foo\n"
+        "    outs:\n"
+        "    - remote://workspace/bar\n"
+    )
+
+    assert (tmp_dir / "dvc.yaml").read_text() == dvc_yaml
+    assert not (tmp_dir / "dvc.lock").exists()
+
+    workspace.gen("bar", "bar")
+    dvc.commit("dvc.yaml", force=True)
+
+    assert (tmp_dir / "dvc.yaml").read_text() == dvc_yaml
+    assert (tmp_dir / "dvc.lock").read_text() == (
+        "mystage:\n"
+        "  cmd: mycmd\n"
+        "  deps:\n"
+        "  - path: remote://workspace/foo\n"
+        f"    {hash_name}: {foo_hash}\n"
+        "  outs:\n"
+        "  - path: remote://workspace/bar\n"
+        f"    {hash_name}: {bar_hash}\n"
+    )
+
+    assert (workspace / "foo").read_text() == "foo"
+    assert (workspace / "bar").read_text() == "bar"
+    assert (
+        workspace / "cache" / bar_hash[:2] / bar_hash[2:]
+    ).read_text() == "bar"

tests/func/test_update.py

@@ -10,26 +10,69 @@ def test_update_import(tmp_dir, dvc, erepo_dir, cached):
     gen = erepo_dir.dvc_gen if cached else erepo_dir.scm_gen
 
     with erepo_dir.branch("branch", new=True), erepo_dir.chdir():
-        gen("version", "branch", "add version file")
+        gen(
+            {
+                "version": "branch",
+                "dir": {"version": "branch", "subdir": {"file": "file"}},
+            },
+            commit="add version file",
+        )
         old_rev = erepo_dir.scm.get_rev()
 
     stage = dvc.imp(os.fspath(erepo_dir), "version", "version", rev="branch")
+    dir_stage = dvc.imp(os.fspath(erepo_dir), "dir", "dir", rev="branch")
+    assert dvc.status() == {}
 
     assert (tmp_dir / "version").read_text() == "branch"
+    assert (tmp_dir / "dir").read_text() == {
+        "version": "branch",
+        "subdir": {"file": "file"},
+    }
     assert stage.deps[0].def_repo["rev_lock"] == old_rev
+    assert dir_stage.deps[0].def_repo["rev_lock"] == old_rev
 
     # Update version file
     with erepo_dir.branch("branch", new=False), erepo_dir.chdir():
-        gen("version", "updated", "update version content")
+        gen(
+            {
+                "version": "updated",
+                "dir": {"version": "updated", "subdir": {"file": "file"}},
+            },
+            commit="update version content",
+        )
         new_rev = erepo_dir.scm.get_rev()
 
     assert old_rev != new_rev
 
-    dvc.update([stage.path])
+    assert dvc.status() == {
+        "dir.dvc": [
+            {
+                "changed deps": {
+                    f"dir ({os.fspath(erepo_dir)})": "update available"
+                }
+            }
+        ],
+        "version.dvc": [
+            {
+                "changed deps": {
+                    f"version ({os.fspath(erepo_dir)})": "update available"
+                }
+            }
+        ],
+    }
+
+    (stage,) = dvc.update(stage.path)
+    (dir_stage,) = dvc.update(dir_stage.path)
+    assert dvc.status() == {}
+
     assert (tmp_dir / "version").read_text() == "updated"
+    assert (tmp_dir / "dir").read_text() == {
+        "version": "updated",
+        "subdir": {"file": "file"},
+    }
 
-    stage = Dvcfile(dvc, stage.path).stage
     assert stage.deps[0].def_repo["rev_lock"] == new_rev
+    assert dir_stage.deps[0].def_repo["rev_lock"] == new_rev
 
 
 def test_update_import_after_remote_updates_to_dvc(tmp_dir, dvc, erepo_dir):
@@ -111,19 +154,33 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
     assert dvc.status([stage.path]) == {}
 
 
-def test_update_import_url(tmp_dir, dvc, tmp_path_factory):
-    import_src = tmp_path_factory.mktemp("import_url_source")
-    src = import_src / "file"
-    src.write_text("file content")
+@pytest.mark.parametrize(
+    "workspace",
+    [
+        pytest.lazy_fixture("local_cloud"),
+        pytest.lazy_fixture("s3"),
+        pytest.lazy_fixture("gs"),
+        pytest.lazy_fixture("hdfs"),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+    ],
+    indirect=True,
+)
+def test_update_import_url(tmp_dir, dvc, workspace):
+    workspace.gen("file", "file content")
 
     dst = tmp_dir / "imported_file"
-    stage = dvc.imp_url(os.fspath(src), os.fspath(dst))
+    stage = dvc.imp_url("remote://workspace/file", os.fspath(dst))
 
     assert dst.is_file()
     assert dst.read_text() == "file content"
 
     # update data
-    src.write_text("updated file content")
+    workspace.gen("file", "updated file content")
 
     assert dvc.status([stage.path]) == {}
     dvc.update([stage.path])

tests/remotes/__init__.py

@@ -1,3 +1,14 @@
+import subprocess
+
+import pytest
+
+from .azure import Azure, azure, azure_server  # noqa: F401
+from .hdfs import HDFS, hadoop, hdfs, hdfs_server  # noqa: F401
+from .http import HTTP, http, http_server  # noqa: F401
+from .local import Local, local_cloud, local_remote  # noqa: F401
+from .oss import OSS, TEST_OSS_REPO_BUCKET, oss, oss_server  # noqa: F401
+from .s3 import S3, TEST_AWS_REPO_BUCKET, real_s3, s3  # noqa: F401
+
 TEST_REMOTE = "upstream"
 TEST_CONFIG = {
     "cache": {},
@@ -5,31 +16,90 @@ TEST_CONFIG = {
     "remote": {TEST_REMOTE: {"url": ""}},
 }
 
-from .azure import Azure, azure, azure_remote  # noqa: F401
-from .hdfs import HDFS, hdfs, hdfs_remote  # noqa: F401
-from .http import HTTP, http, http_remote, http_server  # noqa: F401
-from .local import Local, local_cloud, local_remote  # noqa: F401
-from .oss import OSS, TEST_OSS_REPO_BUCKET, oss, oss_remote  # noqa: F401
-from .s3 import S3, TEST_AWS_REPO_BUCKET, S3Mocked, s3, s3_remote  # noqa: F401
 
 from .gdrive import (  # noqa: F401; noqa: F401
     TEST_GDRIVE_REPO_BUCKET,
     GDrive,
     gdrive,
-    gdrive_remote,
 )
 from .gs import (  # noqa: F401; noqa: F401
     GCP,
     TEST_GCP_CREDS_FILE,
     TEST_GCP_REPO_BUCKET,
     gs,
-    gs_remote,
 )
 from .ssh import (  # noqa: F401; noqa: F401
     SSH,
     SSHMocked,
     ssh,
     ssh_connection,
-    ssh_remote,
     ssh_server,
 )
+
+
+@pytest.fixture(scope="session")
+def docker_compose():
+    try:
+        subprocess.check_output("docker-compose version", shell=True)
+    except (subprocess.CalledProcessError, OSError):
+        pytest.skip("no docker-compose installed")
+
+
+@pytest.fixture(scope="session")
+def docker_compose_project_name():
+    return "pytest-dvc-test"
+
+
+@pytest.fixture(scope="session")
+def docker_services(
+    docker_compose_file, docker_compose_project_name, tmp_path_factory
+):
+    # overriding `docker_services` fixture from `pytest_docker` plugin to
+    # only launch docker images once.
+
+    from filelock import FileLock
+    from pytest_docker.plugin import DockerComposeExecutor, Services
+
+    executor = DockerComposeExecutor(
+        docker_compose_file, docker_compose_project_name,
+    )
+
+    # making sure we don't accidentally launch docker-compose in parallel,
+    # as it might result in network conflicts. Inspired by:
+    # https://github.com/pytest-dev/pytest-xdist#making-session-scoped-fixtures-execute-only-once
+    lockfile = tmp_path_factory.getbasetemp().parent / "docker-compose.lock"
+    with FileLock(str(lockfile)):
+        executor.execute("up --build -d")
+
+    return Services(executor)
+
+
+@pytest.fixture
+def remote(tmp_dir, dvc, request):
+    cloud = request.param
+    assert cloud
+    tmp_dir.add_remote(config=cloud.config)
+    yield cloud
+
+
+@pytest.fixture
+def workspace(tmp_dir, dvc, request):
+    from dvc.cache import Cache
+
+    cloud = request.param
+
+    assert cloud
+
+    tmp_dir.add_remote(name="workspace", config=cloud.config, default=False)
+    tmp_dir.add_remote(
+        name="cache", url="remote://workspace/cache", default=False
+    )
+
+    scheme = getattr(cloud, "scheme", "local")
+    if scheme != "http":
+        with dvc.config.edit() as conf:
+            conf["cache"][scheme] = "cache"
+
+        dvc.cache = Cache(dvc)
+
+    return cloud

tests/remotes/azure.py

@@ -1,39 +1,61 @@
-import os
+# pylint:disable=abstract-method
+
 import uuid
 
 import pytest
 
-from dvc.utils import env2bool
+from dvc.path_info import CloudURLInfo
 
 from .base import Base
 
+TEST_AZURE_CONTAINER = "tests"
+TEST_AZURE_CONNECTION_STRING = (
+    "DefaultEndpointsProtocol=http;"
+    "AccountName=devstoreaccount1;"
+    "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSR"
+    "Z6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;"
+    "BlobEndpoint=http://127.0.0.1:{port}/devstoreaccount1;"
+)
 
-class Azure(Base):
-    @staticmethod
-    def should_test():
-        do_test = env2bool("DVC_TEST_AZURE", undefined=None)
-        if do_test is not None:
-            return do_test
 
-        return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv(
-            "AZURE_STORAGE_CONNECTION_STRING"
-        )
+class Azure(Base, CloudURLInfo):
+    pass
 
-    @staticmethod
-    def get_url():
-        container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
-        assert container_name is not None
-        return "azure://{}/{}".format(container_name, str(uuid.uuid4()))
+
+@pytest.fixture(scope="session")
+def azure_server(docker_compose, docker_services):
+    from azure.storage.blob import (  # pylint: disable=no-name-in-module
+        BlockBlobService,
+    )
+    from azure.common import (  # pylint: disable=no-name-in-module
+        AzureException,
+    )
+
+    port = docker_services.port_for("azurite", 10000)
+    connection_string = TEST_AZURE_CONNECTION_STRING.format(port=port)
+
+    def _check():
+        try:
+            BlockBlobService(
+                connection_string=connection_string,
+            ).list_containers()
+            return True
+        except AzureException:
+            return False
+
+    docker_services.wait_until_responsive(
+        timeout=60.0, pause=0.1, check=_check
+    )
+
+    return connection_string
 
 
 @pytest.fixture
-def azure():
-    if not Azure.should_test():
-        pytest.skip("no azure running")
-    yield Azure()
-
-
-@pytest.fixture
-def azure_remote(tmp_dir, dvc, azure):
-    tmp_dir.add_remote(config=azure.config)
-    yield azure
+def azure(azure_server):
+    url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}"
+    ret = Azure(url)
+    ret.config = {
+        "url": url,
+        "connection_string": azure_server,
+    }
+    return ret

tests/remotes/base.py

@@ -1,7 +1,61 @@
+import pathlib
+
 from funcy import cached_property
 
+from dvc.path_info import URLInfo
+
+
+class Base(URLInfo):
+    def is_file(self):
+        raise NotImplementedError
+
+    def is_dir(self):
+        raise NotImplementedError
+
+    def exists(self):
+        raise NotImplementedError
+
+    def mkdir(self, mode=0o777, parents=False, exist_ok=False):
+        raise NotImplementedError
+
+    def write_text(self, contents, encoding=None, errors=None):
+        raise NotImplementedError
+
+    def write_bytes(self, contents):
+        raise NotImplementedError
+
+    def read_text(self, encoding=None, errors=None):
+        raise NotImplementedError
+
+    def read_bytes(self):
+        raise NotImplementedError
+
+    def _gen(self, struct, prefix=None):
+        for name, contents in struct.items():
+            path = (prefix or self) / name
+
+            if isinstance(contents, dict):
+                if not contents:
+                    path.mkdir(parents=True)
+                else:
+                    self._gen(contents, prefix=path)
+            else:
+                path.parent.mkdir(parents=True)
+                if isinstance(contents, bytes):
+                    path.write_bytes(contents)
+                else:
+                    path.write_text(contents, encoding="utf-8")
+
+    def gen(self, struct, text=""):
+        if isinstance(struct, (str, bytes, pathlib.PurePath)):
+            struct = {struct: text}
+
+        self._gen(struct)
+        return struct.keys()
+
+    def close(self):
+        pass
 
-class Base:
     @staticmethod
     def should_test():
         return True
@@ -10,10 +64,6 @@ class Base:
     def get_url():
         raise NotImplementedError
 
-    @cached_property
-    def url(self):
-        return self.get_url()
-
     @cached_property
     def config(self):
         return {"url": self.url}

tests/remotes/gdrive.py

@@ -1,9 +1,11 @@
+# pylint:disable=abstract-method
 import os
 import uuid
 
 import pytest
 from funcy import cached_property
 
+from dvc.path_info import CloudURLInfo
 from dvc.remote.gdrive import GDriveRemoteTree
 
 from .base import Base
@@ -11,7 +13,7 @@ from .base import Base
 TEST_GDRIVE_REPO_BUCKET = "root"
 
 
-class GDrive(Base):
+class GDrive(Base, CloudURLInfo):
     @staticmethod
     def should_test():
         return os.getenv(GDriveRemoteTree.GDRIVE_CREDENTIALS_DATA) is not None
@@ -25,10 +27,6 @@ class GDrive(Base):
             "gdrive_use_service_account": True,
         }
 
-    def __init__(self, dvc):
-        tree = GDriveRemoteTree(dvc, self.config)
-        tree._gdrive_create_dir("root", tree.path_info.path)
-
     @staticmethod
     def _get_storagepath():
         return TEST_GDRIVE_REPO_BUCKET + "/" + str(uuid.uuid4())
@@ -46,10 +44,8 @@ def gdrive(make_tmp_dir):
 
     # NOTE: temporary workaround
     tmp_dir = make_tmp_dir("gdrive", dvc=True)
-    return GDrive(tmp_dir.dvc)
 
-
-@pytest.fixture
-def gdrive_remote(tmp_dir, dvc, gdrive):
-    tmp_dir.add_remote(config=gdrive.config)
-    return gdrive
+    ret = GDrive(GDrive.get_url())
+    tree = GDriveRemoteTree(tmp_dir.dvc, ret.config)
+    tree._gdrive_create_dir("root", tree.path_info.path)
+    return ret

tests/remotes/gs.py

@@ -1,12 +1,11 @@
+import locale
 import os
 import uuid
-from contextlib import contextmanager
 
 import pytest
 from funcy import cached_property
 
-from dvc.remote.base import Remote
-from dvc.remote.gs import GSRemoteTree
+from dvc.path_info import CloudURLInfo
 from dvc.utils import env2bool
 
 from .base import Base
@@ -23,7 +22,7 @@ TEST_GCP_CREDS_FILE = os.path.abspath(
 os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE
 
 
-class GCP(Base):
+class GCP(Base, CloudURLInfo):
     @staticmethod
     def should_test():
         from subprocess import CalledProcessError, check_output
@@ -64,27 +63,60 @@ class GCP(Base):
     def get_url():
         return "gs://" + GCP._get_storagepath()
 
-    @classmethod
-    @contextmanager
-    def remote(cls, repo):
-        yield Remote(GSRemoteTree(repo, {"url": cls.get_url()}))
+    @property
+    def _gc(self):
+        from google.cloud.storage import Client
 
-    @staticmethod
-    def put_objects(remote, objects):
-        client = remote.tree.gs
-        bucket = client.get_bucket(remote.path_info.bucket)
-        for key, body in objects.items():
-            bucket.blob((remote.path_info / key).path).upload_from_string(body)
+        return Client.from_service_account_json(TEST_GCP_CREDS_FILE)
+
+    @property
+    def _bucket(self):
+        return self._gc.bucket(self.bucket)
+
+    @property
+    def _blob(self):
+        return self._bucket.blob(self.path)
+
+    def is_file(self):
+        if self.path.endswith("/"):
+            return False
+
+        return self._blob.exists()
+
+    def is_dir(self):
+        dir_path = self / ""
+        return bool(list(self._bucket.list_blobs(prefix=dir_path.path)))
+
+    def exists(self):
+        return self.is_file() or self.is_dir()
+
+    def mkdir(self, mode=0o777, parents=False, exist_ok=False):
+        assert mode == 0o777
+        assert parents
+        assert not exist_ok
+
+    def write_bytes(self, contents):
+        assert isinstance(contents, bytes)
+        self._blob.upload_from_string(contents)
+
+    def write_text(self, contents, encoding=None, errors=None):
+        if not encoding:
+            encoding = locale.getpreferredencoding(False)
+        assert errors is None
+        self.write_bytes(contents.encode(encoding))
+
+    def read_bytes(self):
+        return self._blob.download_as_string()
+
+    def read_text(self, encoding=None, errors=None):
+        if not encoding:
+            encoding = locale.getpreferredencoding(False)
+        assert errors is None
+        return self.read_bytes().decode(encoding)
 
 
 @pytest.fixture
 def gs():
     if not GCP.should_test():
         pytest.skip("no gs")
-    yield GCP()
-
-
-@pytest.fixture
-def gs_remote(tmp_dir, dvc, gs):
-    tmp_dir.add_remote(config=gs.config)
-    yield gs
+    yield GCP(GCP.get_url())

tests/remotes/hdfs.py

@@ -1,57 +1,144 @@
-import getpass
+import locale
 import os
 import platform
-from subprocess import CalledProcessError, Popen, check_output
+import uuid
+from contextlib import contextmanager
 
 import pytest
 
+from dvc.path_info import URLInfo
+
 from .base import Base
-from .local import Local
 
 
-class HDFS(Base):
-    @staticmethod
-    def should_test():
-        if platform.system() != "Linux":
-            return False
+class HDFS(Base, URLInfo):  # pylint: disable=abstract-method
+    @contextmanager
+    def _hdfs(self):
+        import pyarrow
 
+        conn = pyarrow.hdfs.connect(self.host, self.port)
         try:
-            # pylint: disable=unexpected-keyword-arg
-            # see: https://github.com/PyCQA/pylint/issues/3645
-            check_output(
-                ["hadoop", "version"],
-                shell=True,
-                executable=os.getenv("SHELL"),
-            )
-        except (CalledProcessError, OSError):
+            yield conn
+        finally:
+            conn.close()
+
+    def is_file(self):
+        with self._hdfs() as _hdfs:
+            return _hdfs.isfile(self.path)
+
+    def is_dir(self):
+        with self._hdfs() as _hdfs:
+            return _hdfs.isfile(self.path)
+
+    def exists(self):
+        with self._hdfs() as _hdfs:
+            return _hdfs.exists(self.path)
+
+    def mkdir(self, mode=0o777, parents=False, exist_ok=False):
+        assert mode == 0o777
+        assert parents
+        assert not exist_ok
+
+        with self._hdfs() as _hdfs:
+            # NOTE: hdfs.mkdir always creates parents
+            _hdfs.mkdir(self.path)
+
+    def write_bytes(self, contents):
+        with self._hdfs() as _hdfs:
+            # NOTE: hdfs.open only supports 'rb', 'wb' or 'ab'
+            with _hdfs.open(self.path, "wb") as fobj:
+                fobj.write(contents)
+
+    def write_text(self, contents, encoding=None, errors=None):
+        if not encoding:
+            encoding = locale.getpreferredencoding(False)
+        assert errors is None
+        self.write_bytes(contents.encode(encoding))
+
+    def read_bytes(self):
+        with self._hdfs() as _hdfs:
+            # NOTE: hdfs.open only supports 'rb', 'wb' or 'ab'
+            with _hdfs.open(self.path, "rb") as fobj:
+                return fobj.read()
+
+    def read_text(self, encoding=None, errors=None):
+        if not encoding:
+            encoding = locale.getpreferredencoding(False)
+        assert errors is None
+        return self.read_bytes().decode(encoding)
+
+
+@pytest.fixture(scope="session")
+def hadoop():
+    import wget
+    import tarfile
+    from appdirs import user_cache_dir
+
+    if platform.system() != "Linux":
+        pytest.skip("only supported on Linux")
+
+    hadoop_name = "hadoop-2.7.2.tar.gz"
+    java_name = "openjdk-7u75-b13-linux-x64-18_dec_2014.tar.gz"
+
+    base_url = "https://s3-us-east-2.amazonaws.com/dvc-public/dvc-test/"
+    hadoop_url = base_url + hadoop_name
+    java_url = base_url + java_name
+
+    (cache_dir,) = (user_cache_dir("dvc-test", "iterative"),)
+    dname = os.path.join(cache_dir, "hdfs")
+
+    java_tar = os.path.join(dname, java_name)
+    hadoop_tar = os.path.join(dname, hadoop_name)
+
+    java_home = os.path.join(dname, "java-se-7u75-ri")
+    hadoop_home = os.path.join(dname, "hadoop-2.7.2")
+
+    def _get(url, tar, target):
+        if os.path.isdir(target):
+            return
+
+        if not os.path.exists(tar):
+            wget.download(url, out=tar)
+        tar = tarfile.open(tar)
+        tar.extractall(dname)
+        assert os.path.isdir(target)
+
+    os.makedirs(dname, exist_ok=True)
+    _get(hadoop_url, hadoop_tar, hadoop_home)
+    _get(java_url, java_tar, java_home)
+
+    os.environ["JAVA_HOME"] = java_home
+    os.environ["HADOOP_HOME"] = hadoop_home
+    os.environ["PATH"] += f":{hadoop_home}/bin:{hadoop_home}/sbin"
+
+
+@pytest.fixture(scope="session")
+def hdfs_server(hadoop, docker_compose, docker_services):
+    import pyarrow
+
+    port = docker_services.port_for("hdfs", 8020)
+
+    def _check():
+        try:
+            # NOTE: just connecting or even opening something is not enough,
+            # we need to make sure that we are able to write something.
+            conn = pyarrow.hdfs.connect("127.0.0.1", port)
+            try:
+                with conn.open(str(uuid.uuid4()), "wb") as fobj:
+                    fobj.write(b"test")
+            finally:
+                conn.close()
+            return True
+        except (pyarrow.ArrowException, OSError):
             return False
 
-        p = Popen(
-            "hadoop fs -ls hdfs://127.0.0.1/",
-            shell=True,
-            executable=os.getenv("SHELL"),
-        )
-        p.communicate()
-        if p.returncode != 0:
-            return False
+    docker_services.wait_until_responsive(timeout=30.0, pause=5, check=_check)
 
-        return True
-
-    @staticmethod
-    def get_url():
-        return "hdfs://{}@127.0.0.1{}".format(
-            getpass.getuser(), Local.get_storagepath()
-        )
+    return port
 
 
 @pytest.fixture
-def hdfs():
-    if not HDFS.should_test():
-        pytest.skip("no hadoop running")
-    yield HDFS()
-
-
-@pytest.fixture
-def hdfs_remote(tmp_dir, dvc, hdfs):
-    tmp_dir.add_remote(config=hdfs.config)
-    yield hdfs
+def hdfs(hdfs_server):
+    port = hdfs_server
+    url = f"hdfs://127.0.0.1:{port}/{uuid.uuid4()}"
+    yield HDFS(url)

tests/remotes/http.py

@@ -1,31 +1,48 @@
+# pylint:disable=abstract-method
+import locale
+import os
+import uuid
+
 import pytest
+import requests
+
+from dvc.path_info import HTTPURLInfo
 
 from .base import Base
 
 
-class HTTP(Base):
+class HTTP(Base, HTTPURLInfo):
     @staticmethod
     def get_url(port):  # pylint: disable=arguments-differ
-        return f"http://127.0.0.1:{port}"
+        dname = str(uuid.uuid4())
+        return f"http://127.0.0.1:{port}/{dname}"
 
-    def __init__(self, server):
-        self.url = self.get_url(server.server_port)
+    def mkdir(self, mode=0o777, parents=False, exist_ok=False):
+        assert mode == 0o777
+        assert parents
+        assert not exist_ok
+
+    def write_bytes(self, contents):
+        assert isinstance(contents, bytes)
+        response = requests.post(self.url, data=contents)
+        assert response.status_code == 200
+
+    def write_text(self, contents, encoding=None, errors=None):
+        if not encoding:
+            encoding = locale.getpreferredencoding(False)
+        assert errors is None
+        self.write_bytes(contents.encode(encoding))
 
 
-@pytest.fixture
-def http_server(tmp_dir):
-    from tests.utils.httpd import PushRequestHandler, StaticFileServer
+@pytest.fixture(scope="session")
+def http_server(tmp_path_factory):
+    from tests.utils.httpd import StaticFileServer
 
-    with StaticFileServer(handler_class=PushRequestHandler) as httpd:
+    directory = os.fspath(tmp_path_factory.mktemp("http"))
+    with StaticFileServer(directory=directory) as httpd:
         yield httpd
 
 
 @pytest.fixture
 def http(http_server):
-    yield HTTP(http_server)
-
-
-@pytest.fixture
-def http_remote(tmp_dir, dvc, http):
-    tmp_dir.add_remote(config=http.config)
-    yield http
+    yield HTTP(HTTP.get_url(http_server.server_port))

tests/remotes/local.py

@@ -1,4 +1,4 @@
-# pylint: disable=cyclic-import
+# pylint: disable=cyclic-import,disable=abstract-method
 import pytest
 
 from tests.basic_env import TestDvc
@@ -17,8 +17,11 @@ class Local(Base):
 
 
 @pytest.fixture
-def local_cloud():
-    yield Local()
+def local_cloud(make_tmp_dir):
+    ret = make_tmp_dir("local-cloud")
+    ret.url = str(ret)
+    ret.config = {"url": ret.url}
+    return ret
 
 
 @pytest.fixture

tests/remotes/oss.py

@@ -1,45 +1,54 @@
-import os
+# pylint:disable=abstract-method
 import uuid
 
 import pytest
 
-from dvc.utils import env2bool
+from dvc.path_info import CloudURLInfo
 
 from .base import Base
 
 TEST_OSS_REPO_BUCKET = "dvc-test"
+TEST_OSS_ENDPOINT = "127.0.0.1:{port}"
+TEST_OSS_ACCESS_KEY_ID = "AccessKeyID"
+TEST_OSS_ACCESS_KEY_SECRET = "AccessKeySecret"
 
 
-class OSS(Base):
-    @staticmethod
-    def should_test():
-        do_test = env2bool("DVC_TEST_OSS", undefined=None)
-        if do_test is not None:
-            return do_test
+class OSS(Base, CloudURLInfo):
+    pass
 
-        return (
-            os.getenv("OSS_ENDPOINT")
-            and os.getenv("OSS_ACCESS_KEY_ID")
-            and os.getenv("OSS_ACCESS_KEY_SECRET")
-        )
 
-    @staticmethod
-    def _get_storagepath():
-        return f"{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
+@pytest.fixture(scope="session")
+def oss_server(docker_compose, docker_services):
+    import oss2
 
-    @staticmethod
-    def get_url():
-        return f"oss://{OSS._get_storagepath()}"
+    port = docker_services.port_for("oss", 8880)
+    endpoint = TEST_OSS_ENDPOINT.format(port=port)
+
+    def _check():
+        try:
+            auth = oss2.Auth(
+                TEST_OSS_ACCESS_KEY_ID, TEST_OSS_ACCESS_KEY_SECRET
+            )
+            oss2.Bucket(auth, endpoint, "mybucket").get_bucket_info()
+            return True
+        except oss2.exceptions.NoSuchBucket:
+            return True
+        except oss2.exceptions.OssError:
+            return False
+
+    docker_services.wait_until_responsive(timeout=30.0, pause=5, check=_check)
+
+    return endpoint
 
 
 @pytest.fixture
-def oss():
-    if not OSS.should_test():
-        pytest.skip("no oss running")
-    yield OSS()
-
-
-@pytest.fixture
-def oss_remote(tmp_dir, dvc, oss):
-    tmp_dir.add_remote(config=oss.config)
-    yield oss
+def oss(oss_server):
+    url = f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
+    ret = OSS(url)
+    ret.config = {
+        "url": url,
+        "oss_key_id": TEST_OSS_ACCESS_KEY_ID,
+        "oss_key_secret": TEST_OSS_ACCESS_KEY_SECRET,
+        "oss_endpoint": oss_server,
+    }
+    return ret

tests/remotes/s3.py

@@ -1,12 +1,12 @@
+import locale
 import os
 import uuid
-from contextlib import contextmanager
 
 import pytest
-from moto.s3 import mock_s3
+from funcy import cached_property
+from moto import mock_s3
 
-from dvc.remote.base import Remote
-from dvc.remote.s3 import S3RemoteTree
+from dvc.path_info import CloudURLInfo
 from dvc.utils import env2bool
 
 from .base import Base
@@ -14,7 +14,7 @@ from .base import Base
 TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-temp")
 
 
-class S3(Base):
+class S3(Base, CloudURLInfo):
     @staticmethod
     def should_test():
         do_test = env2bool("DVC_TEST_AWS", undefined=None)
@@ -36,33 +36,74 @@ class S3(Base):
     def get_url():
         return "s3://" + S3._get_storagepath()
 
+    @cached_property
+    def _s3(self):
+        import boto3
+
+        return boto3.client("s3")
+
+    def is_file(self):
+        from botocore.exceptions import ClientError
+
+        if self.path.endswith("/"):
+            return False
+
+        try:
+            self._s3.head_object(Bucket=self.bucket, Key=self.path)
+        except ClientError as exc:
+            if exc.response["Error"]["Code"] != "404":
+                raise
+            return False
+
+        return True
+
+    def is_dir(self):
+        path = (self / "").path
+        resp = self._s3.list_objects(Bucket=self.bucket, Prefix=path)
+        return bool(resp.get("Contents"))
+
+    def exists(self):
+        return self.is_file() or self.is_dir()
+
+    def mkdir(self, mode=0o777, parents=False, exist_ok=False):
+        assert mode == 0o777
+        assert parents
+        assert not exist_ok
+
+    def write_bytes(self, contents):
+        self._s3.put_object(
+            Bucket=self.bucket, Key=self.path, Body=contents,
+        )
+
+    def write_text(self, contents, encoding=None, errors=None):
+        if not encoding:
+            encoding = locale.getpreferredencoding(False)
+        assert errors is None
+        self.write_bytes(contents.encode(encoding))
+
+    def read_bytes(self):
+        data = self._s3.get_object(Bucket=self.bucket, Key=self.path)
+        return data["Body"].read()
+
+    def read_text(self, encoding=None, errors=None):
+        if not encoding:
+            encoding = locale.getpreferredencoding(False)
+        assert errors is None
+        return self.read_bytes().decode(encoding)
+
 
 @pytest.fixture
 def s3():
-    if not S3.should_test():
-        pytest.skip("no s3")
-    yield S3()
+    with mock_s3():
+        import boto3
+
+        boto3.client("s3").create_bucket(Bucket=TEST_AWS_REPO_BUCKET)
+
+        yield S3(S3.get_url())
 
 
 @pytest.fixture
-def s3_remote(tmp_dir, dvc, s3):
-    tmp_dir.add_remote(config=s3.config)
-    yield s3
-
-
-class S3Mocked(S3):
-    @classmethod
-    @contextmanager
-    def remote(cls, repo):
-        with mock_s3():
-            yield Remote(S3RemoteTree(repo, {"url": cls.get_url()}))
-
-    @staticmethod
-    def put_objects(remote, objects):
-        client = remote.tree.s3
-        bucket = remote.path_info.bucket
-        client.create_bucket(Bucket=bucket)
-        for key, body in objects.items():
-            client.put_object(
-                Bucket=bucket, Key=(remote.path_info / key).path, Body=body
-            )
+def real_s3():
+    if not S3.should_test():
+        pytest.skip("no real s3")
+    yield S3(S3.get_url())

tests/remotes/ssh.py

@@ -1,10 +1,13 @@
 import getpass
+import locale
 import os
+from contextlib import contextmanager
 from subprocess import CalledProcessError, check_output
 
 import pytest
 from funcy import cached_property
 
+from dvc.path_info import URLInfo
 from dvc.utils import env2bool
 
 from .base import Base
@@ -41,7 +44,7 @@ class SSH:
         )
 
 
-class SSHMocked(Base):
+class SSHMocked(Base, URLInfo):
     @staticmethod
     def get_url(user, port):  # pylint: disable=arguments-differ
         path = Local.get_storagepath()
@@ -63,13 +66,6 @@ class SSHMocked(Base):
         url = f"ssh://{user}@127.0.0.1:{port}{path}"
         return url
 
-    def __init__(self, server):
-        self.server = server
-
-    @cached_property
-    def url(self):
-        return self.get_url(TEST_SSH_USER, self.server.port)
-
     @cached_property
     def config(self):
         return {
@@ -77,6 +73,66 @@ class SSHMocked(Base):
             "keyfile": TEST_SSH_KEY_PATH,
         }
 
+    @contextmanager
+    def _ssh(self):
+        from dvc.remote.ssh.connection import SSHConnection
+
+        conn = SSHConnection(
+            host=self.host,
+            port=self.port,
+            username=TEST_SSH_USER,
+            key_filename=TEST_SSH_KEY_PATH,
+        )
+        try:
+            yield conn
+        finally:
+            conn.close()
+
+    def is_file(self):
+        with self._ssh() as _ssh:
+            return _ssh.isfile(self.path)
+
+    def is_dir(self):
+        with self._ssh() as _ssh:
+            return _ssh.isdir(self.path)
+
+    def exists(self):
+        with self._ssh() as _ssh:
+            return _ssh.exists(self.path)
+
+    def mkdir(self, mode=0o777, parents=False, exist_ok=False):
+        assert mode == 0o777
+        assert parents
+        assert not exist_ok
+
+        with self._ssh() as _ssh:
+            _ssh.makedirs(self.path)
+
+    def write_bytes(self, contents):
+        assert isinstance(contents, bytes)
+        with self._ssh() as _ssh:
+            with _ssh.open(self.path, "w+") as fobj:
+                # NOTE: accepts both str and bytes
+                fobj.write(contents)
+
+    def write_text(self, contents, encoding=None, errors=None):
+        if not encoding:
+            encoding = locale.getpreferredencoding(False)
+        assert errors is None
+        self.write_bytes(contents.encode(encoding))
+
+    def read_bytes(self):
+        with self._ssh() as _ssh:
+            # NOTE: sftp always reads in binary format
+            with _ssh.open(self.path, "r") as fobj:
+                return fobj.read()
+
+    def read_text(self, encoding=None, errors=None):
+        if not encoding:
+            encoding = locale.getpreferredencoding(False)
+        assert errors is None
+        return self.read_bytes().decode(encoding)
+
 
 @pytest.fixture
 def ssh_server():
@@ -106,10 +162,4 @@ def ssh(ssh_server, monkeypatch):
     # NOTE: see http://github.com/iterative/dvc/pull/3501
     monkeypatch.setattr(SSHRemoteTree, "CAN_TRAVERSE", False)
 
-    return SSHMocked(ssh_server)
-
-
-@pytest.fixture
-def ssh_remote(tmp_dir, dvc, ssh):
-    tmp_dir.add_remote(config=ssh.config)
-    yield ssh
+    return SSHMocked(SSHMocked.get_url(TEST_SSH_USER, ssh_server.port))

tests/unit/dependency/test_params.py

@@ -4,6 +4,7 @@ import yaml
 from dvc.dependency import ParamsDependency, loadd_from, loads_params
 from dvc.dependency.param import BadParamFileError, MissingParamsError
 from dvc.stage import Stage
+from dvc.utils.yaml import load_yaml
 
 PARAMS = {
     "foo": 1,
@@ -11,6 +12,7 @@ PARAMS = {
     "baz": "str",
     "qux": None,
 }
+DEFAULT_PARAMS_FILE = ParamsDependency.DEFAULT_PARAMS_FILE
 
 
 def test_loads_params(dvc):
@@ -63,7 +65,7 @@ def test_loadd_from(dvc):
 def test_dumpd_with_info(dvc):
     dep = ParamsDependency(Stage(dvc), None, PARAMS)
     assert dep.dumpd() == {
-        "path": "params.yaml",
+        "path": DEFAULT_PARAMS_FILE,
         "params": PARAMS,
     }
 
@@ -71,7 +73,7 @@ def test_dumpd_with_info(dvc):
 def test_dumpd_without_info(dvc):
     dep = ParamsDependency(Stage(dvc), None, list(PARAMS.keys()))
     assert dep.dumpd() == {
-        "path": "params.yaml",
+        "path": DEFAULT_PARAMS_FILE,
         "params": list(PARAMS.keys()),
     }
 
@@ -82,7 +84,7 @@ def test_read_params_nonexistent_file(dvc):
 
 
 def test_read_params_unsupported_format(tmp_dir, dvc):
-    tmp_dir.gen("params.yaml", b"\0\1\2\3\4\5\6\7")
+    tmp_dir.gen(DEFAULT_PARAMS_FILE, b"\0\1\2\3\4\5\6\7")
     dep = ParamsDependency(Stage(dvc), None, ["foo"])
     with pytest.raises(BadParamFileError):
         dep.read_params()
@@ -90,7 +92,8 @@ def test_read_params_unsupported_format(tmp_dir, dvc):
 
 def test_read_params_nested(tmp_dir, dvc):
     tmp_dir.gen(
-        "params.yaml", yaml.dump({"some": {"path": {"foo": ["val1", "val2"]}}})
+        DEFAULT_PARAMS_FILE,
+        yaml.dump({"some": {"path": {"foo": ["val1", "val2"]}}}),
     )
     dep = ParamsDependency(Stage(dvc), None, ["some.path.foo"])
     assert dep.read_params() == {"some.path.foo": ["val1", "val2"]}
@@ -103,7 +106,24 @@ def test_save_info_missing_config(dvc):
 
 
 def test_save_info_missing_param(tmp_dir, dvc):
-    tmp_dir.gen("params.yaml", "bar: baz")
+    tmp_dir.gen(DEFAULT_PARAMS_FILE, "bar: baz")
     dep = ParamsDependency(Stage(dvc), None, ["foo"])
     with pytest.raises(MissingParamsError):
         dep.save_info()
+
+
+@pytest.mark.parametrize(
+    "param_value",
+    ["", "false", "[]", "{}", "null", "no", "off"]
+    # we use pyyaml to load params.yaml, which only supports YAML 1.1
+    # so, some of the above are boolean values
+)
+def test_params_with_false_values(tmp_dir, dvc, param_value):
+    key = "param"
+    dep = ParamsDependency(Stage(dvc), DEFAULT_PARAMS_FILE, [key])
+    (tmp_dir / DEFAULT_PARAMS_FILE).write_text(f"{key}: {param_value}")
+
+    dep.fill_values(load_yaml(DEFAULT_PARAMS_FILE))
+
+    with dvc.state:
+        assert dep.status() == {}

tests/unit/remote/test_azure.py

@@ -1,8 +1,5 @@
-import pytest
-
 from dvc.path_info import PathInfo
 from dvc.remote.azure import AzureRemoteTree
-from tests.remotes import Azure
 
 container_name = "container-name"
 connection_string = (
@@ -32,14 +29,11 @@ def test_init(dvc):
     assert tree._conn_kwargs["connection_string"] == connection_string
 
 
-def test_get_file_hash(tmp_dir):
-    if not Azure.should_test():
-        pytest.skip("no azurite running")
-
+def test_get_file_hash(tmp_dir, azure):
     tmp_dir.gen("foo", "foo")
 
-    tree = AzureRemoteTree(None, {})
-    to_info = tree.PATH_CLS(Azure.get_url())
+    tree = AzureRemoteTree(None, azure.config)
+    to_info = azure
     tree.upload(PathInfo("foo"), to_info)
     assert tree.exists(to_info)
     hash_ = tree.get_file_hash(to_info)

tests/unit/remote/test_base.py

@@ -142,6 +142,20 @@ def test_list_hashes(dvc):
         assert hashes == ["123456"]
 
 
+def test_list_paths(dvc):
+    tree = BaseRemoteTree(dvc, {})
+    tree.path_info = PathInfo("foo")
+
+    with mock.patch.object(tree, "walk_files", return_value=[]) as walk_mock:
+        for _ in tree.list_paths():
+            pass
+        walk_mock.assert_called_with(tree.path_info, prefix=False)
+
+        for _ in tree.list_paths(prefix="000"):
+            pass
+        walk_mock.assert_called_with(tree.path_info / "00" / "0", prefix=True)
+
+
 @pytest.mark.parametrize(
     "hash_, result",
     [(None, False), ("", False), ("3456.dir", True), ("3456", False)],

tests/unit/remote/test_http.py

@@ -1,20 +1,14 @@
 import pytest
 
 from dvc.exceptions import HTTPError
-from dvc.path_info import URLInfo
 from dvc.remote.http import HTTPRemoteTree
-from tests.utils.httpd import StaticFileServer
 
 
-def test_download_fails_on_error_code(dvc):
-    with StaticFileServer() as httpd:
-        url = f"http://localhost:{httpd.server_port}/"
-        config = {"url": url}
+def test_download_fails_on_error_code(dvc, http):
+    tree = HTTPRemoteTree(dvc, http.config)
 
-        tree = HTTPRemoteTree(dvc, config)
-
-        with pytest.raises(HTTPError):
-            tree._download(URLInfo(url) / "missing.txt", "missing.txt")
+    with pytest.raises(HTTPError):
+        tree._download(http / "missing.txt", "missing.txt")
 
 
 def test_public_auth_method(dvc):

tests/unit/remote/test_remote_tree.py

@@ -3,11 +3,11 @@ import os
 import pytest
 
 from dvc.path_info import PathInfo
+from dvc.remote import get_remote
 from dvc.remote.s3 import S3RemoteTree
 from dvc.utils.fs import walk_files
-from tests.remotes import GCP, S3Mocked
 
-remotes = [GCP, S3Mocked]
+remotes = [pytest.lazy_fixture(fix) for fix in ["gs", "s3"]]
 
 FILE_WITH_CONTENTS = {
     "data1.txt": "",
@@ -27,11 +27,9 @@ FILE_WITH_CONTENTS = {
 
 @pytest.fixture
 def remote(request, dvc):
-    if not request.param.should_test():
-        raise pytest.skip()
-    with request.param.remote(dvc) as _remote:
-        request.param.put_objects(_remote, FILE_WITH_CONTENTS)
-        yield _remote
+    cloud = request.param
+    cloud.gen(FILE_WITH_CONTENTS)
+    return get_remote(dvc, **cloud.config)
 
 
 @pytest.mark.parametrize("remote", remotes, indirect=True)
@@ -86,7 +84,7 @@ def test_walk_files(remote):
     assert list(remote.tree.walk_files(remote.path_info / "data")) == files
 
 
-@pytest.mark.parametrize("remote", [S3Mocked], indirect=True)
+@pytest.mark.parametrize("remote", [pytest.lazy_fixture("s3")], indirect=True)
 def test_copy_preserve_etag_across_buckets(remote, dvc):
     s3 = remote.tree.s3
     s3.create_bucket(Bucket="another")
@@ -115,7 +113,7 @@ def test_makedirs(remote):
     assert tree.isdir(empty_dir)
 
 
-@pytest.mark.parametrize("remote", [GCP, S3Mocked], indirect=True)
+@pytest.mark.parametrize("remote", remotes, indirect=True)
 def test_isfile(remote):
     test_cases = [
         (False, "empty_dir/"),

tests/unit/repo/test_repo.py

@@ -3,7 +3,8 @@ import os
 import pytest
 from funcy import raiser
 
-from dvc.repo import locked
+from dvc.repo import NotDvcRepoError, Repo, locked
+from dvc.utils.fs import remove
 
 
 def test_is_dvc_internal(dvc):
@@ -127,3 +128,27 @@ def test_skip_graph_checks(tmp_dir, dvc, mocker, run_copy):
     dvc.add("quux")
     run_copy("quux", "quuz", single_stage=True)
     assert mock_collect_graph.called
+
+
+def test_branch_config(tmp_dir, scm):
+    tmp_dir.scm_gen("foo", "foo", commit="init")
+
+    scm.checkout("branch", create_new=True)
+    dvc = Repo.init()
+    with dvc.config.edit() as conf:
+        conf["remote"]["branch"] = {"url": "/some/path"}
+    scm.add([".dvc"])
+    scm.commit("init dvc")
+    scm.checkout("master")
+
+    remove(".dvc")
+
+    # sanity check
+    with pytest.raises(NotDvcRepoError):
+        Repo()
+
+    with pytest.raises(NotDvcRepoError):
+        Repo(scm=scm, rev="master")
+
+    dvc = Repo(scm=scm, rev="branch")
+    assert dvc.config["remote"]["branch"]["url"] == "/some/path"

tests/unit/test_ignore.py

@@ -14,34 +14,51 @@ def mock_dvcignore(dvcignore_path, patterns):
     return ignore_patterns
 
 
-def test_ignore_from_file_should_filter_dirs_and_files():
-    dvcignore_path = os.path.join(
-        os.path.sep, "full", "path", "to", "ignore", "file", ".dvcignore"
-    )
-
-    patterns = ["dir_to_ignore", "file_to_ignore"]
-
-    root = os.path.dirname(dvcignore_path)
-    dirs = ["dir1", "dir2", "dir_to_ignore"]
-    files = ["file1", "file2", "file_to_ignore"]
-
-    ignore = mock_dvcignore(dvcignore_path, patterns)
-    new_dirs, new_files = ignore(root, dirs, files)
-
-    assert {"dir1", "dir2"} == set(new_dirs)
-    assert {"file1", "file2"} == set(new_files)
-
-
 @pytest.mark.parametrize(
     "file_to_ignore_relpath, patterns,  expected_match",
     [
+        # all rules from https://git-scm.com/docs/gitignore
         ("to_ignore", ["to_ignore"], True),
+        ("dont_ignore.txt", ["dont_ignore"], False),
+        # A blank line matches no files, so it can serve as a separator for
+        # readability.
+        ("to_ignore", ["", "to_ignore"], True),
+        # A line starting with # serves as a comment.
+        # Put a backslash ("\") in front of the first hash for patterns
+        # that begin with a hash.
+        ("#to_ignore", ["\\#to_ignore"], True),
+        ("#to_ignore", ["#to_ignore"], False),
+        # Trailing spaces are ignored unless they are quoted with
+        # backslash ("\").
+        (" to_ignore", [" to_ignore"], False),
+        (" to_ignore", ["\\ to_ignore"], True),
+        # An optional prefix "!" which negates the pattern; any matching file
+        # excluded by a previous pattern will become included again.
         ("to_ignore.txt", ["to_ignore*"], True),
-        (
-            os.path.join("rel", "p", "p2", "to_ignore"),
-            ["rel/**/to_ignore"],
-            True,
-        ),
+        ("to_ignore.txt", ["to_ignore*", "!to_ignore.txt"], False),
+        ("to_ignore.txt", ["!to_ignore.txt", "to_ignore*"], True),
+        # It is not possible to re-include a file if a parent directory of
+        # that file is excluded.
+        # Git doesn’t list excluded directories for performance reasons,
+        # so any patterns on contained files have no effect,
+        # no matter where they are defined.
+        # see (`tests/func/test_ignore.py::test_ignore_parent_path`)
+        # Put a backslash ("\") in front of the first "!"
+        # for patterns that begin with a literal "!",
+        # for example, "\!important!.txt".
+        ("!to_ignore.txt", ["\\!to_ignore.txt"], True),
+        # The slash / is used as the directory separator.
+        # Separators may occur at the beginning, middle or end of the
+        # .gitignore search pattern.
+        # If there is a separator at the beginning or middle (or both)
+        # of the pattern, then the pattern is relative to the directory
+        # level of the particular .gitignore file itself.
+        # Otherwise the pattern may also match at any level below
+        # the .gitignore level.
+        ("file", ["/file"], True),
+        (os.path.join("data", "file"), ["/file"], False),
+        (os.path.join("data", "file"), ["data/file"], True),
+        (os.path.join("other", "data", "file"), ["data/file"], False),
         (
             os.path.join(
                 os.path.sep,
@@ -55,20 +72,96 @@ def test_ignore_from_file_should_filter_dirs_and_files():
             ["to_ignore"],
             True,
         ),
+        # If there is a separator at the end of the pattern then the pattern
+        # will only match directories,
+        # otherwise the pattern can match both files and directories.
+        # For example, a pattern doc/frotz/ matches doc/frotz directory,
+        # but not a/doc/frotz directory;
+        # see (`tests/func/test_ignore.py::test_ignore_sub_directory`)
+        # however frotz/ matches frotz and a/frotz that is a directory
+        # (all paths are relative from the .gitignore file).
+        # see (`tests/func/test_ignore.py::test_ignore_directory`)
+        # An asterisk "*" matches anything except a slash.
         ("to_ignore.txt", ["/*.txt"], True),
+        (os.path.join("path", "to_ignore.txt"), ["/*.txt"], False),
+        (os.path.join("data", "file.txt"), ["data/*"], True),
+        # wait for Git
+        # (os.path.join("data", "sub", "file.txt"), ["data/*"], True),
         (
             os.path.join("rel", "path", "path2", "to_ignore"),
             ["rel/*/to_ignore"],
             False,
         ),
-        (os.path.join("path", "to_ignore.txt"), ["/*.txt"], False),
+        ("file.txt", ["file.*"], True),
+        # The character "?" matches any one character except "/".
+        ("file.txt", ["fi?e.t?t"], True),
+        ("fi/e.txt", ["fi?e.t?t"], False),
+        # The range notation, e.g. [a-zA-Z], can be used
+        # to match one of the characters in a range. See fnmatch(3) and
+        # the FNM_PATHNAME flag for a more detailed description.
+        ("file.txt", ["[a-zA-Z]ile.txt"], True),
+        ("2ile.txt", ["[a-zA-Z]ile.txt"], False),
+        # Two consecutive asterisks ("**") in patterns matched against
+        # full pathname may have special meaning:
+        # A leading "**" followed by a slash means match in all directories.
+        # For example, "**/foo" matches file or directory "foo" anywhere, the
+        # same as pattern "foo".
+        # "**/foo/bar" matches file or directory "bar" anywhere that is
+        # directly under directory "foo".
+        (os.path.join("rel", "p", "p2", "to_ignore"), ["**/to_ignore"], True,),
+        (
+            os.path.join("rel", "p", "p2", "to_ignore"),
+            ["**/p2/to_ignore"],
+            True,
+        ),
+        (
+            os.path.join("rel", "path", "path2", "dont_ignore"),
+            ["**/to_ignore"],
+            False,
+        ),
+        # A trailing "/**" matches everything inside.
+        # For example, "abc/**" matches all files inside directory "abc",
+        # relative to the location of the .gitignore file, with infinite depth.
+        (os.path.join("rel", "p", "p2", "to_ignore"), ["rel/**"], True,),
+        (os.path.join("rel", "p", "p2", "to_ignore"), ["p/**"], False,),
+        (
+            os.path.join("rel", "path", "path2", "dont_ignore"),
+            ["rel/**"],
+            True,
+        ),
+        # A slash followed by two consecutive asterisks then a slash matches
+        # zero or more directories.
+        # For example, "a/**/b" matches "a/b", "a/x/b", "a/x/y/b" and so on.
+        (os.path.join("rel", "p", "to_ignore"), ["rel/**/to_ignore"], True,),
+        (
+            os.path.join("rel", "p", "p2", "to_ignore"),
+            ["rel/**/to_ignore"],
+            True,
+        ),
         (
             os.path.join("rel", "path", "path2", "dont_ignore"),
             ["rel/**/to_ignore"],
             False,
         ),
-        ("dont_ignore.txt", ["dont_ignore"], False),
-        ("dont_ignore.txt", ["dont*", "!dont_ignore.txt"], False),
+        (
+            os.path.join("rel", "path", "path2", "dont_ignore"),
+            ["path/**/dont_ignore"],
+            False,
+        ),
+        # Other consecutive asterisks are considered regular asterisks
+        # and will match according to the previous rules.
+        ("to_ignore.txt", ["/***.txt"], True),
+        (os.path.join("path", "to_ignore.txt"), ["/****.txt"], False),
+        (os.path.join("path", "to_ignore.txt"), ["****.txt"], True),
+        (os.path.join("data", "file.txt"), ["data/***"], True),
+        # bug from PathSpec
+        # (os.path.join("data", "p", "file.txt"), ["data/***"], False),
+        (os.path.join("data", "p", "file.txt"), ["***/file.txt"], False),
+        (
+            os.path.join("rel", "path", "path2", "to_ignore"),
+            ["rel/***/to_ignore"],
+            False,
+        ),
     ],
 )
 def test_match_ignore_from_file(
@@ -99,18 +192,3 @@ def test_should_ignore_dir(omit_dir):
     new_dirs, _ = ignore(root, dirs, files)
 
     assert set(new_dirs) == {"dir1", "dir2"}
-
-
-def test_ignore_order():
-    dvcignore_path = os.path.join(os.path.sep, "ignore_order", ".dvcignore")
-
-    patterns = ["!ac*", "a*", "!ab*"]
-
-    root = os.path.dirname(dvcignore_path)
-    dirs = ["ignore_order"]
-    files = ["ac", "ab", "aa"]
-
-    ignore = mock_dvcignore(dvcignore_path, patterns)
-    _, new_files = ignore(root, dirs, files)
-
-    assert {"ab"} == set(new_files)

tests/unit/utils/test_http.py

@@ -3,12 +3,9 @@ import io
 import requests
 
 from dvc.utils.http import open_url
-from tests.utils.httpd import StaticFileServer
 
 
-def test_open_url(tmp_path, monkeypatch):
-    monkeypatch.chdir(tmp_path)
-
+def test_open_url(tmp_path, monkeypatch, http):
     # Simulate bad connection
     original_iter_content = requests.Response.iter_content
 
@@ -26,11 +23,9 @@ def test_open_url(tmp_path, monkeypatch):
     # using twice of that plus something tests second resume,
     # this is important because second response is different
     text = "0123456789" * (io.DEFAULT_BUFFER_SIZE // 10 + 1)
-    (tmp_path / "sample.txt").write_text(text * 2)
+    http.gen("sample.txt", text * 2)
 
-    with StaticFileServer() as httpd:
-        url = f"http://localhost:{httpd.server_port}/sample.txt"
-        with open_url(url) as fd:
-            # Test various .read() variants
-            assert fd.read(len(text)) == text
-            assert fd.read() == text
+    with open_url((http / "sample.txt").url) as fd:
+        # Test various .read() variants
+        assert fd.read(len(text)) == text
+        assert fd.read() == text

tests/utils/__init__.py

@@ -1,6 +1,5 @@
 import os
 from contextlib import contextmanager
-from filecmp import dircmp
 
 from dvc.scm import Git
 
@@ -20,15 +19,5 @@ def cd(newdir):
         os.chdir(prevdir)
 
 
-def trees_equal(dir_path_1, dir_path_2):
-
-    comparison = dircmp(dir_path_1, dir_path_2)
-
-    assert set(comparison.left_only) == set(comparison.right_only) == set()
-
-    for d in comparison.common_dirs:
-        trees_equal(os.path.join(dir_path_1, d), os.path.join(dir_path_2, d))
-
-
 def to_posixpath(path):
     return path.replace("\\", "/")

tests/utils/httpd.py

@@ -1,14 +1,48 @@
 import hashlib
 import os
+import sys
 import threading
 from http import HTTPStatus
-from http.server import HTTPServer, SimpleHTTPRequestHandler
+from http.server import HTTPServer
 
 from RangeHTTPServer import RangeRequestHandler
 
 
 class TestRequestHandler(RangeRequestHandler):
-    checksum_header = None
+    def __init__(self, *args, **kwargs):
+        # NOTE: `directory` was introduced in 3.7
+        if sys.version_info < (3, 7):
+            self.directory = kwargs.pop("directory", None) or os.getcwd()
+        super().__init__(*args, **kwargs)
+
+    def translate_path(self, path):
+        import urllib
+        import posixpath
+
+        # NOTE: `directory` was introduced in 3.7
+        if sys.version_info < (3, 7):
+            return super().translate_path(path)
+
+        path = path.split("?", 1)[0]
+        path = path.split("#", 1)[0]
+        # Don't forget explicit trailing slash when normalizing. Issue17324
+        trailing_slash = path.rstrip().endswith("/")
+        try:
+            path = urllib.parse.unquote(path, errors="surrogatepass")
+        except UnicodeDecodeError:
+            path = urllib.parse.unquote(path)
+        path = posixpath.normpath(path)
+        words = path.split("/")
+        words = filter(None, words)
+        path = self.directory
+        for word in words:
+            if os.path.dirname(word) or word in (os.curdir, os.pardir):
+                # Ignore components that are not a simple file/directory name
+                continue
+            path = os.path.join(path, word)
+        if trailing_slash:
+            path += "/"
+        return path
 
     def end_headers(self):
         # RangeRequestHandler only sends Accept-Ranges header if Range header
@@ -17,27 +51,16 @@ class TestRequestHandler(RangeRequestHandler):
             self.send_header("Accept-Ranges", "bytes")
 
         # Add a checksum header
-        if self.checksum_header:
-            file = self.translate_path(self.path)
+        file = self.translate_path(self.path)
 
-            if not os.path.isdir(file) and os.path.exists(file):
-                with open(file) as fd:
-                    encoded_text = fd.read().encode("utf8")
-                    checksum = hashlib.md5(encoded_text).hexdigest()
-                    self.send_header(self.checksum_header, checksum)
+        if not os.path.isdir(file) and os.path.exists(file):
+            with open(file) as fd:
+                encoded_text = fd.read().encode("utf8")
+                checksum = hashlib.md5(encoded_text).hexdigest()
+                self.send_header("Content-MD5", checksum)
 
         RangeRequestHandler.end_headers(self)
 
-
-class ETagHandler(TestRequestHandler):
-    checksum_header = "ETag"
-
-
-class ContentMD5Handler(TestRequestHandler):
-    checksum_header = "Content-MD5"
-
-
-class PushRequestHandler(SimpleHTTPRequestHandler):
     def _chunks(self):
         while True:
             data = self.rfile.readline(65537)
@@ -69,9 +92,13 @@ class PushRequestHandler(SimpleHTTPRequestHandler):
 class StaticFileServer:
     _lock = threading.Lock()
 
-    def __init__(self, handler_class=ETagHandler):
+    def __init__(self, directory):
+        from functools import partial
+
         self._lock.acquire()
-        self._httpd = HTTPServer(("localhost", 0), handler_class)
+        self._httpd = HTTPServer(
+            ("localhost", 0), partial(TestRequestHandler, directory=directory),
+        )
         self._thread = None
 
     def __enter__(self):