dvc: bump to 1.1.11

* dvcignore: move file/dir/parent checks into DvcIgnoreFilter

* dvc: get rid of CleanTree

CleanTree is a very awkward wrapper that spreads the context across
the codebase and results in unexpected behaviour when using it.

This PR starts moving dvcignore-related logic into the trees themselves
(it makes a lot of sense, kinda like `state`) so they could deal with
it however they like.

There are at least two temporary ugly parts about this PR:

1) dvcignore is used by individual trees and not packed into
tree/base.py yet;

2) `dvcignore_root` argument. This one is caused by the dvcignore trying
to collect everything topdown starting from the certain root dir. What
it should do instead is for a certain path that is being checked look up
the tree through the parents until it finds repo root (.dvc dir) and
then stop. That would handle subrepos as well. At the same time we need
to leverage existing dvcignore trie structure to cache those results.

.pylintrc

@@ -15,7 +15,11 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-#init-hook=
+init-hook="import sys; \
+           from pathlib import Path; \
+           from pylint.config import find_pylintrc; \
+           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
+
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use.
@@ -28,7 +32,7 @@ limit-inference-results=100
 
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.
-load-plugins=pylint_pytest
+load-plugins=pylint_pytest,pylint_plugin_disable
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -149,10 +153,8 @@ disable=format,
         exception-escape,
         comprehension-escape,
         # custom disables start here
-        protected-access,  # needs-refactor
         no-self-use,
         invalid-name,
-        blacklisted-name,
         # already a check enabled for `wildcard-import`
         unused-wildcard-import,
         misplaced-comparison-constant,
@@ -176,10 +178,6 @@ disable=format,
         # we use it to speedup/optimize
         import-outside-toplevel,
         fixme,
-        # Enabling soon:
-        no-member,
-        unused-argument,
-        arguments-differ,
 
 
 # Enable the message, report, category or checker with the given id(s). You can
@@ -267,7 +265,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=
+generated-members=pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).
@@ -288,7 +286,7 @@ ignore-on-opaque-inference=yes
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
 # qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local
+ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile
 
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
@@ -329,7 +327,7 @@ dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
 
 # Argument names that match this expression will be ignored. Default to name
 # with leading underscore.
-ignored-argument-names=_.*|^ignored_|^unused_
+ignored-argument-names=_.*|^ignored_|^unused_|args|kwargs
 
 # Tells whether we should check for unused import in __init__ files.
 init-import=no

README.rst

@@ -10,7 +10,7 @@
 
 |Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|
 
-|Snap| |Choco| |Brew| |Conda| |PyPI| |Packages|
+|PyPI| |Packages| |Brew| |Conda| |Choco| |Snap|
 
 |
 
@@ -131,8 +131,6 @@ Conda (Anaconda)
 
    conda install -c conda-forge dvc
 
-Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
-
 pip (PyPI)
 ----------
 

dvc/api.py

@@ -27,10 +27,10 @@ def get_url(path, repo=None, rev=None, remote=None):
     """
     with _make_repo(repo, rev=rev) as _repo:
         if not isinstance(_repo, Repo):
-            raise UrlNotDvcRepoError(_repo.url)
+            raise UrlNotDvcRepoError(_repo.url)  # pylint: disable=no-member
         out = _repo.find_out_by_relpath(path)
         remote_obj = _repo.cloud.get_remote(remote)
-        return str(remote_obj.hash_to_path_info(out.checksum))
+        return str(remote_obj.tree.hash_to_path_info(out.checksum))
 
 
 def open(  # noqa, pylint: disable=redefined-builtin

dvc/cache.py

@@ -106,6 +106,7 @@ class NamedCacheItem:
 
 
 class NamedCache:
+    # pylint: disable=protected-access
     def __init__(self):
         self._items = defaultdict(lambda: defaultdict(NamedCacheItem))
         self.external = defaultdict(set)

dvc/cli.py

@@ -80,34 +80,36 @@ COMMANDS = [
 ]
 
 
+def _find_parser(parser, cmd_cls):
+    defaults = parser._defaults  # pylint: disable=protected-access
+    if not cmd_cls or cmd_cls == defaults.get("func"):
+        parser.print_help()
+        raise DvcParserError()
+
+    actions = parser._actions  # pylint: disable=protected-access
+    for action in actions:
+        if not isinstance(action.choices, dict):
+            # NOTE: we are only interested in subparsers
+            continue
+        for subparser in action.choices.values():
+            _find_parser(subparser, cmd_cls)
+
+
 class DvcParser(argparse.ArgumentParser):
     """Custom parser class for dvc CLI."""
 
-    def error(self, message, command=None):
-        """Custom error method.
-        Args:
-            message (str): error message.
-            command (str): subcommand name for help message
-        Raises:
-            dvc.exceptions.DvcParser: dvc parser exception.
-
-        """
+    def error(self, message, cmd_cls=None):  # pylint: disable=arguments-differ
         logger.error(message)
-        if command is not None:
-            for action in self._actions:
-                if action.dest == "cmd" and command in action.choices:
-                    subparser = action.choices[command]
-                    subparser.print_help()
-                    raise DvcParserError()
-        self.print_help()
-        raise DvcParserError()
+        _find_parser(self, cmd_cls)
 
-    # override this to send subcommand name to error method
     def parse_args(self, args=None, namespace=None):
+        # NOTE: overriding to provide a more granular help message.
+        # E.g. `dvc plots diff --bad-flag` would result in a `dvc plots diff`
+        # help message instead of generic `dvc` usage.
         args, argv = self.parse_known_args(args, namespace)
         if argv:
             msg = "unrecognized arguments: %s"
-            self.error(msg % " ".join(argv), args.cmd)
+            self.error(msg % " ".join(argv), getattr(args, "func", None))
         return args
 
 

dvc/command/add.py

@@ -63,10 +63,6 @@ def add_parser(subparsers, parent_parser):
         metavar="<filename>",
     )
     parser.add_argument(
-        "targets",
-        nargs="+",
-        help="Input files/directories to add.",
-        metavar="targets",
-        choices=completion.Required.FILE,
-    )
+        "targets", nargs="+", help="Input files/directories to add.",
+    ).complete = completion.FILE
     parser.set_defaults(func=CmdAdd)

dvc/command/cache.py

@@ -56,7 +56,5 @@ def add_parser(subparsers, parent_parser):
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
         "config file location.",
-        metavar="value",
-        choices=completion.Required.DIR,
-    )
+    ).complete = completion.DIR
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/checkout.py

@@ -112,9 +112,9 @@ def add_parser(subparsers, parent_parser):
     checkout_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-files to checkout. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
-        metavar="targets",
-        choices=completion.Optional.DVC_FILE,
-    )
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVC_FILE
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -67,7 +67,5 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="DVC-files to commit. Optional. "
         "(Finds all DVC-files in the workspace by default.)",
-        metavar="targets",
-        choices=completion.Optional.DVC_FILE,
-    )
+    ).complete = completion.DVC_FILE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -6,9 +6,11 @@ import shtab
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
-CHOICE_FUNCTIONS = {
-    "bash": {"DVCFile": "_dvc_compgen_DVCFiles"},
-    "zsh": {"DVCFile": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'"},
+FILE = shtab.FILE
+DIR = shtab.DIRECTORY
+DVC_FILE = {
+    "bash": "_dvc_compgen_DVCFiles",
+    "zsh": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'",
 }
 PREAMBLE = {
     "bash": """
@@ -24,26 +26,13 @@ _dvc_compgen_DVCFiles() {
 }
 
 
-class Optional(shtab.Optional):
-    DVC_FILE = [shtab.Choice("DVCFile", required=False)]
-
-
-class Required(shtab.Required):
-    DVC_FILE = [shtab.Choice("DVCFile", required=True)]
-
-
 class CmdCompletion(CmdBaseNoRepo):
     def run(self):
         from dvc.cli import get_main_parser
 
         parser = get_main_parser()
         shell = self.args.shell
-        script = shtab.complete(
-            parser,
-            shell=shell,
-            preamble=PREAMBLE[shell],
-            choice_functions=CHOICE_FUNCTIONS[shell],
-        )
+        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
         print(script)
         return 0
 

dvc/command/daemon.py

@@ -66,9 +66,6 @@ def add_parser(subparsers, parent_parser):
         help=DAEMON_ANALYTICS_HELP,
     )
     daemon_analytics_parser.add_argument(
-        "target",
-        help="Analytics file.",
-        metavar="target",
-        choices=completion.Required.FILE,
-    )
+        "target", help="Analytics file.",
+    ).complete = completion.FILE
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/data_sync.py

@@ -103,11 +103,11 @@ def shared_parent_parser():
     parent_parser.add_argument(
         "targets",
         nargs="*",
-        help="Limit command scope to these DVC-files. "
-        "Using -R, directories to search DVC-files in can also be given.",
-        metavar="targets",
-        choices=completion.Optional.DVC_FILE,
-    )
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVC_FILE
 
     return parent_parser
 

dvc/command/freeze.py

@@ -40,12 +40,8 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     freeze_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="Stages or .dvc files to freeze",
-        metavar="targets",
-        choices=completion.Required.DVC_FILE,
-    )
+        "targets", nargs="+", help="Stages or .dvc files to freeze",
+    ).complete = completion.DVC_FILE
     freeze_parser.set_defaults(func=CmdFreeze)
 
     UNFREEZE_HELP = "Unfreeze stages or .dvc files."
@@ -57,10 +53,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unfreeze_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="Stages or .dvc files to unfreeze",
-        metavar="targets",
-        choices=completion.Required.DVC_FILE,
-    )
+        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
+    ).complete = completion.DVC_FILE
     unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/get.py

@@ -63,19 +63,15 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the repository",
-        metavar="path",
-        choices=completion.Required.FILE,
-    )
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
     get_parser.add_argument(
         "-o",
         "--out",
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     get_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/get_url.py

@@ -34,10 +34,6 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out",
-        nargs="?",
-        help="Destination path to put data to.",
-        metavar="out",
-        choices=completion.Optional.DIR,
-    )
+        "out", nargs="?", help="Destination path to put data to.",
+    ).complete = completion.DIR
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/git_hook.py

@@ -19,6 +19,9 @@ class CmdHookBase(CmdBaseNoRepo):
 
         return self._run()
 
+    def _run(self):
+        raise NotImplementedError
+
 
 class CmdPreCommit(CmdHookBase):
     def _run(self):

dvc/command/imp.py

@@ -44,19 +44,15 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the repository",
-        metavar="path",
-        choices=completion.Required.FILE,
-    )
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
     import_parser.add_argument(
         "-o",
         "--out",
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     import_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/imp_url.py

@@ -58,18 +58,13 @@ def add_parser(subparsers, parent_parser):
         "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out",
-        nargs="?",
-        help="Destination path to put files to.",
-        metavar="out",
-        choices=completion.Optional.DIR,
-    )
+        "out", nargs="?", help="Destination path to put files to.",
+    ).complete = completion.DIR
     import_parser.add_argument(
         "--file",
         help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     import_parser.add_argument(
         "--no-exec",
         action="store_true",

dvc/command/ls/__init__.py

@@ -75,7 +75,5 @@ def add_parser(subparsers, parent_parser):
         "path",
         nargs="?",
         help="Path to directory within the repository to list outputs for",
-        metavar="path",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     list_parser.set_defaults(func=CmdList)

dvc/command/metrics.py

@@ -170,9 +170,7 @@ def add_parser(subparsers, parent_parser):
             "Limit command scope to these metric files. Using -R, "
             "directories to search metric files in can also be given."
         ),
-        metavar="targets",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     metrics_show_parser.add_argument(
         "-a",
         "--all-branches",
@@ -238,8 +236,7 @@ def add_parser(subparsers, parent_parser):
             "directories to search metric files in can also be given."
         ),
         metavar="<paths>",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     metrics_diff_parser.add_argument(
         "-R",
         "--recursive",

dvc/command/move.py

@@ -37,15 +37,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src",
-        help="Source path to a data file or directory.",
-        metavar="src",
-        choices=completion.Required.FILE,
-    )
+        "src", help="Source path to a data file or directory.",
+    ).complete = completion.FILE
     move_parser.add_argument(
-        "dst",
-        help="Destination path.",
-        metavar="dst",
-        choices=completion.Required.FILE,
-    )
+        "dst", help="Destination path.",
+    ).complete = completion.FILE
     move_parser.set_defaults(func=CmdMove)

dvc/command/plots.py

@@ -129,9 +129,7 @@ def add_parser(subparsers, parent_parser):
         "targets",
         nargs="*",
         help="Plots files to visualize. Shows all plots by default.",
-        metavar="targets",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     _add_props_arguments(plots_show_parser)
     _add_output_arguments(plots_show_parser)
     plots_show_parser.set_defaults(func=CmdPlotsShow)
@@ -152,8 +150,7 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="Plots file to visualize. Shows all plots by default.",
         metavar="<path>",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     plots_diff_parser.add_argument(
         "revisions", nargs="*", default=None, help="Git commits to plot from",
     )
@@ -170,11 +167,8 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_modify_parser.add_argument(
-        "target",
-        help="Metric file to set properties to",
-        metavar="target",
-        choices=completion.Required.FILE,
-    )
+        "target", help="Metric file to set properties to",
+    ).complete = completion.FILE
     _add_props_arguments(plots_modify_parser)
     plots_modify_parser.add_argument(
         "--unset",
@@ -198,8 +192,7 @@ def _add_props_arguments(parser):
             )
         ),
         metavar="<path>",
-        choices=completion.Optional.FILE,
-    )
+    ).complete = completion.FILE
     parser.add_argument(
         "-x", default=None, help="Field name for X axis.", metavar="<field>"
     )
@@ -231,8 +224,7 @@ def _add_output_arguments(parser):
         default=None,
         help="Destination path to save plots to",
         metavar="<path>",
-        choices=completion.Optional.DIR,
-    )
+    ).complete = completion.DIR
     parser.add_argument(
         "--show-vega",
         action="store_true",

dvc/command/remove.py

@@ -37,10 +37,6 @@ def add_parser(subparsers, parent_parser):
         help="Remove outputs as well.",
     )
     remove_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="DVC-files to remove.",
-        metavar="targets",
-        choices=completion.Required.DVC_FILE,
-    )
+        "targets", nargs="+", help="DVC-files to remove.",
+    ).complete = completion.DVC_FILE
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -72,9 +72,7 @@ def add_parser(subparsers, parent_parser):
         "targets",
         nargs="*",
         help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
-        metavar="targets",
-        choices=completion.Optional.DVC_FILE,
-    )
+    ).complete = completion.DVC_FILE
     repro_parser.add_argument(
         "-f",
         "--force",

dvc/command/unprotect.py

@@ -33,10 +33,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="Data files/directories to unprotect.",
-        metavar="targets",
-        choices=completion.Required.FILE,
-    )
+        "targets", nargs="+", help="Data files/directories to unprotect.",
+    ).complete = completion.FILE
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -33,12 +33,8 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets",
-        nargs="+",
-        help="DVC-files to update.",
-        metavar="targets",
-        choices=completion.Required.DVC_FILE,
-    )
+        "targets", nargs="+", help="DVC-files to update.",
+    ).complete = completion.DVC_FILE
     update_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/version.py

@@ -122,7 +122,7 @@ class CmdVersion(CmdBaseNoRepo):
 
     @staticmethod
     def get_supported_remotes():
-        from dvc.remote import TREES
+        from dvc.tree import TREES
 
         supported_remotes = []
         for tree_cls in TREES:

dvc/config.py

@@ -232,8 +232,10 @@ class Config(dict):
     CONFIG_LOCAL = "config.local"
 
     def __init__(
-        self, dvc_dir=None, validate=True
+        self, dvc_dir=None, validate=True, tree=None,
     ):  # pylint: disable=super-init-not-called
+        from dvc.tree.local import LocalRemoteTree
+
         self.dvc_dir = dvc_dir
 
         if not dvc_dir:
@@ -246,6 +248,9 @@ class Config(dict):
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
+        self.wtree = LocalRemoteTree(None, {"url": self.dvc_dir})
+        self.tree = tree or self.wtree
+
         self.load(validate=validate)
 
     @classmethod
@@ -304,8 +309,32 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
+    def _load_config(self, level):
+        filename = self.files[level]
+        tree = self.tree if level == "repo" else self.wtree
+
+        if tree.exists(filename):
+            with tree.open(filename) as fobj:
+                conf_obj = configobj.ConfigObj(fobj)
+        else:
+            conf_obj = configobj.ConfigObj()
+        return _parse_remotes(_lower_keys(conf_obj.dict()))
+
+    def _save_config(self, level, conf_dict):
+        filename = self.files[level]
+        tree = self.tree if level == "repo" else self.wtree
+
+        logger.debug(f"Writing '{filename}'.")
+
+        tree.makedirs(os.path.dirname(filename))
+
+        config = configobj.ConfigObj(_pack_remotes(conf_dict))
+        with tree.open(filename, "wb") as fobj:
+            config.write(fobj)
+        config.filename = filename
+
     def load_one(self, level):
-        conf = _load_config(self.files[level])
+        conf = self._load_config(level)
         conf = self._load_paths(conf, self.files[level])
 
         # Auto-verify sections
@@ -375,7 +404,7 @@ class Config(dict):
         _merge(merged_conf, conf)
         self.validate(merged_conf)
 
-        _save_config(self.files[level], conf)
+        self._save_config(level, conf)
         self.load()
 
     @staticmethod
@@ -386,20 +415,6 @@ class Config(dict):
             raise ConfigError(str(exc)) from None
 
 
-def _load_config(filename):
-    conf_obj = configobj.ConfigObj(filename)
-    return _parse_remotes(_lower_keys(conf_obj.dict()))
-
-
-def _save_config(filename, conf_dict):
-    logger.debug(f"Writing '{filename}'.")
-    os.makedirs(os.path.dirname(filename), exist_ok=True)
-
-    config = configobj.ConfigObj(_pack_remotes(conf_dict))
-    config.filename = filename
-    config.write()
-
-
 def _parse_remotes(conf):
     result = {"remote": {}}
 

dvc/data_cloud.py

@@ -92,7 +92,7 @@ class DataCloud:
 
     def _save_pulled_checksums(self, cache):
         for checksum in cache.scheme_keys("local"):
-            cache_file = self.repo.cache.local.hash_to_path_info(checksum)
+            cache_file = self.repo.cache.local.tree.hash_to_path_info(checksum)
             if self.repo.cache.local.tree.exists(cache_file):
                 # We can safely save here, as existing corrupted files will
                 # be removed upon status, while files corrupted during

dvc/dependency/__init__.py

@@ -12,9 +12,9 @@ from dvc.dependency.param import ParamsDependency
 from dvc.dependency.s3 import S3Dependency
 from dvc.dependency.ssh import SSHDependency
 from dvc.output.base import BaseOutput
-from dvc.remote import get_remote
 from dvc.scheme import Schemes
 
+from ..tree import get_cloud_tree
 from .repo import RepoDependency
 
 DEPS = [
@@ -54,8 +54,8 @@ SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 def _get(stage, p, info):
     parsed = urlparse(p) if p else None
     if parsed and parsed.scheme == "remote":
-        remote = get_remote(stage.repo, name=parsed.netloc)
-        return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
+        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
+        return DEP_MAP[tree.scheme](stage, p, info, tree=tree)
 
     if info and info.get(RepoDependency.PARAM_REPO):
         repo = info.pop(RepoDependency.PARAM_REPO)

dvc/dependency/azure.py

@@ -1,6 +1,7 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
-from dvc.remote.azure import AzureRemoteTree
+
+from ..tree.azure import AzureRemoteTree
 
 
 class AzureDependency(BaseDependency, BaseOutput):

dvc/dependency/http.py

@@ -1,6 +1,7 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
-from dvc.remote.http import HTTPRemoteTree
+
+from ..tree.http import HTTPRemoteTree
 
 
 class HTTPDependency(BaseDependency, BaseOutput):

dvc/dependency/https.py

@@ -1,5 +1,4 @@
-from dvc.remote.https import HTTPSRemoteTree
-
+from ..tree.https import HTTPSRemoteTree
 from .http import HTTPDependency
 
 

dvc/dependency/param.py

@@ -45,9 +45,8 @@ class ParamsDependency(LocalDependency):
         if not values:
             return
         for param in self.params:
-            value = values.get(param)
-            if value:
-                self.info[param] = value
+            if param in values:
+                self.info[param] = values[param]
 
     def save(self):
         super().save()

dvc/dependency/repo.py

@@ -26,7 +26,7 @@ class RepoDependency(LocalDependency):
         self.def_repo = def_repo
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, remote, path):
+    def _parse_path(self, tree, path):
         return None
 
     @property
@@ -64,7 +64,7 @@ class RepoDependency(LocalDependency):
 
                 # We are polluting our repo cache with some dir listing here
                 if tree.isdir(path):
-                    return self.repo.cache.local.get_hash(path, tree)
+                    return self.repo.cache.local.tree.get_hash(path, tree=tree)
                 return tree.get_file_hash(path)
 
     def status(self):
@@ -98,3 +98,9 @@ class RepoDependency(LocalDependency):
 
         with self._make_repo(locked=False) as repo:
             self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
+
+    def changed_checksum(self):
+        # From current repo point of view what describes RepoDependency is its
+        # origin project url and rev_lock, and it makes RepoDependency
+        # immutable, hence its impossible for checksum to change.
+        return False

dvc/dvcfile.py

@@ -108,7 +108,7 @@ class FileMixin:
         except MultipleInvalid as exc:
             raise StageFileFormatError(f"'{fname}' format error: {exc}")
 
-    def remove(self, force=False):
+    def remove(self, force=False):  # pylint: disable=unused-argument
         with contextlib.suppress(FileNotFoundError):
             os.unlink(self.path)
 
@@ -141,7 +141,7 @@ class SingleStageFile(FileMixin):
         dump_yaml(self.path, serialize.to_single_stage_file(stage))
         self.repo.scm.track_file(self.relpath)
 
-    def remove_stage(self, stage):
+    def remove_stage(self, stage):  # pylint: disable=unused-argument
         self.remove()
 
 
@@ -154,7 +154,9 @@ class PipelineFile(FileMixin):
     def _lockfile(self):
         return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
 
-    def dump(self, stage, update_pipeline=False, no_lock=False):
+    def dump(
+        self, stage, update_pipeline=False, no_lock=False, **kwargs
+    ):  # pylint: disable=arguments-differ
         """Dumps given stage appropriately in the dvcfile."""
         from dvc.stage import PipelineStage
 
@@ -231,7 +233,11 @@ class PipelineFile(FileMixin):
 
         logger.debug("Removing '%s' from '%s'", stage.name, self.path)
         del d["stages"][stage.name]
-        dump_yaml(self.path, d)
+
+        if d["stages"]:
+            dump_yaml(self.path, d)
+        else:
+            super().remove()
 
 
 class Lockfile(FileMixin):
@@ -284,7 +290,10 @@ class Lockfile(FileMixin):
         logger.debug("Removing '%s' from '%s'", stage.name, self.path)
         del d[stage.name]
 
-        dump_yaml(self.path, d)
+        if d:
+            dump_yaml(self.path, d)
+        else:
+            self.remove()
 
 
 class Dvcfile:

dvc/external_repo.py

@@ -68,9 +68,13 @@ def clean_repos():
 
 
 class BaseExternalRepo:
+    # pylint: disable=no-member
 
     _local_cache = None
 
+    def __str__(self):
+        return self.url
+
     @property
     def local_cache(self):
         if hasattr(self, "cache"):

dvc/ignore.py

@@ -3,12 +3,12 @@ import os
 import re
 from itertools import groupby
 
-from funcy import cached_property
 from pathspec.patterns import GitWildMatchPattern
 from pathspec.util import normalize_file
+from pygtrie import StringTrie
 
 from dvc.path_info import PathInfo
-from dvc.scm.tree import BaseTree
+from dvc.pathspec_math import merge_patterns
 from dvc.system import System
 from dvc.utils import relpath
 
@@ -23,39 +23,46 @@ class DvcIgnore:
 
 
 class DvcIgnorePatterns(DvcIgnore):
-    def __init__(self, ignore_file_path, tree):
+    def __init__(self, pattern_list, dirname):
+
+        self.pattern_list = pattern_list
+        self.dirname = dirname
+        self.prefix = self.dirname + os.sep
+
+        regex_pattern_list = map(
+            GitWildMatchPattern.pattern_to_regex, pattern_list
+        )
+
+        self.ignore_spec = [
+            (ignore, re.compile("|".join(item[0] for item in group)))
+            for ignore, group in groupby(regex_pattern_list, lambda x: x[1])
+            if ignore is not None
+        ]
+
+    @classmethod
+    def from_files(cls, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
-
-        self.ignore_file_path = ignore_file_path
-        self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
-
+        dirname = os.path.normpath(os.path.dirname(ignore_file_path))
         with tree.open(ignore_file_path, encoding="utf-8") as fobj:
-            path_spec_lines = fobj.readlines()
-            regex_pattern_list = map(
-                GitWildMatchPattern.pattern_to_regex, path_spec_lines
-            )
-            self.ignore_spec = [
-                (ignore, re.compile("|".join(item[0] for item in group)))
-                for ignore, group in groupby(
-                    regex_pattern_list, lambda x: x[1]
-                )
-                if ignore is not None
+            path_spec_lines = [
+                line for line in map(str.strip, fobj.readlines()) if line
             ]
 
+        return cls(path_spec_lines, dirname)
+
     def __call__(self, root, dirs, files):
         files = [f for f in files if not self.matches(root, f)]
-        dirs = [d for d in dirs if not self.matches(root, d)]
+        dirs = [d for d in dirs if not self.matches(root, d, True)]
 
         return dirs, files
 
-    def matches(self, dirname, basename):
+    def matches(self, dirname, basename, is_dir=False):
         # NOTE: `relpath` is too slow, so we have to assume that both
         # `dirname` and `self.dirname` are relative or absolute together.
-        prefix = self.dirname + os.sep
         if dirname == self.dirname:
             path = basename
-        elif dirname.startswith(prefix):
-            rel = dirname[len(prefix) :]
+        elif dirname.startswith(self.prefix):
+            rel = dirname[len(self.prefix) :]
             # NOTE: `os.path.join` is ~x5.5 slower
             path = f"{rel}{os.sep}{basename}"
         else:
@@ -63,23 +70,63 @@ class DvcIgnorePatterns(DvcIgnore):
 
         if not System.is_unix():
             path = normalize_file(path)
-        return self.ignore(path)
+        return self.ignore(path, is_dir)
 
-    def ignore(self, path):
+    def ignore(self, path, is_dir):
         result = False
-        for ignore, pattern in self.ignore_spec:
-            if pattern.match(path):
-                result = ignore
+        if is_dir:
+            path_dir = f"{path}/"
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path) or pattern.match(path_dir):
+                    result = ignore
+        else:
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path):
+                    result = ignore
         return result
 
     def __hash__(self):
-        return hash(self.ignore_file_path)
+        return hash(self.dirname + ":" + "\n".join(self.pattern_list))
 
     def __eq__(self, other):
         if not isinstance(other, DvcIgnorePatterns):
             return NotImplemented
+        return (self.dirname == other.dirname) & (
+            self.pattern_list == other.pattern_list
+        )
 
-        return self.ignore_file_path == other.ignore_file_path
+    def __bool__(self):
+        return bool(self.pattern_list)
+
+
+class DvcIgnorePatternsTrie(DvcIgnore):
+    trie = None
+
+    def __init__(self):
+        if self.trie is None:
+            self.trie = StringTrie(separator=os.sep)
+
+    def __call__(self, root, dirs, files):
+        ignore_pattern = self[root]
+        if ignore_pattern:
+            return ignore_pattern(root, dirs, files)
+        return dirs, files
+
+    def __setitem__(self, root, ignore_pattern):
+        base_pattern = self[root]
+        common_dirname, merged_pattern = merge_patterns(
+            base_pattern.dirname,
+            base_pattern.pattern_list,
+            ignore_pattern.dirname,
+            ignore_pattern.pattern_list,
+        )
+        self.trie[root] = DvcIgnorePatterns(merged_pattern, common_dirname)
+
+    def __getitem__(self, root):
+        ignore_pattern = self.trie.longest_prefix(root)
+        if ignore_pattern:
+            return ignore_pattern.value
+        return DvcIgnorePatterns([], root)
 
 
 class DvcIgnoreDirs(DvcIgnore):
@@ -113,6 +160,20 @@ class DvcIgnoreRepo(DvcIgnore):
         return dirs, files
 
 
+class DvcIgnoreFilterNoop:
+    def __init__(self, tree, root_dir):
+        pass
+
+    def __call__(self, root, dirs, files):
+        return dirs, files
+
+    def is_ignored_dir(self, _):
+        return False
+
+    def is_ignored_file(self, _):
+        return False
+
+
 class DvcIgnoreFilter:
     def __init__(self, tree, root_dir):
         self.tree = tree
@@ -121,14 +182,19 @@ class DvcIgnoreFilter:
             DvcIgnoreDirs([".git", ".hg", ".dvc"]),
             DvcIgnoreRepo(),
         }
-        for root, dirs, files in self.tree.walk(self.root_dir):
-            self._update(root)
-            dirs[:], files[:] = self(root, dirs, files)
+        ignore_pattern_trie = DvcIgnorePatternsTrie()
+        for root, dirs, _ in self.tree.walk(self.root_dir):
+            ignore_pattern = self._get_ignore_pattern(root)
+            if ignore_pattern:
+                ignore_pattern_trie[root] = ignore_pattern
+                self.ignores.add(ignore_pattern_trie)
+            dirs[:], _ = self(root, dirs, [])
 
-    def _update(self, dirname):
+    def _get_ignore_pattern(self, dirname):
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
         if self.tree.exists(ignore_file_path):
-            self.ignores.add(DvcIgnorePatterns(ignore_file_path, self.tree))
+            return DvcIgnorePatterns.from_files(ignore_file_path, self.tree)
+        return None
 
     def __call__(self, root, dirs, files):
         for ignore in self.ignores:
@@ -136,115 +202,52 @@ class DvcIgnoreFilter:
 
         return dirs, files
 
+    def is_ignored_dir(self, path):
+        if not self._parents_exist(path):
+            return True
 
-class CleanTree(BaseTree):
-    def __init__(self, tree, tree_root=None):
-        self.tree = tree
-        if tree_root:
-            self._tree_root = tree_root
-        else:
-            self._tree_root = self.tree.tree_root
-
-    @cached_property
-    def dvcignore(self):
-        return DvcIgnoreFilter(self.tree, self.tree_root)
-
-    @property
-    def tree_root(self):
-        return self._tree_root
-
-    def open(self, path, mode="r", encoding="utf-8"):
-        if self.isfile(path):
-            return self.tree.open(path, mode, encoding)
-        raise FileNotFoundError
-
-    def exists(self, path):
-        if self.tree.exists(path) and self._parents_exist(path):
-            if self.tree.isdir(path):
-                return self._valid_dirname(path)
-            return self._valid_filename(path)
-        return False
-
-    def isdir(self, path):
-        return (
-            self.tree.isdir(path)
-            and self._parents_exist(path)
-            and self._valid_dirname(path)
-        )
-
-    def _valid_dirname(self, path):
         path = os.path.abspath(path)
-        if path == self.tree_root:
-            return True
+        if path == self.root_dir:
+            return False
         dirname, basename = os.path.split(path)
-        dirs, _ = self.dvcignore(dirname, [basename], [])
-        if dirs:
+        dirs, _ = self(dirname, [basename], [])
+        return not dirs
+
+    def is_ignored_file(self, path):
+        if not self._parents_exist(path):
             return True
-        return False
 
-    def isfile(self, path):
-        return (
-            self.tree.isfile(path)
-            and self._parents_exist(path)
-            and self._valid_filename(path)
-        )
-
-    def _valid_filename(self, path):
         dirname, basename = os.path.split(os.path.normpath(path))
-        _, files = self.dvcignore(os.path.abspath(dirname), [], [basename])
-        if files:
-            return True
-        return False
-
-    def isexec(self, path):
-        return self.exists(path) and self.tree.isexec(path)
+        _, files = self(os.path.abspath(dirname), [], [basename])
+        return not files
 
     def _parents_exist(self, path):
         from dvc.repo import Repo
 
         path = PathInfo(path)
 
-        # if parent is tree_root or inside a .dvc dir we can skip this check
-        if path.parent == self.tree_root or Repo.DVC_DIR in path.parts:
+        # if parent is root_dir or inside a .dvc dir we can skip this check
+        if path.parent == self.root_dir or Repo.DVC_DIR in path.parts:
             return True
 
-        # paths outside of the CleanTree root should be ignored
-        path = relpath(path, self.tree_root)
+        # paths outside of the repo should be ignored
+        path = relpath(path, self.root_dir)
         if path.startswith("..") or (
             os.name == "nt"
             and not os.path.commonprefix(
-                [os.path.abspath(path), self.tree_root]
+                [os.path.abspath(path), self.root_dir]
             )
         ):
             return False
 
         # check if parent directories are in our ignores, starting from
-        # tree_root
+        # root_dir
         for parent_dir in reversed(PathInfo(path).parents):
             dirname, basename = os.path.split(parent_dir)
             if basename == ".":
-                # parent_dir == tree_root
+                # parent_dir == root_dir
                 continue
-            dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
+            dirs, _ = self(os.path.abspath(dirname), [basename], [])
             if not dirs:
                 return False
         return True
-
-    def walk(self, top, topdown=True, onerror=None):
-        for root, dirs, files in self.tree.walk(
-            top, topdown=topdown, onerror=onerror
-        ):
-            dirs[:], files[:] = self.dvcignore(
-                os.path.abspath(root), dirs, files
-            )
-
-            yield root, dirs, files
-
-    def stat(self, path):
-        if self.exists(path):
-            return self.tree.stat(path)
-        raise FileNotFoundError
-
-    @property
-    def hash_jobs(self):
-        return self.tree.hash_jobs

dvc/lock.py

@@ -119,7 +119,7 @@ class HardlinkLock(flufl.lock.Lock):
     def lockfile(self):
         return self._lockfile
 
-    def lock(self):
+    def lock(self):  # pylint: disable=arguments-differ
         try:
             super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
         except flufl.lock.TimeOutError:

dvc/logger.py

@@ -34,6 +34,7 @@ def addLoggingLevel(levelName, levelNum, methodName=None):
 
     def logForLevel(self, message, *args, **kwargs):
         if self.isEnabledFor(levelNum):
+            # pylint: disable=protected-access
             self._log(levelNum, message, args, **kwargs)
 
     def logToRoot(message, *args, **kwargs):

dvc/main.py

@@ -9,7 +9,7 @@ from dvc.config import ConfigError
 from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
 from dvc.external_repo import clean_repos
 from dvc.logger import FOOTER, disable_other_loggers
-from dvc.remote.pool import close_pools
+from dvc.tree.pool import close_pools
 from dvc.utils import format_link
 
 # Workaround for CPython bug. See [1] and [2] for more info.
@@ -65,8 +65,8 @@ def main(argv=None):
     except DvcException:
         ret = 255
         logger.exception("")
-    except Exception as exc:  # pylint: disable=broad-except
-        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+    except OSError as exc:
+        if exc.errno == errno.EMFILE:
             logger.exception(
                 "too many open files, please visit "
                 "{} to see how to handle this "
@@ -78,10 +78,13 @@ def main(argv=None):
         else:
             logger.exception("unexpected error")
         ret = 255
+    except Exception:  # noqa, pylint: disable=broad-except
+        logger.exception("unexpected error")
+        ret = 255
     finally:
         logger.setLevel(outerLogLevel)
 
-        # Closing pools by-hand to prevent wierd messages when closing SSH
+        # Closing pools by-hand to prevent weird messages when closing SSH
         # connections. See https://github.com/iterative/dvc/issues/3248 for
         # more info.
         close_pools()

dvc/output/__init__.py

@@ -10,12 +10,13 @@ from dvc.output.hdfs import HDFSOutput
 from dvc.output.local import LocalOutput
 from dvc.output.s3 import S3Output
 from dvc.output.ssh import SSHOutput
-from dvc.remote import get_remote
-from dvc.remote.hdfs import HDFSRemoteTree
-from dvc.remote.local import LocalRemoteTree
-from dvc.remote.s3 import S3RemoteTree
 from dvc.scheme import Schemes
 
+from ..tree import get_cloud_tree
+from ..tree.hdfs import HDFSRemoteTree
+from ..tree.local import LocalRemoteTree
+from ..tree.s3 import S3RemoteTree
+
 OUTS = [
     HDFSOutput,
     S3Output,
@@ -66,13 +67,13 @@ def _get(
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":
-        remote = get_remote(stage.repo, name=parsed.netloc)
-        return OUTS_MAP[remote.scheme](
+        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
+        return OUTS_MAP[tree.scheme](
             stage,
             p,
             info,
             cache=cache,
-            remote=remote,
+            tree=tree,
             metric=metric,
             plot=plot,
             persist=persist,
@@ -85,7 +86,7 @@ def _get(
                 p,
                 info,
                 cache=cache,
-                remote=None,
+                tree=None,
                 metric=metric,
                 plot=plot,
                 persist=persist,
@@ -95,7 +96,7 @@ def _get(
         p,
         info,
         cache=cache,
-        remote=None,
+        tree=None,
         metric=metric,
         plot=plot,
         persist=persist,

dvc/output/base.py

@@ -12,7 +12,8 @@ from dvc.exceptions import (
     DvcException,
     RemoteCacheRequiredError,
 )
-from dvc.remote.base import BaseRemoteTree, Remote
+
+from ..tree.base import BaseRemoteTree
 
 logger = logging.getLogger(__name__)
 
@@ -47,7 +48,6 @@ class OutputIsStageFileError(DvcException):
 class BaseOutput:
     IS_DEPENDENCY = False
 
-    REMOTE_CLS = Remote
     TREE_CLS = BaseRemoteTree
 
     PARAM_PATH = "path"
@@ -85,7 +85,7 @@ class BaseOutput:
         stage,
         path,
         info=None,
-        remote=None,
+        tree=None,
         cache=True,
         metric=False,
         plot=False,
@@ -106,24 +106,23 @@ class BaseOutput:
         self.repo = stage.repo if stage else None
         self.def_path = path
         self.info = info
-        if remote:
-            self.remote = remote
+        if tree:
+            self.tree = tree
         else:
-            tree = self.TREE_CLS(self.repo, {})
-            self.remote = self.REMOTE_CLS(tree)
+            self.tree = self.TREE_CLS(self.repo, {})
         self.use_cache = False if self.IS_DEPENDENCY else cache
         self.metric = False if self.IS_DEPENDENCY else metric
         self.plot = False if self.IS_DEPENDENCY else plot
         self.persist = persist
 
-        self.path_info = self._parse_path(remote, path)
+        self.path_info = self._parse_path(tree, path)
         if self.use_cache and self.cache is None:
             raise RemoteCacheRequiredError(self.path_info)
 
-    def _parse_path(self, remote, path):
-        if remote:
+    def _parse_path(self, tree, path):
+        if tree:
             parsed = urlparse(path)
-            return remote.path_info / parsed.path.lstrip("/")
+            return tree.path_info / parsed.path.lstrip("/")
         return self.TREE_CLS.PATH_CLS(path)
 
     def __repr__(self):
@@ -163,33 +162,33 @@ class BaseOutput:
 
     @property
     def cache_path(self):
-        return self.cache.hash_to_path_info(self.checksum).url
+        return self.cache.tree.hash_to_path_info(self.checksum).url
 
     @property
     def checksum_type(self):
-        return self.remote.tree.PARAM_CHECKSUM
+        return self.tree.PARAM_CHECKSUM
 
     @property
     def checksum(self):
-        return self.info.get(self.remote.tree.PARAM_CHECKSUM)
+        return self.info.get(self.tree.PARAM_CHECKSUM)
 
     @checksum.setter
     def checksum(self, checksum):
-        self.info[self.remote.tree.PARAM_CHECKSUM] = checksum
+        self.info[self.tree.PARAM_CHECKSUM] = checksum
 
     def get_checksum(self):
-        return self.remote.get_hash(self.path_info)
+        return self.tree.get_hash(self.path_info)
 
     @property
     def is_dir_checksum(self):
-        return self.remote.is_dir_hash(self.checksum)
+        return self.tree.is_dir_hash(self.checksum)
 
     @property
     def exists(self):
-        return self.remote.tree.exists(self.path_info)
+        return self.tree.exists(self.path_info)
 
     def save_info(self):
-        return self.remote.save_info(self.path_info)
+        return self.tree.save_info(self.path_info)
 
     def changed_checksum(self):
         return self.checksum != self.get_checksum()
@@ -222,13 +221,15 @@ class BaseOutput:
 
     @property
     def is_empty(self):
-        return self.remote.tree.is_empty(self.path_info)
+        return self.tree.is_empty(self.path_info)
 
     def isdir(self):
-        return self.remote.tree.isdir(self.path_info)
+        return self.tree.isdir(self.path_info)
 
     def isfile(self):
-        return self.remote.tree.isfile(self.path_info)
+        return self.tree.isfile(self.path_info)
+
+    # pylint: disable=no-member
 
     def ignore(self):
         if not self.use_scm_ignore:
@@ -245,6 +246,8 @@ class BaseOutput:
 
         self.repo.scm.ignore_remove(self.fspath)
 
+    # pylint: enable=no-member
+
     def save(self):
         if not self.exists:
             raise self.DoesNotExistError(self)
@@ -312,7 +315,7 @@ class BaseOutput:
         raise DvcException(f"verify metric is not supported for {self.scheme}")
 
     def download(self, to):
-        self.remote.tree.download(self.path_info, to.path_info)
+        self.tree.download(self.path_info, to.path_info)
 
     def checkout(
         self,
@@ -338,7 +341,7 @@ class BaseOutput:
         )
 
     def remove(self, ignore_remove=False):
-        self.remote.tree.remove(self.path_info)
+        self.tree.remove(self.path_info)
         if self.scheme != "local":
             return
 
@@ -346,10 +349,11 @@ class BaseOutput:
             self.ignore_remove()
 
     def move(self, out):
+        # pylint: disable=no-member
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore_remove(self.fspath)
 
-        self.remote.tree.move(self.path_info, out.path_info)
+        self.tree.move(self.path_info, out.path_info)
         self.def_path = out.def_path
         self.path_info = out.path_info
         self.save()
@@ -368,7 +372,7 @@ class BaseOutput:
 
     def unprotect(self):
         if self.exists:
-            self.remote.tree.unprotect(self.path_info)
+            self.tree.unprotect(self.path_info)
 
     def get_dir_cache(self, **kwargs):
         if not self.is_dir_checksum:
@@ -428,8 +432,8 @@ class BaseOutput:
         filter_path = str(filter_info) if filter_info else None
         is_win = os.name == "nt"
         for entry in self.dir_cache:
-            checksum = entry[self.remote.tree.PARAM_CHECKSUM]
-            entry_relpath = entry[self.remote.tree.PARAM_RELPATH]
+            checksum = entry[self.tree.PARAM_CHECKSUM]
+            entry_relpath = entry[self.tree.PARAM_RELPATH]
             if is_win:
                 entry_relpath = entry_relpath.replace("/", os.sep)
             entry_path = os.path.join(path, entry_relpath)

dvc/output/gs.py

@@ -1,5 +1,6 @@
 from dvc.output.s3 import S3Output
-from dvc.remote.gs import GSRemoteTree
+
+from ..tree.gs import GSRemoteTree
 
 
 class GSOutput(S3Output):

dvc/output/hdfs.py

@@ -1,5 +1,6 @@
 from dvc.output.base import BaseOutput
-from dvc.remote.hdfs import HDFSRemoteTree
+
+from ..tree.hdfs import HDFSRemoteTree
 
 
 class HDFSOutput(BaseOutput):

dvc/output/local.py

@@ -5,15 +5,16 @@ from urllib.parse import urlparse
 from dvc.exceptions import DvcException
 from dvc.istextfile import istextfile
 from dvc.output.base import BaseOutput
-from dvc.remote.local import LocalRemote, LocalRemoteTree
+from dvc.scm.tree import is_working_tree
 from dvc.utils import relpath
 from dvc.utils.fs import path_isin
 
+from ..tree.local import LocalRemoteTree
+
 logger = logging.getLogger(__name__)
 
 
 class LocalOutput(BaseOutput):
-    REMOTE_CLS = LocalRemote
     TREE_CLS = LocalRemoteTree
     sep = os.sep
 
@@ -22,11 +23,13 @@ class LocalOutput(BaseOutput):
             path = relpath(path, stage.wdir)
 
         super().__init__(stage, path, *args, **kwargs)
+        if self.is_in_repo and self.repo and is_working_tree(self.repo.tree):
+            self.tree = self.repo.tree
 
-    def _parse_path(self, remote, path):
+    def _parse_path(self, tree, path):
         parsed = urlparse(path)
         if parsed.scheme == "remote":
-            p = remote.path_info / parsed.path.lstrip("/")
+            p = tree.path_info / parsed.path.lstrip("/")
         else:
             # NOTE: we can path either from command line or .dvc file,
             # so we should expect both posix and windows style paths.

dvc/output/s3.py

@@ -1,5 +1,6 @@
 from dvc.output.base import BaseOutput
-from dvc.remote.s3 import S3RemoteTree
+
+from ..tree.s3 import S3RemoteTree
 
 
 class S3Output(BaseOutput):

dvc/output/ssh.py

@@ -1,7 +1,7 @@
 from dvc.output.base import BaseOutput
-from dvc.remote.ssh import SSHRemote, SSHRemoteTree
+
+from ..tree.ssh import SSHRemoteTree
 
 
 class SSHOutput(BaseOutput):
-    REMOTE_CLS = SSHRemote
     TREE_CLS = SSHRemoteTree

dvc/path_info.py

@@ -1,3 +1,4 @@
+# pylint: disable=protected-access
 import os
 import pathlib
 import posixpath
@@ -17,7 +18,7 @@ class _BasePath:
         return self.isin_or_eq(other) or other.isin(self)
 
     def isin_or_eq(self, other):
-        return self == other or self.isin(other)
+        return self == other or self.isin(other)  # pylint: disable=no-member
 
 
 class PathInfo(pathlib.PurePath, _BasePath):
@@ -37,7 +38,7 @@ class PathInfo(pathlib.PurePath, _BasePath):
         return cls._from_parts(args)
 
     def as_posix(self):
-        f = self._flavour
+        f = self._flavour  # pylint: disable=no-member
         # Unlike original implementation [1] that uses `str()` we actually need
         # to use `fspath`, because we've overridden `__str__` method to return
         # relative paths, which will break original `as_posix`.
@@ -264,7 +265,7 @@ class HTTPURLInfo(URLInfo):
         params=None,
         query=None,
         fragment=None,
-    ):
+    ):  # pylint: disable=arguments-differ
         assert bool(host) ^ bool(netloc)
 
         if netloc is not None:

dvc/pathspec_math.py

@@ -0,0 +1,85 @@
+# Path Specification Pattern Math
+# Including changing base dir of path specification patterns and merging
+# of two path specification patterns with different base
+# All the operations follow the documents of `gitignore`
+import os
+
+from pathspec.util import normalize_file
+
+
+def _not_ignore(rule):
+    return (True, rule[1:]) if rule.startswith("!") else (False, rule)
+
+
+def _is_comment(rule):
+    return rule.startswith("#")
+
+
+def _remove_slash(rule):
+    if rule.startswith("\\"):
+        return rule[1:]
+    return rule
+
+
+def _match_all_level(rule):
+    if rule[:-1].find("/") >= 0 and not rule.startswith("**/"):
+        if rule.startswith("/"):
+            rule = rule[1:]
+        return False, rule
+    if rule.startswith("**/"):
+        rule = rule[3:]
+    return True, rule
+
+
+def change_rule(rule, rel):
+    rule = rule.strip()
+    if _is_comment(rule):
+        return rule
+    not_ignore, rule = _not_ignore(rule)
+    match_all, rule = _match_all_level(rule)
+    rule = _remove_slash(rule)
+    if not match_all:
+        rule = f"/{rule}"
+    else:
+        rule = f"/**/{rule}"
+    if not_ignore:
+        rule = f"!/{rel}{rule}"
+    else:
+        rule = f"/{rel}{rule}"
+    rule = normalize_file(rule)
+    return rule
+
+
+def _change_dirname(dirname, pattern_list, new_dirname):
+    if new_dirname == dirname:
+        return pattern_list
+    rel = os.path.relpath(dirname, new_dirname)
+    if rel.startswith(".."):
+        raise ValueError("change dirname can only change to parent path")
+
+    return [change_rule(rule, rel) for rule in pattern_list]
+
+
+def merge_patterns(prefix_a, pattern_a, prefix_b, pattern_b):
+    """
+    Merge two path specification patterns.
+
+    This implementation merge two path specification patterns on different
+    bases. It returns the longest common parent directory, and the patterns
+    based on this new base directory.
+    """
+    if not pattern_a:
+        return prefix_b, pattern_b
+    elif not pattern_b:
+        return prefix_a, pattern_a
+
+    longest_common_dir = os.path.commonpath([prefix_a, prefix_b])
+    new_pattern_a = _change_dirname(prefix_a, pattern_a, longest_common_dir)
+    new_pattern_b = _change_dirname(prefix_b, pattern_b, longest_common_dir)
+
+    if len(prefix_a) < len(prefix_b):
+        merged_pattern = new_pattern_a + new_pattern_b
+    else:
+        merged_pattern = new_pattern_b + new_pattern_a
+
+    return longest_common_dir, merged_pattern

dvc/remote/__init__.py

@@ -1,82 +1,7 @@
-import posixpath
-from urllib.parse import urlparse
-
-from dvc.remote.azure import AzureRemoteTree
-from dvc.remote.base import Remote
-from dvc.remote.gdrive import GDriveRemoteTree
-from dvc.remote.gs import GSRemoteTree
-from dvc.remote.hdfs import HDFSRemoteTree
-from dvc.remote.http import HTTPRemoteTree
-from dvc.remote.https import HTTPSRemoteTree
-from dvc.remote.local import LocalRemote, LocalRemoteTree
-from dvc.remote.oss import OSSRemoteTree
-from dvc.remote.s3 import S3RemoteTree
-from dvc.remote.ssh import SSHRemote, SSHRemoteTree
-
-TREES = [
-    AzureRemoteTree,
-    GDriveRemoteTree,
-    GSRemoteTree,
-    HDFSRemoteTree,
-    HTTPRemoteTree,
-    HTTPSRemoteTree,
-    S3RemoteTree,
-    SSHRemoteTree,
-    OSSRemoteTree,
-    # NOTE: LocalRemoteTree is the default
-]
-
-
-def _get_tree(remote_conf):
-    for tree_cls in TREES:
-        if tree_cls.supported(remote_conf):
-            return tree_cls
-    return LocalRemoteTree
-
-
-def _get_conf(repo, **kwargs):
-    name = kwargs.get("name")
-    if name:
-        remote_conf = repo.config["remote"][name.lower()]
-    else:
-        remote_conf = kwargs
-    return _resolve_remote_refs(repo.config, remote_conf)
-
-
-def _resolve_remote_refs(config, remote_conf):
-    # Support for cross referenced remotes.
-    # This will merge the settings, shadowing base ref with remote_conf.
-    # For example, having:
-    #
-    #       dvc remote add server ssh://localhost
-    #       dvc remote modify server user root
-    #       dvc remote modify server ask_password true
-    #
-    #       dvc remote add images remote://server/tmp/pictures
-    #       dvc remote modify images user alice
-    #       dvc remote modify images ask_password false
-    #       dvc remote modify images password asdf1234
-    #
-    # Results on a config dictionary like:
-    #
-    #       {
-    #           "url": "ssh://localhost/tmp/pictures",
-    #           "user": "alice",
-    #           "password": "asdf1234",
-    #           "ask_password": False,
-    #       }
-    parsed = urlparse(remote_conf["url"])
-    if parsed.scheme != "remote":
-        return remote_conf
-
-    base = config["remote"][parsed.netloc]
-    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
-    return {**base, **remote_conf, "url": url}
-
-
-def get_cloud_tree(repo, **kwargs):
-    remote_conf = _get_conf(repo, **kwargs)
-    return _get_tree(remote_conf)(repo, remote_conf)
+from ..tree import get_cloud_tree
+from .base import Remote
+from .local import LocalRemote
+from .ssh import SSHRemote
 
 
 def get_remote(repo, **kwargs):

dvc/remote/base.py

@@ -1,34 +1,19 @@
 import hashlib
-import itertools
 import json
 import logging
-import tempfile
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from copy import copy
-from functools import partial, wraps
-from multiprocessing import cpu_count
-from operator import itemgetter
-from urllib.parse import urlparse
+from functools import wraps
 
 from shortuuid import uuid
 
 import dvc.prompt as prompt
-from dvc.exceptions import (
-    CheckoutError,
-    ConfirmRemoveError,
-    DvcException,
-    DvcIgnoreInCollectedDirError,
-    RemoteCacheRequiredError,
-)
-from dvc.ignore import DvcIgnore
-from dvc.path_info import PathInfo, URLInfo, WindowsPathInfo
+from dvc.exceptions import CheckoutError, ConfirmRemoveError, DvcException
+from dvc.path_info import WindowsPathInfo
 from dvc.progress import Tqdm
 from dvc.remote.index import RemoteIndex, RemoteIndexNoop
 from dvc.remote.slow_link_detection import slow_link_guard
-from dvc.state import StateNoop
-from dvc.utils import tmp_fname
-from dvc.utils.fs import makedirs, move
-from dvc.utils.http import open_url
+
+from ..tree.base import RemoteActionNotImplemented
 
 logger = logging.getLogger(__name__)
 
@@ -46,24 +31,6 @@ STATUS_MAP = {
 }
 
 
-class RemoteCmdError(DvcException):
-    def __init__(self, remote, cmd, ret, err):
-        super().__init__(
-            "{remote} command '{cmd}' finished with non-zero return code"
-            " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
-        )
-
-
-class RemoteActionNotImplemented(DvcException):
-    def __init__(self, action, scheme):
-        m = f"{action} is not supported for {scheme} remotes"
-        super().__init__(m)
-
-
-class RemoteMissingDepsError(DvcException):
-    pass
-
-
 class DirCacheError(DvcException):
     def __init__(self, hash_):
         super().__init__(
@@ -82,623 +49,6 @@ def index_locked(f):
     return wrapper
 
 
-class BaseRemoteTree:
-    scheme = "base"
-    REQUIRES = {}
-    PATH_CLS = URLInfo
-    JOBS = 4 * cpu_count()
-
-    PARAM_RELPATH = "relpath"
-    CHECKSUM_DIR_SUFFIX = ".dir"
-    HASH_JOBS = max(1, min(4, cpu_count() // 2))
-    DEFAULT_VERIFY = False
-    LIST_OBJECT_PAGE_SIZE = 1000
-    TRAVERSE_WEIGHT_MULTIPLIER = 5
-    TRAVERSE_PREFIX_LEN = 3
-    TRAVERSE_THRESHOLD_SIZE = 500000
-    CAN_TRAVERSE = True
-
-    CACHE_MODE = None
-    SHARED_MODE_MAP = {None: (None, None), "group": (None, None)}
-    CHECKSUM_DIR_SUFFIX = ".dir"
-
-    state = StateNoop()
-
-    def __init__(self, repo, config):
-        self.repo = repo
-        self.config = config
-
-        self._check_requires(config)
-
-        shared = config.get("shared")
-        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
-
-        self.hash_jobs = (
-            config.get("hash_jobs")
-            or (self.repo and self.repo.config["core"].get("hash_jobs"))
-            or self.HASH_JOBS
-        )
-        self.verify = config.get("verify", self.DEFAULT_VERIFY)
-
-    @classmethod
-    def get_missing_deps(cls):
-        import importlib
-
-        missing = []
-        for package, module in cls.REQUIRES.items():
-            try:
-                importlib.import_module(module)
-            except ImportError:
-                missing.append(package)
-
-        return missing
-
-    def _check_requires(self, config):
-        missing = self.get_missing_deps()
-        if not missing:
-            return
-
-        url = config.get("url", f"{self.scheme}://")
-        msg = (
-            "URL '{}' is supported but requires these missing "
-            "dependencies: {}. If you have installed dvc using pip, "
-            "choose one of these options to proceed: \n"
-            "\n"
-            "    1) Install specific missing dependencies:\n"
-            "        pip install {}\n"
-            "    2) Install dvc package that includes those missing "
-            "dependencies: \n"
-            "        pip install 'dvc[{}]'\n"
-            "    3) Install dvc package with all possible "
-            "dependencies included: \n"
-            "        pip install 'dvc[all]'\n"
-            "\n"
-            "If you have installed dvc from a binary package and you "
-            "are still seeing this message, please report it to us "
-            "using https://github.com/iterative/dvc/issues. Thank you!"
-        ).format(url, missing, " ".join(missing), self.scheme)
-        raise RemoteMissingDepsError(msg)
-
-    @classmethod
-    def supported(cls, config):
-        if isinstance(config, (str, bytes)):
-            url = config
-        else:
-            url = config["url"]
-
-        # NOTE: silently skipping remote, calling code should handle that
-        parsed = urlparse(url)
-        return parsed.scheme == cls.scheme
-
-    @property
-    def file_mode(self):
-        return self._file_mode
-
-    @property
-    def dir_mode(self):
-        return self._dir_mode
-
-    @property
-    def cache(self):
-        return getattr(self.repo.cache, self.scheme)
-
-    def open(self, path_info, mode="r", encoding=None):
-        if hasattr(self, "_generate_download_url"):
-            get_url = partial(self._generate_download_url, path_info)
-            return open_url(get_url, mode=mode, encoding=encoding)
-
-        raise RemoteActionNotImplemented("open", self.scheme)
-
-    def exists(self, path_info):
-        raise NotImplementedError
-
-    def isdir(self, path_info):
-        """Optional: Overwrite only if the remote has a way to distinguish
-        between a directory and a file.
-        """
-        return False
-
-    def isfile(self, path_info):
-        """Optional: Overwrite only if the remote has a way to distinguish
-        between a directory and a file.
-        """
-        return True
-
-    def iscopy(self, path_info):
-        """Check if this file is an independent copy."""
-        return False  # We can't be sure by default
-
-    def walk_files(self, path_info, **kwargs):
-        """Return a generator with `PathInfo`s to all the files"""
-        raise NotImplementedError
-
-    def is_empty(self, path_info):
-        return False
-
-    def remove(self, path_info):
-        raise RemoteActionNotImplemented("remove", self.scheme)
-
-    def makedirs(self, path_info):
-        """Optional: Implement only if the remote needs to create
-        directories before copying/linking/moving data
-        """
-
-    def move(self, from_info, to_info, mode=None):
-        assert mode is None
-        self.copy(from_info, to_info)
-        self.remove(from_info)
-
-    def copy(self, from_info, to_info):
-        raise RemoteActionNotImplemented("copy", self.scheme)
-
-    def copy_fobj(self, fobj, to_info):
-        raise RemoteActionNotImplemented("copy_fobj", self.scheme)
-
-    def symlink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("symlink", self.scheme)
-
-    def hardlink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("hardlink", self.scheme)
-
-    def reflink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("reflink", self.scheme)
-
-    @staticmethod
-    def protect(path_info):
-        pass
-
-    def is_protected(self, path_info):
-        return False
-
-    @staticmethod
-    def unprotect(path_info):
-        pass
-
-    @classmethod
-    def is_dir_hash(cls, hash_):
-        if not hash_:
-            return False
-        return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
-
-    def get_hash(self, path_info, tree=None, **kwargs):
-        assert isinstance(path_info, str) or path_info.scheme == self.scheme
-
-        if not tree:
-            tree = self
-
-        if not tree.exists(path_info):
-            return None
-
-        if tree == self:
-            # pylint: disable=assignment-from-none
-            hash_ = self.state.get(path_info)
-        else:
-            hash_ = None
-        # If we have dir hash in state db, but dir cache file is lost,
-        # then we need to recollect the dir via .get_dir_hash() call below,
-        # see https://github.com/iterative/dvc/issues/2219 for context
-        if (
-            hash_
-            and self.is_dir_hash(hash_)
-            and not tree.exists(self.cache.hash_to_path_info(hash_))
-        ):
-            hash_ = None
-
-        if hash_:
-            return hash_
-
-        if tree.isdir(path_info):
-            hash_ = self.get_dir_hash(path_info, tree, **kwargs)
-        else:
-            hash_ = tree.get_file_hash(path_info)
-
-        if hash_ and self.exists(path_info):
-            self.state.save(path_info, hash_)
-
-        return hash_
-
-    def get_file_hash(self, path_info):
-        raise NotImplementedError
-
-    def get_dir_hash(self, path_info, tree, **kwargs):
-        if not self.cache:
-            raise RemoteCacheRequiredError(path_info)
-
-        dir_info = self._collect_dir(path_info, tree, **kwargs)
-        return self._save_dir_info(dir_info, path_info)
-
-    def hash_to_path_info(self, hash_):
-        return self.path_info / hash_[0:2] / hash_[2:]
-
-    def path_to_hash(self, path):
-        parts = self.PATH_CLS(path).parts[-2:]
-
-        if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
-            raise ValueError(f"Bad cache file path '{path}'")
-
-        return "".join(parts)
-
-    def save_info(self, path_info, tree=None, **kwargs):
-        return {
-            self.PARAM_CHECKSUM: self.get_hash(path_info, tree=tree, **kwargs)
-        }
-
-    @staticmethod
-    def _calculate_hashes(file_infos, tree):
-        file_infos = list(file_infos)
-        with Tqdm(
-            total=len(file_infos),
-            unit="md5",
-            desc="Computing file/dir hashes (only done once)",
-        ) as pbar:
-            worker = pbar.wrap_fn(tree.get_file_hash)
-            with ThreadPoolExecutor(max_workers=tree.hash_jobs) as executor:
-                tasks = executor.map(worker, file_infos)
-                hashes = dict(zip(file_infos, tasks))
-        return hashes
-
-    def _collect_dir(self, path_info, tree, **kwargs):
-        file_infos = set()
-
-        for fname in tree.walk_files(path_info, **kwargs):
-            if DvcIgnore.DVCIGNORE_FILE == fname.name:
-                raise DvcIgnoreInCollectedDirError(fname.parent)
-
-            file_infos.add(fname)
-
-        hashes = {fi: self.state.get(fi) for fi in file_infos}
-        not_in_state = {fi for fi, hash_ in hashes.items() if hash_ is None}
-
-        new_hashes = self._calculate_hashes(not_in_state, tree)
-        hashes.update(new_hashes)
-
-        result = [
-            {
-                self.PARAM_CHECKSUM: hashes[fi],
-                # NOTE: this is lossy transformation:
-                #   "hey\there" -> "hey/there"
-                #   "hey/there" -> "hey/there"
-                # The latter is fine filename on Windows, which
-                # will transform to dir/file on back transform.
-                #
-                # Yes, this is a BUG, as long as we permit "/" in
-                # filenames on Windows and "\" on Unix
-                self.PARAM_RELPATH: fi.relative_to(path_info).as_posix(),
-            }
-            for fi in file_infos
-        ]
-
-        # Sorting the list by path to ensure reproducibility
-        return sorted(result, key=itemgetter(self.PARAM_RELPATH))
-
-    def _save_dir_info(self, dir_info, path_info):
-        hash_, tmp_info = self._get_dir_info_hash(dir_info)
-        new_info = self.cache.hash_to_path_info(hash_)
-        if self.cache.changed_cache_file(hash_):
-            self.cache.tree.makedirs(new_info.parent)
-            self.cache.tree.move(
-                tmp_info, new_info, mode=self.cache.CACHE_MODE
-            )
-
-        if self.exists(path_info):
-            self.state.save(path_info, hash_)
-        self.state.save(new_info, hash_)
-
-        return hash_
-
-    def _get_dir_info_hash(self, dir_info):
-        tmp = tempfile.NamedTemporaryFile(delete=False).name
-        with open(tmp, "w+") as fobj:
-            json.dump(dir_info, fobj, sort_keys=True)
-
-        tree = self.cache.tree
-        from_info = PathInfo(tmp)
-        to_info = tree.path_info / tmp_fname("")
-        tree.upload(from_info, to_info, no_progress_bar=True)
-
-        hash_ = tree.get_file_hash(to_info) + self.CHECKSUM_DIR_SUFFIX
-        return hash_, to_info
-
-    def upload(self, from_info, to_info, name=None, no_progress_bar=False):
-        if not hasattr(self, "_upload"):
-            raise RemoteActionNotImplemented("upload", self.scheme)
-
-        if to_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        if from_info.scheme != "local":
-            raise NotImplementedError
-
-        logger.debug("Uploading '%s' to '%s'", from_info, to_info)
-
-        name = name or from_info.name
-
-        self._upload(
-            from_info.fspath,
-            to_info,
-            name=name,
-            no_progress_bar=no_progress_bar,
-        )
-
-    def download(
-        self,
-        from_info,
-        to_info,
-        name=None,
-        no_progress_bar=False,
-        file_mode=None,
-        dir_mode=None,
-    ):
-        if not hasattr(self, "_download"):
-            raise RemoteActionNotImplemented("download", self.scheme)
-
-        if from_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        if to_info.scheme == self.scheme != "local":
-            self.copy(from_info, to_info)
-            return 0
-
-        if to_info.scheme != "local":
-            raise NotImplementedError
-
-        if self.isdir(from_info):
-            return self._download_dir(
-                from_info, to_info, name, no_progress_bar, file_mode, dir_mode
-            )
-        return self._download_file(
-            from_info, to_info, name, no_progress_bar, file_mode, dir_mode
-        )
-
-    def _download_dir(
-        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
-    ):
-        from_infos = list(self.walk_files(from_info))
-        to_infos = (
-            to_info / info.relative_to(from_info) for info in from_infos
-        )
-
-        with Tqdm(
-            total=len(from_infos),
-            desc="Downloading directory",
-            unit="Files",
-            disable=no_progress_bar,
-        ) as pbar:
-            download_files = pbar.wrap_fn(
-                partial(
-                    self._download_file,
-                    name=name,
-                    no_progress_bar=True,
-                    file_mode=file_mode,
-                    dir_mode=dir_mode,
-                )
-            )
-            with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
-                futures = [
-                    executor.submit(download_files, from_info, to_info)
-                    for from_info, to_info in zip(from_infos, to_infos)
-                ]
-
-                # NOTE: unlike pulling/fetching cache, where we need to
-                # download everything we can, not raising an error here might
-                # turn very ugly, as the user might think that he has
-                # downloaded a complete directory, while having a partial one,
-                # which might cause unexpected results in his pipeline.
-                for future in as_completed(futures):
-                    # NOTE: executor won't let us raise until all futures that
-                    # it has are finished, so we need to cancel them ourselves
-                    # before re-raising.
-                    exc = future.exception()
-                    if exc:
-                        for entry in futures:
-                            entry.cancel()
-                        raise exc
-
-    def _download_file(
-        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
-    ):
-        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
-
-        logger.debug("Downloading '%s' to '%s'", from_info, to_info)
-        name = name or to_info.name
-
-        tmp_file = tmp_fname(to_info)
-
-        self._download(
-            from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
-        )
-
-        move(tmp_file, to_info, mode=file_mode)
-
-    def list_paths(self, prefix=None, progress_callback=None):
-        if prefix:
-            if len(prefix) > 2:
-                path_info = self.path_info / prefix[:2] / prefix[2:]
-            else:
-                path_info = self.path_info / prefix[:2]
-        else:
-            path_info = self.path_info
-        if progress_callback:
-            for file_info in self.walk_files(path_info):
-                progress_callback()
-                yield file_info.path
-        else:
-            yield from self.walk_files(path_info)
-
-    def list_hashes(self, prefix=None, progress_callback=None):
-        """Iterate over hashes in this tree.
-
-        If `prefix` is specified, only hashes which begin with `prefix`
-        will be returned.
-        """
-        for path in self.list_paths(prefix, progress_callback):
-            try:
-                yield self.path_to_hash(path)
-            except ValueError:
-                logger.debug(
-                    "'%s' doesn't look like a cache file, skipping", path
-                )
-
-    def all(self, jobs=None, name=None):
-        """Iterate over all hashes in this tree.
-
-        Hashes will be fetched in parallel threads according to prefix
-        (except for small remotes) and a progress bar will be displayed.
-        """
-        logger.debug(
-            "Fetching all hashes from '{}'".format(
-                name if name else "remote cache"
-            )
-        )
-
-        if not self.CAN_TRAVERSE:
-            return self.list_hashes()
-
-        remote_size, remote_hashes = self.estimate_remote_size(name=name)
-        return self.list_hashes_traverse(
-            remote_size, remote_hashes, jobs, name
-        )
-
-    def _hashes_with_limit(self, limit, prefix=None, progress_callback=None):
-        count = 0
-        for hash_ in self.list_hashes(prefix, progress_callback):
-            yield hash_
-            count += 1
-            if count > limit:
-                logger.debug(
-                    "`list_hashes()` returned max '{}' hashes, "
-                    "skipping remaining results".format(limit)
-                )
-                return
-
-    def _max_estimation_size(self, hashes):
-        # Max remote size allowed for us to use traverse method
-        return max(
-            self.TRAVERSE_THRESHOLD_SIZE,
-            len(hashes)
-            / self.TRAVERSE_WEIGHT_MULTIPLIER
-            * self.LIST_OBJECT_PAGE_SIZE,
-        )
-
-    def estimate_remote_size(self, hashes=None, name=None):
-        """Estimate tree size based on number of entries beginning with
-        "00..." prefix.
-        """
-        prefix = "0" * self.TRAVERSE_PREFIX_LEN
-        total_prefixes = pow(16, self.TRAVERSE_PREFIX_LEN)
-        if hashes:
-            max_hashes = self._max_estimation_size(hashes)
-        else:
-            max_hashes = None
-
-        with Tqdm(
-            desc="Estimating size of "
-            + (f"cache in '{name}'" if name else "remote cache"),
-            unit="file",
-        ) as pbar:
-
-            def update(n=1):
-                pbar.update(n * total_prefixes)
-
-            if max_hashes:
-                hashes = self._hashes_with_limit(
-                    max_hashes / total_prefixes, prefix, update
-                )
-            else:
-                hashes = self.list_hashes(prefix, update)
-
-            remote_hashes = set(hashes)
-            if remote_hashes:
-                remote_size = total_prefixes * len(remote_hashes)
-            else:
-                remote_size = total_prefixes
-            logger.debug(f"Estimated remote size: {remote_size} files")
-        return remote_size, remote_hashes
-
-    def list_hashes_traverse(
-        self, remote_size, remote_hashes, jobs=None, name=None
-    ):
-        """Iterate over all hashes found in this tree.
-        Hashes are fetched in parallel according to prefix, except in
-        cases where the remote size is very small.
-
-        All hashes from the remote (including any from the size
-        estimation step passed via the `remote_hashes` argument) will be
-        returned.
-
-        NOTE: For large remotes the list of hashes will be very
-        big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list)
-        and we don't really need all of it at the same time, so it makes
-        sense to use a generator to gradually iterate over it, without
-        keeping all of it in memory.
-        """
-        num_pages = remote_size / self.LIST_OBJECT_PAGE_SIZE
-        if num_pages < 256 / self.JOBS:
-            # Fetching prefixes in parallel requires at least 255 more
-            # requests, for small enough remotes it will be faster to fetch
-            # entire cache without splitting it into prefixes.
-            #
-            # NOTE: this ends up re-fetching hashes that were already
-            # fetched during remote size estimation
-            traverse_prefixes = [None]
-            initial = 0
-        else:
-            yield from remote_hashes
-            initial = len(remote_hashes)
-            traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
-            if self.TRAVERSE_PREFIX_LEN > 2:
-                traverse_prefixes += [
-                    "{0:0{1}x}".format(i, self.TRAVERSE_PREFIX_LEN)
-                    for i in range(1, pow(16, self.TRAVERSE_PREFIX_LEN - 2))
-                ]
-        with Tqdm(
-            desc="Querying "
-            + (f"cache in '{name}'" if name else "remote cache"),
-            total=remote_size,
-            initial=initial,
-            unit="file",
-        ) as pbar:
-
-            def list_with_update(prefix):
-                return list(
-                    self.list_hashes(
-                        prefix=prefix, progress_callback=pbar.update
-                    )
-                )
-
-            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
-                in_remote = executor.map(list_with_update, traverse_prefixes,)
-                yield from itertools.chain.from_iterable(in_remote)
-
-    def list_hashes_exists(self, hashes, jobs=None, name=None):
-        """Return list of the specified hashes which exist in this tree.
-        Hashes will be queried individually.
-        """
-        logger.debug(
-            "Querying {} hashes via object_exists".format(len(hashes))
-        )
-        with Tqdm(
-            desc="Querying "
-            + ("cache in " + name if name else "remote cache"),
-            total=len(hashes),
-            unit="file",
-        ) as pbar:
-
-            def exists_with_progress(path_info):
-                ret = self.exists(path_info)
-                pbar.update_msg(str(path_info))
-                return ret
-
-            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
-                path_infos = map(self.hash_to_path_info, hashes)
-                in_remote = executor.map(exists_with_progress, path_infos)
-                ret = list(itertools.compress(hashes, in_remote))
-                return ret
-
-    def _remove_unpacked_dir(self, hash_):
-        pass
-
-
 class Remote:
     """Cloud remote class.
 
@@ -722,41 +72,15 @@ class Remote:
         else:
             self.index = RemoteIndexNoop()
 
-    @property
-    def path_info(self):
-        return self.tree.path_info
-
     def __repr__(self):
         return "{class_name}: '{path_info}'".format(
             class_name=type(self).__name__,
-            path_info=self.path_info or "No path",
+            path_info=self.tree.path_info or "No path",
         )
 
     @property
     def cache(self):
-        return getattr(self.repo.cache, self.scheme)
-
-    @property
-    def scheme(self):
-        return self.tree.scheme
-
-    def is_dir_hash(self, hash_):
-        return self.tree.is_dir_hash(hash_)
-
-    def get_hash(self, path_info, **kwargs):
-        return self.tree.get_hash(path_info, **kwargs)
-
-    def hash_to_path_info(self, hash_):
-        return self.tree.hash_to_path_info(hash_)
-
-    def path_to_hash(self, path):
-        return self.tree.path_to_hash(path)
-
-    def save_info(self, path_info, **kwargs):
-        return self.tree.save_info(path_info, **kwargs)
-
-    def open(self, *args, **kwargs):
-        return self.tree.open(*args, **kwargs)
+        return getattr(self.repo.cache, self.tree.scheme)
 
     def hashes_exist(self, hashes, jobs=None, name=None):
         """Check if the given hashes are stored in the remote.
@@ -866,6 +190,7 @@ class Remote:
             path_info = tree.hash_to_path_info(hash_)
             if tree.is_dir_hash(hash_):
                 # backward compatibility
+                # pylint: disable=protected-access
                 tree._remove_unpacked_dir(hash_)
             tree.remove(path_info)
             removed = True
@@ -879,7 +204,7 @@ class CloudCache:
     """Cloud cache class."""
 
     DEFAULT_CACHE_TYPES = ["copy"]
-    CACHE_MODE = BaseRemoteTree.CACHE_MODE
+    CACHE_MODE = None
 
     def __init__(self, tree):
         self.tree = tree
@@ -891,39 +216,6 @@ class CloudCache:
         self.cache_type_confirmed = False
         self._dir_info = {}
 
-    @property
-    def path_info(self):
-        return self.tree.path_info
-
-    @property
-    def cache(self):
-        return getattr(self.repo.cache, self.scheme)
-
-    @property
-    def scheme(self):
-        return self.tree.scheme
-
-    @property
-    def state(self):
-        return self.tree.state
-
-    def open(self, *args, **kwargs):
-        return self.tree.open(*args, **kwargs)
-
-    def is_dir_hash(self, hash_):
-        return self.tree.is_dir_hash(hash_)
-
-    def get_hash(self, path_info, **kwargs):
-        return self.tree.get_hash(path_info, **kwargs)
-
-    # Override to return path as a string instead of PathInfo for clouds
-    # which support string paths (see local)
-    def hash_to_path(self, hash_):
-        return self.hash_to_path_info(hash_)
-
-    def hash_to_path_info(self, hash_):
-        return self.tree.hash_to_path_info(hash_)
-
     def get_dir_cache(self, hash_):
         assert hash_
 
@@ -940,10 +232,10 @@ class CloudCache:
         return dir_info
 
     def load_dir_cache(self, hash_):
-        path_info = self.hash_to_path_info(hash_)
+        path_info = self.tree.hash_to_path_info(hash_)
 
         try:
-            with self.cache.open(path_info, "r") as fobj:
+            with self.tree.open(path_info, "r") as fobj:
                 d = json.load(fobj)
         except (ValueError, FileNotFoundError) as exc:
             raise DirCacheError(hash_) from exc
@@ -1000,7 +292,7 @@ class CloudCache:
             logger.debug("cache for '%s'('%s') has changed.", path_info, hash_)
             return True
 
-        actual = self.get_hash(path_info)
+        actual = self.tree.get_hash(path_info)
         if hash_ != actual:
             logger.debug(
                 "hash value '%s' for '%s' has changed (actual '%s').",
@@ -1064,7 +356,7 @@ class CloudCache:
     def _save_file(self, path_info, tree, hash_, save_link=True, **kwargs):
         assert hash_
 
-        cache_info = self.hash_to_path_info(hash_)
+        cache_info = self.tree.hash_to_path_info(hash_)
         if tree == self.tree:
             if self.changed_cache(hash_):
                 self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
@@ -1079,11 +371,11 @@ class CloudCache:
                 self.link(cache_info, path_info)
 
             if save_link:
-                self.state.save_link(path_info)
+                self.tree.state.save_link(path_info)
             # we need to update path and cache, since in case of reflink,
             # or copy cache type moving original file results in updates on
             # next executed command, which causes md5 recalculation
-            self.state.save(path_info, hash_)
+            self.tree.state.save(path_info, hash_)
         else:
             if self.changed_cache(hash_):
                 with tree.open(path_info, mode="rb") as fobj:
@@ -1097,7 +389,7 @@ class CloudCache:
                 if callback:
                     callback(1)
 
-        self.state.save(cache_info, hash_)
+        self.tree.state.save(cache_info, hash_)
         return {self.tree.PARAM_CHECKSUM: hash_}
 
     def _cache_is_copy(self, path_info):
@@ -1109,7 +401,7 @@ class CloudCache:
             return True
 
         workspace_file = path_info.with_name("." + uuid())
-        test_cache_file = self.path_info / ".cache_type_test_file"
+        test_cache_file = self.tree.path_info / ".cache_type_test_file"
         if not self.tree.exists(test_cache_file):
             with self.tree.open(test_cache_file, "wb") as fobj:
                 fobj.write(bytes(1))
@@ -1134,18 +426,19 @@ class CloudCache:
             )
 
         if save_link:
-            self.state.save_link(path_info)
+            self.tree.state.save_link(path_info)
         if self.tree.exists(path_info):
-            self.state.save(path_info, hash_)
+            self.tree.state.save(path_info, hash_)
 
-        cache_info = self.hash_to_path_info(hash_)
-        self.state.save(cache_info, hash_)
+        cache_info = self.tree.hash_to_path_info(hash_)
+        self.tree.state.save(cache_info, hash_)
         return {self.tree.PARAM_CHECKSUM: hash_}
 
     def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
-        if path_info.scheme != self.scheme:
+        if path_info.scheme != self.tree.scheme:
             raise RemoteActionNotImplemented(
-                f"save {path_info.scheme} -> {self.scheme}", self.scheme,
+                f"save {path_info.scheme} -> {self.tree.scheme}",
+                self.tree.scheme,
             )
 
         if not hash_info:
@@ -1154,13 +447,18 @@ class CloudCache:
         return self._save(path_info, tree, hash_, save_link, **kwargs)
 
     def _save(self, path_info, tree, hash_, save_link=True, **kwargs):
-        to_info = self.hash_to_path_info(hash_)
+        to_info = self.tree.hash_to_path_info(hash_)
         logger.debug("Saving '%s' to '%s'.", path_info, to_info)
 
         if tree.isdir(path_info):
             return self._save_dir(path_info, tree, hash_, save_link, **kwargs)
         return self._save_file(path_info, tree, hash_, save_link, **kwargs)
 
+    # Override to return path as a string instead of PathInfo for clouds
+    # which support string paths (see local)
+    def hash_to_path(self, hash_):
+        return self.tree.hash_to_path_info(hash_)
+
     def changed_cache_file(self, hash_):
         """Compare the given hash with the (corresponding) actual one.
 
@@ -1181,7 +479,7 @@ class CloudCache:
             )
             return False
 
-        actual = self.get_hash(cache_info)
+        actual = self.tree.get_hash(cache_info)
 
         logger.debug(
             "cache '%s' expected '%s' actual '%s'", cache_info, hash_, actual,
@@ -1220,14 +518,14 @@ class CloudCache:
         return False
 
     def changed_cache(self, hash_, path_info=None, filter_info=None):
-        if self.is_dir_hash(hash_):
+        if self.tree.is_dir_hash(hash_):
             return self._changed_dir_cache(
                 hash_, path_info=path_info, filter_info=filter_info
             )
         return self.changed_cache_file(hash_)
 
     def already_cached(self, path_info):
-        current = self.get_hash(path_info)
+        current = self.tree.get_hash(path_info)
 
         if not current:
             return False
@@ -1254,15 +552,15 @@ class CloudCache:
     ):
         """The file is changed we need to checkout a new copy"""
         added, modified = True, False
-        cache_info = self.hash_to_path_info(hash_)
+        cache_info = self.tree.hash_to_path_info(hash_)
         if self.tree.exists(path_info):
             logger.debug("data '%s' will be replaced.", path_info)
             self.safe_remove(path_info, force=force)
             added, modified = False, True
 
         self.link(cache_info, path_info)
-        self.state.save_link(path_info)
-        self.state.save(path_info, hash_)
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_)
         if progress_callback:
             progress_callback(str(path_info))
 
@@ -1291,7 +589,7 @@ class CloudCache:
         for entry in dir_info:
             relative_path = entry[self.tree.PARAM_RELPATH]
             entry_hash = entry[self.tree.PARAM_CHECKSUM]
-            entry_cache_info = self.hash_to_path_info(entry_hash)
+            entry_cache_info = self.tree.hash_to_path_info(entry_hash)
             entry_info = path_info / relative_path
 
             if filter_info and not entry_info.isin_or_eq(filter_info):
@@ -1302,7 +600,7 @@ class CloudCache:
                 modified = True
                 self.safe_remove(entry_info, force=force)
                 self.link(entry_cache_info, entry_info)
-                self.state.save(entry_info, entry_hash)
+                self.tree.state.save(entry_info, entry_hash)
             if progress_callback:
                 progress_callback(str(entry_info))
 
@@ -1311,8 +609,8 @@ class CloudCache:
             or modified
         )
 
-        self.state.save_link(path_info)
-        self.state.save(path_info, hash_)
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_)
 
         # relink is not modified, assume it as nochange
         return added, not added and modified and not relink
@@ -1338,7 +636,7 @@ class CloudCache:
         relink=False,
         filter_info=None,
     ):
-        if path_info.scheme not in ["local", self.scheme]:
+        if path_info.scheme not in ["local", self.tree.scheme]:
             raise NotImplementedError
 
         hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
@@ -1371,7 +669,9 @@ class CloudCache:
             if progress_callback:
                 progress_callback(
                     str(path_info),
-                    self.get_files_number(self.path_info, hash_, filter_info),
+                    self.get_files_number(
+                        self.tree.path_info, hash_, filter_info
+                    ),
                 )
             if failed:
                 raise CheckoutError([failed])
@@ -1392,7 +692,7 @@ class CloudCache:
         relink=False,
         filter_info=None,
     ):
-        if not self.is_dir_hash(hash_):
+        if not self.tree.is_dir_hash(hash_):
             return self._checkout_file(
                 path_info, hash_, force, progress_callback, relink
             )
@@ -1407,7 +707,7 @@ class CloudCache:
         if not hash_:
             return 0
 
-        if not self.is_dir_hash(hash_):
+        if not self.tree.is_dir_hash(hash_):
             return 1
 
         if not filter_info:

dvc/remote/local.py

@@ -1,14 +1,12 @@
 import errno
 import logging
 import os
-import stat
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from functools import partial, wraps
 
 from funcy import cached_property, concat
-from shortuuid import uuid
 
-from dvc.exceptions import DownloadError, DvcException, UploadError
+from dvc.exceptions import DownloadError, UploadError
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
 from dvc.remote.base import (
@@ -16,311 +14,17 @@ from dvc.remote.base import (
     STATUS_MAP,
     STATUS_MISSING,
     STATUS_NEW,
-    BaseRemoteTree,
     CloudCache,
     Remote,
     index_locked,
 )
 from dvc.remote.index import RemoteIndexNoop
-from dvc.scheme import Schemes
-from dvc.scm.tree import WorkingTree, is_working_tree
-from dvc.system import System
-from dvc.utils import file_md5, relpath, tmp_fname
-from dvc.utils.fs import (
-    copy_fobj_to_file,
-    copyfile,
-    makedirs,
-    move,
-    remove,
-    walk_files,
-)
+
+from ..tree.local import LocalRemoteTree
 
 logger = logging.getLogger(__name__)
 
 
-class LocalRemoteTree(BaseRemoteTree):
-    scheme = Schemes.LOCAL
-    PATH_CLS = PathInfo
-    PARAM_CHECKSUM = "md5"
-    PARAM_PATH = "path"
-    TRAVERSE_PREFIX_LEN = 2
-    UNPACKED_DIR_SUFFIX = ".unpacked"
-
-    CACHE_MODE = 0o444
-    SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
-
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-        url = config.get("url")
-        self.path_info = self.PATH_CLS(url) if url else None
-
-    @property
-    def state(self):
-        from dvc.state import StateNoop
-
-        return self.repo.state if self.repo else StateNoop()
-
-    @cached_property
-    def work_tree(self):
-        # When using repo.brancher, repo.tree may change to/from WorkingTree to
-        # GitTree arbitarily. When repo.tree is GitTree, local cache needs to
-        # use its own WorkingTree instance.
-        if self.repo:
-            return WorkingTree(self.repo.root_dir)
-        return None
-
-    @staticmethod
-    def open(path_info, mode="r", encoding=None):
-        return open(path_info, mode=mode, encoding=encoding)
-
-    def exists(self, path_info):
-        assert isinstance(path_info, str) or path_info.scheme == "local"
-        if not self.repo:
-            return os.path.exists(path_info)
-        return self.work_tree.exists(path_info)
-
-    def isfile(self, path_info):
-        if not self.repo:
-            return os.path.isfile(path_info)
-        return self.work_tree.isfile(path_info)
-
-    def isdir(self, path_info):
-        if not self.repo:
-            return os.path.isdir(path_info)
-        return self.work_tree.isdir(path_info)
-
-    def iscopy(self, path_info):
-        return not (
-            System.is_symlink(path_info) or System.is_hardlink(path_info)
-        )
-
-    def walk_files(self, path_info, **kwargs):
-        for fname in self.work_tree.walk_files(path_info):
-            yield PathInfo(fname)
-
-    def is_empty(self, path_info):
-        path = path_info.fspath
-
-        if self.isfile(path_info) and os.path.getsize(path) == 0:
-            return True
-
-        if self.isdir(path_info) and len(os.listdir(path)) == 0:
-            return True
-
-        return False
-
-    def remove(self, path_info):
-        if isinstance(path_info, PathInfo):
-            if path_info.scheme != "local":
-                raise NotImplementedError
-            path = path_info.fspath
-        else:
-            path = path_info
-
-        if self.exists(path):
-            remove(path)
-
-    def makedirs(self, path_info):
-        makedirs(path_info, exist_ok=True, mode=self.dir_mode)
-
-    def move(self, from_info, to_info, mode=None):
-        if from_info.scheme != "local" or to_info.scheme != "local":
-            raise NotImplementedError
-
-        self.makedirs(to_info.parent)
-
-        if mode is None:
-            if self.isfile(from_info):
-                mode = self.file_mode
-            else:
-                mode = self.dir_mode
-
-        move(from_info, to_info, mode=mode)
-
-    def copy(self, from_info, to_info):
-        tmp_info = to_info.parent / tmp_fname(to_info.name)
-        try:
-            System.copy(from_info, tmp_info)
-            os.chmod(tmp_info, self.file_mode)
-            os.rename(tmp_info, to_info)
-        except Exception:
-            self.remove(tmp_info)
-            raise
-
-    def copy_fobj(self, fobj, to_info):
-        self.makedirs(to_info.parent)
-        tmp_info = to_info.parent / tmp_fname(to_info.name)
-        try:
-            copy_fobj_to_file(fobj, tmp_info)
-            os.chmod(tmp_info, self.file_mode)
-            os.rename(tmp_info, to_info)
-        except Exception:
-            self.remove(tmp_info)
-            raise
-
-    @staticmethod
-    def symlink(from_info, to_info):
-        System.symlink(from_info, to_info)
-
-    @staticmethod
-    def is_symlink(path_info):
-        return System.is_symlink(path_info)
-
-    def hardlink(self, from_info, to_info):
-        # If there are a lot of empty files (which happens a lot in datasets),
-        # and the cache type is `hardlink`, we might reach link limits and
-        # will get something like: `too many links error`
-        #
-        # This is because all those empty files will have the same hash
-        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
-        # linked to the same file in the cache.
-        #
-        # From https://en.wikipedia.org/wiki/Hard_link
-        #   * ext4 limits the number of hard links on a file to 65,000
-        #   * Windows with NTFS has a limit of 1024 hard links on a file
-        #
-        # That's why we simply create an empty file rather than a link.
-        if self.getsize(from_info) == 0:
-            self.open(to_info, "w").close()
-
-            logger.debug(
-                "Created empty file: {src} -> {dest}".format(
-                    src=str(from_info), dest=str(to_info)
-                )
-            )
-            return
-
-        System.hardlink(from_info, to_info)
-
-    @staticmethod
-    def is_hardlink(path_info):
-        return System.is_hardlink(path_info)
-
-    def reflink(self, from_info, to_info):
-        tmp_info = to_info.parent / tmp_fname(to_info.name)
-        System.reflink(from_info, tmp_info)
-        # NOTE: reflink has its own separate inode, so you can set permissions
-        # that are different from the source.
-        os.chmod(tmp_info, self.file_mode)
-        os.rename(tmp_info, to_info)
-
-    def _unprotect_file(self, path):
-        if System.is_symlink(path) or System.is_hardlink(path):
-            logger.debug(f"Unprotecting '{path}'")
-            tmp = os.path.join(os.path.dirname(path), "." + uuid())
-
-            # The operations order is important here - if some application
-            # would access the file during the process of copyfile then it
-            # would get only the part of file. So, at first, the file should be
-            # copied with the temporary name, and then original file should be
-            # replaced by new.
-            copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
-            remove(path)
-            os.rename(tmp, path)
-
-        else:
-            logger.debug(
-                "Skipping copying for '{}', since it is not "
-                "a symlink or a hardlink.".format(path)
-            )
-
-        os.chmod(path, self.file_mode)
-
-    def _unprotect_dir(self, path):
-        assert is_working_tree(self.repo.tree)
-
-        for fname in self.repo.tree.walk_files(path):
-            self._unprotect_file(fname)
-
-    def unprotect(self, path_info):
-        path = path_info.fspath
-        if not os.path.exists(path):
-            raise DvcException(f"can't unprotect non-existing data '{path}'")
-
-        if os.path.isdir(path):
-            self._unprotect_dir(path)
-        else:
-            self._unprotect_file(path)
-
-    def protect(self, path_info):
-        path = os.fspath(path_info)
-        mode = self.CACHE_MODE
-
-        try:
-            os.chmod(path, mode)
-        except OSError as exc:
-            # There is nothing we need to do in case of a read-only file system
-            if exc.errno == errno.EROFS:
-                return
-
-            # In shared cache scenario, we might not own the cache file, so we
-            # need to check if cache file is already protected.
-            if exc.errno not in [errno.EPERM, errno.EACCES]:
-                raise
-
-            actual = stat.S_IMODE(os.stat(path).st_mode)
-            if actual != mode:
-                raise
-
-    def is_protected(self, path_info):
-        try:
-            mode = os.stat(path_info).st_mode
-        except FileNotFoundError:
-            return False
-
-        return stat.S_IMODE(mode) == self.CACHE_MODE
-
-    def get_file_hash(self, path_info):
-        return file_md5(path_info)[0]
-
-    @staticmethod
-    def getsize(path_info):
-        return os.path.getsize(path_info)
-
-    def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
-    ):
-        makedirs(to_info.parent, exist_ok=True)
-
-        tmp_file = tmp_fname(to_info)
-        copyfile(
-            from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
-        )
-
-        self.protect(tmp_file)
-        os.rename(tmp_file, to_info)
-
-    @staticmethod
-    def _download(
-        from_info, to_file, name=None, no_progress_bar=False, **_kwargs
-    ):
-        copyfile(
-            from_info, to_file, no_progress_bar=no_progress_bar, name=name
-        )
-
-    def list_paths(self, prefix=None, progress_callback=None):
-        assert self.path_info is not None
-        if prefix:
-            path_info = self.path_info / prefix[:2]
-            if not self.exists(path_info):
-                return
-        else:
-            path_info = self.path_info
-        # NOTE: use utils.fs walk_files since tree.walk_files will not follow
-        # symlinks
-        if progress_callback:
-            for path in walk_files(path_info):
-                progress_callback()
-                yield path
-        else:
-            yield from walk_files(path_info)
-
-    def _remove_unpacked_dir(self, hash_):
-        info = self.hash_to_path_info(hash_)
-        path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
-        self.remove(path_info)
-
-
 def _log_exceptions(func, operation):
     @wraps(func)
     def wrapper(from_info, to_info, *args, **kwargs):
@@ -330,6 +34,7 @@ def _log_exceptions(func, operation):
         except Exception as exc:  # pylint: disable=broad-except
             # NOTE: this means we ran out of file descriptors and there is no
             # reason to try to proceed, as we will hit this error anyways.
+            # pylint: disable=no-member
             if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
                 raise
 
@@ -362,7 +67,7 @@ class LocalCache(CloudCache):
         self.tree.path_info = PathInfo(value) if value else None
 
     @classmethod
-    def supported(cls, config):
+    def supported(cls, config):  # pylint: disable=unused-argument
         return True
 
     @cached_property
@@ -375,7 +80,9 @@ class LocalCache(CloudCache):
         # being ~5.5 times faster.
         return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
 
-    def hashes_exist(self, hashes, jobs=None, name=None):
+    def hashes_exist(
+        self, hashes, jobs=None, name=None
+    ):  # pylint: disable=unused-argument
         return [
             hash_
             for hash_ in Tqdm(
@@ -390,7 +97,7 @@ class LocalCache(CloudCache):
     def already_cached(self, path_info):
         assert path_info.scheme in ["", "local"]
 
-        current_md5 = self.get_hash(path_info)
+        current_md5 = self.tree.get_hash(path_info)
 
         if not current_md5:
             return False
@@ -437,8 +144,10 @@ class LocalCache(CloudCache):
         {dir_hash: set(file_hash, ...)} which can be used to map
         a .dir file to its file contents.
         """
-        logger.debug(f"Preparing to collect status from {remote.path_info}")
-        md5s = set(named_cache.scheme_keys(self.scheme))
+        logger.debug(
+            f"Preparing to collect status from {remote.tree.path_info}"
+        )
+        md5s = set(named_cache.scheme_keys(self.tree.scheme))
 
         logger.debug("Collecting information from local cache...")
         local_exists = frozenset(
@@ -454,7 +163,7 @@ class LocalCache(CloudCache):
         else:
             logger.debug("Collecting information from remote cache...")
             remote_exists = set()
-            dir_md5s = set(named_cache.dir_keys(self.scheme))
+            dir_md5s = set(named_cache.dir_keys(self.tree.scheme))
             if dir_md5s:
                 remote_exists.update(
                     self._indexed_dir_hashes(named_cache, remote, dir_md5s)
@@ -463,7 +172,7 @@ class LocalCache(CloudCache):
             if md5s:
                 remote_exists.update(
                     remote.hashes_exist(
-                        md5s, jobs=jobs, name=str(remote.path_info)
+                        md5s, jobs=jobs, name=str(remote.tree.path_info)
                     )
                 )
         return self._make_status(
@@ -479,7 +188,7 @@ class LocalCache(CloudCache):
         dir_status = {}
         file_status = {}
         dir_contents = {}
-        for hash_, item in named_cache[self.scheme].items():
+        for hash_, item in named_cache[self.tree.scheme].items():
             if item.children:
                 dir_status[hash_] = make_names(hash_, item.names)
                 dir_contents[hash_] = set()
@@ -524,7 +233,9 @@ class LocalCache(CloudCache):
         # If .dir hash exists on the remote, assume directory contents
         # still exists on the remote
         for dir_hash in dir_exists:
-            file_hashes = list(named_cache.child_keys(self.scheme, dir_hash))
+            file_hashes = list(
+                named_cache.child_keys(self.tree.scheme, dir_hash)
+            )
             if dir_hash not in remote.index:
                 logger.debug(
                     "Indexing new .dir '{}' with '{}' nested files".format(
@@ -554,8 +265,8 @@ class LocalCache(CloudCache):
             status_info.items(), desc="Analysing status", unit="file"
         ):
             if info["status"] == status:
-                cache.append(self.hash_to_path_info(md5))
-                path_infos.append(remote.hash_to_path_info(md5))
+                cache.append(self.tree.hash_to_path_info(md5))
+                path_infos.append(remote.tree.hash_to_path_info(md5))
                 names.append(info["name"])
                 hashes.append(md5)
 
@@ -579,7 +290,7 @@ class LocalCache(CloudCache):
         logger.debug(
             "Preparing to {} '{}'".format(
                 "download data from" if download else "upload data to",
-                remote.path_info,
+                remote.tree.path_info,
             )
         )
 

dvc/remote/ssh.py

@@ -0,0 +1,73 @@
+import errno
+import itertools
+import logging
+from concurrent.futures import ThreadPoolExecutor
+
+from dvc.progress import Tqdm
+from dvc.utils import to_chunks
+
+from .base import Remote
+
+logger = logging.getLogger(__name__)
+
+
+class SSHRemote(Remote):
+    def batch_exists(self, path_infos, callback):
+        def _exists(chunk_and_channel):
+            chunk, channel = chunk_and_channel
+            ret = []
+            for path in chunk:
+                try:
+                    channel.stat(path)
+                    ret.append(True)
+                except OSError as exc:
+                    if exc.errno != errno.ENOENT:
+                        raise
+                    ret.append(False)
+                callback(path)
+            return ret
+
+        with self.tree.ssh(path_infos[0]) as ssh:
+            channels = ssh.open_max_sftp_channels()
+            max_workers = len(channels)
+
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                paths = [path_info.path for path_info in path_infos]
+                chunks = to_chunks(paths, num_chunks=max_workers)
+                chunks_and_channels = zip(chunks, channels)
+                outcome = executor.map(_exists, chunks_and_channels)
+                results = list(itertools.chain.from_iterable(outcome))
+
+            return results
+
+    def hashes_exist(self, hashes, jobs=None, name=None):
+        """This is older implementation used in remote/base.py
+        We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
+        faster than current approach (relying on exists(path_info)) applied in
+        remote/base.
+        """
+        if not self.tree.CAN_TRAVERSE:
+            return list(set(hashes) & set(self.tree.all()))
+
+        # possibly prompt for credentials before "Querying" progress output
+        self.tree.ensure_credentials()
+
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(hashes),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(chunks):
+                return self.batch_exists(chunks, callback=pbar.update_msg)
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                path_infos = [self.tree.hash_to_path_info(x) for x in hashes]
+                chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
+                results = executor.map(exists_with_progress, chunks)
+                in_remote = itertools.chain.from_iterable(results)
+                ret = list(itertools.compress(hashes, in_remote))
+                return ret

dvc/repo/__init__.py

@@ -14,10 +14,8 @@ from dvc.exceptions import (
     NotDvcRepoError,
     OutputNotFoundError,
 )
-from dvc.ignore import CleanTree
 from dvc.path_info import PathInfo
 from dvc.repo.tree import RepoTree
-from dvc.scm.tree import is_working_tree
 from dvc.utils.fs import path_isin
 
 from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
@@ -31,6 +29,7 @@ def locked(f):
     @wraps(f)
     def wrapper(repo, *args, **kwargs):
         with repo.lock, repo.state:
+            # pylint: disable=protected-access
             repo._reset()
             ret = f(repo, *args, **kwargs)
             # Our graph cache is no longer valid after we release the repo.lock
@@ -76,28 +75,34 @@ class Repo:
         from dvc.repo.metrics import Metrics
         from dvc.repo.plots import Plots
         from dvc.repo.params import Params
-        from dvc.scm.tree import WorkingTree
+        from dvc.tree.local import LocalRemoteTree
         from dvc.utils.fs import makedirs
         from dvc.stage.cache import StageCache
 
         if scm:
-            # use GitTree instead of WorkingTree as default repo tree instance
             tree = scm.get_tree(rev)
             self.root_dir = self.find_root(root_dir, tree)
             self.scm = scm
-            self.tree = tree
+            self.tree = scm.get_tree(
+                rev, use_dvcignore=True, dvcignore_root=self.root_dir
+            )
             self.state = StateNoop()
         else:
             root_dir = self.find_root(root_dir)
             self.root_dir = os.path.abspath(os.path.realpath(root_dir))
+            self.tree = LocalRemoteTree(
+                self,
+                {"url": self.root_dir},
+                use_dvcignore=True,
+                dvcignore_root=self.root_dir,
+            )
 
         self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
-        self.config = Config(self.dvc_dir)
+        self.config = Config(self.dvc_dir, tree=self.tree)
 
         if not scm:
             no_scm = self.config["core"].get("no_scm", False)
             self.scm = SCM(self.root_dir, no_scm=no_scm)
-            self.tree = WorkingTree(self.root_dir)
 
         self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
         self.index_dir = os.path.join(self.tmp_dir, "index")
@@ -134,13 +139,7 @@ class Repo:
 
     @tree.setter
     def tree(self, tree):
-        if is_working_tree(tree) or tree.tree_root == self.root_dir:
-            root = None
-        else:
-            root = self.root_dir
-        self._tree = (
-            tree if isinstance(tree, CleanTree) else CleanTree(tree, root)
-        )
+        self._tree = tree
         # Our graph cache is no longer valid, as it was based on the previous
         # tree.
         self._reset()
@@ -605,4 +604,4 @@ class Repo:
         self.__dict__.pop("graph", None)
         self.__dict__.pop("stages", None)
         self.__dict__.pop("pipelines", None)
-        self.__dict__.pop("dvcignore", None)
+        self.tree.__dict__.pop("dvcignore", None)

dvc/repo/add.py

@@ -109,17 +109,17 @@ def add(
 def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
-            fname
-            for fname in Tqdm(
+            os.fspath(path)
+            for path in Tqdm(
                 repo.tree.walk_files(target),
                 desc="Searching " + target,
                 bar_format=Tqdm.BAR_FMT_NOTOTAL,
                 unit="file",
             )
-            if not repo.is_dvc_internal(fname)
-            if not is_dvc_file(fname)
-            if not repo.scm.belongs_to_scm(fname)
-            if not repo.scm.is_tracked(fname)
+            if not repo.is_dvc_internal(os.fspath(path))
+            if not is_dvc_file(os.fspath(path))
+            if not repo.scm.belongs_to_scm(os.fspath(path))
+            if not repo.scm.is_tracked(os.fspath(path))
         ]
     return [target]
 
@@ -147,7 +147,7 @@ def _create_stages(repo, targets, fname, pbar=None, external=False):
         if stage:
             Dvcfile(repo, stage.path).remove()
 
-        repo._reset()
+        repo._reset()  # pylint: disable=protected-access
 
         if not stage:
             if pbar is not None:

dvc/repo/brancher.py

@@ -1,6 +1,6 @@
 from funcy import group_by
 
-from dvc.scm.tree import WorkingTree
+from dvc.tree.local import LocalRemoteTree
 
 
 def brancher(  # noqa: E302
@@ -29,7 +29,9 @@ def brancher(  # noqa: E302
 
     scm = self.scm
 
-    self.tree = WorkingTree(self.root_dir)
+    self.tree = LocalRemoteTree(
+        self, {"url": self.root_dir}, use_dvcignore=True
+    )
     yield "workspace"
 
     if revs and "workspace" in revs:
@@ -47,7 +49,9 @@ def brancher(  # noqa: E302
     try:
         if revs:
             for sha, names in group_by(scm.resolve_rev, revs).items():
-                self.tree = scm.get_tree(sha)
+                self.tree = scm.get_tree(
+                    sha, use_dvcignore=True, dvcignore_root=self.root_dir
+                )
                 # ignore revs that don't contain repo root
                 # (i.e. revs from before a subdir=True repo was init'ed)
                 if self.tree.exists(self.root_dir):

dvc/repo/diff.py

@@ -37,7 +37,7 @@ def diff(self, a_rev="HEAD", b_rev=None):
 
         def _to_checksum(output):
             if on_working_tree:
-                return self.cache.local.get_hash(output.path_info)
+                return self.cache.local.tree.get_hash(output.path_info)
             return output.checksum
 
         def _exists(output):

dvc/repo/gc.py

@@ -52,10 +52,9 @@ def gc(
     from contextlib import ExitStack
     from dvc.repo import Repo
 
-    all_repos = []
-
-    if repos:
-        all_repos = [Repo(path) for path in repos]
+    if not repos:
+        repos = []
+    all_repos = [Repo(path) for path in repos]
 
     with ExitStack() as stack:
         for repo in all_repos:

dvc/repo/metrics/show.py

@@ -60,19 +60,19 @@ def _read_metrics(repo, metrics, rev):
         if not tree.exists(metric):
             continue
 
-        with tree.open(metric, "r") as fobj:
-            try:
+        try:
+            with tree.open(metric, "r") as fobj:
                 # NOTE this also supports JSON
                 val = yaml.safe_load(fobj)
-            except yaml.YAMLError:
-                logger.debug(
-                    "failed to read '%s' on '%s'", metric, rev, exc_info=True
-                )
-                continue
+        except (FileNotFoundError, yaml.YAMLError):
+            logger.debug(
+                "failed to read '%s' on '%s'", metric, rev, exc_info=True
+            )
+            continue
 
-            val = _extract_metrics(val)
-            if val:
-                res[str(metric)] = val
+        val = _extract_metrics(val)
+        if val:
+            res[str(metric)] = val
 
     return res
 

dvc/repo/plots/data.py

@@ -181,7 +181,7 @@ class CSVPlotData(PlotData):
         super().__init__(filename, revision, content)
         self.delimiter = delimiter
 
-    def raw(self, header=True, **kwargs):
+    def raw(self, header=True, **kwargs):  # pylint: disable=arguments-differ
         first_row = first(csv.reader(io.StringIO(self.content)))
 
         if header:

dvc/repo/plots/template.py

@@ -29,9 +29,13 @@ class Template:
     EXTENSION = ".json"
     ANCHOR = "<DVC_METRIC_{}>"
 
+    DEFAULT_CONTENT = None
+    DEFAULT_NAME = None
+
     def __init__(self, content=None, name=None):
         self.content = self.DEFAULT_CONTENT if content is None else content
         self.name = name or self.DEFAULT_NAME
+        assert self.content and self.name
         self.filename = self.name + self.EXTENSION
 
     def render(self, data, props=None):

dvc/repo/pull.py

@@ -22,7 +22,7 @@ def pull(
     if isinstance(targets, str):
         targets = [targets]
 
-    processed_files_count = self._fetch(
+    processed_files_count = self._fetch(  # pylint: disable=protected-access
         targets,
         jobs,
         remote=remote,
@@ -33,7 +33,7 @@ def pull(
         recursive=recursive,
         run_cache=run_cache,
     )
-    stats = self._checkout(
+    stats = self._checkout(  # pylint: disable=protected-access
         targets=targets, with_deps=with_deps, force=force, recursive=recursive
     )
 

dvc/repo/tree.py

@@ -12,7 +12,7 @@ from dvc.utils.fs import copy_fobj_to_file, makedirs
 logger = logging.getLogger(__name__)
 
 
-class DvcTree(BaseTree):
+class DvcTree(BaseTree):  # pylint:disable=abstract-method
     """DVC repo tree.
 
     Args:
@@ -44,18 +44,21 @@ class DvcTree(BaseTree):
         return outs
 
     def _get_granular_checksum(self, path, out, remote=None):
+        assert isinstance(path, PathInfo)
         if not self.fetch and not self.stream:
             raise FileNotFoundError
         dir_cache = out.get_dir_cache(remote=remote)
         for entry in dir_cache:
-            entry_relpath = entry[out.remote.tree.PARAM_RELPATH]
+            entry_relpath = entry[out.tree.PARAM_RELPATH]
             if os.name == "nt":
                 entry_relpath = entry_relpath.replace("/", os.sep)
             if path == out.path_info / entry_relpath:
-                return entry[out.remote.tree.PARAM_CHECKSUM]
+                return entry[out.tree.PARAM_CHECKSUM]
         raise FileNotFoundError
 
-    def open(self, path, mode="r", encoding="utf-8", remote=None):
+    def open(
+        self, path, mode="r", encoding="utf-8", remote=None
+    ):  # pylint: disable=arguments-differ
         try:
             outs = self._find_outs(path, strict=False)
         except OutputNotFoundError as exc:
@@ -82,8 +85,8 @@ class DvcTree(BaseTree):
                 else:
                     checksum = out.checksum
                 try:
-                    remote_info = remote_obj.hash_to_path_info(checksum)
-                    return remote_obj.open(
+                    remote_info = remote_obj.tree.hash_to_path_info(checksum)
+                    return remote_obj.tree.open(
                         remote_info, mode=mode, encoding=encoding
                     )
                 except RemoteActionNotImplemented:
@@ -93,7 +96,7 @@ class DvcTree(BaseTree):
 
         if out.is_dir_checksum:
             checksum = self._get_granular_checksum(path, out)
-            cache_path = out.cache.hash_to_path_info(checksum).url
+            cache_path = out.cache.tree.hash_to_path_info(checksum).url
         else:
             cache_path = out.cache_path
         return open(cache_path, mode=mode, encoding=encoding)
@@ -126,7 +129,7 @@ class DvcTree(BaseTree):
         # for dir checksum, we need to check if this is a file inside the
         # directory
         try:
-            self._get_granular_checksum(path, out)
+            self._get_granular_checksum(path_info, out)
             return False
         except FileNotFoundError:
             return True
@@ -153,7 +156,7 @@ class DvcTree(BaseTree):
                     download_callback(downloaded)
 
         for entry in dir_cache:
-            entry_relpath = entry[out.remote.tree.PARAM_RELPATH]
+            entry_relpath = entry[out.tree.PARAM_RELPATH]
             if os.name == "nt":
                 entry_relpath = entry_relpath.replace("/", os.sep)
             path_info = out.path_info / entry_relpath
@@ -221,7 +224,7 @@ class DvcTree(BaseTree):
             pass
         return False
 
-    def isexec(self, path):
+    def isexec(self, path):  # pylint: disable=unused-argument
         return False
 
     def get_file_hash(self, path_info):
@@ -234,7 +237,7 @@ class DvcTree(BaseTree):
         return out.checksum
 
 
-class RepoTree(BaseTree):
+class RepoTree(BaseTree):  # pylint:disable=abstract-method
     """DVC + git-tracked files tree.
 
     Args:
@@ -345,7 +348,9 @@ class RepoTree(BaseTree):
             elif dirname in repo_set:
                 yield from self._walk_one(repo_walk)
 
-    def walk(self, top, topdown=True, onerror=None, dvcfiles=False, **kwargs):
+    def walk(
+        self, top, topdown=True, onerror=None, dvcfiles=False, **kwargs
+    ):  # pylint: disable=arguments-differ
         """Walk and merge both DVC and repo trees.
 
         Args:
@@ -424,11 +429,12 @@ class RepoTree(BaseTree):
 
         for root, _, files in self.walk(top):
             root = PathInfo(root)
-            makedirs(dest, exist_ok=True)
+            dest_dir = dest / root.relative_to(top)
+            makedirs(dest_dir, exist_ok=True)
             for fname in files:
                 src = root / fname
                 with self.open(src, mode="rb") as fobj:
-                    copy_fobj_to_file(fobj, dest / fname)
+                    copy_fobj_to_file(fobj, dest_dir / fname)
 
     @property
     def hash_jobs(self):

dvc/scm/git/__init__.py

@@ -119,7 +119,7 @@ class Git(Base):
                     progress=pbar.update_git,
                 )
             tmp_repo.close()
-        except git.exc.GitCommandError as exc:
+        except git.exc.GitCommandError as exc:  # pylint: disable=no-member
             raise CloneError(url, to_path) from exc
 
         # NOTE: using our wrapper to make sure that env is fixed in __init__
@@ -128,7 +128,7 @@ class Git(Base):
         if rev:
             try:
                 repo.checkout(rev)
-            except git.exc.GitCommandError as exc:
+            except git.exc.GitCommandError as exc:  # pylint: disable=no-member
                 raise RevError(
                     "failed to access revision '{}' for repo '{}'".format(
                         rev, url
@@ -375,8 +375,8 @@ class Git(Base):
         path_parts = os.path.normpath(path).split(os.path.sep)
         return basename == self.ignore_file or Git.GIT_DIR in path_parts
 
-    def get_tree(self, rev):
-        return GitTree(self.repo, self.resolve_rev(rev))
+    def get_tree(self, rev, **kwargs):
+        return GitTree(self.repo, self.resolve_rev(rev), **kwargs)
 
     def get_rev(self):
         return self.repo.rev_parse("HEAD").hexsha

dvc/scm/git/tree.py

@@ -3,6 +3,8 @@ import io
 import os
 import stat
 
+from funcy import cached_property
+
 from dvc.exceptions import DvcException
 from dvc.scm.tree import BaseTree
 from dvc.utils import relpath
@@ -19,10 +21,10 @@ def _item_basename(item):
     return os.path.basename(item.path)
 
 
-class GitTree(BaseTree):
+class GitTree(BaseTree):  # pylint:disable=abstract-method
     """Proxies the repo file access methods to Git objects"""
 
-    def __init__(self, git, rev):
+    def __init__(self, git, rev, use_dvcignore=False, dvcignore_root=None):
         """Create GitTree instance
 
         Args:
@@ -31,11 +33,25 @@ class GitTree(BaseTree):
         """
         self.git = git
         self.rev = rev
+        self.use_dvcignore = use_dvcignore
+        self.dvcignore_root = dvcignore_root
 
     @property
     def tree_root(self):
         return self.git.working_dir
 
+    @cached_property
+    def dvcignore(self):
+        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
+
+        root = self.dvcignore_root or self.tree_root
+        if not self.use_dvcignore:
+            return DvcIgnoreFilterNoop(self, root)
+        self.use_dvcignore = False
+        ret = DvcIgnoreFilter(self, root)
+        self.use_dvcignore = True
+        return ret
+
     def open(self, path, mode="r", encoding="utf-8"):
         assert mode in {"r", "rb"}
 
@@ -58,20 +74,29 @@ class GitTree(BaseTree):
         return io.StringIO(data.decode(encoding))
 
     def exists(self, path):
-        return self._git_object_by_path(path) is not None
+        if self._git_object_by_path(path) is None:
+            return False
+
+        return not self.dvcignore.is_ignored_file(
+            path
+        ) and not self.dvcignore.is_ignored_dir(path)
 
     def isdir(self, path):
         obj = self._git_object_by_path(path)
         if obj is None:
             return False
-        return obj.mode == GIT_MODE_DIR
+        if obj.mode != GIT_MODE_DIR:
+            return False
+        return not self.dvcignore.is_ignored_dir(path)
 
     def isfile(self, path):
         obj = self._git_object_by_path(path)
         if obj is None:
             return False
         # according to git-fast-import(1) file mode could be 644 or 755
-        return obj.mode & GIT_MODE_FILE == GIT_MODE_FILE
+        if obj.mode & GIT_MODE_FILE != GIT_MODE_FILE:
+            return False
+        return not self.dvcignore.is_ignored_file(path)
 
     @staticmethod
     def _is_tree_and_contains(obj, path):
@@ -94,7 +119,7 @@ class GitTree(BaseTree):
 
         try:
             tree = self.git.tree(self.rev)
-        except git.exc.BadName as exc:
+        except git.exc.BadName as exc:  # pylint: disable=no-member
             raise DvcException(
                 "revision '{}' not found in Git '{}'".format(
                     self.rev, os.path.relpath(self.git.working_dir)
@@ -145,7 +170,11 @@ class GitTree(BaseTree):
                 onerror(NotADirectoryError(top))
             return
 
-        yield from self._walk(tree, topdown=topdown)
+        for root, dirs, files in self._walk(tree, topdown=topdown):
+            dirs[:], files[:] = self.dvcignore(
+                os.path.abspath(root), dirs, files
+            )
+            yield root, dirs, files
 
     def isexec(self, path):
         if not self.exists(path):

dvc/scm/tree.py

@@ -1,8 +1,4 @@
 import os
-import stat
-from multiprocessing import cpu_count
-
-from funcy import cached_property
 
 
 class BaseTree:
@@ -37,61 +33,13 @@ class BaseTree:
                 # NOTE: os.path.join is ~5.5 times slower
                 yield f"{root}{os.sep}{file}"
 
-
-class WorkingTree(BaseTree):
-    """Proxies the repo file access methods to working tree files"""
-
-    def __init__(self, repo_root=None):
-        repo_root = repo_root or os.getcwd()
-        self.repo_root = repo_root
-
-    @property
-    def tree_root(self):
-        return self.repo_root
-
-    def open(self, path, mode="r", encoding="utf-8"):
-        """Open file and return a stream."""
-        if "b" in mode:
-            encoding = None
-        return open(path, mode=mode, encoding=encoding)
-
-    def exists(self, path):
-        """Test whether a path exists."""
-        return os.path.lexists(path)
-
-    def isdir(self, path):
-        """Return true if the pathname refers to an existing directory."""
-        return os.path.isdir(path)
-
-    def isfile(self, path):
-        """Test whether a path is a regular file"""
-        return os.path.isfile(path)
-
-    def walk(self, top, topdown=True, onerror=None):
-        """Directory tree generator.
-
-        See `os.walk` for the docs. Differences:
-        - no support for symlinks
-        """
-        for root, dirs, files in os.walk(
-            top, topdown=topdown, onerror=onerror
-        ):
-            yield os.path.normpath(root), dirs, files
-
-    def isexec(self, path):
-        mode = os.stat(path).st_mode
-        return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
-
-    @staticmethod
-    def stat(path):
-        return os.stat(path)
-
-    @cached_property
-    def hash_jobs(self):
-        return max(1, min(4, cpu_count() // 2))
+    def makedirs(self, path, mode=0o777, exist_ok=True):
+        raise NotImplementedError
 
 
 def is_working_tree(tree):
-    return isinstance(tree, WorkingTree) or isinstance(
-        getattr(tree, "tree", None), WorkingTree
+    from dvc.tree.local import LocalRemoteTree
+
+    return isinstance(tree, LocalRemoteTree) or isinstance(
+        getattr(tree, "tree", None), LocalRemoteTree
     )

dvc/stage/decorators.py

@@ -15,7 +15,7 @@ def rwlocked(call, read=None, write=None):
     if write is None:
         write = []
 
-    stage = call._args[0]
+    stage = call._args[0]  # pylint: disable=protected-access
 
     assert stage.repo.lock.is_locked
 
@@ -40,7 +40,7 @@ def unlocked_repo(f):
     def wrapper(stage, *args, **kwargs):
         stage.repo.state.dump()
         stage.repo.lock.unlock()
-        stage.repo._reset()
+        stage.repo._reset()  # pylint: disable=protected-access
         try:
             ret = f(stage, *args, **kwargs)
         finally:

dvc/stage/loader.py

@@ -125,7 +125,7 @@ class SingleStageLoader(Mapping):
     def load_stage(cls, dvcfile, d, stage_text):
         path, wdir = resolve_paths(dvcfile.path, d.get(Stage.PARAM_WDIR))
         stage = loads_from(Stage, dvcfile.repo, path, wdir, d)
-        stage._stage_text = stage_text
+        stage._stage_text = stage_text  # noqa, pylint:disable=protected-access
         stage.deps = dependency.loadd_from(
             stage, d.get(Stage.PARAM_DEPS) or []
         )

dvc/stage/run.py

@@ -60,7 +60,10 @@ def cmd_run(stage, *args, **kwargs):
         warn_if_fish(executable)
         cmd = _nix_cmd(executable, stage.cmd)
 
-    main_thread = isinstance(threading.current_thread(), threading._MainThread)
+    main_thread = isinstance(
+        threading.current_thread(),
+        threading._MainThread,  # pylint: disable=protected-access
+    )
     old_handler = None
     p = None
 

dvc/stage/serialize.py

@@ -173,8 +173,9 @@ def to_single_stage_file(stage: "Stage"):
     # - reparse the same yaml text with a slow but smart ruamel yaml parser
     # - apply changes to a returned structure
     # - serialize it
-    if stage._stage_text is not None:
-        saved_state = parse_yaml_for_update(stage._stage_text, stage.path)
+    text = stage._stage_text  # noqa, pylint: disable=protected-access
+    if text is not None:
+        saved_state = parse_yaml_for_update(text, stage.path)
         # Stage doesn't work with meta in any way, so .dumpd() doesn't
         # have it. We simply copy it over.
         if "meta" in saved_state:

dvc/stage/utils.py

@@ -8,8 +8,8 @@ from dvc import dependency, output
 from dvc.utils.fs import path_isin
 
 from ..dependency import ParamsDependency
-from ..remote.local import LocalRemoteTree
-from ..remote.s3 import S3RemoteTree
+from ..tree.local import LocalRemoteTree
+from ..tree.s3 import S3RemoteTree
 from ..utils import dict_md5, format_link, relpath
 from .exceptions import (
     MissingDataSource,

dvc/state.py

@@ -35,7 +35,7 @@ class StateNoop:
     def save(self, path_info, checksum):
         pass
 
-    def get(self, path_info):
+    def get(self, path_info):  # pylint: disable=unused-argument
         return None
 
     def save_link(self, path_info):
@@ -89,10 +89,12 @@ class State:  # pylint: disable=too-many-instance-attributes
     MAX_UINT = 2 ** 64 - 2
 
     def __init__(self, local_cache):
+        from dvc.tree.local import LocalRemoteTree
+
         repo = local_cache.repo
         self.repo = repo
         self.root_dir = repo.root_dir
-        self.tree = local_cache.tree.work_tree
+        self.tree = LocalRemoteTree(None, {"url": self.root_dir})
 
         state_config = repo.config.get("state", {})
         self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
@@ -394,8 +396,8 @@ class State:  # pylint: disable=too-many-instance-attributes
         assert isinstance(path_info, str) or path_info.scheme == "local"
         path = os.fspath(path_info)
 
-        # NOTE: use os.path.exists instead of WorkingTree.exists
-        # WorkingTree.exists uses lexists() and will return True for broken
+        # NOTE: use os.path.exists instead of LocalRemoteTree.exists
+        # because it uses lexists() and will return True for broken
         # symlinks that we cannot stat() in get_mtime_and_size
         if not os.path.exists(path):
             return None

dvc/system.py

@@ -77,7 +77,7 @@ class System:
         )
 
     @staticmethod
-    def _reflink_windows(src, dst):
+    def _reflink_windows(_src, _dst):
         return -1
 
     @staticmethod

dvc/tree/__init__.py

@@ -0,0 +1,78 @@
+import posixpath
+from urllib.parse import urlparse
+
+from .azure import AzureRemoteTree
+from .gdrive import GDriveRemoteTree
+from .gs import GSRemoteTree
+from .hdfs import HDFSRemoteTree
+from .http import HTTPRemoteTree
+from .https import HTTPSRemoteTree
+from .local import LocalRemoteTree
+from .oss import OSSRemoteTree
+from .s3 import S3RemoteTree
+from .ssh import SSHRemoteTree
+
+TREES = [
+    AzureRemoteTree,
+    GDriveRemoteTree,
+    GSRemoteTree,
+    HDFSRemoteTree,
+    HTTPRemoteTree,
+    HTTPSRemoteTree,
+    S3RemoteTree,
+    SSHRemoteTree,
+    OSSRemoteTree,
+    # NOTE: LocalRemoteTree is the default
+]
+
+
+def _get_tree(remote_conf):
+    for tree_cls in TREES:
+        if tree_cls.supported(remote_conf):
+            return tree_cls
+    return LocalRemoteTree
+
+
+def _get_conf(repo, **kwargs):
+    name = kwargs.get("name")
+    if name:
+        remote_conf = repo.config["remote"][name.lower()]
+    else:
+        remote_conf = kwargs
+    return _resolve_remote_refs(repo.config, remote_conf)
+
+
+def _resolve_remote_refs(config, remote_conf):
+    # Support for cross referenced remotes.
+    # This will merge the settings, shadowing base ref with remote_conf.
+    # For example, having:
+    #
+    #       dvc remote add server ssh://localhost
+    #       dvc remote modify server user root
+    #       dvc remote modify server ask_password true
+    #
+    #       dvc remote add images remote://server/tmp/pictures
+    #       dvc remote modify images user alice
+    #       dvc remote modify images ask_password false
+    #       dvc remote modify images password asdf1234
+    #
+    # Results on a config dictionary like:
+    #
+    #       {
+    #           "url": "ssh://localhost/tmp/pictures",
+    #           "user": "alice",
+    #           "password": "asdf1234",
+    #           "ask_password": False,
+    #       }
+    parsed = urlparse(remote_conf["url"])
+    if parsed.scheme != "remote":
+        return remote_conf
+
+    base = config["remote"][parsed.netloc]
+    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
+    return {**base, **remote_conf, "url": url}
+
+
+def get_cloud_tree(repo, **kwargs):
+    remote_conf = _get_conf(repo, **kwargs)
+    return _get_tree(remote_conf)(repo, remote_conf)

dvc/tree/azure.py

@@ -7,16 +7,20 @@ from funcy import cached_property, wrap_prop
 
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree
 from dvc.scheme import Schemes
 
+from .base import BaseRemoteTree
+
 logger = logging.getLogger(__name__)
 
 
 class AzureRemoteTree(BaseRemoteTree):
     scheme = Schemes.AZURE
     PATH_CLS = CloudURLInfo
-    REQUIRES = {"azure-storage-blob": "azure.storage.blob"}
+    REQUIRES = {
+        "azure-storage-blob": "azure.storage.blob",
+        "knack": "knack",
+    }
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
     LIST_OBJECT_PAGE_SIZE = 5000
@@ -28,12 +32,31 @@ class AzureRemoteTree(BaseRemoteTree):
         self.path_info = self.PATH_CLS(url)
 
         if not self.path_info.bucket:
-            container = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
+            container = self._az_config.get("storage", "container_name", None)
             self.path_info = self.PATH_CLS(f"azure://{container}")
 
-        self.connection_string = config.get("connection_string") or os.getenv(
-            "AZURE_STORAGE_CONNECTION_STRING"
+        self._conn_kwargs = {
+            opt: config.get(opt) or self._az_config.get("storage", opt, None)
+            for opt in ["connection_string", "sas_token"]
+        }
+        self._conn_kwargs["account_name"] = self._az_config.get(
+            "storage", "account", None
         )
+        self._conn_kwargs["account_key"] = self._az_config.get(
+            "storage", "key", None
+        )
+
+    @cached_property
+    def _az_config(self):
+        # NOTE: ideally we would've used get_default_cli().config from
+        # azure.cli.core, but azure-cli-core has a lot of conflicts with other
+        # dependencies. So instead we are just use knack directly
+        from knack.config import CLIConfig
+
+        config_dir = os.getenv(
+            "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
+        )
+        return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
 
     @wrap_prop(threading.Lock())
     @cached_property
@@ -43,10 +66,8 @@ class AzureRemoteTree(BaseRemoteTree):
         from azure.common import AzureMissingResourceHttpError
 
         logger.debug(f"URL {self.path_info}")
-        logger.debug(f"Connection string {self.connection_string}")
-        blob_service = BlockBlobService(
-            connection_string=self.connection_string
-        )
+        logger.debug(f"Connection options {self._conn_kwargs}")
+        blob_service = BlockBlobService(**self._conn_kwargs)
         logger.debug(f"Container name {self.path_info.bucket}")
         try:  # verify that container exists
             blob_service.list_blobs(
@@ -101,6 +122,8 @@ class AzureRemoteTree(BaseRemoteTree):
             next_marker = blobs.next_marker
 
     def walk_files(self, path_info, **kwargs):
+        if not kwargs.pop("prefix", False):
+            path_info = path_info / ""
         for fname in self._list_paths(
             path_info.bucket, path_info.path, **kwargs
         ):

dvc/tree/base.py

@@ -0,0 +1,692 @@
+import itertools
+import json
+import logging
+import tempfile
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import partial
+from multiprocessing import cpu_count
+from operator import itemgetter
+from urllib.parse import urlparse
+
+from funcy import cached_property
+
+from dvc.exceptions import (
+    DvcException,
+    DvcIgnoreInCollectedDirError,
+    RemoteCacheRequiredError,
+)
+from dvc.ignore import DvcIgnore
+from dvc.path_info import PathInfo, URLInfo
+from dvc.progress import Tqdm
+from dvc.state import StateNoop
+from dvc.utils import tmp_fname
+from dvc.utils.fs import makedirs, move
+from dvc.utils.http import open_url
+
+logger = logging.getLogger(__name__)
+
+STATUS_OK = 1
+STATUS_MISSING = 2
+STATUS_NEW = 3
+STATUS_DELETED = 4
+
+STATUS_MAP = {
+    # (local_exists, remote_exists)
+    (True, True): STATUS_OK,
+    (False, False): STATUS_MISSING,
+    (True, False): STATUS_NEW,
+    (False, True): STATUS_DELETED,
+}
+
+
+class RemoteCmdError(DvcException):
+    def __init__(self, remote, cmd, ret, err):
+        super().__init__(
+            "{remote} command '{cmd}' finished with non-zero return code"
+            " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
+        )
+
+
+class RemoteActionNotImplemented(DvcException):
+    def __init__(self, action, scheme):
+        m = f"{action} is not supported for {scheme} remotes"
+        super().__init__(m)
+
+
+class RemoteMissingDepsError(DvcException):
+    pass
+
+
+class BaseRemoteTree:
+    scheme = "base"
+    REQUIRES = {}
+    PATH_CLS = URLInfo
+    JOBS = 4 * cpu_count()
+
+    PARAM_RELPATH = "relpath"
+    CHECKSUM_DIR_SUFFIX = ".dir"
+    HASH_JOBS = max(1, min(4, cpu_count() // 2))
+    DEFAULT_VERIFY = False
+    LIST_OBJECT_PAGE_SIZE = 1000
+    TRAVERSE_WEIGHT_MULTIPLIER = 5
+    TRAVERSE_PREFIX_LEN = 3
+    TRAVERSE_THRESHOLD_SIZE = 500000
+    CAN_TRAVERSE = True
+
+    CACHE_MODE = None
+    SHARED_MODE_MAP = {None: (None, None), "group": (None, None)}
+    PARAM_CHECKSUM = None
+
+    state = StateNoop()
+
+    def __init__(self, repo, config):
+        self.repo = repo
+        self.config = config
+
+        self._check_requires(config)
+
+        shared = config.get("shared")
+        self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
+
+        self.verify = config.get("verify", self.DEFAULT_VERIFY)
+        self.path_info = None
+
+    @cached_property
+    def hash_jobs(self):
+        return (
+            self.config.get("hash_jobs")
+            or (self.repo and self.repo.config["core"].get("hash_jobs"))
+            or self.HASH_JOBS
+        )
+
+    @classmethod
+    def get_missing_deps(cls):
+        import importlib
+
+        missing = []
+        for package, module in cls.REQUIRES.items():
+            try:
+                importlib.import_module(module)
+            except ImportError:
+                missing.append(package)
+
+        return missing
+
+    def _check_requires(self, config):
+        missing = self.get_missing_deps()
+        if not missing:
+            return
+
+        url = config.get("url", f"{self.scheme}://")
+        msg = (
+            "URL '{}' is supported but requires these missing "
+            "dependencies: {}. If you have installed dvc using pip, "
+            "choose one of these options to proceed: \n"
+            "\n"
+            "    1) Install specific missing dependencies:\n"
+            "        pip install {}\n"
+            "    2) Install dvc package that includes those missing "
+            "dependencies: \n"
+            "        pip install 'dvc[{}]'\n"
+            "    3) Install dvc package with all possible "
+            "dependencies included: \n"
+            "        pip install 'dvc[all]'\n"
+            "\n"
+            "If you have installed dvc from a binary package and you "
+            "are still seeing this message, please report it to us "
+            "using https://github.com/iterative/dvc/issues. Thank you!"
+        ).format(url, missing, " ".join(missing), self.scheme)
+        raise RemoteMissingDepsError(msg)
+
+    @classmethod
+    def supported(cls, config):
+        if isinstance(config, (str, bytes)):
+            url = config
+        else:
+            url = config["url"]
+
+        # NOTE: silently skipping remote, calling code should handle that
+        parsed = urlparse(url)
+        return parsed.scheme == cls.scheme
+
+    @property
+    def file_mode(self):
+        return self._file_mode
+
+    @property
+    def dir_mode(self):
+        return self._dir_mode
+
+    @property
+    def cache(self):
+        return getattr(self.repo.cache, self.scheme)
+
+    def open(self, path_info, mode="r", encoding=None):
+        if hasattr(self, "_generate_download_url"):
+            func = self._generate_download_url  # noqa,pylint:disable=no-member
+            get_url = partial(func, path_info)
+            return open_url(get_url, mode=mode, encoding=encoding)
+
+        raise RemoteActionNotImplemented("open", self.scheme)
+
+    def exists(self, path_info):
+        raise NotImplementedError
+
+    # pylint: disable=unused-argument
+
+    def isdir(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return False
+
+    def isfile(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return True
+
+    def iscopy(self, path_info):
+        """Check if this file is an independent copy."""
+        return False  # We can't be sure by default
+
+    def walk_files(self, path_info, **kwargs):
+        """Return a generator with `PathInfo`s to all the files.
+
+        Optional kwargs:
+            prefix (bool): If true `path_info` will be treated as a prefix
+                rather than directory path.
+        """
+        raise NotImplementedError
+
+    def is_empty(self, path_info):
+        return False
+
+    def remove(self, path_info):
+        raise RemoteActionNotImplemented("remove", self.scheme)
+
+    def makedirs(self, path_info):
+        """Optional: Implement only if the remote needs to create
+        directories before copying/linking/moving data
+        """
+
+    def move(self, from_info, to_info, mode=None):
+        assert mode is None
+        self.copy(from_info, to_info)
+        self.remove(from_info)
+
+    def copy(self, from_info, to_info):
+        raise RemoteActionNotImplemented("copy", self.scheme)
+
+    def copy_fobj(self, fobj, to_info):
+        raise RemoteActionNotImplemented("copy_fobj", self.scheme)
+
+    def symlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("symlink", self.scheme)
+
+    def hardlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("hardlink", self.scheme)
+
+    def reflink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("reflink", self.scheme)
+
+    @staticmethod
+    def protect(path_info):
+        pass
+
+    def is_protected(self, path_info):
+        return False
+
+    # pylint: enable=unused-argument
+
+    @staticmethod
+    def unprotect(path_info):
+        pass
+
+    @classmethod
+    def is_dir_hash(cls, hash_):
+        if not hash_:
+            return False
+        return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
+
+    def get_hash(self, path_info, tree=None, **kwargs):
+        assert path_info and (
+            isinstance(path_info, str) or path_info.scheme == self.scheme
+        )
+
+        if not tree:
+            tree = self
+
+        if not tree.exists(path_info):
+            return None
+
+        if tree == self:
+            # pylint: disable=assignment-from-none
+            hash_ = self.state.get(path_info)
+        else:
+            hash_ = None
+        # If we have dir hash in state db, but dir cache file is lost,
+        # then we need to recollect the dir via .get_dir_hash() call below,
+        # see https://github.com/iterative/dvc/issues/2219 for context
+        if (
+            hash_
+            and self.is_dir_hash(hash_)
+            and not tree.exists(self.cache.tree.hash_to_path_info(hash_))
+        ):
+            hash_ = None
+
+        if hash_:
+            return hash_
+
+        if tree.isdir(path_info):
+            hash_ = self.get_dir_hash(path_info, tree, **kwargs)
+        else:
+            hash_ = tree.get_file_hash(path_info)
+
+        if hash_ and self.exists(path_info):
+            self.state.save(path_info, hash_)
+
+        return hash_
+
+    def get_file_hash(self, path_info):
+        raise NotImplementedError
+
+    def get_dir_hash(self, path_info, tree, **kwargs):
+        if not self.cache:
+            raise RemoteCacheRequiredError(path_info)
+
+        dir_info = self._collect_dir(path_info, tree, **kwargs)
+        return self._save_dir_info(dir_info, path_info)
+
+    def hash_to_path_info(self, hash_):
+        return self.path_info / hash_[0:2] / hash_[2:]
+
+    def path_to_hash(self, path):
+        parts = self.PATH_CLS(path).parts[-2:]
+
+        if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
+            raise ValueError(f"Bad cache file path '{path}'")
+
+        return "".join(parts)
+
+    def save_info(self, path_info, tree=None, **kwargs):
+        return {
+            self.PARAM_CHECKSUM: self.get_hash(path_info, tree=tree, **kwargs)
+        }
+
+    @staticmethod
+    def _calculate_hashes(file_infos, tree):
+        file_infos = list(file_infos)
+        with Tqdm(
+            total=len(file_infos),
+            unit="md5",
+            desc="Computing file/dir hashes (only done once)",
+        ) as pbar:
+            worker = pbar.wrap_fn(tree.get_file_hash)
+            with ThreadPoolExecutor(max_workers=tree.hash_jobs) as executor:
+                tasks = executor.map(worker, file_infos)
+                hashes = dict(zip(file_infos, tasks))
+        return hashes
+
+    def _collect_dir(self, path_info, tree, **kwargs):
+        file_infos = set()
+
+        for fname in tree.walk_files(path_info, **kwargs):
+            if DvcIgnore.DVCIGNORE_FILE == fname.name:
+                raise DvcIgnoreInCollectedDirError(fname.parent)
+
+            file_infos.add(fname)
+
+        hashes = {fi: self.state.get(fi) for fi in file_infos}
+        not_in_state = {fi for fi, hash_ in hashes.items() if hash_ is None}
+
+        new_hashes = self._calculate_hashes(not_in_state, tree)
+        hashes.update(new_hashes)
+
+        result = [
+            {
+                self.PARAM_CHECKSUM: hashes[fi],
+                # NOTE: this is lossy transformation:
+                #   "hey\there" -> "hey/there"
+                #   "hey/there" -> "hey/there"
+                # The latter is fine filename on Windows, which
+                # will transform to dir/file on back transform.
+                #
+                # Yes, this is a BUG, as long as we permit "/" in
+                # filenames on Windows and "\" on Unix
+                self.PARAM_RELPATH: fi.relative_to(path_info).as_posix(),
+            }
+            for fi in file_infos
+        ]
+
+        # Sorting the list by path to ensure reproducibility
+        return sorted(result, key=itemgetter(self.PARAM_RELPATH))
+
+    def _save_dir_info(self, dir_info, path_info):
+        hash_, tmp_info = self._get_dir_info_hash(dir_info)
+        new_info = self.cache.tree.hash_to_path_info(hash_)
+        if self.cache.changed_cache_file(hash_):
+            self.cache.tree.makedirs(new_info.parent)
+            self.cache.tree.move(
+                tmp_info, new_info, mode=self.cache.CACHE_MODE
+            )
+
+        if self.exists(path_info):
+            self.state.save(path_info, hash_)
+        self.state.save(new_info, hash_)
+
+        return hash_
+
+    def _get_dir_info_hash(self, dir_info):
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        with open(tmp, "w+") as fobj:
+            json.dump(dir_info, fobj, sort_keys=True)
+
+        tree = self.cache.tree
+        from_info = PathInfo(tmp)
+        to_info = tree.path_info / tmp_fname("")
+        tree.upload(from_info, to_info, no_progress_bar=True)
+
+        hash_ = tree.get_file_hash(to_info) + self.CHECKSUM_DIR_SUFFIX
+        return hash_, to_info
+
+    def upload(self, from_info, to_info, name=None, no_progress_bar=False):
+        if not hasattr(self, "_upload"):
+            raise RemoteActionNotImplemented("upload", self.scheme)
+
+        if to_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        if from_info.scheme != "local":
+            raise NotImplementedError
+
+        logger.debug("Uploading '%s' to '%s'", from_info, to_info)
+
+        name = name or from_info.name
+
+        self._upload(  # noqa, pylint: disable=no-member
+            from_info.fspath,
+            to_info,
+            name=name,
+            no_progress_bar=no_progress_bar,
+        )
+
+    def download(
+        self,
+        from_info,
+        to_info,
+        name=None,
+        no_progress_bar=False,
+        file_mode=None,
+        dir_mode=None,
+    ):
+        if not hasattr(self, "_download"):
+            raise RemoteActionNotImplemented("download", self.scheme)
+
+        if from_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        if to_info.scheme == self.scheme != "local":
+            self.copy(from_info, to_info)
+            return 0
+
+        if to_info.scheme != "local":
+            raise NotImplementedError
+
+        if self.isdir(from_info):
+            return self._download_dir(
+                from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+            )
+        return self._download_file(
+            from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+        )
+
+    def _download_dir(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        from_infos = list(self.walk_files(from_info))
+        to_infos = (
+            to_info / info.relative_to(from_info) for info in from_infos
+        )
+
+        with Tqdm(
+            total=len(from_infos),
+            desc="Downloading directory",
+            unit="Files",
+            disable=no_progress_bar,
+        ) as pbar:
+            download_files = pbar.wrap_fn(
+                partial(
+                    self._download_file,
+                    name=name,
+                    no_progress_bar=True,
+                    file_mode=file_mode,
+                    dir_mode=dir_mode,
+                )
+            )
+            with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
+                futures = [
+                    executor.submit(download_files, from_info, to_info)
+                    for from_info, to_info in zip(from_infos, to_infos)
+                ]
+
+                # NOTE: unlike pulling/fetching cache, where we need to
+                # download everything we can, not raising an error here might
+                # turn very ugly, as the user might think that he has
+                # downloaded a complete directory, while having a partial one,
+                # which might cause unexpected results in his pipeline.
+                for future in as_completed(futures):
+                    # NOTE: executor won't let us raise until all futures that
+                    # it has are finished, so we need to cancel them ourselves
+                    # before re-raising.
+                    exc = future.exception()
+                    if exc:
+                        for entry in futures:
+                            entry.cancel()
+                        raise exc
+
+    def _download_file(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
+        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
+
+        logger.debug("Downloading '%s' to '%s'", from_info, to_info)
+        name = name or to_info.name
+
+        tmp_file = tmp_fname(to_info)
+
+        self._download(  # noqa, pylint: disable=no-member
+            from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
+        )
+
+        move(tmp_file, to_info, mode=file_mode)
+
+    def list_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            if len(prefix) > 2:
+                path_info = self.path_info / prefix[:2] / prefix[2:]
+            else:
+                path_info = self.path_info / prefix[:2]
+            prefix = True
+        else:
+            path_info = self.path_info
+            prefix = False
+        if progress_callback:
+            for file_info in self.walk_files(path_info, prefix=prefix):
+                progress_callback()
+                yield file_info.path
+        else:
+            yield from self.walk_files(path_info, prefix=prefix)
+
+    def list_hashes(self, prefix=None, progress_callback=None):
+        """Iterate over hashes in this tree.
+
+        If `prefix` is specified, only hashes which begin with `prefix`
+        will be returned.
+        """
+        for path in self.list_paths(prefix, progress_callback):
+            try:
+                yield self.path_to_hash(path)
+            except ValueError:
+                logger.debug(
+                    "'%s' doesn't look like a cache file, skipping", path
+                )
+
+    def all(self, jobs=None, name=None):
+        """Iterate over all hashes in this tree.
+
+        Hashes will be fetched in parallel threads according to prefix
+        (except for small remotes) and a progress bar will be displayed.
+        """
+        logger.debug(
+            "Fetching all hashes from '{}'".format(
+                name if name else "remote cache"
+            )
+        )
+
+        if not self.CAN_TRAVERSE:
+            return self.list_hashes()
+
+        remote_size, remote_hashes = self.estimate_remote_size(name=name)
+        return self.list_hashes_traverse(
+            remote_size, remote_hashes, jobs, name
+        )
+
+    def _hashes_with_limit(self, limit, prefix=None, progress_callback=None):
+        count = 0
+        for hash_ in self.list_hashes(prefix, progress_callback):
+            yield hash_
+            count += 1
+            if count > limit:
+                logger.debug(
+                    "`list_hashes()` returned max '{}' hashes, "
+                    "skipping remaining results".format(limit)
+                )
+                return
+
+    def _max_estimation_size(self, hashes):
+        # Max remote size allowed for us to use traverse method
+        return max(
+            self.TRAVERSE_THRESHOLD_SIZE,
+            len(hashes)
+            / self.TRAVERSE_WEIGHT_MULTIPLIER
+            * self.LIST_OBJECT_PAGE_SIZE,
+        )
+
+    def estimate_remote_size(self, hashes=None, name=None):
+        """Estimate tree size based on number of entries beginning with
+        "00..." prefix.
+        """
+        prefix = "0" * self.TRAVERSE_PREFIX_LEN
+        total_prefixes = pow(16, self.TRAVERSE_PREFIX_LEN)
+        if hashes:
+            max_hashes = self._max_estimation_size(hashes)
+        else:
+            max_hashes = None
+
+        with Tqdm(
+            desc="Estimating size of "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            unit="file",
+        ) as pbar:
+
+            def update(n=1):
+                pbar.update(n * total_prefixes)
+
+            if max_hashes:
+                hashes = self._hashes_with_limit(
+                    max_hashes / total_prefixes, prefix, update
+                )
+            else:
+                hashes = self.list_hashes(prefix, update)
+
+            remote_hashes = set(hashes)
+            if remote_hashes:
+                remote_size = total_prefixes * len(remote_hashes)
+            else:
+                remote_size = total_prefixes
+            logger.debug(f"Estimated remote size: {remote_size} files")
+        return remote_size, remote_hashes
+
+    def list_hashes_traverse(
+        self, remote_size, remote_hashes, jobs=None, name=None
+    ):
+        """Iterate over all hashes found in this tree.
+        Hashes are fetched in parallel according to prefix, except in
+        cases where the remote size is very small.
+
+        All hashes from the remote (including any from the size
+        estimation step passed via the `remote_hashes` argument) will be
+        returned.
+
+        NOTE: For large remotes the list of hashes will be very
+        big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list)
+        and we don't really need all of it at the same time, so it makes
+        sense to use a generator to gradually iterate over it, without
+        keeping all of it in memory.
+        """
+        num_pages = remote_size / self.LIST_OBJECT_PAGE_SIZE
+        if num_pages < 256 / self.JOBS:
+            # Fetching prefixes in parallel requires at least 255 more
+            # requests, for small enough remotes it will be faster to fetch
+            # entire cache without splitting it into prefixes.
+            #
+            # NOTE: this ends up re-fetching hashes that were already
+            # fetched during remote size estimation
+            traverse_prefixes = [None]
+            initial = 0
+        else:
+            yield from remote_hashes
+            initial = len(remote_hashes)
+            traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
+            if self.TRAVERSE_PREFIX_LEN > 2:
+                traverse_prefixes += [
+                    "{0:0{1}x}".format(i, self.TRAVERSE_PREFIX_LEN)
+                    for i in range(1, pow(16, self.TRAVERSE_PREFIX_LEN - 2))
+                ]
+        with Tqdm(
+            desc="Querying "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            total=remote_size,
+            initial=initial,
+            unit="file",
+        ) as pbar:
+
+            def list_with_update(prefix):
+                return list(
+                    self.list_hashes(
+                        prefix=prefix, progress_callback=pbar.update
+                    )
+                )
+
+            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
+                in_remote = executor.map(list_with_update, traverse_prefixes,)
+                yield from itertools.chain.from_iterable(in_remote)
+
+    def list_hashes_exists(self, hashes, jobs=None, name=None):
+        """Return list of the specified hashes which exist in this tree.
+        Hashes will be queried individually.
+        """
+        logger.debug(
+            "Querying {} hashes via object_exists".format(len(hashes))
+        )
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(hashes),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(path_info):
+                ret = self.exists(path_info)
+                pbar.update_msg(str(path_info))
+                return ret
+
+            with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
+                path_infos = map(self.hash_to_path_info, hashes)
+                in_remote = executor.map(exists_with_progress, path_infos)
+                ret = list(itertools.compress(hashes, in_remote))
+                return ret
+
+    def _remove_unpacked_dir(self, hash_):
+        pass

dvc/tree/gdrive.py

@@ -14,11 +14,12 @@ from funcy.py3 import cat
 from dvc.exceptions import DvcException, FileMissingError
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree
 from dvc.scheme import Schemes
 from dvc.utils import format_link, tmp_fname
 from dvc.utils.stream import IterStream
 
+from .base import BaseRemoteTree
+
 logger = logging.getLogger(__name__)
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
 
@@ -553,7 +554,8 @@ class GDriveRemoteTree(BaseRemoteTree):
             )
 
     def walk_files(self, path_info, **kwargs):
-        if path_info == self.path_info:
+        use_prefix = kwargs.pop("prefix", False)
+        if path_info == self.path_info or not use_prefix:
             prefix = None
         else:
             prefix = path_info.path

dvc/tree/gs.py

@@ -9,9 +9,10 @@ from funcy import cached_property, wrap_prop
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree
 from dvc.scheme import Schemes
 
+from .base import BaseRemoteTree
+
 logger = logging.getLogger(__name__)
 
 
@@ -33,6 +34,7 @@ def dynamic_chunk_size(func):
         while True:
             try:
                 # skipcq: PYL-W0212
+                # pylint: disable=protected-access
                 chunk_size = Blob._CHUNK_SIZE_MULTIPLE * multiplier
                 return func(*args, chunk_size=chunk_size, **kwargs)
             except requests.exceptions.ConnectionError:
@@ -102,7 +104,8 @@ class GSRemoteTree(BaseRemoteTree):
             raise FileNotFoundError
 
         if isinstance(
-            blob.client._credentials, google.auth.credentials.Signing
+            blob.client._credentials,  # pylint: disable=protected-access
+            google.auth.credentials.Signing,
         ):
             # sign if we're able to sign with credentials.
             return blob.generate_signed_url(expiration=expiration)
@@ -142,7 +145,9 @@ class GSRemoteTree(BaseRemoteTree):
             yield blob.name
 
     def walk_files(self, path_info, **kwargs):
-        for fname in self._list_paths(path_info / "", **kwargs):
+        if not kwargs.pop("prefix", False):
+            path_info = path_info / ""
+        for fname in self._list_paths(path_info, **kwargs):
             # skip nested empty directories
             if fname.endswith("/"):
                 continue

dvc/tree/hdfs.py

@@ -1,7 +1,6 @@
 import io
 import logging
 import os
-import posixpath
 import re
 import subprocess
 from collections import deque
@@ -112,10 +111,13 @@ class HDFSRemoteTree(BaseRemoteTree):
             with self.hdfs(path_info) as hdfs:
                 hdfs.rm(path_info.path)
 
+    def makedirs(self, path_info):
+        with self.hdfs(path_info) as hdfs:
+            # NOTE: hdfs.mkdir creates parents by default
+            hdfs.mkdir(path_info.path)
+
     def copy(self, from_info, to_info, **_kwargs):
-        dname = posixpath.dirname(to_info.path)
         with self.hdfs(to_info) as hdfs:
-            hdfs.mkdir(dname)
             # NOTE: this is how `hadoop fs -cp` works too: it copies through
             # your local machine.
             with hdfs.open(from_info.path, "rb") as from_fobj:
@@ -163,13 +165,12 @@ class HDFSRemoteTree(BaseRemoteTree):
         # NOTE: pyarrow doesn't support checksum, so we need to use hadoop
         regex = r".*\t.*\t(?P<checksum>.*)"
         stdout = self.hadoop_fs(
-            f"checksum {path_info.path}", user=path_info.user
+            f"checksum {path_info.url}", user=path_info.user
         )
         return self._group(regex, stdout, "checksum")
 
     def _upload(self, from_file, to_info, **_kwargs):
         with self.hdfs(to_info) as hdfs:
-            hdfs.mkdir(posixpath.dirname(to_info.path))
             tmp_file = tmp_fname(to_info.path)
             with open(from_file, "rb") as fobj:
                 hdfs.upload(tmp_file, fobj)

dvc/tree/http.py

@@ -8,9 +8,10 @@ import dvc.prompt as prompt
 from dvc.exceptions import DvcException, HTTPError
 from dvc.path_info import HTTPURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree
 from dvc.scheme import Schemes
 
+from .base import BaseRemoteTree
+
 logger = logging.getLogger(__name__)
 
 

dvc/tree/local.py

@@ -0,0 +1,349 @@
+import errno
+import logging
+import os
+import stat
+
+from funcy import cached_property
+from shortuuid import uuid
+
+from dvc.exceptions import DvcException
+from dvc.path_info import PathInfo
+from dvc.scheme import Schemes
+from dvc.system import System
+from dvc.utils import file_md5, relpath, tmp_fname
+from dvc.utils.fs import (
+    copy_fobj_to_file,
+    copyfile,
+    makedirs,
+    move,
+    remove,
+    walk_files,
+)
+
+from .base import BaseRemoteTree
+
+logger = logging.getLogger(__name__)
+
+
+class LocalRemoteTree(BaseRemoteTree):
+    scheme = Schemes.LOCAL
+    PATH_CLS = PathInfo
+    PARAM_CHECKSUM = "md5"
+    PARAM_PATH = "path"
+    TRAVERSE_PREFIX_LEN = 2
+    UNPACKED_DIR_SUFFIX = ".unpacked"
+
+    CACHE_MODE = 0o444
+    SHARED_MODE_MAP = {None: (0o644, 0o755), "group": (0o664, 0o775)}
+
+    def __init__(self, repo, config, use_dvcignore=False, dvcignore_root=None):
+        super().__init__(repo, config)
+        url = config.get("url")
+        self.path_info = self.PATH_CLS(url) if url else None
+        self.use_dvcignore = use_dvcignore
+        self.dvcignore_root = dvcignore_root
+
+    @property
+    def tree_root(self):
+        return self.config.get("url")
+
+    @property
+    def state(self):
+        from dvc.state import StateNoop
+
+        return self.repo.state if self.repo else StateNoop()
+
+    @cached_property
+    def dvcignore(self):
+        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
+
+        root = self.dvcignore_root or self.tree_root
+        if not self.use_dvcignore:
+            return DvcIgnoreFilterNoop(self, root)
+        self.use_dvcignore = False
+        ret = DvcIgnoreFilter(self, root)
+        self.use_dvcignore = True
+        return ret
+
+    @staticmethod
+    def open(path_info, mode="r", encoding=None):
+        return open(path_info, mode=mode, encoding=encoding)
+
+    def exists(self, path_info):
+        assert isinstance(path_info, str) or path_info.scheme == "local"
+        if self.repo:
+            ret = os.path.lexists(path_info)
+        else:
+            ret = os.path.exists(path_info)
+        if not ret:
+            return False
+
+        return not self.dvcignore.is_ignored_file(
+            path_info
+        ) and not self.dvcignore.is_ignored_dir(path_info)
+
+    def isfile(self, path_info):
+        if not os.path.isfile(path_info):
+            return False
+
+        return not self.dvcignore.is_ignored_file(path_info)
+
+    def isdir(self, path_info):
+        if not os.path.isdir(path_info):
+            return False
+
+        return not self.dvcignore.is_ignored_dir(path_info)
+
+    def iscopy(self, path_info):
+        return not (
+            System.is_symlink(path_info) or System.is_hardlink(path_info)
+        )
+
+    def walk(self, top, topdown=True, onerror=None):
+        """Directory tree generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        """
+        for root, dirs, files in os.walk(
+            top, topdown=topdown, onerror=onerror
+        ):
+            dirs[:], files[:] = self.dvcignore(
+                os.path.abspath(root), dirs, files
+            )
+
+            yield os.path.normpath(root), dirs, files
+
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info):
+            for file in files:
+                # NOTE: os.path.join is ~5.5 times slower
+                yield PathInfo(f"{root}{os.sep}{file}")
+
+    def is_empty(self, path_info):
+        path = path_info.fspath
+
+        if self.isfile(path_info) and os.path.getsize(path) == 0:
+            return True
+
+        if self.isdir(path_info) and len(os.listdir(path)) == 0:
+            return True
+
+        return False
+
+    def remove(self, path_info):
+        if isinstance(path_info, PathInfo):
+            if path_info.scheme != "local":
+                raise NotImplementedError
+            path = path_info.fspath
+        else:
+            path = path_info
+
+        if self.exists(path):
+            remove(path)
+
+    def makedirs(self, path_info):
+        makedirs(path_info, exist_ok=True, mode=self.dir_mode)
+
+    def isexec(self, path):
+        mode = os.stat(path).st_mode
+        return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+    @staticmethod
+    def stat(path):
+        return os.stat(path)
+
+    def move(self, from_info, to_info, mode=None):
+        if from_info.scheme != "local" or to_info.scheme != "local":
+            raise NotImplementedError
+
+        self.makedirs(to_info.parent)
+
+        if mode is None:
+            if self.isfile(from_info):
+                mode = self.file_mode
+            else:
+                mode = self.dir_mode
+
+        move(from_info, to_info, mode=mode)
+
+    def copy(self, from_info, to_info):
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        try:
+            System.copy(from_info, tmp_info)
+            os.chmod(tmp_info, self.file_mode)
+            os.rename(tmp_info, to_info)
+        except Exception:
+            self.remove(tmp_info)
+            raise
+
+    def copy_fobj(self, fobj, to_info):
+        self.makedirs(to_info.parent)
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        try:
+            copy_fobj_to_file(fobj, tmp_info)
+            os.chmod(tmp_info, self.file_mode)
+            os.rename(tmp_info, to_info)
+        except Exception:
+            self.remove(tmp_info)
+            raise
+
+    @staticmethod
+    def symlink(from_info, to_info):
+        System.symlink(from_info, to_info)
+
+    @staticmethod
+    def is_symlink(path_info):
+        return System.is_symlink(path_info)
+
+    def hardlink(self, from_info, to_info):
+        # If there are a lot of empty files (which happens a lot in datasets),
+        # and the cache type is `hardlink`, we might reach link limits and
+        # will get something like: `too many links error`
+        #
+        # This is because all those empty files will have the same hash
+        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
+        # linked to the same file in the cache.
+        #
+        # From https://en.wikipedia.org/wiki/Hard_link
+        #   * ext4 limits the number of hard links on a file to 65,000
+        #   * Windows with NTFS has a limit of 1024 hard links on a file
+        #
+        # That's why we simply create an empty file rather than a link.
+        if self.getsize(from_info) == 0:
+            self.open(to_info, "w").close()
+
+            logger.debug(
+                "Created empty file: {src} -> {dest}".format(
+                    src=str(from_info), dest=str(to_info)
+                )
+            )
+            return
+
+        System.hardlink(from_info, to_info)
+
+    @staticmethod
+    def is_hardlink(path_info):
+        return System.is_hardlink(path_info)
+
+    def reflink(self, from_info, to_info):
+        tmp_info = to_info.parent / tmp_fname(to_info.name)
+        System.reflink(from_info, tmp_info)
+        # NOTE: reflink has its own separate inode, so you can set permissions
+        # that are different from the source.
+        os.chmod(tmp_info, self.file_mode)
+        os.rename(tmp_info, to_info)
+
+    def _unprotect_file(self, path):
+        if System.is_symlink(path) or System.is_hardlink(path):
+            logger.debug(f"Unprotecting '{path}'")
+            tmp = os.path.join(os.path.dirname(path), "." + uuid())
+
+            # The operations order is important here - if some application
+            # would access the file during the process of copyfile then it
+            # would get only the part of file. So, at first, the file should be
+            # copied with the temporary name, and then original file should be
+            # replaced by new.
+            copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
+            remove(path)
+            os.rename(tmp, path)
+
+        else:
+            logger.debug(
+                "Skipping copying for '{}', since it is not "
+                "a symlink or a hardlink.".format(path)
+            )
+
+        os.chmod(path, self.file_mode)
+
+    def _unprotect_dir(self, path):
+        for fname in self.walk_files(path):
+            self._unprotect_file(fname)
+
+    def unprotect(self, path_info):
+        path = path_info.fspath
+        if not os.path.exists(path):
+            raise DvcException(f"can't unprotect non-existing data '{path}'")
+
+        if os.path.isdir(path):
+            self._unprotect_dir(path)
+        else:
+            self._unprotect_file(path)
+
+    def protect(self, path_info):
+        path = os.fspath(path_info)
+        mode = self.CACHE_MODE
+
+        try:
+            os.chmod(path, mode)
+        except OSError as exc:
+            # There is nothing we need to do in case of a read-only file system
+            if exc.errno == errno.EROFS:
+                return
+
+            # In shared cache scenario, we might not own the cache file, so we
+            # need to check if cache file is already protected.
+            if exc.errno not in [errno.EPERM, errno.EACCES]:
+                raise
+
+            actual = stat.S_IMODE(os.stat(path).st_mode)
+            if actual != mode:
+                raise
+
+    def is_protected(self, path_info):
+        try:
+            mode = os.stat(path_info).st_mode
+        except FileNotFoundError:
+            return False
+
+        return stat.S_IMODE(mode) == self.CACHE_MODE
+
+    def get_file_hash(self, path_info):
+        return file_md5(path_info)[0]
+
+    @staticmethod
+    def getsize(path_info):
+        return os.path.getsize(path_info)
+
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+    ):
+        makedirs(to_info.parent, exist_ok=True)
+
+        tmp_file = tmp_fname(to_info)
+        copyfile(
+            from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
+        )
+
+        self.protect(tmp_file)
+        os.rename(tmp_file, to_info)
+
+    @staticmethod
+    def _download(
+        from_info, to_file, name=None, no_progress_bar=False, **_kwargs
+    ):
+        copyfile(
+            from_info, to_file, no_progress_bar=no_progress_bar, name=name
+        )
+
+    def list_paths(self, prefix=None, progress_callback=None):
+        assert self.path_info is not None
+        if prefix:
+            path_info = self.path_info / prefix[:2]
+            if not self.exists(path_info):
+                return
+        else:
+            path_info = self.path_info
+        # NOTE: use utils.fs walk_files since tree.walk_files will not follow
+        # symlinks
+        if progress_callback:
+            for path in walk_files(path_info):
+                progress_callback()
+                yield path
+        else:
+            yield from walk_files(path_info)
+
+    def _remove_unpacked_dir(self, hash_):
+        info = self.hash_to_path_info(hash_)
+        path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
+        self.remove(path_info)

dvc/tree/oss.py

@@ -6,9 +6,10 @@ from funcy import cached_property, wrap_prop
 
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree
 from dvc.scheme import Schemes
 
+from .base import BaseRemoteTree
+
 logger = logging.getLogger(__name__)
 
 
@@ -100,6 +101,8 @@ class OSSRemoteTree(BaseRemoteTree):  # pylint:disable=abstract-method
             yield blob.key
 
     def walk_files(self, path_info, **kwargs):
+        if not kwargs.pop("prefix", False):
+            path_info = path_info / ""
         for fname in self._list_paths(path_info):
             if fname.endswith("/"):
                 continue

dvc/tree/s3.py

@@ -8,9 +8,10 @@ from dvc.config import ConfigError
 from dvc.exceptions import DvcException, ETagMismatchError
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree
 from dvc.scheme import Schemes
 
+from .base import BaseRemoteTree
+
 logger = logging.getLogger(__name__)
 
 
@@ -187,7 +188,9 @@ class S3RemoteTree(BaseRemoteTree):
         )
 
     def walk_files(self, path_info, **kwargs):
-        for fname in self._list_paths(path_info / "", **kwargs):
+        if not kwargs.pop("prefix", False):
+            path_info = path_info / ""
+        for fname in self._list_paths(path_info, **kwargs):
             if fname.endswith("/"):
                 continue
 

dvc/tree/ssh/__init__.py

@@ -1,23 +1,19 @@
-import errno
 import getpass
 import io
-import itertools
 import logging
 import os
 import posixpath
 import threading
-from concurrent.futures import ThreadPoolExecutor
 from contextlib import closing, contextmanager
 from urllib.parse import urlparse
 
 from funcy import first, memoize, silent, wrap_with
 
 import dvc.prompt as prompt
-from dvc.progress import Tqdm
-from dvc.remote.base import BaseRemoteTree, Remote
-from dvc.remote.pool import get_connection
 from dvc.scheme import Schemes
-from dvc.utils import to_chunks
+
+from ..base import BaseRemoteTree
+from ..pool import get_connection
 
 logger = logging.getLogger(__name__)
 
@@ -283,65 +279,3 @@ class SSHRemoteTree(BaseRemoteTree):
                     yield path
             else:
                 yield from ssh.walk_files(root)
-
-
-class SSHRemote(Remote):
-    def batch_exists(self, path_infos, callback):
-        def _exists(chunk_and_channel):
-            chunk, channel = chunk_and_channel
-            ret = []
-            for path in chunk:
-                try:
-                    channel.stat(path)
-                    ret.append(True)
-                except OSError as exc:
-                    if exc.errno != errno.ENOENT:
-                        raise
-                    ret.append(False)
-                callback(path)
-            return ret
-
-        with self.tree.ssh(path_infos[0]) as ssh:
-            channels = ssh.open_max_sftp_channels()
-            max_workers = len(channels)
-
-            with ThreadPoolExecutor(max_workers=max_workers) as executor:
-                paths = [path_info.path for path_info in path_infos]
-                chunks = to_chunks(paths, num_chunks=max_workers)
-                chunks_and_channels = zip(chunks, channels)
-                outcome = executor.map(_exists, chunks_and_channels)
-                results = list(itertools.chain.from_iterable(outcome))
-
-            return results
-
-    def hashes_exist(self, hashes, jobs=None, name=None):
-        """This is older implementation used in remote/base.py
-        We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
-        faster than current approach (relying on exists(path_info)) applied in
-        remote/base.
-        """
-        if not self.tree.CAN_TRAVERSE:
-            return list(set(hashes) & set(self.tree.all()))
-
-        # possibly prompt for credentials before "Querying" progress output
-        self.tree.ensure_credentials()
-
-        with Tqdm(
-            desc="Querying "
-            + ("cache in " + name if name else "remote cache"),
-            total=len(hashes),
-            unit="file",
-        ) as pbar:
-
-            def exists_with_progress(chunks):
-                return self.batch_exists(chunks, callback=pbar.update_msg)
-
-            with ThreadPoolExecutor(
-                max_workers=jobs or self.tree.JOBS
-            ) as executor:
-                path_infos = [self.hash_to_path_info(x) for x in hashes]
-                chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
-                results = executor.map(exists_with_progress, chunks)
-                in_remote = itertools.chain.from_iterable(results)
-                ret = list(itertools.compress(hashes, in_remote))
-                return ret

dvc/tree/ssh/connection.py

@@ -9,9 +9,10 @@ from funcy import cached_property
 
 from dvc.exceptions import DvcException
 from dvc.progress import Tqdm
-from dvc.remote.base import RemoteCmdError
 from dvc.utils import tmp_fname
 
+from ..base import RemoteCmdError
+
 try:
     import paramiko
 except ImportError:
@@ -79,7 +80,7 @@ class SSHConnection:
 
         return 0
 
-    def exists(self, path, sftp=None):
+    def exists(self, path):
         return bool(self.st_mode(path))
 
     def isdir(self, path):

dvc/utils/fs.py

@@ -44,7 +44,7 @@ def get_mtime_and_size(path, tree):
                     raise
                 continue
             size += stats.st_size
-            files_mtimes[file_path] = stats.st_mtime
+            files_mtimes[os.fspath(file_path)] = stats.st_mtime
 
         # We track file changes and moves, which cannot be detected with simply
         # max(mtime(f) for f in non_ignored_files)
@@ -108,7 +108,7 @@ def move(src, dst, mode=None):
     shutil.move(tmp, dst)
 
 
-def _chmod(func, p, excinfo):
+def _chmod(func, p, excinfo):  # pylint: disable=unused-argument
     perm = os.lstat(p).st_mode
     perm |= stat.S_IWRITE
 

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "1.1.1"
+_BASE_VERSION = "1.1.11"
 
 
 def _generate_version(base_version):

fastentrypoints.py

@@ -79,7 +79,7 @@ def get_args(cls, dist, header=None):  # noqa: D205,D400
                 group,
                 name,
             )
-            # pylint: disable=E1101
+            # pylint: disable=E1101,protected-access
             args = cls._get_script_args(type_, name, header, script_text)
             for res in args:
                 yield res

scripts/ci/before_install.sh

@@ -16,27 +16,6 @@ fi
 echo >env.sh
 
 if [[ "$TRAVIS_BUILD_STAGE_NAME" == "test" ]]; then
-  if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then
-    # NOTE: ssh keys for ssh test to be able to ssh to the localhost
-    ssh-keygen -t rsa -N "" -f mykey
-    mkdir -p ~/.ssh
-    cp mykey ~/.ssh/id_rsa
-    cp mykey.pub ~/.ssh/id_rsa.pub
-    cat mykey.pub >>~/.ssh/authorized_keys
-    ssh-keyscan localhost >>~/.ssh/known_hosts
-    ssh localhost ls &>/dev/null
-    ssh-keyscan 127.0.0.1 >>~/.ssh/known_hosts
-    ssh 127.0.0.1 ls &>/dev/null
-    ssh-keyscan 0.0.0.0 >>~/.ssh/known_hosts
-    ssh 0.0.0.0 ls &>/dev/null
-  fi
-
-  if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-    bash "$scriptdir/install_azurite.sh"
-    bash "$scriptdir/install_oss.sh"
-    bash "$scriptdir/install_hadoop.sh"
-  fi
-
   if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
     brew install openssl
     $scriptdir/retry.sh brew cask install google-cloud-sdk

scripts/ci/core-site.xml

@@ -1,6 +0,0 @@
-<configuration>
-<property>
-<name>fs.defaultFS</name>
-<value>hdfs://localhost:8020</value>
-</property>
-</configuration>

scripts/ci/hdfs-site.xml

@@ -1,6 +0,0 @@
-<configuration>
-<property>
-<name>dfs.replication</name>
-<value>1</value>
-</property>
-</configuration>

scripts/ci/install_azurite.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -x
-
-# run azurite
-sudo docker run -d --restart always -e executable=blob -p 10000:10000 \
-  --tmpfs /opt/azurite/folder \
-  mcr.microsoft.com/azure-storage/azurite:3.3.0-preview \
-  azurite -l /data --blobHost 0.0.0.0
-
-# save secrets
-echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >>env.sh
-echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >>env.sh

scripts/ci/install_hadoop.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-# NOTE: based on http://sharafjaffri.blogspot.com/2015/04/installing-single-node-hadoop-26-using.html
-
-set -x
-set -e
-
-sudo add-apt-repository -y ppa:openjdk-r/ppa
-sudo apt-get update -y
-sudo apt-get install openjdk-7-jdk
-java -version
-
-pushd /usr/local
-sudo wget https://s3-us-east-2.amazonaws.com/dvc-public/travis-test/hadoop-2.6.5.tar.gz
-sudo tar xzf hadoop-2.6.5.tar.gz
-sudo mkdir hadoop
-sudo mv hadoop-2.6.5/* hadoop/
-popd
-
-echo "export HADOOP_HOME=/usr/local/hadoop" >>env.sh
-echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >>env.sh
-echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >>env.sh
-echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >>env.sh
-echo "export YARN_HOME=/usr/local/hadoop" >>env.sh
-echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >>env.sh
-
-# PyArrow==0.16.0 regression https://issues.apache.org/jira/browse/ARROW-7841
-# retrieves native library from $HADOOP_HOME directory instead of
-# `$HADOOP_HOME/lib/native`.
-# Fix: force search for `libhdfs.so` inside `$HADOOP_HOME/lib/native`.
-# Note: not needed for PyArrow==0.17.0.
-echo "export ARROW_LIBHDFS_DIR=/usr/local/hadoop/lib/native" >> env.sh
-
-echo "export JAVA_HOME=/usr/" >>env.sh
-echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >>env.sh
-
-source env.sh
-
-sudo bash -c 'echo "export JAVA_HOME=/usr/" >> /usr/local/hadoop/etc/hadoop/hadoop-env.sh'
-
-sudo cp scripts/ci/core-site.xml /usr/local/hadoop/etc/hadoop
-sudo cp scripts/ci/hdfs-site.xml /usr/local/hadoop/etc/hadoop
-
-hadoop fs -help
-pushd ~
-sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/
-sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/
-sudo chmod 777 -R /usr/local/hadoop
-hdfs namenode -format
-start-dfs.sh
-jps
-hadoop fs -ls /
-hadoop fs -ls hdfs://127.0.0.1/
-hadoop fs -mkdir hdfs://127.0.0.1/ololo
-popd

scripts/ci/install_oss.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-git clone https://github.com/iterative/oss-emulator.git
-sudo docker image build -t oss:1.0 oss-emulator
-sudo docker run --detach --restart always -p 8880:8880 --name oss-emulator oss:1.0
-echo "export OSS_ENDPOINT='localhost:8880'" >>env.sh
-echo "export OSS_ACCESS_KEY_ID='AccessKeyID'" >>env.sh
-echo "export OSS_ACCESS_KEY_SECRET='AccessKeySecret'" >>env.sh

scripts/ci/remove_hadoop.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-stop-dfs.sh
-sudo rm -rf /usr/local/hadoop
-sudo rm -rf /usr/local/hadoop-2.6.5
-sudo rm -rf /usr/local/hadoop-2.6.5.tar.gz
-rm env.sh

setup.cfg

@@ -17,7 +17,7 @@ count=true
 [isort]
 include_trailing_comma=true
 known_first_party=dvc,tests
-known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
+known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pygtrie,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
 line_length=79
 force_grid_wrap=0
 use_parentheses=True

setup.py

@@ -68,17 +68,17 @@ install_requires = [
     "tqdm>=4.45.0,<5",
     "packaging>=19.0",
     "zc.lockfile>=1.2.1",
-    "flufl.lock>=3.2",
+    "flufl.lock>=3.2,<4",
     "win-unicode-console>=0.5; sys_platform == 'win32'",
     "pywin32>=225; sys_platform == 'win32'",
-    "networkx>=2.1,<2.4",
+    "networkx>=2.1,<2.5",
     "pydot>=1.2.4",
     "speedcopy>=2.0.1; python_version < '3.8' and sys_platform == 'win32'",
     "flatten_json>=0.1.6",
     "tabulate>=0.8.7",
     "pygtrie==2.3.2",
     "dpath>=2.0.1,<3",
-    "shtab>=1.0.3",
+    "shtab>=1.1.0,<2",
 ]
 
 
@@ -87,7 +87,7 @@ install_requires = [
 gs = ["google-cloud-storage==1.19.0"]
 gdrive = ["pydrive2>=1.4.14"]
 s3 = ["boto3>=1.9.201"]
-azure = ["azure-storage-blob==2.1.0"]
+azure = ["azure-storage-blob==2.1.0", "knack"]
 oss = ["oss2==2.6.1"]
 ssh = ["paramiko>=2.5.0"]
 hdfs = ["pyarrow>=0.17.0"]
@@ -103,6 +103,7 @@ tests_requirements = [
     "wheel>=0.31.1",
     # Test requirements:
     "pytest>=4.6.0",
+    "pytest-docker>=0.7.2",
     "pytest-timeout>=1.3.3",
     "pytest-cov>=2.6.1",
     "pytest-xdist>=1.26.1",
@@ -129,6 +130,9 @@ tests_requirements = [
     "flake8-string-format",
     "pylint",
     "pylint-pytest",
+    "pylint-plugin-utils",
+    "wget",
+    "filelock",
 ]
 
 if (sys.version_info) >= (3, 6):