dvc: bump to 0.40.1

pyinstaller: fix build for oss2

.appveyor.yml

@@ -20,12 +20,16 @@ environment:
     secure: XN4jRtmGE5Bqg8pPZkwNs7kn3UEI73Rkldqc0MGsQISZBm5TNJZOPofDMc1QnUsf
   AZURE_STORAGE_CONTAINER_NAME: appveyor-tests
   AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
+  OSS_ENDPOINT: localhost:50004
+  OSS_ACCESS_KEY_ID: AccessKeyID
+  OSS_ACCESS_KEY_SECRET: AccessKeySecret
 
   matrix:
 
     - PYTHON: "C:\\Python27"
       PYTHON_VERSION: "2.7.x" # currently 2.7.9
       PYTHON_ARCH: "32"
+      RUBY: C:\Ruby25\bin
 
 # Disabled as unnecessary
 #    - PYTHON: "C:\\Python27-x64"
@@ -33,7 +37,12 @@ environment:
 #      PYTHON_ARCH: "64"
 
 install:
-  - ps: Install-Product node
+  - SET PATH=%RUBY%;%PATH%
+  - git clone -q https://github.com/nanaya-tachibana/oss-emulator.git
+  - gem install thor builder
+  - ps: cd oss-emulator
+  - ps: $OSSProcess = Start-Process -FilePath ruby -ArgumentList "bin\emulator -r store -p 50004" -PassThru -NoNewWindow
+  - ps: cd ..
   - npm install -g azurite
   - ps: $AzuriteProcess = Start-Process azurite-blob -PassThru
   - cinst wget
@@ -66,6 +75,7 @@ after_test:
 
 on_finish:
   - ps: Stop-Process $AzuriteProcess
+  - ps: Stop-Process $OSSProcess
 
 artifacts:
   - path: dvc*.exe

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,9 @@
+* [ ] Have you followed the guidelines in our
+      [Contributing document](https://dvc.org/doc/user-guide/contributing)?
+
+* [ ] Does your PR affect documented changes or does it add new functionality
+      that should be documented? If yes, have you created a PR for
+      [dvc.org](https://github.com/iterative/dvc.org) documenting it or at
+      least opened an issue for it? If so, please add a link to it.
+
+-----

README.rst

@@ -1,4 +1,4 @@
-.. image:: https://dvc.org/static/img/logo-owl-readme.png
+.. image:: https://dvc.org/static/img/logo-github-readme.png
   :target: https://dvc.org
   :alt: DVC logo
 
@@ -25,6 +25,10 @@
   :target: https://codecov.io/gh/iterative/dvc
   :alt: Codecov
 
+.. image:: https://img.shields.io/badge/patreon-donate-green.svg
+  :target: https://www.patreon.com/DVCorg/overview
+  :alt: Donate
+
 |
 
 **Data Science Version Control** or **DVC** is an **open-source** tool for data science and
@@ -34,7 +38,7 @@ machine learning projects. Key features:
    any databases. Does not depend on any proprietary online services;
 
 #. it manages and versions **datasets** and **machine learning models**. Data is saved in
-   S3, Google cloud, Azure, SSH server, HDFS or even local HDD RAID;
+   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS or even local HDD RAID;
 
 #. it makes projects **reproducible** and **shareable**, it helps answering question how
    the model was build;
@@ -59,12 +63,12 @@ made right and tailored specifically for ML and Data Science scenarios.
 
 #. ``Git/Git-lfs`` part - DVC helps you storing and sharing data artifacts, models. It connects them with your
    Git repository.
-#. ``Makefiles`` part - DVC describes how one data or model artifact was build from another data. 
+#. ``Makefiles`` part - DVC describes how one data or model artifact was build from another data.
 
 DVC usually runs along with Git. Git is used as usual to store and version code and DVC meta-files. DVC helps
 to store data and model files seamlessly out of Git while preserving almost the same user experience as if they
 were stored in Git itself. To store and share data files cache DVC supports remotes - any cloud (S3, Azure,
-Google Cloud, etc) or any on-premise network storage (via SSH, for example). It does not require 
+Google Cloud, etc) or any on-premise network storage (via SSH, for example).
 
 .. image:: https://dvc.org/static/img/flow.gif
    :target: https://dvc.org/static/img/flow.gif
@@ -114,7 +118,7 @@ pip (PyPI)
    pip install dvc
 
 Depending on the remote storage type you plan to use to keep and share your data, you might need to specify
-one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``ssh``. Or ``all_remotes`` to include them all.
+one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all_remotes`` to include them all.
 The command should look like this: ``pip install dvc[s3]`` - it installs the ``boto3`` library along with
 DVC to support the AWS S3 storage.
 
@@ -143,7 +147,7 @@ Package
 Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
 GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
 
-Ubuntu / Debian (apt)
+Ubuntu / Debian (deb)
 ^^^^^^^^^^^^^^^^^^^^^
 .. code-block:: bash
 
@@ -159,7 +163,7 @@ Fedora / CentOS (rpm)
    sudo yum update
    sudo yum install dvc
 
-Arch linux (AUR)
+Arch Linux (AUR)
 ^^^^^^^^^^^^^^^^
 *Unofficial package*, any inquiries regarding the AUR package,
 `refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.

dvc/analytics.py

@@ -96,11 +96,12 @@ class Analytics(object):
         with open(self.user_id_file, "r") as fobj:
             try:
                 info = json.load(fobj)
-                return info[self.PARAM_USER_ID]
-            except json.JSONDecodeError as exc:
+            except ValueError as exc:
                 logger.debug("Failed to load user_id: {}".format(exc))
                 return None
 
+            return info[self.PARAM_USER_ID]
+
     def _get_user_id(self):
         from dvc.lock import LockError
 

dvc/cli.py

@@ -33,6 +33,7 @@ import dvc.command.daemon as daemon
 import dvc.command.commit as commit
 import dvc.command.tag as tag
 import dvc.command.diff as diff
+import dvc.command.version as version
 from dvc.exceptions import DvcParserError
 
 
@@ -65,6 +66,7 @@ COMMANDS = [
     commit,
     tag,
     diff,
+    version,
 ]
 
 

dvc/command/add.py

@@ -54,14 +54,7 @@ def add_parser(subparsers, parent_parser):
         help="Don't put files/directories into cache.",
     )
     add_parser.add_argument(
-        "-f",
-        "--file",
-        help="Specify name of the stage file. It should be "
-        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
-        "'prepare.dvc', 'clean.dvc', etc) in order for "
-        "dvc to be able to find it later. By default "
-        "the output basename + .dvc is used as a stage filename. "
-        "(NOTE: It can't be used when specifying multiple targets)",
+        "-f", "--file", help="Specify name of the DVC file it generates."
     )
     add_parser.add_argument(
         "targets", nargs="+", help="Input files/directories."

dvc/command/base.py

@@ -65,3 +65,11 @@ class CmdBase(object):
     # Abstract methods that have to be implemented by any inheritance class
     def run(self):
         pass
+
+
+class CmdBaseNoRepo(CmdBase):
+    def __init__(self, args):
+        self.args = args
+
+    def run_cmd(self):
+        return self.run()

dvc/command/config.py

@@ -4,7 +4,7 @@ import os
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.config import Config
 from dvc.exceptions import DvcException
 
@@ -12,11 +12,11 @@ from dvc.exceptions import DvcException
 logger = logging.getLogger(__name__)
 
 
-class CmdConfig(CmdBase):
+class CmdConfig(CmdBaseNoRepo):
     def __init__(self, args):
         from dvc.repo import Repo, NotDvcRepoError
 
-        self.args = args
+        super(CmdConfig, self).__init__(args)
 
         try:
             dvc_dir = os.path.join(Repo.find_root(), Repo.DVC_DIR)
@@ -39,9 +39,6 @@ class CmdConfig(CmdBase):
                 raise saved_exc
             self.configobj = self.config._repo_config
 
-    def run_cmd(self):
-        return self.run()
-
     def _unset(self, section, opt=None, configobj=None):
         if configobj is None:
             configobj = self.configobj

dvc/command/daemon.py

@@ -1,15 +1,10 @@
 from __future__ import unicode_literals
 
-from dvc.command.base import CmdBase, fix_subparsers
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
 
 
-class CmdDaemonBase(CmdBase):
-    def __init__(self, args):
-        self.args = args
-        self.config = None
-
-    def run_cmd(self):
-        return self.run()
+class CmdDaemonBase(CmdBaseNoRepo):
+    pass
 
 
 class CmdDaemonUpdater(CmdDaemonBase):

dvc/command/diff.py

@@ -1,11 +1,13 @@
 from __future__ import unicode_literals
 
+import argparse
+
 import humanize
 import inflect
 import logging
 
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.utils.collections import compact
 import dvc.repo.diff as diff
 
@@ -121,14 +123,17 @@ class CmdDiff(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    description = (
-        "Show diff of a data file or a directory that "
-        "is under DVC control. Some basic statistics "
-        "summary, how many files were deleted/changed."
+    DIFF_DESCRIPTION = (
+        "Show diff of a data file or a directory that is under DVC control.\n"
+        "Some basic statistics summary, how many files were deleted/changed."
     )
-    help = "Show a diff of a DVC controlled data file or a directory."
+    DIFF_HELP = "Show a diff of a DVC controlled data file or a directory."
     diff_parser = subparsers.add_parser(
-        "diff", parents=[parent_parser], description=description, help=help
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(DIFF_DESCRIPTION, "diff"),
+        help=DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     diff_parser.add_argument(
         "-t",

dvc/command/imp.py

@@ -19,7 +19,9 @@ class CmdImport(CmdBase):
 
             out = self.args.out or default_out
 
-            self.repo.imp(self.args.url, out, self.args.resume)
+            self.repo.imp(
+                self.args.url, out, self.args.resume, fname=self.args.file
+            )
         except DvcException:
             logger.exception(
                 "failed to import {}. You could also try downloading "
@@ -62,4 +64,7 @@ def add_parser(subparsers, parent_parser):
     import_parser.add_argument(
         "out", nargs="?", help="Destination path to put files to."
     )
+    import_parser.add_argument(
+        "-f", "--file", help="Specify name of the DVC file it generates."
+    )
     import_parser.set_defaults(func=CmdImport)

dvc/command/init.py

@@ -3,17 +3,14 @@ from __future__ import unicode_literals
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 
 logger = logging.getLogger(__name__)
 
 
-class CmdInit(object):
-    def __init__(self, args):
-        self.args = args
-
-    def run_cmd(self):
+class CmdInit(CmdBaseNoRepo):
+    def run(self):
         from dvc.repo import Repo
         from dvc.exceptions import InitError
 

dvc/command/pipeline.py

@@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
 
 
 class CmdPipelineShow(CmdBase):
-    def _show(self, target, commands, outs):
+    def _show(self, target, commands, outs, locked):
         import networkx
         from dvc.stage import Stage
 
@@ -22,8 +22,12 @@ class CmdPipelineShow(CmdBase):
         G = self.repo.graph()[0]
         stages = networkx.get_node_attributes(G, "stage")
         node = os.path.relpath(stage.path, self.repo.root_dir)
+        nodes = networkx.dfs_postorder_nodes(G, node)
 
-        for n in networkx.dfs_postorder_nodes(G, node):
+        if locked:
+            nodes = [n for n in nodes if stages[n].locked]
+
+        for n in nodes:
             if commands:
                 logger.info(stages[n].cmd)
             elif outs:
@@ -122,7 +126,7 @@ class CmdPipelineShow(CmdBase):
         simple_g.add_edges_from(edges)
         write_dot(simple_g, filename)
 
-    def run(self, unlock=False):
+    def run(self):
         if not self.args.targets:
             self.args.targets = self.default_targets
 
@@ -144,7 +148,12 @@ class CmdPipelineShow(CmdBase):
                         target, self.args.commands, self.args.outs
                     )
                 else:
-                    self._show(target, self.args.commands, self.args.outs)
+                    self._show(
+                        target,
+                        self.args.commands,
+                        self.args.outs,
+                        self.args.locked,
+                    )
             except DvcException:
                 msg = "failed to show pipeline for '{}'".format(target)
                 logger.exception(msg)
@@ -208,6 +217,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Print output files instead of paths to DVC files.",
     )
+    pipeline_show_parser.add_argument(
+        "-l",
+        "--locked",
+        action="store_true",
+        default=False,
+        help="Print locked DVC stages",
+    )
     pipeline_show_parser.add_argument(
         "--ascii",
         action="store_true",

dvc/command/pkg.py

@@ -5,28 +5,26 @@ import logging
 
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
-from dvc.repo.pkg import PackageParams
 
 
 logger = logging.getLogger(__name__)
 
 
-class CmdPkg(CmdBase):
-    def run(self, unlock=False):
+class CmdPkgInstall(CmdBase):
+    def run(self):
         try:
-            pkg_params = PackageParams(
+            self.repo.pkg.install(
                 self.args.address,
                 self.args.target_dir,
                 self.args.select,
                 self.args.file,
             )
-            return self.repo.install_pkg(pkg_params)
+            return 0
         except DvcException:
             logger.exception(
                 "failed to install package '{}'".format(self.args.address)
             )
             return 1
-        pass
 
 
 def add_parser(subparsers, parent_parser):
@@ -88,4 +86,4 @@ def add_parser(subparsers, parent_parser):
         "By default the file has 'mod_' prefix and imported package name "
         "followed by .dvc",
     )
-    pkg_install_parser.set_defaults(func=CmdPkg)
+    pkg_install_parser.set_defaults(func=CmdPkgInstall)

dvc/command/repro.py

@@ -39,6 +39,7 @@ class CmdRepro(CmdBase):
                     all_pipelines=self.args.all_pipelines,
                     ignore_build_cache=self.args.ignore_build_cache,
                     no_commit=self.args.no_commit,
+                    downstream=self.args.downstream,
                 )
 
                 if len(stages) == 0:
@@ -140,4 +141,10 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Don't put files/directories into cache.",
     )
+    repro_parser.add_argument(
+        "--downstream",
+        action="store_true",
+        default=False,
+        help="Reproduce the pipeline starting from the specified stage.",
+    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/root.py

@@ -4,18 +4,16 @@ import argparse
 import os
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.repo import Repo
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 
 logger = logging.getLogger(__name__)
 
 
-class CmdRoot(CmdBase):
-    def run_cmd(self):
-        return self.run()
-
+class CmdRoot(CmdBaseNoRepo):
     def run(self):
-        logger.info(os.path.relpath(self.repo.root_dir))
+        logger.info(os.path.relpath(Repo.find_root()))
         return 0
 
 

dvc/command/run.py

@@ -127,14 +127,7 @@ def add_parser(subparsers, parent_parser):
         "(do not put into DVC cache).",
     )
     run_parser.add_argument(
-        "-f",
-        "--file",
-        help="Specify name of the stage file. It should be "
-        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
-        "'prepare.dvc', 'clean.dvc', etc) in order for "
-        "dvc to be able to find it later. By default "
-        "the first output basename + .dvc is used as "
-        "a stage filename.",
+        "-f", "--file", help="Specify name of the DVC file it generates."
     )
     run_parser.add_argument(
         "-c", "--cwd", default=None, help="Deprecated, use -w and -f instead."

dvc/command/tag.py

@@ -1,6 +1,6 @@
-import yaml
 import logging
 
+from dvc.utils import to_yaml_string
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
 
@@ -50,7 +50,7 @@ class CmdTagList(CmdBase):
                     recursive=self.args.recursive,
                 )
                 if tags:
-                    logger.info(yaml.dump(tags, default_flow_style=False))
+                    logger.info(to_yaml_string(tags))
             except DvcException:
                 logger.exception("failed list tags")
                 return 1

dvc/command/version.py

@@ -0,0 +1,39 @@
+from __future__ import unicode_literals
+
+import platform
+import argparse
+import logging
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.version import __version__
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdVersion(CmdBaseNoRepo):
+    def run(self):
+        info = (
+            "DVC version: {dvc_version}\n"
+            "Python version: {python_version}\n"
+            "Platform: {platform_type}"
+        ).format(
+            dvc_version=__version__,
+            python_version=platform.python_version(),
+            platform_type=platform.platform(),
+        )
+        logger.info(info)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    VERSION_HELP = "Show DVC version and system/environment informaion."
+
+    version_parser = subparsers.add_parser(
+        "version",
+        parents=[parent_parser],
+        description=append_doc_link(VERSION_HELP, "version"),
+        help=VERSION_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    version_parser.set_defaults(func=CmdVersion)

dvc/config.py

@@ -134,14 +134,16 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
+    BOOL_SCHEMA = And(str, is_bool, Use(to_bool))
+
     SECTION_CORE = "core"
     SECTION_CORE_LOGLEVEL = "loglevel"
     SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
     SECTION_CORE_REMOTE = "remote"
-    SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
+    SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA
     SECTION_CORE_INTERACTIVE = "interactive"
     SECTION_CORE_ANALYTICS = "analytics"
-    SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool))
+    SECTION_CORE_ANALYTICS_SCHEMA = BOOL_SCHEMA
 
     SECTION_CACHE = "cache"
     SECTION_CACHE_DIR = "dir"
@@ -154,6 +156,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_CACHE_SSH = "ssh"
     SECTION_CACHE_HDFS = "hdfs"
     SECTION_CACHE_AZURE = "azure"
+    SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
     SECTION_CACHE_SCHEMA = {
         Optional(SECTION_CACHE_LOCAL): str,
         Optional(SECTION_CACHE_S3): str,
@@ -163,10 +166,9 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_CACHE_AZURE): str,
         Optional(SECTION_CACHE_DIR): str,
         Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED, default=False): And(
-            str, is_bool, Use(to_bool)
-        ),
+        Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
         Optional(PRIVATE_CWD): str,
+        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): BOOL_SCHEMA,
     }
 
     # backward compatibility
@@ -175,7 +177,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_CORE_STORAGEPATH = "storagepath"
 
     SECTION_CORE_SCHEMA = {
-        Optional(SECTION_CORE_LOGLEVEL, default="info"): And(
+        Optional(SECTION_CORE_LOGLEVEL): And(
             str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA
         ),
         Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)),
@@ -205,12 +207,8 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_PROFILE): str,
         Optional(SECTION_AWS_CREDENTIALPATH): str,
         Optional(SECTION_AWS_ENDPOINT_URL): str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): And(
-            str, is_bool, Use(to_bool)
-        ),
-        Optional(SECTION_AWS_USE_SSL, default=True): And(
-            str, is_bool, Use(to_bool)
-        ),
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
+        Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
     }
 
     # backward compatibility
@@ -229,6 +227,10 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str}
 
     SECTION_AZURE_CONNECTION_STRING = "connection_string"
+    # Alibabacloud oss options
+    SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
+    SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
+    SECTION_OSS_ENDPOINT = "oss_endpoint"
 
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
@@ -245,24 +247,21 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_PROFILE): str,
         Optional(SECTION_AWS_CREDENTIALPATH): str,
         Optional(SECTION_AWS_ENDPOINT_URL): str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): And(
-            str, is_bool, Use(to_bool)
-        ),
-        Optional(SECTION_AWS_USE_SSL, default=True): And(
-            str, is_bool, Use(to_bool)
-        ),
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
+        Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED, default=False): And(
-            str, is_bool, Use(to_bool)
-        ),
+        Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
         Optional(SECTION_REMOTE_USER): str,
         Optional(SECTION_REMOTE_PORT): Use(int),
         Optional(SECTION_REMOTE_KEY_FILE): str,
         Optional(SECTION_REMOTE_TIMEOUT): Use(int),
         Optional(SECTION_REMOTE_PASSWORD): str,
-        Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
+        Optional(SECTION_REMOTE_ASK_PASSWORD): BOOL_SCHEMA,
         Optional(SECTION_AZURE_CONNECTION_STRING): str,
+        Optional(SECTION_OSS_ACCESS_KEY_ID): str,
+        Optional(SECTION_OSS_ACCESS_KEY_SECRET): str,
+        Optional(SECTION_OSS_ENDPOINT): str,
         Optional(PRIVATE_CWD): str,
     }
 

dvc/data_cloud.py

@@ -9,6 +9,7 @@ from dvc.remote import Remote
 from dvc.remote.s3 import RemoteS3
 from dvc.remote.gs import RemoteGS
 from dvc.remote.azure import RemoteAzure
+from dvc.remote.oss import RemoteOSS
 from dvc.remote.ssh import RemoteSSH
 from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
@@ -34,6 +35,7 @@ class DataCloud(object):
         "aws": RemoteS3,
         "gcp": RemoteGS,
         "azure": RemoteAzure,
+        "oss": RemoteOSS,
         "ssh": RemoteSSH,
         "hdfs": RemoteHDFS,
         "local": RemoteLOCAL,
@@ -48,8 +50,10 @@ class DataCloud(object):
 
     @property
     def _cloud(self):
+        """Returns a Remote instance using the `core.remote` config"""
         remote = self._core.get(Config.SECTION_CORE_REMOTE, "")
-        if remote != "":
+
+        if remote:
             return self._init_remote(remote)
 
         if self._core.get(Config.SECTION_CORE_CLOUD, None):

dvc/dependency/http.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+from dvc.path.http import HTTPPathInfo
 from dvc.utils.compat import urlparse, urljoin
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP
@@ -16,4 +17,4 @@ class DependencyHTTP(DependencyBase, OutputBase):
         if path.startswith("remote"):
             path = urljoin(self.remote.cache_dir, urlparse(path).path)
 
-        self.path_info = {"scheme": urlparse(path).scheme, "path": path}
+        self.path_info = HTTPPathInfo(url=self.url, path=path)

dvc/exceptions.py

@@ -214,11 +214,12 @@ class NoMetricsError(DvcException):
 
 
 class StageFileCorruptedError(DvcException):
-    def __init__(self, path):
+    def __init__(self, path, cause=None):
         path = os.path.relpath(path)
         super(StageFileCorruptedError, self).__init__(
             "unable to read stage file: {} "
-            "YAML file structure is corrupted".format(path)
+            "YAML file structure is corrupted".format(path),
+            cause=cause,
         )
 
 
@@ -255,3 +256,11 @@ class CheckoutErrorSuggestGit(DvcException):
         super(CheckoutErrorSuggestGit, self).__init__(
             "Did you mean 'git checkout {}'?".format(target), cause=cause
         )
+
+
+class ETagMismatchError(DvcException):
+    def __init__(self, etag, cached_etag):
+        super(ETagMismatchError, self).__init__(
+            "ETag mismatch detected when copying file to cache! "
+            "(expected: '{}', actual: '{}')".format(etag, cached_etag)
+        )

dvc/ignore.py

@@ -0,0 +1,134 @@
+import os
+
+from dulwich.ignore import match_pattern, read_ignore_patterns
+from dvc.utils.compat import cast_bytes
+from dvc.utils.fs import get_parent_dirs_up_to
+
+
+class DvcIgnoreFileHandler(object):
+    def __init__(self, tree):
+        self.tree = tree
+
+    def read_patterns(self, path):
+        with self.tree.open(path, binary=True) as stream:
+            return self._read_patterns(stream)
+
+    def get_repo_root(self):
+        return self.tree.tree_root
+
+    def _read_patterns(self, binary_stream):
+        negate_patterns = []
+        patterns = []
+        for pattern in read_ignore_patterns(binary_stream):
+            if pattern.lstrip().startswith(b"!"):
+                negate_patterns.append(pattern)
+            else:
+                patterns.append(pattern)
+
+        return negate_patterns, patterns
+
+
+class DvcIgnore(object):
+    DVCIGNORE_FILE = ".dvcignore"
+
+    def __call__(self, root, dirs, files):
+        raise NotImplementedError
+
+
+class DvcIgnoreFromFile(DvcIgnore):
+    def __init__(self, ignore_file_path, ignore_handler):
+        self.ignore_file_path = ignore_file_path
+        self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
+        self.patterns = []
+        self.negate_patterns = []
+
+        self.negate_patterns, self.patterns = ignore_handler.read_patterns(
+            ignore_file_path
+        )
+
+    def __call__(self, root, dirs, files):
+        files = [f for f in files if not self.matches(root, f)]
+        dirs = [d for d in dirs if not self.matches(root, d)]
+
+        return dirs, files
+
+    def get_match(self, abs_path):
+        rel_path = os.path.relpath(abs_path, self.dirname)
+        if os.name == "nt":
+            rel_path = rel_path.replace("\\", "/")
+        rel_path = cast_bytes(rel_path, "utf-8")
+
+        for pattern in self.patterns:
+            if match_pattern(
+                rel_path, pattern
+            ) and self._no_negate_pattern_matches(rel_path):
+                return (abs_path, pattern, self.ignore_file_path)
+        return None
+
+    def matches(self, dirname, basename):
+        if self.get_match(os.path.join(dirname, basename)):
+            return True
+        return False
+
+    def _no_negate_pattern_matches(self, path):
+        return all([not match_pattern(path, p) for p in self.negate_patterns])
+
+    def __hash__(self):
+        return hash(self.ignore_file_path)
+
+
+class DvcIgnoreConstant(DvcIgnore):
+    def __init__(self, basename):
+        self.basename = basename
+
+
+class DvcIgnoreDir(DvcIgnoreConstant):
+    def __call__(self, root, dirs, files):
+        dirs = [d for d in dirs if not d == self.basename]
+
+        return dirs, files
+
+
+class DvcIgnoreFile(DvcIgnoreConstant):
+    def __call__(self, root, dirs, files):
+        files = [f for f in files if not f == self.basename]
+
+        return dirs, files
+
+
+class DvcIgnoreFilter(object):
+    def __init__(self, wdir, ignore_file_handler=None):
+        self.ignores = [
+            DvcIgnoreDir(".git"),
+            DvcIgnoreDir(".hg"),
+            DvcIgnoreDir(".dvc"),
+            DvcIgnoreFile(".dvcignore"),
+        ]
+
+        self.ignore_file_handler = ignore_file_handler
+        self._process_ignores_in_parent_dirs(wdir)
+
+    def _process_ignores_in_parent_dirs(self, wdir):
+        if self.ignore_file_handler:
+            wdir = os.path.normpath(os.path.abspath(wdir))
+            ignore_search_end_dir = self.ignore_file_handler.get_repo_root()
+            parent_dirs = get_parent_dirs_up_to(wdir, ignore_search_end_dir)
+            for d in parent_dirs:
+                self.update(d)
+
+    def update(self, wdir):
+        ignore_file_path = os.path.join(wdir, DvcIgnore.DVCIGNORE_FILE)
+        if os.path.exists(ignore_file_path):
+            file_ignore = DvcIgnoreFromFile(
+                ignore_file_path, ignore_handler=self.ignore_file_handler
+            )
+            self.ignores.append(file_ignore)
+
+    def __call__(self, root, dirs, files):
+        if self.ignore_file_handler:
+            self.update(root)
+
+        for ignore in self.ignores:
+            dirs, files = ignore(root, dirs, files)
+
+        return dirs, files

dvc/logger.py

@@ -47,6 +47,9 @@ class ColorFormatter(logging.Formatter):
     )
 
     def format(self, record):
+        if self._is_visible(record):
+            self._progress_aware()
+
         if record.levelname == "INFO":
             return record.msg
 
@@ -75,6 +78,12 @@ class ColorFormatter(logging.Formatter):
             msg=record.msg,
         )
 
+    def _current_level(self):
+        return logging.getLogger("dvc").getEffectiveLevel()
+
+    def _is_visible(self, record):
+        return record.levelno >= self._current_level()
+
     def _description(self, message, exception):
         description = ""
 
@@ -117,7 +126,7 @@ class ColorFormatter(logging.Formatter):
 
         exception = ": ".join(exc_list)
 
-        if logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG:
+        if self._current_level() == logging.DEBUG:
             stack_trace = (
                 "\n" "{red}{line}{nc}\n" "{stack_trace}" "{red}{line}{nc}"
             ).format(

dvc/main.py

@@ -24,6 +24,7 @@ def main(argv=None):
     args = None
     cmd = None
 
+    outerLogLevel = logger.level
     try:
         args = parse_args(argv)
 
@@ -46,6 +47,8 @@ def main(argv=None):
     except Exception:  # pylint: disable=broad-except
         logger.exception("unexpected error")
         ret = 255
+    finally:
+        logger.setLevel(outerLogLevel)
 
     Analytics().send_cmd(cmd, args, ret)
 

dvc/output/__init__.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import schema
 
+from dvc.path import Schemes
 from dvc.utils.compat import urlparse, str
 
 from dvc.output.base import OutputBase
@@ -25,11 +26,11 @@ OUTS = [
 ]
 
 OUTS_MAP = {
-    "hdfs": OutputHDFS,
-    "s3": OutputS3,
-    "gs": OutputGS,
-    "ssh": OutputSSH,
-    "local": OutputLOCAL,
+    Schemes.HDFS: OutputHDFS,
+    Schemes.S3: OutputS3,
+    Schemes.GS: OutputGS,
+    Schemes.SSH: OutputSSH,
+    Schemes.LOCAL: OutputLOCAL,
 }
 
 # NOTE: currently there are only 3 possible checksum names:

dvc/output/base.py

@@ -1,6 +1,9 @@
 from __future__ import unicode_literals
 
 import re
+import logging
+from copy import copy
+
 from schema import Or, Optional
 
 from dvc.exceptions import DvcException
@@ -8,6 +11,9 @@ from dvc.utils.compat import str
 from dvc.remote.base import RemoteBase
 
 
+logger = logging.getLogger(__name__)
+
+
 class OutputDoesNotExistError(DvcException):
     def __init__(self, path):
         msg = "output '{}' does not exist".format(path)
@@ -73,10 +79,7 @@ class OutputBase(object):
         self.persist = persist
         self.tags = None if self.IS_DEPENDENCY else (tags or {})
 
-        if (
-            self.use_cache
-            and getattr(self.repo.cache, self.REMOTE.scheme) is None
-        ):
+        if self.use_cache and self.cache is None:
             raise DvcException(
                 "no cache location setup for '{}' outputs.".format(
                     self.REMOTE.scheme
@@ -95,6 +98,14 @@ class OutputBase(object):
     def is_in_repo(self):
         return False
 
+    @property
+    def cache(self):
+        return getattr(self.repo.cache, self.scheme)
+
+    @property
+    def dir_cache(self):
+        return self.cache.get_dir_cache(self.checksum)
+
     def assign_to_stage_file(self, target_repo):
         raise DvcException(
             "change repo is not supported for {}".format(self.scheme)
@@ -120,7 +131,11 @@ class OutputBase(object):
 
     @property
     def path(self):
-        return self.path_info["path"]
+        return self.path_info.path
+
+    @property
+    def cache_path(self):
+        return self.cache.checksum_to_path(self.checksum)
 
     @property
     def sep(self):
@@ -130,6 +145,10 @@ class OutputBase(object):
     def checksum(self):
         return self.info.get(self.remote.PARAM_CHECKSUM)
 
+    @property
+    def is_dir_checksum(self):
+        return self.remote.is_dir_checksum(self.checksum)
+
     @property
     def exists(self):
         return self.remote.exists(self.path_info)
@@ -146,9 +165,7 @@ class OutputBase(object):
         if not self.use_cache or not self.checksum:
             return True
 
-        cache = self.repo.cache.__getattribute__(self.scheme)
-
-        return cache.changed_cache(self.checksum)
+        return self.cache.changed_cache(self.checksum)
 
     def status(self):
         if self.checksum and self.use_cache and self.changed_cache():
@@ -166,22 +183,65 @@ class OutputBase(object):
         return {}
 
     def changed(self):
-        return bool(self.status())
+        status = self.status()
+        logger.debug(str(status))
+        return bool(status)
+
+    @property
+    def is_empty(self):
+        return self.remote.is_empty(self.path_info)
+
+    def isdir(self):
+        return self.remote.isdir(self.path_info)
+
+    def isfile(self):
+        return self.remote.isfile(self.path_info)
 
     def save(self):
         if not self.exists:
             raise self.DoesNotExistError(self)
 
+        if not self.isfile and not self.isdir:
+            raise self.IsNotFileOrDirError(self)
+
+        if self.is_empty:
+            logger.warning("'{}' is empty.".format(self))
+
+        if not self.use_cache:
+            self.info = self.remote.save_info(self.path_info)
+            if self.metric:
+                self.verify_metric()
+            if not self.IS_DEPENDENCY:
+                logger.info(
+                    "Output '{}' doesn't use cache. Skipping saving.".format(
+                        self
+                    )
+                )
+            return
+
+        assert not self.IS_DEPENDENCY
+
+        if not self.changed():
+            logger.info(
+                "Output '{}' didn't change. Skipping saving.".format(self)
+            )
+            return
+
+        if self.is_in_repo:
+            if self.repo.scm.is_tracked(self.path):
+                raise OutputAlreadyTrackedError(self)
+
+            if self.use_cache:
+                self.repo.scm.ignore(self.path)
+
         self.info = self.remote.save_info(self.path_info)
 
     def commit(self):
         if self.use_cache:
-            getattr(self.repo.cache, self.scheme).save(
-                self.path_info, self.info
-            )
+            self.cache.save(self.path_info, self.info)
 
     def dumpd(self):
-        ret = self.info.copy()
+        ret = copy(self.info)
         ret[self.PARAM_PATH] = self.url
 
         if self.IS_DEPENDENCY:
@@ -221,7 +281,7 @@ class OutputBase(object):
         else:
             info = self.info
 
-        getattr(self.repo.cache, self.scheme).checkout(
+        self.cache.checkout(
             self.path_info,
             info,
             force=force,
@@ -242,7 +302,7 @@ class OutputBase(object):
 
         self.remote.move(self.path_info, out.path_info)
         self.url = out.url
-        self.path_info = out.path_info
+        self.path_info = copy(out.path_info)
         self.save()
         self.commit()
 
@@ -250,10 +310,13 @@ class OutputBase(object):
             self.repo.scm.ignore(self.path)
 
     def get_files_number(self):
-        if self.use_cache and self.checksum:
-            return 1
+        if not self.use_cache or not self.checksum:
+            return 0
 
-        return 0
+        if self.is_dir_checksum:
+            return len(self.dir_cache)
+
+        return 1
 
     def unprotect(self):
         if self.exists:

dvc/output/hdfs.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import posixpath
 
+from dvc.path.hdfs import HDFSPathInfo
 from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
 from dvc.remote.hdfs import RemoteHDFS
@@ -34,4 +35,4 @@ class OutputHDFS(OutputBase):
         if remote:
             path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))
         user = remote.user if remote else self.group("user")
-        self.path_info = {"scheme": "hdfs", "user": user, "path": path}
+        self.path_info = HDFSPathInfo(user=user, url=self.url, path=path)

dvc/output/local.py

@@ -3,11 +3,12 @@ from __future__ import unicode_literals
 import os
 import logging
 
+from dvc.path.local import LocalPathInfo
 from dvc.utils.compat import urlparse
 from dvc.istextfile import istextfile
 from dvc.exceptions import DvcException
 from dvc.remote.local import RemoteLOCAL
-from dvc.output.base import OutputBase, OutputAlreadyTrackedError
+from dvc.output.base import OutputBase
 
 
 logger = logging.getLogger(__name__)
@@ -49,9 +50,7 @@ class OutputLOCAL(OutputBase):
             p = os.path.join(stage.wdir, p)
         p = os.path.abspath(os.path.normpath(p))
 
-        self.path_info = {"scheme": "local", "path": p}
-
-        self._dir_cache = {}
+        self.path_info = LocalPathInfo(url=self.url, path=p)
 
     def __str__(self):
         return self.rel_path
@@ -66,7 +65,7 @@ class OutputLOCAL(OutputBase):
         from dvc.repo import Repo
 
         fullpath = os.path.abspath(stage.wdir)
-        self.path_info["path"] = os.path.join(fullpath, self.stage_path)
+        self.path_info.path = os.path.join(fullpath, self.stage_path)
 
         self.repo = Repo(self.path)
 
@@ -85,18 +84,10 @@ class OutputLOCAL(OutputBase):
     def stage_path(self):
         return os.path.relpath(self.path, self.stage.wdir)
 
-    @property
-    def cache(self):
-        return self.repo.cache.local.get(self.checksum)
-
-    @property
-    def is_dir_cache(self):
-        return self.repo.cache.local.is_dir_cache(self.checksum)
-
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
         if self.is_in_repo:
-            path = self.remote.unixpath(
+            path = self.remote.to_posixpath(
                 os.path.relpath(self.path, self.stage.wdir)
             )
         else:
@@ -120,52 +111,3 @@ class OutputLOCAL(OutputBase):
         if not istextfile(self.path):
             msg = "binary file '{}' cannot be used as metrics."
             raise DvcException(msg.format(self.rel_path))
-
-    def save(self):
-        if not os.path.exists(self.path):
-            raise self.DoesNotExistError(self.rel_path)
-
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
-            raise self.IsNotFileOrDirError(self.rel_path)
-
-        if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or (
-            os.path.isdir(self.path) and len(os.listdir(self.path)) == 0
-        ):
-            msg = "file/directory '{}' is empty.".format(self.rel_path)
-            logger.warning(msg)
-
-        if not self.use_cache:
-            self.info = self.remote.save_info(self.path_info)
-            self.verify_metric()
-            if not self.IS_DEPENDENCY:
-                msg = "Output '{}' doesn't use cache. Skipping saving."
-                logger.info(msg.format(self.rel_path))
-            return
-
-        assert not self.IS_DEPENDENCY
-
-        if not self.changed():
-            msg = "Output '{}' didn't change. Skipping saving."
-            logger.info(msg.format(self.rel_path))
-            return
-
-        if self.is_in_repo:
-            if self.repo.scm.is_tracked(self.path):
-                raise OutputAlreadyTrackedError(self.rel_path)
-
-            if self.use_cache:
-                self.repo.scm.ignore(self.path)
-
-        self.info = self.remote.save_info(self.path_info)
-
-    @property
-    def dir_cache(self):
-        return self.repo.cache.local.load_dir_cache(self.checksum)
-
-    def get_files_number(self):
-        if self.cache is None:
-            return 0
-
-        if self.is_dir_cache:
-            return len(self.dir_cache)
-        return 1

dvc/output/s3.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import posixpath
 
+from dvc.path.utils import PathInfo
 from dvc.remote.s3 import RemoteS3
 from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
@@ -35,8 +36,7 @@ class OutputS3(OutputBase):
         path = urlparse(path).path.lstrip("/")
         if remote:
             path = posixpath.join(remote.prefix, path)
-        self.path_info = {
-            "scheme": self.scheme,
-            "bucket": bucket,
-            "path": path,
-        }
+
+        self.path_info = PathInfo(
+            self.scheme, bucket=bucket, path=path, url=self.url
+        )

dvc/output/ssh.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import getpass
 import posixpath
 
+from dvc.path.ssh import SSHPathInfo
 from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
 from dvc.remote.ssh import RemoteSSH
@@ -48,10 +49,6 @@ class OutputSSH(OutputBase):
         else:
             path = parsed.path
 
-        self.path_info = {
-            "scheme": "ssh",
-            "host": host,
-            "port": port,
-            "user": user,
-            "path": path,
-        }
+        self.path_info = SSHPathInfo(
+            host=host, user=user, port=port, url=self.url, path=path
+        )

dvc/path/__init__.py

@@ -0,0 +1,34 @@
+from dvc.utils.compat import urlunsplit
+
+
+class Schemes:
+    SSH = "ssh"
+    HDFS = "hdfs"
+    S3 = "s3"
+    AZURE = "azure"
+    HTTP = "http"
+    GS = "gs"
+    LOCAL = "local"
+    OSS = "oss"
+
+
+class BasePathInfo(object):
+    scheme = None
+
+    def __init__(self, url=None, path=None):
+        self.url = url
+        self.path = path
+
+    def __str__(self):
+        return self.url
+
+
+class DefaultCloudPathInfo(BasePathInfo):
+    def __init__(self, bucket, url=None, path=None):
+        super(DefaultCloudPathInfo, self).__init__(url, path)
+        self.bucket = bucket
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit((self.scheme, self.bucket, self.path, "", ""))
+        return self.url

dvc/path/azure.py

@@ -0,0 +1,5 @@
+from dvc.path import Schemes, DefaultCloudPathInfo
+
+
+class AzurePathInfo(DefaultCloudPathInfo):
+    scheme = Schemes.AZURE

dvc/path/gs.py

@@ -0,0 +1,5 @@
+from dvc.path import Schemes, DefaultCloudPathInfo
+
+
+class GSPathInfo(DefaultCloudPathInfo):
+    scheme = Schemes.GS

dvc/path/hdfs.py

@@ -0,0 +1,15 @@
+from dvc.utils.compat import urlunsplit
+from dvc.path import BasePathInfo, Schemes
+
+
+class HDFSPathInfo(BasePathInfo):
+    scheme = Schemes.HDFS
+
+    def __init__(self, user, url=None, path=None):
+        super(HDFSPathInfo, self).__init__(url, path)
+        self.user = user
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit((self.scheme, self.user, self.path, "", ""))
+        return self.url

dvc/path/http.py

@@ -0,0 +1,16 @@
+from dvc.path import BasePathInfo, Schemes
+from dvc.utils.compat import urlparse, urlunsplit
+
+
+class HTTPPathInfo(BasePathInfo):
+    @property
+    def scheme(self):
+        if self.path:
+            return urlparse(self.path).scheme
+        else:
+            return Schemes.HTTP
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit((self.scheme, self.path, "", "", ""))
+        return self.url

dvc/path/local.py

@@ -0,0 +1,10 @@
+import os
+
+from dvc.path import BasePathInfo, Schemes
+
+
+class LocalPathInfo(BasePathInfo):
+    scheme = Schemes.LOCAL
+
+    def __str__(self):
+        return os.path.relpath(self.path)

dvc/path/oss.py

@@ -0,0 +1,5 @@
+from dvc.path import Schemes, DefaultCloudPathInfo
+
+
+class OSSPathInfo(DefaultCloudPathInfo):
+    scheme = Schemes.OSS

dvc/path/s3.py

@@ -0,0 +1,5 @@
+from dvc.path import Schemes, DefaultCloudPathInfo
+
+
+class S3PathInfo(DefaultCloudPathInfo):
+    scheme = Schemes.S3

dvc/path/ssh.py

@@ -0,0 +1,25 @@
+from dvc.utils.compat import urlunsplit
+from dvc.path import BasePathInfo, Schemes
+
+
+class SSHPathInfo(BasePathInfo):
+    scheme = Schemes.SSH
+
+    def __init__(self, host, user, port, url=None, path=None):
+        super(SSHPathInfo, self).__init__(url, path)
+        self.host = host
+        self.user = user
+        self.port = port
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit(
+                (
+                    self.scheme,
+                    "{}@{}:{}".format(self.user, self.host, self.port),
+                    self.path,
+                    "",
+                    "",
+                )
+            )
+        return self.url

dvc/path/utils.py

@@ -0,0 +1,25 @@
+from dvc.path import Schemes
+from dvc.path.azure import AzurePathInfo
+from dvc.path.gs import GSPathInfo
+from dvc.path.hdfs import HDFSPathInfo
+from dvc.path.http import HTTPPathInfo
+from dvc.path.local import LocalPathInfo
+from dvc.path.oss import OSSPathInfo
+from dvc.path.s3 import S3PathInfo
+from dvc.path.ssh import SSHPathInfo
+
+PATH_MAP = {
+    Schemes.SSH: SSHPathInfo,
+    Schemes.HDFS: HDFSPathInfo,
+    Schemes.S3: S3PathInfo,
+    Schemes.AZURE: AzurePathInfo,
+    Schemes.HTTP: HTTPPathInfo,
+    Schemes.GS: GSPathInfo,
+    Schemes.LOCAL: LocalPathInfo,
+    Schemes.OSS: OSSPathInfo,
+}
+
+
+def PathInfo(scheme, *args, **kwargs):
+    cls = PATH_MAP[scheme]
+    return cls(*args, **kwargs)

dvc/progress.py

@@ -126,6 +126,16 @@ class Progress(object):
             self.refresh()
         self._lock.release()
 
+    def __call__(self, seq, name="", total=None):
+        if total is None:
+            total = len(seq)
+
+        self.update_target(name, 0, total)
+        for done, item in enumerate(seq, start=1):
+            yield item
+            self.update_target(name, done, total)
+        self.finish_target(name)
+
 
 class ProgressCallback(object):
     def __init__(self, total):

dvc/remote/__init__.py

@@ -7,6 +7,7 @@ from dvc.remote.local import RemoteLOCAL
 from dvc.remote.s3 import RemoteS3
 from dvc.remote.ssh import RemoteSSH
 from dvc.remote.http import RemoteHTTP
+from dvc.remote.oss import RemoteOSS
 
 
 REMOTES = [
@@ -16,6 +17,7 @@ REMOTES = [
     RemoteHTTP,
     RemoteS3,
     RemoteSSH,
+    RemoteOSS,
     # NOTE: RemoteLOCAL is the default
 ]
 

dvc/remote/azure.py

@@ -5,6 +5,9 @@ import os
 import re
 import logging
 
+from dvc.path import Schemes
+from dvc.path.azure import AzurePathInfo
+
 try:
     from azure.storage.blob import BlockBlobService
     from azure.common import AzureMissingResourceHttpError
@@ -30,7 +33,7 @@ class Callback(object):
 
 
 class RemoteAzure(RemoteBase):
-    scheme = "azure"
+    scheme = Schemes.AZURE
     REGEX = (
         r"azure://((?P<path>[^=;]*)?|("
         # backward compatibility
@@ -69,8 +72,7 @@ class RemoteAzure(RemoteBase):
             raise ValueError("azure storage connection string missing")
 
         self.__blob_service = None
-
-        self.path_info = {"scheme": self.scheme, "bucket": self.bucket}
+        self.path_info = AzurePathInfo(bucket=self.bucket)
 
     @property
     def blob_service(self):
@@ -90,16 +92,14 @@ class RemoteAzure(RemoteBase):
         return self.__blob_service
 
     def remove(self, path_info):
-        if path_info["scheme"] != self.scheme:
+        if path_info.scheme != self.scheme:
             raise NotImplementedError
 
         logger.debug(
-            "Removing azure://{}/{}".format(
-                path_info["bucket"], path_info["path"]
-            )
+            "Removing azure://{}/{}".format(path_info.bucket, path_info.path)
         )
 
-        self.blob_service.delete_blob(path_info["bucket"], path_info["path"])
+        self.blob_service.delete_blob(path_info.bucket, path_info.path)
 
     def _list_paths(self, bucket, prefix):
         blob_service = self.blob_service
@@ -120,36 +120,36 @@ class RemoteAzure(RemoteBase):
     def list_cache_paths(self):
         return self._list_paths(self.bucket, self.prefix)
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info["scheme"] != self.scheme:
+            if to_info.scheme != self.scheme:
                 raise NotImplementedError
 
-            if from_info["scheme"] != "local":
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
-            bucket = to_info["bucket"]
-            path = to_info["path"]
+            bucket = to_info.bucket
+            path = to_info.path
 
             logger.debug(
                 "Uploading '{}' to '{}/{}'".format(
-                    from_info["path"], bucket, path
+                    from_info.path, bucket, path
                 )
             )
 
             if not name:
-                name = os.path.basename(from_info["path"])
+                name = os.path.basename(from_info.path)
 
-            cb = Callback(name)
+            cb = None if no_progress_bar else Callback(name)
 
             try:
                 self.blob_service.create_blob_from_path(
-                    bucket, path, from_info["path"], progress_callback=cb
+                    bucket, path, from_info.path, progress_callback=cb
                 )
             except Exception:
-                msg = "failed to upload '{}'".format(from_info["path"])
+                msg = "failed to upload '{}'".format(from_info.path)
                 logger.warning(msg)
             else:
                 progress.finish_target(name)
@@ -165,28 +165,28 @@ class RemoteAzure(RemoteBase):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != self.scheme:
+            if from_info.scheme != self.scheme:
                 raise NotImplementedError
 
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
-            bucket = from_info["bucket"]
-            path = from_info["path"]
+            bucket = from_info.bucket
+            path = from_info.path
 
             logger.debug(
                 "Downloading '{}/{}' to '{}'".format(
-                    bucket, path, to_info["path"]
+                    bucket, path, to_info.path
                 )
             )
 
-            tmp_file = tmp_fname(to_info["path"])
+            tmp_file = tmp_fname(to_info.path)
             if not name:
-                name = os.path.basename(to_info["path"])
+                name = os.path.basename(to_info.path)
 
             cb = None if no_progress_bar else Callback(name)
 
-            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
 
             try:
                 self.blob_service.get_blob_to_path(
@@ -196,7 +196,7 @@ class RemoteAzure(RemoteBase):
                 msg = "failed to download '{}/{}'".format(bucket, path)
                 logger.warning(msg)
             else:
-                move(tmp_file, to_info["path"])
+                move(tmp_file, to_info.path)
 
                 if not no_progress_bar:
                     progress.finish_target(name)

dvc/remote/base.py

@@ -1,15 +1,26 @@
 from __future__ import unicode_literals
 
+from copy import copy
+
+from dvc.path.local import LocalPathInfo
 from dvc.utils.compat import str
 
+import os
 import re
-import posixpath
+import json
+import ntpath
 import logging
+import tempfile
+import posixpath
+from operator import itemgetter
 from multiprocessing import cpu_count
 
 import dvc.prompt as prompt
 from dvc.config import Config
 from dvc.exceptions import DvcException, ConfirmRemoveError
+from dvc.progress import progress
+from dvc.utils import LARGE_DIR_SIZE, tmp_fname
+from dvc.state import StateBase
 
 
 logger = logging.getLogger(__name__)
@@ -61,6 +72,9 @@ class RemoteBase(object):
     REQUIRES = {}
     JOBS = 4 * cpu_count()
 
+    PARAM_RELPATH = "relpath"
+    CHECKSUM_DIR_SUFFIX = ".dir"
+
     def __init__(self, repo, config):
         self.repo = repo
         deps_ok = all(self.REQUIRES.values())
@@ -89,6 +103,11 @@ class RemoteBase(object):
             ).format(url, missing, " ".join(missing), self.scheme)
             raise RemoteMissingDepsError(msg)
 
+        self.protected = False
+        self.state = StateBase()
+
+        self._dir_info = {}
+
     def __repr__(self):
         return "{class_name}: '{url}'".format(
             class_name=type(self).__name__, url=(self.url or "No url")
@@ -102,6 +121,27 @@ class RemoteBase(object):
         url = config[Config.SECTION_REMOTE_URL]
         return cls.match(url) is not None
 
+    @staticmethod
+    def _replace_sep(path, from_sep, to_sep):
+        assert not ntpath.isabs(path)
+        assert not posixpath.isabs(path)
+        return path.replace(from_sep, to_sep)
+
+    @classmethod
+    def to_posixpath(cls, path):
+        return cls._replace_sep(path, "\\", "/")
+
+    @classmethod
+    def to_ntpath(cls, path):
+        return cls._replace_sep(path, "/", "\\")
+
+    def to_ospath(self, path):
+        if self.ospath == ntpath:
+            return self.to_ntpath(path)
+        elif self.ospath == posixpath:
+            return self.to_posixpath(path)
+        assert False
+
     @classmethod
     def match(cls, url):
         return re.match(cls.REGEX, url)
@@ -112,9 +152,137 @@ class RemoteBase(object):
             return None
         return m.group(name)
 
-    def save_info(self, path_info):
+    @property
+    def cache(self):
+        return getattr(self.repo.cache, self.scheme)
+
+    def get_file_checksum(self, path_info):
         raise NotImplementedError
 
+    def _collect_dir(self, path_info):
+        dir_info = []
+
+        p_info = copy(path_info)
+        dpath = p_info.path
+        for root, dirs, files in self.walk(path_info):
+            if len(files) > LARGE_DIR_SIZE:
+                msg = (
+                    "Computing md5 for a large directory {}. "
+                    "This is only done once."
+                )
+                relpath = self.ospath.relpath(root)
+                logger.info(msg.format(relpath))
+                files = progress(files, name=relpath)
+
+            for fname in files:
+                path = self.ospath.join(root, fname)
+                p_info.path = path
+                relpath = self.to_posixpath(self.ospath.relpath(path, dpath))
+
+                checksum = self.get_file_checksum(p_info)
+                dir_info.append(
+                    {
+                        self.PARAM_RELPATH: relpath,
+                        self.PARAM_CHECKSUM: checksum,
+                    }
+                )
+
+        # NOTE: sorting the list by path to ensure reproducibility
+        return sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
+
+    def get_dir_checksum(self, path_info):
+        dir_info = self._collect_dir(path_info)
+        checksum, from_info, to_info = self._get_dir_info_checksum(
+            dir_info, path_info
+        )
+        if self.cache.changed_cache_file(checksum):
+            self.cache.move(from_info, to_info)
+
+        self.state.save(path_info, checksum)
+        self.state.save(to_info, checksum)
+
+        return checksum
+
+    def _get_dir_info_checksum(self, dir_info, path_info):
+        to_info = copy(path_info)
+        to_info.path = self.cache.ospath.join(self.cache.prefix, tmp_fname(""))
+
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        with open(tmp, "w+") as fobj:
+            json.dump(dir_info, fobj, sort_keys=True)
+
+        from_info = LocalPathInfo(path=tmp)
+        self.cache.upload([from_info], [to_info], no_progress_bar=True)
+
+        checksum = self.get_file_checksum(to_info) + self.CHECKSUM_DIR_SUFFIX
+        from_info = copy(to_info)
+        to_info.path = self.cache.checksum_to_path(checksum)
+        return checksum, from_info, to_info
+
+    def get_dir_cache(self, checksum):
+        assert checksum
+
+        dir_info = self._dir_info.get(checksum)
+        if dir_info:
+            return dir_info
+
+        dir_info = self.load_dir_cache(checksum)
+        self._dir_info[checksum] = dir_info
+        return dir_info
+
+    def load_dir_cache(self, checksum):
+        path_info = self.checksum_to_path_info(checksum)
+
+        fobj = tempfile.NamedTemporaryFile(delete=False)
+        path = fobj.name
+        to_info = LocalPathInfo(path=path)
+        self.cache.download([path_info], [to_info], no_progress_bar=True)
+
+        try:
+            with open(path, "r") as fobj:
+                d = json.load(fobj)
+        except ValueError:
+            logger.exception("Failed to load dir cache '{}'".format(path_info))
+            return []
+        finally:
+            os.unlink(path)
+
+        if not isinstance(d, list):
+            msg = "dir cache file format error '{}' [skipping the file]"
+            logger.error(msg.format(os.path.relpath(path)))
+            return []
+
+        for info in d:
+            info[self.PARAM_RELPATH] = self.to_ospath(info[self.PARAM_RELPATH])
+
+        return d
+
+    @classmethod
+    def is_dir_checksum(cls, checksum):
+        return checksum.endswith(cls.CHECKSUM_DIR_SUFFIX)
+
+    def get_checksum(self, path_info):
+        if not self.exists(path_info):
+            return None
+
+        checksum = self.state.get(path_info)
+        if checksum:
+            return checksum
+
+        if self.isdir(path_info):
+            checksum = self.get_dir_checksum(path_info)
+        else:
+            checksum = self.get_file_checksum(path_info)
+
+        if checksum:
+            self.state.save(path_info, checksum)
+
+        return checksum
+
+    def save_info(self, path_info):
+        assert path_info.scheme == self.scheme
+        return {self.PARAM_CHECKSUM: self.get_checksum(path_info)}
+
     def changed(self, path_info, checksum_info):
         """Checks if data has changed.
 
@@ -165,22 +333,82 @@ class RemoteBase(object):
         logger.debug("'{}' hasn't changed.".format(path_info))
         return False
 
+    def link(self, from_info, to_info):
+        self.copy(from_info, to_info)
+
+    def _save_file(self, path_info, checksum, save_link=True):
+        assert checksum
+
+        cache_info = self.checksum_to_path_info(checksum)
+        if self.changed_cache(checksum):
+            self.move(path_info, cache_info)
+        else:
+            self.remove(path_info)
+
+        self.link(cache_info, path_info)
+
+        if save_link:
+            self.state.save_link(path_info)
+
+        # we need to update path and cache, since in case of reflink,
+        # or copy cache type moving original file results in updates on
+        # next executed command, which causes md5 recalculation
+        self.state.save(path_info, checksum)
+        self.state.save(cache_info, checksum)
+
+    def _save_dir(self, path_info, checksum):
+        cache_info = self.checksum_to_path_info(checksum)
+        dir_info = self.get_dir_cache(checksum)
+
+        entry_info = copy(path_info)
+        for entry in dir_info:
+            entry_checksum = entry[self.PARAM_CHECKSUM]
+            entry_info.path = self.ospath.join(
+                path_info.path, entry[self.PARAM_RELPATH]
+            )
+
+            self._save_file(entry_info, entry_checksum, save_link=False)
+
+        self.state.save_link(path_info)
+        self.state.save(cache_info, checksum)
+        self.state.save(path_info, checksum)
+
+    def is_empty(self, path_info):
+        return False
+
+    def isfile(self, path_info):
+        raise NotImplementedError
+
+    def isdir(self, path_info):
+        return False
+
+    def walk(self, path_info):
+        raise NotImplementedError
+
+    def protect(self, path_info):
+        pass
+
     def save(self, path_info, checksum_info):
-        if path_info["scheme"] != self.scheme:
+        if path_info.scheme != self.scheme:
             raise RemoteActionNotImplemented(
-                "save {} -> {}".format(path_info["scheme"], self.scheme),
+                "save {} -> {}".format(path_info.scheme, self.scheme),
                 self.scheme,
             )
 
         checksum = checksum_info[self.PARAM_CHECKSUM]
         if not self.changed_cache(checksum):
+            self._checkout(path_info, checksum)
             return
 
+        self._save(path_info, checksum)
+
+    def _save(self, path_info, checksum):
         to_info = self.checksum_to_path_info(checksum)
-
         logger.info("Saving '{}' to '{}'.".format(path_info, to_info))
-
-        self.copy(path_info, to_info)
+        if self.isdir(path_info):
+            self._save_dir(path_info, checksum)
+            return
+        self._save_file(path_info, checksum)
 
     def download(
         self,
@@ -192,14 +420,15 @@ class RemoteBase(object):
     ):
         raise RemoteActionNotImplemented("download", self.scheme)
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         raise RemoteActionNotImplemented("upload", self.scheme)
 
     def remove(self, path_info):
         raise RemoteActionNotImplemented("remove", self.scheme)
 
-    def move(self, path_info):
-        raise RemoteActionNotImplemented("move", self.scheme)
+    def move(self, from_info, to_info):
+        self.copy(from_info, to_info)
+        self.remove(from_info)
 
     def copy(self, from_info, to_info):
         raise RemoteActionNotImplemented("copy", self.scheme)
@@ -226,6 +455,7 @@ class RemoteBase(object):
         return posixpath
 
     def checksum_to_path(self, checksum):
+        assert checksum
         return self.ospath.join(self.prefix, checksum[0:2], checksum[2:])
 
     def path_to_checksum(self, path):
@@ -233,8 +463,8 @@ class RemoteBase(object):
         return self.ospath.dirname(relpath) + self.ospath.basename(relpath)
 
     def checksum_to_path_info(self, checksum):
-        path_info = self.path_info.copy()
-        path_info["path"] = self.checksum_to_path(checksum)
+        path_info = copy(self.path_info)
+        path_info.path = self.checksum_to_path(checksum)
         return path_info
 
     def md5s_to_path_infos(self, md5s):
@@ -255,10 +485,13 @@ class RemoteBase(object):
     def gc(self, cinfos):
         from dvc.remote.local import RemoteLOCAL
 
-        used = [info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]]
+        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
 
         if self.scheme != "":
-            used += [info[self.PARAM_CHECKSUM] for info in cinfos[self.scheme]]
+            used |= {
+                info[self.PARAM_CHECKSUM]
+                for info in cinfos.get(self.scheme, [])
+            }
 
         removed = False
         for checksum in self.all():
@@ -269,28 +502,51 @@ class RemoteBase(object):
             removed = True
         return removed
 
-    def changed_cache(self, checksum):
-        cache = self.checksum_to_path_info(checksum)
-        expected = {self.PARAM_CHECKSUM: checksum}
-
-        if not self.exists(cache):
-            return True
-
-        actual = self.save_info(cache)
+    def changed_cache_file(self, checksum):
+        cache_info = self.checksum_to_path_info(checksum)
+        actual = self.get_checksum(cache_info)
 
         logger.debug(
-            "Cache '{}' actual '{}'.".format(str(expected), str(actual))
+            "cache '{}' expected '{}' actual '{}'".format(
+                str(cache_info), checksum, actual
+            )
         )
 
-        if expected != actual:
-            if self.exists(cache):
-                msg = "corrupted cache file {}"
-                logger.warning(msg.format(str(cache)))
-                self.remove(cache)
+        if not checksum or not actual:
             return True
 
+        if actual.split(".")[0] == checksum.split(".")[0]:
+            return False
+
+        if self.exists(cache_info):
+            logger.warning("corrupted cache file '{}'.".format(cache_info))
+            self.remove(cache_info)
+
+        return True
+
+    def _changed_cache_dir(self):
+        # NOTE: only implemented for RemoteLOCAL
+        return True
+
+    def _changed_dir_cache(self, checksum):
+        if not self._changed_cache_dir():
+            return False
+
+        if self.changed_cache_file(checksum):
+            return True
+
+        for entry in self.get_dir_cache(checksum):
+            checksum = entry[self.PARAM_CHECKSUM]
+            if self.changed_cache_file(checksum):
+                return True
+
         return False
 
+    def changed_cache(self, checksum):
+        if self.is_dir_checksum(checksum):
+            return self._changed_dir_cache(checksum)
+        return self.changed_cache_file(checksum)
+
     def cache_exists(self, checksums):
         # NOTE: The reason for such an odd logic is that most of the remotes
         # take much shorter time to just retrieve everything they have under
@@ -303,7 +559,7 @@ class RemoteBase(object):
         return list(filter(lambda checksum: checksum in checksums, self.all()))
 
     def already_cached(self, path_info):
-        current = self.save_info(path_info)[self.PARAM_CHECKSUM]
+        current = self.get_checksum(path_info)
 
         if not current:
             return False
@@ -325,22 +581,81 @@ class RemoteBase(object):
 
         self.remove(path_info)
 
-    def do_checkout(
-        self, path_info, checksum, force=False, progress_callback=None
+    def _checkout_file(
+        self, path_info, checksum, force, progress_callback=None
     ):
+        cache_info = self.checksum_to_path_info(checksum)
         if self.exists(path_info):
             msg = "data '{}' exists. Removing before checkout."
             logger.warning(msg.format(str(path_info)))
             self.safe_remove(path_info, force=force)
 
-        from_info = self.checksum_to_path_info(checksum)
-        self.copy(from_info, path_info)
+        self.link(cache_info, path_info)
+        self.state.save_link(path_info)
+        if progress_callback:
+            progress_callback.update(path_info.url)
+
+    def makedirs(self, path_info):
+        raise NotImplementedError
+
+    def _checkout_dir(
+        self, path_info, checksum, force, progress_callback=None
+    ):
+        # Create dir separately so that dir is created
+        # even if there are no files in it
+        if not self.exists(path_info):
+            self.makedirs(path_info)
+
+        dir_info = self.get_dir_cache(checksum)
+
+        logger.debug("Linking directory '{}'.".format(path_info))
+
+        entry_info = copy(path_info)
+        for entry in dir_info:
+            relpath = entry[self.PARAM_RELPATH]
+            checksum = entry[self.PARAM_CHECKSUM]
+            entry_cache_info = self.checksum_to_path_info(checksum)
+            entry_info.url = self.ospath.join(path_info.url, relpath)
+            entry_info.path = self.ospath.join(path_info.path, relpath)
+
+            entry_checksum_info = {self.PARAM_CHECKSUM: checksum}
+            if self.changed(entry_info, entry_checksum_info):
+                if self.exists(entry_info):
+                    self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+            if progress_callback:
+                progress_callback.update(entry_info.url)
+
+        self._remove_redundant_files(path_info, dir_info, force)
+
+        self.state.save_link(path_info)
+
+    def _remove_redundant_files(self, path_info, dir_info, force):
+        existing_files = set(
+            self.ospath.join(root, fname)
+            for root, _, files in self.walk(path_info)
+            for fname in files
+        )
+
+        needed_files = set(
+            self.ospath.join(path_info.path, entry[self.PARAM_RELPATH])
+            for entry in dir_info
+        )
+
+        delta = existing_files - needed_files
+
+        d_info = copy(path_info)
+        for path in delta:
+            d_info.path = path
+            self.safe_remove(d_info, force)
 
     def checkout(
         self, path_info, checksum_info, force=False, progress_callback=None
     ):
-        scheme = path_info["scheme"]
-        if scheme not in ["", "local"] and scheme != self.scheme:
+        if (
+            path_info.scheme not in ["local"]
+            and path_info.scheme != self.scheme
+        ):
             raise NotImplementedError
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
@@ -363,11 +678,17 @@ class RemoteBase(object):
         msg = "Checking out '{}' with cache '{}'."
         logger.debug(msg.format(str(path_info), checksum))
 
-        self.do_checkout(
-            path_info,
-            checksum,
-            force=force,
-            progress_callback=progress_callback,
+        return self._checkout(path_info, checksum, force, progress_callback)
+
+    def _checkout(
+        self, path_info, checksum, force=False, progress_callback=None
+    ):
+        if not self.is_dir_checksum(checksum):
+            return self._checkout_file(
+                path_info, checksum, force, progress_callback=progress_callback
+            )
+        return self._checkout_dir(
+            path_info, checksum, force, progress_callback=progress_callback
         )
 
     @staticmethod

dvc/remote/gs.py

@@ -3,6 +3,9 @@ from __future__ import unicode_literals
 import os
 import logging
 
+from dvc.path import Schemes
+from dvc.path.gs import GSPathInfo
+
 try:
     from google.cloud import storage
 except ImportError:
@@ -20,7 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 class RemoteGS(RemoteBase):
-    scheme = "gs"
+    scheme = Schemes.GS
     REGEX = r"^gs://(?P<path>.*)$"
     REQUIRES = {"google.cloud.storage": storage}
     PARAM_CHECKSUM = "md5"
@@ -38,7 +41,7 @@ class RemoteGS(RemoteBase):
         self.bucket = parsed.netloc
         self.prefix = parsed.path.lstrip("/")
 
-        self.path_info = {"scheme": "gs", "bucket": self.bucket}
+        self.path_info = GSPathInfo(bucket=self.bucket)
 
     @staticmethod
     def compat_config(config):
@@ -55,10 +58,12 @@ class RemoteGS(RemoteBase):
             else storage.Client(self.projectname)
         )
 
-    def get_md5(self, bucket, path):
+    def get_file_checksum(self, path_info):
         import base64
         import codecs
 
+        bucket = path_info.bucket
+        path = path_info.path
         blob = self.gs.bucket(bucket).get_blob(path)
         if not blob:
             return None
@@ -67,40 +72,28 @@ class RemoteGS(RemoteBase):
         md5 = base64.b64decode(b64_md5)
         return codecs.getencoder("hex")(md5)[0].decode("utf-8")
 
-    def save_info(self, path_info):
-        if path_info["scheme"] != "gs":
-            raise NotImplementedError
-
-        return {
-            self.PARAM_CHECKSUM: self.get_md5(
-                path_info["bucket"], path_info["path"]
-            )
-        }
-
     def copy(self, from_info, to_info, gs=None):
         gs = gs if gs else self.gs
 
-        blob = gs.bucket(from_info["bucket"]).get_blob(from_info["path"])
+        blob = gs.bucket(from_info.bucket).get_blob(from_info.path)
         if not blob:
-            msg = "'{}' doesn't exist in the cloud".format(from_info["path"])
+            msg = "'{}' doesn't exist in the cloud".format(from_info.path)
             raise DvcException(msg)
 
-        bucket = self.gs.bucket(to_info["bucket"])
+        bucket = self.gs.bucket(to_info.bucket)
         bucket.copy_blob(
-            blob, self.gs.bucket(to_info["bucket"]), new_name=to_info["path"]
+            blob, self.gs.bucket(to_info.bucket), new_name=to_info.path
         )
 
     def remove(self, path_info):
-        if path_info["scheme"] != "gs":
+        if path_info.scheme != "gs":
             raise NotImplementedError
 
         logger.debug(
-            "Removing gs://{}/{}".format(
-                path_info["bucket"], path_info["path"]
-            )
+            "Removing gs://{}/{}".format(path_info.bucket, path_info.path)
         )
 
-        blob = self.gs.bucket(path_info["bucket"]).get_blob(path_info["path"])
+        blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
         if not blob:
             return
 
@@ -115,44 +108,43 @@ class RemoteGS(RemoteBase):
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info["scheme"] == "gs"
+        assert path_info.scheme == "gs"
 
-        paths = self._list_paths(path_info["bucket"], path_info["path"])
-        return any(path_info["path"] == path for path in paths)
+        paths = self._list_paths(path_info.bucket, path_info.path)
+        return any(path_info.path == path for path in paths)
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         gs = self.gs
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info["scheme"] != "gs":
+            if to_info.scheme != "gs":
                 raise NotImplementedError
 
-            if from_info["scheme"] != "local":
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
             logger.debug(
                 "Uploading '{}' to '{}/{}'".format(
-                    from_info["path"], to_info["bucket"], to_info["path"]
+                    from_info.path, to_info.bucket, to_info.path
                 )
             )
 
             if not name:
-                name = os.path.basename(from_info["path"])
+                name = os.path.basename(from_info.path)
 
-            progress.update_target(name, 0, None)
+            if not no_progress_bar:
+                progress.update_target(name, 0, None)
 
             try:
-                bucket = gs.bucket(to_info["bucket"])
-                blob = bucket.blob(to_info["path"])
-                blob.upload_from_filename(from_info["path"])
+                bucket = gs.bucket(to_info.bucket)
+                blob = bucket.blob(to_info.path)
+                blob.upload_from_filename(from_info.path)
             except Exception:
                 msg = "failed to upload '{}' to '{}/{}'"
                 logger.exception(
-                    msg.format(
-                        from_info["path"], to_info["bucket"], to_info["path"]
-                    )
+                    msg.format(from_info.path, to_info.bucket, to_info.path)
                 )
                 continue
 
@@ -171,46 +163,44 @@ class RemoteGS(RemoteBase):
         gs = self.gs
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != "gs":
+            if from_info.scheme != "gs":
                 raise NotImplementedError
 
-            if to_info["scheme"] == "gs":
+            if to_info.scheme == "gs":
                 self.copy(from_info, to_info, gs=gs)
                 continue
 
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
             msg = "Downloading '{}/{}' to '{}'".format(
-                from_info["bucket"], from_info["path"], to_info["path"]
+                from_info.bucket, from_info.path, to_info.path
             )
             logger.debug(msg)
 
-            tmp_file = tmp_fname(to_info["path"])
+            tmp_file = tmp_fname(to_info.path)
             if not name:
-                name = os.path.basename(to_info["path"])
+                name = os.path.basename(to_info.path)
 
             if not no_progress_bar:
                 # percent_cb is not available for download_to_filename, so
                 # lets at least update progress at pathpoints(start, finish)
                 progress.update_target(name, 0, None)
 
-            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
 
             try:
-                bucket = gs.bucket(from_info["bucket"])
-                blob = bucket.get_blob(from_info["path"])
+                bucket = gs.bucket(from_info.bucket)
+                blob = bucket.get_blob(from_info.path)
                 blob.download_to_filename(tmp_file)
             except Exception:
                 msg = "failed to download '{}/{}' to '{}'"
                 logger.exception(
-                    msg.format(
-                        from_info["bucket"], from_info["path"], to_info["path"]
-                    )
+                    msg.format(from_info.bucket, from_info.path, to_info.path)
                 )
                 continue
 
-            move(tmp_file, to_info["path"])
+            move(tmp_file, to_info.path)
 
             if not no_progress_bar:
                 progress.finish_target(name)

dvc/remote/hdfs.py

@@ -8,6 +8,8 @@ import logging
 from subprocess import Popen, PIPE
 
 from dvc.config import Config
+from dvc.path import Schemes
+from dvc.path.hdfs import HDFSPathInfo
 from dvc.remote.base import RemoteBase, RemoteCmdError
 from dvc.utils import fix_env, tmp_fname
 
@@ -16,7 +18,7 @@ logger = logging.getLogger(__name__)
 
 
 class RemoteHDFS(RemoteBase):
-    scheme = "hdfs"
+    scheme = Schemes.HDFS
     REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
     PARAM_CHECKSUM = "checksum"
 
@@ -30,7 +32,7 @@ class RemoteHDFS(RemoteBase):
                 Config.SECTION_REMOTE_USER, getpass.getuser()
             )
 
-        self.path_info = {"scheme": "hdfs", "user": self.user}
+        self.path_info = HDFSPathInfo(user=self.user)
 
     def hadoop_fs(self, cmd, user=None):
         cmd = "hadoop fs -" + cmd
@@ -63,79 +65,68 @@ class RemoteHDFS(RemoteBase):
         assert match is not None
         return match.group(gname)
 
-    def checksum(self, path_info):
+    def get_file_checksum(self, path_info):
         regex = r".*\t.*\t(?P<checksum>.*)"
         stdout = self.hadoop_fs(
-            "checksum {}".format(path_info["path"]), user=path_info["user"]
+            "checksum {}".format(path_info.path), user=path_info.user
         )
         return self._group(regex, stdout, "checksum")
 
     def copy(self, from_info, to_info):
-        dname = posixpath.dirname(to_info["path"])
-        self.hadoop_fs("mkdir -p {}".format(dname), user=to_info["user"])
+        dname = posixpath.dirname(to_info.path)
+        self.hadoop_fs("mkdir -p {}".format(dname), user=to_info.user)
         self.hadoop_fs(
-            "cp -f {} {}".format(from_info["path"], to_info["path"]),
-            user=to_info["user"],
+            "cp -f {} {}".format(from_info.path, to_info.path),
+            user=to_info.user,
         )
 
     def rm(self, path_info):
-        self.hadoop_fs(
-            "rm -f {}".format(path_info["path"]), user=path_info["user"]
-        )
-
-    def save_info(self, path_info):
-        if path_info["scheme"] != "hdfs":
-            raise NotImplementedError
-
-        assert path_info.get("path")
-
-        return {self.PARAM_CHECKSUM: self.checksum(path_info)}
+        self.hadoop_fs("rm -f {}".format(path_info.path), user=path_info.user)
 
     def remove(self, path_info):
-        if path_info["scheme"] != "hdfs":
+        if path_info.scheme != "hdfs":
             raise NotImplementedError
 
-        assert path_info.get("path")
+        assert path_info.path
 
-        logger.debug("Removing {}".format(path_info["path"]))
+        logger.debug("Removing {}".format(path_info.path))
 
         self.rm(path_info)
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info["scheme"] == "hdfs"
+        assert path_info.scheme == "hdfs"
 
         try:
-            self.hadoop_fs("test -e {}".format(path_info["path"]))
+            self.hadoop_fs("test -e {}".format(path_info.path))
             return True
         except RemoteCmdError:
             return False
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info["scheme"] != "hdfs":
+            if to_info.scheme != "hdfs":
                 raise NotImplementedError
 
-            if from_info["scheme"] != "local":
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
             self.hadoop_fs(
-                "mkdir -p {}".format(posixpath.dirname(to_info["path"])),
-                user=to_info["user"],
+                "mkdir -p {}".format(posixpath.dirname(to_info.path)),
+                user=to_info.user,
             )
 
-            tmp_file = tmp_fname(to_info["path"])
+            tmp_file = tmp_fname(to_info.path)
 
             self.hadoop_fs(
-                "copyFromLocal {} {}".format(from_info["path"], tmp_file),
-                user=to_info["user"],
+                "copyFromLocal {} {}".format(from_info.path, tmp_file),
+                user=to_info.user,
             )
 
             self.hadoop_fs(
-                "mv {} {}".format(tmp_file, to_info["path"]),
-                user=to_info["user"],
+                "mv {} {}".format(tmp_file, to_info.path), user=to_info.user
             )
 
     def download(
@@ -149,28 +140,28 @@ class RemoteHDFS(RemoteBase):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != "hdfs":
+            if from_info.scheme != "hdfs":
                 raise NotImplementedError
 
-            if to_info["scheme"] == "hdfs":
+            if to_info.scheme == "hdfs":
                 self.copy(from_info, to_info)
                 continue
 
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
-            dname = os.path.dirname(to_info["path"])
+            dname = os.path.dirname(to_info.path)
             if not os.path.exists(dname):
                 os.makedirs(dname)
 
-            tmp_file = tmp_fname(to_info["path"])
+            tmp_file = tmp_fname(to_info.path)
 
             self.hadoop_fs(
-                "copyToLocal {} {}".format(from_info["path"], tmp_file),
-                user=from_info["user"],
+                "copyToLocal {} {}".format(from_info.path, tmp_file),
+                user=from_info.user,
             )
 
-            os.rename(tmp_file, to_info["path"])
+            os.rename(tmp_file, to_info.path)
 
     def list_cache_paths(self):
         try:

dvc/remote/http.py

@@ -1,4 +1,7 @@
 from __future__ import unicode_literals
+
+from dvc.path import Schemes
+from dvc.path.http import HTTPPathInfo
 from dvc.utils.compat import open, makedirs
 
 import os
@@ -30,7 +33,7 @@ class ProgressBarCallback(object):
 
 
 class RemoteHTTP(RemoteBase):
-    scheme = "http"
+    scheme = Schemes.HTTP
     REGEX = r"^https?://.*$"
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 2 ** 16
@@ -40,7 +43,8 @@ class RemoteHTTP(RemoteBase):
         super(RemoteHTTP, self).__init__(repo, config)
         self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
         self.url = self.cache_dir
-        self.path_info = {"scheme": "http"}
+
+        self.path_info = HTTPPathInfo()
 
     @property
     def prefix(self):
@@ -57,23 +61,23 @@ class RemoteHTTP(RemoteBase):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] not in ["http", "https"]:
+            if from_info.scheme not in ["http", "https"]:
                 raise NotImplementedError
 
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
             msg = "Downloading '{}' to '{}'".format(
-                from_info["path"], to_info["path"]
+                from_info.path, to_info.path
             )
             logger.debug(msg)
 
             if not name:
-                name = os.path.basename(to_info["path"])
+                name = os.path.basename(to_info.path)
 
-            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
 
-            total = self._content_length(from_info["path"])
+            total = self._content_length(from_info.path)
 
             if no_progress_bar or not total:
                 cb = None
@@ -82,14 +86,11 @@ class RemoteHTTP(RemoteBase):
 
             try:
                 self._download_to(
-                    from_info["path"],
-                    to_info["path"],
-                    callback=cb,
-                    resume=resume,
+                    from_info.path, to_info.path, callback=cb, resume=resume
                 )
 
             except Exception:
-                msg = "failed to download '{}'".format(from_info["path"])
+                msg = "failed to download '{}'".format(from_info.path)
                 logger.exception(msg)
                 continue
 
@@ -98,8 +99,8 @@ class RemoteHTTP(RemoteBase):
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info["scheme"] in ["http", "https"]
-        return bool(self._request("HEAD", path_info.get("path")))
+        assert path_info.scheme in ["http", "https"]
+        return bool(self._request("HEAD", path_info.path))
 
     def cache_exists(self, md5s):
         assert isinstance(md5s, list)
@@ -109,16 +110,11 @@ class RemoteHTTP(RemoteBase):
 
         return list(filter(func, md5s))
 
-    def save_info(self, path_info):
-        if path_info["scheme"] not in ["http", "https"]:
-            raise NotImplementedError
-
-        return {self.PARAM_CHECKSUM: self._etag(path_info["path"])}
-
     def _content_length(self, url):
         return self._request("HEAD", url).headers.get("Content-Length")
 
-    def _etag(self, url):
+    def get_file_checksum(self, path_info):
+        url = path_info.path
         etag = self._request("HEAD", url).headers.get("ETag") or self._request(
             "HEAD", url
         ).headers.get("Content-MD5")

dvc/remote/local/__init__.py

@@ -1,16 +1,19 @@
 from __future__ import unicode_literals
 
+from copy import copy
+
+from dvc.path import BasePathInfo, Schemes
+from dvc.path.local import LocalPathInfo
+from dvc.remote.local.slow_link_detection import slow_link_guard
 from dvc.utils.compat import str, makedirs
 
 import os
 import stat
 import uuid
-import json
 import ntpath
 import shutil
 import posixpath
 import logging
-from operator import itemgetter
 
 from dvc.system import System
 from dvc.remote.base import (
@@ -24,12 +27,11 @@ from dvc.utils import (
     remove,
     move,
     copyfile,
-    dict_md5,
     to_chunks,
     tmp_fname,
+    file_md5,
     walk_files,
 )
-from dvc.utils import LARGE_DIR_SIZE
 from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.progress import progress
@@ -42,14 +44,12 @@ logger = logging.getLogger(__name__)
 
 
 class RemoteLOCAL(RemoteBase):
-    scheme = "local"
+    scheme = Schemes.LOCAL
     REGEX = r"^(?P<path>.*)$"
     PARAM_CHECKSUM = "md5"
     PARAM_PATH = "path"
-    PARAM_RELPATH = "relpath"
-    MD5_DIR_SUFFIX = ".dir"
 
-    CACHE_TYPES = ["reflink", "hardlink", "symlink", "copy"]
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
     CACHE_TYPE_MAP = {
         "copy": shutil.copyfile,
         "symlink": System.symlink,
@@ -74,14 +74,13 @@ class RemoteLOCAL(RemoteBase):
                 types = [t.strip() for t in types.split(",")]
             self.cache_types = types
         else:
-            self.cache_types = self.CACHE_TYPES
+            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
 
         if self.cache_dir is not None and not os.path.exists(self.cache_dir):
             os.mkdir(self.cache_dir)
 
-        self.path_info = {"scheme": "local"}
-
         self._dir_info = {}
+        self.path_info = LocalPathInfo()
 
     @staticmethod
     def compat_config(config):
@@ -114,44 +113,22 @@ class RemoteLOCAL(RemoteBase):
         if not md5:
             return None
 
-        return os.path.join(self.cache_dir, md5[0:2], md5[2:])
-
-    def changed_cache_file(self, md5):
-        cache = self.get(md5)
-        if self.state.changed(cache, md5=md5):
-            if os.path.exists(cache):
-                msg = "Corrupted cache file {}."
-                logger.warning(msg.format(os.path.relpath(cache)))
-                remove(cache)
-            return True
-        return False
+        return self.checksum_to_path(md5)
 
     def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info["scheme"] == "local"
-        return os.path.lexists(path_info["path"])
+        assert isinstance(path_info, BasePathInfo)
+        assert path_info.scheme == "local"
+        return os.path.lexists(path_info.path)
 
-    def changed_cache(self, md5):
-        cache = self.get(md5)
-        clist = [(cache, md5)]
+    def makedirs(self, path_info):
+        if not self.exists(path_info):
+            os.makedirs(path_info.path)
 
-        while True:
-            if len(clist) == 0:
-                break
+    @slow_link_guard
+    def link(self, cache_info, path_info):
+        cache = cache_info.path
+        path = path_info.path
 
-            cache, md5 = clist.pop()
-            if self.changed_cache_file(md5):
-                return True
-
-            if self.is_dir_cache(cache) and self._cache_metadata_changed():
-                for entry in self.load_dir_cache(md5):
-                    md5 = entry[self.PARAM_CHECKSUM]
-                    cache = self.get(md5)
-                    clist.append((cache, md5))
-
-        return False
-
-    def link(self, cache, path):
         assert os.path.isfile(cache)
 
         dname = os.path.dirname(path)
@@ -172,7 +149,7 @@ class RemoteLOCAL(RemoteBase):
                 self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
 
                 if self.protected:
-                    os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
+                    self.protect(path_info)
 
                 msg = "Created {}'{}': {} -> {}".format(
                     "protected " if self.protected else "",
@@ -198,190 +175,52 @@ class RemoteLOCAL(RemoteBase):
             return ntpath
         return posixpath
 
-    @classmethod
-    def to_ospath(cls, path):
-        if os.name == "nt":
-            return cls.ntpath(path)
-        return cls.unixpath(path)
-
-    @staticmethod
-    def unixpath(path):
-        assert not ntpath.isabs(path)
-        assert not posixpath.isabs(path)
-        return path.replace("\\", "/")
-
-    @staticmethod
-    def ntpath(path):
-        assert not ntpath.isabs(path)
-        assert not posixpath.isabs(path)
-        return path.replace("/", "\\")
-
-    def collect_dir_cache(self, dname):
-        dir_info = []
-
-        for root, dirs, files in os.walk(str(dname)):
-            bar = False
-
-            if len(files) > LARGE_DIR_SIZE:
-                msg = (
-                    "Computing md5 for a large directory {}. "
-                    "This is only done once."
-                )
-                logger.info(msg.format(os.path.relpath(root)))
-                bar = True
-                title = os.path.relpath(root)
-                processed = 0
-                total = len(files)
-                progress.update_target(title, 0, total)
-
-            for fname in files:
-                path = os.path.join(root, fname)
-                relpath = self.unixpath(os.path.relpath(path, dname))
-
-                if bar:
-                    progress.update_target(title, processed, total)
-                    processed += 1
-
-                md5 = self.state.update(path)
-                dir_info.append(
-                    {self.PARAM_RELPATH: relpath, self.PARAM_CHECKSUM: md5}
-                )
-
-            if bar:
-                progress.finish_target(title)
-
-        # NOTE: sorting the list by path to ensure reproducibility
-        dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
-
-        md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
-        if self.changed_cache_file(md5):
-            self.dump_dir_cache(md5, dir_info)
-
-        return (md5, dir_info)
-
-    def load_dir_cache(self, md5):
-        path = self.get(md5)
-        assert self.is_dir_cache(path)
-
-        dir_info = self._dir_info.get(md5)
-        if dir_info:
-            return dir_info
-
-        try:
-            with open(path, "r") as fd:
-                d = json.load(fd)
-        except Exception:
-            msg = "Failed to load dir cache '{}'"
-            logger.exception(msg.format(os.path.relpath(path)))
-            return []
-
-        if not isinstance(d, list):
-            msg = "dir cache file format error '{}' [skipping the file]"
-            logger.error(msg.format(os.path.relpath(path)))
-            return []
-
-        for info in d:
-            info["relpath"] = self.to_ospath(info["relpath"])
-
-        self._dir_info[md5] = d
-
-        return d
-
-    def dump_dir_cache(self, md5, dir_info):
-        path = self.get(md5)
-        dname = os.path.dirname(path)
-
-        assert self.is_dir_cache(path)
-        assert isinstance(dir_info, list)
-
-        if not os.path.isdir(dname):
-            os.makedirs(dname)
-
-        # NOTE: Writing first and renaming after that
-        # to make sure that the operation is atomic.
-        tmp = "{}.{}".format(path, str(uuid.uuid4()))
-
-        with open(tmp, "w+") as fd:
-            json.dump(dir_info, fd, sort_keys=True)
-        move(tmp, path)
-
-    @classmethod
-    def is_dir_cache(cls, cache):
-        return cache.endswith(cls.MD5_DIR_SUFFIX)
-
-    def do_checkout(
-        self, path_info, checksum, force=False, progress_callback=None
-    ):
-        path = path_info["path"]
-        cache = self.get(checksum)
-
-        if not self.is_dir_cache(cache):
-            if self.exists(path_info):
-                self.safe_remove(path_info, force=force)
-
-            self.link(cache, path)
-            self.state.update_link(path)
-            if progress_callback:
-                progress_callback.update(os.path.relpath(path))
-            return
-
-        # Create dir separately so that dir is created
-        # even if there are no files in it
-        if not os.path.exists(path):
-            os.makedirs(path)
-
-        dir_relpath = os.path.relpath(path)
-
-        logger.debug("Linking directory '{}'.".format(dir_relpath))
-
-        dir_info = self.load_dir_cache(checksum)
-
-        for processed, entry in enumerate(dir_info):
-            relpath = entry[self.PARAM_RELPATH]
-            m = entry[self.PARAM_CHECKSUM]
-            p = os.path.join(path, relpath)
-            c = self.get(m)
-
-            entry_info = {"scheme": path_info["scheme"], self.PARAM_PATH: p}
-
-            entry_checksum_info = {self.PARAM_CHECKSUM: m}
-
-            if self.changed(entry_info, entry_checksum_info):
-                if self.exists(entry_info):
-                    self.safe_remove(entry_info, force=force)
-
-                self.link(c, p)
-
-            if progress_callback:
-                progress_callback.update(os.path.relpath(p))
-
-        self._discard_working_directory_changes(path, dir_info, force=force)
-
-        self.state.update_link(path)
-
     def already_cached(self, path_info):
-        assert path_info["scheme"] in ["", "local"]
+        assert path_info.scheme in ["", "local"]
 
-        current_md5 = self.state.update(path_info["path"])
+        current_md5 = self.get_checksum(path_info)
 
         if not current_md5:
             return False
 
         return not self.changed_cache(current_md5)
 
-    def _discard_working_directory_changes(self, path, dir_info, force=False):
-        working_dir_files = set(path for path in walk_files(path))
+    def is_empty(self, path_info):
+        path = path_info.path
 
-        cached_files = set(
-            os.path.join(path, file["relpath"]) for file in dir_info
-        )
+        if self.isfile(path_info) and os.path.getsize(path) == 0:
+            return True
 
-        delta = working_dir_files - cached_files
+        if self.isdir(path_info) and len(os.listdir(path)) == 0:
+            return True
 
-        for file in delta:
-            self.safe_remove({"scheme": "local", "path": file}, force=force)
+        return False
+
+    def isfile(self, path_info):
+        return os.path.isfile(path_info.path)
+
+    def isdir(self, path_info):
+        return os.path.isdir(path_info.path)
+
+    def walk(self, path_info):
+        return os.walk(path_info.path)
+
+    def get_file_checksum(self, path_info):
+        return file_md5(path_info.path)[0]
+
+    def remove(self, path_info):
+        if path_info.scheme != "local":
+            raise NotImplementedError
+
+        remove(path_info.path)
+
+    def move(self, from_info, to_info):
+        if from_info.scheme != "local" or to_info.scheme != "local":
+            raise NotImplementedError
+
+        inp = from_info.path
+        outp = to_info.path
 
-    def _move(self, inp, outp):
         # moving in two stages to make the whole operation atomic in
         # case inp and outp are in different filesystems and actual
         # physical copying of data is happening
@@ -389,131 +228,42 @@ class RemoteLOCAL(RemoteBase):
         move(inp, tmp)
         move(tmp, outp)
 
-    def _save_file(self, path, md5):
-        assert md5 is not None
-
-        cache = self.get(md5)
-
-        if self.changed_cache(md5):
-            self._move(path, cache)
-        else:
-            remove(path)
-
-        self.link(cache, path)
-        self.state.update_link(path)
-
-        # we need to update path and cache, since in case of reflink,
-        # or copy cache type moving original file results in updates on
-        # next executed command, which causes md5 recalculation
-        self.state.update(path, md5)
-        self.state.update(cache, md5)
-
-        return {self.PARAM_CHECKSUM: md5}
-
-    def _save_dir(self, path, md5):
-        dir_info = self.load_dir_cache(md5)
-        dir_relpath = os.path.relpath(path)
-        dir_size = len(dir_info)
-        bar = dir_size > LARGE_DIR_SIZE
-
-        logger.info("Linking directory '{}'.".format(dir_relpath))
-
-        for processed, entry in enumerate(dir_info):
-            relpath = entry[self.PARAM_RELPATH]
-            m = entry[self.PARAM_CHECKSUM]
-            p = os.path.join(path, relpath)
-            c = self.get(m)
-
-            if self.changed_cache(m):
-                self._move(p, c)
-            else:
-                remove(p)
-
-            self.link(c, p)
-
-            self.state.update(p, m)
-            self.state.update(c, m)
-
-            if bar:
-                progress.update_target(dir_relpath, processed, dir_size)
-
-        self.state.update_link(path)
-
-        cache = self.get(md5)
-        self.state.update(cache)
-        self.state.update(path, md5)
-
-        if bar:
-            progress.finish_target(dir_relpath)
-
-    def save(self, path_info, checksum_info):
-        if path_info["scheme"] != "local":
-            raise NotImplementedError
-
-        path = path_info["path"]
-
-        msg = "Saving '{}' to cache '{}'."
-        logger.info(
-            msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir))
-        )
-
-        md5 = checksum_info[self.PARAM_CHECKSUM]
-        if os.path.isdir(path):
-            self._save_dir(path, md5)
-        else:
-            self._save_file(path, md5)
-
-    def save_info(self, path_info):
-        if path_info["scheme"] != "local":
-            raise NotImplementedError
-
-        return {self.PARAM_CHECKSUM: self.state.update(path_info["path"])}
-
-    def remove(self, path_info):
-        if path_info["scheme"] != "local":
-            raise NotImplementedError
-
-        remove(path_info["path"])
-
-    def move(self, from_info, to_info):
-        if from_info["scheme"] != "local" or to_info["scheme"] != "local":
-            raise NotImplementedError
-
-        move(from_info["path"], to_info["path"])
-
     def cache_exists(self, md5s):
         assert isinstance(md5s, list)
         return list(filter(lambda md5: not self.changed_cache_file(md5), md5s))
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
-            if from_info["scheme"] != "local":
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
             logger.debug(
-                "Uploading '{}' to '{}'".format(
-                    from_info["path"], to_info["path"]
-                )
+                "Uploading '{}' to '{}'".format(from_info.path, to_info.path)
             )
 
             if not name:
-                name = os.path.basename(from_info["path"])
+                name = os.path.basename(from_info.path)
 
-            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
-            tmp_file = tmp_fname(to_info["path"])
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+            tmp_file = tmp_fname(to_info.path)
 
             try:
-                copyfile(from_info["path"], tmp_file, name=name)
-                os.rename(tmp_file, to_info["path"])
+                copyfile(
+                    from_info.path,
+                    tmp_file,
+                    name=name,
+                    no_progress_bar=no_progress_bar,
+                )
+                os.rename(tmp_file, to_info.path)
             except Exception:
                 logger.exception(
                     "failed to upload '{}' to '{}'".format(
-                        from_info["path"], to_info["path"]
+                        from_info.path, to_info.path
                     )
                 )
 
@@ -528,36 +278,34 @@ class RemoteLOCAL(RemoteBase):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != "local":
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
             logger.debug(
-                "Downloading '{}' to '{}'".format(
-                    from_info["path"], to_info["path"]
-                )
+                "Downloading '{}' to '{}'".format(from_info.path, to_info.path)
             )
 
             if not name:
-                name = os.path.basename(to_info["path"])
+                name = os.path.basename(to_info.path)
 
-            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
-            tmp_file = tmp_fname(to_info["path"])
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+            tmp_file = tmp_fname(to_info.path)
             try:
                 copyfile(
-                    from_info["path"],
+                    from_info.path,
                     tmp_file,
                     no_progress_bar=no_progress_bar,
                     name=name,
                 )
 
-                move(tmp_file, to_info["path"])
+                move(tmp_file, to_info.path)
             except Exception:
                 logger.exception(
                     "failed to download '{}' to '{}'".format(
-                        from_info["path"], to_info["path"]
+                        from_info.path, to_info.path
                     )
                 )
 
@@ -585,7 +333,14 @@ class RemoteLOCAL(RemoteBase):
 
         return by_md5
 
-    def status(self, checksum_infos, remote, jobs=None, show_checksums=False):
+    def status(
+        self,
+        checksum_infos,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
         logger.info("Preparing to collect status from {}".format(remote.url))
         title = "Collecting information"
 
@@ -601,12 +356,21 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 30, 100)
 
-        remote_exists = list(remote.cache_exists(md5s))
+        local_exists = self.cache_exists(md5s)
+
+        progress.update_target(title, 40, 100)
+
+        # This is a performance optimization. We can safely assume that,
+        # if the resources that we want to fetch are already cached,
+        # there's no need to check the remote storage for the existance of
+        # those files.
+        if download and sorted(local_exists) == sorted(md5s):
+            remote_exists = local_exists
+        else:
+            remote_exists = list(remote.cache_exists(md5s))
 
         progress.update_target(title, 90, 100)
 
-        local_exists = self.cache_exists(md5s)
-
         progress.finish_target(title)
 
         self._fill_statuses(ret, local_exists, remote_exists)
@@ -616,8 +380,12 @@ class RemoteLOCAL(RemoteBase):
         return ret
 
     def _fill_statuses(self, checksum_info_dir, local_exists, remote_exists):
+        # Using sets because they are way faster for lookups
+        local = set(local_exists)
+        remote = set(remote_exists)
+
         for md5, info in checksum_info_dir.items():
-            status = STATUS_MAP[(md5 in local_exists, md5 in remote_exists)]
+            status = STATUS_MAP[(md5 in local, md5 in remote)]
             info["status"] = status
 
     def _get_chunks(self, download, remote, status_info, status, jobs):
@@ -685,7 +453,11 @@ class RemoteLOCAL(RemoteBase):
             jobs = remote.JOBS
 
         status_info = self.status(
-            checksum_infos, remote, jobs=jobs, show_checksums=show_checksums
+            checksum_infos,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
         )
 
         chunks = self._get_chunks(download, remote, status_info, status, jobs)
@@ -722,17 +494,18 @@ class RemoteLOCAL(RemoteBase):
             download=True,
         )
 
-    def _cache_metadata_changed(self):
+    def _changed_cache_dir(self):
         mtime, size = get_mtime_and_size(self.cache_dir)
         inode = get_inode(self.cache_dir)
 
         existing_record = self.state.get_state_record_for_inode(inode)
-
         if existing_record:
             cached_mtime, cached_size, _, _ = existing_record
-            return not (mtime == cached_mtime and size == cached_size)
+            changed = not (mtime == cached_mtime and size == cached_size)
+        else:
+            changed = True
 
-        return True
+        return changed
 
     def _log_missing_caches(self, checksum_info_dict):
         missing_caches = [
@@ -787,7 +560,7 @@ class RemoteLOCAL(RemoteBase):
 
     @staticmethod
     def unprotect(path_info):
-        path = path_info["path"]
+        path = path_info.path
         if not os.path.exists(path):
             raise DvcException(
                 "can't unprotect non-existing data '{}'".format(path)
@@ -797,3 +570,7 @@ class RemoteLOCAL(RemoteBase):
             RemoteLOCAL._unprotect_dir(path)
         else:
             RemoteLOCAL._unprotect_file(path)
+
+    @staticmethod
+    def protect(path_info):
+        os.chmod(path_info.path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)

dvc/remote/local/slow_link_detection.py

@@ -0,0 +1,63 @@
+import logging
+import time
+
+import colorama
+from dvc.config import Config
+
+logger = logging.getLogger(__name__)
+
+
+class SlowLinkDetectorDecorator(object):
+    LINKING_TIMEOUT_SECONDS = 10.0
+    was_displayed = False
+
+    @classmethod
+    def should_display(cls):
+        if not cls.was_displayed:
+            cls.was_displayed = True
+            return True
+        return False
+
+    def __init__(self, method):
+        self.method = method
+
+    def __call__(self, *args, **kwargs):
+        start = time.time()
+        result = self.method(*args, **kwargs)
+        execution_time_seconds = time.time() - start
+
+        if (
+            execution_time_seconds >= self.LINKING_TIMEOUT_SECONDS
+            and self.should_display()
+        ):
+            msg = (
+                "You can cut execution time considerably. Check:\n"
+                "{blue}https://dvc.org/doc/commands-reference/config#cache{"
+                "reset}"
+                "\nfor "
+                "more information.\nTo disable this message, run:\n'dvc "
+                "config "
+                "cache.slow_link_warning False'".format(
+                    blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
+                )
+            )
+            logger.warning(msg)
+
+        return result
+
+
+def slow_link_guard(method):
+    def call(remote_local, *args, **kwargs):
+        cache_config = remote_local.repo.config.config.get(
+            Config.SECTION_CACHE
+        )
+        should_warn = cache_config.get(
+            Config.SECTION_CACHE_SLOW_LINK_WARNING, True
+        ) and not cache_config.get(Config.SECTION_CACHE_TYPE, None)
+
+        if should_warn:
+            decorated = SlowLinkDetectorDecorator(method)
+            return decorated(remote_local, *args, **kwargs)
+        return method(remote_local, *args, **kwargs)
+
+    return call

dvc/remote/oss.py

@@ -0,0 +1,192 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import os
+import logging
+
+from dvc.path import Schemes
+from dvc.path.oss import OSSPathInfo
+
+try:
+    import oss2
+except ImportError:
+    oss2 = None
+
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
+from dvc.progress import progress
+from dvc.config import Config
+from dvc.remote.base import RemoteBase
+from dvc.remote.azure import Callback
+
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteOSS(RemoteBase):
+    """
+    oss2 document:
+    https://www.alibabacloud.com/help/doc-detail/32026.htm
+
+
+    Examples
+    ----------
+    $ dvc remote add myremote oss://my-bucket/path
+    Set key id, key secret and endpoint using modify command
+    $ dvc remote modify myremote oss_key_id my-key-id
+    $ dvc remote modify myremote oss_key_secret my-key-secret
+    $ dvc remote modify myremote oss_endpoint endpoint
+    or environment variables
+    $ export OSS_ACCESS_KEY_ID="my-key-id"
+    $ export OSS_ACCESS_KEY_SECRET="my-key-secret"
+    $ export OSS_ENDPOINT="endpoint"
+    """
+
+    scheme = Schemes.OSS
+    REGEX = r"^oss://(?P<path>.*)?$"
+    REQUIRES = {"oss2": oss2}
+    PARAM_CHECKSUM = "etag"
+    COPY_POLL_SECONDS = 5
+
+    def __init__(self, repo, config):
+        super(RemoteOSS, self).__init__(repo, config)
+
+        self.url = config.get(Config.SECTION_REMOTE_URL)
+        parsed = urlparse(self.url)
+        self.bucket = parsed.netloc
+        self.prefix = parsed.path.lstrip("/")
+
+        self.endpoint = config.get(Config.SECTION_OSS_ENDPOINT) or os.getenv(
+            "OSS_ENDPOINT"
+        )
+
+        self.key_id = (
+            config.get(Config.SECTION_OSS_ACCESS_KEY_ID)
+            or os.getenv("OSS_ACCESS_KEY_ID")
+            or "defaultId"
+        )
+
+        self.key_secret = (
+            config.get(Config.SECTION_OSS_ACCESS_KEY_SECRET)
+            or os.getenv("OSS_ACCESS_KEY_SECRET")
+            or "defaultSecret"
+        )
+
+        self._bucket = None
+        self.path_info = OSSPathInfo(bucket=self.bucket)
+
+    @property
+    def oss_service(self):
+        if self._bucket is None:
+            logger.debug("URL {}".format(self.url))
+            logger.debug("key id {}".format(self.key_id))
+            logger.debug("key secret {}".format(self.key_secret))
+            auth = oss2.Auth(self.key_id, self.key_secret)
+            logger.debug("bucket name {}".format(self.bucket))
+            self._bucket = oss2.Bucket(auth, self.endpoint, self.bucket)
+            try:  # verify that bucket exists
+                self._bucket.get_bucket_info()
+            except oss2.exceptions.NoSuchBucket:
+                self._bucket.create_bucket(
+                    oss2.BUCKET_ACL_PUBLIC_READ,
+                    oss2.models.BucketCreateConfig(
+                        oss2.BUCKET_STORAGE_CLASS_STANDARD
+                    ),
+                )
+        return self._bucket
+
+    def remove(self, path_info):
+        if path_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        logger.debug(
+            "Removing oss://{}/{}".format(path_info.bucket, path_info.path)
+        )
+
+        self.oss_service.delete_object(path_info.path)
+
+    def _list_paths(self, prefix):
+        for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
+            yield blob.key
+
+    def list_cache_paths(self):
+        return self._list_paths(self.prefix)
+
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        for from_info, to_info, name in zip(from_infos, to_infos, names):
+            if to_info.scheme != self.scheme:
+                raise NotImplementedError
+
+            if from_info.scheme != "local":
+                raise NotImplementedError
+
+            bucket = to_info.bucket
+            path = to_info.path
+
+            logger.debug(
+                "Uploading '{}' to 'oss://{}/{}'".format(
+                    from_info.path, bucket, path
+                )
+            )
+
+            if not name:
+                name = os.path.basename(from_info.path)
+
+            cb = None if no_progress_bar else Callback(name)
+
+            try:
+                self.oss_service.put_object_from_file(
+                    path, from_info.path, progress_callback=cb
+                )
+            except Exception:
+                msg = "failed to upload '{}'".format(from_info.path)
+                logger.warning(msg)
+            else:
+                progress.finish_target(name)
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        names=None,
+        no_progress_bar=False,
+        resume=False,
+    ):
+        names = self._verify_path_args(from_infos, to_infos, names)
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info.scheme != self.scheme:
+                raise NotImplementedError
+            if to_info.scheme != "local":
+                raise NotImplementedError
+
+            bucket = from_info.bucket
+            path = from_info.path
+
+            logger.debug(
+                "Downloading 'oss://{}/{}' to '{}'".format(
+                    bucket, path, to_info.path
+                )
+            )
+
+            tmp_file = tmp_fname(to_info.path)
+            if not name:
+                name = os.path.basename(to_info.path)
+
+            cb = None if no_progress_bar else Callback(name)
+
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+
+            try:
+                self.oss_service.get_object_to_file(
+                    path, tmp_file, progress_callback=cb
+                )
+            except Exception:
+                msg = "failed to download 'oss://{}/{}'".format(bucket, path)
+                logger.warning(msg)
+            else:
+                move(tmp_file, to_info.path)
+
+                if not no_progress_bar:
+                    progress.finish_target(name)

dvc/remote/s3.py

@@ -4,6 +4,9 @@ import os
 import threading
 import logging
 
+from dvc.path import Schemes
+from dvc.path.s3 import S3PathInfo
+
 try:
     import boto3
 except ImportError:
@@ -14,8 +17,7 @@ from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
-from dvc.exceptions import DvcException
-
+from dvc.exceptions import DvcException, ETagMismatchError
 
 logger = logging.getLogger(__name__)
 
@@ -34,7 +36,7 @@ class Callback(object):
 
 
 class RemoteS3(RemoteBase):
-    scheme = "s3"
+    scheme = Schemes.S3
     REGEX = r"^s3://(?P<path>.*)$"
     REQUIRES = {"boto3": boto3}
     PARAM_CHECKSUM = "etag"
@@ -70,7 +72,7 @@ class RemoteS3(RemoteBase):
         self.bucket = parsed.netloc
         self.prefix = parsed.path.lstrip("/")
 
-        self.path_info = {"scheme": self.scheme, "bucket": self.bucket}
+        self.path_info = S3PathInfo(bucket=self.bucket)
 
     @staticmethod
     def compat_config(config):
@@ -89,45 +91,119 @@ class RemoteS3(RemoteBase):
             "s3", endpoint_url=self.endpoint_url, use_ssl=self.use_ssl
         )
 
-    def get_etag(self, bucket, path):
-        try:
-            obj = self.s3.head_object(Bucket=bucket, Key=path)
-        except Exception:
-            raise DvcException(
-                "s3://{}/{} does not exist".format(bucket, path)
-            )
+    @classmethod
+    def get_etag(cls, s3, bucket, path):
+        obj = cls.get_head_object(s3, bucket, path)
 
         return obj["ETag"].strip('"')
 
-    def save_info(self, path_info):
-        if path_info["scheme"] != "s3":
-            raise NotImplementedError
+    def get_file_checksum(self, path_info):
+        return self.get_etag(self.s3, path_info.bucket, path_info.path)
 
-        return {
-            self.PARAM_CHECKSUM: self.get_etag(
-                path_info["bucket"], path_info["path"]
+    @staticmethod
+    def get_head_object(s3, bucket, path, *args, **kwargs):
+
+        try:
+            obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
+        except Exception as exc:
+            raise DvcException(
+                "s3://{}/{} does not exist".format(bucket, path), exc
             )
-        }
+        return obj
+
+    @classmethod
+    def _copy_multipart(cls, s3, from_info, to_info, size, n_parts):
+        mpu = s3.create_multipart_upload(
+            Bucket=to_info.bucket, Key=to_info.path
+        )
+        mpu_id = mpu["UploadId"]
+
+        parts = []
+        byte_position = 0
+        for i in range(1, n_parts + 1):
+            obj = cls.get_head_object(
+                s3, from_info.bucket, from_info.path, PartNumber=i
+            )
+            part_size = obj["ContentLength"]
+            lastbyte = byte_position + part_size - 1
+            if lastbyte > size:
+                lastbyte = size - 1
+
+            srange = "bytes={}-{}".format(byte_position, lastbyte)
+
+            part = s3.upload_part_copy(
+                Bucket=to_info.bucket,
+                Key=to_info.path,
+                PartNumber=i,
+                UploadId=mpu_id,
+                CopySourceRange=srange,
+                CopySource={"Bucket": from_info.bucket, "Key": from_info.path},
+            )
+            parts.append(
+                {"PartNumber": i, "ETag": part["CopyPartResult"]["ETag"]}
+            )
+            byte_position += part_size
+
+        assert n_parts == len(parts)
+
+        s3.complete_multipart_upload(
+            Bucket=to_info.bucket,
+            Key=to_info.path,
+            UploadId=mpu_id,
+            MultipartUpload={"Parts": parts},
+        )
+
+    @classmethod
+    def _copy(cls, s3, from_info, to_info):
+        # NOTE: object's etag depends on the way it was uploaded to s3 or the
+        # way it was copied within the s3. More specifically, it depends on
+        # the chunk size that was used to transfer it, which would affect
+        # whether an object would be uploaded as a single part or as a
+        # multipart.
+        #
+        # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
+        # then it was transfered as a single part, which means that the chunk
+        # size used to transfer it was greater or equal to the ContentLength
+        # of that object. So to preserve that tag over the next transfer, we
+        # could use any value >= ContentLength.
+        #
+        # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
+        # then it was transfered as a multipart, which means that the chunk
+        # size used to transfer it was less than ContentLength of that object.
+        # Unfortunately, in general, it doesn't mean that the chunk size was
+        # the same throughout the transfer, so it means that in order to
+        # preserve etag, we need to transfer each part separately, so the
+        # object is transfered in the same chunks as it was originally.
+
+        obj = cls.get_head_object(s3, from_info.bucket, from_info.path)
+        etag = obj["ETag"].strip('"')
+        size = obj["ContentLength"]
+
+        _, _, parts_suffix = etag.partition("-")
+        if parts_suffix:
+            n_parts = int(parts_suffix)
+            cls._copy_multipart(s3, from_info, to_info, size, n_parts)
+        else:
+            source = {"Bucket": from_info.bucket, "Key": from_info.path}
+            s3.copy(source, to_info.bucket, to_info.path)
+
+        cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
+        if etag != cached_etag:
+            raise ETagMismatchError(etag, cached_etag)
 
     def copy(self, from_info, to_info, s3=None):
         s3 = s3 if s3 else self.s3
-
-        source = {"Bucket": from_info["bucket"], "Key": from_info["path"]}
-        self.s3.copy(source, to_info["bucket"], to_info["path"])
+        self._copy(s3, from_info, to_info)
 
     def remove(self, path_info):
-        if path_info["scheme"] != "s3":
+        if path_info.scheme != "s3":
             raise NotImplementedError
 
         logger.debug(
-            "Removing s3://{}/{}".format(
-                path_info["bucket"], path_info["path"]
-            )
+            "Removing s3://{}/{}".format(path_info.bucket, path_info.path)
         )
 
-        self.s3.delete_object(
-            Bucket=path_info["bucket"], Key=path_info["path"]
-        )
+        self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
 
     def _list_paths(self, bucket, prefix):
         """ Read config for list object api, paginate through list objects."""
@@ -150,44 +226,41 @@ class RemoteS3(RemoteBase):
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info["scheme"] == "s3"
+        assert path_info.scheme == "s3"
 
-        paths = self._list_paths(path_info["bucket"], path_info["path"])
-        return any(path_info["path"] == path for path in paths)
+        paths = self._list_paths(path_info.bucket, path_info.path)
+        return any(path_info.path == path for path in paths)
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         s3 = self.s3
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info["scheme"] != "s3":
+            if to_info.scheme != "s3":
                 raise NotImplementedError
 
-            if from_info["scheme"] != "local":
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
             logger.debug(
                 "Uploading '{}' to '{}/{}'".format(
-                    from_info["path"], to_info["bucket"], to_info["path"]
+                    from_info.path, to_info.bucket, to_info.path
                 )
             )
 
             if not name:
-                name = os.path.basename(from_info["path"])
+                name = os.path.basename(from_info.path)
 
-            total = os.path.getsize(from_info["path"])
-            cb = Callback(name, total)
+            total = os.path.getsize(from_info.path)
+            cb = None if no_progress_bar else Callback(name, total)
 
             try:
                 s3.upload_file(
-                    from_info["path"],
-                    to_info["bucket"],
-                    to_info["path"],
-                    Callback=cb,
+                    from_info.path, to_info.bucket, to_info.path, Callback=cb
                 )
             except Exception:
-                msg = "failed to upload '{}'".format(from_info["path"])
+                msg = "failed to upload '{}'".format(from_info.path)
                 logger.exception(msg)
                 continue
 
@@ -206,50 +279,47 @@ class RemoteS3(RemoteBase):
         s3 = self.s3
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != "s3":
+            if from_info.scheme != "s3":
                 raise NotImplementedError
 
-            if to_info["scheme"] == "s3":
+            if to_info.scheme == "s3":
                 self.copy(from_info, to_info, s3=s3)
                 continue
 
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
             msg = "Downloading '{}/{}' to '{}'".format(
-                from_info["bucket"], from_info["path"], to_info["path"]
+                from_info.bucket, from_info.path, to_info.path
             )
             logger.debug(msg)
 
-            tmp_file = tmp_fname(to_info["path"])
+            tmp_file = tmp_fname(to_info.path)
             if not name:
-                name = os.path.basename(to_info["path"])
+                name = os.path.basename(to_info.path)
 
-            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
 
             try:
                 if no_progress_bar:
                     cb = None
                 else:
                     total = s3.head_object(
-                        Bucket=from_info["bucket"], Key=from_info["path"]
+                        Bucket=from_info.bucket, Key=from_info.path
                     )["ContentLength"]
                     cb = Callback(name, total)
 
                 s3.download_file(
-                    from_info["bucket"],
-                    from_info["path"],
-                    tmp_file,
-                    Callback=cb,
+                    from_info.bucket, from_info.path, tmp_file, Callback=cb
                 )
             except Exception:
                 msg = "failed to download '{}/{}'".format(
-                    from_info["bucket"], from_info["path"]
+                    from_info.bucket, from_info.path
                 )
                 logger.exception(msg)
                 continue
 
-            move(tmp_file, to_info["path"])
+            move(tmp_file, to_info.path)
 
             if not no_progress_bar:
                 progress.finish_target(name)

dvc/remote/ssh/__init__.py

@@ -1,8 +1,12 @@
 from __future__ import unicode_literals
 
+import os
 import getpass
 import logging
 
+from dvc.path import Schemes
+from dvc.path.ssh import SSHPathInfo
+
 try:
     import paramiko
 except ImportError:
@@ -19,7 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 class RemoteSSH(RemoteBase):
-    scheme = "ssh"
+    scheme = Schemes.SSH
 
     # NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and
     # SCP-like (ssh://[user@]host.xz:/absolute/path) urls.
@@ -38,32 +42,68 @@ class RemoteSSH(RemoteBase):
 
         parsed = urlparse(self.url)
         self.host = parsed.hostname
+
+        user_ssh_config = self._load_user_ssh_config(self.host)
+
+        self.host = user_ssh_config.get("hostname", self.host)
         self.user = (
             config.get(Config.SECTION_REMOTE_USER)
             or parsed.username
+            or user_ssh_config.get("user")
             or getpass.getuser()
         )
         self.prefix = parsed.path or "/"
         self.port = (
             config.get(Config.SECTION_REMOTE_PORT)
             or parsed.port
+            or self._try_get_ssh_config_port(user_ssh_config)
             or self.DEFAULT_PORT
         )
-        self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
+        self.keyfile = config.get(
+            Config.SECTION_REMOTE_KEY_FILE
+        ) or self._try_get_ssh_config_keyfile(user_ssh_config)
         self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
         self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
         self.ask_password = config.get(
             Config.SECTION_REMOTE_ASK_PASSWORD, False
         )
 
-        self.path_info = {
-            "scheme": "ssh",
-            "host": self.host,
-            "user": self.user,
-            "port": self.port,
-        }
+        self.path_info = SSHPathInfo(
+            host=self.host, user=self.user, port=self.port
+        )
+
+    @staticmethod
+    def ssh_config_filename():
+        return os.path.expanduser(os.path.join("~", ".ssh", "config"))
+
+    @staticmethod
+    def _load_user_ssh_config(hostname):
+        user_config_file = RemoteSSH.ssh_config_filename()
+        user_ssh_config = dict()
+        if hostname and os.path.exists(user_config_file):
+            with open(user_config_file) as f:
+                ssh_config = paramiko.SSHConfig()
+                ssh_config.parse(f)
+                user_ssh_config = ssh_config.lookup(hostname)
+        return user_ssh_config
+
+    @staticmethod
+    def _try_get_ssh_config_port(user_ssh_config):
+        try:
+            return int(user_ssh_config.get("port"))
+        except (ValueError, TypeError):
+            return None
+
+    @staticmethod
+    def _try_get_ssh_config_keyfile(user_ssh_config):
+        identity_file = user_ssh_config.get("identityfile")
+        if identity_file and len(identity_file) > 0:
+            return identity_file[0]
+        return None
+
+    def ssh(self, path_info):
+        host, user, port = path_info.host, path_info.user, path_info.port
 
-    def ssh(self, host=None, user=None, port=None, **kwargs):
         logger.debug(
             "Establishing ssh connection with '{host}' "
             "through port '{port}' as user '{user}'".format(
@@ -90,40 +130,42 @@ class RemoteSSH(RemoteBase):
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info["scheme"] == self.scheme
+        assert path_info.scheme == self.scheme
 
-        with self.ssh(**path_info) as ssh:
-            return ssh.file_exists(path_info["path"])
+        with self.ssh(path_info) as ssh:
+            return ssh.exists(path_info.path)
 
-    def md5(self, path_info):
-        if path_info["scheme"] != self.scheme:
+    def get_file_checksum(self, path_info):
+        if path_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(**path_info) as ssh:
-            return ssh.md5(path_info["path"])
+        with self.ssh(path_info) as ssh:
+            return ssh.md5(path_info.path)
+
+    def isdir(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.isdir(path_info.path)
 
     def copy(self, from_info, to_info):
-        if (
-            from_info["scheme"] != self.scheme
-            or to_info["scheme"] != self.scheme
-        ):
+        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(**from_info) as ssh:
-            ssh.cp(from_info["path"], to_info["path"])
-
-    def save_info(self, path_info):
-        if path_info["scheme"] != self.scheme:
-            raise NotImplementedError
-
-        return {self.PARAM_CHECKSUM: self.md5(path_info)}
+        with self.ssh(from_info) as ssh:
+            ssh.cp(from_info.path, to_info.path)
 
     def remove(self, path_info):
-        if path_info["scheme"] != self.scheme:
+        if path_info.scheme != self.scheme:
             raise NotImplementedError
 
-        with self.ssh(**path_info) as ssh:
-            ssh.remove(path_info["path"])
+        with self.ssh(path_info) as ssh:
+            ssh.remove(path_info.path)
+
+    def move(self, from_info, to_info):
+        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.move(from_info.path, to_info.path)
 
     def download(
         self,
@@ -134,68 +176,81 @@ class RemoteSSH(RemoteBase):
         resume=False,
     ):
         names = self._verify_path_args(from_infos, to_infos, names)
-        ssh = self.ssh(**from_infos[0])
+        ssh = self.ssh(from_infos[0])
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != self.scheme:
+            if from_info.scheme != self.scheme:
                 raise NotImplementedError
 
-            if to_info["scheme"] == self.scheme:
-                ssh.cp(from_info["path"], to_info["path"])
+            if to_info.scheme == self.scheme:
+                ssh.cp(from_info.path, to_info.path)
                 continue
 
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
             logger.debug(
                 "Downloading '{host}/{path}' to '{dest}'".format(
-                    host=from_info["host"],
-                    path=from_info["path"],
-                    dest=to_info["path"],
+                    host=from_info.host, path=from_info.path, dest=to_info.path
                 )
             )
 
             try:
                 ssh.download(
-                    from_info["path"], to_info["path"], progress_title=name
+                    from_info.path,
+                    to_info.path,
+                    progress_title=name,
+                    no_progress_bar=no_progress_bar,
                 )
             except Exception:
                 logger.exception(
                     "failed to download '{host}/{path}' to '{dest}'".format(
-                        host=from_info["host"],
-                        path=from_info["path"],
-                        dest=to_info["path"],
+                        host=from_info.host,
+                        path=from_info.path,
+                        dest=to_info.path,
                     )
                 )
                 continue
 
         ssh.close()
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
-        with self.ssh(**to_infos[0]) as ssh:
+        with self.ssh(to_infos[0]) as ssh:
             for from_info, to_info, name in zip(from_infos, to_infos, names):
-                if to_info["scheme"] != self.scheme:
+                if to_info.scheme != self.scheme:
                     raise NotImplementedError
 
-                if from_info["scheme"] != "local":
+                if from_info.scheme != "local":
                     raise NotImplementedError
 
                 try:
                     ssh.upload(
-                        from_info["path"], to_info["path"], progress_title=name
+                        from_info.path,
+                        to_info.path,
+                        progress_title=name,
+                        no_progress_bar=no_progress_bar,
                     )
                 except Exception:
                     logger.exception(
                         "failed to upload '{host}/{path}' to '{dest}'".format(
-                            host=from_info["host"],
-                            path=from_info["path"],
-                            dest=to_info["path"],
+                            host=from_info.host,
+                            path=from_info.path,
+                            dest=to_info.path,
                         )
                     )
                     pass
 
     def list_cache_paths(self):
-        with self.ssh(**self.path_info) as ssh:
+        with self.ssh(self.path_info) as ssh:
             return list(ssh.walk_files(self.prefix))
+
+    def walk(self, path_info):
+        with self.ssh(path_info) as ssh:
+            for entry in ssh.walk(path_info.path):
+                yield entry
+
+    def makedirs(self, path_info):
+        with self.ssh(path_info) as ssh:
+            ssh.makedirs(path_info.path)

dvc/remote/ssh/connection.py

@@ -2,6 +2,7 @@ import errno
 import os
 import posixpath
 import logging
+from stat import S_ISDIR
 
 try:
     import paramiko
@@ -72,7 +73,7 @@ class SSHConnection:
 
         self._ssh.close()
 
-    def file_exists(self, path):
+    def exists(self, path):
         self._sftp_connect()
         try:
             return self._sftp.stat(path)
@@ -118,7 +119,7 @@ class SSHConnection:
                 "a file with the same name '{}' already exists".format(path)
             )
 
-        head, tail = os.path.split(path)
+        head, tail = posixpath.split(path)
 
         if head:
             self.makedirs(head)
@@ -126,8 +127,11 @@ class SSHConnection:
         if tail:
             self._sftp.mkdir(path)
 
-    def walk_files(self, directory):
-        from stat import S_ISLNK, S_ISDIR, S_ISREG
+    def walk(self, directory, topdown=True):
+        # NOTE: original os.walk() implementation [1] with default options was
+        # used as a template.
+        #
+        # [1] https://github.com/python/cpython/blob/master/Lib/os.py
 
         self._sftp_connect()
 
@@ -141,28 +145,61 @@ class SSHConnection:
                 cause=exc,
             )
 
+        dirs = []
+        nondirs = []
         for entry in dir_entries:
-            path = posixpath.join(directory, entry.filename)
-
-            if S_ISLNK(entry.st_mode):
-                path = self._sftp.readlink(directory)
-                entry = self._sftp.stat(path)
-
+            name = entry.filename
             if S_ISDIR(entry.st_mode):
-                for inner_path in self.walk_files(path):
-                    yield inner_path
+                dirs.append(name)
+            else:
+                nondirs.append(name)
 
-            elif S_ISREG(entry.st_mode):
-                yield path
+        if topdown:
+            yield directory, dirs, nondirs
 
-    def remove(self, path):
-        self._sftp_connect()
+        for dname in dirs:
+            newpath = posixpath.join(directory, dname)
+            for entry in self.walk(newpath, topdown=topdown):
+                yield entry
+
+        if not topdown:
+            yield directory, dirs, nondirs
+
+    def walk_files(self, directory):
+        for root, dirs, files in self.walk(directory):
+            for fname in files:
+                yield posixpath.join(root, fname)
+
+    def _remove_file(self, path):
         try:
             self._sftp.remove(path)
         except IOError as exc:
             if exc.errno != errno.ENOENT:
                 raise
 
+    def _remove_dir(self, path):
+        for root, dirs, files in self.walk(path, topdown=False):
+            for fname in files:
+                path = posixpath.join(root, fname)
+                self._remove_file(path)
+
+            for dname in dirs:
+                path = posixpath.join(root, dname)
+                self._sftp.rmdir(dname)
+        try:
+            self._sftp.rmdir(path)
+        except IOError as exc:
+            if exc.errno != errno.ENOENT:
+                raise
+
+    def remove(self, path):
+        self._sftp_connect()
+
+        if self.isdir(path):
+            self._remove_dir(path)
+        else:
+            self._remove_file(path)
+
     def download(self, src, dest, no_progress_bar=False, progress_title=None):
         self._sftp_connect()
 
@@ -183,6 +220,11 @@ class SSHConnection:
 
         os.rename(tmp_file, dest)
 
+    def move(self, src, dst):
+        self.makedirs(posixpath.dirname(dst))
+        self._sftp_connect()
+        self._sftp.rename(src, dst)
+
     def upload(self, src, dest, no_progress_bar=False, progress_title=None):
         self._sftp_connect()
 

dvc/repo/__init__.py

@@ -11,7 +11,8 @@ from dvc.exceptions import (
     OutputNotFoundError,
     TargetNotDirectoryError,
 )
-
+from dvc.ignore import DvcIgnoreFileHandler
+from dvc.path.local import LocalPathInfo
 
 logger = logging.getLogger(__name__)
 
@@ -35,7 +36,6 @@ class Repo(object):
     from dvc.repo.status import status
     from dvc.repo.gc import gc
     from dvc.repo.commit import commit
-    from dvc.repo.pkg import install_pkg
     from dvc.repo.diff import diff
     from dvc.repo.brancher import brancher
 
@@ -50,6 +50,7 @@ class Repo(object):
         from dvc.repo.metrics import Metrics
         from dvc.scm.tree import WorkingTree
         from dvc.repo.tag import Tag
+        from dvc.repo.pkg import Pkg
 
         root_dir = self.find_root(root_dir)
 
@@ -58,7 +59,7 @@ class Repo(object):
 
         self.config = Config(self.dvc_dir)
 
-        self.tree = WorkingTree()
+        self.tree = WorkingTree(self.root_dir)
 
         self.scm = SCM(self.root_dir, repo=self)
         self.lock = Lock(self.dvc_dir)
@@ -68,11 +69,9 @@ class Repo(object):
 
         core = self.config.config[Config.SECTION_CORE]
 
-        logger.setLevel(
-            logging.getLevelName(
-                core.get(Config.SECTION_CORE_LOGLEVEL, "info").upper()
-            )
-        )
+        level = core.get(Config.SECTION_CORE_LOGLEVEL)
+        if level:
+            logger.setLevel(level.upper())
 
         self.cache = Cache(self)
         self.cloud = DataCloud(self, config=self.config.config)
@@ -80,6 +79,7 @@ class Repo(object):
 
         self.metrics = Metrics(self)
         self.tag = Tag(self)
+        self.pkg = Pkg(self)
 
         self._ignore()
 
@@ -117,7 +117,7 @@ class Repo(object):
         return Repo(root_dir)
 
     def unprotect(self, target):
-        path_info = {"schema": "local", "path": target}
+        path_info = LocalPathInfo(path=target)
         return self.cache.local.unprotect(path_info)
 
     def _ignore(self):
@@ -204,7 +204,7 @@ class Repo(object):
             else:
                 return ret
 
-        for i in self.cache.local.load_dir_cache(md5):
+        for i in out.dir_cache:
             i["branch"] = branch
             i[r.PARAM_PATH] = os.path.join(
                 info[r.PARAM_PATH], i[r.PARAM_RELPATH]
@@ -233,9 +233,7 @@ class Repo(object):
         if out.scheme != "local":
             return ret
 
-        md5 = info[out.remote.PARAM_CHECKSUM]
-        cache = self.cache.local.get(md5)
-        if not out.remote.is_dir_cache(cache):
+        if not out.is_dir_checksum:
             return ret
 
         return self._collect_dir_cache(
@@ -285,18 +283,61 @@ class Repo(object):
                     )
 
                 for out in stage.outs:
-                    scheme = out.path_info["scheme"]
-                    cache[scheme] += self._collect_used_cache(
-                        out,
-                        branch=branch,
-                        remote=remote,
-                        force=force,
-                        jobs=jobs,
+                    scheme = out.path_info.scheme
+                    cache[scheme].extend(
+                        self._collect_used_cache(
+                            out,
+                            branch=branch,
+                            remote=remote,
+                            force=force,
+                            jobs=jobs,
+                        )
                     )
 
         return cache
 
     def graph(self, stages=None, from_directory=None):
+        """Generate a graph by using the given stages on the given directory
+
+        The nodes of the graph are the stage's path relative to the root.
+
+        Edges are created when the output of one stage is used as a
+        dependency in other stage.
+
+        The direction of the edges goes from the stage to its dependency:
+
+        For example, running the following:
+
+            $ dvc run -o A "echo A > A"
+            $ dvc run -d A -o B "echo B > B"
+            $ dvc run -d B -o C "echo C > C"
+
+        Will create the following graph:
+
+               ancestors <--
+                           |
+                C.dvc -> B.dvc -> A.dvc
+                |          |
+                |          --> descendants
+                |
+                ------- pipeline ------>
+                           |
+                           v
+              (weakly connected components)
+
+        Args:
+            stages (list): used to build a graph, if None given, use the ones
+                on the `from_directory`.
+
+            from_directory (str): directory where to look at for stages, if
+                None is given, use the current working directory
+
+        Raises:
+            OutputDuplicationError: two outputs with the same path
+            StagePathAsOutputError: stage inside an output directory
+            OverlappingOutputPathsError: output inside output directory
+            CyclicGraphError: resulting graph has cycles
+        """
         import networkx as nx
         from dvc.exceptions import (
             OutputDuplicationError,
@@ -334,7 +375,6 @@ class Repo(object):
                 if path_dir.startswith(out.path + os.sep):
                     raise StagePathAsOutputError(stage.wdir, stage.relpath)
 
-        # collect the whole DAG
         for stage in stages:
             node = os.path.relpath(stage.path, self.root_dir)
 
@@ -390,7 +430,11 @@ class Repo(object):
 
         stages = []
         outs = []
-        for root, dirs, files in self.tree.walk(from_directory):
+
+        ignore_file_handler = DvcIgnoreFileHandler(self.tree)
+        for root, dirs, files in self.tree.walk(
+            from_directory, ignore_file_handler=ignore_file_handler
+        ):
             for fname in files:
                 path = os.path.join(root, fname)
                 if not Stage.is_valid_filename(path):

dvc/repo/add.py

@@ -1,11 +1,16 @@
 import os
+import logging
+import colorama
 
 from dvc.repo.scm_context import scm_context
 from dvc.stage import Stage
-from dvc.utils import walk_files
+from dvc.utils import walk_files, LARGE_DIR_SIZE
 from dvc.exceptions import RecursiveAddingWhileUsingFilename
 
 
+logger = logging.getLogger(__name__)
+
+
 @scm_context
 def add(repo, target, recursive=False, no_commit=False, fname=None):
     if recursive and fname:
@@ -13,6 +18,19 @@ def add(repo, target, recursive=False, no_commit=False, fname=None):
 
     targets = _find_all_targets(repo, target, recursive)
 
+    if os.path.isdir(target) and len(targets) > LARGE_DIR_SIZE:
+        logger.warning(
+            "You are adding a large directory '{target}' recursively,"
+            " consider tracking it as a whole instead.\n"
+            "{purple}HINT:{nc} Remove the generated stage files and then"
+            " run {cyan}dvc add {target}{nc}".format(
+                purple=colorama.Fore.MAGENTA,
+                cyan=colorama.Fore.CYAN,
+                nc=colorama.Style.RESET_ALL,
+                target=target,
+            )
+        )
+
     stages = _create_stages(repo, targets, fname, no_commit)
 
     repo.check_dag(repo.stages() + stages)
@@ -25,12 +43,12 @@ def add(repo, target, recursive=False, no_commit=False, fname=None):
 def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
-            file
-            for file in walk_files(target)
-            if not repo.is_dvc_internal(file)
-            if not Stage.is_stage_file(file)
-            if not repo.scm.belongs_to_scm(file)
-            if not repo.scm.is_tracked(file)
+            fname
+            for fname in walk_files(target)
+            if not repo.is_dvc_internal(fname)
+            if not Stage.is_stage_file(fname)
+            if not repo.scm.belongs_to_scm(fname)
+            if not repo.scm.is_tracked(fname)
         ]
     return [target]
 

dvc/repo/brancher.py

@@ -27,7 +27,7 @@ def brancher(  # noqa: E302
     if self.scm.is_dirty():
         from dvc.scm.tree import WorkingTree
 
-        self.tree = WorkingTree()
+        self.tree = WorkingTree(self.root_dir)
         yield "Working Tree"
 
     if all_branches:

dvc/repo/diff.py

@@ -133,9 +133,9 @@ def _check_local_cache(a_out, is_checked):
 
 def _is_dir(path, a_outs, b_outs):
     if a_outs.get(path):
-        return a_outs[path].is_dir_cache
+        return a_outs[path].is_dir_checksum
     else:
-        return b_outs[path].is_dir_cache
+        return b_outs[path].is_dir_checksum
 
 
 def _get_diff_outs(self, diff_dct):

dvc/repo/imp.py

@@ -2,11 +2,13 @@ from dvc.repo.scm_context import scm_context
 
 
 @scm_context
-def imp(self, url, out, resume=False):
+def imp(self, url, out, resume=False, fname=None):
     from dvc.stage import Stage
 
     with self.state:
-        stage = Stage.create(repo=self, cmd=None, deps=[url], outs=[out])
+        stage = Stage.create(
+            repo=self, cmd=None, deps=[url], outs=[out], fname=fname
+        )
 
     if stage is None:
         return None

dvc/repo/metrics/modify.py

@@ -11,7 +11,7 @@ def modify(repo, path, typ=None, xpath=None, delete=False):
 
     if out.scheme != "local":
         msg = "output '{}' scheme '{}' is not supported for metrics"
-        raise DvcException(msg.format(out.path, out.path_info["scheme"]))
+        raise DvcException(msg.format(out.path, out.path_info.scheme))
 
     if typ is not None:
         typ = typ.lower().strip()

dvc/repo/metrics/show.py

@@ -1,14 +1,19 @@
 from __future__ import unicode_literals
 
+import errno
 import os
 import csv
 import json
 import logging
-from jsonpath_rw import parse
+
+from jsonpath_ng.ext import parse
 
 from dvc.exceptions import OutputNotFoundError, BadMetricError, NoMetricsError
 from dvc.utils.compat import builtin_str, open, StringIO, csv_reader
 
+NO_METRICS_FILE_AT_REFERENCE_WARNING = (
+    "Metrics file '{}' does not exist at the reference '{}'."
+)
 
 logger = logging.getLogger(__name__)
 
@@ -147,7 +152,7 @@ def _read_metric(fd, typ=None, xpath=None, rel_path=None, branch=None):
         return None
 
 
-def _collect_metrics(self, path, recursive, typ, xpath, branch):
+def _collect_metrics(repo, path, recursive, typ, xpath, branch):
     """Gather all the metric outputs.
 
     Args:
@@ -165,11 +170,11 @@ def _collect_metrics(self, path, recursive, typ, xpath, branch):
             - typ:
             - xpath:
     """
-    outs = [out for stage in self.stages() for out in stage.outs]
+    outs = [out for stage in repo.stages() for out in stage.outs]
 
     if path:
         try:
-            outs = self.find_outs_by_path(path, outs=outs, recursive=recursive)
+            outs = repo.find_outs_by_path(path, outs=outs, recursive=recursive)
         except OutputNotFoundError:
             logger.debug(
                 "stage file not for found for '{}' in branch '{}'".format(
@@ -195,16 +200,7 @@ def _collect_metrics(self, path, recursive, typ, xpath, branch):
     return res
 
 
-def _read_metrics_filesystem(path, typ, xpath, rel_path, branch):
-    if not os.path.exists(path):
-        return None
-    with open(path, "r") as fd:
-        return _read_metric(
-            fd, typ=typ, xpath=xpath, rel_path=rel_path, branch=branch
-        )
-
-
-def _read_metrics(self, metrics, branch):
+def _read_metrics(repo, metrics, branch):
     """Read the content of each metric file and format it.
 
     Args:
@@ -226,18 +222,31 @@ def _read_metrics(self, metrics, branch):
         if not typ:
             typ = os.path.splitext(out.path.lower())[1].replace(".", "")
         if out.use_cache:
-            metric = _read_metrics_filesystem(
-                self.cache.local.get(out.checksum),
-                typ=typ,
-                xpath=xpath,
-                rel_path=out.rel_path,
-                branch=branch,
-            )
+            open_fun = open
+            path = repo.cache.local.get(out.checksum)
         else:
-            fd = self.tree.open(out.path)
-            metric = _read_metric(
-                fd, typ=typ, xpath=xpath, rel_path=out.rel_path, branch=branch
-            )
+            open_fun = repo.tree.open
+            path = out.path
+        try:
+
+            with open_fun(path) as fd:
+                metric = _read_metric(
+                    fd,
+                    typ=typ,
+                    xpath=xpath,
+                    rel_path=out.rel_path,
+                    branch=branch,
+                )
+        except IOError as e:
+            if e.errno == errno.ENOENT:
+                logger.warning(
+                    NO_METRICS_FILE_AT_REFERENCE_WARNING.format(
+                        out.rel_path, branch
+                    )
+                )
+                metric = None
+            else:
+                raise
 
         if not metric:
             continue
@@ -248,7 +257,7 @@ def _read_metrics(self, metrics, branch):
 
 
 def show(
-    self,
+    repo,
     path=None,
     typ=None,
     xpath=None,
@@ -258,9 +267,9 @@ def show(
 ):
     res = {}
 
-    for branch in self.brancher(all_branches=all_branches, all_tags=all_tags):
-        entries = _collect_metrics(self, path, recursive, typ, xpath, branch)
-        metrics = _read_metrics(self, entries, branch)
+    for branch in repo.brancher(all_branches=all_branches, all_tags=all_tags):
+        entries = _collect_metrics(repo, path, recursive, typ, xpath, branch)
+        metrics = _read_metrics(repo, entries, branch)
         if metrics:
             res[branch] = metrics
 

dvc/repo/move.py

@@ -5,7 +5,7 @@ from dvc.repo.scm_context import scm_context
 
 
 def _expand_target_path(from_path, to_path):
-    if os.path.isdir(to_path) and not os.path.isdir(from_path):
+    if os.path.isdir(to_path):
         return os.path.join(to_path, os.path.basename(from_path))
     return to_path
 
@@ -59,7 +59,7 @@ def move(self, from_path, to_path):
         )
 
     to_out = Output.loads_from(
-        stage, [os.path.basename(to_path)], out.cache, out.metric
+        stage, [os.path.basename(to_path)], out.use_cache, out.metric
     )[0]
 
     with self.state:

dvc/repo/pkg/__init__.py

@@ -0,0 +1,8 @@
+class Pkg(object):
+    def __init__(self, repo):
+        self.repo = repo
+
+    def install(self, *args, **kwargs):
+        from dvc.repo.pkg.install import install
+
+        return install(self.repo, *args, **kwargs)

dvc/repo/pkg/install.py

@@ -12,20 +12,6 @@ from dvc.scm.git.temp_git_repo import TempGitRepo
 logger = logging.getLogger(__name__)
 
 
-class PackageParams(object):
-    def __init__(self, address, target_dir, select=[], file=None):
-        self.address = address
-        self.target_dir = target_dir
-        self.select = select
-        self.file = file
-
-    @property
-    def all_addresses(self):
-        if self.address:
-            return [self.address]
-        return PackageManager.read_packages()
-
-
 class PackageManager(object):
     PACKAGE_FILE = "package.yaml"
 
@@ -49,7 +35,9 @@ class PackageManager(object):
 class Package(object):
     MODULES_DIR = "dvc_mod"
 
-    def install_or_update(self, parent_repo, pkg_param):
+    def install_or_update(
+        self, parent_repo, address, target_dir, select=[], fname=None
+    ):
         raise NotImplementedError(
             "A method of abstract Package class was called"
         )
@@ -61,7 +49,9 @@ class Package(object):
 class GitPackage(Package):
     DEF_DVC_FILE_PREFIX = "mod_"
 
-    def install_or_update(self, parent_repo, pkg_params):
+    def install_or_update(
+        self, parent_repo, address, target_dir, select=[], fname=None
+    ):
         from git.cmd import Git
 
         if not self.is_in_root():
@@ -74,24 +64,19 @@ class GitPackage(Package):
             os.makedirs(self.MODULES_DIR)
             parent_repo.scm.ignore(os.path.abspath(self.MODULES_DIR))
 
-        module_name = (
-            Git.polish_url(pkg_params.address).strip("/").split("/")[-1]
-        )
+        module_name = Git.polish_url(address).strip("/").split("/")[-1]
         if not module_name:
             raise DvcException(
                 "Package address error: unable to extract package name"
             )
 
         with TempGitRepo(
-            pkg_params.address, module_name, Package.MODULES_DIR
+            address, module_name, Package.MODULES_DIR
         ) as tmp_repo:
             outputs_to_copy = tmp_repo.outs
-            if pkg_params.select:
+            if select:
                 outputs_to_copy = list(
-                    filter(
-                        lambda out: out.dvc_path in pkg_params.select,
-                        outputs_to_copy,
-                    )
+                    filter(lambda out: out.dvc_path in select, outputs_to_copy)
                 )
 
             fetched_stage_files = set(
@@ -102,22 +87,17 @@ class GitPackage(Package):
             module_dir = self.create_module_dir(module_name)
             tmp_repo.persist_to(module_dir, parent_repo)
 
-            dvc_file = self.get_dvc_file_name(
-                pkg_params.file, pkg_params.target_dir, module_name
-            )
+            dvc_file = self.get_dvc_file_name(fname, target_dir, module_name)
             try:
                 self.persist_stage_and_scm_state(
-                    parent_repo,
-                    outputs_to_copy,
-                    pkg_params.target_dir,
-                    dvc_file,
+                    parent_repo, outputs_to_copy, target_dir, dvc_file
                 )
             except Exception as ex:
                 raise DvcException(
                     "Package '{}' was installed "
                     "but stage file '{}' "
                     "was not created properly: {}".format(
-                        pkg_params.address, dvc_file, ex
+                        address, dvc_file, ex
                     )
                 )
 
@@ -161,7 +141,7 @@ class GitPackage(Package):
         return dvc_file_path
 
 
-def install_pkg(self, pkg_params):
+def install(self, address, target_dir, select=[], fname=None):
     """
     Install package.
 
@@ -175,31 +155,19 @@ def install_pkg(self, pkg_params):
           Result: tag_classifier package in dvc_mod/ directory
     """
 
-    if not os.path.isdir(pkg_params.target_dir):
-        logger.error(
-            "Unable to install package: "
-            "target directory '{}' does not exist".format(
-                pkg_params.target_dir
-            )
+    if not os.path.isdir(target_dir):
+        raise DvcException(
+            "target directory '{}' does not exist".format(target_dir)
         )
-        return 1
 
     curr_dir = os.path.realpath(os.curdir)
-    if not os.path.realpath(pkg_params.target_dir).startswith(curr_dir):
-        logger.error(
-            "Unable to install package: the current dir should be"
-            " a subdirectory of the target dir {}".format(
-                pkg_params.target_dir
-            )
+    if not os.path.realpath(target_dir).startswith(curr_dir):
+        raise DvcException(
+            "the current directory should be a subdirectory of the target "
+            "dir '{}'".format(target_dir)
         )
-        return 1
 
-    for addr in pkg_params.all_addresses:
-        try:
-            mgr = PackageManager.get_package(addr)
-            mgr.install_or_update(self, pkg_params)
-        except Exception as ex:
-            logger.exception("Unable to install package: ".format(ex))
-            return 1
-
-    return 0
+    addresses = [address] if address else PackageManager.read_packages()
+    for addr in addresses:
+        mgr = PackageManager.get_package(addr)
+        mgr.install_or_update(self, address, target_dir, select, fname)

dvc/repo/reproduce.py

@@ -43,6 +43,7 @@ def reproduce(
     all_pipelines=False,
     ignore_build_cache=False,
     no_commit=False,
+    downstream=False,
 ):
     from dvc.stage import Stage
 
@@ -82,6 +83,7 @@ def reproduce(
                 interactive=interactive,
                 ignore_build_cache=ignore_build_cache,
                 no_commit=no_commit,
+                downstream=downstream,
             )
             ret.extend(stages)
 
@@ -97,6 +99,7 @@ def _reproduce(
     interactive=False,
     ignore_build_cache=False,
     no_commit=False,
+    downstream=False,
 ):
     import networkx as nx
     from dvc.stage import Stage
@@ -116,6 +119,7 @@ def _reproduce(
             interactive,
             ignore_build_cache,
             no_commit,
+            downstream,
         )
     else:
         ret = _reproduce_stage(
@@ -126,12 +130,67 @@ def _reproduce(
 
 
 def _reproduce_stages(
-    G, stages, node, force, dry, interactive, ignore_build_cache, no_commit
+    G,
+    stages,
+    node,
+    force,
+    dry,
+    interactive,
+    ignore_build_cache,
+    no_commit,
+    downstream,
 ):
+    r"""Derive the evaluation of the given node for the given graph.
+
+    When you _reproduce a stage_, you want to _evaluate the descendants_
+    to know if it make sense to _recompute_ it. A post-ordered search
+    will give us an order list of the nodes we want.
+
+    For example, let's say that we have the following pipeline:
+
+                               E
+                              / \
+                             D   F
+                            / \   \
+                           B   C   G
+                            \ /
+                             A
+
+    The derived evaluation of D would be: [A, B, C, D]
+
+    In case that `downstream` option is specifed, the desired effect
+    is to derive the evaluation starting from the given stage up to the
+    ancestors. However, the `networkx.ancestors` returns a set, without
+    any guarantee of any order, so we are going to reverse the graph and
+    use a pre-ordered search using the given stage as a starting point.
+
+                   E                                   A
+                  / \                                 / \
+                 D   F                               B   C   G
+                / \   \        --- reverse -->        \ /   /
+               B   C   G                               D   F
+                \ /                                     \ /
+                 A                                       E
+
+    The derived evaluation of _downstream_ B would be: [B, D, E]
+    """
+
     import networkx as nx
 
+    if downstream:
+        # NOTE (py3 only):
+        # Python's `deepcopy` defaults to pickle/unpickle the object.
+        # Stages are complex objects (with references to `repo`, `outs`,
+        # and `deps`) that cause struggles when you try to serialize them.
+        # We need to create a copy of the graph itself, and then reverse it,
+        # instead of using graph.reverse() directly because it calls
+        # `deepcopy` underneath -- unless copy=False is specified.
+        pipeline = nx.dfs_preorder_nodes(G.copy().reverse(copy=False), node)
+    else:
+        pipeline = nx.dfs_postorder_nodes(G, node)
+
     result = []
-    for n in nx.dfs_postorder_nodes(G, node):
+    for n in pipeline:
         try:
             ret = _reproduce_stage(
                 stages, n, force, dry, interactive, no_commit

dvc/repo/tag/add.py

@@ -1,5 +1,5 @@
 import logging
-
+from copy import copy
 
 logger = logging.getLogger(__name__)
 
@@ -12,7 +12,7 @@ def add(self, tag, target=None, with_deps=False, recursive=False):
             if not out.info:
                 logger.warning("missing checksum info for '{}'".format(out))
                 continue
-            out.tags[tag] = out.info.copy()
+            out.tags[tag] = copy(out.info)
             changed = True
         if changed:
             stage.dump()

dvc/scm/git/__init__.py

@@ -33,6 +33,9 @@ class Git(Base):
     GIT_DIR = ".git"
 
     def __init__(self, root_dir=os.curdir, repo=None):
+        """Git class constructor.
+        Requires `Repo` class from `git` module (from gitpython package).
+        """
         super(Git, self).__init__(root_dir, repo=repo)
 
         import git
@@ -204,17 +207,26 @@ class Git(Base):
         return [t.name for t in self.git.tags]
 
     def _install_hook(self, name, cmd):
+        command = "dvc {}".format(cmd)
+
         hook = os.path.join(self.root_dir, self.GIT_DIR, "hooks", name)
+
         if os.path.isfile(hook):
-            msg = "git hook '{}' already exists."
-            raise SCMError(msg.format(os.path.relpath(hook)))
-        with open(hook, "w+") as fobj:
-            fobj.write("#!/bin/sh\nexec dvc {}\n".format(cmd))
+            with open(hook, "r+") as fobj:
+                if command not in fobj.read():
+                    fobj.write("exec {command}\n".format(command=command))
+        else:
+            with open(hook, "w+") as fobj:
+                fobj.write(
+                    "#!/bin/sh\n" "exec {command}\n".format(command=command)
+                )
+
         os.chmod(hook, 0o777)
 
     def install(self):
         self._install_hook("post-checkout", "checkout")
         self._install_hook("pre-commit", "status")
+        self._install_hook("pre-push", "push")
 
     def cleanup_ignores(self):
         for path in self.ignored_paths:
@@ -247,22 +259,30 @@ class Git(Base):
         return GitTree(self.git, rev)
 
     def _get_diff_trees(self, a_ref, b_ref):
+        """Private method for getting the trees and commit hashes of 2 git
+        references.
+        Requires `gitdb` module (from gitpython package).
 
+        Args:
+            a_ref(str) - git reference
+            b_ref(str) - second git reference. If None, uses HEAD
+
+        Returns:
+            tuple - tuple with elements: (trees, commits)
+        """
         from gitdb.exc import BadObject, BadName
 
         trees = {DIFF_A_TREE: None, DIFF_B_TREE: None}
         commits = []
+        if b_ref is None:
+            b_ref = self.git.head.commit
         try:
-            if b_ref is not None:
-                a_commit = self.git.commit(a_ref)
-                b_commit = self.git.commit(b_ref)
-                commits.append(str(a_commit))
-                commits.append(str(b_commit))
-            else:
-                a_commit = self.git.commit(a_ref)
-                b_commit = self.git.head.commit
-                commits.append(str(a_commit))
-                commits.append(str(b_commit))
+            a_commit = self.git.git.rev_parse(a_ref, short=True)
+            b_commit = self.git.git.rev_parse(b_ref, short=True)
+            # See https://gitpython.readthedocs.io
+            # /en/2.1.11/reference.html#git.objects.base.Object.__str__
+            commits.append(a_commit)
+            commits.append(b_commit)
             trees[DIFF_A_TREE] = self.get_tree(commits[0])
             trees[DIFF_B_TREE] = self.get_tree(commits[1])
         except (BadName, BadObject) as e:
@@ -270,8 +290,8 @@ class Git(Base):
         return trees, commits
 
     def get_diff_trees(self, a_ref, b_ref=None):
-        """Method for getting two repo trees between two git tag commits
-        returns the dvc hash names of changed file/directory
+        """Method for getting two repo trees between two git tag commits.
+        Returns the dvc hash names of changed file/directory
 
         Args:
             a_ref(str) - git reference
@@ -281,10 +301,10 @@ class Git(Base):
             dict - dictionary with keys: (a_tree, b_tree, a_ref, b_ref, equal)
         """
         diff_dct = {DIFF_EQUAL: False}
-        trees, commit_refs = self._get_diff_trees(a_ref, b_ref)
-        diff_dct[DIFF_A_REF] = commit_refs[0]
-        diff_dct[DIFF_B_REF] = commit_refs[1]
-        if commit_refs[0] == commit_refs[1]:
+        trees, commits = self._get_diff_trees(a_ref, b_ref)
+        diff_dct[DIFF_A_REF] = commits[0]
+        diff_dct[DIFF_B_REF] = commits[1]
+        if commits[0] == commits[1]:
             diff_dct[DIFF_EQUAL] = True
             return diff_dct
         diff_dct[DIFF_A_TREE] = trees[DIFF_A_TREE]

dvc/scm/git/tree.py

@@ -1,7 +1,8 @@
 import errno
 import os
 
-from dvc.utils.compat import StringIO
+from dvc.ignore import DvcIgnoreFilter
+from dvc.utils.compat import StringIO, BytesIO
 
 from dvc.scm.tree import BaseTree
 
@@ -24,7 +25,11 @@ class GitTree(BaseTree):
         self.git = git
         self.rev = rev
 
-    def open(self, path):
+    @property
+    def tree_root(self):
+        return self.git.working_dir
+
+    def open(self, path, binary=False):
 
         relpath = os.path.relpath(path, self.git.working_dir)
 
@@ -39,7 +44,10 @@ class GitTree(BaseTree):
         # read it immediately, also it needs to be to decoded if we follow
         # the `open()` behavior (since data_stream.read() returns bytes,
         # and `open` with default "r" mode returns str)
-        return StringIO(obj.data_stream.read().decode("utf-8"))
+        data = obj.data_stream.read()
+        if binary:
+            return BytesIO(data)
+        return StringIO(data.decode("utf-8"))
 
     def exists(self, path):
         return self.git_object_by_path(path) is not None
@@ -81,8 +89,13 @@ class GitTree(BaseTree):
             tree = tree[i]
         return tree
 
-    def _walk(self, tree, topdown=True):
-
+    def _walk(
+        self,
+        tree,
+        topdown=True,
+        ignore_file_handler=None,
+        dvc_ignore_filter=None,
+    ):
         dirs, nondirs = [], []
         for i in tree:
             if i.mode == GIT_MODE_DIR:
@@ -91,14 +104,26 @@ class GitTree(BaseTree):
                 nondirs.append(i.name)
 
         if topdown:
-            yield os.path.normpath(tree.path), dirs, nondirs
-        for i in dirs:
-            for x in self._walk(tree[i], topdown=True):
-                yield x
-        if not topdown:
-            yield os.path.normpath(tree.path), dirs, nondirs
+            if not dvc_ignore_filter:
+                dvc_ignore_filter = DvcIgnoreFilter(
+                    tree.abspath, ignore_file_handler=ignore_file_handler
+                )
+            dirs, nondirs = dvc_ignore_filter(tree.path, dirs, nondirs)
+            yield os.path.normpath(tree.abspath), dirs, nondirs
 
-    def walk(self, top, topdown=True):
+        for i in dirs:
+            for x in self._walk(
+                tree[i],
+                topdown=True,
+                ignore_file_handler=ignore_file_handler,
+                dvc_ignore_filter=dvc_ignore_filter,
+            ):
+                yield x
+
+        if not topdown:
+            yield os.path.normpath(tree.abspath), dirs, nondirs
+
+    def walk(self, top, topdown=True, ignore_file_handler=None):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:

dvc/scm/tree.py

@@ -1,12 +1,17 @@
 import os
 
+from dvc.utils import dvc_walk
 from dvc.utils.compat import open
 
 
 class BaseTree(object):
     """Abstract class to represent access to files"""
 
-    def open(self, path):
+    @property
+    def tree_root(self):
+        pass
+
+    def open(self, path, binary=False):
         """Open file and return a stream."""
 
     def exists(self, path):
@@ -18,7 +23,7 @@ class BaseTree(object):
     def isfile(self, path):
         """Test whether a path is a regular file"""
 
-    def walk(self, top, topdown=True):
+    def walk(self, top, topdown=True, ignore_file_handler=None):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
@@ -30,8 +35,17 @@ class BaseTree(object):
 class WorkingTree(BaseTree):
     """Proxies the repo file access methods to working tree files"""
 
-    def open(self, path):
+    def __init__(self, repo_root=os.getcwd()):
+        self.repo_root = repo_root
+
+    @property
+    def tree_root(self):
+        return self.repo_root
+
+    def open(self, path, binary=False):
         """Open file and return a stream."""
+        if binary:
+            return open(path, "rb")
         return open(path, encoding="utf-8")
 
     def exists(self, path):
@@ -46,7 +60,7 @@ class WorkingTree(BaseTree):
         """Test whether a path is a regular file"""
         return os.path.isfile(path)
 
-    def walk(self, top, topdown=True):
+    def walk(self, top, topdown=True, ignore_file_handler=None):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
@@ -57,9 +71,10 @@ class WorkingTree(BaseTree):
         def onerror(e):
             raise e
 
-        for root, dirs, files in os.walk(
-            top, topdown=topdown, onerror=onerror
+        for root, dirs, files in dvc_walk(
+            os.path.abspath(top),
+            topdown=topdown,
+            onerror=onerror,
+            ignore_file_handler=ignore_file_handler,
         ):
-            if topdown:
-                dirs[:] = [i for i in dirs if i not in (".git", ".hg", ".dvc")]
             yield os.path.normpath(root), dirs, files

dvc/stage.py

@@ -1,10 +1,10 @@
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, open
+from dvc.utils.compat import str
 
+import copy
 import re
 import os
-import yaml
 import subprocess
 import logging
 
@@ -15,7 +15,9 @@ import dvc.prompt as prompt
 import dvc.dependency as dependency
 import dvc.output as output
 from dvc.exceptions import DvcException
-from dvc.utils import dict_md5, fix_env, load_stage_file_fobj
+from dvc.utils import dict_md5, fix_env
+from dvc.utils.collections import apply_diff
+from dvc.utils.stage import load_stage_fd, dump_stage_file
 
 
 logger = logging.getLogger(__name__)
@@ -123,6 +125,7 @@ class Stage(object):
     PARAM_DEPS = "deps"
     PARAM_OUTS = "outs"
     PARAM_LOCKED = "locked"
+    PARAM_META = "meta"
 
     SCHEMA = {
         Optional(PARAM_MD5): Or(str, None),
@@ -131,6 +134,7 @@ class Stage(object):
         Optional(PARAM_DEPS): Or(And(list, Schema([dependency.SCHEMA])), None),
         Optional(PARAM_OUTS): Or(And(list, Schema([output.SCHEMA])), None),
         Optional(PARAM_LOCKED): bool,
+        Optional(PARAM_META): object,
     }
 
     TAG_REGEX = r"^(?P<path>.*)@(?P<tag>[^\\/@:]*)$"
@@ -146,6 +150,7 @@ class Stage(object):
         md5=None,
         locked=False,
         tag=None,
+        state=None,
     ):
         if deps is None:
             deps = []
@@ -161,6 +166,7 @@ class Stage(object):
         self.md5 = md5
         self.locked = locked
         self.tag = tag
+        self._state = state or {}
 
     def __repr__(self):
         return "Stage: '{path}'".format(
@@ -388,7 +394,21 @@ class Stage(object):
             out.pop(RemoteLOCAL.PARAM_CHECKSUM, None)
             out.pop(RemoteS3.PARAM_CHECKSUM, None)
 
-        return old_d == new_d
+        if old_d != new_d:
+            return False
+
+        # NOTE: committing to prevent potential data duplication. For example
+        #
+        #    $ dvc config cache.type hardlink
+        #    $ echo foo > foo
+        #    $ dvc add foo
+        #    $ rm -f foo
+        #    $ echo foo > foo
+        #    $ dvc add foo # should replace foo with a link to cache
+        #
+        old.commit()
+
+        return True
 
     @staticmethod
     def create(
@@ -482,6 +502,10 @@ class Stage(object):
         else:
             stage.unprotect_outs()
 
+        if os.path.exists(path) and any(out.persist for out in stage.outs):
+            logger.warning("Build cache is ignored when persisting outputs.")
+            ignore_build_cache = True
+
         if validate_state:
             if os.path.exists(path):
                 if not ignore_build_cache and stage.is_cached:
@@ -565,7 +589,11 @@ class Stage(object):
         Stage._check_dvc_filename(fname)
         Stage._check_isfile(repo, fname)
 
-        d = load_stage_file_fobj(repo.tree.open(fname), fname)
+        with repo.tree.open(fname) as fd:
+            d = load_stage_fd(fd, fname)
+        # Making a deepcopy since the original structure
+        # looses keys in deps and outs load
+        state = copy.deepcopy(d)
 
         Stage.validate(d, fname=os.path.relpath(fname))
         path = os.path.abspath(fname)
@@ -582,6 +610,7 @@ class Stage(object):
             md5=d.get(Stage.PARAM_MD5),
             locked=d.get(Stage.PARAM_LOCKED, False),
             tag=tag,
+            state=state,
         )
 
         stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
@@ -590,19 +619,20 @@ class Stage(object):
         return stage
 
     def dumpd(self):
-        from dvc.remote.local import RemoteLOCAL
+        from dvc.remote.base import RemoteBase
 
         return {
             key: value
             for key, value in {
                 Stage.PARAM_MD5: self.md5,
                 Stage.PARAM_CMD: self.cmd,
-                Stage.PARAM_WDIR: RemoteLOCAL.unixpath(
+                Stage.PARAM_WDIR: RemoteBase.to_posixpath(
                     os.path.relpath(self.wdir, os.path.dirname(self.path))
                 ),
                 Stage.PARAM_LOCKED: self.locked,
                 Stage.PARAM_DEPS: [d.dumpd() for d in self.deps],
                 Stage.PARAM_OUTS: [o.dumpd() for o in self.outs],
+                Stage.PARAM_META: self._state.get("meta"),
             }.items()
             if value
         }
@@ -618,9 +648,8 @@ class Stage(object):
             )
         )
         d = self.dumpd()
-
-        with open(fname, "w") as fd:
-            yaml.safe_dump(d, fd, default_flow_style=False)
+        apply_diff(d, self._state)
+        dump_stage_file(fname, self._state)
 
         self.repo.scm.track_file(os.path.relpath(fname))
 

dvc/state.py

@@ -7,7 +7,7 @@ import sqlite3
 import logging
 
 from dvc.config import Config
-from dvc.utils import file_md5, remove, current_timestamp
+from dvc.utils import remove, current_timestamp
 from dvc.exceptions import DvcException
 from dvc.utils.fs import get_mtime_and_size, get_inode
 
@@ -29,11 +29,18 @@ class StateVersionTooNewError(DvcException):
         )
 
 
-def _file_metadata_changed(actual_mtime, mtime, actual_size, size):
-    return actual_mtime != mtime or actual_size != size
+class StateBase(object):
+    def save(self, path_info, checksum):
+        pass
+
+    def get(self, path_info):
+        return None
+
+    def save_link(self, path_info):
+        pass
 
 
-class State(object):  # pylint: disable=too-many-instance-attributes
+class State(StateBase):  # pylint: disable=too-many-instance-attributes
     """Class for the state database.
 
     Args:
@@ -78,9 +85,6 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         self.dvc_dir = repo.dvc_dir
         self.root_dir = repo.root_dir
 
-        self.row_limit = 100
-        self.row_cleanup_quota = 50
-
         state_config = config.get(Config.SECTION_STATE, {})
         self.row_limit = state_config.get(
             Config.SECTION_STATE_ROW_LIMIT, self.STATE_ROW_LIMIT
@@ -112,11 +116,6 @@ class State(object):  # pylint: disable=too-many-instance-attributes
     def __exit__(self, typ, value, tbck):
         self.dump()
 
-    def _collect(self, path):
-        if os.path.isdir(path):
-            return self.repo.cache.local.collect_dir_cache(path)
-        return (file_md5(path)[0], None)
-
     def changed(self, path, md5):
         """Check if file/directory has the expected md5.
 
@@ -299,53 +298,9 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         self.cursor = None
         self.inserts = 0
 
-    def _do_update(self, path, known_checksum=None):
-        """
-        Make sure the stored info for the given path is up to date.
-        """
-        if not os.path.exists(path):
-            return None, None
-
-        actual_mtime, actual_size = get_mtime_and_size(path)
-        actual_inode = get_inode(path)
-
-        existing_record = self.get_state_record_for_inode(actual_inode)
-
-        if existing_record:
-            md5, info = self._update_existing_state_record(
-                path,
-                actual_inode,
-                actual_mtime,
-                actual_size,
-                existing_record,
-                known_checksum,
-            )
-        else:
-            md5, info = self._insert_new_state_record(
-                path, actual_inode, actual_mtime, actual_size, known_checksum
-            )
-
-        return md5, info
-
-    def _update_existing_state_record(
-        self,
-        path,
-        actual_inode,
-        actual_mtime,
-        actual_size,
-        existing_record,
-        known_checksum=None,
-    ):
-
-        mtime, size, md5, _ = existing_record
-        if _file_metadata_changed(actual_mtime, mtime, actual_size, size):
-            md5, info = self._update_state_for_path_changed(
-                path, actual_inode, actual_mtime, actual_size, known_checksum
-            )
-        else:
-            info = None
-            self._update_state_record_timestamp_for_inode(actual_inode)
-        return md5, info
+    @staticmethod
+    def _file_metadata_changed(actual_mtime, mtime, actual_size, size):
+        return actual_mtime != mtime or actual_size != size
 
     def _update_state_record_timestamp_for_inode(self, actual_inode):
         cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
@@ -358,17 +313,8 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         )
 
     def _update_state_for_path_changed(
-        self,
-        path,
-        actual_inode,
-        actual_mtime,
-        actual_size,
-        known_checksum=None,
+        self, path, actual_inode, actual_mtime, actual_size, checksum
     ):
-        if known_checksum:
-            md5, info = known_checksum, None
-        else:
-            md5, info = self._collect(path)
         cmd = (
             "UPDATE {} SET "
             'mtime = "{}", size = "{}", '
@@ -380,20 +326,17 @@ class State(object):  # pylint: disable=too-many-instance-attributes
                 self.STATE_TABLE,
                 actual_mtime,
                 actual_size,
-                md5,
+                checksum,
                 current_timestamp(),
                 self._to_sqlite(actual_inode),
             )
         )
-        return md5, info
 
     def _insert_new_state_record(
-        self, path, actual_inode, actual_mtime, actual_size, known_checksum
+        self, path, actual_inode, actual_mtime, actual_size, checksum
     ):
-        if known_checksum:
-            md5, info = known_checksum, None
-        else:
-            md5, info = self._collect(path)
+        assert checksum is not None
+
         cmd = (
             "INSERT INTO {}(inode, mtime, size, md5, timestamp) "
             'VALUES ({}, "{}", "{}", "{}", "{}")'
@@ -404,12 +347,11 @@ class State(object):  # pylint: disable=too-many-instance-attributes
                 self._to_sqlite(actual_inode),
                 actual_mtime,
                 actual_size,
-                md5,
+                checksum,
                 current_timestamp(),
             )
         )
         self.inserts += 1
-        return md5, info
 
     def get_state_record_for_inode(self, inode):
         cmd = "SELECT mtime, size, md5, timestamp from {} " "WHERE inode={}"
@@ -422,43 +364,74 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             return results[0]
         return None
 
-    def update(self, path, known_checksum=None):
-        """Gets the checksum for the specified path. Checksum will be
-        retrieved from the state database if available, otherwise it will be
-        computed and cached in the state database for the further use.
+    def save(self, path_info, checksum):
+        """Save checksum for the specified path info.
 
         Args:
-            path (str): path to get the checksum for.
-
-        Returns:
-            str: checksum for the specified path.
+            path_info (dict): path_info to save checksum for.
+            checksum (str): checksum to save.
         """
-        return self._do_update(path, known_checksum)[0]
+        assert path_info.scheme == "local"
+        assert checksum is not None
 
-    def update_info(self, path):
-        """Gets the checksum and the directory info (if applicable) for the
-        specified path.
+        path = path_info.path
+        assert os.path.exists(path)
+
+        actual_mtime, actual_size = get_mtime_and_size(path)
+        actual_inode = get_inode(path)
+
+        existing_record = self.get_state_record_for_inode(actual_inode)
+        if not existing_record:
+            self._insert_new_state_record(
+                path, actual_inode, actual_mtime, actual_size, checksum
+            )
+            return
+
+        self._update_state_for_path_changed(
+            path, actual_inode, actual_mtime, actual_size, checksum
+        )
+
+    def get(self, path_info):
+        """Gets the checksum for the specified path info. Checksum will be
+        retrieved from the state database if available.
 
         Args:
-            path (str): path to get the checksum and the directory info for.
+            path_info (dict): path info to get the checksum for.
 
         Returns:
-            tuple: checksum for the specified path along with a directory info
-            (list of {relative_path: checksum} entries for each file in the
-            directory) if applicable, otherwise None.
+            str or None: checksum for the specified path info or None if it
+            doesn't exist in the state database.
         """
-        md5, info = self._do_update(path)
-        if not info:
-            info = self.repo.cache.local.load_dir_cache(md5)
-        return (md5, info)
+        assert path_info.scheme == "local"
+        path = path_info.path
 
-    def update_link(self, path):
+        if not os.path.exists(path):
+            return None
+
+        actual_mtime, actual_size = get_mtime_and_size(path)
+        actual_inode = get_inode(path)
+
+        existing_record = self.get_state_record_for_inode(actual_inode)
+        if not existing_record:
+            return None
+
+        mtime, size, checksum, _ = existing_record
+        if self._file_metadata_changed(actual_mtime, mtime, actual_size, size):
+            return None
+
+        self._update_state_record_timestamp_for_inode(actual_inode)
+        return checksum
+
+    def save_link(self, path_info):
         """Adds the specified path to the list of links created by dvc. This
         list is later used on `dvc checkout` to cleanup old links.
 
         Args:
-            path (str): path to add to the list of links.
+            path_info (dict): path info to add to the list of links.
         """
+        assert path_info.scheme == "local"
+        path = path_info.path
+
         if not os.path.exists(path):
             return
 

dvc/system.py

@@ -102,22 +102,13 @@ class System(object):
 
         FICLONE = 0x40049409
 
-        s = open(src, "r")
-        d = open(dst, "w+")
-
         try:
-            ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
-        except IOError:
-            s.close()
-            d.close()
-            os.unlink(dst)
-            raise
-
-        s.close()
-        d.close()
-
-        if ret != 0:
-            os.unlink(dst)
+            ret = 255
+            with open(src, "r") as s, open(dst, "w+") as d:
+                ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
+        finally:
+            if ret != 0:
+                os.unlink(dst)
 
         return ret
 

dvc/utils/__init__.py

@@ -2,8 +2,7 @@
 
 from __future__ import unicode_literals
 
-import yaml
-from dvc.utils.compat import str, builtin_str, open, cast_bytes_py2
+from dvc.utils.compat import str, builtin_str, open, cast_bytes_py2, StringIO
 
 import os
 import sys
@@ -19,7 +18,7 @@ import colorama
 import re
 import logging
 
-from yaml.scanner import ScannerError
+from ruamel.yaml import YAML
 
 
 logger = logging.getLogger(__name__)
@@ -120,28 +119,22 @@ def copyfile(src, dest, no_progress_bar=False, name=None):
     name = name if name else os.path.basename(dest)
     total = os.stat(src).st_size
 
-    fsrc = open(src, "rb")
-
     if os.path.isdir(dest):
-        fdest = open(os.path.join(dest, os.path.basename(src)), "wb+")
-    else:
-        fdest = open(dest, "wb+")
+        dest = os.path.join(dest, os.path.basename(src))
 
-    while True:
-        buf = fsrc.read(LOCAL_CHUNK_SIZE)
-        if not buf:
-            break
-        fdest.write(buf)
-        copied += len(buf)
-        if not no_progress_bar:
-            progress.update_target(name, copied, total)
+    with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
+        while True:
+            buf = fsrc.read(LOCAL_CHUNK_SIZE)
+            if not buf:
+                break
+            fdest.write(buf)
+            copied += len(buf)
+            if not no_progress_bar:
+                progress.update_target(name, copied, total)
 
     if not no_progress_bar:
         progress.finish_target(name)
 
-    fsrc.close()
-    fdest.close()
-
 
 def move(src, dst):
     dst = os.path.abspath(dst)
@@ -243,22 +236,51 @@ def current_timestamp():
     return int(nanotime.timestamp(time.time()))
 
 
-def load_stage_file(path):
-    with open(path, "r") as fobj:
-        return load_stage_file_fobj(fobj, path)
+def from_yaml_string(s):
+    return YAML().load(StringIO(s))
 
 
-def load_stage_file_fobj(fobj, path):
-    from dvc.exceptions import StageFileCorruptedError
-
-    try:
-        return yaml.safe_load(fobj) or {}
-    except ScannerError:
-        raise StageFileCorruptedError(path)
+def to_yaml_string(data):
+    stream = StringIO()
+    yaml = YAML()
+    yaml.default_flow_style = False
+    yaml.dump(data, stream)
+    return stream.getvalue()
 
 
-def walk_files(directory):
-    for root, _, files in os.walk(str(directory)):
+def dvc_walk(
+    top,
+    topdown=True,
+    onerror=None,
+    followlinks=False,
+    ignore_file_handler=None,
+):
+    """
+    Proxy for `os.walk` directory tree generator.
+    Utilizes DvcIgnoreFilter functionality.
+    """
+    ignore_filter = None
+    if topdown:
+        from dvc.ignore import DvcIgnoreFilter
+
+        ignore_filter = DvcIgnoreFilter(
+            top, ignore_file_handler=ignore_file_handler
+        )
+
+    for root, dirs, files in os.walk(
+        top, topdown=topdown, onerror=onerror, followlinks=followlinks
+    ):
+
+        if ignore_filter:
+            dirs[:], files[:] = ignore_filter(root, dirs, files)
+
+        yield root, dirs, files
+
+
+def walk_files(directory, ignore_file_handler=None):
+    for root, _, files in dvc_walk(
+        str(directory), ignore_file_handler=ignore_file_handler
+    ):
         for f in files:
             yield os.path.join(root, f)
 

dvc/utils/collections.py

@@ -1,6 +1,54 @@
-from __future__ import unicode_literals
+from __future__ import absolute_import, unicode_literals
+from dvc.utils.compat import Mapping
 
 
 # just simple check for Nones and emtpy strings
 def compact(args):
     return list(filter(bool, args))
+
+
+def apply_diff(src, dest):
+    """Recursively apply changes from src to dest.
+
+    Preserves dest type and hidden info in dest structure,
+    like ruamel.yaml leaves when parses files. This includes comments,
+    ordering and line foldings.
+
+    Used in Stage load/dump cycle to preserve comments and custom formatting.
+    """
+    Seq = (list, tuple)
+    Container = (Mapping, list, tuple)
+
+    def is_same_type(a, b):
+        return any(
+            isinstance(a, t) and isinstance(b, t)
+            for t in [str, Mapping, Seq, bool]
+        )
+
+    if isinstance(src, Mapping) and isinstance(dest, Mapping):
+        for key, value in src.items():
+            if isinstance(value, Container) and is_same_type(
+                value, dest.get(key)
+            ):
+                apply_diff(value, dest[key])
+            elif key not in dest or value != dest[key]:
+                dest[key] = value
+        for key in set(dest) - set(src):
+            del dest[key]
+    elif isinstance(src, Seq) and isinstance(dest, Seq):
+        if len(src) != len(dest):
+            dest[:] = src
+        else:
+            for i, value in enumerate(src):
+                if isinstance(value, Container) and is_same_type(
+                    value, dest[i]
+                ):
+                    apply_diff(value, dest[i])
+                elif value != dest[i]:
+                    dest[i] = value
+    else:
+        raise AssertionError(
+            "Can't apply diff from {} to {}".format(
+                src.__class__.__name__, dest.__class__.__name__
+            )
+        )

dvc/utils/compat.py

@@ -1,10 +1,10 @@
 """Handle import compatibility between Python 2 and Python 3"""
+from __future__ import absolute_import
 
 import sys
 import os
 import errno
 
-
 # Syntax sugar.
 _ver = sys.version_info
 
@@ -91,12 +91,13 @@ def _makedirs(name, mode=0o777, exist_ok=False):
 
 
 if is_py2:
-    from urlparse import urlparse, urljoin  # noqa: F401
-    from StringIO import StringIO  # noqa: F401
+    from urlparse import urlparse, urljoin, urlsplit, urlunsplit  # noqa: F401
     from BaseHTTPServer import HTTPServer  # noqa: F401
     from SimpleHTTPServer import SimpleHTTPRequestHandler  # noqa: F401
     import ConfigParser  # noqa: F401
     from io import open  # noqa: F401
+    from pathlib2 import Path  # noqa: F401
+    from collections import Mapping  # noqa: F401
 
     builtin_str = str  # noqa: F821
     bytes = str  # noqa: F821
@@ -107,16 +108,42 @@ if is_py2:
     input = raw_input  # noqa: F821
     cast_bytes_py2 = cast_bytes
     makedirs = _makedirs
+    range = xrange  # noqa: F821
+
+    import StringIO
+    import io
+
+    class StringIO(StringIO.StringIO):
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args):
+            self.close()
+
+    class BytesIO(io.BytesIO):
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args):
+            self.close()
+
 
 elif is_py3:
+    from pathlib import Path  # noqa: F401
     from os import makedirs  # noqa: F401
-    from urllib.parse import urlparse, urljoin  # noqa: F401
-    from io import StringIO  # noqa: F401
+    from urllib.parse import (  # noqa: F401
+        urlparse,  # noqa: F401
+        urljoin,  # noqa: F401
+        urlsplit,  # noqa: F401
+        urlunsplit,  # noqa: F401
+    )
+    from io import StringIO, BytesIO  # noqa: F401
     from http.server import (  # noqa: F401
         HTTPServer,  # noqa: F401
         SimpleHTTPRequestHandler,  # noqa: F401
     )  # noqa: F401
     import configparser as ConfigParser  # noqa: F401
+    from collections.abc import Mapping  # noqa: F401
 
     builtin_str = str  # noqa: F821
     str = str  # noqa: F821
@@ -127,3 +154,4 @@ elif is_py3:
     input = input  # noqa: F821
     open = open  # noqa: F821
     cast_bytes_py2 = no_code
+    range = range  # noqa: F821

dvc/utils/fs.py

@@ -7,6 +7,7 @@ import logging
 
 from dvc.exceptions import DvcException
 from dvc.system import System
+from dvc.utils import dvc_walk
 from dvc.utils.compat import str
 
 
@@ -24,7 +25,7 @@ def get_mtime_and_size(path):
     mtime = os.path.getmtime(path)
 
     if os.path.isdir(path):
-        for root, dirs, files in os.walk(str(path)):
+        for root, dirs, files in dvc_walk(str(path)):
             for name in dirs + files:
                 entry = os.path.join(root, name)
                 try:
@@ -63,3 +64,22 @@ def contains_symlink_up_to(path, base_path):
     if os.path.dirname(path) == path:
         return False
     return contains_symlink_up_to(os.path.dirname(path), base_path)
+
+
+def get_parent_dirs_up_to(wdir, root_dir):
+
+    assert os.path.isabs(wdir)
+    assert os.path.isabs(root_dir)
+
+    wdir = os.path.normpath(wdir)
+    root_dir = os.path.normpath(root_dir)
+    if root_dir not in wdir:
+        return []
+
+    dirs = []
+    dirs.append(wdir)
+    while wdir != root_dir:
+        wdir = os.path.dirname(wdir)
+        dirs.append(wdir)
+
+    return dirs

dvc/utils/stage.py

@@ -0,0 +1,26 @@
+from dvc.utils.compat import open
+
+from ruamel.yaml.error import YAMLError
+from ruamel.yaml import YAML
+
+from dvc.exceptions import StageFileCorruptedError
+
+
+def load_stage_file(path):
+    with open(path, "r", encoding="utf-8") as fd:
+        return load_stage_fd(fd, path)
+
+
+def load_stage_fd(fd, path):
+    try:
+        yaml = YAML()
+        return yaml.load(fd) or {}
+    except YAMLError as exc:
+        raise StageFileCorruptedError(path, cause=exc)
+
+
+def dump_stage_file(path, data):
+    with open(path, "w", encoding="utf-8") as fd:
+        yaml = YAML()
+        yaml.default_flow_style = False
+        yaml.dump(data, fd)

dvc/version.py

@@ -7,7 +7,7 @@ import os
 import subprocess
 
 
-_BASE_VERSION = "0.35.7"
+_BASE_VERSION = "0.40.1"
 
 
 def _generate_version(base_version):

requirements.txt

@@ -5,18 +5,17 @@ configparser>=3.5.0
 zc.lockfile>=1.2.1
 future>=0.16.0
 google-cloud-storage==1.13.0
-PyInstaller==3.3.1
+PyInstaller==3.4
 colorama>=0.3.9
 configobj>=5.0.6
 networkx>=2.1
-pyyaml>=3.12
 gitpython>=2.1.8
 setuptools>=34.0.0
 nanotime>=0.5.2
 pyasn1>=0.4.1
 schema>=0.6.7
 paramiko>=2.4.1
-jsonpath-rw==1.4.0
+jsonpath-ng>=1.4.3
 requests>=2.18.4
 wheel>=0.31.1
 futures>=3.2.0; python_version == "2.7"
@@ -28,3 +27,7 @@ appdirs>=1.4.3
 treelib>=1.5.5
 inflect>=2.1.0
 humanize>=0.5.1
+dulwich>=0.19.11
+ruamel.yaml>=0.15.91
+pathlib2==2.3.3; python_version == "2.7"
+oss2==2.6.1

scripts/ci/before_install.sh

@@ -14,8 +14,7 @@ if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
     # NOTE: used to install and run our style checking tools
     osx_python_ver=3.6.2
     eval "$(pyenv init -)"
-    pyenv install --list
-    pyenv install --skip-existing --keep --verbose $osx_python_ver
+    pyenv install --skip-existing --keep --verbose $osx_python_ver &> pyenv.log || tail -n 50 pyenv.log
     pyenv shell $osx_python_ver
     python --version
 fi
@@ -50,6 +49,7 @@ echo > env.sh
 if [ -n "$TRAVIS_OS_NAME" ] && [ "$TRAVIS_OS_NAME" != "osx" ] \
    && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
   bash "$scriptdir/install_azurite.sh"
+  bash "$scriptdir/install_oss.sh"
   bash "$scriptdir/install_hadoop.sh"
 fi
 
@@ -60,7 +60,7 @@ fi
 
 if [[ -n "$PYTHON_VER" ]]; then
     eval "$(pyenv init -)"
-    pyenv install --skip-existing --keep --verbose "$PYTHON_VER"
+    pyenv install --skip-existing --keep --verbose "$PYTHON_VER" &> pyenv.log || tail -n 50 pyenv.log
     echo 'eval "$(pyenv init -)"' >> env.sh
     echo "pyenv shell $PYTHON_VER" >> env.sh
 fi

scripts/ci/install_oss.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -euo pipefail
+
+git clone https://github.com/iterative/oss-emulator.git
+sudo docker image build -t oss:1.0 oss-emulator
+sudo docker run --detach --restart always -p 8880:8880 --name oss-emulator oss:1.0
+echo "export OSS_ENDPOINT='localhost:8880'" >> env.sh
+echo "export OSS_ACCESS_KEY_ID='AccessKeyID'" >> env.sh
+echo "export OSS_ACCESS_KEY_SECRET='AccessKeySecret'" >> env.sh

scripts/completion/dvc.zsh

@@ -157,6 +157,7 @@ _dvc_repro=(
   {-i,--interactive}"[Ask for confirmation before reproducing each stage.]"
   {-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]"
   {-P,--all-pipelines}"[Reproduce all pipelines in the project.]"
+  {-d,--downstream}"[Reproduce the pipeline starting from the specified stage.]"
 )
 
 _dvc_remove=(

scripts/hooks/hook-Crypto.py

@@ -0,0 +1,60 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) 2005-2019, PyInstaller Development Team.
+#
+# Distributed under the terms of the GNU General Public License with exception
+# for distributing bootloader.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# -----------------------------------------------------------------------------
+
+"""
+Hook for PyCryptodome library: https://pypi.python.org/pypi/pycryptodome
+
+PyCryptodome is an almost drop-in replacement for the now unmaintained
+PyCrypto library. The two are mutually exclusive as they live under
+the same package ("Crypto").
+
+PyCryptodome distributes dynamic libraries and builds them as if they were
+Python C extensions (even though they are not extensions - as they can't be
+imported by Python). It might sound a bit weird, but this decision is rooted
+in PyPy and its partial and slow support for C extensions. However, this also
+invalidates several of the existing methods used by PyInstaller to decide the
+right files to pull in.
+
+Even though this hook is meant to help with PyCryptodome only, it will be
+triggered also when PyCrypto is installed, so it must be tested with both.
+
+Tested with PyCryptodome 3.5.1, PyCrypto 2.6.1, Python 2.7 & 3.6, Fedora &
+Windows
+"""
+
+import os
+import glob
+
+from PyInstaller.compat import EXTENSION_SUFFIXES
+from PyInstaller.utils.hooks import get_module_file_attribute
+
+# Include the modules as binaries in a subfolder named like the package.
+# Cryptodome's loader expects to find them inside the package directory for
+# the main module. We cannot use hiddenimports because that would add the
+# modules outside the package.
+
+binaries = []
+binary_module_names = [
+    "Crypto.Math",  # First in the list
+    "Crypto.Cipher",
+    "Crypto.Util",
+    "Crypto.Hash",
+    "Crypto.Protocol",
+]
+
+try:
+    for module_name in binary_module_names:
+        m_dir = os.path.dirname(get_module_file_attribute(module_name))
+        for ext in EXTENSION_SUFFIXES:
+            module_bin = glob.glob(os.path.join(m_dir, "_*%s" % ext))
+            for f in module_bin:
+                binaries.append((f, module_name.replace(".", os.sep)))
+except ImportError:
+    # Do nothing for PyCrypto (Crypto.Math does not exist there)
+    pass

scripts/hooks/hook-aliyunsdkcore.py

@@ -0,0 +1,3 @@
+from PyInstaller.utils.hooks import collect_data_files
+
+datas = collect_data_files("aliyunsdkcore")

scripts/hooks/hook-google.api.py

@@ -1,3 +0,0 @@
-from PyInstaller.utils.hooks import copy_metadata
-
-datas = copy_metadata("google-api-core")

scripts/hooks/hook-google.cloud.py

@@ -1,4 +0,0 @@
-from PyInstaller.utils.hooks import copy_metadata
-
-datas = copy_metadata("google-cloud-core")
-hiddenimports = ["google.api"]

setup.py

@@ -43,13 +43,12 @@ install_requires = [
     "colorama>=0.3.9",
     "configobj>=5.0.6",
     "networkx>=2.1",
-    "pyyaml>=3.12",
     "gitpython>=2.1.8",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
     "schema>=0.6.7",
-    "jsonpath-rw==1.4.0",
+    "jsonpath-ng>=1.4.3",
     "requests>=2.18.4",
     "grandalf==0.6",
     "asciimatics>=1.10.0",
@@ -58,14 +57,17 @@ install_requires = [
     "treelib>=1.5.5",
     "inflect>=2.1.0",
     "humanize>=0.5.1",
+    "dulwich>=0.19.11",
+    "ruamel.yaml>=0.15.91",
 ]
 
 # Extra dependencies for remote integrations
 gs = ["google-cloud-storage==1.13.0"]
 s3 = ["boto3==1.9.115"]
 azure = ["azure-storage-blob==1.3.0"]
+oss = ["oss2==2.6.1"]
 ssh = ["paramiko>=2.4.1"]
-all_remotes = gs + s3 + azure + ssh
+all_remotes = gs + s3 + azure + ssh + oss
 
 setup(
     name="dvc",
@@ -82,9 +84,10 @@ setup(
         "gs": gs,
         "s3": s3,
         "azure": azure,
+        "oss": oss,
         "ssh": ssh,
         # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1
-        ':python_version=="2.7"': ["futures"],
+        ':python_version=="2.7"': ["futures", "pathlib2"],
     },
     keywords="data science, data version control, machine learning",
     python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",

tests/__main__.py

@@ -10,15 +10,11 @@ os.putenv(
     "PATH", "{}:{}".format(os.path.join(REPO_ROOT, "bin"), os.getenv("PATH"))
 )
 os.putenv("DVC_HOME", REPO_ROOT)
-os.putenv("DVC_TEST", "true")
 
-if len(sys.argv) == 1:
-    params = ""
-else:
-    params = " ".join(sys.argv[1:])
+params = " ".join(sys.argv[1:])
 
 cmd = (
-    "py.test -v -n=4 --timeout=600 --timeout_method=thread --cov=dvc "
-    "{params} ".format(params=params)
+    "py.test -v -n=4 --timeout=600 --timeout_method=thread --log-level=debug"
+    " --cov=dvc {params} ".format(params=params)
 )
 check_call(cmd, shell=True)

tests/basic_env.py

@@ -7,6 +7,7 @@ import shutil
 import uuid
 import tempfile
 import logging
+import warnings
 
 from git import Repo
 from git.exc import GitCommandNotFound
@@ -60,7 +61,8 @@ class TestDirFixture(object):
         self._root_dir = os.path.realpath(root_dir)
 
     def _pushd(self, d):
-        self._saved_dir = os.path.realpath(os.curdir)
+        if not hasattr(self, "_saved_dir"):
+            self._saved_dir = os.path.realpath(os.curdir)
         os.chdir(d)
 
     def _popd(self):
@@ -98,6 +100,21 @@ class TestDirFixture(object):
 
     def tearDown(self):
         self._popd()
+        try:
+            shutil.rmtree(self._root_dir)
+        except OSError as exc:
+            # We ignore this under Windows with a warning because it happened
+            # to be really hard to trace all not properly closed files.
+            #
+            # Best guess so far is that gitpython is the culprit:
+            # it opens files and uses __del__ to close them, which can happen
+            # late in current pythons. TestGitFixture and TestDvcFixture try
+            # to close that and it works on most of the tests, but not all.
+            # Repos and thus git repos are created all over the dvc ;)
+            if os.name == "nt" and exc.winerror == 32:
+                warnings.warn("Failed to remove test dir: " + str(exc))
+            else:
+                raise
 
 
 class TestGitFixture(TestDirFixture):
@@ -122,6 +139,9 @@ class TestGitFixture(TestDirFixture):
         self.git.index.add([self.CODE])
         self.git.index.commit("add code")
 
+    def tearDown(self):
+        self.git.close()
+
 
 class TestGitSubmoduleFixture(TestGitFixture):
     def __init__(self, root_dir=None):
@@ -143,7 +163,9 @@ class TestDvcFixture(TestGitFixture):
         super(TestDvcFixture, self).setUp()
         self.dvc = DvcRepo.init(self._root_dir)
         self.dvc.scm.commit("init dvc")
-        logger.setLevel("DEBUG")
+
+    def tearDown(self):
+        self.dvc.scm.git.close()
 
 
 class TestDvcGitInitializedFixture(TestDvcFixture):
@@ -196,6 +218,9 @@ class TestDvcDataFileFixture(TestDvcGitInitializedFixture):
         )
         self.git.index.commit("Hello world commit")
 
+        # Return to the dir we started to not confuse parent fixture
+        os.chdir(self._saved_dir)
+
 
 # NOTE: Inheritance order in the classes below is important.
 

tests/conftest.py

@@ -0,0 +1,99 @@
+from __future__ import unicode_literals
+import mockssh
+import pytest
+import os
+from git import Repo
+from git.exc import GitCommandNotFound
+
+from dvc.utils.compat import cast_bytes_py2
+from dvc.remote.ssh.connection import SSHConnection
+from dvc.repo import Repo as DvcRepo
+from .basic_env import TestDirFixture
+
+
+# Prevent updater and analytics from running their processes
+os.environ[cast_bytes_py2("DVC_TEST")] = cast_bytes_py2("true")
+
+
+@pytest.fixture(autouse=True)
+def reset_loglevel(request, caplog):
+    """
+    Use it to ensure log level at the start of each test
+    regardless of dvc.logger.setup(), Repo configs or whatever.
+    """
+    level = request.config.getoption("--log-level")
+    if level:
+        with caplog.at_level(level.upper(), logger="dvc"):
+            yield
+    else:
+        yield
+
+
+# Wrap class like fixture as pytest-like one to avoid code duplication
+@pytest.fixture
+def repo_dir():
+    old_fixture = TestDirFixture()
+    old_fixture.setUp()
+    try:
+        yield old_fixture
+    finally:
+        old_fixture.tearDown()
+
+
+# NOTE: this duplicates code from GitFixture,
+# would fix itself once class-based fixtures are removed
+@pytest.fixture
+def git(repo_dir):
+    # NOTE: handles EAGAIN error on BSD systems (osx in our case).
+    # Otherwise when running tests you might get this exception:
+    #
+    #    GitCommandNotFound: Cmd('git') not found due to:
+    #        OSError('[Errno 35] Resource temporarily unavailable')
+    retries = 5
+    while retries:
+        try:
+            git = Repo.init()
+        except GitCommandNotFound:
+            retries -= 1
+            continue
+        break
+
+    try:
+        git.index.add([repo_dir.CODE])
+        git.index.commit("add code")
+        yield git
+    finally:
+        git.close()
+
+
+@pytest.fixture
+def dvc(repo_dir, git):
+    try:
+        dvc = DvcRepo.init(repo_dir._root_dir)
+        dvc.scm.commit("init dvc")
+        yield dvc
+    finally:
+        dvc.scm.git.close()
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+user = "user"
+key_path = os.path.join(here, "{0}.key".format(user))
+
+
+@pytest.fixture
+def ssh_server():
+    users = {user: key_path}
+    with mockssh.Server(users) as s:
+        yield s
+
+
+@pytest.fixture
+def ssh(ssh_server):
+    yield SSHConnection(
+        ssh_server.host,
+        username=user,
+        port=ssh_server.port,
+        key_filename=key_path,
+    )

tests/func/test_add.py

@@ -7,12 +7,16 @@ import time
 import shutil
 import filecmp
 import posixpath
+import logging
+import colorama
 
 from dvc.system import System
 from mock import patch
 
 from dvc.main import main
-from dvc.utils import file_md5, load_stage_file
+from dvc.utils import file_md5, LARGE_DIR_SIZE
+from dvc.utils.stage import load_stage_file
+from dvc.utils.compat import range
 from dvc.stage import Stage
 from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 from dvc.output.base import OutputAlreadyTrackedError
@@ -86,6 +90,30 @@ class TestAddCmdDirectoryRecursive(TestDvc):
         ret = main(["add", "--recursive", self.DATA_DIR])
         self.assertEqual(ret, 0)
 
+    def test_warn_about_large_directories(self):
+        warning = (
+            "You are adding a large directory 'large-dir' recursively,"
+            " consider tracking it as a whole instead.\n"
+            "{purple}HINT:{nc} Remove the generated stage files and then"
+            " run {cyan}dvc add large-dir{nc}".format(
+                purple=colorama.Fore.MAGENTA,
+                cyan=colorama.Fore.CYAN,
+                nc=colorama.Style.RESET_ALL,
+            )
+        )
+
+        os.mkdir("large-dir")
+
+        # Create a lot of files
+        for iteration in range(LARGE_DIR_SIZE + 1):
+            path = os.path.join("large-dir", str(iteration))
+            with open(path, "w") as fobj:
+                fobj.write(path)
+
+        with self._caplog.at_level(logging.WARNING, logger="dvc"):
+            assert main(["add", "--recursive", "large-dir"]) == 0
+            assert warning in self._caplog.text
+
 
 class TestAddDirectoryWithForwardSlash(TestDvc):
     def test(self):
@@ -223,9 +251,8 @@ class TestDoubleAddUnchanged(TestDvc):
 
 class TestShouldUpdateStateEntryForFileAfterAdd(TestDvc):
     def test(self):
-        file_md5_counter = spy(dvc.state.file_md5)
-        with patch.object(dvc.state, "file_md5", file_md5_counter):
-
+        file_md5_counter = spy(dvc.remote.local.file_md5)
+        with patch.object(dvc.remote.local, "file_md5", file_md5_counter):
             ret = main(["config", "cache.type", "copy"])
             self.assertEqual(ret, 0)
 
@@ -244,8 +271,8 @@ class TestShouldUpdateStateEntryForFileAfterAdd(TestDvc):
 
 class TestShouldUpdateStateEntryForDirectoryAfterAdd(TestDvc):
     def test(self):
-        file_md5_counter = spy(dvc.state.file_md5)
-        with patch.object(dvc.state, "file_md5", file_md5_counter):
+        file_md5_counter = spy(dvc.remote.local.file_md5)
+        with patch.object(dvc.remote.local, "file_md5", file_md5_counter):
 
             ret = main(["config", "cache.type", "copy"])
             self.assertEqual(ret, 0)
@@ -302,27 +329,14 @@ class TestShouldNotCheckCacheForDirIfCacheMetadataDidNotChange(TestDvc):
 
 
 class TestShouldCollectDirCacheOnlyOnce(TestDvc):
-    NEW_LARGE_DIR_SIZE = 1
-
-    @patch("dvc.remote.local.LARGE_DIR_SIZE", NEW_LARGE_DIR_SIZE)
     def test(self):
         from dvc.remote.local import RemoteLOCAL
 
-        collect_dir_counter = spy(RemoteLOCAL.collect_dir_cache)
+        get_dir_checksum_counter = spy(RemoteLOCAL.get_dir_checksum)
         with patch.object(
-            RemoteLOCAL, "collect_dir_cache", collect_dir_counter
+            RemoteLOCAL, "get_dir_checksum", get_dir_checksum_counter
         ):
-
-            LARGE_DIR_FILES_NUM = self.NEW_LARGE_DIR_SIZE + 1
-            data_dir = "dir"
-
-            os.makedirs(data_dir)
-
-            for i in range(LARGE_DIR_FILES_NUM):
-                with open(os.path.join(data_dir, str(i)), "w+") as f:
-                    f.write(str(i))
-
-            ret = main(["add", data_dir])
+            ret = main(["add", self.DATA_DIR])
             self.assertEqual(0, ret)
 
             ret = main(["status"])
@@ -330,7 +344,7 @@ class TestShouldCollectDirCacheOnlyOnce(TestDvc):
 
             ret = main(["status"])
             self.assertEqual(0, ret)
-        self.assertEqual(1, collect_dir_counter.mock.call_count)
+        self.assertEqual(1, get_dir_checksum_counter.mock.call_count)
 
 
 class SymlinkAddTestBase(TestDvc):
@@ -488,3 +502,24 @@ class TestShouldNotTrackGitInternalFiles(TestDvc):
         created_stages_filenames = stage_creator_spy.mock.call_args[0][0]
         for fname in created_stages_filenames:
             self.assertNotIn(".git", fname)
+
+
+class TestAddUnprotected(TestDvc):
+    def test(self):
+        ret = main(["config", "cache.type", "hardlink"])
+        self.assertEqual(ret, 0)
+
+        ret = main(["config", "cache.protected", "true"])
+        self.assertEqual(ret, 0)
+
+        ret = main(["add", self.FOO])
+        self.assertEqual(ret, 0)
+
+        ret = main(["unprotect", self.FOO])
+        self.assertEqual(ret, 0)
+
+        ret = main(["add", self.FOO])
+        self.assertEqual(ret, 0)
+
+        self.assertFalse(os.access(self.FOO, os.W_OK))
+        self.assertTrue(System.is_hardlink(self.FOO))

tests/func/test_cache.py

@@ -168,3 +168,8 @@ class TestCmdCacheDir(TestDvc):
         self.assertEqual(len(subdirs), 1)
         files = os.listdir(os.path.join(tmpdir, subdirs[0]))
         self.assertEqual(len(files), 1)
+
+
+class TestShouldCacheBeReflinkOrCopyByDefault(TestDvc):
+    def test(self):
+        self.assertEqual(self.dvc.cache.local.cache_types, ["reflink", "copy"])

tests/func/test_checkout.py

@@ -2,7 +2,6 @@ import os
 import sys
 import re
 
-import yaml
 import shutil
 import filecmp
 import collections
@@ -13,8 +12,9 @@ from dvc import progress
 from dvc.repo import Repo as DvcRepo
 from dvc.system import System
 from dvc.utils import walk_files
+from dvc.utils.stage import load_stage_file, dump_stage_file
 from tests.basic_env import TestDvc
-from tests.test_repro import TestRepro
+from tests.func.test_repro import TestRepro
 from dvc.stage import Stage, StageFileBadNameError, StageFileDoesNotExistError
 from dvc.remote.local import RemoteLOCAL
 from dvc.exceptions import (
@@ -69,7 +69,7 @@ class TestCheckoutSingleStage(TestCheckout):
 
 class TestCheckoutCorruptedCacheFile(TestRepro):
     def test(self):
-        cache = self.foo_stage.outs[0].cache
+        cache = self.foo_stage.outs[0].cache_path
 
         with open(cache, "a") as fd:
             fd.write("1")
@@ -126,15 +126,16 @@ class CheckoutBase(TestDvc):
         self.dvc.scm.commit("adding " + fname)
 
     def read_ignored(self):
-        return list(
-            map(lambda s: s.strip("\n"), open(self.GIT_IGNORE).readlines())
-        )
+        with open(self.GIT_IGNORE) as f:
+            return [s.strip("\n") for s in f.readlines()]
 
     def outs_info(self, stage):
         FileInfo = collections.namedtuple("FileInfo", "path inode")
 
         paths = [
-            path for output in stage.outs for path in walk_files(output.path)
+            path
+            for output in stage["outs"]
+            for path in walk_files(output["path"])
         ]
 
         return [
@@ -210,14 +211,19 @@ class TestCheckoutSelectiveRemove(CheckoutBase):
         ret = main(["config", "cache.type", "copy"])
         self.assertEqual(ret, 0)
 
-        stages = self.dvc.add(self.DATA_DIR)
-        self.assertEqual(len(stages), 1)
-        stage = stages[0]
+        ret = main(["add", self.DATA_DIR])
+        self.assertEqual(0, ret)
+
+        stage_path = self.DATA_DIR + Stage.STAGE_FILE_SUFFIX
+        stage = load_stage_file(stage_path)
         staged_files = self.outs_info(stage)
 
-        os.remove(staged_files[0].path)
+        # move instead of remove, to lock inode assigned to stage_files[0].path
+        # if we were to use remove, we might end up with same inode assigned to
+        # newly checked out file
+        shutil.move(staged_files[0].path, "random_name")
 
-        ret = main(["checkout", "--force", stage.relpath])
+        ret = main(["checkout", "--force", stage_path])
         self.assertEqual(ret, 0)
 
         checkedout_files = self.outs_info(stage)
@@ -291,14 +297,10 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
 
 class TestCheckoutMissingMd5InStageFile(TestRepro):
     def test(self):
-        with open(self.file1_stage, "r") as fd:
-            d = yaml.load(fd)
-
+        d = load_stage_file(self.file1_stage)
         del d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM]
         del d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM]
-
-        with open(self.file1_stage, "w") as fd:
-            yaml.dump(d, fd)
+        dump_stage_file(self.file1_stage, d)
 
         self.dvc.checkout(force=True)
 

tests/func/test_commit.py

@@ -0,0 +1,82 @@
+import pytest
+from dvc.utils.stage import load_stage_file, dump_stage_file
+
+from dvc.stage import StageCommitError
+
+
+def test_commit_recursive(dvc, repo_dir):
+    stages = dvc.add(repo_dir.DATA_DIR, recursive=True, no_commit=True)
+    assert len(stages) == 2
+
+    assert dvc.status() != {}
+
+    dvc.commit(repo_dir.DATA_DIR, recursive=True)
+
+    assert dvc.status() == {}
+
+
+def test_commit_force(dvc, repo_dir):
+    stages = dvc.add(repo_dir.FOO, no_commit=True)
+    assert len(stages) == 1
+    stage = stages[0]
+
+    with dvc.state:
+        assert stage.outs[0].changed_cache()
+
+    with open(repo_dir.FOO, "a") as fobj:
+        fobj.write(repo_dir.FOO_CONTENTS)
+
+    with dvc.state:
+        assert stage.outs[0].changed_cache()
+
+    with pytest.raises(StageCommitError):
+        dvc.commit(stage.path)
+
+    with dvc.state:
+        assert stage.outs[0].changed_cache()
+
+    dvc.commit(stage.path, force=True)
+
+    assert dvc.status(stage.path) == {}
+
+
+def test_commit_with_deps(dvc, repo_dir):
+    stages = dvc.add(repo_dir.FOO, no_commit=True)
+    assert len(stages) == 1
+    foo_stage = stages[0]
+    assert foo_stage is not None
+    assert len(foo_stage.outs) == 1
+
+    fname = "file"
+    stage = dvc.run(
+        cmd="python {} {} {}".format(repo_dir.CODE, repo_dir.FOO, fname),
+        outs=[fname],
+        deps=[repo_dir.FOO, repo_dir.CODE],
+        no_commit=True,
+    )
+    assert stage is not None
+    assert len(stage.outs) == 1
+
+    with dvc.state:
+        assert foo_stage.outs[0].changed_cache()
+        assert stage.outs[0].changed_cache()
+
+    dvc.commit(stage.path, with_deps=True)
+    with dvc.state:
+        assert not foo_stage.outs[0].changed_cache()
+        assert not stage.outs[0].changed_cache()
+
+
+def test_commit_changed_md5(dvc, repo_dir):
+    stages = dvc.add(repo_dir.FOO, no_commit=True)
+    assert len(stages) == 1
+    stage = stages[0]
+
+    stage_file_content = load_stage_file(stage.path)
+    stage_file_content["md5"] = "1111111111"
+    dump_stage_file(stage.path, stage_file_content)
+
+    with pytest.raises(StageCommitError):
+        dvc.commit(stage.path)
+
+    dvc.commit(stage.path, force=True)

tests/func/test_data_cloud.py

@@ -6,11 +6,10 @@ import uuid
 import shutil
 import getpass
 import platform
-import yaml
 import copy
 import logging
+import pytest
 
-from dvc.state import State
 from mock import patch
 
 from dvc.utils.compat import str
@@ -21,15 +20,19 @@ from dvc.data_cloud import (
     RemoteS3,
     RemoteGS,
     RemoteAzure,
+    RemoteOSS,
     RemoteLOCAL,
     RemoteSSH,
     RemoteHDFS,
     RemoteHTTP,
 )
 from dvc.remote.base import STATUS_OK, STATUS_NEW, STATUS_DELETED
-from dvc.utils import file_md5, load_stage_file
+from dvc.utils import file_md5
+from dvc.utils.stage import load_stage_file, dump_stage_file
 
 from tests.basic_env import TestDvc
+from tests.conftest import user
+from tests.conftest import key_path
 from tests.utils import spy
 
 
@@ -42,6 +45,7 @@ TEST_CONFIG = {
 
 TEST_AWS_REPO_BUCKET = "dvc-test"
 TEST_GCP_REPO_BUCKET = "dvc-test"
+TEST_OSS_REPO_BUCKET = "dvc-test"
 
 
 def _should_test_aws():
@@ -97,6 +101,19 @@ def _should_test_azure():
     )
 
 
+def _should_test_oss():
+    if os.getenv("DVC_TEST_OSS") == "true":
+        return True
+    elif os.getenv("DVC_TEST_OSS") == "false":
+        return False
+
+    return (
+        os.getenv("OSS_ENDPOINT")
+        and os.getenv("OSS_ACCESS_KEY_ID")
+        and os.getenv("OSS_ACCESS_KEY_SECRET")
+    )
+
+
 def _should_test_ssh():
     if os.getenv("DVC_TEST_SSH") == "true":
         return True
@@ -150,6 +167,25 @@ def get_ssh_url():
     )
 
 
+def get_ssh_url_mocked(user, port):
+    path = get_local_storagepath()
+    if os.name == "nt":
+        # NOTE: On Windows get_local_storagepath() will return an ntpath
+        # that looks something like `C:\some\path`, which is not compatible
+        # with SFTP paths [1], so we need to convert it to a proper posixpath.
+        # To do that, we should construct a posixpath that would be relative
+        # to the server's root. In our case our ssh server is running with
+        # `c:/` as a root, and our URL format requires absolute paths, so the
+        # resulting path would look like `/some/path`.
+        #
+        # [1]https://tools.ietf.org/html/draft-ietf-secsh-filexfer-13#section-6
+        drive, path = os.path.splitdrive(path)
+        assert drive == "c:"
+        path = path.replace("\\", "/")
+    url = "ssh://{}@127.0.0.1:{}{}".format(user, port, path)
+    return url
+
+
 def get_hdfs_url():
     return "hdfs://{}@127.0.0.1{}".format(
         getpass.getuser(), get_local_storagepath()
@@ -187,6 +223,14 @@ def get_azure_url():
     return "azure://{}/{}".format(container_name, str(uuid.uuid4()))
 
 
+def get_oss_storagepath():
+    return "{}/{}".format(TEST_OSS_REPO_BUCKET, (uuid.uuid4()))
+
+
+def get_oss_url():
+    return "oss://{}".format(get_oss_storagepath())
+
+
 class TestDataCloud(TestDvc):
     def _test_cloud(self, config, cl):
         cloud = DataCloud(self.dvc, config=config)
@@ -201,6 +245,7 @@ class TestDataCloud(TestDvc):
             ("ssh://user@localhost:/", RemoteSSH),
             ("http://localhost:8000/", RemoteHTTP),
             ("azure://ContainerName=mybucket;conn_string;", RemoteAzure),
+            ("oss://mybucket/", RemoteOSS),
             (TestDvc.mkdtemp(), RemoteLOCAL),
         ]
 
@@ -220,6 +265,9 @@ class TestDataCloudBase(TestDvc):
     def _get_url(self):
         return ""
 
+    def _get_keyfile(self):
+        return None
+
     def _ensure_should_run(self):
         if not self._should_test():
             raise SkipTest(
@@ -230,9 +278,11 @@ class TestDataCloudBase(TestDvc):
         self._ensure_should_run()
 
         repo = self._get_url()
+        keyfile = self._get_keyfile()
 
         config = copy.deepcopy(TEST_CONFIG)
         config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
+        config[TEST_SECTION][Config.SECTION_REMOTE_KEY_FILE] = keyfile
         self.cloud = DataCloud(self.dvc, config)
 
         self.assertIsInstance(self.cloud._cloud, self._get_cloud_class())
@@ -244,7 +294,7 @@ class TestDataCloudBase(TestDvc):
         self.assertEqual(len(stages), 1)
         stage = stages[0]
         self.assertTrue(stage is not None)
-        cache = stage.outs[0].cache
+        cache = stage.outs[0].cache_path
         info = stage.outs[0].dumpd()
         md5 = info["md5"]
 
@@ -253,7 +303,7 @@ class TestDataCloudBase(TestDvc):
         stage_dir = stages[0]
         self.assertTrue(stage_dir is not None)
 
-        cache_dir = stage_dir.outs[0].cache
+        cache_dir = stage_dir.outs[0].cache_path
         info_dir = stage_dir.outs[0].dumpd()
         md5_dir = info_dir["md5"]
 
@@ -369,6 +419,17 @@ class TestRemoteAzure(TestRemoteAzureCompat):
         return get_azure_url()
 
 
+class TestRemoteOSS(TestDataCloudBase):
+    def _should_test(self):
+        return _should_test_oss()
+
+    def _get_url(self):
+        return get_oss_url()
+
+    def _get_cloud_class(self):
+        return RemoteOSS
+
+
 class TestRemoteLOCAL(TestDataCloudBase):
     def _should_test(self):
         return True
@@ -396,6 +457,26 @@ class TestRemoteSSH(TestDataCloudBase):
         return RemoteSSH
 
 
+@pytest.mark.usefixtures("ssh_server")
+class TestRemoteSSHMocked(TestDataCloudBase):
+    @pytest.fixture(autouse=True)
+    def setup_method_fixture(self, request, ssh_server):
+        self.ssh_server = ssh_server
+        self.method_name = request.function.__name__
+
+    def _get_url(self):
+        return get_ssh_url_mocked(user, self.ssh_server.port)
+
+    def _get_keyfile(self):
+        return key_path
+
+    def _should_test(self):
+        return True
+
+    def _get_cloud_class(self):
+        return RemoteSSH
+
+
 class TestRemoteHDFS(TestDataCloudBase):
     def _should_test(self):
         return _should_test_hdfs()
@@ -423,13 +504,13 @@ class TestDataCloudCLIBase(TestDvc):
         self.assertEqual(len(stages), 1)
         stage = stages[0]
         self.assertTrue(stage is not None)
-        cache = stage.outs[0].cache
+        cache = stage.outs[0].cache_path
 
         stages = self.dvc.add(self.DATA_DIR)
         self.assertEqual(len(stages), 1)
         stage_dir = stages[0]
         self.assertTrue(stage_dir is not None)
-        cache_dir = stage_dir.outs[0].cache
+        cache_dir = stage_dir.outs[0].cache_path
 
         # FIXME check status output
         self.main(["status", "-c", "--show-checksums"] + args)
@@ -486,8 +567,11 @@ class TestDataCloudCLIBase(TestDvc):
         pass
 
     def test(self):
-        if self._should_test():
-            self._test()
+        if not self._should_test():
+            raise SkipTest(
+                "Test {} is disabled".format(self.__class__.__name__)
+            )
+        self._test()
 
 
 class TestCompatRemoteLOCALCLI(TestDataCloudCLIBase):
@@ -601,6 +685,18 @@ class TestRemoteAzureCLI(TestDataCloudCLIBase):
         self._test_cloud(TEST_REMOTE)
 
 
+class TestRemoteOSSCLI(TestDataCloudCLIBase):
+    def _should_test(self):
+        return _should_test_oss()
+
+    def _test(self):
+        url = get_oss_url()
+
+        self.main(["remote", "add", TEST_REMOTE, url])
+
+        self._test_cloud(TEST_REMOTE)
+
+
 class TestDataCloudErrorCLI(TestDvc):
     def main_fail(self, args):
         ret = main(args)
@@ -629,8 +725,7 @@ class TestWarnOnOutdatedStage(TestDvc):
         stage_file_path = stage.relpath
         content = load_stage_file(stage_file_path)
         del content["outs"][0]["md5"]
-        with open(stage_file_path, "w") as stage_file:
-            yaml.dump(content, stage_file)
+        dump_stage_file(stage_file_path, content)
 
         with self._caplog.at_level(logging.WARNING, logger="dvc"):
             self._caplog.clear()
@@ -731,8 +826,10 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
 
 class TestCheckSumRecalculation(TestDvc):
     def test(self):
-        test_collect = spy(State._collect)
-        with patch.object(State, "_collect", test_collect):
+        test_get_file_checksum = spy(RemoteLOCAL.get_file_checksum)
+        with patch.object(
+            RemoteLOCAL, "get_file_checksum", test_get_file_checksum
+        ):
             url = get_local_url()
             ret = main(["remote", "add", "-d", TEST_REMOTE, url])
             self.assertEqual(ret, 0)
@@ -744,7 +841,7 @@ class TestCheckSumRecalculation(TestDvc):
             self.assertEqual(ret, 0)
             ret = main(["run", "-d", self.FOO, "echo foo"])
             self.assertEqual(ret, 0)
-        self.assertEqual(test_collect.mock.call_count, 1)
+        self.assertEqual(test_get_file_checksum.mock.call_count, 1)
 
 
 class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):