dvc: bump to 0.40.1

pyinstaller: fix build for oss2

.appveyor.yml

@@ -3,6 +3,10 @@ branches:
     - master
     - devel
 
+cache:
+  - C:\ProgramData\chocolatey\bin -> appveyor.yml
+  - C:\ProgramData\chocolatey\lib -> appveyor.yml
+
 platform: x86
 
 environment:
@@ -16,36 +20,29 @@ environment:
     secure: XN4jRtmGE5Bqg8pPZkwNs7kn3UEI73Rkldqc0MGsQISZBm5TNJZOPofDMc1QnUsf
   AZURE_STORAGE_CONTAINER_NAME: appveyor-tests
   AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
+  OSS_ENDPOINT: localhost:50004
+  OSS_ACCESS_KEY_ID: AccessKeyID
+  OSS_ACCESS_KEY_SECRET: AccessKeySecret
 
   matrix:
 
     - PYTHON: "C:\\Python27"
       PYTHON_VERSION: "2.7.x" # currently 2.7.9
       PYTHON_ARCH: "32"
+      RUBY: C:\Ruby25\bin
 
 # Disabled as unnecessary
 #    - PYTHON: "C:\\Python27-x64"
 #      PYTHON_VERSION: "2.7.x" # currently 2.7.9
 #      PYTHON_ARCH: "64"
-#
-#    - PYTHON: "C:\\Python33"
-#      PYTHON_VERSION: "3.3.x" # currently 3.3.5
-#      PYTHON_ARCH: "32"
-#
-#    - PYTHON: "C:\\Python33-x64"
-#      PYTHON_VERSION: "3.3.x" # currently 3.3.5
-#      PYTHON_ARCH: "64"
-#
-#    - PYTHON: "C:\\Python34"
-#      PYTHON_VERSION: "3.4.x" # currently 3.4.3
-#      PYTHON_ARCH: "32"
-#
-#    - PYTHON: "C:\\Python34-x64"
-#      PYTHON_VERSION: "3.4.x" # currently 3.4.3
-#      PYTHON_ARCH: "64"
 
 install:
-  - ps: Install-Product node
+  - SET PATH=%RUBY%;%PATH%
+  - git clone -q https://github.com/nanaya-tachibana/oss-emulator.git
+  - gem install thor builder
+  - ps: cd oss-emulator
+  - ps: $OSSProcess = Start-Process -FilePath ruby -ArgumentList "bin\emulator -r store -p 50004" -PassThru -NoNewWindow
+  - ps: cd ..
   - npm install -g azurite
   - ps: $AzuriteProcess = Start-Process azurite-blob -PassThru
   - cinst wget
@@ -65,18 +62,20 @@ before_test:
   - aws configure set aws_secret_access_key "%aws_secret_access_key%"
   - aws configure set region us-east-2
   - openssl enc -d -aes-256-cbc -md md5 -k "%GCP_CREDS%" -in scripts\ci\gcp-creds.json.enc -out scripts\ci\gcp-creds.json & exit 0
-  - pip install -r test-requirements.txt
+  - pip install -r tests/requirements.txt
 
 test_script:
   - python -mtests
 
 after_test:
+  - if "%APPVEYOR_PULL_REQUEST_NUMBER%"=="" pip install codecov
   - if "%APPVEYOR_PULL_REQUEST_NUMBER%"=="" codecov
   - if "%APPVEYOR_REPO_TAG%"=="true" python -m dvc pull
   - if "%APPVEYOR_REPO_TAG%"=="true" .\scripts\build_windows.cmd
 
 on_finish:
   - ps: Stop-Process $AzuriteProcess
+  - ps: Stop-Process $OSSProcess
 
 artifacts:
   - path: dvc*.exe

.dvc/config

@@ -1,4 +1,4 @@
 ['remote "dvc"']
-url = s3://dvc-share/dvc/
+url = https://dvc.org/s3/dvc
 [core]
 remote = dvc

.flake8

@@ -1,4 +1,7 @@
 [flake8]
-ignore = E203, E266, E501, W503, F403, F401
+ignore =
+    E203, # Whitespace before ':'
+    E266, # Too many leading '#' for block comment
+    W503, # Line break occurred before a binary operator
 max-line-length = 79
 select = B,C,E,F,W,T4,B9

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,9 @@
+* [ ] Have you followed the guidelines in our
+      [Contributing document](https://dvc.org/doc/user-guide/contributing)?
+
+* [ ] Does your PR affect documented changes or does it add new functionality
+      that should be documented? If yes, have you created a PR for
+      [dvc.org](https://github.com/iterative/dvc.org) documenting it or at
+      least opened an issue for it? If so, please add a link to it.
+
+-----

.gitignore

@@ -24,6 +24,7 @@ innosetup/config.ini
 
 .coverage
 
-dvc/version.py
-
 *.swp
+
+pip-wheel-metadata/
+.vscode/

.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: stable
+    rev: 19.3b0
     hooks:
     - id: black
-      language_version: python3.6
+      language_version: python3
 -   repo: https://gitlab.com/pycqa/flake8
     rev: master
     hooks:
     - id: flake8
-      language_version: python3.6
+      language_version: python3

.travis.yml

@@ -3,6 +3,7 @@ cache:
   - pip
   - directories:
     - /opt/python
+    - $HOME/.pyenv
 sudo: true
 dist: xenial
 python: "3.7"
@@ -21,9 +22,6 @@ matrix:
   - name: "2.7"
     env:
     - PYTHON_VER=2.7.13
-  - name: "3.4"
-    env:
-    - PYTHON_VER=3.4.6
   - name: "3.5"
     env:
     - PYTHON_VER=3.5.3
@@ -50,7 +48,7 @@ matrix:
     language: ruby
     rvm: 2.4.0
     env:
-    - PYTHON_VER=3.4.3
+    - PYTHON_VER=3.6.2
     - PYTHON_CONFIGURE_OPTS="--enable-framework"
   - if: type = cron OR tag is present
     os: osx
@@ -60,20 +58,34 @@ matrix:
     env:
     - PYTHON_VER=3.5.0
     - PYTHON_CONFIGURE_OPTS="--enable-framework"
+  - if: type = cron OR tag is present
+    os: osx
+    osx_image: xcode10.1
+    language: ruby
+    rvm: 2.4.0
+    env:
+    - PYTHON_VER=2.7.10
+    - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
+    - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
+    - PYTHON_CONFIGURE_OPTS="--enable-framework"
+  - if: type = cron OR tag is present
+    os: osx
+    osx_image: xcode10.1
+    language: ruby
+    rvm: 2.4.0
+    env:
+    - PYTHON_VER=3.5.0
+    - PYTHON_CONFIGURE_OPTS="--enable-framework"
 before_install:
   - bash ./scripts/ci/before_install.sh
-  - if [[ -n "$PYTHON_VER" ]]; then
-      eval "$(pyenv init -)";
-      pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
-      pyenv shell "$PYTHON_VER";
-    fi
   - source env.sh
+  - python --version
 install:
   - bash ./scripts/ci/install.sh
 script:
   - bash ./scripts/ci/script.sh
 deploy:
-  condition: "[[ $(python -c 'import sys; print(sys.version_info[0])') == '2' ]]"
+  condition: "[[ $(./scripts/ci/deploy_condition.sh) == 'true' ]]"
   provider: releases
   api_key:
     secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=

CONTRIBUTING.md

@@ -0,0 +1 @@
+### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing#contributing).

README.rst

@@ -1,4 +1,4 @@
-.. image:: https://dvc.org/static/img/logo-owl-readme.png
+.. image:: https://dvc.org/static/img/logo-github-readme.png
   :target: https://dvc.org
   :alt: DVC logo
 
@@ -6,7 +6,7 @@
 • `Docs <https://dvc.org/doc>`_
 • `Twitter <https://twitter.com/iterativeai>`_
 • `Chat (Community & Support) <https://dvc.org/chat>`_
-• `Tutorial <https://dvc.org/doc/tutorial>`_
+• `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
 .. image:: https://travis-ci.com/iterative/dvc.svg?branch=master
@@ -25,20 +25,25 @@
   :target: https://codecov.io/gh/iterative/dvc
   :alt: Codecov
 
+.. image:: https://img.shields.io/badge/patreon-donate-green.svg
+  :target: https://www.patreon.com/DVCorg/overview
+  :alt: Donate
+
 |
 
 **Data Science Version Control** or **DVC** is an **open-source** tool for data science and
-machine learning projects. With a simple and flexible Git-like architecture and interface it
-helps data scientists:
+machine learning projects. Key features:
 
-#. manage **machine learning models** - versioning, including data sets and transformations (scripts) that were
-   used to generate models;
+#. simple **command line** Git-like experience. Does not require installing and maintaining
+   any databases. Does not depend on any proprietary online services;
 
-#. make projects **reproducible**;
+#. it manages and versions **datasets** and **machine learning models**. Data is saved in
+   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS or even local HDD RAID;
 
-#. make projects **shareable**;
+#. it makes projects **reproducible** and **shareable**, it helps answering question how
+   the model was build;
 
-#. manage experiments with branching and **metrics** tracking;
+#. it helps manage experiments with Git tags or branches and **metrics** tracking;
 
 It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
 a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
@@ -50,14 +55,30 @@ data file suffixes and prefixes.
 How DVC works
 =============
 
-DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
-To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
-network storage (via SSH, for example).
+We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ to better understand what DVC
+is and how does it fit your scenarios.
+
+The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-lfs to be precise) + ``makefiles``
+made right and tailored specifically for ML and Data Science scenarios.
+
+#. ``Git/Git-lfs`` part - DVC helps you storing and sharing data artifacts, models. It connects them with your
+   Git repository.
+#. ``Makefiles`` part - DVC describes how one data or model artifact was build from another data.
+
+DVC usually runs along with Git. Git is used as usual to store and version code and DVC meta-files. DVC helps
+to store data and model files seamlessly out of Git while preserving almost the same user experience as if they
+were stored in Git itself. To store and share data files cache DVC supports remotes - any cloud (S3, Azure,
+Google Cloud, etc) or any on-premise network storage (via SSH, for example).
 
 .. image:: https://dvc.org/static/img/flow.gif
    :target: https://dvc.org/static/img/flow.gif
    :alt: how_dvc_works
 
+DVC pipelines (aka computational graph) feature connects code and data together. In a very explicit way you can
+specify, run, and save information that a certain command with certain dependencies needs to be run to produce
+a model. See the quick start section below or check `Get Started <https://dvc.org/doc/get-started>`_ tutorial to
+learn more.
+
 Quick start
 ===========
 
@@ -66,7 +87,7 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 +-----------------------------------+-------------------------------------------------------------------+
 | Step                              | Command                                                           |
 +===================================+===================================================================+
-| Track code and data together      | | ``$ git add train.py``                                          |
+| Track data                        | | ``$ git add train.py``                                          |
 |                                   | | ``$ dvc add images.zip``                                        |
 +-----------------------------------+-------------------------------------------------------------------+
 | Connect code and data by commands | | ``$ dvc run -d images.zip -o images/ unzip -q images.zip``      |
@@ -79,27 +100,30 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 |                                   | | ``$ git commit -m 'The baseline model'``                        |
 |                                   | | ``$ git push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
-| Share data and ML models          | | ``$ dvc remote add myremote s3://mybucket/image_cnn``           |
-|                                   | | ``$ dvc config core.remote myremote``                           |
+| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``        |
 |                                   | | ``$ dvc push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
 
 Installation
 ============
 
-There are three options to install DVC: ``pip``, Homebrew, or an OS-specific package:
+Read this `instruction <https://dvc.org/doc/get-started/install>`_ to get more details. There are three
+options to install DVC: ``pip``, Homebrew, or an OS-specific package:
 
 pip (PyPI)
 ----------
 
-Stable
-^^^^^^
 .. code-block:: bash
 
    pip install dvc
 
-Development
-^^^^^^^^^^^
+Depending on the remote storage type you plan to use to keep and share your data, you might need to specify
+one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``oss``, ``ssh``. Or ``all_remotes`` to include them all.
+The command should look like this: ``pip install dvc[s3]`` - it installs the ``boto3`` library along with
+DVC to support the AWS S3 storage.
+
+To install the development version, run:
+
 .. code-block:: bash
 
    pip install git+git://github.com/iterative/dvc
@@ -123,7 +147,7 @@ Package
 Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
 GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
 
-Ubuntu / Debian (apt)
+Ubuntu / Debian (deb)
 ^^^^^^^^^^^^^^^^^^^^^
 .. code-block:: bash
 
@@ -139,7 +163,7 @@ Fedora / CentOS (rpm)
    sudo yum update
    sudo yum install dvc
 
-Arch linux (AUR)
+Arch Linux (AUR)
 ^^^^^^^^^^^^^^^^
 *Unofficial package*, any inquiries regarding the AUR package,
 `refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
@@ -206,7 +230,7 @@ details.
 Mailing List
 ============
 
-Want to stay up to date? Want to help improve DVC by participating in our ocassional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic.
+Want to stay up to date? Want to help improve DVC by participating in our occasional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic.
 
 Copyright
 =========

dvc/__init__.py

@@ -6,70 +6,12 @@ Make your data science projects reproducible and shareable.
 
 from __future__ import unicode_literals
 
-import os
+from dvc.version import __version__  # noqa: F401
 import warnings
+import dvc.logger
 
 
-VERSION_BASE = "0.25.0"
-__version__ = VERSION_BASE
-
-PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
-HOMEPATH = os.path.dirname(PACKAGEPATH)
-VERSIONPATH = os.path.join(PACKAGEPATH, "version.py")
-
-
-def _update_version_file():
-    """Dynamically update version file."""
-    from git import Repo
-    from git.exc import InvalidGitRepositoryError
-
-    try:
-        repo = Repo(HOMEPATH)
-    except InvalidGitRepositoryError:
-        return __version__
-
-    sha = repo.head.object.hexsha
-    short_sha = repo.git.rev_parse(sha, short=6)
-    dirty = ".mod" if repo.is_dirty() else ""
-    ver = "{}+{}{}".format(__version__, short_sha, dirty)
-
-    # Write a helper file, that will be installed with the package
-    # and will provide a true version of the installed dvc
-    with open(VERSIONPATH, "w+") as fobj:
-        fobj.write("# AUTOGENERATED by dvc/__init__.py\n")
-        fobj.write('version = "{}"\n'.format(ver))
-
-    return ver
-
-
-def _remove_version_file():
-    """Remove version.py so that it doesn't get into the release."""
-    if os.path.exists(VERSIONPATH):
-        os.unlink(VERSIONPATH)
-
-
-if os.path.exists(os.path.join(HOMEPATH, "setup.py")):
-    # dvc is run directly from source without installation or
-    # __version__ is called from setup.py
-    if (
-        os.getenv("APPVEYOR_REPO_TAG", "").lower() != "true"
-        and os.getenv("TRAVIS_TAG", "") == ""
-    ):
-        __version__ = _update_version_file()
-    else:  # pragma: no cover
-        _remove_version_file()
-else:  # pragma: no cover
-    # dvc was installed with pip or something. Hopefully we have our
-    # auto-generated version.py to help us provide a true version
-    try:
-        from dvc.version import version
-
-        __version__ = version
-    except Exception:
-        pass
-
-VERSION = __version__
-
+dvc.logger.setup()
 
 # Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
 # We don't directly import numpy, but our dependency networkx does, causing
@@ -77,3 +19,18 @@ VERSION = __version__
 # we can simply ignore them so that they don't show up when you are using dvc.
 warnings.filterwarnings("ignore", message="numpy.dtype size changed")
 warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
+
+# Ignore paramiko's warning: https://github.com/paramiko/paramiko/issues/1386.
+# This only affects paramiko 2.4.2 and should be fixed in the next version.
+# Cryptography developers decided that it is a brilliant idea not to inherit
+# from DeprecationWarning [1] because it is invisible by default, and decided
+# to spam everyone instead. So it is their fault and not paramiko's.
+#
+# [1] https://github.com/pyca/cryptography/blob/2.6.1/src/cryptography/
+#     utils.py#L14
+try:
+    from cryptography.utils import CryptographyDeprecationWarning
+
+    warnings.simplefilter("ignore", CryptographyDeprecationWarning)
+except ImportError:
+    pass

dvc/analytics.py

@@ -7,9 +7,12 @@ from dvc.utils.compat import str
 import os
 import json
 import errno
+import logging
 
-import dvc.logger as logger
-from dvc import VERSION
+from dvc import __version__
+
+
+logger = logging.getLogger(__name__)
 
 
 class Analytics(object):
@@ -93,11 +96,12 @@ class Analytics(object):
         with open(self.user_id_file, "r") as fobj:
             try:
                 info = json.load(fobj)
-                return info[self.PARAM_USER_ID]
-            except json.JSONDecodeError as exc:
+            except ValueError as exc:
                 logger.debug("Failed to load user_id: {}".format(exc))
                 return None
 
+            return info[self.PARAM_USER_ID]
+
     def _get_user_id(self):
         from dvc.lock import LockError
 
@@ -161,19 +165,19 @@ class Analytics(object):
         """Collect analytics report."""
         from dvc.scm import SCM
         from dvc.utils import is_binary
-        from dvc.project import Project
-        from dvc.exceptions import NotDvcProjectError
+        from dvc.repo import Repo
+        from dvc.exceptions import NotDvcRepoError
 
-        self.info[self.PARAM_DVC_VERSION] = VERSION
+        self.info[self.PARAM_DVC_VERSION] = __version__
         self.info[self.PARAM_IS_BINARY] = is_binary()
         self.info[self.PARAM_USER_ID] = self._get_user_id()
 
         self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
 
         try:
-            scm = SCM(root_dir=Project.find_root())
+            scm = SCM(root_dir=Repo.find_root())
             self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
-        except NotDvcProjectError:
+        except NotDvcRepoError:
             pass
 
     def collect_cmd(self, args, ret):
@@ -201,11 +205,18 @@ class Analytics(object):
             json.dump(self.info, fobj)
             return fobj.name
 
+    @staticmethod
+    def _is_enabled_config(config):
+        from dvc.config import Config
+
+        core = config.config.get(Config.SECTION_CORE, {})
+        return core.get(Config.SECTION_CORE_ANALYTICS, True)
+
     @staticmethod
     def _is_enabled(cmd=None):
         from dvc.config import Config
-        from dvc.project import Project
-        from dvc.exceptions import NotDvcProjectError
+        from dvc.repo import Repo
+        from dvc.exceptions import NotDvcRepoError
         from dvc.command.daemon import CmdDaemonBase
 
         if os.getenv("DVC_TEST"):
@@ -216,20 +227,19 @@ class Analytics(object):
 
         if cmd is None or not hasattr(cmd, "config"):
             try:
-                dvc_dir = Project.find_dvc_dir()
+                dvc_dir = Repo.find_dvc_dir()
                 config = Config(dvc_dir)
                 assert config is not None
-            except NotDvcProjectError:
+            except NotDvcRepoError:
                 config = Config(validate=False)
                 assert config is not None
         else:
             config = cmd.config
             assert config is not None
 
-        core = config.config.get(Config.SECTION_CORE, {})
-        enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
+        enabled = Analytics._is_enabled_config(config)
         logger.debug(
-            "Analytics is {}abled.".format("en" if enabled else "dis")
+            "Analytics is {}.".format("enabled" if enabled else "disabled")
         )
         return enabled
 

dvc/cache.py

@@ -1,4 +1,4 @@
-"""Manages cache of a dvc project."""
+"""Manages cache of a dvc repo."""
 
 from __future__ import unicode_literals
 
@@ -8,86 +8,84 @@ from dvc.config import Config
 
 
 class Cache(object):
-    """Class that manages cache locations of a dvc project.
+    """Class that manages cache locations of a dvc repo.
 
     Args:
-        project (dvc.project.Project): project instance that this cache
-            belongs to.
+        repo (dvc.repo.Repo): repo instance that this cache belongs to.
     """
 
     CACHE_DIR = "cache"
 
-    def __init__(self, project):
+    def __init__(self, repo):
         from dvc.remote import Remote
 
-        self.project = project
+        self.repo = repo
 
-        config = project.config.config[Config.SECTION_CACHE]
+        config = repo.config.config[Config.SECTION_CACHE]
         local = config.get(Config.SECTION_CACHE_LOCAL)
 
         if local:
             name = Config.SECTION_REMOTE_FMT.format(local)
-            sect = project.config.config[name]
+            settings = repo.config.config[name]
         else:
-            default_cache_dir = os.path.join(project.dvc_dir, self.CACHE_DIR)
+            default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
             cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
             cache_type = config.get(Config.SECTION_CACHE_TYPE)
             protected = config.get(Config.SECTION_CACHE_PROTECTED)
 
-            sect = {
+            settings = {
                 Config.PRIVATE_CWD: config.get(
-                    Config.PRIVATE_CWD, project.dvc_dir
+                    Config.PRIVATE_CWD, repo.dvc_dir
                 ),
                 Config.SECTION_REMOTE_URL: cache_dir,
                 Config.SECTION_CACHE_TYPE: cache_type,
                 Config.SECTION_CACHE_PROTECTED: protected,
             }
 
-        self._local = Remote(project, sect)
-
-        self._s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
-        self._gs = self._get_remote(config, Config.SECTION_CACHE_GS)
-        self._ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
-        self._hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
-        self._azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
-
-    @property
-    def local(self):
-        """Remote instance for local cache."""
-        return self._local
-
-    @property
-    def s3(self):
-        """Remote instance for AWS S3 cache."""
-        return self._s3
-
-    @property
-    def gs(self):
-        """Remote instance for Google Cloud Storage cache."""
-        return self._gs
-
-    @property
-    def ssh(self):
-        """Remote instance for SSH cache."""
-        return self._ssh
-
-    @property
-    def hdfs(self):
-        """Remote instance for HDFS cache."""
-        return self._hdfs
-
-    @property
-    def azure(self):
-        """Remote instance for azure cache."""
-        return self._azure
+        self.local = Remote(repo, settings)
+        self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
+        self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
+        self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
+        self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
+        self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
 
     def _get_remote(self, config, name):
+        """
+        The config file is stored in a way that allows you to have a
+        cache for each remote.
+
+        This is needed when specifying external outputs
+        (as they require you to have an external cache location).
+
+        Imagine a config file like the following:
+
+                ['remote "dvc-storage"']
+                url = ssh://localhost/tmp
+                ask_password = true
+
+                [cache]
+                ssh = dvc-storage
+
+        This method resolves the name under the cache section into the
+        correct Remote instance.
+
+        Args:
+            config (dict): The cache section on the config file
+            name (str): Name of the section we are interested in to retrieve
+
+        Returns:
+            remote (dvc.Remote): Remote instance that the section is referring.
+                None when there's no remote with that name.
+
+        Example:
+            >>> _get_remote(config={'ssh': 'dvc-storage'}, name='ssh')
+        """
         from dvc.remote import Remote
 
-        remote = config.get(name, None)
+        remote = config.get(name)
+
         if not remote:
             return None
 
-        name = Config.SECTION_REMOTE_FMT.format(remote)
-        sect = self.project.config.config[name]
-        return Remote(self.project, sect)
+        settings = self.repo.config.get_remote_settings(remote)
+        return Remote(self.repo, settings)

dvc/cli.py

@@ -5,10 +5,11 @@ from __future__ import unicode_literals
 
 import sys
 import argparse
+import logging
 
-import dvc.logger as logger
 from dvc.command.base import fix_subparsers
 import dvc.command.init as init
+import dvc.command.pkg as pkg
 import dvc.command.destroy as destroy
 import dvc.command.remove as remove
 import dvc.command.move as move
@@ -29,12 +30,18 @@ import dvc.command.root as root
 import dvc.command.lock as lock
 import dvc.command.pipeline as pipeline
 import dvc.command.daemon as daemon
+import dvc.command.commit as commit
+import dvc.command.tag as tag
+import dvc.command.diff as diff
+import dvc.command.version as version
 from dvc.exceptions import DvcParserError
-from dvc import VERSION
 
 
+logger = logging.getLogger(__name__)
+
 COMMANDS = [
     init,
+    pkg,
     destroy,
     add,
     remove,
@@ -56,6 +63,10 @@ COMMANDS = [
     lock,
     pipeline,
     daemon,
+    commit,
+    tag,
+    diff,
+    version,
 ]
 
 
@@ -80,7 +91,9 @@ class VersionAction(argparse.Action):  # pragma: no cover
     """Shows dvc version and exits."""
 
     def __call__(self, parser, namespace, values, option_string=None):
-        print(VERSION)
+        from dvc import __version__
+
+        print(__version__)
         sys.exit(0)
 
 

dvc/command/add.py

@@ -1,25 +1,44 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdAdd(CmdBase):
     def run(self):
-        for target in self.args.targets:
-            try:
-                self.project.add(target, recursive=self.args.recursive)
-            except DvcException:
-                logger.error("failed to add file")
-                return 1
+        try:
+            if len(self.args.targets) > 1 and self.args.file:
+                raise DvcException("can't use '--file' with multiple targets")
+
+            for target in self.args.targets:
+                self.repo.add(
+                    target,
+                    recursive=self.args.recursive,
+                    no_commit=self.args.no_commit,
+                    fname=self.args.file,
+                )
+
+        except DvcException:
+            logger.exception("failed to add file")
+            return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    ADD_HELP = "Add files/directories to dvc."
+    ADD_HELP = "Take data files or directories under DVC control."
+
     add_parser = subparsers.add_parser(
-        "add", parents=[parent_parser], description=ADD_HELP, help=ADD_HELP
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(ADD_HELP, "add"),
+        help=ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     add_parser.add_argument(
         "-R",
@@ -28,6 +47,15 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Recursively add each file under the directory.",
     )
+    add_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    add_parser.add_argument(
+        "-f", "--file", help="Specify name of the DVC file it generates."
+    )
     add_parser.add_argument(
         "targets", nargs="+", help="Input files/directories."
     )

dvc/command/base.py

@@ -1,6 +1,10 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import colorama
+import logging
+
+
+logger = logging.getLogger(__name__)
 
 
 def fix_subparsers(subparsers):
@@ -18,14 +22,26 @@ def fix_subparsers(subparsers):
         subparsers.dest = "cmd"
 
 
+def append_doc_link(help_message, path):
+    if not path:
+        return help_message
+    doc_base = "https://man.dvc.org/"
+    return "{message}\ndocumentation: {blue}{base}{path}{nc}".format(
+        message=help_message,
+        base=doc_base,
+        path=path,
+        blue=colorama.Fore.CYAN,
+        nc=colorama.Fore.RESET,
+    )
+
+
 class CmdBase(object):
     def __init__(self, args):
-        from dvc.project import Project
+        from dvc.repo import Repo
 
-        self.project = Project()
-        self.config = self.project.config
+        self.repo = Repo()
+        self.config = self.repo.config
         self.args = args
-        self.set_loglevel(args)
 
     @property
     def default_targets(self):
@@ -36,25 +52,24 @@ class CmdBase(object):
         logger.warning(msg)
         return [Stage.STAGE_FILE]
 
-    @staticmethod
-    def set_loglevel(args):
-        """Sets log level from CLI arguments."""
-        if args.quiet:
-            logger.be_quiet()
-
-        elif args.verbose:
-            logger.be_verbose()
-
     def run_cmd(self):
         from dvc.lock import LockError
 
         try:
-            with self.project.lock:
+            with self.repo.lock:
                 return self.run()
         except LockError:
-            logger.error("failed to lock before running a command")
+            logger.exception("failed to lock before running a command")
             return 1
 
     # Abstract methods that have to be implemented by any inheritance class
     def run(self):
         pass
+
+
+class CmdBaseNoRepo(CmdBase):
+    def __init__(self, args):
+        self.args = args
+
+    def run_cmd(self):
+        return self.run()

dvc/command/cache.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import argparse
 
-from dvc.command.base import fix_subparsers
+from dvc.command.base import fix_subparsers, append_doc_link
 from dvc.command.remote import CmdRemoteAdd
 from dvc.command.config import CmdConfig
 
@@ -21,11 +21,13 @@ def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
     CACHE_HELP = "Manage cache settings."
+
     cache_parser = subparsers.add_parser(
         "cache",
         parents=[parent_parser],
-        description=CACHE_HELP,
+        description=append_doc_link(CACHE_HELP, "cache"),
         help=CACHE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     cache_subparsers = cache_parser.add_subparsers(
@@ -38,11 +40,13 @@ def add_parser(subparsers, parent_parser):
         add_help=False, parents=[parent_config_parser]
     )
     CACHE_DIR_HELP = "Configure cache directory location."
+
     cache_dir_parser = cache_subparsers.add_parser(
         "dir",
         parents=[parent_cache_config_parser],
-        description=CACHE_DIR_HELP,
+        description=append_doc_link(CACHE_HELP, "cache-dir"),
         help=CACHE_DIR_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     cache_dir_parser.add_argument(
         "-u",
@@ -53,7 +57,6 @@ def add_parser(subparsers, parent_parser):
     )
     cache_dir_parser.add_argument(
         "value",
-        nargs="?",
         default=None,
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "

dvc/command/checkout.py

@@ -1,29 +1,34 @@
 from __future__ import unicode_literals
 
-from dvc.command.base import CmdBase
+import argparse
+
+from dvc.command.base import CmdBase, append_doc_link
 
 
 class CmdCheckout(CmdBase):
     def run(self):
         if not self.args.targets:
-            self.project.checkout(force=self.args.force)
+            self.repo.checkout(force=self.args.force)
         else:
             for target in self.args.targets:
-                self.project.checkout(
+                self.repo.checkout(
                     target=target,
                     with_deps=self.args.with_deps,
                     force=self.args.force,
+                    recursive=self.args.recursive,
                 )
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     CHECKOUT_HELP = "Checkout data files from cache."
+
     checkout_parser = subparsers.add_parser(
         "checkout",
         parents=[parent_parser],
-        description=CHECKOUT_HELP,
+        description=append_doc_link(CHECKOUT_HELP, "checkout"),
         help=CHECKOUT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     checkout_parser.add_argument(
         "-d",
@@ -39,5 +44,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Do not prompt when removing working directory files.",
     )
+    checkout_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Checkout all subdirectories of the specified directory.",
+    )
     checkout_parser.add_argument("targets", nargs="*", help="DVC files.")
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -0,0 +1,70 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdCommit(CmdBase):
+    def run(self):
+        if not self.args.targets:
+            self.args.targets = [None]
+
+        for target in self.args.targets:
+            try:
+                self.repo.commit(
+                    target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                    force=self.args.force,
+                )
+            except DvcException:
+                logger.exception(
+                    "failed to commit{}".format(
+                        (" " + target) if target else ""
+                    )
+                )
+                return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMMIT_HELP = "Save changed data to cache and update DVC files."
+
+    commit_parser = subparsers.add_parser(
+        "commit",
+        parents=[parent_parser],
+        description=append_doc_link(COMMIT_HELP, "commit"),
+        help=COMMIT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    commit_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Commit even if checksums for dependencies/outputs changed.",
+    )
+    commit_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Commit all dependencies of the specified target.",
+    )
+    commit_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Commit cache for subdirectories of the specified directory.",
+    )
+    commit_parser.add_argument(
+        "targets", nargs="*", default=None, help="DVC files."
+    )
+    commit_parser.set_defaults(func=CmdCommit)

dvc/command/config.py

@@ -2,23 +2,26 @@ from __future__ import unicode_literals
 
 import os
 import argparse
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.config import Config
 from dvc.exceptions import DvcException
 
 
-class CmdConfig(CmdBase):
-    def __init__(self, args):
-        from dvc.project import Project, NotDvcProjectError
+logger = logging.getLogger(__name__)
 
-        self.args = args
+
+class CmdConfig(CmdBaseNoRepo):
+    def __init__(self, args):
+        from dvc.repo import Repo, NotDvcRepoError
+
+        super(CmdConfig, self).__init__(args)
 
         try:
-            dvc_dir = os.path.join(Project.find_root(), Project.DVC_DIR)
+            dvc_dir = os.path.join(Repo.find_root(), Repo.DVC_DIR)
             saved_exc = None
-        except NotDvcProjectError as exc:
+        except NotDvcRepoError as exc:
             dvc_dir = None
             saved_exc = exc
 
@@ -34,10 +37,7 @@ class CmdConfig(CmdBase):
         else:
             if dvc_dir is None:
                 raise saved_exc
-            self.configobj = self.config._project_config
-
-    def run_cmd(self):
-        return self.run()
+            self.configobj = self.config._repo_config
 
     def _unset(self, section, opt=None, configobj=None):
         if configobj is None:
@@ -47,7 +47,7 @@ class CmdConfig(CmdBase):
             self.config.unset(configobj, section, opt)
             self.config.save(configobj)
         except DvcException:
-            logger.error("failed to unset '{}'".format(self.args.name))
+            logger.exception("failed to unset '{}'".format(self.args.name))
             return 1
         return 0
 
@@ -55,7 +55,7 @@ class CmdConfig(CmdBase):
         try:
             self.config.show(self.configobj, section, opt)
         except DvcException:
-            logger.error("failed to show '{}'".format(self.args.name))
+            logger.exception("failed to show '{}'".format(self.args.name))
             return 1
         return 0
 
@@ -64,7 +64,7 @@ class CmdConfig(CmdBase):
             self.config.set(self.configobj, section, opt, value)
             self.config.save(self.configobj)
         except DvcException:
-            logger.error(
+            logger.exception(
                 "failed to set '{}.{}' to '{}'".format(section, opt, value)
             )
             return 1
@@ -99,11 +99,13 @@ parent_config_parser.add_argument(
 
 def add_parser(subparsers, parent_parser):
     CONFIG_HELP = "Get or set config options."
+
     config_parser = subparsers.add_parser(
         "config",
         parents=[parent_config_parser, parent_parser],
-        description=CONFIG_HELP,
+        description=append_doc_link(CONFIG_HELP, "config"),
         help=CONFIG_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     config_parser.add_argument(
         "-u",

dvc/command/daemon.py

@@ -1,26 +1,20 @@
 from __future__ import unicode_literals
 
-from dvc.command.base import CmdBase, fix_subparsers
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
 
 
-class CmdDaemonBase(CmdBase):
-    def __init__(self, args):
-        self.args = args
-        self.config = None
-        self.set_loglevel(args)
-
-    def run_cmd(self):
-        return self.run()
+class CmdDaemonBase(CmdBaseNoRepo):
+    pass
 
 
 class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
-        from dvc.project import Project
+        from dvc.repo import Repo
         from dvc.updater import Updater
 
-        root_dir = Project.find_root()
-        dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
+        root_dir = Repo.find_root()
+        dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
         updater = Updater(dvc_dir)
         updater.fetch(detach=False)
 
@@ -43,7 +37,7 @@ def add_parser(subparsers, parent_parser):
         "daemon",
         parents=[parent_parser],
         description=DAEMON_HELP,
-        help=DAEMON_HELP,
+        add_help=False,
     )
 
     daemon_subparsers = daemon_parser.add_subparsers(

dvc/command/data_sync.py

@@ -1,12 +1,17 @@
 from __future__ import unicode_literals
 
 import argparse
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
+    UP_TO_DATE_MSG = "Everything is up to date."
+
     def do_run(self, target):
         pass
 
@@ -20,11 +25,16 @@ class CmdDataBase(CmdBase):
                 ret = 1
         return ret
 
+    @classmethod
+    def check_up_to_date(cls, processed_files_count):
+        if processed_files_count == 0:
+            logger.info(cls.UP_TO_DATE_MSG)
+
 
 class CmdDataPull(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.pull(
+            processed_files_count = self.repo.pull(
                 target=target,
                 jobs=self.args.jobs,
                 remote=self.args.remote,
@@ -36,15 +46,16 @@ class CmdDataPull(CmdDataBase):
                 recursive=self.args.recursive,
             )
         except Exception:
-            logger.error("failed to pull data from the cloud")
+            logger.exception("failed to pull data from the cloud")
             return 1
+        self.check_up_to_date(processed_files_count)
         return 0
 
 
 class CmdDataPush(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.push(
+            processed_files_count = self.repo.push(
                 target=target,
                 jobs=self.args.jobs,
                 remote=self.args.remote,
@@ -55,15 +66,16 @@ class CmdDataPush(CmdDataBase):
                 recursive=self.args.recursive,
             )
         except Exception:
-            logger.error("failed to push data to the cloud")
+            logger.exception("failed to push data to the cloud")
             return 1
+        self.check_up_to_date(processed_files_count)
         return 0
 
 
 class CmdDataFetch(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.fetch(
+            processed_files_count = self.repo.fetch(
                 target=target,
                 jobs=self.args.jobs,
                 remote=self.args.remote,
@@ -74,8 +86,9 @@ class CmdDataFetch(CmdDataBase):
                 recursive=self.args.recursive,
             )
         except Exception:
-            logger.error("failed to fetch data from the cloud")
+            logger.exception("failed to fetch data from the cloud")
             return 1
+        self.check_up_to_date(processed_files_count)
         return 0
 
 
@@ -110,12 +123,14 @@ def add_parser(subparsers, _parent_parser):
     from dvc.command.status import CmdDataStatus
 
     # Pull
-    PULL_HELP = "Pull data files from the cloud."
+    PULL_HELP = "Pull data files from a DVC remote storage."
+
     pull_parser = subparsers.add_parser(
         "pull",
         parents=[shared_parent_parser()],
-        description=PULL_HELP,
+        description=append_doc_link(PULL_HELP, "pull"),
         help=PULL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
         "-r", "--remote", help="Remote repository to pull from."
@@ -158,12 +173,14 @@ def add_parser(subparsers, _parent_parser):
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
-    PUSH_HELP = "Push data files to the cloud."
+    PUSH_HELP = "Push data files to a DVC remote storage."
+
     push_parser = subparsers.add_parser(
         "push",
         parents=[shared_parent_parser()],
-        description=PUSH_HELP,
+        description=append_doc_link(PUSH_HELP, "push"),
         help=PUSH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
         "-r", "--remote", help="Remote repository to push to."
@@ -199,12 +216,14 @@ def add_parser(subparsers, _parent_parser):
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
-    FETCH_HELP = "Fetch data files from the cloud."
+    FETCH_HELP = "Fetch data files from a DVC remote storage."
+
     fetch_parser = subparsers.add_parser(
         "fetch",
         parents=[shared_parent_parser()],
-        description=FETCH_HELP,
+        description=append_doc_link(FETCH_HELP, "fetch"),
         help=FETCH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
         "-r", "--remote", help="Remote repository to fetch from."
@@ -240,13 +259,17 @@ def add_parser(subparsers, _parent_parser):
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
-    STATUS_HELP = "Show the project status."
+    STATUS_HELP = (
+        "Show changed stages, compare local cache and a remote storage."
+    )
+
     status_parser = subparsers.add_parser(
         "status",
         parents=[shared_parent_parser()],
-        description=STATUS_HELP,
+        description=append_doc_link(STATUS_HELP, "status"),
         help=STATUS_HELP,
         conflict_handler="resolve",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     status_parser.add_argument(
         "-q",

dvc/command/destroy.py

@@ -1,11 +1,16 @@
 from __future__ import unicode_literals
 
+import argparse
+import logging
+
 import dvc.prompt as prompt
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdDestroy(CmdBase):
     def run_cmd(self):
         try:
@@ -22,23 +27,22 @@ class CmdDestroy(CmdBase):
                     " Use '-f' to force."
                 )
 
-            self.project.destroy()
+            self.repo.destroy()
         except Exception:
-            logger.error("failed to destroy DVC")
+            logger.exception("failed to destroy DVC")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    DESTROY_HELP = (
-        "Destroy dvc. Will remove all project's information, "
-        "data files and cache."
-    )
+    DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
+
     destroy_parser = subparsers.add_parser(
         "destroy",
         parents=[parent_parser],
-        description=DESTROY_HELP,
+        description=append_doc_link(DESTROY_HELP, "destroy"),
         help=DESTROY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     destroy_parser.add_argument(
         "-f",

dvc/command/diff.py

@@ -0,0 +1,160 @@
+from __future__ import unicode_literals
+
+import argparse
+
+import humanize
+import inflect
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.utils.collections import compact
+import dvc.repo.diff as diff
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdDiff(CmdBase):
+    def _print_size(self, size):
+        if size < 0:
+            change = "decreased by {}"
+        elif size > 0:
+            change = "increased by {}"
+        else:
+            change = "not changed"
+        natur_size = humanize.naturalsize(abs(size))
+        return change.format(natur_size)
+
+    def _get_md5_string(self, sign, file_name, checksum):
+        sample_msg = ""
+        if file_name:
+            sample_msg = "{}{} with md5 {}\n"
+            sample_msg = sample_msg.format(sign, file_name, checksum)
+        return sample_msg
+
+    def _get_dir_changes(self, dct):
+        engine = inflect.engine()
+        changes_msg = (
+            "{} {} not changed, {} {} modified, {} {} added, "
+            "{} {} deleted, size was {}"
+        )
+        changes_msg = changes_msg.format(
+            dct[diff.DIFF_IDENT],
+            engine.plural("file", dct[diff.DIFF_IDENT]),
+            dct[diff.DIFF_CHANGE],
+            engine.plural("file", dct[diff.DIFF_CHANGE]),
+            dct[diff.DIFF_NEW],
+            engine.plural("file", dct[diff.DIFF_NEW]),
+            dct[diff.DIFF_DEL],
+            engine.plural("file", dct[diff.DIFF_DEL]),
+            self._print_size(dct[diff.DIFF_SIZE]),
+        )
+        return changes_msg
+
+    def _get_file_changes(self, dct):
+        if (
+            dct.get(diff.DIFF_OLD_FILE)
+            and dct.get(diff.DIFF_NEW_FILE)
+            and dct[diff.DIFF_SIZE] == 0
+        ):
+            msg = "file size was not changed"
+        elif dct.get(diff.DIFF_NEW_FILE):
+            msg = "added file with size {}".format(
+                humanize.naturalsize(dct[diff.DIFF_SIZE])
+            )
+        elif dct.get(diff.DIFF_OLD_FILE):
+            msg = "deleted file with size {}".format(
+                humanize.naturalsize(abs(dct[diff.DIFF_SIZE]))
+            )
+        else:
+            msg = "file was modified, file size {}".format(
+                self._print_size(dct[diff.DIFF_SIZE])
+            )
+        return msg
+
+    def _get_royal_changes(self, dct):
+        if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
+            if dct.get("is_dir"):
+                return self._get_dir_changes(dct)
+            else:
+                return self._get_file_changes(dct)
+        return "size is ?"
+
+    def _show(self, diff_dct):
+        msg = "dvc diff from {} to {}".format(
+            diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
+        )
+        if diff_dct.get(diff.DIFF_EQUAL):
+            logger.info(msg)
+            return
+        for dct in diff_dct[diff.DIFF_LIST]:
+            msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET])
+            msg += self._get_md5_string(
+                "-",
+                dct.get(diff.DIFF_OLD_FILE),
+                dct.get(diff.DIFF_OLD_CHECKSUM),
+            )
+            msg += self._get_md5_string(
+                "+",
+                dct.get(diff.DIFF_NEW_FILE),
+                dct.get(diff.DIFF_NEW_CHECKSUM),
+            )
+            msg += "\n"
+            msg += self._get_royal_changes(dct)
+        logger.info(msg)
+        return msg
+
+    def run(self):
+        try:
+            msg = self.repo.diff(
+                self.args.a_ref, target=self.args.target, b_ref=self.args.b_ref
+            )
+            self._show(msg)
+        except DvcException:
+            msg = "failed to get 'diff {}'"
+            args = " ".join(
+                compact([self.args.target, self.args.a_ref, self.args.b_ref])
+            )
+            msg = msg.format(args)
+            logger.exception(msg)
+            return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    DIFF_DESCRIPTION = (
+        "Show diff of a data file or a directory that is under DVC control.\n"
+        "Some basic statistics summary, how many files were deleted/changed."
+    )
+    DIFF_HELP = "Show a diff of a DVC controlled data file or a directory."
+    diff_parser = subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(DIFF_DESCRIPTION, "diff"),
+        help=DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    diff_parser.add_argument(
+        "-t",
+        "--target",
+        default=None,
+        help=(
+            "Source path to a data file or directory. Default None. "
+            "If not specified, compares all files and directories "
+            "that are under DVC control in the current working space."
+        ),
+    )
+    diff_parser.add_argument(
+        "a_ref", help="Git reference from which diff calculates"
+    )
+    diff_parser.add_argument(
+        "b_ref",
+        help=(
+            "Git reference untill which diff calculates, if omitted "
+            "diff shows the difference between current HEAD and a_ref"
+        ),
+        nargs="?",
+        default=None,
+    )
+    diff_parser.set_defaults(func=CmdDiff)

dvc/command/gc.py

@@ -1,10 +1,14 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
 import dvc.prompt as prompt
-import dvc.logger as logger
+import logging
 
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdGC(CmdBase):
@@ -19,13 +23,13 @@ class CmdGC(CmdBase):
         else:
             msg += "all git branches and all git tags"
 
-        if self.args.projects is not None and len(self.args.projects) > 0:
-            msg += " of the current and the following projects:"
+        if self.args.repos is not None and len(self.args.repos) > 0:
+            msg += " of the current and the following repos:"
 
-            for project_path in self.args.projects:
-                msg += "\n  - %s" % os.path.abspath(project_path)
+            for repo_path in self.args.repos:
+                msg += "\n  - %s" % os.path.abspath(repo_path)
         else:
-            msg += " of the current project."
+            msg += " of the current repo."
 
         logger.warning(msg)
 
@@ -33,36 +37,44 @@ class CmdGC(CmdBase):
         if not self.args.force and not prompt.confirm(msg):
             return 1
 
-        self.project.gc(
+        self.repo.gc(
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
             cloud=self.args.cloud,
             remote=self.args.remote,
             force=self.args.force,
             jobs=self.args.jobs,
-            projects=self.args.projects,
+            repos=self.args.repos,
         )
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    GC_HELP = "Collect garbage."
+    GC_HELP = "Collect unused data from DVC cache or a remote storage."
+    GC_DESCRIPTION = (
+        "Deletes all files in the cache or a remote which are not in\n"
+        "use by the specified git references (defaults to just HEAD)."
+    )
     gc_parser = subparsers.add_parser(
-        "gc", parents=[parent_parser], description=GC_HELP, help=GC_HELP
+        "gc",
+        parents=[parent_parser],
+        description=append_doc_link(GC_DESCRIPTION, "gc"),
+        help=GC_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     gc_parser.add_argument(
         "-a",
         "--all-branches",
         action="store_true",
         default=False,
-        help="Collect garbage for all branches.",
+        help="Keep data files for the tips of all git branches.",
     )
     gc_parser.add_argument(
         "-T",
         "--all-tags",
         action="store_true",
         default=False,
-        help="Collect garbage for all tags.",
+        help="Keep data files for all git tags.",
     )
     gc_parser.add_argument(
         "-c",
@@ -72,14 +84,14 @@ def add_parser(subparsers, parent_parser):
         help="Collect garbage in remote repository.",
     )
     gc_parser.add_argument(
-        "-r", "--remote", help="Remote repository to collect garbage in."
+        "-r", "--remote", help="Remote storage to collect garbage in."
     )
     gc_parser.add_argument(
         "-f",
         "--force",
         action="store_true",
         default=False,
-        help="Force garbage collection.",
+        help="Force garbage collection - automatically agree to all prompts.",
     )
     gc_parser.add_argument(
         "-j",
@@ -91,9 +103,12 @@ def add_parser(subparsers, parent_parser):
     gc_parser.add_argument(
         "-p",
         "--projects",
+        dest="repos",
         type=str,
         nargs="*",
         default=None,
-        help="Collect garbage for all given projects.",
+        help="Keep data files required by these projects "
+        "in addition to the current one. "
+        "Useful if you share a single cache across repos.",
     )
     gc_parser.set_defaults(func=CmdGC)

dvc/command/imp.py

@@ -1,11 +1,15 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
+import logging
 
-import dvc.logger as logger
 from dvc.utils.compat import urlparse
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdImport(CmdBase):
@@ -15,9 +19,11 @@ class CmdImport(CmdBase):
 
             out = self.args.out or default_out
 
-            self.project.imp(self.args.url, out, self.args.resume)
+            self.repo.imp(
+                self.args.url, out, self.args.resume, fname=self.args.file
+            )
         except DvcException:
-            logger.error(
+            logger.exception(
                 "failed to import {}. You could also try downloading "
                 "it manually and adding it with `dvc add` command.".format(
                     self.args.url
@@ -28,25 +34,26 @@ class CmdImport(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    IMPORT_HELP = "Import files from URL."
+    IMPORT_HELP = "Download or copy files from URL and take under DVC control."
+
     import_parser = subparsers.add_parser(
         "import",
         parents=[parent_parser],
-        description=IMPORT_HELP,
+        description=append_doc_link(IMPORT_HELP, "import"),
         help=IMPORT_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     import_parser.add_argument(
         "url",
-        help="URL. Supported urls: "
-        "'/path/to/file', "
-        "'C:\\\\path\\to\\file', "
-        "'https://example.com/path/to/file', "
-        "'s3://bucket/path/to/file', "
-        "'gs://bucket/path/to/file', "
-        "'hdfs://example.com/path/to/file', "
-        "'ssh://example.com:/path/to/file', "
-        "'remote://myremote/path/to/file'(see "
-        "`dvc remote` commands). ",
+        help="Supported urls:\n"
+        "/path/to/file\n"
+        "C:\\\\path\\to\\file\n"
+        "https://example.com/path/to/file\n"
+        "s3://bucket/path/to/file\n"
+        "gs://bucket/path/to/file\n"
+        "hdfs://example.com/path/to/file\n"
+        "ssh://example.com:/path/to/file\n"
+        "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(
         "--resume",
@@ -54,5 +61,10 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Resume previously started download.",
     )
-    import_parser.add_argument("out", nargs="?", help="Output.")
+    import_parser.add_argument(
+        "out", nargs="?", help="Destination path to put files to."
+    )
+    import_parser.add_argument(
+        "-f", "--file", help="Specify name of the DVC file it generates."
+    )
     import_parser.set_defaults(func=CmdImport)

dvc/command/init.py

@@ -1,33 +1,44 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 
-class CmdInit(object):
-    def __init__(self, args):
-        self.args = args
+logger = logging.getLogger(__name__)
 
-    def run_cmd(self):
-        from dvc.project import Project, InitError
+
+class CmdInit(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+        from dvc.exceptions import InitError
 
         try:
-            self.project = Project.init(
+            self.repo = Repo.init(
                 ".", no_scm=self.args.no_scm, force=self.args.force
             )
-            self.config = self.project.config
+            self.config = self.repo.config
         except InitError:
-            logger.error("failed to initiate dvc")
+            logger.exception("failed to initiate dvc")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     """Setup parser for `dvc init`."""
-    INIT_HELP = (
-        "Initialize dvc over a directory " "(should already be a git dir)."
+    INIT_HELP = "Initialize DVC in the current directory."
+    INIT_DESCRIPTION = (
+        "Initialize DVC in the current directory. Expects directory\n"
+        "to be a Git repository unless --no-scm option is specified."
     )
+
     init_parser = subparsers.add_parser(
-        "init", parents=[parent_parser], description=INIT_HELP, help=INIT_HELP
+        "init",
+        parents=[parent_parser],
+        description=append_doc_link(INIT_DESCRIPTION, "init"),
+        help=INIT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     init_parser.add_argument(
         "--no-scm",
@@ -41,6 +52,9 @@ def add_parser(subparsers, parent_parser):
         "--force",
         action="store_true",
         default=False,
-        help="Overwrite '.dvc' if it exists. Will remove all local cache.",
+        help=(
+            "Overwrite existing '.dvc' directory. "
+            "This operation removes local cache."
+        ),
     )
     init_parser.set_defaults(func=CmdInit)

dvc/command/install.py

@@ -1,25 +1,31 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdInstall(CmdBase):
     def run_cmd(self):
         try:
-            self.project.install()
+            self.repo.install()
         except Exception:
-            logger.error("failed to install dvc hooks")
+            logger.exception("failed to install dvc hooks")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    INSTALL_HELP = "Install dvc hooks into the repository."
+    INSTALL_HELP = "Install DVC git hooks into the repository."
     install_parser = subparsers.add_parser(
         "install",
         parents=[parent_parser],
-        description=INSTALL_HELP,
+        description=append_doc_link(INSTALL_HELP, "install"),
         help=INSTALL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     install_parser.set_defaults(func=CmdInstall)

dvc/command/lock.py

@@ -1,17 +1,22 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdLockBase(CmdBase):
-    def run(self, unlock=False):
+    def _run(self, unlock):
         for target in self.args.targets:
             try:
-                self.project.lock_stage(target, unlock=unlock)
+                self.repo.lock_stage(target, unlock=unlock)
             except DvcException:
-                logger.error(
+                logger.exception(
                     "failed to {}lock '{}'".format(
                         "un" if unlock else "", target
                     )
@@ -23,18 +28,22 @@ class CmdLockBase(CmdBase):
 
 class CmdLock(CmdLockBase):
     def run(self):
-        return super(CmdLock, self).run(False)
+        return self._run(False)
 
 
 class CmdUnlock(CmdLockBase):
     def run(self):
-        return super(CmdUnlock, self).run(True)
+        return self._run(True)
 
 
 def add_parser(subparsers, parent_parser):
     LOCK_HELP = "Lock DVC file."
     lock_parser = subparsers.add_parser(
-        "lock", parents=[parent_parser], description=LOCK_HELP, help=LOCK_HELP
+        "lock",
+        parents=[parent_parser],
+        description=append_doc_link(LOCK_HELP, "lock"),
+        help=LOCK_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     lock_parser.add_argument("targets", nargs="+", help="DVC files.")
     lock_parser.set_defaults(func=CmdLock)
@@ -43,8 +52,9 @@ def add_parser(subparsers, parent_parser):
     unlock_parser = subparsers.add_parser(
         "unlock",
         parents=[parent_parser],
-        description=UNLOCK_HELP,
+        description=append_doc_link(UNLOCK_HELP, "unlock"),
         help=UNLOCK_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unlock_parser.add_argument("targets", nargs="+", help="DVC files.")
     unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/metrics.py

@@ -1,34 +1,44 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+def show_metrics(metrics, all_branches=False, all_tags=False):
+    """
+    Args:
+        metrics (list): Where each element is either a `list`
+            if an xpath was specified, otherwise a `str`
+    """
+    for branch, val in metrics.items():
+        if all_branches or all_tags:
+            logger.info("{branch}:".format(branch=branch))
+
+        for fname, metric in val.items():
+            lines = metric if type(metric) is list else metric.splitlines()
+
+            if len(lines) > 1:
+                logger.info("\t{fname}:".format(fname=fname))
+
+                for line in lines:
+                    logger.info("\t\t{content}".format(content=line))
+
+            else:
+                logger.info("\t{}: {}".format(fname, metric))
 
 
 class CmdMetricsShow(CmdBase):
     def run(self):
+        typ = self.args.type
+        xpath = self.args.xpath
         try:
-            # backward compatibility
-            if self.args.json_path:
-                typ = "json"
-                xpath = self.args.json_path
-            elif self.args.tsv_path:
-                typ = "tsv"
-                xpath = self.args.tsv_path
-            elif self.args.htsv_path:
-                typ = "htsv"
-                xpath = self.args.htsv_path
-            elif self.args.csv_path:
-                typ = "csv"
-                xpath = self.args.csv_path
-            elif self.args.hcsv_path:
-                typ = "hcsv"
-                xpath = self.args.hcsv_path
-            else:
-                typ = self.args.type
-                xpath = self.args.xpath
-
-            self.project.metrics_show(
+            metrics = self.repo.metrics.show(
                 self.args.path,
                 typ=typ,
                 xpath=xpath,
@@ -36,8 +46,10 @@ class CmdMetricsShow(CmdBase):
                 all_tags=self.args.all_tags,
                 recursive=self.args.recursive,
             )
+
+            show_metrics(metrics, self.args.all_branches, self.args.all_tags)
         except DvcException:
-            logger.error("failed to show metrics")
+            logger.exception("failed to show metrics")
             return 1
 
         return 0
@@ -46,11 +58,11 @@ class CmdMetricsShow(CmdBase):
 class CmdMetricsModify(CmdBase):
     def run(self):
         try:
-            self.project.metrics_modify(
+            self.repo.metrics.modify(
                 self.args.path, typ=self.args.type, xpath=self.args.xpath
             )
         except DvcException:
-            logger.error("failed to modify metrics")
+            logger.exception("failed to modify metric file settings")
             return 1
 
         return 0
@@ -59,12 +71,12 @@ class CmdMetricsModify(CmdBase):
 class CmdMetricsAdd(CmdBase):
     def run(self):
         try:
-            self.project.metrics_add(
+            self.repo.metrics.add(
                 self.args.path, self.args.type, self.args.xpath
             )
         except DvcException:
             msg = "failed to add metric file '{}'".format(self.args.path)
-            logger.error(msg)
+            logger.exception(msg)
             return 1
 
         return 0
@@ -73,60 +85,55 @@ class CmdMetricsAdd(CmdBase):
 class CmdMetricsRemove(CmdBase):
     def run(self):
         try:
-            self.project.metrics_remove(self.args.path)
+            self.repo.metrics.remove(self.args.path)
         except DvcException:
             msg = "failed to remove metric file '{}'".format(self.args.path)
-            logger.error(msg)
+            logger.exception(msg)
             return 1
 
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = "Get metrics from all branches."
+    METRICS_HELP = "Commands to add, manage, collect and display metrics."
+
     metrics_parser = subparsers.add_parser(
         "metrics",
         parents=[parent_parser],
-        description=METRICS_HELP,
+        description=append_doc_link(METRICS_HELP, "metrics"),
         help=METRICS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     metrics_subparsers = metrics_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc metrics CMD --help for command-specific help.",
+        help="Use dvc metrics CMD --help to display command-specific help.",
     )
 
     fix_subparsers(metrics_subparsers)
 
-    METRICS_SHOW_HELP = "Show metrics."
+    METRICS_SHOW_HELP = "Output metric values."
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
         parents=[parent_parser],
-        description=METRICS_SHOW_HELP,
+        description=append_doc_link(METRICS_SHOW_HELP, "metrics-show"),
         help=METRICS_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_show_parser.add_argument(
-        "path", nargs="?", help="Path to metrics file or directory"
+        "path", nargs="?", help="Path to a metric file or a directory."
     )
     metrics_show_parser.add_argument(
-        "-t", "--type", help="Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)."
+        "-t",
+        "--type",
+        help=(
+            "Type of metrics (json/tsv/htsv/csv/hcsv). "
+            "It can be detected by the file extension automatically. "
+            "Unsupported types will be treated as raw."
+        ),
     )
     metrics_show_parser.add_argument(
-        "-x", "--xpath", help="JSON/TSV/HTSV/CSV/HCSV path."
-    )
-    metrics_show_group = metrics_show_parser.add_mutually_exclusive_group()
-    metrics_show_group.add_argument("--json-path", help="JSON path.")
-    metrics_show_group.add_argument(
-        "--tsv-path", help="TSV path 'row,column' (e.g. '1,2')."
-    )
-    metrics_show_group.add_argument(
-        "--htsv-path", help="Headed TSV path 'row,column (e.g. 'Name,3')."
-    )
-    metrics_show_group.add_argument(
-        "--csv-path", help="CSV path 'row,column' (e.g. '1,2')."
-    )
-    metrics_show_group.add_argument(
-        "--hcsv-path", help="Headed CSV path 'row,column' (e.g. 'Name,3')."
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
     )
     metrics_show_parser.add_argument(
         "-a",
@@ -147,48 +154,54 @@ def add_parser(subparsers, parent_parser):
         "--recursive",
         action="store_true",
         default=False,
-        help="If path is a directory, recursively search and process metric files in path.",
+        help=(
+            "If path is a directory, recursively search and process metric "
+            "files in path."
+        ),
     )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
-    METRICS_ADD_HELP = "Add metrics."
+    METRICS_ADD_HELP = "Tag file as a metric file."
     metrics_add_parser = metrics_subparsers.add_parser(
         "add",
         parents=[parent_parser],
-        description=METRICS_ADD_HELP,
+        description=append_doc_link(METRICS_ADD_HELP, "metrics-add"),
         help=METRICS_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_add_parser.add_argument(
-        "-t", "--type", help="Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)."
+        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
     )
     metrics_add_parser.add_argument(
-        "-x", "--xpath", help="JSON/TSV/HTSV/CSV/HCSV path."
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
     )
-    metrics_add_parser.add_argument("path", help="Path to metrics file.")
+    metrics_add_parser.add_argument("path", help="Path to a metric file.")
     metrics_add_parser.set_defaults(func=CmdMetricsAdd)
 
-    METRICS_MODIFY_HELP = "Modify metrics."
+    METRICS_MODIFY_HELP = "Modify metric file options."
     metrics_modify_parser = metrics_subparsers.add_parser(
         "modify",
         parents=[parent_parser],
-        description=METRICS_MODIFY_HELP,
+        description=append_doc_link(METRICS_MODIFY_HELP, "metrics-modify"),
         help=METRICS_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_modify_parser.add_argument(
-        "-t", "--type", help="Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)."
+        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
     )
     metrics_modify_parser.add_argument(
-        "-x", "--xpath", help="JSON/TSV/HTSV/CSV/HCSV path."
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
     )
-    metrics_modify_parser.add_argument("path", help="Metrics file.")
+    metrics_modify_parser.add_argument("path", help="Path to a metric file.")
     metrics_modify_parser.set_defaults(func=CmdMetricsModify)
 
-    METRICS_REMOVE_HELP = "Remove metrics."
+    METRICS_REMOVE_HELP = "Remove files's metric tag."
     metrics_remove_parser = metrics_subparsers.add_parser(
         "remove",
         parents=[parent_parser],
-        description=METRICS_REMOVE_HELP,
+        description=append_doc_link(METRICS_REMOVE_HELP, "metrics-remove"),
         help=METRICS_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    metrics_remove_parser.add_argument("path", help="Path to metrics file.")
+    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
     metrics_remove_parser.set_defaults(func=CmdMetricsRemove)

dvc/command/move.py

@@ -1,32 +1,42 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdMove(CmdBase):
     def run(self):
         try:
-            self.project.move(self.args.src, self.args.dst)
+            self.repo.move(self.args.src, self.args.dst)
         except DvcException:
             msg = "failed to move '{}' -> '{}'".format(
                 self.args.src, self.args.dst
             )
-            logger.error(msg)
+            logger.exception(msg)
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    description = (
-        "Rename or move a data file or a directory that "
-        "is under DVC control. It renames and modifies "
-        "the corresponding DVC file to reflect the changes."
+    MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
+    MOVE_DESCRIPTION = (
+        "Rename or move a DVC controlled data file or a directory.\n"
+        "It renames and modifies the corresponding DVC file to reflect the"
+        " changes."
     )
-    help = "Rename or move a DVC controlled data file or a directory."
+
     move_parser = subparsers.add_parser(
-        "move", parents=[parent_parser], description=description, help=help
+        "move",
+        parents=[parent_parser],
+        description=append_doc_link(MOVE_DESCRIPTION, "move"),
+        help=MOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
         "src", help="Source path to a data file or directory."

dvc/command/pipeline.py

@@ -2,24 +2,32 @@ from __future__ import unicode_literals
 
 from dvc.utils.compat import str
 
+import argparse
 import os
+import logging
 
-import dvc.logger as logger
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdPipelineShow(CmdBase):
-    def _show(self, target, commands, outs):
+    def _show(self, target, commands, outs, locked):
         import networkx
         from dvc.stage import Stage
 
-        stage = Stage.load(self.project, target)
-        G = self.project.graph()[0]
+        stage = Stage.load(self.repo, target)
+        G = self.repo.graph()[0]
         stages = networkx.get_node_attributes(G, "stage")
-        node = os.path.relpath(stage.path, self.project.root_dir)
+        node = os.path.relpath(stage.path, self.repo.root_dir)
+        nodes = networkx.dfs_postorder_nodes(G, node)
 
-        for n in networkx.dfs_postorder_nodes(G, node):
+        if locked:
+            nodes = [n for n in nodes if stages[n].locked]
+
+        for n in nodes:
             if commands:
                 logger.info(stages[n].cmd)
             elif outs:
@@ -32,11 +40,11 @@ class CmdPipelineShow(CmdBase):
         import networkx
         from dvc.stage import Stage
 
-        stage = Stage.load(self.project, target)
-        node = os.path.relpath(stage.path, self.project.root_dir)
+        stage = Stage.load(self.repo, target)
+        node = os.path.relpath(stage.path, self.repo.root_dir)
 
         pipelines = list(
-            filter(lambda g: node in g.nodes(), self.project.pipelines())
+            filter(lambda g: node in g.nodes(), self.repo.pipelines())
         )
 
         assert len(pipelines) == 1
@@ -71,30 +79,54 @@ class CmdPipelineShow(CmdBase):
             else:
                 edges.append((from_stage.relpath, to_stage.relpath))
 
-        return nodes, edges
+        return nodes, edges, networkx.is_tree(G)
 
     def _show_ascii(self, target, commands, outs):
         from dvc.dagascii import draw
 
-        nodes, edges = self.__build_graph(target, commands, outs)
+        nodes, edges, _ = self.__build_graph(target, commands, outs)
 
         if not nodes:
             return
 
         draw(nodes, edges)
 
+    def _show_dependencies_tree(self, target, commands, outs):
+        from treelib import Tree
+
+        nodes, edges, is_tree = self.__build_graph(target, commands, outs)
+        if not nodes:
+            return
+        if not is_tree:
+            raise DvcException(
+                "DAG is not a tree, can not print it in tree-structure way, "
+                "please use --ascii instead"
+            )
+
+        tree = Tree()
+        tree.create_node(target, target)  # Root node
+        observe_list = [target]
+        while len(observe_list) > 0:
+            current_root = observe_list[0]
+            for edge in edges:
+                if edge[0] == current_root:
+                    tree.create_node(edge[1], edge[1], parent=current_root)
+                    observe_list.append(edge[1])
+            observe_list.pop(0)
+        tree.show()
+
     def __write_dot(self, target, commands, outs, filename):
         import networkx
         from networkx.drawing.nx_pydot import write_dot
 
-        _, edges = self.__build_graph(target, commands, outs)
+        _, edges, _ = self.__build_graph(target, commands, outs)
         edges = [edge[::-1] for edge in edges]
 
         simple_g = networkx.DiGraph()
         simple_g.add_edges_from(edges)
         write_dot(simple_g, filename)
 
-    def run(self, unlock=False):
+    def run(self):
         if not self.args.targets:
             self.args.targets = self.default_targets
 
@@ -111,11 +143,20 @@ class CmdPipelineShow(CmdBase):
                         self.args.outs,
                         self.args.dot,
                     )
+                elif self.args.tree:
+                    self._show_dependencies_tree(
+                        target, self.args.commands, self.args.outs
+                    )
                 else:
-                    self._show(target, self.args.commands, self.args.outs)
+                    self._show(
+                        target,
+                        self.args.commands,
+                        self.args.outs,
+                        self.args.locked,
+                    )
             except DvcException:
                 msg = "failed to show pipeline for '{}'".format(target)
-                logger.error(msg)
+                logger.exception(msg)
                 return 1
         return 0
 
@@ -124,7 +165,7 @@ class CmdPipelineList(CmdBase):
     def run(self):
         import networkx
 
-        pipelines = self.project.pipelines()
+        pipelines = self.repo.pipelines()
         for p in pipelines:
             stages = networkx.get_node_attributes(p, "stage")
             for stage in stages:
@@ -137,12 +178,13 @@ class CmdPipelineList(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    PIPELINE_HELP = "Manage pipeline."
+    PIPELINE_HELP = "Manage pipelines."
     pipeline_parser = subparsers.add_parser(
         "pipeline",
         parents=[parent_parser],
-        description=PIPELINE_HELP,
+        description=append_doc_link(PIPELINE_HELP, "pipeline"),
         help=PIPELINE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     pipeline_subparsers = pipeline_parser.add_subparsers(
@@ -156,8 +198,9 @@ def add_parser(subparsers, parent_parser):
     pipeline_show_parser = pipeline_subparsers.add_parser(
         "show",
         parents=[parent_parser],
-        description=PIPELINE_SHOW_HELP,
+        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline-show"),
         help=PIPELINE_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
     pipeline_show_group.add_argument(
@@ -174,6 +217,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Print output files instead of paths to DVC files.",
     )
+    pipeline_show_parser.add_argument(
+        "-l",
+        "--locked",
+        action="store_true",
+        default=False,
+        help="Print locked DVC stages",
+    )
     pipeline_show_parser.add_argument(
         "--ascii",
         action="store_true",
@@ -183,6 +233,12 @@ def add_parser(subparsers, parent_parser):
     pipeline_show_parser.add_argument(
         "--dot", help="Write DAG in .dot format."
     )
+    pipeline_show_parser.add_argument(
+        "--tree",
+        action="store_true",
+        default=False,
+        help="Output DAG as Dependencies Tree.",
+    )
     pipeline_show_parser.add_argument(
         "targets", nargs="*", help="DVC files. 'Dvcfile' by default."
     )
@@ -192,7 +248,8 @@ def add_parser(subparsers, parent_parser):
     pipeline_list_parser = pipeline_subparsers.add_parser(
         "list",
         parents=[parent_parser],
-        description=PIPELINE_LIST_HELP,
+        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline-list"),
         help=PIPELINE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pipeline_list_parser.set_defaults(func=CmdPipelineList)

dvc/command/pkg.py

@@ -0,0 +1,89 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdPkgInstall(CmdBase):
+    def run(self):
+        try:
+            self.repo.pkg.install(
+                self.args.address,
+                self.args.target_dir,
+                self.args.select,
+                self.args.file,
+            )
+            return 0
+        except DvcException:
+            logger.exception(
+                "failed to install package '{}'".format(self.args.address)
+            )
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    PKG_HELP = "Manage DVC packages."
+    pkg_parser = subparsers.add_parser(
+        "pkg",
+        parents=[parent_parser],
+        description=append_doc_link(PKG_HELP, "pkg"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        add_help=False,
+    )
+
+    pkg_subparsers = pkg_parser.add_subparsers(
+        dest="cmd", help="Use dvc pkg CMD --help for command-specific help."
+    )
+
+    fix_subparsers(pkg_subparsers)
+
+    PKG_INSTALL_HELP = "Install package."
+    pkg_install_parser = pkg_subparsers.add_parser(
+        "install",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(PKG_INSTALL_HELP, "pkg-install"),
+        help=PKG_INSTALL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    pkg_install_parser.add_argument(
+        "address",
+        nargs="?",
+        default="",
+        help="Package address: git://<url> or https://github.com/...",
+    )
+    pkg_install_parser.add_argument(
+        "target_dir",
+        metavar="target",
+        nargs="?",
+        default=".",
+        help="Target directory to deploy package outputs. "
+        "Default value is the current dir.",
+    )
+    pkg_install_parser.add_argument(
+        "-s",
+        "--select",
+        metavar="OUT",
+        action="append",
+        default=[],
+        help="Select and persist only specified outputs from a package. "
+        "The parameter can be used multiple times. "
+        "All outputs will be selected by default.",
+    )
+    pkg_install_parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the stage file. It should be "
+        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+        "'prepare.dvc', 'clean.dvc', etc). "
+        "By default the file has 'mod_' prefix and imported package name "
+        "followed by .dvc",
+    )
+    pkg_install_parser.set_defaults(func=CmdPkgInstall)

dvc/command/remote.py

@@ -1,13 +1,16 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
 import re
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import fix_subparsers
-from dvc.config import Config
-from dvc.remote import _get, RemoteLOCAL
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
+from dvc.config import Config
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdRemoteAdd(CmdConfig):
@@ -30,13 +33,23 @@ class CmdRemoteAdd(CmdConfig):
         return os.path.relpath(path, os.path.dirname(config_file))
 
     def run(self):
+        from dvc.remote import _get, RemoteLOCAL
+
         remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
-        if remote == RemoteLOCAL:
+        if remote == RemoteLOCAL and not self.args.url.startswith("remote://"):
             self.args.url = self.resolve_path(
                 self.args.url, self.configobj.filename
             )
 
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
+        if (section in self.configobj.keys()) and not self.args.force:
+            logger.error(
+                "Remote with name {} already exists. "
+                "Use -f (--force) to overwrite remote "
+                "with new value".format(self.args.name)
+            )
+            return 1
+
         ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
         if ret != 0:
             return ret
@@ -76,7 +89,7 @@ class CmdRemoteRemove(CmdConfig):
 
         for configobj in [
             self.config._local_config,
-            self.config._project_config,
+            self.config._repo_config,
             self.config._global_config,
             self.config._system_config,
         ]:
@@ -118,12 +131,13 @@ class CmdRemoteList(CmdConfig):
 def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
-    REMOTE_HELP = "Manage set of tracked repositories."
+    REMOTE_HELP = "Manage remote storage configuration."
     remote_parser = subparsers.add_parser(
         "remote",
         parents=[parent_parser],
-        description=REMOTE_HELP,
+        description=append_doc_link(REMOTE_HELP, "remote"),
         help=REMOTE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     remote_subparsers = remote_parser.add_subparsers(
@@ -137,14 +151,14 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser = remote_subparsers.add_parser(
         "add",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_ADD_HELP,
+        description=append_doc_link(REMOTE_ADD_HELP, "remote-add"),
         help=REMOTE_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_add_parser.add_argument("name", help="Name.")
     remote_add_parser.add_argument(
         "url",
-        help="URL. See full list of supported urls at "
-        "https://dvc.org/doc/commands-reference/remote",
+        help="URL. See full list of supported urls at " "man.dvc.org/remote",
     )
     remote_add_parser.add_argument(
         "-d",
@@ -153,14 +167,22 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Set as default remote.",
     )
+    remote_add_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force overwriting existing configs",
+    )
     remote_add_parser.set_defaults(func=CmdRemoteAdd)
 
     REMOTE_DEFAULT_HELP = "Set/unset default remote."
     remote_default_parser = remote_subparsers.add_parser(
         "default",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_DEFAULT_HELP,
+        description=append_doc_link(REMOTE_DEFAULT_HELP, "remote-default"),
         help=REMOTE_DEFAULT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_default_parser.add_argument(
         "name", nargs="?", help="Name of the remote."
@@ -178,8 +200,9 @@ def add_parser(subparsers, parent_parser):
     remote_remove_parser = remote_subparsers.add_parser(
         "remove",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_REMOVE_HELP,
+        description=append_doc_link(REMOTE_REMOVE_HELP, "remote-remove"),
         help=REMOTE_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_remove_parser.add_argument("name", help="Name")
     remote_remove_parser.set_defaults(func=CmdRemoteRemove)
@@ -188,8 +211,9 @@ def add_parser(subparsers, parent_parser):
     remote_modify_parser = remote_subparsers.add_parser(
         "modify",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_MODIFY_HELP,
+        description=append_doc_link(REMOTE_MODIFY_HELP, "remote-modify"),
         help=REMOTE_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_modify_parser.add_argument("name", help="Name.")
     remote_modify_parser.add_argument("option", help="Option.")
@@ -203,11 +227,12 @@ def add_parser(subparsers, parent_parser):
     )
     remote_modify_parser.set_defaults(func=CmdRemoteModify)
 
-    REMOTE_LIST_HELP = "List remotes."
+    REMOTE_LIST_HELP = "List available remotes."
     remote_list_parser = remote_subparsers.add_parser(
         "list",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_LIST_HELP,
+        description=append_doc_link(REMOTE_LIST_HELP, "remote-list"),
         help=REMOTE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_list_parser.set_defaults(func=CmdRemoteList)

dvc/command/remove.py

@@ -1,9 +1,14 @@
 from __future__ import unicode_literals
 
+import argparse
+import logging
+
 import dvc.prompt as prompt
-import dvc.logger as logger
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdRemove(CmdBase):
@@ -30,9 +35,9 @@ class CmdRemove(CmdBase):
         for target in self.args.targets:
             try:
                 outs_only = self._is_outs_only(target)
-                self.project.remove(target, outs_only=outs_only)
+                self.repo.remove(target, outs_only=outs_only)
             except DvcException:
-                logger.error("failed to remove {}".format(target))
+                logger.exception("failed to remove {}".format(target))
                 return 1
         return 0
 
@@ -42,8 +47,9 @@ def add_parser(subparsers, parent_parser):
     remove_parser = subparsers.add_parser(
         "remove",
         parents=[parent_parser],
-        description=REMOVE_HELP,
+        description=append_doc_link(REMOVE_HELP, "remove"),
         help=REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remove_parser_group = remove_parser.add_mutually_exclusive_group()
     remove_parser_group.add_argument(

dvc/command/repro.py

@@ -1,13 +1,18 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdRepro(CmdBase):
     def run(self):
         recursive = not self.args.single_item
@@ -24,7 +29,7 @@ class CmdRepro(CmdBase):
         ret = 0
         for target in self.args.targets:
             try:
-                stages = self.project.reproduce(
+                stages = self.repo.reproduce(
                     target,
                     recursive=recursive,
                     force=self.args.force,
@@ -33,15 +38,18 @@ class CmdRepro(CmdBase):
                     pipeline=self.args.pipeline,
                     all_pipelines=self.args.all_pipelines,
                     ignore_build_cache=self.args.ignore_build_cache,
+                    no_commit=self.args.no_commit,
+                    downstream=self.args.downstream,
                 )
 
                 if len(stages) == 0:
                     logger.info(CmdDataStatus.UP_TO_DATE_MSG)
 
                 if self.args.metrics:
-                    self.project.metrics_show()
+                    metrics = self.repo.metrics.show()
+                    show_metrics(metrics)
             except DvcException:
-                logger.error()
+                logger.exception("")
                 ret = 1
                 break
 
@@ -50,15 +58,18 @@ class CmdRepro(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'."
+    REPRO_HELP = "Check for changes and reproduce DVC file and dependencies."
     repro_parser = subparsers.add_parser(
         "repro",
         parents=[parent_parser],
-        description=REPRO_HELP,
+        description=append_doc_link(REPRO_HELP, "repro"),
         help=REPRO_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     repro_parser.add_argument(
-        "targets", nargs="*", help="DVC file to reproduce."
+        "targets",
+        nargs="*",
+        help="DVC file to reproduce (default - 'Dvcfile').",
     )
     repro_parser.add_argument(
         "-f",
@@ -79,7 +90,7 @@ def add_parser(subparsers, parent_parser):
         "-c",
         "--cwd",
         default=os.path.curdir,
-        help="Directory within your project to reroduce from.",
+        help="Directory within your repo to reproduce from.",
     )
     repro_parser.add_argument(
         "-m",
@@ -115,7 +126,7 @@ def add_parser(subparsers, parent_parser):
         "--all-pipelines",
         action="store_true",
         default=False,
-        help="Reproduce all pipelines in the project.",
+        help="Reproduce all pipelines in the repo.",
     )
     repro_parser.add_argument(
         "--ignore-build-cache",
@@ -124,4 +135,16 @@ def add_parser(subparsers, parent_parser):
         help="Reproduce all descendants of a changed stage even if their "
         "direct dependencies didn't change.",
     )
+    repro_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    repro_parser.add_argument(
+        "--downstream",
+        action="store_true",
+        default=False,
+        help="Reproduce the pipeline starting from the specified stage.",
+    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/root.py

@@ -1,23 +1,29 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.repo import Repo
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 
-class CmdRoot(CmdBase):
-    def run_cmd(self):
-        return self.run()
+logger = logging.getLogger(__name__)
 
+
+class CmdRoot(CmdBaseNoRepo):
     def run(self):
-        logger.info(os.path.relpath(self.project.root_dir))
+        logger.info(os.path.relpath(Repo.find_root()))
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    ROOT_HELP = "Relative path to project's directory."
+    ROOT_HELP = "Relative path to the repository's directory."
     root_parser = subparsers.add_parser(
-        "root", parents=[parent_parser], description=ROOT_HELP, help=ROOT_HELP
+        "root",
+        parents=[parent_parser],
+        description=append_doc_link(ROOT_HELP, "root"),
+        help=ROOT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     root_parser.set_defaults(func=CmdRoot)

dvc/command/run.py

@@ -1,13 +1,15 @@
 from __future__ import unicode_literals
 
-import os
 import argparse
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdRun(CmdBase):
     def run(self):
         overwrite = self.args.yes or self.args.overwrite_dvcfile
@@ -17,31 +19,41 @@ class CmdRun(CmdBase):
                 self.args.deps,
                 self.args.outs,
                 self.args.outs_no_cache,
+                self.args.metrics,
+                self.args.metrics_no_cache,
+                self.args.outs_persist,
+                self.args.outs_persist_no_cache,
                 self.args.command,
             ]
         ):  # pragma: no cover
             logger.error(
-                "too few arguments. Specify at least one: '-d', "
-                "'-o', '-O', 'command'."
+                "too few arguments. Specify at least one: '-d', '-o', '-O',"
+                " '-m', '-M', '--outs-persist', '--outs-persist-no-cache',"
+                " 'command'."
             )
             return 1
 
         try:
-            self.project.run(
+            self.repo.run(
                 cmd=self._parsed_cmd(),
                 outs=self.args.outs,
                 outs_no_cache=self.args.outs_no_cache,
+                metrics=self.args.metrics,
                 metrics_no_cache=self.args.metrics_no_cache,
                 deps=self.args.deps,
                 fname=self.args.file,
                 cwd=self.args.cwd,
+                wdir=self.args.wdir,
                 no_exec=self.args.no_exec,
                 overwrite=overwrite,
                 ignore_build_cache=self.args.ignore_build_cache,
                 remove_outs=self.args.remove_outs,
+                no_commit=self.args.no_commit,
+                outs_persist=self.args.outs_persist,
+                outs_persist_no_cache=self.args.outs_persist_no_cache,
             )
         except DvcException:
-            logger.error("failed to run command")
+            logger.exception("failed to run command")
             return 1
 
         return 0
@@ -69,12 +81,13 @@ class CmdRun(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    RUN_HELP = (
-        "Generate a stage file from a given "
-        "command and execute the command."
-    )
+    RUN_HELP = "Generate a stage file from a command and execute the command."
     run_parser = subparsers.add_parser(
-        "run", parents=[parent_parser], description=RUN_HELP, help=RUN_HELP
+        "run",
+        parents=[parent_parser],
+        description=append_doc_link(RUN_HELP, "run"),
+        help=RUN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     run_parser.add_argument(
         "-d",
@@ -88,39 +101,42 @@ def add_parser(subparsers, parent_parser):
         "--outs",
         action="append",
         default=[],
-        help="Declare output data file or data directory.",
+        help="Declare output file or directory.",
     )
     run_parser.add_argument(
         "-O",
         "--outs-no-cache",
         action="append",
         default=[],
-        help="Declare output regular file or directory "
-        "(sync to Git, not DVC cache).",
+        help="Declare output file or directory "
+        "(do not put into DVC cache).",
+    )
+    run_parser.add_argument(
+        "-m",
+        "--metrics",
+        action="append",
+        default=[],
+        help="Declare output metric file or directory.",
     )
     run_parser.add_argument(
         "-M",
         "--metrics-no-cache",
         action="append",
         default=[],
-        help="Declare output metric file or " "directory (not cached by DVC).",
+        help="Declare output metric file or directory "
+        "(do not put into DVC cache).",
     )
     run_parser.add_argument(
-        "-f",
-        "--file",
-        help="Specify name of the stage file. It should be "
-        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
-        "'prepare.dvc', 'clean.dvc', etc) in order for "
-        "dvc to be able to find it later. By default "
-        "the first output basename + .dvc is used as "
-        "a stage filename.",
+        "-f", "--file", help="Specify name of the DVC file it generates."
     )
     run_parser.add_argument(
-        "-c",
-        "--cwd",
-        default=os.path.curdir,
-        help="Directory within your project to run your command and place "
-        "stage file in.",
+        "-c", "--cwd", default=None, help="Deprecated, use -w and -f instead."
+    )
+    run_parser.add_argument(
+        "-w",
+        "--wdir",
+        default=None,
+        help="Directory within your repo to run your command in.",
     )
     run_parser.add_argument(
         "--no-exec",
@@ -133,9 +149,7 @@ def add_parser(subparsers, parent_parser):
         "--yes",
         action="store_true",
         default=False,
-        help="(OBSOLETED, use --overwrite-dvcfile instead) Automatic 'yes' "
-        "answer to all prompts. E.g. when '.dvc' file exists and dvc "
-        "asks if you want to overwrite it.",
+        help="Deprecated, use --overwrite-dvcfile instead",
     )
     run_parser.add_argument(
         "--overwrite-dvcfile",
@@ -154,7 +168,27 @@ def add_parser(subparsers, parent_parser):
         "--remove-outs",
         action="store_true",
         default=False,
-        help="Remove outputs before running the command.",
+        help="Deprecated, this is now the default behavior",
+    )
+    run_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    run_parser.add_argument(
+        "--outs-persist",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro.",
+    )
+    run_parser.add_argument(
+        "--outs-persist-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro (do not put into DVC cache).",
     )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."

dvc/command/status.py

@@ -1,11 +1,16 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import logging
+
 from dvc.command.data_sync import CmdDataBase
+from dvc.utils.compat import str
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdDataStatus(CmdDataBase):
-    STATUS_LEN = 10
+    STATUS_LEN = 20
     STATUS_INDENT = "\t"
     UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce."
 
@@ -17,18 +22,28 @@ class CmdDataStatus(CmdDataBase):
     def _show(self, status, indent=0):
         ind = indent * self.STATUS_INDENT
 
+        if isinstance(status, str):
+            logger.info("{}{}".format(ind, status))
+            return
+
+        if isinstance(status, list):
+            for entry in status:
+                self._show(entry, indent)
+            return
+
+        assert isinstance(status, dict)
+
         for key, value in status.items():
-            if isinstance(value, dict):
-                logger.info("{}{}".format(ind, key))
+            if isinstance(value, str):
+                logger.info("{}{}{}".format(ind, self._normalize(value), key))
+            elif value:
+                logger.info("{}{}:".format(ind, key))
                 self._show(value, indent + 1)
-            else:
-                msg = "{}{}{}".format(ind, self._normalize(value), key)
-                logger.info(msg)
 
     def do_run(self, target=None):
         indent = 1 if self.args.cloud else 0
         try:
-            st = self.project.status(
+            st = self.repo.status(
                 target=target,
                 jobs=self.args.jobs,
                 cloud=self.args.cloud,
@@ -47,6 +62,6 @@ class CmdDataStatus(CmdDataBase):
                 logger.info(self.UP_TO_DATE_MSG)
 
         except Exception:
-            logger.error("failed to obtain data status")
+            logger.exception("failed to obtain data status")
             return 1
         return 0

dvc/command/tag.py

@@ -0,0 +1,154 @@
+import logging
+
+from dvc.utils import to_yaml_string
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdTagAdd(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.repo.tag.add(
+                    self.args.tag,
+                    target=target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("failed to add tag")
+                return 1
+        return 0
+
+
+class CmdTagRemove(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.repo.tag.remove(
+                    self.args.tag,
+                    target=target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("failed to remove tag")
+                return 1
+        return 0
+
+
+class CmdTagList(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                tags = self.repo.tag.list(
+                    target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+                if tags:
+                    logger.info(to_yaml_string(tags))
+            except DvcException:
+                logger.exception("failed list tags")
+                return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    TAG_HELP = "A set of commands to manage DVC tags."
+    tag_parser = subparsers.add_parser(
+        "tag",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_HELP, "tag"),
+        add_help=False,
+    )
+
+    tag_subparsers = tag_parser.add_subparsers(
+        dest="cmd",
+        help="Use DVC tag CMD --help to display command-specific help.",
+    )
+
+    fix_subparsers(tag_subparsers)
+
+    TAG_ADD_HELP = "Add DVC tag."
+    tag_add_parser = tag_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_ADD_HELP, "tag-add"),
+        help=TAG_ADD_HELP,
+    )
+    tag_add_parser.add_argument("tag", help="Dvc tag.")
+    tag_add_parser.add_argument(
+        "targets", nargs="*", default=[None], help="Dvc files."
+    )
+    tag_add_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Add tag for all dependencies of the specified DVC file.",
+    )
+    tag_add_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Add tag for subdirectories of the specified directory.",
+    )
+    tag_add_parser.set_defaults(func=CmdTagAdd)
+
+    TAG_REMOVE_HELP = "Remove DVC tag."
+    tag_remove_parser = tag_subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_REMOVE_HELP, "tag-remove"),
+        help=TAG_REMOVE_HELP,
+    )
+    tag_remove_parser.add_argument("tag", help="Dvc tag.")
+    tag_remove_parser.add_argument(
+        "targets", nargs="*", default=[None], help="Dvc files."
+    )
+    tag_remove_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Remove tag for all dependencies of the specified DVC file.",
+    )
+    tag_remove_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Remove tag for subdirectories of the specified directory.",
+    )
+    tag_remove_parser.set_defaults(func=CmdTagRemove)
+
+    TAG_LIST_HELP = "List DVC tags."
+    tag_list_parser = tag_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_LIST_HELP, "tag-list"),
+        help=TAG_LIST_HELP,
+    )
+    tag_list_parser.add_argument(
+        "targets", nargs="*", default=[None], help="Dvc files."
+    )
+    tag_list_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="List tags for all dependencies of the specified DVC file.",
+    )
+    tag_list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="List tags for subdirectories of the specified directory.",
+    )
+    tag_list_parser.set_defaults(func=CmdTagList)

dvc/command/unprotect.py

@@ -1,29 +1,35 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdUnprotect(CmdBase):
     def run(self):
         for target in self.args.targets:
             try:
-                self.project.unprotect(target)
+                self.repo.unprotect(target)
             except DvcException:
                 msg = "failed to unprotect '{}'".format(target)
-                logger.error(msg)
+                logger.exception(msg)
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    UNPROTECT_HELP = "Unprotect data file/directory."
+    UNPROTECT_HELP = "Unprotect a data file/directory."
     unprotect_parser = subparsers.add_parser(
         "unprotect",
         parents=[parent_parser],
-        description=UNPROTECT_HELP,
+        description=append_doc_link(UNPROTECT_HELP, "unprotect"),
         help=UNPROTECT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
         "targets", nargs="+", help="Data files/directory."

dvc/command/version.py

@@ -0,0 +1,39 @@
+from __future__ import unicode_literals
+
+import platform
+import argparse
+import logging
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.version import __version__
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdVersion(CmdBaseNoRepo):
+    def run(self):
+        info = (
+            "DVC version: {dvc_version}\n"
+            "Python version: {python_version}\n"
+            "Platform: {platform_type}"
+        ).format(
+            dvc_version=__version__,
+            python_version=platform.python_version(),
+            platform_type=platform.platform(),
+        )
+        logger.info(info)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    VERSION_HELP = "Show DVC version and system/environment informaion."
+
+    version_parser = subparsers.add_parser(
+        "version",
+        parents=[parent_parser],
+        description=append_doc_link(VERSION_HELP, "version"),
+        help=VERSION_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    version_parser.set_defaults(func=CmdVersion)

dvc/config.py

@@ -2,16 +2,18 @@
 
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, open
+from dvc.utils.compat import str, open, urlparse
 
 import os
 import errno
 import configobj
-from schema import Schema, Optional, And, Use, Regex
+import logging
 
-import dvc.logger as logger
+from schema import Schema, Optional, And, Use, Regex
 from dvc.exceptions import DvcException
 
+logger = logging.getLogger(__name__)
+
 
 class ConfigError(DvcException):
     """DVC config exception.
@@ -108,11 +110,11 @@ def is_percent(val):
 
 
 class Config(object):  # pylint: disable=too-many-instance-attributes
-    """Class that manages configuration files for a dvc project.
+    """Class that manages configuration files for a dvc repo.
 
     Args:
         dvc_dir (str): optional path to `.dvc` directory, that is used to
-            access project-specific configs like .dvc/config and
+            access repo-specific configs like .dvc/config and
             .dvc/config.local.
         validate (bool): optional flag to tell dvc if it should validate the
             config or just load it as is. 'True' by default.
@@ -132,14 +134,16 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
+    BOOL_SCHEMA = And(str, is_bool, Use(to_bool))
+
     SECTION_CORE = "core"
     SECTION_CORE_LOGLEVEL = "loglevel"
     SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
     SECTION_CORE_REMOTE = "remote"
-    SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
+    SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA
     SECTION_CORE_INTERACTIVE = "interactive"
     SECTION_CORE_ANALYTICS = "analytics"
-    SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool))
+    SECTION_CORE_ANALYTICS_SCHEMA = BOOL_SCHEMA
 
     SECTION_CACHE = "cache"
     SECTION_CACHE_DIR = "dir"
@@ -152,6 +156,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_CACHE_SSH = "ssh"
     SECTION_CACHE_HDFS = "hdfs"
     SECTION_CACHE_AZURE = "azure"
+    SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
     SECTION_CACHE_SCHEMA = {
         Optional(SECTION_CACHE_LOCAL): str,
         Optional(SECTION_CACHE_S3): str,
@@ -161,10 +166,9 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_CACHE_AZURE): str,
         Optional(SECTION_CACHE_DIR): str,
         Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED, default=False): And(
-            str, is_bool, Use(to_bool)
-        ),
+        Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
         Optional(PRIVATE_CWD): str,
+        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): BOOL_SCHEMA,
     }
 
     # backward compatibility
@@ -173,7 +177,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_CORE_STORAGEPATH = "storagepath"
 
     SECTION_CORE_SCHEMA = {
-        Optional(SECTION_CORE_LOGLEVEL, default="info"): And(
+        Optional(SECTION_CORE_LOGLEVEL): And(
             str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA
         ),
         Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)),
@@ -193,6 +197,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_AWS_STORAGEPATH = "storagepath"
     SECTION_AWS_CREDENTIALPATH = "credentialpath"
     SECTION_AWS_ENDPOINT_URL = "endpointurl"
+    SECTION_AWS_LIST_OBJECTS = "listobjects"
     SECTION_AWS_REGION = "region"
     SECTION_AWS_PROFILE = "profile"
     SECTION_AWS_USE_SSL = "use_ssl"
@@ -202,9 +207,8 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_PROFILE): str,
         Optional(SECTION_AWS_CREDENTIALPATH): str,
         Optional(SECTION_AWS_ENDPOINT_URL): str,
-        Optional(SECTION_AWS_USE_SSL, default=True): And(
-            str, is_bool, Use(to_bool)
-        ),
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
+        Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
     }
 
     # backward compatibility
@@ -223,6 +227,10 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str}
 
     SECTION_AZURE_CONNECTION_STRING = "connection_string"
+    # Alibabacloud oss options
+    SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
+    SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
+    SECTION_OSS_ENDPOINT = "oss_endpoint"
 
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
@@ -239,21 +247,21 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_PROFILE): str,
         Optional(SECTION_AWS_CREDENTIALPATH): str,
         Optional(SECTION_AWS_ENDPOINT_URL): str,
-        Optional(SECTION_AWS_USE_SSL, default=True): And(
-            str, is_bool, Use(to_bool)
-        ),
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
+        Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED, default=False): And(
-            str, is_bool, Use(to_bool)
-        ),
+        Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
         Optional(SECTION_REMOTE_USER): str,
         Optional(SECTION_REMOTE_PORT): Use(int),
         Optional(SECTION_REMOTE_KEY_FILE): str,
         Optional(SECTION_REMOTE_TIMEOUT): Use(int),
         Optional(SECTION_REMOTE_PASSWORD): str,
-        Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
+        Optional(SECTION_REMOTE_ASK_PASSWORD): BOOL_SCHEMA,
         Optional(SECTION_AZURE_CONNECTION_STRING): str,
+        Optional(SECTION_OSS_ACCESS_KEY_ID): str,
+        Optional(SECTION_OSS_ACCESS_KEY_SECRET): str,
+        Optional(SECTION_OSS_ENDPOINT): str,
         Optional(PRIVATE_CWD): str,
     }
 
@@ -295,7 +303,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
 
         self._system_config = None
         self._global_config = None
-        self._project_config = None
+        self._repo_config = None
         self._local_config = None
 
         self.config = None
@@ -347,9 +355,9 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         self._global_config = configobj.ConfigObj(self.global_config_file)
 
         if self.config_file is not None:
-            self._project_config = configobj.ConfigObj(self.config_file)
+            self._repo_config = configobj.ConfigObj(self.config_file)
         else:
-            self._project_config = configobj.ConfigObj()
+            self._repo_config = configobj.ConfigObj()
 
         if self.config_local_file is not None:
             self._local_config = configobj.ConfigObj(self.config_local_file)
@@ -382,6 +390,9 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         cache[self.PRIVATE_CWD] = os.path.dirname(fname)
 
     def _resolve_paths(self, config, fname):
+        if fname is None:
+            return
+
         self._resolve_cache_path(config, fname)
         for section in config.values():
             if self.SECTION_REMOTE_URL not in section.keys():
@@ -451,7 +462,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
             clist = [
                 self._system_config,
                 self._global_config,
-                self._project_config,
+                self._repo_config,
                 self._local_config,
             ]
 
@@ -472,6 +483,56 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
                 msg = "failed to write config '{}'".format(conf.filename)
                 raise ConfigError(msg, exc)
 
+    def get_remote_settings(self, name):
+        import posixpath
+
+        """
+        Args:
+            name (str): The name of the remote that we want to retrieve
+
+        Returns:
+            dict: The content beneath the given remote name.
+
+        Example:
+            >>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
+            >>> get_remote_settings("server")
+            {'url': 'ssh://localhost/'}
+        """
+        settings = self.config[self.SECTION_REMOTE_FMT.format(name)]
+        parsed = urlparse(settings["url"])
+
+        # Support for cross referenced remotes.
+        # This will merge the settings, giving priority to the outer reference.
+        # For example, having:
+        #
+        #       dvc remote add server ssh://localhost
+        #       dvc remote modify server user root
+        #       dvc remote modify server ask_password true
+        #
+        #       dvc remote add images remote://server/tmp/pictures
+        #       dvc remote modify images user alice
+        #       dvc remote modify images ask_password false
+        #       dvc remote modify images password asdf1234
+        #
+        # Results on a config dictionary like:
+        #
+        #       {
+        #           "url": "ssh://localhost/tmp/pictures",
+        #           "user": "alice",
+        #           "password": "asdf1234",
+        #           "ask_password": False,
+        #       }
+        #
+        if parsed.scheme == "remote":
+            reference = self.get_remote_settings(parsed.netloc)
+            url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
+            merged = reference.copy()
+            merged.update(settings)
+            merged["url"] = url
+            return merged
+
+        return settings
+
     @staticmethod
     def unset(config, section, opt=None):
         """Unsets specified option and/or section in the config.

dvc/daemon.py

@@ -5,12 +5,16 @@ from __future__ import unicode_literals
 import os
 import sys
 import inspect
+import logging
 from subprocess import Popen
 
-import dvc.logger as logger
+from dvc.env import DVC_DAEMON
 from dvc.utils import is_binary, fix_env
+from dvc.utils.compat import cast_bytes_py2
 
 
+logger = logging.getLogger(__name__)
+
 CREATE_NEW_PROCESS_GROUP = 0x00000200
 DETACHED_PROCESS = 0x00000008
 
@@ -42,7 +46,7 @@ def _spawn_posix(cmd, env):
         if pid > 0:
             return
     except OSError:
-        logger.error("failed at first fork")
+        logger.exception("failed at first fork")
         os._exit(1)  # pylint: disable=protected-access
 
     os.setsid()
@@ -53,7 +57,7 @@ def _spawn_posix(cmd, env):
         if pid > 0:
             os._exit(0)  # pylint: disable=protected-access
     except OSError:
-        logger.error("failed at second fork")
+        logger.exception("failed at second fork")
         os._exit(1)  # pylint: disable=protected-access
 
     sys.stdin.close()
@@ -65,22 +69,8 @@ def _spawn_posix(cmd, env):
     os._exit(0)  # pylint: disable=protected-access
 
 
-def daemon(args):
-    """Launch a `dvc daemon` command in a detached process.
-
-    Args:
-        args (list): list of arguments to append to `dvc daemon` command.
-    """
-    cmd = [sys.executable]
-    if not is_binary():
-        cmd += ["-m", "dvc"]
-    cmd += ["daemon", "-q"] + args
-
-    env = fix_env()
-    file_path = os.path.abspath(inspect.stack()[0][1])
-    env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
-
-    logger.debug("Trying to spawn '{}' with env '{}'".format(cmd, env))
+def _spawn(cmd, env):
+    logger.debug("Trying to spawn '{}'".format(cmd))
 
     if os.name == "nt":
         _spawn_windows(cmd, env)
@@ -90,3 +80,28 @@ def daemon(args):
         raise NotImplementedError
 
     logger.debug("Spawned '{}'".format(cmd))
+
+
+def daemon(args):
+    """Launch a `dvc daemon` command in a detached process.
+
+    Args:
+        args (list): list of arguments to append to `dvc daemon` command.
+    """
+    if os.environ.get(DVC_DAEMON):
+        logger.debug("skipping launching a new daemon.")
+        return
+
+    cmd = [sys.executable]
+    if not is_binary():
+        cmd += ["-m", "dvc"]
+    cmd += ["daemon", "-q"] + args
+
+    env = fix_env()
+    file_path = os.path.abspath(inspect.stack()[0][1])
+    env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
+        os.path.dirname(os.path.dirname(file_path))
+    )
+    env[cast_bytes_py2(DVC_DAEMON)] = cast_bytes_py2("1")
+
+    _spawn(cmd, env)

dvc/data_cloud.py

@@ -2,26 +2,30 @@
 
 from __future__ import unicode_literals
 
-import dvc.logger as logger
-from dvc.config import Config, ConfigError
+import logging
 
+from dvc.config import Config, ConfigError
 from dvc.remote import Remote
 from dvc.remote.s3 import RemoteS3
 from dvc.remote.gs import RemoteGS
 from dvc.remote.azure import RemoteAzure
+from dvc.remote.oss import RemoteOSS
 from dvc.remote.ssh import RemoteSSH
 from dvc.remote.hdfs import RemoteHDFS
 from dvc.remote.local import RemoteLOCAL
 from dvc.remote.http import RemoteHTTP
 
 
+logger = logging.getLogger(__name__)
+
+
 class DataCloud(object):
     """Class that manages dvc remotes.
 
     Args:
-        project (dvc.project.Project): project instance that belongs to the
-            project that we are working on.
-        config (dict): config of the project.
+        repo (dvc.repo.Repo): repo instance that belongs to the repo that
+            we are working on.
+        config (dict): config of the repo.
 
     Raises:
         config.ConfigError: thrown when config has invalid format.
@@ -31,6 +35,7 @@ class DataCloud(object):
         "aws": RemoteS3,
         "gcp": RemoteGS,
         "azure": RemoteAzure,
+        "oss": RemoteOSS,
         "ssh": RemoteSSH,
         "hdfs": RemoteHDFS,
         "local": RemoteLOCAL,
@@ -38,15 +43,17 @@ class DataCloud(object):
         "https": RemoteHTTP,
     }
 
-    def __init__(self, project, config=None):
-        self.project = project
+    def __init__(self, repo, config=None):
+        self.repo = repo
         self._config = config
         self._core = self._config[Config.SECTION_CORE]
 
     @property
     def _cloud(self):
+        """Returns a Remote instance using the `core.remote` config"""
         remote = self._core.get(Config.SECTION_CORE_REMOTE, "")
-        if remote != "":
+
+        if remote:
             return self._init_remote(remote)
 
         if self._core.get(Config.SECTION_CORE_CLOUD, None):
@@ -64,7 +71,7 @@ class DataCloud(object):
             msg = "can't find remote section '{}' in config"
             raise ConfigError(msg.format(section))
 
-        return Remote(self.project, cloud_config)
+        return Remote(self.repo, cloud_config)
 
     def _init_compat(self):
         name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower()
@@ -94,7 +101,7 @@ class DataCloud(object):
         if global_storage_path:
             logger.warning("using obsoleted config format. Consider updating.")
 
-        cloud = cloud_type(self.project, cloud_config)
+        cloud = cloud_type(self.repo, cloud_config)
         return cloud
 
     def _get_cloud(self, remote, cmd):
@@ -122,7 +129,7 @@ class DataCloud(object):
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
-        return self.project.cache.local.push(
+        return self.repo.cache.local.push(
             targets,
             jobs=jobs,
             remote=self._get_cloud(remote, "push"),
@@ -140,7 +147,7 @@ class DataCloud(object):
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
-        return self.project.cache.local.pull(
+        return self.repo.cache.local.pull(
             targets,
             jobs=jobs,
             remote=self._get_cloud(remote, "pull"),
@@ -160,6 +167,6 @@ class DataCloud(object):
                 information messages.
         """
         cloud = self._get_cloud(remote, "status")
-        return self.project.cache.local.status(
+        return self.repo.cache.local.status(
             targets, jobs=jobs, remote=cloud, show_checksums=show_checksums
         )

dvc/dependency/__init__.py

@@ -2,9 +2,6 @@ from __future__ import unicode_literals
 
 import schema
 
-from dvc.config import Config
-from dvc.utils.compat import urlparse
-
 import dvc.output as output
 from dvc.output.base import OutputBase
 from dvc.dependency.s3 import DependencyS3
@@ -46,11 +43,13 @@ del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)]
 
 
 def _get(stage, p, info):
+    from dvc.utils.compat import urlparse
+
     parsed = urlparse(p)
+
     if parsed.scheme == "remote":
-        name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
-        sect = stage.project.config.config[name]
-        remote = Remote(stage.project, sect)
+        settings = stage.repo.config.get_remote_settings(parsed.netloc)
+        remote = Remote(stage.repo, settings)
         return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
 
     for d in DEPS:

dvc/dependency/http.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+from dvc.path.http import HTTPPathInfo
 from dvc.utils.compat import urlparse, urljoin
 from dvc.output.base import OutputBase
 from dvc.remote.http import RemoteHTTP
@@ -16,4 +17,4 @@ class DependencyHTTP(DependencyBase, OutputBase):
         if path.startswith("remote"):
             path = urljoin(self.remote.cache_dir, urlparse(path).path)
 
-        self.path_info = {"scheme": urlparse(path).scheme, "path": path}
+        self.path_info = HTTPPathInfo(url=self.url, path=path)

dvc/env.py

@@ -0,0 +1 @@
+DVC_DAEMON = "DVC_DAEMON"

dvc/exceptions.py

@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 
 from dvc.utils.compat import str, builtin_str
 
+import os
 import traceback
 
 
@@ -11,7 +12,7 @@ class DvcException(Exception):
     """Base class for all dvc exceptions.
 
     Args:
-        msg (str): message for this exception.
+        msg (unicode): message for this exception.
         cause (Exception): optional cause exception.
     """
 
@@ -33,7 +34,7 @@ class OutputDuplicationError(DvcException):
     stage.
 
     Args:
-        output (str): path to the file/directory.
+        output (unicode): path to the file/directory.
         stages (list): list of paths to stages.
     """
 
@@ -51,9 +52,24 @@ class OutputDuplicationError(DvcException):
         super(OutputDuplicationError, self).__init__(msg)
 
 
-class WorkingDirectoryAsOutputError(DvcException):
+class OutputNotFoundError(DvcException):
+    """Thrown if a file/directory not found in repository pipelines.
+
+    Args:
+        output (unicode): path to the file/directory.
+    """
+
+    def __init__(self, output):
+        super(OutputNotFoundError, self).__init__(
+            "unable to find stage file with output '{path}'".format(
+                path=os.path.relpath(output)
+            )
+        )
+
+
+class StagePathAsOutputError(DvcException):
     """Thrown if directory that stage is going to be saved in is specified as
-    an output of anothe stage.
+    an output of another stage.
 
     Args:
         cwd (str): path to the directory.
@@ -61,16 +77,16 @@ class WorkingDirectoryAsOutputError(DvcException):
             output.
     """
 
-    def __init__(self, cwd, fname):
-        assert isinstance(cwd, str) or isinstance(cwd, builtin_str)
+    def __init__(self, wdir, fname):
+        assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
         assert isinstance(fname, str) or isinstance(fname, builtin_str)
         msg = (
             "current working directory '{cwd}' is specified as an output in"
             " '{fname}'. Use another CWD to prevent any data removal.".format(
-                cwd=cwd, fname=fname
+                cwd=wdir, fname=fname
             )
         )
-        super(WorkingDirectoryAsOutputError, self).__init__(msg)
+        super(StagePathAsOutputError, self).__init__(msg)
 
 
 class CircularDependencyError(DvcException):
@@ -125,16 +141,19 @@ class MoveNotDataSourceError(DvcException):
         super(MoveNotDataSourceError, self).__init__(msg.format(path=path))
 
 
-class NotDvcProjectError(DvcException):
-    """Thrown if a directory is not a dvc project.
+class NotDvcRepoError(DvcException):
+    """Thrown if a directory is not a dvc repo.
 
     Args:
         root (str): path to the directory.
     """
 
     def __init__(self, root):
-        msg = "not a dvc repository (checked up to mount point '{}')"
-        super(NotDvcProjectError, self).__init__(msg.format(root))
+        msg = (
+            "you are not inside of a dvc repository "
+            "(checked up to mount point '{}')"
+        )
+        super(NotDvcRepoError, self).__init__(msg.format(root))
 
 
 class DvcParserError(DvcException):
@@ -163,3 +182,85 @@ class ConfirmRemoveError(DvcException):
             "unable to remove '{}' without a confirmation from the user. Use "
             "'-f' to force.".format(path)
         )
+
+
+class InitError(DvcException):
+    def __init__(self, msg):
+        super(InitError, self).__init__(msg)
+
+
+class ReproductionError(DvcException):
+    def __init__(self, dvc_file_name, ex):
+        self.path = dvc_file_name
+        msg = "failed to reproduce '{}'".format(dvc_file_name)
+        super(ReproductionError, self).__init__(msg, cause=ex)
+
+
+class BadMetricError(DvcException):
+    def __init__(self, path):
+        super(BadMetricError, self).__init__(
+            "'{}' does not exist, not a metric or is malformed".format(
+                os.path.relpath(path)
+            )
+        )
+
+
+class NoMetricsError(DvcException):
+    def __init__(self):
+        super(NoMetricsError, self).__init__(
+            "no metric files in this repository. "
+            "Use 'dvc metrics add' to add a metric file to track."
+        )
+
+
+class StageFileCorruptedError(DvcException):
+    def __init__(self, path, cause=None):
+        path = os.path.relpath(path)
+        super(StageFileCorruptedError, self).__init__(
+            "unable to read stage file: {} "
+            "YAML file structure is corrupted".format(path),
+            cause=cause,
+        )
+
+
+class RecursiveAddingWhileUsingFilename(DvcException):
+    def __init__(self):
+        super(RecursiveAddingWhileUsingFilename, self).__init__(
+            "using fname with recursive is not allowed."
+        )
+
+
+class OverlappingOutputPathsError(DvcException):
+    def __init__(self, out_1, out_2):
+        super(OverlappingOutputPathsError, self).__init__(
+            "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
+            "unpredictable behaviour, rerun command with non overlapping outs "
+            "paths.".format(
+                str(out_1),
+                out_1.stage.relpath,
+                str(out_2),
+                out_2.stage.relpath,
+            )
+        )
+
+
+class TargetNotDirectoryError(DvcException):
+    def __init__(self, path):
+        super(TargetNotDirectoryError, self).__init__(
+            "Target: {} is not a directory".format(path)
+        )
+
+
+class CheckoutErrorSuggestGit(DvcException):
+    def __init__(self, target, cause):
+        super(CheckoutErrorSuggestGit, self).__init__(
+            "Did you mean 'git checkout {}'?".format(target), cause=cause
+        )
+
+
+class ETagMismatchError(DvcException):
+    def __init__(self, etag, cached_etag):
+        super(ETagMismatchError, self).__init__(
+            "ETag mismatch detected when copying file to cache! "
+            "(expected: '{}', actual: '{}')".format(etag, cached_etag)
+        )

dvc/ignore.py

@@ -0,0 +1,134 @@
+import os
+
+from dulwich.ignore import match_pattern, read_ignore_patterns
+from dvc.utils.compat import cast_bytes
+from dvc.utils.fs import get_parent_dirs_up_to
+
+
+class DvcIgnoreFileHandler(object):
+    def __init__(self, tree):
+        self.tree = tree
+
+    def read_patterns(self, path):
+        with self.tree.open(path, binary=True) as stream:
+            return self._read_patterns(stream)
+
+    def get_repo_root(self):
+        return self.tree.tree_root
+
+    def _read_patterns(self, binary_stream):
+        negate_patterns = []
+        patterns = []
+        for pattern in read_ignore_patterns(binary_stream):
+            if pattern.lstrip().startswith(b"!"):
+                negate_patterns.append(pattern)
+            else:
+                patterns.append(pattern)
+
+        return negate_patterns, patterns
+
+
+class DvcIgnore(object):
+    DVCIGNORE_FILE = ".dvcignore"
+
+    def __call__(self, root, dirs, files):
+        raise NotImplementedError
+
+
+class DvcIgnoreFromFile(DvcIgnore):
+    def __init__(self, ignore_file_path, ignore_handler):
+        self.ignore_file_path = ignore_file_path
+        self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
+        self.patterns = []
+        self.negate_patterns = []
+
+        self.negate_patterns, self.patterns = ignore_handler.read_patterns(
+            ignore_file_path
+        )
+
+    def __call__(self, root, dirs, files):
+        files = [f for f in files if not self.matches(root, f)]
+        dirs = [d for d in dirs if not self.matches(root, d)]
+
+        return dirs, files
+
+    def get_match(self, abs_path):
+        rel_path = os.path.relpath(abs_path, self.dirname)
+        if os.name == "nt":
+            rel_path = rel_path.replace("\\", "/")
+        rel_path = cast_bytes(rel_path, "utf-8")
+
+        for pattern in self.patterns:
+            if match_pattern(
+                rel_path, pattern
+            ) and self._no_negate_pattern_matches(rel_path):
+                return (abs_path, pattern, self.ignore_file_path)
+        return None
+
+    def matches(self, dirname, basename):
+        if self.get_match(os.path.join(dirname, basename)):
+            return True
+        return False
+
+    def _no_negate_pattern_matches(self, path):
+        return all([not match_pattern(path, p) for p in self.negate_patterns])
+
+    def __hash__(self):
+        return hash(self.ignore_file_path)
+
+
+class DvcIgnoreConstant(DvcIgnore):
+    def __init__(self, basename):
+        self.basename = basename
+
+
+class DvcIgnoreDir(DvcIgnoreConstant):
+    def __call__(self, root, dirs, files):
+        dirs = [d for d in dirs if not d == self.basename]
+
+        return dirs, files
+
+
+class DvcIgnoreFile(DvcIgnoreConstant):
+    def __call__(self, root, dirs, files):
+        files = [f for f in files if not f == self.basename]
+
+        return dirs, files
+
+
+class DvcIgnoreFilter(object):
+    def __init__(self, wdir, ignore_file_handler=None):
+        self.ignores = [
+            DvcIgnoreDir(".git"),
+            DvcIgnoreDir(".hg"),
+            DvcIgnoreDir(".dvc"),
+            DvcIgnoreFile(".dvcignore"),
+        ]
+
+        self.ignore_file_handler = ignore_file_handler
+        self._process_ignores_in_parent_dirs(wdir)
+
+    def _process_ignores_in_parent_dirs(self, wdir):
+        if self.ignore_file_handler:
+            wdir = os.path.normpath(os.path.abspath(wdir))
+            ignore_search_end_dir = self.ignore_file_handler.get_repo_root()
+            parent_dirs = get_parent_dirs_up_to(wdir, ignore_search_end_dir)
+            for d in parent_dirs:
+                self.update(d)
+
+    def update(self, wdir):
+        ignore_file_path = os.path.join(wdir, DvcIgnore.DVCIGNORE_FILE)
+        if os.path.exists(ignore_file_path):
+            file_ignore = DvcIgnoreFromFile(
+                ignore_file_path, ignore_handler=self.ignore_file_handler
+            )
+            self.ignores.append(file_ignore)
+
+    def __call__(self, root, dirs, files):
+        if self.ignore_file_handler:
+            self.update(root)
+
+        for ignore in self.ignores:
+            dirs, files = ignore(root, dirs, files)
+
+        return dirs, files

dvc/lock.py

@@ -10,11 +10,11 @@ from dvc.exceptions import DvcException
 
 
 class LockError(DvcException):
-    """Thrown when unable to acquire the lock for dvc project."""
+    """Thrown when unable to acquire the lock for dvc repo."""
 
 
 class Lock(object):
-    """Class for dvc project lock.
+    """Class for dvc repo lock.
 
     Args:
         dvc_dir (str): path to the directory that the lock should be created
@@ -39,7 +39,7 @@ class Lock(object):
             )
 
     def lock(self):
-        """Acquire lock for dvc project."""
+        """Acquire lock for dvc repo."""
         try:
             self._do_lock()
             return
@@ -49,7 +49,7 @@ class Lock(object):
         self._do_lock()
 
     def unlock(self):
-        """Release lock for dvc project."""
+        """Release lock for dvc repo."""
         self._lock.close()
         self._lock = None
 

dvc/logger.py

@@ -1,294 +1,182 @@
-"""Manages logger for dvc project."""
+"""Manages logging configuration for dvc repo."""
 
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str
+from dvc.utils.compat import str, StringIO
 
-import re
-import sys
 import logging
-import traceback
-
-from contextlib import contextmanager
-
+import logging.handlers
+import logging.config
 import colorama
 
 
-def info(message):
-    """Prints an info message."""
-    logger.info(message)
+class ExcludeErrorsFilter(logging.Filter):
+    def filter(self, record):
+        return record.levelno < logging.ERROR
 
 
-def debug(message):
-    """Prints a debug message."""
-    prefix = colorize("Debug", color="blue")
+class ColorFormatter(logging.Formatter):
+    """Enable color support when logging to a terminal that supports it.
 
-    out = "{prefix}: {message}".format(prefix=prefix, message=message)
+    Color support on Windows versions that do not support ANSI color codes is
+    enabled by use of the colorama__ library.
+    See the colorama documentation for details.
 
-    logger.debug(out)
+    __ https://pypi.python.org/pypi/colorama
 
-
-def warning(message):
-    """Prints a warning message."""
-    prefix = colorize("Warning", color="yellow")
-
-    out = "{prefix}: {message}".format(prefix=prefix, message=message)
-
-    logger.warning(out)
-
-
-def error(message=None):
-    """Prints an error message."""
-    prefix = colorize("Error", color="red")
-
-    exception, stack_trace = _parse_exc()
-
-    out = (
-        "{prefix}: {description}"
-        "\n"
-        "{stack_trace}"
-        "\n"
-        "{footer}".format(
-            prefix=prefix,
-            description=_description(message, exception),
-            stack_trace=stack_trace,
-            footer=_footer(),
-        )
-    )
-
-    logger.error(out)
-
-
-def box(message, border_color=None):
-    """Prints a message in a box.
-
-    Args:
-        message (str): message to print.
-        border_color (str): name of a color to outline the box with.
+    For records containing `exc_info`, it will use a custom `_walk_exc` to
+    retrieve the whole tracebak.
     """
-    lines = message.split("\n")
-    max_width = max(_visual_width(line) for line in lines)
 
-    padding_horizontal = 5
-    padding_vertical = 1
-
-    box_size_horizontal = max_width + (padding_horizontal * 2)
-
-    chars = {"corner": "+", "horizontal": "-", "vertical": "|", "empty": " "}
-
-    margin = "{corner}{line}{corner}\n".format(
-        corner=chars["corner"], line=chars["horizontal"] * box_size_horizontal
-    )
-
-    padding_lines = [
-        "{border}{space}{border}\n".format(
-            border=colorize(chars["vertical"], color=border_color),
-            space=chars["empty"] * box_size_horizontal,
-        )
-        * padding_vertical
-    ]
-
-    content_lines = [
-        "{border}{space}{content}{space}{border}\n".format(
-            border=colorize(chars["vertical"], color=border_color),
-            space=chars["empty"] * padding_horizontal,
-            content=_visual_center(line, max_width),
-        )
-        for line in lines
-    ]
-
-    box_str = "{margin}{padding}{content}{padding}{margin}".format(
-        margin=colorize(margin, color=border_color),
-        padding="".join(padding_lines),
-        content="".join(content_lines),
-    )
-
-    logger.info(box_str)
-
-
-def level():
-    """Returns current log level."""
-    return logger.getEffectiveLevel()
-
-
-def set_level(level_name):
-    """Sets log level.
-
-    Args:
-        level_name (str): log level name. E.g. info, debug, warning, error,
-            critical.
-    """
-    if not level_name:
-        return
-
-    levels = {
-        "info": logging.INFO,
-        "debug": logging.DEBUG,
-        "warning": logging.WARNING,
-        "error": logging.ERROR,
-        "critical": logging.CRITICAL,
+    color_code = {
+        "DEBUG": colorama.Fore.BLUE,
+        "INFO": "",
+        "WARNING": colorama.Fore.YELLOW,
+        "ERROR": colorama.Fore.RED,
+        "CRITICAL": colorama.Fore.RED,
     }
 
-    logger.setLevel(levels.get(level_name))
-
-
-def be_quiet():
-    """Disables all messages except critical ones."""
-    logger.setLevel(logging.CRITICAL)
-
-
-def be_verbose():
-    """Enables all messages."""
-    logger.setLevel(logging.DEBUG)
-
-
-@contextmanager
-def verbose():
-    """Enables verbose mode for the context."""
-    previous_level = level()
-    be_verbose()
-    yield
-    logger.setLevel(previous_level)
-
-
-@contextmanager
-def quiet():
-    """Enables quiet mode for the context."""
-    previous_level = level()
-    be_quiet()
-    yield
-    logger.setLevel(previous_level)
-
-
-def is_quiet():
-    """Returns whether or not all messages except critical ones are
-    disabled.
-    """
-    return level() == logging.CRITICAL
-
-
-def is_verbose():
-    """Returns whether or not all messages are enabled."""
-    return level() == logging.DEBUG
-
-
-def colorize(message, color=None):
-    """Returns a message in a specified color."""
-    if not color:
-        return message
-
-    colors = {
-        "green": colorama.Fore.GREEN,
-        "yellow": colorama.Fore.YELLOW,
-        "blue": colorama.Fore.BLUE,
-        "red": colorama.Fore.RED,
-    }
-
-    return "{color}{message}{nc}".format(
-        color=colors.get(color, ""), message=message, nc=colorama.Fore.RESET
+    footer = (
+        "{yellow}Having any troubles?{nc}."
+        " Hit us up at {blue}https://dvc.org/support{nc},"
+        " we are always happy to help!"
+    ).format(
+        blue=colorama.Fore.BLUE,
+        nc=colorama.Fore.RESET,
+        yellow=colorama.Fore.YELLOW,
     )
 
+    def format(self, record):
+        if self._is_visible(record):
+            self._progress_aware()
 
-def _init_colorama():
+        if record.levelname == "INFO":
+            return record.msg
+
+        if record.levelname == "ERROR" or record.levelname == "CRITICAL":
+            exception, stack_trace = self._parse_exc(record.exc_info)
+
+            return (
+                "{color}{levelname}{nc}: {description}"
+                "{stack_trace}\n"
+                "\n"
+                "{footer}"
+            ).format(
+                color=self.color_code.get(record.levelname, ""),
+                nc=colorama.Fore.RESET,
+                levelname=record.levelname,
+                description=self._description(record.msg, exception),
+                msg=record.msg,
+                stack_trace=stack_trace,
+                footer=self.footer,
+            )
+
+        return "{color}{levelname}{nc}: {msg}".format(
+            color=self.color_code.get(record.levelname, ""),
+            nc=colorama.Fore.RESET,
+            levelname=record.levelname,
+            msg=record.msg,
+        )
+
+    def _current_level(self):
+        return logging.getLogger("dvc").getEffectiveLevel()
+
+    def _is_visible(self, record):
+        return record.levelno >= self._current_level()
+
+    def _description(self, message, exception):
+        description = ""
+
+        if exception and message:
+            description = "{message} - {exception}"
+        elif exception:
+            description = "{exception}"
+        elif message:
+            description = "{message}"
+
+        return description.format(message=message, exception=exception)
+
+    def _walk_exc(self, exc_info):
+        import traceback
+
+        buffer = StringIO()
+
+        traceback.print_exception(*exc_info, file=buffer)
+
+        exc = exc_info[1]
+        tb = buffer.getvalue()
+
+        exc_list = [str(exc)]
+        tb_list = [tb]
+
+        # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info
+        while hasattr(exc, "cause") and exc.cause:
+            exc_list.append(str(exc.cause))
+            if hasattr(exc, "cause_tb") and exc.cause_tb:
+                tb_list.insert(0, str(exc.cause_tb))
+            exc = exc.cause
+
+        return exc_list, tb_list
+
+    def _parse_exc(self, exc_info):
+        if not exc_info:
+            return (None, "")
+
+        exc_list, tb_list = self._walk_exc(exc_info)
+
+        exception = ": ".join(exc_list)
+
+        if self._current_level() == logging.DEBUG:
+            stack_trace = (
+                "\n" "{red}{line}{nc}\n" "{stack_trace}" "{red}{line}{nc}"
+            ).format(
+                red=colorama.Fore.RED,
+                nc=colorama.Fore.RESET,
+                line="-" * 60,
+                stack_trace="\n".join(tb_list),
+            )
+        else:
+            stack_trace = ""
+
+        return (exception, stack_trace)
+
+    def _progress_aware(self):
+        """Add a new line if progress bar hasn't finished"""
+        from dvc.progress import progress
+
+        if not progress.is_finished:
+            progress._print()
+        progress.clearln()
+
+
+def setup(level=logging.INFO):
     colorama.init()
 
-
-def set_default_level():
-    """Sets default log level."""
-    logger.setLevel(logging.INFO)
-
-
-def _add_handlers():
-    formatter = "%(message)s"
-
-    class _LogLevelFilter(logging.Filter):
-        # pylint: disable=too-few-public-methods
-        def filter(self, record):
-            return record.levelno <= logging.WARNING
-
-    sh_out = logging.StreamHandler(sys.stdout)
-    sh_out.setFormatter(logging.Formatter(formatter))
-    sh_out.setLevel(logging.DEBUG)
-    sh_out.addFilter(_LogLevelFilter())
-
-    sh_err = logging.StreamHandler(sys.stderr)
-    sh_err.setFormatter(logging.Formatter(formatter))
-    sh_err.setLevel(logging.ERROR)
-
-    logger.addHandler(sh_out)
-    logger.addHandler(sh_err)
-
-
-def _walk_exc(exc):
-    exc_list = [str(exc)]
-    tb_list = [traceback.format_exc()]
-
-    # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info.
-    while hasattr(exc, "cause") and exc.cause is not None:
-        exc_list.append(str(exc.cause))
-        if hasattr(exc, "cause_tb") and exc.cause_tb is not None:
-            tb_list.insert(0, str(exc.cause_tb))
-        exc = exc.cause
-
-    return (exc_list, tb_list)
-
-
-def _parse_exc():
-    exc = sys.exc_info()[1]
-    if not exc:
-        return (None, "")
-
-    exc_list, tb_list = _walk_exc(exc)
-
-    exception = ": ".join(exc_list)
-
-    if is_verbose():
-        stack_trace = "{line}\n{stack_trace}{line}\n".format(
-            line=colorize("-" * 60, color="red"),
-            stack_trace="\n".join(tb_list),
-        )
-    else:
-        stack_trace = ""
-
-    return (exception, stack_trace)
-
-
-def _description(message, exception):
-    if exception and message:
-        description = "{message} - {exception}"
-    elif exception:
-        description = "{exception}"
-    elif message:
-        description = "{message}"
-
-    return description.format(message=message, exception=exception)
-
-
-def _footer():
-    return "{phrase} Hit us up at {url}, we are always happy to help!".format(
-        phrase=colorize("Having any troubles?", "yellow"),
-        url=colorize("https://dvc.org/support", "blue"),
+    logging.config.dictConfig(
+        {
+            "version": 1,
+            "filters": {"exclude_errors": {"()": ExcludeErrorsFilter}},
+            "formatters": {"color": {"()": ColorFormatter}},
+            "handlers": {
+                "console": {
+                    "class": "logging.StreamHandler",
+                    "level": "DEBUG",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_errors"],
+                },
+                "console_errors": {
+                    "class": "logging.StreamHandler",
+                    "level": "ERROR",
+                    "formatter": "color",
+                    "stream": "ext://sys.stderr",
+                },
+            },
+            "loggers": {
+                "dvc": {
+                    "level": level,
+                    "handlers": ["console", "console_errors"],
+                }
+            },
+        }
     )
-
-
-def _visual_width(line):
-    """Get the the number of columns required to display a string"""
-
-    return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, "", line))
-
-
-def _visual_center(line, width):
-    """Center align string according to it's visual width"""
-
-    spaces = max(width - _visual_width(line), 0)
-    left_padding = int(spaces / 2)
-    right_padding = spaces - left_padding
-
-    return (left_padding * " ") + line + (right_padding * " ")
-
-
-logger = logging.getLogger("dvc")  # pylint: disable=invalid-name
-
-set_default_level()
-_add_handlers()
-_init_colorama()

dvc/main.py

@@ -2,11 +2,14 @@
 
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import logging
+
 from dvc.cli import parse_args
-from dvc.command.base import CmdBase
 from dvc.analytics import Analytics
-from dvc.exceptions import NotDvcProjectError, DvcParserError
+from dvc.exceptions import NotDvcRepoError, DvcParserError
+
+
+logger = logging.getLogger("dvc")
 
 
 def main(argv=None):
@@ -20,27 +23,32 @@ def main(argv=None):
     """
     args = None
     cmd = None
+
+    outerLogLevel = logger.level
     try:
         args = parse_args(argv)
 
-        # Init loglevel early in case we'll run
-        # into errors before setting it properly
-        CmdBase.set_loglevel(args)
+        if args.quiet:
+            logger.setLevel(logging.CRITICAL)
+
+        elif args.verbose:
+            logger.setLevel(logging.DEBUG)
 
         cmd = args.func(args)
-
         ret = cmd.run_cmd()
     except KeyboardInterrupt:
-        logger.error("interrupted by the user")
+        logger.exception("interrupted by the user")
         ret = 252
-    except NotDvcProjectError:
-        logger.error()
+    except NotDvcRepoError:
+        logger.exception("")
         ret = 253
     except DvcParserError:
         ret = 254
     except Exception:  # pylint: disable=broad-except
-        logger.error("unexpected error")
+        logger.exception("unexpected error")
         ret = 255
+    finally:
+        logger.setLevel(outerLogLevel)
 
     Analytics().send_cmd(cmd, args, ret)
 

dvc/output/__init__.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import schema
 
-from dvc.config import Config
+from dvc.path import Schemes
 from dvc.utils.compat import urlparse, str
 
 from dvc.output.base import OutputBase
@@ -26,45 +26,76 @@ OUTS = [
 ]
 
 OUTS_MAP = {
-    "hdfs": OutputHDFS,
-    "s3": OutputS3,
-    "gs": OutputGS,
-    "ssh": OutputSSH,
-    "local": OutputLOCAL,
+    Schemes.HDFS: OutputHDFS,
+    Schemes.S3: OutputS3,
+    Schemes.GS: OutputGS,
+    Schemes.SSH: OutputSSH,
+    Schemes.LOCAL: OutputLOCAL,
 }
 
-SCHEMA = {
-    OutputBase.PARAM_PATH: str,
-    # NOTE: currently there are only 3 possible checksum names:
-    #
-    #    1) md5 (LOCAL, SSH, GS);
-    #    2) etag (S3);
-    #    3) checksum (HDFS);
-    #
-    # so when a few types of outputs share the same name, we only need
-    # specify it once.
+# NOTE: currently there are only 3 possible checksum names:
+#
+#    1) md5 (LOCAL, SSH, GS);
+#    2) etag (S3);
+#    3) checksum (HDFS);
+#
+# so when a few types of outputs share the same name, we only need
+# specify it once.
+CHECKSUM_SCHEMA = {
     schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None),
     schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None),
     schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
-    schema.Optional(OutputBase.PARAM_CACHE): bool,
-    schema.Optional(OutputBase.PARAM_METRIC): OutputBase.METRIC_SCHEMA,
 }
 
+TAGS_SCHEMA = {schema.Optional(str): CHECKSUM_SCHEMA}
 
-def _get(stage, p, info, cache, metric):
+SCHEMA = CHECKSUM_SCHEMA.copy()
+SCHEMA[OutputBase.PARAM_PATH] = str
+SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)] = bool
+SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] = OutputBase.METRIC_SCHEMA
+SCHEMA[schema.Optional(OutputBase.PARAM_TAGS)] = TAGS_SCHEMA
+SCHEMA[schema.Optional(OutputBase.PARAM_PERSIST)] = bool
+
+
+def _get(stage, p, info, cache, metric, persist=False, tags=None):
     parsed = urlparse(p)
+
     if parsed.scheme == "remote":
-        name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
-        sect = stage.project.config.config[name]
-        remote = Remote(stage.project, sect)
+        settings = stage.repo.config.get_remote_settings(parsed.netloc)
+        remote = Remote(stage.repo, settings)
         return OUTS_MAP[remote.scheme](
-            stage, p, info, cache=cache, remote=remote, metric=metric
+            stage,
+            p,
+            info,
+            cache=cache,
+            remote=remote,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
 
     for o in OUTS:
         if o.supported(p):
-            return o(stage, p, info, cache=cache, remote=None, metric=metric)
-    return OutputLOCAL(stage, p, info, cache=cache, remote=None, metric=metric)
+            return o(
+                stage,
+                p,
+                info,
+                cache=cache,
+                remote=None,
+                metric=metric,
+                persist=persist,
+                tags=tags,
+            )
+    return OutputLOCAL(
+        stage,
+        p,
+        info,
+        cache=cache,
+        remote=None,
+        metric=metric,
+        persist=persist,
+        tags=tags,
+    )
 
 
 def loadd_from(stage, d_list):
@@ -73,12 +104,33 @@ def loadd_from(stage, d_list):
         p = d.pop(OutputBase.PARAM_PATH)
         cache = d.pop(OutputBase.PARAM_CACHE, True)
         metric = d.pop(OutputBase.PARAM_METRIC, False)
-        ret.append(_get(stage, p, info=d, cache=cache, metric=metric))
+        persist = d.pop(OutputBase.PARAM_PERSIST, False)
+        tags = d.pop(OutputBase.PARAM_TAGS, None)
+        ret.append(
+            _get(
+                stage,
+                p,
+                info=d,
+                cache=cache,
+                metric=metric,
+                persist=persist,
+                tags=tags,
+            )
+        )
     return ret
 
 
-def loads_from(stage, s_list, use_cache=True, metric=False):
+def loads_from(stage, s_list, use_cache=True, metric=False, persist=False):
     ret = []
     for s in s_list:
-        ret.append(_get(stage, s, info={}, cache=use_cache, metric=metric))
+        ret.append(
+            _get(
+                stage,
+                s,
+                info={},
+                cache=use_cache,
+                metric=metric,
+                persist=persist,
+            )
+        )
     return ret

dvc/output/base.py

@@ -1,10 +1,17 @@
 from __future__ import unicode_literals
-from dvc.utils.compat import str
 
 import re
+import logging
+from copy import copy
+
 from schema import Or, Optional
 
 from dvc.exceptions import DvcException
+from dvc.utils.compat import str
+from dvc.remote.base import RemoteBase
+
+
+logger = logging.getLogger(__name__)
 
 
 class OutputDoesNotExistError(DvcException):
@@ -28,13 +35,14 @@ class OutputAlreadyTrackedError(DvcException):
 class OutputBase(object):
     IS_DEPENDENCY = False
 
-    REMOTE = None
+    REMOTE = RemoteBase
 
     PARAM_PATH = "path"
     PARAM_CACHE = "cache"
     PARAM_METRIC = "metric"
     PARAM_METRIC_TYPE = "type"
     PARAM_METRIC_XPATH = "xpath"
+    PARAM_PERSIST = "persist"
 
     METRIC_SCHEMA = Or(
         None,
@@ -45,24 +53,33 @@ class OutputBase(object):
         },
     )
 
+    PARAM_TAGS = "tags"
+
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         self.stage = stage
-        self.project = stage.project
+        self.repo = stage.repo
         self.url = path
         self.info = info
-        self.remote = remote or self.REMOTE(self.project, {})
+        self.remote = remote or self.REMOTE(self.repo, {})
         self.use_cache = False if self.IS_DEPENDENCY else cache
         self.metric = False if self.IS_DEPENDENCY else metric
+        self.persist = persist
+        self.tags = None if self.IS_DEPENDENCY else (tags or {})
 
-        if (
-            self.use_cache
-            and getattr(self.project.cache, self.REMOTE.scheme) is None
-        ):
+        if self.use_cache and self.cache is None:
             raise DvcException(
                 "no cache location setup for '{}' outputs.".format(
                     self.REMOTE.scheme
@@ -77,6 +94,23 @@ class OutputBase(object):
     def __str__(self):
         return self.url
 
+    @property
+    def is_in_repo(self):
+        return False
+
+    @property
+    def cache(self):
+        return getattr(self.repo.cache, self.scheme)
+
+    @property
+    def dir_cache(self):
+        return self.cache.get_dir_cache(self.checksum)
+
+    def assign_to_stage_file(self, target_repo):
+        raise DvcException(
+            "change repo is not supported for {}".format(self.scheme)
+        )
+
     @classmethod
     def match(cls, url):
         return re.match(cls.REMOTE.REGEX, url)
@@ -97,7 +131,11 @@ class OutputBase(object):
 
     @property
     def path(self):
-        return self.path_info["path"]
+        return self.path_info.path
+
+    @property
+    def cache_path(self):
+        return self.cache.checksum_to_path(self.checksum)
 
     @property
     def sep(self):
@@ -107,45 +145,103 @@ class OutputBase(object):
     def checksum(self):
         return self.info.get(self.remote.PARAM_CHECKSUM)
 
+    @property
+    def is_dir_checksum(self):
+        return self.remote.is_dir_checksum(self.checksum)
+
     @property
     def exists(self):
         return self.remote.exists(self.path_info)
 
-    def changed(self):
-        if not self.exists:
-            return True
-
-        if not self.use_cache:
-            return self.info != self.remote.save_info(self.path_info)
-
-        return getattr(self.project.cache, self.scheme).changed(
-            self.path_info, self.info
+    def changed_checksum(self):
+        return (
+            self.checksum
+            != self.remote.save_info(self.path_info)[
+                self.remote.PARAM_CHECKSUM
+            ]
         )
 
     def changed_cache(self):
         if not self.use_cache or not self.checksum:
             return True
 
-        cache = self.project.cache.__getattribute__(self.scheme)
-
-        return cache.changed_cache(self.checksum)
+        return self.cache.changed_cache(self.checksum)
 
     def status(self):
-        if self.changed():
-            # FIXME better msgs
-            return {str(self): "changed"}
+        if self.checksum and self.use_cache and self.changed_cache():
+            return {str(self): "not in cache"}
+
+        if not self.exists:
+            return {str(self): "deleted"}
+
+        if self.changed_checksum():
+            return {str(self): "modified"}
+
+        if not self.checksum:
+            return {str(self): "new"}
+
         return {}
 
+    def changed(self):
+        status = self.status()
+        logger.debug(str(status))
+        return bool(status)
+
+    @property
+    def is_empty(self):
+        return self.remote.is_empty(self.path_info)
+
+    def isdir(self):
+        return self.remote.isdir(self.path_info)
+
+    def isfile(self):
+        return self.remote.isfile(self.path_info)
+
     def save(self):
+        if not self.exists:
+            raise self.DoesNotExistError(self)
+
+        if not self.isfile and not self.isdir:
+            raise self.IsNotFileOrDirError(self)
+
+        if self.is_empty:
+            logger.warning("'{}' is empty.".format(self))
+
         if not self.use_cache:
             self.info = self.remote.save_info(self.path_info)
-        else:
-            self.info = getattr(self.project.cache, self.scheme).save(
-                self.path_info
+            if self.metric:
+                self.verify_metric()
+            if not self.IS_DEPENDENCY:
+                logger.info(
+                    "Output '{}' doesn't use cache. Skipping saving.".format(
+                        self
+                    )
+                )
+            return
+
+        assert not self.IS_DEPENDENCY
+
+        if not self.changed():
+            logger.info(
+                "Output '{}' didn't change. Skipping saving.".format(self)
             )
+            return
+
+        if self.is_in_repo:
+            if self.repo.scm.is_tracked(self.path):
+                raise OutputAlreadyTrackedError(self)
+
+            if self.use_cache:
+                self.repo.scm.ignore(self.path)
+
+        self.info = self.remote.save_info(self.path_info)
+
+    def commit(self):
+        if self.use_cache:
+            self.cache.save(self.path_info, self.info)
 
     def dumpd(self):
-        ret = self.info.copy()
+        ret = copy(self.info)
         ret[self.PARAM_PATH] = self.url
 
         if self.IS_DEPENDENCY:
@@ -161,18 +257,35 @@ class OutputBase(object):
                 del self.metric[self.PARAM_METRIC_XPATH]
 
         ret[self.PARAM_METRIC] = self.metric
+        ret[self.PARAM_PERSIST] = self.persist
+
+        if self.tags:
+            ret[self.PARAM_TAGS] = self.tags
 
         return ret
 
+    def verify_metric(self):
+        raise DvcException(
+            "verify metric is not supported for {}".format(self.scheme)
+        )
+
     def download(self, to_info, resume=False):
         self.remote.download([self.path_info], [to_info], resume=resume)
 
-    def checkout(self, force=False):
+    def checkout(self, force=False, progress_callback=None, tag=None):
         if not self.use_cache:
             return
 
-        getattr(self.project.cache, self.scheme).checkout(
-            self.path_info, self.info, force=force
+        if tag:
+            info = self.tags[tag]
+        else:
+            info = self.info
+
+        self.cache.checkout(
+            self.path_info,
+            info,
+            force=force,
+            progress_callback=progress_callback,
         )
 
     def remove(self, ignore_remove=False):
@@ -180,17 +293,31 @@ class OutputBase(object):
         if self.scheme != "local":
             return
 
-        if ignore_remove and self.use_cache and self.is_local:
-            self.project.scm.ignore_remove(self.path)
+        if ignore_remove and self.use_cache and self.is_in_repo:
+            self.repo.scm.ignore_remove(self.path)
 
     def move(self, out):
-        if self.scheme == "local" and self.use_cache and self.is_local:
-            self.project.scm.ignore_remove(self.path)
+        if self.scheme == "local" and self.use_cache and self.is_in_repo:
+            self.repo.scm.ignore_remove(self.path)
 
         self.remote.move(self.path_info, out.path_info)
         self.url = out.url
-        self.path_info = out.path_info
+        self.path_info = copy(out.path_info)
         self.save()
+        self.commit()
 
-        if self.scheme == "local" and self.use_cache and self.is_local:
-            self.project.scm.ignore(self.path)
+        if self.scheme == "local" and self.use_cache and self.is_in_repo:
+            self.repo.scm.ignore(self.path)
+
+    def get_files_number(self):
+        if not self.use_cache or not self.checksum:
+            return 0
+
+        if self.is_dir_checksum:
+            return len(self.dir_cache)
+
+        return 1
+
+    def unprotect(self):
+        if self.exists:
+            self.remote.unprotect(self.path_info)

dvc/output/hdfs.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import posixpath
 
+from dvc.path.hdfs import HDFSPathInfo
 from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
 from dvc.remote.hdfs import RemoteHDFS
@@ -11,12 +12,27 @@ class OutputHDFS(OutputBase):
     REMOTE = RemoteHDFS
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         super(OutputHDFS, self).__init__(
-            stage, path, info=info, remote=remote, cache=cache, metric=metric
+            stage,
+            path,
+            info=info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
         if remote:
             path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))
         user = remote.user if remote else self.group("user")
-        self.path_info = {"scheme": "hdfs", "user": user, "path": path}
+        self.path_info = HDFSPathInfo(user=user, url=self.url, path=path)

dvc/output/local.py

@@ -1,23 +1,42 @@
 from __future__ import unicode_literals
 
 import os
+import logging
 
-import dvc.logger as logger
+from dvc.path.local import LocalPathInfo
 from dvc.utils.compat import urlparse
 from dvc.istextfile import istextfile
 from dvc.exceptions import DvcException
 from dvc.remote.local import RemoteLOCAL
-from dvc.output.base import OutputBase, OutputAlreadyTrackedError
+from dvc.output.base import OutputBase
+
+
+logger = logging.getLogger(__name__)
 
 
 class OutputLOCAL(OutputBase):
     REMOTE = RemoteLOCAL
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         super(OutputLOCAL, self).__init__(
-            stage, path, info, remote=remote, cache=cache, metric=metric
+            stage,
+            path,
+            info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
         if remote:
             p = os.path.join(
@@ -28,20 +47,31 @@ class OutputLOCAL(OutputBase):
 
         if not os.path.isabs(p):
             p = self.remote.to_ospath(p)
-            p = os.path.join(stage.cwd, p)
+            p = os.path.join(stage.wdir, p)
         p = os.path.abspath(os.path.normpath(p))
 
-        self.path_info = {"scheme": "local", "path": p}
+        self.path_info = LocalPathInfo(url=self.url, path=p)
 
     def __str__(self):
         return self.rel_path
 
     @property
-    def is_local(self):
+    def is_in_repo(self):
         return urlparse(self.url).scheme != "remote" and not os.path.isabs(
             self.url
         )
 
+    def assign_to_stage_file(self, stage):
+        from dvc.repo import Repo
+
+        fullpath = os.path.abspath(stage.wdir)
+        self.path_info.path = os.path.join(fullpath, self.stage_path)
+
+        self.repo = Repo(self.path)
+
+        self.stage = stage
+        return self
+
     @property
     def sep(self):
         return os.sep
@@ -51,14 +81,14 @@ class OutputLOCAL(OutputBase):
         return os.path.relpath(self.path)
 
     @property
-    def cache(self):
-        return self.project.cache.local.get(self.checksum)
+    def stage_path(self):
+        return os.path.relpath(self.path, self.stage.wdir)
 
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
-        if self.is_local:
-            path = self.remote.unixpath(
-                os.path.relpath(self.path, self.stage.cwd)
+        if self.is_in_repo:
+            path = self.remote.to_posixpath(
+                os.path.relpath(self.path, self.stage.wdir)
             )
         else:
             path = self.url
@@ -67,7 +97,7 @@ class OutputLOCAL(OutputBase):
 
         return ret
 
-    def _verify_metric(self):
+    def verify_metric(self):
         if not self.metric:
             return
 
@@ -81,40 +111,3 @@ class OutputLOCAL(OutputBase):
         if not istextfile(self.path):
             msg = "binary file '{}' cannot be used as metrics."
             raise DvcException(msg.format(self.rel_path))
-
-    def save(self):
-        if not os.path.exists(self.path):
-            raise self.DoesNotExistError(self.rel_path)
-
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
-            raise self.IsNotFileOrDirError(self.rel_path)
-
-        if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or (
-            os.path.isdir(self.path) and len(os.listdir(self.path)) == 0
-        ):
-            msg = "file/directory '{}' is empty.".format(self.rel_path)
-            logger.warning(msg)
-
-        if not self.use_cache:
-            self.info = self.remote.save_info(self.path_info)
-            self._verify_metric()
-            if not self.IS_DEPENDENCY:
-                msg = "Output '{}' doesn't use cache. Skipping saving."
-                logger.info(msg.format(self.rel_path))
-            return
-
-        assert not self.IS_DEPENDENCY
-
-        if not self.changed():
-            msg = "Output '{}' didn't change. Skipping saving."
-            logger.info(msg.format(self.rel_path))
-            return
-
-        if self.is_local:
-            if self.project.scm.is_tracked(self.path):
-                raise OutputAlreadyTrackedError(self.rel_path)
-
-            if self.use_cache:
-                self.project.scm.ignore(self.path)
-
-        self.info = self.project.cache.local.save(self.path_info)

dvc/output/s3.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import posixpath
 
+from dvc.path.utils import PathInfo
 from dvc.remote.s3 import RemoteS3
 from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
@@ -11,17 +12,31 @@ class OutputS3(OutputBase):
     REMOTE = RemoteS3
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         super(OutputS3, self).__init__(
-            stage, path, info=info, remote=remote, cache=cache, metric=metric
+            stage,
+            path,
+            info=info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
         bucket = remote.bucket if remote else urlparse(path).netloc
         path = urlparse(path).path.lstrip("/")
         if remote:
             path = posixpath.join(remote.prefix, path)
-        self.path_info = {
-            "scheme": self.scheme,
-            "bucket": bucket,
-            "path": path,
-        }
+
+        self.path_info = PathInfo(
+            self.scheme, bucket=bucket, path=path, url=self.url
+        )

dvc/output/ssh.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import getpass
 import posixpath
 
+from dvc.path.ssh import SSHPathInfo
 from dvc.utils.compat import urlparse
 from dvc.output.base import OutputBase
 from dvc.remote.ssh import RemoteSSH
@@ -12,10 +13,25 @@ class OutputSSH(OutputBase):
     REMOTE = RemoteSSH
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         super(OutputSSH, self).__init__(
-            stage, path, info=info, remote=remote, cache=cache, metric=metric
+            stage,
+            path,
+            info=info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
         parsed = urlparse(path)
         host = remote.host if remote else parsed.hostname
@@ -33,10 +49,6 @@ class OutputSSH(OutputBase):
         else:
             path = parsed.path
 
-        self.path_info = {
-            "scheme": "ssh",
-            "host": host,
-            "port": port,
-            "user": user,
-            "path": path,
-        }
+        self.path_info = SSHPathInfo(
+            host=host, user=user, port=port, url=self.url, path=path
+        )

dvc/path/__init__.py

@@ -0,0 +1,34 @@
+from dvc.utils.compat import urlunsplit
+
+
+class Schemes:
+    SSH = "ssh"
+    HDFS = "hdfs"
+    S3 = "s3"
+    AZURE = "azure"
+    HTTP = "http"
+    GS = "gs"
+    LOCAL = "local"
+    OSS = "oss"
+
+
+class BasePathInfo(object):
+    scheme = None
+
+    def __init__(self, url=None, path=None):
+        self.url = url
+        self.path = path
+
+    def __str__(self):
+        return self.url
+
+
+class DefaultCloudPathInfo(BasePathInfo):
+    def __init__(self, bucket, url=None, path=None):
+        super(DefaultCloudPathInfo, self).__init__(url, path)
+        self.bucket = bucket
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit((self.scheme, self.bucket, self.path, "", ""))
+        return self.url

dvc/path/azure.py

@@ -0,0 +1,5 @@
+from dvc.path import Schemes, DefaultCloudPathInfo
+
+
+class AzurePathInfo(DefaultCloudPathInfo):
+    scheme = Schemes.AZURE

dvc/path/gs.py

@@ -0,0 +1,5 @@
+from dvc.path import Schemes, DefaultCloudPathInfo
+
+
+class GSPathInfo(DefaultCloudPathInfo):
+    scheme = Schemes.GS

dvc/path/hdfs.py

@@ -0,0 +1,15 @@
+from dvc.utils.compat import urlunsplit
+from dvc.path import BasePathInfo, Schemes
+
+
+class HDFSPathInfo(BasePathInfo):
+    scheme = Schemes.HDFS
+
+    def __init__(self, user, url=None, path=None):
+        super(HDFSPathInfo, self).__init__(url, path)
+        self.user = user
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit((self.scheme, self.user, self.path, "", ""))
+        return self.url

dvc/path/http.py

@@ -0,0 +1,16 @@
+from dvc.path import BasePathInfo, Schemes
+from dvc.utils.compat import urlparse, urlunsplit
+
+
+class HTTPPathInfo(BasePathInfo):
+    @property
+    def scheme(self):
+        if self.path:
+            return urlparse(self.path).scheme
+        else:
+            return Schemes.HTTP
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit((self.scheme, self.path, "", "", ""))
+        return self.url

dvc/path/local.py

@@ -0,0 +1,10 @@
+import os
+
+from dvc.path import BasePathInfo, Schemes
+
+
+class LocalPathInfo(BasePathInfo):
+    scheme = Schemes.LOCAL
+
+    def __str__(self):
+        return os.path.relpath(self.path)

dvc/path/oss.py

@@ -0,0 +1,5 @@
+from dvc.path import Schemes, DefaultCloudPathInfo
+
+
+class OSSPathInfo(DefaultCloudPathInfo):
+    scheme = Schemes.OSS

dvc/path/s3.py

@@ -0,0 +1,5 @@
+from dvc.path import Schemes, DefaultCloudPathInfo
+
+
+class S3PathInfo(DefaultCloudPathInfo):
+    scheme = Schemes.S3

dvc/path/ssh.py

@@ -0,0 +1,25 @@
+from dvc.utils.compat import urlunsplit
+from dvc.path import BasePathInfo, Schemes
+
+
+class SSHPathInfo(BasePathInfo):
+    scheme = Schemes.SSH
+
+    def __init__(self, host, user, port, url=None, path=None):
+        super(SSHPathInfo, self).__init__(url, path)
+        self.host = host
+        self.user = user
+        self.port = port
+
+    def __str__(self):
+        if not self.url:
+            return urlunsplit(
+                (
+                    self.scheme,
+                    "{}@{}:{}".format(self.user, self.host, self.port),
+                    self.path,
+                    "",
+                    "",
+                )
+            )
+        return self.url

dvc/path/utils.py

@@ -0,0 +1,25 @@
+from dvc.path import Schemes
+from dvc.path.azure import AzurePathInfo
+from dvc.path.gs import GSPathInfo
+from dvc.path.hdfs import HDFSPathInfo
+from dvc.path.http import HTTPPathInfo
+from dvc.path.local import LocalPathInfo
+from dvc.path.oss import OSSPathInfo
+from dvc.path.s3 import S3PathInfo
+from dvc.path.ssh import SSHPathInfo
+
+PATH_MAP = {
+    Schemes.SSH: SSHPathInfo,
+    Schemes.HDFS: HDFSPathInfo,
+    Schemes.S3: S3PathInfo,
+    Schemes.AZURE: AzurePathInfo,
+    Schemes.HTTP: HTTPPathInfo,
+    Schemes.GS: GSPathInfo,
+    Schemes.LOCAL: LocalPathInfo,
+    Schemes.OSS: OSSPathInfo,
+}
+
+
+def PathInfo(scheme, *args, **kwargs):
+    cls = PATH_MAP[scheme]
+    return cls(*args, **kwargs)

dvc/progress.py

@@ -1,4 +1,4 @@
-"""Manages progress bars for dvc project."""
+"""Manages progress bars for dvc repo."""
 
 from __future__ import print_function
 from __future__ import unicode_literals
@@ -7,7 +7,8 @@ from dvc.utils.compat import str
 
 import sys
 import threading
-import dvc.logger as logger
+
+CLEARLINE_PATTERN = "\r\x1b[K"
 
 
 class Progress(object):
@@ -31,14 +32,20 @@ class Progress(object):
         """Returns if all targets have finished."""
         return self._n_total == self._n_finished
 
-    def _clearln(self):
-        self._print("\r\x1b[K", end="")
+    def clearln(self):
+        self._print(CLEARLINE_PATTERN, end="")
 
     def _writeln(self, line):
-        self._clearln()
+        self.clearln()
         self._print(line, end="")
         sys.stdout.flush()
 
+    def reset(self):
+        with self._lock:
+            self._n_total = 0
+            self._n_finished = 0
+            self._line = None
+
     def refresh(self, line=None):
         """Refreshes progress bar."""
         # Just go away if it is locked. Will update next time
@@ -65,7 +72,7 @@ class Progress(object):
             pbar = self._bar(name, 100, 100)
 
             if sys.stdout.isatty():
-                self._clearln()
+                self.clearln()
 
             self._print(pbar)
 
@@ -100,7 +107,11 @@ class Progress(object):
 
     @staticmethod
     def _print(*args, **kwargs):
-        if logger.is_quiet():
+        import logging
+
+        logger = logging.getLogger(__name__)
+
+        if logger.getEffectiveLevel() == "CRITICAL":
             return
 
         print(*args, **kwargs)
@@ -108,12 +119,36 @@ class Progress(object):
     def __enter__(self):
         self._lock.acquire(True)
         if self._line is not None:
-            self._clearln()
+            self.clearln()
 
     def __exit__(self, typ, value, tbck):
         if self._line is not None:
             self.refresh()
         self._lock.release()
 
+    def __call__(self, seq, name="", total=None):
+        if total is None:
+            total = len(seq)
+
+        self.update_target(name, 0, total)
+        for done, item in enumerate(seq, start=1):
+            yield item
+            self.update_target(name, done, total)
+        self.finish_target(name)
+
+
+class ProgressCallback(object):
+    def __init__(self, total):
+        self.total = total
+        self.current = 0
+        progress.reset()
+
+    def update(self, name, progress_to_add=1):
+        self.current += progress_to_add
+        progress.update_target(name, self.current, self.total)
+
+    def finish(self, name):
+        progress.finish_target(name)
+
 
 progress = Progress()  # pylint: disable=invalid-name

dvc/prompt.py

@@ -4,11 +4,15 @@ from __future__ import unicode_literals
 from __future__ import print_function
 
 import sys
+import logging
 from getpass import getpass
 
 from dvc.utils.compat import input
 
 
+logger = logging.getLogger(__name__)
+
+
 def _ask(prompt, limited_to=None):
     if not sys.stdout.isatty():
         return None
@@ -25,10 +29,9 @@ def _ask(prompt, limited_to=None):
         if answer in limited_to:
             return answer
 
-        print(
-            "Your response must be one of: {options}. Please try again.".format(
-                options=limited_to
-            )
+        logger.info(
+            "Your response must be one of: {options}. "
+            "Please try again.".format(options=limited_to)
         )
 
 
@@ -36,7 +39,7 @@ def confirm(statement):
     """Ask the user for confirmation about the specified statement.
 
     Args:
-        statement (str): statement to ask the user confirmation about.
+        statement (unicode): statement to ask the user confirmation about.
 
     Returns:
         bool: whether or not specified statement was confirmed.
@@ -55,5 +58,5 @@ def password(statement):
     Returns:
         str: password entered by the user.
     """
-    prompt = "{statement}: ".format(statement=statement)
-    return getpass(prompt)
+    logger.info("{statement}: ".format(statement=statement))
+    return getpass("")

dvc/remote/__init__.py

@@ -7,6 +7,7 @@ from dvc.remote.local import RemoteLOCAL
 from dvc.remote.s3 import RemoteS3
 from dvc.remote.ssh import RemoteSSH
 from dvc.remote.http import RemoteHTTP
+from dvc.remote.oss import RemoteOSS
 
 
 REMOTES = [
@@ -16,6 +17,7 @@ REMOTES = [
     RemoteHTTP,
     RemoteS3,
     RemoteSSH,
+    RemoteOSS,
     # NOTE: RemoteLOCAL is the default
 ]
 
@@ -27,5 +29,5 @@ def _get(config):
     return RemoteLOCAL
 
 
-def Remote(project, config):
-    return _get(config)(project, config)
+def Remote(repo, config):
+    return _get(config)(repo, config)

dvc/remote/azure.py

@@ -3,19 +3,27 @@ from __future__ import unicode_literals
 
 import os
 import re
+import logging
+
+from dvc.path import Schemes
+from dvc.path.azure import AzurePathInfo
 
 try:
     from azure.storage.blob import BlockBlobService
+    from azure.common import AzureMissingResourceHttpError
 except ImportError:
     BlockBlobService = None
 
-import dvc.logger as logger
-from dvc.utils.compat import urlparse
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
 
 
+logger = logging.getLogger(__name__)
+
+
 class Callback(object):
     def __init__(self, name):
         self.name = name
@@ -25,7 +33,7 @@ class Callback(object):
 
 
 class RemoteAzure(RemoteBase):
-    scheme = "azure"
+    scheme = Schemes.AZURE
     REGEX = (
         r"azure://((?P<path>[^=;]*)?|("
         # backward compatibility
@@ -36,8 +44,8 @@ class RemoteAzure(RemoteBase):
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
 
-    def __init__(self, project, config):
-        super(RemoteAzure, self).__init__(project, config)
+    def __init__(self, repo, config):
+        super(RemoteAzure, self).__init__(repo, config)
 
         self.url = config.get(Config.SECTION_REMOTE_URL, "azure://")
         match = re.match(self.REGEX, self.url)  # backward compatibility
@@ -64,8 +72,7 @@ class RemoteAzure(RemoteBase):
             raise ValueError("azure storage connection string missing")
 
         self.__blob_service = None
-
-        self.path_info = {"scheme": self.scheme, "bucket": self.bucket}
+        self.path_info = AzurePathInfo(bucket=self.bucket)
 
     @property
     def blob_service(self):
@@ -76,20 +83,23 @@ class RemoteAzure(RemoteBase):
                 connection_string=self.connection_string
             )
             logger.debug("Container name {}".format(self.bucket))
-            self.__blob_service.create_container(self.bucket)
+            try:  # verify that container exists
+                self.__blob_service.list_blobs(
+                    self.bucket, delimiter="/", num_results=1
+                )
+            except AzureMissingResourceHttpError:
+                self.__blob_service.create_container(self.bucket)
         return self.__blob_service
 
     def remove(self, path_info):
-        if path_info["scheme"] != self.scheme:
+        if path_info.scheme != self.scheme:
             raise NotImplementedError
 
         logger.debug(
-            "Removing azure://{}/{}".format(
-                path_info["bucket"], path_info["path"]
-            )
+            "Removing azure://{}/{}".format(path_info.bucket, path_info.path)
         )
 
-        self.blob_service.delete_blob(path_info["bucket"], path_info["path"])
+        self.blob_service.delete_blob(path_info.bucket, path_info.path)
 
     def _list_paths(self, bucket, prefix):
         blob_service = self.blob_service
@@ -110,36 +120,36 @@ class RemoteAzure(RemoteBase):
     def list_cache_paths(self):
         return self._list_paths(self.bucket, self.prefix)
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info["scheme"] != self.scheme:
+            if to_info.scheme != self.scheme:
                 raise NotImplementedError
 
-            if from_info["scheme"] != "local":
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
-            bucket = to_info["bucket"]
-            path = to_info["path"]
+            bucket = to_info.bucket
+            path = to_info.path
 
             logger.debug(
                 "Uploading '{}' to '{}/{}'".format(
-                    from_info["path"], bucket, path
+                    from_info.path, bucket, path
                 )
             )
 
             if not name:
-                name = os.path.basename(from_info["path"])
+                name = os.path.basename(from_info.path)
 
-            cb = Callback(name)
+            cb = None if no_progress_bar else Callback(name)
 
             try:
                 self.blob_service.create_blob_from_path(
-                    bucket, path, from_info["path"], progress_callback=cb
+                    bucket, path, from_info.path, progress_callback=cb
                 )
             except Exception:
-                msg = "failed to upload '{}'".format(from_info["path"])
+                msg = "failed to upload '{}'".format(from_info.path)
                 logger.warning(msg)
             else:
                 progress.finish_target(name)
@@ -155,28 +165,28 @@ class RemoteAzure(RemoteBase):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != self.scheme:
+            if from_info.scheme != self.scheme:
                 raise NotImplementedError
 
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
-            bucket = from_info["bucket"]
-            path = from_info["path"]
+            bucket = from_info.bucket
+            path = from_info.path
 
             logger.debug(
                 "Downloading '{}/{}' to '{}'".format(
-                    bucket, path, to_info["path"]
+                    bucket, path, to_info.path
                 )
             )
 
-            tmp_file = self.tmp_file(to_info["path"])
+            tmp_file = tmp_fname(to_info.path)
             if not name:
-                name = os.path.basename(to_info["path"])
+                name = os.path.basename(to_info.path)
 
             cb = None if no_progress_bar else Callback(name)
 
-            self._makedirs(to_info["path"])
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
 
             try:
                 self.blob_service.get_blob_to_path(
@@ -186,7 +196,7 @@ class RemoteAzure(RemoteBase):
                 msg = "failed to download '{}/{}'".format(bucket, path)
                 logger.warning(msg)
             else:
-                os.rename(tmp_file, to_info["path"])
+                move(tmp_file, to_info.path)
 
                 if not no_progress_bar:
                     progress.finish_target(name)

dvc/remote/base.py

@@ -1,20 +1,33 @@
 from __future__ import unicode_literals
 
+from copy import copy
+
+from dvc.path.local import LocalPathInfo
 from dvc.utils.compat import str
 
 import os
 import re
-import errno
+import json
+import ntpath
+import logging
+import tempfile
 import posixpath
+from operator import itemgetter
 from multiprocessing import cpu_count
 
 import dvc.prompt as prompt
-import dvc.logger as logger
 from dvc.config import Config
 from dvc.exceptions import DvcException, ConfirmRemoveError
+from dvc.progress import progress
+from dvc.utils import LARGE_DIR_SIZE, tmp_fname
+from dvc.state import StateBase
+
+
+logger = logging.getLogger(__name__)
 
 
 STATUS_OK = 1
+STATUS_MISSING = 2
 STATUS_NEW = 3
 STATUS_DELETED = 4
 
@@ -22,7 +35,7 @@ STATUS_DELETED = 4
 STATUS_MAP = {
     # (local_exists, remote_exists)
     (True, True): STATUS_OK,
-    (False, False): STATUS_OK,
+    (False, False): STATUS_MISSING,
     (True, False): STATUS_NEW,
     (False, True): STATUS_DELETED,
 }
@@ -35,10 +48,18 @@ class DataCloudError(DvcException):
         super(DataCloudError, self).__init__("Data sync error: {}".format(msg))
 
 
-class RemoteBaseCmdError(DvcException):
-    def __init__(self, cmd, ret, err):
-        m = "SSH command '{}' finished with non-zero return code '{}': {}"
-        super(RemoteBaseCmdError, self).__init__(m.format(cmd, ret, err))
+class RemoteCmdError(DvcException):
+    def __init__(self, remote, cmd, ret, err):
+        super(RemoteCmdError, self).__init__(
+            "{remote} command '{cmd}' finished with non-zero return code"
+            " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
+        )
+
+
+class RemoteActionNotImplemented(DvcException):
+    def __init__(self, action, scheme):
+        m = "{} is not supported by {} remote".format(action, scheme)
+        super(RemoteActionNotImplemented, self).__init__(m)
 
 
 class RemoteMissingDepsError(DvcException):
@@ -51,8 +72,11 @@ class RemoteBase(object):
     REQUIRES = {}
     JOBS = 4 * cpu_count()
 
-    def __init__(self, project, config):
-        self.project = project
+    PARAM_RELPATH = "relpath"
+    CHECKSUM_DIR_SUFFIX = ".dir"
+
+    def __init__(self, repo, config):
+        self.repo = repo
         deps_ok = all(self.REQUIRES.values())
         if not deps_ok:
             missing = [k for k, v in self.REQUIRES.items() if v is None]
@@ -75,12 +99,15 @@ class RemoteBase(object):
                 "\n"
                 "If you have installed dvc from a binary package and you "
                 "are still seeing this message, please report it to us "
-                "using https://github.com/iterative/dvc/issues. Thank you!".format(
-                    url, missing, " ".join(missing), self.scheme
-                )
-            )
+                "using https://github.com/iterative/dvc/issues. Thank you!"
+            ).format(url, missing, " ".join(missing), self.scheme)
             raise RemoteMissingDepsError(msg)
 
+        self.protected = False
+        self.state = StateBase()
+
+        self._dir_info = {}
+
     def __repr__(self):
         return "{class_name}: '{url}'".format(
             class_name=type(self).__name__, url=(self.url or "No url")
@@ -94,6 +121,27 @@ class RemoteBase(object):
         url = config[Config.SECTION_REMOTE_URL]
         return cls.match(url) is not None
 
+    @staticmethod
+    def _replace_sep(path, from_sep, to_sep):
+        assert not ntpath.isabs(path)
+        assert not posixpath.isabs(path)
+        return path.replace(from_sep, to_sep)
+
+    @classmethod
+    def to_posixpath(cls, path):
+        return cls._replace_sep(path, "\\", "/")
+
+    @classmethod
+    def to_ntpath(cls, path):
+        return cls._replace_sep(path, "/", "\\")
+
+    def to_ospath(self, path):
+        if self.ospath == ntpath:
+            return self.to_ntpath(path)
+        elif self.ospath == posixpath:
+            return self.to_posixpath(path)
+        assert False
+
     @classmethod
     def match(cls, url):
         return re.match(cls.REGEX, url)
@@ -104,15 +152,136 @@ class RemoteBase(object):
             return None
         return m.group(name)
 
-    @staticmethod
-    def tmp_file(fname):
-        """ Temporary name for a partial download """
-        import uuid
+    @property
+    def cache(self):
+        return getattr(self.repo.cache, self.scheme)
 
-        return fname + "." + str(uuid.uuid4())
+    def get_file_checksum(self, path_info):
+        raise NotImplementedError
+
+    def _collect_dir(self, path_info):
+        dir_info = []
+
+        p_info = copy(path_info)
+        dpath = p_info.path
+        for root, dirs, files in self.walk(path_info):
+            if len(files) > LARGE_DIR_SIZE:
+                msg = (
+                    "Computing md5 for a large directory {}. "
+                    "This is only done once."
+                )
+                relpath = self.ospath.relpath(root)
+                logger.info(msg.format(relpath))
+                files = progress(files, name=relpath)
+
+            for fname in files:
+                path = self.ospath.join(root, fname)
+                p_info.path = path
+                relpath = self.to_posixpath(self.ospath.relpath(path, dpath))
+
+                checksum = self.get_file_checksum(p_info)
+                dir_info.append(
+                    {
+                        self.PARAM_RELPATH: relpath,
+                        self.PARAM_CHECKSUM: checksum,
+                    }
+                )
+
+        # NOTE: sorting the list by path to ensure reproducibility
+        return sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
+
+    def get_dir_checksum(self, path_info):
+        dir_info = self._collect_dir(path_info)
+        checksum, from_info, to_info = self._get_dir_info_checksum(
+            dir_info, path_info
+        )
+        if self.cache.changed_cache_file(checksum):
+            self.cache.move(from_info, to_info)
+
+        self.state.save(path_info, checksum)
+        self.state.save(to_info, checksum)
+
+        return checksum
+
+    def _get_dir_info_checksum(self, dir_info, path_info):
+        to_info = copy(path_info)
+        to_info.path = self.cache.ospath.join(self.cache.prefix, tmp_fname(""))
+
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        with open(tmp, "w+") as fobj:
+            json.dump(dir_info, fobj, sort_keys=True)
+
+        from_info = LocalPathInfo(path=tmp)
+        self.cache.upload([from_info], [to_info], no_progress_bar=True)
+
+        checksum = self.get_file_checksum(to_info) + self.CHECKSUM_DIR_SUFFIX
+        from_info = copy(to_info)
+        to_info.path = self.cache.checksum_to_path(checksum)
+        return checksum, from_info, to_info
+
+    def get_dir_cache(self, checksum):
+        assert checksum
+
+        dir_info = self._dir_info.get(checksum)
+        if dir_info:
+            return dir_info
+
+        dir_info = self.load_dir_cache(checksum)
+        self._dir_info[checksum] = dir_info
+        return dir_info
+
+    def load_dir_cache(self, checksum):
+        path_info = self.checksum_to_path_info(checksum)
+
+        fobj = tempfile.NamedTemporaryFile(delete=False)
+        path = fobj.name
+        to_info = LocalPathInfo(path=path)
+        self.cache.download([path_info], [to_info], no_progress_bar=True)
+
+        try:
+            with open(path, "r") as fobj:
+                d = json.load(fobj)
+        except ValueError:
+            logger.exception("Failed to load dir cache '{}'".format(path_info))
+            return []
+        finally:
+            os.unlink(path)
+
+        if not isinstance(d, list):
+            msg = "dir cache file format error '{}' [skipping the file]"
+            logger.error(msg.format(os.path.relpath(path)))
+            return []
+
+        for info in d:
+            info[self.PARAM_RELPATH] = self.to_ospath(info[self.PARAM_RELPATH])
+
+        return d
+
+    @classmethod
+    def is_dir_checksum(cls, checksum):
+        return checksum.endswith(cls.CHECKSUM_DIR_SUFFIX)
+
+    def get_checksum(self, path_info):
+        if not self.exists(path_info):
+            return None
+
+        checksum = self.state.get(path_info)
+        if checksum:
+            return checksum
+
+        if self.isdir(path_info):
+            checksum = self.get_dir_checksum(path_info)
+        else:
+            checksum = self.get_file_checksum(path_info)
+
+        if checksum:
+            self.state.save(path_info, checksum)
+
+        return checksum
 
     def save_info(self, path_info):
-        raise NotImplementedError
+        assert path_info.scheme == self.scheme
+        return {self.PARAM_CHECKSUM: self.get_checksum(path_info)}
 
     def changed(self, path_info, checksum_info):
         """Checks if data has changed.
@@ -164,9 +333,83 @@ class RemoteBase(object):
         logger.debug("'{}' hasn't changed.".format(path_info))
         return False
 
-    def save(self, path_info):
+    def link(self, from_info, to_info):
+        self.copy(from_info, to_info)
+
+    def _save_file(self, path_info, checksum, save_link=True):
+        assert checksum
+
+        cache_info = self.checksum_to_path_info(checksum)
+        if self.changed_cache(checksum):
+            self.move(path_info, cache_info)
+        else:
+            self.remove(path_info)
+
+        self.link(cache_info, path_info)
+
+        if save_link:
+            self.state.save_link(path_info)
+
+        # we need to update path and cache, since in case of reflink,
+        # or copy cache type moving original file results in updates on
+        # next executed command, which causes md5 recalculation
+        self.state.save(path_info, checksum)
+        self.state.save(cache_info, checksum)
+
+    def _save_dir(self, path_info, checksum):
+        cache_info = self.checksum_to_path_info(checksum)
+        dir_info = self.get_dir_cache(checksum)
+
+        entry_info = copy(path_info)
+        for entry in dir_info:
+            entry_checksum = entry[self.PARAM_CHECKSUM]
+            entry_info.path = self.ospath.join(
+                path_info.path, entry[self.PARAM_RELPATH]
+            )
+
+            self._save_file(entry_info, entry_checksum, save_link=False)
+
+        self.state.save_link(path_info)
+        self.state.save(cache_info, checksum)
+        self.state.save(path_info, checksum)
+
+    def is_empty(self, path_info):
+        return False
+
+    def isfile(self, path_info):
         raise NotImplementedError
 
+    def isdir(self, path_info):
+        return False
+
+    def walk(self, path_info):
+        raise NotImplementedError
+
+    def protect(self, path_info):
+        pass
+
+    def save(self, path_info, checksum_info):
+        if path_info.scheme != self.scheme:
+            raise RemoteActionNotImplemented(
+                "save {} -> {}".format(path_info.scheme, self.scheme),
+                self.scheme,
+            )
+
+        checksum = checksum_info[self.PARAM_CHECKSUM]
+        if not self.changed_cache(checksum):
+            self._checkout(path_info, checksum)
+            return
+
+        self._save(path_info, checksum)
+
+    def _save(self, path_info, checksum):
+        to_info = self.checksum_to_path_info(checksum)
+        logger.info("Saving '{}' to '{}'.".format(path_info, to_info))
+        if self.isdir(path_info):
+            self._save_dir(path_info, checksum)
+            return
+        self._save_file(path_info, checksum)
+
     def download(
         self,
         from_infos,
@@ -175,28 +418,20 @@ class RemoteBase(object):
         name=None,
         resume=False,
     ):
-        raise NotImplementedError
+        raise RemoteActionNotImplemented("download", self.scheme)
 
-    def upload(self, from_infos, to_infos, path_info, names=None):
-        raise NotImplementedError
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
+        raise RemoteActionNotImplemented("upload", self.scheme)
 
     def remove(self, path_info):
-        raise NotImplementedError
+        raise RemoteActionNotImplemented("remove", self.scheme)
 
-    def move(self, path_info):
-        raise NotImplementedError
+    def move(self, from_info, to_info):
+        self.copy(from_info, to_info)
+        self.remove(from_info)
 
     def copy(self, from_info, to_info):
-        raise NotImplementedError
-
-    def _makedirs(self, fname):
-        dname = os.path.dirname(fname)
-
-        try:
-            os.makedirs(dname)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
+        raise RemoteActionNotImplemented("copy", self.scheme)
 
     def exists(self, path_infos):
         raise NotImplementedError
@@ -220,6 +455,7 @@ class RemoteBase(object):
         return posixpath
 
     def checksum_to_path(self, checksum):
+        assert checksum
         return self.ospath.join(self.prefix, checksum[0:2], checksum[2:])
 
     def path_to_checksum(self, path):
@@ -227,8 +463,8 @@ class RemoteBase(object):
         return self.ospath.dirname(relpath) + self.ospath.basename(relpath)
 
     def checksum_to_path_info(self, checksum):
-        path_info = self.path_info.copy()
-        path_info["path"] = self.checksum_to_path(checksum)
+        path_info = copy(self.path_info)
+        path_info.path = self.checksum_to_path(checksum)
         return path_info
 
     def md5s_to_path_infos(self, md5s):
@@ -249,10 +485,13 @@ class RemoteBase(object):
     def gc(self, cinfos):
         from dvc.remote.local import RemoteLOCAL
 
-        used = [info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]]
+        used = {info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]}
 
         if self.scheme != "":
-            used += [info[self.PARAM_CHECKSUM] for info in cinfos[self.scheme]]
+            used |= {
+                info[self.PARAM_CHECKSUM]
+                for info in cinfos.get(self.scheme, [])
+            }
 
         removed = False
         for checksum in self.all():
@@ -263,24 +502,51 @@ class RemoteBase(object):
             removed = True
         return removed
 
-    def changed_cache(self, checksum):
-        cache = self.checksum_to_path_info(checksum)
-        expected = {self.PARAM_CHECKSUM: checksum}
-        actual = self.save_info(cache)
+    def changed_cache_file(self, checksum):
+        cache_info = self.checksum_to_path_info(checksum)
+        actual = self.get_checksum(cache_info)
 
         logger.debug(
-            "Cache '{}' actual '{}'.".format(str(expected), str(actual))
+            "cache '{}' expected '{}' actual '{}'".format(
+                str(cache_info), checksum, actual
+            )
         )
 
-        if expected != actual:
-            if self.exists(cache):
-                msg = "corrupted cache file {}"
-                logger.warning(msg.format(str(cache)))
-                self.remove(cache)
+        if not checksum or not actual:
             return True
 
+        if actual.split(".")[0] == checksum.split(".")[0]:
+            return False
+
+        if self.exists(cache_info):
+            logger.warning("corrupted cache file '{}'.".format(cache_info))
+            self.remove(cache_info)
+
+        return True
+
+    def _changed_cache_dir(self):
+        # NOTE: only implemented for RemoteLOCAL
+        return True
+
+    def _changed_dir_cache(self, checksum):
+        if not self._changed_cache_dir():
+            return False
+
+        if self.changed_cache_file(checksum):
+            return True
+
+        for entry in self.get_dir_cache(checksum):
+            checksum = entry[self.PARAM_CHECKSUM]
+            if self.changed_cache_file(checksum):
+                return True
+
         return False
 
+    def changed_cache(self, checksum):
+        if self.is_dir_checksum(checksum):
+            return self._changed_dir_cache(checksum)
+        return self.changed_cache_file(checksum)
+
     def cache_exists(self, checksums):
         # NOTE: The reason for such an odd logic is that most of the remotes
         # take much shorter time to just retrieve everything they have under
@@ -293,7 +559,7 @@ class RemoteBase(object):
         return list(filter(lambda checksum: checksum in checksums, self.all()))
 
     def already_cached(self, path_info):
-        current = self.save_info(path_info)[self.PARAM_CHECKSUM]
+        current = self.get_checksum(path_info)
 
         if not current:
             return False
@@ -315,29 +581,92 @@ class RemoteBase(object):
 
         self.remove(path_info)
 
-    def do_checkout(self, path_info, checksum, force=False):
+    def _checkout_file(
+        self, path_info, checksum, force, progress_callback=None
+    ):
+        cache_info = self.checksum_to_path_info(checksum)
         if self.exists(path_info):
             msg = "data '{}' exists. Removing before checkout."
             logger.warning(msg.format(str(path_info)))
             self.safe_remove(path_info, force=force)
 
-        from_info = self.checksum_to_path_info(checksum)
-        self.copy(from_info, path_info)
+        self.link(cache_info, path_info)
+        self.state.save_link(path_info)
+        if progress_callback:
+            progress_callback.update(path_info.url)
 
-    def checkout(self, path_info, checksum_info, force=False):
-        scheme = path_info["scheme"]
-        if scheme not in ["", "local"] and scheme != self.scheme:
+    def makedirs(self, path_info):
+        raise NotImplementedError
+
+    def _checkout_dir(
+        self, path_info, checksum, force, progress_callback=None
+    ):
+        # Create dir separately so that dir is created
+        # even if there are no files in it
+        if not self.exists(path_info):
+            self.makedirs(path_info)
+
+        dir_info = self.get_dir_cache(checksum)
+
+        logger.debug("Linking directory '{}'.".format(path_info))
+
+        entry_info = copy(path_info)
+        for entry in dir_info:
+            relpath = entry[self.PARAM_RELPATH]
+            checksum = entry[self.PARAM_CHECKSUM]
+            entry_cache_info = self.checksum_to_path_info(checksum)
+            entry_info.url = self.ospath.join(path_info.url, relpath)
+            entry_info.path = self.ospath.join(path_info.path, relpath)
+
+            entry_checksum_info = {self.PARAM_CHECKSUM: checksum}
+            if self.changed(entry_info, entry_checksum_info):
+                if self.exists(entry_info):
+                    self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+            if progress_callback:
+                progress_callback.update(entry_info.url)
+
+        self._remove_redundant_files(path_info, dir_info, force)
+
+        self.state.save_link(path_info)
+
+    def _remove_redundant_files(self, path_info, dir_info, force):
+        existing_files = set(
+            self.ospath.join(root, fname)
+            for root, _, files in self.walk(path_info)
+            for fname in files
+        )
+
+        needed_files = set(
+            self.ospath.join(path_info.path, entry[self.PARAM_RELPATH])
+            for entry in dir_info
+        )
+
+        delta = existing_files - needed_files
+
+        d_info = copy(path_info)
+        for path in delta:
+            d_info.path = path
+            self.safe_remove(d_info, force)
+
+    def checkout(
+        self, path_info, checksum_info, force=False, progress_callback=None
+    ):
+        if (
+            path_info.scheme not in ["local"]
+            and path_info.scheme != self.scheme
+        ):
             raise NotImplementedError
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
         if not checksum:
             msg = "No checksum info for '{}'."
-            logger.info(msg.format(str(path_info)))
+            logger.debug(msg.format(str(path_info)))
             return
 
         if not self.changed(path_info, checksum_info):
             msg = "Data '{}' didn't change."
-            logger.info(msg.format(str(path_info)))
+            logger.debug(msg.format(str(path_info)))
             return
 
         if self.changed_cache(checksum):
@@ -347,6 +676,21 @@ class RemoteBase(object):
             return
 
         msg = "Checking out '{}' with cache '{}'."
-        logger.info(msg.format(str(path_info), checksum))
+        logger.debug(msg.format(str(path_info), checksum))
 
-        self.do_checkout(path_info, checksum, force=force)
+        return self._checkout(path_info, checksum, force, progress_callback)
+
+    def _checkout(
+        self, path_info, checksum, force=False, progress_callback=None
+    ):
+        if not self.is_dir_checksum(checksum):
+            return self._checkout_file(
+                path_info, checksum, force, progress_callback=progress_callback
+            )
+        return self._checkout_dir(
+            path_info, checksum, force, progress_callback=progress_callback
+        )
+
+    @staticmethod
+    def unprotect(path_info):
+        pass

dvc/remote/gs.py

@@ -1,29 +1,35 @@
 from __future__ import unicode_literals
 
 import os
+import logging
+
+from dvc.path import Schemes
+from dvc.path.gs import GSPathInfo
 
 try:
     from google.cloud import storage
-    from google.auth.credentials import Credentials
 except ImportError:
     storage = None
 
-import dvc.logger as logger
-from dvc.utils.compat import urlparse
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
 from dvc.remote.base import RemoteBase
 from dvc.config import Config
 from dvc.progress import progress
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class RemoteGS(RemoteBase):
-    scheme = "gs"
+    scheme = Schemes.GS
     REGEX = r"^gs://(?P<path>.*)$"
     REQUIRES = {"google.cloud.storage": storage}
     PARAM_CHECKSUM = "md5"
 
-    def __init__(self, project, config):
-        super(RemoteGS, self).__init__(project, config)
+    def __init__(self, repo, config):
+        super(RemoteGS, self).__init__(repo, config)
         storagepath = "gs://"
         storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, "/")
         storagepath.lstrip("/")
@@ -35,7 +41,7 @@ class RemoteGS(RemoteBase):
         self.bucket = parsed.netloc
         self.prefix = parsed.path.lstrip("/")
 
-        self.path_info = {"scheme": "gs", "bucket": self.bucket}
+        self.path_info = GSPathInfo(bucket=self.bucket)
 
     @staticmethod
     def compat_config(config):
@@ -52,10 +58,12 @@ class RemoteGS(RemoteBase):
             else storage.Client(self.projectname)
         )
 
-    def get_md5(self, bucket, path):
+    def get_file_checksum(self, path_info):
         import base64
         import codecs
 
+        bucket = path_info.bucket
+        path = path_info.path
         blob = self.gs.bucket(bucket).get_blob(path)
         if not blob:
             return None
@@ -64,52 +72,28 @@ class RemoteGS(RemoteBase):
         md5 = base64.b64decode(b64_md5)
         return codecs.getencoder("hex")(md5)[0].decode("utf-8")
 
-    def save_info(self, path_info):
-        if path_info["scheme"] != "gs":
-            raise NotImplementedError
-
-        return {
-            self.PARAM_CHECKSUM: self.get_md5(
-                path_info["bucket"], path_info["path"]
-            )
-        }
-
     def copy(self, from_info, to_info, gs=None):
         gs = gs if gs else self.gs
 
-        blob = gs.bucket(from_info["bucket"]).get_blob(from_info["path"])
+        blob = gs.bucket(from_info.bucket).get_blob(from_info.path)
         if not blob:
-            msg = "'{}' doesn't exist in the cloud".format(from_info["path"])
+            msg = "'{}' doesn't exist in the cloud".format(from_info.path)
             raise DvcException(msg)
 
-        bucket = self.gs.bucket(to_info["bucket"])
+        bucket = self.gs.bucket(to_info.bucket)
         bucket.copy_blob(
-            blob, self.gs.bucket(to_info["bucket"]), new_name=to_info["path"]
+            blob, self.gs.bucket(to_info.bucket), new_name=to_info.path
         )
 
-    def save(self, path_info):
-        if path_info["scheme"] != "gs":
-            raise NotImplementedError
-
-        md5 = self.get_md5(path_info["bucket"], path_info["path"])
-        path = self.checksum_to_path(md5)
-        to_info = {"scheme": "gs", "bucket": self.bucket, "path": path}
-
-        self.copy(path_info, to_info)
-
-        return {self.PARAM_CHECKSUM: md5}
-
     def remove(self, path_info):
-        if path_info["scheme"] != "gs":
+        if path_info.scheme != "gs":
             raise NotImplementedError
 
         logger.debug(
-            "Removing gs://{}/{}".format(
-                path_info["bucket"], path_info["path"]
-            )
+            "Removing gs://{}/{}".format(path_info.bucket, path_info.path)
         )
 
-        blob = self.gs.bucket(path_info["bucket"]).get_blob(path_info["path"])
+        blob = self.gs.bucket(path_info.bucket).get_blob(path_info.path)
         if not blob:
             return
 
@@ -124,44 +108,43 @@ class RemoteGS(RemoteBase):
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info["scheme"] == "gs"
+        assert path_info.scheme == "gs"
 
-        paths = self._list_paths(path_info["bucket"], path_info["path"])
-        return any(path_info["path"] == path for path in paths)
+        paths = self._list_paths(path_info.bucket, path_info.path)
+        return any(path_info.path == path for path in paths)
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         gs = self.gs
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info["scheme"] != "gs":
+            if to_info.scheme != "gs":
                 raise NotImplementedError
 
-            if from_info["scheme"] != "local":
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
             logger.debug(
                 "Uploading '{}' to '{}/{}'".format(
-                    from_info["path"], to_info["bucket"], to_info["path"]
+                    from_info.path, to_info.bucket, to_info.path
                 )
             )
 
             if not name:
-                name = os.path.basename(from_info["path"])
+                name = os.path.basename(from_info.path)
 
-            progress.update_target(name, 0, None)
+            if not no_progress_bar:
+                progress.update_target(name, 0, None)
 
             try:
-                bucket = gs.bucket(to_info["bucket"])
-                blob = bucket.blob(to_info["path"])
-                blob.upload_from_filename(from_info["path"])
+                bucket = gs.bucket(to_info.bucket)
+                blob = bucket.blob(to_info.path)
+                blob.upload_from_filename(from_info.path)
             except Exception:
                 msg = "failed to upload '{}' to '{}/{}'"
-                logger.error(
-                    msg.format(
-                        from_info["path"], to_info["bucket"], to_info["path"]
-                    )
+                logger.exception(
+                    msg.format(from_info.path, to_info.bucket, to_info.path)
                 )
                 continue
 
@@ -180,46 +163,44 @@ class RemoteGS(RemoteBase):
         gs = self.gs
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != "gs":
+            if from_info.scheme != "gs":
                 raise NotImplementedError
 
-            if to_info["scheme"] == "gs":
+            if to_info.scheme == "gs":
                 self.copy(from_info, to_info, gs=gs)
                 continue
 
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
             msg = "Downloading '{}/{}' to '{}'".format(
-                from_info["bucket"], from_info["path"], to_info["path"]
+                from_info.bucket, from_info.path, to_info.path
             )
             logger.debug(msg)
 
-            tmp_file = self.tmp_file(to_info["path"])
+            tmp_file = tmp_fname(to_info.path)
             if not name:
-                name = os.path.basename(to_info["path"])
+                name = os.path.basename(to_info.path)
 
             if not no_progress_bar:
                 # percent_cb is not available for download_to_filename, so
                 # lets at least update progress at pathpoints(start, finish)
                 progress.update_target(name, 0, None)
 
-            self._makedirs(to_info["path"])
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
 
             try:
-                bucket = gs.bucket(from_info["bucket"])
-                blob = bucket.get_blob(from_info["path"])
+                bucket = gs.bucket(from_info.bucket)
+                blob = bucket.get_blob(from_info.path)
                 blob.download_to_filename(tmp_file)
             except Exception:
                 msg = "failed to download '{}/{}' to '{}'"
-                logger.error(
-                    msg.format(
-                        from_info["bucket"], from_info["path"], to_info["path"]
-                    )
+                logger.exception(
+                    msg.format(from_info.bucket, from_info.path, to_info.path)
                 )
                 continue
 
-            os.rename(tmp_file, to_info["path"])
+            move(tmp_file, to_info.path)
 
             if not no_progress_bar:
                 progress.finish_target(name)

dvc/remote/hdfs.py

@@ -4,25 +4,26 @@ import os
 import re
 import getpass
 import posixpath
+import logging
 from subprocess import Popen, PIPE
 
-import dvc.logger as logger
 from dvc.config import Config
-from dvc.remote.base import RemoteBase, RemoteBaseCmdError
-from dvc.utils import fix_env
+from dvc.path import Schemes
+from dvc.path.hdfs import HDFSPathInfo
+from dvc.remote.base import RemoteBase, RemoteCmdError
+from dvc.utils import fix_env, tmp_fname
 
 
-class RemoteHDFSCmdError(RemoteBaseCmdError):
-    pass
+logger = logging.getLogger(__name__)
 
 
 class RemoteHDFS(RemoteBase):
-    scheme = "hdfs"
+    scheme = Schemes.HDFS
     REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
     PARAM_CHECKSUM = "checksum"
 
-    def __init__(self, project, config):
-        super(RemoteHDFS, self).__init__(project, config)
+    def __init__(self, repo, config):
+        super(RemoteHDFS, self).__init__(repo, config)
         self.url = config.get(Config.SECTION_REMOTE_URL, "/")
         self.prefix = self.url
         self.user = self.group("user")
@@ -31,7 +32,7 @@ class RemoteHDFS(RemoteBase):
                 Config.SECTION_REMOTE_USER, getpass.getuser()
             )
 
-        self.path_info = {"scheme": "hdfs", "user": self.user}
+        self.path_info = HDFSPathInfo(user=self.user)
 
     def hadoop_fs(self, cmd, user=None):
         cmd = "hadoop fs -" + cmd
@@ -55,7 +56,7 @@ class RemoteHDFS(RemoteBase):
         )
         out, err = p.communicate()
         if p.returncode != 0:
-            raise RemoteHDFSCmdError(cmd, p.returncode, err)
+            raise RemoteCmdError(self.scheme, cmd, p.returncode, err)
         return out.decode("utf-8")
 
     @staticmethod
@@ -64,85 +65,69 @@ class RemoteHDFS(RemoteBase):
         assert match is not None
         return match.group(gname)
 
-    def checksum(self, path_info):
+    def get_file_checksum(self, path_info):
         regex = r".*\t.*\t(?P<checksum>.*)"
         stdout = self.hadoop_fs(
-            "checksum {}".format(path_info["path"]), user=path_info["user"]
+            "checksum {}".format(path_info.path), user=path_info.user
         )
         return self._group(regex, stdout, "checksum")
 
     def copy(self, from_info, to_info):
-        dname = posixpath.dirname(to_info["path"])
-        self.hadoop_fs("mkdir -p {}".format(dname), user=to_info["user"])
+        dname = posixpath.dirname(to_info.path)
+        self.hadoop_fs("mkdir -p {}".format(dname), user=to_info.user)
         self.hadoop_fs(
-            "cp -f {} {}".format(from_info["path"], to_info["path"]),
-            user=to_info["user"],
+            "cp -f {} {}".format(from_info.path, to_info.path),
+            user=to_info.user,
         )
 
     def rm(self, path_info):
-        self.hadoop_fs(
-            "rm {}".format(path_info["path"]), user=path_info["user"]
-        )
-
-    def save_info(self, path_info):
-        if path_info["scheme"] != "hdfs":
-            raise NotImplementedError
-
-        assert path_info.get("path")
-
-        return {self.PARAM_CHECKSUM: self.checksum(path_info)}
-
-    def save(self, path_info):
-        if path_info["scheme"] != "hdfs":
-            raise NotImplementedError
-
-        assert path_info.get("path")
-
-        checksum = self.checksum(path_info)
-        dest = path_info.copy()
-        dest["path"] = self.checksum_to_path(checksum)
-
-        self.copy(path_info, dest)
-
-        return {self.PARAM_CHECKSUM: checksum}
+        self.hadoop_fs("rm -f {}".format(path_info.path), user=path_info.user)
 
     def remove(self, path_info):
-        if path_info["scheme"] != "hdfs":
+        if path_info.scheme != "hdfs":
             raise NotImplementedError
 
-        assert path_info.get("path")
+        assert path_info.path
 
-        logger.debug("Removing {}".format(path_info["path"]))
+        logger.debug("Removing {}".format(path_info.path))
 
         self.rm(path_info)
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info["scheme"] == "hdfs"
+        assert path_info.scheme == "hdfs"
 
         try:
-            self.hadoop_fs("test -e {}".format(path_info["path"]))
+            self.hadoop_fs("test -e {}".format(path_info.path))
             return True
-        except RemoteHDFSCmdError:
+        except RemoteCmdError:
             return False
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info["scheme"] != "hdfs":
+            if to_info.scheme != "hdfs":
                 raise NotImplementedError
 
-            if from_info["scheme"] != "local":
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
-            cmd = "mkdir -p {}".format(posixpath.dirname(to_info["path"]))
-            self.hadoop_fs(cmd, user=to_info["user"])
+            self.hadoop_fs(
+                "mkdir -p {}".format(posixpath.dirname(to_info.path)),
+                user=to_info.user,
+            )
 
-            cmd = "copyFromLocal {} {}".format(
-                from_info["path"], to_info["path"]
+            tmp_file = tmp_fname(to_info.path)
+
+            self.hadoop_fs(
+                "copyFromLocal {} {}".format(from_info.path, tmp_file),
+                user=to_info.user,
+            )
+
+            self.hadoop_fs(
+                "mv {} {}".format(tmp_file, to_info.path), user=to_info.user
             )
-            self.hadoop_fs(cmd, user=to_info["user"])
 
     def download(
         self,
@@ -155,29 +140,33 @@ class RemoteHDFS(RemoteBase):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != "hdfs":
+            if from_info.scheme != "hdfs":
                 raise NotImplementedError
 
-            if to_info["scheme"] == "hdfs":
+            if to_info.scheme == "hdfs":
                 self.copy(from_info, to_info)
                 continue
 
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
-            dname = os.path.dirname(to_info["path"])
+            dname = os.path.dirname(to_info.path)
             if not os.path.exists(dname):
                 os.makedirs(dname)
 
-            cmd = "copyToLocal {} {}".format(
-                from_info["path"], to_info["path"]
+            tmp_file = tmp_fname(to_info.path)
+
+            self.hadoop_fs(
+                "copyToLocal {} {}".format(from_info.path, tmp_file),
+                user=from_info.user,
             )
-            self.hadoop_fs(cmd, user=from_info["user"])
+
+            os.rename(tmp_file, to_info.path)
 
     def list_cache_paths(self):
         try:
             self.hadoop_fs("test -e {}".format(self.prefix))
-        except RemoteHDFSCmdError:
+        except RemoteCmdError:
             return []
 
         stdout = self.hadoop_fs("ls -R {}".format(self.prefix))

dvc/remote/http.py

@@ -1,15 +1,22 @@
 from __future__ import unicode_literals
-from dvc.utils.compat import open
+
+from dvc.path import Schemes
+from dvc.path.http import HTTPPathInfo
+from dvc.utils.compat import open, makedirs
 
 import os
 import threading
 import requests
+import logging
 
-import dvc.logger as logger
 from dvc.progress import progress
 from dvc.exceptions import DvcException
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
+from dvc.utils import move
+
+
+logger = logging.getLogger(__name__)
 
 
 class ProgressBarCallback(object):
@@ -26,17 +33,18 @@ class ProgressBarCallback(object):
 
 
 class RemoteHTTP(RemoteBase):
-    scheme = "http"
+    scheme = Schemes.HTTP
     REGEX = r"^https?://.*$"
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 2 ** 16
     PARAM_CHECKSUM = "etag"
 
-    def __init__(self, project, config):
-        super(RemoteHTTP, self).__init__(project, config)
+    def __init__(self, repo, config):
+        super(RemoteHTTP, self).__init__(repo, config)
         self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
         self.url = self.cache_dir
-        self.path_info = {"scheme": "http"}
+
+        self.path_info = HTTPPathInfo()
 
     @property
     def prefix(self):
@@ -53,23 +61,23 @@ class RemoteHTTP(RemoteBase):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] not in ["http", "https"]:
+            if from_info.scheme not in ["http", "https"]:
                 raise NotImplementedError
 
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
             msg = "Downloading '{}' to '{}'".format(
-                from_info["path"], to_info["path"]
+                from_info.path, to_info.path
             )
             logger.debug(msg)
 
             if not name:
-                name = os.path.basename(to_info["path"])
+                name = os.path.basename(to_info.path)
 
-            self._makedirs(to_info["path"])
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
 
-            total = self._content_length(from_info["path"])
+            total = self._content_length(from_info.path)
 
             if no_progress_bar or not total:
                 cb = None
@@ -78,15 +86,12 @@ class RemoteHTTP(RemoteBase):
 
             try:
                 self._download_to(
-                    from_info["path"],
-                    to_info["path"],
-                    callback=cb,
-                    resume=resume,
+                    from_info.path, to_info.path, callback=cb, resume=resume
                 )
 
             except Exception:
-                msg = "failed to download '{}'".format(from_info["path"])
-                logger.error(msg)
+                msg = "failed to download '{}'".format(from_info.path)
+                logger.exception(msg)
                 continue
 
             if not no_progress_bar:
@@ -94,8 +99,8 @@ class RemoteHTTP(RemoteBase):
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info["scheme"] in ["http", "https"]
-        return bool(self._request("HEAD", path_info.get("path")))
+        assert path_info.scheme in ["http", "https"]
+        return bool(self._request("HEAD", path_info.path))
 
     def cache_exists(self, md5s):
         assert isinstance(md5s, list)
@@ -105,25 +110,19 @@ class RemoteHTTP(RemoteBase):
 
         return list(filter(func, md5s))
 
-    def save_info(self, path_info):
-        if path_info["scheme"] not in ["http", "https"]:
-            raise NotImplementedError
-
-        return {self.PARAM_CHECKSUM: self._etag(path_info["path"])}
-
     def _content_length(self, url):
         return self._request("HEAD", url).headers.get("Content-Length")
 
-    def _etag(self, url):
+    def get_file_checksum(self, path_info):
+        url = path_info.path
         etag = self._request("HEAD", url).headers.get("ETag") or self._request(
             "HEAD", url
         ).headers.get("Content-MD5")
 
         if not etag:
             raise DvcException(
-                "could not find an ETag or Content-MD5 header for '{url}'".format(
-                    url=url
-                )
+                "could not find an ETag or "
+                "Content-MD5 header for '{url}'".format(url=url)
             )
 
         if etag.startswith("W/"):
@@ -148,7 +147,7 @@ class RemoteHTTP(RemoteBase):
             mode, partial_file, request, transferred_bytes, callback
         )
 
-        os.rename(partial_file, target_file)
+        move(partial_file, target_file)
 
     def _write_request_content(
         self, mode, partial_file, request, transferred_bytes, callback=None

dvc/remote/local.py

@@ -1,687 +0,0 @@
-from __future__ import unicode_literals
-
-from dvc.utils.compat import str
-
-import os
-import stat
-import uuid
-import json
-import ntpath
-import shutil
-import posixpath
-from operator import itemgetter
-
-import dvc.logger as logger
-from dvc.system import System
-from dvc.remote.base import RemoteBase, STATUS_MAP, STATUS_NEW, STATUS_DELETED
-from dvc.utils import remove, move, copyfile, dict_md5, to_chunks
-from dvc.utils import LARGE_DIR_SIZE
-from dvc.config import Config
-from dvc.exceptions import DvcException
-from dvc.progress import progress
-from concurrent.futures import ThreadPoolExecutor
-
-
-class RemoteLOCAL(RemoteBase):
-    scheme = "local"
-    REGEX = r"^(?P<path>.*)$"
-    PARAM_CHECKSUM = "md5"
-    PARAM_PATH = "path"
-    PARAM_RELPATH = "relpath"
-    MD5_DIR_SUFFIX = ".dir"
-
-    CACHE_TYPES = ["reflink", "hardlink", "symlink", "copy"]
-    CACHE_TYPE_MAP = {
-        "copy": shutil.copyfile,
-        "symlink": System.symlink,
-        "hardlink": System.hardlink,
-        "reflink": System.reflink,
-    }
-
-    def __init__(self, project, config):
-        super(RemoteLOCAL, self).__init__(project, config)
-        self.state = self.project.state if self.project else None
-        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
-        storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
-        self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
-
-        if self.cache_dir is not None and not os.path.isabs(self.cache_dir):
-            cwd = config[Config.PRIVATE_CWD]
-            self.cache_dir = os.path.abspath(os.path.join(cwd, self.cache_dir))
-
-        types = config.get(Config.SECTION_CACHE_TYPE, None)
-        if types:
-            if isinstance(types, str):
-                types = [t.strip() for t in types.split(",")]
-            self.cache_types = types
-        else:
-            self.cache_types = self.CACHE_TYPES
-
-        if self.cache_dir is not None and not os.path.exists(self.cache_dir):
-            os.mkdir(self.cache_dir)
-
-        self.path_info = {"scheme": "local"}
-
-    @staticmethod
-    def compat_config(config):
-        ret = config.copy()
-        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, "")
-        ret[Config.SECTION_REMOTE_URL] = url
-        return ret
-
-    @property
-    def url(self):
-        return self.cache_dir
-
-    @property
-    def prefix(self):
-        return self.cache_dir
-
-    def list_cache_paths(self):
-        clist = []
-        for entry in os.listdir(self.cache_dir):
-            subdir = os.path.join(self.cache_dir, entry)
-            if not os.path.isdir(subdir):
-                continue
-
-            for cache in os.listdir(subdir):
-                clist.append(os.path.join(subdir, cache))
-
-        return clist
-
-    def get(self, md5):
-        if not md5:
-            return None
-
-        return os.path.join(self.cache_dir, md5[0:2], md5[2:])
-
-    def changed_cache_file(self, md5):
-        cache = self.get(md5)
-        if self.state.changed(cache, md5=md5):
-            if os.path.exists(cache):
-                msg = "Corrupted cache file {}."
-                logger.warning(msg.format(os.path.relpath(cache)))
-                remove(cache)
-            return True
-        return False
-
-    def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info["scheme"] == "local"
-        return os.path.exists(path_info["path"])
-
-    def changed_cache(self, md5):
-        cache = self.get(md5)
-        clist = [(cache, md5)]
-
-        while True:
-            if len(clist) == 0:
-                break
-
-            cache, md5 = clist.pop()
-            if self.changed_cache_file(md5):
-                return True
-
-            if not self.is_dir_cache(cache):
-                continue
-
-            for entry in self.load_dir_cache(md5):
-                md5 = entry[self.PARAM_CHECKSUM]
-                cache = self.get(md5)
-                clist.append((cache, md5))
-
-        return False
-
-    def link(self, cache, path):
-        assert os.path.isfile(cache)
-
-        dname = os.path.dirname(path)
-        if not os.path.exists(dname):
-            os.makedirs(dname)
-
-        # NOTE: just create an empty file for an empty cache
-        if os.path.getsize(cache) == 0:
-            open(path, "w+").close()
-
-            msg = "Created empty file: {} -> {}".format(cache, path)
-            logger.debug(msg)
-            return
-
-        i = len(self.cache_types)
-        while i > 0:
-            try:
-                self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
-
-                if self.protected:
-                    os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
-
-                msg = "Created {}'{}': {} -> {}".format(
-                    "protected " if self.protected else "",
-                    self.cache_types[0],
-                    cache,
-                    path,
-                )
-
-                logger.debug(msg)
-                return
-
-            except DvcException as exc:
-                msg = "Cache type '{}' is not supported: {}"
-                logger.debug(msg.format(self.cache_types[0], str(exc)))
-                del self.cache_types[0]
-                i -= 1
-
-        raise DvcException("no possible cache types left to try out.")
-
-    @property
-    def ospath(self):
-        if os.name == "nt":
-            return ntpath
-        return posixpath
-
-    @classmethod
-    def to_ospath(cls, path):
-        if os.name == "nt":
-            return cls.ntpath(path)
-        return cls.unixpath(path)
-
-    @staticmethod
-    def unixpath(path):
-        assert not ntpath.isabs(path)
-        assert not posixpath.isabs(path)
-        return path.replace("\\", "/")
-
-    @staticmethod
-    def ntpath(path):
-        assert not ntpath.isabs(path)
-        assert not posixpath.isabs(path)
-        return path.replace("/", "\\")
-
-    def collect_dir_cache(self, dname):
-        dir_info = []
-
-        for root, dirs, files in os.walk(str(dname)):
-            bar = False
-
-            if len(files) > LARGE_DIR_SIZE:
-                msg = (
-                    "Computing md5 for a large directory {}. "
-                    "This is only done once."
-                )
-                logger.info(msg.format(os.path.relpath(root)))
-                bar = True
-                title = os.path.relpath(root)
-                processed = 0
-                total = len(files)
-                progress.update_target(title, 0, total)
-
-            for fname in files:
-                path = os.path.join(root, fname)
-                relpath = self.unixpath(os.path.relpath(path, dname))
-
-                if bar:
-                    progress.update_target(title, processed, total)
-                    processed += 1
-
-                md5 = self.state.update(path)
-                dir_info.append(
-                    {self.PARAM_RELPATH: relpath, self.PARAM_CHECKSUM: md5}
-                )
-
-            if bar:
-                progress.finish_target(title)
-
-        # NOTE: sorting the list by path to ensure reproducibility
-        dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
-
-        md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
-        if self.changed_cache_file(md5):
-            self.dump_dir_cache(md5, dir_info)
-
-        return (md5, dir_info)
-
-    def load_dir_cache(self, md5):
-        path = self.get(md5)
-
-        assert self.is_dir_cache(path)
-
-        try:
-            with open(path, "r") as fd:
-                d = json.load(fd)
-        except Exception:
-            msg = "Failed to load dir cache '{}'"
-            logger.error(msg.format(os.path.relpath(path)))
-            return []
-
-        if not isinstance(d, list):
-            msg = "dir cache file format error '{}' [skipping the file]"
-            logger.error(msg.format(os.path.relpath(path)))
-            return []
-
-        for info in d:
-            info["relpath"] = self.to_ospath(info["relpath"])
-
-        return d
-
-    def dump_dir_cache(self, md5, dir_info):
-        path = self.get(md5)
-        dname = os.path.dirname(path)
-
-        assert self.is_dir_cache(path)
-        assert isinstance(dir_info, list)
-
-        if not os.path.isdir(dname):
-            os.makedirs(dname)
-
-        # NOTE: Writing first and renaming after that
-        # to make sure that the operation is atomic.
-        tmp = "{}.{}".format(path, str(uuid.uuid4()))
-
-        with open(tmp, "w+") as fd:
-            json.dump(dir_info, fd, sort_keys=True)
-        move(tmp, path)
-
-    @classmethod
-    def is_dir_cache(cls, cache):
-        return cache.endswith(cls.MD5_DIR_SUFFIX)
-
-    def do_checkout(self, path_info, checksum, force=False):
-        path = path_info["path"]
-        md5 = checksum
-        cache = self.get(md5)
-
-        if not self.is_dir_cache(cache):
-            if os.path.exists(path):
-                self.safe_remove(path_info, force=force)
-
-            self.link(cache, path)
-            self.state.update_link(path)
-            return
-
-        # Create dir separately so that dir is created
-        # even if there are no files in it
-        if not os.path.exists(path):
-            os.makedirs(path)
-
-        dir_info = self.load_dir_cache(md5)
-        dir_relpath = os.path.relpath(path)
-        dir_size = len(dir_info)
-        bar = dir_size > LARGE_DIR_SIZE
-
-        logger.info("Linking directory '{}'.".format(dir_relpath))
-
-        for processed, entry in enumerate(dir_info):
-            relpath = entry[self.PARAM_RELPATH]
-            m = entry[self.PARAM_CHECKSUM]
-            p = os.path.join(path, relpath)
-            c = self.get(m)
-
-            entry_info = {"scheme": path_info["scheme"], self.PARAM_PATH: p}
-
-            entry_checksum_info = {self.PARAM_CHECKSUM: m}
-
-            if self.changed(entry_info, entry_checksum_info):
-                if os.path.exists(p):
-                    self.safe_remove(entry_info, force=force)
-
-                self.link(c, p)
-
-            if bar:
-                progress.update_target(dir_relpath, processed, dir_size)
-
-        self._discard_working_directory_changes(path, dir_info, force=force)
-
-        self.state.update_link(path)
-
-        if bar:
-            progress.finish_target(dir_relpath)
-
-    def already_cached(self, path_info):
-        assert path_info["scheme"] in ["", "local"]
-
-        current_md5 = self.state.update(path_info["path"])
-
-        if not current_md5:
-            return False
-
-        return not self.changed_cache(current_md5)
-
-    def _discard_working_directory_changes(self, path, dir_info, force=False):
-        working_dir_files = set(
-            os.path.join(root, file)
-            for root, _, files in os.walk(str(path))
-            for file in files
-        )
-
-        cached_files = set(
-            os.path.join(path, file["relpath"]) for file in dir_info
-        )
-
-        delta = working_dir_files - cached_files
-
-        for file in delta:
-            self.safe_remove({"scheme": "local", "path": file}, force=force)
-
-    def _move(self, inp, outp):
-        # moving in two stages to make the whole operation atomic in
-        # case inp and outp are in different filesystems and actual
-        # physical copying of data is happening
-        tmp = "{}.{}".format(outp, str(uuid.uuid4()))
-        move(inp, tmp)
-        move(tmp, outp)
-
-    def _save_file(self, path_info):
-        path = path_info["path"]
-        md5 = self.state.update(path)
-        assert md5 is not None
-
-        cache = self.get(md5)
-
-        if self.changed_cache(md5):
-            self._move(path, cache)
-        else:
-            remove(path)
-
-        self.link(cache, path)
-        self.state.update_link(path)
-
-        return {self.PARAM_CHECKSUM: md5}
-
-    def _save_dir(self, path_info):
-        path = path_info["path"]
-        md5, dir_info = self.state.update_info(path)
-        dir_relpath = os.path.relpath(path)
-        dir_size = len(dir_info)
-        bar = dir_size > LARGE_DIR_SIZE
-
-        logger.info("Linking directory '{}'.".format(dir_relpath))
-
-        for processed, entry in enumerate(dir_info):
-            relpath = entry[self.PARAM_RELPATH]
-            m = entry[self.PARAM_CHECKSUM]
-            p = os.path.join(path, relpath)
-            c = self.get(m)
-
-            if self.changed_cache(m):
-                self._move(p, c)
-            else:
-                remove(p)
-
-            self.link(c, p)
-
-            if bar:
-                progress.update_target(dir_relpath, processed, dir_size)
-
-        self.state.update_link(path)
-
-        if bar:
-            progress.finish_target(dir_relpath)
-
-        return {self.PARAM_CHECKSUM: md5}
-
-    def save(self, path_info):
-        if path_info["scheme"] != "local":
-            raise NotImplementedError
-
-        path = path_info["path"]
-
-        msg = "Saving '{}' to cache '{}'."
-        logger.info(
-            msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir))
-        )
-
-        if os.path.isdir(path):
-            return self._save_dir(path_info)
-        else:
-            return self._save_file(path_info)
-
-    def save_info(self, path_info):
-        if path_info["scheme"] != "local":
-            raise NotImplementedError
-
-        return {self.PARAM_CHECKSUM: self.state.update(path_info["path"])}
-
-    def remove(self, path_info):
-        if path_info["scheme"] != "local":
-            raise NotImplementedError
-
-        remove(path_info["path"])
-
-    def move(self, from_info, to_info):
-        if from_info["scheme"] != "local" or to_info["scheme"] != "local":
-            raise NotImplementedError
-
-        move(from_info["path"], to_info["path"])
-
-    def cache_exists(self, md5s):
-        assert isinstance(md5s, list)
-        return list(filter(lambda md5: not self.changed_cache_file(md5), md5s))
-
-    def upload(self, from_infos, to_infos, names=None):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info["scheme"] != "local":
-                raise NotImplementedError
-
-            if from_info["scheme"] != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Uploading '{}' to '{}'".format(
-                    from_info["path"], to_info["path"]
-                )
-            )
-
-            if not name:
-                name = os.path.basename(from_info["path"])
-
-            self._makedirs(to_info["path"])
-
-            try:
-                copyfile(from_info["path"], to_info["path"], name=name)
-            except Exception:
-                logger.error(
-                    "failed to upload '{}' to '{}'".format(
-                        from_info["path"], to_info["path"]
-                    )
-                )
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
-    ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != "local":
-                raise NotImplementedError
-
-            if to_info["scheme"] != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Downloading '{}' to '{}'".format(
-                    from_info["path"], to_info["path"]
-                )
-            )
-
-            if not name:
-                name = os.path.basename(to_info["path"])
-
-            self._makedirs(to_info["path"])
-            tmp_file = self.tmp_file(to_info["path"])
-            try:
-                copyfile(
-                    from_info["path"],
-                    tmp_file,
-                    no_progress_bar=no_progress_bar,
-                    name=name,
-                )
-            except Exception:
-                logger.error(
-                    "failed to download '{}' to '{}'".format(
-                        from_info["path"], to_info["path"]
-                    )
-                )
-
-                continue
-
-            os.rename(tmp_file, to_info["path"])
-
-    def _group(self, checksum_infos, show_checksums=False):
-        by_md5 = {}
-
-        for info in checksum_infos:
-            md5 = info[self.PARAM_CHECKSUM]
-
-            if show_checksums:
-                by_md5[md5] = {"name": md5}
-                continue
-
-            name = info[self.PARAM_PATH]
-            branch = info.get("branch")
-            if branch:
-                name += "({})".format(branch)
-
-            if md5 not in by_md5.keys():
-                by_md5[md5] = {"name": name}
-            else:
-                by_md5[md5]["name"] += " " + name
-
-        return by_md5
-
-    def status(self, checksum_infos, remote, jobs=None, show_checksums=False):
-        logger.info("Preparing to collect status from {}".format(remote.url))
-        title = "Collecting information"
-
-        ret = {}
-
-        progress.set_n_total(1)
-        progress.update_target(title, 0, 100)
-
-        progress.update_target(title, 10, 100)
-
-        ret = self._group(checksum_infos, show_checksums=show_checksums)
-        md5s = list(ret.keys())
-
-        progress.update_target(title, 30, 100)
-
-        remote_exists = list(remote.cache_exists(md5s))
-
-        progress.update_target(title, 90, 100)
-
-        local_exists = self.cache_exists(md5s)
-
-        progress.finish_target(title)
-
-        for md5, info in ret.items():
-            info["status"] = STATUS_MAP[
-                (md5 in local_exists, md5 in remote_exists)
-            ]
-
-        return ret
-
-    def _get_chunks(self, download, remote, status_info, status, jobs):
-        title = "Analysing status."
-
-        progress.set_n_total(1)
-        total = len(status_info)
-        current = 0
-
-        cache = []
-        path_infos = []
-        names = []
-        for md5, info in status_info.items():
-            if info["status"] == status:
-                cache.append(self.checksum_to_path_info(md5))
-                path_infos.append(remote.checksum_to_path_info(md5))
-                names.append(info["name"])
-            current += 1
-            progress.update_target(title, current, total)
-
-        progress.finish_target(title)
-
-        progress.set_n_total(len(names))
-
-        if download:
-            to_infos = cache
-            from_infos = path_infos
-        else:
-            to_infos = path_infos
-            from_infos = cache
-
-        return list(
-            zip(
-                to_chunks(from_infos, jobs),
-                to_chunks(to_infos, jobs),
-                to_chunks(names, jobs),
-            )
-        )
-
-    def _process(
-        self,
-        checksum_infos,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        msg = "Preparing to {} data {} '{}'"
-        logger.info(
-            msg.format(
-                "download" if download else "upload",
-                "from" if download else "to",
-                remote.url,
-            )
-        )
-
-        if download:
-            func = remote.download
-            status = STATUS_DELETED
-        else:
-            func = remote.upload
-            status = STATUS_NEW
-
-        if jobs is None:
-            jobs = remote.JOBS
-
-        status_info = self.status(
-            checksum_infos, remote, jobs=jobs, show_checksums=show_checksums
-        )
-
-        chunks = self._get_chunks(download, remote, status_info, status, jobs)
-
-        if len(chunks) == 0:
-            return
-
-        futures = []
-        with ThreadPoolExecutor(max_workers=jobs) as executor:
-            for from_infos, to_infos, names in chunks:
-                res = executor.submit(func, from_infos, to_infos, names=names)
-                futures.append(res)
-
-        for f in futures:
-            f.result()
-
-    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
-        self._process(
-            checksum_infos,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=False,
-        )
-
-    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
-        self._process(
-            checksum_infos,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=True,
-        )

dvc/remote/local/__init__.py

@@ -0,0 +1,576 @@
+from __future__ import unicode_literals
+
+from copy import copy
+
+from dvc.path import BasePathInfo, Schemes
+from dvc.path.local import LocalPathInfo
+from dvc.remote.local.slow_link_detection import slow_link_guard
+from dvc.utils.compat import str, makedirs
+
+import os
+import stat
+import uuid
+import ntpath
+import shutil
+import posixpath
+import logging
+
+from dvc.system import System
+from dvc.remote.base import (
+    RemoteBase,
+    STATUS_MAP,
+    STATUS_NEW,
+    STATUS_DELETED,
+    STATUS_MISSING,
+)
+from dvc.utils import (
+    remove,
+    move,
+    copyfile,
+    to_chunks,
+    tmp_fname,
+    file_md5,
+    walk_files,
+)
+from dvc.config import Config
+from dvc.exceptions import DvcException
+from dvc.progress import progress
+from concurrent.futures import ThreadPoolExecutor
+
+from dvc.utils.fs import get_mtime_and_size, get_inode
+
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteLOCAL(RemoteBase):
+    scheme = Schemes.LOCAL
+    REGEX = r"^(?P<path>.*)$"
+    PARAM_CHECKSUM = "md5"
+    PARAM_PATH = "path"
+
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
+    CACHE_TYPE_MAP = {
+        "copy": shutil.copyfile,
+        "symlink": System.symlink,
+        "hardlink": System.hardlink,
+        "reflink": System.reflink,
+    }
+
+    def __init__(self, repo, config):
+        super(RemoteLOCAL, self).__init__(repo, config)
+        self.state = self.repo.state if self.repo else None
+        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
+        storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
+        self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
+
+        if self.cache_dir is not None and not os.path.isabs(self.cache_dir):
+            cwd = config[Config.PRIVATE_CWD]
+            self.cache_dir = os.path.abspath(os.path.join(cwd, self.cache_dir))
+
+        types = config.get(Config.SECTION_CACHE_TYPE, None)
+        if types:
+            if isinstance(types, str):
+                types = [t.strip() for t in types.split(",")]
+            self.cache_types = types
+        else:
+            self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
+
+        if self.cache_dir is not None and not os.path.exists(self.cache_dir):
+            os.mkdir(self.cache_dir)
+
+        self._dir_info = {}
+        self.path_info = LocalPathInfo()
+
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, "")
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
+
+    @property
+    def url(self):
+        return self.cache_dir
+
+    @property
+    def prefix(self):
+        return self.cache_dir
+
+    def list_cache_paths(self):
+        clist = []
+        for entry in os.listdir(self.cache_dir):
+            subdir = os.path.join(self.cache_dir, entry)
+            if not os.path.isdir(subdir):
+                continue
+
+            for cache in os.listdir(subdir):
+                clist.append(os.path.join(subdir, cache))
+
+        return clist
+
+    def get(self, md5):
+        if not md5:
+            return None
+
+        return self.checksum_to_path(md5)
+
+    def exists(self, path_info):
+        assert isinstance(path_info, BasePathInfo)
+        assert path_info.scheme == "local"
+        return os.path.lexists(path_info.path)
+
+    def makedirs(self, path_info):
+        if not self.exists(path_info):
+            os.makedirs(path_info.path)
+
+    @slow_link_guard
+    def link(self, cache_info, path_info):
+        cache = cache_info.path
+        path = path_info.path
+
+        assert os.path.isfile(cache)
+
+        dname = os.path.dirname(path)
+        if not os.path.exists(dname):
+            os.makedirs(dname)
+
+        # NOTE: just create an empty file for an empty cache
+        if os.path.getsize(cache) == 0:
+            open(path, "w+").close()
+
+            msg = "Created empty file: {} -> {}".format(cache, path)
+            logger.debug(msg)
+            return
+
+        i = len(self.cache_types)
+        while i > 0:
+            try:
+                self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
+
+                if self.protected:
+                    self.protect(path_info)
+
+                msg = "Created {}'{}': {} -> {}".format(
+                    "protected " if self.protected else "",
+                    self.cache_types[0],
+                    cache,
+                    path,
+                )
+
+                logger.debug(msg)
+                return
+
+            except DvcException as exc:
+                msg = "Cache type '{}' is not supported: {}"
+                logger.debug(msg.format(self.cache_types[0], str(exc)))
+                del self.cache_types[0]
+                i -= 1
+
+        raise DvcException("no possible cache types left to try out.")
+
+    @property
+    def ospath(self):
+        if os.name == "nt":
+            return ntpath
+        return posixpath
+
+    def already_cached(self, path_info):
+        assert path_info.scheme in ["", "local"]
+
+        current_md5 = self.get_checksum(path_info)
+
+        if not current_md5:
+            return False
+
+        return not self.changed_cache(current_md5)
+
+    def is_empty(self, path_info):
+        path = path_info.path
+
+        if self.isfile(path_info) and os.path.getsize(path) == 0:
+            return True
+
+        if self.isdir(path_info) and len(os.listdir(path)) == 0:
+            return True
+
+        return False
+
+    def isfile(self, path_info):
+        return os.path.isfile(path_info.path)
+
+    def isdir(self, path_info):
+        return os.path.isdir(path_info.path)
+
+    def walk(self, path_info):
+        return os.walk(path_info.path)
+
+    def get_file_checksum(self, path_info):
+        return file_md5(path_info.path)[0]
+
+    def remove(self, path_info):
+        if path_info.scheme != "local":
+            raise NotImplementedError
+
+        remove(path_info.path)
+
+    def move(self, from_info, to_info):
+        if from_info.scheme != "local" or to_info.scheme != "local":
+            raise NotImplementedError
+
+        inp = from_info.path
+        outp = to_info.path
+
+        # moving in two stages to make the whole operation atomic in
+        # case inp and outp are in different filesystems and actual
+        # physical copying of data is happening
+        tmp = "{}.{}".format(outp, str(uuid.uuid4()))
+        move(inp, tmp)
+        move(tmp, outp)
+
+    def cache_exists(self, md5s):
+        assert isinstance(md5s, list)
+        return list(filter(lambda md5: not self.changed_cache_file(md5), md5s))
+
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        for from_info, to_info, name in zip(from_infos, to_infos, names):
+            if to_info.scheme != "local":
+                raise NotImplementedError
+
+            if from_info.scheme != "local":
+                raise NotImplementedError
+
+            logger.debug(
+                "Uploading '{}' to '{}'".format(from_info.path, to_info.path)
+            )
+
+            if not name:
+                name = os.path.basename(from_info.path)
+
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+            tmp_file = tmp_fname(to_info.path)
+
+            try:
+                copyfile(
+                    from_info.path,
+                    tmp_file,
+                    name=name,
+                    no_progress_bar=no_progress_bar,
+                )
+                os.rename(tmp_file, to_info.path)
+            except Exception:
+                logger.exception(
+                    "failed to upload '{}' to '{}'".format(
+                        from_info.path, to_info.path
+                    )
+                )
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
+        names = self._verify_path_args(from_infos, to_infos, names)
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info.scheme != "local":
+                raise NotImplementedError
+
+            if to_info.scheme != "local":
+                raise NotImplementedError
+
+            logger.debug(
+                "Downloading '{}' to '{}'".format(from_info.path, to_info.path)
+            )
+
+            if not name:
+                name = os.path.basename(to_info.path)
+
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+            tmp_file = tmp_fname(to_info.path)
+            try:
+                copyfile(
+                    from_info.path,
+                    tmp_file,
+                    no_progress_bar=no_progress_bar,
+                    name=name,
+                )
+
+                move(tmp_file, to_info.path)
+            except Exception:
+                logger.exception(
+                    "failed to download '{}' to '{}'".format(
+                        from_info.path, to_info.path
+                    )
+                )
+
+                continue
+
+    def _group(self, checksum_infos, show_checksums=False):
+        by_md5 = {}
+
+        for info in checksum_infos:
+            md5 = info[self.PARAM_CHECKSUM]
+
+            if show_checksums:
+                by_md5[md5] = {"name": md5}
+                continue
+
+            name = info[self.PARAM_PATH]
+            branch = info.get("branch")
+            if branch:
+                name += "({})".format(branch)
+
+            if md5 not in by_md5.keys():
+                by_md5[md5] = {"name": name}
+            else:
+                by_md5[md5]["name"] += " " + name
+
+        return by_md5
+
+    def status(
+        self,
+        checksum_infos,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        logger.info("Preparing to collect status from {}".format(remote.url))
+        title = "Collecting information"
+
+        ret = {}
+
+        progress.set_n_total(1)
+        progress.update_target(title, 0, 100)
+
+        progress.update_target(title, 10, 100)
+
+        ret = self._group(checksum_infos, show_checksums=show_checksums)
+        md5s = list(ret.keys())
+
+        progress.update_target(title, 30, 100)
+
+        local_exists = self.cache_exists(md5s)
+
+        progress.update_target(title, 40, 100)
+
+        # This is a performance optimization. We can safely assume that,
+        # if the resources that we want to fetch are already cached,
+        # there's no need to check the remote storage for the existance of
+        # those files.
+        if download and sorted(local_exists) == sorted(md5s):
+            remote_exists = local_exists
+        else:
+            remote_exists = list(remote.cache_exists(md5s))
+
+        progress.update_target(title, 90, 100)
+
+        progress.finish_target(title)
+
+        self._fill_statuses(ret, local_exists, remote_exists)
+
+        self._log_missing_caches(ret)
+
+        return ret
+
+    def _fill_statuses(self, checksum_info_dir, local_exists, remote_exists):
+        # Using sets because they are way faster for lookups
+        local = set(local_exists)
+        remote = set(remote_exists)
+
+        for md5, info in checksum_info_dir.items():
+            status = STATUS_MAP[(md5 in local, md5 in remote)]
+            info["status"] = status
+
+    def _get_chunks(self, download, remote, status_info, status, jobs):
+        title = "Analysing status."
+
+        progress.set_n_total(1)
+        total = len(status_info)
+        current = 0
+
+        cache = []
+        path_infos = []
+        names = []
+        for md5, info in status_info.items():
+            if info["status"] == status:
+                cache.append(self.checksum_to_path_info(md5))
+                path_infos.append(remote.checksum_to_path_info(md5))
+                names.append(info["name"])
+            current += 1
+            progress.update_target(title, current, total)
+
+        progress.finish_target(title)
+
+        progress.set_n_total(len(names))
+
+        if download:
+            to_infos = cache
+            from_infos = path_infos
+        else:
+            to_infos = path_infos
+            from_infos = cache
+
+        return list(
+            zip(
+                to_chunks(from_infos, jobs),
+                to_chunks(to_infos, jobs),
+                to_chunks(names, jobs),
+            )
+        )
+
+    def _process(
+        self,
+        checksum_infos,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        msg = "Preparing to {} data {} '{}'"
+        logger.info(
+            msg.format(
+                "download" if download else "upload",
+                "from" if download else "to",
+                remote.url,
+            )
+        )
+
+        if download:
+            func = remote.download
+            status = STATUS_DELETED
+        else:
+            func = remote.upload
+            status = STATUS_NEW
+
+        if jobs is None:
+            jobs = remote.JOBS
+
+        status_info = self.status(
+            checksum_infos,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+        )
+
+        chunks = self._get_chunks(download, remote, status_info, status, jobs)
+
+        if len(chunks) == 0:
+            return 0
+
+        futures = []
+        with ThreadPoolExecutor(max_workers=jobs) as executor:
+            for from_infos, to_infos, names in chunks:
+                res = executor.submit(func, from_infos, to_infos, names=names)
+                futures.append(res)
+
+        for f in futures:
+            f.result()
+
+        return len(chunks)
+
+    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
+        return self._process(
+            checksum_infos,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=False,
+        )
+
+    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
+        return self._process(
+            checksum_infos,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=True,
+        )
+
+    def _changed_cache_dir(self):
+        mtime, size = get_mtime_and_size(self.cache_dir)
+        inode = get_inode(self.cache_dir)
+
+        existing_record = self.state.get_state_record_for_inode(inode)
+        if existing_record:
+            cached_mtime, cached_size, _, _ = existing_record
+            changed = not (mtime == cached_mtime and size == cached_size)
+        else:
+            changed = True
+
+        return changed
+
+    def _log_missing_caches(self, checksum_info_dict):
+        missing_caches = [
+            (md5, info)
+            for md5, info in checksum_info_dict.items()
+            if info["status"] == STATUS_MISSING
+        ]
+        if missing_caches:
+            missing_desc = "".join(
+                [
+                    "\nname: {}, md5: {}".format(info["name"], md5)
+                    for md5, info in missing_caches
+                ]
+            )
+            msg = (
+                "Some of the cache files do not exist neither locally "
+                "nor on remote. Missing cache files: {}".format(missing_desc)
+            )
+            logger.warning(msg)
+
+    @staticmethod
+    def _unprotect_file(path):
+        if System.is_symlink(path) or System.is_hardlink(path):
+            logger.debug("Unprotecting '{}'".format(path))
+            tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))
+
+            # The operations order is important here - if some application
+            # would access the file during the process of copyfile then it
+            # would get only the part of file. So, at first, the file should be
+            # copied with the temporary name, and then original file should be
+            # replaced by new.
+            copyfile(
+                path,
+                tmp,
+                name="Unprotecting '{}'".format(os.path.relpath(path)),
+            )
+            remove(path)
+            os.rename(tmp, path)
+
+        else:
+            logger.debug(
+                "Skipping copying for '{}', since it is not "
+                "a symlink or a hardlink.".format(path)
+            )
+
+        os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
+
+    @staticmethod
+    def _unprotect_dir(path):
+        for path in walk_files(path):
+            RemoteLOCAL._unprotect_file(path)
+
+    @staticmethod
+    def unprotect(path_info):
+        path = path_info.path
+        if not os.path.exists(path):
+            raise DvcException(
+                "can't unprotect non-existing data '{}'".format(path)
+            )
+
+        if os.path.isdir(path):
+            RemoteLOCAL._unprotect_dir(path)
+        else:
+            RemoteLOCAL._unprotect_file(path)
+
+    @staticmethod
+    def protect(path_info):
+        os.chmod(path_info.path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)

dvc/remote/local/slow_link_detection.py

@@ -0,0 +1,63 @@
+import logging
+import time
+
+import colorama
+from dvc.config import Config
+
+logger = logging.getLogger(__name__)
+
+
+class SlowLinkDetectorDecorator(object):
+    LINKING_TIMEOUT_SECONDS = 10.0
+    was_displayed = False
+
+    @classmethod
+    def should_display(cls):
+        if not cls.was_displayed:
+            cls.was_displayed = True
+            return True
+        return False
+
+    def __init__(self, method):
+        self.method = method
+
+    def __call__(self, *args, **kwargs):
+        start = time.time()
+        result = self.method(*args, **kwargs)
+        execution_time_seconds = time.time() - start
+
+        if (
+            execution_time_seconds >= self.LINKING_TIMEOUT_SECONDS
+            and self.should_display()
+        ):
+            msg = (
+                "You can cut execution time considerably. Check:\n"
+                "{blue}https://dvc.org/doc/commands-reference/config#cache{"
+                "reset}"
+                "\nfor "
+                "more information.\nTo disable this message, run:\n'dvc "
+                "config "
+                "cache.slow_link_warning False'".format(
+                    blue=colorama.Fore.BLUE, reset=colorama.Fore.RESET
+                )
+            )
+            logger.warning(msg)
+
+        return result
+
+
+def slow_link_guard(method):
+    def call(remote_local, *args, **kwargs):
+        cache_config = remote_local.repo.config.config.get(
+            Config.SECTION_CACHE
+        )
+        should_warn = cache_config.get(
+            Config.SECTION_CACHE_SLOW_LINK_WARNING, True
+        ) and not cache_config.get(Config.SECTION_CACHE_TYPE, None)
+
+        if should_warn:
+            decorated = SlowLinkDetectorDecorator(method)
+            return decorated(remote_local, *args, **kwargs)
+        return method(remote_local, *args, **kwargs)
+
+    return call

dvc/remote/oss.py

@@ -0,0 +1,192 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import os
+import logging
+
+from dvc.path import Schemes
+from dvc.path.oss import OSSPathInfo
+
+try:
+    import oss2
+except ImportError:
+    oss2 = None
+
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
+from dvc.progress import progress
+from dvc.config import Config
+from dvc.remote.base import RemoteBase
+from dvc.remote.azure import Callback
+
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteOSS(RemoteBase):
+    """
+    oss2 document:
+    https://www.alibabacloud.com/help/doc-detail/32026.htm
+
+
+    Examples
+    ----------
+    $ dvc remote add myremote oss://my-bucket/path
+    Set key id, key secret and endpoint using modify command
+    $ dvc remote modify myremote oss_key_id my-key-id
+    $ dvc remote modify myremote oss_key_secret my-key-secret
+    $ dvc remote modify myremote oss_endpoint endpoint
+    or environment variables
+    $ export OSS_ACCESS_KEY_ID="my-key-id"
+    $ export OSS_ACCESS_KEY_SECRET="my-key-secret"
+    $ export OSS_ENDPOINT="endpoint"
+    """
+
+    scheme = Schemes.OSS
+    REGEX = r"^oss://(?P<path>.*)?$"
+    REQUIRES = {"oss2": oss2}
+    PARAM_CHECKSUM = "etag"
+    COPY_POLL_SECONDS = 5
+
+    def __init__(self, repo, config):
+        super(RemoteOSS, self).__init__(repo, config)
+
+        self.url = config.get(Config.SECTION_REMOTE_URL)
+        parsed = urlparse(self.url)
+        self.bucket = parsed.netloc
+        self.prefix = parsed.path.lstrip("/")
+
+        self.endpoint = config.get(Config.SECTION_OSS_ENDPOINT) or os.getenv(
+            "OSS_ENDPOINT"
+        )
+
+        self.key_id = (
+            config.get(Config.SECTION_OSS_ACCESS_KEY_ID)
+            or os.getenv("OSS_ACCESS_KEY_ID")
+            or "defaultId"
+        )
+
+        self.key_secret = (
+            config.get(Config.SECTION_OSS_ACCESS_KEY_SECRET)
+            or os.getenv("OSS_ACCESS_KEY_SECRET")
+            or "defaultSecret"
+        )
+
+        self._bucket = None
+        self.path_info = OSSPathInfo(bucket=self.bucket)
+
+    @property
+    def oss_service(self):
+        if self._bucket is None:
+            logger.debug("URL {}".format(self.url))
+            logger.debug("key id {}".format(self.key_id))
+            logger.debug("key secret {}".format(self.key_secret))
+            auth = oss2.Auth(self.key_id, self.key_secret)
+            logger.debug("bucket name {}".format(self.bucket))
+            self._bucket = oss2.Bucket(auth, self.endpoint, self.bucket)
+            try:  # verify that bucket exists
+                self._bucket.get_bucket_info()
+            except oss2.exceptions.NoSuchBucket:
+                self._bucket.create_bucket(
+                    oss2.BUCKET_ACL_PUBLIC_READ,
+                    oss2.models.BucketCreateConfig(
+                        oss2.BUCKET_STORAGE_CLASS_STANDARD
+                    ),
+                )
+        return self._bucket
+
+    def remove(self, path_info):
+        if path_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        logger.debug(
+            "Removing oss://{}/{}".format(path_info.bucket, path_info.path)
+        )
+
+        self.oss_service.delete_object(path_info.path)
+
+    def _list_paths(self, prefix):
+        for blob in oss2.ObjectIterator(self.oss_service, prefix=prefix):
+            yield blob.key
+
+    def list_cache_paths(self):
+        return self._list_paths(self.prefix)
+
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        for from_info, to_info, name in zip(from_infos, to_infos, names):
+            if to_info.scheme != self.scheme:
+                raise NotImplementedError
+
+            if from_info.scheme != "local":
+                raise NotImplementedError
+
+            bucket = to_info.bucket
+            path = to_info.path
+
+            logger.debug(
+                "Uploading '{}' to 'oss://{}/{}'".format(
+                    from_info.path, bucket, path
+                )
+            )
+
+            if not name:
+                name = os.path.basename(from_info.path)
+
+            cb = None if no_progress_bar else Callback(name)
+
+            try:
+                self.oss_service.put_object_from_file(
+                    path, from_info.path, progress_callback=cb
+                )
+            except Exception:
+                msg = "failed to upload '{}'".format(from_info.path)
+                logger.warning(msg)
+            else:
+                progress.finish_target(name)
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        names=None,
+        no_progress_bar=False,
+        resume=False,
+    ):
+        names = self._verify_path_args(from_infos, to_infos, names)
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info.scheme != self.scheme:
+                raise NotImplementedError
+            if to_info.scheme != "local":
+                raise NotImplementedError
+
+            bucket = from_info.bucket
+            path = from_info.path
+
+            logger.debug(
+                "Downloading 'oss://{}/{}' to '{}'".format(
+                    bucket, path, to_info.path
+                )
+            )
+
+            tmp_file = tmp_fname(to_info.path)
+            if not name:
+                name = os.path.basename(to_info.path)
+
+            cb = None if no_progress_bar else Callback(name)
+
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
+
+            try:
+                self.oss_service.get_object_to_file(
+                    path, tmp_file, progress_callback=cb
+                )
+            except Exception:
+                msg = "failed to download 'oss://{}/{}'".format(bucket, path)
+                logger.warning(msg)
+            else:
+                move(tmp_file, to_info.path)
+
+                if not no_progress_bar:
+                    progress.finish_target(name)

dvc/remote/s3.py

@@ -2,18 +2,24 @@ from __future__ import unicode_literals
 
 import os
 import threading
+import logging
+
+from dvc.path import Schemes
+from dvc.path.s3 import S3PathInfo
 
 try:
     import boto3
 except ImportError:
     boto3 = None
 
-import dvc.logger as logger
-from dvc.utils.compat import urlparse
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, ETagMismatchError
+
+logger = logging.getLogger(__name__)
 
 
 class Callback(object):
@@ -30,13 +36,13 @@ class Callback(object):
 
 
 class RemoteS3(RemoteBase):
-    scheme = "s3"
+    scheme = Schemes.S3
     REGEX = r"^s3://(?P<path>.*)$"
     REQUIRES = {"boto3": boto3}
     PARAM_CHECKSUM = "etag"
 
-    def __init__(self, project, config):
-        super(RemoteS3, self).__init__(project, config)
+    def __init__(self, repo, config):
+        super(RemoteS3, self).__init__(repo, config)
 
         storagepath = "s3://{}".format(
             config.get(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
@@ -54,6 +60,8 @@ class RemoteS3(RemoteBase):
 
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
 
+        self.list_objects = config.get(Config.SECTION_AWS_LIST_OBJECTS)
+
         self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
 
         shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
@@ -64,7 +72,7 @@ class RemoteS3(RemoteBase):
         self.bucket = parsed.netloc
         self.prefix = parsed.path.lstrip("/")
 
-        self.path_info = {"scheme": self.scheme, "bucket": self.bucket}
+        self.path_info = S3PathInfo(bucket=self.bucket)
 
     @staticmethod
     def compat_config(config):
@@ -83,125 +91,177 @@ class RemoteS3(RemoteBase):
             "s3", endpoint_url=self.endpoint_url, use_ssl=self.use_ssl
         )
 
-    def get_etag(self, bucket, path):
-        try:
-            obj = self.s3.head_object(Bucket=bucket, Key=path)
-        except Exception:
-            raise DvcException(
-                "s3://{}/{} does not exist".format(bucket, path)
-            )
+    @classmethod
+    def get_etag(cls, s3, bucket, path):
+        obj = cls.get_head_object(s3, bucket, path)
 
         return obj["ETag"].strip('"')
 
-    def save_info(self, path_info):
-        if path_info["scheme"] != "s3":
-            raise NotImplementedError
+    def get_file_checksum(self, path_info):
+        return self.get_etag(self.s3, path_info.bucket, path_info.path)
 
-        return {
-            self.PARAM_CHECKSUM: self.get_etag(
-                path_info["bucket"], path_info["path"]
+    @staticmethod
+    def get_head_object(s3, bucket, path, *args, **kwargs):
+
+        try:
+            obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
+        except Exception as exc:
+            raise DvcException(
+                "s3://{}/{} does not exist".format(bucket, path), exc
             )
-        }
+        return obj
+
+    @classmethod
+    def _copy_multipart(cls, s3, from_info, to_info, size, n_parts):
+        mpu = s3.create_multipart_upload(
+            Bucket=to_info.bucket, Key=to_info.path
+        )
+        mpu_id = mpu["UploadId"]
+
+        parts = []
+        byte_position = 0
+        for i in range(1, n_parts + 1):
+            obj = cls.get_head_object(
+                s3, from_info.bucket, from_info.path, PartNumber=i
+            )
+            part_size = obj["ContentLength"]
+            lastbyte = byte_position + part_size - 1
+            if lastbyte > size:
+                lastbyte = size - 1
+
+            srange = "bytes={}-{}".format(byte_position, lastbyte)
+
+            part = s3.upload_part_copy(
+                Bucket=to_info.bucket,
+                Key=to_info.path,
+                PartNumber=i,
+                UploadId=mpu_id,
+                CopySourceRange=srange,
+                CopySource={"Bucket": from_info.bucket, "Key": from_info.path},
+            )
+            parts.append(
+                {"PartNumber": i, "ETag": part["CopyPartResult"]["ETag"]}
+            )
+            byte_position += part_size
+
+        assert n_parts == len(parts)
+
+        s3.complete_multipart_upload(
+            Bucket=to_info.bucket,
+            Key=to_info.path,
+            UploadId=mpu_id,
+            MultipartUpload={"Parts": parts},
+        )
+
+    @classmethod
+    def _copy(cls, s3, from_info, to_info):
+        # NOTE: object's etag depends on the way it was uploaded to s3 or the
+        # way it was copied within the s3. More specifically, it depends on
+        # the chunk size that was used to transfer it, which would affect
+        # whether an object would be uploaded as a single part or as a
+        # multipart.
+        #
+        # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
+        # then it was transfered as a single part, which means that the chunk
+        # size used to transfer it was greater or equal to the ContentLength
+        # of that object. So to preserve that tag over the next transfer, we
+        # could use any value >= ContentLength.
+        #
+        # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
+        # then it was transfered as a multipart, which means that the chunk
+        # size used to transfer it was less than ContentLength of that object.
+        # Unfortunately, in general, it doesn't mean that the chunk size was
+        # the same throughout the transfer, so it means that in order to
+        # preserve etag, we need to transfer each part separately, so the
+        # object is transfered in the same chunks as it was originally.
+
+        obj = cls.get_head_object(s3, from_info.bucket, from_info.path)
+        etag = obj["ETag"].strip('"')
+        size = obj["ContentLength"]
+
+        _, _, parts_suffix = etag.partition("-")
+        if parts_suffix:
+            n_parts = int(parts_suffix)
+            cls._copy_multipart(s3, from_info, to_info, size, n_parts)
+        else:
+            source = {"Bucket": from_info.bucket, "Key": from_info.path}
+            s3.copy(source, to_info.bucket, to_info.path)
+
+        cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
+        if etag != cached_etag:
+            raise ETagMismatchError(etag, cached_etag)
 
     def copy(self, from_info, to_info, s3=None):
         s3 = s3 if s3 else self.s3
-
-        source = {"Bucket": from_info["bucket"], "Key": from_info["path"]}
-        self.s3.copy(source, to_info["bucket"], to_info["path"])
-
-    def save(self, path_info):
-        if path_info["scheme"] != "s3":
-            raise NotImplementedError
-
-        etag = self.get_etag(path_info["bucket"], path_info["path"])
-        path = self.checksum_to_path(etag)
-        to_info = {"scheme": "s3", "bucket": self.bucket, "path": path}
-
-        self.copy(path_info, to_info)
-
-        return {self.PARAM_CHECKSUM: etag}
+        self._copy(s3, from_info, to_info)
 
     def remove(self, path_info):
-        if path_info["scheme"] != "s3":
+        if path_info.scheme != "s3":
             raise NotImplementedError
 
         logger.debug(
-            "Removing s3://{}/{}".format(
-                path_info["bucket"], path_info["path"]
-            )
+            "Removing s3://{}/{}".format(path_info.bucket, path_info.path)
         )
 
-        self.s3.delete_object(
-            Bucket=path_info["bucket"], Key=path_info["path"]
-        )
+        self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
 
     def _list_paths(self, bucket, prefix):
+        """ Read config for list object api, paginate through list objects."""
         s3 = self.s3
-
         kwargs = {"Bucket": bucket, "Prefix": prefix}
-        while True:
-            # NOTE: list_objects_v2() is 90% faster than head_object [1]
-            #
-            # [1] https://www.peterbe.com/plog/
-            #     fastest-way-to-find-out-if-a-file-exists-in-s3
-            resp = s3.list_objects_v2(**kwargs)
-            contents = resp.get("Contents", None)
+        if self.list_objects:
+            list_objects_api = "list_objects"
+        else:
+            list_objects_api = "list_objects_v2"
+        paginator = s3.get_paginator(list_objects_api)
+        for page in paginator.paginate(**kwargs):
+            contents = page.get("Contents", None)
             if not contents:
-                break
-
-            for obj in contents:
-                yield obj["Key"]
-
-            token = resp.get("NextContinuationToken", None)
-            if not token:
-                break
-
-            kwargs["ContinuationToken"] = token
+                continue
+            for item in contents:
+                yield item["Key"]
 
     def list_cache_paths(self):
         return self._list_paths(self.bucket, self.prefix)
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info["scheme"] == "s3"
+        assert path_info.scheme == "s3"
 
-        paths = self._list_paths(path_info["bucket"], path_info["path"])
-        return any(path_info["path"] == path for path in paths)
+        paths = self._list_paths(path_info.bucket, path_info.path)
+        return any(path_info.path == path for path in paths)
 
-    def upload(self, from_infos, to_infos, names=None):
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         s3 = self.s3
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info["scheme"] != "s3":
+            if to_info.scheme != "s3":
                 raise NotImplementedError
 
-            if from_info["scheme"] != "local":
+            if from_info.scheme != "local":
                 raise NotImplementedError
 
             logger.debug(
                 "Uploading '{}' to '{}/{}'".format(
-                    from_info["path"], to_info["bucket"], to_info["path"]
+                    from_info.path, to_info.bucket, to_info.path
                 )
             )
 
             if not name:
-                name = os.path.basename(from_info["path"])
+                name = os.path.basename(from_info.path)
 
-            total = os.path.getsize(from_info["path"])
-            cb = Callback(name, total)
+            total = os.path.getsize(from_info.path)
+            cb = None if no_progress_bar else Callback(name, total)
 
             try:
                 s3.upload_file(
-                    from_info["path"],
-                    to_info["bucket"],
-                    to_info["path"],
-                    Callback=cb,
+                    from_info.path, to_info.bucket, to_info.path, Callback=cb
                 )
             except Exception:
-                msg = "failed to upload '{}'".format(from_info["path"])
-                logger.error(msg)
+                msg = "failed to upload '{}'".format(from_info.path)
+                logger.exception(msg)
                 continue
 
             progress.finish_target(name)
@@ -219,50 +279,47 @@ class RemoteS3(RemoteBase):
         s3 = self.s3
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != "s3":
+            if from_info.scheme != "s3":
                 raise NotImplementedError
 
-            if to_info["scheme"] == "s3":
+            if to_info.scheme == "s3":
                 self.copy(from_info, to_info, s3=s3)
                 continue
 
-            if to_info["scheme"] != "local":
+            if to_info.scheme != "local":
                 raise NotImplementedError
 
             msg = "Downloading '{}/{}' to '{}'".format(
-                from_info["bucket"], from_info["path"], to_info["path"]
+                from_info.bucket, from_info.path, to_info.path
             )
             logger.debug(msg)
 
-            tmp_file = self.tmp_file(to_info["path"])
+            tmp_file = tmp_fname(to_info.path)
             if not name:
-                name = os.path.basename(to_info["path"])
+                name = os.path.basename(to_info.path)
 
-            self._makedirs(to_info["path"])
+            makedirs(os.path.dirname(to_info.path), exist_ok=True)
 
             try:
                 if no_progress_bar:
                     cb = None
                 else:
                     total = s3.head_object(
-                        Bucket=from_info["bucket"], Key=from_info["path"]
+                        Bucket=from_info.bucket, Key=from_info.path
                     )["ContentLength"]
                     cb = Callback(name, total)
 
                 s3.download_file(
-                    from_info["bucket"],
-                    from_info["path"],
-                    tmp_file,
-                    Callback=cb,
+                    from_info.bucket, from_info.path, tmp_file, Callback=cb
                 )
             except Exception:
                 msg = "failed to download '{}/{}'".format(
-                    from_info["bucket"], from_info["path"]
+                    from_info.bucket, from_info.path
                 )
-                logger.error(msg)
+                logger.exception(msg)
                 continue
 
-            os.rename(tmp_file, to_info["path"])
+            move(tmp_file, to_info.path)
 
             if not no_progress_bar:
                 progress.finish_target(name)

dvc/remote/ssh.py

@@ -1,394 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-import getpass
-import posixpath
-
-try:
-    import paramiko
-except ImportError:
-    paramiko = None
-
-import dvc.prompt as prompt
-import dvc.logger as logger
-
-from dvc.config import Config
-from dvc.progress import progress
-from dvc.utils.compat import urlparse
-from dvc.exceptions import DvcException
-from dvc.remote.base import RemoteBase, RemoteBaseCmdError
-
-
-def sizeof_fmt(num, suffix="B"):
-    """ Convert number of bytes to human-readable string """
-    for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
-        if abs(num) < 1024.0:
-            return "%3.1f%s%s" % (num, unit, suffix)
-        num /= 1024.0
-    return "%.1f%s%s" % (num, "Y", suffix)
-
-
-def percent_cb(name, complete, total):
-    """ Callback for updating target progress """
-    logger.debug(
-        "{}: {} transferred out of {}".format(
-            name, sizeof_fmt(complete), sizeof_fmt(total)
-        )
-    )
-    progress.update_target(name, complete, total)
-
-
-def create_cb(name):
-    """ Create callback function for multipart object """
-    return lambda cur, tot: percent_cb(name, cur, tot)
-
-
-class RemoteSSHCmdError(RemoteBaseCmdError):
-    pass
-
-
-class RemoteSSH(RemoteBase):
-    scheme = "ssh"
-
-    # NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and
-    # SCP-like (ssh://[user@]host.xz:path) urls.
-    REGEX = r"^ssh://.*$"
-
-    REQUIRES = {"paramiko": paramiko}
-
-    JOBS = 4
-
-    PARAM_CHECKSUM = "md5"
-
-    DEFAULT_PORT = 22
-    TIMEOUT = 1800
-
-    def __init__(self, project, config):
-        super(RemoteSSH, self).__init__(project, config)
-        self.url = config.get(Config.SECTION_REMOTE_URL, "ssh://")
-
-        parsed = urlparse(self.url)
-        self.host = parsed.hostname
-        self.user = (
-            config.get(Config.SECTION_REMOTE_USER)
-            or parsed.username
-            or getpass.getuser()
-        )
-        self.prefix = parsed.path
-        self.port = (
-            config.get(Config.SECTION_REMOTE_PORT)
-            or parsed.port
-            or self.DEFAULT_PORT
-        )
-        self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
-        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
-        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
-        self.ask_password = config.get(
-            Config.SECTION_REMOTE_ASK_PASSWORD, False
-        )
-
-        self.path_info = {
-            "scheme": "ssh",
-            "host": self.host,
-            "user": self.user,
-            "port": self.port,
-        }
-
-    def ssh(self, host=None, user=None, port=None):
-        msg = (
-            "Establishing ssh connection with '{}' "
-            "through port '{}' as user '{}'"
-        )
-        logger.debug(msg.format(host, port, user))
-
-        ssh = paramiko.SSHClient()
-
-        ssh.load_system_host_keys()
-        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-        if self.ask_password and self.password is None:
-            msg = (
-                "Enter a private key passphrase or a password for "
-                "host '{}' port '{}' user '{}'"
-            ).format(host, port, user)
-            self.password = prompt.password(msg)
-
-        ssh.connect(
-            host,
-            username=user,
-            port=port,
-            key_filename=self.keyfile,
-            timeout=self.timeout,
-            password=self.password,
-        )
-
-        return ssh
-
-    def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info["scheme"] == "ssh"
-
-        with self.ssh(
-            path_info["host"], path_info["user"], path_info["port"]
-        ) as ssh:
-            try:
-                self._exec(ssh, "test -e {}".format(path_info["path"]))
-                exists = True
-            except RemoteSSHCmdError:
-                exists = False
-
-        return exists
-
-    def _exec(self, ssh, cmd):
-        stdin, stdout, stderr = ssh.exec_command(cmd)
-        channel = stdout.channel
-
-        stdin.close()
-        channel.shutdown_write()
-
-        stdout_chunks = []
-        stderr_chunks = []
-        while (
-            not channel.closed
-            or channel.recv_ready()
-            or channel.recv_stderr_ready()
-        ):
-            import select
-
-            got_chunk = False
-            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
-            for c in readq:
-                if c.recv_ready():
-                    stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
-                    got_chunk = True
-
-                if c.recv_stderr_ready():
-                    stderr_len = len(c.in_stderr_buffer)
-                    s = stderr.channel.recv_stderr(stderr_len)
-                    stderr_chunks.append(s)
-                    got_chunk = True
-
-            if (
-                not got_chunk
-                and stdout.channel.exit_status_ready()
-                and not stderr.channel.recv_stderr_ready()
-                and not stdout.channel.recv_ready()
-            ):
-                stdout.channel.shutdown_read()
-                stdout.channel.close()
-                break
-
-        stdout.close()
-        stderr.close()
-
-        ret = stdout.channel.recv_exit_status()
-        if ret != 0:
-            err = b"".join(stderr_chunks).decode("utf-8")
-            raise RemoteSSHCmdError(cmd, ret, err)
-
-        return b"".join(stdout_chunks).decode("utf-8")
-
-    def md5(self, path_info):
-        if path_info["scheme"] != "ssh":
-            raise NotImplementedError
-
-        ssh = self.ssh(
-            host=path_info["host"],
-            user=path_info["user"],
-            port=path_info["port"],
-        )
-
-        # Use different md5 commands depending on os
-        stdout = self._exec(ssh, "uname").strip()
-        if stdout == "Darwin":
-            md5cmd = "md5"
-            index = -1
-        elif stdout == "Linux":
-            md5cmd = "md5sum"
-            index = 0
-        else:
-            msg = "'{}' is not supported as a remote".format(stdout)
-            raise DvcException(msg)
-
-        stdout = self._exec(ssh, "{} {}".format(md5cmd, path_info["path"]))
-        md5 = stdout.split()[index]
-        ssh.close()
-
-        assert len(md5) == 32
-
-        return md5
-
-    def copy(self, from_info, to_info, ssh=None):
-        if from_info["scheme"] != "ssh" or to_info["scheme"] != "ssh":
-            raise NotImplementedError
-
-        assert from_info["host"] == to_info["host"]
-        assert from_info["user"] == to_info["user"]
-
-        s = (
-            ssh
-            if ssh
-            else self.ssh(
-                host=from_info["host"],
-                user=from_info["user"],
-                port=from_info["port"],
-            )
-        )
-
-        dname = posixpath.dirname(to_info["path"])
-        self._exec(s, "mkdir -p {}".format(dname))
-        self._exec(s, "cp {} {}".format(from_info["path"], to_info["path"]))
-
-        if not ssh:
-            s.close()
-
-    def save_info(self, path_info):
-        if path_info["scheme"] != "ssh":
-            raise NotImplementedError
-
-        return {self.PARAM_CHECKSUM: self.md5(path_info)}
-
-    def save(self, path_info):
-        if path_info["scheme"] != "ssh":
-            raise NotImplementedError
-
-        md5 = self.md5(path_info)
-        dest = path_info.copy()
-        dest["path"] = self.checksum_to_path(md5)
-
-        self.copy(path_info, dest)
-
-        return {self.PARAM_CHECKSUM: md5}
-
-    def remove(self, path_info):
-        if path_info["scheme"] != "ssh":
-            raise NotImplementedError
-
-        logger.debug(
-            "Removing ssh://{}@{}/{}".format(
-                path_info["user"], path_info["host"], path_info["path"]
-            )
-        )
-
-        ssh = self.ssh(
-            host=path_info["host"],
-            user=path_info["user"],
-            port=path_info["port"],
-        )
-        ssh.open_sftp().remove(path_info["path"])
-        ssh.close()
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
-    ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        ssh = self.ssh(
-            host=from_infos[0]["host"],
-            user=from_infos[0]["user"],
-            port=from_infos[0]["port"],
-        )
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != "ssh":
-                raise NotImplementedError
-
-            if to_info["scheme"] == "ssh":
-                assert from_info["host"] == to_info["host"]
-                assert from_info["port"] == to_info["port"]
-                assert from_info["user"] == to_info["user"]
-                self.copy(from_info, to_info, ssh=ssh)
-                continue
-
-            if to_info["scheme"] != "local":
-                raise NotImplementedError
-
-            msg = "Downloading '{}/{}' to '{}'".format(
-                from_info["host"], from_info["path"], to_info["path"]
-            )
-            logger.debug(msg)
-
-            if not name:
-                name = os.path.basename(to_info["path"])
-
-            self._makedirs(to_info["path"])
-            tmp_file = self.tmp_file(to_info["path"])
-            try:
-                ssh.open_sftp().get(
-                    from_info["path"], tmp_file, callback=create_cb(name)
-                )
-            except Exception:
-                msg = "failed to download '{}/{}' to '{}'"
-                logger.error(
-                    msg.format(
-                        from_info["host"], from_info["path"], to_info["path"]
-                    )
-                )
-                continue
-
-            os.rename(tmp_file, to_info["path"])
-            progress.finish_target(name)
-
-        ssh.close()
-
-    def upload(self, from_infos, to_infos, names=None):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        ssh = self.ssh(
-            host=to_infos[0]["host"],
-            user=to_infos[0]["user"],
-            port=to_infos[0]["port"],
-        )
-        sftp = ssh.open_sftp()
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info["scheme"] != "ssh":
-                raise NotImplementedError
-
-            if from_info["scheme"] != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Uploading '{}' to '{}/{}'".format(
-                    from_info["path"], to_info["host"], to_info["path"]
-                )
-            )
-
-            if not name:
-                name = os.path.basename(from_info["path"])
-
-            dname = posixpath.dirname(to_info["path"])
-            self._exec(ssh, "mkdir -p {}".format(dname))
-
-            try:
-                sftp.put(
-                    from_info["path"],
-                    to_info["path"],
-                    callback=create_cb(name),
-                )
-            except Exception:
-                msg = "failed to upload '{}' to '{}/{}'"
-                logger.error(
-                    msg.format(
-                        from_info["path"], to_info["host"], to_info["path"]
-                    )
-                )
-                continue
-
-            progress.finish_target(name)
-
-        sftp.close()
-        ssh.close()
-
-    def list_cache_paths(self):
-        ssh = self.ssh(host=self.host, user=self.user, port=self.port)
-        cmd = "find {} -type f -follow -print".format(self.prefix)
-        stdout = self._exec(ssh, cmd)
-        flist = stdout.split()
-        ssh.close()
-        return flist

dvc/remote/ssh/__init__.py

@@ -0,0 +1,256 @@
+from __future__ import unicode_literals
+
+import os
+import getpass
+import logging
+
+from dvc.path import Schemes
+from dvc.path.ssh import SSHPathInfo
+
+try:
+    import paramiko
+except ImportError:
+    paramiko = None
+
+import dvc.prompt as prompt
+from dvc.remote.ssh.connection import SSHConnection
+from dvc.config import Config
+from dvc.utils.compat import urlparse
+from dvc.remote.base import RemoteBase
+
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteSSH(RemoteBase):
+    scheme = Schemes.SSH
+
+    # NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and
+    # SCP-like (ssh://[user@]host.xz:/absolute/path) urls.
+    REGEX = r"^ssh://.*$"
+
+    REQUIRES = {"paramiko": paramiko}
+
+    JOBS = 4
+    PARAM_CHECKSUM = "md5"
+    DEFAULT_PORT = 22
+    TIMEOUT = 1800
+
+    def __init__(self, repo, config):
+        super(RemoteSSH, self).__init__(repo, config)
+        self.url = config.get(Config.SECTION_REMOTE_URL, "ssh://")
+
+        parsed = urlparse(self.url)
+        self.host = parsed.hostname
+
+        user_ssh_config = self._load_user_ssh_config(self.host)
+
+        self.host = user_ssh_config.get("hostname", self.host)
+        self.user = (
+            config.get(Config.SECTION_REMOTE_USER)
+            or parsed.username
+            or user_ssh_config.get("user")
+            or getpass.getuser()
+        )
+        self.prefix = parsed.path or "/"
+        self.port = (
+            config.get(Config.SECTION_REMOTE_PORT)
+            or parsed.port
+            or self._try_get_ssh_config_port(user_ssh_config)
+            or self.DEFAULT_PORT
+        )
+        self.keyfile = config.get(
+            Config.SECTION_REMOTE_KEY_FILE
+        ) or self._try_get_ssh_config_keyfile(user_ssh_config)
+        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
+        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
+        self.ask_password = config.get(
+            Config.SECTION_REMOTE_ASK_PASSWORD, False
+        )
+
+        self.path_info = SSHPathInfo(
+            host=self.host, user=self.user, port=self.port
+        )
+
+    @staticmethod
+    def ssh_config_filename():
+        return os.path.expanduser(os.path.join("~", ".ssh", "config"))
+
+    @staticmethod
+    def _load_user_ssh_config(hostname):
+        user_config_file = RemoteSSH.ssh_config_filename()
+        user_ssh_config = dict()
+        if hostname and os.path.exists(user_config_file):
+            with open(user_config_file) as f:
+                ssh_config = paramiko.SSHConfig()
+                ssh_config.parse(f)
+                user_ssh_config = ssh_config.lookup(hostname)
+        return user_ssh_config
+
+    @staticmethod
+    def _try_get_ssh_config_port(user_ssh_config):
+        try:
+            return int(user_ssh_config.get("port"))
+        except (ValueError, TypeError):
+            return None
+
+    @staticmethod
+    def _try_get_ssh_config_keyfile(user_ssh_config):
+        identity_file = user_ssh_config.get("identityfile")
+        if identity_file and len(identity_file) > 0:
+            return identity_file[0]
+        return None
+
+    def ssh(self, path_info):
+        host, user, port = path_info.host, path_info.user, path_info.port
+
+        logger.debug(
+            "Establishing ssh connection with '{host}' "
+            "through port '{port}' as user '{user}'".format(
+                host=host, user=user, port=port
+            )
+        )
+
+        if self.ask_password and not self.password:
+            self.password = prompt.password(
+                "Enter a private key passphrase or a password for "
+                "host '{host}' port '{port}' user '{user}'".format(
+                    host=host, port=port, user=user
+                )
+            )
+
+        return SSHConnection(
+            host,
+            username=user,
+            port=port,
+            key_filename=self.keyfile,
+            timeout=self.timeout,
+            password=self.password,
+        )
+
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info.scheme == self.scheme
+
+        with self.ssh(path_info) as ssh:
+            return ssh.exists(path_info.path)
+
+    def get_file_checksum(self, path_info):
+        if path_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(path_info) as ssh:
+            return ssh.md5(path_info.path)
+
+    def isdir(self, path_info):
+        with self.ssh(path_info) as ssh:
+            return ssh.isdir(path_info.path)
+
+    def copy(self, from_info, to_info):
+        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.cp(from_info.path, to_info.path)
+
+    def remove(self, path_info):
+        if path_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(path_info) as ssh:
+            ssh.remove(path_info.path)
+
+    def move(self, from_info, to_info):
+        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(from_info) as ssh:
+            ssh.move(from_info.path, to_info.path)
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
+        names = self._verify_path_args(from_infos, to_infos, names)
+        ssh = self.ssh(from_infos[0])
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info.scheme != self.scheme:
+                raise NotImplementedError
+
+            if to_info.scheme == self.scheme:
+                ssh.cp(from_info.path, to_info.path)
+                continue
+
+            if to_info.scheme != "local":
+                raise NotImplementedError
+
+            logger.debug(
+                "Downloading '{host}/{path}' to '{dest}'".format(
+                    host=from_info.host, path=from_info.path, dest=to_info.path
+                )
+            )
+
+            try:
+                ssh.download(
+                    from_info.path,
+                    to_info.path,
+                    progress_title=name,
+                    no_progress_bar=no_progress_bar,
+                )
+            except Exception:
+                logger.exception(
+                    "failed to download '{host}/{path}' to '{dest}'".format(
+                        host=from_info.host,
+                        path=from_info.path,
+                        dest=to_info.path,
+                    )
+                )
+                continue
+
+        ssh.close()
+
+    def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        with self.ssh(to_infos[0]) as ssh:
+            for from_info, to_info, name in zip(from_infos, to_infos, names):
+                if to_info.scheme != self.scheme:
+                    raise NotImplementedError
+
+                if from_info.scheme != "local":
+                    raise NotImplementedError
+
+                try:
+                    ssh.upload(
+                        from_info.path,
+                        to_info.path,
+                        progress_title=name,
+                        no_progress_bar=no_progress_bar,
+                    )
+                except Exception:
+                    logger.exception(
+                        "failed to upload '{host}/{path}' to '{dest}'".format(
+                            host=from_info.host,
+                            path=from_info.path,
+                            dest=to_info.path,
+                        )
+                    )
+                    pass
+
+    def list_cache_paths(self):
+        with self.ssh(self.path_info) as ssh:
+            return list(ssh.walk_files(self.prefix))
+
+    def walk(self, path_info):
+        with self.ssh(path_info) as ssh:
+            for entry in ssh.walk(path_info.path):
+                yield entry
+
+    def makedirs(self, path_info):
+        with self.ssh(path_info) as ssh:
+            ssh.makedirs(path_info.path)

dvc/remote/ssh/connection.py

@@ -0,0 +1,322 @@
+import errno
+import os
+import posixpath
+import logging
+from stat import S_ISDIR
+
+try:
+    import paramiko
+except ImportError:
+    paramiko = None
+
+from dvc.utils import tmp_fname
+from dvc.utils.compat import makedirs
+from dvc.progress import progress
+from dvc.exceptions import DvcException
+from dvc.remote.base import RemoteCmdError
+
+
+logger = logging.getLogger(__name__)
+
+
+def sizeof_fmt(num, suffix="B"):
+    """ Convert number of bytes to human-readable string """
+    for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, "Y", suffix)
+
+
+def percent_cb(name, complete, total):
+    """ Callback for updating target progress """
+    logger.debug(
+        "{}: {} transferred out of {}".format(
+            name, sizeof_fmt(complete), sizeof_fmt(total)
+        )
+    )
+    progress.update_target(name, complete, total)
+
+
+def create_cb(name):
+    """ Create callback function for multipart object """
+    return lambda cur, tot: percent_cb(name, cur, tot)
+
+
+class SSHConnection:
+    def __init__(self, *args, **kwargs):
+        self.timeout = kwargs.get("timeout", 1800)
+
+        self._ssh = paramiko.SSHClient()
+        self._ssh.load_system_host_keys()
+        self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        self._ssh.connect(*args, **kwargs)
+        self._sftp = None
+        self._sftp_alive = False
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        self.close()
+
+    def _sftp_connect(self):
+        if not self._sftp or not self._sftp_alive:
+            self._sftp = self._ssh.open_sftp()
+            self._sftp_alive = True
+
+    def close(self):
+        if self._sftp:
+            self._sftp.close()
+            self._sftp_alive = False
+
+        self._ssh.close()
+
+    def exists(self, path):
+        self._sftp_connect()
+        try:
+            return self._sftp.stat(path)
+        except IOError:
+            return False
+        pass
+
+    def isdir(self, path):
+        from stat import S_ISDIR
+
+        self._sftp_connect()
+        try:
+            return S_ISDIR(self._sftp.stat(path).st_mode)
+        except IOError:
+            return False
+
+    def isfile(self, path):
+        from stat import S_ISREG
+
+        self._sftp_connect()
+        try:
+            return S_ISREG(self._sftp.stat(path).st_mode)
+        except IOError:
+            return False
+
+    def islink(self, path):
+        from stat import S_ISLNK
+
+        self._sftp_connect()
+        try:
+            return S_ISLNK(self._sftp.stat(path).st_mode)
+        except IOError:
+            return False
+
+    def makedirs(self, path):
+        self._sftp_connect()
+
+        if self.isdir(path):
+            return
+
+        if self.isfile(path) or self.islink(path):
+            raise DvcException(
+                "a file with the same name '{}' already exists".format(path)
+            )
+
+        head, tail = posixpath.split(path)
+
+        if head:
+            self.makedirs(head)
+
+        if tail:
+            self._sftp.mkdir(path)
+
+    def walk(self, directory, topdown=True):
+        # NOTE: original os.walk() implementation [1] with default options was
+        # used as a template.
+        #
+        # [1] https://github.com/python/cpython/blob/master/Lib/os.py
+
+        self._sftp_connect()
+
+        try:
+            dir_entries = self._sftp.listdir_attr(directory)
+        except IOError as exc:
+            raise DvcException(
+                "couldn't get the '{}' remote directory files list".format(
+                    directory
+                ),
+                cause=exc,
+            )
+
+        dirs = []
+        nondirs = []
+        for entry in dir_entries:
+            name = entry.filename
+            if S_ISDIR(entry.st_mode):
+                dirs.append(name)
+            else:
+                nondirs.append(name)
+
+        if topdown:
+            yield directory, dirs, nondirs
+
+        for dname in dirs:
+            newpath = posixpath.join(directory, dname)
+            for entry in self.walk(newpath, topdown=topdown):
+                yield entry
+
+        if not topdown:
+            yield directory, dirs, nondirs
+
+    def walk_files(self, directory):
+        for root, dirs, files in self.walk(directory):
+            for fname in files:
+                yield posixpath.join(root, fname)
+
+    def _remove_file(self, path):
+        try:
+            self._sftp.remove(path)
+        except IOError as exc:
+            if exc.errno != errno.ENOENT:
+                raise
+
+    def _remove_dir(self, path):
+        for root, dirs, files in self.walk(path, topdown=False):
+            for fname in files:
+                path = posixpath.join(root, fname)
+                self._remove_file(path)
+
+            for dname in dirs:
+                path = posixpath.join(root, dname)
+                self._sftp.rmdir(dname)
+        try:
+            self._sftp.rmdir(path)
+        except IOError as exc:
+            if exc.errno != errno.ENOENT:
+                raise
+
+    def remove(self, path):
+        self._sftp_connect()
+
+        if self.isdir(path):
+            self._remove_dir(path)
+        else:
+            self._remove_file(path)
+
+    def download(self, src, dest, no_progress_bar=False, progress_title=None):
+        self._sftp_connect()
+
+        makedirs(os.path.dirname(dest), exist_ok=True)
+        tmp_file = tmp_fname(dest)
+
+        if no_progress_bar:
+            self._sftp.get(src, tmp_file)
+        else:
+            if not progress_title:
+                progress_title = os.path.basename(dest)
+
+            self._sftp.get(src, tmp_file, callback=create_cb(progress_title))
+            progress.finish_target(progress_title)
+
+        if os.path.exists(dest):
+            os.remove(dest)
+
+        os.rename(tmp_file, dest)
+
+    def move(self, src, dst):
+        self.makedirs(posixpath.dirname(dst))
+        self._sftp_connect()
+        self._sftp.rename(src, dst)
+
+    def upload(self, src, dest, no_progress_bar=False, progress_title=None):
+        self._sftp_connect()
+
+        self.makedirs(posixpath.dirname(dest))
+        tmp_file = tmp_fname(dest)
+
+        if no_progress_bar:
+            self._sftp.put(src, tmp_file)
+        else:
+            if not progress_title:
+                progress_title = posixpath.basename(dest)
+
+            self._sftp.put(src, tmp_file, callback=create_cb(progress_title))
+            progress.finish_target(progress_title)
+
+        self._sftp.rename(tmp_file, dest)
+
+    def execute(self, cmd):
+        stdin, stdout, stderr = self._ssh.exec_command(cmd)
+        channel = stdout.channel
+
+        stdin.close()
+        channel.shutdown_write()
+
+        stdout_chunks = []
+        stderr_chunks = []
+        while (
+            not channel.closed
+            or channel.recv_ready()
+            or channel.recv_stderr_ready()
+        ):
+            import select
+
+            got_chunk = False
+            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
+            for c in readq:
+                if c.recv_ready():
+                    stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
+                    got_chunk = True
+
+                if c.recv_stderr_ready():
+                    stderr_len = len(c.in_stderr_buffer)
+                    s = stderr.channel.recv_stderr(stderr_len)
+                    stderr_chunks.append(s)
+                    got_chunk = True
+
+            if (
+                not got_chunk
+                and stdout.channel.exit_status_ready()
+                and not stderr.channel.recv_stderr_ready()
+                and not stdout.channel.recv_ready()
+            ):
+                stdout.channel.shutdown_read()
+                stdout.channel.close()
+                break
+
+        stdout.close()
+        stderr.close()
+
+        ret = stdout.channel.recv_exit_status()
+        if ret != 0:
+            err = b"".join(stderr_chunks).decode("utf-8")
+            raise RemoteCmdError("ssh", cmd, ret, err)
+
+        return b"".join(stdout_chunks).decode("utf-8")
+
+    def md5(self, path):
+        """
+        Use different md5 commands depending on the OS:
+
+         - Darwin's `md5` returns BSD-style checksums by default
+         - Linux's `md5sum` needs the `--tag` flag for a similar output
+
+         Example:
+              MD5 (foo.txt) = f3d220a856b52aabbf294351e8a24300
+        """
+        uname = self.execute("uname").strip()
+
+        command = {
+            "Darwin": "md5 {}".format(path),
+            "Linux": "md5sum --tag {}".format(path),
+        }.get(uname)
+
+        if not command:
+            raise DvcException(
+                "'{uname}' is not supported as a remote".format(uname=uname)
+            )
+
+        md5 = self.execute(command).split()[-1]
+        assert len(md5) == 32
+        return md5
+
+    def cp(self, src, dest):
+        self.makedirs(posixpath.dirname(dest))
+        self.execute("cp {} {}".format(src, dest))

dvc/repo/__init__.py

@@ -0,0 +1,499 @@
+from __future__ import unicode_literals
+
+import os
+import logging
+
+import dvc.prompt as prompt
+
+from dvc.exceptions import (
+    DvcException,
+    NotDvcRepoError,
+    OutputNotFoundError,
+    TargetNotDirectoryError,
+)
+from dvc.ignore import DvcIgnoreFileHandler
+from dvc.path.local import LocalPathInfo
+
+logger = logging.getLogger(__name__)
+
+
+class Repo(object):
+    DVC_DIR = ".dvc"
+
+    from dvc.repo.destroy import destroy
+    from dvc.repo.install import install
+    from dvc.repo.add import add
+    from dvc.repo.remove import remove
+    from dvc.repo.lock import lock as lock_stage
+    from dvc.repo.move import move
+    from dvc.repo.run import run
+    from dvc.repo.imp import imp
+    from dvc.repo.reproduce import reproduce
+    from dvc.repo.checkout import checkout
+    from dvc.repo.push import push
+    from dvc.repo.fetch import fetch
+    from dvc.repo.pull import pull
+    from dvc.repo.status import status
+    from dvc.repo.gc import gc
+    from dvc.repo.commit import commit
+    from dvc.repo.diff import diff
+    from dvc.repo.brancher import brancher
+
+    def __init__(self, root_dir=None):
+        from dvc.config import Config
+        from dvc.state import State
+        from dvc.lock import Lock
+        from dvc.scm import SCM
+        from dvc.cache import Cache
+        from dvc.data_cloud import DataCloud
+        from dvc.updater import Updater
+        from dvc.repo.metrics import Metrics
+        from dvc.scm.tree import WorkingTree
+        from dvc.repo.tag import Tag
+        from dvc.repo.pkg import Pkg
+
+        root_dir = self.find_root(root_dir)
+
+        self.root_dir = os.path.abspath(os.path.realpath(root_dir))
+        self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
+
+        self.config = Config(self.dvc_dir)
+
+        self.tree = WorkingTree(self.root_dir)
+
+        self.scm = SCM(self.root_dir, repo=self)
+        self.lock = Lock(self.dvc_dir)
+        # NOTE: storing state and link_state in the repository itself to avoid
+        # any possible state corruption in 'shared cache dir' scenario.
+        self.state = State(self, self.config.config)
+
+        core = self.config.config[Config.SECTION_CORE]
+
+        level = core.get(Config.SECTION_CORE_LOGLEVEL)
+        if level:
+            logger.setLevel(level.upper())
+
+        self.cache = Cache(self)
+        self.cloud = DataCloud(self, config=self.config.config)
+        self.updater = Updater(self.dvc_dir)
+
+        self.metrics = Metrics(self)
+        self.tag = Tag(self)
+        self.pkg = Pkg(self)
+
+        self._ignore()
+
+        self.updater.check()
+
+    def __repr__(self):
+        return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
+
+    @staticmethod
+    def find_root(root=None):
+        if root is None:
+            root = os.getcwd()
+        else:
+            root = os.path.abspath(os.path.realpath(root))
+
+        while True:
+            dvc_dir = os.path.join(root, Repo.DVC_DIR)
+            if os.path.isdir(dvc_dir):
+                return root
+            if os.path.ismount(root):
+                break
+            root = os.path.dirname(root)
+        raise NotDvcRepoError(root)
+
+    @staticmethod
+    def find_dvc_dir(root=None):
+        root_dir = Repo.find_root(root)
+        return os.path.join(root_dir, Repo.DVC_DIR)
+
+    @staticmethod
+    def init(root_dir=os.curdir, no_scm=False, force=False):
+        from dvc.repo.init import init
+
+        init(root_dir=root_dir, no_scm=no_scm, force=force)
+        return Repo(root_dir)
+
+    def unprotect(self, target):
+        path_info = LocalPathInfo(path=target)
+        return self.cache.local.unprotect(path_info)
+
+    def _ignore(self):
+        flist = [
+            self.state.state_file,
+            self.lock.lock_file,
+            self.config.config_local_file,
+            self.updater.updater_file,
+            self.updater.lock.lock_file,
+        ] + self.state.temp_files
+
+        if self.cache.local.cache_dir.startswith(self.root_dir):
+            flist += [self.cache.local.cache_dir]
+
+        self.scm.ignore_list(flist)
+
+    def check_dag(self, stages):
+        """Generate graph including the new stage to check for errors"""
+        self.graph(stages=stages)
+
+    @staticmethod
+    def _check_cyclic_graph(graph):
+        import networkx as nx
+        from dvc.exceptions import CyclicGraphError
+
+        cycles = list(nx.simple_cycles(graph))
+
+        if cycles:
+            raise CyclicGraphError(cycles[0])
+
+    def _get_pipeline(self, node):
+        pipelines = [i for i in self.pipelines() if i.has_node(node)]
+        assert len(pipelines) == 1
+        return pipelines[0]
+
+    def collect(self, target, with_deps=False, recursive=False):
+        import networkx as nx
+        from dvc.stage import Stage
+
+        if not target or recursive:
+            return self.active_stages(target)
+
+        stage = Stage.load(self, target)
+        if not with_deps:
+            return [stage]
+
+        node = os.path.relpath(stage.path, self.root_dir)
+        G = self._get_pipeline(node)
+
+        ret = []
+        for n in nx.dfs_postorder_nodes(G, node):
+            ret.append(G.node[n]["stage"])
+
+        return ret
+
+    def _collect_dir_cache(
+        self, out, branch=None, remote=None, force=False, jobs=None
+    ):
+        info = out.dumpd()
+        ret = [info]
+        r = out.remote
+        md5 = info[r.PARAM_CHECKSUM]
+
+        if self.cache.local.changed_cache_file(md5):
+            try:
+                self.cloud.pull(
+                    ret, jobs=jobs, remote=remote, show_checksums=False
+                )
+            except DvcException as exc:
+                msg = "Failed to pull cache for '{}': {}"
+                logger.debug(msg.format(out, exc))
+
+        if self.cache.local.changed_cache_file(md5):
+            msg = (
+                "Missing cache for directory '{}'. "
+                "Cache for files inside will be lost. "
+                "Would you like to continue? Use '-f' to force. "
+            )
+            if not force and not prompt.confirm(msg):
+                raise DvcException(
+                    "unable to fully collect used cache"
+                    " without cache for directory '{}'".format(out)
+                )
+            else:
+                return ret
+
+        for i in out.dir_cache:
+            i["branch"] = branch
+            i[r.PARAM_PATH] = os.path.join(
+                info[r.PARAM_PATH], i[r.PARAM_RELPATH]
+            )
+            ret.append(i)
+
+        return ret
+
+    def _collect_used_cache(
+        self, out, branch=None, remote=None, force=False, jobs=None
+    ):
+        if not out.use_cache or not out.info:
+            if not out.info:
+                logger.warning(
+                    "Output '{}'({}) is missing version "
+                    "info. Cache for it will not be collected. "
+                    "Use dvc repro to get your pipeline up to "
+                    "date.".format(out, out.stage)
+                )
+            return []
+
+        info = out.dumpd()
+        info["branch"] = branch
+        ret = [info]
+
+        if out.scheme != "local":
+            return ret
+
+        if not out.is_dir_checksum:
+            return ret
+
+        return self._collect_dir_cache(
+            out, branch=branch, remote=remote, force=force, jobs=jobs
+        )
+
+    def used_cache(
+        self,
+        target=None,
+        all_branches=False,
+        active=True,
+        with_deps=False,
+        all_tags=False,
+        remote=None,
+        force=False,
+        jobs=None,
+        recursive=False,
+    ):
+        cache = {}
+        cache["local"] = []
+        cache["s3"] = []
+        cache["gs"] = []
+        cache["hdfs"] = []
+        cache["ssh"] = []
+        cache["azure"] = []
+
+        for branch in self.brancher(
+            all_branches=all_branches, all_tags=all_tags
+        ):
+            if target:
+                if recursive:
+                    stages = self.stages(target)
+                else:
+                    stages = self.collect(target, with_deps=with_deps)
+            elif active:
+                stages = self.active_stages()
+            else:
+                stages = self.stages()
+
+            for stage in stages:
+                if active and not target and stage.locked:
+                    logger.warning(
+                        "DVC file '{path}' is locked. Its dependencies are"
+                        " not going to be pushed/pulled/fetched.".format(
+                            path=stage.relpath
+                        )
+                    )
+
+                for out in stage.outs:
+                    scheme = out.path_info.scheme
+                    cache[scheme].extend(
+                        self._collect_used_cache(
+                            out,
+                            branch=branch,
+                            remote=remote,
+                            force=force,
+                            jobs=jobs,
+                        )
+                    )
+
+        return cache
+
+    def graph(self, stages=None, from_directory=None):
+        """Generate a graph by using the given stages on the given directory
+
+        The nodes of the graph are the stage's path relative to the root.
+
+        Edges are created when the output of one stage is used as a
+        dependency in other stage.
+
+        The direction of the edges goes from the stage to its dependency:
+
+        For example, running the following:
+
+            $ dvc run -o A "echo A > A"
+            $ dvc run -d A -o B "echo B > B"
+            $ dvc run -d B -o C "echo C > C"
+
+        Will create the following graph:
+
+               ancestors <--
+                           |
+                C.dvc -> B.dvc -> A.dvc
+                |          |
+                |          --> descendants
+                |
+                ------- pipeline ------>
+                           |
+                           v
+              (weakly connected components)
+
+        Args:
+            stages (list): used to build a graph, if None given, use the ones
+                on the `from_directory`.
+
+            from_directory (str): directory where to look at for stages, if
+                None is given, use the current working directory
+
+        Raises:
+            OutputDuplicationError: two outputs with the same path
+            StagePathAsOutputError: stage inside an output directory
+            OverlappingOutputPathsError: output inside output directory
+            CyclicGraphError: resulting graph has cycles
+        """
+        import networkx as nx
+        from dvc.exceptions import (
+            OutputDuplicationError,
+            StagePathAsOutputError,
+            OverlappingOutputPathsError,
+        )
+
+        G = nx.DiGraph()
+        G_active = nx.DiGraph()
+        stages = stages or self.stages(from_directory, check_dag=False)
+        stages = [stage for stage in stages if stage]
+        outs = []
+
+        for stage in stages:
+            for out in stage.outs:
+                existing = []
+                for o in outs:
+                    if o.path == out.path:
+                        existing.append(o.stage)
+
+                    in_o_dir = out.path.startswith(o.path + o.sep)
+                    in_out_dir = o.path.startswith(out.path + out.sep)
+                    if in_o_dir or in_out_dir:
+                        raise OverlappingOutputPathsError(o, out)
+
+                if existing:
+                    stages = [stage.relpath, existing[0].relpath]
+                    raise OutputDuplicationError(out.path, stages)
+
+                outs.append(out)
+
+        for stage in stages:
+            path_dir = os.path.dirname(stage.path) + os.sep
+            for out in outs:
+                if path_dir.startswith(out.path + os.sep):
+                    raise StagePathAsOutputError(stage.wdir, stage.relpath)
+
+        for stage in stages:
+            node = os.path.relpath(stage.path, self.root_dir)
+
+            G.add_node(node, stage=stage)
+            G_active.add_node(node, stage=stage)
+
+            for dep in stage.deps:
+                for out in outs:
+                    if (
+                        out.path != dep.path
+                        and not dep.path.startswith(out.path + out.sep)
+                        and not out.path.startswith(dep.path + dep.sep)
+                    ):
+                        continue
+
+                    dep_stage = out.stage
+                    dep_node = os.path.relpath(dep_stage.path, self.root_dir)
+                    G.add_node(dep_node, stage=dep_stage)
+                    G.add_edge(node, dep_node)
+                    if not stage.locked:
+                        G_active.add_node(dep_node, stage=dep_stage)
+                        G_active.add_edge(node, dep_node)
+
+        self._check_cyclic_graph(G)
+
+        return G, G_active
+
+    def pipelines(self, from_directory=None):
+        import networkx as nx
+
+        G, G_active = self.graph(from_directory=from_directory)
+
+        return [
+            G.subgraph(c).copy() for c in nx.weakly_connected_components(G)
+        ]
+
+    def stages(self, from_directory=None, check_dag=True):
+        """
+        Walks down the root directory looking for Dvcfiles,
+        skipping the directories that are related with
+        any SCM (e.g. `.git`), DVC itself (`.dvc`), or directories
+        tracked by DVC (e.g. `dvc add data` would skip `data/`)
+
+        NOTE: For large repos, this could be an expensive
+              operation. Consider using some memoization.
+        """
+        from dvc.stage import Stage
+
+        if not from_directory:
+            from_directory = self.root_dir
+        elif not os.path.isdir(from_directory):
+            raise TargetNotDirectoryError(from_directory)
+
+        stages = []
+        outs = []
+
+        ignore_file_handler = DvcIgnoreFileHandler(self.tree)
+        for root, dirs, files in self.tree.walk(
+            from_directory, ignore_file_handler=ignore_file_handler
+        ):
+            for fname in files:
+                path = os.path.join(root, fname)
+                if not Stage.is_valid_filename(path):
+                    continue
+                stage = Stage.load(self, path)
+                for out in stage.outs:
+                    outs.append(out.path + out.sep)
+                stages.append(stage)
+
+            def filter_dirs(dname, root=root):
+                path = os.path.join(root, dname)
+                if path in (self.dvc_dir, self.scm.dir):
+                    return False
+                for out in outs:
+                    if path == os.path.normpath(out) or path.startswith(out):
+                        return False
+                return True
+
+            dirs[:] = list(filter(filter_dirs, dirs))
+
+        if check_dag:
+            self.check_dag(stages)
+
+        return stages
+
+    def active_stages(self, from_directory=None):
+        import networkx as nx
+
+        stages = []
+        for G in self.pipelines(from_directory):
+            stages.extend(list(nx.get_node_attributes(G, "stage").values()))
+        return stages
+
+    def find_outs_by_path(self, path, outs=None, recursive=False):
+        if not outs:
+            # there is no `from_directory=path` argument because some data
+            # files might be generated to an upper level, and so it is
+            # needed to look at all the files (project root_dir)
+            stages = self.stages()
+            outs = [out for stage in stages for out in stage.outs]
+
+        abs_path = os.path.abspath(path)
+        is_dir = self.tree.isdir(abs_path)
+
+        def func(out):
+            if out.path == abs_path:
+                return True
+
+            if is_dir and recursive and out.path.startswith(abs_path + os.sep):
+                return True
+
+            return False
+
+        matched = list(filter(func, outs))
+        if not matched:
+            raise OutputNotFoundError(path)
+
+        return matched
+
+    def is_dvc_internal(self, path):
+        path_parts = os.path.normpath(path).split(os.path.sep)
+        return self.DVC_DIR in path_parts

dvc/repo/add.py

@@ -0,0 +1,73 @@
+import os
+import logging
+import colorama
+
+from dvc.repo.scm_context import scm_context
+from dvc.stage import Stage
+from dvc.utils import walk_files, LARGE_DIR_SIZE
+from dvc.exceptions import RecursiveAddingWhileUsingFilename
+
+
+logger = logging.getLogger(__name__)
+
+
+@scm_context
+def add(repo, target, recursive=False, no_commit=False, fname=None):
+    if recursive and fname:
+        raise RecursiveAddingWhileUsingFilename()
+
+    targets = _find_all_targets(repo, target, recursive)
+
+    if os.path.isdir(target) and len(targets) > LARGE_DIR_SIZE:
+        logger.warning(
+            "You are adding a large directory '{target}' recursively,"
+            " consider tracking it as a whole instead.\n"
+            "{purple}HINT:{nc} Remove the generated stage files and then"
+            " run {cyan}dvc add {target}{nc}".format(
+                purple=colorama.Fore.MAGENTA,
+                cyan=colorama.Fore.CYAN,
+                nc=colorama.Style.RESET_ALL,
+                target=target,
+            )
+        )
+
+    stages = _create_stages(repo, targets, fname, no_commit)
+
+    repo.check_dag(repo.stages() + stages)
+
+    for stage in stages:
+        stage.dump()
+    return stages
+
+
+def _find_all_targets(repo, target, recursive):
+    if os.path.isdir(target) and recursive:
+        return [
+            fname
+            for fname in walk_files(target)
+            if not repo.is_dvc_internal(fname)
+            if not Stage.is_stage_file(fname)
+            if not repo.scm.belongs_to_scm(fname)
+            if not repo.scm.is_tracked(fname)
+        ]
+    return [target]
+
+
+def _create_stages(repo, targets, fname, no_commit):
+    stages = []
+
+    with repo.state:
+        for out in targets:
+            stage = Stage.create(repo=repo, outs=[out], add=True, fname=fname)
+
+            if not stage:
+                continue
+
+            stage.save()
+
+            if not no_commit:
+                stage.commit()
+
+            stages.append(stage)
+
+    return stages

dvc/repo/brancher.py

@@ -0,0 +1,56 @@
+def brancher(  # noqa: E302
+    self, branches=None, all_branches=False, tags=None, all_tags=False
+):
+    """Generator that iterates over specified revisions.
+
+    Args:
+        branches (list): a list of branches to iterate over.
+        all_branches (bool): iterate over all available branches.
+        tags (list): a list of tags to iterate over.
+        all_tags (bool): iterate over all available tags.
+
+    Yields:
+        str: the display name for the currently selected tree, it could be:
+            - a git revision identifier
+            - empty string it there is no branches to iterate over
+            - "Working Tree" if there are uncommited changes in the SCM repo
+    """
+    if not any([branches, all_branches, tags, all_tags]):
+        yield ""
+        return
+
+    saved_tree = self.tree
+    revs = []
+
+    scm = self.scm
+
+    if self.scm.is_dirty():
+        from dvc.scm.tree import WorkingTree
+
+        self.tree = WorkingTree(self.root_dir)
+        yield "Working Tree"
+
+    if all_branches:
+        branches = scm.list_branches()
+
+    if all_tags:
+        tags = scm.list_tags()
+
+    if branches is None:
+        revs.extend([scm.active_branch()])
+    else:
+        revs.extend(branches)
+
+    if tags is not None:
+        revs.extend(tags)
+
+    # NOTE: it might be a good idea to wrap this loop in try/finally block
+    # to don't leave the tree on some unexpected branch after the
+    # `brancher()`, but this could cause problems on exception handling
+    # code which might expect the tree on which exception was raised to
+    # stay in place. This behavior is a subject to change.
+    for rev in revs:
+        self.tree = scm.get_tree(rev)
+        yield rev
+
+    self.tree = saved_tree

dvc/repo/checkout.py

@@ -0,0 +1,56 @@
+from __future__ import unicode_literals
+
+import logging
+
+from dvc.exceptions import CheckoutErrorSuggestGit
+from dvc.progress import ProgressCallback
+
+
+logger = logging.getLogger(__name__)
+
+
+def _cleanup_unused_links(self, all_stages):
+    used = []
+    for stage in all_stages:
+        for out in stage.outs:
+            used.append(out.path)
+    self.state.remove_unused_links(used)
+
+
+def get_all_files_numbers(stages):
+    return sum(stage.get_all_files_number() for stage in stages)
+
+
+def get_progress_callback(stages):
+    total_files_num = get_all_files_numbers(stages)
+    if total_files_num == 0:
+        return None
+    return ProgressCallback(total_files_num)
+
+
+def checkout(self, target=None, with_deps=False, force=False, recursive=False):
+    from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
+
+    all_stages = self.stages()
+
+    try:
+        stages = self.collect(target, with_deps=with_deps, recursive=recursive)
+    except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+        if not target:
+            raise
+        raise CheckoutErrorSuggestGit(target, exc)
+
+    with self.state:
+        _cleanup_unused_links(self, all_stages)
+        progress_callback = get_progress_callback(stages)
+
+        for stage in stages:
+            if stage.locked:
+                logger.warning(
+                    "DVC file '{path}' is locked. Its dependencies are"
+                    " not going to be checked out.".format(path=stage.relpath)
+                )
+
+            stage.checkout(force=force, progress_callback=progress_callback)
+        if progress_callback:
+            progress_callback.finish("Checkout finished!")

dvc/repo/commit.py

@@ -0,0 +1,7 @@
+def commit(self, target, with_deps=False, recursive=False, force=False):
+    stages = self.collect(target, with_deps=with_deps, recursive=recursive)
+    with self.state:
+        for stage in stages:
+            stage.check_can_commit(force=force)
+            stage.commit()
+            stage.dump()

dvc/repo/destroy.py

@@ -0,0 +1,8 @@
+import shutil
+
+
+def destroy(self):
+    for stage in self.stages():
+        stage.remove()
+
+    shutil.rmtree(self.dvc_dir)

dvc/repo/diff.py

@@ -0,0 +1,252 @@
+from __future__ import unicode_literals
+
+import os
+from errno import ENOENT
+
+
+import dvc.logger as logger
+from dvc.scm.base import FileNotInCommitError
+from dvc.scm.git import (
+    DIFF_A_TREE,
+    DIFF_B_TREE,
+    DIFF_A_REF,
+    DIFF_B_REF,
+    DIFF_EQUAL,
+)
+
+
+DIFF_TARGET = "target"
+DIFF_IS_DIR = "is_dir"
+DIFF_OLD_FILE = "old_file"
+DIFF_OLD_CHECKSUM = "old_checksum"
+DIFF_NEW_FILE = "new_file"
+DIFF_NEW_CHECKSUM = "new_checksum"
+DIFF_SIZE = "size_diff"
+DIFF_DEL = "del"
+DIFF_IDENT = "ident"
+DIFF_CHANGE = "changes"
+DIFF_NEW = "new"
+DIFF_MOVE = "moves"
+DIFF_LIST = "diffs"
+DIFF_SIZE_UNKNOWN = "?"
+DIFF_A_OUTPUT = "a_output"
+DIFF_B_OUTPUT = "b_output"
+DIFF_DELETED = "deleted_file"
+DIFF_IS_NEW = "created_file"
+
+
+def _file_not_exists(error, result):
+    if error.errno == ENOENT:
+        result.update({DIFF_SIZE: DIFF_SIZE_UNKNOWN})
+    else:
+        raise error
+
+
+def _extract_dir(self, dir_not_exists, output):
+    """Extract the content of dvc tree file
+    Args:
+        self(object) - Repo class instance
+        dir_not_exists(bool) - flag for directory existence
+        output(object) - OutputLOCAL class instance
+    Returns:
+        dict - dictionary with keys - paths to file in .dvc/cache
+                               values -checksums for that files
+    """
+    if not dir_not_exists:
+        lst = output.dir_cache
+        return {i["relpath"]: i["md5"] for i in lst}
+    return {}
+
+
+def _get_dir_info(dir_not_exists, a_output):
+    if not dir_not_exists:
+        return str(a_output), a_output.checksum
+    return "", ""
+
+
+def _ident_files(a_entries, b_entries):
+    keys = [
+        key for key in a_entries.keys() if b_entries.get(key) == a_entries[key]
+    ]
+    return len(keys)
+
+
+def _modified_files(self, a_entries, b_entries):
+    keys = [key for key in a_entries.keys() if key in b_entries]
+    diff_size = 0
+    modified_count = 0
+    for key in keys:
+        if a_entries[key] != b_entries[key]:
+            modified_count
+            diff_size += os.path.getsize(
+                self.cache.local.get(b_entries[key])
+            ) - os.path.getsize(self.cache.local.get(a_entries[key]))
+    return modified_count, diff_size
+
+
+def _deleted_files(self, a_entries, b_entries):
+    diff_size = 0
+    deleted_count = 0
+    for key, value in a_entries.items():
+        if key not in b_entries:
+            deleted_count += 1
+            diff_size -= os.path.getsize(self.cache.local.get(a_entries[key]))
+    return deleted_count, diff_size
+
+
+def _new_files(self, a_entries, b_entries):
+    diff_size = 0
+    new_count = 0
+    for key, value in b_entries.items():
+        if key not in a_entries:
+            new_count += 1
+            diff_size += os.path.getsize(self.cache.local.get(b_entries[key]))
+    return new_count, diff_size
+
+
+def _get_tree_changes(self, a_entries, b_entries):
+    result = {
+        DIFF_DEL: 0,
+        DIFF_IDENT: 0,
+        DIFF_CHANGE: 0,
+        DIFF_NEW: 0,
+        DIFF_MOVE: 0,
+    }
+    result[DIFF_IDENT] = _ident_files(a_entries, b_entries)
+    result[DIFF_CHANGE], diff_size = _modified_files(
+        self, a_entries, b_entries
+    )
+    result[DIFF_SIZE] = diff_size
+    result[DIFF_DEL], diff_size = _deleted_files(self, a_entries, b_entries)
+    result[DIFF_SIZE] += diff_size
+    result[DIFF_NEW], diff_size = _new_files(self, a_entries, b_entries)
+    result[DIFF_SIZE] += diff_size
+    return result
+
+
+def _check_local_cache(a_out, is_checked):
+    if a_out is not None and a_out.scheme != "local":
+        is_checked.append(str(a_out))
+        return True
+    return False
+
+
+def _is_dir(path, a_outs, b_outs):
+    if a_outs.get(path):
+        return a_outs[path].is_dir_checksum
+    else:
+        return b_outs[path].is_dir_checksum
+
+
+def _get_diff_outs(self, diff_dct):
+    self.tree = diff_dct[DIFF_A_TREE]
+    a_outs = {str(out): out for st in self.stages() for out in st.outs}
+    self.tree = diff_dct[DIFF_B_TREE]
+    b_outs = {str(out): out for st in self.stages() for out in st.outs}
+    outs_paths = set(a_outs.keys())
+    outs_paths.update(b_outs.keys())
+    results = {}
+    non_local_cache = []
+    for path in outs_paths:
+        check1 = _check_local_cache(a_outs.get(path), non_local_cache)
+        check2 = _check_local_cache(b_outs.get(path), non_local_cache)
+        # skip files/directories with non-local cache for now
+        if check1 or check2:
+            continue
+        results[path] = {}
+        results[path][DIFF_A_OUTPUT] = a_outs.get(path)
+        results[path][DIFF_B_OUTPUT] = b_outs.get(path)
+        results[path][DIFF_IS_NEW] = path not in a_outs
+        results[path][DIFF_DELETED] = path not in b_outs
+        results[path][DIFF_IS_DIR] = _is_dir(path, a_outs, b_outs)
+    if non_local_cache:
+        logger.warning(
+            "Diff is not supported for non-local outputs. Ignoring: {}".format(
+                non_local_cache
+            )
+        )
+
+    return results
+
+
+def _diff_dir(self, target, diff_dct):
+    result = {DIFF_TARGET: target}
+    result[DIFF_IS_DIR] = True
+    a_entries, b_entries = {}, {}
+    try:
+        a_entries = _extract_dir(
+            self, diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
+        )
+        b_entries = _extract_dir(
+            self, diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
+        )
+        result[DIFF_OLD_FILE], result[DIFF_OLD_CHECKSUM] = _get_dir_info(
+            diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
+        )
+        result[DIFF_NEW_FILE], result[DIFF_NEW_CHECKSUM] = _get_dir_info(
+            diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
+        )
+        result.update(_get_tree_changes(self, a_entries, b_entries))
+    except IOError as e:
+        _file_not_exists(e, result)
+    return result
+
+
+def _diff_file(self, target, diff_dct):
+    result = {DIFF_TARGET: target}
+    size = 0
+    try:
+        if not diff_dct[DIFF_IS_NEW]:
+            result[DIFF_OLD_FILE] = target
+            result[DIFF_OLD_CHECKSUM] = diff_dct[DIFF_A_OUTPUT].checksum
+            size -= os.path.getsize(
+                self.cache.local.get(diff_dct[DIFF_A_OUTPUT].checksum)
+            )
+        if not diff_dct[DIFF_DELETED]:
+            result[DIFF_NEW_FILE] = target
+            result[DIFF_NEW_CHECKSUM] = diff_dct[DIFF_B_OUTPUT].checksum
+            size += os.path.getsize(
+                self.cache.local.get(diff_dct[DIFF_B_OUTPUT].checksum)
+            )
+        result[DIFF_SIZE] = size
+    except IOError as e:
+        _file_not_exists(e, result)
+    return result
+
+
+def _diff_royal(self, target, diff_dct):
+    if diff_dct[DIFF_IS_DIR]:
+        return _diff_dir(self, target, diff_dct)
+    return _diff_file(self, target, diff_dct)
+
+
+def diff(self, a_ref, target=None, b_ref=None):
+    """Gerenates diff message string output
+
+    Args:
+        target(str) - file/directory to check diff of
+        a_ref(str) - first tag
+        (optional) b_ref(str) - second git tag
+
+    Returns:
+        string: string of output message with diff info
+    """
+    result = {}
+    diff_dct = self.scm.get_diff_trees(a_ref, b_ref=b_ref)
+    result[DIFF_A_REF] = diff_dct[DIFF_A_REF]
+    result[DIFF_B_REF] = diff_dct[DIFF_B_REF]
+    if diff_dct[DIFF_EQUAL]:
+        result[DIFF_EQUAL] = True
+        return result
+    result[DIFF_LIST] = []
+    diff_outs = _get_diff_outs(self, diff_dct)
+    if target is None:
+        result[DIFF_LIST] = [
+            _diff_royal(self, path, diff_outs[path]) for path in diff_outs
+        ]
+    elif target in diff_outs:
+        result[DIFF_LIST] = [_diff_royal(self, target, diff_outs[target])]
+    else:
+        msg = "Have not found file/directory '{}' in the commits"
+        raise FileNotInCommitError(msg.format(target))
+    return result

dvc/repo/fetch.py

@@ -0,0 +1,28 @@
+from __future__ import unicode_literals
+
+
+def fetch(
+    self,
+    target=None,
+    jobs=1,
+    remote=None,
+    all_branches=False,
+    show_checksums=False,
+    with_deps=False,
+    all_tags=False,
+    recursive=False,
+):
+    with self.state:
+        used = self.used_cache(
+            target,
+            all_branches=all_branches,
+            all_tags=all_tags,
+            with_deps=with_deps,
+            force=True,
+            remote=remote,
+            jobs=jobs,
+            recursive=recursive,
+        )["local"]
+        return self.cloud.pull(
+            used, jobs, remote=remote, show_checksums=show_checksums
+        )

dvc/repo/gc.py

@@ -0,0 +1,113 @@
+from __future__ import unicode_literals
+
+import collections
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+def _merge_cache_lists(clists):
+    merged_cache = collections.defaultdict(list)
+
+    for cache_list in clists:
+        for scheme, cache in cache_list.items():
+            for item in cache:
+                if item not in merged_cache[scheme]:
+                    merged_cache[scheme].append(item)
+
+    return merged_cache
+
+
+def _load_all_used_cache(
+    repos,
+    target=None,
+    all_branches=False,
+    active=True,
+    with_deps=False,
+    all_tags=False,
+    remote=None,
+    force=False,
+    jobs=None,
+):
+    clists = []
+
+    for repo in repos:
+        with repo.state:
+            repo_clist = repo.used_cache(
+                target=None,
+                all_branches=all_branches,
+                active=False,
+                with_deps=with_deps,
+                all_tags=all_tags,
+                remote=remote,
+                force=force,
+                jobs=jobs,
+            )
+
+            clists.append(repo_clist)
+
+    return clists
+
+
+def _do_gc(typ, func, clist):
+    removed = func(clist)
+    if not removed:
+        logger.info("No unused {} cache to remove.".format(typ))
+
+
+def gc(
+    self,
+    all_branches=False,
+    cloud=False,
+    remote=None,
+    with_deps=False,
+    all_tags=False,
+    force=False,
+    jobs=None,
+    repos=None,
+):
+    from dvc.repo import Repo
+
+    all_repos = [self]
+
+    if repos:
+        all_repos.extend(Repo(path) for path in repos)
+
+    all_clists = _load_all_used_cache(
+        all_repos,
+        target=None,
+        all_branches=all_branches,
+        active=False,
+        with_deps=with_deps,
+        all_tags=all_tags,
+        remote=remote,
+        force=force,
+        jobs=jobs,
+    )
+
+    if len(all_clists) > 1:
+        clist = _merge_cache_lists(all_clists)
+    else:
+        clist = all_clists[0]
+
+    with self.state:
+        _do_gc("local", self.cache.local.gc, clist)
+
+        if self.cache.s3:
+            _do_gc("s3", self.cache.s3.gc, clist)
+
+        if self.cache.gs:
+            _do_gc("gs", self.cache.gs.gc, clist)
+
+        if self.cache.ssh:
+            _do_gc("ssh", self.cache.ssh.gc, clist)
+
+        if self.cache.hdfs:
+            _do_gc("hdfs", self.cache.hdfs.gc, clist)
+
+        if self.cache.azure:
+            _do_gc("azure", self.cache.azure.gc, clist)
+
+        if cloud:
+            _do_gc("remote", self.cloud._get_cloud(remote, "gc -c").gc, clist)

dvc/repo/imp.py

@@ -0,0 +1,23 @@
+from dvc.repo.scm_context import scm_context
+
+
+@scm_context
+def imp(self, url, out, resume=False, fname=None):
+    from dvc.stage import Stage
+
+    with self.state:
+        stage = Stage.create(
+            repo=self, cmd=None, deps=[url], outs=[out], fname=fname
+        )
+
+    if stage is None:
+        return None
+
+    self.check_dag(self.stages() + [stage])
+
+    with self.state:
+        stage.run(resume=resume)
+
+    stage.dump()
+
+    return stage

dvc/repo/init.py

@@ -0,0 +1,96 @@
+import os
+import shutil
+import colorama
+import logging
+
+from dvc.repo import Repo
+from dvc.scm import SCM, NoSCM
+from dvc.config import Config
+from dvc.exceptions import InitError
+from dvc.utils import boxify
+
+
+logger = logging.getLogger(__name__)
+
+
+def _welcome_message():
+    logger.info(
+        boxify(
+            "DVC has enabled anonymous aggregate usage analytics.\n"
+            "Read the analytics documentation (and how to opt-out) here:\n"
+            "{blue}https://dvc.org/doc/user-guide/analytics{nc}".format(
+                blue=colorama.Fore.BLUE, nc=colorama.Fore.RESET
+            ),
+            border_color="red",
+        )
+    )
+
+    msg = (
+        "{yellow}What's next?{nc}\n"
+        "{yellow}------------{nc}\n"
+        "- Check out the documentation: {blue}https://dvc.org/doc{nc}\n"
+        "- Get help and share ideas: {blue}https://dvc.org/chat{nc}\n"
+        "- Star us on GitHub: {blue}https://github.com/iterative/dvc{nc}"
+    ).format(
+        yellow=colorama.Fore.YELLOW,
+        blue=colorama.Fore.BLUE,
+        nc=colorama.Fore.RESET,
+    )
+
+    logger.info(msg)
+
+
+def init(root_dir=os.curdir, no_scm=False, force=False):
+    """
+    Creates an empty repo on the given directory -- basically a
+    `.dvc` directory with subdirectories for configuration and cache.
+
+    It should be tracked by a SCM or use the `--no-scm` flag.
+
+    If the given directory is not empty, you must use the `--force`
+    flag to override it.
+
+    Args:
+        root_dir: Path to repo's root directory.
+
+    Returns:
+        Repo instance.
+
+    Raises:
+        KeyError: Raises an exception.
+    """
+    root_dir = os.path.abspath(root_dir)
+    dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
+    scm = SCM(root_dir)
+    if isinstance(scm, NoSCM) and not no_scm:
+        raise InitError(
+            "{repo} is not tracked by any supported scm tool (e.g. git). "
+            "Use '--no-scm' if you don't want to use any scm.".format(
+                repo=root_dir
+            )
+        )
+
+    if os.path.isdir(dvc_dir):
+        if not force:
+            raise InitError(
+                "'{repo}' exists. Use '-f' to force.".format(
+                    repo=os.path.relpath(dvc_dir)
+                )
+            )
+
+        shutil.rmtree(dvc_dir)
+
+    os.mkdir(dvc_dir)
+
+    config = Config.init(dvc_dir)
+    proj = Repo(root_dir)
+
+    scm.add([config.config_file])
+
+    if scm.ignore_file:
+        scm.add([os.path.join(dvc_dir, scm.ignore_file)])
+        logger.info("\nYou can now commit the changes to git.\n")
+
+    _welcome_message()
+
+    return proj

dvc/repo/install.py

@@ -0,0 +1,2 @@
+def install(self):
+    self.scm.install()

dvc/repo/lock.py

@@ -0,0 +1,8 @@
+def lock(self, target, unlock=False):
+    from dvc.stage import Stage
+
+    stage = Stage.load(self, target)
+    stage.locked = False if unlock else True
+    stage.dump()
+
+    return stage

dvc/repo/metrics/__init__.py

@@ -0,0 +1,23 @@
+class Metrics(object):
+    def __init__(self, repo):
+        self.repo = repo
+
+    def add(self, *args, **kwargs):
+        from dvc.repo.metrics.add import add
+
+        return add(self.repo, *args, **kwargs)
+
+    def modify(self, *args, **kwargs):
+        from dvc.repo.metrics.modify import modify
+
+        return modify(self.repo, *args, **kwargs)
+
+    def show(self, *args, **kwargs):
+        from dvc.repo.metrics.show import show
+
+        return show(self.repo, *args, **kwargs)
+
+    def remove(self, *args, **kwargs):
+        from dvc.repo.metrics.remove import remove
+
+        return remove(self.repo, *args, **kwargs)

dvc/repo/metrics/add.py

@@ -0,0 +1,7 @@
+from dvc.repo.metrics.modify import modify
+
+
+def add(repo, path, typ=None, xpath=None):
+    if not typ:
+        typ = "raw"
+    modify(repo, path, typ, xpath)