dvc: bump to 0.35.4

checkout/remove: properly handle broken symlinks

.appveyor.yml

@@ -31,22 +31,6 @@ environment:
 #    - PYTHON: "C:\\Python27-x64"
 #      PYTHON_VERSION: "2.7.x" # currently 2.7.9
 #      PYTHON_ARCH: "64"
-#
-#    - PYTHON: "C:\\Python33"
-#      PYTHON_VERSION: "3.3.x" # currently 3.3.5
-#      PYTHON_ARCH: "32"
-#
-#    - PYTHON: "C:\\Python33-x64"
-#      PYTHON_VERSION: "3.3.x" # currently 3.3.5
-#      PYTHON_ARCH: "64"
-#
-#    - PYTHON: "C:\\Python34"
-#      PYTHON_VERSION: "3.4.x" # currently 3.4.3
-#      PYTHON_ARCH: "32"
-#
-#    - PYTHON: "C:\\Python34-x64"
-#      PYTHON_VERSION: "3.4.x" # currently 3.4.3
-#      PYTHON_ARCH: "64"
 
 install:
   - ps: Install-Product node
@@ -75,6 +59,7 @@ test_script:
   - python -mtests
 
 after_test:
+  - if "%APPVEYOR_PULL_REQUEST_NUMBER%"=="" pip install codecov
   - if "%APPVEYOR_PULL_REQUEST_NUMBER%"=="" codecov
   - if "%APPVEYOR_REPO_TAG%"=="true" python -m dvc pull
   - if "%APPVEYOR_REPO_TAG%"=="true" .\scripts\build_windows.cmd

.dvc/config

@@ -1,4 +1,4 @@
 ['remote "dvc"']
-url = s3://dvc-share/dvc/
+url = https://dvc.org/s3/dvc
 [core]
 remote = dvc

.flake8

@@ -1,4 +1,7 @@
 [flake8]
-ignore = E203, E266, E501, W503, F403, F401
+ignore =
+    E203, # Whitespace before ':'
+    E266, # Too many leading '#' for block comment
+    W503, # Line break occurred before a binary operator
 max-line-length = 79
 select = B,C,E,F,W,T4,B9

.gitignore

@@ -24,6 +24,7 @@ innosetup/config.ini
 
 .coverage
 
-dvc/version.py
-
 *.swp
+
+pip-wheel-metadata/
+.vscode/

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: stable
+    rev: 19.3b0
     hooks:
     - id: black
       language_version: python3

.travis.yml

@@ -3,6 +3,7 @@ cache:
   - pip
   - directories:
     - /opt/python
+    - $HOME/.pyenv
 sudo: true
 dist: xenial
 python: "3.7"
@@ -21,9 +22,6 @@ matrix:
   - name: "2.7"
     env:
     - PYTHON_VER=2.7.13
-  - name: "3.4"
-    env:
-    - PYTHON_VER=3.4.6
   - name: "3.5"
     env:
     - PYTHON_VER=3.5.3
@@ -50,7 +48,7 @@ matrix:
     language: ruby
     rvm: 2.4.0
     env:
-    - PYTHON_VER=3.4.3
+    - PYTHON_VER=3.6.2
     - PYTHON_CONFIGURE_OPTS="--enable-framework"
   - if: type = cron OR tag is present
     os: osx

CONTRIBUTING.md

@@ -0,0 +1 @@
+### See our contribution guide at [dvc.org](https://dvc.org/doc/user-guide/contributing#contributing).

README.rst

@@ -6,7 +6,7 @@
 • `Docs <https://dvc.org/doc>`_
 • `Twitter <https://twitter.com/iterativeai>`_
 • `Chat (Community & Support) <https://dvc.org/chat>`_
-• `Tutorial <https://dvc.org/doc/tutorial>`_
+• `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
 .. image:: https://travis-ci.com/iterative/dvc.svg?branch=master
@@ -28,17 +28,18 @@
 |
 
 **Data Science Version Control** or **DVC** is an **open-source** tool for data science and
-machine learning projects. With a simple and flexible Git-like architecture and interface it
-helps data scientists:
+machine learning projects. Key features:
 
-#. manage **machine learning models** - versioning, including data sets and transformations (scripts) that were
-   used to generate models;
+#. simple **command line** Git-like experience. Does not require installing and maintaining
+   any databases. Does not depend on any proprietary online services;
 
-#. make projects **reproducible**;
+#. it manages and versions **datasets** and **machine learning models**. Data is saved in
+   S3, Google cloud, Azure, SSH server, HDFS or even local HDD RAID;
 
-#. make projects **shareable**;
+#. it makes projects **reproducible** and **shareable**, it helps answering question how
+   the model was build;
 
-#. manage experiments with branching and **metrics** tracking;
+#. it helps manage experiments with Git tags or branches and **metrics** tracking;
 
 It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
 a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
@@ -50,14 +51,30 @@ data file suffixes and prefixes.
 How DVC works
 =============
 
-DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
-To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
-network storage (via SSH, for example).
+We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ to better understand what DVC
+is and how does it fit your scenarios.
+
+The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-lfs to be precise) + ``makefiles``
+made right and tailored specifically for ML and Data Science scenarios.
+
+#. ``Git/Git-lfs`` part - DVC helps you storing and sharing data artifacts, models. It connects them with your
+   Git repository.
+#. ``Makefiles`` part - DVC describes how one data or model artifact was build from another data. 
+
+DVC usually runs along with Git. Git is used as usual to store and version code and DVC meta-files. DVC helps
+to store data and model files seamlessly out of Git while preserving almost the same user experience as if they
+were stored in Git itself. To store and share data files cache DVC supports remotes - any cloud (S3, Azure,
+Google Cloud, etc) or any on-premise network storage (via SSH, for example). It does not require 
 
 .. image:: https://dvc.org/static/img/flow.gif
    :target: https://dvc.org/static/img/flow.gif
    :alt: how_dvc_works
 
+DVC pipelines (aka computational graph) feature connects code and data together. In a very explicit way you can
+specify, run, and save information that a certain command with certain dependencies needs to be run to produce
+a model. See the quick start section below or check `Get Started <https://dvc.org/doc/get-started>`_ tutorial to
+learn more.
+
 Quick start
 ===========
 
@@ -66,7 +83,7 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 +-----------------------------------+-------------------------------------------------------------------+
 | Step                              | Command                                                           |
 +===================================+===================================================================+
-| Track code and data together      | | ``$ git add train.py``                                          |
+| Track data                        | | ``$ git add train.py``                                          |
 |                                   | | ``$ dvc add images.zip``                                        |
 +-----------------------------------+-------------------------------------------------------------------+
 | Connect code and data by commands | | ``$ dvc run -d images.zip -o images/ unzip -q images.zip``      |
@@ -79,27 +96,30 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 |                                   | | ``$ git commit -m 'The baseline model'``                        |
 |                                   | | ``$ git push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
-| Share data and ML models          | | ``$ dvc remote add myremote s3://mybucket/image_cnn``           |
-|                                   | | ``$ dvc config core.remote myremote``                           |
+| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``        |
 |                                   | | ``$ dvc push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
 
 Installation
 ============
 
-There are three options to install DVC: ``pip``, Homebrew, or an OS-specific package:
+Read this `instruction <https://dvc.org/doc/get-started/install>`_ to get more details. There are three
+options to install DVC: ``pip``, Homebrew, or an OS-specific package:
 
 pip (PyPI)
 ----------
 
-Stable
-^^^^^^
 .. code-block:: bash
 
    pip install dvc
 
-Development
-^^^^^^^^^^^
+Depending on the remote storage type you plan to use to keep and share your data, you might need to specify
+one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``ssh``. Or ``all_remotes`` to include them all.
+The command should look like this: ``pip install dvc[s3]`` - it installs the ``boto3`` library along with
+DVC to support the AWS S3 storage.
+
+To install the development version, run:
+
 .. code-block:: bash
 
    pip install git+git://github.com/iterative/dvc

dvc/__init__.py

@@ -6,73 +6,12 @@ Make your data science projects reproducible and shareable.
 
 from __future__ import unicode_literals
 
-import os
+from dvc.version import __version__  # noqa: F401
 import warnings
+import dvc.logger
 
 
-VERSION_BASE = "0.28.1"
-__version__ = VERSION_BASE
-
-PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
-HOMEPATH = os.path.dirname(PACKAGEPATH)
-VERSIONPATH = os.path.join(PACKAGEPATH, "version.py")
-
-
-def _update_version_file():
-    """Dynamically update version file."""
-    try:
-        from git import Repo
-        from git.exc import InvalidGitRepositoryError
-    except ImportError:
-        return __version__
-
-    try:
-        repo = Repo(HOMEPATH)
-    except InvalidGitRepositoryError:
-        return __version__
-
-    sha = repo.head.object.hexsha
-    short_sha = repo.git.rev_parse(sha, short=6)
-    dirty = ".mod" if repo.is_dirty() else ""
-    ver = "{}+{}{}".format(__version__, short_sha, dirty)
-
-    # Write a helper file, that will be installed with the package
-    # and will provide a true version of the installed dvc
-    with open(VERSIONPATH, "w+") as fobj:
-        fobj.write("# AUTOGENERATED by dvc/__init__.py\n")
-        fobj.write('version = "{}"\n'.format(ver))
-
-    return ver
-
-
-def _remove_version_file():
-    """Remove version.py so that it doesn't get into the release."""
-    if os.path.exists(VERSIONPATH):
-        os.unlink(VERSIONPATH)
-
-
-if os.path.exists(os.path.join(HOMEPATH, "setup.py")):
-    # dvc is run directly from source without installation or
-    # __version__ is called from setup.py
-    if (
-        os.getenv("APPVEYOR_REPO_TAG", "").lower() != "true"
-        and os.getenv("TRAVIS_TAG", "") == ""
-    ):
-        __version__ = _update_version_file()
-    else:  # pragma: no cover
-        _remove_version_file()
-else:  # pragma: no cover
-    # dvc was installed with pip or something. Hopefully we have our
-    # auto-generated version.py to help us provide a true version
-    try:
-        from dvc.version import version
-
-        __version__ = version
-    except Exception:
-        pass
-
-VERSION = __version__
-
+dvc.logger.setup()
 
 # Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
 # We don't directly import numpy, but our dependency networkx does, causing
@@ -80,3 +19,18 @@ VERSION = __version__
 # we can simply ignore them so that they don't show up when you are using dvc.
 warnings.filterwarnings("ignore", message="numpy.dtype size changed")
 warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
+
+# Ignore paramiko's warning: https://github.com/paramiko/paramiko/issues/1386.
+# This only affects paramiko 2.4.2 and should be fixed in the next version.
+# Cryptography developers decided that it is a brilliant idea not to inherit
+# from DeprecationWarning [1] because it is invisible by default, and decided
+# to spam everyone instead. So it is their fault and not paramiko's.
+#
+# [1] https://github.com/pyca/cryptography/blob/2.6.1/src/cryptography/
+#     utils.py#L14
+try:
+    from cryptography.utils import CryptographyDeprecationWarning
+
+    warnings.simplefilter("ignore", CryptographyDeprecationWarning)
+except ImportError:
+    pass

dvc/analytics.py

@@ -7,9 +7,12 @@ from dvc.utils.compat import str
 import os
 import json
 import errno
+import logging
 
-import dvc.logger as logger
-from dvc import VERSION
+from dvc import __version__
+
+
+logger = logging.getLogger(__name__)
 
 
 class Analytics(object):
@@ -164,7 +167,7 @@ class Analytics(object):
         from dvc.repo import Repo
         from dvc.exceptions import NotDvcRepoError
 
-        self.info[self.PARAM_DVC_VERSION] = VERSION
+        self.info[self.PARAM_DVC_VERSION] = __version__
         self.info[self.PARAM_IS_BINARY] = is_binary()
         self.info[self.PARAM_USER_ID] = self._get_user_id()
 
@@ -201,6 +204,13 @@ class Analytics(object):
             json.dump(self.info, fobj)
             return fobj.name
 
+    @staticmethod
+    def _is_enabled_config(config):
+        from dvc.config import Config
+
+        core = config.config.get(Config.SECTION_CORE, {})
+        return core.get(Config.SECTION_CORE_ANALYTICS, True)
+
     @staticmethod
     def _is_enabled(cmd=None):
         from dvc.config import Config
@@ -226,10 +236,9 @@ class Analytics(object):
             config = cmd.config
             assert config is not None
 
-        core = config.config.get(Config.SECTION_CORE, {})
-        enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
+        enabled = Analytics._is_enabled_config(config)
         logger.debug(
-            "Analytics is {}abled.".format("en" if enabled else "dis")
+            "Analytics is {}.".format("enabled" if enabled else "disabled")
         )
         return enabled
 

dvc/cli.py

@@ -5,10 +5,11 @@ from __future__ import unicode_literals
 
 import sys
 import argparse
+import logging
 
-import dvc.logger as logger
 from dvc.command.base import fix_subparsers
 import dvc.command.init as init
+import dvc.command.pkg as pkg
 import dvc.command.destroy as destroy
 import dvc.command.remove as remove
 import dvc.command.move as move
@@ -30,12 +31,16 @@ import dvc.command.lock as lock
 import dvc.command.pipeline as pipeline
 import dvc.command.daemon as daemon
 import dvc.command.commit as commit
+import dvc.command.tag as tag
+import dvc.command.diff as diff
 from dvc.exceptions import DvcParserError
-from dvc import VERSION
 
 
+logger = logging.getLogger(__name__)
+
 COMMANDS = [
     init,
+    pkg,
     destroy,
     add,
     remove,
@@ -58,6 +63,8 @@ COMMANDS = [
     pipeline,
     daemon,
     commit,
+    tag,
+    diff,
 ]
 
 
@@ -82,7 +89,9 @@ class VersionAction(argparse.Action):  # pragma: no cover
     """Shows dvc version and exits."""
 
     def __call__(self, parser, namespace, values, option_string=None):
-        print(VERSION)
+        from dvc import __version__
+
+        print(__version__)
         sys.exit(0)
 
 

dvc/command/add.py

@@ -1,29 +1,44 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdAdd(CmdBase):
     def run(self):
-        for target in self.args.targets:
-            try:
+        try:
+            if len(self.args.targets) > 1 and self.args.file:
+                raise DvcException("can't use '--file' with multiple targets")
+
+            for target in self.args.targets:
                 self.repo.add(
                     target,
                     recursive=self.args.recursive,
                     no_commit=self.args.no_commit,
+                    fname=self.args.file,
                 )
-            except DvcException:
-                logger.error("failed to add file")
-                return 1
+
+        except DvcException:
+            logger.exception("failed to add file")
+            return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    ADD_HELP = "Add files/directories to dvc."
+    ADD_HELP = "Take data files or directories under DVC control."
+
     add_parser = subparsers.add_parser(
-        "add", parents=[parent_parser], description=ADD_HELP, help=ADD_HELP
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(ADD_HELP, "add"),
+        help=ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     add_parser.add_argument(
         "-R",
@@ -38,6 +53,16 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Don't put files/directories into cache.",
     )
+    add_parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the stage file. It should be "
+        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+        "'prepare.dvc', 'clean.dvc', etc) in order for "
+        "dvc to be able to find it later. By default "
+        "the output basename + .dvc is used as a stage filename. "
+        "(NOTE: It can't be used when specifying multiple targets)",
+    )
     add_parser.add_argument(
         "targets", nargs="+", help="Input files/directories."
     )

dvc/command/base.py

@@ -1,6 +1,10 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import colorama
+import logging
+
+
+logger = logging.getLogger(__name__)
 
 
 def fix_subparsers(subparsers):
@@ -18,6 +22,19 @@ def fix_subparsers(subparsers):
         subparsers.dest = "cmd"
 
 
+def append_doc_link(help_message, path):
+    if not path:
+        return help_message
+    doc_base = "https://man.dvc.org/"
+    return "{message}\ndocumentation: {blue}{base}{path}{nc}".format(
+        message=help_message,
+        base=doc_base,
+        path=path,
+        blue=colorama.Fore.CYAN,
+        nc=colorama.Fore.RESET,
+    )
+
+
 class CmdBase(object):
     def __init__(self, args):
         from dvc.repo import Repo
@@ -25,7 +42,6 @@ class CmdBase(object):
         self.repo = Repo()
         self.config = self.repo.config
         self.args = args
-        self.set_loglevel(args)
 
     @property
     def default_targets(self):
@@ -36,15 +52,6 @@ class CmdBase(object):
         logger.warning(msg)
         return [Stage.STAGE_FILE]
 
-    @staticmethod
-    def set_loglevel(args):
-        """Sets log level from CLI arguments."""
-        if args.quiet:
-            logger.be_quiet()
-
-        elif args.verbose:
-            logger.be_verbose()
-
     def run_cmd(self):
         from dvc.lock import LockError
 
@@ -52,7 +59,7 @@ class CmdBase(object):
             with self.repo.lock:
                 return self.run()
         except LockError:
-            logger.error("failed to lock before running a command")
+            logger.exception("failed to lock before running a command")
             return 1
 
     # Abstract methods that have to be implemented by any inheritance class

dvc/command/cache.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import argparse
 
-from dvc.command.base import fix_subparsers
+from dvc.command.base import fix_subparsers, append_doc_link
 from dvc.command.remote import CmdRemoteAdd
 from dvc.command.config import CmdConfig
 
@@ -21,11 +21,13 @@ def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
     CACHE_HELP = "Manage cache settings."
+
     cache_parser = subparsers.add_parser(
         "cache",
         parents=[parent_parser],
-        description=CACHE_HELP,
+        description=append_doc_link(CACHE_HELP, "cache"),
         help=CACHE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     cache_subparsers = cache_parser.add_subparsers(
@@ -38,11 +40,13 @@ def add_parser(subparsers, parent_parser):
         add_help=False, parents=[parent_config_parser]
     )
     CACHE_DIR_HELP = "Configure cache directory location."
+
     cache_dir_parser = cache_subparsers.add_parser(
         "dir",
         parents=[parent_cache_config_parser],
-        description=CACHE_DIR_HELP,
+        description=append_doc_link(CACHE_HELP, "cache-dir"),
         help=CACHE_DIR_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     cache_dir_parser.add_argument(
         "-u",
@@ -53,7 +57,6 @@ def add_parser(subparsers, parent_parser):
     )
     cache_dir_parser.add_argument(
         "value",
-        nargs="?",
         default=None,
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "

dvc/command/checkout.py

@@ -1,6 +1,8 @@
 from __future__ import unicode_literals
 
-from dvc.command.base import CmdBase
+import argparse
+
+from dvc.command.base import CmdBase, append_doc_link
 
 
 class CmdCheckout(CmdBase):
@@ -13,17 +15,20 @@ class CmdCheckout(CmdBase):
                     target=target,
                     with_deps=self.args.with_deps,
                     force=self.args.force,
+                    recursive=self.args.recursive,
                 )
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     CHECKOUT_HELP = "Checkout data files from cache."
+
     checkout_parser = subparsers.add_parser(
         "checkout",
         parents=[parent_parser],
-        description=CHECKOUT_HELP,
+        description=append_doc_link(CHECKOUT_HELP, "checkout"),
         help=CHECKOUT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     checkout_parser.add_argument(
         "-d",
@@ -39,5 +44,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Do not prompt when removing working directory files.",
     )
+    checkout_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Checkout all subdirectories of the specified directory.",
+    )
     checkout_parser.add_argument("targets", nargs="*", help="DVC files.")
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -1,8 +1,13 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdCommit(CmdBase):
@@ -19,7 +24,7 @@ class CmdCommit(CmdBase):
                     force=self.args.force,
                 )
             except DvcException:
-                logger.error(
+                logger.exception(
                     "failed to commit{}".format(
                         (" " + target) if target else ""
                     )
@@ -29,12 +34,14 @@ class CmdCommit(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    COMMIT_HELP = "Record changes to the repository."
+    COMMIT_HELP = "Save changed data to cache and update DVC files."
+
     commit_parser = subparsers.add_parser(
         "commit",
         parents=[parent_parser],
-        description=COMMIT_HELP,
+        description=append_doc_link(COMMIT_HELP, "commit"),
         help=COMMIT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     commit_parser.add_argument(
         "-f",

dvc/command/config.py

@@ -2,13 +2,16 @@ from __future__ import unicode_literals
 
 import os
 import argparse
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.config import Config
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdConfig(CmdBase):
     def __init__(self, args):
         from dvc.repo import Repo, NotDvcRepoError
@@ -47,7 +50,7 @@ class CmdConfig(CmdBase):
             self.config.unset(configobj, section, opt)
             self.config.save(configobj)
         except DvcException:
-            logger.error("failed to unset '{}'".format(self.args.name))
+            logger.exception("failed to unset '{}'".format(self.args.name))
             return 1
         return 0
 
@@ -55,7 +58,7 @@ class CmdConfig(CmdBase):
         try:
             self.config.show(self.configobj, section, opt)
         except DvcException:
-            logger.error("failed to show '{}'".format(self.args.name))
+            logger.exception("failed to show '{}'".format(self.args.name))
             return 1
         return 0
 
@@ -64,7 +67,7 @@ class CmdConfig(CmdBase):
             self.config.set(self.configobj, section, opt, value)
             self.config.save(self.configobj)
         except DvcException:
-            logger.error(
+            logger.exception(
                 "failed to set '{}.{}' to '{}'".format(section, opt, value)
             )
             return 1
@@ -99,11 +102,13 @@ parent_config_parser.add_argument(
 
 def add_parser(subparsers, parent_parser):
     CONFIG_HELP = "Get or set config options."
+
     config_parser = subparsers.add_parser(
         "config",
         parents=[parent_config_parser, parent_parser],
-        description=CONFIG_HELP,
+        description=append_doc_link(CONFIG_HELP, "config"),
         help=CONFIG_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     config_parser.add_argument(
         "-u",

dvc/command/daemon.py

@@ -7,7 +7,6 @@ class CmdDaemonBase(CmdBase):
     def __init__(self, args):
         self.args = args
         self.config = None
-        self.set_loglevel(args)
 
     def run_cmd(self):
         return self.run()
@@ -43,7 +42,7 @@ def add_parser(subparsers, parent_parser):
         "daemon",
         parents=[parent_parser],
         description=DAEMON_HELP,
-        help=DAEMON_HELP,
+        add_help=False,
     )
 
     daemon_subparsers = daemon_parser.add_subparsers(

dvc/command/data_sync.py

@@ -1,9 +1,12 @@
 from __future__ import unicode_literals
 
 import argparse
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
@@ -43,7 +46,7 @@ class CmdDataPull(CmdDataBase):
                 recursive=self.args.recursive,
             )
         except Exception:
-            logger.error("failed to pull data from the cloud")
+            logger.exception("failed to pull data from the cloud")
             return 1
         self.check_up_to_date(processed_files_count)
         return 0
@@ -63,7 +66,7 @@ class CmdDataPush(CmdDataBase):
                 recursive=self.args.recursive,
             )
         except Exception:
-            logger.error("failed to push data to the cloud")
+            logger.exception("failed to push data to the cloud")
             return 1
         self.check_up_to_date(processed_files_count)
         return 0
@@ -83,7 +86,7 @@ class CmdDataFetch(CmdDataBase):
                 recursive=self.args.recursive,
             )
         except Exception:
-            logger.error("failed to fetch data from the cloud")
+            logger.exception("failed to fetch data from the cloud")
             return 1
         self.check_up_to_date(processed_files_count)
         return 0
@@ -120,12 +123,14 @@ def add_parser(subparsers, _parent_parser):
     from dvc.command.status import CmdDataStatus
 
     # Pull
-    PULL_HELP = "Pull data files from the cloud."
+    PULL_HELP = "Pull data files from a DVC remote storage."
+
     pull_parser = subparsers.add_parser(
         "pull",
         parents=[shared_parent_parser()],
-        description=PULL_HELP,
+        description=append_doc_link(PULL_HELP, "pull"),
         help=PULL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
         "-r", "--remote", help="Remote repository to pull from."
@@ -168,12 +173,14 @@ def add_parser(subparsers, _parent_parser):
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
-    PUSH_HELP = "Push data files to the cloud."
+    PUSH_HELP = "Push data files to a DVC remote storage."
+
     push_parser = subparsers.add_parser(
         "push",
         parents=[shared_parent_parser()],
-        description=PUSH_HELP,
+        description=append_doc_link(PUSH_HELP, "push"),
         help=PUSH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
         "-r", "--remote", help="Remote repository to push to."
@@ -209,12 +216,14 @@ def add_parser(subparsers, _parent_parser):
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
-    FETCH_HELP = "Fetch data files from the cloud."
+    FETCH_HELP = "Fetch data files from a DVC remote storage."
+
     fetch_parser = subparsers.add_parser(
         "fetch",
         parents=[shared_parent_parser()],
-        description=FETCH_HELP,
+        description=append_doc_link(FETCH_HELP, "fetch"),
         help=FETCH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
         "-r", "--remote", help="Remote repository to fetch from."
@@ -250,13 +259,17 @@ def add_parser(subparsers, _parent_parser):
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
-    STATUS_HELP = "Show the repo status."
+    STATUS_HELP = (
+        "Show changed stages, compare local cache and a remote storage."
+    )
+
     status_parser = subparsers.add_parser(
         "status",
         parents=[shared_parent_parser()],
-        description=STATUS_HELP,
+        description=append_doc_link(STATUS_HELP, "status"),
         help=STATUS_HELP,
         conflict_handler="resolve",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     status_parser.add_argument(
         "-q",

dvc/command/destroy.py

@@ -1,11 +1,16 @@
 from __future__ import unicode_literals
 
+import argparse
+import logging
+
 import dvc.prompt as prompt
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdDestroy(CmdBase):
     def run_cmd(self):
         try:
@@ -24,21 +29,20 @@ class CmdDestroy(CmdBase):
 
             self.repo.destroy()
         except Exception:
-            logger.error("failed to destroy DVC")
+            logger.exception("failed to destroy DVC")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    DESTROY_HELP = (
-        "Destroy dvc. Will remove all repo's information, "
-        "data files and cache."
-    )
+    DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
+
     destroy_parser = subparsers.add_parser(
         "destroy",
         parents=[parent_parser],
-        description=DESTROY_HELP,
+        description=append_doc_link(DESTROY_HELP, "destroy"),
         help=DESTROY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     destroy_parser.add_argument(
         "-f",

dvc/command/diff.py

@@ -0,0 +1,155 @@
+from __future__ import unicode_literals
+
+import humanize
+import inflect
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+from dvc.utils.collections import compact
+import dvc.repo.diff as diff
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdDiff(CmdBase):
+    def _print_size(self, size):
+        if size < 0:
+            change = "decreased by {}"
+        elif size > 0:
+            change = "increased by {}"
+        else:
+            change = "not changed"
+        natur_size = humanize.naturalsize(abs(size))
+        return change.format(natur_size)
+
+    def _get_md5_string(self, sign, file_name, checksum):
+        sample_msg = ""
+        if file_name:
+            sample_msg = "{}{} with md5 {}\n"
+            sample_msg = sample_msg.format(sign, file_name, checksum)
+        return sample_msg
+
+    def _get_dir_changes(self, dct):
+        engine = inflect.engine()
+        changes_msg = (
+            "{} {} not changed, {} {} modified, {} {} added, "
+            "{} {} deleted, size was {}"
+        )
+        changes_msg = changes_msg.format(
+            dct[diff.DIFF_IDENT],
+            engine.plural("file", dct[diff.DIFF_IDENT]),
+            dct[diff.DIFF_CHANGE],
+            engine.plural("file", dct[diff.DIFF_CHANGE]),
+            dct[diff.DIFF_NEW],
+            engine.plural("file", dct[diff.DIFF_NEW]),
+            dct[diff.DIFF_DEL],
+            engine.plural("file", dct[diff.DIFF_DEL]),
+            self._print_size(dct[diff.DIFF_SIZE]),
+        )
+        return changes_msg
+
+    def _get_file_changes(self, dct):
+        if (
+            dct.get(diff.DIFF_OLD_FILE)
+            and dct.get(diff.DIFF_NEW_FILE)
+            and dct[diff.DIFF_SIZE] == 0
+        ):
+            msg = "file size was not changed"
+        elif dct.get(diff.DIFF_NEW_FILE):
+            msg = "added file with size {}".format(
+                humanize.naturalsize(dct[diff.DIFF_SIZE])
+            )
+        elif dct.get(diff.DIFF_OLD_FILE):
+            msg = "deleted file with size {}".format(
+                humanize.naturalsize(abs(dct[diff.DIFF_SIZE]))
+            )
+        else:
+            msg = "file was modified, file size {}".format(
+                self._print_size(dct[diff.DIFF_SIZE])
+            )
+        return msg
+
+    def _get_royal_changes(self, dct):
+        if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
+            if dct.get("is_dir"):
+                return self._get_dir_changes(dct)
+            else:
+                return self._get_file_changes(dct)
+        return "size is ?"
+
+    def _show(self, diff_dct):
+        msg = "dvc diff from {} to {}".format(
+            diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
+        )
+        if diff_dct.get(diff.DIFF_EQUAL):
+            logger.info(msg)
+            return
+        for dct in diff_dct[diff.DIFF_LIST]:
+            msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET])
+            msg += self._get_md5_string(
+                "-",
+                dct.get(diff.DIFF_OLD_FILE),
+                dct.get(diff.DIFF_OLD_CHECKSUM),
+            )
+            msg += self._get_md5_string(
+                "+",
+                dct.get(diff.DIFF_NEW_FILE),
+                dct.get(diff.DIFF_NEW_CHECKSUM),
+            )
+            msg += "\n"
+            msg += self._get_royal_changes(dct)
+        logger.info(msg)
+        return msg
+
+    def run(self):
+        try:
+            msg = self.repo.diff(
+                self.args.a_ref, target=self.args.target, b_ref=self.args.b_ref
+            )
+            self._show(msg)
+        except DvcException:
+            msg = "failed to get 'diff {}'"
+            args = " ".join(
+                compact([self.args.target, self.args.a_ref, self.args.b_ref])
+            )
+            msg = msg.format(args)
+            logger.exception(msg)
+            return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    description = (
+        "Show diff of a data file or a directory that "
+        "is under DVC control. Some basic statistics "
+        "summary, how many files were deleted/changed."
+    )
+    help = "Show a diff of a DVC controlled data file or a directory."
+    diff_parser = subparsers.add_parser(
+        "diff", parents=[parent_parser], description=description, help=help
+    )
+    diff_parser.add_argument(
+        "-t",
+        "--target",
+        default=None,
+        help=(
+            "Source path to a data file or directory. Default None,"
+            "If not specified, compares all files and directories "
+            "that are under DVC control in the current working space."
+        ),
+    )
+    diff_parser.add_argument(
+        "a_ref", help="Git reference from which diff calculates"
+    )
+    diff_parser.add_argument(
+        "b_ref",
+        help=(
+            "Git reference till which diff calculates, if omitted "
+            "diff shows the difference between current HEAD and a_ref"
+        ),
+        nargs="?",
+        default=None,
+    )
+    diff_parser.set_defaults(func=CmdDiff)

dvc/command/gc.py

@@ -1,10 +1,14 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
 import dvc.prompt as prompt
-import dvc.logger as logger
+import logging
 
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdGC(CmdBase):
@@ -46,23 +50,31 @@ class CmdGC(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    GC_HELP = "Collect garbage."
+    GC_HELP = "Collect unused data from DVC cache or a remote storage."
+    GC_DESCRIPTION = (
+        "Deletes all files in the cache or a remote which are not in\n"
+        "use by the specified git references (defaults to just HEAD)."
+    )
     gc_parser = subparsers.add_parser(
-        "gc", parents=[parent_parser], description=GC_HELP, help=GC_HELP
+        "gc",
+        parents=[parent_parser],
+        description=append_doc_link(GC_DESCRIPTION, "gc"),
+        help=GC_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     gc_parser.add_argument(
         "-a",
         "--all-branches",
         action="store_true",
         default=False,
-        help="Collect garbage for all branches.",
+        help="Keep data files for the tips of all git branches.",
     )
     gc_parser.add_argument(
         "-T",
         "--all-tags",
         action="store_true",
         default=False,
-        help="Collect garbage for all tags.",
+        help="Keep data files for all git tags.",
     )
     gc_parser.add_argument(
         "-c",
@@ -72,14 +84,14 @@ def add_parser(subparsers, parent_parser):
         help="Collect garbage in remote repository.",
     )
     gc_parser.add_argument(
-        "-r", "--remote", help="Remote repository to collect garbage in."
+        "-r", "--remote", help="Remote storage to collect garbage in."
     )
     gc_parser.add_argument(
         "-f",
         "--force",
         action="store_true",
         default=False,
-        help="Force garbage collection.",
+        help="Force garbage collection - automatically agree to all prompts.",
     )
     gc_parser.add_argument(
         "-j",
@@ -95,6 +107,8 @@ def add_parser(subparsers, parent_parser):
         type=str,
         nargs="*",
         default=None,
-        help="Collect garbage for all given projects.",
+        help="Keep data files required by these projects "
+        "in addition to the current one. "
+        "Useful if you share a single cache across repos.",
     )
     gc_parser.set_defaults(func=CmdGC)

dvc/command/imp.py

@@ -1,11 +1,15 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
+import logging
 
-import dvc.logger as logger
 from dvc.utils.compat import urlparse
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdImport(CmdBase):
@@ -17,7 +21,7 @@ class CmdImport(CmdBase):
 
             self.repo.imp(self.args.url, out, self.args.resume)
         except DvcException:
-            logger.error(
+            logger.exception(
                 "failed to import {}. You could also try downloading "
                 "it manually and adding it with `dvc add` command.".format(
                     self.args.url
@@ -28,25 +32,26 @@ class CmdImport(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    IMPORT_HELP = "Import files from URL."
+    IMPORT_HELP = "Download or copy files from URL and take under DVC control."
+
     import_parser = subparsers.add_parser(
         "import",
         parents=[parent_parser],
-        description=IMPORT_HELP,
+        description=append_doc_link(IMPORT_HELP, "import"),
         help=IMPORT_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     import_parser.add_argument(
         "url",
-        help="URL. Supported urls: "
-        "'/path/to/file', "
-        "'C:\\\\path\\to\\file', "
-        "'https://example.com/path/to/file', "
-        "'s3://bucket/path/to/file', "
-        "'gs://bucket/path/to/file', "
-        "'hdfs://example.com/path/to/file', "
-        "'ssh://example.com:/path/to/file', "
-        "'remote://myremote/path/to/file'(see "
-        "`dvc remote` commands). ",
+        help="Supported urls:\n"
+        "/path/to/file\n"
+        "C:\\\\path\\to\\file\n"
+        "https://example.com/path/to/file\n"
+        "s3://bucket/path/to/file\n"
+        "gs://bucket/path/to/file\n"
+        "hdfs://example.com/path/to/file\n"
+        "ssh://example.com:/path/to/file\n"
+        "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(
         "--resume",
@@ -54,5 +59,7 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Resume previously started download.",
     )
-    import_parser.add_argument("out", nargs="?", help="Output.")
+    import_parser.add_argument(
+        "out", nargs="?", help="Destination path to put files to."
+    )
     import_parser.set_defaults(func=CmdImport)

dvc/command/init.py

@@ -1,6 +1,12 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
+from dvc.command.base import append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdInit(object):
@@ -17,18 +23,25 @@ class CmdInit(object):
             )
             self.config = self.repo.config
         except InitError:
-            logger.error("failed to initiate dvc")
+            logger.exception("failed to initiate dvc")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     """Setup parser for `dvc init`."""
-    INIT_HELP = (
-        "Initialize dvc over a directory " "(should already be a git dir)."
+    INIT_HELP = "Initialize DVC in the current directory."
+    INIT_DESCRIPTION = (
+        "Initialize DVC in the current directory. Expects directory\n"
+        "to be a Git repository unless --no-scm option is specified."
     )
+
     init_parser = subparsers.add_parser(
-        "init", parents=[parent_parser], description=INIT_HELP, help=INIT_HELP
+        "init",
+        parents=[parent_parser],
+        description=append_doc_link(INIT_DESCRIPTION, "init"),
+        help=INIT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     init_parser.add_argument(
         "--no-scm",
@@ -42,6 +55,9 @@ def add_parser(subparsers, parent_parser):
         "--force",
         action="store_true",
         default=False,
-        help="Overwrite '.dvc' if it exists. Will remove all local cache.",
+        help=(
+            "Overwrite existing '.dvc' directory. "
+            "This operation removes local cache."
+        ),
     )
     init_parser.set_defaults(func=CmdInit)

dvc/command/install.py

@@ -1,7 +1,12 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdInstall(CmdBase):
@@ -9,17 +14,18 @@ class CmdInstall(CmdBase):
         try:
             self.repo.install()
         except Exception:
-            logger.error("failed to install dvc hooks")
+            logger.exception("failed to install dvc hooks")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    INSTALL_HELP = "Install dvc hooks into the repository."
+    INSTALL_HELP = "Install DVC git hooks into the repository."
     install_parser = subparsers.add_parser(
         "install",
         parents=[parent_parser],
-        description=INSTALL_HELP,
+        description=append_doc_link(INSTALL_HELP, "install"),
         help=INSTALL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     install_parser.set_defaults(func=CmdInstall)

dvc/command/lock.py

@@ -1,17 +1,22 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdLockBase(CmdBase):
-    def run(self, unlock=False):
+    def _run(self, unlock):
         for target in self.args.targets:
             try:
                 self.repo.lock_stage(target, unlock=unlock)
             except DvcException:
-                logger.error(
+                logger.exception(
                     "failed to {}lock '{}'".format(
                         "un" if unlock else "", target
                     )
@@ -23,18 +28,22 @@ class CmdLockBase(CmdBase):
 
 class CmdLock(CmdLockBase):
     def run(self):
-        return super(CmdLock, self).run(False)
+        return self._run(False)
 
 
 class CmdUnlock(CmdLockBase):
     def run(self):
-        return super(CmdUnlock, self).run(True)
+        return self._run(True)
 
 
 def add_parser(subparsers, parent_parser):
     LOCK_HELP = "Lock DVC file."
     lock_parser = subparsers.add_parser(
-        "lock", parents=[parent_parser], description=LOCK_HELP, help=LOCK_HELP
+        "lock",
+        parents=[parent_parser],
+        description=append_doc_link(LOCK_HELP, "lock"),
+        help=LOCK_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     lock_parser.add_argument("targets", nargs="+", help="DVC files.")
     lock_parser.set_defaults(func=CmdLock)
@@ -43,8 +52,9 @@ def add_parser(subparsers, parent_parser):
     unlock_parser = subparsers.add_parser(
         "unlock",
         parents=[parent_parser],
-        description=UNLOCK_HELP,
+        description=append_doc_link(UNLOCK_HELP, "unlock"),
         help=UNLOCK_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unlock_parser.add_argument("targets", nargs="+", help="DVC files.")
     unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/metrics.py

@@ -1,44 +1,42 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+def show_metrics(metrics, all_branches=False, all_tags=False):
+    """
+    Args:
+        metrics (list): Where each element is either a `list`
+            if an xpath was specified, otherwise a `str`
+    """
+    for branch, val in metrics.items():
+        if all_branches or all_tags:
+            logger.info("{branch}:".format(branch=branch))
+
+        for fname, metric in val.items():
+            lines = metric if type(metric) is list else metric.splitlines()
+
+            if len(lines) > 1:
+                logger.info("\t{fname}:".format(fname=fname))
+
+                for line in lines:
+                    logger.info("\t\t{content}".format(content=line))
+
+            else:
+                logger.info("\t{}: {}".format(fname, metric))
 
 
 class CmdMetricsShow(CmdBase):
-    def _show(self, metrics):
-        for branch, val in metrics.items():
-            if self.args.all_branches or self.args.all_tags:
-                logger.info("{}:".format(branch))
-
-            for fname, metric in val.items():
-                logger.info("\t{}: {}".format(fname, metric))
-
-    def _get_type_xpath(self):
-        # backward compatibility
-        if self.args.json_path:
-            typ = "json"
-            xpath = self.args.json_path
-        elif self.args.tsv_path:
-            typ = "tsv"
-            xpath = self.args.tsv_path
-        elif self.args.htsv_path:
-            typ = "htsv"
-            xpath = self.args.htsv_path
-        elif self.args.csv_path:
-            typ = "csv"
-            xpath = self.args.csv_path
-        elif self.args.hcsv_path:
-            typ = "hcsv"
-            xpath = self.args.hcsv_path
-        else:
-            typ = self.args.type
-            xpath = self.args.xpath
-
-        return typ, xpath
-
     def run(self):
-        typ, xpath = self._get_type_xpath()
+        typ = self.args.type
+        xpath = self.args.xpath
         try:
             metrics = self.repo.metrics.show(
                 self.args.path,
@@ -49,9 +47,9 @@ class CmdMetricsShow(CmdBase):
                 recursive=self.args.recursive,
             )
 
-            self._show(metrics)
+            show_metrics(metrics, self.args.all_branches, self.args.all_tags)
         except DvcException:
-            logger.error("failed to show metrics")
+            logger.exception("failed to show metrics")
             return 1
 
         return 0
@@ -64,7 +62,7 @@ class CmdMetricsModify(CmdBase):
                 self.args.path, typ=self.args.type, xpath=self.args.xpath
             )
         except DvcException:
-            logger.error("failed to modify metrics")
+            logger.exception("failed to modify metric file settings")
             return 1
 
         return 0
@@ -78,7 +76,7 @@ class CmdMetricsAdd(CmdBase):
             )
         except DvcException:
             msg = "failed to add metric file '{}'".format(self.args.path)
-            logger.error(msg)
+            logger.exception(msg)
             return 1
 
         return 0
@@ -90,57 +88,52 @@ class CmdMetricsRemove(CmdBase):
             self.repo.metrics.remove(self.args.path)
         except DvcException:
             msg = "failed to remove metric file '{}'".format(self.args.path)
-            logger.error(msg)
+            logger.exception(msg)
             return 1
 
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = "Get metrics from all branches."
+    METRICS_HELP = "Commands to add, manage, collect and display metrics."
+
     metrics_parser = subparsers.add_parser(
         "metrics",
         parents=[parent_parser],
-        description=METRICS_HELP,
+        description=append_doc_link(METRICS_HELP, "metrics"),
         help=METRICS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     metrics_subparsers = metrics_parser.add_subparsers(
         dest="cmd",
-        help="Use dvc metrics CMD --help for command-specific help.",
+        help="Use dvc metrics CMD --help to display command-specific help.",
     )
 
     fix_subparsers(metrics_subparsers)
 
-    METRICS_SHOW_HELP = "Show metrics."
+    METRICS_SHOW_HELP = "Output metric values."
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
         parents=[parent_parser],
-        description=METRICS_SHOW_HELP,
+        description=append_doc_link(METRICS_SHOW_HELP, "metrics-show"),
         help=METRICS_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_show_parser.add_argument(
-        "path", nargs="?", help="Path to metrics file or directory"
+        "path", nargs="?", help="Path to a metric file or a directory."
     )
     metrics_show_parser.add_argument(
-        "-t", "--type", help="Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)."
+        "-t",
+        "--type",
+        help=(
+            "Type of metrics (json/tsv/htsv/csv/hcsv). "
+            "It can be detected by the file extension automatically. "
+            "Unsupported types will be treated as raw."
+        ),
     )
     metrics_show_parser.add_argument(
-        "-x", "--xpath", help="JSON/TSV/HTSV/CSV/HCSV path."
-    )
-    metrics_show_group = metrics_show_parser.add_mutually_exclusive_group()
-    metrics_show_group.add_argument("--json-path", help="JSON path.")
-    metrics_show_group.add_argument(
-        "--tsv-path", help="TSV path 'row,column' (e.g. '1,2')."
-    )
-    metrics_show_group.add_argument(
-        "--htsv-path", help="Headed TSV path 'row,column (e.g. 'Name,3')."
-    )
-    metrics_show_group.add_argument(
-        "--csv-path", help="CSV path 'row,column' (e.g. '1,2')."
-    )
-    metrics_show_group.add_argument(
-        "--hcsv-path", help="Headed CSV path 'row,column' (e.g. 'Name,3')."
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
     )
     metrics_show_parser.add_argument(
         "-a",
@@ -161,48 +154,54 @@ def add_parser(subparsers, parent_parser):
         "--recursive",
         action="store_true",
         default=False,
-        help="If path is a directory, recursively search and process metric files in path.",
+        help=(
+            "If path is a directory, recursively search and process metric "
+            "files in path."
+        ),
     )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
-    METRICS_ADD_HELP = "Add metrics."
+    METRICS_ADD_HELP = "Tag file as a metric file."
     metrics_add_parser = metrics_subparsers.add_parser(
         "add",
         parents=[parent_parser],
-        description=METRICS_ADD_HELP,
+        description=append_doc_link(METRICS_ADD_HELP, "metrics-add"),
         help=METRICS_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_add_parser.add_argument(
-        "-t", "--type", help="Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)."
+        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
     )
     metrics_add_parser.add_argument(
-        "-x", "--xpath", help="JSON/TSV/HTSV/CSV/HCSV path."
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
     )
-    metrics_add_parser.add_argument("path", help="Path to metrics file.")
+    metrics_add_parser.add_argument("path", help="Path to a metric file.")
     metrics_add_parser.set_defaults(func=CmdMetricsAdd)
 
-    METRICS_MODIFY_HELP = "Modify metrics."
+    METRICS_MODIFY_HELP = "Modify metric file options."
     metrics_modify_parser = metrics_subparsers.add_parser(
         "modify",
         parents=[parent_parser],
-        description=METRICS_MODIFY_HELP,
+        description=append_doc_link(METRICS_MODIFY_HELP, "metrics-modify"),
         help=METRICS_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_modify_parser.add_argument(
-        "-t", "--type", help="Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)."
+        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
     )
     metrics_modify_parser.add_argument(
-        "-x", "--xpath", help="JSON/TSV/HTSV/CSV/HCSV path."
+        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
     )
-    metrics_modify_parser.add_argument("path", help="Metrics file.")
+    metrics_modify_parser.add_argument("path", help="Path to a metric file.")
     metrics_modify_parser.set_defaults(func=CmdMetricsModify)
 
-    METRICS_REMOVE_HELP = "Remove metrics."
+    METRICS_REMOVE_HELP = "Remove files's metric tag."
     metrics_remove_parser = metrics_subparsers.add_parser(
         "remove",
         parents=[parent_parser],
-        description=METRICS_REMOVE_HELP,
+        description=append_doc_link(METRICS_REMOVE_HELP, "metrics-remove"),
         help=METRICS_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    metrics_remove_parser.add_argument("path", help="Path to metrics file.")
+    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
     metrics_remove_parser.set_defaults(func=CmdMetricsRemove)

dvc/command/move.py

@@ -1,8 +1,13 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdMove(CmdBase):
@@ -13,20 +18,25 @@ class CmdMove(CmdBase):
             msg = "failed to move '{}' -> '{}'".format(
                 self.args.src, self.args.dst
             )
-            logger.error(msg)
+            logger.exception(msg)
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    description = (
-        "Rename or move a data file or a directory that "
-        "is under DVC control. It renames and modifies "
-        "the corresponding DVC file to reflect the changes."
+    MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
+    MOVE_DESCRIPTION = (
+        "Rename or move a DVC controlled data file or a directory.\n"
+        "It renames and modifies the corresponding DVC file to reflect the"
+        " changes."
     )
-    help = "Rename or move a DVC controlled data file or a directory."
+
     move_parser = subparsers.add_parser(
-        "move", parents=[parent_parser], description=description, help=help
+        "move",
+        parents=[parent_parser],
+        description=append_doc_link(MOVE_DESCRIPTION, "move"),
+        help=MOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
         "src", help="Source path to a data file or directory."

dvc/command/pipeline.py

@@ -2,11 +2,15 @@ from __future__ import unicode_literals
 
 from dvc.utils.compat import str
 
+import argparse
 import os
+import logging
 
-import dvc.logger as logger
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdPipelineShow(CmdBase):
@@ -71,23 +75,47 @@ class CmdPipelineShow(CmdBase):
             else:
                 edges.append((from_stage.relpath, to_stage.relpath))
 
-        return nodes, edges
+        return nodes, edges, networkx.is_tree(G)
 
     def _show_ascii(self, target, commands, outs):
         from dvc.dagascii import draw
 
-        nodes, edges = self.__build_graph(target, commands, outs)
+        nodes, edges, _ = self.__build_graph(target, commands, outs)
 
         if not nodes:
             return
 
         draw(nodes, edges)
 
+    def _show_dependencies_tree(self, target, commands, outs):
+        from treelib import Tree
+
+        nodes, edges, is_tree = self.__build_graph(target, commands, outs)
+        if not nodes:
+            return
+        if not is_tree:
+            raise DvcException(
+                "DAG is not a tree, can not print it in tree-structure way, "
+                "please use --ascii instead"
+            )
+
+        tree = Tree()
+        tree.create_node(target, target)  # Root node
+        observe_list = [target]
+        while len(observe_list) > 0:
+            current_root = observe_list[0]
+            for edge in edges:
+                if edge[0] == current_root:
+                    tree.create_node(edge[1], edge[1], parent=current_root)
+                    observe_list.append(edge[1])
+            observe_list.pop(0)
+        tree.show()
+
     def __write_dot(self, target, commands, outs, filename):
         import networkx
         from networkx.drawing.nx_pydot import write_dot
 
-        _, edges = self.__build_graph(target, commands, outs)
+        _, edges, _ = self.__build_graph(target, commands, outs)
         edges = [edge[::-1] for edge in edges]
 
         simple_g = networkx.DiGraph()
@@ -111,11 +139,15 @@ class CmdPipelineShow(CmdBase):
                         self.args.outs,
                         self.args.dot,
                     )
+                elif self.args.tree:
+                    self._show_dependencies_tree(
+                        target, self.args.commands, self.args.outs
+                    )
                 else:
                     self._show(target, self.args.commands, self.args.outs)
             except DvcException:
                 msg = "failed to show pipeline for '{}'".format(target)
-                logger.error(msg)
+                logger.exception(msg)
                 return 1
         return 0
 
@@ -137,12 +169,13 @@ class CmdPipelineList(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    PIPELINE_HELP = "Manage pipeline."
+    PIPELINE_HELP = "Manage pipelines."
     pipeline_parser = subparsers.add_parser(
         "pipeline",
         parents=[parent_parser],
-        description=PIPELINE_HELP,
+        description=append_doc_link(PIPELINE_HELP, "pipeline"),
         help=PIPELINE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     pipeline_subparsers = pipeline_parser.add_subparsers(
@@ -156,8 +189,9 @@ def add_parser(subparsers, parent_parser):
     pipeline_show_parser = pipeline_subparsers.add_parser(
         "show",
         parents=[parent_parser],
-        description=PIPELINE_SHOW_HELP,
+        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline-show"),
         help=PIPELINE_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
     pipeline_show_group.add_argument(
@@ -183,6 +217,12 @@ def add_parser(subparsers, parent_parser):
     pipeline_show_parser.add_argument(
         "--dot", help="Write DAG in .dot format."
     )
+    pipeline_show_parser.add_argument(
+        "--tree",
+        action="store_true",
+        default=False,
+        help="Output DAG as Dependencies Tree.",
+    )
     pipeline_show_parser.add_argument(
         "targets", nargs="*", help="DVC files. 'Dvcfile' by default."
     )
@@ -192,7 +232,8 @@ def add_parser(subparsers, parent_parser):
     pipeline_list_parser = pipeline_subparsers.add_parser(
         "list",
         parents=[parent_parser],
-        description=PIPELINE_LIST_HELP,
+        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline-list"),
         help=PIPELINE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pipeline_list_parser.set_defaults(func=CmdPipelineList)

dvc/command/pkg.py

@@ -0,0 +1,91 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.repo.pkg import PackageParams
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdPkg(CmdBase):
+    def run(self, unlock=False):
+        try:
+            pkg_params = PackageParams(
+                self.args.address,
+                self.args.target_dir,
+                self.args.select,
+                self.args.file,
+            )
+            return self.repo.install_pkg(pkg_params)
+        except DvcException:
+            logger.exception(
+                "failed to install package '{}'".format(self.args.address)
+            )
+            return 1
+        pass
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    PKG_HELP = "Manage DVC packages."
+    pkg_parser = subparsers.add_parser(
+        "pkg",
+        parents=[parent_parser],
+        description=append_doc_link(PKG_HELP, "pkg"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        add_help=False,
+    )
+
+    pkg_subparsers = pkg_parser.add_subparsers(
+        dest="cmd", help="Use dvc pkg CMD --help for command-specific help."
+    )
+
+    fix_subparsers(pkg_subparsers)
+
+    PKG_INSTALL_HELP = "Install package."
+    pkg_install_parser = pkg_subparsers.add_parser(
+        "install",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(PKG_INSTALL_HELP, "pkg-install"),
+        help=PKG_INSTALL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    pkg_install_parser.add_argument(
+        "address",
+        nargs="?",
+        default="",
+        help="Package address: git://<url> or https://github.com/...",
+    )
+    pkg_install_parser.add_argument(
+        "target_dir",
+        metavar="target",
+        nargs="?",
+        default=".",
+        help="Target directory to deploy package outputs. "
+        "Default value is the current dir.",
+    )
+    pkg_install_parser.add_argument(
+        "-s",
+        "--select",
+        metavar="OUT",
+        action="append",
+        default=[],
+        help="Select and persist only specified outputs from a package. "
+        "The parameter can be used multiple times. "
+        "All outputs will be selected by default.",
+    )
+    pkg_install_parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the stage file. It should be "
+        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+        "'prepare.dvc', 'clean.dvc', etc). "
+        "By default the file has 'mod_' prefix and imported package name "
+        "followed by .dvc",
+    )
+    pkg_install_parser.set_defaults(func=CmdPkg)

dvc/command/remote.py

@@ -1,12 +1,16 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
 import re
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import fix_subparsers
-from dvc.config import Config
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
+from dvc.config import Config
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdRemoteAdd(CmdConfig):
@@ -38,6 +42,14 @@ class CmdRemoteAdd(CmdConfig):
             )
 
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
+        if (section in self.configobj.keys()) and not self.args.force:
+            logger.error(
+                "Remote with name {} already exists. "
+                "Use -f (--force) to overwrite remote "
+                "with new value".format(self.args.name)
+            )
+            return 1
+
         ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
         if ret != 0:
             return ret
@@ -119,12 +131,13 @@ class CmdRemoteList(CmdConfig):
 def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
-    REMOTE_HELP = "Manage set of tracked repositories."
+    REMOTE_HELP = "Manage remote storage configuration."
     remote_parser = subparsers.add_parser(
         "remote",
         parents=[parent_parser],
-        description=REMOTE_HELP,
+        description=append_doc_link(REMOTE_HELP, "remote"),
         help=REMOTE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     remote_subparsers = remote_parser.add_subparsers(
@@ -138,14 +151,14 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser = remote_subparsers.add_parser(
         "add",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_ADD_HELP,
+        description=append_doc_link(REMOTE_ADD_HELP, "remote-add"),
         help=REMOTE_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_add_parser.add_argument("name", help="Name.")
     remote_add_parser.add_argument(
         "url",
-        help="URL. See full list of supported urls at "
-        "https://dvc.org/doc/commands-reference/remote",
+        help="URL. See full list of supported urls at " "man.dvc.org/remote",
     )
     remote_add_parser.add_argument(
         "-d",
@@ -154,14 +167,22 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Set as default remote.",
     )
+    remote_add_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force overwriting existing configs",
+    )
     remote_add_parser.set_defaults(func=CmdRemoteAdd)
 
     REMOTE_DEFAULT_HELP = "Set/unset default remote."
     remote_default_parser = remote_subparsers.add_parser(
         "default",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_DEFAULT_HELP,
+        description=append_doc_link(REMOTE_DEFAULT_HELP, "remote-default"),
         help=REMOTE_DEFAULT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_default_parser.add_argument(
         "name", nargs="?", help="Name of the remote."
@@ -179,8 +200,9 @@ def add_parser(subparsers, parent_parser):
     remote_remove_parser = remote_subparsers.add_parser(
         "remove",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_REMOVE_HELP,
+        description=append_doc_link(REMOTE_REMOVE_HELP, "remote-remove"),
         help=REMOTE_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_remove_parser.add_argument("name", help="Name")
     remote_remove_parser.set_defaults(func=CmdRemoteRemove)
@@ -189,8 +211,9 @@ def add_parser(subparsers, parent_parser):
     remote_modify_parser = remote_subparsers.add_parser(
         "modify",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_MODIFY_HELP,
+        description=append_doc_link(REMOTE_MODIFY_HELP, "remote-modify"),
         help=REMOTE_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_modify_parser.add_argument("name", help="Name.")
     remote_modify_parser.add_argument("option", help="Option.")
@@ -204,11 +227,12 @@ def add_parser(subparsers, parent_parser):
     )
     remote_modify_parser.set_defaults(func=CmdRemoteModify)
 
-    REMOTE_LIST_HELP = "List remotes."
+    REMOTE_LIST_HELP = "List available remotes."
     remote_list_parser = remote_subparsers.add_parser(
         "list",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_LIST_HELP,
+        description=append_doc_link(REMOTE_LIST_HELP, "remote-list"),
         help=REMOTE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_list_parser.set_defaults(func=CmdRemoteList)

dvc/command/remove.py

@@ -1,9 +1,14 @@
 from __future__ import unicode_literals
 
+import argparse
+import logging
+
 import dvc.prompt as prompt
-import dvc.logger as logger
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdRemove(CmdBase):
@@ -32,7 +37,7 @@ class CmdRemove(CmdBase):
                 outs_only = self._is_outs_only(target)
                 self.repo.remove(target, outs_only=outs_only)
             except DvcException:
-                logger.error("failed to remove {}".format(target))
+                logger.exception("failed to remove {}".format(target))
                 return 1
         return 0
 
@@ -42,8 +47,9 @@ def add_parser(subparsers, parent_parser):
     remove_parser = subparsers.add_parser(
         "remove",
         parents=[parent_parser],
-        description=REMOVE_HELP,
+        description=append_doc_link(REMOVE_HELP, "remove"),
         help=REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remove_parser_group = remove_parser.add_mutually_exclusive_group()
     remove_parser_group.add_argument(

dvc/command/repro.py

@@ -1,13 +1,18 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdRepro(CmdBase):
     def run(self):
         recursive = not self.args.single_item
@@ -40,9 +45,10 @@ class CmdRepro(CmdBase):
                     logger.info(CmdDataStatus.UP_TO_DATE_MSG)
 
                 if self.args.metrics:
-                    self.repo.metrics.show()
+                    metrics = self.repo.metrics.show()
+                    show_metrics(metrics)
             except DvcException:
-                logger.error()
+                logger.exception()
                 ret = 1
                 break
 
@@ -51,15 +57,18 @@ class CmdRepro(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'."
+    REPRO_HELP = "Check for changes and reproduce DVC file and dependencies."
     repro_parser = subparsers.add_parser(
         "repro",
         parents=[parent_parser],
-        description=REPRO_HELP,
+        description=append_doc_link(REPRO_HELP, "repro"),
         help=REPRO_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     repro_parser.add_argument(
-        "targets", nargs="*", help="DVC file to reproduce."
+        "targets",
+        nargs="*",
+        help="DVC file to reproduce (default - 'Dvcfile').",
     )
     repro_parser.add_argument(
         "-f",
@@ -80,7 +89,7 @@ def add_parser(subparsers, parent_parser):
         "-c",
         "--cwd",
         default=os.path.curdir,
-        help="Directory within your repo to reroduce from.",
+        help="Directory within your repo to reproduce from.",
     )
     repro_parser.add_argument(
         "-m",

dvc/command/root.py

@@ -1,9 +1,13 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdRoot(CmdBase):
@@ -16,8 +20,12 @@ class CmdRoot(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    ROOT_HELP = "Relative path to repo's directory."
+    ROOT_HELP = "Relative path to the repository's directory."
     root_parser = subparsers.add_parser(
-        "root", parents=[parent_parser], description=ROOT_HELP, help=ROOT_HELP
+        "root",
+        parents=[parent_parser],
+        description=append_doc_link(ROOT_HELP, "root"),
+        help=ROOT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     root_parser.set_defaults(func=CmdRoot)

dvc/command/run.py

@@ -1,13 +1,15 @@
 from __future__ import unicode_literals
 
-import os
 import argparse
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdRun(CmdBase):
     def run(self):
         overwrite = self.args.yes or self.args.overwrite_dvcfile
@@ -19,12 +21,15 @@ class CmdRun(CmdBase):
                 self.args.outs_no_cache,
                 self.args.metrics,
                 self.args.metrics_no_cache,
+                self.args.outs_persist,
+                self.args.outs_persist_no_cache,
                 self.args.command,
             ]
         ):  # pragma: no cover
             logger.error(
-                "too few arguments. Specify at least one: '-d', "
-                "'-o', '-O', '-m', '-M', 'command'."
+                "too few arguments. Specify at least one: '-d', '-o', '-O',"
+                " '-m', '-M', '--outs-persist', '--outs-persist-no-cache',"
+                " 'command'."
             )
             return 1
 
@@ -38,14 +43,17 @@ class CmdRun(CmdBase):
                 deps=self.args.deps,
                 fname=self.args.file,
                 cwd=self.args.cwd,
+                wdir=self.args.wdir,
                 no_exec=self.args.no_exec,
                 overwrite=overwrite,
                 ignore_build_cache=self.args.ignore_build_cache,
                 remove_outs=self.args.remove_outs,
                 no_commit=self.args.no_commit,
+                outs_persist=self.args.outs_persist,
+                outs_persist_no_cache=self.args.outs_persist_no_cache,
             )
         except DvcException:
-            logger.error("failed to run command")
+            logger.exception("failed to run command")
             return 1
 
         return 0
@@ -73,12 +81,13 @@ class CmdRun(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    RUN_HELP = (
-        "Generate a stage file from a given "
-        "command and execute the command."
-    )
+    RUN_HELP = "Generate a stage file from a command and execute the command."
     run_parser = subparsers.add_parser(
-        "run", parents=[parent_parser], description=RUN_HELP, help=RUN_HELP
+        "run",
+        parents=[parent_parser],
+        description=append_doc_link(RUN_HELP, "run"),
+        help=RUN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     run_parser.add_argument(
         "-d",
@@ -128,11 +137,13 @@ def add_parser(subparsers, parent_parser):
         "a stage filename.",
     )
     run_parser.add_argument(
-        "-c",
-        "--cwd",
-        default=os.path.curdir,
-        help="Directory within your repo to run your command and place "
-        "stage file in.",
+        "-c", "--cwd", default=None, help="Deprecated, use -w and -f instead."
+    )
+    run_parser.add_argument(
+        "-w",
+        "--wdir",
+        default=None,
+        help="Directory within your repo to run your command in.",
     )
     run_parser.add_argument(
         "--no-exec",
@@ -145,9 +156,7 @@ def add_parser(subparsers, parent_parser):
         "--yes",
         action="store_true",
         default=False,
-        help="(OBSOLETED, use --overwrite-dvcfile instead) Automatic 'yes' "
-        "answer to all prompts. E.g. when '.dvc' file exists and dvc "
-        "asks if you want to overwrite it.",
+        help="Deprecated, use --overwrite-dvcfile instead",
     )
     run_parser.add_argument(
         "--overwrite-dvcfile",
@@ -174,6 +183,20 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Don't put files/directories into cache.",
     )
+    run_parser.add_argument(
+        "--outs-persist",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro.",
+    )
+    run_parser.add_argument(
+        "--outs-persist-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro (do not put into DVC cache).",
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/command/status.py

@@ -1,10 +1,14 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import logging
+
 from dvc.command.data_sync import CmdDataBase
 from dvc.utils.compat import str
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 20
     STATUS_INDENT = "\t"
@@ -58,6 +62,6 @@ class CmdDataStatus(CmdDataBase):
                 logger.info(self.UP_TO_DATE_MSG)
 
         except Exception:
-            logger.error("failed to obtain data status")
+            logger.exception("failed to obtain data status")
             return 1
         return 0

dvc/command/tag.py

@@ -0,0 +1,154 @@
+import yaml
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdTagAdd(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.repo.tag.add(
+                    self.args.tag,
+                    target=target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("failed to add tag")
+                return 1
+        return 0
+
+
+class CmdTagRemove(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.repo.tag.remove(
+                    self.args.tag,
+                    target=target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("failed to remove tag")
+                return 1
+        return 0
+
+
+class CmdTagList(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                tags = self.repo.tag.list(
+                    target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+                if tags:
+                    logger.info(yaml.dump(tags, default_flow_style=False))
+            except DvcException:
+                logger.exception("failed list tags")
+                return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    TAG_HELP = "A set of commands to manage DVC tags."
+    tag_parser = subparsers.add_parser(
+        "tag",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_HELP, "tag"),
+        add_help=False,
+    )
+
+    tag_subparsers = tag_parser.add_subparsers(
+        dest="cmd",
+        help="Use DVC tag CMD --help to display command-specific help.",
+    )
+
+    fix_subparsers(tag_subparsers)
+
+    TAG_ADD_HELP = "Add DVC tag."
+    tag_add_parser = tag_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_ADD_HELP, "tag-add"),
+        help=TAG_ADD_HELP,
+    )
+    tag_add_parser.add_argument("tag", help="Dvc tag.")
+    tag_add_parser.add_argument(
+        "targets", nargs="*", default=[None], help="Dvc files."
+    )
+    tag_add_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Add tag for all dependencies of the specified DVC file.",
+    )
+    tag_add_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Add tag for subdirectories of the specified directory.",
+    )
+    tag_add_parser.set_defaults(func=CmdTagAdd)
+
+    TAG_REMOVE_HELP = "Remove DVC tag."
+    tag_remove_parser = tag_subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_REMOVE_HELP, "tag-remove"),
+        help=TAG_REMOVE_HELP,
+    )
+    tag_remove_parser.add_argument("tag", help="Dvc tag.")
+    tag_remove_parser.add_argument(
+        "targets", nargs="*", default=[None], help="Dvc files."
+    )
+    tag_remove_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Remove tag for all dependencies of the specified DVC file.",
+    )
+    tag_remove_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Remove tag for subdirectories of the specified directory.",
+    )
+    tag_remove_parser.set_defaults(func=CmdTagRemove)
+
+    TAG_LIST_HELP = "List DVC tags."
+    tag_list_parser = tag_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_LIST_HELP, "tag-list"),
+        help=TAG_LIST_HELP,
+    )
+    tag_list_parser.add_argument(
+        "targets", nargs="*", default=[None], help="Dvc files."
+    )
+    tag_list_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="List tags for all dependencies of the specified DVC file.",
+    )
+    tag_list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="List tags for subdirectories of the specified directory.",
+    )
+    tag_list_parser.set_defaults(func=CmdTagList)

dvc/command/unprotect.py

@@ -1,8 +1,13 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdUnprotect(CmdBase):
@@ -12,18 +17,19 @@ class CmdUnprotect(CmdBase):
                 self.repo.unprotect(target)
             except DvcException:
                 msg = "failed to unprotect '{}'".format(target)
-                logger.error(msg)
+                logger.exception(msg)
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    UNPROTECT_HELP = "Unprotect data file/directory."
+    UNPROTECT_HELP = "Unprotect a data file/directory."
     unprotect_parser = subparsers.add_parser(
         "unprotect",
         parents=[parent_parser],
-        description=UNPROTECT_HELP,
+        description=append_doc_link(UNPROTECT_HELP, "unprotect"),
         help=UNPROTECT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
         "targets", nargs="+", help="Data files/directory."

dvc/config.py

@@ -7,11 +7,13 @@ from dvc.utils.compat import str, open
 import os
 import errno
 import configobj
-from schema import Schema, Optional, And, Use, Regex
+import logging
 
-import dvc.logger as logger
+from schema import Schema, Optional, And, Use, Regex
 from dvc.exceptions import DvcException
 
+logger = logging.getLogger(__name__)
+
 
 class ConfigError(DvcException):
     """DVC config exception.
@@ -193,6 +195,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     SECTION_AWS_STORAGEPATH = "storagepath"
     SECTION_AWS_CREDENTIALPATH = "credentialpath"
     SECTION_AWS_ENDPOINT_URL = "endpointurl"
+    SECTION_AWS_LIST_OBJECTS = "listobjects"
     SECTION_AWS_REGION = "region"
     SECTION_AWS_PROFILE = "profile"
     SECTION_AWS_USE_SSL = "use_ssl"
@@ -202,6 +205,9 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_PROFILE): str,
         Optional(SECTION_AWS_CREDENTIALPATH): str,
         Optional(SECTION_AWS_ENDPOINT_URL): str,
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): And(
+            str, is_bool, Use(to_bool)
+        ),
         Optional(SECTION_AWS_USE_SSL, default=True): And(
             str, is_bool, Use(to_bool)
         ),
@@ -239,6 +245,9 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_AWS_PROFILE): str,
         Optional(SECTION_AWS_CREDENTIALPATH): str,
         Optional(SECTION_AWS_ENDPOINT_URL): str,
+        Optional(SECTION_AWS_LIST_OBJECTS, default=False): And(
+            str, is_bool, Use(to_bool)
+        ),
         Optional(SECTION_AWS_USE_SSL, default=True): And(
             str, is_bool, Use(to_bool)
         ),

dvc/daemon.py

@@ -5,13 +5,16 @@ from __future__ import unicode_literals
 import os
 import sys
 import inspect
+import logging
 from subprocess import Popen
 
-import dvc.logger as logger
+from dvc.env import DVC_DAEMON
 from dvc.utils import is_binary, fix_env
 from dvc.utils.compat import cast_bytes_py2
 
 
+logger = logging.getLogger(__name__)
+
 CREATE_NEW_PROCESS_GROUP = 0x00000200
 DETACHED_PROCESS = 0x00000008
 
@@ -43,7 +46,7 @@ def _spawn_posix(cmd, env):
         if pid > 0:
             return
     except OSError:
-        logger.error("failed at first fork")
+        logger.exception("failed at first fork")
         os._exit(1)  # pylint: disable=protected-access
 
     os.setsid()
@@ -54,7 +57,7 @@ def _spawn_posix(cmd, env):
         if pid > 0:
             os._exit(0)  # pylint: disable=protected-access
     except OSError:
-        logger.error("failed at second fork")
+        logger.exception("failed at second fork")
         os._exit(1)  # pylint: disable=protected-access
 
     sys.stdin.close()
@@ -66,12 +69,29 @@ def _spawn_posix(cmd, env):
     os._exit(0)  # pylint: disable=protected-access
 
 
+def _spawn(cmd, env):
+    logger.debug("Trying to spawn '{}'".format(cmd))
+
+    if os.name == "nt":
+        _spawn_windows(cmd, env)
+    elif os.name == "posix":
+        _spawn_posix(cmd, env)
+    else:
+        raise NotImplementedError
+
+    logger.debug("Spawned '{}'".format(cmd))
+
+
 def daemon(args):
     """Launch a `dvc daemon` command in a detached process.
 
     Args:
         args (list): list of arguments to append to `dvc daemon` command.
     """
+    if os.environ.get(DVC_DAEMON):
+        logger.debug("skipping launching a new daemon.")
+        return
+
     cmd = [sys.executable]
     if not is_binary():
         cmd += ["-m", "dvc"]
@@ -82,14 +102,6 @@ def daemon(args):
     env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
         os.path.dirname(os.path.dirname(file_path))
     )
+    env[cast_bytes_py2(DVC_DAEMON)] = cast_bytes_py2("1")
 
-    logger.debug("Trying to spawn '{}' with env '{}'".format(cmd, env))
-
-    if os.name == "nt":
-        _spawn_windows(cmd, env)
-    elif os.name == "posix":
-        _spawn_posix(cmd, env)
-    else:
-        raise NotImplementedError
-
-    logger.debug("Spawned '{}'".format(cmd))
+    _spawn(cmd, env)

dvc/data_cloud.py

@@ -2,9 +2,9 @@
 
 from __future__ import unicode_literals
 
-import dvc.logger as logger
-from dvc.config import Config, ConfigError
+import logging
 
+from dvc.config import Config, ConfigError
 from dvc.remote import Remote
 from dvc.remote.s3 import RemoteS3
 from dvc.remote.gs import RemoteGS
@@ -15,6 +15,9 @@ from dvc.remote.local import RemoteLOCAL
 from dvc.remote.http import RemoteHTTP
 
 
+logger = logging.getLogger(__name__)
+
+
 class DataCloud(object):
     """Class that manages dvc remotes.
 

dvc/env.py

@@ -0,0 +1 @@
+DVC_DAEMON = "DVC_DAEMON"

dvc/exceptions.py

@@ -67,9 +67,9 @@ class OutputNotFoundError(DvcException):
         )
 
 
-class WorkingDirectoryAsOutputError(DvcException):
+class StagePathAsOutputError(DvcException):
     """Thrown if directory that stage is going to be saved in is specified as
-    an output of anothe stage.
+    an output of another stage.
 
     Args:
         cwd (str): path to the directory.
@@ -77,16 +77,16 @@ class WorkingDirectoryAsOutputError(DvcException):
             output.
     """
 
-    def __init__(self, cwd, fname):
-        assert isinstance(cwd, str) or isinstance(cwd, builtin_str)
+    def __init__(self, wdir, fname):
+        assert isinstance(wdir, str) or isinstance(wdir, builtin_str)
         assert isinstance(fname, str) or isinstance(fname, builtin_str)
         msg = (
             "current working directory '{cwd}' is specified as an output in"
             " '{fname}'. Use another CWD to prevent any data removal.".format(
-                cwd=cwd, fname=fname
+                cwd=wdir, fname=fname
             )
         )
-        super(WorkingDirectoryAsOutputError, self).__init__(msg)
+        super(StagePathAsOutputError, self).__init__(msg)
 
 
 class CircularDependencyError(DvcException):
@@ -149,7 +149,10 @@ class NotDvcRepoError(DvcException):
     """
 
     def __init__(self, root):
-        msg = "not a dvc repository (checked up to mount point '{}')"
+        msg = (
+            "you are not inside of a dvc repository "
+            "(checked up to mount point '{}')"
+        )
         super(NotDvcRepoError, self).__init__(msg.format(root))
 
 
@@ -208,3 +211,47 @@ class NoMetricsError(DvcException):
             "no metric files in this repository. "
             "Use 'dvc metrics add' to add a metric file to track."
         )
+
+
+class StageFileCorruptedError(DvcException):
+    def __init__(self, path):
+        path = os.path.relpath(path)
+        super(StageFileCorruptedError, self).__init__(
+            "unable to read stage file: {} "
+            "YAML file structure is corrupted".format(path)
+        )
+
+
+class RecursiveAddingWhileUsingFilename(DvcException):
+    def __init__(self):
+        super(RecursiveAddingWhileUsingFilename, self).__init__(
+            "using fname with recursive is not allowed."
+        )
+
+
+class OverlappingOutputPathsError(DvcException):
+    def __init__(self, out_1, out_2):
+        super(OverlappingOutputPathsError, self).__init__(
+            "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
+            "unpredictable behaviour, rerun command with non overlapping outs "
+            "paths.".format(
+                str(out_1),
+                out_1.stage.relpath,
+                str(out_2),
+                out_2.stage.relpath,
+            )
+        )
+
+
+class TargetNotDirectoryError(DvcException):
+    def __init__(self, path):
+        super(TargetNotDirectoryError, self).__init__(
+            "Target: {} is not a directory".format(path)
+        )
+
+
+class CheckoutErrorSuggestGit(DvcException):
+    def __init__(self, target, cause):
+        super(CheckoutErrorSuggestGit, self).__init__(
+            "Did you mean 'git checkout {}'?".format(target), cause=cause
+        )

dvc/logger.py

@@ -1,299 +1,173 @@
-"""Manages logger for dvc repo."""
+"""Manages logging configuration for dvc repo."""
 
 from __future__ import unicode_literals
 
-from dvc.exceptions import DvcException
-from dvc.utils.compat import str
+from dvc.utils.compat import str, StringIO
 
-import re
-import sys
 import logging
-import traceback
-
-from contextlib import contextmanager
-
+import logging.handlers
+import logging.config
 import colorama
 
 
-def info(message):
-    """Prints an info message."""
-    logger.info(message)
+class ExcludeErrorsFilter(logging.Filter):
+    def filter(self, record):
+        return record.levelno < logging.ERROR
 
 
-def debug(message):
-    """Prints a debug message."""
-    prefix = colorize("Debug", color="blue")
+class ColorFormatter(logging.Formatter):
+    """Enable color support when logging to a terminal that supports it.
 
-    out = "{prefix}: {message}".format(prefix=prefix, message=message)
+    Color support on Windows versions that do not support ANSI color codes is
+    enabled by use of the colorama__ library.
+    See the colorama documentation for details.
 
-    logger.debug(out)
+    __ https://pypi.python.org/pypi/colorama
 
-
-def warning(message):
-    """Prints a warning message."""
-    prefix = colorize("Warning", color="yellow")
-
-    out = "{prefix}: {message}".format(prefix=prefix, message=message)
-
-    logger.warning(out)
-
-
-def error(message=None):
-    """Prints an error message."""
-    prefix = colorize("Error", color="red")
-
-    exception, stack_trace = _parse_exc()
-
-    out = (
-        "{prefix}: {description}"
-        "\n"
-        "{stack_trace}"
-        "\n"
-        "{footer}".format(
-            prefix=prefix,
-            description=_description(message, exception),
-            stack_trace=stack_trace,
-            footer=_footer(),
-        )
-    )
-
-    logger.error(out)
-
-
-def box(message, border_color=None):
-    """Prints a message in a box.
-
-    Args:
-        message (unicode): message to print.
-        border_color (unicode): name of a color to outline the box with.
+    For records containing `exc_info`, it will use a custom `_walk_exc` to
+    retrieve the whole tracebak.
     """
-    lines = message.split("\n")
-    max_width = max(_visual_width(line) for line in lines)
 
-    padding_horizontal = 5
-    padding_vertical = 1
-
-    box_size_horizontal = max_width + (padding_horizontal * 2)
-
-    chars = {"corner": "+", "horizontal": "-", "vertical": "|", "empty": " "}
-
-    margin = "{corner}{line}{corner}\n".format(
-        corner=chars["corner"], line=chars["horizontal"] * box_size_horizontal
-    )
-
-    padding_lines = [
-        "{border}{space}{border}\n".format(
-            border=colorize(chars["vertical"], color=border_color),
-            space=chars["empty"] * box_size_horizontal,
-        )
-        * padding_vertical
-    ]
-
-    content_lines = [
-        "{border}{space}{content}{space}{border}\n".format(
-            border=colorize(chars["vertical"], color=border_color),
-            space=chars["empty"] * padding_horizontal,
-            content=_visual_center(line, max_width),
-        )
-        for line in lines
-    ]
-
-    box_str = "{margin}{padding}{content}{padding}{margin}".format(
-        margin=colorize(margin, color=border_color),
-        padding="".join(padding_lines),
-        content="".join(content_lines),
-    )
-
-    logger.info(box_str)
-
-
-def level():
-    """Returns current log level."""
-    return logger.getEffectiveLevel()
-
-
-def set_level(level_name):
-    """Sets log level.
-
-    Args:
-        level_name (str): log level name. E.g. info, debug, warning, error,
-            critical.
-    """
-    if not level_name:
-        return
-
-    levels = {
-        "info": logging.INFO,
-        "debug": logging.DEBUG,
-        "warning": logging.WARNING,
-        "error": logging.ERROR,
-        "critical": logging.CRITICAL,
+    color_code = {
+        "DEBUG": colorama.Fore.BLUE,
+        "INFO": "",
+        "WARNING": colorama.Fore.YELLOW,
+        "ERROR": colorama.Fore.RED,
+        "CRITICAL": colorama.Fore.RED,
     }
 
-    logger.setLevel(levels.get(level_name))
-
-
-def be_quiet():
-    """Disables all messages except critical ones."""
-    logger.setLevel(logging.CRITICAL)
-
-
-def be_verbose():
-    """Enables all messages."""
-    logger.setLevel(logging.DEBUG)
-
-
-@contextmanager
-def verbose():
-    """Enables verbose mode for the context."""
-    previous_level = level()
-    be_verbose()
-    yield
-    logger.setLevel(previous_level)
-
-
-@contextmanager
-def quiet():
-    """Enables quiet mode for the context."""
-    previous_level = level()
-    be_quiet()
-    yield
-    logger.setLevel(previous_level)
-
-
-def is_quiet():
-    """Returns whether or not all messages except critical ones are
-    disabled.
-    """
-    return level() == logging.CRITICAL
-
-
-def is_verbose():
-    """Returns whether or not all messages are enabled."""
-    return level() == logging.DEBUG
-
-
-def colorize(message, color=None):
-    """Returns a message in a specified color."""
-    if not color:
-        return message
-
-    colors = {
-        "green": colorama.Fore.GREEN,
-        "yellow": colorama.Fore.YELLOW,
-        "blue": colorama.Fore.BLUE,
-        "red": colorama.Fore.RED,
-    }
-
-    return "{color}{message}{nc}".format(
-        color=colors.get(color, ""), message=message, nc=colorama.Fore.RESET
+    footer = (
+        "{yellow}Having any troubles?{nc}."
+        " Hit us up at {blue}https://dvc.org/support{nc},"
+        " we are always happy to help!"
+    ).format(
+        blue=colorama.Fore.BLUE,
+        nc=colorama.Fore.RESET,
+        yellow=colorama.Fore.YELLOW,
     )
 
+    def format(self, record):
+        if record.levelname == "INFO":
+            return record.msg
 
-def _init_colorama():
+        if record.levelname == "ERROR" or record.levelname == "CRITICAL":
+            exception, stack_trace = self._parse_exc(record.exc_info)
+
+            return (
+                "{color}{levelname}{nc}: {description}"
+                "{stack_trace}\n"
+                "\n"
+                "{footer}"
+            ).format(
+                color=self.color_code.get(record.levelname, ""),
+                nc=colorama.Fore.RESET,
+                levelname=record.levelname,
+                description=self._description(record.msg, exception),
+                msg=record.msg,
+                stack_trace=stack_trace,
+                footer=self.footer,
+            )
+
+        return "{color}{levelname}{nc}: {msg}".format(
+            color=self.color_code.get(record.levelname, ""),
+            nc=colorama.Fore.RESET,
+            levelname=record.levelname,
+            msg=record.msg,
+        )
+
+    def _description(self, message, exception):
+        description = ""
+
+        if exception and message:
+            description = "{message} - {exception}"
+        elif exception:
+            description = "{exception}"
+        elif message:
+            description = "{message}"
+
+        return description.format(message=message, exception=exception)
+
+    def _walk_exc(self, exc_info):
+        import traceback
+
+        buffer = StringIO()
+
+        traceback.print_exception(*exc_info, file=buffer)
+
+        exc = exc_info[1]
+        tb = buffer.getvalue()
+
+        exc_list = [str(exc)]
+        tb_list = [tb]
+
+        # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info
+        while hasattr(exc, "cause") and exc.cause:
+            exc_list.append(str(exc.cause))
+            if hasattr(exc, "cause_tb") and exc.cause_tb:
+                tb_list.insert(0, str(exc.cause_tb))
+            exc = exc.cause
+
+        return exc_list, tb_list
+
+    def _parse_exc(self, exc_info):
+        if not exc_info:
+            return (None, "")
+
+        exc_list, tb_list = self._walk_exc(exc_info)
+
+        exception = ": ".join(exc_list)
+
+        if logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG:
+            stack_trace = (
+                "\n" "{red}{line}{nc}\n" "{stack_trace}" "{red}{line}{nc}"
+            ).format(
+                red=colorama.Fore.RED,
+                nc=colorama.Fore.RESET,
+                line="-" * 60,
+                stack_trace="\n".join(tb_list),
+            )
+        else:
+            stack_trace = ""
+
+        return (exception, stack_trace)
+
+    def _progress_aware(self):
+        """Add a new line if progress bar hasn't finished"""
+        from dvc.progress import progress
+
+        if not progress.is_finished:
+            progress._print()
+        progress.clearln()
+
+
+def setup(level=logging.INFO):
     colorama.init()
 
-
-def set_default_level():
-    """Sets default log level."""
-    logger.setLevel(logging.INFO)
-
-
-def _add_handlers():
-    formatter = "%(message)s"
-
-    class _LogLevelFilter(logging.Filter):
-        # pylint: disable=too-few-public-methods
-        def filter(self, record):
-            return record.levelno <= logging.WARNING
-
-    sh_out = logging.StreamHandler(sys.stdout)
-    sh_out.setFormatter(logging.Formatter(formatter))
-    sh_out.setLevel(logging.DEBUG)
-    sh_out.addFilter(_LogLevelFilter())
-
-    sh_err = logging.StreamHandler(sys.stderr)
-    sh_err.setFormatter(logging.Formatter(formatter))
-    sh_err.setLevel(logging.ERROR)
-
-    logger.addHandler(sh_out)
-    logger.addHandler(sh_err)
-
-
-def _walk_exc(exc):
-    exc_list = [str(exc)]
-    tb_list = [traceback.format_exc()]
-
-    # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info.
-    while hasattr(exc, "cause") and exc.cause is not None:
-        exc_list.append(str(exc.cause))
-        if hasattr(exc, "cause_tb") and exc.cause_tb is not None:
-            tb_list.insert(0, str(exc.cause_tb))
-        exc = exc.cause
-
-    return exc_list, tb_list
-
-
-def _parse_exc():
-    exc = sys.exc_info()[1]
-    if not exc:
-        return (None, "")
-
-    exc_list, tb_list = _walk_exc(exc)
-
-    exception = ": ".join(exc_list)
-
-    if is_verbose():
-        stack_trace = "{line}\n{stack_trace}{line}\n".format(
-            line=colorize("-" * 60, color="red"),
-            stack_trace="\n".join(tb_list),
-        )
-    else:
-        stack_trace = ""
-
-    return (exception, stack_trace)
-
-
-def _description(message, exception):
-    if exception and message:
-        description = "{message} - {exception}"
-    elif exception:
-        description = "{exception}"
-    elif message:
-        description = "{message}"
-    else:
-        raise DvcException(
-            "Unexpected error - either exception or message must be provided"
-        )
-
-    return description.format(message=message, exception=exception)
-
-
-def _footer():
-    return "{phrase} Hit us up at {url}, we are always happy to help!".format(
-        phrase=colorize("Having any troubles?", "yellow"),
-        url=colorize("https://dvc.org/support", "blue"),
+    logging.config.dictConfig(
+        {
+            "version": 1,
+            "filters": {"exclude_errors": {"()": ExcludeErrorsFilter}},
+            "formatters": {"color": {"()": ColorFormatter}},
+            "handlers": {
+                "console": {
+                    "class": "logging.StreamHandler",
+                    "level": "DEBUG",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_errors"],
+                },
+                "console_errors": {
+                    "class": "logging.StreamHandler",
+                    "level": "ERROR",
+                    "formatter": "color",
+                    "stream": "ext://sys.stderr",
+                },
+            },
+            "loggers": {
+                "dvc": {
+                    "level": level,
+                    "handlers": ["console", "console_errors"],
+                }
+            },
+        }
     )
-
-
-def _visual_width(line):
-    """Get the the number of columns required to display a string"""
-
-    return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, "", line))
-
-
-def _visual_center(line, width):
-    """Center align string according to it's visual width"""
-
-    spaces = max(width - _visual_width(line), 0)
-    left_padding = int(spaces / 2)
-    right_padding = spaces - left_padding
-
-    return (left_padding * " ") + line + (right_padding * " ")
-
-
-logger = logging.getLogger("dvc")  # pylint: disable=invalid-name
-
-set_default_level()
-_add_handlers()
-_init_colorama()

dvc/main.py

@@ -2,13 +2,16 @@
 
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import logging
+
 from dvc.cli import parse_args
-from dvc.command.base import CmdBase
 from dvc.analytics import Analytics
 from dvc.exceptions import NotDvcRepoError, DvcParserError
 
 
+logger = logging.getLogger("dvc")
+
+
 def main(argv=None):
     """Run dvc CLI command.
 
@@ -20,26 +23,28 @@ def main(argv=None):
     """
     args = None
     cmd = None
+
     try:
         args = parse_args(argv)
 
-        # Init loglevel early in case we'll run
-        # into errors before setting it properly
-        CmdBase.set_loglevel(args)
+        if args.quiet:
+            logger.setLevel(logging.CRITICAL)
+
+        elif args.verbose:
+            logger.setLevel(logging.DEBUG)
 
         cmd = args.func(args)
-
         ret = cmd.run_cmd()
     except KeyboardInterrupt:
-        logger.error("interrupted by the user")
+        logger.exception("interrupted by the user")
         ret = 252
     except NotDvcRepoError:
-        logger.error()
+        logger.exception("")
         ret = 253
     except DvcParserError:
         ret = 254
     except Exception:  # pylint: disable=broad-except
-        logger.error("unexpected error")
+        logger.exception("unexpected error")
         ret = 255
 
     Analytics().send_cmd(cmd, args, ret)

dvc/output/__init__.py

@@ -33,38 +33,69 @@ OUTS_MAP = {
     "local": OutputLOCAL,
 }
 
-SCHEMA = {
-    OutputBase.PARAM_PATH: str,
-    # NOTE: currently there are only 3 possible checksum names:
-    #
-    #    1) md5 (LOCAL, SSH, GS);
-    #    2) etag (S3);
-    #    3) checksum (HDFS);
-    #
-    # so when a few types of outputs share the same name, we only need
-    # specify it once.
+# NOTE: currently there are only 3 possible checksum names:
+#
+#    1) md5 (LOCAL, SSH, GS);
+#    2) etag (S3);
+#    3) checksum (HDFS);
+#
+# so when a few types of outputs share the same name, we only need
+# specify it once.
+CHECKSUM_SCHEMA = {
     schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None),
     schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None),
     schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
-    schema.Optional(OutputBase.PARAM_CACHE): bool,
-    schema.Optional(OutputBase.PARAM_METRIC): OutputBase.METRIC_SCHEMA,
 }
 
+TAGS_SCHEMA = {schema.Optional(str): CHECKSUM_SCHEMA}
 
-def _get(stage, p, info, cache, metric):
+SCHEMA = CHECKSUM_SCHEMA.copy()
+SCHEMA[OutputBase.PARAM_PATH] = str
+SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)] = bool
+SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] = OutputBase.METRIC_SCHEMA
+SCHEMA[schema.Optional(OutputBase.PARAM_TAGS)] = TAGS_SCHEMA
+SCHEMA[schema.Optional(OutputBase.PARAM_PERSIST)] = bool
+
+
+def _get(stage, p, info, cache, metric, persist=False, tags=None):
     parsed = urlparse(p)
     if parsed.scheme == "remote":
         name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
         sect = stage.repo.config.config[name]
         remote = Remote(stage.repo, sect)
         return OUTS_MAP[remote.scheme](
-            stage, p, info, cache=cache, remote=remote, metric=metric
+            stage,
+            p,
+            info,
+            cache=cache,
+            remote=remote,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
 
     for o in OUTS:
         if o.supported(p):
-            return o(stage, p, info, cache=cache, remote=None, metric=metric)
-    return OutputLOCAL(stage, p, info, cache=cache, remote=None, metric=metric)
+            return o(
+                stage,
+                p,
+                info,
+                cache=cache,
+                remote=None,
+                metric=metric,
+                persist=persist,
+                tags=tags,
+            )
+    return OutputLOCAL(
+        stage,
+        p,
+        info,
+        cache=cache,
+        remote=None,
+        metric=metric,
+        persist=persist,
+        tags=tags,
+    )
 
 
 def loadd_from(stage, d_list):
@@ -73,12 +104,33 @@ def loadd_from(stage, d_list):
         p = d.pop(OutputBase.PARAM_PATH)
         cache = d.pop(OutputBase.PARAM_CACHE, True)
         metric = d.pop(OutputBase.PARAM_METRIC, False)
-        ret.append(_get(stage, p, info=d, cache=cache, metric=metric))
+        persist = d.pop(OutputBase.PARAM_PERSIST, False)
+        tags = d.pop(OutputBase.PARAM_TAGS, None)
+        ret.append(
+            _get(
+                stage,
+                p,
+                info=d,
+                cache=cache,
+                metric=metric,
+                persist=persist,
+                tags=tags,
+            )
+        )
     return ret
 
 
-def loads_from(stage, s_list, use_cache=True, metric=False):
+def loads_from(stage, s_list, use_cache=True, metric=False, persist=False):
     ret = []
     for s in s_list:
-        ret.append(_get(stage, s, info={}, cache=use_cache, metric=metric))
+        ret.append(
+            _get(
+                stage,
+                s,
+                info={},
+                cache=use_cache,
+                metric=metric,
+                persist=persist,
+            )
+        )
     return ret

dvc/output/base.py

@@ -1,10 +1,11 @@
 from __future__ import unicode_literals
-from dvc.utils.compat import str
 
 import re
 from schema import Or, Optional
 
 from dvc.exceptions import DvcException
+from dvc.utils.compat import str
+from dvc.remote.base import RemoteBase
 
 
 class OutputDoesNotExistError(DvcException):
@@ -28,13 +29,14 @@ class OutputAlreadyTrackedError(DvcException):
 class OutputBase(object):
     IS_DEPENDENCY = False
 
-    REMOTE = None
+    REMOTE = RemoteBase
 
     PARAM_PATH = "path"
     PARAM_CACHE = "cache"
     PARAM_METRIC = "metric"
     PARAM_METRIC_TYPE = "type"
     PARAM_METRIC_XPATH = "xpath"
+    PARAM_PERSIST = "persist"
 
     METRIC_SCHEMA = Or(
         None,
@@ -45,11 +47,21 @@ class OutputBase(object):
         },
     )
 
+    PARAM_TAGS = "tags"
+
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         self.stage = stage
         self.repo = stage.repo
@@ -58,6 +70,8 @@ class OutputBase(object):
         self.remote = remote or self.REMOTE(self.repo, {})
         self.use_cache = False if self.IS_DEPENDENCY else cache
         self.metric = False if self.IS_DEPENDENCY else metric
+        self.persist = persist
+        self.tags = None if self.IS_DEPENDENCY else (tags or {})
 
         if (
             self.use_cache
@@ -77,6 +91,15 @@ class OutputBase(object):
     def __str__(self):
         return self.url
 
+    @property
+    def is_in_repo(self):
+        return False
+
+    def assign_to_stage_file(self, target_repo):
+        raise DvcException(
+            "change repo is not supported for {}".format(self.scheme)
+        )
+
     @classmethod
     def match(cls, url):
         return re.match(cls.REMOTE.REGEX, url)
@@ -146,6 +169,9 @@ class OutputBase(object):
         return bool(self.status())
 
     def save(self):
+        if not self.exists:
+            raise self.DoesNotExistError(self)
+
         self.info = self.remote.save_info(self.path_info)
 
     def commit(self):
@@ -171,18 +197,35 @@ class OutputBase(object):
                 del self.metric[self.PARAM_METRIC_XPATH]
 
         ret[self.PARAM_METRIC] = self.metric
+        ret[self.PARAM_PERSIST] = self.persist
+
+        if self.tags:
+            ret[self.PARAM_TAGS] = self.tags
 
         return ret
 
+    def verify_metric(self):
+        raise DvcException(
+            "verify metric is not supported for {}".format(self.scheme)
+        )
+
     def download(self, to_info, resume=False):
         self.remote.download([self.path_info], [to_info], resume=resume)
 
-    def checkout(self, force=False):
+    def checkout(self, force=False, progress_callback=None, tag=None):
         if not self.use_cache:
             return
 
+        if tag:
+            info = self.tags[tag]
+        else:
+            info = self.info
+
         getattr(self.repo.cache, self.scheme).checkout(
-            self.path_info, self.info, force=force
+            self.path_info,
+            info,
+            force=force,
+            progress_callback=progress_callback,
         )
 
     def remove(self, ignore_remove=False):
@@ -190,11 +233,11 @@ class OutputBase(object):
         if self.scheme != "local":
             return
 
-        if ignore_remove and self.use_cache and self.is_local:
+        if ignore_remove and self.use_cache and self.is_in_repo:
             self.repo.scm.ignore_remove(self.path)
 
     def move(self, out):
-        if self.scheme == "local" and self.use_cache and self.is_local:
+        if self.scheme == "local" and self.use_cache and self.is_in_repo:
             self.repo.scm.ignore_remove(self.path)
 
         self.remote.move(self.path_info, out.path_info)
@@ -203,5 +246,15 @@ class OutputBase(object):
         self.save()
         self.commit()
 
-        if self.scheme == "local" and self.use_cache and self.is_local:
+        if self.scheme == "local" and self.use_cache and self.is_in_repo:
             self.repo.scm.ignore(self.path)
+
+    def get_files_number(self):
+        if self.use_cache and self.checksum:
+            return 1
+
+        return 0
+
+    def unprotect(self):
+        if self.exists:
+            self.remote.unprotect(self.path_info)

dvc/output/hdfs.py

@@ -11,10 +11,25 @@ class OutputHDFS(OutputBase):
     REMOTE = RemoteHDFS
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         super(OutputHDFS, self).__init__(
-            stage, path, info=info, remote=remote, cache=cache, metric=metric
+            stage,
+            path,
+            info=info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
         if remote:
             path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))

dvc/output/local.py

@@ -1,8 +1,8 @@
 from __future__ import unicode_literals
 
 import os
+import logging
 
-import dvc.logger as logger
 from dvc.utils.compat import urlparse
 from dvc.istextfile import istextfile
 from dvc.exceptions import DvcException
@@ -10,14 +10,32 @@ from dvc.remote.local import RemoteLOCAL
 from dvc.output.base import OutputBase, OutputAlreadyTrackedError
 
 
+logger = logging.getLogger(__name__)
+
+
 class OutputLOCAL(OutputBase):
     REMOTE = RemoteLOCAL
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         super(OutputLOCAL, self).__init__(
-            stage, path, info, remote=remote, cache=cache, metric=metric
+            stage,
+            path,
+            info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
         if remote:
             p = os.path.join(
@@ -28,20 +46,33 @@ class OutputLOCAL(OutputBase):
 
         if not os.path.isabs(p):
             p = self.remote.to_ospath(p)
-            p = os.path.join(stage.cwd, p)
+            p = os.path.join(stage.wdir, p)
         p = os.path.abspath(os.path.normpath(p))
 
         self.path_info = {"scheme": "local", "path": p}
 
+        self._dir_cache = {}
+
     def __str__(self):
         return self.rel_path
 
     @property
-    def is_local(self):
+    def is_in_repo(self):
         return urlparse(self.url).scheme != "remote" and not os.path.isabs(
             self.url
         )
 
+    def assign_to_stage_file(self, stage):
+        from dvc.repo import Repo
+
+        fullpath = os.path.abspath(stage.wdir)
+        self.path_info["path"] = os.path.join(fullpath, self.stage_path)
+
+        self.repo = Repo(self.path)
+
+        self.stage = stage
+        return self
+
     @property
     def sep(self):
         return os.sep
@@ -50,15 +81,23 @@ class OutputLOCAL(OutputBase):
     def rel_path(self):
         return os.path.relpath(self.path)
 
+    @property
+    def stage_path(self):
+        return os.path.relpath(self.path, self.stage.wdir)
+
     @property
     def cache(self):
         return self.repo.cache.local.get(self.checksum)
 
+    @property
+    def is_dir_cache(self):
+        return self.repo.cache.local.is_dir_cache(self.checksum)
+
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
-        if self.is_local:
+        if self.is_in_repo:
             path = self.remote.unixpath(
-                os.path.relpath(self.path, self.stage.cwd)
+                os.path.relpath(self.path, self.stage.wdir)
             )
         else:
             path = self.url
@@ -67,7 +106,7 @@ class OutputLOCAL(OutputBase):
 
         return ret
 
-    def _verify_metric(self):
+    def verify_metric(self):
         if not self.metric:
             return
 
@@ -97,7 +136,7 @@ class OutputLOCAL(OutputBase):
 
         if not self.use_cache:
             self.info = self.remote.save_info(self.path_info)
-            self._verify_metric()
+            self.verify_metric()
             if not self.IS_DEPENDENCY:
                 msg = "Output '{}' doesn't use cache. Skipping saving."
                 logger.info(msg.format(self.rel_path))
@@ -110,7 +149,7 @@ class OutputLOCAL(OutputBase):
             logger.info(msg.format(self.rel_path))
             return
 
-        if self.is_local:
+        if self.is_in_repo:
             if self.repo.scm.is_tracked(self.path):
                 raise OutputAlreadyTrackedError(self.rel_path)
 
@@ -118,3 +157,15 @@ class OutputLOCAL(OutputBase):
                 self.repo.scm.ignore(self.path)
 
         self.info = self.remote.save_info(self.path_info)
+
+    @property
+    def dir_cache(self):
+        return self.repo.cache.local.load_dir_cache(self.checksum)
+
+    def get_files_number(self):
+        if self.cache is None:
+            return 0
+
+        if self.is_dir_cache:
+            return len(self.dir_cache)
+        return 1

dvc/output/s3.py

@@ -11,10 +11,25 @@ class OutputS3(OutputBase):
     REMOTE = RemoteS3
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         super(OutputS3, self).__init__(
-            stage, path, info=info, remote=remote, cache=cache, metric=metric
+            stage,
+            path,
+            info=info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
         bucket = remote.bucket if remote else urlparse(path).netloc
         path = urlparse(path).path.lstrip("/")

dvc/output/ssh.py

@@ -12,10 +12,25 @@ class OutputSSH(OutputBase):
     REMOTE = RemoteSSH
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         super(OutputSSH, self).__init__(
-            stage, path, info=info, remote=remote, cache=cache, metric=metric
+            stage,
+            path,
+            info=info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
         parsed = urlparse(path)
         host = remote.host if remote else parsed.hostname

dvc/progress.py

@@ -7,7 +7,8 @@ from dvc.utils.compat import str
 
 import sys
 import threading
-import dvc.logger as logger
+
+CLEARLINE_PATTERN = "\r\x1b[K"
 
 
 class Progress(object):
@@ -31,14 +32,20 @@ class Progress(object):
         """Returns if all targets have finished."""
         return self._n_total == self._n_finished
 
-    def _clearln(self):
-        self._print("\r\x1b[K", end="")
+    def clearln(self):
+        self._print(CLEARLINE_PATTERN, end="")
 
     def _writeln(self, line):
-        self._clearln()
+        self.clearln()
         self._print(line, end="")
         sys.stdout.flush()
 
+    def reset(self):
+        with self._lock:
+            self._n_total = 0
+            self._n_finished = 0
+            self._line = None
+
     def refresh(self, line=None):
         """Refreshes progress bar."""
         # Just go away if it is locked. Will update next time
@@ -65,7 +72,7 @@ class Progress(object):
             pbar = self._bar(name, 100, 100)
 
             if sys.stdout.isatty():
-                self._clearln()
+                self.clearln()
 
             self._print(pbar)
 
@@ -100,7 +107,11 @@ class Progress(object):
 
     @staticmethod
     def _print(*args, **kwargs):
-        if logger.is_quiet():
+        import logging
+
+        logger = logging.getLogger(__name__)
+
+        if logger.getEffectiveLevel() == "CRITICAL":
             return
 
         print(*args, **kwargs)
@@ -108,7 +119,7 @@ class Progress(object):
     def __enter__(self):
         self._lock.acquire(True)
         if self._line is not None:
-            self._clearln()
+            self.clearln()
 
     def __exit__(self, typ, value, tbck):
         if self._line is not None:
@@ -116,4 +127,18 @@ class Progress(object):
         self._lock.release()
 
 
+class ProgressCallback(object):
+    def __init__(self, total):
+        self.total = total
+        self.current = 0
+        progress.reset()
+
+    def update(self, name, progress_to_add=1):
+        self.current += progress_to_add
+        progress.update_target(name, self.current, self.total)
+
+    def finish(self, name):
+        progress.finish_target(name)
+
+
 progress = Progress()  # pylint: disable=invalid-name

dvc/prompt.py

@@ -4,11 +4,15 @@ from __future__ import unicode_literals
 from __future__ import print_function
 
 import sys
+import logging
 from getpass import getpass
 
 from dvc.utils.compat import input
 
 
+logger = logging.getLogger(__name__)
+
+
 def _ask(prompt, limited_to=None):
     if not sys.stdout.isatty():
         return None
@@ -25,10 +29,9 @@ def _ask(prompt, limited_to=None):
         if answer in limited_to:
             return answer
 
-        print(
-            "Your response must be one of: {options}. Please try again.".format(
-                options=limited_to
-            )
+        logger.info(
+            "Your response must be one of: {options}. "
+            "Please try again.".format(options=limited_to)
         )
 
 
@@ -55,5 +58,5 @@ def password(statement):
     Returns:
         str: password entered by the user.
     """
-    prompt = "{statement}: ".format(statement=statement)
-    return getpass(prompt)
+    logger.info("{statement}: ".format(statement=statement))
+    return getpass("")

dvc/remote/azure.py

@@ -3,19 +3,24 @@ from __future__ import unicode_literals
 
 import os
 import re
+import logging
 
 try:
     from azure.storage.blob import BlockBlobService
+    from azure.common import AzureMissingResourceHttpError
 except ImportError:
     BlockBlobService = None
 
-import dvc.logger as logger
-from dvc.utils.compat import urlparse
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
 
 
+logger = logging.getLogger(__name__)
+
+
 class Callback(object):
     def __init__(self, name):
         self.name = name
@@ -76,7 +81,12 @@ class RemoteAzure(RemoteBase):
                 connection_string=self.connection_string
             )
             logger.debug("Container name {}".format(self.bucket))
-            self.__blob_service.create_container(self.bucket)
+            try:  # verify that container exists
+                self.__blob_service.list_blobs(
+                    self.bucket, delimiter="/", num_results=1
+                )
+            except AzureMissingResourceHttpError:
+                self.__blob_service.create_container(self.bucket)
         return self.__blob_service
 
     def remove(self, path_info):
@@ -170,13 +180,13 @@ class RemoteAzure(RemoteBase):
                 )
             )
 
-            tmp_file = self.tmp_file(to_info["path"])
+            tmp_file = tmp_fname(to_info["path"])
             if not name:
                 name = os.path.basename(to_info["path"])
 
             cb = None if no_progress_bar else Callback(name)
 
-            self._makedirs(to_info["path"])
+            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
 
             try:
                 self.blob_service.get_blob_to_path(
@@ -186,7 +196,7 @@ class RemoteAzure(RemoteBase):
                 msg = "failed to download '{}/{}'".format(bucket, path)
                 logger.warning(msg)
             else:
-                os.rename(tmp_file, to_info["path"])
+                move(tmp_file, to_info["path"])
 
                 if not no_progress_bar:
                     progress.finish_target(name)

dvc/remote/base.py

@@ -2,19 +2,21 @@ from __future__ import unicode_literals
 
 from dvc.utils.compat import str
 
-import os
 import re
-import errno
 import posixpath
+import logging
 from multiprocessing import cpu_count
 
 import dvc.prompt as prompt
-import dvc.logger as logger
 from dvc.config import Config
 from dvc.exceptions import DvcException, ConfirmRemoveError
 
 
+logger = logging.getLogger(__name__)
+
+
 STATUS_OK = 1
+STATUS_MISSING = 2
 STATUS_NEW = 3
 STATUS_DELETED = 4
 
@@ -22,7 +24,7 @@ STATUS_DELETED = 4
 STATUS_MAP = {
     # (local_exists, remote_exists)
     (True, True): STATUS_OK,
-    (False, False): STATUS_OK,
+    (False, False): STATUS_MISSING,
     (True, False): STATUS_NEW,
     (False, True): STATUS_DELETED,
 }
@@ -83,10 +85,8 @@ class RemoteBase(object):
                 "\n"
                 "If you have installed dvc from a binary package and you "
                 "are still seeing this message, please report it to us "
-                "using https://github.com/iterative/dvc/issues. Thank you!".format(
-                    url, missing, " ".join(missing), self.scheme
-                )
-            )
+                "using https://github.com/iterative/dvc/issues. Thank you!"
+            ).format(url, missing, " ".join(missing), self.scheme)
             raise RemoteMissingDepsError(msg)
 
     def __repr__(self):
@@ -112,13 +112,6 @@ class RemoteBase(object):
             return None
         return m.group(name)
 
-    @staticmethod
-    def tmp_file(fname):
-        """ Temporary name for a partial download """
-        import uuid
-
-        return fname + "." + str(uuid.uuid4())
-
     def save_info(self, path_info):
         raise NotImplementedError
 
@@ -173,7 +166,21 @@ class RemoteBase(object):
         return False
 
     def save(self, path_info, checksum_info):
-        raise RemoteActionNotImplemented("save", self.scheme)
+        if path_info["scheme"] != self.scheme:
+            raise RemoteActionNotImplemented(
+                "save {} -> {}".format(path_info["scheme"], self.scheme),
+                self.scheme,
+            )
+
+        checksum = checksum_info[self.PARAM_CHECKSUM]
+        if not self.changed_cache(checksum):
+            return
+
+        to_info = self.checksum_to_path_info(checksum)
+
+        logger.info("Saving '{}' to '{}'.".format(path_info, to_info))
+
+        self.copy(path_info, to_info)
 
     def download(
         self,
@@ -197,15 +204,6 @@ class RemoteBase(object):
     def copy(self, from_info, to_info):
         raise RemoteActionNotImplemented("copy", self.scheme)
 
-    def _makedirs(self, fname):
-        dname = os.path.dirname(fname)
-
-        try:
-            os.makedirs(dname)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
-
     def exists(self, path_infos):
         raise NotImplementedError
 
@@ -274,6 +272,10 @@ class RemoteBase(object):
     def changed_cache(self, checksum):
         cache = self.checksum_to_path_info(checksum)
         expected = {self.PARAM_CHECKSUM: checksum}
+
+        if not self.exists(cache):
+            return True
+
         actual = self.save_info(cache)
 
         logger.debug(
@@ -323,7 +325,9 @@ class RemoteBase(object):
 
         self.remove(path_info)
 
-    def do_checkout(self, path_info, checksum, force=False):
+    def do_checkout(
+        self, path_info, checksum, force=False, progress_callback=None
+    ):
         if self.exists(path_info):
             msg = "data '{}' exists. Removing before checkout."
             logger.warning(msg.format(str(path_info)))
@@ -332,7 +336,9 @@ class RemoteBase(object):
         from_info = self.checksum_to_path_info(checksum)
         self.copy(from_info, path_info)
 
-    def checkout(self, path_info, checksum_info, force=False):
+    def checkout(
+        self, path_info, checksum_info, force=False, progress_callback=None
+    ):
         scheme = path_info["scheme"]
         if scheme not in ["", "local"] and scheme != self.scheme:
             raise NotImplementedError
@@ -340,12 +346,12 @@ class RemoteBase(object):
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
         if not checksum:
             msg = "No checksum info for '{}'."
-            logger.info(msg.format(str(path_info)))
+            logger.debug(msg.format(str(path_info)))
             return
 
         if not self.changed(path_info, checksum_info):
             msg = "Data '{}' didn't change."
-            logger.info(msg.format(str(path_info)))
+            logger.debug(msg.format(str(path_info)))
             return
 
         if self.changed_cache(checksum):
@@ -355,6 +361,15 @@ class RemoteBase(object):
             return
 
         msg = "Checking out '{}' with cache '{}'."
-        logger.info(msg.format(str(path_info), checksum))
+        logger.debug(msg.format(str(path_info), checksum))
 
-        self.do_checkout(path_info, checksum, force=force)
+        self.do_checkout(
+            path_info,
+            checksum,
+            force=force,
+            progress_callback=progress_callback,
+        )
+
+    @staticmethod
+    def unprotect(path_info):
+        pass

dvc/remote/gs.py

@@ -1,21 +1,24 @@
 from __future__ import unicode_literals
 
 import os
+import logging
 
 try:
     from google.cloud import storage
-    from google.auth.credentials import Credentials
 except ImportError:
     storage = None
 
-import dvc.logger as logger
-from dvc.utils.compat import urlparse
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
 from dvc.remote.base import RemoteBase
 from dvc.config import Config
 from dvc.progress import progress
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class RemoteGS(RemoteBase):
     scheme = "gs"
     REGEX = r"^gs://(?P<path>.*)$"
@@ -87,18 +90,6 @@ class RemoteGS(RemoteBase):
             blob, self.gs.bucket(to_info["bucket"]), new_name=to_info["path"]
         )
 
-    def save(self, path_info):
-        if path_info["scheme"] != "gs":
-            raise NotImplementedError
-
-        md5 = self.get_md5(path_info["bucket"], path_info["path"])
-        path = self.checksum_to_path(md5)
-        to_info = {"scheme": "gs", "bucket": self.bucket, "path": path}
-
-        self.copy(path_info, to_info)
-
-        return {self.PARAM_CHECKSUM: md5}
-
     def remove(self, path_info):
         if path_info["scheme"] != "gs":
             raise NotImplementedError
@@ -158,7 +149,7 @@ class RemoteGS(RemoteBase):
                 blob.upload_from_filename(from_info["path"])
             except Exception:
                 msg = "failed to upload '{}' to '{}/{}'"
-                logger.error(
+                logger.exception(
                     msg.format(
                         from_info["path"], to_info["bucket"], to_info["path"]
                     )
@@ -195,7 +186,7 @@ class RemoteGS(RemoteBase):
             )
             logger.debug(msg)
 
-            tmp_file = self.tmp_file(to_info["path"])
+            tmp_file = tmp_fname(to_info["path"])
             if not name:
                 name = os.path.basename(to_info["path"])
 
@@ -204,7 +195,7 @@ class RemoteGS(RemoteBase):
                 # lets at least update progress at pathpoints(start, finish)
                 progress.update_target(name, 0, None)
 
-            self._makedirs(to_info["path"])
+            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
 
             try:
                 bucket = gs.bucket(from_info["bucket"])
@@ -212,14 +203,14 @@ class RemoteGS(RemoteBase):
                 blob.download_to_filename(tmp_file)
             except Exception:
                 msg = "failed to download '{}/{}' to '{}'"
-                logger.error(
+                logger.exception(
                     msg.format(
                         from_info["bucket"], from_info["path"], to_info["path"]
                     )
                 )
                 continue
 
-            os.rename(tmp_file, to_info["path"])
+            move(tmp_file, to_info["path"])
 
             if not no_progress_bar:
                 progress.finish_target(name)

dvc/remote/hdfs.py

@@ -4,12 +4,15 @@ import os
 import re
 import getpass
 import posixpath
+import logging
 from subprocess import Popen, PIPE
 
-import dvc.logger as logger
 from dvc.config import Config
 from dvc.remote.base import RemoteBase, RemoteCmdError
-from dvc.utils import fix_env
+from dvc.utils import fix_env, tmp_fname
+
+
+logger = logging.getLogger(__name__)
 
 
 class RemoteHDFS(RemoteBase):
@@ -77,7 +80,7 @@ class RemoteHDFS(RemoteBase):
 
     def rm(self, path_info):
         self.hadoop_fs(
-            "rm {}".format(path_info["path"]), user=path_info["user"]
+            "rm -f {}".format(path_info["path"]), user=path_info["user"]
         )
 
     def save_info(self, path_info):
@@ -88,18 +91,6 @@ class RemoteHDFS(RemoteBase):
 
         return {self.PARAM_CHECKSUM: self.checksum(path_info)}
 
-    def save(self, path_info, checksum_info):
-        if path_info["scheme"] != "hdfs":
-            raise NotImplementedError
-
-        assert path_info.get("path")
-
-        checksum = checksum_info[self.PARAM_CHECKSUM]
-        dest = path_info.copy()
-        dest["path"] = self.checksum_to_path(checksum)
-
-        self.copy(path_info, dest)
-
     def remove(self, path_info):
         if path_info["scheme"] != "hdfs":
             raise NotImplementedError
@@ -130,13 +121,22 @@ class RemoteHDFS(RemoteBase):
             if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            cmd = "mkdir -p {}".format(posixpath.dirname(to_info["path"]))
-            self.hadoop_fs(cmd, user=to_info["user"])
+            self.hadoop_fs(
+                "mkdir -p {}".format(posixpath.dirname(to_info["path"])),
+                user=to_info["user"],
+            )
 
-            cmd = "copyFromLocal {} {}".format(
-                from_info["path"], to_info["path"]
+            tmp_file = tmp_fname(to_info["path"])
+
+            self.hadoop_fs(
+                "copyFromLocal {} {}".format(from_info["path"], tmp_file),
+                user=to_info["user"],
+            )
+
+            self.hadoop_fs(
+                "mv {} {}".format(tmp_file, to_info["path"]),
+                user=to_info["user"],
             )
-            self.hadoop_fs(cmd, user=to_info["user"])
 
     def download(
         self,
@@ -163,10 +163,14 @@ class RemoteHDFS(RemoteBase):
             if not os.path.exists(dname):
                 os.makedirs(dname)
 
-            cmd = "copyToLocal {} {}".format(
-                from_info["path"], to_info["path"]
+            tmp_file = tmp_fname(to_info["path"])
+
+            self.hadoop_fs(
+                "copyToLocal {} {}".format(from_info["path"], tmp_file),
+                user=from_info["user"],
             )
-            self.hadoop_fs(cmd, user=from_info["user"])
+
+            os.rename(tmp_file, to_info["path"])
 
     def list_cache_paths(self):
         try:

dvc/remote/http.py

@@ -1,15 +1,19 @@
 from __future__ import unicode_literals
-from dvc.utils.compat import open
+from dvc.utils.compat import open, makedirs
 
 import os
 import threading
 import requests
+import logging
 
-import dvc.logger as logger
 from dvc.progress import progress
 from dvc.exceptions import DvcException
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
+from dvc.utils import move
+
+
+logger = logging.getLogger(__name__)
 
 
 class ProgressBarCallback(object):
@@ -67,7 +71,7 @@ class RemoteHTTP(RemoteBase):
             if not name:
                 name = os.path.basename(to_info["path"])
 
-            self._makedirs(to_info["path"])
+            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
 
             total = self._content_length(from_info["path"])
 
@@ -121,9 +125,8 @@ class RemoteHTTP(RemoteBase):
 
         if not etag:
             raise DvcException(
-                "could not find an ETag or Content-MD5 header for '{url}'".format(
-                    url=url
-                )
+                "could not find an ETag or "
+                "Content-MD5 header for '{url}'".format(url=url)
             )
 
         if etag.startswith("W/"):
@@ -148,7 +151,7 @@ class RemoteHTTP(RemoteBase):
             mode, partial_file, request, transferred_bytes, callback
         )
 
-        os.rename(partial_file, target_file)
+        move(partial_file, target_file)
 
     def _write_request_content(
         self, mode, partial_file, request, transferred_bytes, callback=None

dvc/remote/local.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str
+from dvc.utils.compat import str, makedirs
 
 import os
 import stat
@@ -9,18 +9,37 @@ import json
 import ntpath
 import shutil
 import posixpath
+import logging
 from operator import itemgetter
 
-import dvc.logger as logger
 from dvc.system import System
-from dvc.remote.base import RemoteBase, STATUS_MAP, STATUS_NEW, STATUS_DELETED
-from dvc.utils import remove, move, copyfile, dict_md5, to_chunks
+from dvc.remote.base import (
+    RemoteBase,
+    STATUS_MAP,
+    STATUS_NEW,
+    STATUS_DELETED,
+    STATUS_MISSING,
+)
+from dvc.utils import (
+    remove,
+    move,
+    copyfile,
+    dict_md5,
+    to_chunks,
+    tmp_fname,
+    walk_files,
+)
 from dvc.utils import LARGE_DIR_SIZE
 from dvc.config import Config
 from dvc.exceptions import DvcException
 from dvc.progress import progress
 from concurrent.futures import ThreadPoolExecutor
 
+from dvc.utils.fs import get_mtime_and_size, get_inode
+
+
+logger = logging.getLogger(__name__)
+
 
 class RemoteLOCAL(RemoteBase):
     scheme = "local"
@@ -62,6 +81,8 @@ class RemoteLOCAL(RemoteBase):
 
         self.path_info = {"scheme": "local"}
 
+        self._dir_info = {}
+
     @staticmethod
     def compat_config(config):
         ret = config.copy()
@@ -108,7 +129,7 @@ class RemoteLOCAL(RemoteBase):
     def exists(self, path_info):
         assert not isinstance(path_info, list)
         assert path_info["scheme"] == "local"
-        return os.path.exists(path_info["path"])
+        return os.path.lexists(path_info["path"])
 
     def changed_cache(self, md5):
         cache = self.get(md5)
@@ -122,13 +143,11 @@ class RemoteLOCAL(RemoteBase):
             if self.changed_cache_file(md5):
                 return True
 
-            if not self.is_dir_cache(cache):
-                continue
-
-            for entry in self.load_dir_cache(md5):
-                md5 = entry[self.PARAM_CHECKSUM]
-                cache = self.get(md5)
-                clist.append((cache, md5))
+            if self.is_dir_cache(cache) and self._cache_metadata_changed():
+                for entry in self.load_dir_cache(md5):
+                    md5 = entry[self.PARAM_CHECKSUM]
+                    cache = self.get(md5)
+                    clist.append((cache, md5))
 
         return False
 
@@ -242,9 +261,12 @@ class RemoteLOCAL(RemoteBase):
 
     def load_dir_cache(self, md5):
         path = self.get(md5)
-
         assert self.is_dir_cache(path)
 
+        dir_info = self._dir_info.get(md5)
+        if dir_info:
+            return dir_info
+
         try:
             with open(path, "r") as fd:
                 d = json.load(fd)
@@ -261,6 +283,8 @@ class RemoteLOCAL(RemoteBase):
         for info in d:
             info["relpath"] = self.to_ospath(info["relpath"])
 
+        self._dir_info[md5] = d
+
         return d
 
     def dump_dir_cache(self, md5, dir_info):
@@ -285,17 +309,20 @@ class RemoteLOCAL(RemoteBase):
     def is_dir_cache(cls, cache):
         return cache.endswith(cls.MD5_DIR_SUFFIX)
 
-    def do_checkout(self, path_info, checksum, force=False):
+    def do_checkout(
+        self, path_info, checksum, force=False, progress_callback=None
+    ):
         path = path_info["path"]
-        md5 = checksum
-        cache = self.get(md5)
+        cache = self.get(checksum)
 
         if not self.is_dir_cache(cache):
-            if os.path.exists(path):
+            if self.exists(path_info):
                 self.safe_remove(path_info, force=force)
 
             self.link(cache, path)
             self.state.update_link(path)
+            if progress_callback:
+                progress_callback.update(os.path.relpath(path))
             return
 
         # Create dir separately so that dir is created
@@ -303,12 +330,11 @@ class RemoteLOCAL(RemoteBase):
         if not os.path.exists(path):
             os.makedirs(path)
 
-        dir_info = self.load_dir_cache(md5)
         dir_relpath = os.path.relpath(path)
-        dir_size = len(dir_info)
-        bar = dir_size > LARGE_DIR_SIZE
 
-        logger.info("Linking directory '{}'.".format(dir_relpath))
+        logger.debug("Linking directory '{}'.".format(dir_relpath))
+
+        dir_info = self.load_dir_cache(checksum)
 
         for processed, entry in enumerate(dir_info):
             relpath = entry[self.PARAM_RELPATH]
@@ -321,21 +347,18 @@ class RemoteLOCAL(RemoteBase):
             entry_checksum_info = {self.PARAM_CHECKSUM: m}
 
             if self.changed(entry_info, entry_checksum_info):
-                if os.path.exists(p):
+                if self.exists(entry_info):
                     self.safe_remove(entry_info, force=force)
 
                 self.link(c, p)
 
-            if bar:
-                progress.update_target(dir_relpath, processed, dir_size)
+            if progress_callback:
+                progress_callback.update(os.path.relpath(p))
 
         self._discard_working_directory_changes(path, dir_info, force=force)
 
         self.state.update_link(path)
 
-        if bar:
-            progress.finish_target(dir_relpath)
-
     def already_cached(self, path_info):
         assert path_info["scheme"] in ["", "local"]
 
@@ -347,11 +370,7 @@ class RemoteLOCAL(RemoteBase):
         return not self.changed_cache(current_md5)
 
     def _discard_working_directory_changes(self, path, dir_info, force=False):
-        working_dir_files = set(
-            os.path.join(root, file)
-            for root, _, files in os.walk(str(path))
-            for file in files
-        )
+        working_dir_files = set(path for path in walk_files(path))
 
         cached_files = set(
             os.path.join(path, file["relpath"]) for file in dir_info
@@ -422,6 +441,7 @@ class RemoteLOCAL(RemoteBase):
 
         cache = self.get(md5)
         self.state.update(cache)
+        self.state.update(path, md5)
 
         if bar:
             progress.finish_target(dir_relpath)
@@ -484,10 +504,12 @@ class RemoteLOCAL(RemoteBase):
             if not name:
                 name = os.path.basename(from_info["path"])
 
-            self._makedirs(to_info["path"])
+            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
+            tmp_file = tmp_fname(to_info["path"])
 
             try:
-                copyfile(from_info["path"], to_info["path"], name=name)
+                copyfile(from_info["path"], tmp_file, name=name)
+                os.rename(tmp_file, to_info["path"])
             except Exception:
                 logger.error(
                     "failed to upload '{}' to '{}'".format(
@@ -521,8 +543,8 @@ class RemoteLOCAL(RemoteBase):
             if not name:
                 name = os.path.basename(to_info["path"])
 
-            self._makedirs(to_info["path"])
-            tmp_file = self.tmp_file(to_info["path"])
+            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
+            tmp_file = tmp_fname(to_info["path"])
             try:
                 copyfile(
                     from_info["path"],
@@ -530,6 +552,8 @@ class RemoteLOCAL(RemoteBase):
                     no_progress_bar=no_progress_bar,
                     name=name,
                 )
+
+                move(tmp_file, to_info["path"])
             except Exception:
                 logger.error(
                     "failed to download '{}' to '{}'".format(
@@ -539,8 +563,6 @@ class RemoteLOCAL(RemoteBase):
 
                 continue
 
-            os.rename(tmp_file, to_info["path"])
-
     def _group(self, checksum_infos, show_checksums=False):
         by_md5 = {}
 
@@ -587,13 +609,17 @@ class RemoteLOCAL(RemoteBase):
 
         progress.finish_target(title)
 
-        for md5, info in ret.items():
-            info["status"] = STATUS_MAP[
-                (md5 in local_exists, md5 in remote_exists)
-            ]
+        self._fill_statuses(ret, local_exists, remote_exists)
+
+        self._log_missing_caches(ret)
 
         return ret
 
+    def _fill_statuses(self, checksum_info_dir, local_exists, remote_exists):
+        for md5, info in checksum_info_dir.items():
+            status = STATUS_MAP[(md5 in local_exists, md5 in remote_exists)]
+            info["status"] = status
+
     def _get_chunks(self, download, remote, status_info, status, jobs):
         title = "Analysing status."
 
@@ -695,3 +721,79 @@ class RemoteLOCAL(RemoteBase):
             show_checksums=show_checksums,
             download=True,
         )
+
+    def _cache_metadata_changed(self):
+        mtime, size = get_mtime_and_size(self.cache_dir)
+        inode = get_inode(self.cache_dir)
+
+        existing_record = self.state.get_state_record_for_inode(inode)
+
+        if existing_record:
+            cached_mtime, cached_size, _, _ = existing_record
+            return not (mtime == cached_mtime and size == cached_size)
+
+        return True
+
+    def _log_missing_caches(self, checksum_info_dict):
+        missing_caches = [
+            (md5, info)
+            for md5, info in checksum_info_dict.items()
+            if info["status"] == STATUS_MISSING
+        ]
+        if missing_caches:
+            missing_desc = "".join(
+                [
+                    "\nname: {}, md5: {}".format(info["name"], md5)
+                    for md5, info in missing_caches
+                ]
+            )
+            msg = (
+                "Some of the cache files do not exist neither locally "
+                "nor on remote. Missing cache files: {}".format(missing_desc)
+            )
+            logger.warning(msg)
+
+    @staticmethod
+    def _unprotect_file(path):
+        if System.is_symlink(path) or System.is_hardlink(path):
+            logger.debug("Unprotecting '{}'".format(path))
+            tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))
+
+            # The operations order is important here - if some application
+            # would access the file during the process of copyfile then it
+            # would get only the part of file. So, at first, the file should be
+            # copied with the temporary name, and then original file should be
+            # replaced by new.
+            copyfile(
+                path,
+                tmp,
+                name="Unprotecting '{}'".format(os.path.relpath(path)),
+            )
+            remove(path)
+            os.rename(tmp, path)
+
+        else:
+            logger.debug(
+                "Skipping copying for '{}', since it is not "
+                "a symlink or a hardlink.".format(path)
+            )
+
+        os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
+
+    @staticmethod
+    def _unprotect_dir(path):
+        for path in walk_files(path):
+            RemoteLOCAL._unprotect_file(path)
+
+    @staticmethod
+    def unprotect(path_info):
+        path = path_info["path"]
+        if not os.path.exists(path):
+            raise DvcException(
+                "can't unprotect non-existing data '{}'".format(path)
+            )
+
+        if os.path.isdir(path):
+            RemoteLOCAL._unprotect_dir(path)
+        else:
+            RemoteLOCAL._unprotect_file(path)

dvc/remote/s3.py

@@ -2,20 +2,24 @@ from __future__ import unicode_literals
 
 import os
 import threading
+import logging
 
 try:
     import boto3
 except ImportError:
     boto3 = None
 
-import dvc.logger as logger
-from dvc.utils.compat import urlparse
+from dvc.utils import tmp_fname, move
+from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class Callback(object):
     def __init__(self, name, total):
         self.name = name
@@ -54,6 +58,8 @@ class RemoteS3(RemoteBase):
 
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
 
+        self.list_objects = config.get(Config.SECTION_AWS_LIST_OBJECTS)
+
         self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
 
         shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
@@ -109,16 +115,6 @@ class RemoteS3(RemoteBase):
         source = {"Bucket": from_info["bucket"], "Key": from_info["path"]}
         self.s3.copy(source, to_info["bucket"], to_info["path"])
 
-    def save(self, path_info, checksum_info):
-        if path_info["scheme"] != "s3":
-            raise NotImplementedError
-
-        etag = checksum_info[self.PARAM_CHECKSUM]
-        path = self.checksum_to_path(etag)
-        to_info = {"scheme": "s3", "bucket": self.bucket, "path": path}
-
-        self.copy(path_info, to_info)
-
     def remove(self, path_info):
         if path_info["scheme"] != "s3":
             raise NotImplementedError
@@ -134,27 +130,20 @@ class RemoteS3(RemoteBase):
         )
 
     def _list_paths(self, bucket, prefix):
+        """ Read config for list object api, paginate through list objects."""
         s3 = self.s3
-
         kwargs = {"Bucket": bucket, "Prefix": prefix}
-        while True:
-            # NOTE: list_objects_v2() is 90% faster than head_object [1]
-            #
-            # [1] https://www.peterbe.com/plog/
-            #     fastest-way-to-find-out-if-a-file-exists-in-s3
-            resp = s3.list_objects_v2(**kwargs)
-            contents = resp.get("Contents", None)
+        if self.list_objects:
+            list_objects_api = "list_objects"
+        else:
+            list_objects_api = "list_objects_v2"
+        paginator = s3.get_paginator(list_objects_api)
+        for page in paginator.paginate(**kwargs):
+            contents = page.get("Contents", None)
             if not contents:
-                break
-
-            for obj in contents:
-                yield obj["Key"]
-
-            token = resp.get("NextContinuationToken", None)
-            if not token:
-                break
-
-            kwargs["ContinuationToken"] = token
+                continue
+            for item in contents:
+                yield item["Key"]
 
     def list_cache_paths(self):
         return self._list_paths(self.bucket, self.prefix)
@@ -232,11 +221,11 @@ class RemoteS3(RemoteBase):
             )
             logger.debug(msg)
 
-            tmp_file = self.tmp_file(to_info["path"])
+            tmp_file = tmp_fname(to_info["path"])
             if not name:
                 name = os.path.basename(to_info["path"])
 
-            self._makedirs(to_info["path"])
+            makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
 
             try:
                 if no_progress_bar:
@@ -260,7 +249,7 @@ class RemoteS3(RemoteBase):
                 logger.error(msg)
                 continue
 
-            os.rename(tmp_file, to_info["path"])
+            move(tmp_file, to_info["path"])
 
             if not no_progress_bar:
                 progress.finish_target(name)

dvc/remote/ssh.py

@@ -1,388 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-import getpass
-import posixpath
-
-try:
-    import paramiko
-except ImportError:
-    paramiko = None
-
-import dvc.prompt as prompt
-import dvc.logger as logger
-
-from dvc.config import Config
-from dvc.progress import progress
-from dvc.utils.compat import urlparse
-from dvc.exceptions import DvcException
-from dvc.remote.base import RemoteBase, RemoteCmdError
-
-
-def sizeof_fmt(num, suffix="B"):
-    """ Convert number of bytes to human-readable string """
-    for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
-        if abs(num) < 1024.0:
-            return "%3.1f%s%s" % (num, unit, suffix)
-        num /= 1024.0
-    return "%.1f%s%s" % (num, "Y", suffix)
-
-
-def percent_cb(name, complete, total):
-    """ Callback for updating target progress """
-    logger.debug(
-        "{}: {} transferred out of {}".format(
-            name, sizeof_fmt(complete), sizeof_fmt(total)
-        )
-    )
-    progress.update_target(name, complete, total)
-
-
-def create_cb(name):
-    """ Create callback function for multipart object """
-    return lambda cur, tot: percent_cb(name, cur, tot)
-
-
-class RemoteSSH(RemoteBase):
-    scheme = "ssh"
-
-    # NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and
-    # SCP-like (ssh://[user@]host.xz:path) urls.
-    REGEX = r"^ssh://.*$"
-
-    REQUIRES = {"paramiko": paramiko}
-
-    JOBS = 4
-
-    PARAM_CHECKSUM = "md5"
-
-    DEFAULT_PORT = 22
-    TIMEOUT = 1800
-
-    def __init__(self, repo, config):
-        super(RemoteSSH, self).__init__(repo, config)
-        self.url = config.get(Config.SECTION_REMOTE_URL, "ssh://")
-
-        parsed = urlparse(self.url)
-        self.host = parsed.hostname
-        self.user = (
-            config.get(Config.SECTION_REMOTE_USER)
-            or parsed.username
-            or getpass.getuser()
-        )
-        self.prefix = parsed.path or "/"
-        self.port = (
-            config.get(Config.SECTION_REMOTE_PORT)
-            or parsed.port
-            or self.DEFAULT_PORT
-        )
-        self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
-        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
-        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
-        self.ask_password = config.get(
-            Config.SECTION_REMOTE_ASK_PASSWORD, False
-        )
-
-        self.path_info = {
-            "scheme": "ssh",
-            "host": self.host,
-            "user": self.user,
-            "port": self.port,
-        }
-
-    def ssh(self, host=None, user=None, port=None):
-        msg = (
-            "Establishing ssh connection with '{}' "
-            "through port '{}' as user '{}'"
-        )
-        logger.debug(msg.format(host, port, user))
-
-        ssh = paramiko.SSHClient()
-
-        ssh.load_system_host_keys()
-        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-        if self.ask_password and self.password is None:
-            msg = (
-                "Enter a private key passphrase or a password for "
-                "host '{}' port '{}' user '{}'"
-            ).format(host, port, user)
-            self.password = prompt.password(msg)
-
-        ssh.connect(
-            host,
-            username=user,
-            port=port,
-            key_filename=self.keyfile,
-            timeout=self.timeout,
-            password=self.password,
-        )
-
-        return ssh
-
-    def exists(self, path_info):
-        assert not isinstance(path_info, list)
-        assert path_info["scheme"] == "ssh"
-
-        with self.ssh(
-            path_info["host"], path_info["user"], path_info["port"]
-        ) as ssh:
-            try:
-                self._exec(ssh, "test -e {}".format(path_info["path"]))
-                exists = True
-            except RemoteCmdError:
-                exists = False
-
-        return exists
-
-    def _exec(self, ssh, cmd):
-        stdin, stdout, stderr = ssh.exec_command(cmd)
-        channel = stdout.channel
-
-        stdin.close()
-        channel.shutdown_write()
-
-        stdout_chunks = []
-        stderr_chunks = []
-        while (
-            not channel.closed
-            or channel.recv_ready()
-            or channel.recv_stderr_ready()
-        ):
-            import select
-
-            got_chunk = False
-            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
-            for c in readq:
-                if c.recv_ready():
-                    stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
-                    got_chunk = True
-
-                if c.recv_stderr_ready():
-                    stderr_len = len(c.in_stderr_buffer)
-                    s = stderr.channel.recv_stderr(stderr_len)
-                    stderr_chunks.append(s)
-                    got_chunk = True
-
-            if (
-                not got_chunk
-                and stdout.channel.exit_status_ready()
-                and not stderr.channel.recv_stderr_ready()
-                and not stdout.channel.recv_ready()
-            ):
-                stdout.channel.shutdown_read()
-                stdout.channel.close()
-                break
-
-        stdout.close()
-        stderr.close()
-
-        ret = stdout.channel.recv_exit_status()
-        if ret != 0:
-            err = b"".join(stderr_chunks).decode("utf-8")
-            raise RemoteCmdError(self.scheme, cmd, ret, err)
-
-        return b"".join(stdout_chunks).decode("utf-8")
-
-    def md5(self, path_info):
-        if path_info["scheme"] != "ssh":
-            raise NotImplementedError
-
-        ssh = self.ssh(
-            host=path_info["host"],
-            user=path_info["user"],
-            port=path_info["port"],
-        )
-
-        # Use different md5 commands depending on os
-        stdout = self._exec(ssh, "uname").strip()
-        if stdout == "Darwin":
-            md5cmd = "md5"
-            index = -1
-        elif stdout == "Linux":
-            md5cmd = "md5sum"
-            index = 0
-        else:
-            msg = "'{}' is not supported as a remote".format(stdout)
-            raise DvcException(msg)
-
-        stdout = self._exec(ssh, "{} {}".format(md5cmd, path_info["path"]))
-        md5 = stdout.split()[index]
-        ssh.close()
-
-        assert len(md5) == 32
-
-        return md5
-
-    def copy(self, from_info, to_info, ssh=None):
-        if from_info["scheme"] != "ssh" or to_info["scheme"] != "ssh":
-            raise NotImplementedError
-
-        assert from_info["host"] == to_info["host"]
-        assert from_info["user"] == to_info["user"]
-
-        s = (
-            ssh
-            if ssh
-            else self.ssh(
-                host=from_info["host"],
-                user=from_info["user"],
-                port=from_info["port"],
-            )
-        )
-
-        dname = posixpath.dirname(to_info["path"])
-        self._exec(s, "mkdir -p {}".format(dname))
-        self._exec(s, "cp {} {}".format(from_info["path"], to_info["path"]))
-
-        if not ssh:
-            s.close()
-
-    def save_info(self, path_info):
-        if path_info["scheme"] != "ssh":
-            raise NotImplementedError
-
-        return {self.PARAM_CHECKSUM: self.md5(path_info)}
-
-    def save(self, path_info, checksum_info):
-        if path_info["scheme"] != "ssh":
-            raise NotImplementedError
-
-        md5 = checksum_info[self.PARAM_CHECKSUM]
-        dest = path_info.copy()
-        dest["path"] = self.checksum_to_path(md5)
-
-        self.copy(path_info, dest)
-
-    def remove(self, path_info):
-        if path_info["scheme"] != "ssh":
-            raise NotImplementedError
-
-        logger.debug(
-            "Removing ssh://{}@{}/{}".format(
-                path_info["user"], path_info["host"], path_info["path"]
-            )
-        )
-
-        ssh = self.ssh(
-            host=path_info["host"],
-            user=path_info["user"],
-            port=path_info["port"],
-        )
-        ssh.open_sftp().remove(path_info["path"])
-        ssh.close()
-
-    def download(
-        self,
-        from_infos,
-        to_infos,
-        no_progress_bar=False,
-        names=None,
-        resume=False,
-    ):
-        names = self._verify_path_args(from_infos, to_infos, names)
-
-        ssh = self.ssh(
-            host=from_infos[0]["host"],
-            user=from_infos[0]["user"],
-            port=from_infos[0]["port"],
-        )
-
-        for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info["scheme"] != "ssh":
-                raise NotImplementedError
-
-            if to_info["scheme"] == "ssh":
-                assert from_info["host"] == to_info["host"]
-                assert from_info["port"] == to_info["port"]
-                assert from_info["user"] == to_info["user"]
-                self.copy(from_info, to_info, ssh=ssh)
-                continue
-
-            if to_info["scheme"] != "local":
-                raise NotImplementedError
-
-            msg = "Downloading '{}/{}' to '{}'".format(
-                from_info["host"], from_info["path"], to_info["path"]
-            )
-            logger.debug(msg)
-
-            if not name:
-                name = os.path.basename(to_info["path"])
-
-            self._makedirs(to_info["path"])
-            tmp_file = self.tmp_file(to_info["path"])
-            try:
-                ssh.open_sftp().get(
-                    from_info["path"], tmp_file, callback=create_cb(name)
-                )
-            except Exception:
-                msg = "failed to download '{}/{}' to '{}'"
-                logger.error(
-                    msg.format(
-                        from_info["host"], from_info["path"], to_info["path"]
-                    )
-                )
-                continue
-
-            os.rename(tmp_file, to_info["path"])
-            progress.finish_target(name)
-
-        ssh.close()
-
-    def upload(self, from_infos, to_infos, names=None):
-        names = self._verify_path_args(to_infos, from_infos, names)
-
-        ssh = self.ssh(
-            host=to_infos[0]["host"],
-            user=to_infos[0]["user"],
-            port=to_infos[0]["port"],
-        )
-        sftp = ssh.open_sftp()
-
-        for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info["scheme"] != "ssh":
-                raise NotImplementedError
-
-            if from_info["scheme"] != "local":
-                raise NotImplementedError
-
-            logger.debug(
-                "Uploading '{}' to '{}/{}'".format(
-                    from_info["path"], to_info["host"], to_info["path"]
-                )
-            )
-
-            if not name:
-                name = os.path.basename(from_info["path"])
-
-            dname = posixpath.dirname(to_info["path"])
-            self._exec(ssh, "mkdir -p {}".format(dname))
-
-            try:
-                sftp.put(
-                    from_info["path"],
-                    to_info["path"],
-                    callback=create_cb(name),
-                )
-            except Exception:
-                msg = "failed to upload '{}' to '{}/{}'"
-                logger.error(
-                    msg.format(
-                        from_info["path"], to_info["host"], to_info["path"]
-                    )
-                )
-                continue
-
-            progress.finish_target(name)
-
-        sftp.close()
-        ssh.close()
-
-    def list_cache_paths(self):
-        ssh = self.ssh(host=self.host, user=self.user, port=self.port)
-        cmd = "find {} -type f -follow -print".format(self.prefix)
-        stdout = self._exec(ssh, cmd)
-        flist = stdout.split()
-        ssh.close()
-        return flist

dvc/remote/ssh/__init__.py

@@ -0,0 +1,201 @@
+from __future__ import unicode_literals
+
+import getpass
+import logging
+
+try:
+    import paramiko
+except ImportError:
+    paramiko = None
+
+import dvc.prompt as prompt
+from dvc.remote.ssh.connection import SSHConnection
+from dvc.config import Config
+from dvc.utils.compat import urlparse
+from dvc.remote.base import RemoteBase
+
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteSSH(RemoteBase):
+    scheme = "ssh"
+
+    # NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and
+    # SCP-like (ssh://[user@]host.xz:/absolute/path) urls.
+    REGEX = r"^ssh://.*$"
+
+    REQUIRES = {"paramiko": paramiko}
+
+    JOBS = 4
+    PARAM_CHECKSUM = "md5"
+    DEFAULT_PORT = 22
+    TIMEOUT = 1800
+
+    def __init__(self, repo, config):
+        super(RemoteSSH, self).__init__(repo, config)
+        self.url = config.get(Config.SECTION_REMOTE_URL, "ssh://")
+
+        parsed = urlparse(self.url)
+        self.host = parsed.hostname
+        self.user = (
+            config.get(Config.SECTION_REMOTE_USER)
+            or parsed.username
+            or getpass.getuser()
+        )
+        self.prefix = parsed.path or "/"
+        self.port = (
+            config.get(Config.SECTION_REMOTE_PORT)
+            or parsed.port
+            or self.DEFAULT_PORT
+        )
+        self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
+        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
+        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
+        self.ask_password = config.get(
+            Config.SECTION_REMOTE_ASK_PASSWORD, False
+        )
+
+        self.path_info = {
+            "scheme": "ssh",
+            "host": self.host,
+            "user": self.user,
+            "port": self.port,
+        }
+
+    def ssh(self, host=None, user=None, port=None, **kwargs):
+        logger.debug(
+            "Establishing ssh connection with '{host}' "
+            "through port '{port}' as user '{user}'".format(
+                host=host, user=user, port=port
+            )
+        )
+
+        if self.ask_password and not self.password:
+            self.password = prompt.password(
+                "Enter a private key passphrase or a password for "
+                "host '{host}' port '{port}' user '{user}'".format(
+                    host=host, port=port, user=user
+                )
+            )
+
+        return SSHConnection(
+            host,
+            username=user,
+            port=port,
+            key_filename=self.keyfile,
+            timeout=self.timeout,
+            password=self.password,
+        )
+
+    def exists(self, path_info):
+        assert not isinstance(path_info, list)
+        assert path_info["scheme"] == self.scheme
+
+        with self.ssh(**path_info) as ssh:
+            return ssh.file_exists(path_info["path"])
+
+    def md5(self, path_info):
+        if path_info["scheme"] != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(**path_info) as ssh:
+            return ssh.md5(path_info["path"])
+
+    def copy(self, from_info, to_info):
+        if (
+            from_info["scheme"] != self.scheme
+            or to_info["scheme"] != self.scheme
+        ):
+            raise NotImplementedError
+
+        with self.ssh(**from_info) as ssh:
+            ssh.cp(from_info["path"], to_info["path"])
+
+    def save_info(self, path_info):
+        if path_info["scheme"] != self.scheme:
+            raise NotImplementedError
+
+        return {self.PARAM_CHECKSUM: self.md5(path_info)}
+
+    def remove(self, path_info):
+        if path_info["scheme"] != self.scheme:
+            raise NotImplementedError
+
+        with self.ssh(**path_info) as ssh:
+            ssh.remove(path_info["path"])
+
+    def download(
+        self,
+        from_infos,
+        to_infos,
+        no_progress_bar=False,
+        names=None,
+        resume=False,
+    ):
+        names = self._verify_path_args(from_infos, to_infos, names)
+        ssh = self.ssh(**from_infos[0])
+
+        for to_info, from_info, name in zip(to_infos, from_infos, names):
+            if from_info["scheme"] != self.scheme:
+                raise NotImplementedError
+
+            if to_info["scheme"] == self.scheme:
+                ssh.cp(from_info["path"], to_info["path"])
+                continue
+
+            if to_info["scheme"] != "local":
+                raise NotImplementedError
+
+            logger.debug(
+                "Downloading '{host}/{path}' to '{dest}'".format(
+                    host=from_info["host"],
+                    path=from_info["path"],
+                    dest=to_info["path"],
+                )
+            )
+
+            try:
+                ssh.download(
+                    from_info["path"], to_info["path"], progress_title=name
+                )
+            except Exception:
+                logger.error(
+                    "failed to download '{host}/{path}' to '{dest}'".format(
+                        host=from_info["host"],
+                        path=from_info["path"],
+                        dest=to_info["path"],
+                    )
+                )
+                continue
+
+        ssh.close()
+
+    def upload(self, from_infos, to_infos, names=None):
+        names = self._verify_path_args(to_infos, from_infos, names)
+
+        with self.ssh(**to_infos[0]) as ssh:
+            for from_info, to_info, name in zip(from_infos, to_infos, names):
+                if to_info["scheme"] != self.scheme:
+                    raise NotImplementedError
+
+                if from_info["scheme"] != "local":
+                    raise NotImplementedError
+
+                try:
+                    ssh.upload(
+                        from_info["path"], to_info["path"], progress_title=name
+                    )
+                except Exception:
+                    logger.exception(
+                        "failed to upload '{host}/{path}' to '{dest}'".format(
+                            host=from_info["host"],
+                            path=from_info["path"],
+                            dest=to_info["path"],
+                        )
+                    )
+                    pass
+
+    def list_cache_paths(self):
+        with self.ssh(**self.path_info) as ssh:
+            return list(ssh.walk_files(self.prefix))

dvc/remote/ssh/connection.py

@@ -0,0 +1,280 @@
+import errno
+import os
+import posixpath
+import logging
+
+try:
+    import paramiko
+except ImportError:
+    paramiko = None
+
+from dvc.utils import tmp_fname
+from dvc.utils.compat import makedirs
+from dvc.progress import progress
+from dvc.exceptions import DvcException
+from dvc.remote.base import RemoteCmdError
+
+
+logger = logging.getLogger(__name__)
+
+
+def sizeof_fmt(num, suffix="B"):
+    """ Convert number of bytes to human-readable string """
+    for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, "Y", suffix)
+
+
+def percent_cb(name, complete, total):
+    """ Callback for updating target progress """
+    logger.debug(
+        "{}: {} transferred out of {}".format(
+            name, sizeof_fmt(complete), sizeof_fmt(total)
+        )
+    )
+    progress.update_target(name, complete, total)
+
+
+def create_cb(name):
+    """ Create callback function for multipart object """
+    return lambda cur, tot: percent_cb(name, cur, tot)
+
+
+class SSHConnection:
+    def __init__(self, *args, **kwargs):
+        self.timeout = kwargs.get("timeout", 1800)
+
+        self._ssh = paramiko.SSHClient()
+        self._ssh.load_system_host_keys()
+        self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        self._ssh.connect(*args, **kwargs)
+        self._sftp = None
+        self._sftp_alive = False
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        self.close()
+
+    def _sftp_connect(self):
+        if not self._sftp or not self._sftp_alive:
+            self._sftp = self._ssh.open_sftp()
+            self._sftp_alive = True
+
+    def close(self):
+        if self._sftp:
+            self._sftp.close()
+            self._sftp_alive = False
+
+        self._ssh.close()
+
+    def file_exists(self, path):
+        self._sftp_connect()
+        try:
+            return self._sftp.stat(path)
+        except IOError:
+            return False
+        pass
+
+    def isdir(self, path):
+        from stat import S_ISDIR
+
+        self._sftp_connect()
+        try:
+            return S_ISDIR(self._sftp.stat(path).st_mode)
+        except IOError:
+            return False
+
+    def isfile(self, path):
+        from stat import S_ISREG
+
+        self._sftp_connect()
+        try:
+            return S_ISREG(self._sftp.stat(path).st_mode)
+        except IOError:
+            return False
+
+    def islink(self, path):
+        from stat import S_ISLNK
+
+        self._sftp_connect()
+        try:
+            return S_ISLNK(self._sftp.stat(path).st_mode)
+        except IOError:
+            return False
+
+    def makedirs(self, path):
+        self._sftp_connect()
+
+        if self.isdir(path):
+            return
+
+        if self.isfile(path) or self.islink(path):
+            raise DvcException(
+                "a file with the same name '{}' already exists".format(path)
+            )
+
+        head, tail = os.path.split(path)
+
+        if head:
+            self.makedirs(head)
+
+        if tail:
+            self._sftp.mkdir(path)
+
+    def walk_files(self, directory):
+        from stat import S_ISLNK, S_ISDIR, S_ISREG
+
+        self._sftp_connect()
+
+        try:
+            dir_entries = self._sftp.listdir_attr(directory)
+        except IOError as exc:
+            raise DvcException(
+                "couldn't get the '{}' remote directory files list".format(
+                    directory
+                ),
+                cause=exc,
+            )
+
+        for entry in dir_entries:
+            path = posixpath.join(directory, entry.filename)
+
+            if S_ISLNK(entry.st_mode):
+                path = self._sftp.readlink(directory)
+                entry = self._sftp.stat(path)
+
+            if S_ISDIR(entry.st_mode):
+                for inner_path in self.walk_files(path):
+                    yield inner_path
+
+            elif S_ISREG(entry.st_mode):
+                yield path
+
+    def remove(self, path):
+        self._sftp_connect()
+        try:
+            self._sftp.remove(path)
+        except IOError as exc:
+            if exc.errno != errno.ENOENT:
+                raise
+
+    def download(self, src, dest, no_progress_bar=False, progress_title=None):
+        self._sftp_connect()
+
+        makedirs(os.path.dirname(dest), exist_ok=True)
+        tmp_file = tmp_fname(dest)
+
+        if no_progress_bar:
+            self._sftp.get(src, tmp_file)
+        else:
+            if not progress_title:
+                progress_title = os.path.basename(dest)
+
+            self._sftp.get(src, tmp_file, callback=create_cb(progress_title))
+            progress.finish_target(progress_title)
+
+        if os.path.exists(dest):
+            os.remove(dest)
+
+        os.rename(tmp_file, dest)
+
+    def upload(self, src, dest, no_progress_bar=False, progress_title=None):
+        self._sftp_connect()
+
+        self.makedirs(posixpath.dirname(dest))
+        tmp_file = tmp_fname(dest)
+
+        if no_progress_bar:
+            self._sftp.put(src, tmp_file)
+        else:
+            if not progress_title:
+                progress_title = posixpath.basename(dest)
+
+            self._sftp.put(src, tmp_file, callback=create_cb(progress_title))
+            progress.finish_target(progress_title)
+
+        self._sftp.rename(tmp_file, dest)
+
+    def execute(self, cmd):
+        stdin, stdout, stderr = self._ssh.exec_command(cmd)
+        channel = stdout.channel
+
+        stdin.close()
+        channel.shutdown_write()
+
+        stdout_chunks = []
+        stderr_chunks = []
+        while (
+            not channel.closed
+            or channel.recv_ready()
+            or channel.recv_stderr_ready()
+        ):
+            import select
+
+            got_chunk = False
+            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
+            for c in readq:
+                if c.recv_ready():
+                    stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
+                    got_chunk = True
+
+                if c.recv_stderr_ready():
+                    stderr_len = len(c.in_stderr_buffer)
+                    s = stderr.channel.recv_stderr(stderr_len)
+                    stderr_chunks.append(s)
+                    got_chunk = True
+
+            if (
+                not got_chunk
+                and stdout.channel.exit_status_ready()
+                and not stderr.channel.recv_stderr_ready()
+                and not stdout.channel.recv_ready()
+            ):
+                stdout.channel.shutdown_read()
+                stdout.channel.close()
+                break
+
+        stdout.close()
+        stderr.close()
+
+        ret = stdout.channel.recv_exit_status()
+        if ret != 0:
+            err = b"".join(stderr_chunks).decode("utf-8")
+            raise RemoteCmdError("ssh", cmd, ret, err)
+
+        return b"".join(stdout_chunks).decode("utf-8")
+
+    def md5(self, path):
+        """
+        Use different md5 commands depending on the OS:
+
+         - Darwin's `md5` returns BSD-style checksums by default
+         - Linux's `md5sum` needs the `--tag` flag for a similar output
+
+         Example:
+              MD5 (foo.txt) = f3d220a856b52aabbf294351e8a24300
+        """
+        uname = self.execute("uname").strip()
+
+        command = {
+            "Darwin": "md5 {}".format(path),
+            "Linux": "md5sum --tag {}".format(path),
+        }.get(uname)
+
+        if not command:
+            raise DvcException(
+                "'{uname}' is not supported as a remote".format(uname=uname)
+            )
+
+        md5 = self.execute(command).split()[-1]
+        assert len(md5) == 32
+        return md5
+
+    def cp(self, src, dest):
+        self.makedirs(posixpath.dirname(dest))
+        self.execute("cp {} {}".format(src, dest))

dvc/repo/__init__.py

@@ -1,13 +1,19 @@
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str, builtin_str, open
-
-import collections
 import os
-import dvc.prompt as prompt
-import dvc.logger as logger
+import logging
 
-from dvc.exceptions import DvcException, NotDvcRepoError, OutputNotFoundError
+import dvc.prompt as prompt
+
+from dvc.exceptions import (
+    DvcException,
+    NotDvcRepoError,
+    OutputNotFoundError,
+    TargetNotDirectoryError,
+)
+
+
+logger = logging.getLogger(__name__)
 
 
 class Repo(object):
@@ -29,6 +35,9 @@ class Repo(object):
     from dvc.repo.status import status
     from dvc.repo.gc import gc
     from dvc.repo.commit import commit
+    from dvc.repo.pkg import install_pkg
+    from dvc.repo.diff import diff
+    from dvc.repo.brancher import brancher
 
     def __init__(self, root_dir=None):
         from dvc.config import Config
@@ -39,6 +48,8 @@ class Repo(object):
         from dvc.data_cloud import DataCloud
         from dvc.updater import Updater
         from dvc.repo.metrics import Metrics
+        from dvc.scm.tree import WorkingTree
+        from dvc.repo.tag import Tag
 
         root_dir = self.find_root(root_dir)
 
@@ -47,6 +58,8 @@ class Repo(object):
 
         self.config = Config(self.dvc_dir)
 
+        self.tree = WorkingTree()
+
         self.scm = SCM(self.root_dir, repo=self)
         self.lock = Lock(self.dvc_dir)
         # NOTE: storing state and link_state in the repository itself to avoid
@@ -55,15 +68,18 @@ class Repo(object):
 
         core = self.config.config[Config.SECTION_CORE]
 
-        logger.set_level(core.get(Config.SECTION_CORE_LOGLEVEL))
+        logger.setLevel(
+            logging.getLevelName(
+                core.get(Config.SECTION_CORE_LOGLEVEL, "info").upper()
+            )
+        )
 
         self.cache = Cache(self)
         self.cloud = DataCloud(self, config=self.config.config)
         self.updater = Updater(self.dvc_dir)
 
         self.metrics = Metrics(self)
-
-        self.files_to_git_add = []
+        self.tag = Tag(self)
 
         self._ignore()
 
@@ -93,17 +109,6 @@ class Repo(object):
         root_dir = Repo.find_root(root)
         return os.path.join(root_dir, Repo.DVC_DIR)
 
-    def remind_to_git_add(self):
-        if not self.files_to_git_add:
-            return
-
-        logger.info(
-            "\n"
-            "To track the changes with git run:\n"
-            "\n"
-            "\tgit add {files}".format(files=" ".join(self.files_to_git_add))
-        )
-
     @staticmethod
     def init(root_dir=os.curdir, no_scm=False, force=False):
         from dvc.repo.init import init
@@ -111,11 +116,9 @@ class Repo(object):
         init(root_dir=root_dir, no_scm=no_scm, force=force)
         return Repo(root_dir)
 
-    @staticmethod
-    def unprotect(target):
-        from dvc.repo.unprotect import unprotect
-
-        return unprotect(target)
+    def unprotect(self, target):
+        path_info = {"schema": "local", "path": target}
+        return self.cache.local.unprotect(path_info)
 
     def _ignore(self):
         flist = [
@@ -146,25 +149,27 @@ class Repo(object):
             raise CyclicGraphError(cycles[0])
 
     def _get_pipeline(self, node):
-        pipelines = list(filter(lambda g: node in g.nodes(), self.pipelines()))
+        pipelines = [i for i in self.pipelines() if i.has_node(node)]
         assert len(pipelines) == 1
         return pipelines[0]
 
-    def collect(self, target, with_deps=False):
+    def collect(self, target, with_deps=False, recursive=False):
         import networkx as nx
         from dvc.stage import Stage
 
+        if not target or recursive:
+            return self.active_stages(target)
+
         stage = Stage.load(self, target)
         if not with_deps:
             return [stage]
 
         node = os.path.relpath(stage.path, self.root_dir)
         G = self._get_pipeline(node)
-        stages = nx.get_node_attributes(G, "stage")
 
-        ret = [stage]
+        ret = []
         for n in nx.dfs_postorder_nodes(G, node):
-            ret.append(stages[n])
+            ret.append(G.node[n]["stage"])
 
         return ret
 
@@ -257,7 +262,7 @@ class Repo(object):
         cache["ssh"] = []
         cache["azure"] = []
 
-        for branch in self.scm.brancher(
+        for branch in self.brancher(
             all_branches=all_branches, all_tags=all_tags
         ):
             if target:
@@ -295,7 +300,8 @@ class Repo(object):
         import networkx as nx
         from dvc.exceptions import (
             OutputDuplicationError,
-            WorkingDirectoryAsOutputError,
+            StagePathAsOutputError,
+            OverlappingOutputPathsError,
         )
 
         G = nx.DiGraph()
@@ -306,7 +312,15 @@ class Repo(object):
 
         for stage in stages:
             for out in stage.outs:
-                existing = [o.stage for o in outs if o.path == out.path]
+                existing = []
+                for o in outs:
+                    if o.path == out.path:
+                        existing.append(o.stage)
+
+                    in_o_dir = out.path.startswith(o.path + o.sep)
+                    in_out_dir = o.path.startswith(out.path + out.sep)
+                    if in_o_dir or in_out_dir:
+                        raise OverlappingOutputPathsError(o, out)
 
                 if existing:
                     stages = [stage.relpath, existing[0].relpath]
@@ -315,15 +329,10 @@ class Repo(object):
                 outs.append(out)
 
         for stage in stages:
+            path_dir = os.path.dirname(stage.path) + os.sep
             for out in outs:
-                overlaps = stage.cwd == out.path or stage.cwd.startswith(
-                    out.path + os.sep
-                )
-
-                if overlaps:
-                    raise WorkingDirectoryAsOutputError(
-                        stage.cwd, stage.relpath
-                    )
+                if path_dir.startswith(out.path + os.sep):
+                    raise StagePathAsOutputError(stage.wdir, stage.relpath)
 
         # collect the whole DAG
         for stage in stages:
@@ -376,13 +385,15 @@ class Repo(object):
 
         if not from_directory:
             from_directory = self.root_dir
+        elif not os.path.isdir(from_directory):
+            raise TargetNotDirectoryError(from_directory)
 
         stages = []
         outs = []
-        for root, dirs, files in os.walk(str(from_directory)):
+        for root, dirs, files in self.tree.walk(from_directory):
             for fname in files:
                 path = os.path.join(root, fname)
-                if not Stage.is_stage_file(path):
+                if not Stage.is_valid_filename(path):
                     continue
                 stage = Stage.load(self, path)
                 for out in stage.outs:
@@ -415,10 +426,14 @@ class Repo(object):
 
     def find_outs_by_path(self, path, outs=None, recursive=False):
         if not outs:
-            outs = [out for stage in self.stages() for out in stage.outs]
+            # there is no `from_directory=path` argument because some data
+            # files might be generated to an upper level, and so it is
+            # needed to look at all the files (project root_dir)
+            stages = self.stages()
+            outs = [out for stage in stages for out in stage.outs]
 
         abs_path = os.path.abspath(path)
-        is_dir = os.path.isdir(abs_path)
+        is_dir = self.tree.isdir(abs_path)
 
         def func(out):
             if out.path == abs_path:
@@ -434,3 +449,7 @@ class Repo(object):
             raise OutputNotFoundError(path)
 
         return matched
+
+    def is_dvc_internal(self, path):
+        path_parts = os.path.normpath(path).split(os.path.sep)
+        return self.DVC_DIR in path_parts

dvc/repo/add.py

@@ -1,45 +1,55 @@
 import os
 
+from dvc.repo.scm_context import scm_context
+from dvc.stage import Stage
+from dvc.utils import walk_files
+from dvc.exceptions import RecursiveAddingWhileUsingFilename
 
-def add(self, fname, recursive=False, no_commit=False):
-    from dvc.stage import Stage
 
-    fnames = []
-    if recursive and os.path.isdir(fname):
-        for root, dirs, files in os.walk(str(fname)):
-            for f in files:
-                path = os.path.join(root, f)
-                if Stage.is_stage_file(path):
-                    continue
-                if os.path.basename(path) == self.scm.ignore_file:
-                    continue
-                if self.scm.is_tracked(path):
-                    continue
-                fnames.append(path)
-    else:
-        fnames = [fname]
+@scm_context
+def add(repo, target, recursive=False, no_commit=False, fname=None):
+    if recursive and fname:
+        raise RecursiveAddingWhileUsingFilename()
 
+    targets = _find_all_targets(repo, target, recursive)
+
+    stages = _create_stages(repo, targets, fname, no_commit)
+
+    repo.check_dag(repo.stages() + stages)
+
+    for stage in stages:
+        stage.dump()
+    return stages
+
+
+def _find_all_targets(repo, target, recursive):
+    if os.path.isdir(target) and recursive:
+        return [
+            file
+            for file in walk_files(target)
+            if not repo.is_dvc_internal(file)
+            if not Stage.is_stage_file(file)
+            if not repo.scm.belongs_to_scm(file)
+            if not repo.scm.is_tracked(file)
+        ]
+    return [target]
+
+
+def _create_stages(repo, targets, fname, no_commit):
     stages = []
-    self.files_to_git_add = []
-    with self.state:
-        for f in fnames:
-            stage = Stage.create(repo=self, outs=[f], add=True)
 
-            if stage is None:
-                stages.append(stage)
+    with repo.state:
+        for out in targets:
+            stage = Stage.create(repo=repo, outs=[out], add=True, fname=fname)
+
+            if not stage:
                 continue
 
             stage.save()
+
             if not no_commit:
                 stage.commit()
+
             stages.append(stage)
 
-    self.check_dag(self.stages() + stages)
-
-    for stage in stages:
-        if stage is not None:
-            stage.dump()
-
-    self.remind_to_git_add()
-
     return stages

dvc/repo/brancher.py

@@ -0,0 +1,56 @@
+def brancher(  # noqa: E302
+    self, branches=None, all_branches=False, tags=None, all_tags=False
+):
+    """Generator that iterates over specified revisions.
+
+    Args:
+        branches (list): a list of branches to iterate over.
+        all_branches (bool): iterate over all available branches.
+        tags (list): a list of tags to iterate over.
+        all_tags (bool): iterate over all available tags.
+
+    Yields:
+        str: the display name for the currently selected tree, it could be:
+            - a git revision identifier
+            - empty string it there is no branches to iterate over
+            - "Working Tree" if there are uncommited changes in the SCM repo
+    """
+    if not any([branches, all_branches, tags, all_tags]):
+        yield ""
+        return
+
+    saved_tree = self.tree
+    revs = []
+
+    scm = self.scm
+
+    if self.scm.is_dirty():
+        from dvc.scm.tree import WorkingTree
+
+        self.tree = WorkingTree()
+        yield "Working Tree"
+
+    if all_branches:
+        branches = scm.list_branches()
+
+    if all_tags:
+        tags = scm.list_tags()
+
+    if branches is None:
+        revs.extend([scm.active_branch()])
+    else:
+        revs.extend(branches)
+
+    if tags is not None:
+        revs.extend(tags)
+
+    # NOTE: it might be a good idea to wrap this loop in try/finally block
+    # to don't leave the tree on some unexpected branch after the
+    # `brancher()`, but this could cause problems on exception handling
+    # code which might expect the tree on which exception was raised to
+    # stay in place. This behavior is a subject to change.
+    for rev in revs:
+        self.tree = scm.get_tree(rev)
+        yield rev
+
+    self.tree = saved_tree

dvc/repo/checkout.py

@@ -1,7 +1,12 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
-from dvc.exceptions import DvcException
+import logging
+
+from dvc.exceptions import CheckoutErrorSuggestGit
+from dvc.progress import ProgressCallback
+
+
+logger = logging.getLogger(__name__)
 
 
 def _cleanup_unused_links(self, all_stages):
@@ -12,23 +17,32 @@ def _cleanup_unused_links(self, all_stages):
     self.state.remove_unused_links(used)
 
 
+def get_all_files_numbers(stages):
+    return sum(stage.get_all_files_number() for stage in stages)
+
+
+def get_progress_callback(stages):
+    total_files_num = get_all_files_numbers(stages)
+    if total_files_num == 0:
+        return None
+    return ProgressCallback(total_files_num)
+
+
 def checkout(self, target=None, with_deps=False, force=False, recursive=False):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    if target and not recursive:
-        all_stages = self.active_stages()
-        try:
-            stages = self.collect(target, with_deps=with_deps)
-        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
-            raise DvcException(
-                str(exc) + " Did you mean 'git checkout {}'?".format(target)
-            )
-    else:
-        all_stages = self.active_stages(target)
-        stages = all_stages
+    all_stages = self.stages()
+
+    try:
+        stages = self.collect(target, with_deps=with_deps, recursive=recursive)
+    except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+        if not target:
+            raise
+        raise CheckoutErrorSuggestGit(target, exc)
 
     with self.state:
         _cleanup_unused_links(self, all_stages)
+        progress_callback = get_progress_callback(stages)
 
         for stage in stages:
             if stage.locked:
@@ -37,4 +51,6 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False):
                     " not going to be checked out.".format(path=stage.relpath)
                 )
 
-            stage.checkout(force=force)
+            stage.checkout(force=force, progress_callback=progress_callback)
+        if progress_callback:
+            progress_callback.finish("Checkout finished!")

dvc/repo/commit.py

@@ -1,9 +1,5 @@
 def commit(self, target, with_deps=False, recursive=False, force=False):
-    if target and not recursive:
-        stages = self.collect(target, with_deps=with_deps)
-    else:
-        stages = self.active_stages(target)
-
+    stages = self.collect(target, with_deps=with_deps, recursive=recursive)
     with self.state:
         for stage in stages:
             stage.check_can_commit(force=force)

dvc/repo/diff.py

@@ -0,0 +1,252 @@
+from __future__ import unicode_literals
+
+import os
+from errno import ENOENT
+
+
+import dvc.logger as logger
+from dvc.scm.base import FileNotInCommitError
+from dvc.scm.git import (
+    DIFF_A_TREE,
+    DIFF_B_TREE,
+    DIFF_A_REF,
+    DIFF_B_REF,
+    DIFF_EQUAL,
+)
+
+
+DIFF_TARGET = "target"
+DIFF_IS_DIR = "is_dir"
+DIFF_OLD_FILE = "old_file"
+DIFF_OLD_CHECKSUM = "old_checksum"
+DIFF_NEW_FILE = "new_file"
+DIFF_NEW_CHECKSUM = "new_checksum"
+DIFF_SIZE = "size_diff"
+DIFF_DEL = "del"
+DIFF_IDENT = "ident"
+DIFF_CHANGE = "changes"
+DIFF_NEW = "new"
+DIFF_MOVE = "moves"
+DIFF_LIST = "diffs"
+DIFF_SIZE_UNKNOWN = "?"
+DIFF_A_OUTPUT = "a_output"
+DIFF_B_OUTPUT = "b_output"
+DIFF_DELETED = "deleted_file"
+DIFF_IS_NEW = "created_file"
+
+
+def _file_not_exists(error, result):
+    if error.errno == ENOENT:
+        result.update({DIFF_SIZE: DIFF_SIZE_UNKNOWN})
+    else:
+        raise error
+
+
+def _extract_dir(self, dir_not_exists, output):
+    """Extract the content of dvc tree file
+    Args:
+        self(object) - Repo class instance
+        dir_not_exists(bool) - flag for directory existence
+        output(object) - OutputLOCAL class instance
+    Returns:
+        dict - dictionary with keys - paths to file in .dvc/cache
+                               values -checksums for that files
+    """
+    if not dir_not_exists:
+        lst = output.dir_cache
+        return {i["relpath"]: i["md5"] for i in lst}
+    return {}
+
+
+def _get_dir_info(dir_not_exists, a_output):
+    if not dir_not_exists:
+        return str(a_output), a_output.checksum
+    return "", ""
+
+
+def _ident_files(a_entries, b_entries):
+    keys = [
+        key for key in a_entries.keys() if b_entries.get(key) == a_entries[key]
+    ]
+    return len(keys)
+
+
+def _modified_files(self, a_entries, b_entries):
+    keys = [key for key in a_entries.keys() if key in b_entries]
+    diff_size = 0
+    modified_count = 0
+    for key in keys:
+        if a_entries[key] != b_entries[key]:
+            modified_count
+            diff_size += os.path.getsize(
+                self.cache.local.get(b_entries[key])
+            ) - os.path.getsize(self.cache.local.get(a_entries[key]))
+    return modified_count, diff_size
+
+
+def _deleted_files(self, a_entries, b_entries):
+    diff_size = 0
+    deleted_count = 0
+    for key, value in a_entries.items():
+        if key not in b_entries:
+            deleted_count += 1
+            diff_size -= os.path.getsize(self.cache.local.get(a_entries[key]))
+    return deleted_count, diff_size
+
+
+def _new_files(self, a_entries, b_entries):
+    diff_size = 0
+    new_count = 0
+    for key, value in b_entries.items():
+        if key not in a_entries:
+            new_count += 1
+            diff_size += os.path.getsize(self.cache.local.get(b_entries[key]))
+    return new_count, diff_size
+
+
+def _get_tree_changes(self, a_entries, b_entries):
+    result = {
+        DIFF_DEL: 0,
+        DIFF_IDENT: 0,
+        DIFF_CHANGE: 0,
+        DIFF_NEW: 0,
+        DIFF_MOVE: 0,
+    }
+    result[DIFF_IDENT] = _ident_files(a_entries, b_entries)
+    result[DIFF_CHANGE], diff_size = _modified_files(
+        self, a_entries, b_entries
+    )
+    result[DIFF_SIZE] = diff_size
+    result[DIFF_DEL], diff_size = _deleted_files(self, a_entries, b_entries)
+    result[DIFF_SIZE] += diff_size
+    result[DIFF_NEW], diff_size = _new_files(self, a_entries, b_entries)
+    result[DIFF_SIZE] += diff_size
+    return result
+
+
+def _check_local_cache(a_out, is_checked):
+    if a_out is not None and a_out.scheme != "local":
+        is_checked.append(str(a_out))
+        return True
+    return False
+
+
+def _is_dir(path, a_outs, b_outs):
+    if a_outs.get(path):
+        return a_outs[path].is_dir_cache
+    else:
+        return b_outs[path].is_dir_cache
+
+
+def _get_diff_outs(self, diff_dct):
+    self.tree = diff_dct[DIFF_A_TREE]
+    a_outs = {str(out): out for st in self.stages() for out in st.outs}
+    self.tree = diff_dct[DIFF_B_TREE]
+    b_outs = {str(out): out for st in self.stages() for out in st.outs}
+    outs_paths = set(a_outs.keys())
+    outs_paths.update(b_outs.keys())
+    results = {}
+    non_local_cache = []
+    for path in outs_paths:
+        check1 = _check_local_cache(a_outs.get(path), non_local_cache)
+        check2 = _check_local_cache(b_outs.get(path), non_local_cache)
+        # skip files/directories with non-local cache for now
+        if check1 or check2:
+            continue
+        results[path] = {}
+        results[path][DIFF_A_OUTPUT] = a_outs.get(path)
+        results[path][DIFF_B_OUTPUT] = b_outs.get(path)
+        results[path][DIFF_IS_NEW] = path not in a_outs
+        results[path][DIFF_DELETED] = path not in b_outs
+        results[path][DIFF_IS_DIR] = _is_dir(path, a_outs, b_outs)
+    if non_local_cache:
+        logger.warning(
+            "Diff is not supported for non-local outputs. Ignoring: {}".format(
+                non_local_cache
+            )
+        )
+
+    return results
+
+
+def _diff_dir(self, target, diff_dct):
+    result = {DIFF_TARGET: target}
+    result[DIFF_IS_DIR] = True
+    a_entries, b_entries = {}, {}
+    try:
+        a_entries = _extract_dir(
+            self, diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
+        )
+        b_entries = _extract_dir(
+            self, diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
+        )
+        result[DIFF_OLD_FILE], result[DIFF_OLD_CHECKSUM] = _get_dir_info(
+            diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
+        )
+        result[DIFF_NEW_FILE], result[DIFF_NEW_CHECKSUM] = _get_dir_info(
+            diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
+        )
+        result.update(_get_tree_changes(self, a_entries, b_entries))
+    except IOError as e:
+        _file_not_exists(e, result)
+    return result
+
+
+def _diff_file(self, target, diff_dct):
+    result = {DIFF_TARGET: target}
+    size = 0
+    try:
+        if not diff_dct[DIFF_IS_NEW]:
+            result[DIFF_OLD_FILE] = target
+            result[DIFF_OLD_CHECKSUM] = diff_dct[DIFF_A_OUTPUT].checksum
+            size -= os.path.getsize(
+                self.cache.local.get(diff_dct[DIFF_A_OUTPUT].checksum)
+            )
+        if not diff_dct[DIFF_DELETED]:
+            result[DIFF_NEW_FILE] = target
+            result[DIFF_NEW_CHECKSUM] = diff_dct[DIFF_B_OUTPUT].checksum
+            size += os.path.getsize(
+                self.cache.local.get(diff_dct[DIFF_B_OUTPUT].checksum)
+            )
+        result[DIFF_SIZE] = size
+    except IOError as e:
+        _file_not_exists(e, result)
+    return result
+
+
+def _diff_royal(self, target, diff_dct):
+    if diff_dct[DIFF_IS_DIR]:
+        return _diff_dir(self, target, diff_dct)
+    return _diff_file(self, target, diff_dct)
+
+
+def diff(self, a_ref, target=None, b_ref=None):
+    """Gerenates diff message string output
+
+    Args:
+        target(str) - file/directory to check diff of
+        a_ref(str) - first tag
+        (optional) b_ref(str) - second git tag
+
+    Returns:
+        string: string of output message with diff info
+    """
+    result = {}
+    diff_dct = self.scm.get_diff_trees(a_ref, b_ref=b_ref)
+    result[DIFF_A_REF] = diff_dct[DIFF_A_REF]
+    result[DIFF_B_REF] = diff_dct[DIFF_B_REF]
+    if diff_dct[DIFF_EQUAL]:
+        result[DIFF_EQUAL] = True
+        return result
+    result[DIFF_LIST] = []
+    diff_outs = _get_diff_outs(self, diff_dct)
+    if target is None:
+        result[DIFF_LIST] = [
+            _diff_royal(self, path, diff_outs[path]) for path in diff_outs
+        ]
+    elif target in diff_outs:
+        result[DIFF_LIST] = [_diff_royal(self, target, diff_outs[target])]
+    else:
+        msg = "Have not found file/directory '{}' in the commits"
+        raise FileNotInCommitError(msg.format(target))
+    return result

dvc/repo/gc.py

@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
 import collections
+import logging
 
-import dvc.logger as logger
+
+logger = logging.getLogger(__name__)
 
 
 def _merge_cache_lists(clists):

dvc/repo/imp.py

@@ -1,19 +1,21 @@
+from dvc.repo.scm_context import scm_context
+
+
+@scm_context
 def imp(self, url, out, resume=False):
     from dvc.stage import Stage
 
-    stage = Stage.create(repo=self, cmd=None, deps=[url], outs=[out])
+    with self.state:
+        stage = Stage.create(repo=self, cmd=None, deps=[url], outs=[out])
 
     if stage is None:
         return None
 
     self.check_dag(self.stages() + [stage])
 
-    self.files_to_git_add = []
     with self.state:
         stage.run(resume=resume)
 
     stage.dump()
 
-    self.remind_to_git_add()
-
     return stage

dvc/repo/init.py

@@ -1,36 +1,44 @@
 import os
 import shutil
 import colorama
+import logging
 
-import dvc.logger as logger
 from dvc.repo import Repo
-from dvc.scm import SCM, Base
+from dvc.scm import SCM, NoSCM
 from dvc.config import Config
 from dvc.exceptions import InitError
+from dvc.utils import boxify
+
+
+logger = logging.getLogger(__name__)
 
 
 def _welcome_message():
-    logger.box(
-        "DVC has enabled anonymous aggregate usage analytics.\n"
-        "Read the analytics documentation (and how to opt-out) here:\n"
-        "{blue}https://dvc.org/doc/user-guide/analytics{nc}".format(
-            blue=colorama.Fore.BLUE, nc=colorama.Fore.RESET
-        ),
-        border_color="red",
+    logger.info(
+        boxify(
+            "DVC has enabled anonymous aggregate usage analytics.\n"
+            "Read the analytics documentation (and how to opt-out) here:\n"
+            "{blue}https://dvc.org/doc/user-guide/analytics{nc}".format(
+                blue=colorama.Fore.BLUE, nc=colorama.Fore.RESET
+            ),
+            border_color="red",
+        )
     )
 
-    logger.info(
+    msg = (
         "{yellow}What's next?{nc}\n"
         "{yellow}------------{nc}\n"
         "- Check out the documentation: {blue}https://dvc.org/doc{nc}\n"
         "- Get help and share ideas: {blue}https://dvc.org/chat{nc}\n"
-        "- Star us on GitHub: {blue}https://github.com/iterative/dvc{nc}".format(
-            yellow=colorama.Fore.YELLOW,
-            blue=colorama.Fore.BLUE,
-            nc=colorama.Fore.RESET,
-        )
+        "- Star us on GitHub: {blue}https://github.com/iterative/dvc{nc}"
+    ).format(
+        yellow=colorama.Fore.YELLOW,
+        blue=colorama.Fore.BLUE,
+        nc=colorama.Fore.RESET,
     )
 
+    logger.info(msg)
+
 
 def init(root_dir=os.curdir, no_scm=False, force=False):
     """
@@ -54,10 +62,10 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
     root_dir = os.path.abspath(root_dir)
     dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
     scm = SCM(root_dir)
-    if type(scm) == Base and not no_scm:
+    if isinstance(scm, NoSCM) and not no_scm:
         raise InitError(
-            "{repo} is not tracked by any supported scm tool"
-            " (e.g. git). Use '--no-scm' if you don't want to use any scm.".format(
+            "{repo} is not tracked by any supported scm tool (e.g. git). "
+            "Use '--no-scm' if you don't want to use any scm.".format(
                 repo=root_dir
             )
         )

dvc/repo/metrics/modify.py

@@ -1,11 +1,10 @@
 from __future__ import unicode_literals
 
-import os
-
 from dvc.exceptions import DvcException
 
 
 def modify(repo, path, typ=None, xpath=None, delete=False):
+    supported_types = ["raw", "json", "csv", "tsv", "hcsv", "htsv"]
     outs = repo.find_outs_by_path(path)
     assert len(outs) == 1
     out = outs[0]
@@ -14,12 +13,21 @@ def modify(repo, path, typ=None, xpath=None, delete=False):
         msg = "output '{}' scheme '{}' is not supported for metrics"
         raise DvcException(msg.format(out.path, out.path_info["scheme"]))
 
-    if typ:
+    if typ is not None:
+        typ = typ.lower().strip()
+        if typ not in ["raw", "json", "csv", "tsv", "hcsv", "htsv"]:
+            msg = (
+                "metric type '{typ}' is not supported, "
+                "must be one of [{types}]"
+            )
+            raise DvcException(
+                msg.format(typ=typ, types=", ".join(supported_types))
+            )
         if not isinstance(out.metric, dict):
             out.metric = {}
         out.metric[out.PARAM_METRIC_TYPE] = typ
 
-    if xpath:
+    if xpath is not None:
         if not isinstance(out.metric, dict):
             out.metric = {}
         out.metric[out.PARAM_METRIC_XPATH] = xpath
@@ -27,6 +35,6 @@ def modify(repo, path, typ=None, xpath=None, delete=False):
     if delete:
         out.metric = None
 
-    out._verify_metric()
+    out.verify_metric()
 
     out.stage.dump()

dvc/repo/metrics/show.py

@@ -3,17 +3,14 @@ from __future__ import unicode_literals
 import os
 import csv
 import json
+import logging
 from jsonpath_rw import parse
 
-import dvc.logger as logger
-from dvc.exceptions import (
-    OutputDuplicationError,
-    DvcException,
-    OutputNotFoundError,
-    BadMetricError,
-    NoMetricsError,
-)
-from dvc.utils.compat import str, builtin_str, open
+from dvc.exceptions import OutputNotFoundError, BadMetricError, NoMetricsError
+from dvc.utils.compat import builtin_str, open, StringIO, csv_reader
+
+
+logger = logging.getLogger(__name__)
 
 
 def _read_metric_json(fd, json_path):
@@ -21,64 +18,164 @@ def _read_metric_json(fd, json_path):
     return [x.value for x in parser.find(json.load(fd))]
 
 
+def _get_values(row):
+    if isinstance(row, dict):
+        return list(row.values())
+    else:
+        return row
+
+
 def _do_read_metric_xsv(reader, row, col):
     if col is not None and row is not None:
         return [reader[row][col]]
     elif col is not None:
         return [r[col] for r in reader]
     elif row is not None:
-        return reader[row]
-    return None
+        return _get_values(reader[row])
+    return [_get_values(r) for r in reader]
 
 
 def _read_metric_hxsv(fd, hxsv_path, delimiter):
-    col, row = hxsv_path.split(",")
-    row = int(row)
+    indices = hxsv_path.split(",")
+    row = indices[0]
+    row = int(row) if row else None
+    col = indices[1] if len(indices) > 1 and indices[1] else None
     reader = list(csv.DictReader(fd, delimiter=builtin_str(delimiter)))
     return _do_read_metric_xsv(reader, row, col)
 
 
 def _read_metric_xsv(fd, xsv_path, delimiter):
-    col, row = xsv_path.split(",")
-    row = int(row)
-    col = int(col)
+    indices = xsv_path.split(",")
+    row = indices[0]
+    row = int(row) if row else None
+    col = int(indices[1]) if len(indices) > 1 and indices[1] else None
     reader = list(csv.reader(fd, delimiter=builtin_str(delimiter)))
     return _do_read_metric_xsv(reader, row, col)
 
 
-def _read_metric(path, typ=None, xpath=None):
-    ret = None
-
-    if not os.path.exists(path):
-        return ret
-
-    try:
-        with open(path, "r") as fd:
-            if typ == "json":
-                ret = _read_metric_json(fd, xpath)
-            elif typ == "csv":
-                ret = _read_metric_xsv(fd, xpath, ",")
-            elif typ == "tsv":
-                ret = _read_metric_xsv(fd, xpath, "\t")
-            elif typ == "hcsv":
-                ret = _read_metric_hxsv(fd, xpath, ",")
-            elif typ == "htsv":
-                ret = _read_metric_hxsv(fd, xpath, "\t")
-            else:
-                ret = fd.read().strip()
-    except Exception:
-        logger.error("unable to read metric in '{}'".format(path))
-
+def _read_typed_metric(typ, xpath, fd):
+    if typ == "json":
+        ret = _read_metric_json(fd, xpath)
+    elif typ == "csv":
+        ret = _read_metric_xsv(fd, xpath, ",")
+    elif typ == "tsv":
+        ret = _read_metric_xsv(fd, xpath, "\t")
+    elif typ == "hcsv":
+        ret = _read_metric_hxsv(fd, xpath, ",")
+    elif typ == "htsv":
+        ret = _read_metric_hxsv(fd, xpath, "\t")
+    else:
+        ret = fd.read().strip()
     return ret
 
 
-def _collect_metrics(self, path, recursive, typ, xpath):
+def _format_csv(content, delimiter):
+    """Format delimited text to have same column width.
+
+    Args:
+        content (str): The content of a metric.
+        delimiter (str): Value separator
+
+    Returns:
+        str: Formatted content.
+
+    Example:
+
+        >>> content = (
+            "value_mse,deviation_mse,data_set\n"
+            "0.421601,0.173461,train\n"
+            "0.67528,0.289545,testing\n"
+            "0.671502,0.297848,validation\n"
+        )
+        >>> _format_csv(content, ",")
+
+        "value_mse  deviation_mse   data_set\n"
+        "0.421601   0.173461        train\n"
+        "0.67528    0.289545        testing\n"
+        "0.671502   0.297848        validation\n"
+    """
+    reader = csv_reader(StringIO(content), delimiter=builtin_str(delimiter))
+    rows = [row for row in reader]
+    max_widths = [max(map(len, column)) for column in zip(*rows)]
+
+    lines = [
+        " ".join(
+            "{entry:{width}}".format(entry=entry, width=width + 2)
+            for entry, width in zip(row, max_widths)
+        )
+        for row in rows
+    ]
+
+    return "\n".join(lines)
+
+
+def _format_output(content, typ):
+    """Tabularize the content according to its type.
+
+    Args:
+        content (str): The content of a metric.
+        typ (str): The type of metric -- (raw|json|tsv|htsv|csv|hcsv).
+
+    Returns:
+        str: Content in a raw or tabular format.
+    """
+
+    if "csv" in str(typ):
+        return _format_csv(content, delimiter=",")
+
+    if "tsv" in str(typ):
+        return _format_csv(content, delimiter="\t")
+
+    return content
+
+
+def _read_metric(fd, typ=None, xpath=None, rel_path=None, branch=None):
+    typ = typ.lower().strip() if typ else typ
+    try:
+        if xpath:
+            return _read_typed_metric(typ, xpath.strip(), fd)
+        else:
+            return _format_output(fd.read().strip(), typ)
+    # Json path library has to be replaced or wrapped in
+    # order to fix this too broad except clause.
+    except Exception:
+        logger.exception(
+            "unable to read metric in '{}' in branch '{}'".format(
+                rel_path, branch
+            )
+        )
+        return None
+
+
+def _collect_metrics(self, path, recursive, typ, xpath, branch):
+    """Gather all the metric outputs.
+
+    Args:
+        path (str): Path to a metric file or a directory.
+        recursive (bool): If path is a directory, do a recursive search for
+            metrics on the given path.
+        typ (str): The type of metric to search for, could be one of the
+            following (raw|json|tsv|htsv|csv|hcsv).
+        xpath (str): Path to search for.
+        branch (str): Branch to look up for metrics.
+
+    Returns:
+        list(tuple): (output, typ, xpath)
+            - output:
+            - typ:
+            - xpath:
+    """
     outs = [out for stage in self.stages() for out in stage.outs]
 
     if path:
         try:
             outs = self.find_outs_by_path(path, outs=outs, recursive=recursive)
         except OutputNotFoundError:
+            logger.debug(
+                "stage file not for found for '{}' in branch '{}'".format(
+                    path, branch
+                )
+            )
             return []
 
     res = []
@@ -98,16 +195,50 @@ def _collect_metrics(self, path, recursive, typ, xpath):
     return res
 
 
-def _read_metrics(self, metrics):
+def _read_metrics_filesystem(path, typ, xpath, rel_path, branch):
+    if not os.path.exists(path):
+        return None
+    with open(path, "r") as fd:
+        return _read_metric(
+            fd, typ=typ, xpath=xpath, rel_path=rel_path, branch=branch
+        )
+
+
+def _read_metrics(self, metrics, branch):
+    """Read the content of each metric file and format it.
+
+    Args:
+        metrics (list): List of metric touples
+        branch (str): Branch to look up for metrics.
+
+    Returns:
+        A dict mapping keys with metrics path name and content.
+        For example:
+
+        {'metric.csv': ("value_mse  deviation_mse   data_set\n"
+                        "0.421601   0.173461        train\n"
+                        "0.67528    0.289545        testing\n"
+                        "0.671502   0.297848        validation\n")}
+    """
     res = {}
     for out, typ, xpath in metrics:
         assert out.scheme == "local"
+        if not typ:
+            typ = os.path.splitext(out.path.lower())[1].replace(".", "")
         if out.use_cache:
-            path = self.cache.local.get(out.checksum)
+            metric = _read_metrics_filesystem(
+                self.cache.local.get(out.checksum),
+                typ=typ,
+                xpath=xpath,
+                rel_path=out.rel_path,
+                branch=branch,
+            )
         else:
-            path = out.path
+            fd = self.tree.open(out.path)
+            metric = _read_metric(
+                fd, typ=typ, xpath=xpath, rel_path=out.rel_path, branch=branch
+            )
 
-        metric = _read_metric(path, typ=typ, xpath=xpath)
         if not metric:
             continue
 
@@ -126,11 +257,10 @@ def show(
     recursive=False,
 ):
     res = {}
-    for branch in self.scm.brancher(
-        all_branches=all_branches, all_tags=all_tags
-    ):
-        entries = _collect_metrics(self, path, recursive, typ, xpath)
-        metrics = _read_metrics(self, entries)
+
+    for branch in self.brancher(all_branches=all_branches, all_tags=all_tags):
+        entries = _collect_metrics(self, path, recursive, typ, xpath, branch)
+        metrics = _read_metrics(self, entries, branch)
         if metrics:
             res[branch] = metrics
 

dvc/repo/move.py

@@ -1,6 +1,7 @@
 import os
 
-from dvc.exceptions import MoveNotDataSourceError, DvcException
+from dvc.exceptions import MoveNotDataSourceError
+from dvc.repo.scm_context import scm_context
 
 
 def _expand_target_path(from_path, to_path):
@@ -9,6 +10,7 @@ def _expand_target_path(from_path, to_path):
     return to_path
 
 
+@scm_context
 def move(self, from_path, to_path):
     """
     Renames an output file and modifies the stage associated
@@ -30,7 +32,7 @@ def move(self, from_path, to_path):
     import dvc.output as Output
     from dvc.stage import Stage
 
-    from_out = Output.loads_from(Stage(self, cwd=os.curdir), [from_path])[0]
+    from_out = Output.loads_from(Stage(self), [from_path])[0]
 
     to_path = _expand_target_path(from_path, to_path)
 
@@ -52,7 +54,9 @@ def move(self, from_path, to_path):
             os.path.basename(to_path) + Stage.STAGE_FILE_SUFFIX,
         )
 
-        stage.cwd = os.path.join(self.root_dir, os.path.dirname(to_path))
+        stage.wdir = os.path.abspath(
+            os.path.join(os.curdir, os.path.dirname(to_path))
+        )
 
     to_out = Output.loads_from(
         stage, [os.path.basename(to_path)], out.cache, out.metric
@@ -62,5 +66,3 @@ def move(self, from_path, to_path):
         out.move(to_out)
 
     stage.dump()
-
-    self.remind_to_git_add()

dvc/repo/pkg.py

@@ -0,0 +1,205 @@
+from __future__ import unicode_literals
+
+import os
+import shutil
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.stage import Stage
+from dvc.scm.git.temp_git_repo import TempGitRepo
+
+
+logger = logging.getLogger(__name__)
+
+
+class PackageParams(object):
+    def __init__(self, address, target_dir, select=[], file=None):
+        self.address = address
+        self.target_dir = target_dir
+        self.select = select
+        self.file = file
+
+    @property
+    def all_addresses(self):
+        if self.address:
+            return [self.address]
+        return PackageManager.read_packages()
+
+
+class PackageManager(object):
+    PACKAGE_FILE = "package.yaml"
+
+    @staticmethod
+    def read_packages():
+        return []
+
+    @staticmethod
+    def get_package(addr):
+        for pkg_class in [GitPackage]:
+            try:
+                return pkg_class()
+            except Exception:
+                pass
+        return None
+
+    def __init__(self, addr):
+        self._addr = addr
+
+
+class Package(object):
+    MODULES_DIR = "dvc_mod"
+
+    def install_or_update(self, parent_repo, pkg_param):
+        raise NotImplementedError(
+            "A method of abstract Package class was called"
+        )
+
+    def is_in_root(self):
+        return True
+
+
+class GitPackage(Package):
+    DEF_DVC_FILE_PREFIX = "mod_"
+
+    def install_or_update(self, parent_repo, pkg_params):
+        from git.cmd import Git
+
+        if not self.is_in_root():
+            raise DvcException(
+                "This command can be run only from a repository root"
+            )
+
+        if not os.path.exists(self.MODULES_DIR):
+            logger.debug("Creating modules dir {}".format(self.MODULES_DIR))
+            os.makedirs(self.MODULES_DIR)
+            parent_repo.scm.ignore(os.path.abspath(self.MODULES_DIR))
+
+        module_name = (
+            Git.polish_url(pkg_params.address).strip("/").split("/")[-1]
+        )
+        if not module_name:
+            raise DvcException(
+                "Package address error: unable to extract package name"
+            )
+
+        with TempGitRepo(
+            pkg_params.address, module_name, Package.MODULES_DIR
+        ) as tmp_repo:
+            outputs_to_copy = tmp_repo.outs
+            if pkg_params.select:
+                outputs_to_copy = list(
+                    filter(
+                        lambda out: out.dvc_path in pkg_params.select,
+                        outputs_to_copy,
+                    )
+                )
+
+            fetched_stage_files = set(
+                map(lambda o: o.stage.path, outputs_to_copy)
+            )
+            tmp_repo.fetch(fetched_stage_files)
+
+            module_dir = self.create_module_dir(module_name)
+            tmp_repo.persist_to(module_dir, parent_repo)
+
+            dvc_file = self.get_dvc_file_name(
+                pkg_params.file, pkg_params.target_dir, module_name
+            )
+            try:
+                self.persist_stage_and_scm_state(
+                    parent_repo,
+                    outputs_to_copy,
+                    pkg_params.target_dir,
+                    dvc_file,
+                )
+            except Exception as ex:
+                raise DvcException(
+                    "Package '{}' was installed "
+                    "but stage file '{}' "
+                    "was not created properly: {}".format(
+                        pkg_params.address, dvc_file, ex
+                    )
+                )
+
+        parent_repo.checkout(dvc_file)
+
+    @staticmethod
+    def persist_stage_and_scm_state(
+        parent_repo, outputs_to_copy, target_dir, dvc_file
+    ):
+        stage = Stage.create(
+            repo=parent_repo,
+            fname=dvc_file,
+            validate_state=False,
+            wdir=target_dir,
+        )
+        stage.outs = list(
+            map(lambda o: o.assign_to_stage_file(stage), outputs_to_copy)
+        )
+
+        for out in stage.outs:
+            parent_repo.scm.ignore(out.path, in_curr_dir=True)
+
+        stage.dump()
+
+    @staticmethod
+    def create_module_dir(module_name):
+        module_dir = os.path.join(GitPackage.MODULES_DIR, module_name)
+        if os.path.exists(module_dir):
+            logger.info("Updating package {}".format(module_name))
+            shutil.rmtree(module_dir)
+        else:
+            logger.info("Adding package {}".format(module_name))
+        return module_dir
+
+    def get_dvc_file_name(self, stage_file, target_dir, module_name):
+        if stage_file:
+            dvc_file_path = stage_file
+        else:
+            dvc_file_name = self.DEF_DVC_FILE_PREFIX + module_name + ".dvc"
+            dvc_file_path = os.path.join(target_dir, dvc_file_name)
+        return dvc_file_path
+
+
+def install_pkg(self, pkg_params):
+    """
+    Install package.
+
+    The command can be run only from DVC project root.
+
+    E.g.
+          Having: DVC package in https://github.com/dmpetrov/tag_classifier
+
+          $ dvc pkg install https://github.com/dmpetrov/tag_classifier
+
+          Result: tag_classifier package in dvc_mod/ directory
+    """
+
+    if not os.path.isdir(pkg_params.target_dir):
+        logger.error(
+            "Unable to install package: "
+            "target directory '{}' does not exist".format(
+                pkg_params.target_dir
+            )
+        )
+        return 1
+
+    curr_dir = os.path.realpath(os.curdir)
+    if not os.path.realpath(pkg_params.target_dir).startswith(curr_dir):
+        logger.error(
+            "Unable to install package: the current dir should be"
+            " a subdirectory of the target dir {}".format(
+                pkg_params.target_dir
+            )
+        )
+        return 1
+
+    for addr in pkg_params.all_addresses:
+        try:
+            mgr = PackageManager.get_package(addr)
+            mgr.install_or_update(self, pkg_params)
+        except Exception as ex:
+            logger.exception("Unable to install package: ".format(ex))
+            return 1
+
+    return 0

dvc/repo/remove.py

@@ -3,8 +3,8 @@ def remove(self, target, outs_only=False):
 
     stage = Stage.load(self, target)
     if outs_only:
-        stage.remove_outs()
+        stage.remove_outs(force=True)
     else:
-        stage.remove()
+        stage.remove(force=True)
 
     return stage

dvc/repo/reproduce.py

@@ -1,9 +1,13 @@
 from __future__ import unicode_literals
 
 import os
+import logging
 
-import dvc.logger as logger
 from dvc.exceptions import ReproductionError
+from dvc.repo.scm_context import scm_context
+
+
+logger = logging.getLogger(__name__)
 
 
 def _reproduce_stage(stages, node, force, dry, interactive, no_commit):
@@ -27,6 +31,7 @@ def _reproduce_stage(stages, node, force, dry, interactive, no_commit):
     return [stage]
 
 
+@scm_context
 def reproduce(
     self,
     target=None,
@@ -65,8 +70,6 @@ def reproduce(
     else:
         targets.append(target)
 
-    self.files_to_git_add = []
-
     ret = []
     with self.state:
         for target in targets:
@@ -82,8 +85,6 @@ def reproduce(
             )
             ret.extend(stages)
 
-    self.remind_to_git_add()
-
     return ret
 
 

dvc/repo/run.py

@@ -1,6 +1,9 @@
-import os
+from __future__ import unicode_literals
+
+from dvc.repo.scm_context import scm_context
 
 
+@scm_context
 def run(
     self,
     cmd=None,
@@ -10,12 +13,15 @@ def run(
     metrics=None,
     metrics_no_cache=None,
     fname=None,
-    cwd=os.curdir,
+    cwd=None,
+    wdir=None,
     no_exec=False,
     overwrite=False,
     ignore_build_cache=False,
     remove_outs=False,
     no_commit=False,
+    outs_persist=None,
+    outs_persist_no_cache=None,
 ):
     from dvc.stage import Stage
 
@@ -29,6 +35,10 @@ def run(
         metrics = []
     if metrics_no_cache is None:
         metrics_no_cache = []
+    if outs_persist is None:
+        outs_persist = []
+    if outs_persist_no_cache is None:
+        outs_persist_no_cache = []
 
     with self.state:
         stage = Stage.create(
@@ -36,6 +46,7 @@ def run(
             fname=fname,
             cmd=cmd,
             cwd=cwd,
+            wdir=wdir,
             outs=outs,
             outs_no_cache=outs_no_cache,
             metrics=metrics,
@@ -44,6 +55,8 @@ def run(
             overwrite=overwrite,
             ignore_build_cache=ignore_build_cache,
             remove_outs=remove_outs,
+            outs_persist=outs_persist,
+            outs_persist_no_cache=outs_persist_no_cache,
         )
 
     if stage is None:
@@ -51,13 +64,10 @@ def run(
 
     self.check_dag(self.stages() + [stage])
 
-    self.files_to_git_add = []
     with self.state:
         if not no_exec:
             stage.run(no_commit=no_commit)
 
     stage.dump()
 
-    self.remind_to_git_add()
-
     return stage

dvc/repo/scm_context.py

@@ -0,0 +1,12 @@
+def scm_context(method):
+    def run(repo, *args, **kw):
+        try:
+            result = method(repo, *args, **kw)
+            repo.scm.reset_ignores()
+            repo.scm.remind_to_track()
+            return result
+        except Exception:
+            repo.scm.cleanup_ignores()
+            raise
+
+    return run

dvc/repo/status.py

@@ -1,15 +1,15 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import logging
+
+
+logger = logging.getLogger(__name__)
 
 
 def _local_status(self, target=None, with_deps=False):
     status = {}
 
-    if target:
-        stages = self.collect(target, with_deps=with_deps)
-    else:
-        stages = self.active_stages()
+    stages = self.collect(target, with_deps=with_deps)
 
     for stage in stages:
         if stage.locked:
@@ -54,7 +54,7 @@ def _cloud_status(
     for md5, info in status_info.items():
         name = info["name"]
         status = info["status"]
-        if status == cloud.STATUS_OK:
+        if status in [cloud.STATUS_OK, cloud.STATUS_MISSING]:
             continue
 
         prefix_map = {cloud.STATUS_DELETED: "deleted", cloud.STATUS_NEW: "new"}
@@ -76,7 +76,7 @@ def status(
     all_tags=False,
 ):
     with self.state:
-        if cloud:
+        if cloud or remote:
             return _cloud_status(
                 self,
                 target,

dvc/repo/tag/__init__.py

@@ -0,0 +1,18 @@
+class Tag(object):
+    def __init__(self, repo):
+        self.repo = repo
+
+    def add(self, *args, **kwargs):
+        from dvc.repo.tag.add import add
+
+        return add(self.repo, *args, **kwargs)
+
+    def list(self, *args, **kwargs):
+        from dvc.repo.tag.list import list
+
+        return list(self.repo, *args, **kwargs)
+
+    def remove(self, *args, **kwargs):
+        from dvc.repo.tag.remove import remove
+
+        return remove(self.repo, *args, **kwargs)

dvc/repo/tag/add.py

@@ -0,0 +1,18 @@
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+def add(self, tag, target=None, with_deps=False, recursive=False):
+    stages = self.collect(target, with_deps=with_deps, recursive=recursive)
+    for stage in stages:
+        changed = False
+        for out in stage.outs:
+            if not out.info:
+                logger.warning("missing checksum info for '{}'".format(out))
+                continue
+            out.tags[tag] = out.info.copy()
+            changed = True
+        if changed:
+            stage.dump()

dvc/repo/tag/list.py

@@ -0,0 +1,11 @@
+def list(self, target=None, with_deps=False, recursive=False):
+    ret = {}
+    stages = self.collect(target, with_deps=with_deps, recursive=recursive)
+    for stage in stages:
+        outs_tags = {}
+        for out in stage.outs:
+            if out.tags:
+                outs_tags[str(out)] = out.tags.copy()
+        if outs_tags:
+            ret.update({stage.relpath: outs_tags})
+    return ret

dvc/repo/tag/remove.py

@@ -0,0 +1,18 @@
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+def remove(self, tag, target=None, with_deps=False, recursive=False):
+    stages = self.collect(target, with_deps=with_deps, recursive=recursive)
+    for stage in stages:
+        changed = False
+        for out in stage.outs:
+            if tag not in out.tags.keys():
+                logger.warning("tag '{}' not found for '{}'".format(tag, out))
+                continue
+            del out.tags[tag]
+            changed = True
+        if changed:
+            stage.dump()

dvc/repo/unprotect.py

@@ -1,48 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-import stat
-import uuid
-
-import dvc.logger as logger
-from dvc.system import System
-from dvc.utils import copyfile, move, remove
-from dvc.exceptions import DvcException
-
-
-def _unprotect_file(path):
-    if System.is_symlink(path) or System.is_hardlink(path):
-        logger.debug("Unprotecting '{}'".format(path))
-
-        tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))
-        move(path, tmp)
-
-        copyfile(tmp, path)
-
-        remove(tmp)
-    else:
-        logger.debug(
-            "Skipping copying for '{}', since it is not "
-            "a symlink or a hardlink.".format(path)
-        )
-
-    os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
-
-
-def _unprotect_dir(path):
-    for root, dirs, files in os.walk(str(path)):
-        for f in files:
-            path = os.path.join(root, f)
-            _unprotect_file(path)
-
-
-def unprotect(path):
-    if not os.path.exists(path):
-        raise DvcException(
-            "can't unprotect non-existing data '{}'".format(path)
-        )
-
-    if os.path.isdir(path):
-        _unprotect_dir(path)
-    else:
-        _unprotect_file(path)

dvc/scm/__init__.py

@@ -6,6 +6,12 @@ from dvc.scm.base import Base
 from dvc.scm.git import Git
 
 
+# just a sugar to point that this is an actual implementation for a dvc
+# project under no SCM control
+class NoSCM(Base):
+    pass
+
+
 def SCM(root_dir, repo=None):  # pylint: disable=invalid-name
     """Returns SCM instance that corresponds to a repo at the specified
     path.
@@ -20,4 +26,4 @@ def SCM(root_dir, repo=None):  # pylint: disable=invalid-name
     if Git.is_repo(root_dir) or Git.is_submodule(root_dir):
         return Git(root_dir, repo=repo)
 
-    return Base(root_dir, repo=repo)
+    return NoSCM(root_dir, repo=repo)

dvc/scm/base.py

@@ -17,12 +17,23 @@ class FileNotInRepoError(DvcException):
     """
 
 
+class FileNotInCommitError(DvcException):
+    """Thrown when trying to find a file/directory that is not
+    in the specified commit in the repository.
+    """
+
+
+class FileNotInTargetSubdirError(DvcException):
+    """Thrown when trying to place .gitignore for a file that not in
+    the file subdirectory."""
+
+
 class Base(object):
     """Base class for source control management driver implementations."""
 
     def __init__(self, root_dir=os.curdir, repo=None):
         self.repo = repo
-        self.root_dir = root_dir
+        self.root_dir = os.path.realpath(root_dir)
 
     def __repr__(self):
         return "{class_name}: '{directory}'".format(
@@ -76,47 +87,6 @@ class Base(object):
     def tag(self, tag):
         """Makes SCM create a tag with a specified name."""
 
-    def brancher(
-        self, branches=None, all_branches=False, tags=None, all_tags=False
-    ):
-        """Generator that iterates over specified revisions.
-
-        Args:
-            branches (list): a list of branches to iterate over.
-            all_branches (bool): iterate over all available branches.
-            tags (list): a list of tags to iterate over.
-            all_tags (bool): iterate over all available tags.
-
-        Yields:
-            str: the current revision.
-        """
-        if not branches and not all_branches and not tags and not all_tags:
-            yield ""
-            return
-
-        saved = self.active_branch()
-        revs = []
-
-        if all_branches:
-            branches = self.list_branches()
-
-        if all_tags:
-            tags = self.list_tags()
-
-        if branches is None:
-            revs.extend([saved])
-        else:
-            revs.extend(branches)
-
-        if tags is not None:
-            revs.extend(tags)
-
-        for rev in revs:
-            self.checkout(rev)
-            yield rev
-
-        self.checkout(saved)
-
     def untracked_files(self):  # pylint: disable=no-self-use
         """Returns a list of untracked files."""
         return []
@@ -125,6 +95,10 @@ class Base(object):
         """Returns whether or not a specified path is tracked."""
         return False
 
+    def is_dirty(self):
+        """Return whether the SCM contains uncommited changes."""
+        return False
+
     def active_branch(self):  # pylint: disable=no-self-use
         """Returns current branch in the repo."""
         return ""
@@ -139,3 +113,29 @@ class Base(object):
 
     def install(self):
         """Adds dvc commands to SCM hooks for the repo."""
+
+    def cleanup_ignores(self):
+        """
+        This method should clean up ignores (eg. entries in .gitignore),
+        use, when method editing ignores (eg. add, run, import) fails to
+        perform its task.
+        """
+
+    def reset_ignores(self):
+        """
+        Method to reset in-memory ignore storing mechanism.
+        """
+
+    def remind_to_track(self):
+        """
+        Method to remind user to track newly created files handled by scm
+        """
+
+    def track_file(self, path):
+        """
+        Method to add file to mechanism that will remind user
+        to track new files
+        """
+
+    def belongs_to_scm(self, path):
+        """Return boolean whether file belongs to scm"""

dvc/scm/git.py

@@ -1,174 +0,0 @@
-"""Manages Git."""
-
-from __future__ import unicode_literals
-
-# NOTE: need this one because python 2 would re-import this module
-# (called git.py) instead of importing gitpython's git module.
-from __future__ import absolute_import
-
-from dvc.utils.compat import str, open
-
-import os
-
-import dvc.logger as logger
-from dvc.utils import fix_env
-from dvc.scm.base import Base, SCMError, FileNotInRepoError
-
-
-class Git(Base):
-    """Class for managing Git."""
-
-    GITIGNORE = ".gitignore"
-    GIT_DIR = ".git"
-
-    def __init__(self, root_dir=os.curdir, repo=None):
-        super(Git, self).__init__(root_dir, repo=repo)
-
-        import git
-        from git.exc import InvalidGitRepositoryError
-
-        try:
-            self.git = git.Repo(root_dir)
-        except InvalidGitRepositoryError:
-            msg = "{} is not a git repository"
-            raise SCMError(msg.format(root_dir))
-
-        # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
-        # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
-        env = fix_env(None)
-        libpath = env.get("LD_LIBRARY_PATH", None)
-        self.git.git.update_environment(LD_LIBRARY_PATH=libpath)
-
-    @staticmethod
-    def is_repo(root_dir):
-        return os.path.isdir(Git._get_git_dir(root_dir))
-
-    @staticmethod
-    def is_submodule(root_dir):
-        return os.path.isfile(Git._get_git_dir(root_dir))
-
-    @staticmethod
-    def _get_git_dir(root_dir):
-        return os.path.join(root_dir, Git.GIT_DIR)
-
-    @property
-    def dir(self):
-        return self.git.git_dir
-
-    @property
-    def ignore_file(self):
-        return self.GITIGNORE
-
-    def _get_gitignore(self, path):
-        assert os.path.isabs(path)
-        # NOTE: using '/' prefix to make path unambiguous
-        entry = "/" + os.path.basename(path)
-        gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE)
-
-        if not gitignore.startswith(self.root_dir):
-            raise FileNotInRepoError(path)
-
-        return entry, gitignore
-
-    def ignore(self, path):
-        entry, gitignore = self._get_gitignore(path)
-
-        ignore_list = []
-        if os.path.exists(gitignore):
-            ignore_list = open(gitignore, "r").readlines()
-            filtered = list(
-                filter(lambda x: x.strip() == entry.strip(), ignore_list)
-            )
-            if filtered:
-                return
-
-        msg = "Adding '{}' to '{}'.".format(
-            os.path.relpath(path), os.path.relpath(gitignore)
-        )
-        logger.info(msg)
-
-        content = entry
-        if ignore_list:
-            content = "\n" + content
-
-        with open(gitignore, "a") as fobj:
-            fobj.write(content)
-
-        if self.repo is not None:
-            self.repo.files_to_git_add.append(os.path.relpath(gitignore))
-
-    def ignore_remove(self, path):
-        entry, gitignore = self._get_gitignore(path)
-
-        if not os.path.exists(gitignore):
-            return
-
-        with open(gitignore, "r") as fobj:
-            lines = fobj.readlines()
-
-        filtered = list(filter(lambda x: x.strip() != entry.strip(), lines))
-
-        with open(gitignore, "w") as fobj:
-            fobj.writelines(filtered)
-
-        if self.repo is not None:
-            self.repo.files_to_git_add.append(os.path.relpath(gitignore))
-
-    def add(self, paths):
-        # NOTE: GitPython is not currently able to handle index version >= 3.
-        # See https://github.com/iterative/dvc/issues/610 for more details.
-        try:
-            self.git.index.add(paths)
-        except AssertionError:
-            msg = (
-                "failed to add '{}' to git. You can add those files"
-                " manually using 'git add'."
-                " See 'https://github.com/iterative/dvc/issues/610'"
-                " for more details.".format(str(paths))
-            )
-
-            logger.error(msg)
-
-    def commit(self, msg):
-        self.git.index.commit(msg)
-
-    def checkout(self, branch, create_new=False):
-        if create_new:
-            self.git.git.checkout("HEAD", b=branch)
-        else:
-            self.git.git.checkout(branch)
-
-    def branch(self, branch):
-        self.git.git.branch(branch)
-
-    def tag(self, tag):
-        self.git.git.tag(tag)
-
-    def untracked_files(self):
-        files = self.git.untracked_files
-        return [os.path.join(self.git.working_dir, fname) for fname in files]
-
-    def is_tracked(self, path):
-        return len(self.git.git.ls_files(path)) != 0
-
-    def active_branch(self):
-        return self.git.active_branch.name
-
-    def list_branches(self):
-        return [h.name for h in self.git.heads]
-
-    def list_tags(self):
-        return [t.name for t in self.git.tags]
-
-    def _install_hook(self, name, cmd):
-        hook = os.path.join(self.root_dir, self.GIT_DIR, "hooks", name)
-        if os.path.isfile(hook):
-            msg = "git hook '{}' already exists."
-            raise SCMError(msg.format(os.path.relpath(hook)))
-        with open(hook, "w+") as fobj:
-            fobj.write("#!/bin/sh\nexec dvc {}\n".format(cmd))
-        os.chmod(hook, 0o777)
-
-    def install(self):
-        self._install_hook("post-checkout", "checkout")
-        self._install_hook("pre-commit", "status")

dvc/scm/git/__init__.py

@@ -0,0 +1,292 @@
+"""Manages Git."""
+
+from __future__ import unicode_literals
+
+import os
+import logging
+
+from dvc.utils.compat import str, open
+from dvc.utils import fix_env
+from dvc.scm.base import (
+    Base,
+    SCMError,
+    FileNotInRepoError,
+    FileNotInTargetSubdirError,
+)
+from dvc.scm.git.tree import GitTree
+
+
+logger = logging.getLogger(__name__)
+
+
+DIFF_A_TREE = "a_tree"
+DIFF_B_TREE = "b_tree"
+DIFF_A_REF = "a_ref"
+DIFF_B_REF = "b_ref"
+DIFF_EQUAL = "equal"
+
+
+class Git(Base):
+    """Class for managing Git."""
+
+    GITIGNORE = ".gitignore"
+    GIT_DIR = ".git"
+
+    def __init__(self, root_dir=os.curdir, repo=None):
+        super(Git, self).__init__(root_dir, repo=repo)
+
+        import git
+        from git.exc import InvalidGitRepositoryError
+
+        try:
+            self.git = git.Repo(root_dir)
+        except InvalidGitRepositoryError:
+            msg = "{} is not a git repository"
+            raise SCMError(msg.format(root_dir))
+
+        # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
+        # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
+        env = fix_env(None)
+        libpath = env.get("LD_LIBRARY_PATH", None)
+        self.git.git.update_environment(LD_LIBRARY_PATH=libpath)
+
+        self.ignored_paths = []
+        self.files_to_track = []
+
+    @staticmethod
+    def is_repo(root_dir):
+        return os.path.isdir(Git._get_git_dir(root_dir))
+
+    @staticmethod
+    def is_submodule(root_dir):
+        return os.path.isfile(Git._get_git_dir(root_dir))
+
+    @staticmethod
+    def _get_git_dir(root_dir):
+        return os.path.join(root_dir, Git.GIT_DIR)
+
+    @property
+    def dir(self):
+        return self.git.git_dir
+
+    @property
+    def ignore_file(self):
+        return self.GITIGNORE
+
+    def _get_gitignore(self, path, ignore_file_dir=None):
+        if not ignore_file_dir:
+            ignore_file_dir = os.path.dirname(os.path.realpath(path))
+
+        assert os.path.isabs(path)
+        assert os.path.isabs(ignore_file_dir)
+
+        if not path.startswith(ignore_file_dir):
+            msg = (
+                "{} file has to be located in one of '{}' subdirectories"
+                ", not outside '{}'"
+            )
+            raise FileNotInTargetSubdirError(
+                msg.format(self.GITIGNORE, path, ignore_file_dir)
+            )
+
+        entry = os.path.relpath(path, ignore_file_dir).replace(os.sep, "/")
+        # NOTE: using '/' prefix to make path unambiguous
+        if len(entry) > 0 and entry[0] != "/":
+            entry = "/" + entry
+
+        gitignore = os.path.join(ignore_file_dir, self.GITIGNORE)
+
+        if not gitignore.startswith(os.path.realpath(self.root_dir)):
+            raise FileNotInRepoError(path)
+
+        return entry, gitignore
+
+    def ignore(self, path, in_curr_dir=False):
+        base_dir = (
+            os.path.realpath(os.curdir)
+            if in_curr_dir
+            else os.path.dirname(path)
+        )
+        entry, gitignore = self._get_gitignore(path, base_dir)
+
+        ignore_list = []
+        if os.path.exists(gitignore):
+            with open(gitignore, "r") as f:
+                ignore_list = f.readlines()
+            if any(filter(lambda x: x.strip() == entry.strip(), ignore_list)):
+                return
+
+        msg = "Adding '{}' to '{}'.".format(
+            os.path.relpath(path), os.path.relpath(gitignore)
+        )
+        logger.info(msg)
+
+        self._add_entry_to_gitignore(entry, gitignore, ignore_list)
+
+        self.track_file(os.path.relpath(gitignore))
+
+        self.ignored_paths.append(path)
+
+    @staticmethod
+    def _add_entry_to_gitignore(entry, gitignore, ignore_list):
+        content = entry
+        if ignore_list:
+            content = "\n" + content
+        with open(gitignore, "a", encoding="utf-8") as fobj:
+            fobj.write(content)
+
+    def ignore_remove(self, path):
+        entry, gitignore = self._get_gitignore(path)
+
+        if not os.path.exists(gitignore):
+            return
+
+        with open(gitignore, "r") as fobj:
+            lines = fobj.readlines()
+
+        filtered = list(filter(lambda x: x.strip() != entry.strip(), lines))
+
+        with open(gitignore, "w") as fobj:
+            fobj.writelines(filtered)
+
+        self.track_file(os.path.relpath(gitignore))
+
+    def add(self, paths):
+        # NOTE: GitPython is not currently able to handle index version >= 3.
+        # See https://github.com/iterative/dvc/issues/610 for more details.
+        try:
+            self.git.index.add(paths)
+        except AssertionError:
+            msg = (
+                "failed to add '{}' to git. You can add those files"
+                " manually using 'git add'."
+                " See 'https://github.com/iterative/dvc/issues/610'"
+                " for more details.".format(str(paths))
+            )
+
+            logger.exception(msg)
+
+    def commit(self, msg):
+        self.git.index.commit(msg)
+
+    def checkout(self, branch, create_new=False):
+        if create_new:
+            self.git.git.checkout("HEAD", b=branch)
+        else:
+            self.git.git.checkout(branch)
+
+    def branch(self, branch):
+        self.git.git.branch(branch)
+
+    def tag(self, tag):
+        self.git.git.tag(tag)
+
+    def untracked_files(self):
+        files = self.git.untracked_files
+        return [os.path.join(self.git.working_dir, fname) for fname in files]
+
+    def is_tracked(self, path):
+        # it is equivalent to `bool(self.git.git.ls_files(path))` by
+        # functionality, but ls_files fails on unicode filenames
+        path = os.path.relpath(path, self.root_dir)
+        return path in [i[0] for i in self.git.index.entries]
+
+    def is_dirty(self):
+        return self.git.is_dirty()
+
+    def active_branch(self):
+        return self.git.active_branch.name
+
+    def list_branches(self):
+        return [h.name for h in self.git.heads]
+
+    def list_tags(self):
+        return [t.name for t in self.git.tags]
+
+    def _install_hook(self, name, cmd):
+        hook = os.path.join(self.root_dir, self.GIT_DIR, "hooks", name)
+        if os.path.isfile(hook):
+            msg = "git hook '{}' already exists."
+            raise SCMError(msg.format(os.path.relpath(hook)))
+        with open(hook, "w+") as fobj:
+            fobj.write("#!/bin/sh\nexec dvc {}\n".format(cmd))
+        os.chmod(hook, 0o777)
+
+    def install(self):
+        self._install_hook("post-checkout", "checkout")
+        self._install_hook("pre-commit", "status")
+
+    def cleanup_ignores(self):
+        for path in self.ignored_paths:
+            self.ignore_remove(path)
+        self.reset_ignores()
+
+    def reset_ignores(self):
+        self.ignored_paths = []
+
+    def remind_to_track(self):
+        if not self.files_to_track:
+            return
+
+        logger.info(
+            "\n"
+            "To track the changes with git run:\n"
+            "\n"
+            "\tgit add {files}".format(files=" ".join(self.files_to_track))
+        )
+
+    def track_file(self, path):
+        self.files_to_track.append(path)
+
+    def belongs_to_scm(self, path):
+        basename = os.path.basename(path)
+        path_parts = os.path.normpath(path).split(os.path.sep)
+        return basename == self.ignore_file or Git.GIT_DIR in path_parts
+
+    def get_tree(self, rev):
+        return GitTree(self.git, rev)
+
+    def _get_diff_trees(self, a_ref, b_ref):
+
+        from gitdb.exc import BadObject, BadName
+
+        trees = {DIFF_A_TREE: None, DIFF_B_TREE: None}
+        commits = []
+        try:
+            if b_ref is not None:
+                a_commit = self.git.commit(a_ref)
+                b_commit = self.git.commit(b_ref)
+                commits.append(str(a_commit))
+                commits.append(str(b_commit))
+            else:
+                a_commit = self.git.commit(a_ref)
+                b_commit = self.git.head.commit
+                commits.append(str(a_commit))
+                commits.append(str(b_commit))
+            trees[DIFF_A_TREE] = self.get_tree(commits[0])
+            trees[DIFF_B_TREE] = self.get_tree(commits[1])
+        except (BadName, BadObject) as e:
+            raise SCMError("git problem", cause=e)
+        return trees, commits
+
+    def get_diff_trees(self, a_ref, b_ref=None):
+        """Method for getting two repo trees between two git tag commits
+        returns the dvc hash names of changed file/directory
+
+        Args:
+            a_ref(str) - git reference
+            b_ref(str) - optional second git reference, default None
+
+        Returns:
+            dict - dictionary with keys: (a_tree, b_tree, a_ref, b_ref, equal)
+        """
+        diff_dct = {DIFF_EQUAL: False}
+        trees, commit_refs = self._get_diff_trees(a_ref, b_ref)
+        diff_dct[DIFF_A_REF] = commit_refs[0]
+        diff_dct[DIFF_B_REF] = commit_refs[1]
+        if commit_refs[0] == commit_refs[1]:
+            diff_dct[DIFF_EQUAL] = True
+            return diff_dct
+        diff_dct[DIFF_A_TREE] = trees[DIFF_A_TREE]
+        diff_dct[DIFF_B_TREE] = trees[DIFF_B_TREE]
+        return diff_dct

dvc/scm/git/temp_git_repo.py

@@ -0,0 +1,148 @@
+import os
+import shutil
+import tempfile
+import logging
+
+from dvc.exceptions import DvcException
+
+
+logger = logging.getLogger(__name__)
+
+
+class TempRepoException(DvcException):
+    """Raise when temporary package repository has not properly created"""
+
+    def __init__(self, temp_repo, msg, cause=None):
+        m = "temp repository '{}' error: {}".format(temp_repo.addr, msg)
+        super(TempRepoException, self).__init__(m, cause)
+
+
+class TempGitRepo(object):
+    GIT_DIR_TO_REMOVE = ".git"
+    GIT_FILES_TO_REMOVE = [".gitignore", ".gitmodules"]
+    SUFFIX = "tmp_DVC_mod"
+
+    def __init__(self, addr, module_name, modules_dir):
+        self.addr = addr
+        self.modules_dir = modules_dir
+        self._tmp_mod_prefix = module_name + "_" + self.SUFFIX + "_"
+        self._reset_state()
+
+    def _reset_state(self):
+        self._set_state(None, [])
+
+    def _set_state(self, cloned_tmp_dir, outs):
+        from dvc.repo import Repo
+
+        self.repo = Repo(cloned_tmp_dir) if cloned_tmp_dir else None
+
+        self._cloned_tmp_dir = cloned_tmp_dir
+        self.outs = outs
+        self._moved_files = []
+
+    def fetch(self, targets):
+        for target in targets:
+            try:
+                self.repo.fetch(target)
+            except Exception as ex:
+                msg = "error in fetching data from {}: {}".format(
+                    os.path.basename(target), ex
+                )
+                raise TempRepoException(self, msg)
+
+    @property
+    def is_state_set(self):
+        return self._cloned_tmp_dir is not None
+
+    def __enter__(self):
+        if self.is_state_set:
+            raise TempRepoException(self, "Git repo cloning duplication")
+
+        module_temp_dir = None
+        try:
+            module_temp_dir = self._clone_to_temp_dir()
+            self._remove_git_files_and_dirs(module_temp_dir)
+
+            outputs = self._read_outputs(module_temp_dir)
+            self._set_state(module_temp_dir, outputs)
+        finally:
+            self._clean_mod_temp_dir(module_temp_dir)
+
+        return self
+
+    def _clone_to_temp_dir(self):
+        import git
+
+        result = tempfile.mktemp(
+            prefix=self._tmp_mod_prefix, dir=self.modules_dir
+        )
+        logger.debug(
+            "Cloning git repository {} to temp dir {}".format(
+                self.addr, result
+            )
+        )
+        git.Repo.clone_from(self.addr, result, depth=1)
+        return result
+
+    def _remove_git_files_and_dirs(self, module_temp_dir):
+        logger.debug(
+            "Removing git meta files from {}: {} dif and {}".format(
+                module_temp_dir,
+                TempGitRepo.GIT_DIR_TO_REMOVE,
+                ", ".join(TempGitRepo.GIT_FILES_TO_REMOVE),
+            )
+        )
+        shutil.rmtree(
+            os.path.join(module_temp_dir, TempGitRepo.GIT_DIR_TO_REMOVE)
+        )
+        for item in TempGitRepo.GIT_FILES_TO_REMOVE:
+            fname = os.path.join(module_temp_dir, item)
+            if os.path.exists(fname):
+                os.remove(fname)
+
+    def _clean_mod_temp_dir(self, module_temp_dir):
+        if not self.is_state_set:
+            if module_temp_dir and os.path.exists(module_temp_dir):
+                shutil.rmtree(module_temp_dir, ignore_errors=True)
+
+    def _read_outputs(self, module_temp_dir):
+        from dvc.repo import Repo
+
+        pkg_repo = Repo(module_temp_dir)
+        stages = pkg_repo.stages()
+        return [out for s in stages for out in s.outs]
+
+    def persist_to(self, module_dir, parent_repo):
+        if not self.is_state_set:
+            raise TempRepoException(self, "cannot persist")
+
+        tmp_repo_cache = self.repo.cache.local.url
+
+        for prefix in os.listdir(tmp_repo_cache):
+            if len(prefix) != 2:
+                logger.warning(
+                    "wrong dir format in cache {}: dir {}".format(
+                        tmp_repo_cache, prefix
+                    )
+                )
+            self._move_all_cache_files(parent_repo, prefix, tmp_repo_cache)
+
+        shutil.move(self._cloned_tmp_dir, module_dir)
+        self._reset_state()
+
+    @staticmethod
+    def _move_all_cache_files(parent_repo, prefix, tmp_repo_cache):
+        obj_name = os.path.join(tmp_repo_cache, prefix)
+        for suffix in os.listdir(obj_name):
+            src_path = os.path.join(tmp_repo_cache, prefix, suffix)
+            pre = os.path.join(parent_repo.cache.local.url, prefix)
+            if not os.path.exists(pre):
+                os.mkdir(pre)
+            dest = os.path.join(pre, suffix)
+            shutil.move(src_path, dest)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.is_state_set:
+            shutil.rmtree(self._cloned_tmp_dir)
+            self._reset_state()
+        pass

dvc/scm/git/tree.py

@@ -0,0 +1,114 @@
+import errno
+import os
+
+from dvc.utils.compat import StringIO
+
+from dvc.scm.tree import BaseTree
+
+
+# see git-fast-import(1)
+GIT_MODE_DIR = 0o40000
+GIT_MODE_FILE = 0o644
+
+
+class GitTree(BaseTree):
+    """Proxies the repo file access methods to Git objects"""
+
+    def __init__(self, git, rev):
+        """Create GitTree instance
+
+        Args:
+            git (dvc.scm.Git):
+            branch:
+        """
+        self.git = git
+        self.rev = rev
+
+    def open(self, path):
+
+        relpath = os.path.relpath(path, self.git.working_dir)
+
+        obj = self.git_object_by_path(path)
+        if obj is None:
+            msg = "No such file in branch '{}'".format(self.rev)
+            raise IOError(errno.ENOENT, msg, relpath)
+        if obj.mode == GIT_MODE_DIR:
+            raise IOError(errno.EISDIR, "Is a directory", relpath)
+
+        # GitPython's obj.data_stream is a fragile thing, it is better to
+        # read it immediately, also it needs to be to decoded if we follow
+        # the `open()` behavior (since data_stream.read() returns bytes,
+        # and `open` with default "r" mode returns str)
+        return StringIO(obj.data_stream.read().decode("utf-8"))
+
+    def exists(self, path):
+        return self.git_object_by_path(path) is not None
+
+    def isdir(self, path):
+        obj = self.git_object_by_path(path)
+        if obj is None:
+            return False
+        return obj.mode == GIT_MODE_DIR
+
+    def isfile(self, path):
+        obj = self.git_object_by_path(path)
+        if obj is None:
+            return False
+        # according to git-fast-import(1) file mode could be 644 or 755
+        return obj.mode & GIT_MODE_FILE == GIT_MODE_FILE
+
+    @staticmethod
+    def _is_tree_and_contains(obj, path):
+        if obj.mode != GIT_MODE_DIR:
+            return False
+        # see https://github.com/gitpython-developers/GitPython/issues/851
+        # `return (i in tree)` doesn't work so here is a workaround:
+        for i in obj:
+            if i.name == path:
+                return True
+        return False
+
+    def git_object_by_path(self, path):
+        path = os.path.relpath(os.path.realpath(path), self.git.working_dir)
+        assert path.split(os.sep, 1)[0] != ".."
+        tree = self.git.tree(self.rev)
+        if not path or path == ".":
+            return tree
+        for i in path.split(os.sep):
+            if not self._is_tree_and_contains(tree, i):
+                # there is no tree for specified path
+                return None
+            tree = tree[i]
+        return tree
+
+    def _walk(self, tree, topdown=True):
+
+        dirs, nondirs = [], []
+        for i in tree:
+            if i.mode == GIT_MODE_DIR:
+                dirs.append(i.name)
+            else:
+                nondirs.append(i.name)
+
+        if topdown:
+            yield os.path.normpath(tree.path), dirs, nondirs
+        for i in dirs:
+            for x in self._walk(tree[i], topdown=True):
+                yield x
+        if not topdown:
+            yield os.path.normpath(tree.path), dirs, nondirs
+
+    def walk(self, top, topdown=True):
+        """Directory tree generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        - it could raise exceptions, there is no onerror argument
+        """
+
+        tree = self.git_object_by_path(top)
+        if tree is None:
+            raise IOError(errno.ENOENT, "No such file")
+
+        for x in self._walk(tree, topdown):
+            yield x

dvc/scm/tree.py

@@ -0,0 +1,65 @@
+import os
+
+from dvc.utils.compat import open
+
+
+class BaseTree(object):
+    """Abstract class to represent access to files"""
+
+    def open(self, path):
+        """Open file and return a stream."""
+
+    def exists(self, path):
+        """Test whether a path exists."""
+
+    def isdir(self, path):
+        """Return true if the pathname refers to an existing directory."""
+
+    def isfile(self, path):
+        """Test whether a path is a regular file"""
+
+    def walk(self, top, topdown=True):
+        """Directory tree generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        - it could raise exceptions, there is no onerror argument
+        """
+
+
+class WorkingTree(BaseTree):
+    """Proxies the repo file access methods to working tree files"""
+
+    def open(self, path):
+        """Open file and return a stream."""
+        return open(path, encoding="utf-8")
+
+    def exists(self, path):
+        """Test whether a path exists."""
+        return os.path.exists(path)
+
+    def isdir(self, path):
+        """Return true if the pathname refers to an existing directory."""
+        return os.path.isdir(path)
+
+    def isfile(self, path):
+        """Test whether a path is a regular file"""
+        return os.path.isfile(path)
+
+    def walk(self, top, topdown=True):
+        """Directory tree generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        - it could raise exceptions, there is no onerror argument
+        """
+
+        def onerror(e):
+            raise e
+
+        for root, dirs, files in os.walk(
+            top, topdown=topdown, onerror=onerror
+        ):
+            if topdown:
+                dirs[:] = [i for i in dirs if i not in (".git", ".hg", ".dvc")]
+            yield os.path.normpath(root), dirs, files

dvc/stage.py

@@ -2,18 +2,23 @@ from __future__ import unicode_literals
 
 from dvc.utils.compat import str, open
 
+import re
 import os
 import yaml
-import posixpath
 import subprocess
+import logging
+
+from dvc.utils.fs import contains_symlink_up_to
 from schema import Schema, SchemaError, Optional, Or, And
 
 import dvc.prompt as prompt
-import dvc.logger as logger
 import dvc.dependency as dependency
 import dvc.output as output
 from dvc.exceptions import DvcException
-from dvc.utils import dict_md5, fix_env
+from dvc.utils import dict_md5, fix_env, load_stage_file_fobj
+
+
+logger = logging.getLogger(__name__)
 
 
 class StageCmdFailedError(DvcException):
@@ -61,10 +66,22 @@ class StageFileBadNameError(DvcException):
         super(StageFileBadNameError, self).__init__(msg)
 
 
-class StageBadCwdError(DvcException):
-    def __init__(self, cwd):
-        msg = "stage cwd '{}' is outside of the current dvc repo"
-        super(StageBadCwdError, self).__init__(msg.format(cwd))
+class StagePathOutsideError(DvcException):
+    def __init__(self, path):
+        msg = "stage working or file path '{}' is outside of dvc repo"
+        super(StagePathOutsideError, self).__init__(msg.format(path))
+
+
+class StagePathNotFoundError(DvcException):
+    def __init__(self, path):
+        msg = "stage working or file path '{}' does not exist"
+        super(StagePathNotFoundError, self).__init__(msg.format(path))
+
+
+class StagePathNotDirectoryError(DvcException):
+    def __init__(self, path):
+        msg = "stage working or file path '{}' is not directory"
+        super(StagePathNotDirectoryError, self).__init__(msg.format(path))
 
 
 class StageCommitError(DvcException):
@@ -102,6 +119,7 @@ class Stage(object):
 
     PARAM_MD5 = "md5"
     PARAM_CMD = "cmd"
+    PARAM_WDIR = "wdir"
     PARAM_DEPS = "deps"
     PARAM_OUTS = "outs"
     PARAM_LOCKED = "locked"
@@ -109,21 +127,25 @@ class Stage(object):
     SCHEMA = {
         Optional(PARAM_MD5): Or(str, None),
         Optional(PARAM_CMD): Or(str, None),
+        Optional(PARAM_WDIR): Or(str, None),
         Optional(PARAM_DEPS): Or(And(list, Schema([dependency.SCHEMA])), None),
         Optional(PARAM_OUTS): Or(And(list, Schema([output.SCHEMA])), None),
         Optional(PARAM_LOCKED): bool,
     }
 
+    TAG_REGEX = r"^(?P<path>.*)@(?P<tag>[^\\/@:]*)$"
+
     def __init__(
         self,
         repo,
         path=None,
         cmd=None,
-        cwd=os.curdir,
+        wdir=os.curdir,
         deps=None,
         outs=None,
         md5=None,
         locked=False,
+        tag=None,
     ):
         if deps is None:
             deps = []
@@ -133,11 +155,12 @@ class Stage(object):
         self.repo = repo
         self.path = path
         self.cmd = cmd
-        self.cwd = cwd
+        self.wdir = wdir
         self.outs = outs
         self.deps = deps
         self.md5 = md5
         self.locked = locked
+        self.tag = tag
 
     def __repr__(self):
         return "Stage: '{path}'".format(
@@ -156,7 +179,10 @@ class Stage(object):
     @staticmethod
     def is_valid_filename(path):
         return (
-            path.endswith(Stage.STAGE_FILE_SUFFIX)
+            # path.endswith doesn't work for encoded unicode filenames on
+            # Python 2 and since Stage.STAGE_FILE_SUFFIX is ascii then it is
+            # not needed to decode the path from py2's str
+            path[-len(Stage.STAGE_FILE_SUFFIX) :] == Stage.STAGE_FILE_SUFFIX
             or os.path.basename(path) == Stage.STAGE_FILE
         )
 
@@ -186,18 +212,19 @@ class Stage(object):
 
         if self.is_callback:
             logger.warning(
-                "Dvc file '{fname}' is a 'callback' stage (has a command and"
-                " no dependencies) and thus always considered as changed.".format(
-                    fname=self.relpath
-                )
+                "Dvc file '{fname}' is a 'callback' stage "
+                "(has a command and no dependencies) and thus always "
+                "considered as changed.".format(fname=self.relpath)
             )
             return True
 
         for dep in self.deps:
-            if dep.changed():
+            status = dep.status()
+            if status:
                 logger.warning(
-                    "Dependency '{dep}' of '{stage}' changed.".format(
-                        dep=dep, stage=self.relpath
+                    "Dependency '{dep}' of '{stage}' changed because it is "
+                    "'{status}'.".format(
+                        dep=dep, stage=self.relpath, status=status[str(dep)]
                     )
                 )
                 return True
@@ -206,10 +233,12 @@ class Stage(object):
 
     def _changed_outs(self):
         for out in self.outs:
-            if out.changed():
+            status = out.status()
+            if status:
                 logger.warning(
-                    "Output '{out}' of '{stage}' changed.".format(
-                        out=out, stage=self.relpath
+                    "Output '{out}' of '{stage}' changed because it is "
+                    "'{status}'".format(
+                        out=out, stage=self.relpath, status=status[str(out)]
                     )
                 )
                 return True
@@ -228,31 +257,31 @@ class Stage(object):
         )
 
         if ret:
-            msg = "Stage '{}' changed.".format(self.relpath)
-            color = "yellow"
+            logger.warning("Stage '{}' changed.".format(self.relpath))
         else:
-            msg = "Stage '{}' didn't change.".format(self.relpath)
-            color = "green"
-
-        logger.info(logger.colorize(msg, color))
+            logger.info("Stage '{}' didn't change.".format(self.relpath))
 
         return ret
 
-    def remove_outs(self, ignore_remove=False):
-        """
-        Used mainly for `dvc remove --outs`
-        """
+    def remove_outs(self, ignore_remove=False, force=False):
+        """Used mainly for `dvc remove --outs` and :func:`Stage.reproduce`."""
         for out in self.outs:
-            out.remove(ignore_remove=ignore_remove)
+            if out.persist and not force:
+                out.unprotect()
+            else:
+                logger.debug(
+                    "Removing output '{out}' of '{stage}'.".format(
+                        out=out, stage=self.relpath
+                    )
+                )
+                out.remove(ignore_remove=ignore_remove)
 
     def unprotect_outs(self):
         for out in self.outs:
-            if out.scheme != "local" or not out.exists:
-                continue
-            self.repo.unprotect(out.path)
+            out.unprotect()
 
-    def remove(self):
-        self.remove_outs(ignore_remove=True)
+    def remove(self, force=False):
+        self.remove_outs(ignore_remove=True, force=force)
         os.unlink(self.path)
 
     def reproduce(
@@ -261,12 +290,9 @@ class Stage(object):
         if not self.changed() and not force:
             return None
 
-        if (self.cmd or self.is_import) and not self.locked and not dry:
-            # Removing outputs only if we actually have command to reproduce
-            self.remove_outs(ignore_remove=False)
-
-        msg = "Going to reproduce '{stage}'. Are you sure you want to continue?".format(
-            stage=self.relpath
+        msg = (
+            "Going to reproduce '{stage}'. "
+            "Are you sure you want to continue?".format(stage=self.relpath)
         )
 
         if interactive and not prompt.confirm(msg):
@@ -274,7 +300,7 @@ class Stage(object):
 
         logger.info("Reproducing '{stage}'".format(stage=self.relpath))
 
-        self.run(dry=dry, no_commit=no_commit)
+        self.run(dry=dry, no_commit=no_commit, force=force)
 
         logger.debug("'{stage}' was reproduced".format(stage=self.relpath))
 
@@ -290,33 +316,48 @@ class Stage(object):
             raise StageFileFormatError(fname, exc)
 
     @classmethod
-    def _stage_fname_cwd(cls, fname, cwd, outs, add):
-        if fname and cwd:
-            return (fname, cwd)
+    def _stage_fname(cls, fname, outs, add):
+        if fname:
+            return fname
 
         if not outs:
-            return (cls.STAGE_FILE, cwd if cwd else os.getcwd())
+            return cls.STAGE_FILE
 
         out = outs[0]
-        if out.scheme == "local":
-            path = os.path
-        else:
-            path = posixpath
+        path_handler = out.remote.ospath
 
-        if not fname:
-            fname = path.basename(out.path) + cls.STAGE_FILE_SUFFIX
+        fname = path_handler.basename(out.path) + cls.STAGE_FILE_SUFFIX
 
-        if not cwd or (add and out.is_local):
-            cwd = path.dirname(out.path)
+        fname = Stage._expand_to_path_on_add_local(
+            add, fname, out, path_handler
+        )
 
-        return (fname, cwd)
+        return fname
 
     @staticmethod
-    def _check_inside_repo(repo, cwd):
+    def _expand_to_path_on_add_local(add, fname, out, path_handler):
+        if (
+            add
+            and out.is_in_repo
+            and not contains_symlink_up_to(out.path, out.repo.root_dir)
+        ):
+            fname = path_handler.join(path_handler.dirname(out.path), fname)
+        return fname
+
+    @staticmethod
+    def _check_stage_path(repo, path):
         assert repo is not None
-        proj_dir = os.path.realpath(repo.root_dir)
-        if not os.path.realpath(cwd).startswith(proj_dir):
-            raise StageBadCwdError(cwd)
+
+        real_path = os.path.realpath(path)
+        if not os.path.exists(real_path):
+            raise StagePathNotFoundError(path)
+
+        if not os.path.isdir(real_path):
+            raise StagePathNotDirectoryError(path)
+
+        proj_dir = os.path.realpath(repo.root_dir) + os.path.sep
+        if not (real_path + os.path.sep).startswith(proj_dir):
+            raise StagePathOutsideError(path)
 
     @property
     def is_cached(self):
@@ -359,14 +400,17 @@ class Stage(object):
         metrics=None,
         metrics_no_cache=None,
         fname=None,
-        cwd=os.curdir,
+        cwd=None,
+        wdir=None,
         locked=False,
         add=False,
         overwrite=True,
         ignore_build_cache=False,
         remove_outs=False,
+        validate_state=True,
+        outs_persist=None,
+        outs_persist_no_cache=None,
     ):
-
         if outs is None:
             outs = []
         if deps is None:
@@ -377,38 +421,52 @@ class Stage(object):
             metrics = []
         if metrics_no_cache is None:
             metrics_no_cache = []
+        if outs_persist is None:
+            outs_persist = []
+        if outs_persist_no_cache is None:
+            outs_persist_no_cache = []
 
-        stage = Stage(repo=repo, cwd=cwd, cmd=cmd, locked=locked)
+        # Backward compatibility for `cwd` option
+        if wdir is None and cwd is not None:
+            if fname is not None and os.path.basename(fname) != fname:
+                raise StageFileBadNameError(
+                    "stage file name '{fname}' may not contain subdirectories"
+                    " if '-c|--cwd' (deprecated) is specified. Use '-w|--wdir'"
+                    " along with '-f' to specify stage file path and working"
+                    " directory.".format(fname=fname)
+                )
+            wdir = cwd
+        else:
+            wdir = os.curdir if wdir is None else wdir
 
-        stage.outs = output.loads_from(stage, outs, use_cache=True)
-        stage.outs += output.loads_from(
-            stage, metrics, use_cache=True, metric=True
-        )
-        stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False)
-        stage.outs += output.loads_from(
-            stage, metrics_no_cache, use_cache=False, metric=True
+        stage = Stage(repo=repo, wdir=wdir, cmd=cmd, locked=locked)
+
+        Stage._fill_stage_outputs(
+            stage,
+            outs,
+            outs_no_cache,
+            metrics,
+            metrics_no_cache,
+            outs_persist,
+            outs_persist_no_cache,
         )
         stage.deps = dependency.loads_from(stage, deps)
 
         stage._check_circular_dependency()
         stage._check_duplicated_arguments()
 
-        if fname is not None and os.path.basename(fname) != fname:
-            raise StageFileBadNameError(
-                "stage file name '{fname}' should not contain subdirectories."
-                " Use '-c|--cwd' to change location of the stage file.".format(
-                    fname=fname
-                )
-            )
+        fname = Stage._stage_fname(fname, stage.outs, add=add)
+        wdir = os.path.abspath(wdir)
 
-        fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add)
+        if cwd is not None:
+            path = os.path.join(wdir, fname)
+        else:
+            path = os.path.abspath(fname)
 
-        Stage._check_inside_repo(repo, cwd)
+        Stage._check_stage_path(repo, wdir)
+        Stage._check_stage_path(repo, os.path.dirname(path))
 
-        cwd = os.path.abspath(cwd)
-        path = os.path.join(cwd, fname)
-
-        stage.cwd = cwd
+        stage.wdir = wdir
         stage.path = path
 
         # NOTE: remove outs before we check build cache
@@ -419,23 +477,49 @@ class Stage(object):
         else:
             stage.unprotect_outs()
 
-        if os.path.exists(path):
-            if not ignore_build_cache and stage.is_cached:
-                logger.info("Stage is cached, skipping.")
-                return None
+        if validate_state:
+            if os.path.exists(path):
+                if not ignore_build_cache and stage.is_cached:
+                    logger.info("Stage is cached, skipping.")
+                    return None
 
-            msg = (
-                "'{}' already exists. Do you wish to run the command and "
-                "overwrite it?".format(stage.relpath)
-            )
+                msg = (
+                    "'{}' already exists. Do you wish to run the command and "
+                    "overwrite it?".format(stage.relpath)
+                )
 
-            if not overwrite and not prompt.confirm(msg):
-                raise StageFileAlreadyExistsError(stage.relpath)
+                if not overwrite and not prompt.confirm(msg):
+                    raise StageFileAlreadyExistsError(stage.relpath)
 
-            os.unlink(path)
+                os.unlink(path)
 
         return stage
 
+    @staticmethod
+    def _fill_stage_outputs(
+        stage,
+        outs,
+        outs_no_cache,
+        metrics,
+        metrics_no_cache,
+        outs_persist,
+        outs_persist_no_cache,
+    ):
+        stage.outs = output.loads_from(stage, outs, use_cache=True)
+        stage.outs += output.loads_from(
+            stage, metrics, use_cache=True, metric=True
+        )
+        stage.outs += output.loads_from(
+            stage, outs_persist, use_cache=True, persist=True
+        )
+        stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False)
+        stage.outs += output.loads_from(
+            stage, metrics_no_cache, use_cache=False, metric=True
+        )
+        stage.outs += output.loads_from(
+            stage, outs_persist_no_cache, use_cache=False, persist=True
+        )
+
     @staticmethod
     def _check_dvc_filename(fname):
         if not Stage.is_valid_filename(fname):
@@ -447,30 +531,52 @@ class Stage(object):
             )
 
     @staticmethod
-    def _check_file_exists(fname):
-        if not os.path.exists(fname):
+    def _check_file_exists(repo, fname):
+        if not repo.tree.exists(fname):
             raise StageFileDoesNotExistError(fname)
 
     @staticmethod
-    def load(repo, fname):
-        Stage._check_file_exists(fname)
-        Stage._check_dvc_filename(fname)
-
-        if not Stage.is_stage_file(fname):
+    def _check_isfile(repo, fname):
+        if not repo.tree.isfile(fname):
             raise StageFileIsNotDvcFileError(fname)
 
-        with open(fname, "r") as fd:
-            d = yaml.safe_load(fd) or {}
+    @classmethod
+    def _get_path_tag(cls, s):
+        regex = re.compile(cls.TAG_REGEX)
+        match = regex.match(s)
+        if not match:
+            return s, None
+        return match.group("path"), match.group("tag")
+
+    @staticmethod
+    def load(repo, fname):
+        fname, tag = Stage._get_path_tag(fname)
+
+        # it raises the proper exceptions by priority:
+        # 1. when the file doesn't exists
+        # 2. filename is not a dvc filename
+        # 3. path doesn't represent a regular file
+        Stage._check_file_exists(repo, fname)
+        Stage._check_dvc_filename(fname)
+        Stage._check_isfile(repo, fname)
+
+        d = load_stage_file_fobj(repo.tree.open(fname), fname)
 
         Stage.validate(d, fname=os.path.relpath(fname))
+        path = os.path.abspath(fname)
 
         stage = Stage(
             repo=repo,
-            path=os.path.abspath(fname),
-            cwd=os.path.dirname(os.path.abspath(fname)),
+            path=path,
+            wdir=os.path.abspath(
+                os.path.join(
+                    os.path.dirname(path), d.get(Stage.PARAM_WDIR, ".")
+                )
+            ),
             cmd=d.get(Stage.PARAM_CMD),
             md5=d.get(Stage.PARAM_MD5),
             locked=d.get(Stage.PARAM_LOCKED, False),
+            tag=tag,
         )
 
         stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
@@ -479,11 +585,16 @@ class Stage(object):
         return stage
 
     def dumpd(self):
+        from dvc.remote.local import RemoteLOCAL
+
         return {
             key: value
             for key, value in {
                 Stage.PARAM_MD5: self.md5,
                 Stage.PARAM_CMD: self.cmd,
+                Stage.PARAM_WDIR: RemoteLOCAL.unixpath(
+                    os.path.relpath(self.wdir, os.path.dirname(self.path))
+                ),
                 Stage.PARAM_LOCKED: self.locked,
                 Stage.PARAM_DEPS: [d.dumpd() for d in self.deps],
                 Stage.PARAM_OUTS: [o.dumpd() for o in self.outs],
@@ -491,8 +602,8 @@ class Stage(object):
             if value
         }
 
-    def dump(self, fname=None):
-        fname = fname or self.path
+    def dump(self):
+        fname = self.path
 
         self._check_dvc_filename(fname)
 
@@ -501,14 +612,15 @@ class Stage(object):
                 file=os.path.relpath(fname)
             )
         )
+        d = self.dumpd()
 
         with open(fname, "w") as fd:
-            yaml.safe_dump(self.dumpd(), fd, default_flow_style=False)
+            yaml.safe_dump(d, fd, default_flow_style=False)
 
-        self.repo.files_to_git_add.append(os.path.relpath(fname))
+        self.repo.scm.track_file(os.path.relpath(fname))
 
     def _compute_md5(self):
-        from dvc.output.local import OutputLOCAL
+        from dvc.output.base import OutputBase
 
         d = self.dumpd()
 
@@ -516,12 +628,27 @@ class Stage(object):
         if self.PARAM_MD5 in d.keys():
             del d[self.PARAM_MD5]
 
+        # Ignore the wdir default value. In this case stage file w/o
+        # wdir has the same md5 as a file with the default value specified.
+        # It's important for backward compatibility with pipelines that
+        # didn't have WDIR in their stage files.
+        if d.get(self.PARAM_WDIR) == ".":
+            del d[self.PARAM_WDIR]
+
         # NOTE: excluding parameters that don't affect the state of the
         # pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if
         # it has changed, we might not have that output in our cache.
-        return dict_md5(
-            d, exclude=[self.PARAM_LOCKED, OutputLOCAL.PARAM_METRIC]
+        m = dict_md5(
+            d,
+            exclude=[
+                self.PARAM_LOCKED,
+                OutputBase.PARAM_METRIC,
+                OutputBase.PARAM_TAGS,
+                OutputBase.PARAM_PERSIST,
+            ],
         )
+        logger.debug("Computed stage '{}' md5: '{}'".format(self.relpath, m))
+        return m
 
     def save(self):
         for dep in self.deps:
@@ -614,7 +741,7 @@ class Stage(object):
 
         p = subprocess.Popen(
             self.cmd,
-            cwd=self.cwd,
+            cwd=self.wdir,
             shell=True,
             env=fix_env(os.environ),
             executable=executable,
@@ -624,7 +751,10 @@ class Stage(object):
         if p.returncode != 0:
             raise StageCmdFailedError(self)
 
-    def run(self, dry=False, resume=False, no_commit=False):
+    def run(self, dry=False, resume=False, no_commit=False, force=False):
+        if (self.cmd or self.is_import) and not self.locked and not dry:
+            self.remove_outs(ignore_remove=False, force=False)
+
         if self.locked:
             logger.info(
                 "Verifying outputs in locked stage '{stage}'".format(
@@ -641,7 +771,7 @@ class Stage(object):
                 )
             )
             if not dry:
-                if self._already_cached():
+                if self._already_cached() and not force:
                     self.outs[0].checkout()
                 else:
                     self.deps[0].download(
@@ -657,7 +787,11 @@ class Stage(object):
         else:
             logger.info("Running command:\n\t{}".format(self.cmd))
             if not dry:
-                if self._already_cached():
+                if (
+                    not force
+                    and not self.is_callback
+                    and self._already_cached()
+                ):
                     self.checkout()
                 else:
                     self._run()
@@ -677,9 +811,11 @@ class Stage(object):
         if paths:
             raise MissingDataSource(paths)
 
-    def checkout(self, force=False):
+    def checkout(self, force=False, progress_callback=None):
         for out in self.outs:
-            out.checkout(force=force)
+            out.checkout(
+                force=force, tag=self.tag, progress_callback=progress_callback
+            )
 
     @staticmethod
     def _status(entries):
@@ -722,3 +858,6 @@ class Stage(object):
                 for out in self.outs
             )
         )
+
+    def get_all_files_number(self):
+        return sum(out.get_files_number() for out in self.outs)

dvc/state.py

@@ -2,18 +2,17 @@
 
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str
-
 import os
-import time
 import sqlite3
-import nanotime
+import logging
 
-import dvc.logger as logger
 from dvc.config import Config
-from dvc.system import System
-from dvc.utils import file_md5, remove
+from dvc.utils import file_md5, remove, current_timestamp
 from dvc.exceptions import DvcException
+from dvc.utils.fs import get_mtime_and_size, get_inode
+
+
+logger = logging.getLogger(__name__)
 
 
 class StateVersionTooNewError(DvcException):
@@ -168,7 +167,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         return num
 
     def _prepare_db(self, empty=False):
-        from dvc import VERSION
+        from dvc import __version__
 
         if not empty:
             cmd = "PRAGMA user_version;"
@@ -180,7 +179,9 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             version = ret[0][0]
 
             if version > self.VERSION:
-                raise StateVersionTooNewError(VERSION, self.VERSION, version)
+                raise StateVersionTooNewError(
+                    __version__, self.VERSION, version
+                )
             elif version < self.VERSION:
                 msg = (
                     "State file version '{}' is too old. "
@@ -289,6 +290,8 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             )
         )
 
+        self._update_cache_directory_state()
+
         self.database.commit()
         self.cursor.close()
         self.database.close()
@@ -296,30 +299,6 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         self.cursor = None
         self.inserts = 0
 
-    @staticmethod
-    def _mtime_and_size(path):
-        size = os.path.getsize(path)
-        mtime = os.path.getmtime(path)
-
-        if os.path.isdir(path):
-            for root, dirs, files in os.walk(str(path)):
-                for name in dirs + files:
-                    entry = os.path.join(root, name)
-                    stat = os.stat(entry)
-                    size += stat.st_size
-                    entry_mtime = stat.st_mtime
-                    if entry_mtime > mtime:
-                        mtime = entry_mtime
-
-        # State of files handled by dvc is stored in db as TEXT.
-        # We cast results to string for later comparisons with stored values.
-        return str(int(nanotime.timestamp(mtime))), str(size)
-
-    @staticmethod
-    def _inode(path):
-        logger.debug("Path {} inode {}".format(path, System.inode(path)))
-        return System.inode(path)
-
     def _do_update(self, path, known_checksum=None):
         """
         Make sure the stored info for the given path is up to date.
@@ -327,25 +306,24 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         if not os.path.exists(path):
             return None, None
 
-        actual_mtime, actual_size = self._mtime_and_size(path)
-        actual_inode = self._inode(path)
+        actual_mtime, actual_size = get_mtime_and_size(path)
+        actual_inode = get_inode(path)
 
-        existing_records = self._get_state_records_for_inode(actual_inode)
-        should_insert_new_record = not existing_records
+        existing_record = self.get_state_record_for_inode(actual_inode)
 
-        if should_insert_new_record:
-            md5, info = self._insert_new_state_record(
-                path, actual_inode, actual_mtime, actual_size, known_checksum
-            )
-        else:
+        if existing_record:
             md5, info = self._update_existing_state_record(
                 path,
                 actual_inode,
                 actual_mtime,
                 actual_size,
-                existing_records,
+                existing_record,
                 known_checksum,
             )
+        else:
+            md5, info = self._insert_new_state_record(
+                path, actual_inode, actual_mtime, actual_size, known_checksum
+            )
 
         return md5, info
 
@@ -355,13 +333,11 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         actual_inode,
         actual_mtime,
         actual_size,
-        existing_records,
+        existing_record,
         known_checksum=None,
     ):
 
-        md5, mtime, size = self._get_existing_record_data(
-            actual_inode, actual_mtime, actual_size, existing_records
-        )
+        mtime, size, md5, _ = existing_record
         if _file_metadata_changed(actual_mtime, mtime, actual_size, size):
             md5, info = self._update_state_for_path_changed(
                 path, actual_inode, actual_mtime, actual_size, known_checksum
@@ -371,28 +347,12 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             self._update_state_record_timestamp_for_inode(actual_inode)
         return md5, info
 
-    def _get_existing_record_data(
-        self, actual_inode, actual_mtime, actual_size, existing_records
-    ):
-        assert len(existing_records) == 1
-        assert len(existing_records[0]) == 5
-        inode, mtime, size, md5, _ = existing_records[0]
-        inode = self._from_sqlite(inode)
-        assert inode == actual_inode
-        logger.debug(
-            "Inode '{}', mtime '{}', actual mtime '{}', size '{}', "
-            "actual size '{}'.".format(
-                inode, mtime, actual_mtime, size, actual_size
-            )
-        )
-        return md5, mtime, size
-
     def _update_state_record_timestamp_for_inode(self, actual_inode):
         cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
         self._execute(
             cmd.format(
                 self.STATE_TABLE,
-                int(nanotime.timestamp(time.time())),
+                current_timestamp(),
                 self._to_sqlite(actual_inode),
             )
         )
@@ -421,7 +381,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
                 actual_mtime,
                 actual_size,
                 md5,
-                int(nanotime.timestamp(time.time())),
+                current_timestamp(),
                 self._to_sqlite(actual_inode),
             )
         )
@@ -445,19 +405,22 @@ class State(object):  # pylint: disable=too-many-instance-attributes
                 actual_mtime,
                 actual_size,
                 md5,
-                int(nanotime.timestamp(time.time())),
+                current_timestamp(),
             )
         )
         self.inserts += 1
         return md5, info
 
-    def _get_state_records_for_inode(self, actual_inode):
-        cmd = "SELECT * from {} WHERE inode={}".format(
-            self.STATE_TABLE, self._to_sqlite(actual_inode)
-        )
+    def get_state_record_for_inode(self, inode):
+        cmd = "SELECT mtime, size, md5, timestamp from {} " "WHERE inode={}"
+        cmd = cmd.format(self.STATE_TABLE, self._to_sqlite(inode))
         self._execute(cmd)
-        ret = self._fetchall()
-        return ret
+        results = self._fetchall()
+        if results:
+            # uniquness constrain on inode
+            assert len(results) == 1
+            return results[0]
+        return None
 
     def update(self, path, known_checksum=None):
         """Gets the checksum for the specified path. Checksum will be
@@ -499,8 +462,8 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         if not os.path.exists(path):
             return
 
-        mtime, _ = self._mtime_and_size(path)
-        inode = self._inode(path)
+        mtime, _ = get_mtime_and_size(path)
+        inode = get_inode(path)
         relpath = os.path.relpath(path, self.root_dir)
 
         cmd = (
@@ -531,8 +494,8 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             if not os.path.exists(path):
                 continue
 
-            actual_inode = self._inode(path)
-            actual_mtime, _ = self._mtime_and_size(path)
+            actual_inode = get_inode(path)
+            actual_mtime, _ = get_mtime_and_size(path)
 
             if inode == actual_inode and mtime == actual_mtime:
                 logger.debug("Removing '{}' as unused link.".format(path))
@@ -542,3 +505,20 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         for relpath in unused:
             cmd = 'DELETE FROM {} WHERE path = "{}"'
             self._execute(cmd.format(self.LINK_STATE_TABLE, relpath))
+
+    def _update_cache_directory_state(self):
+        cache_path = self.repo.cache.local.cache_dir
+        mtime, size = get_mtime_and_size(cache_path)
+        inode = get_inode(cache_path)
+
+        cmd = (
+            "INSERT OR REPLACE INTO {}(inode, size, mtime, timestamp, md5) "
+            'VALUES ({}, "{}", "{}", "{}", "")'.format(
+                self.STATE_TABLE,
+                self._to_sqlite(inode),
+                size,
+                mtime,
+                current_timestamp(),
+            )
+        )
+        self._execute(cmd)

dvc/system.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 from dvc.utils.compat import str, open
 
 import os
+import errno
 
 
 class System(object):
@@ -61,8 +62,22 @@ class System(object):
     @staticmethod
     def _reflink_darwin(src, dst):
         import ctypes
+        import dvc.logger as logger
+
+        LIBC = "libc.dylib"
+        LIBC_FALLBACK = "/usr/lib/libSystem.dylib"
+        try:
+            clib = ctypes.CDLL(LIBC)
+        except OSError as exc:
+            logger.debug(
+                "unable to access '{}' (errno '{}'). "
+                "Falling back to '{}'.".format(LIBC, exc.errno, LIBC_FALLBACK)
+            )
+            if exc.errno != errno.ENOENT:
+                raise
+            # NOTE: trying to bypass System Integrity Protection (SIP)
+            clib = ctypes.CDLL(LIBC_FALLBACK)
 
-        clib = ctypes.CDLL("libc.dylib")
         if not hasattr(clib, "clonefile"):
             return -1
 
@@ -267,8 +282,11 @@ class System(object):
         # https://docs.microsoft.com/en-us/windows/desktop/fileio/
         # file-attribute-constants
         FILE_ATTRIBUTE_REPARSE_POINT = 0x400
-        info = System.getdirinfo(path)
-        return info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT
+
+        if os.path.lexists(path):
+            info = System.getdirinfo(path)
+            return info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT
+        return False
 
     @staticmethod
     def is_hardlink(path):

dvc/updater.py

@@ -1,16 +1,19 @@
 from __future__ import unicode_literals
-from dvc.utils.compat import str
 
 import sys
 import os
 import time
+import logging
 import requests
 import colorama
+from pkg_resources import parse_version
 
-import dvc.logger as logger
-from dvc import VERSION_BASE
+from dvc import __version__
 from dvc.lock import Lock, LockError
-from dvc.utils import is_binary
+from dvc.utils import is_binary, boxify
+
+
+logger = logging.getLogger(__name__)
 
 
 class Updater(object):  # pragma: no cover
@@ -23,7 +26,7 @@ class Updater(object):  # pragma: no cover
         self.dvc_dir = dvc_dir
         self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
         self.lock = Lock(dvc_dir, self.updater_file + ".lock")
-        self.current = VERSION_BASE
+        self.current = parse_version(__version__).base_version
 
     def _is_outdated_file(self):
         ctime = os.path.getmtime(self.updater_file)
@@ -90,16 +93,7 @@ class Updater(object):  # pragma: no cover
             json.dump(info, fobj)
 
     def _is_outdated(self):
-        l_major, l_minor, l_patch = [int(x) for x in self.latest.split(".")]
-        c_major, c_minor, c_patch = [int(x) for x in self.current.split(".")]
-
-        if l_major != c_major:
-            return l_major > c_major
-
-        if l_minor != c_minor:
-            return l_minor > c_minor
-
-        return l_patch > c_patch
+        return parse_version(self.current) < parse_version(self.latest)
 
     def _notify(self):
         if not sys.stdout.isatty():
@@ -119,7 +113,7 @@ class Updater(object):  # pragma: no cover
             latest=self.latest,
         )
 
-        logger.box(message, border_color="yellow")
+        logger.info(boxify(message, border_color="yellow"))
 
     def _get_update_instructions(self):
         instructions = {

dvc/utils/__init__.py

@@ -1,6 +1,8 @@
 """Helpers for other modules."""
 
 from __future__ import unicode_literals
+
+import yaml
 from dvc.utils.compat import str, builtin_str, open, cast_bytes_py2
 
 import os
@@ -8,8 +10,19 @@ import sys
 import stat
 import math
 import json
+import errno
 import shutil
 import hashlib
+import nanotime
+import time
+import colorama
+import re
+import logging
+
+from yaml.scanner import ScannerError
+
+
+logger = logging.getLogger(__name__)
 
 LOCAL_CHUNK_SIZE = 1024 * 1024
 LARGE_FILE_SIZE = 1024 * 1024 * 1024
@@ -22,7 +35,6 @@ def dos2unix(data):
 
 def file_md5(fname):
     """ get the (md5 hexdigest, md5 digest) of a file """
-    import dvc.logger as logger
     from dvc.progress import progress
     from dvc.istextfile import istextfile
 
@@ -145,24 +157,30 @@ def move(src, dst):
     shutil.move(src, dst)
 
 
-def remove(path):
-    import dvc.logger as logger
-
-    if not os.path.exists(path):
-        return
-
-    logger.debug("Removing '{}'".format(os.path.relpath(path)))
-
-    def _chmod(func, p, excinfo):
+def _chmod(func, p, excinfo):
+    try:
         perm = os.stat(p).st_mode
         perm |= stat.S_IWRITE
         os.chmod(p, perm)
-        func(p)
+    except OSError as exc:
+        # NOTE: broken symlink case.
+        if exc.errno != errno.ENOENT:
+            raise
 
-    if os.path.isfile(path):
-        _chmod(os.unlink, path, None)
-    else:
-        shutil.rmtree(path, onerror=_chmod)
+    func(p)
+
+
+def remove(path):
+    logger.debug("Removing '{}'".format(os.path.relpath(path)))
+
+    try:
+        if os.path.isdir(path):
+            shutil.rmtree(path, onerror=_chmod)
+        else:
+            _chmod(os.unlink, path, None)
+    except OSError as exc:
+        if exc.errno != errno.ENOENT:
+            raise
 
 
 def to_chunks(l, jobs):
@@ -212,3 +230,114 @@ def convert_to_unicode(data):
         return type(data)(map(convert_to_unicode, data))
     else:
         return data
+
+
+def tmp_fname(fname):
+    """ Temporary name for a partial download """
+    from uuid import uuid4
+
+    return fname + "." + str(uuid4()) + ".tmp"
+
+
+def current_timestamp():
+    return int(nanotime.timestamp(time.time()))
+
+
+def load_stage_file(path):
+    with open(path, "r") as fobj:
+        return load_stage_file_fobj(fobj, path)
+
+
+def load_stage_file_fobj(fobj, path):
+    from dvc.exceptions import StageFileCorruptedError
+
+    try:
+        return yaml.safe_load(fobj) or {}
+    except ScannerError:
+        raise StageFileCorruptedError(path)
+
+
+def walk_files(directory):
+    for root, _, files in os.walk(str(directory)):
+        for f in files:
+            yield os.path.join(root, f)
+
+
+def colorize(message, color=None):
+    """Returns a message in a specified color."""
+    if not color:
+        return message
+
+    colors = {
+        "green": colorama.Fore.GREEN,
+        "yellow": colorama.Fore.YELLOW,
+        "blue": colorama.Fore.BLUE,
+        "red": colorama.Fore.RED,
+    }
+
+    return "{color}{message}{nc}".format(
+        color=colors.get(color, ""), message=message, nc=colorama.Fore.RESET
+    )
+
+
+def boxify(message, border_color=None):
+    """Put a message inside a box.
+
+    Args:
+        message (unicode): message to decorate.
+        border_color (unicode): name of the color to outline the box with.
+    """
+    lines = message.split("\n")
+    max_width = max(_visual_width(line) for line in lines)
+
+    padding_horizontal = 5
+    padding_vertical = 1
+
+    box_size_horizontal = max_width + (padding_horizontal * 2)
+
+    chars = {"corner": "+", "horizontal": "-", "vertical": "|", "empty": " "}
+
+    margin = "{corner}{line}{corner}\n".format(
+        corner=chars["corner"], line=chars["horizontal"] * box_size_horizontal
+    )
+
+    padding_lines = [
+        "{border}{space}{border}\n".format(
+            border=colorize(chars["vertical"], color=border_color),
+            space=chars["empty"] * box_size_horizontal,
+        )
+        * padding_vertical
+    ]
+
+    content_lines = [
+        "{border}{space}{content}{space}{border}\n".format(
+            border=colorize(chars["vertical"], color=border_color),
+            space=chars["empty"] * padding_horizontal,
+            content=_visual_center(line, max_width),
+        )
+        for line in lines
+    ]
+
+    box_str = "{margin}{padding}{content}{padding}{margin}".format(
+        margin=colorize(margin, color=border_color),
+        padding="".join(padding_lines),
+        content="".join(content_lines),
+    )
+
+    return box_str
+
+
+def _visual_width(line):
+    """Get the the number of columns required to display a string"""
+
+    return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, "", line))
+
+
+def _visual_center(line, width):
+    """Center align string according to it's visual width"""
+
+    spaces = max(width - _visual_width(line), 0)
+    left_padding = int(spaces / 2)
+    right_padding = spaces - left_padding
+
+    return (left_padding * " ") + line + (right_padding * " ")

dvc/utils/collections.py

@@ -0,0 +1,6 @@
+from __future__ import unicode_literals
+
+
+# just simple check for Nones and emtpy strings
+def compact(args):
+    return list(filter(bool, args))

dvc/utils/compat.py

@@ -1,6 +1,8 @@
 """Handle import compatibility between Python 2 and Python 3"""
 
 import sys
+import os
+import errno
 
 
 # Syntax sugar.
@@ -12,8 +14,6 @@ is_py2 = _ver[0] == 2
 #: Python 3.x?
 is_py3 = _ver[0] == 3
 
-# NOTE: cast_bytes_py2 is taken from https://github.com/ipython/ipython_genutils
-
 # simplified version of ipython_genutils/encoding.py
 DEFAULT_ENCODING = sys.getdefaultencoding()
 
@@ -27,12 +27,69 @@ def encode(u, encoding=None):
     return u.encode(encoding, "replace")
 
 
+def csv_reader(unicode_csv_data, dialect=None, **kwargs):
+    """csv.reader doesn't support Unicode input, so need to use some tricks
+    to work around this.
+
+    Source: https://docs.python.org/2/library/csv.html#csv-examples
+    """
+    import csv
+
+    dialect = dialect or csv.excel
+
+    if is_py3:
+        # Python3 supports encoding by default, so just return the object
+        for row in csv.reader(unicode_csv_data, dialect=dialect, **kwargs):
+            yield [cell for cell in row]
+
+    else:
+        # csv.py doesn't do Unicode; encode temporarily as UTF-8:
+        reader = csv.reader(
+            utf_8_encoder(unicode_csv_data), dialect=dialect, **kwargs
+        )
+        for row in reader:
+            # decode UTF-8 back to Unicode, cell by cell:
+            yield [unicode(cell, "utf-8") for cell in row]  # noqa: F821
+
+
+def utf_8_encoder(unicode_csv_data):
+    """Source: https://docs.python.org/2/library/csv.html#csv-examples"""
+    for line in unicode_csv_data:
+        yield line.encode("utf-8")
+
+
 def cast_bytes(s, encoding=None):
+    """Source: https://github.com/ipython/ipython_genutils"""
     if not isinstance(s, bytes):
         return encode(s, encoding)
     return s
 
 
+def _makedirs(name, mode=0o777, exist_ok=False):
+    """Source: https://github.com/python/cpython/blob/
+        3ce3dea60646d8a5a1c952469a2eb65f937875b3/Lib/os.py#L196-L226
+    """
+    head, tail = os.path.split(name)
+    if not tail:
+        head, tail = os.path.split(head)
+    if head and tail and not os.path.exists(head):
+        try:
+            _makedirs(head, exist_ok=exist_ok)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+        cdir = os.curdir
+        if isinstance(tail, bytes):
+            cdir = bytes(os.curdir, "ASCII")
+        if tail == cdir:
+            return
+    try:
+        os.mkdir(name, mode)
+    except OSError:
+        if not exist_ok or not os.path.isdir(name):
+            raise
+
+
 if is_py2:
     from urlparse import urlparse, urljoin  # noqa: F401
     from StringIO import StringIO  # noqa: F401
@@ -49,8 +106,10 @@ if is_py2:
     integer_types = (int, long)  # noqa: F821
     input = raw_input  # noqa: F821
     cast_bytes_py2 = cast_bytes
+    makedirs = _makedirs
 
 elif is_py3:
+    from os import makedirs  # noqa: F401
     from urllib.parse import urlparse, urljoin  # noqa: F401
     from io import StringIO  # noqa: F401
     from http.server import (  # noqa: F401