dvc: bump to 0.35.3

version: privatize version.py methods and vars

.appveyor.yml

@@ -31,22 +31,6 @@ environment:
 #    - PYTHON: "C:\\Python27-x64"
 #      PYTHON_VERSION: "2.7.x" # currently 2.7.9
 #      PYTHON_ARCH: "64"
-#
-#    - PYTHON: "C:\\Python33"
-#      PYTHON_VERSION: "3.3.x" # currently 3.3.5
-#      PYTHON_ARCH: "32"
-#
-#    - PYTHON: "C:\\Python33-x64"
-#      PYTHON_VERSION: "3.3.x" # currently 3.3.5
-#      PYTHON_ARCH: "64"
-#
-#    - PYTHON: "C:\\Python34"
-#      PYTHON_VERSION: "3.4.x" # currently 3.4.3
-#      PYTHON_ARCH: "32"
-#
-#    - PYTHON: "C:\\Python34-x64"
-#      PYTHON_VERSION: "3.4.x" # currently 3.4.3
-#      PYTHON_ARCH: "64"
 
 install:
   - ps: Install-Product node
@@ -75,6 +59,7 @@ test_script:
   - python -mtests
 
 after_test:
+  - if "%APPVEYOR_PULL_REQUEST_NUMBER%"=="" pip install codecov
   - if "%APPVEYOR_PULL_REQUEST_NUMBER%"=="" codecov
   - if "%APPVEYOR_REPO_TAG%"=="true" python -m dvc pull
   - if "%APPVEYOR_REPO_TAG%"=="true" .\scripts\build_windows.cmd

.dvc/config

@@ -1,4 +1,4 @@
 ['remote "dvc"']
-url = s3://dvc-share/dvc/
+url = https://dvc.org/s3/dvc
 [core]
 remote = dvc

.gitignore

@@ -24,6 +24,7 @@ innosetup/config.ini
 
 .coverage
 
-dvc/version.py
-
 *.swp
+
+pip-wheel-metadata/
+.vscode/

.travis.yml

@@ -22,9 +22,6 @@ matrix:
   - name: "2.7"
     env:
     - PYTHON_VER=2.7.13
-  - name: "3.4"
-    env:
-    - PYTHON_VER=3.4.6
   - name: "3.5"
     env:
     - PYTHON_VER=3.5.3
@@ -51,7 +48,7 @@ matrix:
     language: ruby
     rvm: 2.4.0
     env:
-    - PYTHON_VER=3.4.3
+    - PYTHON_VER=3.6.2
     - PYTHON_CONFIGURE_OPTS="--enable-framework"
   - if: type = cron OR tag is present
     os: osx

README.rst

@@ -6,7 +6,7 @@
 • `Docs <https://dvc.org/doc>`_
 • `Twitter <https://twitter.com/iterativeai>`_
 • `Chat (Community & Support) <https://dvc.org/chat>`_
-• `Tutorial <https://dvc.org/doc/tutorial>`_
+• `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
 .. image:: https://travis-ci.com/iterative/dvc.svg?branch=master
@@ -28,17 +28,18 @@
 |
 
 **Data Science Version Control** or **DVC** is an **open-source** tool for data science and
-machine learning projects. With a simple and flexible Git-like architecture and interface it
-helps data scientists:
+machine learning projects. Key features:
 
-#. manage **machine learning models** - versioning, including data sets and transformations (scripts) that were
-   used to generate models;
+#. simple **command line** Git-like experience. Does not require installing and maintaining
+   any databases. Does not depend on any proprietary online services;
 
-#. make projects **reproducible**;
+#. it manages and versions **datasets** and **machine learning models**. Data is saved in
+   S3, Google cloud, Azure, SSH server, HDFS or even local HDD RAID;
 
-#. make projects **shareable**;
+#. it makes projects **reproducible** and **shareable**, it helps answering question how
+   the model was build;
 
-#. manage experiments with branching and **metrics** tracking;
+#. it helps manage experiments with Git tags or branches and **metrics** tracking;
 
 It aims to replace tools like Excel and Docs that are being commonly used as a knowledge repo and
 a ledger for the team, ad-hoc scripts to track and move deploy different model versions, ad-hoc
@@ -50,14 +51,30 @@ data file suffixes and prefixes.
 How DVC works
 =============
 
-DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
-To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
-network storage (via SSH, for example).
+We encourage you to read our `Get Started <https://dvc.org/doc/get-started>`_ to better understand what DVC
+is and how does it fit your scenarios.
+
+The easiest (but not perfect!) *analogy* to describe it: DVC is Git (or Git-lfs to be precise) + ``makefiles``
+made right and tailored specifically for ML and Data Science scenarios.
+
+#. ``Git/Git-lfs`` part - DVC helps you storing and sharing data artifacts, models. It connects them with your
+   Git repository.
+#. ``Makefiles`` part - DVC describes how one data or model artifact was build from another data. 
+
+DVC usually runs along with Git. Git is used as usual to store and version code and DVC meta-files. DVC helps
+to store data and model files seamlessly out of Git while preserving almost the same user experience as if they
+were stored in Git itself. To store and share data files cache DVC supports remotes - any cloud (S3, Azure,
+Google Cloud, etc) or any on-premise network storage (via SSH, for example). It does not require 
 
 .. image:: https://dvc.org/static/img/flow.gif
    :target: https://dvc.org/static/img/flow.gif
    :alt: how_dvc_works
 
+DVC pipelines (aka computational graph) feature connects code and data together. In a very explicit way you can
+specify, run, and save information that a certain command with certain dependencies needs to be run to produce
+a model. See the quick start section below or check `Get Started <https://dvc.org/doc/get-started>`_ tutorial to
+learn more.
+
 Quick start
 ===========
 
@@ -66,7 +83,7 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 +-----------------------------------+-------------------------------------------------------------------+
 | Step                              | Command                                                           |
 +===================================+===================================================================+
-| Track code and data together      | | ``$ git add train.py``                                          |
+| Track data                        | | ``$ git add train.py``                                          |
 |                                   | | ``$ dvc add images.zip``                                        |
 +-----------------------------------+-------------------------------------------------------------------+
 | Connect code and data by commands | | ``$ dvc run -d images.zip -o images/ unzip -q images.zip``      |
@@ -79,27 +96,30 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 |                                   | | ``$ git commit -m 'The baseline model'``                        |
 |                                   | | ``$ git push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
-| Share data and ML models          | | ``$ dvc remote add myremote s3://mybucket/image_cnn``           |
-|                                   | | ``$ dvc config core.remote myremote``                           |
+| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``        |
 |                                   | | ``$ dvc push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
 
 Installation
 ============
 
-There are three options to install DVC: ``pip``, Homebrew, or an OS-specific package:
+Read this `instruction <https://dvc.org/doc/get-started/install>`_ to get more details. There are three
+options to install DVC: ``pip``, Homebrew, or an OS-specific package:
 
 pip (PyPI)
 ----------
 
-Stable
-^^^^^^
 .. code-block:: bash
 
    pip install dvc
 
-Development
-^^^^^^^^^^^
+Depending on the remote storage type you plan to use to keep and share your data, you might need to specify
+one of the optional dependencies: ``s3``, ``gs``, ``azure``, ``ssh``. Or ``all_remotes`` to include them all.
+The command should look like this: ``pip install dvc[s3]`` - it installs the ``boto3`` library along with
+DVC to support the AWS S3 storage.
+
+To install the development version, run:
+
 .. code-block:: bash
 
    pip install git+git://github.com/iterative/dvc

dvc/__init__.py

@@ -6,73 +6,12 @@ Make your data science projects reproducible and shareable.
 
 from __future__ import unicode_literals
 
-import os
+from dvc.version import __version__  # noqa: F401
 import warnings
+import dvc.logger
 
 
-VERSION_BASE = "0.32.1"
-__version__ = VERSION_BASE
-
-PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
-HOMEPATH = os.path.dirname(PACKAGEPATH)
-VERSIONPATH = os.path.join(PACKAGEPATH, "version.py")
-
-
-def _update_version_file():
-    """Dynamically update version file."""
-    try:
-        from git import Repo
-        from git.exc import InvalidGitRepositoryError
-    except ImportError:
-        return __version__
-
-    try:
-        repo = Repo(HOMEPATH)
-    except InvalidGitRepositoryError:
-        return __version__
-
-    sha = repo.head.object.hexsha
-    short_sha = repo.git.rev_parse(sha, short=6)
-    dirty = ".mod" if repo.is_dirty() else ""
-    ver = "{}+{}{}".format(__version__, short_sha, dirty)
-
-    # Write a helper file, that will be installed with the package
-    # and will provide a true version of the installed dvc
-    with open(VERSIONPATH, "w+") as fobj:
-        fobj.write("# AUTOGENERATED by dvc/__init__.py\n")
-        fobj.write('version = "{}"\n'.format(ver))
-
-    return ver
-
-
-def _remove_version_file():
-    """Remove version.py so that it doesn't get into the release."""
-    if os.path.exists(VERSIONPATH):
-        os.unlink(VERSIONPATH)
-
-
-if os.path.exists(os.path.join(HOMEPATH, "setup.py")):
-    # dvc is run directly from source without installation or
-    # __version__ is called from setup.py
-    if (
-        os.getenv("APPVEYOR_REPO_TAG", "").lower() != "true"
-        and os.getenv("TRAVIS_TAG", "") == ""
-    ):
-        __version__ = _update_version_file()
-    else:  # pragma: no cover
-        _remove_version_file()
-else:  # pragma: no cover
-    # dvc was installed with pip or something. Hopefully we have our
-    # auto-generated version.py to help us provide a true version
-    try:
-        from dvc.version import version
-
-        __version__ = version
-    except Exception:
-        pass
-
-VERSION = __version__
-
+dvc.logger.setup()
 
 # Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
 # We don't directly import numpy, but our dependency networkx does, causing

dvc/analytics.py

@@ -7,9 +7,12 @@ from dvc.utils.compat import str
 import os
 import json
 import errno
+import logging
 
-import dvc.logger as logger
-from dvc import VERSION
+from dvc import __version__
+
+
+logger = logging.getLogger(__name__)
 
 
 class Analytics(object):
@@ -164,7 +167,7 @@ class Analytics(object):
         from dvc.repo import Repo
         from dvc.exceptions import NotDvcRepoError
 
-        self.info[self.PARAM_DVC_VERSION] = VERSION
+        self.info[self.PARAM_DVC_VERSION] = __version__
         self.info[self.PARAM_IS_BINARY] = is_binary()
         self.info[self.PARAM_USER_ID] = self._get_user_id()
 
@@ -201,6 +204,13 @@ class Analytics(object):
             json.dump(self.info, fobj)
             return fobj.name
 
+    @staticmethod
+    def _is_enabled_config(config):
+        from dvc.config import Config
+
+        core = config.config.get(Config.SECTION_CORE, {})
+        return core.get(Config.SECTION_CORE_ANALYTICS, True)
+
     @staticmethod
     def _is_enabled(cmd=None):
         from dvc.config import Config
@@ -226,10 +236,9 @@ class Analytics(object):
             config = cmd.config
             assert config is not None
 
-        core = config.config.get(Config.SECTION_CORE, {})
-        enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
+        enabled = Analytics._is_enabled_config(config)
         logger.debug(
-            "Analytics is {}abled.".format("en" if enabled else "dis")
+            "Analytics is {}.".format("enabled" if enabled else "disabled")
         )
         return enabled
 

dvc/cli.py

@@ -5,10 +5,11 @@ from __future__ import unicode_literals
 
 import sys
 import argparse
+import logging
 
-import dvc.logger as logger
 from dvc.command.base import fix_subparsers
 import dvc.command.init as init
+import dvc.command.pkg as pkg
 import dvc.command.destroy as destroy
 import dvc.command.remove as remove
 import dvc.command.move as move
@@ -30,12 +31,16 @@ import dvc.command.lock as lock
 import dvc.command.pipeline as pipeline
 import dvc.command.daemon as daemon
 import dvc.command.commit as commit
+import dvc.command.tag as tag
+import dvc.command.diff as diff
 from dvc.exceptions import DvcParserError
-from dvc import VERSION
 
 
+logger = logging.getLogger(__name__)
+
 COMMANDS = [
     init,
+    pkg,
     destroy,
     add,
     remove,
@@ -58,6 +63,8 @@ COMMANDS = [
     pipeline,
     daemon,
     commit,
+    tag,
+    diff,
 ]
 
 
@@ -82,7 +89,9 @@ class VersionAction(argparse.Action):  # pragma: no cover
     """Shows dvc version and exits."""
 
     def __call__(self, parser, namespace, values, option_string=None):
-        print(VERSION)
+        from dvc import __version__
+
+        print(__version__)
         sys.exit(0)
 
 

dvc/command/add.py

@@ -1,8 +1,13 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdAdd(CmdBase):
@@ -20,15 +25,20 @@ class CmdAdd(CmdBase):
                 )
 
         except DvcException:
-            logger.error("failed to add file")
+            logger.exception("failed to add file")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    ADD_HELP = "Add files/directories to dvc."
+    ADD_HELP = "Take data files or directories under DVC control."
+
     add_parser = subparsers.add_parser(
-        "add", parents=[parent_parser], description=ADD_HELP, help=ADD_HELP
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(ADD_HELP, "add"),
+        help=ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     add_parser.add_argument(
         "-R",

dvc/command/base.py

@@ -1,6 +1,10 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import colorama
+import logging
+
+
+logger = logging.getLogger(__name__)
 
 
 def fix_subparsers(subparsers):
@@ -18,6 +22,19 @@ def fix_subparsers(subparsers):
         subparsers.dest = "cmd"
 
 
+def append_doc_link(help_message, path):
+    if not path:
+        return help_message
+    doc_base = "https://man.dvc.org/"
+    return "{message}\ndocumentation: {blue}{base}{path}{nc}".format(
+        message=help_message,
+        base=doc_base,
+        path=path,
+        blue=colorama.Fore.CYAN,
+        nc=colorama.Fore.RESET,
+    )
+
+
 class CmdBase(object):
     def __init__(self, args):
         from dvc.repo import Repo
@@ -25,7 +42,6 @@ class CmdBase(object):
         self.repo = Repo()
         self.config = self.repo.config
         self.args = args
-        self.set_loglevel(args)
 
     @property
     def default_targets(self):
@@ -36,15 +52,6 @@ class CmdBase(object):
         logger.warning(msg)
         return [Stage.STAGE_FILE]
 
-    @staticmethod
-    def set_loglevel(args):
-        """Sets log level from CLI arguments."""
-        if args.quiet:
-            logger.be_quiet()
-
-        elif args.verbose:
-            logger.be_verbose()
-
     def run_cmd(self):
         from dvc.lock import LockError
 
@@ -52,7 +59,7 @@ class CmdBase(object):
             with self.repo.lock:
                 return self.run()
         except LockError:
-            logger.error("failed to lock before running a command")
+            logger.exception("failed to lock before running a command")
             return 1
 
     # Abstract methods that have to be implemented by any inheritance class

dvc/command/cache.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import argparse
 
-from dvc.command.base import fix_subparsers
+from dvc.command.base import fix_subparsers, append_doc_link
 from dvc.command.remote import CmdRemoteAdd
 from dvc.command.config import CmdConfig
 
@@ -21,11 +21,13 @@ def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
     CACHE_HELP = "Manage cache settings."
+
     cache_parser = subparsers.add_parser(
         "cache",
         parents=[parent_parser],
-        description=CACHE_HELP,
+        description=append_doc_link(CACHE_HELP, "cache"),
         help=CACHE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     cache_subparsers = cache_parser.add_subparsers(
@@ -38,11 +40,13 @@ def add_parser(subparsers, parent_parser):
         add_help=False, parents=[parent_config_parser]
     )
     CACHE_DIR_HELP = "Configure cache directory location."
+
     cache_dir_parser = cache_subparsers.add_parser(
         "dir",
         parents=[parent_cache_config_parser],
-        description=CACHE_DIR_HELP,
+        description=append_doc_link(CACHE_HELP, "cache-dir"),
         help=CACHE_DIR_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     cache_dir_parser.add_argument(
         "-u",
@@ -53,7 +57,6 @@ def add_parser(subparsers, parent_parser):
     )
     cache_dir_parser.add_argument(
         "value",
-        nargs="?",
         default=None,
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "

dvc/command/checkout.py

@@ -1,6 +1,8 @@
 from __future__ import unicode_literals
 
-from dvc.command.base import CmdBase
+import argparse
+
+from dvc.command.base import CmdBase, append_doc_link
 
 
 class CmdCheckout(CmdBase):
@@ -13,17 +15,20 @@ class CmdCheckout(CmdBase):
                     target=target,
                     with_deps=self.args.with_deps,
                     force=self.args.force,
+                    recursive=self.args.recursive,
                 )
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     CHECKOUT_HELP = "Checkout data files from cache."
+
     checkout_parser = subparsers.add_parser(
         "checkout",
         parents=[parent_parser],
-        description=CHECKOUT_HELP,
+        description=append_doc_link(CHECKOUT_HELP, "checkout"),
         help=CHECKOUT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     checkout_parser.add_argument(
         "-d",
@@ -39,5 +44,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Do not prompt when removing working directory files.",
     )
+    checkout_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Checkout all subdirectories of the specified directory.",
+    )
     checkout_parser.add_argument("targets", nargs="*", help="DVC files.")
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -1,8 +1,13 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdCommit(CmdBase):
@@ -19,7 +24,7 @@ class CmdCommit(CmdBase):
                     force=self.args.force,
                 )
             except DvcException:
-                logger.error(
+                logger.exception(
                     "failed to commit{}".format(
                         (" " + target) if target else ""
                     )
@@ -29,12 +34,14 @@ class CmdCommit(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    COMMIT_HELP = "Record changes to the repository."
+    COMMIT_HELP = "Save changed data to cache and update DVC files."
+
     commit_parser = subparsers.add_parser(
         "commit",
         parents=[parent_parser],
-        description=COMMIT_HELP,
+        description=append_doc_link(COMMIT_HELP, "commit"),
         help=COMMIT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     commit_parser.add_argument(
         "-f",

dvc/command/config.py

@@ -2,13 +2,16 @@ from __future__ import unicode_literals
 
 import os
 import argparse
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.config import Config
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdConfig(CmdBase):
     def __init__(self, args):
         from dvc.repo import Repo, NotDvcRepoError
@@ -47,7 +50,7 @@ class CmdConfig(CmdBase):
             self.config.unset(configobj, section, opt)
             self.config.save(configobj)
         except DvcException:
-            logger.error("failed to unset '{}'".format(self.args.name))
+            logger.exception("failed to unset '{}'".format(self.args.name))
             return 1
         return 0
 
@@ -55,7 +58,7 @@ class CmdConfig(CmdBase):
         try:
             self.config.show(self.configobj, section, opt)
         except DvcException:
-            logger.error("failed to show '{}'".format(self.args.name))
+            logger.exception("failed to show '{}'".format(self.args.name))
             return 1
         return 0
 
@@ -64,7 +67,7 @@ class CmdConfig(CmdBase):
             self.config.set(self.configobj, section, opt, value)
             self.config.save(self.configobj)
         except DvcException:
-            logger.error(
+            logger.exception(
                 "failed to set '{}.{}' to '{}'".format(section, opt, value)
             )
             return 1
@@ -99,11 +102,13 @@ parent_config_parser.add_argument(
 
 def add_parser(subparsers, parent_parser):
     CONFIG_HELP = "Get or set config options."
+
     config_parser = subparsers.add_parser(
         "config",
         parents=[parent_config_parser, parent_parser],
-        description=CONFIG_HELP,
+        description=append_doc_link(CONFIG_HELP, "config"),
         help=CONFIG_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     config_parser.add_argument(
         "-u",

dvc/command/daemon.py

@@ -7,7 +7,6 @@ class CmdDaemonBase(CmdBase):
     def __init__(self, args):
         self.args = args
         self.config = None
-        self.set_loglevel(args)
 
     def run_cmd(self):
         return self.run()
@@ -43,7 +42,7 @@ def add_parser(subparsers, parent_parser):
         "daemon",
         parents=[parent_parser],
         description=DAEMON_HELP,
-        help=DAEMON_HELP,
+        add_help=False,
     )
 
     daemon_subparsers = daemon_parser.add_subparsers(

dvc/command/data_sync.py

@@ -1,9 +1,12 @@
 from __future__ import unicode_literals
 
 import argparse
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
@@ -43,7 +46,7 @@ class CmdDataPull(CmdDataBase):
                 recursive=self.args.recursive,
             )
         except Exception:
-            logger.error("failed to pull data from the cloud")
+            logger.exception("failed to pull data from the cloud")
             return 1
         self.check_up_to_date(processed_files_count)
         return 0
@@ -63,7 +66,7 @@ class CmdDataPush(CmdDataBase):
                 recursive=self.args.recursive,
             )
         except Exception:
-            logger.error("failed to push data to the cloud")
+            logger.exception("failed to push data to the cloud")
             return 1
         self.check_up_to_date(processed_files_count)
         return 0
@@ -83,7 +86,7 @@ class CmdDataFetch(CmdDataBase):
                 recursive=self.args.recursive,
             )
         except Exception:
-            logger.error("failed to fetch data from the cloud")
+            logger.exception("failed to fetch data from the cloud")
             return 1
         self.check_up_to_date(processed_files_count)
         return 0
@@ -120,12 +123,14 @@ def add_parser(subparsers, _parent_parser):
     from dvc.command.status import CmdDataStatus
 
     # Pull
-    PULL_HELP = "Pull data files from the cloud."
+    PULL_HELP = "Pull data files from a DVC remote storage."
+
     pull_parser = subparsers.add_parser(
         "pull",
         parents=[shared_parent_parser()],
-        description=PULL_HELP,
+        description=append_doc_link(PULL_HELP, "pull"),
         help=PULL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
         "-r", "--remote", help="Remote repository to pull from."
@@ -168,12 +173,14 @@ def add_parser(subparsers, _parent_parser):
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
-    PUSH_HELP = "Push data files to the cloud."
+    PUSH_HELP = "Push data files to a DVC remote storage."
+
     push_parser = subparsers.add_parser(
         "push",
         parents=[shared_parent_parser()],
-        description=PUSH_HELP,
+        description=append_doc_link(PUSH_HELP, "push"),
         help=PUSH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
         "-r", "--remote", help="Remote repository to push to."
@@ -209,12 +216,14 @@ def add_parser(subparsers, _parent_parser):
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
-    FETCH_HELP = "Fetch data files from the cloud."
+    FETCH_HELP = "Fetch data files from a DVC remote storage."
+
     fetch_parser = subparsers.add_parser(
         "fetch",
         parents=[shared_parent_parser()],
-        description=FETCH_HELP,
+        description=append_doc_link(FETCH_HELP, "fetch"),
         help=FETCH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
         "-r", "--remote", help="Remote repository to fetch from."
@@ -250,13 +259,17 @@ def add_parser(subparsers, _parent_parser):
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
-    STATUS_HELP = "Show the repo status."
+    STATUS_HELP = (
+        "Show changed stages, compare local cache and a remote storage."
+    )
+
     status_parser = subparsers.add_parser(
         "status",
         parents=[shared_parent_parser()],
-        description=STATUS_HELP,
+        description=append_doc_link(STATUS_HELP, "status"),
         help=STATUS_HELP,
         conflict_handler="resolve",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     status_parser.add_argument(
         "-q",

dvc/command/destroy.py

@@ -1,11 +1,16 @@
 from __future__ import unicode_literals
 
+import argparse
+import logging
+
 import dvc.prompt as prompt
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdDestroy(CmdBase):
     def run_cmd(self):
         try:
@@ -24,21 +29,20 @@ class CmdDestroy(CmdBase):
 
             self.repo.destroy()
         except Exception:
-            logger.error("failed to destroy DVC")
+            logger.exception("failed to destroy DVC")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    DESTROY_HELP = (
-        "Destroy dvc. Will remove all repo's information, "
-        "data files and cache."
-    )
+    DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
+
     destroy_parser = subparsers.add_parser(
         "destroy",
         parents=[parent_parser],
-        description=DESTROY_HELP,
+        description=append_doc_link(DESTROY_HELP, "destroy"),
         help=DESTROY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     destroy_parser.add_argument(
         "-f",

dvc/command/diff.py

@@ -0,0 +1,155 @@
+from __future__ import unicode_literals
+
+import humanize
+import inflect
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+from dvc.utils.collections import compact
+import dvc.repo.diff as diff
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdDiff(CmdBase):
+    def _print_size(self, size):
+        if size < 0:
+            change = "decreased by {}"
+        elif size > 0:
+            change = "increased by {}"
+        else:
+            change = "not changed"
+        natur_size = humanize.naturalsize(abs(size))
+        return change.format(natur_size)
+
+    def _get_md5_string(self, sign, file_name, checksum):
+        sample_msg = ""
+        if file_name:
+            sample_msg = "{}{} with md5 {}\n"
+            sample_msg = sample_msg.format(sign, file_name, checksum)
+        return sample_msg
+
+    def _get_dir_changes(self, dct):
+        engine = inflect.engine()
+        changes_msg = (
+            "{} {} not changed, {} {} modified, {} {} added, "
+            "{} {} deleted, size was {}"
+        )
+        changes_msg = changes_msg.format(
+            dct[diff.DIFF_IDENT],
+            engine.plural("file", dct[diff.DIFF_IDENT]),
+            dct[diff.DIFF_CHANGE],
+            engine.plural("file", dct[diff.DIFF_CHANGE]),
+            dct[diff.DIFF_NEW],
+            engine.plural("file", dct[diff.DIFF_NEW]),
+            dct[diff.DIFF_DEL],
+            engine.plural("file", dct[diff.DIFF_DEL]),
+            self._print_size(dct[diff.DIFF_SIZE]),
+        )
+        return changes_msg
+
+    def _get_file_changes(self, dct):
+        if (
+            dct.get(diff.DIFF_OLD_FILE)
+            and dct.get(diff.DIFF_NEW_FILE)
+            and dct[diff.DIFF_SIZE] == 0
+        ):
+            msg = "file size was not changed"
+        elif dct.get(diff.DIFF_NEW_FILE):
+            msg = "added file with size {}".format(
+                humanize.naturalsize(dct[diff.DIFF_SIZE])
+            )
+        elif dct.get(diff.DIFF_OLD_FILE):
+            msg = "deleted file with size {}".format(
+                humanize.naturalsize(abs(dct[diff.DIFF_SIZE]))
+            )
+        else:
+            msg = "file was modified, file size {}".format(
+                self._print_size(dct[diff.DIFF_SIZE])
+            )
+        return msg
+
+    def _get_royal_changes(self, dct):
+        if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
+            if dct.get("is_dir"):
+                return self._get_dir_changes(dct)
+            else:
+                return self._get_file_changes(dct)
+        return "size is ?"
+
+    def _show(self, diff_dct):
+        msg = "dvc diff from {} to {}".format(
+            diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
+        )
+        if diff_dct.get(diff.DIFF_EQUAL):
+            logger.info(msg)
+            return
+        for dct in diff_dct[diff.DIFF_LIST]:
+            msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET])
+            msg += self._get_md5_string(
+                "-",
+                dct.get(diff.DIFF_OLD_FILE),
+                dct.get(diff.DIFF_OLD_CHECKSUM),
+            )
+            msg += self._get_md5_string(
+                "+",
+                dct.get(diff.DIFF_NEW_FILE),
+                dct.get(diff.DIFF_NEW_CHECKSUM),
+            )
+            msg += "\n"
+            msg += self._get_royal_changes(dct)
+        logger.info(msg)
+        return msg
+
+    def run(self):
+        try:
+            msg = self.repo.diff(
+                self.args.a_ref, target=self.args.target, b_ref=self.args.b_ref
+            )
+            self._show(msg)
+        except DvcException:
+            msg = "failed to get 'diff {}'"
+            args = " ".join(
+                compact([self.args.target, self.args.a_ref, self.args.b_ref])
+            )
+            msg = msg.format(args)
+            logger.exception(msg)
+            return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    description = (
+        "Show diff of a data file or a directory that "
+        "is under DVC control. Some basic statistics "
+        "summary, how many files were deleted/changed."
+    )
+    help = "Show a diff of a DVC controlled data file or a directory."
+    diff_parser = subparsers.add_parser(
+        "diff", parents=[parent_parser], description=description, help=help
+    )
+    diff_parser.add_argument(
+        "-t",
+        "--target",
+        default=None,
+        help=(
+            "Source path to a data file or directory. Default None,"
+            "If not specified, compares all files and directories "
+            "that are under DVC control in the current working space."
+        ),
+    )
+    diff_parser.add_argument(
+        "a_ref", help="Git reference from which diff calculates"
+    )
+    diff_parser.add_argument(
+        "b_ref",
+        help=(
+            "Git reference till which diff calculates, if omitted "
+            "diff shows the difference between current HEAD and a_ref"
+        ),
+        nargs="?",
+        default=None,
+    )
+    diff_parser.set_defaults(func=CmdDiff)

dvc/command/gc.py

@@ -1,10 +1,14 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
 import dvc.prompt as prompt
-import dvc.logger as logger
+import logging
 
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdGC(CmdBase):
@@ -46,13 +50,17 @@ class CmdGC(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    GC_HELP = (
-        "Collect garbage from your DVC cache. \n"
-        "Deletes all files in the cache which are not in use by the "
-        "specified git references (defaults to just HEAD)."
+    GC_HELP = "Collect unused data from DVC cache or a remote storage."
+    GC_DESCRIPTION = (
+        "Deletes all files in the cache or a remote which are not in\n"
+        "use by the specified git references (defaults to just HEAD)."
     )
     gc_parser = subparsers.add_parser(
-        "gc", parents=[parent_parser], description=GC_HELP, help=GC_HELP
+        "gc",
+        parents=[parent_parser],
+        description=append_doc_link(GC_DESCRIPTION, "gc"),
+        help=GC_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     gc_parser.add_argument(
         "-a",
@@ -76,7 +84,7 @@ def add_parser(subparsers, parent_parser):
         help="Collect garbage in remote repository.",
     )
     gc_parser.add_argument(
-        "-r", "--remote", help="Remote repository to collect garbage in."
+        "-r", "--remote", help="Remote storage to collect garbage in."
     )
     gc_parser.add_argument(
         "-f",

dvc/command/imp.py

@@ -1,11 +1,15 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
+import logging
 
-import dvc.logger as logger
 from dvc.utils.compat import urlparse
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdImport(CmdBase):
@@ -17,7 +21,7 @@ class CmdImport(CmdBase):
 
             self.repo.imp(self.args.url, out, self.args.resume)
         except DvcException:
-            logger.error(
+            logger.exception(
                 "failed to import {}. You could also try downloading "
                 "it manually and adding it with `dvc add` command.".format(
                     self.args.url
@@ -28,25 +32,26 @@ class CmdImport(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    IMPORT_HELP = "Import files from URL."
+    IMPORT_HELP = "Download or copy files from URL and take under DVC control."
+
     import_parser = subparsers.add_parser(
         "import",
         parents=[parent_parser],
-        description=IMPORT_HELP,
+        description=append_doc_link(IMPORT_HELP, "import"),
         help=IMPORT_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     import_parser.add_argument(
         "url",
-        help="URL. Supported urls: "
-        "'/path/to/file', "
-        "'C:\\\\path\\to\\file', "
-        "'https://example.com/path/to/file', "
-        "'s3://bucket/path/to/file', "
-        "'gs://bucket/path/to/file', "
-        "'hdfs://example.com/path/to/file', "
-        "'ssh://example.com:/path/to/file', "
-        "'remote://myremote/path/to/file'(see "
-        "`dvc remote` commands). ",
+        help="Supported urls:\n"
+        "/path/to/file\n"
+        "C:\\\\path\\to\\file\n"
+        "https://example.com/path/to/file\n"
+        "s3://bucket/path/to/file\n"
+        "gs://bucket/path/to/file\n"
+        "hdfs://example.com/path/to/file\n"
+        "ssh://example.com:/path/to/file\n"
+        "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(
         "--resume",
@@ -54,5 +59,7 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Resume previously started download.",
     )
-    import_parser.add_argument("out", nargs="?", help="Output.")
+    import_parser.add_argument(
+        "out", nargs="?", help="Destination path to put files to."
+    )
     import_parser.set_defaults(func=CmdImport)

dvc/command/init.py

@@ -1,6 +1,12 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
+from dvc.command.base import append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdInit(object):
@@ -17,18 +23,25 @@ class CmdInit(object):
             )
             self.config = self.repo.config
         except InitError:
-            logger.error("failed to initiate dvc")
+            logger.exception("failed to initiate dvc")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     """Setup parser for `dvc init`."""
-    INIT_HELP = (
-        "Initialize dvc over a directory " "(should already be a git dir)."
+    INIT_HELP = "Initialize DVC in the current directory."
+    INIT_DESCRIPTION = (
+        "Initialize DVC in the current directory. Expects directory\n"
+        "to be a Git repository unless --no-scm option is specified."
     )
+
     init_parser = subparsers.add_parser(
-        "init", parents=[parent_parser], description=INIT_HELP, help=INIT_HELP
+        "init",
+        parents=[parent_parser],
+        description=append_doc_link(INIT_DESCRIPTION, "init"),
+        help=INIT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     init_parser.add_argument(
         "--no-scm",
@@ -42,6 +55,9 @@ def add_parser(subparsers, parent_parser):
         "--force",
         action="store_true",
         default=False,
-        help="Overwrite '.dvc' if it exists. Will remove all local cache.",
+        help=(
+            "Overwrite existing '.dvc' directory. "
+            "This operation removes local cache."
+        ),
     )
     init_parser.set_defaults(func=CmdInit)

dvc/command/install.py

@@ -1,7 +1,12 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdInstall(CmdBase):
@@ -9,17 +14,18 @@ class CmdInstall(CmdBase):
         try:
             self.repo.install()
         except Exception:
-            logger.error("failed to install dvc hooks")
+            logger.exception("failed to install dvc hooks")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    INSTALL_HELP = "Install dvc hooks into the repository."
+    INSTALL_HELP = "Install DVC git hooks into the repository."
     install_parser = subparsers.add_parser(
         "install",
         parents=[parent_parser],
-        description=INSTALL_HELP,
+        description=append_doc_link(INSTALL_HELP, "install"),
         help=INSTALL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     install_parser.set_defaults(func=CmdInstall)

dvc/command/lock.py

@@ -1,17 +1,22 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdLockBase(CmdBase):
-    def run(self, unlock=False):
+    def _run(self, unlock):
         for target in self.args.targets:
             try:
                 self.repo.lock_stage(target, unlock=unlock)
             except DvcException:
-                logger.error(
+                logger.exception(
                     "failed to {}lock '{}'".format(
                         "un" if unlock else "", target
                     )
@@ -23,18 +28,22 @@ class CmdLockBase(CmdBase):
 
 class CmdLock(CmdLockBase):
     def run(self):
-        return super(CmdLock, self).run(False)
+        return self._run(False)
 
 
 class CmdUnlock(CmdLockBase):
     def run(self):
-        return super(CmdUnlock, self).run(True)
+        return self._run(True)
 
 
 def add_parser(subparsers, parent_parser):
     LOCK_HELP = "Lock DVC file."
     lock_parser = subparsers.add_parser(
-        "lock", parents=[parent_parser], description=LOCK_HELP, help=LOCK_HELP
+        "lock",
+        parents=[parent_parser],
+        description=append_doc_link(LOCK_HELP, "lock"),
+        help=LOCK_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     lock_parser.add_argument("targets", nargs="+", help="DVC files.")
     lock_parser.set_defaults(func=CmdLock)
@@ -43,8 +52,9 @@ def add_parser(subparsers, parent_parser):
     unlock_parser = subparsers.add_parser(
         "unlock",
         parents=[parent_parser],
-        description=UNLOCK_HELP,
+        description=append_doc_link(UNLOCK_HELP, "unlock"),
         help=UNLOCK_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unlock_parser.add_argument("targets", nargs="+", help="DVC files.")
     unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/metrics.py

@@ -1,19 +1,39 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+def show_metrics(metrics, all_branches=False, all_tags=False):
+    """
+    Args:
+        metrics (list): Where each element is either a `list`
+            if an xpath was specified, otherwise a `str`
+    """
+    for branch, val in metrics.items():
+        if all_branches or all_tags:
+            logger.info("{branch}:".format(branch=branch))
+
+        for fname, metric in val.items():
+            lines = metric if type(metric) is list else metric.splitlines()
+
+            if len(lines) > 1:
+                logger.info("\t{fname}:".format(fname=fname))
+
+                for line in lines:
+                    logger.info("\t\t{content}".format(content=line))
+
+            else:
+                logger.info("\t{}: {}".format(fname, metric))
 
 
 class CmdMetricsShow(CmdBase):
-    def _show(self, metrics):
-        for branch, val in metrics.items():
-            if self.args.all_branches or self.args.all_tags:
-                logger.info("{}:".format(branch))
-
-            for fname, metric in val.items():
-                logger.info("\t{}: {}".format(fname, metric))
-
     def run(self):
         typ = self.args.type
         xpath = self.args.xpath
@@ -27,9 +47,9 @@ class CmdMetricsShow(CmdBase):
                 recursive=self.args.recursive,
             )
 
-            self._show(metrics)
+            show_metrics(metrics, self.args.all_branches, self.args.all_tags)
         except DvcException:
-            logger.error("failed to show metrics")
+            logger.exception("failed to show metrics")
             return 1
 
         return 0
@@ -42,7 +62,7 @@ class CmdMetricsModify(CmdBase):
                 self.args.path, typ=self.args.type, xpath=self.args.xpath
             )
         except DvcException:
-            logger.error("failed to modify metric file settings")
+            logger.exception("failed to modify metric file settings")
             return 1
 
         return 0
@@ -56,7 +76,7 @@ class CmdMetricsAdd(CmdBase):
             )
         except DvcException:
             msg = "failed to add metric file '{}'".format(self.args.path)
-            logger.error(msg)
+            logger.exception(msg)
             return 1
 
         return 0
@@ -68,22 +88,21 @@ class CmdMetricsRemove(CmdBase):
             self.repo.metrics.remove(self.args.path)
         except DvcException:
             msg = "failed to remove metric file '{}'".format(self.args.path)
-            logger.error(msg)
+            logger.exception(msg)
             return 1
 
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = (
-        "A set of commands to add, manage, collect and display project "
-        "metrics."
-    )
+    METRICS_HELP = "Commands to add, manage, collect and display metrics."
+
     metrics_parser = subparsers.add_parser(
         "metrics",
         parents=[parent_parser],
-        description=METRICS_HELP,
+        description=append_doc_link(METRICS_HELP, "metrics"),
         help=METRICS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     metrics_subparsers = metrics_parser.add_subparsers(
@@ -97,14 +116,21 @@ def add_parser(subparsers, parent_parser):
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
         parents=[parent_parser],
-        description=METRICS_SHOW_HELP,
+        description=append_doc_link(METRICS_SHOW_HELP, "metrics-show"),
         help=METRICS_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_show_parser.add_argument(
         "path", nargs="?", help="Path to a metric file or a directory."
     )
     metrics_show_parser.add_argument(
-        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
+        "-t",
+        "--type",
+        help=(
+            "Type of metrics (json/tsv/htsv/csv/hcsv). "
+            "It can be detected by the file extension automatically. "
+            "Unsupported types will be treated as raw."
+        ),
     )
     metrics_show_parser.add_argument(
         "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
@@ -139,8 +165,9 @@ def add_parser(subparsers, parent_parser):
     metrics_add_parser = metrics_subparsers.add_parser(
         "add",
         parents=[parent_parser],
-        description=METRICS_ADD_HELP,
+        description=append_doc_link(METRICS_ADD_HELP, "metrics-add"),
         help=METRICS_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_add_parser.add_argument(
         "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
@@ -155,8 +182,9 @@ def add_parser(subparsers, parent_parser):
     metrics_modify_parser = metrics_subparsers.add_parser(
         "modify",
         parents=[parent_parser],
-        description=METRICS_MODIFY_HELP,
+        description=append_doc_link(METRICS_MODIFY_HELP, "metrics-modify"),
         help=METRICS_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_modify_parser.add_argument(
         "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
@@ -171,8 +199,9 @@ def add_parser(subparsers, parent_parser):
     metrics_remove_parser = metrics_subparsers.add_parser(
         "remove",
         parents=[parent_parser],
-        description=METRICS_REMOVE_HELP,
+        description=append_doc_link(METRICS_REMOVE_HELP, "metrics-remove"),
         help=METRICS_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_remove_parser.add_argument("path", help="Path to a metric file.")
     metrics_remove_parser.set_defaults(func=CmdMetricsRemove)

dvc/command/move.py

@@ -1,8 +1,13 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdMove(CmdBase):
@@ -13,20 +18,25 @@ class CmdMove(CmdBase):
             msg = "failed to move '{}' -> '{}'".format(
                 self.args.src, self.args.dst
             )
-            logger.error(msg)
+            logger.exception(msg)
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    description = (
-        "Rename or move a data file or a directory that "
-        "is under DVC control. It renames and modifies "
-        "the corresponding DVC file to reflect the changes."
+    MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
+    MOVE_DESCRIPTION = (
+        "Rename or move a DVC controlled data file or a directory.\n"
+        "It renames and modifies the corresponding DVC file to reflect the"
+        " changes."
     )
-    help = "Rename or move a DVC controlled data file or a directory."
+
     move_parser = subparsers.add_parser(
-        "move", parents=[parent_parser], description=description, help=help
+        "move",
+        parents=[parent_parser],
+        description=append_doc_link(MOVE_DESCRIPTION, "move"),
+        help=MOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
         "src", help="Source path to a data file or directory."

dvc/command/pipeline.py

@@ -2,11 +2,15 @@ from __future__ import unicode_literals
 
 from dvc.utils.compat import str
 
+import argparse
 import os
+import logging
 
-import dvc.logger as logger
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase, fix_subparsers
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdPipelineShow(CmdBase):
@@ -71,23 +75,47 @@ class CmdPipelineShow(CmdBase):
             else:
                 edges.append((from_stage.relpath, to_stage.relpath))
 
-        return nodes, edges
+        return nodes, edges, networkx.is_tree(G)
 
     def _show_ascii(self, target, commands, outs):
         from dvc.dagascii import draw
 
-        nodes, edges = self.__build_graph(target, commands, outs)
+        nodes, edges, _ = self.__build_graph(target, commands, outs)
 
         if not nodes:
             return
 
         draw(nodes, edges)
 
+    def _show_dependencies_tree(self, target, commands, outs):
+        from treelib import Tree
+
+        nodes, edges, is_tree = self.__build_graph(target, commands, outs)
+        if not nodes:
+            return
+        if not is_tree:
+            raise DvcException(
+                "DAG is not a tree, can not print it in tree-structure way, "
+                "please use --ascii instead"
+            )
+
+        tree = Tree()
+        tree.create_node(target, target)  # Root node
+        observe_list = [target]
+        while len(observe_list) > 0:
+            current_root = observe_list[0]
+            for edge in edges:
+                if edge[0] == current_root:
+                    tree.create_node(edge[1], edge[1], parent=current_root)
+                    observe_list.append(edge[1])
+            observe_list.pop(0)
+        tree.show()
+
     def __write_dot(self, target, commands, outs, filename):
         import networkx
         from networkx.drawing.nx_pydot import write_dot
 
-        _, edges = self.__build_graph(target, commands, outs)
+        _, edges, _ = self.__build_graph(target, commands, outs)
         edges = [edge[::-1] for edge in edges]
 
         simple_g = networkx.DiGraph()
@@ -111,11 +139,15 @@ class CmdPipelineShow(CmdBase):
                         self.args.outs,
                         self.args.dot,
                     )
+                elif self.args.tree:
+                    self._show_dependencies_tree(
+                        target, self.args.commands, self.args.outs
+                    )
                 else:
                     self._show(target, self.args.commands, self.args.outs)
             except DvcException:
                 msg = "failed to show pipeline for '{}'".format(target)
-                logger.error(msg)
+                logger.exception(msg)
                 return 1
         return 0
 
@@ -137,12 +169,13 @@ class CmdPipelineList(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    PIPELINE_HELP = "Manage pipeline."
+    PIPELINE_HELP = "Manage pipelines."
     pipeline_parser = subparsers.add_parser(
         "pipeline",
         parents=[parent_parser],
-        description=PIPELINE_HELP,
+        description=append_doc_link(PIPELINE_HELP, "pipeline"),
         help=PIPELINE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     pipeline_subparsers = pipeline_parser.add_subparsers(
@@ -156,8 +189,9 @@ def add_parser(subparsers, parent_parser):
     pipeline_show_parser = pipeline_subparsers.add_parser(
         "show",
         parents=[parent_parser],
-        description=PIPELINE_SHOW_HELP,
+        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline-show"),
         help=PIPELINE_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
     pipeline_show_group.add_argument(
@@ -183,6 +217,12 @@ def add_parser(subparsers, parent_parser):
     pipeline_show_parser.add_argument(
         "--dot", help="Write DAG in .dot format."
     )
+    pipeline_show_parser.add_argument(
+        "--tree",
+        action="store_true",
+        default=False,
+        help="Output DAG as Dependencies Tree.",
+    )
     pipeline_show_parser.add_argument(
         "targets", nargs="*", help="DVC files. 'Dvcfile' by default."
     )
@@ -192,7 +232,8 @@ def add_parser(subparsers, parent_parser):
     pipeline_list_parser = pipeline_subparsers.add_parser(
         "list",
         parents=[parent_parser],
-        description=PIPELINE_LIST_HELP,
+        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline-list"),
         help=PIPELINE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pipeline_list_parser.set_defaults(func=CmdPipelineList)

dvc/command/pkg.py

@@ -0,0 +1,91 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+from dvc.repo.pkg import PackageParams
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdPkg(CmdBase):
+    def run(self, unlock=False):
+        try:
+            pkg_params = PackageParams(
+                self.args.address,
+                self.args.target_dir,
+                self.args.select,
+                self.args.file,
+            )
+            return self.repo.install_pkg(pkg_params)
+        except DvcException:
+            logger.exception(
+                "failed to install package '{}'".format(self.args.address)
+            )
+            return 1
+        pass
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    PKG_HELP = "Manage DVC packages."
+    pkg_parser = subparsers.add_parser(
+        "pkg",
+        parents=[parent_parser],
+        description=append_doc_link(PKG_HELP, "pkg"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        add_help=False,
+    )
+
+    pkg_subparsers = pkg_parser.add_subparsers(
+        dest="cmd", help="Use dvc pkg CMD --help for command-specific help."
+    )
+
+    fix_subparsers(pkg_subparsers)
+
+    PKG_INSTALL_HELP = "Install package."
+    pkg_install_parser = pkg_subparsers.add_parser(
+        "install",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(PKG_INSTALL_HELP, "pkg-install"),
+        help=PKG_INSTALL_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    pkg_install_parser.add_argument(
+        "address",
+        nargs="?",
+        default="",
+        help="Package address: git://<url> or https://github.com/...",
+    )
+    pkg_install_parser.add_argument(
+        "target_dir",
+        metavar="target",
+        nargs="?",
+        default=".",
+        help="Target directory to deploy package outputs. "
+        "Default value is the current dir.",
+    )
+    pkg_install_parser.add_argument(
+        "-s",
+        "--select",
+        metavar="OUT",
+        action="append",
+        default=[],
+        help="Select and persist only specified outputs from a package. "
+        "The parameter can be used multiple times. "
+        "All outputs will be selected by default.",
+    )
+    pkg_install_parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the stage file. It should be "
+        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+        "'prepare.dvc', 'clean.dvc', etc). "
+        "By default the file has 'mod_' prefix and imported package name "
+        "followed by .dvc",
+    )
+    pkg_install_parser.set_defaults(func=CmdPkg)

dvc/command/remote.py

@@ -1,12 +1,16 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
 import re
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import fix_subparsers
-from dvc.config import Config
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
+from dvc.config import Config
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdRemoteAdd(CmdConfig):
@@ -38,6 +42,14 @@ class CmdRemoteAdd(CmdConfig):
             )
 
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
+        if (section in self.configobj.keys()) and not self.args.force:
+            logger.error(
+                "Remote with name {} already exists. "
+                "Use -f (--force) to overwrite remote "
+                "with new value".format(self.args.name)
+            )
+            return 1
+
         ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
         if ret != 0:
             return ret
@@ -119,12 +131,13 @@ class CmdRemoteList(CmdConfig):
 def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
-    REMOTE_HELP = "Manage set of tracked repositories."
+    REMOTE_HELP = "Manage remote storage configuration."
     remote_parser = subparsers.add_parser(
         "remote",
         parents=[parent_parser],
-        description=REMOTE_HELP,
+        description=append_doc_link(REMOTE_HELP, "remote"),
         help=REMOTE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     remote_subparsers = remote_parser.add_subparsers(
@@ -138,14 +151,14 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser = remote_subparsers.add_parser(
         "add",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_ADD_HELP,
+        description=append_doc_link(REMOTE_ADD_HELP, "remote-add"),
         help=REMOTE_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_add_parser.add_argument("name", help="Name.")
     remote_add_parser.add_argument(
         "url",
-        help="URL. See full list of supported urls at "
-        "https://dvc.org/doc/commands-reference/remote",
+        help="URL. See full list of supported urls at " "man.dvc.org/remote",
     )
     remote_add_parser.add_argument(
         "-d",
@@ -154,14 +167,22 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Set as default remote.",
     )
+    remote_add_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force overwriting existing configs",
+    )
     remote_add_parser.set_defaults(func=CmdRemoteAdd)
 
     REMOTE_DEFAULT_HELP = "Set/unset default remote."
     remote_default_parser = remote_subparsers.add_parser(
         "default",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_DEFAULT_HELP,
+        description=append_doc_link(REMOTE_DEFAULT_HELP, "remote-default"),
         help=REMOTE_DEFAULT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_default_parser.add_argument(
         "name", nargs="?", help="Name of the remote."
@@ -179,8 +200,9 @@ def add_parser(subparsers, parent_parser):
     remote_remove_parser = remote_subparsers.add_parser(
         "remove",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_REMOVE_HELP,
+        description=append_doc_link(REMOTE_REMOVE_HELP, "remote-remove"),
         help=REMOTE_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_remove_parser.add_argument("name", help="Name")
     remote_remove_parser.set_defaults(func=CmdRemoteRemove)
@@ -189,8 +211,9 @@ def add_parser(subparsers, parent_parser):
     remote_modify_parser = remote_subparsers.add_parser(
         "modify",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_MODIFY_HELP,
+        description=append_doc_link(REMOTE_MODIFY_HELP, "remote-modify"),
         help=REMOTE_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_modify_parser.add_argument("name", help="Name.")
     remote_modify_parser.add_argument("option", help="Option.")
@@ -204,11 +227,12 @@ def add_parser(subparsers, parent_parser):
     )
     remote_modify_parser.set_defaults(func=CmdRemoteModify)
 
-    REMOTE_LIST_HELP = "List remotes."
+    REMOTE_LIST_HELP = "List available remotes."
     remote_list_parser = remote_subparsers.add_parser(
         "list",
         parents=[parent_config_parser, parent_parser],
-        description=REMOTE_LIST_HELP,
+        description=append_doc_link(REMOTE_LIST_HELP, "remote-list"),
         help=REMOTE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_list_parser.set_defaults(func=CmdRemoteList)

dvc/command/remove.py

@@ -1,9 +1,14 @@
 from __future__ import unicode_literals
 
+import argparse
+import logging
+
 import dvc.prompt as prompt
-import dvc.logger as logger
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdRemove(CmdBase):
@@ -32,7 +37,7 @@ class CmdRemove(CmdBase):
                 outs_only = self._is_outs_only(target)
                 self.repo.remove(target, outs_only=outs_only)
             except DvcException:
-                logger.error("failed to remove {}".format(target))
+                logger.exception("failed to remove {}".format(target))
                 return 1
         return 0
 
@@ -42,8 +47,9 @@ def add_parser(subparsers, parent_parser):
     remove_parser = subparsers.add_parser(
         "remove",
         parents=[parent_parser],
-        description=REMOVE_HELP,
+        description=append_doc_link(REMOVE_HELP, "remove"),
         help=REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remove_parser_group = remove_parser.add_mutually_exclusive_group()
     remove_parser_group.add_argument(

dvc/command/repro.py

@@ -1,13 +1,18 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdRepro(CmdBase):
     def run(self):
         recursive = not self.args.single_item
@@ -40,9 +45,10 @@ class CmdRepro(CmdBase):
                     logger.info(CmdDataStatus.UP_TO_DATE_MSG)
 
                 if self.args.metrics:
-                    self.repo.metrics.show()
+                    metrics = self.repo.metrics.show()
+                    show_metrics(metrics)
             except DvcException:
-                logger.error()
+                logger.exception()
                 ret = 1
                 break
 
@@ -51,15 +57,18 @@ class CmdRepro(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'."
+    REPRO_HELP = "Check for changes and reproduce DVC file and dependencies."
     repro_parser = subparsers.add_parser(
         "repro",
         parents=[parent_parser],
-        description=REPRO_HELP,
+        description=append_doc_link(REPRO_HELP, "repro"),
         help=REPRO_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     repro_parser.add_argument(
-        "targets", nargs="*", help="DVC file to reproduce."
+        "targets",
+        nargs="*",
+        help="DVC file to reproduce (default - 'Dvcfile').",
     )
     repro_parser.add_argument(
         "-f",
@@ -80,7 +89,7 @@ def add_parser(subparsers, parent_parser):
         "-c",
         "--cwd",
         default=os.path.curdir,
-        help="Directory within your repo to reroduce from.",
+        help="Directory within your repo to reproduce from.",
     )
     repro_parser.add_argument(
         "-m",

dvc/command/root.py

@@ -1,9 +1,13 @@
 from __future__ import unicode_literals
 
+import argparse
 import os
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdRoot(CmdBase):
@@ -16,8 +20,12 @@ class CmdRoot(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    ROOT_HELP = "Relative path to repo's directory."
+    ROOT_HELP = "Relative path to the repository's directory."
     root_parser = subparsers.add_parser(
-        "root", parents=[parent_parser], description=ROOT_HELP, help=ROOT_HELP
+        "root",
+        parents=[parent_parser],
+        description=append_doc_link(ROOT_HELP, "root"),
+        help=ROOT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     root_parser.set_defaults(func=CmdRoot)

dvc/command/run.py

@@ -1,12 +1,15 @@
 from __future__ import unicode_literals
 
 import argparse
+import logging
 
-import dvc.logger as logger
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdRun(CmdBase):
     def run(self):
         overwrite = self.args.yes or self.args.overwrite_dvcfile
@@ -18,12 +21,15 @@ class CmdRun(CmdBase):
                 self.args.outs_no_cache,
                 self.args.metrics,
                 self.args.metrics_no_cache,
+                self.args.outs_persist,
+                self.args.outs_persist_no_cache,
                 self.args.command,
             ]
         ):  # pragma: no cover
             logger.error(
-                "too few arguments. Specify at least one: '-d', "
-                "'-o', '-O', '-m', '-M', 'command'."
+                "too few arguments. Specify at least one: '-d', '-o', '-O',"
+                " '-m', '-M', '--outs-persist', '--outs-persist-no-cache',"
+                " 'command'."
             )
             return 1
 
@@ -43,9 +49,11 @@ class CmdRun(CmdBase):
                 ignore_build_cache=self.args.ignore_build_cache,
                 remove_outs=self.args.remove_outs,
                 no_commit=self.args.no_commit,
+                outs_persist=self.args.outs_persist,
+                outs_persist_no_cache=self.args.outs_persist_no_cache,
             )
         except DvcException:
-            logger.error("failed to run command")
+            logger.exception("failed to run command")
             return 1
 
         return 0
@@ -73,12 +81,13 @@ class CmdRun(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    RUN_HELP = (
-        "Generate a stage file from a given "
-        "command and execute the command."
-    )
+    RUN_HELP = "Generate a stage file from a command and execute the command."
     run_parser = subparsers.add_parser(
-        "run", parents=[parent_parser], description=RUN_HELP, help=RUN_HELP
+        "run",
+        parents=[parent_parser],
+        description=append_doc_link(RUN_HELP, "run"),
+        help=RUN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     run_parser.add_argument(
         "-d",
@@ -174,6 +183,20 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Don't put files/directories into cache.",
     )
+    run_parser.add_argument(
+        "--outs-persist",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro.",
+    )
+    run_parser.add_argument(
+        "--outs-persist-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro (do not put into DVC cache).",
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/command/status.py

@@ -1,10 +1,14 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import logging
+
 from dvc.command.data_sync import CmdDataBase
 from dvc.utils.compat import str
 
 
+logger = logging.getLogger(__name__)
+
+
 class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 20
     STATUS_INDENT = "\t"
@@ -58,6 +62,6 @@ class CmdDataStatus(CmdDataBase):
                 logger.info(self.UP_TO_DATE_MSG)
 
         except Exception:
-            logger.error("failed to obtain data status")
+            logger.exception("failed to obtain data status")
             return 1
         return 0

dvc/command/tag.py

@@ -0,0 +1,154 @@
+import yaml
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, fix_subparsers, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdTagAdd(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.repo.tag.add(
+                    self.args.tag,
+                    target=target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("failed to add tag")
+                return 1
+        return 0
+
+
+class CmdTagRemove(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.repo.tag.remove(
+                    self.args.tag,
+                    target=target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("failed to remove tag")
+                return 1
+        return 0
+
+
+class CmdTagList(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                tags = self.repo.tag.list(
+                    target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+                if tags:
+                    logger.info(yaml.dump(tags, default_flow_style=False))
+            except DvcException:
+                logger.exception("failed list tags")
+                return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    TAG_HELP = "A set of commands to manage DVC tags."
+    tag_parser = subparsers.add_parser(
+        "tag",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_HELP, "tag"),
+        add_help=False,
+    )
+
+    tag_subparsers = tag_parser.add_subparsers(
+        dest="cmd",
+        help="Use DVC tag CMD --help to display command-specific help.",
+    )
+
+    fix_subparsers(tag_subparsers)
+
+    TAG_ADD_HELP = "Add DVC tag."
+    tag_add_parser = tag_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_ADD_HELP, "tag-add"),
+        help=TAG_ADD_HELP,
+    )
+    tag_add_parser.add_argument("tag", help="Dvc tag.")
+    tag_add_parser.add_argument(
+        "targets", nargs="*", default=[None], help="Dvc files."
+    )
+    tag_add_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Add tag for all dependencies of the specified DVC file.",
+    )
+    tag_add_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Add tag for subdirectories of the specified directory.",
+    )
+    tag_add_parser.set_defaults(func=CmdTagAdd)
+
+    TAG_REMOVE_HELP = "Remove DVC tag."
+    tag_remove_parser = tag_subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_REMOVE_HELP, "tag-remove"),
+        help=TAG_REMOVE_HELP,
+    )
+    tag_remove_parser.add_argument("tag", help="Dvc tag.")
+    tag_remove_parser.add_argument(
+        "targets", nargs="*", default=[None], help="Dvc files."
+    )
+    tag_remove_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Remove tag for all dependencies of the specified DVC file.",
+    )
+    tag_remove_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Remove tag for subdirectories of the specified directory.",
+    )
+    tag_remove_parser.set_defaults(func=CmdTagRemove)
+
+    TAG_LIST_HELP = "List DVC tags."
+    tag_list_parser = tag_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(TAG_LIST_HELP, "tag-list"),
+        help=TAG_LIST_HELP,
+    )
+    tag_list_parser.add_argument(
+        "targets", nargs="*", default=[None], help="Dvc files."
+    )
+    tag_list_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="List tags for all dependencies of the specified DVC file.",
+    )
+    tag_list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="List tags for subdirectories of the specified directory.",
+    )
+    tag_list_parser.set_defaults(func=CmdTagList)

dvc/command/unprotect.py

@@ -1,8 +1,13 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import argparse
+import logging
+
 from dvc.exceptions import DvcException
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
 
 
 class CmdUnprotect(CmdBase):
@@ -12,18 +17,19 @@ class CmdUnprotect(CmdBase):
                 self.repo.unprotect(target)
             except DvcException:
                 msg = "failed to unprotect '{}'".format(target)
-                logger.error(msg)
+                logger.exception(msg)
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    UNPROTECT_HELP = "Unprotect data file/directory."
+    UNPROTECT_HELP = "Unprotect a data file/directory."
     unprotect_parser = subparsers.add_parser(
         "unprotect",
         parents=[parent_parser],
-        description=UNPROTECT_HELP,
+        description=append_doc_link(UNPROTECT_HELP, "unprotect"),
         help=UNPROTECT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
         "targets", nargs="+", help="Data files/directory."

dvc/config.py

@@ -7,11 +7,13 @@ from dvc.utils.compat import str, open
 import os
 import errno
 import configobj
-from schema import Schema, Optional, And, Use, Regex
+import logging
 
-import dvc.logger as logger
+from schema import Schema, Optional, And, Use, Regex
 from dvc.exceptions import DvcException
 
+logger = logging.getLogger(__name__)
+
 
 class ConfigError(DvcException):
     """DVC config exception.

dvc/daemon.py

@@ -5,13 +5,16 @@ from __future__ import unicode_literals
 import os
 import sys
 import inspect
+import logging
 from subprocess import Popen
 
-import dvc.logger as logger
+from dvc.env import DVC_DAEMON
 from dvc.utils import is_binary, fix_env
 from dvc.utils.compat import cast_bytes_py2
 
 
+logger = logging.getLogger(__name__)
+
 CREATE_NEW_PROCESS_GROUP = 0x00000200
 DETACHED_PROCESS = 0x00000008
 
@@ -43,7 +46,7 @@ def _spawn_posix(cmd, env):
         if pid > 0:
             return
     except OSError:
-        logger.error("failed at first fork")
+        logger.exception("failed at first fork")
         os._exit(1)  # pylint: disable=protected-access
 
     os.setsid()
@@ -54,7 +57,7 @@ def _spawn_posix(cmd, env):
         if pid > 0:
             os._exit(0)  # pylint: disable=protected-access
     except OSError:
-        logger.error("failed at second fork")
+        logger.exception("failed at second fork")
         os._exit(1)  # pylint: disable=protected-access
 
     sys.stdin.close()
@@ -66,12 +69,29 @@ def _spawn_posix(cmd, env):
     os._exit(0)  # pylint: disable=protected-access
 
 
+def _spawn(cmd, env):
+    logger.debug("Trying to spawn '{}'".format(cmd))
+
+    if os.name == "nt":
+        _spawn_windows(cmd, env)
+    elif os.name == "posix":
+        _spawn_posix(cmd, env)
+    else:
+        raise NotImplementedError
+
+    logger.debug("Spawned '{}'".format(cmd))
+
+
 def daemon(args):
     """Launch a `dvc daemon` command in a detached process.
 
     Args:
         args (list): list of arguments to append to `dvc daemon` command.
     """
+    if os.environ.get(DVC_DAEMON):
+        logger.debug("skipping launching a new daemon.")
+        return
+
     cmd = [sys.executable]
     if not is_binary():
         cmd += ["-m", "dvc"]
@@ -82,14 +102,6 @@ def daemon(args):
     env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
         os.path.dirname(os.path.dirname(file_path))
     )
+    env[cast_bytes_py2(DVC_DAEMON)] = cast_bytes_py2("1")
 
-    logger.debug("Trying to spawn '{}' with env '{}'".format(cmd, env))
-
-    if os.name == "nt":
-        _spawn_windows(cmd, env)
-    elif os.name == "posix":
-        _spawn_posix(cmd, env)
-    else:
-        raise NotImplementedError
-
-    logger.debug("Spawned '{}'".format(cmd))
+    _spawn(cmd, env)

dvc/data_cloud.py

@@ -2,9 +2,9 @@
 
 from __future__ import unicode_literals
 
-import dvc.logger as logger
-from dvc.config import Config, ConfigError
+import logging
 
+from dvc.config import Config, ConfigError
 from dvc.remote import Remote
 from dvc.remote.s3 import RemoteS3
 from dvc.remote.gs import RemoteGS
@@ -15,6 +15,9 @@ from dvc.remote.local import RemoteLOCAL
 from dvc.remote.http import RemoteHTTP
 
 
+logger = logging.getLogger(__name__)
+
+
 class DataCloud(object):
     """Class that manages dvc remotes.
 

dvc/env.py

@@ -0,0 +1 @@
+DVC_DAEMON = "DVC_DAEMON"

dvc/exceptions.py

@@ -227,3 +227,31 @@ class RecursiveAddingWhileUsingFilename(DvcException):
         super(RecursiveAddingWhileUsingFilename, self).__init__(
             "using fname with recursive is not allowed."
         )
+
+
+class OverlappingOutputPathsError(DvcException):
+    def __init__(self, out_1, out_2):
+        super(OverlappingOutputPathsError, self).__init__(
+            "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
+            "unpredictable behaviour, rerun command with non overlapping outs "
+            "paths.".format(
+                str(out_1),
+                out_1.stage.relpath,
+                str(out_2),
+                out_2.stage.relpath,
+            )
+        )
+
+
+class TargetNotDirectoryError(DvcException):
+    def __init__(self, path):
+        super(TargetNotDirectoryError, self).__init__(
+            "Target: {} is not a directory".format(path)
+        )
+
+
+class CheckoutErrorSuggestGit(DvcException):
+    def __init__(self, target, cause):
+        super(CheckoutErrorSuggestGit, self).__init__(
+            "Did you mean 'git checkout {}'?".format(target), cause=cause
+        )

dvc/logger.py

@@ -1,312 +1,173 @@
-"""Manages logger for dvc repo."""
+"""Manages logging configuration for dvc repo."""
 
 from __future__ import unicode_literals
 
-from dvc.exceptions import DvcException
-from dvc.utils.compat import str
-from dvc.progress import progress_aware
+from dvc.utils.compat import str, StringIO
 
-import re
-import sys
 import logging
-import traceback
-
-from contextlib import contextmanager
-
+import logging.handlers
+import logging.config
 import colorama
 
 
-@progress_aware
-def info(message):
-    """Prints an info message."""
-    logger.info(message)
+class ExcludeErrorsFilter(logging.Filter):
+    def filter(self, record):
+        return record.levelno < logging.ERROR
 
 
-def debug(message):
-    """Prints a debug message."""
-    prefix = colorize("Debug", color="blue")
+class ColorFormatter(logging.Formatter):
+    """Enable color support when logging to a terminal that supports it.
 
-    out = "{prefix}: {message}".format(prefix=prefix, message=message)
+    Color support on Windows versions that do not support ANSI color codes is
+    enabled by use of the colorama__ library.
+    See the colorama documentation for details.
 
-    logger.debug(out)
+    __ https://pypi.python.org/pypi/colorama
 
-
-@progress_aware
-def warning(message, parse_exception=False):
-    """Prints a warning message."""
-    prefix = colorize("Warning", color="yellow")
-
-    exception, stack_trace = None, ""
-    if parse_exception:
-        exception, stack_trace = _parse_exc()
-
-    out = "{prefix}: {description}".format(
-        prefix=prefix, description=_description(message, exception)
-    )
-
-    if stack_trace:
-        out += "\n{stack_trace}".format(stack_trace=stack_trace)
-
-    logger.warning(out)
-
-
-@progress_aware
-def error(message=None):
-    """Prints an error message."""
-    prefix = colorize("Error", color="red")
-
-    exception, stack_trace = _parse_exc()
-
-    out = (
-        "{prefix}: {description}"
-        "\n"
-        "{stack_trace}"
-        "\n"
-        "{footer}".format(
-            prefix=prefix,
-            description=_description(message, exception),
-            stack_trace=stack_trace,
-            footer=_footer(),
-        )
-    )
-
-    logger.error(out)
-
-
-def box(message, border_color=None):
-    """Prints a message in a box.
-
-    Args:
-        message (unicode): message to print.
-        border_color (unicode): name of a color to outline the box with.
+    For records containing `exc_info`, it will use a custom `_walk_exc` to
+    retrieve the whole tracebak.
     """
-    lines = message.split("\n")
-    max_width = max(_visual_width(line) for line in lines)
 
-    padding_horizontal = 5
-    padding_vertical = 1
-
-    box_size_horizontal = max_width + (padding_horizontal * 2)
-
-    chars = {"corner": "+", "horizontal": "-", "vertical": "|", "empty": " "}
-
-    margin = "{corner}{line}{corner}\n".format(
-        corner=chars["corner"], line=chars["horizontal"] * box_size_horizontal
-    )
-
-    padding_lines = [
-        "{border}{space}{border}\n".format(
-            border=colorize(chars["vertical"], color=border_color),
-            space=chars["empty"] * box_size_horizontal,
-        )
-        * padding_vertical
-    ]
-
-    content_lines = [
-        "{border}{space}{content}{space}{border}\n".format(
-            border=colorize(chars["vertical"], color=border_color),
-            space=chars["empty"] * padding_horizontal,
-            content=_visual_center(line, max_width),
-        )
-        for line in lines
-    ]
-
-    box_str = "{margin}{padding}{content}{padding}{margin}".format(
-        margin=colorize(margin, color=border_color),
-        padding="".join(padding_lines),
-        content="".join(content_lines),
-    )
-
-    logger.info(box_str)
-
-
-def level():
-    """Returns current log level."""
-    return logger.getEffectiveLevel()
-
-
-def set_level(level_name):
-    """Sets log level.
-
-    Args:
-        level_name (str): log level name. E.g. info, debug, warning, error,
-            critical.
-    """
-    if not level_name:
-        return
-
-    levels = {
-        "info": logging.INFO,
-        "debug": logging.DEBUG,
-        "warning": logging.WARNING,
-        "error": logging.ERROR,
-        "critical": logging.CRITICAL,
+    color_code = {
+        "DEBUG": colorama.Fore.BLUE,
+        "INFO": "",
+        "WARNING": colorama.Fore.YELLOW,
+        "ERROR": colorama.Fore.RED,
+        "CRITICAL": colorama.Fore.RED,
     }
 
-    logger.setLevel(levels.get(level_name))
-
-
-def be_quiet():
-    """Disables all messages except critical ones."""
-    logger.setLevel(logging.CRITICAL)
-
-
-def be_verbose():
-    """Enables all messages."""
-    logger.setLevel(logging.DEBUG)
-
-
-@contextmanager
-def verbose():
-    """Enables verbose mode for the context."""
-    previous_level = level()
-    be_verbose()
-    yield
-    logger.setLevel(previous_level)
-
-
-@contextmanager
-def quiet():
-    """Enables quiet mode for the context."""
-    previous_level = level()
-    be_quiet()
-    yield
-    logger.setLevel(previous_level)
-
-
-def is_quiet():
-    """Returns whether or not all messages except critical ones are
-    disabled.
-    """
-    return level() == logging.CRITICAL
-
-
-def is_verbose():
-    """Returns whether or not all messages are enabled."""
-    return level() == logging.DEBUG
-
-
-def colorize(message, color=None):
-    """Returns a message in a specified color."""
-    if not color:
-        return message
-
-    colors = {
-        "green": colorama.Fore.GREEN,
-        "yellow": colorama.Fore.YELLOW,
-        "blue": colorama.Fore.BLUE,
-        "red": colorama.Fore.RED,
-    }
-
-    return "{color}{message}{nc}".format(
-        color=colors.get(color, ""), message=message, nc=colorama.Fore.RESET
+    footer = (
+        "{yellow}Having any troubles?{nc}."
+        " Hit us up at {blue}https://dvc.org/support{nc},"
+        " we are always happy to help!"
+    ).format(
+        blue=colorama.Fore.BLUE,
+        nc=colorama.Fore.RESET,
+        yellow=colorama.Fore.YELLOW,
     )
 
+    def format(self, record):
+        if record.levelname == "INFO":
+            return record.msg
 
-def _init_colorama():
+        if record.levelname == "ERROR" or record.levelname == "CRITICAL":
+            exception, stack_trace = self._parse_exc(record.exc_info)
+
+            return (
+                "{color}{levelname}{nc}: {description}"
+                "{stack_trace}\n"
+                "\n"
+                "{footer}"
+            ).format(
+                color=self.color_code.get(record.levelname, ""),
+                nc=colorama.Fore.RESET,
+                levelname=record.levelname,
+                description=self._description(record.msg, exception),
+                msg=record.msg,
+                stack_trace=stack_trace,
+                footer=self.footer,
+            )
+
+        return "{color}{levelname}{nc}: {msg}".format(
+            color=self.color_code.get(record.levelname, ""),
+            nc=colorama.Fore.RESET,
+            levelname=record.levelname,
+            msg=record.msg,
+        )
+
+    def _description(self, message, exception):
+        description = ""
+
+        if exception and message:
+            description = "{message} - {exception}"
+        elif exception:
+            description = "{exception}"
+        elif message:
+            description = "{message}"
+
+        return description.format(message=message, exception=exception)
+
+    def _walk_exc(self, exc_info):
+        import traceback
+
+        buffer = StringIO()
+
+        traceback.print_exception(*exc_info, file=buffer)
+
+        exc = exc_info[1]
+        tb = buffer.getvalue()
+
+        exc_list = [str(exc)]
+        tb_list = [tb]
+
+        # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info
+        while hasattr(exc, "cause") and exc.cause:
+            exc_list.append(str(exc.cause))
+            if hasattr(exc, "cause_tb") and exc.cause_tb:
+                tb_list.insert(0, str(exc.cause_tb))
+            exc = exc.cause
+
+        return exc_list, tb_list
+
+    def _parse_exc(self, exc_info):
+        if not exc_info:
+            return (None, "")
+
+        exc_list, tb_list = self._walk_exc(exc_info)
+
+        exception = ": ".join(exc_list)
+
+        if logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG:
+            stack_trace = (
+                "\n" "{red}{line}{nc}\n" "{stack_trace}" "{red}{line}{nc}"
+            ).format(
+                red=colorama.Fore.RED,
+                nc=colorama.Fore.RESET,
+                line="-" * 60,
+                stack_trace="\n".join(tb_list),
+            )
+        else:
+            stack_trace = ""
+
+        return (exception, stack_trace)
+
+    def _progress_aware(self):
+        """Add a new line if progress bar hasn't finished"""
+        from dvc.progress import progress
+
+        if not progress.is_finished:
+            progress._print()
+        progress.clearln()
+
+
+def setup(level=logging.INFO):
     colorama.init()
 
-
-def set_default_level():
-    """Sets default log level."""
-    logger.setLevel(logging.INFO)
-
-
-def _add_handlers():
-    formatter = "%(message)s"
-
-    class _LogLevelFilter(logging.Filter):
-        # pylint: disable=too-few-public-methods
-        def filter(self, record):
-            return record.levelno <= logging.WARNING
-
-    sh_out = logging.StreamHandler(sys.stdout)
-    sh_out.setFormatter(logging.Formatter(formatter))
-    sh_out.setLevel(logging.DEBUG)
-    sh_out.addFilter(_LogLevelFilter())
-
-    sh_err = logging.StreamHandler(sys.stderr)
-    sh_err.setFormatter(logging.Formatter(formatter))
-    sh_err.setLevel(logging.ERROR)
-
-    logger.addHandler(sh_out)
-    logger.addHandler(sh_err)
-
-
-def _walk_exc(exc):
-    exc_list = [str(exc)]
-    tb_list = [traceback.format_exc()]
-
-    # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info.
-    while hasattr(exc, "cause") and exc.cause is not None:
-        exc_list.append(str(exc.cause))
-        if hasattr(exc, "cause_tb") and exc.cause_tb is not None:
-            tb_list.insert(0, str(exc.cause_tb))
-        exc = exc.cause
-
-    return exc_list, tb_list
-
-
-def _parse_exc():
-    exc = sys.exc_info()[1]
-    if not exc:
-        return (None, "")
-
-    exc_list, tb_list = _walk_exc(exc)
-
-    exception = ": ".join(exc_list)
-
-    if is_verbose():
-        stack_trace = "{line}\n{stack_trace}{line}\n".format(
-            line=colorize("-" * 60, color="red"),
-            stack_trace="\n".join(tb_list),
-        )
-    else:
-        stack_trace = ""
-
-    return (exception, stack_trace)
-
-
-def _description(message, exception):
-    if exception and message:
-        description = "{message} - {exception}"
-    elif exception:
-        description = "{exception}"
-    elif message:
-        description = "{message}"
-    else:
-        raise DvcException(
-            "Unexpected error - either exception or message must be provided"
-        )
-
-    return description.format(message=message, exception=exception)
-
-
-def _footer():
-    return "{phrase} Hit us up at {url}, we are always happy to help!".format(
-        phrase=colorize("Having any troubles?", "yellow"),
-        url=colorize("https://dvc.org/support", "blue"),
+    logging.config.dictConfig(
+        {
+            "version": 1,
+            "filters": {"exclude_errors": {"()": ExcludeErrorsFilter}},
+            "formatters": {"color": {"()": ColorFormatter}},
+            "handlers": {
+                "console": {
+                    "class": "logging.StreamHandler",
+                    "level": "DEBUG",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_errors"],
+                },
+                "console_errors": {
+                    "class": "logging.StreamHandler",
+                    "level": "ERROR",
+                    "formatter": "color",
+                    "stream": "ext://sys.stderr",
+                },
+            },
+            "loggers": {
+                "dvc": {
+                    "level": level,
+                    "handlers": ["console", "console_errors"],
+                }
+            },
+        }
     )
-
-
-def _visual_width(line):
-    """Get the the number of columns required to display a string"""
-
-    return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, "", line))
-
-
-def _visual_center(line, width):
-    """Center align string according to it's visual width"""
-
-    spaces = max(width - _visual_width(line), 0)
-    left_padding = int(spaces / 2)
-    right_padding = spaces - left_padding
-
-    return (left_padding * " ") + line + (right_padding * " ")
-
-
-logger = logging.getLogger("dvc")  # pylint: disable=invalid-name
-
-set_default_level()
-_add_handlers()
-_init_colorama()

dvc/main.py

@@ -2,13 +2,16 @@
 
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import logging
+
 from dvc.cli import parse_args
-from dvc.command.base import CmdBase
 from dvc.analytics import Analytics
 from dvc.exceptions import NotDvcRepoError, DvcParserError
 
 
+logger = logging.getLogger("dvc")
+
+
 def main(argv=None):
     """Run dvc CLI command.
 
@@ -20,26 +23,28 @@ def main(argv=None):
     """
     args = None
     cmd = None
+
     try:
         args = parse_args(argv)
 
-        # Init loglevel early in case we'll run
-        # into errors before setting it properly
-        CmdBase.set_loglevel(args)
+        if args.quiet:
+            logger.setLevel(logging.CRITICAL)
+
+        elif args.verbose:
+            logger.setLevel(logging.DEBUG)
 
         cmd = args.func(args)
-
         ret = cmd.run_cmd()
     except KeyboardInterrupt:
-        logger.error("interrupted by the user")
+        logger.exception("interrupted by the user")
         ret = 252
     except NotDvcRepoError:
-        logger.error()
+        logger.exception("")
         ret = 253
     except DvcParserError:
         ret = 254
     except Exception:  # pylint: disable=broad-except
-        logger.error("unexpected error")
+        logger.exception("unexpected error")
         ret = 255
 
     Analytics().send_cmd(cmd, args, ret)

dvc/output/__init__.py

@@ -33,38 +33,69 @@ OUTS_MAP = {
     "local": OutputLOCAL,
 }
 
-SCHEMA = {
-    OutputBase.PARAM_PATH: str,
-    # NOTE: currently there are only 3 possible checksum names:
-    #
-    #    1) md5 (LOCAL, SSH, GS);
-    #    2) etag (S3);
-    #    3) checksum (HDFS);
-    #
-    # so when a few types of outputs share the same name, we only need
-    # specify it once.
+# NOTE: currently there are only 3 possible checksum names:
+#
+#    1) md5 (LOCAL, SSH, GS);
+#    2) etag (S3);
+#    3) checksum (HDFS);
+#
+# so when a few types of outputs share the same name, we only need
+# specify it once.
+CHECKSUM_SCHEMA = {
     schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None),
     schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None),
     schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
-    schema.Optional(OutputBase.PARAM_CACHE): bool,
-    schema.Optional(OutputBase.PARAM_METRIC): OutputBase.METRIC_SCHEMA,
 }
 
+TAGS_SCHEMA = {schema.Optional(str): CHECKSUM_SCHEMA}
 
-def _get(stage, p, info, cache, metric):
+SCHEMA = CHECKSUM_SCHEMA.copy()
+SCHEMA[OutputBase.PARAM_PATH] = str
+SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)] = bool
+SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] = OutputBase.METRIC_SCHEMA
+SCHEMA[schema.Optional(OutputBase.PARAM_TAGS)] = TAGS_SCHEMA
+SCHEMA[schema.Optional(OutputBase.PARAM_PERSIST)] = bool
+
+
+def _get(stage, p, info, cache, metric, persist=False, tags=None):
     parsed = urlparse(p)
     if parsed.scheme == "remote":
         name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
         sect = stage.repo.config.config[name]
         remote = Remote(stage.repo, sect)
         return OUTS_MAP[remote.scheme](
-            stage, p, info, cache=cache, remote=remote, metric=metric
+            stage,
+            p,
+            info,
+            cache=cache,
+            remote=remote,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
 
     for o in OUTS:
         if o.supported(p):
-            return o(stage, p, info, cache=cache, remote=None, metric=metric)
-    return OutputLOCAL(stage, p, info, cache=cache, remote=None, metric=metric)
+            return o(
+                stage,
+                p,
+                info,
+                cache=cache,
+                remote=None,
+                metric=metric,
+                persist=persist,
+                tags=tags,
+            )
+    return OutputLOCAL(
+        stage,
+        p,
+        info,
+        cache=cache,
+        remote=None,
+        metric=metric,
+        persist=persist,
+        tags=tags,
+    )
 
 
 def loadd_from(stage, d_list):
@@ -73,12 +104,33 @@ def loadd_from(stage, d_list):
         p = d.pop(OutputBase.PARAM_PATH)
         cache = d.pop(OutputBase.PARAM_CACHE, True)
         metric = d.pop(OutputBase.PARAM_METRIC, False)
-        ret.append(_get(stage, p, info=d, cache=cache, metric=metric))
+        persist = d.pop(OutputBase.PARAM_PERSIST, False)
+        tags = d.pop(OutputBase.PARAM_TAGS, None)
+        ret.append(
+            _get(
+                stage,
+                p,
+                info=d,
+                cache=cache,
+                metric=metric,
+                persist=persist,
+                tags=tags,
+            )
+        )
     return ret
 
 
-def loads_from(stage, s_list, use_cache=True, metric=False):
+def loads_from(stage, s_list, use_cache=True, metric=False, persist=False):
     ret = []
     for s in s_list:
-        ret.append(_get(stage, s, info={}, cache=use_cache, metric=metric))
+        ret.append(
+            _get(
+                stage,
+                s,
+                info={},
+                cache=use_cache,
+                metric=metric,
+                persist=persist,
+            )
+        )
     return ret

dvc/output/base.py

@@ -36,6 +36,7 @@ class OutputBase(object):
     PARAM_METRIC = "metric"
     PARAM_METRIC_TYPE = "type"
     PARAM_METRIC_XPATH = "xpath"
+    PARAM_PERSIST = "persist"
 
     METRIC_SCHEMA = Or(
         None,
@@ -46,11 +47,21 @@ class OutputBase(object):
         },
     )
 
+    PARAM_TAGS = "tags"
+
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         self.stage = stage
         self.repo = stage.repo
@@ -59,6 +70,8 @@ class OutputBase(object):
         self.remote = remote or self.REMOTE(self.repo, {})
         self.use_cache = False if self.IS_DEPENDENCY else cache
         self.metric = False if self.IS_DEPENDENCY else metric
+        self.persist = persist
+        self.tags = None if self.IS_DEPENDENCY else (tags or {})
 
         if (
             self.use_cache
@@ -79,9 +92,12 @@ class OutputBase(object):
         return self.url
 
     @property
-    def is_local(self):
+    def is_in_repo(self):
+        return False
+
+    def assign_to_stage_file(self, target_repo):
         raise DvcException(
-            "is local is not supported for {}".format(self.scheme)
+            "change repo is not supported for {}".format(self.scheme)
         )
 
     @classmethod
@@ -181,6 +197,10 @@ class OutputBase(object):
                 del self.metric[self.PARAM_METRIC_XPATH]
 
         ret[self.PARAM_METRIC] = self.metric
+        ret[self.PARAM_PERSIST] = self.persist
+
+        if self.tags:
+            ret[self.PARAM_TAGS] = self.tags
 
         return ret
 
@@ -192,12 +212,20 @@ class OutputBase(object):
     def download(self, to_info, resume=False):
         self.remote.download([self.path_info], [to_info], resume=resume)
 
-    def checkout(self, force=False):
+    def checkout(self, force=False, progress_callback=None, tag=None):
         if not self.use_cache:
             return
 
+        if tag:
+            info = self.tags[tag]
+        else:
+            info = self.info
+
         getattr(self.repo.cache, self.scheme).checkout(
-            self.path_info, self.info, force=force
+            self.path_info,
+            info,
+            force=force,
+            progress_callback=progress_callback,
         )
 
     def remove(self, ignore_remove=False):
@@ -205,11 +233,11 @@ class OutputBase(object):
         if self.scheme != "local":
             return
 
-        if ignore_remove and self.use_cache and self.is_local:
+        if ignore_remove and self.use_cache and self.is_in_repo:
             self.repo.scm.ignore_remove(self.path)
 
     def move(self, out):
-        if self.scheme == "local" and self.use_cache and self.is_local:
+        if self.scheme == "local" and self.use_cache and self.is_in_repo:
             self.repo.scm.ignore_remove(self.path)
 
         self.remote.move(self.path_info, out.path_info)
@@ -218,5 +246,15 @@ class OutputBase(object):
         self.save()
         self.commit()
 
-        if self.scheme == "local" and self.use_cache and self.is_local:
+        if self.scheme == "local" and self.use_cache and self.is_in_repo:
             self.repo.scm.ignore(self.path)
+
+    def get_files_number(self):
+        if self.use_cache and self.checksum:
+            return 1
+
+        return 0
+
+    def unprotect(self):
+        if self.exists:
+            self.remote.unprotect(self.path_info)

dvc/output/hdfs.py

@@ -11,10 +11,25 @@ class OutputHDFS(OutputBase):
     REMOTE = RemoteHDFS
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         super(OutputHDFS, self).__init__(
-            stage, path, info=info, remote=remote, cache=cache, metric=metric
+            stage,
+            path,
+            info=info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
         if remote:
             path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))

dvc/output/local.py

@@ -1,8 +1,8 @@
 from __future__ import unicode_literals
 
 import os
+import logging
 
-import dvc.logger as logger
 from dvc.utils.compat import urlparse
 from dvc.istextfile import istextfile
 from dvc.exceptions import DvcException
@@ -10,14 +10,32 @@ from dvc.remote.local import RemoteLOCAL
 from dvc.output.base import OutputBase, OutputAlreadyTrackedError
 
 
+logger = logging.getLogger(__name__)
+
+
 class OutputLOCAL(OutputBase):
     REMOTE = RemoteLOCAL
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         super(OutputLOCAL, self).__init__(
-            stage, path, info, remote=remote, cache=cache, metric=metric
+            stage,
+            path,
+            info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
         if remote:
             p = os.path.join(
@@ -33,15 +51,28 @@ class OutputLOCAL(OutputBase):
 
         self.path_info = {"scheme": "local", "path": p}
 
+        self._dir_cache = {}
+
     def __str__(self):
         return self.rel_path
 
     @property
-    def is_local(self):
+    def is_in_repo(self):
         return urlparse(self.url).scheme != "remote" and not os.path.isabs(
             self.url
         )
 
+    def assign_to_stage_file(self, stage):
+        from dvc.repo import Repo
+
+        fullpath = os.path.abspath(stage.wdir)
+        self.path_info["path"] = os.path.join(fullpath, self.stage_path)
+
+        self.repo = Repo(self.path)
+
+        self.stage = stage
+        return self
+
     @property
     def sep(self):
         return os.sep
@@ -50,13 +81,21 @@ class OutputLOCAL(OutputBase):
     def rel_path(self):
         return os.path.relpath(self.path)
 
+    @property
+    def stage_path(self):
+        return os.path.relpath(self.path, self.stage.wdir)
+
     @property
     def cache(self):
         return self.repo.cache.local.get(self.checksum)
 
+    @property
+    def is_dir_cache(self):
+        return self.repo.cache.local.is_dir_cache(self.checksum)
+
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
-        if self.is_local:
+        if self.is_in_repo:
             path = self.remote.unixpath(
                 os.path.relpath(self.path, self.stage.wdir)
             )
@@ -110,7 +149,7 @@ class OutputLOCAL(OutputBase):
             logger.info(msg.format(self.rel_path))
             return
 
-        if self.is_local:
+        if self.is_in_repo:
             if self.repo.scm.is_tracked(self.path):
                 raise OutputAlreadyTrackedError(self.rel_path)
 
@@ -118,3 +157,15 @@ class OutputLOCAL(OutputBase):
                 self.repo.scm.ignore(self.path)
 
         self.info = self.remote.save_info(self.path_info)
+
+    @property
+    def dir_cache(self):
+        return self.repo.cache.local.load_dir_cache(self.checksum)
+
+    def get_files_number(self):
+        if self.cache is None:
+            return 0
+
+        if self.is_dir_cache:
+            return len(self.dir_cache)
+        return 1

dvc/output/s3.py

@@ -11,10 +11,25 @@ class OutputS3(OutputBase):
     REMOTE = RemoteS3
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         super(OutputS3, self).__init__(
-            stage, path, info=info, remote=remote, cache=cache, metric=metric
+            stage,
+            path,
+            info=info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
         bucket = remote.bucket if remote else urlparse(path).netloc
         path = urlparse(path).path.lstrip("/")

dvc/output/ssh.py

@@ -12,10 +12,25 @@ class OutputSSH(OutputBase):
     REMOTE = RemoteSSH
 
     def __init__(
-        self, stage, path, info=None, remote=None, cache=True, metric=False
+        self,
+        stage,
+        path,
+        info=None,
+        remote=None,
+        cache=True,
+        metric=False,
+        persist=False,
+        tags=None,
     ):
         super(OutputSSH, self).__init__(
-            stage, path, info=info, remote=remote, cache=cache, metric=metric
+            stage,
+            path,
+            info=info,
+            remote=remote,
+            cache=cache,
+            metric=metric,
+            persist=persist,
+            tags=tags,
         )
         parsed = urlparse(path)
         host = remote.host if remote else parsed.hostname

dvc/progress.py

@@ -8,6 +8,8 @@ from dvc.utils.compat import str
 import sys
 import threading
 
+CLEARLINE_PATTERN = "\r\x1b[K"
+
 
 class Progress(object):
     """
@@ -30,14 +32,20 @@ class Progress(object):
         """Returns if all targets have finished."""
         return self._n_total == self._n_finished
 
-    def _clearln(self):
-        self._print("\r\x1b[K", end="")
+    def clearln(self):
+        self._print(CLEARLINE_PATTERN, end="")
 
     def _writeln(self, line):
-        self._clearln()
+        self.clearln()
         self._print(line, end="")
         sys.stdout.flush()
 
+    def reset(self):
+        with self._lock:
+            self._n_total = 0
+            self._n_finished = 0
+            self._line = None
+
     def refresh(self, line=None):
         """Refreshes progress bar."""
         # Just go away if it is locked. Will update next time
@@ -64,7 +72,7 @@ class Progress(object):
             pbar = self._bar(name, 100, 100)
 
             if sys.stdout.isatty():
-                self._clearln()
+                self.clearln()
 
             self._print(pbar)
 
@@ -99,9 +107,11 @@ class Progress(object):
 
     @staticmethod
     def _print(*args, **kwargs):
-        import dvc.logger as logger
+        import logging
 
-        if logger.is_quiet():
+        logger = logging.getLogger(__name__)
+
+        if logger.getEffectiveLevel() == "CRITICAL":
             return
 
         print(*args, **kwargs)
@@ -109,7 +119,7 @@ class Progress(object):
     def __enter__(self):
         self._lock.acquire(True)
         if self._line is not None:
-            self._clearln()
+            self.clearln()
 
     def __exit__(self, typ, value, tbck):
         if self._line is not None:
@@ -117,18 +127,18 @@ class Progress(object):
         self._lock.release()
 
 
-def progress_aware(f):
-    """ Decorator to add a new line if progress bar hasn't finished  """
-    from functools import wraps
+class ProgressCallback(object):
+    def __init__(self, total):
+        self.total = total
+        self.current = 0
+        progress.reset()
 
-    @wraps(f)
-    def wrapper(*args, **kwargs):
-        if not progress.is_finished:
-            progress._print()
+    def update(self, name, progress_to_add=1):
+        self.current += progress_to_add
+        progress.update_target(name, self.current, self.total)
 
-        return f(*args, **kwargs)
-
-    return wrapper
+    def finish(self, name):
+        progress.finish_target(name)
 
 
 progress = Progress()  # pylint: disable=invalid-name

dvc/prompt.py

@@ -4,12 +4,15 @@ from __future__ import unicode_literals
 from __future__ import print_function
 
 import sys
+import logging
 from getpass import getpass
 
-from dvc.progress import progress_aware
 from dvc.utils.compat import input
 
 
+logger = logging.getLogger(__name__)
+
+
 def _ask(prompt, limited_to=None):
     if not sys.stdout.isatty():
         return None
@@ -26,13 +29,12 @@ def _ask(prompt, limited_to=None):
         if answer in limited_to:
             return answer
 
-        print(
+        logger.info(
             "Your response must be one of: {options}. "
             "Please try again.".format(options=limited_to)
         )
 
 
-@progress_aware
 def confirm(statement):
     """Ask the user for confirmation about the specified statement.
 
@@ -47,7 +49,6 @@ def confirm(statement):
     return answer and answer.startswith("y")
 
 
-@progress_aware
 def password(statement):
     """Ask the user for a password.
 
@@ -57,5 +58,5 @@ def password(statement):
     Returns:
         str: password entered by the user.
     """
-    prompt = "{statement}: ".format(statement=statement)
-    return getpass(prompt)
+    logger.info("{statement}: ".format(statement=statement))
+    return getpass("")

dvc/remote/azure.py

@@ -3,13 +3,14 @@ from __future__ import unicode_literals
 
 import os
 import re
+import logging
 
 try:
     from azure.storage.blob import BlockBlobService
+    from azure.common import AzureMissingResourceHttpError
 except ImportError:
     BlockBlobService = None
 
-import dvc.logger as logger
 from dvc.utils import tmp_fname, move
 from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
@@ -17,6 +18,9 @@ from dvc.config import Config
 from dvc.remote.base import RemoteBase
 
 
+logger = logging.getLogger(__name__)
+
+
 class Callback(object):
     def __init__(self, name):
         self.name = name
@@ -77,7 +81,12 @@ class RemoteAzure(RemoteBase):
                 connection_string=self.connection_string
             )
             logger.debug("Container name {}".format(self.bucket))
-            self.__blob_service.create_container(self.bucket)
+            try:  # verify that container exists
+                self.__blob_service.list_blobs(
+                    self.bucket, delimiter="/", num_results=1
+                )
+            except AzureMissingResourceHttpError:
+                self.__blob_service.create_container(self.bucket)
         return self.__blob_service
 
     def remove(self, path_info):

dvc/remote/base.py

@@ -4,14 +4,17 @@ from dvc.utils.compat import str
 
 import re
 import posixpath
+import logging
 from multiprocessing import cpu_count
 
 import dvc.prompt as prompt
-import dvc.logger as logger
 from dvc.config import Config
 from dvc.exceptions import DvcException, ConfirmRemoveError
 
 
+logger = logging.getLogger(__name__)
+
+
 STATUS_OK = 1
 STATUS_MISSING = 2
 STATUS_NEW = 3
@@ -322,7 +325,9 @@ class RemoteBase(object):
 
         self.remove(path_info)
 
-    def do_checkout(self, path_info, checksum, force=False):
+    def do_checkout(
+        self, path_info, checksum, force=False, progress_callback=None
+    ):
         if self.exists(path_info):
             msg = "data '{}' exists. Removing before checkout."
             logger.warning(msg.format(str(path_info)))
@@ -331,7 +336,9 @@ class RemoteBase(object):
         from_info = self.checksum_to_path_info(checksum)
         self.copy(from_info, path_info)
 
-    def checkout(self, path_info, checksum_info, force=False):
+    def checkout(
+        self, path_info, checksum_info, force=False, progress_callback=None
+    ):
         scheme = path_info["scheme"]
         if scheme not in ["", "local"] and scheme != self.scheme:
             raise NotImplementedError
@@ -339,12 +346,12 @@ class RemoteBase(object):
         checksum = checksum_info.get(self.PARAM_CHECKSUM)
         if not checksum:
             msg = "No checksum info for '{}'."
-            logger.info(msg.format(str(path_info)))
+            logger.debug(msg.format(str(path_info)))
             return
 
         if not self.changed(path_info, checksum_info):
             msg = "Data '{}' didn't change."
-            logger.info(msg.format(str(path_info)))
+            logger.debug(msg.format(str(path_info)))
             return
 
         if self.changed_cache(checksum):
@@ -354,6 +361,15 @@ class RemoteBase(object):
             return
 
         msg = "Checking out '{}' with cache '{}'."
-        logger.info(msg.format(str(path_info), checksum))
+        logger.debug(msg.format(str(path_info), checksum))
 
-        self.do_checkout(path_info, checksum, force=force)
+        self.do_checkout(
+            path_info,
+            checksum,
+            force=force,
+            progress_callback=progress_callback,
+        )
+
+    @staticmethod
+    def unprotect(path_info):
+        pass

dvc/remote/gs.py

@@ -1,13 +1,13 @@
 from __future__ import unicode_literals
 
 import os
+import logging
 
 try:
     from google.cloud import storage
 except ImportError:
     storage = None
 
-import dvc.logger as logger
 from dvc.utils import tmp_fname, move
 from dvc.utils.compat import urlparse, makedirs
 from dvc.remote.base import RemoteBase
@@ -16,6 +16,9 @@ from dvc.progress import progress
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class RemoteGS(RemoteBase):
     scheme = "gs"
     REGEX = r"^gs://(?P<path>.*)$"
@@ -146,7 +149,7 @@ class RemoteGS(RemoteBase):
                 blob.upload_from_filename(from_info["path"])
             except Exception:
                 msg = "failed to upload '{}' to '{}/{}'"
-                logger.error(
+                logger.exception(
                     msg.format(
                         from_info["path"], to_info["bucket"], to_info["path"]
                     )
@@ -200,7 +203,7 @@ class RemoteGS(RemoteBase):
                 blob.download_to_filename(tmp_file)
             except Exception:
                 msg = "failed to download '{}/{}' to '{}'"
-                logger.error(
+                logger.exception(
                     msg.format(
                         from_info["bucket"], from_info["path"], to_info["path"]
                     )

dvc/remote/hdfs.py

@@ -4,14 +4,17 @@ import os
 import re
 import getpass
 import posixpath
+import logging
 from subprocess import Popen, PIPE
 
-import dvc.logger as logger
 from dvc.config import Config
 from dvc.remote.base import RemoteBase, RemoteCmdError
 from dvc.utils import fix_env, tmp_fname
 
 
+logger = logging.getLogger(__name__)
+
+
 class RemoteHDFS(RemoteBase):
     scheme = "hdfs"
     REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
@@ -77,7 +80,7 @@ class RemoteHDFS(RemoteBase):
 
     def rm(self, path_info):
         self.hadoop_fs(
-            "rm {}".format(path_info["path"]), user=path_info["user"]
+            "rm -f {}".format(path_info["path"]), user=path_info["user"]
         )
 
     def save_info(self, path_info):

dvc/remote/http.py

@@ -4,8 +4,8 @@ from dvc.utils.compat import open, makedirs
 import os
 import threading
 import requests
+import logging
 
-import dvc.logger as logger
 from dvc.progress import progress
 from dvc.exceptions import DvcException
 from dvc.config import Config
@@ -13,6 +13,9 @@ from dvc.remote.base import RemoteBase
 from dvc.utils import move
 
 
+logger = logging.getLogger(__name__)
+
+
 class ProgressBarCallback(object):
     def __init__(self, name, total):
         self.name = name

dvc/remote/local.py

@@ -9,9 +9,9 @@ import json
 import ntpath
 import shutil
 import posixpath
+import logging
 from operator import itemgetter
 
-import dvc.logger as logger
 from dvc.system import System
 from dvc.remote.base import (
     RemoteBase,
@@ -20,7 +20,15 @@ from dvc.remote.base import (
     STATUS_DELETED,
     STATUS_MISSING,
 )
-from dvc.utils import remove, move, copyfile, dict_md5, to_chunks, tmp_fname
+from dvc.utils import (
+    remove,
+    move,
+    copyfile,
+    dict_md5,
+    to_chunks,
+    tmp_fname,
+    walk_files,
+)
 from dvc.utils import LARGE_DIR_SIZE
 from dvc.config import Config
 from dvc.exceptions import DvcException
@@ -30,6 +38,9 @@ from concurrent.futures import ThreadPoolExecutor
 from dvc.utils.fs import get_mtime_and_size, get_inode
 
 
+logger = logging.getLogger(__name__)
+
+
 class RemoteLOCAL(RemoteBase):
     scheme = "local"
     REGEX = r"^(?P<path>.*)$"
@@ -70,6 +81,8 @@ class RemoteLOCAL(RemoteBase):
 
         self.path_info = {"scheme": "local"}
 
+        self._dir_info = {}
+
     @staticmethod
     def compat_config(config):
         ret = config.copy()
@@ -248,9 +261,12 @@ class RemoteLOCAL(RemoteBase):
 
     def load_dir_cache(self, md5):
         path = self.get(md5)
-
         assert self.is_dir_cache(path)
 
+        dir_info = self._dir_info.get(md5)
+        if dir_info:
+            return dir_info
+
         try:
             with open(path, "r") as fd:
                 d = json.load(fd)
@@ -267,6 +283,8 @@ class RemoteLOCAL(RemoteBase):
         for info in d:
             info["relpath"] = self.to_ospath(info["relpath"])
 
+        self._dir_info[md5] = d
+
         return d
 
     def dump_dir_cache(self, md5, dir_info):
@@ -291,10 +309,11 @@ class RemoteLOCAL(RemoteBase):
     def is_dir_cache(cls, cache):
         return cache.endswith(cls.MD5_DIR_SUFFIX)
 
-    def do_checkout(self, path_info, checksum, force=False):
+    def do_checkout(
+        self, path_info, checksum, force=False, progress_callback=None
+    ):
         path = path_info["path"]
-        md5 = checksum
-        cache = self.get(md5)
+        cache = self.get(checksum)
 
         if not self.is_dir_cache(cache):
             if os.path.exists(path):
@@ -302,6 +321,8 @@ class RemoteLOCAL(RemoteBase):
 
             self.link(cache, path)
             self.state.update_link(path)
+            if progress_callback:
+                progress_callback.update(os.path.relpath(path))
             return
 
         # Create dir separately so that dir is created
@@ -309,12 +330,11 @@ class RemoteLOCAL(RemoteBase):
         if not os.path.exists(path):
             os.makedirs(path)
 
-        dir_info = self.load_dir_cache(md5)
         dir_relpath = os.path.relpath(path)
-        dir_size = len(dir_info)
-        bar = dir_size > LARGE_DIR_SIZE
 
-        logger.info("Linking directory '{}'.".format(dir_relpath))
+        logger.debug("Linking directory '{}'.".format(dir_relpath))
+
+        dir_info = self.load_dir_cache(checksum)
 
         for processed, entry in enumerate(dir_info):
             relpath = entry[self.PARAM_RELPATH]
@@ -332,16 +352,13 @@ class RemoteLOCAL(RemoteBase):
 
                 self.link(c, p)
 
-            if bar:
-                progress.update_target(dir_relpath, processed, dir_size)
+            if progress_callback:
+                progress_callback.update(os.path.relpath(p))
 
         self._discard_working_directory_changes(path, dir_info, force=force)
 
         self.state.update_link(path)
 
-        if bar:
-            progress.finish_target(dir_relpath)
-
     def already_cached(self, path_info):
         assert path_info["scheme"] in ["", "local"]
 
@@ -353,11 +370,7 @@ class RemoteLOCAL(RemoteBase):
         return not self.changed_cache(current_md5)
 
     def _discard_working_directory_changes(self, path, dir_info, force=False):
-        working_dir_files = set(
-            os.path.join(root, file)
-            for root, _, files in os.walk(str(path))
-            for file in files
-        )
+        working_dir_files = set(path for path in walk_files(path))
 
         cached_files = set(
             os.path.join(path, file["relpath"]) for file in dir_info
@@ -739,3 +752,48 @@ class RemoteLOCAL(RemoteBase):
                 "nor on remote. Missing cache files: {}".format(missing_desc)
             )
             logger.warning(msg)
+
+    @staticmethod
+    def _unprotect_file(path):
+        if System.is_symlink(path) or System.is_hardlink(path):
+            logger.debug("Unprotecting '{}'".format(path))
+            tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))
+
+            # The operations order is important here - if some application
+            # would access the file during the process of copyfile then it
+            # would get only the part of file. So, at first, the file should be
+            # copied with the temporary name, and then original file should be
+            # replaced by new.
+            copyfile(
+                path,
+                tmp,
+                name="Unprotecting '{}'".format(os.path.relpath(path)),
+            )
+            remove(path)
+            os.rename(tmp, path)
+
+        else:
+            logger.debug(
+                "Skipping copying for '{}', since it is not "
+                "a symlink or a hardlink.".format(path)
+            )
+
+        os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
+
+    @staticmethod
+    def _unprotect_dir(path):
+        for path in walk_files(path):
+            RemoteLOCAL._unprotect_file(path)
+
+    @staticmethod
+    def unprotect(path_info):
+        path = path_info["path"]
+        if not os.path.exists(path):
+            raise DvcException(
+                "can't unprotect non-existing data '{}'".format(path)
+            )
+
+        if os.path.isdir(path):
+            RemoteLOCAL._unprotect_dir(path)
+        else:
+            RemoteLOCAL._unprotect_file(path)

dvc/remote/s3.py

@@ -2,13 +2,13 @@ from __future__ import unicode_literals
 
 import os
 import threading
+import logging
 
 try:
     import boto3
 except ImportError:
     boto3 = None
 
-import dvc.logger as logger
 from dvc.utils import tmp_fname, move
 from dvc.utils.compat import urlparse, makedirs
 from dvc.progress import progress
@@ -17,6 +17,9 @@ from dvc.remote.base import RemoteBase
 from dvc.exceptions import DvcException
 
 
+logger = logging.getLogger(__name__)
+
+
 class Callback(object):
     def __init__(self, name, total):
         self.name = name

dvc/remote/ssh/__init__.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import getpass
+import logging
 
 try:
     import paramiko
@@ -8,13 +9,15 @@ except ImportError:
     paramiko = None
 
 import dvc.prompt as prompt
-import dvc.logger as logger
 from dvc.remote.ssh.connection import SSHConnection
 from dvc.config import Config
 from dvc.utils.compat import urlparse
 from dvc.remote.base import RemoteBase
 
 
+logger = logging.getLogger(__name__)
+
+
 class RemoteSSH(RemoteBase):
     scheme = "ssh"
 
@@ -184,7 +187,7 @@ class RemoteSSH(RemoteBase):
                         from_info["path"], to_info["path"], progress_title=name
                     )
                 except Exception:
-                    logger.error(
+                    logger.exception(
                         "failed to upload '{host}/{path}' to '{dest}'".format(
                             host=from_info["host"],
                             path=from_info["path"],

dvc/remote/ssh/connection.py

@@ -1,12 +1,13 @@
+import errno
 import os
 import posixpath
+import logging
 
 try:
     import paramiko
 except ImportError:
     paramiko = None
 
-import dvc.logger as logger
 from dvc.utils import tmp_fname
 from dvc.utils.compat import makedirs
 from dvc.progress import progress
@@ -14,6 +15,9 @@ from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteCmdError
 
 
+logger = logging.getLogger(__name__)
+
+
 def sizeof_fmt(num, suffix="B"):
     """ Convert number of bytes to human-readable string """
     for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
@@ -127,7 +131,17 @@ class SSHConnection:
 
         self._sftp_connect()
 
-        for entry in self._sftp.listdir_attr(directory):
+        try:
+            dir_entries = self._sftp.listdir_attr(directory)
+        except IOError as exc:
+            raise DvcException(
+                "couldn't get the '{}' remote directory files list".format(
+                    directory
+                ),
+                cause=exc,
+            )
+
+        for entry in dir_entries:
             path = posixpath.join(directory, entry.filename)
 
             if S_ISLNK(entry.st_mode):
@@ -143,7 +157,11 @@ class SSHConnection:
 
     def remove(self, path):
         self._sftp_connect()
-        self._sftp.remove(path)
+        try:
+            self._sftp.remove(path)
+        except IOError as exc:
+            if exc.errno != errno.ENOENT:
+                raise
 
     def download(self, src, dest, no_progress_bar=False, progress_title=None):
         self._sftp_connect()

dvc/repo/__init__.py

@@ -1,12 +1,19 @@
 from __future__ import unicode_literals
 
-from dvc.utils.compat import str
-
 import os
-import dvc.prompt as prompt
-import dvc.logger as logger
+import logging
 
-from dvc.exceptions import DvcException, NotDvcRepoError, OutputNotFoundError
+import dvc.prompt as prompt
+
+from dvc.exceptions import (
+    DvcException,
+    NotDvcRepoError,
+    OutputNotFoundError,
+    TargetNotDirectoryError,
+)
+
+
+logger = logging.getLogger(__name__)
 
 
 class Repo(object):
@@ -28,6 +35,9 @@ class Repo(object):
     from dvc.repo.status import status
     from dvc.repo.gc import gc
     from dvc.repo.commit import commit
+    from dvc.repo.pkg import install_pkg
+    from dvc.repo.diff import diff
+    from dvc.repo.brancher import brancher
 
     def __init__(self, root_dir=None):
         from dvc.config import Config
@@ -38,6 +48,8 @@ class Repo(object):
         from dvc.data_cloud import DataCloud
         from dvc.updater import Updater
         from dvc.repo.metrics import Metrics
+        from dvc.scm.tree import WorkingTree
+        from dvc.repo.tag import Tag
 
         root_dir = self.find_root(root_dir)
 
@@ -46,6 +58,8 @@ class Repo(object):
 
         self.config = Config(self.dvc_dir)
 
+        self.tree = WorkingTree()
+
         self.scm = SCM(self.root_dir, repo=self)
         self.lock = Lock(self.dvc_dir)
         # NOTE: storing state and link_state in the repository itself to avoid
@@ -54,13 +68,18 @@ class Repo(object):
 
         core = self.config.config[Config.SECTION_CORE]
 
-        logger.set_level(core.get(Config.SECTION_CORE_LOGLEVEL))
+        logger.setLevel(
+            logging.getLevelName(
+                core.get(Config.SECTION_CORE_LOGLEVEL, "info").upper()
+            )
+        )
 
         self.cache = Cache(self)
         self.cloud = DataCloud(self, config=self.config.config)
         self.updater = Updater(self.dvc_dir)
 
         self.metrics = Metrics(self)
+        self.tag = Tag(self)
 
         self._ignore()
 
@@ -97,11 +116,9 @@ class Repo(object):
         init(root_dir=root_dir, no_scm=no_scm, force=force)
         return Repo(root_dir)
 
-    @staticmethod
-    def unprotect(target):
-        from dvc.repo.unprotect import unprotect
-
-        return unprotect(target)
+    def unprotect(self, target):
+        path_info = {"schema": "local", "path": target}
+        return self.cache.local.unprotect(path_info)
 
     def _ignore(self):
         flist = [
@@ -132,25 +149,27 @@ class Repo(object):
             raise CyclicGraphError(cycles[0])
 
     def _get_pipeline(self, node):
-        pipelines = list(filter(lambda g: node in g.nodes(), self.pipelines()))
+        pipelines = [i for i in self.pipelines() if i.has_node(node)]
         assert len(pipelines) == 1
         return pipelines[0]
 
-    def collect(self, target, with_deps=False):
+    def collect(self, target, with_deps=False, recursive=False):
         import networkx as nx
         from dvc.stage import Stage
 
+        if not target or recursive:
+            return self.active_stages(target)
+
         stage = Stage.load(self, target)
         if not with_deps:
             return [stage]
 
         node = os.path.relpath(stage.path, self.root_dir)
         G = self._get_pipeline(node)
-        stages = nx.get_node_attributes(G, "stage")
 
-        ret = [stage]
+        ret = []
         for n in nx.dfs_postorder_nodes(G, node):
-            ret.append(stages[n])
+            ret.append(G.node[n]["stage"])
 
         return ret
 
@@ -243,7 +262,7 @@ class Repo(object):
         cache["ssh"] = []
         cache["azure"] = []
 
-        for branch in self.scm.brancher(
+        for branch in self.brancher(
             all_branches=all_branches, all_tags=all_tags
         ):
             if target:
@@ -282,6 +301,7 @@ class Repo(object):
         from dvc.exceptions import (
             OutputDuplicationError,
             StagePathAsOutputError,
+            OverlappingOutputPathsError,
         )
 
         G = nx.DiGraph()
@@ -292,7 +312,15 @@ class Repo(object):
 
         for stage in stages:
             for out in stage.outs:
-                existing = [o.stage for o in outs if o.path == out.path]
+                existing = []
+                for o in outs:
+                    if o.path == out.path:
+                        existing.append(o.stage)
+
+                    in_o_dir = out.path.startswith(o.path + o.sep)
+                    in_out_dir = o.path.startswith(out.path + out.sep)
+                    if in_o_dir or in_out_dir:
+                        raise OverlappingOutputPathsError(o, out)
 
                 if existing:
                     stages = [stage.relpath, existing[0].relpath]
@@ -357,13 +385,15 @@ class Repo(object):
 
         if not from_directory:
             from_directory = self.root_dir
+        elif not os.path.isdir(from_directory):
+            raise TargetNotDirectoryError(from_directory)
 
         stages = []
         outs = []
-        for root, dirs, files in os.walk(str(from_directory)):
+        for root, dirs, files in self.tree.walk(from_directory):
             for fname in files:
                 path = os.path.join(root, fname)
-                if not Stage.is_stage_file(path):
+                if not Stage.is_valid_filename(path):
                     continue
                 stage = Stage.load(self, path)
                 for out in stage.outs:
@@ -396,10 +426,14 @@ class Repo(object):
 
     def find_outs_by_path(self, path, outs=None, recursive=False):
         if not outs:
-            outs = [out for stage in self.stages() for out in stage.outs]
+            # there is no `from_directory=path` argument because some data
+            # files might be generated to an upper level, and so it is
+            # needed to look at all the files (project root_dir)
+            stages = self.stages()
+            outs = [out for stage in stages for out in stage.outs]
 
         abs_path = os.path.abspath(path)
-        is_dir = os.path.isdir(abs_path)
+        is_dir = self.tree.isdir(abs_path)
 
         def func(out):
             if out.path == abs_path:

dvc/repo/add.py

@@ -1,6 +1,6 @@
 import os
 
-from dvc.scm import scm_context
+from dvc.repo.scm_context import scm_context
 from dvc.stage import Stage
 from dvc.utils import walk_files
 from dvc.exceptions import RecursiveAddingWhileUsingFilename

dvc/repo/brancher.py

@@ -0,0 +1,56 @@
+def brancher(  # noqa: E302
+    self, branches=None, all_branches=False, tags=None, all_tags=False
+):
+    """Generator that iterates over specified revisions.
+
+    Args:
+        branches (list): a list of branches to iterate over.
+        all_branches (bool): iterate over all available branches.
+        tags (list): a list of tags to iterate over.
+        all_tags (bool): iterate over all available tags.
+
+    Yields:
+        str: the display name for the currently selected tree, it could be:
+            - a git revision identifier
+            - empty string it there is no branches to iterate over
+            - "Working Tree" if there are uncommited changes in the SCM repo
+    """
+    if not any([branches, all_branches, tags, all_tags]):
+        yield ""
+        return
+
+    saved_tree = self.tree
+    revs = []
+
+    scm = self.scm
+
+    if self.scm.is_dirty():
+        from dvc.scm.tree import WorkingTree
+
+        self.tree = WorkingTree()
+        yield "Working Tree"
+
+    if all_branches:
+        branches = scm.list_branches()
+
+    if all_tags:
+        tags = scm.list_tags()
+
+    if branches is None:
+        revs.extend([scm.active_branch()])
+    else:
+        revs.extend(branches)
+
+    if tags is not None:
+        revs.extend(tags)
+
+    # NOTE: it might be a good idea to wrap this loop in try/finally block
+    # to don't leave the tree on some unexpected branch after the
+    # `brancher()`, but this could cause problems on exception handling
+    # code which might expect the tree on which exception was raised to
+    # stay in place. This behavior is a subject to change.
+    for rev in revs:
+        self.tree = scm.get_tree(rev)
+        yield rev
+
+    self.tree = saved_tree

dvc/repo/checkout.py

@@ -1,7 +1,12 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
-from dvc.exceptions import DvcException
+import logging
+
+from dvc.exceptions import CheckoutErrorSuggestGit
+from dvc.progress import ProgressCallback
+
+
+logger = logging.getLogger(__name__)
 
 
 def _cleanup_unused_links(self, all_stages):
@@ -12,23 +17,32 @@ def _cleanup_unused_links(self, all_stages):
     self.state.remove_unused_links(used)
 
 
+def get_all_files_numbers(stages):
+    return sum(stage.get_all_files_number() for stage in stages)
+
+
+def get_progress_callback(stages):
+    total_files_num = get_all_files_numbers(stages)
+    if total_files_num == 0:
+        return None
+    return ProgressCallback(total_files_num)
+
+
 def checkout(self, target=None, with_deps=False, force=False, recursive=False):
     from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
 
-    if target and not recursive:
-        all_stages = self.active_stages()
-        try:
-            stages = self.collect(target, with_deps=with_deps)
-        except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
-            raise DvcException(
-                str(exc) + " Did you mean 'git checkout {}'?".format(target)
-            )
-    else:
-        all_stages = self.active_stages(target)
-        stages = all_stages
+    all_stages = self.stages()
+
+    try:
+        stages = self.collect(target, with_deps=with_deps, recursive=recursive)
+    except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
+        if not target:
+            raise
+        raise CheckoutErrorSuggestGit(target, exc)
 
     with self.state:
         _cleanup_unused_links(self, all_stages)
+        progress_callback = get_progress_callback(stages)
 
         for stage in stages:
             if stage.locked:
@@ -37,4 +51,6 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False):
                     " not going to be checked out.".format(path=stage.relpath)
                 )
 
-            stage.checkout(force=force)
+            stage.checkout(force=force, progress_callback=progress_callback)
+        if progress_callback:
+            progress_callback.finish("Checkout finished!")

dvc/repo/commit.py

@@ -1,9 +1,5 @@
 def commit(self, target, with_deps=False, recursive=False, force=False):
-    if target and not recursive:
-        stages = self.collect(target, with_deps=with_deps)
-    else:
-        stages = self.active_stages(target)
-
+    stages = self.collect(target, with_deps=with_deps, recursive=recursive)
     with self.state:
         for stage in stages:
             stage.check_can_commit(force=force)

dvc/repo/diff.py

@@ -0,0 +1,252 @@
+from __future__ import unicode_literals
+
+import os
+from errno import ENOENT
+
+
+import dvc.logger as logger
+from dvc.scm.base import FileNotInCommitError
+from dvc.scm.git import (
+    DIFF_A_TREE,
+    DIFF_B_TREE,
+    DIFF_A_REF,
+    DIFF_B_REF,
+    DIFF_EQUAL,
+)
+
+
+DIFF_TARGET = "target"
+DIFF_IS_DIR = "is_dir"
+DIFF_OLD_FILE = "old_file"
+DIFF_OLD_CHECKSUM = "old_checksum"
+DIFF_NEW_FILE = "new_file"
+DIFF_NEW_CHECKSUM = "new_checksum"
+DIFF_SIZE = "size_diff"
+DIFF_DEL = "del"
+DIFF_IDENT = "ident"
+DIFF_CHANGE = "changes"
+DIFF_NEW = "new"
+DIFF_MOVE = "moves"
+DIFF_LIST = "diffs"
+DIFF_SIZE_UNKNOWN = "?"
+DIFF_A_OUTPUT = "a_output"
+DIFF_B_OUTPUT = "b_output"
+DIFF_DELETED = "deleted_file"
+DIFF_IS_NEW = "created_file"
+
+
+def _file_not_exists(error, result):
+    if error.errno == ENOENT:
+        result.update({DIFF_SIZE: DIFF_SIZE_UNKNOWN})
+    else:
+        raise error
+
+
+def _extract_dir(self, dir_not_exists, output):
+    """Extract the content of dvc tree file
+    Args:
+        self(object) - Repo class instance
+        dir_not_exists(bool) - flag for directory existence
+        output(object) - OutputLOCAL class instance
+    Returns:
+        dict - dictionary with keys - paths to file in .dvc/cache
+                               values -checksums for that files
+    """
+    if not dir_not_exists:
+        lst = output.dir_cache
+        return {i["relpath"]: i["md5"] for i in lst}
+    return {}
+
+
+def _get_dir_info(dir_not_exists, a_output):
+    if not dir_not_exists:
+        return str(a_output), a_output.checksum
+    return "", ""
+
+
+def _ident_files(a_entries, b_entries):
+    keys = [
+        key for key in a_entries.keys() if b_entries.get(key) == a_entries[key]
+    ]
+    return len(keys)
+
+
+def _modified_files(self, a_entries, b_entries):
+    keys = [key for key in a_entries.keys() if key in b_entries]
+    diff_size = 0
+    modified_count = 0
+    for key in keys:
+        if a_entries[key] != b_entries[key]:
+            modified_count
+            diff_size += os.path.getsize(
+                self.cache.local.get(b_entries[key])
+            ) - os.path.getsize(self.cache.local.get(a_entries[key]))
+    return modified_count, diff_size
+
+
+def _deleted_files(self, a_entries, b_entries):
+    diff_size = 0
+    deleted_count = 0
+    for key, value in a_entries.items():
+        if key not in b_entries:
+            deleted_count += 1
+            diff_size -= os.path.getsize(self.cache.local.get(a_entries[key]))
+    return deleted_count, diff_size
+
+
+def _new_files(self, a_entries, b_entries):
+    diff_size = 0
+    new_count = 0
+    for key, value in b_entries.items():
+        if key not in a_entries:
+            new_count += 1
+            diff_size += os.path.getsize(self.cache.local.get(b_entries[key]))
+    return new_count, diff_size
+
+
+def _get_tree_changes(self, a_entries, b_entries):
+    result = {
+        DIFF_DEL: 0,
+        DIFF_IDENT: 0,
+        DIFF_CHANGE: 0,
+        DIFF_NEW: 0,
+        DIFF_MOVE: 0,
+    }
+    result[DIFF_IDENT] = _ident_files(a_entries, b_entries)
+    result[DIFF_CHANGE], diff_size = _modified_files(
+        self, a_entries, b_entries
+    )
+    result[DIFF_SIZE] = diff_size
+    result[DIFF_DEL], diff_size = _deleted_files(self, a_entries, b_entries)
+    result[DIFF_SIZE] += diff_size
+    result[DIFF_NEW], diff_size = _new_files(self, a_entries, b_entries)
+    result[DIFF_SIZE] += diff_size
+    return result
+
+
+def _check_local_cache(a_out, is_checked):
+    if a_out is not None and a_out.scheme != "local":
+        is_checked.append(str(a_out))
+        return True
+    return False
+
+
+def _is_dir(path, a_outs, b_outs):
+    if a_outs.get(path):
+        return a_outs[path].is_dir_cache
+    else:
+        return b_outs[path].is_dir_cache
+
+
+def _get_diff_outs(self, diff_dct):
+    self.tree = diff_dct[DIFF_A_TREE]
+    a_outs = {str(out): out for st in self.stages() for out in st.outs}
+    self.tree = diff_dct[DIFF_B_TREE]
+    b_outs = {str(out): out for st in self.stages() for out in st.outs}
+    outs_paths = set(a_outs.keys())
+    outs_paths.update(b_outs.keys())
+    results = {}
+    non_local_cache = []
+    for path in outs_paths:
+        check1 = _check_local_cache(a_outs.get(path), non_local_cache)
+        check2 = _check_local_cache(b_outs.get(path), non_local_cache)
+        # skip files/directories with non-local cache for now
+        if check1 or check2:
+            continue
+        results[path] = {}
+        results[path][DIFF_A_OUTPUT] = a_outs.get(path)
+        results[path][DIFF_B_OUTPUT] = b_outs.get(path)
+        results[path][DIFF_IS_NEW] = path not in a_outs
+        results[path][DIFF_DELETED] = path not in b_outs
+        results[path][DIFF_IS_DIR] = _is_dir(path, a_outs, b_outs)
+    if non_local_cache:
+        logger.warning(
+            "Diff is not supported for non-local outputs. Ignoring: {}".format(
+                non_local_cache
+            )
+        )
+
+    return results
+
+
+def _diff_dir(self, target, diff_dct):
+    result = {DIFF_TARGET: target}
+    result[DIFF_IS_DIR] = True
+    a_entries, b_entries = {}, {}
+    try:
+        a_entries = _extract_dir(
+            self, diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
+        )
+        b_entries = _extract_dir(
+            self, diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
+        )
+        result[DIFF_OLD_FILE], result[DIFF_OLD_CHECKSUM] = _get_dir_info(
+            diff_dct[DIFF_IS_NEW], diff_dct[DIFF_A_OUTPUT]
+        )
+        result[DIFF_NEW_FILE], result[DIFF_NEW_CHECKSUM] = _get_dir_info(
+            diff_dct[DIFF_DELETED], diff_dct[DIFF_B_OUTPUT]
+        )
+        result.update(_get_tree_changes(self, a_entries, b_entries))
+    except IOError as e:
+        _file_not_exists(e, result)
+    return result
+
+
+def _diff_file(self, target, diff_dct):
+    result = {DIFF_TARGET: target}
+    size = 0
+    try:
+        if not diff_dct[DIFF_IS_NEW]:
+            result[DIFF_OLD_FILE] = target
+            result[DIFF_OLD_CHECKSUM] = diff_dct[DIFF_A_OUTPUT].checksum
+            size -= os.path.getsize(
+                self.cache.local.get(diff_dct[DIFF_A_OUTPUT].checksum)
+            )
+        if not diff_dct[DIFF_DELETED]:
+            result[DIFF_NEW_FILE] = target
+            result[DIFF_NEW_CHECKSUM] = diff_dct[DIFF_B_OUTPUT].checksum
+            size += os.path.getsize(
+                self.cache.local.get(diff_dct[DIFF_B_OUTPUT].checksum)
+            )
+        result[DIFF_SIZE] = size
+    except IOError as e:
+        _file_not_exists(e, result)
+    return result
+
+
+def _diff_royal(self, target, diff_dct):
+    if diff_dct[DIFF_IS_DIR]:
+        return _diff_dir(self, target, diff_dct)
+    return _diff_file(self, target, diff_dct)
+
+
+def diff(self, a_ref, target=None, b_ref=None):
+    """Gerenates diff message string output
+
+    Args:
+        target(str) - file/directory to check diff of
+        a_ref(str) - first tag
+        (optional) b_ref(str) - second git tag
+
+    Returns:
+        string: string of output message with diff info
+    """
+    result = {}
+    diff_dct = self.scm.get_diff_trees(a_ref, b_ref=b_ref)
+    result[DIFF_A_REF] = diff_dct[DIFF_A_REF]
+    result[DIFF_B_REF] = diff_dct[DIFF_B_REF]
+    if diff_dct[DIFF_EQUAL]:
+        result[DIFF_EQUAL] = True
+        return result
+    result[DIFF_LIST] = []
+    diff_outs = _get_diff_outs(self, diff_dct)
+    if target is None:
+        result[DIFF_LIST] = [
+            _diff_royal(self, path, diff_outs[path]) for path in diff_outs
+        ]
+    elif target in diff_outs:
+        result[DIFF_LIST] = [_diff_royal(self, target, diff_outs[target])]
+    else:
+        msg = "Have not found file/directory '{}' in the commits"
+        raise FileNotInCommitError(msg.format(target))
+    return result

dvc/repo/gc.py

@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
 import collections
+import logging
 
-import dvc.logger as logger
+
+logger = logging.getLogger(__name__)
 
 
 def _merge_cache_lists(clists):

dvc/repo/imp.py

@@ -1,4 +1,4 @@
-from dvc.scm import scm_context
+from dvc.repo.scm_context import scm_context
 
 
 @scm_context

dvc/repo/init.py

@@ -1,22 +1,28 @@
 import os
 import shutil
 import colorama
+import logging
 
-import dvc.logger as logger
 from dvc.repo import Repo
-from dvc.scm import SCM, Base
+from dvc.scm import SCM, NoSCM
 from dvc.config import Config
 from dvc.exceptions import InitError
+from dvc.utils import boxify
+
+
+logger = logging.getLogger(__name__)
 
 
 def _welcome_message():
-    logger.box(
-        "DVC has enabled anonymous aggregate usage analytics.\n"
-        "Read the analytics documentation (and how to opt-out) here:\n"
-        "{blue}https://dvc.org/doc/user-guide/analytics{nc}".format(
-            blue=colorama.Fore.BLUE, nc=colorama.Fore.RESET
-        ),
-        border_color="red",
+    logger.info(
+        boxify(
+            "DVC has enabled anonymous aggregate usage analytics.\n"
+            "Read the analytics documentation (and how to opt-out) here:\n"
+            "{blue}https://dvc.org/doc/user-guide/analytics{nc}".format(
+                blue=colorama.Fore.BLUE, nc=colorama.Fore.RESET
+            ),
+            border_color="red",
+        )
     )
 
     msg = (
@@ -56,7 +62,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
     root_dir = os.path.abspath(root_dir)
     dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
     scm = SCM(root_dir)
-    if type(scm) == Base and not no_scm:
+    if isinstance(scm, NoSCM) and not no_scm:
         raise InitError(
             "{repo} is not tracked by any supported scm tool (e.g. git). "
             "Use '--no-scm' if you don't want to use any scm.".format(

dvc/repo/metrics/show.py

@@ -3,11 +3,14 @@ from __future__ import unicode_literals
 import os
 import csv
 import json
+import logging
 from jsonpath_rw import parse
 
-import dvc.logger as logger
 from dvc.exceptions import OutputNotFoundError, BadMetricError, NoMetricsError
-from dvc.utils.compat import builtin_str, open
+from dvc.utils.compat import builtin_str, open, StringIO, csv_reader
+
+
+logger = logging.getLogger(__name__)
 
 
 def _read_metric_json(fd, json_path):
@@ -66,33 +69,102 @@ def _read_typed_metric(typ, xpath, fd):
     return ret
 
 
-def _read_metric(path, typ=None, xpath=None, branch=None):
-    ret = None
-    if not os.path.exists(path):
-        return ret
+def _format_csv(content, delimiter):
+    """Format delimited text to have same column width.
 
+    Args:
+        content (str): The content of a metric.
+        delimiter (str): Value separator
+
+    Returns:
+        str: Formatted content.
+
+    Example:
+
+        >>> content = (
+            "value_mse,deviation_mse,data_set\n"
+            "0.421601,0.173461,train\n"
+            "0.67528,0.289545,testing\n"
+            "0.671502,0.297848,validation\n"
+        )
+        >>> _format_csv(content, ",")
+
+        "value_mse  deviation_mse   data_set\n"
+        "0.421601   0.173461        train\n"
+        "0.67528    0.289545        testing\n"
+        "0.671502   0.297848        validation\n"
+    """
+    reader = csv_reader(StringIO(content), delimiter=builtin_str(delimiter))
+    rows = [row for row in reader]
+    max_widths = [max(map(len, column)) for column in zip(*rows)]
+
+    lines = [
+        " ".join(
+            "{entry:{width}}".format(entry=entry, width=width + 2)
+            for entry, width in zip(row, max_widths)
+        )
+        for row in rows
+    ]
+
+    return "\n".join(lines)
+
+
+def _format_output(content, typ):
+    """Tabularize the content according to its type.
+
+    Args:
+        content (str): The content of a metric.
+        typ (str): The type of metric -- (raw|json|tsv|htsv|csv|hcsv).
+
+    Returns:
+        str: Content in a raw or tabular format.
+    """
+
+    if "csv" in str(typ):
+        return _format_csv(content, delimiter=",")
+
+    if "tsv" in str(typ):
+        return _format_csv(content, delimiter="\t")
+
+    return content
+
+
+def _read_metric(fd, typ=None, xpath=None, rel_path=None, branch=None):
     typ = typ.lower().strip() if typ else typ
-    xpath = xpath.strip() if xpath else xpath
     try:
-        with open(path, "r") as fd:
-            if not xpath:
-                ret = fd.read().strip()
-            else:
-                ret = _read_typed_metric(typ, xpath, fd)
+        if xpath:
+            return _read_typed_metric(typ, xpath.strip(), fd)
+        else:
+            return _format_output(fd.read().strip(), typ)
     # Json path library has to be replaced or wrapped in
     # order to fix this too broad except clause.
     except Exception:
-        logger.warning(
+        logger.exception(
             "unable to read metric in '{}' in branch '{}'".format(
-                path, branch
-            ),
-            parse_exception=True,
+                rel_path, branch
+            )
         )
-
-    return ret
+        return None
 
 
 def _collect_metrics(self, path, recursive, typ, xpath, branch):
+    """Gather all the metric outputs.
+
+    Args:
+        path (str): Path to a metric file or a directory.
+        recursive (bool): If path is a directory, do a recursive search for
+            metrics on the given path.
+        typ (str): The type of metric to search for, could be one of the
+            following (raw|json|tsv|htsv|csv|hcsv).
+        xpath (str): Path to search for.
+        branch (str): Branch to look up for metrics.
+
+    Returns:
+        list(tuple): (output, typ, xpath)
+            - output:
+            - typ:
+            - xpath:
+    """
     outs = [out for stage in self.stages() for out in stage.outs]
 
     if path:
@@ -123,16 +195,50 @@ def _collect_metrics(self, path, recursive, typ, xpath, branch):
     return res
 
 
+def _read_metrics_filesystem(path, typ, xpath, rel_path, branch):
+    if not os.path.exists(path):
+        return None
+    with open(path, "r") as fd:
+        return _read_metric(
+            fd, typ=typ, xpath=xpath, rel_path=rel_path, branch=branch
+        )
+
+
 def _read_metrics(self, metrics, branch):
+    """Read the content of each metric file and format it.
+
+    Args:
+        metrics (list): List of metric touples
+        branch (str): Branch to look up for metrics.
+
+    Returns:
+        A dict mapping keys with metrics path name and content.
+        For example:
+
+        {'metric.csv': ("value_mse  deviation_mse   data_set\n"
+                        "0.421601   0.173461        train\n"
+                        "0.67528    0.289545        testing\n"
+                        "0.671502   0.297848        validation\n")}
+    """
     res = {}
     for out, typ, xpath in metrics:
         assert out.scheme == "local"
+        if not typ:
+            typ = os.path.splitext(out.path.lower())[1].replace(".", "")
         if out.use_cache:
-            path = self.cache.local.get(out.checksum)
+            metric = _read_metrics_filesystem(
+                self.cache.local.get(out.checksum),
+                typ=typ,
+                xpath=xpath,
+                rel_path=out.rel_path,
+                branch=branch,
+            )
         else:
-            path = out.path
+            fd = self.tree.open(out.path)
+            metric = _read_metric(
+                fd, typ=typ, xpath=xpath, rel_path=out.rel_path, branch=branch
+            )
 
-        metric = _read_metric(path, typ=typ, xpath=xpath, branch=branch)
         if not metric:
             continue
 
@@ -151,9 +257,8 @@ def show(
     recursive=False,
 ):
     res = {}
-    for branch in self.scm.brancher(
-        all_branches=all_branches, all_tags=all_tags
-    ):
+
+    for branch in self.brancher(all_branches=all_branches, all_tags=all_tags):
         entries = _collect_metrics(self, path, recursive, typ, xpath, branch)
         metrics = _read_metrics(self, entries, branch)
         if metrics:

dvc/repo/move.py

@@ -1,7 +1,7 @@
 import os
 
 from dvc.exceptions import MoveNotDataSourceError
-from dvc.scm import scm_context
+from dvc.repo.scm_context import scm_context
 
 
 def _expand_target_path(from_path, to_path):

dvc/repo/pkg.py

@@ -0,0 +1,205 @@
+from __future__ import unicode_literals
+
+import os
+import shutil
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.stage import Stage
+from dvc.scm.git.temp_git_repo import TempGitRepo
+
+
+logger = logging.getLogger(__name__)
+
+
+class PackageParams(object):
+    def __init__(self, address, target_dir, select=[], file=None):
+        self.address = address
+        self.target_dir = target_dir
+        self.select = select
+        self.file = file
+
+    @property
+    def all_addresses(self):
+        if self.address:
+            return [self.address]
+        return PackageManager.read_packages()
+
+
+class PackageManager(object):
+    PACKAGE_FILE = "package.yaml"
+
+    @staticmethod
+    def read_packages():
+        return []
+
+    @staticmethod
+    def get_package(addr):
+        for pkg_class in [GitPackage]:
+            try:
+                return pkg_class()
+            except Exception:
+                pass
+        return None
+
+    def __init__(self, addr):
+        self._addr = addr
+
+
+class Package(object):
+    MODULES_DIR = "dvc_mod"
+
+    def install_or_update(self, parent_repo, pkg_param):
+        raise NotImplementedError(
+            "A method of abstract Package class was called"
+        )
+
+    def is_in_root(self):
+        return True
+
+
+class GitPackage(Package):
+    DEF_DVC_FILE_PREFIX = "mod_"
+
+    def install_or_update(self, parent_repo, pkg_params):
+        from git.cmd import Git
+
+        if not self.is_in_root():
+            raise DvcException(
+                "This command can be run only from a repository root"
+            )
+
+        if not os.path.exists(self.MODULES_DIR):
+            logger.debug("Creating modules dir {}".format(self.MODULES_DIR))
+            os.makedirs(self.MODULES_DIR)
+            parent_repo.scm.ignore(os.path.abspath(self.MODULES_DIR))
+
+        module_name = (
+            Git.polish_url(pkg_params.address).strip("/").split("/")[-1]
+        )
+        if not module_name:
+            raise DvcException(
+                "Package address error: unable to extract package name"
+            )
+
+        with TempGitRepo(
+            pkg_params.address, module_name, Package.MODULES_DIR
+        ) as tmp_repo:
+            outputs_to_copy = tmp_repo.outs
+            if pkg_params.select:
+                outputs_to_copy = list(
+                    filter(
+                        lambda out: out.dvc_path in pkg_params.select,
+                        outputs_to_copy,
+                    )
+                )
+
+            fetched_stage_files = set(
+                map(lambda o: o.stage.path, outputs_to_copy)
+            )
+            tmp_repo.fetch(fetched_stage_files)
+
+            module_dir = self.create_module_dir(module_name)
+            tmp_repo.persist_to(module_dir, parent_repo)
+
+            dvc_file = self.get_dvc_file_name(
+                pkg_params.file, pkg_params.target_dir, module_name
+            )
+            try:
+                self.persist_stage_and_scm_state(
+                    parent_repo,
+                    outputs_to_copy,
+                    pkg_params.target_dir,
+                    dvc_file,
+                )
+            except Exception as ex:
+                raise DvcException(
+                    "Package '{}' was installed "
+                    "but stage file '{}' "
+                    "was not created properly: {}".format(
+                        pkg_params.address, dvc_file, ex
+                    )
+                )
+
+        parent_repo.checkout(dvc_file)
+
+    @staticmethod
+    def persist_stage_and_scm_state(
+        parent_repo, outputs_to_copy, target_dir, dvc_file
+    ):
+        stage = Stage.create(
+            repo=parent_repo,
+            fname=dvc_file,
+            validate_state=False,
+            wdir=target_dir,
+        )
+        stage.outs = list(
+            map(lambda o: o.assign_to_stage_file(stage), outputs_to_copy)
+        )
+
+        for out in stage.outs:
+            parent_repo.scm.ignore(out.path, in_curr_dir=True)
+
+        stage.dump()
+
+    @staticmethod
+    def create_module_dir(module_name):
+        module_dir = os.path.join(GitPackage.MODULES_DIR, module_name)
+        if os.path.exists(module_dir):
+            logger.info("Updating package {}".format(module_name))
+            shutil.rmtree(module_dir)
+        else:
+            logger.info("Adding package {}".format(module_name))
+        return module_dir
+
+    def get_dvc_file_name(self, stage_file, target_dir, module_name):
+        if stage_file:
+            dvc_file_path = stage_file
+        else:
+            dvc_file_name = self.DEF_DVC_FILE_PREFIX + module_name + ".dvc"
+            dvc_file_path = os.path.join(target_dir, dvc_file_name)
+        return dvc_file_path
+
+
+def install_pkg(self, pkg_params):
+    """
+    Install package.
+
+    The command can be run only from DVC project root.
+
+    E.g.
+          Having: DVC package in https://github.com/dmpetrov/tag_classifier
+
+          $ dvc pkg install https://github.com/dmpetrov/tag_classifier
+
+          Result: tag_classifier package in dvc_mod/ directory
+    """
+
+    if not os.path.isdir(pkg_params.target_dir):
+        logger.error(
+            "Unable to install package: "
+            "target directory '{}' does not exist".format(
+                pkg_params.target_dir
+            )
+        )
+        return 1
+
+    curr_dir = os.path.realpath(os.curdir)
+    if not os.path.realpath(pkg_params.target_dir).startswith(curr_dir):
+        logger.error(
+            "Unable to install package: the current dir should be"
+            " a subdirectory of the target dir {}".format(
+                pkg_params.target_dir
+            )
+        )
+        return 1
+
+    for addr in pkg_params.all_addresses:
+        try:
+            mgr = PackageManager.get_package(addr)
+            mgr.install_or_update(self, pkg_params)
+        except Exception as ex:
+            logger.exception("Unable to install package: ".format(ex))
+            return 1
+
+    return 0

dvc/repo/remove.py

@@ -3,8 +3,8 @@ def remove(self, target, outs_only=False):
 
     stage = Stage.load(self, target)
     if outs_only:
-        stage.remove_outs()
+        stage.remove_outs(force=True)
     else:
-        stage.remove()
+        stage.remove(force=True)
 
     return stage

dvc/repo/reproduce.py

@@ -1,10 +1,13 @@
 from __future__ import unicode_literals
 
 import os
+import logging
 
-import dvc.logger as logger
 from dvc.exceptions import ReproductionError
-from dvc.scm import scm_context
+from dvc.repo.scm_context import scm_context
+
+
+logger = logging.getLogger(__name__)
 
 
 def _reproduce_stage(stages, node, force, dry, interactive, no_commit):

dvc/repo/run.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from dvc.scm import scm_context
+from dvc.repo.scm_context import scm_context
 
 
 @scm_context
@@ -20,6 +20,8 @@ def run(
     ignore_build_cache=False,
     remove_outs=False,
     no_commit=False,
+    outs_persist=None,
+    outs_persist_no_cache=None,
 ):
     from dvc.stage import Stage
 
@@ -33,6 +35,10 @@ def run(
         metrics = []
     if metrics_no_cache is None:
         metrics_no_cache = []
+    if outs_persist is None:
+        outs_persist = []
+    if outs_persist_no_cache is None:
+        outs_persist_no_cache = []
 
     with self.state:
         stage = Stage.create(
@@ -49,6 +55,8 @@ def run(
             overwrite=overwrite,
             ignore_build_cache=ignore_build_cache,
             remove_outs=remove_outs,
+            outs_persist=outs_persist,
+            outs_persist_no_cache=outs_persist_no_cache,
         )
 
     if stage is None:

dvc/repo/scm_context.py

@@ -0,0 +1,12 @@
+def scm_context(method):
+    def run(repo, *args, **kw):
+        try:
+            result = method(repo, *args, **kw)
+            repo.scm.reset_ignores()
+            repo.scm.remind_to_track()
+            return result
+        except Exception as e:
+            repo.scm.cleanup_ignores()
+            raise e
+
+    return run

dvc/repo/status.py

@@ -1,15 +1,15 @@
 from __future__ import unicode_literals
 
-import dvc.logger as logger
+import logging
+
+
+logger = logging.getLogger(__name__)
 
 
 def _local_status(self, target=None, with_deps=False):
     status = {}
 
-    if target:
-        stages = self.collect(target, with_deps=with_deps)
-    else:
-        stages = self.active_stages()
+    stages = self.collect(target, with_deps=with_deps)
 
     for stage in stages:
         if stage.locked:
@@ -54,7 +54,7 @@ def _cloud_status(
     for md5, info in status_info.items():
         name = info["name"]
         status = info["status"]
-        if status == cloud.STATUS_OK:
+        if status in [cloud.STATUS_OK, cloud.STATUS_MISSING]:
             continue
 
         prefix_map = {cloud.STATUS_DELETED: "deleted", cloud.STATUS_NEW: "new"}

dvc/repo/tag/__init__.py

@@ -0,0 +1,18 @@
+class Tag(object):
+    def __init__(self, repo):
+        self.repo = repo
+
+    def add(self, *args, **kwargs):
+        from dvc.repo.tag.add import add
+
+        return add(self.repo, *args, **kwargs)
+
+    def list(self, *args, **kwargs):
+        from dvc.repo.tag.list import list
+
+        return list(self.repo, *args, **kwargs)
+
+    def remove(self, *args, **kwargs):
+        from dvc.repo.tag.remove import remove
+
+        return remove(self.repo, *args, **kwargs)

dvc/repo/tag/add.py

@@ -0,0 +1,18 @@
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+def add(self, tag, target=None, with_deps=False, recursive=False):
+    stages = self.collect(target, with_deps=with_deps, recursive=recursive)
+    for stage in stages:
+        changed = False
+        for out in stage.outs:
+            if not out.info:
+                logger.warning("missing checksum info for '{}'".format(out))
+                continue
+            out.tags[tag] = out.info.copy()
+            changed = True
+        if changed:
+            stage.dump()

dvc/repo/tag/list.py

@@ -0,0 +1,11 @@
+def list(self, target=None, with_deps=False, recursive=False):
+    ret = {}
+    stages = self.collect(target, with_deps=with_deps, recursive=recursive)
+    for stage in stages:
+        outs_tags = {}
+        for out in stage.outs:
+            if out.tags:
+                outs_tags[str(out)] = out.tags.copy()
+        if outs_tags:
+            ret.update({stage.relpath: outs_tags})
+    return ret

dvc/repo/tag/remove.py

@@ -0,0 +1,18 @@
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+def remove(self, tag, target=None, with_deps=False, recursive=False):
+    stages = self.collect(target, with_deps=with_deps, recursive=recursive)
+    for stage in stages:
+        changed = False
+        for out in stage.outs:
+            if tag not in out.tags.keys():
+                logger.warning("tag '{}' not found for '{}'".format(tag, out))
+                continue
+            del out.tags[tag]
+            changed = True
+        if changed:
+            stage.dump()

dvc/repo/unprotect.py

@@ -1,48 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-import stat
-import uuid
-
-import dvc.logger as logger
-from dvc.system import System
-from dvc.utils import copyfile, move, remove
-from dvc.exceptions import DvcException
-
-
-def _unprotect_file(path):
-    if System.is_symlink(path) or System.is_hardlink(path):
-        logger.debug("Unprotecting '{}'".format(path))
-
-        tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))
-        move(path, tmp)
-
-        copyfile(tmp, path)
-
-        remove(tmp)
-    else:
-        logger.debug(
-            "Skipping copying for '{}', since it is not "
-            "a symlink or a hardlink.".format(path)
-        )
-
-    os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
-
-
-def _unprotect_dir(path):
-    for root, dirs, files in os.walk(str(path)):
-        for f in files:
-            path = os.path.join(root, f)
-            _unprotect_file(path)
-
-
-def unprotect(path):
-    if not os.path.exists(path):
-        raise DvcException(
-            "can't unprotect non-existing data '{}'".format(path)
-        )
-
-    if os.path.isdir(path):
-        _unprotect_dir(path)
-    else:
-        _unprotect_file(path)

dvc/scm/__init__.py

@@ -5,7 +5,11 @@ from __future__ import unicode_literals
 from dvc.scm.base import Base
 from dvc.scm.git import Git
 
-import dvc
+
+# just a sugar to point that this is an actual implementation for a dvc
+# project under no SCM control
+class NoSCM(Base):
+    pass
 
 
 def SCM(root_dir, repo=None):  # pylint: disable=invalid-name
@@ -22,20 +26,4 @@ def SCM(root_dir, repo=None):  # pylint: disable=invalid-name
     if Git.is_repo(root_dir) or Git.is_submodule(root_dir):
         return Git(root_dir, repo=repo)
 
-    return Base(root_dir, repo=repo)
-
-
-def scm_context(method):
-    def run(*args, **kw):
-        repo = args[0]
-        assert isinstance(repo, dvc.repo.Repo)
-        try:
-            result = method(*args, **kw)
-            repo.scm.reset_ignores()
-            repo.scm.remind_to_track()
-            return result
-        except Exception as e:
-            repo.scm.cleanup_ignores()
-            raise e
-
-    return run
+    return NoSCM(root_dir, repo=repo)

dvc/scm/base.py

@@ -17,12 +17,23 @@ class FileNotInRepoError(DvcException):
     """
 
 
+class FileNotInCommitError(DvcException):
+    """Thrown when trying to find a file/directory that is not
+    in the specified commit in the repository.
+    """
+
+
+class FileNotInTargetSubdirError(DvcException):
+    """Thrown when trying to place .gitignore for a file that not in
+    the file subdirectory."""
+
+
 class Base(object):
     """Base class for source control management driver implementations."""
 
     def __init__(self, root_dir=os.curdir, repo=None):
         self.repo = repo
-        self.root_dir = root_dir
+        self.root_dir = os.path.realpath(root_dir)
 
     def __repr__(self):
         return "{class_name}: '{directory}'".format(
@@ -76,47 +87,6 @@ class Base(object):
     def tag(self, tag):
         """Makes SCM create a tag with a specified name."""
 
-    def brancher(
-        self, branches=None, all_branches=False, tags=None, all_tags=False
-    ):
-        """Generator that iterates over specified revisions.
-
-        Args:
-            branches (list): a list of branches to iterate over.
-            all_branches (bool): iterate over all available branches.
-            tags (list): a list of tags to iterate over.
-            all_tags (bool): iterate over all available tags.
-
-        Yields:
-            str: the current revision.
-        """
-        if not branches and not all_branches and not tags and not all_tags:
-            yield ""
-            return
-
-        saved = self.active_branch()
-        revs = []
-
-        if all_branches:
-            branches = self.list_branches()
-
-        if all_tags:
-            tags = self.list_tags()
-
-        if branches is None:
-            revs.extend([saved])
-        else:
-            revs.extend(branches)
-
-        if tags is not None:
-            revs.extend(tags)
-
-        for rev in revs:
-            self.checkout(rev)
-            yield rev
-
-        self.checkout(saved)
-
     def untracked_files(self):  # pylint: disable=no-self-use
         """Returns a list of untracked files."""
         return []
@@ -125,6 +95,10 @@ class Base(object):
         """Returns whether or not a specified path is tracked."""
         return False
 
+    def is_dirty(self):
+        """Return whether the SCM contains uncommited changes."""
+        return False
+
     def active_branch(self):  # pylint: disable=no-self-use
         """Returns current branch in the repo."""
         return ""

dvc/scm/git/__init__.py

@@ -2,17 +2,28 @@
 
 from __future__ import unicode_literals
 
-# NOTE: need this one because python 2 would re-import this module
-# (called git.py) instead of importing gitpython's git module.
-from __future__ import absolute_import
+import os
+import logging
 
 from dvc.utils.compat import str, open
-
-import os
-
-import dvc.logger as logger
 from dvc.utils import fix_env
-from dvc.scm.base import Base, SCMError, FileNotInRepoError
+from dvc.scm.base import (
+    Base,
+    SCMError,
+    FileNotInRepoError,
+    FileNotInTargetSubdirError,
+)
+from dvc.scm.git.tree import GitTree
+
+
+logger = logging.getLogger(__name__)
+
+
+DIFF_A_TREE = "a_tree"
+DIFF_B_TREE = "b_tree"
+DIFF_A_REF = "a_ref"
+DIFF_B_REF = "b_ref"
+DIFF_EQUAL = "equal"
 
 
 class Git(Base):
@@ -62,28 +73,47 @@ class Git(Base):
     def ignore_file(self):
         return self.GITIGNORE
 
-    def _get_gitignore(self, path):
-        assert os.path.isabs(path)
-        # NOTE: using '/' prefix to make path unambiguous
-        entry = "/" + os.path.basename(path)
-        gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE)
+    def _get_gitignore(self, path, ignore_file_dir=None):
+        if not ignore_file_dir:
+            ignore_file_dir = os.path.dirname(os.path.realpath(path))
 
-        if not gitignore.startswith(self.root_dir):
+        assert os.path.isabs(path)
+        assert os.path.isabs(ignore_file_dir)
+
+        if not path.startswith(ignore_file_dir):
+            msg = (
+                "{} file has to be located in one of '{}' subdirectories"
+                ", not outside '{}'"
+            )
+            raise FileNotInTargetSubdirError(
+                msg.format(self.GITIGNORE, path, ignore_file_dir)
+            )
+
+        entry = os.path.relpath(path, ignore_file_dir).replace(os.sep, "/")
+        # NOTE: using '/' prefix to make path unambiguous
+        if len(entry) > 0 and entry[0] != "/":
+            entry = "/" + entry
+
+        gitignore = os.path.join(ignore_file_dir, self.GITIGNORE)
+
+        if not gitignore.startswith(os.path.realpath(self.root_dir)):
             raise FileNotInRepoError(path)
 
         return entry, gitignore
 
-    def ignore(self, path):
-        entry, gitignore = self._get_gitignore(path)
+    def ignore(self, path, in_curr_dir=False):
+        base_dir = (
+            os.path.realpath(os.curdir)
+            if in_curr_dir
+            else os.path.dirname(path)
+        )
+        entry, gitignore = self._get_gitignore(path, base_dir)
 
         ignore_list = []
         if os.path.exists(gitignore):
             with open(gitignore, "r") as f:
                 ignore_list = f.readlines()
-            filtered = list(
-                filter(lambda x: x.strip() == entry.strip(), ignore_list)
-            )
-            if filtered:
+            if any(filter(lambda x: x.strip() == entry.strip(), ignore_list)):
                 return
 
         msg = "Adding '{}' to '{}'.".format(
@@ -91,17 +121,20 @@ class Git(Base):
         )
         logger.info(msg)
 
-        content = entry
-        if ignore_list:
-            content = "\n" + content
-
-        with open(gitignore, "a") as fobj:
-            fobj.write(content)
+        self._add_entry_to_gitignore(entry, gitignore, ignore_list)
 
         self.track_file(os.path.relpath(gitignore))
 
         self.ignored_paths.append(path)
 
+    @staticmethod
+    def _add_entry_to_gitignore(entry, gitignore, ignore_list):
+        content = entry
+        if ignore_list:
+            content = "\n" + content
+        with open(gitignore, "a", encoding="utf-8") as fobj:
+            fobj.write(content)
+
     def ignore_remove(self, path):
         entry, gitignore = self._get_gitignore(path)
 
@@ -131,7 +164,7 @@ class Git(Base):
                 " for more details.".format(str(paths))
             )
 
-            logger.error(msg)
+            logger.exception(msg)
 
     def commit(self, msg):
         self.git.index.commit(msg)
@@ -153,7 +186,13 @@ class Git(Base):
         return [os.path.join(self.git.working_dir, fname) for fname in files]
 
     def is_tracked(self, path):
-        return len(self.git.git.ls_files(path)) != 0
+        # it is equivalent to `bool(self.git.git.ls_files(path))` by
+        # functionality, but ls_files fails on unicode filenames
+        path = os.path.relpath(path, self.root_dir)
+        return path in [i[0] for i in self.git.index.entries]
+
+    def is_dirty(self):
+        return self.git.is_dirty()
 
     def active_branch(self):
         return self.git.active_branch.name
@@ -203,3 +242,51 @@ class Git(Base):
         basename = os.path.basename(path)
         path_parts = os.path.normpath(path).split(os.path.sep)
         return basename == self.ignore_file or Git.GIT_DIR in path_parts
+
+    def get_tree(self, rev):
+        return GitTree(self.git, rev)
+
+    def _get_diff_trees(self, a_ref, b_ref):
+
+        from gitdb.exc import BadObject, BadName
+
+        trees = {DIFF_A_TREE: None, DIFF_B_TREE: None}
+        commits = []
+        try:
+            if b_ref is not None:
+                a_commit = self.git.commit(a_ref)
+                b_commit = self.git.commit(b_ref)
+                commits.append(str(a_commit))
+                commits.append(str(b_commit))
+            else:
+                a_commit = self.git.commit(a_ref)
+                b_commit = self.git.head.commit
+                commits.append(str(a_commit))
+                commits.append(str(b_commit))
+            trees[DIFF_A_TREE] = self.get_tree(commits[0])
+            trees[DIFF_B_TREE] = self.get_tree(commits[1])
+        except (BadName, BadObject) as e:
+            raise SCMError("git problem", cause=e)
+        return trees, commits
+
+    def get_diff_trees(self, a_ref, b_ref=None):
+        """Method for getting two repo trees between two git tag commits
+        returns the dvc hash names of changed file/directory
+
+        Args:
+            a_ref(str) - git reference
+            b_ref(str) - optional second git reference, default None
+
+        Returns:
+            dict - dictionary with keys: (a_tree, b_tree, a_ref, b_ref, equal)
+        """
+        diff_dct = {DIFF_EQUAL: False}
+        trees, commit_refs = self._get_diff_trees(a_ref, b_ref)
+        diff_dct[DIFF_A_REF] = commit_refs[0]
+        diff_dct[DIFF_B_REF] = commit_refs[1]
+        if commit_refs[0] == commit_refs[1]:
+            diff_dct[DIFF_EQUAL] = True
+            return diff_dct
+        diff_dct[DIFF_A_TREE] = trees[DIFF_A_TREE]
+        diff_dct[DIFF_B_TREE] = trees[DIFF_B_TREE]
+        return diff_dct

dvc/scm/git/temp_git_repo.py

@@ -0,0 +1,148 @@
+import os
+import shutil
+import tempfile
+import logging
+
+from dvc.exceptions import DvcException
+
+
+logger = logging.getLogger(__name__)
+
+
+class TempRepoException(DvcException):
+    """Raise when temporary package repository has not properly created"""
+
+    def __init__(self, temp_repo, msg, cause=None):
+        m = "temp repository '{}' error: {}".format(temp_repo.addr, msg)
+        super(TempRepoException, self).__init__(m, cause)
+
+
+class TempGitRepo(object):
+    GIT_DIR_TO_REMOVE = ".git"
+    GIT_FILES_TO_REMOVE = [".gitignore", ".gitmodules"]
+    SUFFIX = "tmp_DVC_mod"
+
+    def __init__(self, addr, module_name, modules_dir):
+        self.addr = addr
+        self.modules_dir = modules_dir
+        self._tmp_mod_prefix = module_name + "_" + self.SUFFIX + "_"
+        self._reset_state()
+
+    def _reset_state(self):
+        self._set_state(None, [])
+
+    def _set_state(self, cloned_tmp_dir, outs):
+        from dvc.repo import Repo
+
+        self.repo = Repo(cloned_tmp_dir) if cloned_tmp_dir else None
+
+        self._cloned_tmp_dir = cloned_tmp_dir
+        self.outs = outs
+        self._moved_files = []
+
+    def fetch(self, targets):
+        for target in targets:
+            try:
+                self.repo.fetch(target)
+            except Exception as ex:
+                msg = "error in fetching data from {}: {}".format(
+                    os.path.basename(target), ex
+                )
+                raise TempRepoException(self, msg)
+
+    @property
+    def is_state_set(self):
+        return self._cloned_tmp_dir is not None
+
+    def __enter__(self):
+        if self.is_state_set:
+            raise TempRepoException(self, "Git repo cloning duplication")
+
+        module_temp_dir = None
+        try:
+            module_temp_dir = self._clone_to_temp_dir()
+            self._remove_git_files_and_dirs(module_temp_dir)
+
+            outputs = self._read_outputs(module_temp_dir)
+            self._set_state(module_temp_dir, outputs)
+        finally:
+            self._clean_mod_temp_dir(module_temp_dir)
+
+        return self
+
+    def _clone_to_temp_dir(self):
+        import git
+
+        result = tempfile.mktemp(
+            prefix=self._tmp_mod_prefix, dir=self.modules_dir
+        )
+        logger.debug(
+            "Cloning git repository {} to temp dir {}".format(
+                self.addr, result
+            )
+        )
+        git.Repo.clone_from(self.addr, result, depth=1)
+        return result
+
+    def _remove_git_files_and_dirs(self, module_temp_dir):
+        logger.debug(
+            "Removing git meta files from {}: {} dif and {}".format(
+                module_temp_dir,
+                TempGitRepo.GIT_DIR_TO_REMOVE,
+                ", ".join(TempGitRepo.GIT_FILES_TO_REMOVE),
+            )
+        )
+        shutil.rmtree(
+            os.path.join(module_temp_dir, TempGitRepo.GIT_DIR_TO_REMOVE)
+        )
+        for item in TempGitRepo.GIT_FILES_TO_REMOVE:
+            fname = os.path.join(module_temp_dir, item)
+            if os.path.exists(fname):
+                os.remove(fname)
+
+    def _clean_mod_temp_dir(self, module_temp_dir):
+        if not self.is_state_set:
+            if module_temp_dir and os.path.exists(module_temp_dir):
+                shutil.rmtree(module_temp_dir, ignore_errors=True)
+
+    def _read_outputs(self, module_temp_dir):
+        from dvc.repo import Repo
+
+        pkg_repo = Repo(module_temp_dir)
+        stages = pkg_repo.stages()
+        return [out for s in stages for out in s.outs]
+
+    def persist_to(self, module_dir, parent_repo):
+        if not self.is_state_set:
+            raise TempRepoException(self, "cannot persist")
+
+        tmp_repo_cache = self.repo.cache.local.url
+
+        for prefix in os.listdir(tmp_repo_cache):
+            if len(prefix) != 2:
+                logger.warning(
+                    "wrong dir format in cache {}: dir {}".format(
+                        tmp_repo_cache, prefix
+                    )
+                )
+            self._move_all_cache_files(parent_repo, prefix, tmp_repo_cache)
+
+        shutil.move(self._cloned_tmp_dir, module_dir)
+        self._reset_state()
+
+    @staticmethod
+    def _move_all_cache_files(parent_repo, prefix, tmp_repo_cache):
+        obj_name = os.path.join(tmp_repo_cache, prefix)
+        for suffix in os.listdir(obj_name):
+            src_path = os.path.join(tmp_repo_cache, prefix, suffix)
+            pre = os.path.join(parent_repo.cache.local.url, prefix)
+            if not os.path.exists(pre):
+                os.mkdir(pre)
+            dest = os.path.join(pre, suffix)
+            shutil.move(src_path, dest)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.is_state_set:
+            shutil.rmtree(self._cloned_tmp_dir)
+            self._reset_state()
+        pass

dvc/scm/git/tree.py

@@ -0,0 +1,114 @@
+import errno
+import os
+
+from dvc.utils.compat import StringIO
+
+from dvc.scm.tree import BaseTree
+
+
+# see git-fast-import(1)
+GIT_MODE_DIR = 0o40000
+GIT_MODE_FILE = 0o644
+
+
+class GitTree(BaseTree):
+    """Proxies the repo file access methods to Git objects"""
+
+    def __init__(self, git, rev):
+        """Create GitTree instance
+
+        Args:
+            git (dvc.scm.Git):
+            branch:
+        """
+        self.git = git
+        self.rev = rev
+
+    def open(self, path):
+
+        relpath = os.path.relpath(path, self.git.working_dir)
+
+        obj = self.git_object_by_path(path)
+        if obj is None:
+            msg = "No such file in branch '{}'".format(self.rev)
+            raise IOError(errno.ENOENT, msg, relpath)
+        if obj.mode == GIT_MODE_DIR:
+            raise IOError(errno.EISDIR, "Is a directory", relpath)
+
+        # GitPython's obj.data_stream is a fragile thing, it is better to
+        # read it immediately, also it needs to be to decoded if we follow
+        # the `open()` behavior (since data_stream.read() returns bytes,
+        # and `open` with default "r" mode returns str)
+        return StringIO(obj.data_stream.read().decode("utf-8"))
+
+    def exists(self, path):
+        return self.git_object_by_path(path) is not None
+
+    def isdir(self, path):
+        obj = self.git_object_by_path(path)
+        if obj is None:
+            return False
+        return obj.mode == GIT_MODE_DIR
+
+    def isfile(self, path):
+        obj = self.git_object_by_path(path)
+        if obj is None:
+            return False
+        # according to git-fast-import(1) file mode could be 644 or 755
+        return obj.mode & GIT_MODE_FILE == GIT_MODE_FILE
+
+    @staticmethod
+    def _is_tree_and_contains(obj, path):
+        if obj.mode != GIT_MODE_DIR:
+            return False
+        # see https://github.com/gitpython-developers/GitPython/issues/851
+        # `return (i in tree)` doesn't work so here is a workaround:
+        for i in obj:
+            if i.name == path:
+                return True
+        return False
+
+    def git_object_by_path(self, path):
+        path = os.path.relpath(os.path.realpath(path), self.git.working_dir)
+        assert path.split(os.sep, 1)[0] != ".."
+        tree = self.git.tree(self.rev)
+        if not path or path == ".":
+            return tree
+        for i in path.split(os.sep):
+            if not self._is_tree_and_contains(tree, i):
+                # there is no tree for specified path
+                return None
+            tree = tree[i]
+        return tree
+
+    def _walk(self, tree, topdown=True):
+
+        dirs, nondirs = [], []
+        for i in tree:
+            if i.mode == GIT_MODE_DIR:
+                dirs.append(i.name)
+            else:
+                nondirs.append(i.name)
+
+        if topdown:
+            yield os.path.normpath(tree.path), dirs, nondirs
+        for i in dirs:
+            for x in self._walk(tree[i], topdown=True):
+                yield x
+        if not topdown:
+            yield os.path.normpath(tree.path), dirs, nondirs
+
+    def walk(self, top, topdown=True):
+        """Directory tree generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        - it could raise exceptions, there is no onerror argument
+        """
+
+        tree = self.git_object_by_path(top)
+        if tree is None:
+            raise IOError(errno.ENOENT, "No such file")
+
+        for x in self._walk(tree, topdown):
+            yield x

dvc/scm/tree.py

@@ -0,0 +1,65 @@
+import os
+
+from dvc.utils.compat import open
+
+
+class BaseTree(object):
+    """Abstract class to represent access to files"""
+
+    def open(self, path):
+        """Open file and return a stream."""
+
+    def exists(self, path):
+        """Test whether a path exists."""
+
+    def isdir(self, path):
+        """Return true if the pathname refers to an existing directory."""
+
+    def isfile(self, path):
+        """Test whether a path is a regular file"""
+
+    def walk(self, top, topdown=True):
+        """Directory tree generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        - it could raise exceptions, there is no onerror argument
+        """
+
+
+class WorkingTree(BaseTree):
+    """Proxies the repo file access methods to working tree files"""
+
+    def open(self, path):
+        """Open file and return a stream."""
+        return open(path, encoding="utf-8")
+
+    def exists(self, path):
+        """Test whether a path exists."""
+        return os.path.exists(path)
+
+    def isdir(self, path):
+        """Return true if the pathname refers to an existing directory."""
+        return os.path.isdir(path)
+
+    def isfile(self, path):
+        """Test whether a path is a regular file"""
+        return os.path.isfile(path)
+
+    def walk(self, top, topdown=True):
+        """Directory tree generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        - it could raise exceptions, there is no onerror argument
+        """
+
+        def onerror(e):
+            raise e
+
+        for root, dirs, files in os.walk(
+            top, topdown=topdown, onerror=onerror
+        ):
+            if topdown:
+                dirs[:] = [i for i in dirs if i not in (".git", ".hg", ".dvc")]
+            yield os.path.normpath(root), dirs, files

dvc/stage.py

@@ -2,19 +2,23 @@ from __future__ import unicode_literals
 
 from dvc.utils.compat import str, open
 
+import re
 import os
 import yaml
 import subprocess
+import logging
 
 from dvc.utils.fs import contains_symlink_up_to
 from schema import Schema, SchemaError, Optional, Or, And
 
 import dvc.prompt as prompt
-import dvc.logger as logger
 import dvc.dependency as dependency
 import dvc.output as output
 from dvc.exceptions import DvcException
-from dvc.utils import dict_md5, fix_env, load_stage_file
+from dvc.utils import dict_md5, fix_env, load_stage_file_fobj
+
+
+logger = logging.getLogger(__name__)
 
 
 class StageCmdFailedError(DvcException):
@@ -129,6 +133,8 @@ class Stage(object):
         Optional(PARAM_LOCKED): bool,
     }
 
+    TAG_REGEX = r"^(?P<path>.*)@(?P<tag>[^\\/@:]*)$"
+
     def __init__(
         self,
         repo,
@@ -139,6 +145,7 @@ class Stage(object):
         outs=None,
         md5=None,
         locked=False,
+        tag=None,
     ):
         if deps is None:
             deps = []
@@ -153,6 +160,7 @@ class Stage(object):
         self.deps = deps
         self.md5 = md5
         self.locked = locked
+        self.tag = tag
 
     def __repr__(self):
         return "Stage: '{path}'".format(
@@ -171,7 +179,10 @@ class Stage(object):
     @staticmethod
     def is_valid_filename(path):
         return (
-            path.endswith(Stage.STAGE_FILE_SUFFIX)
+            # path.endswith doesn't work for encoded unicode filenames on
+            # Python 2 and since Stage.STAGE_FILE_SUFFIX is ascii then it is
+            # not needed to decode the path from py2's str
+            path[-len(Stage.STAGE_FILE_SUFFIX) :] == Stage.STAGE_FILE_SUFFIX
             or os.path.basename(path) == Stage.STAGE_FILE
         )
 
@@ -208,10 +219,12 @@ class Stage(object):
             return True
 
         for dep in self.deps:
-            if dep.changed():
+            status = dep.status()
+            if status:
                 logger.warning(
-                    "Dependency '{dep}' of '{stage}' changed.".format(
-                        dep=dep, stage=self.relpath
+                    "Dependency '{dep}' of '{stage}' changed because it is "
+                    "'{status}'.".format(
+                        dep=dep, stage=self.relpath, status=status[str(dep)]
                     )
                 )
                 return True
@@ -220,10 +233,12 @@ class Stage(object):
 
     def _changed_outs(self):
         for out in self.outs:
-            if out.changed():
+            status = out.status()
+            if status:
                 logger.warning(
-                    "Output '{out}' of '{stage}' changed.".format(
-                        out=out, stage=self.relpath
+                    "Output '{out}' of '{stage}' changed because it is "
+                    "'{status}'".format(
+                        out=out, stage=self.relpath, status=status[str(out)]
                     )
                 )
                 return True
@@ -242,31 +257,31 @@ class Stage(object):
         )
 
         if ret:
-            msg = "Stage '{}' changed.".format(self.relpath)
-            color = "yellow"
+            logger.warning("Stage '{}' changed.".format(self.relpath))
         else:
-            msg = "Stage '{}' didn't change.".format(self.relpath)
-            color = "green"
-
-        logger.info(logger.colorize(msg, color))
+            logger.info("Stage '{}' didn't change.".format(self.relpath))
 
         return ret
 
-    def remove_outs(self, ignore_remove=False):
-        """
-        Used mainly for `dvc remove --outs`
-        """
+    def remove_outs(self, ignore_remove=False, force=False):
+        """Used mainly for `dvc remove --outs` and :func:`Stage.reproduce`."""
         for out in self.outs:
-            out.remove(ignore_remove=ignore_remove)
+            if out.persist and not force:
+                out.unprotect()
+            else:
+                logger.debug(
+                    "Removing output '{out}' of '{stage}'.".format(
+                        out=out, stage=self.relpath
+                    )
+                )
+                out.remove(ignore_remove=ignore_remove)
 
     def unprotect_outs(self):
         for out in self.outs:
-            if out.scheme != "local" or not out.exists:
-                continue
-            self.repo.unprotect(out.path)
+            out.unprotect()
 
-    def remove(self):
-        self.remove_outs(ignore_remove=True)
+    def remove(self, force=False):
+        self.remove_outs(ignore_remove=True, force=force)
         os.unlink(self.path)
 
     def reproduce(
@@ -275,10 +290,6 @@ class Stage(object):
         if not self.changed() and not force:
             return None
 
-        if (self.cmd or self.is_import) and not self.locked and not dry:
-            # Removing outputs only if we actually have command to reproduce
-            self.remove_outs(ignore_remove=False)
-
         msg = (
             "Going to reproduce '{stage}'. "
             "Are you sure you want to continue?".format(stage=self.relpath)
@@ -327,7 +338,7 @@ class Stage(object):
     def _expand_to_path_on_add_local(add, fname, out, path_handler):
         if (
             add
-            and out.is_local
+            and out.is_in_repo
             and not contains_symlink_up_to(out.path, out.repo.root_dir)
         ):
             fname = path_handler.join(path_handler.dirname(out.path), fname)
@@ -396,6 +407,9 @@ class Stage(object):
         overwrite=True,
         ignore_build_cache=False,
         remove_outs=False,
+        validate_state=True,
+        outs_persist=None,
+        outs_persist_no_cache=None,
     ):
         if outs is None:
             outs = []
@@ -407,6 +421,10 @@ class Stage(object):
             metrics = []
         if metrics_no_cache is None:
             metrics_no_cache = []
+        if outs_persist is None:
+            outs_persist = []
+        if outs_persist_no_cache is None:
+            outs_persist_no_cache = []
 
         # Backward compatibility for `cwd` option
         if wdir is None and cwd is not None:
@@ -423,13 +441,14 @@ class Stage(object):
 
         stage = Stage(repo=repo, wdir=wdir, cmd=cmd, locked=locked)
 
-        stage.outs = output.loads_from(stage, outs, use_cache=True)
-        stage.outs += output.loads_from(
-            stage, metrics, use_cache=True, metric=True
-        )
-        stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False)
-        stage.outs += output.loads_from(
-            stage, metrics_no_cache, use_cache=False, metric=True
+        Stage._fill_stage_outputs(
+            stage,
+            outs,
+            outs_no_cache,
+            metrics,
+            metrics_no_cache,
+            outs_persist,
+            outs_persist_no_cache,
         )
         stage.deps = dependency.loads_from(stage, deps)
 
@@ -458,23 +477,49 @@ class Stage(object):
         else:
             stage.unprotect_outs()
 
-        if os.path.exists(path):
-            if not ignore_build_cache and stage.is_cached:
-                logger.info("Stage is cached, skipping.")
-                return None
+        if validate_state:
+            if os.path.exists(path):
+                if not ignore_build_cache and stage.is_cached:
+                    logger.info("Stage is cached, skipping.")
+                    return None
 
-            msg = (
-                "'{}' already exists. Do you wish to run the command and "
-                "overwrite it?".format(stage.relpath)
-            )
+                msg = (
+                    "'{}' already exists. Do you wish to run the command and "
+                    "overwrite it?".format(stage.relpath)
+                )
 
-            if not overwrite and not prompt.confirm(msg):
-                raise StageFileAlreadyExistsError(stage.relpath)
+                if not overwrite and not prompt.confirm(msg):
+                    raise StageFileAlreadyExistsError(stage.relpath)
 
-            os.unlink(path)
+                os.unlink(path)
 
         return stage
 
+    @staticmethod
+    def _fill_stage_outputs(
+        stage,
+        outs,
+        outs_no_cache,
+        metrics,
+        metrics_no_cache,
+        outs_persist,
+        outs_persist_no_cache,
+    ):
+        stage.outs = output.loads_from(stage, outs, use_cache=True)
+        stage.outs += output.loads_from(
+            stage, metrics, use_cache=True, metric=True
+        )
+        stage.outs += output.loads_from(
+            stage, outs_persist, use_cache=True, persist=True
+        )
+        stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False)
+        stage.outs += output.loads_from(
+            stage, metrics_no_cache, use_cache=False, metric=True
+        )
+        stage.outs += output.loads_from(
+            stage, outs_persist_no_cache, use_cache=False, persist=True
+        )
+
     @staticmethod
     def _check_dvc_filename(fname):
         if not Stage.is_valid_filename(fname):
@@ -486,19 +531,36 @@ class Stage(object):
             )
 
     @staticmethod
-    def _check_file_exists(fname):
-        if not os.path.exists(fname):
+    def _check_file_exists(repo, fname):
+        if not repo.tree.exists(fname):
             raise StageFileDoesNotExistError(fname)
 
     @staticmethod
-    def load(repo, fname):
-        Stage._check_file_exists(fname)
-        Stage._check_dvc_filename(fname)
-
-        if not Stage.is_stage_file(fname):
+    def _check_isfile(repo, fname):
+        if not repo.tree.isfile(fname):
             raise StageFileIsNotDvcFileError(fname)
 
-        d = load_stage_file(fname)
+    @classmethod
+    def _get_path_tag(cls, s):
+        regex = re.compile(cls.TAG_REGEX)
+        match = regex.match(s)
+        if not match:
+            return s, None
+        return match.group("path"), match.group("tag")
+
+    @staticmethod
+    def load(repo, fname):
+        fname, tag = Stage._get_path_tag(fname)
+
+        # it raises the proper exceptions by priority:
+        # 1. when the file doesn't exists
+        # 2. filename is not a dvc filename
+        # 3. path doesn't represent a regular file
+        Stage._check_file_exists(repo, fname)
+        Stage._check_dvc_filename(fname)
+        Stage._check_isfile(repo, fname)
+
+        d = load_stage_file_fobj(repo.tree.open(fname), fname)
 
         Stage.validate(d, fname=os.path.relpath(fname))
         path = os.path.abspath(fname)
@@ -514,6 +576,7 @@ class Stage(object):
             cmd=d.get(Stage.PARAM_CMD),
             md5=d.get(Stage.PARAM_MD5),
             locked=d.get(Stage.PARAM_LOCKED, False),
+            tag=tag,
         )
 
         stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
@@ -557,7 +620,7 @@ class Stage(object):
         self.repo.scm.track_file(os.path.relpath(fname))
 
     def _compute_md5(self):
-        from dvc.output.local import OutputLOCAL
+        from dvc.output.base import OutputBase
 
         d = self.dumpd()
 
@@ -575,7 +638,15 @@ class Stage(object):
         # NOTE: excluding parameters that don't affect the state of the
         # pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if
         # it has changed, we might not have that output in our cache.
-        m = dict_md5(d, exclude=[self.PARAM_LOCKED, OutputLOCAL.PARAM_METRIC])
+        m = dict_md5(
+            d,
+            exclude=[
+                self.PARAM_LOCKED,
+                OutputBase.PARAM_METRIC,
+                OutputBase.PARAM_TAGS,
+                OutputBase.PARAM_PERSIST,
+            ],
+        )
         logger.debug("Computed stage '{}' md5: '{}'".format(self.relpath, m))
         return m
 
@@ -681,6 +752,9 @@ class Stage(object):
             raise StageCmdFailedError(self)
 
     def run(self, dry=False, resume=False, no_commit=False, force=False):
+        if (self.cmd or self.is_import) and not self.locked and not dry:
+            self.remove_outs(ignore_remove=False, force=False)
+
         if self.locked:
             logger.info(
                 "Verifying outputs in locked stage '{stage}'".format(
@@ -737,9 +811,11 @@ class Stage(object):
         if paths:
             raise MissingDataSource(paths)
 
-    def checkout(self, force=False):
+    def checkout(self, force=False, progress_callback=None):
         for out in self.outs:
-            out.checkout(force=force)
+            out.checkout(
+                force=force, tag=self.tag, progress_callback=progress_callback
+            )
 
     @staticmethod
     def _status(entries):
@@ -782,3 +858,6 @@ class Stage(object):
                 for out in self.outs
             )
         )
+
+    def get_all_files_number(self):
+        return sum(out.get_files_number() for out in self.outs)

dvc/state.py

@@ -4,14 +4,17 @@ from __future__ import unicode_literals
 
 import os
 import sqlite3
+import logging
 
-import dvc.logger as logger
 from dvc.config import Config
 from dvc.utils import file_md5, remove, current_timestamp
 from dvc.exceptions import DvcException
 from dvc.utils.fs import get_mtime_and_size, get_inode
 
 
+logger = logging.getLogger(__name__)
+
+
 class StateVersionTooNewError(DvcException):
     """Thrown when dvc version is older than the state database version."""
 
@@ -164,7 +167,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         return num
 
     def _prepare_db(self, empty=False):
-        from dvc import VERSION
+        from dvc import __version__
 
         if not empty:
             cmd = "PRAGMA user_version;"
@@ -176,7 +179,9 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             version = ret[0][0]
 
             if version > self.VERSION:
-                raise StateVersionTooNewError(VERSION, self.VERSION, version)
+                raise StateVersionTooNewError(
+                    __version__, self.VERSION, version
+                )
             elif version < self.VERSION:
                 msg = (
                     "State file version '{}' is too old. "

dvc/system.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 from dvc.utils.compat import str, open
 
 import os
+import errno
 
 
 class System(object):
@@ -61,8 +62,22 @@ class System(object):
     @staticmethod
     def _reflink_darwin(src, dst):
         import ctypes
+        import dvc.logger as logger
+
+        LIBC = "libc.dylib"
+        LIBC_FALLBACK = "/usr/lib/libSystem.dylib"
+        try:
+            clib = ctypes.CDLL(LIBC)
+        except OSError as exc:
+            logger.debug(
+                "unable to access '{}' (errno '{}'). "
+                "Falling back to '{}'.".format(LIBC, exc.errno, LIBC_FALLBACK)
+            )
+            if exc.errno != errno.ENOENT:
+                raise
+            # NOTE: trying to bypass System Integrity Protection (SIP)
+            clib = ctypes.CDLL(LIBC_FALLBACK)
 
-        clib = ctypes.CDLL("libc.dylib")
         if not hasattr(clib, "clonefile"):
             return -1
 

dvc/updater.py

@@ -3,13 +3,17 @@ from __future__ import unicode_literals
 import sys
 import os
 import time
+import logging
 import requests
 import colorama
+from pkg_resources import parse_version
 
-import dvc.logger as logger
-from dvc import VERSION_BASE
+from dvc import __version__
 from dvc.lock import Lock, LockError
-from dvc.utils import is_binary
+from dvc.utils import is_binary, boxify
+
+
+logger = logging.getLogger(__name__)
 
 
 class Updater(object):  # pragma: no cover
@@ -22,7 +26,7 @@ class Updater(object):  # pragma: no cover
         self.dvc_dir = dvc_dir
         self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
         self.lock = Lock(dvc_dir, self.updater_file + ".lock")
-        self.current = VERSION_BASE
+        self.current = parse_version(__version__).base_version
 
     def _is_outdated_file(self):
         ctime = os.path.getmtime(self.updater_file)
@@ -89,16 +93,7 @@ class Updater(object):  # pragma: no cover
             json.dump(info, fobj)
 
     def _is_outdated(self):
-        l_major, l_minor, l_patch = [int(x) for x in self.latest.split(".")]
-        c_major, c_minor, c_patch = [int(x) for x in self.current.split(".")]
-
-        if l_major != c_major:
-            return l_major > c_major
-
-        if l_minor != c_minor:
-            return l_minor > c_minor
-
-        return l_patch > c_patch
+        return parse_version(self.current) < parse_version(self.latest)
 
     def _notify(self):
         if not sys.stdout.isatty():
@@ -118,7 +113,7 @@ class Updater(object):  # pragma: no cover
             latest=self.latest,
         )
 
-        logger.box(message, border_color="yellow")
+        logger.info(boxify(message, border_color="yellow"))
 
     def _get_update_instructions(self):
         instructions = {

dvc/utils/__init__.py

@@ -14,9 +14,15 @@ import shutil
 import hashlib
 import nanotime
 import time
+import colorama
+import re
+import logging
 
 from yaml.scanner import ScannerError
 
+
+logger = logging.getLogger(__name__)
+
 LOCAL_CHUNK_SIZE = 1024 * 1024
 LARGE_FILE_SIZE = 1024 * 1024 * 1024
 LARGE_DIR_SIZE = 100
@@ -28,7 +34,6 @@ def dos2unix(data):
 
 def file_md5(fname):
     """ get the (md5 hexdigest, md5 digest) of a file """
-    import dvc.logger as logger
     from dvc.progress import progress
     from dvc.istextfile import istextfile
 
@@ -152,8 +157,6 @@ def move(src, dst):
 
 
 def remove(path):
-    import dvc.logger as logger
-
     if not os.path.exists(path):
         return
 
@@ -232,16 +235,100 @@ def current_timestamp():
 
 
 def load_stage_file(path):
+    with open(path, "r") as fobj:
+        return load_stage_file_fobj(fobj, path)
+
+
+def load_stage_file_fobj(fobj, path):
     from dvc.exceptions import StageFileCorruptedError
 
-    with open(path, "r") as fobj:
-        try:
-            return yaml.safe_load(fobj) or {}
-        except ScannerError:
-            raise StageFileCorruptedError(path)
+    try:
+        return yaml.safe_load(fobj) or {}
+    except ScannerError:
+        raise StageFileCorruptedError(path)
 
 
 def walk_files(directory):
-    for root, _, files in os.walk(directory):
+    for root, _, files in os.walk(str(directory)):
         for f in files:
             yield os.path.join(root, f)
+
+
+def colorize(message, color=None):
+    """Returns a message in a specified color."""
+    if not color:
+        return message
+
+    colors = {
+        "green": colorama.Fore.GREEN,
+        "yellow": colorama.Fore.YELLOW,
+        "blue": colorama.Fore.BLUE,
+        "red": colorama.Fore.RED,
+    }
+
+    return "{color}{message}{nc}".format(
+        color=colors.get(color, ""), message=message, nc=colorama.Fore.RESET
+    )
+
+
+def boxify(message, border_color=None):
+    """Put a message inside a box.
+
+    Args:
+        message (unicode): message to decorate.
+        border_color (unicode): name of the color to outline the box with.
+    """
+    lines = message.split("\n")
+    max_width = max(_visual_width(line) for line in lines)
+
+    padding_horizontal = 5
+    padding_vertical = 1
+
+    box_size_horizontal = max_width + (padding_horizontal * 2)
+
+    chars = {"corner": "+", "horizontal": "-", "vertical": "|", "empty": " "}
+
+    margin = "{corner}{line}{corner}\n".format(
+        corner=chars["corner"], line=chars["horizontal"] * box_size_horizontal
+    )
+
+    padding_lines = [
+        "{border}{space}{border}\n".format(
+            border=colorize(chars["vertical"], color=border_color),
+            space=chars["empty"] * box_size_horizontal,
+        )
+        * padding_vertical
+    ]
+
+    content_lines = [
+        "{border}{space}{content}{space}{border}\n".format(
+            border=colorize(chars["vertical"], color=border_color),
+            space=chars["empty"] * padding_horizontal,
+            content=_visual_center(line, max_width),
+        )
+        for line in lines
+    ]
+
+    box_str = "{margin}{padding}{content}{padding}{margin}".format(
+        margin=colorize(margin, color=border_color),
+        padding="".join(padding_lines),
+        content="".join(content_lines),
+    )
+
+    return box_str
+
+
+def _visual_width(line):
+    """Get the the number of columns required to display a string"""
+
+    return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, "", line))
+
+
+def _visual_center(line, width):
+    """Center align string according to it's visual width"""
+
+    spaces = max(width - _visual_width(line), 0)
+    left_padding = int(spaces / 2)
+    right_padding = spaces - left_padding
+
+    return (left_padding * " ") + line + (right_padding * " ")

dvc/utils/collections.py

@@ -0,0 +1,6 @@
+from __future__ import unicode_literals
+
+
+# just simple check for Nones and emtpy strings
+def compact(args):
+    return list(filter(bool, args))

dvc/utils/compat.py

@@ -27,18 +27,48 @@ def encode(u, encoding=None):
     return u.encode(encoding, "replace")
 
 
-# NOTE: cast_bytes_py2 is taken from
-# https://github.com/ipython/ipython_genutils
+def csv_reader(unicode_csv_data, dialect=None, **kwargs):
+    """csv.reader doesn't support Unicode input, so need to use some tricks
+    to work around this.
+
+    Source: https://docs.python.org/2/library/csv.html#csv-examples
+    """
+    import csv
+
+    dialect = dialect or csv.excel
+
+    if is_py3:
+        # Python3 supports encoding by default, so just return the object
+        for row in csv.reader(unicode_csv_data, dialect=dialect, **kwargs):
+            yield [cell for cell in row]
+
+    else:
+        # csv.py doesn't do Unicode; encode temporarily as UTF-8:
+        reader = csv.reader(
+            utf_8_encoder(unicode_csv_data), dialect=dialect, **kwargs
+        )
+        for row in reader:
+            # decode UTF-8 back to Unicode, cell by cell:
+            yield [unicode(cell, "utf-8") for cell in row]  # noqa: F821
+
+
+def utf_8_encoder(unicode_csv_data):
+    """Source: https://docs.python.org/2/library/csv.html#csv-examples"""
+    for line in unicode_csv_data:
+        yield line.encode("utf-8")
+
+
 def cast_bytes(s, encoding=None):
+    """Source: https://github.com/ipython/ipython_genutils"""
     if not isinstance(s, bytes):
         return encode(s, encoding)
     return s
 
 
-# NOTE _makedirs is taken from
-# https://github.com/python/cpython/blob/
-# 3ce3dea60646d8a5a1c952469a2eb65f937875b3/Lib/os.py#L196-L226
 def _makedirs(name, mode=0o777, exist_ok=False):
+    """Source: https://github.com/python/cpython/blob/
+        3ce3dea60646d8a5a1c952469a2eb65f937875b3/Lib/os.py#L196-L226
+    """
     head, tail = os.path.split(name)
     if not tail:
         head, tail = os.path.split(head)

dvc/utils/fs.py

@@ -2,13 +2,16 @@ from __future__ import unicode_literals
 
 import nanotime
 import os
+import logging
 
-import dvc.logger as logger
 from dvc.exceptions import DvcException
 from dvc.system import System
 from dvc.utils.compat import str
 
 
+logger = logging.getLogger(__name__)
+
+
 def get_inode(path):
     inode = System.inode(path)
     logger.debug("Path {} inode {}".format(path, inode))

dvc/version.py

@@ -0,0 +1,73 @@
+# Used in setup.py, so don't pull any additional dependencies
+#
+# Based on:
+#   - https://github.com/python/mypy/blob/master/mypy/version.py
+#   - https://github.com/python/mypy/blob/master/mypy/git.py
+import os
+import subprocess
+
+
+_BASE_VERSION = "0.35.3"
+
+
+def _generate_version(base_version):
+    """Generate a version with information about the git repository"""
+    pkg_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+
+    if not _is_git_repo(pkg_dir) or not _have_git():
+        return base_version
+
+    if _is_release(pkg_dir, base_version):
+        return base_version
+
+    return "{base_version}+{short_sha}{dirty}".format(
+        base_version=base_version,
+        short_sha=_git_revision(pkg_dir).decode("utf-8")[0:6],
+        dirty=".mod" if _is_dirty(pkg_dir) else "",
+    )
+
+
+def _is_git_repo(dir_path):
+    """Is the given directory version-controlled with git?"""
+    return os.path.exists(os.path.join(dir_path, ".git"))
+
+
+def _have_git():
+    """Can we run the git executable?"""
+    try:
+        subprocess.check_output(["git", "--help"])
+        return True
+    except subprocess.CalledProcessError:
+        return False
+    except OSError:
+        return False
+
+
+def _is_release(dir_path, base_version):
+    try:
+        output = subprocess.check_output(
+            ["git", "describe", "--tags", "--exact-match"], cwd=dir_path
+        ).decode("utf-8")
+        tag = output.strip()
+        return tag == base_version
+    except subprocess.CalledProcessError:
+        return False
+
+
+def _git_revision(dir_path):
+    """Get the SHA-1 of the HEAD of a git repository."""
+    return subprocess.check_output(
+        ["git", "rev-parse", "HEAD"], cwd=dir_path
+    ).strip()
+
+
+def _is_dirty(dir_path):
+    """Check whether a git repository has uncommitted changes."""
+    try:
+        subprocess.check_call(["git", "diff", "--quiet"], cwd=dir_path)
+        return False
+    except subprocess.CalledProcessError:
+        return True
+
+
+__version__ = _generate_version(_BASE_VERSION)

requirements.txt

@@ -1,5 +1,5 @@
 azure-storage-blob==1.3.0
-boto3==1.9.86
+boto3==1.9.115
 ply>=3.9 # See https://github.com/pyinstaller/pyinstaller/issues/1945
 configparser>=3.5.0
 zc.lockfile>=1.2.1
@@ -25,3 +25,6 @@ pydot>=1.2.4
 asciimatics>=1.10.0
 distro>=1.3.0
 appdirs>=1.4.3
+treelib>=1.5.5
+inflect>=2.1.0
+humanize>=0.5.1

scripts/build_posix.sh

@@ -42,7 +42,7 @@ command_exists()
 fpm_build()
 {
 	print_info "Building $1..."
-	VERSION=$(python -c "import dvc; from dvc import VERSION; print(str(VERSION))")
+	VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))")
 	fpm -s dir \
 	    -f \
 	    -t $1 \

scripts/ci/deploy_condition.sh

@@ -9,7 +9,7 @@ if [[ $python_ver != "2" ]]; then
     exit 0
 fi
 
-if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_OSX_IMAGE != "xcode10.1" ]]; then
+if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_OSX_IMAGE != "xcode8.3" ]]; then
     echo false
     exit 0
 fi

scripts/ci/script.sh

@@ -8,33 +8,42 @@ python -mtests
 if [[ "$TRAVIS_PULL_REQUEST" == "false" && \
       "$TRAVIS_SECURE_ENV_VARS" == "true" && \
       "$(python -c 'import sys; print(sys.version_info[0])')" == '2' ]]; then
+	pip install codecov
 	codecov
 fi
 
-if [[ -n "$TRAVIS_TAG" || "$TRAVIS_EVENT_TYPE" == "cron" ]]; then
-	./scripts/build_package.sh
-
-        if [[ -n "$TRAVIS_TAG" ]]; then
-            # Test version
-            pip uninstall -y dvc
-            if [ -x "$(command -v dvc)" ]; then
-                echo "ERROR: dvc command already exists!!!"
-                exit 1
-            fi
-
-            pip install dist/dvc-*.whl
-            if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
-                echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'"
-                exit 1
-            fi
-            pip uninstall -y dvc
-        fi
-
-        PY_VER=$(python -c 'import sys; print(sys.version_info[0:2])')
-        if [[ "$PY_VER" != '(3, 7)' ]]; then
-            ./scripts/build_posix.sh
-        else
-            echo "Not building dvc binary package on Python 3.7, since it is "\
-                 "not supported by PyInstaller yet."
-        fi
+if [[ -z "$TRAVIS_TAG" && "$TRAVIS_EVENT_TYPE" != "cron" ]]; then
+    echo "Skipping building package."
+    exit 0
 fi
+
+./scripts/build_package.sh
+
+if [[ -n "$TRAVIS_TAG" ]]; then
+    # Test version
+    pip uninstall -y dvc
+    if [ -x "$(command -v dvc)" ]; then
+        echo "ERROR: dvc command already exists!!!"
+        exit 1
+    fi
+
+    pip install dist/dvc-*.whl
+    if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
+        echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'"
+        exit 1
+    fi
+    pip uninstall -y dvc
+fi
+
+PY_VER=$(python -c 'import sys; print(sys.version_info[0:2])')
+if [[ "$PY_VER" != '(2, 7)' ]]; then
+    echo "Skipping building binary package because of a wrong python version."
+    exit 0
+fi
+
+if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_OSX_IMAGE" != "xcode7.3" ]]; then
+    echo "Skipping building binary package because of a wrong xcode version."
+    exit 0
+fi
+
+./scripts/build_posix.sh