dvc: bump to 0.19.8

hdfs: don't use close_fds on Windows

.coveragerc

@@ -8,3 +8,4 @@ exclude_lines =
     pass
     raise NotImplementedError
     # pragma: no cover
+    #  pragma: no cover

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,10 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+
+**Please provide information about your setup**
+DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,6 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+cache: pip
 sudo: required
 dist: trusty
 group: deprecated-2017Q4
@@ -15,18 +16,24 @@ python:
 # version, that causes fpm to crash.
 matrix:
   include:
+  # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
+  - python: "3.7"
+    dist: xenial
+    sudo: true
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
     - PYTHON_VER=2.7.10
     - PYENV_ROOT="$HOME/.pyenv"
     - PATH="$PYENV_ROOT/bin:$PATH"
+    - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
+    - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
@@ -35,23 +42,20 @@ matrix:
     - PATH="$PYENV_ROOT/bin:$PATH"
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
     - PYTHON_VER=3.5.0
     - PYENV_ROOT="$HOME/.pyenv"
     - PATH="$PYENV_ROOT/bin:$PATH"
-  # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
-  - python: "3.7"
-    dist: xenial
-    sudo: true
 before_install:
   - bash ./scripts/ci/before_install.sh
   - if  [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       eval "$(pyenv init -)";
       PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
       pyenv shell "$PYTHON_VER";
+      brew install openssl;
       brew cask install google-cloud-sdk;
     fi
   - which python
@@ -77,6 +81,8 @@ install:
   - bash ./scripts/ci/install.sh
   - pip install flake8
 before_script:
+  # stop the build if there are any readme formatting errors
+  - python setup.py checkdocs
   # stop the build if there are any flake8 errors
   - flake8 dvc
 script:

README.rst

@@ -2,9 +2,12 @@
   :target: https://dvc.org
   :alt: DVC logo
 
-`Website <https://dvc.org>`_ • `Docs <https://dvc.org/doc>`_ • `Discuss <https://discuss.dvc.org/>`_ •
-`Blog <https://blog.dataversioncontrol.com/>`_ • `Twitter <https://twitter.com/iterativeai>`_ •
-`Tutorial <https://dvc.org/doc/tutorial>`_
+`Website <https://dvc.org>`_
+• `Docs <https://dvc.org/doc>`_
+• `Twitter <https://twitter.com/iterativeai>`_
+• `Chat (Community & Support) <https://dvc.org/chat>`_
+• `Tutorial <https://dvc.org/doc/tutorial>`_
+• `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
 .. image:: https://travis-ci.org/iterative/dvc.svg?branch=master
   :target: https://travis-ci.org/iterative/dvc
@@ -24,10 +27,12 @@
 
 |
 
-**Data Science Version Control** or **DVC** is an **open-source** tool for data science projects.
-With a simple and flexible Git-like architecture and interface it helps data scientists:
+**Data Science Version Control** or **DVC** is an **open-source** tool for data science and 
+machine learning projects. With a simple and flexible Git-like architecture and interface it
+helps data scientists:
 
-#. manage code together with data files and models;
+#. manage **machine learning models** - versioning, including data sets and transformations (scripts) that were
+   used to generate models;
 
 #. make projects **reproducible**;
 
@@ -42,6 +47,17 @@ data file suffixes and prefixes.
 .. contents:: **Contents**
   :backlinks: none
 
+How DVC works
+=============
+
+DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
+To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
+network storage (via SSH, for example).
+
+.. image:: https://dvc.org/static/img/flow.gif
+   :target: https://dvc.org/static/img/flow.gif
+   :alt: how_dvc_works
+
 Quick start
 ===========
 
@@ -88,13 +104,24 @@ Homebrew
 
 .. code-block:: bash
 
-   brew install iterative/homebrew-dvc/dvc
+   $ brew install iterative/homebrew-dvc/dvc
 
 or:
 
 .. code-block:: bash
 
-   brew cask install iterative/homebrew-dvc/dvc
+   $ brew cask install iterative/homebrew-dvc/dvc
+
+AUR
+---
+
+*Unofficial package*, any inquiries regarding the AUR package,
+`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
+
+.. code-block:: bash
+
+  # https://aur.archlinux.org/packages/dvc
+  $ yay -S dvc
 
 Related technologies
 ====================
@@ -112,23 +139,48 @@ Related technologies
 #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-How DVC works
-=============
-
-DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
-To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
-network storage (via SSH, for example).
-
-.. image:: https://dvc.org/static/img/flow.png
-   :target: https://dvc.org/static/img/flow.png
-   :alt: how_dvc_works
-
-
 Contributing
 ============
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
 details.
 
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+  :alt: 0
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+  :alt: 1
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+  :alt: 2   
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3   
+  :alt: 3
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+  :alt: 4   
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+  :alt: 5
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+  :alt: 6   
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+  :alt: 7
+
+Mailing List
+============
+
+Want to stay up to date? Want to help improve DVC by participating in our ocassional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic.
+
 Copyright
 =========
 

dvc/__init__.py

@@ -4,8 +4,10 @@ DVC
 Make your data science projects reproducible and shareable.
 """
 import os
+import warnings
 
-VERSION_BASE = '0.16.0'
+
+VERSION_BASE = '0.19.8'
 __version__ = VERSION_BASE
 
 PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
@@ -31,13 +33,13 @@ if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
             with open(VERSIONPATH, 'w+') as fd:
                 fd.write('# AUTOGENERATED by dvc/__init__.py\n')
                 fd.write('version = "{}"\n'.format(__version__))
-        except Exception:
+        except Exception:  # pragma: no cover
             pass
-    else:
+    else:  # pragma: no cover
         # Remove version.py so that it doesn't get into the release
         if os.path.exists(VERSIONPATH):
             os.unlink(VERSIONPATH)
-else:
+else:  # pragma: no cover
     # dvc was installed with pip or something. Hopefully we have our
     # auto-generated version.py to help us provide a true version
     try:
@@ -47,3 +49,11 @@ else:
         pass
 
 VERSION = __version__
+
+
+# Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
+# We don't directly import numpy, but our dependency networkx does, causing
+# these warnings in some environments. Luckily these warnings are benign and
+# we can simply ignore them so that they don't show up when you are using dvc.
+warnings.filterwarnings("ignore", message="numpy.dtype size changed")
+warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

dvc/cli.py

@@ -3,7 +3,6 @@ from __future__ import print_function
 import os
 import sys
 import argparse
-from multiprocessing import cpu_count
 
 from dvc.command.init import CmdInit
 from dvc.command.destroy import CmdDestroy
@@ -26,6 +25,7 @@ from dvc.command.install import CmdInstall
 from dvc.command.root import CmdRoot
 from dvc.command.lock import CmdLock, CmdUnlock
 from dvc.command.pipeline import CmdPipelineShow
+from dvc.logger import Logger
 from dvc import VERSION
 
 
@@ -39,6 +39,19 @@ def _fix_subparsers(subparsers):
         subparsers.dest = 'cmd'
 
 
+class DvcParser(argparse.ArgumentParser):
+    def error(self, message):
+        sys.stderr.write('{}{}\n'.format(Logger.error_prefix(), message))
+        self.print_help()
+        sys.exit(2)
+
+
+class VersionAction(argparse.Action):  # pragma: no cover
+    def __call__(self, parser, namespace, values, option_string=None):
+        print(VERSION)
+        sys.exit(0)
+
+
 def parse_args(argv=None):
     # Common args
     parent_parser = argparse.ArgumentParser(add_help=False)
@@ -57,17 +70,18 @@ def parse_args(argv=None):
 
     # Main parser
     desc = 'Data Version Control'
-    parser = argparse.ArgumentParser(
-                        prog='dvc',
-                        description=desc,
-                        parents=[parent_parser],
-                        formatter_class=argparse.RawTextHelpFormatter)
+    parser = DvcParser(prog='dvc',
+                       description=desc,
+                       parents=[parent_parser],
+                       formatter_class=argparse.RawTextHelpFormatter)
 
+    # NOTE: On some python versions action='version' prints to stderr
+    # instead of stdout https://bugs.python.org/issue18920
     parser.add_argument('-V',
                         '--version',
-                        action='version',
-                        version='%(prog)s ' + VERSION,
-                        help='Show program\'s version')
+                        action=VersionAction,
+                        nargs=0,
+                        help='Show program\'s version.')
 
     # Sub commands
     subparsers = parser.add_subparsers(
@@ -75,84 +89,104 @@ def parse_args(argv=None):
                         metavar='COMMAND',
                         dest='cmd',
                         help='Use dvc COMMAND --help for command-specific '
-                             'help')
+                             'help.')
 
     _fix_subparsers(subparsers)
 
     # Init
+    INIT_HELP = 'Initialize dvc over a directory ' \
+                '(should already be a git dir).'
     init_parser = subparsers.add_parser(
                         'init',
                         parents=[parent_parser],
-                        help='Initialize dvc over a directory '
-                             '(should already be a git dir)')
+                        description=INIT_HELP,
+                        help=INIT_HELP)
     init_parser.add_argument(
                         '--no-scm',
                         action='store_true',
                         default=False,
                         help="Initiate dvc in directory that is "
-                             "not tracked by any scm tool(e.g. git)")
+                             "not tracked by any scm tool(e.g. git).")
     init_parser.add_argument(
                         '-f',
                         '--force',
                         action='store_true',
                         default=False,
-                        help='Force initialization even if \'.dvc\' exists')
+                        help="Overwrite '.dvc' if it exists. Will remove "
+                             "all local cache.")
     init_parser.set_defaults(func=CmdInit)
 
     # Destroy
+    DESTROY_HELP = "Destroy dvc. Will remove all project's information, " \
+                   "data files and cache."
     destroy_parser = subparsers.add_parser(
                         'destroy',
                         parents=[parent_parser],
-                        help='Destroy dvc')
+                        description=DESTROY_HELP,
+                        help=DESTROY_HELP)
     destroy_parser.add_argument(
                         '-f',
                         '--force',
                         action='store_true',
                         default=False,
-                        help='Force destruction')
+                        help='Force destruction.')
     destroy_parser.set_defaults(func=CmdDestroy)
 
     # Add
+    ADD_HELP = 'Add files/directories to dvc.'
     add_parser = subparsers.add_parser(
                         'add',
                         parents=[parent_parser],
-                        help='Add files/directories to dvc')
+                        description=ADD_HELP,
+                        help=ADD_HELP)
+    add_parser.add_argument(
+                        '-R',
+                        '--recursive',
+                        action='store_true',
+                        default=False,
+                        help='Recursively add each file under the directory.')
     add_parser.add_argument(
                         'targets',
                         nargs='+',
-                        help='Input files/directories')
+                        help='Input files/directories.')
     add_parser.set_defaults(func=CmdAdd)
 
     # Import
+    IMPORT_HELP = 'Import files from URL.'
     import_parser = subparsers.add_parser(
                         'import',
                         parents=[parent_parser],
-                        help='Import files from URL')
+                        description=IMPORT_HELP,
+                        help=IMPORT_HELP)
     import_parser.add_argument(
                         'url',
-                        help='URL')
+                        help='URL.')
     import_parser.add_argument(
                         'out',
-                        help='Output')
+                        help='Output.')
     import_parser.set_defaults(func=CmdImport)
 
     # Checkout
+    CHECKOUT_HELP = 'Checkout data files from cache.'
     checkout_parser = subparsers.add_parser(
                         'checkout',
                         parents=[parent_parser],
-                        help='Checkout data files from cache')
+                        description=CHECKOUT_HELP,
+                        help=CHECKOUT_HELP)
     checkout_parser.add_argument(
                         'targets',
                         nargs='*',
-                        help='DVC files')
+                        help='DVC files.')
     checkout_parser.set_defaults(func=CmdCheckout)
 
     # Run
+    RUN_HELP = 'Generate a stage file from a given ' \
+               'command and execute the command.'
     run_parser = subparsers.add_parser(
                         'run',
                         parents=[parent_parser],
-                        help='Generate a stage file from a given '
-                             'command and execute the command')
+                        description=RUN_HELP,
+                        help=RUN_HELP)
     run_parser.add_argument(
                         '-d',
                         '--deps',
@@ -182,23 +216,36 @@ def parse_args(argv=None):
     run_parser.add_argument(
                         '-f',
                         '--file',
-                        help='Specify name of the state file')
+                        help="Specify name of the stage file. It should be "
+                             "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+                             "'prepare.dvc', 'clean.dvc', etc) in order for "
+                             "dvc to be able to find it later. By default "
+                             "the first output basename + .dvc is used as "
+                             "a stage filename.")
     run_parser.add_argument(
                         '-c',
                         '--cwd',
                         default=os.path.curdir,
                         help='Directory to run your command and place state '
-                             'file in')
+                             'file in.')
     run_parser.add_argument(
                         '--no-exec',
                         action='store_true',
                         default=False,
                         help="Only create stage file without actually "
-                             "running it")
+                             "running it.")
+    run_parser.add_argument(
+                        '-y',
+                        '--yes',
+                        action='store_true',
+                        default=False,
+                        help="Automatic 'yes' answer to all prompts. E.g. "
+                             "when '.dvc' file exists and dvc asks if you "
+                             "want to overwrite it.")
     run_parser.add_argument(
                         'command',
                         nargs=argparse.REMAINDER,
-                        help='Command or command file to execute')
+                        help='Command or command file to execute.')
     run_parser.set_defaults(func=CmdRun)
 
     # Parent parser used in pull/push/status
@@ -209,7 +256,7 @@ def parse_args(argv=None):
                         '-j',
                         '--jobs',
                         type=int,
-                        default=8*cpu_count(),
+                        default=None,
                         help='Number of jobs to run simultaneously.')
     parent_cache_parser.add_argument(
                         '--show-checksums',
@@ -223,84 +270,153 @@ def parse_args(argv=None):
                         help='DVC files.')
 
     # Pull
+    PULL_HELP = 'Pull data files from the cloud.'
     pull_parser = subparsers.add_parser(
                         'pull',
                         parents=[parent_cache_parser],
-                        help='Pull data files from the cloud')
+                        description=PULL_HELP,
+                        help=PULL_HELP)
     pull_parser.add_argument(
                         '-r',
                         '--remote',
-                        help='Remote repository to pull from')
+                        help='Remote repository to pull from.')
     pull_parser.add_argument(
                         '-a',
                         '--all-branches',
                         action='store_true',
                         default=False,
                         help='Fetch cache for all branches.')
+    pull_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all tags.')
+    pull_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all dependencies of the '
+                             'specified target.')
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
+    PUSH_HELP = 'Push data files to the cloud.'
     push_parser = subparsers.add_parser(
                         'push',
                         parents=[parent_cache_parser],
-                        help='Push data files to the cloud')
+                        description=PUSH_HELP,
+                        help=PUSH_HELP)
     push_parser.add_argument(
                         '-r',
                         '--remote',
-                        help='Remote repository to push to')
+                        help='Remote repository to push to.')
     push_parser.add_argument(
                         '-a',
                         '--all-branches',
                         action='store_true',
                         default=False,
                         help='Push cache for all branches.')
+    push_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Push cache for all tags.')
+    push_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Push cache for all dependencies of the '
+                             'specified target.')
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
+    FETCH_HELP = 'Fetch data files from the cloud.'
     fetch_parser = subparsers.add_parser(
                         'fetch',
                         parents=[parent_cache_parser],
-                        help='Fetch data files from the cloud')
+                        description=FETCH_HELP,
+                        help=FETCH_HELP)
     fetch_parser.add_argument(
                         '-r',
                         '--remote',
-                        help='Remote repository to fetch from')
+                        help='Remote repository to fetch from.')
     fetch_parser.add_argument(
                         '-a',
                         '--all-branches',
                         action='store_true',
                         default=False,
                         help='Fetch cache for all branches.')
+    fetch_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all tags.')
+    fetch_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all dependencies of the '
+                             'specified target.')
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
+    STATUS_HELP = 'Show the project status.'
     status_parser = subparsers.add_parser(
                         'status',
                         parents=[parent_cache_parser],
-                        help='Show the project status')
+                        description=STATUS_HELP,
+                        help=STATUS_HELP)
     status_parser.add_argument(
                         '-c',
                         '--cloud',
                         action='store_true',
                         default=False,
                         help='Show status of a local cache compared to a '
-                             'remote repository')
+                             'remote repository.')
     status_parser.add_argument(
                         '-r',
                         '--remote',
-                        help='Remote repository to compare local cache to')
+                        help='Remote repository to compare local cache to.')
+    status_parser.add_argument(
+                        '-a',
+                        '--all-branches',
+                        action='store_true',
+                        default=False,
+                        help='Show status of a local cache compared to a '
+                             'remote repository for all branches.')
+    status_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Show status of a local cache compared to a '
+                             'remote repository for all tags.')
+    status_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Show status of a local cache compared to a '
+                             'remote repository for all dependencies of the '
+                             'specified target.')
     status_parser.set_defaults(func=CmdDataStatus)
 
     # Repro
+    REPRO_HELP = 'Reproduce DVC file. Default file name - \'Dvcfile\'.'
     repro_parser = subparsers.add_parser(
                         'repro',
                         parents=[parent_parser],
-                        help='Reproduce DVC file. Default file name '
-                             '- \'Dvcfile\'')
+                        description=REPRO_HELP,
+                        help=REPRO_HELP)
     repro_parser.add_argument(
                         'targets',
                         nargs='*',
-                        default=['Dvcfile'],
                         help='DVC file to reproduce.')
     repro_parser.add_argument(
                         '-f',
@@ -326,7 +442,7 @@ def parse_args(argv=None):
                         '--metrics',
                         action='store_true',
                         default=False,
-                        help='Show metrics after reproduction')
+                        help='Show metrics after reproduction.')
     repro_parser.add_argument(
                         '--dry',
                         action='store_true',
@@ -340,26 +456,41 @@ def parse_args(argv=None):
                         default=False,
                         help='Ask for confirmation before reproducing each '
                              'stage.')
+    repro_parser.add_argument(
+                        '-p',
+                        '--pipeline',
+                        action='store_true',
+                        default=False,
+                        help='Reproduce the whole pipeline that the '
+                             'specified stage file belongs to.')
+    repro_parser.add_argument(
+                        '-P',
+                        '--all-pipelines',
+                        action='store_true',
+                        default=False,
+                        help='Reproduce all pipelines in the project.')
     repro_parser.set_defaults(func=CmdRepro)
 
     # Remove
+    REMOVE_HELP = 'Remove outputs of DVC file.'
     remove_parser = subparsers.add_parser(
                         'remove',
                         parents=[parent_parser],
-                        help='Remove outputs of DVC file.')
+                        description=REMOVE_HELP,
+                        help=REMOVE_HELP)
     remove_parser_group = remove_parser.add_mutually_exclusive_group()
     remove_parser_group.add_argument(
                         '-o',
                         '--outs',
                         action='store_true',
                         default=True,
-                        help='Only remove DVC file outputs.')
+                        help='Only remove DVC file outputs.(default)')
     remove_parser_group.add_argument(
                         '-p',
                         '--purge',
                         action='store_true',
                         default=False,
-                        help='Remove DVC file and all its outputs')
+                        help='Remove DVC file and all its outputs.')
     remove_parser.add_argument(
                         'targets',
                         nargs='+',
@@ -367,107 +498,131 @@ def parse_args(argv=None):
     remove_parser.set_defaults(func=CmdRemove)
 
     # Move
+    MOVE_HELP = 'Move output of DVC file.'
     move_parser = subparsers.add_parser(
                         'move',
                         parents=[parent_parser],
-                        help='Move output of DVC file.')
+                        description=MOVE_HELP,
+                        help=MOVE_HELP)
     move_parser.add_argument(
                         'src',
-                        help='Source')
+                        help='Source.')
     move_parser.add_argument(
                         'dst',
-                        help='Destination')
+                        help='Destination.')
     move_parser.set_defaults(func=CmdMove)
 
     # Garbage collector
+    GC_HELP = 'Collect garbage.'
     gc_parser = subparsers.add_parser(
                         'gc',
                         parents=[parent_parser],
-                        help='Collect garbage')
+                        description=GC_HELP,
+                        help=GC_HELP)
     gc_parser.add_argument(
                         '-a',
                         '--all-branches',
                         action='store_true',
                         default=False,
                         help='Collect garbage for all branches.')
+    gc_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Collect garbage for all tags.')
     gc_parser.add_argument(
                         '-c',
                         '--cloud',
                         action='store_true',
                         default=False,
-                        help='Collect garbage in remote repository')
+                        help='Collect garbage in remote repository.')
     gc_parser.add_argument(
                         '-r',
                         '--remote',
-                        help='Remote repository to collect garbage in')
+                        help='Remote repository to collect garbage in.')
+    gc_parser.add_argument(
+                        '-f',
+                        '--force',
+                        action='store_true',
+                        default=False,
+                        help='Force garbage collection.')
     gc_parser.set_defaults(func=CmdGC)
 
     # Config
+    CONFIG_HELP = 'Get or set config options.'
     config_parser = subparsers.add_parser(
                         'config',
                         parents=[parent_parser],
-                        help='Get or set config options')
+                        description=CONFIG_HELP,
+                        help=CONFIG_HELP)
     config_parser.add_argument(
                         '-u',
                         '--unset',
                         default=False,
                         action='store_true',
-                        help='Unset option')
+                        help='Unset option.')
     config_parser.add_argument(
                         'name',
-                        help='Option name')
+                        help='Option name.')
     config_parser.add_argument(
                         'value',
                         nargs='?',
                         default=None,
-                        help='Option value')
+                        help='Option value.')
     config_parser.add_argument(
                         '--local',
                         action='store_true',
                         default=False,
-                        help='Use local config')
+                        help='Use local config.')
     config_parser.set_defaults(func=CmdConfig)
 
     # Remote
+    REMOTE_HELP = 'Manage set of tracked repositories.'
     remote_parser = subparsers.add_parser(
                         'remote',
                         parents=[parent_parser],
-                        help='Manage set of tracked repositories')
+                        description=REMOTE_HELP,
+                        help=REMOTE_HELP)
 
     remote_subparsers = remote_parser.add_subparsers(
                         dest='cmd',
                         help='Use dvc remote CMD --help for '
-                             'command-specific help')
+                             'command-specific help.')
 
     _fix_subparsers(remote_subparsers)
 
+    REMOTE_ADD_HELP = 'Add remote.'
     remote_add_parser = remote_subparsers.add_parser(
                         'add',
                         parents=[parent_parser],
-                        help='Add remote')
+                        description=REMOTE_ADD_HELP,
+                        help=REMOTE_ADD_HELP)
     remote_add_parser.add_argument(
                         'name',
-                        help='Name')
+                        help='Name.')
     remote_add_parser.add_argument(
                         'url',
-                        help='Url')
+                        help='URL.')
     remote_add_parser.add_argument(
                         '--local',
                         action='store_true',
                         default=False,
-                        help='Use local config')
+                        help='Use local config.')
     remote_add_parser.add_argument(
                         '-d',
                         '--default',
                         action='store_true',
                         default=False,
-                        help='Set as default remote')
+                        help='Set as default remote.')
     remote_add_parser.set_defaults(func=CmdRemoteAdd)
 
+    REMOTE_REMOVE_HELP = 'Remove remote.'
     remote_remove_parser = remote_subparsers.add_parser(
                         'remove',
                         parents=[parent_parser],
-                        help='Remove remote')
+                        description=REMOTE_REMOVE_HELP,
+                        help=REMOTE_REMOVE_HELP)
     remote_remove_parser.add_argument(
                         'name',
                         help='Name')
@@ -475,162 +630,188 @@ def parse_args(argv=None):
                         '--local',
                         action='store_true',
                         default=False,
-                        help='Use local config')
+                        help='Use local config.')
     remote_remove_parser.set_defaults(func=CmdRemoteRemove)
 
+    REMOTE_MODIFY_HELP = 'Modify remote.'
     remote_modify_parser = remote_subparsers.add_parser(
                         'modify',
                         parents=[parent_parser],
-                        help='Modify remote')
+                        description=REMOTE_MODIFY_HELP,
+                        help=REMOTE_MODIFY_HELP)
     remote_modify_parser.add_argument(
                         'name',
-                        help='Name')
+                        help='Name.')
     remote_modify_parser.add_argument(
                         'option',
-                        help='Option')
+                        help='Option.')
     remote_modify_parser.add_argument(
                         'value',
                         nargs='?',
-                        help='Value')
+                        help='Value.')
     remote_modify_parser.add_argument(
                         '-u',
                         '--unset',
                         default=False,
                         action='store_true',
-                        help='Unset option')
+                        help='Unset option.')
     remote_modify_parser.add_argument(
                         '--local',
                         action='store_true',
                         default=False,
-                        help='Use local config')
+                        help='Use local config.')
     remote_modify_parser.set_defaults(func=CmdRemoteModify)
 
+    REMOTE_LIST_HELP = 'List remotes.'
     remote_list_parser = remote_subparsers.add_parser(
                         'list',
                         parents=[parent_parser],
-                        help='List remotes')
+                        description=REMOTE_LIST_HELP,
+                        help=REMOTE_LIST_HELP)
     remote_list_parser.add_argument(
                         '--local',
                         action='store_true',
                         default=False,
-                        help='Use local config')
+                        help='Use local config.')
     remote_list_parser.set_defaults(func=CmdRemoteList)
 
     # Metrics
+    METRICS_HELP = 'Get metrics from all branches.'
     metrics_parser = subparsers.add_parser(
                         'metrics',
                         parents=[parent_parser],
-                        help='Get metrics from all branches')
+                        description=METRICS_HELP,
+                        help=METRICS_HELP)
 
     metrics_subparsers = metrics_parser.add_subparsers(
                         dest='cmd',
                         help='Use dvc metrics CMD --help for '
-                             'command-specific help')
+                             'command-specific help.')
 
     _fix_subparsers(metrics_subparsers)
 
+    METRICS_SHOW_HELP = 'Show metrics.'
     metrics_show_parser = metrics_subparsers.add_parser(
                         'show',
                         parents=[parent_parser],
-                        help='Show metrics')
+                        description=METRICS_SHOW_HELP,
+                        help=METRICS_SHOW_HELP)
     metrics_show_parser.add_argument(
                         'path',
                         nargs='?',
-                        help='Path to metrics file')
+                        help='Path to metrics file.')
     metrics_show_parser.add_argument(
                         '-t',
                         '--type',
-                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)')
+                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
     metrics_show_parser.add_argument(
                         '-x',
                         '--xpath',
-                        help='JSON/TSV/HTSV/CSV/HCSV path')
+                        help='JSON/TSV/HTSV/CSV/HCSV path.')
     metrics_show_group = metrics_show_parser.add_mutually_exclusive_group()
     metrics_show_group.add_argument(
                         '--json-path',
-                        help='JSON path')
+                        help='JSON path.')
     metrics_show_group.add_argument(
                         '--tsv-path',
-                        help='TSV path \'row,column\'(e.g. \'1,2\')')
+                        help='TSV path \'row,column\'(e.g. \'1,2\').')
     metrics_show_group.add_argument(
                         '--htsv-path',
-                        help='Headed TSV path \'row,column\'(e.g. \'Name,3\'')
+                        help='Headed TSV path \'row,column\'(e.g. \'Name,3\'.')
     metrics_show_group.add_argument(
                         '--csv-path',
-                        help='CSV path \'row,column\'(e.g. \'1,2\')')
+                        help='CSV path \'row,column\'(e.g. \'1,2\').')
     metrics_show_group.add_argument(
                         '--hcsv-path',
-                        help='Headed CSV path \'row,column\'(e.g. \'Name,3\'')
+                        help='Headed CSV path \'row,column\'(e.g. \'Name,3\'.')
     metrics_show_parser.add_argument(
                         '-a',
                         '--all-branches',
                         action='store_true',
                         default=False,
-                        help='Show metrics for all branches')
+                        help='Show metrics for all branches.')
+    metrics_show_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Show metrics for all tags.')
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
+    METRICS_ADD_HELP = 'Add metrics.'
     metrics_add_parser = metrics_subparsers.add_parser(
                         'add',
                         parents=[parent_parser],
-                        help='Add metrics')
+                        description=METRICS_ADD_HELP,
+                        help=METRICS_ADD_HELP)
     metrics_add_parser.add_argument(
                         '-t',
                         '--type',
-                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)')
+                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
     metrics_add_parser.add_argument(
                         '-x',
                         '--xpath',
-                        help='JSON/TSV/HTSV/CSV/HCSV path')
+                        help='JSON/TSV/HTSV/CSV/HCSV path.')
     metrics_add_parser.add_argument(
                         'path',
-                        help='Path to metrics file')
+                        help='Path to metrics file.')
     metrics_add_parser.set_defaults(func=CmdMetricsAdd)
 
+    METRICS_MODIFY_HELP = 'Modify metrics.'
     metrics_modify_parser = metrics_subparsers.add_parser(
                         'modify',
                         parents=[parent_parser],
-                        help='Modify metrics')
+                        description=METRICS_MODIFY_HELP,
+                        help=METRICS_MODIFY_HELP)
     metrics_modify_parser.add_argument(
                         '-t',
                         '--type',
-                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)')
+                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
     metrics_modify_parser.add_argument(
                         '-x',
                         '--xpath',
-                        help='JSON/TSV/HTSV/CSV/HCSV path')
+                        help='JSON/TSV/HTSV/CSV/HCSV path.')
     metrics_modify_parser.add_argument(
                         'path',
-                        help='Metrics file')
+                        help='Metrics file.')
     metrics_modify_parser.set_defaults(func=CmdMetricsModify)
 
+    METRICS_REMOVE_HELP = 'Remove metrics.'
     metrics_remove_parser = metrics_subparsers.add_parser(
                         'remove',
                         parents=[parent_parser],
-                        help='Remove metrics')
+                        description=METRICS_REMOVE_HELP,
+                        help=METRICS_REMOVE_HELP)
     metrics_remove_parser.add_argument(
                         'path',
-                        help='Path to metrics file')
+                        help='Path to metrics file.')
     metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
 
     # Install
+    INSTALL_HELP = 'Install dvc hooks into the repository.'
     install_parser = subparsers.add_parser(
                         'install',
                         parents=[parent_parser],
-                        help='Install dvc hooks into the repository')
+                        description=INSTALL_HELP,
+                        help=INSTALL_HELP)
     install_parser.set_defaults(func=CmdInstall)
 
     # Root
+    ROOT_HELP = 'Relative path to project\'s directory.'
     root_parser = subparsers.add_parser(
                         'root',
                         parents=[parent_parser],
-                        help='Relative path to project\'s directory')
+                        description=ROOT_HELP,
+                        help=ROOT_HELP)
     root_parser.set_defaults(func=CmdRoot)
 
     # Lock
+    LOCK_HELP = 'Lock DVC file.'
     lock_parser = subparsers.add_parser(
                         'lock',
                         parents=[parent_parser],
-                        help='Lock DVC file')
+                        description=LOCK_HELP,
+                        help=LOCK_HELP)
     lock_parser.add_argument(
                         'targets',
                         nargs='+',
@@ -638,10 +819,12 @@ def parse_args(argv=None):
     lock_parser.set_defaults(func=CmdLock)
 
     # Unlock
+    UNLOCK_HELP = 'Unlock DVC file.'
     unlock_parser = subparsers.add_parser(
                         'unlock',
                         parents=[parent_parser],
-                        help='Unlock DVC file')
+                        description=UNLOCK_HELP,
+                        help=UNLOCK_HELP)
     unlock_parser.add_argument(
                         'targets',
                         nargs='+',
@@ -649,22 +832,26 @@ def parse_args(argv=None):
     unlock_parser.set_defaults(func=CmdUnlock)
 
     # Pipeline
+    PIPELINE_HELP = 'Manage pipeline.'
     pipeline_parser = subparsers.add_parser(
                         'pipeline',
                         parents=[parent_parser],
-                        help='Manage pipeline')
+                        description=PIPELINE_HELP,
+                        help=PIPELINE_HELP)
 
     pipeline_subparsers = pipeline_parser.add_subparsers(
                         dest='cmd',
                         help='Use dvc pipeline CMD --help '
-                             'for command-specific help')
+                             'for command-specific help.')
 
     _fix_subparsers(pipeline_subparsers)
 
+    PIPELINE_SHOW_HELP = 'Show pipeline.'
     pipeline_show_parser = pipeline_subparsers.add_parser(
                         'show',
                         parents=[parent_parser],
-                        help='Show pipeline')
+                        description=PIPELINE_SHOW_HELP,
+                        help=PIPELINE_SHOW_HELP)
     pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
     pipeline_show_group.add_argument(
                         '-c',
@@ -679,12 +866,48 @@ def parse_args(argv=None):
                         default=False,
                         help='Print output files instead of '
                              'paths to DVC files.')
+    pipeline_show_parser.add_argument(
+                        '--ascii',
+                        action='store_true',
+                        default=False,
+                        help='Output DAG as ASCII.')
+    pipeline_show_parser.add_argument(
+                        '--dot',
+                        help='Write DAG in .dot format.'
+    )
     pipeline_show_parser.add_argument(
                         'targets',
-                        nargs='+',
-                        help='DVC files.')
+                        nargs='*',
+                        help="DVC files. 'Dvcfile' by default.")
     pipeline_show_parser.set_defaults(func=CmdPipelineShow)
 
     args = parser.parse_args(argv)
 
+    if (issubclass(args.func, CmdRepro)
+        or issubclass(args.func, CmdPipelineShow)) \
+       and hasattr(args, 'targets') \
+       and len(args.targets) == 0 \
+       and not (hasattr(args, 'all_pipelines')
+       and args.all_pipelines):  # pragma: no cover
+        if hasattr(args, 'cwd'):
+            cwd = args.cwd
+        else:
+            cwd = os.curdir
+        path = os.path.join(cwd, 'Dvcfile')
+        if not os.path.exists(path):
+            msg = "default target '{}' does not exist.".format(path)
+            if issubclass(args.func, CmdRepro):
+                repro_parser.error(msg)
+            elif issubclass(args.func, CmdPipelineShow):
+                pipeline_show_parser.error(msg)
+        args.targets = ['Dvcfile']
+
+    if issubclass(args.func, CmdRun) \
+       and len(args.deps) == 0 \
+       and len(args.outs) == 0 \
+       and len(args.outs_no_cache) == 0 \
+       and len(args.command) == 0:  # pragma: no cover
+        run_parser.error("Too few arguments. Specify at least one: "
+                         "'-d', '-o', '-O', 'command'.")
+
     return args

dvc/command/add.py

@@ -7,7 +7,7 @@ class CmdAdd(CmdBase):
     def run(self):
         for target in self.args.targets:
             try:
-                self.project.add(target)
+                self.project.add(target, recursive=self.args.recursive)
             except DvcException as ex:
                 Logger.error('Failed to add \'{}\''.format(target), ex)
                 return 1

dvc/command/data_sync.py

@@ -23,7 +23,9 @@ class CmdDataPull(CmdDataBase):
                               jobs=self.args.jobs,
                               remote=self.args.remote,
                               show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches)
+                              all_branches=self.args.all_branches,
+                              all_tags=self.args.all_tags,
+                              with_deps=self.args.with_deps)
         except Exception as exc:
             self.project.logger.error('Failed to pull data from the cloud',
                                       exc)
@@ -38,7 +40,9 @@ class CmdDataPush(CmdDataBase):
                               jobs=self.args.jobs,
                               remote=self.args.remote,
                               show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches)
+                              all_branches=self.args.all_branches,
+                              all_tags=self.args.all_tags,
+                              with_deps=self.args.with_deps)
         except Exception as exc:
             self.project.logger.error('Failed to push data to the cloud', exc)
             return 1
@@ -52,7 +56,9 @@ class CmdDataFetch(CmdDataBase):
                                jobs=self.args.jobs,
                                remote=self.args.remote,
                                show_checksums=self.args.show_checksums,
-                               all_branches=self.args.all_branches)
+                               all_branches=self.args.all_branches,
+                               all_tags=self.args.all_tags,
+                               with_deps=self.args.with_deps)
         except Exception as exc:
             self.project.logger.error('Failed to fetch data from the cloud',
                                       exc)

dvc/command/destroy.py

@@ -1,16 +1,16 @@
 from dvc.command.base import CmdBase
-from dvc.prompt import prompt
 from dvc.exceptions import DvcException
 
 
 class CmdDestroy(CmdBase):
     def run_cmd(self):
         try:
-            msg = u'This will destroy all information about your pipelines ' \
-                  u'as well as cache in .dvc/cache.\n' \
+            msg = u'This will destroy all information about your pipelines, ' \
+                  u'all data files, as well as cache in .dvc/cache.\n' \
                   u'Are you sure you want to continue?'
 
-            if not self.args.force and not prompt(msg, False):
+            if not self.args.force \
+               and not self.project.prompt.prompt(msg, False):
                 msg = u'Cannot destroy without a confirmation from the ' \
                       u'user. Use \'-f\' to force.'
                 raise DvcException(msg)

dvc/command/gc.py

@@ -3,7 +3,24 @@ from dvc.command.base import CmdBase
 
 class CmdGC(CmdBase):
     def run(self):
+        msg = 'This will remove all cache except the cache that is used in '
+        if not self.args.all_branches and not self.args.all_tags:
+            msg += 'the current git branch.'
+        elif self.args.all_branches and not self.args.all_tags:
+            msg += 'all git branches.'
+        elif not self.args.all_branches and self.args.all_tags:
+            msg += 'all git tags.'
+        else:
+            msg += 'all git branches and all git tags.'
+
+        self.project.logger.warn(msg)
+
+        msg = 'Are you sure you want to proceed?'
+        if not self.args.force and not self.project.prompt.prompt(msg):
+            return 1
+
         self.project.gc(all_branches=self.args.all_branches,
+                        all_tags=self.args.all_tags,
                         cloud=self.args.cloud,
                         remote=self.args.remote)
         return 0

dvc/command/metrics.py

@@ -28,7 +28,8 @@ class CmdMetricsShow(CmdBase):
             self.project.metrics_show(self.args.path,
                                       typ=typ,
                                       xpath=xpath,
-                                      all_branches=self.args.all_branches)
+                                      all_branches=self.args.all_branches,
+                                      all_tags=self.args.all_tags)
         except DvcException as exc:
             self.project.logger.error('Failed to show metrics', exc)
             return 1

dvc/command/pipeline.py

@@ -2,32 +2,109 @@ import os
 
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
+from dvc.dagascii import Dagascii
 
 
 class CmdPipelineShow(CmdBase):
     def _show(self, target, commands, outs):
         import networkx
-        from dvc.project import NotDvcFileError
+        from dvc.stage import Stage
 
+        stage = Stage.load(self.project, target)
         G = self.project.graph()[0]
         stages = networkx.get_node_attributes(G, 'stage')
-        node = os.path.relpath(os.path.abspath(target), self.project.root_dir)
-        if node not in stages:
-            raise NotDvcFileError(node)
+        node = os.path.relpath(stage.path, self.project.root_dir)
 
         for n in networkx.dfs_postorder_nodes(G, node):
             if commands:
                 self.project.logger.info(stages[n].cmd)
             elif outs:
                 for out in stages[n].outs:
-                    self.project.logger.info(out.rel_path)
+                    self.project.logger.info(str(out))
             else:
                 self.project.logger.info(n)
 
+    def __build_graph(self, target, commands, outs):
+        import networkx
+        from dvc.stage import Stage
+
+        stage = Stage.load(self.project, target)
+        node = os.path.relpath(stage.path, self.project.root_dir)
+
+        pipelines = list(filter(lambda g: node in g.nodes(),
+                                self.project.pipelines()))
+
+        assert len(pipelines) == 1
+        G = pipelines[0]
+        stages = networkx.get_node_attributes(G, 'stage')
+
+        nodes = []
+        for n in G.nodes():
+            stage = stages[n]
+            if commands:
+                if stage.cmd is None:
+                    continue
+                nodes.append(stage.cmd)
+            elif outs:
+                for out in stage.outs:
+                    nodes.append(str(out))
+            else:
+                nodes.append(stage.relpath)
+
+        edges = []
+        for e in G.edges():
+            from_stage = stages[e[0]]
+            to_stage = stages[e[1]]
+            if commands:
+                if to_stage.cmd is None:
+                    continue
+                edges.append((from_stage.cmd, to_stage.cmd))
+            elif outs:
+                for from_out in from_stage.outs:
+                    for to_out in to_stage.outs:
+                        edges.append((str(from_out),
+                                      str(to_out)))
+            else:
+                edges.append((from_stage.relpath, to_stage.relpath))
+
+        return nodes, edges
+
+    def _show_ascii(self, target, commands, outs):
+        nodes, edges = self.__build_graph(target, commands, outs)
+
+        if not nodes:
+            return
+
+        d = Dagascii(nodes, edges)
+        d.draw()
+
+    def __write_dot(self, target, commands, outs, filename):
+        import networkx
+        from networkx.drawing.nx_pydot import write_dot
+
+        _, edges = self.__build_graph(target, commands, outs)
+        edges = [edge[::-1] for edge in edges]
+
+        simple_g = networkx.DiGraph()
+        simple_g.add_edges_from(edges)
+        write_dot(simple_g, filename)
+
     def run(self, unlock=False):
         for target in self.args.targets:
             try:
-                self._show(target, self.args.commands, self.args.outs)
+                if self.args.ascii:
+                    self._show_ascii(target,
+                                     self.args.commands,
+                                     self.args.outs)
+                elif self.args.dot:
+                    self.__write_dot(target,
+                                     self.args.commands,
+                                     self.args.outs,
+                                     self.args.dot)
+                else:
+                    self._show(target,
+                               self.args.commands,
+                               self.args.outs)
             except DvcException as ex:
                 msg = 'Failed to show pipeline for \'{}\''.format(target)
                 self.project.logger.error(msg, ex)

dvc/command/remote.py

@@ -1,4 +1,6 @@
+import os
 import re
+import configobj
 
 from dvc.config import Config
 from dvc.command.config import CmdConfig
@@ -23,9 +25,35 @@ class CmdRemoteAdd(CmdConfig):
 
 
 class CmdRemoteRemove(CmdConfig):
+    def _remove_default(self, config_file, remote):
+        path = os.path.join(os.path.dirname(self.config_file),
+                            config_file)
+        config = configobj.ConfigObj(path)
+
+        core = config.get(Config.SECTION_CORE, None)
+        if core is None:
+            return
+
+        default = core.get(Config.SECTION_CORE_REMOTE, None)
+        if default is None:
+            return
+
+        if default == remote:
+            del config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE]
+            if len(config[Config.SECTION_CORE]) == 0:
+                del config[Config.SECTION_CORE]
+
+        config.write()
+
     def run(self):
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        return self.unset(section, Config.SECTION_REMOTE_URL)
+        ret = self.unset(section)
+        if ret != 0:
+            return ret
+
+        self._remove_default(Config.CONFIG, self.args.name)
+        self._remove_default(Config.CONFIG_LOCAL, self.args.name)
+        return 0
 
 
 class CmdRemoteModify(CmdConfig):

dvc/command/repro.py

@@ -1,12 +1,12 @@
 import os
 
 from dvc.command.base import CmdBase
+from dvc.command.status import CmdDataStatus
+from dvc.exceptions import DvcException
 
 
 class CmdRepro(CmdBase):
     def run(self):
-        from dvc.project import ReproductionError
-
         recursive = not self.args.single_item
         saved_dir = os.path.realpath(os.curdir)
         if self.args.cwd:
@@ -15,23 +15,25 @@ class CmdRepro(CmdBase):
         ret = 0
         for target in self.args.targets:
             try:
-                self.project.reproduce(target,
+                stages = self.project.reproduce(
+                                       target,
                                        recursive=recursive,
                                        force=self.args.force,
                                        dry=self.args.dry,
-                                       interactive=self.args.interactive)
+                                       interactive=self.args.interactive,
+                                       pipeline=self.args.pipeline,
+                                       all_pipelines=self.args.all_pipelines)
+
+                if len(stages) == 0:
+                    self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+
                 if self.args.metrics:
                     self.project.metrics_show()
-            except ReproductionError as ex:
+            except DvcException as ex:
                 msg = 'Failed to reproduce \'{}\''.format(target)
                 self.project.logger.error(msg, ex)
                 ret = 1
                 break
-            except Exception as ex:
-                msg = 'Failed to reproduce \'{}\' - unexpected error'
-                self.project.logger.error(msg.format(target), ex)
-                ret = 1
-                break
 
         os.chdir(saved_dir)
         return ret

dvc/command/run.py

@@ -5,6 +5,9 @@ from dvc.exceptions import DvcException
 class CmdRun(CmdBase):
     def run(self):
         try:
+            if self.args.yes:
+                self.project.prompt.default = True
+
             self.project.run(cmd=' '.join(self.args.command),
                              outs=self.args.outs,
                              outs_no_cache=self.args.outs_no_cache,

dvc/command/status.py

@@ -4,6 +4,7 @@ from dvc.command.data_sync import CmdDataBase
 class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 10
     STATUS_INDENT = '\t'
+    UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce."
 
     def _normalize(self, s):
         s += ':'
@@ -28,8 +29,15 @@ class CmdDataStatus(CmdDataBase):
                                      jobs=self.args.jobs,
                                      cloud=self.args.cloud,
                                      show_checksums=self.args.show_checksums,
-                                     remote=self.args.remote)
-            self._show(st, indent)
+                                     remote=self.args.remote,
+                                     all_branches=self.args.all_branches,
+                                     all_tags=self.args.all_tags,
+                                     with_deps=self.args.with_deps)
+            if st:
+                self._show(st, indent)
+            else:
+                self.project.logger.info(self.UP_TO_DATE_MSG)
+
         except Exception as exc:
             self.project.logger.error('Failed to obtain data status', exc)
             return 1

dvc/config.py

@@ -10,8 +10,9 @@ from dvc.exceptions import DvcException
 
 class ConfigError(DvcException):
     """ DVC config exception """
-    def __init__(self, ex=None):
-        super(ConfigError, self).__init__('Config file error', ex)
+    def __init__(self, msg, ex=None):
+        super(ConfigError, self).__init__('Config file error: {}'.format(msg),
+                                          ex)
 
 
 def supported_url(url):
@@ -36,6 +37,22 @@ def supported_cloud(cloud):
     return cloud in ['aws', 'gcp', 'local', '']
 
 
+def is_bool(val):
+    return val.lower() in ['true', 'false']
+
+
+def to_bool(val):
+    return val.lower() == 'true'
+
+
+def is_whole(val):
+    return int(val) >= 0
+
+
+def is_percent(val):
+    return int(val) >= 0 and int(val) <= 100
+
+
 class Config(object):
     CONFIG = 'config'
     CONFIG_LOCAL = 'config.local'
@@ -44,6 +61,8 @@ class Config(object):
     SECTION_CORE_LOGLEVEL = 'loglevel'
     SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
     SECTION_CORE_REMOTE = 'remote'
+    SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
+    SECTION_CORE_INTERACTIVE = 'interactive'
 
     SECTION_CACHE = 'cache'
     SECTION_CACHE_DIR = 'dir'
@@ -78,6 +97,8 @@ class Config(object):
                  default='info'): And(str, Use(str.lower),
                                       SECTION_CORE_LOGLEVEL_SCHEMA),
         Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)),
+        Optional(SECTION_CORE_INTERACTIVE,
+                 default=False): SECTION_CORE_INTERACTIVE_SCHEMA,
 
         # backward compatibility
         Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
@@ -119,6 +140,11 @@ class Config(object):
     SECTION_REMOTE_FMT = 'remote "{}"'
     SECTION_REMOTE_URL = 'url'
     SECTION_REMOTE_USER = 'user'
+    SECTION_REMOTE_PORT = 'port'
+    SECTION_REMOTE_KEY_FILE = 'keyfile'
+    SECTION_REMOTE_TIMEOUT = 'timeout'
+    SECTION_REMOTE_PASSWORD = 'password'
+    SECTION_REMOTE_ASK_PASSWORD = 'ask_password'
     SECTION_REMOTE_SCHEMA = {
         SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
         Optional(SECTION_AWS_REGION): str,
@@ -128,12 +154,26 @@ class Config(object):
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
         Optional(SECTION_REMOTE_USER): str,
+        Optional(SECTION_REMOTE_PORT): Use(int),
+        Optional(SECTION_REMOTE_KEY_FILE): str,
+        Optional(SECTION_REMOTE_TIMEOUT): Use(int),
+        Optional(SECTION_REMOTE_PASSWORD): str,
+        Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
+    }
+
+    SECTION_STATE = 'state'
+    SECTION_STATE_ROW_LIMIT = 'row_limit'
+    SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota'
+    SECTION_STATE_SCHEMA = {
+        Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
+        Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
     }
 
     SCHEMA = {
         Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
         Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
         Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
+        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
 
         # backward compatibility
         Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
@@ -154,7 +194,7 @@ class Config(object):
             # need to convert our config to dict before passing it to
             self._config = self._lower(self._config)
             local = self._lower(local)
-            self._config.update(local)
+            self._config = self._merge(self._config, local)
 
             self._config = Schema(self.SCHEMA).validate(self._config)
 
@@ -165,6 +205,17 @@ class Config(object):
         except Exception as ex:
             raise ConfigError(ex)
 
+    @staticmethod
+    def _merge(first, second):
+        res = {}
+        sections = list(first.keys()) + list(second.keys())
+        for section in sections:
+            f = first.get(section, {}).copy()
+            s = second.get(section, {}).copy()
+            f.update(s)
+            res[section] = f
+        return res
+
     @staticmethod
     def _lower(config):
         new_config = {}

dvc/dagascii.py

@@ -0,0 +1,278 @@
+import sys
+import math
+import select
+
+from asciimatics.screen import Screen
+from asciimatics.event import KeyboardEvent
+
+from grandalf.graphs import Vertex, Edge, Graph
+from grandalf.layouts import SugiyamaLayout
+from grandalf.routing import route_with_lines, EdgeViewer
+
+
+class AsciiCanvas(object):
+    def __init__(self, cols, lines):
+        assert cols > 1
+        assert lines > 1
+
+        self.cols = cols
+        self.lines = lines
+
+        self.canvas = [[' '] * cols for l in range(lines)]
+
+    def draw(self):
+        if sys.stdout.isatty():  # pragma: no cover
+            Screen.wrapper(self._do_draw)
+        else:
+            for line in self.canvas:
+                print(''.join(line))
+
+    def _do_draw(self, screen):  # pragma: no cover
+        offset_x = 0
+        offset_y = 0
+        smaxrow, smaxcol = screen.dimensions
+        assert smaxrow > 1
+        assert smaxcol > 1
+        smaxrow -= 1
+        smaxcol -= 1
+
+        if self.lines + 1 > smaxrow:
+            max_y = self.lines + 1 - smaxrow
+        else:
+            max_y = 0
+
+        if self.cols + 1 > smaxcol:
+            max_x = self.cols + 1 - smaxcol
+        else:
+            max_x = 0
+
+        while True:
+            for y in range(smaxrow + 1):
+                y_index = offset_y + y
+                line = []
+                for x in range(smaxcol + 1):
+                    x_index = offset_x + x
+                    if len(self.canvas) > y_index \
+                       and len(self.canvas[y_index]) > x_index:
+                            line.append(self.canvas[y_index][x_index])
+                    else:
+                        line.append(' ')
+                assert len(line) == (smaxcol + 1)
+                screen.print_at(''.join(line), 0, y)
+
+            screen.refresh()
+
+            # NOTE: get_event() doesn't block by itself,
+            # so we have to do the blocking ourselves.
+            select.select([sys.stdin], [], [], None)
+
+            event = screen.get_event()
+            if not isinstance(event, KeyboardEvent):
+                continue
+
+            k = event.key_code
+            if k == screen.KEY_DOWN or k == ord('s'):
+                offset_y += 1
+            elif k == screen.KEY_PAGE_DOWN or k == ord('S'):
+                offset_y += smaxrow
+            elif k == screen.KEY_UP or k == ord('w'):
+                offset_y -= 1
+            elif k == screen.KEY_PAGE_UP or k == ord('W'):
+                offset_y -= smaxrow
+            elif k == screen.KEY_RIGHT or k == ord('d'):
+                offset_x += 1
+            elif k == ord('D'):
+                offset_x += smaxcol
+            elif k == screen.KEY_LEFT or k == ord('a'):
+                offset_x -= 1
+            elif k == ord('A'):
+                offset_x -= smaxcol
+            elif k == ord('q') or k == ord('Q'):
+                break
+
+            if offset_y > max_y:
+                offset_y = max_y
+            elif offset_y < 0:
+                offset_y = 0
+
+            if offset_x > max_x:
+                offset_x = max_x
+            elif offset_x < 0:
+                offset_x = 0
+
+    def point(self, x, y, char):
+        assert x >= 0
+        assert x < self.cols
+        assert y >= 0
+        assert y < self.lines
+
+        self.canvas[y][x] = char
+
+    def line(self, x0, y0, x1, y1, char):
+        if x0 > x1:
+            x1, x0 = x0, x1
+            y1, y0 = y0, y1
+
+        dx = x1 - x0
+        dy = y1 - y0
+
+        if dx == 0 and dy == 0:
+            self.point(x0, y0, char)
+        elif abs(dx) >= abs(dy):
+            for x in range(x0, x1 + 1):
+                if dx == 0:
+                    y = y0
+                else:
+                    y = y0 + int(round((x - x0) * dy / float((dx))))
+                self.point(x, y, char)
+        elif y0 < y1:
+            for y in range(y0, y1 + 1):
+                if dy == 0:
+                    x = x0
+                else:
+                    x = x0 + int(round((y - y0) * dx / float((dy))))
+                self.point(x, y, char)
+        else:
+            for y in range(y1, y0 + 1):
+                if dy == 0:
+                    x = x0
+                else:
+                    x = x1 + int(round((y - y1) * dx / float((dy))))
+                self.point(x, y, char)
+
+    def text(self, x, y, text):
+        for i, c in enumerate(text):
+            self.point(x + i, y, c)
+
+    def box(self, x0, y0, w, h):
+        assert w > 1
+        assert h > 1
+
+        w -= 1
+        h -= 1
+
+        for x in range(x0, x0 + w):
+            self.point(x, y0, '-')
+            self.point(x, y0 + h, '-')
+
+        for y in range(y0, y0 + h):
+            self.point(x0, y, '|')
+            self.point(x0 + w, y, '|')
+
+        self.point(x0, y0, '+')
+        self.point(x0 + w, y0, '+')
+        self.point(x0, y0 + h, '+')
+        self.point(x0 + w, y0 + h, '+')
+
+
+class Dagascii(object):
+    def __init__(self, vertexes, edges):
+        #
+        # Just a reminder about naming conventions:
+        # +------------X
+        # |
+        # |
+        # |
+        # |
+        # Y
+        #
+
+        V = {v: Vertex(" {} ".format(v)) for v in vertexes}
+        # NOTE: reverting edges to correctly orientate the graph
+        E = [Edge(V[e], V[s]) for s, e in edges]
+        V = V.values()
+        g = Graph(V, E)
+
+        class VertexViewer(object):
+            h = 3  # top and bottom box edges + text
+
+            def __init__(self, name):
+                self.w = len(name) + 2  # right and left bottom edges + text
+
+        for v in V:
+            v.view = VertexViewer(v.data)
+
+        # NOTE: determine min box length to create the best layout
+        minw = min([v.view.w for v in V])
+
+        for e in E:
+            e.view = EdgeViewer()
+
+        sug = SugiyamaLayout(g.C[0])
+        gr = g.C[0]
+        r = list(filter(lambda x: len(x.e_in()) == 0, gr.sV))
+
+        sug.init_all(roots=r, optimize=True)
+
+        sug.yspace = VertexViewer.h
+        sug.xspace = minw
+        sug.route_edge = route_with_lines
+
+        sug.draw()
+
+        self.sug = sug
+
+    def draw(self):
+        # NOTE: coordinates might me negative, so we need to shift
+        # everything to the positive plane before we actually draw it.
+        Xs = []
+        Ys = []
+
+        for v in self.sug.g.sV:
+            # NOTE: moving boxes w/2 to the left
+            Xs.append(v.view.xy[0] - v.view.w/2.0)
+            Xs.append(v.view.xy[0] + v.view.w/2.0)
+            Ys.append(v.view.xy[1])
+            Ys.append(v.view.xy[1] + v.view.h)
+
+        for e in self.sug.g.sE:
+            for xy in e.view._pts:
+                Xs.append(xy[0])
+                Ys.append(xy[1])
+
+        minx = min(Xs)
+        miny = min(Ys)
+        maxx = max(Xs)
+        maxy = max(Ys)
+
+        canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
+        canvas_lines = int(round(maxy - miny))
+
+        canvas = AsciiCanvas(canvas_cols, canvas_lines)
+
+        # NOTE: first draw edges so that node boxes could overwrite them
+        for e in self.sug.g.sE:
+            assert len(e.view._pts) > 1
+            for index in range(1, len(e.view._pts)):
+                start = e.view._pts[index - 1]
+                end = e.view._pts[index]
+
+                start_x = int(round(start[0] - minx))
+                start_y = int(round(start[1] - miny))
+                end_x = int(round(end[0] - minx))
+                end_y = int(round(end[1] - miny))
+
+                assert start_x >= 0
+                assert start_y >= 0
+                assert end_x >= 0
+                assert end_y >= 0
+
+                canvas.line(start_x, start_y,
+                            end_x, end_y,
+                            '*')
+
+        for v in self.sug.g.sV:
+            # NOTE: moving boxes w/2 to the left
+            x = v.view.xy[0] - v.view.w/2.0
+            y = v.view.xy[1]
+
+            canvas.box(int(round(x - minx)),
+                       int(round(y - miny)),
+                       v.view.w,
+                       v.view.h)
+
+            canvas.text(int(round(x - minx)) + 1,
+                        int(round(y - miny)) + 1,
+                        v.data)
+
+        canvas.draw()

dvc/data_cloud.py

@@ -83,15 +83,18 @@ class DataCloud(object):
         cloud = cloud_type(self.project, cloud_config)
         return cloud
 
-    def _get_cloud(self, remote):
+    def _get_cloud(self, remote, cmd):
         if remote:
             return self._init_remote(remote)
 
         if self._cloud:
             return self._cloud
 
-        msg = "No remote repository specified. Setup default repository " \
-              "with 'dvc config core.remote <name>' or use '-r <name>'."
+        msg = "No remote repository specified. Setup default " \
+              "repository with\n"\
+              "    dvc config core.remote <name>\n" \
+              "or use:\n" \
+              "    dvc {} -r <name>".format(cmd)
         raise ConfigError(msg)
 
     def push(self, targets, jobs=1, remote=None, show_checksums=False):
@@ -100,7 +103,8 @@ class DataCloud(object):
         """
         return self.project.cache.local.push(targets,
                                              jobs=jobs,
-                                             remote=self._get_cloud(remote),
+                                             remote=self._get_cloud(remote,
+                                                                    'push'),
                                              show_checksums=show_checksums)
 
     def pull(self, targets, jobs=1, remote=None, show_checksums=False):
@@ -109,14 +113,16 @@ class DataCloud(object):
         """
         return self.project.cache.local.pull(targets,
                                              jobs=jobs,
-                                             remote=self._get_cloud(remote),
+                                             remote=self._get_cloud(remote,
+                                                                    'pull'),
                                              show_checksums=show_checksums)
 
     def status(self, targets, jobs=1, remote=None, show_checksums=False):
         """
         Check status of data items in a cloud-agnostic way.
         """
+        cloud = self._get_cloud(remote, 'status')
         return self.project.cache.local.status(targets,
                                                jobs=jobs,
-                                               remote=self._get_cloud(remote),
+                                               remote=cloud,
                                                show_checksums=show_checksums)

dvc/dependency/base.py

@@ -31,6 +31,9 @@ class DependencyBase(object):
         self.project = stage.project
         self.path = path
 
+    def __str__(self):
+        return self.path
+
     @classmethod
     def match(cls, url):
         return re.match(cls.REGEX, url)
@@ -51,7 +54,7 @@ class DependencyBase(object):
 
     @property
     def exists(self):
-        return self.remote.exists([self.path_info])
+        return self.remote.exists([self.path_info])[0]
 
     def changed(self):
         raise NotImplementedError
@@ -59,7 +62,11 @@ class DependencyBase(object):
     def status(self):
         if self.changed():
             # FIXME better msgs
-            return {self.rel_path: 'changed'}
+            if self.path_info['scheme'] == 'local':
+                p = self.rel_path
+            else:
+                p = self.path
+            return {p: 'changed'}
         return {}
 
     def save(self):

dvc/dependency/local.py

@@ -38,6 +38,15 @@ class DependencyLOCAL(DependencyBase):
         self.path_info = {'scheme': 'local',
                           'path': self.path}
 
+    def __str__(self):
+        return self.rel_path
+
+    @property
+    def is_local(self):
+        assert os.path.isabs(self.path)
+        assert os.path.isabs(self.project.root_dir)
+        return self.path.startswith(self.project.root_dir)
+
     @property
     def sep(self):
         return os.sep
@@ -58,7 +67,8 @@ class DependencyLOCAL(DependencyBase):
         if not self.exists:
             raise self.DoesNotExistError(self.rel_path)
 
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
+        if not os.path.isfile(self.path) \
+           and not os.path.isdir(self.path):  # pragma: no cover
             raise self.IsNotFileOrDirError(self.rel_path)
 
         if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
@@ -68,7 +78,7 @@ class DependencyLOCAL(DependencyBase):
         self.info = self.remote.save_info(self.path_info)
 
     def dumpd(self):
-        if self.path.startswith(self.stage.project.root_dir):
+        if self.is_local:
             path = self.remote.unixpath(os.path.relpath(self.path,
                                                         self.stage.cwd))
         else:

dvc/dependency/ssh.py

@@ -18,6 +18,7 @@ class DependencySSH(DependencyBase):
         self.remote = remote if remote else RemoteSSH(stage.project, {})
 
         host = remote.host if remote else self.group('host')
+        port = remote.port if remote else RemoteSSH.DEFAULT_PORT
         user = remote.user if remote else self.group('user')
         if remote:
             path = posixpath.join(remote.prefix,
@@ -27,6 +28,7 @@ class DependencySSH(DependencyBase):
 
         self.path_info = {'scheme': 'ssh',
                           'host': host,
+                          'port': port,
                           'user': user,
                           'path': path}
 

dvc/exceptions.py

@@ -8,7 +8,7 @@ class DvcException(Exception):
         if cause:
             try:
                 self.cause_tb = traceback.format_exc()
-            except Exception:
+            except Exception:  # pragma: no cover
                 pass
         super(DvcException, self).__init__(msg)
 
@@ -24,6 +24,17 @@ class OutputDuplicationError(DvcException):
         assert isinstance(output, str)
         assert isinstance(stages, list)
         assert all(isinstance(stage, str) for stage in stages)
-        msg = "File '{}' is specified as an output in more than one stage: {}"
-        super(OutputDuplicationError, self).__init__(msg.format(output,
-                                                                str(stages)))
+        msg = "File '{}' is specified as an output in more than one stage:{}"
+        s = ""
+        for stage in stages:
+            s += "\n    " + stage
+        super(OutputDuplicationError, self).__init__(msg.format(output, s))
+
+
+class MoveNotDataSourceError(DvcException):
+    def __init__(self, path):
+        msg = "Move is not permitted for stages that are not data sources. " \
+              "You need to either move '{path}' to a new location and edit " \
+              "it by hand, or remove '{path}' and create a new one at the " \
+              "desired location."
+        super(MoveNotDataSourceError, self).__init__(msg.format(path=path))

dvc/logger.py

@@ -82,22 +82,41 @@ class Logger(object):
 
         return (str_exc, str_tb)
 
+    @staticmethod
+    def _prefix(msg, typ):
+        return Logger.colorize('{}: '.format(msg), typ)
+
+    @staticmethod
+    def error_prefix():
+        return Logger._prefix('Error', 'error')
+
+    @staticmethod
+    def warning_prefix():
+        return Logger._prefix('Warning', 'warn')
+
+    @staticmethod
+    def debug_prefix():
+        return Logger._prefix('Debug', 'debug')
+
     @staticmethod
     def error(msg, exc=None):
+        prefix = Logger.error_prefix()
         str_exc, str_tb = Logger.parse_exc(exc)
         if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
             str_tb = str_tb if str_tb else traceback.format_exc()
-            Logger.logger().error(str_tb)
-        return Logger.logger().error(Logger.colorize(msg + str_exc, 'error'))
+            Logger.logger().error(prefix + str_tb)
+        chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \
+               "we are always happy to help!"
+        return Logger.logger().error(prefix + msg + str_exc + chat)
 
     @staticmethod
     def warn(msg):
-        return Logger.logger().warn(Logger.colorize(msg, 'warn'))
+        return Logger.logger().warn(Logger.warning_prefix() + msg)
 
     @staticmethod
     def debug(msg):
-        return Logger.logger().debug(Logger.colorize(msg, 'debug'))
+        return Logger.logger().debug(Logger.debug_prefix() + msg)
 
     @staticmethod
     def info(msg):
-        return Logger.logger().info(Logger.colorize(msg, 'info'))
+        return Logger.logger().info(msg)

dvc/output/gs.py

@@ -30,7 +30,7 @@ class OutputGS(DependencyGS):
             return True
 
         if self.use_cache \
-           and self.info != self.project.cache.gs.save_info(self.path_info):
+           and self.project.cache.gs.changed(self.path_info, self.info):
             return True
 
         return False

dvc/output/hdfs.py

@@ -30,7 +30,7 @@ class OutputHDFS(DependencyHDFS):
             return True
 
         if self.use_cache \
-           and self.info != self.project.cache.hdfs.save_info(self.path_info):
+           and self.project.cache.hdfs.changed(self.path_info, self.info):
             return True
 
         return False

dvc/output/local.py

@@ -40,10 +40,6 @@ class OutputLOCAL(DependencyLOCAL):
     def cache(self):
         return self.project.cache.local.get(self.md5)
 
-    @property
-    def is_local(self):
-        return self.path.startswith(self.project.root_dir)
-
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
         ret[self.PARAM_CACHE] = self.use_cache
@@ -62,17 +58,12 @@ class OutputLOCAL(DependencyLOCAL):
         if not self.use_cache:
             return super(OutputLOCAL, self).changed()
 
-        return self.info != self.project.cache.local.save_info(self.path_info)
+        return self.project.cache.local.changed(self.path_info, self.info)
 
     def checkout(self):
         if not self.use_cache:
             return
 
-        if not self.changed():
-            msg = u'Data file \'{}\' didn\'t change, skipping checkout.'
-            self.project.logger.debug(msg.format(self.rel_path))
-            return
-
         self.project.cache.local.checkout(self.path_info, self.info)
 
     def _verify_metric(self):
@@ -92,13 +83,14 @@ class OutputLOCAL(DependencyLOCAL):
             super(OutputLOCAL, self).save()
             self._verify_metric()
             msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
-            self.project.logger.debug(msg.format(self.rel_path))
+            self.project.logger.info(msg.format(self.rel_path))
             return
 
         if not os.path.exists(self.path):
             raise self.DoesNotExistError(self.rel_path)
 
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
+        if not os.path.isfile(self.path) \
+           and not os.path.isdir(self.path):  # pragma: no cover
             raise self.IsNotFileOrDirError(self.rel_path)
 
         if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
@@ -108,7 +100,7 @@ class OutputLOCAL(DependencyLOCAL):
 
         if not self.changed():
             msg = 'Output \'{}\' didn\'t change. Skipping saving.'
-            self.project.logger.debug(msg.format(self.rel_path))
+            self.project.logger.info(msg.format(self.rel_path))
             return
 
         if self.is_local:

dvc/output/s3.py

@@ -30,7 +30,7 @@ class OutputS3(DependencyS3):
             return True
 
         if self.use_cache \
-           and self.info != self.project.cache.s3.save_info(self.path_info):
+           and self.project.cache.s3.changed(self.path_info, self.info):
             return True
 
         return False

dvc/output/ssh.py

@@ -30,7 +30,7 @@ class OutputSSH(DependencySSH):
             return True
 
         if self.use_cache \
-           and self.info != self.project.cache.ssh.save_info(self.path_info):
+           and self.project.cache.ssh.changed(self.path_info, self.info):
             return True
 
         return False

dvc/progress.py

@@ -16,6 +16,10 @@ class Progress(object):
         self._n_total = total
         self._n_finished = 0
 
+    @property
+    def is_finished(self):
+        return self._n_total == self._n_finished
+
     def _clearln(self):
         print('\r\x1b[K', end='')
 
@@ -63,7 +67,7 @@ class Progress(object):
             progress = int((100 * current)/total) if current < total else 100
             percent = str(progress) + "% "
 
-        if self._n_total:
+        if self._n_total > 1:
             num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
         else:
             num = ""

dvc/project.py

@@ -1,6 +1,6 @@
 import os
 
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, MoveNotDataSourceError
 from dvc.stage import Stage
 
 
@@ -9,15 +9,6 @@ class InitError(DvcException):
         super(InitError, self).__init__(msg)
 
 
-class NotDvcFileError(DvcException):
-    def __init__(self, path):
-        msg = '\'{}\' is not a DVC file'.format(path)
-        p = path + Stage.STAGE_FILE_SUFFIX
-        if Stage.is_stage_file(p):
-            msg += '. Maybe you meant \'{}\'?'.format(p)
-        super(NotDvcFileError, self).__init__(msg)
-
-
 class ReproductionError(DvcException):
     def __init__(self, dvc_file_name, ex):
         self.path = dvc_file_name
@@ -31,23 +22,23 @@ class Project(object):
     def __init__(self, root_dir):
         from dvc.logger import Logger
         from dvc.config import Config
-        from dvc.state import LinkState, State
+        from dvc.state import State
         from dvc.lock import Lock
         from dvc.scm import SCM
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
         from dvc.updater import Updater
+        from dvc.prompt import Prompt
 
         self.root_dir = os.path.abspath(os.path.realpath(root_dir))
         self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
 
         self.config = Config(self.dvc_dir)
-        self.scm = SCM(self.root_dir)
+        self.scm = SCM(self.root_dir, project=self)
         self.lock = Lock(self.dvc_dir)
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
-        self.state = State(self)
-        self.link_state = LinkState(self)
+        self.state = State(self, self.config._config)
 
         core = self.config._config[Config.SECTION_CORE]
         self.logger = Logger(core.get(Config.SECTION_CORE_LOGLEVEL, None))
@@ -55,11 +46,23 @@ class Project(object):
         self.cache = Cache(self)
         self.cloud = DataCloud(self, config=self.config._config)
         self.updater = Updater(self.dvc_dir)
+        self.prompt = Prompt()
+
+        self._files_to_git_add = []
 
         self._ignore()
 
         self.updater.check()
 
+    def _remind_to_git_add(self):
+        if len(self._files_to_git_add) == 0:
+            return
+
+        msg = '\nTo track the changes with git run:\n\n'
+        msg += '\tgit add ' + " ".join(self._files_to_git_add)
+
+        self.logger.info(msg)
+
     @staticmethod
     def init(root_dir=os.curdir, no_scm=False, force=False):
         """
@@ -77,10 +80,10 @@ class Project(object):
         import shutil
         from dvc.scm import SCM, Base
         from dvc.config import Config
+        from dvc.logger import Logger
 
         root_dir = os.path.abspath(root_dir)
         dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
-
         scm = SCM(root_dir)
         if type(scm) == Base and not no_scm:
             msg = "{} is not tracked by any supported scm tool(e.g. git)."
@@ -101,6 +104,8 @@ class Project(object):
         if scm.ignore_file():
             scm.add([os.path.join(dvc_dir, scm.ignore_file())])
 
+        Logger.info('You can now commit the changes to git.')
+
         return proj
 
     def destroy(self):
@@ -113,9 +118,6 @@ class Project(object):
 
     def _ignore(self):
         flist = [self.state.state_file,
-                 self.state._lock_file.lock_file,
-                 self.link_state.state_file,
-                 self.link_state._lock_file.lock_file,
                  self.lock.lock_file,
                  self.config.config_local_file,
                  self.updater.updater_file]
@@ -131,19 +133,52 @@ class Project(object):
     def to_dvc_path(self, path):
         return os.path.relpath(path, self.root_dir)
 
-    def add(self, fname):
-        stage = Stage.loads(project=self,
-                            outs=[fname],
-                            add=True)
+    def _check_output_duplication(self, outs):
+        from dvc.exceptions import OutputDuplicationError
 
-        stage.save()
-        stage.dump()
-        return stage
+        for stage in self.stages():
+            for o in stage.outs:
+                for out in outs:
+                    if o.path == out.path and o.stage.path != out.stage.path:
+                        stages = [o.stage.relpath, out.stage.relpath]
+                        raise OutputDuplicationError(o.path, stages)
+
+    def add(self, fname, recursive=False):
+        fnames = []
+        if recursive and os.path.isdir(fname):
+            fnames = []
+            for root, dirs, files in os.walk(fname):
+                for f in files:
+                    path = os.path.join(root, f)
+                    if Stage.is_stage_file(path):
+                        continue
+                    if os.path.basename(path) == self.scm.ignore_file():
+                        continue
+                    if self.scm.is_tracked(path):
+                        continue
+                    fnames.append(path)
+        else:
+            fnames = [fname]
+
+        stages = []
+        self._files_to_git_add = []
+        with self.state:
+            for f in fnames:
+                stage = Stage.loads(project=self,
+                                    outs=[f],
+                                    add=True)
+
+                self._check_output_duplication(stage.outs)
+
+                stage.save()
+                stage.dump()
+                stages.append(stage)
+
+        self._remind_to_git_add()
+
+        return stages
 
     def remove(self, target, outs_only=False):
-        if not Stage.is_stage_file(target):
-            raise NotDvcFileError(target)
-
         stage = Stage.load(self, target)
         if outs_only:
             stage.remove_outs()
@@ -153,9 +188,6 @@ class Project(object):
         return stage
 
     def lock_stage(self, target, unlock=False):
-        if not Stage.is_stage_file(target):
-            raise NotDvcFileError(target)
-
         stage = Stage.load(self, target)
         stage.locked = False if unlock else True
         stage.dump()
@@ -169,34 +201,37 @@ class Project(object):
                                      [from_path])[0]
 
         found = False
-        for stage in self.stages():
-            for out in stage.outs:
-                if out.path != from_out.path:
-                    continue
+        self._files_to_git_add = []
+        with self.state:
+            for stage in self.stages():
+                for out in stage.outs:
+                    if out.path != from_out.path:
+                        continue
 
-                if not stage.is_data_source:
-                    msg = 'Dvcfile \'{}\' is not a data source.'
-                    raise DvcException(msg.format(stage.rel_path))
+                    if not stage.is_data_source:
+                        raise MoveNotDataSourceError(stage.relpath)
 
-                found = True
-                to_out = Output.loads_from(out.stage,
-                                           [to_path],
-                                           out.cache,
-                                           out.metric)[0]
-                out.move(to_out)
+                    found = True
+                    to_out = Output.loads_from(out.stage,
+                                               [to_path],
+                                               out.cache,
+                                               out.metric)[0]
+                    out.move(to_out)
 
-                stage_base = os.path.basename(stage.path)
-                stage_base = stage_base.rstrip(Stage.STAGE_FILE_SUFFIX)
+                    stage_base = os.path.basename(stage.path)
+                    stage_base = stage_base.rstrip(Stage.STAGE_FILE_SUFFIX)
 
-                stage_dir = os.path.dirname(stage.path)
-                from_base = os.path.basename(from_path)
-                to_base = os.path.basename(to_path)
-                if stage_base == from_base:
-                    os.unlink(stage.path)
-                    path = to_base + Stage.STAGE_FILE_SUFFIX
-                    stage.path = os.path.join(stage_dir, path)
+                    stage_dir = os.path.dirname(stage.path)
+                    from_base = os.path.basename(from_path)
+                    to_base = os.path.basename(to_path)
+                    if stage_base == from_base:
+                        os.unlink(stage.path)
+                        path = to_base + Stage.STAGE_FILE_SUFFIX
+                        stage.path = os.path.join(stage_dir, path)
 
-            stage.dump()
+                stage.dump()
+
+        self._remind_to_git_add()
 
         if not found:
             msg = 'Unable to find dvcfile with output \'{}\''
@@ -210,7 +245,8 @@ class Project(object):
             metrics_no_cache=[],
             fname=Stage.STAGE_FILE,
             cwd=os.curdir,
-            no_exec=False):
+            no_exec=False,
+            overwrite=False):
         stage = Stage.loads(project=self,
                             fname=fname,
                             cmd=cmd,
@@ -219,10 +255,19 @@ class Project(object):
                             outs_no_cache=outs_no_cache,
                             metrics_no_cache=metrics_no_cache,
                             deps=deps,
-                            overwrite=False)
-        if not no_exec:
-            stage.run()
+                            overwrite=overwrite)
+
+        self._check_output_duplication(stage.outs)
+
+        self._files_to_git_add = []
+        with self.state:
+            if not no_exec:
+                stage.run()
+
         stage.dump()
+
+        self._remind_to_git_add()
+
         return stage
 
     def imp(self, url, out):
@@ -231,15 +276,23 @@ class Project(object):
                             deps=[url],
                             outs=[out])
 
-        stage.run()
+        self._check_output_duplication(stage.outs)
+
+        self._files_to_git_add = []
+        with self.state:
+            stage.run()
+
         stage.dump()
+
+        self._remind_to_git_add()
+
         return stage
 
     def _reproduce_stage(self, stages, node, force, dry, interactive):
         stage = stages[node]
 
         if stage.locked:
-            msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
+            msg = 'DVC file \'{}\' is locked. Its dependencies are not ' \
                   'going to be reproduced.'
             self.logger.warn(msg.format(stage.relpath))
 
@@ -253,28 +306,82 @@ class Project(object):
         return [stage]
 
     def reproduce(self,
-                  target,
+                  target=None,
                   recursive=True,
                   force=False,
                   dry=False,
-                  interactive=False):
+                  interactive=False,
+                  pipeline=False,
+                  all_pipelines=False):
+
+        if target is None and not all_pipelines:
+            raise ValueError()
+
+        if not interactive:
+            config = self.config
+            core = config._config[config.SECTION_CORE]
+            interactive = core.get(config.SECTION_CORE_INTERACTIVE, False)
+
+        targets = []
+        if pipeline or all_pipelines:
+            if pipeline:
+                stage = Stage.load(self, target)
+                node = os.path.relpath(stage.path, self.root_dir)
+                pipelines = [self._get_pipeline(node)]
+            else:
+                pipelines = self.pipelines()
+
+            for G in pipelines:
+                for node in G.nodes():
+                    if G.in_degree(node) == 0:
+                        targets.append(os.path.join(self.root_dir, node))
+        else:
+            targets.append(target)
+
+        self._files_to_git_add = []
+
+        ret = []
+        with self.state:
+            for target in targets:
+                stages = self._reproduce(target,
+                                         recursive=recursive,
+                                         force=force,
+                                         dry=dry,
+                                         interactive=interactive)
+                ret.extend(stages)
+
+        self._remind_to_git_add()
+
+        return ret
+
+    def _reproduce(self,
+                   target,
+                   recursive=True,
+                   force=False,
+                   dry=False,
+                   interactive=False):
         import networkx as nx
 
+        stage = Stage.load(self, target)
         G = self.graph()[1]
         stages = nx.get_node_attributes(G, 'stage')
-        node = os.path.relpath(os.path.abspath(target), self.root_dir)
-        if node not in stages:
-            raise NotDvcFileError(target)
+        node = os.path.relpath(stage.path, self.root_dir)
 
         if recursive:
-            return self._reproduce_stages(G,
-                                          stages,
-                                          node,
-                                          force,
-                                          dry,
-                                          interactive)
+            ret = self._reproduce_stages(G,
+                                         stages,
+                                         node,
+                                         force,
+                                         dry,
+                                         interactive)
+        else:
+            ret = self._reproduce_stage(stages,
+                                        node,
+                                        force,
+                                        dry,
+                                        interactive)
 
-        return self._reproduce_stage(stages, node, force, dry, interactive)
+        return ret
 
     def _reproduce_stages(self, G, stages, node, force, dry, interactive):
         import networkx as nx
@@ -296,29 +403,56 @@ class Project(object):
         for stage in all_stages:
             for out in stage.outs:
                 used.append(out.path)
-        self.link_state.remove_unused(used)
+        self.state.remove_unused_links(used)
 
     def checkout(self, target=None):
         all_stages = self.active_stages()
 
         if target:
-            if not Stage.is_stage_file(target):
-                raise NotDvcFileError(target)
             stages = [Stage.load(self, target)]
         else:
             stages = all_stages
 
-        self._cleanup_unused_links(all_stages)
+        with self.state:
+            self._cleanup_unused_links(all_stages)
 
-        for stage in stages:
-            if stage.locked:
-                msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
-                      'going to be checked out.'
-                self.logger.warn(msg.format(stage.relpath))
+            for stage in stages:
+                if stage.locked:
+                    msg = 'DVC file \'{}\' is locked. Its dependencies are ' \
+                          'not going to be checked out.'
+                    self.logger.warn(msg.format(stage.relpath))
 
-            stage.checkout()
+                stage.checkout()
 
-    def _used_cache(self, target=None, all_branches=False, active=True):
+    def _get_pipeline(self, node):
+        pipelines = list(filter(lambda g: node in g.nodes(),
+                                self.pipelines()))
+        assert len(pipelines) == 1
+        return pipelines[0]
+
+    def _collect(self, target, with_deps=False):
+        import networkx as nx
+
+        stage = Stage.load(self, target)
+        if not with_deps:
+            return [stage]
+
+        node = os.path.relpath(stage.path, self.root_dir)
+        G = self._get_pipeline(node)
+        stages = nx.get_node_attributes(G, 'stage')
+
+        ret = [stage]
+        for n in nx.dfs_postorder_nodes(G, node):
+            ret.append(stages[n])
+
+        return ret
+
+    def _used_cache(self,
+                    target=None,
+                    all_branches=False,
+                    active=True,
+                    with_deps=False,
+                    all_tags=False):
         cache = {}
         cache['local'] = []
         cache['s3'] = []
@@ -326,9 +460,11 @@ class Project(object):
         cache['hdfs'] = []
         cache['ssh'] = []
 
-        for branch in self.scm.brancher(all_branches=all_branches):
+        for branch in self.scm.brancher(all_branches=all_branches,
+                                        all_tags=all_tags):
             if target:
-                stages = [Stage.load(self, target)]
+                stages = self._collect(target,
+                                       with_deps=with_deps)
             elif active:
                 stages = self.active_stages()
             else:
@@ -336,7 +472,7 @@ class Project(object):
 
             for stage in stages:
                 if active and not target and stage.locked:
-                    msg = 'DVC file \'{}\' is locked. Its dependecies are ' \
+                    msg = 'DVC file \'{}\' is locked. Its dependencies are ' \
                           'not going to be pushed/pulled/fetched.'
                     self.logger.warn(msg.format(stage.relpath))
 
@@ -351,63 +487,95 @@ class Project(object):
 
         return cache
 
-    def gc(self, all_branches=False, cloud=False, remote=None):
-        clist = self._used_cache(target=None,
-                                 all_branches=all_branches,
-                                 active=False)
-        self.cache.local.gc(clist)
+    def _do_gc(self, typ, func, clist):
+        removed = func(clist)
+        if not removed:
+            self.logger.info("No unused {} cache to remove.".format(typ))
 
-        if self.cache.s3:
-            self.cache.s3.gc(clist)
+    def gc(self,
+           all_branches=False,
+           cloud=False,
+           remote=None,
+           with_deps=False,
+           all_tags=False):
+        with self.state:
+            clist = self._used_cache(target=None,
+                                     all_branches=all_branches,
+                                     active=False,
+                                     with_deps=with_deps,
+                                     all_tags=all_tags)
+            self._do_gc('local', self.cache.local.gc, clist)
 
-        if self.cache.gs:
-            self.cache.gs.gc(clist)
+            if self.cache.s3:
+                self._do_gc('s3', self.cache.s3.gc, clist)
 
-        if self.cache.ssh:
-            self.cache.ssh.gc(clist)
+            if self.cache.gs:
+                self._do_gc('gs', self.cache.gs.gc, clist)
 
-        if self.cache.hdfs:
-            self.cache.hdfs.gc(clist)
+            if self.cache.ssh:
+                self._do_gc('ssh', self.cache.ssh.gc, clist)
 
-        if self.cache.azure:
-            self.cache.azure.gc(clist)
+            if self.cache.hdfs:
+                self._do_gc('hdfs', self.cache.hdfs.gc, clist)
 
-        if cloud:
-            self.cloud._get_cloud(remote).gc(clist)
+            if self.cache.azure:
+                self._do_gc('azure', self.cache.azure.gc, clist)
+
+            if cloud:
+                self._do_gc('remote', self.cloud._get_cloud(remote,
+                                                            'gc -c').gc, clist)
 
     def push(self,
              target=None,
              jobs=1,
              remote=None,
              all_branches=False,
-             show_checksums=False):
-        self.cloud.push(self._used_cache(target, all_branches)['local'],
-                        jobs,
-                        remote=remote,
-                        show_checksums=show_checksums)
+             show_checksums=False,
+             with_deps=False,
+             all_tags=False):
+        with self.state:
+            used = self._used_cache(target,
+                                    all_branches=all_branches,
+                                    all_tags=all_tags,
+                                    with_deps=with_deps)['local']
+            self.cloud.push(used,
+                            jobs,
+                            remote=remote,
+                            show_checksums=show_checksums)
 
     def fetch(self,
               target=None,
               jobs=1,
               remote=None,
               all_branches=False,
-              show_checksums=False):
-        self.cloud.pull(self._used_cache(target, all_branches)['local'],
-                        jobs,
-                        remote=remote,
-                        show_checksums=show_checksums)
+              show_checksums=False,
+              with_deps=False,
+              all_tags=False):
+        with self.state:
+            used = self._used_cache(target,
+                                    all_branches=all_branches,
+                                    all_tags=all_tags,
+                                    with_deps=with_deps)['local']
+            self.cloud.pull(used,
+                            jobs,
+                            remote=remote,
+                            show_checksums=show_checksums)
 
     def pull(self,
              target=None,
              jobs=1,
              remote=None,
              all_branches=False,
-             show_checksums=False):
+             show_checksums=False,
+             with_deps=False,
+             all_tags=False):
         self.fetch(target,
                    jobs,
                    remote=remote,
                    all_branches=all_branches,
-                   show_checksums=show_checksums)
+                   all_tags=all_tags,
+                   show_checksums=show_checksums,
+                   with_deps=with_deps)
         self.checkout(target=target)
 
     def _local_status(self, target=None):
@@ -420,8 +588,8 @@ class Project(object):
 
         for stage in stages:
             if stage.locked:
-                msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
-                      'going to be shown in status output.'
+                msg = 'DVC file \'{}\' is locked. Its dependencies are not ' \
+                      'going to be shown in the status output.'
                 self.logger.warn(msg.format(stage.relpath))
 
             status.update(stage.status())
@@ -432,11 +600,19 @@ class Project(object):
                       target=None,
                       jobs=1,
                       remote=None,
-                      show_checksums=False):
+                      show_checksums=False,
+                      all_branches=False,
+                      with_deps=False,
+                      all_tags=False):
         import dvc.remote.base as cloud
 
+        used = self._used_cache(target,
+                                all_branches=all_branches,
+                                all_tags=all_tags,
+                                with_deps=with_deps)['local']
+
         status = {}
-        for md5, ret in self.cloud.status(self._used_cache(target)['local'],
+        for md5, ret in self.cloud.status(used,
                                           jobs,
                                           remote=remote,
                                           show_checksums=show_checksums):
@@ -457,13 +633,20 @@ class Project(object):
                jobs=1,
                cloud=False,
                remote=None,
-               show_checksums=False):
-        if cloud:
-            return self._cloud_status(target,
-                                      jobs,
-                                      remote=remote,
-                                      show_checksums=show_checksums)
-        return self._local_status(target)
+               show_checksums=False,
+               all_branches=False,
+               with_deps=False,
+               all_tags=False):
+        with self.state:
+            if cloud:
+                return self._cloud_status(target,
+                                          jobs,
+                                          remote=remote,
+                                          show_checksums=show_checksums,
+                                          all_branches=all_branches,
+                                          with_deps=with_deps,
+                                          all_tags=all_tags)
+            return self._local_status(target)
 
     def _read_metric_json(self, fd, json_path):
         import json
@@ -533,7 +716,7 @@ class Project(object):
 
         abs_path = os.path.abspath(path)
         matched = [out for out in outs if out.path == abs_path]
-        stages = [out.stage.path for out in matched]
+        stages = [out.stage.relpath for out in matched]
         if len(stages) > 1:
             raise OutputDuplicationError(path, stages)
 
@@ -543,9 +726,11 @@ class Project(object):
                      path=None,
                      typ=None,
                      xpath=None,
-                     all_branches=False):
+                     all_branches=False,
+                     all_tags=False):
         res = {}
-        for branch in self.scm.brancher(all_branches=all_branches):
+        for branch in self.scm.brancher(all_branches=all_branches,
+                                        all_tags=all_tags):
             astages = self.active_stages()
             outs = [out for stage in astages for out in stage.outs]
 
@@ -566,8 +751,8 @@ class Project(object):
                 entries = []
                 for o in metrics:
                     if not typ and isinstance(o.metric, dict):
-                        t = o.metric.get(o.PARAM_METRIC_TYPE, None)
-                        x = o.metric.get(o.PARAM_METRIC_XPATH, None)
+                        t = o.metric.get(o.PARAM_METRIC_TYPE, typ)
+                        x = o.metric.get(o.PARAM_METRIC_XPATH, xpath)
                     else:
                         t = typ
                         x = xpath
@@ -590,7 +775,7 @@ class Project(object):
                 res[branch][rel] = metric
 
         for branch, val in res.items():
-            if all_branches:
+            if all_branches or all_tags:
                 self.logger.info('{}:'.format(branch))
             for fname, metric in val.items():
                 self.logger.info('\t{}: {}'.format(fname, metric))
@@ -649,14 +834,22 @@ class Project(object):
 
     def graph(self):
         import networkx as nx
+        from dvc.exceptions import OutputDuplicationError
 
         G = nx.DiGraph()
         G_active = nx.DiGraph()
         stages = self.stages()
 
         outs = []
+        outs_by_path = {}
         for stage in stages:
-            outs += stage.outs
+            for o in stage.outs:
+                existing = outs_by_path.get(o.path, None)
+                if existing is not None:
+                    stages = [o.stage.relpath, existing.stage.relpath]
+                    raise OutputDuplicationError(o.path, stages)
+                outs.append(o)
+                outs_by_path[o.path] = o
 
         # collect the whole DAG
         for stage in stages:
@@ -708,12 +901,28 @@ class Project(object):
 
     def stages(self):
         stages = []
+        outs = []
         for root, dirs, files in os.walk(self.root_dir):
             for fname in files:
                 path = os.path.join(root, fname)
                 if not Stage.is_stage_file(path):
                     continue
-                stages.append(Stage.load(self, path))
+                stage = Stage.load(self, path)
+                for out in stage.outs:
+                    outs.append(out.path + out.sep)
+                stages.append(stage)
+
+            def filter_dirs(dname):
+                path = os.path.join(root, dname)
+                if path == self.dvc_dir or path == self.scm.dir:
+                    return False
+                for out in outs:
+                    if path == os.path.normpath(out) or path.startswith(out):
+                        return False
+                return True
+
+            dirs[:] = list(filter(filter_dirs, dirs))
+
         return stages
 
     def active_stages(self):

dvc/prompt.py

@@ -1,5 +1,7 @@
 import sys
 
+from dvc.progress import progress
+
 try:
     # NOTE: in Python3 raw_input() was renamed to input()
     input = raw_input
@@ -7,12 +9,32 @@ except NameError:
     pass
 
 
-def prompt(msg, default=False):
-    if not sys.stdout.isatty():
-        return default
+class Prompt(object):
+    def __init__(self):
+        self.default = None
 
-    answer = input(msg + u'\n').lower()
-    while answer not in ['yes', 'no', 'y', 'n']:
-        answer = input('Enter \'yes\' or \'no\'.\n').lower()
+    def prompt(self, msg, default=False):  # pragma: no cover
+        if self.default is not None:
+            return self.default
 
-    return answer[0] == "y"
+        if not sys.stdout.isatty():
+            return default
+
+        answer = input(msg + u' (y/n)\n').lower()
+        while answer not in ['yes', 'no', 'y', 'n']:
+            answer = input('Enter \'yes\' or \'no\'.\n').lower()
+
+        return answer[0] == "y"
+
+    def prompt_password(self, msg):  # pragma: no cover
+        import getpass
+
+        if not sys.stdout.isatty():
+            return None
+
+        msg = 'Enter password for {}:\n'.format(msg)
+
+        if not progress.is_finished:
+            msg = u'\n' + msg
+
+        return getpass.getpass(msg)

dvc/remote/azure.py

@@ -92,6 +92,9 @@ class RemoteAzure(RemoteBase):
 #
 #        return {self.PARAM_ETAG: etag}
 #
+#    def changed(self, path_info, checksum_info):
+#        return checksum_info != self.save_info(path_info)
+#
 #    def _copy_blob(self, to_bucket, to_key, from_bucket, from_key):
 #        source = self.blob_service.make_blob_url(from_bucket, from_key)
 #
@@ -227,6 +230,7 @@ class RemoteAzure(RemoteBase):
 #
 #        all_blobs = self.blob_service.list_blobs(self.bucket)
 #
+#        removed = False
 #        for blob in all_blobs:
 #            etag = blob.properties.etag
 #            if etag in used:
@@ -235,3 +239,6 @@ class RemoteAzure(RemoteBase):
 #                         'key': blob.name,
 #                         'bucket': self.bucket}
 #            self.remove(path_info)
+#            removed = True
+#
+#        return removed

dvc/remote/base.py

@@ -1,6 +1,7 @@
 import os
 import re
 import errno
+from multiprocessing import cpu_count
 
 from dvc.config import Config
 from dvc.logger import Logger
@@ -30,6 +31,7 @@ class DataCloudError(DvcException):
 class RemoteBase(object):
     REGEX = None
     REQUIRES = {}
+    JOBS = 4 * cpu_count()
 
     def __init__(self, project, config):
         pass
@@ -81,6 +83,9 @@ class RemoteBase(object):
     def save_info(self, path_info):
         raise NotImplementedError
 
+    def changed(self, path_info, checksum_info):
+        raise NotImplementedError
+
     def save(self, path_info):
         raise NotImplementedError
 

dvc/remote/gs.py

@@ -59,6 +59,37 @@ class RemoteGS(RemoteBase):
         return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
                                                path_info['key'])}
 
+    @staticmethod
+    def to_string(path_info):
+        return "{}://{}/{}".format(path_info['scheme'],
+                                   path_info['bucket'],
+                                   path_info['key'])
+
+    def changed_cache(self, etag):
+        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
+        cache = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
+
+        if {self.PARAM_ETAG: etag} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+        return False
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        etag = checksum_info.get(self.PARAM_ETAG, None)
+        if etag is None:
+            return True
+
+        if self.changed_cache(etag):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def _copy(self, from_info, to_info, gs=None):
         gs = gs if gs else self.gs
 
@@ -92,6 +123,25 @@ class RemoteGS(RemoteBase):
         if not etag:
             return
 
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(etag):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(etag, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), etag))
+
         key = posixpath.join(self.prefix, etag[0:2], etag[2:])
         from_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
 
@@ -229,6 +279,7 @@ class RemoteGS(RemoteBase):
         used = [info[self.PARAM_ETAG] for info in cinfos['gs']]
         used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
 
+        removed = False
         for etag in self._all_etags():
             if etag in used:
                 continue
@@ -237,3 +288,6 @@ class RemoteGS(RemoteBase):
                          'key': posixpath.join(self.prefix,
                                                etag[0:2], etag[2:])}
             self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/hdfs.py

@@ -9,6 +9,7 @@ from dvc.remote.base import RemoteBase
 from dvc.remote.local import RemoteLOCAL
 from dvc.exceptions import DvcException
 from dvc.logger import Logger
+from dvc.utils import fix_env
 
 
 class RemoteHDFS(RemoteBase):
@@ -28,10 +29,16 @@ class RemoteHDFS(RemoteBase):
         cmd = 'hadoop fs -' + cmd
         if user:
             cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
+
+        # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
+        # See https://github.com/iterative/dvc/issues/1197.
+        close_fds = (os.name != 'nt')
+
         p = Popen(cmd,
                   shell=True,
-                  close_fds=True,
+                  close_fds=close_fds,
                   executable=os.getenv('SHELL'),
+                  env=fix_env(os.environ),
                   stdin=PIPE,
                   stdout=PIPE,
                   stderr=PIPE)
@@ -70,6 +77,39 @@ class RemoteHDFS(RemoteBase):
 
         return {self.PARAM_CHECKSUM: self.checksum(path_info)}
 
+    @staticmethod
+    def to_string(path_info):
+        return "{}://{}".format(path_info['scheme'],
+                                path_info['url'])
+
+    def changed_cache(self, checksum):
+        cache = {}
+        cache['scheme'] = 'hdfs'
+        cache['user'] = self.user
+        cache['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
+
+        if {self.PARAM_CHECKSUM: checksum} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
+        if checksum is None:
+            return True
+
+        if self.changed_cache(self, checksum):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def save(self, path_info):
         if path_info['scheme'] != 'hdfs':
             raise NotImplementedError
@@ -94,6 +134,25 @@ class RemoteHDFS(RemoteBase):
         if not checksum:
             return
 
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(checksum):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(checksum, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), checksum))
+
         src = path_info.copy()
         src['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
 
@@ -197,6 +256,7 @@ class RemoteHDFS(RemoteBase):
         used = [info[self.PARAM_CHECKSUM] for info in cinfos['hdfs']]
         used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
 
+        removed = False
         for checksum in self._all_checksums():
             if checksum in used:
                 continue
@@ -205,3 +265,6 @@ class RemoteHDFS(RemoteBase):
                          'url': posixpath.join(self.url,
                                                checksum[0:2], checksum[2:])}
             self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/local.py

@@ -36,7 +36,6 @@ class RemoteLOCAL(RemoteBase):
     def __init__(self, project, config):
         self.project = project
         self.state = self.project.state
-        self.link_state = project.link_state
         storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
         self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
 
@@ -82,7 +81,7 @@ class RemoteLOCAL(RemoteBase):
         relpath = os.path.relpath(path, self.cache_dir)
         return os.path.dirname(relpath) + os.path.basename(relpath)
 
-    def changed(self, md5):
+    def changed_cache(self, md5):
         cache = self.get(md5)
         if self.state.changed(cache, md5=md5):
             if os.path.exists(cache):
@@ -104,8 +103,12 @@ class RemoteLOCAL(RemoteBase):
         while i > 0:
             try:
                 self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
+                msg = "Created '{}': {} -> {}"
+                Logger.info(msg.format(self.cache_types[0],
+                                       os.path.relpath(cache),
+                                       os.path.relpath(path)))
                 return
-            except Exception as exc:
+            except DvcException as exc:
                 msg = 'Cache type \'{}\' is not supported: {}'
                 Logger.debug(msg.format(self.cache_types[0], str(exc)))
                 del self.cache_types[0]
@@ -134,8 +137,8 @@ class RemoteLOCAL(RemoteBase):
     def collect_dir_cache(self, dname):
         dir_info = []
 
+        bar = False
         for root, dirs, files in os.walk(dname):
-            bar = False
             if len(files) > LARGE_DIR_SIZE:
                 msg = "Computing md5 for a large directory {}. " \
                       "This is only done once."
@@ -154,25 +157,18 @@ class RemoteLOCAL(RemoteBase):
                     progress.update_target(title, processed, total)
                     processed += 1
 
-                # FIXME: we could've used md5 = state.update(path, dump=False)
-                # here, but it is around twice as slow(on ssd, don't know about
-                # hdd) for a directory with small files. What we could do here
-                # is introduce some kind of a limit for file size, after which
-                # we would actually register it in our state file.
-                md5 = self.state.update(path, dump=False)
+                md5 = self.state.update(path)
                 dir_info.append({self.PARAM_RELPATH: relpath,
                                  self.PARAM_MD5: md5})
 
-            self.state.dump()
-
-            if bar:
-                progress.finish_target(title)
+        if bar:
+            progress.finish_target(title)
 
         # NOTE: sorting the list by path to ensure reproducibility
         dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
 
         md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
-        if self.changed(md5):
+        if self.changed_cache(md5):
             self.dump_dir_cache(md5, dir_info)
 
         return (md5, dir_info)
@@ -231,23 +227,28 @@ class RemoteLOCAL(RemoteBase):
             Logger.warn(msg.format(os.path.relpath(path)))
             return
 
-        if self.changed(md5):
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(os.path.relpath(path)))
+            return
+
+        if self.changed_cache(md5):
             msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.'
             Logger.warn(msg.format(md5, os.path.relpath(path)))
             remove(path)
             return
 
         if os.path.exists(path):
-            msg = u'Data \'{}\' exists. Removing before checkout'
-            Logger.debug(msg.format(os.path.relpath(path)))
+            msg = u'Data \'{}\' exists. Removing before checkout.'
+            Logger.warn(msg.format(os.path.relpath(path)))
             remove(path)
 
-        msg = u'Checking out \'{}\' with cache \'{}\''
-        Logger.debug(msg.format(os.path.relpath(path), md5))
+        msg = u'Checking out \'{}\' with cache \'{}\'.'
+        Logger.info(msg.format(os.path.relpath(path), md5))
 
         if not self.is_dir_cache(cache):
             self.link(cache, path)
-            self.link_state.update(path)
+            self.state.update_link(path)
             return
 
         # Create dir separately so that dir is created
@@ -261,7 +262,7 @@ class RemoteLOCAL(RemoteBase):
             relpath = entry[self.PARAM_RELPATH]
             p = os.path.join(path, relpath)
             self.link(c, p)
-        self.link_state.update(path)
+        self.state.update_link(path)
 
     def _move(self, inp, outp):
         # moving in two stages to make last the move atomic in
@@ -277,13 +278,13 @@ class RemoteLOCAL(RemoteBase):
 
         cache = self.get(md5)
 
-        if self.changed(md5):
+        if self.changed_cache(md5):
             self._move(path, cache)
         else:
             remove(path)
 
         self.link(cache, path)
-        self.link_state.update(path)
+        self.state.update_link(path)
 
         return {self.PARAM_MD5: md5}
 
@@ -297,14 +298,14 @@ class RemoteLOCAL(RemoteBase):
             p = os.path.join(path, relpath)
             c = self.get(m)
 
-            if self.changed(m):
+            if self.changed_cache(m):
                 self._move(p, c)
             else:
                 remove(p)
 
             self.link(c, p)
 
-        self.link_state.update(path)
+        self.state.update_link(path)
 
         return {self.PARAM_MD5: md5}
 
@@ -314,6 +315,10 @@ class RemoteLOCAL(RemoteBase):
 
         path = path_info['path']
 
+        msg = "Saving '{}' to cache '{}'."
+        Logger.info(msg.format(os.path.relpath(path),
+                               os.path.relpath(self.cache_dir)))
+
         if os.path.isdir(path):
             return self._save_dir(path_info)
         else:
@@ -325,6 +330,19 @@ class RemoteLOCAL(RemoteBase):
 
         return {self.PARAM_MD5: self.state.update(path_info['path'])}
 
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        md5 = checksum_info.get(self.PARAM_MD5, None)
+        if md5 is None:
+            return True
+
+        if self.changed_cache(md5):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def remove(self, path_info):
         if path_info['scheme'] != 'local':
             raise NotImplementedError
@@ -425,8 +443,8 @@ class RemoteLOCAL(RemoteBase):
             for i in self.load_dir_cache(md5):
                 if info.get('branch'):
                     i['branch'] = info['branch']
-                i[self.PARAM_PATH] = posixpath.join(info[self.PARAM_PATH],
-                                                    i[self.PARAM_RELPATH])
+                i[self.PARAM_PATH] = os.path.join(info[self.PARAM_PATH],
+                                                  i[self.PARAM_RELPATH])
                 collected.append(i.copy())
 
         collected.extend(checksum_infos)
@@ -460,16 +478,20 @@ class RemoteLOCAL(RemoteBase):
         checksum_infos = self._collect(checksum_infos['local'])[0]
         used_md5s = [info[self.PARAM_MD5] for info in checksum_infos]
 
+        removed = False
         for md5 in self.all():
             if md5 in used_md5s:
                 continue
             remove(self.get(md5))
+            removed = True
 
-    def status(self, checksum_infos, remote, jobs=1, show_checksums=False):
+        return removed
+
+    def status(self, checksum_infos, remote, jobs=None, show_checksums=False):
         Logger.info("Preparing to pull data from {}".format(remote.url))
         title = "Collecting information"
 
-        progress.set_n_total(0)
+        progress.set_n_total(1)
         progress.update_target(title, 0, 100)
 
         checksum_infos, missing = self._collect(checksum_infos)
@@ -490,7 +512,7 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 90, 100)
 
-        local_exists = [not self.changed(md5) for md5 in md5s]
+        local_exists = [not self.changed_cache(md5) for md5 in md5s]
 
         progress.finish_target(title)
 
@@ -501,11 +523,11 @@ class RemoteLOCAL(RemoteBase):
     def _do_pull(self,
                  checksum_infos,
                  remote,
-                 jobs=1,
+                 jobs=None,
                  show_checksums=False):
         title = "Collecting information"
 
-        progress.set_n_total(0)
+        progress.set_n_total(1)
         progress.update_target(title, 0, 100)
 
         grouped = zip(*self._group(checksum_infos,
@@ -517,7 +539,7 @@ class RemoteLOCAL(RemoteBase):
         names = []
         # NOTE: filter files that are not corrupted
         for md5, name in grouped:
-            if self.changed(md5):
+            if self.changed_cache(md5):
                 md5s.append(md5)
                 names.append(name)
 
@@ -531,8 +553,17 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 60, 100)
 
+        # NOTE: dummy call to try to establish a connection
+        # to see if we need to ask user for a password.
+        remote.exists(remote.md5s_to_path_infos(['000']))
+
+        progress.update_target(title, 70, 100)
+
         assert len(path_infos) == len(cache) == len(md5s) == len(names)
 
+        if jobs is None:
+            jobs = remote.JOBS
+
         chunks = list(zip(to_chunks(path_infos, jobs),
                           to_chunks(cache, jobs),
                           to_chunks(names, jobs)))
@@ -556,7 +587,7 @@ class RemoteLOCAL(RemoteBase):
         for f in futures:
             f.result()
 
-    def pull(self, checksum_infos, remote, jobs=1, show_checksums=False):
+    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
         Logger.info("Preparing to pull data from {}".format(remote.url))
 
         # NOTE: try fetching missing dir info
@@ -573,11 +604,11 @@ class RemoteLOCAL(RemoteBase):
                       jobs,
                       show_checksums=show_checksums)
 
-    def push(self, checksum_infos, remote, jobs=1, show_checksums=False):
+    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
         Logger.info("Preparing to push data to {}".format(remote.url))
         title = "Collecting information"
 
-        progress.set_n_total(0)
+        progress.set_n_total(1)
         progress.update_target(title, 0, 100)
 
         checksum_infos = self._collect(checksum_infos)[0]
@@ -586,7 +617,7 @@ class RemoteLOCAL(RemoteBase):
 
         # NOTE: verifying that our cache is not corrupted
         def func(info):
-            return not self.changed(info[self.PARAM_MD5])
+            return not self.changed_cache(info[self.PARAM_MD5])
         checksum_infos = list(filter(func, checksum_infos))
 
         progress.update_target(title, 20, 100)
@@ -618,6 +649,9 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 90, 100)
 
+        if jobs is None:
+            jobs = remote.JOBS
+
         chunks = list(zip(to_chunks(path_infos, jobs),
                           to_chunks(cache, jobs),
                           to_chunks(names, jobs)))

dvc/remote/s3.py

@@ -47,7 +47,9 @@ class RemoteS3(RemoteBase):
                                            '').lstrip('/')
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
         self.region = config.get(Config.SECTION_AWS_REGION, None)
-        self.profile = config.get(Config.SECTION_AWS_PROFILE, 'default')
+        self.profile = os.getenv('AWS_PROFILE',
+                                 config.get(Config.SECTION_AWS_PROFILE,
+                                            'default'))
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL, None)
 
         credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
@@ -55,8 +57,9 @@ class RemoteS3(RemoteBase):
             creds_conf = configobj.ConfigObj(credentialpath)
             creds = creds_conf.get(self.profile, {})
         else:
-            creds = {}
+            creds = self._get_aws_config_from_env()
 
+        self.creds = creds
         self.region = creds.get('region', self.region)
         self.aws_access_key_id = creds.get('aws_access_key_id', None)
         self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
@@ -71,12 +74,22 @@ class RemoteS3(RemoteBase):
 
     @property
     def s3(self):
-        session = boto3.session.Session(profile_name=self.profile)
-        return session.client('s3',
-                              aws_access_key_id=self.aws_access_key_id,
-                              aws_secret_access_key=self.aws_secret_access_key,
-                              region_name=self.region,
-                              endpoint_url=self.endpoint_url)
+        if not self.creds:
+            session = boto3.session.Session(profile_name=self.profile)
+        else:
+            session = boto3.session.Session(
+                             aws_access_key_id=self.aws_access_key_id,
+                             aws_secret_access_key=self.aws_secret_access_key,
+                             region_name=self.region)
+        return session.client('s3', endpoint_url=self.endpoint_url)
+
+    def _get_aws_config_from_env(self):
+        creds = {
+            'region': os.getenv('AWS_DEFAULT_REGION'),
+            'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
+            'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
+        }
+        return {k: v for k, v in creds.items() if v is not None}
 
     def get_etag(self, bucket, key):
         try:
@@ -93,6 +106,19 @@ class RemoteS3(RemoteBase):
         return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
                                                path_info['key'])}
 
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        etag = checksum_info.get(self.PARAM_ETAG, None)
+        if etag is None:
+            return True
+
+        if self.changed_cache(etag):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def _copy(self, from_info, to_info, s3=None):
         s3 = s3 if s3 else self.s3
 
@@ -112,6 +138,23 @@ class RemoteS3(RemoteBase):
 
         return {self.PARAM_ETAG: etag}
 
+    @staticmethod
+    def to_string(path_info):
+        return "s3://{}/{}".format(path_info['bucket'], path_info['key'])
+
+    def changed_cache(self, etag):
+        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
+        cache = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
+
+        if {self.PARAM_ETAG: etag} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
     def checkout(self, path_info, checksum_info):
         if path_info['scheme'] != 's3':
             raise NotImplementedError
@@ -120,6 +163,25 @@ class RemoteS3(RemoteBase):
         if not etag:
             return
 
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(etag):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(etag, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), etag))
+
         key = posixpath.join(self.prefix, etag[0:2], etag[2:])
         from_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
 
@@ -281,6 +343,7 @@ class RemoteS3(RemoteBase):
         used_etags = [info[self.PARAM_ETAG] for info in cinfos['s3']]
         used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
 
+        removed = False
         for etag in self._all():
             if etag in used_etags:
                 continue
@@ -289,3 +352,6 @@ class RemoteS3(RemoteBase):
                                                etag[0:2], etag[2:]),
                          'bucket': self.bucket}
             self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/ssh.py

@@ -44,8 +44,14 @@ class RemoteSSH(RemoteBase):
     REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'
 
     REQUIRES = {'paramiko': paramiko}
+
+    JOBS = 4
+
     PARAM_MD5 = 'md5'
 
+    DEFAULT_PORT = 22
+    TIMEOUT = 1800
+
     def __init__(self, project, config):
         self.project = project
         self.url = config.get(Config.SECTION_REMOTE_URL, '/')
@@ -55,30 +61,49 @@ class RemoteSSH(RemoteBase):
             self.user = config.get(Config.SECTION_REMOTE_USER,
                                    getpass.getuser())
         self.prefix = self.group('path')
+        self.port = config.get(Config.SECTION_REMOTE_PORT, self.DEFAULT_PORT)
+        self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
+        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
+        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
+        self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD,
+                                       False)
 
     def md5s_to_path_infos(self, md5s):
         return [{'scheme': 'ssh',
                  'host': self.host,
                  'user': self.user,
+                 'port': self.port,
                  'path': posixpath.join(self.prefix,
                                         md5[0:2], md5[2:])} for md5 in md5s]
 
-    def ssh(self, host=None, user=None):
-        msg = "Establishing ssh connection with '{}' as user '{}'"
-        Logger.debug(msg.format(host, user))
+    def ssh(self, host=None, user=None, port=None):
+        msg = "Establishing ssh connection with '{}' " \
+              "through port '{}' as user '{}'"
+        Logger.debug(msg.format(host, port, user))
 
         ssh = paramiko.SSHClient()
 
         ssh.load_system_host_keys()
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-        ssh.connect(host, username=user)
+        if self.ask_password and self.password is None:
+            msg = "host '{}' port '{}' user '{}'".format(host, port, user)
+            self.password = self.project.prompt.prompt_password(msg)
+
+        ssh.connect(host,
+                    username=user,
+                    port=port,
+                    key_filename=self.keyfile,
+                    timeout=self.timeout,
+                    password=self.password)
 
         return ssh
 
     def exists(self, path_infos):
         ret = []
-        ssh = self.ssh(host=self.host, user=self.user)
+        ssh = self.ssh(host=self.host,
+                       user=self.user,
+                       port=self.port)
         cmd = 'find {} -type f -follow -print'.format(self.prefix)
         stdout = self._exec(ssh, cmd)
         plist = stdout.split()
@@ -94,16 +119,55 @@ class RemoteSSH(RemoteBase):
 
     def _exec(self, ssh, cmd):
         stdin, stdout, stderr = ssh.exec_command(cmd)
+        channel = stdout.channel
+
+        stdin.close()
+        channel.shutdown_write()
+
+        stdout_chunks = []
+        stderr_chunks = []
+        while (not channel.closed
+               or channel.recv_ready()
+               or channel.recv_stderr_ready()):
+            import select
+            got_chunk = False
+            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
+            for c in readq:
+                if c.recv_ready():
+                    stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
+                    got_chunk = True
+
+                if c.recv_stderr_ready():
+                    stderr_len = len(c.in_stderr_buffer)
+                    s = stderr.channel.recv_stderr(stderr_len)
+                    stderr_chunks.append(s)
+                    got_chunk = True
+
+            if not got_chunk \
+               and stdout.channel.exit_status_ready() \
+               and not stderr.channel.recv_stderr_ready() \
+               and not stdout.channel.recv_ready():
+                stdout.channel.shutdown_read()
+                stdout.channel.close()
+                break
+
+        stdout.close()
+        stderr.close()
+
         if stdout.channel.recv_exit_status() != 0:
-            DvcException('SSH command \'{}\' failed: {}'.format(cmd,
-                                                                stderr.read()))
-        return stdout.read().decode('utf-8')
+            err = b''.join(stderr_chunks).decode('utf-8')
+            msg = 'SSH command \'{}\' failed: {}'.format(cmd, err)
+            raise DvcException(msg)
+
+        return b''.join(stdout_chunks).decode('utf-8')
 
     def md5(self, path_info):
         if path_info['scheme'] != 'ssh':
             raise NotImplementedError
 
-        ssh = self.ssh(host=path_info['host'], user=path_info['user'])
+        ssh = self.ssh(host=path_info['host'],
+                       user=path_info['user'],
+                       port=path_info['port'])
 
         # Use different md5 commands depending on os
         stdout = self._exec(ssh, 'uname').strip()
@@ -133,7 +197,8 @@ class RemoteSSH(RemoteBase):
         assert from_info['user'] == to_info['user']
 
         s = ssh if ssh else self.ssh(host=from_info['host'],
-                                     user=from_info['user'])
+                                     user=from_info['user'],
+                                     port=from_info['port'])
 
         dname = posixpath.dirname(to_info['path'])
         self._exec(s, 'mkdir -p {}'.format(dname))
@@ -148,6 +213,43 @@ class RemoteSSH(RemoteBase):
 
         return {self.PARAM_MD5: self.md5(path_info)}
 
+    @staticmethod
+    def to_string(path_info):
+        return "{}://{}@{}:{}".format(path_info['scheme'],
+                                      path_info['user'],
+                                      path_info['host'],
+                                      path_info['path'])
+
+    def changed_cache(self, md5):
+        cache = {}
+        cache['scheme'] = 'ssh'
+        cache['host'] = self.host
+        cache['port'] = self.port
+        cache['user'] = self.user
+        cache['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
+
+        if {self.PARAM_MD5: md5} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        md5 = checksum_info.get(self.PARAM_MD5, None)
+        if md5 is None:
+            return True
+
+        if self.changed_code(md5):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def save(self, path_info):
         if path_info['scheme'] != 'ssh':
             raise NotImplementedError
@@ -168,6 +270,25 @@ class RemoteSSH(RemoteBase):
         if not md5:
             return
 
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(md5):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(md5, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), md5))
+
         src = path_info.copy()
         src['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
 
@@ -181,7 +302,9 @@ class RemoteSSH(RemoteBase):
                                                       path_info['host'],
                                                       path_info['path']))
 
-        ssh = self.ssh(host=path_info['host'], user=path_info['user'])
+        ssh = self.ssh(host=path_info['host'],
+                       user=path_info['user'],
+                       port=path_info['port'])
         ssh.open_sftp().remove(path_info['path'])
         ssh.close()
 
@@ -192,7 +315,9 @@ class RemoteSSH(RemoteBase):
                  names=None):
         names = self._verify_path_args(from_infos, to_infos, names)
 
-        ssh = self.ssh(host=from_infos[0]['host'], user=from_infos[0]['user'])
+        ssh = self.ssh(host=from_infos[0]['host'],
+                       user=from_infos[0]['user'],
+                       port=from_infos[0]['port'])
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
             if from_info['scheme'] != 'ssh':
@@ -200,6 +325,7 @@ class RemoteSSH(RemoteBase):
 
             if to_info['scheme'] == 'ssh':
                 assert from_info['host'] == to_info['host']
+                assert from_info['port'] == to_info['port']
                 assert from_info['user'] == to_info['user']
                 self.cp(from_info, to_info, ssh=ssh)
                 continue
@@ -236,7 +362,9 @@ class RemoteSSH(RemoteBase):
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 
-        ssh = self.ssh(host=to_infos[0]['host'], user=to_infos[0]['user'])
+        ssh = self.ssh(host=to_infos[0]['host'],
+                       user=to_infos[0]['user'],
+                       port=to_infos[0]['port'])
         sftp = ssh.open_sftp()
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
@@ -277,7 +405,9 @@ class RemoteSSH(RemoteBase):
         return posixpath.dirname(relpath) + posixpath.basename(relpath)
 
     def _all_md5s(self):
-        ssh = self.ssh(host=self.host, user=self.user)
+        ssh = self.ssh(host=self.host,
+                       user=self.user,
+                       port=self.port)
         cmd = 'find {} -type f -follow -print'.format(self.prefix)
         stdout = self._exec(ssh, cmd)
         flist = stdout.split()
@@ -289,12 +419,17 @@ class RemoteSSH(RemoteBase):
         used = [info[self.PARAM_MD5] for info in cinfos['ssh']]
         used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
 
+        removed = False
         for md5 in self._all_md5s():
             if md5 in used:
                 continue
             path_info = {'scheme': 'ssh',
                          'user': self.user,
                          'host': self.host,
+                         'port': self.port,
                          'path': posixpath.join(self.prefix,
                                                 md5[0:2], md5[2:])}
             self.remove(path_info)
+            removed = True
+
+        return removed

dvc/scm.py

@@ -2,6 +2,7 @@ import os
 
 from dvc.exceptions import DvcException
 from dvc.logger import Logger
+from dvc.utils import fix_env
 
 
 class SCMError(DvcException):
@@ -13,13 +14,18 @@ class FileNotInRepoError(DvcException):
 
 
 class Base(object):
-    def __init__(self, root_dir=os.curdir):
-        self.root_dir = os.path.abspath(os.path.realpath(root_dir))
+    def __init__(self, root_dir=os.curdir, project=None):
+        self.project = project
+        self.root_dir = root_dir
 
     @staticmethod
     def is_repo(root_dir):
         return True
 
+    @staticmethod
+    def is_submodule(root_dir):
+        return True
+
     def ignore(self, path):
         pass
 
@@ -44,17 +50,34 @@ class Base(object):
     def branch(self, branch):
         pass
 
-    def brancher(self, branches=None, all_branches=False):
-        if not branches and not all_branches:
+    def brancher(self,
+                 branches=None,
+                 all_branches=False,
+                 tags=None,
+                 all_tags=False):
+        if not branches and not all_branches \
+           and not tags and not all_tags:
             yield ''
             return
 
         saved = self.active_branch()
-        if not branches:
-            branches = self.list_branches() if all_branches else [saved]
-        for branch in branches:
-            self.checkout(branch)
-            yield branch
+        revs = []
+
+        if branches is not None:
+            revs.extend(branches)
+        elif all_branches:
+            revs.extend(self.list_branches())
+        elif tags is not None:
+            revs.extend(tags)
+        elif all_tags:
+            revs.extend(self.list_tags())
+        else:
+            revs.extend([saved])
+
+        for rev in revs:
+            self.checkout(rev)
+            yield rev
+
         self.checkout(saved)
 
     def untracked_files(self):
@@ -69,6 +92,9 @@ class Base(object):
     def list_branches(self):
         pass
 
+    def list_tags(self):
+        pass
+
     def install(self):
         pass
 
@@ -77,29 +103,46 @@ class Git(Base):
     GITIGNORE = '.gitignore'
     GIT_DIR = '.git'
 
-    def __init__(self, root_dir=os.curdir):
-        super(Git, self).__init__(root_dir)
+    def __init__(self, root_dir=os.curdir, project=None):
+        super(Git, self).__init__(root_dir, project=project)
 
         import git
         from git.exc import InvalidGitRepositoryError
         try:
             self.repo = git.Repo(root_dir)
         except InvalidGitRepositoryError:
-            raise SCMError('{} is not a git repository'.format(root_dir))
+            msg = '{} is not a git repository'
+            raise SCMError(msg.format(root_dir))
+
+        # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
+        # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
+        env = fix_env(None)
+        lp = env.get('LD_LIBRARY_PATH', None)
+        self.repo.git.update_environment(LD_LIBRARY_PATH=lp)
 
     @staticmethod
     def is_repo(root_dir):
-        git_dir = os.path.join(root_dir, Git.GIT_DIR)
-        return os.path.isdir(git_dir)
+        return os.path.isdir(Git._get_git_dir(root_dir))
+
+    @staticmethod
+    def is_submodule(root_dir):
+        return os.path.isfile(Git._get_git_dir(root_dir))
+
+    @staticmethod
+    def _get_git_dir(root_dir):
+        return os.path.join(root_dir, Git.GIT_DIR)
+
+    @property
+    def dir(self):
+        return self.repo.git_dir
 
     def ignore_file(self):
         return self.GITIGNORE
 
     def _get_gitignore(self, path):
+        assert os.path.isabs(path)
         entry = os.path.basename(path)
-        gitignore = os.path.join(os.path.abspath(os.path.dirname(path)),
-                                 self.GITIGNORE)
-        gitignore = os.path.abspath(os.path.realpath(gitignore))
+        gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE)
 
         if not gitignore.startswith(self.root_dir):
             raise FileNotInRepoError(path)
@@ -117,11 +160,19 @@ class Git(Base):
             if len(filtered) != 0:
                 return
 
+        msg = "Adding '{}' to '{}'.".format(os.path.relpath(path),
+                                            os.path.relpath(gitignore))
+        Logger.info(msg)
+
         content = entry
         if len(ignore_list) > 0:
             content = '\n' + content
 
-        open(gitignore, 'a').write(content)
+        with open(gitignore, 'a') as fd:
+            fd.write(content)
+
+        if self.project is not None:
+            self.project._files_to_git_add.append(os.path.relpath(gitignore))
 
     def ignore_remove(self, path):
         entry, gitignore = self._get_gitignore(path)
@@ -137,6 +188,9 @@ class Git(Base):
         with open(gitignore, 'w') as fd:
             fd.writelines(filtered)
 
+        if self.project is not None:
+            self.project._files_to_git_add.append(os.path.relpath(gitignore))
+
     def add(self, paths):
         # NOTE: GitPython is not currently able to handle index version >= 3.
         # See https://github.com/iterative/dvc/issues/610 for more details.
@@ -174,6 +228,9 @@ class Git(Base):
     def list_branches(self):
         return [h.name for h in self.repo.heads]
 
+    def list_tags(self):
+        return [t.name for t in self.repo.tags]
+
     def install(self):
         hook = os.path.join(self.root_dir,
                             self.GIT_DIR,
@@ -187,8 +244,8 @@ class Git(Base):
         os.chmod(hook, 0o777)
 
 
-def SCM(root_dir=os.curdir, no_scm=False):
-    if Git.is_repo(root_dir):
-        return Git(root_dir)
+def SCM(root_dir, no_scm=False, project=None):
+    if Git.is_repo(root_dir) or Git.is_submodule(root_dir):
+        return Git(root_dir, project=project)
 
-    return Base(root_dir)
+    return Base(root_dir, project=project)

dvc/signal_handler.py

@@ -1,17 +0,0 @@
-import signal
-import threading
-
-from dvc.logger import Logger
-
-
-class SignalHandler(object):
-    def __enter__(self):
-        if isinstance(threading.current_thread(), threading._MainThread):
-            self.old_handler = signal.signal(signal.SIGINT, self.handler)
-
-    def handler(self, sig, frame):
-        Logger.debug('Ignoring SIGINT during critical parts of code...')
-
-    def __exit__(self, type, value, traceback):
-        if isinstance(threading.current_thread(), threading._MainThread):
-            signal.signal(signal.SIGINT, self.old_handler)

dvc/stage.py

@@ -1,16 +1,15 @@
 import os
 import yaml
 import itertools
+import posixpath
 import subprocess
 from schema import Schema, SchemaError, Optional, Or, And
-import posixpath
 
 import dvc.dependency as dependency
 import dvc.output as output
 from dvc.exceptions import DvcException
-from dvc.prompt import prompt
 from dvc.logger import Logger
-from dvc.utils import dict_md5
+from dvc.utils import dict_md5, fix_env
 
 
 class StageCmdFailedError(DvcException):
@@ -20,8 +19,39 @@ class StageCmdFailedError(DvcException):
 
 
 class StageFileFormatError(DvcException):
-    def __init__(self):
-        super(StageFileFormatError, self).__init__('Stage file format error')
+    def __init__(self, fname, e):
+        msg = "Stage file '{}' format error: {}".format(fname, str(e))
+        super(StageFileFormatError, self).__init__(msg)
+
+
+class StageFileDoesNotExistError(DvcException):
+    def __init__(self, fname):
+        msg = "'{}' does not exist.".format(fname)
+        super(StageFileDoesNotExistError, self).__init__(msg)
+
+
+class StageFileIsNotDvcFileError(DvcException):
+    def __init__(self, fname):
+        msg = "'{}' is not a dvc file".format(fname)
+        super(StageFileIsNotDvcFileError, self).__init__(msg)
+
+
+class StageFileBadNameError(DvcException):
+    def __init__(self, msg):
+        super(StageFileBadNameError, self).__init__(msg)
+
+
+class MissingDep(DvcException):
+    def __init__(self, deps):
+        assert len(deps) > 0
+
+        if len(deps) > 1:
+            dep = 'dependencies'
+        else:
+            dep = 'dependency'
+
+        msg = u'missing {}: {}'.format(dep, ', '.join(map(str, deps)))
+        super(MissingDep, self).__init__(msg)
 
 
 class MissingDataSource(DvcException):
@@ -81,16 +111,20 @@ class Stage(object):
         return self.cmd is None
 
     @staticmethod
-    def is_stage_file(path):
-        if not os.path.isfile(path):
-            return False
-
+    def is_stage_filename(path):
         if not path.endswith(Stage.STAGE_FILE_SUFFIX) \
            and os.path.basename(path) != Stage.STAGE_FILE:
             return False
 
         return True
 
+    @staticmethod
+    def is_stage_file(path):
+        if not os.path.isfile(path):
+            return False
+
+        return Stage.is_stage_filename(path)
+
     def changed_md5(self):
         md5 = self.dumpd().get(self.PARAM_MD5, None)
 
@@ -160,7 +194,8 @@ class Stage(object):
 
         msg = "Going to reproduce '{}'. Are you sure you want to continue?"
         msg = msg.format(self.relpath)
-        if interactive and not prompt(msg):
+        if interactive \
+           and not self.project.prompt.prompt(msg):
             raise DvcException('Reproduction aborted by the user')
 
         self.project.logger.info(u'Reproducing \'{}\''.format(self.relpath))
@@ -173,17 +208,15 @@ class Stage(object):
         return self
 
     @staticmethod
-    def validate(d):
+    def validate(d, fname=None):
         try:
             Schema(Stage.SCHEMA).validate(d)
         except SchemaError as exc:
-            Logger.debug(str(exc))
-            raise StageFileFormatError()
+            raise StageFileFormatError(fname, exc)
 
     @staticmethod
     def loadd(project, d, path):
-        Stage.validate(d)
-
+        Stage.validate(d, fname=os.path.relpath(path))
         path = os.path.abspath(path)
         cwd = os.path.dirname(path)
         cmd = d.get(Stage.PARAM_CMD, None)
@@ -219,7 +252,8 @@ class Stage(object):
         if not fname:
             fname = path.basename(out.path) + cls.STAGE_FILE_SUFFIX
 
-        cwd = path.dirname(out.path) if not cwd or add else cwd
+        if not cwd or (add and out.is_local):
+            cwd = path.dirname(out.path)
 
         return (fname, cwd)
 
@@ -246,6 +280,11 @@ class Stage(object):
                                         use_cache=False, metric=True)
         stage.deps = dependency.loads_from(stage, deps)
 
+        if fname is not None and os.path.basename(fname) != fname:
+            msg = "Stage file name '{}' should not contain subdirectories. " \
+                  "Use '-c|--cwd' to change location of the stage file."
+            raise StageFileBadNameError(msg.format(fname))
+
         fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add)
 
         cwd = os.path.abspath(cwd)
@@ -253,17 +292,43 @@ class Stage(object):
 
         if os.path.exists(path):
             relpath = os.path.relpath(path)
-            msg = "{} already exists. Do you wish to overwrite it?"
-            if not overwrite and not prompt(msg.format(relpath), False):
-                raise DvcException("{} already exists".format(relpath))
+            msg = "'{}' already exists. " \
+                  "Do you wish to run the command and overwrite it?"
+            if not overwrite \
+               and not project.prompt.prompt(msg.format(relpath), False):
+                raise DvcException("'{}' already exists".format(relpath))
 
         stage.cwd = cwd
         stage.path = path
 
         return stage
 
+    @staticmethod
+    def _check_dvc_filename(fname):
+        if not Stage.is_stage_filename(fname):
+            msg = "Bad stage filename '{}'. Stage files should be named " \
+                  "'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc')."
+            raise StageFileBadNameError(msg.format(os.path.relpath(fname),
+                                                   os.path.basename(fname)))
+
+    @staticmethod
+    def _check_dvc_file(fname):
+        sname = fname + Stage.STAGE_FILE_SUFFIX
+        if Stage.is_stage_file(sname):
+            Logger.info("Do you mean '{}'?".format(sname))
+
     @staticmethod
     def load(project, fname):
+        if not os.path.exists(fname):
+            Stage._check_dvc_file(fname)
+            raise StageFileDoesNotExistError(fname)
+
+        Stage._check_dvc_filename(fname)
+
+        if not Stage.is_stage_file(fname):
+            Stage._check_dvc_file(fname)
+            raise StageFileIsNotDvcFileError(fname)
+
         with open(fname, 'r') as fd:
             return Stage.loadd(project, yaml.safe_load(fd), fname)
 
@@ -292,9 +357,16 @@ class Stage(object):
         if not fname:
             fname = self.path
 
+        self._check_dvc_filename(fname)
+
+        msg = "Saving information to '{}'.".format(os.path.relpath(fname))
+        Logger.info(msg)
+
         with open(fname, 'w') as fd:
             yaml.safe_dump(self.dumpd(), fd, default_flow_style=False)
 
+        self.project._files_to_git_add.append(os.path.relpath(fname))
+
     def save(self):
         for dep in self.deps:
             dep.save()
@@ -302,6 +374,15 @@ class Stage(object):
         for out in self.outs:
             out.save()
 
+    def _check_missing_deps(self):
+        missing = []
+        for dep in self.deps:
+            if not dep.exists:
+                missing.append(dep)
+
+        if len(missing) > 0:
+            raise MissingDep(missing)
+
     def run(self, dry=False):
         if self.locked:
             msg = u'Verifying outputs in locked stage \'{}\''
@@ -325,10 +406,11 @@ class Stage(object):
             self.project.logger.info(msg)
 
             if not dry:
+                self._check_missing_deps()
                 p = subprocess.Popen(self.cmd,
                                      cwd=self.cwd,
                                      shell=True,
-                                     env=os.environ,
+                                     env=fix_env(os.environ),
                                      executable=os.getenv('SHELL'))
                 p.communicate()
                 if p.returncode != 0:

dvc/state.py

@@ -1,60 +1,68 @@
 import os
-import json
+import time
+import sqlite3
 import nanotime
-import threading
 
-from dvc.lock import Lock
+from dvc.config import Config
 from dvc.system import System
 from dvc.utils import file_md5, remove
 from dvc.exceptions import DvcException
-from dvc.signal_handler import SignalHandler
 from dvc.logger import Logger
 
 
-class StateEntry(object):
-    PARAM_MTIME = 'mtime'
-    PARAM_MD5 = 'md5'
-
-    def __init__(self, md5, mtime):
-        self.mtime = mtime
-        self.md5 = md5
-
-    def update(self, md5, mtime):
-        self.mtime = mtime
-        self.md5 = md5
-
-    @staticmethod
-    def loadd(d):
-        mtime = d[StateEntry.PARAM_MTIME]
-        md5 = d[StateEntry.PARAM_MD5]
-        return StateEntry(md5, mtime)
-
-    def dumpd(self):
-        return {
-            self.PARAM_MD5: self.md5,
-            self.PARAM_MTIME: self.mtime,
-        }
-
-
 class StateDuplicateError(DvcException):
     pass
 
 
 class State(object):
+    VERSION = 0
     STATE_FILE = 'state'
-    STATE_LOCK_FILE = 'state.lock'
+    STATE_TABLE = 'state'
+    STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
+                         "mtime TEXT NOT NULL, " \
+                         "md5 TEXT NOT NULL, " \
+                         "timestamp TEXT NOT NULL"
 
-    def __init__(self, project):
+    STATE_INFO_TABLE = 'state_info'
+    STATE_INFO_TABLE_LAYOUT = 'count INTEGER'
+    STATE_INFO_ROW = 1
+
+    LINK_STATE_TABLE = 'link_state'
+    LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
+                              "inode INTEGER NOT NULL, " \
+                              "mtime TEXT NOT NULL"
+
+    STATE_ROW_LIMIT = 10000000
+    STATE_ROW_CLEANUP_QUOTA = 50
+
+    def __init__(self, project, config):
         self.project = project
         self.dvc_dir = project.dvc_dir
-        self._lock = threading.Lock()
-        if self.dvc_dir:
-            self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
-            self._lock_file = Lock(self.dvc_dir, self.STATE_LOCK_FILE)
-        else:
+        self.root_dir = project.root_dir
+
+        self.row_limit = 100
+        self.row_cleanup_quota = 50
+
+        c = config.get(Config.SECTION_STATE, {})
+        self.row_limit = c.get(Config.SECTION_STATE_ROW_LIMIT,
+                               self.STATE_ROW_LIMIT)
+        self.row_cleanup_quota = c.get(Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
+                                       self.STATE_ROW_CLEANUP_QUOTA)
+
+        if not self.dvc_dir:
             self.state_file = None
-            self._lock_file = threading.Lock()
-        self._db = self.load()
+            return
+
+        self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
+        self.db = None
+        self.c = None
+        self.inserts = 0
+
+    def __enter__(self):
+        self.load()
+
+    def __exit__(self, type, value, tb):
+        self.dump()
 
     @staticmethod
     def init(project):
@@ -74,137 +82,216 @@ class State(object):
 
         return actual.split('.')[0] != md5.split('.')[0]
 
-    def load(self):
-        if not self.state_file or not os.path.isfile(self.state_file):
-            return {}
+    def _load(self):
+        from dvc import VERSION
 
-        with open(self.state_file, 'r') as fd:
+        cmd = "PRAGMA user_version;"
+        self.c.execute(cmd)
+        ret = self.c.fetchall()
+        assert len(ret) == 1
+        assert len(ret[0]) == 1
+        assert isinstance(ret[0][0], int)
+        self.version = ret[0][0]
+
+        if self.version > self.VERSION:
+            msg = "You are using an old version '{}' of dvc that is using " \
+                  "state file version '{}' which is not compatible with the " \
+                  "state file version '{}' that is used in this projet. " \
+                  "Please upgrade right now!"
+            raise DvcException(msg.format(VERSION,
+                                          self.VERSION,
+                                          self.version))
+        elif self.version < self.VERSION:
+            msg = "State file version '{}' is too old. " \
+                  "Reformatting to the current version '{}'."
+            self.project.logger.warn(msg.format(self.version, self.VERSION))
+            cmd = "DROP TABLE IF EXISTS {};"
+            self.c.execute(cmd.format(self.STATE_TABLE))
+            self.c.execute(cmd.format(self.STATE_INFO_TABLE))
+            self.c.execute(cmd.format(self.LINK_STATE_TABLE))
+
+        # Check that the state file is indeed a database
+        cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
+        self.c.execute(cmd.format(self.STATE_TABLE,
+                                  self.STATE_TABLE_LAYOUT))
+        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
+                                  self.STATE_INFO_TABLE_LAYOUT))
+        self.c.execute(cmd.format(self.LINK_STATE_TABLE,
+                                  self.LINK_STATE_TABLE_LAYOUT))
+
+        cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
+              "WHERE NOT EXISTS (SELECT * FROM {})"
+        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
+                                  self.STATE_INFO_TABLE))
+
+        cmd = "PRAGMA user_version = {};"
+        self.c.execute(cmd.format(self.VERSION))
+
+    def load(self):
+        retries = 1
+        while True:
+            assert self.db is None
+            assert self.c is None
+            assert self.inserts == 0
+            self.db = sqlite3.connect(self.state_file)
+            self.c = self.db.cursor()
+
+            # Try loading once to check that the file is indeed a database
+            # and reformat it if it is not.
             try:
-                return json.load(fd)
-            except ValueError as exc:
-                Logger.error('Failed to load \'{}\''.format(self.state_file),
-                             exc)
-                return {}
+                self._load()
+                return
+            except sqlite3.DatabaseError:
+                self.c.close()
+                self.db.close()
+                self.db = None
+                self.c = None
+                self.inserts = 0
+                if retries > 0:
+                    os.unlink(self.state_file)
+                    retries -= 1
+                else:
+                    raise
 
     def dump(self):
-        if not self.state_file:
-            return
+        assert self.db is not None
 
-        with SignalHandler():
-            with open(self.state_file, 'w+') as fd:
-                json.dump(self._db, fd)
+        cmd = "SELECT count from {} WHERE rowid={}"
+        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
+                                  self.STATE_INFO_ROW))
+        ret = self.c.fetchall()
+        assert len(ret) == 1
+        assert len(ret[0]) == 1
+        count = ret[0][0] + self.inserts
+
+        if count > self.row_limit:
+            msg = "Cleaning up state. This might take a while."
+            self.project.logger.warn(msg)
+
+            delete = (count - self.row_limit)
+            delete += int(self.row_limit * (self.row_cleanup_quota/100.))
+            cmd = "DELETE FROM {} WHERE timestamp IN (" \
+                  "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
+            self.c.execute(cmd.format(self.STATE_TABLE,
+                                      self.STATE_TABLE,
+                                      delete))
+
+            self.c.execute("VACUUM")
+
+            cmd = "SELECT COUNT(*) FROM {}"
+
+            self.c.execute(cmd.format(self.STATE_TABLE))
+            ret = self.c.fetchall()
+            assert len(ret) == 1
+            assert len(ret[0]) == 1
+            count = ret[0][0]
+
+        cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
+        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
+                                  count,
+                                  self.STATE_INFO_ROW))
+
+        self.db.commit()
+        self.c.close()
+        self.db.close()
+        self.db = None
+        self.c = None
+        self.inserts = 0
 
     @staticmethod
     def mtime(path):
-        return str(int(nanotime.timestamp(os.path.getmtime(path))))
+        mtime = os.path.getmtime(path)
+        if os.path.isdir(path):
+            for root, dirs, files in os.walk(path):
+                for entry in list(dirs) + list(files):
+                    p = os.path.join(root, entry)
+                    m = os.path.getmtime(p)
+                    if m > mtime:
+                        mtime = m
+
+        return str(int(nanotime.timestamp(mtime)))
 
     @staticmethod
     def inode(path):
-        return str(System.inode(path))
+        return System.inode(path)
 
-    def _do_update(self, path, dump=True):
+    def _do_update(self, path):
         if not os.path.exists(path):
             return (None, None)
 
         mtime = self.mtime(path)
         inode = self.inode(path)
 
-        md5 = self._get(inode, mtime)
-        if md5:
-            return (md5, None)
+        cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
+                                                       inode)
 
-        md5, info = self._collect(path)
-        state = StateEntry(md5, mtime)
-        d = state.dumpd()
-
-        with self._lock:
-            with self._lock_file:
-                self._db[inode] = d
-
-                if dump:
-                    self.dump()
+        self.c.execute(cmd)
+        ret = self.c.fetchall()
+        if len(ret) == 0:
+            md5, info = self._collect(path)
+            cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
+                  'VALUES ({}, "{}", "{}", "{}")'
+            self.c.execute(cmd.format(self.STATE_TABLE,
+                                      inode,
+                                      mtime,
+                                      md5,
+                                      int(nanotime.timestamp(time.time()))))
+            self.inserts += 1
+        else:
+            assert len(ret) == 1
+            assert len(ret[0]) == 4
+            i, m, md5, timestamp = ret[0]
+            assert i == inode
+            if mtime != m:
+                md5, info = self._collect(path)
+                cmd = 'UPDATE {} SET ' \
+                      'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
+                      'WHERE inode = {}'
+                self.c.execute(cmd.format(self.STATE_TABLE,
+                                          mtime,
+                                          md5,
+                                          int(nanotime.timestamp(time.time())),
+                                          inode))
+            else:
+                info = None
+                cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
+                self.c.execute(cmd.format(self.STATE_TABLE,
+                                          int(nanotime.timestamp(time.time())),
+                                          inode))
 
         return (md5, info)
 
-    def update(self, path, dump=True):
-        return self._do_update(path, dump=dump)[0]
+    def update(self, path):
+        return self._do_update(path)[0]
 
-    def update_info(self, path, dump=True):
-        md5, info = self._do_update(path, dump=dump)
+    def update_info(self, path):
+        md5, info = self._do_update(path)
         if not info:
             info = self.project.cache.local.load_dir_cache(md5)
         return (md5, info)
 
-    def _get(self, inode, mtime):
-        with self._lock:
-            with self._lock_file:
-                d = self._db.get(inode, None)
-
-        if not d:
-            return None
-
-        state = StateEntry.loadd(d)
-        if mtime == state.mtime:
-            return state.md5
-
-        return None
-
-    def get(self, path):
-        mtime = self.mtime(path)
-        inode = self.inode(path)
-
-        return self._get(inode, mtime)
-
-
-class LinkStateEntry(object):
-    PARAM_MTIME = 'mtime'
-    PARAM_INODE = 'inode'
-
-    def __init__(self, inode, mtime):
-        self.mtime = mtime
-        self.inode = inode
-
-    @staticmethod
-    def loadd(d):
-        mtime = d[LinkStateEntry.PARAM_MTIME]
-        inode = d[LinkStateEntry.PARAM_INODE]
-        return LinkStateEntry(inode, mtime)
-
-    def dumpd(self):
-        return {
-            self.PARAM_INODE: self.inode,
-            self.PARAM_MTIME: self.mtime,
-        }
-
-
-class LinkState(State):
-    STATE_FILE = 'link.state'
-    STATE_LOCK_FILE = STATE_FILE + '.lock'
-
-    def __init__(self, project):
-        super(LinkState, self).__init__(project)
-        self.root_dir = project.root_dir
-
-    def update(self, path, dump=True):
+    def update_link(self, path):
         if not os.path.exists(path):
             return
 
         mtime = self.mtime(path)
         inode = self.inode(path)
+        relpath = os.path.relpath(path, self.root_dir)
 
-        with self._lock:
-            state = LinkStateEntry(inode, mtime)
-            d = state.dumpd()
-            self._db[os.path.relpath(path, self.root_dir)] = d
-            if dump:
-                with self._lock_file:
-                    self.dump()
+        cmd = 'REPLACE INTO {}(path, inode, mtime) ' \
+              'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE,
+                                               relpath,
+                                               inode,
+                                               mtime)
+        self.c.execute(cmd)
 
-    def _do_remove_unused(self, used):
-        items = self._db.copy().items()
-        for p, s in items:
+    def remove_unused_links(self, used):
+        unused = []
+
+        self.c.execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
+        for row in self.c:
+            p, i, m = row
             path = os.path.join(self.root_dir, p)
-            state = LinkStateEntry.loadd(s)
 
             if path in used:
                 continue
@@ -215,13 +302,11 @@ class LinkState(State):
             inode = self.inode(path)
             mtime = self.mtime(path)
 
-            if inode == state.inode and mtime == state.mtime:
+            if i == inode and m == mtime:
                 Logger.debug('Removing \'{}\' as unused link.'.format(path))
                 remove(path)
-                del self._db[p]
+                unused.append(p)
 
-    def remove_unused(self, used):
-        with self._lock:
-            with self._lock_file:
-                self._do_remove_unused(used)
-                self.dump()
+        for p in unused:
+            cmd = 'DELETE FROM {} WHERE path = "{}"'
+            self.c.execute(cmd.format(self.LINK_STATE_TABLE, p))

dvc/system.py

@@ -1,9 +1,4 @@
 import os
-import ctypes
-import reflink
-
-if os.name == 'nt':
-    import ntfsutils.hardlink as winlink
 
 
 class System(object):
@@ -13,13 +8,24 @@ class System(object):
 
     @staticmethod
     def hardlink(source, link_name):
+        import ctypes
         if System.is_unix():
-            return os.link(source, link_name)
+            os.link(source, link_name)
+            return
 
-        return winlink.create(source, link_name)
+        CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
+        CreateHardLink.argtypes = [ctypes.c_wchar_p,
+                                   ctypes.c_wchar_p,
+                                   ctypes.c_void_p]
+        CreateHardLink.restype = ctypes.wintypes.BOOL
+
+        res = CreateHardLink(link_name, source, None)
+        if res == 0:
+            raise ctypes.WinError()
 
     @staticmethod
     def symlink(source, link_name):
+        import ctypes
         if System.is_unix():
             return os.symlink(source, link_name)
 
@@ -35,33 +41,138 @@ class System(object):
             raise ctypes.WinError()
 
     @staticmethod
+    def _reflink_darwin(src, dst):
+        import ctypes
+
+        clib = ctypes.CDLL('libc.dylib')
+        if not hasattr(clib, 'clonefile'):
+            return -1
+
+        clonefile = clib.clonefile
+        clonefile.argtypes = [ctypes.c_char_p,
+                              ctypes.c_char_p,
+                              ctypes.c_int]
+        clonefile.restype = ctypes.c_int
+
+        return clonefile(ctypes.c_char_p(src.encode('utf-8')),
+                         ctypes.c_char_p(src.encode('utf-8')),
+                         ctypes.c_int(0))
+
+    @staticmethod
+    def _reflink_windows(src, dst):
+        return -1
+
+    @staticmethod
+    def _reflink_linux(src, dst):
+        import os
+        import fcntl
+
+        FICLONE = 0x40049409
+
+        s = open(src, 'r')
+        d = open(dst, 'w+')
+
+        try:
+            ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
+        except IOError:
+            s.close()
+            d.close()
+            os.unlink(dst)
+            raise
+
+        s.close()
+        d.close()
+
+        if ret != 0:
+            os.unlink(dst)
+
+        return ret
+
+    @staticmethod
     def reflink(source, link_name):
-        return reflink.reflink(source, link_name)
+        import platform
+        from dvc.exceptions import DvcException
+
+        system = platform.system()
+        try:
+            if system == 'Windows':
+                ret = System._reflink_windows(source, link_name)
+            elif system == 'Darwin':
+                ret = System._reflink_darwin(source, link_name)
+            elif system == 'Linux':
+                ret = System._reflink_linux(source, link_name)
+            else:
+                ret = -1
+        except IOError:
+            ret = -1
+
+        if ret != 0:
+            raise DvcException('Reflink is not supported')
 
-    # FIXME
-    # Temporary fix while waiting for the PR to be merged and released:
-    # https://github.com/sunshowers/ntfs/pull/11
     @staticmethod
     def getdirinfo(path):
-        from ntfsutils.fs import FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_READ
-        from ntfsutils.fs import OPEN_EXISTING, BY_HANDLE_FILE_INFORMATION
-        from ntfsutils.fs import CreateFile, WinError
-        from ntfsutils.fs import GetFileInformationByHandle, CloseHandle
+        import ctypes
+        from ctypes import c_void_p, c_wchar_p, Structure, WinError
+        from ctypes.wintypes import DWORD, HANDLE, POINTER, BOOL
+
+        FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+        FILE_SHARE_READ = 0x00000001
+        OPEN_EXISTING = 3
+
+        class FILETIME(Structure):
+            _fields_ = [("dwLowDateTime", DWORD),
+                        ("dwHighDateTime", DWORD)]
+
+        class BY_HANDLE_FILE_INFORMATION(Structure):
+            _fields_ = [("dwFileAttributes", DWORD),
+                        ("ftCreationTime", FILETIME),
+                        ("ftLastAccessTime", FILETIME),
+                        ("ftLastWriteTime", FILETIME),
+                        ("dwVolumeSerialNumber", DWORD),
+                        ("nFileSizeHigh", DWORD),
+                        ("nFileSizeLow", DWORD),
+                        ("nNumberOfLinks", DWORD),
+                        ("nFileIndexHigh", DWORD),
+                        ("nFileIndexLow", DWORD)]
+
         flags = FILE_FLAG_BACKUP_SEMANTICS
-        hfile = CreateFile(path,
-                           0,
-                           FILE_SHARE_READ,
-                           None,
-                           OPEN_EXISTING,
-                           flags,
-                           None)
+
+        func = ctypes.windll.kernel32.CreateFileW
+        func.argtypes = [c_wchar_p,
+                         DWORD,
+                         DWORD,
+                         c_void_p,
+                         DWORD,
+                         DWORD,
+                         HANDLE]
+        func.restype = HANDLE
+
+        hfile = func(path,
+                     0,
+                     FILE_SHARE_READ,
+                     None,
+                     OPEN_EXISTING,
+                     flags,
+                     None)
         if hfile is None:
             raise WinError()
+
+        func = ctypes.windll.kernel32.GetFileInformationByHandle
+        func.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
+        func.restype = BOOL
+
         info = BY_HANDLE_FILE_INFORMATION()
-        rv = GetFileInformationByHandle(hfile, info)
-        CloseHandle(hfile)
+        rv = func(hfile, info)
+
+        func = ctypes.windll.kernel32.CloseHandle
+        func.argtypes = [HANDLE]
+        func.restype = BOOL
+
+        func(hfile)
+
         if rv == 0:
             raise WinError()
+
         return info
 
     @staticmethod

dvc/updater.py

@@ -6,11 +6,11 @@ from dvc import VERSION_BASE
 from dvc.logger import Logger
 
 
-class Updater(object):
+class Updater(object):  # pragma: no cover
     URL = 'https://4ki8820rsf.execute-api.us-east-2.amazonaws.com/' \
           'prod/latest-version'
     UPDATER_FILE = 'updater'
-    TIMEOUT = 7 * 24 * 60 * 60  # every week
+    TIMEOUT = 24 * 60 * 60  # every day
     TIMEOUT_GET = 10
 
     def __init__(self, dvc_dir):

dvc/utils.py

@@ -2,6 +2,7 @@
 Helpers for other modules.
 """
 import os
+import sys
 import math
 import json
 import shutil
@@ -134,3 +135,24 @@ def to_chunks(l, jobs):
         n = 1
 
     return [l[x:x+n] for x in range(0, len(l), n)]
+
+
+# NOTE: Fix env variables modified by PyInstaller
+# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
+def fix_env(env):
+    if env is None:
+        env = os.environ.copy()
+    else:
+        env = env.copy()
+
+    # NOTE: Check if we are in a bundle
+    # https://pythonhosted.org/PyInstaller/runtime-information.html
+    if getattr(sys, 'frozen', False):
+        lp_key = 'LD_LIBRARY_PATH'
+        lp_orig = env.get(lp_key + '_ORIG', None)
+        if lp_orig is not None:
+            env[lp_key] = lp_orig
+        else:
+            env.pop(lp_key, None)
+
+    return env

requirements.txt

@@ -4,21 +4,22 @@ ply>=3.9 # See https://github.com/pyinstaller/pyinstaller/issues/1945
 configparser>=3.5.0
 zc.lockfile>=1.2.1
 future>=0.16.0
-google-cloud==0.32.0
+google-cloud-storage==1.12.0
 PyInstaller==3.3.1
 colorama>=0.3.9
 configobj>=5.0.6
-networkx==2.1
+networkx>=2.1
 pyyaml>=3.12
 gitpython>=2.1.8
-ntfsutils>=0.1.4
 setuptools>=34.0.0
 nanotime>=0.5.2
 pyasn1>=0.4.1
 schema>=0.6.7
 paramiko>=2.4.1
 jsonpath-rw==1.4.0
-reflink==0.2.0
 requests>=2.18.4
 wheel>=0.31.1
 futures>=3.2.0; python_version == "2.7"
+grandalf==0.6
+pydot>=1.2.4
+asciimatics>=1.10.0

scripts/ci/install.sh

@@ -3,9 +3,21 @@
 set -x
 set -e
 
-pip install --upgrade pip
-pip install -r requirements.txt
-pip install -r test-requirements.txt
+N_RETRIES=3
+
+function retry {
+    for i in $(seq $N_RETRIES); do
+        $@ && break
+    done
+}
+
+# NOTE: it is not uncommon for pip to hang on travis for what seems to be
+# networking issues. Thus, let's retry a few times to see if it will eventially
+# work or not.
+retry pip install --upgrade pip setuptools wheel
+retry pip install -r requirements.txt
+retry pip install -r test-requirements.txt
+
 git config --global user.email "[email protected]"
 git config --global user.name "DVC Tester"
 

scripts/ci/script.sh

@@ -6,13 +6,28 @@ set -e
 python -mtests
 
 if [[ "$TRAVIS_PULL_REQUEST" == "false" && \
-      "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
+      "$TRAVIS_SECURE_ENV_VARS" == "true" && \
+      "$(python -c 'import sys; print(sys.version_info[0])')" == '2' ]]; then
 	codecov
 fi
 
 if [[ ! -z "$TRAVIS_TAG" ]]; then
 	./scripts/build_package.sh
 
+        # Test version
+        pip uninstall -y dvc
+        if [ -x "$(command -v dvc)" ]; then
+            echo "ERROR: dvc command already exists!!!"
+            exit 1
+        fi
+
+        pip install dist/dvc-*.whl
+        if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
+            echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'"
+            exit 1
+        fi
+        pip uninstall -y dvc
+
         PY_VER=$(python -c 'import sys; print(sys.version_info[0:2])')
         if [[ "$PY_VER" != '(3, 7)' ]]; then
             ./scripts/build_posix.sh

setup.py

@@ -1,3 +1,4 @@
+import sys
 import platform
 from setuptools import setup, find_packages
 from distutils.errors import DistutilsPlatformError
@@ -11,23 +12,22 @@ install_requires = [
     "future>=0.16.0",
     "colorama>=0.3.9",
     "configobj>=5.0.6",
-    "networkx==2.1",
+    "networkx>=2.1",
     "pyyaml>=3.12",
     "gitpython>=2.1.8",
-    "ntfsutils>=0.1.4",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
     "schema>=0.6.7",
     "jsonpath-rw==1.4.0",
-    "reflink==0.2.0",
     "requests>=2.18.4",
-    "futures>=3.2.0; python_version == \"2.7\"",
+    "grandalf==0.6",
+    "asciimatics>=1.10.0",
 ]
 
 # Extra dependencies for remote integrations
 gs = [
-    "google-cloud==0.32.0",
+    "google-cloud-storage==1.12.0",
 ]
 s3 = [
     "boto3==1.7.4",
@@ -56,6 +56,8 @@ setup(
         's3': s3,
         'azure': azure,
         'ssh': ssh,
+        # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1
+        ':python_version=="2.7"': ['futures'],
     },
     keywords='data science, data version control, machine learning',
     classifiers=[
@@ -63,11 +65,11 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 3',
     ],
-    packages=find_packages(exclude=['bin', 'tests', 'functests']),
+    packages=find_packages(exclude=['tests']),
     include_package_data=True,
     url='http://dataversioncontrol.com',
     entry_points={
         'console_scripts': ['dvc = dvc.main:main']
     },
-    zip_safe=False
+    zip_safe=False,
 )

test-requirements.txt

@@ -1,9 +1,11 @@
 pytest>=3.3.1
 nose>=1.3.7
 mock>=2.0.0
-coverage
-codecov
+coverage>=4.5.1
+codecov>=2.0.15
 xmltodict>=0.11.0
 awscli>=1.14.18
 google-compute-engine
 pywin32; sys_platform == 'win32'
+Pygments
+collective.checkdocs

tests/__main__.py

@@ -7,4 +7,5 @@ os.chdir(REPO_ROOT)
 os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), os.getenv('PATH')))
 os.putenv('DVC_HOME', REPO_ROOT)
 
-check_call('nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage', shell=True)
+cmd = 'nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage'
+check_call(cmd, shell=True)

tests/basic_env.py

@@ -61,6 +61,15 @@ class TestGit(TestDir):
         self.git.index.commit('add code')
 
 
+class TestGitSubmodule(TestGit):
+    def setUp(self):
+        super(TestGitSubmodule, self).setUp()
+        subrepo = Repo.init()
+        subrepo_path = 'subrepo'
+        self.git.create_submodule(subrepo_path, subrepo_path, subrepo.git_dir)
+        self._pushd(subrepo_path)
+
+
 class TestDvc(TestGit):
     def setUp(self):
         super(TestDvc, self).setUp()

tests/test_add.py

@@ -1,6 +1,8 @@
 import os
 import stat
 import shutil
+import filecmp
+import tempfile
 
 from dvc.main import main
 from dvc.utils import file_md5
@@ -17,7 +19,10 @@ class TestAdd(TestDvc):
     def test(self):
         md5 = file_md5(self.FOO)[0]
 
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
 
         self.assertIsInstance(stage, Stage)
         self.assertTrue(os.path.isfile(stage.path))
@@ -38,8 +43,10 @@ class TestAddDirectory(TestDvc):
         dname = 'directory'
         os.mkdir(dname)
         self.create(os.path.join(dname, 'file'), 'file')
-        stage = self.dvc.add(dname)
-        self.assertNotEqual(stage, None)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(len(stage.outs), 1)
 
@@ -50,12 +57,29 @@ class TestAddDirectory(TestDvc):
             self.assertTrue('\\' not in info['relpath'])
 
 
+class TestAddDirectoryRecursive(TestDvc):
+    def test(self):
+        stages = self.dvc.add(self.DATA_DIR, recursive=True)
+        self.assertEqual(len(stages), 2)
+
+
+class TestAddCmdDirectoryRecursive(TestDvc):
+    def test(self):
+        ret = main(['add',
+                    '--recursive',
+                    self.DATA_DIR])
+        self.assertEqual(ret, 0)
+
+
 class TestAddDirectoryWithForwardSlash(TestDvc):
     def test(self):
         dname = 'directory/'
         os.mkdir(dname)
         self.create(os.path.join(dname, 'file'), 'file')
-        stage = self.dvc.add(dname)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         self.assertEquals(os.path.abspath('directory.dvc'), stage.path)
 
 class TestAddTrackedFile(TestDvc):
@@ -76,26 +100,54 @@ class TestAddDirWithExistingCache(TestDvc):
         os.mkdir(dname)
         shutil.copyfile(self.FOO, fname)
 
-        self.dvc.add(self.FOO)
-        self.dvc.add(dname)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
 
 
 class TestAddModifiedDir(TestDvc):
     def test(self):
-        self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
         os.unlink(self.DATA)
-        self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
 
 
 class TestAddFileInDir(TestDvc):
     def test(self):
-        stage = self.dvc.add(self.DATA_SUB)
+        stages = self.dvc.add(self.DATA_SUB)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
         self.assertNotEqual(stage, None)
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(len(stage.outs), 1)
         self.assertEqual(stage.relpath, self.DATA_SUB + '.dvc')
 
 
+class TestAddExternalLocalFile(TestDvc):
+    def test(self):
+        dname = tempfile.mkdtemp()
+        fname = os.path.join(dname, 'foo')
+        shutil.copyfile(self.FOO, fname)
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertNotEqual(stage, None)
+        self.assertEqual(len(stage.deps), 0)
+        self.assertEqual(len(stage.outs), 1)
+        self.assertEqual(stage.relpath, 'foo.dvc')
+        self.assertEqual(len(os.listdir(dname)), 1)
+        self.assertTrue(os.path.isfile(fname))
+        self.assertTrue(filecmp.cmp(fname, 'foo', shallow=False))
+
+
 class TestCmdAdd(TestDvc):
     def test(self):
         ret = main(['add',

tests/test_cache.py

@@ -103,3 +103,12 @@ class TestSharedCacheDir(TestDir):
         self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1)
         self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1)
         self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1)
+
+
+class TestCacheLinkType(TestDvc):
+    def test(self):
+        ret = main(['config', 'cache.type', 'reflink,copy'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['add', self.FOO])
+        self.assertEqual(ret, 0)

tests/test_checkout.py

@@ -16,7 +16,11 @@ class TestCheckout(TestRepro):
     def setUp(self):
         super(TestCheckout, self).setUp()
 
-        self.data_dir_stage = self.dvc.add(self.DATA_DIR)
+
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.data_dir_stage = stages[0]
+        self.assertTrue(self.data_dir_stage is not None)
 
         self.orig = 'orig'
         shutil.copy(self.FOO, self.orig)
@@ -74,7 +78,9 @@ class CheckoutBase(TestDvc):
     def commit_data_file(self, fname, content='random text'):
         with open(fname, 'w') as fd:
             fd.write(content)
-        self.dvc.add(fname)
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
         self.dvc.scm.add([fname + '.dvc', '.gitignore'])
         self.dvc.scm.commit('adding ' + fname)
 
@@ -93,7 +99,8 @@ class TestRemoveFilesWhenCheckout(CheckoutBase):
 
         # add the file into a separate branch
         self.dvc.scm.checkout(branch_1, True)
-        main(['checkout'])
+        ret = main(['checkout'])
+        self.assertEqual(ret, 0)
         self.commit_data_file(fname)
 
         # Checkout back in master
@@ -102,7 +109,8 @@ class TestRemoveFilesWhenCheckout(CheckoutBase):
 
         # Make sure `dvc checkout` removes the file
         # self.dvc.checkout()
-        main(['checkout'])
+        ret = main(['checkout'])
+        self.assertEqual(ret, 0)
         self.assertFalse(os.path.exists(fname))
 
 
@@ -145,11 +153,13 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
         self.commit_data_file(fname_master)
 
         self.dvc.scm.checkout(branch_1, True)
-        main(['checkout'])
+        ret = main(['checkout'])
+        self.assertEqual(ret, 0)
         self.commit_data_file(fname_branch)
 
         self.dvc.scm.checkout(branch_master)
-        main(['checkout'])
+        ret = main(['checkout'])
+        self.assertEqual(ret, 0)
 
         ignored = self.read_ignored()
 
@@ -157,7 +167,8 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
         self.assertIn(fname_master, ignored)
 
         self.dvc.scm.checkout(branch_1)
-        main(['checkout'])
+        ret = main(['checkout'])
+        self.assertEqual(ret, 0)
         ignored = self.read_ignored()
         self.assertIn(fname_branch, ignored)
 
@@ -181,7 +192,10 @@ class TestCheckoutEmptyDir(TestDvc):
         dname = 'empty_dir'
         os.mkdir(dname)
 
-        stage = self.dvc.add(dname)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         self.assertEqual(len(stage.outs), 1)
 
         stage.outs[0].remove()
@@ -191,3 +205,15 @@ class TestCheckoutEmptyDir(TestDvc):
 
         self.assertTrue(os.path.isdir(dname))
         self.assertEqual(len(os.listdir(dname)), 0)
+
+
+class TestCheckoutNotCachedFile(TestDvc):
+    def test(self):
+        cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
+
+        self.dvc.add(self.FOO)
+        stage = self.dvc.run(cmd=cmd,
+                             deps=[self.FOO, self.CODE],
+                             outs_no_cache=['out'])
+
+        self.dvc.checkout()

tests/test_data_cloud.py

@@ -1,5 +1,5 @@
 from subprocess import CalledProcessError
-from subprocess import check_output
+from subprocess import check_output, Popen
 from unittest import SkipTest
 import os
 import time
@@ -94,6 +94,13 @@ def _should_test_hdfs():
     except (CalledProcessError, IOError):
         return False
 
+    p = Popen('hadoop fs -ls hdfs://127.0.0.1/',
+              shell=True,
+              executable=os.getenv('SHELL'))
+    p.communicate()
+    if p.returncode != 0:
+        return False
+
     return True
 
 
@@ -204,65 +211,73 @@ class TestDataCloudBase(TestDvc):
     def _test_cloud(self):
         self._setup_cloud()
 
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         cache = stage.outs[0].cache
         info = stage.outs[0].dumpd()
         md5 = info['md5']
 
-        stage_dir = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage_dir = stages[0]
+        self.assertTrue(stage_dir is not None)
+
         cache_dir = stage_dir.outs[0].cache
         info_dir = stage_dir.outs[0].dumpd()
         md5_dir = info_dir['md5']
 
-        # Check status
-        sleep()
-        status = self.cloud.status([info], show_checksums=True)
-        self.assertEqual([(md5, STATUS_NEW)], status)
+        with self.cloud.project.state:
+            # Check status
+            sleep()
+            status = self.cloud.status([info], show_checksums=True)
+            self.assertEqual([(md5, STATUS_NEW)], status)
 
-        status_dir = self.cloud.status([info_dir], show_checksums=True)
-        self.assertTrue((md5_dir, STATUS_NEW) in status_dir)
+            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            self.assertTrue((md5_dir, STATUS_NEW) in status_dir)
 
-        # Push and check status
-        self.cloud.push([info])
-        self.assertTrue(os.path.exists(cache))
-        self.assertTrue(os.path.isfile(cache))
+            # Push and check status
+            self.cloud.push([info])
+            self.assertTrue(os.path.exists(cache))
+            self.assertTrue(os.path.isfile(cache))
 
-        self.cloud.push([info_dir])
-        self.assertTrue(os.path.isfile(cache_dir))
+            self.cloud.push([info_dir])
+            self.assertTrue(os.path.isfile(cache_dir))
 
-        sleep()
-        status = self.cloud.status([info], show_checksums=True)
-        self.assertEqual([(md5, STATUS_OK)], status)
+            sleep()
+            status = self.cloud.status([info], show_checksums=True)
+            self.assertEqual([(md5, STATUS_OK)], status)
 
-        status_dir = self.cloud.status([info_dir], show_checksums=True)
-        self.assertTrue((md5_dir, STATUS_OK) in status_dir)
+            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            self.assertTrue((md5_dir, STATUS_OK) in status_dir)
 
-        # Remove and check status
-        sleep()
-        shutil.rmtree(self.dvc.cache.local.cache_dir)
+            # Remove and check status
+            sleep()
+            shutil.rmtree(self.dvc.cache.local.cache_dir)
 
-        status = self.cloud.status([info], show_checksums=True)
-        self.assertEqual([(md5, STATUS_DELETED)], status)
+            status = self.cloud.status([info], show_checksums=True)
+            self.assertEqual([(md5, STATUS_DELETED)], status)
 
-        status_dir = self.cloud.status([info_dir], show_checksums=True)
-        self.assertEqual([(md5_dir, STATUS_DELETED)], status_dir)
+            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            self.assertEqual([(md5_dir, STATUS_DELETED)], status_dir)
 
-        # Pull and check status
-        self.cloud.pull([info])
-        self.assertTrue(os.path.exists(cache))
-        self.assertTrue(os.path.isfile(cache))
-        with open(cache, 'r') as fd:
-            self.assertEqual(fd.read(), self.FOO_CONTENTS)
+            # Pull and check status
+            self.cloud.pull([info])
+            self.assertTrue(os.path.exists(cache))
+            self.assertTrue(os.path.isfile(cache))
+            with open(cache, 'r') as fd:
+                self.assertEqual(fd.read(), self.FOO_CONTENTS)
 
-        self.cloud.pull([info_dir])
-        self.assertTrue(os.path.isfile(cache_dir))
+            self.cloud.pull([info_dir])
+            self.assertTrue(os.path.isfile(cache_dir))
 
-        sleep()
-        status = self.cloud.status([info], show_checksums=True)
-        self.assertEqual([(md5, STATUS_OK)], status)
+            sleep()
+            status = self.cloud.status([info], show_checksums=True)
+            self.assertEqual([(md5, STATUS_OK)], status)
 
-        status_dir = self.cloud.status([info_dir], show_checksums=True)
-        self.assertTrue((md5_dir, STATUS_OK) in status_dir)
+            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            self.assertTrue((md5_dir, STATUS_OK) in status_dir)
 
     def test(self):
         self._ensure_should_run()
@@ -328,9 +343,6 @@ class TestRemoteSSH(TestDataCloudBase):
     def _get_cloud_class(self):
         return RemoteSSH
 
-    def test(self):
-        super(TestRemoteSSH, self).test()
-
 
 class TestRemoteHDFS(TestDataCloudBase):
     def _should_test(self):
@@ -342,9 +354,6 @@ class TestRemoteHDFS(TestDataCloudBase):
     def _get_cloud_class(self):
         return RemoteHDFS
 
-    def test(self):
-        super(TestRemoteHDFS, self).test()
-
 
 class TestDataCloudCLIBase(TestDvc):
     def main(self, args):
@@ -358,10 +367,16 @@ class TestDataCloudCLIBase(TestDvc):
         else:
             args += []
 
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         cache = stage.outs[0].cache
 
-        stage_dir = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage_dir = stages[0]
+        self.assertTrue(stage_dir is not None)
         cache_dir = stage_dir.outs[0].cache
 
         # FIXME check status output

tests/test_gc.py

@@ -29,7 +29,8 @@ class TestGC(TestDvc):
         self._test_gc()
 
     def test_cli(self):
-        ret = main(['gc'])
+        ret = main(['gc', '-f'])
+        self.assertEqual(ret, 0)
         self._test_gc()
 
     def _test_gc(self):

tests/test_metrics.py

@@ -219,7 +219,9 @@ class TestCachedMetrics(TestDvc):
         with open('metrics.json', 'w+') as fd:
             json.dump({'metrics': branch}, fd)
 
-        stage = self.dvc.add('metrics.json')
+        stages = self.dvc.add('metrics.json')
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
         self.assertNotEqual(stage, None)
 
         self.dvc.scm.add(['.gitignore', 'metrics.json.dvc'])

tests/test_move.py

@@ -1,9 +1,10 @@
 import os
 
 from dvc.main import main
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, MoveNotDataSourceError
 
 from tests.basic_env import TestDvc
+from tests.test_repro import TestRepro
 
 
 class TestMove(TestDvc):
@@ -39,3 +40,15 @@ class TestCmdMove(TestDvc):
 
         ret = main(['move', 'non-existing-file', 'dst'])
         self.assertNotEqual(ret, 0)
+
+
+class TestMoveNotDataSource(TestRepro):
+    def test(self):
+        from dvc.project import Project
+
+        self.dvc = Project(self._root_dir)
+        with self.assertRaises(MoveNotDataSourceError):
+            self.dvc.move(self.file1, 'dst')
+
+        ret = main(['move', self.file1, 'dst'])
+        self.assertNotEqual(ret, 0)

tests/test_pipeline.py

@@ -1,9 +1,71 @@
 from dvc.main import main
 
-from tests.test_repro import TestRepro
+from tests.basic_env import TestDvc
+from tests.test_repro import TestRepro, TestReproChangedDeepData
+import os
+
+
+class TestPipelineShowSingle(TestDvc):
+    def setUp(self):
+        super(TestPipelineShowSingle, self).setUp()
+        self.stage = 'foo.dvc'
+        self.dotFile = 'graph.dot'
+        ret = main(['add', self.FOO])
+        self.assertEqual(ret, 0)
+
+    def test(self):
+        ret = main(['pipeline', 'show', self.stage])
+        self.assertEqual(ret, 0)
+
+    def test_commands(self):
+        ret = main(['pipeline', 'show', self.stage, '--commands'])
+        self.assertEqual(ret, 0)        
+
+    def test_outs(self):
+        ret = main(['pipeline', 'show', self.stage, '--outs'])
+        self.assertEqual(ret, 0)        
+
+    def test_ascii(self):
+        ret = main(['pipeline', 'show', '--ascii', self.stage])
+        self.assertEqual(ret, 0)
+
+    def test_dot(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_ascii_commands(self):
+        ret = main(['pipeline', 'show', '--ascii', self.stage, '--commands'])
+        self.assertEqual(ret, 0)
+
+    def test_ascii_outs(self):
+        ret = main(['pipeline', 'show', '--ascii', self.stage, '--outs'])
+        self.assertEqual(ret, 0)
+
+    def test_dot_commands(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, '--commands'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_dot_outs(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, '--outs'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_not_dvc_file(self):
+        ret = main(['pipeline', 'show', self.FOO])
+        self.assertNotEqual(ret, 0)
+
+    def test_non_existing(self):
+        ret = main(['pipeline', 'show', 'non-existing'])
+        self.assertNotEqual(ret, 0)
 
 
 class TestPipelineShow(TestRepro):
+    def setUp(self):
+        super(TestPipelineShow, self).setUp()
+        self.dotFile = 'graph.dot'
+
     def test(self):
         ret = main(['pipeline', 'show', self.file1_stage])
         self.assertEqual(ret, 0)
@@ -16,6 +78,86 @@ class TestPipelineShow(TestRepro):
         ret = main(['pipeline', 'show', self.file1_stage, '--outs'])
         self.assertEqual(ret, 0)        
 
+    def test_ascii(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+    def test_dot(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_ascii_commands(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+
+    def test_ascii_outs(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+
+    def test_dot_commands(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_dot_outs(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_not_dvc_file(self):
+        ret = main(['pipeline', 'show', self.file1])
+        self.assertNotEqual(ret, 0)
+
+    def test_non_existing(self):
+        ret = main(['pipeline', 'show', 'non-existing'])
+        self.assertNotEqual(ret, 0)
+
+
+class TestPipelineShowDeep(TestReproChangedDeepData):
+    def setUp(self):
+        super(TestPipelineShowDeep, self).setUp()
+        self.dotFile = 'graph.dot'
+
+    def test(self):
+        ret = main(['pipeline', 'show', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+    def test_commands(self):
+        ret = main(['pipeline', 'show', self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)        
+
+    def test_outs(self):
+        ret = main(['pipeline', 'show', self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)        
+
+    def test_ascii(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+    def test_dot(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_ascii_commands(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+
+    def test_ascii_outs(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+
+    def test_dot_commands(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_dot_outs(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_not_dvc_file(self):
         ret = main(['pipeline', 'show', self.file1])
         self.assertNotEqual(ret, 0)

tests/test_remote.py

@@ -1,4 +1,8 @@
+import os
+import configobj
+
 from dvc.main import main
+from dvc.project import Project
 from dvc.command.remote import CmdRemoteAdd
 from dvc.exceptions import UnsupportedRemoteError
 from dvc.remote import Remote
@@ -41,3 +45,43 @@ class TestRemote(TestDvc):
     def test_unsupported(self):
         with self.assertRaises(UnsupportedRemoteError):
             Remote(self.dvc, {Config.SECTION_REMOTE_URL: 'unsupported://url'})
+
+
+class TestRemoteRemoveDefault(TestDvc):
+    def test(self):
+        remote = 'mys3'
+        self.assertEqual(main(['remote', 'add', '--default', remote, 's3://bucket/name']), 0)
+        self.assertEqual(main(['remote', 'modify', remote, 'profile', 'default']), 0)
+        self.assertEqual(main(['config', '--local', 'core.remote', remote]), 0)
+
+        config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG))
+        local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL))
+        self.assertEqual(config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote)
+        self.assertEqual(local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote)
+
+        self.assertEqual(main(['remote', 'remove', remote]), 0)
+        config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG))
+        local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL))
+        section = Config.SECTION_REMOTE_FMT.format(remote)
+        self.assertTrue(section not in config.keys())
+
+        core = config.get(Config.SECTION_CORE, None)
+        if core is not None:
+            self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys())
+
+        core = local_config.get(Config.SECTION_CORE, None)
+        if core is not None:
+            self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys())
+
+
+class TestRemoteRemove(TestDvc):
+    def test(self):
+        ret = main(['config', 'core.jobs', '1'])
+        self.assertEqual(ret, 0)
+
+        remote = 'mys3'
+        ret = main(['remote', 'add', remote, 's3://bucket/name'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['remote', 'remove', remote])
+        self.assertEqual(ret, 0)

tests/test_remove.py

@@ -1,8 +1,7 @@
 import os
 
 from dvc.main import main
-from dvc.stage import Stage
-from dvc.project import NotDvcFileError
+from dvc.stage import Stage, StageFileDoesNotExistError
 from dvc.command.remove import CmdRemove
 
 from tests.basic_env import TestDvc
@@ -10,7 +9,10 @@ from tests.basic_env import TestDvc
 
 class TestRemove(TestDvc):
     def test(self):
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         stage_removed = self.dvc.remove(stage.path, outs_only=True)
 
         self.assertIsInstance(stage_removed, Stage)
@@ -26,13 +28,16 @@ class TestRemove(TestDvc):
 
 class TestRemoveNonExistentFile(TestDvc):
     def test(self):
-        with self.assertRaises(NotDvcFileError) as cx:
+        with self.assertRaises(StageFileDoesNotExistError) as cx:
             self.dvc.remove('non_existent_dvc_file')
 
 
 class TestRemoveDirectory(TestDvc):
     def test(self):
-        stage_add = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage_add = stages[0]
+        self.assertTrue(stage_add is not None)
         stage_removed = self.dvc.remove(stage_add.path)
         self.assertEqual(stage_add.path, stage_removed.path)
         self.assertFalse(os.path.exists(self.DATA_DIR))
@@ -41,7 +46,10 @@ class TestRemoveDirectory(TestDvc):
 
 class TestCmdRemove(TestDvc):
     def test(self):
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         ret = main(['remove',
                     stage.path])
         self.assertEqual(ret, 0)

tests/test_repro.py

@@ -14,10 +14,11 @@ from google.cloud import storage as gc
 
 from dvc.main import main
 from dvc.command.repro import CmdRepro
-from dvc.project import Project, ReproductionError, NotDvcFileError
+from dvc.project import Project, ReproductionError
 from dvc.utils import file_md5
 from dvc.remote.local import RemoteLOCAL
-from dvc.stage import Stage
+from dvc.stage import Stage, StageFileDoesNotExistError
+from dvc.system import System
 
 from tests.basic_env import TestDvc
 from tests.test_data_cloud import _should_test_aws, TEST_AWS_REPO_BUCKET
@@ -30,7 +31,10 @@ class TestRepro(TestDvc):
     def setUp(self):
         super(TestRepro, self).setUp()
 
-        self.foo_stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.foo_stage = stages[0]
+        self.assertTrue(self.foo_stage is not None)
 
         self.file1 = 'file1'
         self.file1_stage = self.file1 + '.dvc'
@@ -50,7 +54,10 @@ class TestReproFail(TestRepro):
 
 class TestReproDepUnderDir(TestDvc):
     def test(self):
-        self.dir_stage = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.dir_stage = stages[0]
+        self.assertTrue(self.dir_stage is not None)
 
         sleep()
         
@@ -134,28 +141,131 @@ class TestReproDry(TestReproChangedData):
         self.assertTrue(len(stages), 2)
         self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
 
-        ret = main(['repro', '--dry'])
+        ret = main(['repro', '--dry', self.file1_stage])
+        self.assertEqual(ret, 0)
         self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
 
 
-class TestReproChangedDeepData(TestReproChangedData):
+class TestReproUpToDate(TestRepro):
     def test(self):
-        file2 = 'file2'
-        file2_stage = file2 + '.dvc'
-        self.dvc.run(fname=file2_stage,
-                     outs=[file2],
-                     deps=[self.file1, self.CODE],
-                     cmd='python {} {} {}'.format(self.CODE, self.file1, file2))
+        ret = main(['repro', self.file1_stage])
+        self.assertEqual(ret, 0)
 
+
+class TestReproDryNoExec(TestDvc):
+    def test(self):
+        deps = []
+        for d in range(3):
+            idir = 'idir{}'.format(d)
+            odir = 'odir{}'.format(d)
+
+            deps.append('-d')
+            deps.append(odir)
+
+            os.mkdir(idir)
+
+            f = os.path.join(idir, 'file')
+            with open(f, 'w+') as fobj:
+                fobj.write(str(d))
+
+            if os.name == 'nt':
+                cp = 'copy'
+            else:
+                cp = 'cp'
+
+            ret = main(['run',
+                        '--no-exec',
+                        '-d', idir,
+                        '-o', odir,
+                        "python -c 'import shutil; "
+                        "shutil.copytree(\"{}\", \"{}\")'".format(idir, odir)])
+            self.assertEqual(ret, 0)
+
+        ret = main(['run', '--no-exec', '-f', 'Dvcfile'] + deps)
+        self.assertEqual(ret, 0)
+
+        ret = main(['repro', '--dry'])
+        self.assertEqual(ret, 0)
+
+
+class TestReproChangedDeepData(TestReproChangedData):
+    def setUp(self):
+        super(TestReproChangedDeepData, self).setUp()
+
+        self.file2 = 'file2'
+        self.file2_stage = self.file2 + '.dvc'
+        self.dvc.run(fname=self.file2_stage,
+                     outs=[self.file2],
+                     deps=[self.file1, self.CODE],
+                     cmd='python {} {} {}'.format(self.CODE,
+                                                  self.file1,
+                                                  self.file2))
+
+    def test(self):
         self.swap_foo_with_bar()
 
-        stages = self.dvc.reproduce(file2_stage)
+        stages = self.dvc.reproduce(self.file2_stage)
 
         self.assertTrue(filecmp.cmp(self.file1, self.BAR, shallow=False))
-        self.assertTrue(filecmp.cmp(file2, self.BAR, shallow=False))
+        self.assertTrue(filecmp.cmp(self.file2, self.BAR, shallow=False))
         self.assertEqual(len(stages), 3)
 
 
+class TestReproPipeline(TestReproChangedDeepData):
+    def test(self):
+        stages = self.dvc.reproduce(self.file1_stage,
+                                    force=True,
+                                    pipeline=True)
+        self.assertEqual(len(stages), 3)
+
+    def test_cli(self):
+        ret = main(['repro', '--pipeline', '-f', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+
+class TestReproPipelines(TestDvc):
+    def setUp(self):
+        super(TestReproPipelines, self).setUp()
+
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.foo_stage = stages[0]
+        self.assertTrue(self.foo_stage is not None)
+
+        stages = self.dvc.add(self.BAR)
+        self.assertEqual(len(stages), 1)
+        self.bar_stage = stages[0]
+        self.assertTrue(self.bar_stage is not None)
+
+        self.file1 = 'file1'
+        self.file1_stage = self.file1 + '.dvc'
+        self.dvc.run(fname=self.file1_stage,
+                     outs=[self.file1],
+                     deps=[self.FOO, self.CODE],
+                     cmd='python {} {} {}'.format(self.CODE, self.FOO, self.file1))
+
+        self.file2 = 'file2'
+        self.file2_stage = self.file2 + '.dvc'
+        self.dvc.run(fname=self.file2_stage,
+                     outs=[self.file2],
+                     deps=[self.BAR, self.CODE],
+                     cmd='python {} {} {}'.format(self.CODE, self.BAR, self.file2))
+
+
+    def test(self):
+        stages = self.dvc.reproduce(all_pipelines=True, force=True)
+        self.assertEqual(len(stages), 4)
+        names = [stage.relpath for stage in stages]
+        self.assertTrue(self.foo_stage.relpath in names)
+        self.assertTrue(self.bar_stage.relpath in names)
+        self.assertTrue(self.file1_stage in names)
+        self.assertTrue(self.file2_stage in names)
+
+    def test_cli(self):
+        ret = main(['repro', '-f', '-P'])
+        self.assertEqual(ret, 0)
+
+
 class TestReproLocked(TestReproChangedData):
     def test(self):
         file2 = 'file2'
@@ -180,7 +290,7 @@ class TestReproLocked(TestReproChangedData):
         self.assertEqual(len(stages), 3)
 
     def test_non_existing(self):
-        with self.assertRaises(NotDvcFileError):
+        with self.assertRaises(StageFileDoesNotExistError):
             self.dvc.lock_stage('non-existing-stage')
 
         ret = main(['lock', 'non-existing-stage'])
@@ -243,6 +353,55 @@ class TestReproChangedDir(TestDvc):
         self.assertEqual(len(stages), 1)
 
 
+class TestReproChangedDirData(TestDvc):
+    def test(self):
+        dir_name = 'dir'
+        dir_code = 'dir_code.py'
+
+        sleep()
+
+        with open(dir_code, 'w+') as fd:
+            fd.write("import os; import sys; import shutil; shutil.copytree(sys.argv[1], sys.argv[2])")
+
+        sleep()
+            
+        stage = self.dvc.run(outs=[dir_name],
+                             deps=[self.DATA_DIR, dir_code],
+                             cmd="python {} {} {}".format(dir_code,
+                                                          self.DATA_DIR,
+                                                          dir_name))
+        self.assertTrue(stage is not None)
+
+        stages = self.dvc.reproduce(stage.path)
+        self.assertEqual(len(stages), 0)
+
+        sleep()
+
+        with open(self.DATA_SUB, 'a') as fd:
+            fd.write('add')
+
+        sleep()
+        
+        stages = self.dvc.reproduce(stage.path)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+        sleep()
+        
+        # Check that dvc indeed registers changed output dir
+        shutil.move(self.BAR, dir_name)
+        sleep()
+        stages = self.dvc.reproduce(stage.path)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+
+        # Check that dvc registers mtime change for the directory.
+        System.hardlink(self.DATA_SUB, self.DATA_SUB + '.lnk')
+        sleep()
+        stages = self.dvc.reproduce(stage.path)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+
+
 class TestReproMissingMd5InStageFile(TestRepro):
     def test(self):
         with open(self.file1_stage, 'r') as fd:
@@ -373,6 +532,8 @@ class TestReproExternalBase(TestDvc):
 
         sleep()
 
+        self.dvc.status()
+
         stages = self.dvc.reproduce(import_stage.path)
         self.assertEqual(len(stages), 1)
         self.assertTrue(os.path.exists('import'))

tests/test_run.py

@@ -3,8 +3,9 @@ import filecmp
 
 from dvc.main import main
 from dvc.utils import file_md5
-from dvc.stage import Stage
+from dvc.stage import Stage, StageFileBadNameError, MissingDep
 from dvc.command.run import CmdRun
+from dvc.exceptions import OutputDuplicationError
 
 from tests.basic_env import TestDvc
 
@@ -15,7 +16,7 @@ class TestRun(TestDvc):
         deps = [self.FOO, self.CODE]
         outs = [os.path.join(self.dvc.root_dir, 'out')]
         outs_no_cache = []
-        fname = os.path.join(self.dvc.root_dir, 'out.dvc')
+        fname = 'out.dvc'
         cwd = os.curdir
 
         self.dvc.add(self.FOO)
@@ -35,6 +36,14 @@ class TestRun(TestDvc):
         self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])  
         self.assertTrue(stage.path, fname)
 
+        with self.assertRaises(OutputDuplicationError):
+            stage = self.dvc.run(cmd=cmd,
+                                 deps=deps,
+                                 outs=outs,
+                                 outs_no_cache=outs_no_cache,
+                                 fname='duplicate' + fname,
+                                 cwd=cwd)
+
 
 class TestRunEmpty(TestDvc):
     def test(self):
@@ -46,6 +55,37 @@ class TestRunEmpty(TestDvc):
                      cwd=os.curdir)
 
 
+class TestRunMissingDep(TestDvc):
+    def test(self):
+        with self.assertRaises(MissingDep):
+            self.dvc.run(cmd='',
+                         deps=['non-existing-dep'],
+                         outs=[],
+                         outs_no_cache=[],
+                         fname='empty.dvc',
+                         cwd=os.curdir)
+
+
+class TestRunBadStageFilename(TestDvc):
+    def test(self):
+        with self.assertRaises(StageFileBadNameError):
+            self.dvc.run(cmd='',
+                         deps=[],
+                         outs=[],
+                         outs_no_cache=[],
+                         fname='empty',
+                         cwd=os.curdir)
+
+        with self.assertRaises(StageFileBadNameError):
+            self.dvc.run(cmd='',
+                         deps=[],
+                         outs=[],
+                         outs_no_cache=[],
+                         fname=os.path.join(self.DATA_DIR, 'empty.dvc'),
+                         cwd=os.curdir)
+
+
+
 class TestRunNoExec(TestDvc):
     def test(self):
         self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, 'out'),

tests/test_scm.py

@@ -4,7 +4,7 @@ from git import Repo
 
 from dvc.scm import SCM, Base, Git
 
-from tests.basic_env import TestDir, TestGit
+from tests.basic_env import TestDir, TestGit, TestGitSubmodule
 
 
 class TestSCM(TestDir):
@@ -27,6 +27,20 @@ class TestSCMGit(TestGit):
         self.assertTrue('foo' in self.git.git.ls_files())
 
 
+class TestSCMGitSubmodule(TestGitSubmodule):
+    def test_git_submodule(self):
+        self.assertIsInstance(SCM(os.curdir), Git)
+
+    def test_is_submodule(self):
+        self.assertTrue(Git.is_submodule(os.curdir))
+
+    def test_commit_in_submodule(self):
+        G = Git(self._root_dir)
+        G.add(['foo'])
+        G.commit('add')
+        self.assertTrue('foo' in self.git.git.ls_files())
+
+
 class TestIgnore(TestGit):
     def _count_gitignore(self):
         with open(Git.GITIGNORE, 'r') as fd:
@@ -35,13 +49,14 @@ class TestIgnore(TestGit):
 
     def test(self):
         git = Git(self._root_dir)
+        foo = os.path.join(self._root_dir, self.FOO)
 
-        git.ignore('foo')
+        git.ignore(foo)
         self.assertTrue(os.path.isfile(Git.GITIGNORE))
         self.assertEqual(self._count_gitignore(), 1)
 
-        git.ignore('foo')
+        git.ignore(foo)
         self.assertEqual(self._count_gitignore(), 1)
 
-        git.ignore_remove('foo')
+        git.ignore_remove(foo)
         self.assertEqual(self._count_gitignore(), 0)

tests/test_state.py

@@ -3,7 +3,7 @@ import stat
 import time
 
 from dvc.system import System
-from dvc.state import State, StateEntry
+from dvc.state import State
 from dvc.utils import file_md5
 
 from tests.basic_env import TestDvc
@@ -16,31 +16,26 @@ class TestState(TestDvc):
         mtime = os.path.getmtime(path)
         inode = System.inode(path)
 
-        state = State(self.dvc)
+        state = State(self.dvc, self.dvc.config._config)
 
-        entry_md5 = state.update(path)
-        self.assertEqual(entry_md5, md5)
+        with state:
+            entry_md5 = state.update(path)
+            self.assertEqual(entry_md5, md5)
 
-        # Sleep some time to simulate realistic behavior.
-        # Some filesystems have a bad date resolution for
-        # mtime(i.e. 1sec for HFS) that cause problems with
-        # our 'state' system not being able to distinguish
-        # files that were modified within that delta.
-        time.sleep(1)        
+            # Sleep some time to simulate realistic behavior.
+            # Some filesystems have a bad date resolution for
+            # mtime(i.e. 1sec for HFS) that cause problems with
+            # our 'state' system not being able to distinguish
+            # files that were modified within that delta.
+            time.sleep(1)        
 
-        os.unlink(path)
-        with open(path, 'w+') as fd:
-            fd.write('1')
+            os.unlink(path)
+            with open(path, 'w+') as fd:
+                fd.write('1')
 
-        md5 = file_md5(path)[0]
-        mtime = os.path.getmtime(path)
-        inode = System.inode(path)
+            md5 = file_md5(path)[0]
+            mtime = os.path.getmtime(path)
+            inode = System.inode(path)
 
-        entry_md5 = state.update(path)
-        self.assertEqual(entry_md5, md5)
-
-        # Reload state db to make sure that it stays the same
-        old_db = state._db
-        state = State(self.dvc)
-        new_db = state._db
-        self.assertEqual(old_db, new_db)
+            entry_md5 = state.update(path)
+            self.assertEqual(entry_md5, md5)