dvc: update to 0.18.7

Misc

.coveragerc

@@ -8,3 +8,4 @@ exclude_lines =
     pass
     raise NotImplementedError
     # pragma: no cover
+    #  pragma: no cover

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,10 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+
+**Please provide information about your setup**
+DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,6 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+cache: pip
 sudo: required
 dist: trusty
 group: deprecated-2017Q4

README.rst

@@ -4,7 +4,7 @@
 
 `Website <https://dvc.org>`_ • `Docs <https://dvc.org/doc>`_ • `Discuss <https://discuss.dvc.org/>`_ •
 `Blog <https://blog.dataversioncontrol.com/>`_ • `Twitter <https://twitter.com/iterativeai>`_ •
-`Tutorial <https://dvc.org/doc/tutorial>`_
+`Tutorial <https://dvc.org/doc/tutorial>`_ • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
 .. image:: https://travis-ci.org/iterative/dvc.svg?branch=master
   :target: https://travis-ci.org/iterative/dvc
@@ -42,6 +42,17 @@ data file suffixes and prefixes.
 .. contents:: **Contents**
   :backlinks: none
 
+How DVC works
+=============
+
+DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
+To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
+network storage (via SSH, for example).
+
+.. image:: https://dvc.org/static/img/flow.gif
+   :target: https://dvc.org/static/img/flow.gif
+   :alt: how_dvc_works
+
 Quick start
 ===========
 
@@ -88,13 +99,13 @@ Homebrew
 
 .. code-block:: bash
 
-   brew install iterative/homebrew-dvc/dvc
+   $ brew install iterative/homebrew-dvc/dvc
 
 or:
 
 .. code-block:: bash
 
-   brew cask install iterative/homebrew-dvc/dvc
+   $ brew cask install iterative/homebrew-dvc/dvc
 
 Related technologies
 ====================
@@ -112,23 +123,48 @@ Related technologies
 #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-How DVC works
-=============
-
-DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
-To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
-network storage (via SSH, for example).
-
-.. image:: https://dvc.org/static/img/flow.png
-   :target: https://dvc.org/static/img/flow.png
-   :alt: how_dvc_works
-
-
 Contributing
 ============
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
 details.
 
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+  :alt: 0
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+  :alt: 1
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+  :alt: 2   
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3   
+  :alt: 3
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+  :alt: 4   
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+  :alt: 5
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+  :alt: 6   
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+  :alt: 7
+
+Mailing List
+=========
+
+Want to stay up to date? Want to help improve DVC by participating in our ocassional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic.
+
 Copyright
 =========
 

dvc/__init__.py

@@ -4,8 +4,10 @@ DVC
 Make your data science projects reproducible and shareable.
 """
 import os
+import warnings
 
-VERSION_BASE = '0.16.0'
+
+VERSION_BASE = '0.18.7'
 __version__ = VERSION_BASE
 
 PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
@@ -47,3 +49,11 @@ else:
         pass
 
 VERSION = __version__
+
+
+# Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
+# We don't directly import numpy, but our dependency networkx does, causing
+# these warnings in some environments. Luckily these warnings are benign and
+# we can simply ignore them so that they don't show up when you are using dvc.
+warnings.filterwarnings("ignore", message="numpy.dtype size changed")
+warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

dvc/cli.py

@@ -26,6 +26,7 @@ from dvc.command.install import CmdInstall
 from dvc.command.root import CmdRoot
 from dvc.command.lock import CmdLock, CmdUnlock
 from dvc.command.pipeline import CmdPipelineShow
+from dvc.logger import Logger
 from dvc import VERSION
 
 
@@ -39,6 +40,19 @@ def _fix_subparsers(subparsers):
         subparsers.dest = 'cmd'
 
 
+class DvcParser(argparse.ArgumentParser):
+    def error(self, message):
+        sys.stderr.write('{}{}\n'.format(Logger.error_prefix(), message))
+        self.print_help()
+        sys.exit(2)
+
+
+class VersionAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        print(VERSION)
+        sys.exit(0)
+
+
 def parse_args(argv=None):
     # Common args
     parent_parser = argparse.ArgumentParser(add_help=False)
@@ -57,17 +71,18 @@ def parse_args(argv=None):
 
     # Main parser
     desc = 'Data Version Control'
-    parser = argparse.ArgumentParser(
-                        prog='dvc',
-                        description=desc,
-                        parents=[parent_parser],
-                        formatter_class=argparse.RawTextHelpFormatter)
+    parser = DvcParser(prog='dvc',
+                       description=desc,
+                       parents=[parent_parser],
+                       formatter_class=argparse.RawTextHelpFormatter)
 
+    # NOTE: On some python versions action='version' prints to stderr
+    # instead of stdout https://bugs.python.org/issue18920
     parser.add_argument('-V',
                         '--version',
-                        action='version',
-                        version='%(prog)s ' + VERSION,
-                        help='Show program\'s version')
+                        action=VersionAction,
+                        nargs=0,
+                        help='Show program\'s version.')
 
     # Sub commands
     subparsers = parser.add_subparsers(
@@ -75,84 +90,104 @@ def parse_args(argv=None):
                         metavar='COMMAND',
                         dest='cmd',
                         help='Use dvc COMMAND --help for command-specific '
-                             'help')
+                             'help.')
 
     _fix_subparsers(subparsers)
 
     # Init
+    INIT_HELP = 'Initialize dvc over a directory ' \
+                '(should already be a git dir).'
     init_parser = subparsers.add_parser(
                         'init',
                         parents=[parent_parser],
-                        help='Initialize dvc over a directory '
-                             '(should already be a git dir)')
+                        description=INIT_HELP,
+                        help=INIT_HELP)
     init_parser.add_argument(
                         '--no-scm',
                         action='store_true',
                         default=False,
                         help="Initiate dvc in directory that is "
-                             "not tracked by any scm tool(e.g. git)")
+                             "not tracked by any scm tool(e.g. git).")
     init_parser.add_argument(
                         '-f',
                         '--force',
                         action='store_true',
                         default=False,
-                        help='Force initialization even if \'.dvc\' exists')
+                        help="Overwrite '.dvc' if it exists. Will remove "
+                             "all local cache.")
     init_parser.set_defaults(func=CmdInit)
 
     # Destroy
+    DESTROY_HELP = "Destroy dvc. Will remove all project's information, " \
+                   "data files and cache."
     destroy_parser = subparsers.add_parser(
                         'destroy',
                         parents=[parent_parser],
-                        help='Destroy dvc')
+                        description=DESTROY_HELP,
+                        help=DESTROY_HELP)
     destroy_parser.add_argument(
                         '-f',
                         '--force',
                         action='store_true',
                         default=False,
-                        help='Force destruction')
+                        help='Force destruction.')
     destroy_parser.set_defaults(func=CmdDestroy)
 
     # Add
+    ADD_HELP = 'Add files/directories to dvc.'
     add_parser = subparsers.add_parser(
                         'add',
                         parents=[parent_parser],
-                        help='Add files/directories to dvc')
+                        description=ADD_HELP,
+                        help=ADD_HELP)
+    add_parser.add_argument(
+                        '-R',
+                        '--recursive',
+                        action='store_true',
+                        default=False,
+                        help='Recursively add each file under the directory.')
     add_parser.add_argument(
                         'targets',
                         nargs='+',
-                        help='Input files/directories')
+                        help='Input files/directories.')
     add_parser.set_defaults(func=CmdAdd)
 
     # Import
+    IMPORT_HELP = 'Import files from URL.'
     import_parser = subparsers.add_parser(
                         'import',
                         parents=[parent_parser],
-                        help='Import files from URL')
+                        description=IMPORT_HELP,
+                        help=IMPORT_HELP)
     import_parser.add_argument(
                         'url',
-                        help='URL')
+                        help='URL.')
     import_parser.add_argument(
                         'out',
-                        help='Output')
+                        help='Output.')
     import_parser.set_defaults(func=CmdImport)
 
     # Checkout
+    CHECKOUT_HELP = 'Checkout data files from cache.'
     checkout_parser = subparsers.add_parser(
                         'checkout',
                         parents=[parent_parser],
-                        help='Checkout data files from cache')
+                        description=CHECKOUT_HELP,
+                        help=CHECKOUT_HELP)
     checkout_parser.add_argument(
                         'targets',
                         nargs='*',
-                        help='DVC files')
+                        help='DVC files.')
     checkout_parser.set_defaults(func=CmdCheckout)
 
     # Run
+    RUN_HELP = 'Generate a stage file from a given ' \
+               'command and execute the command.'
     run_parser = subparsers.add_parser(
                         'run',
                         parents=[parent_parser],
-                        help='Generate a stage file from a given '
-                             'command and execute the command')
+                        description=RUN_HELP,
+                        help=RUN_HELP)
     run_parser.add_argument(
                         '-d',
                         '--deps',
@@ -182,23 +217,28 @@ def parse_args(argv=None):
     run_parser.add_argument(
                         '-f',
                         '--file',
-                        help='Specify name of the state file')
+                        help="Specify name of the stage file. It should be "
+                             "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+                             "'prepare.dvc', 'clean.dvc', etc) in order for "
+                             "dvc to be able to find it later. By default "
+                             "the first output basename + .dvc is used as "
+                             "a stage filename.")
     run_parser.add_argument(
                         '-c',
                         '--cwd',
                         default=os.path.curdir,
                         help='Directory to run your command and place state '
-                             'file in')
+                             'file in.')
     run_parser.add_argument(
                         '--no-exec',
                         action='store_true',
                         default=False,
                         help="Only create stage file without actually "
-                             "running it")
+                             "running it.")
     run_parser.add_argument(
                         'command',
                         nargs=argparse.REMAINDER,
-                        help='Command or command file to execute')
+                        help='Command or command file to execute.')
     run_parser.set_defaults(func=CmdRun)
 
     # Parent parser used in pull/push/status
@@ -223,14 +263,16 @@ def parse_args(argv=None):
                         help='DVC files.')
 
     # Pull
+    PULL_HELP = 'Pull data files from the cloud.'
     pull_parser = subparsers.add_parser(
                         'pull',
                         parents=[parent_cache_parser],
-                        help='Pull data files from the cloud')
+                        description=PULL_HELP,
+                        help=PULL_HELP)
     pull_parser.add_argument(
                         '-r',
                         '--remote',
-                        help='Remote repository to pull from')
+                        help='Remote repository to pull from.')
     pull_parser.add_argument(
                         '-a',
                         '--all-branches',
@@ -240,14 +282,16 @@ def parse_args(argv=None):
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
+    PUSH_HELP = 'Push data files to the cloud.'
     push_parser = subparsers.add_parser(
                         'push',
                         parents=[parent_cache_parser],
-                        help='Push data files to the cloud')
+                        description=PUSH_HELP,
+                        help=PUSH_HELP)
     push_parser.add_argument(
                         '-r',
                         '--remote',
-                        help='Remote repository to push to')
+                        help='Remote repository to push to.')
     push_parser.add_argument(
                         '-a',
                         '--all-branches',
@@ -257,14 +301,16 @@ def parse_args(argv=None):
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
+    FETCH_HELP = 'Fetch data files from the cloud.'
     fetch_parser = subparsers.add_parser(
                         'fetch',
                         parents=[parent_cache_parser],
-                        help='Fetch data files from the cloud')
+                        description=FETCH_HELP,
+                        help=FETCH_HELP)
     fetch_parser.add_argument(
                         '-r',
                         '--remote',
-                        help='Remote repository to fetch from')
+                        help='Remote repository to fetch from.')
     fetch_parser.add_argument(
                         '-a',
                         '--all-branches',
@@ -274,33 +320,35 @@ def parse_args(argv=None):
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
+    STATUS_HELP = 'Show the project status.'
     status_parser = subparsers.add_parser(
                         'status',
                         parents=[parent_cache_parser],
-                        help='Show the project status')
+                        description=STATUS_HELP,
+                        help=STATUS_HELP)
     status_parser.add_argument(
                         '-c',
                         '--cloud',
                         action='store_true',
                         default=False,
                         help='Show status of a local cache compared to a '
-                             'remote repository')
+                             'remote repository.')
     status_parser.add_argument(
                         '-r',
                         '--remote',
-                        help='Remote repository to compare local cache to')
+                        help='Remote repository to compare local cache to.')
     status_parser.set_defaults(func=CmdDataStatus)
 
     # Repro
+    REPRO_HELP = 'Reproduce DVC file. Default file name - \'Dvcfile\'.'
     repro_parser = subparsers.add_parser(
                         'repro',
                         parents=[parent_parser],
-                        help='Reproduce DVC file. Default file name '
-                             '- \'Dvcfile\'')
+                        description=REPRO_HELP,
+                        help=REPRO_HELP)
     repro_parser.add_argument(
                         'targets',
                         nargs='*',
-                        default=['Dvcfile'],
                         help='DVC file to reproduce.')
     repro_parser.add_argument(
                         '-f',
@@ -326,7 +374,7 @@ def parse_args(argv=None):
                         '--metrics',
                         action='store_true',
                         default=False,
-                        help='Show metrics after reproduction')
+                        help='Show metrics after reproduction.')
     repro_parser.add_argument(
                         '--dry',
                         action='store_true',
@@ -343,10 +391,12 @@ def parse_args(argv=None):
     repro_parser.set_defaults(func=CmdRepro)
 
     # Remove
+    REMOVE_HELP = 'Remove outputs of DVC file.'
     remove_parser = subparsers.add_parser(
                         'remove',
                         parents=[parent_parser],
-                        help='Remove outputs of DVC file.')
+                        description=REMOVE_HELP,
+                        help=REMOVE_HELP)
     remove_parser_group = remove_parser.add_mutually_exclusive_group()
     remove_parser_group.add_argument(
                         '-o',
@@ -359,7 +409,7 @@ def parse_args(argv=None):
                         '--purge',
                         action='store_true',
                         default=False,
-                        help='Remove DVC file and all its outputs')
+                        help='Remove DVC file and all its outputs.')
     remove_parser.add_argument(
                         'targets',
                         nargs='+',
@@ -367,23 +417,27 @@ def parse_args(argv=None):
     remove_parser.set_defaults(func=CmdRemove)
 
     # Move
+    MOVE_HELP = 'Move output of DVC file.'
     move_parser = subparsers.add_parser(
                         'move',
                         parents=[parent_parser],
-                        help='Move output of DVC file.')
+                        description=MOVE_HELP,
+                        help=MOVE_HELP)
     move_parser.add_argument(
                         'src',
-                        help='Source')
+                        help='Source.')
     move_parser.add_argument(
                         'dst',
-                        help='Destination')
+                        help='Destination.')
     move_parser.set_defaults(func=CmdMove)
 
     # Garbage collector
+    GC_HELP = 'Collect garbage.'
     gc_parser = subparsers.add_parser(
                         'gc',
                         parents=[parent_parser],
-                        help='Collect garbage')
+                        description=GC_HELP,
+                        help=GC_HELP)
     gc_parser.add_argument(
                         '-a',
                         '--all-branches',
@@ -395,79 +449,87 @@ def parse_args(argv=None):
                         '--cloud',
                         action='store_true',
                         default=False,
-                        help='Collect garbage in remote repository')
+                        help='Collect garbage in remote repository.')
     gc_parser.add_argument(
                         '-r',
                         '--remote',
-                        help='Remote repository to collect garbage in')
+                        help='Remote repository to collect garbage in.')
     gc_parser.set_defaults(func=CmdGC)
 
     # Config
+    CONFIG_HELP = 'Get or set config options.'
     config_parser = subparsers.add_parser(
                         'config',
                         parents=[parent_parser],
-                        help='Get or set config options')
+                        description=CONFIG_HELP,
+                        help=CONFIG_HELP)
     config_parser.add_argument(
                         '-u',
                         '--unset',
                         default=False,
                         action='store_true',
-                        help='Unset option')
+                        help='Unset option.')
     config_parser.add_argument(
                         'name',
-                        help='Option name')
+                        help='Option name.')
     config_parser.add_argument(
                         'value',
                         nargs='?',
                         default=None,
-                        help='Option value')
+                        help='Option value.')
     config_parser.add_argument(
                         '--local',
                         action='store_true',
                         default=False,
-                        help='Use local config')
+                        help='Use local config.')
     config_parser.set_defaults(func=CmdConfig)
 
     # Remote
+    REMOTE_HELP = 'Manage set of tracked repositories.'
     remote_parser = subparsers.add_parser(
                         'remote',
                         parents=[parent_parser],
-                        help='Manage set of tracked repositories')
+                        description=REMOTE_HELP,
+                        help=REMOTE_HELP)
 
     remote_subparsers = remote_parser.add_subparsers(
                         dest='cmd',
                         help='Use dvc remote CMD --help for '
-                             'command-specific help')
+                             'command-specific help.')
 
     _fix_subparsers(remote_subparsers)
 
+    REMOTE_ADD_HELP = 'Add remote.'
     remote_add_parser = remote_subparsers.add_parser(
                         'add',
                         parents=[parent_parser],
-                        help='Add remote')
+                        description=REMOTE_ADD_HELP,
+                        help=REMOTE_ADD_HELP)
     remote_add_parser.add_argument(
                         'name',
-                        help='Name')
+                        help='Name.')
     remote_add_parser.add_argument(
                         'url',
-                        help='Url')
+                        help='URL.')
     remote_add_parser.add_argument(
                         '--local',
                         action='store_true',
                         default=False,
-                        help='Use local config')
+                        help='Use local config.')
     remote_add_parser.add_argument(
                         '-d',
                         '--default',
                         action='store_true',
                         default=False,
-                        help='Set as default remote')
+                        help='Set as default remote.')
     remote_add_parser.set_defaults(func=CmdRemoteAdd)
 
+    REMOTE_REMOVE_HELP = 'Remove remote.'
     remote_remove_parser = remote_subparsers.add_parser(
                         'remove',
                         parents=[parent_parser],
-                        help='Remove remote')
+                        description=REMOTE_REMOVE_HELP,
+                        help=REMOTE_REMOVE_HELP)
     remote_remove_parser.add_argument(
                         'name',
                         help='Name')
@@ -475,162 +537,182 @@ def parse_args(argv=None):
                         '--local',
                         action='store_true',
                         default=False,
-                        help='Use local config')
+                        help='Use local config.')
     remote_remove_parser.set_defaults(func=CmdRemoteRemove)
 
+    REMOTE_MODIFY_HELP = 'Modify remote.'
     remote_modify_parser = remote_subparsers.add_parser(
                         'modify',
                         parents=[parent_parser],
-                        help='Modify remote')
+                        description=REMOTE_MODIFY_HELP,
+                        help=REMOTE_MODIFY_HELP)
     remote_modify_parser.add_argument(
                         'name',
-                        help='Name')
+                        help='Name.')
     remote_modify_parser.add_argument(
                         'option',
-                        help='Option')
+                        help='Option.')
     remote_modify_parser.add_argument(
                         'value',
                         nargs='?',
-                        help='Value')
+                        help='Value.')
     remote_modify_parser.add_argument(
                         '-u',
                         '--unset',
                         default=False,
                         action='store_true',
-                        help='Unset option')
+                        help='Unset option.')
     remote_modify_parser.add_argument(
                         '--local',
                         action='store_true',
                         default=False,
-                        help='Use local config')
+                        help='Use local config.')
     remote_modify_parser.set_defaults(func=CmdRemoteModify)
 
+    REMOTE_LIST_HELP = 'List remotes.'
     remote_list_parser = remote_subparsers.add_parser(
                         'list',
                         parents=[parent_parser],
-                        help='List remotes')
+                        description=REMOTE_LIST_HELP,
+                        help=REMOTE_LIST_HELP)
     remote_list_parser.add_argument(
                         '--local',
                         action='store_true',
                         default=False,
-                        help='Use local config')
+                        help='Use local config.')
     remote_list_parser.set_defaults(func=CmdRemoteList)
 
     # Metrics
+    METRICS_HELP = 'Get metrics from all branches.'
     metrics_parser = subparsers.add_parser(
                         'metrics',
                         parents=[parent_parser],
-                        help='Get metrics from all branches')
+                        description=METRICS_HELP,
+                        help=METRICS_HELP)
 
     metrics_subparsers = metrics_parser.add_subparsers(
                         dest='cmd',
                         help='Use dvc metrics CMD --help for '
-                             'command-specific help')
+                             'command-specific help.')
 
     _fix_subparsers(metrics_subparsers)
 
+    METRICS_SHOW_HELP = 'Show metrics.'
     metrics_show_parser = metrics_subparsers.add_parser(
                         'show',
                         parents=[parent_parser],
-                        help='Show metrics')
+                        description=METRICS_SHOW_HELP,
+                        help=METRICS_SHOW_HELP)
     metrics_show_parser.add_argument(
                         'path',
                         nargs='?',
-                        help='Path to metrics file')
+                        help='Path to metrics file.')
     metrics_show_parser.add_argument(
                         '-t',
                         '--type',
-                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)')
+                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
     metrics_show_parser.add_argument(
                         '-x',
                         '--xpath',
-                        help='JSON/TSV/HTSV/CSV/HCSV path')
+                        help='JSON/TSV/HTSV/CSV/HCSV path.')
     metrics_show_group = metrics_show_parser.add_mutually_exclusive_group()
     metrics_show_group.add_argument(
                         '--json-path',
-                        help='JSON path')
+                        help='JSON path.')
     metrics_show_group.add_argument(
                         '--tsv-path',
-                        help='TSV path \'row,column\'(e.g. \'1,2\')')
+                        help='TSV path \'row,column\'(e.g. \'1,2\').')
     metrics_show_group.add_argument(
                         '--htsv-path',
-                        help='Headed TSV path \'row,column\'(e.g. \'Name,3\'')
+                        help='Headed TSV path \'row,column\'(e.g. \'Name,3\'.')
     metrics_show_group.add_argument(
                         '--csv-path',
-                        help='CSV path \'row,column\'(e.g. \'1,2\')')
+                        help='CSV path \'row,column\'(e.g. \'1,2\').')
     metrics_show_group.add_argument(
                         '--hcsv-path',
-                        help='Headed CSV path \'row,column\'(e.g. \'Name,3\'')
+                        help='Headed CSV path \'row,column\'(e.g. \'Name,3\'.')
     metrics_show_parser.add_argument(
                         '-a',
                         '--all-branches',
                         action='store_true',
                         default=False,
-                        help='Show metrics for all branches')
+                        help='Show metrics for all branches.')
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
+    METRICS_ADD_HELP = 'Add metrics.'
     metrics_add_parser = metrics_subparsers.add_parser(
                         'add',
                         parents=[parent_parser],
-                        help='Add metrics')
+                        description=METRICS_ADD_HELP,
+                        help=METRICS_ADD_HELP)
     metrics_add_parser.add_argument(
                         '-t',
                         '--type',
-                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)')
+                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
     metrics_add_parser.add_argument(
                         '-x',
                         '--xpath',
-                        help='JSON/TSV/HTSV/CSV/HCSV path')
+                        help='JSON/TSV/HTSV/CSV/HCSV path.')
     metrics_add_parser.add_argument(
                         'path',
-                        help='Path to metrics file')
+                        help='Path to metrics file.')
     metrics_add_parser.set_defaults(func=CmdMetricsAdd)
 
+    METRICS_MODIFY_HELP = 'Modify metrics.'
     metrics_modify_parser = metrics_subparsers.add_parser(
                         'modify',
                         parents=[parent_parser],
-                        help='Modify metrics')
+                        description=METRICS_MODIFY_HELP,
+                        help=METRICS_MODIFY_HELP)
     metrics_modify_parser.add_argument(
                         '-t',
                         '--type',
-                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)')
+                        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
     metrics_modify_parser.add_argument(
                         '-x',
                         '--xpath',
-                        help='JSON/TSV/HTSV/CSV/HCSV path')
+                        help='JSON/TSV/HTSV/CSV/HCSV path.')
     metrics_modify_parser.add_argument(
                         'path',
-                        help='Metrics file')
+                        help='Metrics file.')
     metrics_modify_parser.set_defaults(func=CmdMetricsModify)
 
+    METRICS_REMOVE_HELP = 'Remove metrics.'
     metrics_remove_parser = metrics_subparsers.add_parser(
                         'remove',
                         parents=[parent_parser],
-                        help='Remove metrics')
+                        description=METRICS_REMOVE_HELP,
+                        help=METRICS_REMOVE_HELP)
     metrics_remove_parser.add_argument(
                         'path',
-                        help='Path to metrics file')
+                        help='Path to metrics file.')
     metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
 
     # Install
+    INSTALL_HELP = 'Install dvc hooks into the repository.'
     install_parser = subparsers.add_parser(
                         'install',
                         parents=[parent_parser],
-                        help='Install dvc hooks into the repository')
+                        description=INSTALL_HELP,
+                        help=INSTALL_HELP)
     install_parser.set_defaults(func=CmdInstall)
 
     # Root
+    ROOT_HELP = 'Relative path to project\'s directory.'
     root_parser = subparsers.add_parser(
                         'root',
                         parents=[parent_parser],
-                        help='Relative path to project\'s directory')
+                        description=ROOT_HELP,
+                        help=ROOT_HELP)
     root_parser.set_defaults(func=CmdRoot)
 
     # Lock
+    LOCK_HELP = 'Lock DVC file.'
     lock_parser = subparsers.add_parser(
                         'lock',
                         parents=[parent_parser],
-                        help='Lock DVC file')
+                        description=LOCK_HELP,
+                        help=LOCK_HELP)
     lock_parser.add_argument(
                         'targets',
                         nargs='+',
@@ -638,10 +720,12 @@ def parse_args(argv=None):
     lock_parser.set_defaults(func=CmdLock)
 
     # Unlock
+    UNLOCK_HELP = 'Unlock DVC file.'
     unlock_parser = subparsers.add_parser(
                         'unlock',
                         parents=[parent_parser],
-                        help='Unlock DVC file')
+                        description=UNLOCK_HELP,
+                        help=UNLOCK_HELP)
     unlock_parser.add_argument(
                         'targets',
                         nargs='+',
@@ -649,22 +733,26 @@ def parse_args(argv=None):
     unlock_parser.set_defaults(func=CmdUnlock)
 
     # Pipeline
+    PIPELINE_HELP = 'Manage pipeline.'
     pipeline_parser = subparsers.add_parser(
                         'pipeline',
                         parents=[parent_parser],
-                        help='Manage pipeline')
+                        description=PIPELINE_HELP,
+                        help=PIPELINE_HELP)
 
     pipeline_subparsers = pipeline_parser.add_subparsers(
                         dest='cmd',
                         help='Use dvc pipeline CMD --help '
-                             'for command-specific help')
+                             'for command-specific help.')
 
     _fix_subparsers(pipeline_subparsers)
 
+    PIPELINE_SHOW_HELP = 'Show pipeline.'
     pipeline_show_parser = pipeline_subparsers.add_parser(
                         'show',
                         parents=[parent_parser],
-                        help='Show pipeline')
+                        description=PIPELINE_SHOW_HELP,
+                        help=PIPELINE_SHOW_HELP)
     pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
     pipeline_show_group.add_argument(
                         '-c',
@@ -679,12 +767,34 @@ def parse_args(argv=None):
                         default=False,
                         help='Print output files instead of '
                              'paths to DVC files.')
+    pipeline_show_parser.add_argument(
+                        '--ascii',
+                        action='store_true',
+                        default=False,
+                        help='Output DAG as ASCII.')
     pipeline_show_parser.add_argument(
                         'targets',
-                        nargs='+',
-                        help='DVC files.')
+                        nargs='*',
+                        help="DVC files. 'Dvcfile' by default.")
     pipeline_show_parser.set_defaults(func=CmdPipelineShow)
 
     args = parser.parse_args(argv)
 
+    if (issubclass(args.func, CmdRepro)
+        or issubclass(args.func, CmdPipelineShow)) \
+       and hasattr(args, 'targets') \
+       and len(args.targets) == 0:
+        if hasattr(args, 'cwd'):
+            cwd = args.cwd
+        else:
+            cwd = os.curdir
+        path = os.path.join(cwd, 'Dvcfile')
+        if not os.path.exists(path):
+            msg = "default target '{}' does not exist.".format(path)
+            if issubclass(args.func, CmdRepro):
+                repro_parser.error(msg)
+            elif issubclass(args.func, CmdPipelineShow):
+                pipeline_show_parser.error(msg)
+        args.targets = ['Dvcfile']
+
     return args

dvc/command/add.py

@@ -7,7 +7,7 @@ class CmdAdd(CmdBase):
     def run(self):
         for target in self.args.targets:
             try:
-                self.project.add(target)
+                self.project.add(target, recursive=self.args.recursive)
             except DvcException as ex:
                 Logger.error('Failed to add \'{}\''.format(target), ex)
                 return 1

dvc/command/destroy.py

@@ -6,8 +6,8 @@ from dvc.exceptions import DvcException
 class CmdDestroy(CmdBase):
     def run_cmd(self):
         try:
-            msg = u'This will destroy all information about your pipelines ' \
-                  u'as well as cache in .dvc/cache.\n' \
+            msg = u'This will destroy all information about your pipelines, ' \
+                  u'all data files, as well as cache in .dvc/cache.\n' \
                   u'Are you sure you want to continue?'
 
             if not self.args.force and not prompt(msg, False):

dvc/command/pipeline.py

@@ -2,32 +2,88 @@ import os
 
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
+from dvc.dagascii import Dagascii
 
 
 class CmdPipelineShow(CmdBase):
     def _show(self, target, commands, outs):
         import networkx
-        from dvc.project import NotDvcFileError
+        from dvc.stage import Stage
 
+        stage = Stage.load(self.project, target)
         G = self.project.graph()[0]
         stages = networkx.get_node_attributes(G, 'stage')
-        node = os.path.relpath(os.path.abspath(target), self.project.root_dir)
-        if node not in stages:
-            raise NotDvcFileError(node)
+        node = os.path.relpath(stage.path, self.project.root_dir)
 
         for n in networkx.dfs_postorder_nodes(G, node):
             if commands:
                 self.project.logger.info(stages[n].cmd)
             elif outs:
                 for out in stages[n].outs:
-                    self.project.logger.info(out.rel_path)
+                    self.project.logger.info(str(out))
             else:
                 self.project.logger.info(n)
 
+    def _show_ascii(self, target, commands, outs):
+        import networkx
+        from dvc.stage import Stage
+
+        stage = Stage.load(self.project, target)
+        node = os.path.relpath(stage.path, self.project.root_dir)
+
+        pipelines = list(filter(lambda g: node in g.nodes(),
+                                self.project.pipelines()))
+
+        assert len(pipelines) == 1
+        G = pipelines[0]
+        stages = networkx.get_node_attributes(G, 'stage')
+
+        nodes = []
+        for n in G.nodes():
+            stage = stages[n]
+            if commands:
+                if stage.cmd is None:
+                    continue
+                nodes.append(stage.cmd)
+            elif outs:
+                for out in stage.outs:
+                    nodes.append(str(out))
+            else:
+                nodes.append(stage.relpath)
+
+        if len(nodes) == 0:
+            return
+
+        edges = []
+        for e in G.edges():
+            from_stage = stages[e[0]]
+            to_stage = stages[e[1]]
+            if commands:
+                if to_stage.cmd is None:
+                    continue
+                edges.append((from_stage.cmd, to_stage.cmd))
+            elif outs:
+                for from_out in from_stage.outs:
+                    for to_out in to_stage.outs:
+                        edges.append((str(from_out),
+                                      str(to_out)))
+            else:
+                edges.append((from_stage.relpath, to_stage.relpath))
+
+        d = Dagascii(nodes, edges)
+        d.draw()
+
     def run(self, unlock=False):
         for target in self.args.targets:
             try:
-                self._show(target, self.args.commands, self.args.outs)
+                if self.args.ascii:
+                    self._show_ascii(target,
+                                     self.args.commands,
+                                     self.args.outs)
+                else:
+                    self._show(target,
+                               self.args.commands,
+                               self.args.outs)
             except DvcException as ex:
                 msg = 'Failed to show pipeline for \'{}\''.format(target)
                 self.project.logger.error(msg, ex)

dvc/command/repro.py

@@ -1,12 +1,12 @@
 import os
 
 from dvc.command.base import CmdBase
+from dvc.command.status import CmdDataStatus
+from dvc.exceptions import DvcException
 
 
 class CmdRepro(CmdBase):
     def run(self):
-        from dvc.project import ReproductionError
-
         recursive = not self.args.single_item
         saved_dir = os.path.realpath(os.curdir)
         if self.args.cwd:
@@ -15,23 +15,23 @@ class CmdRepro(CmdBase):
         ret = 0
         for target in self.args.targets:
             try:
-                self.project.reproduce(target,
+                stages = self.project.reproduce(
+                                       target,
                                        recursive=recursive,
                                        force=self.args.force,
                                        dry=self.args.dry,
                                        interactive=self.args.interactive)
+
+                if len(stages) == 0:
+                    self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+
                 if self.args.metrics:
                     self.project.metrics_show()
-            except ReproductionError as ex:
+            except DvcException as ex:
                 msg = 'Failed to reproduce \'{}\''.format(target)
                 self.project.logger.error(msg, ex)
                 ret = 1
                 break
-            except Exception as ex:
-                msg = 'Failed to reproduce \'{}\' - unexpected error'
-                self.project.logger.error(msg.format(target), ex)
-                ret = 1
-                break
 
         os.chdir(saved_dir)
         return ret

dvc/command/status.py

@@ -4,6 +4,7 @@ from dvc.command.data_sync import CmdDataBase
 class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 10
     STATUS_INDENT = '\t'
+    UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce."
 
     def _normalize(self, s):
         s += ':'
@@ -29,7 +30,11 @@ class CmdDataStatus(CmdDataBase):
                                      cloud=self.args.cloud,
                                      show_checksums=self.args.show_checksums,
                                      remote=self.args.remote)
-            self._show(st, indent)
+            if st:
+                self._show(st, indent)
+            else:
+                self.project.logger.info(self.UP_TO_DATE_MSG)
+
         except Exception as exc:
             self.project.logger.error('Failed to obtain data status', exc)
             return 1

dvc/config.py

@@ -10,8 +10,9 @@ from dvc.exceptions import DvcException
 
 class ConfigError(DvcException):
     """ DVC config exception """
-    def __init__(self, ex=None):
-        super(ConfigError, self).__init__('Config file error', ex)
+    def __init__(self, msg, ex=None):
+        super(ConfigError, self).__init__('Config file error: {}'.format(msg),
+                                          ex)
 
 
 def supported_url(url):
@@ -36,6 +37,14 @@ def supported_cloud(cloud):
     return cloud in ['aws', 'gcp', 'local', '']
 
 
+def is_bool(val):
+    return val.lower() in ['true', 'false']
+
+
+def to_bool(val):
+    return val.lower() == 'true'
+
+
 class Config(object):
     CONFIG = 'config'
     CONFIG_LOCAL = 'config.local'
@@ -44,6 +53,8 @@ class Config(object):
     SECTION_CORE_LOGLEVEL = 'loglevel'
     SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
     SECTION_CORE_REMOTE = 'remote'
+    SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
+    SECTION_CORE_INTERACTIVE = 'interactive'
 
     SECTION_CACHE = 'cache'
     SECTION_CACHE_DIR = 'dir'
@@ -78,6 +89,8 @@ class Config(object):
                  default='info'): And(str, Use(str.lower),
                                       SECTION_CORE_LOGLEVEL_SCHEMA),
         Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)),
+        Optional(SECTION_CORE_INTERACTIVE,
+                 default=False): SECTION_CORE_INTERACTIVE_SCHEMA,
 
         # backward compatibility
         Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
@@ -119,6 +132,8 @@ class Config(object):
     SECTION_REMOTE_FMT = 'remote "{}"'
     SECTION_REMOTE_URL = 'url'
     SECTION_REMOTE_USER = 'user'
+    SECTION_REMOTE_PORT = 'port'
+    SECTION_REMOTE_KEY_FILE = 'keyfile'
     SECTION_REMOTE_SCHEMA = {
         SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
         Optional(SECTION_AWS_REGION): str,
@@ -128,6 +143,8 @@ class Config(object):
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
         Optional(SECTION_REMOTE_USER): str,
+        Optional(SECTION_REMOTE_PORT): Use(int),
+        Optional(SECTION_REMOTE_KEY_FILE): str,
     }
 
     SCHEMA = {
@@ -154,7 +171,7 @@ class Config(object):
             # need to convert our config to dict before passing it to
             self._config = self._lower(self._config)
             local = self._lower(local)
-            self._config.update(local)
+            self._config = self._merge(self._config, local)
 
             self._config = Schema(self.SCHEMA).validate(self._config)
 
@@ -165,6 +182,17 @@ class Config(object):
         except Exception as ex:
             raise ConfigError(ex)
 
+    @staticmethod
+    def _merge(first, second):
+        res = {}
+        sections = list(first.keys()) + list(second.keys())
+        for section in sections:
+            f = first.get(section, {}).copy()
+            s = second.get(section, {}).copy()
+            f.update(s)
+            res[section] = f
+        return res
+
     @staticmethod
     def _lower(config):
         new_config = {}

dvc/dagascii.py

@@ -0,0 +1,121 @@
+import math
+
+from asciicanvas.asciicanvas import AsciiCanvas
+from asciicanvas.style import Style
+
+from grandalf.graphs import Vertex, Edge, Graph
+from grandalf.layouts import SugiyamaLayout
+from grandalf.routing import route_with_lines, EdgeViewer
+
+
+class Dagascii(object):
+    def __init__(self, vertexes, edges):
+        #
+        # Just a reminder about naming conventions:
+        # +------------X
+        # |
+        # |
+        # |
+        # |
+        # Y
+        #
+
+        V = {v: Vertex(" {} ".format(v)) for v in vertexes}
+        # NOTE: reverting edges to correctly orientate the graph
+        E = [Edge(V[e], V[s]) for s, e in edges]
+        V = V.values()
+        g = Graph(V, E)
+
+        class VertexViewer(object):
+            h = 3  # top and bottom box edges + text
+
+            def __init__(self, name):
+                self.w = len(name) + 2  # right and left bottom edges + text
+
+        for v in V:
+            v.view = VertexViewer(v.data)
+
+        # NOTE: determine min box length to create the best layout
+        minw = min([v.view.w for v in V])
+
+        for e in E:
+            e.view = EdgeViewer()
+
+        sug = SugiyamaLayout(g.C[0])
+        gr = g.C[0]
+        r = list(filter(lambda x: len(x.e_in()) == 0, gr.sV))
+
+        sug.init_all(roots=r, optimize=True)
+
+        sug.yspace = VertexViewer.h
+        sug.xspace = minw
+        sug.route_edge = route_with_lines
+
+        sug.draw()
+
+        self.sug = sug
+
+    def draw(self):
+        # NOTE: coordinates might me negative, so we need to shift
+        # everything to the positive plane before we actually draw it.
+        Xs = []
+        Ys = []
+
+        for v in self.sug.g.sV:
+            # NOTE: moving boxes w/2 to the left
+            Xs.append(v.view.xy[0] - v.view.w/2.0)
+            Xs.append(v.view.xy[0] + v.view.w/2.0)
+            Ys.append(v.view.xy[1])
+            Ys.append(v.view.xy[1] + v.view.h)
+
+        for e in self.sug.g.sE:
+            for xy in e.view._pts:
+                Xs.append(xy[0])
+                Ys.append(xy[1])
+
+        minx = min(Xs)
+        miny = min(Ys)
+        maxx = max(Xs)
+        maxy = max(Ys)
+
+        canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
+        canvas_lines = int(round(maxy - miny))
+
+        canvas = AsciiCanvas(canvas_cols, canvas_lines)
+
+        # NOTE: first draw edges so that node boxes could overwrite them
+        for e in self.sug.g.sE:
+            assert len(e.view._pts) > 1
+            for index in range(1, len(e.view._pts)):
+                start = e.view._pts[index - 1]
+                end = e.view._pts[index]
+
+                start_x = int(round(start[0] - minx))
+                start_y = int(round(start[1] - miny))
+                end_x = int(round(end[0] - minx))
+                end_y = int(round(end[1] - miny))
+
+                assert start_x >= 0
+                assert start_y >= 0
+                assert end_x >= 0
+                assert end_y >= 0
+
+                canvas.add_line(start_x, start_y,
+                                end_x, end_y,
+                                style=Style('*'))
+
+        for v in self.sug.g.sV:
+            # NOTE: moving boxes w/2 to the left
+            x = v.view.xy[0] - v.view.w/2.0
+            y = v.view.xy[1]
+
+            canvas.add_nine_patch_rect(int(round(x - minx)),
+                                       int(round(y - miny)),
+                                       v.view.w,
+                                       v.view.h)
+
+            canvas.add_text(int(round(x - minx)) + 1,
+                            int(round(y - miny)) + 1,
+                            v.data)
+
+        canvas.print_out()

dvc/data_cloud.py

@@ -83,15 +83,18 @@ class DataCloud(object):
         cloud = cloud_type(self.project, cloud_config)
         return cloud
 
-    def _get_cloud(self, remote):
+    def _get_cloud(self, remote, cmd):
         if remote:
             return self._init_remote(remote)
 
         if self._cloud:
             return self._cloud
 
-        msg = "No remote repository specified. Setup default repository " \
-              "with 'dvc config core.remote <name>' or use '-r <name>'."
+        msg = "No remote repository specified. Setup default " \
+              "repository with\n"\
+              "    dvc config core.remote <name>\n" \
+              "or use:\n" \
+              "    dvc {} -r <name>".format(cmd)
         raise ConfigError(msg)
 
     def push(self, targets, jobs=1, remote=None, show_checksums=False):
@@ -100,7 +103,8 @@ class DataCloud(object):
         """
         return self.project.cache.local.push(targets,
                                              jobs=jobs,
-                                             remote=self._get_cloud(remote),
+                                             remote=self._get_cloud(remote,
+                                                                    'push'),
                                              show_checksums=show_checksums)
 
     def pull(self, targets, jobs=1, remote=None, show_checksums=False):
@@ -109,14 +113,16 @@ class DataCloud(object):
         """
         return self.project.cache.local.pull(targets,
                                              jobs=jobs,
-                                             remote=self._get_cloud(remote),
+                                             remote=self._get_cloud(remote,
+                                                                    'pull'),
                                              show_checksums=show_checksums)
 
     def status(self, targets, jobs=1, remote=None, show_checksums=False):
         """
         Check status of data items in a cloud-agnostic way.
         """
+        cloud = self._get_cloud(remote, 'status')
         return self.project.cache.local.status(targets,
                                                jobs=jobs,
-                                               remote=self._get_cloud(remote),
+                                               remote=cloud,
                                                show_checksums=show_checksums)

dvc/dependency/base.py

@@ -31,6 +31,9 @@ class DependencyBase(object):
         self.project = stage.project
         self.path = path
 
+    def __str__(self):
+        return self.path
+
     @classmethod
     def match(cls, url):
         return re.match(cls.REGEX, url)
@@ -59,7 +62,11 @@ class DependencyBase(object):
     def status(self):
         if self.changed():
             # FIXME better msgs
-            return {self.rel_path: 'changed'}
+            if self.path_info['scheme'] == 'local':
+                p = self.rel_path
+            else:
+                p = self.path
+            return {p: 'changed'}
         return {}
 
     def save(self):

dvc/dependency/local.py

@@ -38,6 +38,9 @@ class DependencyLOCAL(DependencyBase):
         self.path_info = {'scheme': 'local',
                           'path': self.path}
 
+    def __str__(self):
+        return self.rel_path
+
     @property
     def sep(self):
         return os.sep

dvc/dependency/ssh.py

@@ -18,6 +18,7 @@ class DependencySSH(DependencyBase):
         self.remote = remote if remote else RemoteSSH(stage.project, {})
 
         host = remote.host if remote else self.group('host')
+        port = remote.port if remote else RemoteSSH.DEFAULT_PORT
         user = remote.user if remote else self.group('user')
         if remote:
             path = posixpath.join(remote.prefix,
@@ -27,6 +28,7 @@ class DependencySSH(DependencyBase):
 
         self.path_info = {'scheme': 'ssh',
                           'host': host,
+                          'port': port,
                           'user': user,
                           'path': path}
 

dvc/exceptions.py

@@ -24,6 +24,8 @@ class OutputDuplicationError(DvcException):
         assert isinstance(output, str)
         assert isinstance(stages, list)
         assert all(isinstance(stage, str) for stage in stages)
-        msg = "File '{}' is specified as an output in more than one stage: {}"
-        super(OutputDuplicationError, self).__init__(msg.format(output,
-                                                                str(stages)))
+        msg = "File '{}' is specified as an output in more than one stage:{}"
+        s = ""
+        for stage in stages:
+            s += "\n    " + stage
+        super(OutputDuplicationError, self).__init__(msg.format(output, s))

dvc/logger.py

@@ -82,22 +82,39 @@ class Logger(object):
 
         return (str_exc, str_tb)
 
+    @staticmethod
+    def _prefix(msg, typ):
+        return Logger.colorize('{}: '.format(msg), typ)
+
+    @staticmethod
+    def error_prefix():
+        return Logger._prefix('Error', 'error')
+
+    @staticmethod
+    def warning_prefix():
+        return Logger._prefix('Warning', 'warn')
+
+    @staticmethod
+    def debug_prefix():
+        return Logger._prefix('Debug', 'debug')
+
     @staticmethod
     def error(msg, exc=None):
+        prefix = Logger.error_prefix()
         str_exc, str_tb = Logger.parse_exc(exc)
         if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
             str_tb = str_tb if str_tb else traceback.format_exc()
-            Logger.logger().error(str_tb)
-        return Logger.logger().error(Logger.colorize(msg + str_exc, 'error'))
+            Logger.logger().error(prefix + str_tb)
+        return Logger.logger().error(prefix + msg + str_exc)
 
     @staticmethod
     def warn(msg):
-        return Logger.logger().warn(Logger.colorize(msg, 'warn'))
+        return Logger.logger().warn(Logger.warning_prefix() + msg)
 
     @staticmethod
     def debug(msg):
-        return Logger.logger().debug(Logger.colorize(msg, 'debug'))
+        return Logger.logger().debug(Logger.debug_prefix() + msg)
 
     @staticmethod
     def info(msg):
-        return Logger.logger().info(Logger.colorize(msg, 'info'))
+        return Logger.logger().info(msg)

dvc/output/gs.py

@@ -30,7 +30,7 @@ class OutputGS(DependencyGS):
             return True
 
         if self.use_cache \
-           and self.info != self.project.cache.gs.save_info(self.path_info):
+           and self.project.cache.gs.changed(self.path_info, self.info):
             return True
 
         return False

dvc/output/hdfs.py

@@ -30,7 +30,7 @@ class OutputHDFS(DependencyHDFS):
             return True
 
         if self.use_cache \
-           and self.info != self.project.cache.hdfs.save_info(self.path_info):
+           and self.project.cache.hdfs.changed(self.path_info, self.info):
             return True
 
         return False

dvc/output/local.py

@@ -62,17 +62,12 @@ class OutputLOCAL(DependencyLOCAL):
         if not self.use_cache:
             return super(OutputLOCAL, self).changed()
 
-        return self.info != self.project.cache.local.save_info(self.path_info)
+        return self.project.cache.local.changed(self.path_info, self.info)
 
     def checkout(self):
         if not self.use_cache:
             return
 
-        if not self.changed():
-            msg = u'Data file \'{}\' didn\'t change, skipping checkout.'
-            self.project.logger.debug(msg.format(self.rel_path))
-            return
-
         self.project.cache.local.checkout(self.path_info, self.info)
 
     def _verify_metric(self):

dvc/output/s3.py

@@ -30,7 +30,7 @@ class OutputS3(DependencyS3):
             return True
 
         if self.use_cache \
-           and self.info != self.project.cache.s3.save_info(self.path_info):
+           and self.project.cache.s3.changed(self.path_info, self.info):
             return True
 
         return False

dvc/output/ssh.py

@@ -30,7 +30,7 @@ class OutputSSH(DependencySSH):
             return True
 
         if self.use_cache \
-           and self.info != self.project.cache.ssh.save_info(self.path_info):
+           and self.project.cache.ssh.changed(self.path_info, self.info):
             return True
 
         return False

dvc/project.py

@@ -9,15 +9,6 @@ class InitError(DvcException):
         super(InitError, self).__init__(msg)
 
 
-class NotDvcFileError(DvcException):
-    def __init__(self, path):
-        msg = '\'{}\' is not a DVC file'.format(path)
-        p = path + Stage.STAGE_FILE_SUFFIX
-        if Stage.is_stage_file(p):
-            msg += '. Maybe you meant \'{}\'?'.format(p)
-        super(NotDvcFileError, self).__init__(msg)
-
-
 class ReproductionError(DvcException):
     def __init__(self, dvc_file_name, ex):
         self.path = dvc_file_name
@@ -131,19 +122,48 @@ class Project(object):
     def to_dvc_path(self, path):
         return os.path.relpath(path, self.root_dir)
 
-    def add(self, fname):
-        stage = Stage.loads(project=self,
-                            outs=[fname],
-                            add=True)
+    def _check_output_duplication(self, outs):
+        from dvc.exceptions import OutputDuplicationError
 
-        stage.save()
-        stage.dump()
-        return stage
+        for stage in self.stages():
+            for o in stage.outs:
+                for out in outs:
+                    if o.path == out.path and o.stage.path != out.stage.path:
+                        stages = [o.stage.relpath, out.stage.relpath]
+                        raise OutputDuplicationError(o.path, stages)
+
+    def add(self, fname, recursive=False):
+        fnames = []
+        if recursive and os.path.isdir(fname):
+            fnames = []
+            for root, dirs, files in os.walk(fname):
+                for f in files:
+                    path = os.path.join(root, f)
+                    if Stage.is_stage_file(path):
+                        continue
+                    if os.path.basename(path) == self.scm.ignore_file():
+                        continue
+                    if self.scm.is_tracked(path):
+                        continue
+                    fnames.append(path)
+        else:
+            fnames = [fname]
+
+        stages = []
+        for f in fnames:
+            stage = Stage.loads(project=self,
+                                outs=[f],
+                                add=True)
+
+            self._check_output_duplication(stage.outs)
+
+            stage.save()
+            stage.dump()
+            stages.append(stage)
+
+        return stages
 
     def remove(self, target, outs_only=False):
-        if not Stage.is_stage_file(target):
-            raise NotDvcFileError(target)
-
         stage = Stage.load(self, target)
         if outs_only:
             stage.remove_outs()
@@ -153,9 +173,6 @@ class Project(object):
         return stage
 
     def lock_stage(self, target, unlock=False):
-        if not Stage.is_stage_file(target):
-            raise NotDvcFileError(target)
-
         stage = Stage.load(self, target)
         stage.locked = False if unlock else True
         stage.dump()
@@ -220,6 +237,9 @@ class Project(object):
                             metrics_no_cache=metrics_no_cache,
                             deps=deps,
                             overwrite=False)
+
+        self._check_output_duplication(stage.outs)
+
         if not no_exec:
             stage.run()
         stage.dump()
@@ -231,6 +251,8 @@ class Project(object):
                             deps=[url],
                             outs=[out])
 
+        self._check_output_duplication(stage.outs)
+
         stage.run()
         stage.dump()
         return stage
@@ -260,11 +282,15 @@ class Project(object):
                   interactive=False):
         import networkx as nx
 
+        stage = Stage.load(self, target)
         G = self.graph()[1]
         stages = nx.get_node_attributes(G, 'stage')
-        node = os.path.relpath(os.path.abspath(target), self.root_dir)
-        if node not in stages:
-            raise NotDvcFileError(target)
+        node = os.path.relpath(stage.path, self.root_dir)
+
+        if not interactive:
+            config = self.config
+            core = config._config[config.SECTION_CORE]
+            interactive = core.get(config.SECTION_CORE_INTERACTIVE, False)
 
         if recursive:
             return self._reproduce_stages(G,
@@ -302,8 +328,6 @@ class Project(object):
         all_stages = self.active_stages()
 
         if target:
-            if not Stage.is_stage_file(target):
-                raise NotDvcFileError(target)
             stages = [Stage.load(self, target)]
         else:
             stages = all_stages
@@ -351,29 +375,35 @@ class Project(object):
 
         return cache
 
+    def _do_gc(self, typ, func, clist):
+        removed = func(clist)
+        if not removed:
+            self.logger.info("No unused {} cache to remove.".format(typ))
+
     def gc(self, all_branches=False, cloud=False, remote=None):
         clist = self._used_cache(target=None,
                                  all_branches=all_branches,
                                  active=False)
-        self.cache.local.gc(clist)
+        self._do_gc('local', self.cache.local.gc, clist)
 
         if self.cache.s3:
-            self.cache.s3.gc(clist)
+            self._do_gc('s3', self.cache.s3.gc, clist)
 
         if self.cache.gs:
-            self.cache.gs.gc(clist)
+            self._do_gc('gs', self.cache.gs.gc, clist)
 
         if self.cache.ssh:
-            self.cache.ssh.gc(clist)
+            self._do_gc('ssh', self.cache.ssh.gc, clist)
 
         if self.cache.hdfs:
-            self.cache.hdfs.gc(clist)
+            self._do_gc('hdfs', self.cache.hdfs.gc, clist)
 
         if self.cache.azure:
-            self.cache.azure.gc(clist)
+            self._do_gc('azure', self.cache.azure.gc, clist)
 
         if cloud:
-            self.cloud._get_cloud(remote).gc(clist)
+            self._do_gc('remote', self.cloud._get_cloud(remote,
+                                                        'gc -c').gc, clist)
 
     def push(self,
              target=None,
@@ -533,7 +563,7 @@ class Project(object):
 
         abs_path = os.path.abspath(path)
         matched = [out for out in outs if out.path == abs_path]
-        stages = [out.stage.path for out in matched]
+        stages = [out.stage.relpath for out in matched]
         if len(stages) > 1:
             raise OutputDuplicationError(path, stages)
 
@@ -566,8 +596,8 @@ class Project(object):
                 entries = []
                 for o in metrics:
                     if not typ and isinstance(o.metric, dict):
-                        t = o.metric.get(o.PARAM_METRIC_TYPE, None)
-                        x = o.metric.get(o.PARAM_METRIC_XPATH, None)
+                        t = o.metric.get(o.PARAM_METRIC_TYPE, typ)
+                        x = o.metric.get(o.PARAM_METRIC_XPATH, xpath)
                     else:
                         t = typ
                         x = xpath
@@ -649,14 +679,22 @@ class Project(object):
 
     def graph(self):
         import networkx as nx
+        from dvc.exceptions import OutputDuplicationError
 
         G = nx.DiGraph()
         G_active = nx.DiGraph()
         stages = self.stages()
 
         outs = []
+        outs_by_path = {}
         for stage in stages:
-            outs += stage.outs
+            for o in stage.outs:
+                existing = outs_by_path.get(o.path, None)
+                if existing is not None:
+                    stages = [o.stage.relpath, existing.stage.relpath]
+                    raise OutputDuplicationError(o.path, stages)
+                outs.append(o)
+                outs_by_path[o.path] = o
 
         # collect the whole DAG
         for stage in stages:

dvc/prompt.py

@@ -7,11 +7,11 @@ except NameError:
     pass
 
 
-def prompt(msg, default=False):
+def prompt(msg, default=False):  # pragma: no cover
     if not sys.stdout.isatty():
         return default
 
-    answer = input(msg + u'\n').lower()
+    answer = input(msg + u' (y/n)\n').lower()
     while answer not in ['yes', 'no', 'y', 'n']:
         answer = input('Enter \'yes\' or \'no\'.\n').lower()
 

dvc/remote/azure.py

@@ -92,6 +92,9 @@ class RemoteAzure(RemoteBase):
 #
 #        return {self.PARAM_ETAG: etag}
 #
+#    def changed(self, path_info, checksum_info):
+#        return checksum_info != self.save_info(path_info)
+#
 #    def _copy_blob(self, to_bucket, to_key, from_bucket, from_key):
 #        source = self.blob_service.make_blob_url(from_bucket, from_key)
 #
@@ -227,6 +230,7 @@ class RemoteAzure(RemoteBase):
 #
 #        all_blobs = self.blob_service.list_blobs(self.bucket)
 #
+#        removed = False
 #        for blob in all_blobs:
 #            etag = blob.properties.etag
 #            if etag in used:
@@ -235,3 +239,6 @@ class RemoteAzure(RemoteBase):
 #                         'key': blob.name,
 #                         'bucket': self.bucket}
 #            self.remove(path_info)
+#            removed = True
+#
+#        return removed

dvc/remote/base.py

@@ -81,6 +81,9 @@ class RemoteBase(object):
     def save_info(self, path_info):
         raise NotImplementedError
 
+    def changed(self, path_info, checksum_info):
+        raise NotImplementedError
+
     def save(self, path_info):
         raise NotImplementedError
 

dvc/remote/gs.py

@@ -59,6 +59,37 @@ class RemoteGS(RemoteBase):
         return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
                                                path_info['key'])}
 
+    @staticmethod
+    def to_string(path_info):
+        return "{}://{}/{}".format(path_info['scheme'],
+                                   path_info['bucket'],
+                                   path_info['key'])
+
+    def changed_cache(self, etag):
+        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
+        cache = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
+
+        if {self.PARAM_ETAG: etag} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+        return False
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        etag = checksum_info.get(self.PARAM_ETAG, None)
+        if etag is None:
+            return True
+
+        if self.changed_cache(etag):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def _copy(self, from_info, to_info, gs=None):
         gs = gs if gs else self.gs
 
@@ -92,6 +123,25 @@ class RemoteGS(RemoteBase):
         if not etag:
             return
 
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(etag):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(etag, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), etag))
+
         key = posixpath.join(self.prefix, etag[0:2], etag[2:])
         from_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
 
@@ -229,6 +279,7 @@ class RemoteGS(RemoteBase):
         used = [info[self.PARAM_ETAG] for info in cinfos['gs']]
         used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
 
+        removed = False
         for etag in self._all_etags():
             if etag in used:
                 continue
@@ -237,3 +288,6 @@ class RemoteGS(RemoteBase):
                          'key': posixpath.join(self.prefix,
                                                etag[0:2], etag[2:])}
             self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/hdfs.py

@@ -9,6 +9,7 @@ from dvc.remote.base import RemoteBase
 from dvc.remote.local import RemoteLOCAL
 from dvc.exceptions import DvcException
 from dvc.logger import Logger
+from dvc.utils import fix_env
 
 
 class RemoteHDFS(RemoteBase):
@@ -32,6 +33,7 @@ class RemoteHDFS(RemoteBase):
                   shell=True,
                   close_fds=True,
                   executable=os.getenv('SHELL'),
+                  env=fix_env(os.environ),
                   stdin=PIPE,
                   stdout=PIPE,
                   stderr=PIPE)
@@ -70,6 +72,39 @@ class RemoteHDFS(RemoteBase):
 
         return {self.PARAM_CHECKSUM: self.checksum(path_info)}
 
+    @staticmethod
+    def to_string(path_info):
+        return "{}://{}".format(path_info['scheme'],
+                                path_info['url'])
+
+    def changed_cache(self, checksum):
+        cache = {}
+        cache['scheme'] = 'hdfs'
+        cache['user'] = self.user
+        cache['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
+
+        if {self.PARAM_CHECKSUM: checksum} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
+        if checksum is None:
+            return True
+
+        if self.changed_cache(self, checksum):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def save(self, path_info):
         if path_info['scheme'] != 'hdfs':
             raise NotImplementedError
@@ -94,6 +129,25 @@ class RemoteHDFS(RemoteBase):
         if not checksum:
             return
 
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(checksum):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(checksum, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), checksum))
+
         src = path_info.copy()
         src['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
 
@@ -197,6 +251,7 @@ class RemoteHDFS(RemoteBase):
         used = [info[self.PARAM_CHECKSUM] for info in cinfos['hdfs']]
         used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
 
+        removed = False
         for checksum in self._all_checksums():
             if checksum in used:
                 continue
@@ -205,3 +260,6 @@ class RemoteHDFS(RemoteBase):
                          'url': posixpath.join(self.url,
                                                checksum[0:2], checksum[2:])}
             self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/local.py

@@ -82,7 +82,7 @@ class RemoteLOCAL(RemoteBase):
         relpath = os.path.relpath(path, self.cache_dir)
         return os.path.dirname(relpath) + os.path.basename(relpath)
 
-    def changed(self, md5):
+    def changed_cache(self, md5):
         cache = self.get(md5)
         if self.state.changed(cache, md5=md5):
             if os.path.exists(cache):
@@ -104,6 +104,10 @@ class RemoteLOCAL(RemoteBase):
         while i > 0:
             try:
                 self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
+                msg = "Created '{}': {} -> {}"
+                Logger.info(msg.format(self.cache_types[0],
+                                       os.path.relpath(cache),
+                                       os.path.relpath(path)))
                 return
             except Exception as exc:
                 msg = 'Cache type \'{}\' is not supported: {}'
@@ -172,7 +176,7 @@ class RemoteLOCAL(RemoteBase):
         dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
 
         md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
-        if self.changed(md5):
+        if self.changed_cache(md5):
             self.dump_dir_cache(md5, dir_info)
 
         return (md5, dir_info)
@@ -231,19 +235,24 @@ class RemoteLOCAL(RemoteBase):
             Logger.warn(msg.format(os.path.relpath(path)))
             return
 
-        if self.changed(md5):
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(os.path.relpath(path)))
+            return
+
+        if self.changed_cache(md5):
             msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.'
             Logger.warn(msg.format(md5, os.path.relpath(path)))
             remove(path)
             return
 
         if os.path.exists(path):
-            msg = u'Data \'{}\' exists. Removing before checkout'
-            Logger.debug(msg.format(os.path.relpath(path)))
+            msg = u'Data \'{}\' exists. Removing before checkout.'
+            Logger.warn(msg.format(os.path.relpath(path)))
             remove(path)
 
-        msg = u'Checking out \'{}\' with cache \'{}\''
-        Logger.debug(msg.format(os.path.relpath(path), md5))
+        msg = u'Checking out \'{}\' with cache \'{}\'.'
+        Logger.info(msg.format(os.path.relpath(path), md5))
 
         if not self.is_dir_cache(cache):
             self.link(cache, path)
@@ -277,7 +286,7 @@ class RemoteLOCAL(RemoteBase):
 
         cache = self.get(md5)
 
-        if self.changed(md5):
+        if self.changed_cache(md5):
             self._move(path, cache)
         else:
             remove(path)
@@ -297,7 +306,7 @@ class RemoteLOCAL(RemoteBase):
             p = os.path.join(path, relpath)
             c = self.get(m)
 
-            if self.changed(m):
+            if self.changed_cache(m):
                 self._move(p, c)
             else:
                 remove(p)
@@ -314,6 +323,10 @@ class RemoteLOCAL(RemoteBase):
 
         path = path_info['path']
 
+        msg = "Saving '{}' to cache '{}'."
+        Logger.info(msg.format(os.path.relpath(path),
+                               os.path.relpath(self.cache_dir)))
+
         if os.path.isdir(path):
             return self._save_dir(path_info)
         else:
@@ -325,6 +338,19 @@ class RemoteLOCAL(RemoteBase):
 
         return {self.PARAM_MD5: self.state.update(path_info['path'])}
 
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        md5 = checksum_info.get(self.PARAM_MD5, None)
+        if md5 is None:
+            return True
+
+        if self.changed_cache(md5):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def remove(self, path_info):
         if path_info['scheme'] != 'local':
             raise NotImplementedError
@@ -460,10 +486,14 @@ class RemoteLOCAL(RemoteBase):
         checksum_infos = self._collect(checksum_infos['local'])[0]
         used_md5s = [info[self.PARAM_MD5] for info in checksum_infos]
 
+        removed = False
         for md5 in self.all():
             if md5 in used_md5s:
                 continue
             remove(self.get(md5))
+            removed = True
+
+        return removed
 
     def status(self, checksum_infos, remote, jobs=1, show_checksums=False):
         Logger.info("Preparing to pull data from {}".format(remote.url))
@@ -490,7 +520,7 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 90, 100)
 
-        local_exists = [not self.changed(md5) for md5 in md5s]
+        local_exists = [not self.changed_cache(md5) for md5 in md5s]
 
         progress.finish_target(title)
 
@@ -517,7 +547,7 @@ class RemoteLOCAL(RemoteBase):
         names = []
         # NOTE: filter files that are not corrupted
         for md5, name in grouped:
-            if self.changed(md5):
+            if self.changed_cache(md5):
                 md5s.append(md5)
                 names.append(name)
 
@@ -586,7 +616,7 @@ class RemoteLOCAL(RemoteBase):
 
         # NOTE: verifying that our cache is not corrupted
         def func(info):
-            return not self.changed(info[self.PARAM_MD5])
+            return not self.changed_cache(info[self.PARAM_MD5])
         checksum_infos = list(filter(func, checksum_infos))
 
         progress.update_target(title, 20, 100)

dvc/remote/s3.py

@@ -57,6 +57,7 @@ class RemoteS3(RemoteBase):
         else:
             creds = {}
 
+        self.creds = creds
         self.region = creds.get('region', self.region)
         self.aws_access_key_id = creds.get('aws_access_key_id', None)
         self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
@@ -71,12 +72,14 @@ class RemoteS3(RemoteBase):
 
     @property
     def s3(self):
-        session = boto3.session.Session(profile_name=self.profile)
-        return session.client('s3',
-                              aws_access_key_id=self.aws_access_key_id,
-                              aws_secret_access_key=self.aws_secret_access_key,
-                              region_name=self.region,
-                              endpoint_url=self.endpoint_url)
+        if not self.creds:
+            session = boto3.session.Session(profile_name=self.profile)
+        else:
+            session = boto3.session.Session(
+                             aws_access_key_id=self.aws_access_key_id,
+                             aws_secret_access_key=self.aws_secret_access_key,
+                             region_name=self.region)
+        return session.client('s3', endpoint_url=self.endpoint_url)
 
     def get_etag(self, bucket, key):
         try:
@@ -93,6 +96,19 @@ class RemoteS3(RemoteBase):
         return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
                                                path_info['key'])}
 
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        etag = checksum_info.get(self.PARAM_ETAG, None)
+        if etag is None:
+            return True
+
+        if self.changed_cache(etag):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def _copy(self, from_info, to_info, s3=None):
         s3 = s3 if s3 else self.s3
 
@@ -112,6 +128,23 @@ class RemoteS3(RemoteBase):
 
         return {self.PARAM_ETAG: etag}
 
+    @staticmethod
+    def to_string(path_info):
+        return "s3://{}/{}".format(path_info['bucket'], path_info['key'])
+
+    def changed_cache(self, etag):
+        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
+        cache = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
+
+        if {self.PARAM_ETAG: etag} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
     def checkout(self, path_info, checksum_info):
         if path_info['scheme'] != 's3':
             raise NotImplementedError
@@ -120,6 +153,25 @@ class RemoteS3(RemoteBase):
         if not etag:
             return
 
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(etag):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(etag, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), etag))
+
         key = posixpath.join(self.prefix, etag[0:2], etag[2:])
         from_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
 
@@ -281,6 +333,7 @@ class RemoteS3(RemoteBase):
         used_etags = [info[self.PARAM_ETAG] for info in cinfos['s3']]
         used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
 
+        removed = False
         for etag in self._all():
             if etag in used_etags:
                 continue
@@ -289,3 +342,6 @@ class RemoteS3(RemoteBase):
                                                etag[0:2], etag[2:]),
                          'bucket': self.bucket}
             self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/ssh.py

@@ -46,6 +46,8 @@ class RemoteSSH(RemoteBase):
     REQUIRES = {'paramiko': paramiko}
     PARAM_MD5 = 'md5'
 
+    DEFAULT_PORT = 22
+
     def __init__(self, project, config):
         self.project = project
         self.url = config.get(Config.SECTION_REMOTE_URL, '/')
@@ -55,30 +57,39 @@ class RemoteSSH(RemoteBase):
             self.user = config.get(Config.SECTION_REMOTE_USER,
                                    getpass.getuser())
         self.prefix = self.group('path')
+        self.port = config.get(Config.SECTION_REMOTE_PORT, self.DEFAULT_PORT)
+        self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
 
     def md5s_to_path_infos(self, md5s):
         return [{'scheme': 'ssh',
                  'host': self.host,
                  'user': self.user,
+                 'port': self.port,
                  'path': posixpath.join(self.prefix,
                                         md5[0:2], md5[2:])} for md5 in md5s]
 
-    def ssh(self, host=None, user=None):
-        msg = "Establishing ssh connection with '{}' as user '{}'"
-        Logger.debug(msg.format(host, user))
+    def ssh(self, host=None, user=None, port=None):
+        msg = "Establishing ssh connection with '{}' " \
+              "through port '{}' as user '{}'"
+        Logger.debug(msg.format(host, port, user))
 
         ssh = paramiko.SSHClient()
 
         ssh.load_system_host_keys()
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-        ssh.connect(host, username=user)
+        ssh.connect(host,
+                    username=user,
+                    port=port,
+                    key_filename=self.keyfile)
 
         return ssh
 
     def exists(self, path_infos):
         ret = []
-        ssh = self.ssh(host=self.host, user=self.user)
+        ssh = self.ssh(host=self.host,
+                       user=self.user,
+                       port=self.port)
         cmd = 'find {} -type f -follow -print'.format(self.prefix)
         stdout = self._exec(ssh, cmd)
         plist = stdout.split()
@@ -103,7 +114,9 @@ class RemoteSSH(RemoteBase):
         if path_info['scheme'] != 'ssh':
             raise NotImplementedError
 
-        ssh = self.ssh(host=path_info['host'], user=path_info['user'])
+        ssh = self.ssh(host=path_info['host'],
+                       user=path_info['user'],
+                       port=path_info['port'])
 
         # Use different md5 commands depending on os
         stdout = self._exec(ssh, 'uname').strip()
@@ -133,7 +146,8 @@ class RemoteSSH(RemoteBase):
         assert from_info['user'] == to_info['user']
 
         s = ssh if ssh else self.ssh(host=from_info['host'],
-                                     user=from_info['user'])
+                                     user=from_info['user'],
+                                     port=from_info['port'])
 
         dname = posixpath.dirname(to_info['path'])
         self._exec(s, 'mkdir -p {}'.format(dname))
@@ -148,6 +162,43 @@ class RemoteSSH(RemoteBase):
 
         return {self.PARAM_MD5: self.md5(path_info)}
 
+    @staticmethod
+    def to_string(path_info):
+        return "{}://{}@{}:{}".format(path_info['scheme'],
+                                      path_info['user'],
+                                      path_info['host'],
+                                      path_info['path'])
+
+    def changed_cache(self, md5):
+        cache = {}
+        cache['scheme'] = 'ssh'
+        cache['host'] = self.host
+        cache['port'] = self.port
+        cache['user'] = self.user
+        cache['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
+
+        if {self.PARAM_MD5: md5} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        md5 = checksum_info.get(self.PARAM_MD5, None)
+        if md5 is None:
+            return True
+
+        if self.changed_code(md5):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def save(self, path_info):
         if path_info['scheme'] != 'ssh':
             raise NotImplementedError
@@ -168,6 +219,25 @@ class RemoteSSH(RemoteBase):
         if not md5:
             return
 
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(md5):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(md5, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), md5))
+
         src = path_info.copy()
         src['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
 
@@ -181,7 +251,9 @@ class RemoteSSH(RemoteBase):
                                                       path_info['host'],
                                                       path_info['path']))
 
-        ssh = self.ssh(host=path_info['host'], user=path_info['user'])
+        ssh = self.ssh(host=path_info['host'],
+                       user=path_info['user'],
+                       port=path_info['port'])
         ssh.open_sftp().remove(path_info['path'])
         ssh.close()
 
@@ -192,7 +264,9 @@ class RemoteSSH(RemoteBase):
                  names=None):
         names = self._verify_path_args(from_infos, to_infos, names)
 
-        ssh = self.ssh(host=from_infos[0]['host'], user=from_infos[0]['user'])
+        ssh = self.ssh(host=from_infos[0]['host'],
+                       user=from_infos[0]['user'],
+                       port=from_infos[0]['port'])
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
             if from_info['scheme'] != 'ssh':
@@ -200,6 +274,7 @@ class RemoteSSH(RemoteBase):
 
             if to_info['scheme'] == 'ssh':
                 assert from_info['host'] == to_info['host']
+                assert from_info['port'] == to_info['port']
                 assert from_info['user'] == to_info['user']
                 self.cp(from_info, to_info, ssh=ssh)
                 continue
@@ -236,7 +311,9 @@ class RemoteSSH(RemoteBase):
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 
-        ssh = self.ssh(host=to_infos[0]['host'], user=to_infos[0]['user'])
+        ssh = self.ssh(host=to_infos[0]['host'],
+                       user=to_infos[0]['user'],
+                       port=to_infos[0]['port'])
         sftp = ssh.open_sftp()
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
@@ -277,7 +354,9 @@ class RemoteSSH(RemoteBase):
         return posixpath.dirname(relpath) + posixpath.basename(relpath)
 
     def _all_md5s(self):
-        ssh = self.ssh(host=self.host, user=self.user)
+        ssh = self.ssh(host=self.host,
+                       user=self.user,
+                       port=self.port)
         cmd = 'find {} -type f -follow -print'.format(self.prefix)
         stdout = self._exec(ssh, cmd)
         flist = stdout.split()
@@ -289,12 +368,17 @@ class RemoteSSH(RemoteBase):
         used = [info[self.PARAM_MD5] for info in cinfos['ssh']]
         used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
 
+        removed = False
         for md5 in self._all_md5s():
             if md5 in used:
                 continue
             path_info = {'scheme': 'ssh',
                          'user': self.user,
                          'host': self.host,
+                         'port': self.port,
                          'path': posixpath.join(self.prefix,
                                                 md5[0:2], md5[2:])}
             self.remove(path_info)
+            removed = True
+
+        return removed

dvc/scm.py

@@ -2,6 +2,7 @@ import os
 
 from dvc.exceptions import DvcException
 from dvc.logger import Logger
+from dvc.utils import fix_env
 
 
 class SCMError(DvcException):
@@ -87,6 +88,12 @@ class Git(Base):
         except InvalidGitRepositoryError:
             raise SCMError('{} is not a git repository'.format(root_dir))
 
+        # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
+        # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
+        env = fix_env(None)
+        lp = env.get('LD_LIBRARY_PATH', None)
+        self.repo.git.update_environment(LD_LIBRARY_PATH=lp)
+
     @staticmethod
     def is_repo(root_dir):
         git_dir = os.path.join(root_dir, Git.GIT_DIR)
@@ -117,6 +124,10 @@ class Git(Base):
             if len(filtered) != 0:
                 return
 
+        msg = "Adding '{}' to '{}'.".format(os.path.relpath(path),
+                                            os.path.relpath(gitignore))
+        Logger.info(msg)
+
         content = entry
         if len(ignore_list) > 0:
             content = '\n' + content

dvc/stage.py

@@ -1,16 +1,16 @@
 import os
 import yaml
 import itertools
+import posixpath
 import subprocess
 from schema import Schema, SchemaError, Optional, Or, And
-import posixpath
 
 import dvc.dependency as dependency
 import dvc.output as output
 from dvc.exceptions import DvcException
 from dvc.prompt import prompt
 from dvc.logger import Logger
-from dvc.utils import dict_md5
+from dvc.utils import dict_md5, fix_env
 
 
 class StageCmdFailedError(DvcException):
@@ -24,6 +24,25 @@ class StageFileFormatError(DvcException):
         super(StageFileFormatError, self).__init__('Stage file format error')
 
 
+class StageFileDoesNotExistError(DvcException):
+    def __init__(self, fname):
+        msg = "'{}' does not exist.".format(fname)
+        super(StageFileDoesNotExistError, self).__init__(msg)
+
+
+class StageFileIsNotDvcFileError(DvcException):
+    def __init__(self, fname):
+        msg = "'{}' is not a dvc file".format(fname)
+        super(StageFileIsNotDvcFileError, self).__init__(msg)
+
+
+class StageFileBadNameError(DvcException):
+    def __init__(self, fname):
+        msg = "Bad stage filename '{}'. Stage files should be named " \
+              "'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc')."
+        super(StageFileBadNameError, self).__init__(msg.format(fname, fname))
+
+
 class MissingDataSource(DvcException):
     def __init__(self, missing_files):
         assert len(missing_files) > 0
@@ -81,16 +100,20 @@ class Stage(object):
         return self.cmd is None
 
     @staticmethod
-    def is_stage_file(path):
-        if not os.path.isfile(path):
-            return False
-
+    def is_stage_filename(path):
         if not path.endswith(Stage.STAGE_FILE_SUFFIX) \
            and os.path.basename(path) != Stage.STAGE_FILE:
             return False
 
         return True
 
+    @staticmethod
+    def is_stage_file(path):
+        if not os.path.isfile(path):
+            return False
+
+        return Stage.is_stage_filename(path)
+
     def changed_md5(self):
         md5 = self.dumpd().get(self.PARAM_MD5, None)
 
@@ -219,7 +242,8 @@ class Stage(object):
         if not fname:
             fname = path.basename(out.path) + cls.STAGE_FILE_SUFFIX
 
-        cwd = path.dirname(out.path) if not cwd or add else cwd
+        if not cwd or (add and out.is_local):
+            cwd = path.dirname(out.path)
 
         return (fname, cwd)
 
@@ -253,17 +277,39 @@ class Stage(object):
 
         if os.path.exists(path):
             relpath = os.path.relpath(path)
-            msg = "{} already exists. Do you wish to overwrite it?"
+            msg = "'{}' already exists. " \
+                  "Do you wish to run the command and overwrite it?"
             if not overwrite and not prompt(msg.format(relpath), False):
-                raise DvcException("{} already exists".format(relpath))
+                raise DvcException("'{}' already exists".format(relpath))
 
         stage.cwd = cwd
         stage.path = path
 
         return stage
 
+    @staticmethod
+    def _check_dvc_filename(fname):
+        if not Stage.is_stage_filename(fname):
+            raise StageFileBadNameError(os.path.relpath(fname))
+
+    @staticmethod
+    def _check_dvc_file(fname):
+        sname = fname + Stage.STAGE_FILE_SUFFIX
+        if Stage.is_stage_file(sname):
+            Logger.info("Do you mean '{}'?".format(sname))
+
     @staticmethod
     def load(project, fname):
+        if not os.path.exists(fname):
+            Stage._check_dvc_file(fname)
+            raise StageFileDoesNotExistError(fname)
+
+        Stage._check_dvc_filename(fname)
+
+        if not Stage.is_stage_file(fname):
+            Stage._check_dvc_file(fname)
+            raise StageFileIsNotDvcFileError(fname)
+
         with open(fname, 'r') as fd:
             return Stage.loadd(project, yaml.safe_load(fd), fname)
 
@@ -292,6 +338,11 @@ class Stage(object):
         if not fname:
             fname = self.path
 
+        self._check_dvc_filename(fname)
+
+        msg = "Saving information to '{}'.".format(os.path.relpath(fname))
+        Logger.info(msg)
+
         with open(fname, 'w') as fd:
             yaml.safe_dump(self.dumpd(), fd, default_flow_style=False)
 
@@ -328,7 +379,7 @@ class Stage(object):
                 p = subprocess.Popen(self.cmd,
                                      cwd=self.cwd,
                                      shell=True,
-                                     env=os.environ,
+                                     env=fix_env(os.environ),
                                      executable=os.getenv('SHELL'))
                 p.communicate()
                 if p.returncode != 0:

dvc/state.py

@@ -96,7 +96,20 @@ class State(object):
 
     @staticmethod
     def mtime(path):
-        return str(int(nanotime.timestamp(os.path.getmtime(path))))
+        if os.path.isdir(path):
+            mtime = -1
+            for root, dirs, files in os.walk(path):
+                for fname in files:
+                    f = os.path.join(root, fname)
+                    m = os.path.getmtime(f)
+                    if m > mtime:
+                        mtime = m
+            if mtime == -1:
+                mtime = os.path.getmtime(path)
+        else:
+            mtime = os.path.getmtime(path)
+
+        return str(int(nanotime.timestamp(mtime)))
 
     @staticmethod
     def inode(path):

dvc/utils.py

@@ -2,6 +2,7 @@
 Helpers for other modules.
 """
 import os
+import sys
 import math
 import json
 import shutil
@@ -134,3 +135,24 @@ def to_chunks(l, jobs):
         n = 1
 
     return [l[x:x+n] for x in range(0, len(l), n)]
+
+
+# NOTE: Fix env variables modified by PyInstaller
+# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
+def fix_env(env):
+    if env is None:
+        env = os.environ.copy()
+    else:
+        env = env.copy()
+
+    # NOTE: Check if we are in a bundle
+    # https://pythonhosted.org/PyInstaller/runtime-information.html
+    if getattr(sys, 'frozen', False):
+        lp_key = 'LD_LIBRARY_PATH'
+        lp_orig = env.get(lp_key + '_ORIG', None)
+        if lp_orig is not None:
+            env[lp_key] = lp_orig
+        else:
+            env.pop(lp_key, None)
+
+    return env

requirements.txt

@@ -22,3 +22,5 @@ reflink==0.2.0
 requests>=2.18.4
 wheel>=0.31.1
 futures>=3.2.0; python_version == "2.7"
+grandalf==0.6
+asciicanvas==0.0.3

scripts/ci/install.sh

@@ -3,9 +3,21 @@
 set -x
 set -e
 
-pip install --upgrade pip
-pip install -r requirements.txt
-pip install -r test-requirements.txt
+N_RETRIES=3
+
+function retry {
+    for i in $(seq $N_RETRIES); do
+        $@ && break
+    done
+}
+
+# NOTE: it is not uncommon for pip to hang on travis for what seems to be
+# networking issues. Thus, let's retry a few times to see if it will eventially
+# work or not.
+retry pip install --upgrade pip
+retry pip install -r requirements.txt
+retry pip install -r test-requirements.txt
+
 git config --global user.email "[email protected]"
 git config --global user.name "DVC Tester"
 

scripts/ci/script.sh

@@ -13,6 +13,20 @@ fi
 if [[ ! -z "$TRAVIS_TAG" ]]; then
 	./scripts/build_package.sh
 
+        # Test version
+        pip uninstall -y dvc
+        if [ -x "$(command -v dvc)" ]; then
+            echo "ERROR: dvc command already exists!!!"
+            exit 1
+        fi
+
+        pip install dist/dvc-*.whl
+        if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
+            echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'"
+            exit 1
+        fi
+        pip uninstall -y dvc
+
         PY_VER=$(python -c 'import sys; print(sys.version_info[0:2])')
         if [[ "$PY_VER" != '(3, 7)' ]]; then
             ./scripts/build_posix.sh

setup.py

@@ -1,3 +1,4 @@
+import sys
 import platform
 from setuptools import setup, find_packages
 from distutils.errors import DistutilsPlatformError
@@ -22,7 +23,8 @@ install_requires = [
     "jsonpath-rw==1.4.0",
     "reflink==0.2.0",
     "requests>=2.18.4",
-    "futures>=3.2.0; python_version == \"2.7\"",
+    "grandalf==0.6",
+    "asciicanvas==0.0.3",
 ]
 
 # Extra dependencies for remote integrations
@@ -56,6 +58,8 @@ setup(
         's3': s3,
         'azure': azure,
         'ssh': ssh,
+        # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1
+        ':python_version=="2.7"': ['futures'],
     },
     keywords='data science, data version control, machine learning',
     classifiers=[

tests/test_add.py

@@ -1,6 +1,8 @@
 import os
 import stat
 import shutil
+import filecmp
+import tempfile
 
 from dvc.main import main
 from dvc.utils import file_md5
@@ -17,7 +19,10 @@ class TestAdd(TestDvc):
     def test(self):
         md5 = file_md5(self.FOO)[0]
 
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
 
         self.assertIsInstance(stage, Stage)
         self.assertTrue(os.path.isfile(stage.path))
@@ -38,8 +43,10 @@ class TestAddDirectory(TestDvc):
         dname = 'directory'
         os.mkdir(dname)
         self.create(os.path.join(dname, 'file'), 'file')
-        stage = self.dvc.add(dname)
-        self.assertNotEqual(stage, None)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(len(stage.outs), 1)
 
@@ -50,12 +57,29 @@ class TestAddDirectory(TestDvc):
             self.assertTrue('\\' not in info['relpath'])
 
 
+class TestAddDirectoryRecursive(TestDvc):
+    def test(self):
+        stages = self.dvc.add(self.DATA_DIR, recursive=True)
+        self.assertEqual(len(stages), 2)
+
+
+class TestAddCmdDirectoryRecursive(TestDvc):
+    def test(self):
+        ret = main(['add',
+                    '--recursive',
+                    self.DATA_DIR])
+        self.assertEqual(ret, 0)
+
+
 class TestAddDirectoryWithForwardSlash(TestDvc):
     def test(self):
         dname = 'directory/'
         os.mkdir(dname)
         self.create(os.path.join(dname, 'file'), 'file')
-        stage = self.dvc.add(dname)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         self.assertEquals(os.path.abspath('directory.dvc'), stage.path)
 
 class TestAddTrackedFile(TestDvc):
@@ -76,26 +100,54 @@ class TestAddDirWithExistingCache(TestDvc):
         os.mkdir(dname)
         shutil.copyfile(self.FOO, fname)
 
-        self.dvc.add(self.FOO)
-        self.dvc.add(dname)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
 
 
 class TestAddModifiedDir(TestDvc):
     def test(self):
-        self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
         os.unlink(self.DATA)
-        self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
 
 
 class TestAddFileInDir(TestDvc):
     def test(self):
-        stage = self.dvc.add(self.DATA_SUB)
+        stages = self.dvc.add(self.DATA_SUB)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
         self.assertNotEqual(stage, None)
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(len(stage.outs), 1)
         self.assertEqual(stage.relpath, self.DATA_SUB + '.dvc')
 
 
+class TestAddExternalLocalFile(TestDvc):
+    def test(self):
+        dname = tempfile.mkdtemp()
+        fname = os.path.join(dname, 'foo')
+        shutil.copyfile(self.FOO, fname)
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertNotEqual(stage, None)
+        self.assertEqual(len(stage.deps), 0)
+        self.assertEqual(len(stage.outs), 1)
+        self.assertEqual(stage.relpath, 'foo.dvc')
+        self.assertEqual(len(os.listdir(dname)), 1)
+        self.assertTrue(os.path.isfile(fname))
+        self.assertTrue(filecmp.cmp(fname, 'foo', shallow=False))
+
+
 class TestCmdAdd(TestDvc):
     def test(self):
         ret = main(['add',

tests/test_checkout.py

@@ -16,7 +16,11 @@ class TestCheckout(TestRepro):
     def setUp(self):
         super(TestCheckout, self).setUp()
 
-        self.data_dir_stage = self.dvc.add(self.DATA_DIR)
+
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.data_dir_stage = stages[0]
+        self.assertTrue(self.data_dir_stage is not None)
 
         self.orig = 'orig'
         shutil.copy(self.FOO, self.orig)
@@ -74,7 +78,9 @@ class CheckoutBase(TestDvc):
     def commit_data_file(self, fname, content='random text'):
         with open(fname, 'w') as fd:
             fd.write(content)
-        self.dvc.add(fname)
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
         self.dvc.scm.add([fname + '.dvc', '.gitignore'])
         self.dvc.scm.commit('adding ' + fname)
 
@@ -181,7 +187,10 @@ class TestCheckoutEmptyDir(TestDvc):
         dname = 'empty_dir'
         os.mkdir(dname)
 
-        stage = self.dvc.add(dname)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         self.assertEqual(len(stage.outs), 1)
 
         stage.outs[0].remove()

tests/test_data_cloud.py

@@ -204,12 +204,19 @@ class TestDataCloudBase(TestDvc):
     def _test_cloud(self):
         self._setup_cloud()
 
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         cache = stage.outs[0].cache
         info = stage.outs[0].dumpd()
         md5 = info['md5']
 
-        stage_dir = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage_dir = stages[0]
+        self.assertTrue(stage_dir is not None)
+
         cache_dir = stage_dir.outs[0].cache
         info_dir = stage_dir.outs[0].dumpd()
         md5_dir = info_dir['md5']
@@ -358,10 +365,16 @@ class TestDataCloudCLIBase(TestDvc):
         else:
             args += []
 
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         cache = stage.outs[0].cache
 
-        stage_dir = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage_dir = stages[0]
+        self.assertTrue(stage_dir is not None)
         cache_dir = stage_dir.outs[0].cache
 
         # FIXME check status output

tests/test_metrics.py

@@ -219,7 +219,9 @@ class TestCachedMetrics(TestDvc):
         with open('metrics.json', 'w+') as fd:
             json.dump({'metrics': branch}, fd)
 
-        stage = self.dvc.add('metrics.json')
+        stages = self.dvc.add('metrics.json')
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
         self.assertNotEqual(stage, None)
 
         self.dvc.scm.add(['.gitignore', 'metrics.json.dvc'])

tests/test_pipeline.py

@@ -1,6 +1,47 @@
 from dvc.main import main
 
-from tests.test_repro import TestRepro
+from tests.basic_env import TestDvc
+from tests.test_repro import TestRepro, TestReproChangedDeepData
+
+
+class TestPipelineShowSingle(TestDvc):
+    def setUp(self):
+        super(TestPipelineShowSingle, self).setUp()
+        self.stage = 'foo.dvc'
+        ret = main(['add', self.FOO])
+        self.assertEqual(ret, 0)
+
+    def test(self):
+        ret = main(['pipeline', 'show', self.stage])
+        self.assertEqual(ret, 0)
+
+    def test_commands(self):
+        ret = main(['pipeline', 'show', self.stage, '--commands'])
+        self.assertEqual(ret, 0)        
+
+    def test_outs(self):
+        ret = main(['pipeline', 'show', self.stage, '--outs'])
+        self.assertEqual(ret, 0)        
+
+    def test_ascii(self):
+        ret = main(['pipeline', 'show', '--ascii', self.stage])
+        self.assertEqual(ret, 0)
+
+    def test_ascii_commands(self):
+        ret = main(['pipeline', 'show', '--ascii', self.stage, '--commands'])
+        self.assertEqual(ret, 0)
+
+    def test_ascii_outs(self):
+        ret = main(['pipeline', 'show', '--ascii', self.stage, '--outs'])
+        self.assertEqual(ret, 0)
+
+    def test_not_dvc_file(self):
+        ret = main(['pipeline', 'show', self.FOO])
+        self.assertNotEqual(ret, 0)
+
+    def test_non_existing(self):
+        ret = main(['pipeline', 'show', 'non-existing'])
+        self.assertNotEqual(ret, 0)
 
 
 class TestPipelineShow(TestRepro):
@@ -16,6 +57,52 @@ class TestPipelineShow(TestRepro):
         ret = main(['pipeline', 'show', self.file1_stage, '--outs'])
         self.assertEqual(ret, 0)        
 
+    def test_ascii(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+    def test_ascii_commands(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+
+    def test_ascii_outs(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+
+    def test_not_dvc_file(self):
+        ret = main(['pipeline', 'show', self.file1])
+        self.assertNotEqual(ret, 0)
+
+    def test_non_existing(self):
+        ret = main(['pipeline', 'show', 'non-existing'])
+        self.assertNotEqual(ret, 0)
+
+
+class TestPipelineShowDeep(TestReproChangedDeepData):
+    def test(self):
+        ret = main(['pipeline', 'show', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+    def test_commands(self):
+        ret = main(['pipeline', 'show', self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)        
+
+    def test_outs(self):
+        ret = main(['pipeline', 'show', self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)        
+
+    def test_ascii(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+    def test_ascii_commands(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+
+    def test_ascii_outs(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+
     def test_not_dvc_file(self):
         ret = main(['pipeline', 'show', self.file1])
         self.assertNotEqual(ret, 0)

tests/test_remove.py

@@ -1,8 +1,7 @@
 import os
 
 from dvc.main import main
-from dvc.stage import Stage
-from dvc.project import NotDvcFileError
+from dvc.stage import Stage, StageFileDoesNotExistError
 from dvc.command.remove import CmdRemove
 
 from tests.basic_env import TestDvc
@@ -10,7 +9,10 @@ from tests.basic_env import TestDvc
 
 class TestRemove(TestDvc):
     def test(self):
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         stage_removed = self.dvc.remove(stage.path, outs_only=True)
 
         self.assertIsInstance(stage_removed, Stage)
@@ -26,13 +28,16 @@ class TestRemove(TestDvc):
 
 class TestRemoveNonExistentFile(TestDvc):
     def test(self):
-        with self.assertRaises(NotDvcFileError) as cx:
+        with self.assertRaises(StageFileDoesNotExistError) as cx:
             self.dvc.remove('non_existent_dvc_file')
 
 
 class TestRemoveDirectory(TestDvc):
     def test(self):
-        stage_add = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage_add = stages[0]
+        self.assertTrue(stage_add is not None)
         stage_removed = self.dvc.remove(stage_add.path)
         self.assertEqual(stage_add.path, stage_removed.path)
         self.assertFalse(os.path.exists(self.DATA_DIR))
@@ -41,7 +46,10 @@ class TestRemoveDirectory(TestDvc):
 
 class TestCmdRemove(TestDvc):
     def test(self):
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         ret = main(['remove',
                     stage.path])
         self.assertEqual(ret, 0)

tests/test_repro.py

@@ -14,10 +14,10 @@ from google.cloud import storage as gc
 
 from dvc.main import main
 from dvc.command.repro import CmdRepro
-from dvc.project import Project, ReproductionError, NotDvcFileError
+from dvc.project import Project, ReproductionError
 from dvc.utils import file_md5
 from dvc.remote.local import RemoteLOCAL
-from dvc.stage import Stage
+from dvc.stage import Stage, StageFileDoesNotExistError
 
 from tests.basic_env import TestDvc
 from tests.test_data_cloud import _should_test_aws, TEST_AWS_REPO_BUCKET
@@ -30,7 +30,10 @@ class TestRepro(TestDvc):
     def setUp(self):
         super(TestRepro, self).setUp()
 
-        self.foo_stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.foo_stage = stages[0]
+        self.assertTrue(self.foo_stage is not None)
 
         self.file1 = 'file1'
         self.file1_stage = self.file1 + '.dvc'
@@ -50,7 +53,10 @@ class TestReproFail(TestRepro):
 
 class TestReproDepUnderDir(TestDvc):
     def test(self):
-        self.dir_stage = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.dir_stage = stages[0]
+        self.assertTrue(self.dir_stage is not None)
 
         sleep()
         
@@ -134,25 +140,31 @@ class TestReproDry(TestReproChangedData):
         self.assertTrue(len(stages), 2)
         self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
 
-        ret = main(['repro', '--dry'])
+        ret = main(['repro', '--dry', self.file1_stage])
+        self.assertEqual(ret, 0)
         self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
 
 
 class TestReproChangedDeepData(TestReproChangedData):
-    def test(self):
-        file2 = 'file2'
-        file2_stage = file2 + '.dvc'
-        self.dvc.run(fname=file2_stage,
-                     outs=[file2],
-                     deps=[self.file1, self.CODE],
-                     cmd='python {} {} {}'.format(self.CODE, self.file1, file2))
+    def setUp(self):
+        super(TestReproChangedDeepData, self).setUp()
 
+        self.file2 = 'file2'
+        self.file2_stage = self.file2 + '.dvc'
+        self.dvc.run(fname=self.file2_stage,
+                     outs=[self.file2],
+                     deps=[self.file1, self.CODE],
+                     cmd='python {} {} {}'.format(self.CODE,
+                                                  self.file1,
+                                                  self.file2))
+
+    def test(self):
         self.swap_foo_with_bar()
 
-        stages = self.dvc.reproduce(file2_stage)
+        stages = self.dvc.reproduce(self.file2_stage)
 
         self.assertTrue(filecmp.cmp(self.file1, self.BAR, shallow=False))
-        self.assertTrue(filecmp.cmp(file2, self.BAR, shallow=False))
+        self.assertTrue(filecmp.cmp(self.file2, self.BAR, shallow=False))
         self.assertEqual(len(stages), 3)
 
 
@@ -180,7 +192,7 @@ class TestReproLocked(TestReproChangedData):
         self.assertEqual(len(stages), 3)
 
     def test_non_existing(self):
-        with self.assertRaises(NotDvcFileError):
+        with self.assertRaises(StageFileDoesNotExistError):
             self.dvc.lock_stage('non-existing-stage')
 
         ret = main(['lock', 'non-existing-stage'])
@@ -243,6 +255,33 @@ class TestReproChangedDir(TestDvc):
         self.assertEqual(len(stages), 1)
 
 
+class TestReproChangedDirData(TestDvc):
+    def test(self):
+        dir_name = 'dir'
+        dir_code = 'dir_code.py'
+        with open(dir_code, 'w+') as fd:
+            fd.write("import os; import sys; import shutil; shutil.copytree(sys.argv[1], sys.argv[2])")
+
+        stage = self.dvc.run(outs=[dir_name],
+                             deps=[self.DATA_DIR, dir_code],
+                             cmd="python {} {} {}".format(dir_code,
+                                                          self.DATA_DIR,
+                                                          dir_name))
+        self.assertTrue(stage is not None)
+
+        stages = self.dvc.reproduce(stage.path)
+        self.assertEqual(len(stages), 0)
+
+        sleep()
+
+        with open(self.DATA_SUB, 'a') as fd:
+            fd.write('add')
+
+        stages = self.dvc.reproduce(stage.path)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+
+
 class TestReproMissingMd5InStageFile(TestRepro):
     def test(self):
         with open(self.file1_stage, 'r') as fd:
@@ -373,6 +412,8 @@ class TestReproExternalBase(TestDvc):
 
         sleep()
 
+        self.dvc.status()
+
         stages = self.dvc.reproduce(import_stage.path)
         self.assertEqual(len(stages), 1)
         self.assertTrue(os.path.exists('import'))

tests/test_run.py

@@ -3,8 +3,9 @@ import filecmp
 
 from dvc.main import main
 from dvc.utils import file_md5
-from dvc.stage import Stage
+from dvc.stage import Stage, StageFileBadNameError
 from dvc.command.run import CmdRun
+from dvc.exceptions import OutputDuplicationError
 
 from tests.basic_env import TestDvc
 
@@ -35,6 +36,14 @@ class TestRun(TestDvc):
         self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])  
         self.assertTrue(stage.path, fname)
 
+        with self.assertRaises(OutputDuplicationError):
+            stage = self.dvc.run(cmd=cmd,
+                                 deps=deps,
+                                 outs=outs,
+                                 outs_no_cache=outs_no_cache,
+                                 fname='duplicate' + fname,
+                                 cwd=cwd)
+
 
 class TestRunEmpty(TestDvc):
     def test(self):
@@ -46,6 +55,17 @@ class TestRunEmpty(TestDvc):
                      cwd=os.curdir)
 
 
+class TestRunBadStageFilename(TestDvc):
+    def test(self):
+        with self.assertRaises(StageFileBadNameError):
+            self.dvc.run(cmd='',
+                         deps=[],
+                         outs=[],
+                         outs_no_cache=[],
+                         fname='empty',
+                         cwd=os.curdir)
+
+
 class TestRunNoExec(TestDvc):
     def test(self):
         self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, 'out'),