dvc: bump to 0.20.7

dagascii: don't use unreleased wait_for_input method

.appveyor.yml

@@ -54,7 +54,7 @@ install:
   - cinst gcloudsdk
   - refreshenv
   - cinst gsutil
-  - cinst openssl.light
+  - cinst openssl.light --version 1.1.1
   - python -m pip install -U pip setuptools wheel
   - pip install -r requirements.txt
   - python setup.py install

.codeclimate.yml

@@ -1,14 +1,5 @@
 languages:
     Python: true
 
-checks:
-    argument-count:
-        enabled: false
-    method-complexity:
-        enabled: false
-    method-lines:
-        config:
-            threshold: 50
-
 exclude_paths:
     - tests/

.coveragerc

@@ -8,3 +8,4 @@ exclude_lines =
     pass
     raise NotImplementedError
     # pragma: no cover
+    #  pragma: no cover

.dvc/.gitignore

@@ -5,4 +5,6 @@ config.local
 updater
 cache
 state
-state.lock
+state.lock
+state-journal
+state-wal

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,10 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+
+**Please provide information about your setup**
+DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,6 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+cache: pip
 sudo: required
 dist: trusty
 group: deprecated-2017Q4
@@ -15,18 +16,24 @@ python:
 # version, that causes fpm to crash.
 matrix:
   include:
+  # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
+  - python: "3.7"
+    dist: xenial
+    sudo: true
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
     - PYTHON_VER=2.7.10
     - PYENV_ROOT="$HOME/.pyenv"
     - PATH="$PYENV_ROOT/bin:$PATH"
+    - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
+    - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
@@ -35,23 +42,20 @@ matrix:
     - PATH="$PYENV_ROOT/bin:$PATH"
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
     - PYTHON_VER=3.5.0
     - PYENV_ROOT="$HOME/.pyenv"
     - PATH="$PYENV_ROOT/bin:$PATH"
-  # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
-  - python: "3.7"
-    dist: xenial
-    sudo: true
 before_install:
   - bash ./scripts/ci/before_install.sh
   - if  [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       eval "$(pyenv init -)";
       PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
       pyenv shell "$PYTHON_VER";
+      brew install openssl;
       brew cask install google-cloud-sdk;
     fi
   - which python
@@ -77,6 +81,8 @@ install:
   - bash ./scripts/ci/install.sh
   - pip install flake8
 before_script:
+  # stop the build if there are any readme formatting errors
+  - python setup.py checkdocs
   # stop the build if there are any flake8 errors
   - flake8 dvc
 script:

README.rst

@@ -2,9 +2,12 @@
   :target: https://dvc.org
   :alt: DVC logo
 
-`Website <https://dvc.org>`_ • `Docs <https://dvc.org/doc>`_ • `Discuss <https://discuss.dvc.org/>`_ •
-`Blog <https://blog.dataversioncontrol.com/>`_ • `Twitter <https://twitter.com/iterativeai>`_ •
-`Tutorial <https://dvc.org/doc/tutorial>`_
+`Website <https://dvc.org>`_
+• `Docs <https://dvc.org/doc>`_
+• `Twitter <https://twitter.com/iterativeai>`_
+• `Chat (Community & Support) <https://dvc.org/chat>`_
+• `Tutorial <https://dvc.org/doc/tutorial>`_
+• `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
 .. image:: https://travis-ci.org/iterative/dvc.svg?branch=master
   :target: https://travis-ci.org/iterative/dvc
@@ -24,10 +27,12 @@
 
 |
 
-**Data Science Version Control** or **DVC** is an **open-source** tool for data science projects.
-With a simple and flexible Git-like architecture and interface it helps data scientists:
+**Data Science Version Control** or **DVC** is an **open-source** tool for data science and
+machine learning projects. With a simple and flexible Git-like architecture and interface it
+helps data scientists:
 
-#. manage code together with data files and models;
+#. manage **machine learning models** - versioning, including data sets and transformations (scripts) that were
+   used to generate models;
 
 #. make projects **reproducible**;
 
@@ -42,6 +47,17 @@ data file suffixes and prefixes.
 .. contents:: **Contents**
   :backlinks: none
 
+How DVC works
+=============
+
+DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
+To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
+network storage (via SSH, for example).
+
+.. image:: https://dvc.org/static/img/flow.gif
+   :target: https://dvc.org/static/img/flow.gif
+   :alt: how_dvc_works
+
 Quick start
 ===========
 
@@ -64,7 +80,7 @@ Please read `Get Started <https://dvc.org/doc/get-started>`_ for the full versio
 |                                   | | ``$ git push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
 | Share data and ML models          | | ``$ dvc remote add myremote s3://mybucket/image_cnn``           |
-|                                   | | ``$ dvc core.remote myremote``                                  |
+|                                   | | ``$ dvc config core.remote myremote``                           |
 |                                   | | ``$ dvc push``                                                  |
 +-----------------------------------+-------------------------------------------------------------------+
 
@@ -73,15 +89,20 @@ Installation
 
 There are three options to install DVC: ``pip``, Homebrew, or an OS-specific package:
 
+pip (PyPI)
+----------
+
+Stable
+^^^^^^
 .. code-block:: bash
 
    pip install dvc
 
-Package
--------
+Development
+^^^^^^^^^^^
+.. code-block:: bash
 
-Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
-GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+   pip install git+git://github.com/iterative/dvc
 
 Homebrew
 --------
@@ -96,6 +117,37 @@ or:
 
    brew cask install iterative/homebrew-dvc/dvc
 
+Package
+-------
+
+Self-contained packages for Windows, Linux, Mac are available. The latest version of the packages can be found at
+GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+
+Ubuntu / Debian (apt)
+^^^^^^^^^^^^^^^^^^^^^
+.. code-block:: bash
+
+   sudo wget https://dvc.org/deb/dvc.list -O etc/apt/sources.list.d/dvc.list
+   sudo apt-get update
+   sudo apt-get install dvc
+
+Fedora / CentOS (rpm)
+^^^^^^^^^^^^^^^^^^^^^
+.. code-block:: bash
+
+   sudo wget https://dvc.org/rpm/dvc.repo -O /etc/yum.repos.d/dvc.repo
+   sudo yum update
+   sudo yum install dvc
+
+Arch linux (AUR)
+^^^^^^^^^^^^^^^^
+*Unofficial package*, any inquiries regarding the AUR package,
+`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
+
+.. code-block:: bash
+
+   yay -S dvc
+
 Related technologies
 ====================
 
@@ -112,23 +164,48 @@ Related technologies
 #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_. DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-How DVC works
-=============
-
-DVC is compatible with Git for storing code and the dependency graph (DAG), but not data files cache.
-To store and share data files cache DVC supports remotes - any cloud (S3, Azure, Google Cloud, etc) or any on-premise
-network storage (via SSH, for example).
-
-.. image:: https://dvc.org/static/img/flow.png
-   :target: https://dvc.org/static/img/flow.png
-   :alt: how_dvc_works
-
-
 Contributing
 ============
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/>`_ for more
 details.
 
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+  :alt: 0
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+  :alt: 1
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+  :alt: 2
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
+  :alt: 3
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+  :alt: 4
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+  :alt: 5
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+  :alt: 6
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
+  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+  :alt: 7
+
+Mailing List
+============
+
+Want to stay up to date? Want to help improve DVC by participating in our ocassional polls? Subscribe to our `mailing list <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_. No spam, really low traffic.
+
 Copyright
 =========
 

dvc/__init__.py

@@ -4,8 +4,10 @@ DVC
 Make your data science projects reproducible and shareable.
 """
 import os
+import warnings
 
-VERSION_BASE = '0.16.4'
+
+VERSION_BASE = '0.20.7'
 __version__ = VERSION_BASE
 
 PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
@@ -31,13 +33,13 @@ if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
             with open(VERSIONPATH, 'w+') as fd:
                 fd.write('# AUTOGENERATED by dvc/__init__.py\n')
                 fd.write('version = "{}"\n'.format(__version__))
-        except Exception:
+        except Exception:  # pragma: no cover
             pass
-    else:
+    else:  # pragma: no cover
         # Remove version.py so that it doesn't get into the release
         if os.path.exists(VERSIONPATH):
             os.unlink(VERSIONPATH)
-else:
+else:  # pragma: no cover
     # dvc was installed with pip or something. Hopefully we have our
     # auto-generated version.py to help us provide a true version
     try:
@@ -47,3 +49,11 @@ else:
         pass
 
 VERSION = __version__
+
+
+# Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
+# We don't directly import numpy, but our dependency networkx does, causing
+# these warnings in some environments. Luckily these warnings are benign and
+# we can simply ignore them so that they don't show up when you are using dvc.
+warnings.filterwarnings("ignore", message="numpy.dtype size changed")
+warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

dvc/cache.py

@@ -12,21 +12,25 @@ class Cache(object):
         self.project = project
 
         config = project.config._config[Config.SECTION_CACHE]
+        local = config.get(Config.SECTION_CACHE_LOCAL)
 
-        local = config.get(Config.SECTION_CACHE_LOCAL, None)
         if local:
             name = Config.SECTION_REMOTE_FMT.format(local)
             sect = project.config._config[name]
         else:
-            sect = {}
             cache_dir = config.get(Config.SECTION_CACHE_DIR, self.CACHE_DIR)
+            cache_type = config.get(Config.SECTION_CACHE_TYPE)
+            protected = config.get(Config.SECTION_CACHE_PROTECTED)
+
             if not os.path.isabs(cache_dir):
                 cache_dir = os.path.join(project.dvc_dir, cache_dir)
                 cache_dir = os.path.abspath(os.path.realpath(cache_dir))
-            sect[Config.SECTION_REMOTE_URL] = cache_dir
-            t = config.get(Config.SECTION_CACHE_TYPE, None)
-            if t:
-                sect[Config.SECTION_CACHE_TYPE] = t
+
+            sect = {
+                Config.SECTION_REMOTE_URL: cache_dir,
+                Config.SECTION_CACHE_TYPE: cache_type,
+                Config.SECTION_CACHE_PROTECTED: protected
+            }
 
         self.local = Remote(project, sect)
 

dvc/command/add.py

@@ -7,7 +7,7 @@ class CmdAdd(CmdBase):
     def run(self):
         for target in self.args.targets:
             try:
-                self.project.add(target)
+                self.project.add(target, recursive=self.args.recursive)
             except DvcException as ex:
                 Logger.error('Failed to add \'{}\''.format(target), ex)
                 return 1

dvc/command/base.py

@@ -15,9 +15,10 @@ class CmdBase(object):
 
     @staticmethod
     def _set_loglevel(args):
-        if args.quiet and not args.verbose:
+        if args.quiet:
             Logger.be_quiet()
-        elif not args.quiet and args.verbose:
+
+        elif args.verbose:
             Logger.be_verbose()
 
     def _find_root(self):

dvc/command/checkout.py

@@ -4,8 +4,10 @@ from dvc.command.base import CmdBase
 class CmdCheckout(CmdBase):
     def run(self):
         if not self.args.targets:
-            self.project.checkout()
+            self.project.checkout(force=self.args.force)
         else:
             for target in self.args.targets:
-                self.project.checkout(target=target)
+                self.project.checkout(target=target,
+                                      with_deps=self.args.with_deps,
+                                      force=self.args.force)
         return 0

dvc/command/daemon.py

@@ -0,0 +1,19 @@
+from dvc.command.base import CmdBase
+
+
+class CmdDaemonUpdater(CmdBase):
+    def __init__(self, args):
+        pass
+
+    def run_cmd(self):
+        return self.run()
+
+    def run(self):
+        import os
+        from dvc.project import Project
+        from dvc.updater import Updater
+
+        root_dir = self._find_root()
+        dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
+        updater = Updater(dvc_dir)
+        updater.fetch(detach=False)

dvc/command/data_sync.py

@@ -23,7 +23,10 @@ class CmdDataPull(CmdDataBase):
                               jobs=self.args.jobs,
                               remote=self.args.remote,
                               show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches)
+                              all_branches=self.args.all_branches,
+                              all_tags=self.args.all_tags,
+                              with_deps=self.args.with_deps,
+                              force=self.args.force)
         except Exception as exc:
             self.project.logger.error('Failed to pull data from the cloud',
                                       exc)
@@ -38,7 +41,9 @@ class CmdDataPush(CmdDataBase):
                               jobs=self.args.jobs,
                               remote=self.args.remote,
                               show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches)
+                              all_branches=self.args.all_branches,
+                              all_tags=self.args.all_tags,
+                              with_deps=self.args.with_deps)
         except Exception as exc:
             self.project.logger.error('Failed to push data to the cloud', exc)
             return 1
@@ -52,7 +57,9 @@ class CmdDataFetch(CmdDataBase):
                                jobs=self.args.jobs,
                                remote=self.args.remote,
                                show_checksums=self.args.show_checksums,
-                               all_branches=self.args.all_branches)
+                               all_branches=self.args.all_branches,
+                               all_tags=self.args.all_tags,
+                               with_deps=self.args.with_deps)
         except Exception as exc:
             self.project.logger.error('Failed to fetch data from the cloud',
                                       exc)

dvc/command/destroy.py

@@ -1,16 +1,16 @@
 from dvc.command.base import CmdBase
-from dvc.prompt import prompt
 from dvc.exceptions import DvcException
 
 
 class CmdDestroy(CmdBase):
     def run_cmd(self):
         try:
-            msg = u'This will destroy all information about your pipelines ' \
-                  u'as well as cache in .dvc/cache.\n' \
+            msg = u'This will destroy all information about your pipelines, ' \
+                  u'all data files, as well as cache in .dvc/cache.\n' \
                   u'Are you sure you want to continue?'
 
-            if not self.args.force and not prompt(msg, False):
+            if not self.args.force \
+               and not self.project.prompt.prompt(msg, False):
                 msg = u'Cannot destroy without a confirmation from the ' \
                       u'user. Use \'-f\' to force.'
                 raise DvcException(msg)

dvc/command/gc.py

@@ -1,9 +1,39 @@
+import os
+
 from dvc.command.base import CmdBase
 
 
 class CmdGC(CmdBase):
     def run(self):
+        msg = 'This will remove all cache except the cache that is used in '
+        if not self.args.all_branches and not self.args.all_tags:
+            msg += 'the current git branch'
+        elif self.args.all_branches and not self.args.all_tags:
+            msg += 'all git branches'
+        elif not self.args.all_branches and self.args.all_tags:
+            msg += 'all git tags'
+        else:
+            msg += 'all git branches and all git tags'
+
+        if self.args.projects is not None and len(self.args.projects) > 0:
+            msg += ' of the current and the following projects:'
+
+            for project_path in self.args.projects:
+                msg += '\n  - %s' % os.path.abspath(project_path)
+        else:
+            msg += ' of the current project.'
+
+        self.project.logger.warn(msg)
+
+        msg = 'Are you sure you want to proceed?'
+        if not self.args.force and not self.project.prompt.prompt(msg):
+            return 1
+
         self.project.gc(all_branches=self.args.all_branches,
+                        all_tags=self.args.all_tags,
                         cloud=self.args.cloud,
-                        remote=self.args.remote)
+                        remote=self.args.remote,
+                        force=self.args.force,
+                        jobs=self.args.jobs,
+                        projects=self.args.projects)
         return 0

dvc/command/metrics.py

@@ -28,7 +28,8 @@ class CmdMetricsShow(CmdBase):
             self.project.metrics_show(self.args.path,
                                       typ=typ,
                                       xpath=xpath,
-                                      all_branches=self.args.all_branches)
+                                      all_branches=self.args.all_branches,
+                                      all_tags=self.args.all_tags)
         except DvcException as exc:
             self.project.logger.error('Failed to show metrics', exc)
             return 1

dvc/command/pipeline.py

@@ -2,34 +2,127 @@ import os
 
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase
+from dvc.dagascii import Dagascii
 
 
 class CmdPipelineShow(CmdBase):
     def _show(self, target, commands, outs):
         import networkx
-        from dvc.project import NotDvcFileError
+        from dvc.stage import Stage
 
+        stage = Stage.load(self.project, target)
         G = self.project.graph()[0]
         stages = networkx.get_node_attributes(G, 'stage')
-        node = os.path.relpath(os.path.abspath(target), self.project.root_dir)
-        if node not in stages:
-            raise NotDvcFileError(node)
+        node = os.path.relpath(stage.path, self.project.root_dir)
 
         for n in networkx.dfs_postorder_nodes(G, node):
             if commands:
                 self.project.logger.info(stages[n].cmd)
             elif outs:
                 for out in stages[n].outs:
-                    self.project.logger.info(out.rel_path)
+                    self.project.logger.info(str(out))
             else:
                 self.project.logger.info(n)
 
+    def __build_graph(self, target, commands, outs):
+        import networkx
+        from dvc.stage import Stage
+
+        stage = Stage.load(self.project, target)
+        node = os.path.relpath(stage.path, self.project.root_dir)
+
+        pipelines = list(filter(lambda g: node in g.nodes(),
+                                self.project.pipelines()))
+
+        assert len(pipelines) == 1
+        G = pipelines[0]
+        stages = networkx.get_node_attributes(G, 'stage')
+
+        nodes = []
+        for n in G.nodes():
+            stage = stages[n]
+            if commands:
+                if stage.cmd is None:
+                    continue
+                nodes.append(stage.cmd)
+            elif outs:
+                for out in stage.outs:
+                    nodes.append(str(out))
+            else:
+                nodes.append(stage.relpath)
+
+        edges = []
+        for e in G.edges():
+            from_stage = stages[e[0]]
+            to_stage = stages[e[1]]
+            if commands:
+                if to_stage.cmd is None:
+                    continue
+                edges.append((from_stage.cmd, to_stage.cmd))
+            elif outs:
+                for from_out in from_stage.outs:
+                    for to_out in to_stage.outs:
+                        edges.append((str(from_out),
+                                      str(to_out)))
+            else:
+                edges.append((from_stage.relpath, to_stage.relpath))
+
+        return nodes, edges
+
+    def _show_ascii(self, target, commands, outs):
+        nodes, edges = self.__build_graph(target, commands, outs)
+
+        if not nodes:
+            return
+
+        d = Dagascii(nodes, edges)
+        d.draw()
+
+    def __write_dot(self, target, commands, outs, filename):
+        import networkx
+        from networkx.drawing.nx_pydot import write_dot
+
+        _, edges = self.__build_graph(target, commands, outs)
+        edges = [edge[::-1] for edge in edges]
+
+        simple_g = networkx.DiGraph()
+        simple_g.add_edges_from(edges)
+        write_dot(simple_g, filename)
+
     def run(self, unlock=False):
         for target in self.args.targets:
             try:
-                self._show(target, self.args.commands, self.args.outs)
+                if self.args.ascii:
+                    self._show_ascii(target,
+                                     self.args.commands,
+                                     self.args.outs)
+                elif self.args.dot:
+                    self.__write_dot(target,
+                                     self.args.commands,
+                                     self.args.outs,
+                                     self.args.dot)
+                else:
+                    self._show(target,
+                               self.args.commands,
+                               self.args.outs)
             except DvcException as ex:
                 msg = 'Failed to show pipeline for \'{}\''.format(target)
                 self.project.logger.error(msg, ex)
                 return 1
         return 0
+
+
+class CmdPipelineList(CmdBase):
+    def run(self):
+        import networkx
+
+        pipelines = self.project.pipelines()
+        for p in pipelines:
+            stages = networkx.get_node_attributes(p, 'stage')
+            for stage in stages:
+                self.project.logger.info(stage)
+            if len(stages) != 0:
+                self.project.logger.info("="*80)
+        self.project.logger.info("{} pipeline(s) total".format(len(pipelines)))
+
+        return 0

dvc/command/remote.py

@@ -1,4 +1,6 @@
+import os
 import re
+import configobj
 
 from dvc.config import Config
 from dvc.command.config import CmdConfig
@@ -23,9 +25,35 @@ class CmdRemoteAdd(CmdConfig):
 
 
 class CmdRemoteRemove(CmdConfig):
+    def _remove_default(self, config_file, remote):
+        path = os.path.join(os.path.dirname(self.config_file),
+                            config_file)
+        config = configobj.ConfigObj(path)
+
+        core = config.get(Config.SECTION_CORE, None)
+        if core is None:
+            return
+
+        default = core.get(Config.SECTION_CORE_REMOTE, None)
+        if default is None:
+            return
+
+        if default == remote:
+            del config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE]
+            if len(config[Config.SECTION_CORE]) == 0:
+                del config[Config.SECTION_CORE]
+
+        config.write()
+
     def run(self):
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        return self.unset(section, Config.SECTION_REMOTE_URL)
+        ret = self.unset(section)
+        if ret != 0:
+            return ret
+
+        self._remove_default(Config.CONFIG, self.args.name)
+        self._remove_default(Config.CONFIG_LOCAL, self.args.name)
+        return 0
 
 
 class CmdRemoteModify(CmdConfig):

dvc/command/remove.py

@@ -4,10 +4,32 @@ from dvc.logger import Logger
 
 
 class CmdRemove(CmdBase):
+    def _is_outs_only(self, target):
+        if not self.args.purge:
+            return True
+
+        if self.args.force:
+            return False
+
+        msg = (
+            u'Are you sure you want to remove {} with its outputs?'
+            .format(target)
+        )
+
+        confirmed = self.project.prompt.prompt(msg)
+
+        if confirmed:
+            return False
+
+        raise DvcException(
+            u'Cannot purge without a confirmation from the user.'
+            u" Use '-f' to force."
+        )
+
     def run(self):
-        outs_only = not self.args.purge
         for target in self.args.targets:
             try:
+                outs_only = self._is_outs_only(target)
                 self.project.remove(target, outs_only=outs_only)
             except DvcException as ex:
                 Logger.error('Failed to remove {}'.format(target), ex)

dvc/command/repro.py

@@ -1,12 +1,12 @@
 import os
 
 from dvc.command.base import CmdBase
+from dvc.command.status import CmdDataStatus
+from dvc.exceptions import DvcException
 
 
 class CmdRepro(CmdBase):
     def run(self):
-        from dvc.project import ReproductionError
-
         recursive = not self.args.single_item
         saved_dir = os.path.realpath(os.curdir)
         if self.args.cwd:
@@ -15,23 +15,25 @@ class CmdRepro(CmdBase):
         ret = 0
         for target in self.args.targets:
             try:
-                self.project.reproduce(target,
+                stages = self.project.reproduce(
+                                       target,
                                        recursive=recursive,
                                        force=self.args.force,
                                        dry=self.args.dry,
-                                       interactive=self.args.interactive)
+                                       interactive=self.args.interactive,
+                                       pipeline=self.args.pipeline,
+                                       all_pipelines=self.args.all_pipelines)
+
+                if len(stages) == 0:
+                    self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+
                 if self.args.metrics:
                     self.project.metrics_show()
-            except ReproductionError as ex:
+            except DvcException as ex:
                 msg = 'Failed to reproduce \'{}\''.format(target)
                 self.project.logger.error(msg, ex)
                 ret = 1
                 break
-            except Exception as ex:
-                msg = 'Failed to reproduce \'{}\' - unexpected error'
-                self.project.logger.error(msg.format(target), ex)
-                ret = 1
-                break
 
         os.chdir(saved_dir)
         return ret

dvc/command/root.py

@@ -4,6 +4,9 @@ from dvc.command.base import CmdBase
 
 
 class CmdRoot(CmdBase):
+    def run_cmd(self):
+        return self.run()
+
     def run(self):
         self.project.logger.info(os.path.relpath(self.project.root_dir))
         return 0

dvc/command/run.py

@@ -5,6 +5,9 @@ from dvc.exceptions import DvcException
 class CmdRun(CmdBase):
     def run(self):
         try:
+            if self.args.yes:
+                self.project.prompt.default = True
+
             self.project.run(cmd=' '.join(self.args.command),
                              outs=self.args.outs,
                              outs_no_cache=self.args.outs_no_cache,

dvc/command/status.py

@@ -4,6 +4,7 @@ from dvc.command.data_sync import CmdDataBase
 class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 10
     STATUS_INDENT = '\t'
+    UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce."
 
     def _normalize(self, s):
         s += ':'
@@ -28,8 +29,15 @@ class CmdDataStatus(CmdDataBase):
                                      jobs=self.args.jobs,
                                      cloud=self.args.cloud,
                                      show_checksums=self.args.show_checksums,
-                                     remote=self.args.remote)
-            self._show(st, indent)
+                                     remote=self.args.remote,
+                                     all_branches=self.args.all_branches,
+                                     all_tags=self.args.all_tags,
+                                     with_deps=self.args.with_deps)
+            if st:
+                self._show(st, indent)
+            else:
+                self.project.logger.info(self.UP_TO_DATE_MSG)
+
         except Exception as exc:
             self.project.logger.error('Failed to obtain data status', exc)
             return 1

dvc/command/unprotect.py

@@ -0,0 +1,14 @@
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+
+
+class CmdUnprotect(CmdBase):
+    def run(self):
+        for target in self.args.targets:
+            try:
+                self.project.unprotect(target)
+            except DvcException as ex:
+                msg = "Failed to unprotect '{}'".format(target)
+                self.project.logger.error(msg, ex)
+                return 1
+        return 0

dvc/config.py

@@ -10,8 +10,9 @@ from dvc.exceptions import DvcException
 
 class ConfigError(DvcException):
     """ DVC config exception """
-    def __init__(self, ex=None):
-        super(ConfigError, self).__init__('Config file error', ex)
+    def __init__(self, msg, ex=None):
+        super(ConfigError, self).__init__('Config file error: {}'.format(msg),
+                                          ex)
 
 
 def supported_url(url):
@@ -36,6 +37,22 @@ def supported_cloud(cloud):
     return cloud in ['aws', 'gcp', 'local', '']
 
 
+def is_bool(val):
+    return val.lower() in ['true', 'false']
+
+
+def to_bool(val):
+    return val.lower() == 'true'
+
+
+def is_whole(val):
+    return int(val) >= 0
+
+
+def is_percent(val):
+    return int(val) >= 0 and int(val) <= 100
+
+
 class Config(object):
     CONFIG = 'config'
     CONFIG_LOCAL = 'config.local'
@@ -44,11 +61,14 @@ class Config(object):
     SECTION_CORE_LOGLEVEL = 'loglevel'
     SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
     SECTION_CORE_REMOTE = 'remote'
+    SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
+    SECTION_CORE_INTERACTIVE = 'interactive'
 
     SECTION_CACHE = 'cache'
     SECTION_CACHE_DIR = 'dir'
     SECTION_CACHE_TYPE = 'type'
     SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
+    SECTION_CACHE_PROTECTED = 'protected'
     SECTION_CACHE_LOCAL = 'local'
     SECTION_CACHE_S3 = 's3'
     SECTION_CACHE_GS = 'gs'
@@ -63,9 +83,10 @@ class Config(object):
         Optional(SECTION_CACHE_SSH): str,
         Optional(SECTION_CACHE_AZURE): str,
 
-        # backward compatibility
         Optional(SECTION_CACHE_DIR, default='cache'): str,
         Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
+        Optional(SECTION_CACHE_PROTECTED,
+                 default=False): And(str, is_bool, Use(to_bool)),
     }
 
     # backward compatibility
@@ -78,6 +99,8 @@ class Config(object):
                  default='info'): And(str, Use(str.lower),
                                       SECTION_CORE_LOGLEVEL_SCHEMA),
         Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)),
+        Optional(SECTION_CORE_INTERACTIVE,
+                 default=False): SECTION_CORE_INTERACTIVE_SCHEMA,
 
         # backward compatibility
         Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
@@ -94,9 +117,9 @@ class Config(object):
     SECTION_AWS_SCHEMA = {
         SECTION_AWS_STORAGEPATH: str,
         Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE, default='default'): str,
-        Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
-        Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
+        Optional(SECTION_AWS_PROFILE): str,
+        Optional(SECTION_AWS_CREDENTIALPATH): str,
+        Optional(SECTION_AWS_ENDPOINT_URL): str,
     }
 
     # backward compatibility
@@ -119,21 +142,42 @@ class Config(object):
     SECTION_REMOTE_FMT = 'remote "{}"'
     SECTION_REMOTE_URL = 'url'
     SECTION_REMOTE_USER = 'user'
+    SECTION_REMOTE_PORT = 'port'
+    SECTION_REMOTE_KEY_FILE = 'keyfile'
+    SECTION_REMOTE_TIMEOUT = 'timeout'
+    SECTION_REMOTE_PASSWORD = 'password'
+    SECTION_REMOTE_ASK_PASSWORD = 'ask_password'
     SECTION_REMOTE_SCHEMA = {
         SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
         Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE, default='default'): str,
-        Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
-        Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
+        Optional(SECTION_AWS_PROFILE): str,
+        Optional(SECTION_AWS_CREDENTIALPATH): str,
+        Optional(SECTION_AWS_ENDPOINT_URL): str,
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
+        Optional(SECTION_CACHE_PROTECTED,
+                 default=False): And(str, is_bool, Use(to_bool)),
         Optional(SECTION_REMOTE_USER): str,
+        Optional(SECTION_REMOTE_PORT): Use(int),
+        Optional(SECTION_REMOTE_KEY_FILE): str,
+        Optional(SECTION_REMOTE_TIMEOUT): Use(int),
+        Optional(SECTION_REMOTE_PASSWORD): str,
+        Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
+    }
+
+    SECTION_STATE = 'state'
+    SECTION_STATE_ROW_LIMIT = 'row_limit'
+    SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota'
+    SECTION_STATE_SCHEMA = {
+        Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
+        Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
     }
 
     SCHEMA = {
         Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
         Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
         Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
+        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
 
         # backward compatibility
         Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
@@ -154,7 +198,7 @@ class Config(object):
             # need to convert our config to dict before passing it to
             self._config = self._lower(self._config)
             local = self._lower(local)
-            self._config.update(local)
+            self._config = self._merge(self._config, local)
 
             self._config = Schema(self.SCHEMA).validate(self._config)
 
@@ -165,6 +209,17 @@ class Config(object):
         except Exception as ex:
             raise ConfigError(ex)
 
+    @staticmethod
+    def _merge(first, second):
+        res = {}
+        sections = list(first.keys()) + list(second.keys())
+        for section in sections:
+            f = first.get(section, {}).copy()
+            s = second.get(section, {}).copy()
+            f.update(s)
+            res[section] = f
+        return res
+
     @staticmethod
     def _lower(config):
         new_config = {}

dvc/daemon.py

@@ -0,0 +1,62 @@
+import os
+import sys
+from subprocess import Popen
+from dvc.logger import Logger
+
+
+class Daemon(object):  # pragma: no cover
+    def _spawn_windows(self, cmd):
+        CREATE_NEW_PROCESS_GROUP = 0x00000200
+        DETACHED_PROCESS = 0x00000008
+        creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
+        Popen(cmd,
+              close_fds=True,
+              shell=False,
+              creationflags=creationflags)
+
+    def _spawn_posix(self, cmd):
+        try:
+            pid = os.fork()
+            if pid > 0:
+                return
+        except OSError as exc:
+            Logger.error("Failed at first fork", exc)
+            sys.exit(1)
+
+        os.setsid()
+        os.umask(0)
+
+        try:
+            pid = os.fork()
+            if pid > 0:
+                sys.exit(0)
+        except OSError as exc:
+            Logger.error("Failed at second fork", exc)
+            sys.exit(1)
+
+        sys.stdin.close()
+        sys.stdout.close()
+        sys.stderr.close()
+
+        Popen(cmd,
+              close_fds=True,
+              shell=False)
+
+    def __call__(self, name):
+        from dvc.utils import is_binary
+
+        cmd = [sys.executable]
+        if not is_binary():
+            cmd += ['-m', 'dvc']
+        cmd += ['daemon', name, '-q']
+
+        Logger.debug("Trying to spawn '{}'".format(cmd))
+
+        if os.name == 'nt':
+            self._spawn_windows(cmd)
+        elif os.name == 'posix':
+            self._spawn_posix(cmd)
+        else:
+            raise NotImplementedError
+
+        Logger.debug("Spawned '{}'".format(cmd))

dvc/dagascii.py

@@ -0,0 +1,286 @@
+import sys
+import math
+
+from grandalf.graphs import Vertex, Edge, Graph
+from grandalf.layouts import SugiyamaLayout
+from grandalf.routing import route_with_lines, EdgeViewer
+
+
+class AsciiCanvas(object):
+    TIMEOUT = 10
+
+    def __init__(self, cols, lines):
+        assert cols > 1
+        assert lines > 1
+
+        self.cols = cols
+        self.lines = lines
+
+        self.canvas = [[' '] * cols for l in range(lines)]
+
+    def draw(self):
+        if sys.stdout.isatty():  # pragma: no cover
+            from asciimatics.screen import Screen
+            Screen.wrapper(self._do_draw)
+        else:
+            for line in self.canvas:
+                print(''.join(line))
+
+    def _do_draw(self, screen):  # pragma: no cover
+        from dvc.system import System
+        from asciimatics.event import KeyboardEvent
+
+        offset_x = 0
+        offset_y = 0
+        smaxrow, smaxcol = screen.dimensions
+        assert smaxrow > 1
+        assert smaxcol > 1
+        smaxrow -= 1
+        smaxcol -= 1
+
+        if self.lines + 1 > smaxrow:
+            max_y = self.lines + 1 - smaxrow
+        else:
+            max_y = 0
+
+        if self.cols + 1 > smaxcol:
+            max_x = self.cols + 1 - smaxcol
+        else:
+            max_x = 0
+
+        while True:
+            for y in range(smaxrow + 1):
+                y_index = offset_y + y
+                line = []
+                for x in range(smaxcol + 1):
+                    x_index = offset_x + x
+                    if len(self.canvas) > y_index \
+                       and len(self.canvas[y_index]) > x_index:
+                            line.append(self.canvas[y_index][x_index])
+                    else:
+                        line.append(' ')
+                assert len(line) == (smaxcol + 1)
+                screen.print_at(''.join(line), 0, y)
+
+            screen.refresh()
+
+            # NOTE: get_event() doesn't block by itself,
+            # so we have to do the blocking ourselves.
+            #
+            # NOTE: using this workaround while waiting for PR [1]
+            # to get merged and released. After that need to adjust
+            # asciimatics version requirements.
+            #
+            # [1] https://github.com/peterbrittain/asciimatics/pull/188
+            System.wait_for_input(self.TIMEOUT)
+
+            event = screen.get_event()
+            if not isinstance(event, KeyboardEvent):
+                continue
+
+            k = event.key_code
+            if k == screen.KEY_DOWN or k == ord('s'):
+                offset_y += 1
+            elif k == screen.KEY_PAGE_DOWN or k == ord('S'):
+                offset_y += smaxrow
+            elif k == screen.KEY_UP or k == ord('w'):
+                offset_y -= 1
+            elif k == screen.KEY_PAGE_UP or k == ord('W'):
+                offset_y -= smaxrow
+            elif k == screen.KEY_RIGHT or k == ord('d'):
+                offset_x += 1
+            elif k == ord('D'):
+                offset_x += smaxcol
+            elif k == screen.KEY_LEFT or k == ord('a'):
+                offset_x -= 1
+            elif k == ord('A'):
+                offset_x -= smaxcol
+            elif k == ord('q') or k == ord('Q'):
+                break
+
+            if offset_y > max_y:
+                offset_y = max_y
+            elif offset_y < 0:
+                offset_y = 0
+
+            if offset_x > max_x:
+                offset_x = max_x
+            elif offset_x < 0:
+                offset_x = 0
+
+    def point(self, x, y, char):
+        assert x >= 0
+        assert x < self.cols
+        assert y >= 0
+        assert y < self.lines
+
+        self.canvas[y][x] = char
+
+    def line(self, x0, y0, x1, y1, char):
+        if x0 > x1:
+            x1, x0 = x0, x1
+            y1, y0 = y0, y1
+
+        dx = x1 - x0
+        dy = y1 - y0
+
+        if dx == 0 and dy == 0:
+            self.point(x0, y0, char)
+        elif abs(dx) >= abs(dy):
+            for x in range(x0, x1 + 1):
+                if dx == 0:
+                    y = y0
+                else:
+                    y = y0 + int(round((x - x0) * dy / float((dx))))
+                self.point(x, y, char)
+        elif y0 < y1:
+            for y in range(y0, y1 + 1):
+                if dy == 0:
+                    x = x0
+                else:
+                    x = x0 + int(round((y - y0) * dx / float((dy))))
+                self.point(x, y, char)
+        else:
+            for y in range(y1, y0 + 1):
+                if dy == 0:
+                    x = x0
+                else:
+                    x = x1 + int(round((y - y1) * dx / float((dy))))
+                self.point(x, y, char)
+
+    def text(self, x, y, text):
+        for i, c in enumerate(text):
+            self.point(x + i, y, c)
+
+    def box(self, x0, y0, w, h):
+        assert w > 1
+        assert h > 1
+
+        w -= 1
+        h -= 1
+
+        for x in range(x0, x0 + w):
+            self.point(x, y0, '-')
+            self.point(x, y0 + h, '-')
+
+        for y in range(y0, y0 + h):
+            self.point(x0, y, '|')
+            self.point(x0 + w, y, '|')
+
+        self.point(x0, y0, '+')
+        self.point(x0 + w, y0, '+')
+        self.point(x0, y0 + h, '+')
+        self.point(x0 + w, y0 + h, '+')
+
+
+class Dagascii(object):
+    def __init__(self, vertexes, edges):
+        #
+        # Just a reminder about naming conventions:
+        # +------------X
+        # |
+        # |
+        # |
+        # |
+        # Y
+        #
+
+        V = {v: Vertex(" {} ".format(v)) for v in vertexes}
+        # NOTE: reverting edges to correctly orientate the graph
+        E = [Edge(V[e], V[s]) for s, e in edges]
+        V = V.values()
+        g = Graph(V, E)
+
+        class VertexViewer(object):
+            h = 3  # top and bottom box edges + text
+
+            def __init__(self, name):
+                self.w = len(name) + 2  # right and left bottom edges + text
+
+        for v in V:
+            v.view = VertexViewer(v.data)
+
+        # NOTE: determine min box length to create the best layout
+        minw = min([v.view.w for v in V])
+
+        for e in E:
+            e.view = EdgeViewer()
+
+        sug = SugiyamaLayout(g.C[0])
+        gr = g.C[0]
+        r = list(filter(lambda x: len(x.e_in()) == 0, gr.sV))
+
+        sug.init_all(roots=r, optimize=True)
+
+        sug.yspace = VertexViewer.h
+        sug.xspace = minw
+        sug.route_edge = route_with_lines
+
+        sug.draw()
+
+        self.sug = sug
+
+    def draw(self):
+        # NOTE: coordinates might me negative, so we need to shift
+        # everything to the positive plane before we actually draw it.
+        Xs = []
+        Ys = []
+
+        for v in self.sug.g.sV:
+            # NOTE: moving boxes w/2 to the left
+            Xs.append(v.view.xy[0] - v.view.w/2.0)
+            Xs.append(v.view.xy[0] + v.view.w/2.0)
+            Ys.append(v.view.xy[1])
+            Ys.append(v.view.xy[1] + v.view.h)
+
+        for e in self.sug.g.sE:
+            for xy in e.view._pts:
+                Xs.append(xy[0])
+                Ys.append(xy[1])
+
+        minx = min(Xs)
+        miny = min(Ys)
+        maxx = max(Xs)
+        maxy = max(Ys)
+
+        canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
+        canvas_lines = int(round(maxy - miny))
+
+        canvas = AsciiCanvas(canvas_cols, canvas_lines)
+
+        # NOTE: first draw edges so that node boxes could overwrite them
+        for e in self.sug.g.sE:
+            assert len(e.view._pts) > 1
+            for index in range(1, len(e.view._pts)):
+                start = e.view._pts[index - 1]
+                end = e.view._pts[index]
+
+                start_x = int(round(start[0] - minx))
+                start_y = int(round(start[1] - miny))
+                end_x = int(round(end[0] - minx))
+                end_y = int(round(end[1] - miny))
+
+                assert start_x >= 0
+                assert start_y >= 0
+                assert end_x >= 0
+                assert end_y >= 0
+
+                canvas.line(start_x, start_y,
+                            end_x, end_y,
+                            '*')
+
+        for v in self.sug.g.sV:
+            # NOTE: moving boxes w/2 to the left
+            x = v.view.xy[0] - v.view.w/2.0
+            y = v.view.xy[1]
+
+            canvas.box(int(round(x - minx)),
+                       int(round(y - miny)),
+                       v.view.w,
+                       v.view.h)
+
+            canvas.text(int(round(x - minx)) + 1,
+                        int(round(y - miny)) + 1,
+                        v.data)
+
+        canvas.draw()

dvc/data_cloud.py

@@ -73,6 +73,11 @@ class DataCloud(object):
             msg = "Can't find cloud section '{}' in config"
             raise ConfigError(msg.format(name))
 
+        # NOTE: check if the class itself has everything needed for operation.
+        # E.g. all the imported packages.
+        if not cloud_type.supported(cloud_type.compat_config(cloud_config)):
+            raise ConfigError("Unsupported cloud '{}'".format(name))
+
         return self._init_cloud(cloud_config, cloud_type)
 
     def _init_cloud(self, cloud_config, cloud_type):
@@ -83,15 +88,18 @@ class DataCloud(object):
         cloud = cloud_type(self.project, cloud_config)
         return cloud
 
-    def _get_cloud(self, remote):
+    def _get_cloud(self, remote, cmd):
         if remote:
             return self._init_remote(remote)
 
         if self._cloud:
             return self._cloud
 
-        msg = "No remote repository specified. Setup default repository " \
-              "with 'dvc config core.remote <name>' or use '-r <name>'."
+        msg = "No remote repository specified. Setup default " \
+              "repository with\n"\
+              "    dvc config core.remote <name>\n" \
+              "or use:\n" \
+              "    dvc {} -r <name>".format(cmd)
         raise ConfigError(msg)
 
     def push(self, targets, jobs=1, remote=None, show_checksums=False):
@@ -100,7 +108,8 @@ class DataCloud(object):
         """
         return self.project.cache.local.push(targets,
                                              jobs=jobs,
-                                             remote=self._get_cloud(remote),
+                                             remote=self._get_cloud(remote,
+                                                                    'push'),
                                              show_checksums=show_checksums)
 
     def pull(self, targets, jobs=1, remote=None, show_checksums=False):
@@ -109,14 +118,16 @@ class DataCloud(object):
         """
         return self.project.cache.local.pull(targets,
                                              jobs=jobs,
-                                             remote=self._get_cloud(remote),
+                                             remote=self._get_cloud(remote,
+                                                                    'pull'),
                                              show_checksums=show_checksums)
 
     def status(self, targets, jobs=1, remote=None, show_checksums=False):
         """
         Check status of data items in a cloud-agnostic way.
         """
+        cloud = self._get_cloud(remote, 'status')
         return self.project.cache.local.status(targets,
                                                jobs=jobs,
-                                               remote=self._get_cloud(remote),
+                                               remote=cloud,
                                                show_checksums=show_checksums)

dvc/dependency/base.py

@@ -5,8 +5,8 @@ from dvc.exceptions import DvcException
 
 class DependencyError(DvcException):
     def __init__(self, path, msg):
-        msg = 'Dependency \'{}\' error: {}'
-        super(DependencyError, self).__init__(msg.format(path, msg))
+        msg = 'Dependency \'{}\' error: {}'.format(path, msg)
+        super(DependencyError, self).__init__(msg)
 
 
 class DependencyDoesNotExistError(DependencyError):
@@ -31,6 +31,9 @@ class DependencyBase(object):
         self.project = stage.project
         self.path = path
 
+    def __str__(self):
+        return self.path
+
     @classmethod
     def match(cls, url):
         return re.match(cls.REGEX, url)
@@ -51,7 +54,7 @@ class DependencyBase(object):
 
     @property
     def exists(self):
-        return self.remote.exists([self.path_info])
+        return self.remote.exists([self.path_info])[0]
 
     def changed(self):
         raise NotImplementedError
@@ -59,7 +62,11 @@ class DependencyBase(object):
     def status(self):
         if self.changed():
             # FIXME better msgs
-            return {self.rel_path: 'changed'}
+            if self.path_info['scheme'] == 'local':
+                p = self.rel_path
+            else:
+                p = self.path
+            return {p: 'changed'}
         return {}
 
     def save(self):

dvc/dependency/local.py

@@ -38,6 +38,15 @@ class DependencyLOCAL(DependencyBase):
         self.path_info = {'scheme': 'local',
                           'path': self.path}
 
+    def __str__(self):
+        return self.rel_path
+
+    @property
+    def is_local(self):
+        assert os.path.isabs(self.path)
+        assert os.path.isabs(self.project.root_dir)
+        return self.path.startswith(self.project.root_dir)
+
     @property
     def sep(self):
         return os.sep
@@ -58,7 +67,8 @@ class DependencyLOCAL(DependencyBase):
         if not self.exists:
             raise self.DoesNotExistError(self.rel_path)
 
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
+        if not os.path.isfile(self.path) \
+           and not os.path.isdir(self.path):  # pragma: no cover
             raise self.IsNotFileOrDirError(self.rel_path)
 
         if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
@@ -68,7 +78,7 @@ class DependencyLOCAL(DependencyBase):
         self.info = self.remote.save_info(self.path_info)
 
     def dumpd(self):
-        if self.path.startswith(self.stage.project.root_dir):
+        if self.is_local:
             path = self.remote.unixpath(os.path.relpath(self.path,
                                                         self.stage.cwd))
         else:

dvc/dependency/ssh.py

@@ -18,6 +18,7 @@ class DependencySSH(DependencyBase):
         self.remote = remote if remote else RemoteSSH(stage.project, {})
 
         host = remote.host if remote else self.group('host')
+        port = remote.port if remote else RemoteSSH.DEFAULT_PORT
         user = remote.user if remote else self.group('user')
         if remote:
             path = posixpath.join(remote.prefix,
@@ -27,6 +28,7 @@ class DependencySSH(DependencyBase):
 
         self.path_info = {'scheme': 'ssh',
                           'host': host,
+                          'port': port,
                           'user': user,
                           'path': path}
 

dvc/exceptions.py

@@ -4,11 +4,11 @@ import traceback
 class DvcException(Exception):
     def __init__(self, msg, cause=None):
         self.cause = cause
-        self.case_tb = None
+        self.cause_tb = None
         if cause:
             try:
                 self.cause_tb = traceback.format_exc()
-            except Exception:
+            except Exception:  # pragma: no cover
                 pass
         super(DvcException, self).__init__(msg)
 
@@ -24,6 +24,24 @@ class OutputDuplicationError(DvcException):
         assert isinstance(output, str)
         assert isinstance(stages, list)
         assert all(isinstance(stage, str) for stage in stages)
-        msg = "File '{}' is specified as an output in more than one stage: {}"
-        super(OutputDuplicationError, self).__init__(msg.format(output,
-                                                                str(stages)))
+        msg = "File '{}' is specified as an output in more than one stage:{}"
+        s = ""
+        for stage in stages:
+            s += "\n    " + stage
+        super(OutputDuplicationError, self).__init__(msg.format(output, s))
+
+
+class CircularDependencyError(DvcException):
+    def __init__(self, dependency):
+        assert isinstance(dependency, str)
+        msg = "File '{}' is specified as an output and as a dependency."
+        super(CircularDependencyError, self).__init__(msg.format(dependency))
+
+
+class MoveNotDataSourceError(DvcException):
+    def __init__(self, path):
+        msg = "Move is not permitted for stages that are not data sources. " \
+              "You need to either move '{path}' to a new location and edit " \
+              "it by hand, or remove '{path}' and create a new one at the " \
+              "desired location."
+        super(MoveNotDataSourceError, self).__init__(msg.format(path=path))

dvc/logger.py

@@ -2,11 +2,28 @@ import sys
 import logging
 import colorama
 import traceback
+import re
 
 
 colorama.init()
 
 
+def visual_width(line):
+    """ Get the the number of columns required to display a string """
+
+    return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, '', line))
+
+
+def visual_center(line, width):
+    """ Center align string according to it's visual width """
+
+    spaces = max(width - visual_width(line), 0)
+    left_padding = int(spaces / 2)
+    right_padding = (spaces - left_padding)
+
+    return (left_padding * ' ') + line + (right_padding * ' ')
+
+
 class Logger(object):
     FMT = '%(message)s'
     DEFAULT_LEVEL = logging.INFO
@@ -19,9 +36,16 @@ class Logger(object):
     }
 
     COLOR_MAP = {
-        'debug': colorama.Fore.BLUE,
-        'warn': colorama.Fore.YELLOW,
-        'error': colorama.Fore.RED
+        'green': colorama.Fore.GREEN,
+        'yellow': colorama.Fore.YELLOW,
+        'blue': colorama.Fore.BLUE,
+        'red': colorama.Fore.RED,
+    }
+
+    LEVEL_COLOR_MAP = {
+        'debug': 'blue',
+        'warn': 'yellow',
+        'error': 'red',
     }
 
     def __init__(self, loglevel=None):
@@ -30,11 +54,22 @@ class Logger(object):
 
     @staticmethod
     def init():
-        sh = logging.StreamHandler(sys.stdout)
-        sh.setFormatter(logging.Formatter(Logger.FMT))
-        sh.setLevel(logging.DEBUG)
 
-        Logger.logger().addHandler(sh)
+        class LogLevelFilter(logging.Filter):
+            def filter(self, record):
+                return record.levelno <= logging.WARNING
+
+        sh_out = logging.StreamHandler(sys.stdout)
+        sh_out.setFormatter(logging.Formatter(Logger.FMT))
+        sh_out.setLevel(logging.DEBUG)
+        sh_out.addFilter(LogLevelFilter())
+
+        sh_err = logging.StreamHandler(sys.stderr)
+        sh_err.setFormatter(logging.Formatter(Logger.FMT))
+        sh_err.setLevel(logging.ERROR)
+
+        Logger.logger().addHandler(sh_out)
+        Logger.logger().addHandler(sh_err)
         Logger.set_level()
 
     @staticmethod
@@ -58,12 +93,12 @@ class Logger(object):
         Logger.logger().setLevel(logging.DEBUG)
 
     @staticmethod
-    def colorize(msg, typ):
+    def colorize(msg, color):
         header = ''
         footer = ''
 
         if sys.stdout.isatty():  # pragma: no cover
-            header = Logger.COLOR_MAP.get(typ.lower(), '')
+            header = Logger.COLOR_MAP.get(color.lower(), '')
             footer = colorama.Style.RESET_ALL
 
         return u'{}{}{}'.format(header, msg, footer)
@@ -83,21 +118,113 @@ class Logger(object):
         return (str_exc, str_tb)
 
     @staticmethod
-    def error(msg, exc=None):
+    def _prefix(msg, typ):
+        color = Logger.LEVEL_COLOR_MAP.get(typ.lower(), '')
+        return Logger.colorize('{}: '.format(msg), color)
+
+    @staticmethod
+    def error_prefix():
+        return Logger._prefix('Error', 'error')
+
+    @staticmethod
+    def warning_prefix():
+        return Logger._prefix('Warning', 'warn')
+
+    @staticmethod
+    def debug_prefix():
+        return Logger._prefix('Debug', 'debug')
+
+    @staticmethod
+    def _with_progress(func, msg):
+        from dvc.progress import progress
+        with progress:
+            func(msg)
+
+    @staticmethod
+    def _error_exc(exc):
+        if exc is None:
+            return
+
+        if Logger.logger().getEffectiveLevel() != logging.DEBUG:
+            return
+
+        prefix = Logger.error_prefix()
         str_exc, str_tb = Logger.parse_exc(exc)
-        if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
-            str_tb = str_tb if str_tb else traceback.format_exc()
-            Logger.logger().error(str_tb)
-        return Logger.logger().error(Logger.colorize(msg + str_exc, 'error'))
+        str_tb = str_tb if str_tb else traceback.format_exc()
+        Logger._with_progress(Logger.logger().error, prefix + str_tb)
 
     @staticmethod
-    def warn(msg):
-        return Logger.logger().warn(Logger.colorize(msg, 'warn'))
+    def _with_exc(func, msg, suffix="", exc=None):
+        Logger._error_exc(exc)
+        msg = msg + Logger.parse_exc(exc)[0] + suffix
+        Logger._with_progress(func, msg)
 
     @staticmethod
-    def debug(msg):
-        return Logger.logger().debug(Logger.colorize(msg, 'debug'))
+    def error(msg, exc=None):
+        chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \
+               "we are always happy to help!"
+        Logger._with_exc(Logger.logger().error,
+                         Logger.error_prefix() + msg,
+                         suffix=chat,
+                         exc=exc)
+
+    @classmethod
+    def warn(cls, msg, exc=None):
+        cls._with_exc(cls.logger().warning,
+                      cls.warning_prefix() + msg,
+                      exc=exc)
+
+    @classmethod
+    def debug(cls, msg, exc=None):
+        cls._with_exc(cls.logger().debug,
+                      cls.debug_prefix() + msg,
+                      exc=exc)
 
     @staticmethod
     def info(msg):
-        return Logger.logger().info(Logger.colorize(msg, 'info'))
+        Logger._with_progress(Logger.logger().info, msg)
+
+    @classmethod
+    def box(cls, msg, border_color=''):
+        lines = msg.split('\n')
+        max_width = max(visual_width(line) for line in lines)
+
+        padding_horizontal = 5
+        padding_vertical = 1
+
+        box_size_horizontal = (max_width + (padding_horizontal * 2))
+
+        chars = {
+            'corner':     '+',
+            'horizontal': '-',
+            'vertical':   '|',
+            'empty':      ' ',
+        }
+
+        margin = "{corner}{line}{corner}\n".format(
+            corner=chars['corner'],
+            line=chars['horizontal'] * box_size_horizontal,
+        )
+
+        padding_lines = [
+            "{border}{space}{border}\n".format(
+                border=cls.colorize(chars['vertical'], border_color),
+                space=chars['empty'] * box_size_horizontal,
+            ) * padding_vertical
+        ]
+
+        content_lines = [
+            "{border}{space}{content}{space}{border}\n".format(
+                border=cls.colorize(chars['vertical'], border_color),
+                space=chars['empty'] * padding_horizontal,
+                content=visual_center(line, max_width),
+            ) for line in lines
+        ]
+
+        box = "{margin}{padding}{content}{padding}{margin}".format(
+                margin=cls.colorize(margin, border_color),
+                padding=''.join(padding_lines),
+                content=''.join(content_lines),
+            )
+
+        print(box)

dvc/output/gs.py

@@ -30,12 +30,12 @@ class OutputGS(DependencyGS):
             return True
 
         if self.use_cache \
-           and self.info != self.project.cache.gs.save_info(self.path_info):
+           and self.project.cache.gs.changed(self.path_info, self.info):
             return True
 
         return False
 
-    def checkout(self):
+    def checkout(self, force=False):
         if not self.use_cache:
             return
 

dvc/output/hdfs.py

@@ -30,12 +30,12 @@ class OutputHDFS(DependencyHDFS):
             return True
 
         if self.use_cache \
-           and self.info != self.project.cache.hdfs.save_info(self.path_info):
+           and self.project.cache.hdfs.changed(self.path_info, self.info):
             return True
 
         return False
 
-    def checkout(self):
+    def checkout(self, force=False):
         if not self.use_cache:
             return
 

dvc/output/local.py

@@ -40,10 +40,6 @@ class OutputLOCAL(DependencyLOCAL):
     def cache(self):
         return self.project.cache.local.get(self.md5)
 
-    @property
-    def is_local(self):
-        return self.path.startswith(self.project.root_dir)
-
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
         ret[self.PARAM_CACHE] = self.use_cache
@@ -62,18 +58,15 @@ class OutputLOCAL(DependencyLOCAL):
         if not self.use_cache:
             return super(OutputLOCAL, self).changed()
 
-        return self.info != self.project.cache.local.save_info(self.path_info)
+        return self.project.cache.local.changed(self.path_info, self.info)
 
-    def checkout(self):
+    def checkout(self, force=False):
         if not self.use_cache:
             return
 
-        if not self.changed():
-            msg = u'Data file \'{}\' didn\'t change, skipping checkout.'
-            self.project.logger.debug(msg.format(self.rel_path))
-            return
-
-        self.project.cache.local.checkout(self.path_info, self.info)
+        self.project.cache.local.checkout(self.path_info,
+                                          self.info,
+                                          force=force)
 
     def _verify_metric(self):
         if not self.metric:
@@ -92,13 +85,14 @@ class OutputLOCAL(DependencyLOCAL):
             super(OutputLOCAL, self).save()
             self._verify_metric()
             msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
-            self.project.logger.debug(msg.format(self.rel_path))
+            self.project.logger.info(msg.format(self.rel_path))
             return
 
         if not os.path.exists(self.path):
             raise self.DoesNotExistError(self.rel_path)
 
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
+        if not os.path.isfile(self.path) \
+           and not os.path.isdir(self.path):  # pragma: no cover
             raise self.IsNotFileOrDirError(self.rel_path)
 
         if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
@@ -108,7 +102,7 @@ class OutputLOCAL(DependencyLOCAL):
 
         if not self.changed():
             msg = 'Output \'{}\' didn\'t change. Skipping saving.'
-            self.project.logger.debug(msg.format(self.rel_path))
+            self.project.logger.info(msg.format(self.rel_path))
             return
 
         if self.is_local:

dvc/output/s3.py

@@ -30,12 +30,12 @@ class OutputS3(DependencyS3):
             return True
 
         if self.use_cache \
-           and self.info != self.project.cache.s3.save_info(self.path_info):
+           and self.project.cache.s3.changed(self.path_info, self.info):
             return True
 
         return False
 
-    def checkout(self):
+    def checkout(self, force=False):
         if not self.use_cache:
             return
 

dvc/output/ssh.py

@@ -30,12 +30,12 @@ class OutputSSH(DependencySSH):
             return True
 
         if self.use_cache \
-           and self.info != self.project.cache.ssh.save_info(self.path_info):
+           and self.project.cache.ssh.changed(self.path_info, self.info):
             return True
 
         return False
 
-    def checkout(self):
+    def checkout(self, force=False):
         if not self.use_cache:
             return
 

dvc/progress.py

@@ -11,11 +11,16 @@ class Progress(object):
         self._n_total = 0
         self._n_finished = 0
         self._lock = threading.Lock()
+        self._line = None
 
     def set_n_total(self, total):
         self._n_total = total
         self._n_finished = 0
 
+    @property
+    def is_finished(self):
+        return self._n_total == self._n_finished
+
     def _clearln(self):
         print('\r\x1b[K', end='')
 
@@ -24,18 +29,23 @@ class Progress(object):
         print(line, end='')
         sys.stdout.flush()
 
-    def update_target(self, name, current, total):
+    def refresh(self, line=None):
         # Just go away if it is locked. Will update next time
         if not self._lock.acquire(False):
             return
 
-        bar = self._bar(name, current, total)
+        if line is None:
+            line = self._line
 
-        if sys.stdout.isatty():
-            self._writeln(bar)
+        if sys.stdout.isatty() and line is not None:
+            self._writeln(line)
+            self._line = line
 
         self._lock.release()
 
+    def update_target(self, name, current, total):
+        self.refresh(self._bar(name, current, total))
+
     def finish_target(self, name):
         # We have to write a msg about finished target
         with self._lock:
@@ -47,6 +57,7 @@ class Progress(object):
             print(bar)
 
             self._n_finished += 1
+            self._line = None
 
     def _bar(self, target_name, current, total):
         """
@@ -63,7 +74,7 @@ class Progress(object):
             progress = int((100 * current)/total) if current < total else 100
             percent = str(progress) + "% "
 
-        if self._n_total:
+        if self._n_total > 1:
             num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
         else:
             num = ""
@@ -74,5 +85,15 @@ class Progress(object):
 
         return num + bar + percent + target_name
 
+    def __enter__(self):
+        self._lock.acquire(True)
+        if self._line is not None:
+            self._clearln()
+
+    def __exit__(self, type, value, tb):
+        if self._line is not None:
+            self.refresh()
+        self._lock.release()
+
 
 progress = Progress()

dvc/prompt.py

@@ -1,5 +1,7 @@
 import sys
 
+from dvc.progress import progress
+
 try:
     # NOTE: in Python3 raw_input() was renamed to input()
     input = raw_input
@@ -7,12 +9,32 @@ except NameError:
     pass
 
 
-def prompt(msg, default=False):
-    if not sys.stdout.isatty():
-        return default
+class Prompt(object):
+    def __init__(self):
+        self.default = None
 
-    answer = input(msg + u'\n').lower()
-    while answer not in ['yes', 'no', 'y', 'n']:
-        answer = input('Enter \'yes\' or \'no\'.\n').lower()
+    def prompt(self, msg, default=False):  # pragma: no cover
+        if self.default is not None:
+            return self.default
 
-    return answer[0] == "y"
+        if not sys.stdout.isatty():
+            return default
+
+        answer = input(msg + u' (y/n)\n').lower()
+        while answer not in ['yes', 'no', 'y', 'n']:
+            answer = input('Enter \'yes\' or \'no\'.\n').lower()
+
+        return answer[0] == "y"
+
+    def prompt_password(self, msg):  # pragma: no cover
+        import getpass
+
+        if not sys.stdout.isatty():
+            return None
+
+        msg = 'Enter password for {}:\n'.format(msg)
+
+        if not progress.is_finished:
+            msg = u'\n' + msg
+
+        return getpass.getpass(msg)

dvc/remote/__init__.py

@@ -14,14 +14,11 @@ REMOTES = [RemoteHDFS, RemoteSSH, RemoteS3, RemoteGS, RemoteAzure, RemoteLOCAL]
 
 def supported_url(url):
     config = {Config.SECTION_REMOTE_URL: url}
-    for r in REMOTES:
-        if r.supported(config):
-            return True
-    return False
+    return any(remote.supported(config) for remote in REMOTES)
 
 
 def Remote(project, config):
-    for r in REMOTES:
-        if r.supported(config):
-            return r(project, config)
+    for remote in REMOTES:
+        if remote.supported(config):
+            return remote(project, config)
     raise UnsupportedRemoteError(str(config))

dvc/remote/azure.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 import os
 import re
+import posixpath
 
 try:
     from azure.storage.blob import BlockBlobService
@@ -92,6 +93,9 @@ class RemoteAzure(RemoteBase):
 #
 #        return {self.PARAM_ETAG: etag}
 #
+#    def changed(self, path_info, checksum_info):
+#        return checksum_info != self.save_info(path_info)
+#
 #    def _copy_blob(self, to_bucket, to_key, from_bucket, from_key):
 #        source = self.blob_service.make_blob_url(from_bucket, from_key)
 #
@@ -118,14 +122,14 @@ class RemoteAzure(RemoteBase):
 #                        from_bucket=self.bucket,
 #                        from_key='{}/{}'.format(etag[0:2], etag[2:]))
 #
-#    def remove(self, path_info):
-#        if path_info['scheme'] != self.scheme:
-#            raise NotImplementedError
-#
-#        Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
-#                                                     path_info['key']))
-#
-#        self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
+    def remove(self, path_info):
+        if path_info['scheme'] != self.scheme:
+            raise NotImplementedError
+
+        Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
+                                                     path_info['key']))
+
+        self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
 
     def md5s_to_path_infos(self, md5s):
         return [{
@@ -134,10 +138,12 @@ class RemoteAzure(RemoteBase):
             'key': '{}/{}'.format(md5[0:2], md5[2:])
         } for md5 in md5s]
 
-    def exists(self, path_infos):
-        keys = {blob.name
+    def _all_keys(self):
+        return {blob.name
                 for blob in self.blob_service.list_blobs(self.bucket)}
 
+    def exists(self, path_infos):
+        keys = self._all_keys()
         ret = []
         for path_info in path_infos:
             if path_info['scheme'] != self.scheme:
@@ -173,8 +179,8 @@ class RemoteAzure(RemoteBase):
                 self.blob_service.create_blob_from_path(
                     bucket, key, from_info['path'], progress_callback=cb)
             except Exception as ex:
-                Logger.error("Failed to upload '{}'".format(from_info['path']),
-                             ex)
+                msg = "Failed to upload '{}'".format(from_info['path'])
+                Logger.warn(msg, ex)
             else:
                 progress.finish_target(name)
 
@@ -210,28 +216,36 @@ class RemoteAzure(RemoteBase):
                 self.blob_service.get_blob_to_path(
                     bucket, key, tmp_file, progress_callback=cb)
             except Exception as exc:
-                Logger.error("Failed to download '{}/{}'".format(
-                    bucket, key), exc)
+                msg = "Failed to download '{}/{}'".format(bucket, key)
+                Logger.warn(msg, exc)
             else:
                 os.rename(tmp_file, to_info['path'])
 
                 if not no_progress_bar:
                     progress.finish_target(name)
 
-# FIXME: temporarily disabled because of the lack of test for external azure
-# dependencies/outputs/cache.
-#
-#    def gc(self, cinfos):
-#        used = [info[self.PARAM_ETAG] for info in cinfos['azure']]
-#        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
-#
-#        all_blobs = self.blob_service.list_blobs(self.bucket)
-#
-#        for blob in all_blobs:
-#            etag = blob.properties.etag
-#            if etag in used:
-#                continue
-#            path_info = {'scheme': self.scheme,
-#                         'key': blob.name,
-#                         'bucket': self.bucket}
-#            self.remove(path_info)
+    def _path_to_etag(self, path):
+        assert len(path.split('/')) == 2
+        return posixpath.dirname(path) + posixpath.basename(path)
+
+    def _all(self):
+        keys = self._all_keys()
+        return [self._path_to_etag(key) for key in keys]
+
+    def gc(self, cinfos):
+        from dvc.remote.local import RemoteLOCAL
+
+        used = [info[self.PARAM_ETAG] for info in cinfos['azure']]
+        used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
+
+        removed = False
+        for etag in self._all():
+            if etag in used:
+                continue
+            path_info = {'scheme': self.scheme,
+                         'key': posixpath.join(etag[0:2], etag[2:]),
+                         'bucket': self.bucket}
+            self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/base.py

@@ -1,6 +1,7 @@
 import os
 import re
 import errno
+from multiprocessing import cpu_count
 
 from dvc.config import Config
 from dvc.logger import Logger
@@ -30,14 +31,18 @@ class DataCloudError(DvcException):
 class RemoteBase(object):
     REGEX = None
     REQUIRES = {}
+    JOBS = 4 * cpu_count()
 
     def __init__(self, project, config):
         pass
 
+    def compat_config(config):
+        return config.copy()
+
     @classmethod
     def supported(cls, config):
         url = config[Config.SECTION_REMOTE_URL]
-        url_ok = cls.match(url)
+        url_ok = cls.match(url) is not None
         deps_ok = all(cls.REQUIRES.values())
         if url_ok and not deps_ok:
             missing = [k for k, v in cls.REQUIRES.items() if v is None]
@@ -75,16 +80,19 @@ class RemoteBase(object):
     @staticmethod
     def tmp_file(fname):
         """ Temporary name for a partial download """
-        # FIXME probably better use uuid()
-        return fname + '.part'
+        import uuid
+        return fname + '.' + str(uuid.uuid4())
 
     def save_info(self, path_info):
         raise NotImplementedError
 
+    def changed(self, path_info, checksum_info):
+        raise NotImplementedError
+
     def save(self, path_info):
         raise NotImplementedError
 
-    def checkout(self, path_info, checksum_info):
+    def checkout(self, path_info, checksum_info, force=False):
         raise NotImplementedError
 
     def download(self, from_infos, to_infos, no_progress_bar=False, name=None):

dvc/remote/gs.py

@@ -33,6 +33,13 @@ class RemoteGS(RemoteBase):
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
 
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = 'gs://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
+
     @property
     def bucket(self):
         return urlparse(self.url).netloc
@@ -59,6 +66,37 @@ class RemoteGS(RemoteBase):
         return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
                                                path_info['key'])}
 
+    @staticmethod
+    def to_string(path_info):
+        return "{}://{}/{}".format(path_info['scheme'],
+                                   path_info['bucket'],
+                                   path_info['key'])
+
+    def changed_cache(self, etag):
+        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
+        cache = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
+
+        if {self.PARAM_ETAG: etag} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+        return False
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        etag = checksum_info.get(self.PARAM_ETAG, None)
+        if etag is None:
+            return True
+
+        if self.changed_cache(etag):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def _copy(self, from_info, to_info, gs=None):
         gs = gs if gs else self.gs
 
@@ -92,6 +130,25 @@ class RemoteGS(RemoteBase):
         if not etag:
             return
 
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(etag):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(etag, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), etag))
+
         key = posixpath.join(self.prefix, etag[0:2], etag[2:])
         from_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
 
@@ -157,9 +214,9 @@ class RemoteGS(RemoteBase):
                 blob.upload_from_filename(from_info['path'])
             except Exception as exc:
                 msg = "Failed to upload '{}' to '{}/{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['bucket'],
-                                        to_info['key']), exc)
+                Logger.warn(msg.format(from_info['path'],
+                                       to_info['bucket'],
+                                       to_info['key']), exc)
                 continue
 
             progress.finish_target(name)
@@ -206,9 +263,9 @@ class RemoteGS(RemoteBase):
                 blob.download_to_filename(tmp_file)
             except Exception as exc:
                 msg = "Failed to download '{}/{}' to '{}'"
-                Logger.error(msg.format(from_info['bucket'],
-                                        from_info['key'],
-                                        to_info['path']), exc)
+                Logger.warn(msg.format(from_info['bucket'],
+                                       from_info['key'],
+                                       to_info['path']), exc)
                 continue
 
             os.rename(tmp_file, to_info['path'])
@@ -229,6 +286,7 @@ class RemoteGS(RemoteBase):
         used = [info[self.PARAM_ETAG] for info in cinfos['gs']]
         used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
 
+        removed = False
         for etag in self._all_etags():
             if etag in used:
                 continue
@@ -237,3 +295,6 @@ class RemoteGS(RemoteBase):
                          'key': posixpath.join(self.prefix,
                                                etag[0:2], etag[2:])}
             self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/hdfs.py

@@ -29,10 +29,16 @@ class RemoteHDFS(RemoteBase):
         cmd = 'hadoop fs -' + cmd
         if user:
             cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
+
+        # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
+        # See https://github.com/iterative/dvc/issues/1197.
+        close_fds = (os.name != 'nt')
+
+        executable = os.getenv('SHELL') if os.name != 'nt' else None
         p = Popen(cmd,
                   shell=True,
-                  close_fds=True,
-                  executable=os.getenv('SHELL'),
+                  close_fds=close_fds,
+                  executable=executable,
                   env=fix_env(os.environ),
                   stdin=PIPE,
                   stdout=PIPE,
@@ -72,6 +78,39 @@ class RemoteHDFS(RemoteBase):
 
         return {self.PARAM_CHECKSUM: self.checksum(path_info)}
 
+    @staticmethod
+    def to_string(path_info):
+        return "{}://{}".format(path_info['scheme'],
+                                path_info['url'])
+
+    def changed_cache(self, checksum):
+        cache = {}
+        cache['scheme'] = 'hdfs'
+        cache['user'] = self.user
+        cache['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
+
+        if {self.PARAM_CHECKSUM: checksum} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
+        if checksum is None:
+            return True
+
+        if self.changed_cache(self, checksum):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def save(self, path_info):
         if path_info['scheme'] != 'hdfs':
             raise NotImplementedError
@@ -96,6 +135,25 @@ class RemoteHDFS(RemoteBase):
         if not checksum:
             return
 
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(checksum):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(checksum, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), checksum))
+
         src = path_info.copy()
         src['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
 
@@ -199,6 +257,7 @@ class RemoteHDFS(RemoteBase):
         used = [info[self.PARAM_CHECKSUM] for info in cinfos['hdfs']]
         used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
 
+        removed = False
         for checksum in self._all_checksums():
             if checksum in used:
                 continue
@@ -207,3 +266,6 @@ class RemoteHDFS(RemoteBase):
                          'url': posixpath.join(self.url,
                                                checksum[0:2], checksum[2:])}
             self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/local.py

@@ -1,4 +1,5 @@
 import os
+import stat
 import uuid
 import json
 import ntpath
@@ -36,7 +37,7 @@ class RemoteLOCAL(RemoteBase):
     def __init__(self, project, config):
         self.project = project
         self.state = self.project.state
-        self.link_state = project.link_state
+        self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
         storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
         self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
 
@@ -51,6 +52,13 @@ class RemoteLOCAL(RemoteBase):
         if self.cache_dir is not None and not os.path.exists(self.cache_dir):
             os.mkdir(self.cache_dir)
 
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, '')
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
+
     @property
     def url(self):
         return self.cache_dir
@@ -82,14 +90,35 @@ class RemoteLOCAL(RemoteBase):
         relpath = os.path.relpath(path, self.cache_dir)
         return os.path.dirname(relpath) + os.path.basename(relpath)
 
-    def changed(self, md5):
+    def changed_cache_file(self, md5):
         cache = self.get(md5)
         if self.state.changed(cache, md5=md5):
             if os.path.exists(cache):
-                msg = 'Corrupted cache file {}'
+                msg = 'Corrupted cache file {}.'
                 Logger.warn(msg.format(os.path.relpath(cache)))
                 remove(cache)
             return True
+        return False
+
+    def changed_cache(self, md5):
+        cache = self.get(md5)
+        clist = [(cache, md5)]
+
+        while True:
+            if len(clist) == 0:
+                break
+
+            cache, md5 = clist.pop()
+            if self.changed_cache_file(md5):
+                return True
+
+            if not self.is_dir_cache(cache):
+                continue
+
+            for entry in self.load_dir_cache(md5):
+                md5 = entry[self.PARAM_MD5]
+                cache = self.get(md5)
+                clist.append((cache, md5))
 
         return False
 
@@ -100,12 +129,36 @@ class RemoteLOCAL(RemoteBase):
         if not os.path.exists(dname):
             os.makedirs(dname)
 
+        # NOTE: just create an empty file for an empty cache
+        if os.path.getsize(cache) == 0:
+            open(path, 'w+').close()
+
+            msg = "Created empty file: {} -> {}".format(
+                os.path.relpath(cache),
+                os.path.relpath(path),
+            )
+
+            Logger.debug(msg)
+            return
+
         i = len(self.cache_types)
         while i > 0:
             try:
                 self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
+
+                if self.protected:
+                    os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
+
+                msg = "Created {}'{}': {} -> {}".format(
+                    'protected ' if self.protected else '',
+                    self.cache_types[0],
+                    os.path.relpath(cache),
+                    os.path.relpath(path)
+                )
+
+                Logger.debug(msg)
                 return
-            except Exception as exc:
+            except DvcException as exc:
                 msg = 'Cache type \'{}\' is not supported: {}'
                 Logger.debug(msg.format(self.cache_types[0], str(exc)))
                 del self.cache_types[0]
@@ -136,12 +189,13 @@ class RemoteLOCAL(RemoteBase):
 
         for root, dirs, files in os.walk(dname):
             bar = False
+
             if len(files) > LARGE_DIR_SIZE:
                 msg = "Computing md5 for a large directory {}. " \
                       "This is only done once."
-                Logger.info(msg.format(os.path.relpath(dname)))
+                Logger.info(msg.format(os.path.relpath(root)))
                 bar = True
-                title = os.path.relpath(dname)
+                title = os.path.relpath(root)
                 processed = 0
                 total = len(files)
                 progress.update_target(title, 0, total)
@@ -154,17 +208,10 @@ class RemoteLOCAL(RemoteBase):
                     progress.update_target(title, processed, total)
                     processed += 1
 
-                # FIXME: we could've used md5 = state.update(path, dump=False)
-                # here, but it is around twice as slow(on ssd, don't know about
-                # hdd) for a directory with small files. What we could do here
-                # is introduce some kind of a limit for file size, after which
-                # we would actually register it in our state file.
-                md5 = self.state.update(path, dump=False)
+                md5 = self.state.update(path)
                 dir_info.append({self.PARAM_RELPATH: relpath,
                                  self.PARAM_MD5: md5})
 
-            self.state.dump()
-
             if bar:
                 progress.finish_target(title)
 
@@ -172,7 +219,7 @@ class RemoteLOCAL(RemoteBase):
         dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
 
         md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
-        if self.changed(md5):
+        if self.changed_cache(md5):
             self.dump_dir_cache(md5, dir_info)
 
         return (md5, dir_info)
@@ -221,9 +268,9 @@ class RemoteLOCAL(RemoteBase):
     def is_dir_cache(cls, cache):
         return cache.endswith(cls.MD5_DIR_SUFFIX)
 
-    def checkout(self, path_info, checksum_info):
+    def checkout(self, path_info, checksum_info, force=False):
         path = path_info['path']
-        md5 = checksum_info.get(self.PARAM_MD5, None)
+        md5 = checksum_info.get(self.PARAM_MD5)
         cache = self.get(md5)
 
         if not cache:
@@ -231,23 +278,27 @@ class RemoteLOCAL(RemoteBase):
             Logger.warn(msg.format(os.path.relpath(path)))
             return
 
-        if self.changed(md5):
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(os.path.relpath(path)))
+            return
+
+        if self.changed_cache(md5):
             msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.'
             Logger.warn(msg.format(md5, os.path.relpath(path)))
             remove(path)
             return
 
-        if os.path.exists(path):
-            msg = u'Data \'{}\' exists. Removing before checkout'
-            Logger.warn(msg.format(os.path.relpath(path)))
-            remove(path)
-
         msg = u'Checking out \'{}\' with cache \'{}\'.'
         Logger.info(msg.format(os.path.relpath(path), md5))
 
         if not self.is_dir_cache(cache):
+
+            if os.path.exists(path):
+                remove(path) if force else self._safe_remove(path)
+
             self.link(cache, path)
-            self.link_state.update(path)
+            self.state.update_link(path)
             return
 
         # Create dir separately so that dir is created
@@ -255,13 +306,72 @@ class RemoteLOCAL(RemoteBase):
         if not os.path.exists(path):
             os.makedirs(path)
 
-        for entry in self.load_dir_cache(md5):
-            md5 = entry[self.PARAM_MD5]
-            c = self.get(md5)
+        dir_info = self.load_dir_cache(md5)
+        dir_relpath = os.path.relpath(path)
+        dir_size = len(dir_info)
+        bar = dir_size > LARGE_DIR_SIZE
+
+        Logger.info("Linking directory '{}'.".format(dir_relpath))
+
+        for processed, entry in enumerate(dir_info):
             relpath = entry[self.PARAM_RELPATH]
+            m = entry[self.PARAM_MD5]
             p = os.path.join(path, relpath)
-            self.link(c, p)
-        self.link_state.update(path)
+            c = self.get(m)
+
+            entry_info = {
+                'scheme': path_info['scheme'],
+                self.PARAM_PATH: p,
+            }
+
+            entry_checksum_info = {
+                self.PARAM_MD5: m,
+            }
+
+            if self.changed(entry_info, entry_checksum_info):
+                remove(p) if force else self._safe_remove(p)
+                self.link(c, p)
+
+            if bar:
+                progress.update_target(dir_relpath, processed, dir_size)
+
+        self._discard_working_directory_changes(path, dir_info, force=force)
+
+        self.state.update_link(path)
+
+        if bar:
+            progress.finish_target(dir_relpath)
+
+    def _discard_working_directory_changes(self, path, dir_info, force=False):
+        working_dir_files = set(
+            os.path.join(root, file)
+            for root, _, files in os.walk(path)
+            for file in files
+        )
+
+        cached_files = set(
+            os.path.join(path, file['relpath'])
+            for file in dir_info
+        )
+
+        delta = working_dir_files - cached_files
+
+        for file in delta:
+            remove(file) if force else self._safe_remove(file)
+
+    def _safe_remove(self, file):
+        msg = (
+            'File "{}" is going to be removed. '
+            'Are you sure you want to proceed?'
+            .format(file)
+        )
+
+        confirmed = self.project.prompt.prompt(msg, False)
+
+        if not confirmed:
+            raise DvcException('Unable to remove {} without a confirmation'
+                               " from the user. Use '-f' to force."
+                               .format(file))
 
     def _move(self, inp, outp):
         # moving in two stages to make last the move atomic in
@@ -277,34 +387,45 @@ class RemoteLOCAL(RemoteBase):
 
         cache = self.get(md5)
 
-        if self.changed(md5):
+        if self.changed_cache(md5):
             self._move(path, cache)
         else:
             remove(path)
 
         self.link(cache, path)
-        self.link_state.update(path)
+        self.state.update_link(path)
 
         return {self.PARAM_MD5: md5}
 
     def _save_dir(self, path_info):
         path = path_info['path']
         md5, dir_info = self.state.update_info(path)
+        dir_relpath = os.path.relpath(path)
+        dir_size = len(dir_info)
+        bar = dir_size > LARGE_DIR_SIZE
 
-        for entry in dir_info:
+        Logger.info("Linking directory '{}'.".format(dir_relpath))
+
+        for processed, entry in enumerate(dir_info):
             relpath = entry[self.PARAM_RELPATH]
             m = entry[self.PARAM_MD5]
             p = os.path.join(path, relpath)
             c = self.get(m)
 
-            if self.changed(m):
+            if self.changed_cache(m):
                 self._move(p, c)
             else:
                 remove(p)
 
             self.link(c, p)
 
-        self.link_state.update(path)
+            if bar:
+                progress.update_target(dir_relpath, processed, dir_size)
+
+        self.state.update_link(path)
+
+        if bar:
+            progress.finish_target(dir_relpath)
 
         return {self.PARAM_MD5: md5}
 
@@ -329,6 +450,19 @@ class RemoteLOCAL(RemoteBase):
 
         return {self.PARAM_MD5: self.state.update(path_info['path'])}
 
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        md5 = checksum_info.get(self.PARAM_MD5, None)
+        if md5 is None:
+            return True
+
+        if self.changed_cache(md5):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def remove(self, path_info):
         if path_info['scheme'] != 'local':
             raise NotImplementedError
@@ -374,8 +508,8 @@ class RemoteLOCAL(RemoteBase):
                 copyfile(from_info['path'], to_info['path'], name=name)
             except Exception as exc:
                 msg = "Failed to upload '{}' tp '{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['path']), exc)
+                Logger.warn(msg.format(from_info['path'],
+                                       to_info['path']), exc)
 
     def download(self,
                  from_infos,
@@ -406,36 +540,12 @@ class RemoteLOCAL(RemoteBase):
                          name=name)
             except Exception as exc:
                 msg = "Failed to download '{}' to '{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['path']), exc)
+                Logger.warn(msg.format(from_info['path'],
+                                       to_info['path']), exc)
                 continue
 
             os.rename(tmp_file, to_info['path'])
 
-    def _collect(self, checksum_infos, push=False):
-        missing = []
-        collected = []
-        for info in checksum_infos:
-            md5 = info[self.PARAM_MD5]
-            cache = self.get(md5)
-
-            if not self.is_dir_cache(info[self.PARAM_MD5]):
-                continue
-
-            if not os.path.exists(cache):
-                missing.append(info)
-                continue
-
-            for i in self.load_dir_cache(md5):
-                if info.get('branch'):
-                    i['branch'] = info['branch']
-                i[self.PARAM_PATH] = posixpath.join(info[self.PARAM_PATH],
-                                                    i[self.PARAM_RELPATH])
-                collected.append(i.copy())
-
-        collected.extend(checksum_infos)
-        return collected, missing
-
     def _group(self, checksum_infos, show_checksums=False):
         by_md5 = {}
 
@@ -461,24 +571,25 @@ class RemoteLOCAL(RemoteBase):
         return list(by_md5.keys()), list(by_md5.values())
 
     def gc(self, checksum_infos):
-        checksum_infos = self._collect(checksum_infos['local'])[0]
+        checksum_infos = checksum_infos['local']
         used_md5s = [info[self.PARAM_MD5] for info in checksum_infos]
 
+        removed = False
         for md5 in self.all():
             if md5 in used_md5s:
                 continue
             remove(self.get(md5))
+            removed = True
 
-    def status(self, checksum_infos, remote, jobs=1, show_checksums=False):
+        return removed
+
+    def status(self, checksum_infos, remote, jobs=None, show_checksums=False):
         Logger.info("Preparing to pull data from {}".format(remote.url))
         title = "Collecting information"
 
-        progress.set_n_total(0)
+        progress.set_n_total(1)
         progress.update_target(title, 0, 100)
 
-        checksum_infos, missing = self._collect(checksum_infos)
-        checksum_infos += missing
-
         progress.update_target(title, 10, 100)
 
         md5s, names = self._group(checksum_infos,
@@ -494,7 +605,7 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 90, 100)
 
-        local_exists = [not self.changed(md5) for md5 in md5s]
+        local_exists = [not self.changed_cache_file(md5) for md5 in md5s]
 
         progress.finish_target(title)
 
@@ -502,14 +613,14 @@ class RemoteLOCAL(RemoteBase):
                                                                local_exists,
                                                                remote_exists)]
 
-    def _do_pull(self,
-                 checksum_infos,
-                 remote,
-                 jobs=1,
-                 show_checksums=False):
+    def pull(self,
+             checksum_infos,
+             remote,
+             jobs=None,
+             show_checksums=False):
         title = "Collecting information"
 
-        progress.set_n_total(0)
+        progress.set_n_total(1)
         progress.update_target(title, 0, 100)
 
         grouped = zip(*self._group(checksum_infos,
@@ -521,7 +632,7 @@ class RemoteLOCAL(RemoteBase):
         names = []
         # NOTE: filter files that are not corrupted
         for md5, name in grouped:
-            if self.changed(md5):
+            if self.changed_cache_file(md5):
                 md5s.append(md5)
                 names.append(name)
 
@@ -535,8 +646,17 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 60, 100)
 
+        # NOTE: dummy call to try to establish a connection
+        # to see if we need to ask user for a password.
+        remote.exists(remote.md5s_to_path_infos(['000']))
+
+        progress.update_target(title, 70, 100)
+
         assert len(path_infos) == len(cache) == len(md5s) == len(names)
 
+        if jobs is None:
+            jobs = remote.JOBS
+
         chunks = list(zip(to_chunks(path_infos, jobs),
                           to_chunks(cache, jobs),
                           to_chunks(names, jobs)))
@@ -560,37 +680,16 @@ class RemoteLOCAL(RemoteBase):
         for f in futures:
             f.result()
 
-    def pull(self, checksum_infos, remote, jobs=1, show_checksums=False):
-        Logger.info("Preparing to pull data from {}".format(remote.url))
-
-        # NOTE: try fetching missing dir info
-        checksum_infos, missing = self._collect(checksum_infos)
-        if len(missing) > 0:
-            self._do_pull(missing,
-                          remote,
-                          jobs,
-                          show_checksums=show_checksums)
-            checksum_infos += self._collect(missing)[0]
-
-        self._do_pull(checksum_infos,
-                      remote,
-                      jobs,
-                      show_checksums=show_checksums)
-
-    def push(self, checksum_infos, remote, jobs=1, show_checksums=False):
+    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
         Logger.info("Preparing to push data to {}".format(remote.url))
         title = "Collecting information"
 
-        progress.set_n_total(0)
+        progress.set_n_total(1)
         progress.update_target(title, 0, 100)
 
-        checksum_infos = self._collect(checksum_infos)[0]
-
-        progress.update_target(title, 10, 100)
-
         # NOTE: verifying that our cache is not corrupted
         def func(info):
-            return not self.changed(info[self.PARAM_MD5])
+            return not self.changed_cache_file(info[self.PARAM_MD5])
         checksum_infos = list(filter(func, checksum_infos))
 
         progress.update_target(title, 20, 100)
@@ -622,6 +721,9 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 90, 100)
 
+        if jobs is None:
+            jobs = remote.JOBS
+
         chunks = list(zip(to_chunks(path_infos, jobs),
                           to_chunks(cache, jobs),
                           to_chunks(names, jobs)))

dvc/remote/s3.py

@@ -40,26 +40,36 @@ class RemoteS3(RemoteBase):
     PARAM_ETAG = 'etag'
 
     def __init__(self, project, config):
-        import configobj
-
         self.project = project
-        storagepath = 's3://' + config.get(Config.SECTION_AWS_STORAGEPATH,
-                                           '').lstrip('/')
+
+        storagepath = 's3://{}'.format(
+            config.get(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        )
+
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
-        self.region = config.get(Config.SECTION_AWS_REGION, None)
-        self.profile = config.get(Config.SECTION_AWS_PROFILE, 'default')
-        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL, None)
 
-        credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
-        if credentialpath:
-            creds_conf = configobj.ConfigObj(credentialpath)
-            creds = creds_conf.get(self.profile, {})
-        else:
-            creds = {}
+        self.region = (
+            os.environ.get('AWS_DEFAULT_REGION') or
+            config.get(Config.SECTION_AWS_REGION)
+        )
 
-        self.region = creds.get('region', self.region)
-        self.aws_access_key_id = creds.get('aws_access_key_id', None)
-        self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
+        self.profile = (
+            os.environ.get('AWS_PROFILE') or
+            config.get(Config.SECTION_AWS_PROFILE)
+        )
+
+        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
+
+        shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
+        if shared_creds:
+            os.environ.setdefault('AWS_SHARED_CREDENTIALS_FILE', shared_creds)
+
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = 's3://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
 
     @property
     def bucket(self):
@@ -71,12 +81,10 @@ class RemoteS3(RemoteBase):
 
     @property
     def s3(self):
-        session = boto3.session.Session(profile_name=self.profile)
-        return session.client('s3',
-                              aws_access_key_id=self.aws_access_key_id,
-                              aws_secret_access_key=self.aws_secret_access_key,
-                              region_name=self.region,
-                              endpoint_url=self.endpoint_url)
+        session = boto3.session.Session(profile_name=self.profile,
+                                        region_name=self.region)
+
+        return session.client('s3', endpoint_url=self.endpoint_url)
 
     def get_etag(self, bucket, key):
         try:
@@ -93,6 +101,19 @@ class RemoteS3(RemoteBase):
         return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
                                                path_info['key'])}
 
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        etag = checksum_info.get(self.PARAM_ETAG, None)
+        if etag is None:
+            return True
+
+        if self.changed_cache(etag):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def _copy(self, from_info, to_info, s3=None):
         s3 = s3 if s3 else self.s3
 
@@ -112,6 +133,23 @@ class RemoteS3(RemoteBase):
 
         return {self.PARAM_ETAG: etag}
 
+    @staticmethod
+    def to_string(path_info):
+        return "s3://{}/{}".format(path_info['bucket'], path_info['key'])
+
+    def changed_cache(self, etag):
+        key = posixpath.join(self.prefix, etag[0:2], etag[2:])
+        cache = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
+
+        if {self.PARAM_ETAG: etag} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
     def checkout(self, path_info, checksum_info):
         if path_info['scheme'] != 's3':
             raise NotImplementedError
@@ -120,6 +158,25 @@ class RemoteS3(RemoteBase):
         if not etag:
             return
 
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(etag):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(etag, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), etag))
+
         key = posixpath.join(self.prefix, etag[0:2], etag[2:])
         from_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
 
@@ -210,7 +267,7 @@ class RemoteS3(RemoteBase):
                                Callback=cb)
             except Exception as exc:
                 msg = "Failed to upload '{}'".format(from_info['path'])
-                Logger.error(msg, exc)
+                Logger.warn(msg, exc)
                 continue
 
             progress.finish_target(name)
@@ -244,16 +301,17 @@ class RemoteS3(RemoteBase):
             if not name:
                 name = os.path.basename(to_info['path'])
 
-            if no_progress_bar:
-                cb = None
-            else:
-                total = s3.head_object(Bucket=from_info['bucket'],
-                                       Key=from_info['key'])['ContentLength']
-                cb = Callback(name, total)
-
             self._makedirs(to_info['path'])
 
             try:
+                if no_progress_bar:
+                    cb = None
+                else:
+                    total = s3.head_object(
+                                    Bucket=from_info['bucket'],
+                                    Key=from_info['key'])['ContentLength']
+                    cb = Callback(name, total)
+
                 s3.download_file(from_info['bucket'],
                                  from_info['key'],
                                  tmp_file,
@@ -261,8 +319,8 @@ class RemoteS3(RemoteBase):
             except Exception as exc:
                 msg = "Failed to download '{}/{}'".format(from_info['bucket'],
                                                           from_info['key'])
-                Logger.error(msg, exc)
-                return
+                Logger.warn(msg, exc)
+                continue
 
             os.rename(tmp_file, to_info['path'])
 
@@ -281,6 +339,7 @@ class RemoteS3(RemoteBase):
         used_etags = [info[self.PARAM_ETAG] for info in cinfos['s3']]
         used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
 
+        removed = False
         for etag in self._all():
             if etag in used_etags:
                 continue
@@ -289,3 +348,6 @@ class RemoteS3(RemoteBase):
                                                etag[0:2], etag[2:]),
                          'bucket': self.bucket}
             self.remove(path_info)
+            removed = True
+
+        return removed

dvc/remote/ssh.py

@@ -44,8 +44,14 @@ class RemoteSSH(RemoteBase):
     REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'
 
     REQUIRES = {'paramiko': paramiko}
+
+    JOBS = 4
+
     PARAM_MD5 = 'md5'
 
+    DEFAULT_PORT = 22
+    TIMEOUT = 1800
+
     def __init__(self, project, config):
         self.project = project
         self.url = config.get(Config.SECTION_REMOTE_URL, '/')
@@ -55,30 +61,49 @@ class RemoteSSH(RemoteBase):
             self.user = config.get(Config.SECTION_REMOTE_USER,
                                    getpass.getuser())
         self.prefix = self.group('path')
+        self.port = config.get(Config.SECTION_REMOTE_PORT, self.DEFAULT_PORT)
+        self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
+        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
+        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
+        self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD,
+                                       False)
 
     def md5s_to_path_infos(self, md5s):
         return [{'scheme': 'ssh',
                  'host': self.host,
                  'user': self.user,
+                 'port': self.port,
                  'path': posixpath.join(self.prefix,
                                         md5[0:2], md5[2:])} for md5 in md5s]
 
-    def ssh(self, host=None, user=None):
-        msg = "Establishing ssh connection with '{}' as user '{}'"
-        Logger.debug(msg.format(host, user))
+    def ssh(self, host=None, user=None, port=None):
+        msg = "Establishing ssh connection with '{}' " \
+              "through port '{}' as user '{}'"
+        Logger.debug(msg.format(host, port, user))
 
         ssh = paramiko.SSHClient()
 
         ssh.load_system_host_keys()
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-        ssh.connect(host, username=user)
+        if self.ask_password and self.password is None:
+            msg = "host '{}' port '{}' user '{}'".format(host, port, user)
+            self.password = self.project.prompt.prompt_password(msg)
+
+        ssh.connect(host,
+                    username=user,
+                    port=port,
+                    key_filename=self.keyfile,
+                    timeout=self.timeout,
+                    password=self.password)
 
         return ssh
 
     def exists(self, path_infos):
         ret = []
-        ssh = self.ssh(host=self.host, user=self.user)
+        ssh = self.ssh(host=self.host,
+                       user=self.user,
+                       port=self.port)
         cmd = 'find {} -type f -follow -print'.format(self.prefix)
         stdout = self._exec(ssh, cmd)
         plist = stdout.split()
@@ -94,16 +119,55 @@ class RemoteSSH(RemoteBase):
 
     def _exec(self, ssh, cmd):
         stdin, stdout, stderr = ssh.exec_command(cmd)
+        channel = stdout.channel
+
+        stdin.close()
+        channel.shutdown_write()
+
+        stdout_chunks = []
+        stderr_chunks = []
+        while (not channel.closed
+               or channel.recv_ready()
+               or channel.recv_stderr_ready()):
+            import select
+            got_chunk = False
+            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
+            for c in readq:
+                if c.recv_ready():
+                    stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
+                    got_chunk = True
+
+                if c.recv_stderr_ready():
+                    stderr_len = len(c.in_stderr_buffer)
+                    s = stderr.channel.recv_stderr(stderr_len)
+                    stderr_chunks.append(s)
+                    got_chunk = True
+
+            if not got_chunk \
+               and stdout.channel.exit_status_ready() \
+               and not stderr.channel.recv_stderr_ready() \
+               and not stdout.channel.recv_ready():
+                stdout.channel.shutdown_read()
+                stdout.channel.close()
+                break
+
+        stdout.close()
+        stderr.close()
+
         if stdout.channel.recv_exit_status() != 0:
-            DvcException('SSH command \'{}\' failed: {}'.format(cmd,
-                                                                stderr.read()))
-        return stdout.read().decode('utf-8')
+            err = b''.join(stderr_chunks).decode('utf-8')
+            msg = 'SSH command \'{}\' failed: {}'.format(cmd, err)
+            raise DvcException(msg)
+
+        return b''.join(stdout_chunks).decode('utf-8')
 
     def md5(self, path_info):
         if path_info['scheme'] != 'ssh':
             raise NotImplementedError
 
-        ssh = self.ssh(host=path_info['host'], user=path_info['user'])
+        ssh = self.ssh(host=path_info['host'],
+                       user=path_info['user'],
+                       port=path_info['port'])
 
         # Use different md5 commands depending on os
         stdout = self._exec(ssh, 'uname').strip()
@@ -133,7 +197,8 @@ class RemoteSSH(RemoteBase):
         assert from_info['user'] == to_info['user']
 
         s = ssh if ssh else self.ssh(host=from_info['host'],
-                                     user=from_info['user'])
+                                     user=from_info['user'],
+                                     port=from_info['port'])
 
         dname = posixpath.dirname(to_info['path'])
         self._exec(s, 'mkdir -p {}'.format(dname))
@@ -148,6 +213,43 @@ class RemoteSSH(RemoteBase):
 
         return {self.PARAM_MD5: self.md5(path_info)}
 
+    @staticmethod
+    def to_string(path_info):
+        return "{}://{}@{}:{}".format(path_info['scheme'],
+                                      path_info['user'],
+                                      path_info['host'],
+                                      path_info['path'])
+
+    def changed_cache(self, md5):
+        cache = {}
+        cache['scheme'] = 'ssh'
+        cache['host'] = self.host
+        cache['port'] = self.port
+        cache['user'] = self.user
+        cache['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
+
+        if {self.PARAM_MD5: md5} != self.save_info(cache):
+            if self.exists([cache])[0]:
+                msg = 'Corrupted cache file {}'
+                Logger.warn(msg.format(self.to_string(cache)))
+                self.remove(cache)
+            return True
+
+        return False
+
+    def changed(self, path_info, checksum_info):
+        if not self.exists([path_info])[0]:
+            return True
+
+        md5 = checksum_info.get(self.PARAM_MD5, None)
+        if md5 is None:
+            return True
+
+        if self.changed_code(md5):
+            return True
+
+        return checksum_info != self.save_info(path_info)
+
     def save(self, path_info):
         if path_info['scheme'] != 'ssh':
             raise NotImplementedError
@@ -168,6 +270,25 @@ class RemoteSSH(RemoteBase):
         if not md5:
             return
 
+        if not self.changed(path_info, checksum_info):
+            msg = "Data '{}' didn't change."
+            Logger.info(msg.format(self.to_string(path_info)))
+            return
+
+        if self.changed_cache(md5):
+            msg = "Cache '{}' not found. File '{}' won't be created."
+            Logger.warn(msg.format(md5, self.to_string(path_info)))
+            return
+
+        if self.exists([path_info])[0]:
+            msg = "Data '{}' exists. Removing before checkout."
+            Logger.warn(msg.format(self.to_string(path_info)))
+            self.remove(path_info)
+            return
+
+        msg = "Checking out '{}' with cache '{}'."
+        Logger.info(msg.format(self.to_string(path_info), md5))
+
         src = path_info.copy()
         src['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
 
@@ -181,7 +302,9 @@ class RemoteSSH(RemoteBase):
                                                       path_info['host'],
                                                       path_info['path']))
 
-        ssh = self.ssh(host=path_info['host'], user=path_info['user'])
+        ssh = self.ssh(host=path_info['host'],
+                       user=path_info['user'],
+                       port=path_info['port'])
         ssh.open_sftp().remove(path_info['path'])
         ssh.close()
 
@@ -192,7 +315,9 @@ class RemoteSSH(RemoteBase):
                  names=None):
         names = self._verify_path_args(from_infos, to_infos, names)
 
-        ssh = self.ssh(host=from_infos[0]['host'], user=from_infos[0]['user'])
+        ssh = self.ssh(host=from_infos[0]['host'],
+                       user=from_infos[0]['user'],
+                       port=from_infos[0]['port'])
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
             if from_info['scheme'] != 'ssh':
@@ -200,6 +325,7 @@ class RemoteSSH(RemoteBase):
 
             if to_info['scheme'] == 'ssh':
                 assert from_info['host'] == to_info['host']
+                assert from_info['port'] == to_info['port']
                 assert from_info['user'] == to_info['user']
                 self.cp(from_info, to_info, ssh=ssh)
                 continue
@@ -223,9 +349,9 @@ class RemoteSSH(RemoteBase):
                                     callback=create_cb(name))
             except Exception as exc:
                 msg = "Failed to download '{}/{}' to '{}'"
-                Logger.error(msg.format(from_info['host'],
-                                        from_info['path'],
-                                        to_info['path']), exc)
+                Logger.warn(msg.format(from_info['host'],
+                                       from_info['path'],
+                                       to_info['path']), exc)
                 continue
 
             os.rename(tmp_file, to_info['path'])
@@ -236,7 +362,9 @@ class RemoteSSH(RemoteBase):
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 
-        ssh = self.ssh(host=to_infos[0]['host'], user=to_infos[0]['user'])
+        ssh = self.ssh(host=to_infos[0]['host'],
+                       user=to_infos[0]['user'],
+                       port=to_infos[0]['port'])
         sftp = ssh.open_sftp()
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
@@ -262,9 +390,9 @@ class RemoteSSH(RemoteBase):
                          callback=create_cb(name))
             except Exception as exc:
                 msg = "Failed to upload '{}' to '{}/{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['host'],
-                                        to_info['path'], exc))
+                Logger.warn(msg.format(from_info['path'],
+                                       to_info['host'],
+                                       to_info['path'], exc))
                 continue
 
             progress.finish_target(name)
@@ -277,7 +405,9 @@ class RemoteSSH(RemoteBase):
         return posixpath.dirname(relpath) + posixpath.basename(relpath)
 
     def _all_md5s(self):
-        ssh = self.ssh(host=self.host, user=self.user)
+        ssh = self.ssh(host=self.host,
+                       user=self.user,
+                       port=self.port)
         cmd = 'find {} -type f -follow -print'.format(self.prefix)
         stdout = self._exec(ssh, cmd)
         flist = stdout.split()
@@ -289,12 +419,17 @@ class RemoteSSH(RemoteBase):
         used = [info[self.PARAM_MD5] for info in cinfos['ssh']]
         used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
 
+        removed = False
         for md5 in self._all_md5s():
             if md5 in used:
                 continue
             path_info = {'scheme': 'ssh',
                          'user': self.user,
                          'host': self.host,
+                         'port': self.port,
                          'path': posixpath.join(self.prefix,
                                                 md5[0:2], md5[2:])}
             self.remove(path_info)
+            removed = True
+
+        return removed

dvc/scm.py

@@ -14,13 +14,22 @@ class FileNotInRepoError(DvcException):
 
 
 class Base(object):
-    def __init__(self, root_dir=os.curdir):
-        self.root_dir = os.path.abspath(os.path.realpath(root_dir))
+    def __init__(self, root_dir=os.curdir, project=None):
+        self.project = project
+        self.root_dir = root_dir
+
+    @property
+    def dir(self):
+        return None
 
     @staticmethod
     def is_repo(root_dir):
         return True
 
+    @staticmethod
+    def is_submodule(root_dir):
+        return True
+
     def ignore(self, path):
         pass
 
@@ -45,17 +54,40 @@ class Base(object):
     def branch(self, branch):
         pass
 
-    def brancher(self, branches=None, all_branches=False):
-        if not branches and not all_branches:
+    def tag(self, tag):
+        pass
+
+    def brancher(self,
+                 branches=None,
+                 all_branches=False,
+                 tags=None,
+                 all_tags=False):
+        if not branches and not all_branches \
+           and not tags and not all_tags:
             yield ''
             return
 
         saved = self.active_branch()
-        if not branches:
-            branches = self.list_branches() if all_branches else [saved]
-        for branch in branches:
-            self.checkout(branch)
-            yield branch
+        revs = []
+
+        if all_branches:
+            branches = self.list_branches()
+
+        if all_tags:
+            tags = self.list_tags()
+
+        if branches is None:
+            revs.extend([saved])
+        else:
+            revs.extend(branches)
+
+        if tags is not None:
+            revs.extend(tags)
+
+        for rev in revs:
+            self.checkout(rev)
+            yield rev
+
         self.checkout(saved)
 
     def untracked_files(self):
@@ -70,6 +102,9 @@ class Base(object):
     def list_branches(self):
         pass
 
+    def list_tags(self):
+        pass
+
     def install(self):
         pass
 
@@ -78,15 +113,16 @@ class Git(Base):
     GITIGNORE = '.gitignore'
     GIT_DIR = '.git'
 
-    def __init__(self, root_dir=os.curdir):
-        super(Git, self).__init__(root_dir)
+    def __init__(self, root_dir=os.curdir, project=None):
+        super(Git, self).__init__(root_dir, project=project)
 
         import git
         from git.exc import InvalidGitRepositoryError
         try:
             self.repo = git.Repo(root_dir)
         except InvalidGitRepositoryError:
-            raise SCMError('{} is not a git repository'.format(root_dir))
+            msg = '{} is not a git repository'
+            raise SCMError(msg.format(root_dir))
 
         # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
         # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
@@ -96,17 +132,28 @@ class Git(Base):
 
     @staticmethod
     def is_repo(root_dir):
-        git_dir = os.path.join(root_dir, Git.GIT_DIR)
-        return os.path.isdir(git_dir)
+        return os.path.isdir(Git._get_git_dir(root_dir))
+
+    @staticmethod
+    def is_submodule(root_dir):
+        return os.path.isfile(Git._get_git_dir(root_dir))
+
+    @staticmethod
+    def _get_git_dir(root_dir):
+        return os.path.join(root_dir, Git.GIT_DIR)
+
+    @property
+    def dir(self):
+        return self.repo.git_dir
 
     def ignore_file(self):
         return self.GITIGNORE
 
     def _get_gitignore(self, path):
-        entry = os.path.basename(path)
-        gitignore = os.path.join(os.path.abspath(os.path.dirname(path)),
-                                 self.GITIGNORE)
-        gitignore = os.path.abspath(os.path.realpath(gitignore))
+        assert os.path.isabs(path)
+        # NOTE: using '/' prefix to make path unambiguous
+        entry = '/' + os.path.basename(path)
+        gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE)
 
         if not gitignore.startswith(self.root_dir):
             raise FileNotInRepoError(path)
@@ -132,7 +179,11 @@ class Git(Base):
         if len(ignore_list) > 0:
             content = '\n' + content
 
-        open(gitignore, 'a').write(content)
+        with open(gitignore, 'a') as fd:
+            fd.write(content)
+
+        if self.project is not None:
+            self.project._files_to_git_add.append(os.path.relpath(gitignore))
 
     def ignore_remove(self, path):
         entry, gitignore = self._get_gitignore(path)
@@ -148,6 +199,9 @@ class Git(Base):
         with open(gitignore, 'w') as fd:
             fd.writelines(filtered)
 
+        if self.project is not None:
+            self.project._files_to_git_add.append(os.path.relpath(gitignore))
+
     def add(self, paths):
         # NOTE: GitPython is not currently able to handle index version >= 3.
         # See https://github.com/iterative/dvc/issues/610 for more details.
@@ -172,6 +226,9 @@ class Git(Base):
     def branch(self, branch):
         self.repo.git.branch(branch)
 
+    def tag(self, tag):
+        self.repo.git.tag(tag)
+
     def untracked_files(self):
         files = self.repo.untracked_files
         return [os.path.join(self.repo.working_dir, fname) for fname in files]
@@ -185,21 +242,28 @@ class Git(Base):
     def list_branches(self):
         return [h.name for h in self.repo.heads]
 
-    def install(self):
+    def list_tags(self):
+        return [t.name for t in self.repo.tags]
+
+    def _install_hook(self, name, cmd):
         hook = os.path.join(self.root_dir,
                             self.GIT_DIR,
                             'hooks',
-                            'post-checkout')
+                            name)
         if os.path.isfile(hook):
             msg = 'Git hook \'{}\' already exists.'
             raise SCMError(msg.format(os.path.relpath(hook)))
         with open(hook, 'w+') as fd:
-            fd.write('#!/bin/sh\nexec dvc checkout\n')
+            fd.write('#!/bin/sh\nexec dvc {}\n'.format(cmd))
         os.chmod(hook, 0o777)
 
+    def install(self):
+        self._install_hook('post-checkout', 'checkout')
+        self._install_hook('pre-commit', 'status')
 
-def SCM(root_dir=os.curdir, no_scm=False):
-    if Git.is_repo(root_dir):
-        return Git(root_dir)
 
-    return Base(root_dir)
+def SCM(root_dir, no_scm=False, project=None):
+    if Git.is_repo(root_dir) or Git.is_submodule(root_dir):
+        return Git(root_dir, project=project)
+
+    return Base(root_dir, project=project)

dvc/signal_handler.py

@@ -1,17 +0,0 @@
-import signal
-import threading
-
-from dvc.logger import Logger
-
-
-class SignalHandler(object):
-    def __enter__(self):
-        if isinstance(threading.current_thread(), threading._MainThread):
-            self.old_handler = signal.signal(signal.SIGINT, self.handler)
-
-    def handler(self, sig, frame):
-        Logger.debug('Ignoring SIGINT during critical parts of code...')
-
-    def __exit__(self, type, value, traceback):
-        if isinstance(threading.current_thread(), threading._MainThread):
-            signal.signal(signal.SIGINT, self.old_handler)

dvc/stage.py

@@ -1,6 +1,5 @@
 import os
 import yaml
-import itertools
 import posixpath
 import subprocess
 from schema import Schema, SchemaError, Optional, Or, And
@@ -8,7 +7,6 @@ from schema import Schema, SchemaError, Optional, Or, And
 import dvc.dependency as dependency
 import dvc.output as output
 from dvc.exceptions import DvcException
-from dvc.prompt import prompt
 from dvc.logger import Logger
 from dvc.utils import dict_md5, fix_env
 
@@ -20,8 +18,45 @@ class StageCmdFailedError(DvcException):
 
 
 class StageFileFormatError(DvcException):
-    def __init__(self):
-        super(StageFileFormatError, self).__init__('Stage file format error')
+    def __init__(self, fname, e):
+        msg = "Stage file '{}' format error: {}".format(fname, str(e))
+        super(StageFileFormatError, self).__init__(msg)
+
+
+class StageFileDoesNotExistError(DvcException):
+    def __init__(self, fname):
+        msg = "'{}' does not exist.".format(fname)
+        super(StageFileDoesNotExistError, self).__init__(msg)
+
+
+class StageFileIsNotDvcFileError(DvcException):
+    def __init__(self, fname):
+        msg = "'{}' is not a dvc file".format(fname)
+        super(StageFileIsNotDvcFileError, self).__init__(msg)
+
+
+class StageFileBadNameError(DvcException):
+    def __init__(self, msg):
+        super(StageFileBadNameError, self).__init__(msg)
+
+
+class StageBadCwdError(DvcException):
+    def __init__(self, cwd):
+        msg = "Stage cwd '{}' is outside of the current dvc project"
+        super(StageBadCwdError, self).__init__(msg.format(cwd))
+
+
+class MissingDep(DvcException):
+    def __init__(self, deps):
+        assert len(deps) > 0
+
+        if len(deps) > 1:
+            dep = 'dependencies'
+        else:
+            dep = 'dependency'
+
+        msg = u'missing {}: {}'.format(dep, ', '.join(map(str, deps)))
+        super(MissingDep, self).__init__(msg)
 
 
 class MissingDataSource(DvcException):
@@ -81,16 +116,20 @@ class Stage(object):
         return self.cmd is None
 
     @staticmethod
-    def is_stage_file(path):
-        if not os.path.isfile(path):
-            return False
-
+    def is_stage_filename(path):
         if not path.endswith(Stage.STAGE_FILE_SUFFIX) \
            and os.path.basename(path) != Stage.STAGE_FILE:
             return False
 
         return True
 
+    @staticmethod
+    def is_stage_file(path):
+        if not os.path.isfile(path):
+            return False
+
+        return Stage.is_stage_filename(path)
+
     def changed_md5(self):
         md5 = self.dumpd().get(self.PARAM_MD5, None)
 
@@ -101,8 +140,6 @@ class Stage(object):
         if self.md5 and md5 and self.md5 == md5:
             return False
 
-        msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
-        self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
         return True
 
     @property
@@ -115,30 +152,51 @@ class Stage(object):
                len(self.deps) == 1 and \
                len(self.outs) == 1
 
-    def changed(self):
+    def changed(self, print_info=False):
         ret = False
 
+        if print_info:
+            log = self.project.logger.info
+        else:
+            log = self.project.logger.debug
+
         if self.is_callback:
+            msg = "Dvc file '{}' is a 'callback' stage (has a command and " \
+                  "no dependencies) and thus always considered as changed."
+            self.project.logger.warn(msg.format(self.relpath))
             ret = True
 
-        if self.locked:
-            entries = self.outs
-        else:
-            entries = itertools.chain(self.outs, self.deps)
-
-        for entry in entries:
-            if entry.changed():
+        if not self.locked:
+            for dep in self.deps:
+                if not dep.changed():
+                    continue
+                if print_info:
+                    msg = "Dependency '{}' of '{}' changed."
+                    log(msg.format(dep, self.relpath))
                 ret = True
 
+        for out in self.outs:
+            if not out.changed():
+                continue
+            if print_info:
+                msg = "Output '{}' of '{}' changed."
+                log(msg.format(out, self.relpath))
+            ret = True
+
         if self.changed_md5():
+            if print_info:
+                msg = "Dvc file '{}' changed."
+                log(msg.format(self.relpath))
             ret = True
 
         if ret:
-            msg = u'Dvc file \'{}\' changed'.format(self.relpath)
+            msg = "Stage '{}' changed.".format(self.relpath)
+            color = 'yellow'
         else:
-            msg = u'Dvc file \'{}\' didn\'t change'.format(self.relpath)
+            msg = "Stage '{}' didn't change.".format(self.relpath)
+            color = 'green'
 
-        self.project.logger.debug(msg)
+        log(Logger.colorize(msg, color))
 
         return ret
 
@@ -151,7 +209,7 @@ class Stage(object):
         os.unlink(self.path)
 
     def reproduce(self, force=False, dry=False, interactive=False):
-        if not self.changed() and not force:
+        if not self.changed(print_info=True) and not force:
             return None
 
         if (self.cmd or self.is_import) and not self.locked and not dry:
@@ -160,7 +218,8 @@ class Stage(object):
 
         msg = "Going to reproduce '{}'. Are you sure you want to continue?"
         msg = msg.format(self.relpath)
-        if interactive and not prompt(msg):
+        if interactive \
+           and not self.project.prompt.prompt(msg):
             raise DvcException('Reproduction aborted by the user')
 
         self.project.logger.info(u'Reproducing \'{}\''.format(self.relpath))
@@ -173,17 +232,15 @@ class Stage(object):
         return self
 
     @staticmethod
-    def validate(d):
+    def validate(d, fname=None):
         try:
             Schema(Stage.SCHEMA).validate(d)
         except SchemaError as exc:
-            Logger.debug(str(exc))
-            raise StageFileFormatError()
+            raise StageFileFormatError(fname, exc)
 
     @staticmethod
     def loadd(project, d, path):
-        Stage.validate(d)
-
+        Stage.validate(d, fname=os.path.relpath(path))
         path = os.path.abspath(path)
         cwd = os.path.dirname(path)
         cmd = d.get(Stage.PARAM_CMD, None)
@@ -219,10 +276,18 @@ class Stage(object):
         if not fname:
             fname = path.basename(out.path) + cls.STAGE_FILE_SUFFIX
 
-        cwd = path.dirname(out.path) if not cwd or add else cwd
+        if not cwd or (add and out.is_local):
+            cwd = path.dirname(out.path)
 
         return (fname, cwd)
 
+    @staticmethod
+    def _check_inside_project(project, cwd):
+        assert project is not None
+        proj_dir = os.path.realpath(project.root_dir)
+        if not os.path.realpath(cwd).startswith(proj_dir):
+            raise StageBadCwdError(cwd)
+
     @staticmethod
     def loads(project=None,
               cmd=None,
@@ -246,26 +311,59 @@ class Stage(object):
                                         use_cache=False, metric=True)
         stage.deps = dependency.loads_from(stage, deps)
 
+        if fname is not None and os.path.basename(fname) != fname:
+            msg = "Stage file name '{}' should not contain subdirectories. " \
+                  "Use '-c|--cwd' to change location of the stage file."
+            raise StageFileBadNameError(msg.format(fname))
+
         fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add)
 
+        Stage._check_inside_project(project, cwd)
+
         cwd = os.path.abspath(cwd)
         path = os.path.join(cwd, fname)
 
         if os.path.exists(path):
             relpath = os.path.relpath(path)
-            msg = "{} already exists. Do you wish to overwrite it?"
-            if not overwrite and not prompt(msg.format(relpath), False):
-                raise DvcException("{} already exists".format(relpath))
+            msg = "'{}' already exists. " \
+                  "Do you wish to run the command and overwrite it?"
+            if not overwrite \
+               and not project.prompt.prompt(msg.format(relpath), False):
+                raise DvcException("'{}' already exists".format(relpath))
 
         stage.cwd = cwd
         stage.path = path
 
         return stage
 
+    @staticmethod
+    def _check_dvc_filename(fname):
+        if not Stage.is_stage_filename(fname):
+            msg = "Bad stage filename '{}'. Stage files should be named " \
+                  "'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc')."
+            raise StageFileBadNameError(msg.format(os.path.relpath(fname),
+                                                   os.path.basename(fname)))
+
+    @staticmethod
+    def _check_dvc_file(fname):
+        sname = fname + Stage.STAGE_FILE_SUFFIX
+        if Stage.is_stage_file(sname):
+            Logger.info("Do you mean '{}'?".format(sname))
+
     @staticmethod
     def load(project, fname):
+        if not os.path.exists(fname):
+            Stage._check_dvc_file(fname)
+            raise StageFileDoesNotExistError(fname)
+
+        Stage._check_dvc_filename(fname)
+
+        if not Stage.is_stage_file(fname):
+            Stage._check_dvc_file(fname)
+            raise StageFileIsNotDvcFileError(fname)
+
         with open(fname, 'r') as fd:
-            return Stage.loadd(project, yaml.safe_load(fd), fname)
+            return Stage.loadd(project, yaml.safe_load(fd) or dict(), fname)
 
     def dumpd(self):
         deps = [x.dumpd() for x in self.deps]
@@ -292,12 +390,16 @@ class Stage(object):
         if not fname:
             fname = self.path
 
+        self._check_dvc_filename(fname)
+
         msg = "Saving information to '{}'.".format(os.path.relpath(fname))
         Logger.info(msg)
 
         with open(fname, 'w') as fd:
             yaml.safe_dump(self.dumpd(), fd, default_flow_style=False)
 
+        self.project._files_to_git_add.append(os.path.relpath(fname))
+
     def save(self):
         for dep in self.deps:
             dep.save()
@@ -305,6 +407,42 @@ class Stage(object):
         for out in self.outs:
             out.save()
 
+    def _check_missing_deps(self):
+        missing = []
+        for dep in self.deps:
+            if not dep.exists:
+                missing.append(dep)
+
+        if len(missing) > 0:
+            raise MissingDep(missing)
+
+    def _check_if_fish(self, executable):  # pragma: no cover
+        if executable is None \
+           or os.path.basename(os.path.realpath(executable)) != 'fish':
+            return
+
+        msg = "DVC detected that you are using fish as your default " \
+              "shell. Be aware that it might cause problems by overwriting " \
+              "your current environment variables with values defined " \
+              "in '.fishrc', which might affect your command. See " \
+              "https://github.com/iterative/dvc/issues/1307. "
+        self.project.logger.warn(msg)
+
+    def _run(self):
+        self._check_missing_deps()
+        executable = os.getenv('SHELL') if os.name != 'nt' else None
+        self._check_if_fish(executable)
+
+        p = subprocess.Popen(self.cmd,
+                             cwd=self.cwd,
+                             shell=True,
+                             env=fix_env(os.environ),
+                             executable=executable)
+        p.communicate()
+
+        if p.returncode != 0:
+            raise StageCmdFailedError(self)
+
     def run(self, dry=False):
         if self.locked:
             msg = u'Verifying outputs in locked stage \'{}\''
@@ -328,14 +466,7 @@ class Stage(object):
             self.project.logger.info(msg)
 
             if not dry:
-                p = subprocess.Popen(self.cmd,
-                                     cwd=self.cwd,
-                                     shell=True,
-                                     env=fix_env(os.environ),
-                                     executable=os.getenv('SHELL'))
-                p.communicate()
-                if p.returncode != 0:
-                    raise StageCmdFailedError(self)
+                self._run()
 
         if not dry:
             self.save()
@@ -347,9 +478,9 @@ class Stage(object):
         if paths:
             raise MissingDataSource(paths)
 
-    def checkout(self):
+    def checkout(self, force=False):
         for out in self.outs:
-            out.checkout()
+            out.checkout(force=force)
 
     def _status(self, entries, name):
         ret = {}

dvc/state.py

@@ -1,60 +1,78 @@
 import os
-import json
+import time
+import sqlite3
 import nanotime
-import threading
 
-from dvc.lock import Lock
+from dvc.config import Config
 from dvc.system import System
 from dvc.utils import file_md5, remove
 from dvc.exceptions import DvcException
-from dvc.signal_handler import SignalHandler
 from dvc.logger import Logger
 
 
-class StateEntry(object):
-    PARAM_MTIME = 'mtime'
-    PARAM_MD5 = 'md5'
-
-    def __init__(self, md5, mtime):
-        self.mtime = mtime
-        self.md5 = md5
-
-    def update(self, md5, mtime):
-        self.mtime = mtime
-        self.md5 = md5
-
-    @staticmethod
-    def loadd(d):
-        mtime = d[StateEntry.PARAM_MTIME]
-        md5 = d[StateEntry.PARAM_MD5]
-        return StateEntry(md5, mtime)
-
-    def dumpd(self):
-        return {
-            self.PARAM_MD5: self.md5,
-            self.PARAM_MTIME: self.mtime,
-        }
-
-
 class StateDuplicateError(DvcException):
     pass
 
 
 class State(object):
+    VERSION = 2
     STATE_FILE = 'state'
-    STATE_LOCK_FILE = 'state.lock'
+    STATE_TABLE = 'state'
+    STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
+                         "mtime TEXT NOT NULL, " \
+                         "md5 TEXT NOT NULL, " \
+                         "timestamp TEXT NOT NULL"
 
-    def __init__(self, project):
+    STATE_INFO_TABLE = 'state_info'
+    STATE_INFO_TABLE_LAYOUT = 'count INTEGER'
+    STATE_INFO_ROW = 1
+
+    LINK_STATE_TABLE = 'link_state'
+    LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
+                              "inode INTEGER NOT NULL, " \
+                              "mtime TEXT NOT NULL"
+
+    STATE_ROW_LIMIT = 100000000
+    STATE_ROW_CLEANUP_QUOTA = 50
+
+    MAX_INT = 2**63 - 1
+    MAX_UINT = 2**64 - 2
+
+    def __init__(self, project, config):
         self.project = project
         self.dvc_dir = project.dvc_dir
-        self._lock = threading.Lock()
-        if self.dvc_dir:
-            self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
-            self._lock_file = Lock(self.dvc_dir, self.STATE_LOCK_FILE)
-        else:
+        self.root_dir = project.root_dir
+
+        self.row_limit = 100
+        self.row_cleanup_quota = 50
+
+        c = config.get(Config.SECTION_STATE, {})
+        self.row_limit = c.get(Config.SECTION_STATE_ROW_LIMIT,
+                               self.STATE_ROW_LIMIT)
+        self.row_cleanup_quota = c.get(Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
+                                       self.STATE_ROW_CLEANUP_QUOTA)
+
+        if not self.dvc_dir:
             self.state_file = None
-            self._lock_file = threading.Lock()
-        self._db = self.load()
+            return
+
+        self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
+
+        # https://www.sqlite.org/tempfiles.html
+        self.temp_files = [
+            self.state_file + '-journal',
+            self.state_file + '-wal',
+        ]
+
+        self.db = None
+        self.c = None
+        self.inserts = 0
+
+    def __enter__(self):
+        self.load()
+
+    def __exit__(self, type, value, tb):
+        self.dump()
 
     @staticmethod
     def init(project):
@@ -69,142 +87,266 @@ class State(object):
     def changed(self, path, md5):
         actual = self.update(path)
 
+        msg = "File '{}', md5 '{}', actual '{}'"
+        Logger.debug(msg.format(path, md5, actual))
+
         if not md5 or not actual:
             return True
 
         return actual.split('.')[0] != md5.split('.')[0]
 
-    def load(self):
-        if not self.state_file or not os.path.isfile(self.state_file):
-            return {}
+    def _execute(self, cmd):
+        Logger.debug(cmd)
+        return self.c.execute(cmd)
 
-        with open(self.state_file, 'r') as fd:
+    def _fetchall(self):
+        ret = self.c.fetchall()
+        Logger.debug("fetched: {}".format(ret))
+        return ret
+
+    def _to_sqlite(self, n):
+        assert n >= 0
+        assert n < self.MAX_UINT
+        # NOTE: sqlite stores unit as signed ints, so maximum uint is 2^63-1
+        # see http://jakegoulding.com/blog/2011/02/06/sqlite-64-bit-integers/
+        if n > self.MAX_INT:
+            ret = -(n - self.MAX_INT)
+        else:
+            ret = n
+        assert self._from_sqlite(ret) == n
+        return ret
+
+    def _from_sqlite(self, n):
+        assert abs(n) <= self.MAX_INT
+        if n < 0:
+            return abs(n) + self.MAX_INT
+        assert n < self.MAX_UINT
+        assert n >= 0
+        return n
+
+    def _load(self, empty=False):
+        from dvc import VERSION
+
+        if not empty:
+            cmd = "PRAGMA user_version;"
+            self._execute(cmd)
+            ret = self._fetchall()
+            assert len(ret) == 1
+            assert len(ret[0]) == 1
+            assert isinstance(ret[0][0], int)
+            version = ret[0][0]
+
+            if version > self.VERSION:
+                msg = "You are using an old version '{}' of dvc that is " \
+                      "using state file version '{}' which is not " \
+                      "compatible with the state file version '{}' that " \
+                      "is used in this projet. Please upgrade right now!"
+                raise DvcException(msg.format(VERSION,
+                                              self.VERSION,
+                                              version))
+            elif version < self.VERSION:
+                msg = "State file version '{}' is too old. " \
+                      "Reformatting to the current version '{}'."
+                self.project.logger.warn(msg.format(version, self.VERSION))
+                cmd = "DROP TABLE IF EXISTS {};"
+                self._execute(cmd.format(self.STATE_TABLE))
+                self._execute(cmd.format(self.STATE_INFO_TABLE))
+                self._execute(cmd.format(self.LINK_STATE_TABLE))
+
+        # Check that the state file is indeed a database
+        cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
+        self._execute(cmd.format(self.STATE_TABLE,
+                                 self.STATE_TABLE_LAYOUT))
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_TABLE_LAYOUT))
+        self._execute(cmd.format(self.LINK_STATE_TABLE,
+                                 self.LINK_STATE_TABLE_LAYOUT))
+
+        cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
+              "WHERE NOT EXISTS (SELECT * FROM {})"
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_TABLE))
+
+        cmd = "PRAGMA user_version = {};"
+        self._execute(cmd.format(self.VERSION))
+
+    def load(self):
+        retries = 1
+        while True:
+            assert self.db is None
+            assert self.c is None
+            assert self.inserts == 0
+            empty = not os.path.exists(self.state_file)
+            self.db = sqlite3.connect(self.state_file)
+            self.c = self.db.cursor()
+
+            # Try loading once to check that the file is indeed a database
+            # and reformat it if it is not.
             try:
-                return json.load(fd)
-            except ValueError as exc:
-                Logger.error('Failed to load \'{}\''.format(self.state_file),
-                             exc)
-                return {}
+                self._load(empty=empty)
+                return
+            except sqlite3.DatabaseError:
+                self.c.close()
+                self.db.close()
+                self.db = None
+                self.c = None
+                self.inserts = 0
+                if retries > 0:
+                    os.unlink(self.state_file)
+                    retries -= 1
+                else:
+                    raise
+
+    def _vacuum(self):
+        # NOTE: see https://bugs.python.org/issue28518
+        self.db.isolation_level = None
+        self._execute("VACUUM")
+        self.db.isolation_level = ''
 
     def dump(self):
-        if not self.state_file:
-            return
+        assert self.db is not None
 
-        with SignalHandler():
-            with open(self.state_file, 'w+') as fd:
-                json.dump(self._db, fd)
+        cmd = "SELECT count from {} WHERE rowid={}"
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_ROW))
+        ret = self._fetchall()
+        assert len(ret) == 1
+        assert len(ret[0]) == 1
+        count = self._from_sqlite(ret[0][0]) + self.inserts
+
+        if count > self.row_limit:
+            msg = "Cleaning up state. This might take a while."
+            self.project.logger.warn(msg)
+
+            delete = (count - self.row_limit)
+            delete += int(self.row_limit * (self.row_cleanup_quota/100.))
+            cmd = "DELETE FROM {} WHERE timestamp IN (" \
+                  "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
+            self._execute(cmd.format(self.STATE_TABLE,
+                                     self.STATE_TABLE,
+                                     delete))
+
+            self._vacuum()
+
+            cmd = "SELECT COUNT(*) FROM {}"
+
+            self._execute(cmd.format(self.STATE_TABLE))
+            ret = self._fetchall()
+            assert len(ret) == 1
+            assert len(ret[0]) == 1
+            count = ret[0][0]
+
+        cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self._to_sqlite(count),
+                                 self.STATE_INFO_ROW))
+
+        self.db.commit()
+        self.c.close()
+        self.db.close()
+        self.db = None
+        self.c = None
+        self.inserts = 0
 
     @staticmethod
     def mtime(path):
-        return str(int(nanotime.timestamp(os.path.getmtime(path))))
+        mtime = os.path.getmtime(path)
+        if os.path.isdir(path):
+            for root, dirs, files in os.walk(path):
+                for entry in list(dirs) + list(files):
+                    p = os.path.join(root, entry)
+                    m = os.path.getmtime(p)
+                    if m > mtime:
+                        mtime = m
+
+        return str(int(nanotime.timestamp(mtime)))
 
     @staticmethod
     def inode(path):
-        return str(System.inode(path))
+        Logger.debug('Path {} inode {}'.format(path, System.inode(path)))
+        return System.inode(path)
 
-    def _do_update(self, path, dump=True):
+    def _do_update(self, path):
         if not os.path.exists(path):
             return (None, None)
 
         mtime = self.mtime(path)
         inode = self.inode(path)
 
-        md5 = self._get(inode, mtime)
-        if md5:
-            return (md5, None)
+        cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
+                                                       self._to_sqlite(inode))
 
-        md5, info = self._collect(path)
-        state = StateEntry(md5, mtime)
-        d = state.dumpd()
-
-        with self._lock:
-            with self._lock_file:
-                self._db[inode] = d
-
-                if dump:
-                    self.dump()
+        self._execute(cmd)
+        ret = self._fetchall()
+        if len(ret) == 0:
+            md5, info = self._collect(path)
+            cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
+                  'VALUES ({}, "{}", "{}", "{}")'
+            self._execute(cmd.format(self.STATE_TABLE,
+                                     self._to_sqlite(inode),
+                                     mtime,
+                                     md5,
+                                     int(nanotime.timestamp(time.time()))))
+            self.inserts += 1
+        else:
+            assert len(ret) == 1
+            assert len(ret[0]) == 4
+            i, m, md5, timestamp = ret[0]
+            i = self._from_sqlite(i)
+            assert i == inode
+            msg = "Inode '{}', mtime '{}', actual mtime '{}'."
+            Logger.debug(msg.format(i, m, mtime))
+            if mtime != m:
+                md5, info = self._collect(path)
+                cmd = 'UPDATE {} SET ' \
+                      'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
+                      'WHERE inode = {}'
+                self._execute(cmd.format(self.STATE_TABLE,
+                                         mtime,
+                                         md5,
+                                         int(nanotime.timestamp(time.time())),
+                                         self._to_sqlite(inode)))
+            else:
+                info = None
+                cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
+                self._execute(cmd.format(self.STATE_TABLE,
+                                         int(nanotime.timestamp(time.time())),
+                                         self._to_sqlite(inode)))
 
         return (md5, info)
 
-    def update(self, path, dump=True):
-        return self._do_update(path, dump=dump)[0]
+    def update(self, path):
+        return self._do_update(path)[0]
 
-    def update_info(self, path, dump=True):
-        md5, info = self._do_update(path, dump=dump)
+    def update_info(self, path):
+        md5, info = self._do_update(path)
         if not info:
             info = self.project.cache.local.load_dir_cache(md5)
         return (md5, info)
 
-    def _get(self, inode, mtime):
-        with self._lock:
-            with self._lock_file:
-                d = self._db.get(inode, None)
-
-        if not d:
-            return None
-
-        state = StateEntry.loadd(d)
-        if mtime == state.mtime:
-            return state.md5
-
-        return None
-
-    def get(self, path):
-        mtime = self.mtime(path)
-        inode = self.inode(path)
-
-        return self._get(inode, mtime)
-
-
-class LinkStateEntry(object):
-    PARAM_MTIME = 'mtime'
-    PARAM_INODE = 'inode'
-
-    def __init__(self, inode, mtime):
-        self.mtime = mtime
-        self.inode = inode
-
-    @staticmethod
-    def loadd(d):
-        mtime = d[LinkStateEntry.PARAM_MTIME]
-        inode = d[LinkStateEntry.PARAM_INODE]
-        return LinkStateEntry(inode, mtime)
-
-    def dumpd(self):
-        return {
-            self.PARAM_INODE: self.inode,
-            self.PARAM_MTIME: self.mtime,
-        }
-
-
-class LinkState(State):
-    STATE_FILE = 'link.state'
-    STATE_LOCK_FILE = STATE_FILE + '.lock'
-
-    def __init__(self, project):
-        super(LinkState, self).__init__(project)
-        self.root_dir = project.root_dir
-
-    def update(self, path, dump=True):
+    def update_link(self, path):
         if not os.path.exists(path):
             return
 
         mtime = self.mtime(path)
         inode = self.inode(path)
+        relpath = os.path.relpath(path, self.root_dir)
 
-        with self._lock:
-            state = LinkStateEntry(inode, mtime)
-            d = state.dumpd()
-            self._db[os.path.relpath(path, self.root_dir)] = d
-            if dump:
-                with self._lock_file:
-                    self.dump()
+        cmd = 'REPLACE INTO {}(path, inode, mtime) ' \
+              'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE,
+                                               relpath,
+                                               self._to_sqlite(inode),
+                                               mtime)
+        self._execute(cmd)
 
-    def _do_remove_unused(self, used):
-        items = self._db.copy().items()
-        for p, s in items:
+    def remove_unused_links(self, used):
+        unused = []
+
+        self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
+        for row in self.c:
+            p, i, m = row
+            i = self._from_sqlite(i)
             path = os.path.join(self.root_dir, p)
-            state = LinkStateEntry.loadd(s)
 
             if path in used:
                 continue
@@ -215,13 +357,11 @@ class LinkState(State):
             inode = self.inode(path)
             mtime = self.mtime(path)
 
-            if inode == state.inode and mtime == state.mtime:
+            if i == inode and m == mtime:
                 Logger.debug('Removing \'{}\' as unused link.'.format(path))
                 remove(path)
-                del self._db[p]
+                unused.append(p)
 
-    def remove_unused(self, used):
-        with self._lock:
-            with self._lock_file:
-                self._do_remove_unused(used)
-                self.dump()
+        for p in unused:
+            cmd = 'DELETE FROM {} WHERE path = "{}"'
+            self._execute(cmd.format(self.LINK_STATE_TABLE, p))

dvc/system.py

@@ -1,9 +1,4 @@
 import os
-import ctypes
-import reflink
-
-if os.name == 'nt':
-    import ntfsutils.hardlink as winlink
 
 
 class System(object):
@@ -13,15 +8,38 @@ class System(object):
 
     @staticmethod
     def hardlink(source, link_name):
-        if System.is_unix():
-            return os.link(source, link_name)
+        import ctypes
+        from dvc.exceptions import DvcException
 
-        return winlink.create(source, link_name)
+        if System.is_unix():
+            try:
+                os.link(source, link_name)
+                return
+            except Exception as exc:
+                raise DvcException('link', cause=exc)
+
+        CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
+        CreateHardLink.argtypes = [ctypes.c_wchar_p,
+                                   ctypes.c_wchar_p,
+                                   ctypes.c_void_p]
+        CreateHardLink.restype = ctypes.wintypes.BOOL
+
+        res = CreateHardLink(link_name, source, None)
+        if res == 0:
+            raise DvcException('CreateHardLinkW', cause=ctypes.WinError())
 
     @staticmethod
     def symlink(source, link_name):
+        import ctypes
+        from dvc.exceptions import DvcException
+
         if System.is_unix():
-            return os.symlink(source, link_name)
+            try:
+                os.symlink(source, link_name)
+                return
+            except Exception as exc:
+                msg = "Failed to symlink '{}' -> '{}': {}"
+                raise DvcException(msg.format(source, link_name, str(exc)))
 
         flags = 0
         if source is not None and os.path.isdir(source):
@@ -32,45 +50,193 @@ class System(object):
         func.restype = ctypes.c_ubyte
 
         if func(link_name, source, flags) == 0:
-            raise ctypes.WinError()
+            raise DvcException('CreateSymbolicLinkW', cause=ctypes.WinError())
+
+    @staticmethod
+    def _reflink_darwin(src, dst):
+        import ctypes
+
+        clib = ctypes.CDLL('libc.dylib')
+        if not hasattr(clib, 'clonefile'):
+            return -1
+
+        clonefile = clib.clonefile
+        clonefile.argtypes = [ctypes.c_char_p,
+                              ctypes.c_char_p,
+                              ctypes.c_int]
+        clonefile.restype = ctypes.c_int
+
+        return clonefile(ctypes.c_char_p(src.encode('utf-8')),
+                         ctypes.c_char_p(dst.encode('utf-8')),
+                         ctypes.c_int(0))
+
+    @staticmethod
+    def _reflink_windows(src, dst):
+        return -1
+
+    @staticmethod
+    def _reflink_linux(src, dst):
+        import os
+        import fcntl
+
+        FICLONE = 0x40049409
+
+        s = open(src, 'r')
+        d = open(dst, 'w+')
+
+        try:
+            ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
+        except IOError:
+            s.close()
+            d.close()
+            os.unlink(dst)
+            raise
+
+        s.close()
+        d.close()
+
+        if ret != 0:
+            os.unlink(dst)
+
+        return ret
 
     @staticmethod
     def reflink(source, link_name):
-        return reflink.reflink(source, link_name)
+        import platform
+        from dvc.exceptions import DvcException
+
+        system = platform.system()
+        try:
+            if system == 'Windows':
+                ret = System._reflink_windows(source, link_name)
+            elif system == 'Darwin':
+                ret = System._reflink_darwin(source, link_name)
+            elif system == 'Linux':
+                ret = System._reflink_linux(source, link_name)
+            else:
+                ret = -1
+        except IOError:
+            ret = -1
+
+        if ret != 0:
+            raise DvcException('Reflink is not supported')
 
-    # FIXME
-    # Temporary fix while waiting for the PR to be merged and released:
-    # https://github.com/sunshowers/ntfs/pull/11
     @staticmethod
     def getdirinfo(path):
-        from ntfsutils.fs import FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_READ
-        from ntfsutils.fs import OPEN_EXISTING, BY_HANDLE_FILE_INFORMATION
-        from ntfsutils.fs import CreateFile, WinError
-        from ntfsutils.fs import GetFileInformationByHandle, CloseHandle
+        import ctypes
+        from ctypes import c_void_p, c_wchar_p, Structure, WinError, POINTER
+        from ctypes.wintypes import DWORD, HANDLE, BOOL
+
+        FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+        FILE_SHARE_READ = 0x00000001
+        OPEN_EXISTING = 3
+
+        class FILETIME(Structure):
+            _fields_ = [("dwLowDateTime", DWORD),
+                        ("dwHighDateTime", DWORD)]
+
+        class BY_HANDLE_FILE_INFORMATION(Structure):
+            _fields_ = [("dwFileAttributes", DWORD),
+                        ("ftCreationTime", FILETIME),
+                        ("ftLastAccessTime", FILETIME),
+                        ("ftLastWriteTime", FILETIME),
+                        ("dwVolumeSerialNumber", DWORD),
+                        ("nFileSizeHigh", DWORD),
+                        ("nFileSizeLow", DWORD),
+                        ("nNumberOfLinks", DWORD),
+                        ("nFileIndexHigh", DWORD),
+                        ("nFileIndexLow", DWORD)]
+
         flags = FILE_FLAG_BACKUP_SEMANTICS
-        hfile = CreateFile(path,
-                           0,
-                           FILE_SHARE_READ,
-                           None,
-                           OPEN_EXISTING,
-                           flags,
-                           None)
+
+        func = ctypes.windll.kernel32.CreateFileW
+        func.argtypes = [c_wchar_p,
+                         DWORD,
+                         DWORD,
+                         c_void_p,
+                         DWORD,
+                         DWORD,
+                         HANDLE]
+        func.restype = HANDLE
+
+        hfile = func(path,
+                     0,
+                     FILE_SHARE_READ,
+                     None,
+                     OPEN_EXISTING,
+                     flags,
+                     None)
         if hfile is None:
             raise WinError()
+
+        func = ctypes.windll.kernel32.GetFileInformationByHandle
+        func.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
+        func.restype = BOOL
+
         info = BY_HANDLE_FILE_INFORMATION()
-        rv = GetFileInformationByHandle(hfile, info)
-        CloseHandle(hfile)
+        rv = func(hfile, info)
+
+        func = ctypes.windll.kernel32.CloseHandle
+        func.argtypes = [HANDLE]
+        func.restype = BOOL
+
+        func(hfile)
+
         if rv == 0:
             raise WinError()
+
         return info
 
     @staticmethod
     def inode(path):
         if System.is_unix():
-            return os.lstat(path).st_ino
+            import ctypes
+            inode = os.lstat(path).st_ino
+            # NOTE: See https://bugs.python.org/issue29619 and
+            # https://stackoverflow.com/questions/34643289/
+            # pythons-os-stat-is-returning-wrong-inode-value
+            inode = ctypes.c_ulong(inode).value
+        else:
+            # getdirinfo from ntfsutils works on both files and dirs
+            info = System.getdirinfo(path)
+            inode = abs(hash((info.dwVolumeSerialNumber,
+                              info.nFileIndexHigh,
+                              info.nFileIndexLow)))
+        assert inode >= 0
+        assert inode < 2**64
+        return inode
 
-        # getdirinfo from ntfsutils works on both files and dirs
-        info = System.getdirinfo(path)
-        return hash((info.dwVolumeSerialNumber,
-                     info.nFileIndexHigh,
-                     info.nFileIndexLow))
+    @staticmethod
+    def _wait_for_input_windows(timeout):
+        import sys
+        import ctypes
+        import msvcrt
+        from ctypes.wintypes import DWORD, HANDLE
+
+        # https://docs.microsoft.com/en-us/windows/desktop/api/synchapi/nf-synchapi-waitforsingleobject
+        WAIT_OBJECT_0 = 0
+        WAIT_TIMEOUT = 0x00000102
+
+        func = ctypes.windll.kernel32.WaitForSingleObject
+        func.argtypes = [HANDLE, DWORD]
+        func.restype = DWORD
+
+        rc = func(msvcrt.get_osfhandle(sys.stdin.fileno()), timeout * 1000)
+        if rc not in [WAIT_OBJECT_0, WAIT_TIMEOUT]:
+            raise RuntimeError(rc)
+
+    @staticmethod
+    def _wait_for_input_posix(timeout):
+        import sys
+        import select
+        try:
+            select.select([sys.stdin], [], [], timeout)
+        except select.error:
+            pass
+
+    @staticmethod
+    def wait_for_input(timeout):
+        if System.is_unix():
+            return System._wait_for_input_posix(timeout)
+        else:
+            return System._wait_for_input_windows(timeout)

dvc/updater.py

@@ -1,61 +1,189 @@
+import sys
 import os
 import time
 import requests
+import colorama
 
 from dvc import VERSION_BASE
+from dvc.lock import Lock, LockError
 from dvc.logger import Logger
+from dvc.utils import is_binary
 
 
-class Updater(object):
+class Updater(object):  # pragma: no cover
     URL = 'https://4ki8820rsf.execute-api.us-east-2.amazonaws.com/' \
           'prod/latest-version'
     UPDATER_FILE = 'updater'
-    TIMEOUT = 7 * 24 * 60 * 60  # every week
+    TIMEOUT = 24 * 60 * 60  # every day
     TIMEOUT_GET = 10
 
     def __init__(self, dvc_dir):
         self.dvc_dir = dvc_dir
         self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
+        self.lock = Lock(dvc_dir, self.updater_file + '.lock')
+        self.current = VERSION_BASE
 
-    @staticmethod
-    def init(dvc_dir):
-        return Updater(dvc_dir)
+    def _is_outdated_file(self):
+        ctime = os.path.getmtime(self.updater_file)
+        outdated = (time.time() - ctime >= self.TIMEOUT)
+        if outdated:
+            Logger.debug("'{}' is outdated(".format(self.updater_file))
+        return outdated
+
+    def _with_lock(self, func, action):
+        try:
+            with self.lock:
+                func()
+        except LockError:
+            msg = "Failed to acquire '{}' before {} updates"
+            Logger.debug(msg.format(self.lock.lock_file, action))
 
     def check(self):
-        current = VERSION_BASE
-
-        if os.getenv('CI'):
+        if os.getenv('CI') or os.getenv('DVC_TEST'):
             return
 
-        if os.path.isfile(self.updater_file):
-            ctime = os.path.getctime(self.updater_file)
-            if time.time() - ctime < self.TIMEOUT:
-                msg = '{} is not old enough to check for updates'
-                Logger.debug(msg.format(self.UPDATER_FILE))
+        self._with_lock(self._check, 'checking')
+
+    def _check(self):
+        if not os.path.exists(self.updater_file) or self._is_outdated_file():
+            self.fetch()
+            return
+
+        with open(self.updater_file, 'r') as fobj:
+            import json
+
+            try:
+                info = json.load(fobj)
+                self.latest = info['version']
+            except Exception as exc:
+                msg = "'{}' is not a valid json: {}"
+                Logger.debug(msg.format(self.updater_file, exc))
+                self.fetch()
                 return
 
-            os.unlink(self.updater_file)
+        if self._is_outdated():
+            self._notify()
 
-        Logger.info('Checking for updates...')
+    def fetch(self, detach=True):
+        from dvc.daemon import Daemon
+
+        if detach:
+            d = Daemon()
+            d('updater')
+            return
+
+        self._with_lock(self._get_latest_version, 'fetching')
+
+    def _get_latest_version(self):
+        import json
 
         try:
             r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
-            j = r.json()
-            latest = j['version']
-            open(self.updater_file, 'w+').close()
-        except Exception as exc:
-            msg = 'Failed to obtain latest version: {}'.format(str(exc))
-            Logger.debug(msg)
+            info = r.json()
+        except requests.exceptions.RequestException as exc:
+            msg = "Failed to retrieve latest version: {}"
+            Logger.debug(msg.format(exc))
             return
 
-        l_major, l_minor, l_patch = [int(x) for x in latest.split('.')]
-        c_major, c_minor, c_patch = [int(x) for x in current.split('.')]
+        with open(self.updater_file, 'w+') as fobj:
+            json.dump(info, fobj)
 
-        if l_major <= c_major and \
-           l_minor <= c_minor and \
-           l_patch <= c_patch:
-            return
+    def _is_outdated(self):
+        l_major, l_minor, l_patch = [int(x) for x in self.latest.split('.')]
+        c_major, c_minor, c_patch = [int(x) for x in self.current.split('.')]
 
-        msg = 'You are using dvc version {}, however version {} is ' \
-              'available. Consider upgrading.'
-        Logger.warn(msg.format(current, latest))
+        return (l_major > c_major or
+                l_minor > c_minor or
+                l_patch > c_patch)
+
+    def _notify(self):
+        message = (
+            'Update available {red}{current}{reset} -> {green}{latest}{reset}'
+            + '\n'
+            + self._get_update_instructions()
+        ).format(red=colorama.Fore.RED,
+                 reset=colorama.Fore.RESET,
+                 green=colorama.Fore.GREEN,
+                 yellow=colorama.Fore.YELLOW,
+                 blue=colorama.Fore.BLUE,
+                 current=self.current,
+                 latest=self.latest)
+
+        if sys.stdout.isatty():
+            Logger.box(message, border_color='yellow')
+
+    def _get_update_instructions(self):
+        instructions = {
+            'pip': 'Run {yellow}pip{reset} install dvc {blue}--upgrade{reset}',
+            'yum': 'Run {yellow}yum{reset} update dvc',
+            'yay': 'Run {yellow}yay{reset} {blue}-S{reset} dvc',
+            'formula': 'Run {yellow}brew{reset} upgrade dvc',
+            'cask': 'Run {yellow}brew cask{reset} upgrade dvc',
+            'apt': ('Run {yellow}apt-get{reset} install'
+                    ' {blue}--only-upgrade{reset} dvc'),
+            'binary': ('To upgrade follow this steps:\n'
+                       '1. Uninstall dvc binary\n'
+                       '2. Go to {blue}https://dvc.org{reset}\n'
+                       '3. Download and install new binary'),
+            None: ('Find the latest release at\n{blue}'
+                   'https://github.com/iterative/dvc/releases/latest'
+                   '{reset}'),
+        }
+
+        package_manager = self._get_package_manager()
+
+        return instructions[package_manager]
+
+    def _get_linux(self):
+        import distro
+
+        if not is_binary():
+            return 'pip'
+
+        package_managers = {
+            'rhel':     'yum',
+            'centos':   'yum',
+            'fedora':   'yum',
+            'amazon':   'yum',
+            'opensuse': 'yum',
+            'ubuntu':   'apt',
+            'debian':   'apt',
+        }
+
+        return package_managers.get(distro.id())
+
+    def _get_darwin(self):
+        if not is_binary():
+            if __file__.startswith('/usr/local/Cellar'):
+                return 'formula'
+            else:
+                return 'pip'
+
+        # NOTE: both pkg and cask put dvc binary into /usr/local/bin,
+        # so in order to know which method of installation was used,
+        # we need to actually call `brew cask`
+        ret = os.system('brew cask ls dvc')
+        if ret == 0:
+            return 'cask'
+
+        return None
+
+    def _get_windows(self):
+        return None if is_binary() else 'pip'
+
+    def _get_package_manager(self):
+        import platform
+        from dvc.exceptions import DvcException
+
+        m = {
+            'Windows': self._get_windows,
+            'Darwin': self._get_darwin,
+            'Linux': self._get_linux,
+        }
+
+        system = platform.system()
+        func = m.get(system)
+        if func is None:
+            raise DvcException("Not supported system '{}'".format(system))
+
+        return func()

dvc/utils.py

@@ -3,6 +3,7 @@ Helpers for other modules.
 """
 import os
 import sys
+import stat
 import math
 import json
 import shutil
@@ -102,6 +103,7 @@ def copyfile(src, dest, no_progress_bar=False, name=None):
 
 
 def move(src, dst):
+    dst = os.path.abspath(dst)
     dname = os.path.dirname(dst)
     if not os.path.exists(dname):
         os.makedirs(dname)
@@ -119,10 +121,17 @@ def remove(path):
         return
 
     Logger.debug(u'Removing \'{}\''.format(os.path.relpath(path)))
+
+    def _chmod(func, p, excinfo):
+        perm = os.stat(p).st_mode
+        perm |= stat.S_IWRITE
+        os.chmod(p, perm)
+        func(p)
+
     if os.path.isfile(path):
-        os.unlink(path)
+        _chmod(os.unlink, path, None)
     else:
-        shutil.rmtree(path)
+        shutil.rmtree(path, onerror=_chmod)
 
 
 def to_chunks(l, jobs):
@@ -137,6 +146,12 @@ def to_chunks(l, jobs):
     return [l[x:x+n] for x in range(0, len(l), n)]
 
 
+# NOTE: Check if we are in a bundle
+# https://pythonhosted.org/PyInstaller/runtime-information.html
+def is_binary():
+    return getattr(sys, 'frozen', False)
+
+
 # NOTE: Fix env variables modified by PyInstaller
 # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
 def fix_env(env):
@@ -145,9 +160,7 @@ def fix_env(env):
     else:
         env = env.copy()
 
-    # NOTE: Check if we are in a bundle
-    # https://pythonhosted.org/PyInstaller/runtime-information.html
-    if getattr(sys, 'frozen', False):
+    if is_binary():
         lp_key = 'LD_LIBRARY_PATH'
         lp_orig = env.get(lp_key + '_ORIG', None)
         if lp_orig is not None:

requirements.txt

@@ -4,21 +4,23 @@ ply>=3.9 # See https://github.com/pyinstaller/pyinstaller/issues/1945
 configparser>=3.5.0
 zc.lockfile>=1.2.1
 future>=0.16.0
-google-cloud==0.32.0
+google-cloud-storage==1.13.0
 PyInstaller==3.3.1
 colorama>=0.3.9
 configobj>=5.0.6
-networkx==2.1
+networkx>=2.1
 pyyaml>=3.12
 gitpython>=2.1.8
-ntfsutils>=0.1.4
 setuptools>=34.0.0
 nanotime>=0.5.2
 pyasn1>=0.4.1
 schema>=0.6.7
 paramiko>=2.4.1
 jsonpath-rw==1.4.0
-reflink==0.2.0
 requests>=2.18.4
 wheel>=0.31.1
 futures>=3.2.0; python_version == "2.7"
+grandalf==0.6
+pydot>=1.2.4
+asciimatics>=1.10.0
+distro>=1.3.0

scripts/ci/install.sh

@@ -3,9 +3,21 @@
 set -x
 set -e
 
-pip install --upgrade pip
-pip install -r requirements.txt
-pip install -r test-requirements.txt
+N_RETRIES=3
+
+function retry {
+    for i in $(seq $N_RETRIES); do
+        $@ && break
+    done
+}
+
+# NOTE: it is not uncommon for pip to hang on travis for what seems to be
+# networking issues. Thus, let's retry a few times to see if it will eventially
+# work or not.
+retry pip install --upgrade pip setuptools wheel
+retry pip install -r requirements.txt
+retry pip install -r test-requirements.txt
+
 git config --global user.email "[email protected]"
 git config --global user.name "DVC Tester"
 

scripts/ci/script.sh

@@ -6,7 +6,8 @@ set -e
 python -mtests
 
 if [[ "$TRAVIS_PULL_REQUEST" == "false" && \
-      "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
+      "$TRAVIS_SECURE_ENV_VARS" == "true" && \
+      "$(python -c 'import sys; print(sys.version_info[0])')" == '2' ]]; then
 	codecov
 fi
 

scripts/completion/dvc.bash

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+#----------------------------------------------------------
+# Repository:  https://github.com/iterative/dvc
+#
+# References:
+#   - https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion.html
+#   - https://opensource.com/article/18/3/creating-bash-completion-script
+#----------------------------------------------------------
+
+_dvc_commands='init destroy add import checkout run pull push fetch \
+              status repro remove move gc config remote metrics install \
+              root lock unlock pipeline'
+
+_dvc_global_options="-h --help -q --quiet -V --verbose"
+_dvc_options="-h --help -v --version"
+_dvc_init="--no-scm -f --force"
+_dvc_destroy="-f --force"
+_dvc_add="-R --recursive"
+_dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes"
+_dvc_pull="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps"
+_dvc_push="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps"
+_dvc_fetch="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps"
+_dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud"
+_dvc_repro="--dry -f --force -s --single-item -c --cwd -m --metrics -i --interactive -p --pipeline -P --all-pipelines"
+_dvc_remove="--dry -o --outs -p --purge"
+_dvc_gc="-a --all-branches -T --all-tags -c --cloud -r --remote"
+_dvc_config="-u --unset --local"
+_dvc_checkout="$(compgen -G '*.dvc')"
+_dvc_move="$(compgen -G '*')"
+_dvc_lock="$(compgen -G '*.dvc')"
+_dvc_unlock="$(compgen -G '*.dvc')"
+_dvc_import=""
+_dvc_remote=""
+_dvc_metrics=""
+_dvc_install=""
+_dvc_root=""
+_dvc_pipeline=""
+
+# Notes:
+#
+# `COMPREPLY` contains what will be rendered after completion is triggered
+#
+# `word` refers to the current typed word
+#
+# `${!var}` is to evaluate the content of `var` and expand its content as a variable
+#
+#       hello="world"
+#       x="hello"
+#       ${!x} ->  ${hello} ->  "world"
+#
+_dvc () {
+  local word="${COMP_WORDS[COMP_CWORD]}"
+
+  COMPREPLY=()
+
+  if [ "${COMP_CWORD}" -eq 1 ]; then
+    case "$word" in
+      -*) COMPREPLY=($(compgen -W "$_dvc_options" -- "$word")) ;;
+      *)  COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
+    esac
+  elif [ "${COMP_CWORD}" -eq 2 ]; then
+    local options_list="_dvc_${COMP_WORDS[1]}"
+
+    COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
+  fi
+
+  return 0
+}
+
+complete -F _dvc dvc

scripts/completion/dvc.zsh

@@ -0,0 +1,218 @@
+#compdef dvc
+
+#----------------------------------------------------------
+# Repository:  https://github.com/iterative/dvc
+#
+# References:
+#   - https://github.com/zsh-users/zsh-completions
+#   - http://zsh.sourceforge.net/Doc/Release/Completion-System.html
+#   - https://mads-hartmann.com/2017/08/06/writing-zsh-completion-scripts.html
+#   - http://www.linux-mag.com/id/1106/
+#----------------------------------------------------------
+
+_dvc_commands() {
+  local _commands=(
+    "init:Initialize dvc over a directory (should already be a git dir)."
+    "destroy:Destroy dvc. Will remove all project's information, data files and cache."
+    "add:Add files/directories to dvc."
+    "import:Import files from URL."
+    "checkout:Checkout data files from cache."
+    "run:Generate a stage file from a given command and execute the command."
+    "pull:Pull data files from the cloud."
+    "push:Push data files to the cloud."
+    "fetch:Fetch data files from the cloud."
+    "status:Show the project status."
+    "repro:Reproduce DVC file. Default file name - 'Dvcfile'."
+    "remove:Remove outputs of DVC file."
+    "move:Move output of DVC file."
+    "gc:Collect garbage."
+    "config:Get or set config options."
+    "remote:Manage set of tracked repositories."
+    "metrics:Get metrics from all branches."
+    "install:Install dvc hooks into the repository."
+    "root:Relative path to project's directory."
+    "lock:Lock DVC file."
+    "unlock:Unlock DVC file."
+    "pipeline:Manage pipeline."
+  )
+
+  _describe 'dvc commands' _commands
+}
+
+_dvc_global_options=(
+  "(-)"{-h,--help}"[Show help message related to the command]"
+  "(-)"{-q,--quiet}"[Be quiet.]"
+  "(-)"{-V,--verbose}"[Be verbose.]"
+)
+
+_dvc_options=(
+  "(-)"{-h,--help}"[Show this help message and exit]"
+  "(-)"{-v,--version}"[Show program's version]"
+)
+
+_dvc_init=(
+  "--no-scm[Initiate dvc in directory that is not tracket by any scm tool]"
+  {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
+)
+
+_dvc_destroy=(
+  {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
+)
+
+_dvc_add=(
+  {-R,--recursive}"[Recursively add each file under the directory.]"
+)
+
+_dvc_import=(
+  "1:URL:"
+  "2:Output:"
+)
+
+_dvc_checkout=(
+  "1:DVC files:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_run=(
+  "--no-exec[Only create stage file without actually running it.]"
+  {-f,--file}"[Specify name of the stage file.]:File:_files"
+  {-c,--cwd}"[Directory to run your command and place state file in.]:CWD:_files -/"
+  "*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files"
+  "*"{-o,--outs}"[Declare output data file or data directory.]:Output data:_files"
+  "*"{-O,--outs-no-cache}"[Declare output regular file or directory.]:Output regular:_files"
+  "*"{-M,--metrics-no-cache}"[Declare output metric file or directory]:Metrics (no cache):_files"
+  {-y,--yes}"[Automatic 'yes' answer to all prompts.]"
+)
+
+_dvc_pull=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+)
+
+_dvc_push=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+)
+
+_dvc_fetch=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+)
+
+_dvc_status=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+  {-c,--cloud}"[Show status of a local cache compared to a remote repository]"
+)
+
+_dvc_repro=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--dry[Only print the commands that would be executed without actually executing]"
+  {-f,--force}"[Reproduce even if dependencies were not changed.]"
+  {-s,--single-item}"[Reproduce only single data item without recursive dependencies check.]"
+  {-c,--cwd}"[Directory to reproduce from]:CWD:_files -/"
+  {-m,--metrics}"[Show metrics after reproduction.]"
+  {-i,--interactive}"[Ask for confirmation before reproducing each stage.]"
+  {-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]"
+  {-P,--all-pipelines}"[Reproduce all pipelines in the project.]"
+)
+
+_dvc_remove=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--dry[Only print the commands that would be executed without actually executing]"
+  {-o,--outs}"[Only remove DVC file outputs.(default)]"
+  {-p,--purge}"[Remove DVC file and all its outputs]"
+)
+
+_dvc_move=(
+  "1:Source:_files"
+  "2:Destination:"
+)
+
+_dvc_gc=(
+  {-a,--all-branches}"[Collect garbage for all branches]"
+  {-T,--all-tags}"[Collect garbage for all tags]"
+  {-c,--cloud}"[Collect garbage in remote repository]"
+  {-r,--remote}"[Remote repository to collect garbage in]:Remote repository:"
+)
+
+_dvc_config=(
+  {-u,--unset}"[Unset option.]"
+  "--local[Unset option.]"
+)
+
+_dvc_remote=(
+  "1:Task:(add remove modify list)"
+)
+
+_dvc_metrics=(
+  "1:Task:(show add modify remove)"
+)
+
+_dvc_install=()
+
+_dvc_root=()
+
+_dvc_lock=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_unlock=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_pipeline=(
+  "1:Task:(show)"
+)
+
+typeset -A opt_args
+local context state line curcontext="$curcontext"
+
+_arguments \
+  $_dvc_options \
+  '1: :_dvc_commands' \
+  '*::args:->args'
+
+case $words[1] in
+  init)     _arguments $_dvc_global_options   $_dvc_init     ;;
+  destroy)  _arguments $_dvc_global_options   $_dvc_destroy  ;;
+  add)      _arguments $_dvc_global_options   $_dvc_add      ;;
+  import)   _arguments $_dvc_global_options   $_dvc_import   ;;
+  checkout) _arguments $_dvc_global_options   $_dvc_checkout ;;
+  run)      _arguments $_dvc_global_options   $_dvc_run      ;;
+  pull)     _arguments $_dvc_global_options   $_dvc_pull     ;;
+  push)     _arguments $_dvc_global_options   $_dvc_push     ;;
+  fetch)    _arguments $_dvc_global_options   $_dvc_fetch    ;;
+  status)   _arguments $_dvc_global_options   $_dvc_status   ;;
+  repro)    _arguments $_dvc_global_options   $_dvc_repro    ;;
+  remove)   _arguments $_dvc_global_options   $_dvc_remove   ;;
+  move)     _arguments $_dvc_global_options   $_dvc_move     ;;
+  gc)       _arguments $_dvc_global_options   $_dvc_gc       ;;
+  config)   _arguments $_dvc_global_options   $_dvc_config   ;;
+  remote)   _arguments $_dvc_global_options   $_dvc_remote   ;;
+  metrics)  _arguments $_dvc_global_options   $_dvc_metrics  ;;
+  install)  _arguments $_dvc_global_options   $_dvc_install  ;;
+  root)     _arguments $_dvc_global_options   $_dvc_root     ;;
+  lock)     _arguments $_dvc_global_options   $_dvc_lock     ;;
+  unlock)   _arguments $_dvc_global_options   $_dvc_unlock   ;;
+  pipeline) _arguments $_dvc_global_options   $_dvc_pipeline ;;
+esac

setup.py

@@ -12,23 +12,23 @@ install_requires = [
     "future>=0.16.0",
     "colorama>=0.3.9",
     "configobj>=5.0.6",
-    "networkx==2.1",
+    "networkx>=2.1",
     "pyyaml>=3.12",
     "gitpython>=2.1.8",
-    "ntfsutils>=0.1.4",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
     "schema>=0.6.7",
     "jsonpath-rw==1.4.0",
-    "reflink==0.2.0",
     "requests>=2.18.4",
-    'futures; python_version == "2.7"',
+    "grandalf==0.6",
+    "asciimatics>=1.10.0",
+    "distro>=1.3.0",
 ]
 
 # Extra dependencies for remote integrations
 gs = [
-    "google-cloud==0.32.0",
+    "google-cloud-storage==1.13.0",
 ]
 s3 = [
     "boto3==1.7.4",
@@ -57,6 +57,8 @@ setup(
         's3': s3,
         'azure': azure,
         'ssh': ssh,
+        # NOTE: https://github.com/inveniosoftware/troubleshooting/issues/1
+        ':python_version=="2.7"': ['futures'],
     },
     keywords='data science, data version control, machine learning',
     classifiers=[
@@ -64,11 +66,11 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 3',
     ],
-    packages=find_packages(exclude=['bin', 'tests', 'functests']),
+    packages=find_packages(exclude=['tests']),
     include_package_data=True,
     url='http://dataversioncontrol.com',
     entry_points={
         'console_scripts': ['dvc = dvc.main:main']
     },
-    zip_safe=False
+    zip_safe=False,
 )

test-requirements.txt

@@ -1,9 +1,11 @@
 pytest>=3.3.1
 nose>=1.3.7
 mock>=2.0.0
-coverage
-codecov
+coverage>=4.5.1
+codecov>=2.0.15
 xmltodict>=0.11.0
 awscli>=1.14.18
 google-compute-engine
 pywin32; sys_platform == 'win32'
+Pygments
+collective.checkdocs

tests/__main__.py

@@ -6,5 +6,7 @@ os.chdir(REPO_ROOT)
 
 os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), os.getenv('PATH')))
 os.putenv('DVC_HOME', REPO_ROOT)
+os.putenv('DVC_TEST', 'true')
 
-check_call('nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage', shell=True)
+cmd = 'nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage'
+check_call(cmd, shell=True)

tests/basic_env.py

@@ -1,4 +1,5 @@
 import os
+import uuid
 import shutil
 import tempfile
 from git import Repo
@@ -38,8 +39,15 @@ class TestDir(TestCase):
         with open(name, 'a') as f:
             f.write(contents)
 
+    @staticmethod
+    def mkdtemp():
+        prefix = 'dvc-test.{}.'.format(os.getpid())
+        suffix = '.{}'.format(uuid.uuid4())
+        return tempfile.mkdtemp(prefix=prefix, suffix=suffix)
+
     def setUp(self):
-        self._root_dir = tempfile.mkdtemp()
+        self._root_dir = TestDir.mkdtemp()
+
         self._pushd(self._root_dir)
         self.create(self.FOO, self.FOO_CONTENTS)
         self.create(self.BAR, self.BAR_CONTENTS)
@@ -61,6 +69,15 @@ class TestGit(TestDir):
         self.git.index.commit('add code')
 
 
+class TestGitSubmodule(TestGit):
+    def setUp(self):
+        super(TestGitSubmodule, self).setUp()
+        subrepo = Repo.init()
+        subrepo_path = 'subrepo'
+        self.git.create_submodule(subrepo_path, subrepo_path, subrepo.git_dir)
+        self._pushd(subrepo_path)
+
+
 class TestDvc(TestGit):
     def setUp(self):
         super(TestDvc, self).setUp()

tests/test_add.py

@@ -1,6 +1,7 @@
 import os
 import stat
 import shutil
+import filecmp
 
 from dvc.main import main
 from dvc.utils import file_md5
@@ -17,7 +18,10 @@ class TestAdd(TestDvc):
     def test(self):
         md5 = file_md5(self.FOO)[0]
 
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
 
         self.assertIsInstance(stage, Stage)
         self.assertTrue(os.path.isfile(stage.path))
@@ -38,8 +42,10 @@ class TestAddDirectory(TestDvc):
         dname = 'directory'
         os.mkdir(dname)
         self.create(os.path.join(dname, 'file'), 'file')
-        stage = self.dvc.add(dname)
-        self.assertNotEqual(stage, None)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(len(stage.outs), 1)
 
@@ -50,12 +56,29 @@ class TestAddDirectory(TestDvc):
             self.assertTrue('\\' not in info['relpath'])
 
 
+class TestAddDirectoryRecursive(TestDvc):
+    def test(self):
+        stages = self.dvc.add(self.DATA_DIR, recursive=True)
+        self.assertEqual(len(stages), 2)
+
+
+class TestAddCmdDirectoryRecursive(TestDvc):
+    def test(self):
+        ret = main(['add',
+                    '--recursive',
+                    self.DATA_DIR])
+        self.assertEqual(ret, 0)
+
+
 class TestAddDirectoryWithForwardSlash(TestDvc):
     def test(self):
         dname = 'directory/'
         os.mkdir(dname)
         self.create(os.path.join(dname, 'file'), 'file')
-        stage = self.dvc.add(dname)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         self.assertEquals(os.path.abspath('directory.dvc'), stage.path)
 
 class TestAddTrackedFile(TestDvc):
@@ -76,26 +99,54 @@ class TestAddDirWithExistingCache(TestDvc):
         os.mkdir(dname)
         shutil.copyfile(self.FOO, fname)
 
-        self.dvc.add(self.FOO)
-        self.dvc.add(dname)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
 
 
 class TestAddModifiedDir(TestDvc):
     def test(self):
-        self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
         os.unlink(self.DATA)
-        self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
 
 
 class TestAddFileInDir(TestDvc):
     def test(self):
-        stage = self.dvc.add(self.DATA_SUB)
+        stages = self.dvc.add(self.DATA_SUB)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
         self.assertNotEqual(stage, None)
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(len(stage.outs), 1)
         self.assertEqual(stage.relpath, self.DATA_SUB + '.dvc')
 
 
+class TestAddExternalLocalFile(TestDvc):
+    def test(self):
+        dname = TestDvc.mkdtemp()
+        fname = os.path.join(dname, 'foo')
+        shutil.copyfile(self.FOO, fname)
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertNotEqual(stage, None)
+        self.assertEqual(len(stage.deps), 0)
+        self.assertEqual(len(stage.outs), 1)
+        self.assertEqual(stage.relpath, 'foo.dvc')
+        self.assertEqual(len(os.listdir(dname)), 1)
+        self.assertTrue(os.path.isfile(fname))
+        self.assertTrue(filecmp.cmp(fname, 'foo', shallow=False))
+
+
 class TestCmdAdd(TestDvc):
     def test(self):
         ret = main(['add',

tests/test_cache.py

@@ -1,6 +1,5 @@
 import os
 import shutil
-import tempfile
 
 from dvc.cache import Cache
 from dvc.system import System
@@ -52,7 +51,7 @@ class TestCacheLoadBadDirCache(TestDvc):
 
 class TestExternalCacheDir(TestDvc):
     def test(self):
-        cache_dir = tempfile.mkdtemp()
+        cache_dir = TestDvc.mkdtemp()
 
         ret = main(['config', 'cache.dir', cache_dir])
         self.assertEqual(ret, 0)
@@ -103,3 +102,12 @@ class TestSharedCacheDir(TestDir):
         self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1)
         self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1)
         self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1)
+
+
+class TestCacheLinkType(TestDvc):
+    def test(self):
+        ret = main(['config', 'cache.type', 'reflink,copy'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['add', self.FOO])
+        self.assertEqual(ret, 0)

tests/test_checkout.py

@@ -4,19 +4,28 @@ import time
 import stat
 import shutil
 import filecmp
+import collections
 
 from dvc.main import main
+from dvc.project import Project
+from dvc.system import System
 from tests.basic_env import TestDvc
 from tests.test_repro import TestRepro
 from dvc.stage import Stage
 from dvc.remote.local import RemoteLOCAL
+from dvc.exceptions import DvcException
 
+from mock import patch
 
 class TestCheckout(TestRepro):
     def setUp(self):
         super(TestCheckout, self).setUp()
 
-        self.data_dir_stage = self.dvc.add(self.DATA_DIR)
+
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.data_dir_stage = stages[0]
+        self.assertTrue(self.data_dir_stage is not None)
 
         self.orig = 'orig'
         shutil.copy(self.FOO, self.orig)
@@ -27,7 +36,7 @@ class TestCheckout(TestRepro):
         shutil.rmtree(self.DATA_DIR)
 
     def test(self):
-        self.dvc.checkout()
+        self.dvc.checkout(force=True)
         self._test_checkout()
 
     def _test_checkout(self):
@@ -37,10 +46,10 @@ class TestCheckout(TestRepro):
 
 class TestCheckoutSingleStage(TestCheckout):
     def test(self):
-        ret = main(['checkout', self.foo_stage.path])
+        ret = main(['checkout', '--force', self.foo_stage.path])
         self.assertEqual(ret, 0)
 
-        ret = main(['checkout', self.data_dir_stage.path])
+        ret = main(['checkout', '--force', self.data_dir_stage.path])
         self.assertEqual(ret, 0)
 
         self._test_checkout()
@@ -55,15 +64,43 @@ class TestCheckoutCorruptedCacheFile(TestRepro):
         with open(cache, 'a') as fd:
             fd.write('1')
 
-        self.dvc.checkout()
+        self.dvc.checkout(force=True)
 
         self.assertFalse(os.path.isfile(self.FOO))
         self.assertFalse(os.path.isfile(cache))
 
 
+class TestCheckoutCorruptedCacheDir(TestDvc):
+    def test(self):
+        time.sleep(1)
+
+        # NOTE: using 'copy' so that cache and link don't have same inode
+        ret = main(['config', 'cache.type', 'copy'])
+        self.assertEqual(ret, 0)
+
+        self.dvc = Project('.')
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.assertEqual(len(stages[0].outs), 1)
+        out = stages[0].outs[0]
+
+        # NOTE: modifying cache file for one of the files inside the directory
+        # to check if dvc will detect that the cache is corrupted.
+        entry = self.dvc.cache.local.load_dir_cache(out.md5)[0]
+        md5 = entry[self.dvc.cache.local.PARAM_MD5]
+        cache = self.dvc.cache.local.get(md5)
+
+        with open(cache, 'w+') as fobj:
+            fobj.write('1')
+
+        self.dvc.checkout(force=True)
+
+        self.assertFalse(os.path.exists(cache))
+
+
 class TestCmdCheckout(TestCheckout):
     def test(self):
-        ret = main(['checkout'])
+        ret = main(['checkout', '--force'])
         self.assertEqual(ret, 0)
         self._test_checkout()
 
@@ -74,13 +111,29 @@ class CheckoutBase(TestDvc):
     def commit_data_file(self, fname, content='random text'):
         with open(fname, 'w') as fd:
             fd.write(content)
-        self.dvc.add(fname)
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
         self.dvc.scm.add([fname + '.dvc', '.gitignore'])
         self.dvc.scm.commit('adding ' + fname)
 
     def read_ignored(self):
         return list(map(lambda s: s.strip('\n'), open(self.GIT_IGNORE).readlines()))
 
+    def outs_info(self, stage):
+        FileInfo = collections.namedtuple('FileInfo', 'path inode')
+
+        paths = [
+            os.path.join(root, file)
+            for output in stage.outs
+            for root, _, files in os.walk(output.path)
+            for file in files
+        ]
+
+        return [
+            FileInfo(path=path, inode=System.inode(path))
+            for path in paths
+        ]
 
 class TestRemoveFilesWhenCheckout(CheckoutBase):
     def test(self):
@@ -93,7 +146,8 @@ class TestRemoveFilesWhenCheckout(CheckoutBase):
 
         # add the file into a separate branch
         self.dvc.scm.checkout(branch_1, True)
-        main(['checkout'])
+        ret = main(['checkout', '--force'])
+        self.assertEqual(ret, 0)
         self.commit_data_file(fname)
 
         # Checkout back in master
@@ -102,10 +156,88 @@ class TestRemoveFilesWhenCheckout(CheckoutBase):
 
         # Make sure `dvc checkout` removes the file
         # self.dvc.checkout()
-        main(['checkout'])
+        ret = main(['checkout', '--force'])
+        self.assertEqual(ret, 0)
         self.assertFalse(os.path.exists(fname))
 
 
+class TestCheckoutCleanWorkingDir(CheckoutBase):
+    def test(self):
+        from tests.test_data_cloud import sleep
+
+        stages = self.dvc.add(self.DATA_DIR)
+        stage = stages[0]
+
+        sleep()
+
+        working_dir_change = os.path.join(self.DATA_DIR, 'not_cached.txt')
+        with open(working_dir_change, 'w') as f:
+            f.write('not_cached')
+
+        sleep()
+
+        ret = main(['checkout', '--force', stage.relpath])
+        self.assertEqual(ret, 0)
+        self.assertFalse(os.path.exists(working_dir_change))
+
+    @patch('dvc.prompt.Prompt.prompt')
+    def test_force(self, mock_prompt):
+        from tests.test_data_cloud import sleep
+
+        mock_prompt.return_value = False
+
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        
+        sleep()
+        
+        working_dir_change = os.path.join(self.DATA_DIR, 'not_cached.txt')
+        with open(working_dir_change, 'w') as f:
+            f.write('not_cached')
+
+        sleep()
+            
+        ret = main(['checkout', stage.relpath])
+        self.assertNotEqual(ret, 0)
+
+        mock_prompt.assert_called()
+        self.assertNotEqual(ret, 0)
+        self.assertRaises(DvcException)
+
+
+class TestCheckoutSelectiveRemove(CheckoutBase):
+    def test(self):
+        from tests.test_data_cloud import sleep
+
+        # Use copy to test for changes in the inodes
+        ret = main(['config', 'cache.type', 'copy'])
+        self.assertEqual(ret, 0)
+
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        staged_files = self.outs_info(stage)
+
+        sleep()
+
+        os.remove(staged_files[0].path)
+
+        sleep()
+
+        ret = main(['checkout', '--force', stage.relpath])
+        self.assertEqual(ret, 0)
+
+        sleep()
+
+        checkedout_files = self.outs_info(stage)
+
+        self.assertEqual(len(staged_files), len(checkedout_files))
+        self.assertEqual(staged_files[0].path, checkedout_files[0].path)
+        self.assertNotEqual(staged_files[0].inode, checkedout_files[0].inode)
+        self.assertEqual(staged_files[1].inode, checkedout_files[1].inode)
+
+
 class TestGitIgnoreBasic(CheckoutBase):
     def test(self):
         fname1 = 'file_1'
@@ -129,8 +261,8 @@ class TestGitIgnoreBasic(CheckoutBase):
 
         self.assertEqual(len(ignored), 2)
 
-        self.assertIn(fname1, ignored)
-        self.assertIn(fname2, ignored)
+        self.assertIn('/' + fname1, ignored)
+        self.assertIn('/' + fname2, ignored)
 
 
 class TestGitIgnoreWhenCheckout(CheckoutBase):
@@ -145,21 +277,24 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
         self.commit_data_file(fname_master)
 
         self.dvc.scm.checkout(branch_1, True)
-        main(['checkout'])
+        ret = main(['checkout', '--force'])
+        self.assertEqual(ret, 0)
         self.commit_data_file(fname_branch)
 
         self.dvc.scm.checkout(branch_master)
-        main(['checkout'])
+        ret = main(['checkout', '--force'])
+        self.assertEqual(ret, 0)
 
         ignored = self.read_ignored()
 
         self.assertEqual(len(ignored), 1)
-        self.assertIn(fname_master, ignored)
+        self.assertIn('/' + fname_master, ignored)
 
         self.dvc.scm.checkout(branch_1)
-        main(['checkout'])
+        ret = main(['checkout', '--force'])
+        self.assertEqual(ret, 0)
         ignored = self.read_ignored()
-        self.assertIn(fname_branch, ignored)
+        self.assertIn('/' + fname_branch, ignored)
 
 
 class TestCheckoutMissingMd5InStageFile(TestRepro):
@@ -173,7 +308,7 @@ class TestCheckoutMissingMd5InStageFile(TestRepro):
         with open(self.file1_stage, 'w') as fd:
             yaml.dump(d, fd)
 
-        self.dvc.checkout()
+        self.dvc.checkout(force=True)
 
 
 class TestCheckoutEmptyDir(TestDvc):
@@ -181,13 +316,43 @@ class TestCheckoutEmptyDir(TestDvc):
         dname = 'empty_dir'
         os.mkdir(dname)
 
-        stage = self.dvc.add(dname)
+        stages = self.dvc.add(dname)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         self.assertEqual(len(stage.outs), 1)
 
         stage.outs[0].remove()
         self.assertFalse(os.path.exists(dname))
 
-        self.dvc.checkout()
+        self.dvc.checkout(force=True)
 
         self.assertTrue(os.path.isdir(dname))
         self.assertEqual(len(os.listdir(dname)), 0)
+
+
+class TestCheckoutNotCachedFile(TestDvc):
+    def test(self):
+        cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
+
+        self.dvc.add(self.FOO)
+        stage = self.dvc.run(cmd=cmd,
+                             deps=[self.FOO, self.CODE],
+                             outs_no_cache=['out'])
+
+        self.dvc.checkout(force=True)
+
+
+class TestCheckoutWithDeps(TestRepro):
+    def test(self):
+        os.unlink(self.FOO)
+        os.unlink(self.file1)
+
+        self.assertFalse(os.path.exists(self.FOO))
+        self.assertFalse(os.path.exists(self.file1))
+
+        ret = main(['checkout', '--force', self.file1_stage, '--with-deps'])
+        self.assertEqual(ret, 0)
+
+        self.assertTrue(os.path.exists(self.FOO))
+        self.assertTrue(os.path.exists(self.file1))

tests/test_cli.py

@@ -129,6 +129,20 @@ class TestGC(TestDvc):
         self.assertIsInstance(args.func(args), CmdGC)
 
 
+class TestGCMultipleProjects(TestDvc):
+    def test(self):
+        args = parse_args([
+            'gc',
+            '-p',
+            '/tmp/asdf',
+            '/tmp/xyz'
+        ])
+
+        self.assertIsInstance(args.func(args), CmdGC)
+
+        self.assertEqual(args.projects, ['/tmp/asdf', '/tmp/xyz'])
+
+
 class TestConfig(TestDvc):
     def test(self):
         name = 'param'

tests/test_config.py

@@ -28,6 +28,11 @@ class TestConfigCLI(TestDvc):
         ret = main(['root'])
         self.assertEqual(ret, 0)
 
+        # NOTE: check that `dvc root` is not blocked with dvc lock
+        with self.dvc.lock:
+            ret = main(['root'])
+        self.assertEqual(ret, 0)
+
     def _do_test(self, local=False):
         section = 'setsection'
         field = 'setfield'

tests/test_data_cloud.py

@@ -1,12 +1,11 @@
 from subprocess import CalledProcessError
-from subprocess import check_output
+from subprocess import check_output, Popen
 from unittest import SkipTest
 import os
 import time
 import uuid
 import shutil
 import getpass
-import tempfile
 import platform
 
 from dvc.main import main
@@ -94,11 +93,18 @@ def _should_test_hdfs():
     except (CalledProcessError, IOError):
         return False
 
+    p = Popen('hadoop fs -ls hdfs://127.0.0.1/',
+              shell=True,
+              executable=os.getenv('SHELL'))
+    p.communicate()
+    if p.returncode != 0:
+        return False
+
     return True
 
 
 def get_local_storagepath():
-    return tempfile.mkdtemp()
+    return TestDvc.mkdtemp()
 
 
 def get_local_url():
@@ -154,10 +160,10 @@ class TestDataCloud(TestDvc):
     def test(self):
         config = TEST_CONFIG
 
-        clist = [('s3://', RemoteS3),
-                 ('gs://', RemoteGS),
+        clist = [('s3://mybucket/', RemoteS3),
+                 ('gs://mybucket/', RemoteGS),
                  ('ssh://user@localhost:/', RemoteSSH),
-                 (tempfile.mkdtemp(), RemoteLOCAL)]
+                 (TestDvc.mkdtemp(), RemoteLOCAL)]
 
         if _should_test_azure():
             clist.append(('azure://ContainerName=', RemoteAzure))
@@ -204,65 +210,73 @@ class TestDataCloudBase(TestDvc):
     def _test_cloud(self):
         self._setup_cloud()
 
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         cache = stage.outs[0].cache
         info = stage.outs[0].dumpd()
         md5 = info['md5']
 
-        stage_dir = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage_dir = stages[0]
+        self.assertTrue(stage_dir is not None)
+
         cache_dir = stage_dir.outs[0].cache
         info_dir = stage_dir.outs[0].dumpd()
         md5_dir = info_dir['md5']
 
-        # Check status
-        sleep()
-        status = self.cloud.status([info], show_checksums=True)
-        self.assertEqual([(md5, STATUS_NEW)], status)
+        with self.cloud.project.state:
+            # Check status
+            sleep()
+            status = self.cloud.status([info], show_checksums=True)
+            self.assertEqual([(md5, STATUS_NEW)], status)
 
-        status_dir = self.cloud.status([info_dir], show_checksums=True)
-        self.assertTrue((md5_dir, STATUS_NEW) in status_dir)
+            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            self.assertTrue((md5_dir, STATUS_NEW) in status_dir)
 
-        # Push and check status
-        self.cloud.push([info])
-        self.assertTrue(os.path.exists(cache))
-        self.assertTrue(os.path.isfile(cache))
+            # Push and check status
+            self.cloud.push([info])
+            self.assertTrue(os.path.exists(cache))
+            self.assertTrue(os.path.isfile(cache))
 
-        self.cloud.push([info_dir])
-        self.assertTrue(os.path.isfile(cache_dir))
+            self.cloud.push([info_dir])
+            self.assertTrue(os.path.isfile(cache_dir))
 
-        sleep()
-        status = self.cloud.status([info], show_checksums=True)
-        self.assertEqual([(md5, STATUS_OK)], status)
+            sleep()
+            status = self.cloud.status([info], show_checksums=True)
+            self.assertEqual([(md5, STATUS_OK)], status)
 
-        status_dir = self.cloud.status([info_dir], show_checksums=True)
-        self.assertTrue((md5_dir, STATUS_OK) in status_dir)
+            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            self.assertTrue((md5_dir, STATUS_OK) in status_dir)
 
-        # Remove and check status
-        sleep()
-        shutil.rmtree(self.dvc.cache.local.cache_dir)
+            # Remove and check status
+            sleep()
+            shutil.rmtree(self.dvc.cache.local.cache_dir)
 
-        status = self.cloud.status([info], show_checksums=True)
-        self.assertEqual([(md5, STATUS_DELETED)], status)
+            status = self.cloud.status([info], show_checksums=True)
+            self.assertEqual([(md5, STATUS_DELETED)], status)
 
-        status_dir = self.cloud.status([info_dir], show_checksums=True)
-        self.assertEqual([(md5_dir, STATUS_DELETED)], status_dir)
+            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            self.assertEqual([(md5_dir, STATUS_DELETED)], status_dir)
 
-        # Pull and check status
-        self.cloud.pull([info])
-        self.assertTrue(os.path.exists(cache))
-        self.assertTrue(os.path.isfile(cache))
-        with open(cache, 'r') as fd:
-            self.assertEqual(fd.read(), self.FOO_CONTENTS)
+            # Pull and check status
+            self.cloud.pull([info])
+            self.assertTrue(os.path.exists(cache))
+            self.assertTrue(os.path.isfile(cache))
+            with open(cache, 'r') as fd:
+                self.assertEqual(fd.read(), self.FOO_CONTENTS)
 
-        self.cloud.pull([info_dir])
-        self.assertTrue(os.path.isfile(cache_dir))
+            self.cloud.pull([info_dir])
+            self.assertTrue(os.path.isfile(cache_dir))
 
-        sleep()
-        status = self.cloud.status([info], show_checksums=True)
-        self.assertEqual([(md5, STATUS_OK)], status)
+            sleep()
+            status = self.cloud.status([info], show_checksums=True)
+            self.assertEqual([(md5, STATUS_OK)], status)
 
-        status_dir = self.cloud.status([info_dir], show_checksums=True)
-        self.assertTrue((md5_dir, STATUS_OK) in status_dir)
+            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            self.assertTrue((md5_dir, STATUS_OK) in status_dir)
 
     def test(self):
         self._ensure_should_run()
@@ -328,9 +342,6 @@ class TestRemoteSSH(TestDataCloudBase):
     def _get_cloud_class(self):
         return RemoteSSH
 
-    def test(self):
-        super(TestRemoteSSH, self).test()
-
 
 class TestRemoteHDFS(TestDataCloudBase):
     def _should_test(self):
@@ -342,9 +353,6 @@ class TestRemoteHDFS(TestDataCloudBase):
     def _get_cloud_class(self):
         return RemoteHDFS
 
-    def test(self):
-        super(TestRemoteHDFS, self).test()
-
 
 class TestDataCloudCLIBase(TestDvc):
     def main(self, args):
@@ -358,10 +366,16 @@ class TestDataCloudCLIBase(TestDvc):
         else:
             args += []
 
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         cache = stage.outs[0].cache
 
-        stage_dir = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage_dir = stages[0]
+        self.assertTrue(stage_dir is not None)
         cache_dir = stage_dir.outs[0].cache
 
         # FIXME check status output
@@ -400,6 +414,21 @@ class TestDataCloudCLIBase(TestDvc):
 
         self.main(['status', '-c', '--show-checksums'] + args)
 
+        # NOTE: check if remote gc works correctly on directories
+        self.main(['gc', '-c', '-f'] + args)
+        shutil.move(self.dvc.cache.local.cache_dir, self.dvc.cache.local.cache_dir + '.back')
+        
+        sleep()
+        self.main(['fetch'] + args)
+
+        sleep()
+        self.main(['pull'] + args)
+        self.assertTrue(os.path.exists(cache))
+        self.assertTrue(os.path.isfile(cache))
+        self.assertTrue(os.path.isfile(cache_dir))
+        self.assertTrue(os.path.isfile(self.FOO))
+        self.assertTrue(os.path.isdir(self.DATA_DIR))
+
     def _should_test(self):
         return True
 

tests/test_gc.py

@@ -1,13 +1,18 @@
 import os
 import shutil
 import filecmp
+import configobj
+
+from git import Repo
 
 from dvc.main import main
 from dvc.utils import file_md5
 from dvc.stage import Stage
+from dvc.project import Project
 from dvc.command.gc import CmdGC
 
 from tests.basic_env import TestDvc
+from tests.basic_env import TestDir
 
 
 class TestGC(TestDvc):
@@ -16,7 +21,8 @@ class TestGC(TestDvc):
 
         self.dvc.add(self.FOO)
         self.dvc.add(self.DATA_DIR)
-        self.good_cache = [self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all()]
+        self.good_cache = [self.dvc.cache.local.get(
+            md5) for md5 in self.dvc.cache.local.all()]
 
         self.bad_cache = []
         for i in ['123', '234', '345']:
@@ -29,7 +35,8 @@ class TestGC(TestDvc):
         self._test_gc()
 
     def test_cli(self):
-        ret = main(['gc'])
+        ret = main(['gc', '-f'])
+        self.assertEqual(ret, 0)
         self._test_gc()
 
     def _test_gc(self):
@@ -39,3 +46,135 @@ class TestGC(TestDvc):
 
         for c in self.good_cache:
             self.assertTrue(os.path.exists(c))
+
+
+class TestGCBranchesTags(TestDvc):
+    def _check_cache(self, num):
+        total = 0
+        for root, dirs, files in os.walk(os.path.join('.dvc', 'cache')):
+            total += len(files)
+        self.assertEqual(total, num)
+
+    def test(self):
+        fname = 'file'
+
+        with open(fname, 'w+') as fobj:
+            fobj.write('v1.0')
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        self.dvc.scm.add(['.gitignore', stages[0].relpath])
+        self.dvc.scm.commit('v1.0')
+        self.dvc.scm.tag('v1.0')
+
+        self.dvc.scm.checkout('test', create_new=True)
+        self.dvc.remove(stages[0].relpath, outs_only=True)
+        with open(fname, 'w+') as fobj:
+            fobj.write('test')
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        self.dvc.scm.add(['.gitignore', stages[0].relpath])
+        self.dvc.scm.commit('test')
+
+        self.dvc.scm.checkout('master')
+        self.dvc.remove(stages[0].relpath, outs_only=True)
+        with open(fname, 'w+') as fobj:
+            fobj.write('trash')
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        self.dvc.scm.add(['.gitignore', stages[0].relpath])
+        self.dvc.scm.commit('trash')
+
+        self.dvc.remove(stages[0].relpath, outs_only=True)
+        with open(fname, 'w+') as fobj:
+            fobj.write('master')
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+        self.dvc.scm.add(['.gitignore', stages[0].relpath])
+        self.dvc.scm.commit('master')
+
+        self._check_cache(4)
+
+        self.dvc.gc(all_tags=True, all_branches=True)
+
+        self._check_cache(3)
+
+        self.dvc.gc(all_tags=False, all_branches=True)
+
+        self._check_cache(2)
+
+        self.dvc.gc(all_tags=True, all_branches=False)
+
+        self._check_cache(1)
+
+
+class TestGCMultipleProjects(TestDvc):
+    def _check_cache(self, num):
+        total = 0
+        for root, dirs, files in os.walk(os.path.join('.dvc', 'cache')):
+            total += len(files)
+        self.assertEqual(total, num)
+
+    def setUp(self):
+        super(TestGCMultipleProjects, self).setUp()
+        self.additional_path = TestDir.mkdtemp()
+        self.additional_git = Repo.init(self.additional_path)
+        self.additional_dvc = Project.init(self.additional_path)
+
+        cache_path = os.path.join(self._root_dir, '.dvc', 'cache')
+        config_path = os.path.join(self.additional_path, '.dvc',
+                                   'config.local')
+        cfg = configobj.ConfigObj()
+        cfg.filename = config_path
+        cfg['cache'] = {
+            'dir': cache_path
+        }
+        cfg.write()
+
+        self.additional_dvc = Project(self.additional_path)
+
+    def test(self):
+
+        # ADD FILE ONLY IN MAIN PROJECT
+        fname = 'only_in_first'
+        with open(fname, 'w+') as fobj:
+            fobj.write('only in main project')
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        # ADD FILE IN MAIN PROJECT THAT IS ALSO IN SECOND PROJECT
+        fname = 'in_both'
+        with open(fname, 'w+') as fobj:
+            fobj.write('in both projects')
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        cwd = os.getcwd()
+        os.chdir(self.additional_path)
+        # ADD FILE ONLY IN SECOND PROJECT
+        fname = os.path.join(self.additional_path, 'only_in_second')
+        with open(fname, 'w+') as fobj:
+            fobj.write('only in additional project')
+
+        stages = self.additional_dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        # ADD FILE IN SECOND PROJECT THAT IS ALSO IN MAIN PROJECT
+        fname = os.path.join(self.additional_path, 'in_both')
+        with open(fname, 'w+') as fobj:
+            fobj.write('in both projects')
+
+        stages = self.additional_dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        os.chdir(cwd)
+
+        self._check_cache(3)
+
+        self.dvc.gc(projects=[self.additional_path])
+        self._check_cache(3)
+
+        self.dvc.gc()
+        self._check_cache(2)

tests/test_install.py

@@ -13,7 +13,11 @@ class TestInstall(TestDvc):
         ret = main(['install'])
         self.assertNotEqual(ret, 0)
 
-        self.assertTrue(os.path.isfile('.git/hooks/post-checkout'))
+        def hook(name):
+            return os.path.join('.git', 'hooks', name)
+
+        self.assertTrue(os.path.isfile(hook('post-checkout')))
+        self.assertTrue(os.path.isfile(hook('pre-commit')))
 
         self.dvc.add(self.FOO)
         os.unlink(self.FOO)

tests/test_logger.py

@@ -1,13 +1,26 @@
+from mock import patch
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
 import colorama
 import logging
 
 from dvc.command.base import CmdBase
 from dvc.config import Config
 from dvc.logger import Logger
+from dvc.main import main
 
 from tests.basic_env import TestDvc
 
 
+class StringContaining(str):
+    def __eq__(self, other):
+        return self in other
+
+
 class TestLogger(TestDvc):
     def test_config(self):
         Logger('debug')
@@ -34,6 +47,28 @@ class TestLogger(TestDvc):
             # This is not a tty, so it should not colorize anything
             self.assertEqual(msg, name)
 
+    @patch('sys.stderr', new_callable=StringIO)
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_stderr(self, mock_stdout, mock_stderr):
+        error_message = 'error msg'
+        Logger.init()
+        Logger.error(error_message)
+        self.assertEqual('', mock_stdout.getvalue())
+        self.assertEqual('Error: {}\n\nHaving any troubles? '
+                                   'Hit us up at dvc.org/support, we '
+                                   'are always happy to help!\n'.format(error_message),
+                                   mock_stderr.getvalue())
+
+    @patch('sys.stderr', new_callable=StringIO)
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_stdout(self, mock_stdout, mock_stderr):
+        non_error_message = 'non-error message'
+        Logger.init()
+        Logger.info(non_error_message)
+        self.assertEqual('', mock_stderr.getvalue())
+        self.assertEqual('{}\n'.format(non_error_message),
+                                   mock_stdout.getvalue())
+
 
 class TestLoggerCLI(TestDvc):
     def test(self):

tests/test_metrics.py

@@ -214,12 +214,14 @@ class TestNoMetrics(TestDvc):
 class TestCachedMetrics(TestDvc):
     def _do_write(self, branch):
         self.dvc.scm.checkout(branch)
-        self.dvc.checkout()
+        self.dvc.checkout(force=True)
 
         with open('metrics.json', 'w+') as fd:
             json.dump({'metrics': branch}, fd)
 
-        stage = self.dvc.add('metrics.json')
+        stages = self.dvc.add('metrics.json')
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
         self.assertNotEqual(stage, None)
 
         self.dvc.scm.add(['.gitignore', 'metrics.json.dvc'])

tests/test_move.py

@@ -1,9 +1,10 @@
 import os
 
 from dvc.main import main
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, MoveNotDataSourceError
 
 from tests.basic_env import TestDvc
+from tests.test_repro import TestRepro
 
 
 class TestMove(TestDvc):
@@ -39,3 +40,15 @@ class TestCmdMove(TestDvc):
 
         ret = main(['move', 'non-existing-file', 'dst'])
         self.assertNotEqual(ret, 0)
+
+
+class TestMoveNotDataSource(TestRepro):
+    def test(self):
+        from dvc.project import Project
+
+        self.dvc = Project(self._root_dir)
+        with self.assertRaises(MoveNotDataSourceError):
+            self.dvc.move(self.file1, 'dst')
+
+        ret = main(['move', self.file1, 'dst'])
+        self.assertNotEqual(ret, 0)

tests/test_pipeline.py

@@ -1,9 +1,71 @@
 from dvc.main import main
 
-from tests.test_repro import TestRepro
+from tests.basic_env import TestDvc
+from tests.test_repro import TestRepro, TestReproChangedDeepData
+import os
+
+
+class TestPipelineShowSingle(TestDvc):
+    def setUp(self):
+        super(TestPipelineShowSingle, self).setUp()
+        self.stage = 'foo.dvc'
+        self.dotFile = 'graph.dot'
+        ret = main(['add', self.FOO])
+        self.assertEqual(ret, 0)
+
+    def test(self):
+        ret = main(['pipeline', 'show', self.stage])
+        self.assertEqual(ret, 0)
+
+    def test_commands(self):
+        ret = main(['pipeline', 'show', self.stage, '--commands'])
+        self.assertEqual(ret, 0)        
+
+    def test_outs(self):
+        ret = main(['pipeline', 'show', self.stage, '--outs'])
+        self.assertEqual(ret, 0)        
+
+    def test_ascii(self):
+        ret = main(['pipeline', 'show', '--ascii', self.stage])
+        self.assertEqual(ret, 0)
+
+    def test_dot(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_ascii_commands(self):
+        ret = main(['pipeline', 'show', '--ascii', self.stage, '--commands'])
+        self.assertEqual(ret, 0)
+
+    def test_ascii_outs(self):
+        ret = main(['pipeline', 'show', '--ascii', self.stage, '--outs'])
+        self.assertEqual(ret, 0)
+
+    def test_dot_commands(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, '--commands'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_dot_outs(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, '--outs'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_not_dvc_file(self):
+        ret = main(['pipeline', 'show', self.FOO])
+        self.assertNotEqual(ret, 0)
+
+    def test_non_existing(self):
+        ret = main(['pipeline', 'show', 'non-existing'])
+        self.assertNotEqual(ret, 0)
 
 
 class TestPipelineShow(TestRepro):
+    def setUp(self):
+        super(TestPipelineShow, self).setUp()
+        self.dotFile = 'graph.dot'
+
     def test(self):
         ret = main(['pipeline', 'show', self.file1_stage])
         self.assertEqual(ret, 0)
@@ -16,6 +78,33 @@ class TestPipelineShow(TestRepro):
         ret = main(['pipeline', 'show', self.file1_stage, '--outs'])
         self.assertEqual(ret, 0)        
 
+    def test_ascii(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+    def test_dot(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_ascii_commands(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+
+    def test_ascii_outs(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+
+    def test_dot_commands(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_dot_outs(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_not_dvc_file(self):
         ret = main(['pipeline', 'show', self.file1])
         self.assertNotEqual(ret, 0)
@@ -23,3 +112,68 @@ class TestPipelineShow(TestRepro):
     def test_non_existing(self):
         ret = main(['pipeline', 'show', 'non-existing'])
         self.assertNotEqual(ret, 0)
+
+
+class TestPipelineShowDeep(TestReproChangedDeepData):
+    def setUp(self):
+        super(TestPipelineShowDeep, self).setUp()
+        self.dotFile = 'graph.dot'
+
+    def test(self):
+        ret = main(['pipeline', 'show', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+    def test_commands(self):
+        ret = main(['pipeline', 'show', self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)        
+
+    def test_outs(self):
+        ret = main(['pipeline', 'show', self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)        
+
+    def test_ascii(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+    def test_dot(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_ascii_commands(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+
+    def test_ascii_outs(self):
+        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+
+    def test_dot_commands(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_dot_outs(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_not_dvc_file(self):
+        ret = main(['pipeline', 'show', self.file1])
+        self.assertNotEqual(ret, 0)
+
+    def test_non_existing(self):
+        ret = main(['pipeline', 'show', 'non-existing'])
+        self.assertNotEqual(ret, 0)
+
+
+class TestPipelineListEmpty(TestDvc):
+    def test(self):
+        ret = main(['pipeline', 'list'])
+        self.assertEqual(ret, 0)
+
+
+class TestPipelineListSingle(TestPipelineShowDeep):
+    def test(self):
+        ret = main(['pipeline', 'list'])
+        self.assertEqual(ret, 0)

tests/test_remote.py

@@ -1,4 +1,8 @@
+import os
+import configobj
+
 from dvc.main import main
+from dvc.project import Project
 from dvc.command.remote import CmdRemoteAdd
 from dvc.exceptions import UnsupportedRemoteError
 from dvc.remote import Remote
@@ -41,3 +45,43 @@ class TestRemote(TestDvc):
     def test_unsupported(self):
         with self.assertRaises(UnsupportedRemoteError):
             Remote(self.dvc, {Config.SECTION_REMOTE_URL: 'unsupported://url'})
+
+
+class TestRemoteRemoveDefault(TestDvc):
+    def test(self):
+        remote = 'mys3'
+        self.assertEqual(main(['remote', 'add', '--default', remote, 's3://bucket/name']), 0)
+        self.assertEqual(main(['remote', 'modify', remote, 'profile', 'default']), 0)
+        self.assertEqual(main(['config', '--local', 'core.remote', remote]), 0)
+
+        config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG))
+        local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL))
+        self.assertEqual(config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote)
+        self.assertEqual(local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote)
+
+        self.assertEqual(main(['remote', 'remove', remote]), 0)
+        config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG))
+        local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL))
+        section = Config.SECTION_REMOTE_FMT.format(remote)
+        self.assertTrue(section not in config.keys())
+
+        core = config.get(Config.SECTION_CORE, None)
+        if core is not None:
+            self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys())
+
+        core = local_config.get(Config.SECTION_CORE, None)
+        if core is not None:
+            self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys())
+
+
+class TestRemoteRemove(TestDvc):
+    def test(self):
+        ret = main(['config', 'core.jobs', '1'])
+        self.assertEqual(ret, 0)
+
+        remote = 'mys3'
+        ret = main(['remote', 'add', remote, 's3://bucket/name'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['remote', 'remove', remote])
+        self.assertEqual(ret, 0)

tests/test_remove.py

@@ -1,16 +1,20 @@
 import os
 
 from dvc.main import main
-from dvc.stage import Stage
-from dvc.project import NotDvcFileError
+from dvc.stage import Stage, StageFileDoesNotExistError
+from dvc.exceptions import DvcException
 from dvc.command.remove import CmdRemove
 
 from tests.basic_env import TestDvc
+from mock import patch
 
 
 class TestRemove(TestDvc):
     def test(self):
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         stage_removed = self.dvc.remove(stage.path, outs_only=True)
 
         self.assertIsInstance(stage_removed, Stage)
@@ -26,13 +30,16 @@ class TestRemove(TestDvc):
 
 class TestRemoveNonExistentFile(TestDvc):
     def test(self):
-        with self.assertRaises(NotDvcFileError) as cx:
+        with self.assertRaises(StageFileDoesNotExistError) as cx:
             self.dvc.remove('non_existent_dvc_file')
 
 
 class TestRemoveDirectory(TestDvc):
     def test(self):
-        stage_add = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        stage_add = stages[0]
+        self.assertTrue(stage_add is not None)
         stage_removed = self.dvc.remove(stage_add.path)
         self.assertEqual(stage_add.path, stage_removed.path)
         self.assertFalse(os.path.exists(self.DATA_DIR))
@@ -41,7 +48,10 @@ class TestRemoveDirectory(TestDvc):
 
 class TestCmdRemove(TestDvc):
     def test(self):
-        stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
         ret = main(['remove',
                     stage.path])
         self.assertEqual(ret, 0)
@@ -49,3 +59,23 @@ class TestCmdRemove(TestDvc):
         ret = main(['remove',
                     'non-existing-dvc-file'])
         self.assertNotEqual(ret, 0)
+
+class TestRemovePurge(TestDvc):
+    def test(self):
+        dvcfile = self.dvc.add(self.FOO)[0].path
+        ret = main(['remove', '--purge', '--force', dvcfile])
+
+        self.assertEqual(ret, 0)
+        self.assertFalse(os.path.exists(self.FOO))
+        self.assertFalse(os.path.exists(dvcfile))
+
+    @patch('dvc.prompt.Prompt.prompt')
+    def test_force(self, mock_prompt):
+        mock_prompt.return_value = False
+
+        dvcfile = self.dvc.add(self.FOO)[0].path
+        ret = main(['remove', '--purge', dvcfile])
+
+        mock_prompt.assert_called()
+        self.assertEqual(ret, 1)
+        self.assertRaises(DvcException)

tests/test_repro.py

@@ -4,7 +4,6 @@ import stat
 import shutil
 import filecmp
 import getpass
-import tempfile
 import posixpath
 from subprocess import Popen, PIPE
 
@@ -14,10 +13,11 @@ from google.cloud import storage as gc
 
 from dvc.main import main
 from dvc.command.repro import CmdRepro
-from dvc.project import Project, ReproductionError, NotDvcFileError
+from dvc.project import Project, ReproductionError
 from dvc.utils import file_md5
 from dvc.remote.local import RemoteLOCAL
-from dvc.stage import Stage
+from dvc.stage import Stage, StageFileDoesNotExistError
+from dvc.system import System
 
 from tests.basic_env import TestDvc
 from tests.test_data_cloud import _should_test_aws, TEST_AWS_REPO_BUCKET
@@ -30,7 +30,10 @@ class TestRepro(TestDvc):
     def setUp(self):
         super(TestRepro, self).setUp()
 
-        self.foo_stage = self.dvc.add(self.FOO)
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.foo_stage = stages[0]
+        self.assertTrue(self.foo_stage is not None)
 
         self.file1 = 'file1'
         self.file1_stage = self.file1 + '.dvc'
@@ -50,7 +53,10 @@ class TestReproFail(TestRepro):
 
 class TestReproDepUnderDir(TestDvc):
     def test(self):
-        self.dir_stage = self.dvc.add(self.DATA_DIR)
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.dir_stage = stages[0]
+        self.assertTrue(self.dir_stage is not None)
 
         sleep()
         
@@ -73,6 +79,47 @@ class TestReproDepUnderDir(TestDvc):
         self.assertTrue(filecmp.cmp(self.file1, self.FOO, shallow=False))
 
 
+class TestReproDepDirWithOutputsUnderIt(TestDvc):
+    def test(self):
+        stages = self.dvc.add(self.DATA)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+
+        sleep()
+
+        stages = self.dvc.add(self.DATA_SUB)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+
+        sleep()
+
+        stage = self.dvc.run(fname='Dvcfile',
+                             deps=[self.DATA, self.DATA_SUB])
+        self.assertTrue(stage is not None)
+
+        sleep()
+
+        file1 = 'file1'
+        file1_stage = file1 + '.dvc'
+        stage = self.dvc.run(fname=file1_stage,
+                             deps=[self.DATA_DIR],
+                             outs=[file1],
+                             cmd='python {} {} {}'.format(self.CODE,
+                                                          self.DATA,
+                                                          file1))
+        self.assertTrue(stage is not None)
+
+        sleep()
+
+        os.unlink(self.DATA)
+        shutil.copyfile(self.FOO, self.DATA)
+
+        sleep()
+
+        stages = self.dvc.reproduce(file1_stage)
+        self.assertEqual(len(stages), 2)
+
+
 class TestReproNoDeps(TestRepro):
     def test(self):
         out = 'out'
@@ -134,28 +181,131 @@ class TestReproDry(TestReproChangedData):
         self.assertTrue(len(stages), 2)
         self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
 
-        ret = main(['repro', '--dry'])
+        ret = main(['repro', '--dry', self.file1_stage])
+        self.assertEqual(ret, 0)
         self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
 
 
-class TestReproChangedDeepData(TestReproChangedData):
+class TestReproUpToDate(TestRepro):
     def test(self):
-        file2 = 'file2'
-        file2_stage = file2 + '.dvc'
-        self.dvc.run(fname=file2_stage,
-                     outs=[file2],
-                     deps=[self.file1, self.CODE],
-                     cmd='python {} {} {}'.format(self.CODE, self.file1, file2))
+        ret = main(['repro', self.file1_stage])
+        self.assertEqual(ret, 0)
 
+
+class TestReproDryNoExec(TestDvc):
+    def test(self):
+        deps = []
+        for d in range(3):
+            idir = 'idir{}'.format(d)
+            odir = 'odir{}'.format(d)
+
+            deps.append('-d')
+            deps.append(odir)
+
+            os.mkdir(idir)
+
+            f = os.path.join(idir, 'file')
+            with open(f, 'w+') as fobj:
+                fobj.write(str(d))
+
+            if os.name == 'nt':
+                cp = 'copy'
+            else:
+                cp = 'cp'
+
+            ret = main(['run',
+                        '--no-exec',
+                        '-d', idir,
+                        '-o', odir,
+                        "python -c 'import shutil; "
+                        "shutil.copytree(\"{}\", \"{}\")'".format(idir, odir)])
+            self.assertEqual(ret, 0)
+
+        ret = main(['run', '--no-exec', '-f', 'Dvcfile'] + deps)
+        self.assertEqual(ret, 0)
+
+        ret = main(['repro', '--dry'])
+        self.assertEqual(ret, 0)
+
+
+class TestReproChangedDeepData(TestReproChangedData):
+    def setUp(self):
+        super(TestReproChangedDeepData, self).setUp()
+
+        self.file2 = 'file2'
+        self.file2_stage = self.file2 + '.dvc'
+        self.dvc.run(fname=self.file2_stage,
+                     outs=[self.file2],
+                     deps=[self.file1, self.CODE],
+                     cmd='python {} {} {}'.format(self.CODE,
+                                                  self.file1,
+                                                  self.file2))
+
+    def test(self):
         self.swap_foo_with_bar()
 
-        stages = self.dvc.reproduce(file2_stage)
+        stages = self.dvc.reproduce(self.file2_stage)
 
         self.assertTrue(filecmp.cmp(self.file1, self.BAR, shallow=False))
-        self.assertTrue(filecmp.cmp(file2, self.BAR, shallow=False))
+        self.assertTrue(filecmp.cmp(self.file2, self.BAR, shallow=False))
         self.assertEqual(len(stages), 3)
 
 
+class TestReproPipeline(TestReproChangedDeepData):
+    def test(self):
+        stages = self.dvc.reproduce(self.file1_stage,
+                                    force=True,
+                                    pipeline=True)
+        self.assertEqual(len(stages), 3)
+
+    def test_cli(self):
+        ret = main(['repro', '--pipeline', '-f', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+
+class TestReproPipelines(TestDvc):
+    def setUp(self):
+        super(TestReproPipelines, self).setUp()
+
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.foo_stage = stages[0]
+        self.assertTrue(self.foo_stage is not None)
+
+        stages = self.dvc.add(self.BAR)
+        self.assertEqual(len(stages), 1)
+        self.bar_stage = stages[0]
+        self.assertTrue(self.bar_stage is not None)
+
+        self.file1 = 'file1'
+        self.file1_stage = self.file1 + '.dvc'
+        self.dvc.run(fname=self.file1_stage,
+                     outs=[self.file1],
+                     deps=[self.FOO, self.CODE],
+                     cmd='python {} {} {}'.format(self.CODE, self.FOO, self.file1))
+
+        self.file2 = 'file2'
+        self.file2_stage = self.file2 + '.dvc'
+        self.dvc.run(fname=self.file2_stage,
+                     outs=[self.file2],
+                     deps=[self.BAR, self.CODE],
+                     cmd='python {} {} {}'.format(self.CODE, self.BAR, self.file2))
+
+
+    def test(self):
+        stages = self.dvc.reproduce(all_pipelines=True, force=True)
+        self.assertEqual(len(stages), 4)
+        names = [stage.relpath for stage in stages]
+        self.assertTrue(self.foo_stage.relpath in names)
+        self.assertTrue(self.bar_stage.relpath in names)
+        self.assertTrue(self.file1_stage in names)
+        self.assertTrue(self.file2_stage in names)
+
+    def test_cli(self):
+        ret = main(['repro', '-f', '-P'])
+        self.assertEqual(ret, 0)
+
+
 class TestReproLocked(TestReproChangedData):
     def test(self):
         file2 = 'file2'
@@ -180,7 +330,7 @@ class TestReproLocked(TestReproChangedData):
         self.assertEqual(len(stages), 3)
 
     def test_non_existing(self):
-        with self.assertRaises(NotDvcFileError):
+        with self.assertRaises(StageFileDoesNotExistError):
             self.dvc.lock_stage('non-existing-stage')
 
         ret = main(['lock', 'non-existing-stage'])
@@ -243,6 +393,55 @@ class TestReproChangedDir(TestDvc):
         self.assertEqual(len(stages), 1)
 
 
+class TestReproChangedDirData(TestDvc):
+    def test(self):
+        dir_name = 'dir'
+        dir_code = 'dir_code.py'
+
+        sleep()
+
+        with open(dir_code, 'w+') as fd:
+            fd.write("import os; import sys; import shutil; shutil.copytree(sys.argv[1], sys.argv[2])")
+
+        sleep()
+            
+        stage = self.dvc.run(outs=[dir_name],
+                             deps=[self.DATA_DIR, dir_code],
+                             cmd="python {} {} {}".format(dir_code,
+                                                          self.DATA_DIR,
+                                                          dir_name))
+        self.assertTrue(stage is not None)
+
+        stages = self.dvc.reproduce(stage.path)
+        self.assertEqual(len(stages), 0)
+
+        sleep()
+
+        with open(self.DATA_SUB, 'a') as fd:
+            fd.write('add')
+
+        sleep()
+        
+        stages = self.dvc.reproduce(stage.path)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+        sleep()
+        
+        # Check that dvc indeed registers changed output dir
+        shutil.move(self.BAR, dir_name)
+        sleep()
+        stages = self.dvc.reproduce(stage.path)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+
+        # Check that dvc registers mtime change for the directory.
+        System.hardlink(self.DATA_SUB, self.DATA_SUB + '.lnk')
+        sleep()
+        stages = self.dvc.reproduce(stage.path)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+
+
 class TestReproMissingMd5InStageFile(TestRepro):
     def test(self):
         with open(self.file1_stage, 'r') as fd:
@@ -373,6 +572,8 @@ class TestReproExternalBase(TestDvc):
 
         sleep()
 
+        self.dvc.status()
+
         stages = self.dvc.reproduce(import_stage.path)
         self.assertEqual(len(stages), 1)
         self.assertTrue(os.path.exists('import'))
@@ -384,7 +585,7 @@ class TestReproExternalBase(TestDvc):
         self.dvc.gc()
 
         self.dvc.remove(cmd_stage.path, outs_only=True)
-        self.dvc.checkout(cmd_stage.path)
+        self.dvc.checkout(cmd_stage.path, force=True)
 
 
 class TestReproExternalS3(TestReproExternalBase):
@@ -494,7 +695,7 @@ class TestReproExternalSSH(TestReproExternalBase):
     @property
     def bucket(self):
         if not self._dir:
-            self._dir = tempfile.mkdtemp()
+            self._dir = TestDvc.mkdtemp()
         return '{}@127.0.0.1:{}'.format(getpass.getuser(), self._dir)
 
     def cmd(self, i, o):
@@ -541,7 +742,7 @@ class TestReproExternalSSH(TestReproExternalBase):
 class TestReproExternalLOCAL(TestReproExternalBase):
     def setUp(self):
         super(TestReproExternalLOCAL, self).setUp()
-        self.tmpdir = tempfile.mkdtemp()
+        self.tmpdir = TestDvc.mkdtemp()
 
     def should_test(self):
         return True
@@ -603,3 +804,9 @@ class TestReproShell(TestDvc):
 
         with open(fname, 'r') as fd:
             self.assertEqual(os.getenv('SHELL'), fd.read().strip())
+
+class TestReproNoSCM(TestRepro):
+    def test(self):
+        shutil.rmtree(self.dvc.scm.dir)
+        ret = main(['repro', self.file1_stage])
+        self.assertEqual(ret, 0)

tests/test_run.py

@@ -3,8 +3,10 @@ import filecmp
 
 from dvc.main import main
 from dvc.utils import file_md5
-from dvc.stage import Stage
+from dvc.stage import Stage, StageFileBadNameError, MissingDep
+from dvc.stage import StageBadCwdError
 from dvc.command.run import CmdRun
+from dvc.exceptions import OutputDuplicationError, CircularDependencyError
 
 from tests.basic_env import TestDvc
 
@@ -15,7 +17,7 @@ class TestRun(TestDvc):
         deps = [self.FOO, self.CODE]
         outs = [os.path.join(self.dvc.root_dir, 'out')]
         outs_no_cache = []
-        fname = os.path.join(self.dvc.root_dir, 'out.dvc')
+        fname = 'out.dvc'
         cwd = os.curdir
 
         self.dvc.add(self.FOO)
@@ -35,6 +37,14 @@ class TestRun(TestDvc):
         self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])  
         self.assertTrue(stage.path, fname)
 
+        with self.assertRaises(OutputDuplicationError):
+            stage = self.dvc.run(cmd=cmd,
+                                 deps=deps,
+                                 outs=outs,
+                                 outs_no_cache=outs_no_cache,
+                                 fname='duplicate' + fname,
+                                 cwd=cwd)
+
 
 class TestRunEmpty(TestDvc):
     def test(self):
@@ -46,6 +56,37 @@ class TestRunEmpty(TestDvc):
                      cwd=os.curdir)
 
 
+class TestRunMissingDep(TestDvc):
+    def test(self):
+        with self.assertRaises(MissingDep):
+            self.dvc.run(cmd='',
+                         deps=['non-existing-dep'],
+                         outs=[],
+                         outs_no_cache=[],
+                         fname='empty.dvc',
+                         cwd=os.curdir)
+
+
+class TestRunBadStageFilename(TestDvc):
+    def test(self):
+        with self.assertRaises(StageFileBadNameError):
+            self.dvc.run(cmd='',
+                         deps=[],
+                         outs=[],
+                         outs_no_cache=[],
+                         fname='empty',
+                         cwd=os.curdir)
+
+        with self.assertRaises(StageFileBadNameError):
+            self.dvc.run(cmd='',
+                         deps=[],
+                         outs=[],
+                         outs_no_cache=[],
+                         fname=os.path.join(self.DATA_DIR, 'empty.dvc'),
+                         cwd=os.curdir)
+
+
+
 class TestRunNoExec(TestDvc):
     def test(self):
         self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, 'out'),
@@ -53,6 +94,36 @@ class TestRunNoExec(TestDvc):
         self.assertFalse(os.path.exists('out'))
 
 
+class TestRunCircularDependency(TestDvc):
+    def test(self):
+        with self.assertRaises(CircularDependencyError):
+            self.dvc.run(cmd='',
+                         deps=[self.FOO],
+                         outs=[self.FOO],
+                         fname='circular-dependency.dvc')
+
+    def test_outs_no_cache(self):
+        with self.assertRaises(CircularDependencyError):
+            self.dvc.run(cmd='',
+                         deps=[self.FOO],
+                         outs_no_cache=[self.FOO],
+                         fname='circular-dependency.dvc')
+
+    def test_non_normalized_paths(self):
+        with self.assertRaises(CircularDependencyError):
+            self.dvc.run(cmd='',
+                         deps=['./foo'],
+                         outs=['foo'],
+                         fname='circular-dependency.dvc')
+
+
+class TestRunBadCwd(TestDvc):
+    def test(self):
+        with self.assertRaises(StageBadCwdError):
+            self.dvc.run(cmd='',
+                         cwd=self.mkdtemp())
+
+
 class TestCmdRun(TestDvc):
     def test_run(self):
         ret = main(['run',

tests/test_scm.py

@@ -4,7 +4,7 @@ from git import Repo
 
 from dvc.scm import SCM, Base, Git
 
-from tests.basic_env import TestDir, TestGit
+from tests.basic_env import TestDir, TestGit, TestGitSubmodule
 
 
 class TestSCM(TestDir):
@@ -27,21 +27,37 @@ class TestSCMGit(TestGit):
         self.assertTrue('foo' in self.git.git.ls_files())
 
 
+class TestSCMGitSubmodule(TestGitSubmodule):
+    def test_git_submodule(self):
+        self.assertIsInstance(SCM(os.curdir), Git)
+
+    def test_is_submodule(self):
+        self.assertTrue(Git.is_submodule(os.curdir))
+
+    def test_commit_in_submodule(self):
+        G = Git(self._root_dir)
+        G.add(['foo'])
+        G.commit('add')
+        self.assertTrue('foo' in self.git.git.ls_files())
+
+
 class TestIgnore(TestGit):
     def _count_gitignore(self):
         with open(Git.GITIGNORE, 'r') as fd:
             lines = fd.readlines()
-            return len(list(filter(lambda x: x.strip() == 'foo', lines)))
+            return len(list(filter(lambda x: x.strip() == '/' + self.FOO,
+                                   lines)))
 
     def test(self):
         git = Git(self._root_dir)
+        foo = os.path.join(self._root_dir, self.FOO)
 
-        git.ignore('foo')
+        git.ignore(foo)
         self.assertTrue(os.path.isfile(Git.GITIGNORE))
         self.assertEqual(self._count_gitignore(), 1)
 
-        git.ignore('foo')
+        git.ignore(foo)
         self.assertEqual(self._count_gitignore(), 1)
 
-        git.ignore_remove('foo')
+        git.ignore_remove(foo)
         self.assertEqual(self._count_gitignore(), 0)

tests/test_state.py

@@ -3,8 +3,9 @@ import stat
 import time
 
 from dvc.system import System
-from dvc.state import State, StateEntry
+from dvc.state import State
 from dvc.utils import file_md5
+from dvc.main import main
 
 from tests.basic_env import TestDvc
 
@@ -16,31 +17,43 @@ class TestState(TestDvc):
         mtime = os.path.getmtime(path)
         inode = System.inode(path)
 
-        state = State(self.dvc)
+        state = State(self.dvc, self.dvc.config._config)
 
-        entry_md5 = state.update(path)
-        self.assertEqual(entry_md5, md5)
+        with state:
+            entry_md5 = state.update(path)
+            self.assertEqual(entry_md5, md5)
 
-        # Sleep some time to simulate realistic behavior.
-        # Some filesystems have a bad date resolution for
-        # mtime(i.e. 1sec for HFS) that cause problems with
-        # our 'state' system not being able to distinguish
-        # files that were modified within that delta.
-        time.sleep(1)        
+            # Sleep some time to simulate realistic behavior.
+            # Some filesystems have a bad date resolution for
+            # mtime(i.e. 1sec for HFS) that cause problems with
+            # our 'state' system not being able to distinguish
+            # files that were modified within that delta.
+            time.sleep(1)        
 
-        os.unlink(path)
-        with open(path, 'w+') as fd:
-            fd.write('1')
+            os.unlink(path)
+            with open(path, 'w+') as fd:
+                fd.write('1')
 
-        md5 = file_md5(path)[0]
-        mtime = os.path.getmtime(path)
-        inode = System.inode(path)
+            md5 = file_md5(path)[0]
+            mtime = os.path.getmtime(path)
+            inode = System.inode(path)
 
-        entry_md5 = state.update(path)
-        self.assertEqual(entry_md5, md5)
+            entry_md5 = state.update(path)
+            self.assertEqual(entry_md5, md5)
 
-        # Reload state db to make sure that it stays the same
-        old_db = state._db
-        state = State(self.dvc)
-        new_db = state._db
-        self.assertEqual(old_db, new_db)
+
+class TestStateOverflow(TestDvc):
+    def test(self):
+        # NOTE: trying to add more entries than state can handle,
+        # to see if it will clean up and vacuum successfully
+        ret = main(['config', 'state.row_limit', '10'])
+        self.assertEqual(ret, 0)
+
+        dname = 'dir'
+        os.mkdir(dname)
+        for i in range(20):
+            with open(os.path.join(dname, str(i)), 'w+') as fobj:
+                fobj.write(str(i))
+
+        ret = main(['add', 'dir'])
+        self.assertEqual(ret, 0)

tests/test_unprotect.py

@@ -0,0 +1,31 @@
+import os
+import stat
+
+from dvc.main import main
+from dvc.project import Project
+
+from tests.basic_env import TestDvc
+
+
+class TestUnprotect(TestDvc):
+    def test(self):
+        ret = main(['config', 'cache.protected', 'true'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['add', self.FOO])
+        self.assertEqual(ret, 0)
+
+        self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD|
+                                                         stat.S_IRGRP|
+                                                         stat.S_IROTH), 0)
+        self.assertEqual(os.stat(self.FOO).st_mode & (stat.S_IWRITE|
+                                                      stat.S_IWGRP|
+                                                      stat.S_IWOTH), 0)
+
+        ret = main(['unprotect', self.FOO])
+        self.assertEqual(ret, 0)
+
+        self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD|
+                                                         stat.S_IRGRP|
+                                                         stat.S_IROTH|
+                                                         stat.S_IWRITE), 0)