dvc: bump to 0.19.12

state file fixes

.travis.yml

@@ -16,18 +16,24 @@ python:
 # version, that causes fpm to crash.
 matrix:
   include:
+  # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
+  - python: "3.7"
+    dist: xenial
+    sudo: true
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
     - PYTHON_VER=2.7.10
     - PYENV_ROOT="$HOME/.pyenv"
     - PATH="$PYENV_ROOT/bin:$PATH"
+    - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
+    - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
@@ -36,23 +42,20 @@ matrix:
     - PATH="$PYENV_ROOT/bin:$PATH"
   - if: type = cron OR tag is present
     os: osx
-    osx_image: xcode9.3
+    osx_image: xcode7.3
     language: ruby
     rvm: 2.4.0
     env:
     - PYTHON_VER=3.5.0
     - PYENV_ROOT="$HOME/.pyenv"
     - PATH="$PYENV_ROOT/bin:$PATH"
-  # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
-  - python: "3.7"
-    dist: xenial
-    sudo: true
 before_install:
   - bash ./scripts/ci/before_install.sh
   - if  [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       eval "$(pyenv init -)";
       PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
       pyenv shell "$PYTHON_VER";
+      brew install openssl;
       brew cask install google-cloud-sdk;
     fi
   - which python

README.rst

@@ -4,10 +4,8 @@
 
 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_
-• `Discuss <https://discuss.dvc.org/>`_
-• `Blog <https://blog.dataversioncontrol.com/>`_
 • `Twitter <https://twitter.com/iterativeai>`_
-• `Slack <https://dvc-org.slack.com>`_
+• `Chat (Community & Support) <https://dvc.org/chat>`_
 • `Tutorial <https://dvc.org/doc/tutorial>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
@@ -29,10 +27,12 @@
 
 |
 
-**Data Science Version Control** or **DVC** is an **open-source** tool for data science projects.
-With a simple and flexible Git-like architecture and interface it helps data scientists:
+**Data Science Version Control** or **DVC** is an **open-source** tool for data science and 
+machine learning projects. With a simple and flexible Git-like architecture and interface it
+helps data scientists:
 
-#. manage code together with data files and models;
+#. manage **machine learning models** - versioning, including data sets and transformations (scripts) that were
+   used to generate models;
 
 #. make projects **reproducible**;
 
@@ -112,6 +112,17 @@ or:
 
    $ brew cask install iterative/homebrew-dvc/dvc
 
+AUR
+---
+
+*Unofficial package*, any inquiries regarding the AUR package,
+`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
+
+.. code-block:: bash
+
+  # https://aur.archlinux.org/packages/dvc
+  $ yay -S dvc
+
 Related technologies
 ====================
 

dvc/__init__.py

@@ -7,7 +7,7 @@ import os
 import warnings
 
 
-VERSION_BASE = '0.18.10'
+VERSION_BASE = '0.19.12'
 __version__ = VERSION_BASE
 
 PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
@@ -33,13 +33,13 @@ if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
             with open(VERSIONPATH, 'w+') as fd:
                 fd.write('# AUTOGENERATED by dvc/__init__.py\n')
                 fd.write('version = "{}"\n'.format(__version__))
-        except Exception:
+        except Exception:  # pragma: no cover
             pass
-    else:
+    else:  # pragma: no cover
         # Remove version.py so that it doesn't get into the release
         if os.path.exists(VERSIONPATH):
             os.unlink(VERSIONPATH)
-else:
+else:  # pragma: no cover
     # dvc was installed with pip or something. Hopefully we have our
     # auto-generated version.py to help us provide a true version
     try:

dvc/cli.py

@@ -3,7 +3,6 @@ from __future__ import print_function
 import os
 import sys
 import argparse
-from multiprocessing import cpu_count
 
 from dvc.command.init import CmdInit
 from dvc.command.destroy import CmdDestroy
@@ -47,7 +46,7 @@ class DvcParser(argparse.ArgumentParser):
         sys.exit(2)
 
 
-class VersionAction(argparse.Action):
+class VersionAction(argparse.Action):  # pragma: no cover
     def __call__(self, parser, namespace, values, option_string=None):
         print(VERSION)
         sys.exit(0)
@@ -161,7 +160,15 @@ def parse_args(argv=None):
                         help=IMPORT_HELP)
     import_parser.add_argument(
                         'url',
-                        help='URL.')
+                        help="URL. Supported urls: "
+                             "'/path/to/file', "
+                             "'C:\\\\path\\to\\file', "
+                             "'s3://bucket/path/to/file', "
+                             "'gs://bucket/path/to/file', "
+                             "'hdfs://example.com/path/to/file', "
+                             "'ssh://example.com:/path/to/file', "
+                             "'remote://myremote/path/to/file'(see "
+                             "`dvc remote` commands). ")
     import_parser.add_argument(
                         'out',
                         help='Output.')
@@ -257,7 +264,7 @@ def parse_args(argv=None):
                         '-j',
                         '--jobs',
                         type=int,
-                        default=8*cpu_count(),
+                        default=None,
                         help='Number of jobs to run simultaneously.')
     parent_cache_parser.add_argument(
                         '--show-checksums',
@@ -287,6 +294,19 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Fetch cache for all branches.')
+    pull_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all tags.')
+    pull_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all dependencies of the '
+                             'specified target.')
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
@@ -306,6 +326,19 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Push cache for all branches.')
+    push_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Push cache for all tags.')
+    push_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Push cache for all dependencies of the '
+                             'specified target.')
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
@@ -325,6 +358,19 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Fetch cache for all branches.')
+    fetch_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all tags.')
+    fetch_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all dependencies of the '
+                             'specified target.')
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
@@ -345,6 +391,28 @@ def parse_args(argv=None):
                         '-r',
                         '--remote',
                         help='Remote repository to compare local cache to.')
+    status_parser.add_argument(
+                        '-a',
+                        '--all-branches',
+                        action='store_true',
+                        default=False,
+                        help='Show status of a local cache compared to a '
+                             'remote repository for all branches.')
+    status_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Show status of a local cache compared to a '
+                             'remote repository for all tags.')
+    status_parser.add_argument(
+                        '-d',
+                        '--with-deps',
+                        action='store_true',
+                        default=False,
+                        help='Show status of a local cache compared to a '
+                             'remote repository for all dependencies of the '
+                             'specified target.')
     status_parser.set_defaults(func=CmdDataStatus)
 
     # Repro
@@ -396,6 +464,19 @@ def parse_args(argv=None):
                         default=False,
                         help='Ask for confirmation before reproducing each '
                              'stage.')
+    repro_parser.add_argument(
+                        '-p',
+                        '--pipeline',
+                        action='store_true',
+                        default=False,
+                        help='Reproduce the whole pipeline that the '
+                             'specified stage file belongs to.')
+    repro_parser.add_argument(
+                        '-P',
+                        '--all-pipelines',
+                        action='store_true',
+                        default=False,
+                        help='Reproduce all pipelines in the project.')
     repro_parser.set_defaults(func=CmdRepro)
 
     # Remove
@@ -411,7 +492,7 @@ def parse_args(argv=None):
                         '--outs',
                         action='store_true',
                         default=True,
-                        help='Only remove DVC file outputs.')
+                        help='Only remove DVC file outputs.(default)')
     remove_parser_group.add_argument(
                         '-p',
                         '--purge',
@@ -452,6 +533,12 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Collect garbage for all branches.')
+    gc_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Collect garbage for all tags.')
     gc_parser.add_argument(
                         '-c',
                         '--cloud',
@@ -462,6 +549,12 @@ def parse_args(argv=None):
                         '-r',
                         '--remote',
                         help='Remote repository to collect garbage in.')
+    gc_parser.add_argument(
+                        '-f',
+                        '--force',
+                        action='store_true',
+                        default=False,
+                        help='Force garbage collection.')
     gc_parser.set_defaults(func=CmdGC)
 
     # Config
@@ -632,19 +725,27 @@ def parse_args(argv=None):
                         help='TSV path \'row,column\'(e.g. \'1,2\').')
     metrics_show_group.add_argument(
                         '--htsv-path',
-                        help='Headed TSV path \'row,column\'(e.g. \'Name,3\'.')
+                        help='Headed TSV path \'row,column\''
+                             '(e.g. \'Name,3\').')
     metrics_show_group.add_argument(
                         '--csv-path',
                         help='CSV path \'row,column\'(e.g. \'1,2\').')
     metrics_show_group.add_argument(
                         '--hcsv-path',
-                        help='Headed CSV path \'row,column\'(e.g. \'Name,3\'.')
+                        help='Headed CSV path \'row,column\''
+                             '(e.g. \'Name,3\').')
     metrics_show_parser.add_argument(
                         '-a',
                         '--all-branches',
                         action='store_true',
                         default=False,
                         help='Show metrics for all branches.')
+    metrics_show_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Show metrics for all tags.')
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
     METRICS_ADD_HELP = 'Add metrics.'
@@ -780,6 +881,10 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Output DAG as ASCII.')
+    pipeline_show_parser.add_argument(
+                        '--dot',
+                        help='Write DAG in .dot format.'
+    )
     pipeline_show_parser.add_argument(
                         'targets',
                         nargs='*',
@@ -791,7 +896,9 @@ def parse_args(argv=None):
     if (issubclass(args.func, CmdRepro)
         or issubclass(args.func, CmdPipelineShow)) \
        and hasattr(args, 'targets') \
-       and len(args.targets) == 0:
+       and len(args.targets) == 0 \
+       and not (hasattr(args, 'all_pipelines')
+       and args.all_pipelines):  # pragma: no cover
         if hasattr(args, 'cwd'):
             cwd = args.cwd
         else:
@@ -805,4 +912,12 @@ def parse_args(argv=None):
                 pipeline_show_parser.error(msg)
         args.targets = ['Dvcfile']
 
+    if issubclass(args.func, CmdRun) \
+       and len(args.deps) == 0 \
+       and len(args.outs) == 0 \
+       and len(args.outs_no_cache) == 0 \
+       and len(args.command) == 0:  # pragma: no cover
+        run_parser.error("Too few arguments. Specify at least one: "
+                         "'-d', '-o', '-O', 'command'.")
+
     return args

dvc/command/data_sync.py

@@ -23,7 +23,9 @@ class CmdDataPull(CmdDataBase):
                               jobs=self.args.jobs,
                               remote=self.args.remote,
                               show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches)
+                              all_branches=self.args.all_branches,
+                              all_tags=self.args.all_tags,
+                              with_deps=self.args.with_deps)
         except Exception as exc:
             self.project.logger.error('Failed to pull data from the cloud',
                                       exc)
@@ -38,7 +40,9 @@ class CmdDataPush(CmdDataBase):
                               jobs=self.args.jobs,
                               remote=self.args.remote,
                               show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches)
+                              all_branches=self.args.all_branches,
+                              all_tags=self.args.all_tags,
+                              with_deps=self.args.with_deps)
         except Exception as exc:
             self.project.logger.error('Failed to push data to the cloud', exc)
             return 1
@@ -52,7 +56,9 @@ class CmdDataFetch(CmdDataBase):
                                jobs=self.args.jobs,
                                remote=self.args.remote,
                                show_checksums=self.args.show_checksums,
-                               all_branches=self.args.all_branches)
+                               all_branches=self.args.all_branches,
+                               all_tags=self.args.all_tags,
+                               with_deps=self.args.with_deps)
         except Exception as exc:
             self.project.logger.error('Failed to fetch data from the cloud',
                                       exc)

dvc/command/gc.py

@@ -3,7 +3,24 @@ from dvc.command.base import CmdBase
 
 class CmdGC(CmdBase):
     def run(self):
+        msg = 'This will remove all cache except the cache that is used in '
+        if not self.args.all_branches and not self.args.all_tags:
+            msg += 'the current git branch.'
+        elif self.args.all_branches and not self.args.all_tags:
+            msg += 'all git branches.'
+        elif not self.args.all_branches and self.args.all_tags:
+            msg += 'all git tags.'
+        else:
+            msg += 'all git branches and all git tags.'
+
+        self.project.logger.warn(msg)
+
+        msg = 'Are you sure you want to proceed?'
+        if not self.args.force and not self.project.prompt.prompt(msg):
+            return 1
+
         self.project.gc(all_branches=self.args.all_branches,
+                        all_tags=self.args.all_tags,
                         cloud=self.args.cloud,
                         remote=self.args.remote)
         return 0

dvc/command/metrics.py

@@ -28,7 +28,8 @@ class CmdMetricsShow(CmdBase):
             self.project.metrics_show(self.args.path,
                                       typ=typ,
                                       xpath=xpath,
-                                      all_branches=self.args.all_branches)
+                                      all_branches=self.args.all_branches,
+                                      all_tags=self.args.all_tags)
         except DvcException as exc:
             self.project.logger.error('Failed to show metrics', exc)
             return 1

dvc/command/pipeline.py

@@ -24,7 +24,7 @@ class CmdPipelineShow(CmdBase):
             else:
                 self.project.logger.info(n)
 
-    def _show_ascii(self, target, commands, outs):
+    def __build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
 
@@ -51,9 +51,6 @@ class CmdPipelineShow(CmdBase):
             else:
                 nodes.append(stage.relpath)
 
-        if len(nodes) == 0:
-            return
-
         edges = []
         for e in G.edges():
             from_stage = stages[e[0]]
@@ -70,9 +67,28 @@ class CmdPipelineShow(CmdBase):
             else:
                 edges.append((from_stage.relpath, to_stage.relpath))
 
+        return nodes, edges
+
+    def _show_ascii(self, target, commands, outs):
+        nodes, edges = self.__build_graph(target, commands, outs)
+
+        if not nodes:
+            return
+
         d = Dagascii(nodes, edges)
         d.draw()
 
+    def __write_dot(self, target, commands, outs, filename):
+        import networkx
+        from networkx.drawing.nx_pydot import write_dot
+
+        _, edges = self.__build_graph(target, commands, outs)
+        edges = [edge[::-1] for edge in edges]
+
+        simple_g = networkx.DiGraph()
+        simple_g.add_edges_from(edges)
+        write_dot(simple_g, filename)
+
     def run(self, unlock=False):
         for target in self.args.targets:
             try:
@@ -80,6 +96,11 @@ class CmdPipelineShow(CmdBase):
                     self._show_ascii(target,
                                      self.args.commands,
                                      self.args.outs)
+                elif self.args.dot:
+                    self.__write_dot(target,
+                                     self.args.commands,
+                                     self.args.outs,
+                                     self.args.dot)
                 else:
                     self._show(target,
                                self.args.commands,

dvc/command/repro.py

@@ -20,7 +20,9 @@ class CmdRepro(CmdBase):
                                        recursive=recursive,
                                        force=self.args.force,
                                        dry=self.args.dry,
-                                       interactive=self.args.interactive)
+                                       interactive=self.args.interactive,
+                                       pipeline=self.args.pipeline,
+                                       all_pipelines=self.args.all_pipelines)
 
                 if len(stages) == 0:
                     self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG)

dvc/command/status.py

@@ -29,7 +29,10 @@ class CmdDataStatus(CmdDataBase):
                                      jobs=self.args.jobs,
                                      cloud=self.args.cloud,
                                      show_checksums=self.args.show_checksums,
-                                     remote=self.args.remote)
+                                     remote=self.args.remote,
+                                     all_branches=self.args.all_branches,
+                                     all_tags=self.args.all_tags,
+                                     with_deps=self.args.with_deps)
             if st:
                 self._show(st, indent)
             else:

dvc/config.py

@@ -45,6 +45,14 @@ def to_bool(val):
     return val.lower() == 'true'
 
 
+def is_whole(val):
+    return int(val) >= 0
+
+
+def is_percent(val):
+    return int(val) >= 0 and int(val) <= 100
+
+
 class Config(object):
     CONFIG = 'config'
     CONFIG_LOCAL = 'config.local'
@@ -134,6 +142,9 @@ class Config(object):
     SECTION_REMOTE_USER = 'user'
     SECTION_REMOTE_PORT = 'port'
     SECTION_REMOTE_KEY_FILE = 'keyfile'
+    SECTION_REMOTE_TIMEOUT = 'timeout'
+    SECTION_REMOTE_PASSWORD = 'password'
+    SECTION_REMOTE_ASK_PASSWORD = 'ask_password'
     SECTION_REMOTE_SCHEMA = {
         SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
         Optional(SECTION_AWS_REGION): str,
@@ -145,12 +156,24 @@ class Config(object):
         Optional(SECTION_REMOTE_USER): str,
         Optional(SECTION_REMOTE_PORT): Use(int),
         Optional(SECTION_REMOTE_KEY_FILE): str,
+        Optional(SECTION_REMOTE_TIMEOUT): Use(int),
+        Optional(SECTION_REMOTE_PASSWORD): str,
+        Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
+    }
+
+    SECTION_STATE = 'state'
+    SECTION_STATE_ROW_LIMIT = 'row_limit'
+    SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota'
+    SECTION_STATE_SCHEMA = {
+        Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
+        Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
     }
 
     SCHEMA = {
         Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
         Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
         Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
+        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
 
         # backward compatibility
         Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,

dvc/dagascii.py

@@ -1,13 +1,170 @@
+import sys
 import math
-
-from asciicanvas.asciicanvas import AsciiCanvas
-from asciicanvas.style import Style
+import select
 
 from grandalf.graphs import Vertex, Edge, Graph
 from grandalf.layouts import SugiyamaLayout
 from grandalf.routing import route_with_lines, EdgeViewer
 
 
+class AsciiCanvas(object):
+    def __init__(self, cols, lines):
+        assert cols > 1
+        assert lines > 1
+
+        self.cols = cols
+        self.lines = lines
+
+        self.canvas = [[' '] * cols for l in range(lines)]
+
+    def draw(self):
+        if sys.stdout.isatty():  # pragma: no cover
+            from asciimatics.screen import Screen
+            Screen.wrapper(self._do_draw)
+        else:
+            for line in self.canvas:
+                print(''.join(line))
+
+    def _do_draw(self, screen):  # pragma: no cover
+        from asciimatics.event import KeyboardEvent
+
+        offset_x = 0
+        offset_y = 0
+        smaxrow, smaxcol = screen.dimensions
+        assert smaxrow > 1
+        assert smaxcol > 1
+        smaxrow -= 1
+        smaxcol -= 1
+
+        if self.lines + 1 > smaxrow:
+            max_y = self.lines + 1 - smaxrow
+        else:
+            max_y = 0
+
+        if self.cols + 1 > smaxcol:
+            max_x = self.cols + 1 - smaxcol
+        else:
+            max_x = 0
+
+        while True:
+            for y in range(smaxrow + 1):
+                y_index = offset_y + y
+                line = []
+                for x in range(smaxcol + 1):
+                    x_index = offset_x + x
+                    if len(self.canvas) > y_index \
+                       and len(self.canvas[y_index]) > x_index:
+                            line.append(self.canvas[y_index][x_index])
+                    else:
+                        line.append(' ')
+                assert len(line) == (smaxcol + 1)
+                screen.print_at(''.join(line), 0, y)
+
+            screen.refresh()
+
+            # NOTE: get_event() doesn't block by itself,
+            # so we have to do the blocking ourselves.
+            select.select([sys.stdin], [], [], None)
+
+            event = screen.get_event()
+            if not isinstance(event, KeyboardEvent):
+                continue
+
+            k = event.key_code
+            if k == screen.KEY_DOWN or k == ord('s'):
+                offset_y += 1
+            elif k == screen.KEY_PAGE_DOWN or k == ord('S'):
+                offset_y += smaxrow
+            elif k == screen.KEY_UP or k == ord('w'):
+                offset_y -= 1
+            elif k == screen.KEY_PAGE_UP or k == ord('W'):
+                offset_y -= smaxrow
+            elif k == screen.KEY_RIGHT or k == ord('d'):
+                offset_x += 1
+            elif k == ord('D'):
+                offset_x += smaxcol
+            elif k == screen.KEY_LEFT or k == ord('a'):
+                offset_x -= 1
+            elif k == ord('A'):
+                offset_x -= smaxcol
+            elif k == ord('q') or k == ord('Q'):
+                break
+
+            if offset_y > max_y:
+                offset_y = max_y
+            elif offset_y < 0:
+                offset_y = 0
+
+            if offset_x > max_x:
+                offset_x = max_x
+            elif offset_x < 0:
+                offset_x = 0
+
+    def point(self, x, y, char):
+        assert x >= 0
+        assert x < self.cols
+        assert y >= 0
+        assert y < self.lines
+
+        self.canvas[y][x] = char
+
+    def line(self, x0, y0, x1, y1, char):
+        if x0 > x1:
+            x1, x0 = x0, x1
+            y1, y0 = y0, y1
+
+        dx = x1 - x0
+        dy = y1 - y0
+
+        if dx == 0 and dy == 0:
+            self.point(x0, y0, char)
+        elif abs(dx) >= abs(dy):
+            for x in range(x0, x1 + 1):
+                if dx == 0:
+                    y = y0
+                else:
+                    y = y0 + int(round((x - x0) * dy / float((dx))))
+                self.point(x, y, char)
+        elif y0 < y1:
+            for y in range(y0, y1 + 1):
+                if dy == 0:
+                    x = x0
+                else:
+                    x = x0 + int(round((y - y0) * dx / float((dy))))
+                self.point(x, y, char)
+        else:
+            for y in range(y1, y0 + 1):
+                if dy == 0:
+                    x = x0
+                else:
+                    x = x1 + int(round((y - y1) * dx / float((dy))))
+                self.point(x, y, char)
+
+    def text(self, x, y, text):
+        for i, c in enumerate(text):
+            self.point(x + i, y, c)
+
+    def box(self, x0, y0, w, h):
+        assert w > 1
+        assert h > 1
+
+        w -= 1
+        h -= 1
+
+        for x in range(x0, x0 + w):
+            self.point(x, y0, '-')
+            self.point(x, y0 + h, '-')
+
+        for y in range(y0, y0 + h):
+            self.point(x0, y, '|')
+            self.point(x0 + w, y, '|')
+
+        self.point(x0, y0, '+')
+        self.point(x0 + w, y0, '+')
+        self.point(x0, y0 + h, '+')
+        self.point(x0 + w, y0 + h, '+')
+
+
 class Dagascii(object):
     def __init__(self, vertexes, edges):
         #
@@ -100,22 +257,22 @@ class Dagascii(object):
                 assert end_x >= 0
                 assert end_y >= 0
 
-                canvas.add_line(start_x, start_y,
-                                end_x, end_y,
-                                style=Style('*'))
+                canvas.line(start_x, start_y,
+                            end_x, end_y,
+                            '*')
 
         for v in self.sug.g.sV:
             # NOTE: moving boxes w/2 to the left
             x = v.view.xy[0] - v.view.w/2.0
             y = v.view.xy[1]
 
-            canvas.add_nine_patch_rect(int(round(x - minx)),
-                                       int(round(y - miny)),
-                                       v.view.w,
-                                       v.view.h)
+            canvas.box(int(round(x - minx)),
+                       int(round(y - miny)),
+                       v.view.w,
+                       v.view.h)
 
-            canvas.add_text(int(round(x - minx)) + 1,
-                            int(round(y - miny)) + 1,
-                            v.data)
+            canvas.text(int(round(x - minx)) + 1,
+                        int(round(y - miny)) + 1,
+                        v.data)
 
-        canvas.print_out()
+        canvas.draw()

dvc/dependency/base.py

@@ -54,7 +54,7 @@ class DependencyBase(object):
 
     @property
     def exists(self):
-        return self.remote.exists([self.path_info])
+        return self.remote.exists([self.path_info])[0]
 
     def changed(self):
         raise NotImplementedError

dvc/dependency/local.py

@@ -41,6 +41,12 @@ class DependencyLOCAL(DependencyBase):
     def __str__(self):
         return self.rel_path
 
+    @property
+    def is_local(self):
+        assert os.path.isabs(self.path)
+        assert os.path.isabs(self.project.root_dir)
+        return self.path.startswith(self.project.root_dir)
+
     @property
     def sep(self):
         return os.sep
@@ -61,7 +67,8 @@ class DependencyLOCAL(DependencyBase):
         if not self.exists:
             raise self.DoesNotExistError(self.rel_path)
 
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
+        if not os.path.isfile(self.path) \
+           and not os.path.isdir(self.path):  # pragma: no cover
             raise self.IsNotFileOrDirError(self.rel_path)
 
         if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
@@ -71,7 +78,7 @@ class DependencyLOCAL(DependencyBase):
         self.info = self.remote.save_info(self.path_info)
 
     def dumpd(self):
-        if self.path.startswith(self.stage.project.root_dir):
+        if self.is_local:
             path = self.remote.unixpath(os.path.relpath(self.path,
                                                         self.stage.cwd))
         else:

dvc/exceptions.py

@@ -8,7 +8,7 @@ class DvcException(Exception):
         if cause:
             try:
                 self.cause_tb = traceback.format_exc()
-            except Exception:
+            except Exception:  # pragma: no cover
                 pass
         super(DvcException, self).__init__(msg)
 
@@ -29,3 +29,12 @@ class OutputDuplicationError(DvcException):
         for stage in stages:
             s += "\n    " + stage
         super(OutputDuplicationError, self).__init__(msg.format(output, s))
+
+
+class MoveNotDataSourceError(DvcException):
+    def __init__(self, path):
+        msg = "Move is not permitted for stages that are not data sources. " \
+              "You need to either move '{path}' to a new location and edit " \
+              "it by hand, or remove '{path}' and create a new one at the " \
+              "desired location."
+        super(MoveNotDataSourceError, self).__init__(msg.format(path=path))

dvc/logger.py

@@ -19,9 +19,16 @@ class Logger(object):
     }
 
     COLOR_MAP = {
-        'debug': colorama.Fore.BLUE,
-        'warn': colorama.Fore.YELLOW,
-        'error': colorama.Fore.RED
+        'green': colorama.Fore.GREEN,
+        'yellow': colorama.Fore.YELLOW,
+        'blue': colorama.Fore.BLUE,
+        'red': colorama.Fore.RED,
+    }
+
+    LEVEL_COLOR_MAP = {
+        'debug': 'blue',
+        'warn': 'yellow',
+        'error': 'red',
     }
 
     def __init__(self, loglevel=None):
@@ -58,12 +65,12 @@ class Logger(object):
         Logger.logger().setLevel(logging.DEBUG)
 
     @staticmethod
-    def colorize(msg, typ):
+    def colorize(msg, color):
         header = ''
         footer = ''
 
         if sys.stdout.isatty():  # pragma: no cover
-            header = Logger.COLOR_MAP.get(typ.lower(), '')
+            header = Logger.COLOR_MAP.get(color.lower(), '')
             footer = colorama.Style.RESET_ALL
 
         return u'{}{}{}'.format(header, msg, footer)
@@ -84,7 +91,8 @@ class Logger(object):
 
     @staticmethod
     def _prefix(msg, typ):
-        return Logger.colorize('{}: '.format(msg), typ)
+        color = Logger.LEVEL_COLOR_MAP.get(typ.lower(), '')
+        return Logger.colorize('{}: '.format(msg), color)
 
     @staticmethod
     def error_prefix():
@@ -105,7 +113,9 @@ class Logger(object):
         if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
             str_tb = str_tb if str_tb else traceback.format_exc()
             Logger.logger().error(prefix + str_tb)
-        return Logger.logger().error(prefix + msg + str_exc)
+        chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \
+               "we are always happy to help!"
+        return Logger.logger().error(prefix + msg + str_exc + chat)
 
     @staticmethod
     def warn(msg):

dvc/output/local.py

@@ -40,10 +40,6 @@ class OutputLOCAL(DependencyLOCAL):
     def cache(self):
         return self.project.cache.local.get(self.md5)
 
-    @property
-    def is_local(self):
-        return self.path.startswith(self.project.root_dir)
-
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
         ret[self.PARAM_CACHE] = self.use_cache
@@ -87,13 +83,14 @@ class OutputLOCAL(DependencyLOCAL):
             super(OutputLOCAL, self).save()
             self._verify_metric()
             msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
-            self.project.logger.debug(msg.format(self.rel_path))
+            self.project.logger.info(msg.format(self.rel_path))
             return
 
         if not os.path.exists(self.path):
             raise self.DoesNotExistError(self.rel_path)
 
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
+        if not os.path.isfile(self.path) \
+           and not os.path.isdir(self.path):  # pragma: no cover
             raise self.IsNotFileOrDirError(self.rel_path)
 
         if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
@@ -103,7 +100,7 @@ class OutputLOCAL(DependencyLOCAL):
 
         if not self.changed():
             msg = 'Output \'{}\' didn\'t change. Skipping saving.'
-            self.project.logger.debug(msg.format(self.rel_path))
+            self.project.logger.info(msg.format(self.rel_path))
             return
 
         if self.is_local:

dvc/progress.py

@@ -16,6 +16,10 @@ class Progress(object):
         self._n_total = total
         self._n_finished = 0
 
+    @property
+    def is_finished(self):
+        return self._n_total == self._n_finished
+
     def _clearln(self):
         print('\r\x1b[K', end='')
 
@@ -63,7 +67,7 @@ class Progress(object):
             progress = int((100 * current)/total) if current < total else 100
             percent = str(progress) + "% "
 
-        if self._n_total:
+        if self._n_total > 1:
             num = "({}/{}): ".format(self._n_finished + 1, self._n_total)
         else:
             num = ""

dvc/project.py

@@ -1,6 +1,6 @@
 import os
 
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, MoveNotDataSourceError
 from dvc.stage import Stage
 
 
@@ -22,7 +22,7 @@ class Project(object):
     def __init__(self, root_dir):
         from dvc.logger import Logger
         from dvc.config import Config
-        from dvc.state import LinkState, State
+        from dvc.state import State
         from dvc.lock import Lock
         from dvc.scm import SCM
         from dvc.cache import Cache
@@ -34,12 +34,11 @@ class Project(object):
         self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
 
         self.config = Config(self.dvc_dir)
-        self.scm = SCM(self.root_dir)
+        self.scm = SCM(self.root_dir, project=self)
         self.lock = Lock(self.dvc_dir)
         # NOTE: storing state and link_state in the repository itself to avoid
         # any possible state corruption in 'shared cache dir' scenario.
-        self.state = State(self)
-        self.link_state = LinkState(self)
+        self.state = State(self, self.config._config)
 
         core = self.config._config[Config.SECTION_CORE]
         self.logger = Logger(core.get(Config.SECTION_CORE_LOGLEVEL, None))
@@ -49,10 +48,21 @@ class Project(object):
         self.updater = Updater(self.dvc_dir)
         self.prompt = Prompt()
 
+        self._files_to_git_add = []
+
         self._ignore()
 
         self.updater.check()
 
+    def _remind_to_git_add(self):
+        if len(self._files_to_git_add) == 0:
+            return
+
+        msg = '\nTo track the changes with git run:\n\n'
+        msg += '\tgit add ' + " ".join(self._files_to_git_add)
+
+        self.logger.info(msg)
+
     @staticmethod
     def init(root_dir=os.curdir, no_scm=False, force=False):
         """
@@ -70,10 +80,10 @@ class Project(object):
         import shutil
         from dvc.scm import SCM, Base
         from dvc.config import Config
+        from dvc.logger import Logger
 
         root_dir = os.path.abspath(root_dir)
         dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
-
         scm = SCM(root_dir)
         if type(scm) == Base and not no_scm:
             msg = "{} is not tracked by any supported scm tool(e.g. git)."
@@ -94,6 +104,8 @@ class Project(object):
         if scm.ignore_file():
             scm.add([os.path.join(dvc_dir, scm.ignore_file())])
 
+        Logger.info('You can now commit the changes to git.')
+
         return proj
 
     def destroy(self):
@@ -106,9 +118,6 @@ class Project(object):
 
     def _ignore(self):
         flist = [self.state.state_file,
-                 self.state._lock_file.lock_file,
-                 self.link_state.state_file,
-                 self.link_state._lock_file.lock_file,
                  self.lock.lock_file,
                  self.config.config_local_file,
                  self.updater.updater_file]
@@ -152,16 +161,20 @@ class Project(object):
             fnames = [fname]
 
         stages = []
-        for f in fnames:
-            stage = Stage.loads(project=self,
-                                outs=[f],
-                                add=True)
+        self._files_to_git_add = []
+        with self.state:
+            for f in fnames:
+                stage = Stage.loads(project=self,
+                                    outs=[f],
+                                    add=True)
 
-            self._check_output_duplication(stage.outs)
+                self._check_output_duplication(stage.outs)
 
-            stage.save()
-            stage.dump()
-            stages.append(stage)
+                stage.save()
+                stage.dump()
+                stages.append(stage)
+
+        self._remind_to_git_add()
 
         return stages
 
@@ -188,34 +201,37 @@ class Project(object):
                                      [from_path])[0]
 
         found = False
-        for stage in self.stages():
-            for out in stage.outs:
-                if out.path != from_out.path:
-                    continue
+        self._files_to_git_add = []
+        with self.state:
+            for stage in self.stages():
+                for out in stage.outs:
+                    if out.path != from_out.path:
+                        continue
 
-                if not stage.is_data_source:
-                    msg = 'Dvcfile \'{}\' is not a data source.'
-                    raise DvcException(msg.format(stage.rel_path))
+                    if not stage.is_data_source:
+                        raise MoveNotDataSourceError(stage.relpath)
 
-                found = True
-                to_out = Output.loads_from(out.stage,
-                                           [to_path],
-                                           out.cache,
-                                           out.metric)[0]
-                out.move(to_out)
+                    found = True
+                    to_out = Output.loads_from(out.stage,
+                                               [to_path],
+                                               out.cache,
+                                               out.metric)[0]
+                    out.move(to_out)
 
-                stage_base = os.path.basename(stage.path)
-                stage_base = stage_base.rstrip(Stage.STAGE_FILE_SUFFIX)
+                    stage_base = os.path.basename(stage.path)
+                    stage_base = stage_base.rstrip(Stage.STAGE_FILE_SUFFIX)
 
-                stage_dir = os.path.dirname(stage.path)
-                from_base = os.path.basename(from_path)
-                to_base = os.path.basename(to_path)
-                if stage_base == from_base:
-                    os.unlink(stage.path)
-                    path = to_base + Stage.STAGE_FILE_SUFFIX
-                    stage.path = os.path.join(stage_dir, path)
+                    stage_dir = os.path.dirname(stage.path)
+                    from_base = os.path.basename(from_path)
+                    to_base = os.path.basename(to_path)
+                    if stage_base == from_base:
+                        os.unlink(stage.path)
+                        path = to_base + Stage.STAGE_FILE_SUFFIX
+                        stage.path = os.path.join(stage_dir, path)
 
-            stage.dump()
+                stage.dump()
+
+        self._remind_to_git_add()
 
         if not found:
             msg = 'Unable to find dvcfile with output \'{}\''
@@ -243,9 +259,15 @@ class Project(object):
 
         self._check_output_duplication(stage.outs)
 
-        if not no_exec:
-            stage.run()
+        self._files_to_git_add = []
+        with self.state:
+            if not no_exec:
+                stage.run()
+
         stage.dump()
+
+        self._remind_to_git_add()
+
         return stage
 
     def imp(self, url, out):
@@ -256,15 +278,21 @@ class Project(object):
 
         self._check_output_duplication(stage.outs)
 
-        stage.run()
+        self._files_to_git_add = []
+        with self.state:
+            stage.run()
+
         stage.dump()
+
+        self._remind_to_git_add()
+
         return stage
 
     def _reproduce_stage(self, stages, node, force, dry, interactive):
         stage = stages[node]
 
         if stage.locked:
-            msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
+            msg = 'DVC file \'{}\' is locked. Its dependencies are not ' \
                   'going to be reproduced.'
             self.logger.warn(msg.format(stage.relpath))
 
@@ -278,11 +306,60 @@ class Project(object):
         return [stage]
 
     def reproduce(self,
-                  target,
+                  target=None,
                   recursive=True,
                   force=False,
                   dry=False,
-                  interactive=False):
+                  interactive=False,
+                  pipeline=False,
+                  all_pipelines=False):
+
+        if target is None and not all_pipelines:
+            raise ValueError()
+
+        if not interactive:
+            config = self.config
+            core = config._config[config.SECTION_CORE]
+            interactive = core.get(config.SECTION_CORE_INTERACTIVE, False)
+
+        targets = []
+        if pipeline or all_pipelines:
+            if pipeline:
+                stage = Stage.load(self, target)
+                node = os.path.relpath(stage.path, self.root_dir)
+                pipelines = [self._get_pipeline(node)]
+            else:
+                pipelines = self.pipelines()
+
+            for G in pipelines:
+                for node in G.nodes():
+                    if G.in_degree(node) == 0:
+                        targets.append(os.path.join(self.root_dir, node))
+        else:
+            targets.append(target)
+
+        self._files_to_git_add = []
+
+        ret = []
+        with self.state:
+            for target in targets:
+                stages = self._reproduce(target,
+                                         recursive=recursive,
+                                         force=force,
+                                         dry=dry,
+                                         interactive=interactive)
+                ret.extend(stages)
+
+        self._remind_to_git_add()
+
+        return ret
+
+    def _reproduce(self,
+                   target,
+                   recursive=True,
+                   force=False,
+                   dry=False,
+                   interactive=False):
         import networkx as nx
 
         stage = Stage.load(self, target)
@@ -290,20 +367,21 @@ class Project(object):
         stages = nx.get_node_attributes(G, 'stage')
         node = os.path.relpath(stage.path, self.root_dir)
 
-        if not interactive:
-            config = self.config
-            core = config._config[config.SECTION_CORE]
-            interactive = core.get(config.SECTION_CORE_INTERACTIVE, False)
-
         if recursive:
-            return self._reproduce_stages(G,
-                                          stages,
-                                          node,
-                                          force,
-                                          dry,
-                                          interactive)
+            ret = self._reproduce_stages(G,
+                                         stages,
+                                         node,
+                                         force,
+                                         dry,
+                                         interactive)
+        else:
+            ret = self._reproduce_stage(stages,
+                                        node,
+                                        force,
+                                        dry,
+                                        interactive)
 
-        return self._reproduce_stage(stages, node, force, dry, interactive)
+        return ret
 
     def _reproduce_stages(self, G, stages, node, force, dry, interactive):
         import networkx as nx
@@ -325,7 +403,7 @@ class Project(object):
         for stage in all_stages:
             for out in stage.outs:
                 used.append(out.path)
-        self.link_state.remove_unused(used)
+        self.state.remove_unused_links(used)
 
     def checkout(self, target=None):
         all_stages = self.active_stages()
@@ -335,17 +413,46 @@ class Project(object):
         else:
             stages = all_stages
 
-        self._cleanup_unused_links(all_stages)
+        with self.state:
+            self._cleanup_unused_links(all_stages)
 
-        for stage in stages:
-            if stage.locked:
-                msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
-                      'going to be checked out.'
-                self.logger.warn(msg.format(stage.relpath))
+            for stage in stages:
+                if stage.locked:
+                    msg = 'DVC file \'{}\' is locked. Its dependencies are ' \
+                          'not going to be checked out.'
+                    self.logger.warn(msg.format(stage.relpath))
 
-            stage.checkout()
+                stage.checkout()
 
-    def _used_cache(self, target=None, all_branches=False, active=True):
+    def _get_pipeline(self, node):
+        pipelines = list(filter(lambda g: node in g.nodes(),
+                                self.pipelines()))
+        assert len(pipelines) == 1
+        return pipelines[0]
+
+    def _collect(self, target, with_deps=False):
+        import networkx as nx
+
+        stage = Stage.load(self, target)
+        if not with_deps:
+            return [stage]
+
+        node = os.path.relpath(stage.path, self.root_dir)
+        G = self._get_pipeline(node)
+        stages = nx.get_node_attributes(G, 'stage')
+
+        ret = [stage]
+        for n in nx.dfs_postorder_nodes(G, node):
+            ret.append(stages[n])
+
+        return ret
+
+    def _used_cache(self,
+                    target=None,
+                    all_branches=False,
+                    active=True,
+                    with_deps=False,
+                    all_tags=False):
         cache = {}
         cache['local'] = []
         cache['s3'] = []
@@ -353,9 +460,11 @@ class Project(object):
         cache['hdfs'] = []
         cache['ssh'] = []
 
-        for branch in self.scm.brancher(all_branches=all_branches):
+        for branch in self.scm.brancher(all_branches=all_branches,
+                                        all_tags=all_tags):
             if target:
-                stages = [Stage.load(self, target)]
+                stages = self._collect(target,
+                                       with_deps=with_deps)
             elif active:
                 stages = self.active_stages()
             else:
@@ -363,7 +472,7 @@ class Project(object):
 
             for stage in stages:
                 if active and not target and stage.locked:
-                    msg = 'DVC file \'{}\' is locked. Its dependecies are ' \
+                    msg = 'DVC file \'{}\' is locked. Its dependencies are ' \
                           'not going to be pushed/pulled/fetched.'
                     self.logger.warn(msg.format(stage.relpath))
 
@@ -383,64 +492,90 @@ class Project(object):
         if not removed:
             self.logger.info("No unused {} cache to remove.".format(typ))
 
-    def gc(self, all_branches=False, cloud=False, remote=None):
-        clist = self._used_cache(target=None,
-                                 all_branches=all_branches,
-                                 active=False)
-        self._do_gc('local', self.cache.local.gc, clist)
+    def gc(self,
+           all_branches=False,
+           cloud=False,
+           remote=None,
+           with_deps=False,
+           all_tags=False):
+        with self.state:
+            clist = self._used_cache(target=None,
+                                     all_branches=all_branches,
+                                     active=False,
+                                     with_deps=with_deps,
+                                     all_tags=all_tags)
+            self._do_gc('local', self.cache.local.gc, clist)
 
-        if self.cache.s3:
-            self._do_gc('s3', self.cache.s3.gc, clist)
+            if self.cache.s3:
+                self._do_gc('s3', self.cache.s3.gc, clist)
 
-        if self.cache.gs:
-            self._do_gc('gs', self.cache.gs.gc, clist)
+            if self.cache.gs:
+                self._do_gc('gs', self.cache.gs.gc, clist)
 
-        if self.cache.ssh:
-            self._do_gc('ssh', self.cache.ssh.gc, clist)
+            if self.cache.ssh:
+                self._do_gc('ssh', self.cache.ssh.gc, clist)
 
-        if self.cache.hdfs:
-            self._do_gc('hdfs', self.cache.hdfs.gc, clist)
+            if self.cache.hdfs:
+                self._do_gc('hdfs', self.cache.hdfs.gc, clist)
 
-        if self.cache.azure:
-            self._do_gc('azure', self.cache.azure.gc, clist)
+            if self.cache.azure:
+                self._do_gc('azure', self.cache.azure.gc, clist)
 
-        if cloud:
-            self._do_gc('remote', self.cloud._get_cloud(remote,
-                                                        'gc -c').gc, clist)
+            if cloud:
+                self._do_gc('remote', self.cloud._get_cloud(remote,
+                                                            'gc -c').gc, clist)
 
     def push(self,
              target=None,
              jobs=1,
              remote=None,
              all_branches=False,
-             show_checksums=False):
-        self.cloud.push(self._used_cache(target, all_branches)['local'],
-                        jobs,
-                        remote=remote,
-                        show_checksums=show_checksums)
+             show_checksums=False,
+             with_deps=False,
+             all_tags=False):
+        with self.state:
+            used = self._used_cache(target,
+                                    all_branches=all_branches,
+                                    all_tags=all_tags,
+                                    with_deps=with_deps)['local']
+            self.cloud.push(used,
+                            jobs,
+                            remote=remote,
+                            show_checksums=show_checksums)
 
     def fetch(self,
               target=None,
               jobs=1,
               remote=None,
               all_branches=False,
-              show_checksums=False):
-        self.cloud.pull(self._used_cache(target, all_branches)['local'],
-                        jobs,
-                        remote=remote,
-                        show_checksums=show_checksums)
+              show_checksums=False,
+              with_deps=False,
+              all_tags=False):
+        with self.state:
+            used = self._used_cache(target,
+                                    all_branches=all_branches,
+                                    all_tags=all_tags,
+                                    with_deps=with_deps)['local']
+            self.cloud.pull(used,
+                            jobs,
+                            remote=remote,
+                            show_checksums=show_checksums)
 
     def pull(self,
              target=None,
              jobs=1,
              remote=None,
              all_branches=False,
-             show_checksums=False):
+             show_checksums=False,
+             with_deps=False,
+             all_tags=False):
         self.fetch(target,
                    jobs,
                    remote=remote,
                    all_branches=all_branches,
-                   show_checksums=show_checksums)
+                   all_tags=all_tags,
+                   show_checksums=show_checksums,
+                   with_deps=with_deps)
         self.checkout(target=target)
 
     def _local_status(self, target=None):
@@ -453,8 +588,8 @@ class Project(object):
 
         for stage in stages:
             if stage.locked:
-                msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
-                      'going to be shown in status output.'
+                msg = 'DVC file \'{}\' is locked. Its dependencies are not ' \
+                      'going to be shown in the status output.'
                 self.logger.warn(msg.format(stage.relpath))
 
             status.update(stage.status())
@@ -465,11 +600,19 @@ class Project(object):
                       target=None,
                       jobs=1,
                       remote=None,
-                      show_checksums=False):
+                      show_checksums=False,
+                      all_branches=False,
+                      with_deps=False,
+                      all_tags=False):
         import dvc.remote.base as cloud
 
+        used = self._used_cache(target,
+                                all_branches=all_branches,
+                                all_tags=all_tags,
+                                with_deps=with_deps)['local']
+
         status = {}
-        for md5, ret in self.cloud.status(self._used_cache(target)['local'],
+        for md5, ret in self.cloud.status(used,
                                           jobs,
                                           remote=remote,
                                           show_checksums=show_checksums):
@@ -490,13 +633,20 @@ class Project(object):
                jobs=1,
                cloud=False,
                remote=None,
-               show_checksums=False):
-        if cloud:
-            return self._cloud_status(target,
-                                      jobs,
-                                      remote=remote,
-                                      show_checksums=show_checksums)
-        return self._local_status(target)
+               show_checksums=False,
+               all_branches=False,
+               with_deps=False,
+               all_tags=False):
+        with self.state:
+            if cloud:
+                return self._cloud_status(target,
+                                          jobs,
+                                          remote=remote,
+                                          show_checksums=show_checksums,
+                                          all_branches=all_branches,
+                                          with_deps=with_deps,
+                                          all_tags=all_tags)
+            return self._local_status(target)
 
     def _read_metric_json(self, fd, json_path):
         import json
@@ -576,9 +726,11 @@ class Project(object):
                      path=None,
                      typ=None,
                      xpath=None,
-                     all_branches=False):
+                     all_branches=False,
+                     all_tags=False):
         res = {}
-        for branch in self.scm.brancher(all_branches=all_branches):
+        for branch in self.scm.brancher(all_branches=all_branches,
+                                        all_tags=all_tags):
             astages = self.active_stages()
             outs = [out for stage in astages for out in stage.outs]
 
@@ -623,7 +775,7 @@ class Project(object):
                 res[branch][rel] = metric
 
         for branch, val in res.items():
-            if all_branches:
+            if all_branches or all_tags:
                 self.logger.info('{}:'.format(branch))
             for fname, metric in val.items():
                 self.logger.info('\t{}: {}'.format(fname, metric))
@@ -749,12 +901,28 @@ class Project(object):
 
     def stages(self):
         stages = []
+        outs = []
         for root, dirs, files in os.walk(self.root_dir):
             for fname in files:
                 path = os.path.join(root, fname)
                 if not Stage.is_stage_file(path):
                     continue
-                stages.append(Stage.load(self, path))
+                stage = Stage.load(self, path)
+                for out in stage.outs:
+                    outs.append(out.path + out.sep)
+                stages.append(stage)
+
+            def filter_dirs(dname):
+                path = os.path.join(root, dname)
+                if path == self.dvc_dir or path == self.scm.dir:
+                    return False
+                for out in outs:
+                    if path == os.path.normpath(out) or path.startswith(out):
+                        return False
+                return True
+
+            dirs[:] = list(filter(filter_dirs, dirs))
+
         return stages
 
     def active_stages(self):

dvc/prompt.py

@@ -1,5 +1,7 @@
 import sys
 
+from dvc.progress import progress
+
 try:
     # NOTE: in Python3 raw_input() was renamed to input()
     input = raw_input
@@ -23,3 +25,16 @@ class Prompt(object):
             answer = input('Enter \'yes\' or \'no\'.\n').lower()
 
         return answer[0] == "y"
+
+    def prompt_password(self, msg):  # pragma: no cover
+        import getpass
+
+        if not sys.stdout.isatty():
+            return None
+
+        msg = 'Enter password for {}:\n'.format(msg)
+
+        if not progress.is_finished:
+            msg = u'\n' + msg
+
+        return getpass.getpass(msg)

dvc/remote/base.py

@@ -1,6 +1,7 @@
 import os
 import re
 import errno
+from multiprocessing import cpu_count
 
 from dvc.config import Config
 from dvc.logger import Logger
@@ -30,6 +31,7 @@ class DataCloudError(DvcException):
 class RemoteBase(object):
     REGEX = None
     REQUIRES = {}
+    JOBS = 4 * cpu_count()
 
     def __init__(self, project, config):
         pass

dvc/remote/hdfs.py

@@ -29,9 +29,14 @@ class RemoteHDFS(RemoteBase):
         cmd = 'hadoop fs -' + cmd
         if user:
             cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
+
+        # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
+        # See https://github.com/iterative/dvc/issues/1197.
+        close_fds = (os.name != 'nt')
+
         p = Popen(cmd,
                   shell=True,
-                  close_fds=True,
+                  close_fds=close_fds,
                   executable=os.getenv('SHELL'),
                   env=fix_env(os.environ),
                   stdin=PIPE,

dvc/remote/local.py

@@ -36,7 +36,6 @@ class RemoteLOCAL(RemoteBase):
     def __init__(self, project, config):
         self.project = project
         self.state = self.project.state
-        self.link_state = project.link_state
         storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
         self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
 
@@ -82,14 +81,35 @@ class RemoteLOCAL(RemoteBase):
         relpath = os.path.relpath(path, self.cache_dir)
         return os.path.dirname(relpath) + os.path.basename(relpath)
 
-    def changed_cache(self, md5):
+    def changed_cache_file(self, md5):
         cache = self.get(md5)
         if self.state.changed(cache, md5=md5):
             if os.path.exists(cache):
-                msg = 'Corrupted cache file {}'
+                msg = 'Corrupted cache file {}.'
                 Logger.warn(msg.format(os.path.relpath(cache)))
                 remove(cache)
             return True
+        return False
+
+    def changed_cache(self, md5):
+        cache = self.get(md5)
+        clist = [(cache, md5)]
+
+        while True:
+            if len(clist) == 0:
+                break
+
+            cache, md5 = clist.pop()
+            if self.changed_cache_file(md5):
+                return True
+
+            if not self.is_dir_cache(cache):
+                continue
+
+            for entry in self.load_dir_cache(md5):
+                md5 = entry[self.PARAM_MD5]
+                cache = self.get(md5)
+                clist.append((cache, md5))
 
         return False
 
@@ -109,7 +129,7 @@ class RemoteLOCAL(RemoteBase):
                                        os.path.relpath(cache),
                                        os.path.relpath(path)))
                 return
-            except Exception as exc:
+            except DvcException as exc:
                 msg = 'Cache type \'{}\' is not supported: {}'
                 Logger.debug(msg.format(self.cache_types[0], str(exc)))
                 del self.cache_types[0]
@@ -138,8 +158,8 @@ class RemoteLOCAL(RemoteBase):
     def collect_dir_cache(self, dname):
         dir_info = []
 
+        bar = False
         for root, dirs, files in os.walk(dname):
-            bar = False
             if len(files) > LARGE_DIR_SIZE:
                 msg = "Computing md5 for a large directory {}. " \
                       "This is only done once."
@@ -158,19 +178,12 @@ class RemoteLOCAL(RemoteBase):
                     progress.update_target(title, processed, total)
                     processed += 1
 
-                # FIXME: we could've used md5 = state.update(path, dump=False)
-                # here, but it is around twice as slow(on ssd, don't know about
-                # hdd) for a directory with small files. What we could do here
-                # is introduce some kind of a limit for file size, after which
-                # we would actually register it in our state file.
-                md5 = self.state.update(path, dump=False)
+                md5 = self.state.update(path)
                 dir_info.append({self.PARAM_RELPATH: relpath,
                                  self.PARAM_MD5: md5})
 
-            self.state.dump()
-
-            if bar:
-                progress.finish_target(title)
+        if bar:
+            progress.finish_target(title)
 
         # NOTE: sorting the list by path to ensure reproducibility
         dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
@@ -256,7 +269,7 @@ class RemoteLOCAL(RemoteBase):
 
         if not self.is_dir_cache(cache):
             self.link(cache, path)
-            self.link_state.update(path)
+            self.state.update_link(path)
             return
 
         # Create dir separately so that dir is created
@@ -270,7 +283,7 @@ class RemoteLOCAL(RemoteBase):
             relpath = entry[self.PARAM_RELPATH]
             p = os.path.join(path, relpath)
             self.link(c, p)
-        self.link_state.update(path)
+        self.state.update_link(path)
 
     def _move(self, inp, outp):
         # moving in two stages to make last the move atomic in
@@ -292,7 +305,7 @@ class RemoteLOCAL(RemoteBase):
             remove(path)
 
         self.link(cache, path)
-        self.link_state.update(path)
+        self.state.update_link(path)
 
         return {self.PARAM_MD5: md5}
 
@@ -313,7 +326,7 @@ class RemoteLOCAL(RemoteBase):
 
             self.link(c, p)
 
-        self.link_state.update(path)
+        self.state.update_link(path)
 
         return {self.PARAM_MD5: md5}
 
@@ -451,8 +464,8 @@ class RemoteLOCAL(RemoteBase):
             for i in self.load_dir_cache(md5):
                 if info.get('branch'):
                     i['branch'] = info['branch']
-                i[self.PARAM_PATH] = posixpath.join(info[self.PARAM_PATH],
-                                                    i[self.PARAM_RELPATH])
+                i[self.PARAM_PATH] = os.path.join(info[self.PARAM_PATH],
+                                                  i[self.PARAM_RELPATH])
                 collected.append(i.copy())
 
         collected.extend(checksum_infos)
@@ -495,11 +508,11 @@ class RemoteLOCAL(RemoteBase):
 
         return removed
 
-    def status(self, checksum_infos, remote, jobs=1, show_checksums=False):
+    def status(self, checksum_infos, remote, jobs=None, show_checksums=False):
         Logger.info("Preparing to pull data from {}".format(remote.url))
         title = "Collecting information"
 
-        progress.set_n_total(0)
+        progress.set_n_total(1)
         progress.update_target(title, 0, 100)
 
         checksum_infos, missing = self._collect(checksum_infos)
@@ -520,7 +533,7 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 90, 100)
 
-        local_exists = [not self.changed_cache(md5) for md5 in md5s]
+        local_exists = [not self.changed_cache_file(md5) for md5 in md5s]
 
         progress.finish_target(title)
 
@@ -531,11 +544,11 @@ class RemoteLOCAL(RemoteBase):
     def _do_pull(self,
                  checksum_infos,
                  remote,
-                 jobs=1,
+                 jobs=None,
                  show_checksums=False):
         title = "Collecting information"
 
-        progress.set_n_total(0)
+        progress.set_n_total(1)
         progress.update_target(title, 0, 100)
 
         grouped = zip(*self._group(checksum_infos,
@@ -547,7 +560,7 @@ class RemoteLOCAL(RemoteBase):
         names = []
         # NOTE: filter files that are not corrupted
         for md5, name in grouped:
-            if self.changed_cache(md5):
+            if self.changed_cache_file(md5):
                 md5s.append(md5)
                 names.append(name)
 
@@ -561,8 +574,17 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 60, 100)
 
+        # NOTE: dummy call to try to establish a connection
+        # to see if we need to ask user for a password.
+        remote.exists(remote.md5s_to_path_infos(['000']))
+
+        progress.update_target(title, 70, 100)
+
         assert len(path_infos) == len(cache) == len(md5s) == len(names)
 
+        if jobs is None:
+            jobs = remote.JOBS
+
         chunks = list(zip(to_chunks(path_infos, jobs),
                           to_chunks(cache, jobs),
                           to_chunks(names, jobs)))
@@ -586,7 +608,7 @@ class RemoteLOCAL(RemoteBase):
         for f in futures:
             f.result()
 
-    def pull(self, checksum_infos, remote, jobs=1, show_checksums=False):
+    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
         Logger.info("Preparing to pull data from {}".format(remote.url))
 
         # NOTE: try fetching missing dir info
@@ -603,11 +625,11 @@ class RemoteLOCAL(RemoteBase):
                       jobs,
                       show_checksums=show_checksums)
 
-    def push(self, checksum_infos, remote, jobs=1, show_checksums=False):
+    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
         Logger.info("Preparing to push data to {}".format(remote.url))
         title = "Collecting information"
 
-        progress.set_n_total(0)
+        progress.set_n_total(1)
         progress.update_target(title, 0, 100)
 
         checksum_infos = self._collect(checksum_infos)[0]
@@ -616,7 +638,7 @@ class RemoteLOCAL(RemoteBase):
 
         # NOTE: verifying that our cache is not corrupted
         def func(info):
-            return not self.changed_cache(info[self.PARAM_MD5])
+            return not self.changed_cache_file(info[self.PARAM_MD5])
         checksum_infos = list(filter(func, checksum_infos))
 
         progress.update_target(title, 20, 100)
@@ -648,6 +670,9 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 90, 100)
 
+        if jobs is None:
+            jobs = remote.JOBS
+
         chunks = list(zip(to_chunks(path_infos, jobs),
                           to_chunks(cache, jobs),
                           to_chunks(names, jobs)))

dvc/remote/s3.py

@@ -47,7 +47,9 @@ class RemoteS3(RemoteBase):
                                            '').lstrip('/')
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
         self.region = config.get(Config.SECTION_AWS_REGION, None)
-        self.profile = config.get(Config.SECTION_AWS_PROFILE, 'default')
+        self.profile = os.getenv('AWS_PROFILE',
+                                 config.get(Config.SECTION_AWS_PROFILE,
+                                            'default'))
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL, None)
 
         credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
@@ -55,7 +57,7 @@ class RemoteS3(RemoteBase):
             creds_conf = configobj.ConfigObj(credentialpath)
             creds = creds_conf.get(self.profile, {})
         else:
-            creds = {}
+            creds = self._get_aws_config_from_env()
 
         self.creds = creds
         self.region = creds.get('region', self.region)
@@ -81,6 +83,14 @@ class RemoteS3(RemoteBase):
                              region_name=self.region)
         return session.client('s3', endpoint_url=self.endpoint_url)
 
+    def _get_aws_config_from_env(self):
+        creds = {
+            'region': os.getenv('AWS_DEFAULT_REGION'),
+            'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
+            'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
+        }
+        return {k: v for k, v in creds.items() if v is not None}
+
     def get_etag(self, bucket, key):
         try:
             obj = self.s3.head_object(Bucket=bucket, Key=key)

dvc/remote/ssh.py

@@ -44,9 +44,13 @@ class RemoteSSH(RemoteBase):
     REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'
 
     REQUIRES = {'paramiko': paramiko}
+
+    JOBS = 4
+
     PARAM_MD5 = 'md5'
 
     DEFAULT_PORT = 22
+    TIMEOUT = 1800
 
     def __init__(self, project, config):
         self.project = project
@@ -59,6 +63,10 @@ class RemoteSSH(RemoteBase):
         self.prefix = self.group('path')
         self.port = config.get(Config.SECTION_REMOTE_PORT, self.DEFAULT_PORT)
         self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
+        self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
+        self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
+        self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD,
+                                       False)
 
     def md5s_to_path_infos(self, md5s):
         return [{'scheme': 'ssh',
@@ -78,10 +86,16 @@ class RemoteSSH(RemoteBase):
         ssh.load_system_host_keys()
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
+        if self.ask_password and self.password is None:
+            msg = "host '{}' port '{}' user '{}'".format(host, port, user)
+            self.password = self.project.prompt.prompt_password(msg)
+
         ssh.connect(host,
                     username=user,
                     port=port,
-                    key_filename=self.keyfile)
+                    key_filename=self.keyfile,
+                    timeout=self.timeout,
+                    password=self.password)
 
         return ssh
 
@@ -105,10 +119,47 @@ class RemoteSSH(RemoteBase):
 
     def _exec(self, ssh, cmd):
         stdin, stdout, stderr = ssh.exec_command(cmd)
+        channel = stdout.channel
+
+        stdin.close()
+        channel.shutdown_write()
+
+        stdout_chunks = []
+        stderr_chunks = []
+        while (not channel.closed
+               or channel.recv_ready()
+               or channel.recv_stderr_ready()):
+            import select
+            got_chunk = False
+            readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
+            for c in readq:
+                if c.recv_ready():
+                    stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
+                    got_chunk = True
+
+                if c.recv_stderr_ready():
+                    stderr_len = len(c.in_stderr_buffer)
+                    s = stderr.channel.recv_stderr(stderr_len)
+                    stderr_chunks.append(s)
+                    got_chunk = True
+
+            if not got_chunk \
+               and stdout.channel.exit_status_ready() \
+               and not stderr.channel.recv_stderr_ready() \
+               and not stdout.channel.recv_ready():
+                stdout.channel.shutdown_read()
+                stdout.channel.close()
+                break
+
+        stdout.close()
+        stderr.close()
+
         if stdout.channel.recv_exit_status() != 0:
-            DvcException('SSH command \'{}\' failed: {}'.format(cmd,
-                                                                stderr.read()))
-        return stdout.read().decode('utf-8')
+            err = b''.join(stderr_chunks).decode('utf-8')
+            msg = 'SSH command \'{}\' failed: {}'.format(cmd, err)
+            raise DvcException(msg)
+
+        return b''.join(stdout_chunks).decode('utf-8')
 
     def md5(self, path_info):
         if path_info['scheme'] != 'ssh':

dvc/scm.py

@@ -14,13 +14,18 @@ class FileNotInRepoError(DvcException):
 
 
 class Base(object):
-    def __init__(self, root_dir=os.curdir):
-        self.root_dir = os.path.abspath(os.path.realpath(root_dir))
+    def __init__(self, root_dir=os.curdir, project=None):
+        self.project = project
+        self.root_dir = root_dir
 
     @staticmethod
     def is_repo(root_dir):
         return True
 
+    @staticmethod
+    def is_submodule(root_dir):
+        return True
+
     def ignore(self, path):
         pass
 
@@ -45,17 +50,34 @@ class Base(object):
     def branch(self, branch):
         pass
 
-    def brancher(self, branches=None, all_branches=False):
-        if not branches and not all_branches:
+    def brancher(self,
+                 branches=None,
+                 all_branches=False,
+                 tags=None,
+                 all_tags=False):
+        if not branches and not all_branches \
+           and not tags and not all_tags:
             yield ''
             return
 
         saved = self.active_branch()
-        if not branches:
-            branches = self.list_branches() if all_branches else [saved]
-        for branch in branches:
-            self.checkout(branch)
-            yield branch
+        revs = []
+
+        if branches is not None:
+            revs.extend(branches)
+        elif all_branches:
+            revs.extend(self.list_branches())
+        elif tags is not None:
+            revs.extend(tags)
+        elif all_tags:
+            revs.extend(self.list_tags())
+        else:
+            revs.extend([saved])
+
+        for rev in revs:
+            self.checkout(rev)
+            yield rev
+
         self.checkout(saved)
 
     def untracked_files(self):
@@ -70,6 +92,9 @@ class Base(object):
     def list_branches(self):
         pass
 
+    def list_tags(self):
+        pass
+
     def install(self):
         pass
 
@@ -78,15 +103,16 @@ class Git(Base):
     GITIGNORE = '.gitignore'
     GIT_DIR = '.git'
 
-    def __init__(self, root_dir=os.curdir):
-        super(Git, self).__init__(root_dir)
+    def __init__(self, root_dir=os.curdir, project=None):
+        super(Git, self).__init__(root_dir, project=project)
 
         import git
         from git.exc import InvalidGitRepositoryError
         try:
             self.repo = git.Repo(root_dir)
         except InvalidGitRepositoryError:
-            raise SCMError('{} is not a git repository'.format(root_dir))
+            msg = '{} is not a git repository'
+            raise SCMError(msg.format(root_dir))
 
         # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
         # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
@@ -96,17 +122,27 @@ class Git(Base):
 
     @staticmethod
     def is_repo(root_dir):
-        git_dir = os.path.join(root_dir, Git.GIT_DIR)
-        return os.path.isdir(git_dir)
+        return os.path.isdir(Git._get_git_dir(root_dir))
+
+    @staticmethod
+    def is_submodule(root_dir):
+        return os.path.isfile(Git._get_git_dir(root_dir))
+
+    @staticmethod
+    def _get_git_dir(root_dir):
+        return os.path.join(root_dir, Git.GIT_DIR)
+
+    @property
+    def dir(self):
+        return self.repo.git_dir
 
     def ignore_file(self):
         return self.GITIGNORE
 
     def _get_gitignore(self, path):
+        assert os.path.isabs(path)
         entry = os.path.basename(path)
-        gitignore = os.path.join(os.path.abspath(os.path.dirname(path)),
-                                 self.GITIGNORE)
-        gitignore = os.path.abspath(os.path.realpath(gitignore))
+        gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE)
 
         if not gitignore.startswith(self.root_dir):
             raise FileNotInRepoError(path)
@@ -132,7 +168,11 @@ class Git(Base):
         if len(ignore_list) > 0:
             content = '\n' + content
 
-        open(gitignore, 'a').write(content)
+        with open(gitignore, 'a') as fd:
+            fd.write(content)
+
+        if self.project is not None:
+            self.project._files_to_git_add.append(os.path.relpath(gitignore))
 
     def ignore_remove(self, path):
         entry, gitignore = self._get_gitignore(path)
@@ -148,6 +188,9 @@ class Git(Base):
         with open(gitignore, 'w') as fd:
             fd.writelines(filtered)
 
+        if self.project is not None:
+            self.project._files_to_git_add.append(os.path.relpath(gitignore))
+
     def add(self, paths):
         # NOTE: GitPython is not currently able to handle index version >= 3.
         # See https://github.com/iterative/dvc/issues/610 for more details.
@@ -185,6 +228,9 @@ class Git(Base):
     def list_branches(self):
         return [h.name for h in self.repo.heads]
 
+    def list_tags(self):
+        return [t.name for t in self.repo.tags]
+
     def install(self):
         hook = os.path.join(self.root_dir,
                             self.GIT_DIR,
@@ -198,8 +244,8 @@ class Git(Base):
         os.chmod(hook, 0o777)
 
 
-def SCM(root_dir=os.curdir, no_scm=False):
-    if Git.is_repo(root_dir):
-        return Git(root_dir)
+def SCM(root_dir, no_scm=False, project=None):
+    if Git.is_repo(root_dir) or Git.is_submodule(root_dir):
+        return Git(root_dir, project=project)
 
-    return Base(root_dir)
+    return Base(root_dir, project=project)

dvc/signal_handler.py

@@ -1,17 +0,0 @@
-import signal
-import threading
-
-from dvc.logger import Logger
-
-
-class SignalHandler(object):
-    def __enter__(self):
-        if isinstance(threading.current_thread(), threading._MainThread):
-            self.old_handler = signal.signal(signal.SIGINT, self.handler)
-
-    def handler(self, sig, frame):
-        Logger.debug('Ignoring SIGINT during critical parts of code...')
-
-    def __exit__(self, type, value, traceback):
-        if isinstance(threading.current_thread(), threading._MainThread):
-            signal.signal(signal.SIGINT, self.old_handler)

dvc/stage.py

@@ -1,6 +1,5 @@
 import os
 import yaml
-import itertools
 import posixpath
 import subprocess
 from schema import Schema, SchemaError, Optional, Or, And
@@ -19,8 +18,9 @@ class StageCmdFailedError(DvcException):
 
 
 class StageFileFormatError(DvcException):
-    def __init__(self):
-        super(StageFileFormatError, self).__init__('Stage file format error')
+    def __init__(self, fname, e):
+        msg = "Stage file '{}' format error: {}".format(fname, str(e))
+        super(StageFileFormatError, self).__init__(msg)
 
 
 class StageFileDoesNotExistError(DvcException):
@@ -36,10 +36,21 @@ class StageFileIsNotDvcFileError(DvcException):
 
 
 class StageFileBadNameError(DvcException):
-    def __init__(self, fname):
-        msg = "Bad stage filename '{}'. Stage files should be named " \
-              "'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc')."
-        super(StageFileBadNameError, self).__init__(msg.format(fname, fname))
+    def __init__(self, msg):
+        super(StageFileBadNameError, self).__init__(msg)
+
+
+class MissingDep(DvcException):
+    def __init__(self, deps):
+        assert len(deps) > 0
+
+        if len(deps) > 1:
+            dep = 'dependencies'
+        else:
+            dep = 'dependency'
+
+        msg = u'missing {}: {}'.format(dep, ', '.join(map(str, deps)))
+        super(MissingDep, self).__init__(msg)
 
 
 class MissingDataSource(DvcException):
@@ -123,8 +134,6 @@ class Stage(object):
         if self.md5 and md5 and self.md5 == md5:
             return False
 
-        msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
-        self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
         return True
 
     @property
@@ -137,30 +146,51 @@ class Stage(object):
                len(self.deps) == 1 and \
                len(self.outs) == 1
 
-    def changed(self):
+    def changed(self, print_info=False):
         ret = False
 
+        if print_info:
+            log = self.project.logger.info
+        else:
+            log = self.project.logger.debug
+
         if self.is_callback:
+            msg = "Dvc file '{}' is a 'callback' stage (has a command and " \
+                  "no dependencies) and thus always considered as changed."
+            self.project.logger.warn(msg.format(self.relpath))
             ret = True
 
-        if self.locked:
-            entries = self.outs
-        else:
-            entries = itertools.chain(self.outs, self.deps)
-
-        for entry in entries:
-            if entry.changed():
+        if not self.locked:
+            for dep in self.deps:
+                if not dep.changed():
+                    continue
+                if print_info:
+                    msg = "Dependency '{}' of '{}' changed."
+                    log(msg.format(dep, self.relpath))
                 ret = True
 
+        for out in self.outs:
+            if not out.changed():
+                continue
+            if print_info:
+                msg = "Output '{}' of '{}' changed."
+                log(msg.format(out, self.relpath))
+            ret = True
+
         if self.changed_md5():
+            if print_info:
+                msg = "Dvc file '{}' changed."
+                log(msg.format(self.relpath))
             ret = True
 
         if ret:
-            msg = u'Dvc file \'{}\' changed'.format(self.relpath)
+            msg = "Stage '{}' changed.".format(self.relpath)
+            color = 'yellow'
         else:
-            msg = u'Dvc file \'{}\' didn\'t change'.format(self.relpath)
+            msg = "Stage '{}' didn't change.".format(self.relpath)
+            color = 'green'
 
-        self.project.logger.debug(msg)
+        log(Logger.colorize(msg, color))
 
         return ret
 
@@ -173,7 +203,7 @@ class Stage(object):
         os.unlink(self.path)
 
     def reproduce(self, force=False, dry=False, interactive=False):
-        if not self.changed() and not force:
+        if not self.changed(print_info=True) and not force:
             return None
 
         if (self.cmd or self.is_import) and not self.locked and not dry:
@@ -196,17 +226,15 @@ class Stage(object):
         return self
 
     @staticmethod
-    def validate(d):
+    def validate(d, fname=None):
         try:
             Schema(Stage.SCHEMA).validate(d)
         except SchemaError as exc:
-            Logger.debug(str(exc))
-            raise StageFileFormatError()
+            raise StageFileFormatError(fname, exc)
 
     @staticmethod
     def loadd(project, d, path):
-        Stage.validate(d)
-
+        Stage.validate(d, fname=os.path.relpath(path))
         path = os.path.abspath(path)
         cwd = os.path.dirname(path)
         cmd = d.get(Stage.PARAM_CMD, None)
@@ -270,6 +298,11 @@ class Stage(object):
                                         use_cache=False, metric=True)
         stage.deps = dependency.loads_from(stage, deps)
 
+        if fname is not None and os.path.basename(fname) != fname:
+            msg = "Stage file name '{}' should not contain subdirectories. " \
+                  "Use '-c|--cwd' to change location of the stage file."
+            raise StageFileBadNameError(msg.format(fname))
+
         fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add)
 
         cwd = os.path.abspath(cwd)
@@ -291,7 +324,10 @@ class Stage(object):
     @staticmethod
     def _check_dvc_filename(fname):
         if not Stage.is_stage_filename(fname):
-            raise StageFileBadNameError(os.path.relpath(fname))
+            msg = "Bad stage filename '{}'. Stage files should be named " \
+                  "'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc')."
+            raise StageFileBadNameError(msg.format(os.path.relpath(fname),
+                                                   os.path.basename(fname)))
 
     @staticmethod
     def _check_dvc_file(fname):
@@ -347,6 +383,8 @@ class Stage(object):
         with open(fname, 'w') as fd:
             yaml.safe_dump(self.dumpd(), fd, default_flow_style=False)
 
+        self.project._files_to_git_add.append(os.path.relpath(fname))
+
     def save(self):
         for dep in self.deps:
             dep.save()
@@ -354,6 +392,15 @@ class Stage(object):
         for out in self.outs:
             out.save()
 
+    def _check_missing_deps(self):
+        missing = []
+        for dep in self.deps:
+            if not dep.exists:
+                missing.append(dep)
+
+        if len(missing) > 0:
+            raise MissingDep(missing)
+
     def run(self, dry=False):
         if self.locked:
             msg = u'Verifying outputs in locked stage \'{}\''
@@ -377,6 +424,7 @@ class Stage(object):
             self.project.logger.info(msg)
 
             if not dry:
+                self._check_missing_deps()
                 p = subprocess.Popen(self.cmd,
                                      cwd=self.cwd,
                                      shell=True,

dvc/state.py

@@ -1,60 +1,71 @@
 import os
-import json
+import time
+import sqlite3
 import nanotime
-import threading
 
-from dvc.lock import Lock
+from dvc.config import Config
 from dvc.system import System
 from dvc.utils import file_md5, remove
 from dvc.exceptions import DvcException
-from dvc.signal_handler import SignalHandler
 from dvc.logger import Logger
 
 
-class StateEntry(object):
-    PARAM_MTIME = 'mtime'
-    PARAM_MD5 = 'md5'
-
-    def __init__(self, md5, mtime):
-        self.mtime = mtime
-        self.md5 = md5
-
-    def update(self, md5, mtime):
-        self.mtime = mtime
-        self.md5 = md5
-
-    @staticmethod
-    def loadd(d):
-        mtime = d[StateEntry.PARAM_MTIME]
-        md5 = d[StateEntry.PARAM_MD5]
-        return StateEntry(md5, mtime)
-
-    def dumpd(self):
-        return {
-            self.PARAM_MD5: self.md5,
-            self.PARAM_MTIME: self.mtime,
-        }
-
-
 class StateDuplicateError(DvcException):
     pass
 
 
 class State(object):
+    VERSION = 2
     STATE_FILE = 'state'
-    STATE_LOCK_FILE = 'state.lock'
+    STATE_TABLE = 'state'
+    STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
+                         "mtime TEXT NOT NULL, " \
+                         "md5 TEXT NOT NULL, " \
+                         "timestamp TEXT NOT NULL"
 
-    def __init__(self, project):
+    STATE_INFO_TABLE = 'state_info'
+    STATE_INFO_TABLE_LAYOUT = 'count INTEGER'
+    STATE_INFO_ROW = 1
+
+    LINK_STATE_TABLE = 'link_state'
+    LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
+                              "inode INTEGER NOT NULL, " \
+                              "mtime TEXT NOT NULL"
+
+    STATE_ROW_LIMIT = 10000000
+    STATE_ROW_CLEANUP_QUOTA = 50
+
+    MAX_INT = 2**63 - 1
+    MAX_UINT = 2**64 - 2
+
+    def __init__(self, project, config):
         self.project = project
         self.dvc_dir = project.dvc_dir
-        self._lock = threading.Lock()
-        if self.dvc_dir:
-            self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
-            self._lock_file = Lock(self.dvc_dir, self.STATE_LOCK_FILE)
-        else:
+        self.root_dir = project.root_dir
+
+        self.row_limit = 100
+        self.row_cleanup_quota = 50
+
+        c = config.get(Config.SECTION_STATE, {})
+        self.row_limit = c.get(Config.SECTION_STATE_ROW_LIMIT,
+                               self.STATE_ROW_LIMIT)
+        self.row_cleanup_quota = c.get(Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
+                                       self.STATE_ROW_CLEANUP_QUOTA)
+
+        if not self.dvc_dir:
             self.state_file = None
-            self._lock_file = threading.Lock()
-        self._db = self.load()
+            return
+
+        self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
+        self.db = None
+        self.c = None
+        self.inserts = 0
+
+    def __enter__(self):
+        self.load()
+
+    def __exit__(self, type, value, tb):
+        self.dump()
 
     @staticmethod
     def init(project):
@@ -74,34 +85,161 @@ class State(object):
 
         return actual.split('.')[0] != md5.split('.')[0]
 
-    def load(self):
-        if not self.state_file or not os.path.isfile(self.state_file):
-            return {}
+    def _execute(self, cmd):
+        Logger.debug(cmd)
+        return self.c.execute(cmd)
 
-        with open(self.state_file, 'r') as fd:
+    def _fetchall(self):
+        ret = self.c.fetchall()
+        Logger.debug("fetched: {}".format(ret))
+        return ret
+
+    def _to_sqlite(self, n):
+        assert n >= 0
+        assert n < self.MAX_UINT
+        # NOTE: sqlite stores unit as signed ints, so maximum uint is 2^63-1
+        # see http://jakegoulding.com/blog/2011/02/06/sqlite-64-bit-integers/
+        if n > self.MAX_INT:
+            ret = -(n - self.MAX_INT)
+        else:
+            ret = n
+        assert self._from_sqlite(ret) == n
+        return ret
+
+    def _from_sqlite(self, n):
+        assert abs(n) <= self.MAX_INT
+        if n < 0:
+            return abs(n) + self.MAX_INT
+        assert n < self.MAX_UINT
+        assert n >= 0
+        return n
+
+    def _load(self, empty=False):
+        from dvc import VERSION
+
+        if not empty:
+            cmd = "PRAGMA user_version;"
+            self._execute(cmd)
+            ret = self._fetchall()
+            assert len(ret) == 1
+            assert len(ret[0]) == 1
+            assert isinstance(ret[0][0], int)
+            version = ret[0][0]
+
+            if version > self.VERSION:
+                msg = "You are using an old version '{}' of dvc that is " \
+                      "using state file version '{}' which is not " \
+                      "compatible with the state file version '{}' that " \
+                      "is used in this projet. Please upgrade right now!"
+                raise DvcException(msg.format(VERSION,
+                                              self.VERSION,
+                                              version))
+            elif version < self.VERSION:
+                msg = "State file version '{}' is too old. " \
+                      "Reformatting to the current version '{}'."
+                self.project.logger.warn(msg.format(version, self.VERSION))
+                cmd = "DROP TABLE IF EXISTS {};"
+                self._execute(cmd.format(self.STATE_TABLE))
+                self._execute(cmd.format(self.STATE_INFO_TABLE))
+                self._execute(cmd.format(self.LINK_STATE_TABLE))
+
+        # Check that the state file is indeed a database
+        cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
+        self._execute(cmd.format(self.STATE_TABLE,
+                                 self.STATE_TABLE_LAYOUT))
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_TABLE_LAYOUT))
+        self._execute(cmd.format(self.LINK_STATE_TABLE,
+                                 self.LINK_STATE_TABLE_LAYOUT))
+
+        cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
+              "WHERE NOT EXISTS (SELECT * FROM {})"
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_TABLE))
+
+        cmd = "PRAGMA user_version = {};"
+        self._execute(cmd.format(self.VERSION))
+
+    def load(self):
+        retries = 1
+        while True:
+            assert self.db is None
+            assert self.c is None
+            assert self.inserts == 0
+            empty = not os.path.exists(self.state_file)
+            self.db = sqlite3.connect(self.state_file)
+            self.c = self.db.cursor()
+
+            # Try loading once to check that the file is indeed a database
+            # and reformat it if it is not.
             try:
-                return json.load(fd)
-            except ValueError as exc:
-                Logger.error('Failed to load \'{}\''.format(self.state_file),
-                             exc)
-                return {}
+                self._load(empty=empty)
+                return
+            except sqlite3.DatabaseError:
+                self.c.close()
+                self.db.close()
+                self.db = None
+                self.c = None
+                self.inserts = 0
+                if retries > 0:
+                    os.unlink(self.state_file)
+                    retries -= 1
+                else:
+                    raise
 
     def dump(self):
-        if not self.state_file:
-            return
+        assert self.db is not None
 
-        with SignalHandler():
-            with open(self.state_file, 'w+') as fd:
-                json.dump(self._db, fd)
+        cmd = "SELECT count from {} WHERE rowid={}"
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_ROW))
+        ret = self._fetchall()
+        assert len(ret) == 1
+        assert len(ret[0]) == 1
+        count = self._from_sqlite(ret[0][0]) + self.inserts
+
+        if count > self.row_limit:
+            msg = "Cleaning up state. This might take a while."
+            self.project.logger.warn(msg)
+
+            delete = (count - self.row_limit)
+            delete += int(self.row_limit * (self.row_cleanup_quota/100.))
+            cmd = "DELETE FROM {} WHERE timestamp IN (" \
+                  "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
+            self._execute(cmd.format(self.STATE_TABLE,
+                                     self.STATE_TABLE,
+                                     delete))
+
+            self._execute("VACUUM")
+
+            cmd = "SELECT COUNT(*) FROM {}"
+
+            self._execute(cmd.format(self.STATE_TABLE))
+            ret = self._fetchall()
+            assert len(ret) == 1
+            assert len(ret[0]) == 1
+            count = ret[0][0]
+
+        cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self._to_sqlite(count),
+                                 self.STATE_INFO_ROW))
+
+        self.db.commit()
+        self.c.close()
+        self.db.close()
+        self.db = None
+        self.c = None
+        self.inserts = 0
 
     @staticmethod
     def mtime(path):
         mtime = os.path.getmtime(path)
         if os.path.isdir(path):
             for root, dirs, files in os.walk(path):
-                for fname in files:
-                    f = os.path.join(root, fname)
-                    m = os.path.getmtime(f)
+                for entry in list(dirs) + list(files):
+                    p = os.path.join(root, entry)
+                    m = os.path.getmtime(p)
                     if m > mtime:
                         mtime = m
 
@@ -109,111 +247,87 @@ class State(object):
 
     @staticmethod
     def inode(path):
-        return str(System.inode(path))
+        return System.inode(path)
 
-    def _do_update(self, path, dump=True):
+    def _do_update(self, path):
         if not os.path.exists(path):
             return (None, None)
 
         mtime = self.mtime(path)
         inode = self.inode(path)
 
-        md5 = self._get(inode, mtime)
-        if md5:
-            return (md5, None)
+        cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
+                                                       self._to_sqlite(inode))
 
-        md5, info = self._collect(path)
-        state = StateEntry(md5, mtime)
-        d = state.dumpd()
-
-        with self._lock:
-            with self._lock_file:
-                self._db[inode] = d
-
-                if dump:
-                    self.dump()
+        self._execute(cmd)
+        ret = self._fetchall()
+        if len(ret) == 0:
+            md5, info = self._collect(path)
+            cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
+                  'VALUES ({}, "{}", "{}", "{}")'
+            self._execute(cmd.format(self.STATE_TABLE,
+                                     self._to_sqlite(inode),
+                                     mtime,
+                                     md5,
+                                     int(nanotime.timestamp(time.time()))))
+            self.inserts += 1
+        else:
+            assert len(ret) == 1
+            assert len(ret[0]) == 4
+            i, m, md5, timestamp = ret[0]
+            i = self._from_sqlite(i)
+            assert i == inode
+            if mtime != m:
+                md5, info = self._collect(path)
+                cmd = 'UPDATE {} SET ' \
+                      'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
+                      'WHERE inode = {}'
+                self._execute(cmd.format(self.STATE_TABLE,
+                                         mtime,
+                                         md5,
+                                         int(nanotime.timestamp(time.time())),
+                                         self._to_sqlite(inode)))
+            else:
+                info = None
+                cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
+                self._execute(cmd.format(self.STATE_TABLE,
+                                         int(nanotime.timestamp(time.time())),
+                                         self._to_sqlite(inode)))
 
         return (md5, info)
 
-    def update(self, path, dump=True):
-        return self._do_update(path, dump=dump)[0]
+    def update(self, path):
+        return self._do_update(path)[0]
 
-    def update_info(self, path, dump=True):
-        md5, info = self._do_update(path, dump=dump)
+    def update_info(self, path):
+        md5, info = self._do_update(path)
         if not info:
             info = self.project.cache.local.load_dir_cache(md5)
         return (md5, info)
 
-    def _get(self, inode, mtime):
-        with self._lock:
-            with self._lock_file:
-                d = self._db.get(inode, None)
-
-        if not d:
-            return None
-
-        state = StateEntry.loadd(d)
-        if mtime == state.mtime:
-            return state.md5
-
-        return None
-
-    def get(self, path):
-        mtime = self.mtime(path)
-        inode = self.inode(path)
-
-        return self._get(inode, mtime)
-
-
-class LinkStateEntry(object):
-    PARAM_MTIME = 'mtime'
-    PARAM_INODE = 'inode'
-
-    def __init__(self, inode, mtime):
-        self.mtime = mtime
-        self.inode = inode
-
-    @staticmethod
-    def loadd(d):
-        mtime = d[LinkStateEntry.PARAM_MTIME]
-        inode = d[LinkStateEntry.PARAM_INODE]
-        return LinkStateEntry(inode, mtime)
-
-    def dumpd(self):
-        return {
-            self.PARAM_INODE: self.inode,
-            self.PARAM_MTIME: self.mtime,
-        }
-
-
-class LinkState(State):
-    STATE_FILE = 'link.state'
-    STATE_LOCK_FILE = STATE_FILE + '.lock'
-
-    def __init__(self, project):
-        super(LinkState, self).__init__(project)
-        self.root_dir = project.root_dir
-
-    def update(self, path, dump=True):
+    def update_link(self, path):
         if not os.path.exists(path):
             return
 
         mtime = self.mtime(path)
         inode = self.inode(path)
+        relpath = os.path.relpath(path, self.root_dir)
 
-        with self._lock:
-            state = LinkStateEntry(inode, mtime)
-            d = state.dumpd()
-            self._db[os.path.relpath(path, self.root_dir)] = d
-            if dump:
-                with self._lock_file:
-                    self.dump()
+        cmd = 'REPLACE INTO {}(path, inode, mtime) ' \
+              'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE,
+                                               relpath,
+                                               self._to_sqlite(inode),
+                                               mtime)
+        self._execute(cmd)
 
-    def _do_remove_unused(self, used):
-        items = self._db.copy().items()
-        for p, s in items:
+    def remove_unused_links(self, used):
+        unused = []
+
+        self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
+        for row in self.c:
+            p, i, m = row
+            i = self._from_sqlite(i)
             path = os.path.join(self.root_dir, p)
-            state = LinkStateEntry.loadd(s)
 
             if path in used:
                 continue
@@ -224,13 +338,11 @@ class LinkState(State):
             inode = self.inode(path)
             mtime = self.mtime(path)
 
-            if inode == state.inode and mtime == state.mtime:
+            if i == inode and m == mtime:
                 Logger.debug('Removing \'{}\' as unused link.'.format(path))
                 remove(path)
-                del self._db[p]
+                unused.append(p)
 
-    def remove_unused(self, used):
-        with self._lock:
-            with self._lock_file:
-                self._do_remove_unused(used)
-                self.dump()
+        for p in unused:
+            cmd = 'DELETE FROM {} WHERE path = "{}"'
+            self._execute(cmd.format(self.LINK_STATE_TABLE, p))

dvc/system.py

@@ -1,9 +1,4 @@
 import os
-import ctypes
-import reflink
-
-if os.name == 'nt':
-    import ntfsutils.hardlink as winlink
 
 
 class System(object):
@@ -13,13 +8,24 @@ class System(object):
 
     @staticmethod
     def hardlink(source, link_name):
+        import ctypes
         if System.is_unix():
-            return os.link(source, link_name)
+            os.link(source, link_name)
+            return
 
-        return winlink.create(source, link_name)
+        CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
+        CreateHardLink.argtypes = [ctypes.c_wchar_p,
+                                   ctypes.c_wchar_p,
+                                   ctypes.c_void_p]
+        CreateHardLink.restype = ctypes.wintypes.BOOL
+
+        res = CreateHardLink(link_name, source, None)
+        if res == 0:
+            raise ctypes.WinError()
 
     @staticmethod
     def symlink(source, link_name):
+        import ctypes
         if System.is_unix():
             return os.symlink(source, link_name)
 
@@ -35,42 +41,150 @@ class System(object):
             raise ctypes.WinError()
 
     @staticmethod
+    def _reflink_darwin(src, dst):
+        import ctypes
+
+        clib = ctypes.CDLL('libc.dylib')
+        if not hasattr(clib, 'clonefile'):
+            return -1
+
+        clonefile = clib.clonefile
+        clonefile.argtypes = [ctypes.c_char_p,
+                              ctypes.c_char_p,
+                              ctypes.c_int]
+        clonefile.restype = ctypes.c_int
+
+        return clonefile(ctypes.c_char_p(src.encode('utf-8')),
+                         ctypes.c_char_p(dst.encode('utf-8')),
+                         ctypes.c_int(0))
+
+    @staticmethod
+    def _reflink_windows(src, dst):
+        return -1
+
+    @staticmethod
+    def _reflink_linux(src, dst):
+        import os
+        import fcntl
+
+        FICLONE = 0x40049409
+
+        s = open(src, 'r')
+        d = open(dst, 'w+')
+
+        try:
+            ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
+        except IOError:
+            s.close()
+            d.close()
+            os.unlink(dst)
+            raise
+
+        s.close()
+        d.close()
+
+        if ret != 0:
+            os.unlink(dst)
+
+        return ret
+
+    @staticmethod
     def reflink(source, link_name):
-        return reflink.reflink(source, link_name)
+        import platform
+        from dvc.exceptions import DvcException
+
+        system = platform.system()
+        try:
+            if system == 'Windows':
+                ret = System._reflink_windows(source, link_name)
+            elif system == 'Darwin':
+                ret = System._reflink_darwin(source, link_name)
+            elif system == 'Linux':
+                ret = System._reflink_linux(source, link_name)
+            else:
+                ret = -1
+        except IOError:
+            ret = -1
+
+        if ret != 0:
+            raise DvcException('Reflink is not supported')
 
-    # FIXME
-    # Temporary fix while waiting for the PR to be merged and released:
-    # https://github.com/sunshowers/ntfs/pull/11
     @staticmethod
     def getdirinfo(path):
-        from ntfsutils.fs import FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_READ
-        from ntfsutils.fs import OPEN_EXISTING, BY_HANDLE_FILE_INFORMATION
-        from ntfsutils.fs import CreateFile, WinError
-        from ntfsutils.fs import GetFileInformationByHandle, CloseHandle
+        import ctypes
+        from ctypes import c_void_p, c_wchar_p, Structure, WinError, POINTER
+        from ctypes.wintypes import DWORD, HANDLE, BOOL
+
+        FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+        FILE_SHARE_READ = 0x00000001
+        OPEN_EXISTING = 3
+
+        class FILETIME(Structure):
+            _fields_ = [("dwLowDateTime", DWORD),
+                        ("dwHighDateTime", DWORD)]
+
+        class BY_HANDLE_FILE_INFORMATION(Structure):
+            _fields_ = [("dwFileAttributes", DWORD),
+                        ("ftCreationTime", FILETIME),
+                        ("ftLastAccessTime", FILETIME),
+                        ("ftLastWriteTime", FILETIME),
+                        ("dwVolumeSerialNumber", DWORD),
+                        ("nFileSizeHigh", DWORD),
+                        ("nFileSizeLow", DWORD),
+                        ("nNumberOfLinks", DWORD),
+                        ("nFileIndexHigh", DWORD),
+                        ("nFileIndexLow", DWORD)]
+
         flags = FILE_FLAG_BACKUP_SEMANTICS
-        hfile = CreateFile(path,
-                           0,
-                           FILE_SHARE_READ,
-                           None,
-                           OPEN_EXISTING,
-                           flags,
-                           None)
+
+        func = ctypes.windll.kernel32.CreateFileW
+        func.argtypes = [c_wchar_p,
+                         DWORD,
+                         DWORD,
+                         c_void_p,
+                         DWORD,
+                         DWORD,
+                         HANDLE]
+        func.restype = HANDLE
+
+        hfile = func(path,
+                     0,
+                     FILE_SHARE_READ,
+                     None,
+                     OPEN_EXISTING,
+                     flags,
+                     None)
         if hfile is None:
             raise WinError()
+
+        func = ctypes.windll.kernel32.GetFileInformationByHandle
+        func.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
+        func.restype = BOOL
+
         info = BY_HANDLE_FILE_INFORMATION()
-        rv = GetFileInformationByHandle(hfile, info)
-        CloseHandle(hfile)
+        rv = func(hfile, info)
+
+        func = ctypes.windll.kernel32.CloseHandle
+        func.argtypes = [HANDLE]
+        func.restype = BOOL
+
+        func(hfile)
+
         if rv == 0:
             raise WinError()
+
         return info
 
     @staticmethod
     def inode(path):
         if System.is_unix():
-            return os.lstat(path).st_ino
-
-        # getdirinfo from ntfsutils works on both files and dirs
-        info = System.getdirinfo(path)
-        return hash((info.dwVolumeSerialNumber,
-                     info.nFileIndexHigh,
-                     info.nFileIndexLow))
+            inode = os.lstat(path).st_ino
+        else:
+            # getdirinfo from ntfsutils works on both files and dirs
+            info = System.getdirinfo(path)
+            inode = abs(hash((info.dwVolumeSerialNumber,
+                              info.nFileIndexHigh,
+                              info.nFileIndexLow)))
+        assert inode >= 0
+        assert inode < 2**64
+        return inode

dvc/updater.py

@@ -6,11 +6,11 @@ from dvc import VERSION_BASE
 from dvc.logger import Logger
 
 
-class Updater(object):
+class Updater(object):  # pragma: no cover
     URL = 'https://4ki8820rsf.execute-api.us-east-2.amazonaws.com/' \
           'prod/latest-version'
     UPDATER_FILE = 'updater'
-    TIMEOUT = 7 * 24 * 60 * 60  # every week
+    TIMEOUT = 24 * 60 * 60  # every day
     TIMEOUT_GET = 10
 
     def __init__(self, dvc_dir):

requirements.txt

@@ -4,23 +4,22 @@ ply>=3.9 # See https://github.com/pyinstaller/pyinstaller/issues/1945
 configparser>=3.5.0
 zc.lockfile>=1.2.1
 future>=0.16.0
-google-cloud==0.32.0
+google-cloud-storage==1.12.0
 PyInstaller==3.3.1
 colorama>=0.3.9
 configobj>=5.0.6
-networkx==2.1
+networkx>=2.1
 pyyaml>=3.12
 gitpython>=2.1.8
-ntfsutils>=0.1.4
 setuptools>=34.0.0
 nanotime>=0.5.2
 pyasn1>=0.4.1
 schema>=0.6.7
 paramiko>=2.4.1
 jsonpath-rw==1.4.0
-reflink==0.2.0
 requests>=2.18.4
 wheel>=0.31.1
 futures>=3.2.0; python_version == "2.7"
 grandalf==0.6
-asciicanvas==0.0.3
+pydot>=1.2.4
+asciimatics>=1.10.0

scripts/ci/install.sh

@@ -14,7 +14,7 @@ function retry {
 # NOTE: it is not uncommon for pip to hang on travis for what seems to be
 # networking issues. Thus, let's retry a few times to see if it will eventially
 # work or not.
-retry pip install --upgrade pip
+retry pip install --upgrade pip setuptools wheel
 retry pip install -r requirements.txt
 retry pip install -r test-requirements.txt
 

scripts/ci/script.sh

@@ -6,7 +6,8 @@ set -e
 python -mtests
 
 if [[ "$TRAVIS_PULL_REQUEST" == "false" && \
-      "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
+      "$TRAVIS_SECURE_ENV_VARS" == "true" && \
+      "$(python -c 'import sys; print(sys.version_info[0])')" == '2' ]]; then
 	codecov
 fi
 

scripts/completion/dvc.bash

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+#----------------------------------------------------------
+# Repository:  https://github.com/iterative/dvc
+#
+# References:
+#   - https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion.html
+#   - https://opensource.com/article/18/3/creating-bash-completion-script
+#----------------------------------------------------------
+
+_dvc_commands='init destroy add import checkout run pull push fetch \
+              status repro remove move gc config remote metrics install \
+              root lock unlock pipeline'
+
+_dvc_global_options="-h --help -q --quiet -V --verbose"
+_dvc_options="-h --help -v --version"
+_dvc_init="--no-scm -f --force"
+_dvc_destroy="-f --force"
+_dvc_add="-R --recursive"
+_dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes"
+_dvc_pull="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps"
+_dvc_push="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps"
+_dvc_fetch="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps"
+_dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud"
+_dvc_repro="--dry -f --force -s --single-item -c --cwd -m --metrics -i --interactive -p --pipeline -P --all-pipelines"
+_dvc_remove="--dry -o --outs -p --purge"
+_dvc_gc="-a --all-branches -T --all-tags -c --cloud -r --remote"
+_dvc_config="-u --unset --local"
+_dvc_checkout="$(compgen -G '*.dvc')"
+_dvc_move="$(compgen -G '*')"
+_dvc_lock="$(compgen -G '*.dvc')"
+_dvc_unlock="$(compgen -G '*.dvc')"
+_dvc_import=""
+_dvc_remote=""
+_dvc_metrics=""
+_dvc_install=""
+_dvc_root=""
+_dvc_pipeline=""
+
+# Notes:
+#
+# `COMPREPLY` contains what will be rendered after completion is triggered
+#
+# `word` refers to the current typed word
+#
+# `${!var}` is to evaluate the content of `var` and expand its content as a variable
+#
+#       hello="world"
+#       x="hello"
+#       ${!x} ->  ${hello} ->  "world"
+#
+_dvc () {
+  local word="${COMP_WORDS[COMP_CWORD]}"
+
+  COMPREPLY=()
+
+  if [ "${COMP_CWORD}" -eq 1 ]; then
+    case "$word" in
+      -*) COMPREPLY=($(compgen -W "$_dvc_options" -- "$word")) ;;
+      *)  COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
+    esac
+  elif [ "${COMP_CWORD}" -eq 2 ]; then
+    local options_list="_dvc_${COMP_WORDS[1]}"
+
+    COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
+  fi
+
+  return 0
+}
+
+complete -F _dvc dvc

scripts/completion/dvc.zsh

@@ -0,0 +1,218 @@
+#compdef dvc
+
+#----------------------------------------------------------
+# Repository:  https://github.com/iterative/dvc
+#
+# References:
+#   - https://github.com/zsh-users/zsh-completions
+#   - http://zsh.sourceforge.net/Doc/Release/Completion-System.html
+#   - https://mads-hartmann.com/2017/08/06/writing-zsh-completion-scripts.html
+#   - http://www.linux-mag.com/id/1106/
+#----------------------------------------------------------
+
+_dvc_commands() {
+  local _commands=(
+    "init:Initialize dvc over a directory (should already be a git dir)."
+    "destroy:Destroy dvc. Will remove all project's information, data files and cache."
+    "add:Add files/directories to dvc."
+    "import:Import files from URL."
+    "checkout:Checkout data files from cache."
+    "run:Generate a stage file from a given command and execute the command."
+    "pull:Pull data files from the cloud."
+    "push:Push data files to the cloud."
+    "fetch:Fetch data files from the cloud."
+    "status:Show the project status."
+    "repro:Reproduce DVC file. Default file name - 'Dvcfile'."
+    "remove:Remove outputs of DVC file."
+    "move:Move output of DVC file."
+    "gc:Collect garbage."
+    "config:Get or set config options."
+    "remote:Manage set of tracked repositories."
+    "metrics:Get metrics from all branches."
+    "install:Install dvc hooks into the repository."
+    "root:Relative path to project's directory."
+    "lock:Lock DVC file."
+    "unlock:Unlock DVC file."
+    "pipeline:Manage pipeline."
+  )
+
+  _describe 'dvc commands' _commands
+}
+
+_dvc_global_options=(
+  "(-)"{-h,--help}"[Show help message related to the command]"
+  "(-)"{-q,--quiet}"[Be quiet.]"
+  "(-)"{-V,--verbose}"[Be verbose.]"
+)
+
+_dvc_options=(
+  "(-)"{-h,--help}"[Show this help message and exit]"
+  "(-)"{-v,--version}"[Show program's version]"
+)
+
+_dvc_init=(
+  "--no-scm[Initiate dvc in directory that is not tracket by any scm tool]"
+  {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
+)
+
+_dvc_destroy=(
+  {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
+)
+
+_dvc_add=(
+  {-R,--recursive}"[Recursively add each file under the directory.]"
+)
+
+_dvc_import=(
+  "1:URL:"
+  "2:Output:"
+)
+
+_dvc_checkout=(
+  "1:DVC files:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_run=(
+  "--no-exec[Only create stage file without actually running it.]"
+  {-f,--file}"[Specify name of the stage file.]:File:_files"
+  {-c,--cwd}"[Directory to run your command and place state file in.]:CWD:_files -/"
+  "*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files"
+  "*"{-o,--outs}"[Declare output data file or data directory.]:Output data:_files"
+  "*"{-O,--outs-no-cache}"[Declare output regular file or directory.]:Output regular:_files"
+  "*"{-M,--metrics-no-cache}"[Declare output metric file or directory]:Metrics (no cache):_files"
+  {-y,--yes}"[Automatic 'yes' answer to all prompts.]"
+)
+
+_dvc_pull=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+)
+
+_dvc_push=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+)
+
+_dvc_fetch=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+)
+
+_dvc_status=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+  {-c,--cloud}"[Show status of a local cache compared to a remote repository]"
+)
+
+_dvc_repro=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--dry[Only print the commands that would be executed without actually executing]"
+  {-f,--force}"[Reproduce even if dependencies were not changed.]"
+  {-s,--single-item}"[Reproduce only single data item without recursive dependencies check.]"
+  {-c,--cwd}"[Directory to reproduce from]:CWD:_files -/"
+  {-m,--metrics}"[Show metrics after reproduction.]"
+  {-i,--interactive}"[Ask for confirmation before reproducing each stage.]"
+  {-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]"
+  {-P,--all-pipelines}"[Reproduce all pipelines in the project.]"
+)
+
+_dvc_remove=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--dry[Only print the commands that would be executed without actually executing]"
+  {-o,--outs}"[Only remove DVC file outputs.(default)]"
+  {-p,--purge}"[Remove DVC file and all its outputs]"
+)
+
+_dvc_move=(
+  "1:Source:_files"
+  "2:Destination:"
+)
+
+_dvc_gc=(
+  {-a,--all-branches}"[Collect garbage for all branches]"
+  {-T,--all-tags}"[Collect garbage for all tags]"
+  {-c,--cloud}"[Collect garbage in remote repository]"
+  {-r,--remote}"[Remote repository to collect garbage in]:Remote repository:"
+)
+
+_dvc_config=(
+  {-u,--unset}"[Unset option.]"
+  "--local[Unset option.]"
+)
+
+_dvc_remote=(
+  "1:Task:(add remove modify list)"
+)
+
+_dvc_metrics=(
+  "1:Task:(show add modify remove)"
+)
+
+_dvc_install=()
+
+_dvc_root=()
+
+_dvc_lock=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_unlock=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_pipeline=(
+  "1:Task:(show)"
+)
+
+typeset -A opt_args
+local context state line curcontext="$curcontext"
+
+_arguments \
+  $_dvc_options \
+  '1: :_dvc_commands' \
+  '*::args:->args'
+
+case $words[1] in
+  init)     _arguments $_dvc_global_options   $_dvc_init     ;;
+  destroy)  _arguments $_dvc_global_options   $_dvc_destroy  ;;
+  add)      _arguments $_dvc_global_options   $_dvc_add      ;;
+  import)   _arguments $_dvc_global_options   $_dvc_import   ;;
+  checkout) _arguments $_dvc_global_options   $_dvc_checkout ;;
+  run)      _arguments $_dvc_global_options   $_dvc_run      ;;
+  pull)     _arguments $_dvc_global_options   $_dvc_pull     ;;
+  push)     _arguments $_dvc_global_options   $_dvc_push     ;;
+  fetch)    _arguments $_dvc_global_options   $_dvc_fetch    ;;
+  status)   _arguments $_dvc_global_options   $_dvc_status   ;;
+  repro)    _arguments $_dvc_global_options   $_dvc_repro    ;;
+  remove)   _arguments $_dvc_global_options   $_dvc_remove   ;;
+  move)     _arguments $_dvc_global_options   $_dvc_move     ;;
+  gc)       _arguments $_dvc_global_options   $_dvc_gc       ;;
+  config)   _arguments $_dvc_global_options   $_dvc_config   ;;
+  remote)   _arguments $_dvc_global_options   $_dvc_remote   ;;
+  metrics)  _arguments $_dvc_global_options   $_dvc_metrics  ;;
+  install)  _arguments $_dvc_global_options   $_dvc_install  ;;
+  root)     _arguments $_dvc_global_options   $_dvc_root     ;;
+  lock)     _arguments $_dvc_global_options   $_dvc_lock     ;;
+  unlock)   _arguments $_dvc_global_options   $_dvc_unlock   ;;
+  pipeline) _arguments $_dvc_global_options   $_dvc_pipeline ;;
+esac

setup.py

@@ -12,24 +12,22 @@ install_requires = [
     "future>=0.16.0",
     "colorama>=0.3.9",
     "configobj>=5.0.6",
-    "networkx==2.1",
+    "networkx>=2.1",
     "pyyaml>=3.12",
     "gitpython>=2.1.8",
-    "ntfsutils>=0.1.4",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
     "schema>=0.6.7",
     "jsonpath-rw==1.4.0",
-    "reflink==0.2.0",
     "requests>=2.18.4",
     "grandalf==0.6",
-    "asciicanvas==0.0.3",
+    "asciimatics>=1.10.0",
 ]
 
 # Extra dependencies for remote integrations
 gs = [
-    "google-cloud==0.32.0",
+    "google-cloud-storage==1.12.0",
 ]
 s3 = [
     "boto3==1.7.4",
@@ -67,11 +65,11 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 3',
     ],
-    packages=find_packages(exclude=['bin', 'tests', 'functests']),
+    packages=find_packages(exclude=['tests']),
     include_package_data=True,
     url='http://dataversioncontrol.com',
     entry_points={
         'console_scripts': ['dvc = dvc.main:main']
     },
-    zip_safe=False
+    zip_safe=False,
 )

test-requirements.txt

@@ -1,8 +1,8 @@
 pytest>=3.3.1
 nose>=1.3.7
 mock>=2.0.0
-coverage
-codecov
+coverage>=4.5.1
+codecov>=2.0.15
 xmltodict>=0.11.0
 awscli>=1.14.18
 google-compute-engine

tests/__main__.py

@@ -7,4 +7,5 @@ os.chdir(REPO_ROOT)
 os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), os.getenv('PATH')))
 os.putenv('DVC_HOME', REPO_ROOT)
 
-check_call('nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage', shell=True)
+cmd = 'nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage'
+check_call(cmd, shell=True)

tests/basic_env.py

@@ -1,4 +1,5 @@
 import os
+import uuid
 import shutil
 import tempfile
 from git import Repo
@@ -38,8 +39,15 @@ class TestDir(TestCase):
         with open(name, 'a') as f:
             f.write(contents)
 
+    @staticmethod
+    def mkdtemp():
+        prefix = 'dvc-test.{}.'.format(os.getpid())
+        suffix = '.{}'.format(uuid.uuid4())
+        return tempfile.mkdtemp(prefix=prefix, suffix=suffix)
+
     def setUp(self):
-        self._root_dir = tempfile.mkdtemp()
+        self._root_dir = TestDir.mkdtemp()
+
         self._pushd(self._root_dir)
         self.create(self.FOO, self.FOO_CONTENTS)
         self.create(self.BAR, self.BAR_CONTENTS)
@@ -61,6 +69,15 @@ class TestGit(TestDir):
         self.git.index.commit('add code')
 
 
+class TestGitSubmodule(TestGit):
+    def setUp(self):
+        super(TestGitSubmodule, self).setUp()
+        subrepo = Repo.init()
+        subrepo_path = 'subrepo'
+        self.git.create_submodule(subrepo_path, subrepo_path, subrepo.git_dir)
+        self._pushd(subrepo_path)
+
+
 class TestDvc(TestGit):
     def setUp(self):
         super(TestDvc, self).setUp()

tests/test_add.py

@@ -2,7 +2,6 @@ import os
 import stat
 import shutil
 import filecmp
-import tempfile
 
 from dvc.main import main
 from dvc.utils import file_md5
@@ -132,7 +131,7 @@ class TestAddFileInDir(TestDvc):
 
 class TestAddExternalLocalFile(TestDvc):
     def test(self):
-        dname = tempfile.mkdtemp()
+        dname = TestDvc.mkdtemp()
         fname = os.path.join(dname, 'foo')
         shutil.copyfile(self.FOO, fname)
 

tests/test_cache.py

@@ -1,6 +1,5 @@
 import os
 import shutil
-import tempfile
 
 from dvc.cache import Cache
 from dvc.system import System
@@ -52,7 +51,7 @@ class TestCacheLoadBadDirCache(TestDvc):
 
 class TestExternalCacheDir(TestDvc):
     def test(self):
-        cache_dir = tempfile.mkdtemp()
+        cache_dir = TestDvc.mkdtemp()
 
         ret = main(['config', 'cache.dir', cache_dir])
         self.assertEqual(ret, 0)
@@ -103,3 +102,12 @@ class TestSharedCacheDir(TestDir):
         self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1)
         self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1)
         self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1)
+
+
+class TestCacheLinkType(TestDvc):
+    def test(self):
+        ret = main(['config', 'cache.type', 'reflink,copy'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['add', self.FOO])
+        self.assertEqual(ret, 0)

tests/test_checkout.py

@@ -6,6 +6,7 @@ import shutil
 import filecmp
 
 from dvc.main import main
+from dvc.project import Project
 from tests.basic_env import TestDvc
 from tests.test_repro import TestRepro
 from dvc.stage import Stage
@@ -65,6 +66,34 @@ class TestCheckoutCorruptedCacheFile(TestRepro):
         self.assertFalse(os.path.isfile(cache))
 
 
+class TestCheckoutCorruptedCacheDir(TestDvc):
+    def test(self):
+        time.sleep(1)
+        
+        # NOTE: using 'copy' so that cache and link don't have same inode
+        ret = main(['config', 'cache.type', 'copy'])
+        self.assertEqual(ret, 0)
+
+        self.dvc = Project('.')
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.assertEqual(len(stages[0].outs), 1)
+        out = stages[0].outs[0]
+
+        # NOTE: modifying cache file for one of the files inside the directory
+        # to check if dvc will detect that the cache is corrupted.
+        entry = self.dvc.cache.local.load_dir_cache(out.md5)[0]
+        md5 = entry[self.dvc.cache.local.PARAM_MD5]
+        cache = self.dvc.cache.local.get(md5)
+
+        with open(cache, 'w+') as fobj:
+            fobj.write('1')
+
+        self.dvc.checkout()
+
+        self.assertFalse(os.path.exists(cache))
+
+
 class TestCmdCheckout(TestCheckout):
     def test(self):
         ret = main(['checkout'])
@@ -99,7 +128,8 @@ class TestRemoveFilesWhenCheckout(CheckoutBase):
 
         # add the file into a separate branch
         self.dvc.scm.checkout(branch_1, True)
-        main(['checkout'])
+        ret = main(['checkout'])
+        self.assertEqual(ret, 0)
         self.commit_data_file(fname)
 
         # Checkout back in master
@@ -108,7 +138,8 @@ class TestRemoveFilesWhenCheckout(CheckoutBase):
 
         # Make sure `dvc checkout` removes the file
         # self.dvc.checkout()
-        main(['checkout'])
+        ret = main(['checkout'])
+        self.assertEqual(ret, 0)
         self.assertFalse(os.path.exists(fname))
 
 
@@ -151,11 +182,13 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
         self.commit_data_file(fname_master)
 
         self.dvc.scm.checkout(branch_1, True)
-        main(['checkout'])
+        ret = main(['checkout'])
+        self.assertEqual(ret, 0)
         self.commit_data_file(fname_branch)
 
         self.dvc.scm.checkout(branch_master)
-        main(['checkout'])
+        ret = main(['checkout'])
+        self.assertEqual(ret, 0)
 
         ignored = self.read_ignored()
 
@@ -163,7 +196,8 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
         self.assertIn(fname_master, ignored)
 
         self.dvc.scm.checkout(branch_1)
-        main(['checkout'])
+        ret = main(['checkout'])
+        self.assertEqual(ret, 0)
         ignored = self.read_ignored()
         self.assertIn(fname_branch, ignored)
 
@@ -200,3 +234,15 @@ class TestCheckoutEmptyDir(TestDvc):
 
         self.assertTrue(os.path.isdir(dname))
         self.assertEqual(len(os.listdir(dname)), 0)
+
+
+class TestCheckoutNotCachedFile(TestDvc):
+    def test(self):
+        cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
+
+        self.dvc.add(self.FOO)
+        stage = self.dvc.run(cmd=cmd,
+                             deps=[self.FOO, self.CODE],
+                             outs_no_cache=['out'])
+
+        self.dvc.checkout()

tests/test_data_cloud.py

@@ -1,12 +1,11 @@
 from subprocess import CalledProcessError
-from subprocess import check_output
+from subprocess import check_output, Popen
 from unittest import SkipTest
 import os
 import time
 import uuid
 import shutil
 import getpass
-import tempfile
 import platform
 
 from dvc.main import main
@@ -94,11 +93,18 @@ def _should_test_hdfs():
     except (CalledProcessError, IOError):
         return False
 
+    p = Popen('hadoop fs -ls hdfs://127.0.0.1/',
+              shell=True,
+              executable=os.getenv('SHELL'))
+    p.communicate()
+    if p.returncode != 0:
+        return False
+
     return True
 
 
 def get_local_storagepath():
-    return tempfile.mkdtemp()
+    return TestDvc.mkdtemp()
 
 
 def get_local_url():
@@ -157,7 +163,7 @@ class TestDataCloud(TestDvc):
         clist = [('s3://', RemoteS3),
                  ('gs://', RemoteGS),
                  ('ssh://user@localhost:/', RemoteSSH),
-                 (tempfile.mkdtemp(), RemoteLOCAL)]
+                 (TestDvc.mkdtemp(), RemoteLOCAL)]
 
         if _should_test_azure():
             clist.append(('azure://ContainerName=', RemoteAzure))
@@ -221,55 +227,56 @@ class TestDataCloudBase(TestDvc):
         info_dir = stage_dir.outs[0].dumpd()
         md5_dir = info_dir['md5']
 
-        # Check status
-        sleep()
-        status = self.cloud.status([info], show_checksums=True)
-        self.assertEqual([(md5, STATUS_NEW)], status)
+        with self.cloud.project.state:
+            # Check status
+            sleep()
+            status = self.cloud.status([info], show_checksums=True)
+            self.assertEqual([(md5, STATUS_NEW)], status)
 
-        status_dir = self.cloud.status([info_dir], show_checksums=True)
-        self.assertTrue((md5_dir, STATUS_NEW) in status_dir)
+            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            self.assertTrue((md5_dir, STATUS_NEW) in status_dir)
 
-        # Push and check status
-        self.cloud.push([info])
-        self.assertTrue(os.path.exists(cache))
-        self.assertTrue(os.path.isfile(cache))
+            # Push and check status
+            self.cloud.push([info])
+            self.assertTrue(os.path.exists(cache))
+            self.assertTrue(os.path.isfile(cache))
 
-        self.cloud.push([info_dir])
-        self.assertTrue(os.path.isfile(cache_dir))
+            self.cloud.push([info_dir])
+            self.assertTrue(os.path.isfile(cache_dir))
 
-        sleep()
-        status = self.cloud.status([info], show_checksums=True)
-        self.assertEqual([(md5, STATUS_OK)], status)
+            sleep()
+            status = self.cloud.status([info], show_checksums=True)
+            self.assertEqual([(md5, STATUS_OK)], status)
 
-        status_dir = self.cloud.status([info_dir], show_checksums=True)
-        self.assertTrue((md5_dir, STATUS_OK) in status_dir)
+            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            self.assertTrue((md5_dir, STATUS_OK) in status_dir)
 
-        # Remove and check status
-        sleep()
-        shutil.rmtree(self.dvc.cache.local.cache_dir)
+            # Remove and check status
+            sleep()
+            shutil.rmtree(self.dvc.cache.local.cache_dir)
 
-        status = self.cloud.status([info], show_checksums=True)
-        self.assertEqual([(md5, STATUS_DELETED)], status)
+            status = self.cloud.status([info], show_checksums=True)
+            self.assertEqual([(md5, STATUS_DELETED)], status)
 
-        status_dir = self.cloud.status([info_dir], show_checksums=True)
-        self.assertEqual([(md5_dir, STATUS_DELETED)], status_dir)
+            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            self.assertEqual([(md5_dir, STATUS_DELETED)], status_dir)
 
-        # Pull and check status
-        self.cloud.pull([info])
-        self.assertTrue(os.path.exists(cache))
-        self.assertTrue(os.path.isfile(cache))
-        with open(cache, 'r') as fd:
-            self.assertEqual(fd.read(), self.FOO_CONTENTS)
+            # Pull and check status
+            self.cloud.pull([info])
+            self.assertTrue(os.path.exists(cache))
+            self.assertTrue(os.path.isfile(cache))
+            with open(cache, 'r') as fd:
+                self.assertEqual(fd.read(), self.FOO_CONTENTS)
 
-        self.cloud.pull([info_dir])
-        self.assertTrue(os.path.isfile(cache_dir))
+            self.cloud.pull([info_dir])
+            self.assertTrue(os.path.isfile(cache_dir))
 
-        sleep()
-        status = self.cloud.status([info], show_checksums=True)
-        self.assertEqual([(md5, STATUS_OK)], status)
+            sleep()
+            status = self.cloud.status([info], show_checksums=True)
+            self.assertEqual([(md5, STATUS_OK)], status)
 
-        status_dir = self.cloud.status([info_dir], show_checksums=True)
-        self.assertTrue((md5_dir, STATUS_OK) in status_dir)
+            status_dir = self.cloud.status([info_dir], show_checksums=True)
+            self.assertTrue((md5_dir, STATUS_OK) in status_dir)
 
     def test(self):
         self._ensure_should_run()
@@ -335,9 +342,6 @@ class TestRemoteSSH(TestDataCloudBase):
     def _get_cloud_class(self):
         return RemoteSSH
 
-    def test(self):
-        super(TestRemoteSSH, self).test()
-
 
 class TestRemoteHDFS(TestDataCloudBase):
     def _should_test(self):
@@ -349,9 +353,6 @@ class TestRemoteHDFS(TestDataCloudBase):
     def _get_cloud_class(self):
         return RemoteHDFS
 
-    def test(self):
-        super(TestRemoteHDFS, self).test()
-
 
 class TestDataCloudCLIBase(TestDvc):
     def main(self, args):

tests/test_gc.py

@@ -29,7 +29,8 @@ class TestGC(TestDvc):
         self._test_gc()
 
     def test_cli(self):
-        ret = main(['gc'])
+        ret = main(['gc', '-f'])
+        self.assertEqual(ret, 0)
         self._test_gc()
 
     def _test_gc(self):

tests/test_move.py

@@ -1,9 +1,10 @@
 import os
 
 from dvc.main import main
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, MoveNotDataSourceError
 
 from tests.basic_env import TestDvc
+from tests.test_repro import TestRepro
 
 
 class TestMove(TestDvc):
@@ -39,3 +40,15 @@ class TestCmdMove(TestDvc):
 
         ret = main(['move', 'non-existing-file', 'dst'])
         self.assertNotEqual(ret, 0)
+
+
+class TestMoveNotDataSource(TestRepro):
+    def test(self):
+        from dvc.project import Project
+
+        self.dvc = Project(self._root_dir)
+        with self.assertRaises(MoveNotDataSourceError):
+            self.dvc.move(self.file1, 'dst')
+
+        ret = main(['move', self.file1, 'dst'])
+        self.assertNotEqual(ret, 0)

tests/test_pipeline.py

@@ -2,12 +2,14 @@ from dvc.main import main
 
 from tests.basic_env import TestDvc
 from tests.test_repro import TestRepro, TestReproChangedDeepData
+import os
 
 
 class TestPipelineShowSingle(TestDvc):
     def setUp(self):
         super(TestPipelineShowSingle, self).setUp()
         self.stage = 'foo.dvc'
+        self.dotFile = 'graph.dot'
         ret = main(['add', self.FOO])
         self.assertEqual(ret, 0)
 
@@ -27,6 +29,11 @@ class TestPipelineShowSingle(TestDvc):
         ret = main(['pipeline', 'show', '--ascii', self.stage])
         self.assertEqual(ret, 0)
 
+    def test_dot(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_ascii_commands(self):
         ret = main(['pipeline', 'show', '--ascii', self.stage, '--commands'])
         self.assertEqual(ret, 0)
@@ -35,6 +42,16 @@ class TestPipelineShowSingle(TestDvc):
         ret = main(['pipeline', 'show', '--ascii', self.stage, '--outs'])
         self.assertEqual(ret, 0)
 
+    def test_dot_commands(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, '--commands'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_dot_outs(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, '--outs'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_not_dvc_file(self):
         ret = main(['pipeline', 'show', self.FOO])
         self.assertNotEqual(ret, 0)
@@ -45,6 +62,10 @@ class TestPipelineShowSingle(TestDvc):
 
 
 class TestPipelineShow(TestRepro):
+    def setUp(self):
+        super(TestPipelineShow, self).setUp()
+        self.dotFile = 'graph.dot'
+
     def test(self):
         ret = main(['pipeline', 'show', self.file1_stage])
         self.assertEqual(ret, 0)
@@ -61,6 +82,11 @@ class TestPipelineShow(TestRepro):
         ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
         self.assertEqual(ret, 0)
 
+    def test_dot(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_ascii_commands(self):
         ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands'])
         self.assertEqual(ret, 0)
@@ -69,6 +95,16 @@ class TestPipelineShow(TestRepro):
         ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs'])
         self.assertEqual(ret, 0)
 
+    def test_dot_commands(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_dot_outs(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_not_dvc_file(self):
         ret = main(['pipeline', 'show', self.file1])
         self.assertNotEqual(ret, 0)
@@ -79,6 +115,10 @@ class TestPipelineShow(TestRepro):
 
 
 class TestPipelineShowDeep(TestReproChangedDeepData):
+    def setUp(self):
+        super(TestPipelineShowDeep, self).setUp()
+        self.dotFile = 'graph.dot'
+
     def test(self):
         ret = main(['pipeline', 'show', self.file1_stage])
         self.assertEqual(ret, 0)
@@ -95,6 +135,11 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
         ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
         self.assertEqual(ret, 0)
 
+    def test_dot(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_ascii_commands(self):
         ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands'])
         self.assertEqual(ret, 0)
@@ -103,6 +148,16 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
         ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs'])
         self.assertEqual(ret, 0)
 
+    def test_dot_commands(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_dot_outs(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_not_dvc_file(self):
         ret = main(['pipeline', 'show', self.file1])
         self.assertNotEqual(ret, 0)

tests/test_remote.py

@@ -72,3 +72,16 @@ class TestRemoteRemoveDefault(TestDvc):
         core = local_config.get(Config.SECTION_CORE, None)
         if core is not None:
             self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys())
+
+
+class TestRemoteRemove(TestDvc):
+    def test(self):
+        ret = main(['config', 'core.jobs', '1'])
+        self.assertEqual(ret, 0)
+
+        remote = 'mys3'
+        ret = main(['remote', 'add', remote, 's3://bucket/name'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['remote', 'remove', remote])
+        self.assertEqual(ret, 0)

tests/test_repro.py

@@ -4,7 +4,6 @@ import stat
 import shutil
 import filecmp
 import getpass
-import tempfile
 import posixpath
 from subprocess import Popen, PIPE
 
@@ -18,6 +17,7 @@ from dvc.project import Project, ReproductionError
 from dvc.utils import file_md5
 from dvc.remote.local import RemoteLOCAL
 from dvc.stage import Stage, StageFileDoesNotExistError
+from dvc.system import System
 
 from tests.basic_env import TestDvc
 from tests.test_data_cloud import _should_test_aws, TEST_AWS_REPO_BUCKET
@@ -145,6 +145,48 @@ class TestReproDry(TestReproChangedData):
         self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
 
 
+class TestReproUpToDate(TestRepro):
+    def test(self):
+        ret = main(['repro', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+
+class TestReproDryNoExec(TestDvc):
+    def test(self):
+        deps = []
+        for d in range(3):
+            idir = 'idir{}'.format(d)
+            odir = 'odir{}'.format(d)
+
+            deps.append('-d')
+            deps.append(odir)
+
+            os.mkdir(idir)
+
+            f = os.path.join(idir, 'file')
+            with open(f, 'w+') as fobj:
+                fobj.write(str(d))
+
+            if os.name == 'nt':
+                cp = 'copy'
+            else:
+                cp = 'cp'
+
+            ret = main(['run',
+                        '--no-exec',
+                        '-d', idir,
+                        '-o', odir,
+                        "python -c 'import shutil; "
+                        "shutil.copytree(\"{}\", \"{}\")'".format(idir, odir)])
+            self.assertEqual(ret, 0)
+
+        ret = main(['run', '--no-exec', '-f', 'Dvcfile'] + deps)
+        self.assertEqual(ret, 0)
+
+        ret = main(['repro', '--dry'])
+        self.assertEqual(ret, 0)
+
+
 class TestReproChangedDeepData(TestReproChangedData):
     def setUp(self):
         super(TestReproChangedDeepData, self).setUp()
@@ -168,6 +210,61 @@ class TestReproChangedDeepData(TestReproChangedData):
         self.assertEqual(len(stages), 3)
 
 
+class TestReproPipeline(TestReproChangedDeepData):
+    def test(self):
+        stages = self.dvc.reproduce(self.file1_stage,
+                                    force=True,
+                                    pipeline=True)
+        self.assertEqual(len(stages), 3)
+
+    def test_cli(self):
+        ret = main(['repro', '--pipeline', '-f', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+
+class TestReproPipelines(TestDvc):
+    def setUp(self):
+        super(TestReproPipelines, self).setUp()
+
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.foo_stage = stages[0]
+        self.assertTrue(self.foo_stage is not None)
+
+        stages = self.dvc.add(self.BAR)
+        self.assertEqual(len(stages), 1)
+        self.bar_stage = stages[0]
+        self.assertTrue(self.bar_stage is not None)
+
+        self.file1 = 'file1'
+        self.file1_stage = self.file1 + '.dvc'
+        self.dvc.run(fname=self.file1_stage,
+                     outs=[self.file1],
+                     deps=[self.FOO, self.CODE],
+                     cmd='python {} {} {}'.format(self.CODE, self.FOO, self.file1))
+
+        self.file2 = 'file2'
+        self.file2_stage = self.file2 + '.dvc'
+        self.dvc.run(fname=self.file2_stage,
+                     outs=[self.file2],
+                     deps=[self.BAR, self.CODE],
+                     cmd='python {} {} {}'.format(self.CODE, self.BAR, self.file2))
+
+
+    def test(self):
+        stages = self.dvc.reproduce(all_pipelines=True, force=True)
+        self.assertEqual(len(stages), 4)
+        names = [stage.relpath for stage in stages]
+        self.assertTrue(self.foo_stage.relpath in names)
+        self.assertTrue(self.bar_stage.relpath in names)
+        self.assertTrue(self.file1_stage in names)
+        self.assertTrue(self.file2_stage in names)
+
+    def test_cli(self):
+        ret = main(['repro', '-f', '-P'])
+        self.assertEqual(ret, 0)
+
+
 class TestReproLocked(TestReproChangedData):
     def test(self):
         file2 = 'file2'
@@ -296,6 +393,13 @@ class TestReproChangedDirData(TestDvc):
         self.assertEqual(len(stages), 1)
         self.assertTrue(stages[0] is not None)
 
+        # Check that dvc registers mtime change for the directory.
+        System.hardlink(self.DATA_SUB, self.DATA_SUB + '.lnk')
+        sleep()
+        stages = self.dvc.reproduce(stage.path)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+
 
 class TestReproMissingMd5InStageFile(TestRepro):
     def test(self):
@@ -550,7 +654,7 @@ class TestReproExternalSSH(TestReproExternalBase):
     @property
     def bucket(self):
         if not self._dir:
-            self._dir = tempfile.mkdtemp()
+            self._dir = TestDvc.mkdtemp()
         return '{}@127.0.0.1:{}'.format(getpass.getuser(), self._dir)
 
     def cmd(self, i, o):
@@ -597,7 +701,7 @@ class TestReproExternalSSH(TestReproExternalBase):
 class TestReproExternalLOCAL(TestReproExternalBase):
     def setUp(self):
         super(TestReproExternalLOCAL, self).setUp()
-        self.tmpdir = tempfile.mkdtemp()
+        self.tmpdir = TestDvc.mkdtemp()
 
     def should_test(self):
         return True

tests/test_run.py

@@ -3,7 +3,7 @@ import filecmp
 
 from dvc.main import main
 from dvc.utils import file_md5
-from dvc.stage import Stage, StageFileBadNameError
+from dvc.stage import Stage, StageFileBadNameError, MissingDep
 from dvc.command.run import CmdRun
 from dvc.exceptions import OutputDuplicationError
 
@@ -16,7 +16,7 @@ class TestRun(TestDvc):
         deps = [self.FOO, self.CODE]
         outs = [os.path.join(self.dvc.root_dir, 'out')]
         outs_no_cache = []
-        fname = os.path.join(self.dvc.root_dir, 'out.dvc')
+        fname = 'out.dvc'
         cwd = os.curdir
 
         self.dvc.add(self.FOO)
@@ -55,6 +55,17 @@ class TestRunEmpty(TestDvc):
                      cwd=os.curdir)
 
 
+class TestRunMissingDep(TestDvc):
+    def test(self):
+        with self.assertRaises(MissingDep):
+            self.dvc.run(cmd='',
+                         deps=['non-existing-dep'],
+                         outs=[],
+                         outs_no_cache=[],
+                         fname='empty.dvc',
+                         cwd=os.curdir)
+
+
 class TestRunBadStageFilename(TestDvc):
     def test(self):
         with self.assertRaises(StageFileBadNameError):
@@ -65,6 +76,15 @@ class TestRunBadStageFilename(TestDvc):
                          fname='empty',
                          cwd=os.curdir)
 
+        with self.assertRaises(StageFileBadNameError):
+            self.dvc.run(cmd='',
+                         deps=[],
+                         outs=[],
+                         outs_no_cache=[],
+                         fname=os.path.join(self.DATA_DIR, 'empty.dvc'),
+                         cwd=os.curdir)
+
+
 
 class TestRunNoExec(TestDvc):
     def test(self):

tests/test_scm.py

@@ -4,7 +4,7 @@ from git import Repo
 
 from dvc.scm import SCM, Base, Git
 
-from tests.basic_env import TestDir, TestGit
+from tests.basic_env import TestDir, TestGit, TestGitSubmodule
 
 
 class TestSCM(TestDir):
@@ -27,6 +27,20 @@ class TestSCMGit(TestGit):
         self.assertTrue('foo' in self.git.git.ls_files())
 
 
+class TestSCMGitSubmodule(TestGitSubmodule):
+    def test_git_submodule(self):
+        self.assertIsInstance(SCM(os.curdir), Git)
+
+    def test_is_submodule(self):
+        self.assertTrue(Git.is_submodule(os.curdir))
+
+    def test_commit_in_submodule(self):
+        G = Git(self._root_dir)
+        G.add(['foo'])
+        G.commit('add')
+        self.assertTrue('foo' in self.git.git.ls_files())
+
+
 class TestIgnore(TestGit):
     def _count_gitignore(self):
         with open(Git.GITIGNORE, 'r') as fd:
@@ -35,13 +49,14 @@ class TestIgnore(TestGit):
 
     def test(self):
         git = Git(self._root_dir)
+        foo = os.path.join(self._root_dir, self.FOO)
 
-        git.ignore('foo')
+        git.ignore(foo)
         self.assertTrue(os.path.isfile(Git.GITIGNORE))
         self.assertEqual(self._count_gitignore(), 1)
 
-        git.ignore('foo')
+        git.ignore(foo)
         self.assertEqual(self._count_gitignore(), 1)
 
-        git.ignore_remove('foo')
+        git.ignore_remove(foo)
         self.assertEqual(self._count_gitignore(), 0)

tests/test_state.py

@@ -3,7 +3,7 @@ import stat
 import time
 
 from dvc.system import System
-from dvc.state import State, StateEntry
+from dvc.state import State
 from dvc.utils import file_md5
 
 from tests.basic_env import TestDvc
@@ -16,31 +16,26 @@ class TestState(TestDvc):
         mtime = os.path.getmtime(path)
         inode = System.inode(path)
 
-        state = State(self.dvc)
+        state = State(self.dvc, self.dvc.config._config)
 
-        entry_md5 = state.update(path)
-        self.assertEqual(entry_md5, md5)
+        with state:
+            entry_md5 = state.update(path)
+            self.assertEqual(entry_md5, md5)
 
-        # Sleep some time to simulate realistic behavior.
-        # Some filesystems have a bad date resolution for
-        # mtime(i.e. 1sec for HFS) that cause problems with
-        # our 'state' system not being able to distinguish
-        # files that were modified within that delta.
-        time.sleep(1)        
+            # Sleep some time to simulate realistic behavior.
+            # Some filesystems have a bad date resolution for
+            # mtime(i.e. 1sec for HFS) that cause problems with
+            # our 'state' system not being able to distinguish
+            # files that were modified within that delta.
+            time.sleep(1)        
 
-        os.unlink(path)
-        with open(path, 'w+') as fd:
-            fd.write('1')
+            os.unlink(path)
+            with open(path, 'w+') as fd:
+                fd.write('1')
 
-        md5 = file_md5(path)[0]
-        mtime = os.path.getmtime(path)
-        inode = System.inode(path)
+            md5 = file_md5(path)[0]
+            mtime = os.path.getmtime(path)
+            inode = System.inode(path)
 
-        entry_md5 = state.update(path)
-        self.assertEqual(entry_md5, md5)
-
-        # Reload state db to make sure that it stays the same
-        old_db = state._db
-        state = State(self.dvc)
-        new_db = state._db
-        self.assertEqual(old_db, new_db)
+            entry_md5 = state.update(path)
+            self.assertEqual(entry_md5, md5)