dvc: update to 0.19.13

system: add workaround for buggy os.lstat().st_ino

.codeclimate.yml

@@ -1,14 +1,5 @@
 languages:
     Python: true
 
-checks:
-    argument-count:
-        enabled: false
-    method-complexity:
-        enabled: false
-    method-lines:
-        config:
-            threshold: 50
-
 exclude_paths:
     - tests/

README.rst

@@ -4,10 +4,8 @@
 
 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_
-• `Discuss <https://discuss.dvc.org/>`_
-• `Blog <https://blog.dataversioncontrol.com/>`_
 • `Twitter <https://twitter.com/iterativeai>`_
-• `Chat <https://discord.gg/tW6t4gV>`_
+• `Chat (Community & Support) <https://dvc.org/chat>`_
 • `Tutorial <https://dvc.org/doc/tutorial>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
@@ -29,10 +27,12 @@
 
 |
 
-**Data Science Version Control** or **DVC** is an **open-source** tool for data science projects.
-With a simple and flexible Git-like architecture and interface it helps data scientists:
+**Data Science Version Control** or **DVC** is an **open-source** tool for data science and 
+machine learning projects. With a simple and flexible Git-like architecture and interface it
+helps data scientists:
 
-#. manage code together with data files and models;
+#. manage **machine learning models** - versioning, including data sets and transformations (scripts) that were
+   used to generate models;
 
 #. make projects **reproducible**;
 
@@ -112,6 +112,17 @@ or:
 
    $ brew cask install iterative/homebrew-dvc/dvc
 
+AUR
+---
+
+*Unofficial package*, any inquiries regarding the AUR package,
+`refer to the maintainer <https://github.com/mroutis/pkgbuilds>`_.
+
+.. code-block:: bash
+
+  # https://aur.archlinux.org/packages/dvc
+  $ yay -S dvc
+
 Related technologies
 ====================
 

dvc/__init__.py

@@ -7,7 +7,7 @@ import os
 import warnings
 
 
-VERSION_BASE = '0.19.1'
+VERSION_BASE = '0.19.13'
 __version__ = VERSION_BASE
 
 PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
@@ -33,13 +33,13 @@ if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
             with open(VERSIONPATH, 'w+') as fd:
                 fd.write('# AUTOGENERATED by dvc/__init__.py\n')
                 fd.write('version = "{}"\n'.format(__version__))
-        except Exception:
+        except Exception:  # pragma: no cover
             pass
-    else:
+    else:  # pragma: no cover
         # Remove version.py so that it doesn't get into the release
         if os.path.exists(VERSIONPATH):
             os.unlink(VERSIONPATH)
-else:
+else:  # pragma: no cover
     # dvc was installed with pip or something. Hopefully we have our
     # auto-generated version.py to help us provide a true version
     try:

dvc/cli.py

@@ -3,7 +3,6 @@ from __future__ import print_function
 import os
 import sys
 import argparse
-from multiprocessing import cpu_count
 
 from dvc.command.init import CmdInit
 from dvc.command.destroy import CmdDestroy
@@ -47,7 +46,7 @@ class DvcParser(argparse.ArgumentParser):
         sys.exit(2)
 
 
-class VersionAction(argparse.Action):
+class VersionAction(argparse.Action):  # pragma: no cover
     def __call__(self, parser, namespace, values, option_string=None):
         print(VERSION)
         sys.exit(0)
@@ -161,7 +160,15 @@ def parse_args(argv=None):
                         help=IMPORT_HELP)
     import_parser.add_argument(
                         'url',
-                        help='URL.')
+                        help="URL. Supported urls: "
+                             "'/path/to/file', "
+                             "'C:\\\\path\\to\\file', "
+                             "'s3://bucket/path/to/file', "
+                             "'gs://bucket/path/to/file', "
+                             "'hdfs://example.com/path/to/file', "
+                             "'ssh://example.com:/path/to/file', "
+                             "'remote://myremote/path/to/file'(see "
+                             "`dvc remote` commands). ")
     import_parser.add_argument(
                         'out',
                         help='Output.')
@@ -257,7 +264,7 @@ def parse_args(argv=None):
                         '-j',
                         '--jobs',
                         type=int,
-                        default=4*cpu_count(),
+                        default=None,
                         help='Number of jobs to run simultaneously.')
     parent_cache_parser.add_argument(
                         '--show-checksums',
@@ -287,6 +294,12 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Fetch cache for all branches.')
+    pull_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all tags.')
     pull_parser.add_argument(
                         '-d',
                         '--with-deps',
@@ -313,6 +326,12 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Push cache for all branches.')
+    push_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Push cache for all tags.')
     push_parser.add_argument(
                         '-d',
                         '--with-deps',
@@ -339,6 +358,12 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Fetch cache for all branches.')
+    fetch_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Fetch cache for all tags.')
     fetch_parser.add_argument(
                         '-d',
                         '--with-deps',
@@ -373,12 +398,19 @@ def parse_args(argv=None):
                         default=False,
                         help='Show status of a local cache compared to a '
                              'remote repository for all branches.')
+    status_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Show status of a local cache compared to a '
+                             'remote repository for all tags.')
     status_parser.add_argument(
                         '-d',
                         '--with-deps',
                         action='store_true',
                         default=False,
-                        help='Show statusof a local cache compared to a '
+                        help='Show status of a local cache compared to a '
                              'remote repository for all dependencies of the '
                              'specified target.')
     status_parser.set_defaults(func=CmdDataStatus)
@@ -439,6 +471,12 @@ def parse_args(argv=None):
                         default=False,
                         help='Reproduce the whole pipeline that the '
                              'specified stage file belongs to.')
+    repro_parser.add_argument(
+                        '-P',
+                        '--all-pipelines',
+                        action='store_true',
+                        default=False,
+                        help='Reproduce all pipelines in the project.')
     repro_parser.set_defaults(func=CmdRepro)
 
     # Remove
@@ -454,7 +492,7 @@ def parse_args(argv=None):
                         '--outs',
                         action='store_true',
                         default=True,
-                        help='Only remove DVC file outputs.')
+                        help='Only remove DVC file outputs.(default)')
     remove_parser_group.add_argument(
                         '-p',
                         '--purge',
@@ -495,6 +533,12 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Collect garbage for all branches.')
+    gc_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Collect garbage for all tags.')
     gc_parser.add_argument(
                         '-c',
                         '--cloud',
@@ -505,6 +549,12 @@ def parse_args(argv=None):
                         '-r',
                         '--remote',
                         help='Remote repository to collect garbage in.')
+    gc_parser.add_argument(
+                        '-f',
+                        '--force',
+                        action='store_true',
+                        default=False,
+                        help='Force garbage collection.')
     gc_parser.set_defaults(func=CmdGC)
 
     # Config
@@ -675,19 +725,27 @@ def parse_args(argv=None):
                         help='TSV path \'row,column\'(e.g. \'1,2\').')
     metrics_show_group.add_argument(
                         '--htsv-path',
-                        help='Headed TSV path \'row,column\'(e.g. \'Name,3\'.')
+                        help='Headed TSV path \'row,column\''
+                             '(e.g. \'Name,3\').')
     metrics_show_group.add_argument(
                         '--csv-path',
                         help='CSV path \'row,column\'(e.g. \'1,2\').')
     metrics_show_group.add_argument(
                         '--hcsv-path',
-                        help='Headed CSV path \'row,column\'(e.g. \'Name,3\'.')
+                        help='Headed CSV path \'row,column\''
+                             '(e.g. \'Name,3\').')
     metrics_show_parser.add_argument(
                         '-a',
                         '--all-branches',
                         action='store_true',
                         default=False,
                         help='Show metrics for all branches.')
+    metrics_show_parser.add_argument(
+                        '-T',
+                        '--all-tags',
+                        action='store_true',
+                        default=False,
+                        help='Show metrics for all tags.')
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
     METRICS_ADD_HELP = 'Add metrics.'
@@ -823,6 +881,10 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Output DAG as ASCII.')
+    pipeline_show_parser.add_argument(
+                        '--dot',
+                        help='Write DAG in .dot format.'
+    )
     pipeline_show_parser.add_argument(
                         'targets',
                         nargs='*',
@@ -834,7 +896,9 @@ def parse_args(argv=None):
     if (issubclass(args.func, CmdRepro)
         or issubclass(args.func, CmdPipelineShow)) \
        and hasattr(args, 'targets') \
-       and len(args.targets) == 0:
+       and len(args.targets) == 0 \
+       and not (hasattr(args, 'all_pipelines')
+       and args.all_pipelines):  # pragma: no cover
         if hasattr(args, 'cwd'):
             cwd = args.cwd
         else:
@@ -852,7 +916,7 @@ def parse_args(argv=None):
        and len(args.deps) == 0 \
        and len(args.outs) == 0 \
        and len(args.outs_no_cache) == 0 \
-       and len(args.command) == 0:
+       and len(args.command) == 0:  # pragma: no cover
         run_parser.error("Too few arguments. Specify at least one: "
                          "'-d', '-o', '-O', 'command'.")
 

dvc/command/data_sync.py

@@ -24,6 +24,7 @@ class CmdDataPull(CmdDataBase):
                               remote=self.args.remote,
                               show_checksums=self.args.show_checksums,
                               all_branches=self.args.all_branches,
+                              all_tags=self.args.all_tags,
                               with_deps=self.args.with_deps)
         except Exception as exc:
             self.project.logger.error('Failed to pull data from the cloud',
@@ -40,6 +41,7 @@ class CmdDataPush(CmdDataBase):
                               remote=self.args.remote,
                               show_checksums=self.args.show_checksums,
                               all_branches=self.args.all_branches,
+                              all_tags=self.args.all_tags,
                               with_deps=self.args.with_deps)
         except Exception as exc:
             self.project.logger.error('Failed to push data to the cloud', exc)
@@ -55,6 +57,7 @@ class CmdDataFetch(CmdDataBase):
                                remote=self.args.remote,
                                show_checksums=self.args.show_checksums,
                                all_branches=self.args.all_branches,
+                               all_tags=self.args.all_tags,
                                with_deps=self.args.with_deps)
         except Exception as exc:
             self.project.logger.error('Failed to fetch data from the cloud',

dvc/command/gc.py

@@ -3,7 +3,24 @@ from dvc.command.base import CmdBase
 
 class CmdGC(CmdBase):
     def run(self):
+        msg = 'This will remove all cache except the cache that is used in '
+        if not self.args.all_branches and not self.args.all_tags:
+            msg += 'the current git branch.'
+        elif self.args.all_branches and not self.args.all_tags:
+            msg += 'all git branches.'
+        elif not self.args.all_branches and self.args.all_tags:
+            msg += 'all git tags.'
+        else:
+            msg += 'all git branches and all git tags.'
+
+        self.project.logger.warn(msg)
+
+        msg = 'Are you sure you want to proceed?'
+        if not self.args.force and not self.project.prompt.prompt(msg):
+            return 1
+
         self.project.gc(all_branches=self.args.all_branches,
+                        all_tags=self.args.all_tags,
                         cloud=self.args.cloud,
                         remote=self.args.remote)
         return 0

dvc/command/metrics.py

@@ -28,7 +28,8 @@ class CmdMetricsShow(CmdBase):
             self.project.metrics_show(self.args.path,
                                       typ=typ,
                                       xpath=xpath,
-                                      all_branches=self.args.all_branches)
+                                      all_branches=self.args.all_branches,
+                                      all_tags=self.args.all_tags)
         except DvcException as exc:
             self.project.logger.error('Failed to show metrics', exc)
             return 1

dvc/command/pipeline.py

@@ -24,7 +24,7 @@ class CmdPipelineShow(CmdBase):
             else:
                 self.project.logger.info(n)
 
-    def _show_ascii(self, target, commands, outs):
+    def __build_graph(self, target, commands, outs):
         import networkx
         from dvc.stage import Stage
 
@@ -51,9 +51,6 @@ class CmdPipelineShow(CmdBase):
             else:
                 nodes.append(stage.relpath)
 
-        if len(nodes) == 0:
-            return
-
         edges = []
         for e in G.edges():
             from_stage = stages[e[0]]
@@ -70,9 +67,28 @@ class CmdPipelineShow(CmdBase):
             else:
                 edges.append((from_stage.relpath, to_stage.relpath))
 
+        return nodes, edges
+
+    def _show_ascii(self, target, commands, outs):
+        nodes, edges = self.__build_graph(target, commands, outs)
+
+        if not nodes:
+            return
+
         d = Dagascii(nodes, edges)
         d.draw()
 
+    def __write_dot(self, target, commands, outs, filename):
+        import networkx
+        from networkx.drawing.nx_pydot import write_dot
+
+        _, edges = self.__build_graph(target, commands, outs)
+        edges = [edge[::-1] for edge in edges]
+
+        simple_g = networkx.DiGraph()
+        simple_g.add_edges_from(edges)
+        write_dot(simple_g, filename)
+
     def run(self, unlock=False):
         for target in self.args.targets:
             try:
@@ -80,6 +96,11 @@ class CmdPipelineShow(CmdBase):
                     self._show_ascii(target,
                                      self.args.commands,
                                      self.args.outs)
+                elif self.args.dot:
+                    self.__write_dot(target,
+                                     self.args.commands,
+                                     self.args.outs,
+                                     self.args.dot)
                 else:
                     self._show(target,
                                self.args.commands,

dvc/command/repro.py

@@ -21,7 +21,8 @@ class CmdRepro(CmdBase):
                                        force=self.args.force,
                                        dry=self.args.dry,
                                        interactive=self.args.interactive,
-                                       pipeline=self.args.pipeline)
+                                       pipeline=self.args.pipeline,
+                                       all_pipelines=self.args.all_pipelines)
 
                 if len(stages) == 0:
                     self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG)

dvc/command/status.py

@@ -31,6 +31,7 @@ class CmdDataStatus(CmdDataBase):
                                      show_checksums=self.args.show_checksums,
                                      remote=self.args.remote,
                                      all_branches=self.args.all_branches,
+                                     all_tags=self.args.all_tags,
                                      with_deps=self.args.with_deps)
             if st:
                 self._show(st, indent)

dvc/dagascii.py

@@ -1,13 +1,170 @@
+import sys
 import math
-
-from asciicanvas.asciicanvas import AsciiCanvas
-from asciicanvas.style import Style
+import select
 
 from grandalf.graphs import Vertex, Edge, Graph
 from grandalf.layouts import SugiyamaLayout
 from grandalf.routing import route_with_lines, EdgeViewer
 
 
+class AsciiCanvas(object):
+    def __init__(self, cols, lines):
+        assert cols > 1
+        assert lines > 1
+
+        self.cols = cols
+        self.lines = lines
+
+        self.canvas = [[' '] * cols for l in range(lines)]
+
+    def draw(self):
+        if sys.stdout.isatty():  # pragma: no cover
+            from asciimatics.screen import Screen
+            Screen.wrapper(self._do_draw)
+        else:
+            for line in self.canvas:
+                print(''.join(line))
+
+    def _do_draw(self, screen):  # pragma: no cover
+        from asciimatics.event import KeyboardEvent
+
+        offset_x = 0
+        offset_y = 0
+        smaxrow, smaxcol = screen.dimensions
+        assert smaxrow > 1
+        assert smaxcol > 1
+        smaxrow -= 1
+        smaxcol -= 1
+
+        if self.lines + 1 > smaxrow:
+            max_y = self.lines + 1 - smaxrow
+        else:
+            max_y = 0
+
+        if self.cols + 1 > smaxcol:
+            max_x = self.cols + 1 - smaxcol
+        else:
+            max_x = 0
+
+        while True:
+            for y in range(smaxrow + 1):
+                y_index = offset_y + y
+                line = []
+                for x in range(smaxcol + 1):
+                    x_index = offset_x + x
+                    if len(self.canvas) > y_index \
+                       and len(self.canvas[y_index]) > x_index:
+                            line.append(self.canvas[y_index][x_index])
+                    else:
+                        line.append(' ')
+                assert len(line) == (smaxcol + 1)
+                screen.print_at(''.join(line), 0, y)
+
+            screen.refresh()
+
+            # NOTE: get_event() doesn't block by itself,
+            # so we have to do the blocking ourselves.
+            select.select([sys.stdin], [], [], None)
+
+            event = screen.get_event()
+            if not isinstance(event, KeyboardEvent):
+                continue
+
+            k = event.key_code
+            if k == screen.KEY_DOWN or k == ord('s'):
+                offset_y += 1
+            elif k == screen.KEY_PAGE_DOWN or k == ord('S'):
+                offset_y += smaxrow
+            elif k == screen.KEY_UP or k == ord('w'):
+                offset_y -= 1
+            elif k == screen.KEY_PAGE_UP or k == ord('W'):
+                offset_y -= smaxrow
+            elif k == screen.KEY_RIGHT or k == ord('d'):
+                offset_x += 1
+            elif k == ord('D'):
+                offset_x += smaxcol
+            elif k == screen.KEY_LEFT or k == ord('a'):
+                offset_x -= 1
+            elif k == ord('A'):
+                offset_x -= smaxcol
+            elif k == ord('q') or k == ord('Q'):
+                break
+
+            if offset_y > max_y:
+                offset_y = max_y
+            elif offset_y < 0:
+                offset_y = 0
+
+            if offset_x > max_x:
+                offset_x = max_x
+            elif offset_x < 0:
+                offset_x = 0
+
+    def point(self, x, y, char):
+        assert x >= 0
+        assert x < self.cols
+        assert y >= 0
+        assert y < self.lines
+
+        self.canvas[y][x] = char
+
+    def line(self, x0, y0, x1, y1, char):
+        if x0 > x1:
+            x1, x0 = x0, x1
+            y1, y0 = y0, y1
+
+        dx = x1 - x0
+        dy = y1 - y0
+
+        if dx == 0 and dy == 0:
+            self.point(x0, y0, char)
+        elif abs(dx) >= abs(dy):
+            for x in range(x0, x1 + 1):
+                if dx == 0:
+                    y = y0
+                else:
+                    y = y0 + int(round((x - x0) * dy / float((dx))))
+                self.point(x, y, char)
+        elif y0 < y1:
+            for y in range(y0, y1 + 1):
+                if dy == 0:
+                    x = x0
+                else:
+                    x = x0 + int(round((y - y0) * dx / float((dy))))
+                self.point(x, y, char)
+        else:
+            for y in range(y1, y0 + 1):
+                if dy == 0:
+                    x = x0
+                else:
+                    x = x1 + int(round((y - y1) * dx / float((dy))))
+                self.point(x, y, char)
+
+    def text(self, x, y, text):
+        for i, c in enumerate(text):
+            self.point(x + i, y, c)
+
+    def box(self, x0, y0, w, h):
+        assert w > 1
+        assert h > 1
+
+        w -= 1
+        h -= 1
+
+        for x in range(x0, x0 + w):
+            self.point(x, y0, '-')
+            self.point(x, y0 + h, '-')
+
+        for y in range(y0, y0 + h):
+            self.point(x0, y, '|')
+            self.point(x0 + w, y, '|')
+
+        self.point(x0, y0, '+')
+        self.point(x0 + w, y0, '+')
+        self.point(x0, y0 + h, '+')
+        self.point(x0 + w, y0 + h, '+')
+
+
 class Dagascii(object):
     def __init__(self, vertexes, edges):
         #
@@ -100,22 +257,22 @@ class Dagascii(object):
                 assert end_x >= 0
                 assert end_y >= 0
 
-                canvas.add_line(start_x, start_y,
-                                end_x, end_y,
-                                style=Style('*'))
+                canvas.line(start_x, start_y,
+                            end_x, end_y,
+                            '*')
 
         for v in self.sug.g.sV:
             # NOTE: moving boxes w/2 to the left
             x = v.view.xy[0] - v.view.w/2.0
             y = v.view.xy[1]
 
-            canvas.add_nine_patch_rect(int(round(x - minx)),
-                                       int(round(y - miny)),
-                                       v.view.w,
-                                       v.view.h)
+            canvas.box(int(round(x - minx)),
+                       int(round(y - miny)),
+                       v.view.w,
+                       v.view.h)
 
-            canvas.add_text(int(round(x - minx)) + 1,
-                            int(round(y - miny)) + 1,
-                            v.data)
+            canvas.text(int(round(x - minx)) + 1,
+                        int(round(y - miny)) + 1,
+                        v.data)
 
-        canvas.print_out()
+        canvas.draw()

dvc/data_cloud.py

@@ -73,6 +73,11 @@ class DataCloud(object):
             msg = "Can't find cloud section '{}' in config"
             raise ConfigError(msg.format(name))
 
+        # NOTE: check if the class itself has everything needed for operation.
+        # E.g. all the imported packages.
+        if not cloud_type.supported(cloud_type.compat_config(cloud_config)):
+            raise ConfigError("Unsupported cloud '{}'".format(name))
+
         return self._init_cloud(cloud_config, cloud_type)
 
     def _init_cloud(self, cloud_config, cloud_type):

dvc/dependency/local.py

@@ -67,7 +67,8 @@ class DependencyLOCAL(DependencyBase):
         if not self.exists:
             raise self.DoesNotExistError(self.rel_path)
 
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
+        if not os.path.isfile(self.path) \
+           and not os.path.isdir(self.path):  # pragma: no cover
             raise self.IsNotFileOrDirError(self.rel_path)
 
         if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \

dvc/exceptions.py

@@ -8,7 +8,7 @@ class DvcException(Exception):
         if cause:
             try:
                 self.cause_tb = traceback.format_exc()
-            except Exception:
+            except Exception:  # pragma: no cover
                 pass
         super(DvcException, self).__init__(msg)
 
@@ -29,3 +29,12 @@ class OutputDuplicationError(DvcException):
         for stage in stages:
             s += "\n    " + stage
         super(OutputDuplicationError, self).__init__(msg.format(output, s))
+
+
+class MoveNotDataSourceError(DvcException):
+    def __init__(self, path):
+        msg = "Move is not permitted for stages that are not data sources. " \
+              "You need to either move '{path}' to a new location and edit " \
+              "it by hand, or remove '{path}' and create a new one at the " \
+              "desired location."
+        super(MoveNotDataSourceError, self).__init__(msg.format(path=path))

dvc/logger.py

@@ -19,9 +19,16 @@ class Logger(object):
     }
 
     COLOR_MAP = {
-        'debug': colorama.Fore.BLUE,
-        'warn': colorama.Fore.YELLOW,
-        'error': colorama.Fore.RED
+        'green': colorama.Fore.GREEN,
+        'yellow': colorama.Fore.YELLOW,
+        'blue': colorama.Fore.BLUE,
+        'red': colorama.Fore.RED,
+    }
+
+    LEVEL_COLOR_MAP = {
+        'debug': 'blue',
+        'warn': 'yellow',
+        'error': 'red',
     }
 
     def __init__(self, loglevel=None):
@@ -58,12 +65,12 @@ class Logger(object):
         Logger.logger().setLevel(logging.DEBUG)
 
     @staticmethod
-    def colorize(msg, typ):
+    def colorize(msg, color):
         header = ''
         footer = ''
 
         if sys.stdout.isatty():  # pragma: no cover
-            header = Logger.COLOR_MAP.get(typ.lower(), '')
+            header = Logger.COLOR_MAP.get(color.lower(), '')
             footer = colorama.Style.RESET_ALL
 
         return u'{}{}{}'.format(header, msg, footer)
@@ -84,7 +91,8 @@ class Logger(object):
 
     @staticmethod
     def _prefix(msg, typ):
-        return Logger.colorize('{}: '.format(msg), typ)
+        color = Logger.LEVEL_COLOR_MAP.get(typ.lower(), '')
+        return Logger.colorize('{}: '.format(msg), color)
 
     @staticmethod
     def error_prefix():
@@ -99,22 +107,47 @@ class Logger(object):
         return Logger._prefix('Debug', 'debug')
 
     @staticmethod
-    def error(msg, exc=None):
+    def _with_progress(func, msg):
+        from dvc.progress import progress
+        with progress:
+            func(msg)
+
+    @staticmethod
+    def _error_exc(exc):
+        if exc is None:
+            return
+
+        if Logger.logger().getEffectiveLevel() != logging.DEBUG:
+            return
+
         prefix = Logger.error_prefix()
         str_exc, str_tb = Logger.parse_exc(exc)
-        if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
-            str_tb = str_tb if str_tb else traceback.format_exc()
-            Logger.logger().error(prefix + str_tb)
-        return Logger.logger().error(prefix + msg + str_exc)
+        str_tb = str_tb if str_tb else traceback.format_exc()
+        Logger._with_progress(Logger.logger().error, prefix + str_tb)
 
     @staticmethod
-    def warn(msg):
-        return Logger.logger().warn(Logger.warning_prefix() + msg)
+    def _with_exc(func, msg, suffix="", exc=None):
+        Logger._error_exc(exc)
+        msg = msg + Logger.parse_exc(exc)[0] + suffix
+        Logger._with_progress(func, msg)
 
     @staticmethod
-    def debug(msg):
-        return Logger.logger().debug(Logger.debug_prefix() + msg)
+    def error(msg, exc=None):
+        chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \
+               "we are always happy to help!"
+        Logger._with_exc(Logger.logger().error,
+                         Logger.error_prefix() + msg,
+                         suffix=chat,
+                         exc=exc)
+
+    @classmethod
+    def warn(cls, msg, exc=None):
+        cls._with_exc(cls.logger().warn, cls.warning_prefix() + msg, exc=exc)
+
+    @classmethod
+    def debug(cls, msg, exc=None):
+        cls._with_exc(cls.logger().debug, cls.debug_prefix() + msg, exc=exc)
 
     @staticmethod
     def info(msg):
-        return Logger.logger().info(msg)
+        Logger._with_progress(Logger.logger().info, msg)

dvc/output/local.py

@@ -83,13 +83,14 @@ class OutputLOCAL(DependencyLOCAL):
             super(OutputLOCAL, self).save()
             self._verify_metric()
             msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
-            self.project.logger.debug(msg.format(self.rel_path))
+            self.project.logger.info(msg.format(self.rel_path))
             return
 
         if not os.path.exists(self.path):
             raise self.DoesNotExistError(self.rel_path)
 
-        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
+        if not os.path.isfile(self.path) \
+           and not os.path.isdir(self.path):  # pragma: no cover
             raise self.IsNotFileOrDirError(self.rel_path)
 
         if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
@@ -99,7 +100,7 @@ class OutputLOCAL(DependencyLOCAL):
 
         if not self.changed():
             msg = 'Output \'{}\' didn\'t change. Skipping saving.'
-            self.project.logger.debug(msg.format(self.rel_path))
+            self.project.logger.info(msg.format(self.rel_path))
             return
 
         if self.is_local:

dvc/progress.py

@@ -11,6 +11,7 @@ class Progress(object):
         self._n_total = 0
         self._n_finished = 0
         self._lock = threading.Lock()
+        self._line = None
 
     def set_n_total(self, total):
         self._n_total = total
@@ -28,18 +29,23 @@ class Progress(object):
         print(line, end='')
         sys.stdout.flush()
 
-    def update_target(self, name, current, total):
+    def refresh(self, line=None):
         # Just go away if it is locked. Will update next time
         if not self._lock.acquire(False):
             return
 
-        bar = self._bar(name, current, total)
+        if line is None:
+            line = self._line
 
-        if sys.stdout.isatty():
-            self._writeln(bar)
+        if sys.stdout.isatty() and line is not None:
+            self._writeln(line)
+            self._line = line
 
         self._lock.release()
 
+    def update_target(self, name, current, total):
+        self.refresh(self._bar(name, current, total))
+
     def finish_target(self, name):
         # We have to write a msg about finished target
         with self._lock:
@@ -51,6 +57,7 @@ class Progress(object):
             print(bar)
 
             self._n_finished += 1
+            self._line = None
 
     def _bar(self, target_name, current, total):
         """
@@ -78,5 +85,15 @@ class Progress(object):
 
         return num + bar + percent + target_name
 
+    def __enter__(self):
+        self._lock.acquire(True)
+        if self._line is not None:
+            self._clearln()
+
+    def __exit__(self, type, value, tb):
+        if self._line is not None:
+            self.refresh()
+        self._lock.release()
+
 
 progress = Progress()

dvc/project.py

@@ -1,6 +1,6 @@
 import os
 
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, MoveNotDataSourceError
 from dvc.stage import Stage
 
 
@@ -209,8 +209,7 @@ class Project(object):
                         continue
 
                     if not stage.is_data_source:
-                        msg = 'Dvcfile \'{}\' is not a data source.'
-                        raise DvcException(msg.format(stage.rel_path))
+                        raise MoveNotDataSourceError(stage.relpath)
 
                     found = True
                     to_out = Output.loads_from(out.stage,
@@ -293,7 +292,7 @@ class Project(object):
         stage = stages[node]
 
         if stage.locked:
-            msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
+            msg = 'DVC file \'{}\' is locked. Its dependencies are not ' \
                   'going to be reproduced.'
             self.logger.warn(msg.format(stage.relpath))
 
@@ -307,12 +306,16 @@ class Project(object):
         return [stage]
 
     def reproduce(self,
-                  target,
+                  target=None,
                   recursive=True,
                   force=False,
                   dry=False,
                   interactive=False,
-                  pipeline=False):
+                  pipeline=False,
+                  all_pipelines=False):
+
+        if target is None and not all_pipelines:
+            raise ValueError()
 
         if not interactive:
             config = self.config
@@ -320,13 +323,18 @@ class Project(object):
             interactive = core.get(config.SECTION_CORE_INTERACTIVE, False)
 
         targets = []
-        if pipeline:
-            stage = Stage.load(self, target)
-            node = os.path.relpath(stage.path, self.root_dir)
-            G = self._get_pipeline(node)
-            for node in G.nodes():
-                if G.in_degree(node) == 0:
-                    targets.append(os.path.join(self.root_dir, node))
+        if pipeline or all_pipelines:
+            if pipeline:
+                stage = Stage.load(self, target)
+                node = os.path.relpath(stage.path, self.root_dir)
+                pipelines = [self._get_pipeline(node)]
+            else:
+                pipelines = self.pipelines()
+
+            for G in pipelines:
+                for node in G.nodes():
+                    if G.in_degree(node) == 0:
+                        targets.append(os.path.join(self.root_dir, node))
         else:
             targets.append(target)
 
@@ -410,7 +418,7 @@ class Project(object):
 
             for stage in stages:
                 if stage.locked:
-                    msg = 'DVC file \'{}\' is locked. Its dependecies are ' \
+                    msg = 'DVC file \'{}\' is locked. Its dependencies are ' \
                           'not going to be checked out.'
                     self.logger.warn(msg.format(stage.relpath))
 
@@ -443,7 +451,8 @@ class Project(object):
                     target=None,
                     all_branches=False,
                     active=True,
-                    with_deps=False):
+                    with_deps=False,
+                    all_tags=False):
         cache = {}
         cache['local'] = []
         cache['s3'] = []
@@ -451,7 +460,8 @@ class Project(object):
         cache['hdfs'] = []
         cache['ssh'] = []
 
-        for branch in self.scm.brancher(all_branches=all_branches):
+        for branch in self.scm.brancher(all_branches=all_branches,
+                                        all_tags=all_tags):
             if target:
                 stages = self._collect(target,
                                        with_deps=with_deps)
@@ -462,7 +472,7 @@ class Project(object):
 
             for stage in stages:
                 if active and not target and stage.locked:
-                    msg = 'DVC file \'{}\' is locked. Its dependecies are ' \
+                    msg = 'DVC file \'{}\' is locked. Its dependencies are ' \
                           'not going to be pushed/pulled/fetched.'
                     self.logger.warn(msg.format(stage.relpath))
 
@@ -486,12 +496,14 @@ class Project(object):
            all_branches=False,
            cloud=False,
            remote=None,
-           with_deps=False):
+           with_deps=False,
+           all_tags=False):
         with self.state:
             clist = self._used_cache(target=None,
                                      all_branches=all_branches,
                                      active=False,
-                                     with_deps=with_deps)
+                                     with_deps=with_deps,
+                                     all_tags=all_tags)
             self._do_gc('local', self.cache.local.gc, clist)
 
             if self.cache.s3:
@@ -519,10 +531,12 @@ class Project(object):
              remote=None,
              all_branches=False,
              show_checksums=False,
-             with_deps=False):
+             with_deps=False,
+             all_tags=False):
         with self.state:
             used = self._used_cache(target,
                                     all_branches=all_branches,
+                                    all_tags=all_tags,
                                     with_deps=with_deps)['local']
             self.cloud.push(used,
                             jobs,
@@ -535,10 +549,12 @@ class Project(object):
               remote=None,
               all_branches=False,
               show_checksums=False,
-              with_deps=False):
+              with_deps=False,
+              all_tags=False):
         with self.state:
             used = self._used_cache(target,
                                     all_branches=all_branches,
+                                    all_tags=all_tags,
                                     with_deps=with_deps)['local']
             self.cloud.pull(used,
                             jobs,
@@ -551,11 +567,13 @@ class Project(object):
              remote=None,
              all_branches=False,
              show_checksums=False,
-             with_deps=False):
+             with_deps=False,
+             all_tags=False):
         self.fetch(target,
                    jobs,
                    remote=remote,
                    all_branches=all_branches,
+                   all_tags=all_tags,
                    show_checksums=show_checksums,
                    with_deps=with_deps)
         self.checkout(target=target)
@@ -570,8 +588,8 @@ class Project(object):
 
         for stage in stages:
             if stage.locked:
-                msg = 'DVC file \'{}\' is locked. Its dependecies are not ' \
-                      'going to be shown in status output.'
+                msg = 'DVC file \'{}\' is locked. Its dependencies are not ' \
+                      'going to be shown in the status output.'
                 self.logger.warn(msg.format(stage.relpath))
 
             status.update(stage.status())
@@ -584,11 +602,13 @@ class Project(object):
                       remote=None,
                       show_checksums=False,
                       all_branches=False,
-                      with_deps=False):
+                      with_deps=False,
+                      all_tags=False):
         import dvc.remote.base as cloud
 
         used = self._used_cache(target,
                                 all_branches=all_branches,
+                                all_tags=all_tags,
                                 with_deps=with_deps)['local']
 
         status = {}
@@ -615,7 +635,8 @@ class Project(object):
                remote=None,
                show_checksums=False,
                all_branches=False,
-               with_deps=False):
+               with_deps=False,
+               all_tags=False):
         with self.state:
             if cloud:
                 return self._cloud_status(target,
@@ -623,7 +644,8 @@ class Project(object):
                                           remote=remote,
                                           show_checksums=show_checksums,
                                           all_branches=all_branches,
-                                          with_deps=with_deps)
+                                          with_deps=with_deps,
+                                          all_tags=all_tags)
             return self._local_status(target)
 
     def _read_metric_json(self, fd, json_path):
@@ -704,9 +726,11 @@ class Project(object):
                      path=None,
                      typ=None,
                      xpath=None,
-                     all_branches=False):
+                     all_branches=False,
+                     all_tags=False):
         res = {}
-        for branch in self.scm.brancher(all_branches=all_branches):
+        for branch in self.scm.brancher(all_branches=all_branches,
+                                        all_tags=all_tags):
             astages = self.active_stages()
             outs = [out for stage in astages for out in stage.outs]
 
@@ -751,7 +775,7 @@ class Project(object):
                 res[branch][rel] = metric
 
         for branch, val in res.items():
-            if all_branches:
+            if all_branches or all_tags:
                 self.logger.info('{}:'.format(branch))
             for fname, metric in val.items():
                 self.logger.info('\t{}: {}'.format(fname, metric))

dvc/reflink_ffi.py

@@ -1,88 +0,0 @@
-import platform
-from cffi import FFI
-
-
-WINDOWS_SRC = """
-int reflink(char *src, char *dst)
-{
-    return -1;
-}
-"""
-
-DARWIN_SRC = """
-#include <AvailabilityMacros.h>
-/* NOTE: sys/clonefile.h is available since OS X 10.12 */
-#if MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12
-#include <sys/clonefile.h> /* for clonefile(2) */
-#include <errno.h>
-int reflink(char *src, char *dst)
-{
-    return clonefile(src, dst, 0);
-}
-#else
-int reflink(char *src, char *dst)
-{
-    return -1;
-}
-#endif
-"""
-
-LINUX_SRC = """
-#include <sys/stat.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <linux/fs.h>
-#include <sys/ioctl.h>
-#include <assert.h>
-#include <errno.h>
-int reflink(char *src, char *dst)
-{
-    int ret = -1;
-#ifdef FICLONE
-    int src_fd = -1;
-    int dst_fd = -1;
-
-    src_fd = open(src, O_RDONLY);
-    if (src_fd < 0)
-        return ret;
-
-    dst_fd = open(dst, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR);
-    if (src_fd < 0) {
-        close(src_fd);
-        return ret;
-    }
-
-    ret = ioctl(dst_fd, FICLONE, src_fd);
-    if (ret != 0)
-        unlink(dst);
-
-out:
-    if (src_fd >= 0)
-        close(src_fd);
-    if (dst_fd >= 0)
-        close(dst_fd);
-#endif
-    return ret;
-}
-"""
-
-ffibuilder = FFI()
-
-ffibuilder.cdef("""
-int reflink(char *src, char *dst);
-""")
-
-system = platform.system()
-if system == 'Windows':
-    src = WINDOWS_SRC
-elif system == 'Linux':
-    src = LINUX_SRC
-elif system == 'Darwin':
-    src = DARWIN_SRC
-else:
-    raise NotImplementedError
-
-ffibuilder.set_source("reflink", src)
-
-if __file__ == '__main__':
-    ffibuilder.compile(verbose=True)

dvc/remote/azure.py

@@ -176,8 +176,8 @@ class RemoteAzure(RemoteBase):
                 self.blob_service.create_blob_from_path(
                     bucket, key, from_info['path'], progress_callback=cb)
             except Exception as ex:
-                Logger.error("Failed to upload '{}'".format(from_info['path']),
-                             ex)
+                msg = "Failed to upload '{}'".format(from_info['path'])
+                Logger.warn(msg, ex)
             else:
                 progress.finish_target(name)
 
@@ -213,8 +213,8 @@ class RemoteAzure(RemoteBase):
                 self.blob_service.get_blob_to_path(
                     bucket, key, tmp_file, progress_callback=cb)
             except Exception as exc:
-                Logger.error("Failed to download '{}/{}'".format(
-                    bucket, key), exc)
+                msg = "Failed to download '{}/{}'".format(bucket, key)
+                Logger.warn(msg, exc)
             else:
                 os.rename(tmp_file, to_info['path'])
 

dvc/remote/base.py

@@ -1,6 +1,7 @@
 import os
 import re
 import errno
+from multiprocessing import cpu_count
 
 from dvc.config import Config
 from dvc.logger import Logger
@@ -30,14 +31,18 @@ class DataCloudError(DvcException):
 class RemoteBase(object):
     REGEX = None
     REQUIRES = {}
+    JOBS = 4 * cpu_count()
 
     def __init__(self, project, config):
         pass
 
+    def compat_config(config):
+        return config.copy()
+
     @classmethod
     def supported(cls, config):
         url = config[Config.SECTION_REMOTE_URL]
-        url_ok = cls.match(url)
+        url_ok = cls.match(url) is not None
         deps_ok = all(cls.REQUIRES.values())
         if url_ok and not deps_ok:
             missing = [k for k, v in cls.REQUIRES.items() if v is None]

dvc/remote/gs.py

@@ -33,6 +33,13 @@ class RemoteGS(RemoteBase):
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
 
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = 'gs://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
+
     @property
     def bucket(self):
         return urlparse(self.url).netloc
@@ -207,9 +214,9 @@ class RemoteGS(RemoteBase):
                 blob.upload_from_filename(from_info['path'])
             except Exception as exc:
                 msg = "Failed to upload '{}' to '{}/{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['bucket'],
-                                        to_info['key']), exc)
+                Logger.warn(msg.format(from_info['path'],
+                                       to_info['bucket'],
+                                       to_info['key']), exc)
                 continue
 
             progress.finish_target(name)
@@ -256,9 +263,9 @@ class RemoteGS(RemoteBase):
                 blob.download_to_filename(tmp_file)
             except Exception as exc:
                 msg = "Failed to download '{}/{}' to '{}'"
-                Logger.error(msg.format(from_info['bucket'],
-                                        from_info['key'],
-                                        to_info['path']), exc)
+                Logger.warn(msg.format(from_info['bucket'],
+                                       from_info['key'],
+                                       to_info['path']), exc)
                 continue
 
             os.rename(tmp_file, to_info['path'])

dvc/remote/hdfs.py

@@ -29,9 +29,14 @@ class RemoteHDFS(RemoteBase):
         cmd = 'hadoop fs -' + cmd
         if user:
             cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
+
+        # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
+        # See https://github.com/iterative/dvc/issues/1197.
+        close_fds = (os.name != 'nt')
+
         p = Popen(cmd,
                   shell=True,
-                  close_fds=True,
+                  close_fds=close_fds,
                   executable=os.getenv('SHELL'),
                   env=fix_env(os.environ),
                   stdin=PIPE,

dvc/remote/local.py

@@ -50,6 +50,13 @@ class RemoteLOCAL(RemoteBase):
         if self.cache_dir is not None and not os.path.exists(self.cache_dir):
             os.mkdir(self.cache_dir)
 
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, '')
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
+
     @property
     def url(self):
         return self.cache_dir
@@ -81,14 +88,35 @@ class RemoteLOCAL(RemoteBase):
         relpath = os.path.relpath(path, self.cache_dir)
         return os.path.dirname(relpath) + os.path.basename(relpath)
 
-    def changed_cache(self, md5):
+    def changed_cache_file(self, md5):
         cache = self.get(md5)
         if self.state.changed(cache, md5=md5):
             if os.path.exists(cache):
-                msg = 'Corrupted cache file {}'
+                msg = 'Corrupted cache file {}.'
                 Logger.warn(msg.format(os.path.relpath(cache)))
                 remove(cache)
             return True
+        return False
+
+    def changed_cache(self, md5):
+        cache = self.get(md5)
+        clist = [(cache, md5)]
+
+        while True:
+            if len(clist) == 0:
+                break
+
+            cache, md5 = clist.pop()
+            if self.changed_cache_file(md5):
+                return True
+
+            if not self.is_dir_cache(cache):
+                continue
+
+            for entry in self.load_dir_cache(md5):
+                md5 = entry[self.PARAM_MD5]
+                cache = self.get(md5)
+                clist.append((cache, md5))
 
         return False
 
@@ -108,7 +136,7 @@ class RemoteLOCAL(RemoteBase):
                                        os.path.relpath(cache),
                                        os.path.relpath(path)))
                 return
-            except Exception as exc:
+            except DvcException as exc:
                 msg = 'Cache type \'{}\' is not supported: {}'
                 Logger.debug(msg.format(self.cache_types[0], str(exc)))
                 del self.cache_types[0]
@@ -137,14 +165,15 @@ class RemoteLOCAL(RemoteBase):
     def collect_dir_cache(self, dname):
         dir_info = []
 
-        bar = False
         for root, dirs, files in os.walk(dname):
+            bar = False
+
             if len(files) > LARGE_DIR_SIZE:
                 msg = "Computing md5 for a large directory {}. " \
                       "This is only done once."
-                Logger.info(msg.format(os.path.relpath(dname)))
+                Logger.info(msg.format(os.path.relpath(root)))
                 bar = True
-                title = os.path.relpath(dname)
+                title = os.path.relpath(root)
                 processed = 0
                 total = len(files)
                 progress.update_target(title, 0, total)
@@ -161,8 +190,8 @@ class RemoteLOCAL(RemoteBase):
                 dir_info.append({self.PARAM_RELPATH: relpath,
                                  self.PARAM_MD5: md5})
 
-        if bar:
-            progress.finish_target(title)
+            if bar:
+                progress.finish_target(title)
 
         # NOTE: sorting the list by path to ensure reproducibility
         dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
@@ -388,8 +417,8 @@ class RemoteLOCAL(RemoteBase):
                 copyfile(from_info['path'], to_info['path'], name=name)
             except Exception as exc:
                 msg = "Failed to upload '{}' tp '{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['path']), exc)
+                Logger.warn(msg.format(from_info['path'],
+                                       to_info['path']), exc)
 
     def download(self,
                  from_infos,
@@ -420,8 +449,8 @@ class RemoteLOCAL(RemoteBase):
                          name=name)
             except Exception as exc:
                 msg = "Failed to download '{}' to '{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['path']), exc)
+                Logger.warn(msg.format(from_info['path'],
+                                       to_info['path']), exc)
                 continue
 
             os.rename(tmp_file, to_info['path'])
@@ -487,7 +516,7 @@ class RemoteLOCAL(RemoteBase):
 
         return removed
 
-    def status(self, checksum_infos, remote, jobs=1, show_checksums=False):
+    def status(self, checksum_infos, remote, jobs=None, show_checksums=False):
         Logger.info("Preparing to pull data from {}".format(remote.url))
         title = "Collecting information"
 
@@ -512,7 +541,7 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 90, 100)
 
-        local_exists = [not self.changed_cache(md5) for md5 in md5s]
+        local_exists = [not self.changed_cache_file(md5) for md5 in md5s]
 
         progress.finish_target(title)
 
@@ -523,7 +552,7 @@ class RemoteLOCAL(RemoteBase):
     def _do_pull(self,
                  checksum_infos,
                  remote,
-                 jobs=1,
+                 jobs=None,
                  show_checksums=False):
         title = "Collecting information"
 
@@ -539,7 +568,7 @@ class RemoteLOCAL(RemoteBase):
         names = []
         # NOTE: filter files that are not corrupted
         for md5, name in grouped:
-            if self.changed_cache(md5):
+            if self.changed_cache_file(md5):
                 md5s.append(md5)
                 names.append(name)
 
@@ -553,8 +582,17 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 60, 100)
 
+        # NOTE: dummy call to try to establish a connection
+        # to see if we need to ask user for a password.
+        remote.exists(remote.md5s_to_path_infos(['000']))
+
+        progress.update_target(title, 70, 100)
+
         assert len(path_infos) == len(cache) == len(md5s) == len(names)
 
+        if jobs is None:
+            jobs = remote.JOBS
+
         chunks = list(zip(to_chunks(path_infos, jobs),
                           to_chunks(cache, jobs),
                           to_chunks(names, jobs)))
@@ -578,7 +616,7 @@ class RemoteLOCAL(RemoteBase):
         for f in futures:
             f.result()
 
-    def pull(self, checksum_infos, remote, jobs=1, show_checksums=False):
+    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
         Logger.info("Preparing to pull data from {}".format(remote.url))
 
         # NOTE: try fetching missing dir info
@@ -595,7 +633,7 @@ class RemoteLOCAL(RemoteBase):
                       jobs,
                       show_checksums=show_checksums)
 
-    def push(self, checksum_infos, remote, jobs=1, show_checksums=False):
+    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
         Logger.info("Preparing to push data to {}".format(remote.url))
         title = "Collecting information"
 
@@ -608,7 +646,7 @@ class RemoteLOCAL(RemoteBase):
 
         # NOTE: verifying that our cache is not corrupted
         def func(info):
-            return not self.changed_cache(info[self.PARAM_MD5])
+            return not self.changed_cache_file(info[self.PARAM_MD5])
         checksum_infos = list(filter(func, checksum_infos))
 
         progress.update_target(title, 20, 100)
@@ -640,6 +678,9 @@ class RemoteLOCAL(RemoteBase):
 
         progress.update_target(title, 90, 100)
 
+        if jobs is None:
+            jobs = remote.JOBS
+
         chunks = list(zip(to_chunks(path_infos, jobs),
                           to_chunks(cache, jobs),
                           to_chunks(names, jobs)))

dvc/remote/s3.py

@@ -47,7 +47,9 @@ class RemoteS3(RemoteBase):
                                            '').lstrip('/')
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
         self.region = config.get(Config.SECTION_AWS_REGION, None)
-        self.profile = config.get(Config.SECTION_AWS_PROFILE, 'default')
+        self.profile = os.getenv('AWS_PROFILE',
+                                 config.get(Config.SECTION_AWS_PROFILE,
+                                            'default'))
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL, None)
 
         credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
@@ -55,13 +57,20 @@ class RemoteS3(RemoteBase):
             creds_conf = configobj.ConfigObj(credentialpath)
             creds = creds_conf.get(self.profile, {})
         else:
-            creds = {}
+            creds = self._get_aws_config_from_env()
 
         self.creds = creds
         self.region = creds.get('region', self.region)
         self.aws_access_key_id = creds.get('aws_access_key_id', None)
         self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
 
+    @staticmethod
+    def compat_config(config):
+        ret = config.copy()
+        url = 's3://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        ret[Config.SECTION_REMOTE_URL] = url
+        return ret
+
     @property
     def bucket(self):
         return urlparse(self.url).netloc
@@ -81,6 +90,14 @@ class RemoteS3(RemoteBase):
                              region_name=self.region)
         return session.client('s3', endpoint_url=self.endpoint_url)
 
+    def _get_aws_config_from_env(self):
+        creds = {
+            'region': os.getenv('AWS_DEFAULT_REGION'),
+            'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
+            'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
+        }
+        return {k: v for k, v in creds.items() if v is not None}
+
     def get_etag(self, bucket, key):
         try:
             obj = self.s3.head_object(Bucket=bucket, Key=key)
@@ -262,7 +279,7 @@ class RemoteS3(RemoteBase):
                                Callback=cb)
             except Exception as exc:
                 msg = "Failed to upload '{}'".format(from_info['path'])
-                Logger.error(msg, exc)
+                Logger.warn(msg, exc)
                 continue
 
             progress.finish_target(name)
@@ -296,16 +313,17 @@ class RemoteS3(RemoteBase):
             if not name:
                 name = os.path.basename(to_info['path'])
 
-            if no_progress_bar:
-                cb = None
-            else:
-                total = s3.head_object(Bucket=from_info['bucket'],
-                                       Key=from_info['key'])['ContentLength']
-                cb = Callback(name, total)
-
             self._makedirs(to_info['path'])
 
             try:
+                if no_progress_bar:
+                    cb = None
+                else:
+                    total = s3.head_object(
+                                    Bucket=from_info['bucket'],
+                                    Key=from_info['key'])['ContentLength']
+                    cb = Callback(name, total)
+
                 s3.download_file(from_info['bucket'],
                                  from_info['key'],
                                  tmp_file,
@@ -313,8 +331,8 @@ class RemoteS3(RemoteBase):
             except Exception as exc:
                 msg = "Failed to download '{}/{}'".format(from_info['bucket'],
                                                           from_info['key'])
-                Logger.error(msg, exc)
-                return
+                Logger.warn(msg, exc)
+                continue
 
             os.rename(tmp_file, to_info['path'])
 

dvc/remote/ssh.py

@@ -44,6 +44,9 @@ class RemoteSSH(RemoteBase):
     REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'
 
     REQUIRES = {'paramiko': paramiko}
+
+    JOBS = 4
+
     PARAM_MD5 = 'md5'
 
     DEFAULT_PORT = 22
@@ -346,9 +349,9 @@ class RemoteSSH(RemoteBase):
                                     callback=create_cb(name))
             except Exception as exc:
                 msg = "Failed to download '{}/{}' to '{}'"
-                Logger.error(msg.format(from_info['host'],
-                                        from_info['path'],
-                                        to_info['path']), exc)
+                Logger.warn(msg.format(from_info['host'],
+                                       from_info['path'],
+                                       to_info['path']), exc)
                 continue
 
             os.rename(tmp_file, to_info['path'])
@@ -387,9 +390,9 @@ class RemoteSSH(RemoteBase):
                          callback=create_cb(name))
             except Exception as exc:
                 msg = "Failed to upload '{}' to '{}/{}'"
-                Logger.error(msg.format(from_info['path'],
-                                        to_info['host'],
-                                        to_info['path'], exc))
+                Logger.warn(msg.format(from_info['path'],
+                                       to_info['host'],
+                                       to_info['path'], exc))
                 continue
 
             progress.finish_target(name)

dvc/scm.py

@@ -50,17 +50,34 @@ class Base(object):
     def branch(self, branch):
         pass
 
-    def brancher(self, branches=None, all_branches=False):
-        if not branches and not all_branches:
+    def brancher(self,
+                 branches=None,
+                 all_branches=False,
+                 tags=None,
+                 all_tags=False):
+        if not branches and not all_branches \
+           and not tags and not all_tags:
             yield ''
             return
 
         saved = self.active_branch()
-        if not branches:
-            branches = self.list_branches() if all_branches else [saved]
-        for branch in branches:
-            self.checkout(branch)
-            yield branch
+        revs = []
+
+        if branches is not None:
+            revs.extend(branches)
+        elif all_branches:
+            revs.extend(self.list_branches())
+        elif tags is not None:
+            revs.extend(tags)
+        elif all_tags:
+            revs.extend(self.list_tags())
+        else:
+            revs.extend([saved])
+
+        for rev in revs:
+            self.checkout(rev)
+            yield rev
+
         self.checkout(saved)
 
     def untracked_files(self):
@@ -75,6 +92,9 @@ class Base(object):
     def list_branches(self):
         pass
 
+    def list_tags(self):
+        pass
+
     def install(self):
         pass
 
@@ -208,6 +228,9 @@ class Git(Base):
     def list_branches(self):
         return [h.name for h in self.repo.heads]
 
+    def list_tags(self):
+        return [t.name for t in self.repo.tags]
+
     def install(self):
         hook = os.path.join(self.root_dir,
                             self.GIT_DIR,

dvc/signal_handler.py

@@ -1,17 +0,0 @@
-import signal
-import threading
-
-from dvc.logger import Logger
-
-
-class SignalHandler(object):
-    def __enter__(self):
-        if isinstance(threading.current_thread(), threading._MainThread):
-            self.old_handler = signal.signal(signal.SIGINT, self.handler)
-
-    def handler(self, sig, frame):
-        Logger.debug('Ignoring SIGINT during critical parts of code...')
-
-    def __exit__(self, type, value, traceback):
-        if isinstance(threading.current_thread(), threading._MainThread):
-            signal.signal(signal.SIGINT, self.old_handler)

dvc/stage.py

@@ -1,6 +1,5 @@
 import os
 import yaml
-import itertools
 import posixpath
 import subprocess
 from schema import Schema, SchemaError, Optional, Or, And
@@ -19,8 +18,9 @@ class StageCmdFailedError(DvcException):
 
 
 class StageFileFormatError(DvcException):
-    def __init__(self):
-        super(StageFileFormatError, self).__init__('Stage file format error')
+    def __init__(self, fname, e):
+        msg = "Stage file '{}' format error: {}".format(fname, str(e))
+        super(StageFileFormatError, self).__init__(msg)
 
 
 class StageFileDoesNotExistError(DvcException):
@@ -134,8 +134,6 @@ class Stage(object):
         if self.md5 and md5 and self.md5 == md5:
             return False
 
-        msg = "Dvc file '{}' md5 changed(expected '{}', actual '{}')"
-        self.project.logger.debug(msg.format(self.relpath, self.md5, md5))
         return True
 
     @property
@@ -148,30 +146,51 @@ class Stage(object):
                len(self.deps) == 1 and \
                len(self.outs) == 1
 
-    def changed(self):
+    def changed(self, print_info=False):
         ret = False
 
+        if print_info:
+            log = self.project.logger.info
+        else:
+            log = self.project.logger.debug
+
         if self.is_callback:
+            msg = "Dvc file '{}' is a 'callback' stage (has a command and " \
+                  "no dependencies) and thus always considered as changed."
+            self.project.logger.warn(msg.format(self.relpath))
             ret = True
 
-        if self.locked:
-            entries = self.outs
-        else:
-            entries = itertools.chain(self.outs, self.deps)
-
-        for entry in entries:
-            if entry.changed():
+        if not self.locked:
+            for dep in self.deps:
+                if not dep.changed():
+                    continue
+                if print_info:
+                    msg = "Dependency '{}' of '{}' changed."
+                    log(msg.format(dep, self.relpath))
                 ret = True
 
+        for out in self.outs:
+            if not out.changed():
+                continue
+            if print_info:
+                msg = "Output '{}' of '{}' changed."
+                log(msg.format(out, self.relpath))
+            ret = True
+
         if self.changed_md5():
+            if print_info:
+                msg = "Dvc file '{}' changed."
+                log(msg.format(self.relpath))
             ret = True
 
         if ret:
-            msg = u'Dvc file \'{}\' changed'.format(self.relpath)
+            msg = "Stage '{}' changed.".format(self.relpath)
+            color = 'yellow'
         else:
-            msg = u'Dvc file \'{}\' didn\'t change'.format(self.relpath)
+            msg = "Stage '{}' didn't change.".format(self.relpath)
+            color = 'green'
 
-        self.project.logger.debug(msg)
+        log(Logger.colorize(msg, color))
 
         return ret
 
@@ -184,7 +203,7 @@ class Stage(object):
         os.unlink(self.path)
 
     def reproduce(self, force=False, dry=False, interactive=False):
-        if not self.changed() and not force:
+        if not self.changed(print_info=True) and not force:
             return None
 
         if (self.cmd or self.is_import) and not self.locked and not dry:
@@ -207,17 +226,15 @@ class Stage(object):
         return self
 
     @staticmethod
-    def validate(d):
+    def validate(d, fname=None):
         try:
             Schema(Stage.SCHEMA).validate(d)
         except SchemaError as exc:
-            Logger.debug(str(exc))
-            raise StageFileFormatError()
+            raise StageFileFormatError(fname, exc)
 
     @staticmethod
     def loadd(project, d, path):
-        Stage.validate(d)
-
+        Stage.validate(d, fname=os.path.relpath(path))
         path = os.path.abspath(path)
         cwd = os.path.dirname(path)
         cmd = d.get(Stage.PARAM_CMD, None)

dvc/state.py

@@ -15,6 +15,7 @@ class StateDuplicateError(DvcException):
 
 
 class State(object):
+    VERSION = 2
     STATE_FILE = 'state'
     STATE_TABLE = 'state'
     STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
@@ -34,6 +35,9 @@ class State(object):
     STATE_ROW_LIMIT = 10000000
     STATE_ROW_CLEANUP_QUOTA = 50
 
+    MAX_INT = 2**63 - 1
+    MAX_UINT = 2**64 - 2
+
     def __init__(self, project, config):
         self.project = project
         self.dvc_dir = project.dvc_dir
@@ -81,32 +85,95 @@ class State(object):
 
         return actual.split('.')[0] != md5.split('.')[0]
 
+    def _execute(self, cmd):
+        Logger.debug(cmd)
+        return self.c.execute(cmd)
+
+    def _fetchall(self):
+        ret = self.c.fetchall()
+        Logger.debug("fetched: {}".format(ret))
+        return ret
+
+    def _to_sqlite(self, n):
+        assert n >= 0
+        assert n < self.MAX_UINT
+        # NOTE: sqlite stores unit as signed ints, so maximum uint is 2^63-1
+        # see http://jakegoulding.com/blog/2011/02/06/sqlite-64-bit-integers/
+        if n > self.MAX_INT:
+            ret = -(n - self.MAX_INT)
+        else:
+            ret = n
+        assert self._from_sqlite(ret) == n
+        return ret
+
+    def _from_sqlite(self, n):
+        assert abs(n) <= self.MAX_INT
+        if n < 0:
+            return abs(n) + self.MAX_INT
+        assert n < self.MAX_UINT
+        assert n >= 0
+        return n
+
+    def _load(self, empty=False):
+        from dvc import VERSION
+
+        if not empty:
+            cmd = "PRAGMA user_version;"
+            self._execute(cmd)
+            ret = self._fetchall()
+            assert len(ret) == 1
+            assert len(ret[0]) == 1
+            assert isinstance(ret[0][0], int)
+            version = ret[0][0]
+
+            if version > self.VERSION:
+                msg = "You are using an old version '{}' of dvc that is " \
+                      "using state file version '{}' which is not " \
+                      "compatible with the state file version '{}' that " \
+                      "is used in this projet. Please upgrade right now!"
+                raise DvcException(msg.format(VERSION,
+                                              self.VERSION,
+                                              version))
+            elif version < self.VERSION:
+                msg = "State file version '{}' is too old. " \
+                      "Reformatting to the current version '{}'."
+                self.project.logger.warn(msg.format(version, self.VERSION))
+                cmd = "DROP TABLE IF EXISTS {};"
+                self._execute(cmd.format(self.STATE_TABLE))
+                self._execute(cmd.format(self.STATE_INFO_TABLE))
+                self._execute(cmd.format(self.LINK_STATE_TABLE))
+
+        # Check that the state file is indeed a database
+        cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
+        self._execute(cmd.format(self.STATE_TABLE,
+                                 self.STATE_TABLE_LAYOUT))
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_TABLE_LAYOUT))
+        self._execute(cmd.format(self.LINK_STATE_TABLE,
+                                 self.LINK_STATE_TABLE_LAYOUT))
+
+        cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
+              "WHERE NOT EXISTS (SELECT * FROM {})"
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_TABLE))
+
+        cmd = "PRAGMA user_version = {};"
+        self._execute(cmd.format(self.VERSION))
+
     def load(self):
         retries = 1
         while True:
             assert self.db is None
             assert self.c is None
             assert self.inserts == 0
+            empty = not os.path.exists(self.state_file)
             self.db = sqlite3.connect(self.state_file)
             self.c = self.db.cursor()
 
             # Try loading once to check that the file is indeed a database
             # and reformat it if it is not.
             try:
-                # Check that the state file is indeed a database
-                cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
-                self.c.execute(cmd.format(self.STATE_TABLE,
-                                          self.STATE_TABLE_LAYOUT))
-                self.c.execute(cmd.format(self.STATE_INFO_TABLE,
-                                          self.STATE_INFO_TABLE_LAYOUT))
-                self.c.execute(cmd.format(self.LINK_STATE_TABLE,
-                                          self.LINK_STATE_TABLE_LAYOUT))
-
-                cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
-                      "WHERE NOT EXISTS (SELECT * FROM {})"
-                self.c.execute(cmd.format(self.STATE_INFO_TABLE,
-                                          self.STATE_INFO_TABLE))
-
+                self._load(empty=empty)
                 return
             except sqlite3.DatabaseError:
                 self.c.close()
@@ -124,12 +191,12 @@ class State(object):
         assert self.db is not None
 
         cmd = "SELECT count from {} WHERE rowid={}"
-        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
-                                  self.STATE_INFO_ROW))
-        ret = self.c.fetchall()
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_ROW))
+        ret = self._fetchall()
         assert len(ret) == 1
         assert len(ret[0]) == 1
-        count = ret[0][0] + self.inserts
+        count = self._from_sqlite(ret[0][0]) + self.inserts
 
         if count > self.row_limit:
             msg = "Cleaning up state. This might take a while."
@@ -139,24 +206,24 @@ class State(object):
             delete += int(self.row_limit * (self.row_cleanup_quota/100.))
             cmd = "DELETE FROM {} WHERE timestamp IN (" \
                   "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
-            self.c.execute(cmd.format(self.STATE_TABLE,
-                                      self.STATE_TABLE,
-                                      delete))
+            self._execute(cmd.format(self.STATE_TABLE,
+                                     self.STATE_TABLE,
+                                     delete))
 
-            self.c.execute("VACUUM")
+            self._execute("VACUUM")
 
             cmd = "SELECT COUNT(*) FROM {}"
 
-            self.c.execute(cmd.format(self.STATE_TABLE))
-            ret = self.c.fetchall()
+            self._execute(cmd.format(self.STATE_TABLE))
+            ret = self._fetchall()
             assert len(ret) == 1
             assert len(ret[0]) == 1
             count = ret[0][0]
 
         cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
-        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
-                                  count,
-                                  self.STATE_INFO_ROW))
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self._to_sqlite(count),
+                                 self.STATE_INFO_ROW))
 
         self.db.commit()
         self.c.close()
@@ -190,41 +257,42 @@ class State(object):
         inode = self.inode(path)
 
         cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
-                                                       inode)
+                                                       self._to_sqlite(inode))
 
-        self.c.execute(cmd)
-        ret = self.c.fetchall()
+        self._execute(cmd)
+        ret = self._fetchall()
         if len(ret) == 0:
             md5, info = self._collect(path)
             cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
                   'VALUES ({}, "{}", "{}", "{}")'
-            self.c.execute(cmd.format(self.STATE_TABLE,
-                                      inode,
-                                      mtime,
-                                      md5,
-                                      int(nanotime.timestamp(time.time()))))
+            self._execute(cmd.format(self.STATE_TABLE,
+                                     self._to_sqlite(inode),
+                                     mtime,
+                                     md5,
+                                     int(nanotime.timestamp(time.time()))))
             self.inserts += 1
         else:
             assert len(ret) == 1
             assert len(ret[0]) == 4
             i, m, md5, timestamp = ret[0]
+            i = self._from_sqlite(i)
             assert i == inode
             if mtime != m:
                 md5, info = self._collect(path)
                 cmd = 'UPDATE {} SET ' \
                       'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
                       'WHERE inode = {}'
-                self.c.execute(cmd.format(self.STATE_TABLE,
-                                          mtime,
-                                          md5,
-                                          int(nanotime.timestamp(time.time())),
-                                          inode))
+                self._execute(cmd.format(self.STATE_TABLE,
+                                         mtime,
+                                         md5,
+                                         int(nanotime.timestamp(time.time())),
+                                         self._to_sqlite(inode)))
             else:
                 info = None
                 cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
-                self.c.execute(cmd.format(self.STATE_TABLE,
-                                          int(nanotime.timestamp(time.time())),
-                                          inode))
+                self._execute(cmd.format(self.STATE_TABLE,
+                                         int(nanotime.timestamp(time.time())),
+                                         self._to_sqlite(inode)))
 
         return (md5, info)
 
@@ -248,16 +316,17 @@ class State(object):
         cmd = 'REPLACE INTO {}(path, inode, mtime) ' \
               'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE,
                                                relpath,
-                                               inode,
+                                               self._to_sqlite(inode),
                                                mtime)
-        self.c.execute(cmd)
+        self._execute(cmd)
 
     def remove_unused_links(self, used):
         unused = []
 
-        self.c.execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
+        self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
         for row in self.c:
             p, i, m = row
+            i = self._from_sqlite(i)
             path = os.path.join(self.root_dir, p)
 
             if path in used:
@@ -276,4 +345,4 @@ class State(object):
 
         for p in unused:
             cmd = 'DELETE FROM {} WHERE path = "{}"'
-            self.c.execute(cmd.format(self.LINK_STATE_TABLE, p))
+            self._execute(cmd.format(self.LINK_STATE_TABLE, p))

dvc/system.py

@@ -1,8 +1,4 @@
 import os
-import ctypes
-
-if os.name == 'nt':
-    import ntfsutils.hardlink as winlink
 
 
 class System(object):
@@ -12,13 +8,24 @@ class System(object):
 
     @staticmethod
     def hardlink(source, link_name):
+        import ctypes
         if System.is_unix():
-            return os.link(source, link_name)
+            os.link(source, link_name)
+            return
 
-        return winlink.create(source, link_name)
+        CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
+        CreateHardLink.argtypes = [ctypes.c_wchar_p,
+                                   ctypes.c_wchar_p,
+                                   ctypes.c_void_p]
+        CreateHardLink.restype = ctypes.wintypes.BOOL
+
+        res = CreateHardLink(link_name, source, None)
+        if res == 0:
+            raise ctypes.WinError()
 
     @staticmethod
     def symlink(source, link_name):
+        import ctypes
         if System.is_unix():
             return os.symlink(source, link_name)
 
@@ -33,51 +40,156 @@ class System(object):
         if func(link_name, source, flags) == 0:
             raise ctypes.WinError()
 
+    @staticmethod
+    def _reflink_darwin(src, dst):
+        import ctypes
+
+        clib = ctypes.CDLL('libc.dylib')
+        if not hasattr(clib, 'clonefile'):
+            return -1
+
+        clonefile = clib.clonefile
+        clonefile.argtypes = [ctypes.c_char_p,
+                              ctypes.c_char_p,
+                              ctypes.c_int]
+        clonefile.restype = ctypes.c_int
+
+        return clonefile(ctypes.c_char_p(src.encode('utf-8')),
+                         ctypes.c_char_p(dst.encode('utf-8')),
+                         ctypes.c_int(0))
+
+    @staticmethod
+    def _reflink_windows(src, dst):
+        return -1
+
+    @staticmethod
+    def _reflink_linux(src, dst):
+        import os
+        import fcntl
+
+        FICLONE = 0x40049409
+
+        s = open(src, 'r')
+        d = open(dst, 'w+')
+
+        try:
+            ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
+        except IOError:
+            s.close()
+            d.close()
+            os.unlink(dst)
+            raise
+
+        s.close()
+        d.close()
+
+        if ret != 0:
+            os.unlink(dst)
+
+        return ret
+
     @staticmethod
     def reflink(source, link_name):
-        from dvc.reflink import reflink
+        import platform
         from dvc.exceptions import DvcException
 
-        ret = reflink(source, link_name)
+        system = platform.system()
+        try:
+            if system == 'Windows':
+                ret = System._reflink_windows(source, link_name)
+            elif system == 'Darwin':
+                ret = System._reflink_darwin(source, link_name)
+            elif system == 'Linux':
+                ret = System._reflink_linux(source, link_name)
+            else:
+                ret = -1
+        except IOError:
+            ret = -1
+
         if ret != 0:
-            raise DvcException("Failed to reflink '{}' -> '{}' returned "
-                               "non-zero code '{}'".format(source,
-                                                           link_name,
-                                                           ret))
+            raise DvcException('Reflink is not supported')
 
-    # FIXME
-    # Temporary fix while waiting for the PR to be merged and released:
-    # https://github.com/sunshowers/ntfs/pull/11
     @staticmethod
     def getdirinfo(path):
-        from ntfsutils.fs import FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_READ
-        from ntfsutils.fs import OPEN_EXISTING, BY_HANDLE_FILE_INFORMATION
-        from ntfsutils.fs import CreateFile, WinError
-        from ntfsutils.fs import GetFileInformationByHandle, CloseHandle
+        import ctypes
+        from ctypes import c_void_p, c_wchar_p, Structure, WinError, POINTER
+        from ctypes.wintypes import DWORD, HANDLE, BOOL
+
+        FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+        FILE_SHARE_READ = 0x00000001
+        OPEN_EXISTING = 3
+
+        class FILETIME(Structure):
+            _fields_ = [("dwLowDateTime", DWORD),
+                        ("dwHighDateTime", DWORD)]
+
+        class BY_HANDLE_FILE_INFORMATION(Structure):
+            _fields_ = [("dwFileAttributes", DWORD),
+                        ("ftCreationTime", FILETIME),
+                        ("ftLastAccessTime", FILETIME),
+                        ("ftLastWriteTime", FILETIME),
+                        ("dwVolumeSerialNumber", DWORD),
+                        ("nFileSizeHigh", DWORD),
+                        ("nFileSizeLow", DWORD),
+                        ("nNumberOfLinks", DWORD),
+                        ("nFileIndexHigh", DWORD),
+                        ("nFileIndexLow", DWORD)]
+
         flags = FILE_FLAG_BACKUP_SEMANTICS
-        hfile = CreateFile(path,
-                           0,
-                           FILE_SHARE_READ,
-                           None,
-                           OPEN_EXISTING,
-                           flags,
-                           None)
+
+        func = ctypes.windll.kernel32.CreateFileW
+        func.argtypes = [c_wchar_p,
+                         DWORD,
+                         DWORD,
+                         c_void_p,
+                         DWORD,
+                         DWORD,
+                         HANDLE]
+        func.restype = HANDLE
+
+        hfile = func(path,
+                     0,
+                     FILE_SHARE_READ,
+                     None,
+                     OPEN_EXISTING,
+                     flags,
+                     None)
         if hfile is None:
             raise WinError()
+
+        func = ctypes.windll.kernel32.GetFileInformationByHandle
+        func.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)]
+        func.restype = BOOL
+
         info = BY_HANDLE_FILE_INFORMATION()
-        rv = GetFileInformationByHandle(hfile, info)
-        CloseHandle(hfile)
+        rv = func(hfile, info)
+
+        func = ctypes.windll.kernel32.CloseHandle
+        func.argtypes = [HANDLE]
+        func.restype = BOOL
+
+        func(hfile)
+
         if rv == 0:
             raise WinError()
+
         return info
 
     @staticmethod
     def inode(path):
         if System.is_unix():
-            return os.lstat(path).st_ino
-
-        # getdirinfo from ntfsutils works on both files and dirs
-        info = System.getdirinfo(path)
-        return hash((info.dwVolumeSerialNumber,
-                     info.nFileIndexHigh,
-                     info.nFileIndexLow))
+            import ctypes
+            inode = os.lstat(path).st_ino
+            # NOTE: See https://bugs.python.org/issue29619 and
+            # https://stackoverflow.com/questions/34643289/
+            # pythons-os-stat-is-returning-wrong-inode-value
+            inode = ctypes.c_ulong(inode).value
+        else:
+            # getdirinfo from ntfsutils works on both files and dirs
+            info = System.getdirinfo(path)
+            inode = abs(hash((info.dwVolumeSerialNumber,
+                              info.nFileIndexHigh,
+                              info.nFileIndexLow)))
+        assert inode >= 0
+        assert inode < 2**64
+        return inode

dvc/updater.py

@@ -6,11 +6,11 @@ from dvc import VERSION_BASE
 from dvc.logger import Logger
 
 
-class Updater(object):
+class Updater(object):  # pragma: no cover
     URL = 'https://4ki8820rsf.execute-api.us-east-2.amazonaws.com/' \
           'prod/latest-version'
     UPDATER_FILE = 'updater'
-    TIMEOUT = 7 * 24 * 60 * 60  # every week
+    TIMEOUT = 24 * 60 * 60  # every day
     TIMEOUT_GET = 10
 
     def __init__(self, dvc_dir):

requirements.txt

@@ -4,14 +4,13 @@ ply>=3.9 # See https://github.com/pyinstaller/pyinstaller/issues/1945
 configparser>=3.5.0
 zc.lockfile>=1.2.1
 future>=0.16.0
-google-cloud==0.32.0
+google-cloud-storage==1.12.0
 PyInstaller==3.3.1
 colorama>=0.3.9
 configobj>=5.0.6
-networkx==2.1
+networkx>=2.1
 pyyaml>=3.12
 gitpython>=2.1.8
-ntfsutils>=0.1.4
 setuptools>=34.0.0
 nanotime>=0.5.2
 pyasn1>=0.4.1
@@ -22,5 +21,5 @@ requests>=2.18.4
 wheel>=0.31.1
 futures>=3.2.0; python_version == "2.7"
 grandalf==0.6
-asciicanvas==0.0.3
-cffi
+pydot>=1.2.4
+asciimatics>=1.10.0

scripts/ci/script.sh

@@ -6,7 +6,8 @@ set -e
 python -mtests
 
 if [[ "$TRAVIS_PULL_REQUEST" == "false" && \
-      "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
+      "$TRAVIS_SECURE_ENV_VARS" == "true" && \
+      "$(python -c 'import sys; print(sys.version_info[0])')" == '2' ]]; then
 	codecov
 fi
 

scripts/completion/dvc.bash

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+#----------------------------------------------------------
+# Repository:  https://github.com/iterative/dvc
+#
+# References:
+#   - https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion.html
+#   - https://opensource.com/article/18/3/creating-bash-completion-script
+#----------------------------------------------------------
+
+_dvc_commands='init destroy add import checkout run pull push fetch \
+              status repro remove move gc config remote metrics install \
+              root lock unlock pipeline'
+
+_dvc_global_options="-h --help -q --quiet -V --verbose"
+_dvc_options="-h --help -v --version"
+_dvc_init="--no-scm -f --force"
+_dvc_destroy="-f --force"
+_dvc_add="-R --recursive"
+_dvc_run="--no-exec -f --file -c --cwd -d --deps -o --outs -O --outs-no-cache -M --metrics-no-cache -y --yes"
+_dvc_pull="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps"
+_dvc_push="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps"
+_dvc_fetch="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps"
+_dvc_status="--show-checksums -j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud"
+_dvc_repro="--dry -f --force -s --single-item -c --cwd -m --metrics -i --interactive -p --pipeline -P --all-pipelines"
+_dvc_remove="--dry -o --outs -p --purge"
+_dvc_gc="-a --all-branches -T --all-tags -c --cloud -r --remote"
+_dvc_config="-u --unset --local"
+_dvc_checkout="$(compgen -G '*.dvc')"
+_dvc_move="$(compgen -G '*')"
+_dvc_lock="$(compgen -G '*.dvc')"
+_dvc_unlock="$(compgen -G '*.dvc')"
+_dvc_import=""
+_dvc_remote=""
+_dvc_metrics=""
+_dvc_install=""
+_dvc_root=""
+_dvc_pipeline=""
+
+# Notes:
+#
+# `COMPREPLY` contains what will be rendered after completion is triggered
+#
+# `word` refers to the current typed word
+#
+# `${!var}` is to evaluate the content of `var` and expand its content as a variable
+#
+#       hello="world"
+#       x="hello"
+#       ${!x} ->  ${hello} ->  "world"
+#
+_dvc () {
+  local word="${COMP_WORDS[COMP_CWORD]}"
+
+  COMPREPLY=()
+
+  if [ "${COMP_CWORD}" -eq 1 ]; then
+    case "$word" in
+      -*) COMPREPLY=($(compgen -W "$_dvc_options" -- "$word")) ;;
+      *)  COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;;
+    esac
+  elif [ "${COMP_CWORD}" -eq 2 ]; then
+    local options_list="_dvc_${COMP_WORDS[1]}"
+
+    COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word"))
+  fi
+
+  return 0
+}
+
+complete -F _dvc dvc

scripts/completion/dvc.zsh

@@ -0,0 +1,218 @@
+#compdef dvc
+
+#----------------------------------------------------------
+# Repository:  https://github.com/iterative/dvc
+#
+# References:
+#   - https://github.com/zsh-users/zsh-completions
+#   - http://zsh.sourceforge.net/Doc/Release/Completion-System.html
+#   - https://mads-hartmann.com/2017/08/06/writing-zsh-completion-scripts.html
+#   - http://www.linux-mag.com/id/1106/
+#----------------------------------------------------------
+
+_dvc_commands() {
+  local _commands=(
+    "init:Initialize dvc over a directory (should already be a git dir)."
+    "destroy:Destroy dvc. Will remove all project's information, data files and cache."
+    "add:Add files/directories to dvc."
+    "import:Import files from URL."
+    "checkout:Checkout data files from cache."
+    "run:Generate a stage file from a given command and execute the command."
+    "pull:Pull data files from the cloud."
+    "push:Push data files to the cloud."
+    "fetch:Fetch data files from the cloud."
+    "status:Show the project status."
+    "repro:Reproduce DVC file. Default file name - 'Dvcfile'."
+    "remove:Remove outputs of DVC file."
+    "move:Move output of DVC file."
+    "gc:Collect garbage."
+    "config:Get or set config options."
+    "remote:Manage set of tracked repositories."
+    "metrics:Get metrics from all branches."
+    "install:Install dvc hooks into the repository."
+    "root:Relative path to project's directory."
+    "lock:Lock DVC file."
+    "unlock:Unlock DVC file."
+    "pipeline:Manage pipeline."
+  )
+
+  _describe 'dvc commands' _commands
+}
+
+_dvc_global_options=(
+  "(-)"{-h,--help}"[Show help message related to the command]"
+  "(-)"{-q,--quiet}"[Be quiet.]"
+  "(-)"{-V,--verbose}"[Be verbose.]"
+)
+
+_dvc_options=(
+  "(-)"{-h,--help}"[Show this help message and exit]"
+  "(-)"{-v,--version}"[Show program's version]"
+)
+
+_dvc_init=(
+  "--no-scm[Initiate dvc in directory that is not tracket by any scm tool]"
+  {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
+)
+
+_dvc_destroy=(
+  {-f,--force}"[Overwrite '.dvc' if it exists. Will remove all local cache.]"
+)
+
+_dvc_add=(
+  {-R,--recursive}"[Recursively add each file under the directory.]"
+)
+
+_dvc_import=(
+  "1:URL:"
+  "2:Output:"
+)
+
+_dvc_checkout=(
+  "1:DVC files:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_run=(
+  "--no-exec[Only create stage file without actually running it.]"
+  {-f,--file}"[Specify name of the stage file.]:File:_files"
+  {-c,--cwd}"[Directory to run your command and place state file in.]:CWD:_files -/"
+  "*"{-d,--deps}"[Declare dependencies for reproducible cmd.]:Dependency:_files"
+  "*"{-o,--outs}"[Declare output data file or data directory.]:Output data:_files"
+  "*"{-O,--outs-no-cache}"[Declare output regular file or directory.]:Output regular:_files"
+  "*"{-M,--metrics-no-cache}"[Declare output metric file or directory]:Metrics (no cache):_files"
+  {-y,--yes}"[Automatic 'yes' answer to all prompts.]"
+)
+
+_dvc_pull=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+)
+
+_dvc_push=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+)
+
+_dvc_fetch=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+)
+
+_dvc_status=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--show-checksums[Show checksums instead of file names]"
+  {-j,--jobs}"[Number of jobs to run simultaneously]:Number of jobs:"
+  {-r,--remote}"[Remote repository to pull from]:Remote repository:"
+  {-a,--all-branches}"[Fetch cache for all branches]"
+  {-T,--all-tags}"[Fetch cache for all tags]"
+  {-d,--with-deps}"[Fetch cache for all dependencies of the specified target]"
+  {-c,--cloud}"[Show status of a local cache compared to a remote repository]"
+)
+
+_dvc_repro=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--dry[Only print the commands that would be executed without actually executing]"
+  {-f,--force}"[Reproduce even if dependencies were not changed.]"
+  {-s,--single-item}"[Reproduce only single data item without recursive dependencies check.]"
+  {-c,--cwd}"[Directory to reproduce from]:CWD:_files -/"
+  {-m,--metrics}"[Show metrics after reproduction.]"
+  {-i,--interactive}"[Ask for confirmation before reproducing each stage.]"
+  {-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]"
+  {-P,--all-pipelines}"[Reproduce all pipelines in the project.]"
+)
+
+_dvc_remove=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+  "--dry[Only print the commands that would be executed without actually executing]"
+  {-o,--outs}"[Only remove DVC file outputs.(default)]"
+  {-p,--purge}"[Remove DVC file and all its outputs]"
+)
+
+_dvc_move=(
+  "1:Source:_files"
+  "2:Destination:"
+)
+
+_dvc_gc=(
+  {-a,--all-branches}"[Collect garbage for all branches]"
+  {-T,--all-tags}"[Collect garbage for all tags]"
+  {-c,--cloud}"[Collect garbage in remote repository]"
+  {-r,--remote}"[Remote repository to collect garbage in]:Remote repository:"
+)
+
+_dvc_config=(
+  {-u,--unset}"[Unset option.]"
+  "--local[Unset option.]"
+)
+
+_dvc_remote=(
+  "1:Task:(add remove modify list)"
+)
+
+_dvc_metrics=(
+  "1:Task:(show add modify remove)"
+)
+
+_dvc_install=()
+
+_dvc_root=()
+
+_dvc_lock=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_unlock=(
+  "*:DVC files:_files -g '(*.dvc|Dvcfile)'"
+)
+
+_dvc_pipeline=(
+  "1:Task:(show)"
+)
+
+typeset -A opt_args
+local context state line curcontext="$curcontext"
+
+_arguments \
+  $_dvc_options \
+  '1: :_dvc_commands' \
+  '*::args:->args'
+
+case $words[1] in
+  init)     _arguments $_dvc_global_options   $_dvc_init     ;;
+  destroy)  _arguments $_dvc_global_options   $_dvc_destroy  ;;
+  add)      _arguments $_dvc_global_options   $_dvc_add      ;;
+  import)   _arguments $_dvc_global_options   $_dvc_import   ;;
+  checkout) _arguments $_dvc_global_options   $_dvc_checkout ;;
+  run)      _arguments $_dvc_global_options   $_dvc_run      ;;
+  pull)     _arguments $_dvc_global_options   $_dvc_pull     ;;
+  push)     _arguments $_dvc_global_options   $_dvc_push     ;;
+  fetch)    _arguments $_dvc_global_options   $_dvc_fetch    ;;
+  status)   _arguments $_dvc_global_options   $_dvc_status   ;;
+  repro)    _arguments $_dvc_global_options   $_dvc_repro    ;;
+  remove)   _arguments $_dvc_global_options   $_dvc_remove   ;;
+  move)     _arguments $_dvc_global_options   $_dvc_move     ;;
+  gc)       _arguments $_dvc_global_options   $_dvc_gc       ;;
+  config)   _arguments $_dvc_global_options   $_dvc_config   ;;
+  remote)   _arguments $_dvc_global_options   $_dvc_remote   ;;
+  metrics)  _arguments $_dvc_global_options   $_dvc_metrics  ;;
+  install)  _arguments $_dvc_global_options   $_dvc_install  ;;
+  root)     _arguments $_dvc_global_options   $_dvc_root     ;;
+  lock)     _arguments $_dvc_global_options   $_dvc_lock     ;;
+  unlock)   _arguments $_dvc_global_options   $_dvc_unlock   ;;
+  pipeline) _arguments $_dvc_global_options   $_dvc_pipeline ;;
+esac

setup.py

@@ -12,10 +12,9 @@ install_requires = [
     "future>=0.16.0",
     "colorama>=0.3.9",
     "configobj>=5.0.6",
-    "networkx==2.1",
+    "networkx>=2.1",
     "pyyaml>=3.12",
     "gitpython>=2.1.8",
-    "ntfsutils>=0.1.4",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
@@ -23,13 +22,12 @@ install_requires = [
     "jsonpath-rw==1.4.0",
     "requests>=2.18.4",
     "grandalf==0.6",
-    "asciicanvas==0.0.3",
-    "cffi",
+    "asciimatics>=1.10.0",
 ]
 
 # Extra dependencies for remote integrations
 gs = [
-    "google-cloud==0.32.0",
+    "google-cloud-storage==1.12.0",
 ]
 s3 = [
     "boto3==1.7.4",
@@ -67,11 +65,11 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 3',
     ],
-    packages=find_packages(exclude=['bin', 'tests', 'functests']),
+    packages=find_packages(exclude=['tests']),
     include_package_data=True,
     url='http://dataversioncontrol.com',
     entry_points={
         'console_scripts': ['dvc = dvc.main:main']
     },
-    zip_safe=False
+    zip_safe=False,
 )

test-requirements.txt

@@ -1,8 +1,8 @@
 pytest>=3.3.1
 nose>=1.3.7
 mock>=2.0.0
-coverage
-codecov
+coverage>=4.5.1
+codecov>=2.0.15
 xmltodict>=0.11.0
 awscli>=1.14.18
 google-compute-engine

tests/__main__.py

@@ -7,4 +7,5 @@ os.chdir(REPO_ROOT)
 os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), os.getenv('PATH')))
 os.putenv('DVC_HOME', REPO_ROOT)
 
-check_call('nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage', shell=True)
+cmd = 'nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage'
+check_call(cmd, shell=True)

tests/basic_env.py

@@ -1,4 +1,5 @@
 import os
+import uuid
 import shutil
 import tempfile
 from git import Repo
@@ -38,8 +39,15 @@ class TestDir(TestCase):
         with open(name, 'a') as f:
             f.write(contents)
 
+    @staticmethod
+    def mkdtemp():
+        prefix = 'dvc-test.{}.'.format(os.getpid())
+        suffix = '.{}'.format(uuid.uuid4())
+        return tempfile.mkdtemp(prefix=prefix, suffix=suffix)
+
     def setUp(self):
-        self._root_dir = tempfile.mkdtemp()
+        self._root_dir = TestDir.mkdtemp()
+
         self._pushd(self._root_dir)
         self.create(self.FOO, self.FOO_CONTENTS)
         self.create(self.BAR, self.BAR_CONTENTS)

tests/test_add.py

@@ -2,7 +2,6 @@ import os
 import stat
 import shutil
 import filecmp
-import tempfile
 
 from dvc.main import main
 from dvc.utils import file_md5
@@ -132,7 +131,7 @@ class TestAddFileInDir(TestDvc):
 
 class TestAddExternalLocalFile(TestDvc):
     def test(self):
-        dname = tempfile.mkdtemp()
+        dname = TestDvc.mkdtemp()
         fname = os.path.join(dname, 'foo')
         shutil.copyfile(self.FOO, fname)
 

tests/test_cache.py

@@ -1,6 +1,5 @@
 import os
 import shutil
-import tempfile
 
 from dvc.cache import Cache
 from dvc.system import System
@@ -52,7 +51,7 @@ class TestCacheLoadBadDirCache(TestDvc):
 
 class TestExternalCacheDir(TestDvc):
     def test(self):
-        cache_dir = tempfile.mkdtemp()
+        cache_dir = TestDvc.mkdtemp()
 
         ret = main(['config', 'cache.dir', cache_dir])
         self.assertEqual(ret, 0)
@@ -103,3 +102,12 @@ class TestSharedCacheDir(TestDir):
         self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1)
         self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1)
         self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1)
+
+
+class TestCacheLinkType(TestDvc):
+    def test(self):
+        ret = main(['config', 'cache.type', 'reflink,copy'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['add', self.FOO])
+        self.assertEqual(ret, 0)

tests/test_checkout.py

@@ -6,6 +6,7 @@ import shutil
 import filecmp
 
 from dvc.main import main
+from dvc.project import Project
 from tests.basic_env import TestDvc
 from tests.test_repro import TestRepro
 from dvc.stage import Stage
@@ -65,6 +66,34 @@ class TestCheckoutCorruptedCacheFile(TestRepro):
         self.assertFalse(os.path.isfile(cache))
 
 
+class TestCheckoutCorruptedCacheDir(TestDvc):
+    def test(self):
+        time.sleep(1)
+        
+        # NOTE: using 'copy' so that cache and link don't have same inode
+        ret = main(['config', 'cache.type', 'copy'])
+        self.assertEqual(ret, 0)
+
+        self.dvc = Project('.')
+        stages = self.dvc.add(self.DATA_DIR)
+        self.assertEqual(len(stages), 1)
+        self.assertEqual(len(stages[0].outs), 1)
+        out = stages[0].outs[0]
+
+        # NOTE: modifying cache file for one of the files inside the directory
+        # to check if dvc will detect that the cache is corrupted.
+        entry = self.dvc.cache.local.load_dir_cache(out.md5)[0]
+        md5 = entry[self.dvc.cache.local.PARAM_MD5]
+        cache = self.dvc.cache.local.get(md5)
+
+        with open(cache, 'w+') as fobj:
+            fobj.write('1')
+
+        self.dvc.checkout()
+
+        self.assertFalse(os.path.exists(cache))
+
+
 class TestCmdCheckout(TestCheckout):
     def test(self):
         ret = main(['checkout'])
@@ -205,3 +234,15 @@ class TestCheckoutEmptyDir(TestDvc):
 
         self.assertTrue(os.path.isdir(dname))
         self.assertEqual(len(os.listdir(dname)), 0)
+
+
+class TestCheckoutNotCachedFile(TestDvc):
+    def test(self):
+        cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
+
+        self.dvc.add(self.FOO)
+        stage = self.dvc.run(cmd=cmd,
+                             deps=[self.FOO, self.CODE],
+                             outs_no_cache=['out'])
+
+        self.dvc.checkout()

tests/test_data_cloud.py

@@ -1,12 +1,11 @@
 from subprocess import CalledProcessError
-from subprocess import check_output
+from subprocess import check_output, Popen
 from unittest import SkipTest
 import os
 import time
 import uuid
 import shutil
 import getpass
-import tempfile
 import platform
 
 from dvc.main import main
@@ -94,11 +93,18 @@ def _should_test_hdfs():
     except (CalledProcessError, IOError):
         return False
 
+    p = Popen('hadoop fs -ls hdfs://127.0.0.1/',
+              shell=True,
+              executable=os.getenv('SHELL'))
+    p.communicate()
+    if p.returncode != 0:
+        return False
+
     return True
 
 
 def get_local_storagepath():
-    return tempfile.mkdtemp()
+    return TestDvc.mkdtemp()
 
 
 def get_local_url():
@@ -154,10 +160,10 @@ class TestDataCloud(TestDvc):
     def test(self):
         config = TEST_CONFIG
 
-        clist = [('s3://', RemoteS3),
-                 ('gs://', RemoteGS),
+        clist = [('s3://mybucket/', RemoteS3),
+                 ('gs://mybucket/', RemoteGS),
                  ('ssh://user@localhost:/', RemoteSSH),
-                 (tempfile.mkdtemp(), RemoteLOCAL)]
+                 (TestDvc.mkdtemp(), RemoteLOCAL)]
 
         if _should_test_azure():
             clist.append(('azure://ContainerName=', RemoteAzure))

tests/test_gc.py

@@ -29,7 +29,8 @@ class TestGC(TestDvc):
         self._test_gc()
 
     def test_cli(self):
-        ret = main(['gc'])
+        ret = main(['gc', '-f'])
+        self.assertEqual(ret, 0)
         self._test_gc()
 
     def _test_gc(self):

tests/test_move.py

@@ -1,9 +1,10 @@
 import os
 
 from dvc.main import main
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, MoveNotDataSourceError
 
 from tests.basic_env import TestDvc
+from tests.test_repro import TestRepro
 
 
 class TestMove(TestDvc):
@@ -39,3 +40,15 @@ class TestCmdMove(TestDvc):
 
         ret = main(['move', 'non-existing-file', 'dst'])
         self.assertNotEqual(ret, 0)
+
+
+class TestMoveNotDataSource(TestRepro):
+    def test(self):
+        from dvc.project import Project
+
+        self.dvc = Project(self._root_dir)
+        with self.assertRaises(MoveNotDataSourceError):
+            self.dvc.move(self.file1, 'dst')
+
+        ret = main(['move', self.file1, 'dst'])
+        self.assertNotEqual(ret, 0)

tests/test_pipeline.py

@@ -2,12 +2,14 @@ from dvc.main import main
 
 from tests.basic_env import TestDvc
 from tests.test_repro import TestRepro, TestReproChangedDeepData
+import os
 
 
 class TestPipelineShowSingle(TestDvc):
     def setUp(self):
         super(TestPipelineShowSingle, self).setUp()
         self.stage = 'foo.dvc'
+        self.dotFile = 'graph.dot'
         ret = main(['add', self.FOO])
         self.assertEqual(ret, 0)
 
@@ -27,6 +29,11 @@ class TestPipelineShowSingle(TestDvc):
         ret = main(['pipeline', 'show', '--ascii', self.stage])
         self.assertEqual(ret, 0)
 
+    def test_dot(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_ascii_commands(self):
         ret = main(['pipeline', 'show', '--ascii', self.stage, '--commands'])
         self.assertEqual(ret, 0)
@@ -35,6 +42,16 @@ class TestPipelineShowSingle(TestDvc):
         ret = main(['pipeline', 'show', '--ascii', self.stage, '--outs'])
         self.assertEqual(ret, 0)
 
+    def test_dot_commands(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, '--commands'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_dot_outs(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, '--outs'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_not_dvc_file(self):
         ret = main(['pipeline', 'show', self.FOO])
         self.assertNotEqual(ret, 0)
@@ -45,6 +62,10 @@ class TestPipelineShowSingle(TestDvc):
 
 
 class TestPipelineShow(TestRepro):
+    def setUp(self):
+        super(TestPipelineShow, self).setUp()
+        self.dotFile = 'graph.dot'
+
     def test(self):
         ret = main(['pipeline', 'show', self.file1_stage])
         self.assertEqual(ret, 0)
@@ -61,6 +82,11 @@ class TestPipelineShow(TestRepro):
         ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
         self.assertEqual(ret, 0)
 
+    def test_dot(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_ascii_commands(self):
         ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands'])
         self.assertEqual(ret, 0)
@@ -69,6 +95,16 @@ class TestPipelineShow(TestRepro):
         ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs'])
         self.assertEqual(ret, 0)
 
+    def test_dot_commands(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_dot_outs(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_not_dvc_file(self):
         ret = main(['pipeline', 'show', self.file1])
         self.assertNotEqual(ret, 0)
@@ -79,6 +115,10 @@ class TestPipelineShow(TestRepro):
 
 
 class TestPipelineShowDeep(TestReproChangedDeepData):
+    def setUp(self):
+        super(TestPipelineShowDeep, self).setUp()
+        self.dotFile = 'graph.dot'
+
     def test(self):
         ret = main(['pipeline', 'show', self.file1_stage])
         self.assertEqual(ret, 0)
@@ -95,6 +135,11 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
         ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
         self.assertEqual(ret, 0)
 
+    def test_dot(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_ascii_commands(self):
         ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--commands'])
         self.assertEqual(ret, 0)
@@ -103,6 +148,16 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
         ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs'])
         self.assertEqual(ret, 0)
 
+    def test_dot_commands(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--commands'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
+    def test_dot_outs(self):
+        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.file1_stage, '--outs'])
+        self.assertEqual(ret, 0)
+        self.assertTrue(os.path.isfile(self.dotFile))
+
     def test_not_dvc_file(self):
         ret = main(['pipeline', 'show', self.file1])
         self.assertNotEqual(ret, 0)

tests/test_remote.py

@@ -72,3 +72,16 @@ class TestRemoteRemoveDefault(TestDvc):
         core = local_config.get(Config.SECTION_CORE, None)
         if core is not None:
             self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys())
+
+
+class TestRemoteRemove(TestDvc):
+    def test(self):
+        ret = main(['config', 'core.jobs', '1'])
+        self.assertEqual(ret, 0)
+
+        remote = 'mys3'
+        ret = main(['remote', 'add', remote, 's3://bucket/name'])
+        self.assertEqual(ret, 0)
+
+        ret = main(['remote', 'remove', remote])
+        self.assertEqual(ret, 0)

tests/test_repro.py

@@ -4,7 +4,6 @@ import stat
 import shutil
 import filecmp
 import getpass
-import tempfile
 import posixpath
 from subprocess import Popen, PIPE
 
@@ -146,6 +145,12 @@ class TestReproDry(TestReproChangedData):
         self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
 
 
+class TestReproUpToDate(TestRepro):
+    def test(self):
+        ret = main(['repro', self.file1_stage])
+        self.assertEqual(ret, 0)
+
+
 class TestReproDryNoExec(TestDvc):
     def test(self):
         deps = []
@@ -217,6 +222,49 @@ class TestReproPipeline(TestReproChangedDeepData):
         self.assertEqual(ret, 0)
 
 
+class TestReproPipelines(TestDvc):
+    def setUp(self):
+        super(TestReproPipelines, self).setUp()
+
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.foo_stage = stages[0]
+        self.assertTrue(self.foo_stage is not None)
+
+        stages = self.dvc.add(self.BAR)
+        self.assertEqual(len(stages), 1)
+        self.bar_stage = stages[0]
+        self.assertTrue(self.bar_stage is not None)
+
+        self.file1 = 'file1'
+        self.file1_stage = self.file1 + '.dvc'
+        self.dvc.run(fname=self.file1_stage,
+                     outs=[self.file1],
+                     deps=[self.FOO, self.CODE],
+                     cmd='python {} {} {}'.format(self.CODE, self.FOO, self.file1))
+
+        self.file2 = 'file2'
+        self.file2_stage = self.file2 + '.dvc'
+        self.dvc.run(fname=self.file2_stage,
+                     outs=[self.file2],
+                     deps=[self.BAR, self.CODE],
+                     cmd='python {} {} {}'.format(self.CODE, self.BAR, self.file2))
+
+
+    def test(self):
+        stages = self.dvc.reproduce(all_pipelines=True, force=True)
+        self.assertEqual(len(stages), 4)
+        names = [stage.relpath for stage in stages]
+        self.assertTrue(self.foo_stage.relpath in names)
+        self.assertTrue(self.bar_stage.relpath in names)
+        self.assertTrue(self.file1_stage in names)
+        self.assertTrue(self.file2_stage in names)
+
+    def test_cli(self):
+        ret = main(['repro', '-f', '-P'])
+        self.assertEqual(ret, 0)
+
+
 class TestReproLocked(TestReproChangedData):
     def test(self):
         file2 = 'file2'
@@ -606,7 +654,7 @@ class TestReproExternalSSH(TestReproExternalBase):
     @property
     def bucket(self):
         if not self._dir:
-            self._dir = tempfile.mkdtemp()
+            self._dir = TestDvc.mkdtemp()
         return '{}@127.0.0.1:{}'.format(getpass.getuser(), self._dir)
 
     def cmd(self, i, o):
@@ -653,7 +701,7 @@ class TestReproExternalSSH(TestReproExternalBase):
 class TestReproExternalLOCAL(TestReproExternalBase):
     def setUp(self):
         super(TestReproExternalLOCAL, self).setUp()
-        self.tmpdir = tempfile.mkdtemp()
+        self.tmpdir = TestDvc.mkdtemp()
 
     def should_test(self):
         return True