Merge pull request #977 from efiop/master

repro: introduce --dry and --interactive

dvc/cli.py

@@ -324,6 +324,19 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Show metrics after reproduction')
+    repro_parser.add_argument(
+                        '--dry',
+                        action='store_true',
+                        default=False,
+                        help='Only print the commands that would be executed '
+                             'without actually executing.')
+    repro_parser.add_argument(
+                        '-i',
+                        '--interactive',
+                        action='store_true',
+                        default=False,
+                        help='Ask for confirmation before reproducing each '
+                             'stage.')
     repro_parser.set_defaults(func=CmdRepro)
 
     # Remove

dvc/command/repro.py

@@ -17,7 +17,9 @@ class CmdRepro(CmdBase):
             try:
                 self.project.reproduce(target,
                                        recursive=recursive,
-                                       force=self.args.force)
+                                       force=self.args.force,
+                                       dry=self.args.dry,
+                                       interactive=self.args.interactive)
                 if self.args.metrics:
                     self.project.metrics_show()
             except ReproductionError as ex:

dvc/project.py

@@ -235,7 +235,7 @@ class Project(object):
         stage.dump()
         return stage
 
-    def _reproduce_stage(self, stages, node, force):
+    def _reproduce_stage(self, stages, node, force, dry, interactive):
         stage = stages[node]
 
         if stage.locked:
@@ -243,13 +243,21 @@ class Project(object):
                   'going to be reproduced.'
             self.logger.warn(msg.format(stage.relpath))
 
-        stage = stage.reproduce(force=force)
+        stage = stage.reproduce(force=force, dry=dry, interactive=interactive)
         if not stage:
             return []
-        stage.dump()
+
+        if not dry:
+            stage.dump()
+
         return [stage]
 
-    def reproduce(self, target, recursive=True, force=False):
+    def reproduce(self,
+                  target,
+                  recursive=True,
+                  force=False,
+                  dry=False,
+                  interactive=False):
         import networkx as nx
 
         G = self.graph()[1]
@@ -259,17 +267,26 @@ class Project(object):
             raise NotDvcFileError(target)
 
         if recursive:
-            return self._reproduce_stages(G, stages, node, force)
+            return self._reproduce_stages(G,
+                                          stages,
+                                          node,
+                                          force,
+                                          dry,
+                                          interactive)
 
-        return self._reproduce_stage(stages, node, force)
+        return self._reproduce_stage(stages, node, force, dry, interactive)
 
-    def _reproduce_stages(self, G, stages, node, force):
+    def _reproduce_stages(self, G, stages, node, force, dry, interactive):
         import networkx as nx
 
         result = []
         for n in nx.dfs_postorder_nodes(G, node):
             try:
-                result += self._reproduce_stage(stages, n, force)
+                result += self._reproduce_stage(stages,
+                                                n,
+                                                force,
+                                                dry,
+                                                interactive)
             except Exception as ex:
                 raise ReproductionError(stages[n].relpath, ex)
         return result

dvc/stage.py

@@ -150,17 +150,22 @@ class Stage(object):
         self.remove_outs(ignore_remove=True)
         os.unlink(self.path)
 
-    def reproduce(self, force=False):
+    def reproduce(self, force=False, dry=False, interactive=False):
         if not self.changed() and not force:
             return None
 
-        if (self.cmd or self.is_import) and not self.locked:
+        if (self.cmd or self.is_import) and not self.locked and not dry:
             # Removing outputs only if we actually have command to reproduce
             self.remove_outs(ignore_remove=False)
 
+        msg = "Going to reproduce '{}'. Are you sure you want to continue?"
+        msg = msg.format(self.relpath)
+        if interactive and not prompt(msg):
+            raise DvcException('Reproduction aborted by the user')
+
         self.project.logger.info(u'Reproducing \'{}\''.format(self.relpath))
 
-        self.run()
+        self.run(dry=dry)
 
         msg = u'\'{}\' was reproduced'.format(self.relpath)
         self.project.logger.debug(msg)
@@ -297,35 +302,40 @@ class Stage(object):
         for out in self.outs:
             out.save()
 
-    def run(self):
+    def run(self, dry=False):
         if self.locked:
             msg = u'Verifying outputs in locked stage \'{}\''
             self.project.logger.info(msg.format(self.relpath))
-            self.check_missing_outputs()
+            if not dry:
+                self.check_missing_outputs()
         elif self.is_import:
             msg = u'Importing \'{}\' -> \'{}\''
             self.project.logger.info(msg.format(self.deps[0].path,
                                                 self.outs[0].path))
 
-            self.deps[0].download(self.outs[0].path_info)
+            if not dry:
+                self.deps[0].download(self.outs[0].path_info)
         elif self.is_data_source:
             msg = u'Verifying data sources in \'{}\''.format(self.relpath)
             self.project.logger.info(msg)
-            self.check_missing_outputs()
+            if not dry:
+                self.check_missing_outputs()
         else:
             msg = u'Running command:\n\t{}'.format(self.cmd)
             self.project.logger.info(msg)
 
-            p = subprocess.Popen(self.cmd,
-                                 cwd=self.cwd,
-                                 shell=True,
-                                 env=os.environ,
-                                 executable=os.getenv('SHELL'))
-            p.communicate()
-            if p.returncode != 0:
-                raise StageCmdFailedError(self)
+            if not dry:
+                p = subprocess.Popen(self.cmd,
+                                     cwd=self.cwd,
+                                     shell=True,
+                                     env=os.environ,
+                                     executable=os.getenv('SHELL'))
+                p.communicate()
+                if p.returncode != 0:
+                    raise StageCmdFailedError(self)
 
-        self.save()
+        if not dry:
+            self.save()
 
     def check_missing_outputs(self):
         outs = [out for out in self.outs if not out.exists]

tests/test_add.py

@@ -1,7 +1,6 @@
 import os
 import stat
 import shutil
-import filecmp
 
 from dvc.main import main
 from dvc.utils import file_md5

tests/test_checkout.py

@@ -32,7 +32,7 @@ class TestCheckout(TestRepro):
 
     def _test_checkout(self):
         self.assertTrue(os.path.isfile(self.FOO))
-        self.assertTrue(filecmp.cmp(self.FOO, self.orig))
+        self.assertTrue(filecmp.cmp(self.FOO, self.orig, shallow=False))
 
 
 class TestCheckoutSingleStage(TestCheckout):

tests/test_import.py

@@ -1,7 +1,6 @@
 import os
 import stat
 import shutil
-import filecmp
 
 from dvc.main import main
 from dvc.command.imp import CmdImport

tests/test_repro.py

@@ -61,7 +61,7 @@ class TestReproDepUnderDir(TestDvc):
                      deps=[self.DATA, self.CODE],
                      cmd='python {} {} {}'.format(self.CODE, self.DATA, self.file1))
 
-        self.assertTrue(filecmp.cmp(self.file1, self.DATA))
+        self.assertTrue(filecmp.cmp(self.file1, self.DATA, shallow=False))
 
         os.unlink(self.DATA)
         shutil.copyfile(self.FOO, self.DATA)
@@ -70,7 +70,7 @@ class TestReproDepUnderDir(TestDvc):
 
         stages = self.dvc.reproduce(self.file1_stage)
         self.assertEqual(len(stages), 2)
-        self.assertTrue(filecmp.cmp(self.file1, self.FOO))
+        self.assertTrue(filecmp.cmp(self.file1, self.FOO, shallow=False))
 
 
 class TestReproNoDeps(TestRepro):
@@ -101,7 +101,7 @@ class TestReproChangedCode(TestRepro):
 
         stages = self.dvc.reproduce(self.file1_stage)
 
-        self.assertTrue(filecmp.cmp(self.file1, self.BAR))
+        self.assertTrue(filecmp.cmp(self.file1, self.BAR, shallow=False))
         self.assertEqual(len(stages), 1)
 
     def swap_code(self):
@@ -117,7 +117,7 @@ class TestReproChangedData(TestRepro):
 
         stages = self.dvc.reproduce(self.file1_stage)
 
-        self.assertTrue(filecmp.cmp(self.file1, self.BAR))
+        self.assertTrue(filecmp.cmp(self.file1, self.BAR, shallow=False))
         self.assertEqual(len(stages), 2)
 
     def swap_foo_with_bar(self):
@@ -125,6 +125,19 @@ class TestReproChangedData(TestRepro):
         shutil.copyfile(self.BAR, self.FOO)
 
 
+class TestReproDry(TestReproChangedData):
+    def test(self):
+        self.swap_foo_with_bar()
+
+        stages = self.dvc.reproduce(self.file1_stage, dry=True)
+
+        self.assertTrue(len(stages), 2)
+        self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
+
+        ret = main(['repro', '--dry'])
+        self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False))
+
+
 class TestReproChangedDeepData(TestReproChangedData):
     def test(self):
         file2 = 'file2'
@@ -138,8 +151,8 @@ class TestReproChangedDeepData(TestReproChangedData):
 
         stages = self.dvc.reproduce(file2_stage)
 
-        self.assertTrue(filecmp.cmp(self.file1, self.BAR))
-        self.assertTrue(filecmp.cmp(file2, self.BAR))
+        self.assertTrue(filecmp.cmp(self.file1, self.BAR, shallow=False))
+        self.assertTrue(filecmp.cmp(file2, self.BAR, shallow=False))
         self.assertEqual(len(stages), 3)
 
 
@@ -162,8 +175,8 @@ class TestReproLocked(TestReproChangedData):
         ret = main(['unlock', file2_stage])
         self.assertEqual(ret, 0)
         stages = self.dvc.reproduce(file2_stage)
-        self.assertTrue(filecmp.cmp(self.file1, self.BAR))
-        self.assertTrue(filecmp.cmp(file2, self.BAR))
+        self.assertTrue(filecmp.cmp(self.file1, self.BAR, shallow=False))
+        self.assertTrue(filecmp.cmp(file2, self.BAR, shallow=False))
         self.assertEqual(len(stages), 3)
 
     def test_non_existing(self):
@@ -182,7 +195,7 @@ class TestReproPhony(TestReproChangedData):
 
         self.dvc.reproduce(stage.path)
 
-        self.assertTrue(filecmp.cmp(self.file1, self.BAR))
+        self.assertTrue(filecmp.cmp(self.file1, self.BAR, shallow=False))
 
 
 class TestNonExistingOutput(TestRepro):
@@ -199,7 +212,7 @@ class TestReproDataSource(TestReproChangedData):
 
         stages = self.dvc.reproduce(self.foo_stage.path)
 
-        self.assertTrue(filecmp.cmp(self.FOO, self.BAR))
+        self.assertTrue(filecmp.cmp(self.FOO, self.BAR, shallow=False))
         self.assertEqual(stages[0].outs[0].md5, file_md5(self.BAR)[0])
 
 
@@ -280,7 +293,7 @@ class TestCmdReproChdir(TestDvc):
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(foo))
         self.assertTrue(os.path.isfile(bar))
-        self.assertTrue(filecmp.cmp(foo, bar))
+        self.assertTrue(filecmp.cmp(foo, bar, shallow=False))
 
         os.unlink(bar)
 
@@ -289,7 +302,7 @@ class TestCmdReproChdir(TestDvc):
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(foo))
         self.assertTrue(os.path.isfile(bar))
-        self.assertTrue(filecmp.cmp(foo, bar))
+        self.assertTrue(filecmp.cmp(foo, bar, shallow=False))
 
 
 class TestReproExternalBase(TestDvc):
@@ -348,7 +361,7 @@ class TestReproExternalBase(TestDvc):
 
         import_stage = self.dvc.imp(out_foo_path, 'import')
         self.assertTrue(os.path.exists('import'))
-        self.assertTrue(filecmp.cmp('import', self.FOO))
+        self.assertTrue(filecmp.cmp('import', self.FOO, shallow=False))
 
         import_remote_stage = self.dvc.imp(out_foo_path, out_foo_path + '_imported')
 
@@ -363,7 +376,7 @@ class TestReproExternalBase(TestDvc):
         stages = self.dvc.reproduce(import_stage.path)
         self.assertEqual(len(stages), 1)
         self.assertTrue(os.path.exists('import'))
-        self.assertTrue(filecmp.cmp('import', self.BAR))
+        self.assertTrue(filecmp.cmp('import', self.BAR, shallow=False))
 
         stages = self.dvc.reproduce(cmd_stage.path)
         self.assertEqual(len(stages), 1)

tests/test_run.py

@@ -26,7 +26,7 @@ class TestRun(TestDvc):
                              fname=fname,
                              cwd=cwd)
 
-        self.assertTrue(filecmp.cmp(self.FOO, 'out'))
+        self.assertTrue(filecmp.cmp(self.FOO, 'out', shallow=False))
         self.assertTrue(os.path.isfile(stage.path))
         self.assertEqual(stage.cmd, cmd)
         self.assertEqual(len(stage.deps), len(deps))
@@ -64,7 +64,7 @@ class TestCmdRun(TestDvc):
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile('out'))
         self.assertTrue(os.path.isfile('out.dvc'))
-        self.assertTrue(filecmp.cmp(self.FOO, 'out'))
+        self.assertTrue(filecmp.cmp(self.FOO, 'out', shallow=False))
 
     def test_run_bad_command(self):
         ret = main(['run',

tests/test_utils.py

@@ -18,10 +18,10 @@ class TestUtils(TestDvc):
         os.mkdir(dest_dir)
 
         utils.copyfile(src, dest)
-        self.assertTrue(filecmp.cmp(src, dest))
+        self.assertTrue(filecmp.cmp(src, dest, shallow=False))
 
         utils.copyfile(src, dest_dir)
-        self.assertTrue(filecmp.cmp(src, '{}/{}'.format(dest_dir, src)))
+        self.assertTrue(filecmp.cmp(src, '{}/{}'.format(dest_dir, src), shallow=False))
 
         shutil.rmtree(dest_dir)
         os.remove(src)