From c6f642fd6bb0a4e65e4882c7053ee9452f48b1b5 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 21 Aug 2018 10:34:57 +0300 Subject: [PATCH] dvc: add --recursive Fixes #1026 Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 6 +++++ dvc/command/add.py | 2 +- dvc/project.py | 36 +++++++++++++++++++++------ tests/test_add.py | 54 ++++++++++++++++++++++++++++++++-------- tests/test_checkout.py | 15 ++++++++--- tests/test_data_cloud.py | 21 +++++++++++++--- tests/test_metrics.py | 4 ++- tests/test_remove.py | 15 ++++++++--- tests/test_repro.py | 10 ++++++-- 9 files changed, 131 insertions(+), 32 deletions(-) diff --git a/dvc/cli.py b/dvc/cli.py index 4291202b8..ff05e968a 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -139,6 +139,12 @@ def parse_args(argv=None): parents=[parent_parser], description=ADD_HELP, help=ADD_HELP) + add_parser.add_argument( + '-R', + '--recursive', + action='store_true', + default=False, + help='Recursively add each file under the directory.') add_parser.add_argument( 'targets', nargs='+', diff --git a/dvc/command/add.py b/dvc/command/add.py index d21bc1493..ae11035bb 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -7,7 +7,7 @@ class CmdAdd(CmdBase): def run(self): for target in self.args.targets: try: - self.project.add(target) + self.project.add(target, recursive=self.args.recursive) except DvcException as ex: Logger.error('Failed to add \'{}\''.format(target), ex) return 1 diff --git a/dvc/project.py b/dvc/project.py index ad2dbde98..464b01437 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -132,16 +132,36 @@ class Project(object): stages = [o.stage.relpath, out.stage.relpath] raise OutputDuplicationError(o.path, stages) - def add(self, fname): - stage = Stage.loads(project=self, - outs=[fname], - add=True) + def add(self, fname, recursive=False): + fnames = [] + if recursive and os.path.isdir(fname): + fnames = [] + for root, dirs, files in os.walk(fname): + for f in files: + path = os.path.join(root, f) + if Stage.is_stage_file(path): + continue + if os.path.basename(path) == self.scm.ignore_file(): + continue + if self.scm.is_tracked(path): + continue + fnames.append(path) + else: + fnames = [fname] - self._check_output_duplication(stage.outs) + stages = [] + for f in fnames: + stage = Stage.loads(project=self, + outs=[f], + add=True) - stage.save() - stage.dump() - return stage + self._check_output_duplication(stage.outs) + + stage.save() + stage.dump() + stages.append(stage) + + return stages def remove(self, target, outs_only=False): stage = Stage.load(self, target) diff --git a/tests/test_add.py b/tests/test_add.py index 54741a1eb..9fa7436f5 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -19,7 +19,10 @@ class TestAdd(TestDvc): def test(self): md5 = file_md5(self.FOO)[0] - stage = self.dvc.add(self.FOO) + stages = self.dvc.add(self.FOO) + self.assertEqual(len(stages), 1) + stage = stages[0] + self.assertTrue(stage is not None) self.assertIsInstance(stage, Stage) self.assertTrue(os.path.isfile(stage.path)) @@ -40,8 +43,10 @@ class TestAddDirectory(TestDvc): dname = 'directory' os.mkdir(dname) self.create(os.path.join(dname, 'file'), 'file') - stage = self.dvc.add(dname) - self.assertNotEqual(stage, None) + stages = self.dvc.add(dname) + self.assertEqual(len(stages), 1) + stage = stages[0] + self.assertTrue(stage is not None) self.assertEqual(len(stage.deps), 0) self.assertEqual(len(stage.outs), 1) @@ -52,12 +57,29 @@ class TestAddDirectory(TestDvc): self.assertTrue('\\' not in info['relpath']) +class TestAddDirectoryRecursive(TestDvc): + def test(self): + stages = self.dvc.add(self.DATA_DIR, recursive=True) + self.assertEqual(len(stages), 2) + + +class TestAddCmdDirectoryRecursive(TestDvc): + def test(self): + ret = main(['add', + '--recursive', + self.DATA_DIR]) + self.assertEqual(ret, 0) + + class TestAddDirectoryWithForwardSlash(TestDvc): def test(self): dname = 'directory/' os.mkdir(dname) self.create(os.path.join(dname, 'file'), 'file') - stage = self.dvc.add(dname) + stages = self.dvc.add(dname) + self.assertEqual(len(stages), 1) + stage = stages[0] + self.assertTrue(stage is not None) self.assertEquals(os.path.abspath('directory.dvc'), stage.path) class TestAddTrackedFile(TestDvc): @@ -78,20 +100,30 @@ class TestAddDirWithExistingCache(TestDvc): os.mkdir(dname) shutil.copyfile(self.FOO, fname) - self.dvc.add(self.FOO) - self.dvc.add(dname) + stages = self.dvc.add(self.FOO) + self.assertEqual(len(stages), 1) + self.assertTrue(stages[0] is not None) + stages = self.dvc.add(dname) + self.assertEqual(len(stages), 1) + self.assertTrue(stages[0] is not None) class TestAddModifiedDir(TestDvc): def test(self): - self.dvc.add(self.DATA_DIR) + stages = self.dvc.add(self.DATA_DIR) + self.assertEqual(len(stages), 1) + self.assertTrue(stages[0] is not None) os.unlink(self.DATA) - self.dvc.add(self.DATA_DIR) + stages = self.dvc.add(self.DATA_DIR) + self.assertEqual(len(stages), 1) + self.assertTrue(stages[0] is not None) class TestAddFileInDir(TestDvc): def test(self): - stage = self.dvc.add(self.DATA_SUB) + stages = self.dvc.add(self.DATA_SUB) + self.assertEqual(len(stages), 1) + stage = stages[0] self.assertNotEqual(stage, None) self.assertEqual(len(stage.deps), 0) self.assertEqual(len(stage.outs), 1) @@ -104,7 +136,9 @@ class TestAddExternalLocalFile(TestDvc): fname = os.path.join(dname, 'foo') shutil.copyfile(self.FOO, fname) - stage = self.dvc.add(fname) + stages = self.dvc.add(fname) + self.assertEqual(len(stages), 1) + stage = stages[0] self.assertNotEqual(stage, None) self.assertEqual(len(stage.deps), 0) self.assertEqual(len(stage.outs), 1) diff --git a/tests/test_checkout.py b/tests/test_checkout.py index fc226ae39..fb0037bff 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -16,7 +16,11 @@ class TestCheckout(TestRepro): def setUp(self): super(TestCheckout, self).setUp() - self.data_dir_stage = self.dvc.add(self.DATA_DIR) + + stages = self.dvc.add(self.DATA_DIR) + self.assertEqual(len(stages), 1) + self.data_dir_stage = stages[0] + self.assertTrue(self.data_dir_stage is not None) self.orig = 'orig' shutil.copy(self.FOO, self.orig) @@ -74,7 +78,9 @@ class CheckoutBase(TestDvc): def commit_data_file(self, fname, content='random text'): with open(fname, 'w') as fd: fd.write(content) - self.dvc.add(fname) + stages = self.dvc.add(fname) + self.assertEqual(len(stages), 1) + self.assertTrue(stages[0] is not None) self.dvc.scm.add([fname + '.dvc', '.gitignore']) self.dvc.scm.commit('adding ' + fname) @@ -181,7 +187,10 @@ class TestCheckoutEmptyDir(TestDvc): dname = 'empty_dir' os.mkdir(dname) - stage = self.dvc.add(dname) + stages = self.dvc.add(dname) + self.assertEqual(len(stages), 1) + stage = stages[0] + self.assertTrue(stage is not None) self.assertEqual(len(stage.outs), 1) stage.outs[0].remove() diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 1c38a20e8..134273983 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -204,12 +204,19 @@ class TestDataCloudBase(TestDvc): def _test_cloud(self): self._setup_cloud() - stage = self.dvc.add(self.FOO) + stages = self.dvc.add(self.FOO) + self.assertEqual(len(stages), 1) + stage = stages[0] + self.assertTrue(stage is not None) cache = stage.outs[0].cache info = stage.outs[0].dumpd() md5 = info['md5'] - stage_dir = self.dvc.add(self.DATA_DIR) + stages = self.dvc.add(self.DATA_DIR) + self.assertEqual(len(stages), 1) + stage_dir = stages[0] + self.assertTrue(stage_dir is not None) + cache_dir = stage_dir.outs[0].cache info_dir = stage_dir.outs[0].dumpd() md5_dir = info_dir['md5'] @@ -358,10 +365,16 @@ class TestDataCloudCLIBase(TestDvc): else: args += [] - stage = self.dvc.add(self.FOO) + stages = self.dvc.add(self.FOO) + self.assertEqual(len(stages), 1) + stage = stages[0] + self.assertTrue(stage is not None) cache = stage.outs[0].cache - stage_dir = self.dvc.add(self.DATA_DIR) + stages = self.dvc.add(self.DATA_DIR) + self.assertEqual(len(stages), 1) + stage_dir = stages[0] + self.assertTrue(stage_dir is not None) cache_dir = stage_dir.outs[0].cache # FIXME check status output diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 370a97a3b..3aa814221 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -219,7 +219,9 @@ class TestCachedMetrics(TestDvc): with open('metrics.json', 'w+') as fd: json.dump({'metrics': branch}, fd) - stage = self.dvc.add('metrics.json') + stages = self.dvc.add('metrics.json') + self.assertEqual(len(stages), 1) + stage = stages[0] self.assertNotEqual(stage, None) self.dvc.scm.add(['.gitignore', 'metrics.json.dvc']) diff --git a/tests/test_remove.py b/tests/test_remove.py index cd21f82b9..49b44c7df 100644 --- a/tests/test_remove.py +++ b/tests/test_remove.py @@ -9,7 +9,10 @@ from tests.basic_env import TestDvc class TestRemove(TestDvc): def test(self): - stage = self.dvc.add(self.FOO) + stages = self.dvc.add(self.FOO) + self.assertEqual(len(stages), 1) + stage = stages[0] + self.assertTrue(stage is not None) stage_removed = self.dvc.remove(stage.path, outs_only=True) self.assertIsInstance(stage_removed, Stage) @@ -31,7 +34,10 @@ class TestRemoveNonExistentFile(TestDvc): class TestRemoveDirectory(TestDvc): def test(self): - stage_add = self.dvc.add(self.DATA_DIR) + stages = self.dvc.add(self.DATA_DIR) + self.assertEqual(len(stages), 1) + stage_add = stages[0] + self.assertTrue(stage_add is not None) stage_removed = self.dvc.remove(stage_add.path) self.assertEqual(stage_add.path, stage_removed.path) self.assertFalse(os.path.exists(self.DATA_DIR)) @@ -40,7 +46,10 @@ class TestRemoveDirectory(TestDvc): class TestCmdRemove(TestDvc): def test(self): - stage = self.dvc.add(self.FOO) + stages = self.dvc.add(self.FOO) + self.assertEqual(len(stages), 1) + stage = stages[0] + self.assertTrue(stage is not None) ret = main(['remove', stage.path]) self.assertEqual(ret, 0) diff --git a/tests/test_repro.py b/tests/test_repro.py index 95627272c..ad5147cea 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -30,7 +30,10 @@ class TestRepro(TestDvc): def setUp(self): super(TestRepro, self).setUp() - self.foo_stage = self.dvc.add(self.FOO) + stages = self.dvc.add(self.FOO) + self.assertEqual(len(stages), 1) + self.foo_stage = stages[0] + self.assertTrue(self.foo_stage is not None) self.file1 = 'file1' self.file1_stage = self.file1 + '.dvc' @@ -50,7 +53,10 @@ class TestReproFail(TestRepro): class TestReproDepUnderDir(TestDvc): def test(self): - self.dir_stage = self.dvc.add(self.DATA_DIR) + stages = self.dvc.add(self.DATA_DIR) + self.assertEqual(len(stages), 1) + self.dir_stage = stages[0] + self.assertTrue(self.dir_stage is not None) sleep()