Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-10 15:28:34 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1027 from efiop/master

Browse Source 
dvc: add --recursive
This commit is contained in:
Ruslan Kuprieiev
2018-08-21 10:51:07 +03:00
committed by GitHub
parent ee0dcc384d c6f642fd6b
commit ff5152ee5f
9 changed files with 131 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cli.py
+6
View File
@@ -139,6 +139,12 @@ def parse_args(argv=None):
parents=[parent_parser],
description=ADD_HELP,
help=ADD_HELP)
add_parser.add_argument(
'-R',
'--recursive',
action='store_true',
default=False,
help='Recursively add each file under the directory.')
add_parser.add_argument(
'targets',
nargs='+',
dvc/command/add.py
+1 -1
View File
@@ -7,7 +7,7 @@ class CmdAdd(CmdBase):
def run(self):
for target in self.args.targets:
try:
self.project.add(target)
self.project.add(target, recursive=self.args.recursive)
except DvcException as ex:
Logger.error('Failed to add \'{}\''.format(target), ex)
return 1
dvc/project.py
+28 -8
View File
@@ -132,16 +132,36 @@ class Project(object):
stages = [o.stage.relpath, out.stage.relpath]
raise OutputDuplicationError(o.path, stages)
def add(self, fname):
stage = Stage.loads(project=self,
outs=[fname],
add=True)
def add(self, fname, recursive=False):
fnames = []
if recursive and os.path.isdir(fname):
fnames = []
for root, dirs, files in os.walk(fname):
for f in files:
path = os.path.join(root, f)
if Stage.is_stage_file(path):
continue
if os.path.basename(path) == self.scm.ignore_file():
continue
if self.scm.is_tracked(path):
continue
fnames.append(path)
else:
fnames = [fname]
self._check_output_duplication(stage.outs)
stages = []
for f in fnames:
stage = Stage.loads(project=self,
outs=[f],
add=True)
stage.save()
stage.dump()
return stage
self._check_output_duplication(stage.outs)
stage.save()
stage.dump()
stages.append(stage)
return stages
def remove(self, target, outs_only=False):
stage = Stage.load(self, target)
tests/test_add.py
+44 -10
View File
@@ -19,7 +19,10 @@ class TestAdd(TestDvc):
def test(self):
md5 = file_md5(self.FOO)[0]
stage = self.dvc.add(self.FOO)
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
self.assertIsInstance(stage, Stage)
self.assertTrue(os.path.isfile(stage.path))
@@ -40,8 +43,10 @@ class TestAddDirectory(TestDvc):
dname = 'directory'
os.mkdir(dname)
self.create(os.path.join(dname, 'file'), 'file')
stage = self.dvc.add(dname)
self.assertNotEqual(stage, None)
stages = self.dvc.add(dname)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
self.assertEqual(len(stage.deps), 0)
self.assertEqual(len(stage.outs), 1)
@@ -52,12 +57,29 @@ class TestAddDirectory(TestDvc):
self.assertTrue('\\' not in info['relpath'])
class TestAddDirectoryRecursive(TestDvc):
def test(self):
stages = self.dvc.add(self.DATA_DIR, recursive=True)
self.assertEqual(len(stages), 2)
class TestAddCmdDirectoryRecursive(TestDvc):
def test(self):
ret = main(['add',
'--recursive',
self.DATA_DIR])
self.assertEqual(ret, 0)
class TestAddDirectoryWithForwardSlash(TestDvc):
def test(self):
dname = 'directory/'
os.mkdir(dname)
self.create(os.path.join(dname, 'file'), 'file')
stage = self.dvc.add(dname)
stages = self.dvc.add(dname)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
self.assertEquals(os.path.abspath('directory.dvc'), stage.path)
class TestAddTrackedFile(TestDvc):
@@ -78,20 +100,30 @@ class TestAddDirWithExistingCache(TestDvc):
os.mkdir(dname)
shutil.copyfile(self.FOO, fname)
self.dvc.add(self.FOO)
self.dvc.add(dname)
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
self.assertTrue(stages[0] is not None)
stages = self.dvc.add(dname)
self.assertEqual(len(stages), 1)
self.assertTrue(stages[0] is not None)
class TestAddModifiedDir(TestDvc):
def test(self):
self.dvc.add(self.DATA_DIR)
stages = self.dvc.add(self.DATA_DIR)
self.assertEqual(len(stages), 1)
self.assertTrue(stages[0] is not None)
os.unlink(self.DATA)
self.dvc.add(self.DATA_DIR)
stages = self.dvc.add(self.DATA_DIR)
self.assertEqual(len(stages), 1)
self.assertTrue(stages[0] is not None)
class TestAddFileInDir(TestDvc):
def test(self):
stage = self.dvc.add(self.DATA_SUB)
stages = self.dvc.add(self.DATA_SUB)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertNotEqual(stage, None)
self.assertEqual(len(stage.deps), 0)
self.assertEqual(len(stage.outs), 1)
@@ -104,7 +136,9 @@ class TestAddExternalLocalFile(TestDvc):
fname = os.path.join(dname, 'foo')
shutil.copyfile(self.FOO, fname)
stage = self.dvc.add(fname)
stages = self.dvc.add(fname)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertNotEqual(stage, None)
self.assertEqual(len(stage.deps), 0)
self.assertEqual(len(stage.outs), 1)
tests/test_checkout.py
+12 -3
View File
@@ -16,7 +16,11 @@ class TestCheckout(TestRepro):
def setUp(self):
super(TestCheckout, self).setUp()
self.data_dir_stage = self.dvc.add(self.DATA_DIR)
stages = self.dvc.add(self.DATA_DIR)
self.assertEqual(len(stages), 1)
self.data_dir_stage = stages[0]
self.assertTrue(self.data_dir_stage is not None)
self.orig = 'orig'
shutil.copy(self.FOO, self.orig)
@@ -74,7 +78,9 @@ class CheckoutBase(TestDvc):
def commit_data_file(self, fname, content='random text'):
with open(fname, 'w') as fd:
fd.write(content)
self.dvc.add(fname)
stages = self.dvc.add(fname)
self.assertEqual(len(stages), 1)
self.assertTrue(stages[0] is not None)
self.dvc.scm.add([fname + '.dvc', '.gitignore'])
self.dvc.scm.commit('adding ' + fname)
@@ -181,7 +187,10 @@ class TestCheckoutEmptyDir(TestDvc):
dname = 'empty_dir'
os.mkdir(dname)
stage = self.dvc.add(dname)
stages = self.dvc.add(dname)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
self.assertEqual(len(stage.outs), 1)
stage.outs[0].remove()
tests/test_data_cloud.py
+17 -4
View File
@@ -204,12 +204,19 @@ class TestDataCloudBase(TestDvc):
def _test_cloud(self):
self._setup_cloud()
stage = self.dvc.add(self.FOO)
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
cache = stage.outs[0].cache
info = stage.outs[0].dumpd()
md5 = info['md5']
stage_dir = self.dvc.add(self.DATA_DIR)
stages = self.dvc.add(self.DATA_DIR)
self.assertEqual(len(stages), 1)
stage_dir = stages[0]
self.assertTrue(stage_dir is not None)
cache_dir = stage_dir.outs[0].cache
info_dir = stage_dir.outs[0].dumpd()
md5_dir = info_dir['md5']
@@ -358,10 +365,16 @@ class TestDataCloudCLIBase(TestDvc):
else:
args += []
stage = self.dvc.add(self.FOO)
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
cache = stage.outs[0].cache
stage_dir = self.dvc.add(self.DATA_DIR)
stages = self.dvc.add(self.DATA_DIR)
self.assertEqual(len(stages), 1)
stage_dir = stages[0]
self.assertTrue(stage_dir is not None)
cache_dir = stage_dir.outs[0].cache
# FIXME check status output
tests/test_metrics.py
+3 -1
View File
@@ -219,7 +219,9 @@ class TestCachedMetrics(TestDvc):
with open('metrics.json', 'w+') as fd:
json.dump({'metrics': branch}, fd)
stage = self.dvc.add('metrics.json')
stages = self.dvc.add('metrics.json')
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertNotEqual(stage, None)
self.dvc.scm.add(['.gitignore', 'metrics.json.dvc'])
tests/test_remove.py
+12 -3
View File
@@ -9,7 +9,10 @@ from tests.basic_env import TestDvc
class TestRemove(TestDvc):
def test(self):
stage = self.dvc.add(self.FOO)
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
stage_removed = self.dvc.remove(stage.path, outs_only=True)
self.assertIsInstance(stage_removed, Stage)
@@ -31,7 +34,10 @@ class TestRemoveNonExistentFile(TestDvc):
class TestRemoveDirectory(TestDvc):
def test(self):
stage_add = self.dvc.add(self.DATA_DIR)
stages = self.dvc.add(self.DATA_DIR)
self.assertEqual(len(stages), 1)
stage_add = stages[0]
self.assertTrue(stage_add is not None)
stage_removed = self.dvc.remove(stage_add.path)
self.assertEqual(stage_add.path, stage_removed.path)
self.assertFalse(os.path.exists(self.DATA_DIR))
@@ -40,7 +46,10 @@ class TestRemoveDirectory(TestDvc):
class TestCmdRemove(TestDvc):
def test(self):
stage = self.dvc.add(self.FOO)
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
ret = main(['remove',
stage.path])
self.assertEqual(ret, 0)
tests/test_repro.py
+8 -2
View File
@@ -30,7 +30,10 @@ class TestRepro(TestDvc):
def setUp(self):
super(TestRepro, self).setUp()
self.foo_stage = self.dvc.add(self.FOO)
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
self.foo_stage = stages[0]
self.assertTrue(self.foo_stage is not None)
self.file1 = 'file1'
self.file1_stage = self.file1 + '.dvc'
@@ -50,7 +53,10 @@ class TestReproFail(TestRepro):
class TestReproDepUnderDir(TestDvc):
def test(self):
self.dir_stage = self.dvc.add(self.DATA_DIR)
stages = self.dvc.add(self.DATA_DIR)
self.assertEqual(len(stages), 1)
self.dir_stage = stages[0]
self.assertTrue(self.dir_stage is not None)
sleep()