From 2e21c39c00964310d267d1dd964af2c015b3d763 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 8 Mar 2018 14:03:49 +0200 Subject: [PATCH 1/2] cli: move push/pull from under 'dvc cache' Fixes #528 Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 19 +++++-------------- tests/test_cli.py | 6 +++--- tests/test_data_cloud.py | 12 ++++++------ 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/dvc/cli.py b/dvc/cli.py index f275f788b..c6c4b0d94 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -138,38 +138,29 @@ def parse_args(argv=None): default=None, help='DVC files.') - # Cache - cache_parser = subparsers.add_parser( - 'cache', - parents=[parent_parser], - help='Cache operations') - cache_subparsers = cache_parser.add_subparsers( - dest='cmd', - help='Use `dvc cache CMD` --help for command-specific help') - # Pull - pull_parser = cache_subparsers.add_parser( + pull_parser = subparsers.add_parser( 'pull', parents=[parent_cache_parser], help='Pull data files from the cloud') pull_parser.set_defaults(func=CmdDataPull) # Push - push_parser = cache_subparsers.add_parser( + push_parser = subparsers.add_parser( 'push', parents=[parent_cache_parser], help='Push data files to the cloud') push_parser.set_defaults(func=CmdDataPush) - #Fetch - fetch_parser = cache_subparsers.add_parser( + # Fetch + fetch_parser = subparsers.add_parser( 'fetch', parents=[parent_cache_parser], help='Fetch data files from the cloud') fetch_parser.set_defaults(func=CmdDataFetch) # Status - status_parser = cache_subparsers.add_parser( + status_parser = subparsers.add_parser( 'status', parents=[parent_cache_parser], help='Show mismatches between local cache and cloud cache') diff --git a/tests/test_cli.py b/tests/test_cli.py index d4500d110..8f9a8cf52 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -62,19 +62,19 @@ class TestRun(TestDvc): class TestPull(TestDvc): def test(self): - args = parse_args(['cache', 'pull']) + args = parse_args(['pull']) self.assertIsInstance(args.func(args), CmdDataPull) class TestPush(TestDvc): def test(self): - args = parse_args(['cache', 'push']) + args = parse_args(['push']) self.assertIsInstance(args.func(args), CmdDataPush) class TestStatus(TestDvc): def test(self): - args = parse_args(['cache', 'status']) + args = parse_args(['status']) self.assertIsInstance(args.func(args), CmdDataStatus) diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 9ce221615..9b5826d9f 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -204,26 +204,26 @@ class TestDataCloudLocalCli(TestDvc): cache_dir = stage_dir.outs[0].cache #FIXME check status output - self.main(['cache', 'status']) + self.main(['status']) - self.main(['cache', 'push']) + self.main(['push']) self.assertTrue(os.path.exists(cache)) self.assertTrue(os.path.isfile(cache)) self.assertTrue(os.path.isfile(cache_dir)) - self.main(['cache', 'status']) + self.main(['status']) os.chmod(cache, 0o777) os.unlink(cache) self.dvc._remove_cache(cache_dir) - self.main(['cache', 'status']) + self.main(['status']) - self.main(['cache', 'pull']) + self.main(['pull']) self.assertTrue(os.path.exists(cache)) self.assertTrue(os.path.isfile(cache)) with open(cache, 'r') as fd: self.assertEqual(fd.read(), self.FOO_CONTENTS) self.assertTrue(os.path.isfile(cache_dir)) - self.main(['cache', 'status']) + self.main(['status']) From 54a6532360ea51edadf6f6123c6bd4cc1adc478e Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 8 Mar 2018 19:45:58 +0200 Subject: [PATCH 2/2] cache: create cache dirs only when needed Signed-off-by: Ruslan Kuprieiev --- dvc/cache.py | 18 ++++++------------ dvc/output.py | 14 +++++++++----- tests/basic_env.py | 4 ++++ tests/test_gc.py | 1 - 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/dvc/cache.py b/dvc/cache.py index 9a3f13cf3..be66b7a23 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -11,29 +11,23 @@ class Cache(object): if not os.path.exists(self.cache_dir): os.mkdir(self.cache_dir) - for subdir in self._cache_subdirs: - if not os.path.exists(subdir): - os.mkdir(subdir) - @staticmethod def init(dvc_dir): cache_dir = os.path.join(dvc_dir, Cache.CACHE_DIR) os.mkdir(cache_dir) return Cache(dvc_dir) - @property - def _cache_subdirs(self): - dirs = [] - for x in range(0, 256): - dirs.append(os.path.join(self.cache_dir, format(x, '02x'))) - return dirs - def all(self): clist = [] - for subdir in self._cache_subdirs: + for entry in os.listdir(self.cache_dir): + subdir = os.path.join(self.cache_dir, entry) + if not os.path.isdir(subdir): + continue + for cache in os.listdir(subdir): path = os.path.join(subdir, cache) clist.append(path) + return clist def get(self, md5): diff --git a/dvc/output.py b/dvc/output.py index 815c28304..4ebdb6948 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -232,6 +232,11 @@ class Output(Dependency): def hardlink(self, src, link): self.project.logger.debug(u'creating hardlink {} -> {}'.format(src, link)) + + dname = os.path.dirname(link) + if not os.path.exists(dname): + os.makedirs(dname) + System.hardlink(src, link) os.chmod(src, stat.S_IREAD) @@ -304,11 +309,6 @@ class Output(Dependency): for relpath, cache in self.dir_cache().items(): path = os.path.join(self.path, relpath) - dname = os.path.dirname(path) - - if not os.path.exists(dname): - os.makedirs(dname) - self.hardlink(cache, path) def _collect_dir(self): @@ -330,6 +330,7 @@ class Output(Dependency): return dir_info def _save_dir(self): + dname = os.path.dirname(self.cache) dir_info = self._collect_dir() for entry in dir_info: @@ -344,6 +345,9 @@ class Output(Dependency): else: self.hardlink(path, cache) + if not os.path.isdir(dname): + os.makedirs(dname) + with open(self.cache, 'w+') as fd: yaml.safe_dump(dir_info, fd, default_flow_style=False) diff --git a/tests/basic_env.py b/tests/basic_env.py index 68ed97a89..2445f5afd 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -31,6 +31,10 @@ class TestDir(TestCase): self._saved_dir = None def create(self, name, contents): + dname = os.path.dirname(name) + if len(dname) > 0 and not os.path.isdir(dname): + os.makedirs(dname) + with open(name, 'a') as f: f.write(contents) diff --git a/tests/test_gc.py b/tests/test_gc.py index 82fdcdb32..e9de7ad65 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -28,7 +28,6 @@ class TestGC(TestDvc): self.bad_cache.append(path) path = os.path.join(self.dvc.cache.cache_dir, '45', '6', 'data') - os.mkdir(os.path.dirname(path)) self.create(path, 'md5: "123"') self.bad_cache.append(path)