From d80b7da3efacf1b695e217c8de697eea2fefe3da Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 3 Mar 2018 04:09:48 +0200 Subject: [PATCH 1/2] cloud: makedirs: catch exception instead of checking in advance Signed-off-by: Ruslan Kuprieiev --- dvc/cloud/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dvc/cloud/base.py b/dvc/cloud/base.py index 2bbb2beed..7ea3400e5 100644 --- a/dvc/cloud/base.py +++ b/dvc/cloud/base.py @@ -125,8 +125,11 @@ class DataCloudBase(object): def _makedirs(self, fname): dname = os.path.dirname(fname) - if not os.path.exists(dname): + try: os.makedirs(dname) + except OSError as e: + if e.errno != os.errno.EEXIST: + raise def _pull_key(self, key, path): """ Cloud-specific method of pulling keys """ From f814269b4c38c40ba46fc65c1f6623e0c2138668 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 3 Mar 2018 04:12:09 +0200 Subject: [PATCH 2/2] dvc: properly collect used cache Signed-off-by: Ruslan Kuprieiev --- dvc/project.py | 20 ++++++++++++++------ tests/test_gc.py | 9 +++++++-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/dvc/project.py b/dvc/project.py index 4674accdb..99363beeb 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -159,14 +159,22 @@ class Project(object): for stage in self.stages(): stage.checkout() - def _used_cache(self): + def _used_cache(self, collect_dir=False): clist = [] for stage in self.stages(): for out in stage.outs: if not out.use_cache: continue - if out.cache not in clist: + if not os.path.isdir(out.cache) and \ + out.cache not in clist: clist.append(out.cache) + if os.path.isdir(out.cache): + dir_cache = out.dir_cache() + clist.extend(dir_cache.values()) + if collect_dir: + clist.extend([os.path.join(out.cache, f) for f in dir_cache.keys()]) + else: + clist.append(out.cache) return clist def _remove_cache_file(self, cache): @@ -188,7 +196,7 @@ class Project(object): os.rmdir(cache) def gc(self): - clist = self._used_cache() + clist = self._used_cache(collect_dir=False) for cache in self.cache.all(): if cache in clist: continue @@ -196,14 +204,14 @@ class Project(object): self.logger.info(u'\'{}\' was removed'.format(self.to_dvc_path(cache))) def push(self, jobs=1): - self.cloud.push(self._used_cache(), jobs) + self.cloud.push(self._used_cache(collect_dir=True), jobs) def pull(self, jobs=1): - self.cloud.pull(self._used_cache(), jobs) + self.cloud.pull(self._used_cache(collect_dir=False), jobs) self.checkout() def status(self, jobs=1): - return self.cloud.status(self._used_cache(), jobs) + return self.cloud.status(self._used_cache(collect_dir=True), jobs) def graph(self): G = nx.DiGraph() diff --git a/tests/test_gc.py b/tests/test_gc.py index 253d67c23..82fdcdb32 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -17,7 +17,8 @@ class TestGC(TestDvc): def setUp(self): super(TestGC, self).setUp() - stage = self.dvc.add(self.FOO) + self.dvc.add(self.FOO) + self.dvc.add(self.DATA_DIR) self.good_cache = self.dvc.cache.all() self.bad_cache = [] @@ -26,6 +27,11 @@ class TestGC(TestDvc): self.create(path, i) self.bad_cache.append(path) + path = os.path.join(self.dvc.cache.cache_dir, '45', '6', 'data') + os.mkdir(os.path.dirname(path)) + self.create(path, 'md5: "123"') + self.bad_cache.append(path) + def test_api(self): self.dvc.gc() self._test_gc() @@ -41,4 +47,3 @@ class TestGC(TestDvc): for c in self.good_cache: self.assertTrue(os.path.exists(c)) - self.assertTrue(os.path.isfile(c))