From add9498e82d9030996b9b4ab74e3a2cd5223ae69 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 28 Jun 2018 18:05:55 +0300 Subject: [PATCH 1/2] gc: move gc logic to Remote Signed-off-by: Ruslan Kuprieiev --- dvc/project.py | 57 ++++++++++++++++++++++----------------------- dvc/remote/local.py | 10 +++++++- tests/test_cache.py | 8 +++---- tests/test_gc.py | 2 +- 4 files changed, 42 insertions(+), 35 deletions(-) diff --git a/dvc/project.py b/dvc/project.py index 3ffef039b..08aaa3be6 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -253,45 +253,44 @@ class Project(object): for stage in stages: stage.checkout() - def _used_cache(self, target=None): - cache_set = set() + def _used_cache(self, target=None, all_branches=False): + cache = [] - if target: - stages = [Stage.load(self, target)] - else: - stages = self.stages() + for branch in self.scm.brancher(all_branches=all_branches): + if target: + stages = [Stage.load(self, target)] + else: + stages = self.stages() - for stage in stages: - for out in stage.outs: - if out.path_info['scheme'] != 'local': - continue + for stage in stages: + for out in stage.outs: + if out.path_info['scheme'] != 'local': + continue - if not out.use_cache or not out.cache: - continue + if not out.use_cache or not out.info: + continue - cache_set |= set([out.cache]) - if self.cache.local.is_dir_cache(out.cache) and os.path.isfile(out.cache): - dir_cache = self.cache.local.dir_cache(out.cache) - cache_set |= set(dir_cache.values()) + cache.append(out.info) - return list(cache_set) + if self.cache.local.is_dir_cache(out.cache) and os.path.isfile(out.cache): + dir_cache = self.cache.local.load_dir_cache(out.cache) + cache += dir_cache + + return cache + + def _used_cache_files(self, target=None, all_branches=False): + infos = self._used_cache(target=target, all_branches=all_branches) + return [self.cache.local.get(info[self.cache.local.PARAM_MD5]) for info in infos] def gc(self, all_branches=False): - for branch in self.scm.brancher(all_branches=all_branches): - clist = self._used_cache() - for cache in self.cache.local.all(): - if cache in clist: - continue - os.unlink(cache) - self.logger.info(u'\'{}\' was removed'.format(self.to_dvc_path(cache))) + clist = self._used_cache(target=None, all_branches=all_branches) + self.cache.local.gc(clist) def push(self, target=None, jobs=1, remote=None, all_branches=False): - for branch in self.scm.brancher(all_branches=all_branches): - self.cloud.push(self._used_cache(target), jobs, remote=remote) + self.cloud.push(self._used_cache_files(target, all_branches), jobs, remote=remote) def fetch(self, target=None, jobs=1, remote=None, all_branches=False): - for branch in self.scm.brancher(all_branches=all_branches): - self.cloud.pull(self._used_cache(target), jobs, remote=remote) + self.cloud.pull(self._used_cache_files(target, all_branches), jobs, remote=remote) def pull(self, target=None, jobs=1, remote=None, all_branches=False): self.fetch(target, jobs, remote=remote, all_branches=all_branches) @@ -314,7 +313,7 @@ class Project(object): import dvc.remote.base as cloud status = {} - for target, ret in self.cloud.status(self._used_cache(target), jobs, remote=remote): + for target, ret in self.cloud.status(self._used_cache_files(target), jobs, remote=remote): if ret == cloud.STATUS_UNKNOWN or ret == cloud.STATUS_OK: continue diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 473e6691f..66721812e 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -58,7 +58,7 @@ class RemoteLOCAL(RemoteBase): for cache in os.listdir(subdir): path = os.path.join(subdir, cache) - clist.append(path) + clist.append(self.path_to_md5(path)) return clist @@ -317,3 +317,11 @@ class RemoteLOCAL(RemoteBase): os.rename(tmp_file, path) return path + + def gc(self, checksum_infos): + used_md5s = [info[self.PARAM_MD5] for info in checksum_infos] + + for md5 in self.all(): + if md5 in used_md5s: + continue + remove(self.get(md5)) diff --git a/tests/test_cache.py b/tests/test_cache.py index e5b92291f..37963f2ce 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -25,10 +25,10 @@ class TestCache(TestDvc): self.create(self.cache2, '2') def test_all(self): - flist = Cache(self.dvc).local.all() - self.assertEquals(len(flist), 2) - self.assertTrue(self.cache1 in flist) - self.assertTrue(self.cache2 in flist) + md5_list = Cache(self.dvc).local.all() + self.assertEquals(len(md5_list), 2) + self.assertTrue(self.cache1_md5 in md5_list) + self.assertTrue(self.cache2_md5 in md5_list) def test_get(self): cache = Cache(self.dvc).local.get(self.cache1_md5) diff --git a/tests/test_gc.py b/tests/test_gc.py index 6af35b1ba..b2276ea5e 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -16,7 +16,7 @@ class TestGC(TestDvc): self.dvc.add(self.FOO) self.dvc.add(self.DATA_DIR) - self.good_cache = self.dvc.cache.local.all() + self.good_cache = [self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all()] self.bad_cache = [] for i in ['123', '234', '345']: From 50578a5923874111505e3a1bd45eb934c2ba32a0 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 28 Jun 2018 21:32:41 +0300 Subject: [PATCH 2/2] gc: support s3/gs/ssh/hdfs Fixes #738 Signed-off-by: Ruslan Kuprieiev --- dvc/log.log | 296 ++++++++++++++++++++++++++++++++++++++++++++ dvc/project.py | 33 +++-- dvc/remote/gs.py | 22 ++++ dvc/remote/hdfs.py | 28 +++++ dvc/remote/s3.py | 22 ++++ dvc/remote/ssh.py | 44 ++++++- tests/test_repro.py | 2 + 7 files changed, 437 insertions(+), 10 deletions(-) create mode 100644 dvc/log.log diff --git a/dvc/log.log b/dvc/log.log new file mode 100644 index 000000000..83c7183be --- /dev/null +++ b/dvc/log.log @@ -0,0 +1,296 @@ +drwxr-xr-x - root supergroup 0 2018-06-28 19:27 hdfs://root@172.17.0.2/0318d092-5c34-460b-b50c-2310f69e5210 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:27 hdfs://root@172.17.0.2/0318d092-5c34-460b-b50c-2310f69e5210/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:26 hdfs://root@172.17.0.2/0318d092-5c34-460b-b50c-2310f69e5210/foo +drwxr-xr-x - root supergroup 0 2018-06-28 19:46 hdfs://root@172.17.0.2/054f45bb-8b25-4a43-968a-39f68d85b295 +drwxr-xr-x - root supergroup 0 2018-06-28 19:47 hdfs://root@172.17.0.2/054f45bb-8b25-4a43-968a-39f68d85b295/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:46 hdfs://root@172.17.0.2/054f45bb-8b25-4a43-968a-39f68d85b295/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:47 hdfs://root@172.17.0.2/054f45bb-8b25-4a43-968a-39f68d85b295/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:36 hdfs://root@172.17.0.2/0e2fdefa-819c-44d5-9aef-cac883c33256 +drwxr-xr-x - root supergroup 0 2018-06-28 19:36 hdfs://root@172.17.0.2/0e2fdefa-819c-44d5-9aef-cac883c33256/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:36 hdfs://root@172.17.0.2/0e2fdefa-819c-44d5-9aef-cac883c33256/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:36 hdfs://root@172.17.0.2/0e2fdefa-819c-44d5-9aef-cac883c33256/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:40 hdfs://root@172.17.0.2/139c95a1-2663-4bf8-8565-7dbc35afe934 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:40 hdfs://root@172.17.0.2/139c95a1-2663-4bf8-8565-7dbc35afe934/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:40 hdfs://root@172.17.0.2/139c95a1-2663-4bf8-8565-7dbc35afe934/foo +drwxr-xr-x - root supergroup 0 2018-06-27 21:25 hdfs://root@172.17.0.2/1a63abc1-850b-4e60-a7ae-d581061e0a33 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:25 hdfs://root@172.17.0.2/1a63abc1-850b-4e60-a7ae-d581061e0a33/bar +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:25 hdfs://root@172.17.0.2/1a63abc1-850b-4e60-a7ae-d581061e0a33/foo +drwxr-xr-x - root supergroup 0 2018-06-28 19:55 hdfs://root@172.17.0.2/1a83a18d-62d8-4359-ab08-e53fa571593a +drwxr-xr-x - root supergroup 0 2018-06-28 19:55 hdfs://root@172.17.0.2/1a83a18d-62d8-4359-ab08-e53fa571593a/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:55 hdfs://root@172.17.0.2/1a83a18d-62d8-4359-ab08-e53fa571593a/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:55 hdfs://root@172.17.0.2/1a83a18d-62d8-4359-ab08-e53fa571593a/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 20:54 hdfs://root@172.17.0.2/26d990b9-2df0-4e28-be9f-41e62483e97c +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:54 hdfs://root@172.17.0.2/26d990b9-2df0-4e28-be9f-41e62483e97c/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:54 hdfs://root@172.17.0.2/26d990b9-2df0-4e28-be9f-41e62483e97c/foo +drwxr-xr-x - root supergroup 0 2018-06-27 21:21 hdfs://root@172.17.0.2/288aca9b-086e-4b1b-995d-0c95333d76ec +drwxr-xr-x - root supergroup 0 2018-06-27 21:22 hdfs://root@172.17.0.2/288aca9b-086e-4b1b-995d-0c95333d76ec/00 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:21 hdfs://root@172.17.0.2/288aca9b-086e-4b1b-995d-0c95333d76ec/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:22 hdfs://root@172.17.0.2/288aca9b-086e-4b1b-995d-0c95333d76ec/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:44 hdfs://root@172.17.0.2/2a13bae3-1c60-47c9-836f-ee2c3beb95b1 +drwxr-xr-x - root supergroup 0 2018-06-28 19:44 hdfs://root@172.17.0.2/2a13bae3-1c60-47c9-836f-ee2c3beb95b1/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:44 hdfs://root@172.17.0.2/2a13bae3-1c60-47c9-836f-ee2c3beb95b1/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:44 hdfs://root@172.17.0.2/2a13bae3-1c60-47c9-836f-ee2c3beb95b1/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:22 hdfs://root@172.17.0.2/32773fd1-8655-4ba2-b921-16abf5347964 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:22 hdfs://root@172.17.0.2/32773fd1-8655-4ba2-b921-16abf5347964/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:21 hdfs://root@172.17.0.2/32773fd1-8655-4ba2-b921-16abf5347964/foo +drwxr-xr-x - root supergroup 0 2018-06-28 20:43 hdfs://root@172.17.0.2/352567a7-eea7-4da5-b817-88a5c6aa127f +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:43 hdfs://root@172.17.0.2/352567a7-eea7-4da5-b817-88a5c6aa127f/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:43 hdfs://root@172.17.0.2/352567a7-eea7-4da5-b817-88a5c6aa127f/foo +drwxr-xr-x - root supergroup 0 2018-06-28 20:48 hdfs://root@172.17.0.2/3897c510-1fcc-467c-8726-b02e90b8b5f7 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:48 hdfs://root@172.17.0.2/3897c510-1fcc-467c-8726-b02e90b8b5f7/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:48 hdfs://root@172.17.0.2/3897c510-1fcc-467c-8726-b02e90b8b5f7/foo +drwxr-xr-x - root supergroup 0 2018-06-28 21:03 hdfs://root@172.17.0.2/3da96e91-6ab0-40e7-927a-3e4a6c41eebe +-rw-r--r-- 1 root supergroup 3 2018-06-28 21:03 hdfs://root@172.17.0.2/3da96e91-6ab0-40e7-927a-3e4a6c41eebe/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 21:02 hdfs://root@172.17.0.2/3da96e91-6ab0-40e7-927a-3e4a6c41eebe/foo +drwxr-xr-x - root supergroup 0 2018-06-28 16:55 hdfs://root@172.17.0.2/48047054-1b39-4ba0-828f-ca3340d0f4c9 +-rw-r--r-- 1 root supergroup 3 2018-06-28 16:55 hdfs://root@172.17.0.2/48047054-1b39-4ba0-828f-ca3340d0f4c9/foo +drwxr-xr-x - root supergroup 0 2018-06-28 20:00 hdfs://root@172.17.0.2/487ce305-ced1-4ef7-bde2-59925d9f391b +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:00 hdfs://root@172.17.0.2/487ce305-ced1-4ef7-bde2-59925d9f391b/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:00 hdfs://root@172.17.0.2/487ce305-ced1-4ef7-bde2-59925d9f391b/foo +drwxr-xr-x - root supergroup 0 2018-06-27 21:29 hdfs://root@172.17.0.2/4aca4651-0f84-4b0d-a1c9-e718ce96637a +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:29 hdfs://root@172.17.0.2/4aca4651-0f84-4b0d-a1c9-e718ce96637a/bar +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:29 hdfs://root@172.17.0.2/4aca4651-0f84-4b0d-a1c9-e718ce96637a/foo +drwxr-xr-x - root supergroup 0 2018-06-27 21:21 hdfs://root@172.17.0.2/4b0a0875-e425-4628-a42d-69b7b9c8b252 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:21 hdfs://root@172.17.0.2/4b0a0875-e425-4628-a42d-69b7b9c8b252/bar +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:21 hdfs://root@172.17.0.2/4b0a0875-e425-4628-a42d-69b7b9c8b252/foo +drwxr-xr-x - root supergroup 0 2018-06-28 17:38 hdfs://root@172.17.0.2/4c05e7e1-a7e5-4ea6-83a1-1a73e56fd629 +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:38 hdfs://root@172.17.0.2/4c05e7e1-a7e5-4ea6-83a1-1a73e56fd629/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:38 hdfs://root@172.17.0.2/4c05e7e1-a7e5-4ea6-83a1-1a73e56fd629/foo +drwxr-xr-x - root supergroup 0 2018-06-28 19:21 hdfs://root@172.17.0.2/4e6b437e-2b7f-4008-a01c-83b30df2d07e +drwxr-xr-x - root supergroup 0 2018-06-28 19:22 hdfs://root@172.17.0.2/4e6b437e-2b7f-4008-a01c-83b30df2d07e/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:21 hdfs://root@172.17.0.2/4e6b437e-2b7f-4008-a01c-83b30df2d07e/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:22 hdfs://root@172.17.0.2/4e6b437e-2b7f-4008-a01c-83b30df2d07e/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 17:49 hdfs://root@172.17.0.2/4fa2d2f5-9371-42e3-8261-acc144c4fc6b +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:49 hdfs://root@172.17.0.2/4fa2d2f5-9371-42e3-8261-acc144c4fc6b/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:48 hdfs://root@172.17.0.2/4fa2d2f5-9371-42e3-8261-acc144c4fc6b/foo +drwxr-xr-x - root supergroup 0 2018-06-28 19:26 hdfs://root@172.17.0.2/500476f0-2ca8-4534-98d5-0a240df85f54 +drwxr-xr-x - root supergroup 0 2018-06-28 19:27 hdfs://root@172.17.0.2/500476f0-2ca8-4534-98d5-0a240df85f54/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:26 hdfs://root@172.17.0.2/500476f0-2ca8-4534-98d5-0a240df85f54/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:27 hdfs://root@172.17.0.2/500476f0-2ca8-4534-98d5-0a240df85f54/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:58 hdfs://root@172.17.0.2/50c16b53-dfb4-49da-a546-a509b35cf436 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:58 hdfs://root@172.17.0.2/50c16b53-dfb4-49da-a546-a509b35cf436/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:58 hdfs://root@172.17.0.2/50c16b53-dfb4-49da-a546-a509b35cf436/foo +drwxr-xr-x - root supergroup 0 2018-06-28 21:04 hdfs://root@172.17.0.2/516b7677-4453-4ae7-997e-7729751aca03 +-rw-r--r-- 1 root supergroup 3 2018-06-28 21:04 hdfs://root@172.17.0.2/516b7677-4453-4ae7-997e-7729751aca03/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 21:04 hdfs://root@172.17.0.2/516b7677-4453-4ae7-997e-7729751aca03/foo +drwxr-xr-x - root supergroup 0 2018-06-28 16:51 hdfs://root@172.17.0.2/58bfae16-b476-46f6-b2c9-1e836bfafd45 +drwxr-xr-x - root supergroup 0 2018-06-28 16:51 hdfs://root@172.17.0.2/58bfae16-b476-46f6-b2c9-1e836bfafd45/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 16:51 hdfs://root@172.17.0.2/58bfae16-b476-46f6-b2c9-1e836bfafd45/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 16:51 hdfs://root@172.17.0.2/58bfae16-b476-46f6-b2c9-1e836bfafd45/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 17:38 hdfs://root@172.17.0.2/59caf017-c5df-4047-ad31-6c60da6494d7 +drwxr-xr-x - root supergroup 0 2018-06-28 17:38 hdfs://root@172.17.0.2/59caf017-c5df-4047-ad31-6c60da6494d7/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:38 hdfs://root@172.17.0.2/59caf017-c5df-4047-ad31-6c60da6494d7/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:38 hdfs://root@172.17.0.2/59caf017-c5df-4047-ad31-6c60da6494d7/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:44 hdfs://root@172.17.0.2/5c89b716-5913-41ca-a17f-2f0da51c2480 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:44 hdfs://root@172.17.0.2/5c89b716-5913-41ca-a17f-2f0da51c2480/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:44 hdfs://root@172.17.0.2/5c89b716-5913-41ca-a17f-2f0da51c2480/foo +drwxr-xr-x - root supergroup 0 2018-06-28 16:56 hdfs://root@172.17.0.2/60fb05f3-a9ad-4024-9412-ba6e040209c6 +-rw-r--r-- 1 root supergroup 3 2018-06-28 16:56 hdfs://root@172.17.0.2/60fb05f3-a9ad-4024-9412-ba6e040209c6/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 16:56 hdfs://root@172.17.0.2/60fb05f3-a9ad-4024-9412-ba6e040209c6/foo +drwxr-xr-x - root supergroup 0 2018-06-28 19:23 hdfs://root@172.17.0.2/628e4cf0-4fd6-47d6-9812-0986dd73d401 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:23 hdfs://root@172.17.0.2/628e4cf0-4fd6-47d6-9812-0986dd73d401/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:23 hdfs://root@172.17.0.2/628e4cf0-4fd6-47d6-9812-0986dd73d401/foo +drwxr-xr-x - root supergroup 0 2018-06-28 17:08 hdfs://root@172.17.0.2/69631f60-6fd7-4035-904d-917441cef16c +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:08 hdfs://root@172.17.0.2/69631f60-6fd7-4035-904d-917441cef16c/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:08 hdfs://root@172.17.0.2/69631f60-6fd7-4035-904d-917441cef16c/foo +drwxr-xr-x - root supergroup 0 2018-06-28 20:25 hdfs://root@172.17.0.2/6997718f-b0de-46ec-9fec-de0c26f17221 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:25 hdfs://root@172.17.0.2/6997718f-b0de-46ec-9fec-de0c26f17221/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:25 hdfs://root@172.17.0.2/6997718f-b0de-46ec-9fec-de0c26f17221/foo +drwxr-xr-x - root supergroup 0 2018-06-28 18:05 hdfs://root@172.17.0.2/6b8ccaf0-442d-4f6b-92b3-50e82ed4e5e7 +-rw-r--r-- 1 root supergroup 3 2018-06-28 18:05 hdfs://root@172.17.0.2/6b8ccaf0-442d-4f6b-92b3-50e82ed4e5e7/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 18:05 hdfs://root@172.17.0.2/6b8ccaf0-442d-4f6b-92b3-50e82ed4e5e7/foo +drwxr-xr-x - root supergroup 0 2018-06-28 18:04 hdfs://root@172.17.0.2/70f0ea02-fbeb-4998-b3f5-05b55c2b7317 +drwxr-xr-x - root supergroup 0 2018-06-28 18:05 hdfs://root@172.17.0.2/70f0ea02-fbeb-4998-b3f5-05b55c2b7317/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 18:05 hdfs://root@172.17.0.2/70f0ea02-fbeb-4998-b3f5-05b55c2b7317/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 18:05 hdfs://root@172.17.0.2/70f0ea02-fbeb-4998-b3f5-05b55c2b7317/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 16:56 hdfs://root@172.17.0.2/719dbe1b-909e-477c-a2cd-89144c03157b +drwxr-xr-x - root supergroup 0 2018-06-28 16:56 hdfs://root@172.17.0.2/719dbe1b-909e-477c-a2cd-89144c03157b/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 16:56 hdfs://root@172.17.0.2/719dbe1b-909e-477c-a2cd-89144c03157b/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 16:56 hdfs://root@172.17.0.2/719dbe1b-909e-477c-a2cd-89144c03157b/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 20:28 hdfs://root@172.17.0.2/737dbc26-8414-4603-8e75-5adcc7ce13bd +drwxr-xr-x - root supergroup 0 2018-06-28 20:28 hdfs://root@172.17.0.2/737dbc26-8414-4603-8e75-5adcc7ce13bd/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:28 hdfs://root@172.17.0.2/737dbc26-8414-4603-8e75-5adcc7ce13bd/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:28 hdfs://root@172.17.0.2/737dbc26-8414-4603-8e75-5adcc7ce13bd/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-27 21:30 hdfs://root@172.17.0.2/73f6e3cb-8594-4f14-b01b-eaa4e3efd1fc +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:30 hdfs://root@172.17.0.2/73f6e3cb-8594-4f14-b01b-eaa4e3efd1fc/bar +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:30 hdfs://root@172.17.0.2/73f6e3cb-8594-4f14-b01b-eaa4e3efd1fc/foo +drwxr-xr-x - root supergroup 0 2018-06-27 21:30 hdfs://root@172.17.0.2/788ed6c0-19a3-44a4-81f5-61c5ce9c6ec1 +drwxr-xr-x - root supergroup 0 2018-06-27 21:30 hdfs://root@172.17.0.2/788ed6c0-19a3-44a4-81f5-61c5ce9c6ec1/00 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:30 hdfs://root@172.17.0.2/788ed6c0-19a3-44a4-81f5-61c5ce9c6ec1/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:30 hdfs://root@172.17.0.2/788ed6c0-19a3-44a4-81f5-61c5ce9c6ec1/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 20:24 hdfs://root@172.17.0.2/79cffe59-efc2-4a44-93f8-9aa5afa8b556 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:24 hdfs://root@172.17.0.2/79cffe59-efc2-4a44-93f8-9aa5afa8b556/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:24 hdfs://root@172.17.0.2/79cffe59-efc2-4a44-93f8-9aa5afa8b556/foo +drwxr-xr-x - root supergroup 0 2018-06-27 20:53 hdfs://root@172.17.0.2/7f8dce77-e734-4128-8e5b-49a36f2fd4f2 +-rw-r--r-- 1 root supergroup 3 2018-06-27 20:53 hdfs://root@172.17.0.2/7f8dce77-e734-4128-8e5b-49a36f2fd4f2/bar +-rw-r--r-- 1 root supergroup 3 2018-06-27 20:53 hdfs://root@172.17.0.2/7f8dce77-e734-4128-8e5b-49a36f2fd4f2/foo +drwxr-xr-x - root supergroup 0 2018-06-28 19:36 hdfs://root@172.17.0.2/88d1735e-22bc-4c15-9fde-3e2ba63a2874 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:36 hdfs://root@172.17.0.2/88d1735e-22bc-4c15-9fde-3e2ba63a2874/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:36 hdfs://root@172.17.0.2/88d1735e-22bc-4c15-9fde-3e2ba63a2874/foo +drwxr-xr-x - root supergroup 0 2018-06-27 20:53 hdfs://root@172.17.0.2/8afdb1a7-b2c3-4141-b4e2-94dc4a8e587e +drwxr-xr-x - root supergroup 0 2018-06-27 20:53 hdfs://root@172.17.0.2/8afdb1a7-b2c3-4141-b4e2-94dc4a8e587e/00 +-rw-r--r-- 1 root supergroup 3 2018-06-27 20:53 hdfs://root@172.17.0.2/8afdb1a7-b2c3-4141-b4e2-94dc4a8e587e/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-27 20:53 hdfs://root@172.17.0.2/8afdb1a7-b2c3-4141-b4e2-94dc4a8e587e/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:30 hdfs://root@172.17.0.2/8c5687a1-cd77-4588-8764-7e8a4ac48d9c +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:30 hdfs://root@172.17.0.2/8c5687a1-cd77-4588-8764-7e8a4ac48d9c/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:30 hdfs://root@172.17.0.2/8c5687a1-cd77-4588-8764-7e8a4ac48d9c/foo +drwxr-xr-x - root supergroup 0 2018-06-28 20:30 hdfs://root@172.17.0.2/8db7e93a-3c89-442a-9a73-af316e558aee +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:30 hdfs://root@172.17.0.2/8db7e93a-3c89-442a-9a73-af316e558aee/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:29 hdfs://root@172.17.0.2/8db7e93a-3c89-442a-9a73-af316e558aee/foo +drwxr-xr-x - root supergroup 0 2018-06-28 19:28 hdfs://root@172.17.0.2/915847d3-a0a3-40bb-997a-168cd34d037e +drwxr-xr-x - root supergroup 0 2018-06-28 19:28 hdfs://root@172.17.0.2/915847d3-a0a3-40bb-997a-168cd34d037e/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:28 hdfs://root@172.17.0.2/915847d3-a0a3-40bb-997a-168cd34d037e/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:28 hdfs://root@172.17.0.2/915847d3-a0a3-40bb-997a-168cd34d037e/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-27 21:19 hdfs://root@172.17.0.2/96b09f65-391a-43df-8b14-25a424ba5ed3 +drwxr-xr-x - root supergroup 0 2018-06-27 21:19 hdfs://root@172.17.0.2/96b09f65-391a-43df-8b14-25a424ba5ed3/00 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:19 hdfs://root@172.17.0.2/96b09f65-391a-43df-8b14-25a424ba5ed3/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:19 hdfs://root@172.17.0.2/96b09f65-391a-43df-8b14-25a424ba5ed3/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 20:04 hdfs://root@172.17.0.2/9bedddc9-754d-418b-966a-44c516b7a34e +drwxr-xr-x - root supergroup 0 2018-06-28 20:04 hdfs://root@172.17.0.2/9bedddc9-754d-418b-966a-44c516b7a34e/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:04 hdfs://root@172.17.0.2/9bedddc9-754d-418b-966a-44c516b7a34e/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:04 hdfs://root@172.17.0.2/9bedddc9-754d-418b-966a-44c516b7a34e/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-27 21:24 hdfs://root@172.17.0.2/a7351b7f-fb0a-4d66-a6f3-9534d7e45a70 +drwxr-xr-x - root supergroup 0 2018-06-27 21:25 hdfs://root@172.17.0.2/a7351b7f-fb0a-4d66-a6f3-9534d7e45a70/00 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:24 hdfs://root@172.17.0.2/a7351b7f-fb0a-4d66-a6f3-9534d7e45a70/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:25 hdfs://root@172.17.0.2/a7351b7f-fb0a-4d66-a6f3-9534d7e45a70/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 16:51 hdfs://root@172.17.0.2/aa3a9275-466b-44ea-846d-d38d1724289f +-rw-r--r-- 1 root supergroup 3 2018-06-28 16:51 hdfs://root@172.17.0.2/aa3a9275-466b-44ea-846d-d38d1724289f/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 16:51 hdfs://root@172.17.0.2/aa3a9275-466b-44ea-846d-d38d1724289f/foo +drwxr-xr-x - root supergroup 0 2018-06-28 20:48 hdfs://root@172.17.0.2/aacea74a-0584-457b-8d42-74aaeaebc378 +drwxr-xr-x - root supergroup 0 2018-06-28 20:48 hdfs://root@172.17.0.2/aacea74a-0584-457b-8d42-74aaeaebc378/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:48 hdfs://root@172.17.0.2/aacea74a-0584-457b-8d42-74aaeaebc378/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:48 hdfs://root@172.17.0.2/aacea74a-0584-457b-8d42-74aaeaebc378/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 20:04 hdfs://root@172.17.0.2/b0ccee7d-7f1b-4742-ba25-dc01009bbd1a +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:04 hdfs://root@172.17.0.2/b0ccee7d-7f1b-4742-ba25-dc01009bbd1a/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:04 hdfs://root@172.17.0.2/b0ccee7d-7f1b-4742-ba25-dc01009bbd1a/foo +drwxr-xr-x - root supergroup 0 2018-06-28 19:23 hdfs://root@172.17.0.2/b3dc69d0-1392-46a5-8524-28db403c31f5 +drwxr-xr-x - root supergroup 0 2018-06-28 19:23 hdfs://root@172.17.0.2/b3dc69d0-1392-46a5-8524-28db403c31f5/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:23 hdfs://root@172.17.0.2/b3dc69d0-1392-46a5-8524-28db403c31f5/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:23 hdfs://root@172.17.0.2/b3dc69d0-1392-46a5-8524-28db403c31f5/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 17:59 hdfs://root@172.17.0.2/b4e4d71d-26bd-4288-8749-da2b225ee9ca +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:59 hdfs://root@172.17.0.2/b4e4d71d-26bd-4288-8749-da2b225ee9ca/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:59 hdfs://root@172.17.0.2/b4e4d71d-26bd-4288-8749-da2b225ee9ca/foo +drwxr-xr-x - root supergroup 0 2018-06-28 17:45 hdfs://root@172.17.0.2/b5855fb7-6e97-4d83-ba60-5aa1214a65cb +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:45 hdfs://root@172.17.0.2/b5855fb7-6e97-4d83-ba60-5aa1214a65cb/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:45 hdfs://root@172.17.0.2/b5855fb7-6e97-4d83-ba60-5aa1214a65cb/foo +drwxr-xr-x - root supergroup 0 2018-06-28 17:59 hdfs://root@172.17.0.2/b6ab34d9-21c1-4a94-9b68-9334a61b1730 +drwxr-xr-x - root supergroup 0 2018-06-28 17:59 hdfs://root@172.17.0.2/b6ab34d9-21c1-4a94-9b68-9334a61b1730/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:59 hdfs://root@172.17.0.2/b6ab34d9-21c1-4a94-9b68-9334a61b1730/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:59 hdfs://root@172.17.0.2/b6ab34d9-21c1-4a94-9b68-9334a61b1730/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:40 hdfs://root@172.17.0.2/b87345e9-e136-4369-b6ab-7fb25cb29431 +drwxr-xr-x - root supergroup 0 2018-06-28 19:40 hdfs://root@172.17.0.2/b87345e9-e136-4369-b6ab-7fb25cb29431/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:40 hdfs://root@172.17.0.2/b87345e9-e136-4369-b6ab-7fb25cb29431/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:40 hdfs://root@172.17.0.2/b87345e9-e136-4369-b6ab-7fb25cb29431/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 20:39 hdfs://root@172.17.0.2/b8baa132-bfce-47bc-88bd-6ade14d4e8f2 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:39 hdfs://root@172.17.0.2/b8baa132-bfce-47bc-88bd-6ade14d4e8f2/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:38 hdfs://root@172.17.0.2/b8baa132-bfce-47bc-88bd-6ade14d4e8f2/foo +drwxr-xr-x - root supergroup 0 2018-06-27 21:34 hdfs://root@172.17.0.2/b9696820-f7a7-4271-b960-81b685528ab0 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:34 hdfs://root@172.17.0.2/b9696820-f7a7-4271-b960-81b685528ab0/bar +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:34 hdfs://root@172.17.0.2/b9696820-f7a7-4271-b960-81b685528ab0/foo +drwxr-xr-x - root supergroup 0 2018-06-28 20:36 hdfs://root@172.17.0.2/ba729273-3c1c-4ad9-a20b-9fad648fb64e +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:36 hdfs://root@172.17.0.2/ba729273-3c1c-4ad9-a20b-9fad648fb64e/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:36 hdfs://root@172.17.0.2/ba729273-3c1c-4ad9-a20b-9fad648fb64e/foo +drwxr-xr-x - root supergroup 0 2018-06-28 17:45 hdfs://root@172.17.0.2/bdb3ea14-0e14-45e2-a05b-4590073bf5d4 +drwxr-xr-x - root supergroup 0 2018-06-28 17:45 hdfs://root@172.17.0.2/bdb3ea14-0e14-45e2-a05b-4590073bf5d4/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:45 hdfs://root@172.17.0.2/bdb3ea14-0e14-45e2-a05b-4590073bf5d4/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:45 hdfs://root@172.17.0.2/bdb3ea14-0e14-45e2-a05b-4590073bf5d4/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 21:02 hdfs://root@172.17.0.2/be4fd3a6-8950-4a9d-830c-0ea578a9cce1 +drwxr-xr-x - root supergroup 0 2018-06-28 21:03 hdfs://root@172.17.0.2/be4fd3a6-8950-4a9d-830c-0ea578a9cce1/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 21:02 hdfs://root@172.17.0.2/be4fd3a6-8950-4a9d-830c-0ea578a9cce1/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 21:03 hdfs://root@172.17.0.2/be4fd3a6-8950-4a9d-830c-0ea578a9cce1/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:19 hdfs://root@172.17.0.2/be74bec6-d330-4adb-9fa1-6d7c8fe112c5 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:19 hdfs://root@172.17.0.2/be74bec6-d330-4adb-9fa1-6d7c8fe112c5/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:19 hdfs://root@172.17.0.2/be74bec6-d330-4adb-9fa1-6d7c8fe112c5/foo +drwxr-xr-x - root supergroup 0 2018-06-27 21:29 hdfs://root@172.17.0.2/c15c2894-00c3-4ff7-bf62-c93e27cfa1be +drwxr-xr-x - root supergroup 0 2018-06-27 21:29 hdfs://root@172.17.0.2/c15c2894-00c3-4ff7-bf62-c93e27cfa1be/00 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:29 hdfs://root@172.17.0.2/c15c2894-00c3-4ff7-bf62-c93e27cfa1be/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:29 hdfs://root@172.17.0.2/c15c2894-00c3-4ff7-bf62-c93e27cfa1be/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:19 hdfs://root@172.17.0.2/c3ef1cea-b03b-4506-b635-f01936e62e7a +drwxr-xr-x - root supergroup 0 2018-06-28 19:19 hdfs://root@172.17.0.2/c3ef1cea-b03b-4506-b635-f01936e62e7a/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:19 hdfs://root@172.17.0.2/c3ef1cea-b03b-4506-b635-f01936e62e7a/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:19 hdfs://root@172.17.0.2/c3ef1cea-b03b-4506-b635-f01936e62e7a/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:57 hdfs://root@172.17.0.2/cb4e44d9-58c8-4922-aaf2-a110d0b07c87 +drwxr-xr-x - root supergroup 0 2018-06-28 19:58 hdfs://root@172.17.0.2/cb4e44d9-58c8-4922-aaf2-a110d0b07c87/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:57 hdfs://root@172.17.0.2/cb4e44d9-58c8-4922-aaf2-a110d0b07c87/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:58 hdfs://root@172.17.0.2/cb4e44d9-58c8-4922-aaf2-a110d0b07c87/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:17 hdfs://root@172.17.0.2/cec93744-110a-4453-a310-3402dd571319 +drwxr-xr-x - root supergroup 0 2018-06-28 19:17 hdfs://root@172.17.0.2/cec93744-110a-4453-a310-3402dd571319/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:17 hdfs://root@172.17.0.2/cec93744-110a-4453-a310-3402dd571319/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:17 hdfs://root@172.17.0.2/cec93744-110a-4453-a310-3402dd571319/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-27 21:34 hdfs://root@172.17.0.2/cf8fb233-ed96-4f36-b0fe-53e373f26466 +drwxr-xr-x - root supergroup 0 2018-06-27 21:34 hdfs://root@172.17.0.2/cf8fb233-ed96-4f36-b0fe-53e373f26466/00 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:34 hdfs://root@172.17.0.2/cf8fb233-ed96-4f36-b0fe-53e373f26466/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:34 hdfs://root@172.17.0.2/cf8fb233-ed96-4f36-b0fe-53e373f26466/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-27 21:19 hdfs://root@172.17.0.2/d17aeeab-69c2-42c7-957c-38887d1537af +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:19 hdfs://root@172.17.0.2/d17aeeab-69c2-42c7-957c-38887d1537af/bar +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:19 hdfs://root@172.17.0.2/d17aeeab-69c2-42c7-957c-38887d1537af/foo +drwxr-xr-x - root supergroup 0 2018-06-28 20:38 hdfs://root@172.17.0.2/d57a2f41-3410-4dbe-b9d5-54d5c7082aa1 +drwxr-xr-x - root supergroup 0 2018-06-28 20:39 hdfs://root@172.17.0.2/d57a2f41-3410-4dbe-b9d5-54d5c7082aa1/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:38 hdfs://root@172.17.0.2/d57a2f41-3410-4dbe-b9d5-54d5c7082aa1/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:39 hdfs://root@172.17.0.2/d57a2f41-3410-4dbe-b9d5-54d5c7082aa1/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:55 hdfs://root@172.17.0.2/d7728cf0-a583-4874-8c86-bf721f87aaca +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:55 hdfs://root@172.17.0.2/d7728cf0-a583-4874-8c86-bf721f87aaca/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:55 hdfs://root@172.17.0.2/d7728cf0-a583-4874-8c86-bf721f87aaca/foo +drwxr-xr-x - root supergroup 0 2018-06-28 19:28 hdfs://root@172.17.0.2/dbdda09a-514b-495b-af13-5b0b82a4e8bb +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:28 hdfs://root@172.17.0.2/dbdda09a-514b-495b-af13-5b0b82a4e8bb/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:28 hdfs://root@172.17.0.2/dbdda09a-514b-495b-af13-5b0b82a4e8bb/foo +drwxr-xr-x - root supergroup 0 2018-06-28 20:36 hdfs://root@172.17.0.2/deee941a-ce30-4667-894b-05c321055c98 +drwxr-xr-x - root supergroup 0 2018-06-28 20:36 hdfs://root@172.17.0.2/deee941a-ce30-4667-894b-05c321055c98/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:36 hdfs://root@172.17.0.2/deee941a-ce30-4667-894b-05c321055c98/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:36 hdfs://root@172.17.0.2/deee941a-ce30-4667-894b-05c321055c98/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-27 21:23 hdfs://root@172.17.0.2/e043a5c5-d48f-440e-b40b-f60096908de8 +drwxr-xr-x - root supergroup 0 2018-06-27 21:24 hdfs://root@172.17.0.2/e043a5c5-d48f-440e-b40b-f60096908de8/00 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:23 hdfs://root@172.17.0.2/e043a5c5-d48f-440e-b40b-f60096908de8/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:24 hdfs://root@172.17.0.2/e043a5c5-d48f-440e-b40b-f60096908de8/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 20:25 hdfs://root@172.17.0.2/e0786a91-9ce8-4789-911b-efbaac3046e3 +drwxr-xr-x - root supergroup 0 2018-06-28 20:25 hdfs://root@172.17.0.2/e0786a91-9ce8-4789-911b-efbaac3046e3/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:25 hdfs://root@172.17.0.2/e0786a91-9ce8-4789-911b-efbaac3046e3/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:25 hdfs://root@172.17.0.2/e0786a91-9ce8-4789-911b-efbaac3046e3/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 21:04 hdfs://root@172.17.0.2/e1feadbe-8a17-4dee-90fc-815861065429 +drwxr-xr-x - root supergroup 0 2018-06-28 21:04 hdfs://root@172.17.0.2/e1feadbe-8a17-4dee-90fc-815861065429/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 21:04 hdfs://root@172.17.0.2/e1feadbe-8a17-4dee-90fc-815861065429/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 21:04 hdfs://root@172.17.0.2/e1feadbe-8a17-4dee-90fc-815861065429/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 20:29 hdfs://root@172.17.0.2/e210facd-1b37-497d-99f9-4cf9feabb15b +drwxr-xr-x - root supergroup 0 2018-06-28 20:30 hdfs://root@172.17.0.2/e210facd-1b37-497d-99f9-4cf9feabb15b/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:29 hdfs://root@172.17.0.2/e210facd-1b37-497d-99f9-4cf9feabb15b/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:30 hdfs://root@172.17.0.2/e210facd-1b37-497d-99f9-4cf9feabb15b/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-27 21:23 hdfs://root@172.17.0.2/e5ba97c5-01f1-459d-ac74-b3c4f7819a53 +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:23 hdfs://root@172.17.0.2/e5ba97c5-01f1-459d-ac74-b3c4f7819a53/bar +-rw-r--r-- 1 root supergroup 3 2018-06-27 21:23 hdfs://root@172.17.0.2/e5ba97c5-01f1-459d-ac74-b3c4f7819a53/foo +drwxr-xr-x - root supergroup 0 2018-06-28 17:07 hdfs://root@172.17.0.2/e78a3d9b-7d20-4b83-b43c-5708b072653c +drwxr-xr-x - root supergroup 0 2018-06-28 17:08 hdfs://root@172.17.0.2/e78a3d9b-7d20-4b83-b43c-5708b072653c/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:07 hdfs://root@172.17.0.2/e78a3d9b-7d20-4b83-b43c-5708b072653c/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:08 hdfs://root@172.17.0.2/e78a3d9b-7d20-4b83-b43c-5708b072653c/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 17:48 hdfs://root@172.17.0.2/e8ea879e-9929-463c-8550-a9ce1a644f03 +drwxr-xr-x - root supergroup 0 2018-06-28 17:49 hdfs://root@172.17.0.2/e8ea879e-9929-463c-8550-a9ce1a644f03/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:48 hdfs://root@172.17.0.2/e8ea879e-9929-463c-8550-a9ce1a644f03/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 17:49 hdfs://root@172.17.0.2/e8ea879e-9929-463c-8550-a9ce1a644f03/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 20:43 hdfs://root@172.17.0.2/ea7a8505-d345-444b-b9fb-3cf89e3b4a9b +drwxr-xr-x - root supergroup 0 2018-06-28 20:43 hdfs://root@172.17.0.2/ea7a8505-d345-444b-b9fb-3cf89e3b4a9b/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:43 hdfs://root@172.17.0.2/ea7a8505-d345-444b-b9fb-3cf89e3b4a9b/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:43 hdfs://root@172.17.0.2/ea7a8505-d345-444b-b9fb-3cf89e3b4a9b/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 20:28 hdfs://root@172.17.0.2/eb12b8cb-576e-4357-ac6b-a74946ab08ef +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:28 hdfs://root@172.17.0.2/eb12b8cb-576e-4357-ac6b-a74946ab08ef/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:28 hdfs://root@172.17.0.2/eb12b8cb-576e-4357-ac6b-a74946ab08ef/foo +drwxr-xr-x - root supergroup 0 2018-06-28 19:47 hdfs://root@172.17.0.2/ec1f09b6-7b10-4179-b5b1-33d328a38362 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:47 hdfs://root@172.17.0.2/ec1f09b6-7b10-4179-b5b1-33d328a38362/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:46 hdfs://root@172.17.0.2/ec1f09b6-7b10-4179-b5b1-33d328a38362/foo +drwxr-xr-x - root supergroup 0 2018-06-28 19:17 hdfs://root@172.17.0.2/eff121fa-c710-4920-9d71-7b3310229652 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:17 hdfs://root@172.17.0.2/eff121fa-c710-4920-9d71-7b3310229652/bar +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:17 hdfs://root@172.17.0.2/eff121fa-c710-4920-9d71-7b3310229652/foo +drwxr-xr-x - root supergroup 0 2018-06-28 20:53 hdfs://root@172.17.0.2/f05611a1-d925-4027-b75d-1f4cc66f93d1 +drwxr-xr-x - root supergroup 0 2018-06-28 20:54 hdfs://root@172.17.0.2/f05611a1-d925-4027-b75d-1f4cc66f93d1/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:53 hdfs://root@172.17.0.2/f05611a1-d925-4027-b75d-1f4cc66f93d1/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:54 hdfs://root@172.17.0.2/f05611a1-d925-4027-b75d-1f4cc66f93d1/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 20:00 hdfs://root@172.17.0.2/f4e7daed-9b08-44a8-ba9f-fb235f291f24 +drwxr-xr-x - root supergroup 0 2018-06-28 20:01 hdfs://root@172.17.0.2/f4e7daed-9b08-44a8-ba9f-fb235f291f24/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:00 hdfs://root@172.17.0.2/f4e7daed-9b08-44a8-ba9f-fb235f291f24/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:01 hdfs://root@172.17.0.2/f4e7daed-9b08-44a8-ba9f-fb235f291f24/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 20:24 hdfs://root@172.17.0.2/fce2c4aa-fbc2-41a3-af94-64a153f81dad +drwxr-xr-x - root supergroup 0 2018-06-28 20:24 hdfs://root@172.17.0.2/fce2c4aa-fbc2-41a3-af94-64a153f81dad/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:24 hdfs://root@172.17.0.2/fce2c4aa-fbc2-41a3-af94-64a153f81dad/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 20:24 hdfs://root@172.17.0.2/fce2c4aa-fbc2-41a3-af94-64a153f81dad/00/000200000000000000000075433c81259d3c38e364b348af52e84d +drwxr-xr-x - root supergroup 0 2018-06-28 19:30 hdfs://root@172.17.0.2/fe4b1bad-5fad-4097-92c2-e57d6e091fb6 +drwxr-xr-x - root supergroup 0 2018-06-28 19:30 hdfs://root@172.17.0.2/fe4b1bad-5fad-4097-92c2-e57d6e091fb6/00 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:30 hdfs://root@172.17.0.2/fe4b1bad-5fad-4097-92c2-e57d6e091fb6/00/00020000000000000000003dba826b9be9c6a8e2f8310a770555c4 +-rw-r--r-- 1 root supergroup 3 2018-06-28 19:30 hdfs://root@172.17.0.2/fe4b1bad-5fad-4097-92c2-e57d6e091fb6/00/000200000000000000000075433c81259d3c38e364b348af52e84d diff --git a/dvc/project.py b/dvc/project.py index 08aaa3be6..c99de489a 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -254,7 +254,12 @@ class Project(object): stage.checkout() def _used_cache(self, target=None, all_branches=False): - cache = [] + cache = {} + cache['local'] = [] + cache['s3'] = [] + cache['gs'] = [] + cache['hdfs'] = [] + cache['ssh'] = [] for branch in self.scm.brancher(all_branches=all_branches): if target: @@ -264,27 +269,41 @@ class Project(object): for stage in stages: for out in stage.outs: - if out.path_info['scheme'] != 'local': - continue + scheme = out.path_info['scheme'] if not out.use_cache or not out.info: continue - cache.append(out.info) + cache[scheme] += [out.info] + + if scheme != 'local': + continue if self.cache.local.is_dir_cache(out.cache) and os.path.isfile(out.cache): dir_cache = self.cache.local.load_dir_cache(out.cache) - cache += dir_cache + cache[scheme] += dir_cache return cache def _used_cache_files(self, target=None, all_branches=False): - infos = self._used_cache(target=target, all_branches=all_branches) + infos = self._used_cache(target=target, all_branches=all_branches)['local'] return [self.cache.local.get(info[self.cache.local.PARAM_MD5]) for info in infos] def gc(self, all_branches=False): clist = self._used_cache(target=None, all_branches=all_branches) - self.cache.local.gc(clist) + self.cache.local.gc(clist['local']) + + if self.cache.s3: + self.cache.s3.gc(clist['s3']) + + if self.cache.gs: + self.cache.gs.gc(clist['gs']) + + if self.cache.ssh: + self.cache.ssh.gc(clist['ssh']) + + if self.cache.hdfs: + self.cache.hdfs.gc(clist['hdfs']) def push(self, target=None, jobs=1, remote=None, all_branches=False): self.cloud.push(self._used_cache_files(target, all_branches), jobs, remote=remote) diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 602a56388..7c09a5981 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -88,6 +88,9 @@ class RemoteGS(RemoteBase): if path_info['scheme'] != 'gs': raise NotImplementedError + Logger.debug("Removing gs://{}/{}".format(path_info['bucket'], + path_info['key'])) + blob = self.gs.bucket(path_info['bucket']).get_blob(path_info['key']) if not blob: return @@ -166,3 +169,22 @@ class RemoteGS(RemoteBase): progress.finish_target(name) return path + + def _path_to_etag(self, path): + relpath = posixpath.relpath(path, self.prefix) + return posixpath.dirname(relpath) + posixpath.basename(relpath) + + def _all_etags(self): + blobs = list(self.gs.bucket(self.bucket).list_blobs(prefix=self.prefix)) + return [self._path_to_etag(blob.name) for blob in blobs] + + def gc(self, checksum_infos): + used_etags = [info[self.PARAM_ETAG] for info in checksum_infos] + + for etag in self._all_etags(): + if etag in used_etags: + continue + path_info = {'scheme': 'gs', + 'bucket': self.bucket, + 'key': posixpath.join(self.prefix, etag[0:2], etag[2:])} + self.remove(path_info) diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index 6dc98cea6..aa8c8634e 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -12,6 +12,7 @@ except ImportError: from dvc.config import Config from dvc.remote.base import RemoteBase from dvc.exceptions import DvcException +from dvc.logger import Logger class RemoteHDFS(RemoteBase): @@ -103,6 +104,8 @@ class RemoteHDFS(RemoteBase): assert path_info.get('url') + Logger.debug('Removing {}'.format(path_info['url'])) + self.rm(path_info) def upload(self, path, path_info): @@ -121,3 +124,28 @@ class RemoteHDFS(RemoteBase): os.makedirs(dname) self.hadoop_fs('copyToLocal {} {}'.format(path_info['url'], path), user=path_info['user']) + + def _path_to_checksum(self, path): + relpath = posixpath.relpath(path, self.url) + return posixpath.dirname(relpath) + posixpath.basename(relpath) + + def _all_checksums(self): + stdout = self.hadoop_fs('ls -R {}'.format(self.url)) + lines = stdout.split('\n') + flist = [] + for line in lines: + if not line.startswith('-'): + continue + flist.append(line.split()[-1]) + return [self._path_to_checksum(path) for path in flist] + + def gc(self, checksum_infos): + used_checksums = [info[self.PARAM_CHECKSUM] for info in checksum_infos] + + for checksum in self._all_checksums(): + if checksum in used_checksums: + continue + path_info = {'scheme': 'hdfs', + 'user': self.user, + 'url': posixpath.join(self.url, checksum[0:2], checksum[2:])} + self.remove(path_info) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 54eeebf20..ee481ffe7 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -103,6 +103,9 @@ class RemoteS3(RemoteBase): if path_info['scheme'] != 's3': raise NotImplementedError + Logger.debug('Removing s3://{}/{}'.format(path_info['bucket'], + path_info['key'])) + try: obj = self.s3.Object(path_info['bucket'], path_info['key']).get() obj.delete() @@ -183,3 +186,22 @@ class RemoteS3(RemoteBase): progress.finish_target(name) return fname + + def _path_to_etag(self, path): + relpath = posixpath.relpath(path, self.prefix) + return posixpath.dirname(relpath) + posixpath.basename(relpath) + + def _all(self): + objects = self.s3.Bucket(self.bucket).objects.filter(Prefix=self.prefix) + return [self._path_to_etag(obj.key) for obj in objects] + + def gc(self, checksum_infos): + used_etags = [info[self.PARAM_ETAG] for info in checksum_infos] + + for etag in self._all(): + if etag in used_etags: + continue + path_info = {'scheme': 's3', + 'key': posixpath.join(self.prefix, etag[0:2], etag[2:]), + 'bucket': self.bucket} + self.remove(path_info) diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index 2944c0736..4203f9c8b 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -123,13 +123,19 @@ class RemoteSSH(RemoteBase): self._makedirs_remote(ret) return ret + def _exec(self, ssh, cmd): + stdin, stdout, stderr = ssh.exec_command(cmd) + if stdout.channel.recv_exit_status() != 0: + DvcException('SSH command \'{}\' failed: {}'.format(cmd, stderr.read())) + return stdout.read().decode('utf-8') + def md5(self, path_info): if path_info['scheme'] != 'ssh': raise NotImplementedError ssh = self.ssh(host=path_info['host'], user=path_info['user']) - stdin, stdout, stderr = ssh.exec_command('md5sum {}'.format(path_info['path'])) - md5 = re.match(r'^(?P.*) .*$', stdout.read().decode('utf-8')).group('md5') + stdout = self._exec(ssh, 'md5sum {}'.format(path_info['path'])) + md5 = re.match(r'^(?P.*) .*$', stdout).group('md5') ssh.close() return md5 @@ -142,7 +148,11 @@ class RemoteSSH(RemoteBase): assert from_info['user'] == to_info['user'] ssh = self.ssh(host=from_info['host'], user=from_info['user']) - ssh.exec_command('cp {} {}'.format(from_info['path'], to_info['path'])) + + dname = posixpath.dirname(to_info['path']) + self._exec(ssh, 'mkdir -p {}'.format(dname)) + self._exec(ssh, 'cp {} {}'.format(from_info['path'], to_info['path'])) + ssh.close() def save_info(self, path_info): @@ -180,6 +190,10 @@ class RemoteSSH(RemoteBase): if path_info['scheme'] != 'ssh': raise NotImplementedError + Logger.debug('Removing ssh://{}@{}/{}'.format(path_info['user'], + path_info['host'], + path_info['path'])) + ssh = self.ssh(host=path_info['host'], user=path_info['user']) ssh.open_sftp().remove(path_info['path']) ssh.close() @@ -235,3 +249,27 @@ class RemoteSSH(RemoteBase): progress.finish_target(name) return path + + def _path_to_md5(self, path): + relpath = posixpath.relpath(path, self.prefix) + return posixpath.dirname(relpath) + posixpath.basename(relpath) + + def _all_md5s(self): + ssh = self.ssh(host=self.host, user=self.user) + stdout = self._exec(ssh, 'find {} -type f -follow -print'.format(self.prefix)) + flist = stdout.split() + ssh.close() + + return [self._path_to_md5(path) for path in flist] + + def gc(self, checksum_infos): + used_md5s = [info[self.PARAM_MD5] for info in checksum_infos] + + for md5 in self._all_md5s(): + if md5 in used_md5s: + continue + path_info = {'scheme': 'ssh', + 'user': self.user, + 'host': self.host, + 'path': posixpath.join(self.prefix, md5[0:2], md5[2:])} + self.remove(path_info) diff --git a/tests/test_repro.py b/tests/test_repro.py index 92991d206..a6688a929 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -344,6 +344,8 @@ class TestReproExternalBase(TestDvc): stages = self.dvc.reproduce(cmd_stage.path) self.assertEqual(len(stages), 1) + self.dvc.gc() + class TestReproExternalS3(TestReproExternalBase): def should_test(self):