From e600e3a7363499294b78bb3b8b604e6026b9ba96 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Fri, 18 Jan 2019 18:51:33 -0600 Subject: [PATCH 1/2] remote: Use `checksum` instead of MD5/ETag --- dvc/output/base.py | 4 ++++ dvc/output/local.py | 6 +----- dvc/remote/azure.py | 2 +- dvc/remote/http.py | 4 ++-- tests/test_checkout.py | 6 +++--- tests/test_repro.py | 2 +- tests/test_run.py | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dvc/output/base.py b/dvc/output/base.py index 694a8ddf7..b52727d58 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -95,6 +95,10 @@ class OutputBase(object): def sep(self): return '/' + @property + def checksum(self): + return self.info.get(self.remote.PARAM_CHECKSUM) + @property def exists(self): return self.remote.exists(self.path_info) diff --git a/dvc/output/local.py b/dvc/output/local.py index 4e1f8d3ff..17eb1d927 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -54,13 +54,9 @@ class OutputLOCAL(OutputBase): def rel_path(self): return os.path.relpath(self.path) - @property - def md5(self): - return self.info.get(self.project.cache.local.PARAM_CHECKSUM, None) - @property def cache(self): - return self.project.cache.local.get(self.md5) + return self.project.cache.local.get(self.checksum) def dumpd(self): ret = super(OutputLOCAL, self).dumpd() diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index abb9d5c13..749fb4d63 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -29,7 +29,7 @@ class RemoteAzure(RemoteBase): r'(ContainerName=(?P[^;]+);?)?' r'(?P.+)?)?)$') REQUIRES = {'azure-storage-blob': BlockBlobService} - PARAM_ETAG = 'etag' + PARAM_CHECKSUM = 'etag' COPY_POLL_SECONDS = 5 def __init__(self, project, config): diff --git a/dvc/remote/http.py b/dvc/remote/http.py index a7b2e2262..2faf871a8 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -27,7 +27,7 @@ class RemoteHTTP(RemoteBase): REGEX = r'^https?://.*$' REQUEST_TIMEOUT = 10 CHUNK_SIZE = 1000000 # Megabyte - PARAM_ETAG = 'etag' + PARAM_CHECKSUM = 'etag' def __init__(self, project, config): super(RemoteHTTP, self).__init__(project, config) @@ -99,7 +99,7 @@ class RemoteHTTP(RemoteBase): if path_info['scheme'] not in ['http', 'https']: raise NotImplementedError - return {self.PARAM_ETAG: self._etag(path_info['path'])} + return {self.PARAM_CHECKSUM: self._etag(path_info['path'])} def _content_length(self, url): return self._request('HEAD', url).headers.get('Content-Length') diff --git a/tests/test_checkout.py b/tests/test_checkout.py index 706b05236..2fe43da65 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -80,9 +80,9 @@ class TestCheckoutCorruptedCacheDir(TestDvc): # NOTE: modifying cache file for one of the files inside the directory # to check if dvc will detect that the cache is corrupted. - entry = self.dvc.cache.local.load_dir_cache(out.md5)[0] - md5 = entry[self.dvc.cache.local.PARAM_CHECKSUM] - cache = self.dvc.cache.local.get(md5) + entry = self.dvc.cache.local.load_dir_cache(out.checksum)[0] + checksum = entry[self.dvc.cache.local.PARAM_CHECKSUM] + cache = self.dvc.cache.local.get(checksum) with open(cache, 'w+') as fobj: fobj.write('1') diff --git a/tests/test_repro.py b/tests/test_repro.py index a943657e2..33ac54108 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -464,7 +464,7 @@ class TestReproDataSource(TestReproChangedData): stages = self.dvc.reproduce(self.foo_stage.path) self.assertTrue(filecmp.cmp(self.FOO, self.BAR, shallow=False)) - self.assertEqual(stages[0].outs[0].md5, file_md5(self.BAR)[0]) + self.assertEqual(stages[0].outs[0].checksum, file_md5(self.BAR)[0]) class TestReproChangedDir(TestDvc): diff --git a/tests/test_run.py b/tests/test_run.py index 71f64ce45..f939b7ff3 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -41,7 +41,7 @@ class TestRun(TestDvc): self.assertEqual(len(stage.deps), len(deps)) self.assertEqual(len(stage.outs), len(outs + outs_no_cache)) self.assertEqual(stage.outs[0].path, outs[0]) - self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0]) + self.assertEqual(stage.outs[0].checksum, file_md5(self.FOO)[0]) self.assertTrue(stage.path, fname) with self.assertRaises(OutputDuplicationError): From 823077aa2b998781957fd4dde2afe356a0d6ebda Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Fri, 18 Jan 2019 23:05:00 -0600 Subject: [PATCH 2/2] stage: Checkout cached output when possible Fix #1348 --- dvc/output/base.py | 8 ++++++ dvc/stage.py | 18 ++++++++++++-- tests/test_repro.py | 59 +++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 81 insertions(+), 4 deletions(-) diff --git a/dvc/output/base.py b/dvc/output/base.py index b52727d58..3b8b2b445 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -113,6 +113,14 @@ class OutputBase(object): return getattr(self.project.cache, self.scheme).changed(self.path_info, self.info) + def changed_cache(self): + if not self.use_cache or not self.checksum: + return True + + cache = self.project.cache.__getattribute__(self.scheme) + + return cache.changed_cache(self.checksum) + def status(self): if self.changed(): # FIXME better msgs diff --git a/dvc/stage.py b/dvc/stage.py index 7bb718f7e..024c24efb 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -545,7 +545,10 @@ class Stage(object): logger.info("Importing '{dep}' -> '{out}'" .format(dep=self.deps[0].path, out=self.outs[0].path)) if not dry: - self.deps[0].download(self.outs[0].path_info) + if self._already_cached(): + self.outs[0].checkout() + else: + self.deps[0].download(self.outs[0].path_info) elif self.is_data_source: msg = u"Verifying data sources in '{}'".format(self.relpath) @@ -556,7 +559,10 @@ class Stage(object): else: logger.info('Running command:\n\t{}'.format(self.cmd)) if not dry: - self._run() + if self._already_cached(): + self.checkout() + else: + self._run() if not dry: self.save() @@ -598,3 +604,11 @@ class Stage(object): return {self.relpath: ret} return {} + + def _already_cached(self): + return (not self.changed_md5() + and all(not dep.changed() for dep in self.deps) + and all(not out.changed_cache() + if out.use_cache + else not out.changed() + for out in self.outs)) diff --git a/tests/test_repro.py b/tests/test_repro.py index 33ac54108..871e71b32 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -617,7 +617,57 @@ class TestReproExternalBase(TestDvc): def sep(self): return '/' - def test(self): + def check_already_cached(self, stage): + stage.outs[0].remove() + + patch_download = patch.object(stage.deps[0], + 'download', + wraps=stage.deps[0].download) + + patch_checkout = patch.object(stage.outs[0], + 'checkout', + wraps=stage.outs[0].checkout) + + patch_run = patch.object(stage, '_run', wraps=stage._run) + + with self.dvc.state, \ + patch_download as mock_download, \ + patch_checkout as mock_checkout, \ + patch_run as mock_run: + + stage.run() + + mock_run.assert_not_called() + mock_download.assert_not_called() + mock_checkout.assert_called_once() + + def corrupted_cache(self): + os.unlink('bar.dvc') + + stage = self.dvc.run(deps=[self.FOO], + outs=[self.BAR], + cmd='echo bar > bar') + + with open(self.BAR, 'w') as fd: + fd.write('corrupting the cache') + + patch_checkout = patch.object(stage.outs[0], + 'checkout', + wraps=stage.outs[0].checkout) + + patch_run = patch.object(stage, '_run', wraps=stage._run) + + with self.dvc.state, \ + patch_checkout as mock_checkout, \ + patch_run as mock_run: + + stage.run() + + mock_run.assert_called_once() + mock_checkout.assert_not_called() + + @patch('dvc.prompt.confirm', return_value=True) + def test(self, mock_prompt): if not self.should_test(): return @@ -654,9 +704,11 @@ class TestReproExternalBase(TestDvc): self.write(self.bucket, foo_key, self.FOO_CONTENTS) import_stage = self.dvc.imp(out_foo_path, 'import') + self.assertTrue(os.path.exists('import')) self.assertTrue(filecmp.cmp('import', self.FOO, shallow=False)) self.assertEqual(self.dvc.status(import_stage.path), {}) + self.check_already_cached(import_stage) import_remote_stage = self.dvc.imp(out_foo_path, out_foo_path + '_imported') @@ -665,9 +717,10 @@ class TestReproExternalBase(TestDvc): cmd_stage = self.dvc.run(outs=[out_bar_path], deps=[out_foo_path], cmd=self.cmd(foo_path, bar_path)) - self.assertEqual(self.dvc.status(cmd_stage.path), {}) + self.assertEqual(self.dvc.status(cmd_stage.path), {}) self.assertEqual(self.dvc.status(), {}) + self.check_already_cached(cmd_stage) self.write(self.bucket, foo_key, self.BAR_CONTENTS) @@ -697,6 +750,8 @@ class TestReproExternalBase(TestDvc): self.dvc.checkout(cmd_stage.path, force=True) self.assertEqual(self.dvc.status(cmd_stage.path), {}) + self.corrupted_cache() + class TestReproExternalS3(TestReproExternalBase): def should_test(self):