Merge pull request #1431 from mroutis/fix-1348

stage: Checkout cached output when possible
This commit is contained in:
Ruslan Kuprieiev
2019-01-23 03:31:47 +02:00
committed by GitHub
8 changed files with 94 additions and 17 deletions
+12
View File
@@ -95,6 +95,10 @@ class OutputBase(object):
def sep(self):
return '/'
@property
def checksum(self):
return self.info.get(self.remote.PARAM_CHECKSUM)
@property
def exists(self):
return self.remote.exists(self.path_info)
@@ -109,6 +113,14 @@ class OutputBase(object):
return getattr(self.project.cache, self.scheme).changed(self.path_info,
self.info)
def changed_cache(self):
if not self.use_cache or not self.checksum:
return True
cache = self.project.cache.__getattribute__(self.scheme)
return cache.changed_cache(self.checksum)
def status(self):
if self.changed():
# FIXME better msgs
+1 -5
View File
@@ -54,13 +54,9 @@ class OutputLOCAL(OutputBase):
def rel_path(self):
return os.path.relpath(self.path)
@property
def md5(self):
return self.info.get(self.project.cache.local.PARAM_CHECKSUM, None)
@property
def cache(self):
return self.project.cache.local.get(self.md5)
return self.project.cache.local.get(self.checksum)
def dumpd(self):
ret = super(OutputLOCAL, self).dumpd()
+1 -1
View File
@@ -29,7 +29,7 @@ class RemoteAzure(RemoteBase):
r'(ContainerName=(?P<container_name>[^;]+);?)?'
r'(?P<connection_string>.+)?)?)$')
REQUIRES = {'azure-storage-blob': BlockBlobService}
PARAM_ETAG = 'etag'
PARAM_CHECKSUM = 'etag'
COPY_POLL_SECONDS = 5
def __init__(self, project, config):
+2 -2
View File
@@ -27,7 +27,7 @@ class RemoteHTTP(RemoteBase):
REGEX = r'^https?://.*$'
REQUEST_TIMEOUT = 10
CHUNK_SIZE = 1000000 # Megabyte
PARAM_ETAG = 'etag'
PARAM_CHECKSUM = 'etag'
def __init__(self, project, config):
super(RemoteHTTP, self).__init__(project, config)
@@ -99,7 +99,7 @@ class RemoteHTTP(RemoteBase):
if path_info['scheme'] not in ['http', 'https']:
raise NotImplementedError
return {self.PARAM_ETAG: self._etag(path_info['path'])}
return {self.PARAM_CHECKSUM: self._etag(path_info['path'])}
def _content_length(self, url):
return self._request('HEAD', url).headers.get('Content-Length')
+16 -2
View File
@@ -547,7 +547,10 @@ class Stage(object):
logger.info("Importing '{dep}' -> '{out}'"
.format(dep=self.deps[0].path, out=self.outs[0].path))
if not dry:
self.deps[0].download(self.outs[0].path_info)
if self._already_cached():
self.outs[0].checkout()
else:
self.deps[0].download(self.outs[0].path_info)
elif self.is_data_source:
msg = u"Verifying data sources in '{}'".format(self.relpath)
@@ -558,7 +561,10 @@ class Stage(object):
else:
logger.info('Running command:\n\t{}'.format(self.cmd))
if not dry:
self._run()
if self._already_cached():
self.checkout()
else:
self._run()
if not dry:
self.save()
@@ -600,3 +606,11 @@ class Stage(object):
return {self.relpath: ret}
return {}
def _already_cached(self):
return (not self.changed_md5()
and all(not dep.changed() for dep in self.deps)
and all(not out.changed_cache()
if out.use_cache
else not out.changed()
for out in self.outs))
+3 -3
View File
@@ -80,9 +80,9 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
# NOTE: modifying cache file for one of the files inside the directory
# to check if dvc will detect that the cache is corrupted.
entry = self.dvc.cache.local.load_dir_cache(out.md5)[0]
md5 = entry[self.dvc.cache.local.PARAM_CHECKSUM]
cache = self.dvc.cache.local.get(md5)
entry = self.dvc.cache.local.load_dir_cache(out.checksum)[0]
checksum = entry[self.dvc.cache.local.PARAM_CHECKSUM]
cache = self.dvc.cache.local.get(checksum)
with open(cache, 'w+') as fobj:
fobj.write('1')
+58 -3
View File
@@ -604,7 +604,7 @@ class TestReproDataSource(TestReproChangedData):
stages = self.dvc.reproduce(self.foo_stage.path)
self.assertTrue(filecmp.cmp(self.FOO, self.BAR, shallow=False))
self.assertEqual(stages[0].outs[0].md5, file_md5(self.BAR)[0])
self.assertEqual(stages[0].outs[0].checksum, file_md5(self.BAR)[0])
class TestReproChangedDir(TestDvc):
@@ -757,7 +757,57 @@ class TestReproExternalBase(TestDvc):
def sep(self):
return '/'
def test(self):
def check_already_cached(self, stage):
stage.outs[0].remove()
patch_download = patch.object(stage.deps[0],
'download',
wraps=stage.deps[0].download)
patch_checkout = patch.object(stage.outs[0],
'checkout',
wraps=stage.outs[0].checkout)
patch_run = patch.object(stage, '_run', wraps=stage._run)
with self.dvc.state, \
patch_download as mock_download, \
patch_checkout as mock_checkout, \
patch_run as mock_run:
stage.run()
mock_run.assert_not_called()
mock_download.assert_not_called()
mock_checkout.assert_called_once()
def corrupted_cache(self):
os.unlink('bar.dvc')
stage = self.dvc.run(deps=[self.FOO],
outs=[self.BAR],
cmd='echo bar > bar')
with open(self.BAR, 'w') as fd:
fd.write('corrupting the cache')
patch_checkout = patch.object(stage.outs[0],
'checkout',
wraps=stage.outs[0].checkout)
patch_run = patch.object(stage, '_run', wraps=stage._run)
with self.dvc.state, \
patch_checkout as mock_checkout, \
patch_run as mock_run:
stage.run()
mock_run.assert_called_once()
mock_checkout.assert_not_called()
@patch('dvc.prompt.confirm', return_value=True)
def test(self, mock_prompt):
if not self.should_test():
return
@@ -794,9 +844,11 @@ class TestReproExternalBase(TestDvc):
self.write(self.bucket, foo_key, self.FOO_CONTENTS)
import_stage = self.dvc.imp(out_foo_path, 'import')
self.assertTrue(os.path.exists('import'))
self.assertTrue(filecmp.cmp('import', self.FOO, shallow=False))
self.assertEqual(self.dvc.status(import_stage.path), {})
self.check_already_cached(import_stage)
import_remote_stage = self.dvc.imp(out_foo_path,
out_foo_path + '_imported')
@@ -805,9 +857,10 @@ class TestReproExternalBase(TestDvc):
cmd_stage = self.dvc.run(outs=[out_bar_path],
deps=[out_foo_path],
cmd=self.cmd(foo_path, bar_path))
self.assertEqual(self.dvc.status(cmd_stage.path), {})
self.assertEqual(self.dvc.status(cmd_stage.path), {})
self.assertEqual(self.dvc.status(), {})
self.check_already_cached(cmd_stage)
self.write(self.bucket, foo_key, self.BAR_CONTENTS)
@@ -837,6 +890,8 @@ class TestReproExternalBase(TestDvc):
self.dvc.checkout(cmd_stage.path, force=True)
self.assertEqual(self.dvc.status(cmd_stage.path), {})
self.corrupted_cache()
class TestReproExternalS3(TestReproExternalBase):
def should_test(self):
+1 -1
View File
@@ -42,7 +42,7 @@ class TestRun(TestDvc):
self.assertEqual(len(stage.deps), len(deps))
self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
self.assertEqual(stage.outs[0].path, outs[0])
self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])
self.assertEqual(stage.outs[0].checksum, file_md5(self.FOO)[0])
self.assertTrue(stage.path, fname)
with self.assertRaises(OutputDuplicationError):