remote: Use checksum instead of MD5/ETag

This commit is contained in:
Mr. Outis
2019-01-19 00:02:28 -06:00
parent d768607733
commit e600e3a736
7 changed files with 13 additions and 13 deletions
+4
View File
@@ -95,6 +95,10 @@ class OutputBase(object):
def sep(self):
return '/'
@property
def checksum(self):
return self.info.get(self.remote.PARAM_CHECKSUM)
@property
def exists(self):
return self.remote.exists(self.path_info)
+1 -5
View File
@@ -54,13 +54,9 @@ class OutputLOCAL(OutputBase):
def rel_path(self):
return os.path.relpath(self.path)
@property
def md5(self):
return self.info.get(self.project.cache.local.PARAM_CHECKSUM, None)
@property
def cache(self):
return self.project.cache.local.get(self.md5)
return self.project.cache.local.get(self.checksum)
def dumpd(self):
ret = super(OutputLOCAL, self).dumpd()
+1 -1
View File
@@ -29,7 +29,7 @@ class RemoteAzure(RemoteBase):
r'(ContainerName=(?P<container_name>[^;]+);?)?'
r'(?P<connection_string>.+)?)?)$')
REQUIRES = {'azure-storage-blob': BlockBlobService}
PARAM_ETAG = 'etag'
PARAM_CHECKSUM = 'etag'
COPY_POLL_SECONDS = 5
def __init__(self, project, config):
+2 -2
View File
@@ -27,7 +27,7 @@ class RemoteHTTP(RemoteBase):
REGEX = r'^https?://.*$'
REQUEST_TIMEOUT = 10
CHUNK_SIZE = 1000000 # Megabyte
PARAM_ETAG = 'etag'
PARAM_CHECKSUM = 'etag'
def __init__(self, project, config):
super(RemoteHTTP, self).__init__(project, config)
@@ -99,7 +99,7 @@ class RemoteHTTP(RemoteBase):
if path_info['scheme'] not in ['http', 'https']:
raise NotImplementedError
return {self.PARAM_ETAG: self._etag(path_info['path'])}
return {self.PARAM_CHECKSUM: self._etag(path_info['path'])}
def _content_length(self, url):
return self._request('HEAD', url).headers.get('Content-Length')
+3 -3
View File
@@ -80,9 +80,9 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
# NOTE: modifying cache file for one of the files inside the directory
# to check if dvc will detect that the cache is corrupted.
entry = self.dvc.cache.local.load_dir_cache(out.md5)[0]
md5 = entry[self.dvc.cache.local.PARAM_CHECKSUM]
cache = self.dvc.cache.local.get(md5)
entry = self.dvc.cache.local.load_dir_cache(out.checksum)[0]
checksum = entry[self.dvc.cache.local.PARAM_CHECKSUM]
cache = self.dvc.cache.local.get(checksum)
with open(cache, 'w+') as fobj:
fobj.write('1')
+1 -1
View File
@@ -464,7 +464,7 @@ class TestReproDataSource(TestReproChangedData):
stages = self.dvc.reproduce(self.foo_stage.path)
self.assertTrue(filecmp.cmp(self.FOO, self.BAR, shallow=False))
self.assertEqual(stages[0].outs[0].md5, file_md5(self.BAR)[0])
self.assertEqual(stages[0].outs[0].checksum, file_md5(self.BAR)[0])
class TestReproChangedDir(TestDvc):
+1 -1
View File
@@ -41,7 +41,7 @@ class TestRun(TestDvc):
self.assertEqual(len(stage.deps), len(deps))
self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
self.assertEqual(stage.outs[0].path, outs[0])
self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])
self.assertEqual(stage.outs[0].checksum, file_md5(self.FOO)[0])
self.assertTrue(stage.path, fname)
with self.assertRaises(OutputDuplicationError):