mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 15:28:28 +01:00
remote: Use checksum instead of MD5/ETag
This commit is contained in:
@@ -95,6 +95,10 @@ class OutputBase(object):
|
||||
def sep(self):
|
||||
return '/'
|
||||
|
||||
@property
|
||||
def checksum(self):
|
||||
return self.info.get(self.remote.PARAM_CHECKSUM)
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return self.remote.exists(self.path_info)
|
||||
|
||||
+1
-5
@@ -54,13 +54,9 @@ class OutputLOCAL(OutputBase):
|
||||
def rel_path(self):
|
||||
return os.path.relpath(self.path)
|
||||
|
||||
@property
|
||||
def md5(self):
|
||||
return self.info.get(self.project.cache.local.PARAM_CHECKSUM, None)
|
||||
|
||||
@property
|
||||
def cache(self):
|
||||
return self.project.cache.local.get(self.md5)
|
||||
return self.project.cache.local.get(self.checksum)
|
||||
|
||||
def dumpd(self):
|
||||
ret = super(OutputLOCAL, self).dumpd()
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ class RemoteAzure(RemoteBase):
|
||||
r'(ContainerName=(?P<container_name>[^;]+);?)?'
|
||||
r'(?P<connection_string>.+)?)?)$')
|
||||
REQUIRES = {'azure-storage-blob': BlockBlobService}
|
||||
PARAM_ETAG = 'etag'
|
||||
PARAM_CHECKSUM = 'etag'
|
||||
COPY_POLL_SECONDS = 5
|
||||
|
||||
def __init__(self, project, config):
|
||||
|
||||
+2
-2
@@ -27,7 +27,7 @@ class RemoteHTTP(RemoteBase):
|
||||
REGEX = r'^https?://.*$'
|
||||
REQUEST_TIMEOUT = 10
|
||||
CHUNK_SIZE = 1000000 # Megabyte
|
||||
PARAM_ETAG = 'etag'
|
||||
PARAM_CHECKSUM = 'etag'
|
||||
|
||||
def __init__(self, project, config):
|
||||
super(RemoteHTTP, self).__init__(project, config)
|
||||
@@ -99,7 +99,7 @@ class RemoteHTTP(RemoteBase):
|
||||
if path_info['scheme'] not in ['http', 'https']:
|
||||
raise NotImplementedError
|
||||
|
||||
return {self.PARAM_ETAG: self._etag(path_info['path'])}
|
||||
return {self.PARAM_CHECKSUM: self._etag(path_info['path'])}
|
||||
|
||||
def _content_length(self, url):
|
||||
return self._request('HEAD', url).headers.get('Content-Length')
|
||||
|
||||
@@ -80,9 +80,9 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
|
||||
|
||||
# NOTE: modifying cache file for one of the files inside the directory
|
||||
# to check if dvc will detect that the cache is corrupted.
|
||||
entry = self.dvc.cache.local.load_dir_cache(out.md5)[0]
|
||||
md5 = entry[self.dvc.cache.local.PARAM_CHECKSUM]
|
||||
cache = self.dvc.cache.local.get(md5)
|
||||
entry = self.dvc.cache.local.load_dir_cache(out.checksum)[0]
|
||||
checksum = entry[self.dvc.cache.local.PARAM_CHECKSUM]
|
||||
cache = self.dvc.cache.local.get(checksum)
|
||||
|
||||
with open(cache, 'w+') as fobj:
|
||||
fobj.write('1')
|
||||
|
||||
+1
-1
@@ -464,7 +464,7 @@ class TestReproDataSource(TestReproChangedData):
|
||||
stages = self.dvc.reproduce(self.foo_stage.path)
|
||||
|
||||
self.assertTrue(filecmp.cmp(self.FOO, self.BAR, shallow=False))
|
||||
self.assertEqual(stages[0].outs[0].md5, file_md5(self.BAR)[0])
|
||||
self.assertEqual(stages[0].outs[0].checksum, file_md5(self.BAR)[0])
|
||||
|
||||
|
||||
class TestReproChangedDir(TestDvc):
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ class TestRun(TestDvc):
|
||||
self.assertEqual(len(stage.deps), len(deps))
|
||||
self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
|
||||
self.assertEqual(stage.outs[0].path, outs[0])
|
||||
self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])
|
||||
self.assertEqual(stage.outs[0].checksum, file_md5(self.FOO)[0])
|
||||
self.assertTrue(stage.path, fname)
|
||||
|
||||
with self.assertRaises(OutputDuplicationError):
|
||||
|
||||
Reference in New Issue
Block a user