mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 15:28:28 +01:00
gs: use md5_hash instead of etag for external deps/outs
Fixes #1409 Related #1402 Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
+32
-27
@@ -23,7 +23,7 @@ class RemoteGS(RemoteBase):
|
||||
scheme = 'gs'
|
||||
REGEX = r'^gs://(?P<path>.*)$'
|
||||
REQUIRES = {'google.cloud.storage': storage}
|
||||
PARAM_ETAG = 'etag'
|
||||
PARAM_MD5 = 'md5'
|
||||
|
||||
def __init__(self, project, config):
|
||||
self.project = project
|
||||
@@ -52,19 +52,23 @@ class RemoteGS(RemoteBase):
|
||||
def gs(self):
|
||||
return storage.Client()
|
||||
|
||||
def get_etag(self, bucket, key):
|
||||
def get_md5(self, bucket, key):
|
||||
import base64
|
||||
|
||||
blob = self.gs.bucket(bucket).get_blob(key)
|
||||
if not blob:
|
||||
return None
|
||||
|
||||
return blob.etag
|
||||
b64_md5 = blob.md5_hash
|
||||
md5 = base64.b64decode(b64_md5)
|
||||
return md5.encode('hex')
|
||||
|
||||
def save_info(self, path_info):
|
||||
if path_info['scheme'] != 'gs':
|
||||
raise NotImplementedError
|
||||
|
||||
return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
|
||||
path_info['key'])}
|
||||
return {self.PARAM_MD5: self.get_md5(path_info['bucket'],
|
||||
path_info['key'])}
|
||||
|
||||
@staticmethod
|
||||
def to_string(path_info):
|
||||
@@ -72,27 +76,28 @@ class RemoteGS(RemoteBase):
|
||||
path_info['bucket'],
|
||||
path_info['key'])
|
||||
|
||||
def changed_cache(self, etag):
|
||||
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
|
||||
def changed_cache(self, md5):
|
||||
key = posixpath.join(self.prefix, md5[0:2], md5[2:])
|
||||
cache = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
|
||||
|
||||
if {self.PARAM_ETAG: etag} != self.save_info(cache):
|
||||
if {self.PARAM_MD5: md5} != self.save_info(cache):
|
||||
if self.exists([cache])[0]:
|
||||
msg = 'Corrupted cache file {}'
|
||||
Logger.warn(msg.format(self.to_string(cache)))
|
||||
self.remove(cache)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def changed(self, path_info, checksum_info):
|
||||
if not self.exists([path_info])[0]:
|
||||
return True
|
||||
|
||||
etag = checksum_info.get(self.PARAM_ETAG, None)
|
||||
if etag is None:
|
||||
md5 = checksum_info.get(self.PARAM_MD5, None)
|
||||
if md5 is None:
|
||||
return True
|
||||
|
||||
if self.changed_cache(etag):
|
||||
if self.changed_cache(md5):
|
||||
return True
|
||||
|
||||
return checksum_info != self.save_info(path_info)
|
||||
@@ -114,20 +119,20 @@ class RemoteGS(RemoteBase):
|
||||
if path_info['scheme'] != 'gs':
|
||||
raise NotImplementedError
|
||||
|
||||
etag = self.get_etag(path_info['bucket'], path_info['key'])
|
||||
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
|
||||
md5 = self.get_md5(path_info['bucket'], path_info['key'])
|
||||
key = posixpath.join(self.prefix, md5[0:2], md5[2:])
|
||||
to_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
|
||||
|
||||
self._copy(path_info, to_info)
|
||||
|
||||
return {self.PARAM_ETAG: etag}
|
||||
return {self.PARAM_MD5: md5}
|
||||
|
||||
def checkout(self, path_info, checksum_info):
|
||||
if path_info['scheme'] != 'gs':
|
||||
raise NotImplementedError
|
||||
|
||||
etag = checksum_info.get(self.PARAM_ETAG, None)
|
||||
if not etag:
|
||||
md5 = checksum_info.get(self.PARAM_MD5, None)
|
||||
if not md5:
|
||||
return
|
||||
|
||||
if not self.changed(path_info, checksum_info):
|
||||
@@ -135,9 +140,9 @@ class RemoteGS(RemoteBase):
|
||||
Logger.info(msg.format(self.to_string(path_info)))
|
||||
return
|
||||
|
||||
if self.changed_cache(etag):
|
||||
if self.changed_cache(md5):
|
||||
msg = "Cache '{}' not found. File '{}' won't be created."
|
||||
Logger.warn(msg.format(etag, self.to_string(path_info)))
|
||||
Logger.warn(msg.format(md5, self.to_string(path_info)))
|
||||
return
|
||||
|
||||
if self.exists([path_info])[0]:
|
||||
@@ -147,9 +152,9 @@ class RemoteGS(RemoteBase):
|
||||
return
|
||||
|
||||
msg = "Checking out '{}' with cache '{}'."
|
||||
Logger.info(msg.format(self.to_string(path_info), etag))
|
||||
Logger.info(msg.format(self.to_string(path_info), md5))
|
||||
|
||||
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
|
||||
key = posixpath.join(self.prefix, md5[0:2], md5[2:])
|
||||
from_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
|
||||
|
||||
self._copy(from_info, path_info)
|
||||
@@ -281,27 +286,27 @@ class RemoteGS(RemoteBase):
|
||||
if not no_progress_bar:
|
||||
progress.finish_target(name)
|
||||
|
||||
def _path_to_etag(self, path):
|
||||
def _path_to_md5(self, path):
|
||||
relpath = posixpath.relpath(path, self.prefix)
|
||||
return posixpath.dirname(relpath) + posixpath.basename(relpath)
|
||||
|
||||
def _all_etags(self):
|
||||
def _all_md5s(self):
|
||||
blobs = self.gs.bucket(self.bucket).list_blobs(prefix=self.prefix)
|
||||
blobs = list(blobs)
|
||||
return [self._path_to_etag(blob.name) for blob in blobs]
|
||||
return [self._path_to_md5(blob.name) for blob in blobs]
|
||||
|
||||
def gc(self, cinfos):
|
||||
used = [info[self.PARAM_ETAG] for info in cinfos['gs']]
|
||||
used = [info[self.PARAM_MD5] for info in cinfos['gs']]
|
||||
used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']]
|
||||
|
||||
removed = False
|
||||
for etag in self._all_etags():
|
||||
if etag in used:
|
||||
for md5 in self._all_md5s():
|
||||
if md5 in used:
|
||||
continue
|
||||
path_info = {'scheme': 'gs',
|
||||
'bucket': self.bucket,
|
||||
'key': posixpath.join(self.prefix,
|
||||
etag[0:2], etag[2:])}
|
||||
md5[0:2], md5[2:])}
|
||||
self.remove(path_info)
|
||||
removed = True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user