Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-07 07:18:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

remote: add additional checks and msgs for checkout

Browse Source 
Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-08-11 21:27:07 +03:00
parent 3814e0da4e
commit 3ec0a5c4ae
7 changed files with 210 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/output/hdfs.py
+1 -1
View File
@@ -30,7 +30,7 @@ class OutputHDFS(DependencyHDFS):
return True
if self.use_cache \
and self.project.cache.hdfs.save_info(self.path_info, self.info):
and self.project.cache.hdfs.changed(self.path_info, self.info):
return True
return False
dvc/output/local.py
-5
View File
@@ -68,11 +68,6 @@ class OutputLOCAL(DependencyLOCAL):
if not self.use_cache:
return
if not self.changed():
msg = u'Data file \'{}\' didn\'t change.'
self.project.logger.info(msg.format(self.rel_path))
return
self.project.cache.local.checkout(self.path_info, self.info)
def _verify_metric(self):
dvc/remote/gs.py
+47
View File
@@ -59,7 +59,35 @@ class RemoteGS(RemoteBase):
return {self.PARAM_ETAG: self.get_etag(path_info['bucket'],
path_info['key'])}
@staticmethod
def to_string(path_info):
return "{}://{}/{}".format(path_info['scheme'],
path_info['bucket'],
path_info['key'])
def changed_cache(self, etag):
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
cache = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
if {self.PARAM_ETAG: etag} != self.save_info(cache):
if self.exists([cache])[0]:
msg = 'Corrupted cache file {}'
Logger.warn(msg.format(self.to_string(cache)))
self.remove(cache)
return True
return False
def changed(self, path_info, checksum_info):
if not self.exists([path_info])[0]:
return True
etag = checksum_info.get(self.PARAM_ETAG, None)
if etag is None:
return True
if self.changed_cache(etag):
return True
return checksum_info != self.save_info(path_info)
def _copy(self, from_info, to_info, gs=None):
@@ -95,6 +123,25 @@ class RemoteGS(RemoteBase):
if not etag:
return
if not self.changed(path_info, checksum_info):
msg = "Data '{}' didn't change."
Logger.info(msg.format(self.to_string(path_info)))
return
if self.changed_cache(etag):
msg = "Cache '{}' not found. File '{}' won't be created."
Logger.warn(msg.format(etag, self.to_string(path_info)))
return
if self.exists([path_info])[0]:
msg = "Data '{}' exists. Removing before checkout."
Logger.warn(msg.format(self.to_string(path_info)))
self.remove(path_info)
return
msg = "Checking out '{}' with cache '{}'."
Logger.info(msg.format(self.to_string(path_info), etag))
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
from_info = {'scheme': 'gs', 'bucket': self.bucket, 'key': key}
dvc/remote/hdfs.py
+49
View File
@@ -72,7 +72,37 @@ class RemoteHDFS(RemoteBase):
return {self.PARAM_CHECKSUM: self.checksum(path_info)}
@staticmethod
def to_string(path_info):
return "{}://{}".format(path_info['scheme'],
path_info['url'])
def changed_cache(self, checksum):
cache = {}
cache['scheme'] = 'hdfs'
cache['user'] = self.user
cache['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
if {self.PARAM_CHECKSUM: checksum} != self.save_info(cache):
if self.exists([cache])[0]:
msg = 'Corrupted cache file {}'
Logger.warn(msg.format(self.to_string(cache)))
self.remove(cache)
return True
return False
def changed(self, path_info, checksum_info):
if not self.exists([path_info])[0]:
return True
checksum = checksum_info.get(self.PARAM_CHECKSUM, None)
if checksum is None:
return True
if self.changed_cache(self, checksum):
return True
return checksum_info != self.save_info(path_info)
def save(self, path_info):
@@ -99,6 +129,25 @@ class RemoteHDFS(RemoteBase):
if not checksum:
return
if not self.changed(path_info, checksum_info):
msg = "Data '{}' didn't change."
Logger.info(msg.format(self.to_string(path_info)))
return
if self.changed_cache(checksum):
msg = "Cache '{}' not found. File '{}' won't be created."
Logger.warn(msg.format(checksum, self.to_string(path_info)))
return
if self.exists([path_info])[0]:
msg = "Data '{}' exists. Removing before checkout."
Logger.warn(msg.format(self.to_string(path_info)))
self.remove(path_info)
return
msg = "Checking out '{}' with cache '{}'."
Logger.info(msg.format(self.to_string(path_info), checksum))
src = path_info.copy()
src['url'] = posixpath.join(self.url, checksum[0:2], checksum[2:])
dvc/remote/local.py
+15
View File
@@ -235,6 +235,11 @@ class RemoteLOCAL(RemoteBase):
Logger.warn(msg.format(os.path.relpath(path)))
return
if not self.changed(path_info, checksum_info):
msg = "Data '{}' didn't change."
Logger.info(msg.format(os.path.relpath(path)))
return
if self.changed_cache(md5):
msg = u'Cache \'{}\' not found. File \'{}\' won\'t be created.'
Logger.warn(msg.format(md5, os.path.relpath(path)))
@@ -334,6 +339,16 @@ class RemoteLOCAL(RemoteBase):
return {self.PARAM_MD5: self.state.update(path_info['path'])}
def changed(self, path_info, checksum_info):
if not self.exists([path_info])[0]:
return True
md5 = checksum_info.get(self.PARAM_MD5, None)
if md5 is None:
return True
if self.changed_cache(md5):
return True
return checksum_info != self.save_info(path_info)
def remove(self, path_info):
dvc/remote/s3.py
+46
View File
@@ -94,6 +94,16 @@ class RemoteS3(RemoteBase):
path_info['key'])}
def changed(self, path_info, checksum_info):
if not self.exists([path_info])[0]:
return True
etag = checksum_info.get(self.PARAM_ETAG, None)
if etag is None:
return True
if self.changed_cache(etag):
return True
return checksum_info != self.save_info(path_info)
def _copy(self, from_info, to_info, s3=None):
@@ -115,6 +125,23 @@ class RemoteS3(RemoteBase):
return {self.PARAM_ETAG: etag}
@staticmethod
def to_string(path_info):
return "s3://{}/{}".format(path_info['bucket'], path_info['key'])
def changed_cache(self, etag):
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
cache = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
if {self.PARAM_ETAG: etag} != self.save_info(cache):
if self.exists([cache])[0]:
msg = 'Corrupted cache file {}'
Logger.warn(msg.format(self.to_string(cache)))
self.remove(cache)
return True
return False
def checkout(self, path_info, checksum_info):
if path_info['scheme'] != 's3':
raise NotImplementedError
@@ -123,6 +150,25 @@ class RemoteS3(RemoteBase):
if not etag:
return
if not self.changed(path_info, checksum_info):
msg = "Data '{}' didn't change."
Logger.info(msg.format(self.to_string(path_info)))
return
if self.changed_cache(etag):
msg = "Cache '{}' not found. File '{}' won't be created."
Logger.warn(msg.format(etag, self.to_string(path_info)))
return
if self.exists([path_info])[0]:
msg = "Data '{}' exists. Removing before checkout."
Logger.warn(msg.format(self.to_string(path_info)))
self.remove(path_info)
return
msg = "Checking out '{}' with cache '{}'."
Logger.info(msg.format(self.to_string(path_info), etag))
key = posixpath.join(self.prefix, etag[0:2], etag[2:])
from_info = {'scheme': 's3', 'bucket': self.bucket, 'key': key}
dvc/remote/ssh.py
+52
View File
@@ -148,7 +148,40 @@ class RemoteSSH(RemoteBase):
return {self.PARAM_MD5: self.md5(path_info)}
@staticmethod
def to_string(path_info):
return "{}://{}@{}:{}".format(path_info['scheme'],
path_info['user'],
path_info['host'],
path_info['path'])
def changed_cache(self, md5):
cache = {}
cache['scheme'] = 'ssh'
cache['host'] = self.host
cache['user'] = self.user
cache['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])
if {self.PARAM_MD5: md5} != self.save_info(cache):
if self.exists([cache])[0]:
msg = 'Corrupted cache file {}'
Logger.warn(msg.format(self.to_string(cache)))
self.remove(cache)
return True
return False
def changed(self, path_info, checksum_info):
if not self.exists([path_info])[0]:
return True
md5 = checksum_info.get(self.PARAM_MD5, None)
if md5 is None:
return True
if self.changed_code(md5):
return True
return checksum_info != self.save_info(path_info)
def save(self, path_info):
@@ -171,6 +204,25 @@ class RemoteSSH(RemoteBase):
if not md5:
return
if not self.changed(path_info, checksum_info):
msg = "Data '{}' didn't change."
Logger.info(msg.format(self.to_string(path_info)))
return
if self.changed_cache(md5):
msg = "Cache '{}' not found. File '{}' won't be created."
Logger.warn(msg.format(md5, self.to_string(path_info)))
return
if self.exists([path_info])[0]:
msg = "Data '{}' exists. Removing before checkout."
Logger.warn(msg.format(self.to_string(path_info)))
self.remove(path_info)
return
msg = "Checking out '{}' with cache '{}'."
Logger.info(msg.format(self.to_string(path_info), md5))
src = path_info.copy()
src['path'] = posixpath.join(self.prefix, md5[0:2], md5[2:])