diff --git a/dvc/dependency/gs.py b/dvc/dependency/gs.py index 448d6ef8d..e297f452e 100644 --- a/dvc/dependency/gs.py +++ b/dvc/dependency/gs.py @@ -12,12 +12,14 @@ from dvc.cloud.gcp import DataCloudGCP class DependencyGS(DependencyS3): REGEX = DataCloudGCP.REGEX + @property + def client(self): + return gc.Client() + def get_etag(self): - o = urlparse(self.path) - bucket_name = o.netloc - key = o.path.lstrip('/') + bucket = self.client.bucket(self.bucket) + blob = bucket.get_blob(self.key) + if not blob: + return None - client = gc.Client() - bucket = client.bucket(bucket_name) - blob = bucket.get_blob(key) return blob.etag diff --git a/dvc/dependency/s3.py b/dvc/dependency/s3.py index 9b8bfb971..19ebe0cfe 100644 --- a/dvc/dependency/s3.py +++ b/dvc/dependency/s3.py @@ -18,14 +18,25 @@ class DependencyS3(DependencyBase): super(DependencyS3, self).__init__(stage, path) self.etag = etag - def get_etag(self): - o = urlparse(self.path) - bucket = o.netloc - key = o.path.lstrip('/') + @property + def bucket(self): + return urlparse(self.path).netloc + @property + def key(self): + return urlparse(self.path).path.lstrip('/') + + @property + def s3(self): session = boto3.Session() - s3 = session.resource('s3') - obj = s3.Object(bucket, key).get() + return session.resource('s3') + + def get_etag(self): + try: + obj = self.s3.Object(self.bucket, self.key).get() + except Exception: + return None + return obj['ETag'].strip('"') def changed(self):