outs: support gs/s3 as an output without cache

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-06-04 22:28:14 +03:00
parent 2e78bb2a69
commit e0a42f1265
3 changed files with 179 additions and 38 deletions
+52 -34
View File
@@ -186,56 +186,74 @@ class TestCmdRepro(TestRepro):
self.assertNotEqual(ret, 0)
class TestReproDependencyS3(TestDvc):
class TestReproExternalBase(TestDvc):
def should_test(self):
return False
def test(self):
if not _should_test_aws():
if not self.should_test():
return
key = str(uuid.uuid4()) + '/' + self.FOO
path = 's3://' + TEST_AWS_REPO_BUCKET + '/' + key
foo_key = str(uuid.uuid4()) + '/' + self.FOO
bar_key = str(uuid.uuid4()) + '/' + self.BAR
s3 = boto3.resource('s3')
foo_path = self.schema + self.bucket + '/' + foo_key
bar_path = self.schema + self.bucket + '/' + bar_key
with open(self.FOO, 'rb') as fd:
s3.Bucket(TEST_AWS_REPO_BUCKET).put_object(Key=key, Body=fd.read())
self.write(self.bucket, foo_key, 'foo')
stage = self.dvc.run(outs=[self.FOO],
deps=[path],
cmd='aws s3 cp {} {}'.format(path, self.FOO))
stage = self.dvc.run(outs_no_cache=[bar_path],
deps=[foo_path],
cmd='{} {} {}'.format(self.cmd, foo_path, bar_path))
with open(self.BAR, 'rb') as fd:
s3.Bucket(TEST_AWS_REPO_BUCKET).put_object(Key=key, Body=fd.read())
self.write(self.bucket, foo_key, 'bar')
stages = self.dvc.reproduce(stage.path)
self.assertEqual(len(stages), 1)
self.assertTrue(filecmp.cmp(self.FOO, self.BAR))
class TestReproDependencyGS(TestDvc):
def test(self):
if not _should_test_gcp():
return
class TestReproExternalS3(TestReproExternalBase):
def should_test(self):
return _should_test_aws()
@property
def schema(self):
return 's3://'
@property
def bucket(self):
return TEST_AWS_REPO_BUCKET
@property
def cmd(self):
return 'aws s3 cp'
def write(self, bucket, key, body):
s3 = boto3.resource('s3')
s3.Bucket(bucket).put_object(Key=key, Body=body)
class TestReproExternalGS(TestReproExternalBase):
def should_test(self):
#FIXME enable
if os.getenv('CI'):
return
return False
return _should_test_gcp()
key = str(uuid.uuid4()) + '/' + self.FOO
path = 'gs://' + TEST_GCP_REPO_BUCKET + '/' + key
@property
def schema(self):
return 'gs://'
@property
def bucket(self):
return TEST_GCP_REPO_BUCKET
@property
def cmd(self):
return 'gsutil cp'
def write(self, bucket, key, body):
client = gc.Client()
bucket = client.bucket(TEST_GCP_REPO_BUCKET)
bucket.blob(key).upload_from_filename(self.FOO)
stage = self.dvc.run(outs=[self.FOO],
deps=[path],
cmd='gsutil cp {} {}'.format(path, self.FOO))
bucket.blob(key).upload_from_filename(self.BAR)
stages = self.dvc.reproduce(stage.path)
self.assertEqual(len(stages), 1)
self.assertTrue(filecmp.cmp(self.FOO, self.BAR))
bucket = client.bucket(bucket)
bucket.blob(key).upload_from_string(body)