deps: introduce s3/gs dependencies

E.g.:
dvc run -d gs://bucket/file -o file gsutil cp gs://bucket/file file
dvc run -d s3://bucket/file -o file aws s3 cp s3://bucket/file file

Etags are used to track deps in both cases.

Signed-off-by: Ruslan Kuprieiev <[email protected]>

tests/test_add.py

@@ -6,8 +6,8 @@ import filecmp
 from dvc.main import main
 from dvc.utils import file_md5
 from dvc.stage import Stage
-from dvc.output import OutputOutsideOfRepoError, OutputAlreadyTrackedError
-from dvc.output import OutputDoesNotExistError, OutputIsNotFileOrDirError
+from dvc.output.base import OutputAlreadyTrackedError
+from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError
 from dvc.project import StageNotFoundError
 from dvc.command.add import CmdAdd
 
@@ -34,12 +34,6 @@ class TestAddNonExistentFile(TestDvc):
             self.dvc.add('non_existent_file')
 
 
-class TestAddFileOutsideOfRepo(TestDvc):
-    def test(self):
-        with self.assertRaises(OutputOutsideOfRepoError) as cx:
-            self.dvc.add(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO))
-
-
 class TestAddDirectory(TestDvc):
     def test(self):
         dname = 'directory'

tests/test_checkout.py

@@ -9,7 +9,7 @@ from dvc.main import main
 from tests.basic_env import TestDvc
 from tests.test_repro import TestRepro
 from dvc.stage import Stage
-from dvc.output import Output
+from dvc.output.local import OutputLOCAL
 
 
 class TestCheckout(TestRepro):
@@ -163,8 +163,8 @@ class TestCheckoutMissingMd5InStageFile(TestRepro):
         with open(self.file1_stage, 'r') as fd:
             d = yaml.load(fd)
 
-        del(d[Stage.PARAM_OUTS][0][Output.PARAM_MD5])
-        del(d[Stage.PARAM_DEPS][0][Output.PARAM_MD5])
+        del(d[Stage.PARAM_OUTS][0][OutputLOCAL.PARAM_MD5])
+        del(d[Stage.PARAM_DEPS][0][OutputLOCAL.PARAM_MD5])
 
         with open(self.file1_stage, 'w') as fd:
             yaml.dump(d, fd)

tests/test_repro.py

@@ -4,14 +4,20 @@ import stat
 import shutil
 import filecmp
 
+import boto3
+import uuid
+from google.cloud import storage as gc
+
 from dvc.main import main
 from dvc.command.repro import CmdRepro
 from dvc.project import ReproductionError
 from dvc.utils import file_md5
-from dvc.output import Output
+from dvc.output.local import OutputLOCAL
 from dvc.stage import Stage
 
 from tests.basic_env import TestDvc
+from tests.test_data_cloud import _should_test_aws, TEST_AWS_REPO_BUCKET
+from tests.test_data_cloud import _should_test_gcp, TEST_GCP_REPO_BUCKET
 
 
 class TestRepro(TestDvc):
@@ -159,8 +165,8 @@ class TestReproMissingMd5InStageFile(TestRepro):
         with open(self.file1_stage, 'r') as fd:
             d = yaml.load(fd)
 
-        del(d[Stage.PARAM_OUTS][0][Output.PARAM_MD5])
-        del(d[Stage.PARAM_DEPS][0][Output.PARAM_MD5])
+        del(d[Stage.PARAM_OUTS][0][OutputLOCAL.PARAM_MD5])
+        del(d[Stage.PARAM_DEPS][0][OutputLOCAL.PARAM_MD5])
 
         with open(self.file1_stage, 'w') as fd:
             yaml.dump(d, fd)
@@ -178,3 +184,54 @@ class TestCmdRepro(TestRepro):
         ret = main(['repro',
                     'non-existing-file'])
         self.assertNotEqual(ret, 0)
+
+
+class TestReproDependencyS3(TestDvc):
+    def test(self):
+        if not _should_test_aws():
+            return
+
+        key = str(uuid.uuid4()) + '/' + self.FOO
+        path = 's3://' + TEST_AWS_REPO_BUCKET + '/' + key
+
+        s3 = boto3.resource('s3')
+
+        with open(self.FOO, 'rb') as fd:
+            s3.Bucket(TEST_AWS_REPO_BUCKET).put_object(Key=key, Body=fd.read())
+
+        stage = self.dvc.run(outs=[self.FOO],
+                             deps=[path],
+                             cmd='aws s3 cp {} {}'.format(path, self.FOO))
+
+        with open(self.BAR, 'rb') as fd:
+            s3.Bucket(TEST_AWS_REPO_BUCKET).put_object(Key=key, Body=fd.read())
+
+        stages = self.dvc.reproduce(stage.path)
+
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(filecmp.cmp(self.FOO, self.BAR))
+
+
+class TestReproDependencyGS(TestDvc):
+    def test(self):
+        if not _should_test_gcp():
+            return
+
+        key = str(uuid.uuid4()) + '/' + self.FOO
+        path = 'gs://' + TEST_GCP_REPO_BUCKET + '/' + key
+
+        client = gc.Client()
+        bucket = client.bucket(TEST_GCP_REPO_BUCKET)
+
+        bucket.blob(key).upload_from_filename(self.FOO)
+
+        stage = self.dvc.run(outs=[self.FOO],
+                             deps=[path],
+                             cmd='gsutil cp {} {}'.format(path, self.FOO))
+
+        bucket.blob(key).upload_from_filename(self.BAR)
+
+        stages = self.dvc.reproduce(stage.path)
+
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(filecmp.cmp(self.FOO, self.BAR))

tests/test_stage.py

@@ -1,4 +1,4 @@
-from dvc.output import Output
+from dvc.output.local import OutputLOCAL
 from dvc.stage import Stage, StageFileFormatError
 
 from tests.basic_env import TestDvc
@@ -51,13 +51,13 @@ class TestSchemaDepsOuts(TestSchema):
         Stage.validate(d)
 
     def test_list(self):
-        l = [{Output.PARAM_PATH: 'foo', Output.PARAM_MD5: '123'},
-             {Output.PARAM_PATH: 'bar', Output.PARAM_MD5: None},
-             {Output.PARAM_PATH: 'baz'}]
+        l = [{OutputLOCAL.PARAM_PATH: 'foo', OutputLOCAL.PARAM_MD5: '123'},
+             {OutputLOCAL.PARAM_PATH: 'bar', OutputLOCAL.PARAM_MD5: None},
+             {OutputLOCAL.PARAM_PATH: 'baz'}]
         d = {Stage.PARAM_DEPS: l}
         Stage.validate(d)
 
-        l[0][Output.PARAM_CACHE] = True
-        l[1][Output.PARAM_CACHE] = False
+        l[0][OutputLOCAL.PARAM_CACHE] = True
+        l[1][OutputLOCAL.PARAM_CACHE] = False
         d = {Stage.PARAM_OUTS: l}
         Stage.validate(d)