remote/gs: support adding directories

dvc/remote/gs.py

@@ -5,6 +5,7 @@ from datetime import timedelta
 from functools import wraps
 import io
 import os.path
+import posixpath
 
 from funcy import cached_property
 
@@ -143,9 +144,19 @@ class RemoteGS(RemoteBASE):
     def list_cache_paths(self):
         return self._list_paths(self.path_info.bucket, self.path_info.path)
 
+    def walk_files(self, path_info):
+        for fname in self._list_paths(path_info.bucket, path_info.path):
+            yield path_info / posixpath.relpath(fname, path_info.path)
+
+    def isdir(self, path_info):
+        dir_path = path_info / ""
+        file = next(self._list_paths(path_info.bucket, dir_path.path), "")
+        return file.startswith(dir_path.path)
+
     def exists(self, path_info):
-        paths = set(self._list_paths(path_info.bucket, path_info.path))
-        return any(path_info.path == path for path in paths)
+        dir_path = path_info / ""
+        file = next(self._list_paths(path_info.bucket, path_info.path), "")
+        return path_info.path == file or file.startswith(dir_path.path)
 
     def _upload(self, from_file, to_info, name=None, no_progress_bar=True):
         bucket = self.gs.bucket(to_info.bucket)

tests/func/test_remote_dir.py

@@ -1,12 +1,25 @@
 # -*- coding: utf-8 -*-
 import pytest
+import uuid
+
 from moto import mock_s3
 
+from dvc.remote.gs import RemoteGS
 from dvc.remote.s3 import RemoteS3
 
+from tests.func.test_data_cloud import _should_test_gcp
+
+test_gs = pytest.mark.skipif(not _should_test_gcp(), reason="Skipping on gs.")
+
+
+def create_object_gs(client, bucket, key, body):
+    bucket = client.get_bucket(bucket)
+    blob = bucket.blob(key)
+    blob.upload_from_string(body)
+
 
 @pytest.fixture
-def remote():
+def remote_s3():
     """Returns a RemoteS3 connected to a bucket with the following structure:
 
         bucket
@@ -38,7 +51,54 @@ def remote():
         yield remote
 
 
-def test_isdir(remote):
+@pytest.fixture
+def remote_gs():
+    """Returns a RemoteGS connected to a bucket with the following structure:
+        bucket
+        ├── data
+        │  ├── alice
+        │  ├── alpha
+        │  └── subdir
+        │     ├── 1
+        │     ├── 2
+        │     └── 3
+        ├── empty_dir
+        ├── empty_file
+        └── foo
+    """
+    prefix = str(uuid.uuid4())
+    REMOTE_URL = "gs://dvc-test/" + prefix
+    remote = RemoteGS(None, {"url": REMOTE_URL})
+    teardowns = []
+
+    def put_object(file, content):
+        create_object_gs(remote.gs, "dvc-test", prefix + "/" + file, content)
+        teardowns.append(lambda: remote.remove(remote.path_info / file))
+
+    put_object("empty_dir/", "")
+    put_object("empty_file", "")
+    put_object("foo", "foo")
+    put_object("data/alice", "alice")
+    put_object("data/alpha", "alpha")
+    put_object("data/subdir/1", "1")
+    put_object("data/subdir/2", "2")
+    put_object("data/subdir/3", "3")
+
+    yield remote
+
+    for teardown in teardowns:
+        teardown()
+
+
+remote_parameterized = pytest.mark.parametrize(
+    "remote_name", [pytest.param("remote_gs", marks=test_gs), "remote_s3"]
+)
+
+
+@remote_parameterized
+def test_isdir(request, remote_name):
+    remote = request.getfixturevalue(remote_name)
+
     test_cases = [
         (True, "data"),
         (True, "data/"),
@@ -54,7 +114,10 @@ def test_isdir(remote):
         assert remote.isdir(remote.path_info / path) == expected
 
 
-def test_exists(remote):
+@remote_parameterized
+def test_exists(request, remote_name):
+    remote = request.getfixturevalue(remote_name)
+
     test_cases = [
         (True, "data"),
         (True, "data/"),
@@ -72,7 +135,10 @@ def test_exists(remote):
         assert remote.exists(remote.path_info / path) == expected
 
 
-def test_walk_files(remote):
+@remote_parameterized
+def test_walk_files(request, remote_name):
+    remote = request.getfixturevalue(remote_name)
+
     files = [
         remote.path_info / "data/alice",
         remote.path_info / "data/alpha",
@@ -84,16 +150,16 @@ def test_walk_files(remote):
     assert list(remote.walk_files(remote.path_info / "data")) == files
 
 
-def test_copy_preserve_etag_across_buckets(remote):
-    s3 = remote.s3
+def test_copy_preserve_etag_across_buckets(remote_s3):
+    s3 = remote_s3.s3
     s3.create_bucket(Bucket="another")
 
     another = RemoteS3(None, {"url": "s3://another", "region": "us-east-1"})
 
-    from_info = remote.path_info / "foo"
+    from_info = remote_s3.path_info / "foo"
     to_info = another.path_info / "foo"
 
-    remote.copy(from_info, to_info)
+    remote_s3.copy(from_info, to_info)
 
     from_etag = RemoteS3.get_etag(s3, "bucket", "foo")
     to_etag = RemoteS3.get_etag(s3, "another", "foo")