diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 8ba47c292..66408abc9 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -5,6 +5,7 @@ from datetime import timedelta from functools import wraps import io import os.path +import posixpath from funcy import cached_property @@ -143,9 +144,19 @@ class RemoteGS(RemoteBASE): def list_cache_paths(self): return self._list_paths(self.path_info.bucket, self.path_info.path) + def walk_files(self, path_info): + for fname in self._list_paths(path_info.bucket, path_info.path): + yield path_info / posixpath.relpath(fname, path_info.path) + + def isdir(self, path_info): + dir_path = path_info / "" + file = next(self._list_paths(path_info.bucket, dir_path.path), "") + return file.startswith(dir_path.path) + def exists(self, path_info): - paths = set(self._list_paths(path_info.bucket, path_info.path)) - return any(path_info.path == path for path in paths) + dir_path = path_info / "" + file = next(self._list_paths(path_info.bucket, path_info.path), "") + return path_info.path == file or file.startswith(dir_path.path) def _upload(self, from_file, to_info, name=None, no_progress_bar=True): bucket = self.gs.bucket(to_info.bucket) diff --git a/tests/unit/remote/test_s3.py b/tests/func/test_remote_dir.py similarity index 55% rename from tests/unit/remote/test_s3.py rename to tests/func/test_remote_dir.py index a82d005c4..81fa7b362 100644 --- a/tests/unit/remote/test_s3.py +++ b/tests/func/test_remote_dir.py @@ -1,12 +1,25 @@ # -*- coding: utf-8 -*- import pytest +import uuid + from moto import mock_s3 +from dvc.remote.gs import RemoteGS from dvc.remote.s3 import RemoteS3 +from tests.func.test_data_cloud import _should_test_gcp + +test_gs = pytest.mark.skipif(not _should_test_gcp(), reason="Skipping on gs.") + + +def create_object_gs(client, bucket, key, body): + bucket = client.get_bucket(bucket) + blob = bucket.blob(key) + blob.upload_from_string(body) + @pytest.fixture -def remote(): +def remote_s3(): """Returns a RemoteS3 connected to a bucket with the following structure: bucket @@ -38,7 +51,54 @@ def remote(): yield remote -def test_isdir(remote): +@pytest.fixture +def remote_gs(): + """Returns a RemoteGS connected to a bucket with the following structure: + bucket + ├── data + │ ├── alice + │ ├── alpha + │ └── subdir + │ ├── 1 + │ ├── 2 + │ └── 3 + ├── empty_dir + ├── empty_file + └── foo + """ + prefix = str(uuid.uuid4()) + REMOTE_URL = "gs://dvc-test/" + prefix + remote = RemoteGS(None, {"url": REMOTE_URL}) + teardowns = [] + + def put_object(file, content): + create_object_gs(remote.gs, "dvc-test", prefix + "/" + file, content) + teardowns.append(lambda: remote.remove(remote.path_info / file)) + + put_object("empty_dir/", "") + put_object("empty_file", "") + put_object("foo", "foo") + put_object("data/alice", "alice") + put_object("data/alpha", "alpha") + put_object("data/subdir/1", "1") + put_object("data/subdir/2", "2") + put_object("data/subdir/3", "3") + + yield remote + + for teardown in teardowns: + teardown() + + +remote_parameterized = pytest.mark.parametrize( + "remote_name", [pytest.param("remote_gs", marks=test_gs), "remote_s3"] +) + + +@remote_parameterized +def test_isdir(request, remote_name): + remote = request.getfixturevalue(remote_name) + test_cases = [ (True, "data"), (True, "data/"), @@ -54,7 +114,10 @@ def test_isdir(remote): assert remote.isdir(remote.path_info / path) == expected -def test_exists(remote): +@remote_parameterized +def test_exists(request, remote_name): + remote = request.getfixturevalue(remote_name) + test_cases = [ (True, "data"), (True, "data/"), @@ -72,7 +135,10 @@ def test_exists(remote): assert remote.exists(remote.path_info / path) == expected -def test_walk_files(remote): +@remote_parameterized +def test_walk_files(request, remote_name): + remote = request.getfixturevalue(remote_name) + files = [ remote.path_info / "data/alice", remote.path_info / "data/alpha", @@ -84,16 +150,16 @@ def test_walk_files(remote): assert list(remote.walk_files(remote.path_info / "data")) == files -def test_copy_preserve_etag_across_buckets(remote): - s3 = remote.s3 +def test_copy_preserve_etag_across_buckets(remote_s3): + s3 = remote_s3.s3 s3.create_bucket(Bucket="another") another = RemoteS3(None, {"url": "s3://another", "region": "us-east-1"}) - from_info = remote.path_info / "foo" + from_info = remote_s3.path_info / "foo" to_info = another.path_info / "foo" - remote.copy(from_info, to_info) + remote_s3.copy(from_info, to_info) from_etag = RemoteS3.get_etag(s3, "bucket", "foo") to_etag = RemoteS3.get_etag(s3, "another", "foo")