fsspec: support raw buckets on path splitting (#5658)

This commit is contained in:
Batuhan Taskaya
2021-03-22 02:01:06 +02:00
committed by GitHub
parent f2b216261e
commit 21fc06e947
2 changed files with 28 additions and 2 deletions
+8 -2
View File
@@ -19,8 +19,14 @@ class FSSpecWrapper(BaseFileSystem):
return path
def _strip_bucket(self, entry):
_, entry = entry.split("/", 1)
return entry
try:
bucket, path = entry.split("/", 1)
except ValueError:
# If there is no path attached, only returns
# the bucket (top-level).
bucket, path = entry, None
return path or bucket
def _strip_buckets(self, entries, detail, prefix=None):
for entry in entries:
+20
View File
@@ -349,3 +349,23 @@ def test_fs_ls_with_etag(dvc, cloud):
fs.info(path_info.replace(path=details["name"]))["etag"]
== details["etag"]
)
@pytest.mark.parametrize(
"cloud", [pytest.lazy_fixture("azure"), pytest.lazy_fixture("gs")]
)
def test_fs_fsspec_path_management(dvc, cloud):
cloud.gen({"foo": "foo", "data": {"bar": "bar", "baz": {"foo": "foo"}}})
fs = get_cloud_fs(dvc, **cloud.config)
root = cloud.parents[len(cloud.parents) - 1]
bucket_details = fs.info(root)
# special conditions: name always points to the bucket name
assert bucket_details["name"] == root.bucket
assert bucket_details["type"] == "directory"
data = cloud / "data"
data_details = fs.info(data)
assert data_details["name"].rstrip("/") == data.path
assert data_details["type"] == "directory"