Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-12 07:18:23 +01:00
Code Issues Packages Projects Releases Wiki Activity

azure: support directory imports with azure (#6245)

Browse Source 
* azure: support directory imports with azure

* upgrade azurite to 3.13

* tmp: use azurite 3.12?

* upgrade to azurite 3.13.1

* bump adlfs to 2021.7.0
This commit is contained in:
Batuhan Taskaya
2021-07-13 13:39:30 +03:00
committed by GitHub
parent f45311e658
commit 58f0911234
3 changed files with 24 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
setup.py
+1 -1
View File
@@ -99,7 +99,7 @@ install_requires = [
gs = ["gcsfs==2021.6.1"]
gdrive = ["pydrive2>=1.8.1", "six >= 1.13.0"]
s3 = ["s3fs==2021.6.1", "aiobotocore[boto3]==1.3.0"]
azure = ["adlfs==0.7.1", "azure-identity>=1.4.0", "knack"]
azure = ["adlfs==2021.7.0", "azure-identity>=1.4.0", "knack"]
# https://github.com/Legrandin/pycryptodome/issues/465
oss = ["oss2==2.6.1", "pycryptodome>=3.10"]
ssh = ["paramiko[invoke]>=2.7.0"]
tests/docker-compose.yml
+1 -1
View File
@@ -2,7 +2,7 @@
version: '3.2'
services:
azurite:
image: mcr.microsoft.com/azure-storage/azurite:3.10.0
image: mcr.microsoft.com/azure-storage/azurite:3.13.1
command: azurite -L -l /data --blobHost 0.0.0.0 --queueHost 0.0.0.0
ports:
- "10000"
tests/func/test_import_url.py
+22 -15
View File
@@ -161,6 +161,12 @@ def test_import_url(tmp_dir, dvc, workspace):
"2aa17f8daa26996b3f7a4cf8888ac9ac",
"ec602a6ba97b2dd07bd6d2cd89674a60.dir",
),
(
pytest.lazy_fixture("azure"),
None, # ETags for azure are not consistent with the actual
None, # content of the object, so they will change every time
# we create new objects. We'll skip stage content check for Azure.
),
(
pytest.lazy_fixture("hdfs"),
"ec0943f83357f702033c98e70b853c8c",
@@ -192,23 +198,24 @@ def test_import_url_dir(tmp_dir, dvc, workspace, stage_md5, dir_md5):
assert list(os.listdir(tmp_dir / "dir" / "subdir")) == ["subfile"]
assert (tmp_dir / "dir" / "subdir" / "subfile").read_text() == "subfile"
assert (tmp_dir / "dir.dvc").read_text() == (
f"md5: {stage_md5}\n"
"frozen: true\n"
"deps:\n"
f"- md5: {dir_md5}\n"
" size: 11\n"
" nfiles: 2\n"
" path: remote://workspace/dir\n"
"outs:\n"
"- md5: b6dcab6ccd17ca0a8bf4a215a37d14cc.dir\n"
" size: 11\n"
" nfiles: 2\n"
" path: dir\n"
)
assert dvc.status() == {}
if stage_md5 is not None and dir_md5 is not None:
assert (tmp_dir / "dir.dvc").read_text() == (
f"md5: {stage_md5}\n"
"frozen: true\n"
"deps:\n"
f"- md5: {dir_md5}\n"
" size: 11\n"
" nfiles: 2\n"
" path: remote://workspace/dir\n"
"outs:\n"
"- md5: b6dcab6ccd17ca0a8bf4a215a37d14cc.dir\n"
" size: 11\n"
" nfiles: 2\n"
" path: dir\n"
)
def test_import_url_preserve_meta(tmp_dir, dvc):
text = textwrap.dedent(