mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-08 15:28:26 +01:00
Batuhan Taskaya
d4a3df73df
* s3: migrate to s3fs * s3: strip etag * s3: disable instance cache * tests: use the proper remote:// form in test_checkout * tests: move/drop some tests * setup: depend on s3fs * tests: normalize s3 tests with some cleanups * Use _download/_upload with old implementation * fsspec: cache .exists() results * setup: pin the boto version * setup: install s3fs[boto3] * s3: proper exception management * s3: config improvements * setup: reference without quotes * setup: install boto separately * setup: install moto server * s3: mocking adjustments for moto_server * s3: mark some fixtures with scope=session * s3: s3_fake_creds_file scopes to func * fsspec: add some docs regarding exists() optimization * fsspec: exists() fallback * fsspec: revert exists() optimization * tests: ssl_verify => verify * setup: temporarily fetch from master * s3: add boto as a requirement * tests: do an extra check regarding race conditions * tests: use motoserver docker image * tests: wrap config deletion with try/finally * tests: disable s3 by default * setup: upgrade fsspec
54 lines
1.5 KiB
Python
54 lines
1.5 KiB
Python
import pytest
|
|
|
|
from dvc.fs import get_cloud_fs
|
|
from dvc.fs.gs import GSFileSystem
|
|
from dvc.fs.s3 import S3FileSystem
|
|
|
|
|
|
def test_remote_with_hash_jobs(dvc):
|
|
dvc.config["remote"]["with_hash_jobs"] = {
|
|
"url": "s3://bucket/name",
|
|
"checksum_jobs": 100,
|
|
}
|
|
dvc.config["core"]["checksum_jobs"] = 200
|
|
|
|
fs = get_cloud_fs(dvc, name="with_hash_jobs")
|
|
assert fs.hash_jobs == 100
|
|
|
|
|
|
def test_remote_with_jobs(dvc):
|
|
dvc.config["remote"]["with_jobs"] = {
|
|
"url": "s3://bucket/name",
|
|
"jobs": 100,
|
|
}
|
|
dvc.config["core"]["jobs"] = 200
|
|
|
|
fs = get_cloud_fs(dvc, name="with_jobs")
|
|
assert fs.jobs == 100
|
|
|
|
|
|
def test_remote_without_hash_jobs(dvc):
|
|
dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}
|
|
dvc.config["core"]["checksum_jobs"] = 200
|
|
|
|
fs = get_cloud_fs(dvc, name="without_hash_jobs")
|
|
assert fs.hash_jobs == 200
|
|
|
|
|
|
def test_remote_without_hash_jobs_default(dvc):
|
|
dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}
|
|
|
|
fs = get_cloud_fs(dvc, name="without_hash_jobs")
|
|
assert fs.hash_jobs == fs.HASH_JOBS
|
|
|
|
|
|
@pytest.mark.parametrize("fs_cls", [GSFileSystem, S3FileSystem])
|
|
def test_makedirs_not_create_for_top_level_path(fs_cls, dvc, mocker):
|
|
url = f"{fs_cls.scheme}://bucket/"
|
|
fs = fs_cls(dvc, {"url": url})
|
|
mocked_client = mocker.PropertyMock()
|
|
mocker.patch.object(fs_cls, "fs", mocked_client)
|
|
|
|
fs.makedirs(fs.path_info)
|
|
assert not mocked_client.called
|