mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
tests: introduce dvc.testing
This is a very early version of test utils for dvc plugins. This module consists of existing tests with minimal changes and is going to evolve a lot as our plugins are getting filled in and pytest-test-utils matures.
This commit is contained in:
+16
-105
@@ -9,120 +9,31 @@ import dvc as dvc_module
|
||||
from dvc.external_repo import clean_repos
|
||||
from dvc.main import main
|
||||
from dvc.stage.exceptions import StageNotFound
|
||||
from dvc.utils.fs import move, remove
|
||||
from dvc.testing.test_remote import TestRemote as _TestRemote
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
all_clouds = [
|
||||
pytest.lazy_fixture(cloud)
|
||||
for cloud in [
|
||||
"s3",
|
||||
"gs",
|
||||
"azure",
|
||||
"ssh",
|
||||
"http",
|
||||
"hdfs",
|
||||
"webdav",
|
||||
"webhdfs",
|
||||
"oss",
|
||||
"gdrive",
|
||||
]
|
||||
"s3",
|
||||
"gs",
|
||||
"azure",
|
||||
"ssh",
|
||||
"http",
|
||||
"hdfs",
|
||||
"webdav",
|
||||
"webhdfs",
|
||||
"oss",
|
||||
"gdrive",
|
||||
]
|
||||
|
||||
# Clouds that implement the general methods that can be tested
|
||||
# for functional tests that require extensive apis (e.g traversing
|
||||
# via walk_files())
|
||||
full_clouds = [
|
||||
pytest.lazy_fixture(cloud)
|
||||
for cloud in ["s3", "gs", "azure", "ssh", "hdfs"]
|
||||
]
|
||||
full_clouds = ["s3", "gs", "azure", "ssh", "hdfs"]
|
||||
|
||||
|
||||
@pytest.mark.needs_internet
|
||||
@pytest.mark.parametrize("remote", all_clouds, indirect=True)
|
||||
def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
out = stage.outs[0]
|
||||
cache = out.cache_path
|
||||
foo_hash = out.hash_info
|
||||
foo_hashes = out.get_used_objs().get(None, set())
|
||||
|
||||
(stage_dir,) = tmp_dir.dvc_gen(
|
||||
{
|
||||
"data_dir": {
|
||||
"data_sub_dir": {"data_sub": "data_sub"},
|
||||
"data": "data",
|
||||
"empty": "",
|
||||
}
|
||||
}
|
||||
)
|
||||
out_dir = stage_dir.outs[0]
|
||||
cache_dir = out_dir.cache_path
|
||||
dir_hash = out_dir.hash_info
|
||||
dir_hashes = {dir_hash} | {oid for _, _, oid in out_dir.obj}
|
||||
|
||||
def _check_status(status, **kwargs):
|
||||
for key in ("ok", "missing", "new", "deleted"):
|
||||
expected = kwargs.get(key, set())
|
||||
assert expected == set(getattr(status, key))
|
||||
|
||||
# Check status
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
_check_status(status, new={foo_hash})
|
||||
|
||||
status_dir = dvc.cloud.status(dir_hashes)
|
||||
_check_status(status_dir, new=dir_hashes)
|
||||
|
||||
# Move cache and check status
|
||||
# See issue https://github.com/iterative/dvc/issues/4383 for details
|
||||
backup_dir = dvc.odb.local.cache_dir + ".backup"
|
||||
move(dvc.odb.local.cache_dir, backup_dir)
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
_check_status(status, missing={foo_hash})
|
||||
|
||||
status_dir = dvc.cloud.status(dir_hashes)
|
||||
_check_status(status_dir, missing=dir_hashes)
|
||||
|
||||
# Restore original cache:
|
||||
remove(dvc.odb.local.cache_dir)
|
||||
move(backup_dir, dvc.odb.local.cache_dir)
|
||||
|
||||
# Push and check status
|
||||
dvc.cloud.push(foo_hashes)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
|
||||
dvc.cloud.push(dir_hashes)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
_check_status(status, ok={foo_hash})
|
||||
|
||||
status_dir = dvc.cloud.status(dir_hashes)
|
||||
_check_status(status_dir, ok=dir_hashes)
|
||||
|
||||
# Remove and check status
|
||||
remove(dvc.odb.local.cache_dir)
|
||||
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
_check_status(status, deleted={foo_hash})
|
||||
|
||||
status_dir = dvc.cloud.status(dir_hashes)
|
||||
_check_status(status_dir, deleted=dir_hashes)
|
||||
|
||||
# Pull and check status
|
||||
dvc.cloud.pull(foo_hashes)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
with open(cache, encoding="utf-8") as fd:
|
||||
assert fd.read() == "foo"
|
||||
|
||||
dvc.cloud.pull(dir_hashes)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
_check_status(status, ok={foo_hash})
|
||||
|
||||
status_dir = dvc.cloud.status(dir_hashes)
|
||||
_check_status(status_dir, ok=dir_hashes)
|
||||
class TestRemote(_TestRemote):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.needs_internet
|
||||
@@ -448,7 +359,7 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote):
|
||||
assert set(dvc.pull()["added"]) == set(outs)
|
||||
|
||||
# clean everything in remote and push
|
||||
from tests.dir_helpers import TmpDir
|
||||
from dvc.testing.tmp_dir import TmpDir
|
||||
|
||||
clean(TmpDir(path).iterdir())
|
||||
dvc.push(["dvc.yaml:copy-ipsum-baz"])
|
||||
|
||||
Reference in New Issue
Block a user