mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-17 07:18:26 +01:00
tests: cover run cache with functional tests (#4018)
* tests: cover run cache with functional tests * run-cache: use _log_exceptions wrapper Co-authored-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
co-authored by
Ruslan Kuprieiev
parent
d7788887f0
commit
2e12a31c5a
+1
-1
@@ -72,7 +72,7 @@ def _fetch(
|
||||
if failed:
|
||||
raise DownloadError(failed)
|
||||
|
||||
return downloaded
|
||||
return downloaded + len(used_run_cache)
|
||||
|
||||
|
||||
def _fetch_external(self, repo_url, repo_rev, files, jobs):
|
||||
|
||||
+1
-1
@@ -32,4 +32,4 @@ def push(
|
||||
used_run_cache=used_run_cache,
|
||||
)
|
||||
|
||||
return self.cloud.push(used, jobs, remote=remote)
|
||||
return len(used_run_cache) + self.cloud.push(used, jobs, remote=remote)
|
||||
|
||||
@@ -63,6 +63,7 @@ def reproduce(
|
||||
):
|
||||
from dvc.utils import parse_target
|
||||
|
||||
assert target is None or isinstance(target, str)
|
||||
if not target and not all_pipelines:
|
||||
raise InvalidArgumentError(
|
||||
"Neither `target` nor `--all-pipelines` are specified."
|
||||
|
||||
+11
-2
@@ -6,6 +6,7 @@ import yaml
|
||||
from funcy import first
|
||||
from voluptuous import Invalid
|
||||
|
||||
from dvc.remote.local import _log_exceptions
|
||||
from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA
|
||||
from dvc.utils import dict_sha256, relpath
|
||||
from dvc.utils.fs import makedirs
|
||||
@@ -177,11 +178,19 @@ class StageCache:
|
||||
|
||||
def push(self, remote):
|
||||
remote = self.repo.cloud.get_remote(remote)
|
||||
return self._transfer(remote.upload, self.repo.cache.local, remote)
|
||||
return self._transfer(
|
||||
_log_exceptions(remote.tree.upload, "upload"),
|
||||
self.repo.cache.local.tree,
|
||||
remote.tree,
|
||||
)
|
||||
|
||||
def pull(self, remote):
|
||||
remote = self.repo.cloud.get_remote(remote)
|
||||
return self._transfer(remote.download, remote, self.repo.cache.local)
|
||||
return self._transfer(
|
||||
_log_exceptions(remote.tree.download, "download"),
|
||||
remote.tree,
|
||||
self.repo.cache.local.tree,
|
||||
)
|
||||
|
||||
def get_used_cache(self, used_run_cache, *args, **kwargs):
|
||||
from dvc.cache import NamedCache
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import os
|
||||
|
||||
from dvc.dvcfile import PIPELINE_LOCK
|
||||
from dvc.utils import relpath
|
||||
|
||||
|
||||
def _recurse_count_files(path):
|
||||
return len([os.path.join(r, f) for r, _, fs in os.walk(path) for f in fs])
|
||||
|
||||
|
||||
def test_push_pull(tmp_dir, dvc, erepo_dir, run_copy, setup_remote):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
url = setup_remote(dvc)
|
||||
assert dvc.push(run_cache=True) == 2
|
||||
setup_remote(erepo_dir.dvc, url)
|
||||
with erepo_dir.chdir():
|
||||
assert not os.path.exists(erepo_dir.dvc.stage_cache.cache_dir)
|
||||
assert erepo_dir.dvc.pull(run_cache=True)["fetched"] == 2
|
||||
assert os.listdir(erepo_dir.dvc.stage_cache.cache_dir)
|
||||
|
||||
|
||||
def test_restore(tmp_dir, dvc, run_copy, mocker):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
|
||||
mock_restore = mocker.spy(dvc.stage_cache, "restore")
|
||||
mock_run = mocker.patch("dvc.stage.run.cmd_run")
|
||||
|
||||
# removing any information that `dvc` could use to re-generate from
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / PIPELINE_LOCK).unlink()
|
||||
|
||||
(stage,) = dvc.reproduce("copy-foo-bar")
|
||||
|
||||
mock_restore.assert_called_once_with(stage)
|
||||
mock_run.assert_not_called()
|
||||
assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()
|
||||
assert (tmp_dir / PIPELINE_LOCK).exists()
|
||||
|
||||
|
||||
def test_save(tmp_dir, dvc, run_copy):
|
||||
run_cache_dir = dvc.stage_cache.cache_dir
|
||||
assert not os.path.exists(run_cache_dir)
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
assert _recurse_count_files(run_cache_dir) == 1
|
||||
with dvc.state:
|
||||
assert dvc.stage_cache.is_cached(stage)
|
||||
|
||||
|
||||
def test_do_not_save_on_no_exec_and_dry(tmp_dir, dvc, run_copy):
|
||||
run_cache_dir = dvc.stage_cache.cache_dir
|
||||
assert not os.path.exists(run_cache_dir)
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", name="copy-foo-bar", no_exec=True)
|
||||
|
||||
assert _recurse_count_files(run_cache_dir) == 0
|
||||
with dvc.state:
|
||||
assert not dvc.stage_cache.is_cached(stage)
|
||||
|
||||
(stage,) = dvc.reproduce("copy-foo-bar", dry=True)
|
||||
|
||||
assert _recurse_count_files(run_cache_dir) == 0
|
||||
with dvc.state:
|
||||
assert not dvc.stage_cache.is_cached(stage)
|
||||
|
||||
|
||||
def test_uncached_outs_are_cached(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = dvc.run(
|
||||
deps=["foo"],
|
||||
cmd="cp foo bar",
|
||||
outs_no_cache=["bar"],
|
||||
name="copy-foo-bar",
|
||||
)
|
||||
with dvc.state:
|
||||
stage.outs[0].checksum = stage.outs[0].get_checksum()
|
||||
assert os.path.exists(relpath(stage.outs[0].cache_path))
|
||||
|
||||
|
||||
def test_memory_for_multiple_runs_of_same_stage(
|
||||
tmp_dir, dvc, run_copy, mocker
|
||||
):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
assert not os.path.exists(dvc.stage_cache.cache_dir)
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
assert _recurse_count_files(dvc.stage_cache.cache_dir) == 1
|
||||
tmp_dir.gen("foo", "foobar")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
assert _recurse_count_files(dvc.stage_cache.cache_dir) == 2
|
||||
|
||||
from dvc.stage import run as _run
|
||||
|
||||
mock_restore = mocker.spy(dvc.stage_cache, "restore")
|
||||
mock_run = mocker.spy(_run, "cmd_run")
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / PIPELINE_LOCK).unlink()
|
||||
(stage,) = dvc.reproduce("copy-foo-bar")
|
||||
|
||||
assert (tmp_dir / PIPELINE_LOCK).exists()
|
||||
assert (tmp_dir / "bar").read_text() == "foobar"
|
||||
mock_run.assert_not_called()
|
||||
mock_restore.assert_called_once_with(stage)
|
||||
mock_restore.reset_mock()
|
||||
|
||||
(tmp_dir / PIPELINE_LOCK).unlink()
|
||||
tmp_dir.gen("foo", "foo")
|
||||
dvc.reproduce("copy-foo-bar")
|
||||
|
||||
assert (tmp_dir / "bar").read_text() == "foo"
|
||||
mock_run.assert_not_called()
|
||||
mock_restore.assert_called_once_with(stage)
|
||||
assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()
|
||||
assert (tmp_dir / PIPELINE_LOCK).exists()
|
||||
|
||||
|
||||
def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
assert not os.path.exists(dvc.stage_cache.cache_dir)
|
||||
|
||||
run_copy("foo", "foo.bak", name="backup-foo")
|
||||
assert _recurse_count_files(dvc.stage_cache.cache_dir) == 1
|
||||
|
||||
tmp_dir.gen("bar", "bar")
|
||||
run_copy("bar", "bar.bak", name="backup-bar")
|
||||
assert _recurse_count_files(dvc.stage_cache.cache_dir) == 2
|
||||
|
||||
from dvc.stage import run as _run
|
||||
|
||||
mock_restore = mocker.spy(dvc.stage_cache, "restore")
|
||||
mock_run = mocker.spy(_run, "cmd_run")
|
||||
|
||||
(tmp_dir / "foo.bak").unlink()
|
||||
(tmp_dir / "bar.bak").unlink()
|
||||
(tmp_dir / PIPELINE_LOCK).unlink()
|
||||
(stage,) = dvc.reproduce("backup-foo")
|
||||
|
||||
assert (tmp_dir / "foo.bak").read_text() == "foo"
|
||||
assert (tmp_dir / PIPELINE_LOCK).exists()
|
||||
mock_run.assert_not_called()
|
||||
mock_restore.assert_called_once_with(stage)
|
||||
mock_restore.reset_mock()
|
||||
|
||||
(stage,) = dvc.reproduce("backup-bar")
|
||||
|
||||
assert (tmp_dir / "bar.bak").read_text() == "bar"
|
||||
assert (tmp_dir / PIPELINE_LOCK).exists()
|
||||
mock_run.assert_not_called()
|
||||
mock_restore.assert_called_once_with(stage)
|
||||
Reference in New Issue
Block a user