tests: cover run cache with functional tests (#4018)

* tests: cover run cache with functional tests

* run-cache: use _log_exceptions wrapper

Co-authored-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Saugat Pachhai
2020-06-11 18:44:21 +03:00
committed by GitHub
co-authored by Ruslan Kuprieiev
parent d7788887f0
commit 2e12a31c5a
5 changed files with 167 additions and 4 deletions
+1 -1
View File
@@ -72,7 +72,7 @@ def _fetch(
if failed:
raise DownloadError(failed)
return downloaded
return downloaded + len(used_run_cache)
def _fetch_external(self, repo_url, repo_rev, files, jobs):
+1 -1
View File
@@ -32,4 +32,4 @@ def push(
used_run_cache=used_run_cache,
)
return self.cloud.push(used, jobs, remote=remote)
return len(used_run_cache) + self.cloud.push(used, jobs, remote=remote)
+1
View File
@@ -63,6 +63,7 @@ def reproduce(
):
from dvc.utils import parse_target
assert target is None or isinstance(target, str)
if not target and not all_pipelines:
raise InvalidArgumentError(
"Neither `target` nor `--all-pipelines` are specified."
+11 -2
View File
@@ -6,6 +6,7 @@ import yaml
from funcy import first
from voluptuous import Invalid
from dvc.remote.local import _log_exceptions
from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA
from dvc.utils import dict_sha256, relpath
from dvc.utils.fs import makedirs
@@ -177,11 +178,19 @@ class StageCache:
def push(self, remote):
remote = self.repo.cloud.get_remote(remote)
return self._transfer(remote.upload, self.repo.cache.local, remote)
return self._transfer(
_log_exceptions(remote.tree.upload, "upload"),
self.repo.cache.local.tree,
remote.tree,
)
def pull(self, remote):
remote = self.repo.cloud.get_remote(remote)
return self._transfer(remote.download, remote, self.repo.cache.local)
return self._transfer(
_log_exceptions(remote.tree.download, "download"),
remote.tree,
self.repo.cache.local.tree,
)
def get_used_cache(self, used_run_cache, *args, **kwargs):
from dvc.cache import NamedCache
+153
View File
@@ -0,0 +1,153 @@
import os
from dvc.dvcfile import PIPELINE_LOCK
from dvc.utils import relpath
def _recurse_count_files(path):
return len([os.path.join(r, f) for r, _, fs in os.walk(path) for f in fs])
def test_push_pull(tmp_dir, dvc, erepo_dir, run_copy, setup_remote):
tmp_dir.gen("foo", "foo")
run_copy("foo", "bar", name="copy-foo-bar")
url = setup_remote(dvc)
assert dvc.push(run_cache=True) == 2
setup_remote(erepo_dir.dvc, url)
with erepo_dir.chdir():
assert not os.path.exists(erepo_dir.dvc.stage_cache.cache_dir)
assert erepo_dir.dvc.pull(run_cache=True)["fetched"] == 2
assert os.listdir(erepo_dir.dvc.stage_cache.cache_dir)
def test_restore(tmp_dir, dvc, run_copy, mocker):
tmp_dir.gen("foo", "foo")
run_copy("foo", "bar", name="copy-foo-bar")
mock_restore = mocker.spy(dvc.stage_cache, "restore")
mock_run = mocker.patch("dvc.stage.run.cmd_run")
# removing any information that `dvc` could use to re-generate from
(tmp_dir / "bar").unlink()
(tmp_dir / PIPELINE_LOCK).unlink()
(stage,) = dvc.reproduce("copy-foo-bar")
mock_restore.assert_called_once_with(stage)
mock_run.assert_not_called()
assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()
assert (tmp_dir / PIPELINE_LOCK).exists()
def test_save(tmp_dir, dvc, run_copy):
run_cache_dir = dvc.stage_cache.cache_dir
assert not os.path.exists(run_cache_dir)
tmp_dir.gen("foo", "foo")
stage = run_copy("foo", "bar", name="copy-foo-bar")
assert _recurse_count_files(run_cache_dir) == 1
with dvc.state:
assert dvc.stage_cache.is_cached(stage)
def test_do_not_save_on_no_exec_and_dry(tmp_dir, dvc, run_copy):
run_cache_dir = dvc.stage_cache.cache_dir
assert not os.path.exists(run_cache_dir)
tmp_dir.gen("foo", "foo")
stage = run_copy("foo", "bar", name="copy-foo-bar", no_exec=True)
assert _recurse_count_files(run_cache_dir) == 0
with dvc.state:
assert not dvc.stage_cache.is_cached(stage)
(stage,) = dvc.reproduce("copy-foo-bar", dry=True)
assert _recurse_count_files(run_cache_dir) == 0
with dvc.state:
assert not dvc.stage_cache.is_cached(stage)
def test_uncached_outs_are_cached(tmp_dir, dvc, run_copy):
tmp_dir.gen("foo", "foo")
stage = dvc.run(
deps=["foo"],
cmd="cp foo bar",
outs_no_cache=["bar"],
name="copy-foo-bar",
)
with dvc.state:
stage.outs[0].checksum = stage.outs[0].get_checksum()
assert os.path.exists(relpath(stage.outs[0].cache_path))
def test_memory_for_multiple_runs_of_same_stage(
tmp_dir, dvc, run_copy, mocker
):
tmp_dir.gen("foo", "foo")
assert not os.path.exists(dvc.stage_cache.cache_dir)
run_copy("foo", "bar", name="copy-foo-bar")
assert _recurse_count_files(dvc.stage_cache.cache_dir) == 1
tmp_dir.gen("foo", "foobar")
run_copy("foo", "bar", name="copy-foo-bar")
assert _recurse_count_files(dvc.stage_cache.cache_dir) == 2
from dvc.stage import run as _run
mock_restore = mocker.spy(dvc.stage_cache, "restore")
mock_run = mocker.spy(_run, "cmd_run")
(tmp_dir / "bar").unlink()
(tmp_dir / PIPELINE_LOCK).unlink()
(stage,) = dvc.reproduce("copy-foo-bar")
assert (tmp_dir / PIPELINE_LOCK).exists()
assert (tmp_dir / "bar").read_text() == "foobar"
mock_run.assert_not_called()
mock_restore.assert_called_once_with(stage)
mock_restore.reset_mock()
(tmp_dir / PIPELINE_LOCK).unlink()
tmp_dir.gen("foo", "foo")
dvc.reproduce("copy-foo-bar")
assert (tmp_dir / "bar").read_text() == "foo"
mock_run.assert_not_called()
mock_restore.assert_called_once_with(stage)
assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()
assert (tmp_dir / PIPELINE_LOCK).exists()
def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker):
tmp_dir.gen("foo", "foo")
assert not os.path.exists(dvc.stage_cache.cache_dir)
run_copy("foo", "foo.bak", name="backup-foo")
assert _recurse_count_files(dvc.stage_cache.cache_dir) == 1
tmp_dir.gen("bar", "bar")
run_copy("bar", "bar.bak", name="backup-bar")
assert _recurse_count_files(dvc.stage_cache.cache_dir) == 2
from dvc.stage import run as _run
mock_restore = mocker.spy(dvc.stage_cache, "restore")
mock_run = mocker.spy(_run, "cmd_run")
(tmp_dir / "foo.bak").unlink()
(tmp_dir / "bar.bak").unlink()
(tmp_dir / PIPELINE_LOCK).unlink()
(stage,) = dvc.reproduce("backup-foo")
assert (tmp_dir / "foo.bak").read_text() == "foo"
assert (tmp_dir / PIPELINE_LOCK).exists()
mock_run.assert_not_called()
mock_restore.assert_called_once_with(stage)
mock_restore.reset_mock()
(stage,) = dvc.reproduce("backup-bar")
assert (tmp_dir / "bar.bak").read_text() == "bar"
assert (tmp_dir / PIPELINE_LOCK).exists()
mock_run.assert_not_called()
mock_restore.assert_called_once_with(stage)