diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index dcab08ec4..fe3cd9c6f 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -72,7 +72,7 @@ def _fetch( if failed: raise DownloadError(failed) - return downloaded + return downloaded + len(used_run_cache) def _fetch_external(self, repo_url, repo_rev, files, jobs): diff --git a/dvc/repo/push.py b/dvc/repo/push.py index f0c88e298..a950dd284 100644 --- a/dvc/repo/push.py +++ b/dvc/repo/push.py @@ -32,4 +32,4 @@ def push( used_run_cache=used_run_cache, ) - return self.cloud.push(used, jobs, remote=remote) + return len(used_run_cache) + self.cloud.push(used, jobs, remote=remote) diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index 422635797..1c33e23b4 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -63,6 +63,7 @@ def reproduce( ): from dvc.utils import parse_target + assert target is None or isinstance(target, str) if not target and not all_pipelines: raise InvalidArgumentError( "Neither `target` nor `--all-pipelines` are specified." diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index f20be749f..2ba6c1cb3 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -6,6 +6,7 @@ import yaml from funcy import first from voluptuous import Invalid +from dvc.remote.local import _log_exceptions from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA from dvc.utils import dict_sha256, relpath from dvc.utils.fs import makedirs @@ -177,11 +178,19 @@ class StageCache: def push(self, remote): remote = self.repo.cloud.get_remote(remote) - return self._transfer(remote.upload, self.repo.cache.local, remote) + return self._transfer( + _log_exceptions(remote.tree.upload, "upload"), + self.repo.cache.local.tree, + remote.tree, + ) def pull(self, remote): remote = self.repo.cloud.get_remote(remote) - return self._transfer(remote.download, remote, self.repo.cache.local) + return self._transfer( + _log_exceptions(remote.tree.download, "download"), + remote.tree, + self.repo.cache.local.tree, + ) def get_used_cache(self, used_run_cache, *args, **kwargs): from dvc.cache import NamedCache diff --git a/tests/func/test_run_cache.py b/tests/func/test_run_cache.py new file mode 100644 index 000000000..7b510b5a3 --- /dev/null +++ b/tests/func/test_run_cache.py @@ -0,0 +1,153 @@ +import os + +from dvc.dvcfile import PIPELINE_LOCK +from dvc.utils import relpath + + +def _recurse_count_files(path): + return len([os.path.join(r, f) for r, _, fs in os.walk(path) for f in fs]) + + +def test_push_pull(tmp_dir, dvc, erepo_dir, run_copy, setup_remote): + tmp_dir.gen("foo", "foo") + run_copy("foo", "bar", name="copy-foo-bar") + url = setup_remote(dvc) + assert dvc.push(run_cache=True) == 2 + setup_remote(erepo_dir.dvc, url) + with erepo_dir.chdir(): + assert not os.path.exists(erepo_dir.dvc.stage_cache.cache_dir) + assert erepo_dir.dvc.pull(run_cache=True)["fetched"] == 2 + assert os.listdir(erepo_dir.dvc.stage_cache.cache_dir) + + +def test_restore(tmp_dir, dvc, run_copy, mocker): + tmp_dir.gen("foo", "foo") + run_copy("foo", "bar", name="copy-foo-bar") + + mock_restore = mocker.spy(dvc.stage_cache, "restore") + mock_run = mocker.patch("dvc.stage.run.cmd_run") + + # removing any information that `dvc` could use to re-generate from + (tmp_dir / "bar").unlink() + (tmp_dir / PIPELINE_LOCK).unlink() + + (stage,) = dvc.reproduce("copy-foo-bar") + + mock_restore.assert_called_once_with(stage) + mock_run.assert_not_called() + assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink() + assert (tmp_dir / PIPELINE_LOCK).exists() + + +def test_save(tmp_dir, dvc, run_copy): + run_cache_dir = dvc.stage_cache.cache_dir + assert not os.path.exists(run_cache_dir) + + tmp_dir.gen("foo", "foo") + stage = run_copy("foo", "bar", name="copy-foo-bar") + assert _recurse_count_files(run_cache_dir) == 1 + with dvc.state: + assert dvc.stage_cache.is_cached(stage) + + +def test_do_not_save_on_no_exec_and_dry(tmp_dir, dvc, run_copy): + run_cache_dir = dvc.stage_cache.cache_dir + assert not os.path.exists(run_cache_dir) + + tmp_dir.gen("foo", "foo") + stage = run_copy("foo", "bar", name="copy-foo-bar", no_exec=True) + + assert _recurse_count_files(run_cache_dir) == 0 + with dvc.state: + assert not dvc.stage_cache.is_cached(stage) + + (stage,) = dvc.reproduce("copy-foo-bar", dry=True) + + assert _recurse_count_files(run_cache_dir) == 0 + with dvc.state: + assert not dvc.stage_cache.is_cached(stage) + + +def test_uncached_outs_are_cached(tmp_dir, dvc, run_copy): + tmp_dir.gen("foo", "foo") + stage = dvc.run( + deps=["foo"], + cmd="cp foo bar", + outs_no_cache=["bar"], + name="copy-foo-bar", + ) + with dvc.state: + stage.outs[0].checksum = stage.outs[0].get_checksum() + assert os.path.exists(relpath(stage.outs[0].cache_path)) + + +def test_memory_for_multiple_runs_of_same_stage( + tmp_dir, dvc, run_copy, mocker +): + tmp_dir.gen("foo", "foo") + assert not os.path.exists(dvc.stage_cache.cache_dir) + run_copy("foo", "bar", name="copy-foo-bar") + assert _recurse_count_files(dvc.stage_cache.cache_dir) == 1 + tmp_dir.gen("foo", "foobar") + run_copy("foo", "bar", name="copy-foo-bar") + assert _recurse_count_files(dvc.stage_cache.cache_dir) == 2 + + from dvc.stage import run as _run + + mock_restore = mocker.spy(dvc.stage_cache, "restore") + mock_run = mocker.spy(_run, "cmd_run") + + (tmp_dir / "bar").unlink() + (tmp_dir / PIPELINE_LOCK).unlink() + (stage,) = dvc.reproduce("copy-foo-bar") + + assert (tmp_dir / PIPELINE_LOCK).exists() + assert (tmp_dir / "bar").read_text() == "foobar" + mock_run.assert_not_called() + mock_restore.assert_called_once_with(stage) + mock_restore.reset_mock() + + (tmp_dir / PIPELINE_LOCK).unlink() + tmp_dir.gen("foo", "foo") + dvc.reproduce("copy-foo-bar") + + assert (tmp_dir / "bar").read_text() == "foo" + mock_run.assert_not_called() + mock_restore.assert_called_once_with(stage) + assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink() + assert (tmp_dir / PIPELINE_LOCK).exists() + + +def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker): + tmp_dir.gen("foo", "foo") + assert not os.path.exists(dvc.stage_cache.cache_dir) + + run_copy("foo", "foo.bak", name="backup-foo") + assert _recurse_count_files(dvc.stage_cache.cache_dir) == 1 + + tmp_dir.gen("bar", "bar") + run_copy("bar", "bar.bak", name="backup-bar") + assert _recurse_count_files(dvc.stage_cache.cache_dir) == 2 + + from dvc.stage import run as _run + + mock_restore = mocker.spy(dvc.stage_cache, "restore") + mock_run = mocker.spy(_run, "cmd_run") + + (tmp_dir / "foo.bak").unlink() + (tmp_dir / "bar.bak").unlink() + (tmp_dir / PIPELINE_LOCK).unlink() + (stage,) = dvc.reproduce("backup-foo") + + assert (tmp_dir / "foo.bak").read_text() == "foo" + assert (tmp_dir / PIPELINE_LOCK).exists() + mock_run.assert_not_called() + mock_restore.assert_called_once_with(stage) + mock_restore.reset_mock() + + (stage,) = dvc.reproduce("backup-bar") + + assert (tmp_dir / "bar.bak").read_text() == "bar" + assert (tmp_dir / PIPELINE_LOCK).exists() + mock_run.assert_not_called() + mock_restore.assert_called_once_with(stage)