run-cache: skip unhashable stages (#5185)

Fixes #5182
This commit is contained in:
Ruslan Kuprieiev
2020-12-31 06:28:07 +02:00
committed by GitHub
parent ceaba68f62
commit 948633782e
3 changed files with 30 additions and 17 deletions
+15 -10
View File
@@ -34,18 +34,26 @@ def _get_cache_hash(cache, key=False):
)
def _get_stage_hash(stage):
if not (stage.cmd and stage.deps and stage.outs):
return None
def _can_hash(stage):
if stage.is_callback or stage.always_changed:
return False
if not all([stage.cmd, stage.deps, stage.outs]):
return False
for dep in stage.deps:
if not (dep.scheme == "local" and dep.def_path and dep.get_hash()):
return None
return False
for out in stage.outs:
if out.scheme != "local" or not out.def_path or out.persist:
return None
return False
return True
def _get_stage_hash(stage):
assert _can_hash(stage)
return _get_cache_hash(to_single_stage_lockfile(stage), key=True)
@@ -136,13 +144,10 @@ class StageCache:
yield out
def save(self, stage):
if stage.is_callback or stage.always_changed:
if not _can_hash(stage):
return
cache_key = _get_stage_hash(stage)
if not cache_key:
return
cache = to_single_stage_lockfile(stage)
cache_value = _get_cache_hash(cache)
@@ -167,7 +172,7 @@ class StageCache:
self.tree.move(PathInfo(tmp), path)
def restore(self, stage, run_cache=True, pull=False):
if stage.is_callback or stage.always_changed:
if not _can_hash(stage):
raise RunCacheNotFoundError(stage)
if (
+1 -1
View File
@@ -139,7 +139,7 @@ def run_stage(stage, dry=False, force=False, checkpoint_func=None, **kwargs):
stage.repo.stage_cache.restore(stage, **kwargs)
return
except RunCacheNotFoundError:
pass
stage.save_deps()
run = cmd_run if dry else unlocked_repo(cmd_run)
run(stage, dry=dry, checkpoint_func=checkpoint_func)
+14 -6
View File
@@ -208,14 +208,22 @@ def test_shared_stage_cache(tmp_dir, dvc, run_copy):
assert _mode(cache_file) == file_mode
def test_always_changed(mocker):
from dvc.repo import Repo
from dvc.stage import Stage
@pytest.mark.parametrize(
"kwargs",
[
{},
{"cmd": "cmd"},
{"cmd": "cmd", "deps": ["path"]},
{"cmd": "cmd", "outs": ["path"]},
{"always_changed": True},
],
)
def test_unhashable(tmp_dir, dvc, mocker, kwargs):
from dvc.stage import Stage, create_stage
from dvc.stage.cache import RunCacheNotFoundError, StageCache
repo = mocker.Mock(spec=Repo)
cache = StageCache(repo)
stage = Stage(repo, always_changed=True)
cache = StageCache(dvc)
stage = create_stage(Stage, path="stage.dvc", repo=dvc, **kwargs)
get_stage_hash = mocker.patch("dvc.stage.cache._get_stage_hash")
assert cache.save(stage) is None
assert get_stage_hash.not_called