From 718c9c98ce35d2bbdc16206e4d258416ad098315 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 25 Aug 2020 18:34:09 +0300 Subject: [PATCH] serialization: dont rely on the default hash type (#4466) Prerequisite for #4144 and #3069 --- dvc/output/base.py | 8 ------ dvc/stage/loader.py | 4 ++- dvc/stage/serialize.py | 7 +---- tests/func/test_run_cache.py | 2 +- .../stage/test_serialize_pipeline_lock.py | 27 ++++++++++++++++--- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/dvc/output/base.py b/dvc/output/base.py index 01572a4b5..a6bc95d41 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -172,18 +172,10 @@ class BaseOutput: def cache_path(self): return self.cache.tree.hash_to_path_info(self.checksum).url - @property - def checksum_type(self): - return self.tree.PARAM_CHECKSUM - @property def checksum(self): return self.info.get(self.tree.PARAM_CHECKSUM) - @checksum.setter - def checksum(self, checksum): - self.info[self.tree.PARAM_CHECKSUM] = checksum - def get_checksum(self): return self.tree.get_hash(self.path_info)[1] diff --git a/dvc/stage/loader.py b/dvc/stage/loader.py index 246733d35..25098d9bd 100644 --- a/dvc/stage/loader.py +++ b/dvc/stage/loader.py @@ -42,7 +42,9 @@ class StageLoader(Mapping): if isinstance(item, dependency.ParamsDependency): item.fill_values(get_in(lock_data, [stage.PARAM_PARAMS, path])) continue - item.checksum = get_in(checksums, [key, path, item.checksum_type]) + item.info = get_in(checksums, [key, path], {}) + item.info = item.info.copy() + item.info.pop("path", None) @classmethod def load_stage(cls, dvcfile, name, stage_data, lock_data=None): diff --git a/dvc/stage/serialize.py b/dvc/stage/serialize.py index 3b02fdbc4..b39dd4414 100644 --- a/dvc/stage/serialize.py +++ b/dvc/stage/serialize.py @@ -138,12 +138,7 @@ def to_single_stage_lockfile(stage: "Stage") -> dict: params, deps = split_params_deps(stage) deps, outs = [ [ - OrderedDict( - [ - (PARAM_PATH, item.def_path), - (item.checksum_type, item.checksum), - ] - ) + OrderedDict([(PARAM_PATH, item.def_path), *item.info.items()]) for item in sort_by_path(items) ] for items in [deps, stage.outs] diff --git a/tests/func/test_run_cache.py b/tests/func/test_run_cache.py index 5d995304e..b9d6b107d 100644 --- a/tests/func/test_run_cache.py +++ b/tests/func/test_run_cache.py @@ -76,7 +76,7 @@ def test_uncached_outs_are_cached(tmp_dir, dvc, run_copy): name="copy-foo-bar", ) with dvc.state: - stage.outs[0].checksum = stage.outs[0].get_checksum() + stage.outs[0].info = stage.outs[0].save_info() assert os.path.exists(relpath(stage.outs[0].cache_path)) diff --git a/tests/unit/stage/test_serialize_pipeline_lock.py b/tests/unit/stage/test_serialize_pipeline_lock.py index 85721e07c..fe2b6b8fe 100644 --- a/tests/unit/stage/test_serialize_pipeline_lock.py +++ b/tests/unit/stage/test_serialize_pipeline_lock.py @@ -144,11 +144,31 @@ def test_lock_outs_order(dvc, typ): ) -def test_dump_appropriate_checksums(dvc): +def test_dump_nondefault_hash(dvc): stage = create_stage( PipelineStage, dvc, deps=["s3://dvc-temp/file"], **kwargs ) - stage.deps[0].info = {"etag": "is-it-etag", "md5": "or-md5?"} + stage.deps[0].info = {"md5": "value"} + assert to_single_stage_lockfile(stage) == OrderedDict( + [ + ("cmd", "command"), + ( + "deps", + [ + OrderedDict( + [("path", "s3://dvc-temp/file"), ("md5", "value")] + ) + ], + ), + ] + ) + + +def test_dump_multiple_hashes(dvc): + stage = create_stage( + PipelineStage, dvc, deps=["s3://dvc-temp/file"], **kwargs + ) + stage.deps[0].info = {"etag": "etag-value", "md5": "md5-value"} assert to_single_stage_lockfile(stage) == OrderedDict( [ ("cmd", "command"), @@ -158,7 +178,8 @@ def test_dump_appropriate_checksums(dvc): OrderedDict( [ ("path", "s3://dvc-temp/file"), - ("etag", "is-it-etag"), + ("etag", "etag-value"), + ("md5", "md5-value"), ] ) ],