diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 1eb7762ed..bf2c64204 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -35,7 +35,8 @@ def loadd_from(stage, d_list): for d in d_list: p = d.pop(Output.PARAM_PATH, None) files = d.pop(Output.PARAM_FILES, None) - ret.append(_get(stage, p, d, files=files)) + hash_name = d.pop(Output.PARAM_HASH, None) + ret.append(_get(stage, p, d, files=files, hash_name=hash_name)) return ret diff --git a/dvc/dependency/param.py b/dvc/dependency/param.py index e29b78f64..411690286 100644 --- a/dvc/dependency/param.py +++ b/dvc/dependency/param.py @@ -46,6 +46,7 @@ class ParamsDependency(Dependency): repo = repo or stage.repo path = path or os.path.join(repo.root_dir, self.DEFAULT_PARAMS_FILE) super().__init__(stage, path, repo=repo) + self.hash_name = self.PARAM_PARAMS self.hash_info = hash_info def dumpd(self, **kwargs): diff --git a/dvc/output.py b/dvc/output.py index 3bc492376..0d89bda82 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -27,6 +27,7 @@ from dvc_data.hashfile import load as oload from dvc_data.hashfile.build import build from dvc_data.hashfile.checkout import checkout from dvc_data.hashfile.db import HashFileDB, add_update_tree +from dvc_data.hashfile.hash import DEFAULT_ALGORITHM from dvc_data.hashfile.hash_info import HashInfo from dvc_data.hashfile.istextfile import istextfile from dvc_data.hashfile.meta import Meta @@ -64,7 +65,7 @@ CASE_SENSITIVE_CHECKSUM_SCHEMA = Any( # NOTE: currently there are only 3 possible checksum names: # -# 1) md5 (LOCAL, SSH); +# 1) md5 (LOCAL, SSH) (actually DVC 2.x md5-dos2unix) # 2) etag (S3, GS, OSS, AZURE, HTTP); # 3) checksum (HDFS); # @@ -73,7 +74,7 @@ CASE_SENSITIVE_CHECKSUM_SCHEMA = Any( HDFS_PARAM_CHECKSUM = "checksum" S3_PARAM_CHECKSUM = "etag" CHECKSUMS_SCHEMA = { - LocalFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA, + "md5": CHECKSUM_SCHEMA, # DVC 2.x md5-dos2unix HDFS_PARAM_CHECKSUM: CHECKSUM_SCHEMA, S3_PARAM_CHECKSUM: CASE_SENSITIVE_CHECKSUM_SCHEMA, } @@ -95,6 +96,7 @@ def loadd_from(stage, d_list): annot = {field: d.pop(field, None) for field in ANNOTATION_FIELDS} files = d.pop(Output.PARAM_FILES, None) push = d.pop(Output.PARAM_PUSH, True) + hash_name = d.pop(Output.PARAM_HASH, None) ret.append( _get( stage, @@ -108,6 +110,7 @@ def loadd_from(stage, d_list): **annot, files=files, push=push, + hash_name=hash_name, ) ) return ret @@ -227,17 +230,29 @@ def merge_file_meta_from_cloud(entry: Dict) -> Dict: return entry -def _serialize_tree_obj_to_files(obj: "Tree") -> List[Dict[str, Any]]: +def _serialize_tree_obj_to_files(obj: Tree) -> List[Dict[str, Any]]: key = obj.PARAM_RELPATH return sorted( ( - {key: posixpath.sep.join(parts), **hi.to_dict(), **meta.to_dict()} + { + key: posixpath.sep.join(parts), + **_serialize_hi_to_dict(hi), + **meta.to_dict(), + } for parts, meta, hi in obj ), key=itemgetter(key), ) +def _serialize_hi_to_dict(hash_info: Optional[HashInfo]) -> Dict[str, Any]: + if hash_info: + if hash_info.name == "md5-dos2unix": + return {"md5": hash_info.value} + return hash_info.to_dict() + return {} + + class OutputDoesNotExistError(DvcException): def __init__(self, path): msg = f"output '{path}' does not exist" @@ -297,6 +312,7 @@ class Output: PARAM_REMOTE = "remote" PARAM_PUSH = "push" PARAM_CLOUD = "cloud" + PARAM_HASH = "hash" DoesNotExistError: Type[DvcException] = OutputDoesNotExistError IsNotFileOrDirError: Type[DvcException] = OutputIsNotFileOrDirError @@ -321,6 +337,7 @@ class Output: fs_config=None, files: Optional[List[Dict[str, Any]]] = None, push: bool = True, + hash_name: Optional[str] = DEFAULT_ALGORITHM, ): self.annot = Annotation( desc=desc, type=type, labels=labels or [], meta=meta or {} @@ -392,17 +409,30 @@ class Output: ) self.meta.version_id = version_id - if self.is_in_repo: - self.hash_name = "md5" - else: - self.hash_name = self.fs.PARAM_CHECKSUM - - self.hash_info = HashInfo( - name=self.hash_name, - value=getattr(self.meta, self.hash_name, None), - ) + self.hash_name, self.hash_info = self._compute_hash_info_from_meta(hash_name) self._compute_meta_hash_info_from_files() + def _compute_hash_info_from_meta( + self, hash_name: Optional[str] + ) -> Tuple[str, HashInfo]: + if self.is_in_repo: + if hash_name is None: + # Legacy 2.x output, use "md5-dos2unix" but read "md5" from + # file meta + hash_name = "md5-dos2unix" + meta_name = "md5" + else: + meta_name = hash_name + else: + hash_name = meta_name = self.fs.PARAM_CHECKSUM + assert hash_name + + hash_info = HashInfo( + name=hash_name, + value=getattr(self.meta, meta_name, None), + ) + return hash_name, hash_info + def _compute_meta_hash_info_from_files(self) -> None: if self.files: tree = Tree.from_list(self.files, hash_name=self.hash_name) @@ -417,7 +447,7 @@ class Output: self.meta.remote = first(f.get("remote") for f in self.files) elif self.meta.nfiles or self.hash_info and self.hash_info.isdir: self.meta.isdir = True - if not self.hash_info and self.hash_name != "md5": + if not self.hash_info and self.hash_name not in ("md5", "md5-dos2unix"): md5 = getattr(self.meta, "md5", None) if md5: self.hash_info = HashInfo("md5", md5) @@ -494,7 +524,12 @@ class Output: @property def cache(self): - odb_name = "repo" if self.is_in_repo else self.protocol + from dvc.cachemgr import LEGACY_HASH_NAMES + + if self.is_in_repo: + odb_name = "legacy" if self.hash_name in LEGACY_HASH_NAMES else "repo" + else: + odb_name = self.protocol odb = getattr(self.repo.cache, odb_name) if self.use_cache and odb is None: raise RemoteCacheRequiredError(self.fs.protocol, self.fs_path) @@ -663,6 +698,8 @@ class Output: if self.metric: self.verify_metric() + if self.hash_name == "md5-dos2unix": + self.hash_name = "md5" if self.use_cache: _, self.meta, self.obj = build( self.cache, @@ -772,6 +809,8 @@ class Output: return checkout_obj def dumpd(self, **kwargs): # noqa: C901, PLR0912 + from dvc.cachemgr import LEGACY_HASH_NAMES + ret: Dict[str, Any] = {} with_files = ( (not self.IS_DEPENDENCY or self.stage.is_import) @@ -782,7 +821,12 @@ class Output: if not with_files: meta_d = self.meta.to_dict() meta_d.pop("isdir", None) - ret.update(self.hash_info.to_dict()) + if self.hash_name in LEGACY_HASH_NAMES: + # 2.x checksums get serialized with file meta + name = "md5" if self.hash_name == "md5-dos2unix" else self.hash_name + ret.update({name: self.hash_info.value}) + else: + ret.update(self.hash_info.to_dict()) ret.update(split_file_meta_from_cloud(meta_d)) if self.is_in_repo: @@ -790,6 +834,9 @@ class Output: else: path = self.def_path + if self.hash_name not in LEGACY_HASH_NAMES: + ret[self.PARAM_HASH] = "md5" + ret[self.PARAM_PATH] = path if not self.IS_DEPENDENCY: @@ -969,7 +1016,7 @@ class Output: odb, from_info, from_fs, - "md5", + DEFAULT_ALGORITHM, upload=upload, no_progress_bar=no_progress_bar, ) @@ -1110,7 +1157,9 @@ class Output: return {} if self.remote: - remote = self.repo.cloud.get_remote_odb(name=self.remote) + remote = self.repo.cloud.get_remote_odb( + name=self.remote, hash_name=self.hash_name + ) else: remote = None @@ -1149,13 +1198,14 @@ class Output: self.hash_name, Meta.PARAM_SIZE, Meta.PARAM_NFILES, + Output.PARAM_HASH, ] for opt in ignored: my.pop(opt, None) other.pop(opt, None) - if my != other: + if my != other or self.hash_name != out.hash_name: raise MergeError("unable to auto-merge outputs with different options") if not out.is_dir_checksum: @@ -1418,6 +1468,7 @@ ARTIFACT_SCHEMA = { Required(Output.PARAM_PATH): str, Output.PARAM_PERSIST: bool, Output.PARAM_CLOUD: CLOUD_SCHEMA, + Output.PARAM_HASH: str, } DIR_FILES_SCHEMA: Dict[str, Any] = { diff --git a/dvc/schema.py b/dvc/schema.py index 8841bf362..07552232e 100644 --- a/dvc/schema.py +++ b/dvc/schema.py @@ -33,6 +33,7 @@ DATA_SCHEMA = { Required("path"): str, Output.PARAM_CLOUD: CLOUD_SCHEMA, Output.PARAM_FILES: [DIR_FILES_SCHEMA], + Output.PARAM_HASH: str, } LOCK_FILE_STAGE_SCHEMA = { Required(StageParams.PARAM_CMD): Any(str, list), diff --git a/dvc/stage/loader.py b/dvc/stage/loader.py index c58685475..c284c87de 100644 --- a/dvc/stage/loader.py +++ b/dvc/stage/loader.py @@ -9,7 +9,6 @@ from funcy import get_in, lcat, once, project from dvc import dependency, output from dvc.parsing import FOREACH_KWD, JOIN, EntryNotFound from dvc.utils.objects import cached_property -from dvc_data.hashfile.hash_info import HashInfo from dvc_data.hashfile.meta import Meta from . import PipelineStage, Stage, loads_from @@ -19,7 +18,6 @@ from .utils import fill_stage_dependencies, resolve_paths if TYPE_CHECKING: from dvc.dvcfile import ProjectFile, SingleStageFile - from dvc.output import Output logger = logging.getLogger(__name__) @@ -52,7 +50,7 @@ class StageLoader(Mapping): if not lock_data: return - from dvc.output import merge_file_meta_from_cloud + from dvc.output import Output, merge_file_meta_from_cloud assert isinstance(lock_data, dict) items: Iterable[Tuple[str, "Output"]] = chain( @@ -73,9 +71,12 @@ class StageLoader(Mapping): info = info.copy() info.pop("path", None) + hash_name = info.pop(Output.PARAM_HASH, None) item.meta = Meta.from_dict(merge_file_meta_from_cloud(info)) - hash_value = getattr(item.meta, item.hash_name, None) - item.hash_info = HashInfo(item.hash_name, hash_value) + # pylint: disable-next=protected-access + item.hash_name, item.hash_info = item._compute_hash_info_from_meta( + hash_name + ) files = get_in(checksums, [key, path, item.PARAM_FILES], None) if files: item.files = [merge_file_meta_from_cloud(f) for f in files] diff --git a/dvc/stage/serialize.py b/dvc/stage/serialize.py index 0e1634935..a22b0ad94 100644 --- a/dvc/stage/serialize.py +++ b/dvc/stage/serialize.py @@ -149,13 +149,20 @@ def to_pipeline_file(stage: "PipelineStage"): def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict: - from dvc.output import _serialize_tree_obj_to_files, split_file_meta_from_cloud + from dvc.cachemgr import LEGACY_HASH_NAMES + from dvc.output import ( + _serialize_hi_to_dict, + _serialize_tree_obj_to_files, + split_file_meta_from_cloud, + ) from dvc_data.hashfile.tree import Tree assert stage.cmd def _dumpd(item: "Output"): ret: Dict[str, Any] = {item.PARAM_PATH: item.def_path} + if item.hash_name not in LEGACY_HASH_NAMES: + ret[item.PARAM_HASH] = "md5" if item.hash_info.isdir and kwargs.get("with_files"): obj = item.obj or item.get_obj() if obj: @@ -167,7 +174,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict: else: meta_d = item.meta.to_dict() meta_d.pop("isdir", None) - ret.update(item.hash_info.to_dict()) + ret.update(_serialize_hi_to_dict(item.hash_info)) ret.update(split_file_meta_from_cloud(meta_d)) return ret diff --git a/dvc/testing/workspace_tests.py b/dvc/testing/workspace_tests.py index bef340d13..75d463ce3 100644 --- a/dvc/testing/workspace_tests.py +++ b/dvc/testing/workspace_tests.py @@ -54,11 +54,13 @@ class TestImport: f"- md5: {dir_md5}\n" " size: 11\n" " nfiles: 2\n" + " hash: md5\n" " path: remote://workspace/dir\n" "outs:\n" "- md5: b6dcab6ccd17ca0a8bf4a215a37d14cc.dir\n" " size: 11\n" " nfiles: 2\n" + " hash: md5\n" " path: dir\n" ) @@ -177,6 +179,7 @@ class TestAdd: "outs:\n" f"- {hash_name}: {hash_value}\n" " size: 4\n" + " hash: md5\n" " path: remote://workspace/file\n" ) assert (workspace / "file").read_text() == "file" diff --git a/tests/func/test_add.py b/tests/func/test_add.py index ecf226889..03fea3413 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -51,6 +51,7 @@ def test_add(tmp_dir, dvc): "md5": "acbd18db4cc2f85cedef654fccc4a4d8", "path": "foo", "size": 3, + "hash": "md5", } ] } @@ -70,6 +71,7 @@ def test_add_executable(tmp_dir, dvc): "path": "foo", "size": 3, "isexec": True, + "hash": "md5", } ] } @@ -258,6 +260,7 @@ def test_add_external_relpath(tmp_dir, dvc, local_cloud): "outs:\n" "- md5: 8c7dd922ad47494fc02c388e12c00eac\n" " size: 4\n" + " hash: md5\n" f" path: {rel}\n" ) assert fpath.read_text() == "file" diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index df73178d1..b87279b70 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -329,8 +329,8 @@ def test_checkout_moved_cache_dir_with_symlinks(tmp_dir, dvc): assert system.is_symlink(os.path.join("data", "file")) old_data_link = os.path.realpath(os.path.join("data", "file")) - old_cache_dir = dvc.cache.local.path - new_cache_dir = old_cache_dir + "_new" + old_cache_dir = str(tmp_dir / ".dvc" / "cache") + new_cache_dir = str(tmp_dir / ".dvc" / "cache_new") os.rename(old_cache_dir, new_cache_dir) ret = main(["cache", "dir", new_cache_dir]) @@ -801,6 +801,7 @@ def test_checkout_dir_compat(tmp_dir, dvc): f"""\ outs: - md5: {stage.outs[0].hash_info.value} + hash: md5 path: data """ ), diff --git a/tests/func/test_commit.py b/tests/func/test_commit.py index d97f9817f..962fd58b1 100644 --- a/tests/func/test_commit.py +++ b/tests/func/test_commit.py @@ -56,6 +56,7 @@ def test_commit_preserve_fields(tmp_dir, dvc): key1: value1 key2: value2 remote: testremote + hash: md5 meta: some metadata """ ) @@ -77,6 +78,7 @@ def test_commit_preserve_fields(tmp_dir, dvc): key1: value1 key2: value2 remote: testremote + hash: md5 md5: acbd18db4cc2f85cedef654fccc4a4d8 size: 3 meta: some metadata @@ -129,7 +131,10 @@ def test_commit_no_exec(tmp_dir, dvc): def test_commit_granular_output(tmp_dir, dvc): dvc.run( name="mystage", - cmd=["echo foo>foo", "echo bar>bar"], + cmd=[ + "python -c \"open('foo', 'wb').write(b'foo\\n')\"", + "python -c \"open('bar', 'wb').write(b'bar\\n')\"", + ], outs=["foo", "bar"], no_commit=True, ) @@ -248,6 +253,7 @@ def test_commit_updates_to_cloud_versioning_dir(tmp_dir, dvc): "outs": [ { "path": "data", + "hash": "md5", "files": [ { "size": 3, @@ -280,6 +286,7 @@ def test_commit_updates_to_cloud_versioning_dir(tmp_dir, dvc): "outs": [ { "path": "data", + "hash": "md5", "files": [ { "size": 4, @@ -297,3 +304,21 @@ def test_commit_updates_to_cloud_versioning_dir(tmp_dir, dvc): } ] } + + +def test_commit_dos2unix(tmp_dir, dvc): + tmp_dir.gen("foo", "foo") + (tmp_dir / "foo.dvc").dump( + { + "outs": [ + {"path": "foo", "md5": "acbd18db4cc2f85cedef654fccc4a4d8", "size": 3}, + ] + } + ) + legacy_content = (tmp_dir / "foo.dvc").read_text() + + dvc.commit("foo.dvc") + assert (tmp_dir / "foo.dvc").read_text() == legacy_content + dvc.commit("foo.dvc", force=True) + content = (tmp_dir / "foo.dvc").read_text() + assert "hash: md5" in content diff --git a/tests/func/test_dvcfile.py b/tests/func/test_dvcfile.py index 966712f5f..c433d5a50 100644 --- a/tests/func/test_dvcfile.py +++ b/tests/func/test_dvcfile.py @@ -402,3 +402,15 @@ def test_dvcfile_load_with_plots(tmp_dir, dvc): assert all( name in top_level_plots for name in ("path/to/plot", "path/to/another/plot") ) + + +def test_dvcfile_dos2unix(tmp_dir, dvc): + from dvc_data.hashfile.hash import HashInfo + + (tmp_dir / "foo.dvc").dump({"outs": [{"md5": "abc123", "size": 3, "path": "foo"}]}) + orig_content = (tmp_dir / "foo.dvc").read_text() + stage = dvc.stage.load_one("foo.dvc") + assert stage.outs[0].hash_name == "md5-dos2unix" + assert stage.outs[0].hash_info == HashInfo("md5-dos2unix", "abc123") + stage.dump() + assert (tmp_dir / "foo.dvc").read_text() == orig_content diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 0bf273c40..9a7010f01 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -214,10 +214,20 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir): shallow=False, hardlink=True, ) + if os.name == "nt": + expected_gitignore_path = ( + cache_dir / "d5" / "61e684092f0ff908aa82ee9cc1e594" + ) + expected_dir_path = cache_dir / "0d" / "2086760aea091f1504eafc8843bb18.dir" + else: + expected_gitignore_path = ( + cache_dir / "94" / "7d2b84e5aa88170e80dff467a5bfb6" + ) + expected_dir_path = cache_dir / "e1" / "d9e8eae5374860ae025ec84cfd85c7.dir" assert set(cache_dir.glob("??/*")) == { - cache_dir / "e1" / "d9e8eae5374860ae025ec84cfd85c7.dir", + expected_dir_path, + expected_gitignore_path, cache_dir / "37" / "b51d194a7513e45b56f6524f2d51f2", - cache_dir / "94" / "7d2b84e5aa88170e80dff467a5bfb6", cache_dir / "ac" / "bd18db4cc2f85cedef654fccc4a4d8", } diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index 5e47af953..75087adec 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -428,6 +428,7 @@ def test_gc_not_in_remote_with_remote_field( outs: - path: foo remote: other_remote + hash: md5 """ ) tmp_dir.gen("foo.dvc", text) @@ -459,6 +460,7 @@ def test_gc_cloud_remote_field(tmp_dir, scm, dvc, tmp_path_factory, mocker): outs: - path: foo remote: other_remote + hash: md5 """ ) tmp_dir.gen("foo.dvc", text) diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 34274a788..f67b9ceae 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -247,7 +247,11 @@ def test_pull_import_no_download(tmp_dir, scm, dvc, erepo_dir): stage = load_file(dvc, "foo_imported.dvc").stage - assert stage.outs[0].hash_info.value == "bdb8641831d8fcb03939637e09011c21.dir" + if os.name == "nt": + expected_hash = "2e798234df5f782340ac3ce046f8dfae.dir" + else: + expected_hash = "bdb8641831d8fcb03939637e09011c21.dir" + assert stage.outs[0].hash_info.value == expected_hash assert stage.outs[0].meta.size == size assert stage.outs[0].meta.nfiles == 3 diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index f2fe89c8d..cdace1c85 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -108,7 +108,7 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir): class TestImport(_TestImport): @pytest.fixture def stage_md5(self): - return "dc24e1271084ee317ac3c2656fb8812b" + return "7033ee831f78a4dfec2fc71405516067" @pytest.fixture def dir_md5(self): @@ -150,6 +150,7 @@ def test_import_url_preserve_fields(tmp_dir, dvc): - path: foo # dep comment md5: acbd18db4cc2f85cedef654fccc4a4d8 size: 3 + hash: md5 outs: - path: bar # out comment desc: out desc @@ -161,8 +162,9 @@ def test_import_url_preserve_fields(tmp_dir, dvc): key: value md5: acbd18db4cc2f85cedef654fccc4a4d8 size: 3 + hash: md5 meta: some metadata - md5: be7ade0aa89cc8d56e320867a9de9740 + md5: 8fc199641730e3f512deac0bd9a0e0b6 frozen: true """ ) diff --git a/tests/func/test_install.py b/tests/func/test_install.py index 43ceb2b72..00f902c6f 100644 --- a/tests/func/test_install.py +++ b/tests/func/test_install.py @@ -118,6 +118,7 @@ def test_merge_driver_no_ancestor(tmp_dir, scm, dvc): "- md5: 5ea40360f5b4ec688df672a4db9c17d1.dir\n" " size: 6\n" " nfiles: 2\n" + " hash: md5\n" " path: data\n" ) @@ -153,6 +154,7 @@ def test_merge_driver(tmp_dir, scm, dvc): "- md5: 839ef9371606817569c1ee0e5f4ed233.dir\n" " size: 12\n" " nfiles: 3\n" + " hash: md5\n" " path: data\n" ) diff --git a/tests/func/test_lockfile.py b/tests/func/test_lockfile.py index 23e329611..9738bc278 100644 --- a/tests/func/test_lockfile.py +++ b/tests/func/test_lockfile.py @@ -43,8 +43,12 @@ def test_deps_outs_are_sorted_by_path(tmp_dir, dvc, run_head): assert list(lock.keys()) == ["cmd", "deps", "outs"] # `path` key appear first and then the `md5` - assert all(list(dep.keys()) == ["path", "md5", "size"] for dep in lock["deps"]) - assert all(list(out.keys()) == ["path", "md5", "size"] for out in lock["outs"]) + assert all( + list(dep.keys()) == ["path", "hash", "md5", "size"] for dep in lock["deps"] + ) + assert all( + list(out.keys()) == ["path", "hash", "md5", "size"] for out in lock["outs"] + ) # deps are always sorted by the file path naming assert list(map(itemgetter("path"), lock["deps"])) == sorted(deps) diff --git a/tests/func/test_merge_driver.py b/tests/func/test_merge_driver.py index 865b036fe..bc0e1de2f 100644 --- a/tests/func/test_merge_driver.py +++ b/tests/func/test_merge_driver.py @@ -127,10 +127,16 @@ def test_merge_conflict(tmp_dir, dvc, ancestor, our, their, error, caplog): def test_merge_different_output_options(tmp_dir, dvc, caplog): (tmp_dir / "ancestor").touch() - (tmp_dir / "our").write_text("outs:\n- md5: f123456789.dir\n path: path\n") + (tmp_dir / "our").write_text( + "outs:\n- md5: f123456789.dir\n hash: md5\n path: path\n" + ) (tmp_dir / "their").write_text( - "outs:\n- md5: f987654321.dir\n path: path\n cache: false\n" + "outs:\n" + "- md5: f987654321.dir\n" + " hash: md5\n" + " path: path\n" + " cache: false\n" ) assert ( @@ -156,9 +162,13 @@ def test_merge_different_output_options(tmp_dir, dvc, caplog): def test_merge_file(tmp_dir, dvc, caplog): (tmp_dir / "ancestor").touch() - (tmp_dir / "our").write_text("outs:\n- md5: f123456789.dir\n path: path\n") + (tmp_dir / "our").write_text( + "outs:\n- md5: f123456789.dir\n hash: md5\n path: path\n" + ) - (tmp_dir / "their").write_text("outs:\n- md5: f987654321\n path: path\n") + (tmp_dir / "their").write_text( + "outs:\n- md5: f987654321\n hash: md5\n path: path\n" + ) assert ( main( @@ -186,12 +196,16 @@ def test_merge_non_dvc_add(tmp_dir, dvc, caplog): (tmp_dir / "our").write_text( "outs:\n" "- md5: f123456789.dir\n" + " hash: md5\n" " path: path\n" "- md5: ff123456789.dir\n" + " hash: md5\n" " path: another\n" ) - (tmp_dir / "their").write_text("outs:\n- md5: f987654321\n path: path\n") + (tmp_dir / "their").write_text( + "outs:\n- md5: f987654321\n hash: md5\n path: path\n" + ) assert ( main( diff --git a/tests/func/test_move.py b/tests/func/test_move.py index 5d91fda45..8e15cbfab 100644 --- a/tests/func/test_move.py +++ b/tests/func/test_move.py @@ -194,6 +194,7 @@ def test_move_meta(tmp_dir, dvc): outs: - md5: acbd18db4cc2f85cedef654fccc4a4d8 size: 3 + hash: md5 path: bar meta: custom_key: 42 diff --git a/tests/func/test_run.py b/tests/func/test_run.py index a6d6871d8..94c134294 100644 --- a/tests/func/test_run.py +++ b/tests/func/test_run.py @@ -562,10 +562,12 @@ def test_run_external_outputs( " cmd: mycmd\n" " deps:\n" " - path: remote://workspace/foo\n" + " hash: md5\n" f" {hash_name}: {foo_hash}\n" " size: 3\n" " outs:\n" " - path: remote://workspace/bar\n" + " hash: md5\n" f" {hash_name}: {bar_hash}\n" " size: 3\n" ) diff --git a/tests/func/test_stage.py b/tests/func/test_stage.py index 261b53479..76ffc1500 100644 --- a/tests/func/test_stage.py +++ b/tests/func/test_stage.py @@ -139,6 +139,7 @@ def test_md5_ignores_annotations(tmp_dir, dvc): { "md5": "d3b07384d113edec49eaa6238ad5ff00", "size": 4, + "hash": "md5", "path": "foo", "desc": "foo desc", "type": "mytype", @@ -149,10 +150,20 @@ def test_md5_ignores_annotations(tmp_dir, dvc): } (tmp_dir / "foo.dvc").dump(data) stage = dvc.stage.load_one("foo.dvc") - assert compute_md5(stage) == "1822617147b53ae6f9eb4b3c87c0b6f3" + assert compute_md5(stage) == "cde267b60ef5a00e9a35cc1999ab83a3" assert ( - dict_md5({"outs": [{"md5": "d3b07384d113edec49eaa6238ad5ff00", "path": "foo"}]}) - == "1822617147b53ae6f9eb4b3c87c0b6f3" + dict_md5( + { + "outs": [ + { + "md5": "d3b07384d113edec49eaa6238ad5ff00", + "hash": "md5", + "path": "foo", + } + ] + } + ) + == "cde267b60ef5a00e9a35cc1999ab83a3" ) @@ -164,6 +175,7 @@ def test_meta_desc_is_preserved(tmp_dir, dvc): { "md5": "d3b07384d113edec49eaa6238ad5ff00", "size": 4, + "hash": "md5", "path": "foo", "desc": "foo desc", "type": "mytype", diff --git a/tests/scripts.py b/tests/scripts.py index 5d8e4242c..257ebaed3 100644 --- a/tests/scripts.py +++ b/tests/scripts.py @@ -9,11 +9,13 @@ if os.path.isfile(sys.argv[1]): shutil.copyfile(sys.argv[1], sys.argv[2]) else: shutil.copytree(sys.argv[1], sys.argv[2]) -""" +""".replace( + "\r\n", "\n" +) def _add_script(tmp_dir, path, contents=""): - script, *_ = tmp_dir.gen(path, contents) + script, *_ = tmp_dir.gen(path, contents.encode("utf-8")) if hasattr(tmp_dir, "scm"): tmp_dir.scm_add(path, commit=f"add {path}") return script.fs_path @@ -30,7 +32,9 @@ from pathlib import Path with Path(sys.argv[1]).open("a+", encoding="utf-8") as f: f.write("foo") -""", +""".replace( + "\r\n", "\n" + ), ) @@ -55,5 +59,7 @@ import sys for file in sys.argv[1:]: with open(file) as f, open(file +"-1","w+") as w: w.write(f.readline()) -""", +""".replace( + "\r\n", "\n" + ), ) diff --git a/tests/unit/fs/test_dvc.py b/tests/unit/fs/test_dvc.py index f3519b375..90e266d6b 100644 --- a/tests/unit/fs/test_dvc.py +++ b/tests/unit/fs/test_dvc.py @@ -542,7 +542,11 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc): fs = DVCFileSystem(repo=dvc) _, _, obj = build(dvc.cache.local, "dir", fs, "md5") - assert obj.hash_info == HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir") + if os.name == "nt": + expected_hash = "0d2086760aea091f1504eafc8843bb18.dir" + else: + expected_hash = "e1d9e8eae5374860ae025ec84cfd85c7.dir" + assert obj.hash_info == HashInfo("md5", expected_hash) def test_get_hash_dirty_file(tmp_dir, dvc): diff --git a/tests/unit/output/test_output.py b/tests/unit/output/test_output.py index d2c07ed88..d734f3bbf 100644 --- a/tests/unit/output/test_output.py +++ b/tests/unit/output/test_output.py @@ -175,7 +175,7 @@ def test_dumpd_cloud_versioning_dir(mocker): out = Output(stage, "path", files=files) dumpd = out.dumpd() - assert dumpd == {"path": "path", "files": files} + assert dumpd == {"path": "path", "hash": "md5", "files": files} def test_version_aware_is_set_based_on_files(mocker): diff --git a/tests/unit/stage/test_cache.py b/tests/unit/stage/test_cache.py index ae706eaed..bd6d93042 100644 --- a/tests/unit/stage/test_cache.py +++ b/tests/unit/stage/test_cache.py @@ -27,12 +27,12 @@ def test_stage_cache(tmp_dir, dvc, mocker): cache_dir = os.path.join( dvc.stage_cache.cache_dir, - "4b", - "4b495dc2b14e1ca5bd5f2765a99ddd8785523a8342efe6bcadac012c2db01165", + "c7", + "c7a85d71de1912c43d9c5b6218c71630d6277e8fc11e4ccf5e12b3c202234838", ) cache_file = os.path.join( cache_dir, - "78c427104a8e20216031c36d9c7620733066fff33ada254ad3fca551c1f8152b", + "5e2824029f8da9ef6c57131a638ceb65f27c02f16c85d71e85671c27daed0501", ) assert os.path.isdir(cache_dir) @@ -79,7 +79,7 @@ def test_stage_cache_params(tmp_dir, dvc, mocker): ) cache_file = os.path.join( cache_dir, - "202ea269108bf98bea3e15f978e7929864728956c9df8d927a5c7d74fc4fedc8", + "9ce1963a69beb1299800188647cd960b7afff101be19fd46226e32bb8be8ee44", ) assert os.path.isdir(cache_dir) @@ -122,12 +122,12 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker): cache_dir = os.path.join( dvc.stage_cache.cache_dir, - "b5", - "b5d5548c43725139aa3419eb50717e062fe9b81f866f401fd6fd778d2a97822d", + "6a", + "6ad5cce3347e9e96c77d4353d84e5c8cae8c9151c486c4ea3d3d79e9051800f1", ) cache_file = os.path.join( cache_dir, - "e0a59f6a5bd193032a4d1500abd89c9b08a2e9b26c724015e0bc6374295a3b9f", + "1fa3e1a9f785f8364ad185d62bd0813ce8794afcfc79410e985eac0443cc5462", ) assert os.path.isdir(cache_dir) @@ -162,14 +162,14 @@ def test_shared_stage_cache(tmp_dir, dvc, run_copy): run_copy("foo", "bar", name="copy-foo-bar") - parent_cache_dir = os.path.join(dvc.stage_cache.cache_dir, "fd") + parent_cache_dir = os.path.join(dvc.stage_cache.cache_dir, "6d") cache_dir = os.path.join( parent_cache_dir, - "fdbe7847136ebe88f59a29ee5a568f893cab031f74f7d3ab050828b53fd0033a", + "6d4c6de74e7c0d60d2122e2063b4724a8c78a6799def6c7cf5093f45e7f2f3b7", ) cache_file = os.path.join( cache_dir, - "8716052b2074f5acf1a379529d47d6ef1c1f50c2281a7489b8d7ce251f234f86", + "435e34f80692059e79ec53346cbfe29fd9ee65b62f5f06f17f5950ce5a7408ea", ) # sanity check diff --git a/tests/unit/stage/test_loader_pipeline_file.py b/tests/unit/stage/test_loader_pipeline_file.py index a4cfe4f7f..7cf1b4a5b 100644 --- a/tests/unit/stage/test_loader_pipeline_file.py +++ b/tests/unit/stage/test_loader_pipeline_file.py @@ -20,8 +20,8 @@ def stage_data(): def lock_data(): return { "cmd": "command", - "deps": [{"path": "foo", "md5": "foo_checksum"}], - "outs": [{"path": "bar", "md5": "bar_checksum"}], + "deps": [{"path": "foo", "md5": "foo_checksum", "hash": "md5"}], + "outs": [{"path": "bar", "md5": "bar_checksum", "hash": "md5"}], } @@ -46,7 +46,9 @@ def test_fill_from_lock_outs_isexec(dvc): stage, { "cmd": "command", - "outs": [{"path": "foo", "md5": "foo_checksum", "isexec": True}], + "outs": [ + {"path": "foo", "md5": "foo_checksum", "isexec": True, "hash": "md5"} + ], }, ) @@ -253,3 +255,20 @@ def test_load_stage_mapping(dvc, stage_data, lock_data): assert "stage1" not in dvcfile.stages assert dvcfile.stages.keys() == {"stage"} assert isinstance(dvcfile.stages["stage"], PipelineStage) + + +def test_fill_from_lock_dos2unix(dvc): + lock_data = { + "cmd": "command", + "deps": [{"path": "foo", "md5": "foo_checksum"}], + "outs": [{"path": "bar", "md5": "bar_checksum"}], + } + stage = create_stage(PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"]) + + for item in chain(stage.deps, stage.outs): + assert not item.hash_info + + StageLoader.fill_from_lock(stage, lock_data) + + assert stage.deps[0].hash_info == HashInfo("md5-dos2unix", "foo_checksum") + assert stage.outs[0].hash_info == HashInfo("md5-dos2unix", "bar_checksum") diff --git a/tests/unit/stage/test_serialize_pipeline_lock.py b/tests/unit/stage/test_serialize_pipeline_lock.py index ff64121ec..f24cd7d27 100644 --- a/tests/unit/stage/test_serialize_pipeline_lock.py +++ b/tests/unit/stage/test_serialize_pipeline_lock.py @@ -33,7 +33,10 @@ def test_lock_deps(dvc): assert to_single_stage_lockfile(stage) == OrderedDict( [ ("cmd", "command"), - ("deps", [OrderedDict([("path", "input"), ("md5", "md-five")])]), + ( + "deps", + [OrderedDict([("hash", "md5"), ("path", "input"), ("md5", "md-five")])], + ), ] ) @@ -48,8 +51,12 @@ def test_lock_deps_order(dvc): ( "deps", [ - OrderedDict([("path", "input0"), ("md5", "md-zer0")]), - OrderedDict([("path", "input1"), ("md5", "md-one1")]), + OrderedDict( + [("hash", "md5"), ("path", "input0"), ("md5", "md-zer0")] + ), + OrderedDict( + [("hash", "md5"), ("path", "input1"), ("md5", "md-one1")] + ), ], ), ] @@ -134,7 +141,10 @@ def test_lock_outs(dvc, typ): assert to_single_stage_lockfile(stage) == OrderedDict( [ ("cmd", "command"), - ("outs", [OrderedDict([("path", "input"), ("md5", "md-five")])]), + ( + "outs", + [OrderedDict([("hash", "md5"), ("path", "input"), ("md5", "md-five")])], + ), ] ) @@ -152,6 +162,7 @@ def test_lock_outs_isexec(dvc, typ): [ OrderedDict( [ + ("hash", "md5"), ("path", "input"), ("md5", "md-five"), ("isexec", True), @@ -174,8 +185,12 @@ def test_lock_outs_order(dvc, typ): ( "outs", [ - OrderedDict([("path", "input0"), ("md5", "md-zer0")]), - OrderedDict([("path", "input1"), ("md5", "md-one1")]), + OrderedDict( + [("hash", "md5"), ("path", "input0"), ("md5", "md-zer0")] + ), + OrderedDict( + [("hash", "md5"), ("path", "input1"), ("md5", "md-one1")] + ), ], ), ] @@ -190,7 +205,15 @@ def test_dump_nondefault_hash(dvc): ("cmd", "command"), ( "deps", - [OrderedDict([("path", "s3://dvc-temp/file"), ("md5", "value")])], + [ + OrderedDict( + [ + ("hash", "md5"), + ("path", "s3://dvc-temp/file"), + ("md5", "value"), + ] + ) + ], ), ] ) @@ -214,9 +237,9 @@ def test_order(dvc): assert to_single_stage_lockfile(stage) == OrderedDict( [ ("cmd", "command"), - ("deps", [{"path": "input", "md5": "md-five"}]), + ("deps", [{"hash": "md5", "path": "input", "md5": "md-five"}]), ("params", {"params.yaml": {"foo-param": "value"}}), - ("outs", [{"path": "output", "md5": "md5-output"}]), + ("outs", [{"hash": "md5", "path": "output", "md5": "md5-output"}]), ] ) @@ -231,7 +254,7 @@ def test_to_lockfile(dvc): "something": OrderedDict( [ ("cmd", "command"), - ("deps", [{"path": "input", "md5": "md-five"}]), + ("deps", [{"hash": "md5", "path": "input", "md5": "md-five"}]), ] ) } @@ -259,4 +282,4 @@ def test_to_single_stage_lockfile_cloud_versioning_dir(dvc): stage.outs[0].files = files e = _to_single_stage_lockfile(stage, with_files=True) assert Schema(e) - assert e["outs"][0] == {"path": "dir", "files": files} + assert e["outs"][0] == {"hash": "md5", "path": "dir", "files": files}