Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

output: serialize 3.x hash name in 'hash' field

Browse Source
This commit is contained in:
Peter Rowlands
2023-06-09 15:49:44 +03:00
committed by Ruslan Kuprieiev
parent 603f5fdba1
commit 2cd33e10ac
27 changed files with 286 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/dependency/__init__.py
+2 -1
View File
@@ -35,7 +35,8 @@ def loadd_from(stage, d_list):
for d in d_list:
p = d.pop(Output.PARAM_PATH, None)
files = d.pop(Output.PARAM_FILES, None)
ret.append(_get(stage, p, d, files=files))
hash_name = d.pop(Output.PARAM_HASH, None)
ret.append(_get(stage, p, d, files=files, hash_name=hash_name))
return ret
dvc/dependency/param.py
+1
View File
@@ -46,6 +46,7 @@ class ParamsDependency(Dependency):
repo = repo or stage.repo
path = path or os.path.join(repo.root_dir, self.DEFAULT_PARAMS_FILE)
super().__init__(stage, path, repo=repo)
self.hash_name = self.PARAM_PARAMS
self.hash_info = hash_info
def dumpd(self, **kwargs):
dvc/output.py
+70 -19
View File
@@ -27,6 +27,7 @@ from dvc_data.hashfile import load as oload
from dvc_data.hashfile.build import build
from dvc_data.hashfile.checkout import checkout
from dvc_data.hashfile.db import HashFileDB, add_update_tree
from dvc_data.hashfile.hash import DEFAULT_ALGORITHM
from dvc_data.hashfile.hash_info import HashInfo
from dvc_data.hashfile.istextfile import istextfile
from dvc_data.hashfile.meta import Meta
@@ -64,7 +65,7 @@ CASE_SENSITIVE_CHECKSUM_SCHEMA = Any(
# NOTE: currently there are only 3 possible checksum names:
#
# 1) md5 (LOCAL, SSH);
# 1) md5 (LOCAL, SSH) (actually DVC 2.x md5-dos2unix)
# 2) etag (S3, GS, OSS, AZURE, HTTP);
# 3) checksum (HDFS);
#
@@ -73,7 +74,7 @@ CASE_SENSITIVE_CHECKSUM_SCHEMA = Any(
HDFS_PARAM_CHECKSUM = "checksum"
S3_PARAM_CHECKSUM = "etag"
CHECKSUMS_SCHEMA = {
LocalFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
"md5": CHECKSUM_SCHEMA, # DVC 2.x md5-dos2unix
HDFS_PARAM_CHECKSUM: CHECKSUM_SCHEMA,
S3_PARAM_CHECKSUM: CASE_SENSITIVE_CHECKSUM_SCHEMA,
}
@@ -95,6 +96,7 @@ def loadd_from(stage, d_list):
annot = {field: d.pop(field, None) for field in ANNOTATION_FIELDS}
files = d.pop(Output.PARAM_FILES, None)
push = d.pop(Output.PARAM_PUSH, True)
hash_name = d.pop(Output.PARAM_HASH, None)
ret.append(
_get(
stage,
@@ -108,6 +110,7 @@ def loadd_from(stage, d_list):
**annot,
files=files,
push=push,
hash_name=hash_name,
)
)
return ret
@@ -227,17 +230,29 @@ def merge_file_meta_from_cloud(entry: Dict) -> Dict:
return entry
def _serialize_tree_obj_to_files(obj: "Tree") -> List[Dict[str, Any]]:
def _serialize_tree_obj_to_files(obj: Tree) -> List[Dict[str, Any]]:
key = obj.PARAM_RELPATH
return sorted(
(
{key: posixpath.sep.join(parts), **hi.to_dict(), **meta.to_dict()}
{
key: posixpath.sep.join(parts),
**_serialize_hi_to_dict(hi),
**meta.to_dict(),
}
for parts, meta, hi in obj
),
key=itemgetter(key),
)
def _serialize_hi_to_dict(hash_info: Optional[HashInfo]) -> Dict[str, Any]:
if hash_info:
if hash_info.name == "md5-dos2unix":
return {"md5": hash_info.value}
return hash_info.to_dict()
return {}
class OutputDoesNotExistError(DvcException):
def __init__(self, path):
msg = f"output '{path}' does not exist"
@@ -297,6 +312,7 @@ class Output:
PARAM_REMOTE = "remote"
PARAM_PUSH = "push"
PARAM_CLOUD = "cloud"
PARAM_HASH = "hash"
DoesNotExistError: Type[DvcException] = OutputDoesNotExistError
IsNotFileOrDirError: Type[DvcException] = OutputIsNotFileOrDirError
@@ -321,6 +337,7 @@ class Output:
fs_config=None,
files: Optional[List[Dict[str, Any]]] = None,
push: bool = True,
hash_name: Optional[str] = DEFAULT_ALGORITHM,
):
self.annot = Annotation(
desc=desc, type=type, labels=labels or [], meta=meta or {}
@@ -392,17 +409,30 @@ class Output:
)
self.meta.version_id = version_id
if self.is_in_repo:
self.hash_name = "md5"
else:
self.hash_name = self.fs.PARAM_CHECKSUM
self.hash_info = HashInfo(
name=self.hash_name,
value=getattr(self.meta, self.hash_name, None),
)
self.hash_name, self.hash_info = self._compute_hash_info_from_meta(hash_name)
self._compute_meta_hash_info_from_files()
def _compute_hash_info_from_meta(
self, hash_name: Optional[str]
) -> Tuple[str, HashInfo]:
if self.is_in_repo:
if hash_name is None:
# Legacy 2.x output, use "md5-dos2unix" but read "md5" from
# file meta
hash_name = "md5-dos2unix"
meta_name = "md5"
else:
meta_name = hash_name
else:
hash_name = meta_name = self.fs.PARAM_CHECKSUM
assert hash_name
hash_info = HashInfo(
name=hash_name,
value=getattr(self.meta, meta_name, None),
)
return hash_name, hash_info
def _compute_meta_hash_info_from_files(self) -> None:
if self.files:
tree = Tree.from_list(self.files, hash_name=self.hash_name)
@@ -417,7 +447,7 @@ class Output:
self.meta.remote = first(f.get("remote") for f in self.files)
elif self.meta.nfiles or self.hash_info and self.hash_info.isdir:
self.meta.isdir = True
if not self.hash_info and self.hash_name != "md5":
if not self.hash_info and self.hash_name not in ("md5", "md5-dos2unix"):
md5 = getattr(self.meta, "md5", None)
if md5:
self.hash_info = HashInfo("md5", md5)
@@ -494,7 +524,12 @@ class Output:
@property
def cache(self):
odb_name = "repo" if self.is_in_repo else self.protocol
from dvc.cachemgr import LEGACY_HASH_NAMES
if self.is_in_repo:
odb_name = "legacy" if self.hash_name in LEGACY_HASH_NAMES else "repo"
else:
odb_name = self.protocol
odb = getattr(self.repo.cache, odb_name)
if self.use_cache and odb is None:
raise RemoteCacheRequiredError(self.fs.protocol, self.fs_path)
@@ -663,6 +698,8 @@ class Output:
if self.metric:
self.verify_metric()
if self.hash_name == "md5-dos2unix":
self.hash_name = "md5"
if self.use_cache:
_, self.meta, self.obj = build(
self.cache,
@@ -772,6 +809,8 @@ class Output:
return checkout_obj
def dumpd(self, **kwargs): # noqa: C901, PLR0912
from dvc.cachemgr import LEGACY_HASH_NAMES
ret: Dict[str, Any] = {}
with_files = (
(not self.IS_DEPENDENCY or self.stage.is_import)
@@ -782,7 +821,12 @@ class Output:
if not with_files:
meta_d = self.meta.to_dict()
meta_d.pop("isdir", None)
ret.update(self.hash_info.to_dict())
if self.hash_name in LEGACY_HASH_NAMES:
# 2.x checksums get serialized with file meta
name = "md5" if self.hash_name == "md5-dos2unix" else self.hash_name
ret.update({name: self.hash_info.value})
else:
ret.update(self.hash_info.to_dict())
ret.update(split_file_meta_from_cloud(meta_d))
if self.is_in_repo:
@@ -790,6 +834,9 @@ class Output:
else:
path = self.def_path
if self.hash_name not in LEGACY_HASH_NAMES:
ret[self.PARAM_HASH] = "md5"
ret[self.PARAM_PATH] = path
if not self.IS_DEPENDENCY:
@@ -969,7 +1016,7 @@ class Output:
odb,
from_info,
from_fs,
"md5",
DEFAULT_ALGORITHM,
upload=upload,
no_progress_bar=no_progress_bar,
)
@@ -1110,7 +1157,9 @@ class Output:
return {}
if self.remote:
remote = self.repo.cloud.get_remote_odb(name=self.remote)
remote = self.repo.cloud.get_remote_odb(
name=self.remote, hash_name=self.hash_name
)
else:
remote = None
@@ -1149,13 +1198,14 @@ class Output:
self.hash_name,
Meta.PARAM_SIZE,
Meta.PARAM_NFILES,
Output.PARAM_HASH,
]
for opt in ignored:
my.pop(opt, None)
other.pop(opt, None)
if my != other:
if my != other or self.hash_name != out.hash_name:
raise MergeError("unable to auto-merge outputs with different options")
if not out.is_dir_checksum:
@@ -1418,6 +1468,7 @@ ARTIFACT_SCHEMA = {
Required(Output.PARAM_PATH): str,
Output.PARAM_PERSIST: bool,
Output.PARAM_CLOUD: CLOUD_SCHEMA,
Output.PARAM_HASH: str,
}
DIR_FILES_SCHEMA: Dict[str, Any] = {
dvc/schema.py
+1
View File
@@ -33,6 +33,7 @@ DATA_SCHEMA = {
Required("path"): str,
Output.PARAM_CLOUD: CLOUD_SCHEMA,
Output.PARAM_FILES: [DIR_FILES_SCHEMA],
Output.PARAM_HASH: str,
}
LOCK_FILE_STAGE_SCHEMA = {
Required(StageParams.PARAM_CMD): Any(str, list),
dvc/stage/loader.py
+6 -5
View File
@@ -9,7 +9,6 @@ from funcy import get_in, lcat, once, project
from dvc import dependency, output
from dvc.parsing import FOREACH_KWD, JOIN, EntryNotFound
from dvc.utils.objects import cached_property
from dvc_data.hashfile.hash_info import HashInfo
from dvc_data.hashfile.meta import Meta
from . import PipelineStage, Stage, loads_from
@@ -19,7 +18,6 @@ from .utils import fill_stage_dependencies, resolve_paths
if TYPE_CHECKING:
from dvc.dvcfile import ProjectFile, SingleStageFile
from dvc.output import Output
logger = logging.getLogger(__name__)
@@ -52,7 +50,7 @@ class StageLoader(Mapping):
if not lock_data:
return
from dvc.output import merge_file_meta_from_cloud
from dvc.output import Output, merge_file_meta_from_cloud
assert isinstance(lock_data, dict)
items: Iterable[Tuple[str, "Output"]] = chain(
@@ -73,9 +71,12 @@ class StageLoader(Mapping):
info = info.copy()
info.pop("path", None)
hash_name = info.pop(Output.PARAM_HASH, None)
item.meta = Meta.from_dict(merge_file_meta_from_cloud(info))
hash_value = getattr(item.meta, item.hash_name, None)
item.hash_info = HashInfo(item.hash_name, hash_value)
# pylint: disable-next=protected-access
item.hash_name, item.hash_info = item._compute_hash_info_from_meta(
hash_name
)
files = get_in(checksums, [key, path, item.PARAM_FILES], None)
if files:
item.files = [merge_file_meta_from_cloud(f) for f in files]
dvc/stage/serialize.py
+9 -2
View File
@@ -149,13 +149,20 @@ def to_pipeline_file(stage: "PipelineStage"):
def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
from dvc.output import _serialize_tree_obj_to_files, split_file_meta_from_cloud
from dvc.cachemgr import LEGACY_HASH_NAMES
from dvc.output import (
_serialize_hi_to_dict,
_serialize_tree_obj_to_files,
split_file_meta_from_cloud,
)
from dvc_data.hashfile.tree import Tree
assert stage.cmd
def _dumpd(item: "Output"):
ret: Dict[str, Any] = {item.PARAM_PATH: item.def_path}
if item.hash_name not in LEGACY_HASH_NAMES:
ret[item.PARAM_HASH] = "md5"
if item.hash_info.isdir and kwargs.get("with_files"):
obj = item.obj or item.get_obj()
if obj:
@@ -167,7 +174,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
else:
meta_d = item.meta.to_dict()
meta_d.pop("isdir", None)
ret.update(item.hash_info.to_dict())
ret.update(_serialize_hi_to_dict(item.hash_info))
ret.update(split_file_meta_from_cloud(meta_d))
return ret
dvc/testing/workspace_tests.py
+3
View File
@@ -54,11 +54,13 @@ class TestImport:
f"- md5: {dir_md5}\n"
" size: 11\n"
" nfiles: 2\n"
" hash: md5\n"
" path: remote://workspace/dir\n"
"outs:\n"
"- md5: b6dcab6ccd17ca0a8bf4a215a37d14cc.dir\n"
" size: 11\n"
" nfiles: 2\n"
" hash: md5\n"
" path: dir\n"
)
@@ -177,6 +179,7 @@ class TestAdd:
"outs:\n"
f"- {hash_name}: {hash_value}\n"
" size: 4\n"
" hash: md5\n"
" path: remote://workspace/file\n"
)
assert (workspace / "file").read_text() == "file"
tests/func/test_add.py
+3
View File
@@ -51,6 +51,7 @@ def test_add(tmp_dir, dvc):
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
"path": "foo",
"size": 3,
"hash": "md5",
}
]
}
@@ -70,6 +71,7 @@ def test_add_executable(tmp_dir, dvc):
"path": "foo",
"size": 3,
"isexec": True,
"hash": "md5",
}
]
}
@@ -258,6 +260,7 @@ def test_add_external_relpath(tmp_dir, dvc, local_cloud):
"outs:\n"
"- md5: 8c7dd922ad47494fc02c388e12c00eac\n"
" size: 4\n"
" hash: md5\n"
f" path: {rel}\n"
)
assert fpath.read_text() == "file"
tests/func/test_checkout.py
+3 -2
View File
@@ -329,8 +329,8 @@ def test_checkout_moved_cache_dir_with_symlinks(tmp_dir, dvc):
assert system.is_symlink(os.path.join("data", "file"))
old_data_link = os.path.realpath(os.path.join("data", "file"))
old_cache_dir = dvc.cache.local.path
new_cache_dir = old_cache_dir + "_new"
old_cache_dir = str(tmp_dir / ".dvc" / "cache")
new_cache_dir = str(tmp_dir / ".dvc" / "cache_new")
os.rename(old_cache_dir, new_cache_dir)
ret = main(["cache", "dir", new_cache_dir])
@@ -801,6 +801,7 @@ def test_checkout_dir_compat(tmp_dir, dvc):
f"""\
outs:
- md5: {stage.outs[0].hash_info.value}
hash: md5
path: data
"""
),
tests/func/test_commit.py
+26 -1
View File
@@ -56,6 +56,7 @@ def test_commit_preserve_fields(tmp_dir, dvc):
key1: value1
key2: value2
remote: testremote
hash: md5
meta: some metadata
"""
)
@@ -77,6 +78,7 @@ def test_commit_preserve_fields(tmp_dir, dvc):
key1: value1
key2: value2
remote: testremote
hash: md5
md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
meta: some metadata
@@ -129,7 +131,10 @@ def test_commit_no_exec(tmp_dir, dvc):
def test_commit_granular_output(tmp_dir, dvc):
dvc.run(
name="mystage",
cmd=["echo foo>foo", "echo bar>bar"],
cmd=[
"python -c \"open('foo', 'wb').write(b'foo\\n')\"",
"python -c \"open('bar', 'wb').write(b'bar\\n')\"",
],
outs=["foo", "bar"],
no_commit=True,
)
@@ -248,6 +253,7 @@ def test_commit_updates_to_cloud_versioning_dir(tmp_dir, dvc):
"outs": [
{
"path": "data",
"hash": "md5",
"files": [
{
"size": 3,
@@ -280,6 +286,7 @@ def test_commit_updates_to_cloud_versioning_dir(tmp_dir, dvc):
"outs": [
{
"path": "data",
"hash": "md5",
"files": [
{
"size": 4,
@@ -297,3 +304,21 @@ def test_commit_updates_to_cloud_versioning_dir(tmp_dir, dvc):
}
]
}
def test_commit_dos2unix(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
(tmp_dir / "foo.dvc").dump(
{
"outs": [
{"path": "foo", "md5": "acbd18db4cc2f85cedef654fccc4a4d8", "size": 3},
]
}
)
legacy_content = (tmp_dir / "foo.dvc").read_text()
dvc.commit("foo.dvc")
assert (tmp_dir / "foo.dvc").read_text() == legacy_content
dvc.commit("foo.dvc", force=True)
content = (tmp_dir / "foo.dvc").read_text()
assert "hash: md5" in content
tests/func/test_dvcfile.py
+12
View File
@@ -402,3 +402,15 @@ def test_dvcfile_load_with_plots(tmp_dir, dvc):
assert all(
name in top_level_plots for name in ("path/to/plot", "path/to/another/plot")
)
def test_dvcfile_dos2unix(tmp_dir, dvc):
from dvc_data.hashfile.hash import HashInfo
(tmp_dir / "foo.dvc").dump({"outs": [{"md5": "abc123", "size": 3, "path": "foo"}]})
orig_content = (tmp_dir / "foo.dvc").read_text()
stage = dvc.stage.load_one("foo.dvc")
assert stage.outs[0].hash_name == "md5-dos2unix"
assert stage.outs[0].hash_info == HashInfo("md5-dos2unix", "abc123")
stage.dump()
assert (tmp_dir / "foo.dvc").read_text() == orig_content
tests/func/test_external_repo.py
+12 -2
View File
@@ -214,10 +214,20 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
shallow=False,
hardlink=True,
)
if os.name == "nt":
expected_gitignore_path = (
cache_dir / "d5" / "61e684092f0ff908aa82ee9cc1e594"
)
expected_dir_path = cache_dir / "0d" / "2086760aea091f1504eafc8843bb18.dir"
else:
expected_gitignore_path = (
cache_dir / "94" / "7d2b84e5aa88170e80dff467a5bfb6"
)
expected_dir_path = cache_dir / "e1" / "d9e8eae5374860ae025ec84cfd85c7.dir"
assert set(cache_dir.glob("??/*")) == {
cache_dir / "e1" / "d9e8eae5374860ae025ec84cfd85c7.dir",
expected_dir_path,
expected_gitignore_path,
cache_dir / "37" / "b51d194a7513e45b56f6524f2d51f2",
cache_dir / "94" / "7d2b84e5aa88170e80dff467a5bfb6",
cache_dir / "ac" / "bd18db4cc2f85cedef654fccc4a4d8",
}
tests/func/test_gc.py
+2
View File
@@ -428,6 +428,7 @@ def test_gc_not_in_remote_with_remote_field(
outs:
- path: foo
remote: other_remote
hash: md5
"""
)
tmp_dir.gen("foo.dvc", text)
@@ -459,6 +460,7 @@ def test_gc_cloud_remote_field(tmp_dir, scm, dvc, tmp_path_factory, mocker):
outs:
- path: foo
remote: other_remote
hash: md5
"""
)
tmp_dir.gen("foo.dvc", text)
tests/func/test_import.py
+5 -1
View File
@@ -247,7 +247,11 @@ def test_pull_import_no_download(tmp_dir, scm, dvc, erepo_dir):
stage = load_file(dvc, "foo_imported.dvc").stage
assert stage.outs[0].hash_info.value == "bdb8641831d8fcb03939637e09011c21.dir"
if os.name == "nt":
expected_hash = "2e798234df5f782340ac3ce046f8dfae.dir"
else:
expected_hash = "bdb8641831d8fcb03939637e09011c21.dir"
assert stage.outs[0].hash_info.value == expected_hash
assert stage.outs[0].meta.size == size
assert stage.outs[0].meta.nfiles == 3
tests/func/test_import_url.py
+4 -2
View File
@@ -108,7 +108,7 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
class TestImport(_TestImport):
@pytest.fixture
def stage_md5(self):
return "dc24e1271084ee317ac3c2656fb8812b"
return "7033ee831f78a4dfec2fc71405516067"
@pytest.fixture
def dir_md5(self):
@@ -150,6 +150,7 @@ def test_import_url_preserve_fields(tmp_dir, dvc):
- path: foo # dep comment
md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
hash: md5
outs:
- path: bar # out comment
desc: out desc
@@ -161,8 +162,9 @@ def test_import_url_preserve_fields(tmp_dir, dvc):
key: value
md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
hash: md5
meta: some metadata
md5: be7ade0aa89cc8d56e320867a9de9740
md5: 8fc199641730e3f512deac0bd9a0e0b6
frozen: true
"""
)
tests/func/test_install.py
+2
View File
@@ -118,6 +118,7 @@ def test_merge_driver_no_ancestor(tmp_dir, scm, dvc):
"- md5: 5ea40360f5b4ec688df672a4db9c17d1.dir\n"
" size: 6\n"
" nfiles: 2\n"
" hash: md5\n"
" path: data\n"
)
@@ -153,6 +154,7 @@ def test_merge_driver(tmp_dir, scm, dvc):
"- md5: 839ef9371606817569c1ee0e5f4ed233.dir\n"
" size: 12\n"
" nfiles: 3\n"
" hash: md5\n"
" path: data\n"
)
tests/func/test_lockfile.py
+6 -2
View File
@@ -43,8 +43,12 @@ def test_deps_outs_are_sorted_by_path(tmp_dir, dvc, run_head):
assert list(lock.keys()) == ["cmd", "deps", "outs"]
# `path` key appear first and then the `md5`
assert all(list(dep.keys()) == ["path", "md5", "size"] for dep in lock["deps"])
assert all(list(out.keys()) == ["path", "md5", "size"] for out in lock["outs"])
assert all(
list(dep.keys()) == ["path", "hash", "md5", "size"] for dep in lock["deps"]
)
assert all(
list(out.keys()) == ["path", "hash", "md5", "size"] for out in lock["outs"]
)
# deps are always sorted by the file path naming
assert list(map(itemgetter("path"), lock["deps"])) == sorted(deps)
tests/func/test_merge_driver.py
+19 -5
View File
@@ -127,10 +127,16 @@ def test_merge_conflict(tmp_dir, dvc, ancestor, our, their, error, caplog):
def test_merge_different_output_options(tmp_dir, dvc, caplog):
(tmp_dir / "ancestor").touch()
(tmp_dir / "our").write_text("outs:\n- md5: f123456789.dir\n path: path\n")
(tmp_dir / "our").write_text(
"outs:\n- md5: f123456789.dir\n hash: md5\n path: path\n"
)
(tmp_dir / "their").write_text(
"outs:\n- md5: f987654321.dir\n path: path\n cache: false\n"
"outs:\n"
"- md5: f987654321.dir\n"
" hash: md5\n"
" path: path\n"
" cache: false\n"
)
assert (
@@ -156,9 +162,13 @@ def test_merge_different_output_options(tmp_dir, dvc, caplog):
def test_merge_file(tmp_dir, dvc, caplog):
(tmp_dir / "ancestor").touch()
(tmp_dir / "our").write_text("outs:\n- md5: f123456789.dir\n path: path\n")
(tmp_dir / "our").write_text(
"outs:\n- md5: f123456789.dir\n hash: md5\n path: path\n"
)
(tmp_dir / "their").write_text("outs:\n- md5: f987654321\n path: path\n")
(tmp_dir / "their").write_text(
"outs:\n- md5: f987654321\n hash: md5\n path: path\n"
)
assert (
main(
@@ -186,12 +196,16 @@ def test_merge_non_dvc_add(tmp_dir, dvc, caplog):
(tmp_dir / "our").write_text(
"outs:\n"
"- md5: f123456789.dir\n"
" hash: md5\n"
" path: path\n"
"- md5: ff123456789.dir\n"
" hash: md5\n"
" path: another\n"
)
(tmp_dir / "their").write_text("outs:\n- md5: f987654321\n path: path\n")
(tmp_dir / "their").write_text(
"outs:\n- md5: f987654321\n hash: md5\n path: path\n"
)
assert (
main(
tests/func/test_move.py
+1
View File
@@ -194,6 +194,7 @@ def test_move_meta(tmp_dir, dvc):
outs:
- md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
hash: md5
path: bar
meta:
custom_key: 42
tests/func/test_run.py
+2
View File
@@ -562,10 +562,12 @@ def test_run_external_outputs(
" cmd: mycmd\n"
" deps:\n"
" - path: remote://workspace/foo\n"
" hash: md5\n"
f" {hash_name}: {foo_hash}\n"
" size: 3\n"
" outs:\n"
" - path: remote://workspace/bar\n"
" hash: md5\n"
f" {hash_name}: {bar_hash}\n"
" size: 3\n"
)
tests/func/test_stage.py
+15 -3
View File
@@ -139,6 +139,7 @@ def test_md5_ignores_annotations(tmp_dir, dvc):
{
"md5": "d3b07384d113edec49eaa6238ad5ff00",
"size": 4,
"hash": "md5",
"path": "foo",
"desc": "foo desc",
"type": "mytype",
@@ -149,10 +150,20 @@ def test_md5_ignores_annotations(tmp_dir, dvc):
}
(tmp_dir / "foo.dvc").dump(data)
stage = dvc.stage.load_one("foo.dvc")
assert compute_md5(stage) == "1822617147b53ae6f9eb4b3c87c0b6f3"
assert compute_md5(stage) == "cde267b60ef5a00e9a35cc1999ab83a3"
assert (
dict_md5({"outs": [{"md5": "d3b07384d113edec49eaa6238ad5ff00", "path": "foo"}]})
== "1822617147b53ae6f9eb4b3c87c0b6f3"
dict_md5(
{
"outs": [
{
"md5": "d3b07384d113edec49eaa6238ad5ff00",
"hash": "md5",
"path": "foo",
}
]
}
)
== "cde267b60ef5a00e9a35cc1999ab83a3"
)
@@ -164,6 +175,7 @@ def test_meta_desc_is_preserved(tmp_dir, dvc):
{
"md5": "d3b07384d113edec49eaa6238ad5ff00",
"size": 4,
"hash": "md5",
"path": "foo",
"desc": "foo desc",
"type": "mytype",
tests/scripts.py
+10 -4
View File
@@ -9,11 +9,13 @@ if os.path.isfile(sys.argv[1]):
shutil.copyfile(sys.argv[1], sys.argv[2])
else:
shutil.copytree(sys.argv[1], sys.argv[2])
"""
""".replace(
"\r\n", "\n"
)
def _add_script(tmp_dir, path, contents=""):
script, *_ = tmp_dir.gen(path, contents)
script, *_ = tmp_dir.gen(path, contents.encode("utf-8"))
if hasattr(tmp_dir, "scm"):
tmp_dir.scm_add(path, commit=f"add {path}")
return script.fs_path
@@ -30,7 +32,9 @@ from pathlib import Path
with Path(sys.argv[1]).open("a+", encoding="utf-8") as f:
f.write("foo")
""",
""".replace(
"\r\n", "\n"
),
)
@@ -55,5 +59,7 @@ import sys
for file in sys.argv[1:]:
with open(file) as f, open(file +"-1","w+") as w:
w.write(f.readline())
""",
""".replace(
"\r\n", "\n"
),
)
tests/unit/fs/test_dvc.py
+5 -1
View File
@@ -542,7 +542,11 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
fs = DVCFileSystem(repo=dvc)
_, _, obj = build(dvc.cache.local, "dir", fs, "md5")
assert obj.hash_info == HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir")
if os.name == "nt":
expected_hash = "0d2086760aea091f1504eafc8843bb18.dir"
else:
expected_hash = "e1d9e8eae5374860ae025ec84cfd85c7.dir"
assert obj.hash_info == HashInfo("md5", expected_hash)
def test_get_hash_dirty_file(tmp_dir, dvc):
tests/unit/output/test_output.py
+1 -1
View File
@@ -175,7 +175,7 @@ def test_dumpd_cloud_versioning_dir(mocker):
out = Output(stage, "path", files=files)
dumpd = out.dumpd()
assert dumpd == {"path": "path", "files": files}
assert dumpd == {"path": "path", "hash": "md5", "files": files}
def test_version_aware_is_set_based_on_files(mocker):
tests/unit/stage/test_cache.py
+10 -10
View File
@@ -27,12 +27,12 @@ def test_stage_cache(tmp_dir, dvc, mocker):
cache_dir = os.path.join(
dvc.stage_cache.cache_dir,
"4b",
"4b495dc2b14e1ca5bd5f2765a99ddd8785523a8342efe6bcadac012c2db01165",
"c7",
"c7a85d71de1912c43d9c5b6218c71630d6277e8fc11e4ccf5e12b3c202234838",
)
cache_file = os.path.join(
cache_dir,
"78c427104a8e20216031c36d9c7620733066fff33ada254ad3fca551c1f8152b",
"5e2824029f8da9ef6c57131a638ceb65f27c02f16c85d71e85671c27daed0501",
)
assert os.path.isdir(cache_dir)
@@ -79,7 +79,7 @@ def test_stage_cache_params(tmp_dir, dvc, mocker):
)
cache_file = os.path.join(
cache_dir,
"202ea269108bf98bea3e15f978e7929864728956c9df8d927a5c7d74fc4fedc8",
"9ce1963a69beb1299800188647cd960b7afff101be19fd46226e32bb8be8ee44",
)
assert os.path.isdir(cache_dir)
@@ -122,12 +122,12 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker):
cache_dir = os.path.join(
dvc.stage_cache.cache_dir,
"b5",
"b5d5548c43725139aa3419eb50717e062fe9b81f866f401fd6fd778d2a97822d",
"6a",
"6ad5cce3347e9e96c77d4353d84e5c8cae8c9151c486c4ea3d3d79e9051800f1",
)
cache_file = os.path.join(
cache_dir,
"e0a59f6a5bd193032a4d1500abd89c9b08a2e9b26c724015e0bc6374295a3b9f",
"1fa3e1a9f785f8364ad185d62bd0813ce8794afcfc79410e985eac0443cc5462",
)
assert os.path.isdir(cache_dir)
@@ -162,14 +162,14 @@ def test_shared_stage_cache(tmp_dir, dvc, run_copy):
run_copy("foo", "bar", name="copy-foo-bar")
parent_cache_dir = os.path.join(dvc.stage_cache.cache_dir, "fd")
parent_cache_dir = os.path.join(dvc.stage_cache.cache_dir, "6d")
cache_dir = os.path.join(
parent_cache_dir,
"fdbe7847136ebe88f59a29ee5a568f893cab031f74f7d3ab050828b53fd0033a",
"6d4c6de74e7c0d60d2122e2063b4724a8c78a6799def6c7cf5093f45e7f2f3b7",
)
cache_file = os.path.join(
cache_dir,
"8716052b2074f5acf1a379529d47d6ef1c1f50c2281a7489b8d7ce251f234f86",
"435e34f80692059e79ec53346cbfe29fd9ee65b62f5f06f17f5950ce5a7408ea",
)
# sanity check
tests/unit/stage/test_loader_pipeline_file.py
+22 -3
View File
@@ -20,8 +20,8 @@ def stage_data():
def lock_data():
return {
"cmd": "command",
"deps": [{"path": "foo", "md5": "foo_checksum"}],
"outs": [{"path": "bar", "md5": "bar_checksum"}],
"deps": [{"path": "foo", "md5": "foo_checksum", "hash": "md5"}],
"outs": [{"path": "bar", "md5": "bar_checksum", "hash": "md5"}],
}
@@ -46,7 +46,9 @@ def test_fill_from_lock_outs_isexec(dvc):
stage,
{
"cmd": "command",
"outs": [{"path": "foo", "md5": "foo_checksum", "isexec": True}],
"outs": [
{"path": "foo", "md5": "foo_checksum", "isexec": True, "hash": "md5"}
],
},
)
@@ -253,3 +255,20 @@ def test_load_stage_mapping(dvc, stage_data, lock_data):
assert "stage1" not in dvcfile.stages
assert dvcfile.stages.keys() == {"stage"}
assert isinstance(dvcfile.stages["stage"], PipelineStage)
def test_fill_from_lock_dos2unix(dvc):
lock_data = {
"cmd": "command",
"deps": [{"path": "foo", "md5": "foo_checksum"}],
"outs": [{"path": "bar", "md5": "bar_checksum"}],
}
stage = create_stage(PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"])
for item in chain(stage.deps, stage.outs):
assert not item.hash_info
StageLoader.fill_from_lock(stage, lock_data)
assert stage.deps[0].hash_info == HashInfo("md5-dos2unix", "foo_checksum")
assert stage.outs[0].hash_info == HashInfo("md5-dos2unix", "bar_checksum")
tests/unit/stage/test_serialize_pipeline_lock.py
+34 -11
View File
@@ -33,7 +33,10 @@ def test_lock_deps(dvc):
assert to_single_stage_lockfile(stage) == OrderedDict(
[
("cmd", "command"),
("deps", [OrderedDict([("path", "input"), ("md5", "md-five")])]),
(
"deps",
[OrderedDict([("hash", "md5"), ("path", "input"), ("md5", "md-five")])],
),
]
)
@@ -48,8 +51,12 @@ def test_lock_deps_order(dvc):
(
"deps",
[
OrderedDict([("path", "input0"), ("md5", "md-zer0")]),
OrderedDict([("path", "input1"), ("md5", "md-one1")]),
OrderedDict(
[("hash", "md5"), ("path", "input0"), ("md5", "md-zer0")]
),
OrderedDict(
[("hash", "md5"), ("path", "input1"), ("md5", "md-one1")]
),
],
),
]
@@ -134,7 +141,10 @@ def test_lock_outs(dvc, typ):
assert to_single_stage_lockfile(stage) == OrderedDict(
[
("cmd", "command"),
("outs", [OrderedDict([("path", "input"), ("md5", "md-five")])]),
(
"outs",
[OrderedDict([("hash", "md5"), ("path", "input"), ("md5", "md-five")])],
),
]
)
@@ -152,6 +162,7 @@ def test_lock_outs_isexec(dvc, typ):
[
OrderedDict(
[
("hash", "md5"),
("path", "input"),
("md5", "md-five"),
("isexec", True),
@@ -174,8 +185,12 @@ def test_lock_outs_order(dvc, typ):
(
"outs",
[
OrderedDict([("path", "input0"), ("md5", "md-zer0")]),
OrderedDict([("path", "input1"), ("md5", "md-one1")]),
OrderedDict(
[("hash", "md5"), ("path", "input0"), ("md5", "md-zer0")]
),
OrderedDict(
[("hash", "md5"), ("path", "input1"), ("md5", "md-one1")]
),
],
),
]
@@ -190,7 +205,15 @@ def test_dump_nondefault_hash(dvc):
("cmd", "command"),
(
"deps",
[OrderedDict([("path", "s3://dvc-temp/file"), ("md5", "value")])],
[
OrderedDict(
[
("hash", "md5"),
("path", "s3://dvc-temp/file"),
("md5", "value"),
]
)
],
),
]
)
@@ -214,9 +237,9 @@ def test_order(dvc):
assert to_single_stage_lockfile(stage) == OrderedDict(
[
("cmd", "command"),
("deps", [{"path": "input", "md5": "md-five"}]),
("deps", [{"hash": "md5", "path": "input", "md5": "md-five"}]),
("params", {"params.yaml": {"foo-param": "value"}}),
("outs", [{"path": "output", "md5": "md5-output"}]),
("outs", [{"hash": "md5", "path": "output", "md5": "md5-output"}]),
]
)
@@ -231,7 +254,7 @@ def test_to_lockfile(dvc):
"something": OrderedDict(
[
("cmd", "command"),
("deps", [{"path": "input", "md5": "md-five"}]),
("deps", [{"hash": "md5", "path": "input", "md5": "md-five"}]),
]
)
}
@@ -259,4 +282,4 @@ def test_to_single_stage_lockfile_cloud_versioning_dir(dvc):
stage.outs[0].files = files
e = _to_single_stage_lockfile(stage, with_files=True)
assert Schema(e)
assert e["outs"][0] == {"path": "dir", "files": files}
assert e["outs"][0] == {"hash": "md5", "path": "dir", "files": files}