Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

output: serialize 3.x hash name in 'hash' field

Browse Source
This commit is contained in:
Peter Rowlands
2023-06-09 15:49:44 +03:00
committed by Ruslan Kuprieiev
parent 603f5fdba1
commit 2cd33e10ac
27 changed files with 286 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/func/test_add.py
+3
View File
@@ -51,6 +51,7 @@ def test_add(tmp_dir, dvc):
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
"path": "foo",
"size": 3,
"hash": "md5",
}
]
}
@@ -70,6 +71,7 @@ def test_add_executable(tmp_dir, dvc):
"path": "foo",
"size": 3,
"isexec": True,
"hash": "md5",
}
]
}
@@ -258,6 +260,7 @@ def test_add_external_relpath(tmp_dir, dvc, local_cloud):
"outs:\n"
"- md5: 8c7dd922ad47494fc02c388e12c00eac\n"
" size: 4\n"
" hash: md5\n"
f" path: {rel}\n"
)
assert fpath.read_text() == "file"
tests/func/test_checkout.py
+3 -2
View File
@@ -329,8 +329,8 @@ def test_checkout_moved_cache_dir_with_symlinks(tmp_dir, dvc):
assert system.is_symlink(os.path.join("data", "file"))
old_data_link = os.path.realpath(os.path.join("data", "file"))
old_cache_dir = dvc.cache.local.path
new_cache_dir = old_cache_dir + "_new"
old_cache_dir = str(tmp_dir / ".dvc" / "cache")
new_cache_dir = str(tmp_dir / ".dvc" / "cache_new")
os.rename(old_cache_dir, new_cache_dir)
ret = main(["cache", "dir", new_cache_dir])
@@ -801,6 +801,7 @@ def test_checkout_dir_compat(tmp_dir, dvc):
f"""\
outs:
- md5: {stage.outs[0].hash_info.value}
hash: md5
path: data
"""
),
tests/func/test_commit.py
+26 -1
View File
@@ -56,6 +56,7 @@ def test_commit_preserve_fields(tmp_dir, dvc):
key1: value1
key2: value2
remote: testremote
hash: md5
meta: some metadata
"""
)
@@ -77,6 +78,7 @@ def test_commit_preserve_fields(tmp_dir, dvc):
key1: value1
key2: value2
remote: testremote
hash: md5
md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
meta: some metadata
@@ -129,7 +131,10 @@ def test_commit_no_exec(tmp_dir, dvc):
def test_commit_granular_output(tmp_dir, dvc):
dvc.run(
name="mystage",
cmd=["echo foo>foo", "echo bar>bar"],
cmd=[
"python -c \"open('foo', 'wb').write(b'foo\\n')\"",
"python -c \"open('bar', 'wb').write(b'bar\\n')\"",
],
outs=["foo", "bar"],
no_commit=True,
)
@@ -248,6 +253,7 @@ def test_commit_updates_to_cloud_versioning_dir(tmp_dir, dvc):
"outs": [
{
"path": "data",
"hash": "md5",
"files": [
{
"size": 3,
@@ -280,6 +286,7 @@ def test_commit_updates_to_cloud_versioning_dir(tmp_dir, dvc):
"outs": [
{
"path": "data",
"hash": "md5",
"files": [
{
"size": 4,
@@ -297,3 +304,21 @@ def test_commit_updates_to_cloud_versioning_dir(tmp_dir, dvc):
}
]
}
def test_commit_dos2unix(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
(tmp_dir / "foo.dvc").dump(
{
"outs": [
{"path": "foo", "md5": "acbd18db4cc2f85cedef654fccc4a4d8", "size": 3},
]
}
)
legacy_content = (tmp_dir / "foo.dvc").read_text()
dvc.commit("foo.dvc")
assert (tmp_dir / "foo.dvc").read_text() == legacy_content
dvc.commit("foo.dvc", force=True)
content = (tmp_dir / "foo.dvc").read_text()
assert "hash: md5" in content
tests/func/test_dvcfile.py
+12
View File
@@ -402,3 +402,15 @@ def test_dvcfile_load_with_plots(tmp_dir, dvc):
assert all(
name in top_level_plots for name in ("path/to/plot", "path/to/another/plot")
)
def test_dvcfile_dos2unix(tmp_dir, dvc):
from dvc_data.hashfile.hash import HashInfo
(tmp_dir / "foo.dvc").dump({"outs": [{"md5": "abc123", "size": 3, "path": "foo"}]})
orig_content = (tmp_dir / "foo.dvc").read_text()
stage = dvc.stage.load_one("foo.dvc")
assert stage.outs[0].hash_name == "md5-dos2unix"
assert stage.outs[0].hash_info == HashInfo("md5-dos2unix", "abc123")
stage.dump()
assert (tmp_dir / "foo.dvc").read_text() == orig_content
tests/func/test_external_repo.py
+12 -2
View File
@@ -214,10 +214,20 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
shallow=False,
hardlink=True,
)
if os.name == "nt":
expected_gitignore_path = (
cache_dir / "d5" / "61e684092f0ff908aa82ee9cc1e594"
)
expected_dir_path = cache_dir / "0d" / "2086760aea091f1504eafc8843bb18.dir"
else:
expected_gitignore_path = (
cache_dir / "94" / "7d2b84e5aa88170e80dff467a5bfb6"
)
expected_dir_path = cache_dir / "e1" / "d9e8eae5374860ae025ec84cfd85c7.dir"
assert set(cache_dir.glob("??/*")) == {
cache_dir / "e1" / "d9e8eae5374860ae025ec84cfd85c7.dir",
expected_dir_path,
expected_gitignore_path,
cache_dir / "37" / "b51d194a7513e45b56f6524f2d51f2",
cache_dir / "94" / "7d2b84e5aa88170e80dff467a5bfb6",
cache_dir / "ac" / "bd18db4cc2f85cedef654fccc4a4d8",
}
tests/func/test_gc.py
+2
View File
@@ -428,6 +428,7 @@ def test_gc_not_in_remote_with_remote_field(
outs:
- path: foo
remote: other_remote
hash: md5
"""
)
tmp_dir.gen("foo.dvc", text)
@@ -459,6 +460,7 @@ def test_gc_cloud_remote_field(tmp_dir, scm, dvc, tmp_path_factory, mocker):
outs:
- path: foo
remote: other_remote
hash: md5
"""
)
tmp_dir.gen("foo.dvc", text)
tests/func/test_import.py
+5 -1
View File
@@ -247,7 +247,11 @@ def test_pull_import_no_download(tmp_dir, scm, dvc, erepo_dir):
stage = load_file(dvc, "foo_imported.dvc").stage
assert stage.outs[0].hash_info.value == "bdb8641831d8fcb03939637e09011c21.dir"
if os.name == "nt":
expected_hash = "2e798234df5f782340ac3ce046f8dfae.dir"
else:
expected_hash = "bdb8641831d8fcb03939637e09011c21.dir"
assert stage.outs[0].hash_info.value == expected_hash
assert stage.outs[0].meta.size == size
assert stage.outs[0].meta.nfiles == 3
tests/func/test_import_url.py
+4 -2
View File
@@ -108,7 +108,7 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
class TestImport(_TestImport):
@pytest.fixture
def stage_md5(self):
return "dc24e1271084ee317ac3c2656fb8812b"
return "7033ee831f78a4dfec2fc71405516067"
@pytest.fixture
def dir_md5(self):
@@ -150,6 +150,7 @@ def test_import_url_preserve_fields(tmp_dir, dvc):
- path: foo # dep comment
md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
hash: md5
outs:
- path: bar # out comment
desc: out desc
@@ -161,8 +162,9 @@ def test_import_url_preserve_fields(tmp_dir, dvc):
key: value
md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
hash: md5
meta: some metadata
md5: be7ade0aa89cc8d56e320867a9de9740
md5: 8fc199641730e3f512deac0bd9a0e0b6
frozen: true
"""
)
tests/func/test_install.py
+2
View File
@@ -118,6 +118,7 @@ def test_merge_driver_no_ancestor(tmp_dir, scm, dvc):
"- md5: 5ea40360f5b4ec688df672a4db9c17d1.dir\n"
" size: 6\n"
" nfiles: 2\n"
" hash: md5\n"
" path: data\n"
)
@@ -153,6 +154,7 @@ def test_merge_driver(tmp_dir, scm, dvc):
"- md5: 839ef9371606817569c1ee0e5f4ed233.dir\n"
" size: 12\n"
" nfiles: 3\n"
" hash: md5\n"
" path: data\n"
)
tests/func/test_lockfile.py
+6 -2
View File
@@ -43,8 +43,12 @@ def test_deps_outs_are_sorted_by_path(tmp_dir, dvc, run_head):
assert list(lock.keys()) == ["cmd", "deps", "outs"]
# `path` key appear first and then the `md5`
assert all(list(dep.keys()) == ["path", "md5", "size"] for dep in lock["deps"])
assert all(list(out.keys()) == ["path", "md5", "size"] for out in lock["outs"])
assert all(
list(dep.keys()) == ["path", "hash", "md5", "size"] for dep in lock["deps"]
)
assert all(
list(out.keys()) == ["path", "hash", "md5", "size"] for out in lock["outs"]
)
# deps are always sorted by the file path naming
assert list(map(itemgetter("path"), lock["deps"])) == sorted(deps)
tests/func/test_merge_driver.py
+19 -5
View File
@@ -127,10 +127,16 @@ def test_merge_conflict(tmp_dir, dvc, ancestor, our, their, error, caplog):
def test_merge_different_output_options(tmp_dir, dvc, caplog):
(tmp_dir / "ancestor").touch()
(tmp_dir / "our").write_text("outs:\n- md5: f123456789.dir\n path: path\n")
(tmp_dir / "our").write_text(
"outs:\n- md5: f123456789.dir\n hash: md5\n path: path\n"
)
(tmp_dir / "their").write_text(
"outs:\n- md5: f987654321.dir\n path: path\n cache: false\n"
"outs:\n"
"- md5: f987654321.dir\n"
" hash: md5\n"
" path: path\n"
" cache: false\n"
)
assert (
@@ -156,9 +162,13 @@ def test_merge_different_output_options(tmp_dir, dvc, caplog):
def test_merge_file(tmp_dir, dvc, caplog):
(tmp_dir / "ancestor").touch()
(tmp_dir / "our").write_text("outs:\n- md5: f123456789.dir\n path: path\n")
(tmp_dir / "our").write_text(
"outs:\n- md5: f123456789.dir\n hash: md5\n path: path\n"
)
(tmp_dir / "their").write_text("outs:\n- md5: f987654321\n path: path\n")
(tmp_dir / "their").write_text(
"outs:\n- md5: f987654321\n hash: md5\n path: path\n"
)
assert (
main(
@@ -186,12 +196,16 @@ def test_merge_non_dvc_add(tmp_dir, dvc, caplog):
(tmp_dir / "our").write_text(
"outs:\n"
"- md5: f123456789.dir\n"
" hash: md5\n"
" path: path\n"
"- md5: ff123456789.dir\n"
" hash: md5\n"
" path: another\n"
)
(tmp_dir / "their").write_text("outs:\n- md5: f987654321\n path: path\n")
(tmp_dir / "their").write_text(
"outs:\n- md5: f987654321\n hash: md5\n path: path\n"
)
assert (
main(
tests/func/test_move.py
+1
View File
@@ -194,6 +194,7 @@ def test_move_meta(tmp_dir, dvc):
outs:
- md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
hash: md5
path: bar
meta:
custom_key: 42
tests/func/test_run.py
+2
View File
@@ -562,10 +562,12 @@ def test_run_external_outputs(
" cmd: mycmd\n"
" deps:\n"
" - path: remote://workspace/foo\n"
" hash: md5\n"
f" {hash_name}: {foo_hash}\n"
" size: 3\n"
" outs:\n"
" - path: remote://workspace/bar\n"
" hash: md5\n"
f" {hash_name}: {bar_hash}\n"
" size: 3\n"
)
tests/func/test_stage.py
+15 -3
View File
@@ -139,6 +139,7 @@ def test_md5_ignores_annotations(tmp_dir, dvc):
{
"md5": "d3b07384d113edec49eaa6238ad5ff00",
"size": 4,
"hash": "md5",
"path": "foo",
"desc": "foo desc",
"type": "mytype",
@@ -149,10 +150,20 @@ def test_md5_ignores_annotations(tmp_dir, dvc):
}
(tmp_dir / "foo.dvc").dump(data)
stage = dvc.stage.load_one("foo.dvc")
assert compute_md5(stage) == "1822617147b53ae6f9eb4b3c87c0b6f3"
assert compute_md5(stage) == "cde267b60ef5a00e9a35cc1999ab83a3"
assert (
dict_md5({"outs": [{"md5": "d3b07384d113edec49eaa6238ad5ff00", "path": "foo"}]})
== "1822617147b53ae6f9eb4b3c87c0b6f3"
dict_md5(
{
"outs": [
{
"md5": "d3b07384d113edec49eaa6238ad5ff00",
"hash": "md5",
"path": "foo",
}
]
}
)
== "cde267b60ef5a00e9a35cc1999ab83a3"
)
@@ -164,6 +175,7 @@ def test_meta_desc_is_preserved(tmp_dir, dvc):
{
"md5": "d3b07384d113edec49eaa6238ad5ff00",
"size": 4,
"hash": "md5",
"path": "foo",
"desc": "foo desc",
"type": "mytype",
tests/scripts.py
+10 -4
View File
@@ -9,11 +9,13 @@ if os.path.isfile(sys.argv[1]):
shutil.copyfile(sys.argv[1], sys.argv[2])
else:
shutil.copytree(sys.argv[1], sys.argv[2])
"""
""".replace(
"\r\n", "\n"
)
def _add_script(tmp_dir, path, contents=""):
script, *_ = tmp_dir.gen(path, contents)
script, *_ = tmp_dir.gen(path, contents.encode("utf-8"))
if hasattr(tmp_dir, "scm"):
tmp_dir.scm_add(path, commit=f"add {path}")
return script.fs_path
@@ -30,7 +32,9 @@ from pathlib import Path
with Path(sys.argv[1]).open("a+", encoding="utf-8") as f:
f.write("foo")
""",
""".replace(
"\r\n", "\n"
),
)
@@ -55,5 +59,7 @@ import sys
for file in sys.argv[1:]:
with open(file) as f, open(file +"-1","w+") as w:
w.write(f.readline())
""",
""".replace(
"\r\n", "\n"
),
)
tests/unit/fs/test_dvc.py
+5 -1
View File
@@ -542,7 +542,11 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
fs = DVCFileSystem(repo=dvc)
_, _, obj = build(dvc.cache.local, "dir", fs, "md5")
assert obj.hash_info == HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir")
if os.name == "nt":
expected_hash = "0d2086760aea091f1504eafc8843bb18.dir"
else:
expected_hash = "e1d9e8eae5374860ae025ec84cfd85c7.dir"
assert obj.hash_info == HashInfo("md5", expected_hash)
def test_get_hash_dirty_file(tmp_dir, dvc):
tests/unit/output/test_output.py
+1 -1
View File
@@ -175,7 +175,7 @@ def test_dumpd_cloud_versioning_dir(mocker):
out = Output(stage, "path", files=files)
dumpd = out.dumpd()
assert dumpd == {"path": "path", "files": files}
assert dumpd == {"path": "path", "hash": "md5", "files": files}
def test_version_aware_is_set_based_on_files(mocker):
tests/unit/stage/test_cache.py
+10 -10
View File
@@ -27,12 +27,12 @@ def test_stage_cache(tmp_dir, dvc, mocker):
cache_dir = os.path.join(
dvc.stage_cache.cache_dir,
"4b",
"4b495dc2b14e1ca5bd5f2765a99ddd8785523a8342efe6bcadac012c2db01165",
"c7",
"c7a85d71de1912c43d9c5b6218c71630d6277e8fc11e4ccf5e12b3c202234838",
)
cache_file = os.path.join(
cache_dir,
"78c427104a8e20216031c36d9c7620733066fff33ada254ad3fca551c1f8152b",
"5e2824029f8da9ef6c57131a638ceb65f27c02f16c85d71e85671c27daed0501",
)
assert os.path.isdir(cache_dir)
@@ -79,7 +79,7 @@ def test_stage_cache_params(tmp_dir, dvc, mocker):
)
cache_file = os.path.join(
cache_dir,
"202ea269108bf98bea3e15f978e7929864728956c9df8d927a5c7d74fc4fedc8",
"9ce1963a69beb1299800188647cd960b7afff101be19fd46226e32bb8be8ee44",
)
assert os.path.isdir(cache_dir)
@@ -122,12 +122,12 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker):
cache_dir = os.path.join(
dvc.stage_cache.cache_dir,
"b5",
"b5d5548c43725139aa3419eb50717e062fe9b81f866f401fd6fd778d2a97822d",
"6a",
"6ad5cce3347e9e96c77d4353d84e5c8cae8c9151c486c4ea3d3d79e9051800f1",
)
cache_file = os.path.join(
cache_dir,
"e0a59f6a5bd193032a4d1500abd89c9b08a2e9b26c724015e0bc6374295a3b9f",
"1fa3e1a9f785f8364ad185d62bd0813ce8794afcfc79410e985eac0443cc5462",
)
assert os.path.isdir(cache_dir)
@@ -162,14 +162,14 @@ def test_shared_stage_cache(tmp_dir, dvc, run_copy):
run_copy("foo", "bar", name="copy-foo-bar")
parent_cache_dir = os.path.join(dvc.stage_cache.cache_dir, "fd")
parent_cache_dir = os.path.join(dvc.stage_cache.cache_dir, "6d")
cache_dir = os.path.join(
parent_cache_dir,
"fdbe7847136ebe88f59a29ee5a568f893cab031f74f7d3ab050828b53fd0033a",
"6d4c6de74e7c0d60d2122e2063b4724a8c78a6799def6c7cf5093f45e7f2f3b7",
)
cache_file = os.path.join(
cache_dir,
"8716052b2074f5acf1a379529d47d6ef1c1f50c2281a7489b8d7ce251f234f86",
"435e34f80692059e79ec53346cbfe29fd9ee65b62f5f06f17f5950ce5a7408ea",
)
# sanity check
tests/unit/stage/test_loader_pipeline_file.py
+22 -3
View File
@@ -20,8 +20,8 @@ def stage_data():
def lock_data():
return {
"cmd": "command",
"deps": [{"path": "foo", "md5": "foo_checksum"}],
"outs": [{"path": "bar", "md5": "bar_checksum"}],
"deps": [{"path": "foo", "md5": "foo_checksum", "hash": "md5"}],
"outs": [{"path": "bar", "md5": "bar_checksum", "hash": "md5"}],
}
@@ -46,7 +46,9 @@ def test_fill_from_lock_outs_isexec(dvc):
stage,
{
"cmd": "command",
"outs": [{"path": "foo", "md5": "foo_checksum", "isexec": True}],
"outs": [
{"path": "foo", "md5": "foo_checksum", "isexec": True, "hash": "md5"}
],
},
)
@@ -253,3 +255,20 @@ def test_load_stage_mapping(dvc, stage_data, lock_data):
assert "stage1" not in dvcfile.stages
assert dvcfile.stages.keys() == {"stage"}
assert isinstance(dvcfile.stages["stage"], PipelineStage)
def test_fill_from_lock_dos2unix(dvc):
lock_data = {
"cmd": "command",
"deps": [{"path": "foo", "md5": "foo_checksum"}],
"outs": [{"path": "bar", "md5": "bar_checksum"}],
}
stage = create_stage(PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"])
for item in chain(stage.deps, stage.outs):
assert not item.hash_info
StageLoader.fill_from_lock(stage, lock_data)
assert stage.deps[0].hash_info == HashInfo("md5-dos2unix", "foo_checksum")
assert stage.outs[0].hash_info == HashInfo("md5-dos2unix", "bar_checksum")
tests/unit/stage/test_serialize_pipeline_lock.py
+34 -11
View File
@@ -33,7 +33,10 @@ def test_lock_deps(dvc):
assert to_single_stage_lockfile(stage) == OrderedDict(
[
("cmd", "command"),
("deps", [OrderedDict([("path", "input"), ("md5", "md-five")])]),
(
"deps",
[OrderedDict([("hash", "md5"), ("path", "input"), ("md5", "md-five")])],
),
]
)
@@ -48,8 +51,12 @@ def test_lock_deps_order(dvc):
(
"deps",
[
OrderedDict([("path", "input0"), ("md5", "md-zer0")]),
OrderedDict([("path", "input1"), ("md5", "md-one1")]),
OrderedDict(
[("hash", "md5"), ("path", "input0"), ("md5", "md-zer0")]
),
OrderedDict(
[("hash", "md5"), ("path", "input1"), ("md5", "md-one1")]
),
],
),
]
@@ -134,7 +141,10 @@ def test_lock_outs(dvc, typ):
assert to_single_stage_lockfile(stage) == OrderedDict(
[
("cmd", "command"),
("outs", [OrderedDict([("path", "input"), ("md5", "md-five")])]),
(
"outs",
[OrderedDict([("hash", "md5"), ("path", "input"), ("md5", "md-five")])],
),
]
)
@@ -152,6 +162,7 @@ def test_lock_outs_isexec(dvc, typ):
[
OrderedDict(
[
("hash", "md5"),
("path", "input"),
("md5", "md-five"),
("isexec", True),
@@ -174,8 +185,12 @@ def test_lock_outs_order(dvc, typ):
(
"outs",
[
OrderedDict([("path", "input0"), ("md5", "md-zer0")]),
OrderedDict([("path", "input1"), ("md5", "md-one1")]),
OrderedDict(
[("hash", "md5"), ("path", "input0"), ("md5", "md-zer0")]
),
OrderedDict(
[("hash", "md5"), ("path", "input1"), ("md5", "md-one1")]
),
],
),
]
@@ -190,7 +205,15 @@ def test_dump_nondefault_hash(dvc):
("cmd", "command"),
(
"deps",
[OrderedDict([("path", "s3://dvc-temp/file"), ("md5", "value")])],
[
OrderedDict(
[
("hash", "md5"),
("path", "s3://dvc-temp/file"),
("md5", "value"),
]
)
],
),
]
)
@@ -214,9 +237,9 @@ def test_order(dvc):
assert to_single_stage_lockfile(stage) == OrderedDict(
[
("cmd", "command"),
("deps", [{"path": "input", "md5": "md-five"}]),
("deps", [{"hash": "md5", "path": "input", "md5": "md-five"}]),
("params", {"params.yaml": {"foo-param": "value"}}),
("outs", [{"path": "output", "md5": "md5-output"}]),
("outs", [{"hash": "md5", "path": "output", "md5": "md5-output"}]),
]
)
@@ -231,7 +254,7 @@ def test_to_lockfile(dvc):
"something": OrderedDict(
[
("cmd", "command"),
("deps", [{"path": "input", "md5": "md-five"}]),
("deps", [{"hash": "md5", "path": "input", "md5": "md-five"}]),
]
)
}
@@ -259,4 +282,4 @@ def test_to_single_stage_lockfile_cloud_versioning_dir(dvc):
stage.outs[0].files = files
e = _to_single_stage_lockfile(stage, with_files=True)
assert Schema(e)
assert e["outs"][0] == {"path": "dir", "files": files}
assert e["outs"][0] == {"hash": "md5", "path": "dir", "files": files}