From 59a4c4f67c8e841177b387dcb05458943e995aa1 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 1 Mar 2023 12:56:04 +0200 Subject: [PATCH] remote: set hash_name as md5 Fixes #7977 --- dvc/data_cloud.py | 2 +- dvc/testing/workspace_tests.py | 82 ++++++++++++++++++++++++++++++++++ tests/func/test_add.py | 20 --------- tests/func/test_import_url.py | 58 ------------------------ 4 files changed, 83 insertions(+), 79 deletions(-) diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 6e9893b8c..b5094ca6e 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -35,7 +35,7 @@ class Remote: if self.worktree: path = self.fs.path.join(path, ".dvc", "cache") - return get_odb(self.fs, path, **self.config) + return get_odb(self.fs, path, hash_name="md5", **self.config) class DataCloud: diff --git a/dvc/testing/workspace_tests.py b/dvc/testing/workspace_tests.py index 139ad194a..1f3cc794f 100644 --- a/dvc/testing/workspace_tests.py +++ b/dvc/testing/workspace_tests.py @@ -282,3 +282,85 @@ class TestGetUrl: def test_get_url_nonexistent(self, cloud): with pytest.raises(URLMissingError): Repo.get_url(str(cloud / "nonexistent"), config=cloud.config) + + +class TestToRemote: + def test_add_to_remote(self, tmp_dir, dvc, remote, workspace): + workspace.gen("foo", "foo") + + url = "remote://workspace/foo" + [stage] = dvc.add(url, to_remote=True) + + assert not (tmp_dir / "foo").exists() + assert (tmp_dir / "foo.dvc").exists() + + assert len(stage.deps) == 0 + assert len(stage.outs) == 1 + + hash_info = stage.outs[0].hash_info + meta = stage.outs[0].meta + assert hash_info.name == "md5" + assert hash_info.value == "acbd18db4cc2f85cedef654fccc4a4d8" + assert (remote / "ac" / "bd18db4cc2f85cedef654fccc4a4d8").read_text() == "foo" + assert meta.size == len("foo") + + def test_import_url_to_remote_file(self, tmp_dir, dvc, workspace, remote): + workspace.gen("foo", "foo") + + url = "remote://workspace/foo" + stage = dvc.imp_url(url, to_remote=True) + + assert stage.deps[0].hash_info.value is not None + assert not (tmp_dir / "foo").exists() + assert (tmp_dir / "foo.dvc").exists() + + assert len(stage.deps) == 1 + assert stage.deps[0].def_path == url + assert len(stage.outs) == 1 + + hash_info = stage.outs[0].hash_info + assert hash_info.name == "md5" + assert hash_info.value == "acbd18db4cc2f85cedef654fccc4a4d8" + assert (remote / "ac" / "bd18db4cc2f85cedef654fccc4a4d8").read_text() == "foo" + assert stage.outs[0].meta.size == len("foo") + + def test_import_url_to_remote_dir(self, tmp_dir, dvc, workspace, remote): + import json + + workspace.gen( + { + "data": { + "foo": "foo", + "bar": "bar", + "sub_dir": {"baz": "sub_dir/baz"}, + } + } + ) + + url = "remote://workspace/data" + stage = dvc.imp_url(url, to_remote=True) + + assert not (tmp_dir / "data").exists() + assert (tmp_dir / "data.dvc").exists() + + assert len(stage.deps) == 1 + assert stage.deps[0].def_path == url + assert len(stage.outs) == 1 + + hash_info = stage.outs[0].hash_info + assert hash_info.name == "md5" + assert hash_info.value == "55d05978954d1b2cd7b06aedda9b9e43.dir" + file_parts = json.loads( + (remote / "55" / "d05978954d1b2cd7b06aedda9b9e43.dir").read_text() + ) + + assert len(file_parts) == 3 + assert {file_part["relpath"] for file_part in file_parts} == { + "foo", + "bar", + "sub_dir/baz", + } + + for file_part in file_parts: + md5 = file_part["md5"] + assert (remote / md5[:2] / md5[2:]).read_text() == file_part["relpath"] diff --git a/tests/func/test_add.py b/tests/func/test_add.py index ab48de101..bc9e49896 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -898,26 +898,6 @@ def test_add_long_fname(tmp_dir, dvc): assert (tmp_dir / "data").read_text() == {name: "foo"} -def test_add_to_remote(tmp_dir, dvc, remote, workspace): - workspace.gen("foo", "foo") - - url = "remote://workspace/foo" - [stage] = dvc.add(url, to_remote=True) - - assert not (tmp_dir / "foo").exists() - assert (tmp_dir / "foo.dvc").exists() - - assert len(stage.deps) == 0 - assert len(stage.outs) == 1 - - hash_info = stage.outs[0].hash_info - meta = stage.outs[0].meta - with open(remote.oid_to_path(hash_info.value), encoding="utf-8") as stream: - assert stream.read() == "foo" - - assert meta.size == len("foo") - - def test_add_to_remote_absolute(tmp_dir, make_tmp_dir, dvc, remote): tmp_abs_dir = make_tmp_dir("abs") tmp_foo = tmp_abs_dir / "foo" diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index 4380afe7c..1735d5e65 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -1,4 +1,3 @@ -import json import os import textwrap from uuid import uuid4 @@ -171,63 +170,6 @@ def test_import_url_preserve_fields(tmp_dir, dvc): ) -def test_import_url_to_remote_single_file(tmp_dir, dvc, workspace, local_remote): - workspace.gen("foo", "foo") - - url = "remote://workspace/foo" - stage = dvc.imp_url(url, to_remote=True) - - assert stage.deps[0].hash_info.value is not None - assert not (tmp_dir / "foo").exists() - assert (tmp_dir / "foo.dvc").exists() - - assert len(stage.deps) == 1 - assert stage.deps[0].def_path == url - assert len(stage.outs) == 1 - - hash_info = stage.outs[0].hash_info - with open(local_remote.oid_to_path(hash_info.value), encoding="utf-8") as fobj: - assert fobj.read() == "foo" - assert stage.outs[0].meta.size == len("foo") - - -def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote): - workspace.gen( - { - "data": { - "foo": "foo", - "bar": "bar", - "sub_dir": {"baz": "sub_dir/baz"}, - } - } - ) - - url = "remote://workspace/data" - stage = dvc.imp_url(url, to_remote=True) - - assert not (tmp_dir / "data").exists() - assert (tmp_dir / "data.dvc").exists() - - assert len(stage.deps) == 1 - assert stage.deps[0].def_path == url - assert len(stage.outs) == 1 - - hash_info = stage.outs[0].hash_info - with open(local_remote.oid_to_path(hash_info.value), encoding="utf-8") as stream: - file_parts = json.load(stream) - - assert len(file_parts) == 3 - assert {file_part["relpath"] for file_part in file_parts} == { - "foo", - "bar", - "sub_dir/baz", - } - - for file_part in file_parts: - with open(local_remote.oid_to_path(file_part["md5"]), encoding="utf-8") as fobj: - assert fobj.read() == file_part["relpath"] - - def test_import_url_to_remote_absolute(tmp_dir, make_tmp_dir, dvc, local_remote): tmp_abs_dir = make_tmp_dir("abs") tmp_foo = tmp_abs_dir / "foo"