Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvc: drop cached external outputs

Browse Source 
Fixes #9531

Docs https://github.com/iterative/dvc.org/pull/4574 Kudos @dberenbaum
This commit is contained in:
Ruslan Kuprieiev
2023-06-09 18:34:36 +03:00
parent 8b1c1a31f3
commit af1fb722c8
20 changed files with 69 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/func/test_add.py
+6 -68
View File
@@ -22,7 +22,6 @@ from dvc.output import (
)
from dvc.stage import Stage
from dvc.stage.exceptions import StageExternalOutputsError, StagePathNotFoundError
from dvc.testing.workspace_tests import TestAdd
from dvc.utils.fs import path_isin
from dvc.utils.serialize import YAMLFileCorruptedError
from dvc_data.hashfile.hash import file_md5
@@ -234,55 +233,6 @@ def test_add_filtered_files_in_dir(
assert stage.outs[0].def_path in expected_def_paths
class TestAddExternal(TestAdd):
@pytest.fixture
def hash_name(self):
return "md5"
@pytest.fixture
def hash_value(self):
return "8c7dd922ad47494fc02c388e12c00eac"
@pytest.fixture
def dir_hash_value(self):
return "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir"
def test_add_external_relpath(tmp_dir, dvc, local_cloud):
(fpath,) = local_cloud.gen("file", "file")
rel = os.path.relpath(fpath)
with pytest.raises(StageExternalOutputsError):
dvc.add(rel)
dvc.add(rel, external=True)
assert (tmp_dir / "file.dvc").read_text() == (
"outs:\n"
"- md5: 8c7dd922ad47494fc02c388e12c00eac\n"
" size: 4\n"
" hash: md5\n"
f" path: {rel}\n"
)
assert fpath.read_text() == "file"
assert dvc.status() == {}
def test_add_local_remote_file(tmp_dir, dvc):
"""
Making sure that 'remote' syntax is handled properly for local outs.
"""
tmp_dir.gen({"foo": "foo", "bar": "bar"})
tmp_dir.add_remote(url=tmp_dir.fs_path, name="myremote")
assert main(["add", "remote://myremote/foo"]) == 0
d = (tmp_dir / "foo.dvc").load_yaml()
assert d["outs"][0]["path"] == "remote://myremote/foo"
assert main(["add", (tmp_dir / "bar").fs_path]) == 0
d = (tmp_dir / "bar.dvc").load_yaml()
assert d["outs"][0]["path"] == "bar"
def test_cmd_add(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
ret = main(["add", "foo"])
@@ -755,15 +705,9 @@ def test_add_symlink_file(tmp_dir, dvc):
dvc.add(os.path.join("dir", "foo"))
@pytest.mark.parametrize("external", [True, False])
def test_add_symlink_dir(make_tmp_dir, tmp_dir, dvc, external):
if external:
data_dir = make_tmp_dir("data")
data_dir.gen({"foo": "foo"})
target = os.fspath(data_dir)
else:
tmp_dir.gen({"data": {"foo": "foo"}})
target = os.path.join(".", "data")
def test_add_symlink_dir(make_tmp_dir, tmp_dir, dvc):
tmp_dir.gen({"data": {"foo": "foo"}})
target = os.path.join(".", "data")
tmp_dir.gen({"data": {"foo": "foo"}})
@@ -774,15 +718,9 @@ def test_add_symlink_dir(make_tmp_dir, tmp_dir, dvc, external):
dvc.add("dir")
@pytest.mark.parametrize("external", [True, False])
def test_add_file_in_symlink_dir(make_tmp_dir, tmp_dir, dvc, external):
if external:
data_dir = make_tmp_dir("data")
data_dir.gen({"dir": {"foo": "foo"}})
target = os.fspath(data_dir / "dir")
else:
tmp_dir.gen({"data": {"foo": "foo"}})
target = os.path.join(".", "data")
def test_add_file_in_symlink_dir(make_tmp_dir, tmp_dir, dvc):
tmp_dir.gen({"data": {"foo": "foo"}})
target = os.path.join(".", "data")
(tmp_dir / "dir").symlink_to(target)
tests/func/test_gc.py
-28
View File
@@ -251,34 +251,6 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
assert _count_files(dvc.cache.local.path) == 0
def test_gc_external_output(tmp_dir, dvc, workspace):
workspace.gen({"foo": "foo", "bar": "bar"})
(foo_stage,) = dvc.add("remote://workspace/foo")
(bar_stage,) = dvc.add("remote://workspace/bar")
foo_hash = foo_stage.outs[0].hash_info.value
bar_hash = bar_stage.outs[0].hash_info.value
assert (
workspace / "cache" / "files" / "md5" / foo_hash[:2] / foo_hash[2:]
).read_text() == "foo"
assert (
workspace / "cache" / "files" / "md5" / bar_hash[:2] / bar_hash[2:]
).read_text() == "bar"
(tmp_dir / "foo.dvc").unlink()
dvc.gc(workspace=True)
assert not (
workspace / "cache" / "files" / "md5" / foo_hash[:2] / foo_hash[2:]
).exists()
assert (
workspace / "cache" / "files" / "md5" / bar_hash[:2] / bar_hash[2:]
).read_text() == "bar"
def test_gc_all_experiments(tmp_dir, scm, dvc):
from dvc.repo.experiments.refs import ExpRefInfo
tests/func/test_get.py
-14
View File
@@ -129,20 +129,6 @@ def test_get_a_dvc_file(tmp_dir, erepo_dir):
Repo.get(os.fspath(erepo_dir), "some_file.dvc")
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
def test_get_full_dvc_path(tmp_dir, erepo_dir, tmp_path_factory):
path = tmp_path_factory.mktemp("ext")
external_data = path / "ext_data"
external_data.write_text("ext_data")
with erepo_dir.chdir():
erepo_dir.dvc.add(os.fspath(external_data), external=True)
erepo_dir.scm_add("ext_data.dvc", commit="add external data")
Repo.get(os.fspath(erepo_dir), os.fspath(external_data), "ext_data_imported")
assert (tmp_dir / "ext_data_imported").read_text() == "ext_data"
def test_non_cached_output(tmp_dir, erepo_dir):
src = "non_cached_file"
dst = src + "_imported"
tests/func/test_odb.py
-11
View File
@@ -81,17 +81,6 @@ def test_external_cache_dir(tmp_dir, dvc, make_tmp_dir):
assert len(os.listdir(cache_dir)) != 0
def test_remote_cache_references(tmp_dir, dvc):
with dvc.config.edit() as conf:
conf["remote"]["storage"] = {"url": "ssh://user@localhost:23"}
conf["remote"]["cache"] = {"url": "remote://storage/tmp"}
conf["cache"]["ssh"] = "cache"
dvc.cache = CacheManager(dvc)
assert dvc.cache.ssh.path == "/tmp/files/md5"
def test_shared_cache_dir(tmp_dir):
cache_dir = os.path.abspath(os.path.join(os.curdir, "cache"))
for d in ["dir1", "dir2"]:
tests/func/test_remote.py
+1 -17
View File
@@ -9,7 +9,7 @@ import pytest
from dvc.cli import main
from dvc.config import Config
from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError
from dvc.exceptions import DownloadError, UploadError
from dvc.utils.fs import remove
@@ -257,22 +257,6 @@ def test_remote_modify_local_on_repo_config(tmp_dir, dvc):
}
def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
# https://github.com/iterative/dvc/issues/2647, is some situations
# (external dir dependency) cache is required to calculate dir md5
external_dir = tmp_path_factory.mktemp("external_dir")
file = external_dir / "file"
dvc.cache.local = None
with pytest.raises(RemoteCacheRequiredError):
dvc.run(
cmd=f"echo content > {file}",
outs=[os.fspath(file)],
name="echo",
external=True,
)
def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
from dvc_objects.fs import generic
tests/func/test_repo_index.py
+25 -34
View File
@@ -143,17 +143,7 @@ def test_skip_graph_checks(dvc, mocker):
assert mock_build_graph.called
def get_index(dvc, rev):
if rev:
brancher = dvc.brancher(revs=[rev])
if rev != "workspace":
assert next(brancher) == "workspace"
next(brancher)
return Index.from_repo(dvc)
@pytest.mark.parametrize("rev", ["workspace", "HEAD"])
def test_used_objs(tmp_dir, scm, dvc, run_copy, rev):
def test_used_objs(tmp_dir, scm, dvc, run_copy):
from dvc_data.hashfile.hash_info import HashInfo
dvc.scm_context.autostage = True
@@ -161,30 +151,31 @@ def test_used_objs(tmp_dir, scm, dvc, run_copy, rev):
run_copy("foo", "bar", name="copy-foo-bar")
scm.commit("commit")
index = get_index(dvc, rev)
for _ in dvc.brancher(revs=["HEAD"]):
expected_objs = [
HashInfo(
name="md5",
value="acbd18db4cc2f85cedef654fccc4a4d8",
obj_name="bar",
),
HashInfo(
name="md5",
value="8c7dd922ad47494fc02c388e12c00eac",
obj_name="dir/subdir/file",
),
HashInfo(
name="md5",
value="d28c9e28591aeb7e303dc6772ffa6f6b.dir",
obj_name="dir",
),
]
expected_objs = [
HashInfo(
name="md5",
value="acbd18db4cc2f85cedef654fccc4a4d8",
obj_name="bar",
),
HashInfo(
name="md5",
value="8c7dd922ad47494fc02c388e12c00eac",
obj_name="dir/subdir/file",
),
HashInfo(
name="md5",
value="d28c9e28591aeb7e303dc6772ffa6f6b.dir",
obj_name="dir",
),
]
assert index.used_objs() == {None: set(expected_objs)}
assert index.used_objs("dir") == {None: set(expected_objs[1:])}
assert index.used_objs(".", recursive=True) == {None: set(expected_objs)}
assert index.used_objs("copy-foo-bar", with_deps=True) == {None: {expected_objs[0]}}
assert dvc.index.used_objs() == {None: set(expected_objs)}
assert dvc.index.used_objs("dir") == {None: set(expected_objs[1:])}
assert dvc.index.used_objs(".", recursive=True) == {None: set(expected_objs)}
assert dvc.index.used_objs("copy-foo-bar", with_deps=True) == {
None: {expected_objs[0]}
}
def test_view_granular_dir(tmp_dir, scm, dvc, run_copy):
tests/func/test_run.py
+4 -5
View File
@@ -534,7 +534,7 @@ def test_run_external_outputs(
name="mystage",
cmd="mycmd",
deps=["remote://workspace/foo"],
outs=["remote://workspace/bar"],
outs_no_cache=["remote://workspace/bar"],
no_exec=True,
)
@@ -545,7 +545,8 @@ def test_run_external_outputs(
" deps:\n"
" - remote://workspace/foo\n"
" outs:\n"
" - remote://workspace/bar\n"
" - remote://workspace/bar:\n"
" cache: false\n"
)
assert (tmp_dir / "dvc.yaml").read_text() == dvc_yaml
@@ -574,9 +575,7 @@ def test_run_external_outputs(
assert (local_workspace / "foo").read_text() == "foo"
assert (local_workspace / "bar").read_text() == "bar"
assert (
local_workspace / "cache" / "files" / "md5" / bar_hash[:2] / bar_hash[2:]
).read_text() == "bar"
assert not (local_workspace / "cache").exists()
def test_rerun_callback(dvc):