mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
dvc: drop cached external outputs
Fixes #9531 Docs https://github.com/iterative/dvc.org/pull/4574 Kudos @dberenbaum
This commit is contained in:
@@ -32,13 +32,6 @@ def _get_odb(
|
|||||||
|
|
||||||
class CacheManager:
|
class CacheManager:
|
||||||
CACHE_DIR = "cache"
|
CACHE_DIR = "cache"
|
||||||
CLOUD_SCHEMES = [
|
|
||||||
Schemes.S3,
|
|
||||||
Schemes.GS,
|
|
||||||
Schemes.SSH,
|
|
||||||
Schemes.HDFS,
|
|
||||||
Schemes.WEBHDFS,
|
|
||||||
]
|
|
||||||
FILES_DIR = "files"
|
FILES_DIR = "files"
|
||||||
|
|
||||||
def __init__(self, repo):
|
def __init__(self, repo):
|
||||||
@@ -91,16 +84,12 @@ class CacheManager:
|
|||||||
)
|
)
|
||||||
|
|
||||||
def __getattr__(self, name):
|
def __getattr__(self, name):
|
||||||
if name not in self._odb and name in self.CLOUD_SCHEMES:
|
|
||||||
self._init_odb([name])
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return self._odb[name]
|
return self._odb[name]
|
||||||
except KeyError as exc:
|
except KeyError as exc:
|
||||||
raise AttributeError from exc
|
raise AttributeError from exc
|
||||||
|
|
||||||
def by_scheme(self):
|
def by_scheme(self):
|
||||||
self._init_odb(self.CLOUD_SCHEMES)
|
|
||||||
yield from self._odb.items()
|
yield from self._odb.items()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|||||||
@@ -24,8 +24,6 @@ class CmdAdd(CmdBase):
|
|||||||
invalid_opt = "--glob option"
|
invalid_opt = "--glob option"
|
||||||
elif args.no_commit:
|
elif args.no_commit:
|
||||||
invalid_opt = "--no-commit option"
|
invalid_opt = "--no-commit option"
|
||||||
elif args.external:
|
|
||||||
invalid_opt = "--external option"
|
|
||||||
else:
|
else:
|
||||||
message = "{option} can't be used without --to-remote"
|
message = "{option} can't be used without --to-remote"
|
||||||
if args.remote:
|
if args.remote:
|
||||||
@@ -49,7 +47,6 @@ class CmdAdd(CmdBase):
|
|||||||
self.repo.add(
|
self.repo.add(
|
||||||
self.args.targets,
|
self.args.targets,
|
||||||
no_commit=self.args.no_commit,
|
no_commit=self.args.no_commit,
|
||||||
external=self.args.external,
|
|
||||||
glob=self.args.glob,
|
glob=self.args.glob,
|
||||||
out=self.args.out,
|
out=self.args.out,
|
||||||
remote=self.args.remote,
|
remote=self.args.remote,
|
||||||
@@ -82,12 +79,6 @@ def add_parser(subparsers, parent_parser):
|
|||||||
default=False,
|
default=False,
|
||||||
help="Don't put files/directories into cache.",
|
help="Don't put files/directories into cache.",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
|
||||||
"--external",
|
|
||||||
action="store_true",
|
|
||||||
default=False,
|
|
||||||
help="Allow targets that are outside of the DVC repository.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--glob",
|
"--glob",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
|
|||||||
@@ -154,13 +154,13 @@ SCHEMA = {
|
|||||||
"machine": Lower,
|
"machine": Lower,
|
||||||
},
|
},
|
||||||
"cache": {
|
"cache": {
|
||||||
"local": str,
|
"local": str, # obsoleted
|
||||||
"s3": str,
|
"s3": str, # obsoleted
|
||||||
"gs": str,
|
"gs": str, # obsoleted
|
||||||
"hdfs": str,
|
"hdfs": str, # obsoleted
|
||||||
"webhdfs": str,
|
"webhdfs": str, # obsoleted
|
||||||
"ssh": str,
|
"ssh": str, # obsoleted
|
||||||
"azure": str,
|
"azure": str, # obsoleted
|
||||||
# This is for default local cache
|
# This is for default local cache
|
||||||
"dir": str,
|
"dir": str,
|
||||||
**LOCAL_COMMON,
|
**LOCAL_COMMON,
|
||||||
|
|||||||
@@ -299,21 +299,6 @@ class URLMissingError(DvcException):
|
|||||||
super().__init__(f"The path '{url}' does not exist")
|
super().__init__(f"The path '{url}' does not exist")
|
||||||
|
|
||||||
|
|
||||||
class RemoteCacheRequiredError(DvcException):
|
|
||||||
def __init__(self, scheme, fs_path):
|
|
||||||
super().__init__(
|
|
||||||
(
|
|
||||||
"Current operation was unsuccessful because '{}' requires "
|
|
||||||
"existing cache on '{}' remote. See {} for information on how "
|
|
||||||
"to set up remote cache."
|
|
||||||
).format(
|
|
||||||
fs_path,
|
|
||||||
scheme,
|
|
||||||
format_link("https://man.dvc.org/config#cache"),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class IsADirectoryError(DvcException): # noqa,pylint:disable=redefined-builtin
|
class IsADirectoryError(DvcException): # noqa,pylint:disable=redefined-builtin
|
||||||
"""Raised when a file operation is requested on a directory."""
|
"""Raised when a file operation is requested on a directory."""
|
||||||
|
|
||||||
|
|||||||
+3
-9
@@ -19,7 +19,6 @@ from dvc.exceptions import (
|
|||||||
ConfirmRemoveError,
|
ConfirmRemoveError,
|
||||||
DvcException,
|
DvcException,
|
||||||
MergeError,
|
MergeError,
|
||||||
RemoteCacheRequiredError,
|
|
||||||
)
|
)
|
||||||
from dvc.utils.objects import cached_property
|
from dvc.utils.objects import cached_property
|
||||||
from dvc_data.hashfile import check as ocheck
|
from dvc_data.hashfile import check as ocheck
|
||||||
@@ -526,14 +525,9 @@ class Output:
|
|||||||
def cache(self):
|
def cache(self):
|
||||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||||
|
|
||||||
if self.is_in_repo:
|
assert self.is_in_repo
|
||||||
odb_name = "legacy" if self.hash_name in LEGACY_HASH_NAMES else "repo"
|
odb_name = "legacy" if self.hash_name in LEGACY_HASH_NAMES else "repo"
|
||||||
else:
|
return getattr(self.repo.cache, odb_name)
|
||||||
odb_name = self.protocol
|
|
||||||
odb = getattr(self.repo.cache, odb_name)
|
|
||||||
if self.use_cache and odb is None:
|
|
||||||
raise RemoteCacheRequiredError(self.fs.protocol, self.fs_path)
|
|
||||||
return odb
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def local_cache(self):
|
def local_cache(self):
|
||||||
|
|||||||
@@ -54,7 +54,6 @@ PIPELINE_TRACKED_UPDATE_FMT = (
|
|||||||
def get_or_create_stage(
|
def get_or_create_stage(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
target: str,
|
target: str,
|
||||||
external: bool = False,
|
|
||||||
out: Optional[str] = None,
|
out: Optional[str] = None,
|
||||||
to_remote: bool = False,
|
to_remote: bool = False,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
@@ -79,7 +78,6 @@ def get_or_create_stage(
|
|||||||
fname=path,
|
fname=path,
|
||||||
wdir=wdir,
|
wdir=wdir,
|
||||||
outs=[out],
|
outs=[out],
|
||||||
external=external,
|
|
||||||
force=force,
|
force=force,
|
||||||
)
|
)
|
||||||
return StageInfo(stage, output_exists=False)
|
return StageInfo(stage, output_exists=False)
|
||||||
@@ -199,7 +197,6 @@ def add(
|
|||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
targets: Union["StrOrBytesPath", Iterator["StrOrBytesPath"]],
|
targets: Union["StrOrBytesPath", Iterator["StrOrBytesPath"]],
|
||||||
no_commit: bool = False,
|
no_commit: bool = False,
|
||||||
external: bool = False,
|
|
||||||
glob: bool = False,
|
glob: bool = False,
|
||||||
out: Optional[str] = None,
|
out: Optional[str] = None,
|
||||||
remote: Optional[str] = None,
|
remote: Optional[str] = None,
|
||||||
@@ -215,7 +212,6 @@ def add(
|
|||||||
target: get_or_create_stage(
|
target: get_or_create_stage(
|
||||||
repo,
|
repo,
|
||||||
target,
|
target,
|
||||||
external=external,
|
|
||||||
out=out,
|
out=out,
|
||||||
to_remote=to_remote,
|
to_remote=to_remote,
|
||||||
force=force,
|
force=force,
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ class RawData:
|
|||||||
generated_from: Optional[str] = None
|
generated_from: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
def create_stage(cls: Type[_T], repo, path, external=False, **kwargs) -> _T:
|
def create_stage(cls: Type[_T], repo, path, **kwargs) -> _T:
|
||||||
from dvc.dvcfile import check_dvcfile_path
|
from dvc.dvcfile import check_dvcfile_path
|
||||||
|
|
||||||
wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
|
wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
|
||||||
@@ -101,8 +101,7 @@ def create_stage(cls: Type[_T], repo, path, external=False, **kwargs) -> _T:
|
|||||||
|
|
||||||
stage = loads_from(cls, repo, path, wdir, kwargs)
|
stage = loads_from(cls, repo, path, wdir, kwargs)
|
||||||
fill_stage_outputs(stage, **kwargs)
|
fill_stage_outputs(stage, **kwargs)
|
||||||
if not external:
|
check_no_externals(stage)
|
||||||
check_no_externals(stage)
|
|
||||||
fill_stage_dependencies(
|
fill_stage_dependencies(
|
||||||
stage, **project(kwargs, ["deps", "erepo", "params", "fs_config"])
|
stage, **project(kwargs, ["deps", "erepo", "params", "fs_config"])
|
||||||
)
|
)
|
||||||
|
|||||||
+7
-3
@@ -44,7 +44,12 @@ def _can_hash(stage):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
for out in stage.outs:
|
for out in stage.outs:
|
||||||
if out.protocol != "local" or not out.def_path or out.persist:
|
if (
|
||||||
|
out.protocol != "local"
|
||||||
|
or not out.def_path
|
||||||
|
or out.persist
|
||||||
|
or not out.is_in_repo
|
||||||
|
):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
return True
|
return True
|
||||||
@@ -112,7 +117,6 @@ class StageCache:
|
|||||||
cmd=cache["cmd"],
|
cmd=cache["cmd"],
|
||||||
wdir=wdir,
|
wdir=wdir,
|
||||||
outs=[out["path"] for out in cache["outs"]],
|
outs=[out["path"] for out in cache["outs"]],
|
||||||
external=True,
|
|
||||||
)
|
)
|
||||||
StageLoader.fill_from_lock(stage, cache)
|
StageLoader.fill_from_lock(stage, cache)
|
||||||
return stage
|
return stage
|
||||||
@@ -137,7 +141,7 @@ class StageCache:
|
|||||||
# NOTE: using copy link to make it look like a git-tracked file
|
# NOTE: using copy link to make it look like a git-tracked file
|
||||||
with self._cache_type_copy():
|
with self._cache_type_copy():
|
||||||
for out in cached_stage.outs:
|
for out in cached_stage.outs:
|
||||||
if out.def_path in outs_no_cache:
|
if out.def_path in outs_no_cache and out.is_in_repo:
|
||||||
yield out
|
yield out
|
||||||
|
|
||||||
def save(self, stage):
|
def save(self, stage):
|
||||||
|
|||||||
+8
-12
@@ -87,28 +87,24 @@ def fill_stage_dependencies(stage, deps=None, erepo=None, params=None, fs_config
|
|||||||
|
|
||||||
|
|
||||||
def check_no_externals(stage):
|
def check_no_externals(stage):
|
||||||
from urllib.parse import urlparse
|
|
||||||
|
|
||||||
from dvc.utils import format_link
|
from dvc.utils import format_link
|
||||||
|
|
||||||
# NOTE: preventing users from accidentally using external outputs. See
|
def _is_cached_external(out):
|
||||||
# https://github.com/iterative/dvc/issues/1545 for more details.
|
if out.is_in_repo or not out.use_cache:
|
||||||
|
|
||||||
def _is_external(out):
|
|
||||||
# NOTE: in case of `remote://` notation, the user clearly knows that
|
|
||||||
# this is an advanced feature and so we shouldn't error-out.
|
|
||||||
if out.is_in_repo or urlparse(out.def_path).scheme == "remote":
|
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
outs = [str(out) for out in stage.outs if _is_external(out)]
|
outs = [str(out) for out in stage.outs if _is_cached_external(out)]
|
||||||
if not outs:
|
if not outs:
|
||||||
return
|
return
|
||||||
|
|
||||||
str_outs = ", ".join(outs)
|
str_outs = ", ".join(outs)
|
||||||
link = format_link("https://dvc.org/doc/user-guide/managing-external-data")
|
link = format_link(
|
||||||
|
"https://dvc.org/doc/user-guide/pipelines/external-dependencies-and-outputs"
|
||||||
|
)
|
||||||
raise StageExternalOutputsError(
|
raise StageExternalOutputsError(
|
||||||
f"Output(s) outside of DVC project: {str_outs}. See {link} for more info."
|
f"Cached output(s) outside of DVC project: {str_outs}. "
|
||||||
|
f"See {link} for more info."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -153,57 +153,6 @@ class TestImportURLVersionAware:
|
|||||||
assert dvc.status() == {}
|
assert dvc.status() == {}
|
||||||
|
|
||||||
|
|
||||||
class TestAdd:
|
|
||||||
@pytest.fixture
|
|
||||||
def hash_name(self):
|
|
||||||
pytest.skip()
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def hash_value(self):
|
|
||||||
pytest.skip()
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def dir_hash_value(self):
|
|
||||||
pytest.skip()
|
|
||||||
|
|
||||||
def test_add(self, tmp_dir, dvc, workspace, hash_name, hash_value):
|
|
||||||
from dvc.stage.exceptions import StageExternalOutputsError
|
|
||||||
|
|
||||||
workspace.gen("file", "file")
|
|
||||||
|
|
||||||
with pytest.raises(StageExternalOutputsError):
|
|
||||||
dvc.add(workspace.url)
|
|
||||||
|
|
||||||
dvc.add("remote://workspace/file")
|
|
||||||
assert (tmp_dir / "file.dvc").read_text() == (
|
|
||||||
"outs:\n"
|
|
||||||
f"- {hash_name}: {hash_value}\n"
|
|
||||||
" size: 4\n"
|
|
||||||
" hash: md5\n"
|
|
||||||
" path: remote://workspace/file\n"
|
|
||||||
)
|
|
||||||
assert (workspace / "file").read_text() == "file"
|
|
||||||
assert (
|
|
||||||
workspace / "cache" / "files" / "md5" / hash_value[:2] / hash_value[2:]
|
|
||||||
).read_text() == "file"
|
|
||||||
|
|
||||||
assert dvc.status() == {}
|
|
||||||
|
|
||||||
# pylint: disable-next=unused-argument
|
|
||||||
def test_add_dir(self, tmp_dir, dvc, workspace, hash_name, dir_hash_value):
|
|
||||||
workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})
|
|
||||||
|
|
||||||
dvc.add("remote://workspace/dir")
|
|
||||||
assert (
|
|
||||||
workspace
|
|
||||||
/ "cache"
|
|
||||||
/ "files"
|
|
||||||
/ "md5"
|
|
||||||
/ dir_hash_value[:2]
|
|
||||||
/ dir_hash_value[2:]
|
|
||||||
).is_file()
|
|
||||||
|
|
||||||
|
|
||||||
def match_files(fs, entries, expected):
|
def match_files(fs, entries, expected):
|
||||||
entries_content = {(fs.path.normpath(d["path"]), d["isdir"]) for d in entries}
|
entries_content = {(fs.path.normpath(d["path"]), d["isdir"]) for d in entries}
|
||||||
expected_content = {(fs.path.normpath(d["path"]), d["isdir"]) for d in expected}
|
expected_content = {(fs.path.normpath(d["path"]), d["isdir"]) for d in expected}
|
||||||
|
|||||||
+6
-68
@@ -22,7 +22,6 @@ from dvc.output import (
|
|||||||
)
|
)
|
||||||
from dvc.stage import Stage
|
from dvc.stage import Stage
|
||||||
from dvc.stage.exceptions import StageExternalOutputsError, StagePathNotFoundError
|
from dvc.stage.exceptions import StageExternalOutputsError, StagePathNotFoundError
|
||||||
from dvc.testing.workspace_tests import TestAdd
|
|
||||||
from dvc.utils.fs import path_isin
|
from dvc.utils.fs import path_isin
|
||||||
from dvc.utils.serialize import YAMLFileCorruptedError
|
from dvc.utils.serialize import YAMLFileCorruptedError
|
||||||
from dvc_data.hashfile.hash import file_md5
|
from dvc_data.hashfile.hash import file_md5
|
||||||
@@ -234,55 +233,6 @@ def test_add_filtered_files_in_dir(
|
|||||||
assert stage.outs[0].def_path in expected_def_paths
|
assert stage.outs[0].def_path in expected_def_paths
|
||||||
|
|
||||||
|
|
||||||
class TestAddExternal(TestAdd):
|
|
||||||
@pytest.fixture
|
|
||||||
def hash_name(self):
|
|
||||||
return "md5"
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def hash_value(self):
|
|
||||||
return "8c7dd922ad47494fc02c388e12c00eac"
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def dir_hash_value(self):
|
|
||||||
return "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir"
|
|
||||||
|
|
||||||
|
|
||||||
def test_add_external_relpath(tmp_dir, dvc, local_cloud):
|
|
||||||
(fpath,) = local_cloud.gen("file", "file")
|
|
||||||
rel = os.path.relpath(fpath)
|
|
||||||
|
|
||||||
with pytest.raises(StageExternalOutputsError):
|
|
||||||
dvc.add(rel)
|
|
||||||
|
|
||||||
dvc.add(rel, external=True)
|
|
||||||
assert (tmp_dir / "file.dvc").read_text() == (
|
|
||||||
"outs:\n"
|
|
||||||
"- md5: 8c7dd922ad47494fc02c388e12c00eac\n"
|
|
||||||
" size: 4\n"
|
|
||||||
" hash: md5\n"
|
|
||||||
f" path: {rel}\n"
|
|
||||||
)
|
|
||||||
assert fpath.read_text() == "file"
|
|
||||||
assert dvc.status() == {}
|
|
||||||
|
|
||||||
|
|
||||||
def test_add_local_remote_file(tmp_dir, dvc):
|
|
||||||
"""
|
|
||||||
Making sure that 'remote' syntax is handled properly for local outs.
|
|
||||||
"""
|
|
||||||
tmp_dir.gen({"foo": "foo", "bar": "bar"})
|
|
||||||
tmp_dir.add_remote(url=tmp_dir.fs_path, name="myremote")
|
|
||||||
|
|
||||||
assert main(["add", "remote://myremote/foo"]) == 0
|
|
||||||
d = (tmp_dir / "foo.dvc").load_yaml()
|
|
||||||
assert d["outs"][0]["path"] == "remote://myremote/foo"
|
|
||||||
|
|
||||||
assert main(["add", (tmp_dir / "bar").fs_path]) == 0
|
|
||||||
d = (tmp_dir / "bar.dvc").load_yaml()
|
|
||||||
assert d["outs"][0]["path"] == "bar"
|
|
||||||
|
|
||||||
|
|
||||||
def test_cmd_add(tmp_dir, dvc):
|
def test_cmd_add(tmp_dir, dvc):
|
||||||
tmp_dir.gen("foo", "foo")
|
tmp_dir.gen("foo", "foo")
|
||||||
ret = main(["add", "foo"])
|
ret = main(["add", "foo"])
|
||||||
@@ -755,15 +705,9 @@ def test_add_symlink_file(tmp_dir, dvc):
|
|||||||
dvc.add(os.path.join("dir", "foo"))
|
dvc.add(os.path.join("dir", "foo"))
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("external", [True, False])
|
def test_add_symlink_dir(make_tmp_dir, tmp_dir, dvc):
|
||||||
def test_add_symlink_dir(make_tmp_dir, tmp_dir, dvc, external):
|
tmp_dir.gen({"data": {"foo": "foo"}})
|
||||||
if external:
|
target = os.path.join(".", "data")
|
||||||
data_dir = make_tmp_dir("data")
|
|
||||||
data_dir.gen({"foo": "foo"})
|
|
||||||
target = os.fspath(data_dir)
|
|
||||||
else:
|
|
||||||
tmp_dir.gen({"data": {"foo": "foo"}})
|
|
||||||
target = os.path.join(".", "data")
|
|
||||||
|
|
||||||
tmp_dir.gen({"data": {"foo": "foo"}})
|
tmp_dir.gen({"data": {"foo": "foo"}})
|
||||||
|
|
||||||
@@ -774,15 +718,9 @@ def test_add_symlink_dir(make_tmp_dir, tmp_dir, dvc, external):
|
|||||||
dvc.add("dir")
|
dvc.add("dir")
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("external", [True, False])
|
def test_add_file_in_symlink_dir(make_tmp_dir, tmp_dir, dvc):
|
||||||
def test_add_file_in_symlink_dir(make_tmp_dir, tmp_dir, dvc, external):
|
tmp_dir.gen({"data": {"foo": "foo"}})
|
||||||
if external:
|
target = os.path.join(".", "data")
|
||||||
data_dir = make_tmp_dir("data")
|
|
||||||
data_dir.gen({"dir": {"foo": "foo"}})
|
|
||||||
target = os.fspath(data_dir / "dir")
|
|
||||||
else:
|
|
||||||
tmp_dir.gen({"data": {"foo": "foo"}})
|
|
||||||
target = os.path.join(".", "data")
|
|
||||||
|
|
||||||
(tmp_dir / "dir").symlink_to(target)
|
(tmp_dir / "dir").symlink_to(target)
|
||||||
|
|
||||||
|
|||||||
@@ -251,34 +251,6 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
|
|||||||
assert _count_files(dvc.cache.local.path) == 0
|
assert _count_files(dvc.cache.local.path) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_gc_external_output(tmp_dir, dvc, workspace):
|
|
||||||
workspace.gen({"foo": "foo", "bar": "bar"})
|
|
||||||
|
|
||||||
(foo_stage,) = dvc.add("remote://workspace/foo")
|
|
||||||
(bar_stage,) = dvc.add("remote://workspace/bar")
|
|
||||||
|
|
||||||
foo_hash = foo_stage.outs[0].hash_info.value
|
|
||||||
bar_hash = bar_stage.outs[0].hash_info.value
|
|
||||||
|
|
||||||
assert (
|
|
||||||
workspace / "cache" / "files" / "md5" / foo_hash[:2] / foo_hash[2:]
|
|
||||||
).read_text() == "foo"
|
|
||||||
assert (
|
|
||||||
workspace / "cache" / "files" / "md5" / bar_hash[:2] / bar_hash[2:]
|
|
||||||
).read_text() == "bar"
|
|
||||||
|
|
||||||
(tmp_dir / "foo.dvc").unlink()
|
|
||||||
|
|
||||||
dvc.gc(workspace=True)
|
|
||||||
|
|
||||||
assert not (
|
|
||||||
workspace / "cache" / "files" / "md5" / foo_hash[:2] / foo_hash[2:]
|
|
||||||
).exists()
|
|
||||||
assert (
|
|
||||||
workspace / "cache" / "files" / "md5" / bar_hash[:2] / bar_hash[2:]
|
|
||||||
).read_text() == "bar"
|
|
||||||
|
|
||||||
|
|
||||||
def test_gc_all_experiments(tmp_dir, scm, dvc):
|
def test_gc_all_experiments(tmp_dir, scm, dvc):
|
||||||
from dvc.repo.experiments.refs import ExpRefInfo
|
from dvc.repo.experiments.refs import ExpRefInfo
|
||||||
|
|
||||||
|
|||||||
@@ -129,20 +129,6 @@ def test_get_a_dvc_file(tmp_dir, erepo_dir):
|
|||||||
Repo.get(os.fspath(erepo_dir), "some_file.dvc")
|
Repo.get(os.fspath(erepo_dir), "some_file.dvc")
|
||||||
|
|
||||||
|
|
||||||
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
|
|
||||||
def test_get_full_dvc_path(tmp_dir, erepo_dir, tmp_path_factory):
|
|
||||||
path = tmp_path_factory.mktemp("ext")
|
|
||||||
external_data = path / "ext_data"
|
|
||||||
external_data.write_text("ext_data")
|
|
||||||
|
|
||||||
with erepo_dir.chdir():
|
|
||||||
erepo_dir.dvc.add(os.fspath(external_data), external=True)
|
|
||||||
erepo_dir.scm_add("ext_data.dvc", commit="add external data")
|
|
||||||
|
|
||||||
Repo.get(os.fspath(erepo_dir), os.fspath(external_data), "ext_data_imported")
|
|
||||||
assert (tmp_dir / "ext_data_imported").read_text() == "ext_data"
|
|
||||||
|
|
||||||
|
|
||||||
def test_non_cached_output(tmp_dir, erepo_dir):
|
def test_non_cached_output(tmp_dir, erepo_dir):
|
||||||
src = "non_cached_file"
|
src = "non_cached_file"
|
||||||
dst = src + "_imported"
|
dst = src + "_imported"
|
||||||
|
|||||||
@@ -81,17 +81,6 @@ def test_external_cache_dir(tmp_dir, dvc, make_tmp_dir):
|
|||||||
assert len(os.listdir(cache_dir)) != 0
|
assert len(os.listdir(cache_dir)) != 0
|
||||||
|
|
||||||
|
|
||||||
def test_remote_cache_references(tmp_dir, dvc):
|
|
||||||
with dvc.config.edit() as conf:
|
|
||||||
conf["remote"]["storage"] = {"url": "ssh://user@localhost:23"}
|
|
||||||
conf["remote"]["cache"] = {"url": "remote://storage/tmp"}
|
|
||||||
conf["cache"]["ssh"] = "cache"
|
|
||||||
|
|
||||||
dvc.cache = CacheManager(dvc)
|
|
||||||
|
|
||||||
assert dvc.cache.ssh.path == "/tmp/files/md5"
|
|
||||||
|
|
||||||
|
|
||||||
def test_shared_cache_dir(tmp_dir):
|
def test_shared_cache_dir(tmp_dir):
|
||||||
cache_dir = os.path.abspath(os.path.join(os.curdir, "cache"))
|
cache_dir = os.path.abspath(os.path.join(os.curdir, "cache"))
|
||||||
for d in ["dir1", "dir2"]:
|
for d in ["dir1", "dir2"]:
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import pytest
|
|||||||
|
|
||||||
from dvc.cli import main
|
from dvc.cli import main
|
||||||
from dvc.config import Config
|
from dvc.config import Config
|
||||||
from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError
|
from dvc.exceptions import DownloadError, UploadError
|
||||||
from dvc.utils.fs import remove
|
from dvc.utils.fs import remove
|
||||||
|
|
||||||
|
|
||||||
@@ -257,22 +257,6 @@ def test_remote_modify_local_on_repo_config(tmp_dir, dvc):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
|
|
||||||
# https://github.com/iterative/dvc/issues/2647, is some situations
|
|
||||||
# (external dir dependency) cache is required to calculate dir md5
|
|
||||||
external_dir = tmp_path_factory.mktemp("external_dir")
|
|
||||||
file = external_dir / "file"
|
|
||||||
|
|
||||||
dvc.cache.local = None
|
|
||||||
with pytest.raises(RemoteCacheRequiredError):
|
|
||||||
dvc.run(
|
|
||||||
cmd=f"echo content > {file}",
|
|
||||||
outs=[os.fspath(file)],
|
|
||||||
name="echo",
|
|
||||||
external=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
|
def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
|
||||||
from dvc_objects.fs import generic
|
from dvc_objects.fs import generic
|
||||||
|
|
||||||
|
|||||||
@@ -143,17 +143,7 @@ def test_skip_graph_checks(dvc, mocker):
|
|||||||
assert mock_build_graph.called
|
assert mock_build_graph.called
|
||||||
|
|
||||||
|
|
||||||
def get_index(dvc, rev):
|
def test_used_objs(tmp_dir, scm, dvc, run_copy):
|
||||||
if rev:
|
|
||||||
brancher = dvc.brancher(revs=[rev])
|
|
||||||
if rev != "workspace":
|
|
||||||
assert next(brancher) == "workspace"
|
|
||||||
next(brancher)
|
|
||||||
return Index.from_repo(dvc)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("rev", ["workspace", "HEAD"])
|
|
||||||
def test_used_objs(tmp_dir, scm, dvc, run_copy, rev):
|
|
||||||
from dvc_data.hashfile.hash_info import HashInfo
|
from dvc_data.hashfile.hash_info import HashInfo
|
||||||
|
|
||||||
dvc.scm_context.autostage = True
|
dvc.scm_context.autostage = True
|
||||||
@@ -161,30 +151,31 @@ def test_used_objs(tmp_dir, scm, dvc, run_copy, rev):
|
|||||||
run_copy("foo", "bar", name="copy-foo-bar")
|
run_copy("foo", "bar", name="copy-foo-bar")
|
||||||
scm.commit("commit")
|
scm.commit("commit")
|
||||||
|
|
||||||
index = get_index(dvc, rev)
|
for _ in dvc.brancher(revs=["HEAD"]):
|
||||||
|
expected_objs = [
|
||||||
|
HashInfo(
|
||||||
|
name="md5",
|
||||||
|
value="acbd18db4cc2f85cedef654fccc4a4d8",
|
||||||
|
obj_name="bar",
|
||||||
|
),
|
||||||
|
HashInfo(
|
||||||
|
name="md5",
|
||||||
|
value="8c7dd922ad47494fc02c388e12c00eac",
|
||||||
|
obj_name="dir/subdir/file",
|
||||||
|
),
|
||||||
|
HashInfo(
|
||||||
|
name="md5",
|
||||||
|
value="d28c9e28591aeb7e303dc6772ffa6f6b.dir",
|
||||||
|
obj_name="dir",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
expected_objs = [
|
assert dvc.index.used_objs() == {None: set(expected_objs)}
|
||||||
HashInfo(
|
assert dvc.index.used_objs("dir") == {None: set(expected_objs[1:])}
|
||||||
name="md5",
|
assert dvc.index.used_objs(".", recursive=True) == {None: set(expected_objs)}
|
||||||
value="acbd18db4cc2f85cedef654fccc4a4d8",
|
assert dvc.index.used_objs("copy-foo-bar", with_deps=True) == {
|
||||||
obj_name="bar",
|
None: {expected_objs[0]}
|
||||||
),
|
}
|
||||||
HashInfo(
|
|
||||||
name="md5",
|
|
||||||
value="8c7dd922ad47494fc02c388e12c00eac",
|
|
||||||
obj_name="dir/subdir/file",
|
|
||||||
),
|
|
||||||
HashInfo(
|
|
||||||
name="md5",
|
|
||||||
value="d28c9e28591aeb7e303dc6772ffa6f6b.dir",
|
|
||||||
obj_name="dir",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
assert index.used_objs() == {None: set(expected_objs)}
|
|
||||||
assert index.used_objs("dir") == {None: set(expected_objs[1:])}
|
|
||||||
assert index.used_objs(".", recursive=True) == {None: set(expected_objs)}
|
|
||||||
assert index.used_objs("copy-foo-bar", with_deps=True) == {None: {expected_objs[0]}}
|
|
||||||
|
|
||||||
|
|
||||||
def test_view_granular_dir(tmp_dir, scm, dvc, run_copy):
|
def test_view_granular_dir(tmp_dir, scm, dvc, run_copy):
|
||||||
|
|||||||
@@ -534,7 +534,7 @@ def test_run_external_outputs(
|
|||||||
name="mystage",
|
name="mystage",
|
||||||
cmd="mycmd",
|
cmd="mycmd",
|
||||||
deps=["remote://workspace/foo"],
|
deps=["remote://workspace/foo"],
|
||||||
outs=["remote://workspace/bar"],
|
outs_no_cache=["remote://workspace/bar"],
|
||||||
no_exec=True,
|
no_exec=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -545,7 +545,8 @@ def test_run_external_outputs(
|
|||||||
" deps:\n"
|
" deps:\n"
|
||||||
" - remote://workspace/foo\n"
|
" - remote://workspace/foo\n"
|
||||||
" outs:\n"
|
" outs:\n"
|
||||||
" - remote://workspace/bar\n"
|
" - remote://workspace/bar:\n"
|
||||||
|
" cache: false\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
assert (tmp_dir / "dvc.yaml").read_text() == dvc_yaml
|
assert (tmp_dir / "dvc.yaml").read_text() == dvc_yaml
|
||||||
@@ -574,9 +575,7 @@ def test_run_external_outputs(
|
|||||||
|
|
||||||
assert (local_workspace / "foo").read_text() == "foo"
|
assert (local_workspace / "foo").read_text() == "foo"
|
||||||
assert (local_workspace / "bar").read_text() == "bar"
|
assert (local_workspace / "bar").read_text() == "bar"
|
||||||
assert (
|
assert not (local_workspace / "cache").exists()
|
||||||
local_workspace / "cache" / "files" / "md5" / bar_hash[:2] / bar_hash[2:]
|
|
||||||
).read_text() == "bar"
|
|
||||||
|
|
||||||
|
|
||||||
def test_rerun_callback(dvc):
|
def test_rerun_callback(dvc):
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ def test_add(mocker, dvc):
|
|||||||
[
|
[
|
||||||
"add",
|
"add",
|
||||||
"--no-commit",
|
"--no-commit",
|
||||||
"--external",
|
|
||||||
"--glob",
|
"--glob",
|
||||||
"data",
|
"data",
|
||||||
]
|
]
|
||||||
@@ -25,7 +24,6 @@ def test_add(mocker, dvc):
|
|||||||
["data"],
|
["data"],
|
||||||
no_commit=True,
|
no_commit=True,
|
||||||
glob=True,
|
glob=True,
|
||||||
external=True,
|
|
||||||
out=None,
|
out=None,
|
||||||
remote=None,
|
remote=None,
|
||||||
to_remote=False,
|
to_remote=False,
|
||||||
@@ -57,7 +55,6 @@ def test_add_to_remote(mocker):
|
|||||||
["s3://bucket/foo"],
|
["s3://bucket/foo"],
|
||||||
no_commit=False,
|
no_commit=False,
|
||||||
glob=False,
|
glob=False,
|
||||||
external=False,
|
|
||||||
out="bar",
|
out="bar",
|
||||||
remote="remote",
|
remote="remote",
|
||||||
to_remote=True,
|
to_remote=True,
|
||||||
|
|||||||
@@ -120,5 +120,8 @@ def test_external_outs(tmp_path_factory, dvc):
|
|||||||
with dvc.config.edit() as conf:
|
with dvc.config.edit() as conf:
|
||||||
conf["remote"]["myremote"] = {"url": os.fspath(tmp_path)}
|
conf["remote"]["myremote"] = {"url": os.fspath(tmp_path)}
|
||||||
|
|
||||||
create_stage(Stage, dvc, "path.dvc", outs=["remote://myremote/foo"])
|
with pytest.raises(StageExternalOutputsError):
|
||||||
create_stage(Stage, dvc, "path.dvc", outs=[os.fspath(foo)], external=True)
|
create_stage(Stage, dvc, "path.dvc", outs=["remote://myremote/foo"])
|
||||||
|
|
||||||
|
create_stage(Stage, dvc, "path.dvc", outs_no_cache=["remote://myremote/foo"])
|
||||||
|
create_stage(Stage, dvc, "path.dvc", outs_no_cache=[os.fspath(foo)])
|
||||||
|
|||||||
@@ -85,14 +85,10 @@ def test_info_in_broken_git_repo(tmp_dir, dvc, scm, caplog):
|
|||||||
|
|
||||||
|
|
||||||
def test_caches(tmp_dir, dvc, caplog):
|
def test_caches(tmp_dir, dvc, caplog):
|
||||||
tmp_dir.add_remote(name="sshcache", url="ssh://example.com/path", default=False)
|
|
||||||
with tmp_dir.dvc.config.edit() as conf:
|
|
||||||
conf["cache"]["ssh"] = "sshcache"
|
|
||||||
|
|
||||||
dvc_info = get_dvc_info()
|
dvc_info = get_dvc_info()
|
||||||
|
|
||||||
# Order of cache types is runtime dependent
|
# Order of cache types is runtime dependent
|
||||||
assert re.search("Caches: (local, ssh|ssh, local)", dvc_info)
|
assert re.search("Caches: local", dvc_info)
|
||||||
|
|
||||||
|
|
||||||
def test_remotes_empty(tmp_dir, dvc, caplog):
|
def test_remotes_empty(tmp_dir, dvc, caplog):
|
||||||
|
|||||||
Reference in New Issue
Block a user