mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
odb: use odb.path instead of odb.cache_dir
This commit is contained in:
@@ -108,9 +108,7 @@ class RepoDependency(Dependency):
|
|||||||
|
|
||||||
local_odb = self.repo.odb.local
|
local_odb = self.repo.odb.local
|
||||||
locked = kwargs.pop("locked", True)
|
locked = kwargs.pop("locked", True)
|
||||||
with self._make_repo(
|
with self._make_repo(locked=locked, cache_dir=local_odb.path) as repo:
|
||||||
locked=locked, cache_dir=local_odb.cache_dir
|
|
||||||
) as repo:
|
|
||||||
used_obj_ids = defaultdict(set)
|
used_obj_ids = defaultdict(set)
|
||||||
rev = repo.get_rev()
|
rev = repo.get_rev()
|
||||||
if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
|
if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
|
||||||
|
|||||||
@@ -145,7 +145,7 @@ def _get_remote_config(url):
|
|||||||
name = "auto-generated-upstream"
|
name = "auto-generated-upstream"
|
||||||
return {
|
return {
|
||||||
"core": {"remote": name},
|
"core": {"remote": name},
|
||||||
"remote": {name: {"url": repo.odb.local.cache_dir}},
|
"remote": {name: {"url": repo.odb.local.path}},
|
||||||
}
|
}
|
||||||
|
|
||||||
# Use original remote to make sure that we are using correct url,
|
# Use original remote to make sure that we are using correct url,
|
||||||
|
|||||||
+1
-3
@@ -155,9 +155,7 @@ class _DvcFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
|||||||
self.PARAM_REPO_URL: self.repo_url,
|
self.PARAM_REPO_URL: self.repo_url,
|
||||||
self.PARAM_REPO_ROOT: self.repo.root_dir,
|
self.PARAM_REPO_ROOT: self.repo.root_dir,
|
||||||
self.PARAM_REV: getattr(self.repo.fs, "rev", None),
|
self.PARAM_REV: getattr(self.repo.fs, "rev", None),
|
||||||
self.PARAM_CACHE_DIR: os.path.abspath(
|
self.PARAM_CACHE_DIR: os.path.abspath(self.repo.odb.local.path),
|
||||||
self.repo.odb.local.cache_dir
|
|
||||||
),
|
|
||||||
self.PARAM_CACHE_TYPES: self.repo.odb.local.cache_types,
|
self.PARAM_CACHE_TYPES: self.repo.odb.local.cache_types,
|
||||||
self.PARAM_SUBREPOS: self._traverse_subrepos,
|
self.PARAM_SUBREPOS: self._traverse_subrepos,
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -32,9 +32,9 @@ def get_dvc_info():
|
|||||||
# can't auto-create it, as it might cause issues if the user
|
# can't auto-create it, as it might cause issues if the user
|
||||||
# later decides to enable shared cache mode with
|
# later decides to enable shared cache mode with
|
||||||
# `dvc config cache.shared group`.
|
# `dvc config cache.shared group`.
|
||||||
if os.path.exists(repo.odb.local.cache_dir):
|
if os.path.exists(repo.odb.local.path):
|
||||||
info.append(f"Cache types: {_get_linktype_support_info(repo)}")
|
info.append(f"Cache types: {_get_linktype_support_info(repo)}")
|
||||||
fs_type = get_fs_type(repo.odb.local.cache_dir)
|
fs_type = get_fs_type(repo.odb.local.path)
|
||||||
info.append(f"Cache directory: {fs_type}")
|
info.append(f"Cache directory: {fs_type}")
|
||||||
else:
|
else:
|
||||||
info.append("Cache types: " + error_link("no-dvc-cache"))
|
info.append("Cache types: " + error_link("no-dvc-cache"))
|
||||||
|
|||||||
@@ -369,8 +369,8 @@ class Repo:
|
|||||||
def _ignore(self):
|
def _ignore(self):
|
||||||
flist = [self.config.files["local"], self.tmp_dir]
|
flist = [self.config.files["local"], self.tmp_dir]
|
||||||
|
|
||||||
if path_isin(self.odb.repo.cache_dir, self.root_dir):
|
if path_isin(self.odb.repo.path, self.root_dir):
|
||||||
flist += [self.odb.repo.cache_dir]
|
flist += [self.odb.repo.path]
|
||||||
|
|
||||||
for file in flist:
|
for file in flist:
|
||||||
self.scm_context.ignore(file)
|
self.scm_context.ignore(file)
|
||||||
|
|||||||
@@ -112,7 +112,7 @@ class TempDirExecutor(BaseLocalExecutor):
|
|||||||
|
|
||||||
def init_cache(self, repo: "Repo", rev: str, run_cache: bool = True):
|
def init_cache(self, repo: "Repo", rev: str, run_cache: bool = True):
|
||||||
"""Initialize DVC cache."""
|
"""Initialize DVC cache."""
|
||||||
self._config(repo.odb.repo.cache_dir)
|
self._config(repo.odb.repo.path)
|
||||||
|
|
||||||
def cleanup(self):
|
def cleanup(self):
|
||||||
super().cleanup()
|
super().cleanup()
|
||||||
|
|||||||
+1
-1
@@ -65,7 +65,7 @@ class StageCache:
|
|||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def cache_dir(self):
|
def cache_dir(self):
|
||||||
return os.path.join(self.repo.odb.local.cache_dir, "runs")
|
return os.path.join(self.repo.odb.local.path, "runs")
|
||||||
|
|
||||||
def _get_cache_dir(self, key):
|
def _get_cache_dir(self, key):
|
||||||
return os.path.join(self.cache_dir, key[:2], key)
|
return os.path.join(self.cache_dir, key[:2], key)
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ class TestAPI:
|
|||||||
dvc.push()
|
dvc.push()
|
||||||
|
|
||||||
# Remove cache to force download
|
# Remove cache to force download
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
with api.open("foo") as fd:
|
with api.open("foo") as fd:
|
||||||
assert fd.read() == "foo-text"
|
assert fd.read() == "foo-text"
|
||||||
|
|||||||
@@ -44,8 +44,8 @@ class TestRemote:
|
|||||||
|
|
||||||
# Move cache and check status
|
# Move cache and check status
|
||||||
# See issue https://github.com/iterative/dvc/issues/4383 for details
|
# See issue https://github.com/iterative/dvc/issues/4383 for details
|
||||||
backup_dir = dvc.odb.local.cache_dir + ".backup"
|
backup_dir = dvc.odb.local.path + ".backup"
|
||||||
shutil.move(dvc.odb.local.cache_dir, backup_dir)
|
shutil.move(dvc.odb.local.path, backup_dir)
|
||||||
status = dvc.cloud.status(foo_hashes)
|
status = dvc.cloud.status(foo_hashes)
|
||||||
_check_status(status, missing={foo_hash})
|
_check_status(status, missing={foo_hash})
|
||||||
|
|
||||||
@@ -53,8 +53,8 @@ class TestRemote:
|
|||||||
_check_status(status_dir, missing=dir_hashes)
|
_check_status(status_dir, missing=dir_hashes)
|
||||||
|
|
||||||
# Restore original cache:
|
# Restore original cache:
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
shutil.move(backup_dir, dvc.odb.local.cache_dir)
|
shutil.move(backup_dir, dvc.odb.local.path)
|
||||||
|
|
||||||
# Push and check status
|
# Push and check status
|
||||||
dvc.cloud.push(foo_hashes)
|
dvc.cloud.push(foo_hashes)
|
||||||
@@ -71,7 +71,7 @@ class TestRemote:
|
|||||||
_check_status(status_dir, ok=dir_hashes)
|
_check_status(status_dir, ok=dir_hashes)
|
||||||
|
|
||||||
# Remove and check status
|
# Remove and check status
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
status = dvc.cloud.status(foo_hashes)
|
status = dvc.cloud.status(foo_hashes)
|
||||||
_check_status(status, deleted={foo_hash})
|
_check_status(status, deleted={foo_hash})
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ def test_open_external(tmp_dir, erepo_dir, cloud):
|
|||||||
erepo_dir.dvc.push(all_branches=True)
|
erepo_dir.dvc.push(all_branches=True)
|
||||||
|
|
||||||
# Remove cache to force download
|
# Remove cache to force download
|
||||||
remove(erepo_dir.dvc.odb.local.cache_dir)
|
remove(erepo_dir.dvc.odb.local.path)
|
||||||
|
|
||||||
# Using file url to force clone to tmp repo
|
# Using file url to force clone to tmp repo
|
||||||
repo_url = f"file://{erepo_dir.as_posix()}"
|
repo_url = f"file://{erepo_dir.as_posix()}"
|
||||||
@@ -63,7 +63,7 @@ def test_open_granular(tmp_dir, dvc, remote):
|
|||||||
dvc.push()
|
dvc.push()
|
||||||
|
|
||||||
# Remove cache to force download
|
# Remove cache to force download
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
with api.open("dir/foo") as fd:
|
with api.open("dir/foo") as fd:
|
||||||
assert fd.read() == "foo-text"
|
assert fd.read() == "foo-text"
|
||||||
@@ -73,7 +73,7 @@ def test_missing(tmp_dir, dvc, remote):
|
|||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
|
|
||||||
# Remove cache to make foo missing
|
# Remove cache to make foo missing
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
api.read("foo")
|
api.read("foo")
|
||||||
|
|
||||||
@@ -135,7 +135,7 @@ def test_api_missing_local_cache_exists_on_remote(
|
|||||||
dvc.push()
|
dvc.push()
|
||||||
|
|
||||||
# Remove cache to make foo missing
|
# Remove cache to make foo missing
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
remove(first(files))
|
remove(first(files))
|
||||||
|
|
||||||
repo_url = f"file://{tmp_dir.as_posix()}" if as_external else None
|
repo_url = f"file://{tmp_dir.as_posix()}" if as_external else None
|
||||||
@@ -220,7 +220,7 @@ def test_open_from_remote(tmp_dir, erepo_dir, cloud, local_cloud):
|
|||||||
erepo_dir.add_remote(config=local_cloud.config, default=True)
|
erepo_dir.add_remote(config=local_cloud.config, default=True)
|
||||||
erepo_dir.dvc_gen({"dir": {"foo": "foo content"}}, commit="create file")
|
erepo_dir.dvc_gen({"dir": {"foo": "foo content"}}, commit="create file")
|
||||||
erepo_dir.dvc.push(remote="other")
|
erepo_dir.dvc.push(remote="other")
|
||||||
remove(erepo_dir.dvc.odb.local.cache_dir)
|
remove(erepo_dir.dvc.odb.local.path)
|
||||||
|
|
||||||
with api.open(
|
with api.open(
|
||||||
os.path.join("dir", "foo"),
|
os.path.join("dir", "foo"),
|
||||||
|
|||||||
@@ -329,12 +329,12 @@ def test_push_pull_cache(
|
|||||||
assert os.path.exists(path)
|
assert os.path.exists(path)
|
||||||
assert open(path, encoding="utf-8").read() == str(x)
|
assert open(path, encoding="utf-8").read() == str(x)
|
||||||
|
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
dvc.experiments.pull(remote, [ref_info.name], pull_cache=True)
|
dvc.experiments.pull(remote, [ref_info.name], pull_cache=True)
|
||||||
for x in range(2, checkpoint_stage.iterations + 1):
|
for x in range(2, checkpoint_stage.iterations + 1):
|
||||||
hash_ = digest(str(x))
|
hash_ = digest(str(x))
|
||||||
path = os.path.join(dvc.odb.local.cache_dir, hash_[:2], hash_[2:])
|
path = os.path.join(dvc.odb.local.path, hash_[:2], hash_[2:])
|
||||||
assert os.path.exists(path)
|
assert os.path.exists(path)
|
||||||
assert open(path, encoding="utf-8").read() == str(x)
|
assert open(path, encoding="utf-8").read() == str(x)
|
||||||
|
|
||||||
|
|||||||
@@ -272,7 +272,7 @@ def test_metrics_show_overlap(
|
|||||||
# so as it works even for optimized cases
|
# so as it works even for optimized cases
|
||||||
if clear_before_run:
|
if clear_before_run:
|
||||||
remove(data_dir)
|
remove(data_dir)
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
dvc._reset()
|
dvc._reset()
|
||||||
|
|
||||||
|
|||||||
@@ -163,7 +163,7 @@ def test_plots_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):
|
|||||||
# so as it works even for optimized cases
|
# so as it works even for optimized cases
|
||||||
if clear_before_run:
|
if clear_before_run:
|
||||||
remove(data_dir)
|
remove(data_dir)
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
dvc._reset()
|
dvc._reset()
|
||||||
|
|
||||||
|
|||||||
@@ -447,12 +447,12 @@ class TestAddCommit(TestDvc):
|
|||||||
ret = main(["add", self.FOO, "--no-commit"])
|
ret = main(["add", self.FOO, "--no-commit"])
|
||||||
self.assertEqual(ret, 0)
|
self.assertEqual(ret, 0)
|
||||||
self.assertTrue(os.path.isfile(self.FOO))
|
self.assertTrue(os.path.isfile(self.FOO))
|
||||||
self.assertFalse(os.path.exists(self.dvc.odb.local.cache_dir))
|
self.assertFalse(os.path.exists(self.dvc.odb.local.path))
|
||||||
|
|
||||||
ret = main(["commit", self.FOO + ".dvc"])
|
ret = main(["commit", self.FOO + ".dvc"])
|
||||||
self.assertEqual(ret, 0)
|
self.assertEqual(ret, 0)
|
||||||
self.assertTrue(os.path.isfile(self.FOO))
|
self.assertTrue(os.path.isfile(self.FOO))
|
||||||
self.assertEqual(len(os.listdir(self.dvc.odb.local.cache_dir)), 1)
|
self.assertEqual(len(os.listdir(self.dvc.odb.local.path)), 1)
|
||||||
|
|
||||||
|
|
||||||
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
|
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
|
||||||
|
|||||||
@@ -470,7 +470,7 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
|
|||||||
self.assertTrue(system.is_symlink(self.DATA))
|
self.assertTrue(system.is_symlink(self.DATA))
|
||||||
old_data_link = os.path.realpath(self.DATA)
|
old_data_link = os.path.realpath(self.DATA)
|
||||||
|
|
||||||
old_cache_dir = self.dvc.odb.local.cache_dir
|
old_cache_dir = self.dvc.odb.local.path
|
||||||
new_cache_dir = old_cache_dir + "_new"
|
new_cache_dir = old_cache_dir + "_new"
|
||||||
os.rename(old_cache_dir, new_cache_dir)
|
os.rename(old_cache_dir, new_cache_dir)
|
||||||
|
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker):
|
|||||||
for (_args, _kwargs) in oids_exist.call_args_list
|
for (_args, _kwargs) in oids_exist.call_args_list
|
||||||
)
|
)
|
||||||
|
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
oids_exist.reset_mock()
|
oids_exist.reset_mock()
|
||||||
|
|
||||||
assert main(["fetch"] + args) == 0
|
assert main(["fetch"] + args) == 0
|
||||||
@@ -93,7 +93,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker):
|
|||||||
assert all(
|
assert all(
|
||||||
_kwargs["jobs"] == 2 for (_args, _kwargs) in oids_exist.call_args_list
|
_kwargs["jobs"] == 2 for (_args, _kwargs) in oids_exist.call_args_list
|
||||||
)
|
)
|
||||||
shutil.move(dvc.odb.local.cache_dir, dvc.odb.local.cache_dir + ".back")
|
shutil.move(dvc.odb.local.path, dvc.odb.local.path + ".back")
|
||||||
|
|
||||||
assert main(["fetch"] + args) == 0
|
assert main(["fetch"] + args) == 0
|
||||||
|
|
||||||
@@ -164,7 +164,7 @@ def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
|
|||||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||||
|
|
||||||
# purge cache
|
# purge cache
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
header = (
|
header = (
|
||||||
"Some of the cache files do not exist "
|
"Some of the cache files do not exist "
|
||||||
@@ -205,7 +205,7 @@ def test_verify_hashes(
|
|||||||
# remove artifacts and cache to trigger fetching
|
# remove artifacts and cache to trigger fetching
|
||||||
remove("file")
|
remove("file")
|
||||||
remove("dir")
|
remove("dir")
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
hash_spy = mocker.spy(dvc_data.hashfile.hash, "file_md5")
|
hash_spy = mocker.spy(dvc_data.hashfile.hash, "file_md5")
|
||||||
|
|
||||||
@@ -213,7 +213,7 @@ def test_verify_hashes(
|
|||||||
assert hash_spy.call_count == 0
|
assert hash_spy.call_count == 0
|
||||||
|
|
||||||
# Removing cache will invalidate existing state entries
|
# Removing cache will invalidate existing state entries
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
dvc.config["remote"]["upstream"]["verify"] = True
|
dvc.config["remote"]["upstream"]["verify"] = True
|
||||||
|
|
||||||
@@ -235,9 +235,9 @@ def test_pull_git_imports(tmp_dir, dvc, scm, erepo):
|
|||||||
|
|
||||||
assert dvc.pull()["fetched"] == 0
|
assert dvc.pull()["fetched"] == 0
|
||||||
|
|
||||||
for item in ["foo", "new_dir", dvc.odb.local.cache_dir]:
|
for item in ["foo", "new_dir", dvc.odb.local.path]:
|
||||||
remove(item)
|
remove(item)
|
||||||
os.makedirs(dvc.odb.local.cache_dir, exist_ok=True)
|
os.makedirs(dvc.odb.local.path, exist_ok=True)
|
||||||
clean_repos()
|
clean_repos()
|
||||||
|
|
||||||
assert dvc.pull(force=True)["fetched"] == 3
|
assert dvc.pull(force=True)["fetched"] == 3
|
||||||
@@ -296,12 +296,12 @@ def test_pull_external_dvc_imports_mixed(
|
|||||||
|
|
||||||
def clean(outs, dvc=None):
|
def clean(outs, dvc=None):
|
||||||
if dvc:
|
if dvc:
|
||||||
outs = outs + [dvc.odb.local.cache_dir]
|
outs = outs + [dvc.odb.local.path]
|
||||||
for path in outs:
|
for path in outs:
|
||||||
print(path)
|
print(path)
|
||||||
remove(path)
|
remove(path)
|
||||||
if dvc:
|
if dvc:
|
||||||
os.makedirs(dvc.odb.local.cache_dir, exist_ok=True)
|
os.makedirs(dvc.odb.local.path, exist_ok=True)
|
||||||
clean_repos()
|
clean_repos()
|
||||||
|
|
||||||
|
|
||||||
@@ -474,11 +474,11 @@ def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
|
|||||||
|
|
||||||
dvc.pull("copy-foo-bar")
|
dvc.pull("copy-foo-bar")
|
||||||
assert (tmp_dir / "bar").exists()
|
assert (tmp_dir / "bar").exists()
|
||||||
assert len(recurse_list_dir(dvc.odb.local.cache_dir)) == 1
|
assert len(recurse_list_dir(dvc.odb.local.path)) == 1
|
||||||
clean(["bar"], dvc)
|
clean(["bar"], dvc)
|
||||||
|
|
||||||
dvc.fetch("copy-foo-bar")
|
dvc.fetch("copy-foo-bar")
|
||||||
assert len(recurse_list_dir(dvc.odb.local.cache_dir)) == 1
|
assert len(recurse_list_dir(dvc.odb.local.path)) == 1
|
||||||
|
|
||||||
|
|
||||||
def test_pull_partial(tmp_dir, dvc, local_remote):
|
def test_pull_partial(tmp_dir, dvc, local_remote):
|
||||||
|
|||||||
@@ -101,7 +101,7 @@ def test_not_in_cache(M, tmp_dir, scm, dvc):
|
|||||||
# TODO: investigation required, might return wrong results
|
# TODO: investigation required, might return wrong results
|
||||||
tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
|
tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
|
||||||
tmp_dir.dvc_gen("bar", "bar", commit="add foo")
|
tmp_dir.dvc_gen("bar", "bar", commit="add foo")
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
assert dvc.data_status() == {
|
assert dvc.data_status() == {
|
||||||
**EMPTY_STATUS,
|
**EMPTY_STATUS,
|
||||||
|
|||||||
@@ -252,7 +252,7 @@ def test_diff_no_cache(tmp_dir, scm, dvc):
|
|||||||
)
|
)
|
||||||
scm.tag("v2")
|
scm.tag("v2")
|
||||||
|
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
# invalidate_dir_info to force cache loading
|
# invalidate_dir_info to force cache loading
|
||||||
dvc.odb.local._dir_info = {}
|
dvc.odb.local._dir_info = {}
|
||||||
|
|||||||
@@ -116,7 +116,7 @@ def test_relative_remote(erepo_dir, tmp_dir):
|
|||||||
erepo_dir.dvc.push()
|
erepo_dir.dvc.push()
|
||||||
|
|
||||||
(erepo_dir / "file").unlink()
|
(erepo_dir / "file").unlink()
|
||||||
remove(erepo_dir.dvc.odb.local.cache_dir)
|
remove(erepo_dir.dvc.odb.local.path)
|
||||||
|
|
||||||
url = os.fspath(erepo_dir)
|
url = os.fspath(erepo_dir)
|
||||||
|
|
||||||
@@ -198,7 +198,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
|
|||||||
assert (tmp_dir / "out").read_text() == expected_files
|
assert (tmp_dir / "out").read_text() == expected_files
|
||||||
|
|
||||||
# clear cache to test saving to cache
|
# clear cache to test saving to cache
|
||||||
cache_dir = tmp_dir / repo.odb.local.cache_dir
|
cache_dir = tmp_dir / repo.odb.local.path
|
||||||
remove(cache_dir)
|
remove(cache_dir)
|
||||||
makedirs(cache_dir)
|
makedirs(cache_dir)
|
||||||
|
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ class TestGC(TestDvcGit):
|
|||||||
|
|
||||||
self.bad_cache = [self.dvc.odb.local.oid_to_path(raw_dir_hash)]
|
self.bad_cache = [self.dvc.odb.local.oid_to_path(raw_dir_hash)]
|
||||||
for i in ["123", "234", "345"]:
|
for i in ["123", "234", "345"]:
|
||||||
path = os.path.join(self.dvc.odb.local.cache_dir, i[0:2], i[2:])
|
path = os.path.join(self.dvc.odb.local.path, i[0:2], i[2:])
|
||||||
self.create(path, i)
|
self.create(path, i)
|
||||||
self.bad_cache.append(path)
|
self.bad_cache.append(path)
|
||||||
|
|
||||||
@@ -45,7 +45,7 @@ class TestGC(TestDvcGit):
|
|||||||
self._test_gc()
|
self._test_gc()
|
||||||
|
|
||||||
def _test_gc(self):
|
def _test_gc(self):
|
||||||
self.assertTrue(os.path.isdir(self.dvc.odb.local.cache_dir))
|
self.assertTrue(os.path.isdir(self.dvc.odb.local.path))
|
||||||
for c in self.bad_cache:
|
for c in self.bad_cache:
|
||||||
self.assertFalse(os.path.exists(c))
|
self.assertFalse(os.path.exists(c))
|
||||||
|
|
||||||
@@ -190,11 +190,11 @@ def test_all_commits(tmp_dir, scm, dvc):
|
|||||||
tmp_dir.dvc_gen("testfile", "modified", commit="modified")
|
tmp_dir.dvc_gen("testfile", "modified", commit="modified")
|
||||||
tmp_dir.dvc_gen("testfile", "workspace")
|
tmp_dir.dvc_gen("testfile", "workspace")
|
||||||
|
|
||||||
n = _count_files(dvc.odb.local.cache_dir)
|
n = _count_files(dvc.odb.local.path)
|
||||||
dvc.gc(all_commits=True)
|
dvc.gc(all_commits=True)
|
||||||
|
|
||||||
# Only one uncommitted file should go away
|
# Only one uncommitted file should go away
|
||||||
assert _count_files(dvc.odb.local.cache_dir) == n - 1
|
assert _count_files(dvc.odb.local.path) == n - 1
|
||||||
|
|
||||||
|
|
||||||
def test_gc_no_dir_cache(tmp_dir, dvc):
|
def test_gc_no_dir_cache(tmp_dir, dvc):
|
||||||
@@ -206,9 +206,9 @@ def test_gc_no_dir_cache(tmp_dir, dvc):
|
|||||||
with pytest.raises(CollectCacheError):
|
with pytest.raises(CollectCacheError):
|
||||||
dvc.gc(workspace=True)
|
dvc.gc(workspace=True)
|
||||||
|
|
||||||
assert _count_files(dvc.odb.local.cache_dir) == 4
|
assert _count_files(dvc.odb.local.path) == 4
|
||||||
dvc.gc(force=True, workspace=True)
|
dvc.gc(force=True, workspace=True)
|
||||||
assert _count_files(dvc.odb.local.cache_dir) == 2
|
assert _count_files(dvc.odb.local.path) == 2
|
||||||
|
|
||||||
|
|
||||||
def _count_files(path):
|
def _count_files(path):
|
||||||
@@ -328,14 +328,14 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
|
|||||||
|
|
||||||
run_copy("foo", "foo2", name="copy")
|
run_copy("foo", "foo2", name="copy")
|
||||||
shutil.rmtree(dvc.stage_cache.cache_dir)
|
shutil.rmtree(dvc.stage_cache.cache_dir)
|
||||||
assert _count_files(dvc.odb.local.cache_dir) == 1
|
assert _count_files(dvc.odb.local.path) == 1
|
||||||
dvc.gc(workspace=True, force=True)
|
dvc.gc(workspace=True, force=True)
|
||||||
assert _count_files(dvc.odb.local.cache_dir) == 1
|
assert _count_files(dvc.odb.local.path) == 1
|
||||||
|
|
||||||
# remove pipeline file and lockfile and check
|
# remove pipeline file and lockfile and check
|
||||||
Dvcfile(dvc, PIPELINE_FILE).remove(force=True)
|
Dvcfile(dvc, PIPELINE_FILE).remove(force=True)
|
||||||
dvc.gc(workspace=True, force=True)
|
dvc.gc(workspace=True, force=True)
|
||||||
assert _count_files(dvc.odb.local.cache_dir) == 0
|
assert _count_files(dvc.odb.local.path) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_gc_external_output(tmp_dir, dvc, workspace):
|
def test_gc_external_output(tmp_dir, dvc, workspace):
|
||||||
|
|||||||
@@ -242,7 +242,7 @@ def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir):
|
|||||||
dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported")
|
dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported")
|
||||||
|
|
||||||
remove("dir_imported")
|
remove("dir_imported")
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
dvc.pull(["dir_imported.dvc"])
|
dvc.pull(["dir_imported.dvc"])
|
||||||
|
|
||||||
@@ -258,7 +258,7 @@ def test_pull_wildcard_imported_directory_stage(tmp_dir, dvc, erepo_dir):
|
|||||||
dvc.imp(os.fspath(erepo_dir), "dir123", "dir_imported123")
|
dvc.imp(os.fspath(erepo_dir), "dir123", "dir_imported123")
|
||||||
|
|
||||||
remove("dir_imported123")
|
remove("dir_imported123")
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
dvc.pull(["dir_imported*.dvc"], glob=True)
|
dvc.pull(["dir_imported*.dvc"], glob=True)
|
||||||
|
|
||||||
@@ -483,9 +483,9 @@ def test_pull_imported_stage_from_subrepos(
|
|||||||
dvc.imp(os.fspath(erepo_dir), path, out="out")
|
dvc.imp(os.fspath(erepo_dir), path, out="out")
|
||||||
|
|
||||||
# clean everything
|
# clean everything
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
remove("out")
|
remove("out")
|
||||||
makedirs(dvc.odb.local.cache_dir)
|
makedirs(dvc.odb.local.path)
|
||||||
|
|
||||||
stats = dvc.pull(["out.dvc"])
|
stats = dvc.pull(["out.dvc"])
|
||||||
|
|
||||||
@@ -554,7 +554,7 @@ def test_chained_import(tmp_dir, dvc, make_tmp_dir, erepo_dir, local_cloud):
|
|||||||
with erepo_dir.chdir():
|
with erepo_dir.chdir():
|
||||||
erepo_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="init")
|
erepo_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="init")
|
||||||
erepo_dir.dvc.push()
|
erepo_dir.dvc.push()
|
||||||
remove(erepo_dir.dvc.odb.local.cache_dir)
|
remove(erepo_dir.dvc.odb.local.path)
|
||||||
remove(os.fspath(erepo_dir / "dir"))
|
remove(os.fspath(erepo_dir / "dir"))
|
||||||
|
|
||||||
erepo2 = make_tmp_dir("erepo2", scm=True, dvc=True)
|
erepo2 = make_tmp_dir("erepo2", scm=True, dvc=True)
|
||||||
@@ -562,7 +562,7 @@ def test_chained_import(tmp_dir, dvc, make_tmp_dir, erepo_dir, local_cloud):
|
|||||||
erepo2.dvc.imp(os.fspath(erepo_dir), "dir")
|
erepo2.dvc.imp(os.fspath(erepo_dir), "dir")
|
||||||
erepo2.scm.add("dir.dvc")
|
erepo2.scm.add("dir.dvc")
|
||||||
erepo2.scm.commit("import")
|
erepo2.scm.commit("import")
|
||||||
remove(erepo2.dvc.odb.local.cache_dir)
|
remove(erepo2.dvc.odb.local.path)
|
||||||
remove(os.fspath(erepo2 / "dir"))
|
remove(os.fspath(erepo2 / "dir"))
|
||||||
|
|
||||||
dvc.imp(os.fspath(erepo2), "dir", "dir_imported")
|
dvc.imp(os.fspath(erepo2), "dir", "dir_imported")
|
||||||
@@ -570,14 +570,14 @@ def test_chained_import(tmp_dir, dvc, make_tmp_dir, erepo_dir, local_cloud):
|
|||||||
assert (dst / "foo").read_text() == "foo"
|
assert (dst / "foo").read_text() == "foo"
|
||||||
assert (dst / "bar").read_text() == "bar"
|
assert (dst / "bar").read_text() == "bar"
|
||||||
|
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
remove("dir_imported")
|
remove("dir_imported")
|
||||||
|
|
||||||
# pulled objects should come from the original upstream repo's remote,
|
# pulled objects should come from the original upstream repo's remote,
|
||||||
# no cache or remote should be needed from the intermediate repo
|
# no cache or remote should be needed from the intermediate repo
|
||||||
dvc.pull(["dir_imported.dvc"])
|
dvc.pull(["dir_imported.dvc"])
|
||||||
assert not os.path.exists(erepo_dir.dvc.odb.local.cache_dir)
|
assert not os.path.exists(erepo_dir.dvc.odb.local.path)
|
||||||
assert not os.path.exists(erepo2.dvc.odb.local.cache_dir)
|
assert not os.path.exists(erepo2.dvc.odb.local.path)
|
||||||
assert (dst / "foo").read_text() == "foo"
|
assert (dst / "foo").read_text() == "foo"
|
||||||
assert (dst / "bar").read_text() == "bar"
|
assert (dst / "bar").read_text() == "bar"
|
||||||
|
|
||||||
|
|||||||
@@ -15,12 +15,12 @@ def test_cache(tmp_dir, dvc):
|
|||||||
cache1_md5 = "123"
|
cache1_md5 = "123"
|
||||||
cache2_md5 = "234"
|
cache2_md5 = "234"
|
||||||
cache1 = os.path.join(
|
cache1 = os.path.join(
|
||||||
dvc.odb.local.cache_dir,
|
dvc.odb.local.path,
|
||||||
cache1_md5[0:2],
|
cache1_md5[0:2],
|
||||||
cache1_md5[2:],
|
cache1_md5[2:],
|
||||||
)
|
)
|
||||||
cache2 = os.path.join(
|
cache2 = os.path.join(
|
||||||
dvc.odb.local.cache_dir,
|
dvc.odb.local.path,
|
||||||
cache2_md5[0:2],
|
cache2_md5[0:2],
|
||||||
cache2_md5[2:],
|
cache2_md5[2:],
|
||||||
)
|
)
|
||||||
@@ -63,7 +63,7 @@ def test_external_cache_dir(tmp_dir, dvc, make_tmp_dir):
|
|||||||
|
|
||||||
with dvc.config.edit() as conf:
|
with dvc.config.edit() as conf:
|
||||||
conf["cache"]["dir"] = cache_dir.fs_path
|
conf["cache"]["dir"] = cache_dir.fs_path
|
||||||
assert not os.path.exists(dvc.odb.local.cache_dir)
|
assert not os.path.exists(dvc.odb.local.path)
|
||||||
dvc.odb = ODBManager(dvc)
|
dvc.odb = ODBManager(dvc)
|
||||||
|
|
||||||
tmp_dir.dvc_gen({"foo": "foo"})
|
tmp_dir.dvc_gen({"foo": "foo"})
|
||||||
@@ -186,7 +186,7 @@ def test_shared_cache(tmp_dir, dvc, group):
|
|||||||
with dvc.config.edit() as conf:
|
with dvc.config.edit() as conf:
|
||||||
conf["cache"].update({"shared": "group"})
|
conf["cache"].update({"shared": "group"})
|
||||||
dvc.odb = ODBManager(dvc)
|
dvc.odb = ODBManager(dvc)
|
||||||
cache_dir = dvc.odb.local.cache_dir
|
cache_dir = dvc.odb.local.path
|
||||||
|
|
||||||
assert not os.path.exists(cache_dir)
|
assert not os.path.exists(cache_dir)
|
||||||
|
|
||||||
|
|||||||
@@ -197,7 +197,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
|
|||||||
|
|
||||||
# Push everything and delete local cache
|
# Push everything and delete local cache
|
||||||
dvc.push()
|
dvc.push()
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
baz._collect_used_dir_cache()
|
baz._collect_used_dir_cache()
|
||||||
with patch.object(generic, "transfer", side_effect=Exception):
|
with patch.object(generic, "transfer", side_effect=Exception):
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ def test_remove_broken_symlink(tmp_dir, dvc):
|
|||||||
dvc.odb.local.cache_types = ["symlink"]
|
dvc.odb.local.cache_types = ["symlink"]
|
||||||
|
|
||||||
(stage,) = dvc.add("foo")
|
(stage,) = dvc.add("foo")
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
assert system.is_symlink("foo")
|
assert system.is_symlink("foo")
|
||||||
|
|
||||||
with pytest.raises(ObjectDBError):
|
with pytest.raises(ObjectDBError):
|
||||||
|
|||||||
@@ -831,12 +831,12 @@ class TestReproAllPipelines(SingleStageRun, TestDvc):
|
|||||||
|
|
||||||
class TestReproNoCommit(TestRepro):
|
class TestReproNoCommit(TestRepro):
|
||||||
def test(self):
|
def test(self):
|
||||||
remove(self.dvc.odb.local.cache_dir)
|
remove(self.dvc.odb.local.path)
|
||||||
ret = main(
|
ret = main(
|
||||||
["repro", self._get_stage_target(self.stage), "--no-commit"]
|
["repro", self._get_stage_target(self.stage), "--no-commit"]
|
||||||
)
|
)
|
||||||
self.assertEqual(ret, 0)
|
self.assertEqual(ret, 0)
|
||||||
self.assertEqual(os.listdir(self.dvc.odb.local.cache_dir), ["runs"])
|
self.assertEqual(os.listdir(self.dvc.odb.local.path), ["runs"])
|
||||||
|
|
||||||
|
|
||||||
class TestReproAlreadyCached(TestRepro):
|
class TestReproAlreadyCached(TestRepro):
|
||||||
|
|||||||
@@ -721,12 +721,12 @@ class TestRunCommit(TestDvc):
|
|||||||
)
|
)
|
||||||
self.assertEqual(ret, 0)
|
self.assertEqual(ret, 0)
|
||||||
self.assertTrue(os.path.isfile(fname))
|
self.assertTrue(os.path.isfile(fname))
|
||||||
self.assertFalse(os.path.exists(self.dvc.odb.local.cache_dir))
|
self.assertFalse(os.path.exists(self.dvc.odb.local.path))
|
||||||
|
|
||||||
ret = main(["commit", fname + ".dvc"])
|
ret = main(["commit", fname + ".dvc"])
|
||||||
self.assertEqual(ret, 0)
|
self.assertEqual(ret, 0)
|
||||||
self.assertTrue(os.path.isfile(fname))
|
self.assertTrue(os.path.isfile(fname))
|
||||||
self.assertEqual(len(os.listdir(self.dvc.odb.local.cache_dir)), 1)
|
self.assertEqual(len(os.listdir(self.dvc.odb.local.path)), 1)
|
||||||
|
|
||||||
|
|
||||||
class TestRunPersist(TestDvc):
|
class TestRunPersist(TestDvc):
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ def test_open_dirty_hash(tmp_dir, dvc):
|
|||||||
def test_open_no_remote(tmp_dir, dvc):
|
def test_open_no_remote(tmp_dir, dvc):
|
||||||
tmp_dir.dvc_gen("file", "file")
|
tmp_dir.dvc_gen("file", "file")
|
||||||
(tmp_dir / "file").unlink()
|
(tmp_dir / "file").unlink()
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
fs = DataFileSystem(index=dvc.index.data["repo"])
|
fs = DataFileSystem(index=dvc.index.data["repo"])
|
||||||
with pytest.raises(FileNotFoundError):
|
with pytest.raises(FileNotFoundError):
|
||||||
|
|||||||
@@ -111,7 +111,7 @@ def test_remote_missing_depenency_on_dir_pull(tmp_dir, scm, dvc, mocker):
|
|||||||
conf["core"] = {"remote": "s3"}
|
conf["core"] = {"remote": "s3"}
|
||||||
|
|
||||||
remove("dir")
|
remove("dir")
|
||||||
remove(dvc.odb.local.cache_dir)
|
remove(dvc.odb.local.path)
|
||||||
|
|
||||||
with mocker.patch(
|
with mocker.patch(
|
||||||
"dvc.data_cloud.DataCloud.get_remote_odb",
|
"dvc.data_cloud.DataCloud.get_remote_odb",
|
||||||
|
|||||||
@@ -176,7 +176,7 @@ def test_shared_stage_cache(tmp_dir, dvc, run_copy):
|
|||||||
|
|
||||||
dvc.odb = ODBManager(dvc)
|
dvc.odb = ODBManager(dvc)
|
||||||
|
|
||||||
assert not os.path.exists(dvc.odb.local.cache_dir)
|
assert not os.path.exists(dvc.odb.local.path)
|
||||||
|
|
||||||
run_copy("foo", "bar", name="copy-foo-bar")
|
run_copy("foo", "bar", name="copy-foo-bar")
|
||||||
|
|
||||||
@@ -205,7 +205,7 @@ def test_shared_stage_cache(tmp_dir, dvc, run_copy):
|
|||||||
dir_mode = 0o2775
|
dir_mode = 0o2775
|
||||||
file_mode = 0o664
|
file_mode = 0o664
|
||||||
|
|
||||||
assert _mode(dvc.odb.local.cache_dir) == dir_mode
|
assert _mode(dvc.odb.local.path) == dir_mode
|
||||||
assert _mode(dvc.stage_cache.cache_dir) == dir_mode
|
assert _mode(dvc.stage_cache.cache_dir) == dir_mode
|
||||||
assert _mode(parent_cache_dir) == dir_mode
|
assert _mode(parent_cache_dir) == dir_mode
|
||||||
assert _mode(cache_dir) == dir_mode
|
assert _mode(cache_dir) == dir_mode
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ def test_subrepo_is_constructed_properly(
|
|||||||
|
|
||||||
subrepo = tmp_dir / "subrepo"
|
subrepo = tmp_dir / "subrepo"
|
||||||
make_subrepo(subrepo, scm)
|
make_subrepo(subrepo, scm)
|
||||||
local_cache = subrepo.dvc.odb.local.cache_dir
|
local_cache = subrepo.dvc.odb.local.path
|
||||||
|
|
||||||
tmp_dir.scm_gen("bar", "bar", commit="add bar")
|
tmp_dir.scm_gen("bar", "bar", commit="add bar")
|
||||||
subrepo.dvc_gen("foo", "foo", commit="add foo")
|
subrepo.dvc_gen("foo", "foo", commit="add foo")
|
||||||
@@ -72,8 +72,8 @@ def test_subrepo_is_constructed_properly(
|
|||||||
|
|
||||||
assert repo.url == str(tmp_dir)
|
assert repo.url == str(tmp_dir)
|
||||||
assert repo.config["cache"]["dir"] == str(cache_dir)
|
assert repo.config["cache"]["dir"] == str(cache_dir)
|
||||||
assert repo.odb.local.cache_dir == str(cache_dir)
|
assert repo.odb.local.path == str(cache_dir)
|
||||||
assert subrepo.odb.local.cache_dir == str(cache_dir)
|
assert subrepo.odb.local.path == str(cache_dir)
|
||||||
|
|
||||||
assert repo.config["cache"]["type"] == ["symlink"]
|
assert repo.config["cache"]["type"] == ["symlink"]
|
||||||
assert repo.odb.local.cache_types == ["symlink"]
|
assert repo.odb.local.cache_types == ["symlink"]
|
||||||
@@ -84,7 +84,7 @@ def test_subrepo_is_constructed_properly(
|
|||||||
== local_cache
|
== local_cache
|
||||||
)
|
)
|
||||||
if root_is_dvc:
|
if root_is_dvc:
|
||||||
main_cache = tmp_dir.dvc.odb.local.cache_dir
|
main_cache = tmp_dir.dvc.odb.local.path
|
||||||
assert repo.config["remote"]["auto-generated-upstream"][
|
assert repo.config["remote"]["auto-generated-upstream"][
|
||||||
"url"
|
"url"
|
||||||
] == str(main_cache)
|
] == str(main_cache)
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ def find_supported_remotes(string):
|
|||||||
def test_info_in_repo(scm_init, tmp_dir):
|
def test_info_in_repo(scm_init, tmp_dir):
|
||||||
tmp_dir.init(scm=scm_init, dvc=True)
|
tmp_dir.init(scm=scm_init, dvc=True)
|
||||||
# Create `.dvc/cache`, that is needed to check supported link types.
|
# Create `.dvc/cache`, that is needed to check supported link types.
|
||||||
os.mkdir(tmp_dir.dvc.odb.local.cache_dir)
|
os.mkdir(tmp_dir.dvc.odb.local.path)
|
||||||
|
|
||||||
dvc_info = get_dvc_info()
|
dvc_info = get_dvc_info()
|
||||||
|
|
||||||
@@ -116,7 +116,7 @@ def test_remotes(tmp_dir, dvc, caplog):
|
|||||||
|
|
||||||
|
|
||||||
def test_fs_info_in_repo(tmp_dir, dvc, caplog):
|
def test_fs_info_in_repo(tmp_dir, dvc, caplog):
|
||||||
os.mkdir(dvc.odb.local.cache_dir)
|
os.mkdir(dvc.odb.local.path)
|
||||||
dvc_info = get_dvc_info()
|
dvc_info = get_dvc_info()
|
||||||
|
|
||||||
assert re.search(r"Cache directory: .* on .*", dvc_info)
|
assert re.search(r"Cache directory: .* on .*", dvc_info)
|
||||||
|
|||||||
Reference in New Issue
Block a user