From 603f5fdba1e22de2adf227c37203074df4e098e6 Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Mon, 5 Jun 2023 17:48:17 +0900 Subject: [PATCH] cache: move 3.x data objects into files/md5/ prefix --- dvc/cachemgr.py | 41 ++++++++++++++++++++++++-- dvc/data_cloud.py | 21 +++++++++++-- dvc/dependency/repo.py | 2 +- dvc/info.py | 2 +- dvc/output.py | 14 +++++++-- dvc/repo/__init__.py | 4 +-- dvc/repo/experiments/executor/local.py | 2 +- dvc/repo/open_repo.py | 2 +- dvc/stage/cache.py | 26 +++++++++------- dvc/testing/api_tests.py | 4 ++- dvc/testing/workspace_tests.py | 27 +++++++++++++---- tests/func/api/test_data.py | 26 +++++++++++----- tests/func/test_add.py | 24 +++++++++++++-- tests/func/test_commit.py | 4 +-- tests/func/test_data_cloud.py | 4 +-- tests/func/test_gc.py | 26 +++++++++++----- tests/func/test_install.py | 4 ++- tests/func/test_odb.py | 10 +++---- tests/func/test_run.py | 2 +- tests/func/test_unprotect.py | 4 ++- tests/unit/repo/test_open_repo.py | 12 ++++---- tests/unit/test_info.py | 4 +-- 22 files changed, 195 insertions(+), 70 deletions(-) diff --git a/dvc/cachemgr.py b/dvc/cachemgr.py index 9885a23a2..3c691ba6d 100644 --- a/dvc/cachemgr.py +++ b/dvc/cachemgr.py @@ -1,10 +1,21 @@ import os +from typing import Optional, Tuple from dvc.fs import GitFileSystem, Schemes from dvc_data.hashfile.db import get_odb +from dvc_data.hashfile.hash import DEFAULT_ALGORITHM + +LEGACY_HASH_NAMES = {"md5-dos2unix", "params"} -def _get_odb(repo, settings, fs=None): +def _get_odb( + repo, + settings, + fs=None, + prefix: Optional[Tuple[str, ...]] = None, + hash_name: Optional[str] = None, + **kwargs, +): from dvc.fs import get_cloud_fs if not settings: @@ -12,6 +23,10 @@ def _get_odb(repo, settings, fs=None): cls, config, fs_path = get_cloud_fs(repo, **settings) fs = fs or cls(**config) + if prefix: + fs_path = fs.path.join(fs_path, *prefix) + if hash_name: + config["hash_name"] = hash_name return get_odb(fs, fs_path, state=repo.state, **config) @@ -24,6 +39,7 @@ class CacheManager: Schemes.HDFS, Schemes.WEBHDFS, ] + FILES_DIR = "files" def __init__(self, repo): self._repo = repo @@ -53,15 +69,26 @@ class CacheManager: if not isinstance(repo.fs, GitFileSystem): kwargs["fs"] = repo.fs - odb = _get_odb(repo, settings, **kwargs) + odb = _get_odb( + repo, + settings, + prefix=(self.FILES_DIR, DEFAULT_ALGORITHM), + **kwargs, + ) self._odb["repo"] = odb self._odb[Schemes.LOCAL] = odb + legacy_odb = _get_odb(repo, settings, hash_name="md5-dos2unix", **kwargs) + self._odb["legacy"] = legacy_odb def _init_odb(self, schemes): for scheme in schemes: remote = self.config.get(scheme) settings = {"name": remote} if remote else None - self._odb[scheme] = _get_odb(self._repo, settings) + self._odb[scheme] = _get_odb( + self._repo, + settings, + prefix=(self.FILES_DIR, DEFAULT_ALGORITHM), + ) def __getattr__(self, name): if name not in self._odb and name in self.CLOUD_SCHEMES: @@ -75,3 +102,11 @@ class CacheManager: def by_scheme(self): self._init_odb(self.CLOUD_SCHEMES) yield from self._odb.items() + + @property + def local_cache_dir(self) -> str: + """Return base local cache directory without any prefixes. + + (i.e. `dvc cache dir`). + """ + return self.legacy.path diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 8e64f6ab5..d8f9b038f 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -29,13 +29,25 @@ class Remote: @cached_property def odb(self) -> "HashFileDB": + from dvc.cachemgr import CacheManager from dvc_data.hashfile.db import get_odb + from dvc_data.hashfile.hash import DEFAULT_ALGORITHM path = self.path if self.worktree: - path = self.fs.path.join(path, ".dvc", "cache") + path = self.fs.path.join( + path, ".dvc", CacheManager.FILES_DIR, DEFAULT_ALGORITHM + ) + else: + path = self.fs.path.join(path, CacheManager.FILES_DIR, DEFAULT_ALGORITHM) + return get_odb(self.fs, path, hash_name=DEFAULT_ALGORITHM, **self.config) - return get_odb(self.fs, path, hash_name="md5", **self.config) + @cached_property + def legacy_odb(self) -> "HashFileDB": + from dvc_data.hashfile.db import get_odb + + path = self.path + return get_odb(self.fs, path, hash_name="md5-dos2unix", **self.config) class DataCloud: @@ -101,12 +113,17 @@ class DataCloud: self, name: Optional[str] = None, command: str = "", + hash_name: str = "md5", ) -> "HashFileDB": + from dvc.cachemgr import LEGACY_HASH_NAMES + remote = self.get_remote(name=name, command=command) if remote.fs.version_aware or remote.worktree: raise NoRemoteError( f"'{command}' is unsupported for cloud versioned remotes" ) + if hash_name in LEGACY_HASH_NAMES: + return remote.legacy_odb return remote.odb def _log_missing(self, status: "CompareStatusResult"): diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index e0ffa4cec..8e5956005 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -80,7 +80,7 @@ class RepoDependency(Dependency): from dvc.fs import DVCFileSystem config = {"cache": self.repo.config["cache"]} - config["cache"]["dir"] = self.repo.cache.local.path + config["cache"]["dir"] = self.repo.cache.local_cache_dir return DVCFileSystem( url=self.def_repo[self.PARAM_URL], diff --git a/dvc/info.py b/dvc/info.py index 9a8f5332f..fedf4a99e 100644 --- a/dvc/info.py +++ b/dvc/info.py @@ -67,7 +67,7 @@ def _get_caches(cache): caches = ( cache_type for cache_type, cache_instance in cache.by_scheme() - if cache_instance and cache_type != "repo" + if cache_instance and cache_type not in ("repo", "legacy") ) # Caches will be always non-empty including the local cache diff --git a/dvc/output.py b/dvc/output.py index 3e86a4804..3bc492376 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -500,6 +500,14 @@ class Output: raise RemoteCacheRequiredError(self.fs.protocol, self.fs_path) return odb + @property + def local_cache(self): + from dvc.cachemgr import LEGACY_HASH_NAMES + + if self.hash_name in LEGACY_HASH_NAMES: + return self.repo.cache.legacy + return self.repo.cache.local + @property def cache_path(self): return self.cache.fs.unstrip_protocol( @@ -514,7 +522,7 @@ class Output: if self.use_cache: odb = self.cache else: - odb = self.repo.cache.local + odb = self.local_cache _, meta, obj = build( odb, self.fs_path, @@ -665,7 +673,7 @@ class Output: ) else: _, self.meta, self.obj = build( - self.repo.cache.local, + self.local_cache, self.fs_path, self.fs, self.hash_name, @@ -1283,7 +1291,7 @@ class Output: ) assert self.repo - cache = self.cache if self.use_cache else self.repo.cache.local + cache = self.cache if self.use_cache else self.local_cache assert isinstance(cache, HashFileDB) new: "HashFile" diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index d18b4a693..e51fe3feb 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -420,8 +420,8 @@ class Repo: flist = [self.config.files["local"]] if tmp_dir := self.tmp_dir: flist.append(tmp_dir) - if path_isin(self.cache.repo.path, self.root_dir): - flist.append(self.cache.repo.path) + if path_isin(self.cache.legacy.path, self.root_dir): + flist.append(self.cache.legacy.path) for file in flist: self.scm_context.ignore(file) diff --git a/dvc/repo/experiments/executor/local.py b/dvc/repo/experiments/executor/local.py index 03a4ac753..9ca9f4132 100644 --- a/dvc/repo/experiments/executor/local.py +++ b/dvc/repo/experiments/executor/local.py @@ -152,7 +152,7 @@ class TempDirExecutor(BaseLocalExecutor): self, repo: "Repo", rev: str, run_cache: bool = True # noqa: ARG002 ): """Initialize DVC cache.""" - self._update_config({"cache": {"dir": repo.cache.repo.path}}) + self._update_config({"cache": {"dir": repo.cache.local_cache_dir}}) def cleanup(self, infofile: Optional[str] = None): super().cleanup(infofile) diff --git a/dvc/repo/open_repo.py b/dvc/repo/open_repo.py index f2237a9c6..bc01f9dff 100644 --- a/dvc/repo/open_repo.py +++ b/dvc/repo/open_repo.py @@ -129,7 +129,7 @@ def _get_remote_config(url): name = "auto-generated-upstream" return { "core": {"remote": name}, - "remote": {name: {"url": repo.cache.local.path}}, + "remote": {name: {"url": repo.cache.local_cache_dir}}, } # Use original remote to make sure that we are using correct url, diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index 332059b0d..d236334dc 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -60,7 +60,7 @@ def _get_stage_hash(stage): class StageCache: def __init__(self, repo): self.repo = repo - self.cache_dir = os.path.join(self.repo.cache.local.path, "runs") + self.cache_dir = os.path.join(self.repo.cache.legacy.path, "runs") def _get_cache_dir(self, key): return os.path.join(self.cache_dir, key[:2], key) @@ -119,12 +119,12 @@ class StageCache: @contextmanager def _cache_type_copy(self): - cache_types = self.repo.cache.local.cache_types - self.repo.cache.local.cache_types = ["copy"] + cache_types = self.repo.cache.legacy.cache_types + self.repo.cache.legacy.cache_types = ["copy"] try: yield finally: - self.repo.cache.local.cache_types = cache_types + self.repo.cache.legacy.cache_types = cache_types def _uncached_outs(self, stage, cache): # NOTE: using temporary stage to avoid accidentally modifying original @@ -166,14 +166,14 @@ class StageCache: COMPILED_LOCK_FILE_STAGE_SCHEMA(cache) path = self._get_cache_path(cache_key, cache_value) - local_fs = self.repo.cache.local.fs + local_fs = self.repo.cache.legacy.fs parent = local_fs.path.parent(path) - self.repo.cache.local.makedirs(parent) + self.repo.cache.legacy.makedirs(parent) tmp = local_fs.path.join(parent, fs.utils.tmp_fname()) assert os.path.exists(parent) assert os.path.isdir(parent) dump_yaml(tmp, cache) - self.repo.cache.local.move(tmp, path) + self.repo.cache.legacy.move(tmp, path) def restore(self, stage, run_cache=True, pull=False, dry=False): from .serialize import to_single_stage_lockfile @@ -258,12 +258,16 @@ class StageCache: return ret def push(self, remote: Optional[str], odb: Optional["ObjectDB"] = None): - dest_odb = odb or self.repo.cloud.get_remote_odb(remote, "push --run-cache") - return self.transfer(self.repo.cache.local, dest_odb) + dest_odb = odb or self.repo.cloud.get_remote_odb( + remote, "push --run-cache", hash_name="md5-dos2unix" + ) + return self.transfer(self.repo.cache.legacy, dest_odb) def pull(self, remote: Optional[str], odb: Optional["ObjectDB"] = None): - odb = odb or self.repo.cloud.get_remote_odb(remote, "fetch --run-cache") - return self.transfer(odb, self.repo.cache.local) + odb = odb or self.repo.cloud.get_remote_odb( + remote, "fetch --run-cache", hash_name="md5-dos2unix" + ) + return self.transfer(odb, self.repo.cache.legacy) def get_used_objs(self, used_run_cache, *args, **kwargs): """Return used cache for the specified run-cached stages.""" diff --git a/dvc/testing/api_tests.py b/dvc/testing/api_tests.py index f340b02bb..925b25844 100644 --- a/dvc/testing/api_tests.py +++ b/dvc/testing/api_tests.py @@ -11,7 +11,9 @@ class TestAPI: def test_get_url(self, tmp_dir, dvc, remote): tmp_dir.dvc_gen("foo", "foo") - expected_url = (remote / "ac/bd18db4cc2f85cedef654fccc4a4d8").url + expected_url = ( + remote / "files" / "md5" / "ac/bd18db4cc2f85cedef654fccc4a4d8" + ).url assert api.get_url("foo") == expected_url def test_open(self, tmp_dir, dvc, remote): diff --git a/dvc/testing/workspace_tests.py b/dvc/testing/workspace_tests.py index 1f3cc794f..bef340d13 100644 --- a/dvc/testing/workspace_tests.py +++ b/dvc/testing/workspace_tests.py @@ -181,7 +181,7 @@ class TestAdd: ) assert (workspace / "file").read_text() == "file" assert ( - workspace / "cache" / hash_value[:2] / hash_value[2:] + workspace / "cache" / "files" / "md5" / hash_value[:2] / hash_value[2:] ).read_text() == "file" assert dvc.status() == {} @@ -191,7 +191,14 @@ class TestAdd: workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}}) dvc.add("remote://workspace/dir") - assert (workspace / "cache" / dir_hash_value[:2] / dir_hash_value[2:]).is_file() + assert ( + workspace + / "cache" + / "files" + / "md5" + / dir_hash_value[:2] + / dir_hash_value[2:] + ).is_file() def match_files(fs, entries, expected): @@ -301,7 +308,9 @@ class TestToRemote: meta = stage.outs[0].meta assert hash_info.name == "md5" assert hash_info.value == "acbd18db4cc2f85cedef654fccc4a4d8" - assert (remote / "ac" / "bd18db4cc2f85cedef654fccc4a4d8").read_text() == "foo" + assert ( + remote / "files" / "md5" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8" + ).read_text() == "foo" assert meta.size == len("foo") def test_import_url_to_remote_file(self, tmp_dir, dvc, workspace, remote): @@ -321,7 +330,9 @@ class TestToRemote: hash_info = stage.outs[0].hash_info assert hash_info.name == "md5" assert hash_info.value == "acbd18db4cc2f85cedef654fccc4a4d8" - assert (remote / "ac" / "bd18db4cc2f85cedef654fccc4a4d8").read_text() == "foo" + assert ( + remote / "files" / "md5" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8" + ).read_text() == "foo" assert stage.outs[0].meta.size == len("foo") def test_import_url_to_remote_dir(self, tmp_dir, dvc, workspace, remote): @@ -351,7 +362,9 @@ class TestToRemote: assert hash_info.name == "md5" assert hash_info.value == "55d05978954d1b2cd7b06aedda9b9e43.dir" file_parts = json.loads( - (remote / "55" / "d05978954d1b2cd7b06aedda9b9e43.dir").read_text() + ( + remote / "files" / "md5" / "55" / "d05978954d1b2cd7b06aedda9b9e43.dir" + ).read_text() ) assert len(file_parts) == 3 @@ -363,4 +376,6 @@ class TestToRemote: for file_part in file_parts: md5 = file_part["md5"] - assert (remote / md5[:2] / md5[2:]).read_text() == file_part["relpath"] + assert ( + remote / "files" / "md5" / md5[:2] / md5[2:] + ).read_text() == file_part["relpath"] diff --git a/tests/func/api/test_data.py b/tests/func/api/test_data.py index 4eea99159..c4ab642c8 100644 --- a/tests/func/api/test_data.py +++ b/tests/func/api/test_data.py @@ -17,7 +17,7 @@ def test_get_url_external(tmp_dir, erepo_dir, cloud): # Using file url to force clone to tmp repo repo_url = f"file://{erepo_dir.as_posix()}" - expected_url = (cloud / "ac/bd18db4cc2f85cedef654fccc4a4d8").url + expected_url = (cloud / "files" / "md5" / "ac/bd18db4cc2f85cedef654fccc4a4d8").url assert api.get_url("foo", repo=repo_url) == expected_url @@ -166,16 +166,24 @@ def test_get_url_granular(tmp_dir, dvc, cloud): tmp_dir.add_remote(config=cloud.config) tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar", "nested": {"file": "file"}}}) - expected_url = (cloud / "5f" / "c28ea78987408341668eba6525ebd1.dir").url + expected_url = ( + cloud / "files" / "md5" / "5f" / "c28ea78987408341668eba6525ebd1.dir" + ).url assert api.get_url("dir") == expected_url - expected_url = (cloud / "ac" / "bd18db4cc2f85cedef654fccc4a4d8").url + expected_url = ( + cloud / "files" / "md5" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8" + ).url assert api.get_url("dir/foo") == expected_url - expected_url = (cloud / "37" / "b51d194a7513e45b56f6524f2d51f2").url + expected_url = ( + cloud / "files" / "md5" / "37" / "b51d194a7513e45b56f6524f2d51f2" + ).url assert api.get_url("dir/bar") == expected_url - expected_url = (cloud / "8c" / "7dd922ad47494fc02c388e12c00eac").url + expected_url = ( + cloud / "files" / "md5" / "8c" / "7dd922ad47494fc02c388e12c00eac" + ).url assert api.get_url(os.path.join("dir", "nested", "file")) == expected_url @@ -186,10 +194,14 @@ def test_get_url_subrepos(tmp_dir, scm, local_cloud): subrepo.dvc_gen({"dir": {"foo": "foo"}, "bar": "bar"}, commit="add files") subrepo.dvc.push() - expected_url = os.fspath(local_cloud / "ac" / "bd18db4cc2f85cedef654fccc4a4d8") + expected_url = os.fspath( + local_cloud / "files" / "md5" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8" + ) assert api.get_url(os.path.join("subrepo", "dir", "foo")) == expected_url - expected_url = os.fspath(local_cloud / "37" / "b51d194a7513e45b56f6524f2d51f2") + expected_url = os.fspath( + local_cloud / "files" / "md5" / "37" / "b51d194a7513e45b56f6524f2d51f2" + ) assert api.get_url("subrepo/bar") == expected_url diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 84cc0bf23..ecf226889 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -729,10 +729,22 @@ def test_add_symlink_file(tmp_dir, dvc): assert (tmp_dir / "dir" / "bar").read_text() == "bar" assert ( - tmp_dir / ".dvc" / "cache" / "37" / "b51d194a7513e45b56f6524f2d51f2" + tmp_dir + / ".dvc" + / "cache" + / "files" + / "md5" + / "37" + / "b51d194a7513e45b56f6524f2d51f2" ).read_text() == "bar" assert not ( - tmp_dir / ".dvc" / "cache" / "37" / "b51d194a7513e45b56f6524f2d51f2" + tmp_dir + / ".dvc" + / "cache" + / "files" + / "md5" + / "37" + / "b51d194a7513e45b56f6524f2d51f2" ).is_symlink() # Test that subsequent add succeeds @@ -790,7 +802,13 @@ def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys): assert (tmp_dir / "foo").exists() assert (tmp_dir / "foo.dvc").exists() assert ( - tmp_dir / ".dvc" / "cache" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8" + tmp_dir + / ".dvc" + / "cache" + / "files" + / "md5" + / "ac" + / "bd18db4cc2f85cedef654fccc4a4d8" ).read_text() == "foo" diff --git a/tests/func/test_commit.py b/tests/func/test_commit.py index 7c82d4221..d97f9817f 100644 --- a/tests/func/test_commit.py +++ b/tests/func/test_commit.py @@ -134,7 +134,7 @@ def test_commit_granular_output(tmp_dir, dvc): no_commit=True, ) - cache = tmp_dir / ".dvc" / "cache" + cache = tmp_dir / ".dvc" / "cache" / "files" / "md5" assert not list(cache.glob("*/*")) dvc.commit("foo") @@ -175,7 +175,7 @@ def test_commit_granular_dir(tmp_dir, dvc): ) dvc.add("data", no_commit=True) - cache = tmp_dir / ".dvc" / "cache" + cache = tmp_dir / ".dvc" / "cache" / "files" / "md5" assert set(cache.glob("*/*")) == set() diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 4877497ae..125a47dbd 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -470,11 +470,11 @@ def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote): dvc.pull("copy-foo-bar") assert (tmp_dir / "bar").exists() - assert len(recurse_list_dir(dvc.cache.local.path)) == 2 + assert len(recurse_list_dir(dvc.cache.local.path)) == 1 clean(["bar"], dvc) dvc.fetch("copy-foo-bar") - assert len(recurse_list_dir(dvc.cache.local.path)) == 2 + assert len(recurse_list_dir(dvc.cache.local.path)) == 1 def test_pull_partial(tmp_dir, dvc, local_remote): diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index 1ee79bdca..5e47af953 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -260,15 +260,23 @@ def test_gc_external_output(tmp_dir, dvc, workspace): foo_hash = foo_stage.outs[0].hash_info.value bar_hash = bar_stage.outs[0].hash_info.value - assert (workspace / "cache" / foo_hash[:2] / foo_hash[2:]).read_text() == "foo" - assert (workspace / "cache" / bar_hash[:2] / bar_hash[2:]).read_text() == "bar" + assert ( + workspace / "cache" / "files" / "md5" / foo_hash[:2] / foo_hash[2:] + ).read_text() == "foo" + assert ( + workspace / "cache" / "files" / "md5" / bar_hash[:2] / bar_hash[2:] + ).read_text() == "bar" (tmp_dir / "foo.dvc").unlink() dvc.gc(workspace=True) - assert not (workspace / "cache" / foo_hash[:2] / foo_hash[2:]).exists() - assert (workspace / "cache" / bar_hash[:2] / bar_hash[2:]).read_text() == "bar" + assert not ( + workspace / "cache" / "files" / "md5" / foo_hash[:2] / foo_hash[2:] + ).exists() + assert ( + workspace / "cache" / "files" / "md5" / bar_hash[:2] / bar_hash[2:] + ).read_text() == "bar" def test_gc_all_experiments(tmp_dir, scm, dvc): @@ -288,9 +296,11 @@ def test_gc_all_experiments(tmp_dir, scm, dvc): dvc.gc(all_experiments=True, force=True) - assert not (tmp_dir / ".dvc" / "cache" / foo_hash[:2] / foo_hash[2:]).exists() + assert not ( + tmp_dir / ".dvc" / "cache" / "files" / "md5" / foo_hash[:2] / foo_hash[2:] + ).exists() assert ( - tmp_dir / ".dvc" / "cache" / baz_hash[:2] / baz_hash[2:] + tmp_dir / ".dvc" / "cache" / "files" / "md5" / baz_hash[:2] / baz_hash[2:] ).read_text() == "baz" @@ -306,7 +316,9 @@ def test_gc_rev_num(tmp_dir, scm, dvc): dvc.gc(rev="HEAD", num=num, force=True) for n, i in enumerate(reversed(range(4))): - cache = tmp_dir / ".dvc" / "cache" / hashes[i][:2] / hashes[i][2:] + cache = ( + tmp_dir / ".dvc" / "cache" / "files" / "md5" / hashes[i][:2] / hashes[i][2:] + ) if n >= num: assert not cache.exists() else: diff --git a/tests/func/test_install.py b/tests/func/test_install.py index cae020185..43ceb2b72 100644 --- a/tests/func/test_install.py +++ b/tests/func/test_install.py @@ -77,7 +77,9 @@ class TestInstall: tmp_dir.dvc_gen("file", "file_content", "commit message") file_checksum = file_md5("file", dvc.fs) - expected_storage_path = storage_path / file_checksum[:2] / file_checksum[2:] + expected_storage_path = ( + storage_path / "files" / "md5" / file_checksum[:2] / file_checksum[2:] + ) scm.gitpython.repo.clone(os.fspath(git_remote)) scm.gitpython.repo.create_remote("origin", os.fspath(git_remote)) diff --git a/tests/func/test_odb.py b/tests/func/test_odb.py index 9427ecdc8..5e5f5b275 100644 --- a/tests/func/test_odb.py +++ b/tests/func/test_odb.py @@ -89,7 +89,7 @@ def test_remote_cache_references(tmp_dir, dvc): dvc.cache = CacheManager(dvc) - assert dvc.cache.ssh.path == "/tmp" + assert dvc.cache.ssh.path == "/tmp/files/md5" def test_shared_cache_dir(tmp_dir): @@ -113,13 +113,13 @@ def test_shared_cache_dir(tmp_dir): assert not os.path.exists(os.path.join("dir1", ".dvc", "cache")) assert not os.path.exists(os.path.join("dir2", ".dvc", "cache")) assert os.path.exists( - os.path.join(cache_dir, "dc", "f6c2fa538b445a3a095255c3641dfc") + os.path.join(cache_dir, "files", "md5", "dc", "f6c2fa538b445a3a095255c3641dfc") ) assert os.path.exists( - os.path.join(cache_dir, "b4", "333c8cfa2ebba7ef20ec6c3265902b") + os.path.join(cache_dir, "files", "md5", "b4", "333c8cfa2ebba7ef20ec6c3265902b") ) assert os.path.exists( - os.path.join(cache_dir, "9e", "fab2399c7c560b34de477b9aa0a465") + os.path.join(cache_dir, "files", "md5", "9e", "fab2399c7c560b34de477b9aa0a465") ) @@ -165,7 +165,7 @@ def test_cmd_cache_relative_path(tmp_dir, scm, dvc, make_tmp_dir): tmp_dir.dvc_gen({"foo": "foo"}) assert os.path.exists( - os.path.join(cache_dir, "ac", "bd18db4cc2f85cedef654fccc4a4d8") + os.path.join(cache_dir, "files", "md5", "ac", "bd18db4cc2f85cedef654fccc4a4d8") ) diff --git a/tests/func/test_run.py b/tests/func/test_run.py index 55f99d20c..a6d6871d8 100644 --- a/tests/func/test_run.py +++ b/tests/func/test_run.py @@ -573,7 +573,7 @@ def test_run_external_outputs( assert (local_workspace / "foo").read_text() == "foo" assert (local_workspace / "bar").read_text() == "bar" assert ( - local_workspace / "cache" / bar_hash[:2] / bar_hash[2:] + local_workspace / "cache" / "files" / "md5" / bar_hash[:2] / bar_hash[2:] ).read_text() == "bar" diff --git a/tests/func/test_unprotect.py b/tests/func/test_unprotect.py index abdc59b28..0c3edbb36 100644 --- a/tests/func/test_unprotect.py +++ b/tests/func/test_unprotect.py @@ -6,7 +6,9 @@ def test_unprotect(tmp_dir, dvc): dvc.cache.local.cache_types = ["hardlink"] dvc.add("foo") - cache = os.path.join(".dvc", "cache", "ac", "bd18db4cc2f85cedef654fccc4a4d8") + cache = os.path.join( + ".dvc", "cache", "files", "md5", "ac", "bd18db4cc2f85cedef654fccc4a4d8" + ) assert not os.access("foo", os.W_OK) assert not os.access(cache, os.W_OK) diff --git a/tests/unit/repo/test_open_repo.py b/tests/unit/repo/test_open_repo.py index 0db2a76ab..a8b06c31f 100644 --- a/tests/unit/repo/test_open_repo.py +++ b/tests/unit/repo/test_open_repo.py @@ -55,7 +55,7 @@ def test_subrepo_is_constructed_properly( subrepo = tmp_dir / "subrepo" make_subrepo(subrepo, scm) - local_cache = subrepo.dvc.cache.local.path + local_cache = subrepo.dvc.cache.local_cache_dir tmp_dir.scm_gen("bar", "bar", commit="add bar") subrepo.dvc_gen("foo", "foo", commit="add foo") @@ -75,8 +75,8 @@ def test_subrepo_is_constructed_properly( assert repo.url == str(tmp_dir) assert repo.config["cache"]["dir"] == str(cache_dir) - assert repo.cache.local.path == str(cache_dir) - assert subrepo.cache.local.path == str(cache_dir) + assert repo.cache.local.path == os.path.join(cache_dir, "files", "md5") + assert subrepo.cache.local.path == os.path.join(cache_dir, "files", "md5") assert repo.config["cache"]["type"] == ["symlink"] assert repo.cache.local.cache_types == ["symlink"] @@ -84,7 +84,5 @@ def test_subrepo_is_constructed_properly( assert subrepo.config["remote"]["auto-generated-upstream"]["url"] == local_cache if root_is_dvc: - main_cache = tmp_dir.dvc.cache.local.path - assert repo.config["remote"]["auto-generated-upstream"]["url"] == str( - main_cache - ) + main_cache = tmp_dir.dvc.cache.local_cache_dir + assert repo.config["remote"]["auto-generated-upstream"]["url"] == main_cache diff --git a/tests/unit/test_info.py b/tests/unit/test_info.py index 5de458627..7cf27fe73 100644 --- a/tests/unit/test_info.py +++ b/tests/unit/test_info.py @@ -45,7 +45,7 @@ def find_supported_remotes(string): def test_info_in_repo(scm_init, tmp_dir): tmp_dir.init(scm=scm_init, dvc=True) # Create `.dvc/cache`, that is needed to check supported link types. - os.mkdir(tmp_dir.dvc.cache.local.path) + os.makedirs(tmp_dir.dvc.cache.local.path) dvc_info = get_dvc_info() @@ -113,7 +113,7 @@ def test_remotes(tmp_dir, dvc, caplog): def test_fs_info_in_repo(tmp_dir, dvc, caplog): - os.mkdir(dvc.cache.local.path) + os.makedirs(dvc.cache.local.path) dvc_info = get_dvc_info() assert re.search(r"Cache directory: .* on .*", dvc_info)