mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-01 23:08:43 +01:00
cache: move 3.x data objects into files/md5/ prefix
This commit is contained in:
committed by
Ruslan Kuprieiev
parent
528eab0418
commit
603f5fdba1
+38
-3
@@ -1,10 +1,21 @@
|
|||||||
import os
|
import os
|
||||||
|
from typing import Optional, Tuple
|
||||||
|
|
||||||
from dvc.fs import GitFileSystem, Schemes
|
from dvc.fs import GitFileSystem, Schemes
|
||||||
from dvc_data.hashfile.db import get_odb
|
from dvc_data.hashfile.db import get_odb
|
||||||
|
from dvc_data.hashfile.hash import DEFAULT_ALGORITHM
|
||||||
|
|
||||||
|
LEGACY_HASH_NAMES = {"md5-dos2unix", "params"}
|
||||||
|
|
||||||
|
|
||||||
def _get_odb(repo, settings, fs=None):
|
def _get_odb(
|
||||||
|
repo,
|
||||||
|
settings,
|
||||||
|
fs=None,
|
||||||
|
prefix: Optional[Tuple[str, ...]] = None,
|
||||||
|
hash_name: Optional[str] = None,
|
||||||
|
**kwargs,
|
||||||
|
):
|
||||||
from dvc.fs import get_cloud_fs
|
from dvc.fs import get_cloud_fs
|
||||||
|
|
||||||
if not settings:
|
if not settings:
|
||||||
@@ -12,6 +23,10 @@ def _get_odb(repo, settings, fs=None):
|
|||||||
|
|
||||||
cls, config, fs_path = get_cloud_fs(repo, **settings)
|
cls, config, fs_path = get_cloud_fs(repo, **settings)
|
||||||
fs = fs or cls(**config)
|
fs = fs or cls(**config)
|
||||||
|
if prefix:
|
||||||
|
fs_path = fs.path.join(fs_path, *prefix)
|
||||||
|
if hash_name:
|
||||||
|
config["hash_name"] = hash_name
|
||||||
return get_odb(fs, fs_path, state=repo.state, **config)
|
return get_odb(fs, fs_path, state=repo.state, **config)
|
||||||
|
|
||||||
|
|
||||||
@@ -24,6 +39,7 @@ class CacheManager:
|
|||||||
Schemes.HDFS,
|
Schemes.HDFS,
|
||||||
Schemes.WEBHDFS,
|
Schemes.WEBHDFS,
|
||||||
]
|
]
|
||||||
|
FILES_DIR = "files"
|
||||||
|
|
||||||
def __init__(self, repo):
|
def __init__(self, repo):
|
||||||
self._repo = repo
|
self._repo = repo
|
||||||
@@ -53,15 +69,26 @@ class CacheManager:
|
|||||||
if not isinstance(repo.fs, GitFileSystem):
|
if not isinstance(repo.fs, GitFileSystem):
|
||||||
kwargs["fs"] = repo.fs
|
kwargs["fs"] = repo.fs
|
||||||
|
|
||||||
odb = _get_odb(repo, settings, **kwargs)
|
odb = _get_odb(
|
||||||
|
repo,
|
||||||
|
settings,
|
||||||
|
prefix=(self.FILES_DIR, DEFAULT_ALGORITHM),
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
self._odb["repo"] = odb
|
self._odb["repo"] = odb
|
||||||
self._odb[Schemes.LOCAL] = odb
|
self._odb[Schemes.LOCAL] = odb
|
||||||
|
legacy_odb = _get_odb(repo, settings, hash_name="md5-dos2unix", **kwargs)
|
||||||
|
self._odb["legacy"] = legacy_odb
|
||||||
|
|
||||||
def _init_odb(self, schemes):
|
def _init_odb(self, schemes):
|
||||||
for scheme in schemes:
|
for scheme in schemes:
|
||||||
remote = self.config.get(scheme)
|
remote = self.config.get(scheme)
|
||||||
settings = {"name": remote} if remote else None
|
settings = {"name": remote} if remote else None
|
||||||
self._odb[scheme] = _get_odb(self._repo, settings)
|
self._odb[scheme] = _get_odb(
|
||||||
|
self._repo,
|
||||||
|
settings,
|
||||||
|
prefix=(self.FILES_DIR, DEFAULT_ALGORITHM),
|
||||||
|
)
|
||||||
|
|
||||||
def __getattr__(self, name):
|
def __getattr__(self, name):
|
||||||
if name not in self._odb and name in self.CLOUD_SCHEMES:
|
if name not in self._odb and name in self.CLOUD_SCHEMES:
|
||||||
@@ -75,3 +102,11 @@ class CacheManager:
|
|||||||
def by_scheme(self):
|
def by_scheme(self):
|
||||||
self._init_odb(self.CLOUD_SCHEMES)
|
self._init_odb(self.CLOUD_SCHEMES)
|
||||||
yield from self._odb.items()
|
yield from self._odb.items()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def local_cache_dir(self) -> str:
|
||||||
|
"""Return base local cache directory without any prefixes.
|
||||||
|
|
||||||
|
(i.e. `dvc cache dir`).
|
||||||
|
"""
|
||||||
|
return self.legacy.path
|
||||||
|
|||||||
+19
-2
@@ -29,13 +29,25 @@ class Remote:
|
|||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def odb(self) -> "HashFileDB":
|
def odb(self) -> "HashFileDB":
|
||||||
|
from dvc.cachemgr import CacheManager
|
||||||
from dvc_data.hashfile.db import get_odb
|
from dvc_data.hashfile.db import get_odb
|
||||||
|
from dvc_data.hashfile.hash import DEFAULT_ALGORITHM
|
||||||
|
|
||||||
path = self.path
|
path = self.path
|
||||||
if self.worktree:
|
if self.worktree:
|
||||||
path = self.fs.path.join(path, ".dvc", "cache")
|
path = self.fs.path.join(
|
||||||
|
path, ".dvc", CacheManager.FILES_DIR, DEFAULT_ALGORITHM
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
path = self.fs.path.join(path, CacheManager.FILES_DIR, DEFAULT_ALGORITHM)
|
||||||
|
return get_odb(self.fs, path, hash_name=DEFAULT_ALGORITHM, **self.config)
|
||||||
|
|
||||||
return get_odb(self.fs, path, hash_name="md5", **self.config)
|
@cached_property
|
||||||
|
def legacy_odb(self) -> "HashFileDB":
|
||||||
|
from dvc_data.hashfile.db import get_odb
|
||||||
|
|
||||||
|
path = self.path
|
||||||
|
return get_odb(self.fs, path, hash_name="md5-dos2unix", **self.config)
|
||||||
|
|
||||||
|
|
||||||
class DataCloud:
|
class DataCloud:
|
||||||
@@ -101,12 +113,17 @@ class DataCloud:
|
|||||||
self,
|
self,
|
||||||
name: Optional[str] = None,
|
name: Optional[str] = None,
|
||||||
command: str = "<command>",
|
command: str = "<command>",
|
||||||
|
hash_name: str = "md5",
|
||||||
) -> "HashFileDB":
|
) -> "HashFileDB":
|
||||||
|
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||||
|
|
||||||
remote = self.get_remote(name=name, command=command)
|
remote = self.get_remote(name=name, command=command)
|
||||||
if remote.fs.version_aware or remote.worktree:
|
if remote.fs.version_aware or remote.worktree:
|
||||||
raise NoRemoteError(
|
raise NoRemoteError(
|
||||||
f"'{command}' is unsupported for cloud versioned remotes"
|
f"'{command}' is unsupported for cloud versioned remotes"
|
||||||
)
|
)
|
||||||
|
if hash_name in LEGACY_HASH_NAMES:
|
||||||
|
return remote.legacy_odb
|
||||||
return remote.odb
|
return remote.odb
|
||||||
|
|
||||||
def _log_missing(self, status: "CompareStatusResult"):
|
def _log_missing(self, status: "CompareStatusResult"):
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ class RepoDependency(Dependency):
|
|||||||
from dvc.fs import DVCFileSystem
|
from dvc.fs import DVCFileSystem
|
||||||
|
|
||||||
config = {"cache": self.repo.config["cache"]}
|
config = {"cache": self.repo.config["cache"]}
|
||||||
config["cache"]["dir"] = self.repo.cache.local.path
|
config["cache"]["dir"] = self.repo.cache.local_cache_dir
|
||||||
|
|
||||||
return DVCFileSystem(
|
return DVCFileSystem(
|
||||||
url=self.def_repo[self.PARAM_URL],
|
url=self.def_repo[self.PARAM_URL],
|
||||||
|
|||||||
+1
-1
@@ -67,7 +67,7 @@ def _get_caches(cache):
|
|||||||
caches = (
|
caches = (
|
||||||
cache_type
|
cache_type
|
||||||
for cache_type, cache_instance in cache.by_scheme()
|
for cache_type, cache_instance in cache.by_scheme()
|
||||||
if cache_instance and cache_type != "repo"
|
if cache_instance and cache_type not in ("repo", "legacy")
|
||||||
)
|
)
|
||||||
|
|
||||||
# Caches will be always non-empty including the local cache
|
# Caches will be always non-empty including the local cache
|
||||||
|
|||||||
+11
-3
@@ -500,6 +500,14 @@ class Output:
|
|||||||
raise RemoteCacheRequiredError(self.fs.protocol, self.fs_path)
|
raise RemoteCacheRequiredError(self.fs.protocol, self.fs_path)
|
||||||
return odb
|
return odb
|
||||||
|
|
||||||
|
@property
|
||||||
|
def local_cache(self):
|
||||||
|
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||||
|
|
||||||
|
if self.hash_name in LEGACY_HASH_NAMES:
|
||||||
|
return self.repo.cache.legacy
|
||||||
|
return self.repo.cache.local
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def cache_path(self):
|
def cache_path(self):
|
||||||
return self.cache.fs.unstrip_protocol(
|
return self.cache.fs.unstrip_protocol(
|
||||||
@@ -514,7 +522,7 @@ class Output:
|
|||||||
if self.use_cache:
|
if self.use_cache:
|
||||||
odb = self.cache
|
odb = self.cache
|
||||||
else:
|
else:
|
||||||
odb = self.repo.cache.local
|
odb = self.local_cache
|
||||||
_, meta, obj = build(
|
_, meta, obj = build(
|
||||||
odb,
|
odb,
|
||||||
self.fs_path,
|
self.fs_path,
|
||||||
@@ -665,7 +673,7 @@ class Output:
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
_, self.meta, self.obj = build(
|
_, self.meta, self.obj = build(
|
||||||
self.repo.cache.local,
|
self.local_cache,
|
||||||
self.fs_path,
|
self.fs_path,
|
||||||
self.fs,
|
self.fs,
|
||||||
self.hash_name,
|
self.hash_name,
|
||||||
@@ -1283,7 +1291,7 @@ class Output:
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert self.repo
|
assert self.repo
|
||||||
cache = self.cache if self.use_cache else self.repo.cache.local
|
cache = self.cache if self.use_cache else self.local_cache
|
||||||
assert isinstance(cache, HashFileDB)
|
assert isinstance(cache, HashFileDB)
|
||||||
|
|
||||||
new: "HashFile"
|
new: "HashFile"
|
||||||
|
|||||||
@@ -420,8 +420,8 @@ class Repo:
|
|||||||
flist = [self.config.files["local"]]
|
flist = [self.config.files["local"]]
|
||||||
if tmp_dir := self.tmp_dir:
|
if tmp_dir := self.tmp_dir:
|
||||||
flist.append(tmp_dir)
|
flist.append(tmp_dir)
|
||||||
if path_isin(self.cache.repo.path, self.root_dir):
|
if path_isin(self.cache.legacy.path, self.root_dir):
|
||||||
flist.append(self.cache.repo.path)
|
flist.append(self.cache.legacy.path)
|
||||||
|
|
||||||
for file in flist:
|
for file in flist:
|
||||||
self.scm_context.ignore(file)
|
self.scm_context.ignore(file)
|
||||||
|
|||||||
@@ -152,7 +152,7 @@ class TempDirExecutor(BaseLocalExecutor):
|
|||||||
self, repo: "Repo", rev: str, run_cache: bool = True # noqa: ARG002
|
self, repo: "Repo", rev: str, run_cache: bool = True # noqa: ARG002
|
||||||
):
|
):
|
||||||
"""Initialize DVC cache."""
|
"""Initialize DVC cache."""
|
||||||
self._update_config({"cache": {"dir": repo.cache.repo.path}})
|
self._update_config({"cache": {"dir": repo.cache.local_cache_dir}})
|
||||||
|
|
||||||
def cleanup(self, infofile: Optional[str] = None):
|
def cleanup(self, infofile: Optional[str] = None):
|
||||||
super().cleanup(infofile)
|
super().cleanup(infofile)
|
||||||
|
|||||||
@@ -129,7 +129,7 @@ def _get_remote_config(url):
|
|||||||
name = "auto-generated-upstream"
|
name = "auto-generated-upstream"
|
||||||
return {
|
return {
|
||||||
"core": {"remote": name},
|
"core": {"remote": name},
|
||||||
"remote": {name: {"url": repo.cache.local.path}},
|
"remote": {name: {"url": repo.cache.local_cache_dir}},
|
||||||
}
|
}
|
||||||
|
|
||||||
# Use original remote to make sure that we are using correct url,
|
# Use original remote to make sure that we are using correct url,
|
||||||
|
|||||||
+15
-11
@@ -60,7 +60,7 @@ def _get_stage_hash(stage):
|
|||||||
class StageCache:
|
class StageCache:
|
||||||
def __init__(self, repo):
|
def __init__(self, repo):
|
||||||
self.repo = repo
|
self.repo = repo
|
||||||
self.cache_dir = os.path.join(self.repo.cache.local.path, "runs")
|
self.cache_dir = os.path.join(self.repo.cache.legacy.path, "runs")
|
||||||
|
|
||||||
def _get_cache_dir(self, key):
|
def _get_cache_dir(self, key):
|
||||||
return os.path.join(self.cache_dir, key[:2], key)
|
return os.path.join(self.cache_dir, key[:2], key)
|
||||||
@@ -119,12 +119,12 @@ class StageCache:
|
|||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def _cache_type_copy(self):
|
def _cache_type_copy(self):
|
||||||
cache_types = self.repo.cache.local.cache_types
|
cache_types = self.repo.cache.legacy.cache_types
|
||||||
self.repo.cache.local.cache_types = ["copy"]
|
self.repo.cache.legacy.cache_types = ["copy"]
|
||||||
try:
|
try:
|
||||||
yield
|
yield
|
||||||
finally:
|
finally:
|
||||||
self.repo.cache.local.cache_types = cache_types
|
self.repo.cache.legacy.cache_types = cache_types
|
||||||
|
|
||||||
def _uncached_outs(self, stage, cache):
|
def _uncached_outs(self, stage, cache):
|
||||||
# NOTE: using temporary stage to avoid accidentally modifying original
|
# NOTE: using temporary stage to avoid accidentally modifying original
|
||||||
@@ -166,14 +166,14 @@ class StageCache:
|
|||||||
COMPILED_LOCK_FILE_STAGE_SCHEMA(cache)
|
COMPILED_LOCK_FILE_STAGE_SCHEMA(cache)
|
||||||
|
|
||||||
path = self._get_cache_path(cache_key, cache_value)
|
path = self._get_cache_path(cache_key, cache_value)
|
||||||
local_fs = self.repo.cache.local.fs
|
local_fs = self.repo.cache.legacy.fs
|
||||||
parent = local_fs.path.parent(path)
|
parent = local_fs.path.parent(path)
|
||||||
self.repo.cache.local.makedirs(parent)
|
self.repo.cache.legacy.makedirs(parent)
|
||||||
tmp = local_fs.path.join(parent, fs.utils.tmp_fname())
|
tmp = local_fs.path.join(parent, fs.utils.tmp_fname())
|
||||||
assert os.path.exists(parent)
|
assert os.path.exists(parent)
|
||||||
assert os.path.isdir(parent)
|
assert os.path.isdir(parent)
|
||||||
dump_yaml(tmp, cache)
|
dump_yaml(tmp, cache)
|
||||||
self.repo.cache.local.move(tmp, path)
|
self.repo.cache.legacy.move(tmp, path)
|
||||||
|
|
||||||
def restore(self, stage, run_cache=True, pull=False, dry=False):
|
def restore(self, stage, run_cache=True, pull=False, dry=False):
|
||||||
from .serialize import to_single_stage_lockfile
|
from .serialize import to_single_stage_lockfile
|
||||||
@@ -258,12 +258,16 @@ class StageCache:
|
|||||||
return ret
|
return ret
|
||||||
|
|
||||||
def push(self, remote: Optional[str], odb: Optional["ObjectDB"] = None):
|
def push(self, remote: Optional[str], odb: Optional["ObjectDB"] = None):
|
||||||
dest_odb = odb or self.repo.cloud.get_remote_odb(remote, "push --run-cache")
|
dest_odb = odb or self.repo.cloud.get_remote_odb(
|
||||||
return self.transfer(self.repo.cache.local, dest_odb)
|
remote, "push --run-cache", hash_name="md5-dos2unix"
|
||||||
|
)
|
||||||
|
return self.transfer(self.repo.cache.legacy, dest_odb)
|
||||||
|
|
||||||
def pull(self, remote: Optional[str], odb: Optional["ObjectDB"] = None):
|
def pull(self, remote: Optional[str], odb: Optional["ObjectDB"] = None):
|
||||||
odb = odb or self.repo.cloud.get_remote_odb(remote, "fetch --run-cache")
|
odb = odb or self.repo.cloud.get_remote_odb(
|
||||||
return self.transfer(odb, self.repo.cache.local)
|
remote, "fetch --run-cache", hash_name="md5-dos2unix"
|
||||||
|
)
|
||||||
|
return self.transfer(odb, self.repo.cache.legacy)
|
||||||
|
|
||||||
def get_used_objs(self, used_run_cache, *args, **kwargs):
|
def get_used_objs(self, used_run_cache, *args, **kwargs):
|
||||||
"""Return used cache for the specified run-cached stages."""
|
"""Return used cache for the specified run-cached stages."""
|
||||||
|
|||||||
@@ -11,7 +11,9 @@ class TestAPI:
|
|||||||
def test_get_url(self, tmp_dir, dvc, remote):
|
def test_get_url(self, tmp_dir, dvc, remote):
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
|
|
||||||
expected_url = (remote / "ac/bd18db4cc2f85cedef654fccc4a4d8").url
|
expected_url = (
|
||||||
|
remote / "files" / "md5" / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||||
|
).url
|
||||||
assert api.get_url("foo") == expected_url
|
assert api.get_url("foo") == expected_url
|
||||||
|
|
||||||
def test_open(self, tmp_dir, dvc, remote):
|
def test_open(self, tmp_dir, dvc, remote):
|
||||||
|
|||||||
@@ -181,7 +181,7 @@ class TestAdd:
|
|||||||
)
|
)
|
||||||
assert (workspace / "file").read_text() == "file"
|
assert (workspace / "file").read_text() == "file"
|
||||||
assert (
|
assert (
|
||||||
workspace / "cache" / hash_value[:2] / hash_value[2:]
|
workspace / "cache" / "files" / "md5" / hash_value[:2] / hash_value[2:]
|
||||||
).read_text() == "file"
|
).read_text() == "file"
|
||||||
|
|
||||||
assert dvc.status() == {}
|
assert dvc.status() == {}
|
||||||
@@ -191,7 +191,14 @@ class TestAdd:
|
|||||||
workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})
|
workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})
|
||||||
|
|
||||||
dvc.add("remote://workspace/dir")
|
dvc.add("remote://workspace/dir")
|
||||||
assert (workspace / "cache" / dir_hash_value[:2] / dir_hash_value[2:]).is_file()
|
assert (
|
||||||
|
workspace
|
||||||
|
/ "cache"
|
||||||
|
/ "files"
|
||||||
|
/ "md5"
|
||||||
|
/ dir_hash_value[:2]
|
||||||
|
/ dir_hash_value[2:]
|
||||||
|
).is_file()
|
||||||
|
|
||||||
|
|
||||||
def match_files(fs, entries, expected):
|
def match_files(fs, entries, expected):
|
||||||
@@ -301,7 +308,9 @@ class TestToRemote:
|
|||||||
meta = stage.outs[0].meta
|
meta = stage.outs[0].meta
|
||||||
assert hash_info.name == "md5"
|
assert hash_info.name == "md5"
|
||||||
assert hash_info.value == "acbd18db4cc2f85cedef654fccc4a4d8"
|
assert hash_info.value == "acbd18db4cc2f85cedef654fccc4a4d8"
|
||||||
assert (remote / "ac" / "bd18db4cc2f85cedef654fccc4a4d8").read_text() == "foo"
|
assert (
|
||||||
|
remote / "files" / "md5" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8"
|
||||||
|
).read_text() == "foo"
|
||||||
assert meta.size == len("foo")
|
assert meta.size == len("foo")
|
||||||
|
|
||||||
def test_import_url_to_remote_file(self, tmp_dir, dvc, workspace, remote):
|
def test_import_url_to_remote_file(self, tmp_dir, dvc, workspace, remote):
|
||||||
@@ -321,7 +330,9 @@ class TestToRemote:
|
|||||||
hash_info = stage.outs[0].hash_info
|
hash_info = stage.outs[0].hash_info
|
||||||
assert hash_info.name == "md5"
|
assert hash_info.name == "md5"
|
||||||
assert hash_info.value == "acbd18db4cc2f85cedef654fccc4a4d8"
|
assert hash_info.value == "acbd18db4cc2f85cedef654fccc4a4d8"
|
||||||
assert (remote / "ac" / "bd18db4cc2f85cedef654fccc4a4d8").read_text() == "foo"
|
assert (
|
||||||
|
remote / "files" / "md5" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8"
|
||||||
|
).read_text() == "foo"
|
||||||
assert stage.outs[0].meta.size == len("foo")
|
assert stage.outs[0].meta.size == len("foo")
|
||||||
|
|
||||||
def test_import_url_to_remote_dir(self, tmp_dir, dvc, workspace, remote):
|
def test_import_url_to_remote_dir(self, tmp_dir, dvc, workspace, remote):
|
||||||
@@ -351,7 +362,9 @@ class TestToRemote:
|
|||||||
assert hash_info.name == "md5"
|
assert hash_info.name == "md5"
|
||||||
assert hash_info.value == "55d05978954d1b2cd7b06aedda9b9e43.dir"
|
assert hash_info.value == "55d05978954d1b2cd7b06aedda9b9e43.dir"
|
||||||
file_parts = json.loads(
|
file_parts = json.loads(
|
||||||
(remote / "55" / "d05978954d1b2cd7b06aedda9b9e43.dir").read_text()
|
(
|
||||||
|
remote / "files" / "md5" / "55" / "d05978954d1b2cd7b06aedda9b9e43.dir"
|
||||||
|
).read_text()
|
||||||
)
|
)
|
||||||
|
|
||||||
assert len(file_parts) == 3
|
assert len(file_parts) == 3
|
||||||
@@ -363,4 +376,6 @@ class TestToRemote:
|
|||||||
|
|
||||||
for file_part in file_parts:
|
for file_part in file_parts:
|
||||||
md5 = file_part["md5"]
|
md5 = file_part["md5"]
|
||||||
assert (remote / md5[:2] / md5[2:]).read_text() == file_part["relpath"]
|
assert (
|
||||||
|
remote / "files" / "md5" / md5[:2] / md5[2:]
|
||||||
|
).read_text() == file_part["relpath"]
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ def test_get_url_external(tmp_dir, erepo_dir, cloud):
|
|||||||
|
|
||||||
# Using file url to force clone to tmp repo
|
# Using file url to force clone to tmp repo
|
||||||
repo_url = f"file://{erepo_dir.as_posix()}"
|
repo_url = f"file://{erepo_dir.as_posix()}"
|
||||||
expected_url = (cloud / "ac/bd18db4cc2f85cedef654fccc4a4d8").url
|
expected_url = (cloud / "files" / "md5" / "ac/bd18db4cc2f85cedef654fccc4a4d8").url
|
||||||
assert api.get_url("foo", repo=repo_url) == expected_url
|
assert api.get_url("foo", repo=repo_url) == expected_url
|
||||||
|
|
||||||
|
|
||||||
@@ -166,16 +166,24 @@ def test_get_url_granular(tmp_dir, dvc, cloud):
|
|||||||
tmp_dir.add_remote(config=cloud.config)
|
tmp_dir.add_remote(config=cloud.config)
|
||||||
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar", "nested": {"file": "file"}}})
|
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar", "nested": {"file": "file"}}})
|
||||||
|
|
||||||
expected_url = (cloud / "5f" / "c28ea78987408341668eba6525ebd1.dir").url
|
expected_url = (
|
||||||
|
cloud / "files" / "md5" / "5f" / "c28ea78987408341668eba6525ebd1.dir"
|
||||||
|
).url
|
||||||
assert api.get_url("dir") == expected_url
|
assert api.get_url("dir") == expected_url
|
||||||
|
|
||||||
expected_url = (cloud / "ac" / "bd18db4cc2f85cedef654fccc4a4d8").url
|
expected_url = (
|
||||||
|
cloud / "files" / "md5" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8"
|
||||||
|
).url
|
||||||
assert api.get_url("dir/foo") == expected_url
|
assert api.get_url("dir/foo") == expected_url
|
||||||
|
|
||||||
expected_url = (cloud / "37" / "b51d194a7513e45b56f6524f2d51f2").url
|
expected_url = (
|
||||||
|
cloud / "files" / "md5" / "37" / "b51d194a7513e45b56f6524f2d51f2"
|
||||||
|
).url
|
||||||
assert api.get_url("dir/bar") == expected_url
|
assert api.get_url("dir/bar") == expected_url
|
||||||
|
|
||||||
expected_url = (cloud / "8c" / "7dd922ad47494fc02c388e12c00eac").url
|
expected_url = (
|
||||||
|
cloud / "files" / "md5" / "8c" / "7dd922ad47494fc02c388e12c00eac"
|
||||||
|
).url
|
||||||
assert api.get_url(os.path.join("dir", "nested", "file")) == expected_url
|
assert api.get_url(os.path.join("dir", "nested", "file")) == expected_url
|
||||||
|
|
||||||
|
|
||||||
@@ -186,10 +194,14 @@ def test_get_url_subrepos(tmp_dir, scm, local_cloud):
|
|||||||
subrepo.dvc_gen({"dir": {"foo": "foo"}, "bar": "bar"}, commit="add files")
|
subrepo.dvc_gen({"dir": {"foo": "foo"}, "bar": "bar"}, commit="add files")
|
||||||
subrepo.dvc.push()
|
subrepo.dvc.push()
|
||||||
|
|
||||||
expected_url = os.fspath(local_cloud / "ac" / "bd18db4cc2f85cedef654fccc4a4d8")
|
expected_url = os.fspath(
|
||||||
|
local_cloud / "files" / "md5" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8"
|
||||||
|
)
|
||||||
assert api.get_url(os.path.join("subrepo", "dir", "foo")) == expected_url
|
assert api.get_url(os.path.join("subrepo", "dir", "foo")) == expected_url
|
||||||
|
|
||||||
expected_url = os.fspath(local_cloud / "37" / "b51d194a7513e45b56f6524f2d51f2")
|
expected_url = os.fspath(
|
||||||
|
local_cloud / "files" / "md5" / "37" / "b51d194a7513e45b56f6524f2d51f2"
|
||||||
|
)
|
||||||
assert api.get_url("subrepo/bar") == expected_url
|
assert api.get_url("subrepo/bar") == expected_url
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+21
-3
@@ -729,10 +729,22 @@ def test_add_symlink_file(tmp_dir, dvc):
|
|||||||
assert (tmp_dir / "dir" / "bar").read_text() == "bar"
|
assert (tmp_dir / "dir" / "bar").read_text() == "bar"
|
||||||
|
|
||||||
assert (
|
assert (
|
||||||
tmp_dir / ".dvc" / "cache" / "37" / "b51d194a7513e45b56f6524f2d51f2"
|
tmp_dir
|
||||||
|
/ ".dvc"
|
||||||
|
/ "cache"
|
||||||
|
/ "files"
|
||||||
|
/ "md5"
|
||||||
|
/ "37"
|
||||||
|
/ "b51d194a7513e45b56f6524f2d51f2"
|
||||||
).read_text() == "bar"
|
).read_text() == "bar"
|
||||||
assert not (
|
assert not (
|
||||||
tmp_dir / ".dvc" / "cache" / "37" / "b51d194a7513e45b56f6524f2d51f2"
|
tmp_dir
|
||||||
|
/ ".dvc"
|
||||||
|
/ "cache"
|
||||||
|
/ "files"
|
||||||
|
/ "md5"
|
||||||
|
/ "37"
|
||||||
|
/ "b51d194a7513e45b56f6524f2d51f2"
|
||||||
).is_symlink()
|
).is_symlink()
|
||||||
|
|
||||||
# Test that subsequent add succeeds
|
# Test that subsequent add succeeds
|
||||||
@@ -790,7 +802,13 @@ def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys):
|
|||||||
assert (tmp_dir / "foo").exists()
|
assert (tmp_dir / "foo").exists()
|
||||||
assert (tmp_dir / "foo.dvc").exists()
|
assert (tmp_dir / "foo.dvc").exists()
|
||||||
assert (
|
assert (
|
||||||
tmp_dir / ".dvc" / "cache" / "ac" / "bd18db4cc2f85cedef654fccc4a4d8"
|
tmp_dir
|
||||||
|
/ ".dvc"
|
||||||
|
/ "cache"
|
||||||
|
/ "files"
|
||||||
|
/ "md5"
|
||||||
|
/ "ac"
|
||||||
|
/ "bd18db4cc2f85cedef654fccc4a4d8"
|
||||||
).read_text() == "foo"
|
).read_text() == "foo"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -134,7 +134,7 @@ def test_commit_granular_output(tmp_dir, dvc):
|
|||||||
no_commit=True,
|
no_commit=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
cache = tmp_dir / ".dvc" / "cache"
|
cache = tmp_dir / ".dvc" / "cache" / "files" / "md5"
|
||||||
assert not list(cache.glob("*/*"))
|
assert not list(cache.glob("*/*"))
|
||||||
|
|
||||||
dvc.commit("foo")
|
dvc.commit("foo")
|
||||||
@@ -175,7 +175,7 @@ def test_commit_granular_dir(tmp_dir, dvc):
|
|||||||
)
|
)
|
||||||
dvc.add("data", no_commit=True)
|
dvc.add("data", no_commit=True)
|
||||||
|
|
||||||
cache = tmp_dir / ".dvc" / "cache"
|
cache = tmp_dir / ".dvc" / "cache" / "files" / "md5"
|
||||||
|
|
||||||
assert set(cache.glob("*/*")) == set()
|
assert set(cache.glob("*/*")) == set()
|
||||||
|
|
||||||
|
|||||||
@@ -470,11 +470,11 @@ def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
|
|||||||
|
|
||||||
dvc.pull("copy-foo-bar")
|
dvc.pull("copy-foo-bar")
|
||||||
assert (tmp_dir / "bar").exists()
|
assert (tmp_dir / "bar").exists()
|
||||||
assert len(recurse_list_dir(dvc.cache.local.path)) == 2
|
assert len(recurse_list_dir(dvc.cache.local.path)) == 1
|
||||||
clean(["bar"], dvc)
|
clean(["bar"], dvc)
|
||||||
|
|
||||||
dvc.fetch("copy-foo-bar")
|
dvc.fetch("copy-foo-bar")
|
||||||
assert len(recurse_list_dir(dvc.cache.local.path)) == 2
|
assert len(recurse_list_dir(dvc.cache.local.path)) == 1
|
||||||
|
|
||||||
|
|
||||||
def test_pull_partial(tmp_dir, dvc, local_remote):
|
def test_pull_partial(tmp_dir, dvc, local_remote):
|
||||||
|
|||||||
+19
-7
@@ -260,15 +260,23 @@ def test_gc_external_output(tmp_dir, dvc, workspace):
|
|||||||
foo_hash = foo_stage.outs[0].hash_info.value
|
foo_hash = foo_stage.outs[0].hash_info.value
|
||||||
bar_hash = bar_stage.outs[0].hash_info.value
|
bar_hash = bar_stage.outs[0].hash_info.value
|
||||||
|
|
||||||
assert (workspace / "cache" / foo_hash[:2] / foo_hash[2:]).read_text() == "foo"
|
assert (
|
||||||
assert (workspace / "cache" / bar_hash[:2] / bar_hash[2:]).read_text() == "bar"
|
workspace / "cache" / "files" / "md5" / foo_hash[:2] / foo_hash[2:]
|
||||||
|
).read_text() == "foo"
|
||||||
|
assert (
|
||||||
|
workspace / "cache" / "files" / "md5" / bar_hash[:2] / bar_hash[2:]
|
||||||
|
).read_text() == "bar"
|
||||||
|
|
||||||
(tmp_dir / "foo.dvc").unlink()
|
(tmp_dir / "foo.dvc").unlink()
|
||||||
|
|
||||||
dvc.gc(workspace=True)
|
dvc.gc(workspace=True)
|
||||||
|
|
||||||
assert not (workspace / "cache" / foo_hash[:2] / foo_hash[2:]).exists()
|
assert not (
|
||||||
assert (workspace / "cache" / bar_hash[:2] / bar_hash[2:]).read_text() == "bar"
|
workspace / "cache" / "files" / "md5" / foo_hash[:2] / foo_hash[2:]
|
||||||
|
).exists()
|
||||||
|
assert (
|
||||||
|
workspace / "cache" / "files" / "md5" / bar_hash[:2] / bar_hash[2:]
|
||||||
|
).read_text() == "bar"
|
||||||
|
|
||||||
|
|
||||||
def test_gc_all_experiments(tmp_dir, scm, dvc):
|
def test_gc_all_experiments(tmp_dir, scm, dvc):
|
||||||
@@ -288,9 +296,11 @@ def test_gc_all_experiments(tmp_dir, scm, dvc):
|
|||||||
|
|
||||||
dvc.gc(all_experiments=True, force=True)
|
dvc.gc(all_experiments=True, force=True)
|
||||||
|
|
||||||
assert not (tmp_dir / ".dvc" / "cache" / foo_hash[:2] / foo_hash[2:]).exists()
|
assert not (
|
||||||
|
tmp_dir / ".dvc" / "cache" / "files" / "md5" / foo_hash[:2] / foo_hash[2:]
|
||||||
|
).exists()
|
||||||
assert (
|
assert (
|
||||||
tmp_dir / ".dvc" / "cache" / baz_hash[:2] / baz_hash[2:]
|
tmp_dir / ".dvc" / "cache" / "files" / "md5" / baz_hash[:2] / baz_hash[2:]
|
||||||
).read_text() == "baz"
|
).read_text() == "baz"
|
||||||
|
|
||||||
|
|
||||||
@@ -306,7 +316,9 @@ def test_gc_rev_num(tmp_dir, scm, dvc):
|
|||||||
dvc.gc(rev="HEAD", num=num, force=True)
|
dvc.gc(rev="HEAD", num=num, force=True)
|
||||||
|
|
||||||
for n, i in enumerate(reversed(range(4))):
|
for n, i in enumerate(reversed(range(4))):
|
||||||
cache = tmp_dir / ".dvc" / "cache" / hashes[i][:2] / hashes[i][2:]
|
cache = (
|
||||||
|
tmp_dir / ".dvc" / "cache" / "files" / "md5" / hashes[i][:2] / hashes[i][2:]
|
||||||
|
)
|
||||||
if n >= num:
|
if n >= num:
|
||||||
assert not cache.exists()
|
assert not cache.exists()
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -77,7 +77,9 @@ class TestInstall:
|
|||||||
tmp_dir.dvc_gen("file", "file_content", "commit message")
|
tmp_dir.dvc_gen("file", "file_content", "commit message")
|
||||||
|
|
||||||
file_checksum = file_md5("file", dvc.fs)
|
file_checksum = file_md5("file", dvc.fs)
|
||||||
expected_storage_path = storage_path / file_checksum[:2] / file_checksum[2:]
|
expected_storage_path = (
|
||||||
|
storage_path / "files" / "md5" / file_checksum[:2] / file_checksum[2:]
|
||||||
|
)
|
||||||
|
|
||||||
scm.gitpython.repo.clone(os.fspath(git_remote))
|
scm.gitpython.repo.clone(os.fspath(git_remote))
|
||||||
scm.gitpython.repo.create_remote("origin", os.fspath(git_remote))
|
scm.gitpython.repo.create_remote("origin", os.fspath(git_remote))
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ def test_remote_cache_references(tmp_dir, dvc):
|
|||||||
|
|
||||||
dvc.cache = CacheManager(dvc)
|
dvc.cache = CacheManager(dvc)
|
||||||
|
|
||||||
assert dvc.cache.ssh.path == "/tmp"
|
assert dvc.cache.ssh.path == "/tmp/files/md5"
|
||||||
|
|
||||||
|
|
||||||
def test_shared_cache_dir(tmp_dir):
|
def test_shared_cache_dir(tmp_dir):
|
||||||
@@ -113,13 +113,13 @@ def test_shared_cache_dir(tmp_dir):
|
|||||||
assert not os.path.exists(os.path.join("dir1", ".dvc", "cache"))
|
assert not os.path.exists(os.path.join("dir1", ".dvc", "cache"))
|
||||||
assert not os.path.exists(os.path.join("dir2", ".dvc", "cache"))
|
assert not os.path.exists(os.path.join("dir2", ".dvc", "cache"))
|
||||||
assert os.path.exists(
|
assert os.path.exists(
|
||||||
os.path.join(cache_dir, "dc", "f6c2fa538b445a3a095255c3641dfc")
|
os.path.join(cache_dir, "files", "md5", "dc", "f6c2fa538b445a3a095255c3641dfc")
|
||||||
)
|
)
|
||||||
assert os.path.exists(
|
assert os.path.exists(
|
||||||
os.path.join(cache_dir, "b4", "333c8cfa2ebba7ef20ec6c3265902b")
|
os.path.join(cache_dir, "files", "md5", "b4", "333c8cfa2ebba7ef20ec6c3265902b")
|
||||||
)
|
)
|
||||||
assert os.path.exists(
|
assert os.path.exists(
|
||||||
os.path.join(cache_dir, "9e", "fab2399c7c560b34de477b9aa0a465")
|
os.path.join(cache_dir, "files", "md5", "9e", "fab2399c7c560b34de477b9aa0a465")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -165,7 +165,7 @@ def test_cmd_cache_relative_path(tmp_dir, scm, dvc, make_tmp_dir):
|
|||||||
tmp_dir.dvc_gen({"foo": "foo"})
|
tmp_dir.dvc_gen({"foo": "foo"})
|
||||||
|
|
||||||
assert os.path.exists(
|
assert os.path.exists(
|
||||||
os.path.join(cache_dir, "ac", "bd18db4cc2f85cedef654fccc4a4d8")
|
os.path.join(cache_dir, "files", "md5", "ac", "bd18db4cc2f85cedef654fccc4a4d8")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -573,7 +573,7 @@ def test_run_external_outputs(
|
|||||||
assert (local_workspace / "foo").read_text() == "foo"
|
assert (local_workspace / "foo").read_text() == "foo"
|
||||||
assert (local_workspace / "bar").read_text() == "bar"
|
assert (local_workspace / "bar").read_text() == "bar"
|
||||||
assert (
|
assert (
|
||||||
local_workspace / "cache" / bar_hash[:2] / bar_hash[2:]
|
local_workspace / "cache" / "files" / "md5" / bar_hash[:2] / bar_hash[2:]
|
||||||
).read_text() == "bar"
|
).read_text() == "bar"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,9 @@ def test_unprotect(tmp_dir, dvc):
|
|||||||
|
|
||||||
dvc.cache.local.cache_types = ["hardlink"]
|
dvc.cache.local.cache_types = ["hardlink"]
|
||||||
dvc.add("foo")
|
dvc.add("foo")
|
||||||
cache = os.path.join(".dvc", "cache", "ac", "bd18db4cc2f85cedef654fccc4a4d8")
|
cache = os.path.join(
|
||||||
|
".dvc", "cache", "files", "md5", "ac", "bd18db4cc2f85cedef654fccc4a4d8"
|
||||||
|
)
|
||||||
assert not os.access("foo", os.W_OK)
|
assert not os.access("foo", os.W_OK)
|
||||||
assert not os.access(cache, os.W_OK)
|
assert not os.access(cache, os.W_OK)
|
||||||
|
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ def test_subrepo_is_constructed_properly(
|
|||||||
|
|
||||||
subrepo = tmp_dir / "subrepo"
|
subrepo = tmp_dir / "subrepo"
|
||||||
make_subrepo(subrepo, scm)
|
make_subrepo(subrepo, scm)
|
||||||
local_cache = subrepo.dvc.cache.local.path
|
local_cache = subrepo.dvc.cache.local_cache_dir
|
||||||
|
|
||||||
tmp_dir.scm_gen("bar", "bar", commit="add bar")
|
tmp_dir.scm_gen("bar", "bar", commit="add bar")
|
||||||
subrepo.dvc_gen("foo", "foo", commit="add foo")
|
subrepo.dvc_gen("foo", "foo", commit="add foo")
|
||||||
@@ -75,8 +75,8 @@ def test_subrepo_is_constructed_properly(
|
|||||||
|
|
||||||
assert repo.url == str(tmp_dir)
|
assert repo.url == str(tmp_dir)
|
||||||
assert repo.config["cache"]["dir"] == str(cache_dir)
|
assert repo.config["cache"]["dir"] == str(cache_dir)
|
||||||
assert repo.cache.local.path == str(cache_dir)
|
assert repo.cache.local.path == os.path.join(cache_dir, "files", "md5")
|
||||||
assert subrepo.cache.local.path == str(cache_dir)
|
assert subrepo.cache.local.path == os.path.join(cache_dir, "files", "md5")
|
||||||
|
|
||||||
assert repo.config["cache"]["type"] == ["symlink"]
|
assert repo.config["cache"]["type"] == ["symlink"]
|
||||||
assert repo.cache.local.cache_types == ["symlink"]
|
assert repo.cache.local.cache_types == ["symlink"]
|
||||||
@@ -84,7 +84,5 @@ def test_subrepo_is_constructed_properly(
|
|||||||
|
|
||||||
assert subrepo.config["remote"]["auto-generated-upstream"]["url"] == local_cache
|
assert subrepo.config["remote"]["auto-generated-upstream"]["url"] == local_cache
|
||||||
if root_is_dvc:
|
if root_is_dvc:
|
||||||
main_cache = tmp_dir.dvc.cache.local.path
|
main_cache = tmp_dir.dvc.cache.local_cache_dir
|
||||||
assert repo.config["remote"]["auto-generated-upstream"]["url"] == str(
|
assert repo.config["remote"]["auto-generated-upstream"]["url"] == main_cache
|
||||||
main_cache
|
|
||||||
)
|
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ def find_supported_remotes(string):
|
|||||||
def test_info_in_repo(scm_init, tmp_dir):
|
def test_info_in_repo(scm_init, tmp_dir):
|
||||||
tmp_dir.init(scm=scm_init, dvc=True)
|
tmp_dir.init(scm=scm_init, dvc=True)
|
||||||
# Create `.dvc/cache`, that is needed to check supported link types.
|
# Create `.dvc/cache`, that is needed to check supported link types.
|
||||||
os.mkdir(tmp_dir.dvc.cache.local.path)
|
os.makedirs(tmp_dir.dvc.cache.local.path)
|
||||||
|
|
||||||
dvc_info = get_dvc_info()
|
dvc_info = get_dvc_info()
|
||||||
|
|
||||||
@@ -113,7 +113,7 @@ def test_remotes(tmp_dir, dvc, caplog):
|
|||||||
|
|
||||||
|
|
||||||
def test_fs_info_in_repo(tmp_dir, dvc, caplog):
|
def test_fs_info_in_repo(tmp_dir, dvc, caplog):
|
||||||
os.mkdir(dvc.cache.local.path)
|
os.makedirs(dvc.cache.local.path)
|
||||||
dvc_info = get_dvc_info()
|
dvc_info = get_dvc_info()
|
||||||
|
|
||||||
assert re.search(r"Cache directory: .* on .*", dvc_info)
|
assert re.search(r"Cache directory: .* on .*", dvc_info)
|
||||||
|
|||||||
Reference in New Issue
Block a user