mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-07 07:18:27 +01:00
Peter Rowlands (변기호)
114a07e7a4
* objects.status: use object IDs instead of naive objects * objects.transfer: use object IDs instead of naive objects * objects: add ReferenceHashFile * odb: add ReferenceObjectDB * repo: collect used object IDs instead of naive objs * support basic/naive fs serialization * objects.stage: stage file objects in addition to trees * objects: remove save(), update transfer() * output/dep: update for osave removal * path_info: support both url and path attributes * status/gc: use object IDs instead of naive objs * diff: update staging usage * commit: update staging usage * tests: clean staging when needed * tests: update for save/transfer/stage changes * dvcfs: don't allow config serialization (repofs should be used instead) * utils: return mtime as str, size as int * tests: update unit tests for save/stage/transfer changes * objects: use Repo.open when needed in ReferenceHashFile * fix ls test staging * objects.stage: don't skip hash check when loading from state - individual hashes will still be loaded from state if they are still valid * ReferenceHashFile: serialize repo fs paths relative to repo root * preserve state when staging * objects.tree: skip hash computation in digest() when we have already loaded hash from state * always check file existence in referencehashfile * update repofs test since stage no longer allows nonexistent files * odb: only protect on check() if hash was verified * staging: use unique memfs namespace for staging per dest ODB * tests: clean full staging namespace * refodb: preserve repofs cache dir * repodependency: stage imports to local ODB and strict check circular imports * fetch: fetch staging last * staging: make dest odb mandatory and preserve state * refodb: cache nonlocal fs's * fix rebase errors * repofs: support full instantiation from fs config * repofs: preserve subrepo factory for erepos * tests: clarify repofs dirty file behavior
140 lines
4.0 KiB
Python
140 lines
4.0 KiB
Python
import errno
|
|
import os
|
|
|
|
import pytest
|
|
|
|
from dvc.fs.local import LocalFileSystem
|
|
from dvc.hash_info import HashInfo
|
|
from dvc.objects.db.local import LocalObjectDB
|
|
from dvc.path_info import PathInfo
|
|
|
|
|
|
def test_status_download_optimization(mocker, dvc):
|
|
"""When comparing the status to pull a remote cache,
|
|
And the desired files to fetch are already on the local cache,
|
|
Don't check the existence of the desired files on the remote cache
|
|
"""
|
|
from dvc.objects.status import compare_status
|
|
|
|
odb = LocalObjectDB(LocalFileSystem(), PathInfo("."))
|
|
obj_ids = {
|
|
HashInfo("md5", "acbd18db4cc2f85cedef654fccc4a4d8"),
|
|
HashInfo("md5", "37b51d194a7513e45b56f6524f2d51f2"),
|
|
}
|
|
|
|
local_exists = [hash_info.value for hash_info in obj_ids]
|
|
mocker.patch.object(odb, "hashes_exist", return_value=local_exists)
|
|
|
|
src_odb = mocker.Mock()
|
|
|
|
compare_status(src_odb, odb, obj_ids, check_deleted=False)
|
|
assert src_odb.hashes_exist.call_count == 0
|
|
|
|
|
|
@pytest.mark.parametrize("link_name", ["hardlink", "symlink"])
|
|
def test_is_protected(tmp_dir, dvc, link_name):
|
|
odb = dvc.odb.local
|
|
fs = odb.fs
|
|
link_method = getattr(fs, link_name)
|
|
|
|
(tmp_dir / "foo").write_text("foo")
|
|
|
|
foo = PathInfo(tmp_dir / "foo")
|
|
link = PathInfo(tmp_dir / "link")
|
|
|
|
link_method(foo, link)
|
|
|
|
assert not odb.is_protected(foo)
|
|
assert not odb.is_protected(link)
|
|
|
|
odb.protect(foo)
|
|
|
|
assert odb.is_protected(foo)
|
|
assert odb.is_protected(link)
|
|
|
|
odb.unprotect(link)
|
|
|
|
assert not odb.is_protected(link)
|
|
if os.name == "nt" and link_name == "hardlink":
|
|
# NOTE: NTFS doesn't allow deleting read-only files, which forces us to
|
|
# set write perms on the link, which propagates to the source.
|
|
assert not odb.is_protected(foo)
|
|
else:
|
|
assert odb.is_protected(foo)
|
|
|
|
|
|
@pytest.mark.parametrize("err", [errno.EPERM, errno.EACCES, errno.EROFS])
|
|
def test_protect_ignore_errors(tmp_dir, dvc, mocker, err):
|
|
tmp_dir.gen("foo", "foo")
|
|
|
|
mock_chmod = mocker.patch(
|
|
"os.chmod", side_effect=OSError(err, "something")
|
|
)
|
|
dvc.odb.local.protect(PathInfo("foo"))
|
|
assert mock_chmod.called
|
|
|
|
|
|
@pytest.mark.parametrize("err", [errno.EPERM, errno.EACCES, errno.EROFS])
|
|
def test_set_exec_ignore_errors(tmp_dir, dvc, mocker, err):
|
|
tmp_dir.gen("foo", "foo")
|
|
|
|
mock_chmod = mocker.patch(
|
|
"os.chmod", side_effect=OSError(err, "something")
|
|
)
|
|
dvc.odb.local.set_exec(PathInfo("foo"))
|
|
assert mock_chmod.called
|
|
|
|
|
|
def test_staging_file(tmp_dir, dvc):
|
|
from dvc.objects import check
|
|
from dvc.objects.stage import stage
|
|
from dvc.objects.transfer import transfer
|
|
|
|
tmp_dir.gen("foo", "foo")
|
|
fs = LocalFileSystem()
|
|
|
|
local_odb = dvc.odb.local
|
|
staging_odb, obj = stage(local_odb, tmp_dir / "foo", fs, "md5")
|
|
|
|
assert not local_odb.exists(obj.hash_info)
|
|
assert staging_odb.exists(obj.hash_info)
|
|
|
|
with pytest.raises(FileNotFoundError):
|
|
check(local_odb, obj)
|
|
check(staging_odb, obj)
|
|
|
|
transfer(staging_odb, local_odb, {obj.hash_info}, move=True)
|
|
check(local_odb, obj)
|
|
with pytest.raises(FileNotFoundError):
|
|
check(staging_odb, obj)
|
|
|
|
path_info = local_odb.hash_to_path_info(obj.hash_info.value)
|
|
assert fs.exists(path_info)
|
|
|
|
|
|
def test_staging_dir(tmp_dir, dvc):
|
|
from dvc.objects import check
|
|
from dvc.objects.stage import stage
|
|
from dvc.objects.transfer import transfer
|
|
|
|
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
|
fs = LocalFileSystem()
|
|
local_odb = dvc.odb.local
|
|
|
|
staging_odb, obj = stage(local_odb, tmp_dir / "dir", fs, "md5")
|
|
|
|
assert not local_odb.exists(obj.hash_info)
|
|
assert staging_odb.exists(obj.hash_info)
|
|
|
|
with pytest.raises(FileNotFoundError):
|
|
check(local_odb, obj)
|
|
check(staging_odb, obj)
|
|
|
|
transfer(staging_odb, local_odb, {obj.hash_info}, shallow=False, move=True)
|
|
check(local_odb, obj)
|
|
with pytest.raises(FileNotFoundError):
|
|
check(staging_odb, obj)
|
|
|
|
path_info = local_odb.hash_to_path_info(obj.hash_info.value)
|
|
assert fs.exists(path_info)
|