mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-06 07:18:28 +01:00
Ruslan Kuprieiev
f64018b75e
Initial simple separation to get the process of extracting dvc-objects and dvc-data packages out of dvc. dvc-objects is for: * defining base Object (aka HashFile) and ObjectDB * providing tools to work with those objects (check/transfer/etc) * etc dvc-data is for: * defining higher level objects (e.g. Tree) * bulding objects from the workspace (aka staging) * projecting objects back into the workspace (aka checkout) * comparing Tree objects (aka diff) * defining misc object databases (e.g LocalObjectDB) * higher-level transfer (e.g. with Tree indexes) * etc For example, the objects we use in run-cache are based on dvc-objects and not on dvc-data, as they have absolutely nothing to do with data management. There are still some mixups in the code that will be addressed in the followups.
143 lines
3.9 KiB
Python
143 lines
3.9 KiB
Python
import errno
|
|
import os
|
|
|
|
import pytest
|
|
|
|
from dvc.data.db.local import LocalObjectDB
|
|
from dvc.fs.local import LocalFileSystem
|
|
from dvc.hash_info import HashInfo
|
|
|
|
|
|
def test_status_download_optimization(mocker, dvc):
|
|
"""When comparing the status to pull a remote cache,
|
|
And the desired files to fetch are already on the local cache,
|
|
Don't check the existence of the desired files on the remote cache
|
|
"""
|
|
from dvc.data.status import compare_status
|
|
|
|
odb = LocalObjectDB(LocalFileSystem(), os.getcwd())
|
|
obj_ids = {
|
|
HashInfo("md5", "acbd18db4cc2f85cedef654fccc4a4d8"),
|
|
HashInfo("md5", "37b51d194a7513e45b56f6524f2d51f2"),
|
|
}
|
|
|
|
local_exists = [hash_info.value for hash_info in obj_ids]
|
|
mocker.patch.object(odb, "hashes_exist", return_value=local_exists)
|
|
|
|
src_odb = mocker.Mock()
|
|
|
|
compare_status(src_odb, odb, obj_ids, check_deleted=False)
|
|
assert src_odb.hashes_exist.call_count == 0
|
|
|
|
|
|
@pytest.mark.parametrize("link_name", ["hardlink", "symlink"])
|
|
def test_is_protected(tmp_dir, dvc, link_name):
|
|
odb = dvc.odb.local
|
|
fs = odb.fs
|
|
link_method = getattr(fs, link_name)
|
|
|
|
(tmp_dir / "foo").write_text("foo")
|
|
|
|
foo = tmp_dir / "foo"
|
|
link = tmp_dir / "link"
|
|
|
|
link_method(foo, link)
|
|
|
|
assert not odb.is_protected(foo)
|
|
assert not odb.is_protected(link)
|
|
|
|
odb.protect(foo)
|
|
|
|
assert odb.is_protected(foo)
|
|
assert odb.is_protected(link)
|
|
|
|
odb.unprotect(link)
|
|
|
|
assert not odb.is_protected(link)
|
|
if os.name == "nt" and link_name == "hardlink":
|
|
# NOTE: NTFS doesn't allow deleting read-only files, which forces us to
|
|
# set write perms on the link, which propagates to the source.
|
|
assert not odb.is_protected(foo)
|
|
else:
|
|
assert odb.is_protected(foo)
|
|
|
|
|
|
@pytest.mark.parametrize("err", [errno.EPERM, errno.EACCES, errno.EROFS])
|
|
def test_protect_ignore_errors(tmp_dir, dvc, mocker, err):
|
|
tmp_dir.gen("foo", "foo")
|
|
|
|
mock_chmod = mocker.patch(
|
|
"os.chmod", side_effect=OSError(err, "something")
|
|
)
|
|
dvc.odb.local.protect("foo")
|
|
assert mock_chmod.called
|
|
|
|
|
|
@pytest.mark.parametrize("err", [errno.EPERM, errno.EACCES, errno.EROFS])
|
|
def test_set_exec_ignore_errors(tmp_dir, dvc, mocker, err):
|
|
tmp_dir.gen("foo", "foo")
|
|
|
|
mock_chmod = mocker.patch(
|
|
"os.chmod", side_effect=OSError(err, "something")
|
|
)
|
|
dvc.odb.local.set_exec("foo")
|
|
assert mock_chmod.called
|
|
|
|
|
|
def test_staging_file(tmp_dir, dvc):
|
|
from dvc.data import check
|
|
from dvc.data.stage import stage
|
|
from dvc.data.transfer import transfer
|
|
|
|
tmp_dir.gen("foo", "foo")
|
|
fs = LocalFileSystem()
|
|
|
|
local_odb = dvc.odb.local
|
|
staging_odb, _, obj = stage(
|
|
local_odb, (tmp_dir / "foo").fs_path, fs, "md5"
|
|
)
|
|
|
|
assert not local_odb.exists(obj.hash_info)
|
|
assert staging_odb.exists(obj.hash_info)
|
|
|
|
with pytest.raises(FileNotFoundError):
|
|
check(local_odb, obj)
|
|
check(staging_odb, obj)
|
|
|
|
transfer(staging_odb, local_odb, {obj.hash_info}, hardlink=True)
|
|
check(local_odb, obj)
|
|
check(staging_odb, obj)
|
|
|
|
path = local_odb.hash_to_path(obj.hash_info.value)
|
|
assert fs.exists(path)
|
|
|
|
|
|
def test_staging_dir(tmp_dir, dvc):
|
|
from dvc.data import check
|
|
from dvc.data.stage import stage
|
|
from dvc.data.transfer import transfer
|
|
|
|
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
|
fs = LocalFileSystem()
|
|
local_odb = dvc.odb.local
|
|
|
|
staging_odb, _, obj = stage(
|
|
local_odb, (tmp_dir / "dir").fs_path, fs, "md5"
|
|
)
|
|
|
|
assert not local_odb.exists(obj.hash_info)
|
|
assert staging_odb.exists(obj.hash_info)
|
|
|
|
with pytest.raises(FileNotFoundError):
|
|
check(local_odb, obj)
|
|
check(staging_odb, obj)
|
|
|
|
transfer(
|
|
staging_odb, local_odb, {obj.hash_info}, shallow=False, hardlink=True
|
|
)
|
|
check(local_odb, obj)
|
|
check(staging_odb, obj)
|
|
|
|
path = local_odb.hash_to_path(obj.hash_info.value)
|
|
assert fs.exists(path)
|