mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
Ruslan Kuprieiev
f64018b75e
Initial simple separation to get the process of extracting dvc-objects and dvc-data packages out of dvc. dvc-objects is for: * defining base Object (aka HashFile) and ObjectDB * providing tools to work with those objects (check/transfer/etc) * etc dvc-data is for: * defining higher level objects (e.g. Tree) * bulding objects from the workspace (aka staging) * projecting objects back into the workspace (aka checkout) * comparing Tree objects (aka diff) * defining misc object databases (e.g LocalObjectDB) * higher-level transfer (e.g. with Tree indexes) * etc For example, the objects we use in run-cache are based on dvc-objects and not on dvc-data, as they have absolutely nothing to do with data management. There are still some mixups in the code that will be addressed in the followups.
283 lines
7.7 KiB
Python
283 lines
7.7 KiB
Python
import os
|
|
import shutil
|
|
|
|
import pytest
|
|
|
|
from dvc.config import NoRemoteError
|
|
from dvc.data.stage import stage
|
|
from dvc.fs.dvc import DvcFileSystem
|
|
from dvc.hash_info import HashInfo
|
|
from dvc.utils.fs import remove
|
|
|
|
|
|
def test_exists(tmp_dir, dvc):
|
|
tmp_dir.gen("foo", "foo")
|
|
dvc.add("foo")
|
|
(tmp_dir / "foo").unlink()
|
|
|
|
fs = DvcFileSystem(repo=dvc)
|
|
assert fs.exists("foo")
|
|
|
|
|
|
def test_open(tmp_dir, dvc):
|
|
tmp_dir.gen("foo", "foo")
|
|
dvc.add("foo")
|
|
(tmp_dir / "foo").unlink()
|
|
|
|
fs = DvcFileSystem(repo=dvc)
|
|
with fs.open((tmp_dir / "foo").fs_path, "r") as fobj:
|
|
assert fobj.read() == "foo"
|
|
|
|
|
|
def test_open_dirty_hash(tmp_dir, dvc):
|
|
tmp_dir.dvc_gen("file", "file")
|
|
(tmp_dir / "file").write_text("something")
|
|
|
|
fs = DvcFileSystem(repo=dvc)
|
|
with fs.open((tmp_dir / "file").fs_path, "r") as fobj:
|
|
# NOTE: Unlike RepoFileSystem, DvcFileSystem should not
|
|
# be affected by a dirty workspace.
|
|
assert fobj.read() == "file"
|
|
|
|
|
|
def test_open_no_remote(tmp_dir, dvc):
|
|
tmp_dir.dvc_gen("file", "file")
|
|
(tmp_dir / "file").unlink()
|
|
remove(dvc.odb.local.cache_dir)
|
|
|
|
fs = DvcFileSystem(repo=dvc)
|
|
with pytest.raises(FileNotFoundError) as exc_info:
|
|
with fs.open((tmp_dir / "file").fs_path, "r"):
|
|
pass
|
|
assert isinstance(exc_info.value.__cause__, NoRemoteError)
|
|
|
|
|
|
def test_open_dirty_no_hash(tmp_dir, dvc):
|
|
tmp_dir.gen("file", "file")
|
|
(tmp_dir / "file.dvc").write_text("outs:\n- path: file\n")
|
|
|
|
fs = DvcFileSystem(repo=dvc)
|
|
# NOTE: Unlike RepoFileSystem, DvcFileSystem should not
|
|
# be affected by a dirty workspace.
|
|
with pytest.raises(FileNotFoundError):
|
|
with fs.open((tmp_dir / "file").fs_path, "r"):
|
|
pass
|
|
|
|
|
|
def test_open_in_history(tmp_dir, scm, dvc):
|
|
tmp_dir.gen("foo", "foo")
|
|
dvc.add("foo")
|
|
dvc.scm.add(["foo.dvc", ".gitignore"])
|
|
dvc.scm.commit("foo")
|
|
|
|
tmp_dir.gen("foo", "foofoo")
|
|
dvc.add("foo")
|
|
dvc.scm.add(["foo.dvc", ".gitignore"])
|
|
dvc.scm.commit("foofoo")
|
|
|
|
for rev in dvc.brancher(revs=["HEAD~1"]):
|
|
if rev == "workspace":
|
|
continue
|
|
|
|
fs = DvcFileSystem(repo=dvc)
|
|
with fs.open((tmp_dir / "foo").fs_path, "r") as fobj:
|
|
assert fobj.read() == "foo"
|
|
|
|
|
|
def test_isdir_isfile(tmp_dir, dvc):
|
|
tmp_dir.gen({"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}})
|
|
|
|
fs = DvcFileSystem(repo=dvc)
|
|
assert not fs.isdir("datadir")
|
|
assert not fs.isfile("datadir")
|
|
assert not fs.isdir("datafile")
|
|
assert not fs.isfile("datafile")
|
|
|
|
dvc.add(["datadir", "datafile"])
|
|
shutil.rmtree(tmp_dir / "datadir")
|
|
(tmp_dir / "datafile").unlink()
|
|
|
|
assert fs.isdir("datadir")
|
|
assert not fs.isfile("datadir")
|
|
assert not fs.isdir("datafile")
|
|
assert fs.isfile("datafile")
|
|
|
|
|
|
def test_isdir_mixed(tmp_dir, dvc):
|
|
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
|
|
|
dvc.add(str(tmp_dir / "dir" / "foo"))
|
|
|
|
fs = DvcFileSystem(repo=dvc)
|
|
assert fs.isdir("dir")
|
|
assert not fs.isfile("dir")
|
|
|
|
|
|
def test_walk(tmp_dir, dvc):
|
|
tmp_dir.gen(
|
|
{
|
|
"dir": {
|
|
"subdir1": {"foo1": "foo1", "bar1": "bar1"},
|
|
"subdir2": {"foo2": "foo2"},
|
|
"foo": "foo",
|
|
"bar": "bar",
|
|
}
|
|
}
|
|
)
|
|
|
|
dvc.add("dir", recursive=True)
|
|
fs = DvcFileSystem(repo=dvc)
|
|
|
|
expected = [
|
|
str(tmp_dir / "dir" / "subdir1"),
|
|
str(tmp_dir / "dir" / "subdir2"),
|
|
str(tmp_dir / "dir" / "subdir1" / "foo1"),
|
|
str(tmp_dir / "dir" / "subdir1" / "bar1"),
|
|
str(tmp_dir / "dir" / "subdir2" / "foo2"),
|
|
str(tmp_dir / "dir" / "foo"),
|
|
str(tmp_dir / "dir" / "bar"),
|
|
]
|
|
|
|
actual = []
|
|
for root, dirs, files in fs.walk("dir"):
|
|
for entry in dirs + files:
|
|
actual.append(os.path.join(root, entry))
|
|
|
|
assert set(actual) == set(expected)
|
|
assert len(actual) == len(expected)
|
|
|
|
|
|
def test_walk_dir(tmp_dir, dvc):
|
|
tmp_dir.gen(
|
|
{
|
|
"dir": {
|
|
"subdir1": {"foo1": "foo1", "bar1": "bar1"},
|
|
"subdir2": {"foo2": "foo2"},
|
|
"foo": "foo",
|
|
"bar": "bar",
|
|
}
|
|
}
|
|
)
|
|
|
|
dvc.add("dir")
|
|
fs = DvcFileSystem(repo=dvc)
|
|
|
|
expected = [
|
|
str(tmp_dir / "dir" / "subdir1"),
|
|
str(tmp_dir / "dir" / "subdir2"),
|
|
str(tmp_dir / "dir" / "subdir1" / "foo1"),
|
|
str(tmp_dir / "dir" / "subdir1" / "bar1"),
|
|
str(tmp_dir / "dir" / "subdir2" / "foo2"),
|
|
str(tmp_dir / "dir" / "foo"),
|
|
str(tmp_dir / "dir" / "bar"),
|
|
]
|
|
|
|
actual = []
|
|
for root, dirs, files in fs.walk("dir"):
|
|
for entry in dirs + files:
|
|
actual.append(os.path.join(root, entry))
|
|
|
|
assert set(actual) == set(expected)
|
|
assert len(actual) == len(expected)
|
|
|
|
|
|
def test_walk_onerror(tmp_dir, dvc):
|
|
def onerror(exc):
|
|
raise exc
|
|
|
|
tmp_dir.dvc_gen("foo", "foo")
|
|
fs = DvcFileSystem(repo=dvc)
|
|
|
|
# path does not exist
|
|
for _ in fs.walk("dir"):
|
|
pass
|
|
with pytest.raises(OSError):
|
|
for _ in fs.walk("dir", onerror=onerror):
|
|
pass
|
|
|
|
# path is not a directory
|
|
for _ in fs.walk("foo"):
|
|
pass
|
|
with pytest.raises(OSError):
|
|
for _ in fs.walk("foo", onerror=onerror):
|
|
pass
|
|
|
|
|
|
def test_isdvc(tmp_dir, dvc):
|
|
tmp_dir.gen({"foo": "foo", "bar": "bar"})
|
|
dvc.add("foo")
|
|
fs = DvcFileSystem(repo=dvc)
|
|
assert fs.isdvc("foo")
|
|
assert not fs.isdvc("bar")
|
|
|
|
|
|
def test_get_hash_file(tmp_dir, dvc):
|
|
tmp_dir.dvc_gen({"foo": "foo"})
|
|
fs = DvcFileSystem(repo=dvc)
|
|
assert (
|
|
fs.info((tmp_dir / "foo").fs_path)["md5"]
|
|
== "acbd18db4cc2f85cedef654fccc4a4d8"
|
|
)
|
|
|
|
|
|
def test_get_hash_dir(tmp_dir, dvc, mocker):
|
|
import dvc as dvc_module
|
|
|
|
tmp_dir.dvc_gen(
|
|
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
|
|
)
|
|
fs = DvcFileSystem(repo=dvc)
|
|
get_file_hash_spy = mocker.spy(dvc_module.data.stage, "get_file_hash")
|
|
assert (
|
|
fs.info((tmp_dir / "dir").fs_path)["md5"]
|
|
== "8761c4e9acad696bee718615e23e22db.dir"
|
|
)
|
|
assert not get_file_hash_spy.called
|
|
|
|
|
|
def test_get_hash_granular(tmp_dir, dvc):
|
|
tmp_dir.dvc_gen(
|
|
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
|
|
)
|
|
fs = DvcFileSystem(repo=dvc)
|
|
subdir = tmp_dir / "dir" / "subdir"
|
|
assert fs.info(subdir.fs_path).get("md5") is None
|
|
_, _, obj = stage(dvc.odb.local, subdir.fs_path, fs, "md5", dry_run=True)
|
|
assert obj.hash_info == HashInfo(
|
|
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir"
|
|
)
|
|
assert (
|
|
fs.info((subdir / "data").fs_path)["md5"]
|
|
== "8d777f385d3dfec8815d20f7496026dc"
|
|
)
|
|
_, _, obj = stage(
|
|
dvc.odb.local, (subdir / "data").fs_path, fs, "md5", dry_run=True
|
|
)
|
|
assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc")
|
|
|
|
|
|
def test_get_hash_dirty_file(tmp_dir, dvc):
|
|
tmp_dir.dvc_gen("file", "file")
|
|
(tmp_dir / "file").write_text("something")
|
|
|
|
fs = DvcFileSystem(repo=dvc)
|
|
expected = "8c7dd922ad47494fc02c388e12c00eac"
|
|
assert fs.info((tmp_dir / "file").fs_path).get("md5") == expected
|
|
_, _, obj = stage(
|
|
dvc.odb.local, (tmp_dir / "file").fs_path, fs, "md5", dry_run=True
|
|
)
|
|
assert obj.hash_info == HashInfo("md5", expected)
|
|
|
|
|
|
def test_get_hash_dirty_dir(tmp_dir, dvc):
|
|
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
|
|
(tmp_dir / "dir" / "baz").write_text("baz")
|
|
|
|
fs = DvcFileSystem(repo=dvc)
|
|
expected = "5ea40360f5b4ec688df672a4db9c17d1.dir"
|
|
assert fs.info((tmp_dir / "dir").fs_path).get("md5") == expected
|
|
_, _, obj = stage(
|
|
dvc.odb.local, (tmp_dir / "dir").fs_path, fs, "md5", dry_run=True
|
|
)
|
|
assert obj.hash_info == HashInfo("md5", expected)
|