fs: rename RepoFileSystem to DvcFileSystem

This commit is contained in:
Ruslan Kuprieiev
2022-05-12 00:39:41 +03:00
parent b6d25825c8
commit 005d56eedf
24 changed files with 121 additions and 121 deletions
+2 -2
View File
@@ -19,9 +19,9 @@ def get_url(path, repo=None, rev=None, remote=None):
directory in the remote storage.
"""
with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
fs_path = _repo.repo_fs.from_os_path(path)
fs_path = _repo.dvcfs.from_os_path(path)
with reraise(FileNotFoundError, PathMissingError(path, repo)):
info = _repo.repo_fs.info(fs_path)
info = _repo.dvcfs.info(fs_path)
dvc_info = info.get("dvc_info")
if not dvc_info:
+3 -3
View File
@@ -71,7 +71,7 @@ class ReferenceHashFile(HashFile):
@classmethod
def from_bytes(cls, data: bytes, fs_cache: Optional[dict] = None):
from dvc.fs import get_fs_cls
from dvc.fs.repo import RepoFileSystem, _RepoFileSystem
from dvc.fs.dvc import DvcFileSystem, _DvcFileSystem
try:
dict_ = pickle.loads(data)
@@ -88,8 +88,8 @@ class ReferenceHashFile(HashFile):
fs = fs_cache.get((scheme, config_pairs)) if fs_cache else None
if not fs:
config = dict(config_pairs)
if _RepoFileSystem.PARAM_REPO_URL in config:
fs_cls = RepoFileSystem
if _DvcFileSystem.PARAM_REPO_URL in config:
fs_cls = DvcFileSystem
else:
fs_cls = get_fs_cls(config, scheme=scheme)
fs = fs_cls(**config)
+3 -3
View File
@@ -133,7 +133,7 @@ class RepoDependency(Dependency):
staging, _, staged_obj = stage(
local_odb,
as_posix(self.def_path),
repo.repo_fs,
repo.dvcfs,
local_odb.fs.PARAM_CHECKSUM,
)
except (FileNotFoundError, TreeError) as exc:
@@ -153,7 +153,7 @@ class RepoDependency(Dependency):
from dvc.data.db.reference import ReferenceObjectDB
from dvc.data.tree import Tree
from dvc.exceptions import CircularImportError
from dvc.fs.repo import RepoFileSystem
from dvc.fs.dvc import DvcFileSystem
if not isinstance(odb, ReferenceObjectDB):
return
@@ -168,7 +168,7 @@ class RepoDependency(Dependency):
checked_urls = set()
for obj in iter_objs():
if not isinstance(obj.fs, RepoFileSystem):
if not isinstance(obj.fs, DvcFileSystem):
continue
if (
obj.fs.repo_url in checked_urls
+3 -3
View File
@@ -63,7 +63,7 @@ def _merge_info(repo, fs_info, dvc_info):
return ret
class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
class _DvcFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
"""DVC + git-tracked files fs.
Args:
@@ -436,7 +436,7 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
return dvc_fs.checksum(dvc_path)
class RepoFileSystem(FileSystem):
class DvcFileSystem(FileSystem):
scheme = "local"
PARAM_CHECKSUM = "md5"
@@ -446,7 +446,7 @@ class RepoFileSystem(FileSystem):
@wrap_prop(threading.Lock())
@cached_property
def fs(self):
return _RepoFileSystem(**self.fs_args)
return _DvcFileSystem(**self.fs_args)
def isdvc(self, path, **kwargs):
return self.fs.isdvc(path, **kwargs)
+6 -6
View File
@@ -188,7 +188,7 @@ class Repo:
self.config = Config(self.dvc_dir, fs=self.fs, config=config)
self._uninitialized = uninitialized
# used by RepoFileSystem to determine if it should traverse subrepos
# used by DvcFileSystem to determine if it should traverse subrepos
self.subrepos = subrepos
self.cloud = DataCloud(self)
@@ -490,10 +490,10 @@ class Repo:
return DataFileSystem(repo=self)
@cached_property
def repo_fs(self):
from dvc.fs.repo import RepoFileSystem
def dvcfs(self):
from dvc.fs.dvc import DvcFileSystem
return RepoFileSystem(
return DvcFileSystem(
repo=self, subrepos=self.subrepos, **self._fs_conf
)
@@ -505,13 +505,13 @@ class Repo:
def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
"""Opens a specified resource as a file descriptor"""
from dvc.fs.data import DataFileSystem
from dvc.fs.repo import RepoFileSystem
from dvc.fs.dvc import DvcFileSystem
if os.path.isabs(path):
fs = DataFileSystem(repo=self, workspace="local")
fs_path = path
else:
fs = RepoFileSystem(repo=self, subrepos=True)
fs = DvcFileSystem(repo=self, subrepos=True)
fs_path = fs.from_os_path(path)
try:
+3 -3
View File
@@ -30,9 +30,9 @@ def _collect_paths(
recursive: bool = False,
rev: str = None,
) -> StrPaths:
from dvc.fs.repo import RepoFileSystem
from dvc.fs.dvc import DvcFileSystem
fs = RepoFileSystem(repo=repo)
fs = DvcFileSystem(repo=repo)
fs_paths = [fs.from_os_path(target) for target in targets]
target_paths: StrPaths = []
@@ -58,7 +58,7 @@ def _filter_duplicates(
fs_res_paths = fs_paths
for out in outs:
fs_path = out.repo.repo_fs.from_os_path(out.fs_path)
fs_path = out.repo.dvcfs.from_os_path(out.fs_path)
if fs_path in fs_paths:
res_outs.append(out)
# MUTATING THE SAME LIST!!
+10 -10
View File
@@ -22,9 +22,9 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
if self.scm.no_commits:
return {}
from dvc.fs.repo import RepoFileSystem
from dvc.fs.dvc import DvcFileSystem
repo_fs = RepoFileSystem(repo=self)
dvcfs = DvcFileSystem(repo=self)
b_rev = b_rev if b_rev else "workspace"
results = {}
@@ -39,7 +39,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
if targets is not None:
# convert targets to paths, and capture any missing targets
targets_paths, missing_targets[rev] = _targets_to_paths(
repo_fs, targets
dvcfs, targets
)
results[rev] = _paths_checksums(self, targets_paths)
@@ -61,7 +61,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
if b_rev == "workspace":
# missing status is only applicable when diffing local workspace
# against a commit
missing = sorted(_filter_missing(repo_fs, deleted_or_missing))
missing = sorted(_filter_missing(dvcfs, deleted_or_missing))
else:
missing = []
deleted = sorted(deleted_or_missing - set(missing))
@@ -178,11 +178,11 @@ def _dir_output_paths(fs, fs_path, obj, targets=None):
yield os.path.join(base, *key), oid.value
def _filter_missing(repo_fs, paths):
def _filter_missing(dvcfs, paths):
for path in paths:
fs_path = repo_fs.from_os_path(path)
fs_path = dvcfs.from_os_path(path)
try:
info = repo_fs.info(fs_path)
info = dvcfs.info(fs_path)
dvc_info = info.get("dvc_info")
if (
dvc_info
@@ -194,13 +194,13 @@ def _filter_missing(repo_fs, paths):
pass
def _targets_to_paths(repo_fs, targets):
def _targets_to_paths(dvcfs, targets):
paths = []
missing = []
for target in targets:
if repo_fs.exists(target):
paths.append(repo_fs.repo.fs.path.abspath(target))
if dvcfs.exists(target):
paths.append(dvcfs.repo.fs.path.abspath(target))
else:
missing.append(target)
+1 -1
View File
@@ -57,7 +57,7 @@ def get(url, path, out=None, rev=None, jobs=None):
fs = DataFileSystem(repo=repo, workspace="local")
fs_path = path
else:
fs = repo.repo_fs
fs = repo.dvcfs
fs_path = fs.from_os_path(path)
with FsspecCallback.as_tqdm_callback(
+2 -2
View File
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Optional
from dvc.exceptions import PathMissingError
if TYPE_CHECKING:
from dvc.fs.repo import RepoFileSystem
from dvc.fs.dvc import DvcFileSystem
from . import Repo
@@ -56,7 +56,7 @@ def ls(
def _ls(
repo: "Repo", path: str, recursive: bool = None, dvc_only: bool = False
):
fs: "RepoFileSystem" = repo.repo_fs
fs: "DvcFileSystem" = repo.dvcfs
fs_path = fs.from_os_path(path)
try:
+4 -4
View File
@@ -4,7 +4,7 @@ from typing import List
from scmrepo.exceptions import SCMError
from dvc.fs.repo import RepoFileSystem
from dvc.fs.dvc import DvcFileSystem
from dvc.output import Output
from dvc.repo import locked
from dvc.repo.collect import StrPaths, collect
@@ -24,11 +24,11 @@ def _to_fs_paths(metrics: List[Output]) -> StrPaths:
result = []
for out in metrics:
if out.metric:
result.append(out.repo.repo_fs.from_os_path(out.fs_path))
result.append(out.repo.dvcfs.from_os_path(out.fs_path))
elif out.live:
fs_path = summary_fs_path(out)
if fs_path:
result.append(out.repo.repo_fs.from_os_path(fs_path))
result.append(out.repo.dvcfs.from_os_path(fs_path))
return result
@@ -76,7 +76,7 @@ def _read_metric(path, fs, rev, **kwargs):
def _read_metrics(repo, metrics, rev, onerror=None):
fs = RepoFileSystem(repo=repo)
fs = DvcFileSystem(repo=repo)
relpath = ""
if repo.root_dir != repo.fs.path.getcwd():
+3 -3
View File
@@ -99,9 +99,9 @@ class Plots:
onerror: Optional[Callable] = None,
props: Optional[Dict] = None,
):
from dvc.fs.repo import RepoFileSystem
from dvc.fs.dvc import DvcFileSystem
fs = RepoFileSystem(repo=self.repo)
fs = DvcFileSystem(repo=self.repo)
plots = _collect_plots(self.repo, targets, revision, recursive)
res: Dict[str, Any] = {}
for fs_path, rev_props in plots.items():
@@ -253,7 +253,7 @@ def _collect_plots(
)
result = {
repo.repo_fs.from_os_path(plot.fs_path): _plot_props(plot)
repo.dvcfs.from_os_path(plot.fs_path): _plot_props(plot)
for plot in plots
}
result.update({fs_path: {} for fs_path in fs_paths})
+1 -1
View File
@@ -63,7 +63,7 @@ def _get_stage_files(stage: "Stage") -> typing.Iterator[str]:
if (
not dep.use_scm_ignore
and dep.is_in_repo
and not stage.repo.repo_fs.isdvc(dep.fs_path)
and not stage.repo.dvcfs.isdvc(dep.fs_path)
):
yield dep.fs_path
for out in stage.outs:
+5 -5
View File
@@ -31,7 +31,7 @@ def test_new_checkpoint(
for rev in dvc.brancher([exp]):
if rev == "workspace":
continue
fs = dvc.repo_fs
fs = dvc.dvcfs
with fs.open("foo") as fobj:
assert fobj.read().strip() == str(checkpoint_stage.iterations)
with fs.open("metrics.yaml") as fobj:
@@ -80,7 +80,7 @@ def test_resume_checkpoint(
for rev in dvc.brancher([exp]):
if rev == "workspace":
continue
fs = dvc.repo_fs
fs = dvc.dvcfs
with fs.open("foo") as fobj:
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
with fs.open("metrics.yaml") as fobj:
@@ -111,7 +111,7 @@ def test_reset_checkpoint(
for rev in dvc.brancher([exp]):
if rev == "workspace":
continue
fs = dvc.repo_fs
fs = dvc.dvcfs
with fs.open("foo") as fobj:
assert fobj.read().strip() == str(checkpoint_stage.iterations)
with fs.open("metrics.yaml") as fobj:
@@ -150,7 +150,7 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage, workspace):
for rev in dvc.brancher([checkpoint_a]):
if rev == "workspace":
continue
fs = dvc.repo_fs
fs = dvc.dvcfs
with fs.open("foo") as fobj:
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
with fs.open("metrics.yaml") as fobj:
@@ -159,7 +159,7 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage, workspace):
for rev in dvc.brancher([checkpoint_b]):
if rev == "workspace":
continue
fs = dvc.repo_fs
fs = dvc.dvcfs
with fs.open("foo") as fobj:
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
with fs.open("metrics.yaml") as fobj:
+3 -3
View File
@@ -525,7 +525,7 @@ def test_subdir(tmp_dir, scm, dvc, workspace):
@pytest.mark.parametrize("workspace", [True, False])
def test_subrepo(tmp_dir, scm, workspace):
from tests.unit.fs.test_repo import make_subrepo
from tests.unit.fs.test_dvc import make_subrepo
subrepo = tmp_dir / "dir" / "repo"
make_subrepo(subrepo, scm)
@@ -673,9 +673,9 @@ def test_modified_data_dep(tmp_dir, scm, dvc, workspace, params, target):
for rev in dvc.brancher(revs=[exp]):
if rev != exp:
continue
with dvc.repo_fs.open("metrics.yaml") as fobj:
with dvc.dvcfs.open("metrics.yaml") as fobj:
assert fobj.read().strip() == params
with dvc.repo_fs.open("data") as fobj:
with dvc.dvcfs.open("data") as fobj:
assert fobj.read().strip() == "modified"
if workspace:
+1 -1
View File
@@ -11,7 +11,7 @@ from dvc.exceptions import (
)
from dvc.testing.test_api import TestAPI # noqa, pylint: disable=unused-import
from dvc.utils.fs import remove
from tests.unit.fs.test_repo import make_subrepo
from tests.unit.fs.test_dvc import make_subrepo
def test_get_url_external(tmp_dir, erepo_dir, cloud):
+5 -5
View File
@@ -8,7 +8,7 @@ from dvc.data.transfer import transfer
from dvc.external_repo import CLONES, external_repo
from dvc.utils import relpath
from dvc.utils.fs import makedirs, remove
from tests.unit.fs.test_repo import make_subrepo
from tests.unit.fs.test_dvc import make_subrepo
from tests.utils import clean_staging
@@ -95,7 +95,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
dest = tmp_dir / "file"
with external_repo(os.fspath(erepo_dir)) as repo:
repo.repo_fs.get(
repo.dvcfs.get(
"subdir/file",
os.fspath(dest),
)
@@ -191,7 +191,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
with external_repo(os.fspath(erepo_dir)) as repo:
repo.repo_fs.get(
repo.dvcfs.get(
"dir",
os.fspath(tmp_dir / "out"),
)
@@ -207,7 +207,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
staging, _, obj = stage(
repo.odb.local,
"dir",
repo.repo_fs,
repo.dvcfs,
"md5",
dvcignore=repo.dvcignore,
)
@@ -237,7 +237,7 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
with external_repo(os.fspath(erepo_dir)) as repo:
repo.repo_fs.get(
repo.dvcfs.get(
"dir",
os.fspath(tmp_dir / "out"),
)
+2 -2
View File
@@ -273,11 +273,11 @@ def test_get_dir_callback(tmp_dir, dvc, cloud):
assert (tmp_dir / "dir").read_text() == {"foo": "foo", "bar": "bar"}
def test_callback_on_repo_fs(tmp_dir, dvc, scm, mocker):
def test_callback_on_dvcfs(tmp_dir, dvc, scm, mocker):
tmp_dir.dvc_gen({"dir": {"bar": "bar"}}, commit="dvc")
tmp_dir.scm_gen({"dir": {"foo": "foo"}}, commit="git")
fs = dvc.repo_fs
fs = dvc.dvcfs
callback = FsspecCallback()
fs.get(
+1 -1
View File
@@ -9,7 +9,7 @@ from dvc.repo import Repo
from dvc.repo.get import GetDVCFileError
from dvc.system import System
from dvc.utils.fs import makedirs
from tests.unit.fs.test_repo import make_subrepo
from tests.unit.fs.test_dvc import make_subrepo
def test_get_repo_file(tmp_dir, erepo_dir):
+1 -1
View File
@@ -13,7 +13,7 @@ from dvc.exceptions import DownloadError, PathMissingError
from dvc.stage.exceptions import StagePathNotFoundError
from dvc.system import System
from dvc.utils.fs import makedirs, remove
from tests.unit.fs.test_repo import make_subrepo
from tests.unit.fs.test_dvc import make_subrepo
from tests.utils import clean_staging
+1 -1
View File
@@ -4,7 +4,7 @@ import pytest
from dvc.dvcfile import Dvcfile
from dvc.exceptions import InvalidArgumentError
from tests.unit.fs.test_repo import make_subrepo
from tests.unit.fs.test_dvc import make_subrepo
@pytest.mark.parametrize("cached", [True, False])
+2 -2
View File
@@ -50,7 +50,7 @@ def test_open_dirty_hash(tmp_dir, dvc):
fs = DataFileSystem(repo=dvc)
with fs.open("file", "r") as fobj:
# NOTE: Unlike RepoFileSystem, DataFileSystem should not
# NOTE: Unlike DvcFileSystem, DataFileSystem should not
# be affected by a dirty workspace.
assert fobj.read() == "file"
@@ -72,7 +72,7 @@ def test_open_dirty_no_hash(tmp_dir, dvc):
(tmp_dir / "file.dvc").write_text("outs:\n- path: file\n")
fs = DataFileSystem(repo=dvc)
# NOTE: Unlike RepoFileSystem, DataFileSystem should not
# NOTE: Unlike DvcFileSystem, DataFileSystem should not
# be affected by a dirty workspace.
with pytest.raises(FileNotFoundError):
with fs.open("file", "r"):
@@ -6,7 +6,7 @@ from unittest import mock
import pytest
from dvc.data.stage import stage
from dvc.fs.repo import RepoFileSystem
from dvc.fs.dvc import DvcFileSystem
from dvc.objects.hash_info import HashInfo
from tests.utils import clean_staging
@@ -16,7 +16,7 @@ def test_exists(tmp_dir, dvc):
dvc.add("foo")
(tmp_dir / "foo").unlink()
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
assert fs.exists("foo")
@@ -25,7 +25,7 @@ def test_open(tmp_dir, dvc):
dvc.add("foo")
(tmp_dir / "foo").unlink()
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
with fs.open("foo", "r") as fobj:
assert fobj.read() == "foo"
@@ -34,7 +34,7 @@ def test_open_dirty_hash(tmp_dir, dvc):
tmp_dir.dvc_gen("file", "file")
(tmp_dir / "file").write_text("something")
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
with fs.open("file", "r") as fobj:
assert fobj.read() == "something"
@@ -43,7 +43,7 @@ def test_open_dirty_no_hash(tmp_dir, dvc):
tmp_dir.gen("file", "file")
(tmp_dir / "file.dvc").write_text("outs:\n- path: file\n")
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
with fs.open("file", "r") as fobj:
assert fobj.read() == "file"
@@ -63,7 +63,7 @@ def test_open_in_history(tmp_dir, scm, dvc):
if rev == "workspace":
continue
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
with fs.open("foo", "r") as fobj:
assert fobj.read() == "foo"
@@ -86,7 +86,7 @@ def test_isdir_isfile(tmp_dir, dvc):
},
)
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
assert fs.isdir("datadir")
assert not fs.isfile("datadir")
assert not fs.isdvc("datadir")
@@ -126,7 +126,7 @@ def test_exists_isdir_isfile_dirty(tmp_dir, dvc):
{"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}}
)
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
shutil.rmtree(tmp_dir / "datadir")
(tmp_dir / "datafile").unlink()
@@ -161,7 +161,7 @@ def test_isdir_mixed(tmp_dir, dvc):
dvc.add(str(tmp_dir / "dir" / "foo"))
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
assert fs.isdir("dir")
assert not fs.isfile("dir")
@@ -191,7 +191,7 @@ def test_walk(tmp_dir, dvc, dvcfiles, extra_expected):
)
dvc.add(str(tmp_dir / "dir"), recursive=True)
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
expected = [
"dir/subdir1",
@@ -226,7 +226,7 @@ def test_walk_dirty(tmp_dir, dvc):
tmp_dir.gen({"dir": {"bar": "bar", "subdir3": {"foo3": "foo3"}}})
(tmp_dir / "dir" / "foo").unlink()
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
expected = [
"dir/subdir1",
"dir/subdir2",
@@ -252,7 +252,7 @@ def test_walk_dirty_cached_dir(tmp_dir, scm, dvc):
tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}}, commit="add data")
(tmp_dir / "data" / "foo").unlink()
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
actual = []
for root, dirs, files in fs.walk("data"):
@@ -276,7 +276,7 @@ def test_walk_mixed_dir(tmp_dir, scm, dvc):
)
tmp_dir.scm.commit("add dir")
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
expected = [
"dir/foo",
@@ -293,7 +293,7 @@ def test_walk_mixed_dir(tmp_dir, scm, dvc):
def test_walk_missing(tmp_dir, dvc):
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
for _ in fs.walk("dir"):
pass
@@ -301,7 +301,7 @@ def test_walk_missing(tmp_dir, dvc):
def test_walk_not_a_dir(tmp_dir, dvc):
tmp_dir.dvc_gen("foo", "foo")
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
for _ in fs.walk("foo"):
pass
@@ -311,7 +311,7 @@ def test_isdvc(tmp_dir, dvc):
tmp_dir.gen({"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}})
dvc.add("foo")
dvc.add("dir")
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
assert fs.isdvc("foo")
assert not fs.isdvc("bar")
assert fs.isdvc("dir")
@@ -330,7 +330,7 @@ def make_subrepo(dir_, scm, config=None):
def test_subrepos(tmp_dir, scm, dvc, mocker):
tmp_dir.scm_gen(
{"dir": {"repo.txt": "file to confuse RepoFileSystem"}},
{"dir": {"repo.txt": "file to confuse DvcFileSystem"}},
commit="dir/repo.txt",
)
@@ -348,7 +348,7 @@ def test_subrepos(tmp_dir, scm, dvc, mocker):
)
dvc._reset()
fs = RepoFileSystem(repo=dvc, subrepos=True)
fs = DvcFileSystem(repo=dvc, subrepos=True)
def assert_fs_belongs_to_repo(ret_val):
method = fs.fs._get_repo
@@ -408,7 +408,7 @@ def test_subrepos(tmp_dir, scm, dvc, mocker):
)
def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
tmp_dir.scm_gen(
{"dir": {"repo.txt": "file to confuse RepoFileSystem"}},
{"dir": {"repo.txt": "file to confuse DvcFileSystem"}},
commit="dir/repo.txt",
)
@@ -428,7 +428,7 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
# using fs that does not have dvcignore
dvc._reset()
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
expected = [
"dir/repo",
"dir/repo.txt",
@@ -457,9 +457,9 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
assert len(actual) == len(expected)
def test_repo_fs_no_subrepos(tmp_dir, dvc, scm):
def test_dvcfs_no_subrepos(tmp_dir, dvc, scm):
tmp_dir.scm_gen(
{"dir": {"repo.txt": "file to confuse RepoFileSystem"}},
{"dir": {"repo.txt": "file to confuse DvcFileSystem"}},
commit="dir/repo.txt",
)
tmp_dir.dvc_gen({"lorem": "lorem"}, commit="add foo")
@@ -472,7 +472,7 @@ def test_repo_fs_no_subrepos(tmp_dir, dvc, scm):
# using fs that does not have dvcignore
dvc._reset()
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
expected = [
"/.dvcignore",
"/.gitignore",
@@ -504,7 +504,7 @@ def test_repo_fs_no_subrepos(tmp_dir, dvc, scm):
def test_get_hash_cached_file(tmp_dir, dvc, mocker):
tmp_dir.dvc_gen({"foo": "foo"})
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
expected = "acbd18db4cc2f85cedef654fccc4a4d8"
assert fs.info("foo").get("md5") is None
_, _, obj = stage(dvc.odb.local, "foo", fs, "md5")
@@ -517,7 +517,7 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker):
tmp_dir.dvc_gen(
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
)
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
expected = "8761c4e9acad696bee718615e23e22db.dir"
assert fs.info("dir").get("md5") is None
_, _, obj = stage(dvc.odb.local, "dir", fs, "md5")
@@ -537,7 +537,7 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
tmp_dir.dvc_gen(
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
)
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
subdir = "dir/subdir"
assert fs.info(subdir).get("md5") is None
_, _, obj = stage(dvc.odb.local, subdir, fs, "md5")
@@ -567,7 +567,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
tmp_dir.scm.commit("add dir")
clean_staging()
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
_, _, obj = stage(dvc.odb.local, "dir", fs, "md5")
assert obj.hash_info == HashInfo(
"md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir"
@@ -589,7 +589,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
# file is modified in workspace
# get_file_hash(file) should return workspace hash, not DVC cached hash
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
assert fs.info("file").get("md5") is None
staging, _, obj = stage(dvc.odb.local, "file", fs, "md5")
assert obj.hash_info == something_hash_info
@@ -617,7 +617,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
(tmp_dir / "dir" / "baz").write_text("baz")
clean_staging()
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
_, meta, obj = stage(dvc.odb.local, "dir", fs, "md5")
assert obj.hash_info == HashInfo(
"md5", "ba75a2162ca9c29acecb7957105a0bc2.dir"
@@ -679,7 +679,7 @@ def test_walk_nested_subrepos(tmp_dir, dvc, scm, traverse_subrepos):
expected["/subrepo1"].add("subrepo3")
actual = {}
fs = RepoFileSystem(repo=dvc)
fs = DvcFileSystem(repo=dvc)
for root, dirs, files in fs.walk(
"/", ignore_subrepos=not traverse_subrepos
):
@@ -2,12 +2,12 @@ import os
import pytest
from dvc.fs.repo import RepoFileSystem
from tests.unit.fs.test_repo import make_subrepo
from dvc.fs.dvc import DvcFileSystem
from tests.unit.fs.test_dvc import make_subrepo
@pytest.fixture
def repo_fs(tmp_dir, dvc, scm):
def dvcfs(tmp_dir, dvc, scm):
fs_structure = {
"models": { # mixed dvc + git directory
"train.py": "train dot py",
@@ -38,12 +38,12 @@ def repo_fs(tmp_dir, dvc, scm):
tmp_dir.scm_gen(fs_structure, commit="repo init")
tmp_dir.dvc_gen(dvc_structure, commit="use dvc")
yield RepoFileSystem(repo=dvc, subrepos=True)
yield DvcFileSystem(repo=dvc, subrepos=True)
def test_info_not_existing(repo_fs):
def test_info_not_existing(dvcfs):
with pytest.raises(FileNotFoundError):
repo_fs.info("path/that/does/not/exist")
dvcfs.info("path/that/does/not/exist")
@pytest.mark.parametrize(
@@ -56,10 +56,10 @@ def test_info_not_existing(repo_fs):
"src/utils/serve_model.py",
],
)
def test_info_git_tracked_file(repo_fs, path):
info = repo_fs.info(path)
def test_info_git_tracked_file(dvcfs, path):
info = dvcfs.info(path)
assert info["repo"].root_dir == repo_fs.repo.root_dir
assert info["repo"].root_dir == dvcfs.repo.root_dir
assert "dvc_info" not in info
assert info["type"] == "file"
assert not info["isexec"]
@@ -75,30 +75,30 @@ def test_info_git_tracked_file(repo_fs, path):
"models/transform.pickle",
],
)
def test_info_dvc_tracked_file(repo_fs, path):
info = repo_fs.info(path)
def test_info_dvc_tracked_file(dvcfs, path):
info = dvcfs.info(path)
assert info["repo"].root_dir == repo_fs.repo.root_dir
assert info["repo"].root_dir == dvcfs.repo.root_dir
assert info["dvc_info"]["isdvc"]
assert info["type"] == "file"
assert not info["isexec"]
@pytest.mark.parametrize("path", ["src", "src/utils"])
def test_info_git_only_dirs(repo_fs, path):
info = repo_fs.info(path)
def test_info_git_only_dirs(dvcfs, path):
info = dvcfs.info(path)
assert info["repo"].root_dir == repo_fs.repo.root_dir
assert info["repo"].root_dir == dvcfs.repo.root_dir
assert "dvc_info" not in info
assert info["type"] == "directory"
assert not info["isexec"]
@pytest.mark.parametrize("path", [".", "models"])
def test_info_git_dvc_mixed_dirs(repo_fs, path):
info = repo_fs.info(path)
def test_info_git_dvc_mixed_dirs(dvcfs, path):
info = dvcfs.info(path)
assert info["repo"].root_dir == repo_fs.repo.root_dir
assert info["repo"].root_dir == dvcfs.repo.root_dir
assert not info["dvc_info"]["isdvc"]
assert info["type"] == "directory"
assert not info["isexec"]
@@ -112,16 +112,16 @@ def test_info_git_dvc_mixed_dirs(repo_fs, path):
"data/processed",
],
)
def test_info_dvc_only_dirs(repo_fs, path):
info = repo_fs.info(path)
def test_info_dvc_only_dirs(dvcfs, path):
info = dvcfs.info(path)
assert info["repo"].root_dir == repo_fs.repo.root_dir
assert info["repo"].root_dir == dvcfs.repo.root_dir
assert info["dvc_info"]["isdvc"]
assert info["type"] == "directory"
assert not info["isexec"]
def test_info_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, repo_fs):
def test_info_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, dvcfs):
subrepo = tmp_dir / "subrepo"
make_subrepo(subrepo, scm)
with subrepo.chdir():
@@ -133,7 +133,7 @@ def test_info_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, repo_fs):
"subrepo/foo",
"subrepo/foobar",
]:
info = repo_fs.info(path)
info = dvcfs.info(path)
assert info["repo"].root_dir == str(
subrepo
), f"repo root didn't match for {path}"
+6 -6
View File
@@ -4,7 +4,7 @@ from unittest.mock import call
import pytest
from dvc.external_repo import external_repo
from tests.unit.fs.test_repo import make_subrepo
from tests.unit.fs.test_dvc import make_subrepo
def test_hook_is_called(tmp_dir, erepo_dir, mocker):
@@ -26,9 +26,9 @@ def test_hook_is_called(tmp_dir, erepo_dir, mocker):
repo.dvc_gen("bar", "bar", commit=f"dvc add {repo}/bar")
with external_repo(str(erepo_dir)) as repo:
spy = mocker.spy(repo.repo_fs.fs, "repo_factory")
spy = mocker.spy(repo.dvcfs.fs, "repo_factory")
list(repo.repo_fs.walk("", ignore_subrepos=False)) # drain
list(repo.dvcfs.walk("", ignore_subrepos=False)) # drain
assert spy.call_count == len(subrepos)
paths = ["/" + path.replace("\\", "/") for path in subrepo_paths]
@@ -38,7 +38,7 @@ def test_hook_is_called(tmp_dir, erepo_dir, mocker):
path,
fs=repo.fs,
scm=repo.scm,
repo_factory=repo.repo_fs.fs.repo_factory,
repo_factory=repo.dvcfs.fs.repo_factory,
)
for path in paths
],
@@ -64,9 +64,9 @@ def test_subrepo_is_constructed_properly(
with external_repo(
str(tmp_dir), cache_dir=str(cache_dir), cache_types=["symlink"]
) as repo:
spy = mocker.spy(repo.repo_fs.fs, "repo_factory")
spy = mocker.spy(repo.dvcfs.fs, "repo_factory")
list(repo.repo_fs.walk("", ignore_subrepos=False)) # drain
list(repo.dvcfs.walk("", ignore_subrepos=False)) # drain
assert spy.call_count == 1
subrepo = spy.spy_return