mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
fs: rename RepoFileSystem to DvcFileSystem
This commit is contained in:
+2
-2
@@ -19,9 +19,9 @@ def get_url(path, repo=None, rev=None, remote=None):
|
||||
directory in the remote storage.
|
||||
"""
|
||||
with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
|
||||
fs_path = _repo.repo_fs.from_os_path(path)
|
||||
fs_path = _repo.dvcfs.from_os_path(path)
|
||||
with reraise(FileNotFoundError, PathMissingError(path, repo)):
|
||||
info = _repo.repo_fs.info(fs_path)
|
||||
info = _repo.dvcfs.info(fs_path)
|
||||
|
||||
dvc_info = info.get("dvc_info")
|
||||
if not dvc_info:
|
||||
|
||||
@@ -71,7 +71,7 @@ class ReferenceHashFile(HashFile):
|
||||
@classmethod
|
||||
def from_bytes(cls, data: bytes, fs_cache: Optional[dict] = None):
|
||||
from dvc.fs import get_fs_cls
|
||||
from dvc.fs.repo import RepoFileSystem, _RepoFileSystem
|
||||
from dvc.fs.dvc import DvcFileSystem, _DvcFileSystem
|
||||
|
||||
try:
|
||||
dict_ = pickle.loads(data)
|
||||
@@ -88,8 +88,8 @@ class ReferenceHashFile(HashFile):
|
||||
fs = fs_cache.get((scheme, config_pairs)) if fs_cache else None
|
||||
if not fs:
|
||||
config = dict(config_pairs)
|
||||
if _RepoFileSystem.PARAM_REPO_URL in config:
|
||||
fs_cls = RepoFileSystem
|
||||
if _DvcFileSystem.PARAM_REPO_URL in config:
|
||||
fs_cls = DvcFileSystem
|
||||
else:
|
||||
fs_cls = get_fs_cls(config, scheme=scheme)
|
||||
fs = fs_cls(**config)
|
||||
|
||||
@@ -133,7 +133,7 @@ class RepoDependency(Dependency):
|
||||
staging, _, staged_obj = stage(
|
||||
local_odb,
|
||||
as_posix(self.def_path),
|
||||
repo.repo_fs,
|
||||
repo.dvcfs,
|
||||
local_odb.fs.PARAM_CHECKSUM,
|
||||
)
|
||||
except (FileNotFoundError, TreeError) as exc:
|
||||
@@ -153,7 +153,7 @@ class RepoDependency(Dependency):
|
||||
from dvc.data.db.reference import ReferenceObjectDB
|
||||
from dvc.data.tree import Tree
|
||||
from dvc.exceptions import CircularImportError
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
|
||||
if not isinstance(odb, ReferenceObjectDB):
|
||||
return
|
||||
@@ -168,7 +168,7 @@ class RepoDependency(Dependency):
|
||||
|
||||
checked_urls = set()
|
||||
for obj in iter_objs():
|
||||
if not isinstance(obj.fs, RepoFileSystem):
|
||||
if not isinstance(obj.fs, DvcFileSystem):
|
||||
continue
|
||||
if (
|
||||
obj.fs.repo_url in checked_urls
|
||||
|
||||
@@ -63,7 +63,7 @@ def _merge_info(repo, fs_info, dvc_info):
|
||||
return ret
|
||||
|
||||
|
||||
class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
class _DvcFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
"""DVC + git-tracked files fs.
|
||||
|
||||
Args:
|
||||
@@ -436,7 +436,7 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
return dvc_fs.checksum(dvc_path)
|
||||
|
||||
|
||||
class RepoFileSystem(FileSystem):
|
||||
class DvcFileSystem(FileSystem):
|
||||
scheme = "local"
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
@@ -446,7 +446,7 @@ class RepoFileSystem(FileSystem):
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def fs(self):
|
||||
return _RepoFileSystem(**self.fs_args)
|
||||
return _DvcFileSystem(**self.fs_args)
|
||||
|
||||
def isdvc(self, path, **kwargs):
|
||||
return self.fs.isdvc(path, **kwargs)
|
||||
@@ -188,7 +188,7 @@ class Repo:
|
||||
self.config = Config(self.dvc_dir, fs=self.fs, config=config)
|
||||
self._uninitialized = uninitialized
|
||||
|
||||
# used by RepoFileSystem to determine if it should traverse subrepos
|
||||
# used by DvcFileSystem to determine if it should traverse subrepos
|
||||
self.subrepos = subrepos
|
||||
|
||||
self.cloud = DataCloud(self)
|
||||
@@ -490,10 +490,10 @@ class Repo:
|
||||
return DataFileSystem(repo=self)
|
||||
|
||||
@cached_property
|
||||
def repo_fs(self):
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
def dvcfs(self):
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
|
||||
return RepoFileSystem(
|
||||
return DvcFileSystem(
|
||||
repo=self, subrepos=self.subrepos, **self._fs_conf
|
||||
)
|
||||
|
||||
@@ -505,13 +505,13 @@ class Repo:
|
||||
def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
|
||||
"""Opens a specified resource as a file descriptor"""
|
||||
from dvc.fs.data import DataFileSystem
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
|
||||
if os.path.isabs(path):
|
||||
fs = DataFileSystem(repo=self, workspace="local")
|
||||
fs_path = path
|
||||
else:
|
||||
fs = RepoFileSystem(repo=self, subrepos=True)
|
||||
fs = DvcFileSystem(repo=self, subrepos=True)
|
||||
fs_path = fs.from_os_path(path)
|
||||
|
||||
try:
|
||||
|
||||
+3
-3
@@ -30,9 +30,9 @@ def _collect_paths(
|
||||
recursive: bool = False,
|
||||
rev: str = None,
|
||||
) -> StrPaths:
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
|
||||
fs = RepoFileSystem(repo=repo)
|
||||
fs = DvcFileSystem(repo=repo)
|
||||
fs_paths = [fs.from_os_path(target) for target in targets]
|
||||
|
||||
target_paths: StrPaths = []
|
||||
@@ -58,7 +58,7 @@ def _filter_duplicates(
|
||||
fs_res_paths = fs_paths
|
||||
|
||||
for out in outs:
|
||||
fs_path = out.repo.repo_fs.from_os_path(out.fs_path)
|
||||
fs_path = out.repo.dvcfs.from_os_path(out.fs_path)
|
||||
if fs_path in fs_paths:
|
||||
res_outs.append(out)
|
||||
# MUTATING THE SAME LIST!!
|
||||
|
||||
+10
-10
@@ -22,9 +22,9 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
|
||||
if self.scm.no_commits:
|
||||
return {}
|
||||
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
|
||||
repo_fs = RepoFileSystem(repo=self)
|
||||
dvcfs = DvcFileSystem(repo=self)
|
||||
|
||||
b_rev = b_rev if b_rev else "workspace"
|
||||
results = {}
|
||||
@@ -39,7 +39,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
|
||||
if targets is not None:
|
||||
# convert targets to paths, and capture any missing targets
|
||||
targets_paths, missing_targets[rev] = _targets_to_paths(
|
||||
repo_fs, targets
|
||||
dvcfs, targets
|
||||
)
|
||||
|
||||
results[rev] = _paths_checksums(self, targets_paths)
|
||||
@@ -61,7 +61,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
|
||||
if b_rev == "workspace":
|
||||
# missing status is only applicable when diffing local workspace
|
||||
# against a commit
|
||||
missing = sorted(_filter_missing(repo_fs, deleted_or_missing))
|
||||
missing = sorted(_filter_missing(dvcfs, deleted_or_missing))
|
||||
else:
|
||||
missing = []
|
||||
deleted = sorted(deleted_or_missing - set(missing))
|
||||
@@ -178,11 +178,11 @@ def _dir_output_paths(fs, fs_path, obj, targets=None):
|
||||
yield os.path.join(base, *key), oid.value
|
||||
|
||||
|
||||
def _filter_missing(repo_fs, paths):
|
||||
def _filter_missing(dvcfs, paths):
|
||||
for path in paths:
|
||||
fs_path = repo_fs.from_os_path(path)
|
||||
fs_path = dvcfs.from_os_path(path)
|
||||
try:
|
||||
info = repo_fs.info(fs_path)
|
||||
info = dvcfs.info(fs_path)
|
||||
dvc_info = info.get("dvc_info")
|
||||
if (
|
||||
dvc_info
|
||||
@@ -194,13 +194,13 @@ def _filter_missing(repo_fs, paths):
|
||||
pass
|
||||
|
||||
|
||||
def _targets_to_paths(repo_fs, targets):
|
||||
def _targets_to_paths(dvcfs, targets):
|
||||
paths = []
|
||||
missing = []
|
||||
|
||||
for target in targets:
|
||||
if repo_fs.exists(target):
|
||||
paths.append(repo_fs.repo.fs.path.abspath(target))
|
||||
if dvcfs.exists(target):
|
||||
paths.append(dvcfs.repo.fs.path.abspath(target))
|
||||
else:
|
||||
missing.append(target)
|
||||
|
||||
|
||||
+1
-1
@@ -57,7 +57,7 @@ def get(url, path, out=None, rev=None, jobs=None):
|
||||
fs = DataFileSystem(repo=repo, workspace="local")
|
||||
fs_path = path
|
||||
else:
|
||||
fs = repo.repo_fs
|
||||
fs = repo.dvcfs
|
||||
fs_path = fs.from_os_path(path)
|
||||
|
||||
with FsspecCallback.as_tqdm_callback(
|
||||
|
||||
+2
-2
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Optional
|
||||
from dvc.exceptions import PathMissingError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
|
||||
from . import Repo
|
||||
|
||||
@@ -56,7 +56,7 @@ def ls(
|
||||
def _ls(
|
||||
repo: "Repo", path: str, recursive: bool = None, dvc_only: bool = False
|
||||
):
|
||||
fs: "RepoFileSystem" = repo.repo_fs
|
||||
fs: "DvcFileSystem" = repo.dvcfs
|
||||
fs_path = fs.from_os_path(path)
|
||||
|
||||
try:
|
||||
|
||||
@@ -4,7 +4,7 @@ from typing import List
|
||||
|
||||
from scmrepo.exceptions import SCMError
|
||||
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
from dvc.output import Output
|
||||
from dvc.repo import locked
|
||||
from dvc.repo.collect import StrPaths, collect
|
||||
@@ -24,11 +24,11 @@ def _to_fs_paths(metrics: List[Output]) -> StrPaths:
|
||||
result = []
|
||||
for out in metrics:
|
||||
if out.metric:
|
||||
result.append(out.repo.repo_fs.from_os_path(out.fs_path))
|
||||
result.append(out.repo.dvcfs.from_os_path(out.fs_path))
|
||||
elif out.live:
|
||||
fs_path = summary_fs_path(out)
|
||||
if fs_path:
|
||||
result.append(out.repo.repo_fs.from_os_path(fs_path))
|
||||
result.append(out.repo.dvcfs.from_os_path(fs_path))
|
||||
return result
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ def _read_metric(path, fs, rev, **kwargs):
|
||||
|
||||
|
||||
def _read_metrics(repo, metrics, rev, onerror=None):
|
||||
fs = RepoFileSystem(repo=repo)
|
||||
fs = DvcFileSystem(repo=repo)
|
||||
|
||||
relpath = ""
|
||||
if repo.root_dir != repo.fs.path.getcwd():
|
||||
|
||||
@@ -99,9 +99,9 @@ class Plots:
|
||||
onerror: Optional[Callable] = None,
|
||||
props: Optional[Dict] = None,
|
||||
):
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
|
||||
fs = RepoFileSystem(repo=self.repo)
|
||||
fs = DvcFileSystem(repo=self.repo)
|
||||
plots = _collect_plots(self.repo, targets, revision, recursive)
|
||||
res: Dict[str, Any] = {}
|
||||
for fs_path, rev_props in plots.items():
|
||||
@@ -253,7 +253,7 @@ def _collect_plots(
|
||||
)
|
||||
|
||||
result = {
|
||||
repo.repo_fs.from_os_path(plot.fs_path): _plot_props(plot)
|
||||
repo.dvcfs.from_os_path(plot.fs_path): _plot_props(plot)
|
||||
for plot in plots
|
||||
}
|
||||
result.update({fs_path: {} for fs_path in fs_paths})
|
||||
|
||||
@@ -63,7 +63,7 @@ def _get_stage_files(stage: "Stage") -> typing.Iterator[str]:
|
||||
if (
|
||||
not dep.use_scm_ignore
|
||||
and dep.is_in_repo
|
||||
and not stage.repo.repo_fs.isdvc(dep.fs_path)
|
||||
and not stage.repo.dvcfs.isdvc(dep.fs_path)
|
||||
):
|
||||
yield dep.fs_path
|
||||
for out in stage.outs:
|
||||
|
||||
@@ -31,7 +31,7 @@ def test_new_checkpoint(
|
||||
for rev in dvc.brancher([exp]):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
fs = dvc.repo_fs
|
||||
fs = dvc.dvcfs
|
||||
with fs.open("foo") as fobj:
|
||||
assert fobj.read().strip() == str(checkpoint_stage.iterations)
|
||||
with fs.open("metrics.yaml") as fobj:
|
||||
@@ -80,7 +80,7 @@ def test_resume_checkpoint(
|
||||
for rev in dvc.brancher([exp]):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
fs = dvc.repo_fs
|
||||
fs = dvc.dvcfs
|
||||
with fs.open("foo") as fobj:
|
||||
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
|
||||
with fs.open("metrics.yaml") as fobj:
|
||||
@@ -111,7 +111,7 @@ def test_reset_checkpoint(
|
||||
for rev in dvc.brancher([exp]):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
fs = dvc.repo_fs
|
||||
fs = dvc.dvcfs
|
||||
with fs.open("foo") as fobj:
|
||||
assert fobj.read().strip() == str(checkpoint_stage.iterations)
|
||||
with fs.open("metrics.yaml") as fobj:
|
||||
@@ -150,7 +150,7 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage, workspace):
|
||||
for rev in dvc.brancher([checkpoint_a]):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
fs = dvc.repo_fs
|
||||
fs = dvc.dvcfs
|
||||
with fs.open("foo") as fobj:
|
||||
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
|
||||
with fs.open("metrics.yaml") as fobj:
|
||||
@@ -159,7 +159,7 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage, workspace):
|
||||
for rev in dvc.brancher([checkpoint_b]):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
fs = dvc.repo_fs
|
||||
fs = dvc.dvcfs
|
||||
with fs.open("foo") as fobj:
|
||||
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
|
||||
with fs.open("metrics.yaml") as fobj:
|
||||
|
||||
@@ -525,7 +525,7 @@ def test_subdir(tmp_dir, scm, dvc, workspace):
|
||||
|
||||
@pytest.mark.parametrize("workspace", [True, False])
|
||||
def test_subrepo(tmp_dir, scm, workspace):
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
from tests.unit.fs.test_dvc import make_subrepo
|
||||
|
||||
subrepo = tmp_dir / "dir" / "repo"
|
||||
make_subrepo(subrepo, scm)
|
||||
@@ -673,9 +673,9 @@ def test_modified_data_dep(tmp_dir, scm, dvc, workspace, params, target):
|
||||
for rev in dvc.brancher(revs=[exp]):
|
||||
if rev != exp:
|
||||
continue
|
||||
with dvc.repo_fs.open("metrics.yaml") as fobj:
|
||||
with dvc.dvcfs.open("metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == params
|
||||
with dvc.repo_fs.open("data") as fobj:
|
||||
with dvc.dvcfs.open("data") as fobj:
|
||||
assert fobj.read().strip() == "modified"
|
||||
|
||||
if workspace:
|
||||
|
||||
@@ -11,7 +11,7 @@ from dvc.exceptions import (
|
||||
)
|
||||
from dvc.testing.test_api import TestAPI # noqa, pylint: disable=unused-import
|
||||
from dvc.utils.fs import remove
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
from tests.unit.fs.test_dvc import make_subrepo
|
||||
|
||||
|
||||
def test_get_url_external(tmp_dir, erepo_dir, cloud):
|
||||
|
||||
@@ -8,7 +8,7 @@ from dvc.data.transfer import transfer
|
||||
from dvc.external_repo import CLONES, external_repo
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import makedirs, remove
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
from tests.unit.fs.test_dvc import make_subrepo
|
||||
from tests.utils import clean_staging
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
|
||||
|
||||
dest = tmp_dir / "file"
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
repo.repo_fs.get(
|
||||
repo.dvcfs.get(
|
||||
"subdir/file",
|
||||
os.fspath(dest),
|
||||
)
|
||||
@@ -191,7 +191,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
|
||||
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
|
||||
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
repo.repo_fs.get(
|
||||
repo.dvcfs.get(
|
||||
"dir",
|
||||
os.fspath(tmp_dir / "out"),
|
||||
)
|
||||
@@ -207,7 +207,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
|
||||
staging, _, obj = stage(
|
||||
repo.odb.local,
|
||||
"dir",
|
||||
repo.repo_fs,
|
||||
repo.dvcfs,
|
||||
"md5",
|
||||
dvcignore=repo.dvcignore,
|
||||
)
|
||||
@@ -237,7 +237,7 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
|
||||
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
|
||||
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
repo.repo_fs.get(
|
||||
repo.dvcfs.get(
|
||||
"dir",
|
||||
os.fspath(tmp_dir / "out"),
|
||||
)
|
||||
|
||||
@@ -273,11 +273,11 @@ def test_get_dir_callback(tmp_dir, dvc, cloud):
|
||||
assert (tmp_dir / "dir").read_text() == {"foo": "foo", "bar": "bar"}
|
||||
|
||||
|
||||
def test_callback_on_repo_fs(tmp_dir, dvc, scm, mocker):
|
||||
def test_callback_on_dvcfs(tmp_dir, dvc, scm, mocker):
|
||||
tmp_dir.dvc_gen({"dir": {"bar": "bar"}}, commit="dvc")
|
||||
tmp_dir.scm_gen({"dir": {"foo": "foo"}}, commit="git")
|
||||
|
||||
fs = dvc.repo_fs
|
||||
fs = dvc.dvcfs
|
||||
|
||||
callback = FsspecCallback()
|
||||
fs.get(
|
||||
|
||||
@@ -9,7 +9,7 @@ from dvc.repo import Repo
|
||||
from dvc.repo.get import GetDVCFileError
|
||||
from dvc.system import System
|
||||
from dvc.utils.fs import makedirs
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
from tests.unit.fs.test_dvc import make_subrepo
|
||||
|
||||
|
||||
def test_get_repo_file(tmp_dir, erepo_dir):
|
||||
|
||||
@@ -13,7 +13,7 @@ from dvc.exceptions import DownloadError, PathMissingError
|
||||
from dvc.stage.exceptions import StagePathNotFoundError
|
||||
from dvc.system import System
|
||||
from dvc.utils.fs import makedirs, remove
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
from tests.unit.fs.test_dvc import make_subrepo
|
||||
from tests.utils import clean_staging
|
||||
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import pytest
|
||||
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
from tests.unit.fs.test_dvc import make_subrepo
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cached", [True, False])
|
||||
|
||||
@@ -50,7 +50,7 @@ def test_open_dirty_hash(tmp_dir, dvc):
|
||||
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
with fs.open("file", "r") as fobj:
|
||||
# NOTE: Unlike RepoFileSystem, DataFileSystem should not
|
||||
# NOTE: Unlike DvcFileSystem, DataFileSystem should not
|
||||
# be affected by a dirty workspace.
|
||||
assert fobj.read() == "file"
|
||||
|
||||
@@ -72,7 +72,7 @@ def test_open_dirty_no_hash(tmp_dir, dvc):
|
||||
(tmp_dir / "file.dvc").write_text("outs:\n- path: file\n")
|
||||
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
# NOTE: Unlike RepoFileSystem, DataFileSystem should not
|
||||
# NOTE: Unlike DvcFileSystem, DataFileSystem should not
|
||||
# be affected by a dirty workspace.
|
||||
with pytest.raises(FileNotFoundError):
|
||||
with fs.open("file", "r"):
|
||||
|
||||
@@ -6,7 +6,7 @@ from unittest import mock
|
||||
import pytest
|
||||
|
||||
from dvc.data.stage import stage
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
from dvc.objects.hash_info import HashInfo
|
||||
from tests.utils import clean_staging
|
||||
|
||||
@@ -16,7 +16,7 @@ def test_exists(tmp_dir, dvc):
|
||||
dvc.add("foo")
|
||||
(tmp_dir / "foo").unlink()
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
assert fs.exists("foo")
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ def test_open(tmp_dir, dvc):
|
||||
dvc.add("foo")
|
||||
(tmp_dir / "foo").unlink()
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
with fs.open("foo", "r") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
|
||||
@@ -34,7 +34,7 @@ def test_open_dirty_hash(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("file", "file")
|
||||
(tmp_dir / "file").write_text("something")
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
with fs.open("file", "r") as fobj:
|
||||
assert fobj.read() == "something"
|
||||
|
||||
@@ -43,7 +43,7 @@ def test_open_dirty_no_hash(tmp_dir, dvc):
|
||||
tmp_dir.gen("file", "file")
|
||||
(tmp_dir / "file.dvc").write_text("outs:\n- path: file\n")
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
with fs.open("file", "r") as fobj:
|
||||
assert fobj.read() == "file"
|
||||
|
||||
@@ -63,7 +63,7 @@ def test_open_in_history(tmp_dir, scm, dvc):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
with fs.open("foo", "r") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
|
||||
@@ -86,7 +86,7 @@ def test_isdir_isfile(tmp_dir, dvc):
|
||||
},
|
||||
)
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
assert fs.isdir("datadir")
|
||||
assert not fs.isfile("datadir")
|
||||
assert not fs.isdvc("datadir")
|
||||
@@ -126,7 +126,7 @@ def test_exists_isdir_isfile_dirty(tmp_dir, dvc):
|
||||
{"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}}
|
||||
)
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
shutil.rmtree(tmp_dir / "datadir")
|
||||
(tmp_dir / "datafile").unlink()
|
||||
|
||||
@@ -161,7 +161,7 @@ def test_isdir_mixed(tmp_dir, dvc):
|
||||
|
||||
dvc.add(str(tmp_dir / "dir" / "foo"))
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
assert fs.isdir("dir")
|
||||
assert not fs.isfile("dir")
|
||||
|
||||
@@ -191,7 +191,7 @@ def test_walk(tmp_dir, dvc, dvcfiles, extra_expected):
|
||||
)
|
||||
dvc.add(str(tmp_dir / "dir"), recursive=True)
|
||||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
|
||||
expected = [
|
||||
"dir/subdir1",
|
||||
@@ -226,7 +226,7 @@ def test_walk_dirty(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"bar": "bar", "subdir3": {"foo3": "foo3"}}})
|
||||
(tmp_dir / "dir" / "foo").unlink()
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
expected = [
|
||||
"dir/subdir1",
|
||||
"dir/subdir2",
|
||||
@@ -252,7 +252,7 @@ def test_walk_dirty_cached_dir(tmp_dir, scm, dvc):
|
||||
tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}}, commit="add data")
|
||||
(tmp_dir / "data" / "foo").unlink()
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
|
||||
actual = []
|
||||
for root, dirs, files in fs.walk("data"):
|
||||
@@ -276,7 +276,7 @@ def test_walk_mixed_dir(tmp_dir, scm, dvc):
|
||||
)
|
||||
tmp_dir.scm.commit("add dir")
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
|
||||
expected = [
|
||||
"dir/foo",
|
||||
@@ -293,7 +293,7 @@ def test_walk_mixed_dir(tmp_dir, scm, dvc):
|
||||
|
||||
|
||||
def test_walk_missing(tmp_dir, dvc):
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
|
||||
for _ in fs.walk("dir"):
|
||||
pass
|
||||
@@ -301,7 +301,7 @@ def test_walk_missing(tmp_dir, dvc):
|
||||
|
||||
def test_walk_not_a_dir(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
|
||||
for _ in fs.walk("foo"):
|
||||
pass
|
||||
@@ -311,7 +311,7 @@ def test_isdvc(tmp_dir, dvc):
|
||||
tmp_dir.gen({"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}})
|
||||
dvc.add("foo")
|
||||
dvc.add("dir")
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
assert fs.isdvc("foo")
|
||||
assert not fs.isdvc("bar")
|
||||
assert fs.isdvc("dir")
|
||||
@@ -330,7 +330,7 @@ def make_subrepo(dir_, scm, config=None):
|
||||
|
||||
def test_subrepos(tmp_dir, scm, dvc, mocker):
|
||||
tmp_dir.scm_gen(
|
||||
{"dir": {"repo.txt": "file to confuse RepoFileSystem"}},
|
||||
{"dir": {"repo.txt": "file to confuse DvcFileSystem"}},
|
||||
commit="dir/repo.txt",
|
||||
)
|
||||
|
||||
@@ -348,7 +348,7 @@ def test_subrepos(tmp_dir, scm, dvc, mocker):
|
||||
)
|
||||
|
||||
dvc._reset()
|
||||
fs = RepoFileSystem(repo=dvc, subrepos=True)
|
||||
fs = DvcFileSystem(repo=dvc, subrepos=True)
|
||||
|
||||
def assert_fs_belongs_to_repo(ret_val):
|
||||
method = fs.fs._get_repo
|
||||
@@ -408,7 +408,7 @@ def test_subrepos(tmp_dir, scm, dvc, mocker):
|
||||
)
|
||||
def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
|
||||
tmp_dir.scm_gen(
|
||||
{"dir": {"repo.txt": "file to confuse RepoFileSystem"}},
|
||||
{"dir": {"repo.txt": "file to confuse DvcFileSystem"}},
|
||||
commit="dir/repo.txt",
|
||||
)
|
||||
|
||||
@@ -428,7 +428,7 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
|
||||
|
||||
# using fs that does not have dvcignore
|
||||
dvc._reset()
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
expected = [
|
||||
"dir/repo",
|
||||
"dir/repo.txt",
|
||||
@@ -457,9 +457,9 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
|
||||
assert len(actual) == len(expected)
|
||||
|
||||
|
||||
def test_repo_fs_no_subrepos(tmp_dir, dvc, scm):
|
||||
def test_dvcfs_no_subrepos(tmp_dir, dvc, scm):
|
||||
tmp_dir.scm_gen(
|
||||
{"dir": {"repo.txt": "file to confuse RepoFileSystem"}},
|
||||
{"dir": {"repo.txt": "file to confuse DvcFileSystem"}},
|
||||
commit="dir/repo.txt",
|
||||
)
|
||||
tmp_dir.dvc_gen({"lorem": "lorem"}, commit="add foo")
|
||||
@@ -472,7 +472,7 @@ def test_repo_fs_no_subrepos(tmp_dir, dvc, scm):
|
||||
|
||||
# using fs that does not have dvcignore
|
||||
dvc._reset()
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
expected = [
|
||||
"/.dvcignore",
|
||||
"/.gitignore",
|
||||
@@ -504,7 +504,7 @@ def test_repo_fs_no_subrepos(tmp_dir, dvc, scm):
|
||||
|
||||
def test_get_hash_cached_file(tmp_dir, dvc, mocker):
|
||||
tmp_dir.dvc_gen({"foo": "foo"})
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
expected = "acbd18db4cc2f85cedef654fccc4a4d8"
|
||||
assert fs.info("foo").get("md5") is None
|
||||
_, _, obj = stage(dvc.odb.local, "foo", fs, "md5")
|
||||
@@ -517,7 +517,7 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker):
|
||||
tmp_dir.dvc_gen(
|
||||
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
|
||||
)
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
expected = "8761c4e9acad696bee718615e23e22db.dir"
|
||||
assert fs.info("dir").get("md5") is None
|
||||
_, _, obj = stage(dvc.odb.local, "dir", fs, "md5")
|
||||
@@ -537,7 +537,7 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
|
||||
tmp_dir.dvc_gen(
|
||||
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
|
||||
)
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
subdir = "dir/subdir"
|
||||
assert fs.info(subdir).get("md5") is None
|
||||
_, _, obj = stage(dvc.odb.local, subdir, fs, "md5")
|
||||
@@ -567,7 +567,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
|
||||
tmp_dir.scm.commit("add dir")
|
||||
clean_staging()
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
_, _, obj = stage(dvc.odb.local, "dir", fs, "md5")
|
||||
assert obj.hash_info == HashInfo(
|
||||
"md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir"
|
||||
@@ -589,7 +589,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
|
||||
|
||||
# file is modified in workspace
|
||||
# get_file_hash(file) should return workspace hash, not DVC cached hash
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
assert fs.info("file").get("md5") is None
|
||||
staging, _, obj = stage(dvc.odb.local, "file", fs, "md5")
|
||||
assert obj.hash_info == something_hash_info
|
||||
@@ -617,7 +617,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
|
||||
(tmp_dir / "dir" / "baz").write_text("baz")
|
||||
clean_staging()
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
_, meta, obj = stage(dvc.odb.local, "dir", fs, "md5")
|
||||
assert obj.hash_info == HashInfo(
|
||||
"md5", "ba75a2162ca9c29acecb7957105a0bc2.dir"
|
||||
@@ -679,7 +679,7 @@ def test_walk_nested_subrepos(tmp_dir, dvc, scm, traverse_subrepos):
|
||||
expected["/subrepo1"].add("subrepo3")
|
||||
|
||||
actual = {}
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
for root, dirs, files in fs.walk(
|
||||
"/", ignore_subrepos=not traverse_subrepos
|
||||
):
|
||||
@@ -2,12 +2,12 @@ import os
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
from tests.unit.fs.test_dvc import make_subrepo
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repo_fs(tmp_dir, dvc, scm):
|
||||
def dvcfs(tmp_dir, dvc, scm):
|
||||
fs_structure = {
|
||||
"models": { # mixed dvc + git directory
|
||||
"train.py": "train dot py",
|
||||
@@ -38,12 +38,12 @@ def repo_fs(tmp_dir, dvc, scm):
|
||||
tmp_dir.scm_gen(fs_structure, commit="repo init")
|
||||
tmp_dir.dvc_gen(dvc_structure, commit="use dvc")
|
||||
|
||||
yield RepoFileSystem(repo=dvc, subrepos=True)
|
||||
yield DvcFileSystem(repo=dvc, subrepos=True)
|
||||
|
||||
|
||||
def test_info_not_existing(repo_fs):
|
||||
def test_info_not_existing(dvcfs):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
repo_fs.info("path/that/does/not/exist")
|
||||
dvcfs.info("path/that/does/not/exist")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -56,10 +56,10 @@ def test_info_not_existing(repo_fs):
|
||||
"src/utils/serve_model.py",
|
||||
],
|
||||
)
|
||||
def test_info_git_tracked_file(repo_fs, path):
|
||||
info = repo_fs.info(path)
|
||||
def test_info_git_tracked_file(dvcfs, path):
|
||||
info = dvcfs.info(path)
|
||||
|
||||
assert info["repo"].root_dir == repo_fs.repo.root_dir
|
||||
assert info["repo"].root_dir == dvcfs.repo.root_dir
|
||||
assert "dvc_info" not in info
|
||||
assert info["type"] == "file"
|
||||
assert not info["isexec"]
|
||||
@@ -75,30 +75,30 @@ def test_info_git_tracked_file(repo_fs, path):
|
||||
"models/transform.pickle",
|
||||
],
|
||||
)
|
||||
def test_info_dvc_tracked_file(repo_fs, path):
|
||||
info = repo_fs.info(path)
|
||||
def test_info_dvc_tracked_file(dvcfs, path):
|
||||
info = dvcfs.info(path)
|
||||
|
||||
assert info["repo"].root_dir == repo_fs.repo.root_dir
|
||||
assert info["repo"].root_dir == dvcfs.repo.root_dir
|
||||
assert info["dvc_info"]["isdvc"]
|
||||
assert info["type"] == "file"
|
||||
assert not info["isexec"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["src", "src/utils"])
|
||||
def test_info_git_only_dirs(repo_fs, path):
|
||||
info = repo_fs.info(path)
|
||||
def test_info_git_only_dirs(dvcfs, path):
|
||||
info = dvcfs.info(path)
|
||||
|
||||
assert info["repo"].root_dir == repo_fs.repo.root_dir
|
||||
assert info["repo"].root_dir == dvcfs.repo.root_dir
|
||||
assert "dvc_info" not in info
|
||||
assert info["type"] == "directory"
|
||||
assert not info["isexec"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", [".", "models"])
|
||||
def test_info_git_dvc_mixed_dirs(repo_fs, path):
|
||||
info = repo_fs.info(path)
|
||||
def test_info_git_dvc_mixed_dirs(dvcfs, path):
|
||||
info = dvcfs.info(path)
|
||||
|
||||
assert info["repo"].root_dir == repo_fs.repo.root_dir
|
||||
assert info["repo"].root_dir == dvcfs.repo.root_dir
|
||||
assert not info["dvc_info"]["isdvc"]
|
||||
assert info["type"] == "directory"
|
||||
assert not info["isexec"]
|
||||
@@ -112,16 +112,16 @@ def test_info_git_dvc_mixed_dirs(repo_fs, path):
|
||||
"data/processed",
|
||||
],
|
||||
)
|
||||
def test_info_dvc_only_dirs(repo_fs, path):
|
||||
info = repo_fs.info(path)
|
||||
def test_info_dvc_only_dirs(dvcfs, path):
|
||||
info = dvcfs.info(path)
|
||||
|
||||
assert info["repo"].root_dir == repo_fs.repo.root_dir
|
||||
assert info["repo"].root_dir == dvcfs.repo.root_dir
|
||||
assert info["dvc_info"]["isdvc"]
|
||||
assert info["type"] == "directory"
|
||||
assert not info["isexec"]
|
||||
|
||||
|
||||
def test_info_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, repo_fs):
|
||||
def test_info_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, dvcfs):
|
||||
subrepo = tmp_dir / "subrepo"
|
||||
make_subrepo(subrepo, scm)
|
||||
with subrepo.chdir():
|
||||
@@ -133,7 +133,7 @@ def test_info_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, repo_fs):
|
||||
"subrepo/foo",
|
||||
"subrepo/foobar",
|
||||
]:
|
||||
info = repo_fs.info(path)
|
||||
info = dvcfs.info(path)
|
||||
assert info["repo"].root_dir == str(
|
||||
subrepo
|
||||
), f"repo root didn't match for {path}"
|
||||
@@ -4,7 +4,7 @@ from unittest.mock import call
|
||||
import pytest
|
||||
|
||||
from dvc.external_repo import external_repo
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
from tests.unit.fs.test_dvc import make_subrepo
|
||||
|
||||
|
||||
def test_hook_is_called(tmp_dir, erepo_dir, mocker):
|
||||
@@ -26,9 +26,9 @@ def test_hook_is_called(tmp_dir, erepo_dir, mocker):
|
||||
repo.dvc_gen("bar", "bar", commit=f"dvc add {repo}/bar")
|
||||
|
||||
with external_repo(str(erepo_dir)) as repo:
|
||||
spy = mocker.spy(repo.repo_fs.fs, "repo_factory")
|
||||
spy = mocker.spy(repo.dvcfs.fs, "repo_factory")
|
||||
|
||||
list(repo.repo_fs.walk("", ignore_subrepos=False)) # drain
|
||||
list(repo.dvcfs.walk("", ignore_subrepos=False)) # drain
|
||||
assert spy.call_count == len(subrepos)
|
||||
|
||||
paths = ["/" + path.replace("\\", "/") for path in subrepo_paths]
|
||||
@@ -38,7 +38,7 @@ def test_hook_is_called(tmp_dir, erepo_dir, mocker):
|
||||
path,
|
||||
fs=repo.fs,
|
||||
scm=repo.scm,
|
||||
repo_factory=repo.repo_fs.fs.repo_factory,
|
||||
repo_factory=repo.dvcfs.fs.repo_factory,
|
||||
)
|
||||
for path in paths
|
||||
],
|
||||
@@ -64,9 +64,9 @@ def test_subrepo_is_constructed_properly(
|
||||
with external_repo(
|
||||
str(tmp_dir), cache_dir=str(cache_dir), cache_types=["symlink"]
|
||||
) as repo:
|
||||
spy = mocker.spy(repo.repo_fs.fs, "repo_factory")
|
||||
spy = mocker.spy(repo.dvcfs.fs, "repo_factory")
|
||||
|
||||
list(repo.repo_fs.walk("", ignore_subrepos=False)) # drain
|
||||
list(repo.dvcfs.walk("", ignore_subrepos=False)) # drain
|
||||
assert spy.call_count == 1
|
||||
subrepo = spy.spy_return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user