From 005d56eedf9093b722e67e806b680580087a7bf6 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 12 May 2022 00:15:01 +0300 Subject: [PATCH] fs: rename RepoFileSystem to DvcFileSystem --- dvc/api.py | 4 +- dvc/data/reference.py | 6 +- dvc/dependency/repo.py | 6 +- dvc/fs/{repo.py => dvc.py} | 6 +- dvc/repo/__init__.py | 12 ++-- dvc/repo/collect.py | 6 +- dvc/repo/diff.py | 20 +++---- dvc/repo/get.py | 2 +- dvc/repo/ls.py | 4 +- dvc/repo/metrics/show.py | 8 +-- dvc/repo/plots/__init__.py | 6 +- dvc/repo/reproduce.py | 2 +- tests/func/experiments/test_checkpoints.py | 10 ++-- tests/func/experiments/test_experiments.py | 6 +- tests/func/test_api.py | 2 +- tests/func/test_external_repo.py | 10 ++-- tests/func/test_fs.py | 4 +- tests/func/test_get.py | 2 +- tests/func/test_import.py | 2 +- tests/func/test_update.py | 2 +- tests/unit/fs/test_data.py | 4 +- tests/unit/fs/{test_repo.py => test_dvc.py} | 60 +++++++++---------- .../{test_repo_info.py => test_dvc_info.py} | 46 +++++++------- tests/unit/test_external_repo.py | 12 ++-- 24 files changed, 121 insertions(+), 121 deletions(-) rename dvc/fs/{repo.py => dvc.py} (98%) rename tests/unit/fs/{test_repo.py => test_dvc.py} (94%) rename tests/unit/fs/{test_repo_info.py => test_dvc_info.py} (72%) diff --git a/dvc/api.py b/dvc/api.py index d668d3b6c..a7f9d663d 100644 --- a/dvc/api.py +++ b/dvc/api.py @@ -19,9 +19,9 @@ def get_url(path, repo=None, rev=None, remote=None): directory in the remote storage. """ with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo: - fs_path = _repo.repo_fs.from_os_path(path) + fs_path = _repo.dvcfs.from_os_path(path) with reraise(FileNotFoundError, PathMissingError(path, repo)): - info = _repo.repo_fs.info(fs_path) + info = _repo.dvcfs.info(fs_path) dvc_info = info.get("dvc_info") if not dvc_info: diff --git a/dvc/data/reference.py b/dvc/data/reference.py index 977c46d5a..7cdbdc018 100644 --- a/dvc/data/reference.py +++ b/dvc/data/reference.py @@ -71,7 +71,7 @@ class ReferenceHashFile(HashFile): @classmethod def from_bytes(cls, data: bytes, fs_cache: Optional[dict] = None): from dvc.fs import get_fs_cls - from dvc.fs.repo import RepoFileSystem, _RepoFileSystem + from dvc.fs.dvc import DvcFileSystem, _DvcFileSystem try: dict_ = pickle.loads(data) @@ -88,8 +88,8 @@ class ReferenceHashFile(HashFile): fs = fs_cache.get((scheme, config_pairs)) if fs_cache else None if not fs: config = dict(config_pairs) - if _RepoFileSystem.PARAM_REPO_URL in config: - fs_cls = RepoFileSystem + if _DvcFileSystem.PARAM_REPO_URL in config: + fs_cls = DvcFileSystem else: fs_cls = get_fs_cls(config, scheme=scheme) fs = fs_cls(**config) diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 7560e308a..5651d5501 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -133,7 +133,7 @@ class RepoDependency(Dependency): staging, _, staged_obj = stage( local_odb, as_posix(self.def_path), - repo.repo_fs, + repo.dvcfs, local_odb.fs.PARAM_CHECKSUM, ) except (FileNotFoundError, TreeError) as exc: @@ -153,7 +153,7 @@ class RepoDependency(Dependency): from dvc.data.db.reference import ReferenceObjectDB from dvc.data.tree import Tree from dvc.exceptions import CircularImportError - from dvc.fs.repo import RepoFileSystem + from dvc.fs.dvc import DvcFileSystem if not isinstance(odb, ReferenceObjectDB): return @@ -168,7 +168,7 @@ class RepoDependency(Dependency): checked_urls = set() for obj in iter_objs(): - if not isinstance(obj.fs, RepoFileSystem): + if not isinstance(obj.fs, DvcFileSystem): continue if ( obj.fs.repo_url in checked_urls diff --git a/dvc/fs/repo.py b/dvc/fs/dvc.py similarity index 98% rename from dvc/fs/repo.py rename to dvc/fs/dvc.py index 50b4f71ec..5a6aeff4c 100644 --- a/dvc/fs/repo.py +++ b/dvc/fs/dvc.py @@ -63,7 +63,7 @@ def _merge_info(repo, fs_info, dvc_info): return ret -class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method +class _DvcFileSystem(AbstractFileSystem): # pylint:disable=abstract-method """DVC + git-tracked files fs. Args: @@ -436,7 +436,7 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method return dvc_fs.checksum(dvc_path) -class RepoFileSystem(FileSystem): +class DvcFileSystem(FileSystem): scheme = "local" PARAM_CHECKSUM = "md5" @@ -446,7 +446,7 @@ class RepoFileSystem(FileSystem): @wrap_prop(threading.Lock()) @cached_property def fs(self): - return _RepoFileSystem(**self.fs_args) + return _DvcFileSystem(**self.fs_args) def isdvc(self, path, **kwargs): return self.fs.isdvc(path, **kwargs) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index ed6e677ef..ba81ca948 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -188,7 +188,7 @@ class Repo: self.config = Config(self.dvc_dir, fs=self.fs, config=config) self._uninitialized = uninitialized - # used by RepoFileSystem to determine if it should traverse subrepos + # used by DvcFileSystem to determine if it should traverse subrepos self.subrepos = subrepos self.cloud = DataCloud(self) @@ -490,10 +490,10 @@ class Repo: return DataFileSystem(repo=self) @cached_property - def repo_fs(self): - from dvc.fs.repo import RepoFileSystem + def dvcfs(self): + from dvc.fs.dvc import DvcFileSystem - return RepoFileSystem( + return DvcFileSystem( repo=self, subrepos=self.subrepos, **self._fs_conf ) @@ -505,13 +505,13 @@ class Repo: def open_by_relpath(self, path, remote=None, mode="r", encoding=None): """Opens a specified resource as a file descriptor""" from dvc.fs.data import DataFileSystem - from dvc.fs.repo import RepoFileSystem + from dvc.fs.dvc import DvcFileSystem if os.path.isabs(path): fs = DataFileSystem(repo=self, workspace="local") fs_path = path else: - fs = RepoFileSystem(repo=self, subrepos=True) + fs = DvcFileSystem(repo=self, subrepos=True) fs_path = fs.from_os_path(path) try: diff --git a/dvc/repo/collect.py b/dvc/repo/collect.py index 3f29852b1..dda93ec49 100644 --- a/dvc/repo/collect.py +++ b/dvc/repo/collect.py @@ -30,9 +30,9 @@ def _collect_paths( recursive: bool = False, rev: str = None, ) -> StrPaths: - from dvc.fs.repo import RepoFileSystem + from dvc.fs.dvc import DvcFileSystem - fs = RepoFileSystem(repo=repo) + fs = DvcFileSystem(repo=repo) fs_paths = [fs.from_os_path(target) for target in targets] target_paths: StrPaths = [] @@ -58,7 +58,7 @@ def _filter_duplicates( fs_res_paths = fs_paths for out in outs: - fs_path = out.repo.repo_fs.from_os_path(out.fs_path) + fs_path = out.repo.dvcfs.from_os_path(out.fs_path) if fs_path in fs_paths: res_outs.append(out) # MUTATING THE SAME LIST!! diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index df0c445ef..feeffe14e 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -22,9 +22,9 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None): if self.scm.no_commits: return {} - from dvc.fs.repo import RepoFileSystem + from dvc.fs.dvc import DvcFileSystem - repo_fs = RepoFileSystem(repo=self) + dvcfs = DvcFileSystem(repo=self) b_rev = b_rev if b_rev else "workspace" results = {} @@ -39,7 +39,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None): if targets is not None: # convert targets to paths, and capture any missing targets targets_paths, missing_targets[rev] = _targets_to_paths( - repo_fs, targets + dvcfs, targets ) results[rev] = _paths_checksums(self, targets_paths) @@ -61,7 +61,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None): if b_rev == "workspace": # missing status is only applicable when diffing local workspace # against a commit - missing = sorted(_filter_missing(repo_fs, deleted_or_missing)) + missing = sorted(_filter_missing(dvcfs, deleted_or_missing)) else: missing = [] deleted = sorted(deleted_or_missing - set(missing)) @@ -178,11 +178,11 @@ def _dir_output_paths(fs, fs_path, obj, targets=None): yield os.path.join(base, *key), oid.value -def _filter_missing(repo_fs, paths): +def _filter_missing(dvcfs, paths): for path in paths: - fs_path = repo_fs.from_os_path(path) + fs_path = dvcfs.from_os_path(path) try: - info = repo_fs.info(fs_path) + info = dvcfs.info(fs_path) dvc_info = info.get("dvc_info") if ( dvc_info @@ -194,13 +194,13 @@ def _filter_missing(repo_fs, paths): pass -def _targets_to_paths(repo_fs, targets): +def _targets_to_paths(dvcfs, targets): paths = [] missing = [] for target in targets: - if repo_fs.exists(target): - paths.append(repo_fs.repo.fs.path.abspath(target)) + if dvcfs.exists(target): + paths.append(dvcfs.repo.fs.path.abspath(target)) else: missing.append(target) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index d9b9e5578..9c0ece904 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -57,7 +57,7 @@ def get(url, path, out=None, rev=None, jobs=None): fs = DataFileSystem(repo=repo, workspace="local") fs_path = path else: - fs = repo.repo_fs + fs = repo.dvcfs fs_path = fs.from_os_path(path) with FsspecCallback.as_tqdm_callback( diff --git a/dvc/repo/ls.py b/dvc/repo/ls.py index d6479798f..728c3e590 100644 --- a/dvc/repo/ls.py +++ b/dvc/repo/ls.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Optional from dvc.exceptions import PathMissingError if TYPE_CHECKING: - from dvc.fs.repo import RepoFileSystem + from dvc.fs.dvc import DvcFileSystem from . import Repo @@ -56,7 +56,7 @@ def ls( def _ls( repo: "Repo", path: str, recursive: bool = None, dvc_only: bool = False ): - fs: "RepoFileSystem" = repo.repo_fs + fs: "DvcFileSystem" = repo.dvcfs fs_path = fs.from_os_path(path) try: diff --git a/dvc/repo/metrics/show.py b/dvc/repo/metrics/show.py index d57df8f3b..7558fbe26 100644 --- a/dvc/repo/metrics/show.py +++ b/dvc/repo/metrics/show.py @@ -4,7 +4,7 @@ from typing import List from scmrepo.exceptions import SCMError -from dvc.fs.repo import RepoFileSystem +from dvc.fs.dvc import DvcFileSystem from dvc.output import Output from dvc.repo import locked from dvc.repo.collect import StrPaths, collect @@ -24,11 +24,11 @@ def _to_fs_paths(metrics: List[Output]) -> StrPaths: result = [] for out in metrics: if out.metric: - result.append(out.repo.repo_fs.from_os_path(out.fs_path)) + result.append(out.repo.dvcfs.from_os_path(out.fs_path)) elif out.live: fs_path = summary_fs_path(out) if fs_path: - result.append(out.repo.repo_fs.from_os_path(fs_path)) + result.append(out.repo.dvcfs.from_os_path(fs_path)) return result @@ -76,7 +76,7 @@ def _read_metric(path, fs, rev, **kwargs): def _read_metrics(repo, metrics, rev, onerror=None): - fs = RepoFileSystem(repo=repo) + fs = DvcFileSystem(repo=repo) relpath = "" if repo.root_dir != repo.fs.path.getcwd(): diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index f8354b418..f3f557f92 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -99,9 +99,9 @@ class Plots: onerror: Optional[Callable] = None, props: Optional[Dict] = None, ): - from dvc.fs.repo import RepoFileSystem + from dvc.fs.dvc import DvcFileSystem - fs = RepoFileSystem(repo=self.repo) + fs = DvcFileSystem(repo=self.repo) plots = _collect_plots(self.repo, targets, revision, recursive) res: Dict[str, Any] = {} for fs_path, rev_props in plots.items(): @@ -253,7 +253,7 @@ def _collect_plots( ) result = { - repo.repo_fs.from_os_path(plot.fs_path): _plot_props(plot) + repo.dvcfs.from_os_path(plot.fs_path): _plot_props(plot) for plot in plots } result.update({fs_path: {} for fs_path in fs_paths}) diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index d4fa759ba..d25f54c3d 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -63,7 +63,7 @@ def _get_stage_files(stage: "Stage") -> typing.Iterator[str]: if ( not dep.use_scm_ignore and dep.is_in_repo - and not stage.repo.repo_fs.isdvc(dep.fs_path) + and not stage.repo.dvcfs.isdvc(dep.fs_path) ): yield dep.fs_path for out in stage.outs: diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index eab79e18c..015041b2c 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -31,7 +31,7 @@ def test_new_checkpoint( for rev in dvc.brancher([exp]): if rev == "workspace": continue - fs = dvc.repo_fs + fs = dvc.dvcfs with fs.open("foo") as fobj: assert fobj.read().strip() == str(checkpoint_stage.iterations) with fs.open("metrics.yaml") as fobj: @@ -80,7 +80,7 @@ def test_resume_checkpoint( for rev in dvc.brancher([exp]): if rev == "workspace": continue - fs = dvc.repo_fs + fs = dvc.dvcfs with fs.open("foo") as fobj: assert fobj.read().strip() == str(2 * checkpoint_stage.iterations) with fs.open("metrics.yaml") as fobj: @@ -111,7 +111,7 @@ def test_reset_checkpoint( for rev in dvc.brancher([exp]): if rev == "workspace": continue - fs = dvc.repo_fs + fs = dvc.dvcfs with fs.open("foo") as fobj: assert fobj.read().strip() == str(checkpoint_stage.iterations) with fs.open("metrics.yaml") as fobj: @@ -150,7 +150,7 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage, workspace): for rev in dvc.brancher([checkpoint_a]): if rev == "workspace": continue - fs = dvc.repo_fs + fs = dvc.dvcfs with fs.open("foo") as fobj: assert fobj.read().strip() == str(2 * checkpoint_stage.iterations) with fs.open("metrics.yaml") as fobj: @@ -159,7 +159,7 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage, workspace): for rev in dvc.brancher([checkpoint_b]): if rev == "workspace": continue - fs = dvc.repo_fs + fs = dvc.dvcfs with fs.open("foo") as fobj: assert fobj.read().strip() == str(2 * checkpoint_stage.iterations) with fs.open("metrics.yaml") as fobj: diff --git a/tests/func/experiments/test_experiments.py b/tests/func/experiments/test_experiments.py index 08d1a8063..727022bbf 100644 --- a/tests/func/experiments/test_experiments.py +++ b/tests/func/experiments/test_experiments.py @@ -525,7 +525,7 @@ def test_subdir(tmp_dir, scm, dvc, workspace): @pytest.mark.parametrize("workspace", [True, False]) def test_subrepo(tmp_dir, scm, workspace): - from tests.unit.fs.test_repo import make_subrepo + from tests.unit.fs.test_dvc import make_subrepo subrepo = tmp_dir / "dir" / "repo" make_subrepo(subrepo, scm) @@ -673,9 +673,9 @@ def test_modified_data_dep(tmp_dir, scm, dvc, workspace, params, target): for rev in dvc.brancher(revs=[exp]): if rev != exp: continue - with dvc.repo_fs.open("metrics.yaml") as fobj: + with dvc.dvcfs.open("metrics.yaml") as fobj: assert fobj.read().strip() == params - with dvc.repo_fs.open("data") as fobj: + with dvc.dvcfs.open("data") as fobj: assert fobj.read().strip() == "modified" if workspace: diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 9a6e974e1..715e4dc98 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -11,7 +11,7 @@ from dvc.exceptions import ( ) from dvc.testing.test_api import TestAPI # noqa, pylint: disable=unused-import from dvc.utils.fs import remove -from tests.unit.fs.test_repo import make_subrepo +from tests.unit.fs.test_dvc import make_subrepo def test_get_url_external(tmp_dir, erepo_dir, cloud): diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index b97095635..479cf219a 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -8,7 +8,7 @@ from dvc.data.transfer import transfer from dvc.external_repo import CLONES, external_repo from dvc.utils import relpath from dvc.utils.fs import makedirs, remove -from tests.unit.fs.test_repo import make_subrepo +from tests.unit.fs.test_dvc import make_subrepo from tests.utils import clean_staging @@ -95,7 +95,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir): dest = tmp_dir / "file" with external_repo(os.fspath(erepo_dir)) as repo: - repo.repo_fs.get( + repo.dvcfs.get( "subdir/file", os.fspath(dest), ) @@ -191,7 +191,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir): subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo") with external_repo(os.fspath(erepo_dir)) as repo: - repo.repo_fs.get( + repo.dvcfs.get( "dir", os.fspath(tmp_dir / "out"), ) @@ -207,7 +207,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir): staging, _, obj = stage( repo.odb.local, "dir", - repo.repo_fs, + repo.dvcfs, "md5", dvcignore=repo.dvcignore, ) @@ -237,7 +237,7 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir): subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo") with external_repo(os.fspath(erepo_dir)) as repo: - repo.repo_fs.get( + repo.dvcfs.get( "dir", os.fspath(tmp_dir / "out"), ) diff --git a/tests/func/test_fs.py b/tests/func/test_fs.py index 4b4b48468..0fcf5fd33 100644 --- a/tests/func/test_fs.py +++ b/tests/func/test_fs.py @@ -273,11 +273,11 @@ def test_get_dir_callback(tmp_dir, dvc, cloud): assert (tmp_dir / "dir").read_text() == {"foo": "foo", "bar": "bar"} -def test_callback_on_repo_fs(tmp_dir, dvc, scm, mocker): +def test_callback_on_dvcfs(tmp_dir, dvc, scm, mocker): tmp_dir.dvc_gen({"dir": {"bar": "bar"}}, commit="dvc") tmp_dir.scm_gen({"dir": {"foo": "foo"}}, commit="git") - fs = dvc.repo_fs + fs = dvc.dvcfs callback = FsspecCallback() fs.get( diff --git a/tests/func/test_get.py b/tests/func/test_get.py index a8b368e23..75b0774dc 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -9,7 +9,7 @@ from dvc.repo import Repo from dvc.repo.get import GetDVCFileError from dvc.system import System from dvc.utils.fs import makedirs -from tests.unit.fs.test_repo import make_subrepo +from tests.unit.fs.test_dvc import make_subrepo def test_get_repo_file(tmp_dir, erepo_dir): diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 1c9dd986e..5cc8e2c71 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -13,7 +13,7 @@ from dvc.exceptions import DownloadError, PathMissingError from dvc.stage.exceptions import StagePathNotFoundError from dvc.system import System from dvc.utils.fs import makedirs, remove -from tests.unit.fs.test_repo import make_subrepo +from tests.unit.fs.test_dvc import make_subrepo from tests.utils import clean_staging diff --git a/tests/func/test_update.py b/tests/func/test_update.py index a305327c5..94ba2c9d3 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -4,7 +4,7 @@ import pytest from dvc.dvcfile import Dvcfile from dvc.exceptions import InvalidArgumentError -from tests.unit.fs.test_repo import make_subrepo +from tests.unit.fs.test_dvc import make_subrepo @pytest.mark.parametrize("cached", [True, False]) diff --git a/tests/unit/fs/test_data.py b/tests/unit/fs/test_data.py index 92287afee..f56985af6 100644 --- a/tests/unit/fs/test_data.py +++ b/tests/unit/fs/test_data.py @@ -50,7 +50,7 @@ def test_open_dirty_hash(tmp_dir, dvc): fs = DataFileSystem(repo=dvc) with fs.open("file", "r") as fobj: - # NOTE: Unlike RepoFileSystem, DataFileSystem should not + # NOTE: Unlike DvcFileSystem, DataFileSystem should not # be affected by a dirty workspace. assert fobj.read() == "file" @@ -72,7 +72,7 @@ def test_open_dirty_no_hash(tmp_dir, dvc): (tmp_dir / "file.dvc").write_text("outs:\n- path: file\n") fs = DataFileSystem(repo=dvc) - # NOTE: Unlike RepoFileSystem, DataFileSystem should not + # NOTE: Unlike DvcFileSystem, DataFileSystem should not # be affected by a dirty workspace. with pytest.raises(FileNotFoundError): with fs.open("file", "r"): diff --git a/tests/unit/fs/test_repo.py b/tests/unit/fs/test_dvc.py similarity index 94% rename from tests/unit/fs/test_repo.py rename to tests/unit/fs/test_dvc.py index becb85fc0..9dae47e3a 100644 --- a/tests/unit/fs/test_repo.py +++ b/tests/unit/fs/test_dvc.py @@ -6,7 +6,7 @@ from unittest import mock import pytest from dvc.data.stage import stage -from dvc.fs.repo import RepoFileSystem +from dvc.fs.dvc import DvcFileSystem from dvc.objects.hash_info import HashInfo from tests.utils import clean_staging @@ -16,7 +16,7 @@ def test_exists(tmp_dir, dvc): dvc.add("foo") (tmp_dir / "foo").unlink() - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) assert fs.exists("foo") @@ -25,7 +25,7 @@ def test_open(tmp_dir, dvc): dvc.add("foo") (tmp_dir / "foo").unlink() - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) with fs.open("foo", "r") as fobj: assert fobj.read() == "foo" @@ -34,7 +34,7 @@ def test_open_dirty_hash(tmp_dir, dvc): tmp_dir.dvc_gen("file", "file") (tmp_dir / "file").write_text("something") - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) with fs.open("file", "r") as fobj: assert fobj.read() == "something" @@ -43,7 +43,7 @@ def test_open_dirty_no_hash(tmp_dir, dvc): tmp_dir.gen("file", "file") (tmp_dir / "file.dvc").write_text("outs:\n- path: file\n") - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) with fs.open("file", "r") as fobj: assert fobj.read() == "file" @@ -63,7 +63,7 @@ def test_open_in_history(tmp_dir, scm, dvc): if rev == "workspace": continue - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) with fs.open("foo", "r") as fobj: assert fobj.read() == "foo" @@ -86,7 +86,7 @@ def test_isdir_isfile(tmp_dir, dvc): }, ) - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) assert fs.isdir("datadir") assert not fs.isfile("datadir") assert not fs.isdvc("datadir") @@ -126,7 +126,7 @@ def test_exists_isdir_isfile_dirty(tmp_dir, dvc): {"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}} ) - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) shutil.rmtree(tmp_dir / "datadir") (tmp_dir / "datafile").unlink() @@ -161,7 +161,7 @@ def test_isdir_mixed(tmp_dir, dvc): dvc.add(str(tmp_dir / "dir" / "foo")) - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) assert fs.isdir("dir") assert not fs.isfile("dir") @@ -191,7 +191,7 @@ def test_walk(tmp_dir, dvc, dvcfiles, extra_expected): ) dvc.add(str(tmp_dir / "dir"), recursive=True) tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) expected = [ "dir/subdir1", @@ -226,7 +226,7 @@ def test_walk_dirty(tmp_dir, dvc): tmp_dir.gen({"dir": {"bar": "bar", "subdir3": {"foo3": "foo3"}}}) (tmp_dir / "dir" / "foo").unlink() - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) expected = [ "dir/subdir1", "dir/subdir2", @@ -252,7 +252,7 @@ def test_walk_dirty_cached_dir(tmp_dir, scm, dvc): tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}}, commit="add data") (tmp_dir / "data" / "foo").unlink() - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) actual = [] for root, dirs, files in fs.walk("data"): @@ -276,7 +276,7 @@ def test_walk_mixed_dir(tmp_dir, scm, dvc): ) tmp_dir.scm.commit("add dir") - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) expected = [ "dir/foo", @@ -293,7 +293,7 @@ def test_walk_mixed_dir(tmp_dir, scm, dvc): def test_walk_missing(tmp_dir, dvc): - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) for _ in fs.walk("dir"): pass @@ -301,7 +301,7 @@ def test_walk_missing(tmp_dir, dvc): def test_walk_not_a_dir(tmp_dir, dvc): tmp_dir.dvc_gen("foo", "foo") - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) for _ in fs.walk("foo"): pass @@ -311,7 +311,7 @@ def test_isdvc(tmp_dir, dvc): tmp_dir.gen({"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}}) dvc.add("foo") dvc.add("dir") - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) assert fs.isdvc("foo") assert not fs.isdvc("bar") assert fs.isdvc("dir") @@ -330,7 +330,7 @@ def make_subrepo(dir_, scm, config=None): def test_subrepos(tmp_dir, scm, dvc, mocker): tmp_dir.scm_gen( - {"dir": {"repo.txt": "file to confuse RepoFileSystem"}}, + {"dir": {"repo.txt": "file to confuse DvcFileSystem"}}, commit="dir/repo.txt", ) @@ -348,7 +348,7 @@ def test_subrepos(tmp_dir, scm, dvc, mocker): ) dvc._reset() - fs = RepoFileSystem(repo=dvc, subrepos=True) + fs = DvcFileSystem(repo=dvc, subrepos=True) def assert_fs_belongs_to_repo(ret_val): method = fs.fs._get_repo @@ -408,7 +408,7 @@ def test_subrepos(tmp_dir, scm, dvc, mocker): ) def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected): tmp_dir.scm_gen( - {"dir": {"repo.txt": "file to confuse RepoFileSystem"}}, + {"dir": {"repo.txt": "file to confuse DvcFileSystem"}}, commit="dir/repo.txt", ) @@ -428,7 +428,7 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected): # using fs that does not have dvcignore dvc._reset() - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) expected = [ "dir/repo", "dir/repo.txt", @@ -457,9 +457,9 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected): assert len(actual) == len(expected) -def test_repo_fs_no_subrepos(tmp_dir, dvc, scm): +def test_dvcfs_no_subrepos(tmp_dir, dvc, scm): tmp_dir.scm_gen( - {"dir": {"repo.txt": "file to confuse RepoFileSystem"}}, + {"dir": {"repo.txt": "file to confuse DvcFileSystem"}}, commit="dir/repo.txt", ) tmp_dir.dvc_gen({"lorem": "lorem"}, commit="add foo") @@ -472,7 +472,7 @@ def test_repo_fs_no_subrepos(tmp_dir, dvc, scm): # using fs that does not have dvcignore dvc._reset() - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) expected = [ "/.dvcignore", "/.gitignore", @@ -504,7 +504,7 @@ def test_repo_fs_no_subrepos(tmp_dir, dvc, scm): def test_get_hash_cached_file(tmp_dir, dvc, mocker): tmp_dir.dvc_gen({"foo": "foo"}) - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) expected = "acbd18db4cc2f85cedef654fccc4a4d8" assert fs.info("foo").get("md5") is None _, _, obj = stage(dvc.odb.local, "foo", fs, "md5") @@ -517,7 +517,7 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker): tmp_dir.dvc_gen( {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} ) - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) expected = "8761c4e9acad696bee718615e23e22db.dir" assert fs.info("dir").get("md5") is None _, _, obj = stage(dvc.odb.local, "dir", fs, "md5") @@ -537,7 +537,7 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker): tmp_dir.dvc_gen( {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} ) - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) subdir = "dir/subdir" assert fs.info(subdir).get("md5") is None _, _, obj = stage(dvc.odb.local, subdir, fs, "md5") @@ -567,7 +567,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc): tmp_dir.scm.commit("add dir") clean_staging() - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) _, _, obj = stage(dvc.odb.local, "dir", fs, "md5") assert obj.hash_info == HashInfo( "md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir" @@ -589,7 +589,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc): # file is modified in workspace # get_file_hash(file) should return workspace hash, not DVC cached hash - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) assert fs.info("file").get("md5") is None staging, _, obj = stage(dvc.odb.local, "file", fs, "md5") assert obj.hash_info == something_hash_info @@ -617,7 +617,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): (tmp_dir / "dir" / "baz").write_text("baz") clean_staging() - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) _, meta, obj = stage(dvc.odb.local, "dir", fs, "md5") assert obj.hash_info == HashInfo( "md5", "ba75a2162ca9c29acecb7957105a0bc2.dir" @@ -679,7 +679,7 @@ def test_walk_nested_subrepos(tmp_dir, dvc, scm, traverse_subrepos): expected["/subrepo1"].add("subrepo3") actual = {} - fs = RepoFileSystem(repo=dvc) + fs = DvcFileSystem(repo=dvc) for root, dirs, files in fs.walk( "/", ignore_subrepos=not traverse_subrepos ): diff --git a/tests/unit/fs/test_repo_info.py b/tests/unit/fs/test_dvc_info.py similarity index 72% rename from tests/unit/fs/test_repo_info.py rename to tests/unit/fs/test_dvc_info.py index 5f93d1edf..03bdec1f7 100644 --- a/tests/unit/fs/test_repo_info.py +++ b/tests/unit/fs/test_dvc_info.py @@ -2,12 +2,12 @@ import os import pytest -from dvc.fs.repo import RepoFileSystem -from tests.unit.fs.test_repo import make_subrepo +from dvc.fs.dvc import DvcFileSystem +from tests.unit.fs.test_dvc import make_subrepo @pytest.fixture -def repo_fs(tmp_dir, dvc, scm): +def dvcfs(tmp_dir, dvc, scm): fs_structure = { "models": { # mixed dvc + git directory "train.py": "train dot py", @@ -38,12 +38,12 @@ def repo_fs(tmp_dir, dvc, scm): tmp_dir.scm_gen(fs_structure, commit="repo init") tmp_dir.dvc_gen(dvc_structure, commit="use dvc") - yield RepoFileSystem(repo=dvc, subrepos=True) + yield DvcFileSystem(repo=dvc, subrepos=True) -def test_info_not_existing(repo_fs): +def test_info_not_existing(dvcfs): with pytest.raises(FileNotFoundError): - repo_fs.info("path/that/does/not/exist") + dvcfs.info("path/that/does/not/exist") @pytest.mark.parametrize( @@ -56,10 +56,10 @@ def test_info_not_existing(repo_fs): "src/utils/serve_model.py", ], ) -def test_info_git_tracked_file(repo_fs, path): - info = repo_fs.info(path) +def test_info_git_tracked_file(dvcfs, path): + info = dvcfs.info(path) - assert info["repo"].root_dir == repo_fs.repo.root_dir + assert info["repo"].root_dir == dvcfs.repo.root_dir assert "dvc_info" not in info assert info["type"] == "file" assert not info["isexec"] @@ -75,30 +75,30 @@ def test_info_git_tracked_file(repo_fs, path): "models/transform.pickle", ], ) -def test_info_dvc_tracked_file(repo_fs, path): - info = repo_fs.info(path) +def test_info_dvc_tracked_file(dvcfs, path): + info = dvcfs.info(path) - assert info["repo"].root_dir == repo_fs.repo.root_dir + assert info["repo"].root_dir == dvcfs.repo.root_dir assert info["dvc_info"]["isdvc"] assert info["type"] == "file" assert not info["isexec"] @pytest.mark.parametrize("path", ["src", "src/utils"]) -def test_info_git_only_dirs(repo_fs, path): - info = repo_fs.info(path) +def test_info_git_only_dirs(dvcfs, path): + info = dvcfs.info(path) - assert info["repo"].root_dir == repo_fs.repo.root_dir + assert info["repo"].root_dir == dvcfs.repo.root_dir assert "dvc_info" not in info assert info["type"] == "directory" assert not info["isexec"] @pytest.mark.parametrize("path", [".", "models"]) -def test_info_git_dvc_mixed_dirs(repo_fs, path): - info = repo_fs.info(path) +def test_info_git_dvc_mixed_dirs(dvcfs, path): + info = dvcfs.info(path) - assert info["repo"].root_dir == repo_fs.repo.root_dir + assert info["repo"].root_dir == dvcfs.repo.root_dir assert not info["dvc_info"]["isdvc"] assert info["type"] == "directory" assert not info["isexec"] @@ -112,16 +112,16 @@ def test_info_git_dvc_mixed_dirs(repo_fs, path): "data/processed", ], ) -def test_info_dvc_only_dirs(repo_fs, path): - info = repo_fs.info(path) +def test_info_dvc_only_dirs(dvcfs, path): + info = dvcfs.info(path) - assert info["repo"].root_dir == repo_fs.repo.root_dir + assert info["repo"].root_dir == dvcfs.repo.root_dir assert info["dvc_info"]["isdvc"] assert info["type"] == "directory" assert not info["isexec"] -def test_info_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, repo_fs): +def test_info_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, dvcfs): subrepo = tmp_dir / "subrepo" make_subrepo(subrepo, scm) with subrepo.chdir(): @@ -133,7 +133,7 @@ def test_info_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, repo_fs): "subrepo/foo", "subrepo/foobar", ]: - info = repo_fs.info(path) + info = dvcfs.info(path) assert info["repo"].root_dir == str( subrepo ), f"repo root didn't match for {path}" diff --git a/tests/unit/test_external_repo.py b/tests/unit/test_external_repo.py index d3a2bc663..7f2f680b7 100644 --- a/tests/unit/test_external_repo.py +++ b/tests/unit/test_external_repo.py @@ -4,7 +4,7 @@ from unittest.mock import call import pytest from dvc.external_repo import external_repo -from tests.unit.fs.test_repo import make_subrepo +from tests.unit.fs.test_dvc import make_subrepo def test_hook_is_called(tmp_dir, erepo_dir, mocker): @@ -26,9 +26,9 @@ def test_hook_is_called(tmp_dir, erepo_dir, mocker): repo.dvc_gen("bar", "bar", commit=f"dvc add {repo}/bar") with external_repo(str(erepo_dir)) as repo: - spy = mocker.spy(repo.repo_fs.fs, "repo_factory") + spy = mocker.spy(repo.dvcfs.fs, "repo_factory") - list(repo.repo_fs.walk("", ignore_subrepos=False)) # drain + list(repo.dvcfs.walk("", ignore_subrepos=False)) # drain assert spy.call_count == len(subrepos) paths = ["/" + path.replace("\\", "/") for path in subrepo_paths] @@ -38,7 +38,7 @@ def test_hook_is_called(tmp_dir, erepo_dir, mocker): path, fs=repo.fs, scm=repo.scm, - repo_factory=repo.repo_fs.fs.repo_factory, + repo_factory=repo.dvcfs.fs.repo_factory, ) for path in paths ], @@ -64,9 +64,9 @@ def test_subrepo_is_constructed_properly( with external_repo( str(tmp_dir), cache_dir=str(cache_dir), cache_types=["symlink"] ) as repo: - spy = mocker.spy(repo.repo_fs.fs, "repo_factory") + spy = mocker.spy(repo.dvcfs.fs, "repo_factory") - list(repo.repo_fs.walk("", ignore_subrepos=False)) # drain + list(repo.dvcfs.walk("", ignore_subrepos=False)) # drain assert spy.call_count == 1 subrepo = spy.spy_return