mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-06 15:28:28 +01:00
fs: rename DvcFileSystem to DataFileSystem
Current naming for `DvcFileSystem` and `RepoFileSystem` is confusing, because dvc is not only about data and it is much more natural to expect something named `dvcfs` to include complete view for your whole repo (including git files), which is what `repofs` is doing right now. `DvcFileSystem` can provide a view not only for the repo data, but also for external data (e.g. local external outputs). So it makes more sense to call it a `DataFileSystem`, since it really represents the data management side of dvc.
This commit is contained in:
@@ -14,7 +14,7 @@ if typing.TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _DvcFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
class _DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
"""DVC repo fs.
|
||||
|
||||
Args:
|
||||
@@ -178,7 +178,7 @@ class _DvcFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class DvcFileSystem(FileSystem):
|
||||
class DataFileSystem(FileSystem):
|
||||
scheme = "local"
|
||||
|
||||
PARAM_CHECKSUM = "md5"
|
||||
@@ -189,7 +189,7 @@ class DvcFileSystem(FileSystem):
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def fs(self):
|
||||
return _DvcFileSystem(**self.fs_args)
|
||||
return _DataFileSystem(**self.fs_args)
|
||||
|
||||
def isdvc(self, path, **kwargs):
|
||||
return self.fs.isdvc(path, **kwargs)
|
||||
+8
-8
@@ -10,7 +10,7 @@ from funcy import cached_property, wrap_prop, wrap_with
|
||||
|
||||
from ._callback import DEFAULT_CALLBACK
|
||||
from .base import FileSystem
|
||||
from .dvc import DvcFileSystem
|
||||
from .data import DataFileSystem
|
||||
from .path import Path
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -70,7 +70,7 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
repo: DVC or git repo.
|
||||
subrepos: traverse to subrepos (by default, it ignores subrepos)
|
||||
repo_factory: A function to initialize subrepo with, default is Repo.
|
||||
kwargs: Additional keyword arguments passed to the `DvcFileSystem()`.
|
||||
kwargs: Additional keyword arguments passed to the `DataFileSystem()`.
|
||||
"""
|
||||
|
||||
root_marker = "/"
|
||||
@@ -124,11 +124,11 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
key = self._get_key(self.repo.root_dir)
|
||||
self._subrepos_trie[key] = repo
|
||||
|
||||
self._dvcfss = {}
|
||||
"""Keep a dvcfs instance of each repo."""
|
||||
self._datafss = {}
|
||||
"""Keep a datafs instance of each repo."""
|
||||
|
||||
if hasattr(repo, "dvc_dir"):
|
||||
self._dvcfss[key] = DvcFileSystem(repo=repo)
|
||||
self._datafss[key] = DataFileSystem(repo=repo)
|
||||
|
||||
def _get_key(self, path):
|
||||
parts = self.repo.fs.path.relparts(path, self.repo.root_dir)
|
||||
@@ -245,7 +245,7 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
scm=self.repo.scm,
|
||||
repo_factory=self.repo_factory,
|
||||
)
|
||||
self._dvcfss[key] = DvcFileSystem(repo=repo)
|
||||
self._datafss[key] = DataFileSystem(repo=repo)
|
||||
self._subrepos_trie[key] = repo
|
||||
|
||||
def _is_dvc_repo(self, dir_path):
|
||||
@@ -263,7 +263,7 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
) -> Tuple[
|
||||
Optional[FileSystem],
|
||||
Optional[str],
|
||||
Optional[DvcFileSystem],
|
||||
Optional[DataFileSystem],
|
||||
Optional[str],
|
||||
]:
|
||||
"""
|
||||
@@ -286,7 +286,7 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
dvc_parts = dvc_parts[1:]
|
||||
|
||||
key = self._get_key(repo.root_dir)
|
||||
dvc_fs = self._dvcfss.get(key)
|
||||
dvc_fs = self._datafss.get(key)
|
||||
if dvc_fs:
|
||||
dvc_path = dvc_fs.path.join(*dvc_parts) if dvc_parts else ""
|
||||
else:
|
||||
|
||||
@@ -484,10 +484,10 @@ class Repo:
|
||||
return self.DVC_DIR in path_parts
|
||||
|
||||
@cached_property
|
||||
def dvcfs(self):
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
def datafs(self):
|
||||
from dvc.fs.data import DataFileSystem
|
||||
|
||||
return DvcFileSystem(repo=self)
|
||||
return DataFileSystem(repo=self)
|
||||
|
||||
@cached_property
|
||||
def repo_fs(self):
|
||||
@@ -504,11 +504,11 @@ class Repo:
|
||||
@contextmanager
|
||||
def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
|
||||
"""Opens a specified resource as a file descriptor"""
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
from dvc.fs.data import DataFileSystem
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
|
||||
if os.path.isabs(path):
|
||||
fs = DvcFileSystem(repo=self, workspace="local")
|
||||
fs = DataFileSystem(repo=self, workspace="local")
|
||||
fs_path = path
|
||||
else:
|
||||
fs = RepoFileSystem(repo=self, subrepos=True)
|
||||
|
||||
+2
-2
@@ -52,9 +52,9 @@ def get(url, path, out=None, rev=None, jobs=None):
|
||||
) as repo:
|
||||
|
||||
if os.path.isabs(path):
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
from dvc.fs.data import DataFileSystem
|
||||
|
||||
fs = DvcFileSystem(repo=repo, workspace="local")
|
||||
fs = DataFileSystem(repo=repo, workspace="local")
|
||||
fs_path = path
|
||||
else:
|
||||
fs = repo.repo_fs
|
||||
|
||||
@@ -5,7 +5,7 @@ import pytest
|
||||
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.data.stage import stage
|
||||
from dvc.fs.dvc import DvcFileSystem
|
||||
from dvc.fs.data import DataFileSystem
|
||||
from dvc.objects.hash_info import HashInfo
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
@@ -21,7 +21,7 @@ from dvc.utils.fs import remove
|
||||
],
|
||||
)
|
||||
def test_get_key(tmp_dir, dvc, path, key):
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
assert fs.fs._get_key(path) == key
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ def test_exists(tmp_dir, dvc):
|
||||
dvc.add("foo")
|
||||
(tmp_dir / "foo").unlink()
|
||||
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
assert fs.exists("foo")
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ def test_open(tmp_dir, dvc):
|
||||
dvc.add("foo")
|
||||
(tmp_dir / "foo").unlink()
|
||||
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
with fs.open("foo", "r") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
|
||||
@@ -48,9 +48,9 @@ def test_open_dirty_hash(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("file", "file")
|
||||
(tmp_dir / "file").write_text("something")
|
||||
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
with fs.open("file", "r") as fobj:
|
||||
# NOTE: Unlike RepoFileSystem, DvcFileSystem should not
|
||||
# NOTE: Unlike RepoFileSystem, DataFileSystem should not
|
||||
# be affected by a dirty workspace.
|
||||
assert fobj.read() == "file"
|
||||
|
||||
@@ -60,7 +60,7 @@ def test_open_no_remote(tmp_dir, dvc):
|
||||
(tmp_dir / "file").unlink()
|
||||
remove(dvc.odb.local.cache_dir)
|
||||
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
with pytest.raises(FileNotFoundError) as exc_info:
|
||||
with fs.open("file", "r"):
|
||||
pass
|
||||
@@ -71,8 +71,8 @@ def test_open_dirty_no_hash(tmp_dir, dvc):
|
||||
tmp_dir.gen("file", "file")
|
||||
(tmp_dir / "file.dvc").write_text("outs:\n- path: file\n")
|
||||
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
# NOTE: Unlike RepoFileSystem, DvcFileSystem should not
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
# NOTE: Unlike RepoFileSystem, DataFileSystem should not
|
||||
# be affected by a dirty workspace.
|
||||
with pytest.raises(FileNotFoundError):
|
||||
with fs.open("file", "r"):
|
||||
@@ -94,7 +94,7 @@ def test_open_in_history(tmp_dir, scm, dvc):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
with fs.open("foo", "r") as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
|
||||
@@ -102,7 +102,7 @@ def test_open_in_history(tmp_dir, scm, dvc):
|
||||
def test_isdir_isfile(tmp_dir, dvc):
|
||||
tmp_dir.gen({"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}})
|
||||
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
assert not fs.isdir("datadir")
|
||||
assert not fs.isfile("datadir")
|
||||
assert not fs.isdir("datafile")
|
||||
@@ -123,7 +123,7 @@ def test_isdir_mixed(tmp_dir, dvc):
|
||||
|
||||
dvc.add(str(tmp_dir / "dir" / "foo"))
|
||||
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
assert fs.isdir("dir")
|
||||
assert not fs.isfile("dir")
|
||||
|
||||
@@ -141,7 +141,7 @@ def test_walk(tmp_dir, dvc):
|
||||
)
|
||||
|
||||
dvc.add("dir", recursive=True)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
|
||||
expected = [
|
||||
"dir/subdir1",
|
||||
@@ -175,7 +175,7 @@ def test_walk_dir(tmp_dir, dvc):
|
||||
)
|
||||
|
||||
dvc.add("dir")
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
|
||||
expected = [
|
||||
"dir/subdir1",
|
||||
@@ -197,7 +197,7 @@ def test_walk_dir(tmp_dir, dvc):
|
||||
|
||||
|
||||
def test_walk_missing(tmp_dir, dvc):
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
|
||||
for _ in fs.walk("dir"):
|
||||
pass
|
||||
@@ -205,7 +205,7 @@ def test_walk_missing(tmp_dir, dvc):
|
||||
|
||||
def test_walk_not_a_dir(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
|
||||
for _ in fs.walk("foo"):
|
||||
pass
|
||||
@@ -214,14 +214,14 @@ def test_walk_not_a_dir(tmp_dir, dvc):
|
||||
def test_isdvc(tmp_dir, dvc):
|
||||
tmp_dir.gen({"foo": "foo", "bar": "bar"})
|
||||
dvc.add("foo")
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
assert fs.isdvc("foo")
|
||||
assert not fs.isdvc("bar")
|
||||
|
||||
|
||||
def test_get_hash_file(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen({"foo": "foo"})
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
assert fs.info("foo")["md5"] == "acbd18db4cc2f85cedef654fccc4a4d8"
|
||||
|
||||
|
||||
@@ -231,7 +231,7 @@ def test_get_hash_dir(tmp_dir, dvc, mocker):
|
||||
tmp_dir.dvc_gen(
|
||||
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
|
||||
)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
get_file_hash_spy = mocker.spy(dvc_module.data.stage, "get_file_hash")
|
||||
assert fs.info("dir")["md5"] == "8761c4e9acad696bee718615e23e22db.dir"
|
||||
assert not get_file_hash_spy.called
|
||||
@@ -241,7 +241,7 @@ def test_get_hash_granular(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen(
|
||||
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
|
||||
)
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
subdir = "dir/subdir"
|
||||
assert fs.info(subdir).get("md5") is None
|
||||
_, _, obj = stage(dvc.odb.local, subdir, fs, "md5", dry_run=True)
|
||||
@@ -258,7 +258,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("file", "file")
|
||||
(tmp_dir / "file").write_text("something")
|
||||
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
expected = "8c7dd922ad47494fc02c388e12c00eac"
|
||||
assert fs.info("file").get("md5") == expected
|
||||
_, _, obj = stage(dvc.odb.local, "file", fs, "md5", dry_run=True)
|
||||
@@ -269,7 +269,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
(tmp_dir / "dir" / "baz").write_text("baz")
|
||||
|
||||
fs = DvcFileSystem(repo=dvc)
|
||||
fs = DataFileSystem(repo=dvc)
|
||||
expected = "5ea40360f5b4ec688df672a4db9c17d1.dir"
|
||||
assert fs.info("dir").get("md5") == expected
|
||||
_, _, obj = stage(dvc.odb.local, "dir", fs, "md5", dry_run=True)
|
||||
Reference in New Issue
Block a user