fs: rename DvcFileSystem to DataFileSystem

Current naming for `DvcFileSystem` and `RepoFileSystem` is confusing, because
dvc is not only about data and it is much more natural to expect something
named `dvcfs` to include complete view for your whole repo (including git
files), which is what `repofs` is doing right now.

`DvcFileSystem` can provide a view not only for the repo data, but also for
external data (e.g. local external outputs). So it makes more sense to call
it a `DataFileSystem`, since it really represents the data management side
of dvc.
This commit is contained in:
Ruslan Kuprieiev
2022-05-11 21:49:57 +03:00
parent 799dca8b67
commit b6d25825c8
5 changed files with 40 additions and 40 deletions
+3 -3
View File
@@ -14,7 +14,7 @@ if typing.TYPE_CHECKING:
logger = logging.getLogger(__name__)
class _DvcFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
class _DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
"""DVC repo fs.
Args:
@@ -178,7 +178,7 @@ class _DvcFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
raise NotImplementedError
class DvcFileSystem(FileSystem):
class DataFileSystem(FileSystem):
scheme = "local"
PARAM_CHECKSUM = "md5"
@@ -189,7 +189,7 @@ class DvcFileSystem(FileSystem):
@wrap_prop(threading.Lock())
@cached_property
def fs(self):
return _DvcFileSystem(**self.fs_args)
return _DataFileSystem(**self.fs_args)
def isdvc(self, path, **kwargs):
return self.fs.isdvc(path, **kwargs)
+8 -8
View File
@@ -10,7 +10,7 @@ from funcy import cached_property, wrap_prop, wrap_with
from ._callback import DEFAULT_CALLBACK
from .base import FileSystem
from .dvc import DvcFileSystem
from .data import DataFileSystem
from .path import Path
if TYPE_CHECKING:
@@ -70,7 +70,7 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
repo: DVC or git repo.
subrepos: traverse to subrepos (by default, it ignores subrepos)
repo_factory: A function to initialize subrepo with, default is Repo.
kwargs: Additional keyword arguments passed to the `DvcFileSystem()`.
kwargs: Additional keyword arguments passed to the `DataFileSystem()`.
"""
root_marker = "/"
@@ -124,11 +124,11 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
key = self._get_key(self.repo.root_dir)
self._subrepos_trie[key] = repo
self._dvcfss = {}
"""Keep a dvcfs instance of each repo."""
self._datafss = {}
"""Keep a datafs instance of each repo."""
if hasattr(repo, "dvc_dir"):
self._dvcfss[key] = DvcFileSystem(repo=repo)
self._datafss[key] = DataFileSystem(repo=repo)
def _get_key(self, path):
parts = self.repo.fs.path.relparts(path, self.repo.root_dir)
@@ -245,7 +245,7 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
scm=self.repo.scm,
repo_factory=self.repo_factory,
)
self._dvcfss[key] = DvcFileSystem(repo=repo)
self._datafss[key] = DataFileSystem(repo=repo)
self._subrepos_trie[key] = repo
def _is_dvc_repo(self, dir_path):
@@ -263,7 +263,7 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
) -> Tuple[
Optional[FileSystem],
Optional[str],
Optional[DvcFileSystem],
Optional[DataFileSystem],
Optional[str],
]:
"""
@@ -286,7 +286,7 @@ class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
dvc_parts = dvc_parts[1:]
key = self._get_key(repo.root_dir)
dvc_fs = self._dvcfss.get(key)
dvc_fs = self._datafss.get(key)
if dvc_fs:
dvc_path = dvc_fs.path.join(*dvc_parts) if dvc_parts else ""
else:
+5 -5
View File
@@ -484,10 +484,10 @@ class Repo:
return self.DVC_DIR in path_parts
@cached_property
def dvcfs(self):
from dvc.fs.dvc import DvcFileSystem
def datafs(self):
from dvc.fs.data import DataFileSystem
return DvcFileSystem(repo=self)
return DataFileSystem(repo=self)
@cached_property
def repo_fs(self):
@@ -504,11 +504,11 @@ class Repo:
@contextmanager
def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
"""Opens a specified resource as a file descriptor"""
from dvc.fs.dvc import DvcFileSystem
from dvc.fs.data import DataFileSystem
from dvc.fs.repo import RepoFileSystem
if os.path.isabs(path):
fs = DvcFileSystem(repo=self, workspace="local")
fs = DataFileSystem(repo=self, workspace="local")
fs_path = path
else:
fs = RepoFileSystem(repo=self, subrepos=True)
+2 -2
View File
@@ -52,9 +52,9 @@ def get(url, path, out=None, rev=None, jobs=None):
) as repo:
if os.path.isabs(path):
from dvc.fs.dvc import DvcFileSystem
from dvc.fs.data import DataFileSystem
fs = DvcFileSystem(repo=repo, workspace="local")
fs = DataFileSystem(repo=repo, workspace="local")
fs_path = path
else:
fs = repo.repo_fs
@@ -5,7 +5,7 @@ import pytest
from dvc.config import NoRemoteError
from dvc.data.stage import stage
from dvc.fs.dvc import DvcFileSystem
from dvc.fs.data import DataFileSystem
from dvc.objects.hash_info import HashInfo
from dvc.utils.fs import remove
@@ -21,7 +21,7 @@ from dvc.utils.fs import remove
],
)
def test_get_key(tmp_dir, dvc, path, key):
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
assert fs.fs._get_key(path) == key
@@ -30,7 +30,7 @@ def test_exists(tmp_dir, dvc):
dvc.add("foo")
(tmp_dir / "foo").unlink()
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
assert fs.exists("foo")
@@ -39,7 +39,7 @@ def test_open(tmp_dir, dvc):
dvc.add("foo")
(tmp_dir / "foo").unlink()
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
with fs.open("foo", "r") as fobj:
assert fobj.read() == "foo"
@@ -48,9 +48,9 @@ def test_open_dirty_hash(tmp_dir, dvc):
tmp_dir.dvc_gen("file", "file")
(tmp_dir / "file").write_text("something")
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
with fs.open("file", "r") as fobj:
# NOTE: Unlike RepoFileSystem, DvcFileSystem should not
# NOTE: Unlike RepoFileSystem, DataFileSystem should not
# be affected by a dirty workspace.
assert fobj.read() == "file"
@@ -60,7 +60,7 @@ def test_open_no_remote(tmp_dir, dvc):
(tmp_dir / "file").unlink()
remove(dvc.odb.local.cache_dir)
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
with pytest.raises(FileNotFoundError) as exc_info:
with fs.open("file", "r"):
pass
@@ -71,8 +71,8 @@ def test_open_dirty_no_hash(tmp_dir, dvc):
tmp_dir.gen("file", "file")
(tmp_dir / "file.dvc").write_text("outs:\n- path: file\n")
fs = DvcFileSystem(repo=dvc)
# NOTE: Unlike RepoFileSystem, DvcFileSystem should not
fs = DataFileSystem(repo=dvc)
# NOTE: Unlike RepoFileSystem, DataFileSystem should not
# be affected by a dirty workspace.
with pytest.raises(FileNotFoundError):
with fs.open("file", "r"):
@@ -94,7 +94,7 @@ def test_open_in_history(tmp_dir, scm, dvc):
if rev == "workspace":
continue
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
with fs.open("foo", "r") as fobj:
assert fobj.read() == "foo"
@@ -102,7 +102,7 @@ def test_open_in_history(tmp_dir, scm, dvc):
def test_isdir_isfile(tmp_dir, dvc):
tmp_dir.gen({"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}})
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
assert not fs.isdir("datadir")
assert not fs.isfile("datadir")
assert not fs.isdir("datafile")
@@ -123,7 +123,7 @@ def test_isdir_mixed(tmp_dir, dvc):
dvc.add(str(tmp_dir / "dir" / "foo"))
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
assert fs.isdir("dir")
assert not fs.isfile("dir")
@@ -141,7 +141,7 @@ def test_walk(tmp_dir, dvc):
)
dvc.add("dir", recursive=True)
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
expected = [
"dir/subdir1",
@@ -175,7 +175,7 @@ def test_walk_dir(tmp_dir, dvc):
)
dvc.add("dir")
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
expected = [
"dir/subdir1",
@@ -197,7 +197,7 @@ def test_walk_dir(tmp_dir, dvc):
def test_walk_missing(tmp_dir, dvc):
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
for _ in fs.walk("dir"):
pass
@@ -205,7 +205,7 @@ def test_walk_missing(tmp_dir, dvc):
def test_walk_not_a_dir(tmp_dir, dvc):
tmp_dir.dvc_gen("foo", "foo")
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
for _ in fs.walk("foo"):
pass
@@ -214,14 +214,14 @@ def test_walk_not_a_dir(tmp_dir, dvc):
def test_isdvc(tmp_dir, dvc):
tmp_dir.gen({"foo": "foo", "bar": "bar"})
dvc.add("foo")
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
assert fs.isdvc("foo")
assert not fs.isdvc("bar")
def test_get_hash_file(tmp_dir, dvc):
tmp_dir.dvc_gen({"foo": "foo"})
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
assert fs.info("foo")["md5"] == "acbd18db4cc2f85cedef654fccc4a4d8"
@@ -231,7 +231,7 @@ def test_get_hash_dir(tmp_dir, dvc, mocker):
tmp_dir.dvc_gen(
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
)
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
get_file_hash_spy = mocker.spy(dvc_module.data.stage, "get_file_hash")
assert fs.info("dir")["md5"] == "8761c4e9acad696bee718615e23e22db.dir"
assert not get_file_hash_spy.called
@@ -241,7 +241,7 @@ def test_get_hash_granular(tmp_dir, dvc):
tmp_dir.dvc_gen(
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
)
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
subdir = "dir/subdir"
assert fs.info(subdir).get("md5") is None
_, _, obj = stage(dvc.odb.local, subdir, fs, "md5", dry_run=True)
@@ -258,7 +258,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
tmp_dir.dvc_gen("file", "file")
(tmp_dir / "file").write_text("something")
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
expected = "8c7dd922ad47494fc02c388e12c00eac"
assert fs.info("file").get("md5") == expected
_, _, obj = stage(dvc.odb.local, "file", fs, "md5", dry_run=True)
@@ -269,7 +269,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
(tmp_dir / "dir" / "baz").write_text("baz")
fs = DvcFileSystem(repo=dvc)
fs = DataFileSystem(repo=dvc)
expected = "5ea40360f5b4ec688df672a4db9c17d1.dir"
assert fs.info("dir").get("md5") == expected
_, _, obj = stage(dvc.odb.local, "dir", fs, "md5", dry_run=True)