mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
repofs: don't assume repofs paths are the same as underlying fs paths
This commit is contained in:
+27
-33
@@ -233,7 +233,7 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
else:
|
||||
dvc_path = path
|
||||
|
||||
return repo.fs, dvc_fs, dvc_path
|
||||
return repo.fs, path, dvc_fs, dvc_path
|
||||
|
||||
def open(
|
||||
self, path, mode="r", encoding="utf-8", **kwargs
|
||||
@@ -241,9 +241,9 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
if "b" in mode:
|
||||
encoding = None
|
||||
|
||||
fs, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
try:
|
||||
return fs.open(path, mode=mode, encoding=encoding)
|
||||
return fs.open(fs_path, mode=mode, encoding=encoding)
|
||||
except FileNotFoundError:
|
||||
if not dvc_fs:
|
||||
raise
|
||||
@@ -251,12 +251,10 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
return dvc_fs.open(dvc_path, mode=mode, encoding=encoding, **kwargs)
|
||||
|
||||
def exists(self, path) -> bool:
|
||||
path = os.path.abspath(path)
|
||||
|
||||
fs, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
|
||||
if not dvc_fs:
|
||||
return fs.exists(path)
|
||||
return fs.exists(fs_path)
|
||||
|
||||
if dvc_fs.repo.dvcignore.is_ignored(fs, path):
|
||||
return False
|
||||
@@ -267,7 +265,7 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
if not dvc_fs.exists(dvc_path):
|
||||
return False
|
||||
|
||||
for p in self.path.parents(path):
|
||||
for p in self.path.parents(fs_path):
|
||||
try:
|
||||
if fs.info(p)["type"] != "directory":
|
||||
return False
|
||||
@@ -277,15 +275,13 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
return True
|
||||
|
||||
def isdir(self, path): # pylint: disable=arguments-renamed
|
||||
path = os.path.abspath(path)
|
||||
fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
|
||||
fs, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
|
||||
if dvc_fs and dvc_fs.repo.dvcignore.is_ignored_dir(path):
|
||||
if dvc_fs and dvc_fs.repo.dvcignore.is_ignored_dir(fs_path):
|
||||
return False
|
||||
|
||||
try:
|
||||
info = fs.info(path)
|
||||
info = fs.info(fs_path)
|
||||
return info["type"] == "directory"
|
||||
except (OSError, ValueError):
|
||||
# from CPython's os.path.isdir()
|
||||
@@ -299,7 +295,7 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
for p in self.path.parents(path):
|
||||
for p in fs.path.parents(fs_path):
|
||||
try:
|
||||
if fs.info(p)["type"] != "directory":
|
||||
return False
|
||||
@@ -309,19 +305,17 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
return info["type"] == "directory"
|
||||
|
||||
def isdvc(self, path, **kwargs):
|
||||
_, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
_, _, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
return dvc_fs is not None and dvc_fs.isdvc(dvc_path, **kwargs)
|
||||
|
||||
def isfile(self, path): # pylint: disable=arguments-renamed
|
||||
path = os.path.abspath(path)
|
||||
fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
|
||||
fs, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
|
||||
if dvc_fs and dvc_fs.repo.dvcignore.is_ignored_file(path):
|
||||
if dvc_fs and dvc_fs.repo.dvcignore.is_ignored_file(fs_path):
|
||||
return False
|
||||
|
||||
try:
|
||||
info = fs.info(path)
|
||||
info = fs.info(fs_path)
|
||||
return info["type"] == "file"
|
||||
except (OSError, ValueError):
|
||||
# from CPython's os.path.isfile()
|
||||
@@ -335,7 +329,7 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
for p in self.path.parents(path):
|
||||
for p in fs.path.parents(fs_path):
|
||||
try:
|
||||
if fs.info(p)["type"] != "directory":
|
||||
return False
|
||||
@@ -350,8 +344,8 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
NOTE: subrepo will only be discovered when walking if
|
||||
ignore_subrepos is set to False.
|
||||
"""
|
||||
fs, dvc_fs, dvc_path = self._get_fs_pair(dir_path)
|
||||
yield from self._walk(fs, dir_path, dvc_fs, dvc_path, **kwargs)
|
||||
fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(dir_path)
|
||||
yield from self._walk(fs, fs_path, dvc_fs, dvc_path, **kwargs)
|
||||
|
||||
def _walk(
|
||||
self,
|
||||
@@ -463,13 +457,13 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
dvcignore = repo.dvcignore
|
||||
dvcfiles = kwargs.pop("dvcfiles", False)
|
||||
|
||||
fs, dvc_fs, dvc_path = self._get_fs_pair(top)
|
||||
repo_exists = fs.exists(top)
|
||||
fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(top)
|
||||
repo_exists = fs.exists(fs_path)
|
||||
|
||||
if not dvc_fs or (repo_exists and dvc_fs.isdvc(dvc_path)):
|
||||
yield from self._walk(
|
||||
fs,
|
||||
top,
|
||||
fs_path,
|
||||
None,
|
||||
None,
|
||||
dvcfiles=dvcfiles,
|
||||
@@ -483,7 +477,7 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
|
||||
yield from self._walk(
|
||||
fs,
|
||||
top,
|
||||
fs_path,
|
||||
dvc_fs,
|
||||
dvc_path,
|
||||
dvcfiles=dvcfiles,
|
||||
@@ -499,10 +493,10 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
def get_file(
|
||||
self, from_info, to_file, callback=DEFAULT_CALLBACK, **kwargs
|
||||
):
|
||||
fs, dvc_fs, dvc_path = self._get_fs_pair(from_info)
|
||||
fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(from_info)
|
||||
try:
|
||||
fs.get_file( # pylint: disable=protected-access
|
||||
from_info, to_file, callback=callback, **kwargs
|
||||
fs_path, to_file, callback=callback, **kwargs
|
||||
)
|
||||
return
|
||||
except FileNotFoundError:
|
||||
@@ -514,7 +508,7 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
)
|
||||
|
||||
def info(self, path):
|
||||
fs, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
|
||||
try:
|
||||
dvc_info = dvc_fs.info(dvc_path)
|
||||
@@ -524,7 +518,7 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
try:
|
||||
from dvc.utils import is_exec
|
||||
|
||||
fs_info = fs.info(path)
|
||||
fs_info = fs.info(fs_path)
|
||||
fs_info["repo"] = dvc_fs.repo
|
||||
fs_info["isout"] = (
|
||||
dvc_info.get("isout", False) if dvc_info else False
|
||||
@@ -550,9 +544,9 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
return dvc_info
|
||||
|
||||
def checksum(self, path):
|
||||
fs, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(path)
|
||||
|
||||
try:
|
||||
return fs.checksum(path)
|
||||
return fs.checksum(fs_path)
|
||||
except FileNotFoundError:
|
||||
return dvc_fs.checksum(dvc_path)
|
||||
|
||||
+19
-19
@@ -194,13 +194,13 @@ def test_walk(tmp_dir, dvc, dvcfiles, extra_expected):
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
|
||||
expected = [
|
||||
os.path.join("dir", "subdir1"),
|
||||
os.path.join("dir", "subdir2"),
|
||||
os.path.join("dir", "subdir1", "foo1"),
|
||||
os.path.join("dir", "subdir1", "bar1"),
|
||||
os.path.join("dir", "subdir2", "foo2"),
|
||||
os.path.join("dir", "foo"),
|
||||
os.path.join("dir", "bar"),
|
||||
os.path.join(tmp_dir, "dir", "subdir1"),
|
||||
os.path.join(tmp_dir, "dir", "subdir2"),
|
||||
os.path.join(tmp_dir, "dir", "subdir1", "foo1"),
|
||||
os.path.join(tmp_dir, "dir", "subdir1", "bar1"),
|
||||
os.path.join(tmp_dir, "dir", "subdir2", "foo2"),
|
||||
os.path.join(tmp_dir, "dir", "foo"),
|
||||
os.path.join(tmp_dir, "dir", "bar"),
|
||||
]
|
||||
|
||||
actual = []
|
||||
@@ -208,7 +208,7 @@ def test_walk(tmp_dir, dvc, dvcfiles, extra_expected):
|
||||
for entry in dirs + files:
|
||||
actual.append(os.path.join(root, entry))
|
||||
|
||||
expected += extra_expected
|
||||
expected += [os.path.join(tmp_dir, path) for path in extra_expected]
|
||||
assert set(actual) == set(expected)
|
||||
assert len(actual) == len(expected)
|
||||
|
||||
@@ -228,14 +228,14 @@ def test_walk_dirty(tmp_dir, dvc):
|
||||
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
expected = [
|
||||
os.path.join("dir", "subdir1"),
|
||||
os.path.join("dir", "subdir2"),
|
||||
os.path.join("dir", "subdir3"),
|
||||
os.path.join("dir", "subdir1", "foo1"),
|
||||
os.path.join("dir", "subdir1", "bar1"),
|
||||
os.path.join("dir", "subdir2", "foo2"),
|
||||
os.path.join("dir", "subdir3", "foo3"),
|
||||
os.path.join("dir", "bar"),
|
||||
os.path.join(tmp_dir, "dir", "subdir1"),
|
||||
os.path.join(tmp_dir, "dir", "subdir2"),
|
||||
os.path.join(tmp_dir, "dir", "subdir3"),
|
||||
os.path.join(tmp_dir, "dir", "subdir1", "foo1"),
|
||||
os.path.join(tmp_dir, "dir", "subdir1", "bar1"),
|
||||
os.path.join(tmp_dir, "dir", "subdir2", "foo2"),
|
||||
os.path.join(tmp_dir, "dir", "subdir3", "foo3"),
|
||||
os.path.join(tmp_dir, "dir", "bar"),
|
||||
]
|
||||
|
||||
actual = []
|
||||
@@ -278,9 +278,9 @@ def test_walk_mixed_dir(tmp_dir, scm, dvc):
|
||||
fs = RepoFileSystem(repo=dvc)
|
||||
|
||||
expected = [
|
||||
os.path.join("dir", "foo"),
|
||||
os.path.join("dir", "bar"),
|
||||
os.path.join("dir", ".gitignore"),
|
||||
os.path.join(tmp_dir, "dir", "foo"),
|
||||
os.path.join(tmp_dir, "dir", "bar"),
|
||||
os.path.join(tmp_dir, "dir", ".gitignore"),
|
||||
]
|
||||
actual = []
|
||||
for root, dirs, files in fs.walk("dir"):
|
||||
|
||||
Reference in New Issue
Block a user