deps: update dvc-data to >=3

This commit is contained in:
Ruslan Kuprieiev
2023-12-15 08:02:40 +02:00
parent 766b43bc2d
commit 3e4e143bee
44 changed files with 266 additions and 269 deletions
+2 -2
View File
@@ -27,7 +27,7 @@ def _get_odb(
cls, config, fs_path = get_cloud_fs(repo.config, **settings) cls, config, fs_path = get_cloud_fs(repo.config, **settings)
fs = fs or cls(**config) fs = fs or cls(**config)
if prefix: if prefix:
fs_path = fs.path.join(fs_path, *prefix) fs_path = fs.join(fs_path, *prefix)
if hash_name: if hash_name:
config["hash_name"] = hash_name config["hash_name"] = hash_name
return get_odb(fs, fs_path, state=repo.state, **config) return get_odb(fs, fs_path, state=repo.state, **config)
@@ -86,7 +86,7 @@ class CacheManager:
return FileStorage( return FileStorage(
key=(), key=(),
fs=self.local.fs, fs=self.local.fs,
path=self.local.fs.path.join(self.default_local_cache_dir, self.FS_DIR), path=self.local.fs.join(self.default_local_cache_dir, self.FS_DIR),
) )
def _init_odb(self, schemes): def _init_odb(self, schemes):
+1 -1
View File
@@ -86,7 +86,7 @@ def _collect_targets(repo, target, outs):
targets.extend([str(out) for out in stage.outs]) targets.extend([str(out) for out in stage.outs])
continue continue
for out in outs_trie.itervalues(prefix=repo.fs.path.parts(path)): for out in outs_trie.itervalues(prefix=repo.fs.parts(path)):
targets.extend(str(out)) targets.extend(str(out))
return targets return targets
+8 -8
View File
@@ -14,7 +14,7 @@ from .utils.objects import cached_property
if TYPE_CHECKING: if TYPE_CHECKING:
from dvc.fs import FileSystem from dvc.fs import FileSystem
from dvc.types import DictStrAny, StrPath from dvc.types import DictStrAny
logger = logger.getChild(__name__) logger = logger.getChild(__name__)
@@ -89,8 +89,8 @@ class Config(dict):
def __init__( def __init__(
self, self,
dvc_dir: Optional["StrPath"] = None, dvc_dir: Optional[str] = None,
local_dvc_dir: Optional["StrPath"] = None, local_dvc_dir: Optional[str] = None,
validate: bool = True, validate: bool = True,
fs: Optional["FileSystem"] = None, fs: Optional["FileSystem"] = None,
config: Optional["DictStrAny"] = None, config: Optional["DictStrAny"] = None,
@@ -105,7 +105,7 @@ class Config(dict):
self.fs = fs or self.wfs self.fs = fs or self.wfs
if dvc_dir: if dvc_dir:
self.dvc_dir = self.fs.path.abspath(dvc_dir) self.dvc_dir = self.fs.abspath(dvc_dir)
self.local_dvc_dir = local_dvc_dir self.local_dvc_dir = local_dvc_dir
if not fs and not local_dvc_dir: if not fs and not local_dvc_dir:
@@ -145,10 +145,10 @@ class Config(dict):
} }
if self.dvc_dir is not None: if self.dvc_dir is not None:
files["repo"] = self.fs.path.join(self.dvc_dir, self.CONFIG) files["repo"] = self.fs.join(self.dvc_dir, self.CONFIG)
if self.local_dvc_dir is not None: if self.local_dvc_dir is not None:
files["local"] = self.wfs.path.join(self.local_dvc_dir, self.CONFIG_LOCAL) files["local"] = self.wfs.join(self.local_dvc_dir, self.CONFIG_LOCAL)
return files return files
@@ -302,11 +302,11 @@ class Config(dict):
return path.def_path return path.def_path
if os.path.expanduser(path) != path: if os.path.expanduser(path) != path:
return localfs.path.as_posix(path) return localfs.as_posix(path)
if isinstance(path, RelPath) or not os.path.isabs(path): if isinstance(path, RelPath) or not os.path.isabs(path):
path = relpath(path, conf_dir) path = relpath(path, conf_dir)
return localfs.path.as_posix(path) return localfs.as_posix(path)
return path return path
+2 -4
View File
@@ -35,11 +35,9 @@ class Remote:
path = self.path path = self.path
if self.worktree: if self.worktree:
path = self.fs.path.join( path = self.fs.join(path, ".dvc", CacheManager.FILES_DIR, DEFAULT_ALGORITHM)
path, ".dvc", CacheManager.FILES_DIR, DEFAULT_ALGORITHM
)
else: else:
path = self.fs.path.join(path, CacheManager.FILES_DIR, DEFAULT_ALGORITHM) path = self.fs.join(path, CacheManager.FILES_DIR, DEFAULT_ALGORITHM)
return get_odb(self.fs, path, hash_name=DEFAULT_ALGORITHM, **self.config) return get_odb(self.fs, path, hash_name=DEFAULT_ALGORITHM, **self.config)
@cached_property @cached_property
+4 -4
View File
@@ -33,7 +33,7 @@ class Dependency(Output):
if self.fs.version_aware: if self.fs.version_aware:
old_fs_path = self.fs_path old_fs_path = self.fs_path
try: try:
self.fs_path = self.fs.path.version_path(self.fs_path, None) self.fs_path = self.fs.version_path(self.fs_path, None)
if self.changed_meta(): if self.changed_meta():
return {str(self): "update available"} return {str(self): "update available"}
finally: finally:
@@ -42,9 +42,9 @@ class Dependency(Output):
def update(self, rev=None): def update(self, rev=None):
if self.fs.version_aware: if self.fs.version_aware:
self.fs_path = self.fs.path.version_path(self.fs_path, rev) self.fs_path = self.fs.version_path(self.fs_path, rev)
self.meta = self.get_meta() self.meta = self.get_meta()
self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id) self.fs_path = self.fs.version_path(self.fs_path, self.meta.version_id)
def download(self, to, jobs=None): def download(self, to, jobs=None):
fs_download(self.fs, self.fs_path, to.fs_path, jobs=jobs) fs_download(self.fs, self.fs_path, to.fs_path, jobs=jobs)
@@ -52,7 +52,7 @@ class Dependency(Output):
def save(self): def save(self):
super().save() super().save()
if self.fs.version_aware: if self.fs.version_aware:
self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id) self.fs_path = self.fs.version_path(self.fs_path, self.meta.version_id)
def dumpd(self, **kwargs): def dumpd(self, **kwargs):
if self.fs.version_aware: if self.fs.version_aware:
+1 -1
View File
@@ -294,7 +294,7 @@ class ProjectFile(FileMixin):
def resolver(self) -> "DataResolver": def resolver(self) -> "DataResolver":
from .parsing import DataResolver from .parsing import DataResolver
wdir = self.repo.fs.path.parent(self.path) wdir = self.repo.fs.parent(self.path)
return DataResolver(self.repo, wdir, self.contents) return DataResolver(self.repo, wdir, self.contents)
@cached_property @cached_property
+3 -7
View File
@@ -25,7 +25,6 @@ from dvc_objects.fs.errors import ( # noqa: F401
ConfigError, ConfigError,
RemoteMissingDepsError, RemoteMissingDepsError,
) )
from dvc_objects.fs.path import Path # noqa: F401
from .callbacks import Callback # noqa: F401 from .callbacks import Callback # noqa: F401
from .data import DataFileSystem # noqa: F401 from .data import DataFileSystem # noqa: F401
@@ -54,23 +53,20 @@ def download(
from .callbacks import TqdmCallback from .callbacks import TqdmCallback
with TqdmCallback( with TqdmCallback(
desc=f"Downloading {fs.path.name(fs_path)}", desc=f"Downloading {fs.name(fs_path)}",
unit="files", unit="files",
) as cb: ) as cb:
# NOTE: We use dvc-objects generic.copy over fs.get since it makes file # NOTE: We use dvc-objects generic.copy over fs.get since it makes file
# download atomic and avoids fsspec glob/regex path expansion. # download atomic and avoids fsspec glob/regex path expansion.
if fs.isdir(fs_path): if fs.isdir(fs_path):
from_infos = [ from_infos = [
path path for path in fs.find(fs_path) if not path.endswith(fs.flavour.sep)
for path in fs.find(fs_path)
if not path.endswith(fs.path.flavour.sep)
] ]
if not from_infos: if not from_infos:
localfs.makedirs(to, exist_ok=True) localfs.makedirs(to, exist_ok=True)
return 0 return 0
to_infos = [ to_infos = [
localfs.path.join(to, *fs.path.relparts(info, fs_path)) localfs.join(to, *fs.relparts(info, fs_path)) for info in from_infos
for info in from_infos
] ]
else: else:
from_infos = [fs_path] from_infos = [fs_path]
+3
View File
@@ -29,6 +29,9 @@ class DataFileSystem(FileSystem):
return _DataFileSystem(**self.fs_args) return _DataFileSystem(**self.fs_args)
def getcwd(self):
return self.fs.getcwd()
def isdvc(self, path, **kwargs): def isdvc(self, path, **kwargs):
return self.fs.isdvc(path, **kwargs) return self.fs.isdvc(path, **kwargs)
+57 -24
View File
@@ -13,7 +13,6 @@ from funcy import wrap_with
from dvc.log import logger from dvc.log import logger
from dvc_objects.fs.base import FileSystem from dvc_objects.fs.base import FileSystem
from dvc_objects.fs.path import Path
from .data import DataFileSystem from .data import DataFileSystem
@@ -71,7 +70,7 @@ def _merge_info(repo, key, fs_info, dvc_info):
def _get_dvc_path(dvc_fs, subkey): def _get_dvc_path(dvc_fs, subkey):
return dvc_fs.path.join(*subkey) if subkey else "" return dvc_fs.join(*subkey) if subkey else ""
class _DVCFileSystem(AbstractFileSystem): class _DVCFileSystem(AbstractFileSystem):
@@ -145,7 +144,51 @@ class _DVCFileSystem(AbstractFileSystem):
"remote_config": remote_config, "remote_config": remote_config,
} }
self.path = Path(self.sep, getcwd=self._getcwd) def getcwd(self):
relparts: Tuple[str, ...] = ()
assert self.repo is not None
if self.repo.fs.isin(self.repo.fs.getcwd(), self.repo.root_dir):
relparts = self.repo.fs.relparts(self.repo.fs.getcwd(), self.repo.root_dir)
return self.root_marker + self.sep.join(relparts)
@classmethod
def join(cls, *parts: str) -> str:
return posixpath.join(*parts)
@classmethod
def parts(cls, path: str) -> Tuple[str, ...]:
ret = []
while True:
path, part = posixpath.split(path)
if part:
ret.append(part)
continue
if path:
ret.append(path)
break
ret.reverse()
return tuple(ret)
def normpath(self, path: str) -> str:
return posixpath.normpath(path)
def abspath(self, path: str) -> str:
if not posixpath.isabs(path):
path = self.join(self.getcwd(), path)
return self.normpath(path)
def relpath(self, path: str, start: Optional[str] = None) -> str:
if start is None:
start = "."
return posixpath.relpath(self.abspath(path), start=self.abspath(start))
def relparts(self, path: str, start: Optional[str] = None) -> Tuple[str, ...]:
return self.parts(self.relpath(path, start=start))
@functools.cached_property @functools.cached_property
def repo(self): def repo(self):
@@ -170,15 +213,6 @@ class _DVCFileSystem(AbstractFileSystem):
return self.repo._fs_conf["repo_factory"] return self.repo._fs_conf["repo_factory"]
def _getcwd(self):
relparts: Tuple[str, ...] = ()
assert self.repo is not None
if self.repo.fs.path.isin(self.repo.fs.path.getcwd(), self.repo.root_dir):
relparts = self.repo.fs.path.relparts(
self.repo.fs.path.getcwd(), self.repo.root_dir
)
return self.root_marker + self.sep.join(relparts)
@functools.cached_property @functools.cached_property
def fsid(self) -> str: def fsid(self) -> str:
from fsspec.utils import tokenize from fsspec.utils import tokenize
@@ -192,7 +226,7 @@ class _DVCFileSystem(AbstractFileSystem):
def _get_key(self, path: "StrPath") -> Key: def _get_key(self, path: "StrPath") -> Key:
path = os.fspath(path) path = os.fspath(path)
parts = self.repo.fs.path.relparts(path, self.repo.root_dir) parts = self.repo.fs.relparts(path, self.repo.root_dir)
if parts == (os.curdir,): if parts == (os.curdir,):
return () return ()
return parts return parts
@@ -210,13 +244,13 @@ class _DVCFileSystem(AbstractFileSystem):
def _get_key_from_relative(self, path) -> Key: def _get_key_from_relative(self, path) -> Key:
path = self._strip_protocol(path) path = self._strip_protocol(path)
parts = self.path.relparts(path, self.root_marker) parts = self.relparts(path, self.root_marker)
if parts and parts[0] == os.curdir: if parts and parts[0] == os.curdir:
return parts[1:] return parts[1:]
return parts return parts
def _from_key(self, parts: Key) -> str: def _from_key(self, parts: Key) -> str:
return self.repo.fs.path.join(self.repo.root_dir, *parts) return self.repo.fs.join(self.repo.root_dir, *parts)
@functools.cached_property @functools.cached_property
def _datafss(self): def _datafss(self):
@@ -282,7 +316,7 @@ class _DVCFileSystem(AbstractFileSystem):
from dvc.repo import Repo from dvc.repo import Repo
repo_path = self.repo.fs.path.join(dir_path, Repo.DVC_DIR) repo_path = self.repo.fs.join(dir_path, Repo.DVC_DIR)
return self.repo.fs.isdir(repo_path) return self.repo.fs.isdir(repo_path)
def _get_subrepo_info( def _get_subrepo_info(
@@ -336,7 +370,7 @@ class _DVCFileSystem(AbstractFileSystem):
dvc_path = _get_dvc_path(dvc_fs, subkey) dvc_path = _get_dvc_path(dvc_fs, subkey)
with suppress(FileNotFoundError, NotADirectoryError): with suppress(FileNotFoundError, NotADirectoryError):
for info in dvc_fs.ls(dvc_path, detail=True): for info in dvc_fs.ls(dvc_path, detail=True):
dvc_infos[dvc_fs.path.name(info["name"])] = info dvc_infos[dvc_fs.name(info["name"])] = info
dvc_exists = True dvc_exists = True
fs_exists = False fs_exists = False
@@ -349,7 +383,7 @@ class _DVCFileSystem(AbstractFileSystem):
for info in repo.dvcignore.ls( for info in repo.dvcignore.ls(
fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos
): ):
fs_infos[fs.path.name(info["name"])] = info fs_infos[fs.name(info["name"])] = info
fs_exists = True fs_exists = True
except (FileNotFoundError, NotADirectoryError): except (FileNotFoundError, NotADirectoryError):
pass pass
@@ -368,7 +402,7 @@ class _DVCFileSystem(AbstractFileSystem):
if not dvcfiles and _is_dvc_file(name): if not dvcfiles and _is_dvc_file(name):
continue continue
entry_path = self.path.join(path, name) entry_path = self.join(path, name)
info = _merge_info( info = _merge_info(
repo, (*subkey, name), fs_infos.get(name), dvc_infos.get(name) repo, (*subkey, name), fs_infos.get(name), dvc_infos.get(name)
) )
@@ -416,7 +450,7 @@ class _DVCFileSystem(AbstractFileSystem):
# NOTE: if some parent in fs_path turns out to be a file, it means # NOTE: if some parent in fs_path turns out to be a file, it means
# that the whole repofs branch doesn't exist. # that the whole repofs branch doesn't exist.
if dvc_info and not fs_info: if dvc_info and not fs_info:
for parent in fs.path.parents(fs_path): for parent in fs.parents(fs_path):
try: try:
if fs.info(parent)["type"] != "directory": if fs.info(parent)["type"] != "directory":
dvc_info = None dvc_info = None
@@ -498,6 +532,9 @@ class DVCFileSystem(FileSystem):
def fs(self) -> "_DVCFileSystem": def fs(self) -> "_DVCFileSystem":
return _DVCFileSystem(**self.fs_args) return _DVCFileSystem(**self.fs_args)
def getcwd(self):
return self.fs.getcwd()
@property @property
def fsid(self) -> str: def fsid(self) -> str:
return self.fs.fsid return self.fs.fsid
@@ -505,10 +542,6 @@ class DVCFileSystem(FileSystem):
def isdvc(self, path, **kwargs) -> bool: def isdvc(self, path, **kwargs) -> bool:
return self.fs.isdvc(path, **kwargs) return self.fs.isdvc(path, **kwargs)
@property
def path(self) -> Path:
return self.fs.path
@property @property
def repo(self) -> "Repo": def repo(self) -> "Repo":
return self.fs.repo return self.fs.repo
+5 -3
View File
@@ -46,9 +46,11 @@ class GitFileSystem(FileSystem):
return FsspecGitFileSystem(**self.fs_args) return FsspecGitFileSystem(**self.fs_args)
@functools.cached_property def getcwd(self):
def path(self): return self.fs.getcwd()
return self.fs.path
def chdir(self, path):
self.fs.chdir(path)
@property @property
def rev(self) -> str: def rev(self) -> str:
+21 -21
View File
@@ -51,8 +51,8 @@ class DvcIgnorePatterns(DvcIgnore):
@classmethod @classmethod
def from_file(cls, path, fs, name): def from_file(cls, path, fs, name):
assert fs.path.isabs(path) assert fs.isabs(path)
dirname = fs.path.normpath(fs.path.dirname(path)) dirname = fs.normpath(fs.dirname(path))
with fs.open(path, encoding="utf-8") as fobj: with fs.open(path, encoding="utf-8") as fobj:
path_spec_lines = [ path_spec_lines = [
PatternInfo(line, f"{name}:{line_no + 1}:{line}") PatternInfo(line, f"{name}:{line_no + 1}:{line}")
@@ -194,7 +194,7 @@ class DvcIgnoreFilter:
) )
def _get_key(self, path): def _get_key(self, path):
parts = self.fs.path.relparts(path, self.root_dir) parts = self.fs.relparts(path, self.root_dir)
if parts == (os.curdir,): if parts == (os.curdir,):
return () return ()
return parts return parts
@@ -204,13 +204,13 @@ class DvcIgnoreFilter:
old_pattern = trie.longest_prefix(key).value old_pattern = trie.longest_prefix(key).value
matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False) matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
path = self.fs.path.join(dirname, DvcIgnore.DVCIGNORE_FILE) path = self.fs.join(dirname, DvcIgnore.DVCIGNORE_FILE)
if not matches and self.fs.exists(path): if not matches and self.fs.exists(path):
name = self.fs.path.relpath(path, self.root_dir) name = self.fs.relpath(path, self.root_dir)
new_pattern = DvcIgnorePatterns.from_file(path, self.fs, name) new_pattern = DvcIgnorePatterns.from_file(path, self.fs, name)
if old_pattern: if old_pattern:
plist, prefix = merge_patterns( plist, prefix = merge_patterns(
self.fs.path.flavour, self.fs.flavour,
old_pattern.pattern_list, old_pattern.pattern_list,
old_pattern.dirname, old_pattern.dirname,
new_pattern.pattern_list, new_pattern.pattern_list,
@@ -239,7 +239,7 @@ class DvcIgnoreFilter:
dnames = [] dnames = []
for dname in dnames: for dname in dnames:
self._update_sub_repo(self.fs.path.join(dirname, dname), ignore_trie) self._update_sub_repo(self.fs.join(dirname, dname), ignore_trie)
def _update_sub_repo(self, path, ignore_trie: Trie): def _update_sub_repo(self, path, ignore_trie: Trie):
from dvc.repo import Repo from dvc.repo import Repo
@@ -247,18 +247,18 @@ class DvcIgnoreFilter:
if path == self.root_dir: if path == self.root_dir:
return return
dvc_dir = self.fs.path.join(path, Repo.DVC_DIR) dvc_dir = self.fs.join(path, Repo.DVC_DIR)
if not self.fs.exists(dvc_dir): if not self.fs.exists(dvc_dir):
return return
root, dname = self.fs.path.split(path) root, dname = self.fs.split(path)
key = self._get_key(root) key = self._get_key(root)
pattern_info = PatternInfo(f"/{dname}/", f"in sub_repo:{dname}") pattern_info = PatternInfo(f"/{dname}/", f"in sub_repo:{dname}")
new_pattern = DvcIgnorePatterns([pattern_info], root, self.fs.sep) new_pattern = DvcIgnorePatterns([pattern_info], root, self.fs.sep)
old_pattern = ignore_trie.longest_prefix(key).value old_pattern = ignore_trie.longest_prefix(key).value
if old_pattern: if old_pattern:
plist, prefix = merge_patterns( plist, prefix = merge_patterns(
self.fs.path.flavour, self.fs.flavour,
old_pattern.pattern_list, old_pattern.pattern_list,
old_pattern.dirname, old_pattern.dirname,
new_pattern.pattern_list, new_pattern.pattern_list,
@@ -269,7 +269,7 @@ class DvcIgnoreFilter:
ignore_trie[key] = new_pattern ignore_trie[key] = new_pattern
def __call__(self, root, dirs, files, ignore_subrepos=True): def __call__(self, root, dirs, files, ignore_subrepos=True):
abs_root = self.fs.path.abspath(root) abs_root = self.fs.abspath(root)
ignore_pattern = self._get_trie_pattern( ignore_pattern = self._get_trie_pattern(
abs_root, dnames=dirs, ignore_subrepos=ignore_subrepos abs_root, dnames=dirs, ignore_subrepos=ignore_subrepos
) )
@@ -283,7 +283,7 @@ class DvcIgnoreFilter:
nondirs = [] nondirs = []
for entry in fs.ls(path, detail=True, **kwargs): for entry in fs.ls(path, detail=True, **kwargs):
name = fs.path.name(entry["name"]) name = fs.name(entry["name"])
fs_dict[name] = entry fs_dict[name] = entry
if entry["type"] == "directory": if entry["type"] == "directory":
dirs.append(name) dirs.append(name)
@@ -338,7 +338,7 @@ class DvcIgnoreFilter:
else: else:
ignores_trie = self._ignores_trie_subrepos ignores_trie = self._ignores_trie_subrepos
if not self.fs.path.isin_or_eq(dirname, self.root_dir): if not self.fs.isin_or_eq(dirname, self.root_dir):
# outside of the repo # outside of the repo
return None return None
@@ -349,12 +349,12 @@ class DvcIgnoreFilter:
return ignore_pattern return ignore_pattern
prefix_key = ignores_trie.longest_prefix(key).key or () prefix_key = ignores_trie.longest_prefix(key).key or ()
prefix = self.fs.path.join(self.root_dir, *prefix_key) prefix = self.fs.join(self.root_dir, *prefix_key)
dirs = list( dirs = list(
takewhile( takewhile(
lambda path: path != prefix, lambda path: path != prefix,
(parent for parent in localfs.path.parents(dirname)), (parent for parent in localfs.parents(dirname)),
) )
) )
dirs.reverse() dirs.reverse()
@@ -370,7 +370,7 @@ class DvcIgnoreFilter:
): ):
if self._outside_repo(path): if self._outside_repo(path):
return False return False
dirname, basename = self.fs.path.split(self.fs.path.normpath(path)) dirname, basename = self.fs.split(self.fs.normpath(path))
ignore_pattern = self._get_trie_pattern(dirname, None, ignore_subrepos) ignore_pattern = self._get_trie_pattern(dirname, None, ignore_subrepos)
if ignore_pattern: if ignore_pattern:
return ignore_pattern.matches(dirname, basename, is_dir) return ignore_pattern.matches(dirname, basename, is_dir)
@@ -378,7 +378,7 @@ class DvcIgnoreFilter:
def is_ignored_dir(self, path: str, ignore_subrepos: bool = True) -> bool: def is_ignored_dir(self, path: str, ignore_subrepos: bool = True) -> bool:
# only used in LocalFileSystem # only used in LocalFileSystem
path = self.fs.path.abspath(path) path = self.fs.abspath(path)
if path == self.root_dir: if path == self.root_dir:
return False return False
@@ -386,18 +386,18 @@ class DvcIgnoreFilter:
def is_ignored_file(self, path: str, ignore_subrepos: bool = True) -> bool: def is_ignored_file(self, path: str, ignore_subrepos: bool = True) -> bool:
# only used in LocalFileSystem # only used in LocalFileSystem
path = self.fs.path.abspath(path) path = self.fs.abspath(path)
return self._is_ignored(path, False, ignore_subrepos=ignore_subrepos) return self._is_ignored(path, False, ignore_subrepos=ignore_subrepos)
def _outside_repo(self, path): def _outside_repo(self, path):
return not self.fs.path.isin_or_eq(path, self.root_dir) return not self.fs.isin_or_eq(path, self.root_dir)
def check_ignore(self, target): def check_ignore(self, target):
# NOTE: can only be used in `dvc check-ignore`, see # NOTE: can only be used in `dvc check-ignore`, see
# https://github.com/iterative/dvc/issues/5046 # https://github.com/iterative/dvc/issues/5046
full_target = self.fs.path.abspath(target) full_target = self.fs.abspath(target)
if not self._outside_repo(full_target): if not self._outside_repo(full_target):
dirname, basename = self.fs.path.split(self.fs.path.normpath(full_target)) dirname, basename = self.fs.split(self.fs.normpath(full_target))
pattern = self._get_trie_pattern(dirname) pattern = self._get_trie_pattern(dirname)
if pattern: if pattern:
matches = pattern.matches( matches = pattern.matches(
+22 -28
View File
@@ -378,7 +378,7 @@ class Output:
if ( if (
self.repo self.repo
and self.fs.protocol == "local" and self.fs.protocol == "local"
and not self.fs.path.isabs(self.def_path) and not self.fs.isabs(self.def_path)
): ):
self.fs = self.repo.fs self.fs = self.repo.fs
@@ -411,7 +411,7 @@ class Output:
self.remote = remote self.remote = remote
if self.fs.version_aware: if self.fs.version_aware:
_, version_id = self.fs.path.coalesce_version( _, version_id = self.fs.coalesce_version(
self.def_path, self.meta.version_id self.def_path, self.meta.version_id
) )
self.meta.version_id = version_id self.meta.version_id = version_id
@@ -465,7 +465,7 @@ class Output:
parsed.scheme != "remote" parsed.scheme != "remote"
and self.stage and self.stage
and self.stage.repo.fs == fs and self.stage.repo.fs == fs
and not fs.path.isabs(fs_path) and not fs.isabs(fs_path)
): ):
# NOTE: we can path either from command line or .dvc file, # NOTE: we can path either from command line or .dvc file,
# so we should expect both posix and windows style paths. # so we should expect both posix and windows style paths.
@@ -473,9 +473,9 @@ class Output:
# #
# FIXME: if we have Windows path containing / or posix one with \ # FIXME: if we have Windows path containing / or posix one with \
# then we have #2059 bug and can't really handle that. # then we have #2059 bug and can't really handle that.
fs_path = fs.path.join(self.stage.wdir, fs_path) fs_path = fs.join(self.stage.wdir, fs_path)
return fs.path.abspath(fs.path.normpath(fs_path)) return fs.abspath(fs.normpath(fs_path))
def __repr__(self): def __repr__(self):
return f"{type(self).__name__}: {self.def_path!r}" return f"{type(self).__name__}: {self.def_path!r}"
@@ -491,14 +491,14 @@ class Output:
): ):
return str(self.def_path) return str(self.def_path)
if not self.fs.path.isin(self.fs_path, self.repo.root_dir): if not self.fs.isin(self.fs_path, self.repo.root_dir):
return self.fs_path return self.fs_path
cur_dir = self.fs.path.getcwd() cur_dir = self.fs.getcwd()
if self.fs.path.isin(cur_dir, self.repo.root_dir): if self.fs.isin(cur_dir, self.repo.root_dir):
return self.fs.path.relpath(self.fs_path, cur_dir) return self.fs.relpath(self.fs_path, cur_dir)
return self.fs.path.relpath(self.fs_path, self.repo.root_dir) return self.fs.relpath(self.fs_path, self.repo.root_dir)
def clear(self): def clear(self):
self.hash_info = HashInfo.from_dict({}) self.hash_info = HashInfo.from_dict({})
@@ -515,10 +515,10 @@ class Output:
if urlparse(self.def_path).scheme == "remote": if urlparse(self.def_path).scheme == "remote":
return False return False
if self.fs.path.isabs(self.def_path): if self.fs.isabs(self.def_path):
return False return False
return self.repo and self.fs.path.isin( return self.repo and self.fs.isin(
self.fs_path, self.fs_path,
self.repo.root_dir, self.repo.root_dir,
) )
@@ -607,11 +607,11 @@ class Output:
def index_key(self) -> Tuple[str, "DataIndexKey"]: def index_key(self) -> Tuple[str, "DataIndexKey"]:
if self.is_in_repo: if self.is_in_repo:
workspace = "repo" workspace = "repo"
key = self.repo.fs.path.relparts(self.fs_path, self.repo.root_dir) key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
else: else:
workspace = self.fs.protocol workspace = self.fs.protocol
no_drive = self.fs.path.flavour.splitdrive(self.fs_path)[1] no_drive = self.fs.flavour.splitdrive(self.fs_path)[1]
key = self.fs.path.parts(no_drive)[1:] key = self.fs.parts(no_drive)[1:]
return workspace, key return workspace, key
def changed_checksum(self): def changed_checksum(self):
@@ -799,7 +799,7 @@ class Output:
callback=cb, callback=cb,
) )
if relink: if relink:
rel = self.fs.path.relpath(filter_info or self.fs_path) rel = self.fs.relpath(filter_info or self.fs_path)
with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb: with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb:
self._checkout( self._checkout(
filter_info or self.fs_path, filter_info or self.fs_path,
@@ -814,7 +814,7 @@ class Output:
self.set_exec() self.set_exec()
def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]: def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]:
prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path)) prefix = self.fs.parts(self.fs.relpath(filter_info, self.fs_path))
staging, _, obj = self._build( staging, _, obj = self._build(
self.cache, self.cache,
self.fs_path, self.fs_path,
@@ -862,7 +862,7 @@ class Output:
ret.update(split_file_meta_from_cloud(meta_d)) ret.update(split_file_meta_from_cloud(meta_d))
if self.is_in_repo: if self.is_in_repo:
path = self.fs.path.as_posix(relpath(self.fs_path, self.stage.wdir)) path = self.fs.as_posix(relpath(self.fs_path, self.stage.wdir))
else: else:
path = self.def_path path = self.def_path
@@ -949,7 +949,7 @@ class Output:
return None return None
assert obj assert obj
fs_path = self.fs.path fs_path = self.fs
if filter_info and filter_info != self.fs_path: if filter_info and filter_info != self.fs_path:
prefix = fs_path.relparts(filter_info, self.fs_path) prefix = fs_path.relparts(filter_info, self.fs_path)
assert isinstance(obj, Tree) assert isinstance(obj, Tree)
@@ -1154,7 +1154,7 @@ class Output:
assert obj is None or isinstance(obj, Tree) assert obj is None or isinstance(obj, Tree)
if filter_info and filter_info != self.fs_path: if filter_info and filter_info != self.fs_path:
assert obj assert obj
prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path)) prefix = self.fs.parts(self.fs.relpath(filter_info, self.fs_path))
return obj.filter(prefix) return obj.filter(prefix)
return obj return obj
@@ -1297,10 +1297,7 @@ class Output:
def unstage(self, path: str) -> Tuple["Meta", "Tree"]: def unstage(self, path: str) -> Tuple["Meta", "Tree"]:
from pygtrie import Trie from pygtrie import Trie
from dvc_objects.fs.path import Path rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path)))
assert isinstance(self.fs.path, Path)
rel_key = tuple(self.fs.path.parts(self.fs.path.relpath(path, self.fs_path)))
if self.hash_info: if self.hash_info:
tree = self.get_dir_cache() tree = self.get_dir_cache()
@@ -1333,11 +1330,8 @@ class Output:
) -> Tuple["Meta", "Tree"]: ) -> Tuple["Meta", "Tree"]:
from pygtrie import Trie from pygtrie import Trie
from dvc_objects.fs.path import Path
assert isinstance(self.fs.path, Path)
append_only = True append_only = True
rel_key = tuple(self.fs.path.parts(self.fs.path.relpath(path, self.fs_path))) rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path)))
if self.hash_info: if self.hash_info:
tree = self.get_dir_cache() tree = self.get_dir_cache()
+3 -3
View File
@@ -143,11 +143,11 @@ class DataResolver:
self.parsing_config = repo.config.get("parsing", {}) self.parsing_config = repo.config.get("parsing", {})
if os.path.isabs(wdir): if os.path.isabs(wdir):
wdir = fs.path.relpath(wdir) wdir = fs.relpath(wdir)
wdir = "" if wdir == os.curdir else wdir wdir = "" if wdir == os.curdir else wdir
self.wdir = wdir self.wdir = wdir
self.relpath = fs.path.normpath(fs.path.join(self.wdir, "dvc.yaml")) self.relpath = fs.normpath(fs.join(self.wdir, "dvc.yaml"))
vars_ = d.get(VARS_KWD, []) vars_ = d.get(VARS_KWD, [])
check_interpolations(vars_, VARS_KWD, self.relpath) check_interpolations(vars_, VARS_KWD, self.relpath)
@@ -248,7 +248,7 @@ class EntryDefinition:
wdir = to_str(context.resolve_str(wdir)) wdir = to_str(context.resolve_str(wdir))
except (ContextError, ParseError) as exc: except (ContextError, ParseError) as exc:
format_and_raise(exc, f"'{self.where}.{name}.wdir'", self.relpath) format_and_raise(exc, f"'{self.where}.{name}.wdir'", self.relpath)
return self.resolver.fs.path.join(self.wdir, wdir) return self.resolver.fs.join(self.wdir, wdir)
def resolve(self, **kwargs): def resolve(self, **kwargs):
try: try:
+2 -2
View File
@@ -384,7 +384,7 @@ class Context(CtxDict):
def merge_from(self, fs, item: str, wdir: str, overwrite=False): def merge_from(self, fs, item: str, wdir: str, overwrite=False):
path, _, keys_str = item.partition(":") path, _, keys_str = item.partition(":")
path = fs.path.normpath(fs.path.join(wdir, path)) path = fs.normpath(fs.join(wdir, path))
select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
if path in self.imports: if path in self.imports:
@@ -429,7 +429,7 @@ class Context(CtxDict):
default: Optional[str] = None, default: Optional[str] = None,
): ):
if default: if default:
to_import = fs.path.join(wdir, default) to_import = fs.join(wdir, default)
if fs.exists(to_import): if fs.exists(to_import):
self.merge_from(fs, default, wdir) self.merge_from(fs, default, wdir)
else: else:
+16 -16
View File
@@ -114,7 +114,7 @@ class Repo:
try: try:
root_dir = self.find_root(root_dir, fs) root_dir = self.find_root(root_dir, fs)
fs = fs or localfs fs = fs or localfs
dvc_dir = fs.path.join(root_dir, self.DVC_DIR) dvc_dir = fs.join(root_dir, self.DVC_DIR)
except NotDvcRepoError: except NotDvcRepoError:
if not uninitialized: if not uninitialized:
raise raise
@@ -207,7 +207,7 @@ class Repo:
self.fs.makedirs(self.tmp_dir, exist_ok=True) self.fs.makedirs(self.tmp_dir, exist_ok=True)
self.lock = make_lock( self.lock = make_lock(
self.fs.path.join(self.tmp_dir, "lock"), self.fs.join(self.tmp_dir, "lock"),
tmp_dir=self.tmp_dir, tmp_dir=self.tmp_dir,
hardlink_lock=self.config["core"].get("hardlink_lock", False), hardlink_lock=self.config["core"].get("hardlink_lock", False),
friendly=True, friendly=True,
@@ -251,7 +251,7 @@ class Repo:
) )
@cached_property @cached_property
def local_dvc_dir(self): def local_dvc_dir(self) -> Optional[str]:
from dvc.fs import GitFileSystem, LocalFileSystem from dvc.fs import GitFileSystem, LocalFileSystem
if not self.dvc_dir: if not self.dvc_dir:
@@ -263,10 +263,10 @@ class Repo:
if not isinstance(self.fs, GitFileSystem): if not isinstance(self.fs, GitFileSystem):
return None return None
relparts = () relparts: Tuple[str, ...] = ()
if self.root_dir != "/": if self.root_dir != "/":
# subrepo # subrepo
relparts = self.fs.path.relparts(self.root_dir, "/") relparts = self.fs.relparts(self.root_dir, "/")
dvc_dir = os.path.join( dvc_dir = os.path.join(
self.scm.root_dir, self.scm.root_dir,
@@ -395,7 +395,7 @@ class Repo:
index = subrepo.index.data[workspace] index = subrepo.index.data[workspace]
else: else:
index = self.index.data[workspace] index = self.index.data[workspace]
key = self.fs.path.relparts(path, self.root_dir) key = self.fs.relparts(path, self.root_dir)
try: try:
return index, index[key] return index, index[key]
@@ -411,18 +411,18 @@ class Repo:
fs = fs or localfs fs = fs or localfs
root = root or os.curdir root = root or os.curdir
root_dir = fs.path.abspath(root) root_dir = fs.abspath(root)
if not fs.isdir(root_dir): if not fs.isdir(root_dir):
raise NotDvcRepoError(f"directory '{root}' does not exist") raise NotDvcRepoError(f"directory '{root}' does not exist")
while True: while True:
dvc_dir = fs.path.join(root_dir, cls.DVC_DIR) dvc_dir = fs.join(root_dir, cls.DVC_DIR)
if fs.isdir(dvc_dir): if fs.isdir(dvc_dir):
return root_dir return root_dir
if isinstance(fs, LocalFileSystem) and os.path.ismount(root_dir): if isinstance(fs, LocalFileSystem) and os.path.ismount(root_dir):
break break
parent = fs.path.parent(root_dir) parent = fs.parent(root_dir)
if parent == root_dir: if parent == root_dir:
break break
root_dir = parent root_dir = parent
@@ -440,7 +440,7 @@ class Repo:
fs = fs or localfs fs = fs or localfs
root_dir = cls.find_root(root, fs=fs) root_dir = cls.find_root(root, fs=fs)
return fs.path.join(root_dir, cls.DVC_DIR) return fs.join(root_dir, cls.DVC_DIR)
@staticmethod @staticmethod
def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False) -> "Repo": def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False) -> "Repo":
@@ -542,19 +542,19 @@ class Repo:
# using `outs_graph` to ensure graph checks are run # using `outs_graph` to ensure graph checks are run
outs = outs or self.index.outs_graph outs = outs or self.index.outs_graph
abs_path = self.fs.path.abspath(path) abs_path = self.fs.abspath(path)
fs_path = abs_path fs_path = abs_path
def func(out): def func(out):
def eq(one, two): def eq(one, two):
return one == two return one == two
match = eq if strict else out.fs.path.isin_or_eq match = eq if strict else out.fs.isin_or_eq
if out.protocol == "local" and match(fs_path, out.fs_path): if out.protocol == "local" and match(fs_path, out.fs_path):
return True return True
if recursive and out.fs.path.isin(out.fs_path, fs_path): if recursive and out.fs.isin(out.fs_path, fs_path):
return True return True
return False return False
@@ -566,7 +566,7 @@ class Repo:
return matched return matched
def is_dvc_internal(self, path): def is_dvc_internal(self, path):
path_parts = self.fs.path.normpath(path).split(self.fs.sep) path_parts = self.fs.normpath(path).split(self.fs.sep)
return self.DVC_DIR in path_parts return self.DVC_DIR in path_parts
@cached_property @cached_property
@@ -613,10 +613,10 @@ class Repo:
cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir() cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
if isinstance(self.fs, GitFileSystem): if isinstance(self.fs, GitFileSystem):
relparts = () relparts: Tuple[str, ...] = ()
if self.root_dir != "/": if self.root_dir != "/":
# subrepo # subrepo
relparts = self.fs.path.relparts(self.root_dir, "/") relparts = self.fs.relparts(self.root_dir, "/")
root_dir = os.path.join(self.scm.root_dir, *relparts) root_dir = os.path.join(self.scm.root_dir, *relparts)
else: else:
root_dir = self.root_dir root_dir = self.root_dir
+1 -1
View File
@@ -181,7 +181,7 @@ def _add_transfer(
def _add(stage: "Stage", source: Optional[str] = None, no_commit: bool = False) -> None: def _add(stage: "Stage", source: Optional[str] = None, no_commit: bool = False) -> None:
out = stage.outs[0] out = stage.outs[0]
path = out.fs.path.abspath(source) if source else None path = out.fs.abspath(source) if source else None
try: try:
stage.add_outs(path, no_commit=no_commit) stage.add_outs(path, no_commit=no_commit)
except CacheLinkError: except CacheLinkError:
+2 -2
View File
@@ -229,7 +229,7 @@ class Artifacts:
dvc_studio_config=dvc_studio_config, dvc_studio_config=dvc_studio_config,
**kwargs, **kwargs,
).items(): ).items():
to_info = localfs.path.join(out, path) to_info = localfs.join(out, path)
if localfs.exists(to_info) and not force: if localfs.exists(to_info) and not force:
hint = "\nTo override it, re-run with '--force'." hint = "\nTo override it, re-run with '--force'."
raise FileExistsLocallyError( # noqa: TRY301 raise FileExistsLocallyError( # noqa: TRY301
@@ -254,7 +254,7 @@ class Artifacts:
fs, from_infos, localfs, to_infos, callback=cb, batch_size=jobs fs, from_infos, localfs, to_infos, callback=cb, batch_size=jobs
) )
return len(to_infos), relpath(localfs.path.commonpath(to_infos)) return len(to_infos), relpath(localfs.commonpath(to_infos))
@classmethod @classmethod
def get( # noqa: PLR0913 def get( # noqa: PLR0913
+12 -12
View File
@@ -60,12 +60,12 @@ def brancher(
from dvc.fs import LocalFileSystem from dvc.fs import LocalFileSystem
repo_root_parts: Tuple[str, ...] = () repo_root_parts: Tuple[str, ...] = ()
if self.fs.path.isin(self.root_dir, self.scm.root_dir): if self.fs.isin(self.root_dir, self.scm.root_dir):
repo_root_parts = self.fs.path.relparts(self.root_dir, self.scm.root_dir) repo_root_parts = self.fs.relparts(self.root_dir, self.scm.root_dir)
cwd_parts: Tuple[str, ...] = () cwd_parts: Tuple[str, ...] = ()
if self.fs.path.isin(self.fs.path.getcwd(), self.scm.root_dir): if self.fs.isin(self.fs.getcwd(), self.scm.root_dir):
cwd_parts = self.fs.path.relparts(self.fs.path.getcwd(), self.scm.root_dir) cwd_parts = self.fs.relparts(self.fs.getcwd(), self.scm.root_dir)
saved_fs = self.fs saved_fs = self.fs
saved_root = self.root_dir saved_root = self.root_dir
@@ -125,18 +125,18 @@ def _switch_fs(
logger.trace("switching fs to revision %s", rev[:7]) logger.trace("switching fs to revision %s", rev[:7])
assert isinstance(repo.scm, Git) assert isinstance(repo.scm, Git)
fs = GitFileSystem(scm=repo.scm, rev=rev) fs = GitFileSystem(scm=repo.scm, rev=rev)
root_dir = repo.fs.path.join("/", *repo_root_parts) root_dir = repo.fs.join("/", *repo_root_parts)
if not fs.exists(root_dir): if not fs.exists(root_dir):
raise NotDvcRepoError(f"Commit '{rev[:7]}' does not contain a DVC repo") raise NotDvcRepoError(f"Commit '{rev[:7]}' does not contain a DVC repo")
repo.fs = fs repo.fs = fs
repo.root_dir = root_dir repo.root_dir = root_dir
repo.dvc_dir = fs.path.join(root_dir, repo.DVC_DIR) repo.dvc_dir = fs.join(root_dir, repo.DVC_DIR)
repo._reset() repo._reset()
if cwd_parts: if cwd_parts:
cwd = repo.fs.path.join("/", *cwd_parts) cwd = repo.fs.join("/", *cwd_parts)
repo.fs.path.chdir(cwd) repo.fs.chdir(cwd)
@contextmanager @contextmanager
@@ -148,12 +148,12 @@ def switch(repo: "Repo", rev: str) -> Iterator[str]:
rev = resolve_rev(repo.scm, rev) rev = resolve_rev(repo.scm, rev)
repo_root_parts: Tuple[str, ...] = () repo_root_parts: Tuple[str, ...] = ()
if repo.fs.path.isin(repo.root_dir, repo.scm.root_dir): if repo.fs.isin(repo.root_dir, repo.scm.root_dir):
repo_root_parts = repo.fs.path.relparts(repo.root_dir, repo.scm.root_dir) repo_root_parts = repo.fs.relparts(repo.root_dir, repo.scm.root_dir)
cwd_parts: Tuple[str, ...] = () cwd_parts: Tuple[str, ...] = ()
if repo.fs.path.isin(repo.fs.path.getcwd(), repo.scm.root_dir): if repo.fs.isin(repo.fs.getcwd(), repo.scm.root_dir):
cwd_parts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.scm.root_dir) cwd_parts = repo.fs.relparts(repo.fs.getcwd(), repo.scm.root_dir)
saved_fs = repo.fs saved_fs = repo.fs
saved_root = repo.root_dir saved_root = repo.root_dir
+3 -3
View File
@@ -81,7 +81,7 @@ def _check_can_delete(
if cache_fs.exists(cache_path): if cache_fs.exists(cache_path):
continue continue
entry_paths.append(fs.path.join(path, *(entry.key or ()))) entry_paths.append(fs.join(path, *(entry.key or ())))
if not entry_paths: if not entry_paths:
return return
@@ -172,7 +172,7 @@ def checkout( # noqa: C901
) )
for out_path in out_paths: for out_path in out_paths:
if self.fs.path.isin_or_eq(dest_path, out_path): if self.fs.isin_or_eq(dest_path, out_path):
failed.add(out_path) failed.add(out_path)
with ui.progress( with ui.progress(
@@ -195,7 +195,7 @@ def checkout( # noqa: C901
typ_map = {ADD: "added", DELETE: "deleted", MODIFY: "modified"} typ_map = {ADD: "added", DELETE: "deleted", MODIFY: "modified"}
for key, typ in out_changes.items(): for key, typ in out_changes.items():
out_path = self.fs.path.join(self.root_dir, *key) out_path = self.fs.join(self.root_dir, *key)
if out_path in failed: if out_path in failed:
self.fs.remove(out_path, recursive=True) self.fs.remove(out_path, recursive=True)
+3 -3
View File
@@ -205,7 +205,7 @@ class Status(TypedDict):
def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]: def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]:
"""Transform files rel. to Git root to DVC root, and drop outside files.""" """Transform files rel. to Git root to DVC root, and drop outside files."""
rel = repo.fs.path.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/") rel = repo.fs.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
# if we have repo root in a different location than scm's root, # if we have repo root in a different location than scm's root,
# i.e. subdir repo, all git_paths need to be transformed rel. to the DVC # i.e. subdir repo, all git_paths need to be transformed rel. to the DVC
@@ -215,11 +215,11 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]
length = len(prefix) length = len(prefix)
files = (file[length:] for file in files if file.startswith(prefix)) files = (file[length:] for file in files if file.startswith(prefix))
start = repo.fs.path.relpath(repo.fs.path.getcwd(), repo.root_dir) start = repo.fs.relpath(repo.fs.getcwd(), repo.root_dir)
if start in (os.curdir, ""): if start in (os.curdir, ""):
return list(files) return list(files)
# we need to convert repo relative paths to curdir relative. # we need to convert repo relative paths to curdir relative.
return [repo.fs.path.relpath(file, start) for file in files] return [repo.fs.relpath(file, start) for file in files]
def status( def status(
+1 -1
View File
@@ -171,7 +171,7 @@ class ApplyStash(Stash):
right_fs = self.scm.get_fs(right_rev) right_fs = self.scm.get_fs(right_rev)
paths = [path for path in left_fs.find("/") if not right_fs.exists(path)] paths = [path for path in left_fs.find("/") if not right_fs.exists(path)]
dest_paths = [ dest_paths = [
localfs.path.join(self.scm.root_dir, left_fs.path.relpath(path, "/")) localfs.join(self.scm.root_dir, left_fs.relpath(path, "/"))
for path in paths for path in paths
] ]
for src, dest in zip(paths, dest_paths): for src, dest in zip(paths, dest_paths):
+1 -1
View File
@@ -219,7 +219,7 @@ def _log_unversioned(data: List["DataIndex"]) -> Tuple[List["DataIndex"], int]:
index.storage_map = fs_index.storage_map index.storage_map = fs_index.storage_map
for key, entry in fs_index.iteritems(): for key, entry in fs_index.iteritems():
if entry.meta and not entry.meta.isdir and entry.meta.version_id is None: if entry.meta and not entry.meta.isdir and entry.meta.version_id is None:
unversioned.append(fs.unstrip_protocol(fs.path.join(remote.path, *key))) unversioned.append(fs.unstrip_protocol(fs.join(remote.path, *key)))
else: else:
index[key] = entry index[key] = entry
fs_index.close() fs_index.close()
+3 -3
View File
@@ -126,7 +126,7 @@ def build_graph(stages, outs_trie=None):
outs_trie = outs_trie or build_outs_trie(stages) outs_trie = outs_trie or build_outs_trie(stages)
for stage in stages: for stage in stages:
out = outs_trie.shortest_prefix(localfs.path.parts(stage.path)).value out = outs_trie.shortest_prefix(localfs.parts(stage.path)).value
if out: if out:
raise StagePathAsOutputError(stage, str(out)) raise StagePathAsOutputError(stage, str(out))
@@ -139,7 +139,7 @@ def build_graph(stages, outs_trie=None):
continue continue
for dep in stage.deps: for dep in stage.deps:
dep_key = dep.fs.path.parts(dep.fs_path) dep_key = dep.fs.parts(dep.fs_path)
overlapping = [n.value for n in outs_trie.prefixes(dep_key)] overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
if outs_trie.has_subtrie(dep_key): if outs_trie.has_subtrie(dep_key):
overlapping.extend(outs_trie.values(prefix=dep_key)) overlapping.extend(outs_trie.values(prefix=dep_key))
@@ -165,7 +165,7 @@ def build_outs_graph(graph, outs_trie):
if stage.is_db_import: if stage.is_db_import:
continue continue
for dep in stage.deps: for dep in stage.deps:
dep_key = dep.fs.path.parts(dep.fs_path) dep_key = dep.fs.parts(dep.fs_path)
overlapping = [n.value for n in outs_trie.prefixes(dep_key)] overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
if outs_trie.has_subtrie(dep_key): if outs_trie.has_subtrie(dep_key):
overlapping.extend(outs_trie.values(prefix=dep_key)) overlapping.extend(outs_trie.values(prefix=dep_key))
+11 -11
View File
@@ -97,7 +97,7 @@ def collect_files(
for root, dirs, files in walk_iter: for root, dirs, files in walk_iter:
dvcfile_filter = partial(is_dvcfile_and_not_ignored, root) dvcfile_filter = partial(is_dvcfile_and_not_ignored, root)
for file in filter(dvcfile_filter, files): for file in filter(dvcfile_filter, files):
file_path = fs.path.join(root, file) file_path = fs.join(root, file)
try: try:
index = Index.from_file(repo, file_path) index = Index.from_file(repo, file_path)
except DvcException as exc: except DvcException as exc:
@@ -201,7 +201,7 @@ def _load_storage_from_out(storage_map, key, out):
FileStorage( FileStorage(
key, key,
fs_cache.fs, fs_cache.fs,
fs_cache.fs.path.join( fs_cache.fs.join(
fs_cache.path, dep.fs.protocol, tokenize(dep.fs_path) fs_cache.path, dep.fs.protocol, tokenize(dep.fs_path)
), ),
) )
@@ -414,7 +414,7 @@ class Index:
by_workspace[workspace].add(key) by_workspace[workspace].add(key)
for path in _collect_top_level_metrics(self.repo): for path in _collect_top_level_metrics(self.repo):
key = self.repo.fs.path.relparts(path, self.repo.root_dir) key = self.repo.fs.relparts(path, self.repo.root_dir)
by_workspace["repo"].add(key) by_workspace["repo"].add(key)
return dict(by_workspace) return dict(by_workspace)
@@ -432,7 +432,7 @@ class Index:
param_paths = chain(param_paths, [default_file]) param_paths = chain(param_paths, [default_file])
for path in param_paths: for path in param_paths:
key = self.repo.fs.path.relparts(path, self.repo.root_dir) key = self.repo.fs.relparts(path, self.repo.root_dir)
by_workspace["repo"].add(key) by_workspace["repo"].add(key)
return dict(by_workspace) return dict(by_workspace)
@@ -451,7 +451,7 @@ class Index:
by_workspace[workspace].add(key) by_workspace[workspace].add(key)
for path in self._plot_sources: for path in self._plot_sources:
key = self.repo.fs.path.parts(path) key = self.repo.fs.parts(path)
by_workspace["repo"].add(key) by_workspace["repo"].add(key)
return dict(by_workspace) return dict(by_workspace)
@@ -701,8 +701,8 @@ class IndexView:
if not out.use_cache: if not out.use_cache:
continue continue
workspace, key = out.index_key workspace, key = out.index_key
if filter_info and out.fs.path.isin(filter_info, out.fs_path): if filter_info and out.fs.isin(filter_info, out.fs_path):
key = key + out.fs.path.relparts(filter_info, out.fs_path) key = key + out.fs.relparts(filter_info, out.fs_path)
entry = self._index.data[workspace].get(key) entry = self._index.data[workspace].get(key)
if entry and entry.meta and entry.meta.isdir: if entry and entry.meta and entry.meta.isdir:
prefixes[workspace].recursive.add(key) prefixes[workspace].recursive.add(key)
@@ -718,8 +718,8 @@ class IndexView:
continue continue
workspace, key = out.index_key workspace, key = out.index_key
if filter_info and out.fs.path.isin(filter_info, out.fs_path): if filter_info and out.fs.isin(filter_info, out.fs_path):
key = key + out.fs.path.relparts(filter_info, out.fs_path) key = key + out.fs.relparts(filter_info, out.fs_path)
ret[workspace].add(key) ret[workspace].add(key)
return dict(ret) return dict(ret)
@@ -782,7 +782,7 @@ def build_data_index( # noqa: C901, PLR0912
data = DataIndex() data = DataIndex()
parents = set() parents = set()
for key in index.data_keys.get(workspace, set()): for key in index.data_keys.get(workspace, set()):
out_path = fs.path.join(path, *key) out_path = fs.join(path, *key)
for key_len in range(1, len(key)): for key_len in range(1, len(key)):
parents.add(key[:key_len]) parents.add(key[:key_len])
@@ -827,7 +827,7 @@ def build_data_index( # noqa: C901, PLR0912
callback.relative_update(1) callback.relative_update(1)
for key in parents: for key in parents:
parent_path = fs.path.join(path, *key) parent_path = fs.join(path, *key)
if not fs.exists(parent_path): if not fs.exists(parent_path):
continue continue
direntry = DataIndexEntry(key=key, meta=Meta(isdir=True), loaded=True) direntry = DataIndexEntry(key=key, meta=Meta(isdir=True), loaded=True)
+1 -1
View File
@@ -89,7 +89,7 @@ def _ls(
if not recursive: if not recursive:
files.update(dirs) files.update(dirs)
parts = fs.path.relparts(root, fs_path) parts = fs.relparts(root, fs_path)
if parts == (".",): if parts == (".",):
parts = () parts = ()
+1 -1
View File
@@ -18,7 +18,7 @@ def ls_url(url, *, fs_config=None, recursive=False, config=None):
for info in files.values(): for info in files.values():
ls_info = { ls_info = {
"path": fs.path.relpath(info["name"], fs_path), "path": fs.relpath(info["name"], fs_path),
"isdir": info["type"] == "directory", "isdir": info["type"] == "directory",
"size": info.get("size"), "size": info.get("size"),
} }
+7 -7
View File
@@ -35,10 +35,10 @@ logger = logger.getChild(__name__)
def _collect_top_level_metrics(repo: "Repo") -> Iterator[str]: def _collect_top_level_metrics(repo: "Repo") -> Iterator[str]:
top_metrics = repo.index._metrics top_metrics = repo.index._metrics
for dvcfile, metrics in top_metrics.items(): for dvcfile, metrics in top_metrics.items():
wdir = repo.fs.path.relpath(repo.fs.path.parent(dvcfile), repo.root_dir) wdir = repo.fs.relpath(repo.fs.parent(dvcfile), repo.root_dir)
for file in metrics: for file in metrics:
path = repo.fs.path.join(wdir, as_posix(file)) path = repo.fs.join(wdir, as_posix(file))
yield repo.fs.path.normpath(path) yield repo.fs.normpath(path)
def _extract_metrics(metrics, path: str): def _extract_metrics(metrics, path: str):
@@ -101,7 +101,7 @@ def _collect_metrics(
if not targets or outs_only: if not targets or outs_only:
outs = metrics_from_target(repo, stages) if stages else repo.index.metrics outs = metrics_from_target(repo, stages) if stages else repo.index.metrics
relpath = repo.fs.path.relpath relpath = repo.fs.relpath
metrics.extend(relpath(out.fs_path, repo.root_dir) for out in outs) metrics.extend(relpath(out.fs_path, repo.root_dir) for out in outs)
if not targets and not outs_only and not stages: if not targets and not outs_only and not stages:
@@ -128,8 +128,8 @@ class Result(TypedDict, total=False):
def to_relpath(fs: "FileSystem", root_dir: str, d: Result) -> Result: def to_relpath(fs: "FileSystem", root_dir: str, d: Result) -> Result:
relpath = fs.path.relpath relpath = fs.relpath
cwd = fs.path.getcwd() cwd = fs.getcwd()
start = relpath(cwd, root_dir) start = relpath(cwd, root_dir)
data = d.get("data") data = d.get("data")
@@ -156,7 +156,7 @@ def _gather_metrics(
fs = repo.dvcfs fs = repo.dvcfs
for fs_path, result in _read_metrics(fs, files, cache=True): for fs_path, result in _read_metrics(fs, files, cache=True):
repo_path = fs_path.lstrip(fs.root_marker) repo_path = fs_path.lstrip(fs.root_marker)
repo_os_path = os.sep.join(fs.path.parts(repo_path)) repo_os_path = os.sep.join(fs.parts(repo_path))
if not isinstance(result, Exception): if not isinstance(result, Exception):
data.update({repo_os_path: FileResult(data=result)}) data.update({repo_os_path: FileResult(data=result)})
continue continue
+1 -1
View File
@@ -71,7 +71,7 @@ def erepo_factory(url, root_dir, cache_config):
_config = cache_config.copy() _config = cache_config.copy()
if os.path.isdir(url): if os.path.isdir(url):
fs = fs or localfs fs = fs or localfs
repo_path = os.path.join(url, *fs.path.relparts(path, root_dir)) repo_path = os.path.join(url, *fs.relparts(path, root_dir))
_config.update(_get_remote_config(repo_path)) _config.update(_get_remote_config(repo_path))
return Repo(path, fs=fs, config=_config, **_kwargs) return Repo(path, fs=fs, config=_config, **_kwargs)
+6 -6
View File
@@ -21,10 +21,10 @@ logger = logger.getChild(__name__)
def _collect_top_level_params(repo: "Repo") -> Iterator[str]: def _collect_top_level_params(repo: "Repo") -> Iterator[str]:
top_params = repo.index._params top_params = repo.index._params
for dvcfile, params in top_params.items(): for dvcfile, params in top_params.items():
wdir = repo.fs.path.relpath(repo.fs.path.parent(dvcfile), repo.root_dir) wdir = repo.fs.relpath(repo.fs.parent(dvcfile), repo.root_dir)
for file in params: for file in params:
path = repo.fs.path.join(wdir, as_posix(file)) path = repo.fs.join(wdir, as_posix(file))
yield repo.fs.path.normpath(path) yield repo.fs.normpath(path)
def params_from_target( def params_from_target(
@@ -55,7 +55,7 @@ def _collect_params(
if not targets or stages: if not targets or stages:
deps = params_from_target(repo, stages) if stages else repo.index.params deps = params_from_target(repo, stages) if stages else repo.index.params
relpath = repo.fs.path.relpath relpath = repo.fs.relpath
params.extend( params.extend(
{relpath(dep.fs_path, repo.root_dir): list(dep.params)} for dep in deps {relpath(dep.fs_path, repo.root_dir): list(dep.params)} for dep in deps
) )
@@ -93,7 +93,7 @@ def _collect_vars(repo, params, stages=None) -> Dict:
# to reduce noise and duplication, they are skipped # to reduce noise and duplication, they are skipped
# `file` is relative # `file` is relative
abspath = repo.fs.path.abspath(file) abspath = repo.fs.abspath(file)
repo_path = repo.dvcfs.from_os_path(abspath) repo_path = repo.dvcfs.from_os_path(abspath)
if repo_path in params: if repo_path in params:
continue continue
@@ -138,7 +138,7 @@ def _gather_params(
fs = repo.dvcfs fs = repo.dvcfs
for fs_path, result in _read_params(fs, files_keypaths, cache=True): for fs_path, result in _read_params(fs, files_keypaths, cache=True):
repo_path = fs_path.lstrip(fs.root_marker) repo_path = fs_path.lstrip(fs.root_marker)
repo_os_path = os.sep.join(fs.path.parts(repo_path)) repo_os_path = os.sep.join(fs.parts(repo_path))
if not isinstance(result, Exception): if not isinstance(result, Exception):
data.update({repo_os_path: FileResult(data=result)}) data.update({repo_os_path: FileResult(data=result)})
continue continue
+6 -6
View File
@@ -386,7 +386,7 @@ def _matches(targets, config_file, plot_id):
def _normpath(path): def _normpath(path):
# TODO dvcfs.path.normopath normalizes to windows path on Windows # TODO dvcfs.normopath normalizes to windows path on Windows
# even though other methods work as expected # even though other methods work as expected
import posixpath import posixpath
@@ -399,7 +399,7 @@ def _relpath(fs, path):
# and invoking from some subdir `dvcfile.relpath` returns strange long # and invoking from some subdir `dvcfile.relpath` returns strange long
# relative paths # relative paths
# ("../../../../../../dvc.yaml") - investigate # ("../../../../../../dvc.yaml") - investigate
return fs.path.relpath(fs.path.join("/", fs.from_os_path(path)), fs.path.getcwd()) return fs.relpath(fs.join("/", fs.from_os_path(path)), fs.getcwd())
def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None): def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None):
@@ -413,7 +413,7 @@ def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = No
if _matches(targets, config_path, str(plot)): if _matches(targets, config_path, str(plot)):
unpacked = unpack_if_dir( unpacked = unpack_if_dir(
fs, fs,
_normpath(fs.path.join(wdir_relpath, plot.def_path)), _normpath(fs.join(wdir_relpath, plot.def_path)),
props={**plot_props, **props}, props={**plot_props, **props},
onerror=onerror, onerror=onerror,
) )
@@ -438,7 +438,7 @@ def _adjust_sources(fs, plot_props, config_dir):
old_y = new_plot_props.pop("y", {}) old_y = new_plot_props.pop("y", {})
new_y = {} new_y = {}
for filepath, val in old_y.items(): for filepath, val in old_y.items():
new_y[_normpath(fs.path.join(config_dir, filepath))] = val new_y[_normpath(fs.join(config_dir, filepath))] = val
new_plot_props["y"] = new_y new_plot_props["y"] = new_y
return new_plot_props return new_plot_props
@@ -452,13 +452,13 @@ def _resolve_definitions(
onerror: Optional[Callable[[Any], Any]] = None, onerror: Optional[Callable[[Any], Any]] = None,
): ):
config_path = os.fspath(config_path) config_path = os.fspath(config_path)
config_dir = fs.path.dirname(config_path) config_dir = fs.dirname(config_path)
result: Dict[str, Dict] = {} result: Dict[str, Dict] = {}
for plot_id, plot_props in definitions.items(): for plot_id, plot_props in definitions.items():
if plot_props is None: if plot_props is None:
plot_props = {} plot_props = {}
if _id_is_path(plot_props): if _id_is_path(plot_props):
data_path = _normpath(fs.path.join(config_dir, plot_id)) data_path = _normpath(fs.join(config_dir, plot_id))
if _matches(targets, config_path, plot_id): if _matches(targets, config_path, plot_id):
unpacked = unpack_if_dir( unpacked = unpack_if_dir(
fs, fs,
+3 -3
View File
@@ -219,7 +219,7 @@ class StageLoad:
self, path: Optional[str] = None, name: Optional[str] = None self, path: Optional[str] = None, name: Optional[str] = None
) -> str: ) -> str:
if path: if path:
return self.repo.fs.path.abspath(path) return self.repo.fs.abspath(path)
path = PROJECT_FILE path = PROJECT_FILE
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path) logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
@@ -348,7 +348,7 @@ class StageLoad:
if recursive and self.fs.isdir(target): if recursive and self.fs.isdir(target):
from dvc.repo.graph import collect_inside_path from dvc.repo.graph import collect_inside_path
path = self.fs.path.abspath(target) path = self.fs.abspath(target)
return collect_inside_path(path, graph or self.repo.index.graph) return collect_inside_path(path, graph or self.repo.index.graph)
stages = self.from_target(target, glob=glob) stages = self.from_target(target, glob=glob)
@@ -393,7 +393,7 @@ class StageLoad:
if not (recursive and self.fs.isdir(target)): if not (recursive and self.fs.isdir(target)):
try: try:
(out,) = self.repo.find_outs_by_path(target, strict=False) (out,) = self.repo.find_outs_by_path(target, strict=False)
return [StageInfo(out.stage, self.fs.path.abspath(target))] return [StageInfo(out.stage, self.fs.abspath(target))]
except OutputNotFoundError: except OutputNotFoundError:
pass pass
+1 -1
View File
@@ -9,7 +9,7 @@ def build_outs_trie(stages):
for stage in stages: for stage in stages:
for out in stage.outs: for out in stage.outs:
out_key = out.fs.path.parts(out.fs_path) out_key = out.fs.parts(out.fs_path)
# Check for dup outs # Check for dup outs
if out_key in outs: if out_key in outs:
+7 -6
View File
@@ -131,10 +131,11 @@ def _merge_push_meta( # noqa: C901
entries.append(entry) entries.append(entry)
if entry.meta is not None and entry.meta.isdir: if entry.meta is not None and entry.meta.isdir:
continue continue
fs_path = repo.fs.path.join(repo.root_dir, *subkey) fs_path = repo.fs.join(repo.root_dir, *subkey)
meta, hash_info = old_tree.get( meta, hash_info = old_tree.get(repo.fs.relparts(fs_path, out.fs_path)) or (
repo.fs.path.relparts(fs_path, out.fs_path) None,
) or (None, None) None,
)
entry.hash_info = hash_info entry.hash_info = hash_info
if entry.meta: if entry.meta:
entry.meta.remote = remote entry.meta.remote = remote
@@ -354,11 +355,11 @@ def _get_update_diff_index(
# downloading # downloading
if out.isdir(): if out.isdir():
if not entry.meta.isdir: if not entry.meta.isdir:
fs_path = repo.fs.path.join(repo.root_dir, *entry.key) fs_path = repo.fs.join(repo.root_dir, *entry.key)
tree = out.obj tree = out.obj
assert isinstance(tree, Tree) assert isinstance(tree, Tree)
_, entry.hash_info = tree.get( # type: ignore[misc] _, entry.hash_info = tree.get( # type: ignore[misc]
repo.fs.path.relparts(fs_path, out.fs_path) repo.fs.relparts(fs_path, out.fs_path)
) )
else: else:
entry.hash_info = out.hash_info entry.hash_info = out.hash_info
+3 -3
View File
@@ -43,10 +43,10 @@ class RWLockFileFormatError(DvcException):
@contextmanager @contextmanager
def _edit_rwlock(lock_dir, fs, hardlink): def _edit_rwlock(lock_dir, fs, hardlink):
path = fs.path.join(lock_dir, RWLOCK_FILE) path = fs.join(lock_dir, RWLOCK_FILE)
rwlock_guard = make_lock( rwlock_guard = make_lock(
fs.path.join(lock_dir, RWLOCK_LOCK), fs.join(lock_dir, RWLOCK_LOCK),
tmp_dir=lock_dir, tmp_dir=lock_dir,
hardlink_lock=hardlink, hardlink_lock=hardlink,
) )
@@ -82,7 +82,7 @@ def _check_blockers(tmp_dir, lock, info, *, mode, waiters): # noqa: C901, PLR09
to_release = defaultdict(list) to_release = defaultdict(list)
for path, infos in lock[mode].items(): for path, infos in lock[mode].items():
for waiter_path in waiters: for waiter_path in waiters:
if localfs.path.overlaps(waiter_path, path): if localfs.overlaps(waiter_path, path):
break break
else: else:
continue continue
+1 -1
View File
@@ -656,7 +656,7 @@ class Stage(params.StageParams):
def filter_outs(self, fs_path) -> Iterable["Output"]: def filter_outs(self, fs_path) -> Iterable["Output"]:
def _func(o): def _func(o):
return o.fs.path.isin_or_eq(fs_path, o.fs_path) return o.fs.isin_or_eq(fs_path, o.fs_path)
return filter(_func, self.outs) if fs_path else self.outs return filter(_func, self.outs) if fs_path else self.outs
+9 -9
View File
@@ -181,9 +181,9 @@ class StageCache:
path = self._get_cache_path(cache_key, cache_value) path = self._get_cache_path(cache_key, cache_value)
local_fs = self.repo.cache.legacy.fs local_fs = self.repo.cache.legacy.fs
parent = local_fs.path.parent(path) parent = local_fs.parent(path)
self.repo.cache.legacy.makedirs(parent) self.repo.cache.legacy.makedirs(parent)
tmp = local_fs.path.join(parent, fs.utils.tmp_fname()) tmp = local_fs.join(parent, fs.utils.tmp_fname())
assert os.path.exists(parent) assert os.path.exists(parent)
assert os.path.isdir(parent) assert os.path.isdir(parent)
dump_yaml(tmp, cache) dump_yaml(tmp, cache)
@@ -236,7 +236,7 @@ class StageCache:
from_fs = from_odb.fs from_fs = from_odb.fs
to_fs = to_odb.fs to_fs = to_odb.fs
func = fs.generic.log_exceptions(fs.generic.copy) func = fs.generic.log_exceptions(fs.generic.copy)
runs = from_fs.path.join(from_odb.path, "runs") runs = from_fs.join(from_odb.path, "runs")
http_odb = next( http_odb = next(
(odb for odb in (from_odb, to_odb) if isinstance(odb.fs, HTTPFileSystem)), (odb for odb in (from_odb, to_odb) if isinstance(odb.fs, HTTPFileSystem)),
@@ -252,20 +252,20 @@ class StageCache:
return ret return ret
for src in from_fs.find(runs): for src in from_fs.find(runs):
rel = from_fs.path.relpath(src, from_odb.path) rel = from_fs.relpath(src, from_odb.path)
if not isinstance(to_fs, LocalFileSystem): if not isinstance(to_fs, LocalFileSystem):
rel = from_fs.path.as_posix(rel) rel = from_fs.as_posix(rel)
dst = to_fs.path.join(to_odb.path, rel) dst = to_fs.join(to_odb.path, rel)
key = to_fs.path.parent(dst) key = to_fs.parent(dst)
# check if any build cache already exists for this key # check if any build cache already exists for this key
# TODO: check if MaxKeys=1 or something like that applies # TODO: check if MaxKeys=1 or something like that applies
# or otherwise this will take a lot of time! # or otherwise this will take a lot of time!
if to_fs.exists(key) and first(to_fs.find(key)): if to_fs.exists(key) and first(to_fs.find(key)):
continue continue
src_name = from_fs.path.name(src) src_name = from_fs.name(src)
parent_name = from_fs.path.name(from_fs.path.parent(src)) parent_name = from_fs.name(from_fs.parent(src))
with TqdmCallback( with TqdmCallback(
desc=src_name, desc=src_name,
bytes=True, bytes=True,
+2 -2
View File
@@ -184,9 +184,9 @@ def resolve_wdir(wdir, path):
def resolve_paths(fs, path, wdir=None): def resolve_paths(fs, path, wdir=None):
path = fs.path.abspath(path) path = fs.abspath(path)
wdir = wdir or os.curdir wdir = wdir or os.curdir
wdir = fs.path.abspath(fs.path.join(fs.path.dirname(path), wdir)) wdir = fs.abspath(fs.join(fs.dirname(path), wdir))
return path, wdir return path, wdir
+3 -3
View File
@@ -163,8 +163,8 @@ class TestImportURLVersionAware:
def match_files(fs, entries, expected): def match_files(fs, entries, expected):
entries_content = {(fs.path.normpath(d["path"]), d["isdir"]) for d in entries} entries_content = {(fs.normpath(d["path"]), d["isdir"]) for d in entries}
expected_content = {(fs.path.normpath(d["path"]), d["isdir"]) for d in expected} expected_content = {(fs.normpath(d["path"]), d["isdir"]) for d in expected}
assert entries_content == expected_content assert entries_content == expected_content
@@ -177,7 +177,7 @@ class TestLsUrl:
match_files( match_files(
fs, fs,
result, result,
[{"path": fs.path.join(fs_path, fname), "isdir": False}], [{"path": fs.join(fs_path, fname), "isdir": False}],
) )
def test_dir(self, cloud): def test_dir(self, cloud):
+1 -1
View File
@@ -287,7 +287,7 @@ def resolve_paths(repo, out, always_local=False):
# urlparse interprets windows drive letters as URL scheme # urlparse interprets windows drive letters as URL scheme
scheme = "" scheme = ""
if scheme or not localfs.path.isin_or_eq(abspath, repo.root_dir): if scheme or not localfs.isin_or_eq(abspath, repo.root_dir):
wdir = os.getcwd() wdir = os.getcwd()
elif contains_symlink_up_to(dirname, repo.root_dir) or ( elif contains_symlink_up_to(dirname, repo.root_dir) or (
os.path.isdir(abspath) and localfs.is_symlink(abspath) os.path.isdir(abspath) and localfs.is_symlink(abspath)
+1 -1
View File
@@ -21,7 +21,7 @@ PARSERS.update(
def load_path(fs_path, fs, **kwargs): def load_path(fs_path, fs, **kwargs):
suffix = fs.path.suffix(fs_path).lower() suffix = fs.suffix(fs_path).lower()
loader = LOADERS[suffix] loader = LOADERS[suffix]
return loader(fs_path, fs=fs, **kwargs) return loader(fs_path, fs=fs, **kwargs)
+14 -14
View File
@@ -35,7 +35,7 @@ dependencies = [
"configobj>=5.0.6", "configobj>=5.0.6",
"distro>=1.3", "distro>=1.3",
"dpath<3,>=2.1.0", "dpath<3,>=2.1.0",
"dvc-data>=2.25,<2.26", "dvc-data>=3,<3.1",
"dvc-http>=2.29.0", "dvc-http>=2.29.0",
"dvc-render>=1.0.0,<2", "dvc-render>=1.0.0,<2",
"dvc-studio-client>=0.17.1,<1", "dvc-studio-client>=0.17.1,<1",
@@ -45,7 +45,7 @@ dependencies = [
"flufl.lock>=5,<8", "flufl.lock>=5,<8",
"funcy>=1.14", "funcy>=1.14",
"grandalf<1,>=0.7", "grandalf<1,>=0.7",
"gto>=1.4.0,<2", "gto>=1.6.0,<2",
"hydra-core>=1.1", "hydra-core>=1.1",
"iterative-telemetry>=0.0.7", "iterative-telemetry>=0.0.7",
"networkx>=2.5", "networkx>=2.5",
@@ -59,7 +59,7 @@ dependencies = [
"requests>=2.22", "requests>=2.22",
"rich>=12", "rich>=12",
"ruamel.yaml>=0.17.11", "ruamel.yaml>=0.17.11",
"scmrepo>=1.5.0,<2", "scmrepo>=2.0.2,<3",
"shortuuid>=0.5", "shortuuid>=0.5",
"shtab<2,>=1.3.4", "shtab<2,>=1.3.4",
"tabulate>=0.8.7", "tabulate>=0.8.7",
@@ -71,11 +71,11 @@ dependencies = [
[project.optional-dependencies] [project.optional-dependencies]
all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"] all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"]
azure = ["dvc-azure>=2.23.0"] azure = ["dvc-azure>=3,<4"]
dev = ["dvc[azure,gdrive,gs,hdfs,lint,oss,s3,ssh,tests,webdav,webhdfs]"] dev = ["dvc[azure,gdrive,gs,hdfs,lint,oss,s3,ssh,tests,webdav,webhdfs]"]
gdrive = ["dvc-gdrive==2.20"] gdrive = ["dvc-gdrive>=3,<4"]
gs = ["dvc-gs==2.22.1"] gs = ["dvc-gs>=3,<4"]
hdfs = ["dvc-hdfs==2.19"] hdfs = ["dvc-hdfs>=3,<4"]
lint = [ lint = [
"mypy==1.7.1", "mypy==1.7.1",
"pandas-stubs", "pandas-stubs",
@@ -87,10 +87,10 @@ lint = [
"types-toml", "types-toml",
"types-tqdm", "types-tqdm",
] ]
oss = ["dvc-oss>=2.22.0"] oss = ["dvc-oss>=3,<4"]
s3 = ["dvc-s3==2.23.0"] s3 = ["dvc-s3>=3,<4"]
ssh = ["dvc-ssh>=2.22.1,<3"] ssh = ["dvc-ssh>=3,<4"]
ssh_gssapi = ["dvc-ssh[gssapi]>=2.22.1,<3"] ssh_gssapi = ["dvc-ssh[gssapi]>=3,<4"]
testing = [ testing = [
"pytest-benchmark[histogram]", "pytest-benchmark[histogram]",
"pytest-test-utils", "pytest-test-utils",
@@ -116,9 +116,9 @@ tests = [
"sqlalchemy>=1,<3", # optional dependency for `import-db` "sqlalchemy>=1,<3", # optional dependency for `import-db`
"pandas>=1", "pandas>=1",
] ]
webdav = ["dvc-webdav==2.19.1"] webdav = ["dvc-webdav>=3,<4"]
webhdfs = ["dvc-webhdfs==2.19"] webhdfs = ["dvc-webhdfs>=3,<4"]
webhdfs_kerberos = ["dvc-webhdfs[kerberos]==2.19"] webhdfs_kerberos = ["dvc-webhdfs[kerberos]>=3,<4"]
[project.urls] [project.urls]
Documentation = "https://dvc.org/doc" Documentation = "https://dvc.org/doc"
+8 -8
View File
@@ -13,15 +13,15 @@ def test_cleanfs_subrepo(tmp_dir, dvc, scm, monkeypatch):
path = subrepo_dir.fs_path path = subrepo_dir.fs_path
assert dvc.fs.exists(dvc.fs.path.join(path, "foo")) assert dvc.fs.exists(dvc.fs.join(path, "foo"))
assert dvc.fs.isfile(dvc.fs.path.join(path, "foo")) assert dvc.fs.isfile(dvc.fs.join(path, "foo"))
assert dvc.fs.exists(dvc.fs.path.join(path, "dir")) assert dvc.fs.exists(dvc.fs.join(path, "dir"))
assert dvc.fs.isdir(dvc.fs.path.join(path, "dir")) assert dvc.fs.isdir(dvc.fs.join(path, "dir"))
assert subrepo.fs.exists(subrepo.fs.path.join(path, "foo")) assert subrepo.fs.exists(subrepo.fs.join(path, "foo"))
assert subrepo.fs.isfile(subrepo.fs.path.join(path, "foo")) assert subrepo.fs.isfile(subrepo.fs.join(path, "foo"))
assert subrepo.fs.exists(subrepo.fs.path.join(path, "dir")) assert subrepo.fs.exists(subrepo.fs.join(path, "dir"))
assert subrepo.fs.isdir(subrepo.fs.path.join(path, "dir")) assert subrepo.fs.isdir(subrepo.fs.join(path, "dir"))
def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc): def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
-29
View File
@@ -1,29 +0,0 @@
import pytest
from dvc.fs import Path
@pytest.mark.parametrize("prefix", ["", "/"])
@pytest.mark.parametrize("postfix", ["", "/"])
@pytest.mark.parametrize(
"path,expected",
[
("path", ("path",)),
("some/path", ("some", "path")),
],
)
def test_parts_posix(prefix, postfix, path, expected):
assert Path("/").parts(prefix + path + postfix) == tuple(prefix) + expected
@pytest.mark.parametrize("postfix", ["", "\\"])
@pytest.mark.parametrize(
"path,expected",
[
("path", ("path",)),
("c:\\path", ("c:", "\\", "path")),
("some\\path", ("some", "path")),
],
)
def test_parts_nt(postfix, path, expected):
assert Path("\\").parts(path + postfix) == expected
+4 -5
View File
@@ -173,11 +173,10 @@ def test_match_ignore_from_file(
) )
dvcignore_dirname = os.path.dirname(dvcignore_path) dvcignore_dirname = os.path.dirname(dvcignore_path)
fs = mocker.MagicMock() mocker.patch.object(
fs.path = localfs.path localfs, "open", mocker.mock_open(read_data="\n".join(patterns))
fs.sep = localfs.sep )
mocker.patch.object(fs, "open", mocker.mock_open(read_data="\n".join(patterns))) ignore_file = DvcIgnorePatterns.from_file(dvcignore_path, localfs, "mocked")
ignore_file = DvcIgnorePatterns.from_file(dvcignore_path, fs, "mocked")
assert ( assert (
ignore_file.matches(dvcignore_dirname, file_to_ignore_relpath) == expected_match ignore_file.matches(dvcignore_dirname, file_to_ignore_relpath) == expected_match