mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
deps: update dvc-data to >=3
This commit is contained in:
+2
-2
@@ -27,7 +27,7 @@ def _get_odb(
|
|||||||
cls, config, fs_path = get_cloud_fs(repo.config, **settings)
|
cls, config, fs_path = get_cloud_fs(repo.config, **settings)
|
||||||
fs = fs or cls(**config)
|
fs = fs or cls(**config)
|
||||||
if prefix:
|
if prefix:
|
||||||
fs_path = fs.path.join(fs_path, *prefix)
|
fs_path = fs.join(fs_path, *prefix)
|
||||||
if hash_name:
|
if hash_name:
|
||||||
config["hash_name"] = hash_name
|
config["hash_name"] = hash_name
|
||||||
return get_odb(fs, fs_path, state=repo.state, **config)
|
return get_odb(fs, fs_path, state=repo.state, **config)
|
||||||
@@ -86,7 +86,7 @@ class CacheManager:
|
|||||||
return FileStorage(
|
return FileStorage(
|
||||||
key=(),
|
key=(),
|
||||||
fs=self.local.fs,
|
fs=self.local.fs,
|
||||||
path=self.local.fs.path.join(self.default_local_cache_dir, self.FS_DIR),
|
path=self.local.fs.join(self.default_local_cache_dir, self.FS_DIR),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _init_odb(self, schemes):
|
def _init_odb(self, schemes):
|
||||||
|
|||||||
+1
-1
@@ -86,7 +86,7 @@ def _collect_targets(repo, target, outs):
|
|||||||
targets.extend([str(out) for out in stage.outs])
|
targets.extend([str(out) for out in stage.outs])
|
||||||
continue
|
continue
|
||||||
|
|
||||||
for out in outs_trie.itervalues(prefix=repo.fs.path.parts(path)):
|
for out in outs_trie.itervalues(prefix=repo.fs.parts(path)):
|
||||||
targets.extend(str(out))
|
targets.extend(str(out))
|
||||||
|
|
||||||
return targets
|
return targets
|
||||||
|
|||||||
+8
-8
@@ -14,7 +14,7 @@ from .utils.objects import cached_property
|
|||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from dvc.fs import FileSystem
|
from dvc.fs import FileSystem
|
||||||
from dvc.types import DictStrAny, StrPath
|
from dvc.types import DictStrAny
|
||||||
|
|
||||||
logger = logger.getChild(__name__)
|
logger = logger.getChild(__name__)
|
||||||
|
|
||||||
@@ -89,8 +89,8 @@ class Config(dict):
|
|||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
dvc_dir: Optional["StrPath"] = None,
|
dvc_dir: Optional[str] = None,
|
||||||
local_dvc_dir: Optional["StrPath"] = None,
|
local_dvc_dir: Optional[str] = None,
|
||||||
validate: bool = True,
|
validate: bool = True,
|
||||||
fs: Optional["FileSystem"] = None,
|
fs: Optional["FileSystem"] = None,
|
||||||
config: Optional["DictStrAny"] = None,
|
config: Optional["DictStrAny"] = None,
|
||||||
@@ -105,7 +105,7 @@ class Config(dict):
|
|||||||
self.fs = fs or self.wfs
|
self.fs = fs or self.wfs
|
||||||
|
|
||||||
if dvc_dir:
|
if dvc_dir:
|
||||||
self.dvc_dir = self.fs.path.abspath(dvc_dir)
|
self.dvc_dir = self.fs.abspath(dvc_dir)
|
||||||
|
|
||||||
self.local_dvc_dir = local_dvc_dir
|
self.local_dvc_dir = local_dvc_dir
|
||||||
if not fs and not local_dvc_dir:
|
if not fs and not local_dvc_dir:
|
||||||
@@ -145,10 +145,10 @@ class Config(dict):
|
|||||||
}
|
}
|
||||||
|
|
||||||
if self.dvc_dir is not None:
|
if self.dvc_dir is not None:
|
||||||
files["repo"] = self.fs.path.join(self.dvc_dir, self.CONFIG)
|
files["repo"] = self.fs.join(self.dvc_dir, self.CONFIG)
|
||||||
|
|
||||||
if self.local_dvc_dir is not None:
|
if self.local_dvc_dir is not None:
|
||||||
files["local"] = self.wfs.path.join(self.local_dvc_dir, self.CONFIG_LOCAL)
|
files["local"] = self.wfs.join(self.local_dvc_dir, self.CONFIG_LOCAL)
|
||||||
|
|
||||||
return files
|
return files
|
||||||
|
|
||||||
@@ -302,11 +302,11 @@ class Config(dict):
|
|||||||
return path.def_path
|
return path.def_path
|
||||||
|
|
||||||
if os.path.expanduser(path) != path:
|
if os.path.expanduser(path) != path:
|
||||||
return localfs.path.as_posix(path)
|
return localfs.as_posix(path)
|
||||||
|
|
||||||
if isinstance(path, RelPath) or not os.path.isabs(path):
|
if isinstance(path, RelPath) or not os.path.isabs(path):
|
||||||
path = relpath(path, conf_dir)
|
path = relpath(path, conf_dir)
|
||||||
return localfs.path.as_posix(path)
|
return localfs.as_posix(path)
|
||||||
|
|
||||||
return path
|
return path
|
||||||
|
|
||||||
|
|||||||
+2
-4
@@ -35,11 +35,9 @@ class Remote:
|
|||||||
|
|
||||||
path = self.path
|
path = self.path
|
||||||
if self.worktree:
|
if self.worktree:
|
||||||
path = self.fs.path.join(
|
path = self.fs.join(path, ".dvc", CacheManager.FILES_DIR, DEFAULT_ALGORITHM)
|
||||||
path, ".dvc", CacheManager.FILES_DIR, DEFAULT_ALGORITHM
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
path = self.fs.path.join(path, CacheManager.FILES_DIR, DEFAULT_ALGORITHM)
|
path = self.fs.join(path, CacheManager.FILES_DIR, DEFAULT_ALGORITHM)
|
||||||
return get_odb(self.fs, path, hash_name=DEFAULT_ALGORITHM, **self.config)
|
return get_odb(self.fs, path, hash_name=DEFAULT_ALGORITHM, **self.config)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ class Dependency(Output):
|
|||||||
if self.fs.version_aware:
|
if self.fs.version_aware:
|
||||||
old_fs_path = self.fs_path
|
old_fs_path = self.fs_path
|
||||||
try:
|
try:
|
||||||
self.fs_path = self.fs.path.version_path(self.fs_path, None)
|
self.fs_path = self.fs.version_path(self.fs_path, None)
|
||||||
if self.changed_meta():
|
if self.changed_meta():
|
||||||
return {str(self): "update available"}
|
return {str(self): "update available"}
|
||||||
finally:
|
finally:
|
||||||
@@ -42,9 +42,9 @@ class Dependency(Output):
|
|||||||
|
|
||||||
def update(self, rev=None):
|
def update(self, rev=None):
|
||||||
if self.fs.version_aware:
|
if self.fs.version_aware:
|
||||||
self.fs_path = self.fs.path.version_path(self.fs_path, rev)
|
self.fs_path = self.fs.version_path(self.fs_path, rev)
|
||||||
self.meta = self.get_meta()
|
self.meta = self.get_meta()
|
||||||
self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
|
self.fs_path = self.fs.version_path(self.fs_path, self.meta.version_id)
|
||||||
|
|
||||||
def download(self, to, jobs=None):
|
def download(self, to, jobs=None):
|
||||||
fs_download(self.fs, self.fs_path, to.fs_path, jobs=jobs)
|
fs_download(self.fs, self.fs_path, to.fs_path, jobs=jobs)
|
||||||
@@ -52,7 +52,7 @@ class Dependency(Output):
|
|||||||
def save(self):
|
def save(self):
|
||||||
super().save()
|
super().save()
|
||||||
if self.fs.version_aware:
|
if self.fs.version_aware:
|
||||||
self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
|
self.fs_path = self.fs.version_path(self.fs_path, self.meta.version_id)
|
||||||
|
|
||||||
def dumpd(self, **kwargs):
|
def dumpd(self, **kwargs):
|
||||||
if self.fs.version_aware:
|
if self.fs.version_aware:
|
||||||
|
|||||||
+1
-1
@@ -294,7 +294,7 @@ class ProjectFile(FileMixin):
|
|||||||
def resolver(self) -> "DataResolver":
|
def resolver(self) -> "DataResolver":
|
||||||
from .parsing import DataResolver
|
from .parsing import DataResolver
|
||||||
|
|
||||||
wdir = self.repo.fs.path.parent(self.path)
|
wdir = self.repo.fs.parent(self.path)
|
||||||
return DataResolver(self.repo, wdir, self.contents)
|
return DataResolver(self.repo, wdir, self.contents)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
|
|||||||
+3
-7
@@ -25,7 +25,6 @@ from dvc_objects.fs.errors import ( # noqa: F401
|
|||||||
ConfigError,
|
ConfigError,
|
||||||
RemoteMissingDepsError,
|
RemoteMissingDepsError,
|
||||||
)
|
)
|
||||||
from dvc_objects.fs.path import Path # noqa: F401
|
|
||||||
|
|
||||||
from .callbacks import Callback # noqa: F401
|
from .callbacks import Callback # noqa: F401
|
||||||
from .data import DataFileSystem # noqa: F401
|
from .data import DataFileSystem # noqa: F401
|
||||||
@@ -54,23 +53,20 @@ def download(
|
|||||||
from .callbacks import TqdmCallback
|
from .callbacks import TqdmCallback
|
||||||
|
|
||||||
with TqdmCallback(
|
with TqdmCallback(
|
||||||
desc=f"Downloading {fs.path.name(fs_path)}",
|
desc=f"Downloading {fs.name(fs_path)}",
|
||||||
unit="files",
|
unit="files",
|
||||||
) as cb:
|
) as cb:
|
||||||
# NOTE: We use dvc-objects generic.copy over fs.get since it makes file
|
# NOTE: We use dvc-objects generic.copy over fs.get since it makes file
|
||||||
# download atomic and avoids fsspec glob/regex path expansion.
|
# download atomic and avoids fsspec glob/regex path expansion.
|
||||||
if fs.isdir(fs_path):
|
if fs.isdir(fs_path):
|
||||||
from_infos = [
|
from_infos = [
|
||||||
path
|
path for path in fs.find(fs_path) if not path.endswith(fs.flavour.sep)
|
||||||
for path in fs.find(fs_path)
|
|
||||||
if not path.endswith(fs.path.flavour.sep)
|
|
||||||
]
|
]
|
||||||
if not from_infos:
|
if not from_infos:
|
||||||
localfs.makedirs(to, exist_ok=True)
|
localfs.makedirs(to, exist_ok=True)
|
||||||
return 0
|
return 0
|
||||||
to_infos = [
|
to_infos = [
|
||||||
localfs.path.join(to, *fs.path.relparts(info, fs_path))
|
localfs.join(to, *fs.relparts(info, fs_path)) for info in from_infos
|
||||||
for info in from_infos
|
|
||||||
]
|
]
|
||||||
else:
|
else:
|
||||||
from_infos = [fs_path]
|
from_infos = [fs_path]
|
||||||
|
|||||||
@@ -29,6 +29,9 @@ class DataFileSystem(FileSystem):
|
|||||||
|
|
||||||
return _DataFileSystem(**self.fs_args)
|
return _DataFileSystem(**self.fs_args)
|
||||||
|
|
||||||
|
def getcwd(self):
|
||||||
|
return self.fs.getcwd()
|
||||||
|
|
||||||
def isdvc(self, path, **kwargs):
|
def isdvc(self, path, **kwargs):
|
||||||
return self.fs.isdvc(path, **kwargs)
|
return self.fs.isdvc(path, **kwargs)
|
||||||
|
|
||||||
|
|||||||
+57
-24
@@ -13,7 +13,6 @@ from funcy import wrap_with
|
|||||||
|
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
from dvc_objects.fs.base import FileSystem
|
from dvc_objects.fs.base import FileSystem
|
||||||
from dvc_objects.fs.path import Path
|
|
||||||
|
|
||||||
from .data import DataFileSystem
|
from .data import DataFileSystem
|
||||||
|
|
||||||
@@ -71,7 +70,7 @@ def _merge_info(repo, key, fs_info, dvc_info):
|
|||||||
|
|
||||||
|
|
||||||
def _get_dvc_path(dvc_fs, subkey):
|
def _get_dvc_path(dvc_fs, subkey):
|
||||||
return dvc_fs.path.join(*subkey) if subkey else ""
|
return dvc_fs.join(*subkey) if subkey else ""
|
||||||
|
|
||||||
|
|
||||||
class _DVCFileSystem(AbstractFileSystem):
|
class _DVCFileSystem(AbstractFileSystem):
|
||||||
@@ -145,7 +144,51 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
"remote_config": remote_config,
|
"remote_config": remote_config,
|
||||||
}
|
}
|
||||||
|
|
||||||
self.path = Path(self.sep, getcwd=self._getcwd)
|
def getcwd(self):
|
||||||
|
relparts: Tuple[str, ...] = ()
|
||||||
|
assert self.repo is not None
|
||||||
|
if self.repo.fs.isin(self.repo.fs.getcwd(), self.repo.root_dir):
|
||||||
|
relparts = self.repo.fs.relparts(self.repo.fs.getcwd(), self.repo.root_dir)
|
||||||
|
return self.root_marker + self.sep.join(relparts)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def join(cls, *parts: str) -> str:
|
||||||
|
return posixpath.join(*parts)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def parts(cls, path: str) -> Tuple[str, ...]:
|
||||||
|
ret = []
|
||||||
|
while True:
|
||||||
|
path, part = posixpath.split(path)
|
||||||
|
|
||||||
|
if part:
|
||||||
|
ret.append(part)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if path:
|
||||||
|
ret.append(path)
|
||||||
|
|
||||||
|
break
|
||||||
|
|
||||||
|
ret.reverse()
|
||||||
|
|
||||||
|
return tuple(ret)
|
||||||
|
|
||||||
|
def normpath(self, path: str) -> str:
|
||||||
|
return posixpath.normpath(path)
|
||||||
|
|
||||||
|
def abspath(self, path: str) -> str:
|
||||||
|
if not posixpath.isabs(path):
|
||||||
|
path = self.join(self.getcwd(), path)
|
||||||
|
return self.normpath(path)
|
||||||
|
|
||||||
|
def relpath(self, path: str, start: Optional[str] = None) -> str:
|
||||||
|
if start is None:
|
||||||
|
start = "."
|
||||||
|
return posixpath.relpath(self.abspath(path), start=self.abspath(start))
|
||||||
|
|
||||||
|
def relparts(self, path: str, start: Optional[str] = None) -> Tuple[str, ...]:
|
||||||
|
return self.parts(self.relpath(path, start=start))
|
||||||
|
|
||||||
@functools.cached_property
|
@functools.cached_property
|
||||||
def repo(self):
|
def repo(self):
|
||||||
@@ -170,15 +213,6 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
|
|
||||||
return self.repo._fs_conf["repo_factory"]
|
return self.repo._fs_conf["repo_factory"]
|
||||||
|
|
||||||
def _getcwd(self):
|
|
||||||
relparts: Tuple[str, ...] = ()
|
|
||||||
assert self.repo is not None
|
|
||||||
if self.repo.fs.path.isin(self.repo.fs.path.getcwd(), self.repo.root_dir):
|
|
||||||
relparts = self.repo.fs.path.relparts(
|
|
||||||
self.repo.fs.path.getcwd(), self.repo.root_dir
|
|
||||||
)
|
|
||||||
return self.root_marker + self.sep.join(relparts)
|
|
||||||
|
|
||||||
@functools.cached_property
|
@functools.cached_property
|
||||||
def fsid(self) -> str:
|
def fsid(self) -> str:
|
||||||
from fsspec.utils import tokenize
|
from fsspec.utils import tokenize
|
||||||
@@ -192,7 +226,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
|
|
||||||
def _get_key(self, path: "StrPath") -> Key:
|
def _get_key(self, path: "StrPath") -> Key:
|
||||||
path = os.fspath(path)
|
path = os.fspath(path)
|
||||||
parts = self.repo.fs.path.relparts(path, self.repo.root_dir)
|
parts = self.repo.fs.relparts(path, self.repo.root_dir)
|
||||||
if parts == (os.curdir,):
|
if parts == (os.curdir,):
|
||||||
return ()
|
return ()
|
||||||
return parts
|
return parts
|
||||||
@@ -210,13 +244,13 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
|
|
||||||
def _get_key_from_relative(self, path) -> Key:
|
def _get_key_from_relative(self, path) -> Key:
|
||||||
path = self._strip_protocol(path)
|
path = self._strip_protocol(path)
|
||||||
parts = self.path.relparts(path, self.root_marker)
|
parts = self.relparts(path, self.root_marker)
|
||||||
if parts and parts[0] == os.curdir:
|
if parts and parts[0] == os.curdir:
|
||||||
return parts[1:]
|
return parts[1:]
|
||||||
return parts
|
return parts
|
||||||
|
|
||||||
def _from_key(self, parts: Key) -> str:
|
def _from_key(self, parts: Key) -> str:
|
||||||
return self.repo.fs.path.join(self.repo.root_dir, *parts)
|
return self.repo.fs.join(self.repo.root_dir, *parts)
|
||||||
|
|
||||||
@functools.cached_property
|
@functools.cached_property
|
||||||
def _datafss(self):
|
def _datafss(self):
|
||||||
@@ -282,7 +316,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
|
|
||||||
from dvc.repo import Repo
|
from dvc.repo import Repo
|
||||||
|
|
||||||
repo_path = self.repo.fs.path.join(dir_path, Repo.DVC_DIR)
|
repo_path = self.repo.fs.join(dir_path, Repo.DVC_DIR)
|
||||||
return self.repo.fs.isdir(repo_path)
|
return self.repo.fs.isdir(repo_path)
|
||||||
|
|
||||||
def _get_subrepo_info(
|
def _get_subrepo_info(
|
||||||
@@ -336,7 +370,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
dvc_path = _get_dvc_path(dvc_fs, subkey)
|
dvc_path = _get_dvc_path(dvc_fs, subkey)
|
||||||
with suppress(FileNotFoundError, NotADirectoryError):
|
with suppress(FileNotFoundError, NotADirectoryError):
|
||||||
for info in dvc_fs.ls(dvc_path, detail=True):
|
for info in dvc_fs.ls(dvc_path, detail=True):
|
||||||
dvc_infos[dvc_fs.path.name(info["name"])] = info
|
dvc_infos[dvc_fs.name(info["name"])] = info
|
||||||
dvc_exists = True
|
dvc_exists = True
|
||||||
|
|
||||||
fs_exists = False
|
fs_exists = False
|
||||||
@@ -349,7 +383,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
for info in repo.dvcignore.ls(
|
for info in repo.dvcignore.ls(
|
||||||
fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos
|
fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos
|
||||||
):
|
):
|
||||||
fs_infos[fs.path.name(info["name"])] = info
|
fs_infos[fs.name(info["name"])] = info
|
||||||
fs_exists = True
|
fs_exists = True
|
||||||
except (FileNotFoundError, NotADirectoryError):
|
except (FileNotFoundError, NotADirectoryError):
|
||||||
pass
|
pass
|
||||||
@@ -368,7 +402,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
if not dvcfiles and _is_dvc_file(name):
|
if not dvcfiles and _is_dvc_file(name):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
entry_path = self.path.join(path, name)
|
entry_path = self.join(path, name)
|
||||||
info = _merge_info(
|
info = _merge_info(
|
||||||
repo, (*subkey, name), fs_infos.get(name), dvc_infos.get(name)
|
repo, (*subkey, name), fs_infos.get(name), dvc_infos.get(name)
|
||||||
)
|
)
|
||||||
@@ -416,7 +450,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
# NOTE: if some parent in fs_path turns out to be a file, it means
|
# NOTE: if some parent in fs_path turns out to be a file, it means
|
||||||
# that the whole repofs branch doesn't exist.
|
# that the whole repofs branch doesn't exist.
|
||||||
if dvc_info and not fs_info:
|
if dvc_info and not fs_info:
|
||||||
for parent in fs.path.parents(fs_path):
|
for parent in fs.parents(fs_path):
|
||||||
try:
|
try:
|
||||||
if fs.info(parent)["type"] != "directory":
|
if fs.info(parent)["type"] != "directory":
|
||||||
dvc_info = None
|
dvc_info = None
|
||||||
@@ -498,6 +532,9 @@ class DVCFileSystem(FileSystem):
|
|||||||
def fs(self) -> "_DVCFileSystem":
|
def fs(self) -> "_DVCFileSystem":
|
||||||
return _DVCFileSystem(**self.fs_args)
|
return _DVCFileSystem(**self.fs_args)
|
||||||
|
|
||||||
|
def getcwd(self):
|
||||||
|
return self.fs.getcwd()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def fsid(self) -> str:
|
def fsid(self) -> str:
|
||||||
return self.fs.fsid
|
return self.fs.fsid
|
||||||
@@ -505,10 +542,6 @@ class DVCFileSystem(FileSystem):
|
|||||||
def isdvc(self, path, **kwargs) -> bool:
|
def isdvc(self, path, **kwargs) -> bool:
|
||||||
return self.fs.isdvc(path, **kwargs)
|
return self.fs.isdvc(path, **kwargs)
|
||||||
|
|
||||||
@property
|
|
||||||
def path(self) -> Path:
|
|
||||||
return self.fs.path
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def repo(self) -> "Repo":
|
def repo(self) -> "Repo":
|
||||||
return self.fs.repo
|
return self.fs.repo
|
||||||
|
|||||||
+5
-3
@@ -46,9 +46,11 @@ class GitFileSystem(FileSystem):
|
|||||||
|
|
||||||
return FsspecGitFileSystem(**self.fs_args)
|
return FsspecGitFileSystem(**self.fs_args)
|
||||||
|
|
||||||
@functools.cached_property
|
def getcwd(self):
|
||||||
def path(self):
|
return self.fs.getcwd()
|
||||||
return self.fs.path
|
|
||||||
|
def chdir(self, path):
|
||||||
|
self.fs.chdir(path)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def rev(self) -> str:
|
def rev(self) -> str:
|
||||||
|
|||||||
+21
-21
@@ -51,8 +51,8 @@ class DvcIgnorePatterns(DvcIgnore):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_file(cls, path, fs, name):
|
def from_file(cls, path, fs, name):
|
||||||
assert fs.path.isabs(path)
|
assert fs.isabs(path)
|
||||||
dirname = fs.path.normpath(fs.path.dirname(path))
|
dirname = fs.normpath(fs.dirname(path))
|
||||||
with fs.open(path, encoding="utf-8") as fobj:
|
with fs.open(path, encoding="utf-8") as fobj:
|
||||||
path_spec_lines = [
|
path_spec_lines = [
|
||||||
PatternInfo(line, f"{name}:{line_no + 1}:{line}")
|
PatternInfo(line, f"{name}:{line_no + 1}:{line}")
|
||||||
@@ -194,7 +194,7 @@ class DvcIgnoreFilter:
|
|||||||
)
|
)
|
||||||
|
|
||||||
def _get_key(self, path):
|
def _get_key(self, path):
|
||||||
parts = self.fs.path.relparts(path, self.root_dir)
|
parts = self.fs.relparts(path, self.root_dir)
|
||||||
if parts == (os.curdir,):
|
if parts == (os.curdir,):
|
||||||
return ()
|
return ()
|
||||||
return parts
|
return parts
|
||||||
@@ -204,13 +204,13 @@ class DvcIgnoreFilter:
|
|||||||
old_pattern = trie.longest_prefix(key).value
|
old_pattern = trie.longest_prefix(key).value
|
||||||
matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
|
matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
|
||||||
|
|
||||||
path = self.fs.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
|
path = self.fs.join(dirname, DvcIgnore.DVCIGNORE_FILE)
|
||||||
if not matches and self.fs.exists(path):
|
if not matches and self.fs.exists(path):
|
||||||
name = self.fs.path.relpath(path, self.root_dir)
|
name = self.fs.relpath(path, self.root_dir)
|
||||||
new_pattern = DvcIgnorePatterns.from_file(path, self.fs, name)
|
new_pattern = DvcIgnorePatterns.from_file(path, self.fs, name)
|
||||||
if old_pattern:
|
if old_pattern:
|
||||||
plist, prefix = merge_patterns(
|
plist, prefix = merge_patterns(
|
||||||
self.fs.path.flavour,
|
self.fs.flavour,
|
||||||
old_pattern.pattern_list,
|
old_pattern.pattern_list,
|
||||||
old_pattern.dirname,
|
old_pattern.dirname,
|
||||||
new_pattern.pattern_list,
|
new_pattern.pattern_list,
|
||||||
@@ -239,7 +239,7 @@ class DvcIgnoreFilter:
|
|||||||
dnames = []
|
dnames = []
|
||||||
|
|
||||||
for dname in dnames:
|
for dname in dnames:
|
||||||
self._update_sub_repo(self.fs.path.join(dirname, dname), ignore_trie)
|
self._update_sub_repo(self.fs.join(dirname, dname), ignore_trie)
|
||||||
|
|
||||||
def _update_sub_repo(self, path, ignore_trie: Trie):
|
def _update_sub_repo(self, path, ignore_trie: Trie):
|
||||||
from dvc.repo import Repo
|
from dvc.repo import Repo
|
||||||
@@ -247,18 +247,18 @@ class DvcIgnoreFilter:
|
|||||||
if path == self.root_dir:
|
if path == self.root_dir:
|
||||||
return
|
return
|
||||||
|
|
||||||
dvc_dir = self.fs.path.join(path, Repo.DVC_DIR)
|
dvc_dir = self.fs.join(path, Repo.DVC_DIR)
|
||||||
if not self.fs.exists(dvc_dir):
|
if not self.fs.exists(dvc_dir):
|
||||||
return
|
return
|
||||||
|
|
||||||
root, dname = self.fs.path.split(path)
|
root, dname = self.fs.split(path)
|
||||||
key = self._get_key(root)
|
key = self._get_key(root)
|
||||||
pattern_info = PatternInfo(f"/{dname}/", f"in sub_repo:{dname}")
|
pattern_info = PatternInfo(f"/{dname}/", f"in sub_repo:{dname}")
|
||||||
new_pattern = DvcIgnorePatterns([pattern_info], root, self.fs.sep)
|
new_pattern = DvcIgnorePatterns([pattern_info], root, self.fs.sep)
|
||||||
old_pattern = ignore_trie.longest_prefix(key).value
|
old_pattern = ignore_trie.longest_prefix(key).value
|
||||||
if old_pattern:
|
if old_pattern:
|
||||||
plist, prefix = merge_patterns(
|
plist, prefix = merge_patterns(
|
||||||
self.fs.path.flavour,
|
self.fs.flavour,
|
||||||
old_pattern.pattern_list,
|
old_pattern.pattern_list,
|
||||||
old_pattern.dirname,
|
old_pattern.dirname,
|
||||||
new_pattern.pattern_list,
|
new_pattern.pattern_list,
|
||||||
@@ -269,7 +269,7 @@ class DvcIgnoreFilter:
|
|||||||
ignore_trie[key] = new_pattern
|
ignore_trie[key] = new_pattern
|
||||||
|
|
||||||
def __call__(self, root, dirs, files, ignore_subrepos=True):
|
def __call__(self, root, dirs, files, ignore_subrepos=True):
|
||||||
abs_root = self.fs.path.abspath(root)
|
abs_root = self.fs.abspath(root)
|
||||||
ignore_pattern = self._get_trie_pattern(
|
ignore_pattern = self._get_trie_pattern(
|
||||||
abs_root, dnames=dirs, ignore_subrepos=ignore_subrepos
|
abs_root, dnames=dirs, ignore_subrepos=ignore_subrepos
|
||||||
)
|
)
|
||||||
@@ -283,7 +283,7 @@ class DvcIgnoreFilter:
|
|||||||
nondirs = []
|
nondirs = []
|
||||||
|
|
||||||
for entry in fs.ls(path, detail=True, **kwargs):
|
for entry in fs.ls(path, detail=True, **kwargs):
|
||||||
name = fs.path.name(entry["name"])
|
name = fs.name(entry["name"])
|
||||||
fs_dict[name] = entry
|
fs_dict[name] = entry
|
||||||
if entry["type"] == "directory":
|
if entry["type"] == "directory":
|
||||||
dirs.append(name)
|
dirs.append(name)
|
||||||
@@ -338,7 +338,7 @@ class DvcIgnoreFilter:
|
|||||||
else:
|
else:
|
||||||
ignores_trie = self._ignores_trie_subrepos
|
ignores_trie = self._ignores_trie_subrepos
|
||||||
|
|
||||||
if not self.fs.path.isin_or_eq(dirname, self.root_dir):
|
if not self.fs.isin_or_eq(dirname, self.root_dir):
|
||||||
# outside of the repo
|
# outside of the repo
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -349,12 +349,12 @@ class DvcIgnoreFilter:
|
|||||||
return ignore_pattern
|
return ignore_pattern
|
||||||
|
|
||||||
prefix_key = ignores_trie.longest_prefix(key).key or ()
|
prefix_key = ignores_trie.longest_prefix(key).key or ()
|
||||||
prefix = self.fs.path.join(self.root_dir, *prefix_key)
|
prefix = self.fs.join(self.root_dir, *prefix_key)
|
||||||
|
|
||||||
dirs = list(
|
dirs = list(
|
||||||
takewhile(
|
takewhile(
|
||||||
lambda path: path != prefix,
|
lambda path: path != prefix,
|
||||||
(parent for parent in localfs.path.parents(dirname)),
|
(parent for parent in localfs.parents(dirname)),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
dirs.reverse()
|
dirs.reverse()
|
||||||
@@ -370,7 +370,7 @@ class DvcIgnoreFilter:
|
|||||||
):
|
):
|
||||||
if self._outside_repo(path):
|
if self._outside_repo(path):
|
||||||
return False
|
return False
|
||||||
dirname, basename = self.fs.path.split(self.fs.path.normpath(path))
|
dirname, basename = self.fs.split(self.fs.normpath(path))
|
||||||
ignore_pattern = self._get_trie_pattern(dirname, None, ignore_subrepos)
|
ignore_pattern = self._get_trie_pattern(dirname, None, ignore_subrepos)
|
||||||
if ignore_pattern:
|
if ignore_pattern:
|
||||||
return ignore_pattern.matches(dirname, basename, is_dir)
|
return ignore_pattern.matches(dirname, basename, is_dir)
|
||||||
@@ -378,7 +378,7 @@ class DvcIgnoreFilter:
|
|||||||
|
|
||||||
def is_ignored_dir(self, path: str, ignore_subrepos: bool = True) -> bool:
|
def is_ignored_dir(self, path: str, ignore_subrepos: bool = True) -> bool:
|
||||||
# only used in LocalFileSystem
|
# only used in LocalFileSystem
|
||||||
path = self.fs.path.abspath(path)
|
path = self.fs.abspath(path)
|
||||||
if path == self.root_dir:
|
if path == self.root_dir:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -386,18 +386,18 @@ class DvcIgnoreFilter:
|
|||||||
|
|
||||||
def is_ignored_file(self, path: str, ignore_subrepos: bool = True) -> bool:
|
def is_ignored_file(self, path: str, ignore_subrepos: bool = True) -> bool:
|
||||||
# only used in LocalFileSystem
|
# only used in LocalFileSystem
|
||||||
path = self.fs.path.abspath(path)
|
path = self.fs.abspath(path)
|
||||||
return self._is_ignored(path, False, ignore_subrepos=ignore_subrepos)
|
return self._is_ignored(path, False, ignore_subrepos=ignore_subrepos)
|
||||||
|
|
||||||
def _outside_repo(self, path):
|
def _outside_repo(self, path):
|
||||||
return not self.fs.path.isin_or_eq(path, self.root_dir)
|
return not self.fs.isin_or_eq(path, self.root_dir)
|
||||||
|
|
||||||
def check_ignore(self, target):
|
def check_ignore(self, target):
|
||||||
# NOTE: can only be used in `dvc check-ignore`, see
|
# NOTE: can only be used in `dvc check-ignore`, see
|
||||||
# https://github.com/iterative/dvc/issues/5046
|
# https://github.com/iterative/dvc/issues/5046
|
||||||
full_target = self.fs.path.abspath(target)
|
full_target = self.fs.abspath(target)
|
||||||
if not self._outside_repo(full_target):
|
if not self._outside_repo(full_target):
|
||||||
dirname, basename = self.fs.path.split(self.fs.path.normpath(full_target))
|
dirname, basename = self.fs.split(self.fs.normpath(full_target))
|
||||||
pattern = self._get_trie_pattern(dirname)
|
pattern = self._get_trie_pattern(dirname)
|
||||||
if pattern:
|
if pattern:
|
||||||
matches = pattern.matches(
|
matches = pattern.matches(
|
||||||
|
|||||||
+22
-28
@@ -378,7 +378,7 @@ class Output:
|
|||||||
if (
|
if (
|
||||||
self.repo
|
self.repo
|
||||||
and self.fs.protocol == "local"
|
and self.fs.protocol == "local"
|
||||||
and not self.fs.path.isabs(self.def_path)
|
and not self.fs.isabs(self.def_path)
|
||||||
):
|
):
|
||||||
self.fs = self.repo.fs
|
self.fs = self.repo.fs
|
||||||
|
|
||||||
@@ -411,7 +411,7 @@ class Output:
|
|||||||
self.remote = remote
|
self.remote = remote
|
||||||
|
|
||||||
if self.fs.version_aware:
|
if self.fs.version_aware:
|
||||||
_, version_id = self.fs.path.coalesce_version(
|
_, version_id = self.fs.coalesce_version(
|
||||||
self.def_path, self.meta.version_id
|
self.def_path, self.meta.version_id
|
||||||
)
|
)
|
||||||
self.meta.version_id = version_id
|
self.meta.version_id = version_id
|
||||||
@@ -465,7 +465,7 @@ class Output:
|
|||||||
parsed.scheme != "remote"
|
parsed.scheme != "remote"
|
||||||
and self.stage
|
and self.stage
|
||||||
and self.stage.repo.fs == fs
|
and self.stage.repo.fs == fs
|
||||||
and not fs.path.isabs(fs_path)
|
and not fs.isabs(fs_path)
|
||||||
):
|
):
|
||||||
# NOTE: we can path either from command line or .dvc file,
|
# NOTE: we can path either from command line or .dvc file,
|
||||||
# so we should expect both posix and windows style paths.
|
# so we should expect both posix and windows style paths.
|
||||||
@@ -473,9 +473,9 @@ class Output:
|
|||||||
#
|
#
|
||||||
# FIXME: if we have Windows path containing / or posix one with \
|
# FIXME: if we have Windows path containing / or posix one with \
|
||||||
# then we have #2059 bug and can't really handle that.
|
# then we have #2059 bug and can't really handle that.
|
||||||
fs_path = fs.path.join(self.stage.wdir, fs_path)
|
fs_path = fs.join(self.stage.wdir, fs_path)
|
||||||
|
|
||||||
return fs.path.abspath(fs.path.normpath(fs_path))
|
return fs.abspath(fs.normpath(fs_path))
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
return f"{type(self).__name__}: {self.def_path!r}"
|
return f"{type(self).__name__}: {self.def_path!r}"
|
||||||
@@ -491,14 +491,14 @@ class Output:
|
|||||||
):
|
):
|
||||||
return str(self.def_path)
|
return str(self.def_path)
|
||||||
|
|
||||||
if not self.fs.path.isin(self.fs_path, self.repo.root_dir):
|
if not self.fs.isin(self.fs_path, self.repo.root_dir):
|
||||||
return self.fs_path
|
return self.fs_path
|
||||||
|
|
||||||
cur_dir = self.fs.path.getcwd()
|
cur_dir = self.fs.getcwd()
|
||||||
if self.fs.path.isin(cur_dir, self.repo.root_dir):
|
if self.fs.isin(cur_dir, self.repo.root_dir):
|
||||||
return self.fs.path.relpath(self.fs_path, cur_dir)
|
return self.fs.relpath(self.fs_path, cur_dir)
|
||||||
|
|
||||||
return self.fs.path.relpath(self.fs_path, self.repo.root_dir)
|
return self.fs.relpath(self.fs_path, self.repo.root_dir)
|
||||||
|
|
||||||
def clear(self):
|
def clear(self):
|
||||||
self.hash_info = HashInfo.from_dict({})
|
self.hash_info = HashInfo.from_dict({})
|
||||||
@@ -515,10 +515,10 @@ class Output:
|
|||||||
if urlparse(self.def_path).scheme == "remote":
|
if urlparse(self.def_path).scheme == "remote":
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if self.fs.path.isabs(self.def_path):
|
if self.fs.isabs(self.def_path):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
return self.repo and self.fs.path.isin(
|
return self.repo and self.fs.isin(
|
||||||
self.fs_path,
|
self.fs_path,
|
||||||
self.repo.root_dir,
|
self.repo.root_dir,
|
||||||
)
|
)
|
||||||
@@ -607,11 +607,11 @@ class Output:
|
|||||||
def index_key(self) -> Tuple[str, "DataIndexKey"]:
|
def index_key(self) -> Tuple[str, "DataIndexKey"]:
|
||||||
if self.is_in_repo:
|
if self.is_in_repo:
|
||||||
workspace = "repo"
|
workspace = "repo"
|
||||||
key = self.repo.fs.path.relparts(self.fs_path, self.repo.root_dir)
|
key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
|
||||||
else:
|
else:
|
||||||
workspace = self.fs.protocol
|
workspace = self.fs.protocol
|
||||||
no_drive = self.fs.path.flavour.splitdrive(self.fs_path)[1]
|
no_drive = self.fs.flavour.splitdrive(self.fs_path)[1]
|
||||||
key = self.fs.path.parts(no_drive)[1:]
|
key = self.fs.parts(no_drive)[1:]
|
||||||
return workspace, key
|
return workspace, key
|
||||||
|
|
||||||
def changed_checksum(self):
|
def changed_checksum(self):
|
||||||
@@ -799,7 +799,7 @@ class Output:
|
|||||||
callback=cb,
|
callback=cb,
|
||||||
)
|
)
|
||||||
if relink:
|
if relink:
|
||||||
rel = self.fs.path.relpath(filter_info or self.fs_path)
|
rel = self.fs.relpath(filter_info or self.fs_path)
|
||||||
with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb:
|
with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb:
|
||||||
self._checkout(
|
self._checkout(
|
||||||
filter_info or self.fs_path,
|
filter_info or self.fs_path,
|
||||||
@@ -814,7 +814,7 @@ class Output:
|
|||||||
self.set_exec()
|
self.set_exec()
|
||||||
|
|
||||||
def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]:
|
def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]:
|
||||||
prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
|
prefix = self.fs.parts(self.fs.relpath(filter_info, self.fs_path))
|
||||||
staging, _, obj = self._build(
|
staging, _, obj = self._build(
|
||||||
self.cache,
|
self.cache,
|
||||||
self.fs_path,
|
self.fs_path,
|
||||||
@@ -862,7 +862,7 @@ class Output:
|
|||||||
ret.update(split_file_meta_from_cloud(meta_d))
|
ret.update(split_file_meta_from_cloud(meta_d))
|
||||||
|
|
||||||
if self.is_in_repo:
|
if self.is_in_repo:
|
||||||
path = self.fs.path.as_posix(relpath(self.fs_path, self.stage.wdir))
|
path = self.fs.as_posix(relpath(self.fs_path, self.stage.wdir))
|
||||||
else:
|
else:
|
||||||
path = self.def_path
|
path = self.def_path
|
||||||
|
|
||||||
@@ -949,7 +949,7 @@ class Output:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
assert obj
|
assert obj
|
||||||
fs_path = self.fs.path
|
fs_path = self.fs
|
||||||
if filter_info and filter_info != self.fs_path:
|
if filter_info and filter_info != self.fs_path:
|
||||||
prefix = fs_path.relparts(filter_info, self.fs_path)
|
prefix = fs_path.relparts(filter_info, self.fs_path)
|
||||||
assert isinstance(obj, Tree)
|
assert isinstance(obj, Tree)
|
||||||
@@ -1154,7 +1154,7 @@ class Output:
|
|||||||
assert obj is None or isinstance(obj, Tree)
|
assert obj is None or isinstance(obj, Tree)
|
||||||
if filter_info and filter_info != self.fs_path:
|
if filter_info and filter_info != self.fs_path:
|
||||||
assert obj
|
assert obj
|
||||||
prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
|
prefix = self.fs.parts(self.fs.relpath(filter_info, self.fs_path))
|
||||||
return obj.filter(prefix)
|
return obj.filter(prefix)
|
||||||
return obj
|
return obj
|
||||||
|
|
||||||
@@ -1297,10 +1297,7 @@ class Output:
|
|||||||
def unstage(self, path: str) -> Tuple["Meta", "Tree"]:
|
def unstage(self, path: str) -> Tuple["Meta", "Tree"]:
|
||||||
from pygtrie import Trie
|
from pygtrie import Trie
|
||||||
|
|
||||||
from dvc_objects.fs.path import Path
|
rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path)))
|
||||||
|
|
||||||
assert isinstance(self.fs.path, Path)
|
|
||||||
rel_key = tuple(self.fs.path.parts(self.fs.path.relpath(path, self.fs_path)))
|
|
||||||
|
|
||||||
if self.hash_info:
|
if self.hash_info:
|
||||||
tree = self.get_dir_cache()
|
tree = self.get_dir_cache()
|
||||||
@@ -1333,11 +1330,8 @@ class Output:
|
|||||||
) -> Tuple["Meta", "Tree"]:
|
) -> Tuple["Meta", "Tree"]:
|
||||||
from pygtrie import Trie
|
from pygtrie import Trie
|
||||||
|
|
||||||
from dvc_objects.fs.path import Path
|
|
||||||
|
|
||||||
assert isinstance(self.fs.path, Path)
|
|
||||||
append_only = True
|
append_only = True
|
||||||
rel_key = tuple(self.fs.path.parts(self.fs.path.relpath(path, self.fs_path)))
|
rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path)))
|
||||||
|
|
||||||
if self.hash_info:
|
if self.hash_info:
|
||||||
tree = self.get_dir_cache()
|
tree = self.get_dir_cache()
|
||||||
|
|||||||
@@ -143,11 +143,11 @@ class DataResolver:
|
|||||||
self.parsing_config = repo.config.get("parsing", {})
|
self.parsing_config = repo.config.get("parsing", {})
|
||||||
|
|
||||||
if os.path.isabs(wdir):
|
if os.path.isabs(wdir):
|
||||||
wdir = fs.path.relpath(wdir)
|
wdir = fs.relpath(wdir)
|
||||||
wdir = "" if wdir == os.curdir else wdir
|
wdir = "" if wdir == os.curdir else wdir
|
||||||
|
|
||||||
self.wdir = wdir
|
self.wdir = wdir
|
||||||
self.relpath = fs.path.normpath(fs.path.join(self.wdir, "dvc.yaml"))
|
self.relpath = fs.normpath(fs.join(self.wdir, "dvc.yaml"))
|
||||||
|
|
||||||
vars_ = d.get(VARS_KWD, [])
|
vars_ = d.get(VARS_KWD, [])
|
||||||
check_interpolations(vars_, VARS_KWD, self.relpath)
|
check_interpolations(vars_, VARS_KWD, self.relpath)
|
||||||
@@ -248,7 +248,7 @@ class EntryDefinition:
|
|||||||
wdir = to_str(context.resolve_str(wdir))
|
wdir = to_str(context.resolve_str(wdir))
|
||||||
except (ContextError, ParseError) as exc:
|
except (ContextError, ParseError) as exc:
|
||||||
format_and_raise(exc, f"'{self.where}.{name}.wdir'", self.relpath)
|
format_and_raise(exc, f"'{self.where}.{name}.wdir'", self.relpath)
|
||||||
return self.resolver.fs.path.join(self.wdir, wdir)
|
return self.resolver.fs.join(self.wdir, wdir)
|
||||||
|
|
||||||
def resolve(self, **kwargs):
|
def resolve(self, **kwargs):
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -384,7 +384,7 @@ class Context(CtxDict):
|
|||||||
|
|
||||||
def merge_from(self, fs, item: str, wdir: str, overwrite=False):
|
def merge_from(self, fs, item: str, wdir: str, overwrite=False):
|
||||||
path, _, keys_str = item.partition(":")
|
path, _, keys_str = item.partition(":")
|
||||||
path = fs.path.normpath(fs.path.join(wdir, path))
|
path = fs.normpath(fs.join(wdir, path))
|
||||||
|
|
||||||
select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
|
select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
|
||||||
if path in self.imports:
|
if path in self.imports:
|
||||||
@@ -429,7 +429,7 @@ class Context(CtxDict):
|
|||||||
default: Optional[str] = None,
|
default: Optional[str] = None,
|
||||||
):
|
):
|
||||||
if default:
|
if default:
|
||||||
to_import = fs.path.join(wdir, default)
|
to_import = fs.join(wdir, default)
|
||||||
if fs.exists(to_import):
|
if fs.exists(to_import):
|
||||||
self.merge_from(fs, default, wdir)
|
self.merge_from(fs, default, wdir)
|
||||||
else:
|
else:
|
||||||
|
|||||||
+16
-16
@@ -114,7 +114,7 @@ class Repo:
|
|||||||
try:
|
try:
|
||||||
root_dir = self.find_root(root_dir, fs)
|
root_dir = self.find_root(root_dir, fs)
|
||||||
fs = fs or localfs
|
fs = fs or localfs
|
||||||
dvc_dir = fs.path.join(root_dir, self.DVC_DIR)
|
dvc_dir = fs.join(root_dir, self.DVC_DIR)
|
||||||
except NotDvcRepoError:
|
except NotDvcRepoError:
|
||||||
if not uninitialized:
|
if not uninitialized:
|
||||||
raise
|
raise
|
||||||
@@ -207,7 +207,7 @@ class Repo:
|
|||||||
self.fs.makedirs(self.tmp_dir, exist_ok=True)
|
self.fs.makedirs(self.tmp_dir, exist_ok=True)
|
||||||
|
|
||||||
self.lock = make_lock(
|
self.lock = make_lock(
|
||||||
self.fs.path.join(self.tmp_dir, "lock"),
|
self.fs.join(self.tmp_dir, "lock"),
|
||||||
tmp_dir=self.tmp_dir,
|
tmp_dir=self.tmp_dir,
|
||||||
hardlink_lock=self.config["core"].get("hardlink_lock", False),
|
hardlink_lock=self.config["core"].get("hardlink_lock", False),
|
||||||
friendly=True,
|
friendly=True,
|
||||||
@@ -251,7 +251,7 @@ class Repo:
|
|||||||
)
|
)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def local_dvc_dir(self):
|
def local_dvc_dir(self) -> Optional[str]:
|
||||||
from dvc.fs import GitFileSystem, LocalFileSystem
|
from dvc.fs import GitFileSystem, LocalFileSystem
|
||||||
|
|
||||||
if not self.dvc_dir:
|
if not self.dvc_dir:
|
||||||
@@ -263,10 +263,10 @@ class Repo:
|
|||||||
if not isinstance(self.fs, GitFileSystem):
|
if not isinstance(self.fs, GitFileSystem):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
relparts = ()
|
relparts: Tuple[str, ...] = ()
|
||||||
if self.root_dir != "/":
|
if self.root_dir != "/":
|
||||||
# subrepo
|
# subrepo
|
||||||
relparts = self.fs.path.relparts(self.root_dir, "/")
|
relparts = self.fs.relparts(self.root_dir, "/")
|
||||||
|
|
||||||
dvc_dir = os.path.join(
|
dvc_dir = os.path.join(
|
||||||
self.scm.root_dir,
|
self.scm.root_dir,
|
||||||
@@ -395,7 +395,7 @@ class Repo:
|
|||||||
index = subrepo.index.data[workspace]
|
index = subrepo.index.data[workspace]
|
||||||
else:
|
else:
|
||||||
index = self.index.data[workspace]
|
index = self.index.data[workspace]
|
||||||
key = self.fs.path.relparts(path, self.root_dir)
|
key = self.fs.relparts(path, self.root_dir)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return index, index[key]
|
return index, index[key]
|
||||||
@@ -411,18 +411,18 @@ class Repo:
|
|||||||
|
|
||||||
fs = fs or localfs
|
fs = fs or localfs
|
||||||
root = root or os.curdir
|
root = root or os.curdir
|
||||||
root_dir = fs.path.abspath(root)
|
root_dir = fs.abspath(root)
|
||||||
|
|
||||||
if not fs.isdir(root_dir):
|
if not fs.isdir(root_dir):
|
||||||
raise NotDvcRepoError(f"directory '{root}' does not exist")
|
raise NotDvcRepoError(f"directory '{root}' does not exist")
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
dvc_dir = fs.path.join(root_dir, cls.DVC_DIR)
|
dvc_dir = fs.join(root_dir, cls.DVC_DIR)
|
||||||
if fs.isdir(dvc_dir):
|
if fs.isdir(dvc_dir):
|
||||||
return root_dir
|
return root_dir
|
||||||
if isinstance(fs, LocalFileSystem) and os.path.ismount(root_dir):
|
if isinstance(fs, LocalFileSystem) and os.path.ismount(root_dir):
|
||||||
break
|
break
|
||||||
parent = fs.path.parent(root_dir)
|
parent = fs.parent(root_dir)
|
||||||
if parent == root_dir:
|
if parent == root_dir:
|
||||||
break
|
break
|
||||||
root_dir = parent
|
root_dir = parent
|
||||||
@@ -440,7 +440,7 @@ class Repo:
|
|||||||
|
|
||||||
fs = fs or localfs
|
fs = fs or localfs
|
||||||
root_dir = cls.find_root(root, fs=fs)
|
root_dir = cls.find_root(root, fs=fs)
|
||||||
return fs.path.join(root_dir, cls.DVC_DIR)
|
return fs.join(root_dir, cls.DVC_DIR)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False) -> "Repo":
|
def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False) -> "Repo":
|
||||||
@@ -542,19 +542,19 @@ class Repo:
|
|||||||
# using `outs_graph` to ensure graph checks are run
|
# using `outs_graph` to ensure graph checks are run
|
||||||
outs = outs or self.index.outs_graph
|
outs = outs or self.index.outs_graph
|
||||||
|
|
||||||
abs_path = self.fs.path.abspath(path)
|
abs_path = self.fs.abspath(path)
|
||||||
fs_path = abs_path
|
fs_path = abs_path
|
||||||
|
|
||||||
def func(out):
|
def func(out):
|
||||||
def eq(one, two):
|
def eq(one, two):
|
||||||
return one == two
|
return one == two
|
||||||
|
|
||||||
match = eq if strict else out.fs.path.isin_or_eq
|
match = eq if strict else out.fs.isin_or_eq
|
||||||
|
|
||||||
if out.protocol == "local" and match(fs_path, out.fs_path):
|
if out.protocol == "local" and match(fs_path, out.fs_path):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
if recursive and out.fs.path.isin(out.fs_path, fs_path):
|
if recursive and out.fs.isin(out.fs_path, fs_path):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
return False
|
return False
|
||||||
@@ -566,7 +566,7 @@ class Repo:
|
|||||||
return matched
|
return matched
|
||||||
|
|
||||||
def is_dvc_internal(self, path):
|
def is_dvc_internal(self, path):
|
||||||
path_parts = self.fs.path.normpath(path).split(self.fs.sep)
|
path_parts = self.fs.normpath(path).split(self.fs.sep)
|
||||||
return self.DVC_DIR in path_parts
|
return self.DVC_DIR in path_parts
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
@@ -613,10 +613,10 @@ class Repo:
|
|||||||
cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
|
cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
|
||||||
|
|
||||||
if isinstance(self.fs, GitFileSystem):
|
if isinstance(self.fs, GitFileSystem):
|
||||||
relparts = ()
|
relparts: Tuple[str, ...] = ()
|
||||||
if self.root_dir != "/":
|
if self.root_dir != "/":
|
||||||
# subrepo
|
# subrepo
|
||||||
relparts = self.fs.path.relparts(self.root_dir, "/")
|
relparts = self.fs.relparts(self.root_dir, "/")
|
||||||
root_dir = os.path.join(self.scm.root_dir, *relparts)
|
root_dir = os.path.join(self.scm.root_dir, *relparts)
|
||||||
else:
|
else:
|
||||||
root_dir = self.root_dir
|
root_dir = self.root_dir
|
||||||
|
|||||||
+1
-1
@@ -181,7 +181,7 @@ def _add_transfer(
|
|||||||
|
|
||||||
def _add(stage: "Stage", source: Optional[str] = None, no_commit: bool = False) -> None:
|
def _add(stage: "Stage", source: Optional[str] = None, no_commit: bool = False) -> None:
|
||||||
out = stage.outs[0]
|
out = stage.outs[0]
|
||||||
path = out.fs.path.abspath(source) if source else None
|
path = out.fs.abspath(source) if source else None
|
||||||
try:
|
try:
|
||||||
stage.add_outs(path, no_commit=no_commit)
|
stage.add_outs(path, no_commit=no_commit)
|
||||||
except CacheLinkError:
|
except CacheLinkError:
|
||||||
|
|||||||
@@ -229,7 +229,7 @@ class Artifacts:
|
|||||||
dvc_studio_config=dvc_studio_config,
|
dvc_studio_config=dvc_studio_config,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
).items():
|
).items():
|
||||||
to_info = localfs.path.join(out, path)
|
to_info = localfs.join(out, path)
|
||||||
if localfs.exists(to_info) and not force:
|
if localfs.exists(to_info) and not force:
|
||||||
hint = "\nTo override it, re-run with '--force'."
|
hint = "\nTo override it, re-run with '--force'."
|
||||||
raise FileExistsLocallyError( # noqa: TRY301
|
raise FileExistsLocallyError( # noqa: TRY301
|
||||||
@@ -254,7 +254,7 @@ class Artifacts:
|
|||||||
fs, from_infos, localfs, to_infos, callback=cb, batch_size=jobs
|
fs, from_infos, localfs, to_infos, callback=cb, batch_size=jobs
|
||||||
)
|
)
|
||||||
|
|
||||||
return len(to_infos), relpath(localfs.path.commonpath(to_infos))
|
return len(to_infos), relpath(localfs.commonpath(to_infos))
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get( # noqa: PLR0913
|
def get( # noqa: PLR0913
|
||||||
|
|||||||
+12
-12
@@ -60,12 +60,12 @@ def brancher(
|
|||||||
from dvc.fs import LocalFileSystem
|
from dvc.fs import LocalFileSystem
|
||||||
|
|
||||||
repo_root_parts: Tuple[str, ...] = ()
|
repo_root_parts: Tuple[str, ...] = ()
|
||||||
if self.fs.path.isin(self.root_dir, self.scm.root_dir):
|
if self.fs.isin(self.root_dir, self.scm.root_dir):
|
||||||
repo_root_parts = self.fs.path.relparts(self.root_dir, self.scm.root_dir)
|
repo_root_parts = self.fs.relparts(self.root_dir, self.scm.root_dir)
|
||||||
|
|
||||||
cwd_parts: Tuple[str, ...] = ()
|
cwd_parts: Tuple[str, ...] = ()
|
||||||
if self.fs.path.isin(self.fs.path.getcwd(), self.scm.root_dir):
|
if self.fs.isin(self.fs.getcwd(), self.scm.root_dir):
|
||||||
cwd_parts = self.fs.path.relparts(self.fs.path.getcwd(), self.scm.root_dir)
|
cwd_parts = self.fs.relparts(self.fs.getcwd(), self.scm.root_dir)
|
||||||
|
|
||||||
saved_fs = self.fs
|
saved_fs = self.fs
|
||||||
saved_root = self.root_dir
|
saved_root = self.root_dir
|
||||||
@@ -125,18 +125,18 @@ def _switch_fs(
|
|||||||
logger.trace("switching fs to revision %s", rev[:7])
|
logger.trace("switching fs to revision %s", rev[:7])
|
||||||
assert isinstance(repo.scm, Git)
|
assert isinstance(repo.scm, Git)
|
||||||
fs = GitFileSystem(scm=repo.scm, rev=rev)
|
fs = GitFileSystem(scm=repo.scm, rev=rev)
|
||||||
root_dir = repo.fs.path.join("/", *repo_root_parts)
|
root_dir = repo.fs.join("/", *repo_root_parts)
|
||||||
if not fs.exists(root_dir):
|
if not fs.exists(root_dir):
|
||||||
raise NotDvcRepoError(f"Commit '{rev[:7]}' does not contain a DVC repo")
|
raise NotDvcRepoError(f"Commit '{rev[:7]}' does not contain a DVC repo")
|
||||||
|
|
||||||
repo.fs = fs
|
repo.fs = fs
|
||||||
repo.root_dir = root_dir
|
repo.root_dir = root_dir
|
||||||
repo.dvc_dir = fs.path.join(root_dir, repo.DVC_DIR)
|
repo.dvc_dir = fs.join(root_dir, repo.DVC_DIR)
|
||||||
repo._reset()
|
repo._reset()
|
||||||
|
|
||||||
if cwd_parts:
|
if cwd_parts:
|
||||||
cwd = repo.fs.path.join("/", *cwd_parts)
|
cwd = repo.fs.join("/", *cwd_parts)
|
||||||
repo.fs.path.chdir(cwd)
|
repo.fs.chdir(cwd)
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
@@ -148,12 +148,12 @@ def switch(repo: "Repo", rev: str) -> Iterator[str]:
|
|||||||
rev = resolve_rev(repo.scm, rev)
|
rev = resolve_rev(repo.scm, rev)
|
||||||
|
|
||||||
repo_root_parts: Tuple[str, ...] = ()
|
repo_root_parts: Tuple[str, ...] = ()
|
||||||
if repo.fs.path.isin(repo.root_dir, repo.scm.root_dir):
|
if repo.fs.isin(repo.root_dir, repo.scm.root_dir):
|
||||||
repo_root_parts = repo.fs.path.relparts(repo.root_dir, repo.scm.root_dir)
|
repo_root_parts = repo.fs.relparts(repo.root_dir, repo.scm.root_dir)
|
||||||
|
|
||||||
cwd_parts: Tuple[str, ...] = ()
|
cwd_parts: Tuple[str, ...] = ()
|
||||||
if repo.fs.path.isin(repo.fs.path.getcwd(), repo.scm.root_dir):
|
if repo.fs.isin(repo.fs.getcwd(), repo.scm.root_dir):
|
||||||
cwd_parts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.scm.root_dir)
|
cwd_parts = repo.fs.relparts(repo.fs.getcwd(), repo.scm.root_dir)
|
||||||
|
|
||||||
saved_fs = repo.fs
|
saved_fs = repo.fs
|
||||||
saved_root = repo.root_dir
|
saved_root = repo.root_dir
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ def _check_can_delete(
|
|||||||
if cache_fs.exists(cache_path):
|
if cache_fs.exists(cache_path):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
entry_paths.append(fs.path.join(path, *(entry.key or ())))
|
entry_paths.append(fs.join(path, *(entry.key or ())))
|
||||||
|
|
||||||
if not entry_paths:
|
if not entry_paths:
|
||||||
return
|
return
|
||||||
@@ -172,7 +172,7 @@ def checkout( # noqa: C901
|
|||||||
)
|
)
|
||||||
|
|
||||||
for out_path in out_paths:
|
for out_path in out_paths:
|
||||||
if self.fs.path.isin_or_eq(dest_path, out_path):
|
if self.fs.isin_or_eq(dest_path, out_path):
|
||||||
failed.add(out_path)
|
failed.add(out_path)
|
||||||
|
|
||||||
with ui.progress(
|
with ui.progress(
|
||||||
@@ -195,7 +195,7 @@ def checkout( # noqa: C901
|
|||||||
|
|
||||||
typ_map = {ADD: "added", DELETE: "deleted", MODIFY: "modified"}
|
typ_map = {ADD: "added", DELETE: "deleted", MODIFY: "modified"}
|
||||||
for key, typ in out_changes.items():
|
for key, typ in out_changes.items():
|
||||||
out_path = self.fs.path.join(self.root_dir, *key)
|
out_path = self.fs.join(self.root_dir, *key)
|
||||||
|
|
||||||
if out_path in failed:
|
if out_path in failed:
|
||||||
self.fs.remove(out_path, recursive=True)
|
self.fs.remove(out_path, recursive=True)
|
||||||
|
|||||||
+3
-3
@@ -205,7 +205,7 @@ class Status(TypedDict):
|
|||||||
|
|
||||||
def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]:
|
def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]:
|
||||||
"""Transform files rel. to Git root to DVC root, and drop outside files."""
|
"""Transform files rel. to Git root to DVC root, and drop outside files."""
|
||||||
rel = repo.fs.path.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
|
rel = repo.fs.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
|
||||||
|
|
||||||
# if we have repo root in a different location than scm's root,
|
# if we have repo root in a different location than scm's root,
|
||||||
# i.e. subdir repo, all git_paths need to be transformed rel. to the DVC
|
# i.e. subdir repo, all git_paths need to be transformed rel. to the DVC
|
||||||
@@ -215,11 +215,11 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]
|
|||||||
length = len(prefix)
|
length = len(prefix)
|
||||||
files = (file[length:] for file in files if file.startswith(prefix))
|
files = (file[length:] for file in files if file.startswith(prefix))
|
||||||
|
|
||||||
start = repo.fs.path.relpath(repo.fs.path.getcwd(), repo.root_dir)
|
start = repo.fs.relpath(repo.fs.getcwd(), repo.root_dir)
|
||||||
if start in (os.curdir, ""):
|
if start in (os.curdir, ""):
|
||||||
return list(files)
|
return list(files)
|
||||||
# we need to convert repo relative paths to curdir relative.
|
# we need to convert repo relative paths to curdir relative.
|
||||||
return [repo.fs.path.relpath(file, start) for file in files]
|
return [repo.fs.relpath(file, start) for file in files]
|
||||||
|
|
||||||
|
|
||||||
def status(
|
def status(
|
||||||
|
|||||||
@@ -171,7 +171,7 @@ class ApplyStash(Stash):
|
|||||||
right_fs = self.scm.get_fs(right_rev)
|
right_fs = self.scm.get_fs(right_rev)
|
||||||
paths = [path for path in left_fs.find("/") if not right_fs.exists(path)]
|
paths = [path for path in left_fs.find("/") if not right_fs.exists(path)]
|
||||||
dest_paths = [
|
dest_paths = [
|
||||||
localfs.path.join(self.scm.root_dir, left_fs.path.relpath(path, "/"))
|
localfs.join(self.scm.root_dir, left_fs.relpath(path, "/"))
|
||||||
for path in paths
|
for path in paths
|
||||||
]
|
]
|
||||||
for src, dest in zip(paths, dest_paths):
|
for src, dest in zip(paths, dest_paths):
|
||||||
|
|||||||
+1
-1
@@ -219,7 +219,7 @@ def _log_unversioned(data: List["DataIndex"]) -> Tuple[List["DataIndex"], int]:
|
|||||||
index.storage_map = fs_index.storage_map
|
index.storage_map = fs_index.storage_map
|
||||||
for key, entry in fs_index.iteritems():
|
for key, entry in fs_index.iteritems():
|
||||||
if entry.meta and not entry.meta.isdir and entry.meta.version_id is None:
|
if entry.meta and not entry.meta.isdir and entry.meta.version_id is None:
|
||||||
unversioned.append(fs.unstrip_protocol(fs.path.join(remote.path, *key)))
|
unversioned.append(fs.unstrip_protocol(fs.join(remote.path, *key)))
|
||||||
else:
|
else:
|
||||||
index[key] = entry
|
index[key] = entry
|
||||||
fs_index.close()
|
fs_index.close()
|
||||||
|
|||||||
+3
-3
@@ -126,7 +126,7 @@ def build_graph(stages, outs_trie=None):
|
|||||||
outs_trie = outs_trie or build_outs_trie(stages)
|
outs_trie = outs_trie or build_outs_trie(stages)
|
||||||
|
|
||||||
for stage in stages:
|
for stage in stages:
|
||||||
out = outs_trie.shortest_prefix(localfs.path.parts(stage.path)).value
|
out = outs_trie.shortest_prefix(localfs.parts(stage.path)).value
|
||||||
if out:
|
if out:
|
||||||
raise StagePathAsOutputError(stage, str(out))
|
raise StagePathAsOutputError(stage, str(out))
|
||||||
|
|
||||||
@@ -139,7 +139,7 @@ def build_graph(stages, outs_trie=None):
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
for dep in stage.deps:
|
for dep in stage.deps:
|
||||||
dep_key = dep.fs.path.parts(dep.fs_path)
|
dep_key = dep.fs.parts(dep.fs_path)
|
||||||
overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
|
overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
|
||||||
if outs_trie.has_subtrie(dep_key):
|
if outs_trie.has_subtrie(dep_key):
|
||||||
overlapping.extend(outs_trie.values(prefix=dep_key))
|
overlapping.extend(outs_trie.values(prefix=dep_key))
|
||||||
@@ -165,7 +165,7 @@ def build_outs_graph(graph, outs_trie):
|
|||||||
if stage.is_db_import:
|
if stage.is_db_import:
|
||||||
continue
|
continue
|
||||||
for dep in stage.deps:
|
for dep in stage.deps:
|
||||||
dep_key = dep.fs.path.parts(dep.fs_path)
|
dep_key = dep.fs.parts(dep.fs_path)
|
||||||
overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
|
overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
|
||||||
if outs_trie.has_subtrie(dep_key):
|
if outs_trie.has_subtrie(dep_key):
|
||||||
overlapping.extend(outs_trie.values(prefix=dep_key))
|
overlapping.extend(outs_trie.values(prefix=dep_key))
|
||||||
|
|||||||
+11
-11
@@ -97,7 +97,7 @@ def collect_files(
|
|||||||
for root, dirs, files in walk_iter:
|
for root, dirs, files in walk_iter:
|
||||||
dvcfile_filter = partial(is_dvcfile_and_not_ignored, root)
|
dvcfile_filter = partial(is_dvcfile_and_not_ignored, root)
|
||||||
for file in filter(dvcfile_filter, files):
|
for file in filter(dvcfile_filter, files):
|
||||||
file_path = fs.path.join(root, file)
|
file_path = fs.join(root, file)
|
||||||
try:
|
try:
|
||||||
index = Index.from_file(repo, file_path)
|
index = Index.from_file(repo, file_path)
|
||||||
except DvcException as exc:
|
except DvcException as exc:
|
||||||
@@ -201,7 +201,7 @@ def _load_storage_from_out(storage_map, key, out):
|
|||||||
FileStorage(
|
FileStorage(
|
||||||
key,
|
key,
|
||||||
fs_cache.fs,
|
fs_cache.fs,
|
||||||
fs_cache.fs.path.join(
|
fs_cache.fs.join(
|
||||||
fs_cache.path, dep.fs.protocol, tokenize(dep.fs_path)
|
fs_cache.path, dep.fs.protocol, tokenize(dep.fs_path)
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -414,7 +414,7 @@ class Index:
|
|||||||
by_workspace[workspace].add(key)
|
by_workspace[workspace].add(key)
|
||||||
|
|
||||||
for path in _collect_top_level_metrics(self.repo):
|
for path in _collect_top_level_metrics(self.repo):
|
||||||
key = self.repo.fs.path.relparts(path, self.repo.root_dir)
|
key = self.repo.fs.relparts(path, self.repo.root_dir)
|
||||||
by_workspace["repo"].add(key)
|
by_workspace["repo"].add(key)
|
||||||
|
|
||||||
return dict(by_workspace)
|
return dict(by_workspace)
|
||||||
@@ -432,7 +432,7 @@ class Index:
|
|||||||
param_paths = chain(param_paths, [default_file])
|
param_paths = chain(param_paths, [default_file])
|
||||||
|
|
||||||
for path in param_paths:
|
for path in param_paths:
|
||||||
key = self.repo.fs.path.relparts(path, self.repo.root_dir)
|
key = self.repo.fs.relparts(path, self.repo.root_dir)
|
||||||
by_workspace["repo"].add(key)
|
by_workspace["repo"].add(key)
|
||||||
|
|
||||||
return dict(by_workspace)
|
return dict(by_workspace)
|
||||||
@@ -451,7 +451,7 @@ class Index:
|
|||||||
by_workspace[workspace].add(key)
|
by_workspace[workspace].add(key)
|
||||||
|
|
||||||
for path in self._plot_sources:
|
for path in self._plot_sources:
|
||||||
key = self.repo.fs.path.parts(path)
|
key = self.repo.fs.parts(path)
|
||||||
by_workspace["repo"].add(key)
|
by_workspace["repo"].add(key)
|
||||||
|
|
||||||
return dict(by_workspace)
|
return dict(by_workspace)
|
||||||
@@ -701,8 +701,8 @@ class IndexView:
|
|||||||
if not out.use_cache:
|
if not out.use_cache:
|
||||||
continue
|
continue
|
||||||
workspace, key = out.index_key
|
workspace, key = out.index_key
|
||||||
if filter_info and out.fs.path.isin(filter_info, out.fs_path):
|
if filter_info and out.fs.isin(filter_info, out.fs_path):
|
||||||
key = key + out.fs.path.relparts(filter_info, out.fs_path)
|
key = key + out.fs.relparts(filter_info, out.fs_path)
|
||||||
entry = self._index.data[workspace].get(key)
|
entry = self._index.data[workspace].get(key)
|
||||||
if entry and entry.meta and entry.meta.isdir:
|
if entry and entry.meta and entry.meta.isdir:
|
||||||
prefixes[workspace].recursive.add(key)
|
prefixes[workspace].recursive.add(key)
|
||||||
@@ -718,8 +718,8 @@ class IndexView:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
workspace, key = out.index_key
|
workspace, key = out.index_key
|
||||||
if filter_info and out.fs.path.isin(filter_info, out.fs_path):
|
if filter_info and out.fs.isin(filter_info, out.fs_path):
|
||||||
key = key + out.fs.path.relparts(filter_info, out.fs_path)
|
key = key + out.fs.relparts(filter_info, out.fs_path)
|
||||||
ret[workspace].add(key)
|
ret[workspace].add(key)
|
||||||
|
|
||||||
return dict(ret)
|
return dict(ret)
|
||||||
@@ -782,7 +782,7 @@ def build_data_index( # noqa: C901, PLR0912
|
|||||||
data = DataIndex()
|
data = DataIndex()
|
||||||
parents = set()
|
parents = set()
|
||||||
for key in index.data_keys.get(workspace, set()):
|
for key in index.data_keys.get(workspace, set()):
|
||||||
out_path = fs.path.join(path, *key)
|
out_path = fs.join(path, *key)
|
||||||
|
|
||||||
for key_len in range(1, len(key)):
|
for key_len in range(1, len(key)):
|
||||||
parents.add(key[:key_len])
|
parents.add(key[:key_len])
|
||||||
@@ -827,7 +827,7 @@ def build_data_index( # noqa: C901, PLR0912
|
|||||||
callback.relative_update(1)
|
callback.relative_update(1)
|
||||||
|
|
||||||
for key in parents:
|
for key in parents:
|
||||||
parent_path = fs.path.join(path, *key)
|
parent_path = fs.join(path, *key)
|
||||||
if not fs.exists(parent_path):
|
if not fs.exists(parent_path):
|
||||||
continue
|
continue
|
||||||
direntry = DataIndexEntry(key=key, meta=Meta(isdir=True), loaded=True)
|
direntry = DataIndexEntry(key=key, meta=Meta(isdir=True), loaded=True)
|
||||||
|
|||||||
+1
-1
@@ -89,7 +89,7 @@ def _ls(
|
|||||||
if not recursive:
|
if not recursive:
|
||||||
files.update(dirs)
|
files.update(dirs)
|
||||||
|
|
||||||
parts = fs.path.relparts(root, fs_path)
|
parts = fs.relparts(root, fs_path)
|
||||||
if parts == (".",):
|
if parts == (".",):
|
||||||
parts = ()
|
parts = ()
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -18,7 +18,7 @@ def ls_url(url, *, fs_config=None, recursive=False, config=None):
|
|||||||
|
|
||||||
for info in files.values():
|
for info in files.values():
|
||||||
ls_info = {
|
ls_info = {
|
||||||
"path": fs.path.relpath(info["name"], fs_path),
|
"path": fs.relpath(info["name"], fs_path),
|
||||||
"isdir": info["type"] == "directory",
|
"isdir": info["type"] == "directory",
|
||||||
"size": info.get("size"),
|
"size": info.get("size"),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,10 +35,10 @@ logger = logger.getChild(__name__)
|
|||||||
def _collect_top_level_metrics(repo: "Repo") -> Iterator[str]:
|
def _collect_top_level_metrics(repo: "Repo") -> Iterator[str]:
|
||||||
top_metrics = repo.index._metrics
|
top_metrics = repo.index._metrics
|
||||||
for dvcfile, metrics in top_metrics.items():
|
for dvcfile, metrics in top_metrics.items():
|
||||||
wdir = repo.fs.path.relpath(repo.fs.path.parent(dvcfile), repo.root_dir)
|
wdir = repo.fs.relpath(repo.fs.parent(dvcfile), repo.root_dir)
|
||||||
for file in metrics:
|
for file in metrics:
|
||||||
path = repo.fs.path.join(wdir, as_posix(file))
|
path = repo.fs.join(wdir, as_posix(file))
|
||||||
yield repo.fs.path.normpath(path)
|
yield repo.fs.normpath(path)
|
||||||
|
|
||||||
|
|
||||||
def _extract_metrics(metrics, path: str):
|
def _extract_metrics(metrics, path: str):
|
||||||
@@ -101,7 +101,7 @@ def _collect_metrics(
|
|||||||
|
|
||||||
if not targets or outs_only:
|
if not targets or outs_only:
|
||||||
outs = metrics_from_target(repo, stages) if stages else repo.index.metrics
|
outs = metrics_from_target(repo, stages) if stages else repo.index.metrics
|
||||||
relpath = repo.fs.path.relpath
|
relpath = repo.fs.relpath
|
||||||
metrics.extend(relpath(out.fs_path, repo.root_dir) for out in outs)
|
metrics.extend(relpath(out.fs_path, repo.root_dir) for out in outs)
|
||||||
|
|
||||||
if not targets and not outs_only and not stages:
|
if not targets and not outs_only and not stages:
|
||||||
@@ -128,8 +128,8 @@ class Result(TypedDict, total=False):
|
|||||||
|
|
||||||
|
|
||||||
def to_relpath(fs: "FileSystem", root_dir: str, d: Result) -> Result:
|
def to_relpath(fs: "FileSystem", root_dir: str, d: Result) -> Result:
|
||||||
relpath = fs.path.relpath
|
relpath = fs.relpath
|
||||||
cwd = fs.path.getcwd()
|
cwd = fs.getcwd()
|
||||||
|
|
||||||
start = relpath(cwd, root_dir)
|
start = relpath(cwd, root_dir)
|
||||||
data = d.get("data")
|
data = d.get("data")
|
||||||
@@ -156,7 +156,7 @@ def _gather_metrics(
|
|||||||
fs = repo.dvcfs
|
fs = repo.dvcfs
|
||||||
for fs_path, result in _read_metrics(fs, files, cache=True):
|
for fs_path, result in _read_metrics(fs, files, cache=True):
|
||||||
repo_path = fs_path.lstrip(fs.root_marker)
|
repo_path = fs_path.lstrip(fs.root_marker)
|
||||||
repo_os_path = os.sep.join(fs.path.parts(repo_path))
|
repo_os_path = os.sep.join(fs.parts(repo_path))
|
||||||
if not isinstance(result, Exception):
|
if not isinstance(result, Exception):
|
||||||
data.update({repo_os_path: FileResult(data=result)})
|
data.update({repo_os_path: FileResult(data=result)})
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ def erepo_factory(url, root_dir, cache_config):
|
|||||||
_config = cache_config.copy()
|
_config = cache_config.copy()
|
||||||
if os.path.isdir(url):
|
if os.path.isdir(url):
|
||||||
fs = fs or localfs
|
fs = fs or localfs
|
||||||
repo_path = os.path.join(url, *fs.path.relparts(path, root_dir))
|
repo_path = os.path.join(url, *fs.relparts(path, root_dir))
|
||||||
_config.update(_get_remote_config(repo_path))
|
_config.update(_get_remote_config(repo_path))
|
||||||
return Repo(path, fs=fs, config=_config, **_kwargs)
|
return Repo(path, fs=fs, config=_config, **_kwargs)
|
||||||
|
|
||||||
|
|||||||
@@ -21,10 +21,10 @@ logger = logger.getChild(__name__)
|
|||||||
def _collect_top_level_params(repo: "Repo") -> Iterator[str]:
|
def _collect_top_level_params(repo: "Repo") -> Iterator[str]:
|
||||||
top_params = repo.index._params
|
top_params = repo.index._params
|
||||||
for dvcfile, params in top_params.items():
|
for dvcfile, params in top_params.items():
|
||||||
wdir = repo.fs.path.relpath(repo.fs.path.parent(dvcfile), repo.root_dir)
|
wdir = repo.fs.relpath(repo.fs.parent(dvcfile), repo.root_dir)
|
||||||
for file in params:
|
for file in params:
|
||||||
path = repo.fs.path.join(wdir, as_posix(file))
|
path = repo.fs.join(wdir, as_posix(file))
|
||||||
yield repo.fs.path.normpath(path)
|
yield repo.fs.normpath(path)
|
||||||
|
|
||||||
|
|
||||||
def params_from_target(
|
def params_from_target(
|
||||||
@@ -55,7 +55,7 @@ def _collect_params(
|
|||||||
|
|
||||||
if not targets or stages:
|
if not targets or stages:
|
||||||
deps = params_from_target(repo, stages) if stages else repo.index.params
|
deps = params_from_target(repo, stages) if stages else repo.index.params
|
||||||
relpath = repo.fs.path.relpath
|
relpath = repo.fs.relpath
|
||||||
params.extend(
|
params.extend(
|
||||||
{relpath(dep.fs_path, repo.root_dir): list(dep.params)} for dep in deps
|
{relpath(dep.fs_path, repo.root_dir): list(dep.params)} for dep in deps
|
||||||
)
|
)
|
||||||
@@ -93,7 +93,7 @@ def _collect_vars(repo, params, stages=None) -> Dict:
|
|||||||
# to reduce noise and duplication, they are skipped
|
# to reduce noise and duplication, they are skipped
|
||||||
|
|
||||||
# `file` is relative
|
# `file` is relative
|
||||||
abspath = repo.fs.path.abspath(file)
|
abspath = repo.fs.abspath(file)
|
||||||
repo_path = repo.dvcfs.from_os_path(abspath)
|
repo_path = repo.dvcfs.from_os_path(abspath)
|
||||||
if repo_path in params:
|
if repo_path in params:
|
||||||
continue
|
continue
|
||||||
@@ -138,7 +138,7 @@ def _gather_params(
|
|||||||
fs = repo.dvcfs
|
fs = repo.dvcfs
|
||||||
for fs_path, result in _read_params(fs, files_keypaths, cache=True):
|
for fs_path, result in _read_params(fs, files_keypaths, cache=True):
|
||||||
repo_path = fs_path.lstrip(fs.root_marker)
|
repo_path = fs_path.lstrip(fs.root_marker)
|
||||||
repo_os_path = os.sep.join(fs.path.parts(repo_path))
|
repo_os_path = os.sep.join(fs.parts(repo_path))
|
||||||
if not isinstance(result, Exception):
|
if not isinstance(result, Exception):
|
||||||
data.update({repo_os_path: FileResult(data=result)})
|
data.update({repo_os_path: FileResult(data=result)})
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -386,7 +386,7 @@ def _matches(targets, config_file, plot_id):
|
|||||||
|
|
||||||
|
|
||||||
def _normpath(path):
|
def _normpath(path):
|
||||||
# TODO dvcfs.path.normopath normalizes to windows path on Windows
|
# TODO dvcfs.normopath normalizes to windows path on Windows
|
||||||
# even though other methods work as expected
|
# even though other methods work as expected
|
||||||
import posixpath
|
import posixpath
|
||||||
|
|
||||||
@@ -399,7 +399,7 @@ def _relpath(fs, path):
|
|||||||
# and invoking from some subdir `dvcfile.relpath` returns strange long
|
# and invoking from some subdir `dvcfile.relpath` returns strange long
|
||||||
# relative paths
|
# relative paths
|
||||||
# ("../../../../../../dvc.yaml") - investigate
|
# ("../../../../../../dvc.yaml") - investigate
|
||||||
return fs.path.relpath(fs.path.join("/", fs.from_os_path(path)), fs.path.getcwd())
|
return fs.relpath(fs.join("/", fs.from_os_path(path)), fs.getcwd())
|
||||||
|
|
||||||
|
|
||||||
def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None):
|
def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None):
|
||||||
@@ -413,7 +413,7 @@ def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = No
|
|||||||
if _matches(targets, config_path, str(plot)):
|
if _matches(targets, config_path, str(plot)):
|
||||||
unpacked = unpack_if_dir(
|
unpacked = unpack_if_dir(
|
||||||
fs,
|
fs,
|
||||||
_normpath(fs.path.join(wdir_relpath, plot.def_path)),
|
_normpath(fs.join(wdir_relpath, plot.def_path)),
|
||||||
props={**plot_props, **props},
|
props={**plot_props, **props},
|
||||||
onerror=onerror,
|
onerror=onerror,
|
||||||
)
|
)
|
||||||
@@ -438,7 +438,7 @@ def _adjust_sources(fs, plot_props, config_dir):
|
|||||||
old_y = new_plot_props.pop("y", {})
|
old_y = new_plot_props.pop("y", {})
|
||||||
new_y = {}
|
new_y = {}
|
||||||
for filepath, val in old_y.items():
|
for filepath, val in old_y.items():
|
||||||
new_y[_normpath(fs.path.join(config_dir, filepath))] = val
|
new_y[_normpath(fs.join(config_dir, filepath))] = val
|
||||||
new_plot_props["y"] = new_y
|
new_plot_props["y"] = new_y
|
||||||
return new_plot_props
|
return new_plot_props
|
||||||
|
|
||||||
@@ -452,13 +452,13 @@ def _resolve_definitions(
|
|||||||
onerror: Optional[Callable[[Any], Any]] = None,
|
onerror: Optional[Callable[[Any], Any]] = None,
|
||||||
):
|
):
|
||||||
config_path = os.fspath(config_path)
|
config_path = os.fspath(config_path)
|
||||||
config_dir = fs.path.dirname(config_path)
|
config_dir = fs.dirname(config_path)
|
||||||
result: Dict[str, Dict] = {}
|
result: Dict[str, Dict] = {}
|
||||||
for plot_id, plot_props in definitions.items():
|
for plot_id, plot_props in definitions.items():
|
||||||
if plot_props is None:
|
if plot_props is None:
|
||||||
plot_props = {}
|
plot_props = {}
|
||||||
if _id_is_path(plot_props):
|
if _id_is_path(plot_props):
|
||||||
data_path = _normpath(fs.path.join(config_dir, plot_id))
|
data_path = _normpath(fs.join(config_dir, plot_id))
|
||||||
if _matches(targets, config_path, plot_id):
|
if _matches(targets, config_path, plot_id):
|
||||||
unpacked = unpack_if_dir(
|
unpacked = unpack_if_dir(
|
||||||
fs,
|
fs,
|
||||||
|
|||||||
+3
-3
@@ -219,7 +219,7 @@ class StageLoad:
|
|||||||
self, path: Optional[str] = None, name: Optional[str] = None
|
self, path: Optional[str] = None, name: Optional[str] = None
|
||||||
) -> str:
|
) -> str:
|
||||||
if path:
|
if path:
|
||||||
return self.repo.fs.path.abspath(path)
|
return self.repo.fs.abspath(path)
|
||||||
|
|
||||||
path = PROJECT_FILE
|
path = PROJECT_FILE
|
||||||
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
|
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
|
||||||
@@ -348,7 +348,7 @@ class StageLoad:
|
|||||||
if recursive and self.fs.isdir(target):
|
if recursive and self.fs.isdir(target):
|
||||||
from dvc.repo.graph import collect_inside_path
|
from dvc.repo.graph import collect_inside_path
|
||||||
|
|
||||||
path = self.fs.path.abspath(target)
|
path = self.fs.abspath(target)
|
||||||
return collect_inside_path(path, graph or self.repo.index.graph)
|
return collect_inside_path(path, graph or self.repo.index.graph)
|
||||||
|
|
||||||
stages = self.from_target(target, glob=glob)
|
stages = self.from_target(target, glob=glob)
|
||||||
@@ -393,7 +393,7 @@ class StageLoad:
|
|||||||
if not (recursive and self.fs.isdir(target)):
|
if not (recursive and self.fs.isdir(target)):
|
||||||
try:
|
try:
|
||||||
(out,) = self.repo.find_outs_by_path(target, strict=False)
|
(out,) = self.repo.find_outs_by_path(target, strict=False)
|
||||||
return [StageInfo(out.stage, self.fs.path.abspath(target))]
|
return [StageInfo(out.stage, self.fs.abspath(target))]
|
||||||
except OutputNotFoundError:
|
except OutputNotFoundError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -9,7 +9,7 @@ def build_outs_trie(stages):
|
|||||||
|
|
||||||
for stage in stages:
|
for stage in stages:
|
||||||
for out in stage.outs:
|
for out in stage.outs:
|
||||||
out_key = out.fs.path.parts(out.fs_path)
|
out_key = out.fs.parts(out.fs_path)
|
||||||
|
|
||||||
# Check for dup outs
|
# Check for dup outs
|
||||||
if out_key in outs:
|
if out_key in outs:
|
||||||
|
|||||||
@@ -131,10 +131,11 @@ def _merge_push_meta( # noqa: C901
|
|||||||
entries.append(entry)
|
entries.append(entry)
|
||||||
if entry.meta is not None and entry.meta.isdir:
|
if entry.meta is not None and entry.meta.isdir:
|
||||||
continue
|
continue
|
||||||
fs_path = repo.fs.path.join(repo.root_dir, *subkey)
|
fs_path = repo.fs.join(repo.root_dir, *subkey)
|
||||||
meta, hash_info = old_tree.get(
|
meta, hash_info = old_tree.get(repo.fs.relparts(fs_path, out.fs_path)) or (
|
||||||
repo.fs.path.relparts(fs_path, out.fs_path)
|
None,
|
||||||
) or (None, None)
|
None,
|
||||||
|
)
|
||||||
entry.hash_info = hash_info
|
entry.hash_info = hash_info
|
||||||
if entry.meta:
|
if entry.meta:
|
||||||
entry.meta.remote = remote
|
entry.meta.remote = remote
|
||||||
@@ -354,11 +355,11 @@ def _get_update_diff_index(
|
|||||||
# downloading
|
# downloading
|
||||||
if out.isdir():
|
if out.isdir():
|
||||||
if not entry.meta.isdir:
|
if not entry.meta.isdir:
|
||||||
fs_path = repo.fs.path.join(repo.root_dir, *entry.key)
|
fs_path = repo.fs.join(repo.root_dir, *entry.key)
|
||||||
tree = out.obj
|
tree = out.obj
|
||||||
assert isinstance(tree, Tree)
|
assert isinstance(tree, Tree)
|
||||||
_, entry.hash_info = tree.get( # type: ignore[misc]
|
_, entry.hash_info = tree.get( # type: ignore[misc]
|
||||||
repo.fs.path.relparts(fs_path, out.fs_path)
|
repo.fs.relparts(fs_path, out.fs_path)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
entry.hash_info = out.hash_info
|
entry.hash_info = out.hash_info
|
||||||
|
|||||||
+3
-3
@@ -43,10 +43,10 @@ class RWLockFileFormatError(DvcException):
|
|||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def _edit_rwlock(lock_dir, fs, hardlink):
|
def _edit_rwlock(lock_dir, fs, hardlink):
|
||||||
path = fs.path.join(lock_dir, RWLOCK_FILE)
|
path = fs.join(lock_dir, RWLOCK_FILE)
|
||||||
|
|
||||||
rwlock_guard = make_lock(
|
rwlock_guard = make_lock(
|
||||||
fs.path.join(lock_dir, RWLOCK_LOCK),
|
fs.join(lock_dir, RWLOCK_LOCK),
|
||||||
tmp_dir=lock_dir,
|
tmp_dir=lock_dir,
|
||||||
hardlink_lock=hardlink,
|
hardlink_lock=hardlink,
|
||||||
)
|
)
|
||||||
@@ -82,7 +82,7 @@ def _check_blockers(tmp_dir, lock, info, *, mode, waiters): # noqa: C901, PLR09
|
|||||||
to_release = defaultdict(list)
|
to_release = defaultdict(list)
|
||||||
for path, infos in lock[mode].items():
|
for path, infos in lock[mode].items():
|
||||||
for waiter_path in waiters:
|
for waiter_path in waiters:
|
||||||
if localfs.path.overlaps(waiter_path, path):
|
if localfs.overlaps(waiter_path, path):
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -656,7 +656,7 @@ class Stage(params.StageParams):
|
|||||||
|
|
||||||
def filter_outs(self, fs_path) -> Iterable["Output"]:
|
def filter_outs(self, fs_path) -> Iterable["Output"]:
|
||||||
def _func(o):
|
def _func(o):
|
||||||
return o.fs.path.isin_or_eq(fs_path, o.fs_path)
|
return o.fs.isin_or_eq(fs_path, o.fs_path)
|
||||||
|
|
||||||
return filter(_func, self.outs) if fs_path else self.outs
|
return filter(_func, self.outs) if fs_path else self.outs
|
||||||
|
|
||||||
|
|||||||
+9
-9
@@ -181,9 +181,9 @@ class StageCache:
|
|||||||
|
|
||||||
path = self._get_cache_path(cache_key, cache_value)
|
path = self._get_cache_path(cache_key, cache_value)
|
||||||
local_fs = self.repo.cache.legacy.fs
|
local_fs = self.repo.cache.legacy.fs
|
||||||
parent = local_fs.path.parent(path)
|
parent = local_fs.parent(path)
|
||||||
self.repo.cache.legacy.makedirs(parent)
|
self.repo.cache.legacy.makedirs(parent)
|
||||||
tmp = local_fs.path.join(parent, fs.utils.tmp_fname())
|
tmp = local_fs.join(parent, fs.utils.tmp_fname())
|
||||||
assert os.path.exists(parent)
|
assert os.path.exists(parent)
|
||||||
assert os.path.isdir(parent)
|
assert os.path.isdir(parent)
|
||||||
dump_yaml(tmp, cache)
|
dump_yaml(tmp, cache)
|
||||||
@@ -236,7 +236,7 @@ class StageCache:
|
|||||||
from_fs = from_odb.fs
|
from_fs = from_odb.fs
|
||||||
to_fs = to_odb.fs
|
to_fs = to_odb.fs
|
||||||
func = fs.generic.log_exceptions(fs.generic.copy)
|
func = fs.generic.log_exceptions(fs.generic.copy)
|
||||||
runs = from_fs.path.join(from_odb.path, "runs")
|
runs = from_fs.join(from_odb.path, "runs")
|
||||||
|
|
||||||
http_odb = next(
|
http_odb = next(
|
||||||
(odb for odb in (from_odb, to_odb) if isinstance(odb.fs, HTTPFileSystem)),
|
(odb for odb in (from_odb, to_odb) if isinstance(odb.fs, HTTPFileSystem)),
|
||||||
@@ -252,20 +252,20 @@ class StageCache:
|
|||||||
return ret
|
return ret
|
||||||
|
|
||||||
for src in from_fs.find(runs):
|
for src in from_fs.find(runs):
|
||||||
rel = from_fs.path.relpath(src, from_odb.path)
|
rel = from_fs.relpath(src, from_odb.path)
|
||||||
if not isinstance(to_fs, LocalFileSystem):
|
if not isinstance(to_fs, LocalFileSystem):
|
||||||
rel = from_fs.path.as_posix(rel)
|
rel = from_fs.as_posix(rel)
|
||||||
|
|
||||||
dst = to_fs.path.join(to_odb.path, rel)
|
dst = to_fs.join(to_odb.path, rel)
|
||||||
key = to_fs.path.parent(dst)
|
key = to_fs.parent(dst)
|
||||||
# check if any build cache already exists for this key
|
# check if any build cache already exists for this key
|
||||||
# TODO: check if MaxKeys=1 or something like that applies
|
# TODO: check if MaxKeys=1 or something like that applies
|
||||||
# or otherwise this will take a lot of time!
|
# or otherwise this will take a lot of time!
|
||||||
if to_fs.exists(key) and first(to_fs.find(key)):
|
if to_fs.exists(key) and first(to_fs.find(key)):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
src_name = from_fs.path.name(src)
|
src_name = from_fs.name(src)
|
||||||
parent_name = from_fs.path.name(from_fs.path.parent(src))
|
parent_name = from_fs.name(from_fs.parent(src))
|
||||||
with TqdmCallback(
|
with TqdmCallback(
|
||||||
desc=src_name,
|
desc=src_name,
|
||||||
bytes=True,
|
bytes=True,
|
||||||
|
|||||||
+2
-2
@@ -184,9 +184,9 @@ def resolve_wdir(wdir, path):
|
|||||||
|
|
||||||
|
|
||||||
def resolve_paths(fs, path, wdir=None):
|
def resolve_paths(fs, path, wdir=None):
|
||||||
path = fs.path.abspath(path)
|
path = fs.abspath(path)
|
||||||
wdir = wdir or os.curdir
|
wdir = wdir or os.curdir
|
||||||
wdir = fs.path.abspath(fs.path.join(fs.path.dirname(path), wdir))
|
wdir = fs.abspath(fs.join(fs.dirname(path), wdir))
|
||||||
return path, wdir
|
return path, wdir
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -163,8 +163,8 @@ class TestImportURLVersionAware:
|
|||||||
|
|
||||||
|
|
||||||
def match_files(fs, entries, expected):
|
def match_files(fs, entries, expected):
|
||||||
entries_content = {(fs.path.normpath(d["path"]), d["isdir"]) for d in entries}
|
entries_content = {(fs.normpath(d["path"]), d["isdir"]) for d in entries}
|
||||||
expected_content = {(fs.path.normpath(d["path"]), d["isdir"]) for d in expected}
|
expected_content = {(fs.normpath(d["path"]), d["isdir"]) for d in expected}
|
||||||
assert entries_content == expected_content
|
assert entries_content == expected_content
|
||||||
|
|
||||||
|
|
||||||
@@ -177,7 +177,7 @@ class TestLsUrl:
|
|||||||
match_files(
|
match_files(
|
||||||
fs,
|
fs,
|
||||||
result,
|
result,
|
||||||
[{"path": fs.path.join(fs_path, fname), "isdir": False}],
|
[{"path": fs.join(fs_path, fname), "isdir": False}],
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_dir(self, cloud):
|
def test_dir(self, cloud):
|
||||||
|
|||||||
@@ -287,7 +287,7 @@ def resolve_paths(repo, out, always_local=False):
|
|||||||
# urlparse interprets windows drive letters as URL scheme
|
# urlparse interprets windows drive letters as URL scheme
|
||||||
scheme = ""
|
scheme = ""
|
||||||
|
|
||||||
if scheme or not localfs.path.isin_or_eq(abspath, repo.root_dir):
|
if scheme or not localfs.isin_or_eq(abspath, repo.root_dir):
|
||||||
wdir = os.getcwd()
|
wdir = os.getcwd()
|
||||||
elif contains_symlink_up_to(dirname, repo.root_dir) or (
|
elif contains_symlink_up_to(dirname, repo.root_dir) or (
|
||||||
os.path.isdir(abspath) and localfs.is_symlink(abspath)
|
os.path.isdir(abspath) and localfs.is_symlink(abspath)
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ PARSERS.update(
|
|||||||
|
|
||||||
|
|
||||||
def load_path(fs_path, fs, **kwargs):
|
def load_path(fs_path, fs, **kwargs):
|
||||||
suffix = fs.path.suffix(fs_path).lower()
|
suffix = fs.suffix(fs_path).lower()
|
||||||
loader = LOADERS[suffix]
|
loader = LOADERS[suffix]
|
||||||
return loader(fs_path, fs=fs, **kwargs)
|
return loader(fs_path, fs=fs, **kwargs)
|
||||||
|
|
||||||
|
|||||||
+14
-14
@@ -35,7 +35,7 @@ dependencies = [
|
|||||||
"configobj>=5.0.6",
|
"configobj>=5.0.6",
|
||||||
"distro>=1.3",
|
"distro>=1.3",
|
||||||
"dpath<3,>=2.1.0",
|
"dpath<3,>=2.1.0",
|
||||||
"dvc-data>=2.25,<2.26",
|
"dvc-data>=3,<3.1",
|
||||||
"dvc-http>=2.29.0",
|
"dvc-http>=2.29.0",
|
||||||
"dvc-render>=1.0.0,<2",
|
"dvc-render>=1.0.0,<2",
|
||||||
"dvc-studio-client>=0.17.1,<1",
|
"dvc-studio-client>=0.17.1,<1",
|
||||||
@@ -45,7 +45,7 @@ dependencies = [
|
|||||||
"flufl.lock>=5,<8",
|
"flufl.lock>=5,<8",
|
||||||
"funcy>=1.14",
|
"funcy>=1.14",
|
||||||
"grandalf<1,>=0.7",
|
"grandalf<1,>=0.7",
|
||||||
"gto>=1.4.0,<2",
|
"gto>=1.6.0,<2",
|
||||||
"hydra-core>=1.1",
|
"hydra-core>=1.1",
|
||||||
"iterative-telemetry>=0.0.7",
|
"iterative-telemetry>=0.0.7",
|
||||||
"networkx>=2.5",
|
"networkx>=2.5",
|
||||||
@@ -59,7 +59,7 @@ dependencies = [
|
|||||||
"requests>=2.22",
|
"requests>=2.22",
|
||||||
"rich>=12",
|
"rich>=12",
|
||||||
"ruamel.yaml>=0.17.11",
|
"ruamel.yaml>=0.17.11",
|
||||||
"scmrepo>=1.5.0,<2",
|
"scmrepo>=2.0.2,<3",
|
||||||
"shortuuid>=0.5",
|
"shortuuid>=0.5",
|
||||||
"shtab<2,>=1.3.4",
|
"shtab<2,>=1.3.4",
|
||||||
"tabulate>=0.8.7",
|
"tabulate>=0.8.7",
|
||||||
@@ -71,11 +71,11 @@ dependencies = [
|
|||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"]
|
all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"]
|
||||||
azure = ["dvc-azure>=2.23.0"]
|
azure = ["dvc-azure>=3,<4"]
|
||||||
dev = ["dvc[azure,gdrive,gs,hdfs,lint,oss,s3,ssh,tests,webdav,webhdfs]"]
|
dev = ["dvc[azure,gdrive,gs,hdfs,lint,oss,s3,ssh,tests,webdav,webhdfs]"]
|
||||||
gdrive = ["dvc-gdrive==2.20"]
|
gdrive = ["dvc-gdrive>=3,<4"]
|
||||||
gs = ["dvc-gs==2.22.1"]
|
gs = ["dvc-gs>=3,<4"]
|
||||||
hdfs = ["dvc-hdfs==2.19"]
|
hdfs = ["dvc-hdfs>=3,<4"]
|
||||||
lint = [
|
lint = [
|
||||||
"mypy==1.7.1",
|
"mypy==1.7.1",
|
||||||
"pandas-stubs",
|
"pandas-stubs",
|
||||||
@@ -87,10 +87,10 @@ lint = [
|
|||||||
"types-toml",
|
"types-toml",
|
||||||
"types-tqdm",
|
"types-tqdm",
|
||||||
]
|
]
|
||||||
oss = ["dvc-oss>=2.22.0"]
|
oss = ["dvc-oss>=3,<4"]
|
||||||
s3 = ["dvc-s3==2.23.0"]
|
s3 = ["dvc-s3>=3,<4"]
|
||||||
ssh = ["dvc-ssh>=2.22.1,<3"]
|
ssh = ["dvc-ssh>=3,<4"]
|
||||||
ssh_gssapi = ["dvc-ssh[gssapi]>=2.22.1,<3"]
|
ssh_gssapi = ["dvc-ssh[gssapi]>=3,<4"]
|
||||||
testing = [
|
testing = [
|
||||||
"pytest-benchmark[histogram]",
|
"pytest-benchmark[histogram]",
|
||||||
"pytest-test-utils",
|
"pytest-test-utils",
|
||||||
@@ -116,9 +116,9 @@ tests = [
|
|||||||
"sqlalchemy>=1,<3", # optional dependency for `import-db`
|
"sqlalchemy>=1,<3", # optional dependency for `import-db`
|
||||||
"pandas>=1",
|
"pandas>=1",
|
||||||
]
|
]
|
||||||
webdav = ["dvc-webdav==2.19.1"]
|
webdav = ["dvc-webdav>=3,<4"]
|
||||||
webhdfs = ["dvc-webhdfs==2.19"]
|
webhdfs = ["dvc-webhdfs>=3,<4"]
|
||||||
webhdfs_kerberos = ["dvc-webhdfs[kerberos]==2.19"]
|
webhdfs_kerberos = ["dvc-webhdfs[kerberos]>=3,<4"]
|
||||||
|
|
||||||
[project.urls]
|
[project.urls]
|
||||||
Documentation = "https://dvc.org/doc"
|
Documentation = "https://dvc.org/doc"
|
||||||
|
|||||||
@@ -13,15 +13,15 @@ def test_cleanfs_subrepo(tmp_dir, dvc, scm, monkeypatch):
|
|||||||
|
|
||||||
path = subrepo_dir.fs_path
|
path = subrepo_dir.fs_path
|
||||||
|
|
||||||
assert dvc.fs.exists(dvc.fs.path.join(path, "foo"))
|
assert dvc.fs.exists(dvc.fs.join(path, "foo"))
|
||||||
assert dvc.fs.isfile(dvc.fs.path.join(path, "foo"))
|
assert dvc.fs.isfile(dvc.fs.join(path, "foo"))
|
||||||
assert dvc.fs.exists(dvc.fs.path.join(path, "dir"))
|
assert dvc.fs.exists(dvc.fs.join(path, "dir"))
|
||||||
assert dvc.fs.isdir(dvc.fs.path.join(path, "dir"))
|
assert dvc.fs.isdir(dvc.fs.join(path, "dir"))
|
||||||
|
|
||||||
assert subrepo.fs.exists(subrepo.fs.path.join(path, "foo"))
|
assert subrepo.fs.exists(subrepo.fs.join(path, "foo"))
|
||||||
assert subrepo.fs.isfile(subrepo.fs.path.join(path, "foo"))
|
assert subrepo.fs.isfile(subrepo.fs.join(path, "foo"))
|
||||||
assert subrepo.fs.exists(subrepo.fs.path.join(path, "dir"))
|
assert subrepo.fs.exists(subrepo.fs.join(path, "dir"))
|
||||||
assert subrepo.fs.isdir(subrepo.fs.path.join(path, "dir"))
|
assert subrepo.fs.isdir(subrepo.fs.join(path, "dir"))
|
||||||
|
|
||||||
|
|
||||||
def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
|
def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
|
||||||
|
|||||||
@@ -1,29 +0,0 @@
|
|||||||
import pytest
|
|
||||||
|
|
||||||
from dvc.fs import Path
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("prefix", ["", "/"])
|
|
||||||
@pytest.mark.parametrize("postfix", ["", "/"])
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"path,expected",
|
|
||||||
[
|
|
||||||
("path", ("path",)),
|
|
||||||
("some/path", ("some", "path")),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_parts_posix(prefix, postfix, path, expected):
|
|
||||||
assert Path("/").parts(prefix + path + postfix) == tuple(prefix) + expected
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("postfix", ["", "\\"])
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"path,expected",
|
|
||||||
[
|
|
||||||
("path", ("path",)),
|
|
||||||
("c:\\path", ("c:", "\\", "path")),
|
|
||||||
("some\\path", ("some", "path")),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_parts_nt(postfix, path, expected):
|
|
||||||
assert Path("\\").parts(path + postfix) == expected
|
|
||||||
@@ -173,11 +173,10 @@ def test_match_ignore_from_file(
|
|||||||
)
|
)
|
||||||
dvcignore_dirname = os.path.dirname(dvcignore_path)
|
dvcignore_dirname = os.path.dirname(dvcignore_path)
|
||||||
|
|
||||||
fs = mocker.MagicMock()
|
mocker.patch.object(
|
||||||
fs.path = localfs.path
|
localfs, "open", mocker.mock_open(read_data="\n".join(patterns))
|
||||||
fs.sep = localfs.sep
|
)
|
||||||
mocker.patch.object(fs, "open", mocker.mock_open(read_data="\n".join(patterns)))
|
ignore_file = DvcIgnorePatterns.from_file(dvcignore_path, localfs, "mocked")
|
||||||
ignore_file = DvcIgnorePatterns.from_file(dvcignore_path, fs, "mocked")
|
|
||||||
|
|
||||||
assert (
|
assert (
|
||||||
ignore_file.matches(dvcignore_dirname, file_to_ignore_relpath) == expected_match
|
ignore_file.matches(dvcignore_dirname, file_to_ignore_relpath) == expected_match
|
||||||
|
|||||||
Reference in New Issue
Block a user