mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
dvc: get rid of path_info
Intermediate PR in removing path_info from our code base. It is completely removed from the main codebase and is still being used in tests, which will be migrated in the followups. For fs-specific simple path manipulation, we've introduced `fs.path` helper (similar to `os.path`). Fixes #6774
This commit is contained in:
committed by
Ruslan Kuprieiev
parent
4175f9ffa4
commit
9aec8f1926
+3
-4
@@ -4,7 +4,6 @@ from contextlib import _GeneratorContextManager as GCM
|
||||
from funcy import reraise
|
||||
|
||||
from dvc.exceptions import OutputNotFoundError, PathMissingError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
|
||||
|
||||
@@ -20,15 +19,15 @@ def get_url(path, repo=None, rev=None, remote=None):
|
||||
directory in the remote storage.
|
||||
"""
|
||||
with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
|
||||
path_info = PathInfo(_repo.root_dir) / path
|
||||
fs_path = _repo.fs.path.join(_repo.root_dir, path)
|
||||
with reraise(FileNotFoundError, PathMissingError(path, repo)):
|
||||
metadata = _repo.repo_fs.metadata(path_info)
|
||||
metadata = _repo.repo_fs.metadata(fs_path)
|
||||
|
||||
if not metadata.is_dvc:
|
||||
raise OutputNotFoundError(path, repo)
|
||||
|
||||
cloud = metadata.repo.cloud
|
||||
md5 = metadata.repo.dvcfs.info(path_info)["md5"]
|
||||
md5 = metadata.repo.dvcfs.info(fs_path)["md5"]
|
||||
return cloud.get_url_for(remote, checksum=md5)
|
||||
|
||||
|
||||
|
||||
+5
-3
@@ -38,12 +38,14 @@ def _collect_targets(repo, target, outs):
|
||||
targets = []
|
||||
|
||||
outs_trie = repo.index.outs_trie
|
||||
for stage, info in pairs:
|
||||
if not info:
|
||||
for stage, path in pairs:
|
||||
if not path:
|
||||
targets.extend([str(out) for out in stage.outs])
|
||||
continue
|
||||
|
||||
for out in outs_trie.itervalues(prefix=info.parts): # noqa: B301
|
||||
for out in outs_trie.itervalues(
|
||||
prefix=repo.fs.path.parts(path)
|
||||
): # noqa: B301
|
||||
targets.extend(str(out))
|
||||
|
||||
return targets
|
||||
|
||||
+3
-2
@@ -8,7 +8,6 @@ from functools import partial
|
||||
from funcy import cached_property, compact, memoize, re_find
|
||||
|
||||
from dvc.exceptions import DvcException, NotDvcRepoError
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -229,6 +228,7 @@ class Config(dict):
|
||||
|
||||
@staticmethod
|
||||
def _to_relpath(conf_dir, path):
|
||||
from dvc.fs.local import localfs
|
||||
from dvc.utils import relpath
|
||||
|
||||
from .config_schema import RelPath
|
||||
@@ -238,8 +238,9 @@ class Config(dict):
|
||||
|
||||
if isinstance(path, RelPath) or not os.path.isabs(path):
|
||||
path = relpath(path, conf_dir)
|
||||
return localfs.path.as_posix(path)
|
||||
|
||||
return PathInfo(path).as_posix()
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _save_paths(conf, filename):
|
||||
|
||||
+5
-4
@@ -58,9 +58,9 @@ class DataCloud:
|
||||
from dvc.fs import get_cloud_fs
|
||||
from dvc.objects.db import get_odb
|
||||
|
||||
cls, config, path_info = get_cloud_fs(self.repo, name=name)
|
||||
cls, config, fs_path = get_cloud_fs(self.repo, name=name)
|
||||
config["tmp_dir"] = self.repo.index_db_dir
|
||||
return get_odb(cls(**config), path_info, **config)
|
||||
return get_odb(cls(**config), fs_path, **config)
|
||||
|
||||
def push(
|
||||
self,
|
||||
@@ -156,5 +156,6 @@ class DataCloud:
|
||||
)
|
||||
|
||||
def get_url_for(self, remote, checksum):
|
||||
remote_odb = self.get_remote_odb(remote)
|
||||
return str(remote_odb.hash_to_path_info(checksum))
|
||||
odb = self.get_remote_odb(remote)
|
||||
path = odb.hash_to_path(checksum)
|
||||
return odb.fs.unstrip_protocol(path)
|
||||
|
||||
@@ -92,10 +92,10 @@ class ParamsDependency(Dependency):
|
||||
if not self.exists:
|
||||
return {}
|
||||
|
||||
suffix = self.path_info.suffix.lower()
|
||||
suffix = self.repo.fs.path.suffix(self.fs_path).lower()
|
||||
loader = LOADERS[suffix]
|
||||
try:
|
||||
return loader(self.path_info, fs=self.repo.fs)
|
||||
return loader(self.fs_path, fs=self.repo.fs)
|
||||
except ParseError as exc:
|
||||
raise BadParamFileError(
|
||||
f"Unable to read parameters from '{self}'"
|
||||
|
||||
@@ -5,8 +5,6 @@ from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple
|
||||
|
||||
from voluptuous import Required
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
from .base import Dependency
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -34,7 +32,7 @@ class RepoDependency(Dependency):
|
||||
self._staged_objs: Dict[str, "HashFile"] = {}
|
||||
super().__init__(stage, *args, **kwargs)
|
||||
|
||||
def _parse_path(self, fs, path_info):
|
||||
def _parse_path(self, fs, fs_path):
|
||||
return None
|
||||
|
||||
@property
|
||||
@@ -70,7 +68,7 @@ class RepoDependency(Dependency):
|
||||
|
||||
obj = self.get_obj()
|
||||
checkout(
|
||||
to.path_info,
|
||||
to.fs_path,
|
||||
to.fs,
|
||||
obj,
|
||||
self.repo.odb.local,
|
||||
@@ -114,11 +112,11 @@ class RepoDependency(Dependency):
|
||||
if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
|
||||
self.def_repo[self.PARAM_REV_LOCK] = rev
|
||||
|
||||
path_info = PathInfo(repo.root_dir) / str(self.def_path)
|
||||
path = os.path.abspath(os.path.join(repo.root_dir, self.def_path))
|
||||
if not obj_only:
|
||||
try:
|
||||
for odb, obj_ids in repo.used_objs(
|
||||
[os.fspath(path_info)],
|
||||
[path],
|
||||
force=True,
|
||||
jobs=kwargs.get("jobs"),
|
||||
recursive=True,
|
||||
@@ -134,7 +132,7 @@ class RepoDependency(Dependency):
|
||||
try:
|
||||
staging, _, staged_obj = stage(
|
||||
local_odb,
|
||||
path_info,
|
||||
path,
|
||||
repo.repo_fs,
|
||||
local_odb.fs.PARAM_CHECKSUM,
|
||||
)
|
||||
|
||||
+6
-6
@@ -305,7 +305,7 @@ class PathMissingError(DvcException):
|
||||
|
||||
|
||||
class RemoteCacheRequiredError(DvcException):
|
||||
def __init__(self, path_info):
|
||||
def __init__(self, scheme, fs_path):
|
||||
from dvc.utils import format_link
|
||||
|
||||
super().__init__(
|
||||
@@ -314,8 +314,8 @@ class RemoteCacheRequiredError(DvcException):
|
||||
"existing cache on '{}' remote. See {} for information on how "
|
||||
"to set up remote cache."
|
||||
).format(
|
||||
path_info,
|
||||
path_info.scheme,
|
||||
fs_path,
|
||||
scheme,
|
||||
format_link("https://man.dvc.org/config#cache"),
|
||||
)
|
||||
)
|
||||
@@ -350,12 +350,12 @@ class CacheLinkError(DvcException):
|
||||
)
|
||||
)
|
||||
|
||||
def __init__(self, path_infos):
|
||||
def __init__(self, fs_paths):
|
||||
msg = "No possible cache link types for '{}'. {}".format(
|
||||
", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK
|
||||
", ".join(fs_paths), self.SUPPORT_LINK
|
||||
)
|
||||
super().__init__(msg)
|
||||
self.path_infos = path_infos
|
||||
self.fs_paths = fs_paths
|
||||
|
||||
|
||||
class CircularImportError(DvcException):
|
||||
|
||||
+10
-11
@@ -1,4 +1,3 @@
|
||||
import posixpath
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ..scheme import Schemes
|
||||
@@ -39,7 +38,7 @@ def get_fs_cls(remote_conf, scheme=None):
|
||||
return FS_MAP.get(scheme, LocalFileSystem)
|
||||
|
||||
|
||||
def get_fs_config(config, **kwargs):
|
||||
def get_fs_config(repo, config, **kwargs):
|
||||
name = kwargs.get("name")
|
||||
if name:
|
||||
try:
|
||||
@@ -50,10 +49,10 @@ def get_fs_config(config, **kwargs):
|
||||
raise RemoteNotFoundError(f"remote '{name}' doesn't exist")
|
||||
else:
|
||||
remote_conf = kwargs
|
||||
return _resolve_remote_refs(config, remote_conf)
|
||||
return _resolve_remote_refs(repo, config, remote_conf)
|
||||
|
||||
|
||||
def _resolve_remote_refs(config, remote_conf):
|
||||
def _resolve_remote_refs(repo, config, remote_conf):
|
||||
# Support for cross referenced remotes.
|
||||
# This will merge the settings, shadowing base ref with remote_conf.
|
||||
# For example, having:
|
||||
@@ -79,8 +78,10 @@ def _resolve_remote_refs(config, remote_conf):
|
||||
if parsed.scheme != "remote":
|
||||
return remote_conf
|
||||
|
||||
base = get_fs_config(config, name=parsed.netloc)
|
||||
url = posixpath.join(base["url"], parsed.path.lstrip("/"))
|
||||
base = get_fs_config(repo, config, name=parsed.netloc)
|
||||
cls, _, _ = get_cloud_fs(repo, **base)
|
||||
relpath = parsed.path.lstrip("/").replace("/", cls.sep)
|
||||
url = cls.sep.join((base["url"], relpath))
|
||||
return {**base, **remote_conf, "url": url}
|
||||
|
||||
|
||||
@@ -91,7 +92,7 @@ def get_cloud_fs(repo, **kwargs):
|
||||
repo_config = repo.config if repo else {}
|
||||
core_config = repo_config.get("core", {})
|
||||
|
||||
remote_conf = get_fs_config(repo_config, **kwargs)
|
||||
remote_conf = get_fs_config(repo, repo_config, **kwargs)
|
||||
try:
|
||||
remote_conf = SCHEMA["remote"][str](remote_conf)
|
||||
except Invalid as exc:
|
||||
@@ -113,10 +114,8 @@ def get_cloud_fs(repo, **kwargs):
|
||||
remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir
|
||||
|
||||
url = remote_conf.pop("url")
|
||||
path_info = cls.PATH_CLS(
|
||||
cls._strip_protocol(url) # pylint:disable=protected-access
|
||||
)
|
||||
fs_path = cls._strip_protocol(url) # pylint:disable=protected-access
|
||||
|
||||
extras = cls._get_kwargs_from_urls(url) # pylint:disable=protected-access
|
||||
conf = {**extras, **remote_conf} # remote config takes priority
|
||||
return cls, conf, path_info
|
||||
return cls, conf, fs_path
|
||||
|
||||
+6
-5
@@ -2,7 +2,6 @@ from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, List
|
||||
|
||||
from dvc.output import Output
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.repo import Repo
|
||||
@@ -16,7 +15,7 @@ class Metadata:
|
||||
"""
|
||||
|
||||
# required field
|
||||
path_info: PathInfo
|
||||
fs_path: str
|
||||
repo: "Repo"
|
||||
|
||||
# computed fields
|
||||
@@ -47,11 +46,13 @@ class Metadata:
|
||||
else:
|
||||
out = self.outs[0]
|
||||
# or, the path itself could be an output
|
||||
self.is_output = self.path_info == out.path_info
|
||||
self.is_output = self.fs_path == out.fs_path
|
||||
|
||||
# or, a directory must have an output somewhere deep inside it
|
||||
if not self.is_output:
|
||||
self.contains_outputs = out.path_info.isin(self.path_info)
|
||||
self.contains_outputs = self.repo.fs.path.isin(
|
||||
out.fs_path, self.fs_path
|
||||
)
|
||||
|
||||
# or, the path could be a part of an output, i.e. inside of an output
|
||||
self.part_of_output = not self.is_output and not self.contains_outputs
|
||||
@@ -69,4 +70,4 @@ class Metadata:
|
||||
return not self.isdir
|
||||
|
||||
def __str__(self):
|
||||
return f"Metadata: {self.path_info}"
|
||||
return f"Metadata: {self.fs_path}"
|
||||
|
||||
+7
-7
@@ -7,7 +7,6 @@ from fsspec.utils import infer_storage_options
|
||||
from funcy import cached_property, memoize, wrap_prop
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import format_link
|
||||
|
||||
@@ -41,7 +40,6 @@ def _az_config():
|
||||
# pylint:disable=abstract-method
|
||||
class AzureFileSystem(CallbackMixin, ObjectFSWrapper):
|
||||
scheme = Schemes.AZURE
|
||||
PATH_CLS = CloudURLInfo
|
||||
PARAM_CHECKSUM = "etag"
|
||||
DETAIL_FIELDS = frozenset(("etag", "size"))
|
||||
REQUIRES = {
|
||||
@@ -52,12 +50,14 @@ class AzureFileSystem(CallbackMixin, ObjectFSWrapper):
|
||||
|
||||
@classmethod
|
||||
def _strip_protocol(cls, path: str):
|
||||
bucket = infer_storage_options(path).get("host")
|
||||
if bucket:
|
||||
return path
|
||||
opts = infer_storage_options(path)
|
||||
if opts.get("host"):
|
||||
return "{host}{path}".format(**opts)
|
||||
|
||||
bucket = _az_config().get("storage", "container_name", None)
|
||||
return f"azure://{bucket}"
|
||||
return _az_config().get("storage", "container_name", None)
|
||||
|
||||
def unstrip_protocol(self, path: str) -> str:
|
||||
return "azure://" + path.lstrip("/")
|
||||
|
||||
@staticmethod
|
||||
def _get_kwargs_from_urls(urlpath):
|
||||
|
||||
+41
-36
@@ -4,12 +4,12 @@ import os
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from functools import partialmethod
|
||||
from multiprocessing import cpu_count
|
||||
from typing import Any, ClassVar, Dict, FrozenSet, Optional
|
||||
from typing import ClassVar, Dict, FrozenSet, Optional
|
||||
|
||||
from funcy import cached_property
|
||||
from tqdm.utils import CallbackIOWrapper
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.progress import DEFAULT_CALLBACK, FsspecCallback
|
||||
from dvc.ui import ui
|
||||
from dvc.utils import tmp_fname
|
||||
@@ -41,7 +41,6 @@ class BaseFileSystem:
|
||||
|
||||
scheme = "base"
|
||||
REQUIRES: ClassVar[Dict[str, str]] = {}
|
||||
PATH_CLS = URLInfo # type: Any
|
||||
_JOBS = 4 * cpu_count()
|
||||
|
||||
CHECKSUM_DIR_SUFFIX = ".dir"
|
||||
@@ -69,10 +68,19 @@ class BaseFileSystem:
|
||||
def config(self):
|
||||
return self._config
|
||||
|
||||
@cached_property
|
||||
def path(self):
|
||||
from .path import Path
|
||||
|
||||
return Path(self.sep)
|
||||
|
||||
@classmethod
|
||||
def _strip_protocol(cls, path: str):
|
||||
return path
|
||||
|
||||
def unstrip_protocol(self, path):
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _get_kwargs_from_urls(urlpath): # pylint:disable=unused-argument
|
||||
return {}
|
||||
@@ -129,36 +137,36 @@ class BaseFileSystem:
|
||||
f"dependencies: {missing}. {hint}"
|
||||
)
|
||||
|
||||
def checksum(self, path_info) -> str:
|
||||
def checksum(self, path) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
|
||||
def open(self, path, mode: str = "r", encoding: str = None, **kwargs):
|
||||
raise RemoteActionNotImplemented("open", self.scheme)
|
||||
|
||||
def exists(self, path_info) -> bool:
|
||||
def exists(self, path) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
def isdir(self, path_info):
|
||||
def isdir(self, path):
|
||||
"""Optional: Overwrite only if the remote has a way to distinguish
|
||||
between a directory and a file.
|
||||
"""
|
||||
return False
|
||||
|
||||
def isfile(self, path_info):
|
||||
def isfile(self, path):
|
||||
"""Optional: Overwrite only if the remote has a way to distinguish
|
||||
between a directory and a file.
|
||||
"""
|
||||
return True
|
||||
|
||||
def isexec(self, path_info):
|
||||
def isexec(self, path):
|
||||
"""Optional: Overwrite only if the remote has a way to distinguish
|
||||
between executable and non-executable file.
|
||||
"""
|
||||
return False
|
||||
|
||||
def iscopy(self, path_info):
|
||||
def iscopy(self, path):
|
||||
"""Check if this file is an independent copy."""
|
||||
return False # We can't be sure by default
|
||||
|
||||
@@ -166,28 +174,28 @@ class BaseFileSystem:
|
||||
"""Return a generator with (root, dirs, files)."""
|
||||
raise NotImplementedError
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
"""Return a generator with `PathInfo`s to all the files.
|
||||
def find(self, path, prefix=None):
|
||||
"""Return a generator with `str`s to all the files.
|
||||
|
||||
Optional kwargs:
|
||||
prefix (bool): If true `path_info` will be treated as a prefix
|
||||
prefix (bool): If true `path` will be treated as a prefix
|
||||
rather than directory path.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def is_empty(self, path_info):
|
||||
def is_empty(self, path):
|
||||
return False
|
||||
|
||||
def info(self, path_info):
|
||||
def info(self, path):
|
||||
raise NotImplementedError
|
||||
|
||||
def getsize(self, path_info):
|
||||
return self.info(path_info).get("size")
|
||||
def getsize(self, path):
|
||||
return self.info(path).get("size")
|
||||
|
||||
def remove(self, path_info):
|
||||
def remove(self, path):
|
||||
raise RemoteActionNotImplemented("remove", self.scheme)
|
||||
|
||||
def makedirs(self, path_info, **kwargs):
|
||||
def makedirs(self, path, **kwargs):
|
||||
"""Optional: Implement only if the remote needs to create
|
||||
directories before copying/linking/moving data
|
||||
"""
|
||||
@@ -231,11 +239,10 @@ class BaseFileSystem:
|
||||
if not hasattr(self, method):
|
||||
raise RemoteActionNotImplemented(method, self.scheme)
|
||||
|
||||
if to_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
if not is_file_obj:
|
||||
desc = desc or from_info.name
|
||||
from .local import localfs
|
||||
|
||||
desc = desc or localfs.path.name(from_info)
|
||||
|
||||
stack = contextlib.ExitStack()
|
||||
if not callback:
|
||||
@@ -261,9 +268,6 @@ class BaseFileSystem:
|
||||
# pylint: disable=no-member
|
||||
return self.upload_fobj(wrapped, to_info, size=total)
|
||||
|
||||
if from_info.scheme != "local":
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug("Uploading '%s' to '%s'", from_info, to_info)
|
||||
# pylint: disable=no-member
|
||||
return self.put_file(
|
||||
@@ -281,18 +285,14 @@ class BaseFileSystem:
|
||||
_only_file=False,
|
||||
**kwargs,
|
||||
):
|
||||
from .local import localfs
|
||||
|
||||
if not hasattr(self, "get_file"):
|
||||
raise RemoteActionNotImplemented("get_file", self.scheme)
|
||||
|
||||
if from_info.scheme != self.scheme:
|
||||
raise NotImplementedError
|
||||
|
||||
if to_info.scheme != "local":
|
||||
raise NotImplementedError
|
||||
|
||||
download_dir = not _only_file and self.isdir(from_info)
|
||||
|
||||
desc = name or to_info.name
|
||||
desc = name or localfs.path.name(to_info)
|
||||
stack = contextlib.ExitStack()
|
||||
if not callback:
|
||||
pbar_kwargs = {"unit": "Files"} if download_dir else {}
|
||||
@@ -323,13 +323,16 @@ class BaseFileSystem:
|
||||
jobs=None,
|
||||
**kwargs,
|
||||
):
|
||||
from_infos = list(self.walk_files(from_info, **kwargs))
|
||||
from .local import localfs
|
||||
|
||||
from_infos = list(self.find(from_info, **kwargs))
|
||||
if not from_infos:
|
||||
makedirs(to_info, exist_ok=True)
|
||||
return None
|
||||
|
||||
to_infos = (
|
||||
to_info / info.relative_to(from_info) for info in from_infos
|
||||
localfs.path.join(to_info, self.path.relpath(info, from_info))
|
||||
for info in from_infos
|
||||
)
|
||||
callback.set_size(len(from_infos))
|
||||
|
||||
@@ -362,7 +365,9 @@ class BaseFileSystem:
|
||||
to_info,
|
||||
callback=DEFAULT_CALLBACK,
|
||||
):
|
||||
makedirs(to_info.parent, exist_ok=True)
|
||||
from .local import localfs
|
||||
|
||||
makedirs(localfs.path.parent(to_info), exist_ok=True)
|
||||
tmp_file = tmp_fname(to_info)
|
||||
|
||||
logger.debug("Downloading '%s' to '%s'", from_info, to_info)
|
||||
|
||||
+40
-41
@@ -3,7 +3,6 @@ import os
|
||||
import typing
|
||||
|
||||
from dvc.exceptions import OutputNotFoundError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.utils import relpath
|
||||
|
||||
from ..progress import DEFAULT_CALLBACK
|
||||
@@ -12,7 +11,7 @@ from .base import BaseFileSystem
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from dvc.output import Output
|
||||
|
||||
from dvc.types import AnyPath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -49,12 +48,9 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
|
||||
return outs
|
||||
|
||||
def _get_granular_hash(
|
||||
self, path_info: PathInfo, out: "Output", remote=None
|
||||
):
|
||||
assert isinstance(path_info, PathInfo)
|
||||
def _get_granular_hash(self, path: "AnyPath", out: "Output", remote=None):
|
||||
# NOTE: use string paths here for performance reasons
|
||||
key = tuple(relpath(path_info, out.path_info).split(os.sep))
|
||||
key = tuple(relpath(path, out.fs_path).split(os.sep))
|
||||
out.get_dir_cache(remote=remote)
|
||||
if out.obj is None:
|
||||
raise FileNotFoundError
|
||||
@@ -63,14 +59,14 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
return oid
|
||||
raise FileNotFoundError
|
||||
|
||||
def _get_fs_path(self, path: PathInfo, remote=None):
|
||||
def _get_fs_path(self, path: "AnyPath", remote=None):
|
||||
try:
|
||||
outs = self._find_outs(path, strict=False)
|
||||
except OutputNotFoundError as exc:
|
||||
raise FileNotFoundError from exc
|
||||
|
||||
if len(outs) != 1 or (
|
||||
outs[0].is_dir_checksum and path == outs[0].path_info
|
||||
outs[0].is_dir_checksum and path == outs[0].fs_path
|
||||
):
|
||||
raise IsADirectoryError
|
||||
|
||||
@@ -90,18 +86,20 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
checksum = self._get_granular_hash(path, out).value
|
||||
else:
|
||||
checksum = out.hash_info.value
|
||||
remote_info = remote_odb.hash_to_path_info(checksum)
|
||||
return remote_odb.fs, remote_info
|
||||
remote_fs_path = remote_odb.hash_to_path(checksum)
|
||||
return remote_odb.fs, remote_fs_path
|
||||
|
||||
if out.is_dir_checksum:
|
||||
checksum = self._get_granular_hash(path, out).value
|
||||
cache_path = out.odb.hash_to_path_info(checksum).url
|
||||
cache_path = out.odb.fs.unstrip_protocol(
|
||||
out.odb.hash_to_path(checksum)
|
||||
)
|
||||
else:
|
||||
cache_path = out.cache_path
|
||||
return out.odb.fs, cache_path
|
||||
|
||||
def open( # type: ignore
|
||||
self, path: PathInfo, mode="r", encoding=None, **kwargs
|
||||
self, path: str, mode="r", encoding=None, **kwargs
|
||||
): # pylint: disable=arguments-renamed
|
||||
fs, fspath = self._get_fs_path(path, **kwargs)
|
||||
return fs.open(fspath, mode=mode, encoding=encoding)
|
||||
@@ -120,18 +118,18 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
def check_isdir(self, path_info, outs):
|
||||
def check_isdir(self, path, outs):
|
||||
if len(outs) != 1:
|
||||
return True
|
||||
|
||||
out = outs[0]
|
||||
if not out.is_dir_checksum:
|
||||
return out.path_info != path_info
|
||||
if out.path_info == path_info:
|
||||
return out.fs_path != path
|
||||
if out.fs_path == path:
|
||||
return True
|
||||
|
||||
try:
|
||||
self._get_granular_hash(path_info, out)
|
||||
self._get_granular_hash(path, out)
|
||||
return False
|
||||
except FileNotFoundError:
|
||||
return True
|
||||
@@ -153,7 +151,7 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
def _add_dir(self, trie, out, **kwargs):
|
||||
self._fetch_dir(out, **kwargs)
|
||||
|
||||
base = out.path_info.parts
|
||||
base = out.fs.path.parts(out.fs_path)
|
||||
for key, _, _ in out.obj: # noqa: B301
|
||||
trie[base + key] = None
|
||||
|
||||
@@ -161,14 +159,15 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
dirs = set()
|
||||
files = []
|
||||
|
||||
out = trie.get(root.parts)
|
||||
root_parts = self.path.parts(root)
|
||||
out = trie.get(root_parts)
|
||||
if out and out.is_dir_checksum:
|
||||
self._add_dir(trie, out, **kwargs)
|
||||
|
||||
root_len = len(root.parts)
|
||||
root_len = len(root_parts)
|
||||
try:
|
||||
for key, out in trie.iteritems(prefix=root.parts): # noqa: B301
|
||||
if key == root.parts:
|
||||
for key, out in trie.iteritems(prefix=root_parts): # noqa: B301
|
||||
if key == root_parts:
|
||||
continue
|
||||
|
||||
name = key[root_len]
|
||||
@@ -182,16 +181,16 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
|
||||
assert topdown
|
||||
dirs = list(dirs)
|
||||
yield root.fspath, dirs, files
|
||||
yield root, dirs, files
|
||||
|
||||
for dname in dirs:
|
||||
yield from self._walk(root / dname, trie)
|
||||
yield from self._walk(self.path.join(root, dname), trie)
|
||||
|
||||
def walk(self, top, topdown=True, onerror=None, **kwargs):
|
||||
from pygtrie import Trie
|
||||
|
||||
assert topdown
|
||||
root = PathInfo(os.path.abspath(top))
|
||||
root = os.path.abspath(top)
|
||||
try:
|
||||
meta = self.metadata(root)
|
||||
except FileNotFoundError:
|
||||
@@ -206,18 +205,18 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
|
||||
trie = Trie()
|
||||
for out in meta.outs:
|
||||
trie[out.path_info.parts] = out
|
||||
trie[out.fs.path.parts(out.fs_path)] = out
|
||||
|
||||
if out.is_dir_checksum and root.isin_or_eq(out.path_info):
|
||||
if out.is_dir_checksum and self.path.isin_or_eq(root, out.fs_path):
|
||||
self._add_dir(trie, out, **kwargs)
|
||||
|
||||
yield from self._walk(root, trie, topdown=topdown, **kwargs)
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
for root, _, files in self.walk(path_info):
|
||||
def find(self, path, prefix=None):
|
||||
for root, _, files in self.walk(path):
|
||||
for fname in files:
|
||||
# NOTE: os.path.join is ~5.5 times slower
|
||||
yield PathInfo(f"{root}{os.sep}{fname}")
|
||||
yield f"{root}{os.sep}{fname}"
|
||||
|
||||
def isdvc(self, path, recursive=False, strict=True):
|
||||
try:
|
||||
@@ -228,23 +227,23 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
recurse = recursive or not strict
|
||||
return meta.output_exists if recurse else meta.is_output
|
||||
|
||||
def isexec(self, path_info): # pylint: disable=unused-argument
|
||||
def isexec(self, path): # pylint: disable=unused-argument
|
||||
return False
|
||||
|
||||
def metadata(self, path_info):
|
||||
path_info = PathInfo(os.path.abspath(path_info))
|
||||
def metadata(self, fs_path):
|
||||
abspath = os.path.abspath(fs_path)
|
||||
|
||||
try:
|
||||
outs = self._find_outs(path_info, strict=False, recursive=True)
|
||||
outs = self._find_outs(abspath, strict=False, recursive=True)
|
||||
except OutputNotFoundError as exc:
|
||||
raise FileNotFoundError from exc
|
||||
|
||||
meta = Metadata(path_info=path_info, outs=outs, repo=self.repo)
|
||||
meta.isdir = meta.isdir or self.check_isdir(meta.path_info, meta.outs)
|
||||
meta = Metadata(fs_path=abspath, outs=outs, repo=self.repo)
|
||||
meta.isdir = meta.isdir or self.check_isdir(meta.fs_path, meta.outs)
|
||||
return meta
|
||||
|
||||
def info(self, path_info):
|
||||
meta = self.metadata(path_info)
|
||||
def info(self, path):
|
||||
meta = self.metadata(path)
|
||||
ret = {"type": "directory" if meta.isdir else "file"}
|
||||
if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info:
|
||||
out = meta.outs[0]
|
||||
@@ -252,7 +251,7 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
ret[out.hash_info.name] = out.hash_info.value
|
||||
elif meta.part_of_output:
|
||||
(out,) = meta.outs
|
||||
key = path_info.relative_to(out.path_info).parts
|
||||
key = self.path.parts(self.path.relpath(path, out.fs_path))
|
||||
(obj_meta, oid) = out.obj.trie.get(key) or (None, None)
|
||||
if oid:
|
||||
ret["size"] = obj_meta.size if obj_meta else 0
|
||||
@@ -268,8 +267,8 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
path, to_file, callback=callback, **kwargs
|
||||
)
|
||||
|
||||
def checksum(self, path_info):
|
||||
info = self.info(path_info)
|
||||
def checksum(self, path):
|
||||
info = self.info(path)
|
||||
md5 = info.get("md5")
|
||||
if md5:
|
||||
return md5
|
||||
|
||||
+56
-165
@@ -1,31 +1,18 @@
|
||||
import os
|
||||
import shutil
|
||||
from functools import lru_cache
|
||||
from typing import (
|
||||
IO,
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Iterator,
|
||||
Optional,
|
||||
Union,
|
||||
overload,
|
||||
)
|
||||
from typing import IO, TYPE_CHECKING, Any, Dict, Iterator, Optional, overload
|
||||
|
||||
from funcy import cached_property
|
||||
from tqdm.utils import CallbackIOWrapper
|
||||
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.progress import DEFAULT_CALLBACK
|
||||
|
||||
from .base import BaseFileSystem
|
||||
|
||||
FSPath = URLInfo
|
||||
AnyFSPath = Union[FSPath, str]
|
||||
FSPath = str
|
||||
AnyFSPath = str
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import Iterable
|
||||
|
||||
from typing_extensions import Literal
|
||||
|
||||
|
||||
@@ -56,38 +43,6 @@ class FSSpecWrapper(BaseFileSystem):
|
||||
def fs(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def _with_bucket(self, path: AnyFSPath) -> str:
|
||||
return str(path)
|
||||
|
||||
def _strip_bucket(self, entry: str) -> str:
|
||||
return entry
|
||||
|
||||
@overload
|
||||
def _strip_buckets(
|
||||
self, entries: "Iterable[Entry]", detail: "Literal[True]"
|
||||
) -> "Iterator[Entry]":
|
||||
...
|
||||
|
||||
@overload
|
||||
def _strip_buckets(
|
||||
self, entries: "Iterable[str]", detail: "Literal[False]" = False
|
||||
) -> "Iterator[str]":
|
||||
...
|
||||
|
||||
def _strip_buckets(self, entries, detail=False):
|
||||
for entry in entries:
|
||||
if detail:
|
||||
entry = self._entry_hook(entry.copy())
|
||||
entry["name"] = self._strip_bucket(entry["name"])
|
||||
else:
|
||||
entry = self._strip_bucket(entry)
|
||||
yield entry
|
||||
|
||||
def _entry_hook(self, entry: "Entry") -> "Entry":
|
||||
"""Simple hook method to be overridden when wanted to process
|
||||
entries within info() and ls(detail=True) calls"""
|
||||
return entry
|
||||
|
||||
def _prepare_credentials(
|
||||
self, **config: Dict[str, Any] # pylint: disable=unused-argument
|
||||
) -> Dict[str, Any]:
|
||||
@@ -95,134 +50,95 @@ class FSSpecWrapper(BaseFileSystem):
|
||||
host filesystem"""
|
||||
return {}
|
||||
|
||||
def _isdir(self, path_info: FSPath) -> bool:
|
||||
return self.fs.isdir(self._with_bucket(path_info))
|
||||
def _isdir(self, path: AnyFSPath) -> bool:
|
||||
return self.fs.isdir(path)
|
||||
|
||||
def isdir(self, path_info: FSPath) -> bool:
|
||||
def isdir(self, path: AnyFSPath) -> bool:
|
||||
try:
|
||||
return self._isdir(path_info)
|
||||
return self._isdir(path)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
def isfile(self, path_info: FSPath) -> bool:
|
||||
def isfile(self, path: AnyFSPath) -> bool:
|
||||
try:
|
||||
return not self._isdir(path_info)
|
||||
return not self._isdir(path)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
def is_empty(self, path_info: FSPath) -> bool:
|
||||
entry = self.info(path_info)
|
||||
def is_empty(self, path: AnyFSPath) -> bool:
|
||||
entry = self.info(path)
|
||||
if entry["type"] == "directory":
|
||||
return not self.fs.ls(self._with_bucket(path_info))
|
||||
return not self.fs.ls(path)
|
||||
return entry["size"] == 0
|
||||
|
||||
def open(
|
||||
self,
|
||||
path_info: FSPath,
|
||||
path: AnyFSPath,
|
||||
mode: str = "r",
|
||||
encoding: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> "IO": # pylint: disable=arguments-differ
|
||||
return self.fs.open(self._with_bucket(path_info), mode=mode)
|
||||
return self.fs.open(path, mode=mode)
|
||||
|
||||
def checksum(self, path_info: FSPath) -> str:
|
||||
return self.fs.checksum(self._with_bucket(path_info))
|
||||
def checksum(self, path: AnyFSPath) -> str:
|
||||
return self.fs.checksum(path)
|
||||
|
||||
def copy(self, from_info: FSPath, to_info: FSPath) -> None:
|
||||
self.makedirs(to_info.parent)
|
||||
self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
|
||||
def copy(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
|
||||
self.makedirs(self.path.parent(to_info))
|
||||
self.fs.copy(from_info, to_info)
|
||||
|
||||
def exists(self, path_info: FSPath) -> bool:
|
||||
return self.fs.exists(self._with_bucket(path_info))
|
||||
def exists(self, path: AnyFSPath) -> bool:
|
||||
return self.fs.exists(path)
|
||||
|
||||
@overload
|
||||
def ls(
|
||||
self, path_info: FSPath, detail: "Literal[True]"
|
||||
self, path: AnyFSPath, detail: "Literal[True]"
|
||||
) -> "Iterator[Entry]":
|
||||
...
|
||||
|
||||
@overload
|
||||
def ls(self, path_info: FSPath, detail: "Literal[False]") -> Iterator[str]:
|
||||
def ls(self, path: AnyFSPath, detail: "Literal[False]") -> Iterator[str]:
|
||||
...
|
||||
|
||||
def ls(self, path_info, detail=False):
|
||||
path = self._with_bucket(path_info)
|
||||
files = self.fs.ls(path, detail=detail)
|
||||
yield from self._strip_buckets(files, detail=detail)
|
||||
def ls(self, path, detail=False):
|
||||
yield from self.fs.ls(path, detail=detail)
|
||||
|
||||
@overload
|
||||
def find(
|
||||
self,
|
||||
path_info: FSPath,
|
||||
detail: "Literal[True]",
|
||||
prefix: Optional[bool],
|
||||
) -> "Iterator[Entry]":
|
||||
...
|
||||
def find(self, path, prefix=None):
|
||||
yield from self.fs.find(path)
|
||||
|
||||
@overload
|
||||
def find(
|
||||
self,
|
||||
path_info: FSPath,
|
||||
detail: "Literal[False]",
|
||||
prefix: Optional[bool],
|
||||
) -> Iterator[str]:
|
||||
...
|
||||
def move(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
|
||||
self.fs.move(from_info, to_info)
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
def find(self, path_info, detail=False, prefix=None):
|
||||
path = self._with_bucket(path_info)
|
||||
files = self.fs.find(path, detail=detail)
|
||||
if detail:
|
||||
files = files.values()
|
||||
def remove(self, path: AnyFSPath) -> None:
|
||||
self.fs.rm_file(path)
|
||||
|
||||
yield from self._strip_buckets(files, detail=detail)
|
||||
def info(self, path: AnyFSPath) -> "Entry":
|
||||
return self.fs.info(path)
|
||||
|
||||
def walk_files(self, path_info: FSPath, **kwargs) -> Iterator[FSPath]:
|
||||
for file in self.find(path_info, **kwargs):
|
||||
yield path_info.replace(path=file)
|
||||
|
||||
def move(self, from_info: FSPath, to_info: FSPath) -> None:
|
||||
self.fs.move(self._with_bucket(from_info), self._with_bucket(to_info))
|
||||
|
||||
def remove(self, path_info: FSPath) -> None:
|
||||
self.fs.rm_file(self._with_bucket(path_info))
|
||||
|
||||
def info(self, path_info: FSPath) -> "Entry":
|
||||
info = self.fs.info(self._with_bucket(path_info)).copy()
|
||||
info = self._entry_hook(info)
|
||||
info["name"] = self._strip_bucket(info["name"])
|
||||
return info
|
||||
|
||||
def makedirs(self, path_info: FSPath, **kwargs) -> None:
|
||||
self.fs.makedirs(
|
||||
self._with_bucket(path_info), exist_ok=kwargs.pop("exist_ok", True)
|
||||
)
|
||||
def makedirs(self, path: AnyFSPath, **kwargs) -> None:
|
||||
self.fs.makedirs(path, exist_ok=kwargs.pop("exist_ok", True))
|
||||
|
||||
def put_file(
|
||||
self,
|
||||
from_file: FSPath,
|
||||
to_info: FSPath,
|
||||
from_file: AnyFSPath,
|
||||
to_info: AnyFSPath,
|
||||
callback: Any = DEFAULT_CALLBACK,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.fs.put_file(
|
||||
from_file, self._with_bucket(to_info), callback=callback, **kwargs
|
||||
)
|
||||
self.fs.invalidate_cache(self._with_bucket(to_info.parent))
|
||||
self.fs.put_file(from_file, to_info, callback=callback, **kwargs)
|
||||
self.fs.invalidate_cache(self.path.parent(to_info))
|
||||
|
||||
def get_file(
|
||||
self,
|
||||
from_info: FSPath,
|
||||
to_info: FSPath,
|
||||
from_info: AnyFSPath,
|
||||
to_info: AnyFSPath,
|
||||
callback: Any = DEFAULT_CALLBACK,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.fs.get_file(
|
||||
self._with_bucket(from_info), to_info, callback=callback, **kwargs
|
||||
)
|
||||
self.fs.get_file(from_info, to_info, callback=callback, **kwargs)
|
||||
|
||||
def upload_fobj(self, fobj: IO, to_info: FSPath, **kwargs) -> None:
|
||||
self.makedirs(to_info.parent)
|
||||
def upload_fobj(self, fobj: IO, to_info: AnyFSPath, **kwargs) -> None:
|
||||
self.makedirs(self.path.parent(to_info))
|
||||
with self.open(to_info, "wb") as fdest:
|
||||
shutil.copyfileobj(
|
||||
fobj, fdest, length=getattr(fdest, "blocksize", None)
|
||||
@@ -233,22 +149,7 @@ class FSSpecWrapper(BaseFileSystem):
|
||||
class ObjectFSWrapper(FSSpecWrapper):
|
||||
TRAVERSE_PREFIX_LEN = 3
|
||||
|
||||
@lru_cache(512)
|
||||
def _with_bucket(self, path: AnyFSPath) -> str:
|
||||
if isinstance(path, self.PATH_CLS):
|
||||
return f"{path.bucket}/{path.path}"
|
||||
return str(path)
|
||||
|
||||
def _strip_bucket(self, entry: str) -> str:
|
||||
try:
|
||||
bucket, path = entry.split("/", 1)
|
||||
except ValueError:
|
||||
# If there is no path attached, only returns
|
||||
# the bucket (top-level).
|
||||
bucket, path = entry, ""
|
||||
return path or bucket
|
||||
|
||||
def makedirs(self, path_info: FSPath, **kwargs) -> None:
|
||||
def makedirs(self, path: AnyFSPath, **kwargs) -> None:
|
||||
# For object storages make this method a no-op. The original
|
||||
# fs.makedirs() method will only check if the bucket exists
|
||||
# and create if it doesn't though we don't want to support
|
||||
@@ -256,44 +157,37 @@ class ObjectFSWrapper(FSSpecWrapper):
|
||||
# simply ignore all mkdir()/makedirs() calls.
|
||||
return None
|
||||
|
||||
def _isdir(self, path_info: FSPath) -> bool:
|
||||
def _isdir(self, path: AnyFSPath) -> bool:
|
||||
# Directory in object storages are interpreted differently
|
||||
# among different fsspec providers, so this logic is a temporary
|
||||
# measure for us to adapt as of now. It checks whether it is a
|
||||
# directory (as in a prefix with contents) or whether it is an empty
|
||||
# file where it's name ends with a forward slash
|
||||
|
||||
entry = self.info(path_info)
|
||||
entry = self.info(path)
|
||||
return entry["type"] == "directory" or (
|
||||
entry["size"] == 0
|
||||
and entry["type"] == "file"
|
||||
and entry["name"].endswith("/")
|
||||
)
|
||||
|
||||
def find(self, path_info, detail=False, prefix=None):
|
||||
def find(self, path, prefix=None):
|
||||
if prefix:
|
||||
path = self._with_bucket(path_info.parent)
|
||||
files = self.fs.find(
|
||||
path, detail=detail, prefix=path_info.parts[-1]
|
||||
)
|
||||
with_prefix = self.path.parent(path)
|
||||
files = self.fs.find(with_prefix, prefix=self.path.parts(path)[-1])
|
||||
else:
|
||||
path = self._with_bucket(path_info)
|
||||
files = self.fs.find(path, detail=detail)
|
||||
|
||||
if detail:
|
||||
files = list(files.values())
|
||||
with_prefix = path
|
||||
files = self.fs.find(path)
|
||||
|
||||
# When calling find() on a file, it returns the same file in a list.
|
||||
# For object-based storages, the same behavior applies to empty
|
||||
# directories since they are represented as files. This condition
|
||||
# checks whether we should yield an empty list (if it is an empty
|
||||
# directory) or just yield the file itself.
|
||||
if len(files) == 1 and files[0] == path and self.isdir(path_info):
|
||||
if len(files) == 1 and files[0] == with_prefix and self.isdir(path):
|
||||
return None
|
||||
|
||||
yield from self._strip_buckets( # pylint: disable=not-an-iterable
|
||||
files, detail=detail
|
||||
)
|
||||
yield from files
|
||||
|
||||
|
||||
# pylint: disable=arguments-differ
|
||||
@@ -310,9 +204,6 @@ class NoDirectoriesMixin:
|
||||
def walk(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def walk_files(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def ls(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -329,13 +220,13 @@ class CallbackMixin:
|
||||
):
|
||||
"""Add compatibility support for Callback."""
|
||||
# pylint: disable=protected-access
|
||||
self.makedirs(to_info.parent)
|
||||
self.makedirs(self.path.parent(to_info))
|
||||
size = os.path.getsize(from_file)
|
||||
with open(from_file, "rb") as fobj:
|
||||
callback.set_size(size)
|
||||
wrapped = CallbackIOWrapper(callback.relative_update, fobj)
|
||||
self.upload_fobj(wrapped, to_info)
|
||||
self.fs.invalidate_cache(self._with_bucket(to_info.parent))
|
||||
self.fs.invalidate_cache(self.path.parent(to_info))
|
||||
|
||||
def get_file(
|
||||
self,
|
||||
|
||||
+20
-43
@@ -1,14 +1,10 @@
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import threading
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import format_link, tmp_fname
|
||||
|
||||
@@ -34,24 +30,8 @@ class GDriveAuthError(DvcException):
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class GDriveURLInfo(CloudURLInfo):
|
||||
def __init__(self, url):
|
||||
super().__init__(url)
|
||||
|
||||
# GDrive URL host part is case sensitive,
|
||||
# we are restoring it here.
|
||||
p = urlparse(url)
|
||||
self.host = p.netloc
|
||||
assert self.netloc == self.host
|
||||
|
||||
# Normalize path. Important since we have a cache (path to ID)
|
||||
# and don't want to deal with different variations of path in it.
|
||||
self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
|
||||
|
||||
|
||||
class GDriveFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
|
||||
scheme = Schemes.GDRIVE
|
||||
PATH_CLS = GDriveURLInfo
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
REQUIRES = {"pydrive2": "pydrive2"}
|
||||
# Always prefer traverse for GDrive since API usage quotas are a concern.
|
||||
@@ -65,11 +45,14 @@ class GDriveFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
|
||||
DEFAULT_GDRIVE_CLIENT_SECRET = "a1Fz59uTpVNeG_VGuSKDLJXv"
|
||||
|
||||
def __init__(self, **config):
|
||||
from fsspec.utils import infer_storage_options
|
||||
|
||||
super().__init__(**config)
|
||||
|
||||
self.path_info = self.PATH_CLS(config["url"])
|
||||
self.url = config["url"]
|
||||
opts = infer_storage_options(self.url)
|
||||
|
||||
if not self.path_info.bucket:
|
||||
if not opts["host"]:
|
||||
raise DvcException(
|
||||
"Empty GDrive URL '{}'. Learn more at {}".format(
|
||||
config["url"],
|
||||
@@ -77,8 +60,8 @@ class GDriveFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
|
||||
)
|
||||
)
|
||||
|
||||
self._bucket = self.path_info.bucket
|
||||
self._path = self.path_info.path
|
||||
self._bucket = opts["host"]
|
||||
self._path = opts["path"].lstrip("/")
|
||||
self._trash_only = config.get("gdrive_trash_only")
|
||||
self._use_service_account = config.get("gdrive_use_service_account")
|
||||
self._service_account_user_email = config.get(
|
||||
@@ -106,6 +89,16 @@ class GDriveFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
|
||||
)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _strip_protocol(cls, path):
|
||||
from fsspec.utils import infer_storage_options
|
||||
|
||||
return infer_storage_options(path)["path"].lstrip("/")
|
||||
|
||||
@staticmethod
|
||||
def unstrip_protocol(path):
|
||||
return "gdrive://{path}"
|
||||
|
||||
@staticmethod
|
||||
def _get_kwargs_from_urls(urlpath):
|
||||
return {"url": urlpath}
|
||||
@@ -255,27 +248,11 @@ class GDriveFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
|
||||
os.remove(temporary_save_path)
|
||||
|
||||
return _GDriveFileSystem(
|
||||
self._with_bucket(self.path_info),
|
||||
self.url,
|
||||
gauth,
|
||||
trash_only=self._trash_only,
|
||||
)
|
||||
|
||||
def _with_bucket(self, path):
|
||||
if isinstance(path, self.PATH_CLS):
|
||||
return posixpath.join(path.bucket, path.path)
|
||||
|
||||
return super()._with_bucket(path)
|
||||
|
||||
def _strip_bucket(self, entry):
|
||||
try:
|
||||
bucket, path = entry.split("/", 1)
|
||||
except ValueError:
|
||||
# If there is no path attached, only returns
|
||||
# the bucket (top-level).
|
||||
bucket, path = entry, None
|
||||
return path or bucket
|
||||
|
||||
def upload_fobj(self, fobj, to_info, **kwargs):
|
||||
rpath = self._with_bucket(to_info)
|
||||
self.makedirs(os.path.dirname(rpath))
|
||||
return self.fs.upload_fobj(fobj, rpath, **kwargs)
|
||||
self.makedirs(os.path.dirname(to_info))
|
||||
return self.fs.upload_fobj(fobj, to_info, **kwargs)
|
||||
|
||||
+16
-16
@@ -55,16 +55,16 @@ class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
errno.EISDIR, os.strerror(errno.EISDIR), path
|
||||
) from exc
|
||||
|
||||
def exists(self, path_info) -> bool:
|
||||
key = self._get_key(path_info)
|
||||
def exists(self, path) -> bool:
|
||||
key = self._get_key(path)
|
||||
return self.trie.exists(key)
|
||||
|
||||
def isdir(self, path_info) -> bool:
|
||||
key = self._get_key(path_info)
|
||||
def isdir(self, path) -> bool:
|
||||
key = self._get_key(path)
|
||||
return self.trie.isdir(key)
|
||||
|
||||
def isfile(self, path_info) -> bool:
|
||||
key = self._get_key(path_info)
|
||||
def isfile(self, path) -> bool:
|
||||
key = self._get_key(path)
|
||||
return self.trie.isfile(key)
|
||||
|
||||
def walk(self, top, topdown=True, onerror=None, **kwargs):
|
||||
@@ -94,27 +94,27 @@ class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
root = self._root
|
||||
yield root, dirs, files
|
||||
|
||||
def isexec(self, path_info):
|
||||
if not self.exists(path_info):
|
||||
def isexec(self, path):
|
||||
if not self.exists(path):
|
||||
return False
|
||||
|
||||
mode = self.info(path_info)["mode"]
|
||||
mode = self.info(path)["mode"]
|
||||
return is_exec(mode)
|
||||
|
||||
def info(self, path_info):
|
||||
key = self._get_key(path_info)
|
||||
def info(self, path):
|
||||
key = self._get_key(path)
|
||||
try:
|
||||
return self.trie.info(key)
|
||||
except KeyError:
|
||||
raise FileNotFoundError(
|
||||
errno.ENOENT, os.strerror(errno.ENOENT), path_info
|
||||
errno.ENOENT, os.strerror(errno.ENOENT), path
|
||||
)
|
||||
|
||||
def checksum(self, path_info):
|
||||
return self.info(path_info)["sha"]
|
||||
def checksum(self, path):
|
||||
return self.info(path)["sha"]
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
for root, _, files in self.walk(path_info, **kwargs):
|
||||
def find(self, path, prefix=None):
|
||||
for root, _, files in self.walk(path):
|
||||
for file in files:
|
||||
# NOTE: os.path.join is ~5.5 times slower
|
||||
yield f"{root}{os.sep}{file}"
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import base64
|
||||
import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
# pylint:disable=abstract-method
|
||||
@@ -12,7 +10,6 @@ from .fsspec_wrapper import CallbackMixin, ObjectFSWrapper
|
||||
|
||||
class GSFileSystem(CallbackMixin, ObjectFSWrapper):
|
||||
scheme = Schemes.GS
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"gcsfs": "gcsfs"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
DETAIL_FIELDS = frozenset(("etag", "size"))
|
||||
@@ -23,11 +20,6 @@ class GSFileSystem(CallbackMixin, ObjectFSWrapper):
|
||||
login_info["token"] = config.get("credentialpath")
|
||||
return login_info
|
||||
|
||||
def _entry_hook(self, entry):
|
||||
if "etag" in entry:
|
||||
entry["etag"] = base64.b64decode(entry["etag"]).hex()
|
||||
return entry
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def fs(self):
|
||||
|
||||
+18
-9
@@ -19,10 +19,17 @@ class HDFSFileSystem(CallbackMixin, FSSpecWrapper):
|
||||
REQUIRES = {"fsspec": "fsspec", "pyarrow": "pyarrow"}
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
|
||||
def _with_bucket(self, path):
|
||||
if isinstance(path, self.PATH_CLS):
|
||||
return path.path
|
||||
return super()._with_bucket(path)
|
||||
@classmethod
|
||||
def _strip_protocol(cls, path: str) -> str:
|
||||
from fsspec.utils import infer_storage_options
|
||||
|
||||
return infer_storage_options(path)["path"].lstrip("/")
|
||||
|
||||
def unstrip_protocol(self, path: str) -> str:
|
||||
host = self.fs_args["host"]
|
||||
port = self.fs_args["port"]
|
||||
path = path.lstrip("/")
|
||||
return f"hdfs://{host}:{port}/{path}"
|
||||
|
||||
@staticmethod
|
||||
def _get_kwargs_from_urls(urlpath):
|
||||
@@ -41,17 +48,19 @@ class HDFSFileSystem(CallbackMixin, FSSpecWrapper):
|
||||
|
||||
return HadoopFileSystem(**self.fs_args)
|
||||
|
||||
def checksum(self, path_info):
|
||||
return self._checksum(path_info)
|
||||
def checksum(self, path):
|
||||
return self._checksum(path)
|
||||
|
||||
def _checksum(self, path_info, **kwargs):
|
||||
def _checksum(self, path, **kwargs):
|
||||
# PyArrow doesn't natively support retrieving the
|
||||
# checksum, so we have to use hadoop fs
|
||||
|
||||
url = self.unstrip_protocol(path)
|
||||
|
||||
result = self._run_command(
|
||||
f"checksum {path_info.url}",
|
||||
f"checksum {url}",
|
||||
env=fix_env(os.environ),
|
||||
user=path_info.user,
|
||||
user=self.fs_args["user"],
|
||||
)
|
||||
if result is None:
|
||||
return None
|
||||
|
||||
+2
-5
@@ -3,7 +3,6 @@ import threading
|
||||
from funcy import cached_property, memoize, wrap_with
|
||||
|
||||
from dvc import prompt
|
||||
from dvc.path_info import HTTPURLInfo
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .fsspec_wrapper import FSSpecWrapper, NoDirectoriesMixin
|
||||
@@ -34,7 +33,6 @@ def make_context(ssl_verify):
|
||||
# pylint: disable=abstract-method
|
||||
class HTTPFileSystem(NoDirectoriesMixin, FSSpecWrapper):
|
||||
scheme = Schemes.HTTP
|
||||
PATH_CLS = HTTPURLInfo
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
REQUIRES = {"aiohttp": "aiohttp", "aiohttp-retry": "aiohttp_retry"}
|
||||
CAN_TRAVERSE = False
|
||||
@@ -127,6 +125,5 @@ class HTTPFileSystem(NoDirectoriesMixin, FSSpecWrapper):
|
||||
|
||||
return _HTTPFileSystem(**self.fs_args)
|
||||
|
||||
def _entry_hook(self, entry):
|
||||
entry["checksum"] = entry.get("ETag") or entry.get("Content-MD5")
|
||||
return entry
|
||||
def unstrip_protocol(self, path: str) -> str:
|
||||
return path
|
||||
|
||||
+54
-49
@@ -1,7 +1,8 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.system import System
|
||||
from dvc.utils import is_exec, tmp_fname
|
||||
@@ -9,15 +10,23 @@ from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove
|
||||
|
||||
from ..progress import DEFAULT_CALLBACK
|
||||
from .base import BaseFileSystem
|
||||
from .path import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LocalPath(Path):
|
||||
def parent(self, path):
|
||||
return os.path.dirname(path)
|
||||
|
||||
def drive(self, path):
|
||||
return os.path.splitdrive(path)[0]
|
||||
|
||||
|
||||
class LocalFileSystem(BaseFileSystem):
|
||||
sep = os.sep
|
||||
|
||||
scheme = Schemes.LOCAL
|
||||
PATH_CLS = PathInfo
|
||||
PARAM_CHECKSUM = "md5"
|
||||
PARAM_PATH = "path"
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
@@ -28,33 +37,34 @@ class LocalFileSystem(BaseFileSystem):
|
||||
super().__init__(**config)
|
||||
self.fs = LocalFS()
|
||||
|
||||
@staticmethod
|
||||
def open(path_info, mode="r", encoding=None, **kwargs):
|
||||
return open(path_info, mode=mode, encoding=encoding)
|
||||
@cached_property
|
||||
def path(self):
|
||||
return LocalPath(os.sep)
|
||||
|
||||
def exists(self, path_info) -> bool:
|
||||
assert isinstance(path_info, str) or path_info.scheme == "local"
|
||||
@staticmethod
|
||||
def open(path, mode="r", encoding=None, **kwargs):
|
||||
return open(path, mode=mode, encoding=encoding)
|
||||
|
||||
def exists(self, path) -> bool:
|
||||
# TODO: replace this with os.path.exists once the problem is fixed on
|
||||
# the fsspec https://github.com/intake/filesystem_spec/issues/742
|
||||
return os.path.lexists(path_info)
|
||||
return os.path.lexists(path)
|
||||
|
||||
def checksum(self, path_info) -> str:
|
||||
def checksum(self, path) -> str:
|
||||
from fsspec.utils import tokenize
|
||||
|
||||
st = os.stat(path_info)
|
||||
st = os.stat(path)
|
||||
|
||||
return str(int(tokenize([st.st_ino, st.st_mtime, st.st_size]), 16))
|
||||
|
||||
def isfile(self, path_info) -> bool:
|
||||
return os.path.isfile(path_info)
|
||||
def isfile(self, path) -> bool:
|
||||
return os.path.isfile(path)
|
||||
|
||||
def isdir(self, path_info) -> bool:
|
||||
return os.path.isdir(path_info)
|
||||
def isdir(self, path) -> bool:
|
||||
return os.path.isdir(path)
|
||||
|
||||
def iscopy(self, path_info):
|
||||
return not (
|
||||
System.is_symlink(path_info) or System.is_hardlink(path_info)
|
||||
)
|
||||
def iscopy(self, path):
|
||||
return not (System.is_symlink(path) or System.is_hardlink(path))
|
||||
|
||||
def walk(self, top, topdown=True, onerror=None, **kwargs):
|
||||
"""Directory fs generator.
|
||||
@@ -67,45 +77,37 @@ class LocalFileSystem(BaseFileSystem):
|
||||
):
|
||||
yield os.path.normpath(root), dirs, files
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
for root, _, files in self.walk(path_info):
|
||||
def find(self, path, prefix=None):
|
||||
for root, _, files in self.walk(path):
|
||||
for file in files:
|
||||
# NOTE: os.path.join is ~5.5 times slower
|
||||
yield PathInfo(f"{root}{os.sep}{file}")
|
||||
yield f"{root}{os.sep}{file}"
|
||||
|
||||
def is_empty(self, path_info):
|
||||
if self.isfile(path_info) and os.path.getsize(path_info) == 0:
|
||||
def is_empty(self, path):
|
||||
if self.isfile(path) and os.path.getsize(path) == 0:
|
||||
return True
|
||||
|
||||
if self.isdir(path_info) and len(os.listdir(path_info)) == 0:
|
||||
if self.isdir(path) and len(os.listdir(path)) == 0:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def remove(self, path_info):
|
||||
if isinstance(path_info, PathInfo):
|
||||
if path_info.scheme != "local":
|
||||
raise NotImplementedError
|
||||
remove(path_info)
|
||||
def remove(self, path):
|
||||
remove(path)
|
||||
|
||||
def makedirs(self, path_info, **kwargs):
|
||||
makedirs(path_info, exist_ok=kwargs.pop("exist_ok", True))
|
||||
def makedirs(self, path, **kwargs):
|
||||
makedirs(path, exist_ok=kwargs.pop("exist_ok", True))
|
||||
|
||||
def isexec(self, path_info):
|
||||
mode = self.info(path_info)["mode"]
|
||||
def isexec(self, path):
|
||||
mode = self.info(path)["mode"]
|
||||
return is_exec(mode)
|
||||
|
||||
def move(self, from_info, to_info):
|
||||
if (
|
||||
isinstance(from_info, PathInfo) and from_info.scheme != "local"
|
||||
) or (isinstance(to_info, PathInfo) and to_info.scheme != "local"):
|
||||
raise NotImplementedError
|
||||
|
||||
self.makedirs(to_info.parent)
|
||||
self.makedirs(self.path.parent(to_info))
|
||||
move(from_info, to_info)
|
||||
|
||||
def copy(self, from_info, to_info):
|
||||
tmp_info = to_info.parent / tmp_fname("")
|
||||
tmp_info = self.path.join(self.path.parent(to_info), tmp_fname(""))
|
||||
try:
|
||||
copyfile(from_info, tmp_info)
|
||||
os.rename(tmp_info, to_info)
|
||||
@@ -114,8 +116,8 @@ class LocalFileSystem(BaseFileSystem):
|
||||
raise
|
||||
|
||||
def upload_fobj(self, fobj, to_info, **kwargs):
|
||||
self.makedirs(to_info.parent)
|
||||
tmp_info = to_info.parent / tmp_fname("")
|
||||
self.makedirs(self.path.parent(to_info))
|
||||
tmp_info = self.path.join(self.path.parent(to_info), tmp_fname(""))
|
||||
try:
|
||||
copy_fobj_to_file(fobj, tmp_info)
|
||||
os.rename(tmp_info, to_info)
|
||||
@@ -128,8 +130,8 @@ class LocalFileSystem(BaseFileSystem):
|
||||
System.symlink(from_info, to_info)
|
||||
|
||||
@staticmethod
|
||||
def is_symlink(path_info):
|
||||
return System.is_symlink(path_info)
|
||||
def is_symlink(path):
|
||||
return System.is_symlink(path)
|
||||
|
||||
def hardlink(self, from_info, to_info):
|
||||
# If there are a lot of empty files (which happens a lot in datasets),
|
||||
@@ -154,19 +156,19 @@ class LocalFileSystem(BaseFileSystem):
|
||||
System.hardlink(from_info, to_info)
|
||||
|
||||
@staticmethod
|
||||
def is_hardlink(path_info):
|
||||
return System.is_hardlink(path_info)
|
||||
def is_hardlink(path):
|
||||
return System.is_hardlink(path)
|
||||
|
||||
def reflink(self, from_info, to_info):
|
||||
System.reflink(from_info, to_info)
|
||||
|
||||
def info(self, path_info):
|
||||
return self.fs.info(path_info)
|
||||
def info(self, path):
|
||||
return self.fs.info(path)
|
||||
|
||||
def put_file(
|
||||
self, from_file, to_info, callback=DEFAULT_CALLBACK, **kwargs
|
||||
):
|
||||
makedirs(to_info.parent, exist_ok=True)
|
||||
makedirs(self.path.parent(to_info), exist_ok=True)
|
||||
tmp_file = tmp_fname(to_info)
|
||||
copyfile(from_file, tmp_file, callback=callback)
|
||||
os.replace(tmp_file, to_info)
|
||||
@@ -175,3 +177,6 @@ class LocalFileSystem(BaseFileSystem):
|
||||
self, from_info, to_file, callback=DEFAULT_CALLBACK, **kwargs
|
||||
):
|
||||
copyfile(from_info, to_file, callback=callback)
|
||||
|
||||
|
||||
localfs = LocalFileSystem()
|
||||
|
||||
@@ -2,7 +2,6 @@ import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .fsspec_wrapper import FSSpecWrapper
|
||||
@@ -11,7 +10,6 @@ from .fsspec_wrapper import FSSpecWrapper
|
||||
class MemoryFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
|
||||
scheme = Schemes.MEMORY
|
||||
PARAM_CHECKSUM = "md5"
|
||||
PATH_CLS = CloudURLInfo
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
DEFAULT_BLOCKSIZE = 4096
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .fsspec_wrapper import ObjectFSWrapper
|
||||
@@ -15,7 +14,6 @@ logger = logging.getLogger(__name__)
|
||||
# pylint:disable=abstract-method
|
||||
class OSSFileSystem(ObjectFSWrapper):
|
||||
scheme = Schemes.OSS
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"ossfs": "ossfs"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
COPY_POLL_SECONDS = 5
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import posixpath
|
||||
|
||||
|
||||
class Path:
|
||||
"""
|
||||
Class for operations on simple string paths.
|
||||
|
||||
This is meant to be very efficient and so doesn't use os.path,
|
||||
doesn't have any notion of cwd and assumes that we are always
|
||||
operating on absolute paths.
|
||||
"""
|
||||
|
||||
def __init__(self, sep: str):
|
||||
self.sep = sep
|
||||
|
||||
def join(self, *parts):
|
||||
if len(parts) == 0:
|
||||
return None
|
||||
if len(parts) == 1:
|
||||
return parts[0]
|
||||
|
||||
return self.sep.join(parts)
|
||||
|
||||
def parts(self, path):
|
||||
return tuple(path.split(self.sep))
|
||||
|
||||
def parent(self, path):
|
||||
parts = path.rsplit(self.sep, 1)
|
||||
if len(parts) == 1:
|
||||
return ""
|
||||
return parts[0]
|
||||
|
||||
def parents(self, path):
|
||||
parts = self.parts(path)
|
||||
return tuple(
|
||||
self.join(*parts[:length])
|
||||
for length in range(len(parts) - 1, 0, -1)
|
||||
)
|
||||
|
||||
def name(self, path):
|
||||
return self.parts(path)[-1]
|
||||
|
||||
def suffix(self, path):
|
||||
name = self.name(path)
|
||||
_, dot, suffix = name.partition(".")
|
||||
return dot + suffix
|
||||
|
||||
def with_name(self, path, name):
|
||||
parts = list(self.parts(path))
|
||||
parts[-1] = name
|
||||
return self.join(*parts)
|
||||
|
||||
def with_suffix(self, path, suffix):
|
||||
parts = list(self.parts(path))
|
||||
real_path, _, _ = parts[-1].partition(".")
|
||||
parts[-1] = real_path + suffix
|
||||
return self.join(*parts)
|
||||
|
||||
def isin(self, left, right):
|
||||
left_parts = self.parts(left)
|
||||
right_parts = self.parts(right)
|
||||
left_len = len(left_parts)
|
||||
right_len = len(right_parts)
|
||||
return left_len > right_len and left_parts[:right_len] == right_parts
|
||||
|
||||
def isin_or_eq(self, left, right):
|
||||
return left == right or self.isin(left, right)
|
||||
|
||||
def overlaps(self, left, right):
|
||||
# pylint: disable=arguments-out-of-order
|
||||
return self.isin_or_eq(left, right) or self.isin(right, left)
|
||||
|
||||
def relpath(self, path, base):
|
||||
assert len(path) > len(base)
|
||||
assert path.startswith(base)
|
||||
normpath = path.rstrip(self.sep)
|
||||
normbase = base.rstrip(self.sep)
|
||||
return normpath[len(normbase) + 1 :]
|
||||
|
||||
def as_posix(self, path):
|
||||
return path.replace(self.sep, posixpath.sep)
|
||||
+34
-38
@@ -7,8 +7,6 @@ from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union
|
||||
|
||||
from funcy import lfilter, wrap_with
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
from ..progress import DEFAULT_CALLBACK
|
||||
from .base import BaseFileSystem
|
||||
from .dvc import DvcFileSystem
|
||||
@@ -164,7 +162,7 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
if not prefix:
|
||||
return None
|
||||
|
||||
parents = (parent.fspath for parent in PathInfo(path).parents)
|
||||
parents = (parent for parent in self.path.parents(path))
|
||||
dirs = [path] + list(takewhile(lambda p: p != prefix, parents))
|
||||
dirs.reverse()
|
||||
self._update(dirs, starting_repo=repo)
|
||||
@@ -217,34 +215,33 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
encoding = None
|
||||
|
||||
fs, dvc_fs = self._get_fs_pair(path)
|
||||
path_info = PathInfo(path)
|
||||
try:
|
||||
return fs.open(path_info, mode=mode, encoding=encoding)
|
||||
return fs.open(path, mode=mode, encoding=encoding)
|
||||
except FileNotFoundError:
|
||||
if not dvc_fs:
|
||||
raise
|
||||
|
||||
return dvc_fs.open(path_info, mode=mode, encoding=encoding, **kwargs)
|
||||
return dvc_fs.open(path, mode=mode, encoding=encoding, **kwargs)
|
||||
|
||||
def exists(self, path_info) -> bool:
|
||||
fs, dvc_fs = self._get_fs_pair(path_info)
|
||||
def exists(self, path) -> bool:
|
||||
fs, dvc_fs = self._get_fs_pair(path)
|
||||
|
||||
if not dvc_fs:
|
||||
return fs.exists(path_info)
|
||||
return fs.exists(path)
|
||||
|
||||
if dvc_fs.repo.dvcignore.is_ignored(fs, path_info):
|
||||
if dvc_fs.repo.dvcignore.is_ignored(fs, path):
|
||||
return False
|
||||
|
||||
if fs.exists(path_info):
|
||||
if fs.exists(path):
|
||||
return True
|
||||
|
||||
try:
|
||||
meta = dvc_fs.metadata(path_info)
|
||||
meta = dvc_fs.metadata(path)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
for out in meta.outs:
|
||||
if fs.exists(out.path_info):
|
||||
if fs.exists(out.fs_path):
|
||||
return False
|
||||
|
||||
return True
|
||||
@@ -271,7 +268,7 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
return False
|
||||
|
||||
for out in meta.outs:
|
||||
if fs.exists(out.path_info):
|
||||
if fs.exists(out.fs_path):
|
||||
return False
|
||||
|
||||
return meta.isdir
|
||||
@@ -303,15 +300,15 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
|
||||
(out,) = meta.outs
|
||||
assert len(meta.outs) == 1
|
||||
if fs.exists(out.path_info):
|
||||
if fs.exists(out.fs_path):
|
||||
return False
|
||||
return meta.isfile
|
||||
|
||||
def isexec(self, path_info):
|
||||
fs, dvc_fs = self._get_fs_pair(path_info)
|
||||
if dvc_fs and dvc_fs.exists(path_info):
|
||||
return dvc_fs.isexec(path_info)
|
||||
return fs.isexec(path_info)
|
||||
def isexec(self, path):
|
||||
fs, dvc_fs = self._get_fs_pair(path)
|
||||
if dvc_fs and dvc_fs.exists(path):
|
||||
return dvc_fs.isexec(path)
|
||||
return fs.isexec(path)
|
||||
|
||||
def _dvc_walk(self, walk):
|
||||
try:
|
||||
@@ -444,10 +441,10 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
|
||||
yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
for root, _, files in self.walk(path_info, **kwargs):
|
||||
def find(self, path, prefix=None):
|
||||
for root, _, files in self.walk(path):
|
||||
for fname in files:
|
||||
yield PathInfo(root) / fname
|
||||
yield self.path.join(root, fname)
|
||||
|
||||
def get_file(
|
||||
self, from_info, to_file, callback=DEFAULT_CALLBACK, **kwargs
|
||||
@@ -467,18 +464,17 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
)
|
||||
|
||||
def metadata(self, path):
|
||||
abspath = os.path.abspath(path)
|
||||
path_info = PathInfo(abspath)
|
||||
fs, dvc_fs = self._get_fs_pair(path_info)
|
||||
fs_path = os.path.abspath(path)
|
||||
fs, dvc_fs = self._get_fs_pair(fs_path)
|
||||
|
||||
dvc_meta = None
|
||||
if dvc_fs:
|
||||
with suppress(FileNotFoundError):
|
||||
dvc_meta = dvc_fs.metadata(path_info)
|
||||
dvc_meta = dvc_fs.metadata(fs_path)
|
||||
|
||||
info_result = None
|
||||
with suppress(FileNotFoundError):
|
||||
info_result = fs.info(path_info)
|
||||
info_result = fs.info(fs_path)
|
||||
|
||||
if not info_result and not dvc_meta:
|
||||
raise FileNotFoundError
|
||||
@@ -486,8 +482,8 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
from ._metadata import Metadata
|
||||
|
||||
meta = dvc_meta or Metadata(
|
||||
path_info=path_info,
|
||||
repo=self._get_repo(abspath) or self._main_repo,
|
||||
fs_path=fs_path,
|
||||
repo=self._get_repo(fs_path) or self._main_repo,
|
||||
)
|
||||
|
||||
isdir = bool(info_result) and info_result["type"] == "directory"
|
||||
@@ -499,18 +495,18 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
meta.is_exec = bool(info_result) and is_exec(info_result["mode"])
|
||||
return meta
|
||||
|
||||
def info(self, path_info):
|
||||
fs, dvc_fs = self._get_fs_pair(path_info)
|
||||
def info(self, path):
|
||||
fs, dvc_fs = self._get_fs_pair(path)
|
||||
|
||||
try:
|
||||
return fs.info(path_info)
|
||||
return fs.info(path)
|
||||
except FileNotFoundError:
|
||||
return dvc_fs.info(path_info)
|
||||
return dvc_fs.info(path)
|
||||
|
||||
def checksum(self, path_info):
|
||||
fs, dvc_fs = self._get_fs_pair(path_info)
|
||||
def checksum(self, path):
|
||||
fs, dvc_fs = self._get_fs_pair(path)
|
||||
|
||||
try:
|
||||
return fs.checksum(path_info)
|
||||
return fs.checksum(path)
|
||||
except FileNotFoundError:
|
||||
return dvc_fs.checksum(path_info)
|
||||
return dvc_fs.checksum(path)
|
||||
|
||||
+14
-12
@@ -5,7 +5,6 @@ from collections import defaultdict
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import DEFAULT_CALLBACK
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
@@ -17,7 +16,6 @@ _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
|
||||
# pylint:disable=abstract-method
|
||||
class BaseS3FileSystem(ObjectFSWrapper):
|
||||
scheme = Schemes.S3
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"s3fs": "s3fs", "boto3": "boto3"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
DETAIL_FIELDS = frozenset(("etag", "size"))
|
||||
@@ -148,12 +146,6 @@ class BaseS3FileSystem(ObjectFSWrapper):
|
||||
}
|
||||
)
|
||||
|
||||
def _entry_hook(self, entry):
|
||||
entry = entry.copy()
|
||||
if "ETag" in entry:
|
||||
entry["etag"] = entry["ETag"].strip('"')
|
||||
return entry
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def fs(self):
|
||||
@@ -161,6 +153,15 @@ class BaseS3FileSystem(ObjectFSWrapper):
|
||||
|
||||
return _S3FileSystem(**self.fs_args)
|
||||
|
||||
@classmethod
|
||||
def _strip_protocol(cls, path: str) -> str:
|
||||
from fsspec.utils import infer_storage_options
|
||||
|
||||
return infer_storage_options(path)["path"]
|
||||
|
||||
def unstrip_protocol(self, path):
|
||||
return "s3://" + path.lstrip("/")
|
||||
|
||||
|
||||
def _translate_exceptions(func):
|
||||
@functools.wraps(func)
|
||||
@@ -204,9 +205,10 @@ class S3FileSystem(BaseS3FileSystem): # pylint:disable=abstract-method
|
||||
verify=client_kwargs.get("verify"),
|
||||
)
|
||||
|
||||
def _get_obj(self, path_info):
|
||||
bucket = self.s3.Bucket(path_info.bucket)
|
||||
return bucket.Object(path_info.path)
|
||||
def _get_obj(self, path):
|
||||
bucket_name, key, _ = self.fs.split_path(path)
|
||||
bucket = self.s3.Bucket(bucket_name)
|
||||
return bucket.Object(key)
|
||||
|
||||
@_translate_exceptions
|
||||
def put_file(
|
||||
@@ -220,7 +222,7 @@ class S3FileSystem(BaseS3FileSystem): # pylint:disable=abstract-method
|
||||
ExtraArgs=self.fs_args.get("s3_additional_kwargs"),
|
||||
Config=self._transfer_config,
|
||||
)
|
||||
self.fs.invalidate_cache(self._with_bucket(to_info.parent))
|
||||
self.fs.invalidate_cache(self.path.parent(to_info))
|
||||
|
||||
@_translate_exceptions
|
||||
def get_file(
|
||||
|
||||
+11
-4
@@ -34,10 +34,17 @@ class SSHFileSystem(FSSpecWrapper):
|
||||
DEFAULT_PORT = 22
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
def _with_bucket(self, path):
|
||||
if isinstance(path, self.PATH_CLS):
|
||||
return path.path
|
||||
return super()._with_bucket(path)
|
||||
@classmethod
|
||||
def _strip_protocol(cls, path: str) -> str:
|
||||
from fsspec.utils import infer_storage_options
|
||||
|
||||
return infer_storage_options(path)["path"]
|
||||
|
||||
def unstrip_protocol(self, path: str) -> str:
|
||||
host = self.fs_args["host"]
|
||||
port = self.fs_args["port"]
|
||||
path = path.lstrip("/")
|
||||
return f"ssh://{host}:{port}/{path}"
|
||||
|
||||
def _prepare_credentials(self, **config):
|
||||
self.CAN_TRAVERSE = True
|
||||
|
||||
+3
-5
@@ -7,7 +7,7 @@ from .base import RemoteActionNotImplemented
|
||||
from .local import LocalFileSystem
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.types import AnyPath, DvcPath
|
||||
from dvc.types import AnyPath
|
||||
|
||||
from .base import BaseFileSystem
|
||||
|
||||
@@ -18,7 +18,7 @@ def _link(
|
||||
from_fs: "BaseFileSystem",
|
||||
from_info: "AnyPath",
|
||||
to_fs: "BaseFileSystem",
|
||||
to_info: "DvcPath",
|
||||
to_info: "AnyPath",
|
||||
hardlink: bool = False,
|
||||
) -> None:
|
||||
if not isinstance(from_fs, type(to_fs)):
|
||||
@@ -47,7 +47,7 @@ def transfer(
|
||||
from_fs: "BaseFileSystem",
|
||||
from_info: "AnyPath",
|
||||
to_fs: "BaseFileSystem",
|
||||
to_info: "DvcPath",
|
||||
to_info: "AnyPath",
|
||||
hardlink: bool = False,
|
||||
) -> None:
|
||||
try:
|
||||
@@ -58,8 +58,6 @@ def transfer(
|
||||
raise
|
||||
|
||||
if isinstance(from_fs, LocalFileSystem):
|
||||
if not isinstance(from_info, from_fs.PATH_CLS):
|
||||
from_info = from_fs.PATH_CLS(from_info)
|
||||
return to_fs.upload(from_info, to_info)
|
||||
|
||||
if isinstance(to_fs, LocalFileSystem):
|
||||
|
||||
+27
-18
@@ -1,10 +1,8 @@
|
||||
import logging
|
||||
import threading
|
||||
from functools import lru_cache
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import WebDAVURLInfo
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .fsspec_wrapper import FSSpecWrapper
|
||||
@@ -15,7 +13,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class WebDAVFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
|
||||
scheme = Schemes.WEBDAV
|
||||
PATH_CLS = WebDAVURLInfo
|
||||
CAN_TRAVERSE = True
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
REQUIRES = {"webdav4": "webdav4"}
|
||||
@@ -37,14 +34,35 @@ class WebDAVFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _strip_protocol(cls, path: str) -> str:
|
||||
from fsspec.utils import infer_storage_options
|
||||
|
||||
return infer_storage_options(path)["path"].lstrip("/")
|
||||
|
||||
def unstrip_protocol(self, path: str) -> str:
|
||||
return self.fs_args["base_url"] + path
|
||||
|
||||
@staticmethod
|
||||
def _get_kwargs_from_urls(urlpath):
|
||||
path_info = WebDAVURLInfo(urlpath)
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
|
||||
parsed = urlparse(urlpath)
|
||||
scheme = parsed.scheme.replace("webdav", "http")
|
||||
return {
|
||||
"prefix": path_info.path.rstrip("/"),
|
||||
"host": path_info.replace(path="").url,
|
||||
"url": path_info.url.rstrip("/"),
|
||||
"user": path_info.user,
|
||||
"prefix": parsed.path.rstrip("/"),
|
||||
"host": urlunparse((scheme, parsed.netloc, "", None, None, None)),
|
||||
"url": urlunparse(
|
||||
(
|
||||
scheme,
|
||||
parsed.netloc,
|
||||
parsed.path.rstrip("/"),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
),
|
||||
"user": parsed.username,
|
||||
}
|
||||
|
||||
def _prepare_credentials(self, **config):
|
||||
@@ -71,20 +89,11 @@ class WebDAVFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
|
||||
return WebdavFileSystem(**self.fs_args)
|
||||
|
||||
def upload_fobj(self, fobj, to_info, **kwargs):
|
||||
rpath = self.translate_path_info(to_info)
|
||||
size = kwargs.get("size")
|
||||
# using upload_fileobj to directly upload fileobj rather than buffering
|
||||
# and using overwrite=True to avoid check for an extra exists call,
|
||||
# as caller should ensure that the file does not exist beforehand.
|
||||
return self.fs.upload_fileobj(fobj, rpath, overwrite=True, size=size)
|
||||
|
||||
@lru_cache(512)
|
||||
def translate_path_info(self, path):
|
||||
if isinstance(path, self.PATH_CLS):
|
||||
return path.path[len(self.prefix) :].lstrip("/")
|
||||
return path
|
||||
|
||||
_with_bucket = translate_path_info # type: ignore
|
||||
return self.fs.upload_fileobj(fobj, to_info, overwrite=True, size=size)
|
||||
|
||||
|
||||
class WebDAVSFileSystem(WebDAVFileSystem): # pylint:disable=abstract-method
|
||||
|
||||
+12
-8
@@ -2,7 +2,6 @@ import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
# pylint:disable=abstract-method
|
||||
@@ -11,14 +10,20 @@ from .fsspec_wrapper import CallbackMixin, FSSpecWrapper
|
||||
|
||||
class WebHDFSFileSystem(CallbackMixin, FSSpecWrapper):
|
||||
scheme = Schemes.WEBHDFS
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"fsspec": "fsspec"}
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
|
||||
def _with_bucket(self, path):
|
||||
if isinstance(path, self.PATH_CLS):
|
||||
return f"/{path.path.rstrip('/')}"
|
||||
return path
|
||||
@classmethod
|
||||
def _strip_protocol(cls, path: str) -> str:
|
||||
from fsspec.utils import infer_storage_options
|
||||
|
||||
return infer_storage_options(path)["path"]
|
||||
|
||||
def unstrip_protocol(self, path: str) -> str:
|
||||
host = self.fs_args["host"]
|
||||
port = self.fs_args["port"]
|
||||
path = path.lstrip("/")
|
||||
return f"webhdfs://{host}:{port}/{path}"
|
||||
|
||||
@staticmethod
|
||||
def _get_kwargs_from_urls(urlpath):
|
||||
@@ -46,7 +51,6 @@ class WebHDFSFileSystem(CallbackMixin, FSSpecWrapper):
|
||||
fs.session.verify = self._ssl_verify
|
||||
return fs
|
||||
|
||||
def checksum(self, path_info):
|
||||
path = self._with_bucket(path_info)
|
||||
def checksum(self, path):
|
||||
ukey = self.fs.ukey(path)
|
||||
return ukey["bytes"]
|
||||
|
||||
+9
-11
@@ -8,7 +8,7 @@ from pathspec.patterns import GitWildMatchPattern
|
||||
from pathspec.util import normalize_file
|
||||
|
||||
from dvc.fs.base import BaseFileSystem
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.fs.local import localfs
|
||||
from dvc.pathspec_math import PatternInfo, merge_patterns
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.types import AnyPath, List, Optional
|
||||
@@ -267,25 +267,25 @@ class DvcIgnoreFilter:
|
||||
dirs, files = ignore_pattern(abs_root, dirs, files)
|
||||
return dirs, files
|
||||
|
||||
def walk(self, fs: BaseFileSystem, path_info: AnyPath, **kwargs):
|
||||
def walk(self, fs: BaseFileSystem, path: AnyPath, **kwargs):
|
||||
ignore_subrepos = kwargs.pop("ignore_subrepos", True)
|
||||
if fs.scheme == Schemes.LOCAL:
|
||||
for root, dirs, files in fs.walk(path_info, **kwargs):
|
||||
for root, dirs, files in fs.walk(path, **kwargs):
|
||||
dirs[:], files[:] = self(
|
||||
root, dirs, files, ignore_subrepos=ignore_subrepos
|
||||
)
|
||||
yield root, dirs, files
|
||||
else:
|
||||
yield from fs.walk(path_info, **kwargs)
|
||||
yield from fs.walk(path, **kwargs)
|
||||
|
||||
def walk_files(self, fs: BaseFileSystem, path_info: AnyPath, **kwargs):
|
||||
def find(self, fs: BaseFileSystem, path: AnyPath, **kwargs):
|
||||
if fs.scheme == Schemes.LOCAL:
|
||||
for root, _, files in self.walk(fs, path_info, **kwargs):
|
||||
for root, _, files in self.walk(fs, path, **kwargs):
|
||||
for file in files:
|
||||
# NOTE: os.path.join is ~5.5 times slower
|
||||
yield PathInfo(f"{root}{os.sep}{file}")
|
||||
yield f"{root}{os.sep}{file}"
|
||||
else:
|
||||
yield from fs.walk_files(path_info)
|
||||
yield from fs.find(path)
|
||||
|
||||
def _get_trie_pattern(
|
||||
self, dirname, dnames: Optional["List"] = None, ignore_subrepos=True
|
||||
@@ -307,7 +307,7 @@ class DvcIgnoreFilter:
|
||||
dirs = list(
|
||||
takewhile(
|
||||
lambda path: path != prefix,
|
||||
(parent.fspath for parent in PathInfo(dirname).parents),
|
||||
(parent for parent in localfs.path.parents(dirname)),
|
||||
)
|
||||
)
|
||||
dirs.reverse()
|
||||
@@ -343,8 +343,6 @@ class DvcIgnoreFilter:
|
||||
return self._is_ignored(path, False)
|
||||
|
||||
def _outside_repo(self, path):
|
||||
path = PathInfo(path)
|
||||
|
||||
# paths outside of the repo should be ignored
|
||||
path = relpath(path, self.root_dir)
|
||||
if path.startswith("..") or (
|
||||
|
||||
+1
-1
@@ -75,7 +75,7 @@ def _get_caches(cache):
|
||||
|
||||
def _get_remotes(config):
|
||||
schemes = (
|
||||
get_fs_cls(get_fs_config(config, name=remote)).scheme
|
||||
get_fs_cls(get_fs_config(None, config, name=remote)).scheme
|
||||
for remote in config["remote"]
|
||||
)
|
||||
|
||||
|
||||
+50
-50
@@ -23,36 +23,36 @@ from dvc.types import Optional
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _remove(path_info, fs, in_cache, force=False):
|
||||
if not fs.exists(path_info):
|
||||
def _remove(fs_path, fs, in_cache, force=False):
|
||||
if not fs.exists(fs_path):
|
||||
return
|
||||
|
||||
if force:
|
||||
fs.remove(path_info)
|
||||
fs.remove(fs_path)
|
||||
return
|
||||
|
||||
if not in_cache:
|
||||
msg = (
|
||||
f"file/directory '{path_info}' is going to be removed. "
|
||||
f"file/directory '{fs_path}' is going to be removed. "
|
||||
"Are you sure you want to proceed?"
|
||||
)
|
||||
|
||||
if not prompt.confirm(msg):
|
||||
raise ConfirmRemoveError(str(path_info))
|
||||
raise ConfirmRemoveError(fs_path)
|
||||
|
||||
fs.remove(path_info)
|
||||
fs.remove(fs_path)
|
||||
|
||||
|
||||
def _verify_link(cache, path_info, link_type):
|
||||
if link_type == "hardlink" and cache.fs.getsize(path_info) == 0:
|
||||
def _verify_link(cache, fs_path, link_type):
|
||||
if link_type == "hardlink" and cache.fs.getsize(fs_path) == 0:
|
||||
return
|
||||
|
||||
if cache.cache_type_confirmed:
|
||||
return
|
||||
|
||||
is_link = getattr(cache.fs, f"is_{link_type}", None)
|
||||
if is_link and not is_link(path_info):
|
||||
cache.fs.remove(path_info)
|
||||
if is_link and not is_link(fs_path):
|
||||
cache.fs.remove(fs_path)
|
||||
raise DvcException(f"failed to verify {link_type}")
|
||||
|
||||
cache.cache_type_confirmed = True
|
||||
@@ -90,14 +90,14 @@ def _try_links(cache, from_info, to_info, link_types):
|
||||
|
||||
|
||||
def _link(cache, from_info, to_info):
|
||||
cache.makedirs(to_info.parent)
|
||||
cache.makedirs(cache.fs.path.parent(to_info))
|
||||
try:
|
||||
_try_links(cache, from_info, to_info, cache.cache_types)
|
||||
except FileNotFoundError as exc:
|
||||
raise CheckoutError([str(to_info)]) from exc
|
||||
|
||||
|
||||
def _confirm_cache_type(cache, path_info):
|
||||
def _confirm_cache_type(cache, fs_path):
|
||||
"""Checks whether cache uses copies."""
|
||||
if cache.cache_type_confirmed:
|
||||
return
|
||||
@@ -105,10 +105,12 @@ def _confirm_cache_type(cache, path_info):
|
||||
if set(cache.cache_types) <= {"copy"}:
|
||||
return
|
||||
|
||||
workspace_file = path_info.with_name("." + uuid())
|
||||
test_cache_file = cache.path_info / ".cache_type_test_file"
|
||||
workspace_file = cache.fs.path.with_name(fs_path, "." + uuid())
|
||||
test_cache_file = cache.fs.path.join(
|
||||
cache.fs_path, ".cache_type_test_file"
|
||||
)
|
||||
if not cache.fs.exists(test_cache_file):
|
||||
cache.makedirs(test_cache_file.parent)
|
||||
cache.makedirs(cache.fs.path.parent(test_cache_file))
|
||||
with cache.fs.open(test_cache_file, "wb") as fobj:
|
||||
fobj.write(bytes(1))
|
||||
try:
|
||||
@@ -120,9 +122,9 @@ def _confirm_cache_type(cache, path_info):
|
||||
cache.cache_type_confirmed = True
|
||||
|
||||
|
||||
def _relink(cache, cache_info, fs, path_info, in_cache, force):
|
||||
_remove(path_info, fs, in_cache, force=force)
|
||||
_link(cache, cache_info, path_info)
|
||||
def _relink(cache, cache_info, fs, path, in_cache, force):
|
||||
_remove(path, fs, in_cache, force=force)
|
||||
_link(cache, cache_info, path)
|
||||
# NOTE: Depending on a file system (e.g. on NTFS), `_remove` might reset
|
||||
# read-only permissions in order to delete a hardlink to protected object,
|
||||
# which will also reset it for the object itself, making it unprotected,
|
||||
@@ -131,7 +133,7 @@ def _relink(cache, cache_info, fs, path_info, in_cache, force):
|
||||
|
||||
|
||||
def _checkout_file(
|
||||
path_info,
|
||||
fs_path,
|
||||
fs,
|
||||
change,
|
||||
cache,
|
||||
@@ -142,20 +144,19 @@ def _checkout_file(
|
||||
):
|
||||
"""The file is changed we need to checkout a new copy"""
|
||||
modified = False
|
||||
_confirm_cache_type(cache, fs_path)
|
||||
|
||||
_confirm_cache_type(cache, path_info)
|
||||
|
||||
cache_info = cache.hash_to_path_info(change.new.oid.value)
|
||||
cache_fs_path = cache.hash_to_path(change.new.oid.value)
|
||||
if change.old.oid:
|
||||
if relink:
|
||||
if fs.iscopy(path_info) and cache.cache_types[0] == "copy":
|
||||
cache.unprotect(path_info)
|
||||
if fs.iscopy(fs_path) and cache.cache_types[0] == "copy":
|
||||
cache.unprotect(fs_path)
|
||||
else:
|
||||
_relink(
|
||||
cache,
|
||||
cache_info,
|
||||
cache_fs_path,
|
||||
fs,
|
||||
path_info,
|
||||
fs_path,
|
||||
change.old.in_cache,
|
||||
force=force,
|
||||
)
|
||||
@@ -163,27 +164,27 @@ def _checkout_file(
|
||||
modified = True
|
||||
_relink(
|
||||
cache,
|
||||
cache_info,
|
||||
cache_fs_path,
|
||||
fs,
|
||||
path_info,
|
||||
fs_path,
|
||||
change.old.in_cache,
|
||||
force=force,
|
||||
)
|
||||
else:
|
||||
_link(cache, cache_info, path_info)
|
||||
_link(cache, cache_fs_path, fs_path)
|
||||
modified = True
|
||||
|
||||
if state:
|
||||
state.save(path_info, fs, change.new.oid)
|
||||
state.save(fs_path, fs, change.new.oid)
|
||||
|
||||
if progress_callback:
|
||||
progress_callback(str(path_info))
|
||||
progress_callback(fs_path)
|
||||
|
||||
return modified
|
||||
|
||||
|
||||
def _diff(
|
||||
path_info,
|
||||
fs_path,
|
||||
fs,
|
||||
obj,
|
||||
cache,
|
||||
@@ -194,7 +195,7 @@ def _diff(
|
||||
try:
|
||||
_, _, old = stage(
|
||||
cache,
|
||||
path_info,
|
||||
fs_path,
|
||||
fs,
|
||||
obj.hash_info.name if obj else cache.fs.PARAM_CHECKSUM,
|
||||
dry_run=True,
|
||||
@@ -213,7 +214,7 @@ def _diff(
|
||||
|
||||
def _checkout(
|
||||
diff,
|
||||
path_info,
|
||||
fs_path,
|
||||
fs,
|
||||
cache,
|
||||
force=False,
|
||||
@@ -226,18 +227,18 @@ def _checkout(
|
||||
|
||||
for change in diff.deleted:
|
||||
entry_path = (
|
||||
path_info.joinpath(*change.old.key)
|
||||
fs.path.join(fs_path, *change.old.key)
|
||||
if change.old.key != ROOT
|
||||
else path_info
|
||||
else fs_path
|
||||
)
|
||||
_remove(entry_path, fs, change.old.in_cache, force=force)
|
||||
|
||||
failed = []
|
||||
for change in chain(diff.added, diff.modified):
|
||||
entry_path = (
|
||||
path_info.joinpath(*change.new.key)
|
||||
fs.path.join(fs_path, *change.new.key)
|
||||
if change.new.key != ROOT
|
||||
else path_info
|
||||
else fs_path
|
||||
)
|
||||
if change.new.oid.isdir:
|
||||
fs.makedirs(entry_path)
|
||||
@@ -262,7 +263,7 @@ def _checkout(
|
||||
|
||||
|
||||
def checkout(
|
||||
path_info,
|
||||
fs_path,
|
||||
fs,
|
||||
obj,
|
||||
cache,
|
||||
@@ -273,12 +274,11 @@ def checkout(
|
||||
dvcignore: Optional[DvcIgnoreFilter] = None,
|
||||
state=None,
|
||||
):
|
||||
|
||||
if path_info.scheme not in ["local", cache.fs.scheme]:
|
||||
raise NotImplementedError
|
||||
# if scheme(fs_path) not in ["local", cache.fs.scheme]:
|
||||
# raise NotImplementedError
|
||||
|
||||
diff = _diff(
|
||||
path_info,
|
||||
fs_path,
|
||||
fs,
|
||||
obj,
|
||||
cache,
|
||||
@@ -291,16 +291,16 @@ def checkout(
|
||||
if not quiet:
|
||||
logger.warning(
|
||||
"No file hash info found for '%s'. It won't be created.",
|
||||
path_info,
|
||||
fs_path,
|
||||
)
|
||||
failed.append(str(path_info))
|
||||
failed.append(fs_path)
|
||||
elif not diff:
|
||||
logger.trace("Data '%s' didn't change.", path_info) # type: ignore
|
||||
logger.trace("Data '%s' didn't change.", fs_path) # type: ignore
|
||||
|
||||
try:
|
||||
_checkout(
|
||||
diff,
|
||||
path_info,
|
||||
fs_path,
|
||||
fs,
|
||||
cache,
|
||||
force=force,
|
||||
@@ -312,13 +312,13 @@ def checkout(
|
||||
failed.extend(exc.target_infos)
|
||||
|
||||
if diff and state:
|
||||
state.save_link(path_info, fs)
|
||||
state.save_link(fs_path, fs)
|
||||
if not failed:
|
||||
state.save(path_info, fs, obj.hash_info)
|
||||
state.save(fs_path, fs, obj.hash_info)
|
||||
|
||||
if failed or not diff:
|
||||
if progress_callback and obj:
|
||||
progress_callback(str(path_info), len(obj))
|
||||
progress_callback(fs_path, len(obj))
|
||||
if failed:
|
||||
raise CheckoutError(failed)
|
||||
return
|
||||
|
||||
@@ -6,22 +6,22 @@ if TYPE_CHECKING:
|
||||
from .index import ObjectDBIndexBase
|
||||
|
||||
|
||||
def get_odb(fs, path_info, **config):
|
||||
def get_odb(fs, fs_path, **config):
|
||||
from .base import ObjectDB
|
||||
from .gdrive import GDriveObjectDB
|
||||
from .local import LocalObjectDB
|
||||
from .oss import OSSObjectDB
|
||||
|
||||
if fs.scheme == Schemes.LOCAL:
|
||||
return LocalObjectDB(fs, path_info, **config)
|
||||
return LocalObjectDB(fs, fs_path, **config)
|
||||
|
||||
if fs.scheme == Schemes.GDRIVE:
|
||||
return GDriveObjectDB(fs, path_info, **config)
|
||||
return GDriveObjectDB(fs, fs_path, **config)
|
||||
|
||||
if fs.scheme == Schemes.OSS:
|
||||
return OSSObjectDB(fs, path_info, **config)
|
||||
return OSSObjectDB(fs, fs_path, **config)
|
||||
|
||||
return ObjectDB(fs, path_info, **config)
|
||||
return ObjectDB(fs, fs_path, **config)
|
||||
|
||||
|
||||
def _get_odb(repo, settings):
|
||||
@@ -30,9 +30,9 @@ def _get_odb(repo, settings):
|
||||
if not settings:
|
||||
return None
|
||||
|
||||
cls, config, path_info = get_cloud_fs(repo, **settings)
|
||||
cls, config, fs_path = get_cloud_fs(repo, **settings)
|
||||
config["tmp_dir"] = repo.tmp_dir
|
||||
return get_odb(cls(**config), path_info, state=repo.state, **config)
|
||||
return get_odb(cls(**config), fs_path, state=repo.state, **config)
|
||||
|
||||
|
||||
def get_index(odb) -> "ObjectDBIndexBase":
|
||||
@@ -43,7 +43,9 @@ def get_index(odb) -> "ObjectDBIndexBase":
|
||||
cls = ObjectDBIndex if odb.tmp_dir else ObjectDBIndexNoop
|
||||
return cls(
|
||||
odb.tmp_dir,
|
||||
hashlib.sha256(odb.path_info.url.encode("utf-8")).hexdigest(),
|
||||
hashlib.sha256(
|
||||
odb.fs.unstrip_protocol(odb.fs_path).encode("utf-8")
|
||||
).hexdigest(),
|
||||
odb.fs.CHECKSUM_DIR_SUFFIX,
|
||||
)
|
||||
|
||||
|
||||
+44
-46
@@ -12,7 +12,7 @@ from dvc.progress import Tqdm
|
||||
if TYPE_CHECKING:
|
||||
from dvc.fs.base import BaseFileSystem
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.types import AnyPath, DvcPath
|
||||
from dvc.types import AnyPath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -23,11 +23,11 @@ class ObjectDB:
|
||||
DEFAULT_CACHE_TYPES = ["copy"]
|
||||
CACHE_MODE: Optional[int] = None
|
||||
|
||||
def __init__(self, fs: "BaseFileSystem", path_info: "AnyPath", **config):
|
||||
def __init__(self, fs: "BaseFileSystem", path: str, **config):
|
||||
from dvc.state import StateNoop
|
||||
|
||||
self.fs = fs
|
||||
self.path_info = path_info
|
||||
self.fs_path = path
|
||||
self.state = config.get("state", StateNoop())
|
||||
self.verify = config.get("verify", self.DEFAULT_VERIFY)
|
||||
self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES)
|
||||
@@ -50,26 +50,25 @@ class ObjectDB:
|
||||
def __eq__(self, other):
|
||||
return (
|
||||
self.fs == other.fs
|
||||
and self.path_info == other.path_info
|
||||
and self.fs_path == other.fs_path
|
||||
and self.read_only == other.read_only
|
||||
)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.fs.scheme, self.path_info))
|
||||
return hash((self.fs.scheme, self.fs_path))
|
||||
|
||||
def exists(self, hash_info: "HashInfo"):
|
||||
return self.fs.exists(self.hash_to_path_info(hash_info.value))
|
||||
return self.fs.exists(self.hash_to_path(hash_info.value))
|
||||
|
||||
def move(self, from_info, to_info):
|
||||
self.fs.move(from_info, to_info)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
self.fs.makedirs(path_info)
|
||||
def makedirs(self, fs_path):
|
||||
self.fs.makedirs(fs_path)
|
||||
|
||||
def get(self, hash_info: "HashInfo"):
|
||||
"""get raw object"""
|
||||
return HashFile(
|
||||
# Prefer string path over PathInfo when possible due to performance
|
||||
self.hash_to_path(hash_info.value),
|
||||
self.fs,
|
||||
hash_info,
|
||||
@@ -79,13 +78,13 @@ class ObjectDB:
|
||||
self,
|
||||
from_fs: "BaseFileSystem",
|
||||
from_info: "AnyPath",
|
||||
to_info: "DvcPath",
|
||||
to_info: "AnyPath",
|
||||
_hash_info: "HashInfo",
|
||||
hardlink: bool = False,
|
||||
):
|
||||
from dvc import fs
|
||||
|
||||
self.makedirs(to_info.parent)
|
||||
self.makedirs(self.fs.path.parent(to_info))
|
||||
return fs.utils.transfer(
|
||||
from_fs,
|
||||
from_info,
|
||||
@@ -96,7 +95,7 @@ class ObjectDB:
|
||||
|
||||
def add(
|
||||
self,
|
||||
path_info: "AnyPath",
|
||||
fs_path: "AnyPath",
|
||||
fs: "BaseFileSystem",
|
||||
hash_info: "HashInfo",
|
||||
hardlink: bool = False,
|
||||
@@ -113,35 +112,32 @@ class ObjectDB:
|
||||
except (ObjectFormatError, FileNotFoundError):
|
||||
pass
|
||||
|
||||
cache_info = self.hash_to_path_info(hash_info.value)
|
||||
self._add_file(fs, path_info, cache_info, hash_info, hardlink=hardlink)
|
||||
cache_fs_path = self.hash_to_path(hash_info.value)
|
||||
self._add_file(
|
||||
fs, fs_path, cache_fs_path, hash_info, hardlink=hardlink
|
||||
)
|
||||
|
||||
try:
|
||||
if verify:
|
||||
self.check(hash_info, check_hash=True)
|
||||
self.protect(cache_info)
|
||||
self.state.save(cache_info, self.fs, hash_info)
|
||||
self.protect(cache_fs_path)
|
||||
self.state.save(cache_fs_path, self.fs, hash_info)
|
||||
except (ObjectFormatError, FileNotFoundError):
|
||||
pass
|
||||
|
||||
def hash_to_path_info(self, hash_) -> "DvcPath":
|
||||
return self.path_info / hash_[0:2] / hash_[2:]
|
||||
|
||||
# Override to return path as a string instead of PathInfo for clouds
|
||||
# which support string paths (see local)
|
||||
def hash_to_path(self, hash_):
|
||||
return self.hash_to_path_info(hash_)
|
||||
return self.fs.path.join(self.fs_path, hash_[0:2], hash_[2:])
|
||||
|
||||
def protect(self, path_info): # pylint: disable=unused-argument
|
||||
def protect(self, fs_path): # pylint: disable=unused-argument
|
||||
pass
|
||||
|
||||
def is_protected(self, path_info): # pylint: disable=unused-argument
|
||||
def is_protected(self, fs_path): # pylint: disable=unused-argument
|
||||
return False
|
||||
|
||||
def unprotect(self, path_info): # pylint: disable=unused-argument
|
||||
def unprotect(self, fs_path): # pylint: disable=unused-argument
|
||||
pass
|
||||
|
||||
def set_exec(self, path_info): # pylint: disable=unused-argument
|
||||
def set_exec(self, fs_path): # pylint: disable=unused-argument
|
||||
pass
|
||||
|
||||
def check(
|
||||
@@ -164,45 +160,47 @@ class ObjectDB:
|
||||
"""
|
||||
|
||||
obj = self.get(hash_info)
|
||||
if self.is_protected(obj.path_info):
|
||||
if self.is_protected(obj.fs_path):
|
||||
logger.trace( # type: ignore[attr-defined]
|
||||
"Assuming '%s' is unchanged since it is read-only",
|
||||
obj.path_info,
|
||||
obj.fs_path,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
obj.check(self, check_hash=check_hash)
|
||||
except ObjectFormatError:
|
||||
logger.warning("corrupted cache file '%s'.", obj.path_info)
|
||||
logger.warning("corrupted cache file '%s'.", obj.fs_path)
|
||||
with suppress(FileNotFoundError):
|
||||
self.fs.remove(obj.path_info)
|
||||
self.fs.remove(obj.fs_path)
|
||||
raise
|
||||
|
||||
if check_hash:
|
||||
# making cache file read-only so we don't need to check it
|
||||
# next time
|
||||
self.protect(obj.path_info)
|
||||
self.protect(obj.fs_path)
|
||||
|
||||
def _list_paths(self, prefix=None, progress_callback=None):
|
||||
if prefix:
|
||||
if len(prefix) > 2:
|
||||
path_info = self.path_info / prefix[:2] / prefix[2:]
|
||||
fs_path = self.fs.path.join(
|
||||
self.fs_path, prefix[:2], prefix[2:]
|
||||
)
|
||||
else:
|
||||
path_info = self.path_info / prefix[:2]
|
||||
fs_path = self.fs.path.join(self.fs_path, prefix[:2])
|
||||
prefix = True
|
||||
else:
|
||||
path_info = self.path_info
|
||||
fs_path = self.fs_path
|
||||
prefix = False
|
||||
if progress_callback:
|
||||
for file_info in self.fs.walk_files(path_info, prefix=prefix):
|
||||
for file_info in self.fs.find(fs_path, prefix=prefix):
|
||||
progress_callback()
|
||||
yield file_info.path
|
||||
yield file_info
|
||||
else:
|
||||
yield from self.fs.walk_files(path_info, prefix=prefix)
|
||||
yield from self.fs.find(fs_path, prefix=prefix)
|
||||
|
||||
def _path_to_hash(self, path):
|
||||
parts = self.fs.PATH_CLS(path).parts[-2:]
|
||||
parts = self.fs.path.parts(path)[-2:]
|
||||
|
||||
if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
|
||||
raise ValueError(f"Bad cache file path '{path}'")
|
||||
@@ -378,18 +376,18 @@ class ObjectDB:
|
||||
removed = False
|
||||
# hashes must be sorted to ensure we always remove .dir files first
|
||||
for hash_ in sorted(
|
||||
self.all(jobs, str(self.path_info)),
|
||||
self.all(jobs, self.fs_path),
|
||||
key=self.fs.is_dir_hash,
|
||||
reverse=True,
|
||||
):
|
||||
if hash_ in used_hashes:
|
||||
continue
|
||||
path_info = self.hash_to_path_info(hash_)
|
||||
fs_path = self.hash_to_path(hash_)
|
||||
if self.fs.is_dir_hash(hash_):
|
||||
# backward compatibility
|
||||
# pylint: disable=protected-access
|
||||
self._remove_unpacked_dir(hash_)
|
||||
self.fs.remove(path_info)
|
||||
self.fs.remove(fs_path)
|
||||
removed = True
|
||||
|
||||
return removed
|
||||
@@ -406,16 +404,16 @@ class ObjectDB:
|
||||
unit="file",
|
||||
) as pbar:
|
||||
|
||||
def exists_with_progress(path_info):
|
||||
ret = self.fs.exists(path_info)
|
||||
pbar.update_msg(str(path_info))
|
||||
def exists_with_progress(fs_path):
|
||||
ret = self.fs.exists(fs_path)
|
||||
pbar.update_msg(fs_path)
|
||||
return ret
|
||||
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=jobs or self.fs.jobs
|
||||
) as executor:
|
||||
path_infos = map(self.hash_to_path_info, hashes)
|
||||
in_remote = executor.map(exists_with_progress, path_infos)
|
||||
fs_paths = map(self.hash_to_path, hashes)
|
||||
in_remote = executor.map(exists_with_progress, fs_paths)
|
||||
ret = list(itertools.compress(hashes, in_remote))
|
||||
return ret
|
||||
|
||||
|
||||
+36
-33
@@ -7,7 +7,6 @@ from shortuuid import uuid
|
||||
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.objects.errors import ObjectFormatError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import copyfile, remove, umask, walk_files
|
||||
@@ -22,9 +21,9 @@ class LocalObjectDB(ObjectDB):
|
||||
CACHE_MODE = 0o444
|
||||
UNPACKED_DIR_SUFFIX = ".unpacked"
|
||||
|
||||
def __init__(self, fs, path_info, **config):
|
||||
super().__init__(fs, path_info, **config)
|
||||
self.cache_dir = path_info.fspath
|
||||
def __init__(self, fs, fs_path, **config):
|
||||
super().__init__(fs, fs_path, **config)
|
||||
self.cache_dir = fs_path
|
||||
|
||||
shared = config.get("shared")
|
||||
if shared:
|
||||
@@ -36,11 +35,11 @@ class LocalObjectDB(ObjectDB):
|
||||
|
||||
@property
|
||||
def cache_dir(self):
|
||||
return self.path_info.fspath if self.path_info else None
|
||||
return self.fs_path if self.fs_path else None
|
||||
|
||||
@cache_dir.setter
|
||||
def cache_dir(self, value):
|
||||
self.path_info = PathInfo(value) if value else None
|
||||
self.fs_path = value
|
||||
|
||||
@cached_property
|
||||
def cache_path(self):
|
||||
@@ -50,10 +49,10 @@ class LocalObjectDB(ObjectDB):
|
||||
super().move(from_info, to_info)
|
||||
os.chmod(to_info, self._file_mode)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
def makedirs(self, fs_path):
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
makedirs(path_info, exist_ok=True, mode=self._dir_mode)
|
||||
makedirs(fs_path, exist_ok=True, mode=self._dir_mode)
|
||||
|
||||
def hash_to_path(self, hash_):
|
||||
# NOTE: `self.cache_path` is already normalized so we can simply use
|
||||
@@ -81,26 +80,30 @@ class LocalObjectDB(ObjectDB):
|
||||
return ret
|
||||
|
||||
def _list_paths(self, prefix=None, progress_callback=None):
|
||||
assert self.path_info is not None
|
||||
assert self.fs_path is not None
|
||||
if prefix:
|
||||
path_info = self.path_info / prefix[:2]
|
||||
if not self.fs.exists(path_info):
|
||||
fs_path = self.fs.path.join(self.fs_path, prefix[:2])
|
||||
if not self.fs.exists(fs_path):
|
||||
return
|
||||
else:
|
||||
path_info = self.path_info
|
||||
fs_path = self.fs_path
|
||||
|
||||
# NOTE: use utils.fs walk_files since fs.walk_files will not follow
|
||||
# symlinks
|
||||
if progress_callback:
|
||||
for path in walk_files(path_info):
|
||||
for path in walk_files(fs_path):
|
||||
progress_callback()
|
||||
yield path
|
||||
else:
|
||||
yield from walk_files(path_info)
|
||||
yield from walk_files(fs_path)
|
||||
|
||||
def _remove_unpacked_dir(self, hash_):
|
||||
info = self.hash_to_path_info(hash_)
|
||||
path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
self.fs.remove(path_info)
|
||||
hash_fs_path = self.hash_to_path(hash_)
|
||||
fs_path = self.fs.path.with_name(
|
||||
hash_fs_path,
|
||||
self.fs.path.name(hash_fs_path) + self.UNPACKED_DIR_SUFFIX,
|
||||
)
|
||||
self.fs.remove(fs_path)
|
||||
|
||||
def _unprotect_file(self, path):
|
||||
if self.fs.is_symlink(path) or self.fs.is_hardlink(path):
|
||||
@@ -126,47 +129,47 @@ class LocalObjectDB(ObjectDB):
|
||||
os.chmod(path, self._file_mode)
|
||||
|
||||
def _unprotect_dir(self, path):
|
||||
for fname in self.fs.walk_files(path):
|
||||
for fname in self.fs.find(path):
|
||||
self._unprotect_file(fname)
|
||||
|
||||
def unprotect(self, path_info):
|
||||
if not os.path.exists(path_info):
|
||||
def unprotect(self, fs_path):
|
||||
if not os.path.exists(fs_path):
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
raise DvcException(
|
||||
f"can't unprotect non-existing data '{path_info}'"
|
||||
f"can't unprotect non-existing data '{fs_path}'"
|
||||
)
|
||||
|
||||
if os.path.isdir(path_info):
|
||||
self._unprotect_dir(path_info)
|
||||
if os.path.isdir(fs_path):
|
||||
self._unprotect_dir(fs_path)
|
||||
else:
|
||||
self._unprotect_file(path_info)
|
||||
self._unprotect_file(fs_path)
|
||||
|
||||
def protect(self, path_info):
|
||||
def protect(self, fs_path):
|
||||
try:
|
||||
os.chmod(path_info, self.CACHE_MODE)
|
||||
os.chmod(fs_path, self.CACHE_MODE)
|
||||
except OSError:
|
||||
# NOTE: not being able to protect cache file is not fatal, it
|
||||
# might happen on funky filesystems (e.g. Samba, see #5255),
|
||||
# read-only filesystems or in a shared cache scenario.
|
||||
logger.trace("failed to protect '%s'", path_info, exc_info=True)
|
||||
logger.trace("failed to protect '%s'", fs_path, exc_info=True)
|
||||
|
||||
def is_protected(self, path_info):
|
||||
def is_protected(self, fs_path):
|
||||
try:
|
||||
mode = os.stat(path_info).st_mode
|
||||
mode = os.stat(fs_path).st_mode
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
return stat.S_IMODE(mode) == self.CACHE_MODE
|
||||
|
||||
def set_exec(self, path_info):
|
||||
mode = os.stat(path_info).st_mode | stat.S_IEXEC
|
||||
def set_exec(self, fs_path):
|
||||
mode = os.stat(fs_path).st_mode | stat.S_IEXEC
|
||||
try:
|
||||
os.chmod(path_info, mode)
|
||||
os.chmod(fs_path, mode)
|
||||
except OSError:
|
||||
logger.trace(
|
||||
"failed to chmod '%s' '%s'",
|
||||
oct(mode),
|
||||
path_info,
|
||||
fs_path,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
@@ -12,7 +12,7 @@ from .base import ObjectDB
|
||||
if TYPE_CHECKING:
|
||||
from dvc.fs.base import BaseFileSystem
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.types import AnyPath, DvcPath
|
||||
from dvc.types import AnyPath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -36,9 +36,9 @@ class ReferenceObjectDB(ObjectDB):
|
||||
return self._obj_cache[hash_info]
|
||||
except KeyError:
|
||||
pass
|
||||
path_info = self.hash_to_path(hash_info.value)
|
||||
fs_path = self.hash_to_path(hash_info.value)
|
||||
try:
|
||||
with self.fs.open(path_info, "rb") as fobj:
|
||||
with self.fs.open(fs_path, "rb") as fobj:
|
||||
ref_file = ReferenceHashFile.from_bytes(
|
||||
fobj.read(), fs_cache=self._fs_cache
|
||||
)
|
||||
@@ -47,7 +47,7 @@ class ReferenceObjectDB(ObjectDB):
|
||||
try:
|
||||
ref_file.check(self, check_hash=False)
|
||||
except ObjectFormatError:
|
||||
self.fs.remove(path_info)
|
||||
self.fs.remove(fs_path)
|
||||
raise
|
||||
self._obj_cache[hash_info] = ref_file
|
||||
return ref_file
|
||||
@@ -56,11 +56,11 @@ class ReferenceObjectDB(ObjectDB):
|
||||
self,
|
||||
from_fs: "BaseFileSystem",
|
||||
from_info: "AnyPath",
|
||||
to_info: "DvcPath",
|
||||
to_info: "AnyPath",
|
||||
hash_info: "HashInfo",
|
||||
hardlink: bool = False,
|
||||
):
|
||||
self.makedirs(to_info.parent)
|
||||
self.makedirs(self.fs.path.parent(to_info))
|
||||
if hash_info.isdir:
|
||||
return super()._add_file(
|
||||
from_fs,
|
||||
|
||||
+8
-8
@@ -18,12 +18,12 @@ logger = logging.getLogger(__name__)
|
||||
class HashFile:
|
||||
def __init__(
|
||||
self,
|
||||
path_info: Optional["AnyPath"],
|
||||
fs_path: Optional["AnyPath"],
|
||||
fs: Optional["BaseFileSystem"],
|
||||
hash_info: "HashInfo",
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
self.path_info = path_info
|
||||
self.fs_path = fs_path
|
||||
self.fs = fs
|
||||
self.hash_info = hash_info
|
||||
self.name = name
|
||||
@@ -41,7 +41,7 @@ class HashFile:
|
||||
if not isinstance(other, HashFile):
|
||||
return False
|
||||
return (
|
||||
self.path_info == other.path_info
|
||||
self.fs_path == other.fs_path
|
||||
and self.fs == other.fs
|
||||
and self.hash_info == other.hash_info
|
||||
)
|
||||
@@ -50,7 +50,7 @@ class HashFile:
|
||||
return hash(
|
||||
(
|
||||
self.hash_info,
|
||||
self.path_info,
|
||||
self.fs_path,
|
||||
self.fs.scheme if self.fs else None,
|
||||
)
|
||||
)
|
||||
@@ -58,9 +58,9 @@ class HashFile:
|
||||
def check(self, odb: "ObjectDB", check_hash: bool = True):
|
||||
if not check_hash:
|
||||
assert self.fs
|
||||
if not self.fs.exists(self.path_info):
|
||||
if not self.fs.exists(self.fs_path):
|
||||
raise FileNotFoundError(
|
||||
errno.ENOENT, os.strerror(errno.ENOENT), self.path_info
|
||||
errno.ENOENT, os.strerror(errno.ENOENT), self.fs_path
|
||||
)
|
||||
else:
|
||||
return None
|
||||
@@ -71,12 +71,12 @@ class HashFile:
|
||||
from .stage import get_file_hash
|
||||
|
||||
_, actual = get_file_hash(
|
||||
self.path_info, self.fs, self.hash_info.name, odb.state
|
||||
self.fs_path, self.fs, self.hash_info.name, odb.state
|
||||
)
|
||||
|
||||
logger.trace(
|
||||
"cache '%s' expected '%s' actual '%s'",
|
||||
self.path_info,
|
||||
self.fs_path,
|
||||
self.hash_info,
|
||||
actual,
|
||||
)
|
||||
|
||||
+14
-15
@@ -25,23 +25,23 @@ class ReferenceHashFile(HashFile):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path_info: "AnyPath",
|
||||
fs_path: "AnyPath",
|
||||
fs: "BaseFileSystem",
|
||||
hash_info: "HashInfo",
|
||||
checksum: Optional[str] = None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(path_info, fs, hash_info, **kwargs)
|
||||
self.checksum = checksum or fs.checksum(path_info)
|
||||
super().__init__(fs_path, fs, hash_info, **kwargs)
|
||||
self.checksum = checksum or fs.checksum(fs_path)
|
||||
|
||||
def __str__(self):
|
||||
return f"ref object {self.hash_info} -> {self.path_info}"
|
||||
return f"ref object {self.hash_info} -> {self.fs_path}"
|
||||
|
||||
def check(self, odb: "ObjectDB", check_hash: bool = True):
|
||||
assert self.fs
|
||||
if not self.fs.exists(self.path_info):
|
||||
if not self.fs.exists(self.fs_path):
|
||||
raise FileNotFoundError(
|
||||
errno.ENOENT, os.strerror(errno.ENOENT), self.path_info
|
||||
errno.ENOENT, os.strerror(errno.ENOENT), self.fs_path
|
||||
)
|
||||
if self.checksum != self._get_checksum():
|
||||
raise ObjectFormatError(f"{self} is changed")
|
||||
@@ -50,23 +50,22 @@ class ReferenceHashFile(HashFile):
|
||||
|
||||
def _get_checksum(self) -> str:
|
||||
assert self.fs
|
||||
return self.fs.checksum(self.path_info)
|
||||
return self.fs.checksum(self.fs_path)
|
||||
|
||||
def to_bytes(self):
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
# NOTE: dumping reference FS's this way is insecure, as the
|
||||
# fully parsed remote FS config will include credentials
|
||||
#
|
||||
# ReferenceHashFiles should currently only be serialized in
|
||||
# memory and not to disk
|
||||
path_info = self.path_info
|
||||
fs_path = self.fs_path
|
||||
if isinstance(self.fs, RepoFileSystem):
|
||||
path_info = PathInfo(path_info).relative_to(self.fs.root_dir)
|
||||
fs_path = self.fs.path.relpath(fs_path, self.fs.root_dir)
|
||||
|
||||
dict_ = {
|
||||
self.PARAM_PATH: path_info,
|
||||
self.PARAM_PATH: fs_path,
|
||||
self.PARAM_HASH: self.hash_info,
|
||||
self.PARAM_CHECKSUM: self.checksum,
|
||||
self.PARAM_FS_CONFIG: self.config_tuple(self.fs),
|
||||
@@ -87,7 +86,7 @@ class ReferenceHashFile(HashFile):
|
||||
raise ObjectFormatError("ReferenceHashFile is corrupted") from exc
|
||||
|
||||
try:
|
||||
path_info = dict_[cls.PARAM_PATH]
|
||||
fs_path = dict_[cls.PARAM_PATH]
|
||||
hash_info = dict_[cls.PARAM_HASH]
|
||||
except KeyError as exc:
|
||||
raise ObjectFormatError("ReferenceHashFile is corrupted") from exc
|
||||
@@ -98,12 +97,12 @@ class ReferenceHashFile(HashFile):
|
||||
config = dict(config_pairs)
|
||||
if RepoFileSystem.PARAM_REPO_URL in config:
|
||||
fs = RepoFileSystem(**config)
|
||||
path_info = fs.root_dir / path_info
|
||||
fs_path = fs.path.join(fs.root_dir, fs_path)
|
||||
else:
|
||||
fs_cls = get_fs_cls(config, scheme=path_info.scheme)
|
||||
fs_cls = get_fs_cls(config, scheme=scheme)
|
||||
fs = fs_cls(**config)
|
||||
return ReferenceHashFile(
|
||||
path_info,
|
||||
fs_path,
|
||||
fs,
|
||||
hash_info,
|
||||
checksum=dict_.get(cls.PARAM_CHECKSUM),
|
||||
|
||||
+89
-64
@@ -1,10 +1,9 @@
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Dict, Tuple
|
||||
from typing import TYPE_CHECKING, Dict, Optional, Tuple
|
||||
|
||||
from dvc.exceptions import DvcIgnoreInCollectedDirError
|
||||
from dvc.hash_info import HashInfo
|
||||
@@ -18,7 +17,7 @@ from .meta import Meta
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.fs.base import BaseFileSystem
|
||||
from dvc.types import AnyPath, DvcPath
|
||||
from dvc.types import AnyPath
|
||||
|
||||
from .db.base import ObjectDB
|
||||
|
||||
@@ -28,31 +27,47 @@ logger = logging.getLogger(__name__)
|
||||
_STAGING_MEMFS_PATH = "dvc-staging"
|
||||
|
||||
|
||||
def _upload_file(path_info, fs, odb, upload_odb):
|
||||
def _upload_file(from_fs_path, fs, odb, upload_odb):
|
||||
from dvc.utils import tmp_fname
|
||||
from dvc.utils.stream import HashedStreamReader
|
||||
|
||||
tmp_info = upload_odb.path_info / tmp_fname()
|
||||
with fs.open(path_info, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream:
|
||||
fs_path = upload_odb.fs.path
|
||||
tmp_info = fs_path.join(upload_odb.fs_path, tmp_fname())
|
||||
with fs.open(from_fs_path, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream:
|
||||
stream = HashedStreamReader(stream)
|
||||
size = fs.getsize(path_info)
|
||||
upload_odb.fs.upload(stream, tmp_info, desc=path_info.name, total=size)
|
||||
size = fs.getsize(from_fs_path)
|
||||
upload_odb.fs.upload(
|
||||
stream, tmp_info, desc=fs_path.name(from_fs_path), total=size
|
||||
)
|
||||
|
||||
odb.add(tmp_info, upload_odb.fs, stream.hash_info)
|
||||
meta = Meta(size=size)
|
||||
return path_info, meta, odb.get(stream.hash_info)
|
||||
return from_fs_path, meta, odb.get(stream.hash_info)
|
||||
|
||||
|
||||
def _get_file_hash(path_info, fs, name):
|
||||
info = fs.info(path_info)
|
||||
def _adapt_info(info, scheme):
|
||||
if scheme == "s3" and "ETag" in info:
|
||||
info["etag"] = info["ETag"].strip('"')
|
||||
elif scheme == "gs" and "etag" in info:
|
||||
import base64
|
||||
|
||||
info["etag"] = base64.b64decode(info["etag"]).hex()
|
||||
elif scheme == "http" and ("ETag" in info or "Content-MD5" in info):
|
||||
info["checksum"] = info.get("ETag") or info.get("Content-MD5")
|
||||
return info
|
||||
|
||||
|
||||
def _get_file_hash(fs_path, fs, name):
|
||||
info = _adapt_info(fs.info(fs_path), fs.scheme)
|
||||
|
||||
if name in info:
|
||||
assert not info[name].endswith(".dir")
|
||||
hash_value = info[name]
|
||||
elif hasattr(fs, name):
|
||||
func = getattr(fs, name)
|
||||
hash_value = func(path_info)
|
||||
hash_value = func(fs_path)
|
||||
elif name == "md5":
|
||||
hash_value = file_md5(path_info, fs)
|
||||
hash_value = file_md5(fs_path, fs)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -61,41 +76,41 @@ def _get_file_hash(path_info, fs, name):
|
||||
return meta, hash_info
|
||||
|
||||
|
||||
def get_file_hash(path_info, fs, name, state=None):
|
||||
def get_file_hash(fs_path, fs, name, state=None):
|
||||
if state:
|
||||
meta, hash_info = state.get( # pylint: disable=assignment-from-none
|
||||
path_info, fs
|
||||
fs_path, fs
|
||||
)
|
||||
if hash_info:
|
||||
return meta, hash_info
|
||||
|
||||
meta, hash_info = _get_file_hash(path_info, fs, name)
|
||||
meta, hash_info = _get_file_hash(fs_path, fs, name)
|
||||
|
||||
if state:
|
||||
assert ".dir" not in hash_info.value
|
||||
state.save(path_info, fs, hash_info)
|
||||
state.save(fs_path, fs, hash_info)
|
||||
|
||||
return meta, hash_info
|
||||
|
||||
|
||||
def _stage_file(path_info, fs, name, odb=None, upload_odb=None, dry_run=False):
|
||||
def _stage_file(fs_path, fs, name, odb=None, upload_odb=None, dry_run=False):
|
||||
state = odb.state if odb else None
|
||||
meta, hash_info = get_file_hash(path_info, fs, name, state=state)
|
||||
meta, hash_info = get_file_hash(fs_path, fs, name, state=state)
|
||||
if upload_odb and not dry_run:
|
||||
assert odb and name == "md5"
|
||||
return _upload_file(path_info, fs, odb, upload_odb)
|
||||
return _upload_file(fs_path, fs, odb, upload_odb)
|
||||
|
||||
if dry_run:
|
||||
obj = HashFile(path_info, fs, hash_info)
|
||||
obj = HashFile(fs_path, fs, hash_info)
|
||||
else:
|
||||
odb.add(path_info, fs, hash_info, hardlink=False)
|
||||
odb.add(fs_path, fs, hash_info, hardlink=False)
|
||||
obj = odb.get(hash_info)
|
||||
|
||||
return path_info, meta, obj
|
||||
return fs_path, meta, obj
|
||||
|
||||
|
||||
def _build_objects(
|
||||
path_info,
|
||||
fs_path,
|
||||
fs,
|
||||
name,
|
||||
dvcignore=None,
|
||||
@@ -104,9 +119,9 @@ def _build_objects(
|
||||
**kwargs,
|
||||
):
|
||||
if dvcignore:
|
||||
walk_iterator = dvcignore.walk_files(fs, path_info)
|
||||
walk_iterator = dvcignore.find(fs, fs_path)
|
||||
else:
|
||||
walk_iterator = fs.walk_files(path_info)
|
||||
walk_iterator = fs.find(fs_path)
|
||||
with Tqdm(
|
||||
unit="md5",
|
||||
desc="Computing file/dir hashes (only done once)",
|
||||
@@ -126,18 +141,18 @@ def _build_objects(
|
||||
yield from executor.map(worker, walk_iterator)
|
||||
|
||||
|
||||
def _iter_objects(path_info, fs, name, **kwargs):
|
||||
yield from _build_objects(path_info, fs, name, **kwargs)
|
||||
def _iter_objects(fs_path, fs, name, **kwargs):
|
||||
yield from _build_objects(fs_path, fs, name, **kwargs)
|
||||
|
||||
|
||||
def _build_tree(path_info, fs, name, **kwargs):
|
||||
def _build_tree(fs_path, fs, name, **kwargs):
|
||||
from .tree import Tree
|
||||
|
||||
tree_meta = Meta(size=0, nfiles=0)
|
||||
tree = Tree(None, None, None)
|
||||
for file_info, meta, obj in _iter_objects(path_info, fs, name, **kwargs):
|
||||
if DvcIgnore.DVCIGNORE_FILE == file_info.name:
|
||||
raise DvcIgnoreInCollectedDirError(file_info.parent)
|
||||
for file_fs_path, meta, obj in _iter_objects(fs_path, fs, name, **kwargs):
|
||||
if DvcIgnore.DVCIGNORE_FILE == fs.path.name(file_fs_path):
|
||||
raise DvcIgnoreInCollectedDirError(fs.path.parent(file_fs_path))
|
||||
|
||||
# NOTE: this is lossy transformation:
|
||||
# "hey\there" -> "hey/there"
|
||||
@@ -147,7 +162,11 @@ def _build_tree(path_info, fs, name, **kwargs):
|
||||
#
|
||||
# Yes, this is a BUG, as long as we permit "/" in
|
||||
# filenames on Windows and "\" on Unix
|
||||
tree.add(file_info.relative_to(path_info).parts, meta, obj.hash_info)
|
||||
|
||||
relpath = fs.path.relpath(file_fs_path, fs_path)
|
||||
key = tuple(relpath.split(fs.path.sep))
|
||||
assert key
|
||||
tree.add(key, meta, obj.hash_info)
|
||||
|
||||
tree_meta.size += meta.size
|
||||
tree_meta.nfiles += 1
|
||||
@@ -155,7 +174,7 @@ def _build_tree(path_info, fs, name, **kwargs):
|
||||
return tree_meta, tree
|
||||
|
||||
|
||||
def _stage_tree(path_info, fs, fs_info, name, odb=None, **kwargs):
|
||||
def _stage_tree(fs_path, fs, fs_info, name, odb=None, **kwargs):
|
||||
from .tree import Tree
|
||||
|
||||
value = fs_info.get(name)
|
||||
@@ -167,41 +186,46 @@ def _stage_tree(path_info, fs, fs_info, name, odb=None, **kwargs):
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
meta, tree = _build_tree(path_info, fs, name, odb=odb, **kwargs)
|
||||
meta, tree = _build_tree(fs_path, fs, name, odb=odb, **kwargs)
|
||||
state = odb.state if odb and odb.state else None
|
||||
hash_info = None
|
||||
if state:
|
||||
_, hash_info = state.get( # pylint: disable=assignment-from-none
|
||||
path_info, fs
|
||||
fs_path, fs
|
||||
)
|
||||
tree.digest(hash_info=hash_info)
|
||||
odb.add(tree.path_info, tree.fs, tree.hash_info, hardlink=False)
|
||||
odb.add(tree.fs_path, tree.fs, tree.hash_info, hardlink=False)
|
||||
raw = odb.get(tree.hash_info)
|
||||
# cleanup unneeded memfs tmpfile and return tree based on the
|
||||
# ODB fs/path
|
||||
if odb.fs != tree.fs:
|
||||
tree.fs.remove(tree.path_info)
|
||||
tree.fs.remove(tree.fs_path)
|
||||
tree.fs = raw.fs
|
||||
tree.path_info = raw.path_info
|
||||
tree.fs_path = raw.fs_path
|
||||
return meta, tree
|
||||
|
||||
|
||||
_url_cache: Dict[str, str] = {}
|
||||
|
||||
|
||||
def _make_staging_url(path_info: "AnyPath"):
|
||||
from dvc.path_info import CloudURLInfo
|
||||
def _make_staging_url(
|
||||
fs: "BaseFileSystem", odb: "ObjectDB", fs_path: Optional[str]
|
||||
):
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
url = CloudURLInfo(f"{Schemes.MEMORY}://{_STAGING_MEMFS_PATH}")
|
||||
if path_info:
|
||||
if isinstance(path_info, (str, pathlib.PurePath)):
|
||||
path = os.path.abspath(path_info)
|
||||
else:
|
||||
path = str(path_info)
|
||||
if path not in _url_cache:
|
||||
_url_cache[path] = hashlib.sha256(path.encode("utf-8")).hexdigest()
|
||||
url /= _url_cache[path]
|
||||
url = f"{Schemes.MEMORY}://{_STAGING_MEMFS_PATH}"
|
||||
|
||||
if fs_path is not None:
|
||||
if odb.fs.scheme == Schemes.LOCAL:
|
||||
fs_path = os.path.abspath(fs_path)
|
||||
|
||||
if fs_path not in _url_cache:
|
||||
_url_cache[fs_path] = hashlib.sha256(
|
||||
fs_path.encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
url = fs.path.join(url, _url_cache[fs_path])
|
||||
|
||||
return url
|
||||
|
||||
|
||||
@@ -214,18 +238,18 @@ def _get_staging(odb: "ObjectDB") -> "ObjectDB":
|
||||
from dvc.fs.memory import MemoryFileSystem
|
||||
|
||||
fs = MemoryFileSystem()
|
||||
path_info = _make_staging_url(odb.path_info)
|
||||
fs_path = _make_staging_url(fs, odb, odb.fs_path)
|
||||
state = odb.state
|
||||
return ReferenceObjectDB(fs, path_info, state=state)
|
||||
return ReferenceObjectDB(fs, fs_path, state=state)
|
||||
|
||||
|
||||
def _load_from_state(odb, staging, path_info, fs, name):
|
||||
def _load_from_state(odb, staging, fs_path, fs, name):
|
||||
from . import check, load
|
||||
from .errors import ObjectFormatError
|
||||
from .tree import Tree
|
||||
|
||||
state = odb.state
|
||||
meta, hash_info = state.get(path_info, fs)
|
||||
meta, hash_info = state.get(fs_path, fs)
|
||||
if hash_info:
|
||||
for odb_ in (odb, staging):
|
||||
if odb_.exists(hash_info):
|
||||
@@ -248,10 +272,10 @@ def _stage_external_tree_info(odb, tree, name):
|
||||
# that would be universal for all clouds.
|
||||
assert odb and name != "md5"
|
||||
|
||||
odb.add(tree.path_info, tree.fs, tree.hash_info)
|
||||
odb.add(tree.fs_path, tree.fs, tree.hash_info)
|
||||
raw = odb.get(tree.hash_info)
|
||||
_, hash_info = get_file_hash(raw.path_info, raw.fs, name, state=odb.state)
|
||||
tree.path_info = raw.path_info
|
||||
_, hash_info = get_file_hash(raw.fs_path, raw.fs, name, state=odb.state)
|
||||
tree.fs_path = raw.fs_path
|
||||
tree.fs = raw.fs
|
||||
tree.hash_info.name = hash_info.name
|
||||
tree.hash_info.value = hash_info.value
|
||||
@@ -262,7 +286,7 @@ def _stage_external_tree_info(odb, tree, name):
|
||||
|
||||
def stage(
|
||||
odb: "ObjectDB",
|
||||
path_info: "DvcPath",
|
||||
fs_path: "AnyPath",
|
||||
fs: "BaseFileSystem",
|
||||
name: str,
|
||||
upload: bool = False,
|
||||
@@ -282,19 +306,20 @@ def stage(
|
||||
ODB filesystem, and staged objects will reference the uploaded path rather
|
||||
than the original source path.
|
||||
"""
|
||||
assert path_info and path_info.scheme == fs.scheme
|
||||
assert fs_path
|
||||
# assert scheme(fs_path) == fs.scheme
|
||||
|
||||
details = fs.info(path_info)
|
||||
details = fs.info(fs_path)
|
||||
staging = _get_staging(odb)
|
||||
if odb:
|
||||
try:
|
||||
return _load_from_state(odb, staging, path_info, fs, name)
|
||||
return _load_from_state(odb, staging, fs_path, fs, name)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
if details["type"] == "directory":
|
||||
meta, obj = _stage_tree(
|
||||
path_info,
|
||||
fs_path,
|
||||
fs,
|
||||
details,
|
||||
name,
|
||||
@@ -308,7 +333,7 @@ def stage(
|
||||
obj = _stage_external_tree_info(odb, obj, name)
|
||||
else:
|
||||
_, meta, obj = _stage_file(
|
||||
path_info,
|
||||
fs_path,
|
||||
fs,
|
||||
name,
|
||||
odb=staging,
|
||||
@@ -317,6 +342,6 @@ def stage(
|
||||
)
|
||||
|
||||
if odb and odb.state and obj.hash_info:
|
||||
odb.state.save(path_info, fs, obj.hash_info)
|
||||
odb.state.save(fs_path, fs, obj.hash_info)
|
||||
|
||||
return staging, meta, obj
|
||||
|
||||
@@ -86,7 +86,7 @@ def status(
|
||||
exists: objs that exist in odb
|
||||
missing: objs that do not exist in ODB
|
||||
"""
|
||||
logger.debug("Preparing to collect status from '%s'", odb.path_info)
|
||||
logger.debug("Preparing to collect status from '%s'", odb.fs_path)
|
||||
if not name:
|
||||
name = odb.fs.PARAM_CHECKSUM
|
||||
|
||||
@@ -116,7 +116,7 @@ def status(
|
||||
hashes: Set[str] = set(hash_infos.keys())
|
||||
exists: Set[str] = set()
|
||||
|
||||
logger.debug("Collecting status from '%s'", odb.path_info)
|
||||
logger.debug("Collecting status from '%s'", odb.fs_path)
|
||||
if index and hashes:
|
||||
if dir_objs:
|
||||
exists = hashes.intersection(
|
||||
@@ -128,9 +128,7 @@ def status(
|
||||
hashes.difference_update(exists)
|
||||
|
||||
if hashes:
|
||||
exists.update(
|
||||
odb.hashes_exist(hashes, name=str(odb.path_info), **kwargs)
|
||||
)
|
||||
exists.update(odb.hashes_exist(hashes, name=odb.fs_path, **kwargs))
|
||||
return StatusResult(
|
||||
{hash_infos[hash_] for hash_ in exists},
|
||||
{hash_infos[hash_] for hash_ in (hashes - exists)},
|
||||
|
||||
@@ -20,9 +20,9 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
def _log_exceptions(func):
|
||||
@wraps(func)
|
||||
def wrapper(path_info, *args, **kwargs):
|
||||
def wrapper(fs_path, *args, **kwargs):
|
||||
try:
|
||||
func(path_info, *args, **kwargs)
|
||||
func(fs_path, *args, **kwargs)
|
||||
return 0
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
# NOTE: this means we ran out of file descriptors and there is no
|
||||
@@ -31,7 +31,7 @@ def _log_exceptions(func):
|
||||
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
|
||||
raise
|
||||
|
||||
logger.exception("failed to transfer '%s'", path_info)
|
||||
logger.exception("failed to transfer '%s'", fs_path)
|
||||
return 1
|
||||
|
||||
return wrapper
|
||||
@@ -87,8 +87,8 @@ def _do_transfer(
|
||||
)
|
||||
logger.error(
|
||||
"failed to upload '%s' to '%s'",
|
||||
src.get(dir_obj.hash_info).path_info,
|
||||
dest.get(dir_obj.hash_info).path_info,
|
||||
src.get(dir_obj.hash_info).fs_path,
|
||||
dest.get(dir_obj.hash_info).fs_path,
|
||||
)
|
||||
total_fails += dir_fails + 1
|
||||
elif entry_ids.intersection(missing_ids):
|
||||
@@ -144,8 +144,8 @@ def transfer(
|
||||
|
||||
logger.debug(
|
||||
"Preparing to transfer data from '%s' to '%s'",
|
||||
src.path_info,
|
||||
dest.path_info,
|
||||
src.fs_path,
|
||||
dest.fs_path,
|
||||
)
|
||||
if src == dest:
|
||||
return 0
|
||||
@@ -157,7 +157,7 @@ def transfer(
|
||||
def func(hash_info: "HashInfo") -> None:
|
||||
obj = src.get(hash_info)
|
||||
return dest.add(
|
||||
obj.path_info,
|
||||
obj.fs_path,
|
||||
obj.fs,
|
||||
obj.hash_info,
|
||||
verify=verify,
|
||||
|
||||
+10
-12
@@ -36,19 +36,18 @@ class Tree(HashFile):
|
||||
|
||||
def digest(self, hash_info: Optional["HashInfo"] = None):
|
||||
from dvc.fs.memory import MemoryFileSystem
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.utils import tmp_fname
|
||||
|
||||
memfs = MemoryFileSystem()
|
||||
path_info = CloudURLInfo("memory://{}".format(tmp_fname("")))
|
||||
with memfs.open(path_info, "wb") as fobj:
|
||||
fs_path = "memory://{}".format(tmp_fname(""))
|
||||
with memfs.open(fs_path, "wb") as fobj:
|
||||
fobj.write(self.as_bytes())
|
||||
self.fs = memfs
|
||||
self.path_info = path_info
|
||||
self.fs_path = fs_path
|
||||
if hash_info:
|
||||
self.hash_info = hash_info
|
||||
else:
|
||||
_, self.hash_info = get_file_hash(path_info, memfs, "md5")
|
||||
_, self.hash_info = get_file_hash(fs_path, memfs, "md5")
|
||||
assert self.hash_info.value
|
||||
self.hash_info.value += ".dir"
|
||||
|
||||
@@ -101,7 +100,7 @@ class Tree(HashFile):
|
||||
obj = odb.get(hash_info)
|
||||
|
||||
try:
|
||||
with obj.fs.open(obj.path_info, "r") as fobj:
|
||||
with obj.fs.open(obj.fs_path, "r") as fobj:
|
||||
raw = json.load(fobj)
|
||||
except ValueError as exc:
|
||||
raise ObjectFormatError(f"{obj} is corrupted") from exc
|
||||
@@ -109,12 +108,12 @@ class Tree(HashFile):
|
||||
if not isinstance(raw, list):
|
||||
logger.error(
|
||||
"dir cache file format error '%s' [skipping the file]",
|
||||
obj.path_info,
|
||||
obj.fs_path,
|
||||
)
|
||||
raise ObjectFormatError(f"{obj} is corrupted")
|
||||
|
||||
tree = cls.from_list(raw)
|
||||
tree.path_info = obj.path_info
|
||||
tree.fs_path = obj.fs_path
|
||||
tree.fs = obj.fs
|
||||
tree.hash_info = hash_info
|
||||
|
||||
@@ -125,11 +124,11 @@ class Tree(HashFile):
|
||||
inside prefix.
|
||||
|
||||
The returned tree will contain the original tree's hash_info and
|
||||
path_info.
|
||||
fs_path.
|
||||
|
||||
Returns an empty tree if no object exists at the specified prefix.
|
||||
"""
|
||||
tree = Tree(self.path_info, self.fs, self.hash_info)
|
||||
tree = Tree(self.fs_path, self.fs, self.hash_info)
|
||||
try:
|
||||
for key, (meta, oid) in self.trie.items(prefix):
|
||||
tree.add(key, meta, oid)
|
||||
@@ -160,8 +159,7 @@ class Tree(HashFile):
|
||||
def du(odb, tree):
|
||||
try:
|
||||
return sum(
|
||||
odb.fs.getsize(odb.hash_to_path_info(oid.value))
|
||||
for _, _, oid in tree
|
||||
odb.fs.getsize(odb.hash_to_path(oid.value)) for _, _, oid in tree
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
+62
-55
@@ -292,7 +292,7 @@ class Output:
|
||||
):
|
||||
self.repo = stage.repo if stage else None
|
||||
|
||||
fs_cls, fs_config, path_info = get_cloud_fs(self.repo, url=path)
|
||||
fs_cls, fs_config, fs_path = get_cloud_fs(self.repo, url=path)
|
||||
self.fs = fs_cls(**fs_config)
|
||||
|
||||
if (
|
||||
@@ -309,7 +309,6 @@ class Output:
|
||||
# here is a list and comments:
|
||||
#
|
||||
# .def_path - path from definition in DVC file
|
||||
# .path_info - PathInfo/URLInfo structured resolved path
|
||||
# .fspath - local only, resolved
|
||||
# .__str__ - for presentation purposes, def_path/relpath
|
||||
#
|
||||
@@ -326,32 +325,32 @@ class Output:
|
||||
self.live = live
|
||||
self.desc = desc
|
||||
|
||||
self.path_info = self._parse_path(self.fs, path_info)
|
||||
self.fs_path = self._parse_path(self.fs, fs_path)
|
||||
if self.use_cache and self.odb is None:
|
||||
raise RemoteCacheRequiredError(self.path_info)
|
||||
raise RemoteCacheRequiredError(self.fs.scheme, self.fs_path)
|
||||
|
||||
self.obj = None
|
||||
self.isexec = False if self.IS_DEPENDENCY else isexec
|
||||
|
||||
self.remote = remote
|
||||
|
||||
def _parse_path(self, fs, path_info):
|
||||
def _parse_path(self, fs, fs_path):
|
||||
if fs.scheme != "local":
|
||||
return path_info
|
||||
return fs_path
|
||||
|
||||
parsed = urlparse(self.def_path)
|
||||
if parsed.scheme != "remote":
|
||||
# NOTE: we can path either from command line or .dvc file,
|
||||
# so we should expect both posix and windows style paths.
|
||||
# PathInfo accepts both, i.e. / works everywhere, \ only on win.
|
||||
# paths accepts both, i.e. / works everywhere, \ only on win.
|
||||
#
|
||||
# FIXME: if we have Windows path containing / or posix one with \
|
||||
# then we have #2059 bug and can't really handle that.
|
||||
if self.stage and not path_info.is_absolute():
|
||||
path_info = self.stage.wdir / path_info
|
||||
if self.stage and not os.path.isabs(fs_path):
|
||||
fs_path = fs.path.join(self.stage.wdir, fs_path)
|
||||
|
||||
abs_p = os.path.abspath(os.path.normpath(path_info))
|
||||
return fs.PATH_CLS(abs_p)
|
||||
abs_p = os.path.abspath(os.path.normpath(fs_path))
|
||||
return abs_p
|
||||
|
||||
def __repr__(self):
|
||||
return "{class_name}: '{def_path}'".format(
|
||||
@@ -371,9 +370,9 @@ class Output:
|
||||
|
||||
cur_dir = os.getcwd()
|
||||
if path_isin(cur_dir, self.repo.root_dir):
|
||||
return relpath(self.path_info, cur_dir)
|
||||
return relpath(self.fs_path, cur_dir)
|
||||
|
||||
return relpath(self.path_info, self.repo.root_dir)
|
||||
return relpath(self.fs_path, self.repo.root_dir)
|
||||
|
||||
@property
|
||||
def scheme(self):
|
||||
@@ -391,7 +390,7 @@ class Output:
|
||||
return False
|
||||
|
||||
return self.repo and path_isin(
|
||||
os.path.realpath(self.path_info), self.repo.root_dir
|
||||
os.path.realpath(self.fs_path), self.repo.root_dir
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -407,7 +406,9 @@ class Output:
|
||||
|
||||
@property
|
||||
def cache_path(self):
|
||||
return self.odb.hash_to_path_info(self.hash_info.value).url
|
||||
return self.odb.fs.unstrip_protocol(
|
||||
self.odb.hash_to_path(self.hash_info.value)
|
||||
)
|
||||
|
||||
def get_hash(self):
|
||||
if self.use_cache:
|
||||
@@ -418,7 +419,7 @@ class Output:
|
||||
name = self.fs.PARAM_CHECKSUM
|
||||
_, _, obj = ostage(
|
||||
odb,
|
||||
self.path_info,
|
||||
self.fs_path,
|
||||
self.fs,
|
||||
name,
|
||||
dvcignore=self.dvcignore,
|
||||
@@ -438,10 +439,10 @@ class Output:
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
if self._is_path_dvcignore(self.path_info):
|
||||
if self._is_path_dvcignore(self.fs_path):
|
||||
return False
|
||||
|
||||
return self.fs.exists(self.path_info)
|
||||
return self.fs.exists(self.fs_path)
|
||||
|
||||
def changed_checksum(self):
|
||||
return self.hash_info != self.get_hash()
|
||||
@@ -491,17 +492,17 @@ class Output:
|
||||
|
||||
@property
|
||||
def is_empty(self):
|
||||
return self.fs.is_empty(self.path_info)
|
||||
return self.fs.is_empty(self.fs_path)
|
||||
|
||||
def isdir(self):
|
||||
if self._is_path_dvcignore(self.path_info):
|
||||
if self._is_path_dvcignore(self.fs_path):
|
||||
return False
|
||||
return self.fs.isdir(self.path_info)
|
||||
return self.fs.isdir(self.fs_path)
|
||||
|
||||
def isfile(self):
|
||||
if self._is_path_dvcignore(self.path_info):
|
||||
if self._is_path_dvcignore(self.fs_path):
|
||||
return False
|
||||
return self.fs.isfile(self.path_info)
|
||||
return self.fs.isfile(self.fs_path)
|
||||
|
||||
# pylint: disable=no-member
|
||||
|
||||
@@ -540,7 +541,7 @@ class Output:
|
||||
if not self.use_cache:
|
||||
_, self.meta, obj = ostage(
|
||||
self.repo.odb.local,
|
||||
self.path_info,
|
||||
self.fs_path,
|
||||
self.fs,
|
||||
self.fs.PARAM_CHECKSUM,
|
||||
dvcignore=self.dvcignore,
|
||||
@@ -557,17 +558,17 @@ class Output:
|
||||
|
||||
_, self.meta, self.obj = ostage(
|
||||
self.odb,
|
||||
self.path_info,
|
||||
self.fs_path,
|
||||
self.fs,
|
||||
self.odb.fs.PARAM_CHECKSUM,
|
||||
dvcignore=self.dvcignore,
|
||||
)
|
||||
self.hash_info = self.obj.hash_info
|
||||
self.isexec = self.isfile() and self.fs.isexec(self.path_info)
|
||||
self.isexec = self.isfile() and self.fs.isexec(self.fs_path)
|
||||
|
||||
def set_exec(self):
|
||||
if self.isfile() and self.isexec:
|
||||
self.odb.set_exec(self.path_info)
|
||||
self.odb.set_exec(self.fs_path)
|
||||
|
||||
def commit(self, filter_info=None):
|
||||
if not self.exists:
|
||||
@@ -579,14 +580,14 @@ class Output:
|
||||
granular = (
|
||||
self.is_dir_checksum
|
||||
and filter_info
|
||||
and filter_info != self.path_info
|
||||
and filter_info != self.fs_path
|
||||
)
|
||||
if granular:
|
||||
obj = self._commit_granular_dir(filter_info)
|
||||
else:
|
||||
staging, _, obj = ostage(
|
||||
self.odb,
|
||||
filter_info or self.path_info,
|
||||
filter_info or self.fs_path,
|
||||
self.fs,
|
||||
self.odb.fs.PARAM_CHECKSUM,
|
||||
dvcignore=self.dvcignore,
|
||||
@@ -599,7 +600,7 @@ class Output:
|
||||
hardlink=True,
|
||||
)
|
||||
checkout(
|
||||
filter_info or self.path_info,
|
||||
filter_info or self.fs_path,
|
||||
self.fs,
|
||||
obj,
|
||||
self.odb,
|
||||
@@ -610,10 +611,12 @@ class Output:
|
||||
self.set_exec()
|
||||
|
||||
def _commit_granular_dir(self, filter_info):
|
||||
prefix = filter_info.relative_to(self.path_info).parts
|
||||
prefix = self.fs.path.parts(
|
||||
self.fs.path.relpath(filter_info, self.fs_path)
|
||||
)
|
||||
staging, _, save_obj = ostage(
|
||||
self.odb,
|
||||
self.path_info,
|
||||
self.fs_path,
|
||||
self.fs,
|
||||
self.odb.fs.PARAM_CHECKSUM,
|
||||
dvcignore=self.dvcignore,
|
||||
@@ -633,7 +636,9 @@ class Output:
|
||||
ret = {**self.hash_info.to_dict(), **self.meta.to_dict()}
|
||||
|
||||
if self.is_in_repo:
|
||||
path = self.path_info.relpath(self.stage.wdir).as_posix()
|
||||
path = self.fs.path.as_posix(
|
||||
relpath(self.fs_path, self.stage.wdir)
|
||||
)
|
||||
else:
|
||||
path = self.def_path
|
||||
|
||||
@@ -684,22 +689,21 @@ class Output:
|
||||
if not self.metric or self.plot:
|
||||
return
|
||||
|
||||
path = os.fspath(self.path_info)
|
||||
if not os.path.exists(path):
|
||||
if not os.path.exists(self.fs_path):
|
||||
return
|
||||
|
||||
name = "metrics" if self.metric else "plot"
|
||||
if os.path.isdir(path):
|
||||
if os.path.isdir(self.fs_path):
|
||||
msg = "directory '%s' cannot be used as %s."
|
||||
logger.debug(msg, str(self.path_info), name)
|
||||
logger.debug(msg, str(self), name)
|
||||
return
|
||||
|
||||
if not istextfile(path, self.fs):
|
||||
if not istextfile(self.fs_path, self.fs):
|
||||
msg = "binary file '{}' cannot be used as {}."
|
||||
raise DvcException(msg.format(self.path_info, name))
|
||||
raise DvcException(msg.format(self.fs_path, name))
|
||||
|
||||
def download(self, to, jobs=None):
|
||||
self.fs.download(self.path_info, to.path_info, jobs=jobs)
|
||||
self.fs.download(self.fs_path, to.fs_path, jobs=jobs)
|
||||
|
||||
def get_obj(self, filter_info=None, **kwargs):
|
||||
if self.obj:
|
||||
@@ -712,8 +716,9 @@ class Output:
|
||||
else:
|
||||
return None
|
||||
|
||||
if filter_info and filter_info != self.path_info:
|
||||
prefix = filter_info.relative_to(self.path_info).parts
|
||||
fs_path = self.fs.path
|
||||
if filter_info and filter_info != self.fs_path:
|
||||
prefix = fs_path.parts(fs_path.relpath(filter_info, self.fs_path))
|
||||
obj = obj.get(self.odb, prefix)
|
||||
|
||||
return obj
|
||||
@@ -731,12 +736,12 @@ class Output:
|
||||
if not self.use_cache:
|
||||
if progress_callback:
|
||||
progress_callback(
|
||||
str(self.path_info), self.get_files_number(filter_info)
|
||||
self.fs_path, self.get_files_number(filter_info)
|
||||
)
|
||||
return None
|
||||
|
||||
obj = self.get_obj(filter_info=filter_info)
|
||||
if not obj and (filter_info and filter_info != self.path_info):
|
||||
if not obj and (filter_info and filter_info != self.fs_path):
|
||||
# backward compatibility
|
||||
return None
|
||||
|
||||
@@ -749,7 +754,7 @@ class Output:
|
||||
|
||||
try:
|
||||
modified = checkout(
|
||||
filter_info or self.path_info,
|
||||
filter_info or self.fs_path,
|
||||
self.fs,
|
||||
obj,
|
||||
self.odb,
|
||||
@@ -767,7 +772,7 @@ class Output:
|
||||
return added, False if added else modified
|
||||
|
||||
def remove(self, ignore_remove=False):
|
||||
self.fs.remove(self.path_info)
|
||||
self.fs.remove(self.fs_path)
|
||||
if self.scheme != Schemes.LOCAL:
|
||||
return
|
||||
|
||||
@@ -779,9 +784,9 @@ class Output:
|
||||
if self.scheme == "local" and self.use_scm_ignore:
|
||||
self.repo.scm.ignore_remove(self.fspath)
|
||||
|
||||
self.fs.move(self.path_info, out.path_info)
|
||||
self.fs.move(self.fs_path, out.fs_path)
|
||||
self.def_path = out.def_path
|
||||
self.path_info = out.path_info
|
||||
self.fs_path = out.fs_path
|
||||
self.save()
|
||||
self.commit()
|
||||
|
||||
@@ -836,7 +841,7 @@ class Output:
|
||||
if not self.hash_info.isdir:
|
||||
return 1
|
||||
|
||||
if not filter_info or filter_info == self.path_info:
|
||||
if not filter_info or filter_info == self.fs_path:
|
||||
return self.meta.nfiles or 0
|
||||
|
||||
obj = self.get_obj(filter_info=filter_info)
|
||||
@@ -844,7 +849,7 @@ class Output:
|
||||
|
||||
def unprotect(self):
|
||||
if self.exists:
|
||||
self.odb.unprotect(self.path_info)
|
||||
self.odb.unprotect(self.fs_path)
|
||||
|
||||
def get_dir_cache(self, **kwargs):
|
||||
if not self.is_dir_checksum:
|
||||
@@ -886,7 +891,7 @@ class Output:
|
||||
"Cache for files inside will be lost. "
|
||||
"Would you like to continue? Use '-f' to force."
|
||||
)
|
||||
if not force and not prompt.confirm(msg.format(self.path_info)):
|
||||
if not force and not prompt.confirm(msg.format(self.fs_path)):
|
||||
raise CollectCacheError(
|
||||
"unable to fully collect used cache"
|
||||
" without cache for directory '{}'".format(self)
|
||||
@@ -894,8 +899,10 @@ class Output:
|
||||
return None
|
||||
|
||||
obj = self.get_obj()
|
||||
if filter_info and filter_info != self.path_info:
|
||||
prefix = filter_info.relative_to(self.path_info).parts
|
||||
if filter_info and filter_info != self.fs_path:
|
||||
prefix = self.fs.path.parts(
|
||||
self.fs.path.relpath(filter_info, self.fs_path)
|
||||
)
|
||||
obj = obj.filter(prefix)
|
||||
return obj
|
||||
|
||||
@@ -1021,7 +1028,7 @@ class Output:
|
||||
merged = merge(
|
||||
self.odb, ancestor_info, self.hash_info, other.hash_info
|
||||
)
|
||||
self.odb.add(merged.path_info, merged.fs, merged.hash_info)
|
||||
self.odb.add(merged.fs_path, merged.fs, merged.hash_info)
|
||||
|
||||
self.hash_info = merged.hash_info
|
||||
self.meta = Meta(
|
||||
@@ -1031,7 +1038,7 @@ class Output:
|
||||
|
||||
@property
|
||||
def fspath(self):
|
||||
return self.path_info.fspath
|
||||
return self.fs_path
|
||||
|
||||
@property
|
||||
def is_decorated(self) -> bool:
|
||||
|
||||
@@ -17,7 +17,6 @@ from funcy import cached_property, collecting, first, isa, join, reraise
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.parsing.interpolate import ParseError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.utils import relpath
|
||||
|
||||
from .context import (
|
||||
@@ -134,10 +133,10 @@ def make_definition(
|
||||
|
||||
|
||||
class DataResolver:
|
||||
def __init__(self, repo: "Repo", wdir: PathInfo, d: dict):
|
||||
def __init__(self, repo: "Repo", wdir: str, d: dict):
|
||||
self.fs = fs = repo.fs
|
||||
self.wdir = wdir
|
||||
self.relpath = relpath(self.wdir / "dvc.yaml")
|
||||
self.relpath = relpath(fs.path.join(self.wdir, "dvc.yaml"))
|
||||
|
||||
vars_ = d.get(VARS_KWD, [])
|
||||
check_interpolations(vars_, VARS_KWD, self.relpath)
|
||||
@@ -232,7 +231,7 @@ class EntryDefinition:
|
||||
|
||||
def _resolve_wdir(
|
||||
self, context: Context, name: str, wdir: str = None
|
||||
) -> PathInfo:
|
||||
) -> str:
|
||||
if not wdir:
|
||||
return self.wdir
|
||||
|
||||
@@ -240,7 +239,7 @@ class EntryDefinition:
|
||||
wdir = to_str(context.resolve_str(wdir))
|
||||
except (ContextError, ParseError) as exc:
|
||||
format_and_raise(exc, f"'{self.where}.{name}.wdir'", self.relpath)
|
||||
return self.wdir / wdir
|
||||
return self.resolver.fs.path.join(self.wdir, wdir)
|
||||
|
||||
def resolve(self, **kwargs):
|
||||
try:
|
||||
|
||||
@@ -19,7 +19,6 @@ from dvc.parsing.interpolate import (
|
||||
recurse,
|
||||
str_interpolate,
|
||||
)
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.utils import relpath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -356,7 +355,7 @@ class Context(CtxDict):
|
||||
|
||||
@classmethod
|
||||
def load_from(
|
||||
cls, fs, path: PathInfo, select_keys: List[str] = None
|
||||
cls, fs, path: str, select_keys: List[str] = None
|
||||
) -> "Context":
|
||||
from dvc.utils.serialize import LOADERS
|
||||
|
||||
@@ -396,18 +395,17 @@ class Context(CtxDict):
|
||||
raise ReservedKeyError(matches)
|
||||
return super().merge_update(other, overwrite=overwrite)
|
||||
|
||||
def merge_from(self, fs, item: str, wdir: PathInfo, overwrite=False):
|
||||
def merge_from(self, fs, item: str, wdir: str, overwrite=False):
|
||||
path, _, keys_str = item.partition(":")
|
||||
select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
|
||||
|
||||
abspath = os.path.abspath(wdir / path)
|
||||
path_info = PathInfo(abspath)
|
||||
abspath = os.path.abspath(fs.path.join(wdir, path))
|
||||
if abspath in self.imports:
|
||||
if not select_keys and self.imports[abspath] is None:
|
||||
return # allow specifying complete filepath multiple times
|
||||
self.check_loaded(abspath, item, select_keys)
|
||||
|
||||
ctx = Context.load_from(fs, path_info, select_keys)
|
||||
ctx = Context.load_from(fs, abspath, select_keys)
|
||||
|
||||
try:
|
||||
self.merge_update(ctx, overwrite=overwrite)
|
||||
@@ -439,12 +437,12 @@ class Context(CtxDict):
|
||||
self,
|
||||
fs,
|
||||
vars_: List,
|
||||
wdir: PathInfo,
|
||||
wdir: str,
|
||||
stage_name: str = None,
|
||||
default: str = None,
|
||||
):
|
||||
if default:
|
||||
to_import = wdir / default
|
||||
to_import = fs.path.join(wdir, default)
|
||||
if fs.exists(to_import):
|
||||
self.merge_from(fs, default, wdir)
|
||||
else:
|
||||
|
||||
+10
-7
@@ -11,7 +11,6 @@ from dvc.exceptions import FileMissingError
|
||||
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
|
||||
from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
|
||||
from dvc.ignore import DvcIgnoreFilter
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.utils import env2bool
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
@@ -354,7 +353,7 @@ class Repo:
|
||||
)
|
||||
|
||||
def unprotect(self, target):
|
||||
return self.odb.local.unprotect(PathInfo(target))
|
||||
return self.odb.local.unprotect(target)
|
||||
|
||||
def _ignore(self):
|
||||
flist = [self.config.files["local"], self.tmp_dir]
|
||||
@@ -446,14 +445,18 @@ class Repo:
|
||||
outs = outs or self.index.outs_graph
|
||||
|
||||
abs_path = os.path.abspath(path)
|
||||
path_info = PathInfo(abs_path)
|
||||
match = path_info.__eq__ if strict else path_info.isin_or_eq
|
||||
fs_path = abs_path
|
||||
|
||||
def func(out):
|
||||
if out.scheme == "local" and match(out.path_info):
|
||||
def eq(one, two):
|
||||
return one == two
|
||||
|
||||
match = eq if strict else out.fs.path.isin_or_eq
|
||||
|
||||
if out.scheme == "local" and match(fs_path, out.fs_path):
|
||||
return True
|
||||
|
||||
if recursive and out.path_info.isin(path_info):
|
||||
if recursive and out.fs.path.isin(out.fs_path, fs_path):
|
||||
return True
|
||||
|
||||
return False
|
||||
@@ -490,7 +493,7 @@ class Repo:
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
|
||||
fs = RepoFileSystem(self, subrepos=True)
|
||||
path = PathInfo(self.root_dir) / path
|
||||
path = self.fs.path.join(self.root_dir, path)
|
||||
try:
|
||||
with fs.open(
|
||||
path, mode=mode, encoding=encoding, remote=remote
|
||||
|
||||
+3
-3
@@ -87,8 +87,8 @@ def translate_graph_error(stages: Stages) -> Iterator[None]:
|
||||
yield
|
||||
except OverlappingOutputPathsError as exc:
|
||||
msg = OVERLAPPING_OUTPUT_FMT.format(
|
||||
out=exc.overlapping_out.path_info,
|
||||
parent=exc.parent.path_info,
|
||||
out=exc.overlapping_out.fs_path,
|
||||
parent=exc.parent.fs_path,
|
||||
parent_stage=exc.parent.stage.addressing,
|
||||
)
|
||||
raise OverlappingOutputPathsError(exc.parent, exc.overlapping_out, msg)
|
||||
@@ -228,7 +228,7 @@ def _find_all_targets(
|
||||
from dvc.dvcfile import is_dvc_file
|
||||
|
||||
if os.path.isdir(target) and recursive:
|
||||
files = map(os.fspath, repo.dvcignore.walk_files(repo.fs, target))
|
||||
files = repo.dvcignore.find(repo.fs, target)
|
||||
yield from (
|
||||
path
|
||||
for path in files
|
||||
|
||||
+27
-26
@@ -2,8 +2,7 @@ import logging
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Callable, Iterable, List, Tuple
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.types import DvcPath
|
||||
from dvc.types import AnyPath
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.output import Output
|
||||
@@ -14,7 +13,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
FilterFn = Callable[["Output"], bool]
|
||||
Outputs = List["Output"]
|
||||
DvcPaths = List[DvcPath]
|
||||
AnyPaths = List[AnyPath]
|
||||
StrPaths = List[str]
|
||||
|
||||
|
||||
def _collect_outs(
|
||||
@@ -32,39 +32,40 @@ def _collect_paths(
|
||||
rev: str = None,
|
||||
):
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.utils import relpath
|
||||
|
||||
path_infos = [PathInfo(os.path.abspath(target)) for target in targets]
|
||||
fs_paths = [os.path.abspath(target) for target in targets]
|
||||
fs = RepoFileSystem(repo)
|
||||
|
||||
target_infos = []
|
||||
for path_info in path_infos:
|
||||
target_paths = []
|
||||
for fs_path in fs_paths:
|
||||
|
||||
if recursive and fs.isdir(path_info):
|
||||
target_infos.extend(repo.dvcignore.walk_files(fs, path_info))
|
||||
if recursive and fs.isdir(fs_path):
|
||||
target_paths.extend(repo.dvcignore.find(fs, fs_path))
|
||||
|
||||
if not fs.exists(path_info):
|
||||
if not fs.exists(fs_path):
|
||||
rel = relpath(fs_path)
|
||||
if rev == "workspace" or rev == "":
|
||||
logger.warning(
|
||||
"'%s' was not found in current workspace.", path_info
|
||||
)
|
||||
logger.warning("'%s' was not found in current workspace.", rel)
|
||||
else:
|
||||
logger.warning("'%s' was not found at: '%s'.", path_info, rev)
|
||||
target_infos.append(path_info)
|
||||
return target_infos
|
||||
logger.warning("'%s' was not found at: '%s'.", rel, rev)
|
||||
target_paths.append(fs_path)
|
||||
return target_paths
|
||||
|
||||
|
||||
def _filter_duplicates(
|
||||
outs: Outputs, path_infos: DvcPaths
|
||||
) -> Tuple[Outputs, DvcPaths]:
|
||||
outs: Outputs, fs_paths: StrPaths
|
||||
) -> Tuple[Outputs, StrPaths]:
|
||||
res_outs: Outputs = []
|
||||
res_infos = path_infos
|
||||
fs_res_paths = fs_paths
|
||||
|
||||
for out in outs:
|
||||
if out.path_info in path_infos:
|
||||
if out.fs_path in fs_paths:
|
||||
res_outs.append(out)
|
||||
res_infos.remove(out.path_info)
|
||||
# MUTATING THE SAME LIST!!
|
||||
fs_res_paths.remove(out.fs_path)
|
||||
|
||||
return res_outs, res_infos
|
||||
return res_outs, fs_res_paths
|
||||
|
||||
|
||||
def collect(
|
||||
@@ -74,15 +75,15 @@ def collect(
|
||||
output_filter: FilterFn = None,
|
||||
rev: str = None,
|
||||
recursive: bool = False,
|
||||
) -> Tuple[Outputs, DvcPaths]:
|
||||
) -> Tuple[Outputs, StrPaths]:
|
||||
assert targets or output_filter
|
||||
|
||||
outs: Outputs = _collect_outs(repo, output_filter=output_filter, deps=deps)
|
||||
|
||||
if not targets:
|
||||
path_infos: DvcPaths = []
|
||||
return outs, path_infos
|
||||
fs_paths: StrPaths = []
|
||||
return outs, fs_paths
|
||||
|
||||
target_infos = _collect_paths(repo, targets, recursive=recursive, rev=rev)
|
||||
target_paths = _collect_paths(repo, targets, recursive=recursive, rev=rev)
|
||||
|
||||
return _filter_duplicates(outs, target_infos)
|
||||
return _filter_duplicates(outs, target_paths)
|
||||
|
||||
+22
-16
@@ -37,14 +37,14 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
|
||||
# workspace paths/checksums if b_rev was None
|
||||
continue
|
||||
|
||||
targets_path_infos = None
|
||||
targets_paths = None
|
||||
if targets is not None:
|
||||
# convert targets to path_infos, and capture any missing targets
|
||||
targets_path_infos, missing_targets[rev] = _targets_to_path_infos(
|
||||
# convert targets to paths, and capture any missing targets
|
||||
targets_paths, missing_targets[rev] = _targets_to_paths(
|
||||
repo_fs, targets
|
||||
)
|
||||
|
||||
results[rev] = _paths_checksums(self, targets_path_infos)
|
||||
results[rev] = _paths_checksums(self, targets_paths)
|
||||
|
||||
if targets is not None:
|
||||
# check for overlapping missing targets between a_rev and b_rev
|
||||
@@ -133,13 +133,14 @@ def _output_paths(repo, targets):
|
||||
for output in repo.index.outs:
|
||||
if _exists(output):
|
||||
yield_output = targets is None or any(
|
||||
output.path_info.isin_or_eq(target) for target in targets
|
||||
output.fs.path.isin_or_eq(output.fs_path, target)
|
||||
for target in targets
|
||||
)
|
||||
|
||||
if on_working_fs:
|
||||
_, _, obj = ostage(
|
||||
repo.odb.local,
|
||||
output.path_info,
|
||||
output.fs_path,
|
||||
repo.odb.local.fs,
|
||||
"md5",
|
||||
dry_run=True,
|
||||
@@ -158,19 +159,24 @@ def _output_paths(repo, targets):
|
||||
|
||||
if output.is_dir_checksum and (
|
||||
yield_output
|
||||
or any(target.isin(output.path_info) for target in targets)
|
||||
or any(
|
||||
output.fs.path.isin(target, output.fs_path)
|
||||
for target in targets
|
||||
)
|
||||
):
|
||||
yield from _dir_output_paths(output.path_info, obj, targets)
|
||||
yield from _dir_output_paths(
|
||||
output.fs, output.fs_path, obj, targets
|
||||
)
|
||||
|
||||
|
||||
def _dir_output_paths(path_info, obj, targets=None):
|
||||
def _dir_output_paths(fs, fs_path, obj, targets=None):
|
||||
for key, _, oid in obj:
|
||||
fname = path_info.joinpath(*key)
|
||||
fname = fs.path.join(fs_path, *key)
|
||||
if targets is None or any(
|
||||
fname.isin_or_eq(target) for target in targets
|
||||
fs.path.isin_or_eq(fname, target) for target in targets
|
||||
):
|
||||
# pylint: disable=no-member
|
||||
yield str(fname), oid.value
|
||||
yield fs.path.join(fs.path.name(fs_path), *key), oid.value
|
||||
|
||||
|
||||
def _filter_missing(repo_fs, paths):
|
||||
@@ -185,17 +191,17 @@ def _filter_missing(repo_fs, paths):
|
||||
pass
|
||||
|
||||
|
||||
def _targets_to_path_infos(repo_fs, targets):
|
||||
path_infos = []
|
||||
def _targets_to_paths(repo_fs, targets):
|
||||
paths = []
|
||||
missing = []
|
||||
|
||||
for target in targets:
|
||||
if repo_fs.exists(target):
|
||||
path_infos.append(repo_fs.metadata(target).path_info)
|
||||
paths.append(repo_fs.metadata(target).fs_path)
|
||||
else:
|
||||
missing.append(target)
|
||||
|
||||
return path_infos, missing
|
||||
return paths, missing
|
||||
|
||||
|
||||
def _calculate_renamed(new, old, added, deleted):
|
||||
|
||||
@@ -13,7 +13,6 @@ from funcy import cached_property, first
|
||||
from dvc.dependency.param import MissingParamsError
|
||||
from dvc.env import DVCLIVE_RESUME
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.stage.monitor import CheckpointKilledError
|
||||
from dvc.utils import relpath
|
||||
|
||||
@@ -368,13 +367,12 @@ class Experiments:
|
||||
|
||||
logger.debug("Using experiment params '%s'", params)
|
||||
|
||||
for params_fname in params:
|
||||
path = PathInfo(params_fname)
|
||||
suffix = path.suffix.lower()
|
||||
for path in params:
|
||||
suffix = self.repo.fs.path.suffix(path).lower()
|
||||
modify_data = MODIFIERS[suffix]
|
||||
with modify_data(path, fs=self.repo.fs) as data:
|
||||
try:
|
||||
merge_params(data, params[params_fname], allow_new=False)
|
||||
merge_params(data, params[path], allow_new=False)
|
||||
except NewParamsFound as e:
|
||||
msg = self._format_new_params_msg(e.new_params, path)
|
||||
raise MissingParamsError(msg)
|
||||
@@ -929,7 +927,7 @@ class Experiments:
|
||||
from .executor.base import BaseExecutor, ExecutorInfo
|
||||
|
||||
result = {}
|
||||
for pidfile in self.repo.fs.walk_files(
|
||||
for pidfile in self.repo.fs.find(
|
||||
os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR)
|
||||
):
|
||||
rev, _ = os.path.splitext(os.path.basename(pidfile))
|
||||
|
||||
@@ -16,7 +16,6 @@ from typing import (
|
||||
|
||||
from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.experiments.base import (
|
||||
EXEC_BASELINE,
|
||||
@@ -101,7 +100,7 @@ class BaseExecutor(ABC):
|
||||
self,
|
||||
src: "Git",
|
||||
dvc_dir: str,
|
||||
root_dir: Optional[Union[str, PathInfo]] = None,
|
||||
root_dir: Optional[str] = None,
|
||||
branch: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
@@ -148,12 +148,12 @@ class SSHExecutor(BaseExecutor):
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.push import push
|
||||
|
||||
cache_url = posixpath.join(
|
||||
self.abs_url,
|
||||
self._dvc_dir,
|
||||
cache_path = posixpath.join(
|
||||
self._repo_abspath,
|
||||
Repo.DVC_DIR,
|
||||
ODBManager.CACHE_DIR,
|
||||
)
|
||||
|
||||
with self.sshfs() as fs:
|
||||
odb = get_odb(fs, fs.PATH_CLS(cache_url), **fs.config)
|
||||
odb = get_odb(fs, cache_path, **fs.config)
|
||||
push(dvc, revs=[rev], run_cache=run_cache, odb=odb)
|
||||
|
||||
+4
-4
@@ -2,7 +2,6 @@ import logging
|
||||
import os
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.utils import resolve_output
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
@@ -50,8 +49,9 @@ def get(url, path, out=None, rev=None, jobs=None):
|
||||
with external_repo(
|
||||
url=url, rev=rev, cache_dir=tmp_dir, cache_types=cache_types
|
||||
) as repo:
|
||||
from_info = PathInfo(repo.root_dir) / path
|
||||
to_info = PathInfo(out)
|
||||
repo.repo_fs.download(from_info, to_info, jobs=jobs)
|
||||
from_fs_path = os.path.abspath(os.path.join(repo.root_dir, path))
|
||||
repo.repo_fs.download(
|
||||
from_fs_path, os.path.abspath(out), jobs=jobs
|
||||
)
|
||||
finally:
|
||||
remove(tmp_dir)
|
||||
|
||||
+7
-7
@@ -1,5 +1,6 @@
|
||||
from typing import TYPE_CHECKING, Iterator, List, Set
|
||||
|
||||
from dvc.fs.local import localfs
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -94,7 +95,6 @@ def build_graph(stages, outs_trie=None):
|
||||
|
||||
from dvc.exceptions import StagePathAsOutputError
|
||||
|
||||
from ..path_info import PathInfo
|
||||
from .trie import build_outs_trie
|
||||
|
||||
G = nx.DiGraph()
|
||||
@@ -103,7 +103,7 @@ def build_graph(stages, outs_trie=None):
|
||||
outs_trie = outs_trie or build_outs_trie(stages)
|
||||
|
||||
for stage in stages:
|
||||
out = outs_trie.shortest_prefix(PathInfo(stage.path).parts).value
|
||||
out = outs_trie.shortest_prefix(localfs.path.parts(stage.path)).value
|
||||
if out:
|
||||
raise StagePathAsOutputError(stage, str(out))
|
||||
|
||||
@@ -111,10 +111,10 @@ def build_graph(stages, outs_trie=None):
|
||||
G.add_nodes_from(stages)
|
||||
for stage in stages:
|
||||
for dep in stage.deps:
|
||||
if dep.path_info is None:
|
||||
if dep.fs_path is None:
|
||||
continue
|
||||
|
||||
dep_key = dep.path_info.parts
|
||||
dep_key = dep.fs.path.parts(dep.fs_path)
|
||||
overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
|
||||
if outs_trie.has_subtrie(dep_key):
|
||||
overlapping.extend(outs_trie.values(prefix=dep_key))
|
||||
@@ -136,10 +136,10 @@ def build_outs_graph(graph, outs_trie):
|
||||
G.add_nodes_from(outs_trie.values())
|
||||
for stage in graph.nodes():
|
||||
for dep in stage.deps:
|
||||
if dep.path_info is None:
|
||||
# RepoDependency don't have a path_info
|
||||
if dep.fs_path is None:
|
||||
# RepoDependency don't have a path
|
||||
continue
|
||||
dep_key = dep.path_info.parts
|
||||
dep_key = dep.fs.path.parts(dep.fs_path)
|
||||
overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
|
||||
if outs_trie.has_subtrie(dep_key):
|
||||
overlapping.extend(outs_trie.values(prefix=dep_key))
|
||||
|
||||
+4
-5
@@ -7,21 +7,20 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.output import Output
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
|
||||
|
||||
def create_summary(out):
|
||||
assert out.live and out.live["html"]
|
||||
|
||||
metrics, plots = out.repo.live.show(str(out.path_info))
|
||||
metrics, plots = out.repo.live.show(out.fs_path)
|
||||
|
||||
html_path = out.path_info.fspath + "_dvc_plots"
|
||||
html_path = out.fs_path + "_dvc_plots"
|
||||
|
||||
render(out.repo, plots, metrics=metrics, path=html_path, refresh_seconds=5)
|
||||
|
||||
|
||||
def summary_path_info(out: "Output") -> Optional["PathInfo"]:
|
||||
def summary_fs_path(out: "Output") -> Optional[str]:
|
||||
from dvc.output import Output
|
||||
|
||||
assert out.live
|
||||
@@ -29,7 +28,7 @@ def summary_path_info(out: "Output") -> Optional["PathInfo"]:
|
||||
if isinstance(out.live, dict):
|
||||
has_summary = out.live.get(Output.PARAM_LIVE_SUMMARY, True)
|
||||
if has_summary:
|
||||
return out.path_info.with_suffix(".json")
|
||||
return out.fs.path.with_suffix(out.fs_path, ".json")
|
||||
return None
|
||||
|
||||
|
||||
|
||||
+11
-11
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
from itertools import chain
|
||||
|
||||
from dvc.exceptions import PathMissingError
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
|
||||
def ls(url, path=None, rev=None, recursive=None, dvc_only=False):
|
||||
@@ -29,11 +29,11 @@ def ls(url, path=None, rev=None, recursive=None, dvc_only=False):
|
||||
from . import Repo
|
||||
|
||||
with Repo.open(url, rev=rev, subrepos=True, uninitialized=True) as repo:
|
||||
path_info = PathInfo(repo.root_dir)
|
||||
fs_path = repo.root_dir
|
||||
if path:
|
||||
path_info /= path
|
||||
fs_path = os.path.abspath(repo.fs.path.join(fs_path, path))
|
||||
|
||||
ret = _ls(repo.repo_fs, path_info, recursive, dvc_only)
|
||||
ret = _ls(repo.repo_fs, fs_path, recursive, dvc_only)
|
||||
|
||||
if path and not ret:
|
||||
raise PathMissingError(path, repo, dvc_only=dvc_only)
|
||||
@@ -46,21 +46,21 @@ def ls(url, path=None, rev=None, recursive=None, dvc_only=False):
|
||||
return ret_list
|
||||
|
||||
|
||||
def _ls(fs, path_info, recursive=None, dvc_only=False):
|
||||
def _ls(fs, fs_path, recursive=None, dvc_only=False):
|
||||
def onerror(exc):
|
||||
raise exc
|
||||
|
||||
infos = []
|
||||
try:
|
||||
for root, dirs, files in fs.walk(
|
||||
path_info.fspath, onerror=onerror, dvcfiles=True
|
||||
fs_path, onerror=onerror, dvcfiles=True
|
||||
):
|
||||
entries = chain(files, dirs) if not recursive else files
|
||||
infos.extend(PathInfo(root) / entry for entry in entries)
|
||||
infos.extend(fs.path.join(root, entry) for entry in entries)
|
||||
if not recursive:
|
||||
break
|
||||
except NotADirectoryError:
|
||||
infos.append(path_info)
|
||||
infos.append(fs_path)
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
|
||||
@@ -69,9 +69,9 @@ def _ls(fs, path_info, recursive=None, dvc_only=False):
|
||||
metadata = fs.metadata(info)
|
||||
if metadata.output_exists or not dvc_only:
|
||||
path = (
|
||||
path_info.name
|
||||
if path_info == info
|
||||
else str(info.relative_to(path_info))
|
||||
fs.path.name(fs_path)
|
||||
if fs_path == info
|
||||
else fs.path.relpath(info, fs_path)
|
||||
)
|
||||
ret[path] = {
|
||||
"isout": metadata.is_output,
|
||||
|
||||
+13
-10
@@ -1,11 +1,12 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.output import Output
|
||||
from dvc.repo import locked
|
||||
from dvc.repo.collect import DvcPaths, collect
|
||||
from dvc.repo.live import summary_path_info
|
||||
from dvc.repo.collect import StrPaths, collect
|
||||
from dvc.repo.live import summary_fs_path
|
||||
from dvc.scm.base import SCMError
|
||||
from dvc.utils import error_handler, errored_revisions, onerror_collect
|
||||
from dvc.utils.serialize import load_yaml
|
||||
@@ -17,27 +18,27 @@ def _is_metric(out: Output) -> bool:
|
||||
return bool(out.metric) or bool(out.live)
|
||||
|
||||
|
||||
def _to_path_infos(metrics: List[Output]) -> DvcPaths:
|
||||
def _to_fs_paths(metrics: List[Output]) -> StrPaths:
|
||||
result = []
|
||||
for out in metrics:
|
||||
if out.metric:
|
||||
result.append(out.path_info)
|
||||
result.append(out.fs_path)
|
||||
elif out.live:
|
||||
path_info = summary_path_info(out)
|
||||
if path_info:
|
||||
result.append(path_info)
|
||||
fs_path = summary_fs_path(out)
|
||||
if fs_path:
|
||||
result.append(fs_path)
|
||||
return result
|
||||
|
||||
|
||||
def _collect_metrics(repo, targets, revision, recursive):
|
||||
metrics, path_infos = collect(
|
||||
metrics, fs_paths = collect(
|
||||
repo,
|
||||
targets=targets,
|
||||
output_filter=_is_metric,
|
||||
recursive=recursive,
|
||||
rev=revision,
|
||||
)
|
||||
return _to_path_infos(metrics) + list(path_infos)
|
||||
return _to_fs_paths(metrics) + list(fs_paths)
|
||||
|
||||
|
||||
def _extract_metrics(metrics, path, rev):
|
||||
@@ -80,7 +81,9 @@ def _read_metrics(repo, metrics, rev, onerror=None):
|
||||
if not fs.isfile(metric):
|
||||
continue
|
||||
|
||||
res[str(metric)] = _read_metric(metric, fs, rev, onerror=onerror)
|
||||
res[fs.path.relpath(metric, os.getcwd())] = _read_metric(
|
||||
metric, fs, rev, onerror=onerror
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
+23
-24
@@ -1,10 +1,10 @@
|
||||
import logging
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from copy import copy
|
||||
from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from dvc.dependency.param import ParamsDependency
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import locked
|
||||
from dvc.repo.collect import collect
|
||||
from dvc.scm.base import SCMError
|
||||
@@ -16,7 +16,6 @@ from dvc.utils.serialize import LOADERS
|
||||
if TYPE_CHECKING:
|
||||
from dvc.output import Output
|
||||
from dvc.repo import Repo
|
||||
from dvc.types import DvcPath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -27,56 +26,58 @@ def _is_params(dep: "Output"):
|
||||
|
||||
def _collect_configs(
|
||||
repo: "Repo", rev, targets=None
|
||||
) -> Tuple[List["Output"], List["DvcPath"]]:
|
||||
) -> Tuple[List["Output"], List[str]]:
|
||||
|
||||
params, path_infos = collect(
|
||||
params, fs_paths = collect(
|
||||
repo,
|
||||
targets=targets or [],
|
||||
deps=True,
|
||||
output_filter=_is_params,
|
||||
rev=rev,
|
||||
)
|
||||
all_path_infos = path_infos + [p.path_info for p in params]
|
||||
all_fs_paths = fs_paths + [p.fs_path for p in params]
|
||||
if not targets:
|
||||
default_params = (
|
||||
PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE
|
||||
default_params = os.path.join(
|
||||
repo.root_dir, ParamsDependency.DEFAULT_PARAMS_FILE
|
||||
)
|
||||
if default_params not in all_path_infos and repo.fs.exists(
|
||||
if default_params not in all_fs_paths and repo.fs.exists(
|
||||
default_params
|
||||
):
|
||||
path_infos.append(default_params)
|
||||
return params, path_infos
|
||||
fs_paths.append(default_params)
|
||||
return params, fs_paths
|
||||
|
||||
|
||||
@error_handler
|
||||
def _read_path_info(fs, path_info, **kwargs):
|
||||
suffix = path_info.suffix.lower()
|
||||
def _read_fs_path(fs, fs_path, **kwargs):
|
||||
suffix = fs.path.suffix(fs_path).lower()
|
||||
loader = LOADERS[suffix]
|
||||
return loader(path_info, fs=fs)
|
||||
return loader(fs_path, fs=fs)
|
||||
|
||||
|
||||
def _read_params(
|
||||
repo,
|
||||
params,
|
||||
params_path_infos,
|
||||
params_fs_paths,
|
||||
deps=False,
|
||||
onerror: Optional[Callable] = None,
|
||||
):
|
||||
res: Dict[str, Dict] = defaultdict(dict)
|
||||
path_infos = copy(params_path_infos)
|
||||
fs_paths = copy(params_fs_paths)
|
||||
|
||||
if deps:
|
||||
for param in params:
|
||||
params_dict = error_handler(param.read_params_d)(onerror=onerror)
|
||||
if params_dict:
|
||||
res[str(param.path_info)] = params_dict
|
||||
res[
|
||||
repo.fs.path.relpath(param.fs_path, os.getcwd())
|
||||
] = params_dict
|
||||
else:
|
||||
path_infos += [param.path_info for param in params]
|
||||
fs_paths += [param.fs_path for param in params]
|
||||
|
||||
for path_info in path_infos:
|
||||
from_path = _read_path_info(repo.fs, path_info, onerror=onerror)
|
||||
for fs_path in fs_paths:
|
||||
from_path = _read_fs_path(repo.fs, fs_path, onerror=onerror)
|
||||
if from_path:
|
||||
res[str(path_info)] = from_path
|
||||
res[repo.fs.path.relpath(fs_path, os.getcwd())] = from_path
|
||||
|
||||
return res
|
||||
|
||||
@@ -131,13 +132,11 @@ def show(repo, revs=None, targets=None, deps=False, onerror: Callable = None):
|
||||
|
||||
|
||||
def _gather_params(repo, rev, targets=None, deps=False, onerror=None):
|
||||
param_outs, params_path_infos = _collect_configs(
|
||||
repo, rev, targets=targets
|
||||
)
|
||||
param_outs, params_fs_paths = _collect_configs(repo, rev, targets=targets)
|
||||
params = _read_params(
|
||||
repo,
|
||||
params=param_outs,
|
||||
params_path_infos=params_path_infos,
|
||||
params_fs_paths=params_fs_paths,
|
||||
deps=deps,
|
||||
onerror=onerror,
|
||||
)
|
||||
|
||||
@@ -20,7 +20,6 @@ from dvc.utils.serialize import LOADERS
|
||||
if TYPE_CHECKING:
|
||||
from dvc.output import Output
|
||||
from dvc.repo import Repo
|
||||
from dvc.types import DvcPath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -89,16 +88,14 @@ class Plots:
|
||||
fs = RepoFileSystem(self.repo)
|
||||
plots = _collect_plots(self.repo, targets, revision, recursive)
|
||||
res = {}
|
||||
for path_info, rev_props in plots.items():
|
||||
for fs_path, rev_props in plots.items():
|
||||
|
||||
if fs.isdir(path_info):
|
||||
if fs.isdir(fs_path):
|
||||
plot_files = []
|
||||
for pi in fs.walk_files(path_info):
|
||||
for pi in fs.find(fs_path):
|
||||
plot_files.append((pi, relpath(pi, self.repo.root_dir)))
|
||||
else:
|
||||
plot_files = [
|
||||
(path_info, relpath(path_info, self.repo.root_dir))
|
||||
]
|
||||
plot_files = [(fs_path, relpath(fs_path, self.repo.root_dir))]
|
||||
|
||||
props = props or {}
|
||||
|
||||
@@ -205,10 +202,10 @@ def _collect_plots(
|
||||
targets: List[str] = None,
|
||||
rev: str = None,
|
||||
recursive: bool = False,
|
||||
) -> Dict["DvcPath", Dict]:
|
||||
) -> Dict["List[str]", Dict]:
|
||||
from dvc.repo.collect import collect
|
||||
|
||||
plots, path_infos = collect(
|
||||
plots, fs_paths = collect(
|
||||
repo,
|
||||
output_filter=_is_plot,
|
||||
targets=targets,
|
||||
@@ -216,8 +213,8 @@ def _collect_plots(
|
||||
recursive=recursive,
|
||||
)
|
||||
|
||||
result = {plot.path_info: _plot_props(plot) for plot in plots}
|
||||
result.update({path_info: {} for path_info in path_infos})
|
||||
result = {plot.fs_path: _plot_props(plot) for plot in plots}
|
||||
result.update({fs_path: {} for fs_path in fs_paths})
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -64,16 +64,16 @@ def _track_stage(stage):
|
||||
if (
|
||||
not dep.use_scm_ignore
|
||||
and dep.is_in_repo
|
||||
and not stage.repo.repo_fs.isdvc(dep.path_info)
|
||||
and not stage.repo.repo_fs.isdvc(dep.fs_path)
|
||||
):
|
||||
stage.repo.scm.track_file(relpath(dep.path_info))
|
||||
stage.repo.scm.track_file(relpath(dep.fs_path))
|
||||
for out in stage.outs:
|
||||
if not out.use_scm_ignore and out.is_in_repo:
|
||||
stage.repo.scm.track_file(relpath(out.path_info))
|
||||
stage.repo.scm.track_file(relpath(out.fs_path))
|
||||
if out.live:
|
||||
from dvc.repo.live import summary_path_info
|
||||
from dvc.repo.live import summary_fs_path
|
||||
|
||||
summary = summary_path_info(out)
|
||||
summary = summary_fs_path(out)
|
||||
if summary:
|
||||
stage.repo.scm.track_file(relpath(summary))
|
||||
stage.repo.scm.track_changed_files()
|
||||
|
||||
+3
-5
@@ -20,7 +20,6 @@ from dvc.exceptions import (
|
||||
NoOutputOrStageError,
|
||||
OutputNotFoundError,
|
||||
)
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import lock_repo
|
||||
from dvc.utils import parse_target, relpath
|
||||
|
||||
@@ -39,7 +38,7 @@ PIPELINE_FILE = "dvc.yaml"
|
||||
|
||||
class StageInfo(NamedTuple):
|
||||
stage: "Stage"
|
||||
filter_info: Optional[PathInfo] = None
|
||||
filter_info: Optional[str] = None
|
||||
|
||||
|
||||
StageList = List["Stage"]
|
||||
@@ -393,7 +392,7 @@ class StageLoad:
|
||||
target: if not provided, all of the stages without any filters are
|
||||
returned.
|
||||
If `target` is a path to a dvc-tracked output,
|
||||
a (stage, output_path_info) is returned.
|
||||
a (stage, output_path) is returned.
|
||||
Otherwise, the details above for `target` in `collect()`
|
||||
applies.
|
||||
|
||||
@@ -409,8 +408,7 @@ class StageLoad:
|
||||
if not (recursive and self.fs.isdir(target)):
|
||||
try:
|
||||
(out,) = self.repo.find_outs_by_path(target, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(target))
|
||||
return [StageInfo(out.stage, filter_info)]
|
||||
return [StageInfo(out.stage, os.path.abspath(target))]
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ def build_outs_trie(stages):
|
||||
|
||||
for stage in stages:
|
||||
for out in stage.outs:
|
||||
out_key = out.path_info.parts
|
||||
out_key = out.fs.path.parts(out.fs_path)
|
||||
|
||||
# Check for dup outs
|
||||
if out_key in outs:
|
||||
|
||||
+19
-22
@@ -6,6 +6,7 @@ from contextlib import contextmanager
|
||||
from voluptuous import Invalid, Optional, Required, Schema
|
||||
|
||||
from .exceptions import DvcException
|
||||
from .fs.local import localfs
|
||||
from .utils import relpath
|
||||
|
||||
INFO_SCHEMA = {Required("pid"): int, Required("cmd"): str}
|
||||
@@ -59,11 +60,11 @@ def _infos_to_str(infos):
|
||||
def _check_blockers(lock, info, *, mode, waiters):
|
||||
from .lock import LockError
|
||||
|
||||
for path_info in waiters:
|
||||
for waiter_path in waiters:
|
||||
blockers = [
|
||||
blocker
|
||||
for path, infos in lock[mode].items()
|
||||
if path_info.overlaps(path)
|
||||
if localfs.path.overlaps(waiter_path, path)
|
||||
for blocker in (infos if isinstance(infos, list) else [infos])
|
||||
if blocker != info
|
||||
]
|
||||
@@ -77,38 +78,34 @@ def _check_blockers(lock, info, *, mode, waiters):
|
||||
"\n"
|
||||
"If there are no processes with such PIDs, you can manually "
|
||||
"remove '.dvc/tmp/rwlock' and try again.".format(
|
||||
path=str(path_info), blockers=_infos_to_str(blockers)
|
||||
path=waiter_path, blockers=_infos_to_str(blockers)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _acquire_read(lock, info, path_infos):
|
||||
def _acquire_read(lock, info, paths):
|
||||
changes = []
|
||||
|
||||
for path_info in path_infos:
|
||||
url = path_info.url
|
||||
|
||||
readers = lock["read"][url]
|
||||
for path in paths:
|
||||
readers = lock["read"][path]
|
||||
if info in readers:
|
||||
continue
|
||||
|
||||
changes.append(url)
|
||||
changes.append(path)
|
||||
readers.append(info)
|
||||
|
||||
return changes
|
||||
|
||||
|
||||
def _acquire_write(lock, info, path_infos):
|
||||
def _acquire_write(lock, info, paths):
|
||||
changes = []
|
||||
|
||||
for path_info in path_infos:
|
||||
url = path_info.url
|
||||
|
||||
if lock["write"][url] == info:
|
||||
for path in paths:
|
||||
if lock["write"][path] == info:
|
||||
continue
|
||||
|
||||
changes.append(url)
|
||||
lock["write"][url] = info
|
||||
changes.append(path)
|
||||
lock["write"][path] = info
|
||||
|
||||
return changes
|
||||
|
||||
@@ -137,17 +134,17 @@ def _release_read(lock, info, changes):
|
||||
|
||||
@contextmanager
|
||||
def rwlock(tmp_dir, cmd, read, write):
|
||||
"""Create non-thread-safe RWLock for PathInfos.
|
||||
"""Create non-thread-safe RWLock for file paths.
|
||||
|
||||
Args:
|
||||
tmp_dir (str): existing directory where to create the rwlock file.
|
||||
cmd (str): command that will be working on these PathInfos.
|
||||
read ([PathInfo]): PathInfos that are going to be read.
|
||||
write ([PathInfo]): PathInfos that are going to be written.
|
||||
cmd (str): command that will be working on these file path.
|
||||
read ([str]): file paths that are going to be read.
|
||||
write ([str]): file paths that are going to be written.
|
||||
|
||||
Raises:
|
||||
LockError: raised if PathInfo we want to read is being written to by
|
||||
another command or if PathInfo we want to write is being written
|
||||
LockError: raised if file paths we want to read is being written to by
|
||||
another command or if file paths we want to write is being written
|
||||
to or read from by another command.
|
||||
RWLockFileCorruptedError: raised if rwlock file is not a valid JSON.
|
||||
RWLockFileFormatError: raised if rwlock file is a valid JSON, but
|
||||
|
||||
@@ -18,7 +18,6 @@ from typing import (
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.scm.base import GitAuthError, InvalidRemoteSCMRepo, SCMError
|
||||
from dvc.utils import relpath
|
||||
@@ -105,22 +104,22 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
except NotGitRepository as exc:
|
||||
raise SCMError(f"{root_dir} is not a git repository") from exc
|
||||
|
||||
self._submodules: Dict[str, "PathInfo"] = self._find_submodules()
|
||||
self._submodules: Dict[str, str] = self._find_submodules()
|
||||
self._stashes: dict = {}
|
||||
|
||||
def _find_submodules(self) -> Dict[str, "PathInfo"]:
|
||||
def _find_submodules(self) -> Dict[str, str]:
|
||||
"""Return dict mapping submodule names to submodule paths.
|
||||
|
||||
Submodule paths will be relative to Git repo root.
|
||||
"""
|
||||
from dulwich.config import ConfigFile, parse_submodules
|
||||
|
||||
submodules: Dict[str, "PathInfo"] = {}
|
||||
submodules: Dict[str, str] = {}
|
||||
config_path = os.path.join(self.root_dir, ".gitmodules")
|
||||
if os.path.isfile(config_path):
|
||||
config = ConfigFile.from_path(config_path)
|
||||
for path, _url, section in parse_submodules(config):
|
||||
submodules[os.fsdecode(section)] = PathInfo(os.fsdecode(path))
|
||||
submodules[os.fsdecode(section)] = os.fsdecode(path)
|
||||
return submodules
|
||||
|
||||
def close(self):
|
||||
@@ -163,11 +162,12 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
# parent git repo root). We append path to root_dir here so
|
||||
# that the result of relpath(path, root_dir) is actually the
|
||||
# path relative to the submodule root.
|
||||
path_info = PathInfo(path).relative_to(self.root_dir)
|
||||
fs_path = relpath(path, self.root_dir)
|
||||
for sm_path in self._submodules.values():
|
||||
if path_info.isin(sm_path):
|
||||
if fs_path.startswith(sm_path):
|
||||
path = os.path.join(
|
||||
self.root_dir, path_info.relative_to(sm_path)
|
||||
self.root_dir,
|
||||
relpath(fs_path, sm_path),
|
||||
)
|
||||
break
|
||||
if os.path.isdir(path):
|
||||
@@ -241,7 +241,7 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
return untracked
|
||||
|
||||
def is_tracked(self, path: str) -> bool:
|
||||
rel = PathInfo(path).relative_to(self.root_dir).as_posix().encode()
|
||||
rel = relpath(path, self.root_dir).replace(os.path.sep, "/").encode()
|
||||
rel_dir = rel + b"/"
|
||||
for path in self.repo.open_index():
|
||||
if path == rel or path.startswith(rel_dir):
|
||||
|
||||
@@ -273,7 +273,7 @@ class Stage(params.StageParams):
|
||||
from dvc.output import Output
|
||||
from dvc.schema import LIVE_PROPS
|
||||
|
||||
env[DVCLIVE_PATH] = relpath(out.path_info, self.wdir)
|
||||
env[DVCLIVE_PATH] = relpath(out.fs_path, self.wdir)
|
||||
if isinstance(out.live, dict):
|
||||
config = project(out.live, LIVE_PROPS)
|
||||
|
||||
@@ -511,7 +511,7 @@ class Stage(params.StageParams):
|
||||
if not (allow_missing or out.checkpoint):
|
||||
raise
|
||||
except CacheLinkError:
|
||||
link_failures.append(out.path_info)
|
||||
link_failures.append(out.fs_path)
|
||||
if link_failures:
|
||||
raise CacheLinkError(link_failures)
|
||||
|
||||
@@ -559,11 +559,11 @@ class Stage(params.StageParams):
|
||||
def _check_missing_outputs(self):
|
||||
check_missing_outputs(self)
|
||||
|
||||
def filter_outs(self, path_info):
|
||||
def filter_outs(self, fs_path):
|
||||
def _func(o):
|
||||
return path_info.isin_or_eq(o.path_info)
|
||||
return o.fs.path.isin_or_eq(fs_path, o.fs_path)
|
||||
|
||||
return filter(_func, self.outs) if path_info else self.outs
|
||||
return filter(_func, self.outs) if fs_path else self.outs
|
||||
|
||||
@rwlocked(write=["outs"])
|
||||
def checkout(self, allow_missing=False, **kwargs):
|
||||
@@ -585,7 +585,7 @@ class Stage(params.StageParams):
|
||||
added, modified = result or (None, None)
|
||||
if not (added or modified):
|
||||
return None, []
|
||||
return "modified" if modified else "added", [out.path_info]
|
||||
return "modified" if modified else "added", [str(out)]
|
||||
except CheckoutError as exc:
|
||||
return "failed", exc.target_infos
|
||||
|
||||
|
||||
+19
-16
@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Optional
|
||||
from funcy import cached_property, first
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.utils import dict_sha256, relpath
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -167,13 +166,14 @@ class StageCache:
|
||||
# sanity check
|
||||
COMPILED_LOCK_FILE_STAGE_SCHEMA(cache)
|
||||
|
||||
path = PathInfo(self._get_cache_path(cache_key, cache_value))
|
||||
self.repo.odb.local.makedirs(path.parent)
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, dir=path.parent).name
|
||||
assert os.path.exists(path.parent)
|
||||
assert os.path.isdir(path.parent)
|
||||
path = self._get_cache_path(cache_key, cache_value)
|
||||
parent = self.repo.odb.local.fs.path.parent(path)
|
||||
self.repo.odb.local.makedirs(parent)
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, dir=parent).name
|
||||
assert os.path.exists(parent)
|
||||
assert os.path.isdir(parent)
|
||||
dump_yaml(tmp, cache)
|
||||
self.repo.odb.local.move(PathInfo(tmp), path)
|
||||
self.repo.odb.local.move(tmp, path)
|
||||
|
||||
def restore(self, stage, run_cache=True, pull=False):
|
||||
from .serialize import to_single_stage_lockfile
|
||||
@@ -214,21 +214,24 @@ class StageCache:
|
||||
def _transfer(func, from_remote, to_remote):
|
||||
ret = []
|
||||
|
||||
runs = from_remote.path_info / "runs"
|
||||
runs = from_remote.fs.path.join(from_remote.fs_path, "runs")
|
||||
if not from_remote.fs.exists(runs):
|
||||
return []
|
||||
|
||||
for src in from_remote.fs.walk_files(runs):
|
||||
rel = src.relative_to(from_remote.path_info)
|
||||
dst = to_remote.path_info / rel
|
||||
key = dst.parent
|
||||
from_path = from_remote.fs.path
|
||||
for src in from_remote.fs.find(runs):
|
||||
rel = from_path.relpath(src, from_remote.fs_path)
|
||||
dst = to_remote.fs.path.join(to_remote.fs_path, rel)
|
||||
key = to_remote.fs.path.parent(dst)
|
||||
# check if any build cache already exists for this key
|
||||
if to_remote.fs.exists(key) and first(
|
||||
to_remote.fs.walk_files(key)
|
||||
):
|
||||
# TODO: check if MaxKeys=1 or something like that applies
|
||||
# or otherwise this will take a lot of time!
|
||||
if to_remote.fs.exists(key) and first(to_remote.fs.find(key)):
|
||||
continue
|
||||
func(src, dst)
|
||||
ret.append((src.parent.name, src.name))
|
||||
ret.append(
|
||||
(from_path.name(from_path.parent(src)), from_path.name(src))
|
||||
)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ def rwlocked(call, read=None, write=None):
|
||||
|
||||
def _chain(names):
|
||||
return [
|
||||
item.path_info
|
||||
item.fs_path
|
||||
for attr in names
|
||||
for item in getattr(stage, attr)
|
||||
# There is no need to lock RepoDependency deps, as there is no
|
||||
|
||||
+1
-2
@@ -10,7 +10,6 @@ from dvc.hash_info import HashInfo
|
||||
from dvc.objects.meta import Meta
|
||||
from dvc.parsing import FOREACH_KWD, JOIN, DataResolver, EntryNotFound
|
||||
from dvc.parsing.versions import LOCKFILE_VERSION
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
from . import PipelineStage, Stage, loads_from
|
||||
from .exceptions import StageNameUnspecified, StageNotFound
|
||||
@@ -36,7 +35,7 @@ class StageLoader(Mapping):
|
||||
|
||||
@cached_property
|
||||
def resolver(self):
|
||||
wdir = PathInfo(self.dvcfile.path).parent
|
||||
wdir = self.repo.fs.path.parent(self.dvcfile.path)
|
||||
return DataResolver(self.repo, wdir, self.data)
|
||||
|
||||
@cached_property
|
||||
|
||||
@@ -88,7 +88,7 @@ class LiveTask(MonitorTask):
|
||||
|
||||
def __init__(self, stage: "Stage", out: "Output", proc: subprocess.Popen):
|
||||
self.output_path = os.path.join(
|
||||
os.getcwd(), out.path_info.fspath + "_dvc_plots", "index.html"
|
||||
os.getcwd(), out.fs_path + "_dvc_plots", "index.html"
|
||||
)
|
||||
ui.write(f"Live summary will be created at:\n{self.output_path}")
|
||||
super().__init__(stage, functools.partial(create_summary, out), proc)
|
||||
|
||||
+3
-3
@@ -142,8 +142,8 @@ def check_no_externals(stage):
|
||||
def check_circular_dependency(stage):
|
||||
from dvc.exceptions import CircularDependencyError
|
||||
|
||||
circular_dependencies = {d.path_info for d in stage.deps} & {
|
||||
o.path_info for o in stage.outs
|
||||
circular_dependencies = {d.fs_path for d in stage.deps} & {
|
||||
o.fs_path for o in stage.outs
|
||||
}
|
||||
|
||||
if circular_dependencies:
|
||||
@@ -155,7 +155,7 @@ def check_duplicated_arguments(stage):
|
||||
|
||||
from dvc.exceptions import ArgumentDuplicationError
|
||||
|
||||
path_counts = Counter(edge.path_info for edge in stage.deps + stage.outs)
|
||||
path_counts = Counter(edge.fs_path for edge in stage.deps + stage.outs)
|
||||
|
||||
for path, occurrence in path_counts.items():
|
||||
if occurrence > 1:
|
||||
|
||||
+19
-19
@@ -18,15 +18,15 @@ class StateBase(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def save(self, path_info, fs, hash_info):
|
||||
def save(self, path, fs, hash_info):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get(self, path_info, fs):
|
||||
def get(self, path, fs):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def save_link(self, path_info, fs):
|
||||
def save_link(self, path, fs):
|
||||
pass
|
||||
|
||||
|
||||
@@ -34,13 +34,13 @@ class StateNoop(StateBase):
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
def save(self, path_info, fs, hash_info):
|
||||
def save(self, path, fs, hash_info):
|
||||
pass
|
||||
|
||||
def get(self, path_info, fs): # pylint: disable=unused-argument
|
||||
def get(self, path, fs): # pylint: disable=unused-argument
|
||||
return None, None
|
||||
|
||||
def save_link(self, path_info, fs):
|
||||
def save_link(self, path, fs):
|
||||
pass
|
||||
|
||||
|
||||
@@ -68,19 +68,19 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
self.md5s.close()
|
||||
self.links.close()
|
||||
|
||||
def save(self, path_info, fs, hash_info):
|
||||
def save(self, path, fs, hash_info):
|
||||
"""Save hash for the specified path info.
|
||||
|
||||
Args:
|
||||
path_info (dict): path_info to save hash for.
|
||||
path (str): path to save hash for.
|
||||
hash_info (HashInfo): hash to save.
|
||||
"""
|
||||
|
||||
if not isinstance(fs, LocalFileSystem):
|
||||
return
|
||||
|
||||
mtime, size = get_mtime_and_size(path_info, fs, self.dvcignore)
|
||||
inode = get_inode(path_info)
|
||||
mtime, size = get_mtime_and_size(path, fs, self.dvcignore)
|
||||
inode = get_inode(path)
|
||||
|
||||
logger.debug(
|
||||
"state save (%s, %s, %s) %s",
|
||||
@@ -92,12 +92,12 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
|
||||
self.md5s[inode] = (mtime, str(size), hash_info.value)
|
||||
|
||||
def get(self, path_info, fs):
|
||||
def get(self, path, fs):
|
||||
"""Gets the hash for the specified path info. Hash will be
|
||||
retrieved from the state database if available.
|
||||
|
||||
Args:
|
||||
path_info (dict): path info to get the hash for.
|
||||
path (str): path info to get the hash for.
|
||||
|
||||
Returns:
|
||||
HashInfo or None: hash for the specified path info or None if it
|
||||
@@ -109,11 +109,11 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
return None, None
|
||||
|
||||
try:
|
||||
mtime, size = get_mtime_and_size(path_info, fs, self.dvcignore)
|
||||
mtime, size = get_mtime_and_size(path, fs, self.dvcignore)
|
||||
except FileNotFoundError:
|
||||
return None, None
|
||||
|
||||
inode = get_inode(path_info)
|
||||
inode = get_inode(path)
|
||||
|
||||
value = self.md5s.get(inode)
|
||||
|
||||
@@ -122,23 +122,23 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
|
||||
return Meta(size=size), HashInfo("md5", value[2])
|
||||
|
||||
def save_link(self, path_info, fs):
|
||||
def save_link(self, path, fs):
|
||||
"""Adds the specified path to the list of links created by dvc. This
|
||||
list is later used on `dvc checkout` to cleanup old links.
|
||||
|
||||
Args:
|
||||
path_info (dict): path info to add to the list of links.
|
||||
path (str): path info to add to the list of links.
|
||||
"""
|
||||
if not isinstance(fs, LocalFileSystem):
|
||||
return
|
||||
|
||||
try:
|
||||
mtime, _ = get_mtime_and_size(path_info, fs, self.dvcignore)
|
||||
mtime, _ = get_mtime_and_size(path, fs, self.dvcignore)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
|
||||
inode = get_inode(path_info)
|
||||
relative_path = relpath(path_info, self.root_dir)
|
||||
inode = get_inode(path)
|
||||
relative_path = relpath(path, self.root_dir)
|
||||
|
||||
with self.links as ref:
|
||||
ref[relative_path] = (inode, mtime)
|
||||
|
||||
+1
-4
@@ -1,13 +1,10 @@
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
|
||||
|
||||
from dvc.path_info import PathInfo, URLInfo
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from os import PathLike
|
||||
|
||||
StrPath = Union[str, "PathLike[str]"]
|
||||
DvcPath = Union[PathInfo, URLInfo]
|
||||
AnyPath = Union[str, DvcPath, StrPath]
|
||||
AnyPath = Union[str, StrPath]
|
||||
|
||||
OptStr = Optional[str]
|
||||
TargetType = Union[List[str], str]
|
||||
|
||||
@@ -363,22 +363,24 @@ def resolve_output(inp, out):
|
||||
def resolve_paths(repo, out, always_local=False):
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.fs.local import localfs
|
||||
|
||||
from ..dvcfile import DVC_FILE_SUFFIX
|
||||
from ..exceptions import DvcException
|
||||
from ..path_info import PathInfo
|
||||
from ..system import System
|
||||
from .fs import contains_symlink_up_to
|
||||
|
||||
abspath = PathInfo(os.path.abspath(out))
|
||||
abspath = os.path.abspath(out)
|
||||
dirname = os.path.dirname(abspath)
|
||||
base = os.path.basename(os.path.normpath(out))
|
||||
|
||||
scheme = urlparse(out).scheme
|
||||
if os.name == "nt" and scheme == abspath.drive[0].lower():
|
||||
|
||||
if os.name == "nt" and scheme == localfs.path.drive(abspath)[0].lower():
|
||||
# urlparse interprets windows drive letters as URL scheme
|
||||
scheme = ""
|
||||
|
||||
if scheme or not abspath.isin_or_eq(repo.root_dir):
|
||||
if scheme or not localfs.path.isin_or_eq(abspath, repo.root_dir):
|
||||
wdir = os.getcwd()
|
||||
elif contains_symlink_up_to(dirname, repo.root_dir) or (
|
||||
os.path.isdir(abspath) and System.is_symlink(abspath)
|
||||
|
||||
+4
-4
@@ -42,9 +42,9 @@ def get_mtime_and_size(path, fs, dvcignore=None):
|
||||
size = 0
|
||||
files_mtimes = {}
|
||||
if dvcignore:
|
||||
walk_iterator = dvcignore.walk_files(fs, path)
|
||||
walk_iterator = dvcignore.find(fs, path)
|
||||
else:
|
||||
walk_iterator = fs.walk_files(path)
|
||||
walk_iterator = fs.find(path)
|
||||
for file_path in walk_iterator:
|
||||
try:
|
||||
stats = fs.info(file_path)
|
||||
@@ -54,7 +54,7 @@ def get_mtime_and_size(path, fs, dvcignore=None):
|
||||
raise
|
||||
continue
|
||||
size += stats["size"]
|
||||
files_mtimes[os.fspath(file_path)] = stats["mtime"]
|
||||
files_mtimes[file_path] = stats["mtime"]
|
||||
|
||||
# We track file changes and moves, which cannot be detected with simply
|
||||
# max(mtime(f) for f in non_ignored_files)
|
||||
@@ -244,7 +244,7 @@ def walk_files(directory):
|
||||
def as_atomic(fs, to_info):
|
||||
from dvc.utils import tmp_fname
|
||||
|
||||
tmp_info = to_info.parent / tmp_fname()
|
||||
tmp_info = fs.path.join(fs.path.parent(to_info), tmp_fname())
|
||||
try:
|
||||
yield tmp_info
|
||||
except BaseException:
|
||||
|
||||
@@ -81,6 +81,10 @@ disable_other_loggers()
|
||||
class TmpDir(pathlib.Path):
|
||||
scheme = "local"
|
||||
|
||||
@property
|
||||
def fs_path(self):
|
||||
return str(self)
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if cls is TmpDir:
|
||||
cls = ( # pylint: disable=self-cls-assignment
|
||||
@@ -249,8 +253,8 @@ class TmpDir(pathlib.Path):
|
||||
}
|
||||
return super().read_text(*args, encoding="utf-8", **kwargs)
|
||||
|
||||
def hash_to_path_info(self, hash_):
|
||||
return self / hash_[0:2] / hash_[2:]
|
||||
def hash_to_path(self, hash_):
|
||||
return str(self / hash_[0:2] / hash_[2:])
|
||||
|
||||
def dump(self, *args, **kwargs):
|
||||
return serialize.DUMPERS[self.suffix](self, *args, **kwargs)
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -27,11 +25,7 @@ def run_copy_metrics(tmp_dir, run_copy):
|
||||
|
||||
if hasattr(tmp_dir.dvc, "scm"):
|
||||
files = [stage.path]
|
||||
files += [
|
||||
os.fspath(out.path_info)
|
||||
for out in stage.outs
|
||||
if not out.use_cache
|
||||
]
|
||||
files += [out.fs_path for out in stage.outs if not out.use_cache]
|
||||
tmp_dir.dvc.scm.add(files)
|
||||
if commit:
|
||||
tmp_dir.dvc.scm.commit(commit)
|
||||
|
||||
@@ -5,7 +5,6 @@ from functools import partial
|
||||
import pytest
|
||||
|
||||
from dvc.fs.ssh import SSHFileSystem
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.repo.experiments.base import EXEC_HEAD, EXEC_MERGE
|
||||
from dvc.repo.experiments.executor.ssh import SSHExecutor
|
||||
from tests.func.machine.conftest import * # noqa, pylint: disable=wildcard-import
|
||||
@@ -39,7 +38,7 @@ def test_init_git(tmp_dir, scm, cloud):
|
||||
rev = scm.resolve_rev("stash@{0}")
|
||||
scm.set_ref(EXEC_MERGE, rev)
|
||||
|
||||
root_url = URLInfo(str(cloud)) / SSHExecutor.gen_dirname()
|
||||
root_url = cloud / SSHExecutor.gen_dirname()
|
||||
|
||||
executor = SSHExecutor(
|
||||
scm,
|
||||
@@ -65,7 +64,7 @@ def test_init_cache(tmp_dir, dvc, scm, cloud):
|
||||
rev = scm.get_rev()
|
||||
scm.set_ref(EXEC_HEAD, rev)
|
||||
scm.set_ref(EXEC_MERGE, rev)
|
||||
root_url = URLInfo(str(cloud)) / SSHExecutor.gen_dirname()
|
||||
root_url = cloud / SSHExecutor.gen_dirname()
|
||||
|
||||
executor = SSHExecutor(
|
||||
scm,
|
||||
|
||||
@@ -32,9 +32,9 @@ def test_new_checkpoint(
|
||||
if rev == "workspace":
|
||||
continue
|
||||
fs = dvc.repo_fs
|
||||
with fs.open(tmp_dir / "foo") as fobj:
|
||||
with fs.open((tmp_dir / "foo").fs_path) as fobj:
|
||||
assert fobj.read().strip() == str(checkpoint_stage.iterations)
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
with fs.open((tmp_dir / "metrics.yaml").fs_path) as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
if workspace:
|
||||
@@ -81,9 +81,9 @@ def test_resume_checkpoint(
|
||||
if rev == "workspace":
|
||||
continue
|
||||
fs = dvc.repo_fs
|
||||
with fs.open(tmp_dir / "foo") as fobj:
|
||||
with fs.open((tmp_dir / "foo").fs_path) as fobj:
|
||||
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
with fs.open((tmp_dir / "metrics.yaml").fs_path) as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
if workspace:
|
||||
@@ -112,9 +112,9 @@ def test_reset_checkpoint(
|
||||
if rev == "workspace":
|
||||
continue
|
||||
fs = dvc.repo_fs
|
||||
with fs.open(tmp_dir / "foo") as fobj:
|
||||
with fs.open((tmp_dir / "foo").fs_path) as fobj:
|
||||
assert fobj.read().strip() == str(checkpoint_stage.iterations)
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
with fs.open((tmp_dir / "metrics.yaml").fs_path) as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
if workspace:
|
||||
@@ -151,18 +151,18 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage, workspace):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
fs = dvc.repo_fs
|
||||
with fs.open(tmp_dir / "foo") as fobj:
|
||||
with fs.open((tmp_dir / "foo").fs_path) as fobj:
|
||||
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
with fs.open((tmp_dir / "metrics.yaml").fs_path) as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
for rev in dvc.brancher([checkpoint_b]):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
fs = dvc.repo_fs
|
||||
with fs.open(tmp_dir / "foo") as fobj:
|
||||
with fs.open((tmp_dir / "foo").fs_path) as fobj:
|
||||
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
with fs.open((tmp_dir / "metrics.yaml").fs_path) as fobj:
|
||||
assert fobj.read().strip() == "foo: 100"
|
||||
|
||||
with pytest.raises(MultipleBranchError):
|
||||
|
||||
@@ -670,9 +670,9 @@ def test_modified_data_dep(tmp_dir, scm, dvc, workspace, params, target):
|
||||
for rev in dvc.brancher(revs=[exp]):
|
||||
if rev != exp:
|
||||
continue
|
||||
with dvc.repo_fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
with dvc.repo_fs.open((tmp_dir / "metrics.yaml").fs_path) as fobj:
|
||||
assert fobj.read().strip() == params
|
||||
with dvc.repo_fs.open(tmp_dir / "data") as fobj:
|
||||
with dvc.repo_fs.open((tmp_dir / "data").fs_path) as fobj:
|
||||
assert fobj.read().strip() == "modified"
|
||||
|
||||
if workspace:
|
||||
|
||||
@@ -61,8 +61,8 @@ def test_clear_on_download_err(tmp_dir, dvc, index, mocker):
|
||||
dvc.push()
|
||||
|
||||
for _, _, oid in out.obj:
|
||||
remove(dvc.odb.local.get(oid).path_info)
|
||||
remove(out.path_info)
|
||||
remove(dvc.odb.local.get(oid).fs_path)
|
||||
remove(out.fs_path)
|
||||
|
||||
assert list(index.hashes())
|
||||
|
||||
|
||||
@@ -39,7 +39,10 @@ USED_VARS = {
|
||||
|
||||
def make_entry_definition(wdir, name, data, context=None) -> EntryDefinition:
|
||||
return EntryDefinition(
|
||||
DataResolver(wdir.dvc, wdir, {}), context or Context(), name, data
|
||||
DataResolver(wdir.dvc, wdir.fs_path, {}),
|
||||
context or Context(),
|
||||
name,
|
||||
data,
|
||||
)
|
||||
|
||||
|
||||
@@ -47,7 +50,7 @@ def make_foreach_def(
|
||||
wdir, name, foreach_data, do_data=None, context=None
|
||||
) -> ForeachDefinition:
|
||||
return ForeachDefinition(
|
||||
DataResolver(wdir.dvc, wdir, {}),
|
||||
DataResolver(wdir.dvc, wdir.fs_path, {}),
|
||||
context or Context(),
|
||||
name,
|
||||
{"foreach": foreach_data, "do": do_data or {}},
|
||||
|
||||
@@ -9,7 +9,7 @@ from dvc.parsing.context import Context
|
||||
|
||||
def test_with_simple_list_data(tmp_dir, dvc):
|
||||
"""Testing a simple non-nested list as a foreach data"""
|
||||
resolver = DataResolver(dvc, tmp_dir, {})
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
|
||||
|
||||
context = Context()
|
||||
data = {"foreach": ["foo", "bar", "baz"], "do": {"cmd": "echo ${item}"}}
|
||||
@@ -24,7 +24,7 @@ def test_with_simple_list_data(tmp_dir, dvc):
|
||||
|
||||
|
||||
def test_with_dict_data(tmp_dir, dvc):
|
||||
resolver = DataResolver(dvc, tmp_dir, {})
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
|
||||
context = Context()
|
||||
|
||||
foreach_data = {"model1": "foo", "model2": "bar"}
|
||||
@@ -45,7 +45,7 @@ def test_with_dict_data(tmp_dir, dvc):
|
||||
|
||||
|
||||
def test_with_composite_list(tmp_dir, dvc):
|
||||
resolver = DataResolver(dvc, tmp_dir, {})
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
|
||||
|
||||
context = Context()
|
||||
foreach_data = [{"thresh": "foo"}, {"thresh": "bar"}]
|
||||
@@ -64,7 +64,7 @@ def test_with_composite_list(tmp_dir, dvc):
|
||||
def test_foreach_interpolated_simple_list(tmp_dir, dvc):
|
||||
foreach_data = ["foo", "bar", "baz"]
|
||||
vars_ = {"models": foreach_data}
|
||||
resolver = DataResolver(dvc, tmp_dir, {"vars": [vars_]})
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, {"vars": [vars_]})
|
||||
data = {"foreach": "${models}", "do": {"cmd": "echo ${item}"}}
|
||||
definition = ForeachDefinition(resolver, resolver.context, "build", data)
|
||||
|
||||
@@ -98,7 +98,7 @@ def test_foreach_interpolate_with_composite_data(
|
||||
tmp_dir, dvc, foreach_def, foreach_data, result
|
||||
):
|
||||
vars_ = [{"models": foreach_data}]
|
||||
resolver = DataResolver(dvc, tmp_dir, {"vars": vars_})
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, {"vars": vars_})
|
||||
data = {"foreach": "${models}", "do": {"cmd": f"echo {foreach_def}"}}
|
||||
definition = ForeachDefinition(resolver, resolver.context, "build", data)
|
||||
|
||||
@@ -112,7 +112,7 @@ def test_params_file_with_dict_tracked(tmp_dir, dvc):
|
||||
params = {"models": foreach_data}
|
||||
(tmp_dir / "params.yaml").dump(params)
|
||||
|
||||
resolver = DataResolver(dvc, tmp_dir, {})
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
|
||||
data = {"foreach": "${models}", "do": {"cmd": "echo ${item.thresh}"}}
|
||||
definition = ForeachDefinition(resolver, resolver.context, "build", data)
|
||||
|
||||
@@ -133,7 +133,7 @@ def test_params_file_tracked_for_composite_list(tmp_dir, dvc):
|
||||
params = {"models": foreach_data}
|
||||
(tmp_dir / "params.yaml").dump(params)
|
||||
|
||||
resolver = DataResolver(dvc, tmp_dir, {})
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
|
||||
data = {"foreach": "${models}", "do": {"cmd": "echo ${item.thresh}"}}
|
||||
definition = ForeachDefinition(resolver, resolver.context, "build", data)
|
||||
|
||||
@@ -150,7 +150,7 @@ def test_params_file_tracked_for_composite_list(tmp_dir, dvc):
|
||||
|
||||
def test_foreach_data_from_nested_vars(tmp_dir, dvc):
|
||||
vars_ = {"models": {"lst": [{"thresh": 10}, {"thresh": 15}]}}
|
||||
resolver = DataResolver(dvc, tmp_dir, {"vars": [vars_]})
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, {"vars": [vars_]})
|
||||
data = {"foreach": "${models.lst}", "do": {"cmd": "echo ${item.thresh}"}}
|
||||
definition = ForeachDefinition(resolver, resolver.context, "build", data)
|
||||
|
||||
@@ -163,7 +163,7 @@ def test_foreach_data_from_nested_vars(tmp_dir, dvc):
|
||||
|
||||
|
||||
def test_foreach_partial_interpolations(tmp_dir, dvc):
|
||||
resolver = DataResolver(dvc, tmp_dir, {"vars": [{"bar": "bar"}]})
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, {"vars": [{"bar": "bar"}]})
|
||||
foreach_data = {"model1": "foo", "model2": "${bar}"}
|
||||
data = {"foreach": foreach_data, "do": {"cmd": "echo ${item}"}}
|
||||
definition = ForeachDefinition(resolver, resolver.context, "build", data)
|
||||
@@ -180,7 +180,9 @@ def test_mixed_vars_for_foreach_data(tmp_dir, dvc):
|
||||
(tmp_dir / "params.yaml").dump({"models": {"model1": "foo"}})
|
||||
(tmp_dir / "test_params.yaml").dump({"models": {"model2": "bar"}})
|
||||
|
||||
resolver = DataResolver(dvc, tmp_dir, {"vars": ["test_params.yaml"]})
|
||||
resolver = DataResolver(
|
||||
dvc, tmp_dir.fs_path, {"vars": ["test_params.yaml"]}
|
||||
)
|
||||
data = {"foreach": "${models}", "do": {"cmd": "echo ${item}"}}
|
||||
definition = ForeachDefinition(resolver, resolver.context, "build", data)
|
||||
|
||||
@@ -203,7 +205,9 @@ def test_mixed_vars_for_foreach_data_2(tmp_dir, dvc):
|
||||
{"models": {"model1": {"epochs": 5}, "model2": {"epochs": 10}}},
|
||||
)
|
||||
|
||||
resolver = DataResolver(dvc, tmp_dir, {"vars": ["test_params.yaml"]})
|
||||
resolver = DataResolver(
|
||||
dvc, tmp_dir.fs_path, {"vars": ["test_params.yaml"]}
|
||||
)
|
||||
data = {
|
||||
"foreach": "${models}",
|
||||
"do": {"cmd": "echo ${item.thresh} ${item.epochs}"},
|
||||
@@ -233,7 +237,7 @@ def test_mixed_vars_for_foreach_data_2(tmp_dir, dvc):
|
||||
|
||||
|
||||
def test_foreach_with_interpolated_wdir(tmp_dir, dvc):
|
||||
resolver = DataResolver(dvc, tmp_dir / "data", {})
|
||||
resolver = DataResolver(dvc, (tmp_dir / "data").fs_path, {})
|
||||
foreach_data = ["foo", "bar"]
|
||||
data = {
|
||||
"foreach": foreach_data,
|
||||
@@ -254,7 +258,7 @@ def test_foreach_with_interpolated_wdir(tmp_dir, dvc):
|
||||
|
||||
|
||||
def test_foreach_with_local_vars(tmp_dir, dvc):
|
||||
resolver = DataResolver(dvc, tmp_dir / "data", {})
|
||||
resolver = DataResolver(dvc, (tmp_dir / "data").fs_path, {})
|
||||
foreach_data = ["foo", "bar"]
|
||||
data = {
|
||||
"foreach": foreach_data,
|
||||
@@ -289,7 +293,7 @@ def test_foreach_with_imported_vars(tmp_dir, dvc, local_import):
|
||||
(tmp_dir / "test_params.yaml").dump(
|
||||
{"train": {"epochs": 10}, "prepare": {"nums": 25}}
|
||||
)
|
||||
resolver = DataResolver(dvc, tmp_dir, {})
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
|
||||
foreach_data = ["foo", "bar"]
|
||||
data = {
|
||||
"foreach": foreach_data,
|
||||
@@ -327,7 +331,7 @@ def test_foreach_with_interpolated_wdir_and_local_vars(
|
||||
{"train": {"epochs": 1 + i}, "prepare": {"nums": 10 * i}},
|
||||
)
|
||||
|
||||
resolver = DataResolver(dvc, tmp_dir, {})
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
|
||||
data = {
|
||||
"foreach": [0, 1, 2, 3, 4],
|
||||
"do": {
|
||||
@@ -391,7 +395,7 @@ def test_foreach_do_syntax_is_checked_once(tmp_dir, dvc, mocker):
|
||||
do_def = {"cmd": "python script.py --epochs ${item}"}
|
||||
data = {"foreach": [0, 1, 2, 3, 4], "do": do_def}
|
||||
definition = ForeachDefinition(
|
||||
DataResolver(dvc, tmp_dir, {}), Context(), "build", data
|
||||
DataResolver(dvc, tmp_dir.fs_path, {}), Context(), "build", data
|
||||
)
|
||||
mock = mocker.patch("dvc.parsing.check_syntax_errors", return_value=True)
|
||||
definition.resolve_all()
|
||||
@@ -403,7 +407,7 @@ def test_foreach_data_is_only_resolved_once(tmp_dir, dvc, mocker):
|
||||
context = Context(models=["foo", "bar", "baz"])
|
||||
data = {"foreach": "${models}", "do": {}}
|
||||
definition = ForeachDefinition(
|
||||
DataResolver(dvc, tmp_dir, {}), context, "build", data
|
||||
DataResolver(dvc, tmp_dir.fs_path, {}), context, "build", data
|
||||
)
|
||||
mock = mocker.spy(definition, "_resolve_foreach_data")
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import os
|
||||
from copy import deepcopy
|
||||
|
||||
import pytest
|
||||
@@ -5,7 +6,6 @@ import pytest
|
||||
from dvc.dependency import _merge_params
|
||||
from dvc.parsing import DEFAULT_PARAMS_FILE, DataResolver
|
||||
from dvc.parsing.context import recurse_not_a_node
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
from . import (
|
||||
CONTEXT_DATA,
|
||||
@@ -31,7 +31,7 @@ def assert_stage_equal(d1, d2):
|
||||
def test_simple(tmp_dir, dvc):
|
||||
(tmp_dir / DEFAULT_PARAMS_FILE).dump(CONTEXT_DATA)
|
||||
resolver = DataResolver(
|
||||
dvc, PathInfo(str(tmp_dir)), deepcopy(TEMPLATED_DVC_YAML_DATA)
|
||||
dvc, tmp_dir.fs_path, deepcopy(TEMPLATED_DVC_YAML_DATA)
|
||||
)
|
||||
assert_stage_equal(resolver.resolve(), deepcopy(RESOLVED_DVC_YAML_DATA))
|
||||
assert resolver.tracked_vars == {
|
||||
@@ -48,7 +48,7 @@ def test_vars_import(tmp_dir, dvc):
|
||||
(tmp_dir / "params2.yaml").dump(CONTEXT_DATA)
|
||||
d = deepcopy(TEMPLATED_DVC_YAML_DATA)
|
||||
d["vars"] = ["params2.yaml"]
|
||||
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
|
||||
resolved_data = deepcopy(RESOLVED_DVC_YAML_DATA)
|
||||
assert_stage_equal(resolver.resolve(), resolved_data)
|
||||
@@ -68,7 +68,7 @@ def test_vars_and_params_import(tmp_dir, dvc):
|
||||
"stages": {"stage1": {"cmd": "echo ${dict.foo} ${dict.bar}"}},
|
||||
}
|
||||
(tmp_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"bar": "bar"}})
|
||||
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
|
||||
assert_stage_equal(
|
||||
resolver.resolve(), {"stages": {"stage1": {"cmd": "echo foobar bar"}}}
|
||||
@@ -97,7 +97,7 @@ def test_stage_with_wdir(tmp_dir, dvc):
|
||||
data_dir.mkdir()
|
||||
(tmp_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"bar": "bar"}})
|
||||
(data_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"foo": "foo"}})
|
||||
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
|
||||
assert_stage_equal(
|
||||
resolver.resolve(),
|
||||
@@ -113,7 +113,7 @@ def test_stage_with_wdir(tmp_dir, dvc):
|
||||
)
|
||||
assert resolver.tracked_vars == {
|
||||
"stage1": {
|
||||
str(PathInfo("data") / DEFAULT_PARAMS_FILE): {"dict.foo": "foo"},
|
||||
os.path.join("data", DEFAULT_PARAMS_FILE): {"dict.foo": "foo"},
|
||||
DEFAULT_PARAMS_FILE: {"dict.bar": "bar"},
|
||||
}
|
||||
}
|
||||
@@ -140,7 +140,7 @@ def test_with_templated_wdir(tmp_dir, dvc):
|
||||
data_dir = tmp_dir / "data"
|
||||
data_dir.mkdir()
|
||||
(data_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"foo": "foo"}})
|
||||
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
|
||||
assert_stage_equal(
|
||||
resolver.resolve(),
|
||||
@@ -156,7 +156,7 @@ def test_with_templated_wdir(tmp_dir, dvc):
|
||||
)
|
||||
assert resolver.tracked_vars == {
|
||||
"stage1": {
|
||||
str(PathInfo("data") / DEFAULT_PARAMS_FILE): {"dict.foo": "foo"},
|
||||
os.path.join("data", DEFAULT_PARAMS_FILE): {"dict.foo": "foo"},
|
||||
DEFAULT_PARAMS_FILE: {"dict.bar": "bar", "dict.ws": "data"},
|
||||
}
|
||||
}
|
||||
@@ -178,7 +178,7 @@ def test_resolve_local_tries_to_load_globally_used_files(tmp_dir, dvc):
|
||||
}
|
||||
},
|
||||
}
|
||||
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
assert_stage_equal(
|
||||
resolver.resolve(),
|
||||
{
|
||||
@@ -206,7 +206,7 @@ def test_resolve_local_tries_to_load_globally_used_params_yaml(tmp_dir, dvc):
|
||||
}
|
||||
}
|
||||
}
|
||||
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
assert_stage_equal(
|
||||
resolver.resolve(),
|
||||
{
|
||||
@@ -234,7 +234,7 @@ def test_vars_relpath_overwrite(tmp_dir, dvc):
|
||||
}
|
||||
},
|
||||
}
|
||||
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
resolver.resolve()
|
||||
assert resolver.context.imports == {str(tmp_dir / "params.yaml"): None}
|
||||
|
||||
@@ -257,5 +257,5 @@ def test_vars_load_partial(tmp_dir, dvc, local, vars_):
|
||||
d["stages"]["build"]["vars"] = vars_
|
||||
else:
|
||||
d["vars"] = vars_
|
||||
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
resolver.resolve()
|
||||
|
||||
@@ -13,25 +13,25 @@ DATA = {"models": {"bar": "bar", "foo": "foo"}}
|
||||
|
||||
|
||||
def test_resolver(tmp_dir, dvc):
|
||||
resolver = DataResolver(dvc, tmp_dir, TEMPLATED_DVC_YAML_DATA)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, TEMPLATED_DVC_YAML_DATA)
|
||||
resolver.context.merge_update(Context(CONTEXT_DATA))
|
||||
assert resolver.resolve() == RESOLVED_DVC_YAML_DATA
|
||||
|
||||
|
||||
def test_default_params_file_not_exist(tmp_dir, dvc):
|
||||
d = {"vars": [DATA["models"]]}
|
||||
resolver = DataResolver(dvc, tmp_dir, d)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
assert resolver.context == d["vars"][0]
|
||||
|
||||
|
||||
def test_no_params_yaml_and_vars(tmp_dir, dvc):
|
||||
resolver = DataResolver(dvc, tmp_dir, {})
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
|
||||
assert not resolver.context
|
||||
|
||||
|
||||
def test_local_vars(tmp_dir, dvc):
|
||||
resolver = DataResolver(
|
||||
dvc, tmp_dir, {"vars": [{"foo": "bar", "bar": "foo"}]}
|
||||
dvc, tmp_dir.fs_path, {"vars": [{"foo": "bar", "bar": "foo"}]}
|
||||
)
|
||||
assert resolver.context == {"foo": "bar", "bar": "foo"}
|
||||
|
||||
@@ -39,7 +39,7 @@ def test_local_vars(tmp_dir, dvc):
|
||||
@pytest.mark.parametrize("vars_", ["${file}_params.yaml", {"foo": "${foo}"}])
|
||||
def test_vars_interpolation_errors(tmp_dir, dvc, vars_):
|
||||
with pytest.raises(ResolveError) as exc_info:
|
||||
DataResolver(dvc, tmp_dir, {"vars": [vars_, {"bar": "foo"}]})
|
||||
DataResolver(dvc, tmp_dir.fs_path, {"vars": [vars_, {"bar": "foo"}]})
|
||||
assert (
|
||||
str(exc_info.value)
|
||||
== "failed to parse 'vars' in 'dvc.yaml': interpolating is not allowed"
|
||||
@@ -51,7 +51,7 @@ def test_vars_interpolation_errors(tmp_dir, dvc, vars_):
|
||||
)
|
||||
def test_default_params_file(tmp_dir, dvc, vars_):
|
||||
(tmp_dir / DEFAULT_PARAMS_FILE).dump(DATA)
|
||||
resolver = DataResolver(dvc, tmp_dir, vars_)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, vars_)
|
||||
assert resolver.context == DATA
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ def test_load_vars_from_file(tmp_dir, dvc):
|
||||
datasets = {"datasets": ["foo", "bar"]}
|
||||
(tmp_dir / "params.json").dump(datasets)
|
||||
d = {"vars": [DEFAULT_PARAMS_FILE, "params.json"]}
|
||||
resolver = DataResolver(dvc, tmp_dir, d)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
|
||||
expected = deepcopy(DATA)
|
||||
expected.update(datasets)
|
||||
@@ -77,7 +77,7 @@ def test_load_vars_with_relpath(tmp_dir, scm, dvc):
|
||||
revisions = ["HEAD", "workspace"]
|
||||
for rev in dvc.brancher(revs=["HEAD"]):
|
||||
assert rev == revisions.pop()
|
||||
resolver = DataResolver(dvc, subdir, d)
|
||||
resolver = DataResolver(dvc, subdir.fs_path, d)
|
||||
assert resolver.context == deepcopy(DATA)
|
||||
|
||||
|
||||
@@ -85,7 +85,7 @@ def test_partial_vars_doesnot_exist(tmp_dir, dvc):
|
||||
(tmp_dir / "test_params.yaml").dump({"sub1": "sub1"})
|
||||
|
||||
with pytest.raises(ResolveError) as exc_info:
|
||||
DataResolver(dvc, tmp_dir, {"vars": ["test_params.yaml:sub2"]})
|
||||
DataResolver(dvc, tmp_dir.fs_path, {"vars": ["test_params.yaml:sub2"]})
|
||||
|
||||
assert (
|
||||
str(exc_info.value) == "failed to parse 'vars' in 'dvc.yaml': "
|
||||
@@ -99,7 +99,7 @@ def test_global_overwrite_error_on_imports(tmp_dir, dvc):
|
||||
|
||||
d = {"vars": [DEFAULT_PARAMS_FILE, "params.json"]}
|
||||
with pytest.raises(ResolveError) as exc_info:
|
||||
DataResolver(dvc, tmp_dir, d)
|
||||
DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
|
||||
assert str(exc_info.value) == (
|
||||
"failed to parse 'vars' in 'dvc.yaml':\n"
|
||||
@@ -113,7 +113,7 @@ def test_global_overwrite_vars(tmp_dir, dvc):
|
||||
d = {"vars": [DATA]}
|
||||
|
||||
with pytest.raises(ResolveError) as exc_info:
|
||||
DataResolver(dvc, tmp_dir, d)
|
||||
DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
|
||||
assert str(exc_info.value) == (
|
||||
"failed to parse 'vars' in 'dvc.yaml':\n"
|
||||
@@ -127,7 +127,7 @@ def test_local_declared_vars_overwrite(tmp_dir, dvc):
|
||||
|
||||
d = {"vars": [DATA["models"], DATA["models"]]}
|
||||
with pytest.raises(ResolveError) as exc_info:
|
||||
DataResolver(dvc, tmp_dir, d)
|
||||
DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
|
||||
assert str(exc_info.value) == (
|
||||
"failed to parse 'vars' in 'dvc.yaml':\n"
|
||||
@@ -139,7 +139,7 @@ def test_local_declared_vars_overwrite(tmp_dir, dvc):
|
||||
def test_specified_params_file_not_exist(tmp_dir, dvc):
|
||||
d = {"vars": ["not_existing_params.yaml"]}
|
||||
with pytest.raises(ResolveError) as exc_info:
|
||||
DataResolver(dvc, tmp_dir, d)
|
||||
DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
|
||||
assert str(exc_info.value) == (
|
||||
"failed to parse 'vars' in 'dvc.yaml': "
|
||||
@@ -165,7 +165,7 @@ def test_vars_already_loaded_message(tmp_dir, dvc, local, vars_):
|
||||
d["stages"]["build"]["vars"] = vars_
|
||||
|
||||
with pytest.raises(ResolveError) as exc_info:
|
||||
resolver = DataResolver(dvc, tmp_dir, d)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
resolver.resolve()
|
||||
|
||||
assert "partially" in str(exc_info.value)
|
||||
@@ -180,7 +180,7 @@ def test_local_overwrite_error(tmp_dir, dvc, vars_, loc):
|
||||
|
||||
d = {"stages": {"build": {"cmd": "echo ${models.foo}", "vars": [vars_]}}}
|
||||
|
||||
resolver = DataResolver(dvc, tmp_dir, d)
|
||||
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
|
||||
with pytest.raises(ResolveError) as exc_info:
|
||||
resolver.resolve()
|
||||
|
||||
|
||||
+11
-6
@@ -770,14 +770,14 @@ def test_should_not_checkout_when_adding_cached_copy(tmp_dir, dvc, mocker):
|
||||
def test_should_relink_on_repeated_add(
|
||||
link, new_link, link_test_func, tmp_dir, dvc
|
||||
):
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
dvc.config["cache"]["type"] = link
|
||||
|
||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||
|
||||
os.remove("foo")
|
||||
getattr(dvc.odb.local.fs, link)(PathInfo("bar"), PathInfo("foo"))
|
||||
getattr(dvc.odb.local.fs, link)(
|
||||
(tmp_dir / "bar").fs_path, (tmp_dir / "foo").fs_path
|
||||
)
|
||||
|
||||
dvc.odb.local.cache_types = [new_link]
|
||||
|
||||
@@ -822,6 +822,7 @@ def test_escape_gitignore_entries(tmp_dir, scm, dvc):
|
||||
assert ignored_fname in get_gitignore_content()
|
||||
|
||||
|
||||
@pytest.mark.xfail(reason="error message relpath")
|
||||
def test_add_from_data_dir(tmp_dir, scm, dvc):
|
||||
tmp_dir.dvc_gen({"dir": {"file1": "file1 content"}})
|
||||
|
||||
@@ -866,10 +867,10 @@ def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
|
||||
|
||||
for stage in stages[:2]:
|
||||
# hardlinks are not created for empty files
|
||||
assert not System.is_hardlink(stage.outs[0].path_info)
|
||||
assert not System.is_hardlink(stage.outs[0].fs_path)
|
||||
|
||||
for stage in stages[2:]:
|
||||
assert System.is_hardlink(stage.outs[0].path_info)
|
||||
assert System.is_hardlink(stage.outs[0].fs_path)
|
||||
|
||||
for stage in stages:
|
||||
assert os.path.exists(stage.path)
|
||||
@@ -1031,7 +1032,11 @@ def test_add_to_remote(tmp_dir, dvc, local_cloud, local_remote):
|
||||
|
||||
hash_info = stage.outs[0].hash_info
|
||||
meta = stage.outs[0].meta
|
||||
assert local_remote.hash_to_path_info(hash_info.value).read_text() == "foo"
|
||||
with open(
|
||||
local_remote.hash_to_path(hash_info.value), encoding="utf-8"
|
||||
) as stream:
|
||||
assert stream.read() == "foo"
|
||||
|
||||
assert meta.size == len("foo")
|
||||
|
||||
|
||||
|
||||
+12
-11
@@ -5,7 +5,6 @@ from funcy import first, get_in
|
||||
|
||||
from dvc import api
|
||||
from dvc.exceptions import FileMissingError, OutputNotFoundError
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.utils.fs import remove
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
|
||||
@@ -32,7 +31,7 @@ all_clouds = [pytest.lazy_fixture("local_cloud")] + clouds
|
||||
def test_get_url(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
|
||||
expected_url = URLInfo(remote.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
expected_url = (remote / "ac/bd18db4cc2f85cedef654fccc4a4d8").url
|
||||
assert api.get_url("foo") == expected_url
|
||||
|
||||
|
||||
@@ -45,7 +44,7 @@ def test_get_url_external(tmp_dir, erepo_dir, cloud):
|
||||
|
||||
# Using file url to force clone to tmp repo
|
||||
repo_url = f"file://{erepo_dir}"
|
||||
expected_url = URLInfo(cloud.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
expected_url = (cloud / "ac/bd18db4cc2f85cedef654fccc4a4d8").url
|
||||
assert api.get_url("foo", repo=repo_url) == expected_url
|
||||
|
||||
|
||||
@@ -255,16 +254,16 @@ def test_get_url_granular(tmp_dir, dvc, s3):
|
||||
{"dir": {"foo": "foo", "bar": "bar", "nested": {"file": "file"}}}
|
||||
)
|
||||
|
||||
expected_url = URLInfo(s3.url) / "5f/c28ea78987408341668eba6525ebd1.dir"
|
||||
expected_url = (s3 / "5f/c28ea78987408341668eba6525ebd1.dir").url
|
||||
assert api.get_url("dir") == expected_url
|
||||
|
||||
expected_url = URLInfo(s3.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
expected_url = (s3 / "ac/bd18db4cc2f85cedef654fccc4a4d8").url
|
||||
assert api.get_url("dir/foo") == expected_url
|
||||
|
||||
expected_url = URLInfo(s3.url) / "37/b51d194a7513e45b56f6524f2d51f2"
|
||||
expected_url = (s3 / "37/b51d194a7513e45b56f6524f2d51f2").url
|
||||
assert api.get_url("dir/bar") == expected_url
|
||||
|
||||
expected_url = URLInfo(s3.url) / "8c/7dd922ad47494fc02c388e12c00eac"
|
||||
expected_url = (s3 / "8c/7dd922ad47494fc02c388e12c00eac").url
|
||||
assert api.get_url(os.path.join("dir", "nested", "file")) == expected_url
|
||||
|
||||
|
||||
@@ -277,10 +276,12 @@ def test_get_url_subrepos(tmp_dir, scm, local_cloud):
|
||||
)
|
||||
subrepo.dvc.push()
|
||||
|
||||
path = os.path.relpath(local_cloud.config["url"])
|
||||
|
||||
expected_url = os.path.join(path, "ac", "bd18db4cc2f85cedef654fccc4a4d8")
|
||||
expected_url = os.fspath(
|
||||
local_cloud / "ac" / "bd18db4cc2f85cedef654fccc4a4d8"
|
||||
)
|
||||
assert api.get_url(os.path.join("subrepo", "dir", "foo")) == expected_url
|
||||
|
||||
expected_url = os.path.join(path, "37", "b51d194a7513e45b56f6524f2d51f2")
|
||||
expected_url = os.fspath(
|
||||
local_cloud / "37" / "b51d194a7513e45b56f6524f2d51f2"
|
||||
)
|
||||
assert api.get_url("subrepo/bar") == expected_url
|
||||
|
||||
@@ -102,9 +102,7 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
|
||||
# to check if dvc will detect that the cache is corrupted.
|
||||
obj = load(self.dvc.odb.local, out.hash_info)
|
||||
_, _, entry_oid = list(obj)[0]
|
||||
cache = os.fspath(
|
||||
self.dvc.odb.local.hash_to_path_info(entry_oid.value)
|
||||
)
|
||||
cache = self.dvc.odb.local.hash_to_path(entry_oid.value)
|
||||
|
||||
os.chmod(cache, 0o644)
|
||||
with open(cache, "w+", encoding="utf-8") as fobj:
|
||||
@@ -145,7 +143,7 @@ class CheckoutBase(TestDvcGit):
|
||||
paths = [
|
||||
path
|
||||
for output in stage["outs"]
|
||||
for path in self.dvc.fs.walk_files(output["path"])
|
||||
for path in self.dvc.fs.find(output["path"])
|
||||
]
|
||||
|
||||
return [
|
||||
@@ -599,6 +597,7 @@ def test_stats_on_checkout(tmp_dir, dvc, scm):
|
||||
assert set(stats["deleted"]) == {"foo"}
|
||||
|
||||
|
||||
@pytest.mark.xfail(reason="values relpath")
|
||||
def test_checkout_stats_on_failure(tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen(
|
||||
{"foo": "foo", "dir": {"subdir": {"file": "file"}}, "other": "other"},
|
||||
@@ -763,17 +762,17 @@ def test_checkout_for_external_outputs(tmp_dir, dvc, workspace):
|
||||
dvc.add("remote://workspace/foo")
|
||||
|
||||
odb = dvc.cloud.get_remote_odb("workspace")
|
||||
odb.fs.remove(file_path)
|
||||
odb.fs.remove(str(file_path))
|
||||
assert not file_path.exists()
|
||||
|
||||
stats = dvc.checkout(force=True)
|
||||
assert stats == {**empty_checkout, "added": [str(file_path)]}
|
||||
assert stats == {**empty_checkout, "added": ["remote://workspace/foo"]}
|
||||
assert file_path.exists()
|
||||
|
||||
workspace.gen("foo", "foo\nfoo")
|
||||
|
||||
stats = dvc.checkout(force=True)
|
||||
assert stats == {**empty_checkout, "modified": [str(file_path)]}
|
||||
assert stats == {**empty_checkout, "modified": ["remote://workspace/foo"]}
|
||||
assert file_path.read_text() == "foo"
|
||||
|
||||
|
||||
|
||||
@@ -440,6 +440,7 @@ def test_diff_add_similar_files(tmp_dir, scm, dvc, commit_last):
|
||||
{"dir2": {"file": "text1", "subdir": {"file2": "text2"}}},
|
||||
commit=last_commit_msg,
|
||||
)
|
||||
|
||||
assert dvc.diff(a_rev) == {
|
||||
"added": [
|
||||
{
|
||||
|
||||
@@ -4,7 +4,6 @@ from unittest.mock import ANY, patch
|
||||
from dvc.external_repo import CLONES, external_repo
|
||||
from dvc.objects.stage import stage
|
||||
from dvc.objects.transfer import transfer
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.scm.git import Git
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import makedirs, remove
|
||||
@@ -96,7 +95,8 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
|
||||
dest = tmp_dir / "file"
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
repo.repo_fs.download(
|
||||
PathInfo(repo.root_dir) / "subdir" / "file", PathInfo(dest)
|
||||
os.path.join(repo.root_dir, "subdir", "file"),
|
||||
os.fspath(dest),
|
||||
)
|
||||
|
||||
assert dest.is_file()
|
||||
@@ -191,9 +191,8 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
|
||||
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
repo.repo_fs.download(
|
||||
PathInfo(repo.root_dir) / "dir",
|
||||
PathInfo(tmp_dir / "out"),
|
||||
follow_subrepos=False,
|
||||
os.path.join(repo.root_dir, "dir"),
|
||||
os.fspath(tmp_dir / "out"),
|
||||
)
|
||||
expected_files = {"foo": "foo", "bar": "bar", ".gitignore": "/foo\n"}
|
||||
assert (tmp_dir / "out").read_text() == expected_files
|
||||
@@ -206,7 +205,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
|
||||
|
||||
staging, _, obj = stage(
|
||||
repo.odb.local,
|
||||
PathInfo(repo.root_dir) / "dir",
|
||||
os.path.join(repo.root_dir, "dir"),
|
||||
repo.repo_fs,
|
||||
"md5",
|
||||
dvcignore=repo.dvcignore,
|
||||
@@ -238,9 +237,8 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
|
||||
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
repo.repo_fs.download(
|
||||
PathInfo(repo.root_dir) / "dir",
|
||||
PathInfo(tmp_dir / "out"),
|
||||
follow_subrepos=False,
|
||||
os.path.join(repo.root_dir, "dir"),
|
||||
os.fspath(tmp_dir / "out"),
|
||||
)
|
||||
# subrepo files should not be here
|
||||
assert (tmp_dir / "out").read_text() == scm_files
|
||||
|
||||
+68
-87
@@ -8,7 +8,6 @@ import pytest
|
||||
|
||||
from dvc.fs import get_cloud_fs
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.scm import SCM
|
||||
from tests.basic_env import TestDir, TestGit, TestGitSubmodule
|
||||
@@ -200,17 +199,17 @@ def test_cleanfs_subrepo(tmp_dir, dvc, scm, monkeypatch):
|
||||
subrepo = Repo.init(subdir=True)
|
||||
subrepo_dir.gen({"foo": "foo", "dir": {"bar": "bar"}})
|
||||
|
||||
path = PathInfo(subrepo_dir)
|
||||
path = subrepo_dir.fs_path
|
||||
|
||||
assert dvc.fs.exists(path / "foo")
|
||||
assert dvc.fs.isfile(path / "foo")
|
||||
assert dvc.fs.exists(path / "dir")
|
||||
assert dvc.fs.isdir(path / "dir")
|
||||
assert dvc.fs.exists(dvc.fs.path.join(path, "foo"))
|
||||
assert dvc.fs.isfile(dvc.fs.path.join(path, "foo"))
|
||||
assert dvc.fs.exists(dvc.fs.path.join(path, "dir"))
|
||||
assert dvc.fs.isdir(dvc.fs.path.join(path, "dir"))
|
||||
|
||||
assert subrepo.fs.exists(path / "foo")
|
||||
assert subrepo.fs.isfile(path / "foo")
|
||||
assert subrepo.fs.exists(path / "dir")
|
||||
assert subrepo.fs.isdir(path / "dir")
|
||||
assert subrepo.fs.exists(subrepo.fs.path.join(path, "foo"))
|
||||
assert subrepo.fs.isfile(subrepo.fs.path.join(path, "foo"))
|
||||
assert subrepo.fs.exists(subrepo.fs.path.join(path, "dir"))
|
||||
assert subrepo.fs.isdir(subrepo.fs.path.join(path, "dir"))
|
||||
|
||||
|
||||
def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
|
||||
@@ -246,11 +245,11 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
|
||||
)
|
||||
def test_fs_getsize(dvc, cloud):
|
||||
cloud.gen({"data": {"foo": "foo"}, "baz": "baz baz"})
|
||||
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
|
||||
cls, config, path = get_cloud_fs(dvc, **cloud.config)
|
||||
fs = cls(**config)
|
||||
|
||||
assert fs.getsize(path_info / "baz") == 7
|
||||
assert fs.getsize(path_info / "data" / "foo") == 3
|
||||
assert fs.getsize(fs.path.join(path, "baz")) == 7
|
||||
assert fs.getsize(fs.path.join(path, "data", "foo")) == 3
|
||||
|
||||
|
||||
@pytest.mark.needs_internet
|
||||
@@ -270,11 +269,11 @@ def test_fs_getsize(dvc, cloud):
|
||||
)
|
||||
def test_fs_upload_fobj(dvc, tmp_dir, cloud):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
|
||||
cls, config, path = get_cloud_fs(dvc, **cloud.config)
|
||||
fs = cls(**config)
|
||||
|
||||
from_info = tmp_dir / "foo"
|
||||
to_info = path_info / "foo"
|
||||
to_info = fs.path.join(path, "foo")
|
||||
|
||||
with open(from_info, "rb") as stream:
|
||||
fs.upload_fobj(stream, to_info)
|
||||
@@ -297,73 +296,25 @@ def test_fs_ls(dvc, cloud):
|
||||
}
|
||||
}
|
||||
)
|
||||
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
|
||||
cls, config, path = get_cloud_fs(dvc, **cloud.config)
|
||||
fs = cls(**config)
|
||||
path_info /= "directory"
|
||||
path = os.path.join(path, "directory")
|
||||
|
||||
assert {
|
||||
os.path.basename(file_key.rstrip("/")) for file_key in fs.ls(path_info)
|
||||
os.path.basename(file_key.rstrip("/")) for file_key in fs.ls(path)
|
||||
} == {
|
||||
"foo",
|
||||
"bar",
|
||||
"baz",
|
||||
"empty",
|
||||
}
|
||||
assert set(fs.ls(path_info / "empty")) == set()
|
||||
assert set(fs.ls(fs.path.join(path, "empty"))) == set()
|
||||
assert {
|
||||
(detail["type"], os.path.basename(detail["name"].rstrip("/")))
|
||||
for detail in fs.ls(path_info / "baz", detail=True)
|
||||
for detail in fs.ls(fs.path.join(path, "baz"), detail=True)
|
||||
} == {("file", "quux"), ("directory", "egg")}
|
||||
|
||||
|
||||
@pytest.mark.needs_internet
|
||||
@pytest.mark.parametrize(
|
||||
"cloud",
|
||||
[
|
||||
pytest.lazy_fixture("s3"),
|
||||
pytest.lazy_fixture("azure"),
|
||||
pytest.lazy_fixture("gs"),
|
||||
pytest.lazy_fixture("webdav"),
|
||||
pytest.lazy_fixture("gdrive"),
|
||||
],
|
||||
)
|
||||
def test_fs_find(dvc, cloud):
|
||||
cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}})
|
||||
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
|
||||
fs = cls(**config)
|
||||
|
||||
assert {
|
||||
os.path.basename(file_key) for file_key in fs.find(path_info / "data")
|
||||
} == {"foo", "baz", "quux"}
|
||||
assert {
|
||||
os.path.basename(file_info["name"])
|
||||
for file_info in fs.find(path_info / "data", detail=True)
|
||||
} == {"foo", "baz", "quux"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cloud",
|
||||
[
|
||||
pytest.lazy_fixture("s3"),
|
||||
pytest.lazy_fixture("azure"),
|
||||
pytest.param(
|
||||
pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
|
||||
),
|
||||
pytest.lazy_fixture("webdav"),
|
||||
],
|
||||
)
|
||||
def test_fs_find_with_etag(dvc, cloud):
|
||||
cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}})
|
||||
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
|
||||
fs = cls(**config)
|
||||
|
||||
for details in fs.find(path_info / "data", detail=True):
|
||||
assert (
|
||||
fs.info(path_info.replace(path=details["name"]))["etag"]
|
||||
== details["etag"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cloud",
|
||||
[
|
||||
@@ -379,15 +330,15 @@ def test_fs_fsspec_path_management(dvc, cloud):
|
||||
fs = cls(**config)
|
||||
|
||||
root = cloud.parents[len(cloud.parents) - 1]
|
||||
bucket_details = fs.info(root)
|
||||
bucket_details = fs.info(root.fs_path)
|
||||
|
||||
# special conditions: name always points to the bucket name
|
||||
assert bucket_details["name"] == root.bucket
|
||||
assert bucket_details["type"] == "directory"
|
||||
|
||||
data = cloud / "data"
|
||||
data_details = fs.info(data)
|
||||
assert data_details["name"].rstrip("/") == data.path
|
||||
data_details = fs.info(data.fs_path)
|
||||
assert data_details["name"].rstrip("/") == data.fs_path
|
||||
assert data_details["type"] == "directory"
|
||||
|
||||
|
||||
@@ -406,14 +357,14 @@ def test_fs_makedirs_on_upload_and_copy(dvc, cloud):
|
||||
fs = cls(**config)
|
||||
|
||||
with io.BytesIO(b"foo") as stream:
|
||||
fs.upload(stream, cloud / "dir" / "foo")
|
||||
fs.upload(stream, (cloud / "dir" / "foo").fs_path)
|
||||
|
||||
assert fs.isdir(cloud / "dir")
|
||||
assert fs.exists(cloud / "dir" / "foo")
|
||||
assert fs.isdir((cloud / "dir").fs_path)
|
||||
assert fs.exists((cloud / "dir" / "foo").fs_path)
|
||||
|
||||
fs.copy(cloud / "dir" / "foo", cloud / "dir2" / "foo")
|
||||
assert fs.isdir(cloud / "dir2")
|
||||
assert fs.exists(cloud / "dir2" / "foo")
|
||||
fs.copy((cloud / "dir" / "foo").fs_path, (cloud / "dir2" / "foo").fs_path)
|
||||
assert fs.isdir((cloud / "dir2").fs_path)
|
||||
assert fs.exists((cloud / "dir2" / "foo").fs_path)
|
||||
|
||||
|
||||
@pytest.mark.needs_internet
|
||||
@@ -440,7 +391,11 @@ def test_upload_callback(tmp_dir, dvc, cloud):
|
||||
expected_size = os.path.getsize(tmp_dir / "foo")
|
||||
|
||||
callback = fsspec.Callback()
|
||||
fs.upload(tmp_dir / "foo", cloud / "foo", callback=callback)
|
||||
fs.upload(
|
||||
(tmp_dir / "foo").fs_path,
|
||||
(cloud / "foo").fs_path,
|
||||
callback=callback,
|
||||
)
|
||||
|
||||
assert callback.size == expected_size
|
||||
assert callback.value == expected_size
|
||||
@@ -468,11 +423,15 @@ def test_download_callback(tmp_dir, dvc, cloud, local_cloud):
|
||||
fs = cls(**config)
|
||||
|
||||
(tmp_dir / "to_upload").write_text("foo")
|
||||
fs.upload(tmp_dir / "to_upload", cloud / "foo")
|
||||
expected_size = fs.getsize(cloud / "foo")
|
||||
fs.upload((tmp_dir / "to_upload").fs_path, (cloud / "foo").fs_path)
|
||||
expected_size = fs.getsize((cloud / "foo").fs_path)
|
||||
|
||||
callback = fsspec.Callback()
|
||||
fs.download_file(cloud / "foo", tmp_dir / "foo", callback=callback)
|
||||
fs.download_file(
|
||||
(cloud / "foo").fs_path,
|
||||
(tmp_dir / "foo").fs_path,
|
||||
callback=callback,
|
||||
)
|
||||
|
||||
assert callback.size == expected_size
|
||||
assert callback.value == expected_size
|
||||
@@ -504,7 +463,9 @@ def test_download_dir_callback(tmp_dir, dvc, cloud):
|
||||
cloud.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
|
||||
callback = fsspec.Callback()
|
||||
fs.download(cloud / "dir", tmp_dir / "dir", callback=callback)
|
||||
fs.download(
|
||||
(cloud / "dir").fs_path, (tmp_dir / "dir").fs_path, callback=callback
|
||||
)
|
||||
|
||||
assert callback.size == 2
|
||||
assert callback.value == 2
|
||||
@@ -519,7 +480,11 @@ def test_download_callbacks_on_dvc_git_fs(tmp_dir, dvc, scm, fs_type):
|
||||
fs = dvc.dvcfs if fs_type == "dvc" else scm.get_fs("HEAD")
|
||||
|
||||
callback = fsspec.Callback()
|
||||
fs.download(tmp_dir / "file", tmp_dir / "file2", callback=callback)
|
||||
fs.download(
|
||||
(tmp_dir / "file").fs_path,
|
||||
(tmp_dir / "file2").fs_path,
|
||||
callback=callback,
|
||||
)
|
||||
|
||||
size = os.path.getsize(tmp_dir / "file")
|
||||
assert (tmp_dir / "file2").read_text() == "file"
|
||||
@@ -530,7 +495,11 @@ def test_download_callbacks_on_dvc_git_fs(tmp_dir, dvc, scm, fs_type):
|
||||
pytest.skip("gitfs does not support download_dir")
|
||||
|
||||
callback = fsspec.Callback()
|
||||
fs.download(tmp_dir / "dir", tmp_dir / "dir2", callback=callback)
|
||||
fs.download(
|
||||
(tmp_dir / "dir").fs_path,
|
||||
(tmp_dir / "dir2").fs_path,
|
||||
callback=callback,
|
||||
)
|
||||
|
||||
assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"}
|
||||
assert callback.size == 2
|
||||
@@ -544,14 +513,22 @@ def test_callback_on_repo_fs(tmp_dir, dvc, scm):
|
||||
fs = dvc.repo_fs
|
||||
|
||||
callback = fsspec.Callback()
|
||||
fs.download(tmp_dir / "dir", tmp_dir / "dir2", callback=callback)
|
||||
fs.download(
|
||||
(tmp_dir / "dir").fs_path,
|
||||
(tmp_dir / "dir2").fs_path,
|
||||
callback=callback,
|
||||
)
|
||||
|
||||
assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"}
|
||||
assert callback.size == 2
|
||||
assert callback.value == 2
|
||||
|
||||
callback = fsspec.Callback()
|
||||
fs.download(tmp_dir / "dir" / "foo", tmp_dir / "foo", callback=callback)
|
||||
fs.download(
|
||||
(tmp_dir / "dir" / "foo").fs_path,
|
||||
(tmp_dir / "foo").fs_path,
|
||||
callback=callback,
|
||||
)
|
||||
|
||||
size = os.path.getsize(tmp_dir / "dir" / "foo")
|
||||
assert (tmp_dir / "foo").read_text() == "foo"
|
||||
@@ -559,7 +536,11 @@ def test_callback_on_repo_fs(tmp_dir, dvc, scm):
|
||||
assert callback.value == size
|
||||
|
||||
callback = fsspec.Callback()
|
||||
fs.download(tmp_dir / "dir" / "bar", tmp_dir / "bar", callback=callback)
|
||||
fs.download(
|
||||
(tmp_dir / "dir" / "bar").fs_path,
|
||||
(tmp_dir / "bar").fs_path,
|
||||
callback=callback,
|
||||
)
|
||||
|
||||
size = os.path.getsize(tmp_dir / "dir" / "bar")
|
||||
assert (tmp_dir / "bar").read_text() == "bar"
|
||||
|
||||
@@ -22,7 +22,7 @@ class TestGC(TestDvcGit):
|
||||
self.dvc.add(self.FOO)
|
||||
self.dvc.add(self.DATA_DIR)
|
||||
self.good_cache = [
|
||||
self.dvc.odb.local.hash_to_path_info(md5)
|
||||
self.dvc.odb.local.hash_to_path(md5)
|
||||
for md5 in self.dvc.odb.local.all()
|
||||
]
|
||||
|
||||
|
||||
+40
-25
@@ -18,6 +18,11 @@ def _to_pattern_info_list(str_list: List):
|
||||
return [PatternInfo(a, "") for a in str_list]
|
||||
|
||||
|
||||
def walk_files(dvc, *args):
|
||||
for fs_path in dvc.dvcignore.find(*args):
|
||||
yield fs_path
|
||||
|
||||
|
||||
@pytest.mark.parametrize("filename", ["ignored", "тест"])
|
||||
def test_ignore(tmp_dir, dvc, filename):
|
||||
tmp_dir.gen({"dir": {filename: filename, "other": "text2"}})
|
||||
@@ -25,10 +30,10 @@ def test_ignore(tmp_dir, dvc, filename):
|
||||
|
||||
dvc._reset()
|
||||
|
||||
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir)
|
||||
result = walk_files(dvc, dvc.fs, tmp_dir)
|
||||
assert set(result) == {
|
||||
tmp_dir / DvcIgnore.DVCIGNORE_FILE,
|
||||
tmp_dir / "dir" / "other",
|
||||
(tmp_dir / DvcIgnore.DVCIGNORE_FILE).fs_path,
|
||||
(tmp_dir / "dir" / "other").fs_path,
|
||||
}
|
||||
|
||||
|
||||
@@ -125,11 +130,11 @@ def test_ignore_on_branch(tmp_dir, scm, dvc):
|
||||
|
||||
dvc._reset()
|
||||
|
||||
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir)
|
||||
result = walk_files(dvc, dvc.fs, tmp_dir)
|
||||
assert set(result) == {
|
||||
tmp_dir / "foo",
|
||||
tmp_dir / "bar",
|
||||
tmp_dir / DvcIgnore.DVCIGNORE_FILE,
|
||||
(tmp_dir / "foo").fs_path,
|
||||
(tmp_dir / "bar").fs_path,
|
||||
(tmp_dir / DvcIgnore.DVCIGNORE_FILE).fs_path,
|
||||
}
|
||||
|
||||
dvc.fs = scm.get_fs("branch")
|
||||
@@ -146,8 +151,11 @@ def test_match_nested(tmp_dir, dvc):
|
||||
}
|
||||
)
|
||||
dvc._reset()
|
||||
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir)
|
||||
assert set(result) == {tmp_dir / DvcIgnore.DVCIGNORE_FILE, tmp_dir / "foo"}
|
||||
result = walk_files(dvc, dvc.fs, tmp_dir)
|
||||
assert set(result) == {
|
||||
(tmp_dir / DvcIgnore.DVCIGNORE_FILE).fs_path,
|
||||
(tmp_dir / "foo").fs_path,
|
||||
}
|
||||
|
||||
|
||||
def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
|
||||
@@ -155,8 +163,11 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
|
||||
ext_dir = TmpDir(os.fspath(tmp_path_factory.mktemp("external_dir")))
|
||||
ext_dir.gen({"y.backup": "y", "tmp": {"file": "ext tmp"}})
|
||||
|
||||
result = dvc.dvcignore.walk_files(dvc.fs, ext_dir)
|
||||
assert set(result) == {ext_dir / "y.backup", ext_dir / "tmp" / "file"}
|
||||
result = walk_files(dvc, dvc.fs, ext_dir)
|
||||
assert set(result) == {
|
||||
(ext_dir / "y.backup").fs_path,
|
||||
(ext_dir / "tmp" / "file").fs_path,
|
||||
}
|
||||
assert dvc.dvcignore.is_ignored_dir(os.fspath(ext_dir / "tmp")) is False
|
||||
assert (
|
||||
dvc.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup")) is False
|
||||
@@ -171,7 +182,7 @@ def test_ignore_subrepo(tmp_dir, scm, dvc):
|
||||
|
||||
subrepo_dir = tmp_dir / "subdir"
|
||||
|
||||
result = dvc.dvcignore.walk_files(dvc.fs, subrepo_dir)
|
||||
result = walk_files(dvc, dvc.fs, subrepo_dir)
|
||||
assert set(result) == set()
|
||||
|
||||
with subrepo_dir.chdir():
|
||||
@@ -212,8 +223,8 @@ def test_ignore_blank_line(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "foo\n\ndir/ignored")
|
||||
dvc._reset()
|
||||
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir / "dir")
|
||||
assert set(result) == {tmp_dir / "dir" / "other"}
|
||||
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
|
||||
assert set(result) == {(tmp_dir / "dir" / "other").fs_path}
|
||||
|
||||
|
||||
# It is not possible to re-include a file if a parent directory of
|
||||
@@ -247,8 +258,10 @@ def test_ignore_file_in_parent_path(
|
||||
tmp_dir.gen(data_struct)
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "\n".join(pattern_list))
|
||||
dvc._reset()
|
||||
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir / "dir")
|
||||
assert set(result) == {tmp_dir / relpath for relpath in result_set}
|
||||
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
|
||||
assert set(result) == {
|
||||
(tmp_dir / relpath).fs_path for relpath in result_set
|
||||
}
|
||||
|
||||
|
||||
# If there is a separator at the end of the pattern then the pattern
|
||||
@@ -268,10 +281,10 @@ def test_ignore_sub_directory(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "doc/fortz"}})
|
||||
|
||||
dvc._reset()
|
||||
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir / "dir")
|
||||
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
|
||||
assert set(result) == {
|
||||
tmp_dir / "dir" / "a" / "doc" / "fortz" / "a",
|
||||
tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE,
|
||||
(tmp_dir / "dir" / "a" / "doc" / "fortz" / "a").fs_path,
|
||||
(tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE).fs_path,
|
||||
}
|
||||
|
||||
|
||||
@@ -280,8 +293,10 @@ def test_ignore_directory(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"fortz": {}, "a": {"fortz": {}}}})
|
||||
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "fortz"}})
|
||||
dvc._reset()
|
||||
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir / "dir")
|
||||
assert set(result) == {tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE}
|
||||
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
|
||||
assert set(result) == {
|
||||
(tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE).fs_path
|
||||
}
|
||||
|
||||
|
||||
def test_multi_ignore_file(tmp_dir, dvc, monkeypatch):
|
||||
@@ -289,10 +304,10 @@ def test_multi_ignore_file(tmp_dir, dvc, monkeypatch):
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/subdir/*_ignore")
|
||||
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "!subdir/not_ignore"}})
|
||||
dvc._reset()
|
||||
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir / "dir")
|
||||
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
|
||||
assert set(result) == {
|
||||
tmp_dir / "dir" / "subdir" / "not_ignore",
|
||||
tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE,
|
||||
(tmp_dir / "dir" / "subdir" / "not_ignore").fs_path,
|
||||
(tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE).fs_path,
|
||||
}
|
||||
|
||||
|
||||
@@ -384,7 +399,7 @@ def test_ignore_in_added_dir(tmp_dir, dvc):
|
||||
dvc._reset()
|
||||
|
||||
ignored_path = tmp_dir / "dir" / "sub" / "ignored"
|
||||
result = dvc.dvcignore.walk_files(dvc.fs, ignored_path)
|
||||
result = walk_files(dvc, dvc.fs, ignored_path)
|
||||
assert set(result) == set()
|
||||
assert ignored_path.exists()
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ def test_import_url_to_dir(dname, tmp_dir, dvc):
|
||||
|
||||
dst = tmp_dir / dname / "file"
|
||||
|
||||
assert stage.outs[0].path_info == dst
|
||||
assert stage.outs[0].fs_path == os.fspath(dst)
|
||||
assert os.path.isdir(dname)
|
||||
assert dst.read_text() == "file content"
|
||||
|
||||
@@ -290,7 +290,10 @@ def test_import_url_to_remote_single_file(
|
||||
assert len(stage.outs) == 1
|
||||
|
||||
hash_info = stage.outs[0].hash_info
|
||||
assert local_remote.hash_to_path_info(hash_info.value).read_text() == "foo"
|
||||
with open(
|
||||
local_remote.hash_to_path(hash_info.value), encoding="utf-8"
|
||||
) as fobj:
|
||||
assert fobj.read() == "foo"
|
||||
assert stage.outs[0].meta.size == len("foo")
|
||||
|
||||
|
||||
@@ -335,7 +338,7 @@ def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote):
|
||||
|
||||
hash_info = stage.outs[0].hash_info
|
||||
with open(
|
||||
local_remote.hash_to_path_info(hash_info.value), encoding="utf-8"
|
||||
local_remote.hash_to_path(hash_info.value), encoding="utf-8"
|
||||
) as stream:
|
||||
file_parts = json.load(stream)
|
||||
|
||||
@@ -347,10 +350,10 @@ def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote):
|
||||
}
|
||||
|
||||
for file_part in file_parts:
|
||||
assert (
|
||||
local_remote.hash_to_path_info(file_part["md5"]).read_text()
|
||||
== file_part["relpath"]
|
||||
)
|
||||
with open(
|
||||
local_remote.hash_to_path(file_part["md5"]), encoding="utf-8"
|
||||
) as fobj:
|
||||
assert fobj.read() == file_part["relpath"]
|
||||
|
||||
|
||||
def test_import_url_to_remote_absolute(
|
||||
|
||||
@@ -37,7 +37,7 @@ class TestCache(TestDvc):
|
||||
self.assertIn(self.cache2_md5, md5_list)
|
||||
|
||||
def test_get(self):
|
||||
cache = ODBManager(self.dvc).local.hash_to_path_info(self.cache1_md5)
|
||||
cache = ODBManager(self.dvc).local.hash_to_path(self.cache1_md5)
|
||||
self.assertEqual(os.fspath(cache), self.cache1)
|
||||
|
||||
|
||||
@@ -50,13 +50,13 @@ class TestCacheLoadBadDirCache(TestDvc):
|
||||
from dvc.objects import load
|
||||
|
||||
dir_hash = "123.dir"
|
||||
fname = os.fspath(self.dvc.odb.local.hash_to_path_info(dir_hash))
|
||||
fname = os.fspath(self.dvc.odb.local.hash_to_path(dir_hash))
|
||||
self.create(fname, "<clearly>not,json")
|
||||
with pytest.raises(ObjectFormatError):
|
||||
load(self.dvc.odb.local, HashInfo("md5", dir_hash))
|
||||
|
||||
dir_hash = "234.dir"
|
||||
fname = os.fspath(self.dvc.odb.local.hash_to_path_info(dir_hash))
|
||||
fname = os.fspath(self.dvc.odb.local.hash_to_path(dir_hash))
|
||||
self.create(fname, '{"a": "b"}')
|
||||
with pytest.raises(ObjectFormatError):
|
||||
load(self.dvc.odb.local, HashInfo("md5", dir_hash))
|
||||
@@ -88,7 +88,7 @@ class TestExternalCacheDir(TestDvc):
|
||||
|
||||
self.dvc.__init__()
|
||||
|
||||
assert self.dvc.odb.ssh.path_info == ssh_url + "/tmp"
|
||||
assert self.dvc.odb.ssh.fs_path == "/tmp"
|
||||
|
||||
|
||||
class TestSharedCacheDir(TestDir):
|
||||
@@ -158,7 +158,7 @@ class TestCmdCacheDir(TestDvc):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
config = configobj.ConfigObj(self.dvc.config.files["repo"])
|
||||
self.assertEqual(config["cache"]["dir"], dname.replace("\\", "/"))
|
||||
self.assertEqual(config["cache"]["dir"], dname)
|
||||
|
||||
def test_relative_path(self):
|
||||
tmpdir = os.path.realpath(self.mkdtemp())
|
||||
|
||||
+10
-11
@@ -9,7 +9,6 @@ import pytest
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError
|
||||
from dvc.main import main
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.utils.fs import remove
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.remotes import Local
|
||||
@@ -149,14 +148,14 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
|
||||
tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}})
|
||||
|
||||
path_info = PathInfo("data")
|
||||
path = (tmp_dir / "data").fs_path
|
||||
|
||||
tree = Tree.from_list(
|
||||
[{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}]
|
||||
)
|
||||
tree.digest()
|
||||
with patch("dvc.objects.stage._stage_tree", return_value=(None, tree)):
|
||||
_, _, obj = stage(dvc.odb.local, path_info, dvc.odb.local.fs, "md5")
|
||||
_, _, obj = stage(dvc.odb.local, path, dvc.odb.local.fs, "md5")
|
||||
hash1 = obj.hash_info
|
||||
|
||||
tree = Tree.from_list(
|
||||
@@ -164,7 +163,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
|
||||
)
|
||||
tree.digest()
|
||||
with patch("dvc.objects.stage._stage_tree", return_value=(None, tree)):
|
||||
_, _, obj = stage(dvc.odb.local, path_info, dvc.odb.local.fs, "md5")
|
||||
_, _, obj = stage(dvc.odb.local, path, dvc.odb.local.fs, "md5")
|
||||
hash2 = obj.hash_info
|
||||
|
||||
assert hash1 == hash2
|
||||
@@ -183,7 +182,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
|
||||
|
||||
def unreliable_upload(from_fs, from_info, to_fs, to_info, **kwargs):
|
||||
if os.path.abspath(to_info) == os.path.abspath(
|
||||
odb.get(foo.hash_info).path_info
|
||||
odb.get(foo.hash_info).fs_path
|
||||
):
|
||||
raise Exception("stop foo")
|
||||
return original(from_fs, from_info, to_fs, to_info, **kwargs)
|
||||
@@ -273,8 +272,8 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
|
||||
|
||||
# foo .dir file should be uploaded after bar
|
||||
odb = dvc.cloud.get_remote_odb("upstream")
|
||||
foo_path = odb.hash_to_path_info(foo.hash_info.value)
|
||||
bar_path = odb.hash_to_path_info(foo.obj.trie[("bar",)][1].value)
|
||||
foo_path = odb.hash_to_path(foo.hash_info.value)
|
||||
bar_path = odb.hash_to_path(foo.obj.trie[("bar",)][1].value)
|
||||
paths = [args[3] for args, _ in mocked_upload.call_args_list]
|
||||
assert paths.index(foo_path) > paths.index(bar_path)
|
||||
|
||||
@@ -412,7 +411,7 @@ def test_protect_local_remote(tmp_dir, dvc, local_remote):
|
||||
|
||||
dvc.push()
|
||||
odb = dvc.cloud.get_remote_odb("upstream")
|
||||
remote_cache_file = odb.hash_to_path_info(stage.outs[0].hash_info.value)
|
||||
remote_cache_file = odb.hash_to_path(stage.outs[0].hash_info.value)
|
||||
|
||||
assert os.path.exists(remote_cache_file)
|
||||
assert stat.S_IMODE(os.stat(remote_cache_file).st_mode) == 0o444
|
||||
@@ -427,7 +426,7 @@ def test_push_incomplete_dir(tmp_dir, dvc, mocker, local_remote):
|
||||
file_objs = [entry_obj for _, _, entry_obj in out.obj]
|
||||
|
||||
# remove one of the cache files for directory
|
||||
remove(odb.hash_to_path_info(file_objs[0].value))
|
||||
remove(odb.hash_to_path(file_objs[0].value))
|
||||
|
||||
dvc.push()
|
||||
assert not remote_odb.exists(out.hash_info)
|
||||
@@ -442,8 +441,8 @@ def test_upload_exists(tmp_dir, dvc, local_remote):
|
||||
# normally they are set to read-only for DVC remotes
|
||||
odb.fs.CACHE_MODE = 0o644
|
||||
|
||||
from_info = PathInfo(tmp_dir / "foo")
|
||||
to_info = odb.path_info / "foo"
|
||||
from_info = (tmp_dir / "foo").fs_path
|
||||
to_info = odb.fs.path.join(odb.fs_path, "foo")
|
||||
odb.fs.upload(from_info, to_info)
|
||||
assert odb.fs.exists(to_info)
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ from funcy import raiser
|
||||
|
||||
from dvc.dvcfile import PIPELINE_FILE, FileIsGitIgnored
|
||||
from dvc.exceptions import NoOutputOrStageError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.stage.exceptions import (
|
||||
StageFileDoesNotExistError,
|
||||
@@ -226,7 +225,7 @@ def test_collect_granular_with_target(tmp_dir, dvc, stages):
|
||||
) == [(stages["copy-bar-foobar"], None)]
|
||||
|
||||
assert dvc.stage.collect_granular("foobar") == [
|
||||
(stages["copy-bar-foobar"], PathInfo(tmp_dir / "foobar"))
|
||||
(stages["copy-bar-foobar"], os.path.join(tmp_dir, "foobar"))
|
||||
]
|
||||
|
||||
|
||||
@@ -296,10 +295,10 @@ def test_collect_granular_same_output_name_stage_name(tmp_dir, dvc, run_copy):
|
||||
assert list(map(itemgetter(1), coll)) == [None] * 2
|
||||
|
||||
assert dvc.stage.collect_granular("./copy-foo-bar") == [
|
||||
(stage2, PathInfo(tmp_dir / "copy-foo-bar"))
|
||||
(stage2, os.path.join(tmp_dir / "copy-foo-bar"))
|
||||
]
|
||||
assert dvc.stage.collect_granular("./copy-foo-bar", with_deps=True) == [
|
||||
(stage2, PathInfo(tmp_dir / "copy-foo-bar"))
|
||||
(stage2, os.path.join(tmp_dir / "copy-foo-bar"))
|
||||
]
|
||||
|
||||
|
||||
@@ -330,7 +329,7 @@ def test_collect_granular_collision_output_dir_stage_name(
|
||||
assert dvc.stage.collect_granular("dir") == [(stage3, None)]
|
||||
assert not dvc.stage.collect_granular("dir", recursive=True)
|
||||
assert dvc.stage.collect_granular("./dir") == [
|
||||
(stage1, PathInfo(tmp_dir / "dir"))
|
||||
(stage1, os.path.join(tmp_dir / "dir"))
|
||||
]
|
||||
|
||||
|
||||
@@ -468,24 +467,24 @@ def test_gitignored_file_try_collect_granular_for_data_files(
|
||||
tmp_dir, dvc, scm
|
||||
):
|
||||
(stage,) = tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}})
|
||||
path = PathInfo("data") / "foo"
|
||||
path = os.path.join("data", "foo")
|
||||
|
||||
assert dvc.stage.collect_granular(str(path)) == [
|
||||
(stage, PathInfo(tmp_dir / path))
|
||||
assert dvc.stage.collect_granular(path) == [
|
||||
(stage, os.path.join(tmp_dir, path))
|
||||
]
|
||||
|
||||
scm.ignore(stage.path)
|
||||
dvc._reset()
|
||||
|
||||
with pytest.raises(NoOutputOrStageError):
|
||||
dvc.stage.collect_granular(str(path))
|
||||
dvc.stage.collect_granular(path)
|
||||
|
||||
|
||||
def test_gitignored_file_try_collect_granular_for_dvc_yaml_files(
|
||||
tmp_dir, dvc, scm, stages
|
||||
):
|
||||
assert dvc.stage.collect_granular("bar") == [
|
||||
(stages["copy-foo-bar"], PathInfo(tmp_dir / "bar"))
|
||||
(stages["copy-foo-bar"], os.path.join(tmp_dir, "bar"))
|
||||
]
|
||||
|
||||
scm.ignore(tmp_dir / "dvc.yaml")
|
||||
|
||||
@@ -2,7 +2,6 @@ import os
|
||||
import re
|
||||
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.state import State
|
||||
from dvc.utils import file_md5
|
||||
@@ -11,23 +10,22 @@ from dvc.utils import file_md5
|
||||
def test_state(tmp_dir, dvc):
|
||||
tmp_dir.gen("foo", "foo content")
|
||||
path = tmp_dir / "foo"
|
||||
path_info = PathInfo(path)
|
||||
hash_info = HashInfo("md5", file_md5(path, dvc.fs))
|
||||
|
||||
state = State(dvc.root_dir, dvc.tmp_dir, dvc.dvcignore)
|
||||
|
||||
state.save(path_info, dvc.fs, hash_info)
|
||||
assert state.get(path_info, dvc.fs)[1] == hash_info
|
||||
state.save(path, dvc.fs, hash_info)
|
||||
assert state.get(path, dvc.fs)[1] == hash_info
|
||||
|
||||
path.unlink()
|
||||
path.write_text("1")
|
||||
|
||||
assert state.get(path_info, dvc.fs) == (None, None)
|
||||
assert state.get(path, dvc.fs) == (None, None)
|
||||
|
||||
hash_info = HashInfo("md5", file_md5(path, dvc.fs))
|
||||
state.save(path_info, dvc.fs, hash_info)
|
||||
state.save(path, dvc.fs, hash_info)
|
||||
|
||||
assert state.get(path_info, dvc.fs)[1] == hash_info
|
||||
assert state.get(path, dvc.fs)[1] == hash_info
|
||||
|
||||
|
||||
def test_state_overflow(tmp_dir, dvc):
|
||||
|
||||
@@ -6,9 +6,8 @@ import uuid
|
||||
import pytest
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
|
||||
from .base import Base
|
||||
from .path_info import CloudURLInfo
|
||||
|
||||
TEST_AZURE_CONTAINER = "tests"
|
||||
TEST_AZURE_CONNECTION_STRING = (
|
||||
@@ -57,6 +56,12 @@ class Azure(Base, CloudURLInfo):
|
||||
assert errors is None
|
||||
return self.read_bytes().decode(encoding)
|
||||
|
||||
@property
|
||||
def fs_path(self):
|
||||
bucket = self.bucket.rstrip("/")
|
||||
path = self.path.lstrip("/")
|
||||
return f"{bucket}/{path}"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def azure_server(test_config, docker_compose, docker_services):
|
||||
|
||||
@@ -3,7 +3,7 @@ import pathlib
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.path_info import URLInfo
|
||||
from .path_info import URLInfo
|
||||
|
||||
|
||||
class Base(URLInfo):
|
||||
|
||||
+21
-2
@@ -3,18 +3,37 @@ import json
|
||||
import os
|
||||
import uuid
|
||||
from functools import partialmethod
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pytest
|
||||
from funcy import cached_property, retry
|
||||
|
||||
from dvc.fs.gdrive import GDriveFileSystem, GDriveURLInfo
|
||||
from dvc.fs.gdrive import GDriveFileSystem
|
||||
from dvc.utils import tmp_fname
|
||||
|
||||
from .base import Base
|
||||
from .path_info import CloudURLInfo
|
||||
|
||||
TEST_GDRIVE_REPO_BUCKET = "root"
|
||||
|
||||
|
||||
class GDriveURLInfo(CloudURLInfo):
|
||||
def __init__(self, url):
|
||||
super().__init__(url)
|
||||
|
||||
# GDrive URL host part is case sensitive,
|
||||
# we are restoring it here.
|
||||
p = urlparse(url)
|
||||
self.host = p.netloc
|
||||
assert self.netloc == self.host
|
||||
|
||||
# Normalize path. Important since we have a cache (path to ID)
|
||||
# and don't want to deal with different variations of path in it.
|
||||
import re
|
||||
|
||||
self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
|
||||
|
||||
|
||||
def _gdrive_retry(func):
|
||||
def should_retry(exc):
|
||||
from googleapiclient.errors import HttpError
|
||||
@@ -128,5 +147,5 @@ def gdrive(test_config, make_tmp_dir):
|
||||
fs = GDriveFileSystem(
|
||||
gdrive_credentials_tmp_dir=tmp_dir.dvc.tmp_dir, **ret.config
|
||||
)
|
||||
fs.fs._gdrive_create_dir("root", fs.path_info.path)
|
||||
fs.fs._gdrive_create_dir("root", fs.url)
|
||||
yield ret
|
||||
|
||||
+1
-1
@@ -5,10 +5,10 @@ import uuid
|
||||
import pytest
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.path_info import CloudURLInfo
|
||||
from dvc.utils import env2bool
|
||||
|
||||
from .base import Base
|
||||
from .path_info import CloudURLInfo
|
||||
|
||||
TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test")
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user