dvc: get rid of path_info

Intermediate PR in removing path_info from our code base.

It is completely removed from the main codebase and is still
being used in tests, which will be migrated in the followups.

For fs-specific simple path manipulation, we've introduced
`fs.path` helper (similar to `os.path`).

Fixes #6774
This commit is contained in:
Batuhan Taskaya
2021-11-10 00:52:48 +02:00
committed by Ruslan Kuprieiev
parent 4175f9ffa4
commit 9aec8f1926
129 changed files with 1654 additions and 1906 deletions
+3 -4
View File
@@ -4,7 +4,6 @@ from contextlib import _GeneratorContextManager as GCM
from funcy import reraise
from dvc.exceptions import OutputNotFoundError, PathMissingError
from dvc.path_info import PathInfo
from dvc.repo import Repo
@@ -20,15 +19,15 @@ def get_url(path, repo=None, rev=None, remote=None):
directory in the remote storage.
"""
with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
path_info = PathInfo(_repo.root_dir) / path
fs_path = _repo.fs.path.join(_repo.root_dir, path)
with reraise(FileNotFoundError, PathMissingError(path, repo)):
metadata = _repo.repo_fs.metadata(path_info)
metadata = _repo.repo_fs.metadata(fs_path)
if not metadata.is_dvc:
raise OutputNotFoundError(path, repo)
cloud = metadata.repo.cloud
md5 = metadata.repo.dvcfs.info(path_info)["md5"]
md5 = metadata.repo.dvcfs.info(fs_path)["md5"]
return cloud.get_url_for(remote, checksum=md5)
+5 -3
View File
@@ -38,12 +38,14 @@ def _collect_targets(repo, target, outs):
targets = []
outs_trie = repo.index.outs_trie
for stage, info in pairs:
if not info:
for stage, path in pairs:
if not path:
targets.extend([str(out) for out in stage.outs])
continue
for out in outs_trie.itervalues(prefix=info.parts): # noqa: B301
for out in outs_trie.itervalues(
prefix=repo.fs.path.parts(path)
): # noqa: B301
targets.extend(str(out))
return targets
+3 -2
View File
@@ -8,7 +8,6 @@ from functools import partial
from funcy import cached_property, compact, memoize, re_find
from dvc.exceptions import DvcException, NotDvcRepoError
from dvc.path_info import PathInfo
logger = logging.getLogger(__name__)
@@ -229,6 +228,7 @@ class Config(dict):
@staticmethod
def _to_relpath(conf_dir, path):
from dvc.fs.local import localfs
from dvc.utils import relpath
from .config_schema import RelPath
@@ -238,8 +238,9 @@ class Config(dict):
if isinstance(path, RelPath) or not os.path.isabs(path):
path = relpath(path, conf_dir)
return localfs.path.as_posix(path)
return PathInfo(path).as_posix()
return path
@staticmethod
def _save_paths(conf, filename):
+5 -4
View File
@@ -58,9 +58,9 @@ class DataCloud:
from dvc.fs import get_cloud_fs
from dvc.objects.db import get_odb
cls, config, path_info = get_cloud_fs(self.repo, name=name)
cls, config, fs_path = get_cloud_fs(self.repo, name=name)
config["tmp_dir"] = self.repo.index_db_dir
return get_odb(cls(**config), path_info, **config)
return get_odb(cls(**config), fs_path, **config)
def push(
self,
@@ -156,5 +156,6 @@ class DataCloud:
)
def get_url_for(self, remote, checksum):
remote_odb = self.get_remote_odb(remote)
return str(remote_odb.hash_to_path_info(checksum))
odb = self.get_remote_odb(remote)
path = odb.hash_to_path(checksum)
return odb.fs.unstrip_protocol(path)
+2 -2
View File
@@ -92,10 +92,10 @@ class ParamsDependency(Dependency):
if not self.exists:
return {}
suffix = self.path_info.suffix.lower()
suffix = self.repo.fs.path.suffix(self.fs_path).lower()
loader = LOADERS[suffix]
try:
return loader(self.path_info, fs=self.repo.fs)
return loader(self.fs_path, fs=self.repo.fs)
except ParseError as exc:
raise BadParamFileError(
f"Unable to read parameters from '{self}'"
+5 -7
View File
@@ -5,8 +5,6 @@ from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple
from voluptuous import Required
from dvc.path_info import PathInfo
from .base import Dependency
if TYPE_CHECKING:
@@ -34,7 +32,7 @@ class RepoDependency(Dependency):
self._staged_objs: Dict[str, "HashFile"] = {}
super().__init__(stage, *args, **kwargs)
def _parse_path(self, fs, path_info):
def _parse_path(self, fs, fs_path):
return None
@property
@@ -70,7 +68,7 @@ class RepoDependency(Dependency):
obj = self.get_obj()
checkout(
to.path_info,
to.fs_path,
to.fs,
obj,
self.repo.odb.local,
@@ -114,11 +112,11 @@ class RepoDependency(Dependency):
if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
self.def_repo[self.PARAM_REV_LOCK] = rev
path_info = PathInfo(repo.root_dir) / str(self.def_path)
path = os.path.abspath(os.path.join(repo.root_dir, self.def_path))
if not obj_only:
try:
for odb, obj_ids in repo.used_objs(
[os.fspath(path_info)],
[path],
force=True,
jobs=kwargs.get("jobs"),
recursive=True,
@@ -134,7 +132,7 @@ class RepoDependency(Dependency):
try:
staging, _, staged_obj = stage(
local_odb,
path_info,
path,
repo.repo_fs,
local_odb.fs.PARAM_CHECKSUM,
)
+6 -6
View File
@@ -305,7 +305,7 @@ class PathMissingError(DvcException):
class RemoteCacheRequiredError(DvcException):
def __init__(self, path_info):
def __init__(self, scheme, fs_path):
from dvc.utils import format_link
super().__init__(
@@ -314,8 +314,8 @@ class RemoteCacheRequiredError(DvcException):
"existing cache on '{}' remote. See {} for information on how "
"to set up remote cache."
).format(
path_info,
path_info.scheme,
fs_path,
scheme,
format_link("https://man.dvc.org/config#cache"),
)
)
@@ -350,12 +350,12 @@ class CacheLinkError(DvcException):
)
)
def __init__(self, path_infos):
def __init__(self, fs_paths):
msg = "No possible cache link types for '{}'. {}".format(
", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK
", ".join(fs_paths), self.SUPPORT_LINK
)
super().__init__(msg)
self.path_infos = path_infos
self.fs_paths = fs_paths
class CircularImportError(DvcException):
+10 -11
View File
@@ -1,4 +1,3 @@
import posixpath
from urllib.parse import urlparse
from ..scheme import Schemes
@@ -39,7 +38,7 @@ def get_fs_cls(remote_conf, scheme=None):
return FS_MAP.get(scheme, LocalFileSystem)
def get_fs_config(config, **kwargs):
def get_fs_config(repo, config, **kwargs):
name = kwargs.get("name")
if name:
try:
@@ -50,10 +49,10 @@ def get_fs_config(config, **kwargs):
raise RemoteNotFoundError(f"remote '{name}' doesn't exist")
else:
remote_conf = kwargs
return _resolve_remote_refs(config, remote_conf)
return _resolve_remote_refs(repo, config, remote_conf)
def _resolve_remote_refs(config, remote_conf):
def _resolve_remote_refs(repo, config, remote_conf):
# Support for cross referenced remotes.
# This will merge the settings, shadowing base ref with remote_conf.
# For example, having:
@@ -79,8 +78,10 @@ def _resolve_remote_refs(config, remote_conf):
if parsed.scheme != "remote":
return remote_conf
base = get_fs_config(config, name=parsed.netloc)
url = posixpath.join(base["url"], parsed.path.lstrip("/"))
base = get_fs_config(repo, config, name=parsed.netloc)
cls, _, _ = get_cloud_fs(repo, **base)
relpath = parsed.path.lstrip("/").replace("/", cls.sep)
url = cls.sep.join((base["url"], relpath))
return {**base, **remote_conf, "url": url}
@@ -91,7 +92,7 @@ def get_cloud_fs(repo, **kwargs):
repo_config = repo.config if repo else {}
core_config = repo_config.get("core", {})
remote_conf = get_fs_config(repo_config, **kwargs)
remote_conf = get_fs_config(repo, repo_config, **kwargs)
try:
remote_conf = SCHEMA["remote"][str](remote_conf)
except Invalid as exc:
@@ -113,10 +114,8 @@ def get_cloud_fs(repo, **kwargs):
remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir
url = remote_conf.pop("url")
path_info = cls.PATH_CLS(
cls._strip_protocol(url) # pylint:disable=protected-access
)
fs_path = cls._strip_protocol(url) # pylint:disable=protected-access
extras = cls._get_kwargs_from_urls(url) # pylint:disable=protected-access
conf = {**extras, **remote_conf} # remote config takes priority
return cls, conf, path_info
return cls, conf, fs_path
+6 -5
View File
@@ -2,7 +2,6 @@ from dataclasses import dataclass, field
from typing import TYPE_CHECKING, List
from dvc.output import Output
from dvc.path_info import PathInfo
if TYPE_CHECKING:
from dvc.repo import Repo
@@ -16,7 +15,7 @@ class Metadata:
"""
# required field
path_info: PathInfo
fs_path: str
repo: "Repo"
# computed fields
@@ -47,11 +46,13 @@ class Metadata:
else:
out = self.outs[0]
# or, the path itself could be an output
self.is_output = self.path_info == out.path_info
self.is_output = self.fs_path == out.fs_path
# or, a directory must have an output somewhere deep inside it
if not self.is_output:
self.contains_outputs = out.path_info.isin(self.path_info)
self.contains_outputs = self.repo.fs.path.isin(
out.fs_path, self.fs_path
)
# or, the path could be a part of an output, i.e. inside of an output
self.part_of_output = not self.is_output and not self.contains_outputs
@@ -69,4 +70,4 @@ class Metadata:
return not self.isdir
def __str__(self):
return f"Metadata: {self.path_info}"
return f"Metadata: {self.fs_path}"
+7 -7
View File
@@ -7,7 +7,6 @@ from fsspec.utils import infer_storage_options
from funcy import cached_property, memoize, wrap_prop
from dvc.exceptions import DvcException
from dvc.path_info import CloudURLInfo
from dvc.scheme import Schemes
from dvc.utils import format_link
@@ -41,7 +40,6 @@ def _az_config():
# pylint:disable=abstract-method
class AzureFileSystem(CallbackMixin, ObjectFSWrapper):
scheme = Schemes.AZURE
PATH_CLS = CloudURLInfo
PARAM_CHECKSUM = "etag"
DETAIL_FIELDS = frozenset(("etag", "size"))
REQUIRES = {
@@ -52,12 +50,14 @@ class AzureFileSystem(CallbackMixin, ObjectFSWrapper):
@classmethod
def _strip_protocol(cls, path: str):
bucket = infer_storage_options(path).get("host")
if bucket:
return path
opts = infer_storage_options(path)
if opts.get("host"):
return "{host}{path}".format(**opts)
bucket = _az_config().get("storage", "container_name", None)
return f"azure://{bucket}"
return _az_config().get("storage", "container_name", None)
def unstrip_protocol(self, path: str) -> str:
return "azure://" + path.lstrip("/")
@staticmethod
def _get_kwargs_from_urls(urlpath):
+41 -36
View File
@@ -4,12 +4,12 @@ import os
from concurrent.futures import ThreadPoolExecutor, as_completed
from functools import partialmethod
from multiprocessing import cpu_count
from typing import Any, ClassVar, Dict, FrozenSet, Optional
from typing import ClassVar, Dict, FrozenSet, Optional
from funcy import cached_property
from tqdm.utils import CallbackIOWrapper
from dvc.exceptions import DvcException
from dvc.path_info import URLInfo
from dvc.progress import DEFAULT_CALLBACK, FsspecCallback
from dvc.ui import ui
from dvc.utils import tmp_fname
@@ -41,7 +41,6 @@ class BaseFileSystem:
scheme = "base"
REQUIRES: ClassVar[Dict[str, str]] = {}
PATH_CLS = URLInfo # type: Any
_JOBS = 4 * cpu_count()
CHECKSUM_DIR_SUFFIX = ".dir"
@@ -69,10 +68,19 @@ class BaseFileSystem:
def config(self):
return self._config
@cached_property
def path(self):
from .path import Path
return Path(self.sep)
@classmethod
def _strip_protocol(cls, path: str):
return path
def unstrip_protocol(self, path):
return path
@staticmethod
def _get_kwargs_from_urls(urlpath): # pylint:disable=unused-argument
return {}
@@ -129,36 +137,36 @@ class BaseFileSystem:
f"dependencies: {missing}. {hint}"
)
def checksum(self, path_info) -> str:
def checksum(self, path) -> str:
raise NotImplementedError
def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
def open(self, path, mode: str = "r", encoding: str = None, **kwargs):
raise RemoteActionNotImplemented("open", self.scheme)
def exists(self, path_info) -> bool:
def exists(self, path) -> bool:
raise NotImplementedError
# pylint: disable=unused-argument
def isdir(self, path_info):
def isdir(self, path):
"""Optional: Overwrite only if the remote has a way to distinguish
between a directory and a file.
"""
return False
def isfile(self, path_info):
def isfile(self, path):
"""Optional: Overwrite only if the remote has a way to distinguish
between a directory and a file.
"""
return True
def isexec(self, path_info):
def isexec(self, path):
"""Optional: Overwrite only if the remote has a way to distinguish
between executable and non-executable file.
"""
return False
def iscopy(self, path_info):
def iscopy(self, path):
"""Check if this file is an independent copy."""
return False # We can't be sure by default
@@ -166,28 +174,28 @@ class BaseFileSystem:
"""Return a generator with (root, dirs, files)."""
raise NotImplementedError
def walk_files(self, path_info, **kwargs):
"""Return a generator with `PathInfo`s to all the files.
def find(self, path, prefix=None):
"""Return a generator with `str`s to all the files.
Optional kwargs:
prefix (bool): If true `path_info` will be treated as a prefix
prefix (bool): If true `path` will be treated as a prefix
rather than directory path.
"""
raise NotImplementedError
def is_empty(self, path_info):
def is_empty(self, path):
return False
def info(self, path_info):
def info(self, path):
raise NotImplementedError
def getsize(self, path_info):
return self.info(path_info).get("size")
def getsize(self, path):
return self.info(path).get("size")
def remove(self, path_info):
def remove(self, path):
raise RemoteActionNotImplemented("remove", self.scheme)
def makedirs(self, path_info, **kwargs):
def makedirs(self, path, **kwargs):
"""Optional: Implement only if the remote needs to create
directories before copying/linking/moving data
"""
@@ -231,11 +239,10 @@ class BaseFileSystem:
if not hasattr(self, method):
raise RemoteActionNotImplemented(method, self.scheme)
if to_info.scheme != self.scheme:
raise NotImplementedError
if not is_file_obj:
desc = desc or from_info.name
from .local import localfs
desc = desc or localfs.path.name(from_info)
stack = contextlib.ExitStack()
if not callback:
@@ -261,9 +268,6 @@ class BaseFileSystem:
# pylint: disable=no-member
return self.upload_fobj(wrapped, to_info, size=total)
if from_info.scheme != "local":
raise NotImplementedError
logger.debug("Uploading '%s' to '%s'", from_info, to_info)
# pylint: disable=no-member
return self.put_file(
@@ -281,18 +285,14 @@ class BaseFileSystem:
_only_file=False,
**kwargs,
):
from .local import localfs
if not hasattr(self, "get_file"):
raise RemoteActionNotImplemented("get_file", self.scheme)
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme != "local":
raise NotImplementedError
download_dir = not _only_file and self.isdir(from_info)
desc = name or to_info.name
desc = name or localfs.path.name(to_info)
stack = contextlib.ExitStack()
if not callback:
pbar_kwargs = {"unit": "Files"} if download_dir else {}
@@ -323,13 +323,16 @@ class BaseFileSystem:
jobs=None,
**kwargs,
):
from_infos = list(self.walk_files(from_info, **kwargs))
from .local import localfs
from_infos = list(self.find(from_info, **kwargs))
if not from_infos:
makedirs(to_info, exist_ok=True)
return None
to_infos = (
to_info / info.relative_to(from_info) for info in from_infos
localfs.path.join(to_info, self.path.relpath(info, from_info))
for info in from_infos
)
callback.set_size(len(from_infos))
@@ -362,7 +365,9 @@ class BaseFileSystem:
to_info,
callback=DEFAULT_CALLBACK,
):
makedirs(to_info.parent, exist_ok=True)
from .local import localfs
makedirs(localfs.path.parent(to_info), exist_ok=True)
tmp_file = tmp_fname(to_info)
logger.debug("Downloading '%s' to '%s'", from_info, to_info)
+40 -41
View File
@@ -3,7 +3,6 @@ import os
import typing
from dvc.exceptions import OutputNotFoundError
from dvc.path_info import PathInfo
from dvc.utils import relpath
from ..progress import DEFAULT_CALLBACK
@@ -12,7 +11,7 @@ from .base import BaseFileSystem
if typing.TYPE_CHECKING:
from dvc.output import Output
from dvc.types import AnyPath
logger = logging.getLogger(__name__)
@@ -49,12 +48,9 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
return outs
def _get_granular_hash(
self, path_info: PathInfo, out: "Output", remote=None
):
assert isinstance(path_info, PathInfo)
def _get_granular_hash(self, path: "AnyPath", out: "Output", remote=None):
# NOTE: use string paths here for performance reasons
key = tuple(relpath(path_info, out.path_info).split(os.sep))
key = tuple(relpath(path, out.fs_path).split(os.sep))
out.get_dir_cache(remote=remote)
if out.obj is None:
raise FileNotFoundError
@@ -63,14 +59,14 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
return oid
raise FileNotFoundError
def _get_fs_path(self, path: PathInfo, remote=None):
def _get_fs_path(self, path: "AnyPath", remote=None):
try:
outs = self._find_outs(path, strict=False)
except OutputNotFoundError as exc:
raise FileNotFoundError from exc
if len(outs) != 1 or (
outs[0].is_dir_checksum and path == outs[0].path_info
outs[0].is_dir_checksum and path == outs[0].fs_path
):
raise IsADirectoryError
@@ -90,18 +86,20 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
checksum = self._get_granular_hash(path, out).value
else:
checksum = out.hash_info.value
remote_info = remote_odb.hash_to_path_info(checksum)
return remote_odb.fs, remote_info
remote_fs_path = remote_odb.hash_to_path(checksum)
return remote_odb.fs, remote_fs_path
if out.is_dir_checksum:
checksum = self._get_granular_hash(path, out).value
cache_path = out.odb.hash_to_path_info(checksum).url
cache_path = out.odb.fs.unstrip_protocol(
out.odb.hash_to_path(checksum)
)
else:
cache_path = out.cache_path
return out.odb.fs, cache_path
def open( # type: ignore
self, path: PathInfo, mode="r", encoding=None, **kwargs
self, path: str, mode="r", encoding=None, **kwargs
): # pylint: disable=arguments-renamed
fs, fspath = self._get_fs_path(path, **kwargs)
return fs.open(fspath, mode=mode, encoding=encoding)
@@ -120,18 +118,18 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
except FileNotFoundError:
return False
def check_isdir(self, path_info, outs):
def check_isdir(self, path, outs):
if len(outs) != 1:
return True
out = outs[0]
if not out.is_dir_checksum:
return out.path_info != path_info
if out.path_info == path_info:
return out.fs_path != path
if out.fs_path == path:
return True
try:
self._get_granular_hash(path_info, out)
self._get_granular_hash(path, out)
return False
except FileNotFoundError:
return True
@@ -153,7 +151,7 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
def _add_dir(self, trie, out, **kwargs):
self._fetch_dir(out, **kwargs)
base = out.path_info.parts
base = out.fs.path.parts(out.fs_path)
for key, _, _ in out.obj: # noqa: B301
trie[base + key] = None
@@ -161,14 +159,15 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
dirs = set()
files = []
out = trie.get(root.parts)
root_parts = self.path.parts(root)
out = trie.get(root_parts)
if out and out.is_dir_checksum:
self._add_dir(trie, out, **kwargs)
root_len = len(root.parts)
root_len = len(root_parts)
try:
for key, out in trie.iteritems(prefix=root.parts): # noqa: B301
if key == root.parts:
for key, out in trie.iteritems(prefix=root_parts): # noqa: B301
if key == root_parts:
continue
name = key[root_len]
@@ -182,16 +181,16 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
assert topdown
dirs = list(dirs)
yield root.fspath, dirs, files
yield root, dirs, files
for dname in dirs:
yield from self._walk(root / dname, trie)
yield from self._walk(self.path.join(root, dname), trie)
def walk(self, top, topdown=True, onerror=None, **kwargs):
from pygtrie import Trie
assert topdown
root = PathInfo(os.path.abspath(top))
root = os.path.abspath(top)
try:
meta = self.metadata(root)
except FileNotFoundError:
@@ -206,18 +205,18 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
trie = Trie()
for out in meta.outs:
trie[out.path_info.parts] = out
trie[out.fs.path.parts(out.fs_path)] = out
if out.is_dir_checksum and root.isin_or_eq(out.path_info):
if out.is_dir_checksum and self.path.isin_or_eq(root, out.fs_path):
self._add_dir(trie, out, **kwargs)
yield from self._walk(root, trie, topdown=topdown, **kwargs)
def walk_files(self, path_info, **kwargs):
for root, _, files in self.walk(path_info):
def find(self, path, prefix=None):
for root, _, files in self.walk(path):
for fname in files:
# NOTE: os.path.join is ~5.5 times slower
yield PathInfo(f"{root}{os.sep}{fname}")
yield f"{root}{os.sep}{fname}"
def isdvc(self, path, recursive=False, strict=True):
try:
@@ -228,23 +227,23 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
recurse = recursive or not strict
return meta.output_exists if recurse else meta.is_output
def isexec(self, path_info): # pylint: disable=unused-argument
def isexec(self, path): # pylint: disable=unused-argument
return False
def metadata(self, path_info):
path_info = PathInfo(os.path.abspath(path_info))
def metadata(self, fs_path):
abspath = os.path.abspath(fs_path)
try:
outs = self._find_outs(path_info, strict=False, recursive=True)
outs = self._find_outs(abspath, strict=False, recursive=True)
except OutputNotFoundError as exc:
raise FileNotFoundError from exc
meta = Metadata(path_info=path_info, outs=outs, repo=self.repo)
meta.isdir = meta.isdir or self.check_isdir(meta.path_info, meta.outs)
meta = Metadata(fs_path=abspath, outs=outs, repo=self.repo)
meta.isdir = meta.isdir or self.check_isdir(meta.fs_path, meta.outs)
return meta
def info(self, path_info):
meta = self.metadata(path_info)
def info(self, path):
meta = self.metadata(path)
ret = {"type": "directory" if meta.isdir else "file"}
if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info:
out = meta.outs[0]
@@ -252,7 +251,7 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
ret[out.hash_info.name] = out.hash_info.value
elif meta.part_of_output:
(out,) = meta.outs
key = path_info.relative_to(out.path_info).parts
key = self.path.parts(self.path.relpath(path, out.fs_path))
(obj_meta, oid) = out.obj.trie.get(key) or (None, None)
if oid:
ret["size"] = obj_meta.size if obj_meta else 0
@@ -268,8 +267,8 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
path, to_file, callback=callback, **kwargs
)
def checksum(self, path_info):
info = self.info(path_info)
def checksum(self, path):
info = self.info(path)
md5 = info.get("md5")
if md5:
return md5
+56 -165
View File
@@ -1,31 +1,18 @@
import os
import shutil
from functools import lru_cache
from typing import (
IO,
TYPE_CHECKING,
Any,
Dict,
Iterator,
Optional,
Union,
overload,
)
from typing import IO, TYPE_CHECKING, Any, Dict, Iterator, Optional, overload
from funcy import cached_property
from tqdm.utils import CallbackIOWrapper
from dvc.path_info import URLInfo
from dvc.progress import DEFAULT_CALLBACK
from .base import BaseFileSystem
FSPath = URLInfo
AnyFSPath = Union[FSPath, str]
FSPath = str
AnyFSPath = str
if TYPE_CHECKING:
from typing import Iterable
from typing_extensions import Literal
@@ -56,38 +43,6 @@ class FSSpecWrapper(BaseFileSystem):
def fs(self):
raise NotImplementedError
def _with_bucket(self, path: AnyFSPath) -> str:
return str(path)
def _strip_bucket(self, entry: str) -> str:
return entry
@overload
def _strip_buckets(
self, entries: "Iterable[Entry]", detail: "Literal[True]"
) -> "Iterator[Entry]":
...
@overload
def _strip_buckets(
self, entries: "Iterable[str]", detail: "Literal[False]" = False
) -> "Iterator[str]":
...
def _strip_buckets(self, entries, detail=False):
for entry in entries:
if detail:
entry = self._entry_hook(entry.copy())
entry["name"] = self._strip_bucket(entry["name"])
else:
entry = self._strip_bucket(entry)
yield entry
def _entry_hook(self, entry: "Entry") -> "Entry":
"""Simple hook method to be overridden when wanted to process
entries within info() and ls(detail=True) calls"""
return entry
def _prepare_credentials(
self, **config: Dict[str, Any] # pylint: disable=unused-argument
) -> Dict[str, Any]:
@@ -95,134 +50,95 @@ class FSSpecWrapper(BaseFileSystem):
host filesystem"""
return {}
def _isdir(self, path_info: FSPath) -> bool:
return self.fs.isdir(self._with_bucket(path_info))
def _isdir(self, path: AnyFSPath) -> bool:
return self.fs.isdir(path)
def isdir(self, path_info: FSPath) -> bool:
def isdir(self, path: AnyFSPath) -> bool:
try:
return self._isdir(path_info)
return self._isdir(path)
except FileNotFoundError:
return False
def isfile(self, path_info: FSPath) -> bool:
def isfile(self, path: AnyFSPath) -> bool:
try:
return not self._isdir(path_info)
return not self._isdir(path)
except FileNotFoundError:
return False
def is_empty(self, path_info: FSPath) -> bool:
entry = self.info(path_info)
def is_empty(self, path: AnyFSPath) -> bool:
entry = self.info(path)
if entry["type"] == "directory":
return not self.fs.ls(self._with_bucket(path_info))
return not self.fs.ls(path)
return entry["size"] == 0
def open(
self,
path_info: FSPath,
path: AnyFSPath,
mode: str = "r",
encoding: Optional[str] = None,
**kwargs,
) -> "IO": # pylint: disable=arguments-differ
return self.fs.open(self._with_bucket(path_info), mode=mode)
return self.fs.open(path, mode=mode)
def checksum(self, path_info: FSPath) -> str:
return self.fs.checksum(self._with_bucket(path_info))
def checksum(self, path: AnyFSPath) -> str:
return self.fs.checksum(path)
def copy(self, from_info: FSPath, to_info: FSPath) -> None:
self.makedirs(to_info.parent)
self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
def copy(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
self.makedirs(self.path.parent(to_info))
self.fs.copy(from_info, to_info)
def exists(self, path_info: FSPath) -> bool:
return self.fs.exists(self._with_bucket(path_info))
def exists(self, path: AnyFSPath) -> bool:
return self.fs.exists(path)
@overload
def ls(
self, path_info: FSPath, detail: "Literal[True]"
self, path: AnyFSPath, detail: "Literal[True]"
) -> "Iterator[Entry]":
...
@overload
def ls(self, path_info: FSPath, detail: "Literal[False]") -> Iterator[str]:
def ls(self, path: AnyFSPath, detail: "Literal[False]") -> Iterator[str]:
...
def ls(self, path_info, detail=False):
path = self._with_bucket(path_info)
files = self.fs.ls(path, detail=detail)
yield from self._strip_buckets(files, detail=detail)
def ls(self, path, detail=False):
yield from self.fs.ls(path, detail=detail)
@overload
def find(
self,
path_info: FSPath,
detail: "Literal[True]",
prefix: Optional[bool],
) -> "Iterator[Entry]":
...
def find(self, path, prefix=None):
yield from self.fs.find(path)
@overload
def find(
self,
path_info: FSPath,
detail: "Literal[False]",
prefix: Optional[bool],
) -> Iterator[str]:
...
def move(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
self.fs.move(from_info, to_info)
# pylint: disable=unused-argument
def find(self, path_info, detail=False, prefix=None):
path = self._with_bucket(path_info)
files = self.fs.find(path, detail=detail)
if detail:
files = files.values()
def remove(self, path: AnyFSPath) -> None:
self.fs.rm_file(path)
yield from self._strip_buckets(files, detail=detail)
def info(self, path: AnyFSPath) -> "Entry":
return self.fs.info(path)
def walk_files(self, path_info: FSPath, **kwargs) -> Iterator[FSPath]:
for file in self.find(path_info, **kwargs):
yield path_info.replace(path=file)
def move(self, from_info: FSPath, to_info: FSPath) -> None:
self.fs.move(self._with_bucket(from_info), self._with_bucket(to_info))
def remove(self, path_info: FSPath) -> None:
self.fs.rm_file(self._with_bucket(path_info))
def info(self, path_info: FSPath) -> "Entry":
info = self.fs.info(self._with_bucket(path_info)).copy()
info = self._entry_hook(info)
info["name"] = self._strip_bucket(info["name"])
return info
def makedirs(self, path_info: FSPath, **kwargs) -> None:
self.fs.makedirs(
self._with_bucket(path_info), exist_ok=kwargs.pop("exist_ok", True)
)
def makedirs(self, path: AnyFSPath, **kwargs) -> None:
self.fs.makedirs(path, exist_ok=kwargs.pop("exist_ok", True))
def put_file(
self,
from_file: FSPath,
to_info: FSPath,
from_file: AnyFSPath,
to_info: AnyFSPath,
callback: Any = DEFAULT_CALLBACK,
**kwargs,
) -> None:
self.fs.put_file(
from_file, self._with_bucket(to_info), callback=callback, **kwargs
)
self.fs.invalidate_cache(self._with_bucket(to_info.parent))
self.fs.put_file(from_file, to_info, callback=callback, **kwargs)
self.fs.invalidate_cache(self.path.parent(to_info))
def get_file(
self,
from_info: FSPath,
to_info: FSPath,
from_info: AnyFSPath,
to_info: AnyFSPath,
callback: Any = DEFAULT_CALLBACK,
**kwargs,
) -> None:
self.fs.get_file(
self._with_bucket(from_info), to_info, callback=callback, **kwargs
)
self.fs.get_file(from_info, to_info, callback=callback, **kwargs)
def upload_fobj(self, fobj: IO, to_info: FSPath, **kwargs) -> None:
self.makedirs(to_info.parent)
def upload_fobj(self, fobj: IO, to_info: AnyFSPath, **kwargs) -> None:
self.makedirs(self.path.parent(to_info))
with self.open(to_info, "wb") as fdest:
shutil.copyfileobj(
fobj, fdest, length=getattr(fdest, "blocksize", None)
@@ -233,22 +149,7 @@ class FSSpecWrapper(BaseFileSystem):
class ObjectFSWrapper(FSSpecWrapper):
TRAVERSE_PREFIX_LEN = 3
@lru_cache(512)
def _with_bucket(self, path: AnyFSPath) -> str:
if isinstance(path, self.PATH_CLS):
return f"{path.bucket}/{path.path}"
return str(path)
def _strip_bucket(self, entry: str) -> str:
try:
bucket, path = entry.split("/", 1)
except ValueError:
# If there is no path attached, only returns
# the bucket (top-level).
bucket, path = entry, ""
return path or bucket
def makedirs(self, path_info: FSPath, **kwargs) -> None:
def makedirs(self, path: AnyFSPath, **kwargs) -> None:
# For object storages make this method a no-op. The original
# fs.makedirs() method will only check if the bucket exists
# and create if it doesn't though we don't want to support
@@ -256,44 +157,37 @@ class ObjectFSWrapper(FSSpecWrapper):
# simply ignore all mkdir()/makedirs() calls.
return None
def _isdir(self, path_info: FSPath) -> bool:
def _isdir(self, path: AnyFSPath) -> bool:
# Directory in object storages are interpreted differently
# among different fsspec providers, so this logic is a temporary
# measure for us to adapt as of now. It checks whether it is a
# directory (as in a prefix with contents) or whether it is an empty
# file where it's name ends with a forward slash
entry = self.info(path_info)
entry = self.info(path)
return entry["type"] == "directory" or (
entry["size"] == 0
and entry["type"] == "file"
and entry["name"].endswith("/")
)
def find(self, path_info, detail=False, prefix=None):
def find(self, path, prefix=None):
if prefix:
path = self._with_bucket(path_info.parent)
files = self.fs.find(
path, detail=detail, prefix=path_info.parts[-1]
)
with_prefix = self.path.parent(path)
files = self.fs.find(with_prefix, prefix=self.path.parts(path)[-1])
else:
path = self._with_bucket(path_info)
files = self.fs.find(path, detail=detail)
if detail:
files = list(files.values())
with_prefix = path
files = self.fs.find(path)
# When calling find() on a file, it returns the same file in a list.
# For object-based storages, the same behavior applies to empty
# directories since they are represented as files. This condition
# checks whether we should yield an empty list (if it is an empty
# directory) or just yield the file itself.
if len(files) == 1 and files[0] == path and self.isdir(path_info):
if len(files) == 1 and files[0] == with_prefix and self.isdir(path):
return None
yield from self._strip_buckets( # pylint: disable=not-an-iterable
files, detail=detail
)
yield from files
# pylint: disable=arguments-differ
@@ -310,9 +204,6 @@ class NoDirectoriesMixin:
def walk(self, *args, **kwargs):
raise NotImplementedError
def walk_files(self, *args, **kwargs):
raise NotImplementedError
def ls(self, *args, **kwargs):
raise NotImplementedError
@@ -329,13 +220,13 @@ class CallbackMixin:
):
"""Add compatibility support for Callback."""
# pylint: disable=protected-access
self.makedirs(to_info.parent)
self.makedirs(self.path.parent(to_info))
size = os.path.getsize(from_file)
with open(from_file, "rb") as fobj:
callback.set_size(size)
wrapped = CallbackIOWrapper(callback.relative_update, fobj)
self.upload_fobj(wrapped, to_info)
self.fs.invalidate_cache(self._with_bucket(to_info.parent))
self.fs.invalidate_cache(self.path.parent(to_info))
def get_file(
self,
+20 -43
View File
@@ -1,14 +1,10 @@
import logging
import os
import posixpath
import re
import threading
from urllib.parse import urlparse
from funcy import cached_property, wrap_prop
from dvc.exceptions import DvcException
from dvc.path_info import CloudURLInfo
from dvc.scheme import Schemes
from dvc.utils import format_link, tmp_fname
@@ -34,24 +30,8 @@ class GDriveAuthError(DvcException):
super().__init__(message)
class GDriveURLInfo(CloudURLInfo):
def __init__(self, url):
super().__init__(url)
# GDrive URL host part is case sensitive,
# we are restoring it here.
p = urlparse(url)
self.host = p.netloc
assert self.netloc == self.host
# Normalize path. Important since we have a cache (path to ID)
# and don't want to deal with different variations of path in it.
self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
class GDriveFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
scheme = Schemes.GDRIVE
PATH_CLS = GDriveURLInfo
PARAM_CHECKSUM = "checksum"
REQUIRES = {"pydrive2": "pydrive2"}
# Always prefer traverse for GDrive since API usage quotas are a concern.
@@ -65,11 +45,14 @@ class GDriveFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
DEFAULT_GDRIVE_CLIENT_SECRET = "a1Fz59uTpVNeG_VGuSKDLJXv"
def __init__(self, **config):
from fsspec.utils import infer_storage_options
super().__init__(**config)
self.path_info = self.PATH_CLS(config["url"])
self.url = config["url"]
opts = infer_storage_options(self.url)
if not self.path_info.bucket:
if not opts["host"]:
raise DvcException(
"Empty GDrive URL '{}'. Learn more at {}".format(
config["url"],
@@ -77,8 +60,8 @@ class GDriveFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
)
)
self._bucket = self.path_info.bucket
self._path = self.path_info.path
self._bucket = opts["host"]
self._path = opts["path"].lstrip("/")
self._trash_only = config.get("gdrive_trash_only")
self._use_service_account = config.get("gdrive_use_service_account")
self._service_account_user_email = config.get(
@@ -106,6 +89,16 @@ class GDriveFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
)
)
@classmethod
def _strip_protocol(cls, path):
from fsspec.utils import infer_storage_options
return infer_storage_options(path)["path"].lstrip("/")
@staticmethod
def unstrip_protocol(path):
return "gdrive://{path}"
@staticmethod
def _get_kwargs_from_urls(urlpath):
return {"url": urlpath}
@@ -255,27 +248,11 @@ class GDriveFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
os.remove(temporary_save_path)
return _GDriveFileSystem(
self._with_bucket(self.path_info),
self.url,
gauth,
trash_only=self._trash_only,
)
def _with_bucket(self, path):
if isinstance(path, self.PATH_CLS):
return posixpath.join(path.bucket, path.path)
return super()._with_bucket(path)
def _strip_bucket(self, entry):
try:
bucket, path = entry.split("/", 1)
except ValueError:
# If there is no path attached, only returns
# the bucket (top-level).
bucket, path = entry, None
return path or bucket
def upload_fobj(self, fobj, to_info, **kwargs):
rpath = self._with_bucket(to_info)
self.makedirs(os.path.dirname(rpath))
return self.fs.upload_fobj(fobj, rpath, **kwargs)
self.makedirs(os.path.dirname(to_info))
return self.fs.upload_fobj(fobj, to_info, **kwargs)
+16 -16
View File
@@ -55,16 +55,16 @@ class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method
errno.EISDIR, os.strerror(errno.EISDIR), path
) from exc
def exists(self, path_info) -> bool:
key = self._get_key(path_info)
def exists(self, path) -> bool:
key = self._get_key(path)
return self.trie.exists(key)
def isdir(self, path_info) -> bool:
key = self._get_key(path_info)
def isdir(self, path) -> bool:
key = self._get_key(path)
return self.trie.isdir(key)
def isfile(self, path_info) -> bool:
key = self._get_key(path_info)
def isfile(self, path) -> bool:
key = self._get_key(path)
return self.trie.isfile(key)
def walk(self, top, topdown=True, onerror=None, **kwargs):
@@ -94,27 +94,27 @@ class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method
root = self._root
yield root, dirs, files
def isexec(self, path_info):
if not self.exists(path_info):
def isexec(self, path):
if not self.exists(path):
return False
mode = self.info(path_info)["mode"]
mode = self.info(path)["mode"]
return is_exec(mode)
def info(self, path_info):
key = self._get_key(path_info)
def info(self, path):
key = self._get_key(path)
try:
return self.trie.info(key)
except KeyError:
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), path_info
errno.ENOENT, os.strerror(errno.ENOENT), path
)
def checksum(self, path_info):
return self.info(path_info)["sha"]
def checksum(self, path):
return self.info(path)["sha"]
def walk_files(self, path_info, **kwargs):
for root, _, files in self.walk(path_info, **kwargs):
def find(self, path, prefix=None):
for root, _, files in self.walk(path):
for file in files:
# NOTE: os.path.join is ~5.5 times slower
yield f"{root}{os.sep}{file}"
-8
View File
@@ -1,9 +1,7 @@
import base64
import threading
from funcy import cached_property, wrap_prop
from dvc.path_info import CloudURLInfo
from dvc.scheme import Schemes
# pylint:disable=abstract-method
@@ -12,7 +10,6 @@ from .fsspec_wrapper import CallbackMixin, ObjectFSWrapper
class GSFileSystem(CallbackMixin, ObjectFSWrapper):
scheme = Schemes.GS
PATH_CLS = CloudURLInfo
REQUIRES = {"gcsfs": "gcsfs"}
PARAM_CHECKSUM = "etag"
DETAIL_FIELDS = frozenset(("etag", "size"))
@@ -23,11 +20,6 @@ class GSFileSystem(CallbackMixin, ObjectFSWrapper):
login_info["token"] = config.get("credentialpath")
return login_info
def _entry_hook(self, entry):
if "etag" in entry:
entry["etag"] = base64.b64decode(entry["etag"]).hex()
return entry
@wrap_prop(threading.Lock())
@cached_property
def fs(self):
+18 -9
View File
@@ -19,10 +19,17 @@ class HDFSFileSystem(CallbackMixin, FSSpecWrapper):
REQUIRES = {"fsspec": "fsspec", "pyarrow": "pyarrow"}
PARAM_CHECKSUM = "checksum"
def _with_bucket(self, path):
if isinstance(path, self.PATH_CLS):
return path.path
return super()._with_bucket(path)
@classmethod
def _strip_protocol(cls, path: str) -> str:
from fsspec.utils import infer_storage_options
return infer_storage_options(path)["path"].lstrip("/")
def unstrip_protocol(self, path: str) -> str:
host = self.fs_args["host"]
port = self.fs_args["port"]
path = path.lstrip("/")
return f"hdfs://{host}:{port}/{path}"
@staticmethod
def _get_kwargs_from_urls(urlpath):
@@ -41,17 +48,19 @@ class HDFSFileSystem(CallbackMixin, FSSpecWrapper):
return HadoopFileSystem(**self.fs_args)
def checksum(self, path_info):
return self._checksum(path_info)
def checksum(self, path):
return self._checksum(path)
def _checksum(self, path_info, **kwargs):
def _checksum(self, path, **kwargs):
# PyArrow doesn't natively support retrieving the
# checksum, so we have to use hadoop fs
url = self.unstrip_protocol(path)
result = self._run_command(
f"checksum {path_info.url}",
f"checksum {url}",
env=fix_env(os.environ),
user=path_info.user,
user=self.fs_args["user"],
)
if result is None:
return None
+2 -5
View File
@@ -3,7 +3,6 @@ import threading
from funcy import cached_property, memoize, wrap_with
from dvc import prompt
from dvc.path_info import HTTPURLInfo
from dvc.scheme import Schemes
from .fsspec_wrapper import FSSpecWrapper, NoDirectoriesMixin
@@ -34,7 +33,6 @@ def make_context(ssl_verify):
# pylint: disable=abstract-method
class HTTPFileSystem(NoDirectoriesMixin, FSSpecWrapper):
scheme = Schemes.HTTP
PATH_CLS = HTTPURLInfo
PARAM_CHECKSUM = "checksum"
REQUIRES = {"aiohttp": "aiohttp", "aiohttp-retry": "aiohttp_retry"}
CAN_TRAVERSE = False
@@ -127,6 +125,5 @@ class HTTPFileSystem(NoDirectoriesMixin, FSSpecWrapper):
return _HTTPFileSystem(**self.fs_args)
def _entry_hook(self, entry):
entry["checksum"] = entry.get("ETag") or entry.get("Content-MD5")
return entry
def unstrip_protocol(self, path: str) -> str:
return path
+54 -49
View File
@@ -1,7 +1,8 @@
import logging
import os
from dvc.path_info import PathInfo
from funcy import cached_property
from dvc.scheme import Schemes
from dvc.system import System
from dvc.utils import is_exec, tmp_fname
@@ -9,15 +10,23 @@ from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove
from ..progress import DEFAULT_CALLBACK
from .base import BaseFileSystem
from .path import Path
logger = logging.getLogger(__name__)
class LocalPath(Path):
def parent(self, path):
return os.path.dirname(path)
def drive(self, path):
return os.path.splitdrive(path)[0]
class LocalFileSystem(BaseFileSystem):
sep = os.sep
scheme = Schemes.LOCAL
PATH_CLS = PathInfo
PARAM_CHECKSUM = "md5"
PARAM_PATH = "path"
TRAVERSE_PREFIX_LEN = 2
@@ -28,33 +37,34 @@ class LocalFileSystem(BaseFileSystem):
super().__init__(**config)
self.fs = LocalFS()
@staticmethod
def open(path_info, mode="r", encoding=None, **kwargs):
return open(path_info, mode=mode, encoding=encoding)
@cached_property
def path(self):
return LocalPath(os.sep)
def exists(self, path_info) -> bool:
assert isinstance(path_info, str) or path_info.scheme == "local"
@staticmethod
def open(path, mode="r", encoding=None, **kwargs):
return open(path, mode=mode, encoding=encoding)
def exists(self, path) -> bool:
# TODO: replace this with os.path.exists once the problem is fixed on
# the fsspec https://github.com/intake/filesystem_spec/issues/742
return os.path.lexists(path_info)
return os.path.lexists(path)
def checksum(self, path_info) -> str:
def checksum(self, path) -> str:
from fsspec.utils import tokenize
st = os.stat(path_info)
st = os.stat(path)
return str(int(tokenize([st.st_ino, st.st_mtime, st.st_size]), 16))
def isfile(self, path_info) -> bool:
return os.path.isfile(path_info)
def isfile(self, path) -> bool:
return os.path.isfile(path)
def isdir(self, path_info) -> bool:
return os.path.isdir(path_info)
def isdir(self, path) -> bool:
return os.path.isdir(path)
def iscopy(self, path_info):
return not (
System.is_symlink(path_info) or System.is_hardlink(path_info)
)
def iscopy(self, path):
return not (System.is_symlink(path) or System.is_hardlink(path))
def walk(self, top, topdown=True, onerror=None, **kwargs):
"""Directory fs generator.
@@ -67,45 +77,37 @@ class LocalFileSystem(BaseFileSystem):
):
yield os.path.normpath(root), dirs, files
def walk_files(self, path_info, **kwargs):
for root, _, files in self.walk(path_info):
def find(self, path, prefix=None):
for root, _, files in self.walk(path):
for file in files:
# NOTE: os.path.join is ~5.5 times slower
yield PathInfo(f"{root}{os.sep}{file}")
yield f"{root}{os.sep}{file}"
def is_empty(self, path_info):
if self.isfile(path_info) and os.path.getsize(path_info) == 0:
def is_empty(self, path):
if self.isfile(path) and os.path.getsize(path) == 0:
return True
if self.isdir(path_info) and len(os.listdir(path_info)) == 0:
if self.isdir(path) and len(os.listdir(path)) == 0:
return True
return False
def remove(self, path_info):
if isinstance(path_info, PathInfo):
if path_info.scheme != "local":
raise NotImplementedError
remove(path_info)
def remove(self, path):
remove(path)
def makedirs(self, path_info, **kwargs):
makedirs(path_info, exist_ok=kwargs.pop("exist_ok", True))
def makedirs(self, path, **kwargs):
makedirs(path, exist_ok=kwargs.pop("exist_ok", True))
def isexec(self, path_info):
mode = self.info(path_info)["mode"]
def isexec(self, path):
mode = self.info(path)["mode"]
return is_exec(mode)
def move(self, from_info, to_info):
if (
isinstance(from_info, PathInfo) and from_info.scheme != "local"
) or (isinstance(to_info, PathInfo) and to_info.scheme != "local"):
raise NotImplementedError
self.makedirs(to_info.parent)
self.makedirs(self.path.parent(to_info))
move(from_info, to_info)
def copy(self, from_info, to_info):
tmp_info = to_info.parent / tmp_fname("")
tmp_info = self.path.join(self.path.parent(to_info), tmp_fname(""))
try:
copyfile(from_info, tmp_info)
os.rename(tmp_info, to_info)
@@ -114,8 +116,8 @@ class LocalFileSystem(BaseFileSystem):
raise
def upload_fobj(self, fobj, to_info, **kwargs):
self.makedirs(to_info.parent)
tmp_info = to_info.parent / tmp_fname("")
self.makedirs(self.path.parent(to_info))
tmp_info = self.path.join(self.path.parent(to_info), tmp_fname(""))
try:
copy_fobj_to_file(fobj, tmp_info)
os.rename(tmp_info, to_info)
@@ -128,8 +130,8 @@ class LocalFileSystem(BaseFileSystem):
System.symlink(from_info, to_info)
@staticmethod
def is_symlink(path_info):
return System.is_symlink(path_info)
def is_symlink(path):
return System.is_symlink(path)
def hardlink(self, from_info, to_info):
# If there are a lot of empty files (which happens a lot in datasets),
@@ -154,19 +156,19 @@ class LocalFileSystem(BaseFileSystem):
System.hardlink(from_info, to_info)
@staticmethod
def is_hardlink(path_info):
return System.is_hardlink(path_info)
def is_hardlink(path):
return System.is_hardlink(path)
def reflink(self, from_info, to_info):
System.reflink(from_info, to_info)
def info(self, path_info):
return self.fs.info(path_info)
def info(self, path):
return self.fs.info(path)
def put_file(
self, from_file, to_info, callback=DEFAULT_CALLBACK, **kwargs
):
makedirs(to_info.parent, exist_ok=True)
makedirs(self.path.parent(to_info), exist_ok=True)
tmp_file = tmp_fname(to_info)
copyfile(from_file, tmp_file, callback=callback)
os.replace(tmp_file, to_info)
@@ -175,3 +177,6 @@ class LocalFileSystem(BaseFileSystem):
self, from_info, to_file, callback=DEFAULT_CALLBACK, **kwargs
):
copyfile(from_info, to_file, callback=callback)
localfs = LocalFileSystem()
-2
View File
@@ -2,7 +2,6 @@ import threading
from funcy import cached_property, wrap_prop
from dvc.path_info import CloudURLInfo
from dvc.scheme import Schemes
from .fsspec_wrapper import FSSpecWrapper
@@ -11,7 +10,6 @@ from .fsspec_wrapper import FSSpecWrapper
class MemoryFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
scheme = Schemes.MEMORY
PARAM_CHECKSUM = "md5"
PATH_CLS = CloudURLInfo
TRAVERSE_PREFIX_LEN = 2
DEFAULT_BLOCKSIZE = 4096
-2
View File
@@ -4,7 +4,6 @@ import threading
from funcy import cached_property, wrap_prop
from dvc.path_info import CloudURLInfo
from dvc.scheme import Schemes
from .fsspec_wrapper import ObjectFSWrapper
@@ -15,7 +14,6 @@ logger = logging.getLogger(__name__)
# pylint:disable=abstract-method
class OSSFileSystem(ObjectFSWrapper):
scheme = Schemes.OSS
PATH_CLS = CloudURLInfo
REQUIRES = {"ossfs": "ossfs"}
PARAM_CHECKSUM = "etag"
COPY_POLL_SECONDS = 5
+81
View File
@@ -0,0 +1,81 @@
import posixpath
class Path:
"""
Class for operations on simple string paths.
This is meant to be very efficient and so doesn't use os.path,
doesn't have any notion of cwd and assumes that we are always
operating on absolute paths.
"""
def __init__(self, sep: str):
self.sep = sep
def join(self, *parts):
if len(parts) == 0:
return None
if len(parts) == 1:
return parts[0]
return self.sep.join(parts)
def parts(self, path):
return tuple(path.split(self.sep))
def parent(self, path):
parts = path.rsplit(self.sep, 1)
if len(parts) == 1:
return ""
return parts[0]
def parents(self, path):
parts = self.parts(path)
return tuple(
self.join(*parts[:length])
for length in range(len(parts) - 1, 0, -1)
)
def name(self, path):
return self.parts(path)[-1]
def suffix(self, path):
name = self.name(path)
_, dot, suffix = name.partition(".")
return dot + suffix
def with_name(self, path, name):
parts = list(self.parts(path))
parts[-1] = name
return self.join(*parts)
def with_suffix(self, path, suffix):
parts = list(self.parts(path))
real_path, _, _ = parts[-1].partition(".")
parts[-1] = real_path + suffix
return self.join(*parts)
def isin(self, left, right):
left_parts = self.parts(left)
right_parts = self.parts(right)
left_len = len(left_parts)
right_len = len(right_parts)
return left_len > right_len and left_parts[:right_len] == right_parts
def isin_or_eq(self, left, right):
return left == right or self.isin(left, right)
def overlaps(self, left, right):
# pylint: disable=arguments-out-of-order
return self.isin_or_eq(left, right) or self.isin(right, left)
def relpath(self, path, base):
assert len(path) > len(base)
assert path.startswith(base)
normpath = path.rstrip(self.sep)
normbase = base.rstrip(self.sep)
return normpath[len(normbase) + 1 :]
def as_posix(self, path):
return path.replace(self.sep, posixpath.sep)
+34 -38
View File
@@ -7,8 +7,6 @@ from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union
from funcy import lfilter, wrap_with
from dvc.path_info import PathInfo
from ..progress import DEFAULT_CALLBACK
from .base import BaseFileSystem
from .dvc import DvcFileSystem
@@ -164,7 +162,7 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
if not prefix:
return None
parents = (parent.fspath for parent in PathInfo(path).parents)
parents = (parent for parent in self.path.parents(path))
dirs = [path] + list(takewhile(lambda p: p != prefix, parents))
dirs.reverse()
self._update(dirs, starting_repo=repo)
@@ -217,34 +215,33 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
encoding = None
fs, dvc_fs = self._get_fs_pair(path)
path_info = PathInfo(path)
try:
return fs.open(path_info, mode=mode, encoding=encoding)
return fs.open(path, mode=mode, encoding=encoding)
except FileNotFoundError:
if not dvc_fs:
raise
return dvc_fs.open(path_info, mode=mode, encoding=encoding, **kwargs)
return dvc_fs.open(path, mode=mode, encoding=encoding, **kwargs)
def exists(self, path_info) -> bool:
fs, dvc_fs = self._get_fs_pair(path_info)
def exists(self, path) -> bool:
fs, dvc_fs = self._get_fs_pair(path)
if not dvc_fs:
return fs.exists(path_info)
return fs.exists(path)
if dvc_fs.repo.dvcignore.is_ignored(fs, path_info):
if dvc_fs.repo.dvcignore.is_ignored(fs, path):
return False
if fs.exists(path_info):
if fs.exists(path):
return True
try:
meta = dvc_fs.metadata(path_info)
meta = dvc_fs.metadata(path)
except FileNotFoundError:
return False
for out in meta.outs:
if fs.exists(out.path_info):
if fs.exists(out.fs_path):
return False
return True
@@ -271,7 +268,7 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
return False
for out in meta.outs:
if fs.exists(out.path_info):
if fs.exists(out.fs_path):
return False
return meta.isdir
@@ -303,15 +300,15 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
(out,) = meta.outs
assert len(meta.outs) == 1
if fs.exists(out.path_info):
if fs.exists(out.fs_path):
return False
return meta.isfile
def isexec(self, path_info):
fs, dvc_fs = self._get_fs_pair(path_info)
if dvc_fs and dvc_fs.exists(path_info):
return dvc_fs.isexec(path_info)
return fs.isexec(path_info)
def isexec(self, path):
fs, dvc_fs = self._get_fs_pair(path)
if dvc_fs and dvc_fs.exists(path):
return dvc_fs.isexec(path)
return fs.isexec(path)
def _dvc_walk(self, walk):
try:
@@ -444,10 +441,10 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
def walk_files(self, path_info, **kwargs):
for root, _, files in self.walk(path_info, **kwargs):
def find(self, path, prefix=None):
for root, _, files in self.walk(path):
for fname in files:
yield PathInfo(root) / fname
yield self.path.join(root, fname)
def get_file(
self, from_info, to_file, callback=DEFAULT_CALLBACK, **kwargs
@@ -467,18 +464,17 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
)
def metadata(self, path):
abspath = os.path.abspath(path)
path_info = PathInfo(abspath)
fs, dvc_fs = self._get_fs_pair(path_info)
fs_path = os.path.abspath(path)
fs, dvc_fs = self._get_fs_pair(fs_path)
dvc_meta = None
if dvc_fs:
with suppress(FileNotFoundError):
dvc_meta = dvc_fs.metadata(path_info)
dvc_meta = dvc_fs.metadata(fs_path)
info_result = None
with suppress(FileNotFoundError):
info_result = fs.info(path_info)
info_result = fs.info(fs_path)
if not info_result and not dvc_meta:
raise FileNotFoundError
@@ -486,8 +482,8 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
from ._metadata import Metadata
meta = dvc_meta or Metadata(
path_info=path_info,
repo=self._get_repo(abspath) or self._main_repo,
fs_path=fs_path,
repo=self._get_repo(fs_path) or self._main_repo,
)
isdir = bool(info_result) and info_result["type"] == "directory"
@@ -499,18 +495,18 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
meta.is_exec = bool(info_result) and is_exec(info_result["mode"])
return meta
def info(self, path_info):
fs, dvc_fs = self._get_fs_pair(path_info)
def info(self, path):
fs, dvc_fs = self._get_fs_pair(path)
try:
return fs.info(path_info)
return fs.info(path)
except FileNotFoundError:
return dvc_fs.info(path_info)
return dvc_fs.info(path)
def checksum(self, path_info):
fs, dvc_fs = self._get_fs_pair(path_info)
def checksum(self, path):
fs, dvc_fs = self._get_fs_pair(path)
try:
return fs.checksum(path_info)
return fs.checksum(path)
except FileNotFoundError:
return dvc_fs.checksum(path_info)
return dvc_fs.checksum(path)
+14 -12
View File
@@ -5,7 +5,6 @@ from collections import defaultdict
from funcy import cached_property, wrap_prop
from dvc.path_info import CloudURLInfo
from dvc.progress import DEFAULT_CALLBACK
from dvc.scheme import Schemes
@@ -17,7 +16,6 @@ _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
# pylint:disable=abstract-method
class BaseS3FileSystem(ObjectFSWrapper):
scheme = Schemes.S3
PATH_CLS = CloudURLInfo
REQUIRES = {"s3fs": "s3fs", "boto3": "boto3"}
PARAM_CHECKSUM = "etag"
DETAIL_FIELDS = frozenset(("etag", "size"))
@@ -148,12 +146,6 @@ class BaseS3FileSystem(ObjectFSWrapper):
}
)
def _entry_hook(self, entry):
entry = entry.copy()
if "ETag" in entry:
entry["etag"] = entry["ETag"].strip('"')
return entry
@wrap_prop(threading.Lock())
@cached_property
def fs(self):
@@ -161,6 +153,15 @@ class BaseS3FileSystem(ObjectFSWrapper):
return _S3FileSystem(**self.fs_args)
@classmethod
def _strip_protocol(cls, path: str) -> str:
from fsspec.utils import infer_storage_options
return infer_storage_options(path)["path"]
def unstrip_protocol(self, path):
return "s3://" + path.lstrip("/")
def _translate_exceptions(func):
@functools.wraps(func)
@@ -204,9 +205,10 @@ class S3FileSystem(BaseS3FileSystem): # pylint:disable=abstract-method
verify=client_kwargs.get("verify"),
)
def _get_obj(self, path_info):
bucket = self.s3.Bucket(path_info.bucket)
return bucket.Object(path_info.path)
def _get_obj(self, path):
bucket_name, key, _ = self.fs.split_path(path)
bucket = self.s3.Bucket(bucket_name)
return bucket.Object(key)
@_translate_exceptions
def put_file(
@@ -220,7 +222,7 @@ class S3FileSystem(BaseS3FileSystem): # pylint:disable=abstract-method
ExtraArgs=self.fs_args.get("s3_additional_kwargs"),
Config=self._transfer_config,
)
self.fs.invalidate_cache(self._with_bucket(to_info.parent))
self.fs.invalidate_cache(self.path.parent(to_info))
@_translate_exceptions
def get_file(
+11 -4
View File
@@ -34,10 +34,17 @@ class SSHFileSystem(FSSpecWrapper):
DEFAULT_PORT = 22
PARAM_CHECKSUM = "md5"
def _with_bucket(self, path):
if isinstance(path, self.PATH_CLS):
return path.path
return super()._with_bucket(path)
@classmethod
def _strip_protocol(cls, path: str) -> str:
from fsspec.utils import infer_storage_options
return infer_storage_options(path)["path"]
def unstrip_protocol(self, path: str) -> str:
host = self.fs_args["host"]
port = self.fs_args["port"]
path = path.lstrip("/")
return f"ssh://{host}:{port}/{path}"
def _prepare_credentials(self, **config):
self.CAN_TRAVERSE = True
+3 -5
View File
@@ -7,7 +7,7 @@ from .base import RemoteActionNotImplemented
from .local import LocalFileSystem
if TYPE_CHECKING:
from dvc.types import AnyPath, DvcPath
from dvc.types import AnyPath
from .base import BaseFileSystem
@@ -18,7 +18,7 @@ def _link(
from_fs: "BaseFileSystem",
from_info: "AnyPath",
to_fs: "BaseFileSystem",
to_info: "DvcPath",
to_info: "AnyPath",
hardlink: bool = False,
) -> None:
if not isinstance(from_fs, type(to_fs)):
@@ -47,7 +47,7 @@ def transfer(
from_fs: "BaseFileSystem",
from_info: "AnyPath",
to_fs: "BaseFileSystem",
to_info: "DvcPath",
to_info: "AnyPath",
hardlink: bool = False,
) -> None:
try:
@@ -58,8 +58,6 @@ def transfer(
raise
if isinstance(from_fs, LocalFileSystem):
if not isinstance(from_info, from_fs.PATH_CLS):
from_info = from_fs.PATH_CLS(from_info)
return to_fs.upload(from_info, to_info)
if isinstance(to_fs, LocalFileSystem):
+27 -18
View File
@@ -1,10 +1,8 @@
import logging
import threading
from functools import lru_cache
from funcy import cached_property, wrap_prop
from dvc.path_info import WebDAVURLInfo
from dvc.scheme import Schemes
from .fsspec_wrapper import FSSpecWrapper
@@ -15,7 +13,6 @@ logger = logging.getLogger(__name__)
class WebDAVFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
scheme = Schemes.WEBDAV
PATH_CLS = WebDAVURLInfo
CAN_TRAVERSE = True
TRAVERSE_PREFIX_LEN = 2
REQUIRES = {"webdav4": "webdav4"}
@@ -37,14 +34,35 @@ class WebDAVFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
}
)
@classmethod
def _strip_protocol(cls, path: str) -> str:
from fsspec.utils import infer_storage_options
return infer_storage_options(path)["path"].lstrip("/")
def unstrip_protocol(self, path: str) -> str:
return self.fs_args["base_url"] + path
@staticmethod
def _get_kwargs_from_urls(urlpath):
path_info = WebDAVURLInfo(urlpath)
from urllib.parse import urlparse, urlunparse
parsed = urlparse(urlpath)
scheme = parsed.scheme.replace("webdav", "http")
return {
"prefix": path_info.path.rstrip("/"),
"host": path_info.replace(path="").url,
"url": path_info.url.rstrip("/"),
"user": path_info.user,
"prefix": parsed.path.rstrip("/"),
"host": urlunparse((scheme, parsed.netloc, "", None, None, None)),
"url": urlunparse(
(
scheme,
parsed.netloc,
parsed.path.rstrip("/"),
None,
None,
None,
)
),
"user": parsed.username,
}
def _prepare_credentials(self, **config):
@@ -71,20 +89,11 @@ class WebDAVFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
return WebdavFileSystem(**self.fs_args)
def upload_fobj(self, fobj, to_info, **kwargs):
rpath = self.translate_path_info(to_info)
size = kwargs.get("size")
# using upload_fileobj to directly upload fileobj rather than buffering
# and using overwrite=True to avoid check for an extra exists call,
# as caller should ensure that the file does not exist beforehand.
return self.fs.upload_fileobj(fobj, rpath, overwrite=True, size=size)
@lru_cache(512)
def translate_path_info(self, path):
if isinstance(path, self.PATH_CLS):
return path.path[len(self.prefix) :].lstrip("/")
return path
_with_bucket = translate_path_info # type: ignore
return self.fs.upload_fileobj(fobj, to_info, overwrite=True, size=size)
class WebDAVSFileSystem(WebDAVFileSystem): # pylint:disable=abstract-method
+12 -8
View File
@@ -2,7 +2,6 @@ import threading
from funcy import cached_property, wrap_prop
from dvc.path_info import CloudURLInfo
from dvc.scheme import Schemes
# pylint:disable=abstract-method
@@ -11,14 +10,20 @@ from .fsspec_wrapper import CallbackMixin, FSSpecWrapper
class WebHDFSFileSystem(CallbackMixin, FSSpecWrapper):
scheme = Schemes.WEBHDFS
PATH_CLS = CloudURLInfo
REQUIRES = {"fsspec": "fsspec"}
PARAM_CHECKSUM = "checksum"
def _with_bucket(self, path):
if isinstance(path, self.PATH_CLS):
return f"/{path.path.rstrip('/')}"
return path
@classmethod
def _strip_protocol(cls, path: str) -> str:
from fsspec.utils import infer_storage_options
return infer_storage_options(path)["path"]
def unstrip_protocol(self, path: str) -> str:
host = self.fs_args["host"]
port = self.fs_args["port"]
path = path.lstrip("/")
return f"webhdfs://{host}:{port}/{path}"
@staticmethod
def _get_kwargs_from_urls(urlpath):
@@ -46,7 +51,6 @@ class WebHDFSFileSystem(CallbackMixin, FSSpecWrapper):
fs.session.verify = self._ssl_verify
return fs
def checksum(self, path_info):
path = self._with_bucket(path_info)
def checksum(self, path):
ukey = self.fs.ukey(path)
return ukey["bytes"]
+9 -11
View File
@@ -8,7 +8,7 @@ from pathspec.patterns import GitWildMatchPattern
from pathspec.util import normalize_file
from dvc.fs.base import BaseFileSystem
from dvc.path_info import PathInfo
from dvc.fs.local import localfs
from dvc.pathspec_math import PatternInfo, merge_patterns
from dvc.scheme import Schemes
from dvc.types import AnyPath, List, Optional
@@ -267,25 +267,25 @@ class DvcIgnoreFilter:
dirs, files = ignore_pattern(abs_root, dirs, files)
return dirs, files
def walk(self, fs: BaseFileSystem, path_info: AnyPath, **kwargs):
def walk(self, fs: BaseFileSystem, path: AnyPath, **kwargs):
ignore_subrepos = kwargs.pop("ignore_subrepos", True)
if fs.scheme == Schemes.LOCAL:
for root, dirs, files in fs.walk(path_info, **kwargs):
for root, dirs, files in fs.walk(path, **kwargs):
dirs[:], files[:] = self(
root, dirs, files, ignore_subrepos=ignore_subrepos
)
yield root, dirs, files
else:
yield from fs.walk(path_info, **kwargs)
yield from fs.walk(path, **kwargs)
def walk_files(self, fs: BaseFileSystem, path_info: AnyPath, **kwargs):
def find(self, fs: BaseFileSystem, path: AnyPath, **kwargs):
if fs.scheme == Schemes.LOCAL:
for root, _, files in self.walk(fs, path_info, **kwargs):
for root, _, files in self.walk(fs, path, **kwargs):
for file in files:
# NOTE: os.path.join is ~5.5 times slower
yield PathInfo(f"{root}{os.sep}{file}")
yield f"{root}{os.sep}{file}"
else:
yield from fs.walk_files(path_info)
yield from fs.find(path)
def _get_trie_pattern(
self, dirname, dnames: Optional["List"] = None, ignore_subrepos=True
@@ -307,7 +307,7 @@ class DvcIgnoreFilter:
dirs = list(
takewhile(
lambda path: path != prefix,
(parent.fspath for parent in PathInfo(dirname).parents),
(parent for parent in localfs.path.parents(dirname)),
)
)
dirs.reverse()
@@ -343,8 +343,6 @@ class DvcIgnoreFilter:
return self._is_ignored(path, False)
def _outside_repo(self, path):
path = PathInfo(path)
# paths outside of the repo should be ignored
path = relpath(path, self.root_dir)
if path.startswith("..") or (
+1 -1
View File
@@ -75,7 +75,7 @@ def _get_caches(cache):
def _get_remotes(config):
schemes = (
get_fs_cls(get_fs_config(config, name=remote)).scheme
get_fs_cls(get_fs_config(None, config, name=remote)).scheme
for remote in config["remote"]
)
+50 -50
View File
@@ -23,36 +23,36 @@ from dvc.types import Optional
logger = logging.getLogger(__name__)
def _remove(path_info, fs, in_cache, force=False):
if not fs.exists(path_info):
def _remove(fs_path, fs, in_cache, force=False):
if not fs.exists(fs_path):
return
if force:
fs.remove(path_info)
fs.remove(fs_path)
return
if not in_cache:
msg = (
f"file/directory '{path_info}' is going to be removed. "
f"file/directory '{fs_path}' is going to be removed. "
"Are you sure you want to proceed?"
)
if not prompt.confirm(msg):
raise ConfirmRemoveError(str(path_info))
raise ConfirmRemoveError(fs_path)
fs.remove(path_info)
fs.remove(fs_path)
def _verify_link(cache, path_info, link_type):
if link_type == "hardlink" and cache.fs.getsize(path_info) == 0:
def _verify_link(cache, fs_path, link_type):
if link_type == "hardlink" and cache.fs.getsize(fs_path) == 0:
return
if cache.cache_type_confirmed:
return
is_link = getattr(cache.fs, f"is_{link_type}", None)
if is_link and not is_link(path_info):
cache.fs.remove(path_info)
if is_link and not is_link(fs_path):
cache.fs.remove(fs_path)
raise DvcException(f"failed to verify {link_type}")
cache.cache_type_confirmed = True
@@ -90,14 +90,14 @@ def _try_links(cache, from_info, to_info, link_types):
def _link(cache, from_info, to_info):
cache.makedirs(to_info.parent)
cache.makedirs(cache.fs.path.parent(to_info))
try:
_try_links(cache, from_info, to_info, cache.cache_types)
except FileNotFoundError as exc:
raise CheckoutError([str(to_info)]) from exc
def _confirm_cache_type(cache, path_info):
def _confirm_cache_type(cache, fs_path):
"""Checks whether cache uses copies."""
if cache.cache_type_confirmed:
return
@@ -105,10 +105,12 @@ def _confirm_cache_type(cache, path_info):
if set(cache.cache_types) <= {"copy"}:
return
workspace_file = path_info.with_name("." + uuid())
test_cache_file = cache.path_info / ".cache_type_test_file"
workspace_file = cache.fs.path.with_name(fs_path, "." + uuid())
test_cache_file = cache.fs.path.join(
cache.fs_path, ".cache_type_test_file"
)
if not cache.fs.exists(test_cache_file):
cache.makedirs(test_cache_file.parent)
cache.makedirs(cache.fs.path.parent(test_cache_file))
with cache.fs.open(test_cache_file, "wb") as fobj:
fobj.write(bytes(1))
try:
@@ -120,9 +122,9 @@ def _confirm_cache_type(cache, path_info):
cache.cache_type_confirmed = True
def _relink(cache, cache_info, fs, path_info, in_cache, force):
_remove(path_info, fs, in_cache, force=force)
_link(cache, cache_info, path_info)
def _relink(cache, cache_info, fs, path, in_cache, force):
_remove(path, fs, in_cache, force=force)
_link(cache, cache_info, path)
# NOTE: Depending on a file system (e.g. on NTFS), `_remove` might reset
# read-only permissions in order to delete a hardlink to protected object,
# which will also reset it for the object itself, making it unprotected,
@@ -131,7 +133,7 @@ def _relink(cache, cache_info, fs, path_info, in_cache, force):
def _checkout_file(
path_info,
fs_path,
fs,
change,
cache,
@@ -142,20 +144,19 @@ def _checkout_file(
):
"""The file is changed we need to checkout a new copy"""
modified = False
_confirm_cache_type(cache, fs_path)
_confirm_cache_type(cache, path_info)
cache_info = cache.hash_to_path_info(change.new.oid.value)
cache_fs_path = cache.hash_to_path(change.new.oid.value)
if change.old.oid:
if relink:
if fs.iscopy(path_info) and cache.cache_types[0] == "copy":
cache.unprotect(path_info)
if fs.iscopy(fs_path) and cache.cache_types[0] == "copy":
cache.unprotect(fs_path)
else:
_relink(
cache,
cache_info,
cache_fs_path,
fs,
path_info,
fs_path,
change.old.in_cache,
force=force,
)
@@ -163,27 +164,27 @@ def _checkout_file(
modified = True
_relink(
cache,
cache_info,
cache_fs_path,
fs,
path_info,
fs_path,
change.old.in_cache,
force=force,
)
else:
_link(cache, cache_info, path_info)
_link(cache, cache_fs_path, fs_path)
modified = True
if state:
state.save(path_info, fs, change.new.oid)
state.save(fs_path, fs, change.new.oid)
if progress_callback:
progress_callback(str(path_info))
progress_callback(fs_path)
return modified
def _diff(
path_info,
fs_path,
fs,
obj,
cache,
@@ -194,7 +195,7 @@ def _diff(
try:
_, _, old = stage(
cache,
path_info,
fs_path,
fs,
obj.hash_info.name if obj else cache.fs.PARAM_CHECKSUM,
dry_run=True,
@@ -213,7 +214,7 @@ def _diff(
def _checkout(
diff,
path_info,
fs_path,
fs,
cache,
force=False,
@@ -226,18 +227,18 @@ def _checkout(
for change in diff.deleted:
entry_path = (
path_info.joinpath(*change.old.key)
fs.path.join(fs_path, *change.old.key)
if change.old.key != ROOT
else path_info
else fs_path
)
_remove(entry_path, fs, change.old.in_cache, force=force)
failed = []
for change in chain(diff.added, diff.modified):
entry_path = (
path_info.joinpath(*change.new.key)
fs.path.join(fs_path, *change.new.key)
if change.new.key != ROOT
else path_info
else fs_path
)
if change.new.oid.isdir:
fs.makedirs(entry_path)
@@ -262,7 +263,7 @@ def _checkout(
def checkout(
path_info,
fs_path,
fs,
obj,
cache,
@@ -273,12 +274,11 @@ def checkout(
dvcignore: Optional[DvcIgnoreFilter] = None,
state=None,
):
if path_info.scheme not in ["local", cache.fs.scheme]:
raise NotImplementedError
# if scheme(fs_path) not in ["local", cache.fs.scheme]:
# raise NotImplementedError
diff = _diff(
path_info,
fs_path,
fs,
obj,
cache,
@@ -291,16 +291,16 @@ def checkout(
if not quiet:
logger.warning(
"No file hash info found for '%s'. It won't be created.",
path_info,
fs_path,
)
failed.append(str(path_info))
failed.append(fs_path)
elif not diff:
logger.trace("Data '%s' didn't change.", path_info) # type: ignore
logger.trace("Data '%s' didn't change.", fs_path) # type: ignore
try:
_checkout(
diff,
path_info,
fs_path,
fs,
cache,
force=force,
@@ -312,13 +312,13 @@ def checkout(
failed.extend(exc.target_infos)
if diff and state:
state.save_link(path_info, fs)
state.save_link(fs_path, fs)
if not failed:
state.save(path_info, fs, obj.hash_info)
state.save(fs_path, fs, obj.hash_info)
if failed or not diff:
if progress_callback and obj:
progress_callback(str(path_info), len(obj))
progress_callback(fs_path, len(obj))
if failed:
raise CheckoutError(failed)
return
+10 -8
View File
@@ -6,22 +6,22 @@ if TYPE_CHECKING:
from .index import ObjectDBIndexBase
def get_odb(fs, path_info, **config):
def get_odb(fs, fs_path, **config):
from .base import ObjectDB
from .gdrive import GDriveObjectDB
from .local import LocalObjectDB
from .oss import OSSObjectDB
if fs.scheme == Schemes.LOCAL:
return LocalObjectDB(fs, path_info, **config)
return LocalObjectDB(fs, fs_path, **config)
if fs.scheme == Schemes.GDRIVE:
return GDriveObjectDB(fs, path_info, **config)
return GDriveObjectDB(fs, fs_path, **config)
if fs.scheme == Schemes.OSS:
return OSSObjectDB(fs, path_info, **config)
return OSSObjectDB(fs, fs_path, **config)
return ObjectDB(fs, path_info, **config)
return ObjectDB(fs, fs_path, **config)
def _get_odb(repo, settings):
@@ -30,9 +30,9 @@ def _get_odb(repo, settings):
if not settings:
return None
cls, config, path_info = get_cloud_fs(repo, **settings)
cls, config, fs_path = get_cloud_fs(repo, **settings)
config["tmp_dir"] = repo.tmp_dir
return get_odb(cls(**config), path_info, state=repo.state, **config)
return get_odb(cls(**config), fs_path, state=repo.state, **config)
def get_index(odb) -> "ObjectDBIndexBase":
@@ -43,7 +43,9 @@ def get_index(odb) -> "ObjectDBIndexBase":
cls = ObjectDBIndex if odb.tmp_dir else ObjectDBIndexNoop
return cls(
odb.tmp_dir,
hashlib.sha256(odb.path_info.url.encode("utf-8")).hexdigest(),
hashlib.sha256(
odb.fs.unstrip_protocol(odb.fs_path).encode("utf-8")
).hexdigest(),
odb.fs.CHECKSUM_DIR_SUFFIX,
)
+44 -46
View File
@@ -12,7 +12,7 @@ from dvc.progress import Tqdm
if TYPE_CHECKING:
from dvc.fs.base import BaseFileSystem
from dvc.hash_info import HashInfo
from dvc.types import AnyPath, DvcPath
from dvc.types import AnyPath
logger = logging.getLogger(__name__)
@@ -23,11 +23,11 @@ class ObjectDB:
DEFAULT_CACHE_TYPES = ["copy"]
CACHE_MODE: Optional[int] = None
def __init__(self, fs: "BaseFileSystem", path_info: "AnyPath", **config):
def __init__(self, fs: "BaseFileSystem", path: str, **config):
from dvc.state import StateNoop
self.fs = fs
self.path_info = path_info
self.fs_path = path
self.state = config.get("state", StateNoop())
self.verify = config.get("verify", self.DEFAULT_VERIFY)
self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES)
@@ -50,26 +50,25 @@ class ObjectDB:
def __eq__(self, other):
return (
self.fs == other.fs
and self.path_info == other.path_info
and self.fs_path == other.fs_path
and self.read_only == other.read_only
)
def __hash__(self):
return hash((self.fs.scheme, self.path_info))
return hash((self.fs.scheme, self.fs_path))
def exists(self, hash_info: "HashInfo"):
return self.fs.exists(self.hash_to_path_info(hash_info.value))
return self.fs.exists(self.hash_to_path(hash_info.value))
def move(self, from_info, to_info):
self.fs.move(from_info, to_info)
def makedirs(self, path_info):
self.fs.makedirs(path_info)
def makedirs(self, fs_path):
self.fs.makedirs(fs_path)
def get(self, hash_info: "HashInfo"):
"""get raw object"""
return HashFile(
# Prefer string path over PathInfo when possible due to performance
self.hash_to_path(hash_info.value),
self.fs,
hash_info,
@@ -79,13 +78,13 @@ class ObjectDB:
self,
from_fs: "BaseFileSystem",
from_info: "AnyPath",
to_info: "DvcPath",
to_info: "AnyPath",
_hash_info: "HashInfo",
hardlink: bool = False,
):
from dvc import fs
self.makedirs(to_info.parent)
self.makedirs(self.fs.path.parent(to_info))
return fs.utils.transfer(
from_fs,
from_info,
@@ -96,7 +95,7 @@ class ObjectDB:
def add(
self,
path_info: "AnyPath",
fs_path: "AnyPath",
fs: "BaseFileSystem",
hash_info: "HashInfo",
hardlink: bool = False,
@@ -113,35 +112,32 @@ class ObjectDB:
except (ObjectFormatError, FileNotFoundError):
pass
cache_info = self.hash_to_path_info(hash_info.value)
self._add_file(fs, path_info, cache_info, hash_info, hardlink=hardlink)
cache_fs_path = self.hash_to_path(hash_info.value)
self._add_file(
fs, fs_path, cache_fs_path, hash_info, hardlink=hardlink
)
try:
if verify:
self.check(hash_info, check_hash=True)
self.protect(cache_info)
self.state.save(cache_info, self.fs, hash_info)
self.protect(cache_fs_path)
self.state.save(cache_fs_path, self.fs, hash_info)
except (ObjectFormatError, FileNotFoundError):
pass
def hash_to_path_info(self, hash_) -> "DvcPath":
return self.path_info / hash_[0:2] / hash_[2:]
# Override to return path as a string instead of PathInfo for clouds
# which support string paths (see local)
def hash_to_path(self, hash_):
return self.hash_to_path_info(hash_)
return self.fs.path.join(self.fs_path, hash_[0:2], hash_[2:])
def protect(self, path_info): # pylint: disable=unused-argument
def protect(self, fs_path): # pylint: disable=unused-argument
pass
def is_protected(self, path_info): # pylint: disable=unused-argument
def is_protected(self, fs_path): # pylint: disable=unused-argument
return False
def unprotect(self, path_info): # pylint: disable=unused-argument
def unprotect(self, fs_path): # pylint: disable=unused-argument
pass
def set_exec(self, path_info): # pylint: disable=unused-argument
def set_exec(self, fs_path): # pylint: disable=unused-argument
pass
def check(
@@ -164,45 +160,47 @@ class ObjectDB:
"""
obj = self.get(hash_info)
if self.is_protected(obj.path_info):
if self.is_protected(obj.fs_path):
logger.trace( # type: ignore[attr-defined]
"Assuming '%s' is unchanged since it is read-only",
obj.path_info,
obj.fs_path,
)
return
try:
obj.check(self, check_hash=check_hash)
except ObjectFormatError:
logger.warning("corrupted cache file '%s'.", obj.path_info)
logger.warning("corrupted cache file '%s'.", obj.fs_path)
with suppress(FileNotFoundError):
self.fs.remove(obj.path_info)
self.fs.remove(obj.fs_path)
raise
if check_hash:
# making cache file read-only so we don't need to check it
# next time
self.protect(obj.path_info)
self.protect(obj.fs_path)
def _list_paths(self, prefix=None, progress_callback=None):
if prefix:
if len(prefix) > 2:
path_info = self.path_info / prefix[:2] / prefix[2:]
fs_path = self.fs.path.join(
self.fs_path, prefix[:2], prefix[2:]
)
else:
path_info = self.path_info / prefix[:2]
fs_path = self.fs.path.join(self.fs_path, prefix[:2])
prefix = True
else:
path_info = self.path_info
fs_path = self.fs_path
prefix = False
if progress_callback:
for file_info in self.fs.walk_files(path_info, prefix=prefix):
for file_info in self.fs.find(fs_path, prefix=prefix):
progress_callback()
yield file_info.path
yield file_info
else:
yield from self.fs.walk_files(path_info, prefix=prefix)
yield from self.fs.find(fs_path, prefix=prefix)
def _path_to_hash(self, path):
parts = self.fs.PATH_CLS(path).parts[-2:]
parts = self.fs.path.parts(path)[-2:]
if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
raise ValueError(f"Bad cache file path '{path}'")
@@ -378,18 +376,18 @@ class ObjectDB:
removed = False
# hashes must be sorted to ensure we always remove .dir files first
for hash_ in sorted(
self.all(jobs, str(self.path_info)),
self.all(jobs, self.fs_path),
key=self.fs.is_dir_hash,
reverse=True,
):
if hash_ in used_hashes:
continue
path_info = self.hash_to_path_info(hash_)
fs_path = self.hash_to_path(hash_)
if self.fs.is_dir_hash(hash_):
# backward compatibility
# pylint: disable=protected-access
self._remove_unpacked_dir(hash_)
self.fs.remove(path_info)
self.fs.remove(fs_path)
removed = True
return removed
@@ -406,16 +404,16 @@ class ObjectDB:
unit="file",
) as pbar:
def exists_with_progress(path_info):
ret = self.fs.exists(path_info)
pbar.update_msg(str(path_info))
def exists_with_progress(fs_path):
ret = self.fs.exists(fs_path)
pbar.update_msg(fs_path)
return ret
with ThreadPoolExecutor(
max_workers=jobs or self.fs.jobs
) as executor:
path_infos = map(self.hash_to_path_info, hashes)
in_remote = executor.map(exists_with_progress, path_infos)
fs_paths = map(self.hash_to_path, hashes)
in_remote = executor.map(exists_with_progress, fs_paths)
ret = list(itertools.compress(hashes, in_remote))
return ret
+36 -33
View File
@@ -7,7 +7,6 @@ from shortuuid import uuid
from dvc.hash_info import HashInfo
from dvc.objects.errors import ObjectFormatError
from dvc.path_info import PathInfo
from dvc.progress import Tqdm
from dvc.utils import relpath
from dvc.utils.fs import copyfile, remove, umask, walk_files
@@ -22,9 +21,9 @@ class LocalObjectDB(ObjectDB):
CACHE_MODE = 0o444
UNPACKED_DIR_SUFFIX = ".unpacked"
def __init__(self, fs, path_info, **config):
super().__init__(fs, path_info, **config)
self.cache_dir = path_info.fspath
def __init__(self, fs, fs_path, **config):
super().__init__(fs, fs_path, **config)
self.cache_dir = fs_path
shared = config.get("shared")
if shared:
@@ -36,11 +35,11 @@ class LocalObjectDB(ObjectDB):
@property
def cache_dir(self):
return self.path_info.fspath if self.path_info else None
return self.fs_path if self.fs_path else None
@cache_dir.setter
def cache_dir(self, value):
self.path_info = PathInfo(value) if value else None
self.fs_path = value
@cached_property
def cache_path(self):
@@ -50,10 +49,10 @@ class LocalObjectDB(ObjectDB):
super().move(from_info, to_info)
os.chmod(to_info, self._file_mode)
def makedirs(self, path_info):
def makedirs(self, fs_path):
from dvc.utils.fs import makedirs
makedirs(path_info, exist_ok=True, mode=self._dir_mode)
makedirs(fs_path, exist_ok=True, mode=self._dir_mode)
def hash_to_path(self, hash_):
# NOTE: `self.cache_path` is already normalized so we can simply use
@@ -81,26 +80,30 @@ class LocalObjectDB(ObjectDB):
return ret
def _list_paths(self, prefix=None, progress_callback=None):
assert self.path_info is not None
assert self.fs_path is not None
if prefix:
path_info = self.path_info / prefix[:2]
if not self.fs.exists(path_info):
fs_path = self.fs.path.join(self.fs_path, prefix[:2])
if not self.fs.exists(fs_path):
return
else:
path_info = self.path_info
fs_path = self.fs_path
# NOTE: use utils.fs walk_files since fs.walk_files will not follow
# symlinks
if progress_callback:
for path in walk_files(path_info):
for path in walk_files(fs_path):
progress_callback()
yield path
else:
yield from walk_files(path_info)
yield from walk_files(fs_path)
def _remove_unpacked_dir(self, hash_):
info = self.hash_to_path_info(hash_)
path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
self.fs.remove(path_info)
hash_fs_path = self.hash_to_path(hash_)
fs_path = self.fs.path.with_name(
hash_fs_path,
self.fs.path.name(hash_fs_path) + self.UNPACKED_DIR_SUFFIX,
)
self.fs.remove(fs_path)
def _unprotect_file(self, path):
if self.fs.is_symlink(path) or self.fs.is_hardlink(path):
@@ -126,47 +129,47 @@ class LocalObjectDB(ObjectDB):
os.chmod(path, self._file_mode)
def _unprotect_dir(self, path):
for fname in self.fs.walk_files(path):
for fname in self.fs.find(path):
self._unprotect_file(fname)
def unprotect(self, path_info):
if not os.path.exists(path_info):
def unprotect(self, fs_path):
if not os.path.exists(fs_path):
from dvc.exceptions import DvcException
raise DvcException(
f"can't unprotect non-existing data '{path_info}'"
f"can't unprotect non-existing data '{fs_path}'"
)
if os.path.isdir(path_info):
self._unprotect_dir(path_info)
if os.path.isdir(fs_path):
self._unprotect_dir(fs_path)
else:
self._unprotect_file(path_info)
self._unprotect_file(fs_path)
def protect(self, path_info):
def protect(self, fs_path):
try:
os.chmod(path_info, self.CACHE_MODE)
os.chmod(fs_path, self.CACHE_MODE)
except OSError:
# NOTE: not being able to protect cache file is not fatal, it
# might happen on funky filesystems (e.g. Samba, see #5255),
# read-only filesystems or in a shared cache scenario.
logger.trace("failed to protect '%s'", path_info, exc_info=True)
logger.trace("failed to protect '%s'", fs_path, exc_info=True)
def is_protected(self, path_info):
def is_protected(self, fs_path):
try:
mode = os.stat(path_info).st_mode
mode = os.stat(fs_path).st_mode
except FileNotFoundError:
return False
return stat.S_IMODE(mode) == self.CACHE_MODE
def set_exec(self, path_info):
mode = os.stat(path_info).st_mode | stat.S_IEXEC
def set_exec(self, fs_path):
mode = os.stat(fs_path).st_mode | stat.S_IEXEC
try:
os.chmod(path_info, mode)
os.chmod(fs_path, mode)
except OSError:
logger.trace(
"failed to chmod '%s' '%s'",
oct(mode),
path_info,
fs_path,
exc_info=True,
)
+6 -6
View File
@@ -12,7 +12,7 @@ from .base import ObjectDB
if TYPE_CHECKING:
from dvc.fs.base import BaseFileSystem
from dvc.hash_info import HashInfo
from dvc.types import AnyPath, DvcPath
from dvc.types import AnyPath
logger = logging.getLogger(__name__)
@@ -36,9 +36,9 @@ class ReferenceObjectDB(ObjectDB):
return self._obj_cache[hash_info]
except KeyError:
pass
path_info = self.hash_to_path(hash_info.value)
fs_path = self.hash_to_path(hash_info.value)
try:
with self.fs.open(path_info, "rb") as fobj:
with self.fs.open(fs_path, "rb") as fobj:
ref_file = ReferenceHashFile.from_bytes(
fobj.read(), fs_cache=self._fs_cache
)
@@ -47,7 +47,7 @@ class ReferenceObjectDB(ObjectDB):
try:
ref_file.check(self, check_hash=False)
except ObjectFormatError:
self.fs.remove(path_info)
self.fs.remove(fs_path)
raise
self._obj_cache[hash_info] = ref_file
return ref_file
@@ -56,11 +56,11 @@ class ReferenceObjectDB(ObjectDB):
self,
from_fs: "BaseFileSystem",
from_info: "AnyPath",
to_info: "DvcPath",
to_info: "AnyPath",
hash_info: "HashInfo",
hardlink: bool = False,
):
self.makedirs(to_info.parent)
self.makedirs(self.fs.path.parent(to_info))
if hash_info.isdir:
return super()._add_file(
from_fs,
+8 -8
View File
@@ -18,12 +18,12 @@ logger = logging.getLogger(__name__)
class HashFile:
def __init__(
self,
path_info: Optional["AnyPath"],
fs_path: Optional["AnyPath"],
fs: Optional["BaseFileSystem"],
hash_info: "HashInfo",
name: Optional[str] = None,
):
self.path_info = path_info
self.fs_path = fs_path
self.fs = fs
self.hash_info = hash_info
self.name = name
@@ -41,7 +41,7 @@ class HashFile:
if not isinstance(other, HashFile):
return False
return (
self.path_info == other.path_info
self.fs_path == other.fs_path
and self.fs == other.fs
and self.hash_info == other.hash_info
)
@@ -50,7 +50,7 @@ class HashFile:
return hash(
(
self.hash_info,
self.path_info,
self.fs_path,
self.fs.scheme if self.fs else None,
)
)
@@ -58,9 +58,9 @@ class HashFile:
def check(self, odb: "ObjectDB", check_hash: bool = True):
if not check_hash:
assert self.fs
if not self.fs.exists(self.path_info):
if not self.fs.exists(self.fs_path):
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), self.path_info
errno.ENOENT, os.strerror(errno.ENOENT), self.fs_path
)
else:
return None
@@ -71,12 +71,12 @@ class HashFile:
from .stage import get_file_hash
_, actual = get_file_hash(
self.path_info, self.fs, self.hash_info.name, odb.state
self.fs_path, self.fs, self.hash_info.name, odb.state
)
logger.trace(
"cache '%s' expected '%s' actual '%s'",
self.path_info,
self.fs_path,
self.hash_info,
actual,
)
+14 -15
View File
@@ -25,23 +25,23 @@ class ReferenceHashFile(HashFile):
def __init__(
self,
path_info: "AnyPath",
fs_path: "AnyPath",
fs: "BaseFileSystem",
hash_info: "HashInfo",
checksum: Optional[str] = None,
**kwargs,
):
super().__init__(path_info, fs, hash_info, **kwargs)
self.checksum = checksum or fs.checksum(path_info)
super().__init__(fs_path, fs, hash_info, **kwargs)
self.checksum = checksum or fs.checksum(fs_path)
def __str__(self):
return f"ref object {self.hash_info} -> {self.path_info}"
return f"ref object {self.hash_info} -> {self.fs_path}"
def check(self, odb: "ObjectDB", check_hash: bool = True):
assert self.fs
if not self.fs.exists(self.path_info):
if not self.fs.exists(self.fs_path):
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), self.path_info
errno.ENOENT, os.strerror(errno.ENOENT), self.fs_path
)
if self.checksum != self._get_checksum():
raise ObjectFormatError(f"{self} is changed")
@@ -50,23 +50,22 @@ class ReferenceHashFile(HashFile):
def _get_checksum(self) -> str:
assert self.fs
return self.fs.checksum(self.path_info)
return self.fs.checksum(self.fs_path)
def to_bytes(self):
from dvc.fs.repo import RepoFileSystem
from dvc.path_info import PathInfo
# NOTE: dumping reference FS's this way is insecure, as the
# fully parsed remote FS config will include credentials
#
# ReferenceHashFiles should currently only be serialized in
# memory and not to disk
path_info = self.path_info
fs_path = self.fs_path
if isinstance(self.fs, RepoFileSystem):
path_info = PathInfo(path_info).relative_to(self.fs.root_dir)
fs_path = self.fs.path.relpath(fs_path, self.fs.root_dir)
dict_ = {
self.PARAM_PATH: path_info,
self.PARAM_PATH: fs_path,
self.PARAM_HASH: self.hash_info,
self.PARAM_CHECKSUM: self.checksum,
self.PARAM_FS_CONFIG: self.config_tuple(self.fs),
@@ -87,7 +86,7 @@ class ReferenceHashFile(HashFile):
raise ObjectFormatError("ReferenceHashFile is corrupted") from exc
try:
path_info = dict_[cls.PARAM_PATH]
fs_path = dict_[cls.PARAM_PATH]
hash_info = dict_[cls.PARAM_HASH]
except KeyError as exc:
raise ObjectFormatError("ReferenceHashFile is corrupted") from exc
@@ -98,12 +97,12 @@ class ReferenceHashFile(HashFile):
config = dict(config_pairs)
if RepoFileSystem.PARAM_REPO_URL in config:
fs = RepoFileSystem(**config)
path_info = fs.root_dir / path_info
fs_path = fs.path.join(fs.root_dir, fs_path)
else:
fs_cls = get_fs_cls(config, scheme=path_info.scheme)
fs_cls = get_fs_cls(config, scheme=scheme)
fs = fs_cls(**config)
return ReferenceHashFile(
path_info,
fs_path,
fs,
hash_info,
checksum=dict_.get(cls.PARAM_CHECKSUM),
+89 -64
View File
@@ -1,10 +1,9 @@
import hashlib
import logging
import os
import pathlib
from concurrent.futures import ThreadPoolExecutor
from functools import partial
from typing import TYPE_CHECKING, Dict, Tuple
from typing import TYPE_CHECKING, Dict, Optional, Tuple
from dvc.exceptions import DvcIgnoreInCollectedDirError
from dvc.hash_info import HashInfo
@@ -18,7 +17,7 @@ from .meta import Meta
if TYPE_CHECKING:
from dvc.fs.base import BaseFileSystem
from dvc.types import AnyPath, DvcPath
from dvc.types import AnyPath
from .db.base import ObjectDB
@@ -28,31 +27,47 @@ logger = logging.getLogger(__name__)
_STAGING_MEMFS_PATH = "dvc-staging"
def _upload_file(path_info, fs, odb, upload_odb):
def _upload_file(from_fs_path, fs, odb, upload_odb):
from dvc.utils import tmp_fname
from dvc.utils.stream import HashedStreamReader
tmp_info = upload_odb.path_info / tmp_fname()
with fs.open(path_info, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream:
fs_path = upload_odb.fs.path
tmp_info = fs_path.join(upload_odb.fs_path, tmp_fname())
with fs.open(from_fs_path, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream:
stream = HashedStreamReader(stream)
size = fs.getsize(path_info)
upload_odb.fs.upload(stream, tmp_info, desc=path_info.name, total=size)
size = fs.getsize(from_fs_path)
upload_odb.fs.upload(
stream, tmp_info, desc=fs_path.name(from_fs_path), total=size
)
odb.add(tmp_info, upload_odb.fs, stream.hash_info)
meta = Meta(size=size)
return path_info, meta, odb.get(stream.hash_info)
return from_fs_path, meta, odb.get(stream.hash_info)
def _get_file_hash(path_info, fs, name):
info = fs.info(path_info)
def _adapt_info(info, scheme):
if scheme == "s3" and "ETag" in info:
info["etag"] = info["ETag"].strip('"')
elif scheme == "gs" and "etag" in info:
import base64
info["etag"] = base64.b64decode(info["etag"]).hex()
elif scheme == "http" and ("ETag" in info or "Content-MD5" in info):
info["checksum"] = info.get("ETag") or info.get("Content-MD5")
return info
def _get_file_hash(fs_path, fs, name):
info = _adapt_info(fs.info(fs_path), fs.scheme)
if name in info:
assert not info[name].endswith(".dir")
hash_value = info[name]
elif hasattr(fs, name):
func = getattr(fs, name)
hash_value = func(path_info)
hash_value = func(fs_path)
elif name == "md5":
hash_value = file_md5(path_info, fs)
hash_value = file_md5(fs_path, fs)
else:
raise NotImplementedError
@@ -61,41 +76,41 @@ def _get_file_hash(path_info, fs, name):
return meta, hash_info
def get_file_hash(path_info, fs, name, state=None):
def get_file_hash(fs_path, fs, name, state=None):
if state:
meta, hash_info = state.get( # pylint: disable=assignment-from-none
path_info, fs
fs_path, fs
)
if hash_info:
return meta, hash_info
meta, hash_info = _get_file_hash(path_info, fs, name)
meta, hash_info = _get_file_hash(fs_path, fs, name)
if state:
assert ".dir" not in hash_info.value
state.save(path_info, fs, hash_info)
state.save(fs_path, fs, hash_info)
return meta, hash_info
def _stage_file(path_info, fs, name, odb=None, upload_odb=None, dry_run=False):
def _stage_file(fs_path, fs, name, odb=None, upload_odb=None, dry_run=False):
state = odb.state if odb else None
meta, hash_info = get_file_hash(path_info, fs, name, state=state)
meta, hash_info = get_file_hash(fs_path, fs, name, state=state)
if upload_odb and not dry_run:
assert odb and name == "md5"
return _upload_file(path_info, fs, odb, upload_odb)
return _upload_file(fs_path, fs, odb, upload_odb)
if dry_run:
obj = HashFile(path_info, fs, hash_info)
obj = HashFile(fs_path, fs, hash_info)
else:
odb.add(path_info, fs, hash_info, hardlink=False)
odb.add(fs_path, fs, hash_info, hardlink=False)
obj = odb.get(hash_info)
return path_info, meta, obj
return fs_path, meta, obj
def _build_objects(
path_info,
fs_path,
fs,
name,
dvcignore=None,
@@ -104,9 +119,9 @@ def _build_objects(
**kwargs,
):
if dvcignore:
walk_iterator = dvcignore.walk_files(fs, path_info)
walk_iterator = dvcignore.find(fs, fs_path)
else:
walk_iterator = fs.walk_files(path_info)
walk_iterator = fs.find(fs_path)
with Tqdm(
unit="md5",
desc="Computing file/dir hashes (only done once)",
@@ -126,18 +141,18 @@ def _build_objects(
yield from executor.map(worker, walk_iterator)
def _iter_objects(path_info, fs, name, **kwargs):
yield from _build_objects(path_info, fs, name, **kwargs)
def _iter_objects(fs_path, fs, name, **kwargs):
yield from _build_objects(fs_path, fs, name, **kwargs)
def _build_tree(path_info, fs, name, **kwargs):
def _build_tree(fs_path, fs, name, **kwargs):
from .tree import Tree
tree_meta = Meta(size=0, nfiles=0)
tree = Tree(None, None, None)
for file_info, meta, obj in _iter_objects(path_info, fs, name, **kwargs):
if DvcIgnore.DVCIGNORE_FILE == file_info.name:
raise DvcIgnoreInCollectedDirError(file_info.parent)
for file_fs_path, meta, obj in _iter_objects(fs_path, fs, name, **kwargs):
if DvcIgnore.DVCIGNORE_FILE == fs.path.name(file_fs_path):
raise DvcIgnoreInCollectedDirError(fs.path.parent(file_fs_path))
# NOTE: this is lossy transformation:
# "hey\there" -> "hey/there"
@@ -147,7 +162,11 @@ def _build_tree(path_info, fs, name, **kwargs):
#
# Yes, this is a BUG, as long as we permit "/" in
# filenames on Windows and "\" on Unix
tree.add(file_info.relative_to(path_info).parts, meta, obj.hash_info)
relpath = fs.path.relpath(file_fs_path, fs_path)
key = tuple(relpath.split(fs.path.sep))
assert key
tree.add(key, meta, obj.hash_info)
tree_meta.size += meta.size
tree_meta.nfiles += 1
@@ -155,7 +174,7 @@ def _build_tree(path_info, fs, name, **kwargs):
return tree_meta, tree
def _stage_tree(path_info, fs, fs_info, name, odb=None, **kwargs):
def _stage_tree(fs_path, fs, fs_info, name, odb=None, **kwargs):
from .tree import Tree
value = fs_info.get(name)
@@ -167,41 +186,46 @@ def _stage_tree(path_info, fs, fs_info, name, odb=None, **kwargs):
except FileNotFoundError:
pass
meta, tree = _build_tree(path_info, fs, name, odb=odb, **kwargs)
meta, tree = _build_tree(fs_path, fs, name, odb=odb, **kwargs)
state = odb.state if odb and odb.state else None
hash_info = None
if state:
_, hash_info = state.get( # pylint: disable=assignment-from-none
path_info, fs
fs_path, fs
)
tree.digest(hash_info=hash_info)
odb.add(tree.path_info, tree.fs, tree.hash_info, hardlink=False)
odb.add(tree.fs_path, tree.fs, tree.hash_info, hardlink=False)
raw = odb.get(tree.hash_info)
# cleanup unneeded memfs tmpfile and return tree based on the
# ODB fs/path
if odb.fs != tree.fs:
tree.fs.remove(tree.path_info)
tree.fs.remove(tree.fs_path)
tree.fs = raw.fs
tree.path_info = raw.path_info
tree.fs_path = raw.fs_path
return meta, tree
_url_cache: Dict[str, str] = {}
def _make_staging_url(path_info: "AnyPath"):
from dvc.path_info import CloudURLInfo
def _make_staging_url(
fs: "BaseFileSystem", odb: "ObjectDB", fs_path: Optional[str]
):
from dvc.scheme import Schemes
url = CloudURLInfo(f"{Schemes.MEMORY}://{_STAGING_MEMFS_PATH}")
if path_info:
if isinstance(path_info, (str, pathlib.PurePath)):
path = os.path.abspath(path_info)
else:
path = str(path_info)
if path not in _url_cache:
_url_cache[path] = hashlib.sha256(path.encode("utf-8")).hexdigest()
url /= _url_cache[path]
url = f"{Schemes.MEMORY}://{_STAGING_MEMFS_PATH}"
if fs_path is not None:
if odb.fs.scheme == Schemes.LOCAL:
fs_path = os.path.abspath(fs_path)
if fs_path not in _url_cache:
_url_cache[fs_path] = hashlib.sha256(
fs_path.encode("utf-8")
).hexdigest()
url = fs.path.join(url, _url_cache[fs_path])
return url
@@ -214,18 +238,18 @@ def _get_staging(odb: "ObjectDB") -> "ObjectDB":
from dvc.fs.memory import MemoryFileSystem
fs = MemoryFileSystem()
path_info = _make_staging_url(odb.path_info)
fs_path = _make_staging_url(fs, odb, odb.fs_path)
state = odb.state
return ReferenceObjectDB(fs, path_info, state=state)
return ReferenceObjectDB(fs, fs_path, state=state)
def _load_from_state(odb, staging, path_info, fs, name):
def _load_from_state(odb, staging, fs_path, fs, name):
from . import check, load
from .errors import ObjectFormatError
from .tree import Tree
state = odb.state
meta, hash_info = state.get(path_info, fs)
meta, hash_info = state.get(fs_path, fs)
if hash_info:
for odb_ in (odb, staging):
if odb_.exists(hash_info):
@@ -248,10 +272,10 @@ def _stage_external_tree_info(odb, tree, name):
# that would be universal for all clouds.
assert odb and name != "md5"
odb.add(tree.path_info, tree.fs, tree.hash_info)
odb.add(tree.fs_path, tree.fs, tree.hash_info)
raw = odb.get(tree.hash_info)
_, hash_info = get_file_hash(raw.path_info, raw.fs, name, state=odb.state)
tree.path_info = raw.path_info
_, hash_info = get_file_hash(raw.fs_path, raw.fs, name, state=odb.state)
tree.fs_path = raw.fs_path
tree.fs = raw.fs
tree.hash_info.name = hash_info.name
tree.hash_info.value = hash_info.value
@@ -262,7 +286,7 @@ def _stage_external_tree_info(odb, tree, name):
def stage(
odb: "ObjectDB",
path_info: "DvcPath",
fs_path: "AnyPath",
fs: "BaseFileSystem",
name: str,
upload: bool = False,
@@ -282,19 +306,20 @@ def stage(
ODB filesystem, and staged objects will reference the uploaded path rather
than the original source path.
"""
assert path_info and path_info.scheme == fs.scheme
assert fs_path
# assert scheme(fs_path) == fs.scheme
details = fs.info(path_info)
details = fs.info(fs_path)
staging = _get_staging(odb)
if odb:
try:
return _load_from_state(odb, staging, path_info, fs, name)
return _load_from_state(odb, staging, fs_path, fs, name)
except FileNotFoundError:
pass
if details["type"] == "directory":
meta, obj = _stage_tree(
path_info,
fs_path,
fs,
details,
name,
@@ -308,7 +333,7 @@ def stage(
obj = _stage_external_tree_info(odb, obj, name)
else:
_, meta, obj = _stage_file(
path_info,
fs_path,
fs,
name,
odb=staging,
@@ -317,6 +342,6 @@ def stage(
)
if odb and odb.state and obj.hash_info:
odb.state.save(path_info, fs, obj.hash_info)
odb.state.save(fs_path, fs, obj.hash_info)
return staging, meta, obj
+3 -5
View File
@@ -86,7 +86,7 @@ def status(
exists: objs that exist in odb
missing: objs that do not exist in ODB
"""
logger.debug("Preparing to collect status from '%s'", odb.path_info)
logger.debug("Preparing to collect status from '%s'", odb.fs_path)
if not name:
name = odb.fs.PARAM_CHECKSUM
@@ -116,7 +116,7 @@ def status(
hashes: Set[str] = set(hash_infos.keys())
exists: Set[str] = set()
logger.debug("Collecting status from '%s'", odb.path_info)
logger.debug("Collecting status from '%s'", odb.fs_path)
if index and hashes:
if dir_objs:
exists = hashes.intersection(
@@ -128,9 +128,7 @@ def status(
hashes.difference_update(exists)
if hashes:
exists.update(
odb.hashes_exist(hashes, name=str(odb.path_info), **kwargs)
)
exists.update(odb.hashes_exist(hashes, name=odb.fs_path, **kwargs))
return StatusResult(
{hash_infos[hash_] for hash_ in exists},
{hash_infos[hash_] for hash_ in (hashes - exists)},
+8 -8
View File
@@ -20,9 +20,9 @@ logger = logging.getLogger(__name__)
def _log_exceptions(func):
@wraps(func)
def wrapper(path_info, *args, **kwargs):
def wrapper(fs_path, *args, **kwargs):
try:
func(path_info, *args, **kwargs)
func(fs_path, *args, **kwargs)
return 0
except Exception as exc: # pylint: disable=broad-except
# NOTE: this means we ran out of file descriptors and there is no
@@ -31,7 +31,7 @@ def _log_exceptions(func):
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
raise
logger.exception("failed to transfer '%s'", path_info)
logger.exception("failed to transfer '%s'", fs_path)
return 1
return wrapper
@@ -87,8 +87,8 @@ def _do_transfer(
)
logger.error(
"failed to upload '%s' to '%s'",
src.get(dir_obj.hash_info).path_info,
dest.get(dir_obj.hash_info).path_info,
src.get(dir_obj.hash_info).fs_path,
dest.get(dir_obj.hash_info).fs_path,
)
total_fails += dir_fails + 1
elif entry_ids.intersection(missing_ids):
@@ -144,8 +144,8 @@ def transfer(
logger.debug(
"Preparing to transfer data from '%s' to '%s'",
src.path_info,
dest.path_info,
src.fs_path,
dest.fs_path,
)
if src == dest:
return 0
@@ -157,7 +157,7 @@ def transfer(
def func(hash_info: "HashInfo") -> None:
obj = src.get(hash_info)
return dest.add(
obj.path_info,
obj.fs_path,
obj.fs,
obj.hash_info,
verify=verify,
+10 -12
View File
@@ -36,19 +36,18 @@ class Tree(HashFile):
def digest(self, hash_info: Optional["HashInfo"] = None):
from dvc.fs.memory import MemoryFileSystem
from dvc.path_info import CloudURLInfo
from dvc.utils import tmp_fname
memfs = MemoryFileSystem()
path_info = CloudURLInfo("memory://{}".format(tmp_fname("")))
with memfs.open(path_info, "wb") as fobj:
fs_path = "memory://{}".format(tmp_fname(""))
with memfs.open(fs_path, "wb") as fobj:
fobj.write(self.as_bytes())
self.fs = memfs
self.path_info = path_info
self.fs_path = fs_path
if hash_info:
self.hash_info = hash_info
else:
_, self.hash_info = get_file_hash(path_info, memfs, "md5")
_, self.hash_info = get_file_hash(fs_path, memfs, "md5")
assert self.hash_info.value
self.hash_info.value += ".dir"
@@ -101,7 +100,7 @@ class Tree(HashFile):
obj = odb.get(hash_info)
try:
with obj.fs.open(obj.path_info, "r") as fobj:
with obj.fs.open(obj.fs_path, "r") as fobj:
raw = json.load(fobj)
except ValueError as exc:
raise ObjectFormatError(f"{obj} is corrupted") from exc
@@ -109,12 +108,12 @@ class Tree(HashFile):
if not isinstance(raw, list):
logger.error(
"dir cache file format error '%s' [skipping the file]",
obj.path_info,
obj.fs_path,
)
raise ObjectFormatError(f"{obj} is corrupted")
tree = cls.from_list(raw)
tree.path_info = obj.path_info
tree.fs_path = obj.fs_path
tree.fs = obj.fs
tree.hash_info = hash_info
@@ -125,11 +124,11 @@ class Tree(HashFile):
inside prefix.
The returned tree will contain the original tree's hash_info and
path_info.
fs_path.
Returns an empty tree if no object exists at the specified prefix.
"""
tree = Tree(self.path_info, self.fs, self.hash_info)
tree = Tree(self.fs_path, self.fs, self.hash_info)
try:
for key, (meta, oid) in self.trie.items(prefix):
tree.add(key, meta, oid)
@@ -160,8 +159,7 @@ class Tree(HashFile):
def du(odb, tree):
try:
return sum(
odb.fs.getsize(odb.hash_to_path_info(oid.value))
for _, _, oid in tree
odb.fs.getsize(odb.hash_to_path(oid.value)) for _, _, oid in tree
)
except FileNotFoundError:
return None
+62 -55
View File
@@ -292,7 +292,7 @@ class Output:
):
self.repo = stage.repo if stage else None
fs_cls, fs_config, path_info = get_cloud_fs(self.repo, url=path)
fs_cls, fs_config, fs_path = get_cloud_fs(self.repo, url=path)
self.fs = fs_cls(**fs_config)
if (
@@ -309,7 +309,6 @@ class Output:
# here is a list and comments:
#
# .def_path - path from definition in DVC file
# .path_info - PathInfo/URLInfo structured resolved path
# .fspath - local only, resolved
# .__str__ - for presentation purposes, def_path/relpath
#
@@ -326,32 +325,32 @@ class Output:
self.live = live
self.desc = desc
self.path_info = self._parse_path(self.fs, path_info)
self.fs_path = self._parse_path(self.fs, fs_path)
if self.use_cache and self.odb is None:
raise RemoteCacheRequiredError(self.path_info)
raise RemoteCacheRequiredError(self.fs.scheme, self.fs_path)
self.obj = None
self.isexec = False if self.IS_DEPENDENCY else isexec
self.remote = remote
def _parse_path(self, fs, path_info):
def _parse_path(self, fs, fs_path):
if fs.scheme != "local":
return path_info
return fs_path
parsed = urlparse(self.def_path)
if parsed.scheme != "remote":
# NOTE: we can path either from command line or .dvc file,
# so we should expect both posix and windows style paths.
# PathInfo accepts both, i.e. / works everywhere, \ only on win.
# paths accepts both, i.e. / works everywhere, \ only on win.
#
# FIXME: if we have Windows path containing / or posix one with \
# then we have #2059 bug and can't really handle that.
if self.stage and not path_info.is_absolute():
path_info = self.stage.wdir / path_info
if self.stage and not os.path.isabs(fs_path):
fs_path = fs.path.join(self.stage.wdir, fs_path)
abs_p = os.path.abspath(os.path.normpath(path_info))
return fs.PATH_CLS(abs_p)
abs_p = os.path.abspath(os.path.normpath(fs_path))
return abs_p
def __repr__(self):
return "{class_name}: '{def_path}'".format(
@@ -371,9 +370,9 @@ class Output:
cur_dir = os.getcwd()
if path_isin(cur_dir, self.repo.root_dir):
return relpath(self.path_info, cur_dir)
return relpath(self.fs_path, cur_dir)
return relpath(self.path_info, self.repo.root_dir)
return relpath(self.fs_path, self.repo.root_dir)
@property
def scheme(self):
@@ -391,7 +390,7 @@ class Output:
return False
return self.repo and path_isin(
os.path.realpath(self.path_info), self.repo.root_dir
os.path.realpath(self.fs_path), self.repo.root_dir
)
@property
@@ -407,7 +406,9 @@ class Output:
@property
def cache_path(self):
return self.odb.hash_to_path_info(self.hash_info.value).url
return self.odb.fs.unstrip_protocol(
self.odb.hash_to_path(self.hash_info.value)
)
def get_hash(self):
if self.use_cache:
@@ -418,7 +419,7 @@ class Output:
name = self.fs.PARAM_CHECKSUM
_, _, obj = ostage(
odb,
self.path_info,
self.fs_path,
self.fs,
name,
dvcignore=self.dvcignore,
@@ -438,10 +439,10 @@ class Output:
@property
def exists(self):
if self._is_path_dvcignore(self.path_info):
if self._is_path_dvcignore(self.fs_path):
return False
return self.fs.exists(self.path_info)
return self.fs.exists(self.fs_path)
def changed_checksum(self):
return self.hash_info != self.get_hash()
@@ -491,17 +492,17 @@ class Output:
@property
def is_empty(self):
return self.fs.is_empty(self.path_info)
return self.fs.is_empty(self.fs_path)
def isdir(self):
if self._is_path_dvcignore(self.path_info):
if self._is_path_dvcignore(self.fs_path):
return False
return self.fs.isdir(self.path_info)
return self.fs.isdir(self.fs_path)
def isfile(self):
if self._is_path_dvcignore(self.path_info):
if self._is_path_dvcignore(self.fs_path):
return False
return self.fs.isfile(self.path_info)
return self.fs.isfile(self.fs_path)
# pylint: disable=no-member
@@ -540,7 +541,7 @@ class Output:
if not self.use_cache:
_, self.meta, obj = ostage(
self.repo.odb.local,
self.path_info,
self.fs_path,
self.fs,
self.fs.PARAM_CHECKSUM,
dvcignore=self.dvcignore,
@@ -557,17 +558,17 @@ class Output:
_, self.meta, self.obj = ostage(
self.odb,
self.path_info,
self.fs_path,
self.fs,
self.odb.fs.PARAM_CHECKSUM,
dvcignore=self.dvcignore,
)
self.hash_info = self.obj.hash_info
self.isexec = self.isfile() and self.fs.isexec(self.path_info)
self.isexec = self.isfile() and self.fs.isexec(self.fs_path)
def set_exec(self):
if self.isfile() and self.isexec:
self.odb.set_exec(self.path_info)
self.odb.set_exec(self.fs_path)
def commit(self, filter_info=None):
if not self.exists:
@@ -579,14 +580,14 @@ class Output:
granular = (
self.is_dir_checksum
and filter_info
and filter_info != self.path_info
and filter_info != self.fs_path
)
if granular:
obj = self._commit_granular_dir(filter_info)
else:
staging, _, obj = ostage(
self.odb,
filter_info or self.path_info,
filter_info or self.fs_path,
self.fs,
self.odb.fs.PARAM_CHECKSUM,
dvcignore=self.dvcignore,
@@ -599,7 +600,7 @@ class Output:
hardlink=True,
)
checkout(
filter_info or self.path_info,
filter_info or self.fs_path,
self.fs,
obj,
self.odb,
@@ -610,10 +611,12 @@ class Output:
self.set_exec()
def _commit_granular_dir(self, filter_info):
prefix = filter_info.relative_to(self.path_info).parts
prefix = self.fs.path.parts(
self.fs.path.relpath(filter_info, self.fs_path)
)
staging, _, save_obj = ostage(
self.odb,
self.path_info,
self.fs_path,
self.fs,
self.odb.fs.PARAM_CHECKSUM,
dvcignore=self.dvcignore,
@@ -633,7 +636,9 @@ class Output:
ret = {**self.hash_info.to_dict(), **self.meta.to_dict()}
if self.is_in_repo:
path = self.path_info.relpath(self.stage.wdir).as_posix()
path = self.fs.path.as_posix(
relpath(self.fs_path, self.stage.wdir)
)
else:
path = self.def_path
@@ -684,22 +689,21 @@ class Output:
if not self.metric or self.plot:
return
path = os.fspath(self.path_info)
if not os.path.exists(path):
if not os.path.exists(self.fs_path):
return
name = "metrics" if self.metric else "plot"
if os.path.isdir(path):
if os.path.isdir(self.fs_path):
msg = "directory '%s' cannot be used as %s."
logger.debug(msg, str(self.path_info), name)
logger.debug(msg, str(self), name)
return
if not istextfile(path, self.fs):
if not istextfile(self.fs_path, self.fs):
msg = "binary file '{}' cannot be used as {}."
raise DvcException(msg.format(self.path_info, name))
raise DvcException(msg.format(self.fs_path, name))
def download(self, to, jobs=None):
self.fs.download(self.path_info, to.path_info, jobs=jobs)
self.fs.download(self.fs_path, to.fs_path, jobs=jobs)
def get_obj(self, filter_info=None, **kwargs):
if self.obj:
@@ -712,8 +716,9 @@ class Output:
else:
return None
if filter_info and filter_info != self.path_info:
prefix = filter_info.relative_to(self.path_info).parts
fs_path = self.fs.path
if filter_info and filter_info != self.fs_path:
prefix = fs_path.parts(fs_path.relpath(filter_info, self.fs_path))
obj = obj.get(self.odb, prefix)
return obj
@@ -731,12 +736,12 @@ class Output:
if not self.use_cache:
if progress_callback:
progress_callback(
str(self.path_info), self.get_files_number(filter_info)
self.fs_path, self.get_files_number(filter_info)
)
return None
obj = self.get_obj(filter_info=filter_info)
if not obj and (filter_info and filter_info != self.path_info):
if not obj and (filter_info and filter_info != self.fs_path):
# backward compatibility
return None
@@ -749,7 +754,7 @@ class Output:
try:
modified = checkout(
filter_info or self.path_info,
filter_info or self.fs_path,
self.fs,
obj,
self.odb,
@@ -767,7 +772,7 @@ class Output:
return added, False if added else modified
def remove(self, ignore_remove=False):
self.fs.remove(self.path_info)
self.fs.remove(self.fs_path)
if self.scheme != Schemes.LOCAL:
return
@@ -779,9 +784,9 @@ class Output:
if self.scheme == "local" and self.use_scm_ignore:
self.repo.scm.ignore_remove(self.fspath)
self.fs.move(self.path_info, out.path_info)
self.fs.move(self.fs_path, out.fs_path)
self.def_path = out.def_path
self.path_info = out.path_info
self.fs_path = out.fs_path
self.save()
self.commit()
@@ -836,7 +841,7 @@ class Output:
if not self.hash_info.isdir:
return 1
if not filter_info or filter_info == self.path_info:
if not filter_info or filter_info == self.fs_path:
return self.meta.nfiles or 0
obj = self.get_obj(filter_info=filter_info)
@@ -844,7 +849,7 @@ class Output:
def unprotect(self):
if self.exists:
self.odb.unprotect(self.path_info)
self.odb.unprotect(self.fs_path)
def get_dir_cache(self, **kwargs):
if not self.is_dir_checksum:
@@ -886,7 +891,7 @@ class Output:
"Cache for files inside will be lost. "
"Would you like to continue? Use '-f' to force."
)
if not force and not prompt.confirm(msg.format(self.path_info)):
if not force and not prompt.confirm(msg.format(self.fs_path)):
raise CollectCacheError(
"unable to fully collect used cache"
" without cache for directory '{}'".format(self)
@@ -894,8 +899,10 @@ class Output:
return None
obj = self.get_obj()
if filter_info and filter_info != self.path_info:
prefix = filter_info.relative_to(self.path_info).parts
if filter_info and filter_info != self.fs_path:
prefix = self.fs.path.parts(
self.fs.path.relpath(filter_info, self.fs_path)
)
obj = obj.filter(prefix)
return obj
@@ -1021,7 +1028,7 @@ class Output:
merged = merge(
self.odb, ancestor_info, self.hash_info, other.hash_info
)
self.odb.add(merged.path_info, merged.fs, merged.hash_info)
self.odb.add(merged.fs_path, merged.fs, merged.hash_info)
self.hash_info = merged.hash_info
self.meta = Meta(
@@ -1031,7 +1038,7 @@ class Output:
@property
def fspath(self):
return self.path_info.fspath
return self.fs_path
@property
def is_decorated(self) -> bool:
+4 -5
View File
@@ -17,7 +17,6 @@ from funcy import cached_property, collecting, first, isa, join, reraise
from dvc.exceptions import DvcException
from dvc.parsing.interpolate import ParseError
from dvc.path_info import PathInfo
from dvc.utils import relpath
from .context import (
@@ -134,10 +133,10 @@ def make_definition(
class DataResolver:
def __init__(self, repo: "Repo", wdir: PathInfo, d: dict):
def __init__(self, repo: "Repo", wdir: str, d: dict):
self.fs = fs = repo.fs
self.wdir = wdir
self.relpath = relpath(self.wdir / "dvc.yaml")
self.relpath = relpath(fs.path.join(self.wdir, "dvc.yaml"))
vars_ = d.get(VARS_KWD, [])
check_interpolations(vars_, VARS_KWD, self.relpath)
@@ -232,7 +231,7 @@ class EntryDefinition:
def _resolve_wdir(
self, context: Context, name: str, wdir: str = None
) -> PathInfo:
) -> str:
if not wdir:
return self.wdir
@@ -240,7 +239,7 @@ class EntryDefinition:
wdir = to_str(context.resolve_str(wdir))
except (ContextError, ParseError) as exc:
format_and_raise(exc, f"'{self.where}.{name}.wdir'", self.relpath)
return self.wdir / wdir
return self.resolver.fs.path.join(self.wdir, wdir)
def resolve(self, **kwargs):
try:
+6 -8
View File
@@ -19,7 +19,6 @@ from dvc.parsing.interpolate import (
recurse,
str_interpolate,
)
from dvc.path_info import PathInfo
from dvc.utils import relpath
logger = logging.getLogger(__name__)
@@ -356,7 +355,7 @@ class Context(CtxDict):
@classmethod
def load_from(
cls, fs, path: PathInfo, select_keys: List[str] = None
cls, fs, path: str, select_keys: List[str] = None
) -> "Context":
from dvc.utils.serialize import LOADERS
@@ -396,18 +395,17 @@ class Context(CtxDict):
raise ReservedKeyError(matches)
return super().merge_update(other, overwrite=overwrite)
def merge_from(self, fs, item: str, wdir: PathInfo, overwrite=False):
def merge_from(self, fs, item: str, wdir: str, overwrite=False):
path, _, keys_str = item.partition(":")
select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
abspath = os.path.abspath(wdir / path)
path_info = PathInfo(abspath)
abspath = os.path.abspath(fs.path.join(wdir, path))
if abspath in self.imports:
if not select_keys and self.imports[abspath] is None:
return # allow specifying complete filepath multiple times
self.check_loaded(abspath, item, select_keys)
ctx = Context.load_from(fs, path_info, select_keys)
ctx = Context.load_from(fs, abspath, select_keys)
try:
self.merge_update(ctx, overwrite=overwrite)
@@ -439,12 +437,12 @@ class Context(CtxDict):
self,
fs,
vars_: List,
wdir: PathInfo,
wdir: str,
stage_name: str = None,
default: str = None,
):
if default:
to_import = wdir / default
to_import = fs.path.join(wdir, default)
if fs.exists(to_import):
self.merge_from(fs, default, wdir)
else:
+10 -7
View File
@@ -11,7 +11,6 @@ from dvc.exceptions import FileMissingError
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
from dvc.ignore import DvcIgnoreFilter
from dvc.path_info import PathInfo
from dvc.utils import env2bool
from dvc.utils.fs import path_isin
@@ -354,7 +353,7 @@ class Repo:
)
def unprotect(self, target):
return self.odb.local.unprotect(PathInfo(target))
return self.odb.local.unprotect(target)
def _ignore(self):
flist = [self.config.files["local"], self.tmp_dir]
@@ -446,14 +445,18 @@ class Repo:
outs = outs or self.index.outs_graph
abs_path = os.path.abspath(path)
path_info = PathInfo(abs_path)
match = path_info.__eq__ if strict else path_info.isin_or_eq
fs_path = abs_path
def func(out):
if out.scheme == "local" and match(out.path_info):
def eq(one, two):
return one == two
match = eq if strict else out.fs.path.isin_or_eq
if out.scheme == "local" and match(fs_path, out.fs_path):
return True
if recursive and out.path_info.isin(path_info):
if recursive and out.fs.path.isin(out.fs_path, fs_path):
return True
return False
@@ -490,7 +493,7 @@ class Repo:
from dvc.fs.repo import RepoFileSystem
fs = RepoFileSystem(self, subrepos=True)
path = PathInfo(self.root_dir) / path
path = self.fs.path.join(self.root_dir, path)
try:
with fs.open(
path, mode=mode, encoding=encoding, remote=remote
+3 -3
View File
@@ -87,8 +87,8 @@ def translate_graph_error(stages: Stages) -> Iterator[None]:
yield
except OverlappingOutputPathsError as exc:
msg = OVERLAPPING_OUTPUT_FMT.format(
out=exc.overlapping_out.path_info,
parent=exc.parent.path_info,
out=exc.overlapping_out.fs_path,
parent=exc.parent.fs_path,
parent_stage=exc.parent.stage.addressing,
)
raise OverlappingOutputPathsError(exc.parent, exc.overlapping_out, msg)
@@ -228,7 +228,7 @@ def _find_all_targets(
from dvc.dvcfile import is_dvc_file
if os.path.isdir(target) and recursive:
files = map(os.fspath, repo.dvcignore.walk_files(repo.fs, target))
files = repo.dvcignore.find(repo.fs, target)
yield from (
path
for path in files
+27 -26
View File
@@ -2,8 +2,7 @@ import logging
import os
from typing import TYPE_CHECKING, Callable, Iterable, List, Tuple
from dvc.path_info import PathInfo
from dvc.types import DvcPath
from dvc.types import AnyPath
if TYPE_CHECKING:
from dvc.output import Output
@@ -14,7 +13,8 @@ logger = logging.getLogger(__name__)
FilterFn = Callable[["Output"], bool]
Outputs = List["Output"]
DvcPaths = List[DvcPath]
AnyPaths = List[AnyPath]
StrPaths = List[str]
def _collect_outs(
@@ -32,39 +32,40 @@ def _collect_paths(
rev: str = None,
):
from dvc.fs.repo import RepoFileSystem
from dvc.utils import relpath
path_infos = [PathInfo(os.path.abspath(target)) for target in targets]
fs_paths = [os.path.abspath(target) for target in targets]
fs = RepoFileSystem(repo)
target_infos = []
for path_info in path_infos:
target_paths = []
for fs_path in fs_paths:
if recursive and fs.isdir(path_info):
target_infos.extend(repo.dvcignore.walk_files(fs, path_info))
if recursive and fs.isdir(fs_path):
target_paths.extend(repo.dvcignore.find(fs, fs_path))
if not fs.exists(path_info):
if not fs.exists(fs_path):
rel = relpath(fs_path)
if rev == "workspace" or rev == "":
logger.warning(
"'%s' was not found in current workspace.", path_info
)
logger.warning("'%s' was not found in current workspace.", rel)
else:
logger.warning("'%s' was not found at: '%s'.", path_info, rev)
target_infos.append(path_info)
return target_infos
logger.warning("'%s' was not found at: '%s'.", rel, rev)
target_paths.append(fs_path)
return target_paths
def _filter_duplicates(
outs: Outputs, path_infos: DvcPaths
) -> Tuple[Outputs, DvcPaths]:
outs: Outputs, fs_paths: StrPaths
) -> Tuple[Outputs, StrPaths]:
res_outs: Outputs = []
res_infos = path_infos
fs_res_paths = fs_paths
for out in outs:
if out.path_info in path_infos:
if out.fs_path in fs_paths:
res_outs.append(out)
res_infos.remove(out.path_info)
# MUTATING THE SAME LIST!!
fs_res_paths.remove(out.fs_path)
return res_outs, res_infos
return res_outs, fs_res_paths
def collect(
@@ -74,15 +75,15 @@ def collect(
output_filter: FilterFn = None,
rev: str = None,
recursive: bool = False,
) -> Tuple[Outputs, DvcPaths]:
) -> Tuple[Outputs, StrPaths]:
assert targets or output_filter
outs: Outputs = _collect_outs(repo, output_filter=output_filter, deps=deps)
if not targets:
path_infos: DvcPaths = []
return outs, path_infos
fs_paths: StrPaths = []
return outs, fs_paths
target_infos = _collect_paths(repo, targets, recursive=recursive, rev=rev)
target_paths = _collect_paths(repo, targets, recursive=recursive, rev=rev)
return _filter_duplicates(outs, target_infos)
return _filter_duplicates(outs, target_paths)
+22 -16
View File
@@ -37,14 +37,14 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
# workspace paths/checksums if b_rev was None
continue
targets_path_infos = None
targets_paths = None
if targets is not None:
# convert targets to path_infos, and capture any missing targets
targets_path_infos, missing_targets[rev] = _targets_to_path_infos(
# convert targets to paths, and capture any missing targets
targets_paths, missing_targets[rev] = _targets_to_paths(
repo_fs, targets
)
results[rev] = _paths_checksums(self, targets_path_infos)
results[rev] = _paths_checksums(self, targets_paths)
if targets is not None:
# check for overlapping missing targets between a_rev and b_rev
@@ -133,13 +133,14 @@ def _output_paths(repo, targets):
for output in repo.index.outs:
if _exists(output):
yield_output = targets is None or any(
output.path_info.isin_or_eq(target) for target in targets
output.fs.path.isin_or_eq(output.fs_path, target)
for target in targets
)
if on_working_fs:
_, _, obj = ostage(
repo.odb.local,
output.path_info,
output.fs_path,
repo.odb.local.fs,
"md5",
dry_run=True,
@@ -158,19 +159,24 @@ def _output_paths(repo, targets):
if output.is_dir_checksum and (
yield_output
or any(target.isin(output.path_info) for target in targets)
or any(
output.fs.path.isin(target, output.fs_path)
for target in targets
)
):
yield from _dir_output_paths(output.path_info, obj, targets)
yield from _dir_output_paths(
output.fs, output.fs_path, obj, targets
)
def _dir_output_paths(path_info, obj, targets=None):
def _dir_output_paths(fs, fs_path, obj, targets=None):
for key, _, oid in obj:
fname = path_info.joinpath(*key)
fname = fs.path.join(fs_path, *key)
if targets is None or any(
fname.isin_or_eq(target) for target in targets
fs.path.isin_or_eq(fname, target) for target in targets
):
# pylint: disable=no-member
yield str(fname), oid.value
yield fs.path.join(fs.path.name(fs_path), *key), oid.value
def _filter_missing(repo_fs, paths):
@@ -185,17 +191,17 @@ def _filter_missing(repo_fs, paths):
pass
def _targets_to_path_infos(repo_fs, targets):
path_infos = []
def _targets_to_paths(repo_fs, targets):
paths = []
missing = []
for target in targets:
if repo_fs.exists(target):
path_infos.append(repo_fs.metadata(target).path_info)
paths.append(repo_fs.metadata(target).fs_path)
else:
missing.append(target)
return path_infos, missing
return paths, missing
def _calculate_renamed(new, old, added, deleted):
+4 -6
View File
@@ -13,7 +13,6 @@ from funcy import cached_property, first
from dvc.dependency.param import MissingParamsError
from dvc.env import DVCLIVE_RESUME
from dvc.exceptions import DvcException
from dvc.path_info import PathInfo
from dvc.stage.monitor import CheckpointKilledError
from dvc.utils import relpath
@@ -368,13 +367,12 @@ class Experiments:
logger.debug("Using experiment params '%s'", params)
for params_fname in params:
path = PathInfo(params_fname)
suffix = path.suffix.lower()
for path in params:
suffix = self.repo.fs.path.suffix(path).lower()
modify_data = MODIFIERS[suffix]
with modify_data(path, fs=self.repo.fs) as data:
try:
merge_params(data, params[params_fname], allow_new=False)
merge_params(data, params[path], allow_new=False)
except NewParamsFound as e:
msg = self._format_new_params_msg(e.new_params, path)
raise MissingParamsError(msg)
@@ -929,7 +927,7 @@ class Experiments:
from .executor.base import BaseExecutor, ExecutorInfo
result = {}
for pidfile in self.repo.fs.walk_files(
for pidfile in self.repo.fs.find(
os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR)
):
rev, _ = os.path.splitext(os.path.basename(pidfile))
+1 -2
View File
@@ -16,7 +16,6 @@ from typing import (
from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
from dvc.exceptions import DvcException
from dvc.path_info import PathInfo
from dvc.repo import Repo
from dvc.repo.experiments.base import (
EXEC_BASELINE,
@@ -101,7 +100,7 @@ class BaseExecutor(ABC):
self,
src: "Git",
dvc_dir: str,
root_dir: Optional[Union[str, PathInfo]] = None,
root_dir: Optional[str] = None,
branch: Optional[str] = None,
name: Optional[str] = None,
**kwargs,
+4 -4
View File
@@ -148,12 +148,12 @@ class SSHExecutor(BaseExecutor):
from dvc.repo import Repo
from dvc.repo.push import push
cache_url = posixpath.join(
self.abs_url,
self._dvc_dir,
cache_path = posixpath.join(
self._repo_abspath,
Repo.DVC_DIR,
ODBManager.CACHE_DIR,
)
with self.sshfs() as fs:
odb = get_odb(fs, fs.PATH_CLS(cache_url), **fs.config)
odb = get_odb(fs, cache_path, **fs.config)
push(dvc, revs=[rev], run_cache=run_cache, odb=odb)
+4 -4
View File
@@ -2,7 +2,6 @@ import logging
import os
from dvc.exceptions import DvcException
from dvc.path_info import PathInfo
from dvc.utils import resolve_output
from dvc.utils.fs import remove
@@ -50,8 +49,9 @@ def get(url, path, out=None, rev=None, jobs=None):
with external_repo(
url=url, rev=rev, cache_dir=tmp_dir, cache_types=cache_types
) as repo:
from_info = PathInfo(repo.root_dir) / path
to_info = PathInfo(out)
repo.repo_fs.download(from_info, to_info, jobs=jobs)
from_fs_path = os.path.abspath(os.path.join(repo.root_dir, path))
repo.repo_fs.download(
from_fs_path, os.path.abspath(out), jobs=jobs
)
finally:
remove(tmp_dir)
+7 -7
View File
@@ -1,5 +1,6 @@
from typing import TYPE_CHECKING, Iterator, List, Set
from dvc.fs.local import localfs
from dvc.utils.fs import path_isin
if TYPE_CHECKING:
@@ -94,7 +95,6 @@ def build_graph(stages, outs_trie=None):
from dvc.exceptions import StagePathAsOutputError
from ..path_info import PathInfo
from .trie import build_outs_trie
G = nx.DiGraph()
@@ -103,7 +103,7 @@ def build_graph(stages, outs_trie=None):
outs_trie = outs_trie or build_outs_trie(stages)
for stage in stages:
out = outs_trie.shortest_prefix(PathInfo(stage.path).parts).value
out = outs_trie.shortest_prefix(localfs.path.parts(stage.path)).value
if out:
raise StagePathAsOutputError(stage, str(out))
@@ -111,10 +111,10 @@ def build_graph(stages, outs_trie=None):
G.add_nodes_from(stages)
for stage in stages:
for dep in stage.deps:
if dep.path_info is None:
if dep.fs_path is None:
continue
dep_key = dep.path_info.parts
dep_key = dep.fs.path.parts(dep.fs_path)
overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
if outs_trie.has_subtrie(dep_key):
overlapping.extend(outs_trie.values(prefix=dep_key))
@@ -136,10 +136,10 @@ def build_outs_graph(graph, outs_trie):
G.add_nodes_from(outs_trie.values())
for stage in graph.nodes():
for dep in stage.deps:
if dep.path_info is None:
# RepoDependency don't have a path_info
if dep.fs_path is None:
# RepoDependency don't have a path
continue
dep_key = dep.path_info.parts
dep_key = dep.fs.path.parts(dep.fs_path)
overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
if outs_trie.has_subtrie(dep_key):
overlapping.extend(outs_trie.values(prefix=dep_key))
+4 -5
View File
@@ -7,21 +7,20 @@ logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from dvc.output import Output
from dvc.path_info import PathInfo
from dvc.repo import Repo
def create_summary(out):
assert out.live and out.live["html"]
metrics, plots = out.repo.live.show(str(out.path_info))
metrics, plots = out.repo.live.show(out.fs_path)
html_path = out.path_info.fspath + "_dvc_plots"
html_path = out.fs_path + "_dvc_plots"
render(out.repo, plots, metrics=metrics, path=html_path, refresh_seconds=5)
def summary_path_info(out: "Output") -> Optional["PathInfo"]:
def summary_fs_path(out: "Output") -> Optional[str]:
from dvc.output import Output
assert out.live
@@ -29,7 +28,7 @@ def summary_path_info(out: "Output") -> Optional["PathInfo"]:
if isinstance(out.live, dict):
has_summary = out.live.get(Output.PARAM_LIVE_SUMMARY, True)
if has_summary:
return out.path_info.with_suffix(".json")
return out.fs.path.with_suffix(out.fs_path, ".json")
return None
+11 -11
View File
@@ -1,7 +1,7 @@
import os
from itertools import chain
from dvc.exceptions import PathMissingError
from dvc.path_info import PathInfo
def ls(url, path=None, rev=None, recursive=None, dvc_only=False):
@@ -29,11 +29,11 @@ def ls(url, path=None, rev=None, recursive=None, dvc_only=False):
from . import Repo
with Repo.open(url, rev=rev, subrepos=True, uninitialized=True) as repo:
path_info = PathInfo(repo.root_dir)
fs_path = repo.root_dir
if path:
path_info /= path
fs_path = os.path.abspath(repo.fs.path.join(fs_path, path))
ret = _ls(repo.repo_fs, path_info, recursive, dvc_only)
ret = _ls(repo.repo_fs, fs_path, recursive, dvc_only)
if path and not ret:
raise PathMissingError(path, repo, dvc_only=dvc_only)
@@ -46,21 +46,21 @@ def ls(url, path=None, rev=None, recursive=None, dvc_only=False):
return ret_list
def _ls(fs, path_info, recursive=None, dvc_only=False):
def _ls(fs, fs_path, recursive=None, dvc_only=False):
def onerror(exc):
raise exc
infos = []
try:
for root, dirs, files in fs.walk(
path_info.fspath, onerror=onerror, dvcfiles=True
fs_path, onerror=onerror, dvcfiles=True
):
entries = chain(files, dirs) if not recursive else files
infos.extend(PathInfo(root) / entry for entry in entries)
infos.extend(fs.path.join(root, entry) for entry in entries)
if not recursive:
break
except NotADirectoryError:
infos.append(path_info)
infos.append(fs_path)
except FileNotFoundError:
return {}
@@ -69,9 +69,9 @@ def _ls(fs, path_info, recursive=None, dvc_only=False):
metadata = fs.metadata(info)
if metadata.output_exists or not dvc_only:
path = (
path_info.name
if path_info == info
else str(info.relative_to(path_info))
fs.path.name(fs_path)
if fs_path == info
else fs.path.relpath(info, fs_path)
)
ret[path] = {
"isout": metadata.is_output,
+13 -10
View File
@@ -1,11 +1,12 @@
import logging
import os
from typing import List
from dvc.fs.repo import RepoFileSystem
from dvc.output import Output
from dvc.repo import locked
from dvc.repo.collect import DvcPaths, collect
from dvc.repo.live import summary_path_info
from dvc.repo.collect import StrPaths, collect
from dvc.repo.live import summary_fs_path
from dvc.scm.base import SCMError
from dvc.utils import error_handler, errored_revisions, onerror_collect
from dvc.utils.serialize import load_yaml
@@ -17,27 +18,27 @@ def _is_metric(out: Output) -> bool:
return bool(out.metric) or bool(out.live)
def _to_path_infos(metrics: List[Output]) -> DvcPaths:
def _to_fs_paths(metrics: List[Output]) -> StrPaths:
result = []
for out in metrics:
if out.metric:
result.append(out.path_info)
result.append(out.fs_path)
elif out.live:
path_info = summary_path_info(out)
if path_info:
result.append(path_info)
fs_path = summary_fs_path(out)
if fs_path:
result.append(fs_path)
return result
def _collect_metrics(repo, targets, revision, recursive):
metrics, path_infos = collect(
metrics, fs_paths = collect(
repo,
targets=targets,
output_filter=_is_metric,
recursive=recursive,
rev=revision,
)
return _to_path_infos(metrics) + list(path_infos)
return _to_fs_paths(metrics) + list(fs_paths)
def _extract_metrics(metrics, path, rev):
@@ -80,7 +81,9 @@ def _read_metrics(repo, metrics, rev, onerror=None):
if not fs.isfile(metric):
continue
res[str(metric)] = _read_metric(metric, fs, rev, onerror=onerror)
res[fs.path.relpath(metric, os.getcwd())] = _read_metric(
metric, fs, rev, onerror=onerror
)
return res
+23 -24
View File
@@ -1,10 +1,10 @@
import logging
import os
from collections import defaultdict
from copy import copy
from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
from dvc.dependency.param import ParamsDependency
from dvc.path_info import PathInfo
from dvc.repo import locked
from dvc.repo.collect import collect
from dvc.scm.base import SCMError
@@ -16,7 +16,6 @@ from dvc.utils.serialize import LOADERS
if TYPE_CHECKING:
from dvc.output import Output
from dvc.repo import Repo
from dvc.types import DvcPath
logger = logging.getLogger(__name__)
@@ -27,56 +26,58 @@ def _is_params(dep: "Output"):
def _collect_configs(
repo: "Repo", rev, targets=None
) -> Tuple[List["Output"], List["DvcPath"]]:
) -> Tuple[List["Output"], List[str]]:
params, path_infos = collect(
params, fs_paths = collect(
repo,
targets=targets or [],
deps=True,
output_filter=_is_params,
rev=rev,
)
all_path_infos = path_infos + [p.path_info for p in params]
all_fs_paths = fs_paths + [p.fs_path for p in params]
if not targets:
default_params = (
PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE
default_params = os.path.join(
repo.root_dir, ParamsDependency.DEFAULT_PARAMS_FILE
)
if default_params not in all_path_infos and repo.fs.exists(
if default_params not in all_fs_paths and repo.fs.exists(
default_params
):
path_infos.append(default_params)
return params, path_infos
fs_paths.append(default_params)
return params, fs_paths
@error_handler
def _read_path_info(fs, path_info, **kwargs):
suffix = path_info.suffix.lower()
def _read_fs_path(fs, fs_path, **kwargs):
suffix = fs.path.suffix(fs_path).lower()
loader = LOADERS[suffix]
return loader(path_info, fs=fs)
return loader(fs_path, fs=fs)
def _read_params(
repo,
params,
params_path_infos,
params_fs_paths,
deps=False,
onerror: Optional[Callable] = None,
):
res: Dict[str, Dict] = defaultdict(dict)
path_infos = copy(params_path_infos)
fs_paths = copy(params_fs_paths)
if deps:
for param in params:
params_dict = error_handler(param.read_params_d)(onerror=onerror)
if params_dict:
res[str(param.path_info)] = params_dict
res[
repo.fs.path.relpath(param.fs_path, os.getcwd())
] = params_dict
else:
path_infos += [param.path_info for param in params]
fs_paths += [param.fs_path for param in params]
for path_info in path_infos:
from_path = _read_path_info(repo.fs, path_info, onerror=onerror)
for fs_path in fs_paths:
from_path = _read_fs_path(repo.fs, fs_path, onerror=onerror)
if from_path:
res[str(path_info)] = from_path
res[repo.fs.path.relpath(fs_path, os.getcwd())] = from_path
return res
@@ -131,13 +132,11 @@ def show(repo, revs=None, targets=None, deps=False, onerror: Callable = None):
def _gather_params(repo, rev, targets=None, deps=False, onerror=None):
param_outs, params_path_infos = _collect_configs(
repo, rev, targets=targets
)
param_outs, params_fs_paths = _collect_configs(repo, rev, targets=targets)
params = _read_params(
repo,
params=param_outs,
params_path_infos=params_path_infos,
params_fs_paths=params_fs_paths,
deps=deps,
onerror=onerror,
)
+8 -11
View File
@@ -20,7 +20,6 @@ from dvc.utils.serialize import LOADERS
if TYPE_CHECKING:
from dvc.output import Output
from dvc.repo import Repo
from dvc.types import DvcPath
logger = logging.getLogger(__name__)
@@ -89,16 +88,14 @@ class Plots:
fs = RepoFileSystem(self.repo)
plots = _collect_plots(self.repo, targets, revision, recursive)
res = {}
for path_info, rev_props in plots.items():
for fs_path, rev_props in plots.items():
if fs.isdir(path_info):
if fs.isdir(fs_path):
plot_files = []
for pi in fs.walk_files(path_info):
for pi in fs.find(fs_path):
plot_files.append((pi, relpath(pi, self.repo.root_dir)))
else:
plot_files = [
(path_info, relpath(path_info, self.repo.root_dir))
]
plot_files = [(fs_path, relpath(fs_path, self.repo.root_dir))]
props = props or {}
@@ -205,10 +202,10 @@ def _collect_plots(
targets: List[str] = None,
rev: str = None,
recursive: bool = False,
) -> Dict["DvcPath", Dict]:
) -> Dict["List[str]", Dict]:
from dvc.repo.collect import collect
plots, path_infos = collect(
plots, fs_paths = collect(
repo,
output_filter=_is_plot,
targets=targets,
@@ -216,8 +213,8 @@ def _collect_plots(
recursive=recursive,
)
result = {plot.path_info: _plot_props(plot) for plot in plots}
result.update({path_info: {} for path_info in path_infos})
result = {plot.fs_path: _plot_props(plot) for plot in plots}
result.update({fs_path: {} for fs_path in fs_paths})
return result
+5 -5
View File
@@ -64,16 +64,16 @@ def _track_stage(stage):
if (
not dep.use_scm_ignore
and dep.is_in_repo
and not stage.repo.repo_fs.isdvc(dep.path_info)
and not stage.repo.repo_fs.isdvc(dep.fs_path)
):
stage.repo.scm.track_file(relpath(dep.path_info))
stage.repo.scm.track_file(relpath(dep.fs_path))
for out in stage.outs:
if not out.use_scm_ignore and out.is_in_repo:
stage.repo.scm.track_file(relpath(out.path_info))
stage.repo.scm.track_file(relpath(out.fs_path))
if out.live:
from dvc.repo.live import summary_path_info
from dvc.repo.live import summary_fs_path
summary = summary_path_info(out)
summary = summary_fs_path(out)
if summary:
stage.repo.scm.track_file(relpath(summary))
stage.repo.scm.track_changed_files()
+3 -5
View File
@@ -20,7 +20,6 @@ from dvc.exceptions import (
NoOutputOrStageError,
OutputNotFoundError,
)
from dvc.path_info import PathInfo
from dvc.repo import lock_repo
from dvc.utils import parse_target, relpath
@@ -39,7 +38,7 @@ PIPELINE_FILE = "dvc.yaml"
class StageInfo(NamedTuple):
stage: "Stage"
filter_info: Optional[PathInfo] = None
filter_info: Optional[str] = None
StageList = List["Stage"]
@@ -393,7 +392,7 @@ class StageLoad:
target: if not provided, all of the stages without any filters are
returned.
If `target` is a path to a dvc-tracked output,
a (stage, output_path_info) is returned.
a (stage, output_path) is returned.
Otherwise, the details above for `target` in `collect()`
applies.
@@ -409,8 +408,7 @@ class StageLoad:
if not (recursive and self.fs.isdir(target)):
try:
(out,) = self.repo.find_outs_by_path(target, strict=False)
filter_info = PathInfo(os.path.abspath(target))
return [StageInfo(out.stage, filter_info)]
return [StageInfo(out.stage, os.path.abspath(target))]
except OutputNotFoundError:
pass
+1 -1
View File
@@ -9,7 +9,7 @@ def build_outs_trie(stages):
for stage in stages:
for out in stage.outs:
out_key = out.path_info.parts
out_key = out.fs.path.parts(out.fs_path)
# Check for dup outs
if out_key in outs:
+19 -22
View File
@@ -6,6 +6,7 @@ from contextlib import contextmanager
from voluptuous import Invalid, Optional, Required, Schema
from .exceptions import DvcException
from .fs.local import localfs
from .utils import relpath
INFO_SCHEMA = {Required("pid"): int, Required("cmd"): str}
@@ -59,11 +60,11 @@ def _infos_to_str(infos):
def _check_blockers(lock, info, *, mode, waiters):
from .lock import LockError
for path_info in waiters:
for waiter_path in waiters:
blockers = [
blocker
for path, infos in lock[mode].items()
if path_info.overlaps(path)
if localfs.path.overlaps(waiter_path, path)
for blocker in (infos if isinstance(infos, list) else [infos])
if blocker != info
]
@@ -77,38 +78,34 @@ def _check_blockers(lock, info, *, mode, waiters):
"\n"
"If there are no processes with such PIDs, you can manually "
"remove '.dvc/tmp/rwlock' and try again.".format(
path=str(path_info), blockers=_infos_to_str(blockers)
path=waiter_path, blockers=_infos_to_str(blockers)
)
)
def _acquire_read(lock, info, path_infos):
def _acquire_read(lock, info, paths):
changes = []
for path_info in path_infos:
url = path_info.url
readers = lock["read"][url]
for path in paths:
readers = lock["read"][path]
if info in readers:
continue
changes.append(url)
changes.append(path)
readers.append(info)
return changes
def _acquire_write(lock, info, path_infos):
def _acquire_write(lock, info, paths):
changes = []
for path_info in path_infos:
url = path_info.url
if lock["write"][url] == info:
for path in paths:
if lock["write"][path] == info:
continue
changes.append(url)
lock["write"][url] = info
changes.append(path)
lock["write"][path] = info
return changes
@@ -137,17 +134,17 @@ def _release_read(lock, info, changes):
@contextmanager
def rwlock(tmp_dir, cmd, read, write):
"""Create non-thread-safe RWLock for PathInfos.
"""Create non-thread-safe RWLock for file paths.
Args:
tmp_dir (str): existing directory where to create the rwlock file.
cmd (str): command that will be working on these PathInfos.
read ([PathInfo]): PathInfos that are going to be read.
write ([PathInfo]): PathInfos that are going to be written.
cmd (str): command that will be working on these file path.
read ([str]): file paths that are going to be read.
write ([str]): file paths that are going to be written.
Raises:
LockError: raised if PathInfo we want to read is being written to by
another command or if PathInfo we want to write is being written
LockError: raised if file paths we want to read is being written to by
another command or if file paths we want to write is being written
to or read from by another command.
RWLockFileCorruptedError: raised if rwlock file is not a valid JSON.
RWLockFileFormatError: raised if rwlock file is a valid JSON, but
+9 -9
View File
@@ -18,7 +18,6 @@ from typing import (
from funcy import cached_property
from dvc.path_info import PathInfo
from dvc.progress import Tqdm
from dvc.scm.base import GitAuthError, InvalidRemoteSCMRepo, SCMError
from dvc.utils import relpath
@@ -105,22 +104,22 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
except NotGitRepository as exc:
raise SCMError(f"{root_dir} is not a git repository") from exc
self._submodules: Dict[str, "PathInfo"] = self._find_submodules()
self._submodules: Dict[str, str] = self._find_submodules()
self._stashes: dict = {}
def _find_submodules(self) -> Dict[str, "PathInfo"]:
def _find_submodules(self) -> Dict[str, str]:
"""Return dict mapping submodule names to submodule paths.
Submodule paths will be relative to Git repo root.
"""
from dulwich.config import ConfigFile, parse_submodules
submodules: Dict[str, "PathInfo"] = {}
submodules: Dict[str, str] = {}
config_path = os.path.join(self.root_dir, ".gitmodules")
if os.path.isfile(config_path):
config = ConfigFile.from_path(config_path)
for path, _url, section in parse_submodules(config):
submodules[os.fsdecode(section)] = PathInfo(os.fsdecode(path))
submodules[os.fsdecode(section)] = os.fsdecode(path)
return submodules
def close(self):
@@ -163,11 +162,12 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
# parent git repo root). We append path to root_dir here so
# that the result of relpath(path, root_dir) is actually the
# path relative to the submodule root.
path_info = PathInfo(path).relative_to(self.root_dir)
fs_path = relpath(path, self.root_dir)
for sm_path in self._submodules.values():
if path_info.isin(sm_path):
if fs_path.startswith(sm_path):
path = os.path.join(
self.root_dir, path_info.relative_to(sm_path)
self.root_dir,
relpath(fs_path, sm_path),
)
break
if os.path.isdir(path):
@@ -241,7 +241,7 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
return untracked
def is_tracked(self, path: str) -> bool:
rel = PathInfo(path).relative_to(self.root_dir).as_posix().encode()
rel = relpath(path, self.root_dir).replace(os.path.sep, "/").encode()
rel_dir = rel + b"/"
for path in self.repo.open_index():
if path == rel or path.startswith(rel_dir):
+6 -6
View File
@@ -273,7 +273,7 @@ class Stage(params.StageParams):
from dvc.output import Output
from dvc.schema import LIVE_PROPS
env[DVCLIVE_PATH] = relpath(out.path_info, self.wdir)
env[DVCLIVE_PATH] = relpath(out.fs_path, self.wdir)
if isinstance(out.live, dict):
config = project(out.live, LIVE_PROPS)
@@ -511,7 +511,7 @@ class Stage(params.StageParams):
if not (allow_missing or out.checkpoint):
raise
except CacheLinkError:
link_failures.append(out.path_info)
link_failures.append(out.fs_path)
if link_failures:
raise CacheLinkError(link_failures)
@@ -559,11 +559,11 @@ class Stage(params.StageParams):
def _check_missing_outputs(self):
check_missing_outputs(self)
def filter_outs(self, path_info):
def filter_outs(self, fs_path):
def _func(o):
return path_info.isin_or_eq(o.path_info)
return o.fs.path.isin_or_eq(fs_path, o.fs_path)
return filter(_func, self.outs) if path_info else self.outs
return filter(_func, self.outs) if fs_path else self.outs
@rwlocked(write=["outs"])
def checkout(self, allow_missing=False, **kwargs):
@@ -585,7 +585,7 @@ class Stage(params.StageParams):
added, modified = result or (None, None)
if not (added or modified):
return None, []
return "modified" if modified else "added", [out.path_info]
return "modified" if modified else "added", [str(out)]
except CheckoutError as exc:
return "failed", exc.target_infos
+19 -16
View File
@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Optional
from funcy import cached_property, first
from dvc.exceptions import DvcException
from dvc.path_info import PathInfo
from dvc.utils import dict_sha256, relpath
if TYPE_CHECKING:
@@ -167,13 +166,14 @@ class StageCache:
# sanity check
COMPILED_LOCK_FILE_STAGE_SCHEMA(cache)
path = PathInfo(self._get_cache_path(cache_key, cache_value))
self.repo.odb.local.makedirs(path.parent)
tmp = tempfile.NamedTemporaryFile(delete=False, dir=path.parent).name
assert os.path.exists(path.parent)
assert os.path.isdir(path.parent)
path = self._get_cache_path(cache_key, cache_value)
parent = self.repo.odb.local.fs.path.parent(path)
self.repo.odb.local.makedirs(parent)
tmp = tempfile.NamedTemporaryFile(delete=False, dir=parent).name
assert os.path.exists(parent)
assert os.path.isdir(parent)
dump_yaml(tmp, cache)
self.repo.odb.local.move(PathInfo(tmp), path)
self.repo.odb.local.move(tmp, path)
def restore(self, stage, run_cache=True, pull=False):
from .serialize import to_single_stage_lockfile
@@ -214,21 +214,24 @@ class StageCache:
def _transfer(func, from_remote, to_remote):
ret = []
runs = from_remote.path_info / "runs"
runs = from_remote.fs.path.join(from_remote.fs_path, "runs")
if not from_remote.fs.exists(runs):
return []
for src in from_remote.fs.walk_files(runs):
rel = src.relative_to(from_remote.path_info)
dst = to_remote.path_info / rel
key = dst.parent
from_path = from_remote.fs.path
for src in from_remote.fs.find(runs):
rel = from_path.relpath(src, from_remote.fs_path)
dst = to_remote.fs.path.join(to_remote.fs_path, rel)
key = to_remote.fs.path.parent(dst)
# check if any build cache already exists for this key
if to_remote.fs.exists(key) and first(
to_remote.fs.walk_files(key)
):
# TODO: check if MaxKeys=1 or something like that applies
# or otherwise this will take a lot of time!
if to_remote.fs.exists(key) and first(to_remote.fs.find(key)):
continue
func(src, dst)
ret.append((src.parent.name, src.name))
ret.append(
(from_path.name(from_path.parent(src)), from_path.name(src))
)
return ret
+1 -1
View File
@@ -22,7 +22,7 @@ def rwlocked(call, read=None, write=None):
def _chain(names):
return [
item.path_info
item.fs_path
for attr in names
for item in getattr(stage, attr)
# There is no need to lock RepoDependency deps, as there is no
+1 -2
View File
@@ -10,7 +10,6 @@ from dvc.hash_info import HashInfo
from dvc.objects.meta import Meta
from dvc.parsing import FOREACH_KWD, JOIN, DataResolver, EntryNotFound
from dvc.parsing.versions import LOCKFILE_VERSION
from dvc.path_info import PathInfo
from . import PipelineStage, Stage, loads_from
from .exceptions import StageNameUnspecified, StageNotFound
@@ -36,7 +35,7 @@ class StageLoader(Mapping):
@cached_property
def resolver(self):
wdir = PathInfo(self.dvcfile.path).parent
wdir = self.repo.fs.path.parent(self.dvcfile.path)
return DataResolver(self.repo, wdir, self.data)
@cached_property
+1 -1
View File
@@ -88,7 +88,7 @@ class LiveTask(MonitorTask):
def __init__(self, stage: "Stage", out: "Output", proc: subprocess.Popen):
self.output_path = os.path.join(
os.getcwd(), out.path_info.fspath + "_dvc_plots", "index.html"
os.getcwd(), out.fs_path + "_dvc_plots", "index.html"
)
ui.write(f"Live summary will be created at:\n{self.output_path}")
super().__init__(stage, functools.partial(create_summary, out), proc)
+3 -3
View File
@@ -142,8 +142,8 @@ def check_no_externals(stage):
def check_circular_dependency(stage):
from dvc.exceptions import CircularDependencyError
circular_dependencies = {d.path_info for d in stage.deps} & {
o.path_info for o in stage.outs
circular_dependencies = {d.fs_path for d in stage.deps} & {
o.fs_path for o in stage.outs
}
if circular_dependencies:
@@ -155,7 +155,7 @@ def check_duplicated_arguments(stage):
from dvc.exceptions import ArgumentDuplicationError
path_counts = Counter(edge.path_info for edge in stage.deps + stage.outs)
path_counts = Counter(edge.fs_path for edge in stage.deps + stage.outs)
for path, occurrence in path_counts.items():
if occurrence > 1:
+19 -19
View File
@@ -18,15 +18,15 @@ class StateBase(ABC):
pass
@abstractmethod
def save(self, path_info, fs, hash_info):
def save(self, path, fs, hash_info):
pass
@abstractmethod
def get(self, path_info, fs):
def get(self, path, fs):
pass
@abstractmethod
def save_link(self, path_info, fs):
def save_link(self, path, fs):
pass
@@ -34,13 +34,13 @@ class StateNoop(StateBase):
def close(self):
pass
def save(self, path_info, fs, hash_info):
def save(self, path, fs, hash_info):
pass
def get(self, path_info, fs): # pylint: disable=unused-argument
def get(self, path, fs): # pylint: disable=unused-argument
return None, None
def save_link(self, path_info, fs):
def save_link(self, path, fs):
pass
@@ -68,19 +68,19 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
self.md5s.close()
self.links.close()
def save(self, path_info, fs, hash_info):
def save(self, path, fs, hash_info):
"""Save hash for the specified path info.
Args:
path_info (dict): path_info to save hash for.
path (str): path to save hash for.
hash_info (HashInfo): hash to save.
"""
if not isinstance(fs, LocalFileSystem):
return
mtime, size = get_mtime_and_size(path_info, fs, self.dvcignore)
inode = get_inode(path_info)
mtime, size = get_mtime_and_size(path, fs, self.dvcignore)
inode = get_inode(path)
logger.debug(
"state save (%s, %s, %s) %s",
@@ -92,12 +92,12 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
self.md5s[inode] = (mtime, str(size), hash_info.value)
def get(self, path_info, fs):
def get(self, path, fs):
"""Gets the hash for the specified path info. Hash will be
retrieved from the state database if available.
Args:
path_info (dict): path info to get the hash for.
path (str): path info to get the hash for.
Returns:
HashInfo or None: hash for the specified path info or None if it
@@ -109,11 +109,11 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
return None, None
try:
mtime, size = get_mtime_and_size(path_info, fs, self.dvcignore)
mtime, size = get_mtime_and_size(path, fs, self.dvcignore)
except FileNotFoundError:
return None, None
inode = get_inode(path_info)
inode = get_inode(path)
value = self.md5s.get(inode)
@@ -122,23 +122,23 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
return Meta(size=size), HashInfo("md5", value[2])
def save_link(self, path_info, fs):
def save_link(self, path, fs):
"""Adds the specified path to the list of links created by dvc. This
list is later used on `dvc checkout` to cleanup old links.
Args:
path_info (dict): path info to add to the list of links.
path (str): path info to add to the list of links.
"""
if not isinstance(fs, LocalFileSystem):
return
try:
mtime, _ = get_mtime_and_size(path_info, fs, self.dvcignore)
mtime, _ = get_mtime_and_size(path, fs, self.dvcignore)
except FileNotFoundError:
return
inode = get_inode(path_info)
relative_path = relpath(path_info, self.root_dir)
inode = get_inode(path)
relative_path = relpath(path, self.root_dir)
with self.links as ref:
ref[relative_path] = (inode, mtime)
+1 -4
View File
@@ -1,13 +1,10 @@
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from dvc.path_info import PathInfo, URLInfo
if TYPE_CHECKING:
from os import PathLike
StrPath = Union[str, "PathLike[str]"]
DvcPath = Union[PathInfo, URLInfo]
AnyPath = Union[str, DvcPath, StrPath]
AnyPath = Union[str, StrPath]
OptStr = Optional[str]
TargetType = Union[List[str], str]
+6 -4
View File
@@ -363,22 +363,24 @@ def resolve_output(inp, out):
def resolve_paths(repo, out, always_local=False):
from urllib.parse import urlparse
from dvc.fs.local import localfs
from ..dvcfile import DVC_FILE_SUFFIX
from ..exceptions import DvcException
from ..path_info import PathInfo
from ..system import System
from .fs import contains_symlink_up_to
abspath = PathInfo(os.path.abspath(out))
abspath = os.path.abspath(out)
dirname = os.path.dirname(abspath)
base = os.path.basename(os.path.normpath(out))
scheme = urlparse(out).scheme
if os.name == "nt" and scheme == abspath.drive[0].lower():
if os.name == "nt" and scheme == localfs.path.drive(abspath)[0].lower():
# urlparse interprets windows drive letters as URL scheme
scheme = ""
if scheme or not abspath.isin_or_eq(repo.root_dir):
if scheme or not localfs.path.isin_or_eq(abspath, repo.root_dir):
wdir = os.getcwd()
elif contains_symlink_up_to(dirname, repo.root_dir) or (
os.path.isdir(abspath) and System.is_symlink(abspath)
+4 -4
View File
@@ -42,9 +42,9 @@ def get_mtime_and_size(path, fs, dvcignore=None):
size = 0
files_mtimes = {}
if dvcignore:
walk_iterator = dvcignore.walk_files(fs, path)
walk_iterator = dvcignore.find(fs, path)
else:
walk_iterator = fs.walk_files(path)
walk_iterator = fs.find(path)
for file_path in walk_iterator:
try:
stats = fs.info(file_path)
@@ -54,7 +54,7 @@ def get_mtime_and_size(path, fs, dvcignore=None):
raise
continue
size += stats["size"]
files_mtimes[os.fspath(file_path)] = stats["mtime"]
files_mtimes[file_path] = stats["mtime"]
# We track file changes and moves, which cannot be detected with simply
# max(mtime(f) for f in non_ignored_files)
@@ -244,7 +244,7 @@ def walk_files(directory):
def as_atomic(fs, to_info):
from dvc.utils import tmp_fname
tmp_info = to_info.parent / tmp_fname()
tmp_info = fs.path.join(fs.path.parent(to_info), tmp_fname())
try:
yield tmp_info
except BaseException:
+6 -2
View File
@@ -81,6 +81,10 @@ disable_other_loggers()
class TmpDir(pathlib.Path):
scheme = "local"
@property
def fs_path(self):
return str(self)
def __new__(cls, *args, **kwargs):
if cls is TmpDir:
cls = ( # pylint: disable=self-cls-assignment
@@ -249,8 +253,8 @@ class TmpDir(pathlib.Path):
}
return super().read_text(*args, encoding="utf-8", **kwargs)
def hash_to_path_info(self, hash_):
return self / hash_[0:2] / hash_[2:]
def hash_to_path(self, hash_):
return str(self / hash_[0:2] / hash_[2:])
def dump(self, *args, **kwargs):
return serialize.DUMPERS[self.suffix](self, *args, **kwargs)
+1 -7
View File
@@ -1,5 +1,3 @@
import os
import pytest
@@ -27,11 +25,7 @@ def run_copy_metrics(tmp_dir, run_copy):
if hasattr(tmp_dir.dvc, "scm"):
files = [stage.path]
files += [
os.fspath(out.path_info)
for out in stage.outs
if not out.use_cache
]
files += [out.fs_path for out in stage.outs if not out.use_cache]
tmp_dir.dvc.scm.add(files)
if commit:
tmp_dir.dvc.scm.commit(commit)
+2 -3
View File
@@ -5,7 +5,6 @@ from functools import partial
import pytest
from dvc.fs.ssh import SSHFileSystem
from dvc.path_info import URLInfo
from dvc.repo.experiments.base import EXEC_HEAD, EXEC_MERGE
from dvc.repo.experiments.executor.ssh import SSHExecutor
from tests.func.machine.conftest import * # noqa, pylint: disable=wildcard-import
@@ -39,7 +38,7 @@ def test_init_git(tmp_dir, scm, cloud):
rev = scm.resolve_rev("stash@{0}")
scm.set_ref(EXEC_MERGE, rev)
root_url = URLInfo(str(cloud)) / SSHExecutor.gen_dirname()
root_url = cloud / SSHExecutor.gen_dirname()
executor = SSHExecutor(
scm,
@@ -65,7 +64,7 @@ def test_init_cache(tmp_dir, dvc, scm, cloud):
rev = scm.get_rev()
scm.set_ref(EXEC_HEAD, rev)
scm.set_ref(EXEC_MERGE, rev)
root_url = URLInfo(str(cloud)) / SSHExecutor.gen_dirname()
root_url = cloud / SSHExecutor.gen_dirname()
executor = SSHExecutor(
scm,
+10 -10
View File
@@ -32,9 +32,9 @@ def test_new_checkpoint(
if rev == "workspace":
continue
fs = dvc.repo_fs
with fs.open(tmp_dir / "foo") as fobj:
with fs.open((tmp_dir / "foo").fs_path) as fobj:
assert fobj.read().strip() == str(checkpoint_stage.iterations)
with fs.open(tmp_dir / "metrics.yaml") as fobj:
with fs.open((tmp_dir / "metrics.yaml").fs_path) as fobj:
assert fobj.read().strip() == "foo: 2"
if workspace:
@@ -81,9 +81,9 @@ def test_resume_checkpoint(
if rev == "workspace":
continue
fs = dvc.repo_fs
with fs.open(tmp_dir / "foo") as fobj:
with fs.open((tmp_dir / "foo").fs_path) as fobj:
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
with fs.open(tmp_dir / "metrics.yaml") as fobj:
with fs.open((tmp_dir / "metrics.yaml").fs_path) as fobj:
assert fobj.read().strip() == "foo: 2"
if workspace:
@@ -112,9 +112,9 @@ def test_reset_checkpoint(
if rev == "workspace":
continue
fs = dvc.repo_fs
with fs.open(tmp_dir / "foo") as fobj:
with fs.open((tmp_dir / "foo").fs_path) as fobj:
assert fobj.read().strip() == str(checkpoint_stage.iterations)
with fs.open(tmp_dir / "metrics.yaml") as fobj:
with fs.open((tmp_dir / "metrics.yaml").fs_path) as fobj:
assert fobj.read().strip() == "foo: 2"
if workspace:
@@ -151,18 +151,18 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage, workspace):
if rev == "workspace":
continue
fs = dvc.repo_fs
with fs.open(tmp_dir / "foo") as fobj:
with fs.open((tmp_dir / "foo").fs_path) as fobj:
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
with fs.open(tmp_dir / "metrics.yaml") as fobj:
with fs.open((tmp_dir / "metrics.yaml").fs_path) as fobj:
assert fobj.read().strip() == "foo: 2"
for rev in dvc.brancher([checkpoint_b]):
if rev == "workspace":
continue
fs = dvc.repo_fs
with fs.open(tmp_dir / "foo") as fobj:
with fs.open((tmp_dir / "foo").fs_path) as fobj:
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
with fs.open(tmp_dir / "metrics.yaml") as fobj:
with fs.open((tmp_dir / "metrics.yaml").fs_path) as fobj:
assert fobj.read().strip() == "foo: 100"
with pytest.raises(MultipleBranchError):
+2 -2
View File
@@ -670,9 +670,9 @@ def test_modified_data_dep(tmp_dir, scm, dvc, workspace, params, target):
for rev in dvc.brancher(revs=[exp]):
if rev != exp:
continue
with dvc.repo_fs.open(tmp_dir / "metrics.yaml") as fobj:
with dvc.repo_fs.open((tmp_dir / "metrics.yaml").fs_path) as fobj:
assert fobj.read().strip() == params
with dvc.repo_fs.open(tmp_dir / "data") as fobj:
with dvc.repo_fs.open((tmp_dir / "data").fs_path) as fobj:
assert fobj.read().strip() == "modified"
if workspace:
+2 -2
View File
@@ -61,8 +61,8 @@ def test_clear_on_download_err(tmp_dir, dvc, index, mocker):
dvc.push()
for _, _, oid in out.obj:
remove(dvc.odb.local.get(oid).path_info)
remove(out.path_info)
remove(dvc.odb.local.get(oid).fs_path)
remove(out.fs_path)
assert list(index.hashes())
+5 -2
View File
@@ -39,7 +39,10 @@ USED_VARS = {
def make_entry_definition(wdir, name, data, context=None) -> EntryDefinition:
return EntryDefinition(
DataResolver(wdir.dvc, wdir, {}), context or Context(), name, data
DataResolver(wdir.dvc, wdir.fs_path, {}),
context or Context(),
name,
data,
)
@@ -47,7 +50,7 @@ def make_foreach_def(
wdir, name, foreach_data, do_data=None, context=None
) -> ForeachDefinition:
return ForeachDefinition(
DataResolver(wdir.dvc, wdir, {}),
DataResolver(wdir.dvc, wdir.fs_path, {}),
context or Context(),
name,
{"foreach": foreach_data, "do": do_data or {}},
+21 -17
View File
@@ -9,7 +9,7 @@ from dvc.parsing.context import Context
def test_with_simple_list_data(tmp_dir, dvc):
"""Testing a simple non-nested list as a foreach data"""
resolver = DataResolver(dvc, tmp_dir, {})
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
context = Context()
data = {"foreach": ["foo", "bar", "baz"], "do": {"cmd": "echo ${item}"}}
@@ -24,7 +24,7 @@ def test_with_simple_list_data(tmp_dir, dvc):
def test_with_dict_data(tmp_dir, dvc):
resolver = DataResolver(dvc, tmp_dir, {})
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
context = Context()
foreach_data = {"model1": "foo", "model2": "bar"}
@@ -45,7 +45,7 @@ def test_with_dict_data(tmp_dir, dvc):
def test_with_composite_list(tmp_dir, dvc):
resolver = DataResolver(dvc, tmp_dir, {})
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
context = Context()
foreach_data = [{"thresh": "foo"}, {"thresh": "bar"}]
@@ -64,7 +64,7 @@ def test_with_composite_list(tmp_dir, dvc):
def test_foreach_interpolated_simple_list(tmp_dir, dvc):
foreach_data = ["foo", "bar", "baz"]
vars_ = {"models": foreach_data}
resolver = DataResolver(dvc, tmp_dir, {"vars": [vars_]})
resolver = DataResolver(dvc, tmp_dir.fs_path, {"vars": [vars_]})
data = {"foreach": "${models}", "do": {"cmd": "echo ${item}"}}
definition = ForeachDefinition(resolver, resolver.context, "build", data)
@@ -98,7 +98,7 @@ def test_foreach_interpolate_with_composite_data(
tmp_dir, dvc, foreach_def, foreach_data, result
):
vars_ = [{"models": foreach_data}]
resolver = DataResolver(dvc, tmp_dir, {"vars": vars_})
resolver = DataResolver(dvc, tmp_dir.fs_path, {"vars": vars_})
data = {"foreach": "${models}", "do": {"cmd": f"echo {foreach_def}"}}
definition = ForeachDefinition(resolver, resolver.context, "build", data)
@@ -112,7 +112,7 @@ def test_params_file_with_dict_tracked(tmp_dir, dvc):
params = {"models": foreach_data}
(tmp_dir / "params.yaml").dump(params)
resolver = DataResolver(dvc, tmp_dir, {})
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
data = {"foreach": "${models}", "do": {"cmd": "echo ${item.thresh}"}}
definition = ForeachDefinition(resolver, resolver.context, "build", data)
@@ -133,7 +133,7 @@ def test_params_file_tracked_for_composite_list(tmp_dir, dvc):
params = {"models": foreach_data}
(tmp_dir / "params.yaml").dump(params)
resolver = DataResolver(dvc, tmp_dir, {})
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
data = {"foreach": "${models}", "do": {"cmd": "echo ${item.thresh}"}}
definition = ForeachDefinition(resolver, resolver.context, "build", data)
@@ -150,7 +150,7 @@ def test_params_file_tracked_for_composite_list(tmp_dir, dvc):
def test_foreach_data_from_nested_vars(tmp_dir, dvc):
vars_ = {"models": {"lst": [{"thresh": 10}, {"thresh": 15}]}}
resolver = DataResolver(dvc, tmp_dir, {"vars": [vars_]})
resolver = DataResolver(dvc, tmp_dir.fs_path, {"vars": [vars_]})
data = {"foreach": "${models.lst}", "do": {"cmd": "echo ${item.thresh}"}}
definition = ForeachDefinition(resolver, resolver.context, "build", data)
@@ -163,7 +163,7 @@ def test_foreach_data_from_nested_vars(tmp_dir, dvc):
def test_foreach_partial_interpolations(tmp_dir, dvc):
resolver = DataResolver(dvc, tmp_dir, {"vars": [{"bar": "bar"}]})
resolver = DataResolver(dvc, tmp_dir.fs_path, {"vars": [{"bar": "bar"}]})
foreach_data = {"model1": "foo", "model2": "${bar}"}
data = {"foreach": foreach_data, "do": {"cmd": "echo ${item}"}}
definition = ForeachDefinition(resolver, resolver.context, "build", data)
@@ -180,7 +180,9 @@ def test_mixed_vars_for_foreach_data(tmp_dir, dvc):
(tmp_dir / "params.yaml").dump({"models": {"model1": "foo"}})
(tmp_dir / "test_params.yaml").dump({"models": {"model2": "bar"}})
resolver = DataResolver(dvc, tmp_dir, {"vars": ["test_params.yaml"]})
resolver = DataResolver(
dvc, tmp_dir.fs_path, {"vars": ["test_params.yaml"]}
)
data = {"foreach": "${models}", "do": {"cmd": "echo ${item}"}}
definition = ForeachDefinition(resolver, resolver.context, "build", data)
@@ -203,7 +205,9 @@ def test_mixed_vars_for_foreach_data_2(tmp_dir, dvc):
{"models": {"model1": {"epochs": 5}, "model2": {"epochs": 10}}},
)
resolver = DataResolver(dvc, tmp_dir, {"vars": ["test_params.yaml"]})
resolver = DataResolver(
dvc, tmp_dir.fs_path, {"vars": ["test_params.yaml"]}
)
data = {
"foreach": "${models}",
"do": {"cmd": "echo ${item.thresh} ${item.epochs}"},
@@ -233,7 +237,7 @@ def test_mixed_vars_for_foreach_data_2(tmp_dir, dvc):
def test_foreach_with_interpolated_wdir(tmp_dir, dvc):
resolver = DataResolver(dvc, tmp_dir / "data", {})
resolver = DataResolver(dvc, (tmp_dir / "data").fs_path, {})
foreach_data = ["foo", "bar"]
data = {
"foreach": foreach_data,
@@ -254,7 +258,7 @@ def test_foreach_with_interpolated_wdir(tmp_dir, dvc):
def test_foreach_with_local_vars(tmp_dir, dvc):
resolver = DataResolver(dvc, tmp_dir / "data", {})
resolver = DataResolver(dvc, (tmp_dir / "data").fs_path, {})
foreach_data = ["foo", "bar"]
data = {
"foreach": foreach_data,
@@ -289,7 +293,7 @@ def test_foreach_with_imported_vars(tmp_dir, dvc, local_import):
(tmp_dir / "test_params.yaml").dump(
{"train": {"epochs": 10}, "prepare": {"nums": 25}}
)
resolver = DataResolver(dvc, tmp_dir, {})
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
foreach_data = ["foo", "bar"]
data = {
"foreach": foreach_data,
@@ -327,7 +331,7 @@ def test_foreach_with_interpolated_wdir_and_local_vars(
{"train": {"epochs": 1 + i}, "prepare": {"nums": 10 * i}},
)
resolver = DataResolver(dvc, tmp_dir, {})
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
data = {
"foreach": [0, 1, 2, 3, 4],
"do": {
@@ -391,7 +395,7 @@ def test_foreach_do_syntax_is_checked_once(tmp_dir, dvc, mocker):
do_def = {"cmd": "python script.py --epochs ${item}"}
data = {"foreach": [0, 1, 2, 3, 4], "do": do_def}
definition = ForeachDefinition(
DataResolver(dvc, tmp_dir, {}), Context(), "build", data
DataResolver(dvc, tmp_dir.fs_path, {}), Context(), "build", data
)
mock = mocker.patch("dvc.parsing.check_syntax_errors", return_value=True)
definition.resolve_all()
@@ -403,7 +407,7 @@ def test_foreach_data_is_only_resolved_once(tmp_dir, dvc, mocker):
context = Context(models=["foo", "bar", "baz"])
data = {"foreach": "${models}", "do": {}}
definition = ForeachDefinition(
DataResolver(dvc, tmp_dir, {}), context, "build", data
DataResolver(dvc, tmp_dir.fs_path, {}), context, "build", data
)
mock = mocker.spy(definition, "_resolve_foreach_data")
+12 -12
View File
@@ -1,3 +1,4 @@
import os
from copy import deepcopy
import pytest
@@ -5,7 +6,6 @@ import pytest
from dvc.dependency import _merge_params
from dvc.parsing import DEFAULT_PARAMS_FILE, DataResolver
from dvc.parsing.context import recurse_not_a_node
from dvc.path_info import PathInfo
from . import (
CONTEXT_DATA,
@@ -31,7 +31,7 @@ def assert_stage_equal(d1, d2):
def test_simple(tmp_dir, dvc):
(tmp_dir / DEFAULT_PARAMS_FILE).dump(CONTEXT_DATA)
resolver = DataResolver(
dvc, PathInfo(str(tmp_dir)), deepcopy(TEMPLATED_DVC_YAML_DATA)
dvc, tmp_dir.fs_path, deepcopy(TEMPLATED_DVC_YAML_DATA)
)
assert_stage_equal(resolver.resolve(), deepcopy(RESOLVED_DVC_YAML_DATA))
assert resolver.tracked_vars == {
@@ -48,7 +48,7 @@ def test_vars_import(tmp_dir, dvc):
(tmp_dir / "params2.yaml").dump(CONTEXT_DATA)
d = deepcopy(TEMPLATED_DVC_YAML_DATA)
d["vars"] = ["params2.yaml"]
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
resolved_data = deepcopy(RESOLVED_DVC_YAML_DATA)
assert_stage_equal(resolver.resolve(), resolved_data)
@@ -68,7 +68,7 @@ def test_vars_and_params_import(tmp_dir, dvc):
"stages": {"stage1": {"cmd": "echo ${dict.foo} ${dict.bar}"}},
}
(tmp_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"bar": "bar"}})
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
assert_stage_equal(
resolver.resolve(), {"stages": {"stage1": {"cmd": "echo foobar bar"}}}
@@ -97,7 +97,7 @@ def test_stage_with_wdir(tmp_dir, dvc):
data_dir.mkdir()
(tmp_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"bar": "bar"}})
(data_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"foo": "foo"}})
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
assert_stage_equal(
resolver.resolve(),
@@ -113,7 +113,7 @@ def test_stage_with_wdir(tmp_dir, dvc):
)
assert resolver.tracked_vars == {
"stage1": {
str(PathInfo("data") / DEFAULT_PARAMS_FILE): {"dict.foo": "foo"},
os.path.join("data", DEFAULT_PARAMS_FILE): {"dict.foo": "foo"},
DEFAULT_PARAMS_FILE: {"dict.bar": "bar"},
}
}
@@ -140,7 +140,7 @@ def test_with_templated_wdir(tmp_dir, dvc):
data_dir = tmp_dir / "data"
data_dir.mkdir()
(data_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"foo": "foo"}})
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
assert_stage_equal(
resolver.resolve(),
@@ -156,7 +156,7 @@ def test_with_templated_wdir(tmp_dir, dvc):
)
assert resolver.tracked_vars == {
"stage1": {
str(PathInfo("data") / DEFAULT_PARAMS_FILE): {"dict.foo": "foo"},
os.path.join("data", DEFAULT_PARAMS_FILE): {"dict.foo": "foo"},
DEFAULT_PARAMS_FILE: {"dict.bar": "bar", "dict.ws": "data"},
}
}
@@ -178,7 +178,7 @@ def test_resolve_local_tries_to_load_globally_used_files(tmp_dir, dvc):
}
},
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
assert_stage_equal(
resolver.resolve(),
{
@@ -206,7 +206,7 @@ def test_resolve_local_tries_to_load_globally_used_params_yaml(tmp_dir, dvc):
}
}
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
assert_stage_equal(
resolver.resolve(),
{
@@ -234,7 +234,7 @@ def test_vars_relpath_overwrite(tmp_dir, dvc):
}
},
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
resolver.resolve()
assert resolver.context.imports == {str(tmp_dir / "params.yaml"): None}
@@ -257,5 +257,5 @@ def test_vars_load_partial(tmp_dir, dvc, local, vars_):
d["stages"]["build"]["vars"] = vars_
else:
d["vars"] = vars_
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
resolver.resolve()
+15 -15
View File
@@ -13,25 +13,25 @@ DATA = {"models": {"bar": "bar", "foo": "foo"}}
def test_resolver(tmp_dir, dvc):
resolver = DataResolver(dvc, tmp_dir, TEMPLATED_DVC_YAML_DATA)
resolver = DataResolver(dvc, tmp_dir.fs_path, TEMPLATED_DVC_YAML_DATA)
resolver.context.merge_update(Context(CONTEXT_DATA))
assert resolver.resolve() == RESOLVED_DVC_YAML_DATA
def test_default_params_file_not_exist(tmp_dir, dvc):
d = {"vars": [DATA["models"]]}
resolver = DataResolver(dvc, tmp_dir, d)
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
assert resolver.context == d["vars"][0]
def test_no_params_yaml_and_vars(tmp_dir, dvc):
resolver = DataResolver(dvc, tmp_dir, {})
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
assert not resolver.context
def test_local_vars(tmp_dir, dvc):
resolver = DataResolver(
dvc, tmp_dir, {"vars": [{"foo": "bar", "bar": "foo"}]}
dvc, tmp_dir.fs_path, {"vars": [{"foo": "bar", "bar": "foo"}]}
)
assert resolver.context == {"foo": "bar", "bar": "foo"}
@@ -39,7 +39,7 @@ def test_local_vars(tmp_dir, dvc):
@pytest.mark.parametrize("vars_", ["${file}_params.yaml", {"foo": "${foo}"}])
def test_vars_interpolation_errors(tmp_dir, dvc, vars_):
with pytest.raises(ResolveError) as exc_info:
DataResolver(dvc, tmp_dir, {"vars": [vars_, {"bar": "foo"}]})
DataResolver(dvc, tmp_dir.fs_path, {"vars": [vars_, {"bar": "foo"}]})
assert (
str(exc_info.value)
== "failed to parse 'vars' in 'dvc.yaml': interpolating is not allowed"
@@ -51,7 +51,7 @@ def test_vars_interpolation_errors(tmp_dir, dvc, vars_):
)
def test_default_params_file(tmp_dir, dvc, vars_):
(tmp_dir / DEFAULT_PARAMS_FILE).dump(DATA)
resolver = DataResolver(dvc, tmp_dir, vars_)
resolver = DataResolver(dvc, tmp_dir.fs_path, vars_)
assert resolver.context == DATA
@@ -61,7 +61,7 @@ def test_load_vars_from_file(tmp_dir, dvc):
datasets = {"datasets": ["foo", "bar"]}
(tmp_dir / "params.json").dump(datasets)
d = {"vars": [DEFAULT_PARAMS_FILE, "params.json"]}
resolver = DataResolver(dvc, tmp_dir, d)
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
expected = deepcopy(DATA)
expected.update(datasets)
@@ -77,7 +77,7 @@ def test_load_vars_with_relpath(tmp_dir, scm, dvc):
revisions = ["HEAD", "workspace"]
for rev in dvc.brancher(revs=["HEAD"]):
assert rev == revisions.pop()
resolver = DataResolver(dvc, subdir, d)
resolver = DataResolver(dvc, subdir.fs_path, d)
assert resolver.context == deepcopy(DATA)
@@ -85,7 +85,7 @@ def test_partial_vars_doesnot_exist(tmp_dir, dvc):
(tmp_dir / "test_params.yaml").dump({"sub1": "sub1"})
with pytest.raises(ResolveError) as exc_info:
DataResolver(dvc, tmp_dir, {"vars": ["test_params.yaml:sub2"]})
DataResolver(dvc, tmp_dir.fs_path, {"vars": ["test_params.yaml:sub2"]})
assert (
str(exc_info.value) == "failed to parse 'vars' in 'dvc.yaml': "
@@ -99,7 +99,7 @@ def test_global_overwrite_error_on_imports(tmp_dir, dvc):
d = {"vars": [DEFAULT_PARAMS_FILE, "params.json"]}
with pytest.raises(ResolveError) as exc_info:
DataResolver(dvc, tmp_dir, d)
DataResolver(dvc, tmp_dir.fs_path, d)
assert str(exc_info.value) == (
"failed to parse 'vars' in 'dvc.yaml':\n"
@@ -113,7 +113,7 @@ def test_global_overwrite_vars(tmp_dir, dvc):
d = {"vars": [DATA]}
with pytest.raises(ResolveError) as exc_info:
DataResolver(dvc, tmp_dir, d)
DataResolver(dvc, tmp_dir.fs_path, d)
assert str(exc_info.value) == (
"failed to parse 'vars' in 'dvc.yaml':\n"
@@ -127,7 +127,7 @@ def test_local_declared_vars_overwrite(tmp_dir, dvc):
d = {"vars": [DATA["models"], DATA["models"]]}
with pytest.raises(ResolveError) as exc_info:
DataResolver(dvc, tmp_dir, d)
DataResolver(dvc, tmp_dir.fs_path, d)
assert str(exc_info.value) == (
"failed to parse 'vars' in 'dvc.yaml':\n"
@@ -139,7 +139,7 @@ def test_local_declared_vars_overwrite(tmp_dir, dvc):
def test_specified_params_file_not_exist(tmp_dir, dvc):
d = {"vars": ["not_existing_params.yaml"]}
with pytest.raises(ResolveError) as exc_info:
DataResolver(dvc, tmp_dir, d)
DataResolver(dvc, tmp_dir.fs_path, d)
assert str(exc_info.value) == (
"failed to parse 'vars' in 'dvc.yaml': "
@@ -165,7 +165,7 @@ def test_vars_already_loaded_message(tmp_dir, dvc, local, vars_):
d["stages"]["build"]["vars"] = vars_
with pytest.raises(ResolveError) as exc_info:
resolver = DataResolver(dvc, tmp_dir, d)
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
resolver.resolve()
assert "partially" in str(exc_info.value)
@@ -180,7 +180,7 @@ def test_local_overwrite_error(tmp_dir, dvc, vars_, loc):
d = {"stages": {"build": {"cmd": "echo ${models.foo}", "vars": [vars_]}}}
resolver = DataResolver(dvc, tmp_dir, d)
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
with pytest.raises(ResolveError) as exc_info:
resolver.resolve()
+11 -6
View File
@@ -770,14 +770,14 @@ def test_should_not_checkout_when_adding_cached_copy(tmp_dir, dvc, mocker):
def test_should_relink_on_repeated_add(
link, new_link, link_test_func, tmp_dir, dvc
):
from dvc.path_info import PathInfo
dvc.config["cache"]["type"] = link
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
os.remove("foo")
getattr(dvc.odb.local.fs, link)(PathInfo("bar"), PathInfo("foo"))
getattr(dvc.odb.local.fs, link)(
(tmp_dir / "bar").fs_path, (tmp_dir / "foo").fs_path
)
dvc.odb.local.cache_types = [new_link]
@@ -822,6 +822,7 @@ def test_escape_gitignore_entries(tmp_dir, scm, dvc):
assert ignored_fname in get_gitignore_content()
@pytest.mark.xfail(reason="error message relpath")
def test_add_from_data_dir(tmp_dir, scm, dvc):
tmp_dir.dvc_gen({"dir": {"file1": "file1 content"}})
@@ -866,10 +867,10 @@ def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
for stage in stages[:2]:
# hardlinks are not created for empty files
assert not System.is_hardlink(stage.outs[0].path_info)
assert not System.is_hardlink(stage.outs[0].fs_path)
for stage in stages[2:]:
assert System.is_hardlink(stage.outs[0].path_info)
assert System.is_hardlink(stage.outs[0].fs_path)
for stage in stages:
assert os.path.exists(stage.path)
@@ -1031,7 +1032,11 @@ def test_add_to_remote(tmp_dir, dvc, local_cloud, local_remote):
hash_info = stage.outs[0].hash_info
meta = stage.outs[0].meta
assert local_remote.hash_to_path_info(hash_info.value).read_text() == "foo"
with open(
local_remote.hash_to_path(hash_info.value), encoding="utf-8"
) as stream:
assert stream.read() == "foo"
assert meta.size == len("foo")
+12 -11
View File
@@ -5,7 +5,6 @@ from funcy import first, get_in
from dvc import api
from dvc.exceptions import FileMissingError, OutputNotFoundError
from dvc.path_info import URLInfo
from dvc.utils.fs import remove
from tests.unit.fs.test_repo import make_subrepo
@@ -32,7 +31,7 @@ all_clouds = [pytest.lazy_fixture("local_cloud")] + clouds
def test_get_url(tmp_dir, dvc, remote):
tmp_dir.dvc_gen("foo", "foo")
expected_url = URLInfo(remote.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
expected_url = (remote / "ac/bd18db4cc2f85cedef654fccc4a4d8").url
assert api.get_url("foo") == expected_url
@@ -45,7 +44,7 @@ def test_get_url_external(tmp_dir, erepo_dir, cloud):
# Using file url to force clone to tmp repo
repo_url = f"file://{erepo_dir}"
expected_url = URLInfo(cloud.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
expected_url = (cloud / "ac/bd18db4cc2f85cedef654fccc4a4d8").url
assert api.get_url("foo", repo=repo_url) == expected_url
@@ -255,16 +254,16 @@ def test_get_url_granular(tmp_dir, dvc, s3):
{"dir": {"foo": "foo", "bar": "bar", "nested": {"file": "file"}}}
)
expected_url = URLInfo(s3.url) / "5f/c28ea78987408341668eba6525ebd1.dir"
expected_url = (s3 / "5f/c28ea78987408341668eba6525ebd1.dir").url
assert api.get_url("dir") == expected_url
expected_url = URLInfo(s3.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
expected_url = (s3 / "ac/bd18db4cc2f85cedef654fccc4a4d8").url
assert api.get_url("dir/foo") == expected_url
expected_url = URLInfo(s3.url) / "37/b51d194a7513e45b56f6524f2d51f2"
expected_url = (s3 / "37/b51d194a7513e45b56f6524f2d51f2").url
assert api.get_url("dir/bar") == expected_url
expected_url = URLInfo(s3.url) / "8c/7dd922ad47494fc02c388e12c00eac"
expected_url = (s3 / "8c/7dd922ad47494fc02c388e12c00eac").url
assert api.get_url(os.path.join("dir", "nested", "file")) == expected_url
@@ -277,10 +276,12 @@ def test_get_url_subrepos(tmp_dir, scm, local_cloud):
)
subrepo.dvc.push()
path = os.path.relpath(local_cloud.config["url"])
expected_url = os.path.join(path, "ac", "bd18db4cc2f85cedef654fccc4a4d8")
expected_url = os.fspath(
local_cloud / "ac" / "bd18db4cc2f85cedef654fccc4a4d8"
)
assert api.get_url(os.path.join("subrepo", "dir", "foo")) == expected_url
expected_url = os.path.join(path, "37", "b51d194a7513e45b56f6524f2d51f2")
expected_url = os.fspath(
local_cloud / "37" / "b51d194a7513e45b56f6524f2d51f2"
)
assert api.get_url("subrepo/bar") == expected_url
+6 -7
View File
@@ -102,9 +102,7 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
# to check if dvc will detect that the cache is corrupted.
obj = load(self.dvc.odb.local, out.hash_info)
_, _, entry_oid = list(obj)[0]
cache = os.fspath(
self.dvc.odb.local.hash_to_path_info(entry_oid.value)
)
cache = self.dvc.odb.local.hash_to_path(entry_oid.value)
os.chmod(cache, 0o644)
with open(cache, "w+", encoding="utf-8") as fobj:
@@ -145,7 +143,7 @@ class CheckoutBase(TestDvcGit):
paths = [
path
for output in stage["outs"]
for path in self.dvc.fs.walk_files(output["path"])
for path in self.dvc.fs.find(output["path"])
]
return [
@@ -599,6 +597,7 @@ def test_stats_on_checkout(tmp_dir, dvc, scm):
assert set(stats["deleted"]) == {"foo"}
@pytest.mark.xfail(reason="values relpath")
def test_checkout_stats_on_failure(tmp_dir, dvc, scm):
tmp_dir.dvc_gen(
{"foo": "foo", "dir": {"subdir": {"file": "file"}}, "other": "other"},
@@ -763,17 +762,17 @@ def test_checkout_for_external_outputs(tmp_dir, dvc, workspace):
dvc.add("remote://workspace/foo")
odb = dvc.cloud.get_remote_odb("workspace")
odb.fs.remove(file_path)
odb.fs.remove(str(file_path))
assert not file_path.exists()
stats = dvc.checkout(force=True)
assert stats == {**empty_checkout, "added": [str(file_path)]}
assert stats == {**empty_checkout, "added": ["remote://workspace/foo"]}
assert file_path.exists()
workspace.gen("foo", "foo\nfoo")
stats = dvc.checkout(force=True)
assert stats == {**empty_checkout, "modified": [str(file_path)]}
assert stats == {**empty_checkout, "modified": ["remote://workspace/foo"]}
assert file_path.read_text() == "foo"
+1
View File
@@ -440,6 +440,7 @@ def test_diff_add_similar_files(tmp_dir, scm, dvc, commit_last):
{"dir2": {"file": "text1", "subdir": {"file2": "text2"}}},
commit=last_commit_msg,
)
assert dvc.diff(a_rev) == {
"added": [
{
+7 -9
View File
@@ -4,7 +4,6 @@ from unittest.mock import ANY, patch
from dvc.external_repo import CLONES, external_repo
from dvc.objects.stage import stage
from dvc.objects.transfer import transfer
from dvc.path_info import PathInfo
from dvc.scm.git import Git
from dvc.utils import relpath
from dvc.utils.fs import makedirs, remove
@@ -96,7 +95,8 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
dest = tmp_dir / "file"
with external_repo(os.fspath(erepo_dir)) as repo:
repo.repo_fs.download(
PathInfo(repo.root_dir) / "subdir" / "file", PathInfo(dest)
os.path.join(repo.root_dir, "subdir", "file"),
os.fspath(dest),
)
assert dest.is_file()
@@ -191,9 +191,8 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
with external_repo(os.fspath(erepo_dir)) as repo:
repo.repo_fs.download(
PathInfo(repo.root_dir) / "dir",
PathInfo(tmp_dir / "out"),
follow_subrepos=False,
os.path.join(repo.root_dir, "dir"),
os.fspath(tmp_dir / "out"),
)
expected_files = {"foo": "foo", "bar": "bar", ".gitignore": "/foo\n"}
assert (tmp_dir / "out").read_text() == expected_files
@@ -206,7 +205,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
staging, _, obj = stage(
repo.odb.local,
PathInfo(repo.root_dir) / "dir",
os.path.join(repo.root_dir, "dir"),
repo.repo_fs,
"md5",
dvcignore=repo.dvcignore,
@@ -238,9 +237,8 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
with external_repo(os.fspath(erepo_dir)) as repo:
repo.repo_fs.download(
PathInfo(repo.root_dir) / "dir",
PathInfo(tmp_dir / "out"),
follow_subrepos=False,
os.path.join(repo.root_dir, "dir"),
os.fspath(tmp_dir / "out"),
)
# subrepo files should not be here
assert (tmp_dir / "out").read_text() == scm_files
+68 -87
View File
@@ -8,7 +8,6 @@ import pytest
from dvc.fs import get_cloud_fs
from dvc.fs.local import LocalFileSystem
from dvc.path_info import PathInfo
from dvc.repo import Repo
from dvc.scm import SCM
from tests.basic_env import TestDir, TestGit, TestGitSubmodule
@@ -200,17 +199,17 @@ def test_cleanfs_subrepo(tmp_dir, dvc, scm, monkeypatch):
subrepo = Repo.init(subdir=True)
subrepo_dir.gen({"foo": "foo", "dir": {"bar": "bar"}})
path = PathInfo(subrepo_dir)
path = subrepo_dir.fs_path
assert dvc.fs.exists(path / "foo")
assert dvc.fs.isfile(path / "foo")
assert dvc.fs.exists(path / "dir")
assert dvc.fs.isdir(path / "dir")
assert dvc.fs.exists(dvc.fs.path.join(path, "foo"))
assert dvc.fs.isfile(dvc.fs.path.join(path, "foo"))
assert dvc.fs.exists(dvc.fs.path.join(path, "dir"))
assert dvc.fs.isdir(dvc.fs.path.join(path, "dir"))
assert subrepo.fs.exists(path / "foo")
assert subrepo.fs.isfile(path / "foo")
assert subrepo.fs.exists(path / "dir")
assert subrepo.fs.isdir(path / "dir")
assert subrepo.fs.exists(subrepo.fs.path.join(path, "foo"))
assert subrepo.fs.isfile(subrepo.fs.path.join(path, "foo"))
assert subrepo.fs.exists(subrepo.fs.path.join(path, "dir"))
assert subrepo.fs.isdir(subrepo.fs.path.join(path, "dir"))
def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
@@ -246,11 +245,11 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
)
def test_fs_getsize(dvc, cloud):
cloud.gen({"data": {"foo": "foo"}, "baz": "baz baz"})
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
cls, config, path = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
assert fs.getsize(path_info / "baz") == 7
assert fs.getsize(path_info / "data" / "foo") == 3
assert fs.getsize(fs.path.join(path, "baz")) == 7
assert fs.getsize(fs.path.join(path, "data", "foo")) == 3
@pytest.mark.needs_internet
@@ -270,11 +269,11 @@ def test_fs_getsize(dvc, cloud):
)
def test_fs_upload_fobj(dvc, tmp_dir, cloud):
tmp_dir.gen("foo", "foo")
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
cls, config, path = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
from_info = tmp_dir / "foo"
to_info = path_info / "foo"
to_info = fs.path.join(path, "foo")
with open(from_info, "rb") as stream:
fs.upload_fobj(stream, to_info)
@@ -297,73 +296,25 @@ def test_fs_ls(dvc, cloud):
}
}
)
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
cls, config, path = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
path_info /= "directory"
path = os.path.join(path, "directory")
assert {
os.path.basename(file_key.rstrip("/")) for file_key in fs.ls(path_info)
os.path.basename(file_key.rstrip("/")) for file_key in fs.ls(path)
} == {
"foo",
"bar",
"baz",
"empty",
}
assert set(fs.ls(path_info / "empty")) == set()
assert set(fs.ls(fs.path.join(path, "empty"))) == set()
assert {
(detail["type"], os.path.basename(detail["name"].rstrip("/")))
for detail in fs.ls(path_info / "baz", detail=True)
for detail in fs.ls(fs.path.join(path, "baz"), detail=True)
} == {("file", "quux"), ("directory", "egg")}
@pytest.mark.needs_internet
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("azure"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("webdav"),
pytest.lazy_fixture("gdrive"),
],
)
def test_fs_find(dvc, cloud):
cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}})
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
assert {
os.path.basename(file_key) for file_key in fs.find(path_info / "data")
} == {"foo", "baz", "quux"}
assert {
os.path.basename(file_info["name"])
for file_info in fs.find(path_info / "data", detail=True)
} == {"foo", "baz", "quux"}
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("azure"),
pytest.param(
pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
),
pytest.lazy_fixture("webdav"),
],
)
def test_fs_find_with_etag(dvc, cloud):
cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}})
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
for details in fs.find(path_info / "data", detail=True):
assert (
fs.info(path_info.replace(path=details["name"]))["etag"]
== details["etag"]
)
@pytest.mark.parametrize(
"cloud",
[
@@ -379,15 +330,15 @@ def test_fs_fsspec_path_management(dvc, cloud):
fs = cls(**config)
root = cloud.parents[len(cloud.parents) - 1]
bucket_details = fs.info(root)
bucket_details = fs.info(root.fs_path)
# special conditions: name always points to the bucket name
assert bucket_details["name"] == root.bucket
assert bucket_details["type"] == "directory"
data = cloud / "data"
data_details = fs.info(data)
assert data_details["name"].rstrip("/") == data.path
data_details = fs.info(data.fs_path)
assert data_details["name"].rstrip("/") == data.fs_path
assert data_details["type"] == "directory"
@@ -406,14 +357,14 @@ def test_fs_makedirs_on_upload_and_copy(dvc, cloud):
fs = cls(**config)
with io.BytesIO(b"foo") as stream:
fs.upload(stream, cloud / "dir" / "foo")
fs.upload(stream, (cloud / "dir" / "foo").fs_path)
assert fs.isdir(cloud / "dir")
assert fs.exists(cloud / "dir" / "foo")
assert fs.isdir((cloud / "dir").fs_path)
assert fs.exists((cloud / "dir" / "foo").fs_path)
fs.copy(cloud / "dir" / "foo", cloud / "dir2" / "foo")
assert fs.isdir(cloud / "dir2")
assert fs.exists(cloud / "dir2" / "foo")
fs.copy((cloud / "dir" / "foo").fs_path, (cloud / "dir2" / "foo").fs_path)
assert fs.isdir((cloud / "dir2").fs_path)
assert fs.exists((cloud / "dir2" / "foo").fs_path)
@pytest.mark.needs_internet
@@ -440,7 +391,11 @@ def test_upload_callback(tmp_dir, dvc, cloud):
expected_size = os.path.getsize(tmp_dir / "foo")
callback = fsspec.Callback()
fs.upload(tmp_dir / "foo", cloud / "foo", callback=callback)
fs.upload(
(tmp_dir / "foo").fs_path,
(cloud / "foo").fs_path,
callback=callback,
)
assert callback.size == expected_size
assert callback.value == expected_size
@@ -468,11 +423,15 @@ def test_download_callback(tmp_dir, dvc, cloud, local_cloud):
fs = cls(**config)
(tmp_dir / "to_upload").write_text("foo")
fs.upload(tmp_dir / "to_upload", cloud / "foo")
expected_size = fs.getsize(cloud / "foo")
fs.upload((tmp_dir / "to_upload").fs_path, (cloud / "foo").fs_path)
expected_size = fs.getsize((cloud / "foo").fs_path)
callback = fsspec.Callback()
fs.download_file(cloud / "foo", tmp_dir / "foo", callback=callback)
fs.download_file(
(cloud / "foo").fs_path,
(tmp_dir / "foo").fs_path,
callback=callback,
)
assert callback.size == expected_size
assert callback.value == expected_size
@@ -504,7 +463,9 @@ def test_download_dir_callback(tmp_dir, dvc, cloud):
cloud.gen({"dir": {"foo": "foo", "bar": "bar"}})
callback = fsspec.Callback()
fs.download(cloud / "dir", tmp_dir / "dir", callback=callback)
fs.download(
(cloud / "dir").fs_path, (tmp_dir / "dir").fs_path, callback=callback
)
assert callback.size == 2
assert callback.value == 2
@@ -519,7 +480,11 @@ def test_download_callbacks_on_dvc_git_fs(tmp_dir, dvc, scm, fs_type):
fs = dvc.dvcfs if fs_type == "dvc" else scm.get_fs("HEAD")
callback = fsspec.Callback()
fs.download(tmp_dir / "file", tmp_dir / "file2", callback=callback)
fs.download(
(tmp_dir / "file").fs_path,
(tmp_dir / "file2").fs_path,
callback=callback,
)
size = os.path.getsize(tmp_dir / "file")
assert (tmp_dir / "file2").read_text() == "file"
@@ -530,7 +495,11 @@ def test_download_callbacks_on_dvc_git_fs(tmp_dir, dvc, scm, fs_type):
pytest.skip("gitfs does not support download_dir")
callback = fsspec.Callback()
fs.download(tmp_dir / "dir", tmp_dir / "dir2", callback=callback)
fs.download(
(tmp_dir / "dir").fs_path,
(tmp_dir / "dir2").fs_path,
callback=callback,
)
assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"}
assert callback.size == 2
@@ -544,14 +513,22 @@ def test_callback_on_repo_fs(tmp_dir, dvc, scm):
fs = dvc.repo_fs
callback = fsspec.Callback()
fs.download(tmp_dir / "dir", tmp_dir / "dir2", callback=callback)
fs.download(
(tmp_dir / "dir").fs_path,
(tmp_dir / "dir2").fs_path,
callback=callback,
)
assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"}
assert callback.size == 2
assert callback.value == 2
callback = fsspec.Callback()
fs.download(tmp_dir / "dir" / "foo", tmp_dir / "foo", callback=callback)
fs.download(
(tmp_dir / "dir" / "foo").fs_path,
(tmp_dir / "foo").fs_path,
callback=callback,
)
size = os.path.getsize(tmp_dir / "dir" / "foo")
assert (tmp_dir / "foo").read_text() == "foo"
@@ -559,7 +536,11 @@ def test_callback_on_repo_fs(tmp_dir, dvc, scm):
assert callback.value == size
callback = fsspec.Callback()
fs.download(tmp_dir / "dir" / "bar", tmp_dir / "bar", callback=callback)
fs.download(
(tmp_dir / "dir" / "bar").fs_path,
(tmp_dir / "bar").fs_path,
callback=callback,
)
size = os.path.getsize(tmp_dir / "dir" / "bar")
assert (tmp_dir / "bar").read_text() == "bar"
+1 -1
View File
@@ -22,7 +22,7 @@ class TestGC(TestDvcGit):
self.dvc.add(self.FOO)
self.dvc.add(self.DATA_DIR)
self.good_cache = [
self.dvc.odb.local.hash_to_path_info(md5)
self.dvc.odb.local.hash_to_path(md5)
for md5 in self.dvc.odb.local.all()
]
+40 -25
View File
@@ -18,6 +18,11 @@ def _to_pattern_info_list(str_list: List):
return [PatternInfo(a, "") for a in str_list]
def walk_files(dvc, *args):
for fs_path in dvc.dvcignore.find(*args):
yield fs_path
@pytest.mark.parametrize("filename", ["ignored", "тест"])
def test_ignore(tmp_dir, dvc, filename):
tmp_dir.gen({"dir": {filename: filename, "other": "text2"}})
@@ -25,10 +30,10 @@ def test_ignore(tmp_dir, dvc, filename):
dvc._reset()
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir)
result = walk_files(dvc, dvc.fs, tmp_dir)
assert set(result) == {
tmp_dir / DvcIgnore.DVCIGNORE_FILE,
tmp_dir / "dir" / "other",
(tmp_dir / DvcIgnore.DVCIGNORE_FILE).fs_path,
(tmp_dir / "dir" / "other").fs_path,
}
@@ -125,11 +130,11 @@ def test_ignore_on_branch(tmp_dir, scm, dvc):
dvc._reset()
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir)
result = walk_files(dvc, dvc.fs, tmp_dir)
assert set(result) == {
tmp_dir / "foo",
tmp_dir / "bar",
tmp_dir / DvcIgnore.DVCIGNORE_FILE,
(tmp_dir / "foo").fs_path,
(tmp_dir / "bar").fs_path,
(tmp_dir / DvcIgnore.DVCIGNORE_FILE).fs_path,
}
dvc.fs = scm.get_fs("branch")
@@ -146,8 +151,11 @@ def test_match_nested(tmp_dir, dvc):
}
)
dvc._reset()
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir)
assert set(result) == {tmp_dir / DvcIgnore.DVCIGNORE_FILE, tmp_dir / "foo"}
result = walk_files(dvc, dvc.fs, tmp_dir)
assert set(result) == {
(tmp_dir / DvcIgnore.DVCIGNORE_FILE).fs_path,
(tmp_dir / "foo").fs_path,
}
def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
@@ -155,8 +163,11 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
ext_dir = TmpDir(os.fspath(tmp_path_factory.mktemp("external_dir")))
ext_dir.gen({"y.backup": "y", "tmp": {"file": "ext tmp"}})
result = dvc.dvcignore.walk_files(dvc.fs, ext_dir)
assert set(result) == {ext_dir / "y.backup", ext_dir / "tmp" / "file"}
result = walk_files(dvc, dvc.fs, ext_dir)
assert set(result) == {
(ext_dir / "y.backup").fs_path,
(ext_dir / "tmp" / "file").fs_path,
}
assert dvc.dvcignore.is_ignored_dir(os.fspath(ext_dir / "tmp")) is False
assert (
dvc.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup")) is False
@@ -171,7 +182,7 @@ def test_ignore_subrepo(tmp_dir, scm, dvc):
subrepo_dir = tmp_dir / "subdir"
result = dvc.dvcignore.walk_files(dvc.fs, subrepo_dir)
result = walk_files(dvc, dvc.fs, subrepo_dir)
assert set(result) == set()
with subrepo_dir.chdir():
@@ -212,8 +223,8 @@ def test_ignore_blank_line(tmp_dir, dvc):
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "foo\n\ndir/ignored")
dvc._reset()
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir / "dir")
assert set(result) == {tmp_dir / "dir" / "other"}
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
assert set(result) == {(tmp_dir / "dir" / "other").fs_path}
# It is not possible to re-include a file if a parent directory of
@@ -247,8 +258,10 @@ def test_ignore_file_in_parent_path(
tmp_dir.gen(data_struct)
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "\n".join(pattern_list))
dvc._reset()
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir / "dir")
assert set(result) == {tmp_dir / relpath for relpath in result_set}
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
assert set(result) == {
(tmp_dir / relpath).fs_path for relpath in result_set
}
# If there is a separator at the end of the pattern then the pattern
@@ -268,10 +281,10 @@ def test_ignore_sub_directory(tmp_dir, dvc):
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "doc/fortz"}})
dvc._reset()
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir / "dir")
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
assert set(result) == {
tmp_dir / "dir" / "a" / "doc" / "fortz" / "a",
tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE,
(tmp_dir / "dir" / "a" / "doc" / "fortz" / "a").fs_path,
(tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE).fs_path,
}
@@ -280,8 +293,10 @@ def test_ignore_directory(tmp_dir, dvc):
tmp_dir.gen({"dir": {"fortz": {}, "a": {"fortz": {}}}})
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "fortz"}})
dvc._reset()
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir / "dir")
assert set(result) == {tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE}
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
assert set(result) == {
(tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE).fs_path
}
def test_multi_ignore_file(tmp_dir, dvc, monkeypatch):
@@ -289,10 +304,10 @@ def test_multi_ignore_file(tmp_dir, dvc, monkeypatch):
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/subdir/*_ignore")
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "!subdir/not_ignore"}})
dvc._reset()
result = dvc.dvcignore.walk_files(dvc.fs, tmp_dir / "dir")
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
assert set(result) == {
tmp_dir / "dir" / "subdir" / "not_ignore",
tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE,
(tmp_dir / "dir" / "subdir" / "not_ignore").fs_path,
(tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE).fs_path,
}
@@ -384,7 +399,7 @@ def test_ignore_in_added_dir(tmp_dir, dvc):
dvc._reset()
ignored_path = tmp_dir / "dir" / "sub" / "ignored"
result = dvc.dvcignore.walk_files(dvc.fs, ignored_path)
result = walk_files(dvc, dvc.fs, ignored_path)
assert set(result) == set()
assert ignored_path.exists()
+10 -7
View File
@@ -87,7 +87,7 @@ def test_import_url_to_dir(dname, tmp_dir, dvc):
dst = tmp_dir / dname / "file"
assert stage.outs[0].path_info == dst
assert stage.outs[0].fs_path == os.fspath(dst)
assert os.path.isdir(dname)
assert dst.read_text() == "file content"
@@ -290,7 +290,10 @@ def test_import_url_to_remote_single_file(
assert len(stage.outs) == 1
hash_info = stage.outs[0].hash_info
assert local_remote.hash_to_path_info(hash_info.value).read_text() == "foo"
with open(
local_remote.hash_to_path(hash_info.value), encoding="utf-8"
) as fobj:
assert fobj.read() == "foo"
assert stage.outs[0].meta.size == len("foo")
@@ -335,7 +338,7 @@ def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote):
hash_info = stage.outs[0].hash_info
with open(
local_remote.hash_to_path_info(hash_info.value), encoding="utf-8"
local_remote.hash_to_path(hash_info.value), encoding="utf-8"
) as stream:
file_parts = json.load(stream)
@@ -347,10 +350,10 @@ def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote):
}
for file_part in file_parts:
assert (
local_remote.hash_to_path_info(file_part["md5"]).read_text()
== file_part["relpath"]
)
with open(
local_remote.hash_to_path(file_part["md5"]), encoding="utf-8"
) as fobj:
assert fobj.read() == file_part["relpath"]
def test_import_url_to_remote_absolute(
+5 -5
View File
@@ -37,7 +37,7 @@ class TestCache(TestDvc):
self.assertIn(self.cache2_md5, md5_list)
def test_get(self):
cache = ODBManager(self.dvc).local.hash_to_path_info(self.cache1_md5)
cache = ODBManager(self.dvc).local.hash_to_path(self.cache1_md5)
self.assertEqual(os.fspath(cache), self.cache1)
@@ -50,13 +50,13 @@ class TestCacheLoadBadDirCache(TestDvc):
from dvc.objects import load
dir_hash = "123.dir"
fname = os.fspath(self.dvc.odb.local.hash_to_path_info(dir_hash))
fname = os.fspath(self.dvc.odb.local.hash_to_path(dir_hash))
self.create(fname, "<clearly>not,json")
with pytest.raises(ObjectFormatError):
load(self.dvc.odb.local, HashInfo("md5", dir_hash))
dir_hash = "234.dir"
fname = os.fspath(self.dvc.odb.local.hash_to_path_info(dir_hash))
fname = os.fspath(self.dvc.odb.local.hash_to_path(dir_hash))
self.create(fname, '{"a": "b"}')
with pytest.raises(ObjectFormatError):
load(self.dvc.odb.local, HashInfo("md5", dir_hash))
@@ -88,7 +88,7 @@ class TestExternalCacheDir(TestDvc):
self.dvc.__init__()
assert self.dvc.odb.ssh.path_info == ssh_url + "/tmp"
assert self.dvc.odb.ssh.fs_path == "/tmp"
class TestSharedCacheDir(TestDir):
@@ -158,7 +158,7 @@ class TestCmdCacheDir(TestDvc):
self.assertEqual(ret, 0)
config = configobj.ConfigObj(self.dvc.config.files["repo"])
self.assertEqual(config["cache"]["dir"], dname.replace("\\", "/"))
self.assertEqual(config["cache"]["dir"], dname)
def test_relative_path(self):
tmpdir = os.path.realpath(self.mkdtemp())
+10 -11
View File
@@ -9,7 +9,6 @@ import pytest
from dvc.config import Config
from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError
from dvc.main import main
from dvc.path_info import PathInfo
from dvc.utils.fs import remove
from tests.basic_env import TestDvc
from tests.remotes import Local
@@ -149,14 +148,14 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}})
path_info = PathInfo("data")
path = (tmp_dir / "data").fs_path
tree = Tree.from_list(
[{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}]
)
tree.digest()
with patch("dvc.objects.stage._stage_tree", return_value=(None, tree)):
_, _, obj = stage(dvc.odb.local, path_info, dvc.odb.local.fs, "md5")
_, _, obj = stage(dvc.odb.local, path, dvc.odb.local.fs, "md5")
hash1 = obj.hash_info
tree = Tree.from_list(
@@ -164,7 +163,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
)
tree.digest()
with patch("dvc.objects.stage._stage_tree", return_value=(None, tree)):
_, _, obj = stage(dvc.odb.local, path_info, dvc.odb.local.fs, "md5")
_, _, obj = stage(dvc.odb.local, path, dvc.odb.local.fs, "md5")
hash2 = obj.hash_info
assert hash1 == hash2
@@ -183,7 +182,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
def unreliable_upload(from_fs, from_info, to_fs, to_info, **kwargs):
if os.path.abspath(to_info) == os.path.abspath(
odb.get(foo.hash_info).path_info
odb.get(foo.hash_info).fs_path
):
raise Exception("stop foo")
return original(from_fs, from_info, to_fs, to_info, **kwargs)
@@ -273,8 +272,8 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
# foo .dir file should be uploaded after bar
odb = dvc.cloud.get_remote_odb("upstream")
foo_path = odb.hash_to_path_info(foo.hash_info.value)
bar_path = odb.hash_to_path_info(foo.obj.trie[("bar",)][1].value)
foo_path = odb.hash_to_path(foo.hash_info.value)
bar_path = odb.hash_to_path(foo.obj.trie[("bar",)][1].value)
paths = [args[3] for args, _ in mocked_upload.call_args_list]
assert paths.index(foo_path) > paths.index(bar_path)
@@ -412,7 +411,7 @@ def test_protect_local_remote(tmp_dir, dvc, local_remote):
dvc.push()
odb = dvc.cloud.get_remote_odb("upstream")
remote_cache_file = odb.hash_to_path_info(stage.outs[0].hash_info.value)
remote_cache_file = odb.hash_to_path(stage.outs[0].hash_info.value)
assert os.path.exists(remote_cache_file)
assert stat.S_IMODE(os.stat(remote_cache_file).st_mode) == 0o444
@@ -427,7 +426,7 @@ def test_push_incomplete_dir(tmp_dir, dvc, mocker, local_remote):
file_objs = [entry_obj for _, _, entry_obj in out.obj]
# remove one of the cache files for directory
remove(odb.hash_to_path_info(file_objs[0].value))
remove(odb.hash_to_path(file_objs[0].value))
dvc.push()
assert not remote_odb.exists(out.hash_info)
@@ -442,8 +441,8 @@ def test_upload_exists(tmp_dir, dvc, local_remote):
# normally they are set to read-only for DVC remotes
odb.fs.CACHE_MODE = 0o644
from_info = PathInfo(tmp_dir / "foo")
to_info = odb.path_info / "foo"
from_info = (tmp_dir / "foo").fs_path
to_info = odb.fs.path.join(odb.fs_path, "foo")
odb.fs.upload(from_info, to_info)
assert odb.fs.exists(to_info)
+9 -10
View File
@@ -6,7 +6,6 @@ from funcy import raiser
from dvc.dvcfile import PIPELINE_FILE, FileIsGitIgnored
from dvc.exceptions import NoOutputOrStageError
from dvc.path_info import PathInfo
from dvc.repo import Repo
from dvc.stage.exceptions import (
StageFileDoesNotExistError,
@@ -226,7 +225,7 @@ def test_collect_granular_with_target(tmp_dir, dvc, stages):
) == [(stages["copy-bar-foobar"], None)]
assert dvc.stage.collect_granular("foobar") == [
(stages["copy-bar-foobar"], PathInfo(tmp_dir / "foobar"))
(stages["copy-bar-foobar"], os.path.join(tmp_dir, "foobar"))
]
@@ -296,10 +295,10 @@ def test_collect_granular_same_output_name_stage_name(tmp_dir, dvc, run_copy):
assert list(map(itemgetter(1), coll)) == [None] * 2
assert dvc.stage.collect_granular("./copy-foo-bar") == [
(stage2, PathInfo(tmp_dir / "copy-foo-bar"))
(stage2, os.path.join(tmp_dir / "copy-foo-bar"))
]
assert dvc.stage.collect_granular("./copy-foo-bar", with_deps=True) == [
(stage2, PathInfo(tmp_dir / "copy-foo-bar"))
(stage2, os.path.join(tmp_dir / "copy-foo-bar"))
]
@@ -330,7 +329,7 @@ def test_collect_granular_collision_output_dir_stage_name(
assert dvc.stage.collect_granular("dir") == [(stage3, None)]
assert not dvc.stage.collect_granular("dir", recursive=True)
assert dvc.stage.collect_granular("./dir") == [
(stage1, PathInfo(tmp_dir / "dir"))
(stage1, os.path.join(tmp_dir / "dir"))
]
@@ -468,24 +467,24 @@ def test_gitignored_file_try_collect_granular_for_data_files(
tmp_dir, dvc, scm
):
(stage,) = tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}})
path = PathInfo("data") / "foo"
path = os.path.join("data", "foo")
assert dvc.stage.collect_granular(str(path)) == [
(stage, PathInfo(tmp_dir / path))
assert dvc.stage.collect_granular(path) == [
(stage, os.path.join(tmp_dir, path))
]
scm.ignore(stage.path)
dvc._reset()
with pytest.raises(NoOutputOrStageError):
dvc.stage.collect_granular(str(path))
dvc.stage.collect_granular(path)
def test_gitignored_file_try_collect_granular_for_dvc_yaml_files(
tmp_dir, dvc, scm, stages
):
assert dvc.stage.collect_granular("bar") == [
(stages["copy-foo-bar"], PathInfo(tmp_dir / "bar"))
(stages["copy-foo-bar"], os.path.join(tmp_dir, "bar"))
]
scm.ignore(tmp_dir / "dvc.yaml")
+5 -7
View File
@@ -2,7 +2,6 @@ import os
import re
from dvc.hash_info import HashInfo
from dvc.path_info import PathInfo
from dvc.repo import Repo
from dvc.state import State
from dvc.utils import file_md5
@@ -11,23 +10,22 @@ from dvc.utils import file_md5
def test_state(tmp_dir, dvc):
tmp_dir.gen("foo", "foo content")
path = tmp_dir / "foo"
path_info = PathInfo(path)
hash_info = HashInfo("md5", file_md5(path, dvc.fs))
state = State(dvc.root_dir, dvc.tmp_dir, dvc.dvcignore)
state.save(path_info, dvc.fs, hash_info)
assert state.get(path_info, dvc.fs)[1] == hash_info
state.save(path, dvc.fs, hash_info)
assert state.get(path, dvc.fs)[1] == hash_info
path.unlink()
path.write_text("1")
assert state.get(path_info, dvc.fs) == (None, None)
assert state.get(path, dvc.fs) == (None, None)
hash_info = HashInfo("md5", file_md5(path, dvc.fs))
state.save(path_info, dvc.fs, hash_info)
state.save(path, dvc.fs, hash_info)
assert state.get(path_info, dvc.fs)[1] == hash_info
assert state.get(path, dvc.fs)[1] == hash_info
def test_state_overflow(tmp_dir, dvc):
+7 -2
View File
@@ -6,9 +6,8 @@ import uuid
import pytest
from funcy import cached_property
from dvc.path_info import CloudURLInfo
from .base import Base
from .path_info import CloudURLInfo
TEST_AZURE_CONTAINER = "tests"
TEST_AZURE_CONNECTION_STRING = (
@@ -57,6 +56,12 @@ class Azure(Base, CloudURLInfo):
assert errors is None
return self.read_bytes().decode(encoding)
@property
def fs_path(self):
bucket = self.bucket.rstrip("/")
path = self.path.lstrip("/")
return f"{bucket}/{path}"
@pytest.fixture(scope="session")
def azure_server(test_config, docker_compose, docker_services):
+1 -1
View File
@@ -3,7 +3,7 @@ import pathlib
from funcy import cached_property
from dvc.path_info import URLInfo
from .path_info import URLInfo
class Base(URLInfo):
+21 -2
View File
@@ -3,18 +3,37 @@ import json
import os
import uuid
from functools import partialmethod
from urllib.parse import urlparse
import pytest
from funcy import cached_property, retry
from dvc.fs.gdrive import GDriveFileSystem, GDriveURLInfo
from dvc.fs.gdrive import GDriveFileSystem
from dvc.utils import tmp_fname
from .base import Base
from .path_info import CloudURLInfo
TEST_GDRIVE_REPO_BUCKET = "root"
class GDriveURLInfo(CloudURLInfo):
def __init__(self, url):
super().__init__(url)
# GDrive URL host part is case sensitive,
# we are restoring it here.
p = urlparse(url)
self.host = p.netloc
assert self.netloc == self.host
# Normalize path. Important since we have a cache (path to ID)
# and don't want to deal with different variations of path in it.
import re
self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
def _gdrive_retry(func):
def should_retry(exc):
from googleapiclient.errors import HttpError
@@ -128,5 +147,5 @@ def gdrive(test_config, make_tmp_dir):
fs = GDriveFileSystem(
gdrive_credentials_tmp_dir=tmp_dir.dvc.tmp_dir, **ret.config
)
fs.fs._gdrive_create_dir("root", fs.path_info.path)
fs.fs._gdrive_create_dir("root", fs.url)
yield ret
+1 -1
View File
@@ -5,10 +5,10 @@ import uuid
import pytest
from funcy import cached_property
from dvc.path_info import CloudURLInfo
from dvc.utils import env2bool
from .base import Base
from .path_info import CloudURLInfo
TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test")

Some files were not shown because too many files have changed in this diff Show More