repofs: migrate to fsspec

This commit is contained in:
Ruslan Kuprieiev
2022-04-18 14:17:20 +03:00
parent 53c7f44480
commit 517f45d45f
20 changed files with 300 additions and 325 deletions
+5 -8
View File
@@ -22,17 +22,14 @@ def get_url(path, repo=None, rev=None, remote=None):
with reraise(FileNotFoundError, PathMissingError(path, repo)): with reraise(FileNotFoundError, PathMissingError(path, repo)):
info = _repo.repo_fs.info(path) info = _repo.repo_fs.info(path)
if not info["isdvc"]: dvc_info = info.get("dvc_info")
if not dvc_info:
raise OutputNotFoundError(path, repo) raise OutputNotFoundError(path, repo)
cloud = info["repo"].cloud dvc_repo = info["repo"]
_, _, dvc_fs, dvc_path = _repo.repo_fs._get_fs_pair(path) md5 = dvc_info["md5"]
if not os.path.isabs(path): return dvc_repo.cloud.get_url_for(remote, checksum=md5)
dvc_path = dvc_path.replace("\\", "/")
md5 = info["repo"].dvcfs.info(dvc_path)["md5"]
return cloud.get_url_for(remote, checksum=md5)
def open( # noqa, pylint: disable=redefined-builtin def open( # noqa, pylint: disable=redefined-builtin
+2 -5
View File
@@ -30,7 +30,6 @@ class ReferenceHashFile(HashFile):
checksum: Optional[str] = None, checksum: Optional[str] = None,
**kwargs, **kwargs,
): ):
from dvc.fs.repo import RepoFileSystem
super().__init__(fs_path, fs, hash_info, **kwargs) super().__init__(fs_path, fs, hash_info, **kwargs)
self.checksum = checksum or fs.checksum(fs_path) self.checksum = checksum or fs.checksum(fs_path)
@@ -53,8 +52,6 @@ class ReferenceHashFile(HashFile):
return self.fs.checksum(self.fs_path) return self.fs.checksum(self.fs_path)
def to_bytes(self): def to_bytes(self):
from dvc.fs.repo import RepoFileSystem
# NOTE: dumping reference FS's this way is insecure, as the # NOTE: dumping reference FS's this way is insecure, as the
# fully parsed remote FS config will include credentials # fully parsed remote FS config will include credentials
# #
@@ -75,7 +72,7 @@ class ReferenceHashFile(HashFile):
@classmethod @classmethod
def from_bytes(cls, data: bytes, fs_cache: Optional[dict] = None): def from_bytes(cls, data: bytes, fs_cache: Optional[dict] = None):
from dvc.fs import get_fs_cls from dvc.fs import get_fs_cls
from dvc.fs.repo import RepoFileSystem from dvc.fs.repo import RepoFileSystem, _RepoFileSystem
try: try:
dict_ = pickle.loads(data) dict_ = pickle.loads(data)
@@ -92,7 +89,7 @@ class ReferenceHashFile(HashFile):
fs = fs_cache.get((scheme, config_pairs)) if fs_cache else None fs = fs_cache.get((scheme, config_pairs)) if fs_cache else None
if not fs: if not fs:
config = dict(config_pairs) config = dict(config_pairs)
if RepoFileSystem.PARAM_REPO_URL in config: if _RepoFileSystem.PARAM_REPO_URL in config:
fs_cls = RepoFileSystem fs_cls = RepoFileSystem
else: else:
fs_cls = get_fs_cls(config, scheme=scheme) fs_cls = get_fs_cls(config, scheme=scheme)
+6 -9
View File
@@ -101,6 +101,7 @@ class RepoDependency(Dependency):
from dvc.data.stage import stage from dvc.data.stage import stage
from dvc.data.tree import Tree, TreeError from dvc.data.tree import Tree, TreeError
from dvc.exceptions import NoOutputOrStageError, PathMissingError from dvc.exceptions import NoOutputOrStageError, PathMissingError
from dvc.utils import as_posix
local_odb = self.repo.odb.local local_odb = self.repo.odb.local
locked = kwargs.pop("locked", True) locked = kwargs.pop("locked", True)
@@ -115,11 +116,7 @@ class RepoDependency(Dependency):
if not obj_only: if not obj_only:
try: try:
for odb, obj_ids in repo.used_objs( for odb, obj_ids in repo.used_objs(
[ [os.path.join(repo.root_dir, self.def_path)],
os.path.abspath(
os.path.join(repo.root_dir, self.def_path)
)
],
force=True, force=True,
jobs=kwargs.get("jobs"), jobs=kwargs.get("jobs"),
recursive=True, recursive=True,
@@ -135,7 +132,7 @@ class RepoDependency(Dependency):
try: try:
staging, _, staged_obj = stage( staging, _, staged_obj = stage(
local_odb, local_odb,
self.def_path, as_posix(self.def_path),
repo.repo_fs, repo.repo_fs,
local_odb.fs.PARAM_CHECKSUM, local_odb.fs.PARAM_CHECKSUM,
) )
@@ -175,17 +172,17 @@ class RepoDependency(Dependency):
continue continue
if ( if (
obj.fs.repo_url in checked_urls obj.fs.repo_url in checked_urls
or obj.fs._root_dir in checked_urls or obj.fs.repo.root_dir in checked_urls
): ):
continue continue
self_url = self.repo.url or self.repo.root_dir self_url = self.repo.url or self.repo.root_dir
if ( if (
obj.fs.repo_url is not None obj.fs.repo_url is not None
and obj.fs.repo_url == self_url and obj.fs.repo_url == self_url
or obj.fs._root_dir == self.repo.root_dir or obj.fs.repo.root_dir == self.repo.root_dir
): ):
raise CircularImportError(self, obj.fs.repo_url, self_url) raise CircularImportError(self, obj.fs.repo_url, self_url)
checked_urls.update([obj.fs.repo_url, obj.fs._root_dir]) checked_urls.update([obj.fs.repo_url, obj.fs.repo.root_dir])
def get_obj(self, filter_info=None, **kwargs): def get_obj(self, filter_info=None, **kwargs):
locked = kwargs.get("locked", True) locked = kwargs.get("locked", True)
+2 -2
View File
@@ -51,5 +51,5 @@ class GitFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
def rev(self) -> str: def rev(self) -> str:
return self.fs.rev return self.fs.rev
def ls(self, path, **kwargs): def ls(self, path, detail=True, **kwargs):
return self.fs.ls(path, **kwargs) or [] return self.fs.ls(path, detail=detail, **kwargs) or []
-2
View File
@@ -81,8 +81,6 @@ class Path:
return self.isin_or_eq(left, right) or self.isin(right, left) return self.isin_or_eq(left, right) or self.isin(right, left)
def relpath(self, path, start): def relpath(self, path, start):
if not start:
return path
return self.flavour.relpath(path, start=start) return self.flavour.relpath(path, start=start)
def relparts(self, path, base): def relparts(self, path, base):
+89 -110
View File
@@ -5,11 +5,14 @@ from contextlib import suppress
from itertools import takewhile from itertools import takewhile
from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union
from funcy import wrap_with from fsspec.spec import AbstractFileSystem
from funcy import cached_property, wrap_prop, wrap_with
from ._callback import DEFAULT_CALLBACK from ._callback import DEFAULT_CALLBACK
from .base import FileSystem from .base import FileSystem
from .dvc import DvcFileSystem from .dvc import DvcFileSystem
from .fsspec_wrapper import FSSpecWrapper
from .path import Path
if TYPE_CHECKING: if TYPE_CHECKING:
from dvc.repo import Repo from dvc.repo import Repo
@@ -41,27 +44,27 @@ def _ls(fs, path):
def _merge_info(repo, fs_info, dvc_info): def _merge_info(repo, fs_info, dvc_info):
from dvc.utils import is_exec from dvc.utils import is_exec
if not fs_info: ret = {"repo": repo}
dvc_info["repo"] = repo
dvc_info["isdvc"] = True
return dvc_info
fs_info["repo"] = repo
fs_info["isout"] = dvc_info.get("isout", False) if dvc_info else False
fs_info["outs"] = dvc_info["outs"] if dvc_info else None
fs_info["isdvc"] = dvc_info["isdvc"] if dvc_info else False
fs_info["meta"] = dvc_info.get("meta") if dvc_info else None
isexec = False
if dvc_info: if dvc_info:
isexec = dvc_info["isexec"] ret["dvc_info"] = dvc_info
elif fs_info["type"] == "file": ret["type"] = dvc_info["type"]
isexec = is_exec(fs_info["mode"]) ret["size"] = dvc_info["size"]
fs_info["isexec"] = isexec if not fs_info and "md5" in dvc_info:
return fs_info ret["md5"] = dvc_info["md5"]
if fs_info:
ret["type"] = fs_info["type"]
ret["size"] = fs_info["size"]
isexec = False
if fs_info["type"] == "file":
isexec = is_exec(fs_info["mode"])
ret["isexec"] = isexec
return ret
class RepoFileSystem(FileSystem): # pylint:disable=abstract-method class _RepoFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
"""DVC + git-tracked files fs. """DVC + git-tracked files fs.
Args: Args:
@@ -71,10 +74,6 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
kwargs: Additional keyword arguments passed to the `DvcFileSystem()`. kwargs: Additional keyword arguments passed to the `DvcFileSystem()`.
""" """
sep = os.sep
scheme = "local"
PARAM_CHECKSUM = "md5"
PARAM_REPO_URL = "repo_url" PARAM_REPO_URL = "repo_url"
PARAM_REPO_ROOT = "repo_root" PARAM_REPO_ROOT = "repo_root"
PARAM_REV = "rev" PARAM_REV = "rev"
@@ -105,6 +104,7 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
else: else:
self.repo_factory = repo_factory self.repo_factory = repo_factory
self.path = Path(self.sep)
self.repo = repo self.repo = repo
self.hash_jobs = repo.fs.hash_jobs self.hash_jobs = repo.fs.hash_jobs
self._root_dir: str = repo.root_dir self._root_dir: str = repo.root_dir
@@ -131,7 +131,7 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
def config(self): def config(self):
return { return {
self.PARAM_REPO_URL: self.repo_url, self.PARAM_REPO_URL: self.repo_url,
self.PARAM_REPO_ROOT: self._root_dir, self.PARAM_REPO_ROOT: self.repo.root_dir,
self.PARAM_REV: getattr(self.repo.fs, "rev", None), self.PARAM_REV: getattr(self.repo.fs, "rev", None),
self.PARAM_CACHE_DIR: os.path.abspath( self.PARAM_CACHE_DIR: os.path.abspath(
self.repo.odb.local.cache_dir self.repo.odb.local.cache_dir
@@ -188,7 +188,7 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
) as repo: ) as repo:
return repo, factory return repo, factory
def _get_repo(self, path: str) -> Optional["Repo"]: def _get_repo(self, path: str) -> "Repo":
"""Returns repo that the path falls in, using prefix. """Returns repo that the path falls in, using prefix.
If the path is already tracked/collected, it just returns the repo. If the path is already tracked/collected, it just returns the repo.
@@ -204,7 +204,7 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
if not prefix: if not prefix:
return self.repo return self.repo
parents = (parent for parent in self.path.parents(path)) parents = (parent for parent in self.repo.fs.path.parents(path))
dirs = [path] + list(takewhile(lambda p: p != prefix, parents)) dirs = [path] + list(takewhile(lambda p: p != prefix, parents))
dirs.reverse() dirs.reverse()
self._update(dirs, starting_repo=repo) self._update(dirs, starting_repo=repo)
@@ -236,12 +236,24 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
def _get_fs_pair( def _get_fs_pair(
self, path self, path
) -> Tuple[FileSystem, Optional[DvcFileSystem], str]: ) -> Tuple[
Optional[FileSystem],
Optional[str],
Optional[DvcFileSystem],
Optional[str],
]:
""" """
Returns a pair of fss based on repo the path falls in, using prefix. Returns a pair of fss based on repo the path falls in, using prefix.
""" """
from dvc.utils import as_posix
if os.path.isabs(path): if os.path.isabs(path):
return None, None, self.repo.dvcfs, path if self.repo.fs.path.isin_or_eq(path, self.repo.root_dir):
path = self.repo.fs.path.relpath(path, self.repo.root_dir)
else:
return None, None, self.repo.dvcfs, path
path = as_posix(path)
parts = self.path.parts(path) parts = self.path.parts(path)
if parts and parts[0] == os.curdir: if parts and parts[0] == os.curdir:
@@ -260,13 +272,16 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
dvc_parts = dvc_parts[1:] dvc_parts = dvc_parts[1:]
dvc_fs = self._dvcfss.get(repo.root_dir) dvc_fs = self._dvcfss.get(repo.root_dir)
dvc_path = dvc_fs.path.join(*dvc_parts) if dvc_parts else "" if dvc_fs:
dvc_path = dvc_fs.path.join(*dvc_parts) if dvc_parts else ""
else:
dvc_path = None
return fs, fs_path, dvc_fs, dvc_path return fs, fs_path, dvc_fs, dvc_path
def open( def open(
self, path, mode="r", encoding="utf-8", **kwargs self, path, mode="r", encoding="utf-8", **kwargs
): # pylint: disable=arguments-renamed ): # pylint: disable=arguments-renamed, arguments-differ
if "b" in mode: if "b" in mode:
encoding = None encoding = None
@@ -279,29 +294,10 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
return dvc_fs.open(dvc_path, mode=mode, encoding=encoding, **kwargs) return dvc_fs.open(dvc_path, mode=mode, encoding=encoding, **kwargs)
def exists(self, path, **kwargs) -> bool:
try:
self.info(path, **kwargs)
return True
except FileNotFoundError:
return False
def isdir(self, path, **kwargs): # pylint: disable=arguments-renamed
try:
return self.info(path, **kwargs)["type"] == "directory"
except FileNotFoundError:
return False
def isdvc(self, path, **kwargs): def isdvc(self, path, **kwargs):
_, _, dvc_fs, dvc_path = self._get_fs_pair(path) _, _, dvc_fs, dvc_path = self._get_fs_pair(path)
return dvc_fs is not None and dvc_fs.isdvc(dvc_path, **kwargs) return dvc_fs is not None and dvc_fs.isdvc(dvc_path, **kwargs)
def isfile(self, path, **kwargs): # pylint: disable=arguments-renamed
try:
return self.info(path, **kwargs)["type"] == "file"
except FileNotFoundError:
return False
def ls(self, path, detail=True, **kwargs): def ls(self, path, detail=True, **kwargs):
fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(path) fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(path)
@@ -315,13 +311,14 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
for entry in dvc_fs.ls(dvc_path, detail=False): for entry in dvc_fs.ls(dvc_path, detail=False):
names.add(dvc_fs.path.name(entry)) names.add(dvc_fs.path.name(entry))
try: if fs:
for entry in dvcignore.ls( try:
fs, fs_path, detail=False, ignore_subrepos=ignore_subrepos for entry in dvcignore.ls(
): fs, fs_path, detail=False, ignore_subrepos=ignore_subrepos
names.add(fs.path.name(entry)) ):
except (FileNotFoundError, NotADirectoryError): names.add(fs.path.name(entry))
pass except (FileNotFoundError, NotADirectoryError):
pass
dvcfiles = kwargs.get("dvcfiles", False) dvcfiles = kwargs.get("dvcfiles", False)
@@ -354,61 +351,13 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
return infos return infos
def walk(self, top, topdown=True, **kwargs): def get_file(self, rpath, lpath, callback=DEFAULT_CALLBACK, **kwargs):
"""Walk and merge both DVC and repo fss. fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(rpath)
Args:
top: path to walk from
topdown: if True, fs will be walked from top down.
dvcfiles: if True, dvcfiles will be included in the files list
for walked directories.
Any kwargs will be passed into methods used for fetching and/or
streaming DVC outs from remotes.
"""
dirs = []
nondirs = []
ignore_subrepos = kwargs.get("ignore_subrepos", True)
if not self.exists(top, ignore_subrepos=ignore_subrepos):
return
if not self.isdir(top, ignore_subrepos=ignore_subrepos):
return
for entry in self.ls(top, **kwargs):
name = self.path.name(entry["name"])
if entry["type"] == "directory":
dirs.append(name)
else:
nondirs.append(name)
if topdown:
yield top, dirs, nondirs
for dname in dirs:
yield from self.walk(
self.path.join(top, dname), topdown=topdown, **kwargs
)
if not topdown:
yield top, dirs, nondirs
def find(self, path, prefix=None):
for root, _, files in self.walk(path):
for fname in files:
yield self.path.join(root, fname)
def get_file(
self, from_info, to_file, callback=DEFAULT_CALLBACK, **kwargs
):
fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(from_info)
if fs: if fs:
try: try:
fs.get_file( # pylint: disable=protected-access fs.get_file( # pylint: disable=protected-access
fs_path, to_file, callback=callback, **kwargs fs_path, lpath, callback=callback, **kwargs
) )
return return
except FileNotFoundError: except FileNotFoundError:
@@ -416,7 +365,7 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
raise raise
dvc_fs.get_file( # pylint: disable=protected-access dvc_fs.get_file( # pylint: disable=protected-access
dvc_path, to_file, callback=callback, **kwargs dvc_path, lpath, callback=callback, **kwargs
) )
def info(self, path, **kwargs): def info(self, path, **kwargs):
@@ -446,9 +395,9 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
raise raise
# NOTE: if some parent in fs_path turns out to be a file, it means # NOTE: if some parent in fs_path turns out to be a file, it means
# that that whole repofs branch doesn't exist. # that the whole repofs branch doesn't exist.
if fs and not fs_info and dvc_info: if fs and not fs_info and dvc_info:
for parent in self.path.parents(fs_path): for parent in fs.path.parents(fs_path):
try: try:
if fs.info(parent)["type"] != "directory": if fs.info(parent)["type"] != "directory":
dvc_info = None dvc_info = None
@@ -459,7 +408,9 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
if not dvc_info and not fs_info: if not dvc_info and not fs_info:
raise FileNotFoundError raise FileNotFoundError
return _merge_info(dvc_fs.repo, fs_info, dvc_info) info = _merge_info(dvc_fs.repo, fs_info, dvc_info)
info["name"] = path
return info
def checksum(self, path): def checksum(self, path):
fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(path) fs, fs_path, dvc_fs, dvc_path = self._get_fs_pair(path)
@@ -468,3 +419,31 @@ class RepoFileSystem(FileSystem): # pylint:disable=abstract-method
return fs.checksum(fs_path) return fs.checksum(fs_path)
except FileNotFoundError: except FileNotFoundError:
return dvc_fs.checksum(dvc_path) return dvc_fs.checksum(dvc_path)
class RepoFileSystem(FSSpecWrapper):
scheme = "local"
PARAM_CHECKSUM = "md5"
def _prepare_credentials(self, **config):
return config
@wrap_prop(threading.Lock())
@cached_property
def fs(self):
return _RepoFileSystem(**self.fs_args)
def isdvc(self, path, **kwargs):
return self.fs.isdvc(path, **kwargs)
@property
def repo(self):
return self.fs.repo
@property
def repo_url(self):
return self.fs.repo_url
@property
def config(self):
return self.fs.config
+1 -1
View File
@@ -303,7 +303,7 @@ class DvcIgnoreFilter:
for root, _, files in self.walk(fs, path, **kwargs): for root, _, files in self.walk(fs, path, **kwargs):
for file in files: for file in files:
# NOTE: os.path.join is ~5.5 times slower # NOTE: os.path.join is ~5.5 times slower
yield f"{root}{os.sep}{file}" yield f"{root}{fs.sep}{file}"
else: else:
yield from fs.find(path) yield from fs.find(path)
-5
View File
@@ -386,11 +386,6 @@ class Output:
os.path.realpath(self.fs_path), self.repo.root_dir os.path.realpath(self.fs_path), self.repo.root_dir
) )
@property
def repo_path(self):
assert self.is_in_repo
return relpath(self.fs_path, self.repo.root_dir)
@property @property
def use_scm_ignore(self): def use_scm_ignore(self):
if not self.is_in_repo: if not self.is_in_repo:
+4 -2
View File
@@ -482,7 +482,9 @@ class Repo:
def repo_fs(self): def repo_fs(self):
from dvc.fs.repo import RepoFileSystem from dvc.fs.repo import RepoFileSystem
return RepoFileSystem(self, subrepos=self.subrepos, **self._fs_conf) return RepoFileSystem(
repo=self, subrepos=self.subrepos, **self._fs_conf
)
@cached_property @cached_property
def index_db_dir(self): def index_db_dir(self):
@@ -493,7 +495,7 @@ class Repo:
"""Opens a specified resource as a file descriptor""" """Opens a specified resource as a file descriptor"""
from dvc.fs.repo import RepoFileSystem from dvc.fs.repo import RepoFileSystem
fs = RepoFileSystem(self, subrepos=True) fs = RepoFileSystem(repo=self, subrepos=True)
try: try:
with fs.open( with fs.open(
path, mode=mode, encoding=encoding, remote=remote path, mode=mode, encoding=encoding, remote=remote
+13 -15
View File
@@ -34,22 +34,22 @@ def _collect_paths(
from dvc.fs.repo import RepoFileSystem from dvc.fs.repo import RepoFileSystem
from dvc.utils import relpath from dvc.utils import relpath
fs = RepoFileSystem(repo) fs_paths = [os.path.abspath(target) for target in targets]
fs = RepoFileSystem(repo=repo)
target_paths = [] target_paths = []
for target in targets: for fs_path in fs_paths:
if os.path.isabs(target): if recursive and fs.isdir(fs_path):
target = relpath(target, repo.root_dir) target_paths.extend(fs.find(fs_path))
if recursive and fs.isdir(target): rel = relpath(fs_path)
target_paths.extend(repo.dvcignore.find(fs, target)) if not fs.exists(fs_path):
if not fs.exists(target):
if rev == "workspace" or rev == "": if rev == "workspace" or rev == "":
logger.warning("'%s' was not found in current workspace.", target) logger.warning("'%s' was not found in current workspace.", rel)
else: else:
logger.warning("'%s' was not found at: '%s'.", target, rev) logger.warning("'%s' was not found at: '%s'.", rel, rev)
target_paths.append(target) target_paths.append(fs_path)
return target_paths return target_paths
@@ -58,14 +58,12 @@ def _filter_duplicates(
) -> Tuple[Outputs, StrPaths]: ) -> Tuple[Outputs, StrPaths]:
res_outs: Outputs = [] res_outs: Outputs = []
fs_res_paths = fs_paths fs_res_paths = fs_paths
from dvc.utils import relpath
for out in outs: for out in outs:
rel = relpath(out.fs_path, out.stage.repo.root_dir) if out.fs_path in fs_paths:
if rel in fs_paths:
res_outs.append(out) res_outs.append(out)
# MUTATING THE SAME LIST!! # MUTATING THE SAME LIST!!
fs_res_paths.remove(rel) fs_res_paths.remove(out.fs_path)
return res_outs, fs_res_paths return res_outs, fs_res_paths
+4 -3
View File
@@ -24,7 +24,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
from dvc.fs.repo import RepoFileSystem from dvc.fs.repo import RepoFileSystem
repo_fs = RepoFileSystem(self) repo_fs = RepoFileSystem(repo=self)
b_rev = b_rev if b_rev else "workspace" b_rev = b_rev if b_rev else "workspace"
results = {} results = {}
@@ -188,10 +188,11 @@ def _filter_missing(repo_fs, paths):
for path in paths: for path in paths:
try: try:
info = repo_fs.info(path) info = repo_fs.info(path)
dvc_info = info.get("dvc_info")
if ( if (
info["isdvc"] dvc_info
and info["type"] == "directory" and info["type"] == "directory"
and not info["meta"].obj and not dvc_info["meta"].obj
): ):
yield path yield path
except FileNotFoundError: except FileNotFoundError:
+25 -20
View File
@@ -29,9 +29,9 @@ def ls(url, path=None, rev=None, recursive=None, dvc_only=False):
from . import Repo from . import Repo
with Repo.open(url, rev=rev, subrepos=True, uninitialized=True) as repo: with Repo.open(url, rev=rev, subrepos=True, uninitialized=True) as repo:
fs_path = path or "" path = path or ""
ret = _ls(repo, fs_path, recursive, dvc_only) ret = _ls(repo.repo_fs, path, recursive, dvc_only)
if path and not ret: if path and not ret:
raise PathMissingError(path, repo, dvc_only=dvc_only) raise PathMissingError(path, repo, dvc_only=dvc_only)
@@ -44,31 +44,36 @@ def ls(url, path=None, rev=None, recursive=None, dvc_only=False):
return ret_list return ret_list
def _ls(repo, fs_path, recursive=None, dvc_only=False): def _ls(fs, path, recursive=None, dvc_only=False):
fs = repo.repo_fs try:
infos = [] fs_path = fs.info(path)["name"]
except FileNotFoundError:
return {}
infos = {}
for root, dirs, files in fs.walk(fs_path, dvcfiles=True): for root, dirs, files in fs.walk(fs_path, dvcfiles=True):
entries = chain(files, dirs) if not recursive else files entries = chain(files, dirs) if not recursive else files
infos.extend(fs.path.join(root, entry) for entry in entries)
for entry in entries:
entry_fs_path = fs.path.join(root, entry)
relparts = fs.path.relparts(entry_fs_path, fs_path)
name = os.path.join(*relparts)
infos[name] = fs.info(entry_fs_path)
if not recursive: if not recursive:
break break
if not infos and fs.isfile(fs_path): if not infos and fs.isfile(fs_path):
infos.append(fs_path) infos[os.path.basename(path)] = fs.info(fs_path)
ret = {} ret = {}
for info in infos: for name, info in infos.items():
_info = fs.info(info) dvc_info = info.get("dvc_info", {})
if dvc_info.get("outs") or not dvc_only:
if _info.get("outs") or not dvc_only: ret[name] = {
path = ( "isout": dvc_info.get("isout", False),
fs.path.name(fs_path) "isdir": info["type"] == "directory",
if fs_path == info "isexec": info.get("isexec", False),
else fs.path.relpath(info, fs_path)
)
ret[path] = {
"isout": _info.get("isout", False),
"isdir": _info["type"] == "directory",
"isexec": _info["isexec"],
} }
return ret return ret
+4 -5
View File
@@ -76,16 +76,15 @@ def _read_metric(path, fs, rev, **kwargs):
def _read_metrics(repo, metrics, rev, onerror=None): def _read_metrics(repo, metrics, rev, onerror=None):
fs = RepoFileSystem(repo) fs = RepoFileSystem(repo=repo)
res = {} res = {}
for metric in metrics: for metric in metrics:
fs_path = repo.fs.path.relpath(metric, repo.root_dir) if not fs.isfile(metric):
if not fs.isfile(fs_path):
continue continue
res[fs.path.relpath(metric, os.getcwd())] = _read_metric( res[os.path.relpath(metric, os.getcwd())] = _read_metric(
fs_path, fs, rev, onerror=onerror metric, fs, rev, onerror=onerror
) )
return res return res
+4 -4
View File
@@ -106,7 +106,7 @@ class Plots:
): ):
from dvc.fs.repo import RepoFileSystem from dvc.fs.repo import RepoFileSystem
fs = RepoFileSystem(self.repo) fs = RepoFileSystem(repo=self.repo)
plots = _collect_plots(self.repo, targets, revision, recursive) plots = _collect_plots(self.repo, targets, revision, recursive)
res: Dict[str, Any] = {} res: Dict[str, Any] = {}
for fs_path, rev_props in plots.items(): for fs_path, rev_props in plots.items():
@@ -240,7 +240,7 @@ def _collect_plots(
targets: List[str] = None, targets: List[str] = None,
rev: str = None, rev: str = None,
recursive: bool = False, recursive: bool = False,
) -> Dict["List[str]", Dict]: ) -> Dict[str, Dict]:
from dvc.repo.collect import collect from dvc.repo.collect import collect
plots, fs_paths = collect( plots, fs_paths = collect(
@@ -251,8 +251,8 @@ def _collect_plots(
recursive=recursive, recursive=recursive,
) )
result = {relpath(plot.fs_path, repo.root_dir): _plot_props(plot) for plot in plots} result = {plot.fs_path: _plot_props(plot) for plot in plots}
result.update({relpath(fs_path, repo.root_dir): {} for fs_path in fs_paths}) result.update({fs_path: {} for fs_path in fs_paths})
return result return result
+1 -1
View File
@@ -63,7 +63,7 @@ def _get_stage_files(stage: "Stage") -> typing.Iterator[str]:
if ( if (
not dep.use_scm_ignore not dep.use_scm_ignore
and dep.is_in_repo and dep.is_in_repo
and not stage.repo.repo_fs.isdvc(dep.fs.path.relpath(dep.fs_path, stage.repo.root_dir)) and not stage.repo.repo_fs.isdvc(dep.fs_path)
): ):
yield dep.fs_path yield dep.fs_path
for out in stage.outs: for out in stage.outs:
+7
View File
@@ -332,6 +332,13 @@ def relpath(path, start=os.curdir):
return os.path.relpath(path, start) return os.path.relpath(path, start)
def as_posix(path):
import ntpath
import posixpath
return path.replace(ntpath.sep, posixpath.sep)
def env2bool(var, undefined=False): def env2bool(var, undefined=False):
""" """
undefined: return value if env var is unset undefined: return value if env var is unset
+1 -1
View File
@@ -96,7 +96,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
dest = tmp_dir / "file" dest = tmp_dir / "file"
with external_repo(os.fspath(erepo_dir)) as repo: with external_repo(os.fspath(erepo_dir)) as repo:
repo.repo_fs.download( repo.repo_fs.download(
os.path.join("subdir", "file"), "subdir/file",
os.fspath(dest), os.fspath(dest),
) )
+103 -101
View File
@@ -1,4 +1,5 @@
import os import os
import posixpath
import shutil import shutil
from unittest import mock from unittest import mock
@@ -116,8 +117,8 @@ def test_isdir_isfile(tmp_dir, dvc):
assert fs.isdir("subdir") assert fs.isdir("subdir")
assert not fs.isfile("subdir") assert not fs.isfile("subdir")
assert not fs.isdvc("subdir") assert not fs.isdvc("subdir")
assert fs.isfile(os.path.join("subdir", "baz")) assert fs.isfile("subdir/baz")
assert fs.isdir(os.path.join("subdir", "data")) assert fs.isdir("subdir/data")
def test_exists_isdir_isfile_dirty(tmp_dir, dvc): def test_exists_isdir_isfile_dirty(tmp_dir, dvc):
@@ -129,31 +130,30 @@ def test_exists_isdir_isfile_dirty(tmp_dir, dvc):
shutil.rmtree(tmp_dir / "datadir") shutil.rmtree(tmp_dir / "datadir")
(tmp_dir / "datafile").unlink() (tmp_dir / "datafile").unlink()
root = tmp_dir
assert fs.exists("datafile") assert fs.exists("datafile")
assert fs.exists("datadir") assert fs.exists("datadir")
assert fs.exists(os.path.join("datadir", "foo")) assert fs.exists("datadir/foo")
assert fs.isfile("datafile") assert fs.isfile("datafile")
assert not fs.isfile("datadir") assert not fs.isfile("datadir")
assert fs.isfile(os.path.join("datadir", "foo")) assert fs.isfile("datadir/foo")
assert not fs.isdir("datafile") assert not fs.isdir("datafile")
assert fs.isdir("datadir") assert fs.isdir("datadir")
assert not fs.isdir(os.path.join("datadir", "foo")) assert not fs.isdir("datadir/foo")
# NOTE: creating file instead of dir and dir instead of file # NOTE: creating file instead of dir and dir instead of file
tmp_dir.gen({"datadir": "data", "datafile": {"foo": "foo", "bar": "bar"}}) tmp_dir.gen({"datadir": "data", "datafile": {"foo": "foo", "bar": "bar"}})
assert fs.exists("datafile") assert fs.exists("datafile")
assert fs.exists("datadir") assert fs.exists("datadir")
assert not fs.exists(os.path.join("datadir", "foo")) assert not fs.exists("datadir/foo")
assert fs.exists(os.path.join("datafile", "foo")) assert fs.exists("datafile/foo")
assert not fs.isfile("datafile") assert not fs.isfile("datafile")
assert fs.isfile("datadir") assert fs.isfile("datadir")
assert not fs.isfile(os.path.join("datadir", "foo")) assert not fs.isfile("datadir/foo")
assert fs.isfile(os.path.join("datafile", "foo")) assert fs.isfile("datafile/foo")
assert fs.isdir("datafile") assert fs.isdir("datafile")
assert not fs.isdir("datadir") assert not fs.isdir("datadir")
assert not fs.isdir(os.path.join("datadir", "foo")) assert not fs.isdir("datadir/foo")
assert not fs.isdir(os.path.join("datafile", "foo")) assert not fs.isdir("datafile/foo")
def test_isdir_mixed(tmp_dir, dvc): def test_isdir_mixed(tmp_dir, dvc):
@@ -173,9 +173,9 @@ def test_isdir_mixed(tmp_dir, dvc):
( (
True, True,
[ [
os.path.join("dir", "subdir1", "foo1.dvc"), "dir/subdir1/foo1.dvc",
os.path.join("dir", "subdir1", "bar1.dvc"), "dir/subdir1/bar1.dvc",
os.path.join("dir", "subdir2", "foo2.dvc"), "dir/subdir2/foo2.dvc",
], ],
), ),
], ],
@@ -194,21 +194,21 @@ def test_walk(tmp_dir, dvc, dvcfiles, extra_expected):
fs = RepoFileSystem(repo=dvc) fs = RepoFileSystem(repo=dvc)
expected = [ expected = [
os.path.join("dir", "subdir1"), "dir/subdir1",
os.path.join("dir", "subdir2"), "dir/subdir2",
os.path.join("dir", "subdir1", "foo1"), "dir/subdir1/foo1",
os.path.join("dir", "subdir1", "bar1"), "dir/subdir1/bar1",
os.path.join("dir", "subdir2", "foo2"), "dir/subdir2/foo2",
os.path.join("dir", "foo"), "dir/foo",
os.path.join("dir", "bar"), "dir/bar",
] ]
actual = [] actual = []
for root, dirs, files in fs.walk("dir", dvcfiles=dvcfiles): for root, dirs, files in fs.walk("dir", dvcfiles=dvcfiles):
for entry in dirs + files: for entry in dirs + files:
actual.append(os.path.join(root, entry)) actual.append(posixpath.join(root, entry))
expected += [os.path.join(path) for path in extra_expected] expected += extra_expected
assert set(actual) == set(expected) assert set(actual) == set(expected)
assert len(actual) == len(expected) assert len(actual) == len(expected)
@@ -228,21 +228,21 @@ def test_walk_dirty(tmp_dir, dvc):
fs = RepoFileSystem(repo=dvc) fs = RepoFileSystem(repo=dvc)
expected = [ expected = [
os.path.join("dir", "subdir1"), "dir/subdir1",
os.path.join("dir", "subdir2"), "dir/subdir2",
os.path.join("dir", "subdir3"), "dir/subdir3",
os.path.join("dir", "subdir1", "foo1"), "dir/subdir1/foo1",
os.path.join("dir", "subdir1", "bar1"), "dir/subdir1/bar1",
os.path.join("dir", "subdir2", "foo2"), "dir/subdir2/foo2",
os.path.join("dir", "subdir3", "foo3"), "dir/subdir3/foo3",
os.path.join("dir", "bar"), "dir/bar",
os.path.join("dir", "foo"), "dir/foo",
] ]
actual = [] actual = []
for root, dirs, files in fs.walk("dir"): for root, dirs, files in fs.walk("dir"):
for entry in dirs + files: for entry in dirs + files:
actual.append(os.path.join(root, entry)) actual.append(posixpath.join(root, entry))
assert set(actual) == set(expected) assert set(actual) == set(expected)
assert len(actual) == len(expected) assert len(actual) == len(expected)
@@ -254,14 +254,12 @@ def test_walk_dirty_cached_dir(tmp_dir, scm, dvc):
fs = RepoFileSystem(repo=dvc) fs = RepoFileSystem(repo=dvc)
data = tmp_dir / "data"
actual = [] actual = []
for root, dirs, files in fs.walk("data"): for root, dirs, files in fs.walk("data"):
for entry in dirs + files: for entry in dirs + files:
actual.append(os.path.join(root, entry)) actual.append(posixpath.join(root, entry))
expected = [os.path.join("data", "foo"), os.path.join("data", "bar")] expected = ["data/foo", "data/bar"]
assert set(actual) == set(expected) assert set(actual) == set(expected)
assert len(actual) == len(expected) assert len(actual) == len(expected)
@@ -281,14 +279,14 @@ def test_walk_mixed_dir(tmp_dir, scm, dvc):
fs = RepoFileSystem(repo=dvc) fs = RepoFileSystem(repo=dvc)
expected = [ expected = [
os.path.join("dir", "foo"), "dir/foo",
os.path.join("dir", "bar"), "dir/bar",
os.path.join("dir", ".gitignore"), "dir/.gitignore",
] ]
actual = [] actual = []
for root, dirs, files in fs.walk("dir"): for root, dirs, files in fs.walk("dir"):
for entry in dirs + files: for entry in dirs + files:
actual.append(os.path.join(root, entry)) actual.append(posixpath.join(root, entry))
assert set(actual) == set(expected) assert set(actual) == set(expected)
assert len(actual) == len(expected) assert len(actual) == len(expected)
@@ -317,8 +315,8 @@ def test_isdvc(tmp_dir, dvc):
assert fs.isdvc("foo") assert fs.isdvc("foo")
assert not fs.isdvc("bar") assert not fs.isdvc("bar")
assert fs.isdvc("dir") assert fs.isdvc("dir")
assert fs.isdvc(os.path.join("dir", "baz")) assert fs.isdvc("dir/baz")
assert fs.isdvc(os.path.join("dir", "baz"), recursive=True) assert fs.isdvc("dir/baz", recursive=True)
def make_subrepo(dir_, scm, config=None): def make_subrepo(dir_, scm, config=None):
@@ -353,7 +351,7 @@ def test_subrepos(tmp_dir, scm, dvc, mocker):
fs = RepoFileSystem(repo=dvc, subrepos=True) fs = RepoFileSystem(repo=dvc, subrepos=True)
def assert_fs_belongs_to_repo(ret_val): def assert_fs_belongs_to_repo(ret_val):
method = fs._get_repo method = fs.fs._get_repo
def f(*args, **kwargs): def f(*args, **kwargs):
r = method(*args, **kwargs) r = method(*args, **kwargs)
@@ -363,32 +361,32 @@ def test_subrepos(tmp_dir, scm, dvc, mocker):
return f return f
with mock.patch.object( with mock.patch.object(
fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo1.dvc) fs.fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo1.dvc)
): ):
assert fs.exists(os.path.join("dir", "repo", "foo")) is True assert fs.exists("dir/repo/foo") is True
assert fs.exists(os.path.join("dir", "repo", "bar")) is False assert fs.exists("dir/repo/bar") is False
assert fs.isfile(os.path.join("dir", "repo", "foo")) is True assert fs.isfile("dir/repo/foo") is True
assert fs.isfile(os.path.join("dir", "repo", "dir1", "bar")) is True assert fs.isfile("dir/repo/dir1/bar") is True
assert fs.isfile(os.path.join("dir", "repo", "dir1")) is False assert fs.isfile("dir/repo/dir1") is False
assert fs.isdir(os.path.join("dir", "repo", "dir1")) is True assert fs.isdir("dir/repo/dir1") is True
assert fs.isdir(os.path.join("dir", "repo", "dir1", "bar")) is False assert fs.isdir("dir/repo/dir1/bar") is False
assert fs.isdvc(os.path.join("dir", "repo", "foo")) is True assert fs.isdvc("dir/repo/foo") is True
with mock.patch.object( with mock.patch.object(
fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo2.dvc) fs.fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo2.dvc)
): ):
assert fs.exists(os.path.join("dir", "repo2", "lorem")) is True assert fs.exists("dir/repo2/lorem") is True
assert fs.exists(os.path.join("dir", "repo2", "ipsum")) is False assert fs.exists("dir/repo2/ipsum") is False
assert fs.isfile(os.path.join("dir", "repo2", "lorem")) is True assert fs.isfile("dir/repo2/lorem") is True
assert fs.isfile(os.path.join("dir", "repo2", "dir2", "ipsum")) is True assert fs.isfile("dir/repo2/dir2/ipsum") is True
assert fs.isfile(os.path.join("dir", "repo2", "dir2")) is False assert fs.isfile("dir/repo2/dir2") is False
assert fs.isdir(os.path.join("dir", "repo2", "dir2")) is True assert fs.isdir("dir/repo2/dir2") is True
assert fs.isdir(os.path.join("dir", "repo2", "dir2", "ipsum")) is False assert fs.isdir("dir/repo2/dir2/ipsum") is False
assert fs.isdvc(os.path.join("dir", "repo2", "lorem")) is True assert fs.isdvc("dir/repo2/lorem") is True
@pytest.mark.parametrize( @pytest.mark.parametrize(
@@ -398,12 +396,12 @@ def test_subrepos(tmp_dir, scm, dvc, mocker):
( (
True, True,
[ [
os.path.join("dir", "repo", "foo.dvc"), "dir/repo/foo.dvc",
os.path.join("dir", "repo", ".dvcignore"), "dir/repo/.dvcignore",
os.path.join("dir", "repo", "dir1.dvc"), "dir/repo/dir1.dvc",
os.path.join("dir", "repo2", ".dvcignore"), "dir/repo2/.dvcignore",
os.path.join("dir", "repo2", "lorem.dvc"), "dir/repo2/lorem.dvc",
os.path.join("dir", "repo2", "dir2.dvc"), "dir/repo2/dir2.dvc",
], ],
), ),
], ],
@@ -432,17 +430,17 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
dvc._reset() dvc._reset()
fs = RepoFileSystem(repo=dvc) fs = RepoFileSystem(repo=dvc)
expected = [ expected = [
os.path.join("dir", "repo"), "dir/repo",
os.path.join("dir", "repo.txt"), "dir/repo.txt",
os.path.join("dir", "repo2"), "dir/repo2",
os.path.join("dir", "repo", ".gitignore"), "dir/repo/.gitignore",
os.path.join("dir", "repo", "foo"), "dir/repo/foo",
os.path.join("dir", "repo", "dir1"), "dir/repo/dir1",
os.path.join("dir", "repo", "dir1", "bar"), "dir/repo/dir1/bar",
os.path.join("dir", "repo2", ".gitignore"), "dir/repo2/.gitignore",
os.path.join("dir", "repo2", "lorem"), "dir/repo2/lorem",
os.path.join("dir", "repo2", "dir2"), "dir/repo2/dir2",
os.path.join("dir", "repo2", "dir2", "ipsum"), "dir/repo2/dir2/ipsum",
] ]
actual = [] actual = []
@@ -452,7 +450,7 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
ignore_subrepos=False, ignore_subrepos=False,
): ):
for entry in dirs + files: for entry in dirs + files:
actual.append(os.path.join(root, entry)) actual.append(posixpath.join(root, entry))
expected += extra_expected expected += extra_expected
assert set(actual) == set(expected) assert set(actual) == set(expected)
@@ -481,27 +479,27 @@ def test_repo_fs_no_subrepos(tmp_dir, dvc, scm):
"lorem", "lorem",
"lorem.dvc", "lorem.dvc",
"dir", "dir",
os.path.join("dir", "repo.txt"), "dir/repo.txt",
] ]
actual = [] actual = []
for root, dirs, files in fs.walk("", dvcfiles=True): for root, dirs, files in fs.walk("", dvcfiles=True):
for entry in dirs + files: for entry in dirs + files:
actual.append(os.path.join(root, entry)) actual.append(posixpath.join(root, entry))
assert set(actual) == set(expected) assert set(actual) == set(expected)
assert len(actual) == len(expected) assert len(actual) == len(expected)
assert fs.isfile("lorem") is True assert fs.isfile("lorem") is True
assert fs.isfile(os.path.join("dir", "repo", "foo")) is False assert fs.isfile("dir/repo/foo") is False
assert fs.isdir(os.path.join("dir", "repo")) is False assert fs.isdir("dir/repo") is False
assert fs.isdir("dir") is True assert fs.isdir("dir") is True
assert fs.isdvc("lorem") is True assert fs.isdvc("lorem") is True
assert fs.isdvc(os.path.join("dir", "repo", "dir1")) is False assert fs.isdvc("dir/repo/dir1") is False
assert fs.exists(os.path.join("dir", "repo.txt")) is True assert fs.exists("dir/repo.txt") is True
assert fs.exists(os.path.join("repo", "ipsum")) is False assert fs.exists("repo/ipsum") is False
def test_get_hash_cached_file(tmp_dir, dvc, mocker): def test_get_hash_cached_file(tmp_dir, dvc, mocker):
@@ -540,18 +538,18 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
) )
fs = RepoFileSystem(repo=dvc) fs = RepoFileSystem(repo=dvc)
subdir = os.path.join("dir", "subdir") subdir = "dir/subdir"
assert fs.info(subdir).get("md5") is None assert fs.info(subdir).get("md5") is None
_, _, obj = stage(dvc.odb.local, subdir, fs, "md5") _, _, obj = stage(dvc.odb.local, subdir, fs, "md5")
assert obj.hash_info == HashInfo( assert obj.hash_info == HashInfo(
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir" "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir"
) )
assert fs.info(os.path.join(subdir, "data")).get("md5") is None assert fs.info(posixpath.join(subdir, "data")).get("md5") is None
_, _, obj = stage(dvc.odb.local, os.path.join(subdir, "data"), fs, "md5") _, _, obj = stage(dvc.odb.local, posixpath.join(subdir, "data"), fs, "md5")
assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc") assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc")
(tmp_dir / "dir" / "subdir" / "data").unlink() (tmp_dir / "dir" / "subdir" / "data").unlink()
assert ( assert (
fs.info(os.path.join(subdir, "data"))["md5"] fs.info(posixpath.join(subdir, "data"))["md5"]
== "8d777f385d3dfec8815d20f7496026dc" == "8d777f385d3dfec8815d20f7496026dc"
) )
@@ -593,9 +591,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
# get_file_hash(file) should return workspace hash, not DVC cached hash # get_file_hash(file) should return workspace hash, not DVC cached hash
fs = RepoFileSystem(repo=dvc) fs = RepoFileSystem(repo=dvc)
assert fs.info("file").get("md5") is None assert fs.info("file").get("md5") is None
staging, _, obj = stage( staging, _, obj = stage(dvc.odb.local, "file", fs, "md5")
dvc.odb.local, "file", fs, "md5"
)
assert obj.hash_info == something_hash_info assert obj.hash_info == something_hash_info
check(staging, obj) check(staging, obj)
@@ -607,9 +603,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
# get_file_hash(file) should return DVC cached hash # get_file_hash(file) should return DVC cached hash
assert fs.info("file")["md5"] == file_hash_info.value assert fs.info("file")["md5"] == file_hash_info.value
_, hash_info = get_file_hash( _, hash_info = get_file_hash("file", fs, "md5", state=dvc.state)
"file", fs, "md5", state=dvc.state
)
assert hash_info == file_hash_info assert hash_info == file_hash_info
# tmp_dir/file can be staged even though it is missing in workspace since # tmp_dir/file can be staged even though it is missing in workspace since
@@ -662,14 +656,22 @@ def test_walk_nested_subrepos(tmp_dir, dvc, scm, traverse_subrepos):
repo_dir.dvc_gen(dvc_files, commit=f"dvc add in {repo_dir}") repo_dir.dvc_gen(dvc_files, commit=f"dvc add in {repo_dir}")
if traverse_subrepos or repo_dir == tmp_dir: if traverse_subrepos or repo_dir == tmp_dir:
repo_dir_path = repo_dir.relative_to(tmp_dir).fs_path if repo_dir != tmp_dir else "" repo_dir_path = (
repo_dir.relative_to(tmp_dir).as_posix()
if repo_dir != tmp_dir
else ""
)
expected[repo_dir_path] = set( expected[repo_dir_path] = set(
scm_files.keys() | dvc_files.keys() | extras scm_files.keys() | dvc_files.keys() | extras
) )
# files inside a dvc directory # files inside a dvc directory
expected[os.path.join(repo_dir_path, f"dvc-{base}")] = {f"ipsum-{base}"} expected[posixpath.join(repo_dir_path, f"dvc-{base}")] = {
f"ipsum-{base}"
}
# files inside a git directory # files inside a git directory
expected[os.path.join(repo_dir_path, f"dir-{base}")] = {f"bar-{base}"} expected[posixpath.join(repo_dir_path, f"dir-{base}")] = {
f"bar-{base}"
}
if traverse_subrepos: if traverse_subrepos:
# update subrepos # update subrepos
+26 -28
View File
@@ -38,31 +38,29 @@ def repo_fs(tmp_dir, dvc, scm):
tmp_dir.scm_gen(fs_structure, commit="repo init") tmp_dir.scm_gen(fs_structure, commit="repo init")
tmp_dir.dvc_gen(dvc_structure, commit="use dvc") tmp_dir.dvc_gen(dvc_structure, commit="use dvc")
yield RepoFileSystem(dvc, subrepos=True) yield RepoFileSystem(repo=dvc, subrepos=True)
def test_info_not_existing(repo_fs): def test_info_not_existing(repo_fs):
path = os.path.join("path", "that", "does", "not", "exist")
with pytest.raises(FileNotFoundError): with pytest.raises(FileNotFoundError):
repo_fs.info(path) repo_fs.info("path/that/does/not/exist")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"path", "path",
[ [
"README.md", "README.md",
os.path.join("models", "train.py"), "models/train.py",
os.path.join("models", "test.py"), "models/test.py",
os.path.join("src", "utils", "__init__.py"), "src/utils/__init__.py",
os.path.join("src", "utils", "serve_model.py"), "src/utils/serve_model.py",
], ],
) )
def test_info_git_tracked_file(repo_fs, path): def test_info_git_tracked_file(repo_fs, path):
info = repo_fs.info(path) info = repo_fs.info(path)
assert info["repo"].root_dir == repo_fs._root_dir assert info["repo"].root_dir == repo_fs.repo.root_dir
assert not info["isdvc"] assert "dvc_info" not in info
assert info["type"] == "file" assert info["type"] == "file"
assert not info["isexec"] assert not info["isexec"]
@@ -70,28 +68,28 @@ def test_info_git_tracked_file(repo_fs, path):
@pytest.mark.parametrize( @pytest.mark.parametrize(
"path", "path",
[ [
os.path.join("data", "raw", "raw-1.csv"), "data/raw/raw-1.csv",
os.path.join("data", "raw", "raw-2.csv"), "data/raw/raw-2.csv",
os.path.join("data", "processed", "processed-1.csv"), "data/processed/processed-1.csv",
os.path.join("data", "processed", "processed-2.csv"), "data/processed/processed-2.csv",
os.path.join("models", "transform.pickle"), "models/transform.pickle",
], ],
) )
def test_info_dvc_tracked_file(repo_fs, path): def test_info_dvc_tracked_file(repo_fs, path):
info = repo_fs.info(path) info = repo_fs.info(path)
assert info["repo"].root_dir == repo_fs._root_dir assert info["repo"].root_dir == repo_fs.repo.root_dir
assert info["isdvc"] assert info["dvc_info"]["isdvc"]
assert info["type"] == "file" assert info["type"] == "file"
assert not info["isexec"] assert not info["isexec"]
@pytest.mark.parametrize("path", ["src", os.path.join("src", "utils")]) @pytest.mark.parametrize("path", ["src", "src/utils"])
def test_info_git_only_dirs(repo_fs, path): def test_info_git_only_dirs(repo_fs, path):
info = repo_fs.info(path) info = repo_fs.info(path)
assert info["repo"].root_dir == repo_fs._root_dir assert info["repo"].root_dir == repo_fs.repo.root_dir
assert not info["isdvc"] assert "dvc_info" not in info
assert info["type"] == "directory" assert info["type"] == "directory"
assert not info["isexec"] assert not info["isexec"]
@@ -100,8 +98,8 @@ def test_info_git_only_dirs(repo_fs, path):
def test_info_git_dvc_mixed_dirs(repo_fs, path): def test_info_git_dvc_mixed_dirs(repo_fs, path):
info = repo_fs.info(path) info = repo_fs.info(path)
assert info["repo"].root_dir == repo_fs._root_dir assert info["repo"].root_dir == repo_fs.repo.root_dir
assert not info["isdvc"] assert not info["dvc_info"]["isdvc"]
assert info["type"] == "directory" assert info["type"] == "directory"
assert not info["isexec"] assert not info["isexec"]
@@ -110,15 +108,15 @@ def test_info_git_dvc_mixed_dirs(repo_fs, path):
"path", "path",
[ [
"data", "data",
os.path.join("data", "raw"), "data/raw",
os.path.join("data", "processed"), "data/processed",
], ],
) )
def test_info_dvc_only_dirs(repo_fs, path): def test_info_dvc_only_dirs(repo_fs, path):
info = repo_fs.info(path) info = repo_fs.info(path)
assert info["repo"].root_dir == repo_fs._root_dir assert info["repo"].root_dir == repo_fs.repo.root_dir
assert info["isdvc"] assert info["dvc_info"]["isdvc"]
assert info["type"] == "directory" assert info["type"] == "directory"
assert not info["isexec"] assert not info["isexec"]
@@ -132,8 +130,8 @@ def test_info_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, repo_fs):
for path in [ for path in [
"subrepo", "subrepo",
os.path.join("subrepo", "foo"), "subrepo/foo",
os.path.join("subrepo", "foobar"), "subrepo/foobar",
]: ]:
info = repo_fs.info(path) info = repo_fs.info(path)
assert info["repo"].root_dir == str( assert info["repo"].root_dir == str(
+3 -3
View File
@@ -26,7 +26,7 @@ def test_hook_is_called(tmp_dir, erepo_dir, mocker):
repo.dvc_gen("bar", "bar", commit=f"dvc add {repo}/bar") repo.dvc_gen("bar", "bar", commit=f"dvc add {repo}/bar")
with external_repo(str(erepo_dir)) as repo: with external_repo(str(erepo_dir)) as repo:
spy = mocker.spy(repo.repo_fs, "repo_factory") spy = mocker.spy(repo.repo_fs.fs, "repo_factory")
list(repo.repo_fs.walk("", ignore_subrepos=False)) # drain list(repo.repo_fs.walk("", ignore_subrepos=False)) # drain
assert spy.call_count == len(subrepos) assert spy.call_count == len(subrepos)
@@ -37,7 +37,7 @@ def test_hook_is_called(tmp_dir, erepo_dir, mocker):
call( call(
path, path,
fs=repo.fs, fs=repo.fs,
repo_factory=repo.repo_fs.repo_factory, repo_factory=repo.repo_fs.fs.repo_factory,
) )
for path in paths for path in paths
], ],
@@ -63,7 +63,7 @@ def test_subrepo_is_constructed_properly(
with external_repo( with external_repo(
str(tmp_dir), cache_dir=str(cache_dir), cache_types=["symlink"] str(tmp_dir), cache_dir=str(cache_dir), cache_types=["symlink"]
) as repo: ) as repo:
spy = mocker.spy(repo.repo_fs, "repo_factory") spy = mocker.spy(repo.repo_fs.fs, "repo_factory")
list(repo.repo_fs.walk("", ignore_subrepos=False)) # drain list(repo.repo_fs.walk("", ignore_subrepos=False)) # drain
assert spy.call_count == 1 assert spy.call_count == 1