dvc: rename tree -> fs (#5461)

This commit is contained in:
Ruslan Kuprieiev
2021-02-13 15:40:44 +02:00
committed by GitHub
parent 9ce6bce9b6
commit b0fd40f12c
147 changed files with 1835 additions and 1857 deletions
+2 -2
View File
@@ -23,13 +23,13 @@ def get_url(path, repo=None, rev=None, remote=None):
with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
path_info = PathInfo(_repo.root_dir) / path
with reraise(FileNotFoundError, PathMissingError(path, repo)):
metadata = _repo.repo_tree.metadata(path_info)
metadata = _repo.repo_fs.metadata(path_info)
if not metadata.is_dvc:
raise OutputNotFoundError(path, repo)
cloud = metadata.repo.cloud
hash_info = get_hash(path_info, _repo.repo_tree, "md5")
hash_info = get_hash(path_info, _repo.repo_fs, "md5")
return cloud.get_url_for(remote, checksum=hash_info.value)
+11 -11
View File
@@ -4,32 +4,32 @@ from collections import defaultdict
from ..scheme import Schemes
def get_cloud_cache(tree):
def get_cloud_cache(fs):
from .base import CloudCache
from .gdrive import GDriveCache
from .local import LocalCache
from .ssh import SSHCache
if tree.scheme == Schemes.LOCAL:
return LocalCache(tree)
if fs.scheme == Schemes.LOCAL:
return LocalCache(fs)
if tree.scheme == Schemes.SSH:
return SSHCache(tree)
if fs.scheme == Schemes.SSH:
return SSHCache(fs)
if tree.scheme == Schemes.GDRIVE:
return GDriveCache(tree)
if fs.scheme == Schemes.GDRIVE:
return GDriveCache(fs)
return CloudCache(tree)
return CloudCache(fs)
def _get_cache(repo, settings):
from ..tree import get_cloud_tree
from ..fs import get_cloud_fs
if not settings:
return None
tree = get_cloud_tree(repo, **settings)
return get_cloud_cache(tree)
fs = get_cloud_fs(repo, **settings)
return get_cloud_cache(fs)
class Cache:
+51 -53
View File
@@ -17,32 +17,32 @@ class CloudCache:
DEFAULT_CACHE_TYPES = ["copy"]
CACHE_MODE: Optional[int] = None
def __init__(self, tree):
self.tree = tree
self.repo = tree.repo
def __init__(self, fs):
self.fs = fs
self.repo = fs.repo
self.verify = tree.config.get("verify", self.DEFAULT_VERIFY)
self.cache_types = tree.config.get("type") or copy(
self.verify = fs.config.get("verify", self.DEFAULT_VERIFY)
self.cache_types = fs.config.get("type") or copy(
self.DEFAULT_CACHE_TYPES
)
self.cache_type_confirmed = False
def move(self, from_info, to_info):
self.tree.move(from_info, to_info)
self.fs.move(from_info, to_info)
def makedirs(self, path_info):
self.tree.makedirs(path_info)
self.fs.makedirs(path_info)
def get(self, hash_info):
""" get raw object """
return HashFile(
# Prefer string path over PathInfo when possible due to performance
self.hash_to_path(hash_info.value),
self.tree,
self.fs,
hash_info,
)
def add(self, path_info, tree, hash_info, **kwargs):
def add(self, path_info, fs, hash_info, **kwargs):
try:
self.check(hash_info)
return
@@ -52,21 +52,21 @@ class CloudCache:
cache_info = self.hash_to_path_info(hash_info.value)
# using our makedirs to create dirs with proper permissions
self.makedirs(cache_info.parent)
if isinstance(tree, type(self.tree)):
self.tree.move(path_info, cache_info)
if isinstance(fs, type(self.fs)):
self.fs.move(path_info, cache_info)
else:
with tree.open(path_info, mode="rb") as fobj:
self.tree.upload_fobj(fobj, cache_info)
with fs.open(path_info, mode="rb") as fobj:
self.fs.upload_fobj(fobj, cache_info)
self.protect(cache_info)
with self.tree.state:
self.tree.state.save(cache_info, hash_info)
with self.fs.state:
self.fs.state.save(cache_info, hash_info)
callback = kwargs.get("download_callback")
if callback:
callback(1)
def hash_to_path_info(self, hash_):
return self.tree.path_info / hash_[0:2] / hash_[2:]
return self.fs.path_info / hash_[0:2] / hash_[2:]
# Override to return path as a string instead of PathInfo for clouds
# which support string paths (see local)
@@ -110,7 +110,7 @@ class CloudCache:
obj.check(self)
except ObjectFormatError:
logger.warning("corrupted cache file '%s'.", obj.path_info)
self.tree.remove(obj.path_info)
self.fs.remove(obj.path_info)
raise
# making cache file read-only so we don't need to check it
@@ -120,22 +120,22 @@ class CloudCache:
def _list_paths(self, prefix=None, progress_callback=None):
if prefix:
if len(prefix) > 2:
path_info = self.tree.path_info / prefix[:2] / prefix[2:]
path_info = self.fs.path_info / prefix[:2] / prefix[2:]
else:
path_info = self.tree.path_info / prefix[:2]
path_info = self.fs.path_info / prefix[:2]
prefix = True
else:
path_info = self.tree.path_info
path_info = self.fs.path_info
prefix = False
if progress_callback:
for file_info in self.tree.walk_files(path_info, prefix=prefix):
for file_info in self.fs.walk_files(path_info, prefix=prefix):
progress_callback()
yield file_info.path
else:
yield from self.tree.walk_files(path_info, prefix=prefix)
yield from self.fs.walk_files(path_info, prefix=prefix)
def _path_to_hash(self, path):
parts = self.tree.PATH_CLS(path).parts[-2:]
parts = self.fs.PATH_CLS(path).parts[-2:]
if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
raise ValueError(f"Bad cache file path '{path}'")
@@ -143,7 +143,7 @@ class CloudCache:
return "".join(parts)
def list_hashes(self, prefix=None, progress_callback=None):
"""Iterate over hashes in this tree.
"""Iterate over hashes in this fs.
If `prefix` is specified, only hashes which begin with `prefix`
will be returned.
@@ -171,18 +171,18 @@ class CloudCache:
def _max_estimation_size(self, hashes):
# Max remote size allowed for us to use traverse method
return max(
self.tree.TRAVERSE_THRESHOLD_SIZE,
self.fs.TRAVERSE_THRESHOLD_SIZE,
len(hashes)
/ self.tree.TRAVERSE_WEIGHT_MULTIPLIER
* self.tree.LIST_OBJECT_PAGE_SIZE,
/ self.fs.TRAVERSE_WEIGHT_MULTIPLIER
* self.fs.LIST_OBJECT_PAGE_SIZE,
)
def _estimate_remote_size(self, hashes=None, name=None):
"""Estimate tree size based on number of entries beginning with
"""Estimate fs size based on number of entries beginning with
"00..." prefix.
"""
prefix = "0" * self.tree.TRAVERSE_PREFIX_LEN
total_prefixes = pow(16, self.tree.TRAVERSE_PREFIX_LEN)
prefix = "0" * self.fs.TRAVERSE_PREFIX_LEN
total_prefixes = pow(16, self.fs.TRAVERSE_PREFIX_LEN)
if hashes:
max_hashes = self._max_estimation_size(hashes)
else:
@@ -215,7 +215,7 @@ class CloudCache:
def list_hashes_traverse(
self, remote_size, remote_hashes, jobs=None, name=None
):
"""Iterate over all hashes found in this tree.
"""Iterate over all hashes found in this fs.
Hashes are fetched in parallel according to prefix, except in
cases where the remote size is very small.
@@ -229,8 +229,8 @@ class CloudCache:
sense to use a generator to gradually iterate over it, without
keeping all of it in memory.
"""
num_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
if num_pages < 256 / self.tree.JOBS:
num_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE
if num_pages < 256 / self.fs.JOBS:
# Fetching prefixes in parallel requires at least 255 more
# requests, for small enough remotes it will be faster to fetch
# entire cache without splitting it into prefixes.
@@ -243,12 +243,10 @@ class CloudCache:
yield from remote_hashes
initial = len(remote_hashes)
traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
if self.tree.TRAVERSE_PREFIX_LEN > 2:
if self.fs.TRAVERSE_PREFIX_LEN > 2:
traverse_prefixes += [
"{0:0{1}x}".format(i, self.tree.TRAVERSE_PREFIX_LEN)
for i in range(
1, pow(16, self.tree.TRAVERSE_PREFIX_LEN - 2)
)
"{0:0{1}x}".format(i, self.fs.TRAVERSE_PREFIX_LEN)
for i in range(1, pow(16, self.fs.TRAVERSE_PREFIX_LEN - 2))
]
with Tqdm(
desc="Querying "
@@ -266,13 +264,13 @@ class CloudCache:
)
with ThreadPoolExecutor(
max_workers=jobs or self.tree.JOBS
max_workers=jobs or self.fs.JOBS
) as executor:
in_remote = executor.map(list_with_update, traverse_prefixes,)
yield from itertools.chain.from_iterable(in_remote)
def all(self, jobs=None, name=None):
"""Iterate over all hashes in this tree.
"""Iterate over all hashes in this fs.
Hashes will be fetched in parallel threads according to prefix
(except for small remotes) and a progress bar will be displayed.
@@ -283,7 +281,7 @@ class CloudCache:
)
)
if not self.tree.CAN_TRAVERSE:
if not self.fs.CAN_TRAVERSE:
return self.list_hashes()
remote_size, remote_hashes = self._estimate_remote_size(name=name)
@@ -298,24 +296,24 @@ class CloudCache:
removed = False
# hashes must be sorted to ensure we always remove .dir files first
for hash_ in sorted(
self.all(jobs, str(self.tree.path_info)),
key=self.tree.is_dir_hash,
self.all(jobs, str(self.fs.path_info)),
key=self.fs.is_dir_hash,
reverse=True,
):
if hash_ in used:
continue
path_info = self.hash_to_path_info(hash_)
if self.tree.is_dir_hash(hash_):
if self.fs.is_dir_hash(hash_):
# backward compatibility
# pylint: disable=protected-access
self._remove_unpacked_dir(hash_)
self.tree.remove(path_info)
self.fs.remove(path_info)
removed = True
return removed
def list_hashes_exists(self, hashes, jobs=None, name=None):
"""Return list of the specified hashes which exist in this tree.
"""Return list of the specified hashes which exist in this fs.
Hashes will be queried individually.
"""
logger.debug(
@@ -329,12 +327,12 @@ class CloudCache:
) as pbar:
def exists_with_progress(path_info):
ret = self.tree.exists(path_info)
ret = self.fs.exists(path_info)
pbar.update_msg(str(path_info))
return ret
with ThreadPoolExecutor(
max_workers=jobs or self.tree.JOBS
max_workers=jobs or self.fs.JOBS
) as executor:
path_infos = map(self.hash_to_path_info, hashes)
in_remote = executor.map(exists_with_progress, path_infos)
@@ -376,24 +374,24 @@ class CloudCache:
"""
# Remotes which do not use traverse prefix should override
# hashes_exist() (see ssh, local)
assert self.tree.TRAVERSE_PREFIX_LEN >= 2
assert self.fs.TRAVERSE_PREFIX_LEN >= 2
hashes = set(hashes)
if len(hashes) == 1 or not self.tree.CAN_TRAVERSE:
if len(hashes) == 1 or not self.fs.CAN_TRAVERSE:
remote_hashes = self.list_hashes_exists(hashes, jobs, name)
return remote_hashes
# Max remote size allowed for us to use traverse method
remote_size, remote_hashes = self._estimate_remote_size(hashes, name)
traverse_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
traverse_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE
# For sufficiently large remotes, traverse must be weighted to account
# for performance overhead from large lists/sets.
# From testing with S3, for remotes with 1M+ files, object_exists is
# faster until len(hashes) is at least 10k~100k
if remote_size > self.tree.TRAVERSE_THRESHOLD_SIZE:
if remote_size > self.fs.TRAVERSE_THRESHOLD_SIZE:
traverse_weight = (
traverse_pages * self.tree.TRAVERSE_WEIGHT_MULTIPLIER
traverse_pages * self.fs.TRAVERSE_WEIGHT_MULTIPLIER
)
else:
traverse_weight = traverse_pages
+15 -15
View File
@@ -22,11 +22,11 @@ class LocalCache(CloudCache):
CACHE_MODE = 0o444
UNPACKED_DIR_SUFFIX = ".unpacked"
def __init__(self, tree):
super().__init__(tree)
self.cache_dir = tree.config.get("url")
def __init__(self, fs):
super().__init__(fs)
self.cache_dir = fs.config.get("url")
shared = tree.config.get("shared")
shared = fs.config.get("shared")
if shared:
self._file_mode = 0o664
self._dir_mode = 0o2775
@@ -36,11 +36,11 @@ class LocalCache(CloudCache):
@property
def cache_dir(self):
return self.tree.path_info.fspath if self.tree.path_info else None
return self.fs.path_info.fspath if self.fs.path_info else None
@cache_dir.setter
def cache_dir(self, value):
self.tree.path_info = PathInfo(value) if value else None
self.fs.path_info = PathInfo(value) if value else None
@cached_property
def cache_path(self):
@@ -71,7 +71,7 @@ class LocalCache(CloudCache):
unit="file",
desc="Querying " + ("cache in " + name if name else "local cache"),
):
hash_info = HashInfo(self.tree.PARAM_CHECKSUM, hash_)
hash_info = HashInfo(self.fs.PARAM_CHECKSUM, hash_)
try:
self.check(hash_info)
ret.append(hash_)
@@ -81,14 +81,14 @@ class LocalCache(CloudCache):
return ret
def _list_paths(self, prefix=None, progress_callback=None):
assert self.tree.path_info is not None
assert self.fs.path_info is not None
if prefix:
path_info = self.tree.path_info / prefix[:2]
if not self.tree.exists(path_info):
path_info = self.fs.path_info / prefix[:2]
if not self.fs.exists(path_info):
return
else:
path_info = self.tree.path_info
# NOTE: use utils.fs walk_files since tree.walk_files will not follow
path_info = self.fs.path_info
# NOTE: use utils.fs walk_files since fs.walk_files will not follow
# symlinks
if progress_callback:
for path in walk_files(path_info):
@@ -100,10 +100,10 @@ class LocalCache(CloudCache):
def _remove_unpacked_dir(self, hash_):
info = self.hash_to_path_info(hash_)
path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
self.tree.remove(path_info)
self.fs.remove(path_info)
def _unprotect_file(self, path):
if self.tree.is_symlink(path) or self.tree.is_hardlink(path):
if self.fs.is_symlink(path) or self.fs.is_hardlink(path):
logger.debug("Unprotecting '%s'", path)
tmp = os.path.join(os.path.dirname(path), "." + uuid())
@@ -126,7 +126,7 @@ class LocalCache(CloudCache):
os.chmod(path, self._file_mode)
def _unprotect_dir(self, path):
for fname in self.tree.walk_files(path):
for fname in self.fs.walk_files(path):
self._unprotect_file(fname)
def unprotect(self, path_info):
+8 -8
View File
@@ -28,7 +28,7 @@ class SSHCache(CloudCache):
callback(path)
return ret
with self.tree.ssh(path_infos[0]) as ssh:
with self.fs.ssh(path_infos[0]) as ssh:
channels = ssh.open_max_sftp_channels()
max_workers = len(channels)
@@ -47,11 +47,11 @@ class SSHCache(CloudCache):
faster than current approach (relying on exists(path_info)) applied in
remote/base.
"""
if not self.tree.CAN_TRAVERSE:
if not self.fs.CAN_TRAVERSE:
return list(set(hashes) & set(self.all()))
# possibly prompt for credentials before "Querying" progress output
self.tree.ensure_credentials()
self.fs.ensure_credentials()
with Tqdm(
desc="Querying "
@@ -64,10 +64,10 @@ class SSHCache(CloudCache):
return self.batch_exists(chunks, callback=pbar.update_msg)
with ThreadPoolExecutor(
max_workers=jobs or self.tree.JOBS
max_workers=jobs or self.fs.JOBS
) as executor:
path_infos = [self.hash_to_path_info(x) for x in hashes]
chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
chunks = to_chunks(path_infos, num_chunks=self.fs.JOBS)
results = executor.map(exists_with_progress, chunks)
in_remote = itertools.chain.from_iterable(results)
ret = list(itertools.compress(hashes, in_remote))
@@ -75,10 +75,10 @@ class SSHCache(CloudCache):
def _list_paths(self, prefix=None, progress_callback=None):
if prefix:
root = posixpath.join(self.tree.path_info.path, prefix[:2])
root = posixpath.join(self.fs.path_info.path, prefix[:2])
else:
root = self.tree.path_info.path
with self.tree.ssh(self.tree.path_info) as ssh:
root = self.fs.path_info.path
with self.fs.ssh(self.fs.path_info) as ssh:
if prefix and not ssh.exists(root):
return
# If we simply return an iterator then with above closes instantly
+44 -44
View File
@@ -18,7 +18,7 @@ from dvc.remote.slow_link_detection import ( # type: ignore[attr-defined]
logger = logging.getLogger(__name__)
def _changed(path_info, tree, obj, cache):
def _changed(path_info, fs, obj, cache):
logger.trace("checking if '%s'('%s') has changed.", path_info, obj)
try:
@@ -30,7 +30,7 @@ def _changed(path_info, tree, obj, cache):
return True
try:
actual = get_hash(path_info, tree, obj.hash_info.name)
actual = get_hash(path_info, fs, obj.hash_info.name)
except FileNotFoundError:
logger.debug("'%s' doesn't exist.", path_info)
return True
@@ -48,15 +48,15 @@ def _changed(path_info, tree, obj, cache):
return False
def _remove(path_info, tree, cache, force=False):
if not tree.exists(path_info):
def _remove(path_info, fs, cache, force=False):
if not fs.exists(path_info):
return
if force:
tree.remove(path_info)
fs.remove(path_info)
return
current = get_hash(path_info, tree, tree.PARAM_CHECKSUM)
current = get_hash(path_info, fs, fs.PARAM_CHECKSUM)
try:
obj = load(cache, current)
check(cache, obj)
@@ -69,26 +69,26 @@ def _remove(path_info, tree, cache, force=False):
if not prompt.confirm(msg):
raise ConfirmRemoveError(str(path_info))
tree.remove(path_info)
fs.remove(path_info)
def _verify_link(cache, path_info, link_type):
if link_type == "hardlink" and cache.tree.getsize(path_info) == 0:
if link_type == "hardlink" and cache.fs.getsize(path_info) == 0:
return
if cache.cache_type_confirmed:
return
is_link = getattr(cache.tree, f"is_{link_type}", None)
is_link = getattr(cache.fs, f"is_{link_type}", None)
if is_link and not is_link(path_info):
cache.tree.remove(path_info)
cache.fs.remove(path_info)
raise DvcException(f"failed to verify {link_type}")
cache.cache_type_confirmed = True
def _do_link(cache, from_info, to_info, link_method):
if cache.tree.exists(to_info):
if cache.fs.exists(to_info):
raise DvcException(f"Link '{to_info}' already exists!")
link_method(from_info, to_info)
@@ -101,7 +101,7 @@ def _do_link(cache, from_info, to_info, link_method):
@slow_link_guard
def _try_links(cache, from_info, to_info, link_types):
while link_types:
link_method = getattr(cache.tree, link_types[0])
link_method = getattr(cache.fs, link_types[0])
try:
_do_link(cache, from_info, to_info, link_method)
_verify_link(cache, to_info, link_types[0])
@@ -117,7 +117,7 @@ def _try_links(cache, from_info, to_info, link_types):
def _link(cache, from_info, to_info):
assert cache.tree.isfile(from_info)
assert cache.fs.isfile(from_info)
cache.makedirs(to_info.parent)
_try_links(cache, from_info, to_info, cache.cache_types)
@@ -131,76 +131,76 @@ def _cache_is_copy(cache, path_info):
return True
workspace_file = path_info.with_name("." + uuid())
test_cache_file = cache.tree.path_info / ".cache_type_test_file"
if not cache.tree.exists(test_cache_file):
test_cache_file = cache.fs.path_info / ".cache_type_test_file"
if not cache.fs.exists(test_cache_file):
cache.makedirs(test_cache_file.parent)
with cache.tree.open(test_cache_file, "wb") as fobj:
with cache.fs.open(test_cache_file, "wb") as fobj:
fobj.write(bytes(1))
try:
_link(cache, test_cache_file, workspace_file)
finally:
cache.tree.remove(workspace_file)
cache.tree.remove(test_cache_file)
cache.fs.remove(workspace_file)
cache.fs.remove(test_cache_file)
cache.cache_type_confirmed = True
return cache.cache_types[0] == "copy"
def _checkout_file(
path_info, tree, obj, cache, force, progress_callback=None, relink=False,
path_info, fs, obj, cache, force, progress_callback=None, relink=False,
):
"""The file is changed we need to checkout a new copy"""
modified = False
cache_info = cache.hash_to_path_info(obj.hash_info.value)
if tree.exists(path_info):
if not relink and _changed(path_info, tree, obj, cache):
if fs.exists(path_info):
if not relink and _changed(path_info, fs, obj, cache):
modified = True
_remove(path_info, tree, cache, force=force)
_remove(path_info, fs, cache, force=force)
_link(cache, cache_info, path_info)
else:
if tree.iscopy(path_info) and _cache_is_copy(cache, path_info):
if fs.iscopy(path_info) and _cache_is_copy(cache, path_info):
cache.unprotect(path_info)
else:
_remove(path_info, tree, cache, force=force)
_remove(path_info, fs, cache, force=force)
_link(cache, cache_info, path_info)
else:
_link(cache, cache_info, path_info)
modified = True
tree.state.save(path_info, obj.hash_info)
fs.state.save(path_info, obj.hash_info)
if progress_callback:
progress_callback(str(path_info))
return modified
def _remove_redundant_files(path_info, tree, dir_info, cache, force):
existing_files = set(tree.walk_files(path_info))
def _remove_redundant_files(path_info, fs, dir_info, cache, force):
existing_files = set(fs.walk_files(path_info))
needed_files = {info for info, _ in dir_info.items(path_info)}
redundant_files = existing_files - needed_files
for path in redundant_files:
_remove(path, tree, cache, force)
_remove(path, fs, cache, force)
return bool(redundant_files)
def _checkout_dir(
path_info, tree, obj, cache, force, progress_callback=None, relink=False,
path_info, fs, obj, cache, force, progress_callback=None, relink=False,
):
modified = False, False
# Create dir separately so that dir is created
# even if there are no files in it
if not tree.exists(path_info):
if not fs.exists(path_info):
modified = True
tree.makedirs(path_info)
fs.makedirs(path_info)
logger.debug("Linking directory '%s'.", path_info)
for entry_info, entry_hash_info in obj.hash_info.dir_info.items(path_info):
entry_modified = _checkout_file(
entry_info,
tree,
fs,
cache.get(entry_hash_info),
cache,
force,
@@ -212,12 +212,12 @@ def _checkout_dir(
modified = (
_remove_redundant_files(
path_info, tree, obj.hash_info.dir_info, cache, force
path_info, fs, obj.hash_info.dir_info, cache, force
)
or modified
)
tree.state.save(path_info, obj.hash_info)
fs.state.save(path_info, obj.hash_info)
# relink is not modified, assume it as nochange
return modified and not relink
@@ -225,7 +225,7 @@ def _checkout_dir(
def _checkout(
path_info,
tree,
fs,
obj,
cache,
force=False,
@@ -234,21 +234,21 @@ def _checkout(
):
if not obj.hash_info.isdir:
ret = _checkout_file(
path_info, tree, obj, cache, force, progress_callback, relink
path_info, fs, obj, cache, force, progress_callback, relink
)
else:
ret = _checkout_dir(
path_info, tree, obj, cache, force, progress_callback, relink,
path_info, fs, obj, cache, force, progress_callback, relink,
)
tree.state.save_link(path_info)
fs.state.save_link(path_info)
return ret
def checkout(
path_info,
tree,
fs,
obj,
cache,
force=False,
@@ -256,7 +256,7 @@ def checkout(
relink=False,
quiet=False,
):
if path_info.scheme not in ["local", cache.tree.scheme]:
if path_info.scheme not in ["local", cache.fs.scheme]:
raise NotImplementedError
failed = None
@@ -267,10 +267,10 @@ def checkout(
"No file hash info found for '%s'. It won't be created.",
path_info,
)
_remove(path_info, tree, cache, force=force)
_remove(path_info, fs, cache, force=force)
failed = path_info
elif not relink and not _changed(path_info, tree, obj, cache):
elif not relink and not _changed(path_info, fs, obj, cache):
logger.trace("Data '%s' didn't change.", path_info)
skip = True
else:
@@ -283,7 +283,7 @@ def checkout(
obj.hash_info,
path_info,
)
_remove(path_info, tree, cache, force=force)
_remove(path_info, fs, cache, force=force)
failed = path_info
if failed or skip:
@@ -298,5 +298,5 @@ def checkout(
logger.debug("Checking out '%s' with cache '%s'.", path_info, obj)
return _checkout(
path_info, tree, obj, cache, force, progress_callback, relink,
path_info, fs, obj, cache, force, progress_callback, relink,
)
+1 -1
View File
@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
class CmdCheckIgnore(CmdBase):
def __init__(self, args):
super().__init__(args)
self.ignore_filter = self.repo.tree.dvcignore
self.ignore_filter = self.repo.fs.dvcignore
def _show_results(self, result):
if not result.match and not self.args.non_matching:
+13 -13
View File
@@ -63,9 +63,9 @@ class Config(dict):
CONFIG_LOCAL = "config.local"
def __init__(
self, dvc_dir=None, validate=True, tree=None, config=None,
self, dvc_dir=None, validate=True, fs=None, config=None,
): # pylint: disable=super-init-not-called
from dvc.tree.local import LocalTree
from dvc.fs.local import LocalFileSystem
self.dvc_dir = dvc_dir
@@ -79,8 +79,8 @@ class Config(dict):
else:
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self.wtree = LocalTree(None, {"url": self.dvc_dir})
self.tree = tree or self.wtree
self.wfs = LocalFileSystem(None, {"url": self.dvc_dir})
self.fs = fs or self.wfs
self.load(validate=validate, config=config)
@@ -143,19 +143,19 @@ class Config(dict):
if not self["cache"].get("dir") and self.dvc_dir:
self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
def _get_tree(self, level):
# NOTE: this might be a GitTree, which doesn't see things outside of
def _get_fs(self, level):
# NOTE: this might be a Gitfs, which doesn't see things outside of
# the repo.
return self.tree if level == "repo" else self.wtree
return self.fs if level == "repo" else self.wfs
def _load_config(self, level):
from configobj import ConfigObj
filename = self.files[level]
tree = self._get_tree(level)
fs = self._get_fs(level)
if tree.exists(filename, use_dvcignore=False):
with tree.open(filename) as fobj:
if fs.exists(filename, use_dvcignore=False):
with fs.open(filename) as fobj:
conf_obj = ConfigObj(fobj)
else:
conf_obj = ConfigObj()
@@ -165,14 +165,14 @@ class Config(dict):
from configobj import ConfigObj
filename = self.files[level]
tree = self._get_tree(level)
fs = self._get_fs(level)
logger.debug(f"Writing '{filename}'.")
tree.makedirs(os.path.dirname(filename))
fs.makedirs(os.path.dirname(filename))
config = ConfigObj(_pack_remotes(conf_dict))
with tree.open(filename, "wb") as fobj:
with fs.open(filename, "wb") as fobj:
config.write(fobj)
config.filename = filename
+3 -3
View File
@@ -104,11 +104,11 @@ class DataCloud:
command (str): the command which is benefitting from this function
(to be used for reporting better error messages).
"""
from dvc.tree import get_cloud_tree
from dvc.fs import get_cloud_fs
from_tree = get_cloud_tree(self.repo, url=source)
from_fs = get_cloud_fs(self.repo, url=source)
remote = self.get_remote(remote, command)
return remote.transfer(from_tree, from_tree.path_info, jobs=jobs)
return remote.transfer(from_fs, from_fs.path_info, jobs=jobs)
def status(
self,
+3 -3
View File
@@ -17,7 +17,7 @@ from dvc.dependency.webhdfs import WebHDFSDependency
from dvc.output.base import BaseOutput
from dvc.scheme import Schemes
from ..tree import get_cloud_tree
from ..fs import get_cloud_fs
from .repo import RepoDependency
DEPS = [
@@ -64,8 +64,8 @@ SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
def _get(stage, p, info):
parsed = urlparse(p) if p else None
if parsed and parsed.scheme == "remote":
tree = get_cloud_tree(stage.repo, name=parsed.netloc)
return DEP_MAP[tree.scheme](stage, p, info, tree=tree)
fs = get_cloud_fs(stage.repo, name=parsed.netloc)
return DEP_MAP[fs.scheme](stage, p, info, fs=fs)
if info and info.get(RepoDependency.PARAM_REPO):
repo = info.pop(RepoDependency.PARAM_REPO)
+2 -2
View File
@@ -1,8 +1,8 @@
from dvc.dependency.base import BaseDependency
from dvc.output.base import BaseOutput
from ..tree.azure import AzureTree
from ..fs.azure import AzureFileSystem
class AzureDependency(BaseDependency, BaseOutput):
TREE_CLS = AzureTree
FS_CLS = AzureFileSystem
+2 -2
View File
@@ -1,8 +1,8 @@
from dvc.dependency.base import BaseDependency
from dvc.output.base import BaseOutput
from ..tree.http import HTTPTree
from ..fs.http import HTTPFileSystem
class HTTPDependency(BaseDependency, BaseOutput):
TREE_CLS = HTTPTree
FS_CLS = HTTPFileSystem
+2 -2
View File
@@ -1,6 +1,6 @@
from ..tree.https import HTTPSTree
from ..fs.https import HTTPSFileSystem
from .http import HTTPDependency
class HTTPSDependency(HTTPDependency):
TREE_CLS = HTTPSTree
FS_CLS = HTTPSFileSystem
+1 -1
View File
@@ -91,7 +91,7 @@ class ParamsDependency(LocalDependency):
suffix = self.path_info.suffix.lower()
loader = LOADERS[suffix]
try:
config = loader(self.path_info, tree=self.repo.tree)
config = loader(self.path_info, fs=self.repo.fs)
except ParseError as exc:
raise BadParamFileError(
f"Unable to read parameters from '{self}'"
+4 -4
View File
@@ -23,7 +23,7 @@ class RepoDependency(LocalDependency):
self.def_repo = def_repo
super().__init__(stage, *args, **kwargs)
def _parse_path(self, tree, path):
def _parse_path(self, fs, path):
return None
@property
@@ -52,7 +52,7 @@ class RepoDependency(LocalDependency):
with self._make_repo(locked=locked) as repo:
path_info = PathInfo(repo.root_dir) / self.def_path
return get_hash(
path_info, repo.repo_tree, "md5", follow_subrepos=False
path_info, repo.repo_fs, "md5", follow_subrepos=False
)
def workspace_status(self):
@@ -86,13 +86,13 @@ class RepoDependency(LocalDependency):
obj = stage(
cache,
path_info,
repo.repo_tree,
repo.repo_fs,
jobs=jobs,
follow_subrepos=False,
)
save(cache, obj, jobs=jobs)
checkout(to.path_info, to.tree, obj, cache)
checkout(to.path_info, to.fs, obj, cache)
def update(self, rev=None):
if rev:
+2 -2
View File
@@ -1,8 +1,8 @@
from dvc.dependency.base import BaseDependency
from dvc.output.base import BaseOutput
from ..tree.webdav import WebDAVTree
from ..fs.webdav import WebDAVFileSystem
class WebDAVDependency(BaseDependency, BaseOutput):
TREE_CLS = WebDAVTree
FS_CLS = WebDAVFileSystem
+2 -2
View File
@@ -1,6 +1,6 @@
from ..tree.webdavs import WebDAVSTree
from ..fs.webdavs import WebDAVSFileSystem
from .webdav import WebDAVDependency
class WebDAVSDependency(WebDAVDependency):
TREE_CLS = WebDAVSTree
FS_CLS = WebDAVSFileSystem
+9 -9
View File
@@ -106,13 +106,13 @@ class FileMixin:
return relpath(self.path)
def exists(self):
return self.repo.tree.exists(self.path)
return self.repo.fs.exists(self.path)
def _is_git_ignored(self):
from dvc.tree.local import LocalTree
from dvc.fs.local import LocalFileSystem
return isinstance(
self.repo.tree, LocalTree
self.repo.fs, LocalFileSystem
) and self.repo.scm.is_ignored(self.path)
def _load(self):
@@ -122,16 +122,16 @@ class FileMixin:
# 3. path doesn't represent a regular file
# 4. when the file is git ignored
if not self.exists():
is_ignored = self.repo.tree.exists(self.path, use_dvcignore=False)
is_ignored = self.repo.fs.exists(self.path, use_dvcignore=False)
raise StageFileDoesNotExistError(self.path, dvc_ignored=is_ignored)
if self.verify:
check_dvc_filename(self.path)
if not self.repo.tree.isfile(self.path):
if not self.repo.fs.isfile(self.path):
raise StageFileIsNotDvcFileError(self.path)
if self._is_git_ignored():
raise FileIsGitIgnored(self.path)
with self.repo.tree.open(self.path, encoding="utf-8") as fd:
with self.repo.fs.open(self.path, encoding="utf-8") as fd:
stage_text = fd.read()
d = parse_yaml(stage_text, self.path)
return self.validate(d, self.relpath), stage_text
@@ -232,7 +232,7 @@ class PipelineFile(FileMixin):
self._check_if_parametrized(stage)
stage_data = serialize.to_pipeline_file(stage)
with modify_yaml(self.path, tree=self.repo.tree) as data:
with modify_yaml(self.path, fs=self.repo.fs) as data:
if not data:
logger.info("Creating '%s'", self.relpath)
@@ -334,7 +334,7 @@ class Lockfile(FileMixin):
if not self.exists():
return {}
data = load_yaml(self.path, tree=self.repo.tree)
data = load_yaml(self.path, fs=self.repo.fs)
try:
data = self.validate(data, fname=self.relpath)
except StageFileFormatError as exc:
@@ -351,7 +351,7 @@ class Lockfile(FileMixin):
def dump(self, stage, **kwargs):
stage_data = serialize.to_lockfile(stage)
with modify_yaml(self.path, tree=self.repo.tree) as data:
with modify_yaml(self.path, fs=self.repo.fs) as data:
version = LOCKFILE_VERSION.from_dict(data)
if version == LOCKFILE_VERSION.V1:
logger.info(
+37 -37
View File
@@ -1,45 +1,45 @@
import posixpath
from urllib.parse import urlparse
from .azure import AzureTree
from .gdrive import GDriveTree
from .gs import GSTree
from .hdfs import HDFSTree
from .http import HTTPTree
from .https import HTTPSTree
from .local import LocalTree
from .oss import OSSTree
from .s3 import S3Tree
from .ssh import SSHTree
from .webdav import WebDAVTree
from .webdavs import WebDAVSTree
from .webhdfs import WebHDFSTree
from .azure import AzureFileSystem
from .gdrive import GDriveFileSystem
from .gs import GSFileSystem
from .hdfs import HDFSFileSystem
from .http import HTTPFileSystem
from .https import HTTPSFileSystem
from .local import LocalFileSystem
from .oss import OSSFileSystem
from .s3 import S3FileSystem
from .ssh import SSHFileSystem
from .webdav import WebDAVFileSystem
from .webdavs import WebDAVSFileSystem
from .webhdfs import WebHDFSFileSystem
TREES = [
AzureTree,
GDriveTree,
GSTree,
HDFSTree,
HTTPTree,
HTTPSTree,
S3Tree,
SSHTree,
OSSTree,
WebDAVTree,
WebDAVSTree,
WebHDFSTree
# NOTE: LocalTree is the default
ALL_FS = [
AzureFileSystem,
GDriveFileSystem,
GSFileSystem,
HDFSFileSystem,
HTTPFileSystem,
HTTPSFileSystem,
S3FileSystem,
SSHFileSystem,
OSSFileSystem,
WebDAVFileSystem,
WebDAVSFileSystem,
WebHDFSFileSystem
# NOTE: LocalFileSystem is the default
]
def get_tree_cls(remote_conf):
for tree_cls in TREES:
if tree_cls.supported(remote_conf):
return tree_cls
return LocalTree
def get_fs_cls(remote_conf):
for fs_cls in ALL_FS:
if fs_cls.supported(remote_conf):
return fs_cls
return LocalFileSystem
def get_tree_config(config, **kwargs):
def get_fs_config(config, **kwargs):
name = kwargs.get("name")
if name:
remote_conf = config["remote"][name.lower()]
@@ -74,18 +74,18 @@ def _resolve_remote_refs(config, remote_conf):
if parsed.scheme != "remote":
return remote_conf
base = get_tree_config(config, name=parsed.netloc)
base = get_fs_config(config, name=parsed.netloc)
url = posixpath.join(base["url"], parsed.path.lstrip("/"))
return {**base, **remote_conf, "url": url}
def get_cloud_tree(repo, **kwargs):
def get_cloud_fs(repo, **kwargs):
from dvc.config import ConfigError
from dvc.config_schema import SCHEMA, Invalid
remote_conf = get_tree_config(repo.config, **kwargs)
remote_conf = get_fs_config(repo.config, **kwargs)
try:
remote_conf = SCHEMA["remote"][str](remote_conf)
except Invalid as exc:
raise ConfigError(str(exc)) from None
return get_tree_cls(remote_conf)(repo, remote_conf)
return get_fs_cls(remote_conf)(repo, remote_conf)
+2 -2
View File
@@ -9,12 +9,12 @@ from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from .base import BaseTree
from .base import BaseFileSystem
logger = logging.getLogger(__name__)
class AzureTree(BaseTree):
class AzureFileSystem(BaseFileSystem):
scheme = Schemes.AZURE
PATH_CLS = CloudURLInfo
REQUIRES = {
+3 -3
View File
@@ -38,12 +38,12 @@ class RemoteMissingDepsError(DvcException):
@decorator
def use_state(call):
tree = call._args[0] # pylint: disable=protected-access
with tree.state:
fs = call._args[0] # pylint: disable=protected-access
with fs.state:
return call()
class BaseTree:
class BaseFileSystem:
scheme = "base"
REQUIRES: ClassVar[Dict[str, str]] = {}
PATH_CLS = URLInfo # type: Any
+4 -4
View File
@@ -7,7 +7,7 @@ from dvc.path_info import PathInfo
from dvc.utils import relpath
from ._metadata import Metadata
from .base import BaseTree
from .base import BaseFileSystem
if typing.TYPE_CHECKING:
from dvc.output.base import BaseOutput
@@ -16,8 +16,8 @@ if typing.TYPE_CHECKING:
logger = logging.getLogger(__name__)
class DvcTree(BaseTree): # pylint:disable=abstract-method
"""DVC repo tree.
class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
"""DVC repo fs.
Args:
repo: DVC repo.
@@ -79,7 +79,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method
else:
checksum = out.hash_info.value
remote_info = remote_obj.cache.hash_to_path_info(checksum)
return remote_obj.tree.open(
return remote_obj.fs.open(
remote_info, mode=mode, encoding=encoding
)
+12 -10
View File
@@ -18,7 +18,7 @@ from dvc.scheme import Schemes
from dvc.utils import format_link, tmp_fname
from dvc.utils.stream import IterStream
from .base import BaseTree
from .base import BaseFileSystem
logger = logging.getLogger(__name__)
FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
@@ -85,7 +85,7 @@ class GDriveURLInfo(CloudURLInfo):
self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
class GDriveTree(BaseTree):
class GDriveFileSystem(BaseFileSystem):
scheme = Schemes.GDRIVE
PATH_CLS = GDriveURLInfo
PARAM_CHECKSUM = "checksum"
@@ -131,7 +131,7 @@ class GDriveTree(BaseTree):
self._validate_config()
self._gdrive_user_credentials_path = (
tmp_fname(os.path.join(self.repo.tmp_dir, ""))
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA)
if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
else config.get(
"gdrive_user_credentials_file",
os.path.join(
@@ -173,8 +173,8 @@ class GDriveTree(BaseTree):
Useful for tests, exception messages, etc. Returns either env variable
name if it's set or actual path to the credentials file.
"""
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
return GDriveTree.GDRIVE_CREDENTIALS_DATA
if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
return GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
if os.path.exists(self._gdrive_user_credentials_path):
return self._gdrive_user_credentials_path
return None
@@ -188,7 +188,7 @@ class GDriveTree(BaseTree):
DVC config client id or secret but forgets to remove the cached
credentials file.
"""
if not os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
if not os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
if (
settings["client_config"]["client_id"]
!= auth.credentials.client_id
@@ -211,9 +211,11 @@ class GDriveTree(BaseTree):
from pydrive2.auth import GoogleAuth
from pydrive2.drive import GoogleDrive
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
with open(self._gdrive_user_credentials_path, "w") as cred_file:
cred_file.write(os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA))
cred_file.write(
os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
)
auth_settings = {
"client_config_backend": "settings",
@@ -259,7 +261,7 @@ class GDriveTree(BaseTree):
gauth.ServiceAuth()
else:
gauth.CommandLineAuth()
GDriveTree._validate_credentials(gauth, auth_settings)
GDriveFileSystem._validate_credentials(gauth, auth_settings)
# Handle AuthenticationError, RefreshError and other auth failures
# It's hard to come up with a narrow exception, since PyDrive throws
@@ -268,7 +270,7 @@ class GDriveTree(BaseTree):
except Exception as exc:
raise GDriveAuthError(self.credentials_location) from exc
finally:
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
os.remove(self._gdrive_user_credentials_path)
return GoogleDrive(gauth)
+11 -10
View File
@@ -3,18 +3,19 @@ import os
from funcy import cached_property
from dvc.tree.base import BaseTree
from dvc.utils import is_exec, relpath
from .base import BaseFileSystem
class GitTree(BaseTree): # pylint:disable=abstract-method
class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method
"""Proxies the repo file access methods to Git objects"""
def __init__(
self, root_dir, trie, use_dvcignore=False, dvcignore_root=None
):
super().__init__(None, {})
self._tree_root = root_dir
self._fs_root = root_dir
self.trie = trie
self.use_dvcignore = use_dvcignore
self.dvcignore_root = dvcignore_root
@@ -24,17 +25,17 @@ class GitTree(BaseTree): # pylint:disable=abstract-method
return self.trie.rev
@property
def tree_root(self):
return self._tree_root
def fs_root(self):
return self._fs_root
def _get_key(self, path):
if isinstance(path, str):
if not os.path.isabs(path):
relparts = path.split(os.sep)
else:
relparts = relpath(path, self.tree_root).split(os.sep)
relparts = relpath(path, self.fs_root).split(os.sep)
else:
relparts = path.relative_to(self.tree_root).parts
relparts = path.relative_to(self.fs_root).parts
if relparts == ["."]:
return ()
return tuple(relparts)
@@ -43,7 +44,7 @@ class GitTree(BaseTree): # pylint:disable=abstract-method
def dvcignore(self):
from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
root = self.dvcignore_root or self.tree_root
root = self.dvcignore_root or self.fs_root
cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
return cls(self, root)
@@ -122,9 +123,9 @@ class GitTree(BaseTree): # pylint:disable=abstract-method
key = self._get_key(top)
for prefix, dirs, files in self.trie.walk(key, topdown=topdown):
if prefix:
root = os.path.join(self.tree_root, os.sep.join(prefix))
root = os.path.join(self.fs_root, os.sep.join(prefix))
else:
root = self.tree_root
root = self.fs_root
if use_dvcignore:
dirs[:], files[:] = self.dvcignore(
root, dirs, files, ignore_subrepos=ignore_subrepos,
+3 -3
View File
@@ -11,7 +11,7 @@ from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from .base import BaseTree
from .base import BaseFileSystem
logger = logging.getLogger(__name__)
@@ -50,7 +50,7 @@ def _upload_to_bucket(bucket, fobj, to_info, chunk_size=None):
blob.upload_from_file(fobj)
class GSTree(BaseTree):
class GSFileSystem(BaseFileSystem):
scheme = Schemes.GS
PATH_CLS = CloudURLInfo
REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
@@ -196,7 +196,7 @@ class GSTree(BaseTree):
def _upload_fobj(self, fobj, to_info):
bucket = self.gs.bucket(to_info.bucket)
# With other references being given in the @dynamic_chunk_size
# this function does not respect tree.CHUNK_SIZE, since it is
# this function does not respect fs.CHUNK_SIZE, since it is
# too big for GS to handle. Rather it dynamically calculates the
# best possible chunk size
_upload_to_bucket(bucket, fobj, to_info)
+2 -2
View File
@@ -13,7 +13,7 @@ from dvc.progress import Tqdm
from dvc.scheme import Schemes
from dvc.utils import fix_env, tmp_fname
from .base import BaseTree, RemoteCmdError
from .base import BaseFileSystem, RemoteCmdError
from .pool import get_connection
logger = logging.getLogger(__name__)
@@ -58,7 +58,7 @@ def _hadoop_fs_checksum(path_info):
return _group(regex, stdout, "checksum")
class HDFSTree(BaseTree):
class HDFSFileSystem(BaseFileSystem):
scheme = Schemes.HDFS
REQUIRES = {"pyarrow": "pyarrow"}
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
+2 -2
View File
@@ -10,7 +10,7 @@ from dvc.path_info import HTTPURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from .base import BaseTree
from .base import BaseFileSystem
logger = logging.getLogger(__name__)
@@ -24,7 +24,7 @@ def ask_password(host, user):
)
class HTTPTree(BaseTree): # pylint:disable=abstract-method
class HTTPFileSystem(BaseFileSystem): # pylint:disable=abstract-method
scheme = Schemes.HTTP
PATH_CLS = HTTPURLInfo
PARAM_CHECKSUM = "etag"
+7
View File
@@ -0,0 +1,7 @@
from dvc.scheme import Schemes
from .http import HTTPFileSystem
class HTTPSFileSystem(HTTPFileSystem): # pylint:disable=abstract-method
scheme = Schemes.HTTPS
+5 -5
View File
@@ -10,12 +10,12 @@ from dvc.system import System
from dvc.utils import is_exec, tmp_fname
from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove
from .base import BaseTree
from .base import BaseFileSystem
logger = logging.getLogger(__name__)
class LocalTree(BaseTree):
class LocalFileSystem(BaseFileSystem):
scheme = Schemes.LOCAL
PATH_CLS = PathInfo
PARAM_CHECKSUM = "md5"
@@ -30,7 +30,7 @@ class LocalTree(BaseTree):
self.dvcignore_root = dvcignore_root
@property
def tree_root(self):
def fs_root(self):
return self.config.get("url")
@property
@@ -43,7 +43,7 @@ class LocalTree(BaseTree):
def dvcignore(self):
from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
root = self.dvcignore_root or self.tree_root
root = self.dvcignore_root or self.fs_root
cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
return cls(self, root)
@@ -92,7 +92,7 @@ class LocalTree(BaseTree):
use_dvcignore=True,
ignore_subrepos=True,
):
"""Directory tree generator.
"""Directory fs generator.
See `os.walk` for the docs. Differences:
- no support for symlinks
+4 -4
View File
@@ -1,16 +1,16 @@
from .base import BaseTree
from .base import BaseFileSystem
class MemoryTree(BaseTree):
class MemoryFileSystem(BaseFileSystem):
scheme = "local"
PARAM_CHECKSUM = "md5"
def __init__(self, repo, config):
from fsspec.implementations.memory import MemoryFileSystem
from fsspec.implementations.memory import MemoryFileSystem as MemFS
super().__init__(repo, config)
self.fs = MemoryFileSystem()
self.fs = MemFS()
def exists(self, path_info, use_dvcignore=True):
return self.fs.exists(path_info.path)
+2 -2
View File
@@ -8,12 +8,12 @@ from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from .base import BaseTree
from .base import BaseFileSystem
logger = logging.getLogger(__name__)
class OSSTree(BaseTree): # pylint:disable=abstract-method
class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method
"""
oss2 document:
https://www.alibabacloud.com/help/doc-detail/32026.htm
View File
+66 -64
View File
@@ -10,8 +10,8 @@ from funcy import lfilter, wrap_with
from dvc.path_info import PathInfo
from .base import BaseTree
from .dvc import DvcTree
from .base import BaseFileSystem
from .dvc import DvcFileSystem
if TYPE_CHECKING:
from dvc.repo import Repo
@@ -21,14 +21,14 @@ logger = logging.getLogger(__name__)
RepoFactory = Union[Callable[[str], "Repo"], Type["Repo"]]
class RepoTree(BaseTree): # pylint:disable=abstract-method
"""DVC + git-tracked files tree.
class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
"""DVC + git-tracked files fs.
Args:
repo: DVC or git repo.
subrepos: traverse to subrepos (by default, it ignores subrepos)
repo_factory: A function to initialize subrepo with, default is Repo.
kwargs: Additional keyword arguments passed to the `DvcTree()`.
kwargs: Additional keyword arguments passed to the `DvcFileSystem()`.
"""
scheme = "local"
@@ -57,11 +57,11 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
self._subrepos_trie[self.root_dir] = repo
self._dvctrees = {}
"""Keep a dvctree instance of each repo."""
self._dvcfss = {}
"""Keep a dvcfs instance of each repo."""
if hasattr(repo, "dvc_dir"):
self._dvctrees[repo.root_dir] = DvcTree(repo)
self._dvcfss[repo.root_dir] = DvcFileSystem(repo)
def _get_repo(self, path) -> Optional["Repo"]:
"""Returns repo that the path falls in, using prefix.
@@ -97,7 +97,7 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
rev=self.repo.get_rev(),
repo_factory=self.repo_factory,
)
self._dvctrees[repo.root_dir] = DvcTree(repo)
self._dvcfss[repo.root_dir] = DvcFileSystem(repo)
self._subrepos_trie[d] = repo
def _is_dvc_repo(self, dir_path):
@@ -109,11 +109,13 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
repo_path = os.path.join(dir_path, Repo.DVC_DIR)
# dvcignore will ignore subrepos, therefore using `use_dvcignore=False`
return self._main_repo.tree.isdir(repo_path, use_dvcignore=False)
return self._main_repo.fs.isdir(repo_path, use_dvcignore=False)
def _get_tree_pair(self, path) -> Tuple[BaseTree, Optional[DvcTree]]:
def _get_fs_pair(
self, path
) -> Tuple[BaseFileSystem, Optional[DvcFileSystem]]:
"""
Returns a pair of trees based on repo the path falls in, using prefix.
Returns a pair of fss based on repo the path falls in, using prefix.
"""
path = os.path.abspath(path)
@@ -121,8 +123,8 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
# this can happen if the path is outside of the repo
repo = self._get_repo(path) or self._main_repo
dvc_tree = self._dvctrees.get(repo.root_dir)
return repo.tree, dvc_tree
dvc_fs = self._dvcfss.get(repo.root_dir)
return repo.fs, dvc_fs
def open(
self, path, mode="r", encoding="utf-8", **kwargs
@@ -130,99 +132,99 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
if "b" in mode:
encoding = None
tree, dvc_tree = self._get_tree_pair(path)
fs, dvc_fs = self._get_fs_pair(path)
path_info = PathInfo(path)
try:
return tree.open(path_info, mode=mode, encoding=encoding)
return fs.open(path_info, mode=mode, encoding=encoding)
except FileNotFoundError:
if not dvc_tree:
if not dvc_fs:
raise
return dvc_tree.open(path_info, mode=mode, encoding=encoding, **kwargs)
return dvc_fs.open(path_info, mode=mode, encoding=encoding, **kwargs)
def exists(
self, path, use_dvcignore=True
): # pylint: disable=arguments-differ
tree, dvc_tree = self._get_tree_pair(path)
fs, dvc_fs = self._get_fs_pair(path)
if not dvc_tree:
return tree.exists(path)
if not dvc_fs:
return fs.exists(path)
if tree.exists(path):
if fs.exists(path):
return True
try:
meta = dvc_tree.metadata(path)
meta = dvc_fs.metadata(path)
except FileNotFoundError:
return False
(out,) = meta.outs
assert len(meta.outs) == 1
if tree.exists(out.path_info):
if fs.exists(out.path_info):
return False
return True
def isdir(self, path): # pylint: disable=arguments-differ
tree, dvc_tree = self._get_tree_pair(path)
fs, dvc_fs = self._get_fs_pair(path)
try:
st = tree.stat(path)
st = fs.stat(path)
return stat.S_ISDIR(st.st_mode)
except (OSError, ValueError):
# from CPython's os.path.isdir()
pass
if not dvc_tree:
if not dvc_fs:
return False
try:
meta = dvc_tree.metadata(path)
meta = dvc_fs.metadata(path)
except FileNotFoundError:
return False
(out,) = meta.outs
assert len(meta.outs) == 1
if tree.exists(out.path_info):
if fs.exists(out.path_info):
return False
return meta.isdir
def isdvc(self, path, **kwargs):
_, dvc_tree = self._get_tree_pair(path)
return dvc_tree is not None and dvc_tree.isdvc(path, **kwargs)
_, dvc_fs = self._get_fs_pair(path)
return dvc_fs is not None and dvc_fs.isdvc(path, **kwargs)
def isfile(self, path): # pylint: disable=arguments-differ
tree, dvc_tree = self._get_tree_pair(path)
fs, dvc_fs = self._get_fs_pair(path)
try:
st = tree.stat(path)
st = fs.stat(path)
return stat.S_ISREG(st.st_mode)
except (OSError, ValueError):
# from CPython's os.path.isfile()
pass
if not dvc_tree:
if not dvc_fs:
return False
try:
meta = dvc_tree.metadata(path)
meta = dvc_fs.metadata(path)
except FileNotFoundError:
return False
(out,) = meta.outs
assert len(meta.outs) == 1
if tree.exists(out.path_info):
if fs.exists(out.path_info):
return False
return meta.isfile
def isexec(self, path_info):
tree, dvc_tree = self._get_tree_pair(path_info)
if dvc_tree and dvc_tree.exists(path_info):
return dvc_tree.isexec(path_info)
return tree.isexec(path_info)
fs, dvc_fs = self._get_fs_pair(path_info)
if dvc_fs and dvc_fs.exists(path_info):
return dvc_fs.isexec(path_info)
return fs.isexec(path_info)
def stat(self, path):
tree, _ = self._get_tree_pair(path)
return tree.stat(path)
fs, _ = self._get_fs_pair(path)
return fs.stat(path)
def _dvc_walk(self, walk):
try:
@@ -239,15 +241,15 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
NOTE: subrepo will only be discovered when walking if
ignore_subrepos is set to False.
"""
tree, dvc_tree = self._get_tree_pair(dir_path)
tree_walk = tree.walk(
fs, dvc_fs = self._get_fs_pair(dir_path)
fs_walk = fs.walk(
dir_path, topdown=True, ignore_subrepos=not self._traverse_subrepos
)
if dvc_tree:
dvc_walk = dvc_tree.walk(dir_path, topdown=True, **kwargs)
if dvc_fs:
dvc_walk = dvc_fs.walk(dir_path, topdown=True, **kwargs)
else:
dvc_walk = None
yield from self._walk(tree_walk, dvc_walk, **kwargs)
yield from self._walk(fs_walk, dvc_walk, **kwargs)
def _walk(self, repo_walk, dvc_walk=None, dvcfiles=False):
from dvc.dvcfile import is_valid_filename
@@ -313,11 +315,11 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
follow_subrepos=None,
**kwargs
): # pylint: disable=arguments-differ
"""Walk and merge both DVC and repo trees.
"""Walk and merge both DVC and repo fss.
Args:
top: path to walk from
topdown: if True, tree will be walked from top down.
topdown: if True, fs will be walked from top down.
onerror: if set, onerror function will be called if an error
occurs (by default errors are ignored).
dvcfiles: if True, dvcfiles will be included in the files list
@@ -342,25 +344,25 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
if follow_subrepos is not None:
ignore_subrepos = not follow_subrepos
tree, dvc_tree = self._get_tree_pair(top)
repo_exists = tree.exists(top)
repo_walk = tree.walk(
fs, dvc_fs = self._get_fs_pair(top)
repo_exists = fs.exists(top)
repo_walk = fs.walk(
top,
topdown=topdown,
onerror=onerror,
ignore_subrepos=ignore_subrepos,
)
if not dvc_tree or (repo_exists and dvc_tree.isdvc(top)):
if not dvc_fs or (repo_exists and dvc_fs.isdvc(top)):
yield from self._walk(repo_walk, None, dvcfiles=dvcfiles)
return
if not repo_exists:
yield from dvc_tree.walk(top, topdown=topdown, **kwargs)
yield from dvc_fs.walk(top, topdown=topdown, **kwargs)
dvc_walk = None
if dvc_tree.exists(top):
dvc_walk = dvc_tree.walk(top, topdown=topdown, **kwargs)
if dvc_fs.exists(top):
dvc_walk = dvc_fs.walk(top, topdown=topdown, **kwargs)
yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
@@ -385,21 +387,21 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
@property
def hash_jobs(self): # pylint: disable=invalid-overridden-method
return self._main_repo.tree.hash_jobs
return self._main_repo.fs.hash_jobs
def metadata(self, path):
abspath = os.path.abspath(path)
path_info = PathInfo(abspath)
tree, dvc_tree = self._get_tree_pair(path_info)
fs, dvc_fs = self._get_fs_pair(path_info)
dvc_meta = None
if dvc_tree:
if dvc_fs:
with suppress(FileNotFoundError):
dvc_meta = dvc_tree.metadata(path_info)
dvc_meta = dvc_fs.metadata(path_info)
stat_result = None
with suppress(FileNotFoundError):
stat_result = tree.stat(path_info)
stat_result = fs.stat(path_info)
if not stat_result and not dvc_meta:
raise FileNotFoundError
@@ -421,9 +423,9 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
return meta
def info(self, path_info):
tree, dvc_tree = self._get_tree_pair(path_info)
fs, dvc_fs = self._get_fs_pair(path_info)
try:
return tree.info(path_info)
return fs.info(path_info)
except FileNotFoundError:
return dvc_tree.info(path_info)
return dvc_fs.info(path_info)
+2 -2
View File
@@ -13,14 +13,14 @@ from dvc.progress import Tqdm
from dvc.scheme import Schemes
from dvc.utils import conversions, error_link
from .base import BaseTree
from .base import BaseFileSystem
logger = logging.getLogger(__name__)
_AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
class S3Tree(BaseTree):
class S3FileSystem(BaseFileSystem):
scheme = Schemes.S3
PATH_CLS = CloudURLInfo
REQUIRES = {"boto3": "boto3"}
@@ -13,7 +13,7 @@ import dvc.prompt as prompt
from dvc.hash_info import HashInfo
from dvc.scheme import Schemes
from ..base import BaseTree
from ..base import BaseFileSystem
from ..pool import get_connection
logger = logging.getLogger(__name__)
@@ -30,7 +30,7 @@ def ask_password(host, user, port):
)
class SSHTree(BaseTree):
class SSHFileSystem(BaseFileSystem):
scheme = Schemes.SSH
REQUIRES = {"paramiko": "paramiko"}
JOBS = 4
@@ -101,7 +101,7 @@ class SSHTree(BaseTree):
def _load_user_ssh_config(hostname):
import paramiko
user_config_file = SSHTree.ssh_config_filename()
user_config_file = SSHFileSystem.ssh_config_filename()
user_ssh_config = {}
if hostname and os.path.exists(user_config_file):
ssh_config = paramiko.SSHConfig()
+4 -4
View File
@@ -12,7 +12,7 @@ from dvc.path_info import HTTPURLInfo, WebDAVURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from .base import BaseTree
from .base import BaseFileSystem
from .http import ask_password
logger = logging.getLogger(__name__)
@@ -23,7 +23,7 @@ class WebDAVConnectionError(DvcException):
super().__init__(f"Unable to connect to WebDAV {host}.")
class WebDAVTree(BaseTree): # pylint:disable=abstract-method
class WebDAVFileSystem(BaseFileSystem): # pylint:disable=abstract-method
# Use webdav scheme
scheme = Schemes.WEBDAV
@@ -47,7 +47,7 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
# Constructor
def __init__(self, repo, config):
# Call BaseTree constructor
# Call BaseFileSystem constructor
super().__init__(repo, config)
# Get username from configuration
@@ -235,7 +235,7 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
# Downloads file from remote to file
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
# Progress from HTTPTree
# Progress from HTTPFileSystem
with open(to_file, "wb") as fd:
with Tqdm.wrapattr(
fd,
+7
View File
@@ -0,0 +1,7 @@
from dvc.scheme import Schemes
from .webdav import WebDAVFileSystem
class WebDAVSFileSystem(WebDAVFileSystem): # pylint:disable=abstract-method
scheme = Schemes.WEBDAVS
+2 -2
View File
@@ -13,7 +13,7 @@ from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from .base import BaseTree
from .base import BaseFileSystem
logger = logging.getLogger(__name__)
@@ -30,7 +30,7 @@ def update_pbar(pbar, total):
return update
class WebHDFSTree(BaseTree):
class WebHDFSFileSystem(BaseFileSystem):
scheme = Schemes.WEBHDFS
PATH_CLS = CloudURLInfo
REQUIRES = {"hdfs": "hdfs"}
+20 -22
View File
@@ -49,13 +49,11 @@ class DvcIgnorePatterns(DvcIgnore):
]
@classmethod
def from_files(cls, ignore_file_path, tree):
def from_files(cls, ignore_file_path, fs):
assert os.path.isabs(ignore_file_path)
dirname = os.path.normpath(os.path.dirname(ignore_file_path))
ignore_file_rel_path = os.path.relpath(
ignore_file_path, tree.tree_root
)
with tree.open(ignore_file_path, encoding="utf-8") as fobj:
ignore_file_rel_path = os.path.relpath(ignore_file_path, fs.fs_root)
with fs.open(ignore_file_path, encoding="utf-8") as fobj:
path_spec_lines = [
PatternInfo(
line,
@@ -162,7 +160,7 @@ def _no_match(path):
class DvcIgnoreFilterNoop:
def __init__(self, tree, root_dir):
def __init__(self, fs, root_dir):
pass
def __call__(self, root, dirs, files, **kwargs):
@@ -182,15 +180,15 @@ class DvcIgnoreFilterNoop:
class DvcIgnoreFilter:
def __init__(self, tree, root_dir):
def __init__(self, fs, root_dir):
from dvc.repo import Repo
default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
self.tree = tree
self.fs = fs
self.root_dir = root_dir
self.ignores_trie_tree = PathStringTrie()
self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
self.ignores_trie_fs = PathStringTrie()
self.ignores_trie_fs[root_dir] = DvcIgnorePatterns(
default_ignore_patterns, root_dir
)
self._ignored_subrepos = PathStringTrie()
@@ -199,18 +197,18 @@ class DvcIgnoreFilter:
def _update(self, dirname):
self._update_sub_repo(dirname)
old_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
old_pattern = self.ignores_trie_fs.longest_prefix(dirname).value
matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
if not matches and self.tree.exists(
if not matches and self.fs.exists(
ignore_file_path, use_dvcignore=False
):
new_pattern = DvcIgnorePatterns.from_files(
ignore_file_path, self.tree
ignore_file_path, self.fs
)
if old_pattern:
self.ignores_trie_tree[dirname] = DvcIgnorePatterns(
self.ignores_trie_fs[dirname] = DvcIgnorePatterns(
*merge_patterns(
old_pattern.pattern_list,
old_pattern.dirname,
@@ -219,9 +217,9 @@ class DvcIgnoreFilter:
)
)
else:
self.ignores_trie_tree[dirname] = new_pattern
self.ignores_trie_fs[dirname] = new_pattern
elif old_pattern:
self.ignores_trie_tree[dirname] = old_pattern
self.ignores_trie_fs[dirname] = old_pattern
def _update_sub_repo(self, path):
from dvc.repo import Repo
@@ -239,9 +237,9 @@ class DvcIgnoreFilter:
) | {dname}
pattern_info = PatternInfo(f"/{dname}/", f"in sub_repo:{dname}")
new_pattern = DvcIgnorePatterns([pattern_info], root)
old_pattern = self.ignores_trie_tree.longest_prefix(root).value
old_pattern = self.ignores_trie_fs.longest_prefix(root).value
if old_pattern:
self.ignores_trie_tree[root] = DvcIgnorePatterns(
self.ignores_trie_fs[root] = DvcIgnorePatterns(
*merge_patterns(
old_pattern.pattern_list,
old_pattern.dirname,
@@ -250,7 +248,7 @@ class DvcIgnoreFilter:
)
)
else:
self.ignores_trie_tree[root] = new_pattern
self.ignores_trie_fs[root] = new_pattern
def __call__(self, root, dirs, files, ignore_subrepos=True):
for dname in dirs:
@@ -264,11 +262,11 @@ class DvcIgnoreFilter:
return dirs, files
def _get_trie_pattern(self, dirname):
ignore_pattern = self.ignores_trie_tree.get(dirname)
ignore_pattern = self.ignores_trie_fs.get(dirname)
if ignore_pattern:
return ignore_pattern
prefix = self.ignores_trie_tree.longest_prefix(dirname).key
prefix = self.ignores_trie_fs.longest_prefix(dirname).key
if not prefix:
# outside of the repo
return None
@@ -285,7 +283,7 @@ class DvcIgnoreFilter:
for parent in dirs:
self._update(parent)
return self.ignores_trie_tree.get(dirname)
return self.ignores_trie_fs.get(dirname)
def _is_ignored(self, path, is_dir=False):
if self._outside_repo(path):
+6 -6
View File
@@ -7,10 +7,10 @@ import uuid
import psutil
from dvc.exceptions import DvcException, NotDvcRepoError
from dvc.fs import ALL_FS, get_fs_cls, get_fs_config
from dvc.repo import Repo
from dvc.scm.base import SCMError
from dvc.system import System
from dvc.tree import TREES, get_tree_cls, get_tree_config
from dvc.utils import error_link
from dvc.utils.pkg import PKG
from dvc.version import __version__
@@ -75,7 +75,7 @@ def _get_caches(cache):
def _get_remotes(config):
schemes = (
get_tree_cls(get_tree_config(config, name=remote)).scheme
get_fs_cls(get_fs_config(config, name=remote)).scheme
for remote in config["remote"]
)
@@ -117,11 +117,11 @@ def _get_linktype_support_info(repo):
def _get_supported_remotes():
supported_remotes = []
for tree_cls in TREES:
if not tree_cls.get_missing_deps():
supported_remotes.append(tree_cls.scheme)
for fs_cls in ALL_FS:
if not fs_cls.get_missing_deps():
supported_remotes.append(fs_cls.scheme)
if len(supported_remotes) == len(TREES):
if len(supported_remotes) == len(ALL_FS):
return "All remotes"
if len(supported_remotes) == 1:
+2 -6
View File
@@ -22,16 +22,12 @@ def istextblock(block):
return float(len(nontext)) / len(block) <= 0.30
def istextfile(fname, blocksize=512, tree=None):
def istextfile(fname, fs, blocksize=512):
""" Uses heuristics to guess whether the given file is text or binary,
by reading a single block of bytes from the file.
If more than 30% of the chars in the block are non-text, or there
are NUL ('\x00') bytes in the block, assume this is a binary file.
"""
if tree:
open_func = tree.open
else:
open_func = open
with open_func(fname, "rb") as fobj:
with fs.open(fname, "rb") as fobj:
block = fobj.read(blocksize)
return istextblock(block)
+1 -1
View File
@@ -96,7 +96,7 @@ def main(argv=None): # noqa: C901
finally:
logger.setLevel(outerLogLevel)
from dvc.tree.pool import close_pools
from dvc.fs.pool import close_pools
# Closing pools by-hand to prevent weird messages when closing SSH
# connections. See https://github.com/iterative/dvc/issues/3248 for
+50 -58
View File
@@ -21,9 +21,9 @@ class ObjectFormatError(ObjectError):
class HashFile:
def __init__(self, path_info, tree, hash_info):
def __init__(self, path_info, fs, hash_info):
self.path_info = path_info
self.tree = tree
self.fs = fs
self.hash_info = hash_info
def __len__(self):
@@ -36,7 +36,7 @@ class HashFile:
return bool(self.hash_info)
def check(self, odb):
actual = get_hash(self.path_info, self.tree, odb.tree.PARAM_CHECKSUM)
actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM)
logger.trace(
"cache '%s' expected '%s' actual '%s'",
@@ -54,7 +54,7 @@ class HashFile:
return odb.get(hash_info)
def save(self, odb, **kwargs):
odb.add(self.path_info, self.tree, self.hash_info, **kwargs)
odb.add(self.path_info, self.fs, self.hash_info, **kwargs)
class File(HashFile):
@@ -63,13 +63,11 @@ class File(HashFile):
self.src = None
@classmethod
def stage(cls, odb, path_info, tree, **kwargs):
hash_info = get_hash(
path_info, tree, odb.tree.PARAM_CHECKSUM, **kwargs
)
def stage(cls, odb, path_info, fs, **kwargs):
hash_info = get_hash(path_info, fs, odb.fs.PARAM_CHECKSUM, **kwargs)
raw = odb.get(hash_info)
obj = cls(raw.path_info, raw.tree, hash_info)
obj.src = HashFile(path_info, tree, hash_info)
obj = cls(raw.path_info, raw.fs, hash_info)
obj.src = HashFile(path_info, fs, hash_info)
return obj
def save(self, odb, **kwargs):
@@ -81,7 +79,7 @@ class Tree(HashFile):
super().__init__(*args, **kwargs)
self.src_hash_info = None
self.src_path_info = None
self.src_tree = None
self.src_fs = None
def __len__(self):
return self.hash_info.dir_info.nfiles
@@ -95,7 +93,7 @@ class Tree(HashFile):
@classmethod
def save_dir_info(cls, odb, dir_info, hash_info=None):
if hash_info and hash_info.name == odb.tree.PARAM_CHECKSUM:
if hash_info and hash_info.name == odb.fs.PARAM_CHECKSUM:
try:
odb.check(hash_info)
assert hash_info.dir_info == dir_info
@@ -103,40 +101,38 @@ class Tree(HashFile):
except (FileNotFoundError, ObjectFormatError):
pass
from dvc.fs.memory import MemoryFileSystem
from dvc.path_info import PathInfo
from dvc.tree.memory import MemoryTree
from dvc.utils import tmp_fname
tree = MemoryTree(None, {})
fs = MemoryFileSystem(None, {})
path_info = PathInfo(tmp_fname(""))
with tree.open(path_info, "w") as fobj:
with fs.open(path_info, "w") as fobj:
json.dump(dir_info.to_list(), fobj, sort_keys=True)
tmp_info = odb.tree.path_info / tmp_fname("")
with tree.open(path_info, "rb") as fobj:
odb.tree.upload_fobj(fobj, tmp_info)
tmp_info = odb.fs.path_info / tmp_fname("")
with fs.open(path_info, "rb") as fobj:
odb.fs.upload_fobj(fobj, tmp_info)
hash_info = get_hash(tmp_info, odb.tree, odb.tree.PARAM_CHECKSUM)
hash_info.value += odb.tree.CHECKSUM_DIR_SUFFIX
hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM)
hash_info.value += odb.fs.CHECKSUM_DIR_SUFFIX
hash_info.dir_info = dir_info
hash_info.nfiles = dir_info.nfiles
odb.add(tmp_info, odb.tree, hash_info)
odb.add(tmp_info, odb.fs, hash_info)
return hash_info
@classmethod
def stage(cls, odb, path_info, tree, **kwargs):
hash_info = get_hash(
path_info, tree, odb.tree.PARAM_CHECKSUM, **kwargs
)
def stage(cls, odb, path_info, fs, **kwargs):
hash_info = get_hash(path_info, fs, odb.fs.PARAM_CHECKSUM, **kwargs)
hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info)
hi.size = hash_info.size
raw = odb.get(hi)
obj = cls(raw.path_info, raw.tree, hi)
obj = cls(raw.path_info, raw.fs, hi)
obj.src_hash_info = hash_info
obj.src_path_info = path_info
obj.src_tree = tree
obj.src_fs = fs
return obj
@classmethod
@@ -145,7 +141,7 @@ class Tree(HashFile):
obj = odb.get(hash_info)
try:
with obj.tree.open(obj.path_info, "r") as fobj:
with obj.fs.open(obj.path_info, "r") as fobj:
raw = json.load(fobj)
except ValueError as exc:
raise ObjectFormatError(f"{obj} is corrupted") from exc
@@ -161,7 +157,7 @@ class Tree(HashFile):
hash_info.dir_info = dir_info
hash_info.nfiles = dir_info.nfiles
return cls(obj.path_info, obj.tree, hash_info)
return cls(obj.path_info, obj.fs, hash_info)
def save(self, odb, **kwargs):
assert self.src_hash_info.dir_info
@@ -173,9 +169,9 @@ class Tree(HashFile):
desc="Saving " + self.src_path_info.name,
unit="file",
):
entry_obj = HashFile(entry_info, self.src_tree, entry_hash)
entry_obj = HashFile(entry_info, self.src_fs, entry_hash)
entry_obj.save(odb, **kwargs)
self.src_tree.state.save(self.src_path_info, hi)
self.src_fs.state.save(self.src_path_info, hi)
def filter(self, odb, prefix):
hash_info = self.hash_info.dir_info.trie.get(prefix)
@@ -193,10 +189,10 @@ class Tree(HashFile):
return load(odb, self.save_dir_info(odb, dir_info))
def stage(odb, path_info, tree, **kwargs):
if tree.isdir(path_info):
return Tree.stage(odb, path_info, tree, **kwargs)
return File.stage(odb, path_info, tree, **kwargs)
def stage(odb, path_info, fs, **kwargs):
if fs.isdir(path_info):
return Tree.stage(odb, path_info, fs, **kwargs)
return File.stage(odb, path_info, fs, **kwargs)
def save(odb, obj, **kwargs):
@@ -220,7 +216,7 @@ def load(odb, hash_info):
def _get_dir_size(odb, dir_info):
try:
return sum(
odb.tree.getsize(odb.hash_to_path_info(hi.value))
odb.fs.getsize(odb.hash_to_path_info(hi.value))
for _, hi in dir_info.items()
)
except FileNotFoundError:
@@ -245,22 +241,22 @@ def merge(odb, ancestor_info, our_info, their_info):
return hash_info
def _transfer_file(odb, from_tree, from_info):
def _transfer_file(odb, from_fs, from_info):
from dvc.utils import tmp_fname
from dvc.utils.stream import HashedStreamReader
tmp_info = odb.tree.path_info / tmp_fname()
with from_tree.open(
from_info, mode="rb", chunk_size=from_tree.CHUNK_SIZE
tmp_info = odb.fs.path_info / tmp_fname()
with from_fs.open(
from_info, mode="rb", chunk_size=from_fs.CHUNK_SIZE
) as stream:
stream_reader = HashedStreamReader(stream)
# Since we don't know the hash beforehand, we'll
# upload it to a temporary location and then move
# it.
odb.tree.upload_fobj(
odb.fs.upload_fobj(
stream_reader,
tmp_info,
total=from_tree.getsize(from_info),
total=from_fs.getsize(from_info),
desc=from_info.name,
)
@@ -268,15 +264,15 @@ def _transfer_file(odb, from_tree, from_info):
return tmp_info, hash_info
def _transfer_directory_contents(odb, from_tree, from_info, jobs, pbar):
def _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
rel_path_infos = {}
from_infos = from_tree.walk_files(from_info)
from_infos = from_fs.walk_files(from_info)
def create_tasks(executor, amount):
for entry_info in itertools.islice(from_infos, amount):
pbar.total += 1
task = executor.submit(
pbar.wrap_fn(_transfer_file), odb, from_tree, entry_info
pbar.wrap_fn(_transfer_file), odb, from_fs, entry_info
)
rel_path_infos[task] = entry_info.relative_to(from_info)
yield task
@@ -294,35 +290,31 @@ def _transfer_directory_contents(odb, from_tree, from_info, jobs, pbar):
yield rel_path_infos.pop(task), task.result()
def _transfer_directory(
odb, from_tree, from_info, jobs, no_progress_bar=False
):
def _transfer_directory(odb, from_fs, from_info, jobs, no_progress_bar=False):
dir_info = DirInfo()
with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar:
for (
entry_info,
(entry_tmp_info, entry_hash),
) in _transfer_directory_contents(
odb, from_tree, from_info, jobs, pbar
):
odb.add(entry_tmp_info, odb.tree, entry_hash)
) in _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
odb.add(entry_tmp_info, odb.fs, entry_hash)
dir_info.trie[entry_info.parts] = entry_hash
return Tree.save_dir_info(odb, dir_info)
def transfer(odb, from_tree, from_info, jobs=None, no_progress_bar=False):
jobs = jobs or min((from_tree.jobs, odb.tree.jobs))
def transfer(odb, from_fs, from_info, jobs=None, no_progress_bar=False):
jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
if from_tree.isdir(from_info):
if from_fs.isdir(from_info):
return _transfer_directory(
odb,
from_tree,
from_fs,
from_info,
jobs=jobs,
no_progress_bar=no_progress_bar,
)
tmp_info, hash_info = _transfer_file(odb, from_tree, from_info)
odb.add(tmp_info, odb.tree, hash_info)
tmp_info, hash_info = _transfer_file(odb, from_fs, from_info)
odb.add(tmp_info, odb.fs, hash_info)
return hash_info
+32 -32
View File
@@ -9,27 +9,27 @@ from dvc.progress import Tqdm
from dvc.utils import file_md5
def get_file_hash(path_info, tree, name):
info = tree.info(path_info)
def get_file_hash(path_info, fs, name):
info = fs.info(path_info)
if name in info:
assert not info[name].endswith(".dir")
return HashInfo(name, info[name], size=info["size"])
func = getattr(tree, name, None)
func = getattr(fs, name, None)
if func:
return func(path_info)
if name == "md5":
return HashInfo(
name, file_md5(path_info, tree), size=tree.getsize(path_info)
name, file_md5(path_info, fs), size=fs.getsize(path_info)
)
raise NotImplementedError
def _calculate_hashes(file_infos, tree, name):
def _calculate_hashes(file_infos, fs, name):
def _get_file_hash(path_info):
return get_file_hash(path_info, tree, name)
return get_file_hash(path_info, fs, name)
with Tqdm(
total=len(file_infos),
@@ -37,14 +37,14 @@ def _calculate_hashes(file_infos, tree, name):
desc="Computing file/dir hashes (only done once)",
) as pbar:
worker = pbar.wrap_fn(_get_file_hash)
with ThreadPoolExecutor(max_workers=tree.hash_jobs) as executor:
with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor:
hash_infos = executor.map(worker, file_infos)
return dict(zip(file_infos, hash_infos))
def _iter_hashes(path_info, tree, name, **kwargs):
if name in tree.DETAIL_FIELDS:
for details in tree.ls(path_info, recursive=True, detail=True):
def _iter_hashes(path_info, fs, name, **kwargs):
if name in fs.DETAIL_FIELDS:
for details in fs.ls(path_info, recursive=True, detail=True):
file_info = path_info.replace(path=details["name"])
hash_info = HashInfo(
name, details[name], size=details.get("size"),
@@ -54,8 +54,8 @@ def _iter_hashes(path_info, tree, name, **kwargs):
return None
file_infos = []
for file_info in tree.walk_files(path_info, **kwargs):
hash_info = tree.state.get( # pylint: disable=assignment-from-none
for file_info in fs.walk_files(path_info, **kwargs):
hash_info = fs.state.get( # pylint: disable=assignment-from-none
file_info
)
if not hash_info:
@@ -64,14 +64,14 @@ def _iter_hashes(path_info, tree, name, **kwargs):
assert hash_info.name == name
yield file_info, hash_info
yield from _calculate_hashes(file_infos, tree, name).items()
yield from _calculate_hashes(file_infos, fs, name).items()
def _collect_dir(path_info, tree, name, **kwargs):
def _collect_dir(path_info, fs, name, **kwargs):
from dvc.dir_info import DirInfo
dir_info = DirInfo()
for fi, hi in _iter_hashes(path_info, tree, name, **kwargs):
for fi, hi in _iter_hashes(path_info, fs, name, **kwargs):
if DvcIgnore.DVCIGNORE_FILE == fi.name:
raise DvcIgnoreInCollectedDirError(fi.parent)
@@ -88,29 +88,29 @@ def _collect_dir(path_info, tree, name, **kwargs):
return dir_info
def get_dir_hash(path_info, tree, name, **kwargs):
def get_dir_hash(path_info, fs, name, **kwargs):
from dvc.objects import Tree
dir_info = _collect_dir(path_info, tree, name, **kwargs)
hash_info = Tree.save_dir_info(tree.repo.cache.local, dir_info)
dir_info = _collect_dir(path_info, fs, name, **kwargs)
hash_info = Tree.save_dir_info(fs.repo.cache.local, dir_info)
hash_info.size = dir_info.size
hash_info.dir_info = dir_info
return hash_info
def get_hash(path_info, tree, name, **kwargs):
def get_hash(path_info, fs, name, **kwargs):
assert path_info and (
isinstance(path_info, str) or path_info.scheme == tree.scheme
isinstance(path_info, str) or path_info.scheme == fs.scheme
)
if not tree.exists(path_info):
if not fs.exists(path_info):
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), path_info
)
with tree.state:
with fs.state:
# pylint: disable=assignment-from-none
hash_info = tree.state.get(path_info)
hash_info = fs.state.get(path_info)
# If we have dir hash in state db, but dir cache file is lost,
# then we need to recollect the dir via .get_dir_hash() call below,
@@ -118,8 +118,8 @@ def get_hash(path_info, tree, name, **kwargs):
if (
hash_info
and hash_info.isdir
and not tree.cache.tree.exists(
tree.cache.hash_to_path_info(hash_info.value)
and not fs.cache.fs.exists(
fs.cache.hash_to_path_info(hash_info.value)
)
):
hash_info = None
@@ -128,17 +128,17 @@ def get_hash(path_info, tree, name, **kwargs):
if hash_info.isdir:
from dvc.objects import Tree
# NOTE: loading the tree will restore hash_info.dir_info
Tree.load(tree.cache, hash_info)
# NOTE: loading the fs will restore hash_info.dir_info
Tree.load(fs.cache, hash_info)
assert hash_info.name == name
return hash_info
if tree.isdir(path_info):
hash_info = get_dir_hash(path_info, tree, name, **kwargs)
if fs.isdir(path_info):
hash_info = get_dir_hash(path_info, fs, name, **kwargs)
else:
hash_info = get_file_hash(path_info, tree, name)
hash_info = get_file_hash(path_info, fs, name)
if hash_info and tree.exists(path_info):
tree.state.save(path_info, hash_info)
if hash_info and fs.exists(path_info):
fs.state.save(path_info, hash_info)
return hash_info
+14 -14
View File
@@ -14,11 +14,11 @@ from dvc.output.ssh import SSHOutput
from dvc.output.webhdfs import WebHDFSOutput
from dvc.scheme import Schemes
from ..tree import get_cloud_tree
from ..tree.hdfs import HDFSTree
from ..tree.local import LocalTree
from ..tree.s3 import S3Tree
from ..tree.webhdfs import WebHDFSTree
from ..fs import get_cloud_fs
from ..fs.hdfs import HDFSFileSystem
from ..fs.local import LocalFileSystem
from ..fs.s3 import S3FileSystem
from ..fs.webhdfs import WebHDFSFileSystem
OUTS = [
HDFSOutput,
@@ -53,10 +53,10 @@ CHECKSUM_SCHEMA = Any(
# so when a few types of outputs share the same name, we only need
# specify it once.
CHECKSUMS_SCHEMA = {
LocalTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
S3Tree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
HDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
WebHDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
LocalFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
S3FileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
HDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
WebHDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
}
SCHEMA = CHECKSUMS_SCHEMA.copy()
@@ -88,13 +88,13 @@ def _get(
parsed = urlparse(p)
if parsed.scheme == "remote":
tree = get_cloud_tree(stage.repo, name=parsed.netloc)
return OUTS_MAP[tree.scheme](
fs = get_cloud_fs(stage.repo, name=parsed.netloc)
return OUTS_MAP[fs.scheme](
stage,
p,
info,
cache=cache,
tree=tree,
fs=fs,
metric=metric,
plot=plot,
persist=persist,
@@ -111,7 +111,7 @@ def _get(
p,
info,
cache=cache,
tree=None,
fs=None,
metric=metric,
plot=plot,
persist=persist,
@@ -125,7 +125,7 @@ def _get(
p,
info,
cache=cache,
tree=None,
fs=None,
metric=metric,
plot=plot,
persist=persist,
+30 -32
View File
@@ -20,7 +20,7 @@ from dvc.exceptions import (
from dvc.hash_info import HashInfo
from dvc.oid import get_hash
from ..tree.base import BaseTree
from ..fs.base import BaseFileSystem
logger = logging.getLogger(__name__)
@@ -61,7 +61,7 @@ class OutputIsIgnoredError(DvcException):
class BaseOutput:
IS_DEPENDENCY = False
TREE_CLS = BaseTree
FS_CLS = BaseFileSystem
PARAM_PATH = "path"
PARAM_CACHE = "cache"
@@ -105,7 +105,7 @@ class BaseOutput:
stage,
path,
info=None,
tree=None,
fs=None,
cache=True,
metric=False,
plot=False,
@@ -130,10 +130,10 @@ class BaseOutput:
self.repo = stage.repo if stage else None
self.def_path = path
self.hash_info = HashInfo.from_dict(info)
if tree:
self.tree = tree
if fs:
self.fs = fs
else:
self.tree = self.TREE_CLS(self.repo, {})
self.fs = self.FS_CLS(self.repo, {})
self.use_cache = False if self.IS_DEPENDENCY else cache
self.metric = False if self.IS_DEPENDENCY else metric
self.plot = False if self.IS_DEPENDENCY else plot
@@ -142,18 +142,18 @@ class BaseOutput:
self.live = live
self.desc = desc
self.path_info = self._parse_path(tree, path)
self.path_info = self._parse_path(fs, path)
if self.use_cache and self.cache is None:
raise RemoteCacheRequiredError(self.path_info)
self.obj = None
self.isexec = False if self.IS_DEPENDENCY else isexec
def _parse_path(self, tree, path):
if tree:
def _parse_path(self, fs, path):
if fs:
parsed = urlparse(path)
return tree.path_info / parsed.path.lstrip("/")
return self.TREE_CLS.PATH_CLS(path)
return fs.path_info / parsed.path.lstrip("/")
return self.FS_CLS.PATH_CLS(path)
def __repr__(self):
return "{class_name}: '{def_path}'".format(
@@ -165,7 +165,7 @@ class BaseOutput:
@property
def scheme(self):
return self.TREE_CLS.scheme
return self.FS_CLS.scheme
@property
def is_in_repo(self):
@@ -188,7 +188,7 @@ class BaseOutput:
@classmethod
def supported(cls, url):
return cls.TREE_CLS.supported(url)
return cls.FS_CLS.supported(url)
@property
def cache_path(self):
@@ -196,10 +196,8 @@ class BaseOutput:
def get_hash(self):
if not self.use_cache:
return get_hash(
self.path_info, self.tree, self.tree.PARAM_CHECKSUM
)
return objects.stage(self.cache, self.path_info, self.tree).hash_info
return get_hash(self.path_info, self.fs, self.fs.PARAM_CHECKSUM)
return objects.stage(self.cache, self.path_info, self.fs).hash_info
@property
def is_dir_checksum(self):
@@ -207,7 +205,7 @@ class BaseOutput:
@property
def exists(self):
return self.tree.exists(self.path_info)
return self.fs.exists(self.path_info)
def changed_checksum(self):
return self.hash_info != self.get_hash()
@@ -251,13 +249,13 @@ class BaseOutput:
@property
def is_empty(self):
return self.tree.is_empty(self.path_info)
return self.fs.is_empty(self.path_info)
def isdir(self):
return self.tree.isdir(self.path_info)
return self.fs.isdir(self.path_info)
def isfile(self):
return self.tree.isfile(self.path_info)
return self.fs.isfile(self.path_info)
# pylint: disable=no-member
@@ -307,9 +305,9 @@ class BaseOutput:
logger.debug("Output '%s' didn't change. Skipping saving.", self)
return
self.obj = objects.stage(self.cache, self.path_info, self.tree)
self.obj = objects.stage(self.cache, self.path_info, self.fs)
self.hash_info = self.obj.hash_info
self.isexec = self.isfile() and self.tree.isexec(self.path_info)
self.isexec = self.isfile() and self.fs.isexec(self.path_info)
def set_exec(self):
if self.isfile() and self.isexec:
@@ -323,12 +321,12 @@ class BaseOutput:
if self.use_cache:
obj = objects.stage(
self.cache, filter_info or self.path_info, self.tree
self.cache, filter_info or self.path_info, self.fs
)
objects.save(self.cache, obj)
checkout(
filter_info or self.path_info,
self.tree,
self.fs,
obj,
self.cache,
relink=True,
@@ -379,7 +377,7 @@ class BaseOutput:
raise DvcException(f"verify metric is not supported for {self.scheme}")
def download(self, to, jobs=None):
self.tree.download(self.path_info, to.path_info, jobs=jobs)
self.fs.download(self.path_info, to.path_info, jobs=jobs)
def get_obj(self, filter_info=None):
if self.obj:
@@ -424,7 +422,7 @@ class BaseOutput:
try:
modified = checkout(
filter_info or self.path_info,
self.tree,
self.fs,
obj,
self.cache,
force=force,
@@ -440,7 +438,7 @@ class BaseOutput:
return added, False if added else modified
def remove(self, ignore_remove=False):
self.tree.remove(self.path_info)
self.fs.remove(self.path_info)
if self.scheme != "local":
return
@@ -452,7 +450,7 @@ class BaseOutput:
if self.scheme == "local" and self.use_scm_ignore:
self.repo.scm.ignore_remove(self.fspath)
self.tree.move(self.path_info, out.path_info)
self.fs.move(self.path_info, out.path_info)
self.def_path = out.def_path
self.path_info = out.path_info
self.save()
@@ -624,8 +622,8 @@ class BaseOutput:
if stage:
abs_path = os.path.join(stage.wdir, path)
if stage.repo.tree.dvcignore.is_ignored(abs_path):
check = stage.repo.tree.dvcignore.check_ignore(abs_path)
if stage.repo.fs.dvcignore.is_ignored(abs_path):
check = stage.repo.fs.dvcignore.check_ignore(abs_path)
raise cls.IsIgnoredError(check)
def _check_can_merge(self, out):
@@ -636,7 +634,7 @@ class BaseOutput:
other = out.dumpd()
ignored = [
self.tree.PARAM_CHECKSUM,
self.fs.PARAM_CHECKSUM,
HashInfo.PARAM_SIZE,
HashInfo.PARAM_NFILES,
]
+2 -2
View File
@@ -1,7 +1,7 @@
from dvc.output.base import BaseOutput
from ..tree.gs import GSTree
from ..fs.gs import GSFileSystem
class GSOutput(BaseOutput):
TREE_CLS = GSTree
FS_CLS = GSFileSystem
+2 -2
View File
@@ -1,7 +1,7 @@
from dvc.output.base import BaseOutput
from ..tree.hdfs import HDFSTree
from ..fs.hdfs import HDFSFileSystem
class HDFSOutput(BaseOutput):
TREE_CLS = HDFSTree
FS_CLS = HDFSFileSystem
+9 -9
View File
@@ -8,13 +8,13 @@ from dvc.output.base import BaseOutput
from dvc.utils import relpath
from dvc.utils.fs import path_isin
from ..tree.local import LocalTree
from ..fs.local import LocalFileSystem
logger = logging.getLogger(__name__)
class LocalOutput(BaseOutput):
TREE_CLS = LocalTree
FS_CLS = LocalFileSystem
sep = os.sep
def __init__(self, stage, path, *args, **kwargs):
@@ -25,14 +25,14 @@ class LocalOutput(BaseOutput):
if (
self.is_in_repo
and self.repo
and isinstance(self.repo.tree, LocalTree)
and isinstance(self.repo.fs, LocalFileSystem)
):
self.tree = self.repo.tree
self.fs = self.repo.fs
def _parse_path(self, tree, path):
def _parse_path(self, fs, path):
parsed = urlparse(path)
if parsed.scheme == "remote":
p = tree.path_info / parsed.path.lstrip("/")
p = fs.path_info / parsed.path.lstrip("/")
else:
# NOTE: we can path either from command line or .dvc file,
# so we should expect both posix and windows style paths.
@@ -40,12 +40,12 @@ class LocalOutput(BaseOutput):
#
# FIXME: if we have Windows path containing / or posix one with \
# then we have #2059 bug and can't really handle that.
p = self.TREE_CLS.PATH_CLS(path)
p = self.FS_CLS.PATH_CLS(path)
if self.stage and not p.is_absolute():
p = self.stage.wdir / p
abs_p = os.path.abspath(os.path.normpath(p))
return self.TREE_CLS.PATH_CLS(abs_p)
return self.FS_CLS.PATH_CLS(abs_p)
def __str__(self):
if not self.repo or not self.is_in_repo:
@@ -91,6 +91,6 @@ class LocalOutput(BaseOutput):
logger.debug(msg, str(self.path_info), name)
return
if not istextfile(path):
if not istextfile(path, self.fs):
msg = "binary file '{}' cannot be used as {}."
raise DvcException(msg.format(self.path_info, name))
+2 -2
View File
@@ -1,7 +1,7 @@
from dvc.output.base import BaseOutput
from ..tree.s3 import S3Tree
from ..fs.s3 import S3FileSystem
class S3Output(BaseOutput):
TREE_CLS = S3Tree
FS_CLS = S3FileSystem
+2 -2
View File
@@ -1,7 +1,7 @@
from dvc.output.base import BaseOutput
from ..tree.ssh import SSHTree
from ..fs.ssh import SSHFileSystem
class SSHOutput(BaseOutput):
TREE_CLS = SSHTree
FS_CLS = SSHFileSystem
+2 -2
View File
@@ -1,7 +1,7 @@
from dvc.output.base import BaseOutput
from ..tree.webhdfs import WebHDFSTree
from ..fs.webhdfs import WebHDFSFileSystem
class WebHDFSOutput(BaseOutput):
TREE_CLS = WebHDFSTree
FS_CLS = WebHDFSFileSystem
+4 -4
View File
@@ -135,7 +135,7 @@ def make_definition(
class DataResolver:
def __init__(self, repo: "Repo", wdir: PathInfo, d: dict):
self.tree = tree = repo.tree
self.fs = fs = repo.fs
self.wdir = wdir
self.relpath = relpath(self.wdir / "dvc.yaml")
@@ -144,7 +144,7 @@ class DataResolver:
self.context: Context = Context()
try:
args = tree, vars_, wdir # load from `vars` section
args = fs, vars_, wdir # load from `vars` section
self.context.load_from_vars(*args, default=DEFAULT_PARAMS_FILE)
except ContextError as exc:
format_and_raise(exc, "'vars'", self.relpath)
@@ -270,8 +270,8 @@ class EntryDefinition:
context = Context.clone(context)
try:
tree = self.resolver.tree
context.load_from_vars(tree, vars_, wdir, stage_name=name)
fs = self.resolver.fs
context.load_from_vars(fs, vars_, wdir, stage_name=name)
except VarsAlreadyLoaded as exc:
format_and_raise(exc, f"'{self.where}.{name}.vars'", self.relpath)
+10 -10
View File
@@ -356,20 +356,20 @@ class Context(CtxDict):
@classmethod
def load_from(
cls, tree, path: PathInfo, select_keys: List[str] = None
cls, fs, path: PathInfo, select_keys: List[str] = None
) -> "Context":
from dvc.utils.serialize import LOADERS
file = relpath(path)
if not tree.exists(path):
if not fs.exists(path):
raise ParamsLoadError(f"'{file}' does not exist")
if tree.isdir(path):
if fs.isdir(path):
raise ParamsLoadError(f"'{file}' is a directory")
_, ext = os.path.splitext(file)
loader = LOADERS[ext]
data = loader(path, tree=tree)
data = loader(path, fs=fs)
if not isinstance(data, Mapping):
typ = type(data).__name__
raise ParamsLoadError(
@@ -397,7 +397,7 @@ class Context(CtxDict):
return super().merge_update(other, overwrite=overwrite)
def merge_from(
self, tree, item: str, wdir: PathInfo, overwrite=False,
self, fs, item: str, wdir: PathInfo, overwrite=False,
):
path, _, keys_str = item.partition(":")
select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
@@ -409,7 +409,7 @@ class Context(CtxDict):
return # allow specifying complete filepath multiple times
self.check_loaded(abspath, item, select_keys)
ctx = Context.load_from(tree, path_info, select_keys)
ctx = Context.load_from(fs, path_info, select_keys)
try:
self.merge_update(ctx, overwrite=overwrite)
@@ -439,7 +439,7 @@ class Context(CtxDict):
def load_from_vars(
self,
tree,
fs,
vars_: List,
wdir: PathInfo,
stage_name: str = None,
@@ -447,8 +447,8 @@ class Context(CtxDict):
):
if default:
to_import = wdir / default
if tree.exists(to_import):
self.merge_from(tree, default, wdir)
if fs.exists(to_import):
self.merge_from(fs, default, wdir)
else:
msg = "%s does not exist, it won't be used in parametrization"
logger.trace(msg, to_import) # type: ignore[attr-defined]
@@ -457,7 +457,7 @@ class Context(CtxDict):
for index, item in enumerate(vars_):
assert isinstance(item, (str, dict))
if isinstance(item, str):
self.merge_from(tree, item, wdir)
self.merge_from(fs, item, wdir)
else:
joiner = "." if stage_name else ""
meta = Meta(source=f"{stage_name}{joiner}vars[{index}]")
+5 -5
View File
@@ -1,10 +1,10 @@
from ..tree import get_cloud_tree
from ..fs import get_cloud_fs
from .base import Remote
from .local import LocalRemote
def get_remote(repo, **kwargs):
tree = get_cloud_tree(repo, **kwargs)
if tree.scheme == "local":
return LocalRemote(tree)
return Remote(tree)
fs = get_cloud_fs(repo, **kwargs)
if fs.scheme == "local":
return LocalRemote(fs)
return Remote(fs)
+30 -34
View File
@@ -61,25 +61,25 @@ def index_locked(f):
class Remote:
"""Cloud remote class.
Provides methods for indexing and garbage collecting trees which contain
Provides methods for indexing and garbage collecting fss which contain
DVC remotes.
"""
INDEX_CLS = RemoteIndex
def __init__(self, tree):
def __init__(self, fs):
from dvc.cache import get_cloud_cache
self.tree = tree
self.repo = tree.repo
self.cache = get_cloud_cache(self.tree)
self.fs = fs
self.repo = fs.repo
self.cache = get_cloud_cache(self.fs)
config = tree.config
config = fs.config
url = config.get("url")
if url:
index_name = hashlib.sha256(url.encode("utf-8")).hexdigest()
self.index = self.INDEX_CLS(
self.repo, index_name, dir_suffix=self.tree.CHECKSUM_DIR_SUFFIX
self.repo, index_name, dir_suffix=self.fs.CHECKSUM_DIR_SUFFIX
)
else:
self.index = RemoteIndexNoop()
@@ -87,7 +87,7 @@ class Remote:
def __repr__(self):
return "{class_name}: '{path_info}'".format(
class_name=type(self).__name__,
path_info=self.tree.path_info or "No path",
path_info=self.fs.path_info or "No path",
)
@index_locked
@@ -147,8 +147,8 @@ class Remote:
{dir_hash: set(file_hash, ...)} which can be used to map
a .dir file to its file contents.
"""
logger.debug(f"Preparing to collect status from {self.tree.path_info}")
md5s = set(named_cache.scheme_keys(cache.tree.scheme))
logger.debug(f"Preparing to collect status from {self.fs.path_info}")
md5s = set(named_cache.scheme_keys(cache.fs.scheme))
logger.debug("Collecting information from local cache...")
local_exists = frozenset(
@@ -164,7 +164,7 @@ class Remote:
else:
logger.debug("Collecting information from remote cache...")
remote_exists = set()
dir_md5s = set(named_cache.dir_keys(cache.tree.scheme))
dir_md5s = set(named_cache.dir_keys(cache.fs.scheme))
if dir_md5s:
remote_exists.update(
self._indexed_dir_hashes(cache, named_cache, dir_md5s)
@@ -173,7 +173,7 @@ class Remote:
if md5s:
remote_exists.update(
self.hashes_exist(
md5s, jobs=jobs, name=str(self.tree.path_info)
md5s, jobs=jobs, name=str(self.fs.path_info)
)
)
return self._make_status(
@@ -200,7 +200,7 @@ class Remote:
dir_status = {}
file_status = {}
dir_contents = {}
for hash_, item in named_cache[cache.tree.scheme].items():
for hash_, item in named_cache[cache.fs.scheme].items():
if item.children:
dir_status[hash_] = make_names(hash_, item.names)
dir_contents[hash_] = set()
@@ -245,7 +245,7 @@ class Remote:
# still exists on the remote
for dir_hash in dir_exists:
file_hashes = list(
named_cache.child_keys(cache.tree.scheme, dir_hash)
named_cache.child_keys(cache.fs.scheme, dir_hash)
)
if dir_hash not in self.index:
logger.debug(
@@ -304,21 +304,21 @@ class Remote:
logger.debug(
"Preparing to {} '{}'".format(
"download data from" if download else "upload data to",
self.tree.path_info,
self.fs.path_info,
)
)
if download:
func = _log_exceptions(self.tree.download, "download")
func = _log_exceptions(self.fs.download, "download")
status = STATUS_DELETED
desc = "Downloading"
else:
func = _log_exceptions(self.tree.upload, "upload")
func = _log_exceptions(self.fs.upload, "upload")
status = STATUS_NEW
desc = "Uploading"
if jobs is None:
jobs = self.tree.JOBS
jobs = self.fs.JOBS
dir_status, file_status, dir_contents = self._status(
cache,
@@ -471,15 +471,13 @@ class Remote:
download=False,
)
if self.tree.scheme == "local":
with self.tree.state:
if self.fs.scheme == "local":
with self.fs.state:
for checksum in named_cache.scheme_keys("local"):
cache_file = self.cache.hash_to_path_info(checksum)
if self.tree.exists(cache_file):
hash_info = HashInfo(
self.tree.PARAM_CHECKSUM, checksum
)
self.tree.state.save(cache_file, hash_info)
if self.fs.exists(cache_file):
hash_info = HashInfo(self.fs.PARAM_CHECKSUM, checksum)
self.fs.state.save(cache_file, hash_info)
self.cache.protect(cache_file)
return ret
@@ -495,28 +493,26 @@ class Remote:
)
if not self.cache.verify:
with cache.tree.state:
with cache.fs.state:
for checksum in named_cache.scheme_keys("local"):
cache_file = cache.hash_to_path_info(checksum)
if cache.tree.exists(cache_file):
if cache.fs.exists(cache_file):
# We can safely save here, as existing corrupted files
# will be removed upon status, while files corrupted
# during download will not be moved from tmp_file
# (see `BaseTree.download()`)
hash_info = HashInfo(
cache.tree.PARAM_CHECKSUM, checksum
)
cache.tree.state.save(cache_file, hash_info)
# (see `BaseFileSystem.download()`)
hash_info = HashInfo(cache.fs.PARAM_CHECKSUM, checksum)
cache.fs.state.save(cache_file, hash_info)
cache.protect(cache_file)
return ret
def transfer(self, from_tree, from_info, jobs=None, no_progress_bar=False):
def transfer(self, from_fs, from_info, jobs=None, no_progress_bar=False):
from dvc.objects import transfer
return transfer(
self.cache,
from_tree,
from_fs,
from_info,
jobs=jobs,
no_progress_bar=no_progress_bar,
+30 -28
View File
@@ -16,8 +16,8 @@ from .graph import build_graph, build_outs_graph, get_pipelines
from .trie import build_outs_trie
if TYPE_CHECKING:
from dvc.fs.base import BaseFileSystem
from dvc.scm import Base
from dvc.tree.base import BaseTree
logger = logging.getLogger(__name__)
@@ -99,8 +99,8 @@ class Repo:
dvc_dir = None
tmp_dir = None
try:
tree = scm.get_tree(rev) if isinstance(scm, Git) and rev else None
root_dir = self.find_root(root_dir, tree)
fs = scm.get_fs(rev) if isinstance(scm, Git) and rev else None
root_dir = self.find_root(root_dir, fs)
dvc_dir = os.path.join(root_dir, self.DVC_DIR)
tmp_dir = os.path.join(dvc_dir, "tmp")
makedirs(tmp_dir, exist_ok=True)
@@ -134,6 +134,7 @@ class Repo:
from dvc.cache import Cache
from dvc.config import Config
from dvc.data_cloud import DataCloud
from dvc.fs.local import LocalFileSystem
from dvc.lock import LockNoop, make_lock
from dvc.repo.live import Live
from dvc.repo.metrics import Metrics
@@ -142,10 +143,9 @@ class Repo:
from dvc.repo.stage import StageLoad
from dvc.stage.cache import StageCache
from dvc.state import State, StateNoop
from dvc.tree.local import LocalTree
self.url = url
self._tree_conf = {
self._fs_conf = {
"repo_factory": repo_factory,
}
@@ -153,17 +153,19 @@ class Repo:
root_dir=root_dir, scm=scm, rev=rev, uninitialized=uninitialized
)
tree_kwargs = {"use_dvcignore": True, "dvcignore_root": self.root_dir}
fs_kwargs = {"use_dvcignore": True, "dvcignore_root": self.root_dir}
if scm:
self._tree = scm.get_tree(rev, **tree_kwargs)
self._fs = scm.get_fs(rev, **fs_kwargs)
else:
self._tree = LocalTree(self, {"url": self.root_dir}, **tree_kwargs)
self._fs = LocalFileSystem(
self, {"url": self.root_dir}, **fs_kwargs
)
self.config = Config(self.dvc_dir, tree=self.tree, config=config)
self.config = Config(self.dvc_dir, fs=self.fs, config=config)
self._uninitialized = uninitialized
self._scm = scm
# used by RepoTree to determine if it should traverse subrepos
# used by RepoFileSystem to determine if it should traverse subrepos
self.subrepos = subrepos
self.cache = Cache(self)
@@ -239,12 +241,12 @@ class Repo:
raise
def get_rev(self):
from dvc.tree.local import LocalTree
from dvc.fs.local import LocalFileSystem
assert self.scm
if isinstance(self.tree, LocalTree):
if isinstance(self.fs, LocalFileSystem):
return self.scm.get_rev()
return self.tree.rev
return self.fs.rev
@cached_property
def experiments(self):
@@ -253,25 +255,25 @@ class Repo:
return Experiments(self)
@property
def tree(self) -> "BaseTree":
return self._tree
def fs(self) -> "BaseFileSystem":
return self._fs
@tree.setter
def tree(self, tree: "BaseTree"):
self._tree = tree
@fs.setter
def fs(self, fs: "BaseFileSystem"):
self._fs = fs
# Our graph cache is no longer valid, as it was based on the previous
# tree.
# fs.
self._reset()
def __repr__(self):
return f"{self.__class__.__name__}: '{self.root_dir}'"
@classmethod
def find_root(cls, root=None, tree=None) -> str:
def find_root(cls, root=None, fs=None) -> str:
root_dir = os.path.realpath(root or os.curdir)
if tree:
if tree.isdir(os.path.join(root_dir, cls.DVC_DIR)):
if fs:
if fs.isdir(os.path.join(root_dir, cls.DVC_DIR)):
return root_dir
raise NotDvcRepoError(f"'{root}' does not contain DVC directory")
@@ -462,21 +464,21 @@ class Repo:
return self.DVC_DIR in path_parts
@cached_property
def repo_tree(self):
from dvc.tree.repo import RepoTree
def repo_fs(self):
from dvc.fs.repo import RepoFileSystem
return RepoTree(self, subrepos=self.subrepos, **self._tree_conf)
return RepoFileSystem(self, subrepos=self.subrepos, **self._fs_conf)
@contextmanager
def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
"""Opens a specified resource as a file descriptor"""
from dvc.tree.repo import RepoTree
from dvc.fs.repo import RepoFileSystem
tree = RepoTree(self, subrepos=True)
fs = RepoFileSystem(self, subrepos=True)
path = PathInfo(self.root_dir) / path
try:
with self.state:
with tree.open(
with fs.open(
path, mode=mode, encoding=encoding, remote=remote,
) as fobj:
yield fobj
+4 -7
View File
@@ -165,15 +165,12 @@ def _process_stages(
command="add",
)
else:
from dvc.fs import get_cloud_fs
from dvc.objects import transfer
from dvc.tree import get_cloud_tree
from_tree = get_cloud_tree(repo, url=target)
from_fs = get_cloud_fs(repo, url=target)
out.hash_info = transfer(
out.cache,
from_tree,
from_tree.path_info,
jobs=kwargs.get("jobs"),
out.cache, from_fs, from_fs.path_info, jobs=kwargs.get("jobs"),
)
out.checkout()
@@ -212,7 +209,7 @@ def _find_all_targets(repo, target, recursive):
return [
os.fspath(path)
for path in Tqdm(
repo.tree.walk_files(target),
repo.fs.walk_files(target),
desc="Searching " + target,
bar_format=Tqdm.BAR_FMT_NOTOTAL,
unit="file",
+7 -7
View File
@@ -20,7 +20,7 @@ def brancher( # noqa: E302
sha_only (bool): only return git SHA for a revision.
Yields:
str: the display name for the currently selected tree, it could be:
str: the display name for the currently selected fs, it could be:
- a git revision identifier
- empty string it there is no branches to iterate over
- "workspace" if there are uncommitted changes in the SCM repo
@@ -29,14 +29,14 @@ def brancher( # noqa: E302
yield ""
return
from dvc.tree.local import LocalTree
from dvc.fs.local import LocalFileSystem
saved_tree = self.tree
saved_fs = self.fs
revs = revs.copy() if revs else []
scm = self.scm
self.tree = LocalTree(self, {"url": self.root_dir}, use_dvcignore=True)
self.fs = LocalFileSystem(self, {"url": self.root_dir}, use_dvcignore=True)
yield "workspace"
if revs and "workspace" in revs:
@@ -59,15 +59,15 @@ def brancher( # noqa: E302
try:
if revs:
for sha, names in group_by(scm.resolve_rev, revs).items():
self.tree = scm.get_tree(
self.fs = scm.get_fs(
sha, use_dvcignore=True, dvcignore_root=self.root_dir
)
# ignore revs that don't contain repo root
# (i.e. revs from before a subdir=True repo was init'ed)
if self.tree.exists(self.root_dir):
if self.fs.exists(self.root_dir):
if sha_only:
yield sha
else:
yield ", ".join(names)
finally:
self.tree = saved_tree
self.fs = saved_fs
+5 -5
View File
@@ -34,18 +34,18 @@ def _collect_paths(
recursive: bool = False,
rev: str = None,
):
from dvc.tree.repo import RepoTree
from dvc.fs.repo import RepoFileSystem
path_infos = [PathInfo(os.path.abspath(target)) for target in targets]
tree = RepoTree(repo)
fs = RepoFileSystem(repo)
target_infos = []
for path_info in path_infos:
if recursive and tree.isdir(path_info):
target_infos.extend(tree.walk_files(path_info))
if recursive and fs.isdir(path_info):
target_infos.extend(fs.walk_files(path_info))
if not tree.exists(path_info):
if not fs.exists(path_info):
if not recursive:
if rev == "workspace" or rev == "":
logger.warning(
+25 -27
View File
@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
@locked
def diff(self, a_rev="HEAD", b_rev=None, targets=None):
"""
By default, it compares the workspace with the last commit's tree.
By default, it compares the workspace with the last commit's fs.
This implementation differs from `git diff` since DVC doesn't have
the concept of `index`, but it keeps the same interface, thus,
@@ -21,9 +21,9 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
if self.scm.no_commits:
return {}
from dvc.tree.repo import RepoTree
from dvc.fs.repo import RepoFileSystem
repo_tree = RepoTree(self)
repo_fs = RepoFileSystem(self)
b_rev = b_rev if b_rev else "workspace"
results = {}
@@ -38,10 +38,10 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
if targets is not None:
# convert targets to path_infos, and capture any missing targets
targets_path_infos, missing_targets[rev] = _targets_to_path_infos(
repo_tree, targets
repo_fs, targets
)
results[rev] = _paths_checksums(self, repo_tree, targets_path_infos)
results[rev] = _paths_checksums(self, repo_fs, targets_path_infos)
if targets is not None:
# check for overlapping missing targets between a_rev and b_rev
@@ -53,14 +53,14 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
old = results[a_rev]
new = results[b_rev]
# Compare paths between the old and new tree.
# Compare paths between the old and new fs.
# set() efficiently converts dict keys to a set
added = sorted(set(new) - set(old))
deleted_or_missing = set(old) - set(new)
if b_rev == "workspace":
# missing status is only applicable when diffing local workspace
# against a commit
missing = sorted(_filter_missing(repo_tree, deleted_or_missing))
missing = sorted(_filter_missing(repo_fs, deleted_or_missing))
else:
missing = []
deleted = sorted(deleted_or_missing - set(missing))
@@ -94,10 +94,10 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
return ret if any(ret.values()) else {}
def _paths_checksums(repo, repo_tree, targets):
def _paths_checksums(repo, repo_fs, targets):
"""
A dictionary of checksums addressed by relpaths collected from
the current tree outputs.
the current fs outputs.
To help distinguish between a directory and a file output,
the former one will come with a trailing slash in the path:
@@ -106,16 +106,16 @@ def _paths_checksums(repo, repo_tree, targets):
file: "data"
"""
return dict(_output_paths(repo, repo_tree, targets))
return dict(_output_paths(repo, repo_fs, targets))
def _output_paths(repo, repo_tree, targets):
from dvc.tree.local import LocalTree
def _output_paths(repo, repo_fs, targets):
from dvc.fs.local import LocalFileSystem
on_working_tree = isinstance(repo.tree, LocalTree)
on_working_fs = isinstance(repo.fs, LocalFileSystem)
def _exists(output):
if on_working_tree:
if on_working_fs:
return output.exists
return True
@@ -127,10 +127,8 @@ def _output_paths(repo, repo_tree, targets):
)
def _to_checksum(output):
if on_working_tree:
return get_hash(
output.path_info, repo.cache.local.tree, "md5"
).value
if on_working_fs:
return get_hash(output.path_info, repo.cache.local.fs, "md5").value
return output.hash_info.value
for stage in repo.stages:
@@ -147,26 +145,26 @@ def _output_paths(repo, repo_tree, targets):
yield_output
or any(target.isin(output.path_info) for target in targets)
):
yield from _dir_output_paths(repo_tree, output, targets)
yield from _dir_output_paths(repo_fs, output, targets)
def _dir_output_paths(repo_tree, output, targets=None):
def _dir_output_paths(repo_fs, output, targets=None):
from dvc.config import NoRemoteError
try:
for fname in repo_tree.walk_files(output.path_info):
for fname in repo_fs.walk_files(output.path_info):
if targets is None or any(
fname.isin_or_eq(target) for target in targets
):
yield str(fname), get_file_hash(fname, repo_tree, "md5").value
yield str(fname), get_file_hash(fname, repo_fs, "md5").value
except NoRemoteError:
logger.warning("dir cache entry for '%s' is missing", output)
def _filter_missing(repo_tree, paths):
def _filter_missing(repo_fs, paths):
for path in paths:
try:
metadata = repo_tree.metadata(path)
metadata = repo_fs.metadata(path)
if metadata.is_dvc:
out = metadata.outs[0]
if out.status().get(str(out)) == "not in cache":
@@ -175,13 +173,13 @@ def _filter_missing(repo_tree, paths):
pass
def _targets_to_path_infos(repo_tree, targets):
def _targets_to_path_infos(repo_fs, targets):
path_infos = []
missing = []
for target in targets:
if repo_tree.exists(target):
path_infos.append(repo_tree.metadata(target).path_info)
if repo_fs.exists(target):
path_infos.append(repo_fs.metadata(target).path_info)
else:
missing.append(target)
+3 -3
View File
@@ -251,10 +251,10 @@ class Experiments:
# NOTE: dirty DVC lock files must be restored to index state to
# avoid checking out incorrect persist or checkpoint outs
tree = self.scm.get_tree("HEAD")
fs = self.scm.get_fs("HEAD")
lock_files = [
str(fname)
for fname in tree.walk_files(self.scm.root_dir)
for fname in fs.walk_files(self.scm.root_dir)
if is_lock_file(fname)
]
if lock_files:
@@ -333,7 +333,7 @@ class Experiments:
path = PathInfo(params_fname)
suffix = path.suffix.lower()
modify_data = MODIFIERS[suffix]
with modify_data(path, tree=self.repo.tree) as data:
with modify_data(path, fs=self.repo.fs) as data:
benedict(data).merge(params[params_fname], overwrite=True)
# Force params file changes to be staged in git
+4 -4
View File
@@ -138,11 +138,11 @@ class BaseExecutor(ABC):
# TODO: come up with better way to stash repro arguments
@staticmethod
def pack_repro_args(path, *args, tree=None, extra=None, **kwargs):
def pack_repro_args(path, *args, fs=None, extra=None, **kwargs):
dpath = os.path.dirname(path)
if tree:
open_func = tree.open
tree.makedirs(dpath)
if fs:
open_func = fs.open
fs.makedirs(dpath)
else:
from dvc.utils.fs import makedirs
+1 -1
View File
@@ -113,7 +113,7 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
obj = stage(
cache,
path_info,
repo.repo_tree,
repo.repo_fs,
jobs=jobs,
follow_subrepos=False,
)
+1 -1
View File
@@ -52,7 +52,7 @@ def get(url, path, out=None, rev=None, jobs=None):
) as repo:
from_info = PathInfo(repo.root_dir) / path
to_info = PathInfo(out)
repo.repo_tree.download(
repo.repo_fs.download(
from_info, to_info, jobs=jobs, follow_subrepos=False
)
finally:
+4 -4
View File
@@ -35,7 +35,7 @@ def ls(
if path:
path_info /= path
ret = _ls(repo.repo_tree, path_info, recursive, dvc_only)
ret = _ls(repo.repo_fs, path_info, recursive, dvc_only)
if path and not ret:
raise PathMissingError(path, repo, dvc_only=dvc_only)
@@ -48,13 +48,13 @@ def ls(
return ret_list
def _ls(tree, path_info, recursive=None, dvc_only=False):
def _ls(fs, path_info, recursive=None, dvc_only=False):
def onerror(exc):
raise exc
infos = []
try:
for root, dirs, files in tree.walk(
for root, dirs, files in fs.walk(
path_info.fspath, onerror=onerror, dvcfiles=True
):
entries = chain(files, dirs) if not recursive else files
@@ -68,7 +68,7 @@ def _ls(tree, path_info, recursive=None, dvc_only=False):
ret = {}
for info in infos:
metadata = tree.metadata(info)
metadata = fs.metadata(info)
if metadata.output_exists or not dvc_only:
path = (
path_info.name
+4 -4
View File
@@ -6,13 +6,13 @@ from dvc.exceptions import (
NoMetricsFoundError,
NoMetricsParsedError,
)
from dvc.fs.repo import RepoFileSystem
from dvc.output import BaseOutput
from dvc.path_info import PathInfo
from dvc.repo import locked
from dvc.repo.collect import collect
from dvc.repo.live import summary_path_info
from dvc.scm.base import SCMError
from dvc.tree.repo import RepoTree
from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml
logger = logging.getLogger(__name__)
@@ -71,15 +71,15 @@ def _extract_metrics(metrics, path, rev):
def _read_metrics(repo, metrics, rev):
tree = RepoTree(repo)
fs = RepoFileSystem(repo)
res = {}
for metric in metrics:
if not tree.isfile(metric):
if not fs.isfile(metric):
continue
try:
val = load_yaml(metric, tree=tree)
val = load_yaml(metric, fs=fs)
except (FileNotFoundError, YAMLFileCorruptedError):
logger.debug(
"failed to read '%s' on '%s'", metric, rev, exc_info=True
+2 -2
View File
@@ -48,13 +48,13 @@ def _collect_configs(repo: "Repo", rev, targets=None) -> List["DvcPath"]:
def _read_params(repo, configs, rev):
res = {}
for config in configs:
if not repo.tree.exists(config):
if not repo.fs.exists(config):
continue
suffix = config.suffix.lower()
loader = LOADERS[suffix]
try:
res[str(config)] = loader(config, tree=repo.tree)
res[str(config)] = loader(config, fs=repo.fs)
except ParseError:
logger.debug(
"failed to read '%s' on '%s'", config, rev, exc_info=True
+5 -5
View File
@@ -49,7 +49,7 @@ class Plots:
}}}
Data parsing is postponed, since it's affected by props.
"""
from dvc.tree.repo import RepoTree
from dvc.fs.repo import RepoFileSystem
from dvc.utils.collections import ensure_list
targets = ensure_list(targets)
@@ -60,16 +60,16 @@ class Plots:
continue
rev = rev or "workspace"
tree = RepoTree(self.repo)
fs = RepoFileSystem(self.repo)
plots = _collect_plots(self.repo, targets, rev, recursive)
for path_info, props in plots.items():
if rev not in data:
data[rev] = {}
if tree.isdir(path_info):
if fs.isdir(path_info):
plot_files = []
for pi in tree.walk_files(path_info):
for pi in fs.walk_files(path_info):
plot_files.append(
(pi, relpath(pi, self.repo.root_dir))
)
@@ -83,7 +83,7 @@ class Plots:
# Load data from git or dvc cache
try:
with tree.open(path) as fd:
with fs.open(path) as fd:
data[rev][repo_path]["data"] = fd.read()
except FileNotFoundError:
# This might happen simply because cache is absent
+11 -11
View File
@@ -62,7 +62,7 @@ def _maybe_collect_from_dvc_yaml(
from dvc.stage.exceptions import StageNotFound
stages: StageList = []
if loader.tree.exists(PIPELINE_FILE):
if loader.fs.exists(PIPELINE_FILE):
with suppress(StageNotFound):
stages = loader.load_all(PIPELINE_FILE, target, **load_kwargs)
return _collect_with_deps(stages, loader.graph) if with_deps else stages
@@ -88,7 +88,7 @@ def _collect_specific_target(
# else, we assume it's a output name in the `collect_granular()` below
msg = "Checking if stage '%s' is in '%s'"
logger.debug(msg, target, PIPELINE_FILE)
if not (recursive and loader.tree.isdir(target)):
if not (recursive and loader.fs.isdir(target)):
stages = _maybe_collect_from_dvc_yaml(
loader, target, with_deps, accept_group=accept_group,
)
@@ -304,8 +304,8 @@ class StageLoad:
return self.load_all(path, expr, glob=True)
@property
def tree(self):
return self.repo.tree
def fs(self):
return self.repo.fs
@property
def graph(self) -> "DiGraph":
@@ -353,7 +353,7 @@ class StageLoad:
if not target:
return list(graph) if graph else self.repo.stages
if recursive and self.repo.tree.isdir(target):
if recursive and self.repo.fs.isdir(target):
from dvc.repo.graph import collect_inside_path
path = os.path.abspath(target)
@@ -398,7 +398,7 @@ class StageLoad:
self, target, with_deps, recursive, accept_group
)
if not stages:
if not (recursive and self.tree.isdir(target)):
if not (recursive and self.fs.isdir(target)):
try:
(out,) = self.repo.find_outs_by_path(target, strict=False)
filter_info = PathInfo(os.path.abspath(target))
@@ -443,17 +443,17 @@ class StageLoad:
the collection.
"""
from dvc.dvcfile import is_valid_filename
from dvc.tree.local import LocalTree
from dvc.fs.local import LocalFileSystem
scm = self.repo.scm
sep = os.sep
outs: Set[str] = set()
is_local_tree = isinstance(self.tree, LocalTree)
is_local_fs = isinstance(self.fs, LocalFileSystem)
def is_ignored(path):
# apply only for the local tree
return is_local_tree and scm.is_ignored(path)
# apply only for the local fs
return is_local_fs and scm.is_ignored(path)
def is_dvcfile_and_not_ignored(root, file):
return is_valid_filename(file) and not is_ignored(
@@ -465,7 +465,7 @@ class StageLoad:
return dir_path in outs or is_ignored(dir_path)
stages = []
for root, dirs, files in self.tree.walk(self.repo.root_dir):
for root, dirs, files in self.fs.walk(self.repo.root_dir):
dvcfile_filter = partial(is_dvcfile_and_not_ignored, root)
for file in filter(dvcfile_filter, files):
+3 -3
View File
@@ -340,15 +340,15 @@ class Git(Base):
pass
raise NoGitBackendError(name)
def get_tree(self, rev: str, **kwargs):
from dvc.tree.git import GitTree
def get_fs(self, rev: str, **kwargs):
from dvc.fs.git import GitFileSystem
from .objects import GitTrie
resolved = self.resolve_rev(rev)
tree_obj = self._backend_func("get_tree_obj", rev=resolved)
trie = GitTrie(tree_obj, resolved)
return GitTree(self.root_dir, trie, **kwargs)
return GitFileSystem(self.root_dir, trie, **kwargs)
is_ignored = partialmethod(_backend_func, "is_ignored")
add = partialmethod(_backend_func, "add")
+6 -6
View File
@@ -232,9 +232,9 @@ class StageCache:
remote = self.repo.cloud.get_remote(remote)
return self._transfer(
_log_exceptions(remote.tree.upload, "upload"),
self.repo.cache.local.tree,
remote.tree,
_log_exceptions(remote.fs.upload, "upload"),
self.repo.cache.local.fs,
remote.fs,
)
def pull(self, remote):
@@ -242,9 +242,9 @@ class StageCache:
remote = self.repo.cloud.get_remote(remote)
return self._transfer(
_log_exceptions(remote.tree.download, "download"),
remote.tree,
self.repo.cache.local.tree,
_log_exceptions(remote.fs.download, "download"),
remote.fs,
self.repo.cache.local.fs,
)
def get_used_cache(self, used_run_cache, *args, **kwargs):
+9 -9
View File
@@ -133,13 +133,13 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
MAX_UINT = 2 ** 64 - 2
def __init__(self, repo):
from dvc.tree.local import LocalTree
from dvc.fs.local import LocalFileSystem
super().__init__()
self.repo = repo
self.root_dir = repo.root_dir
self.tree = LocalTree(None, {"url": self.root_dir})
self.fs = LocalFileSystem(None, {"url": self.root_dir})
state_config = repo.config.get("state", {})
self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
@@ -410,7 +410,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
assert isinstance(hash_info, HashInfo)
assert os.path.exists(path_info)
actual_mtime, actual_size = get_mtime_and_size(path_info, self.tree)
actual_mtime, actual_size = get_mtime_and_size(path_info, self.fs)
actual_inode = get_inode(path_info)
existing_record = self.get_state_record_for_inode(actual_inode)
@@ -438,13 +438,13 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
assert isinstance(path_info, str) or path_info.scheme == "local"
path = os.fspath(path_info)
# NOTE: use os.path.exists instead of LocalTree.exists
# NOTE: use os.path.exists instead of LocalFileSystem.exists
# because it uses lexists() and will return True for broken
# symlinks that we cannot stat() in get_mtime_and_size
if not os.path.exists(path):
return None
actual_mtime, actual_size = get_mtime_and_size(path, self.tree)
actual_mtime, actual_size = get_mtime_and_size(path, self.fs)
actual_inode = get_inode(path)
existing_record = self.get_state_record_for_inode(actual_inode)
@@ -467,10 +467,10 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
"""
assert isinstance(path_info, str) or path_info.scheme == "local"
if not self.tree.exists(path_info):
if not self.fs.exists(path_info):
return
mtime, _ = get_mtime_and_size(path_info, self.tree)
mtime, _ = get_mtime_and_size(path_info, self.fs)
inode = get_inode(path_info)
relative_path = relpath(path_info, self.root_dir)
@@ -493,11 +493,11 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
inode = self._from_sqlite(inode)
path = os.path.join(self.root_dir, relative_path)
if path in used or not self.tree.exists(path):
if path in used or not self.fs.exists(path):
continue
actual_inode = get_inode(path)
actual_mtime, _ = get_mtime_and_size(path, self.tree)
actual_mtime, _ = get_mtime_and_size(path, self.fs)
if (inode, mtime) == (actual_inode, actual_mtime):
logger.debug("Removing '%s' as unused link.", path)
-7
View File
@@ -1,7 +0,0 @@
from dvc.scheme import Schemes
from .http import HTTPTree
class HTTPSTree(HTTPTree): # pylint:disable=abstract-method
scheme = Schemes.HTTPS
-7
View File
@@ -1,7 +0,0 @@
from dvc.scheme import Schemes
from .webdav import WebDAVTree
class WebDAVSTree(WebDAVTree): # pylint:disable=abstract-method
scheme = Schemes.WEBDAVS
+4 -4
View File
@@ -41,14 +41,14 @@ def _fobj_md5(fobj, hash_md5, binary, progress_func=None):
progress_func(len(data))
def file_md5(fname, tree):
def file_md5(fname, fs):
""" get the (md5 hexdigest, md5 digest) of a file """
from dvc.istextfile import istextfile
from dvc.progress import Tqdm
hash_md5 = hashlib.md5()
binary = not istextfile(fname, tree=tree)
size = tree.getsize(fname) or 0
binary = not istextfile(fname, fs=fs)
size = fs.getsize(fname) or 0
no_progress_bar = True
if size >= LARGE_FILE_SIZE:
no_progress_bar = False
@@ -65,7 +65,7 @@ def file_md5(fname, tree):
bytes=True,
leave=False,
) as pbar:
with tree.open(fname, "rb") as fobj:
with fs.open(fname, "rb") as fobj:
_fobj_md5(fobj, hash_md5, binary, pbar.update)
return hash_md5.hexdigest()
+5 -5
View File
@@ -30,15 +30,15 @@ def get_inode(path):
return inode
def get_mtime_and_size(path, tree):
def get_mtime_and_size(path, fs):
import nanotime
if tree.isdir(path):
if fs.isdir(path):
size = 0
files_mtimes = {}
for file_path in tree.walk_files(path):
for file_path in fs.walk_files(path):
try:
stats = tree.stat(file_path)
stats = fs.stat(file_path)
except OSError as exc:
# NOTE: broken symlink case.
if exc.errno != errno.ENOENT:
@@ -51,7 +51,7 @@ def get_mtime_and_size(path, tree):
# max(mtime(f) for f in non_ignored_files)
mtime = dict_md5(files_mtimes)
else:
base_stat = tree.stat(path)
base_stat = fs.stat(path)
size = base_stat.st_size
mtime = base_stat.st_mtime
mtime = int(nanotime.timestamp(mtime))
+12 -12
View File
@@ -8,26 +8,26 @@ from typing_extensions import Protocol
from dvc.exceptions import DvcException
if TYPE_CHECKING:
from dvc.tree.base import BaseTree
from dvc.fs.base import BaseFileSystem
from dvc.types import AnyPath
class DumperFn(Protocol):
def __call__(
self, path: "AnyPath", data: Any, tree: "BaseTree" = None
self, path: "AnyPath", data: Any, fs: "BaseFileSystem" = None
) -> Any:
...
class ModifierFn(Protocol):
def __call__(
self, path: "AnyPath", tree: "BaseTree" = None
self, path: "AnyPath", fs: "BaseFileSystem" = None
) -> ContextManager[Dict]:
...
class LoaderFn(Protocol):
def __call__(self, path: "AnyPath", tree: "BaseTree" = None) -> Any:
def __call__(self, path: "AnyPath", fs: "BaseFileSystem" = None) -> Any:
...
@@ -45,14 +45,14 @@ class ParseError(DvcException):
super().__init__(f"unable to read: '{path}', {message}")
def _load_data(path: "AnyPath", parser: ParserFn, tree: "BaseTree" = None):
open_fn = tree.open if tree else open
def _load_data(path: "AnyPath", parser: ParserFn, fs: "BaseFileSystem" = None):
open_fn = fs.open if fs else open
with open_fn(path, encoding="utf-8") as fd: # type: ignore
return parser(fd.read(), path)
def _dump_data(path, data: Any, dumper: DumperFn, tree: "BaseTree" = None):
open_fn = tree.open if tree else open
def _dump_data(path, data: Any, dumper: DumperFn, fs: "BaseFileSystem" = None):
open_fn = fs.open if fs else open
with open_fn(path, "w+", encoding="utf-8") as fd: # type: ignore
dumper(data, fd)
@@ -62,10 +62,10 @@ def _modify_data(
path: "AnyPath",
parser: ParserFn,
dumper: DumperFn,
tree: "BaseTree" = None,
fs: "BaseFileSystem" = None,
):
exists_fn = tree.exists if tree else os.path.exists
exists_fn = fs.exists if fs else os.path.exists
file_exists = exists_fn(path) # type: ignore
data = _load_data(path, parser=parser, tree=tree) if file_exists else {}
data = _load_data(path, parser=parser, fs=fs) if file_exists else {}
yield data
dumper(path, data, tree=tree)
dumper(path, data, fs=fs)
+6 -6
View File
@@ -10,8 +10,8 @@ class JSONFileCorruptedError(ParseError):
super().__init__(path, "JSON file structure is corrupted")
def load_json(path, tree=None):
return _load_data(path, parser=parse_json, tree=tree)
def load_json(path, fs=None):
return _load_data(path, parser=parse_json, fs=fs)
def parse_json(text, path, **kwargs):
@@ -19,11 +19,11 @@ def parse_json(text, path, **kwargs):
return json.loads(text, **kwargs) or {}
def dump_json(path, data, tree=None):
return _dump_data(path, data, dumper=json.dump, tree=tree)
def dump_json(path, data, fs=None):
return _dump_data(path, data, dumper=json.dump, fs=fs)
@contextmanager
def modify_json(path, tree=None):
with _modify_data(path, parse_json, dump_json, tree=tree) as d:
def modify_json(path, fs=None):
with _modify_data(path, parse_json, dump_json, fs=fs) as d:
yield d
+6 -6
View File
@@ -14,8 +14,8 @@ class PythonFileCorruptedError(ParseError):
super().__init__(path, message)
def load_py(path, tree=None):
return _load_data(path, parser=parse_py, tree=tree)
def load_py(path, fs=None):
return _load_data(path, parser=parse_py, fs=fs)
def parse_py(text, path):
@@ -76,13 +76,13 @@ def _dump(data, stream):
stream.close()
def dump_py(path, data, tree=None):
return _dump_data(path, data, dumper=_dump, tree=tree)
def dump_py(path, data, fs=None):
return _dump_data(path, data, dumper=_dump, fs=fs)
@contextmanager
def modify_py(path, tree=None):
with _modify_data(path, parse_py_for_update, dump_py, tree=tree) as d:
def modify_py(path, fs=None):
with _modify_data(path, parse_py_for_update, dump_py, fs=fs) as d:
yield d
+6 -6
View File
@@ -10,8 +10,8 @@ class TOMLFileCorruptedError(ParseError):
super().__init__(path, "TOML file structure is corrupted")
def load_toml(path, tree=None):
return _load_data(path, parser=parse_toml, tree=tree)
def load_toml(path, fs=None):
return _load_data(path, parser=parse_toml, fs=fs)
def parse_toml(text, path, decoder=None):
@@ -40,11 +40,11 @@ def _dump(data, stream):
return toml.dump(data, stream, encoder=toml.TomlPreserveCommentEncoder())
def dump_toml(path, data, tree=None):
return _dump_data(path, data, dumper=_dump, tree=tree)
def dump_toml(path, data, fs=None):
return _dump_data(path, data, dumper=_dump, fs=fs)
@contextmanager
def modify_toml(path, tree=None):
with _modify_data(path, parse_toml_for_update, dump_toml, tree=tree) as d:
def modify_toml(path, fs=None):
with _modify_data(path, parse_toml_for_update, dump_toml, fs=fs) as d:
yield d
+6 -6
View File
@@ -16,8 +16,8 @@ class YAMLFileCorruptedError(YAMLError):
super().__init__(path, "YAML file structure is corrupted")
def load_yaml(path, tree=None):
return _load_data(path, parser=parse_yaml, tree=tree)
def load_yaml(path, fs=None):
return _load_data(path, parser=parse_yaml, fs=fs)
def parse_yaml(text, path, typ="safe"):
@@ -65,8 +65,8 @@ def _dump(data, stream):
return yaml.dump(data, stream)
def dump_yaml(path, data, tree=None):
return _dump_data(path, data, dumper=_dump, tree=tree)
def dump_yaml(path, data, fs=None):
return _dump_data(path, data, dumper=_dump, fs=fs)
def loads_yaml(s, typ="safe"):
@@ -82,6 +82,6 @@ def dumps_yaml(d):
@contextmanager
def modify_yaml(path, tree=None):
with _modify_data(path, parse_yaml_for_update, dump_yaml, tree=tree) as d:
def modify_yaml(path, fs=None):
with _modify_data(path, parse_yaml_for_update, dump_yaml, fs=fs) as d:
yield d
+1 -1
View File
@@ -45,7 +45,7 @@ def reset_loglevel(request, caplog):
@pytest.fixture(scope="session", autouse=True)
def _close_pools():
from dvc.tree.pool import close_pools
from dvc.fs.pool import close_pools
yield
close_pools()
+1 -1
View File
@@ -8,7 +8,7 @@ from dvc import api
from dvc.exceptions import FileMissingError, OutputNotFoundError
from dvc.path_info import URLInfo
from dvc.utils.fs import remove
from tests.unit.tree.test_repo import make_subrepo
from tests.unit.fs.test_repo import make_subrepo
cloud_names = [
"s3",
+15 -15
View File
@@ -20,10 +20,10 @@ def test_new_checkpoint(
for rev in dvc.brancher([exp]):
if rev == "workspace":
continue
tree = dvc.repo_tree
with tree.open(tmp_dir / "foo") as fobj:
fs = dvc.repo_fs
with fs.open(tmp_dir / "foo") as fobj:
assert fobj.read().strip() == str(checkpoint_stage.iterations)
with tree.open(tmp_dir / "metrics.yaml") as fobj:
with fs.open(tmp_dir / "metrics.yaml") as fobj:
assert fobj.read().strip() == "foo: 2"
if workspace:
@@ -69,10 +69,10 @@ def test_resume_checkpoint(
for rev in dvc.brancher([exp]):
if rev == "workspace":
continue
tree = dvc.repo_tree
with tree.open(tmp_dir / "foo") as fobj:
fs = dvc.repo_fs
with fs.open(tmp_dir / "foo") as fobj:
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
with tree.open(tmp_dir / "metrics.yaml") as fobj:
with fs.open(tmp_dir / "metrics.yaml") as fobj:
assert fobj.read().strip() == "foo: 2"
if workspace:
@@ -104,10 +104,10 @@ def test_reset_checkpoint(
for rev in dvc.brancher([exp]):
if rev == "workspace":
continue
tree = dvc.repo_tree
with tree.open(tmp_dir / "foo") as fobj:
fs = dvc.repo_fs
with fs.open(tmp_dir / "foo") as fobj:
assert fobj.read().strip() == str(checkpoint_stage.iterations)
with tree.open(tmp_dir / "metrics.yaml") as fobj:
with fs.open(tmp_dir / "metrics.yaml") as fobj:
assert fobj.read().strip() == "foo: 2"
if workspace:
@@ -143,19 +143,19 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage, workspace):
for rev in dvc.brancher([checkpoint_a]):
if rev == "workspace":
continue
tree = dvc.repo_tree
with tree.open(tmp_dir / "foo") as fobj:
fs = dvc.repo_fs
with fs.open(tmp_dir / "foo") as fobj:
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
with tree.open(tmp_dir / "metrics.yaml") as fobj:
with fs.open(tmp_dir / "metrics.yaml") as fobj:
assert fobj.read().strip() == "foo: 2"
for rev in dvc.brancher([checkpoint_b]):
if rev == "workspace":
continue
tree = dvc.repo_tree
with tree.open(tmp_dir / "foo") as fobj:
fs = dvc.repo_fs
with fs.open(tmp_dir / "foo") as fobj:
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
with tree.open(tmp_dir / "metrics.yaml") as fobj:
with fs.open(tmp_dir / "metrics.yaml") as fobj:
assert fobj.read().strip() == "foo: 100"
with pytest.raises(MultipleBranchError):
+28 -28
View File
@@ -28,8 +28,8 @@ def test_new_simple(tmp_dir, scm, dvc, exp_stage, mocker, name, workspace):
assert ref_info and ref_info.baseline_sha == baseline
new_mock.assert_called_once()
tree = scm.get_tree(exp)
with tree.open(tmp_dir / "metrics.yaml") as fobj:
fs = scm.get_fs(exp)
with fs.open(tmp_dir / "metrics.yaml") as fobj:
assert fobj.read().strip() == "foo: 2"
if workspace:
@@ -68,8 +68,8 @@ def test_experiment_exists(tmp_dir, scm, dvc, exp_stage, mocker, workspace):
)
exp = first(results)
tree = scm.get_tree(exp)
with tree.open(tmp_dir / "metrics.yaml") as fobj:
fs = scm.get_fs(exp)
with fs.open(tmp_dir / "metrics.yaml") as fobj:
assert fobj.read().strip() == "foo: 3"
@@ -142,8 +142,8 @@ def test_modify_params(tmp_dir, scm, dvc, mocker, changes, expected):
exp = first(results)
new_mock.assert_called_once()
tree = scm.get_tree(exp)
with tree.open(tmp_dir / "metrics.yaml") as fobj:
fs = scm.get_fs(exp)
with fs.open(tmp_dir / "metrics.yaml") as fobj:
assert fobj.read().strip() == expected
@@ -238,10 +238,10 @@ def test_update_py_params(tmp_dir, scm, dvc):
)
exp_a = first(results)
tree = scm.get_tree(exp_a)
with tree.open(tmp_dir / "params.py") as fobj:
fs = scm.get_fs(exp_a)
with fs.open(tmp_dir / "params.py") as fobj:
assert fobj.read().strip() == "INT = 2"
with tree.open(tmp_dir / "metrics.py") as fobj:
with fs.open(tmp_dir / "metrics.py") as fobj:
assert fobj.read().strip() == "INT = 2"
tmp_dir.gen(
@@ -280,10 +280,10 @@ def test_update_py_params(tmp_dir, scm, dvc):
# in order to compare with the original
return text.replace("\r\n", "\n")
tree = scm.get_tree(exp_a)
with tree.open(tmp_dir / "params.py") as fobj:
fs = scm.get_fs(exp_a)
with fs.open(tmp_dir / "params.py") as fobj:
assert _dos2unix(fobj.read().strip()) == result
with tree.open(tmp_dir / "metrics.py") as fobj:
with fs.open(tmp_dir / "metrics.py") as fobj:
assert _dos2unix(fobj.read().strip()) == result
tmp_dir.gen("params.py", "INT = 1\n")
@@ -406,11 +406,11 @@ def test_untracked(tmp_dir, scm, dvc, caplog, workspace):
stage.addressing, params=["foo=2"], tmp_dir=not workspace
)
exp = first(results)
tree = scm.get_tree(exp)
assert tree.exists("dvc.yaml")
assert tree.exists("dvc.lock")
assert tree.exists("copy.py")
with tree.open(tmp_dir / "metrics.yaml") as fobj:
fs = scm.get_fs(exp)
assert fs.exists("dvc.yaml")
assert fs.exists("dvc.lock")
assert fs.exists("copy.py")
with fs.open(tmp_dir / "metrics.yaml") as fobj:
assert fobj.read().strip() == "foo: 2"
@@ -428,8 +428,8 @@ def test_dirty_lockfile(tmp_dir, scm, dvc, exp_stage, workspace):
)
exp = first(results)
tree = scm.get_tree(exp)
with tree.open(tmp_dir / "metrics.yaml") as fobj:
fs = scm.get_fs(exp)
with fs.open(tmp_dir / "metrics.yaml") as fobj:
assert fobj.read().strip() == "foo: 2"
if not workspace:
@@ -509,10 +509,10 @@ def test_subdir(tmp_dir, scm, dvc, workspace):
exp = first(results)
ref_info = first(exp_refs_by_rev(scm, exp))
tree = scm.get_tree(exp)
fs = scm.get_fs(exp)
for fname in ["metrics.yaml", "dvc.lock"]:
assert tree.exists(subdir / fname)
with tree.open(subdir / "metrics.yaml") as fobj:
assert fs.exists(subdir / fname)
with fs.open(subdir / "metrics.yaml") as fobj:
assert fobj.read().strip() == "foo: 2"
assert dvc.experiments.get_exact_name(exp) == ref_info.name
@@ -521,7 +521,7 @@ def test_subdir(tmp_dir, scm, dvc, workspace):
@pytest.mark.parametrize("workspace", [True, False])
def test_subrepo(tmp_dir, scm, workspace):
from tests.unit.tree.test_repo import make_subrepo
from tests.unit.fs.test_repo import make_subrepo
subrepo = tmp_dir / "dir" / "repo"
make_subrepo(subrepo, scm)
@@ -554,10 +554,10 @@ def test_subrepo(tmp_dir, scm, workspace):
exp = first(results)
ref_info = first(exp_refs_by_rev(scm, exp))
tree = scm.get_tree(exp)
fs = scm.get_fs(exp)
for fname in ["metrics.yaml", "dvc.lock"]:
assert tree.exists(subrepo / fname)
with tree.open(subrepo / "metrics.yaml") as fobj:
assert fs.exists(subrepo / fname)
with fs.open(subrepo / "metrics.yaml") as fobj:
assert fobj.read().strip() == "foo: 2"
assert subrepo.dvc.experiments.get_exact_name(exp) == ref_info.name
@@ -577,8 +577,8 @@ def test_queue(tmp_dir, scm, dvc, exp_stage, mocker):
expected = {"foo: 2", "foo: 3"}
metrics = set()
for exp in results:
tree = scm.get_tree(exp)
with tree.open(tmp_dir / "metrics.yaml") as fobj:
fs = scm.get_fs(exp)
with fs.open(tmp_dir / "metrics.yaml") as fobj:
metrics.add(fobj.read().strip())
assert expected == metrics
+4 -4
View File
@@ -108,14 +108,14 @@ def test_show_checkpoint(
for i, rev in enumerate(checkpoints):
if i == 0:
name = dvc.experiments.get_exact_name(rev)
tree = "╓"
fs = "╓"
elif i == len(checkpoints) - 1:
name = rev[:7]
tree = "╨"
fs = "╨"
else:
name = rev[:7]
tree = "╟"
assert f"{tree} {name}" in cap.out
fs = "╟"
assert f"{fs} {name}" in cap.out
@pytest.mark.parametrize("workspace", [True, False])
+5 -5
View File
@@ -3,10 +3,10 @@ import os
import pytest
from dvc.exceptions import DownloadError, UploadError
from dvc.fs.local import LocalFileSystem
from dvc.remote.base import Remote
from dvc.remote.index import RemoteIndex
from dvc.remote.local import LocalRemote
from dvc.tree.local import LocalTree
from dvc.utils.fs import remove
@@ -17,7 +17,7 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker):
dvc.config["core"]["remote"] = "upstream"
# patch hashes_exist since the LocalRemote normally overrides
# BaseTree.hashes_exist.
# BaseFileSystem.hashes_exist.
def hashes_exist(self, *args, **kwargs):
return Remote.hashes_exist(self, *args, **kwargs)
@@ -85,7 +85,7 @@ def test_clear_on_download_err(tmp_dir, dvc, tmp_path_factory, remote, mocker):
remove(dvc.cache.local.cache_dir)
mocked_clear = mocker.patch.object(remote.INDEX_CLS, "clear")
mocker.patch.object(LocalTree, "_download", side_effect=Exception)
mocker.patch.object(LocalFileSystem, "_download", side_effect=Exception)
with pytest.raises(DownloadError):
dvc.pull()
mocked_clear.assert_called_once_with()
@@ -95,14 +95,14 @@ def test_partial_upload(tmp_dir, dvc, tmp_path_factory, remote, mocker):
tmp_dir.dvc_gen({"foo": "foo content"})
tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})
original = LocalTree._upload
original = LocalFileSystem._upload
def unreliable_upload(self, from_file, to_info, name=None, **kwargs):
if "baz" in name:
raise Exception("stop baz")
return original(self, from_file, to_info, name, **kwargs)
mocker.patch.object(LocalTree, "_upload", unreliable_upload)
mocker.patch.object(LocalFileSystem, "_upload", unreliable_upload)
with pytest.raises(UploadError):
dvc.push()
with remote.index:
+5 -5
View File
@@ -21,6 +21,7 @@ from dvc.exceptions import (
OverlappingOutputPathsError,
RecursiveAddingWhileUsingFilename,
)
from dvc.fs.local import LocalFileSystem
from dvc.hash_info import HashInfo
from dvc.main import main
from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
@@ -28,7 +29,6 @@ from dvc.repo import Repo as DvcRepo
from dvc.stage import Stage
from dvc.stage.exceptions import StagePathNotFoundError
from dvc.system import System
from dvc.tree.local import LocalTree
from dvc.utils import LARGE_DIR_SIZE, file_md5, relpath
from dvc.utils.fs import path_isin
from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml
@@ -38,7 +38,7 @@ from tests.utils import get_gitignore_content
def test_add(tmp_dir, dvc):
(stage,) = tmp_dir.dvc_gen({"foo": "foo"})
md5 = file_md5("foo", dvc.tree)
md5 = file_md5("foo", dvc.fs)
assert stage is not None
@@ -732,7 +732,7 @@ def test_should_not_checkout_when_adding_cached_copy(tmp_dir, dvc, mocker):
shutil.copy("bar", "foo")
copy_spy = mocker.spy(dvc.cache.local.tree, "copy")
copy_spy = mocker.spy(dvc.cache.local.fs, "copy")
dvc.add("foo")
@@ -758,7 +758,7 @@ def test_should_relink_on_repeated_add(
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
os.remove("foo")
getattr(dvc.cache.local.tree, link)(PathInfo("bar"), PathInfo("foo"))
getattr(dvc.cache.local.fs, link)(PathInfo("bar"), PathInfo("foo"))
dvc.cache.local.cache_types = [new_link]
@@ -838,7 +838,7 @@ def test_add_empty_files(tmp_dir, dvc, link):
def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
dvc.cache.local.cache_types = ["hardlink"]
tmp_dir.gen({"foo": "", "bar": "", "lorem": "lorem", "ipsum": "ipsum"})
m = mocker.spy(LocalTree, "is_hardlink")
m = mocker.spy(LocalFileSystem, "is_hardlink")
stages = dvc.add(["foo", "bar", "lorem", "ipsum"])
assert m.call_count == 1
+1 -1
View File
@@ -89,7 +89,7 @@ class TestExternalCacheDir(TestDvc):
self.dvc.__init__()
assert self.dvc.cache.ssh.tree.path_info == ssh_url + "/tmp"
assert self.dvc.cache.ssh.fs.path_info == ssh_url + "/tmp"
class TestSharedCacheDir(TestDir):
+5 -5
View File
@@ -17,12 +17,12 @@ from dvc.exceptions import (
DvcException,
NoOutputOrStageError,
)
from dvc.fs.local import LocalFileSystem
from dvc.main import main
from dvc.repo import Repo as DvcRepo
from dvc.stage import Stage
from dvc.stage.exceptions import StageFileDoesNotExistError
from dvc.system import System
from dvc.tree.local import LocalTree
from dvc.utils import relpath
from dvc.utils.fs import remove, walk_files
from dvc.utils.serialize import dump_yaml, load_yaml
@@ -148,7 +148,7 @@ class CheckoutBase(TestDvcGit):
paths = [
path
for output in stage["outs"]
for path in self.dvc.tree.walk_files(output["path"])
for path in self.dvc.fs.walk_files(output["path"])
]
return [
@@ -312,8 +312,8 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
class TestCheckoutMissingMd5InStageFile(TestRepro):
def test(self):
d = load_yaml(self.file1_stage)
del d[Stage.PARAM_OUTS][0][LocalTree.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][LocalTree.PARAM_CHECKSUM]
del d[Stage.PARAM_OUTS][0][LocalFileSystem.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][LocalFileSystem.PARAM_CHECKSUM]
dump_yaml(self.file1_stage, d)
with pytest.raises(CheckoutError):
@@ -769,7 +769,7 @@ def test_checkout_for_external_outputs(tmp_dir, dvc, workspace):
dvc.add("remote://workspace/foo")
remote = dvc.cloud.get_remote("workspace")
remote.tree.remove(file_path)
remote.fs.remove(file_path)
assert not file_path.exists()
stats = dvc.checkout(force=True)
+7 -7
View File
@@ -3,13 +3,13 @@ import os
from mock import ANY, patch
from dvc.external_repo import CLONES, external_repo
from dvc.fs.local import LocalFileSystem
from dvc.objects import save, stage
from dvc.path_info import PathInfo
from dvc.scm.git import Git
from dvc.tree.local import LocalTree
from dvc.utils import relpath
from dvc.utils.fs import makedirs, remove
from tests.unit.tree.test_repo import make_subrepo
from tests.unit.fs.test_repo import make_subrepo
def test_external_repo(erepo_dir):
@@ -51,7 +51,7 @@ def test_cache_reused(erepo_dir, mocker, local_cloud):
erepo_dir.dvc_gen("file", "text", commit="add file")
erepo_dir.dvc.push()
download_spy = mocker.spy(LocalTree, "download")
download_spy = mocker.spy(LocalFileSystem, "download")
# Use URL to prevent any fishy optimizations
url = f"file://{erepo_dir}"
@@ -92,7 +92,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
dest = tmp_dir / "file"
with external_repo(os.fspath(erepo_dir)) as repo:
repo.repo_tree.download(
repo.repo_fs.download(
PathInfo(repo.root_dir) / "subdir" / "file", PathInfo(dest),
)
@@ -187,7 +187,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
with external_repo(os.fspath(erepo_dir)) as repo:
repo.repo_tree.download(
repo.repo_fs.download(
PathInfo(repo.root_dir) / "dir",
PathInfo(tmp_dir / "out"),
follow_subrepos=False,
@@ -203,7 +203,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
obj = stage(
repo.cache.local,
PathInfo(repo.root_dir) / "dir",
repo.repo_tree,
repo.repo_fs,
follow_subrepos=False,
)
save(repo.cache.local, obj)
@@ -226,7 +226,7 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
with external_repo(os.fspath(erepo_dir)) as repo:
repo.repo_tree.download(
repo.repo_fs.download(
PathInfo(repo.root_dir) / "dir",
PathInfo(tmp_dir / "out"),
follow_subrepos=False,
+336 -21
View File
@@ -1,36 +1,351 @@
import os
import stat
from operator import itemgetter
from os.path import join
import pytest
from dvc.utils.fs import copyfile, makedirs
from dvc.fs import get_cloud_fs
from dvc.fs.local import LocalFileSystem
from dvc.path_info import PathInfo
from dvc.repo import Repo
from dvc.scm import SCM
from tests.basic_env import TestDir, TestGit, TestGitSubmodule
@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
def test_makedirs_permissions(tmp_dir):
dir_mode = 0o755
intermediate_dir = "тестовая-директория"
test_dir = os.path.join(intermediate_dir, "data")
class TestLocalFileSystem(TestDir):
def setUp(self):
super().setUp()
self.fs = LocalFileSystem(None, {})
assert not os.path.exists(intermediate_dir)
def test_open(self):
with self.fs.open(self.FOO) as fd:
self.assertEqual(fd.read(), self.FOO_CONTENTS)
with self.fs.open(self.UNICODE, encoding="utf-8") as fd:
self.assertEqual(fd.read(), self.UNICODE_CONTENTS)
makedirs(test_dir, mode=dir_mode)
def test_exists(self):
self.assertTrue(self.fs.exists(self.FOO))
self.assertTrue(self.fs.exists(self.UNICODE))
self.assertFalse(self.fs.exists("not-existing-file"))
assert stat.S_IMODE(os.stat(test_dir).st_mode) == dir_mode
assert stat.S_IMODE(os.stat(intermediate_dir).st_mode) == dir_mode
def test_isdir(self):
self.assertTrue(self.fs.isdir(self.DATA_DIR))
self.assertFalse(self.fs.isdir(self.FOO))
self.assertFalse(self.fs.isdir("not-existing-file"))
def test_isfile(self):
self.assertTrue(self.fs.isfile(self.FOO))
self.assertFalse(self.fs.isfile(self.DATA_DIR))
self.assertFalse(self.fs.isfile("not-existing-file"))
def test_copyfile(tmp_dir):
src = "file1"
dest = "file2"
dest_dir = "testdir"
class GitFileSystemTests:
# pylint: disable=no-member
def test_open(self):
self.scm.add([self.FOO, self.UNICODE, self.DATA_DIR])
self.scm.commit("add")
tmp_dir.gen(src, "file1contents")
fs = self.scm.get_fs("master")
with fs.open(self.FOO) as fd:
self.assertEqual(fd.read(), self.FOO_CONTENTS)
with fs.open(self.UNICODE) as fd:
self.assertEqual(fd.read(), self.UNICODE_CONTENTS)
with self.assertRaises(IOError):
fs.open("not-existing-file")
with self.assertRaises(IOError):
fs.open(self.DATA_DIR)
os.mkdir(dest_dir)
def test_exists(self):
fs = self.scm.get_fs("master")
self.assertFalse(fs.exists(self.FOO))
self.assertFalse(fs.exists(self.UNICODE))
self.assertFalse(fs.exists(self.DATA_DIR))
self.scm.add([self.FOO, self.UNICODE, self.DATA])
self.scm.commit("add")
copyfile(src, dest)
assert (tmp_dir / dest).read_text() == "file1contents"
fs = self.scm.get_fs("master")
self.assertTrue(fs.exists(self.FOO))
self.assertTrue(fs.exists(self.UNICODE))
self.assertTrue(fs.exists(self.DATA_DIR))
self.assertFalse(fs.exists("non-existing-file"))
copyfile(src, dest_dir)
assert (tmp_dir / dest_dir / src).read_text() == "file1contents"
def test_isdir(self):
self.scm.add([self.FOO, self.DATA_DIR])
self.scm.commit("add")
fs = self.scm.get_fs("master")
self.assertTrue(fs.isdir(self.DATA_DIR))
self.assertFalse(fs.isdir(self.FOO))
self.assertFalse(fs.isdir("non-existing-file"))
def test_isfile(self):
self.scm.add([self.FOO, self.DATA_DIR])
self.scm.commit("add")
fs = self.scm.get_fs("master")
self.assertTrue(fs.isfile(self.FOO))
self.assertFalse(fs.isfile(self.DATA_DIR))
self.assertFalse(fs.isfile("not-existing-file"))
class TestGitFileSystem(TestGit, GitFileSystemTests):
def setUp(self):
super().setUp()
self.scm = SCM(self._root_dir)
class TestGitSubmoduleFileSystem(TestGitSubmodule, GitFileSystemTests):
def setUp(self):
super().setUp()
self.scm = SCM(self._root_dir)
self._pushd(self._root_dir)
class AssertWalkEqualMixin:
def assertWalkEqual(self, actual, expected, msg=None):
def convert_to_sets(walk_results):
return [
(root, set(dirs), set(nondirs))
for root, dirs, nondirs in walk_results
]
self.assertEqual(
convert_to_sets(actual), convert_to_sets(expected), msg=msg
)
class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir):
def test(self):
fs = LocalFileSystem(None, {"url": self._root_dir})
self.assertWalkEqual(
fs.walk(self._root_dir),
[
(
self._root_dir,
["data_dir"],
["code.py", "bar", "тест", "foo"],
),
(join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]),
(
join(self._root_dir, "data_dir", "data_sub_dir"),
[],
["data_sub"],
),
],
)
def test_subdir(self):
fs = LocalFileSystem(None, {"url": self._root_dir})
self.assertWalkEqual(
fs.walk(join("data_dir", "data_sub_dir")),
[(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
)
class TestWalkInGit(AssertWalkEqualMixin, TestGit):
def test_nobranch(self):
fs = LocalFileSystem(None, {"url": self._root_dir}, use_dvcignore=True)
self.assertWalkEqual(
fs.walk("."),
[
(".", ["data_dir"], ["bar", "тест", "code.py", "foo"]),
(join("data_dir"), ["data_sub_dir"], ["data"]),
(join("data_dir", "data_sub_dir"), [], ["data_sub"]),
],
)
self.assertWalkEqual(
fs.walk(join("data_dir", "data_sub_dir")),
[(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
)
def test_branch(self):
scm = SCM(self._root_dir)
scm.add([self.DATA_SUB_DIR])
scm.commit("add data_dir/data_sub_dir/data_sub")
fs = scm.get_fs("master")
self.assertWalkEqual(
fs.walk("."),
[
(self._root_dir, ["data_dir"], ["code.py"]),
(join(self._root_dir, "data_dir"), ["data_sub_dir"], []),
(
join(self._root_dir, "data_dir", "data_sub_dir"),
[],
["data_sub"],
),
],
)
self.assertWalkEqual(
fs.walk(join("data_dir", "data_sub_dir")),
[
(
join(self._root_dir, "data_dir", "data_sub_dir"),
[],
["data_sub"],
)
],
)
def test_cleanfs_subrepo(tmp_dir, dvc, scm, monkeypatch):
tmp_dir.gen({"subdir": {}})
subrepo_dir = tmp_dir / "subdir"
with subrepo_dir.chdir():
subrepo = Repo.init(subdir=True)
subrepo_dir.gen({"foo": "foo", "dir": {"bar": "bar"}})
path = PathInfo(subrepo_dir)
assert dvc.fs.use_dvcignore
assert not dvc.fs.exists(path / "foo")
assert not dvc.fs.isfile(path / "foo")
assert not dvc.fs.exists(path / "dir")
assert not dvc.fs.isdir(path / "dir")
assert subrepo.fs.use_dvcignore
assert subrepo.fs.exists(path / "foo")
assert subrepo.fs.isfile(path / "foo")
assert subrepo.fs.exists(path / "dir")
assert subrepo.fs.isdir(path / "dir")
def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
tmp_dir.dvc_gen({"foo": "foo"}, commit="add foo")
subrepo_dir = tmp_dir / "subdir"
subrepo_dir.mkdir()
with subrepo_dir.chdir():
Repo.init(subdir=True)
scm.add(["subdir"])
scm.commit("Add subrepo")
dvc_fs = dvc.fs
dvc_fs._reset()
scm_fs = scm.get_fs("HEAD", use_dvcignore=True)
path = os.fspath(tmp_dir)
get_dirs = itemgetter(1)
assert get_dirs(next(dvc_fs.walk(path))) == []
assert get_dirs(next(scm_fs.walk(path))) == []
kw = {"ignore_subrepos": False}
assert get_dirs(next(dvc_fs.walk(path, **kw))) == ["subdir"]
assert get_dirs(next(scm_fs.walk(path, **kw))) == ["subdir"]
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("hdfs"),
pytest.lazy_fixture("http"),
],
)
def test_fs_getsize(dvc, cloud):
cloud.gen({"data": {"foo": "foo"}, "baz": "baz baz"})
fs = get_cloud_fs(dvc, **cloud.config)
path_info = fs.path_info
assert fs.getsize(path_info / "baz") == 7
assert fs.getsize(path_info / "data" / "foo") == 3
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("azure"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("gdrive"),
pytest.lazy_fixture("hdfs"),
pytest.lazy_fixture("http"),
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("oss"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("ssh"),
pytest.lazy_fixture("webhdfs"),
],
)
def test_fs_upload_fobj(dvc, tmp_dir, cloud):
tmp_dir.gen("foo", "foo")
fs = get_cloud_fs(dvc, **cloud.config)
from_info = tmp_dir / "foo"
to_info = fs.path_info / "foo"
with open(from_info, "rb") as stream:
fs.upload_fobj(stream, to_info)
assert fs.exists(to_info)
with fs.open(to_info, "rb") as stream:
assert stream.read() == b"foo"
@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("gdrive")])
def test_fs_ls(dvc, cloud):
cloud.gen(
{
"directory": {
"foo": "foo",
"bar": "bar",
"baz": {"quux": "quux", "egg": {"foo": "foo"}},
"empty": {},
}
}
)
fs = get_cloud_fs(dvc, **cloud.config)
path_info = cloud / "directory"
assert {os.path.basename(file_key) for file_key in fs.ls(path_info)} == {
"foo",
"bar",
"baz",
"empty",
}
assert set(fs.ls(path_info / "empty")) == set()
assert {
(detail["type"], os.path.basename(detail["name"]))
for detail in fs.ls(path_info / "baz", detail=True)
} == {("file", "quux"), ("directory", "egg")}
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("azure"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("webdav"),
pytest.lazy_fixture("gdrive"),
],
)
def test_fs_ls_recursive(dvc, cloud):
cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}})
fs = get_cloud_fs(dvc, **cloud.config)
path_info = fs.path_info
assert {
os.path.basename(file_key)
for file_key in fs.ls(path_info / "data", recursive=True)
} == {"foo", "baz", "quux"}
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("azure"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("webdav"),
],
)
def test_fs_ls_with_etag(dvc, cloud):
cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}})
fs = get_cloud_fs(dvc, **cloud.config)
path_info = fs.path_info
for details in fs.ls(path_info / "data", recursive=True, detail=True):
assert (
fs.info(path_info.replace(path=details["name"]))["etag"]
== details["etag"]
)
+4 -2
View File
@@ -8,9 +8,9 @@ from git import Repo
from dvc.cache.local import LocalCache
from dvc.exceptions import CollectCacheError
from dvc.fs.local import LocalFileSystem
from dvc.main import main
from dvc.repo import Repo as DvcRepo
from dvc.tree.local import LocalTree
from dvc.utils.fs import remove
from tests.basic_env import TestDir, TestDvcGit
@@ -321,7 +321,9 @@ def test_gc_cloud_remove_order(tmp_dir, scm, dvc, tmp_path_factory, mocker):
dvc.remove(dir2.relpath)
dvc.gc(workspace=True)
mocked_remove = mocker.patch.object(LocalTree, "remove", autospec=True)
mocked_remove = mocker.patch.object(
LocalFileSystem, "remove", autospec=True
)
dvc.gc(workspace=True, cloud=True)
assert len(mocked_remove.mock_calls) == 8
# dir (and unpacked dir) should be first 4 checksums removed from
+1 -1
View File
@@ -9,7 +9,7 @@ from dvc.repo import Repo
from dvc.repo.get import GetDVCFileError
from dvc.system import System
from dvc.utils.fs import makedirs
from tests.unit.tree.test_repo import make_subrepo
from tests.unit.fs.test_repo import make_subrepo
def test_get_repo_file(tmp_dir, erepo_dir):
+48 -50
View File
@@ -23,41 +23,41 @@ def test_ignore(tmp_dir, dvc, monkeypatch):
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
dvc.tree._reset()
dvc.fs._reset()
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"}
assert set(dvc.fs.walk_files(path / "dir")) == {path / "dir" / "other"}
def test_ignore_unicode(tmp_dir, dvc):
tmp_dir.gen({"dir": {"other": "text", "тест": "проверка"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест")
dvc.tree._reset()
dvc.fs._reset()
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"}
assert set(dvc.fs.walk_files(path / "dir")) == {path / "dir" / "other"}
def test_rename_ignored_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*")
dvc.tree._reset()
dvc.fs._reset()
mtime, size = get_mtime_and_size("dir", dvc.tree)
mtime, size = get_mtime_and_size("dir", dvc.fs)
shutil.move("dir/ignored", "dir/ignored_new")
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
new_mtime, new_size = get_mtime_and_size("dir", dvc.fs)
assert new_mtime == mtime and new_size == size
def test_rename_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
mtime, size = get_mtime_and_size("dir", dvc.tree)
mtime, size = get_mtime_and_size("dir", dvc.fs)
shutil.move("dir/foo", "dir/foo_new")
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
new_mtime, new_size = get_mtime_and_size("dir", dvc.fs)
assert new_mtime != mtime and new_size == size
@@ -65,22 +65,22 @@ def test_rename_file(tmp_dir, dvc):
def test_remove_ignored_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
dvc.tree._reset()
dvc.fs._reset()
mtime, size = get_mtime_and_size("dir", dvc.tree)
mtime, size = get_mtime_and_size("dir", dvc.fs)
os.remove("dir/ignored")
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
new_mtime, new_size = get_mtime_and_size("dir", dvc.fs)
assert new_mtime == mtime and new_size == size
def test_remove_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
mtime, size = get_mtime_and_size("dir", dvc.tree)
mtime, size = get_mtime_and_size("dir", dvc.fs)
os.remove("dir/foo")
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
new_mtime, new_size = get_mtime_and_size("dir", dvc.fs)
assert new_mtime != mtime and new_size != size
@@ -98,12 +98,12 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE)
top_ignore_file.write_text(os.path.basename(dname))
dvc.tree._reset()
dvc.fs._reset()
ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE
ignore_file.write_text("foo")
dvcignore = dvc.tree.dvcignore
dvcignore = dvc.fs.dvcignore
top_ignore_path = os.path.dirname(os.fspath(top_ignore_file))
@@ -129,16 +129,16 @@ def test_ignore_on_branch(tmp_dir, scm, dvc):
with tmp_dir.branch("branch", new=True):
tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
dvc.tree._reset()
dvc.fs._reset()
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path)) == {
assert set(dvc.fs.walk_files(path)) == {
path / "foo",
path / "bar",
path / ".dvcignore",
}
dvc.tree = scm.get_tree("branch", use_dvcignore=True)
assert set(dvc.tree.walk_files(path)) == {
dvc.fs = scm.get_fs("branch", use_dvcignore=True)
assert set(dvc.fs.walk_files(path)) == {
os.fspath(path / DvcIgnore.DVCIGNORE_FILE),
os.fspath(path / "bar"),
}
@@ -153,8 +153,8 @@ def test_match_nested(tmp_dir, dvc):
"dir": {"x.backup": "x backup", "tmp": "content"},
}
)
dvc.tree._reset()
result = {os.fspath(os.path.normpath(f)) for f in dvc.tree.walk_files(".")}
dvc.fs._reset()
result = {os.fspath(os.path.normpath(f)) for f in dvc.fs.walk_files(".")}
assert result == {".dvcignore", "foo"}
@@ -163,13 +163,11 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
ext_dir = TmpDir(os.fspath(tmp_path_factory.mktemp("external_dir")))
ext_dir.gen({"y.backup": "y", "tmp": {"file": "ext tmp"}})
result = {relpath(f, ext_dir) for f in dvc.tree.walk_files(ext_dir)}
result = {relpath(f, ext_dir) for f in dvc.fs.walk_files(ext_dir)}
assert result == {"y.backup", os.path.join("tmp", "file")}
assert dvc.fs.dvcignore.is_ignored_dir(os.fspath(ext_dir / "tmp")) is False
assert (
dvc.tree.dvcignore.is_ignored_dir(os.fspath(ext_dir / "tmp")) is False
)
assert (
dvc.tree.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup"))
dvc.fs.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup"))
is False
)
@@ -178,10 +176,10 @@ def test_ignore_subrepo(tmp_dir, scm, dvc):
tmp_dir.gen({".dvcignore": "foo", "subdir": {"foo": "foo"}})
scm.add([".dvcignore"])
scm.commit("init parent dvcignore")
dvc.tree._reset()
dvc.fs._reset()
subrepo_dir = tmp_dir / "subdir"
assert not dvc.tree.exists(PathInfo(subrepo_dir / "foo"))
assert not dvc.fs.exists(PathInfo(subrepo_dir / "foo"))
with subrepo_dir.chdir():
subrepo = Repo.init(subdir=True)
@@ -189,7 +187,7 @@ def test_ignore_subrepo(tmp_dir, scm, dvc):
scm.commit("subrepo init")
for _ in subrepo.brancher(all_commits=True):
assert subrepo.tree.exists(PathInfo(subrepo_dir / "foo"))
assert subrepo.fs.exists(PathInfo(subrepo_dir / "foo"))
def test_ignore_resurface_subrepo(tmp_dir, scm, dvc):
@@ -199,17 +197,17 @@ def test_ignore_resurface_subrepo(tmp_dir, scm, dvc):
with subrepo_dir.chdir():
Repo.init(subdir=True)
dvc.tree._reset()
dvc.fs._reset()
dirs = ["subdir"]
files = ["foo"]
assert dvc.tree.dvcignore(os.fspath(tmp_dir), dirs, files) == ([], files)
assert dvc.tree.dvcignore(
assert dvc.fs.dvcignore(os.fspath(tmp_dir), dirs, files) == ([], files)
assert dvc.fs.dvcignore(
os.fspath(tmp_dir), dirs, files, ignore_subrepos=False
) == (dirs, files)
assert dvc.tree.dvcignore.is_ignored_dir(os.fspath(subrepo_dir))
assert not dvc.tree.dvcignore.is_ignored_dir(
assert dvc.fs.dvcignore.is_ignored_dir(os.fspath(subrepo_dir))
assert not dvc.fs.dvcignore.is_ignored_dir(
os.fspath(subrepo_dir), ignore_subrepos=False
)
@@ -217,9 +215,9 @@ def test_ignore_resurface_subrepo(tmp_dir, scm, dvc):
def test_ignore_blank_line(tmp_dir, dvc):
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "foo\n\ndir/ignored")
dvc.tree._reset()
dvc.fs._reset()
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"}
assert set(dvc.fs.walk_files(path / "dir")) == {path / "dir" / "other"}
# It is not possible to re-include a file if a parent directory of
@@ -252,9 +250,9 @@ def test_ignore_file_in_parent_path(
):
tmp_dir.gen(data_struct)
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "\n".join(pattern_list))
dvc.tree._reset()
dvc.fs._reset()
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {
assert set(dvc.fs.walk_files(path / "dir")) == {
path / relpath for relpath in result_set
}
@@ -275,9 +273,9 @@ def test_ignore_sub_directory(tmp_dir, dvc):
)
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "doc/fortz"}})
dvc.tree._reset()
dvc.fs._reset()
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {
assert set(dvc.fs.walk_files(path / "dir")) == {
path / "dir" / "a" / "doc" / "fortz" / "a",
path / "dir" / DvcIgnore.DVCIGNORE_FILE,
}
@@ -287,9 +285,9 @@ def test_ignore_sub_directory(tmp_dir, dvc):
def test_ignore_directory(tmp_dir, dvc):
tmp_dir.gen({"dir": {"fortz": {}, "a": {"fortz": {}}}})
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "fortz"}})
dvc.tree._reset()
dvc.fs._reset()
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {
assert set(dvc.fs.walk_files(path / "dir")) == {
path / "dir" / DvcIgnore.DVCIGNORE_FILE,
}
@@ -298,15 +296,15 @@ def test_multi_ignore_file(tmp_dir, dvc, monkeypatch):
tmp_dir.gen({"dir": {"subdir": {"should_ignore": "1", "not_ignore": "1"}}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/subdir/*_ignore")
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "!subdir/not_ignore"}})
dvc.tree._reset()
dvc.fs._reset()
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {
assert set(dvc.fs.walk_files(path / "dir")) == {
path / "dir" / "subdir" / "not_ignore",
path / "dir" / DvcIgnore.DVCIGNORE_FILE,
}
def test_pattern_trie_tree(tmp_dir, dvc):
def test_pattern_trie_fs(tmp_dir, dvc):
tmp_dir.gen(
{
"top": {
@@ -323,8 +321,8 @@ def test_pattern_trie_tree(tmp_dir, dvc):
"other": {DvcIgnore.DVCIGNORE_FILE: "1\n2\n3"},
}
)
dvc.tree._reset()
dvcignore = dvc.tree.dvcignore
dvc.fs._reset()
dvcignore = dvc.fs.dvcignore
ignore_pattern_top = dvcignore._get_trie_pattern(
os.fspath(tmp_dir / "top")
@@ -391,10 +389,10 @@ def test_ignore_in_added_dir(tmp_dir, dvc):
".dvcignore": "**/ignored",
}
)
dvc.tree._reset()
dvc.fs._reset()
ignored_path = tmp_dir / "dir" / "sub" / "ignored"
assert not dvc.tree.exists(PathInfo(ignored_path))
assert not dvc.fs.exists(PathInfo(ignored_path))
assert ignored_path.exists()
dvc.add("dir")
+2 -2
View File
@@ -13,7 +13,7 @@ from dvc.exceptions import DownloadError
from dvc.stage.exceptions import StagePathNotFoundError
from dvc.system import System
from dvc.utils.fs import makedirs, remove
from tests.unit.tree.test_repo import make_subrepo
from tests.unit.fs.test_repo import make_subrepo
def test_import(tmp_dir, scm, dvc, erepo_dir):
@@ -305,7 +305,7 @@ def test_download_error_pulling_imported_stage(tmp_dir, dvc, erepo_dir):
remove(dst_cache)
with patch(
"dvc.tree.local.LocalTree._download", side_effect=Exception
"dvc.fs.local.LocalFileSystem._download", side_effect=Exception
), pytest.raises(DownloadError):
dvc.pull(["foo_imported.dvc"])
+1 -1
View File
@@ -64,7 +64,7 @@ class TestInstall:
conf["core"]["remote"] = "store"
tmp_dir.dvc_gen("file", "file_content", "commit message")
file_checksum = file_md5("file", dvc.tree)
file_checksum = file_md5("file", dvc.fs)
expected_storage_path = (
storage_path / file_checksum[:2] / file_checksum[2:]
)

Some files were not shown because too many files have changed in this diff Show More