mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-01 23:08:43 +01:00
dvc: rename tree -> fs (#5461)
This commit is contained in:
+2
-2
@@ -23,13 +23,13 @@ def get_url(path, repo=None, rev=None, remote=None):
|
||||
with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
|
||||
path_info = PathInfo(_repo.root_dir) / path
|
||||
with reraise(FileNotFoundError, PathMissingError(path, repo)):
|
||||
metadata = _repo.repo_tree.metadata(path_info)
|
||||
metadata = _repo.repo_fs.metadata(path_info)
|
||||
|
||||
if not metadata.is_dvc:
|
||||
raise OutputNotFoundError(path, repo)
|
||||
|
||||
cloud = metadata.repo.cloud
|
||||
hash_info = get_hash(path_info, _repo.repo_tree, "md5")
|
||||
hash_info = get_hash(path_info, _repo.repo_fs, "md5")
|
||||
return cloud.get_url_for(remote, checksum=hash_info.value)
|
||||
|
||||
|
||||
|
||||
Vendored
+11
-11
@@ -4,32 +4,32 @@ from collections import defaultdict
|
||||
from ..scheme import Schemes
|
||||
|
||||
|
||||
def get_cloud_cache(tree):
|
||||
def get_cloud_cache(fs):
|
||||
from .base import CloudCache
|
||||
from .gdrive import GDriveCache
|
||||
from .local import LocalCache
|
||||
from .ssh import SSHCache
|
||||
|
||||
if tree.scheme == Schemes.LOCAL:
|
||||
return LocalCache(tree)
|
||||
if fs.scheme == Schemes.LOCAL:
|
||||
return LocalCache(fs)
|
||||
|
||||
if tree.scheme == Schemes.SSH:
|
||||
return SSHCache(tree)
|
||||
if fs.scheme == Schemes.SSH:
|
||||
return SSHCache(fs)
|
||||
|
||||
if tree.scheme == Schemes.GDRIVE:
|
||||
return GDriveCache(tree)
|
||||
if fs.scheme == Schemes.GDRIVE:
|
||||
return GDriveCache(fs)
|
||||
|
||||
return CloudCache(tree)
|
||||
return CloudCache(fs)
|
||||
|
||||
|
||||
def _get_cache(repo, settings):
|
||||
from ..tree import get_cloud_tree
|
||||
from ..fs import get_cloud_fs
|
||||
|
||||
if not settings:
|
||||
return None
|
||||
|
||||
tree = get_cloud_tree(repo, **settings)
|
||||
return get_cloud_cache(tree)
|
||||
fs = get_cloud_fs(repo, **settings)
|
||||
return get_cloud_cache(fs)
|
||||
|
||||
|
||||
class Cache:
|
||||
|
||||
Vendored
+51
-53
@@ -17,32 +17,32 @@ class CloudCache:
|
||||
DEFAULT_CACHE_TYPES = ["copy"]
|
||||
CACHE_MODE: Optional[int] = None
|
||||
|
||||
def __init__(self, tree):
|
||||
self.tree = tree
|
||||
self.repo = tree.repo
|
||||
def __init__(self, fs):
|
||||
self.fs = fs
|
||||
self.repo = fs.repo
|
||||
|
||||
self.verify = tree.config.get("verify", self.DEFAULT_VERIFY)
|
||||
self.cache_types = tree.config.get("type") or copy(
|
||||
self.verify = fs.config.get("verify", self.DEFAULT_VERIFY)
|
||||
self.cache_types = fs.config.get("type") or copy(
|
||||
self.DEFAULT_CACHE_TYPES
|
||||
)
|
||||
self.cache_type_confirmed = False
|
||||
|
||||
def move(self, from_info, to_info):
|
||||
self.tree.move(from_info, to_info)
|
||||
self.fs.move(from_info, to_info)
|
||||
|
||||
def makedirs(self, path_info):
|
||||
self.tree.makedirs(path_info)
|
||||
self.fs.makedirs(path_info)
|
||||
|
||||
def get(self, hash_info):
|
||||
""" get raw object """
|
||||
return HashFile(
|
||||
# Prefer string path over PathInfo when possible due to performance
|
||||
self.hash_to_path(hash_info.value),
|
||||
self.tree,
|
||||
self.fs,
|
||||
hash_info,
|
||||
)
|
||||
|
||||
def add(self, path_info, tree, hash_info, **kwargs):
|
||||
def add(self, path_info, fs, hash_info, **kwargs):
|
||||
try:
|
||||
self.check(hash_info)
|
||||
return
|
||||
@@ -52,21 +52,21 @@ class CloudCache:
|
||||
cache_info = self.hash_to_path_info(hash_info.value)
|
||||
# using our makedirs to create dirs with proper permissions
|
||||
self.makedirs(cache_info.parent)
|
||||
if isinstance(tree, type(self.tree)):
|
||||
self.tree.move(path_info, cache_info)
|
||||
if isinstance(fs, type(self.fs)):
|
||||
self.fs.move(path_info, cache_info)
|
||||
else:
|
||||
with tree.open(path_info, mode="rb") as fobj:
|
||||
self.tree.upload_fobj(fobj, cache_info)
|
||||
with fs.open(path_info, mode="rb") as fobj:
|
||||
self.fs.upload_fobj(fobj, cache_info)
|
||||
self.protect(cache_info)
|
||||
with self.tree.state:
|
||||
self.tree.state.save(cache_info, hash_info)
|
||||
with self.fs.state:
|
||||
self.fs.state.save(cache_info, hash_info)
|
||||
|
||||
callback = kwargs.get("download_callback")
|
||||
if callback:
|
||||
callback(1)
|
||||
|
||||
def hash_to_path_info(self, hash_):
|
||||
return self.tree.path_info / hash_[0:2] / hash_[2:]
|
||||
return self.fs.path_info / hash_[0:2] / hash_[2:]
|
||||
|
||||
# Override to return path as a string instead of PathInfo for clouds
|
||||
# which support string paths (see local)
|
||||
@@ -110,7 +110,7 @@ class CloudCache:
|
||||
obj.check(self)
|
||||
except ObjectFormatError:
|
||||
logger.warning("corrupted cache file '%s'.", obj.path_info)
|
||||
self.tree.remove(obj.path_info)
|
||||
self.fs.remove(obj.path_info)
|
||||
raise
|
||||
|
||||
# making cache file read-only so we don't need to check it
|
||||
@@ -120,22 +120,22 @@ class CloudCache:
|
||||
def _list_paths(self, prefix=None, progress_callback=None):
|
||||
if prefix:
|
||||
if len(prefix) > 2:
|
||||
path_info = self.tree.path_info / prefix[:2] / prefix[2:]
|
||||
path_info = self.fs.path_info / prefix[:2] / prefix[2:]
|
||||
else:
|
||||
path_info = self.tree.path_info / prefix[:2]
|
||||
path_info = self.fs.path_info / prefix[:2]
|
||||
prefix = True
|
||||
else:
|
||||
path_info = self.tree.path_info
|
||||
path_info = self.fs.path_info
|
||||
prefix = False
|
||||
if progress_callback:
|
||||
for file_info in self.tree.walk_files(path_info, prefix=prefix):
|
||||
for file_info in self.fs.walk_files(path_info, prefix=prefix):
|
||||
progress_callback()
|
||||
yield file_info.path
|
||||
else:
|
||||
yield from self.tree.walk_files(path_info, prefix=prefix)
|
||||
yield from self.fs.walk_files(path_info, prefix=prefix)
|
||||
|
||||
def _path_to_hash(self, path):
|
||||
parts = self.tree.PATH_CLS(path).parts[-2:]
|
||||
parts = self.fs.PATH_CLS(path).parts[-2:]
|
||||
|
||||
if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
|
||||
raise ValueError(f"Bad cache file path '{path}'")
|
||||
@@ -143,7 +143,7 @@ class CloudCache:
|
||||
return "".join(parts)
|
||||
|
||||
def list_hashes(self, prefix=None, progress_callback=None):
|
||||
"""Iterate over hashes in this tree.
|
||||
"""Iterate over hashes in this fs.
|
||||
|
||||
If `prefix` is specified, only hashes which begin with `prefix`
|
||||
will be returned.
|
||||
@@ -171,18 +171,18 @@ class CloudCache:
|
||||
def _max_estimation_size(self, hashes):
|
||||
# Max remote size allowed for us to use traverse method
|
||||
return max(
|
||||
self.tree.TRAVERSE_THRESHOLD_SIZE,
|
||||
self.fs.TRAVERSE_THRESHOLD_SIZE,
|
||||
len(hashes)
|
||||
/ self.tree.TRAVERSE_WEIGHT_MULTIPLIER
|
||||
* self.tree.LIST_OBJECT_PAGE_SIZE,
|
||||
/ self.fs.TRAVERSE_WEIGHT_MULTIPLIER
|
||||
* self.fs.LIST_OBJECT_PAGE_SIZE,
|
||||
)
|
||||
|
||||
def _estimate_remote_size(self, hashes=None, name=None):
|
||||
"""Estimate tree size based on number of entries beginning with
|
||||
"""Estimate fs size based on number of entries beginning with
|
||||
"00..." prefix.
|
||||
"""
|
||||
prefix = "0" * self.tree.TRAVERSE_PREFIX_LEN
|
||||
total_prefixes = pow(16, self.tree.TRAVERSE_PREFIX_LEN)
|
||||
prefix = "0" * self.fs.TRAVERSE_PREFIX_LEN
|
||||
total_prefixes = pow(16, self.fs.TRAVERSE_PREFIX_LEN)
|
||||
if hashes:
|
||||
max_hashes = self._max_estimation_size(hashes)
|
||||
else:
|
||||
@@ -215,7 +215,7 @@ class CloudCache:
|
||||
def list_hashes_traverse(
|
||||
self, remote_size, remote_hashes, jobs=None, name=None
|
||||
):
|
||||
"""Iterate over all hashes found in this tree.
|
||||
"""Iterate over all hashes found in this fs.
|
||||
Hashes are fetched in parallel according to prefix, except in
|
||||
cases where the remote size is very small.
|
||||
|
||||
@@ -229,8 +229,8 @@ class CloudCache:
|
||||
sense to use a generator to gradually iterate over it, without
|
||||
keeping all of it in memory.
|
||||
"""
|
||||
num_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
|
||||
if num_pages < 256 / self.tree.JOBS:
|
||||
num_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE
|
||||
if num_pages < 256 / self.fs.JOBS:
|
||||
# Fetching prefixes in parallel requires at least 255 more
|
||||
# requests, for small enough remotes it will be faster to fetch
|
||||
# entire cache without splitting it into prefixes.
|
||||
@@ -243,12 +243,10 @@ class CloudCache:
|
||||
yield from remote_hashes
|
||||
initial = len(remote_hashes)
|
||||
traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
|
||||
if self.tree.TRAVERSE_PREFIX_LEN > 2:
|
||||
if self.fs.TRAVERSE_PREFIX_LEN > 2:
|
||||
traverse_prefixes += [
|
||||
"{0:0{1}x}".format(i, self.tree.TRAVERSE_PREFIX_LEN)
|
||||
for i in range(
|
||||
1, pow(16, self.tree.TRAVERSE_PREFIX_LEN - 2)
|
||||
)
|
||||
"{0:0{1}x}".format(i, self.fs.TRAVERSE_PREFIX_LEN)
|
||||
for i in range(1, pow(16, self.fs.TRAVERSE_PREFIX_LEN - 2))
|
||||
]
|
||||
with Tqdm(
|
||||
desc="Querying "
|
||||
@@ -266,13 +264,13 @@ class CloudCache:
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=jobs or self.tree.JOBS
|
||||
max_workers=jobs or self.fs.JOBS
|
||||
) as executor:
|
||||
in_remote = executor.map(list_with_update, traverse_prefixes,)
|
||||
yield from itertools.chain.from_iterable(in_remote)
|
||||
|
||||
def all(self, jobs=None, name=None):
|
||||
"""Iterate over all hashes in this tree.
|
||||
"""Iterate over all hashes in this fs.
|
||||
|
||||
Hashes will be fetched in parallel threads according to prefix
|
||||
(except for small remotes) and a progress bar will be displayed.
|
||||
@@ -283,7 +281,7 @@ class CloudCache:
|
||||
)
|
||||
)
|
||||
|
||||
if not self.tree.CAN_TRAVERSE:
|
||||
if not self.fs.CAN_TRAVERSE:
|
||||
return self.list_hashes()
|
||||
|
||||
remote_size, remote_hashes = self._estimate_remote_size(name=name)
|
||||
@@ -298,24 +296,24 @@ class CloudCache:
|
||||
removed = False
|
||||
# hashes must be sorted to ensure we always remove .dir files first
|
||||
for hash_ in sorted(
|
||||
self.all(jobs, str(self.tree.path_info)),
|
||||
key=self.tree.is_dir_hash,
|
||||
self.all(jobs, str(self.fs.path_info)),
|
||||
key=self.fs.is_dir_hash,
|
||||
reverse=True,
|
||||
):
|
||||
if hash_ in used:
|
||||
continue
|
||||
path_info = self.hash_to_path_info(hash_)
|
||||
if self.tree.is_dir_hash(hash_):
|
||||
if self.fs.is_dir_hash(hash_):
|
||||
# backward compatibility
|
||||
# pylint: disable=protected-access
|
||||
self._remove_unpacked_dir(hash_)
|
||||
self.tree.remove(path_info)
|
||||
self.fs.remove(path_info)
|
||||
removed = True
|
||||
|
||||
return removed
|
||||
|
||||
def list_hashes_exists(self, hashes, jobs=None, name=None):
|
||||
"""Return list of the specified hashes which exist in this tree.
|
||||
"""Return list of the specified hashes which exist in this fs.
|
||||
Hashes will be queried individually.
|
||||
"""
|
||||
logger.debug(
|
||||
@@ -329,12 +327,12 @@ class CloudCache:
|
||||
) as pbar:
|
||||
|
||||
def exists_with_progress(path_info):
|
||||
ret = self.tree.exists(path_info)
|
||||
ret = self.fs.exists(path_info)
|
||||
pbar.update_msg(str(path_info))
|
||||
return ret
|
||||
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=jobs or self.tree.JOBS
|
||||
max_workers=jobs or self.fs.JOBS
|
||||
) as executor:
|
||||
path_infos = map(self.hash_to_path_info, hashes)
|
||||
in_remote = executor.map(exists_with_progress, path_infos)
|
||||
@@ -376,24 +374,24 @@ class CloudCache:
|
||||
"""
|
||||
# Remotes which do not use traverse prefix should override
|
||||
# hashes_exist() (see ssh, local)
|
||||
assert self.tree.TRAVERSE_PREFIX_LEN >= 2
|
||||
assert self.fs.TRAVERSE_PREFIX_LEN >= 2
|
||||
|
||||
hashes = set(hashes)
|
||||
if len(hashes) == 1 or not self.tree.CAN_TRAVERSE:
|
||||
if len(hashes) == 1 or not self.fs.CAN_TRAVERSE:
|
||||
remote_hashes = self.list_hashes_exists(hashes, jobs, name)
|
||||
return remote_hashes
|
||||
|
||||
# Max remote size allowed for us to use traverse method
|
||||
remote_size, remote_hashes = self._estimate_remote_size(hashes, name)
|
||||
|
||||
traverse_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
|
||||
traverse_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE
|
||||
# For sufficiently large remotes, traverse must be weighted to account
|
||||
# for performance overhead from large lists/sets.
|
||||
# From testing with S3, for remotes with 1M+ files, object_exists is
|
||||
# faster until len(hashes) is at least 10k~100k
|
||||
if remote_size > self.tree.TRAVERSE_THRESHOLD_SIZE:
|
||||
if remote_size > self.fs.TRAVERSE_THRESHOLD_SIZE:
|
||||
traverse_weight = (
|
||||
traverse_pages * self.tree.TRAVERSE_WEIGHT_MULTIPLIER
|
||||
traverse_pages * self.fs.TRAVERSE_WEIGHT_MULTIPLIER
|
||||
)
|
||||
else:
|
||||
traverse_weight = traverse_pages
|
||||
|
||||
Vendored
+15
-15
@@ -22,11 +22,11 @@ class LocalCache(CloudCache):
|
||||
CACHE_MODE = 0o444
|
||||
UNPACKED_DIR_SUFFIX = ".unpacked"
|
||||
|
||||
def __init__(self, tree):
|
||||
super().__init__(tree)
|
||||
self.cache_dir = tree.config.get("url")
|
||||
def __init__(self, fs):
|
||||
super().__init__(fs)
|
||||
self.cache_dir = fs.config.get("url")
|
||||
|
||||
shared = tree.config.get("shared")
|
||||
shared = fs.config.get("shared")
|
||||
if shared:
|
||||
self._file_mode = 0o664
|
||||
self._dir_mode = 0o2775
|
||||
@@ -36,11 +36,11 @@ class LocalCache(CloudCache):
|
||||
|
||||
@property
|
||||
def cache_dir(self):
|
||||
return self.tree.path_info.fspath if self.tree.path_info else None
|
||||
return self.fs.path_info.fspath if self.fs.path_info else None
|
||||
|
||||
@cache_dir.setter
|
||||
def cache_dir(self, value):
|
||||
self.tree.path_info = PathInfo(value) if value else None
|
||||
self.fs.path_info = PathInfo(value) if value else None
|
||||
|
||||
@cached_property
|
||||
def cache_path(self):
|
||||
@@ -71,7 +71,7 @@ class LocalCache(CloudCache):
|
||||
unit="file",
|
||||
desc="Querying " + ("cache in " + name if name else "local cache"),
|
||||
):
|
||||
hash_info = HashInfo(self.tree.PARAM_CHECKSUM, hash_)
|
||||
hash_info = HashInfo(self.fs.PARAM_CHECKSUM, hash_)
|
||||
try:
|
||||
self.check(hash_info)
|
||||
ret.append(hash_)
|
||||
@@ -81,14 +81,14 @@ class LocalCache(CloudCache):
|
||||
return ret
|
||||
|
||||
def _list_paths(self, prefix=None, progress_callback=None):
|
||||
assert self.tree.path_info is not None
|
||||
assert self.fs.path_info is not None
|
||||
if prefix:
|
||||
path_info = self.tree.path_info / prefix[:2]
|
||||
if not self.tree.exists(path_info):
|
||||
path_info = self.fs.path_info / prefix[:2]
|
||||
if not self.fs.exists(path_info):
|
||||
return
|
||||
else:
|
||||
path_info = self.tree.path_info
|
||||
# NOTE: use utils.fs walk_files since tree.walk_files will not follow
|
||||
path_info = self.fs.path_info
|
||||
# NOTE: use utils.fs walk_files since fs.walk_files will not follow
|
||||
# symlinks
|
||||
if progress_callback:
|
||||
for path in walk_files(path_info):
|
||||
@@ -100,10 +100,10 @@ class LocalCache(CloudCache):
|
||||
def _remove_unpacked_dir(self, hash_):
|
||||
info = self.hash_to_path_info(hash_)
|
||||
path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
|
||||
self.tree.remove(path_info)
|
||||
self.fs.remove(path_info)
|
||||
|
||||
def _unprotect_file(self, path):
|
||||
if self.tree.is_symlink(path) or self.tree.is_hardlink(path):
|
||||
if self.fs.is_symlink(path) or self.fs.is_hardlink(path):
|
||||
logger.debug("Unprotecting '%s'", path)
|
||||
tmp = os.path.join(os.path.dirname(path), "." + uuid())
|
||||
|
||||
@@ -126,7 +126,7 @@ class LocalCache(CloudCache):
|
||||
os.chmod(path, self._file_mode)
|
||||
|
||||
def _unprotect_dir(self, path):
|
||||
for fname in self.tree.walk_files(path):
|
||||
for fname in self.fs.walk_files(path):
|
||||
self._unprotect_file(fname)
|
||||
|
||||
def unprotect(self, path_info):
|
||||
|
||||
Vendored
+8
-8
@@ -28,7 +28,7 @@ class SSHCache(CloudCache):
|
||||
callback(path)
|
||||
return ret
|
||||
|
||||
with self.tree.ssh(path_infos[0]) as ssh:
|
||||
with self.fs.ssh(path_infos[0]) as ssh:
|
||||
channels = ssh.open_max_sftp_channels()
|
||||
max_workers = len(channels)
|
||||
|
||||
@@ -47,11 +47,11 @@ class SSHCache(CloudCache):
|
||||
faster than current approach (relying on exists(path_info)) applied in
|
||||
remote/base.
|
||||
"""
|
||||
if not self.tree.CAN_TRAVERSE:
|
||||
if not self.fs.CAN_TRAVERSE:
|
||||
return list(set(hashes) & set(self.all()))
|
||||
|
||||
# possibly prompt for credentials before "Querying" progress output
|
||||
self.tree.ensure_credentials()
|
||||
self.fs.ensure_credentials()
|
||||
|
||||
with Tqdm(
|
||||
desc="Querying "
|
||||
@@ -64,10 +64,10 @@ class SSHCache(CloudCache):
|
||||
return self.batch_exists(chunks, callback=pbar.update_msg)
|
||||
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=jobs or self.tree.JOBS
|
||||
max_workers=jobs or self.fs.JOBS
|
||||
) as executor:
|
||||
path_infos = [self.hash_to_path_info(x) for x in hashes]
|
||||
chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
|
||||
chunks = to_chunks(path_infos, num_chunks=self.fs.JOBS)
|
||||
results = executor.map(exists_with_progress, chunks)
|
||||
in_remote = itertools.chain.from_iterable(results)
|
||||
ret = list(itertools.compress(hashes, in_remote))
|
||||
@@ -75,10 +75,10 @@ class SSHCache(CloudCache):
|
||||
|
||||
def _list_paths(self, prefix=None, progress_callback=None):
|
||||
if prefix:
|
||||
root = posixpath.join(self.tree.path_info.path, prefix[:2])
|
||||
root = posixpath.join(self.fs.path_info.path, prefix[:2])
|
||||
else:
|
||||
root = self.tree.path_info.path
|
||||
with self.tree.ssh(self.tree.path_info) as ssh:
|
||||
root = self.fs.path_info.path
|
||||
with self.fs.ssh(self.fs.path_info) as ssh:
|
||||
if prefix and not ssh.exists(root):
|
||||
return
|
||||
# If we simply return an iterator then with above closes instantly
|
||||
|
||||
+44
-44
@@ -18,7 +18,7 @@ from dvc.remote.slow_link_detection import ( # type: ignore[attr-defined]
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _changed(path_info, tree, obj, cache):
|
||||
def _changed(path_info, fs, obj, cache):
|
||||
logger.trace("checking if '%s'('%s') has changed.", path_info, obj)
|
||||
|
||||
try:
|
||||
@@ -30,7 +30,7 @@ def _changed(path_info, tree, obj, cache):
|
||||
return True
|
||||
|
||||
try:
|
||||
actual = get_hash(path_info, tree, obj.hash_info.name)
|
||||
actual = get_hash(path_info, fs, obj.hash_info.name)
|
||||
except FileNotFoundError:
|
||||
logger.debug("'%s' doesn't exist.", path_info)
|
||||
return True
|
||||
@@ -48,15 +48,15 @@ def _changed(path_info, tree, obj, cache):
|
||||
return False
|
||||
|
||||
|
||||
def _remove(path_info, tree, cache, force=False):
|
||||
if not tree.exists(path_info):
|
||||
def _remove(path_info, fs, cache, force=False):
|
||||
if not fs.exists(path_info):
|
||||
return
|
||||
|
||||
if force:
|
||||
tree.remove(path_info)
|
||||
fs.remove(path_info)
|
||||
return
|
||||
|
||||
current = get_hash(path_info, tree, tree.PARAM_CHECKSUM)
|
||||
current = get_hash(path_info, fs, fs.PARAM_CHECKSUM)
|
||||
try:
|
||||
obj = load(cache, current)
|
||||
check(cache, obj)
|
||||
@@ -69,26 +69,26 @@ def _remove(path_info, tree, cache, force=False):
|
||||
if not prompt.confirm(msg):
|
||||
raise ConfirmRemoveError(str(path_info))
|
||||
|
||||
tree.remove(path_info)
|
||||
fs.remove(path_info)
|
||||
|
||||
|
||||
def _verify_link(cache, path_info, link_type):
|
||||
if link_type == "hardlink" and cache.tree.getsize(path_info) == 0:
|
||||
if link_type == "hardlink" and cache.fs.getsize(path_info) == 0:
|
||||
return
|
||||
|
||||
if cache.cache_type_confirmed:
|
||||
return
|
||||
|
||||
is_link = getattr(cache.tree, f"is_{link_type}", None)
|
||||
is_link = getattr(cache.fs, f"is_{link_type}", None)
|
||||
if is_link and not is_link(path_info):
|
||||
cache.tree.remove(path_info)
|
||||
cache.fs.remove(path_info)
|
||||
raise DvcException(f"failed to verify {link_type}")
|
||||
|
||||
cache.cache_type_confirmed = True
|
||||
|
||||
|
||||
def _do_link(cache, from_info, to_info, link_method):
|
||||
if cache.tree.exists(to_info):
|
||||
if cache.fs.exists(to_info):
|
||||
raise DvcException(f"Link '{to_info}' already exists!")
|
||||
|
||||
link_method(from_info, to_info)
|
||||
@@ -101,7 +101,7 @@ def _do_link(cache, from_info, to_info, link_method):
|
||||
@slow_link_guard
|
||||
def _try_links(cache, from_info, to_info, link_types):
|
||||
while link_types:
|
||||
link_method = getattr(cache.tree, link_types[0])
|
||||
link_method = getattr(cache.fs, link_types[0])
|
||||
try:
|
||||
_do_link(cache, from_info, to_info, link_method)
|
||||
_verify_link(cache, to_info, link_types[0])
|
||||
@@ -117,7 +117,7 @@ def _try_links(cache, from_info, to_info, link_types):
|
||||
|
||||
|
||||
def _link(cache, from_info, to_info):
|
||||
assert cache.tree.isfile(from_info)
|
||||
assert cache.fs.isfile(from_info)
|
||||
cache.makedirs(to_info.parent)
|
||||
_try_links(cache, from_info, to_info, cache.cache_types)
|
||||
|
||||
@@ -131,76 +131,76 @@ def _cache_is_copy(cache, path_info):
|
||||
return True
|
||||
|
||||
workspace_file = path_info.with_name("." + uuid())
|
||||
test_cache_file = cache.tree.path_info / ".cache_type_test_file"
|
||||
if not cache.tree.exists(test_cache_file):
|
||||
test_cache_file = cache.fs.path_info / ".cache_type_test_file"
|
||||
if not cache.fs.exists(test_cache_file):
|
||||
cache.makedirs(test_cache_file.parent)
|
||||
with cache.tree.open(test_cache_file, "wb") as fobj:
|
||||
with cache.fs.open(test_cache_file, "wb") as fobj:
|
||||
fobj.write(bytes(1))
|
||||
try:
|
||||
_link(cache, test_cache_file, workspace_file)
|
||||
finally:
|
||||
cache.tree.remove(workspace_file)
|
||||
cache.tree.remove(test_cache_file)
|
||||
cache.fs.remove(workspace_file)
|
||||
cache.fs.remove(test_cache_file)
|
||||
|
||||
cache.cache_type_confirmed = True
|
||||
return cache.cache_types[0] == "copy"
|
||||
|
||||
|
||||
def _checkout_file(
|
||||
path_info, tree, obj, cache, force, progress_callback=None, relink=False,
|
||||
path_info, fs, obj, cache, force, progress_callback=None, relink=False,
|
||||
):
|
||||
"""The file is changed we need to checkout a new copy"""
|
||||
modified = False
|
||||
cache_info = cache.hash_to_path_info(obj.hash_info.value)
|
||||
if tree.exists(path_info):
|
||||
if not relink and _changed(path_info, tree, obj, cache):
|
||||
if fs.exists(path_info):
|
||||
if not relink and _changed(path_info, fs, obj, cache):
|
||||
modified = True
|
||||
_remove(path_info, tree, cache, force=force)
|
||||
_remove(path_info, fs, cache, force=force)
|
||||
_link(cache, cache_info, path_info)
|
||||
else:
|
||||
if tree.iscopy(path_info) and _cache_is_copy(cache, path_info):
|
||||
if fs.iscopy(path_info) and _cache_is_copy(cache, path_info):
|
||||
cache.unprotect(path_info)
|
||||
else:
|
||||
_remove(path_info, tree, cache, force=force)
|
||||
_remove(path_info, fs, cache, force=force)
|
||||
_link(cache, cache_info, path_info)
|
||||
else:
|
||||
_link(cache, cache_info, path_info)
|
||||
modified = True
|
||||
|
||||
tree.state.save(path_info, obj.hash_info)
|
||||
fs.state.save(path_info, obj.hash_info)
|
||||
if progress_callback:
|
||||
progress_callback(str(path_info))
|
||||
|
||||
return modified
|
||||
|
||||
|
||||
def _remove_redundant_files(path_info, tree, dir_info, cache, force):
|
||||
existing_files = set(tree.walk_files(path_info))
|
||||
def _remove_redundant_files(path_info, fs, dir_info, cache, force):
|
||||
existing_files = set(fs.walk_files(path_info))
|
||||
|
||||
needed_files = {info for info, _ in dir_info.items(path_info)}
|
||||
redundant_files = existing_files - needed_files
|
||||
for path in redundant_files:
|
||||
_remove(path, tree, cache, force)
|
||||
_remove(path, fs, cache, force)
|
||||
|
||||
return bool(redundant_files)
|
||||
|
||||
|
||||
def _checkout_dir(
|
||||
path_info, tree, obj, cache, force, progress_callback=None, relink=False,
|
||||
path_info, fs, obj, cache, force, progress_callback=None, relink=False,
|
||||
):
|
||||
modified = False, False
|
||||
# Create dir separately so that dir is created
|
||||
# even if there are no files in it
|
||||
if not tree.exists(path_info):
|
||||
if not fs.exists(path_info):
|
||||
modified = True
|
||||
tree.makedirs(path_info)
|
||||
fs.makedirs(path_info)
|
||||
|
||||
logger.debug("Linking directory '%s'.", path_info)
|
||||
|
||||
for entry_info, entry_hash_info in obj.hash_info.dir_info.items(path_info):
|
||||
entry_modified = _checkout_file(
|
||||
entry_info,
|
||||
tree,
|
||||
fs,
|
||||
cache.get(entry_hash_info),
|
||||
cache,
|
||||
force,
|
||||
@@ -212,12 +212,12 @@ def _checkout_dir(
|
||||
|
||||
modified = (
|
||||
_remove_redundant_files(
|
||||
path_info, tree, obj.hash_info.dir_info, cache, force
|
||||
path_info, fs, obj.hash_info.dir_info, cache, force
|
||||
)
|
||||
or modified
|
||||
)
|
||||
|
||||
tree.state.save(path_info, obj.hash_info)
|
||||
fs.state.save(path_info, obj.hash_info)
|
||||
|
||||
# relink is not modified, assume it as nochange
|
||||
return modified and not relink
|
||||
@@ -225,7 +225,7 @@ def _checkout_dir(
|
||||
|
||||
def _checkout(
|
||||
path_info,
|
||||
tree,
|
||||
fs,
|
||||
obj,
|
||||
cache,
|
||||
force=False,
|
||||
@@ -234,21 +234,21 @@ def _checkout(
|
||||
):
|
||||
if not obj.hash_info.isdir:
|
||||
ret = _checkout_file(
|
||||
path_info, tree, obj, cache, force, progress_callback, relink
|
||||
path_info, fs, obj, cache, force, progress_callback, relink
|
||||
)
|
||||
else:
|
||||
ret = _checkout_dir(
|
||||
path_info, tree, obj, cache, force, progress_callback, relink,
|
||||
path_info, fs, obj, cache, force, progress_callback, relink,
|
||||
)
|
||||
|
||||
tree.state.save_link(path_info)
|
||||
fs.state.save_link(path_info)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def checkout(
|
||||
path_info,
|
||||
tree,
|
||||
fs,
|
||||
obj,
|
||||
cache,
|
||||
force=False,
|
||||
@@ -256,7 +256,7 @@ def checkout(
|
||||
relink=False,
|
||||
quiet=False,
|
||||
):
|
||||
if path_info.scheme not in ["local", cache.tree.scheme]:
|
||||
if path_info.scheme not in ["local", cache.fs.scheme]:
|
||||
raise NotImplementedError
|
||||
|
||||
failed = None
|
||||
@@ -267,10 +267,10 @@ def checkout(
|
||||
"No file hash info found for '%s'. It won't be created.",
|
||||
path_info,
|
||||
)
|
||||
_remove(path_info, tree, cache, force=force)
|
||||
_remove(path_info, fs, cache, force=force)
|
||||
failed = path_info
|
||||
|
||||
elif not relink and not _changed(path_info, tree, obj, cache):
|
||||
elif not relink and not _changed(path_info, fs, obj, cache):
|
||||
logger.trace("Data '%s' didn't change.", path_info)
|
||||
skip = True
|
||||
else:
|
||||
@@ -283,7 +283,7 @@ def checkout(
|
||||
obj.hash_info,
|
||||
path_info,
|
||||
)
|
||||
_remove(path_info, tree, cache, force=force)
|
||||
_remove(path_info, fs, cache, force=force)
|
||||
failed = path_info
|
||||
|
||||
if failed or skip:
|
||||
@@ -298,5 +298,5 @@ def checkout(
|
||||
logger.debug("Checking out '%s' with cache '%s'.", path_info, obj)
|
||||
|
||||
return _checkout(
|
||||
path_info, tree, obj, cache, force, progress_callback, relink,
|
||||
path_info, fs, obj, cache, force, progress_callback, relink,
|
||||
)
|
||||
|
||||
@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
|
||||
class CmdCheckIgnore(CmdBase):
|
||||
def __init__(self, args):
|
||||
super().__init__(args)
|
||||
self.ignore_filter = self.repo.tree.dvcignore
|
||||
self.ignore_filter = self.repo.fs.dvcignore
|
||||
|
||||
def _show_results(self, result):
|
||||
if not result.match and not self.args.non_matching:
|
||||
|
||||
+13
-13
@@ -63,9 +63,9 @@ class Config(dict):
|
||||
CONFIG_LOCAL = "config.local"
|
||||
|
||||
def __init__(
|
||||
self, dvc_dir=None, validate=True, tree=None, config=None,
|
||||
self, dvc_dir=None, validate=True, fs=None, config=None,
|
||||
): # pylint: disable=super-init-not-called
|
||||
from dvc.tree.local import LocalTree
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
|
||||
self.dvc_dir = dvc_dir
|
||||
|
||||
@@ -79,8 +79,8 @@ class Config(dict):
|
||||
else:
|
||||
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
|
||||
|
||||
self.wtree = LocalTree(None, {"url": self.dvc_dir})
|
||||
self.tree = tree or self.wtree
|
||||
self.wfs = LocalFileSystem(None, {"url": self.dvc_dir})
|
||||
self.fs = fs or self.wfs
|
||||
|
||||
self.load(validate=validate, config=config)
|
||||
|
||||
@@ -143,19 +143,19 @@ class Config(dict):
|
||||
if not self["cache"].get("dir") and self.dvc_dir:
|
||||
self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
|
||||
|
||||
def _get_tree(self, level):
|
||||
# NOTE: this might be a GitTree, which doesn't see things outside of
|
||||
def _get_fs(self, level):
|
||||
# NOTE: this might be a Gitfs, which doesn't see things outside of
|
||||
# the repo.
|
||||
return self.tree if level == "repo" else self.wtree
|
||||
return self.fs if level == "repo" else self.wfs
|
||||
|
||||
def _load_config(self, level):
|
||||
from configobj import ConfigObj
|
||||
|
||||
filename = self.files[level]
|
||||
tree = self._get_tree(level)
|
||||
fs = self._get_fs(level)
|
||||
|
||||
if tree.exists(filename, use_dvcignore=False):
|
||||
with tree.open(filename) as fobj:
|
||||
if fs.exists(filename, use_dvcignore=False):
|
||||
with fs.open(filename) as fobj:
|
||||
conf_obj = ConfigObj(fobj)
|
||||
else:
|
||||
conf_obj = ConfigObj()
|
||||
@@ -165,14 +165,14 @@ class Config(dict):
|
||||
from configobj import ConfigObj
|
||||
|
||||
filename = self.files[level]
|
||||
tree = self._get_tree(level)
|
||||
fs = self._get_fs(level)
|
||||
|
||||
logger.debug(f"Writing '{filename}'.")
|
||||
|
||||
tree.makedirs(os.path.dirname(filename))
|
||||
fs.makedirs(os.path.dirname(filename))
|
||||
|
||||
config = ConfigObj(_pack_remotes(conf_dict))
|
||||
with tree.open(filename, "wb") as fobj:
|
||||
with fs.open(filename, "wb") as fobj:
|
||||
config.write(fobj)
|
||||
config.filename = filename
|
||||
|
||||
|
||||
+3
-3
@@ -104,11 +104,11 @@ class DataCloud:
|
||||
command (str): the command which is benefitting from this function
|
||||
(to be used for reporting better error messages).
|
||||
"""
|
||||
from dvc.tree import get_cloud_tree
|
||||
from dvc.fs import get_cloud_fs
|
||||
|
||||
from_tree = get_cloud_tree(self.repo, url=source)
|
||||
from_fs = get_cloud_fs(self.repo, url=source)
|
||||
remote = self.get_remote(remote, command)
|
||||
return remote.transfer(from_tree, from_tree.path_info, jobs=jobs)
|
||||
return remote.transfer(from_fs, from_fs.path_info, jobs=jobs)
|
||||
|
||||
def status(
|
||||
self,
|
||||
|
||||
@@ -17,7 +17,7 @@ from dvc.dependency.webhdfs import WebHDFSDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from ..tree import get_cloud_tree
|
||||
from ..fs import get_cloud_fs
|
||||
from .repo import RepoDependency
|
||||
|
||||
DEPS = [
|
||||
@@ -64,8 +64,8 @@ SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
|
||||
def _get(stage, p, info):
|
||||
parsed = urlparse(p) if p else None
|
||||
if parsed and parsed.scheme == "remote":
|
||||
tree = get_cloud_tree(stage.repo, name=parsed.netloc)
|
||||
return DEP_MAP[tree.scheme](stage, p, info, tree=tree)
|
||||
fs = get_cloud_fs(stage.repo, name=parsed.netloc)
|
||||
return DEP_MAP[fs.scheme](stage, p, info, fs=fs)
|
||||
|
||||
if info and info.get(RepoDependency.PARAM_REPO):
|
||||
repo = info.pop(RepoDependency.PARAM_REPO)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
|
||||
from ..tree.azure import AzureTree
|
||||
from ..fs.azure import AzureFileSystem
|
||||
|
||||
|
||||
class AzureDependency(BaseDependency, BaseOutput):
|
||||
TREE_CLS = AzureTree
|
||||
FS_CLS = AzureFileSystem
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
|
||||
from ..tree.http import HTTPTree
|
||||
from ..fs.http import HTTPFileSystem
|
||||
|
||||
|
||||
class HTTPDependency(BaseDependency, BaseOutput):
|
||||
TREE_CLS = HTTPTree
|
||||
FS_CLS = HTTPFileSystem
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from ..tree.https import HTTPSTree
|
||||
from ..fs.https import HTTPSFileSystem
|
||||
from .http import HTTPDependency
|
||||
|
||||
|
||||
class HTTPSDependency(HTTPDependency):
|
||||
TREE_CLS = HTTPSTree
|
||||
FS_CLS = HTTPSFileSystem
|
||||
|
||||
@@ -91,7 +91,7 @@ class ParamsDependency(LocalDependency):
|
||||
suffix = self.path_info.suffix.lower()
|
||||
loader = LOADERS[suffix]
|
||||
try:
|
||||
config = loader(self.path_info, tree=self.repo.tree)
|
||||
config = loader(self.path_info, fs=self.repo.fs)
|
||||
except ParseError as exc:
|
||||
raise BadParamFileError(
|
||||
f"Unable to read parameters from '{self}'"
|
||||
|
||||
@@ -23,7 +23,7 @@ class RepoDependency(LocalDependency):
|
||||
self.def_repo = def_repo
|
||||
super().__init__(stage, *args, **kwargs)
|
||||
|
||||
def _parse_path(self, tree, path):
|
||||
def _parse_path(self, fs, path):
|
||||
return None
|
||||
|
||||
@property
|
||||
@@ -52,7 +52,7 @@ class RepoDependency(LocalDependency):
|
||||
with self._make_repo(locked=locked) as repo:
|
||||
path_info = PathInfo(repo.root_dir) / self.def_path
|
||||
return get_hash(
|
||||
path_info, repo.repo_tree, "md5", follow_subrepos=False
|
||||
path_info, repo.repo_fs, "md5", follow_subrepos=False
|
||||
)
|
||||
|
||||
def workspace_status(self):
|
||||
@@ -86,13 +86,13 @@ class RepoDependency(LocalDependency):
|
||||
obj = stage(
|
||||
cache,
|
||||
path_info,
|
||||
repo.repo_tree,
|
||||
repo.repo_fs,
|
||||
jobs=jobs,
|
||||
follow_subrepos=False,
|
||||
)
|
||||
save(cache, obj, jobs=jobs)
|
||||
|
||||
checkout(to.path_info, to.tree, obj, cache)
|
||||
checkout(to.path_info, to.fs, obj, cache)
|
||||
|
||||
def update(self, rev=None):
|
||||
if rev:
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
from dvc.dependency.base import BaseDependency
|
||||
from dvc.output.base import BaseOutput
|
||||
|
||||
from ..tree.webdav import WebDAVTree
|
||||
from ..fs.webdav import WebDAVFileSystem
|
||||
|
||||
|
||||
class WebDAVDependency(BaseDependency, BaseOutput):
|
||||
TREE_CLS = WebDAVTree
|
||||
FS_CLS = WebDAVFileSystem
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from ..tree.webdavs import WebDAVSTree
|
||||
from ..fs.webdavs import WebDAVSFileSystem
|
||||
from .webdav import WebDAVDependency
|
||||
|
||||
|
||||
class WebDAVSDependency(WebDAVDependency):
|
||||
TREE_CLS = WebDAVSTree
|
||||
FS_CLS = WebDAVSFileSystem
|
||||
|
||||
+9
-9
@@ -106,13 +106,13 @@ class FileMixin:
|
||||
return relpath(self.path)
|
||||
|
||||
def exists(self):
|
||||
return self.repo.tree.exists(self.path)
|
||||
return self.repo.fs.exists(self.path)
|
||||
|
||||
def _is_git_ignored(self):
|
||||
from dvc.tree.local import LocalTree
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
|
||||
return isinstance(
|
||||
self.repo.tree, LocalTree
|
||||
self.repo.fs, LocalFileSystem
|
||||
) and self.repo.scm.is_ignored(self.path)
|
||||
|
||||
def _load(self):
|
||||
@@ -122,16 +122,16 @@ class FileMixin:
|
||||
# 3. path doesn't represent a regular file
|
||||
# 4. when the file is git ignored
|
||||
if not self.exists():
|
||||
is_ignored = self.repo.tree.exists(self.path, use_dvcignore=False)
|
||||
is_ignored = self.repo.fs.exists(self.path, use_dvcignore=False)
|
||||
raise StageFileDoesNotExistError(self.path, dvc_ignored=is_ignored)
|
||||
|
||||
if self.verify:
|
||||
check_dvc_filename(self.path)
|
||||
if not self.repo.tree.isfile(self.path):
|
||||
if not self.repo.fs.isfile(self.path):
|
||||
raise StageFileIsNotDvcFileError(self.path)
|
||||
if self._is_git_ignored():
|
||||
raise FileIsGitIgnored(self.path)
|
||||
with self.repo.tree.open(self.path, encoding="utf-8") as fd:
|
||||
with self.repo.fs.open(self.path, encoding="utf-8") as fd:
|
||||
stage_text = fd.read()
|
||||
d = parse_yaml(stage_text, self.path)
|
||||
return self.validate(d, self.relpath), stage_text
|
||||
@@ -232,7 +232,7 @@ class PipelineFile(FileMixin):
|
||||
self._check_if_parametrized(stage)
|
||||
stage_data = serialize.to_pipeline_file(stage)
|
||||
|
||||
with modify_yaml(self.path, tree=self.repo.tree) as data:
|
||||
with modify_yaml(self.path, fs=self.repo.fs) as data:
|
||||
if not data:
|
||||
logger.info("Creating '%s'", self.relpath)
|
||||
|
||||
@@ -334,7 +334,7 @@ class Lockfile(FileMixin):
|
||||
if not self.exists():
|
||||
return {}
|
||||
|
||||
data = load_yaml(self.path, tree=self.repo.tree)
|
||||
data = load_yaml(self.path, fs=self.repo.fs)
|
||||
try:
|
||||
data = self.validate(data, fname=self.relpath)
|
||||
except StageFileFormatError as exc:
|
||||
@@ -351,7 +351,7 @@ class Lockfile(FileMixin):
|
||||
def dump(self, stage, **kwargs):
|
||||
stage_data = serialize.to_lockfile(stage)
|
||||
|
||||
with modify_yaml(self.path, tree=self.repo.tree) as data:
|
||||
with modify_yaml(self.path, fs=self.repo.fs) as data:
|
||||
version = LOCKFILE_VERSION.from_dict(data)
|
||||
if version == LOCKFILE_VERSION.V1:
|
||||
logger.info(
|
||||
|
||||
@@ -1,45 +1,45 @@
|
||||
import posixpath
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from .azure import AzureTree
|
||||
from .gdrive import GDriveTree
|
||||
from .gs import GSTree
|
||||
from .hdfs import HDFSTree
|
||||
from .http import HTTPTree
|
||||
from .https import HTTPSTree
|
||||
from .local import LocalTree
|
||||
from .oss import OSSTree
|
||||
from .s3 import S3Tree
|
||||
from .ssh import SSHTree
|
||||
from .webdav import WebDAVTree
|
||||
from .webdavs import WebDAVSTree
|
||||
from .webhdfs import WebHDFSTree
|
||||
from .azure import AzureFileSystem
|
||||
from .gdrive import GDriveFileSystem
|
||||
from .gs import GSFileSystem
|
||||
from .hdfs import HDFSFileSystem
|
||||
from .http import HTTPFileSystem
|
||||
from .https import HTTPSFileSystem
|
||||
from .local import LocalFileSystem
|
||||
from .oss import OSSFileSystem
|
||||
from .s3 import S3FileSystem
|
||||
from .ssh import SSHFileSystem
|
||||
from .webdav import WebDAVFileSystem
|
||||
from .webdavs import WebDAVSFileSystem
|
||||
from .webhdfs import WebHDFSFileSystem
|
||||
|
||||
TREES = [
|
||||
AzureTree,
|
||||
GDriveTree,
|
||||
GSTree,
|
||||
HDFSTree,
|
||||
HTTPTree,
|
||||
HTTPSTree,
|
||||
S3Tree,
|
||||
SSHTree,
|
||||
OSSTree,
|
||||
WebDAVTree,
|
||||
WebDAVSTree,
|
||||
WebHDFSTree
|
||||
# NOTE: LocalTree is the default
|
||||
ALL_FS = [
|
||||
AzureFileSystem,
|
||||
GDriveFileSystem,
|
||||
GSFileSystem,
|
||||
HDFSFileSystem,
|
||||
HTTPFileSystem,
|
||||
HTTPSFileSystem,
|
||||
S3FileSystem,
|
||||
SSHFileSystem,
|
||||
OSSFileSystem,
|
||||
WebDAVFileSystem,
|
||||
WebDAVSFileSystem,
|
||||
WebHDFSFileSystem
|
||||
# NOTE: LocalFileSystem is the default
|
||||
]
|
||||
|
||||
|
||||
def get_tree_cls(remote_conf):
|
||||
for tree_cls in TREES:
|
||||
if tree_cls.supported(remote_conf):
|
||||
return tree_cls
|
||||
return LocalTree
|
||||
def get_fs_cls(remote_conf):
|
||||
for fs_cls in ALL_FS:
|
||||
if fs_cls.supported(remote_conf):
|
||||
return fs_cls
|
||||
return LocalFileSystem
|
||||
|
||||
|
||||
def get_tree_config(config, **kwargs):
|
||||
def get_fs_config(config, **kwargs):
|
||||
name = kwargs.get("name")
|
||||
if name:
|
||||
remote_conf = config["remote"][name.lower()]
|
||||
@@ -74,18 +74,18 @@ def _resolve_remote_refs(config, remote_conf):
|
||||
if parsed.scheme != "remote":
|
||||
return remote_conf
|
||||
|
||||
base = get_tree_config(config, name=parsed.netloc)
|
||||
base = get_fs_config(config, name=parsed.netloc)
|
||||
url = posixpath.join(base["url"], parsed.path.lstrip("/"))
|
||||
return {**base, **remote_conf, "url": url}
|
||||
|
||||
|
||||
def get_cloud_tree(repo, **kwargs):
|
||||
def get_cloud_fs(repo, **kwargs):
|
||||
from dvc.config import ConfigError
|
||||
from dvc.config_schema import SCHEMA, Invalid
|
||||
|
||||
remote_conf = get_tree_config(repo.config, **kwargs)
|
||||
remote_conf = get_fs_config(repo.config, **kwargs)
|
||||
try:
|
||||
remote_conf = SCHEMA["remote"][str](remote_conf)
|
||||
except Invalid as exc:
|
||||
raise ConfigError(str(exc)) from None
|
||||
return get_tree_cls(remote_conf)(repo, remote_conf)
|
||||
return get_fs_cls(remote_conf)(repo, remote_conf)
|
||||
@@ -9,12 +9,12 @@ from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .base import BaseTree
|
||||
from .base import BaseFileSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AzureTree(BaseTree):
|
||||
class AzureFileSystem(BaseFileSystem):
|
||||
scheme = Schemes.AZURE
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {
|
||||
@@ -38,12 +38,12 @@ class RemoteMissingDepsError(DvcException):
|
||||
|
||||
@decorator
|
||||
def use_state(call):
|
||||
tree = call._args[0] # pylint: disable=protected-access
|
||||
with tree.state:
|
||||
fs = call._args[0] # pylint: disable=protected-access
|
||||
with fs.state:
|
||||
return call()
|
||||
|
||||
|
||||
class BaseTree:
|
||||
class BaseFileSystem:
|
||||
scheme = "base"
|
||||
REQUIRES: ClassVar[Dict[str, str]] = {}
|
||||
PATH_CLS = URLInfo # type: Any
|
||||
@@ -7,7 +7,7 @@ from dvc.path_info import PathInfo
|
||||
from dvc.utils import relpath
|
||||
|
||||
from ._metadata import Metadata
|
||||
from .base import BaseTree
|
||||
from .base import BaseFileSystem
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from dvc.output.base import BaseOutput
|
||||
@@ -16,8 +16,8 @@ if typing.TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DvcTree(BaseTree): # pylint:disable=abstract-method
|
||||
"""DVC repo tree.
|
||||
class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
"""DVC repo fs.
|
||||
|
||||
Args:
|
||||
repo: DVC repo.
|
||||
@@ -79,7 +79,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method
|
||||
else:
|
||||
checksum = out.hash_info.value
|
||||
remote_info = remote_obj.cache.hash_to_path_info(checksum)
|
||||
return remote_obj.tree.open(
|
||||
return remote_obj.fs.open(
|
||||
remote_info, mode=mode, encoding=encoding
|
||||
)
|
||||
|
||||
@@ -18,7 +18,7 @@ from dvc.scheme import Schemes
|
||||
from dvc.utils import format_link, tmp_fname
|
||||
from dvc.utils.stream import IterStream
|
||||
|
||||
from .base import BaseTree
|
||||
from .base import BaseFileSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
|
||||
@@ -85,7 +85,7 @@ class GDriveURLInfo(CloudURLInfo):
|
||||
self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
|
||||
|
||||
|
||||
class GDriveTree(BaseTree):
|
||||
class GDriveFileSystem(BaseFileSystem):
|
||||
scheme = Schemes.GDRIVE
|
||||
PATH_CLS = GDriveURLInfo
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
@@ -131,7 +131,7 @@ class GDriveTree(BaseTree):
|
||||
self._validate_config()
|
||||
self._gdrive_user_credentials_path = (
|
||||
tmp_fname(os.path.join(self.repo.tmp_dir, ""))
|
||||
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA)
|
||||
if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
|
||||
else config.get(
|
||||
"gdrive_user_credentials_file",
|
||||
os.path.join(
|
||||
@@ -173,8 +173,8 @@ class GDriveTree(BaseTree):
|
||||
Useful for tests, exception messages, etc. Returns either env variable
|
||||
name if it's set or actual path to the credentials file.
|
||||
"""
|
||||
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
|
||||
return GDriveTree.GDRIVE_CREDENTIALS_DATA
|
||||
if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
|
||||
return GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
|
||||
if os.path.exists(self._gdrive_user_credentials_path):
|
||||
return self._gdrive_user_credentials_path
|
||||
return None
|
||||
@@ -188,7 +188,7 @@ class GDriveTree(BaseTree):
|
||||
DVC config client id or secret but forgets to remove the cached
|
||||
credentials file.
|
||||
"""
|
||||
if not os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
|
||||
if not os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
|
||||
if (
|
||||
settings["client_config"]["client_id"]
|
||||
!= auth.credentials.client_id
|
||||
@@ -211,9 +211,11 @@ class GDriveTree(BaseTree):
|
||||
from pydrive2.auth import GoogleAuth
|
||||
from pydrive2.drive import GoogleDrive
|
||||
|
||||
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
|
||||
if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
|
||||
with open(self._gdrive_user_credentials_path, "w") as cred_file:
|
||||
cred_file.write(os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA))
|
||||
cred_file.write(
|
||||
os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
|
||||
)
|
||||
|
||||
auth_settings = {
|
||||
"client_config_backend": "settings",
|
||||
@@ -259,7 +261,7 @@ class GDriveTree(BaseTree):
|
||||
gauth.ServiceAuth()
|
||||
else:
|
||||
gauth.CommandLineAuth()
|
||||
GDriveTree._validate_credentials(gauth, auth_settings)
|
||||
GDriveFileSystem._validate_credentials(gauth, auth_settings)
|
||||
|
||||
# Handle AuthenticationError, RefreshError and other auth failures
|
||||
# It's hard to come up with a narrow exception, since PyDrive throws
|
||||
@@ -268,7 +270,7 @@ class GDriveTree(BaseTree):
|
||||
except Exception as exc:
|
||||
raise GDriveAuthError(self.credentials_location) from exc
|
||||
finally:
|
||||
if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
|
||||
if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
|
||||
os.remove(self._gdrive_user_credentials_path)
|
||||
|
||||
return GoogleDrive(gauth)
|
||||
@@ -3,18 +3,19 @@ import os
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.tree.base import BaseTree
|
||||
from dvc.utils import is_exec, relpath
|
||||
|
||||
from .base import BaseFileSystem
|
||||
|
||||
class GitTree(BaseTree): # pylint:disable=abstract-method
|
||||
|
||||
class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
"""Proxies the repo file access methods to Git objects"""
|
||||
|
||||
def __init__(
|
||||
self, root_dir, trie, use_dvcignore=False, dvcignore_root=None
|
||||
):
|
||||
super().__init__(None, {})
|
||||
self._tree_root = root_dir
|
||||
self._fs_root = root_dir
|
||||
self.trie = trie
|
||||
self.use_dvcignore = use_dvcignore
|
||||
self.dvcignore_root = dvcignore_root
|
||||
@@ -24,17 +25,17 @@ class GitTree(BaseTree): # pylint:disable=abstract-method
|
||||
return self.trie.rev
|
||||
|
||||
@property
|
||||
def tree_root(self):
|
||||
return self._tree_root
|
||||
def fs_root(self):
|
||||
return self._fs_root
|
||||
|
||||
def _get_key(self, path):
|
||||
if isinstance(path, str):
|
||||
if not os.path.isabs(path):
|
||||
relparts = path.split(os.sep)
|
||||
else:
|
||||
relparts = relpath(path, self.tree_root).split(os.sep)
|
||||
relparts = relpath(path, self.fs_root).split(os.sep)
|
||||
else:
|
||||
relparts = path.relative_to(self.tree_root).parts
|
||||
relparts = path.relative_to(self.fs_root).parts
|
||||
if relparts == ["."]:
|
||||
return ()
|
||||
return tuple(relparts)
|
||||
@@ -43,7 +44,7 @@ class GitTree(BaseTree): # pylint:disable=abstract-method
|
||||
def dvcignore(self):
|
||||
from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
|
||||
|
||||
root = self.dvcignore_root or self.tree_root
|
||||
root = self.dvcignore_root or self.fs_root
|
||||
cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
|
||||
return cls(self, root)
|
||||
|
||||
@@ -122,9 +123,9 @@ class GitTree(BaseTree): # pylint:disable=abstract-method
|
||||
key = self._get_key(top)
|
||||
for prefix, dirs, files in self.trie.walk(key, topdown=topdown):
|
||||
if prefix:
|
||||
root = os.path.join(self.tree_root, os.sep.join(prefix))
|
||||
root = os.path.join(self.fs_root, os.sep.join(prefix))
|
||||
else:
|
||||
root = self.tree_root
|
||||
root = self.fs_root
|
||||
if use_dvcignore:
|
||||
dirs[:], files[:] = self.dvcignore(
|
||||
root, dirs, files, ignore_subrepos=ignore_subrepos,
|
||||
@@ -11,7 +11,7 @@ from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .base import BaseTree
|
||||
from .base import BaseFileSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -50,7 +50,7 @@ def _upload_to_bucket(bucket, fobj, to_info, chunk_size=None):
|
||||
blob.upload_from_file(fobj)
|
||||
|
||||
|
||||
class GSTree(BaseTree):
|
||||
class GSFileSystem(BaseFileSystem):
|
||||
scheme = Schemes.GS
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
|
||||
@@ -196,7 +196,7 @@ class GSTree(BaseTree):
|
||||
def _upload_fobj(self, fobj, to_info):
|
||||
bucket = self.gs.bucket(to_info.bucket)
|
||||
# With other references being given in the @dynamic_chunk_size
|
||||
# this function does not respect tree.CHUNK_SIZE, since it is
|
||||
# this function does not respect fs.CHUNK_SIZE, since it is
|
||||
# too big for GS to handle. Rather it dynamically calculates the
|
||||
# best possible chunk size
|
||||
_upload_to_bucket(bucket, fobj, to_info)
|
||||
@@ -13,7 +13,7 @@ from dvc.progress import Tqdm
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import fix_env, tmp_fname
|
||||
|
||||
from .base import BaseTree, RemoteCmdError
|
||||
from .base import BaseFileSystem, RemoteCmdError
|
||||
from .pool import get_connection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -58,7 +58,7 @@ def _hadoop_fs_checksum(path_info):
|
||||
return _group(regex, stdout, "checksum")
|
||||
|
||||
|
||||
class HDFSTree(BaseTree):
|
||||
class HDFSFileSystem(BaseFileSystem):
|
||||
scheme = Schemes.HDFS
|
||||
REQUIRES = {"pyarrow": "pyarrow"}
|
||||
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
|
||||
@@ -10,7 +10,7 @@ from dvc.path_info import HTTPURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .base import BaseTree
|
||||
from .base import BaseFileSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -24,7 +24,7 @@ def ask_password(host, user):
|
||||
)
|
||||
|
||||
|
||||
class HTTPTree(BaseTree): # pylint:disable=abstract-method
|
||||
class HTTPFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
scheme = Schemes.HTTP
|
||||
PATH_CLS = HTTPURLInfo
|
||||
PARAM_CHECKSUM = "etag"
|
||||
@@ -0,0 +1,7 @@
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .http import HTTPFileSystem
|
||||
|
||||
|
||||
class HTTPSFileSystem(HTTPFileSystem): # pylint:disable=abstract-method
|
||||
scheme = Schemes.HTTPS
|
||||
@@ -10,12 +10,12 @@ from dvc.system import System
|
||||
from dvc.utils import is_exec, tmp_fname
|
||||
from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove
|
||||
|
||||
from .base import BaseTree
|
||||
from .base import BaseFileSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LocalTree(BaseTree):
|
||||
class LocalFileSystem(BaseFileSystem):
|
||||
scheme = Schemes.LOCAL
|
||||
PATH_CLS = PathInfo
|
||||
PARAM_CHECKSUM = "md5"
|
||||
@@ -30,7 +30,7 @@ class LocalTree(BaseTree):
|
||||
self.dvcignore_root = dvcignore_root
|
||||
|
||||
@property
|
||||
def tree_root(self):
|
||||
def fs_root(self):
|
||||
return self.config.get("url")
|
||||
|
||||
@property
|
||||
@@ -43,7 +43,7 @@ class LocalTree(BaseTree):
|
||||
def dvcignore(self):
|
||||
from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
|
||||
|
||||
root = self.dvcignore_root or self.tree_root
|
||||
root = self.dvcignore_root or self.fs_root
|
||||
cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
|
||||
return cls(self, root)
|
||||
|
||||
@@ -92,7 +92,7 @@ class LocalTree(BaseTree):
|
||||
use_dvcignore=True,
|
||||
ignore_subrepos=True,
|
||||
):
|
||||
"""Directory tree generator.
|
||||
"""Directory fs generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
- no support for symlinks
|
||||
@@ -1,16 +1,16 @@
|
||||
from .base import BaseTree
|
||||
from .base import BaseFileSystem
|
||||
|
||||
|
||||
class MemoryTree(BaseTree):
|
||||
class MemoryFileSystem(BaseFileSystem):
|
||||
scheme = "local"
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
def __init__(self, repo, config):
|
||||
from fsspec.implementations.memory import MemoryFileSystem
|
||||
from fsspec.implementations.memory import MemoryFileSystem as MemFS
|
||||
|
||||
super().__init__(repo, config)
|
||||
|
||||
self.fs = MemoryFileSystem()
|
||||
self.fs = MemFS()
|
||||
|
||||
def exists(self, path_info, use_dvcignore=True):
|
||||
return self.fs.exists(path_info.path)
|
||||
@@ -8,12 +8,12 @@ from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .base import BaseTree
|
||||
from .base import BaseFileSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OSSTree(BaseTree): # pylint:disable=abstract-method
|
||||
class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
"""
|
||||
oss2 document:
|
||||
https://www.alibabacloud.com/help/doc-detail/32026.htm
|
||||
@@ -10,8 +10,8 @@ from funcy import lfilter, wrap_with
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
|
||||
from .base import BaseTree
|
||||
from .dvc import DvcTree
|
||||
from .base import BaseFileSystem
|
||||
from .dvc import DvcFileSystem
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.repo import Repo
|
||||
@@ -21,14 +21,14 @@ logger = logging.getLogger(__name__)
|
||||
RepoFactory = Union[Callable[[str], "Repo"], Type["Repo"]]
|
||||
|
||||
|
||||
class RepoTree(BaseTree): # pylint:disable=abstract-method
|
||||
"""DVC + git-tracked files tree.
|
||||
class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
"""DVC + git-tracked files fs.
|
||||
|
||||
Args:
|
||||
repo: DVC or git repo.
|
||||
subrepos: traverse to subrepos (by default, it ignores subrepos)
|
||||
repo_factory: A function to initialize subrepo with, default is Repo.
|
||||
kwargs: Additional keyword arguments passed to the `DvcTree()`.
|
||||
kwargs: Additional keyword arguments passed to the `DvcFileSystem()`.
|
||||
"""
|
||||
|
||||
scheme = "local"
|
||||
@@ -57,11 +57,11 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
|
||||
|
||||
self._subrepos_trie[self.root_dir] = repo
|
||||
|
||||
self._dvctrees = {}
|
||||
"""Keep a dvctree instance of each repo."""
|
||||
self._dvcfss = {}
|
||||
"""Keep a dvcfs instance of each repo."""
|
||||
|
||||
if hasattr(repo, "dvc_dir"):
|
||||
self._dvctrees[repo.root_dir] = DvcTree(repo)
|
||||
self._dvcfss[repo.root_dir] = DvcFileSystem(repo)
|
||||
|
||||
def _get_repo(self, path) -> Optional["Repo"]:
|
||||
"""Returns repo that the path falls in, using prefix.
|
||||
@@ -97,7 +97,7 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
|
||||
rev=self.repo.get_rev(),
|
||||
repo_factory=self.repo_factory,
|
||||
)
|
||||
self._dvctrees[repo.root_dir] = DvcTree(repo)
|
||||
self._dvcfss[repo.root_dir] = DvcFileSystem(repo)
|
||||
self._subrepos_trie[d] = repo
|
||||
|
||||
def _is_dvc_repo(self, dir_path):
|
||||
@@ -109,11 +109,13 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
|
||||
|
||||
repo_path = os.path.join(dir_path, Repo.DVC_DIR)
|
||||
# dvcignore will ignore subrepos, therefore using `use_dvcignore=False`
|
||||
return self._main_repo.tree.isdir(repo_path, use_dvcignore=False)
|
||||
return self._main_repo.fs.isdir(repo_path, use_dvcignore=False)
|
||||
|
||||
def _get_tree_pair(self, path) -> Tuple[BaseTree, Optional[DvcTree]]:
|
||||
def _get_fs_pair(
|
||||
self, path
|
||||
) -> Tuple[BaseFileSystem, Optional[DvcFileSystem]]:
|
||||
"""
|
||||
Returns a pair of trees based on repo the path falls in, using prefix.
|
||||
Returns a pair of fss based on repo the path falls in, using prefix.
|
||||
"""
|
||||
path = os.path.abspath(path)
|
||||
|
||||
@@ -121,8 +123,8 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
|
||||
# this can happen if the path is outside of the repo
|
||||
repo = self._get_repo(path) or self._main_repo
|
||||
|
||||
dvc_tree = self._dvctrees.get(repo.root_dir)
|
||||
return repo.tree, dvc_tree
|
||||
dvc_fs = self._dvcfss.get(repo.root_dir)
|
||||
return repo.fs, dvc_fs
|
||||
|
||||
def open(
|
||||
self, path, mode="r", encoding="utf-8", **kwargs
|
||||
@@ -130,99 +132,99 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
|
||||
if "b" in mode:
|
||||
encoding = None
|
||||
|
||||
tree, dvc_tree = self._get_tree_pair(path)
|
||||
fs, dvc_fs = self._get_fs_pair(path)
|
||||
path_info = PathInfo(path)
|
||||
try:
|
||||
return tree.open(path_info, mode=mode, encoding=encoding)
|
||||
return fs.open(path_info, mode=mode, encoding=encoding)
|
||||
except FileNotFoundError:
|
||||
if not dvc_tree:
|
||||
if not dvc_fs:
|
||||
raise
|
||||
|
||||
return dvc_tree.open(path_info, mode=mode, encoding=encoding, **kwargs)
|
||||
return dvc_fs.open(path_info, mode=mode, encoding=encoding, **kwargs)
|
||||
|
||||
def exists(
|
||||
self, path, use_dvcignore=True
|
||||
): # pylint: disable=arguments-differ
|
||||
tree, dvc_tree = self._get_tree_pair(path)
|
||||
fs, dvc_fs = self._get_fs_pair(path)
|
||||
|
||||
if not dvc_tree:
|
||||
return tree.exists(path)
|
||||
if not dvc_fs:
|
||||
return fs.exists(path)
|
||||
|
||||
if tree.exists(path):
|
||||
if fs.exists(path):
|
||||
return True
|
||||
|
||||
try:
|
||||
meta = dvc_tree.metadata(path)
|
||||
meta = dvc_fs.metadata(path)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
(out,) = meta.outs
|
||||
assert len(meta.outs) == 1
|
||||
if tree.exists(out.path_info):
|
||||
if fs.exists(out.path_info):
|
||||
return False
|
||||
return True
|
||||
|
||||
def isdir(self, path): # pylint: disable=arguments-differ
|
||||
tree, dvc_tree = self._get_tree_pair(path)
|
||||
fs, dvc_fs = self._get_fs_pair(path)
|
||||
|
||||
try:
|
||||
st = tree.stat(path)
|
||||
st = fs.stat(path)
|
||||
return stat.S_ISDIR(st.st_mode)
|
||||
except (OSError, ValueError):
|
||||
# from CPython's os.path.isdir()
|
||||
pass
|
||||
|
||||
if not dvc_tree:
|
||||
if not dvc_fs:
|
||||
return False
|
||||
|
||||
try:
|
||||
meta = dvc_tree.metadata(path)
|
||||
meta = dvc_fs.metadata(path)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
(out,) = meta.outs
|
||||
assert len(meta.outs) == 1
|
||||
if tree.exists(out.path_info):
|
||||
if fs.exists(out.path_info):
|
||||
return False
|
||||
return meta.isdir
|
||||
|
||||
def isdvc(self, path, **kwargs):
|
||||
_, dvc_tree = self._get_tree_pair(path)
|
||||
return dvc_tree is not None and dvc_tree.isdvc(path, **kwargs)
|
||||
_, dvc_fs = self._get_fs_pair(path)
|
||||
return dvc_fs is not None and dvc_fs.isdvc(path, **kwargs)
|
||||
|
||||
def isfile(self, path): # pylint: disable=arguments-differ
|
||||
tree, dvc_tree = self._get_tree_pair(path)
|
||||
fs, dvc_fs = self._get_fs_pair(path)
|
||||
|
||||
try:
|
||||
st = tree.stat(path)
|
||||
st = fs.stat(path)
|
||||
return stat.S_ISREG(st.st_mode)
|
||||
except (OSError, ValueError):
|
||||
# from CPython's os.path.isfile()
|
||||
pass
|
||||
|
||||
if not dvc_tree:
|
||||
if not dvc_fs:
|
||||
return False
|
||||
|
||||
try:
|
||||
meta = dvc_tree.metadata(path)
|
||||
meta = dvc_fs.metadata(path)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
(out,) = meta.outs
|
||||
assert len(meta.outs) == 1
|
||||
if tree.exists(out.path_info):
|
||||
if fs.exists(out.path_info):
|
||||
return False
|
||||
return meta.isfile
|
||||
|
||||
def isexec(self, path_info):
|
||||
tree, dvc_tree = self._get_tree_pair(path_info)
|
||||
if dvc_tree and dvc_tree.exists(path_info):
|
||||
return dvc_tree.isexec(path_info)
|
||||
return tree.isexec(path_info)
|
||||
fs, dvc_fs = self._get_fs_pair(path_info)
|
||||
if dvc_fs and dvc_fs.exists(path_info):
|
||||
return dvc_fs.isexec(path_info)
|
||||
return fs.isexec(path_info)
|
||||
|
||||
def stat(self, path):
|
||||
tree, _ = self._get_tree_pair(path)
|
||||
return tree.stat(path)
|
||||
fs, _ = self._get_fs_pair(path)
|
||||
return fs.stat(path)
|
||||
|
||||
def _dvc_walk(self, walk):
|
||||
try:
|
||||
@@ -239,15 +241,15 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
|
||||
NOTE: subrepo will only be discovered when walking if
|
||||
ignore_subrepos is set to False.
|
||||
"""
|
||||
tree, dvc_tree = self._get_tree_pair(dir_path)
|
||||
tree_walk = tree.walk(
|
||||
fs, dvc_fs = self._get_fs_pair(dir_path)
|
||||
fs_walk = fs.walk(
|
||||
dir_path, topdown=True, ignore_subrepos=not self._traverse_subrepos
|
||||
)
|
||||
if dvc_tree:
|
||||
dvc_walk = dvc_tree.walk(dir_path, topdown=True, **kwargs)
|
||||
if dvc_fs:
|
||||
dvc_walk = dvc_fs.walk(dir_path, topdown=True, **kwargs)
|
||||
else:
|
||||
dvc_walk = None
|
||||
yield from self._walk(tree_walk, dvc_walk, **kwargs)
|
||||
yield from self._walk(fs_walk, dvc_walk, **kwargs)
|
||||
|
||||
def _walk(self, repo_walk, dvc_walk=None, dvcfiles=False):
|
||||
from dvc.dvcfile import is_valid_filename
|
||||
@@ -313,11 +315,11 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
|
||||
follow_subrepos=None,
|
||||
**kwargs
|
||||
): # pylint: disable=arguments-differ
|
||||
"""Walk and merge both DVC and repo trees.
|
||||
"""Walk and merge both DVC and repo fss.
|
||||
|
||||
Args:
|
||||
top: path to walk from
|
||||
topdown: if True, tree will be walked from top down.
|
||||
topdown: if True, fs will be walked from top down.
|
||||
onerror: if set, onerror function will be called if an error
|
||||
occurs (by default errors are ignored).
|
||||
dvcfiles: if True, dvcfiles will be included in the files list
|
||||
@@ -342,25 +344,25 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
|
||||
if follow_subrepos is not None:
|
||||
ignore_subrepos = not follow_subrepos
|
||||
|
||||
tree, dvc_tree = self._get_tree_pair(top)
|
||||
repo_exists = tree.exists(top)
|
||||
repo_walk = tree.walk(
|
||||
fs, dvc_fs = self._get_fs_pair(top)
|
||||
repo_exists = fs.exists(top)
|
||||
repo_walk = fs.walk(
|
||||
top,
|
||||
topdown=topdown,
|
||||
onerror=onerror,
|
||||
ignore_subrepos=ignore_subrepos,
|
||||
)
|
||||
|
||||
if not dvc_tree or (repo_exists and dvc_tree.isdvc(top)):
|
||||
if not dvc_fs or (repo_exists and dvc_fs.isdvc(top)):
|
||||
yield from self._walk(repo_walk, None, dvcfiles=dvcfiles)
|
||||
return
|
||||
|
||||
if not repo_exists:
|
||||
yield from dvc_tree.walk(top, topdown=topdown, **kwargs)
|
||||
yield from dvc_fs.walk(top, topdown=topdown, **kwargs)
|
||||
|
||||
dvc_walk = None
|
||||
if dvc_tree.exists(top):
|
||||
dvc_walk = dvc_tree.walk(top, topdown=topdown, **kwargs)
|
||||
if dvc_fs.exists(top):
|
||||
dvc_walk = dvc_fs.walk(top, topdown=topdown, **kwargs)
|
||||
|
||||
yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
|
||||
|
||||
@@ -385,21 +387,21 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
|
||||
|
||||
@property
|
||||
def hash_jobs(self): # pylint: disable=invalid-overridden-method
|
||||
return self._main_repo.tree.hash_jobs
|
||||
return self._main_repo.fs.hash_jobs
|
||||
|
||||
def metadata(self, path):
|
||||
abspath = os.path.abspath(path)
|
||||
path_info = PathInfo(abspath)
|
||||
tree, dvc_tree = self._get_tree_pair(path_info)
|
||||
fs, dvc_fs = self._get_fs_pair(path_info)
|
||||
|
||||
dvc_meta = None
|
||||
if dvc_tree:
|
||||
if dvc_fs:
|
||||
with suppress(FileNotFoundError):
|
||||
dvc_meta = dvc_tree.metadata(path_info)
|
||||
dvc_meta = dvc_fs.metadata(path_info)
|
||||
|
||||
stat_result = None
|
||||
with suppress(FileNotFoundError):
|
||||
stat_result = tree.stat(path_info)
|
||||
stat_result = fs.stat(path_info)
|
||||
|
||||
if not stat_result and not dvc_meta:
|
||||
raise FileNotFoundError
|
||||
@@ -421,9 +423,9 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
|
||||
return meta
|
||||
|
||||
def info(self, path_info):
|
||||
tree, dvc_tree = self._get_tree_pair(path_info)
|
||||
fs, dvc_fs = self._get_fs_pair(path_info)
|
||||
|
||||
try:
|
||||
return tree.info(path_info)
|
||||
return fs.info(path_info)
|
||||
except FileNotFoundError:
|
||||
return dvc_tree.info(path_info)
|
||||
return dvc_fs.info(path_info)
|
||||
@@ -13,14 +13,14 @@ from dvc.progress import Tqdm
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import conversions, error_link
|
||||
|
||||
from .base import BaseTree
|
||||
from .base import BaseFileSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
|
||||
|
||||
|
||||
class S3Tree(BaseTree):
|
||||
class S3FileSystem(BaseFileSystem):
|
||||
scheme = Schemes.S3
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"boto3": "boto3"}
|
||||
@@ -13,7 +13,7 @@ import dvc.prompt as prompt
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from ..base import BaseTree
|
||||
from ..base import BaseFileSystem
|
||||
from ..pool import get_connection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -30,7 +30,7 @@ def ask_password(host, user, port):
|
||||
)
|
||||
|
||||
|
||||
class SSHTree(BaseTree):
|
||||
class SSHFileSystem(BaseFileSystem):
|
||||
scheme = Schemes.SSH
|
||||
REQUIRES = {"paramiko": "paramiko"}
|
||||
JOBS = 4
|
||||
@@ -101,7 +101,7 @@ class SSHTree(BaseTree):
|
||||
def _load_user_ssh_config(hostname):
|
||||
import paramiko
|
||||
|
||||
user_config_file = SSHTree.ssh_config_filename()
|
||||
user_config_file = SSHFileSystem.ssh_config_filename()
|
||||
user_ssh_config = {}
|
||||
if hostname and os.path.exists(user_config_file):
|
||||
ssh_config = paramiko.SSHConfig()
|
||||
@@ -12,7 +12,7 @@ from dvc.path_info import HTTPURLInfo, WebDAVURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .base import BaseTree
|
||||
from .base import BaseFileSystem
|
||||
from .http import ask_password
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -23,7 +23,7 @@ class WebDAVConnectionError(DvcException):
|
||||
super().__init__(f"Unable to connect to WebDAV {host}.")
|
||||
|
||||
|
||||
class WebDAVTree(BaseTree): # pylint:disable=abstract-method
|
||||
class WebDAVFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
# Use webdav scheme
|
||||
scheme = Schemes.WEBDAV
|
||||
|
||||
@@ -47,7 +47,7 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
|
||||
|
||||
# Constructor
|
||||
def __init__(self, repo, config):
|
||||
# Call BaseTree constructor
|
||||
# Call BaseFileSystem constructor
|
||||
super().__init__(repo, config)
|
||||
|
||||
# Get username from configuration
|
||||
@@ -235,7 +235,7 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
|
||||
|
||||
# Downloads file from remote to file
|
||||
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
|
||||
# Progress from HTTPTree
|
||||
# Progress from HTTPFileSystem
|
||||
with open(to_file, "wb") as fd:
|
||||
with Tqdm.wrapattr(
|
||||
fd,
|
||||
@@ -0,0 +1,7 @@
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .webdav import WebDAVFileSystem
|
||||
|
||||
|
||||
class WebDAVSFileSystem(WebDAVFileSystem): # pylint:disable=abstract-method
|
||||
scheme = Schemes.WEBDAVS
|
||||
@@ -13,7 +13,7 @@ from dvc.path_info import CloudURLInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .base import BaseTree
|
||||
from .base import BaseFileSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -30,7 +30,7 @@ def update_pbar(pbar, total):
|
||||
return update
|
||||
|
||||
|
||||
class WebHDFSTree(BaseTree):
|
||||
class WebHDFSFileSystem(BaseFileSystem):
|
||||
scheme = Schemes.WEBHDFS
|
||||
PATH_CLS = CloudURLInfo
|
||||
REQUIRES = {"hdfs": "hdfs"}
|
||||
+20
-22
@@ -49,13 +49,11 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def from_files(cls, ignore_file_path, tree):
|
||||
def from_files(cls, ignore_file_path, fs):
|
||||
assert os.path.isabs(ignore_file_path)
|
||||
dirname = os.path.normpath(os.path.dirname(ignore_file_path))
|
||||
ignore_file_rel_path = os.path.relpath(
|
||||
ignore_file_path, tree.tree_root
|
||||
)
|
||||
with tree.open(ignore_file_path, encoding="utf-8") as fobj:
|
||||
ignore_file_rel_path = os.path.relpath(ignore_file_path, fs.fs_root)
|
||||
with fs.open(ignore_file_path, encoding="utf-8") as fobj:
|
||||
path_spec_lines = [
|
||||
PatternInfo(
|
||||
line,
|
||||
@@ -162,7 +160,7 @@ def _no_match(path):
|
||||
|
||||
|
||||
class DvcIgnoreFilterNoop:
|
||||
def __init__(self, tree, root_dir):
|
||||
def __init__(self, fs, root_dir):
|
||||
pass
|
||||
|
||||
def __call__(self, root, dirs, files, **kwargs):
|
||||
@@ -182,15 +180,15 @@ class DvcIgnoreFilterNoop:
|
||||
|
||||
|
||||
class DvcIgnoreFilter:
|
||||
def __init__(self, tree, root_dir):
|
||||
def __init__(self, fs, root_dir):
|
||||
from dvc.repo import Repo
|
||||
|
||||
default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
|
||||
|
||||
self.tree = tree
|
||||
self.fs = fs
|
||||
self.root_dir = root_dir
|
||||
self.ignores_trie_tree = PathStringTrie()
|
||||
self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
|
||||
self.ignores_trie_fs = PathStringTrie()
|
||||
self.ignores_trie_fs[root_dir] = DvcIgnorePatterns(
|
||||
default_ignore_patterns, root_dir
|
||||
)
|
||||
self._ignored_subrepos = PathStringTrie()
|
||||
@@ -199,18 +197,18 @@ class DvcIgnoreFilter:
|
||||
def _update(self, dirname):
|
||||
self._update_sub_repo(dirname)
|
||||
|
||||
old_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
|
||||
old_pattern = self.ignores_trie_fs.longest_prefix(dirname).value
|
||||
matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
|
||||
|
||||
ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
|
||||
if not matches and self.tree.exists(
|
||||
if not matches and self.fs.exists(
|
||||
ignore_file_path, use_dvcignore=False
|
||||
):
|
||||
new_pattern = DvcIgnorePatterns.from_files(
|
||||
ignore_file_path, self.tree
|
||||
ignore_file_path, self.fs
|
||||
)
|
||||
if old_pattern:
|
||||
self.ignores_trie_tree[dirname] = DvcIgnorePatterns(
|
||||
self.ignores_trie_fs[dirname] = DvcIgnorePatterns(
|
||||
*merge_patterns(
|
||||
old_pattern.pattern_list,
|
||||
old_pattern.dirname,
|
||||
@@ -219,9 +217,9 @@ class DvcIgnoreFilter:
|
||||
)
|
||||
)
|
||||
else:
|
||||
self.ignores_trie_tree[dirname] = new_pattern
|
||||
self.ignores_trie_fs[dirname] = new_pattern
|
||||
elif old_pattern:
|
||||
self.ignores_trie_tree[dirname] = old_pattern
|
||||
self.ignores_trie_fs[dirname] = old_pattern
|
||||
|
||||
def _update_sub_repo(self, path):
|
||||
from dvc.repo import Repo
|
||||
@@ -239,9 +237,9 @@ class DvcIgnoreFilter:
|
||||
) | {dname}
|
||||
pattern_info = PatternInfo(f"/{dname}/", f"in sub_repo:{dname}")
|
||||
new_pattern = DvcIgnorePatterns([pattern_info], root)
|
||||
old_pattern = self.ignores_trie_tree.longest_prefix(root).value
|
||||
old_pattern = self.ignores_trie_fs.longest_prefix(root).value
|
||||
if old_pattern:
|
||||
self.ignores_trie_tree[root] = DvcIgnorePatterns(
|
||||
self.ignores_trie_fs[root] = DvcIgnorePatterns(
|
||||
*merge_patterns(
|
||||
old_pattern.pattern_list,
|
||||
old_pattern.dirname,
|
||||
@@ -250,7 +248,7 @@ class DvcIgnoreFilter:
|
||||
)
|
||||
)
|
||||
else:
|
||||
self.ignores_trie_tree[root] = new_pattern
|
||||
self.ignores_trie_fs[root] = new_pattern
|
||||
|
||||
def __call__(self, root, dirs, files, ignore_subrepos=True):
|
||||
for dname in dirs:
|
||||
@@ -264,11 +262,11 @@ class DvcIgnoreFilter:
|
||||
return dirs, files
|
||||
|
||||
def _get_trie_pattern(self, dirname):
|
||||
ignore_pattern = self.ignores_trie_tree.get(dirname)
|
||||
ignore_pattern = self.ignores_trie_fs.get(dirname)
|
||||
if ignore_pattern:
|
||||
return ignore_pattern
|
||||
|
||||
prefix = self.ignores_trie_tree.longest_prefix(dirname).key
|
||||
prefix = self.ignores_trie_fs.longest_prefix(dirname).key
|
||||
if not prefix:
|
||||
# outside of the repo
|
||||
return None
|
||||
@@ -285,7 +283,7 @@ class DvcIgnoreFilter:
|
||||
for parent in dirs:
|
||||
self._update(parent)
|
||||
|
||||
return self.ignores_trie_tree.get(dirname)
|
||||
return self.ignores_trie_fs.get(dirname)
|
||||
|
||||
def _is_ignored(self, path, is_dir=False):
|
||||
if self._outside_repo(path):
|
||||
|
||||
+6
-6
@@ -7,10 +7,10 @@ import uuid
|
||||
import psutil
|
||||
|
||||
from dvc.exceptions import DvcException, NotDvcRepoError
|
||||
from dvc.fs import ALL_FS, get_fs_cls, get_fs_config
|
||||
from dvc.repo import Repo
|
||||
from dvc.scm.base import SCMError
|
||||
from dvc.system import System
|
||||
from dvc.tree import TREES, get_tree_cls, get_tree_config
|
||||
from dvc.utils import error_link
|
||||
from dvc.utils.pkg import PKG
|
||||
from dvc.version import __version__
|
||||
@@ -75,7 +75,7 @@ def _get_caches(cache):
|
||||
|
||||
def _get_remotes(config):
|
||||
schemes = (
|
||||
get_tree_cls(get_tree_config(config, name=remote)).scheme
|
||||
get_fs_cls(get_fs_config(config, name=remote)).scheme
|
||||
for remote in config["remote"]
|
||||
)
|
||||
|
||||
@@ -117,11 +117,11 @@ def _get_linktype_support_info(repo):
|
||||
def _get_supported_remotes():
|
||||
|
||||
supported_remotes = []
|
||||
for tree_cls in TREES:
|
||||
if not tree_cls.get_missing_deps():
|
||||
supported_remotes.append(tree_cls.scheme)
|
||||
for fs_cls in ALL_FS:
|
||||
if not fs_cls.get_missing_deps():
|
||||
supported_remotes.append(fs_cls.scheme)
|
||||
|
||||
if len(supported_remotes) == len(TREES):
|
||||
if len(supported_remotes) == len(ALL_FS):
|
||||
return "All remotes"
|
||||
|
||||
if len(supported_remotes) == 1:
|
||||
|
||||
+2
-6
@@ -22,16 +22,12 @@ def istextblock(block):
|
||||
return float(len(nontext)) / len(block) <= 0.30
|
||||
|
||||
|
||||
def istextfile(fname, blocksize=512, tree=None):
|
||||
def istextfile(fname, fs, blocksize=512):
|
||||
""" Uses heuristics to guess whether the given file is text or binary,
|
||||
by reading a single block of bytes from the file.
|
||||
If more than 30% of the chars in the block are non-text, or there
|
||||
are NUL ('\x00') bytes in the block, assume this is a binary file.
|
||||
"""
|
||||
if tree:
|
||||
open_func = tree.open
|
||||
else:
|
||||
open_func = open
|
||||
with open_func(fname, "rb") as fobj:
|
||||
with fs.open(fname, "rb") as fobj:
|
||||
block = fobj.read(blocksize)
|
||||
return istextblock(block)
|
||||
|
||||
+1
-1
@@ -96,7 +96,7 @@ def main(argv=None): # noqa: C901
|
||||
finally:
|
||||
logger.setLevel(outerLogLevel)
|
||||
|
||||
from dvc.tree.pool import close_pools
|
||||
from dvc.fs.pool import close_pools
|
||||
|
||||
# Closing pools by-hand to prevent weird messages when closing SSH
|
||||
# connections. See https://github.com/iterative/dvc/issues/3248 for
|
||||
|
||||
+50
-58
@@ -21,9 +21,9 @@ class ObjectFormatError(ObjectError):
|
||||
|
||||
|
||||
class HashFile:
|
||||
def __init__(self, path_info, tree, hash_info):
|
||||
def __init__(self, path_info, fs, hash_info):
|
||||
self.path_info = path_info
|
||||
self.tree = tree
|
||||
self.fs = fs
|
||||
self.hash_info = hash_info
|
||||
|
||||
def __len__(self):
|
||||
@@ -36,7 +36,7 @@ class HashFile:
|
||||
return bool(self.hash_info)
|
||||
|
||||
def check(self, odb):
|
||||
actual = get_hash(self.path_info, self.tree, odb.tree.PARAM_CHECKSUM)
|
||||
actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM)
|
||||
|
||||
logger.trace(
|
||||
"cache '%s' expected '%s' actual '%s'",
|
||||
@@ -54,7 +54,7 @@ class HashFile:
|
||||
return odb.get(hash_info)
|
||||
|
||||
def save(self, odb, **kwargs):
|
||||
odb.add(self.path_info, self.tree, self.hash_info, **kwargs)
|
||||
odb.add(self.path_info, self.fs, self.hash_info, **kwargs)
|
||||
|
||||
|
||||
class File(HashFile):
|
||||
@@ -63,13 +63,11 @@ class File(HashFile):
|
||||
self.src = None
|
||||
|
||||
@classmethod
|
||||
def stage(cls, odb, path_info, tree, **kwargs):
|
||||
hash_info = get_hash(
|
||||
path_info, tree, odb.tree.PARAM_CHECKSUM, **kwargs
|
||||
)
|
||||
def stage(cls, odb, path_info, fs, **kwargs):
|
||||
hash_info = get_hash(path_info, fs, odb.fs.PARAM_CHECKSUM, **kwargs)
|
||||
raw = odb.get(hash_info)
|
||||
obj = cls(raw.path_info, raw.tree, hash_info)
|
||||
obj.src = HashFile(path_info, tree, hash_info)
|
||||
obj = cls(raw.path_info, raw.fs, hash_info)
|
||||
obj.src = HashFile(path_info, fs, hash_info)
|
||||
return obj
|
||||
|
||||
def save(self, odb, **kwargs):
|
||||
@@ -81,7 +79,7 @@ class Tree(HashFile):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.src_hash_info = None
|
||||
self.src_path_info = None
|
||||
self.src_tree = None
|
||||
self.src_fs = None
|
||||
|
||||
def __len__(self):
|
||||
return self.hash_info.dir_info.nfiles
|
||||
@@ -95,7 +93,7 @@ class Tree(HashFile):
|
||||
|
||||
@classmethod
|
||||
def save_dir_info(cls, odb, dir_info, hash_info=None):
|
||||
if hash_info and hash_info.name == odb.tree.PARAM_CHECKSUM:
|
||||
if hash_info and hash_info.name == odb.fs.PARAM_CHECKSUM:
|
||||
try:
|
||||
odb.check(hash_info)
|
||||
assert hash_info.dir_info == dir_info
|
||||
@@ -103,40 +101,38 @@ class Tree(HashFile):
|
||||
except (FileNotFoundError, ObjectFormatError):
|
||||
pass
|
||||
|
||||
from dvc.fs.memory import MemoryFileSystem
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.tree.memory import MemoryTree
|
||||
from dvc.utils import tmp_fname
|
||||
|
||||
tree = MemoryTree(None, {})
|
||||
fs = MemoryFileSystem(None, {})
|
||||
path_info = PathInfo(tmp_fname(""))
|
||||
with tree.open(path_info, "w") as fobj:
|
||||
with fs.open(path_info, "w") as fobj:
|
||||
json.dump(dir_info.to_list(), fobj, sort_keys=True)
|
||||
|
||||
tmp_info = odb.tree.path_info / tmp_fname("")
|
||||
with tree.open(path_info, "rb") as fobj:
|
||||
odb.tree.upload_fobj(fobj, tmp_info)
|
||||
tmp_info = odb.fs.path_info / tmp_fname("")
|
||||
with fs.open(path_info, "rb") as fobj:
|
||||
odb.fs.upload_fobj(fobj, tmp_info)
|
||||
|
||||
hash_info = get_hash(tmp_info, odb.tree, odb.tree.PARAM_CHECKSUM)
|
||||
hash_info.value += odb.tree.CHECKSUM_DIR_SUFFIX
|
||||
hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM)
|
||||
hash_info.value += odb.fs.CHECKSUM_DIR_SUFFIX
|
||||
hash_info.dir_info = dir_info
|
||||
hash_info.nfiles = dir_info.nfiles
|
||||
|
||||
odb.add(tmp_info, odb.tree, hash_info)
|
||||
odb.add(tmp_info, odb.fs, hash_info)
|
||||
|
||||
return hash_info
|
||||
|
||||
@classmethod
|
||||
def stage(cls, odb, path_info, tree, **kwargs):
|
||||
hash_info = get_hash(
|
||||
path_info, tree, odb.tree.PARAM_CHECKSUM, **kwargs
|
||||
)
|
||||
def stage(cls, odb, path_info, fs, **kwargs):
|
||||
hash_info = get_hash(path_info, fs, odb.fs.PARAM_CHECKSUM, **kwargs)
|
||||
hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info)
|
||||
hi.size = hash_info.size
|
||||
raw = odb.get(hi)
|
||||
obj = cls(raw.path_info, raw.tree, hi)
|
||||
obj = cls(raw.path_info, raw.fs, hi)
|
||||
obj.src_hash_info = hash_info
|
||||
obj.src_path_info = path_info
|
||||
obj.src_tree = tree
|
||||
obj.src_fs = fs
|
||||
return obj
|
||||
|
||||
@classmethod
|
||||
@@ -145,7 +141,7 @@ class Tree(HashFile):
|
||||
obj = odb.get(hash_info)
|
||||
|
||||
try:
|
||||
with obj.tree.open(obj.path_info, "r") as fobj:
|
||||
with obj.fs.open(obj.path_info, "r") as fobj:
|
||||
raw = json.load(fobj)
|
||||
except ValueError as exc:
|
||||
raise ObjectFormatError(f"{obj} is corrupted") from exc
|
||||
@@ -161,7 +157,7 @@ class Tree(HashFile):
|
||||
hash_info.dir_info = dir_info
|
||||
hash_info.nfiles = dir_info.nfiles
|
||||
|
||||
return cls(obj.path_info, obj.tree, hash_info)
|
||||
return cls(obj.path_info, obj.fs, hash_info)
|
||||
|
||||
def save(self, odb, **kwargs):
|
||||
assert self.src_hash_info.dir_info
|
||||
@@ -173,9 +169,9 @@ class Tree(HashFile):
|
||||
desc="Saving " + self.src_path_info.name,
|
||||
unit="file",
|
||||
):
|
||||
entry_obj = HashFile(entry_info, self.src_tree, entry_hash)
|
||||
entry_obj = HashFile(entry_info, self.src_fs, entry_hash)
|
||||
entry_obj.save(odb, **kwargs)
|
||||
self.src_tree.state.save(self.src_path_info, hi)
|
||||
self.src_fs.state.save(self.src_path_info, hi)
|
||||
|
||||
def filter(self, odb, prefix):
|
||||
hash_info = self.hash_info.dir_info.trie.get(prefix)
|
||||
@@ -193,10 +189,10 @@ class Tree(HashFile):
|
||||
return load(odb, self.save_dir_info(odb, dir_info))
|
||||
|
||||
|
||||
def stage(odb, path_info, tree, **kwargs):
|
||||
if tree.isdir(path_info):
|
||||
return Tree.stage(odb, path_info, tree, **kwargs)
|
||||
return File.stage(odb, path_info, tree, **kwargs)
|
||||
def stage(odb, path_info, fs, **kwargs):
|
||||
if fs.isdir(path_info):
|
||||
return Tree.stage(odb, path_info, fs, **kwargs)
|
||||
return File.stage(odb, path_info, fs, **kwargs)
|
||||
|
||||
|
||||
def save(odb, obj, **kwargs):
|
||||
@@ -220,7 +216,7 @@ def load(odb, hash_info):
|
||||
def _get_dir_size(odb, dir_info):
|
||||
try:
|
||||
return sum(
|
||||
odb.tree.getsize(odb.hash_to_path_info(hi.value))
|
||||
odb.fs.getsize(odb.hash_to_path_info(hi.value))
|
||||
for _, hi in dir_info.items()
|
||||
)
|
||||
except FileNotFoundError:
|
||||
@@ -245,22 +241,22 @@ def merge(odb, ancestor_info, our_info, their_info):
|
||||
return hash_info
|
||||
|
||||
|
||||
def _transfer_file(odb, from_tree, from_info):
|
||||
def _transfer_file(odb, from_fs, from_info):
|
||||
from dvc.utils import tmp_fname
|
||||
from dvc.utils.stream import HashedStreamReader
|
||||
|
||||
tmp_info = odb.tree.path_info / tmp_fname()
|
||||
with from_tree.open(
|
||||
from_info, mode="rb", chunk_size=from_tree.CHUNK_SIZE
|
||||
tmp_info = odb.fs.path_info / tmp_fname()
|
||||
with from_fs.open(
|
||||
from_info, mode="rb", chunk_size=from_fs.CHUNK_SIZE
|
||||
) as stream:
|
||||
stream_reader = HashedStreamReader(stream)
|
||||
# Since we don't know the hash beforehand, we'll
|
||||
# upload it to a temporary location and then move
|
||||
# it.
|
||||
odb.tree.upload_fobj(
|
||||
odb.fs.upload_fobj(
|
||||
stream_reader,
|
||||
tmp_info,
|
||||
total=from_tree.getsize(from_info),
|
||||
total=from_fs.getsize(from_info),
|
||||
desc=from_info.name,
|
||||
)
|
||||
|
||||
@@ -268,15 +264,15 @@ def _transfer_file(odb, from_tree, from_info):
|
||||
return tmp_info, hash_info
|
||||
|
||||
|
||||
def _transfer_directory_contents(odb, from_tree, from_info, jobs, pbar):
|
||||
def _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
|
||||
rel_path_infos = {}
|
||||
from_infos = from_tree.walk_files(from_info)
|
||||
from_infos = from_fs.walk_files(from_info)
|
||||
|
||||
def create_tasks(executor, amount):
|
||||
for entry_info in itertools.islice(from_infos, amount):
|
||||
pbar.total += 1
|
||||
task = executor.submit(
|
||||
pbar.wrap_fn(_transfer_file), odb, from_tree, entry_info
|
||||
pbar.wrap_fn(_transfer_file), odb, from_fs, entry_info
|
||||
)
|
||||
rel_path_infos[task] = entry_info.relative_to(from_info)
|
||||
yield task
|
||||
@@ -294,35 +290,31 @@ def _transfer_directory_contents(odb, from_tree, from_info, jobs, pbar):
|
||||
yield rel_path_infos.pop(task), task.result()
|
||||
|
||||
|
||||
def _transfer_directory(
|
||||
odb, from_tree, from_info, jobs, no_progress_bar=False
|
||||
):
|
||||
def _transfer_directory(odb, from_fs, from_info, jobs, no_progress_bar=False):
|
||||
dir_info = DirInfo()
|
||||
|
||||
with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar:
|
||||
for (
|
||||
entry_info,
|
||||
(entry_tmp_info, entry_hash),
|
||||
) in _transfer_directory_contents(
|
||||
odb, from_tree, from_info, jobs, pbar
|
||||
):
|
||||
odb.add(entry_tmp_info, odb.tree, entry_hash)
|
||||
) in _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
|
||||
odb.add(entry_tmp_info, odb.fs, entry_hash)
|
||||
dir_info.trie[entry_info.parts] = entry_hash
|
||||
|
||||
return Tree.save_dir_info(odb, dir_info)
|
||||
|
||||
|
||||
def transfer(odb, from_tree, from_info, jobs=None, no_progress_bar=False):
|
||||
jobs = jobs or min((from_tree.jobs, odb.tree.jobs))
|
||||
def transfer(odb, from_fs, from_info, jobs=None, no_progress_bar=False):
|
||||
jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
|
||||
|
||||
if from_tree.isdir(from_info):
|
||||
if from_fs.isdir(from_info):
|
||||
return _transfer_directory(
|
||||
odb,
|
||||
from_tree,
|
||||
from_fs,
|
||||
from_info,
|
||||
jobs=jobs,
|
||||
no_progress_bar=no_progress_bar,
|
||||
)
|
||||
tmp_info, hash_info = _transfer_file(odb, from_tree, from_info)
|
||||
odb.add(tmp_info, odb.tree, hash_info)
|
||||
tmp_info, hash_info = _transfer_file(odb, from_fs, from_info)
|
||||
odb.add(tmp_info, odb.fs, hash_info)
|
||||
return hash_info
|
||||
|
||||
+32
-32
@@ -9,27 +9,27 @@ from dvc.progress import Tqdm
|
||||
from dvc.utils import file_md5
|
||||
|
||||
|
||||
def get_file_hash(path_info, tree, name):
|
||||
info = tree.info(path_info)
|
||||
def get_file_hash(path_info, fs, name):
|
||||
info = fs.info(path_info)
|
||||
if name in info:
|
||||
assert not info[name].endswith(".dir")
|
||||
return HashInfo(name, info[name], size=info["size"])
|
||||
|
||||
func = getattr(tree, name, None)
|
||||
func = getattr(fs, name, None)
|
||||
if func:
|
||||
return func(path_info)
|
||||
|
||||
if name == "md5":
|
||||
return HashInfo(
|
||||
name, file_md5(path_info, tree), size=tree.getsize(path_info)
|
||||
name, file_md5(path_info, fs), size=fs.getsize(path_info)
|
||||
)
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _calculate_hashes(file_infos, tree, name):
|
||||
def _calculate_hashes(file_infos, fs, name):
|
||||
def _get_file_hash(path_info):
|
||||
return get_file_hash(path_info, tree, name)
|
||||
return get_file_hash(path_info, fs, name)
|
||||
|
||||
with Tqdm(
|
||||
total=len(file_infos),
|
||||
@@ -37,14 +37,14 @@ def _calculate_hashes(file_infos, tree, name):
|
||||
desc="Computing file/dir hashes (only done once)",
|
||||
) as pbar:
|
||||
worker = pbar.wrap_fn(_get_file_hash)
|
||||
with ThreadPoolExecutor(max_workers=tree.hash_jobs) as executor:
|
||||
with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor:
|
||||
hash_infos = executor.map(worker, file_infos)
|
||||
return dict(zip(file_infos, hash_infos))
|
||||
|
||||
|
||||
def _iter_hashes(path_info, tree, name, **kwargs):
|
||||
if name in tree.DETAIL_FIELDS:
|
||||
for details in tree.ls(path_info, recursive=True, detail=True):
|
||||
def _iter_hashes(path_info, fs, name, **kwargs):
|
||||
if name in fs.DETAIL_FIELDS:
|
||||
for details in fs.ls(path_info, recursive=True, detail=True):
|
||||
file_info = path_info.replace(path=details["name"])
|
||||
hash_info = HashInfo(
|
||||
name, details[name], size=details.get("size"),
|
||||
@@ -54,8 +54,8 @@ def _iter_hashes(path_info, tree, name, **kwargs):
|
||||
return None
|
||||
|
||||
file_infos = []
|
||||
for file_info in tree.walk_files(path_info, **kwargs):
|
||||
hash_info = tree.state.get( # pylint: disable=assignment-from-none
|
||||
for file_info in fs.walk_files(path_info, **kwargs):
|
||||
hash_info = fs.state.get( # pylint: disable=assignment-from-none
|
||||
file_info
|
||||
)
|
||||
if not hash_info:
|
||||
@@ -64,14 +64,14 @@ def _iter_hashes(path_info, tree, name, **kwargs):
|
||||
assert hash_info.name == name
|
||||
yield file_info, hash_info
|
||||
|
||||
yield from _calculate_hashes(file_infos, tree, name).items()
|
||||
yield from _calculate_hashes(file_infos, fs, name).items()
|
||||
|
||||
|
||||
def _collect_dir(path_info, tree, name, **kwargs):
|
||||
def _collect_dir(path_info, fs, name, **kwargs):
|
||||
from dvc.dir_info import DirInfo
|
||||
|
||||
dir_info = DirInfo()
|
||||
for fi, hi in _iter_hashes(path_info, tree, name, **kwargs):
|
||||
for fi, hi in _iter_hashes(path_info, fs, name, **kwargs):
|
||||
if DvcIgnore.DVCIGNORE_FILE == fi.name:
|
||||
raise DvcIgnoreInCollectedDirError(fi.parent)
|
||||
|
||||
@@ -88,29 +88,29 @@ def _collect_dir(path_info, tree, name, **kwargs):
|
||||
return dir_info
|
||||
|
||||
|
||||
def get_dir_hash(path_info, tree, name, **kwargs):
|
||||
def get_dir_hash(path_info, fs, name, **kwargs):
|
||||
from dvc.objects import Tree
|
||||
|
||||
dir_info = _collect_dir(path_info, tree, name, **kwargs)
|
||||
hash_info = Tree.save_dir_info(tree.repo.cache.local, dir_info)
|
||||
dir_info = _collect_dir(path_info, fs, name, **kwargs)
|
||||
hash_info = Tree.save_dir_info(fs.repo.cache.local, dir_info)
|
||||
hash_info.size = dir_info.size
|
||||
hash_info.dir_info = dir_info
|
||||
return hash_info
|
||||
|
||||
|
||||
def get_hash(path_info, tree, name, **kwargs):
|
||||
def get_hash(path_info, fs, name, **kwargs):
|
||||
assert path_info and (
|
||||
isinstance(path_info, str) or path_info.scheme == tree.scheme
|
||||
isinstance(path_info, str) or path_info.scheme == fs.scheme
|
||||
)
|
||||
|
||||
if not tree.exists(path_info):
|
||||
if not fs.exists(path_info):
|
||||
raise FileNotFoundError(
|
||||
errno.ENOENT, os.strerror(errno.ENOENT), path_info
|
||||
)
|
||||
|
||||
with tree.state:
|
||||
with fs.state:
|
||||
# pylint: disable=assignment-from-none
|
||||
hash_info = tree.state.get(path_info)
|
||||
hash_info = fs.state.get(path_info)
|
||||
|
||||
# If we have dir hash in state db, but dir cache file is lost,
|
||||
# then we need to recollect the dir via .get_dir_hash() call below,
|
||||
@@ -118,8 +118,8 @@ def get_hash(path_info, tree, name, **kwargs):
|
||||
if (
|
||||
hash_info
|
||||
and hash_info.isdir
|
||||
and not tree.cache.tree.exists(
|
||||
tree.cache.hash_to_path_info(hash_info.value)
|
||||
and not fs.cache.fs.exists(
|
||||
fs.cache.hash_to_path_info(hash_info.value)
|
||||
)
|
||||
):
|
||||
hash_info = None
|
||||
@@ -128,17 +128,17 @@ def get_hash(path_info, tree, name, **kwargs):
|
||||
if hash_info.isdir:
|
||||
from dvc.objects import Tree
|
||||
|
||||
# NOTE: loading the tree will restore hash_info.dir_info
|
||||
Tree.load(tree.cache, hash_info)
|
||||
# NOTE: loading the fs will restore hash_info.dir_info
|
||||
Tree.load(fs.cache, hash_info)
|
||||
assert hash_info.name == name
|
||||
return hash_info
|
||||
|
||||
if tree.isdir(path_info):
|
||||
hash_info = get_dir_hash(path_info, tree, name, **kwargs)
|
||||
if fs.isdir(path_info):
|
||||
hash_info = get_dir_hash(path_info, fs, name, **kwargs)
|
||||
else:
|
||||
hash_info = get_file_hash(path_info, tree, name)
|
||||
hash_info = get_file_hash(path_info, fs, name)
|
||||
|
||||
if hash_info and tree.exists(path_info):
|
||||
tree.state.save(path_info, hash_info)
|
||||
if hash_info and fs.exists(path_info):
|
||||
fs.state.save(path_info, hash_info)
|
||||
|
||||
return hash_info
|
||||
|
||||
+14
-14
@@ -14,11 +14,11 @@ from dvc.output.ssh import SSHOutput
|
||||
from dvc.output.webhdfs import WebHDFSOutput
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from ..tree import get_cloud_tree
|
||||
from ..tree.hdfs import HDFSTree
|
||||
from ..tree.local import LocalTree
|
||||
from ..tree.s3 import S3Tree
|
||||
from ..tree.webhdfs import WebHDFSTree
|
||||
from ..fs import get_cloud_fs
|
||||
from ..fs.hdfs import HDFSFileSystem
|
||||
from ..fs.local import LocalFileSystem
|
||||
from ..fs.s3 import S3FileSystem
|
||||
from ..fs.webhdfs import WebHDFSFileSystem
|
||||
|
||||
OUTS = [
|
||||
HDFSOutput,
|
||||
@@ -53,10 +53,10 @@ CHECKSUM_SCHEMA = Any(
|
||||
# so when a few types of outputs share the same name, we only need
|
||||
# specify it once.
|
||||
CHECKSUMS_SCHEMA = {
|
||||
LocalTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
S3Tree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
HDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
WebHDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
LocalFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
S3FileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
HDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
WebHDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
||||
}
|
||||
|
||||
SCHEMA = CHECKSUMS_SCHEMA.copy()
|
||||
@@ -88,13 +88,13 @@ def _get(
|
||||
parsed = urlparse(p)
|
||||
|
||||
if parsed.scheme == "remote":
|
||||
tree = get_cloud_tree(stage.repo, name=parsed.netloc)
|
||||
return OUTS_MAP[tree.scheme](
|
||||
fs = get_cloud_fs(stage.repo, name=parsed.netloc)
|
||||
return OUTS_MAP[fs.scheme](
|
||||
stage,
|
||||
p,
|
||||
info,
|
||||
cache=cache,
|
||||
tree=tree,
|
||||
fs=fs,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
@@ -111,7 +111,7 @@ def _get(
|
||||
p,
|
||||
info,
|
||||
cache=cache,
|
||||
tree=None,
|
||||
fs=None,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
@@ -125,7 +125,7 @@ def _get(
|
||||
p,
|
||||
info,
|
||||
cache=cache,
|
||||
tree=None,
|
||||
fs=None,
|
||||
metric=metric,
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
|
||||
+30
-32
@@ -20,7 +20,7 @@ from dvc.exceptions import (
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.oid import get_hash
|
||||
|
||||
from ..tree.base import BaseTree
|
||||
from ..fs.base import BaseFileSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -61,7 +61,7 @@ class OutputIsIgnoredError(DvcException):
|
||||
class BaseOutput:
|
||||
IS_DEPENDENCY = False
|
||||
|
||||
TREE_CLS = BaseTree
|
||||
FS_CLS = BaseFileSystem
|
||||
|
||||
PARAM_PATH = "path"
|
||||
PARAM_CACHE = "cache"
|
||||
@@ -105,7 +105,7 @@ class BaseOutput:
|
||||
stage,
|
||||
path,
|
||||
info=None,
|
||||
tree=None,
|
||||
fs=None,
|
||||
cache=True,
|
||||
metric=False,
|
||||
plot=False,
|
||||
@@ -130,10 +130,10 @@ class BaseOutput:
|
||||
self.repo = stage.repo if stage else None
|
||||
self.def_path = path
|
||||
self.hash_info = HashInfo.from_dict(info)
|
||||
if tree:
|
||||
self.tree = tree
|
||||
if fs:
|
||||
self.fs = fs
|
||||
else:
|
||||
self.tree = self.TREE_CLS(self.repo, {})
|
||||
self.fs = self.FS_CLS(self.repo, {})
|
||||
self.use_cache = False if self.IS_DEPENDENCY else cache
|
||||
self.metric = False if self.IS_DEPENDENCY else metric
|
||||
self.plot = False if self.IS_DEPENDENCY else plot
|
||||
@@ -142,18 +142,18 @@ class BaseOutput:
|
||||
self.live = live
|
||||
self.desc = desc
|
||||
|
||||
self.path_info = self._parse_path(tree, path)
|
||||
self.path_info = self._parse_path(fs, path)
|
||||
if self.use_cache and self.cache is None:
|
||||
raise RemoteCacheRequiredError(self.path_info)
|
||||
|
||||
self.obj = None
|
||||
self.isexec = False if self.IS_DEPENDENCY else isexec
|
||||
|
||||
def _parse_path(self, tree, path):
|
||||
if tree:
|
||||
def _parse_path(self, fs, path):
|
||||
if fs:
|
||||
parsed = urlparse(path)
|
||||
return tree.path_info / parsed.path.lstrip("/")
|
||||
return self.TREE_CLS.PATH_CLS(path)
|
||||
return fs.path_info / parsed.path.lstrip("/")
|
||||
return self.FS_CLS.PATH_CLS(path)
|
||||
|
||||
def __repr__(self):
|
||||
return "{class_name}: '{def_path}'".format(
|
||||
@@ -165,7 +165,7 @@ class BaseOutput:
|
||||
|
||||
@property
|
||||
def scheme(self):
|
||||
return self.TREE_CLS.scheme
|
||||
return self.FS_CLS.scheme
|
||||
|
||||
@property
|
||||
def is_in_repo(self):
|
||||
@@ -188,7 +188,7 @@ class BaseOutput:
|
||||
|
||||
@classmethod
|
||||
def supported(cls, url):
|
||||
return cls.TREE_CLS.supported(url)
|
||||
return cls.FS_CLS.supported(url)
|
||||
|
||||
@property
|
||||
def cache_path(self):
|
||||
@@ -196,10 +196,8 @@ class BaseOutput:
|
||||
|
||||
def get_hash(self):
|
||||
if not self.use_cache:
|
||||
return get_hash(
|
||||
self.path_info, self.tree, self.tree.PARAM_CHECKSUM
|
||||
)
|
||||
return objects.stage(self.cache, self.path_info, self.tree).hash_info
|
||||
return get_hash(self.path_info, self.fs, self.fs.PARAM_CHECKSUM)
|
||||
return objects.stage(self.cache, self.path_info, self.fs).hash_info
|
||||
|
||||
@property
|
||||
def is_dir_checksum(self):
|
||||
@@ -207,7 +205,7 @@ class BaseOutput:
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return self.tree.exists(self.path_info)
|
||||
return self.fs.exists(self.path_info)
|
||||
|
||||
def changed_checksum(self):
|
||||
return self.hash_info != self.get_hash()
|
||||
@@ -251,13 +249,13 @@ class BaseOutput:
|
||||
|
||||
@property
|
||||
def is_empty(self):
|
||||
return self.tree.is_empty(self.path_info)
|
||||
return self.fs.is_empty(self.path_info)
|
||||
|
||||
def isdir(self):
|
||||
return self.tree.isdir(self.path_info)
|
||||
return self.fs.isdir(self.path_info)
|
||||
|
||||
def isfile(self):
|
||||
return self.tree.isfile(self.path_info)
|
||||
return self.fs.isfile(self.path_info)
|
||||
|
||||
# pylint: disable=no-member
|
||||
|
||||
@@ -307,9 +305,9 @@ class BaseOutput:
|
||||
logger.debug("Output '%s' didn't change. Skipping saving.", self)
|
||||
return
|
||||
|
||||
self.obj = objects.stage(self.cache, self.path_info, self.tree)
|
||||
self.obj = objects.stage(self.cache, self.path_info, self.fs)
|
||||
self.hash_info = self.obj.hash_info
|
||||
self.isexec = self.isfile() and self.tree.isexec(self.path_info)
|
||||
self.isexec = self.isfile() and self.fs.isexec(self.path_info)
|
||||
|
||||
def set_exec(self):
|
||||
if self.isfile() and self.isexec:
|
||||
@@ -323,12 +321,12 @@ class BaseOutput:
|
||||
|
||||
if self.use_cache:
|
||||
obj = objects.stage(
|
||||
self.cache, filter_info or self.path_info, self.tree
|
||||
self.cache, filter_info or self.path_info, self.fs
|
||||
)
|
||||
objects.save(self.cache, obj)
|
||||
checkout(
|
||||
filter_info or self.path_info,
|
||||
self.tree,
|
||||
self.fs,
|
||||
obj,
|
||||
self.cache,
|
||||
relink=True,
|
||||
@@ -379,7 +377,7 @@ class BaseOutput:
|
||||
raise DvcException(f"verify metric is not supported for {self.scheme}")
|
||||
|
||||
def download(self, to, jobs=None):
|
||||
self.tree.download(self.path_info, to.path_info, jobs=jobs)
|
||||
self.fs.download(self.path_info, to.path_info, jobs=jobs)
|
||||
|
||||
def get_obj(self, filter_info=None):
|
||||
if self.obj:
|
||||
@@ -424,7 +422,7 @@ class BaseOutput:
|
||||
try:
|
||||
modified = checkout(
|
||||
filter_info or self.path_info,
|
||||
self.tree,
|
||||
self.fs,
|
||||
obj,
|
||||
self.cache,
|
||||
force=force,
|
||||
@@ -440,7 +438,7 @@ class BaseOutput:
|
||||
return added, False if added else modified
|
||||
|
||||
def remove(self, ignore_remove=False):
|
||||
self.tree.remove(self.path_info)
|
||||
self.fs.remove(self.path_info)
|
||||
if self.scheme != "local":
|
||||
return
|
||||
|
||||
@@ -452,7 +450,7 @@ class BaseOutput:
|
||||
if self.scheme == "local" and self.use_scm_ignore:
|
||||
self.repo.scm.ignore_remove(self.fspath)
|
||||
|
||||
self.tree.move(self.path_info, out.path_info)
|
||||
self.fs.move(self.path_info, out.path_info)
|
||||
self.def_path = out.def_path
|
||||
self.path_info = out.path_info
|
||||
self.save()
|
||||
@@ -624,8 +622,8 @@ class BaseOutput:
|
||||
|
||||
if stage:
|
||||
abs_path = os.path.join(stage.wdir, path)
|
||||
if stage.repo.tree.dvcignore.is_ignored(abs_path):
|
||||
check = stage.repo.tree.dvcignore.check_ignore(abs_path)
|
||||
if stage.repo.fs.dvcignore.is_ignored(abs_path):
|
||||
check = stage.repo.fs.dvcignore.check_ignore(abs_path)
|
||||
raise cls.IsIgnoredError(check)
|
||||
|
||||
def _check_can_merge(self, out):
|
||||
@@ -636,7 +634,7 @@ class BaseOutput:
|
||||
other = out.dumpd()
|
||||
|
||||
ignored = [
|
||||
self.tree.PARAM_CHECKSUM,
|
||||
self.fs.PARAM_CHECKSUM,
|
||||
HashInfo.PARAM_SIZE,
|
||||
HashInfo.PARAM_NFILES,
|
||||
]
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
|
||||
from ..tree.gs import GSTree
|
||||
from ..fs.gs import GSFileSystem
|
||||
|
||||
|
||||
class GSOutput(BaseOutput):
|
||||
TREE_CLS = GSTree
|
||||
FS_CLS = GSFileSystem
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
|
||||
from ..tree.hdfs import HDFSTree
|
||||
from ..fs.hdfs import HDFSFileSystem
|
||||
|
||||
|
||||
class HDFSOutput(BaseOutput):
|
||||
TREE_CLS = HDFSTree
|
||||
FS_CLS = HDFSFileSystem
|
||||
|
||||
+9
-9
@@ -8,13 +8,13 @@ from dvc.output.base import BaseOutput
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
from ..tree.local import LocalTree
|
||||
from ..fs.local import LocalFileSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LocalOutput(BaseOutput):
|
||||
TREE_CLS = LocalTree
|
||||
FS_CLS = LocalFileSystem
|
||||
sep = os.sep
|
||||
|
||||
def __init__(self, stage, path, *args, **kwargs):
|
||||
@@ -25,14 +25,14 @@ class LocalOutput(BaseOutput):
|
||||
if (
|
||||
self.is_in_repo
|
||||
and self.repo
|
||||
and isinstance(self.repo.tree, LocalTree)
|
||||
and isinstance(self.repo.fs, LocalFileSystem)
|
||||
):
|
||||
self.tree = self.repo.tree
|
||||
self.fs = self.repo.fs
|
||||
|
||||
def _parse_path(self, tree, path):
|
||||
def _parse_path(self, fs, path):
|
||||
parsed = urlparse(path)
|
||||
if parsed.scheme == "remote":
|
||||
p = tree.path_info / parsed.path.lstrip("/")
|
||||
p = fs.path_info / parsed.path.lstrip("/")
|
||||
else:
|
||||
# NOTE: we can path either from command line or .dvc file,
|
||||
# so we should expect both posix and windows style paths.
|
||||
@@ -40,12 +40,12 @@ class LocalOutput(BaseOutput):
|
||||
#
|
||||
# FIXME: if we have Windows path containing / or posix one with \
|
||||
# then we have #2059 bug and can't really handle that.
|
||||
p = self.TREE_CLS.PATH_CLS(path)
|
||||
p = self.FS_CLS.PATH_CLS(path)
|
||||
if self.stage and not p.is_absolute():
|
||||
p = self.stage.wdir / p
|
||||
|
||||
abs_p = os.path.abspath(os.path.normpath(p))
|
||||
return self.TREE_CLS.PATH_CLS(abs_p)
|
||||
return self.FS_CLS.PATH_CLS(abs_p)
|
||||
|
||||
def __str__(self):
|
||||
if not self.repo or not self.is_in_repo:
|
||||
@@ -91,6 +91,6 @@ class LocalOutput(BaseOutput):
|
||||
logger.debug(msg, str(self.path_info), name)
|
||||
return
|
||||
|
||||
if not istextfile(path):
|
||||
if not istextfile(path, self.fs):
|
||||
msg = "binary file '{}' cannot be used as {}."
|
||||
raise DvcException(msg.format(self.path_info, name))
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
|
||||
from ..tree.s3 import S3Tree
|
||||
from ..fs.s3 import S3FileSystem
|
||||
|
||||
|
||||
class S3Output(BaseOutput):
|
||||
TREE_CLS = S3Tree
|
||||
FS_CLS = S3FileSystem
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
|
||||
from ..tree.ssh import SSHTree
|
||||
from ..fs.ssh import SSHFileSystem
|
||||
|
||||
|
||||
class SSHOutput(BaseOutput):
|
||||
TREE_CLS = SSHTree
|
||||
FS_CLS = SSHFileSystem
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from dvc.output.base import BaseOutput
|
||||
|
||||
from ..tree.webhdfs import WebHDFSTree
|
||||
from ..fs.webhdfs import WebHDFSFileSystem
|
||||
|
||||
|
||||
class WebHDFSOutput(BaseOutput):
|
||||
TREE_CLS = WebHDFSTree
|
||||
FS_CLS = WebHDFSFileSystem
|
||||
|
||||
@@ -135,7 +135,7 @@ def make_definition(
|
||||
|
||||
class DataResolver:
|
||||
def __init__(self, repo: "Repo", wdir: PathInfo, d: dict):
|
||||
self.tree = tree = repo.tree
|
||||
self.fs = fs = repo.fs
|
||||
self.wdir = wdir
|
||||
self.relpath = relpath(self.wdir / "dvc.yaml")
|
||||
|
||||
@@ -144,7 +144,7 @@ class DataResolver:
|
||||
self.context: Context = Context()
|
||||
|
||||
try:
|
||||
args = tree, vars_, wdir # load from `vars` section
|
||||
args = fs, vars_, wdir # load from `vars` section
|
||||
self.context.load_from_vars(*args, default=DEFAULT_PARAMS_FILE)
|
||||
except ContextError as exc:
|
||||
format_and_raise(exc, "'vars'", self.relpath)
|
||||
@@ -270,8 +270,8 @@ class EntryDefinition:
|
||||
context = Context.clone(context)
|
||||
|
||||
try:
|
||||
tree = self.resolver.tree
|
||||
context.load_from_vars(tree, vars_, wdir, stage_name=name)
|
||||
fs = self.resolver.fs
|
||||
context.load_from_vars(fs, vars_, wdir, stage_name=name)
|
||||
except VarsAlreadyLoaded as exc:
|
||||
format_and_raise(exc, f"'{self.where}.{name}.vars'", self.relpath)
|
||||
|
||||
|
||||
+10
-10
@@ -356,20 +356,20 @@ class Context(CtxDict):
|
||||
|
||||
@classmethod
|
||||
def load_from(
|
||||
cls, tree, path: PathInfo, select_keys: List[str] = None
|
||||
cls, fs, path: PathInfo, select_keys: List[str] = None
|
||||
) -> "Context":
|
||||
from dvc.utils.serialize import LOADERS
|
||||
|
||||
file = relpath(path)
|
||||
if not tree.exists(path):
|
||||
if not fs.exists(path):
|
||||
raise ParamsLoadError(f"'{file}' does not exist")
|
||||
if tree.isdir(path):
|
||||
if fs.isdir(path):
|
||||
raise ParamsLoadError(f"'{file}' is a directory")
|
||||
|
||||
_, ext = os.path.splitext(file)
|
||||
loader = LOADERS[ext]
|
||||
|
||||
data = loader(path, tree=tree)
|
||||
data = loader(path, fs=fs)
|
||||
if not isinstance(data, Mapping):
|
||||
typ = type(data).__name__
|
||||
raise ParamsLoadError(
|
||||
@@ -397,7 +397,7 @@ class Context(CtxDict):
|
||||
return super().merge_update(other, overwrite=overwrite)
|
||||
|
||||
def merge_from(
|
||||
self, tree, item: str, wdir: PathInfo, overwrite=False,
|
||||
self, fs, item: str, wdir: PathInfo, overwrite=False,
|
||||
):
|
||||
path, _, keys_str = item.partition(":")
|
||||
select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
|
||||
@@ -409,7 +409,7 @@ class Context(CtxDict):
|
||||
return # allow specifying complete filepath multiple times
|
||||
self.check_loaded(abspath, item, select_keys)
|
||||
|
||||
ctx = Context.load_from(tree, path_info, select_keys)
|
||||
ctx = Context.load_from(fs, path_info, select_keys)
|
||||
|
||||
try:
|
||||
self.merge_update(ctx, overwrite=overwrite)
|
||||
@@ -439,7 +439,7 @@ class Context(CtxDict):
|
||||
|
||||
def load_from_vars(
|
||||
self,
|
||||
tree,
|
||||
fs,
|
||||
vars_: List,
|
||||
wdir: PathInfo,
|
||||
stage_name: str = None,
|
||||
@@ -447,8 +447,8 @@ class Context(CtxDict):
|
||||
):
|
||||
if default:
|
||||
to_import = wdir / default
|
||||
if tree.exists(to_import):
|
||||
self.merge_from(tree, default, wdir)
|
||||
if fs.exists(to_import):
|
||||
self.merge_from(fs, default, wdir)
|
||||
else:
|
||||
msg = "%s does not exist, it won't be used in parametrization"
|
||||
logger.trace(msg, to_import) # type: ignore[attr-defined]
|
||||
@@ -457,7 +457,7 @@ class Context(CtxDict):
|
||||
for index, item in enumerate(vars_):
|
||||
assert isinstance(item, (str, dict))
|
||||
if isinstance(item, str):
|
||||
self.merge_from(tree, item, wdir)
|
||||
self.merge_from(fs, item, wdir)
|
||||
else:
|
||||
joiner = "." if stage_name else ""
|
||||
meta = Meta(source=f"{stage_name}{joiner}vars[{index}]")
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
from ..tree import get_cloud_tree
|
||||
from ..fs import get_cloud_fs
|
||||
from .base import Remote
|
||||
from .local import LocalRemote
|
||||
|
||||
|
||||
def get_remote(repo, **kwargs):
|
||||
tree = get_cloud_tree(repo, **kwargs)
|
||||
if tree.scheme == "local":
|
||||
return LocalRemote(tree)
|
||||
return Remote(tree)
|
||||
fs = get_cloud_fs(repo, **kwargs)
|
||||
if fs.scheme == "local":
|
||||
return LocalRemote(fs)
|
||||
return Remote(fs)
|
||||
|
||||
+30
-34
@@ -61,25 +61,25 @@ def index_locked(f):
|
||||
class Remote:
|
||||
"""Cloud remote class.
|
||||
|
||||
Provides methods for indexing and garbage collecting trees which contain
|
||||
Provides methods for indexing and garbage collecting fss which contain
|
||||
DVC remotes.
|
||||
"""
|
||||
|
||||
INDEX_CLS = RemoteIndex
|
||||
|
||||
def __init__(self, tree):
|
||||
def __init__(self, fs):
|
||||
from dvc.cache import get_cloud_cache
|
||||
|
||||
self.tree = tree
|
||||
self.repo = tree.repo
|
||||
self.cache = get_cloud_cache(self.tree)
|
||||
self.fs = fs
|
||||
self.repo = fs.repo
|
||||
self.cache = get_cloud_cache(self.fs)
|
||||
|
||||
config = tree.config
|
||||
config = fs.config
|
||||
url = config.get("url")
|
||||
if url:
|
||||
index_name = hashlib.sha256(url.encode("utf-8")).hexdigest()
|
||||
self.index = self.INDEX_CLS(
|
||||
self.repo, index_name, dir_suffix=self.tree.CHECKSUM_DIR_SUFFIX
|
||||
self.repo, index_name, dir_suffix=self.fs.CHECKSUM_DIR_SUFFIX
|
||||
)
|
||||
else:
|
||||
self.index = RemoteIndexNoop()
|
||||
@@ -87,7 +87,7 @@ class Remote:
|
||||
def __repr__(self):
|
||||
return "{class_name}: '{path_info}'".format(
|
||||
class_name=type(self).__name__,
|
||||
path_info=self.tree.path_info or "No path",
|
||||
path_info=self.fs.path_info or "No path",
|
||||
)
|
||||
|
||||
@index_locked
|
||||
@@ -147,8 +147,8 @@ class Remote:
|
||||
{dir_hash: set(file_hash, ...)} which can be used to map
|
||||
a .dir file to its file contents.
|
||||
"""
|
||||
logger.debug(f"Preparing to collect status from {self.tree.path_info}")
|
||||
md5s = set(named_cache.scheme_keys(cache.tree.scheme))
|
||||
logger.debug(f"Preparing to collect status from {self.fs.path_info}")
|
||||
md5s = set(named_cache.scheme_keys(cache.fs.scheme))
|
||||
|
||||
logger.debug("Collecting information from local cache...")
|
||||
local_exists = frozenset(
|
||||
@@ -164,7 +164,7 @@ class Remote:
|
||||
else:
|
||||
logger.debug("Collecting information from remote cache...")
|
||||
remote_exists = set()
|
||||
dir_md5s = set(named_cache.dir_keys(cache.tree.scheme))
|
||||
dir_md5s = set(named_cache.dir_keys(cache.fs.scheme))
|
||||
if dir_md5s:
|
||||
remote_exists.update(
|
||||
self._indexed_dir_hashes(cache, named_cache, dir_md5s)
|
||||
@@ -173,7 +173,7 @@ class Remote:
|
||||
if md5s:
|
||||
remote_exists.update(
|
||||
self.hashes_exist(
|
||||
md5s, jobs=jobs, name=str(self.tree.path_info)
|
||||
md5s, jobs=jobs, name=str(self.fs.path_info)
|
||||
)
|
||||
)
|
||||
return self._make_status(
|
||||
@@ -200,7 +200,7 @@ class Remote:
|
||||
dir_status = {}
|
||||
file_status = {}
|
||||
dir_contents = {}
|
||||
for hash_, item in named_cache[cache.tree.scheme].items():
|
||||
for hash_, item in named_cache[cache.fs.scheme].items():
|
||||
if item.children:
|
||||
dir_status[hash_] = make_names(hash_, item.names)
|
||||
dir_contents[hash_] = set()
|
||||
@@ -245,7 +245,7 @@ class Remote:
|
||||
# still exists on the remote
|
||||
for dir_hash in dir_exists:
|
||||
file_hashes = list(
|
||||
named_cache.child_keys(cache.tree.scheme, dir_hash)
|
||||
named_cache.child_keys(cache.fs.scheme, dir_hash)
|
||||
)
|
||||
if dir_hash not in self.index:
|
||||
logger.debug(
|
||||
@@ -304,21 +304,21 @@ class Remote:
|
||||
logger.debug(
|
||||
"Preparing to {} '{}'".format(
|
||||
"download data from" if download else "upload data to",
|
||||
self.tree.path_info,
|
||||
self.fs.path_info,
|
||||
)
|
||||
)
|
||||
|
||||
if download:
|
||||
func = _log_exceptions(self.tree.download, "download")
|
||||
func = _log_exceptions(self.fs.download, "download")
|
||||
status = STATUS_DELETED
|
||||
desc = "Downloading"
|
||||
else:
|
||||
func = _log_exceptions(self.tree.upload, "upload")
|
||||
func = _log_exceptions(self.fs.upload, "upload")
|
||||
status = STATUS_NEW
|
||||
desc = "Uploading"
|
||||
|
||||
if jobs is None:
|
||||
jobs = self.tree.JOBS
|
||||
jobs = self.fs.JOBS
|
||||
|
||||
dir_status, file_status, dir_contents = self._status(
|
||||
cache,
|
||||
@@ -471,15 +471,13 @@ class Remote:
|
||||
download=False,
|
||||
)
|
||||
|
||||
if self.tree.scheme == "local":
|
||||
with self.tree.state:
|
||||
if self.fs.scheme == "local":
|
||||
with self.fs.state:
|
||||
for checksum in named_cache.scheme_keys("local"):
|
||||
cache_file = self.cache.hash_to_path_info(checksum)
|
||||
if self.tree.exists(cache_file):
|
||||
hash_info = HashInfo(
|
||||
self.tree.PARAM_CHECKSUM, checksum
|
||||
)
|
||||
self.tree.state.save(cache_file, hash_info)
|
||||
if self.fs.exists(cache_file):
|
||||
hash_info = HashInfo(self.fs.PARAM_CHECKSUM, checksum)
|
||||
self.fs.state.save(cache_file, hash_info)
|
||||
self.cache.protect(cache_file)
|
||||
|
||||
return ret
|
||||
@@ -495,28 +493,26 @@ class Remote:
|
||||
)
|
||||
|
||||
if not self.cache.verify:
|
||||
with cache.tree.state:
|
||||
with cache.fs.state:
|
||||
for checksum in named_cache.scheme_keys("local"):
|
||||
cache_file = cache.hash_to_path_info(checksum)
|
||||
if cache.tree.exists(cache_file):
|
||||
if cache.fs.exists(cache_file):
|
||||
# We can safely save here, as existing corrupted files
|
||||
# will be removed upon status, while files corrupted
|
||||
# during download will not be moved from tmp_file
|
||||
# (see `BaseTree.download()`)
|
||||
hash_info = HashInfo(
|
||||
cache.tree.PARAM_CHECKSUM, checksum
|
||||
)
|
||||
cache.tree.state.save(cache_file, hash_info)
|
||||
# (see `BaseFileSystem.download()`)
|
||||
hash_info = HashInfo(cache.fs.PARAM_CHECKSUM, checksum)
|
||||
cache.fs.state.save(cache_file, hash_info)
|
||||
cache.protect(cache_file)
|
||||
|
||||
return ret
|
||||
|
||||
def transfer(self, from_tree, from_info, jobs=None, no_progress_bar=False):
|
||||
def transfer(self, from_fs, from_info, jobs=None, no_progress_bar=False):
|
||||
from dvc.objects import transfer
|
||||
|
||||
return transfer(
|
||||
self.cache,
|
||||
from_tree,
|
||||
from_fs,
|
||||
from_info,
|
||||
jobs=jobs,
|
||||
no_progress_bar=no_progress_bar,
|
||||
|
||||
+30
-28
@@ -16,8 +16,8 @@ from .graph import build_graph, build_outs_graph, get_pipelines
|
||||
from .trie import build_outs_trie
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.fs.base import BaseFileSystem
|
||||
from dvc.scm import Base
|
||||
from dvc.tree.base import BaseTree
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -99,8 +99,8 @@ class Repo:
|
||||
dvc_dir = None
|
||||
tmp_dir = None
|
||||
try:
|
||||
tree = scm.get_tree(rev) if isinstance(scm, Git) and rev else None
|
||||
root_dir = self.find_root(root_dir, tree)
|
||||
fs = scm.get_fs(rev) if isinstance(scm, Git) and rev else None
|
||||
root_dir = self.find_root(root_dir, fs)
|
||||
dvc_dir = os.path.join(root_dir, self.DVC_DIR)
|
||||
tmp_dir = os.path.join(dvc_dir, "tmp")
|
||||
makedirs(tmp_dir, exist_ok=True)
|
||||
@@ -134,6 +134,7 @@ class Repo:
|
||||
from dvc.cache import Cache
|
||||
from dvc.config import Config
|
||||
from dvc.data_cloud import DataCloud
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
from dvc.lock import LockNoop, make_lock
|
||||
from dvc.repo.live import Live
|
||||
from dvc.repo.metrics import Metrics
|
||||
@@ -142,10 +143,9 @@ class Repo:
|
||||
from dvc.repo.stage import StageLoad
|
||||
from dvc.stage.cache import StageCache
|
||||
from dvc.state import State, StateNoop
|
||||
from dvc.tree.local import LocalTree
|
||||
|
||||
self.url = url
|
||||
self._tree_conf = {
|
||||
self._fs_conf = {
|
||||
"repo_factory": repo_factory,
|
||||
}
|
||||
|
||||
@@ -153,17 +153,19 @@ class Repo:
|
||||
root_dir=root_dir, scm=scm, rev=rev, uninitialized=uninitialized
|
||||
)
|
||||
|
||||
tree_kwargs = {"use_dvcignore": True, "dvcignore_root": self.root_dir}
|
||||
fs_kwargs = {"use_dvcignore": True, "dvcignore_root": self.root_dir}
|
||||
if scm:
|
||||
self._tree = scm.get_tree(rev, **tree_kwargs)
|
||||
self._fs = scm.get_fs(rev, **fs_kwargs)
|
||||
else:
|
||||
self._tree = LocalTree(self, {"url": self.root_dir}, **tree_kwargs)
|
||||
self._fs = LocalFileSystem(
|
||||
self, {"url": self.root_dir}, **fs_kwargs
|
||||
)
|
||||
|
||||
self.config = Config(self.dvc_dir, tree=self.tree, config=config)
|
||||
self.config = Config(self.dvc_dir, fs=self.fs, config=config)
|
||||
self._uninitialized = uninitialized
|
||||
self._scm = scm
|
||||
|
||||
# used by RepoTree to determine if it should traverse subrepos
|
||||
# used by RepoFileSystem to determine if it should traverse subrepos
|
||||
self.subrepos = subrepos
|
||||
|
||||
self.cache = Cache(self)
|
||||
@@ -239,12 +241,12 @@ class Repo:
|
||||
raise
|
||||
|
||||
def get_rev(self):
|
||||
from dvc.tree.local import LocalTree
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
|
||||
assert self.scm
|
||||
if isinstance(self.tree, LocalTree):
|
||||
if isinstance(self.fs, LocalFileSystem):
|
||||
return self.scm.get_rev()
|
||||
return self.tree.rev
|
||||
return self.fs.rev
|
||||
|
||||
@cached_property
|
||||
def experiments(self):
|
||||
@@ -253,25 +255,25 @@ class Repo:
|
||||
return Experiments(self)
|
||||
|
||||
@property
|
||||
def tree(self) -> "BaseTree":
|
||||
return self._tree
|
||||
def fs(self) -> "BaseFileSystem":
|
||||
return self._fs
|
||||
|
||||
@tree.setter
|
||||
def tree(self, tree: "BaseTree"):
|
||||
self._tree = tree
|
||||
@fs.setter
|
||||
def fs(self, fs: "BaseFileSystem"):
|
||||
self._fs = fs
|
||||
# Our graph cache is no longer valid, as it was based on the previous
|
||||
# tree.
|
||||
# fs.
|
||||
self._reset()
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.__class__.__name__}: '{self.root_dir}'"
|
||||
|
||||
@classmethod
|
||||
def find_root(cls, root=None, tree=None) -> str:
|
||||
def find_root(cls, root=None, fs=None) -> str:
|
||||
root_dir = os.path.realpath(root or os.curdir)
|
||||
|
||||
if tree:
|
||||
if tree.isdir(os.path.join(root_dir, cls.DVC_DIR)):
|
||||
if fs:
|
||||
if fs.isdir(os.path.join(root_dir, cls.DVC_DIR)):
|
||||
return root_dir
|
||||
raise NotDvcRepoError(f"'{root}' does not contain DVC directory")
|
||||
|
||||
@@ -462,21 +464,21 @@ class Repo:
|
||||
return self.DVC_DIR in path_parts
|
||||
|
||||
@cached_property
|
||||
def repo_tree(self):
|
||||
from dvc.tree.repo import RepoTree
|
||||
def repo_fs(self):
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
|
||||
return RepoTree(self, subrepos=self.subrepos, **self._tree_conf)
|
||||
return RepoFileSystem(self, subrepos=self.subrepos, **self._fs_conf)
|
||||
|
||||
@contextmanager
|
||||
def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
|
||||
"""Opens a specified resource as a file descriptor"""
|
||||
from dvc.tree.repo import RepoTree
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
|
||||
tree = RepoTree(self, subrepos=True)
|
||||
fs = RepoFileSystem(self, subrepos=True)
|
||||
path = PathInfo(self.root_dir) / path
|
||||
try:
|
||||
with self.state:
|
||||
with tree.open(
|
||||
with fs.open(
|
||||
path, mode=mode, encoding=encoding, remote=remote,
|
||||
) as fobj:
|
||||
yield fobj
|
||||
|
||||
+4
-7
@@ -165,15 +165,12 @@ def _process_stages(
|
||||
command="add",
|
||||
)
|
||||
else:
|
||||
from dvc.fs import get_cloud_fs
|
||||
from dvc.objects import transfer
|
||||
from dvc.tree import get_cloud_tree
|
||||
|
||||
from_tree = get_cloud_tree(repo, url=target)
|
||||
from_fs = get_cloud_fs(repo, url=target)
|
||||
out.hash_info = transfer(
|
||||
out.cache,
|
||||
from_tree,
|
||||
from_tree.path_info,
|
||||
jobs=kwargs.get("jobs"),
|
||||
out.cache, from_fs, from_fs.path_info, jobs=kwargs.get("jobs"),
|
||||
)
|
||||
out.checkout()
|
||||
|
||||
@@ -212,7 +209,7 @@ def _find_all_targets(repo, target, recursive):
|
||||
return [
|
||||
os.fspath(path)
|
||||
for path in Tqdm(
|
||||
repo.tree.walk_files(target),
|
||||
repo.fs.walk_files(target),
|
||||
desc="Searching " + target,
|
||||
bar_format=Tqdm.BAR_FMT_NOTOTAL,
|
||||
unit="file",
|
||||
|
||||
@@ -20,7 +20,7 @@ def brancher( # noqa: E302
|
||||
sha_only (bool): only return git SHA for a revision.
|
||||
|
||||
Yields:
|
||||
str: the display name for the currently selected tree, it could be:
|
||||
str: the display name for the currently selected fs, it could be:
|
||||
- a git revision identifier
|
||||
- empty string it there is no branches to iterate over
|
||||
- "workspace" if there are uncommitted changes in the SCM repo
|
||||
@@ -29,14 +29,14 @@ def brancher( # noqa: E302
|
||||
yield ""
|
||||
return
|
||||
|
||||
from dvc.tree.local import LocalTree
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
|
||||
saved_tree = self.tree
|
||||
saved_fs = self.fs
|
||||
revs = revs.copy() if revs else []
|
||||
|
||||
scm = self.scm
|
||||
|
||||
self.tree = LocalTree(self, {"url": self.root_dir}, use_dvcignore=True)
|
||||
self.fs = LocalFileSystem(self, {"url": self.root_dir}, use_dvcignore=True)
|
||||
yield "workspace"
|
||||
|
||||
if revs and "workspace" in revs:
|
||||
@@ -59,15 +59,15 @@ def brancher( # noqa: E302
|
||||
try:
|
||||
if revs:
|
||||
for sha, names in group_by(scm.resolve_rev, revs).items():
|
||||
self.tree = scm.get_tree(
|
||||
self.fs = scm.get_fs(
|
||||
sha, use_dvcignore=True, dvcignore_root=self.root_dir
|
||||
)
|
||||
# ignore revs that don't contain repo root
|
||||
# (i.e. revs from before a subdir=True repo was init'ed)
|
||||
if self.tree.exists(self.root_dir):
|
||||
if self.fs.exists(self.root_dir):
|
||||
if sha_only:
|
||||
yield sha
|
||||
else:
|
||||
yield ", ".join(names)
|
||||
finally:
|
||||
self.tree = saved_tree
|
||||
self.fs = saved_fs
|
||||
|
||||
+5
-5
@@ -34,18 +34,18 @@ def _collect_paths(
|
||||
recursive: bool = False,
|
||||
rev: str = None,
|
||||
):
|
||||
from dvc.tree.repo import RepoTree
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
|
||||
path_infos = [PathInfo(os.path.abspath(target)) for target in targets]
|
||||
tree = RepoTree(repo)
|
||||
fs = RepoFileSystem(repo)
|
||||
|
||||
target_infos = []
|
||||
for path_info in path_infos:
|
||||
|
||||
if recursive and tree.isdir(path_info):
|
||||
target_infos.extend(tree.walk_files(path_info))
|
||||
if recursive and fs.isdir(path_info):
|
||||
target_infos.extend(fs.walk_files(path_info))
|
||||
|
||||
if not tree.exists(path_info):
|
||||
if not fs.exists(path_info):
|
||||
if not recursive:
|
||||
if rev == "workspace" or rev == "":
|
||||
logger.warning(
|
||||
|
||||
+25
-27
@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
|
||||
@locked
|
||||
def diff(self, a_rev="HEAD", b_rev=None, targets=None):
|
||||
"""
|
||||
By default, it compares the workspace with the last commit's tree.
|
||||
By default, it compares the workspace with the last commit's fs.
|
||||
|
||||
This implementation differs from `git diff` since DVC doesn't have
|
||||
the concept of `index`, but it keeps the same interface, thus,
|
||||
@@ -21,9 +21,9 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
|
||||
if self.scm.no_commits:
|
||||
return {}
|
||||
|
||||
from dvc.tree.repo import RepoTree
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
|
||||
repo_tree = RepoTree(self)
|
||||
repo_fs = RepoFileSystem(self)
|
||||
|
||||
b_rev = b_rev if b_rev else "workspace"
|
||||
results = {}
|
||||
@@ -38,10 +38,10 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
|
||||
if targets is not None:
|
||||
# convert targets to path_infos, and capture any missing targets
|
||||
targets_path_infos, missing_targets[rev] = _targets_to_path_infos(
|
||||
repo_tree, targets
|
||||
repo_fs, targets
|
||||
)
|
||||
|
||||
results[rev] = _paths_checksums(self, repo_tree, targets_path_infos)
|
||||
results[rev] = _paths_checksums(self, repo_fs, targets_path_infos)
|
||||
|
||||
if targets is not None:
|
||||
# check for overlapping missing targets between a_rev and b_rev
|
||||
@@ -53,14 +53,14 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
|
||||
old = results[a_rev]
|
||||
new = results[b_rev]
|
||||
|
||||
# Compare paths between the old and new tree.
|
||||
# Compare paths between the old and new fs.
|
||||
# set() efficiently converts dict keys to a set
|
||||
added = sorted(set(new) - set(old))
|
||||
deleted_or_missing = set(old) - set(new)
|
||||
if b_rev == "workspace":
|
||||
# missing status is only applicable when diffing local workspace
|
||||
# against a commit
|
||||
missing = sorted(_filter_missing(repo_tree, deleted_or_missing))
|
||||
missing = sorted(_filter_missing(repo_fs, deleted_or_missing))
|
||||
else:
|
||||
missing = []
|
||||
deleted = sorted(deleted_or_missing - set(missing))
|
||||
@@ -94,10 +94,10 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
|
||||
return ret if any(ret.values()) else {}
|
||||
|
||||
|
||||
def _paths_checksums(repo, repo_tree, targets):
|
||||
def _paths_checksums(repo, repo_fs, targets):
|
||||
"""
|
||||
A dictionary of checksums addressed by relpaths collected from
|
||||
the current tree outputs.
|
||||
the current fs outputs.
|
||||
|
||||
To help distinguish between a directory and a file output,
|
||||
the former one will come with a trailing slash in the path:
|
||||
@@ -106,16 +106,16 @@ def _paths_checksums(repo, repo_tree, targets):
|
||||
file: "data"
|
||||
"""
|
||||
|
||||
return dict(_output_paths(repo, repo_tree, targets))
|
||||
return dict(_output_paths(repo, repo_fs, targets))
|
||||
|
||||
|
||||
def _output_paths(repo, repo_tree, targets):
|
||||
from dvc.tree.local import LocalTree
|
||||
def _output_paths(repo, repo_fs, targets):
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
|
||||
on_working_tree = isinstance(repo.tree, LocalTree)
|
||||
on_working_fs = isinstance(repo.fs, LocalFileSystem)
|
||||
|
||||
def _exists(output):
|
||||
if on_working_tree:
|
||||
if on_working_fs:
|
||||
return output.exists
|
||||
return True
|
||||
|
||||
@@ -127,10 +127,8 @@ def _output_paths(repo, repo_tree, targets):
|
||||
)
|
||||
|
||||
def _to_checksum(output):
|
||||
if on_working_tree:
|
||||
return get_hash(
|
||||
output.path_info, repo.cache.local.tree, "md5"
|
||||
).value
|
||||
if on_working_fs:
|
||||
return get_hash(output.path_info, repo.cache.local.fs, "md5").value
|
||||
return output.hash_info.value
|
||||
|
||||
for stage in repo.stages:
|
||||
@@ -147,26 +145,26 @@ def _output_paths(repo, repo_tree, targets):
|
||||
yield_output
|
||||
or any(target.isin(output.path_info) for target in targets)
|
||||
):
|
||||
yield from _dir_output_paths(repo_tree, output, targets)
|
||||
yield from _dir_output_paths(repo_fs, output, targets)
|
||||
|
||||
|
||||
def _dir_output_paths(repo_tree, output, targets=None):
|
||||
def _dir_output_paths(repo_fs, output, targets=None):
|
||||
from dvc.config import NoRemoteError
|
||||
|
||||
try:
|
||||
for fname in repo_tree.walk_files(output.path_info):
|
||||
for fname in repo_fs.walk_files(output.path_info):
|
||||
if targets is None or any(
|
||||
fname.isin_or_eq(target) for target in targets
|
||||
):
|
||||
yield str(fname), get_file_hash(fname, repo_tree, "md5").value
|
||||
yield str(fname), get_file_hash(fname, repo_fs, "md5").value
|
||||
except NoRemoteError:
|
||||
logger.warning("dir cache entry for '%s' is missing", output)
|
||||
|
||||
|
||||
def _filter_missing(repo_tree, paths):
|
||||
def _filter_missing(repo_fs, paths):
|
||||
for path in paths:
|
||||
try:
|
||||
metadata = repo_tree.metadata(path)
|
||||
metadata = repo_fs.metadata(path)
|
||||
if metadata.is_dvc:
|
||||
out = metadata.outs[0]
|
||||
if out.status().get(str(out)) == "not in cache":
|
||||
@@ -175,13 +173,13 @@ def _filter_missing(repo_tree, paths):
|
||||
pass
|
||||
|
||||
|
||||
def _targets_to_path_infos(repo_tree, targets):
|
||||
def _targets_to_path_infos(repo_fs, targets):
|
||||
path_infos = []
|
||||
missing = []
|
||||
|
||||
for target in targets:
|
||||
if repo_tree.exists(target):
|
||||
path_infos.append(repo_tree.metadata(target).path_info)
|
||||
if repo_fs.exists(target):
|
||||
path_infos.append(repo_fs.metadata(target).path_info)
|
||||
else:
|
||||
missing.append(target)
|
||||
|
||||
|
||||
@@ -251,10 +251,10 @@ class Experiments:
|
||||
|
||||
# NOTE: dirty DVC lock files must be restored to index state to
|
||||
# avoid checking out incorrect persist or checkpoint outs
|
||||
tree = self.scm.get_tree("HEAD")
|
||||
fs = self.scm.get_fs("HEAD")
|
||||
lock_files = [
|
||||
str(fname)
|
||||
for fname in tree.walk_files(self.scm.root_dir)
|
||||
for fname in fs.walk_files(self.scm.root_dir)
|
||||
if is_lock_file(fname)
|
||||
]
|
||||
if lock_files:
|
||||
@@ -333,7 +333,7 @@ class Experiments:
|
||||
path = PathInfo(params_fname)
|
||||
suffix = path.suffix.lower()
|
||||
modify_data = MODIFIERS[suffix]
|
||||
with modify_data(path, tree=self.repo.tree) as data:
|
||||
with modify_data(path, fs=self.repo.fs) as data:
|
||||
benedict(data).merge(params[params_fname], overwrite=True)
|
||||
|
||||
# Force params file changes to be staged in git
|
||||
|
||||
@@ -138,11 +138,11 @@ class BaseExecutor(ABC):
|
||||
|
||||
# TODO: come up with better way to stash repro arguments
|
||||
@staticmethod
|
||||
def pack_repro_args(path, *args, tree=None, extra=None, **kwargs):
|
||||
def pack_repro_args(path, *args, fs=None, extra=None, **kwargs):
|
||||
dpath = os.path.dirname(path)
|
||||
if tree:
|
||||
open_func = tree.open
|
||||
tree.makedirs(dpath)
|
||||
if fs:
|
||||
open_func = fs.open
|
||||
fs.makedirs(dpath)
|
||||
else:
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
|
||||
obj = stage(
|
||||
cache,
|
||||
path_info,
|
||||
repo.repo_tree,
|
||||
repo.repo_fs,
|
||||
jobs=jobs,
|
||||
follow_subrepos=False,
|
||||
)
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ def get(url, path, out=None, rev=None, jobs=None):
|
||||
) as repo:
|
||||
from_info = PathInfo(repo.root_dir) / path
|
||||
to_info = PathInfo(out)
|
||||
repo.repo_tree.download(
|
||||
repo.repo_fs.download(
|
||||
from_info, to_info, jobs=jobs, follow_subrepos=False
|
||||
)
|
||||
finally:
|
||||
|
||||
+4
-4
@@ -35,7 +35,7 @@ def ls(
|
||||
if path:
|
||||
path_info /= path
|
||||
|
||||
ret = _ls(repo.repo_tree, path_info, recursive, dvc_only)
|
||||
ret = _ls(repo.repo_fs, path_info, recursive, dvc_only)
|
||||
|
||||
if path and not ret:
|
||||
raise PathMissingError(path, repo, dvc_only=dvc_only)
|
||||
@@ -48,13 +48,13 @@ def ls(
|
||||
return ret_list
|
||||
|
||||
|
||||
def _ls(tree, path_info, recursive=None, dvc_only=False):
|
||||
def _ls(fs, path_info, recursive=None, dvc_only=False):
|
||||
def onerror(exc):
|
||||
raise exc
|
||||
|
||||
infos = []
|
||||
try:
|
||||
for root, dirs, files in tree.walk(
|
||||
for root, dirs, files in fs.walk(
|
||||
path_info.fspath, onerror=onerror, dvcfiles=True
|
||||
):
|
||||
entries = chain(files, dirs) if not recursive else files
|
||||
@@ -68,7 +68,7 @@ def _ls(tree, path_info, recursive=None, dvc_only=False):
|
||||
|
||||
ret = {}
|
||||
for info in infos:
|
||||
metadata = tree.metadata(info)
|
||||
metadata = fs.metadata(info)
|
||||
if metadata.output_exists or not dvc_only:
|
||||
path = (
|
||||
path_info.name
|
||||
|
||||
@@ -6,13 +6,13 @@ from dvc.exceptions import (
|
||||
NoMetricsFoundError,
|
||||
NoMetricsParsedError,
|
||||
)
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.output import BaseOutput
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import locked
|
||||
from dvc.repo.collect import collect
|
||||
from dvc.repo.live import summary_path_info
|
||||
from dvc.scm.base import SCMError
|
||||
from dvc.tree.repo import RepoTree
|
||||
from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -71,15 +71,15 @@ def _extract_metrics(metrics, path, rev):
|
||||
|
||||
|
||||
def _read_metrics(repo, metrics, rev):
|
||||
tree = RepoTree(repo)
|
||||
fs = RepoFileSystem(repo)
|
||||
|
||||
res = {}
|
||||
for metric in metrics:
|
||||
if not tree.isfile(metric):
|
||||
if not fs.isfile(metric):
|
||||
continue
|
||||
|
||||
try:
|
||||
val = load_yaml(metric, tree=tree)
|
||||
val = load_yaml(metric, fs=fs)
|
||||
except (FileNotFoundError, YAMLFileCorruptedError):
|
||||
logger.debug(
|
||||
"failed to read '%s' on '%s'", metric, rev, exc_info=True
|
||||
|
||||
@@ -48,13 +48,13 @@ def _collect_configs(repo: "Repo", rev, targets=None) -> List["DvcPath"]:
|
||||
def _read_params(repo, configs, rev):
|
||||
res = {}
|
||||
for config in configs:
|
||||
if not repo.tree.exists(config):
|
||||
if not repo.fs.exists(config):
|
||||
continue
|
||||
|
||||
suffix = config.suffix.lower()
|
||||
loader = LOADERS[suffix]
|
||||
try:
|
||||
res[str(config)] = loader(config, tree=repo.tree)
|
||||
res[str(config)] = loader(config, fs=repo.fs)
|
||||
except ParseError:
|
||||
logger.debug(
|
||||
"failed to read '%s' on '%s'", config, rev, exc_info=True
|
||||
|
||||
@@ -49,7 +49,7 @@ class Plots:
|
||||
}}}
|
||||
Data parsing is postponed, since it's affected by props.
|
||||
"""
|
||||
from dvc.tree.repo import RepoTree
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
from dvc.utils.collections import ensure_list
|
||||
|
||||
targets = ensure_list(targets)
|
||||
@@ -60,16 +60,16 @@ class Plots:
|
||||
continue
|
||||
rev = rev or "workspace"
|
||||
|
||||
tree = RepoTree(self.repo)
|
||||
fs = RepoFileSystem(self.repo)
|
||||
plots = _collect_plots(self.repo, targets, rev, recursive)
|
||||
for path_info, props in plots.items():
|
||||
|
||||
if rev not in data:
|
||||
data[rev] = {}
|
||||
|
||||
if tree.isdir(path_info):
|
||||
if fs.isdir(path_info):
|
||||
plot_files = []
|
||||
for pi in tree.walk_files(path_info):
|
||||
for pi in fs.walk_files(path_info):
|
||||
plot_files.append(
|
||||
(pi, relpath(pi, self.repo.root_dir))
|
||||
)
|
||||
@@ -83,7 +83,7 @@ class Plots:
|
||||
|
||||
# Load data from git or dvc cache
|
||||
try:
|
||||
with tree.open(path) as fd:
|
||||
with fs.open(path) as fd:
|
||||
data[rev][repo_path]["data"] = fd.read()
|
||||
except FileNotFoundError:
|
||||
# This might happen simply because cache is absent
|
||||
|
||||
+11
-11
@@ -62,7 +62,7 @@ def _maybe_collect_from_dvc_yaml(
|
||||
from dvc.stage.exceptions import StageNotFound
|
||||
|
||||
stages: StageList = []
|
||||
if loader.tree.exists(PIPELINE_FILE):
|
||||
if loader.fs.exists(PIPELINE_FILE):
|
||||
with suppress(StageNotFound):
|
||||
stages = loader.load_all(PIPELINE_FILE, target, **load_kwargs)
|
||||
return _collect_with_deps(stages, loader.graph) if with_deps else stages
|
||||
@@ -88,7 +88,7 @@ def _collect_specific_target(
|
||||
# else, we assume it's a output name in the `collect_granular()` below
|
||||
msg = "Checking if stage '%s' is in '%s'"
|
||||
logger.debug(msg, target, PIPELINE_FILE)
|
||||
if not (recursive and loader.tree.isdir(target)):
|
||||
if not (recursive and loader.fs.isdir(target)):
|
||||
stages = _maybe_collect_from_dvc_yaml(
|
||||
loader, target, with_deps, accept_group=accept_group,
|
||||
)
|
||||
@@ -304,8 +304,8 @@ class StageLoad:
|
||||
return self.load_all(path, expr, glob=True)
|
||||
|
||||
@property
|
||||
def tree(self):
|
||||
return self.repo.tree
|
||||
def fs(self):
|
||||
return self.repo.fs
|
||||
|
||||
@property
|
||||
def graph(self) -> "DiGraph":
|
||||
@@ -353,7 +353,7 @@ class StageLoad:
|
||||
if not target:
|
||||
return list(graph) if graph else self.repo.stages
|
||||
|
||||
if recursive and self.repo.tree.isdir(target):
|
||||
if recursive and self.repo.fs.isdir(target):
|
||||
from dvc.repo.graph import collect_inside_path
|
||||
|
||||
path = os.path.abspath(target)
|
||||
@@ -398,7 +398,7 @@ class StageLoad:
|
||||
self, target, with_deps, recursive, accept_group
|
||||
)
|
||||
if not stages:
|
||||
if not (recursive and self.tree.isdir(target)):
|
||||
if not (recursive and self.fs.isdir(target)):
|
||||
try:
|
||||
(out,) = self.repo.find_outs_by_path(target, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(target))
|
||||
@@ -443,17 +443,17 @@ class StageLoad:
|
||||
the collection.
|
||||
"""
|
||||
from dvc.dvcfile import is_valid_filename
|
||||
from dvc.tree.local import LocalTree
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
|
||||
scm = self.repo.scm
|
||||
sep = os.sep
|
||||
outs: Set[str] = set()
|
||||
|
||||
is_local_tree = isinstance(self.tree, LocalTree)
|
||||
is_local_fs = isinstance(self.fs, LocalFileSystem)
|
||||
|
||||
def is_ignored(path):
|
||||
# apply only for the local tree
|
||||
return is_local_tree and scm.is_ignored(path)
|
||||
# apply only for the local fs
|
||||
return is_local_fs and scm.is_ignored(path)
|
||||
|
||||
def is_dvcfile_and_not_ignored(root, file):
|
||||
return is_valid_filename(file) and not is_ignored(
|
||||
@@ -465,7 +465,7 @@ class StageLoad:
|
||||
return dir_path in outs or is_ignored(dir_path)
|
||||
|
||||
stages = []
|
||||
for root, dirs, files in self.tree.walk(self.repo.root_dir):
|
||||
for root, dirs, files in self.fs.walk(self.repo.root_dir):
|
||||
dvcfile_filter = partial(is_dvcfile_and_not_ignored, root)
|
||||
|
||||
for file in filter(dvcfile_filter, files):
|
||||
|
||||
@@ -340,15 +340,15 @@ class Git(Base):
|
||||
pass
|
||||
raise NoGitBackendError(name)
|
||||
|
||||
def get_tree(self, rev: str, **kwargs):
|
||||
from dvc.tree.git import GitTree
|
||||
def get_fs(self, rev: str, **kwargs):
|
||||
from dvc.fs.git import GitFileSystem
|
||||
|
||||
from .objects import GitTrie
|
||||
|
||||
resolved = self.resolve_rev(rev)
|
||||
tree_obj = self._backend_func("get_tree_obj", rev=resolved)
|
||||
trie = GitTrie(tree_obj, resolved)
|
||||
return GitTree(self.root_dir, trie, **kwargs)
|
||||
return GitFileSystem(self.root_dir, trie, **kwargs)
|
||||
|
||||
is_ignored = partialmethod(_backend_func, "is_ignored")
|
||||
add = partialmethod(_backend_func, "add")
|
||||
|
||||
+6
-6
@@ -232,9 +232,9 @@ class StageCache:
|
||||
|
||||
remote = self.repo.cloud.get_remote(remote)
|
||||
return self._transfer(
|
||||
_log_exceptions(remote.tree.upload, "upload"),
|
||||
self.repo.cache.local.tree,
|
||||
remote.tree,
|
||||
_log_exceptions(remote.fs.upload, "upload"),
|
||||
self.repo.cache.local.fs,
|
||||
remote.fs,
|
||||
)
|
||||
|
||||
def pull(self, remote):
|
||||
@@ -242,9 +242,9 @@ class StageCache:
|
||||
|
||||
remote = self.repo.cloud.get_remote(remote)
|
||||
return self._transfer(
|
||||
_log_exceptions(remote.tree.download, "download"),
|
||||
remote.tree,
|
||||
self.repo.cache.local.tree,
|
||||
_log_exceptions(remote.fs.download, "download"),
|
||||
remote.fs,
|
||||
self.repo.cache.local.fs,
|
||||
)
|
||||
|
||||
def get_used_cache(self, used_run_cache, *args, **kwargs):
|
||||
|
||||
+9
-9
@@ -133,13 +133,13 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
MAX_UINT = 2 ** 64 - 2
|
||||
|
||||
def __init__(self, repo):
|
||||
from dvc.tree.local import LocalTree
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
|
||||
super().__init__()
|
||||
|
||||
self.repo = repo
|
||||
self.root_dir = repo.root_dir
|
||||
self.tree = LocalTree(None, {"url": self.root_dir})
|
||||
self.fs = LocalFileSystem(None, {"url": self.root_dir})
|
||||
|
||||
state_config = repo.config.get("state", {})
|
||||
self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
|
||||
@@ -410,7 +410,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
assert isinstance(hash_info, HashInfo)
|
||||
assert os.path.exists(path_info)
|
||||
|
||||
actual_mtime, actual_size = get_mtime_and_size(path_info, self.tree)
|
||||
actual_mtime, actual_size = get_mtime_and_size(path_info, self.fs)
|
||||
actual_inode = get_inode(path_info)
|
||||
|
||||
existing_record = self.get_state_record_for_inode(actual_inode)
|
||||
@@ -438,13 +438,13 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
assert isinstance(path_info, str) or path_info.scheme == "local"
|
||||
path = os.fspath(path_info)
|
||||
|
||||
# NOTE: use os.path.exists instead of LocalTree.exists
|
||||
# NOTE: use os.path.exists instead of LocalFileSystem.exists
|
||||
# because it uses lexists() and will return True for broken
|
||||
# symlinks that we cannot stat() in get_mtime_and_size
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
|
||||
actual_mtime, actual_size = get_mtime_and_size(path, self.tree)
|
||||
actual_mtime, actual_size = get_mtime_and_size(path, self.fs)
|
||||
actual_inode = get_inode(path)
|
||||
|
||||
existing_record = self.get_state_record_for_inode(actual_inode)
|
||||
@@ -467,10 +467,10 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
"""
|
||||
assert isinstance(path_info, str) or path_info.scheme == "local"
|
||||
|
||||
if not self.tree.exists(path_info):
|
||||
if not self.fs.exists(path_info):
|
||||
return
|
||||
|
||||
mtime, _ = get_mtime_and_size(path_info, self.tree)
|
||||
mtime, _ = get_mtime_and_size(path_info, self.fs)
|
||||
inode = get_inode(path_info)
|
||||
relative_path = relpath(path_info, self.root_dir)
|
||||
|
||||
@@ -493,11 +493,11 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
inode = self._from_sqlite(inode)
|
||||
path = os.path.join(self.root_dir, relative_path)
|
||||
|
||||
if path in used or not self.tree.exists(path):
|
||||
if path in used or not self.fs.exists(path):
|
||||
continue
|
||||
|
||||
actual_inode = get_inode(path)
|
||||
actual_mtime, _ = get_mtime_and_size(path, self.tree)
|
||||
actual_mtime, _ = get_mtime_and_size(path, self.fs)
|
||||
|
||||
if (inode, mtime) == (actual_inode, actual_mtime):
|
||||
logger.debug("Removing '%s' as unused link.", path)
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .http import HTTPTree
|
||||
|
||||
|
||||
class HTTPSTree(HTTPTree): # pylint:disable=abstract-method
|
||||
scheme = Schemes.HTTPS
|
||||
@@ -1,7 +0,0 @@
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .webdav import WebDAVTree
|
||||
|
||||
|
||||
class WebDAVSTree(WebDAVTree): # pylint:disable=abstract-method
|
||||
scheme = Schemes.WEBDAVS
|
||||
@@ -41,14 +41,14 @@ def _fobj_md5(fobj, hash_md5, binary, progress_func=None):
|
||||
progress_func(len(data))
|
||||
|
||||
|
||||
def file_md5(fname, tree):
|
||||
def file_md5(fname, fs):
|
||||
""" get the (md5 hexdigest, md5 digest) of a file """
|
||||
from dvc.istextfile import istextfile
|
||||
from dvc.progress import Tqdm
|
||||
|
||||
hash_md5 = hashlib.md5()
|
||||
binary = not istextfile(fname, tree=tree)
|
||||
size = tree.getsize(fname) or 0
|
||||
binary = not istextfile(fname, fs=fs)
|
||||
size = fs.getsize(fname) or 0
|
||||
no_progress_bar = True
|
||||
if size >= LARGE_FILE_SIZE:
|
||||
no_progress_bar = False
|
||||
@@ -65,7 +65,7 @@ def file_md5(fname, tree):
|
||||
bytes=True,
|
||||
leave=False,
|
||||
) as pbar:
|
||||
with tree.open(fname, "rb") as fobj:
|
||||
with fs.open(fname, "rb") as fobj:
|
||||
_fobj_md5(fobj, hash_md5, binary, pbar.update)
|
||||
|
||||
return hash_md5.hexdigest()
|
||||
|
||||
+5
-5
@@ -30,15 +30,15 @@ def get_inode(path):
|
||||
return inode
|
||||
|
||||
|
||||
def get_mtime_and_size(path, tree):
|
||||
def get_mtime_and_size(path, fs):
|
||||
import nanotime
|
||||
|
||||
if tree.isdir(path):
|
||||
if fs.isdir(path):
|
||||
size = 0
|
||||
files_mtimes = {}
|
||||
for file_path in tree.walk_files(path):
|
||||
for file_path in fs.walk_files(path):
|
||||
try:
|
||||
stats = tree.stat(file_path)
|
||||
stats = fs.stat(file_path)
|
||||
except OSError as exc:
|
||||
# NOTE: broken symlink case.
|
||||
if exc.errno != errno.ENOENT:
|
||||
@@ -51,7 +51,7 @@ def get_mtime_and_size(path, tree):
|
||||
# max(mtime(f) for f in non_ignored_files)
|
||||
mtime = dict_md5(files_mtimes)
|
||||
else:
|
||||
base_stat = tree.stat(path)
|
||||
base_stat = fs.stat(path)
|
||||
size = base_stat.st_size
|
||||
mtime = base_stat.st_mtime
|
||||
mtime = int(nanotime.timestamp(mtime))
|
||||
|
||||
@@ -8,26 +8,26 @@ from typing_extensions import Protocol
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.tree.base import BaseTree
|
||||
from dvc.fs.base import BaseFileSystem
|
||||
from dvc.types import AnyPath
|
||||
|
||||
|
||||
class DumperFn(Protocol):
|
||||
def __call__(
|
||||
self, path: "AnyPath", data: Any, tree: "BaseTree" = None
|
||||
self, path: "AnyPath", data: Any, fs: "BaseFileSystem" = None
|
||||
) -> Any:
|
||||
...
|
||||
|
||||
|
||||
class ModifierFn(Protocol):
|
||||
def __call__(
|
||||
self, path: "AnyPath", tree: "BaseTree" = None
|
||||
self, path: "AnyPath", fs: "BaseFileSystem" = None
|
||||
) -> ContextManager[Dict]:
|
||||
...
|
||||
|
||||
|
||||
class LoaderFn(Protocol):
|
||||
def __call__(self, path: "AnyPath", tree: "BaseTree" = None) -> Any:
|
||||
def __call__(self, path: "AnyPath", fs: "BaseFileSystem" = None) -> Any:
|
||||
...
|
||||
|
||||
|
||||
@@ -45,14 +45,14 @@ class ParseError(DvcException):
|
||||
super().__init__(f"unable to read: '{path}', {message}")
|
||||
|
||||
|
||||
def _load_data(path: "AnyPath", parser: ParserFn, tree: "BaseTree" = None):
|
||||
open_fn = tree.open if tree else open
|
||||
def _load_data(path: "AnyPath", parser: ParserFn, fs: "BaseFileSystem" = None):
|
||||
open_fn = fs.open if fs else open
|
||||
with open_fn(path, encoding="utf-8") as fd: # type: ignore
|
||||
return parser(fd.read(), path)
|
||||
|
||||
|
||||
def _dump_data(path, data: Any, dumper: DumperFn, tree: "BaseTree" = None):
|
||||
open_fn = tree.open if tree else open
|
||||
def _dump_data(path, data: Any, dumper: DumperFn, fs: "BaseFileSystem" = None):
|
||||
open_fn = fs.open if fs else open
|
||||
with open_fn(path, "w+", encoding="utf-8") as fd: # type: ignore
|
||||
dumper(data, fd)
|
||||
|
||||
@@ -62,10 +62,10 @@ def _modify_data(
|
||||
path: "AnyPath",
|
||||
parser: ParserFn,
|
||||
dumper: DumperFn,
|
||||
tree: "BaseTree" = None,
|
||||
fs: "BaseFileSystem" = None,
|
||||
):
|
||||
exists_fn = tree.exists if tree else os.path.exists
|
||||
exists_fn = fs.exists if fs else os.path.exists
|
||||
file_exists = exists_fn(path) # type: ignore
|
||||
data = _load_data(path, parser=parser, tree=tree) if file_exists else {}
|
||||
data = _load_data(path, parser=parser, fs=fs) if file_exists else {}
|
||||
yield data
|
||||
dumper(path, data, tree=tree)
|
||||
dumper(path, data, fs=fs)
|
||||
|
||||
@@ -10,8 +10,8 @@ class JSONFileCorruptedError(ParseError):
|
||||
super().__init__(path, "JSON file structure is corrupted")
|
||||
|
||||
|
||||
def load_json(path, tree=None):
|
||||
return _load_data(path, parser=parse_json, tree=tree)
|
||||
def load_json(path, fs=None):
|
||||
return _load_data(path, parser=parse_json, fs=fs)
|
||||
|
||||
|
||||
def parse_json(text, path, **kwargs):
|
||||
@@ -19,11 +19,11 @@ def parse_json(text, path, **kwargs):
|
||||
return json.loads(text, **kwargs) or {}
|
||||
|
||||
|
||||
def dump_json(path, data, tree=None):
|
||||
return _dump_data(path, data, dumper=json.dump, tree=tree)
|
||||
def dump_json(path, data, fs=None):
|
||||
return _dump_data(path, data, dumper=json.dump, fs=fs)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def modify_json(path, tree=None):
|
||||
with _modify_data(path, parse_json, dump_json, tree=tree) as d:
|
||||
def modify_json(path, fs=None):
|
||||
with _modify_data(path, parse_json, dump_json, fs=fs) as d:
|
||||
yield d
|
||||
|
||||
@@ -14,8 +14,8 @@ class PythonFileCorruptedError(ParseError):
|
||||
super().__init__(path, message)
|
||||
|
||||
|
||||
def load_py(path, tree=None):
|
||||
return _load_data(path, parser=parse_py, tree=tree)
|
||||
def load_py(path, fs=None):
|
||||
return _load_data(path, parser=parse_py, fs=fs)
|
||||
|
||||
|
||||
def parse_py(text, path):
|
||||
@@ -76,13 +76,13 @@ def _dump(data, stream):
|
||||
stream.close()
|
||||
|
||||
|
||||
def dump_py(path, data, tree=None):
|
||||
return _dump_data(path, data, dumper=_dump, tree=tree)
|
||||
def dump_py(path, data, fs=None):
|
||||
return _dump_data(path, data, dumper=_dump, fs=fs)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def modify_py(path, tree=None):
|
||||
with _modify_data(path, parse_py_for_update, dump_py, tree=tree) as d:
|
||||
def modify_py(path, fs=None):
|
||||
with _modify_data(path, parse_py_for_update, dump_py, fs=fs) as d:
|
||||
yield d
|
||||
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ class TOMLFileCorruptedError(ParseError):
|
||||
super().__init__(path, "TOML file structure is corrupted")
|
||||
|
||||
|
||||
def load_toml(path, tree=None):
|
||||
return _load_data(path, parser=parse_toml, tree=tree)
|
||||
def load_toml(path, fs=None):
|
||||
return _load_data(path, parser=parse_toml, fs=fs)
|
||||
|
||||
|
||||
def parse_toml(text, path, decoder=None):
|
||||
@@ -40,11 +40,11 @@ def _dump(data, stream):
|
||||
return toml.dump(data, stream, encoder=toml.TomlPreserveCommentEncoder())
|
||||
|
||||
|
||||
def dump_toml(path, data, tree=None):
|
||||
return _dump_data(path, data, dumper=_dump, tree=tree)
|
||||
def dump_toml(path, data, fs=None):
|
||||
return _dump_data(path, data, dumper=_dump, fs=fs)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def modify_toml(path, tree=None):
|
||||
with _modify_data(path, parse_toml_for_update, dump_toml, tree=tree) as d:
|
||||
def modify_toml(path, fs=None):
|
||||
with _modify_data(path, parse_toml_for_update, dump_toml, fs=fs) as d:
|
||||
yield d
|
||||
|
||||
@@ -16,8 +16,8 @@ class YAMLFileCorruptedError(YAMLError):
|
||||
super().__init__(path, "YAML file structure is corrupted")
|
||||
|
||||
|
||||
def load_yaml(path, tree=None):
|
||||
return _load_data(path, parser=parse_yaml, tree=tree)
|
||||
def load_yaml(path, fs=None):
|
||||
return _load_data(path, parser=parse_yaml, fs=fs)
|
||||
|
||||
|
||||
def parse_yaml(text, path, typ="safe"):
|
||||
@@ -65,8 +65,8 @@ def _dump(data, stream):
|
||||
return yaml.dump(data, stream)
|
||||
|
||||
|
||||
def dump_yaml(path, data, tree=None):
|
||||
return _dump_data(path, data, dumper=_dump, tree=tree)
|
||||
def dump_yaml(path, data, fs=None):
|
||||
return _dump_data(path, data, dumper=_dump, fs=fs)
|
||||
|
||||
|
||||
def loads_yaml(s, typ="safe"):
|
||||
@@ -82,6 +82,6 @@ def dumps_yaml(d):
|
||||
|
||||
|
||||
@contextmanager
|
||||
def modify_yaml(path, tree=None):
|
||||
with _modify_data(path, parse_yaml_for_update, dump_yaml, tree=tree) as d:
|
||||
def modify_yaml(path, fs=None):
|
||||
with _modify_data(path, parse_yaml_for_update, dump_yaml, fs=fs) as d:
|
||||
yield d
|
||||
|
||||
+1
-1
@@ -45,7 +45,7 @@ def reset_loglevel(request, caplog):
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def _close_pools():
|
||||
from dvc.tree.pool import close_pools
|
||||
from dvc.fs.pool import close_pools
|
||||
|
||||
yield
|
||||
close_pools()
|
||||
|
||||
@@ -8,7 +8,7 @@ from dvc import api
|
||||
from dvc.exceptions import FileMissingError, OutputNotFoundError
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.utils.fs import remove
|
||||
from tests.unit.tree.test_repo import make_subrepo
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
|
||||
cloud_names = [
|
||||
"s3",
|
||||
|
||||
@@ -20,10 +20,10 @@ def test_new_checkpoint(
|
||||
for rev in dvc.brancher([exp]):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
tree = dvc.repo_tree
|
||||
with tree.open(tmp_dir / "foo") as fobj:
|
||||
fs = dvc.repo_fs
|
||||
with fs.open(tmp_dir / "foo") as fobj:
|
||||
assert fobj.read().strip() == str(checkpoint_stage.iterations)
|
||||
with tree.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
if workspace:
|
||||
@@ -69,10 +69,10 @@ def test_resume_checkpoint(
|
||||
for rev in dvc.brancher([exp]):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
tree = dvc.repo_tree
|
||||
with tree.open(tmp_dir / "foo") as fobj:
|
||||
fs = dvc.repo_fs
|
||||
with fs.open(tmp_dir / "foo") as fobj:
|
||||
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
|
||||
with tree.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
if workspace:
|
||||
@@ -104,10 +104,10 @@ def test_reset_checkpoint(
|
||||
for rev in dvc.brancher([exp]):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
tree = dvc.repo_tree
|
||||
with tree.open(tmp_dir / "foo") as fobj:
|
||||
fs = dvc.repo_fs
|
||||
with fs.open(tmp_dir / "foo") as fobj:
|
||||
assert fobj.read().strip() == str(checkpoint_stage.iterations)
|
||||
with tree.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
if workspace:
|
||||
@@ -143,19 +143,19 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage, workspace):
|
||||
for rev in dvc.brancher([checkpoint_a]):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
tree = dvc.repo_tree
|
||||
with tree.open(tmp_dir / "foo") as fobj:
|
||||
fs = dvc.repo_fs
|
||||
with fs.open(tmp_dir / "foo") as fobj:
|
||||
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
|
||||
with tree.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
for rev in dvc.brancher([checkpoint_b]):
|
||||
if rev == "workspace":
|
||||
continue
|
||||
tree = dvc.repo_tree
|
||||
with tree.open(tmp_dir / "foo") as fobj:
|
||||
fs = dvc.repo_fs
|
||||
with fs.open(tmp_dir / "foo") as fobj:
|
||||
assert fobj.read().strip() == str(2 * checkpoint_stage.iterations)
|
||||
with tree.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == "foo: 100"
|
||||
|
||||
with pytest.raises(MultipleBranchError):
|
||||
|
||||
@@ -28,8 +28,8 @@ def test_new_simple(tmp_dir, scm, dvc, exp_stage, mocker, name, workspace):
|
||||
assert ref_info and ref_info.baseline_sha == baseline
|
||||
|
||||
new_mock.assert_called_once()
|
||||
tree = scm.get_tree(exp)
|
||||
with tree.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
fs = scm.get_fs(exp)
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
if workspace:
|
||||
@@ -68,8 +68,8 @@ def test_experiment_exists(tmp_dir, scm, dvc, exp_stage, mocker, workspace):
|
||||
)
|
||||
exp = first(results)
|
||||
|
||||
tree = scm.get_tree(exp)
|
||||
with tree.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
fs = scm.get_fs(exp)
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == "foo: 3"
|
||||
|
||||
|
||||
@@ -142,8 +142,8 @@ def test_modify_params(tmp_dir, scm, dvc, mocker, changes, expected):
|
||||
exp = first(results)
|
||||
|
||||
new_mock.assert_called_once()
|
||||
tree = scm.get_tree(exp)
|
||||
with tree.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
fs = scm.get_fs(exp)
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == expected
|
||||
|
||||
|
||||
@@ -238,10 +238,10 @@ def test_update_py_params(tmp_dir, scm, dvc):
|
||||
)
|
||||
exp_a = first(results)
|
||||
|
||||
tree = scm.get_tree(exp_a)
|
||||
with tree.open(tmp_dir / "params.py") as fobj:
|
||||
fs = scm.get_fs(exp_a)
|
||||
with fs.open(tmp_dir / "params.py") as fobj:
|
||||
assert fobj.read().strip() == "INT = 2"
|
||||
with tree.open(tmp_dir / "metrics.py") as fobj:
|
||||
with fs.open(tmp_dir / "metrics.py") as fobj:
|
||||
assert fobj.read().strip() == "INT = 2"
|
||||
|
||||
tmp_dir.gen(
|
||||
@@ -280,10 +280,10 @@ def test_update_py_params(tmp_dir, scm, dvc):
|
||||
# in order to compare with the original
|
||||
return text.replace("\r\n", "\n")
|
||||
|
||||
tree = scm.get_tree(exp_a)
|
||||
with tree.open(tmp_dir / "params.py") as fobj:
|
||||
fs = scm.get_fs(exp_a)
|
||||
with fs.open(tmp_dir / "params.py") as fobj:
|
||||
assert _dos2unix(fobj.read().strip()) == result
|
||||
with tree.open(tmp_dir / "metrics.py") as fobj:
|
||||
with fs.open(tmp_dir / "metrics.py") as fobj:
|
||||
assert _dos2unix(fobj.read().strip()) == result
|
||||
|
||||
tmp_dir.gen("params.py", "INT = 1\n")
|
||||
@@ -406,11 +406,11 @@ def test_untracked(tmp_dir, scm, dvc, caplog, workspace):
|
||||
stage.addressing, params=["foo=2"], tmp_dir=not workspace
|
||||
)
|
||||
exp = first(results)
|
||||
tree = scm.get_tree(exp)
|
||||
assert tree.exists("dvc.yaml")
|
||||
assert tree.exists("dvc.lock")
|
||||
assert tree.exists("copy.py")
|
||||
with tree.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
fs = scm.get_fs(exp)
|
||||
assert fs.exists("dvc.yaml")
|
||||
assert fs.exists("dvc.lock")
|
||||
assert fs.exists("copy.py")
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
|
||||
@@ -428,8 +428,8 @@ def test_dirty_lockfile(tmp_dir, scm, dvc, exp_stage, workspace):
|
||||
)
|
||||
exp = first(results)
|
||||
|
||||
tree = scm.get_tree(exp)
|
||||
with tree.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
fs = scm.get_fs(exp)
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
if not workspace:
|
||||
@@ -509,10 +509,10 @@ def test_subdir(tmp_dir, scm, dvc, workspace):
|
||||
exp = first(results)
|
||||
ref_info = first(exp_refs_by_rev(scm, exp))
|
||||
|
||||
tree = scm.get_tree(exp)
|
||||
fs = scm.get_fs(exp)
|
||||
for fname in ["metrics.yaml", "dvc.lock"]:
|
||||
assert tree.exists(subdir / fname)
|
||||
with tree.open(subdir / "metrics.yaml") as fobj:
|
||||
assert fs.exists(subdir / fname)
|
||||
with fs.open(subdir / "metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
assert dvc.experiments.get_exact_name(exp) == ref_info.name
|
||||
@@ -521,7 +521,7 @@ def test_subdir(tmp_dir, scm, dvc, workspace):
|
||||
|
||||
@pytest.mark.parametrize("workspace", [True, False])
|
||||
def test_subrepo(tmp_dir, scm, workspace):
|
||||
from tests.unit.tree.test_repo import make_subrepo
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
|
||||
subrepo = tmp_dir / "dir" / "repo"
|
||||
make_subrepo(subrepo, scm)
|
||||
@@ -554,10 +554,10 @@ def test_subrepo(tmp_dir, scm, workspace):
|
||||
exp = first(results)
|
||||
ref_info = first(exp_refs_by_rev(scm, exp))
|
||||
|
||||
tree = scm.get_tree(exp)
|
||||
fs = scm.get_fs(exp)
|
||||
for fname in ["metrics.yaml", "dvc.lock"]:
|
||||
assert tree.exists(subrepo / fname)
|
||||
with tree.open(subrepo / "metrics.yaml") as fobj:
|
||||
assert fs.exists(subrepo / fname)
|
||||
with fs.open(subrepo / "metrics.yaml") as fobj:
|
||||
assert fobj.read().strip() == "foo: 2"
|
||||
|
||||
assert subrepo.dvc.experiments.get_exact_name(exp) == ref_info.name
|
||||
@@ -577,8 +577,8 @@ def test_queue(tmp_dir, scm, dvc, exp_stage, mocker):
|
||||
expected = {"foo: 2", "foo: 3"}
|
||||
metrics = set()
|
||||
for exp in results:
|
||||
tree = scm.get_tree(exp)
|
||||
with tree.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
fs = scm.get_fs(exp)
|
||||
with fs.open(tmp_dir / "metrics.yaml") as fobj:
|
||||
metrics.add(fobj.read().strip())
|
||||
assert expected == metrics
|
||||
|
||||
|
||||
@@ -108,14 +108,14 @@ def test_show_checkpoint(
|
||||
for i, rev in enumerate(checkpoints):
|
||||
if i == 0:
|
||||
name = dvc.experiments.get_exact_name(rev)
|
||||
tree = "╓"
|
||||
fs = "╓"
|
||||
elif i == len(checkpoints) - 1:
|
||||
name = rev[:7]
|
||||
tree = "╨"
|
||||
fs = "╨"
|
||||
else:
|
||||
name = rev[:7]
|
||||
tree = "╟"
|
||||
assert f"{tree} {name}" in cap.out
|
||||
fs = "╟"
|
||||
assert f"{fs} {name}" in cap.out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("workspace", [True, False])
|
||||
|
||||
@@ -3,10 +3,10 @@ import os
|
||||
import pytest
|
||||
|
||||
from dvc.exceptions import DownloadError, UploadError
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
from dvc.remote.base import Remote
|
||||
from dvc.remote.index import RemoteIndex
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.tree.local import LocalTree
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker):
|
||||
dvc.config["core"]["remote"] = "upstream"
|
||||
|
||||
# patch hashes_exist since the LocalRemote normally overrides
|
||||
# BaseTree.hashes_exist.
|
||||
# BaseFileSystem.hashes_exist.
|
||||
def hashes_exist(self, *args, **kwargs):
|
||||
return Remote.hashes_exist(self, *args, **kwargs)
|
||||
|
||||
@@ -85,7 +85,7 @@ def test_clear_on_download_err(tmp_dir, dvc, tmp_path_factory, remote, mocker):
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
|
||||
mocked_clear = mocker.patch.object(remote.INDEX_CLS, "clear")
|
||||
mocker.patch.object(LocalTree, "_download", side_effect=Exception)
|
||||
mocker.patch.object(LocalFileSystem, "_download", side_effect=Exception)
|
||||
with pytest.raises(DownloadError):
|
||||
dvc.pull()
|
||||
mocked_clear.assert_called_once_with()
|
||||
@@ -95,14 +95,14 @@ def test_partial_upload(tmp_dir, dvc, tmp_path_factory, remote, mocker):
|
||||
tmp_dir.dvc_gen({"foo": "foo content"})
|
||||
tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})
|
||||
|
||||
original = LocalTree._upload
|
||||
original = LocalFileSystem._upload
|
||||
|
||||
def unreliable_upload(self, from_file, to_info, name=None, **kwargs):
|
||||
if "baz" in name:
|
||||
raise Exception("stop baz")
|
||||
return original(self, from_file, to_info, name, **kwargs)
|
||||
|
||||
mocker.patch.object(LocalTree, "_upload", unreliable_upload)
|
||||
mocker.patch.object(LocalFileSystem, "_upload", unreliable_upload)
|
||||
with pytest.raises(UploadError):
|
||||
dvc.push()
|
||||
with remote.index:
|
||||
|
||||
@@ -21,6 +21,7 @@ from dvc.exceptions import (
|
||||
OverlappingOutputPathsError,
|
||||
RecursiveAddingWhileUsingFilename,
|
||||
)
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.main import main
|
||||
from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
|
||||
@@ -28,7 +29,6 @@ from dvc.repo import Repo as DvcRepo
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import StagePathNotFoundError
|
||||
from dvc.system import System
|
||||
from dvc.tree.local import LocalTree
|
||||
from dvc.utils import LARGE_DIR_SIZE, file_md5, relpath
|
||||
from dvc.utils.fs import path_isin
|
||||
from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml
|
||||
@@ -38,7 +38,7 @@ from tests.utils import get_gitignore_content
|
||||
|
||||
def test_add(tmp_dir, dvc):
|
||||
(stage,) = tmp_dir.dvc_gen({"foo": "foo"})
|
||||
md5 = file_md5("foo", dvc.tree)
|
||||
md5 = file_md5("foo", dvc.fs)
|
||||
|
||||
assert stage is not None
|
||||
|
||||
@@ -732,7 +732,7 @@ def test_should_not_checkout_when_adding_cached_copy(tmp_dir, dvc, mocker):
|
||||
|
||||
shutil.copy("bar", "foo")
|
||||
|
||||
copy_spy = mocker.spy(dvc.cache.local.tree, "copy")
|
||||
copy_spy = mocker.spy(dvc.cache.local.fs, "copy")
|
||||
|
||||
dvc.add("foo")
|
||||
|
||||
@@ -758,7 +758,7 @@ def test_should_relink_on_repeated_add(
|
||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||
|
||||
os.remove("foo")
|
||||
getattr(dvc.cache.local.tree, link)(PathInfo("bar"), PathInfo("foo"))
|
||||
getattr(dvc.cache.local.fs, link)(PathInfo("bar"), PathInfo("foo"))
|
||||
|
||||
dvc.cache.local.cache_types = [new_link]
|
||||
|
||||
@@ -838,7 +838,7 @@ def test_add_empty_files(tmp_dir, dvc, link):
|
||||
def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
|
||||
dvc.cache.local.cache_types = ["hardlink"]
|
||||
tmp_dir.gen({"foo": "", "bar": "", "lorem": "lorem", "ipsum": "ipsum"})
|
||||
m = mocker.spy(LocalTree, "is_hardlink")
|
||||
m = mocker.spy(LocalFileSystem, "is_hardlink")
|
||||
stages = dvc.add(["foo", "bar", "lorem", "ipsum"])
|
||||
|
||||
assert m.call_count == 1
|
||||
|
||||
@@ -89,7 +89,7 @@ class TestExternalCacheDir(TestDvc):
|
||||
|
||||
self.dvc.__init__()
|
||||
|
||||
assert self.dvc.cache.ssh.tree.path_info == ssh_url + "/tmp"
|
||||
assert self.dvc.cache.ssh.fs.path_info == ssh_url + "/tmp"
|
||||
|
||||
|
||||
class TestSharedCacheDir(TestDir):
|
||||
|
||||
@@ -17,12 +17,12 @@ from dvc.exceptions import (
|
||||
DvcException,
|
||||
NoOutputOrStageError,
|
||||
)
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
from dvc.main import main
|
||||
from dvc.repo import Repo as DvcRepo
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
from dvc.system import System
|
||||
from dvc.tree.local import LocalTree
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import remove, walk_files
|
||||
from dvc.utils.serialize import dump_yaml, load_yaml
|
||||
@@ -148,7 +148,7 @@ class CheckoutBase(TestDvcGit):
|
||||
paths = [
|
||||
path
|
||||
for output in stage["outs"]
|
||||
for path in self.dvc.tree.walk_files(output["path"])
|
||||
for path in self.dvc.fs.walk_files(output["path"])
|
||||
]
|
||||
|
||||
return [
|
||||
@@ -312,8 +312,8 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
|
||||
class TestCheckoutMissingMd5InStageFile(TestRepro):
|
||||
def test(self):
|
||||
d = load_yaml(self.file1_stage)
|
||||
del d[Stage.PARAM_OUTS][0][LocalTree.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_DEPS][0][LocalTree.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_OUTS][0][LocalFileSystem.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_DEPS][0][LocalFileSystem.PARAM_CHECKSUM]
|
||||
dump_yaml(self.file1_stage, d)
|
||||
|
||||
with pytest.raises(CheckoutError):
|
||||
@@ -769,7 +769,7 @@ def test_checkout_for_external_outputs(tmp_dir, dvc, workspace):
|
||||
dvc.add("remote://workspace/foo")
|
||||
|
||||
remote = dvc.cloud.get_remote("workspace")
|
||||
remote.tree.remove(file_path)
|
||||
remote.fs.remove(file_path)
|
||||
assert not file_path.exists()
|
||||
|
||||
stats = dvc.checkout(force=True)
|
||||
|
||||
@@ -3,13 +3,13 @@ import os
|
||||
from mock import ANY, patch
|
||||
|
||||
from dvc.external_repo import CLONES, external_repo
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
from dvc.objects import save, stage
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.scm.git import Git
|
||||
from dvc.tree.local import LocalTree
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import makedirs, remove
|
||||
from tests.unit.tree.test_repo import make_subrepo
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
|
||||
|
||||
def test_external_repo(erepo_dir):
|
||||
@@ -51,7 +51,7 @@ def test_cache_reused(erepo_dir, mocker, local_cloud):
|
||||
erepo_dir.dvc_gen("file", "text", commit="add file")
|
||||
erepo_dir.dvc.push()
|
||||
|
||||
download_spy = mocker.spy(LocalTree, "download")
|
||||
download_spy = mocker.spy(LocalFileSystem, "download")
|
||||
|
||||
# Use URL to prevent any fishy optimizations
|
||||
url = f"file://{erepo_dir}"
|
||||
@@ -92,7 +92,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
|
||||
|
||||
dest = tmp_dir / "file"
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
repo.repo_tree.download(
|
||||
repo.repo_fs.download(
|
||||
PathInfo(repo.root_dir) / "subdir" / "file", PathInfo(dest),
|
||||
)
|
||||
|
||||
@@ -187,7 +187,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
|
||||
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
|
||||
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
repo.repo_tree.download(
|
||||
repo.repo_fs.download(
|
||||
PathInfo(repo.root_dir) / "dir",
|
||||
PathInfo(tmp_dir / "out"),
|
||||
follow_subrepos=False,
|
||||
@@ -203,7 +203,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
|
||||
obj = stage(
|
||||
repo.cache.local,
|
||||
PathInfo(repo.root_dir) / "dir",
|
||||
repo.repo_tree,
|
||||
repo.repo_fs,
|
||||
follow_subrepos=False,
|
||||
)
|
||||
save(repo.cache.local, obj)
|
||||
@@ -226,7 +226,7 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
|
||||
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
|
||||
|
||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||
repo.repo_tree.download(
|
||||
repo.repo_fs.download(
|
||||
PathInfo(repo.root_dir) / "dir",
|
||||
PathInfo(tmp_dir / "out"),
|
||||
follow_subrepos=False,
|
||||
|
||||
+336
-21
@@ -1,36 +1,351 @@
|
||||
import os
|
||||
import stat
|
||||
from operator import itemgetter
|
||||
from os.path import join
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.utils.fs import copyfile, makedirs
|
||||
from dvc.fs import get_cloud_fs
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.scm import SCM
|
||||
from tests.basic_env import TestDir, TestGit, TestGitSubmodule
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
|
||||
def test_makedirs_permissions(tmp_dir):
|
||||
dir_mode = 0o755
|
||||
intermediate_dir = "тестовая-директория"
|
||||
test_dir = os.path.join(intermediate_dir, "data")
|
||||
class TestLocalFileSystem(TestDir):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.fs = LocalFileSystem(None, {})
|
||||
|
||||
assert not os.path.exists(intermediate_dir)
|
||||
def test_open(self):
|
||||
with self.fs.open(self.FOO) as fd:
|
||||
self.assertEqual(fd.read(), self.FOO_CONTENTS)
|
||||
with self.fs.open(self.UNICODE, encoding="utf-8") as fd:
|
||||
self.assertEqual(fd.read(), self.UNICODE_CONTENTS)
|
||||
|
||||
makedirs(test_dir, mode=dir_mode)
|
||||
def test_exists(self):
|
||||
self.assertTrue(self.fs.exists(self.FOO))
|
||||
self.assertTrue(self.fs.exists(self.UNICODE))
|
||||
self.assertFalse(self.fs.exists("not-existing-file"))
|
||||
|
||||
assert stat.S_IMODE(os.stat(test_dir).st_mode) == dir_mode
|
||||
assert stat.S_IMODE(os.stat(intermediate_dir).st_mode) == dir_mode
|
||||
def test_isdir(self):
|
||||
self.assertTrue(self.fs.isdir(self.DATA_DIR))
|
||||
self.assertFalse(self.fs.isdir(self.FOO))
|
||||
self.assertFalse(self.fs.isdir("not-existing-file"))
|
||||
|
||||
def test_isfile(self):
|
||||
self.assertTrue(self.fs.isfile(self.FOO))
|
||||
self.assertFalse(self.fs.isfile(self.DATA_DIR))
|
||||
self.assertFalse(self.fs.isfile("not-existing-file"))
|
||||
|
||||
|
||||
def test_copyfile(tmp_dir):
|
||||
src = "file1"
|
||||
dest = "file2"
|
||||
dest_dir = "testdir"
|
||||
class GitFileSystemTests:
|
||||
# pylint: disable=no-member
|
||||
def test_open(self):
|
||||
self.scm.add([self.FOO, self.UNICODE, self.DATA_DIR])
|
||||
self.scm.commit("add")
|
||||
|
||||
tmp_dir.gen(src, "file1contents")
|
||||
fs = self.scm.get_fs("master")
|
||||
with fs.open(self.FOO) as fd:
|
||||
self.assertEqual(fd.read(), self.FOO_CONTENTS)
|
||||
with fs.open(self.UNICODE) as fd:
|
||||
self.assertEqual(fd.read(), self.UNICODE_CONTENTS)
|
||||
with self.assertRaises(IOError):
|
||||
fs.open("not-existing-file")
|
||||
with self.assertRaises(IOError):
|
||||
fs.open(self.DATA_DIR)
|
||||
|
||||
os.mkdir(dest_dir)
|
||||
def test_exists(self):
|
||||
fs = self.scm.get_fs("master")
|
||||
self.assertFalse(fs.exists(self.FOO))
|
||||
self.assertFalse(fs.exists(self.UNICODE))
|
||||
self.assertFalse(fs.exists(self.DATA_DIR))
|
||||
self.scm.add([self.FOO, self.UNICODE, self.DATA])
|
||||
self.scm.commit("add")
|
||||
|
||||
copyfile(src, dest)
|
||||
assert (tmp_dir / dest).read_text() == "file1contents"
|
||||
fs = self.scm.get_fs("master")
|
||||
self.assertTrue(fs.exists(self.FOO))
|
||||
self.assertTrue(fs.exists(self.UNICODE))
|
||||
self.assertTrue(fs.exists(self.DATA_DIR))
|
||||
self.assertFalse(fs.exists("non-existing-file"))
|
||||
|
||||
copyfile(src, dest_dir)
|
||||
assert (tmp_dir / dest_dir / src).read_text() == "file1contents"
|
||||
def test_isdir(self):
|
||||
self.scm.add([self.FOO, self.DATA_DIR])
|
||||
self.scm.commit("add")
|
||||
|
||||
fs = self.scm.get_fs("master")
|
||||
self.assertTrue(fs.isdir(self.DATA_DIR))
|
||||
self.assertFalse(fs.isdir(self.FOO))
|
||||
self.assertFalse(fs.isdir("non-existing-file"))
|
||||
|
||||
def test_isfile(self):
|
||||
self.scm.add([self.FOO, self.DATA_DIR])
|
||||
self.scm.commit("add")
|
||||
|
||||
fs = self.scm.get_fs("master")
|
||||
self.assertTrue(fs.isfile(self.FOO))
|
||||
self.assertFalse(fs.isfile(self.DATA_DIR))
|
||||
self.assertFalse(fs.isfile("not-existing-file"))
|
||||
|
||||
|
||||
class TestGitFileSystem(TestGit, GitFileSystemTests):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.scm = SCM(self._root_dir)
|
||||
|
||||
|
||||
class TestGitSubmoduleFileSystem(TestGitSubmodule, GitFileSystemTests):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.scm = SCM(self._root_dir)
|
||||
self._pushd(self._root_dir)
|
||||
|
||||
|
||||
class AssertWalkEqualMixin:
|
||||
def assertWalkEqual(self, actual, expected, msg=None):
|
||||
def convert_to_sets(walk_results):
|
||||
return [
|
||||
(root, set(dirs), set(nondirs))
|
||||
for root, dirs, nondirs in walk_results
|
||||
]
|
||||
|
||||
self.assertEqual(
|
||||
convert_to_sets(actual), convert_to_sets(expected), msg=msg
|
||||
)
|
||||
|
||||
|
||||
class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir):
|
||||
def test(self):
|
||||
fs = LocalFileSystem(None, {"url": self._root_dir})
|
||||
self.assertWalkEqual(
|
||||
fs.walk(self._root_dir),
|
||||
[
|
||||
(
|
||||
self._root_dir,
|
||||
["data_dir"],
|
||||
["code.py", "bar", "тест", "foo"],
|
||||
),
|
||||
(join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]),
|
||||
(
|
||||
join(self._root_dir, "data_dir", "data_sub_dir"),
|
||||
[],
|
||||
["data_sub"],
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
def test_subdir(self):
|
||||
fs = LocalFileSystem(None, {"url": self._root_dir})
|
||||
self.assertWalkEqual(
|
||||
fs.walk(join("data_dir", "data_sub_dir")),
|
||||
[(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
|
||||
)
|
||||
|
||||
|
||||
class TestWalkInGit(AssertWalkEqualMixin, TestGit):
|
||||
def test_nobranch(self):
|
||||
fs = LocalFileSystem(None, {"url": self._root_dir}, use_dvcignore=True)
|
||||
self.assertWalkEqual(
|
||||
fs.walk("."),
|
||||
[
|
||||
(".", ["data_dir"], ["bar", "тест", "code.py", "foo"]),
|
||||
(join("data_dir"), ["data_sub_dir"], ["data"]),
|
||||
(join("data_dir", "data_sub_dir"), [], ["data_sub"]),
|
||||
],
|
||||
)
|
||||
self.assertWalkEqual(
|
||||
fs.walk(join("data_dir", "data_sub_dir")),
|
||||
[(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
|
||||
)
|
||||
|
||||
def test_branch(self):
|
||||
scm = SCM(self._root_dir)
|
||||
scm.add([self.DATA_SUB_DIR])
|
||||
scm.commit("add data_dir/data_sub_dir/data_sub")
|
||||
fs = scm.get_fs("master")
|
||||
self.assertWalkEqual(
|
||||
fs.walk("."),
|
||||
[
|
||||
(self._root_dir, ["data_dir"], ["code.py"]),
|
||||
(join(self._root_dir, "data_dir"), ["data_sub_dir"], []),
|
||||
(
|
||||
join(self._root_dir, "data_dir", "data_sub_dir"),
|
||||
[],
|
||||
["data_sub"],
|
||||
),
|
||||
],
|
||||
)
|
||||
self.assertWalkEqual(
|
||||
fs.walk(join("data_dir", "data_sub_dir")),
|
||||
[
|
||||
(
|
||||
join(self._root_dir, "data_dir", "data_sub_dir"),
|
||||
[],
|
||||
["data_sub"],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_cleanfs_subrepo(tmp_dir, dvc, scm, monkeypatch):
|
||||
tmp_dir.gen({"subdir": {}})
|
||||
subrepo_dir = tmp_dir / "subdir"
|
||||
with subrepo_dir.chdir():
|
||||
subrepo = Repo.init(subdir=True)
|
||||
subrepo_dir.gen({"foo": "foo", "dir": {"bar": "bar"}})
|
||||
|
||||
path = PathInfo(subrepo_dir)
|
||||
|
||||
assert dvc.fs.use_dvcignore
|
||||
assert not dvc.fs.exists(path / "foo")
|
||||
assert not dvc.fs.isfile(path / "foo")
|
||||
assert not dvc.fs.exists(path / "dir")
|
||||
assert not dvc.fs.isdir(path / "dir")
|
||||
|
||||
assert subrepo.fs.use_dvcignore
|
||||
assert subrepo.fs.exists(path / "foo")
|
||||
assert subrepo.fs.isfile(path / "foo")
|
||||
assert subrepo.fs.exists(path / "dir")
|
||||
assert subrepo.fs.isdir(path / "dir")
|
||||
|
||||
|
||||
def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
|
||||
tmp_dir.dvc_gen({"foo": "foo"}, commit="add foo")
|
||||
subrepo_dir = tmp_dir / "subdir"
|
||||
subrepo_dir.mkdir()
|
||||
with subrepo_dir.chdir():
|
||||
Repo.init(subdir=True)
|
||||
scm.add(["subdir"])
|
||||
scm.commit("Add subrepo")
|
||||
|
||||
dvc_fs = dvc.fs
|
||||
dvc_fs._reset()
|
||||
scm_fs = scm.get_fs("HEAD", use_dvcignore=True)
|
||||
path = os.fspath(tmp_dir)
|
||||
get_dirs = itemgetter(1)
|
||||
|
||||
assert get_dirs(next(dvc_fs.walk(path))) == []
|
||||
assert get_dirs(next(scm_fs.walk(path))) == []
|
||||
|
||||
kw = {"ignore_subrepos": False}
|
||||
assert get_dirs(next(dvc_fs.walk(path, **kw))) == ["subdir"]
|
||||
assert get_dirs(next(scm_fs.walk(path, **kw))) == ["subdir"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cloud",
|
||||
[
|
||||
pytest.lazy_fixture("local_cloud"),
|
||||
pytest.lazy_fixture("s3"),
|
||||
pytest.lazy_fixture("gs"),
|
||||
pytest.lazy_fixture("hdfs"),
|
||||
pytest.lazy_fixture("http"),
|
||||
],
|
||||
)
|
||||
def test_fs_getsize(dvc, cloud):
|
||||
cloud.gen({"data": {"foo": "foo"}, "baz": "baz baz"})
|
||||
fs = get_cloud_fs(dvc, **cloud.config)
|
||||
path_info = fs.path_info
|
||||
|
||||
assert fs.getsize(path_info / "baz") == 7
|
||||
assert fs.getsize(path_info / "data" / "foo") == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cloud",
|
||||
[
|
||||
pytest.lazy_fixture("azure"),
|
||||
pytest.lazy_fixture("gs"),
|
||||
pytest.lazy_fixture("gdrive"),
|
||||
pytest.lazy_fixture("hdfs"),
|
||||
pytest.lazy_fixture("http"),
|
||||
pytest.lazy_fixture("local_cloud"),
|
||||
pytest.lazy_fixture("oss"),
|
||||
pytest.lazy_fixture("s3"),
|
||||
pytest.lazy_fixture("ssh"),
|
||||
pytest.lazy_fixture("webhdfs"),
|
||||
],
|
||||
)
|
||||
def test_fs_upload_fobj(dvc, tmp_dir, cloud):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
fs = get_cloud_fs(dvc, **cloud.config)
|
||||
|
||||
from_info = tmp_dir / "foo"
|
||||
to_info = fs.path_info / "foo"
|
||||
|
||||
with open(from_info, "rb") as stream:
|
||||
fs.upload_fobj(stream, to_info)
|
||||
|
||||
assert fs.exists(to_info)
|
||||
with fs.open(to_info, "rb") as stream:
|
||||
assert stream.read() == b"foo"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("gdrive")])
|
||||
def test_fs_ls(dvc, cloud):
|
||||
cloud.gen(
|
||||
{
|
||||
"directory": {
|
||||
"foo": "foo",
|
||||
"bar": "bar",
|
||||
"baz": {"quux": "quux", "egg": {"foo": "foo"}},
|
||||
"empty": {},
|
||||
}
|
||||
}
|
||||
)
|
||||
fs = get_cloud_fs(dvc, **cloud.config)
|
||||
path_info = cloud / "directory"
|
||||
|
||||
assert {os.path.basename(file_key) for file_key in fs.ls(path_info)} == {
|
||||
"foo",
|
||||
"bar",
|
||||
"baz",
|
||||
"empty",
|
||||
}
|
||||
assert set(fs.ls(path_info / "empty")) == set()
|
||||
assert {
|
||||
(detail["type"], os.path.basename(detail["name"]))
|
||||
for detail in fs.ls(path_info / "baz", detail=True)
|
||||
} == {("file", "quux"), ("directory", "egg")}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cloud",
|
||||
[
|
||||
pytest.lazy_fixture("s3"),
|
||||
pytest.lazy_fixture("azure"),
|
||||
pytest.lazy_fixture("gs"),
|
||||
pytest.lazy_fixture("webdav"),
|
||||
pytest.lazy_fixture("gdrive"),
|
||||
],
|
||||
)
|
||||
def test_fs_ls_recursive(dvc, cloud):
|
||||
cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}})
|
||||
fs = get_cloud_fs(dvc, **cloud.config)
|
||||
path_info = fs.path_info
|
||||
|
||||
assert {
|
||||
os.path.basename(file_key)
|
||||
for file_key in fs.ls(path_info / "data", recursive=True)
|
||||
} == {"foo", "baz", "quux"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cloud",
|
||||
[
|
||||
pytest.lazy_fixture("s3"),
|
||||
pytest.lazy_fixture("azure"),
|
||||
pytest.lazy_fixture("gs"),
|
||||
pytest.lazy_fixture("webdav"),
|
||||
],
|
||||
)
|
||||
def test_fs_ls_with_etag(dvc, cloud):
|
||||
cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}})
|
||||
fs = get_cloud_fs(dvc, **cloud.config)
|
||||
path_info = fs.path_info
|
||||
|
||||
for details in fs.ls(path_info / "data", recursive=True, detail=True):
|
||||
assert (
|
||||
fs.info(path_info.replace(path=details["name"]))["etag"]
|
||||
== details["etag"]
|
||||
)
|
||||
|
||||
@@ -8,9 +8,9 @@ from git import Repo
|
||||
|
||||
from dvc.cache.local import LocalCache
|
||||
from dvc.exceptions import CollectCacheError
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
from dvc.main import main
|
||||
from dvc.repo import Repo as DvcRepo
|
||||
from dvc.tree.local import LocalTree
|
||||
from dvc.utils.fs import remove
|
||||
from tests.basic_env import TestDir, TestDvcGit
|
||||
|
||||
@@ -321,7 +321,9 @@ def test_gc_cloud_remove_order(tmp_dir, scm, dvc, tmp_path_factory, mocker):
|
||||
dvc.remove(dir2.relpath)
|
||||
dvc.gc(workspace=True)
|
||||
|
||||
mocked_remove = mocker.patch.object(LocalTree, "remove", autospec=True)
|
||||
mocked_remove = mocker.patch.object(
|
||||
LocalFileSystem, "remove", autospec=True
|
||||
)
|
||||
dvc.gc(workspace=True, cloud=True)
|
||||
assert len(mocked_remove.mock_calls) == 8
|
||||
# dir (and unpacked dir) should be first 4 checksums removed from
|
||||
|
||||
@@ -9,7 +9,7 @@ from dvc.repo import Repo
|
||||
from dvc.repo.get import GetDVCFileError
|
||||
from dvc.system import System
|
||||
from dvc.utils.fs import makedirs
|
||||
from tests.unit.tree.test_repo import make_subrepo
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
|
||||
|
||||
def test_get_repo_file(tmp_dir, erepo_dir):
|
||||
|
||||
+48
-50
@@ -23,41 +23,41 @@ def test_ignore(tmp_dir, dvc, monkeypatch):
|
||||
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
|
||||
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
|
||||
path = PathInfo(tmp_dir)
|
||||
|
||||
assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"}
|
||||
assert set(dvc.fs.walk_files(path / "dir")) == {path / "dir" / "other"}
|
||||
|
||||
|
||||
def test_ignore_unicode(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"other": "text", "тест": "проверка"}})
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест")
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
path = PathInfo(tmp_dir)
|
||||
assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"}
|
||||
assert set(dvc.fs.walk_files(path / "dir")) == {path / "dir" / "other"}
|
||||
|
||||
|
||||
def test_rename_ignored_file(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
|
||||
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*")
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
|
||||
mtime, size = get_mtime_and_size("dir", dvc.tree)
|
||||
mtime, size = get_mtime_and_size("dir", dvc.fs)
|
||||
|
||||
shutil.move("dir/ignored", "dir/ignored_new")
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.fs)
|
||||
|
||||
assert new_mtime == mtime and new_size == size
|
||||
|
||||
|
||||
def test_rename_file(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
mtime, size = get_mtime_and_size("dir", dvc.tree)
|
||||
mtime, size = get_mtime_and_size("dir", dvc.fs)
|
||||
|
||||
shutil.move("dir/foo", "dir/foo_new")
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.fs)
|
||||
|
||||
assert new_mtime != mtime and new_size == size
|
||||
|
||||
@@ -65,22 +65,22 @@ def test_rename_file(tmp_dir, dvc):
|
||||
def test_remove_ignored_file(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
|
||||
mtime, size = get_mtime_and_size("dir", dvc.tree)
|
||||
mtime, size = get_mtime_and_size("dir", dvc.fs)
|
||||
|
||||
os.remove("dir/ignored")
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.fs)
|
||||
|
||||
assert new_mtime == mtime and new_size == size
|
||||
|
||||
|
||||
def test_remove_file(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
mtime, size = get_mtime_and_size("dir", dvc.tree)
|
||||
mtime, size = get_mtime_and_size("dir", dvc.fs)
|
||||
|
||||
os.remove("dir/foo")
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.fs)
|
||||
|
||||
assert new_mtime != mtime and new_size != size
|
||||
|
||||
@@ -98,12 +98,12 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
|
||||
|
||||
top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE)
|
||||
top_ignore_file.write_text(os.path.basename(dname))
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
|
||||
ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE
|
||||
ignore_file.write_text("foo")
|
||||
|
||||
dvcignore = dvc.tree.dvcignore
|
||||
dvcignore = dvc.fs.dvcignore
|
||||
|
||||
top_ignore_path = os.path.dirname(os.fspath(top_ignore_file))
|
||||
|
||||
@@ -129,16 +129,16 @@ def test_ignore_on_branch(tmp_dir, scm, dvc):
|
||||
with tmp_dir.branch("branch", new=True):
|
||||
tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
|
||||
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
path = PathInfo(tmp_dir)
|
||||
assert set(dvc.tree.walk_files(path)) == {
|
||||
assert set(dvc.fs.walk_files(path)) == {
|
||||
path / "foo",
|
||||
path / "bar",
|
||||
path / ".dvcignore",
|
||||
}
|
||||
|
||||
dvc.tree = scm.get_tree("branch", use_dvcignore=True)
|
||||
assert set(dvc.tree.walk_files(path)) == {
|
||||
dvc.fs = scm.get_fs("branch", use_dvcignore=True)
|
||||
assert set(dvc.fs.walk_files(path)) == {
|
||||
os.fspath(path / DvcIgnore.DVCIGNORE_FILE),
|
||||
os.fspath(path / "bar"),
|
||||
}
|
||||
@@ -153,8 +153,8 @@ def test_match_nested(tmp_dir, dvc):
|
||||
"dir": {"x.backup": "x backup", "tmp": "content"},
|
||||
}
|
||||
)
|
||||
dvc.tree._reset()
|
||||
result = {os.fspath(os.path.normpath(f)) for f in dvc.tree.walk_files(".")}
|
||||
dvc.fs._reset()
|
||||
result = {os.fspath(os.path.normpath(f)) for f in dvc.fs.walk_files(".")}
|
||||
assert result == {".dvcignore", "foo"}
|
||||
|
||||
|
||||
@@ -163,13 +163,11 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
|
||||
ext_dir = TmpDir(os.fspath(tmp_path_factory.mktemp("external_dir")))
|
||||
ext_dir.gen({"y.backup": "y", "tmp": {"file": "ext tmp"}})
|
||||
|
||||
result = {relpath(f, ext_dir) for f in dvc.tree.walk_files(ext_dir)}
|
||||
result = {relpath(f, ext_dir) for f in dvc.fs.walk_files(ext_dir)}
|
||||
assert result == {"y.backup", os.path.join("tmp", "file")}
|
||||
assert dvc.fs.dvcignore.is_ignored_dir(os.fspath(ext_dir / "tmp")) is False
|
||||
assert (
|
||||
dvc.tree.dvcignore.is_ignored_dir(os.fspath(ext_dir / "tmp")) is False
|
||||
)
|
||||
assert (
|
||||
dvc.tree.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup"))
|
||||
dvc.fs.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup"))
|
||||
is False
|
||||
)
|
||||
|
||||
@@ -178,10 +176,10 @@ def test_ignore_subrepo(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen({".dvcignore": "foo", "subdir": {"foo": "foo"}})
|
||||
scm.add([".dvcignore"])
|
||||
scm.commit("init parent dvcignore")
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
|
||||
subrepo_dir = tmp_dir / "subdir"
|
||||
assert not dvc.tree.exists(PathInfo(subrepo_dir / "foo"))
|
||||
assert not dvc.fs.exists(PathInfo(subrepo_dir / "foo"))
|
||||
|
||||
with subrepo_dir.chdir():
|
||||
subrepo = Repo.init(subdir=True)
|
||||
@@ -189,7 +187,7 @@ def test_ignore_subrepo(tmp_dir, scm, dvc):
|
||||
scm.commit("subrepo init")
|
||||
|
||||
for _ in subrepo.brancher(all_commits=True):
|
||||
assert subrepo.tree.exists(PathInfo(subrepo_dir / "foo"))
|
||||
assert subrepo.fs.exists(PathInfo(subrepo_dir / "foo"))
|
||||
|
||||
|
||||
def test_ignore_resurface_subrepo(tmp_dir, scm, dvc):
|
||||
@@ -199,17 +197,17 @@ def test_ignore_resurface_subrepo(tmp_dir, scm, dvc):
|
||||
with subrepo_dir.chdir():
|
||||
Repo.init(subdir=True)
|
||||
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
|
||||
dirs = ["subdir"]
|
||||
files = ["foo"]
|
||||
assert dvc.tree.dvcignore(os.fspath(tmp_dir), dirs, files) == ([], files)
|
||||
assert dvc.tree.dvcignore(
|
||||
assert dvc.fs.dvcignore(os.fspath(tmp_dir), dirs, files) == ([], files)
|
||||
assert dvc.fs.dvcignore(
|
||||
os.fspath(tmp_dir), dirs, files, ignore_subrepos=False
|
||||
) == (dirs, files)
|
||||
|
||||
assert dvc.tree.dvcignore.is_ignored_dir(os.fspath(subrepo_dir))
|
||||
assert not dvc.tree.dvcignore.is_ignored_dir(
|
||||
assert dvc.fs.dvcignore.is_ignored_dir(os.fspath(subrepo_dir))
|
||||
assert not dvc.fs.dvcignore.is_ignored_dir(
|
||||
os.fspath(subrepo_dir), ignore_subrepos=False
|
||||
)
|
||||
|
||||
@@ -217,9 +215,9 @@ def test_ignore_resurface_subrepo(tmp_dir, scm, dvc):
|
||||
def test_ignore_blank_line(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "foo\n\ndir/ignored")
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
path = PathInfo(tmp_dir)
|
||||
assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"}
|
||||
assert set(dvc.fs.walk_files(path / "dir")) == {path / "dir" / "other"}
|
||||
|
||||
|
||||
# It is not possible to re-include a file if a parent directory of
|
||||
@@ -252,9 +250,9 @@ def test_ignore_file_in_parent_path(
|
||||
):
|
||||
tmp_dir.gen(data_struct)
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "\n".join(pattern_list))
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
path = PathInfo(tmp_dir)
|
||||
assert set(dvc.tree.walk_files(path / "dir")) == {
|
||||
assert set(dvc.fs.walk_files(path / "dir")) == {
|
||||
path / relpath for relpath in result_set
|
||||
}
|
||||
|
||||
@@ -275,9 +273,9 @@ def test_ignore_sub_directory(tmp_dir, dvc):
|
||||
)
|
||||
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "doc/fortz"}})
|
||||
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
path = PathInfo(tmp_dir)
|
||||
assert set(dvc.tree.walk_files(path / "dir")) == {
|
||||
assert set(dvc.fs.walk_files(path / "dir")) == {
|
||||
path / "dir" / "a" / "doc" / "fortz" / "a",
|
||||
path / "dir" / DvcIgnore.DVCIGNORE_FILE,
|
||||
}
|
||||
@@ -287,9 +285,9 @@ def test_ignore_sub_directory(tmp_dir, dvc):
|
||||
def test_ignore_directory(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"fortz": {}, "a": {"fortz": {}}}})
|
||||
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "fortz"}})
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
path = PathInfo(tmp_dir)
|
||||
assert set(dvc.tree.walk_files(path / "dir")) == {
|
||||
assert set(dvc.fs.walk_files(path / "dir")) == {
|
||||
path / "dir" / DvcIgnore.DVCIGNORE_FILE,
|
||||
}
|
||||
|
||||
@@ -298,15 +296,15 @@ def test_multi_ignore_file(tmp_dir, dvc, monkeypatch):
|
||||
tmp_dir.gen({"dir": {"subdir": {"should_ignore": "1", "not_ignore": "1"}}})
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/subdir/*_ignore")
|
||||
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "!subdir/not_ignore"}})
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
path = PathInfo(tmp_dir)
|
||||
assert set(dvc.tree.walk_files(path / "dir")) == {
|
||||
assert set(dvc.fs.walk_files(path / "dir")) == {
|
||||
path / "dir" / "subdir" / "not_ignore",
|
||||
path / "dir" / DvcIgnore.DVCIGNORE_FILE,
|
||||
}
|
||||
|
||||
|
||||
def test_pattern_trie_tree(tmp_dir, dvc):
|
||||
def test_pattern_trie_fs(tmp_dir, dvc):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
"top": {
|
||||
@@ -323,8 +321,8 @@ def test_pattern_trie_tree(tmp_dir, dvc):
|
||||
"other": {DvcIgnore.DVCIGNORE_FILE: "1\n2\n3"},
|
||||
}
|
||||
)
|
||||
dvc.tree._reset()
|
||||
dvcignore = dvc.tree.dvcignore
|
||||
dvc.fs._reset()
|
||||
dvcignore = dvc.fs.dvcignore
|
||||
|
||||
ignore_pattern_top = dvcignore._get_trie_pattern(
|
||||
os.fspath(tmp_dir / "top")
|
||||
@@ -391,10 +389,10 @@ def test_ignore_in_added_dir(tmp_dir, dvc):
|
||||
".dvcignore": "**/ignored",
|
||||
}
|
||||
)
|
||||
dvc.tree._reset()
|
||||
dvc.fs._reset()
|
||||
|
||||
ignored_path = tmp_dir / "dir" / "sub" / "ignored"
|
||||
assert not dvc.tree.exists(PathInfo(ignored_path))
|
||||
assert not dvc.fs.exists(PathInfo(ignored_path))
|
||||
assert ignored_path.exists()
|
||||
|
||||
dvc.add("dir")
|
||||
|
||||
@@ -13,7 +13,7 @@ from dvc.exceptions import DownloadError
|
||||
from dvc.stage.exceptions import StagePathNotFoundError
|
||||
from dvc.system import System
|
||||
from dvc.utils.fs import makedirs, remove
|
||||
from tests.unit.tree.test_repo import make_subrepo
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
|
||||
|
||||
def test_import(tmp_dir, scm, dvc, erepo_dir):
|
||||
@@ -305,7 +305,7 @@ def test_download_error_pulling_imported_stage(tmp_dir, dvc, erepo_dir):
|
||||
remove(dst_cache)
|
||||
|
||||
with patch(
|
||||
"dvc.tree.local.LocalTree._download", side_effect=Exception
|
||||
"dvc.fs.local.LocalFileSystem._download", side_effect=Exception
|
||||
), pytest.raises(DownloadError):
|
||||
dvc.pull(["foo_imported.dvc"])
|
||||
|
||||
|
||||
@@ -64,7 +64,7 @@ class TestInstall:
|
||||
conf["core"]["remote"] = "store"
|
||||
tmp_dir.dvc_gen("file", "file_content", "commit message")
|
||||
|
||||
file_checksum = file_md5("file", dvc.tree)
|
||||
file_checksum = file_md5("file", dvc.fs)
|
||||
expected_storage_path = (
|
||||
storage_path / file_checksum[:2] / file_checksum[2:]
|
||||
)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user