diff --git a/dvc/api/__init__.py b/dvc/api/__init__.py index 53ce86ccd..ccb88fed6 100644 --- a/dvc/api/__init__.py +++ b/dvc/api/__init__.py @@ -23,13 +23,13 @@ def get_url(path, repo=None, rev=None, remote=None): with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo: path_info = PathInfo(_repo.root_dir) / path with reraise(FileNotFoundError, PathMissingError(path, repo)): - metadata = _repo.repo_tree.metadata(path_info) + metadata = _repo.repo_fs.metadata(path_info) if not metadata.is_dvc: raise OutputNotFoundError(path, repo) cloud = metadata.repo.cloud - hash_info = get_hash(path_info, _repo.repo_tree, "md5") + hash_info = get_hash(path_info, _repo.repo_fs, "md5") return cloud.get_url_for(remote, checksum=hash_info.value) diff --git a/dvc/cache/__init__.py b/dvc/cache/__init__.py index 512778fa9..7238a2378 100644 --- a/dvc/cache/__init__.py +++ b/dvc/cache/__init__.py @@ -4,32 +4,32 @@ from collections import defaultdict from ..scheme import Schemes -def get_cloud_cache(tree): +def get_cloud_cache(fs): from .base import CloudCache from .gdrive import GDriveCache from .local import LocalCache from .ssh import SSHCache - if tree.scheme == Schemes.LOCAL: - return LocalCache(tree) + if fs.scheme == Schemes.LOCAL: + return LocalCache(fs) - if tree.scheme == Schemes.SSH: - return SSHCache(tree) + if fs.scheme == Schemes.SSH: + return SSHCache(fs) - if tree.scheme == Schemes.GDRIVE: - return GDriveCache(tree) + if fs.scheme == Schemes.GDRIVE: + return GDriveCache(fs) - return CloudCache(tree) + return CloudCache(fs) def _get_cache(repo, settings): - from ..tree import get_cloud_tree + from ..fs import get_cloud_fs if not settings: return None - tree = get_cloud_tree(repo, **settings) - return get_cloud_cache(tree) + fs = get_cloud_fs(repo, **settings) + return get_cloud_cache(fs) class Cache: diff --git a/dvc/cache/base.py b/dvc/cache/base.py index 9b5eb99de..cceee2a13 100644 --- a/dvc/cache/base.py +++ b/dvc/cache/base.py @@ -17,32 +17,32 @@ class CloudCache: DEFAULT_CACHE_TYPES = ["copy"] CACHE_MODE: Optional[int] = None - def __init__(self, tree): - self.tree = tree - self.repo = tree.repo + def __init__(self, fs): + self.fs = fs + self.repo = fs.repo - self.verify = tree.config.get("verify", self.DEFAULT_VERIFY) - self.cache_types = tree.config.get("type") or copy( + self.verify = fs.config.get("verify", self.DEFAULT_VERIFY) + self.cache_types = fs.config.get("type") or copy( self.DEFAULT_CACHE_TYPES ) self.cache_type_confirmed = False def move(self, from_info, to_info): - self.tree.move(from_info, to_info) + self.fs.move(from_info, to_info) def makedirs(self, path_info): - self.tree.makedirs(path_info) + self.fs.makedirs(path_info) def get(self, hash_info): """ get raw object """ return HashFile( # Prefer string path over PathInfo when possible due to performance self.hash_to_path(hash_info.value), - self.tree, + self.fs, hash_info, ) - def add(self, path_info, tree, hash_info, **kwargs): + def add(self, path_info, fs, hash_info, **kwargs): try: self.check(hash_info) return @@ -52,21 +52,21 @@ class CloudCache: cache_info = self.hash_to_path_info(hash_info.value) # using our makedirs to create dirs with proper permissions self.makedirs(cache_info.parent) - if isinstance(tree, type(self.tree)): - self.tree.move(path_info, cache_info) + if isinstance(fs, type(self.fs)): + self.fs.move(path_info, cache_info) else: - with tree.open(path_info, mode="rb") as fobj: - self.tree.upload_fobj(fobj, cache_info) + with fs.open(path_info, mode="rb") as fobj: + self.fs.upload_fobj(fobj, cache_info) self.protect(cache_info) - with self.tree.state: - self.tree.state.save(cache_info, hash_info) + with self.fs.state: + self.fs.state.save(cache_info, hash_info) callback = kwargs.get("download_callback") if callback: callback(1) def hash_to_path_info(self, hash_): - return self.tree.path_info / hash_[0:2] / hash_[2:] + return self.fs.path_info / hash_[0:2] / hash_[2:] # Override to return path as a string instead of PathInfo for clouds # which support string paths (see local) @@ -110,7 +110,7 @@ class CloudCache: obj.check(self) except ObjectFormatError: logger.warning("corrupted cache file '%s'.", obj.path_info) - self.tree.remove(obj.path_info) + self.fs.remove(obj.path_info) raise # making cache file read-only so we don't need to check it @@ -120,22 +120,22 @@ class CloudCache: def _list_paths(self, prefix=None, progress_callback=None): if prefix: if len(prefix) > 2: - path_info = self.tree.path_info / prefix[:2] / prefix[2:] + path_info = self.fs.path_info / prefix[:2] / prefix[2:] else: - path_info = self.tree.path_info / prefix[:2] + path_info = self.fs.path_info / prefix[:2] prefix = True else: - path_info = self.tree.path_info + path_info = self.fs.path_info prefix = False if progress_callback: - for file_info in self.tree.walk_files(path_info, prefix=prefix): + for file_info in self.fs.walk_files(path_info, prefix=prefix): progress_callback() yield file_info.path else: - yield from self.tree.walk_files(path_info, prefix=prefix) + yield from self.fs.walk_files(path_info, prefix=prefix) def _path_to_hash(self, path): - parts = self.tree.PATH_CLS(path).parts[-2:] + parts = self.fs.PATH_CLS(path).parts[-2:] if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2): raise ValueError(f"Bad cache file path '{path}'") @@ -143,7 +143,7 @@ class CloudCache: return "".join(parts) def list_hashes(self, prefix=None, progress_callback=None): - """Iterate over hashes in this tree. + """Iterate over hashes in this fs. If `prefix` is specified, only hashes which begin with `prefix` will be returned. @@ -171,18 +171,18 @@ class CloudCache: def _max_estimation_size(self, hashes): # Max remote size allowed for us to use traverse method return max( - self.tree.TRAVERSE_THRESHOLD_SIZE, + self.fs.TRAVERSE_THRESHOLD_SIZE, len(hashes) - / self.tree.TRAVERSE_WEIGHT_MULTIPLIER - * self.tree.LIST_OBJECT_PAGE_SIZE, + / self.fs.TRAVERSE_WEIGHT_MULTIPLIER + * self.fs.LIST_OBJECT_PAGE_SIZE, ) def _estimate_remote_size(self, hashes=None, name=None): - """Estimate tree size based on number of entries beginning with + """Estimate fs size based on number of entries beginning with "00..." prefix. """ - prefix = "0" * self.tree.TRAVERSE_PREFIX_LEN - total_prefixes = pow(16, self.tree.TRAVERSE_PREFIX_LEN) + prefix = "0" * self.fs.TRAVERSE_PREFIX_LEN + total_prefixes = pow(16, self.fs.TRAVERSE_PREFIX_LEN) if hashes: max_hashes = self._max_estimation_size(hashes) else: @@ -215,7 +215,7 @@ class CloudCache: def list_hashes_traverse( self, remote_size, remote_hashes, jobs=None, name=None ): - """Iterate over all hashes found in this tree. + """Iterate over all hashes found in this fs. Hashes are fetched in parallel according to prefix, except in cases where the remote size is very small. @@ -229,8 +229,8 @@ class CloudCache: sense to use a generator to gradually iterate over it, without keeping all of it in memory. """ - num_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE - if num_pages < 256 / self.tree.JOBS: + num_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE + if num_pages < 256 / self.fs.JOBS: # Fetching prefixes in parallel requires at least 255 more # requests, for small enough remotes it will be faster to fetch # entire cache without splitting it into prefixes. @@ -243,12 +243,10 @@ class CloudCache: yield from remote_hashes initial = len(remote_hashes) traverse_prefixes = [f"{i:02x}" for i in range(1, 256)] - if self.tree.TRAVERSE_PREFIX_LEN > 2: + if self.fs.TRAVERSE_PREFIX_LEN > 2: traverse_prefixes += [ - "{0:0{1}x}".format(i, self.tree.TRAVERSE_PREFIX_LEN) - for i in range( - 1, pow(16, self.tree.TRAVERSE_PREFIX_LEN - 2) - ) + "{0:0{1}x}".format(i, self.fs.TRAVERSE_PREFIX_LEN) + for i in range(1, pow(16, self.fs.TRAVERSE_PREFIX_LEN - 2)) ] with Tqdm( desc="Querying " @@ -266,13 +264,13 @@ class CloudCache: ) with ThreadPoolExecutor( - max_workers=jobs or self.tree.JOBS + max_workers=jobs or self.fs.JOBS ) as executor: in_remote = executor.map(list_with_update, traverse_prefixes,) yield from itertools.chain.from_iterable(in_remote) def all(self, jobs=None, name=None): - """Iterate over all hashes in this tree. + """Iterate over all hashes in this fs. Hashes will be fetched in parallel threads according to prefix (except for small remotes) and a progress bar will be displayed. @@ -283,7 +281,7 @@ class CloudCache: ) ) - if not self.tree.CAN_TRAVERSE: + if not self.fs.CAN_TRAVERSE: return self.list_hashes() remote_size, remote_hashes = self._estimate_remote_size(name=name) @@ -298,24 +296,24 @@ class CloudCache: removed = False # hashes must be sorted to ensure we always remove .dir files first for hash_ in sorted( - self.all(jobs, str(self.tree.path_info)), - key=self.tree.is_dir_hash, + self.all(jobs, str(self.fs.path_info)), + key=self.fs.is_dir_hash, reverse=True, ): if hash_ in used: continue path_info = self.hash_to_path_info(hash_) - if self.tree.is_dir_hash(hash_): + if self.fs.is_dir_hash(hash_): # backward compatibility # pylint: disable=protected-access self._remove_unpacked_dir(hash_) - self.tree.remove(path_info) + self.fs.remove(path_info) removed = True return removed def list_hashes_exists(self, hashes, jobs=None, name=None): - """Return list of the specified hashes which exist in this tree. + """Return list of the specified hashes which exist in this fs. Hashes will be queried individually. """ logger.debug( @@ -329,12 +327,12 @@ class CloudCache: ) as pbar: def exists_with_progress(path_info): - ret = self.tree.exists(path_info) + ret = self.fs.exists(path_info) pbar.update_msg(str(path_info)) return ret with ThreadPoolExecutor( - max_workers=jobs or self.tree.JOBS + max_workers=jobs or self.fs.JOBS ) as executor: path_infos = map(self.hash_to_path_info, hashes) in_remote = executor.map(exists_with_progress, path_infos) @@ -376,24 +374,24 @@ class CloudCache: """ # Remotes which do not use traverse prefix should override # hashes_exist() (see ssh, local) - assert self.tree.TRAVERSE_PREFIX_LEN >= 2 + assert self.fs.TRAVERSE_PREFIX_LEN >= 2 hashes = set(hashes) - if len(hashes) == 1 or not self.tree.CAN_TRAVERSE: + if len(hashes) == 1 or not self.fs.CAN_TRAVERSE: remote_hashes = self.list_hashes_exists(hashes, jobs, name) return remote_hashes # Max remote size allowed for us to use traverse method remote_size, remote_hashes = self._estimate_remote_size(hashes, name) - traverse_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE + traverse_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE # For sufficiently large remotes, traverse must be weighted to account # for performance overhead from large lists/sets. # From testing with S3, for remotes with 1M+ files, object_exists is # faster until len(hashes) is at least 10k~100k - if remote_size > self.tree.TRAVERSE_THRESHOLD_SIZE: + if remote_size > self.fs.TRAVERSE_THRESHOLD_SIZE: traverse_weight = ( - traverse_pages * self.tree.TRAVERSE_WEIGHT_MULTIPLIER + traverse_pages * self.fs.TRAVERSE_WEIGHT_MULTIPLIER ) else: traverse_weight = traverse_pages diff --git a/dvc/cache/local.py b/dvc/cache/local.py index 1e94e1c7b..4c60b602e 100644 --- a/dvc/cache/local.py +++ b/dvc/cache/local.py @@ -22,11 +22,11 @@ class LocalCache(CloudCache): CACHE_MODE = 0o444 UNPACKED_DIR_SUFFIX = ".unpacked" - def __init__(self, tree): - super().__init__(tree) - self.cache_dir = tree.config.get("url") + def __init__(self, fs): + super().__init__(fs) + self.cache_dir = fs.config.get("url") - shared = tree.config.get("shared") + shared = fs.config.get("shared") if shared: self._file_mode = 0o664 self._dir_mode = 0o2775 @@ -36,11 +36,11 @@ class LocalCache(CloudCache): @property def cache_dir(self): - return self.tree.path_info.fspath if self.tree.path_info else None + return self.fs.path_info.fspath if self.fs.path_info else None @cache_dir.setter def cache_dir(self, value): - self.tree.path_info = PathInfo(value) if value else None + self.fs.path_info = PathInfo(value) if value else None @cached_property def cache_path(self): @@ -71,7 +71,7 @@ class LocalCache(CloudCache): unit="file", desc="Querying " + ("cache in " + name if name else "local cache"), ): - hash_info = HashInfo(self.tree.PARAM_CHECKSUM, hash_) + hash_info = HashInfo(self.fs.PARAM_CHECKSUM, hash_) try: self.check(hash_info) ret.append(hash_) @@ -81,14 +81,14 @@ class LocalCache(CloudCache): return ret def _list_paths(self, prefix=None, progress_callback=None): - assert self.tree.path_info is not None + assert self.fs.path_info is not None if prefix: - path_info = self.tree.path_info / prefix[:2] - if not self.tree.exists(path_info): + path_info = self.fs.path_info / prefix[:2] + if not self.fs.exists(path_info): return else: - path_info = self.tree.path_info - # NOTE: use utils.fs walk_files since tree.walk_files will not follow + path_info = self.fs.path_info + # NOTE: use utils.fs walk_files since fs.walk_files will not follow # symlinks if progress_callback: for path in walk_files(path_info): @@ -100,10 +100,10 @@ class LocalCache(CloudCache): def _remove_unpacked_dir(self, hash_): info = self.hash_to_path_info(hash_) path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX) - self.tree.remove(path_info) + self.fs.remove(path_info) def _unprotect_file(self, path): - if self.tree.is_symlink(path) or self.tree.is_hardlink(path): + if self.fs.is_symlink(path) or self.fs.is_hardlink(path): logger.debug("Unprotecting '%s'", path) tmp = os.path.join(os.path.dirname(path), "." + uuid()) @@ -126,7 +126,7 @@ class LocalCache(CloudCache): os.chmod(path, self._file_mode) def _unprotect_dir(self, path): - for fname in self.tree.walk_files(path): + for fname in self.fs.walk_files(path): self._unprotect_file(fname) def unprotect(self, path_info): diff --git a/dvc/cache/ssh.py b/dvc/cache/ssh.py index b7aef729e..2033df0c8 100644 --- a/dvc/cache/ssh.py +++ b/dvc/cache/ssh.py @@ -28,7 +28,7 @@ class SSHCache(CloudCache): callback(path) return ret - with self.tree.ssh(path_infos[0]) as ssh: + with self.fs.ssh(path_infos[0]) as ssh: channels = ssh.open_max_sftp_channels() max_workers = len(channels) @@ -47,11 +47,11 @@ class SSHCache(CloudCache): faster than current approach (relying on exists(path_info)) applied in remote/base. """ - if not self.tree.CAN_TRAVERSE: + if not self.fs.CAN_TRAVERSE: return list(set(hashes) & set(self.all())) # possibly prompt for credentials before "Querying" progress output - self.tree.ensure_credentials() + self.fs.ensure_credentials() with Tqdm( desc="Querying " @@ -64,10 +64,10 @@ class SSHCache(CloudCache): return self.batch_exists(chunks, callback=pbar.update_msg) with ThreadPoolExecutor( - max_workers=jobs or self.tree.JOBS + max_workers=jobs or self.fs.JOBS ) as executor: path_infos = [self.hash_to_path_info(x) for x in hashes] - chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS) + chunks = to_chunks(path_infos, num_chunks=self.fs.JOBS) results = executor.map(exists_with_progress, chunks) in_remote = itertools.chain.from_iterable(results) ret = list(itertools.compress(hashes, in_remote)) @@ -75,10 +75,10 @@ class SSHCache(CloudCache): def _list_paths(self, prefix=None, progress_callback=None): if prefix: - root = posixpath.join(self.tree.path_info.path, prefix[:2]) + root = posixpath.join(self.fs.path_info.path, prefix[:2]) else: - root = self.tree.path_info.path - with self.tree.ssh(self.tree.path_info) as ssh: + root = self.fs.path_info.path + with self.fs.ssh(self.fs.path_info) as ssh: if prefix and not ssh.exists(root): return # If we simply return an iterator then with above closes instantly diff --git a/dvc/checkout.py b/dvc/checkout.py index 47826c239..ec98bacf4 100644 --- a/dvc/checkout.py +++ b/dvc/checkout.py @@ -18,7 +18,7 @@ from dvc.remote.slow_link_detection import ( # type: ignore[attr-defined] logger = logging.getLogger(__name__) -def _changed(path_info, tree, obj, cache): +def _changed(path_info, fs, obj, cache): logger.trace("checking if '%s'('%s') has changed.", path_info, obj) try: @@ -30,7 +30,7 @@ def _changed(path_info, tree, obj, cache): return True try: - actual = get_hash(path_info, tree, obj.hash_info.name) + actual = get_hash(path_info, fs, obj.hash_info.name) except FileNotFoundError: logger.debug("'%s' doesn't exist.", path_info) return True @@ -48,15 +48,15 @@ def _changed(path_info, tree, obj, cache): return False -def _remove(path_info, tree, cache, force=False): - if not tree.exists(path_info): +def _remove(path_info, fs, cache, force=False): + if not fs.exists(path_info): return if force: - tree.remove(path_info) + fs.remove(path_info) return - current = get_hash(path_info, tree, tree.PARAM_CHECKSUM) + current = get_hash(path_info, fs, fs.PARAM_CHECKSUM) try: obj = load(cache, current) check(cache, obj) @@ -69,26 +69,26 @@ def _remove(path_info, tree, cache, force=False): if not prompt.confirm(msg): raise ConfirmRemoveError(str(path_info)) - tree.remove(path_info) + fs.remove(path_info) def _verify_link(cache, path_info, link_type): - if link_type == "hardlink" and cache.tree.getsize(path_info) == 0: + if link_type == "hardlink" and cache.fs.getsize(path_info) == 0: return if cache.cache_type_confirmed: return - is_link = getattr(cache.tree, f"is_{link_type}", None) + is_link = getattr(cache.fs, f"is_{link_type}", None) if is_link and not is_link(path_info): - cache.tree.remove(path_info) + cache.fs.remove(path_info) raise DvcException(f"failed to verify {link_type}") cache.cache_type_confirmed = True def _do_link(cache, from_info, to_info, link_method): - if cache.tree.exists(to_info): + if cache.fs.exists(to_info): raise DvcException(f"Link '{to_info}' already exists!") link_method(from_info, to_info) @@ -101,7 +101,7 @@ def _do_link(cache, from_info, to_info, link_method): @slow_link_guard def _try_links(cache, from_info, to_info, link_types): while link_types: - link_method = getattr(cache.tree, link_types[0]) + link_method = getattr(cache.fs, link_types[0]) try: _do_link(cache, from_info, to_info, link_method) _verify_link(cache, to_info, link_types[0]) @@ -117,7 +117,7 @@ def _try_links(cache, from_info, to_info, link_types): def _link(cache, from_info, to_info): - assert cache.tree.isfile(from_info) + assert cache.fs.isfile(from_info) cache.makedirs(to_info.parent) _try_links(cache, from_info, to_info, cache.cache_types) @@ -131,76 +131,76 @@ def _cache_is_copy(cache, path_info): return True workspace_file = path_info.with_name("." + uuid()) - test_cache_file = cache.tree.path_info / ".cache_type_test_file" - if not cache.tree.exists(test_cache_file): + test_cache_file = cache.fs.path_info / ".cache_type_test_file" + if not cache.fs.exists(test_cache_file): cache.makedirs(test_cache_file.parent) - with cache.tree.open(test_cache_file, "wb") as fobj: + with cache.fs.open(test_cache_file, "wb") as fobj: fobj.write(bytes(1)) try: _link(cache, test_cache_file, workspace_file) finally: - cache.tree.remove(workspace_file) - cache.tree.remove(test_cache_file) + cache.fs.remove(workspace_file) + cache.fs.remove(test_cache_file) cache.cache_type_confirmed = True return cache.cache_types[0] == "copy" def _checkout_file( - path_info, tree, obj, cache, force, progress_callback=None, relink=False, + path_info, fs, obj, cache, force, progress_callback=None, relink=False, ): """The file is changed we need to checkout a new copy""" modified = False cache_info = cache.hash_to_path_info(obj.hash_info.value) - if tree.exists(path_info): - if not relink and _changed(path_info, tree, obj, cache): + if fs.exists(path_info): + if not relink and _changed(path_info, fs, obj, cache): modified = True - _remove(path_info, tree, cache, force=force) + _remove(path_info, fs, cache, force=force) _link(cache, cache_info, path_info) else: - if tree.iscopy(path_info) and _cache_is_copy(cache, path_info): + if fs.iscopy(path_info) and _cache_is_copy(cache, path_info): cache.unprotect(path_info) else: - _remove(path_info, tree, cache, force=force) + _remove(path_info, fs, cache, force=force) _link(cache, cache_info, path_info) else: _link(cache, cache_info, path_info) modified = True - tree.state.save(path_info, obj.hash_info) + fs.state.save(path_info, obj.hash_info) if progress_callback: progress_callback(str(path_info)) return modified -def _remove_redundant_files(path_info, tree, dir_info, cache, force): - existing_files = set(tree.walk_files(path_info)) +def _remove_redundant_files(path_info, fs, dir_info, cache, force): + existing_files = set(fs.walk_files(path_info)) needed_files = {info for info, _ in dir_info.items(path_info)} redundant_files = existing_files - needed_files for path in redundant_files: - _remove(path, tree, cache, force) + _remove(path, fs, cache, force) return bool(redundant_files) def _checkout_dir( - path_info, tree, obj, cache, force, progress_callback=None, relink=False, + path_info, fs, obj, cache, force, progress_callback=None, relink=False, ): modified = False, False # Create dir separately so that dir is created # even if there are no files in it - if not tree.exists(path_info): + if not fs.exists(path_info): modified = True - tree.makedirs(path_info) + fs.makedirs(path_info) logger.debug("Linking directory '%s'.", path_info) for entry_info, entry_hash_info in obj.hash_info.dir_info.items(path_info): entry_modified = _checkout_file( entry_info, - tree, + fs, cache.get(entry_hash_info), cache, force, @@ -212,12 +212,12 @@ def _checkout_dir( modified = ( _remove_redundant_files( - path_info, tree, obj.hash_info.dir_info, cache, force + path_info, fs, obj.hash_info.dir_info, cache, force ) or modified ) - tree.state.save(path_info, obj.hash_info) + fs.state.save(path_info, obj.hash_info) # relink is not modified, assume it as nochange return modified and not relink @@ -225,7 +225,7 @@ def _checkout_dir( def _checkout( path_info, - tree, + fs, obj, cache, force=False, @@ -234,21 +234,21 @@ def _checkout( ): if not obj.hash_info.isdir: ret = _checkout_file( - path_info, tree, obj, cache, force, progress_callback, relink + path_info, fs, obj, cache, force, progress_callback, relink ) else: ret = _checkout_dir( - path_info, tree, obj, cache, force, progress_callback, relink, + path_info, fs, obj, cache, force, progress_callback, relink, ) - tree.state.save_link(path_info) + fs.state.save_link(path_info) return ret def checkout( path_info, - tree, + fs, obj, cache, force=False, @@ -256,7 +256,7 @@ def checkout( relink=False, quiet=False, ): - if path_info.scheme not in ["local", cache.tree.scheme]: + if path_info.scheme not in ["local", cache.fs.scheme]: raise NotImplementedError failed = None @@ -267,10 +267,10 @@ def checkout( "No file hash info found for '%s'. It won't be created.", path_info, ) - _remove(path_info, tree, cache, force=force) + _remove(path_info, fs, cache, force=force) failed = path_info - elif not relink and not _changed(path_info, tree, obj, cache): + elif not relink and not _changed(path_info, fs, obj, cache): logger.trace("Data '%s' didn't change.", path_info) skip = True else: @@ -283,7 +283,7 @@ def checkout( obj.hash_info, path_info, ) - _remove(path_info, tree, cache, force=force) + _remove(path_info, fs, cache, force=force) failed = path_info if failed or skip: @@ -298,5 +298,5 @@ def checkout( logger.debug("Checking out '%s' with cache '%s'.", path_info, obj) return _checkout( - path_info, tree, obj, cache, force, progress_callback, relink, + path_info, fs, obj, cache, force, progress_callback, relink, ) diff --git a/dvc/command/check_ignore.py b/dvc/command/check_ignore.py index e1a1bf5ac..f96e46abc 100644 --- a/dvc/command/check_ignore.py +++ b/dvc/command/check_ignore.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) class CmdCheckIgnore(CmdBase): def __init__(self, args): super().__init__(args) - self.ignore_filter = self.repo.tree.dvcignore + self.ignore_filter = self.repo.fs.dvcignore def _show_results(self, result): if not result.match and not self.args.non_matching: diff --git a/dvc/config.py b/dvc/config.py index 72c2a7069..4acc245ae 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -63,9 +63,9 @@ class Config(dict): CONFIG_LOCAL = "config.local" def __init__( - self, dvc_dir=None, validate=True, tree=None, config=None, + self, dvc_dir=None, validate=True, fs=None, config=None, ): # pylint: disable=super-init-not-called - from dvc.tree.local import LocalTree + from dvc.fs.local import LocalFileSystem self.dvc_dir = dvc_dir @@ -79,8 +79,8 @@ class Config(dict): else: self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) - self.wtree = LocalTree(None, {"url": self.dvc_dir}) - self.tree = tree or self.wtree + self.wfs = LocalFileSystem(None, {"url": self.dvc_dir}) + self.fs = fs or self.wfs self.load(validate=validate, config=config) @@ -143,19 +143,19 @@ class Config(dict): if not self["cache"].get("dir") and self.dvc_dir: self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache") - def _get_tree(self, level): - # NOTE: this might be a GitTree, which doesn't see things outside of + def _get_fs(self, level): + # NOTE: this might be a Gitfs, which doesn't see things outside of # the repo. - return self.tree if level == "repo" else self.wtree + return self.fs if level == "repo" else self.wfs def _load_config(self, level): from configobj import ConfigObj filename = self.files[level] - tree = self._get_tree(level) + fs = self._get_fs(level) - if tree.exists(filename, use_dvcignore=False): - with tree.open(filename) as fobj: + if fs.exists(filename, use_dvcignore=False): + with fs.open(filename) as fobj: conf_obj = ConfigObj(fobj) else: conf_obj = ConfigObj() @@ -165,14 +165,14 @@ class Config(dict): from configobj import ConfigObj filename = self.files[level] - tree = self._get_tree(level) + fs = self._get_fs(level) logger.debug(f"Writing '{filename}'.") - tree.makedirs(os.path.dirname(filename)) + fs.makedirs(os.path.dirname(filename)) config = ConfigObj(_pack_remotes(conf_dict)) - with tree.open(filename, "wb") as fobj: + with fs.open(filename, "wb") as fobj: config.write(fobj) config.filename = filename diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 7bd70ef18..652555b1d 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -104,11 +104,11 @@ class DataCloud: command (str): the command which is benefitting from this function (to be used for reporting better error messages). """ - from dvc.tree import get_cloud_tree + from dvc.fs import get_cloud_fs - from_tree = get_cloud_tree(self.repo, url=source) + from_fs = get_cloud_fs(self.repo, url=source) remote = self.get_remote(remote, command) - return remote.transfer(from_tree, from_tree.path_info, jobs=jobs) + return remote.transfer(from_fs, from_fs.path_info, jobs=jobs) def status( self, diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 681fe9a48..cef642821 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -17,7 +17,7 @@ from dvc.dependency.webhdfs import WebHDFSDependency from dvc.output.base import BaseOutput from dvc.scheme import Schemes -from ..tree import get_cloud_tree +from ..fs import get_cloud_fs from .repo import RepoDependency DEPS = [ @@ -64,8 +64,8 @@ SCHEMA.update(ParamsDependency.PARAM_SCHEMA) def _get(stage, p, info): parsed = urlparse(p) if p else None if parsed and parsed.scheme == "remote": - tree = get_cloud_tree(stage.repo, name=parsed.netloc) - return DEP_MAP[tree.scheme](stage, p, info, tree=tree) + fs = get_cloud_fs(stage.repo, name=parsed.netloc) + return DEP_MAP[fs.scheme](stage, p, info, fs=fs) if info and info.get(RepoDependency.PARAM_REPO): repo = info.pop(RepoDependency.PARAM_REPO) diff --git a/dvc/dependency/azure.py b/dvc/dependency/azure.py index 58a741acd..063ea867f 100644 --- a/dvc/dependency/azure.py +++ b/dvc/dependency/azure.py @@ -1,8 +1,8 @@ from dvc.dependency.base import BaseDependency from dvc.output.base import BaseOutput -from ..tree.azure import AzureTree +from ..fs.azure import AzureFileSystem class AzureDependency(BaseDependency, BaseOutput): - TREE_CLS = AzureTree + FS_CLS = AzureFileSystem diff --git a/dvc/dependency/http.py b/dvc/dependency/http.py index 4eae8d695..521bff00a 100644 --- a/dvc/dependency/http.py +++ b/dvc/dependency/http.py @@ -1,8 +1,8 @@ from dvc.dependency.base import BaseDependency from dvc.output.base import BaseOutput -from ..tree.http import HTTPTree +from ..fs.http import HTTPFileSystem class HTTPDependency(BaseDependency, BaseOutput): - TREE_CLS = HTTPTree + FS_CLS = HTTPFileSystem diff --git a/dvc/dependency/https.py b/dvc/dependency/https.py index d7b84efbe..a906bceeb 100644 --- a/dvc/dependency/https.py +++ b/dvc/dependency/https.py @@ -1,6 +1,6 @@ -from ..tree.https import HTTPSTree +from ..fs.https import HTTPSFileSystem from .http import HTTPDependency class HTTPSDependency(HTTPDependency): - TREE_CLS = HTTPSTree + FS_CLS = HTTPSFileSystem diff --git a/dvc/dependency/param.py b/dvc/dependency/param.py index a9068d5b8..5d015f1ae 100644 --- a/dvc/dependency/param.py +++ b/dvc/dependency/param.py @@ -91,7 +91,7 @@ class ParamsDependency(LocalDependency): suffix = self.path_info.suffix.lower() loader = LOADERS[suffix] try: - config = loader(self.path_info, tree=self.repo.tree) + config = loader(self.path_info, fs=self.repo.fs) except ParseError as exc: raise BadParamFileError( f"Unable to read parameters from '{self}'" diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index fbb11fbe5..c70201f3b 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -23,7 +23,7 @@ class RepoDependency(LocalDependency): self.def_repo = def_repo super().__init__(stage, *args, **kwargs) - def _parse_path(self, tree, path): + def _parse_path(self, fs, path): return None @property @@ -52,7 +52,7 @@ class RepoDependency(LocalDependency): with self._make_repo(locked=locked) as repo: path_info = PathInfo(repo.root_dir) / self.def_path return get_hash( - path_info, repo.repo_tree, "md5", follow_subrepos=False + path_info, repo.repo_fs, "md5", follow_subrepos=False ) def workspace_status(self): @@ -86,13 +86,13 @@ class RepoDependency(LocalDependency): obj = stage( cache, path_info, - repo.repo_tree, + repo.repo_fs, jobs=jobs, follow_subrepos=False, ) save(cache, obj, jobs=jobs) - checkout(to.path_info, to.tree, obj, cache) + checkout(to.path_info, to.fs, obj, cache) def update(self, rev=None): if rev: diff --git a/dvc/dependency/webdav.py b/dvc/dependency/webdav.py index 62c1da8c0..bb428bb2a 100644 --- a/dvc/dependency/webdav.py +++ b/dvc/dependency/webdav.py @@ -1,8 +1,8 @@ from dvc.dependency.base import BaseDependency from dvc.output.base import BaseOutput -from ..tree.webdav import WebDAVTree +from ..fs.webdav import WebDAVFileSystem class WebDAVDependency(BaseDependency, BaseOutput): - TREE_CLS = WebDAVTree + FS_CLS = WebDAVFileSystem diff --git a/dvc/dependency/webdavs.py b/dvc/dependency/webdavs.py index 25e320888..acdd164ed 100644 --- a/dvc/dependency/webdavs.py +++ b/dvc/dependency/webdavs.py @@ -1,6 +1,6 @@ -from ..tree.webdavs import WebDAVSTree +from ..fs.webdavs import WebDAVSFileSystem from .webdav import WebDAVDependency class WebDAVSDependency(WebDAVDependency): - TREE_CLS = WebDAVSTree + FS_CLS = WebDAVSFileSystem diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py index 58eb3119a..a5ca24c09 100644 --- a/dvc/dvcfile.py +++ b/dvc/dvcfile.py @@ -106,13 +106,13 @@ class FileMixin: return relpath(self.path) def exists(self): - return self.repo.tree.exists(self.path) + return self.repo.fs.exists(self.path) def _is_git_ignored(self): - from dvc.tree.local import LocalTree + from dvc.fs.local import LocalFileSystem return isinstance( - self.repo.tree, LocalTree + self.repo.fs, LocalFileSystem ) and self.repo.scm.is_ignored(self.path) def _load(self): @@ -122,16 +122,16 @@ class FileMixin: # 3. path doesn't represent a regular file # 4. when the file is git ignored if not self.exists(): - is_ignored = self.repo.tree.exists(self.path, use_dvcignore=False) + is_ignored = self.repo.fs.exists(self.path, use_dvcignore=False) raise StageFileDoesNotExistError(self.path, dvc_ignored=is_ignored) if self.verify: check_dvc_filename(self.path) - if not self.repo.tree.isfile(self.path): + if not self.repo.fs.isfile(self.path): raise StageFileIsNotDvcFileError(self.path) if self._is_git_ignored(): raise FileIsGitIgnored(self.path) - with self.repo.tree.open(self.path, encoding="utf-8") as fd: + with self.repo.fs.open(self.path, encoding="utf-8") as fd: stage_text = fd.read() d = parse_yaml(stage_text, self.path) return self.validate(d, self.relpath), stage_text @@ -232,7 +232,7 @@ class PipelineFile(FileMixin): self._check_if_parametrized(stage) stage_data = serialize.to_pipeline_file(stage) - with modify_yaml(self.path, tree=self.repo.tree) as data: + with modify_yaml(self.path, fs=self.repo.fs) as data: if not data: logger.info("Creating '%s'", self.relpath) @@ -334,7 +334,7 @@ class Lockfile(FileMixin): if not self.exists(): return {} - data = load_yaml(self.path, tree=self.repo.tree) + data = load_yaml(self.path, fs=self.repo.fs) try: data = self.validate(data, fname=self.relpath) except StageFileFormatError as exc: @@ -351,7 +351,7 @@ class Lockfile(FileMixin): def dump(self, stage, **kwargs): stage_data = serialize.to_lockfile(stage) - with modify_yaml(self.path, tree=self.repo.tree) as data: + with modify_yaml(self.path, fs=self.repo.fs) as data: version = LOCKFILE_VERSION.from_dict(data) if version == LOCKFILE_VERSION.V1: logger.info( diff --git a/dvc/tree/__init__.py b/dvc/fs/__init__.py similarity index 57% rename from dvc/tree/__init__.py rename to dvc/fs/__init__.py index cc6d08e8c..a501695bf 100644 --- a/dvc/tree/__init__.py +++ b/dvc/fs/__init__.py @@ -1,45 +1,45 @@ import posixpath from urllib.parse import urlparse -from .azure import AzureTree -from .gdrive import GDriveTree -from .gs import GSTree -from .hdfs import HDFSTree -from .http import HTTPTree -from .https import HTTPSTree -from .local import LocalTree -from .oss import OSSTree -from .s3 import S3Tree -from .ssh import SSHTree -from .webdav import WebDAVTree -from .webdavs import WebDAVSTree -from .webhdfs import WebHDFSTree +from .azure import AzureFileSystem +from .gdrive import GDriveFileSystem +from .gs import GSFileSystem +from .hdfs import HDFSFileSystem +from .http import HTTPFileSystem +from .https import HTTPSFileSystem +from .local import LocalFileSystem +from .oss import OSSFileSystem +from .s3 import S3FileSystem +from .ssh import SSHFileSystem +from .webdav import WebDAVFileSystem +from .webdavs import WebDAVSFileSystem +from .webhdfs import WebHDFSFileSystem -TREES = [ - AzureTree, - GDriveTree, - GSTree, - HDFSTree, - HTTPTree, - HTTPSTree, - S3Tree, - SSHTree, - OSSTree, - WebDAVTree, - WebDAVSTree, - WebHDFSTree - # NOTE: LocalTree is the default +ALL_FS = [ + AzureFileSystem, + GDriveFileSystem, + GSFileSystem, + HDFSFileSystem, + HTTPFileSystem, + HTTPSFileSystem, + S3FileSystem, + SSHFileSystem, + OSSFileSystem, + WebDAVFileSystem, + WebDAVSFileSystem, + WebHDFSFileSystem + # NOTE: LocalFileSystem is the default ] -def get_tree_cls(remote_conf): - for tree_cls in TREES: - if tree_cls.supported(remote_conf): - return tree_cls - return LocalTree +def get_fs_cls(remote_conf): + for fs_cls in ALL_FS: + if fs_cls.supported(remote_conf): + return fs_cls + return LocalFileSystem -def get_tree_config(config, **kwargs): +def get_fs_config(config, **kwargs): name = kwargs.get("name") if name: remote_conf = config["remote"][name.lower()] @@ -74,18 +74,18 @@ def _resolve_remote_refs(config, remote_conf): if parsed.scheme != "remote": return remote_conf - base = get_tree_config(config, name=parsed.netloc) + base = get_fs_config(config, name=parsed.netloc) url = posixpath.join(base["url"], parsed.path.lstrip("/")) return {**base, **remote_conf, "url": url} -def get_cloud_tree(repo, **kwargs): +def get_cloud_fs(repo, **kwargs): from dvc.config import ConfigError from dvc.config_schema import SCHEMA, Invalid - remote_conf = get_tree_config(repo.config, **kwargs) + remote_conf = get_fs_config(repo.config, **kwargs) try: remote_conf = SCHEMA["remote"][str](remote_conf) except Invalid as exc: raise ConfigError(str(exc)) from None - return get_tree_cls(remote_conf)(repo, remote_conf) + return get_fs_cls(remote_conf)(repo, remote_conf) diff --git a/dvc/tree/_metadata.py b/dvc/fs/_metadata.py similarity index 100% rename from dvc/tree/_metadata.py rename to dvc/fs/_metadata.py diff --git a/dvc/tree/azure.py b/dvc/fs/azure.py similarity index 99% rename from dvc/tree/azure.py rename to dvc/fs/azure.py index 2cd02ede8..2271fb775 100644 --- a/dvc/tree/azure.py +++ b/dvc/fs/azure.py @@ -9,12 +9,12 @@ from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm from dvc.scheme import Schemes -from .base import BaseTree +from .base import BaseFileSystem logger = logging.getLogger(__name__) -class AzureTree(BaseTree): +class AzureFileSystem(BaseFileSystem): scheme = Schemes.AZURE PATH_CLS = CloudURLInfo REQUIRES = { diff --git a/dvc/tree/base.py b/dvc/fs/base.py similarity index 99% rename from dvc/tree/base.py rename to dvc/fs/base.py index bc12ebaee..4c7e579d2 100644 --- a/dvc/tree/base.py +++ b/dvc/fs/base.py @@ -38,12 +38,12 @@ class RemoteMissingDepsError(DvcException): @decorator def use_state(call): - tree = call._args[0] # pylint: disable=protected-access - with tree.state: + fs = call._args[0] # pylint: disable=protected-access + with fs.state: return call() -class BaseTree: +class BaseFileSystem: scheme = "base" REQUIRES: ClassVar[Dict[str, str]] = {} PATH_CLS = URLInfo # type: Any diff --git a/dvc/tree/dvc.py b/dvc/fs/dvc.py similarity index 97% rename from dvc/tree/dvc.py rename to dvc/fs/dvc.py index 5d1862a41..c63bb3d65 100644 --- a/dvc/tree/dvc.py +++ b/dvc/fs/dvc.py @@ -7,7 +7,7 @@ from dvc.path_info import PathInfo from dvc.utils import relpath from ._metadata import Metadata -from .base import BaseTree +from .base import BaseFileSystem if typing.TYPE_CHECKING: from dvc.output.base import BaseOutput @@ -16,8 +16,8 @@ if typing.TYPE_CHECKING: logger = logging.getLogger(__name__) -class DvcTree(BaseTree): # pylint:disable=abstract-method - """DVC repo tree. +class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method + """DVC repo fs. Args: repo: DVC repo. @@ -79,7 +79,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method else: checksum = out.hash_info.value remote_info = remote_obj.cache.hash_to_path_info(checksum) - return remote_obj.tree.open( + return remote_obj.fs.open( remote_info, mode=mode, encoding=encoding ) diff --git a/dvc/tree/gdrive.py b/dvc/fs/gdrive.py similarity index 97% rename from dvc/tree/gdrive.py rename to dvc/fs/gdrive.py index 09b264afe..72df10b2a 100644 --- a/dvc/tree/gdrive.py +++ b/dvc/fs/gdrive.py @@ -18,7 +18,7 @@ from dvc.scheme import Schemes from dvc.utils import format_link, tmp_fname from dvc.utils.stream import IterStream -from .base import BaseTree +from .base import BaseFileSystem logger = logging.getLogger(__name__) FOLDER_MIME_TYPE = "application/vnd.google-apps.folder" @@ -85,7 +85,7 @@ class GDriveURLInfo(CloudURLInfo): self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/")) -class GDriveTree(BaseTree): +class GDriveFileSystem(BaseFileSystem): scheme = Schemes.GDRIVE PATH_CLS = GDriveURLInfo PARAM_CHECKSUM = "checksum" @@ -131,7 +131,7 @@ class GDriveTree(BaseTree): self._validate_config() self._gdrive_user_credentials_path = ( tmp_fname(os.path.join(self.repo.tmp_dir, "")) - if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA) + if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA) else config.get( "gdrive_user_credentials_file", os.path.join( @@ -173,8 +173,8 @@ class GDriveTree(BaseTree): Useful for tests, exception messages, etc. Returns either env variable name if it's set or actual path to the credentials file. """ - if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA): - return GDriveTree.GDRIVE_CREDENTIALS_DATA + if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA): + return GDriveFileSystem.GDRIVE_CREDENTIALS_DATA if os.path.exists(self._gdrive_user_credentials_path): return self._gdrive_user_credentials_path return None @@ -188,7 +188,7 @@ class GDriveTree(BaseTree): DVC config client id or secret but forgets to remove the cached credentials file. """ - if not os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA): + if not os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA): if ( settings["client_config"]["client_id"] != auth.credentials.client_id @@ -211,9 +211,11 @@ class GDriveTree(BaseTree): from pydrive2.auth import GoogleAuth from pydrive2.drive import GoogleDrive - if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA): + if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA): with open(self._gdrive_user_credentials_path, "w") as cred_file: - cred_file.write(os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA)) + cred_file.write( + os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA) + ) auth_settings = { "client_config_backend": "settings", @@ -259,7 +261,7 @@ class GDriveTree(BaseTree): gauth.ServiceAuth() else: gauth.CommandLineAuth() - GDriveTree._validate_credentials(gauth, auth_settings) + GDriveFileSystem._validate_credentials(gauth, auth_settings) # Handle AuthenticationError, RefreshError and other auth failures # It's hard to come up with a narrow exception, since PyDrive throws @@ -268,7 +270,7 @@ class GDriveTree(BaseTree): except Exception as exc: raise GDriveAuthError(self.credentials_location) from exc finally: - if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA): + if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA): os.remove(self._gdrive_user_credentials_path) return GoogleDrive(gauth) diff --git a/dvc/tree/git.py b/dvc/fs/git.py similarity index 91% rename from dvc/tree/git.py rename to dvc/fs/git.py index 428566388..ef8c4b684 100644 --- a/dvc/tree/git.py +++ b/dvc/fs/git.py @@ -3,18 +3,19 @@ import os from funcy import cached_property -from dvc.tree.base import BaseTree from dvc.utils import is_exec, relpath +from .base import BaseFileSystem -class GitTree(BaseTree): # pylint:disable=abstract-method + +class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method """Proxies the repo file access methods to Git objects""" def __init__( self, root_dir, trie, use_dvcignore=False, dvcignore_root=None ): super().__init__(None, {}) - self._tree_root = root_dir + self._fs_root = root_dir self.trie = trie self.use_dvcignore = use_dvcignore self.dvcignore_root = dvcignore_root @@ -24,17 +25,17 @@ class GitTree(BaseTree): # pylint:disable=abstract-method return self.trie.rev @property - def tree_root(self): - return self._tree_root + def fs_root(self): + return self._fs_root def _get_key(self, path): if isinstance(path, str): if not os.path.isabs(path): relparts = path.split(os.sep) else: - relparts = relpath(path, self.tree_root).split(os.sep) + relparts = relpath(path, self.fs_root).split(os.sep) else: - relparts = path.relative_to(self.tree_root).parts + relparts = path.relative_to(self.fs_root).parts if relparts == ["."]: return () return tuple(relparts) @@ -43,7 +44,7 @@ class GitTree(BaseTree): # pylint:disable=abstract-method def dvcignore(self): from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop - root = self.dvcignore_root or self.tree_root + root = self.dvcignore_root or self.fs_root cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop return cls(self, root) @@ -122,9 +123,9 @@ class GitTree(BaseTree): # pylint:disable=abstract-method key = self._get_key(top) for prefix, dirs, files in self.trie.walk(key, topdown=topdown): if prefix: - root = os.path.join(self.tree_root, os.sep.join(prefix)) + root = os.path.join(self.fs_root, os.sep.join(prefix)) else: - root = self.tree_root + root = self.fs_root if use_dvcignore: dirs[:], files[:] = self.dvcignore( root, dirs, files, ignore_subrepos=ignore_subrepos, diff --git a/dvc/tree/gs.py b/dvc/fs/gs.py similarity index 98% rename from dvc/tree/gs.py rename to dvc/fs/gs.py index 9b48396a4..2b7847f9b 100644 --- a/dvc/tree/gs.py +++ b/dvc/fs/gs.py @@ -11,7 +11,7 @@ from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm from dvc.scheme import Schemes -from .base import BaseTree +from .base import BaseFileSystem logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def _upload_to_bucket(bucket, fobj, to_info, chunk_size=None): blob.upload_from_file(fobj) -class GSTree(BaseTree): +class GSFileSystem(BaseFileSystem): scheme = Schemes.GS PATH_CLS = CloudURLInfo REQUIRES = {"google-cloud-storage": "google.cloud.storage"} @@ -196,7 +196,7 @@ class GSTree(BaseTree): def _upload_fobj(self, fobj, to_info): bucket = self.gs.bucket(to_info.bucket) # With other references being given in the @dynamic_chunk_size - # this function does not respect tree.CHUNK_SIZE, since it is + # this function does not respect fs.CHUNK_SIZE, since it is # too big for GS to handle. Rather it dynamically calculates the # best possible chunk size _upload_to_bucket(bucket, fobj, to_info) diff --git a/dvc/tree/hdfs.py b/dvc/fs/hdfs.py similarity index 99% rename from dvc/tree/hdfs.py rename to dvc/fs/hdfs.py index f4919759e..1693697a0 100644 --- a/dvc/tree/hdfs.py +++ b/dvc/fs/hdfs.py @@ -13,7 +13,7 @@ from dvc.progress import Tqdm from dvc.scheme import Schemes from dvc.utils import fix_env, tmp_fname -from .base import BaseTree, RemoteCmdError +from .base import BaseFileSystem, RemoteCmdError from .pool import get_connection logger = logging.getLogger(__name__) @@ -58,7 +58,7 @@ def _hadoop_fs_checksum(path_info): return _group(regex, stdout, "checksum") -class HDFSTree(BaseTree): +class HDFSFileSystem(BaseFileSystem): scheme = Schemes.HDFS REQUIRES = {"pyarrow": "pyarrow"} REGEX = r"^hdfs://((?P.*)@)?.*$" diff --git a/dvc/tree/http.py b/dvc/fs/http.py similarity index 98% rename from dvc/tree/http.py rename to dvc/fs/http.py index cf8810166..a843ad1ce 100644 --- a/dvc/tree/http.py +++ b/dvc/fs/http.py @@ -10,7 +10,7 @@ from dvc.path_info import HTTPURLInfo from dvc.progress import Tqdm from dvc.scheme import Schemes -from .base import BaseTree +from .base import BaseFileSystem logger = logging.getLogger(__name__) @@ -24,7 +24,7 @@ def ask_password(host, user): ) -class HTTPTree(BaseTree): # pylint:disable=abstract-method +class HTTPFileSystem(BaseFileSystem): # pylint:disable=abstract-method scheme = Schemes.HTTP PATH_CLS = HTTPURLInfo PARAM_CHECKSUM = "etag" diff --git a/dvc/fs/https.py b/dvc/fs/https.py new file mode 100644 index 000000000..25a93c192 --- /dev/null +++ b/dvc/fs/https.py @@ -0,0 +1,7 @@ +from dvc.scheme import Schemes + +from .http import HTTPFileSystem + + +class HTTPSFileSystem(HTTPFileSystem): # pylint:disable=abstract-method + scheme = Schemes.HTTPS diff --git a/dvc/tree/local.py b/dvc/fs/local.py similarity index 97% rename from dvc/tree/local.py rename to dvc/fs/local.py index f718fbff3..2f48426ba 100644 --- a/dvc/tree/local.py +++ b/dvc/fs/local.py @@ -10,12 +10,12 @@ from dvc.system import System from dvc.utils import is_exec, tmp_fname from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove -from .base import BaseTree +from .base import BaseFileSystem logger = logging.getLogger(__name__) -class LocalTree(BaseTree): +class LocalFileSystem(BaseFileSystem): scheme = Schemes.LOCAL PATH_CLS = PathInfo PARAM_CHECKSUM = "md5" @@ -30,7 +30,7 @@ class LocalTree(BaseTree): self.dvcignore_root = dvcignore_root @property - def tree_root(self): + def fs_root(self): return self.config.get("url") @property @@ -43,7 +43,7 @@ class LocalTree(BaseTree): def dvcignore(self): from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop - root = self.dvcignore_root or self.tree_root + root = self.dvcignore_root or self.fs_root cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop return cls(self, root) @@ -92,7 +92,7 @@ class LocalTree(BaseTree): use_dvcignore=True, ignore_subrepos=True, ): - """Directory tree generator. + """Directory fs generator. See `os.walk` for the docs. Differences: - no support for symlinks diff --git a/dvc/tree/memory.py b/dvc/fs/memory.py similarity index 86% rename from dvc/tree/memory.py rename to dvc/fs/memory.py index 27fa9d7f6..20f8e7e34 100644 --- a/dvc/tree/memory.py +++ b/dvc/fs/memory.py @@ -1,16 +1,16 @@ -from .base import BaseTree +from .base import BaseFileSystem -class MemoryTree(BaseTree): +class MemoryFileSystem(BaseFileSystem): scheme = "local" PARAM_CHECKSUM = "md5" def __init__(self, repo, config): - from fsspec.implementations.memory import MemoryFileSystem + from fsspec.implementations.memory import MemoryFileSystem as MemFS super().__init__(repo, config) - self.fs = MemoryFileSystem() + self.fs = MemFS() def exists(self, path_info, use_dvcignore=True): return self.fs.exists(path_info.path) diff --git a/dvc/tree/oss.py b/dvc/fs/oss.py similarity index 97% rename from dvc/tree/oss.py rename to dvc/fs/oss.py index 619333edc..e16e8272c 100644 --- a/dvc/tree/oss.py +++ b/dvc/fs/oss.py @@ -8,12 +8,12 @@ from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm from dvc.scheme import Schemes -from .base import BaseTree +from .base import BaseFileSystem logger = logging.getLogger(__name__) -class OSSTree(BaseTree): # pylint:disable=abstract-method +class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method """ oss2 document: https://www.alibabacloud.com/help/doc-detail/32026.htm diff --git a/dvc/tree/pool.py b/dvc/fs/pool.py similarity index 100% rename from dvc/tree/pool.py rename to dvc/fs/pool.py diff --git a/dvc/tree/repo.py b/dvc/fs/repo.py similarity index 77% rename from dvc/tree/repo.py rename to dvc/fs/repo.py index b513873eb..2f4dae762 100644 --- a/dvc/tree/repo.py +++ b/dvc/fs/repo.py @@ -10,8 +10,8 @@ from funcy import lfilter, wrap_with from dvc.path_info import PathInfo -from .base import BaseTree -from .dvc import DvcTree +from .base import BaseFileSystem +from .dvc import DvcFileSystem if TYPE_CHECKING: from dvc.repo import Repo @@ -21,14 +21,14 @@ logger = logging.getLogger(__name__) RepoFactory = Union[Callable[[str], "Repo"], Type["Repo"]] -class RepoTree(BaseTree): # pylint:disable=abstract-method - """DVC + git-tracked files tree. +class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method + """DVC + git-tracked files fs. Args: repo: DVC or git repo. subrepos: traverse to subrepos (by default, it ignores subrepos) repo_factory: A function to initialize subrepo with, default is Repo. - kwargs: Additional keyword arguments passed to the `DvcTree()`. + kwargs: Additional keyword arguments passed to the `DvcFileSystem()`. """ scheme = "local" @@ -57,11 +57,11 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method self._subrepos_trie[self.root_dir] = repo - self._dvctrees = {} - """Keep a dvctree instance of each repo.""" + self._dvcfss = {} + """Keep a dvcfs instance of each repo.""" if hasattr(repo, "dvc_dir"): - self._dvctrees[repo.root_dir] = DvcTree(repo) + self._dvcfss[repo.root_dir] = DvcFileSystem(repo) def _get_repo(self, path) -> Optional["Repo"]: """Returns repo that the path falls in, using prefix. @@ -97,7 +97,7 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method rev=self.repo.get_rev(), repo_factory=self.repo_factory, ) - self._dvctrees[repo.root_dir] = DvcTree(repo) + self._dvcfss[repo.root_dir] = DvcFileSystem(repo) self._subrepos_trie[d] = repo def _is_dvc_repo(self, dir_path): @@ -109,11 +109,13 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method repo_path = os.path.join(dir_path, Repo.DVC_DIR) # dvcignore will ignore subrepos, therefore using `use_dvcignore=False` - return self._main_repo.tree.isdir(repo_path, use_dvcignore=False) + return self._main_repo.fs.isdir(repo_path, use_dvcignore=False) - def _get_tree_pair(self, path) -> Tuple[BaseTree, Optional[DvcTree]]: + def _get_fs_pair( + self, path + ) -> Tuple[BaseFileSystem, Optional[DvcFileSystem]]: """ - Returns a pair of trees based on repo the path falls in, using prefix. + Returns a pair of fss based on repo the path falls in, using prefix. """ path = os.path.abspath(path) @@ -121,8 +123,8 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method # this can happen if the path is outside of the repo repo = self._get_repo(path) or self._main_repo - dvc_tree = self._dvctrees.get(repo.root_dir) - return repo.tree, dvc_tree + dvc_fs = self._dvcfss.get(repo.root_dir) + return repo.fs, dvc_fs def open( self, path, mode="r", encoding="utf-8", **kwargs @@ -130,99 +132,99 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method if "b" in mode: encoding = None - tree, dvc_tree = self._get_tree_pair(path) + fs, dvc_fs = self._get_fs_pair(path) path_info = PathInfo(path) try: - return tree.open(path_info, mode=mode, encoding=encoding) + return fs.open(path_info, mode=mode, encoding=encoding) except FileNotFoundError: - if not dvc_tree: + if not dvc_fs: raise - return dvc_tree.open(path_info, mode=mode, encoding=encoding, **kwargs) + return dvc_fs.open(path_info, mode=mode, encoding=encoding, **kwargs) def exists( self, path, use_dvcignore=True ): # pylint: disable=arguments-differ - tree, dvc_tree = self._get_tree_pair(path) + fs, dvc_fs = self._get_fs_pair(path) - if not dvc_tree: - return tree.exists(path) + if not dvc_fs: + return fs.exists(path) - if tree.exists(path): + if fs.exists(path): return True try: - meta = dvc_tree.metadata(path) + meta = dvc_fs.metadata(path) except FileNotFoundError: return False (out,) = meta.outs assert len(meta.outs) == 1 - if tree.exists(out.path_info): + if fs.exists(out.path_info): return False return True def isdir(self, path): # pylint: disable=arguments-differ - tree, dvc_tree = self._get_tree_pair(path) + fs, dvc_fs = self._get_fs_pair(path) try: - st = tree.stat(path) + st = fs.stat(path) return stat.S_ISDIR(st.st_mode) except (OSError, ValueError): # from CPython's os.path.isdir() pass - if not dvc_tree: + if not dvc_fs: return False try: - meta = dvc_tree.metadata(path) + meta = dvc_fs.metadata(path) except FileNotFoundError: return False (out,) = meta.outs assert len(meta.outs) == 1 - if tree.exists(out.path_info): + if fs.exists(out.path_info): return False return meta.isdir def isdvc(self, path, **kwargs): - _, dvc_tree = self._get_tree_pair(path) - return dvc_tree is not None and dvc_tree.isdvc(path, **kwargs) + _, dvc_fs = self._get_fs_pair(path) + return dvc_fs is not None and dvc_fs.isdvc(path, **kwargs) def isfile(self, path): # pylint: disable=arguments-differ - tree, dvc_tree = self._get_tree_pair(path) + fs, dvc_fs = self._get_fs_pair(path) try: - st = tree.stat(path) + st = fs.stat(path) return stat.S_ISREG(st.st_mode) except (OSError, ValueError): # from CPython's os.path.isfile() pass - if not dvc_tree: + if not dvc_fs: return False try: - meta = dvc_tree.metadata(path) + meta = dvc_fs.metadata(path) except FileNotFoundError: return False (out,) = meta.outs assert len(meta.outs) == 1 - if tree.exists(out.path_info): + if fs.exists(out.path_info): return False return meta.isfile def isexec(self, path_info): - tree, dvc_tree = self._get_tree_pair(path_info) - if dvc_tree and dvc_tree.exists(path_info): - return dvc_tree.isexec(path_info) - return tree.isexec(path_info) + fs, dvc_fs = self._get_fs_pair(path_info) + if dvc_fs and dvc_fs.exists(path_info): + return dvc_fs.isexec(path_info) + return fs.isexec(path_info) def stat(self, path): - tree, _ = self._get_tree_pair(path) - return tree.stat(path) + fs, _ = self._get_fs_pair(path) + return fs.stat(path) def _dvc_walk(self, walk): try: @@ -239,15 +241,15 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method NOTE: subrepo will only be discovered when walking if ignore_subrepos is set to False. """ - tree, dvc_tree = self._get_tree_pair(dir_path) - tree_walk = tree.walk( + fs, dvc_fs = self._get_fs_pair(dir_path) + fs_walk = fs.walk( dir_path, topdown=True, ignore_subrepos=not self._traverse_subrepos ) - if dvc_tree: - dvc_walk = dvc_tree.walk(dir_path, topdown=True, **kwargs) + if dvc_fs: + dvc_walk = dvc_fs.walk(dir_path, topdown=True, **kwargs) else: dvc_walk = None - yield from self._walk(tree_walk, dvc_walk, **kwargs) + yield from self._walk(fs_walk, dvc_walk, **kwargs) def _walk(self, repo_walk, dvc_walk=None, dvcfiles=False): from dvc.dvcfile import is_valid_filename @@ -313,11 +315,11 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method follow_subrepos=None, **kwargs ): # pylint: disable=arguments-differ - """Walk and merge both DVC and repo trees. + """Walk and merge both DVC and repo fss. Args: top: path to walk from - topdown: if True, tree will be walked from top down. + topdown: if True, fs will be walked from top down. onerror: if set, onerror function will be called if an error occurs (by default errors are ignored). dvcfiles: if True, dvcfiles will be included in the files list @@ -342,25 +344,25 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method if follow_subrepos is not None: ignore_subrepos = not follow_subrepos - tree, dvc_tree = self._get_tree_pair(top) - repo_exists = tree.exists(top) - repo_walk = tree.walk( + fs, dvc_fs = self._get_fs_pair(top) + repo_exists = fs.exists(top) + repo_walk = fs.walk( top, topdown=topdown, onerror=onerror, ignore_subrepos=ignore_subrepos, ) - if not dvc_tree or (repo_exists and dvc_tree.isdvc(top)): + if not dvc_fs or (repo_exists and dvc_fs.isdvc(top)): yield from self._walk(repo_walk, None, dvcfiles=dvcfiles) return if not repo_exists: - yield from dvc_tree.walk(top, topdown=topdown, **kwargs) + yield from dvc_fs.walk(top, topdown=topdown, **kwargs) dvc_walk = None - if dvc_tree.exists(top): - dvc_walk = dvc_tree.walk(top, topdown=topdown, **kwargs) + if dvc_fs.exists(top): + dvc_walk = dvc_fs.walk(top, topdown=topdown, **kwargs) yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles) @@ -385,21 +387,21 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method @property def hash_jobs(self): # pylint: disable=invalid-overridden-method - return self._main_repo.tree.hash_jobs + return self._main_repo.fs.hash_jobs def metadata(self, path): abspath = os.path.abspath(path) path_info = PathInfo(abspath) - tree, dvc_tree = self._get_tree_pair(path_info) + fs, dvc_fs = self._get_fs_pair(path_info) dvc_meta = None - if dvc_tree: + if dvc_fs: with suppress(FileNotFoundError): - dvc_meta = dvc_tree.metadata(path_info) + dvc_meta = dvc_fs.metadata(path_info) stat_result = None with suppress(FileNotFoundError): - stat_result = tree.stat(path_info) + stat_result = fs.stat(path_info) if not stat_result and not dvc_meta: raise FileNotFoundError @@ -421,9 +423,9 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method return meta def info(self, path_info): - tree, dvc_tree = self._get_tree_pair(path_info) + fs, dvc_fs = self._get_fs_pair(path_info) try: - return tree.info(path_info) + return fs.info(path_info) except FileNotFoundError: - return dvc_tree.info(path_info) + return dvc_fs.info(path_info) diff --git a/dvc/tree/s3.py b/dvc/fs/s3.py similarity index 99% rename from dvc/tree/s3.py rename to dvc/fs/s3.py index 715e0cbe6..91bcb89e8 100644 --- a/dvc/tree/s3.py +++ b/dvc/fs/s3.py @@ -13,14 +13,14 @@ from dvc.progress import Tqdm from dvc.scheme import Schemes from dvc.utils import conversions, error_link -from .base import BaseTree +from .base import BaseFileSystem logger = logging.getLogger(__name__) _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config") -class S3Tree(BaseTree): +class S3FileSystem(BaseFileSystem): scheme = Schemes.S3 PATH_CLS = CloudURLInfo REQUIRES = {"boto3": "boto3"} diff --git a/dvc/tree/ssh/__init__.py b/dvc/fs/ssh/__init__.py similarity index 98% rename from dvc/tree/ssh/__init__.py rename to dvc/fs/ssh/__init__.py index 8e730ae78..54a182368 100644 --- a/dvc/tree/ssh/__init__.py +++ b/dvc/fs/ssh/__init__.py @@ -13,7 +13,7 @@ import dvc.prompt as prompt from dvc.hash_info import HashInfo from dvc.scheme import Schemes -from ..base import BaseTree +from ..base import BaseFileSystem from ..pool import get_connection logger = logging.getLogger(__name__) @@ -30,7 +30,7 @@ def ask_password(host, user, port): ) -class SSHTree(BaseTree): +class SSHFileSystem(BaseFileSystem): scheme = Schemes.SSH REQUIRES = {"paramiko": "paramiko"} JOBS = 4 @@ -101,7 +101,7 @@ class SSHTree(BaseTree): def _load_user_ssh_config(hostname): import paramiko - user_config_file = SSHTree.ssh_config_filename() + user_config_file = SSHFileSystem.ssh_config_filename() user_ssh_config = {} if hostname and os.path.exists(user_config_file): ssh_config = paramiko.SSHConfig() diff --git a/dvc/tree/ssh/connection.py b/dvc/fs/ssh/connection.py similarity index 100% rename from dvc/tree/ssh/connection.py rename to dvc/fs/ssh/connection.py diff --git a/dvc/tree/webdav.py b/dvc/fs/webdav.py similarity index 97% rename from dvc/tree/webdav.py rename to dvc/fs/webdav.py index 40fafd5fd..9b66e635b 100644 --- a/dvc/tree/webdav.py +++ b/dvc/fs/webdav.py @@ -12,7 +12,7 @@ from dvc.path_info import HTTPURLInfo, WebDAVURLInfo from dvc.progress import Tqdm from dvc.scheme import Schemes -from .base import BaseTree +from .base import BaseFileSystem from .http import ask_password logger = logging.getLogger(__name__) @@ -23,7 +23,7 @@ class WebDAVConnectionError(DvcException): super().__init__(f"Unable to connect to WebDAV {host}.") -class WebDAVTree(BaseTree): # pylint:disable=abstract-method +class WebDAVFileSystem(BaseFileSystem): # pylint:disable=abstract-method # Use webdav scheme scheme = Schemes.WEBDAV @@ -47,7 +47,7 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method # Constructor def __init__(self, repo, config): - # Call BaseTree constructor + # Call BaseFileSystem constructor super().__init__(repo, config) # Get username from configuration @@ -235,7 +235,7 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method # Downloads file from remote to file def _download(self, from_info, to_file, name=None, no_progress_bar=False): - # Progress from HTTPTree + # Progress from HTTPFileSystem with open(to_file, "wb") as fd: with Tqdm.wrapattr( fd, diff --git a/dvc/fs/webdavs.py b/dvc/fs/webdavs.py new file mode 100644 index 000000000..984ea5fe5 --- /dev/null +++ b/dvc/fs/webdavs.py @@ -0,0 +1,7 @@ +from dvc.scheme import Schemes + +from .webdav import WebDAVFileSystem + + +class WebDAVSFileSystem(WebDAVFileSystem): # pylint:disable=abstract-method + scheme = Schemes.WEBDAVS diff --git a/dvc/tree/webhdfs.py b/dvc/fs/webhdfs.py similarity index 98% rename from dvc/tree/webhdfs.py rename to dvc/fs/webhdfs.py index 9fa3f71e0..cf6873ae5 100644 --- a/dvc/tree/webhdfs.py +++ b/dvc/fs/webhdfs.py @@ -13,7 +13,7 @@ from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm from dvc.scheme import Schemes -from .base import BaseTree +from .base import BaseFileSystem logger = logging.getLogger(__name__) @@ -30,7 +30,7 @@ def update_pbar(pbar, total): return update -class WebHDFSTree(BaseTree): +class WebHDFSFileSystem(BaseFileSystem): scheme = Schemes.WEBHDFS PATH_CLS = CloudURLInfo REQUIRES = {"hdfs": "hdfs"} diff --git a/dvc/ignore.py b/dvc/ignore.py index e7cace363..8cccfecfe 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -49,13 +49,11 @@ class DvcIgnorePatterns(DvcIgnore): ] @classmethod - def from_files(cls, ignore_file_path, tree): + def from_files(cls, ignore_file_path, fs): assert os.path.isabs(ignore_file_path) dirname = os.path.normpath(os.path.dirname(ignore_file_path)) - ignore_file_rel_path = os.path.relpath( - ignore_file_path, tree.tree_root - ) - with tree.open(ignore_file_path, encoding="utf-8") as fobj: + ignore_file_rel_path = os.path.relpath(ignore_file_path, fs.fs_root) + with fs.open(ignore_file_path, encoding="utf-8") as fobj: path_spec_lines = [ PatternInfo( line, @@ -162,7 +160,7 @@ def _no_match(path): class DvcIgnoreFilterNoop: - def __init__(self, tree, root_dir): + def __init__(self, fs, root_dir): pass def __call__(self, root, dirs, files, **kwargs): @@ -182,15 +180,15 @@ class DvcIgnoreFilterNoop: class DvcIgnoreFilter: - def __init__(self, tree, root_dir): + def __init__(self, fs, root_dir): from dvc.repo import Repo default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)] - self.tree = tree + self.fs = fs self.root_dir = root_dir - self.ignores_trie_tree = PathStringTrie() - self.ignores_trie_tree[root_dir] = DvcIgnorePatterns( + self.ignores_trie_fs = PathStringTrie() + self.ignores_trie_fs[root_dir] = DvcIgnorePatterns( default_ignore_patterns, root_dir ) self._ignored_subrepos = PathStringTrie() @@ -199,18 +197,18 @@ class DvcIgnoreFilter: def _update(self, dirname): self._update_sub_repo(dirname) - old_pattern = self.ignores_trie_tree.longest_prefix(dirname).value + old_pattern = self.ignores_trie_fs.longest_prefix(dirname).value matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False) ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE) - if not matches and self.tree.exists( + if not matches and self.fs.exists( ignore_file_path, use_dvcignore=False ): new_pattern = DvcIgnorePatterns.from_files( - ignore_file_path, self.tree + ignore_file_path, self.fs ) if old_pattern: - self.ignores_trie_tree[dirname] = DvcIgnorePatterns( + self.ignores_trie_fs[dirname] = DvcIgnorePatterns( *merge_patterns( old_pattern.pattern_list, old_pattern.dirname, @@ -219,9 +217,9 @@ class DvcIgnoreFilter: ) ) else: - self.ignores_trie_tree[dirname] = new_pattern + self.ignores_trie_fs[dirname] = new_pattern elif old_pattern: - self.ignores_trie_tree[dirname] = old_pattern + self.ignores_trie_fs[dirname] = old_pattern def _update_sub_repo(self, path): from dvc.repo import Repo @@ -239,9 +237,9 @@ class DvcIgnoreFilter: ) | {dname} pattern_info = PatternInfo(f"/{dname}/", f"in sub_repo:{dname}") new_pattern = DvcIgnorePatterns([pattern_info], root) - old_pattern = self.ignores_trie_tree.longest_prefix(root).value + old_pattern = self.ignores_trie_fs.longest_prefix(root).value if old_pattern: - self.ignores_trie_tree[root] = DvcIgnorePatterns( + self.ignores_trie_fs[root] = DvcIgnorePatterns( *merge_patterns( old_pattern.pattern_list, old_pattern.dirname, @@ -250,7 +248,7 @@ class DvcIgnoreFilter: ) ) else: - self.ignores_trie_tree[root] = new_pattern + self.ignores_trie_fs[root] = new_pattern def __call__(self, root, dirs, files, ignore_subrepos=True): for dname in dirs: @@ -264,11 +262,11 @@ class DvcIgnoreFilter: return dirs, files def _get_trie_pattern(self, dirname): - ignore_pattern = self.ignores_trie_tree.get(dirname) + ignore_pattern = self.ignores_trie_fs.get(dirname) if ignore_pattern: return ignore_pattern - prefix = self.ignores_trie_tree.longest_prefix(dirname).key + prefix = self.ignores_trie_fs.longest_prefix(dirname).key if not prefix: # outside of the repo return None @@ -285,7 +283,7 @@ class DvcIgnoreFilter: for parent in dirs: self._update(parent) - return self.ignores_trie_tree.get(dirname) + return self.ignores_trie_fs.get(dirname) def _is_ignored(self, path, is_dir=False): if self._outside_repo(path): diff --git a/dvc/info.py b/dvc/info.py index 677c5ebb2..e7e186c43 100644 --- a/dvc/info.py +++ b/dvc/info.py @@ -7,10 +7,10 @@ import uuid import psutil from dvc.exceptions import DvcException, NotDvcRepoError +from dvc.fs import ALL_FS, get_fs_cls, get_fs_config from dvc.repo import Repo from dvc.scm.base import SCMError from dvc.system import System -from dvc.tree import TREES, get_tree_cls, get_tree_config from dvc.utils import error_link from dvc.utils.pkg import PKG from dvc.version import __version__ @@ -75,7 +75,7 @@ def _get_caches(cache): def _get_remotes(config): schemes = ( - get_tree_cls(get_tree_config(config, name=remote)).scheme + get_fs_cls(get_fs_config(config, name=remote)).scheme for remote in config["remote"] ) @@ -117,11 +117,11 @@ def _get_linktype_support_info(repo): def _get_supported_remotes(): supported_remotes = [] - for tree_cls in TREES: - if not tree_cls.get_missing_deps(): - supported_remotes.append(tree_cls.scheme) + for fs_cls in ALL_FS: + if not fs_cls.get_missing_deps(): + supported_remotes.append(fs_cls.scheme) - if len(supported_remotes) == len(TREES): + if len(supported_remotes) == len(ALL_FS): return "All remotes" if len(supported_remotes) == 1: diff --git a/dvc/istextfile.py b/dvc/istextfile.py index 917a6e354..d8602c1d6 100644 --- a/dvc/istextfile.py +++ b/dvc/istextfile.py @@ -22,16 +22,12 @@ def istextblock(block): return float(len(nontext)) / len(block) <= 0.30 -def istextfile(fname, blocksize=512, tree=None): +def istextfile(fname, fs, blocksize=512): """ Uses heuristics to guess whether the given file is text or binary, by reading a single block of bytes from the file. If more than 30% of the chars in the block are non-text, or there are NUL ('\x00') bytes in the block, assume this is a binary file. """ - if tree: - open_func = tree.open - else: - open_func = open - with open_func(fname, "rb") as fobj: + with fs.open(fname, "rb") as fobj: block = fobj.read(blocksize) return istextblock(block) diff --git a/dvc/main.py b/dvc/main.py index 6ee3540e1..89eb316f9 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -96,7 +96,7 @@ def main(argv=None): # noqa: C901 finally: logger.setLevel(outerLogLevel) - from dvc.tree.pool import close_pools + from dvc.fs.pool import close_pools # Closing pools by-hand to prevent weird messages when closing SSH # connections. See https://github.com/iterative/dvc/issues/3248 for diff --git a/dvc/objects.py b/dvc/objects.py index 02a2b652d..0a9dff3ac 100644 --- a/dvc/objects.py +++ b/dvc/objects.py @@ -21,9 +21,9 @@ class ObjectFormatError(ObjectError): class HashFile: - def __init__(self, path_info, tree, hash_info): + def __init__(self, path_info, fs, hash_info): self.path_info = path_info - self.tree = tree + self.fs = fs self.hash_info = hash_info def __len__(self): @@ -36,7 +36,7 @@ class HashFile: return bool(self.hash_info) def check(self, odb): - actual = get_hash(self.path_info, self.tree, odb.tree.PARAM_CHECKSUM) + actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM) logger.trace( "cache '%s' expected '%s' actual '%s'", @@ -54,7 +54,7 @@ class HashFile: return odb.get(hash_info) def save(self, odb, **kwargs): - odb.add(self.path_info, self.tree, self.hash_info, **kwargs) + odb.add(self.path_info, self.fs, self.hash_info, **kwargs) class File(HashFile): @@ -63,13 +63,11 @@ class File(HashFile): self.src = None @classmethod - def stage(cls, odb, path_info, tree, **kwargs): - hash_info = get_hash( - path_info, tree, odb.tree.PARAM_CHECKSUM, **kwargs - ) + def stage(cls, odb, path_info, fs, **kwargs): + hash_info = get_hash(path_info, fs, odb.fs.PARAM_CHECKSUM, **kwargs) raw = odb.get(hash_info) - obj = cls(raw.path_info, raw.tree, hash_info) - obj.src = HashFile(path_info, tree, hash_info) + obj = cls(raw.path_info, raw.fs, hash_info) + obj.src = HashFile(path_info, fs, hash_info) return obj def save(self, odb, **kwargs): @@ -81,7 +79,7 @@ class Tree(HashFile): super().__init__(*args, **kwargs) self.src_hash_info = None self.src_path_info = None - self.src_tree = None + self.src_fs = None def __len__(self): return self.hash_info.dir_info.nfiles @@ -95,7 +93,7 @@ class Tree(HashFile): @classmethod def save_dir_info(cls, odb, dir_info, hash_info=None): - if hash_info and hash_info.name == odb.tree.PARAM_CHECKSUM: + if hash_info and hash_info.name == odb.fs.PARAM_CHECKSUM: try: odb.check(hash_info) assert hash_info.dir_info == dir_info @@ -103,40 +101,38 @@ class Tree(HashFile): except (FileNotFoundError, ObjectFormatError): pass + from dvc.fs.memory import MemoryFileSystem from dvc.path_info import PathInfo - from dvc.tree.memory import MemoryTree from dvc.utils import tmp_fname - tree = MemoryTree(None, {}) + fs = MemoryFileSystem(None, {}) path_info = PathInfo(tmp_fname("")) - with tree.open(path_info, "w") as fobj: + with fs.open(path_info, "w") as fobj: json.dump(dir_info.to_list(), fobj, sort_keys=True) - tmp_info = odb.tree.path_info / tmp_fname("") - with tree.open(path_info, "rb") as fobj: - odb.tree.upload_fobj(fobj, tmp_info) + tmp_info = odb.fs.path_info / tmp_fname("") + with fs.open(path_info, "rb") as fobj: + odb.fs.upload_fobj(fobj, tmp_info) - hash_info = get_hash(tmp_info, odb.tree, odb.tree.PARAM_CHECKSUM) - hash_info.value += odb.tree.CHECKSUM_DIR_SUFFIX + hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM) + hash_info.value += odb.fs.CHECKSUM_DIR_SUFFIX hash_info.dir_info = dir_info hash_info.nfiles = dir_info.nfiles - odb.add(tmp_info, odb.tree, hash_info) + odb.add(tmp_info, odb.fs, hash_info) return hash_info @classmethod - def stage(cls, odb, path_info, tree, **kwargs): - hash_info = get_hash( - path_info, tree, odb.tree.PARAM_CHECKSUM, **kwargs - ) + def stage(cls, odb, path_info, fs, **kwargs): + hash_info = get_hash(path_info, fs, odb.fs.PARAM_CHECKSUM, **kwargs) hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info) hi.size = hash_info.size raw = odb.get(hi) - obj = cls(raw.path_info, raw.tree, hi) + obj = cls(raw.path_info, raw.fs, hi) obj.src_hash_info = hash_info obj.src_path_info = path_info - obj.src_tree = tree + obj.src_fs = fs return obj @classmethod @@ -145,7 +141,7 @@ class Tree(HashFile): obj = odb.get(hash_info) try: - with obj.tree.open(obj.path_info, "r") as fobj: + with obj.fs.open(obj.path_info, "r") as fobj: raw = json.load(fobj) except ValueError as exc: raise ObjectFormatError(f"{obj} is corrupted") from exc @@ -161,7 +157,7 @@ class Tree(HashFile): hash_info.dir_info = dir_info hash_info.nfiles = dir_info.nfiles - return cls(obj.path_info, obj.tree, hash_info) + return cls(obj.path_info, obj.fs, hash_info) def save(self, odb, **kwargs): assert self.src_hash_info.dir_info @@ -173,9 +169,9 @@ class Tree(HashFile): desc="Saving " + self.src_path_info.name, unit="file", ): - entry_obj = HashFile(entry_info, self.src_tree, entry_hash) + entry_obj = HashFile(entry_info, self.src_fs, entry_hash) entry_obj.save(odb, **kwargs) - self.src_tree.state.save(self.src_path_info, hi) + self.src_fs.state.save(self.src_path_info, hi) def filter(self, odb, prefix): hash_info = self.hash_info.dir_info.trie.get(prefix) @@ -193,10 +189,10 @@ class Tree(HashFile): return load(odb, self.save_dir_info(odb, dir_info)) -def stage(odb, path_info, tree, **kwargs): - if tree.isdir(path_info): - return Tree.stage(odb, path_info, tree, **kwargs) - return File.stage(odb, path_info, tree, **kwargs) +def stage(odb, path_info, fs, **kwargs): + if fs.isdir(path_info): + return Tree.stage(odb, path_info, fs, **kwargs) + return File.stage(odb, path_info, fs, **kwargs) def save(odb, obj, **kwargs): @@ -220,7 +216,7 @@ def load(odb, hash_info): def _get_dir_size(odb, dir_info): try: return sum( - odb.tree.getsize(odb.hash_to_path_info(hi.value)) + odb.fs.getsize(odb.hash_to_path_info(hi.value)) for _, hi in dir_info.items() ) except FileNotFoundError: @@ -245,22 +241,22 @@ def merge(odb, ancestor_info, our_info, their_info): return hash_info -def _transfer_file(odb, from_tree, from_info): +def _transfer_file(odb, from_fs, from_info): from dvc.utils import tmp_fname from dvc.utils.stream import HashedStreamReader - tmp_info = odb.tree.path_info / tmp_fname() - with from_tree.open( - from_info, mode="rb", chunk_size=from_tree.CHUNK_SIZE + tmp_info = odb.fs.path_info / tmp_fname() + with from_fs.open( + from_info, mode="rb", chunk_size=from_fs.CHUNK_SIZE ) as stream: stream_reader = HashedStreamReader(stream) # Since we don't know the hash beforehand, we'll # upload it to a temporary location and then move # it. - odb.tree.upload_fobj( + odb.fs.upload_fobj( stream_reader, tmp_info, - total=from_tree.getsize(from_info), + total=from_fs.getsize(from_info), desc=from_info.name, ) @@ -268,15 +264,15 @@ def _transfer_file(odb, from_tree, from_info): return tmp_info, hash_info -def _transfer_directory_contents(odb, from_tree, from_info, jobs, pbar): +def _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar): rel_path_infos = {} - from_infos = from_tree.walk_files(from_info) + from_infos = from_fs.walk_files(from_info) def create_tasks(executor, amount): for entry_info in itertools.islice(from_infos, amount): pbar.total += 1 task = executor.submit( - pbar.wrap_fn(_transfer_file), odb, from_tree, entry_info + pbar.wrap_fn(_transfer_file), odb, from_fs, entry_info ) rel_path_infos[task] = entry_info.relative_to(from_info) yield task @@ -294,35 +290,31 @@ def _transfer_directory_contents(odb, from_tree, from_info, jobs, pbar): yield rel_path_infos.pop(task), task.result() -def _transfer_directory( - odb, from_tree, from_info, jobs, no_progress_bar=False -): +def _transfer_directory(odb, from_fs, from_info, jobs, no_progress_bar=False): dir_info = DirInfo() with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar: for ( entry_info, (entry_tmp_info, entry_hash), - ) in _transfer_directory_contents( - odb, from_tree, from_info, jobs, pbar - ): - odb.add(entry_tmp_info, odb.tree, entry_hash) + ) in _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar): + odb.add(entry_tmp_info, odb.fs, entry_hash) dir_info.trie[entry_info.parts] = entry_hash return Tree.save_dir_info(odb, dir_info) -def transfer(odb, from_tree, from_info, jobs=None, no_progress_bar=False): - jobs = jobs or min((from_tree.jobs, odb.tree.jobs)) +def transfer(odb, from_fs, from_info, jobs=None, no_progress_bar=False): + jobs = jobs or min((from_fs.jobs, odb.fs.jobs)) - if from_tree.isdir(from_info): + if from_fs.isdir(from_info): return _transfer_directory( odb, - from_tree, + from_fs, from_info, jobs=jobs, no_progress_bar=no_progress_bar, ) - tmp_info, hash_info = _transfer_file(odb, from_tree, from_info) - odb.add(tmp_info, odb.tree, hash_info) + tmp_info, hash_info = _transfer_file(odb, from_fs, from_info) + odb.add(tmp_info, odb.fs, hash_info) return hash_info diff --git a/dvc/oid.py b/dvc/oid.py index 716456e32..d231de10d 100644 --- a/dvc/oid.py +++ b/dvc/oid.py @@ -9,27 +9,27 @@ from dvc.progress import Tqdm from dvc.utils import file_md5 -def get_file_hash(path_info, tree, name): - info = tree.info(path_info) +def get_file_hash(path_info, fs, name): + info = fs.info(path_info) if name in info: assert not info[name].endswith(".dir") return HashInfo(name, info[name], size=info["size"]) - func = getattr(tree, name, None) + func = getattr(fs, name, None) if func: return func(path_info) if name == "md5": return HashInfo( - name, file_md5(path_info, tree), size=tree.getsize(path_info) + name, file_md5(path_info, fs), size=fs.getsize(path_info) ) raise NotImplementedError -def _calculate_hashes(file_infos, tree, name): +def _calculate_hashes(file_infos, fs, name): def _get_file_hash(path_info): - return get_file_hash(path_info, tree, name) + return get_file_hash(path_info, fs, name) with Tqdm( total=len(file_infos), @@ -37,14 +37,14 @@ def _calculate_hashes(file_infos, tree, name): desc="Computing file/dir hashes (only done once)", ) as pbar: worker = pbar.wrap_fn(_get_file_hash) - with ThreadPoolExecutor(max_workers=tree.hash_jobs) as executor: + with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor: hash_infos = executor.map(worker, file_infos) return dict(zip(file_infos, hash_infos)) -def _iter_hashes(path_info, tree, name, **kwargs): - if name in tree.DETAIL_FIELDS: - for details in tree.ls(path_info, recursive=True, detail=True): +def _iter_hashes(path_info, fs, name, **kwargs): + if name in fs.DETAIL_FIELDS: + for details in fs.ls(path_info, recursive=True, detail=True): file_info = path_info.replace(path=details["name"]) hash_info = HashInfo( name, details[name], size=details.get("size"), @@ -54,8 +54,8 @@ def _iter_hashes(path_info, tree, name, **kwargs): return None file_infos = [] - for file_info in tree.walk_files(path_info, **kwargs): - hash_info = tree.state.get( # pylint: disable=assignment-from-none + for file_info in fs.walk_files(path_info, **kwargs): + hash_info = fs.state.get( # pylint: disable=assignment-from-none file_info ) if not hash_info: @@ -64,14 +64,14 @@ def _iter_hashes(path_info, tree, name, **kwargs): assert hash_info.name == name yield file_info, hash_info - yield from _calculate_hashes(file_infos, tree, name).items() + yield from _calculate_hashes(file_infos, fs, name).items() -def _collect_dir(path_info, tree, name, **kwargs): +def _collect_dir(path_info, fs, name, **kwargs): from dvc.dir_info import DirInfo dir_info = DirInfo() - for fi, hi in _iter_hashes(path_info, tree, name, **kwargs): + for fi, hi in _iter_hashes(path_info, fs, name, **kwargs): if DvcIgnore.DVCIGNORE_FILE == fi.name: raise DvcIgnoreInCollectedDirError(fi.parent) @@ -88,29 +88,29 @@ def _collect_dir(path_info, tree, name, **kwargs): return dir_info -def get_dir_hash(path_info, tree, name, **kwargs): +def get_dir_hash(path_info, fs, name, **kwargs): from dvc.objects import Tree - dir_info = _collect_dir(path_info, tree, name, **kwargs) - hash_info = Tree.save_dir_info(tree.repo.cache.local, dir_info) + dir_info = _collect_dir(path_info, fs, name, **kwargs) + hash_info = Tree.save_dir_info(fs.repo.cache.local, dir_info) hash_info.size = dir_info.size hash_info.dir_info = dir_info return hash_info -def get_hash(path_info, tree, name, **kwargs): +def get_hash(path_info, fs, name, **kwargs): assert path_info and ( - isinstance(path_info, str) or path_info.scheme == tree.scheme + isinstance(path_info, str) or path_info.scheme == fs.scheme ) - if not tree.exists(path_info): + if not fs.exists(path_info): raise FileNotFoundError( errno.ENOENT, os.strerror(errno.ENOENT), path_info ) - with tree.state: + with fs.state: # pylint: disable=assignment-from-none - hash_info = tree.state.get(path_info) + hash_info = fs.state.get(path_info) # If we have dir hash in state db, but dir cache file is lost, # then we need to recollect the dir via .get_dir_hash() call below, @@ -118,8 +118,8 @@ def get_hash(path_info, tree, name, **kwargs): if ( hash_info and hash_info.isdir - and not tree.cache.tree.exists( - tree.cache.hash_to_path_info(hash_info.value) + and not fs.cache.fs.exists( + fs.cache.hash_to_path_info(hash_info.value) ) ): hash_info = None @@ -128,17 +128,17 @@ def get_hash(path_info, tree, name, **kwargs): if hash_info.isdir: from dvc.objects import Tree - # NOTE: loading the tree will restore hash_info.dir_info - Tree.load(tree.cache, hash_info) + # NOTE: loading the fs will restore hash_info.dir_info + Tree.load(fs.cache, hash_info) assert hash_info.name == name return hash_info - if tree.isdir(path_info): - hash_info = get_dir_hash(path_info, tree, name, **kwargs) + if fs.isdir(path_info): + hash_info = get_dir_hash(path_info, fs, name, **kwargs) else: - hash_info = get_file_hash(path_info, tree, name) + hash_info = get_file_hash(path_info, fs, name) - if hash_info and tree.exists(path_info): - tree.state.save(path_info, hash_info) + if hash_info and fs.exists(path_info): + fs.state.save(path_info, hash_info) return hash_info diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index 7123951b8..ea93c847c 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -14,11 +14,11 @@ from dvc.output.ssh import SSHOutput from dvc.output.webhdfs import WebHDFSOutput from dvc.scheme import Schemes -from ..tree import get_cloud_tree -from ..tree.hdfs import HDFSTree -from ..tree.local import LocalTree -from ..tree.s3 import S3Tree -from ..tree.webhdfs import WebHDFSTree +from ..fs import get_cloud_fs +from ..fs.hdfs import HDFSFileSystem +from ..fs.local import LocalFileSystem +from ..fs.s3 import S3FileSystem +from ..fs.webhdfs import WebHDFSFileSystem OUTS = [ HDFSOutput, @@ -53,10 +53,10 @@ CHECKSUM_SCHEMA = Any( # so when a few types of outputs share the same name, we only need # specify it once. CHECKSUMS_SCHEMA = { - LocalTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA, - S3Tree.PARAM_CHECKSUM: CHECKSUM_SCHEMA, - HDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA, - WebHDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA, + LocalFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA, + S3FileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA, + HDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA, + WebHDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA, } SCHEMA = CHECKSUMS_SCHEMA.copy() @@ -88,13 +88,13 @@ def _get( parsed = urlparse(p) if parsed.scheme == "remote": - tree = get_cloud_tree(stage.repo, name=parsed.netloc) - return OUTS_MAP[tree.scheme]( + fs = get_cloud_fs(stage.repo, name=parsed.netloc) + return OUTS_MAP[fs.scheme]( stage, p, info, cache=cache, - tree=tree, + fs=fs, metric=metric, plot=plot, persist=persist, @@ -111,7 +111,7 @@ def _get( p, info, cache=cache, - tree=None, + fs=None, metric=metric, plot=plot, persist=persist, @@ -125,7 +125,7 @@ def _get( p, info, cache=cache, - tree=None, + fs=None, metric=metric, plot=plot, persist=persist, diff --git a/dvc/output/base.py b/dvc/output/base.py index 91ecc79c0..2779d3f9c 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -20,7 +20,7 @@ from dvc.exceptions import ( from dvc.hash_info import HashInfo from dvc.oid import get_hash -from ..tree.base import BaseTree +from ..fs.base import BaseFileSystem logger = logging.getLogger(__name__) @@ -61,7 +61,7 @@ class OutputIsIgnoredError(DvcException): class BaseOutput: IS_DEPENDENCY = False - TREE_CLS = BaseTree + FS_CLS = BaseFileSystem PARAM_PATH = "path" PARAM_CACHE = "cache" @@ -105,7 +105,7 @@ class BaseOutput: stage, path, info=None, - tree=None, + fs=None, cache=True, metric=False, plot=False, @@ -130,10 +130,10 @@ class BaseOutput: self.repo = stage.repo if stage else None self.def_path = path self.hash_info = HashInfo.from_dict(info) - if tree: - self.tree = tree + if fs: + self.fs = fs else: - self.tree = self.TREE_CLS(self.repo, {}) + self.fs = self.FS_CLS(self.repo, {}) self.use_cache = False if self.IS_DEPENDENCY else cache self.metric = False if self.IS_DEPENDENCY else metric self.plot = False if self.IS_DEPENDENCY else plot @@ -142,18 +142,18 @@ class BaseOutput: self.live = live self.desc = desc - self.path_info = self._parse_path(tree, path) + self.path_info = self._parse_path(fs, path) if self.use_cache and self.cache is None: raise RemoteCacheRequiredError(self.path_info) self.obj = None self.isexec = False if self.IS_DEPENDENCY else isexec - def _parse_path(self, tree, path): - if tree: + def _parse_path(self, fs, path): + if fs: parsed = urlparse(path) - return tree.path_info / parsed.path.lstrip("/") - return self.TREE_CLS.PATH_CLS(path) + return fs.path_info / parsed.path.lstrip("/") + return self.FS_CLS.PATH_CLS(path) def __repr__(self): return "{class_name}: '{def_path}'".format( @@ -165,7 +165,7 @@ class BaseOutput: @property def scheme(self): - return self.TREE_CLS.scheme + return self.FS_CLS.scheme @property def is_in_repo(self): @@ -188,7 +188,7 @@ class BaseOutput: @classmethod def supported(cls, url): - return cls.TREE_CLS.supported(url) + return cls.FS_CLS.supported(url) @property def cache_path(self): @@ -196,10 +196,8 @@ class BaseOutput: def get_hash(self): if not self.use_cache: - return get_hash( - self.path_info, self.tree, self.tree.PARAM_CHECKSUM - ) - return objects.stage(self.cache, self.path_info, self.tree).hash_info + return get_hash(self.path_info, self.fs, self.fs.PARAM_CHECKSUM) + return objects.stage(self.cache, self.path_info, self.fs).hash_info @property def is_dir_checksum(self): @@ -207,7 +205,7 @@ class BaseOutput: @property def exists(self): - return self.tree.exists(self.path_info) + return self.fs.exists(self.path_info) def changed_checksum(self): return self.hash_info != self.get_hash() @@ -251,13 +249,13 @@ class BaseOutput: @property def is_empty(self): - return self.tree.is_empty(self.path_info) + return self.fs.is_empty(self.path_info) def isdir(self): - return self.tree.isdir(self.path_info) + return self.fs.isdir(self.path_info) def isfile(self): - return self.tree.isfile(self.path_info) + return self.fs.isfile(self.path_info) # pylint: disable=no-member @@ -307,9 +305,9 @@ class BaseOutput: logger.debug("Output '%s' didn't change. Skipping saving.", self) return - self.obj = objects.stage(self.cache, self.path_info, self.tree) + self.obj = objects.stage(self.cache, self.path_info, self.fs) self.hash_info = self.obj.hash_info - self.isexec = self.isfile() and self.tree.isexec(self.path_info) + self.isexec = self.isfile() and self.fs.isexec(self.path_info) def set_exec(self): if self.isfile() and self.isexec: @@ -323,12 +321,12 @@ class BaseOutput: if self.use_cache: obj = objects.stage( - self.cache, filter_info or self.path_info, self.tree + self.cache, filter_info or self.path_info, self.fs ) objects.save(self.cache, obj) checkout( filter_info or self.path_info, - self.tree, + self.fs, obj, self.cache, relink=True, @@ -379,7 +377,7 @@ class BaseOutput: raise DvcException(f"verify metric is not supported for {self.scheme}") def download(self, to, jobs=None): - self.tree.download(self.path_info, to.path_info, jobs=jobs) + self.fs.download(self.path_info, to.path_info, jobs=jobs) def get_obj(self, filter_info=None): if self.obj: @@ -424,7 +422,7 @@ class BaseOutput: try: modified = checkout( filter_info or self.path_info, - self.tree, + self.fs, obj, self.cache, force=force, @@ -440,7 +438,7 @@ class BaseOutput: return added, False if added else modified def remove(self, ignore_remove=False): - self.tree.remove(self.path_info) + self.fs.remove(self.path_info) if self.scheme != "local": return @@ -452,7 +450,7 @@ class BaseOutput: if self.scheme == "local" and self.use_scm_ignore: self.repo.scm.ignore_remove(self.fspath) - self.tree.move(self.path_info, out.path_info) + self.fs.move(self.path_info, out.path_info) self.def_path = out.def_path self.path_info = out.path_info self.save() @@ -624,8 +622,8 @@ class BaseOutput: if stage: abs_path = os.path.join(stage.wdir, path) - if stage.repo.tree.dvcignore.is_ignored(abs_path): - check = stage.repo.tree.dvcignore.check_ignore(abs_path) + if stage.repo.fs.dvcignore.is_ignored(abs_path): + check = stage.repo.fs.dvcignore.check_ignore(abs_path) raise cls.IsIgnoredError(check) def _check_can_merge(self, out): @@ -636,7 +634,7 @@ class BaseOutput: other = out.dumpd() ignored = [ - self.tree.PARAM_CHECKSUM, + self.fs.PARAM_CHECKSUM, HashInfo.PARAM_SIZE, HashInfo.PARAM_NFILES, ] diff --git a/dvc/output/gs.py b/dvc/output/gs.py index 6e9a6639b..86adcd8ed 100644 --- a/dvc/output/gs.py +++ b/dvc/output/gs.py @@ -1,7 +1,7 @@ from dvc.output.base import BaseOutput -from ..tree.gs import GSTree +from ..fs.gs import GSFileSystem class GSOutput(BaseOutput): - TREE_CLS = GSTree + FS_CLS = GSFileSystem diff --git a/dvc/output/hdfs.py b/dvc/output/hdfs.py index ef1df8558..6a4f16b14 100644 --- a/dvc/output/hdfs.py +++ b/dvc/output/hdfs.py @@ -1,7 +1,7 @@ from dvc.output.base import BaseOutput -from ..tree.hdfs import HDFSTree +from ..fs.hdfs import HDFSFileSystem class HDFSOutput(BaseOutput): - TREE_CLS = HDFSTree + FS_CLS = HDFSFileSystem diff --git a/dvc/output/local.py b/dvc/output/local.py index c3b6688f3..1ab16fcbe 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -8,13 +8,13 @@ from dvc.output.base import BaseOutput from dvc.utils import relpath from dvc.utils.fs import path_isin -from ..tree.local import LocalTree +from ..fs.local import LocalFileSystem logger = logging.getLogger(__name__) class LocalOutput(BaseOutput): - TREE_CLS = LocalTree + FS_CLS = LocalFileSystem sep = os.sep def __init__(self, stage, path, *args, **kwargs): @@ -25,14 +25,14 @@ class LocalOutput(BaseOutput): if ( self.is_in_repo and self.repo - and isinstance(self.repo.tree, LocalTree) + and isinstance(self.repo.fs, LocalFileSystem) ): - self.tree = self.repo.tree + self.fs = self.repo.fs - def _parse_path(self, tree, path): + def _parse_path(self, fs, path): parsed = urlparse(path) if parsed.scheme == "remote": - p = tree.path_info / parsed.path.lstrip("/") + p = fs.path_info / parsed.path.lstrip("/") else: # NOTE: we can path either from command line or .dvc file, # so we should expect both posix and windows style paths. @@ -40,12 +40,12 @@ class LocalOutput(BaseOutput): # # FIXME: if we have Windows path containing / or posix one with \ # then we have #2059 bug and can't really handle that. - p = self.TREE_CLS.PATH_CLS(path) + p = self.FS_CLS.PATH_CLS(path) if self.stage and not p.is_absolute(): p = self.stage.wdir / p abs_p = os.path.abspath(os.path.normpath(p)) - return self.TREE_CLS.PATH_CLS(abs_p) + return self.FS_CLS.PATH_CLS(abs_p) def __str__(self): if not self.repo or not self.is_in_repo: @@ -91,6 +91,6 @@ class LocalOutput(BaseOutput): logger.debug(msg, str(self.path_info), name) return - if not istextfile(path): + if not istextfile(path, self.fs): msg = "binary file '{}' cannot be used as {}." raise DvcException(msg.format(self.path_info, name)) diff --git a/dvc/output/s3.py b/dvc/output/s3.py index fbe7dbbd6..c7d31352a 100644 --- a/dvc/output/s3.py +++ b/dvc/output/s3.py @@ -1,7 +1,7 @@ from dvc.output.base import BaseOutput -from ..tree.s3 import S3Tree +from ..fs.s3 import S3FileSystem class S3Output(BaseOutput): - TREE_CLS = S3Tree + FS_CLS = S3FileSystem diff --git a/dvc/output/ssh.py b/dvc/output/ssh.py index d4b65a3c1..1bc460f39 100644 --- a/dvc/output/ssh.py +++ b/dvc/output/ssh.py @@ -1,7 +1,7 @@ from dvc.output.base import BaseOutput -from ..tree.ssh import SSHTree +from ..fs.ssh import SSHFileSystem class SSHOutput(BaseOutput): - TREE_CLS = SSHTree + FS_CLS = SSHFileSystem diff --git a/dvc/output/webhdfs.py b/dvc/output/webhdfs.py index 7fd28229e..ee52a1bf9 100644 --- a/dvc/output/webhdfs.py +++ b/dvc/output/webhdfs.py @@ -1,7 +1,7 @@ from dvc.output.base import BaseOutput -from ..tree.webhdfs import WebHDFSTree +from ..fs.webhdfs import WebHDFSFileSystem class WebHDFSOutput(BaseOutput): - TREE_CLS = WebHDFSTree + FS_CLS = WebHDFSFileSystem diff --git a/dvc/parsing/__init__.py b/dvc/parsing/__init__.py index 96dde8768..61f91fcac 100644 --- a/dvc/parsing/__init__.py +++ b/dvc/parsing/__init__.py @@ -135,7 +135,7 @@ def make_definition( class DataResolver: def __init__(self, repo: "Repo", wdir: PathInfo, d: dict): - self.tree = tree = repo.tree + self.fs = fs = repo.fs self.wdir = wdir self.relpath = relpath(self.wdir / "dvc.yaml") @@ -144,7 +144,7 @@ class DataResolver: self.context: Context = Context() try: - args = tree, vars_, wdir # load from `vars` section + args = fs, vars_, wdir # load from `vars` section self.context.load_from_vars(*args, default=DEFAULT_PARAMS_FILE) except ContextError as exc: format_and_raise(exc, "'vars'", self.relpath) @@ -270,8 +270,8 @@ class EntryDefinition: context = Context.clone(context) try: - tree = self.resolver.tree - context.load_from_vars(tree, vars_, wdir, stage_name=name) + fs = self.resolver.fs + context.load_from_vars(fs, vars_, wdir, stage_name=name) except VarsAlreadyLoaded as exc: format_and_raise(exc, f"'{self.where}.{name}.vars'", self.relpath) diff --git a/dvc/parsing/context.py b/dvc/parsing/context.py index 08a527beb..ee0e07e75 100644 --- a/dvc/parsing/context.py +++ b/dvc/parsing/context.py @@ -356,20 +356,20 @@ class Context(CtxDict): @classmethod def load_from( - cls, tree, path: PathInfo, select_keys: List[str] = None + cls, fs, path: PathInfo, select_keys: List[str] = None ) -> "Context": from dvc.utils.serialize import LOADERS file = relpath(path) - if not tree.exists(path): + if not fs.exists(path): raise ParamsLoadError(f"'{file}' does not exist") - if tree.isdir(path): + if fs.isdir(path): raise ParamsLoadError(f"'{file}' is a directory") _, ext = os.path.splitext(file) loader = LOADERS[ext] - data = loader(path, tree=tree) + data = loader(path, fs=fs) if not isinstance(data, Mapping): typ = type(data).__name__ raise ParamsLoadError( @@ -397,7 +397,7 @@ class Context(CtxDict): return super().merge_update(other, overwrite=overwrite) def merge_from( - self, tree, item: str, wdir: PathInfo, overwrite=False, + self, fs, item: str, wdir: PathInfo, overwrite=False, ): path, _, keys_str = item.partition(":") select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None @@ -409,7 +409,7 @@ class Context(CtxDict): return # allow specifying complete filepath multiple times self.check_loaded(abspath, item, select_keys) - ctx = Context.load_from(tree, path_info, select_keys) + ctx = Context.load_from(fs, path_info, select_keys) try: self.merge_update(ctx, overwrite=overwrite) @@ -439,7 +439,7 @@ class Context(CtxDict): def load_from_vars( self, - tree, + fs, vars_: List, wdir: PathInfo, stage_name: str = None, @@ -447,8 +447,8 @@ class Context(CtxDict): ): if default: to_import = wdir / default - if tree.exists(to_import): - self.merge_from(tree, default, wdir) + if fs.exists(to_import): + self.merge_from(fs, default, wdir) else: msg = "%s does not exist, it won't be used in parametrization" logger.trace(msg, to_import) # type: ignore[attr-defined] @@ -457,7 +457,7 @@ class Context(CtxDict): for index, item in enumerate(vars_): assert isinstance(item, (str, dict)) if isinstance(item, str): - self.merge_from(tree, item, wdir) + self.merge_from(fs, item, wdir) else: joiner = "." if stage_name else "" meta = Meta(source=f"{stage_name}{joiner}vars[{index}]") diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index 7979ed7cf..aad726221 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -1,10 +1,10 @@ -from ..tree import get_cloud_tree +from ..fs import get_cloud_fs from .base import Remote from .local import LocalRemote def get_remote(repo, **kwargs): - tree = get_cloud_tree(repo, **kwargs) - if tree.scheme == "local": - return LocalRemote(tree) - return Remote(tree) + fs = get_cloud_fs(repo, **kwargs) + if fs.scheme == "local": + return LocalRemote(fs) + return Remote(fs) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index b7cc9b385..a92887c9b 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -61,25 +61,25 @@ def index_locked(f): class Remote: """Cloud remote class. - Provides methods for indexing and garbage collecting trees which contain + Provides methods for indexing and garbage collecting fss which contain DVC remotes. """ INDEX_CLS = RemoteIndex - def __init__(self, tree): + def __init__(self, fs): from dvc.cache import get_cloud_cache - self.tree = tree - self.repo = tree.repo - self.cache = get_cloud_cache(self.tree) + self.fs = fs + self.repo = fs.repo + self.cache = get_cloud_cache(self.fs) - config = tree.config + config = fs.config url = config.get("url") if url: index_name = hashlib.sha256(url.encode("utf-8")).hexdigest() self.index = self.INDEX_CLS( - self.repo, index_name, dir_suffix=self.tree.CHECKSUM_DIR_SUFFIX + self.repo, index_name, dir_suffix=self.fs.CHECKSUM_DIR_SUFFIX ) else: self.index = RemoteIndexNoop() @@ -87,7 +87,7 @@ class Remote: def __repr__(self): return "{class_name}: '{path_info}'".format( class_name=type(self).__name__, - path_info=self.tree.path_info or "No path", + path_info=self.fs.path_info or "No path", ) @index_locked @@ -147,8 +147,8 @@ class Remote: {dir_hash: set(file_hash, ...)} which can be used to map a .dir file to its file contents. """ - logger.debug(f"Preparing to collect status from {self.tree.path_info}") - md5s = set(named_cache.scheme_keys(cache.tree.scheme)) + logger.debug(f"Preparing to collect status from {self.fs.path_info}") + md5s = set(named_cache.scheme_keys(cache.fs.scheme)) logger.debug("Collecting information from local cache...") local_exists = frozenset( @@ -164,7 +164,7 @@ class Remote: else: logger.debug("Collecting information from remote cache...") remote_exists = set() - dir_md5s = set(named_cache.dir_keys(cache.tree.scheme)) + dir_md5s = set(named_cache.dir_keys(cache.fs.scheme)) if dir_md5s: remote_exists.update( self._indexed_dir_hashes(cache, named_cache, dir_md5s) @@ -173,7 +173,7 @@ class Remote: if md5s: remote_exists.update( self.hashes_exist( - md5s, jobs=jobs, name=str(self.tree.path_info) + md5s, jobs=jobs, name=str(self.fs.path_info) ) ) return self._make_status( @@ -200,7 +200,7 @@ class Remote: dir_status = {} file_status = {} dir_contents = {} - for hash_, item in named_cache[cache.tree.scheme].items(): + for hash_, item in named_cache[cache.fs.scheme].items(): if item.children: dir_status[hash_] = make_names(hash_, item.names) dir_contents[hash_] = set() @@ -245,7 +245,7 @@ class Remote: # still exists on the remote for dir_hash in dir_exists: file_hashes = list( - named_cache.child_keys(cache.tree.scheme, dir_hash) + named_cache.child_keys(cache.fs.scheme, dir_hash) ) if dir_hash not in self.index: logger.debug( @@ -304,21 +304,21 @@ class Remote: logger.debug( "Preparing to {} '{}'".format( "download data from" if download else "upload data to", - self.tree.path_info, + self.fs.path_info, ) ) if download: - func = _log_exceptions(self.tree.download, "download") + func = _log_exceptions(self.fs.download, "download") status = STATUS_DELETED desc = "Downloading" else: - func = _log_exceptions(self.tree.upload, "upload") + func = _log_exceptions(self.fs.upload, "upload") status = STATUS_NEW desc = "Uploading" if jobs is None: - jobs = self.tree.JOBS + jobs = self.fs.JOBS dir_status, file_status, dir_contents = self._status( cache, @@ -471,15 +471,13 @@ class Remote: download=False, ) - if self.tree.scheme == "local": - with self.tree.state: + if self.fs.scheme == "local": + with self.fs.state: for checksum in named_cache.scheme_keys("local"): cache_file = self.cache.hash_to_path_info(checksum) - if self.tree.exists(cache_file): - hash_info = HashInfo( - self.tree.PARAM_CHECKSUM, checksum - ) - self.tree.state.save(cache_file, hash_info) + if self.fs.exists(cache_file): + hash_info = HashInfo(self.fs.PARAM_CHECKSUM, checksum) + self.fs.state.save(cache_file, hash_info) self.cache.protect(cache_file) return ret @@ -495,28 +493,26 @@ class Remote: ) if not self.cache.verify: - with cache.tree.state: + with cache.fs.state: for checksum in named_cache.scheme_keys("local"): cache_file = cache.hash_to_path_info(checksum) - if cache.tree.exists(cache_file): + if cache.fs.exists(cache_file): # We can safely save here, as existing corrupted files # will be removed upon status, while files corrupted # during download will not be moved from tmp_file - # (see `BaseTree.download()`) - hash_info = HashInfo( - cache.tree.PARAM_CHECKSUM, checksum - ) - cache.tree.state.save(cache_file, hash_info) + # (see `BaseFileSystem.download()`) + hash_info = HashInfo(cache.fs.PARAM_CHECKSUM, checksum) + cache.fs.state.save(cache_file, hash_info) cache.protect(cache_file) return ret - def transfer(self, from_tree, from_info, jobs=None, no_progress_bar=False): + def transfer(self, from_fs, from_info, jobs=None, no_progress_bar=False): from dvc.objects import transfer return transfer( self.cache, - from_tree, + from_fs, from_info, jobs=jobs, no_progress_bar=no_progress_bar, diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index ce3b6698a..0ca9303d6 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -16,8 +16,8 @@ from .graph import build_graph, build_outs_graph, get_pipelines from .trie import build_outs_trie if TYPE_CHECKING: + from dvc.fs.base import BaseFileSystem from dvc.scm import Base - from dvc.tree.base import BaseTree logger = logging.getLogger(__name__) @@ -99,8 +99,8 @@ class Repo: dvc_dir = None tmp_dir = None try: - tree = scm.get_tree(rev) if isinstance(scm, Git) and rev else None - root_dir = self.find_root(root_dir, tree) + fs = scm.get_fs(rev) if isinstance(scm, Git) and rev else None + root_dir = self.find_root(root_dir, fs) dvc_dir = os.path.join(root_dir, self.DVC_DIR) tmp_dir = os.path.join(dvc_dir, "tmp") makedirs(tmp_dir, exist_ok=True) @@ -134,6 +134,7 @@ class Repo: from dvc.cache import Cache from dvc.config import Config from dvc.data_cloud import DataCloud + from dvc.fs.local import LocalFileSystem from dvc.lock import LockNoop, make_lock from dvc.repo.live import Live from dvc.repo.metrics import Metrics @@ -142,10 +143,9 @@ class Repo: from dvc.repo.stage import StageLoad from dvc.stage.cache import StageCache from dvc.state import State, StateNoop - from dvc.tree.local import LocalTree self.url = url - self._tree_conf = { + self._fs_conf = { "repo_factory": repo_factory, } @@ -153,17 +153,19 @@ class Repo: root_dir=root_dir, scm=scm, rev=rev, uninitialized=uninitialized ) - tree_kwargs = {"use_dvcignore": True, "dvcignore_root": self.root_dir} + fs_kwargs = {"use_dvcignore": True, "dvcignore_root": self.root_dir} if scm: - self._tree = scm.get_tree(rev, **tree_kwargs) + self._fs = scm.get_fs(rev, **fs_kwargs) else: - self._tree = LocalTree(self, {"url": self.root_dir}, **tree_kwargs) + self._fs = LocalFileSystem( + self, {"url": self.root_dir}, **fs_kwargs + ) - self.config = Config(self.dvc_dir, tree=self.tree, config=config) + self.config = Config(self.dvc_dir, fs=self.fs, config=config) self._uninitialized = uninitialized self._scm = scm - # used by RepoTree to determine if it should traverse subrepos + # used by RepoFileSystem to determine if it should traverse subrepos self.subrepos = subrepos self.cache = Cache(self) @@ -239,12 +241,12 @@ class Repo: raise def get_rev(self): - from dvc.tree.local import LocalTree + from dvc.fs.local import LocalFileSystem assert self.scm - if isinstance(self.tree, LocalTree): + if isinstance(self.fs, LocalFileSystem): return self.scm.get_rev() - return self.tree.rev + return self.fs.rev @cached_property def experiments(self): @@ -253,25 +255,25 @@ class Repo: return Experiments(self) @property - def tree(self) -> "BaseTree": - return self._tree + def fs(self) -> "BaseFileSystem": + return self._fs - @tree.setter - def tree(self, tree: "BaseTree"): - self._tree = tree + @fs.setter + def fs(self, fs: "BaseFileSystem"): + self._fs = fs # Our graph cache is no longer valid, as it was based on the previous - # tree. + # fs. self._reset() def __repr__(self): return f"{self.__class__.__name__}: '{self.root_dir}'" @classmethod - def find_root(cls, root=None, tree=None) -> str: + def find_root(cls, root=None, fs=None) -> str: root_dir = os.path.realpath(root or os.curdir) - if tree: - if tree.isdir(os.path.join(root_dir, cls.DVC_DIR)): + if fs: + if fs.isdir(os.path.join(root_dir, cls.DVC_DIR)): return root_dir raise NotDvcRepoError(f"'{root}' does not contain DVC directory") @@ -462,21 +464,21 @@ class Repo: return self.DVC_DIR in path_parts @cached_property - def repo_tree(self): - from dvc.tree.repo import RepoTree + def repo_fs(self): + from dvc.fs.repo import RepoFileSystem - return RepoTree(self, subrepos=self.subrepos, **self._tree_conf) + return RepoFileSystem(self, subrepos=self.subrepos, **self._fs_conf) @contextmanager def open_by_relpath(self, path, remote=None, mode="r", encoding=None): """Opens a specified resource as a file descriptor""" - from dvc.tree.repo import RepoTree + from dvc.fs.repo import RepoFileSystem - tree = RepoTree(self, subrepos=True) + fs = RepoFileSystem(self, subrepos=True) path = PathInfo(self.root_dir) / path try: with self.state: - with tree.open( + with fs.open( path, mode=mode, encoding=encoding, remote=remote, ) as fobj: yield fobj diff --git a/dvc/repo/add.py b/dvc/repo/add.py index 7a0b4974c..fc0f05472 100644 --- a/dvc/repo/add.py +++ b/dvc/repo/add.py @@ -165,15 +165,12 @@ def _process_stages( command="add", ) else: + from dvc.fs import get_cloud_fs from dvc.objects import transfer - from dvc.tree import get_cloud_tree - from_tree = get_cloud_tree(repo, url=target) + from_fs = get_cloud_fs(repo, url=target) out.hash_info = transfer( - out.cache, - from_tree, - from_tree.path_info, - jobs=kwargs.get("jobs"), + out.cache, from_fs, from_fs.path_info, jobs=kwargs.get("jobs"), ) out.checkout() @@ -212,7 +209,7 @@ def _find_all_targets(repo, target, recursive): return [ os.fspath(path) for path in Tqdm( - repo.tree.walk_files(target), + repo.fs.walk_files(target), desc="Searching " + target, bar_format=Tqdm.BAR_FMT_NOTOTAL, unit="file", diff --git a/dvc/repo/brancher.py b/dvc/repo/brancher.py index 8e6503dea..63bce9d18 100644 --- a/dvc/repo/brancher.py +++ b/dvc/repo/brancher.py @@ -20,7 +20,7 @@ def brancher( # noqa: E302 sha_only (bool): only return git SHA for a revision. Yields: - str: the display name for the currently selected tree, it could be: + str: the display name for the currently selected fs, it could be: - a git revision identifier - empty string it there is no branches to iterate over - "workspace" if there are uncommitted changes in the SCM repo @@ -29,14 +29,14 @@ def brancher( # noqa: E302 yield "" return - from dvc.tree.local import LocalTree + from dvc.fs.local import LocalFileSystem - saved_tree = self.tree + saved_fs = self.fs revs = revs.copy() if revs else [] scm = self.scm - self.tree = LocalTree(self, {"url": self.root_dir}, use_dvcignore=True) + self.fs = LocalFileSystem(self, {"url": self.root_dir}, use_dvcignore=True) yield "workspace" if revs and "workspace" in revs: @@ -59,15 +59,15 @@ def brancher( # noqa: E302 try: if revs: for sha, names in group_by(scm.resolve_rev, revs).items(): - self.tree = scm.get_tree( + self.fs = scm.get_fs( sha, use_dvcignore=True, dvcignore_root=self.root_dir ) # ignore revs that don't contain repo root # (i.e. revs from before a subdir=True repo was init'ed) - if self.tree.exists(self.root_dir): + if self.fs.exists(self.root_dir): if sha_only: yield sha else: yield ", ".join(names) finally: - self.tree = saved_tree + self.fs = saved_fs diff --git a/dvc/repo/collect.py b/dvc/repo/collect.py index 7b6a07c7f..a099d391d 100644 --- a/dvc/repo/collect.py +++ b/dvc/repo/collect.py @@ -34,18 +34,18 @@ def _collect_paths( recursive: bool = False, rev: str = None, ): - from dvc.tree.repo import RepoTree + from dvc.fs.repo import RepoFileSystem path_infos = [PathInfo(os.path.abspath(target)) for target in targets] - tree = RepoTree(repo) + fs = RepoFileSystem(repo) target_infos = [] for path_info in path_infos: - if recursive and tree.isdir(path_info): - target_infos.extend(tree.walk_files(path_info)) + if recursive and fs.isdir(path_info): + target_infos.extend(fs.walk_files(path_info)) - if not tree.exists(path_info): + if not fs.exists(path_info): if not recursive: if rev == "workspace" or rev == "": logger.warning( diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index 22734c1d8..d3c0eb374 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) @locked def diff(self, a_rev="HEAD", b_rev=None, targets=None): """ - By default, it compares the workspace with the last commit's tree. + By default, it compares the workspace with the last commit's fs. This implementation differs from `git diff` since DVC doesn't have the concept of `index`, but it keeps the same interface, thus, @@ -21,9 +21,9 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None): if self.scm.no_commits: return {} - from dvc.tree.repo import RepoTree + from dvc.fs.repo import RepoFileSystem - repo_tree = RepoTree(self) + repo_fs = RepoFileSystem(self) b_rev = b_rev if b_rev else "workspace" results = {} @@ -38,10 +38,10 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None): if targets is not None: # convert targets to path_infos, and capture any missing targets targets_path_infos, missing_targets[rev] = _targets_to_path_infos( - repo_tree, targets + repo_fs, targets ) - results[rev] = _paths_checksums(self, repo_tree, targets_path_infos) + results[rev] = _paths_checksums(self, repo_fs, targets_path_infos) if targets is not None: # check for overlapping missing targets between a_rev and b_rev @@ -53,14 +53,14 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None): old = results[a_rev] new = results[b_rev] - # Compare paths between the old and new tree. + # Compare paths between the old and new fs. # set() efficiently converts dict keys to a set added = sorted(set(new) - set(old)) deleted_or_missing = set(old) - set(new) if b_rev == "workspace": # missing status is only applicable when diffing local workspace # against a commit - missing = sorted(_filter_missing(repo_tree, deleted_or_missing)) + missing = sorted(_filter_missing(repo_fs, deleted_or_missing)) else: missing = [] deleted = sorted(deleted_or_missing - set(missing)) @@ -94,10 +94,10 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None): return ret if any(ret.values()) else {} -def _paths_checksums(repo, repo_tree, targets): +def _paths_checksums(repo, repo_fs, targets): """ A dictionary of checksums addressed by relpaths collected from - the current tree outputs. + the current fs outputs. To help distinguish between a directory and a file output, the former one will come with a trailing slash in the path: @@ -106,16 +106,16 @@ def _paths_checksums(repo, repo_tree, targets): file: "data" """ - return dict(_output_paths(repo, repo_tree, targets)) + return dict(_output_paths(repo, repo_fs, targets)) -def _output_paths(repo, repo_tree, targets): - from dvc.tree.local import LocalTree +def _output_paths(repo, repo_fs, targets): + from dvc.fs.local import LocalFileSystem - on_working_tree = isinstance(repo.tree, LocalTree) + on_working_fs = isinstance(repo.fs, LocalFileSystem) def _exists(output): - if on_working_tree: + if on_working_fs: return output.exists return True @@ -127,10 +127,8 @@ def _output_paths(repo, repo_tree, targets): ) def _to_checksum(output): - if on_working_tree: - return get_hash( - output.path_info, repo.cache.local.tree, "md5" - ).value + if on_working_fs: + return get_hash(output.path_info, repo.cache.local.fs, "md5").value return output.hash_info.value for stage in repo.stages: @@ -147,26 +145,26 @@ def _output_paths(repo, repo_tree, targets): yield_output or any(target.isin(output.path_info) for target in targets) ): - yield from _dir_output_paths(repo_tree, output, targets) + yield from _dir_output_paths(repo_fs, output, targets) -def _dir_output_paths(repo_tree, output, targets=None): +def _dir_output_paths(repo_fs, output, targets=None): from dvc.config import NoRemoteError try: - for fname in repo_tree.walk_files(output.path_info): + for fname in repo_fs.walk_files(output.path_info): if targets is None or any( fname.isin_or_eq(target) for target in targets ): - yield str(fname), get_file_hash(fname, repo_tree, "md5").value + yield str(fname), get_file_hash(fname, repo_fs, "md5").value except NoRemoteError: logger.warning("dir cache entry for '%s' is missing", output) -def _filter_missing(repo_tree, paths): +def _filter_missing(repo_fs, paths): for path in paths: try: - metadata = repo_tree.metadata(path) + metadata = repo_fs.metadata(path) if metadata.is_dvc: out = metadata.outs[0] if out.status().get(str(out)) == "not in cache": @@ -175,13 +173,13 @@ def _filter_missing(repo_tree, paths): pass -def _targets_to_path_infos(repo_tree, targets): +def _targets_to_path_infos(repo_fs, targets): path_infos = [] missing = [] for target in targets: - if repo_tree.exists(target): - path_infos.append(repo_tree.metadata(target).path_info) + if repo_fs.exists(target): + path_infos.append(repo_fs.metadata(target).path_info) else: missing.append(target) diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index 2438d2dd8..ffa6d7afb 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -251,10 +251,10 @@ class Experiments: # NOTE: dirty DVC lock files must be restored to index state to # avoid checking out incorrect persist or checkpoint outs - tree = self.scm.get_tree("HEAD") + fs = self.scm.get_fs("HEAD") lock_files = [ str(fname) - for fname in tree.walk_files(self.scm.root_dir) + for fname in fs.walk_files(self.scm.root_dir) if is_lock_file(fname) ] if lock_files: @@ -333,7 +333,7 @@ class Experiments: path = PathInfo(params_fname) suffix = path.suffix.lower() modify_data = MODIFIERS[suffix] - with modify_data(path, tree=self.repo.tree) as data: + with modify_data(path, fs=self.repo.fs) as data: benedict(data).merge(params[params_fname], overwrite=True) # Force params file changes to be staged in git diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index a8ca5a11c..1010b4832 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -138,11 +138,11 @@ class BaseExecutor(ABC): # TODO: come up with better way to stash repro arguments @staticmethod - def pack_repro_args(path, *args, tree=None, extra=None, **kwargs): + def pack_repro_args(path, *args, fs=None, extra=None, **kwargs): dpath = os.path.dirname(path) - if tree: - open_func = tree.open - tree.makedirs(dpath) + if fs: + open_func = fs.open + fs.makedirs(dpath) else: from dvc.utils.fs import makedirs diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 5d491d06f..5282f0586 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -113,7 +113,7 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs): obj = stage( cache, path_info, - repo.repo_tree, + repo.repo_fs, jobs=jobs, follow_subrepos=False, ) diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 52563cda6..cdd908a82 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -52,7 +52,7 @@ def get(url, path, out=None, rev=None, jobs=None): ) as repo: from_info = PathInfo(repo.root_dir) / path to_info = PathInfo(out) - repo.repo_tree.download( + repo.repo_fs.download( from_info, to_info, jobs=jobs, follow_subrepos=False ) finally: diff --git a/dvc/repo/ls.py b/dvc/repo/ls.py index 12c759385..049814e14 100644 --- a/dvc/repo/ls.py +++ b/dvc/repo/ls.py @@ -35,7 +35,7 @@ def ls( if path: path_info /= path - ret = _ls(repo.repo_tree, path_info, recursive, dvc_only) + ret = _ls(repo.repo_fs, path_info, recursive, dvc_only) if path and not ret: raise PathMissingError(path, repo, dvc_only=dvc_only) @@ -48,13 +48,13 @@ def ls( return ret_list -def _ls(tree, path_info, recursive=None, dvc_only=False): +def _ls(fs, path_info, recursive=None, dvc_only=False): def onerror(exc): raise exc infos = [] try: - for root, dirs, files in tree.walk( + for root, dirs, files in fs.walk( path_info.fspath, onerror=onerror, dvcfiles=True ): entries = chain(files, dirs) if not recursive else files @@ -68,7 +68,7 @@ def _ls(tree, path_info, recursive=None, dvc_only=False): ret = {} for info in infos: - metadata = tree.metadata(info) + metadata = fs.metadata(info) if metadata.output_exists or not dvc_only: path = ( path_info.name diff --git a/dvc/repo/metrics/show.py b/dvc/repo/metrics/show.py index 1a669be30..c03379a5f 100644 --- a/dvc/repo/metrics/show.py +++ b/dvc/repo/metrics/show.py @@ -6,13 +6,13 @@ from dvc.exceptions import ( NoMetricsFoundError, NoMetricsParsedError, ) +from dvc.fs.repo import RepoFileSystem from dvc.output import BaseOutput from dvc.path_info import PathInfo from dvc.repo import locked from dvc.repo.collect import collect from dvc.repo.live import summary_path_info from dvc.scm.base import SCMError -from dvc.tree.repo import RepoTree from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml logger = logging.getLogger(__name__) @@ -71,15 +71,15 @@ def _extract_metrics(metrics, path, rev): def _read_metrics(repo, metrics, rev): - tree = RepoTree(repo) + fs = RepoFileSystem(repo) res = {} for metric in metrics: - if not tree.isfile(metric): + if not fs.isfile(metric): continue try: - val = load_yaml(metric, tree=tree) + val = load_yaml(metric, fs=fs) except (FileNotFoundError, YAMLFileCorruptedError): logger.debug( "failed to read '%s' on '%s'", metric, rev, exc_info=True diff --git a/dvc/repo/params/show.py b/dvc/repo/params/show.py index 8ea73b8f8..7ca660302 100644 --- a/dvc/repo/params/show.py +++ b/dvc/repo/params/show.py @@ -48,13 +48,13 @@ def _collect_configs(repo: "Repo", rev, targets=None) -> List["DvcPath"]: def _read_params(repo, configs, rev): res = {} for config in configs: - if not repo.tree.exists(config): + if not repo.fs.exists(config): continue suffix = config.suffix.lower() loader = LOADERS[suffix] try: - res[str(config)] = loader(config, tree=repo.tree) + res[str(config)] = loader(config, fs=repo.fs) except ParseError: logger.debug( "failed to read '%s' on '%s'", config, rev, exc_info=True diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index d0b795d90..3e69ec91e 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -49,7 +49,7 @@ class Plots: }}} Data parsing is postponed, since it's affected by props. """ - from dvc.tree.repo import RepoTree + from dvc.fs.repo import RepoFileSystem from dvc.utils.collections import ensure_list targets = ensure_list(targets) @@ -60,16 +60,16 @@ class Plots: continue rev = rev or "workspace" - tree = RepoTree(self.repo) + fs = RepoFileSystem(self.repo) plots = _collect_plots(self.repo, targets, rev, recursive) for path_info, props in plots.items(): if rev not in data: data[rev] = {} - if tree.isdir(path_info): + if fs.isdir(path_info): plot_files = [] - for pi in tree.walk_files(path_info): + for pi in fs.walk_files(path_info): plot_files.append( (pi, relpath(pi, self.repo.root_dir)) ) @@ -83,7 +83,7 @@ class Plots: # Load data from git or dvc cache try: - with tree.open(path) as fd: + with fs.open(path) as fd: data[rev][repo_path]["data"] = fd.read() except FileNotFoundError: # This might happen simply because cache is absent diff --git a/dvc/repo/stage.py b/dvc/repo/stage.py index cacae58f6..a45fb682b 100644 --- a/dvc/repo/stage.py +++ b/dvc/repo/stage.py @@ -62,7 +62,7 @@ def _maybe_collect_from_dvc_yaml( from dvc.stage.exceptions import StageNotFound stages: StageList = [] - if loader.tree.exists(PIPELINE_FILE): + if loader.fs.exists(PIPELINE_FILE): with suppress(StageNotFound): stages = loader.load_all(PIPELINE_FILE, target, **load_kwargs) return _collect_with_deps(stages, loader.graph) if with_deps else stages @@ -88,7 +88,7 @@ def _collect_specific_target( # else, we assume it's a output name in the `collect_granular()` below msg = "Checking if stage '%s' is in '%s'" logger.debug(msg, target, PIPELINE_FILE) - if not (recursive and loader.tree.isdir(target)): + if not (recursive and loader.fs.isdir(target)): stages = _maybe_collect_from_dvc_yaml( loader, target, with_deps, accept_group=accept_group, ) @@ -304,8 +304,8 @@ class StageLoad: return self.load_all(path, expr, glob=True) @property - def tree(self): - return self.repo.tree + def fs(self): + return self.repo.fs @property def graph(self) -> "DiGraph": @@ -353,7 +353,7 @@ class StageLoad: if not target: return list(graph) if graph else self.repo.stages - if recursive and self.repo.tree.isdir(target): + if recursive and self.repo.fs.isdir(target): from dvc.repo.graph import collect_inside_path path = os.path.abspath(target) @@ -398,7 +398,7 @@ class StageLoad: self, target, with_deps, recursive, accept_group ) if not stages: - if not (recursive and self.tree.isdir(target)): + if not (recursive and self.fs.isdir(target)): try: (out,) = self.repo.find_outs_by_path(target, strict=False) filter_info = PathInfo(os.path.abspath(target)) @@ -443,17 +443,17 @@ class StageLoad: the collection. """ from dvc.dvcfile import is_valid_filename - from dvc.tree.local import LocalTree + from dvc.fs.local import LocalFileSystem scm = self.repo.scm sep = os.sep outs: Set[str] = set() - is_local_tree = isinstance(self.tree, LocalTree) + is_local_fs = isinstance(self.fs, LocalFileSystem) def is_ignored(path): - # apply only for the local tree - return is_local_tree and scm.is_ignored(path) + # apply only for the local fs + return is_local_fs and scm.is_ignored(path) def is_dvcfile_and_not_ignored(root, file): return is_valid_filename(file) and not is_ignored( @@ -465,7 +465,7 @@ class StageLoad: return dir_path in outs or is_ignored(dir_path) stages = [] - for root, dirs, files in self.tree.walk(self.repo.root_dir): + for root, dirs, files in self.fs.walk(self.repo.root_dir): dvcfile_filter = partial(is_dvcfile_and_not_ignored, root) for file in filter(dvcfile_filter, files): diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 441635392..6d4868f3d 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -340,15 +340,15 @@ class Git(Base): pass raise NoGitBackendError(name) - def get_tree(self, rev: str, **kwargs): - from dvc.tree.git import GitTree + def get_fs(self, rev: str, **kwargs): + from dvc.fs.git import GitFileSystem from .objects import GitTrie resolved = self.resolve_rev(rev) tree_obj = self._backend_func("get_tree_obj", rev=resolved) trie = GitTrie(tree_obj, resolved) - return GitTree(self.root_dir, trie, **kwargs) + return GitFileSystem(self.root_dir, trie, **kwargs) is_ignored = partialmethod(_backend_func, "is_ignored") add = partialmethod(_backend_func, "add") diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index 270579505..4e0b5a78b 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -232,9 +232,9 @@ class StageCache: remote = self.repo.cloud.get_remote(remote) return self._transfer( - _log_exceptions(remote.tree.upload, "upload"), - self.repo.cache.local.tree, - remote.tree, + _log_exceptions(remote.fs.upload, "upload"), + self.repo.cache.local.fs, + remote.fs, ) def pull(self, remote): @@ -242,9 +242,9 @@ class StageCache: remote = self.repo.cloud.get_remote(remote) return self._transfer( - _log_exceptions(remote.tree.download, "download"), - remote.tree, - self.repo.cache.local.tree, + _log_exceptions(remote.fs.download, "download"), + remote.fs, + self.repo.cache.local.fs, ) def get_used_cache(self, used_run_cache, *args, **kwargs): diff --git a/dvc/state.py b/dvc/state.py index abc828a96..2827c64cb 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -133,13 +133,13 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes MAX_UINT = 2 ** 64 - 2 def __init__(self, repo): - from dvc.tree.local import LocalTree + from dvc.fs.local import LocalFileSystem super().__init__() self.repo = repo self.root_dir = repo.root_dir - self.tree = LocalTree(None, {"url": self.root_dir}) + self.fs = LocalFileSystem(None, {"url": self.root_dir}) state_config = repo.config.get("state", {}) self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT) @@ -410,7 +410,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes assert isinstance(hash_info, HashInfo) assert os.path.exists(path_info) - actual_mtime, actual_size = get_mtime_and_size(path_info, self.tree) + actual_mtime, actual_size = get_mtime_and_size(path_info, self.fs) actual_inode = get_inode(path_info) existing_record = self.get_state_record_for_inode(actual_inode) @@ -438,13 +438,13 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes assert isinstance(path_info, str) or path_info.scheme == "local" path = os.fspath(path_info) - # NOTE: use os.path.exists instead of LocalTree.exists + # NOTE: use os.path.exists instead of LocalFileSystem.exists # because it uses lexists() and will return True for broken # symlinks that we cannot stat() in get_mtime_and_size if not os.path.exists(path): return None - actual_mtime, actual_size = get_mtime_and_size(path, self.tree) + actual_mtime, actual_size = get_mtime_and_size(path, self.fs) actual_inode = get_inode(path) existing_record = self.get_state_record_for_inode(actual_inode) @@ -467,10 +467,10 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes """ assert isinstance(path_info, str) or path_info.scheme == "local" - if not self.tree.exists(path_info): + if not self.fs.exists(path_info): return - mtime, _ = get_mtime_and_size(path_info, self.tree) + mtime, _ = get_mtime_and_size(path_info, self.fs) inode = get_inode(path_info) relative_path = relpath(path_info, self.root_dir) @@ -493,11 +493,11 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes inode = self._from_sqlite(inode) path = os.path.join(self.root_dir, relative_path) - if path in used or not self.tree.exists(path): + if path in used or not self.fs.exists(path): continue actual_inode = get_inode(path) - actual_mtime, _ = get_mtime_and_size(path, self.tree) + actual_mtime, _ = get_mtime_and_size(path, self.fs) if (inode, mtime) == (actual_inode, actual_mtime): logger.debug("Removing '%s' as unused link.", path) diff --git a/dvc/tree/https.py b/dvc/tree/https.py deleted file mode 100644 index 1e48c2d7e..000000000 --- a/dvc/tree/https.py +++ /dev/null @@ -1,7 +0,0 @@ -from dvc.scheme import Schemes - -from .http import HTTPTree - - -class HTTPSTree(HTTPTree): # pylint:disable=abstract-method - scheme = Schemes.HTTPS diff --git a/dvc/tree/webdavs.py b/dvc/tree/webdavs.py deleted file mode 100644 index 570079507..000000000 --- a/dvc/tree/webdavs.py +++ /dev/null @@ -1,7 +0,0 @@ -from dvc.scheme import Schemes - -from .webdav import WebDAVTree - - -class WebDAVSTree(WebDAVTree): # pylint:disable=abstract-method - scheme = Schemes.WEBDAVS diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index a89034971..0785b6911 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -41,14 +41,14 @@ def _fobj_md5(fobj, hash_md5, binary, progress_func=None): progress_func(len(data)) -def file_md5(fname, tree): +def file_md5(fname, fs): """ get the (md5 hexdigest, md5 digest) of a file """ from dvc.istextfile import istextfile from dvc.progress import Tqdm hash_md5 = hashlib.md5() - binary = not istextfile(fname, tree=tree) - size = tree.getsize(fname) or 0 + binary = not istextfile(fname, fs=fs) + size = fs.getsize(fname) or 0 no_progress_bar = True if size >= LARGE_FILE_SIZE: no_progress_bar = False @@ -65,7 +65,7 @@ def file_md5(fname, tree): bytes=True, leave=False, ) as pbar: - with tree.open(fname, "rb") as fobj: + with fs.open(fname, "rb") as fobj: _fobj_md5(fobj, hash_md5, binary, pbar.update) return hash_md5.hexdigest() diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index 4227e29b9..6b088b00e 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -30,15 +30,15 @@ def get_inode(path): return inode -def get_mtime_and_size(path, tree): +def get_mtime_and_size(path, fs): import nanotime - if tree.isdir(path): + if fs.isdir(path): size = 0 files_mtimes = {} - for file_path in tree.walk_files(path): + for file_path in fs.walk_files(path): try: - stats = tree.stat(file_path) + stats = fs.stat(file_path) except OSError as exc: # NOTE: broken symlink case. if exc.errno != errno.ENOENT: @@ -51,7 +51,7 @@ def get_mtime_and_size(path, tree): # max(mtime(f) for f in non_ignored_files) mtime = dict_md5(files_mtimes) else: - base_stat = tree.stat(path) + base_stat = fs.stat(path) size = base_stat.st_size mtime = base_stat.st_mtime mtime = int(nanotime.timestamp(mtime)) diff --git a/dvc/utils/serialize/_common.py b/dvc/utils/serialize/_common.py index 36f14b00d..8e1a7b77e 100644 --- a/dvc/utils/serialize/_common.py +++ b/dvc/utils/serialize/_common.py @@ -8,26 +8,26 @@ from typing_extensions import Protocol from dvc.exceptions import DvcException if TYPE_CHECKING: - from dvc.tree.base import BaseTree + from dvc.fs.base import BaseFileSystem from dvc.types import AnyPath class DumperFn(Protocol): def __call__( - self, path: "AnyPath", data: Any, tree: "BaseTree" = None + self, path: "AnyPath", data: Any, fs: "BaseFileSystem" = None ) -> Any: ... class ModifierFn(Protocol): def __call__( - self, path: "AnyPath", tree: "BaseTree" = None + self, path: "AnyPath", fs: "BaseFileSystem" = None ) -> ContextManager[Dict]: ... class LoaderFn(Protocol): - def __call__(self, path: "AnyPath", tree: "BaseTree" = None) -> Any: + def __call__(self, path: "AnyPath", fs: "BaseFileSystem" = None) -> Any: ... @@ -45,14 +45,14 @@ class ParseError(DvcException): super().__init__(f"unable to read: '{path}', {message}") -def _load_data(path: "AnyPath", parser: ParserFn, tree: "BaseTree" = None): - open_fn = tree.open if tree else open +def _load_data(path: "AnyPath", parser: ParserFn, fs: "BaseFileSystem" = None): + open_fn = fs.open if fs else open with open_fn(path, encoding="utf-8") as fd: # type: ignore return parser(fd.read(), path) -def _dump_data(path, data: Any, dumper: DumperFn, tree: "BaseTree" = None): - open_fn = tree.open if tree else open +def _dump_data(path, data: Any, dumper: DumperFn, fs: "BaseFileSystem" = None): + open_fn = fs.open if fs else open with open_fn(path, "w+", encoding="utf-8") as fd: # type: ignore dumper(data, fd) @@ -62,10 +62,10 @@ def _modify_data( path: "AnyPath", parser: ParserFn, dumper: DumperFn, - tree: "BaseTree" = None, + fs: "BaseFileSystem" = None, ): - exists_fn = tree.exists if tree else os.path.exists + exists_fn = fs.exists if fs else os.path.exists file_exists = exists_fn(path) # type: ignore - data = _load_data(path, parser=parser, tree=tree) if file_exists else {} + data = _load_data(path, parser=parser, fs=fs) if file_exists else {} yield data - dumper(path, data, tree=tree) + dumper(path, data, fs=fs) diff --git a/dvc/utils/serialize/_json.py b/dvc/utils/serialize/_json.py index ad045016c..210d20bb0 100644 --- a/dvc/utils/serialize/_json.py +++ b/dvc/utils/serialize/_json.py @@ -10,8 +10,8 @@ class JSONFileCorruptedError(ParseError): super().__init__(path, "JSON file structure is corrupted") -def load_json(path, tree=None): - return _load_data(path, parser=parse_json, tree=tree) +def load_json(path, fs=None): + return _load_data(path, parser=parse_json, fs=fs) def parse_json(text, path, **kwargs): @@ -19,11 +19,11 @@ def parse_json(text, path, **kwargs): return json.loads(text, **kwargs) or {} -def dump_json(path, data, tree=None): - return _dump_data(path, data, dumper=json.dump, tree=tree) +def dump_json(path, data, fs=None): + return _dump_data(path, data, dumper=json.dump, fs=fs) @contextmanager -def modify_json(path, tree=None): - with _modify_data(path, parse_json, dump_json, tree=tree) as d: +def modify_json(path, fs=None): + with _modify_data(path, parse_json, dump_json, fs=fs) as d: yield d diff --git a/dvc/utils/serialize/_py.py b/dvc/utils/serialize/_py.py index e51f0d4f3..4911bf31d 100644 --- a/dvc/utils/serialize/_py.py +++ b/dvc/utils/serialize/_py.py @@ -14,8 +14,8 @@ class PythonFileCorruptedError(ParseError): super().__init__(path, message) -def load_py(path, tree=None): - return _load_data(path, parser=parse_py, tree=tree) +def load_py(path, fs=None): + return _load_data(path, parser=parse_py, fs=fs) def parse_py(text, path): @@ -76,13 +76,13 @@ def _dump(data, stream): stream.close() -def dump_py(path, data, tree=None): - return _dump_data(path, data, dumper=_dump, tree=tree) +def dump_py(path, data, fs=None): + return _dump_data(path, data, dumper=_dump, fs=fs) @contextmanager -def modify_py(path, tree=None): - with _modify_data(path, parse_py_for_update, dump_py, tree=tree) as d: +def modify_py(path, fs=None): + with _modify_data(path, parse_py_for_update, dump_py, fs=fs) as d: yield d diff --git a/dvc/utils/serialize/_toml.py b/dvc/utils/serialize/_toml.py index 05eee7a8d..559b4a11f 100644 --- a/dvc/utils/serialize/_toml.py +++ b/dvc/utils/serialize/_toml.py @@ -10,8 +10,8 @@ class TOMLFileCorruptedError(ParseError): super().__init__(path, "TOML file structure is corrupted") -def load_toml(path, tree=None): - return _load_data(path, parser=parse_toml, tree=tree) +def load_toml(path, fs=None): + return _load_data(path, parser=parse_toml, fs=fs) def parse_toml(text, path, decoder=None): @@ -40,11 +40,11 @@ def _dump(data, stream): return toml.dump(data, stream, encoder=toml.TomlPreserveCommentEncoder()) -def dump_toml(path, data, tree=None): - return _dump_data(path, data, dumper=_dump, tree=tree) +def dump_toml(path, data, fs=None): + return _dump_data(path, data, dumper=_dump, fs=fs) @contextmanager -def modify_toml(path, tree=None): - with _modify_data(path, parse_toml_for_update, dump_toml, tree=tree) as d: +def modify_toml(path, fs=None): + with _modify_data(path, parse_toml_for_update, dump_toml, fs=fs) as d: yield d diff --git a/dvc/utils/serialize/_yaml.py b/dvc/utils/serialize/_yaml.py index 15ba436c9..72d14494a 100644 --- a/dvc/utils/serialize/_yaml.py +++ b/dvc/utils/serialize/_yaml.py @@ -16,8 +16,8 @@ class YAMLFileCorruptedError(YAMLError): super().__init__(path, "YAML file structure is corrupted") -def load_yaml(path, tree=None): - return _load_data(path, parser=parse_yaml, tree=tree) +def load_yaml(path, fs=None): + return _load_data(path, parser=parse_yaml, fs=fs) def parse_yaml(text, path, typ="safe"): @@ -65,8 +65,8 @@ def _dump(data, stream): return yaml.dump(data, stream) -def dump_yaml(path, data, tree=None): - return _dump_data(path, data, dumper=_dump, tree=tree) +def dump_yaml(path, data, fs=None): + return _dump_data(path, data, dumper=_dump, fs=fs) def loads_yaml(s, typ="safe"): @@ -82,6 +82,6 @@ def dumps_yaml(d): @contextmanager -def modify_yaml(path, tree=None): - with _modify_data(path, parse_yaml_for_update, dump_yaml, tree=tree) as d: +def modify_yaml(path, fs=None): + with _modify_data(path, parse_yaml_for_update, dump_yaml, fs=fs) as d: yield d diff --git a/tests/conftest.py b/tests/conftest.py index 72797fe58..e5d906fa1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,7 +45,7 @@ def reset_loglevel(request, caplog): @pytest.fixture(scope="session", autouse=True) def _close_pools(): - from dvc.tree.pool import close_pools + from dvc.fs.pool import close_pools yield close_pools() diff --git a/tests/func/api/test_api.py b/tests/func/api/test_api.py index 3fa0f4aa4..5a5555479 100644 --- a/tests/func/api/test_api.py +++ b/tests/func/api/test_api.py @@ -8,7 +8,7 @@ from dvc import api from dvc.exceptions import FileMissingError, OutputNotFoundError from dvc.path_info import URLInfo from dvc.utils.fs import remove -from tests.unit.tree.test_repo import make_subrepo +from tests.unit.fs.test_repo import make_subrepo cloud_names = [ "s3", diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index 3c707d992..11b65ccd6 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -20,10 +20,10 @@ def test_new_checkpoint( for rev in dvc.brancher([exp]): if rev == "workspace": continue - tree = dvc.repo_tree - with tree.open(tmp_dir / "foo") as fobj: + fs = dvc.repo_fs + with fs.open(tmp_dir / "foo") as fobj: assert fobj.read().strip() == str(checkpoint_stage.iterations) - with tree.open(tmp_dir / "metrics.yaml") as fobj: + with fs.open(tmp_dir / "metrics.yaml") as fobj: assert fobj.read().strip() == "foo: 2" if workspace: @@ -69,10 +69,10 @@ def test_resume_checkpoint( for rev in dvc.brancher([exp]): if rev == "workspace": continue - tree = dvc.repo_tree - with tree.open(tmp_dir / "foo") as fobj: + fs = dvc.repo_fs + with fs.open(tmp_dir / "foo") as fobj: assert fobj.read().strip() == str(2 * checkpoint_stage.iterations) - with tree.open(tmp_dir / "metrics.yaml") as fobj: + with fs.open(tmp_dir / "metrics.yaml") as fobj: assert fobj.read().strip() == "foo: 2" if workspace: @@ -104,10 +104,10 @@ def test_reset_checkpoint( for rev in dvc.brancher([exp]): if rev == "workspace": continue - tree = dvc.repo_tree - with tree.open(tmp_dir / "foo") as fobj: + fs = dvc.repo_fs + with fs.open(tmp_dir / "foo") as fobj: assert fobj.read().strip() == str(checkpoint_stage.iterations) - with tree.open(tmp_dir / "metrics.yaml") as fobj: + with fs.open(tmp_dir / "metrics.yaml") as fobj: assert fobj.read().strip() == "foo: 2" if workspace: @@ -143,19 +143,19 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage, workspace): for rev in dvc.brancher([checkpoint_a]): if rev == "workspace": continue - tree = dvc.repo_tree - with tree.open(tmp_dir / "foo") as fobj: + fs = dvc.repo_fs + with fs.open(tmp_dir / "foo") as fobj: assert fobj.read().strip() == str(2 * checkpoint_stage.iterations) - with tree.open(tmp_dir / "metrics.yaml") as fobj: + with fs.open(tmp_dir / "metrics.yaml") as fobj: assert fobj.read().strip() == "foo: 2" for rev in dvc.brancher([checkpoint_b]): if rev == "workspace": continue - tree = dvc.repo_tree - with tree.open(tmp_dir / "foo") as fobj: + fs = dvc.repo_fs + with fs.open(tmp_dir / "foo") as fobj: assert fobj.read().strip() == str(2 * checkpoint_stage.iterations) - with tree.open(tmp_dir / "metrics.yaml") as fobj: + with fs.open(tmp_dir / "metrics.yaml") as fobj: assert fobj.read().strip() == "foo: 100" with pytest.raises(MultipleBranchError): diff --git a/tests/func/experiments/test_experiments.py b/tests/func/experiments/test_experiments.py index 7c81f14f5..0f81e7b2b 100644 --- a/tests/func/experiments/test_experiments.py +++ b/tests/func/experiments/test_experiments.py @@ -28,8 +28,8 @@ def test_new_simple(tmp_dir, scm, dvc, exp_stage, mocker, name, workspace): assert ref_info and ref_info.baseline_sha == baseline new_mock.assert_called_once() - tree = scm.get_tree(exp) - with tree.open(tmp_dir / "metrics.yaml") as fobj: + fs = scm.get_fs(exp) + with fs.open(tmp_dir / "metrics.yaml") as fobj: assert fobj.read().strip() == "foo: 2" if workspace: @@ -68,8 +68,8 @@ def test_experiment_exists(tmp_dir, scm, dvc, exp_stage, mocker, workspace): ) exp = first(results) - tree = scm.get_tree(exp) - with tree.open(tmp_dir / "metrics.yaml") as fobj: + fs = scm.get_fs(exp) + with fs.open(tmp_dir / "metrics.yaml") as fobj: assert fobj.read().strip() == "foo: 3" @@ -142,8 +142,8 @@ def test_modify_params(tmp_dir, scm, dvc, mocker, changes, expected): exp = first(results) new_mock.assert_called_once() - tree = scm.get_tree(exp) - with tree.open(tmp_dir / "metrics.yaml") as fobj: + fs = scm.get_fs(exp) + with fs.open(tmp_dir / "metrics.yaml") as fobj: assert fobj.read().strip() == expected @@ -238,10 +238,10 @@ def test_update_py_params(tmp_dir, scm, dvc): ) exp_a = first(results) - tree = scm.get_tree(exp_a) - with tree.open(tmp_dir / "params.py") as fobj: + fs = scm.get_fs(exp_a) + with fs.open(tmp_dir / "params.py") as fobj: assert fobj.read().strip() == "INT = 2" - with tree.open(tmp_dir / "metrics.py") as fobj: + with fs.open(tmp_dir / "metrics.py") as fobj: assert fobj.read().strip() == "INT = 2" tmp_dir.gen( @@ -280,10 +280,10 @@ def test_update_py_params(tmp_dir, scm, dvc): # in order to compare with the original return text.replace("\r\n", "\n") - tree = scm.get_tree(exp_a) - with tree.open(tmp_dir / "params.py") as fobj: + fs = scm.get_fs(exp_a) + with fs.open(tmp_dir / "params.py") as fobj: assert _dos2unix(fobj.read().strip()) == result - with tree.open(tmp_dir / "metrics.py") as fobj: + with fs.open(tmp_dir / "metrics.py") as fobj: assert _dos2unix(fobj.read().strip()) == result tmp_dir.gen("params.py", "INT = 1\n") @@ -406,11 +406,11 @@ def test_untracked(tmp_dir, scm, dvc, caplog, workspace): stage.addressing, params=["foo=2"], tmp_dir=not workspace ) exp = first(results) - tree = scm.get_tree(exp) - assert tree.exists("dvc.yaml") - assert tree.exists("dvc.lock") - assert tree.exists("copy.py") - with tree.open(tmp_dir / "metrics.yaml") as fobj: + fs = scm.get_fs(exp) + assert fs.exists("dvc.yaml") + assert fs.exists("dvc.lock") + assert fs.exists("copy.py") + with fs.open(tmp_dir / "metrics.yaml") as fobj: assert fobj.read().strip() == "foo: 2" @@ -428,8 +428,8 @@ def test_dirty_lockfile(tmp_dir, scm, dvc, exp_stage, workspace): ) exp = first(results) - tree = scm.get_tree(exp) - with tree.open(tmp_dir / "metrics.yaml") as fobj: + fs = scm.get_fs(exp) + with fs.open(tmp_dir / "metrics.yaml") as fobj: assert fobj.read().strip() == "foo: 2" if not workspace: @@ -509,10 +509,10 @@ def test_subdir(tmp_dir, scm, dvc, workspace): exp = first(results) ref_info = first(exp_refs_by_rev(scm, exp)) - tree = scm.get_tree(exp) + fs = scm.get_fs(exp) for fname in ["metrics.yaml", "dvc.lock"]: - assert tree.exists(subdir / fname) - with tree.open(subdir / "metrics.yaml") as fobj: + assert fs.exists(subdir / fname) + with fs.open(subdir / "metrics.yaml") as fobj: assert fobj.read().strip() == "foo: 2" assert dvc.experiments.get_exact_name(exp) == ref_info.name @@ -521,7 +521,7 @@ def test_subdir(tmp_dir, scm, dvc, workspace): @pytest.mark.parametrize("workspace", [True, False]) def test_subrepo(tmp_dir, scm, workspace): - from tests.unit.tree.test_repo import make_subrepo + from tests.unit.fs.test_repo import make_subrepo subrepo = tmp_dir / "dir" / "repo" make_subrepo(subrepo, scm) @@ -554,10 +554,10 @@ def test_subrepo(tmp_dir, scm, workspace): exp = first(results) ref_info = first(exp_refs_by_rev(scm, exp)) - tree = scm.get_tree(exp) + fs = scm.get_fs(exp) for fname in ["metrics.yaml", "dvc.lock"]: - assert tree.exists(subrepo / fname) - with tree.open(subrepo / "metrics.yaml") as fobj: + assert fs.exists(subrepo / fname) + with fs.open(subrepo / "metrics.yaml") as fobj: assert fobj.read().strip() == "foo: 2" assert subrepo.dvc.experiments.get_exact_name(exp) == ref_info.name @@ -577,8 +577,8 @@ def test_queue(tmp_dir, scm, dvc, exp_stage, mocker): expected = {"foo: 2", "foo: 3"} metrics = set() for exp in results: - tree = scm.get_tree(exp) - with tree.open(tmp_dir / "metrics.yaml") as fobj: + fs = scm.get_fs(exp) + with fs.open(tmp_dir / "metrics.yaml") as fobj: metrics.add(fobj.read().strip()) assert expected == metrics diff --git a/tests/func/experiments/test_show.py b/tests/func/experiments/test_show.py index 8f91ca985..b18bb1556 100644 --- a/tests/func/experiments/test_show.py +++ b/tests/func/experiments/test_show.py @@ -108,14 +108,14 @@ def test_show_checkpoint( for i, rev in enumerate(checkpoints): if i == 0: name = dvc.experiments.get_exact_name(rev) - tree = "╓" + fs = "╓" elif i == len(checkpoints) - 1: name = rev[:7] - tree = "╨" + fs = "╨" else: name = rev[:7] - tree = "╟" - assert f"{tree} {name}" in cap.out + fs = "╟" + assert f"{fs} {name}" in cap.out @pytest.mark.parametrize("workspace", [True, False]) diff --git a/tests/func/remote/test_index.py b/tests/func/remote/test_index.py index cc8ff0053..ea931f8df 100644 --- a/tests/func/remote/test_index.py +++ b/tests/func/remote/test_index.py @@ -3,10 +3,10 @@ import os import pytest from dvc.exceptions import DownloadError, UploadError +from dvc.fs.local import LocalFileSystem from dvc.remote.base import Remote from dvc.remote.index import RemoteIndex from dvc.remote.local import LocalRemote -from dvc.tree.local import LocalTree from dvc.utils.fs import remove @@ -17,7 +17,7 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker): dvc.config["core"]["remote"] = "upstream" # patch hashes_exist since the LocalRemote normally overrides - # BaseTree.hashes_exist. + # BaseFileSystem.hashes_exist. def hashes_exist(self, *args, **kwargs): return Remote.hashes_exist(self, *args, **kwargs) @@ -85,7 +85,7 @@ def test_clear_on_download_err(tmp_dir, dvc, tmp_path_factory, remote, mocker): remove(dvc.cache.local.cache_dir) mocked_clear = mocker.patch.object(remote.INDEX_CLS, "clear") - mocker.patch.object(LocalTree, "_download", side_effect=Exception) + mocker.patch.object(LocalFileSystem, "_download", side_effect=Exception) with pytest.raises(DownloadError): dvc.pull() mocked_clear.assert_called_once_with() @@ -95,14 +95,14 @@ def test_partial_upload(tmp_dir, dvc, tmp_path_factory, remote, mocker): tmp_dir.dvc_gen({"foo": "foo content"}) tmp_dir.dvc_gen({"bar": {"baz": "baz content"}}) - original = LocalTree._upload + original = LocalFileSystem._upload def unreliable_upload(self, from_file, to_info, name=None, **kwargs): if "baz" in name: raise Exception("stop baz") return original(self, from_file, to_info, name, **kwargs) - mocker.patch.object(LocalTree, "_upload", unreliable_upload) + mocker.patch.object(LocalFileSystem, "_upload", unreliable_upload) with pytest.raises(UploadError): dvc.push() with remote.index: diff --git a/tests/func/test_add.py b/tests/func/test_add.py index c1bc89dcb..15438d477 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -21,6 +21,7 @@ from dvc.exceptions import ( OverlappingOutputPathsError, RecursiveAddingWhileUsingFilename, ) +from dvc.fs.local import LocalFileSystem from dvc.hash_info import HashInfo from dvc.main import main from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError @@ -28,7 +29,6 @@ from dvc.repo import Repo as DvcRepo from dvc.stage import Stage from dvc.stage.exceptions import StagePathNotFoundError from dvc.system import System -from dvc.tree.local import LocalTree from dvc.utils import LARGE_DIR_SIZE, file_md5, relpath from dvc.utils.fs import path_isin from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml @@ -38,7 +38,7 @@ from tests.utils import get_gitignore_content def test_add(tmp_dir, dvc): (stage,) = tmp_dir.dvc_gen({"foo": "foo"}) - md5 = file_md5("foo", dvc.tree) + md5 = file_md5("foo", dvc.fs) assert stage is not None @@ -732,7 +732,7 @@ def test_should_not_checkout_when_adding_cached_copy(tmp_dir, dvc, mocker): shutil.copy("bar", "foo") - copy_spy = mocker.spy(dvc.cache.local.tree, "copy") + copy_spy = mocker.spy(dvc.cache.local.fs, "copy") dvc.add("foo") @@ -758,7 +758,7 @@ def test_should_relink_on_repeated_add( tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) os.remove("foo") - getattr(dvc.cache.local.tree, link)(PathInfo("bar"), PathInfo("foo")) + getattr(dvc.cache.local.fs, link)(PathInfo("bar"), PathInfo("foo")) dvc.cache.local.cache_types = [new_link] @@ -838,7 +838,7 @@ def test_add_empty_files(tmp_dir, dvc, link): def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker): dvc.cache.local.cache_types = ["hardlink"] tmp_dir.gen({"foo": "", "bar": "", "lorem": "lorem", "ipsum": "ipsum"}) - m = mocker.spy(LocalTree, "is_hardlink") + m = mocker.spy(LocalFileSystem, "is_hardlink") stages = dvc.add(["foo", "bar", "lorem", "ipsum"]) assert m.call_count == 1 diff --git a/tests/func/test_cache.py b/tests/func/test_cache.py index cd6d03933..62d178cc4 100644 --- a/tests/func/test_cache.py +++ b/tests/func/test_cache.py @@ -89,7 +89,7 @@ class TestExternalCacheDir(TestDvc): self.dvc.__init__() - assert self.dvc.cache.ssh.tree.path_info == ssh_url + "/tmp" + assert self.dvc.cache.ssh.fs.path_info == ssh_url + "/tmp" class TestSharedCacheDir(TestDir): diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 686087841..d9d4ca544 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -17,12 +17,12 @@ from dvc.exceptions import ( DvcException, NoOutputOrStageError, ) +from dvc.fs.local import LocalFileSystem from dvc.main import main from dvc.repo import Repo as DvcRepo from dvc.stage import Stage from dvc.stage.exceptions import StageFileDoesNotExistError from dvc.system import System -from dvc.tree.local import LocalTree from dvc.utils import relpath from dvc.utils.fs import remove, walk_files from dvc.utils.serialize import dump_yaml, load_yaml @@ -148,7 +148,7 @@ class CheckoutBase(TestDvcGit): paths = [ path for output in stage["outs"] - for path in self.dvc.tree.walk_files(output["path"]) + for path in self.dvc.fs.walk_files(output["path"]) ] return [ @@ -312,8 +312,8 @@ class TestGitIgnoreWhenCheckout(CheckoutBase): class TestCheckoutMissingMd5InStageFile(TestRepro): def test(self): d = load_yaml(self.file1_stage) - del d[Stage.PARAM_OUTS][0][LocalTree.PARAM_CHECKSUM] - del d[Stage.PARAM_DEPS][0][LocalTree.PARAM_CHECKSUM] + del d[Stage.PARAM_OUTS][0][LocalFileSystem.PARAM_CHECKSUM] + del d[Stage.PARAM_DEPS][0][LocalFileSystem.PARAM_CHECKSUM] dump_yaml(self.file1_stage, d) with pytest.raises(CheckoutError): @@ -769,7 +769,7 @@ def test_checkout_for_external_outputs(tmp_dir, dvc, workspace): dvc.add("remote://workspace/foo") remote = dvc.cloud.get_remote("workspace") - remote.tree.remove(file_path) + remote.fs.remove(file_path) assert not file_path.exists() stats = dvc.checkout(force=True) diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 764619987..98504c12c 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -3,13 +3,13 @@ import os from mock import ANY, patch from dvc.external_repo import CLONES, external_repo +from dvc.fs.local import LocalFileSystem from dvc.objects import save, stage from dvc.path_info import PathInfo from dvc.scm.git import Git -from dvc.tree.local import LocalTree from dvc.utils import relpath from dvc.utils.fs import makedirs, remove -from tests.unit.tree.test_repo import make_subrepo +from tests.unit.fs.test_repo import make_subrepo def test_external_repo(erepo_dir): @@ -51,7 +51,7 @@ def test_cache_reused(erepo_dir, mocker, local_cloud): erepo_dir.dvc_gen("file", "text", commit="add file") erepo_dir.dvc.push() - download_spy = mocker.spy(LocalTree, "download") + download_spy = mocker.spy(LocalFileSystem, "download") # Use URL to prevent any fishy optimizations url = f"file://{erepo_dir}" @@ -92,7 +92,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir): dest = tmp_dir / "file" with external_repo(os.fspath(erepo_dir)) as repo: - repo.repo_tree.download( + repo.repo_fs.download( PathInfo(repo.root_dir) / "subdir" / "file", PathInfo(dest), ) @@ -187,7 +187,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir): subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo") with external_repo(os.fspath(erepo_dir)) as repo: - repo.repo_tree.download( + repo.repo_fs.download( PathInfo(repo.root_dir) / "dir", PathInfo(tmp_dir / "out"), follow_subrepos=False, @@ -203,7 +203,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir): obj = stage( repo.cache.local, PathInfo(repo.root_dir) / "dir", - repo.repo_tree, + repo.repo_fs, follow_subrepos=False, ) save(repo.cache.local, obj) @@ -226,7 +226,7 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir): subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo") with external_repo(os.fspath(erepo_dir)) as repo: - repo.repo_tree.download( + repo.repo_fs.download( PathInfo(repo.root_dir) / "dir", PathInfo(tmp_dir / "out"), follow_subrepos=False, diff --git a/tests/func/test_fs.py b/tests/func/test_fs.py index b73b5b681..f5330ee53 100644 --- a/tests/func/test_fs.py +++ b/tests/func/test_fs.py @@ -1,36 +1,351 @@ import os -import stat +from operator import itemgetter +from os.path import join import pytest -from dvc.utils.fs import copyfile, makedirs +from dvc.fs import get_cloud_fs +from dvc.fs.local import LocalFileSystem +from dvc.path_info import PathInfo +from dvc.repo import Repo +from dvc.scm import SCM +from tests.basic_env import TestDir, TestGit, TestGitSubmodule -@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.") -def test_makedirs_permissions(tmp_dir): - dir_mode = 0o755 - intermediate_dir = "тестовая-директория" - test_dir = os.path.join(intermediate_dir, "data") +class TestLocalFileSystem(TestDir): + def setUp(self): + super().setUp() + self.fs = LocalFileSystem(None, {}) - assert not os.path.exists(intermediate_dir) + def test_open(self): + with self.fs.open(self.FOO) as fd: + self.assertEqual(fd.read(), self.FOO_CONTENTS) + with self.fs.open(self.UNICODE, encoding="utf-8") as fd: + self.assertEqual(fd.read(), self.UNICODE_CONTENTS) - makedirs(test_dir, mode=dir_mode) + def test_exists(self): + self.assertTrue(self.fs.exists(self.FOO)) + self.assertTrue(self.fs.exists(self.UNICODE)) + self.assertFalse(self.fs.exists("not-existing-file")) - assert stat.S_IMODE(os.stat(test_dir).st_mode) == dir_mode - assert stat.S_IMODE(os.stat(intermediate_dir).st_mode) == dir_mode + def test_isdir(self): + self.assertTrue(self.fs.isdir(self.DATA_DIR)) + self.assertFalse(self.fs.isdir(self.FOO)) + self.assertFalse(self.fs.isdir("not-existing-file")) + + def test_isfile(self): + self.assertTrue(self.fs.isfile(self.FOO)) + self.assertFalse(self.fs.isfile(self.DATA_DIR)) + self.assertFalse(self.fs.isfile("not-existing-file")) -def test_copyfile(tmp_dir): - src = "file1" - dest = "file2" - dest_dir = "testdir" +class GitFileSystemTests: + # pylint: disable=no-member + def test_open(self): + self.scm.add([self.FOO, self.UNICODE, self.DATA_DIR]) + self.scm.commit("add") - tmp_dir.gen(src, "file1contents") + fs = self.scm.get_fs("master") + with fs.open(self.FOO) as fd: + self.assertEqual(fd.read(), self.FOO_CONTENTS) + with fs.open(self.UNICODE) as fd: + self.assertEqual(fd.read(), self.UNICODE_CONTENTS) + with self.assertRaises(IOError): + fs.open("not-existing-file") + with self.assertRaises(IOError): + fs.open(self.DATA_DIR) - os.mkdir(dest_dir) + def test_exists(self): + fs = self.scm.get_fs("master") + self.assertFalse(fs.exists(self.FOO)) + self.assertFalse(fs.exists(self.UNICODE)) + self.assertFalse(fs.exists(self.DATA_DIR)) + self.scm.add([self.FOO, self.UNICODE, self.DATA]) + self.scm.commit("add") - copyfile(src, dest) - assert (tmp_dir / dest).read_text() == "file1contents" + fs = self.scm.get_fs("master") + self.assertTrue(fs.exists(self.FOO)) + self.assertTrue(fs.exists(self.UNICODE)) + self.assertTrue(fs.exists(self.DATA_DIR)) + self.assertFalse(fs.exists("non-existing-file")) - copyfile(src, dest_dir) - assert (tmp_dir / dest_dir / src).read_text() == "file1contents" + def test_isdir(self): + self.scm.add([self.FOO, self.DATA_DIR]) + self.scm.commit("add") + + fs = self.scm.get_fs("master") + self.assertTrue(fs.isdir(self.DATA_DIR)) + self.assertFalse(fs.isdir(self.FOO)) + self.assertFalse(fs.isdir("non-existing-file")) + + def test_isfile(self): + self.scm.add([self.FOO, self.DATA_DIR]) + self.scm.commit("add") + + fs = self.scm.get_fs("master") + self.assertTrue(fs.isfile(self.FOO)) + self.assertFalse(fs.isfile(self.DATA_DIR)) + self.assertFalse(fs.isfile("not-existing-file")) + + +class TestGitFileSystem(TestGit, GitFileSystemTests): + def setUp(self): + super().setUp() + self.scm = SCM(self._root_dir) + + +class TestGitSubmoduleFileSystem(TestGitSubmodule, GitFileSystemTests): + def setUp(self): + super().setUp() + self.scm = SCM(self._root_dir) + self._pushd(self._root_dir) + + +class AssertWalkEqualMixin: + def assertWalkEqual(self, actual, expected, msg=None): + def convert_to_sets(walk_results): + return [ + (root, set(dirs), set(nondirs)) + for root, dirs, nondirs in walk_results + ] + + self.assertEqual( + convert_to_sets(actual), convert_to_sets(expected), msg=msg + ) + + +class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir): + def test(self): + fs = LocalFileSystem(None, {"url": self._root_dir}) + self.assertWalkEqual( + fs.walk(self._root_dir), + [ + ( + self._root_dir, + ["data_dir"], + ["code.py", "bar", "тест", "foo"], + ), + (join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]), + ( + join(self._root_dir, "data_dir", "data_sub_dir"), + [], + ["data_sub"], + ), + ], + ) + + def test_subdir(self): + fs = LocalFileSystem(None, {"url": self._root_dir}) + self.assertWalkEqual( + fs.walk(join("data_dir", "data_sub_dir")), + [(join("data_dir", "data_sub_dir"), [], ["data_sub"])], + ) + + +class TestWalkInGit(AssertWalkEqualMixin, TestGit): + def test_nobranch(self): + fs = LocalFileSystem(None, {"url": self._root_dir}, use_dvcignore=True) + self.assertWalkEqual( + fs.walk("."), + [ + (".", ["data_dir"], ["bar", "тест", "code.py", "foo"]), + (join("data_dir"), ["data_sub_dir"], ["data"]), + (join("data_dir", "data_sub_dir"), [], ["data_sub"]), + ], + ) + self.assertWalkEqual( + fs.walk(join("data_dir", "data_sub_dir")), + [(join("data_dir", "data_sub_dir"), [], ["data_sub"])], + ) + + def test_branch(self): + scm = SCM(self._root_dir) + scm.add([self.DATA_SUB_DIR]) + scm.commit("add data_dir/data_sub_dir/data_sub") + fs = scm.get_fs("master") + self.assertWalkEqual( + fs.walk("."), + [ + (self._root_dir, ["data_dir"], ["code.py"]), + (join(self._root_dir, "data_dir"), ["data_sub_dir"], []), + ( + join(self._root_dir, "data_dir", "data_sub_dir"), + [], + ["data_sub"], + ), + ], + ) + self.assertWalkEqual( + fs.walk(join("data_dir", "data_sub_dir")), + [ + ( + join(self._root_dir, "data_dir", "data_sub_dir"), + [], + ["data_sub"], + ) + ], + ) + + +def test_cleanfs_subrepo(tmp_dir, dvc, scm, monkeypatch): + tmp_dir.gen({"subdir": {}}) + subrepo_dir = tmp_dir / "subdir" + with subrepo_dir.chdir(): + subrepo = Repo.init(subdir=True) + subrepo_dir.gen({"foo": "foo", "dir": {"bar": "bar"}}) + + path = PathInfo(subrepo_dir) + + assert dvc.fs.use_dvcignore + assert not dvc.fs.exists(path / "foo") + assert not dvc.fs.isfile(path / "foo") + assert not dvc.fs.exists(path / "dir") + assert not dvc.fs.isdir(path / "dir") + + assert subrepo.fs.use_dvcignore + assert subrepo.fs.exists(path / "foo") + assert subrepo.fs.isfile(path / "foo") + assert subrepo.fs.exists(path / "dir") + assert subrepo.fs.isdir(path / "dir") + + +def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc): + tmp_dir.dvc_gen({"foo": "foo"}, commit="add foo") + subrepo_dir = tmp_dir / "subdir" + subrepo_dir.mkdir() + with subrepo_dir.chdir(): + Repo.init(subdir=True) + scm.add(["subdir"]) + scm.commit("Add subrepo") + + dvc_fs = dvc.fs + dvc_fs._reset() + scm_fs = scm.get_fs("HEAD", use_dvcignore=True) + path = os.fspath(tmp_dir) + get_dirs = itemgetter(1) + + assert get_dirs(next(dvc_fs.walk(path))) == [] + assert get_dirs(next(scm_fs.walk(path))) == [] + + kw = {"ignore_subrepos": False} + assert get_dirs(next(dvc_fs.walk(path, **kw))) == ["subdir"] + assert get_dirs(next(scm_fs.walk(path, **kw))) == ["subdir"] + + +@pytest.mark.parametrize( + "cloud", + [ + pytest.lazy_fixture("local_cloud"), + pytest.lazy_fixture("s3"), + pytest.lazy_fixture("gs"), + pytest.lazy_fixture("hdfs"), + pytest.lazy_fixture("http"), + ], +) +def test_fs_getsize(dvc, cloud): + cloud.gen({"data": {"foo": "foo"}, "baz": "baz baz"}) + fs = get_cloud_fs(dvc, **cloud.config) + path_info = fs.path_info + + assert fs.getsize(path_info / "baz") == 7 + assert fs.getsize(path_info / "data" / "foo") == 3 + + +@pytest.mark.parametrize( + "cloud", + [ + pytest.lazy_fixture("azure"), + pytest.lazy_fixture("gs"), + pytest.lazy_fixture("gdrive"), + pytest.lazy_fixture("hdfs"), + pytest.lazy_fixture("http"), + pytest.lazy_fixture("local_cloud"), + pytest.lazy_fixture("oss"), + pytest.lazy_fixture("s3"), + pytest.lazy_fixture("ssh"), + pytest.lazy_fixture("webhdfs"), + ], +) +def test_fs_upload_fobj(dvc, tmp_dir, cloud): + tmp_dir.gen("foo", "foo") + fs = get_cloud_fs(dvc, **cloud.config) + + from_info = tmp_dir / "foo" + to_info = fs.path_info / "foo" + + with open(from_info, "rb") as stream: + fs.upload_fobj(stream, to_info) + + assert fs.exists(to_info) + with fs.open(to_info, "rb") as stream: + assert stream.read() == b"foo" + + +@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("gdrive")]) +def test_fs_ls(dvc, cloud): + cloud.gen( + { + "directory": { + "foo": "foo", + "bar": "bar", + "baz": {"quux": "quux", "egg": {"foo": "foo"}}, + "empty": {}, + } + } + ) + fs = get_cloud_fs(dvc, **cloud.config) + path_info = cloud / "directory" + + assert {os.path.basename(file_key) for file_key in fs.ls(path_info)} == { + "foo", + "bar", + "baz", + "empty", + } + assert set(fs.ls(path_info / "empty")) == set() + assert { + (detail["type"], os.path.basename(detail["name"])) + for detail in fs.ls(path_info / "baz", detail=True) + } == {("file", "quux"), ("directory", "egg")} + + +@pytest.mark.parametrize( + "cloud", + [ + pytest.lazy_fixture("s3"), + pytest.lazy_fixture("azure"), + pytest.lazy_fixture("gs"), + pytest.lazy_fixture("webdav"), + pytest.lazy_fixture("gdrive"), + ], +) +def test_fs_ls_recursive(dvc, cloud): + cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}}) + fs = get_cloud_fs(dvc, **cloud.config) + path_info = fs.path_info + + assert { + os.path.basename(file_key) + for file_key in fs.ls(path_info / "data", recursive=True) + } == {"foo", "baz", "quux"} + + +@pytest.mark.parametrize( + "cloud", + [ + pytest.lazy_fixture("s3"), + pytest.lazy_fixture("azure"), + pytest.lazy_fixture("gs"), + pytest.lazy_fixture("webdav"), + ], +) +def test_fs_ls_with_etag(dvc, cloud): + cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}}) + fs = get_cloud_fs(dvc, **cloud.config) + path_info = fs.path_info + + for details in fs.ls(path_info / "data", recursive=True, detail=True): + assert ( + fs.info(path_info.replace(path=details["name"]))["etag"] + == details["etag"] + ) diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index 33a91dff5..95314f208 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -8,9 +8,9 @@ from git import Repo from dvc.cache.local import LocalCache from dvc.exceptions import CollectCacheError +from dvc.fs.local import LocalFileSystem from dvc.main import main from dvc.repo import Repo as DvcRepo -from dvc.tree.local import LocalTree from dvc.utils.fs import remove from tests.basic_env import TestDir, TestDvcGit @@ -321,7 +321,9 @@ def test_gc_cloud_remove_order(tmp_dir, scm, dvc, tmp_path_factory, mocker): dvc.remove(dir2.relpath) dvc.gc(workspace=True) - mocked_remove = mocker.patch.object(LocalTree, "remove", autospec=True) + mocked_remove = mocker.patch.object( + LocalFileSystem, "remove", autospec=True + ) dvc.gc(workspace=True, cloud=True) assert len(mocked_remove.mock_calls) == 8 # dir (and unpacked dir) should be first 4 checksums removed from diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 2890e38fb..202096cd2 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -9,7 +9,7 @@ from dvc.repo import Repo from dvc.repo.get import GetDVCFileError from dvc.system import System from dvc.utils.fs import makedirs -from tests.unit.tree.test_repo import make_subrepo +from tests.unit.fs.test_repo import make_subrepo def test_get_repo_file(tmp_dir, erepo_dir): diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 48b4f4d5b..65f806bcf 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -23,41 +23,41 @@ def test_ignore(tmp_dir, dvc, monkeypatch): tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - dvc.tree._reset() + dvc.fs._reset() path = PathInfo(tmp_dir) - assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"} + assert set(dvc.fs.walk_files(path / "dir")) == {path / "dir" / "other"} def test_ignore_unicode(tmp_dir, dvc): tmp_dir.gen({"dir": {"other": "text", "тест": "проверка"}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест") - dvc.tree._reset() + dvc.fs._reset() path = PathInfo(tmp_dir) - assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"} + assert set(dvc.fs.walk_files(path / "dir")) == {path / "dir" / "other"} def test_rename_ignored_file(tmp_dir, dvc): tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*") - dvc.tree._reset() + dvc.fs._reset() - mtime, size = get_mtime_and_size("dir", dvc.tree) + mtime, size = get_mtime_and_size("dir", dvc.fs) shutil.move("dir/ignored", "dir/ignored_new") - new_mtime, new_size = get_mtime_and_size("dir", dvc.tree) + new_mtime, new_size = get_mtime_and_size("dir", dvc.fs) assert new_mtime == mtime and new_size == size def test_rename_file(tmp_dir, dvc): tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) - mtime, size = get_mtime_and_size("dir", dvc.tree) + mtime, size = get_mtime_and_size("dir", dvc.fs) shutil.move("dir/foo", "dir/foo_new") - new_mtime, new_size = get_mtime_and_size("dir", dvc.tree) + new_mtime, new_size = get_mtime_and_size("dir", dvc.fs) assert new_mtime != mtime and new_size == size @@ -65,22 +65,22 @@ def test_rename_file(tmp_dir, dvc): def test_remove_ignored_file(tmp_dir, dvc): tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - dvc.tree._reset() + dvc.fs._reset() - mtime, size = get_mtime_and_size("dir", dvc.tree) + mtime, size = get_mtime_and_size("dir", dvc.fs) os.remove("dir/ignored") - new_mtime, new_size = get_mtime_and_size("dir", dvc.tree) + new_mtime, new_size = get_mtime_and_size("dir", dvc.fs) assert new_mtime == mtime and new_size == size def test_remove_file(tmp_dir, dvc): tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) - mtime, size = get_mtime_and_size("dir", dvc.tree) + mtime, size = get_mtime_and_size("dir", dvc.fs) os.remove("dir/foo") - new_mtime, new_size = get_mtime_and_size("dir", dvc.tree) + new_mtime, new_size = get_mtime_and_size("dir", dvc.fs) assert new_mtime != mtime and new_size != size @@ -98,12 +98,12 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname): top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE) top_ignore_file.write_text(os.path.basename(dname)) - dvc.tree._reset() + dvc.fs._reset() ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE ignore_file.write_text("foo") - dvcignore = dvc.tree.dvcignore + dvcignore = dvc.fs.dvcignore top_ignore_path = os.path.dirname(os.fspath(top_ignore_file)) @@ -129,16 +129,16 @@ def test_ignore_on_branch(tmp_dir, scm, dvc): with tmp_dir.branch("branch", new=True): tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore") - dvc.tree._reset() + dvc.fs._reset() path = PathInfo(tmp_dir) - assert set(dvc.tree.walk_files(path)) == { + assert set(dvc.fs.walk_files(path)) == { path / "foo", path / "bar", path / ".dvcignore", } - dvc.tree = scm.get_tree("branch", use_dvcignore=True) - assert set(dvc.tree.walk_files(path)) == { + dvc.fs = scm.get_fs("branch", use_dvcignore=True) + assert set(dvc.fs.walk_files(path)) == { os.fspath(path / DvcIgnore.DVCIGNORE_FILE), os.fspath(path / "bar"), } @@ -153,8 +153,8 @@ def test_match_nested(tmp_dir, dvc): "dir": {"x.backup": "x backup", "tmp": "content"}, } ) - dvc.tree._reset() - result = {os.fspath(os.path.normpath(f)) for f in dvc.tree.walk_files(".")} + dvc.fs._reset() + result = {os.fspath(os.path.normpath(f)) for f in dvc.fs.walk_files(".")} assert result == {".dvcignore", "foo"} @@ -163,13 +163,11 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory): ext_dir = TmpDir(os.fspath(tmp_path_factory.mktemp("external_dir"))) ext_dir.gen({"y.backup": "y", "tmp": {"file": "ext tmp"}}) - result = {relpath(f, ext_dir) for f in dvc.tree.walk_files(ext_dir)} + result = {relpath(f, ext_dir) for f in dvc.fs.walk_files(ext_dir)} assert result == {"y.backup", os.path.join("tmp", "file")} + assert dvc.fs.dvcignore.is_ignored_dir(os.fspath(ext_dir / "tmp")) is False assert ( - dvc.tree.dvcignore.is_ignored_dir(os.fspath(ext_dir / "tmp")) is False - ) - assert ( - dvc.tree.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup")) + dvc.fs.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup")) is False ) @@ -178,10 +176,10 @@ def test_ignore_subrepo(tmp_dir, scm, dvc): tmp_dir.gen({".dvcignore": "foo", "subdir": {"foo": "foo"}}) scm.add([".dvcignore"]) scm.commit("init parent dvcignore") - dvc.tree._reset() + dvc.fs._reset() subrepo_dir = tmp_dir / "subdir" - assert not dvc.tree.exists(PathInfo(subrepo_dir / "foo")) + assert not dvc.fs.exists(PathInfo(subrepo_dir / "foo")) with subrepo_dir.chdir(): subrepo = Repo.init(subdir=True) @@ -189,7 +187,7 @@ def test_ignore_subrepo(tmp_dir, scm, dvc): scm.commit("subrepo init") for _ in subrepo.brancher(all_commits=True): - assert subrepo.tree.exists(PathInfo(subrepo_dir / "foo")) + assert subrepo.fs.exists(PathInfo(subrepo_dir / "foo")) def test_ignore_resurface_subrepo(tmp_dir, scm, dvc): @@ -199,17 +197,17 @@ def test_ignore_resurface_subrepo(tmp_dir, scm, dvc): with subrepo_dir.chdir(): Repo.init(subdir=True) - dvc.tree._reset() + dvc.fs._reset() dirs = ["subdir"] files = ["foo"] - assert dvc.tree.dvcignore(os.fspath(tmp_dir), dirs, files) == ([], files) - assert dvc.tree.dvcignore( + assert dvc.fs.dvcignore(os.fspath(tmp_dir), dirs, files) == ([], files) + assert dvc.fs.dvcignore( os.fspath(tmp_dir), dirs, files, ignore_subrepos=False ) == (dirs, files) - assert dvc.tree.dvcignore.is_ignored_dir(os.fspath(subrepo_dir)) - assert not dvc.tree.dvcignore.is_ignored_dir( + assert dvc.fs.dvcignore.is_ignored_dir(os.fspath(subrepo_dir)) + assert not dvc.fs.dvcignore.is_ignored_dir( os.fspath(subrepo_dir), ignore_subrepos=False ) @@ -217,9 +215,9 @@ def test_ignore_resurface_subrepo(tmp_dir, scm, dvc): def test_ignore_blank_line(tmp_dir, dvc): tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "foo\n\ndir/ignored") - dvc.tree._reset() + dvc.fs._reset() path = PathInfo(tmp_dir) - assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"} + assert set(dvc.fs.walk_files(path / "dir")) == {path / "dir" / "other"} # It is not possible to re-include a file if a parent directory of @@ -252,9 +250,9 @@ def test_ignore_file_in_parent_path( ): tmp_dir.gen(data_struct) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "\n".join(pattern_list)) - dvc.tree._reset() + dvc.fs._reset() path = PathInfo(tmp_dir) - assert set(dvc.tree.walk_files(path / "dir")) == { + assert set(dvc.fs.walk_files(path / "dir")) == { path / relpath for relpath in result_set } @@ -275,9 +273,9 @@ def test_ignore_sub_directory(tmp_dir, dvc): ) tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "doc/fortz"}}) - dvc.tree._reset() + dvc.fs._reset() path = PathInfo(tmp_dir) - assert set(dvc.tree.walk_files(path / "dir")) == { + assert set(dvc.fs.walk_files(path / "dir")) == { path / "dir" / "a" / "doc" / "fortz" / "a", path / "dir" / DvcIgnore.DVCIGNORE_FILE, } @@ -287,9 +285,9 @@ def test_ignore_sub_directory(tmp_dir, dvc): def test_ignore_directory(tmp_dir, dvc): tmp_dir.gen({"dir": {"fortz": {}, "a": {"fortz": {}}}}) tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "fortz"}}) - dvc.tree._reset() + dvc.fs._reset() path = PathInfo(tmp_dir) - assert set(dvc.tree.walk_files(path / "dir")) == { + assert set(dvc.fs.walk_files(path / "dir")) == { path / "dir" / DvcIgnore.DVCIGNORE_FILE, } @@ -298,15 +296,15 @@ def test_multi_ignore_file(tmp_dir, dvc, monkeypatch): tmp_dir.gen({"dir": {"subdir": {"should_ignore": "1", "not_ignore": "1"}}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/subdir/*_ignore") tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "!subdir/not_ignore"}}) - dvc.tree._reset() + dvc.fs._reset() path = PathInfo(tmp_dir) - assert set(dvc.tree.walk_files(path / "dir")) == { + assert set(dvc.fs.walk_files(path / "dir")) == { path / "dir" / "subdir" / "not_ignore", path / "dir" / DvcIgnore.DVCIGNORE_FILE, } -def test_pattern_trie_tree(tmp_dir, dvc): +def test_pattern_trie_fs(tmp_dir, dvc): tmp_dir.gen( { "top": { @@ -323,8 +321,8 @@ def test_pattern_trie_tree(tmp_dir, dvc): "other": {DvcIgnore.DVCIGNORE_FILE: "1\n2\n3"}, } ) - dvc.tree._reset() - dvcignore = dvc.tree.dvcignore + dvc.fs._reset() + dvcignore = dvc.fs.dvcignore ignore_pattern_top = dvcignore._get_trie_pattern( os.fspath(tmp_dir / "top") @@ -391,10 +389,10 @@ def test_ignore_in_added_dir(tmp_dir, dvc): ".dvcignore": "**/ignored", } ) - dvc.tree._reset() + dvc.fs._reset() ignored_path = tmp_dir / "dir" / "sub" / "ignored" - assert not dvc.tree.exists(PathInfo(ignored_path)) + assert not dvc.fs.exists(PathInfo(ignored_path)) assert ignored_path.exists() dvc.add("dir") diff --git a/tests/func/test_import.py b/tests/func/test_import.py index c10bc2750..eebbc3e92 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -13,7 +13,7 @@ from dvc.exceptions import DownloadError from dvc.stage.exceptions import StagePathNotFoundError from dvc.system import System from dvc.utils.fs import makedirs, remove -from tests.unit.tree.test_repo import make_subrepo +from tests.unit.fs.test_repo import make_subrepo def test_import(tmp_dir, scm, dvc, erepo_dir): @@ -305,7 +305,7 @@ def test_download_error_pulling_imported_stage(tmp_dir, dvc, erepo_dir): remove(dst_cache) with patch( - "dvc.tree.local.LocalTree._download", side_effect=Exception + "dvc.fs.local.LocalFileSystem._download", side_effect=Exception ), pytest.raises(DownloadError): dvc.pull(["foo_imported.dvc"]) diff --git a/tests/func/test_install.py b/tests/func/test_install.py index be2cc4c82..345183dec 100644 --- a/tests/func/test_install.py +++ b/tests/func/test_install.py @@ -64,7 +64,7 @@ class TestInstall: conf["core"]["remote"] = "store" tmp_dir.dvc_gen("file", "file_content", "commit message") - file_checksum = file_md5("file", dvc.tree) + file_checksum = file_md5("file", dvc.fs) expected_storage_path = ( storage_path / file_checksum[:2] / file_checksum[2:] ) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 970f6e3aa..4ae645c1c 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -9,9 +9,9 @@ from mock import patch from dvc.config import Config from dvc.dir_info import DirInfo from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError +from dvc.fs.local import LocalFileSystem from dvc.main import main from dvc.path_info import PathInfo -from dvc.tree.local import LocalTree from dvc.utils.fs import remove from tests.basic_env import TestDvc from tests.remotes import Local @@ -158,7 +158,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): with patch( "dvc.oid._collect_dir", return_value=dir_info, ): - hash1 = get_hash(path_info, dvc.cache.local.tree, "md5") + hash1 = get_hash(path_info, dvc.cache.local.fs, "md5") dir_info = DirInfo.from_list( [{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}] @@ -166,7 +166,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): with patch( "dvc.oid._collect_dir", return_value=dir_info, ): - hash2 = get_hash(path_info, dvc.cache.local.tree, "md5") + hash2 = get_hash(path_info, dvc.cache.local.fs, "md5") assert hash1 == hash2 @@ -177,26 +177,26 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote): baz = tmp_dir.dvc_gen({"baz": {"foo": "baz content"}})[0].outs[0] # Faulty upload version, failing on foo - original = LocalTree._upload + original = LocalFileSystem._upload def unreliable_upload(self, from_file, to_info, name=None, **kwargs): if "foo" in name: raise Exception("stop foo") return original(self, from_file, to_info, name, **kwargs) - with patch.object(LocalTree, "_upload", unreliable_upload): + with patch.object(LocalFileSystem, "_upload", unreliable_upload): with pytest.raises(UploadError) as upload_error_info: dvc.push() assert upload_error_info.value.amount == 3 remote = dvc.cloud.get_remote("upstream") - assert not remote.tree.exists( + assert not remote.fs.exists( remote.cache.hash_to_path_info(foo.hash_info.value) ) - assert remote.tree.exists( + assert remote.fs.exists( remote.cache.hash_to_path_info(bar.hash_info.value) ) - assert not remote.tree.exists( + assert not remote.fs.exists( remote.cache.hash_to_path_info(baz.hash_info.value) ) @@ -204,7 +204,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote): dvc.push() remove(dvc.cache.local.cache_dir) - with patch.object(LocalTree, "_download", side_effect=Exception): + with patch.object(LocalFileSystem, "_download", side_effect=Exception): with pytest.raises(DownloadError) as download_error_info: dvc.pull() # error count should be len(.dir + standalone file checksums) @@ -218,7 +218,7 @@ def test_raise_on_too_many_open_files( tmp_dir.dvc_gen({"file": "file content"}) mocker.patch.object( - LocalTree, + LocalFileSystem, "_upload", side_effect=OSError(errno.EMFILE, "Too many open files"), ) @@ -269,7 +269,9 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote): foo = tmp_dir.dvc_gen({"foo": {"bar": "bar content"}})[0].outs[0] tmp_dir.dvc_gen({"baz": "baz content"}) - mocked_upload = mocker.patch.object(LocalTree, "_upload", return_value=0) + mocked_upload = mocker.patch.object( + LocalFileSystem, "_upload", return_value=0 + ) dvc.push() # foo .dir file should be uploaded after bar @@ -432,15 +434,13 @@ def test_push_incomplete_dir(tmp_dir, dvc, mocker, local_remote): used = stage.get_used_cache(remote=remote) # remove one of the cache files for directory - file_hashes = list(used.child_keys(cache.tree.scheme, dir_hash)) + file_hashes = list(used.child_keys(cache.fs.scheme, dir_hash)) remove(cache.hash_to_path_info(file_hashes[0])) dvc.push() - assert not remote.tree.exists(remote.cache.hash_to_path_info(dir_hash)) - assert not remote.tree.exists( - remote.cache.hash_to_path_info(file_hashes[0]) - ) - assert remote.tree.exists(remote.cache.hash_to_path_info(file_hashes[1])) + assert not remote.fs.exists(remote.cache.hash_to_path_info(dir_hash)) + assert not remote.fs.exists(remote.cache.hash_to_path_info(file_hashes[0])) + assert remote.fs.exists(remote.cache.hash_to_path_info(file_hashes[1])) def test_upload_exists(tmp_dir, dvc, local_remote): @@ -448,14 +448,14 @@ def test_upload_exists(tmp_dir, dvc, local_remote): remote = dvc.cloud.get_remote("upstream") # allow uploaded files to be writable for this test, # normally they are set to read-only for DVC remotes - remote.tree.CACHE_MODE = 0o644 + remote.fs.CACHE_MODE = 0o644 from_info = PathInfo(tmp_dir / "foo") - to_info = remote.tree.path_info / "foo" - remote.tree.upload(from_info, to_info) - assert remote.tree.exists(to_info) + to_info = remote.fs.path_info / "foo" + remote.fs.upload(from_info, to_info) + assert remote.fs.exists(to_info) tmp_dir.gen("foo", "bar") - remote.tree.upload(from_info, to_info) - with remote.tree.open(to_info) as fobj: + remote.fs.upload(from_info, to_info) + with remote.fs.open(to_info) as fobj: assert fobj.read() == "bar" diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index 602d4f0cd..c8633c938 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -13,12 +13,12 @@ from dvc.exceptions import ( ReproductionError, StagePathAsOutputError, ) +from dvc.fs.local import LocalFileSystem from dvc.main import main from dvc.output.base import BaseOutput from dvc.stage import Stage from dvc.stage.exceptions import StageFileDoesNotExistError from dvc.system import System -from dvc.tree.local import LocalTree from dvc.utils import file_md5, relpath from dvc.utils.fs import remove from dvc.utils.serialize import dump_yaml, load_yaml @@ -652,8 +652,7 @@ class TestReproDataSource(TestReproChangedData): self.assertTrue(filecmp.cmp(self.FOO, self.BAR, shallow=False)) self.assertEqual( - stages[0].outs[0].hash_info.value, - file_md5(self.BAR, self.dvc.tree), + stages[0].outs[0].hash_info.value, file_md5(self.BAR, self.dvc.fs), ) @@ -737,8 +736,8 @@ class TestReproChangedDirData(SingleStageRun, TestDvc): class TestReproMissingMd5InStageFile(TestRepro): def test(self): d = load_yaml(self.file1_stage) - del d[Stage.PARAM_OUTS][0][LocalTree.PARAM_CHECKSUM] - del d[Stage.PARAM_DEPS][0][LocalTree.PARAM_CHECKSUM] + del d[Stage.PARAM_OUTS][0][LocalFileSystem.PARAM_CHECKSUM] + del d[Stage.PARAM_DEPS][0][LocalFileSystem.PARAM_CHECKSUM] dump_yaml(self.file1_stage, d) stages = self.dvc.reproduce(self.file1_stage) @@ -877,9 +876,9 @@ class TestReproAlreadyCached(TestRepro): self.assertEqual(ret, 0) patch_download = patch.object( - LocalTree, + LocalFileSystem, "download", - side_effect=LocalTree.download, + side_effect=LocalFileSystem.download, autospec=True, ) diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index 037e5132a..bb5b942b5 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -61,7 +61,7 @@ class TestRun(TestDvc): self.assertEqual(len(stage.outs), len(outs + outs_no_cache)) self.assertEqual(stage.outs[0].fspath, outs[0]) self.assertEqual( - stage.outs[0].hash_info.value, file_md5(self.FOO, self.dvc.tree) + stage.outs[0].hash_info.value, file_md5(self.FOO, self.dvc.fs) ) self.assertTrue(stage.path, fname) diff --git a/tests/func/test_s3.py b/tests/func/test_s3.py index 4cd6a9b0a..17804b34a 100644 --- a/tests/func/test_s3.py +++ b/tests/func/test_s3.py @@ -9,7 +9,7 @@ import pytest from moto import mock_s3 from dvc.cache.base import CloudCache -from dvc.tree.s3 import S3Tree +from dvc.fs.s3 import S3FileSystem from tests.remotes import S3 # from https://github.com/spulec/moto/blob/v1.3.5/tests/test_s3/test_s3.py#L40 @@ -34,7 +34,7 @@ def reduced_min_part_size(f): def _get_src_dst(): - base_info = S3Tree.PATH_CLS(S3.get_url()) + base_info = S3FileSystem.PATH_CLS(S3.get_url()) return base_info / "from", base_info / "to" @@ -46,35 +46,38 @@ def test_copy_singlepart_preserve_etag(): s3.create_bucket(Bucket=from_info.bucket) s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data") - S3Tree._copy(s3, from_info, to_info, {}) + S3FileSystem._copy(s3, from_info, to_info, {}) @mock_s3 @pytest.mark.parametrize( "base_info", - [S3Tree.PATH_CLS("s3://bucket/"), S3Tree.PATH_CLS("s3://bucket/ns/")], + [ + S3FileSystem.PATH_CLS("s3://bucket/"), + S3FileSystem.PATH_CLS("s3://bucket/ns/"), + ], ) def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm): from dvc.checkout import _link - tree = S3Tree(dvc, {"url": str(base_info.parent)}) - cache = CloudCache(tree) - s3 = cache.tree.s3.meta.client + fs = S3FileSystem(dvc, {"url": str(base_info.parent)}) + cache = CloudCache(fs) + s3 = cache.fs.s3.meta.client s3.create_bucket(Bucket=base_info.bucket) s3.put_object( Bucket=base_info.bucket, Key=(base_info / "from").path, Body="data" ) _link(cache, base_info / "from", base_info / "to") - assert cache.tree.exists(base_info / "from") - assert cache.tree.exists(base_info / "to") + assert cache.fs.exists(base_info / "from") + assert cache.fs.exists(base_info / "to") @mock_s3 def test_makedirs_doesnot_try_on_top_level_paths(tmp_dir, dvc, scm): - base_info = S3Tree.PATH_CLS("s3://bucket/") - tree = S3Tree(dvc, {"url": str(base_info)}) - tree.makedirs(base_info) + base_info = S3FileSystem.PATH_CLS("s3://bucket/") + fs = S3FileSystem(dvc, {"url": str(base_info)}) + fs.makedirs(base_info) def _upload_multipart(s3, Bucket, Key): @@ -115,24 +118,24 @@ def test_copy_multipart_preserve_etag(): s3 = boto3.client("s3") s3.create_bucket(Bucket=from_info.bucket) _upload_multipart(s3, from_info.bucket, from_info.path) - S3Tree._copy(s3, from_info, to_info, {}) + S3FileSystem._copy(s3, from_info, to_info, {}) def test_s3_isdir(tmp_dir, dvc, s3): s3.gen({"data": {"foo": "foo"}}) - tree = S3Tree(dvc, s3.config) + fs = S3FileSystem(dvc, s3.config) - assert not tree.isdir(s3 / "data" / "foo") - assert tree.isdir(s3 / "data") + assert not fs.isdir(s3 / "data" / "foo") + assert fs.isdir(s3 / "data") def test_s3_upload_fobj(tmp_dir, dvc, s3): s3.gen({"data": {"foo": "foo"}}) - tree = S3Tree(dvc, s3.config) + fs = S3FileSystem(dvc, s3.config) to_info = s3 / "data" / "bar" - with tree.open(s3 / "data" / "foo", "rb") as stream: - tree.upload_fobj(stream, to_info, 1) + with fs.open(s3 / "data" / "foo", "rb") as stream: + fs.upload_fobj(stream, to_info, 1) assert to_info.read_text() == "foo" @@ -167,16 +170,16 @@ def test_s3_aws_config(tmp_dir, dvc, s3, monkeypatch): monkeypatch.setenv(var, str(tmp_dir)) # Fresh import to see the effects of changing HOME variable - s3_mod = importlib.reload(sys.modules[S3Tree.__module__]) - tree = s3_mod.S3Tree(dvc, s3.config) - assert tree._transfer_config is None + s3_mod = importlib.reload(sys.modules[S3FileSystem.__module__]) + fs = s3_mod.S3FileSystem(dvc, s3.config) + assert fs._transfer_config is None - with tree._get_s3() as s3: + with fs._get_s3() as s3: s3_config = s3.meta.client.meta.config.s3 assert s3_config["use_accelerate_endpoint"] assert s3_config["addressing_style"] == "path" - transfer_config = tree._transfer_config + transfer_config = fs._transfer_config assert transfer_config.max_io_queue_size == 1000 assert transfer_config.multipart_chunksize == 64 * MB assert transfer_config.multipart_threshold == 1000 * KB @@ -204,13 +207,13 @@ def test_s3_aws_config_different_profile(tmp_dir, dvc, s3, monkeypatch): ) monkeypatch.setenv("AWS_CONFIG_FILE", config_file) - tree = S3Tree(dvc, {**s3.config, "profile": "dev"}) - assert tree._transfer_config is None + fs = S3FileSystem(dvc, {**s3.config, "profile": "dev"}) + assert fs._transfer_config is None - with tree._get_s3() as s3: + with fs._get_s3() as s3: s3_config = s3.meta.client.meta.config.s3 assert s3_config["addresing_style"] == "virtual" assert "use_accelerate_endpoint" not in s3_config - transfer_config = tree._transfer_config + transfer_config = fs._transfer_config assert transfer_config.multipart_threshold == 2 * GB diff --git a/tests/func/test_stage.py b/tests/func/test_stage.py index dc19e868d..d203b264b 100644 --- a/tests/func/test_stage.py +++ b/tests/func/test_stage.py @@ -4,13 +4,13 @@ import tempfile import pytest from dvc.dvcfile import SingleStageFile +from dvc.fs.local import LocalFileSystem from dvc.main import main from dvc.output.local import LocalOutput from dvc.repo import Repo, lock_repo from dvc.stage import PipelineStage, Stage from dvc.stage.exceptions import StageFileFormatError from dvc.stage.run import run_stage -from dvc.tree.local import LocalTree from dvc.utils.serialize import dump_yaml, load_yaml from tests.basic_env import TestDvc @@ -55,8 +55,8 @@ def test_empty_list(): def test_list(): lst = [ - {LocalOutput.PARAM_PATH: "foo", LocalTree.PARAM_CHECKSUM: "123"}, - {LocalOutput.PARAM_PATH: "bar", LocalTree.PARAM_CHECKSUM: None}, + {LocalOutput.PARAM_PATH: "foo", LocalFileSystem.PARAM_CHECKSUM: "123"}, + {LocalOutput.PARAM_PATH: "bar", LocalFileSystem.PARAM_CHECKSUM: None}, {LocalOutput.PARAM_PATH: "baz"}, ] d = {Stage.PARAM_DEPS: lst} diff --git a/tests/func/test_state.py b/tests/func/test_state.py index 2ed075192..c16140f8f 100644 --- a/tests/func/test_state.py +++ b/tests/func/test_state.py @@ -12,7 +12,7 @@ def test_state(tmp_dir, dvc): tmp_dir.gen("foo", "foo content") path = tmp_dir / "foo" path_info = PathInfo(path) - hash_info = HashInfo("md5", file_md5(path, dvc.tree)) + hash_info = HashInfo("md5", file_md5(path, dvc.fs)) state = State(dvc) @@ -25,7 +25,7 @@ def test_state(tmp_dir, dvc): assert state.get(path_info) is None - hash_info = HashInfo("md5", file_md5(path, dvc.tree)) + hash_info = HashInfo("md5", file_md5(path, dvc.fs)) state.save(path_info, hash_info) assert state.get(path_info) == hash_info @@ -60,7 +60,7 @@ def test_get_state_record_for_inode(get_inode_mock, tmp_dir, dvc): assert inode != state._to_sqlite(inode) foo = tmp_dir / "foo" - md5 = file_md5(foo, dvc.tree) + md5 = file_md5(foo, dvc.fs) get_inode_mock.side_effect = mock_get_inode(inode) with state: diff --git a/tests/func/test_tree.py b/tests/func/test_tree.py deleted file mode 100644 index 9176f7832..000000000 --- a/tests/func/test_tree.py +++ /dev/null @@ -1,351 +0,0 @@ -import os -from operator import itemgetter -from os.path import join - -import pytest - -from dvc.path_info import PathInfo -from dvc.repo import Repo -from dvc.scm import SCM -from dvc.tree import get_cloud_tree -from dvc.tree.local import LocalTree -from tests.basic_env import TestDir, TestGit, TestGitSubmodule - - -class TestLocalTree(TestDir): - def setUp(self): - super().setUp() - self.tree = LocalTree(None, {}) - - def test_open(self): - with self.tree.open(self.FOO) as fd: - self.assertEqual(fd.read(), self.FOO_CONTENTS) - with self.tree.open(self.UNICODE, encoding="utf-8") as fd: - self.assertEqual(fd.read(), self.UNICODE_CONTENTS) - - def test_exists(self): - self.assertTrue(self.tree.exists(self.FOO)) - self.assertTrue(self.tree.exists(self.UNICODE)) - self.assertFalse(self.tree.exists("not-existing-file")) - - def test_isdir(self): - self.assertTrue(self.tree.isdir(self.DATA_DIR)) - self.assertFalse(self.tree.isdir(self.FOO)) - self.assertFalse(self.tree.isdir("not-existing-file")) - - def test_isfile(self): - self.assertTrue(self.tree.isfile(self.FOO)) - self.assertFalse(self.tree.isfile(self.DATA_DIR)) - self.assertFalse(self.tree.isfile("not-existing-file")) - - -class GitTreeTests: - # pylint: disable=no-member - def test_open(self): - self.scm.add([self.FOO, self.UNICODE, self.DATA_DIR]) - self.scm.commit("add") - - tree = self.scm.get_tree("master") - with tree.open(self.FOO) as fd: - self.assertEqual(fd.read(), self.FOO_CONTENTS) - with tree.open(self.UNICODE) as fd: - self.assertEqual(fd.read(), self.UNICODE_CONTENTS) - with self.assertRaises(IOError): - tree.open("not-existing-file") - with self.assertRaises(IOError): - tree.open(self.DATA_DIR) - - def test_exists(self): - tree = self.scm.get_tree("master") - self.assertFalse(tree.exists(self.FOO)) - self.assertFalse(tree.exists(self.UNICODE)) - self.assertFalse(tree.exists(self.DATA_DIR)) - self.scm.add([self.FOO, self.UNICODE, self.DATA]) - self.scm.commit("add") - - tree = self.scm.get_tree("master") - self.assertTrue(tree.exists(self.FOO)) - self.assertTrue(tree.exists(self.UNICODE)) - self.assertTrue(tree.exists(self.DATA_DIR)) - self.assertFalse(tree.exists("non-existing-file")) - - def test_isdir(self): - self.scm.add([self.FOO, self.DATA_DIR]) - self.scm.commit("add") - - tree = self.scm.get_tree("master") - self.assertTrue(tree.isdir(self.DATA_DIR)) - self.assertFalse(tree.isdir(self.FOO)) - self.assertFalse(tree.isdir("non-existing-file")) - - def test_isfile(self): - self.scm.add([self.FOO, self.DATA_DIR]) - self.scm.commit("add") - - tree = self.scm.get_tree("master") - self.assertTrue(tree.isfile(self.FOO)) - self.assertFalse(tree.isfile(self.DATA_DIR)) - self.assertFalse(tree.isfile("not-existing-file")) - - -class TestGitTree(TestGit, GitTreeTests): - def setUp(self): - super().setUp() - self.scm = SCM(self._root_dir) - - -class TestGitSubmoduleTree(TestGitSubmodule, GitTreeTests): - def setUp(self): - super().setUp() - self.scm = SCM(self._root_dir) - self._pushd(self._root_dir) - - -class AssertWalkEqualMixin: - def assertWalkEqual(self, actual, expected, msg=None): - def convert_to_sets(walk_results): - return [ - (root, set(dirs), set(nondirs)) - for root, dirs, nondirs in walk_results - ] - - self.assertEqual( - convert_to_sets(actual), convert_to_sets(expected), msg=msg - ) - - -class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir): - def test(self): - tree = LocalTree(None, {"url": self._root_dir}) - self.assertWalkEqual( - tree.walk(self._root_dir), - [ - ( - self._root_dir, - ["data_dir"], - ["code.py", "bar", "тест", "foo"], - ), - (join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]), - ( - join(self._root_dir, "data_dir", "data_sub_dir"), - [], - ["data_sub"], - ), - ], - ) - - def test_subdir(self): - tree = LocalTree(None, {"url": self._root_dir}) - self.assertWalkEqual( - tree.walk(join("data_dir", "data_sub_dir")), - [(join("data_dir", "data_sub_dir"), [], ["data_sub"])], - ) - - -class TestWalkInGit(AssertWalkEqualMixin, TestGit): - def test_nobranch(self): - tree = LocalTree(None, {"url": self._root_dir}, use_dvcignore=True) - self.assertWalkEqual( - tree.walk("."), - [ - (".", ["data_dir"], ["bar", "тест", "code.py", "foo"]), - (join("data_dir"), ["data_sub_dir"], ["data"]), - (join("data_dir", "data_sub_dir"), [], ["data_sub"]), - ], - ) - self.assertWalkEqual( - tree.walk(join("data_dir", "data_sub_dir")), - [(join("data_dir", "data_sub_dir"), [], ["data_sub"])], - ) - - def test_branch(self): - scm = SCM(self._root_dir) - scm.add([self.DATA_SUB_DIR]) - scm.commit("add data_dir/data_sub_dir/data_sub") - tree = scm.get_tree("master") - self.assertWalkEqual( - tree.walk("."), - [ - (self._root_dir, ["data_dir"], ["code.py"]), - (join(self._root_dir, "data_dir"), ["data_sub_dir"], []), - ( - join(self._root_dir, "data_dir", "data_sub_dir"), - [], - ["data_sub"], - ), - ], - ) - self.assertWalkEqual( - tree.walk(join("data_dir", "data_sub_dir")), - [ - ( - join(self._root_dir, "data_dir", "data_sub_dir"), - [], - ["data_sub"], - ) - ], - ) - - -def test_cleantree_subrepo(tmp_dir, dvc, scm, monkeypatch): - tmp_dir.gen({"subdir": {}}) - subrepo_dir = tmp_dir / "subdir" - with subrepo_dir.chdir(): - subrepo = Repo.init(subdir=True) - subrepo_dir.gen({"foo": "foo", "dir": {"bar": "bar"}}) - - path = PathInfo(subrepo_dir) - - assert dvc.tree.use_dvcignore - assert not dvc.tree.exists(path / "foo") - assert not dvc.tree.isfile(path / "foo") - assert not dvc.tree.exists(path / "dir") - assert not dvc.tree.isdir(path / "dir") - - assert subrepo.tree.use_dvcignore - assert subrepo.tree.exists(path / "foo") - assert subrepo.tree.isfile(path / "foo") - assert subrepo.tree.exists(path / "dir") - assert subrepo.tree.isdir(path / "dir") - - -def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc): - tmp_dir.dvc_gen({"foo": "foo"}, commit="add foo") - subrepo_dir = tmp_dir / "subdir" - subrepo_dir.mkdir() - with subrepo_dir.chdir(): - Repo.init(subdir=True) - scm.add(["subdir"]) - scm.commit("Add subrepo") - - dvc_tree = dvc.tree - dvc_tree._reset() - scm_tree = scm.get_tree("HEAD", use_dvcignore=True) - path = os.fspath(tmp_dir) - get_dirs = itemgetter(1) - - assert get_dirs(next(dvc_tree.walk(path))) == [] - assert get_dirs(next(scm_tree.walk(path))) == [] - - kw = {"ignore_subrepos": False} - assert get_dirs(next(dvc_tree.walk(path, **kw))) == ["subdir"] - assert get_dirs(next(scm_tree.walk(path, **kw))) == ["subdir"] - - -@pytest.mark.parametrize( - "cloud", - [ - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("s3"), - pytest.lazy_fixture("gs"), - pytest.lazy_fixture("hdfs"), - pytest.lazy_fixture("http"), - ], -) -def test_tree_getsize(dvc, cloud): - cloud.gen({"data": {"foo": "foo"}, "baz": "baz baz"}) - tree = get_cloud_tree(dvc, **cloud.config) - path_info = tree.path_info - - assert tree.getsize(path_info / "baz") == 7 - assert tree.getsize(path_info / "data" / "foo") == 3 - - -@pytest.mark.parametrize( - "cloud", - [ - pytest.lazy_fixture("azure"), - pytest.lazy_fixture("gs"), - pytest.lazy_fixture("gdrive"), - pytest.lazy_fixture("hdfs"), - pytest.lazy_fixture("http"), - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("oss"), - pytest.lazy_fixture("s3"), - pytest.lazy_fixture("ssh"), - pytest.lazy_fixture("webhdfs"), - ], -) -def test_tree_upload_fobj(dvc, tmp_dir, cloud): - tmp_dir.gen("foo", "foo") - tree = get_cloud_tree(dvc, **cloud.config) - - from_info = tmp_dir / "foo" - to_info = tree.path_info / "foo" - - with open(from_info, "rb") as stream: - tree.upload_fobj(stream, to_info) - - assert tree.exists(to_info) - with tree.open(to_info, "rb") as stream: - assert stream.read() == b"foo" - - -@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("gdrive")]) -def test_tree_ls(dvc, cloud): - cloud.gen( - { - "directory": { - "foo": "foo", - "bar": "bar", - "baz": {"quux": "quux", "egg": {"foo": "foo"}}, - "empty": {}, - } - } - ) - tree = get_cloud_tree(dvc, **cloud.config) - path_info = cloud / "directory" - - assert {os.path.basename(file_key) for file_key in tree.ls(path_info)} == { - "foo", - "bar", - "baz", - "empty", - } - assert set(tree.ls(path_info / "empty")) == set() - assert { - (detail["type"], os.path.basename(detail["name"])) - for detail in tree.ls(path_info / "baz", detail=True) - } == {("file", "quux"), ("directory", "egg")} - - -@pytest.mark.parametrize( - "cloud", - [ - pytest.lazy_fixture("s3"), - pytest.lazy_fixture("azure"), - pytest.lazy_fixture("gs"), - pytest.lazy_fixture("webdav"), - pytest.lazy_fixture("gdrive"), - ], -) -def test_tree_ls_recursive(dvc, cloud): - cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}}) - tree = get_cloud_tree(dvc, **cloud.config) - path_info = tree.path_info - - assert { - os.path.basename(file_key) - for file_key in tree.ls(path_info / "data", recursive=True) - } == {"foo", "baz", "quux"} - - -@pytest.mark.parametrize( - "cloud", - [ - pytest.lazy_fixture("s3"), - pytest.lazy_fixture("azure"), - pytest.lazy_fixture("gs"), - pytest.lazy_fixture("webdav"), - ], -) -def test_tree_ls_with_etag(dvc, cloud): - cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}}) - tree = get_cloud_tree(dvc, **cloud.config) - path_info = tree.path_info - - for details in tree.ls(path_info / "data", recursive=True, detail=True): - assert ( - tree.info(path_info.replace(path=details["name"]))["etag"] - == details["etag"] - ) diff --git a/tests/func/test_update.py b/tests/func/test_update.py index 758f41b1b..e2ff7a2ab 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -3,7 +3,7 @@ import os import pytest from dvc.dvcfile import Dvcfile -from tests.unit.tree.test_repo import make_subrepo +from tests.unit.fs.test_repo import make_subrepo @pytest.mark.parametrize( diff --git a/tests/func/test_utils.py b/tests/func/test_utils.py index 47a2dc71c..4110abeda 100644 --- a/tests/func/test_utils.py +++ b/tests/func/test_utils.py @@ -1,12 +1,12 @@ from dvc import utils -from dvc.tree.local import LocalTree +from dvc.fs.local import LocalFileSystem def test_file_md5_crlf(tmp_dir): - tree = LocalTree(None, {}) + fs = LocalFileSystem(None, {}) tmp_dir.gen("cr", b"a\nb\nc") tmp_dir.gen("crlf", b"a\r\nb\r\nc") - assert utils.file_md5("cr", tree) == utils.file_md5("crlf", tree) + assert utils.file_md5("cr", fs) == utils.file_md5("crlf", fs) def test_dict_md5(): diff --git a/tests/unit/tree/__init__.py b/tests/func/utils/__init__.py similarity index 100% rename from tests/unit/tree/__init__.py rename to tests/func/utils/__init__.py diff --git a/tests/func/utils/test_fs.py b/tests/func/utils/test_fs.py new file mode 100644 index 000000000..b73b5b681 --- /dev/null +++ b/tests/func/utils/test_fs.py @@ -0,0 +1,36 @@ +import os +import stat + +import pytest + +from dvc.utils.fs import copyfile, makedirs + + +@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.") +def test_makedirs_permissions(tmp_dir): + dir_mode = 0o755 + intermediate_dir = "тестовая-директория" + test_dir = os.path.join(intermediate_dir, "data") + + assert not os.path.exists(intermediate_dir) + + makedirs(test_dir, mode=dir_mode) + + assert stat.S_IMODE(os.stat(test_dir).st_mode) == dir_mode + assert stat.S_IMODE(os.stat(intermediate_dir).st_mode) == dir_mode + + +def test_copyfile(tmp_dir): + src = "file1" + dest = "file2" + dest_dir = "testdir" + + tmp_dir.gen(src, "file1contents") + + os.mkdir(dest_dir) + + copyfile(src, dest) + assert (tmp_dir / dest).read_text() == "file1contents" + + copyfile(src, dest_dir) + assert (tmp_dir / dest_dir / src).read_text() == "file1contents" diff --git a/tests/remotes/gdrive.py b/tests/remotes/gdrive.py index ef0eba8d3..c35497fbe 100644 --- a/tests/remotes/gdrive.py +++ b/tests/remotes/gdrive.py @@ -6,8 +6,8 @@ from functools import partial, partialmethod import pytest from funcy import cached_property +from dvc.fs.gdrive import GDriveFileSystem from dvc.path_info import CloudURLInfo -from dvc.tree.gdrive import GDriveTree from dvc.utils import tmp_fname from .base import Base @@ -23,7 +23,7 @@ CREDENTIALS_FILE = os.path.join( class GDrive(Base, CloudURLInfo): @staticmethod def should_test(): - return bool(os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA)) + return bool(os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)) @cached_property def config(self): @@ -50,7 +50,7 @@ class GDrive(Base, CloudURLInfo): tmp_path = tmp_fname() with open(tmp_path, "w") as stream: - stream.write(os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA)) + stream.write(os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)) GoogleDriveFileSystem._connect_cache = partial( pydata_google_auth.load_user_credentials, tmp_path @@ -88,6 +88,6 @@ def gdrive(test_config, make_tmp_dir): tmp_dir = make_tmp_dir("gdrive", dvc=True) ret = GDrive(GDrive.get_url()) - tree = GDriveTree(tmp_dir.dvc, ret.config) - tree._gdrive_create_dir("root", tree.path_info.path) + fs = GDriveFileSystem(tmp_dir.dvc, ret.config) + fs._gdrive_create_dir("root", fs.path_info.path) yield ret diff --git a/tests/remotes/hdfs.py b/tests/remotes/hdfs.py index 690ff2944..a766bee01 100644 --- a/tests/remotes/hdfs.py +++ b/tests/remotes/hdfs.py @@ -250,7 +250,7 @@ def hdfs(mocker): "pyarrow.fs.HadoopFileSystem", FakeHadoopFileSystem, create=True ) - mocker.patch("dvc.tree.hdfs._hadoop_fs_checksum", hadoop_fs_checksum) + mocker.patch("dvc.fs.hdfs._hadoop_fs_checksum", hadoop_fs_checksum) url = f"hdfs://example.com:12345/{uuid.uuid4()}" yield HDFS(url) diff --git a/tests/remotes/ssh.py b/tests/remotes/ssh.py index 2cf386289..53d628af1 100644 --- a/tests/remotes/ssh.py +++ b/tests/remotes/ssh.py @@ -53,7 +53,7 @@ class SSHMocked(Base, URLInfo): @contextmanager def _ssh(self): - from dvc.tree.ssh.connection import SSHConnection + from dvc.fs.ssh.connection import SSHConnection conn = SSHConnection( host=self.host, @@ -117,7 +117,7 @@ def ssh_server(test_config): @pytest.fixture def ssh_connection(ssh_server): - from dvc.tree.ssh.connection import SSHConnection + from dvc.fs.ssh.connection import SSHConnection yield SSHConnection( host=ssh_server.host, @@ -129,9 +129,9 @@ def ssh_connection(ssh_server): @pytest.fixture def ssh(ssh_server, monkeypatch): - from dvc.tree.ssh import SSHTree + from dvc.fs.ssh import SSHFileSystem # NOTE: see http://github.com/iterative/dvc/pull/3501 - monkeypatch.setattr(SSHTree, "CAN_TRAVERSE", False) + monkeypatch.setattr(SSHFileSystem, "CAN_TRAVERSE", False) return SSHMocked(SSHMocked.get_url(TEST_SSH_USER, ssh_server.port)) diff --git a/tests/unit/cache/test_local.py b/tests/unit/cache/test_local.py index 7f7818f8f..62cb4b9e9 100644 --- a/tests/unit/cache/test_local.py +++ b/tests/unit/cache/test_local.py @@ -5,9 +5,9 @@ import pytest from dvc.cache import NamedCache from dvc.cache.local import LocalCache +from dvc.fs.local import LocalFileSystem from dvc.path_info import PathInfo from dvc.remote.index import RemoteIndexNoop -from dvc.tree.local import LocalTree def test_status_download_optimization(mocker, dvc): @@ -15,7 +15,7 @@ def test_status_download_optimization(mocker, dvc): And the desired files to fetch are already on the local cache, Don't check the existence of the desired files on the remote cache """ - cache = LocalCache(LocalTree(dvc, {})) + cache = LocalCache(LocalFileSystem(dvc, {})) infos = NamedCache() infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo") @@ -37,8 +37,8 @@ def test_status_download_optimization(mocker, dvc): @pytest.mark.parametrize("link_name", ["hardlink", "symlink"]) def test_is_protected(tmp_dir, dvc, link_name): cache = dvc.cache.local - tree = cache.tree - link_method = getattr(tree, link_name) + fs = cache.fs + link_method = getattr(fs, link_name) (tmp_dir / "foo").write_text("foo") diff --git a/tests/unit/dependency/test_local.py b/tests/unit/dependency/test_local.py index 49936cecc..a4505fc07 100644 --- a/tests/unit/dependency/test_local.py +++ b/tests/unit/dependency/test_local.py @@ -15,6 +15,6 @@ class TestLocalDependency(TestDvc): def test_save_missing(self): d = self._get_dependency() - with mock.patch.object(d.tree, "exists", return_value=False): + with mock.patch.object(d.fs, "exists", return_value=False): with self.assertRaises(d.DoesNotExistError): d.save() diff --git a/tests/unit/fs/__init__.py b/tests/unit/fs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/tree/test_base.py b/tests/unit/fs/test_base.py similarity index 69% rename from tests/unit/tree/test_base.py rename to tests/unit/fs/test_base.py index 07b8459de..986e86f0d 100644 --- a/tests/unit/tree/test_base.py +++ b/tests/unit/fs/test_base.py @@ -1,6 +1,6 @@ import pytest -from dvc.tree.base import BaseTree, RemoteMissingDepsError +from dvc.fs.base import BaseFileSystem, RemoteMissingDepsError @pytest.mark.parametrize( @@ -13,7 +13,7 @@ from dvc.tree.base import BaseTree, RemoteMissingDepsError ) def test_missing_deps(pkg, msg, mocker): requires = {"missing": "missing"} - mocker.patch.object(BaseTree, "REQUIRES", requires) + mocker.patch.object(BaseFileSystem, "REQUIRES", requires) mocker.patch("dvc.utils.pkg.PKG", pkg) with pytest.raises(RemoteMissingDepsError, match=msg): - BaseTree(None, {}) + BaseFileSystem(None, {}) diff --git a/tests/unit/tree/test_dvc.py b/tests/unit/fs/test_dvc.py similarity index 69% rename from tests/unit/tree/test_dvc.py rename to tests/unit/fs/test_dvc.py index 840cdb4fa..f7efba0dd 100644 --- a/tests/unit/tree/test_dvc.py +++ b/tests/unit/fs/test_dvc.py @@ -3,10 +3,10 @@ import shutil import pytest +from dvc.fs.dvc import DvcFileSystem from dvc.hash_info import HashInfo from dvc.oid import get_hash from dvc.path_info import PathInfo -from dvc.tree.dvc import DvcTree def test_exists(tmp_dir, dvc): @@ -14,8 +14,8 @@ def test_exists(tmp_dir, dvc): dvc.add("foo") (tmp_dir / "foo").unlink() - tree = DvcTree(dvc) - assert tree.exists("foo") + fs = DvcFileSystem(dvc) + assert fs.exists("foo") def test_open(tmp_dir, dvc): @@ -23,8 +23,8 @@ def test_open(tmp_dir, dvc): dvc.add("foo") (tmp_dir / "foo").unlink() - tree = DvcTree(dvc) - with tree.open(PathInfo(tmp_dir) / "foo", "r") as fobj: + fs = DvcFileSystem(dvc) + with fs.open(PathInfo(tmp_dir) / "foo", "r") as fobj: assert fobj.read() == "foo" @@ -32,9 +32,9 @@ def test_open_dirty_hash(tmp_dir, dvc): tmp_dir.dvc_gen("file", "file") (tmp_dir / "file").write_text("something") - tree = DvcTree(dvc) - with tree.open(PathInfo(tmp_dir) / "file", "r") as fobj: - # NOTE: Unlike RepoTree, DvcTree should not + fs = DvcFileSystem(dvc) + with fs.open(PathInfo(tmp_dir) / "file", "r") as fobj: + # NOTE: Unlike RepoFileSystem, DvcFileSystem should not # be affected by a dirty workspace. assert fobj.read() == "file" @@ -43,11 +43,11 @@ def test_open_dirty_no_hash(tmp_dir, dvc): tmp_dir.gen("file", "file") (tmp_dir / "file.dvc").write_text("outs:\n- path: file\n") - tree = DvcTree(dvc) - # NOTE: Unlike RepoTree, DvcTree should not + fs = DvcFileSystem(dvc) + # NOTE: Unlike RepoFileSystem, DvcFileSystem should not # be affected by a dirty workspace. with pytest.raises(FileNotFoundError): - with tree.open(PathInfo(tmp_dir) / "file", "r"): + with fs.open(PathInfo(tmp_dir) / "file", "r"): pass @@ -66,28 +66,28 @@ def test_open_in_history(tmp_dir, scm, dvc): if rev == "workspace": continue - tree = DvcTree(dvc) - with tree.open(PathInfo(tmp_dir) / "foo", "r") as fobj: + fs = DvcFileSystem(dvc) + with fs.open(PathInfo(tmp_dir) / "foo", "r") as fobj: assert fobj.read() == "foo" def test_isdir_isfile(tmp_dir, dvc): tmp_dir.gen({"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}}) - tree = DvcTree(dvc) - assert not tree.isdir("datadir") - assert not tree.isfile("datadir") - assert not tree.isdir("datafile") - assert not tree.isfile("datafile") + fs = DvcFileSystem(dvc) + assert not fs.isdir("datadir") + assert not fs.isfile("datadir") + assert not fs.isdir("datafile") + assert not fs.isfile("datafile") dvc.add(["datadir", "datafile"]) shutil.rmtree(tmp_dir / "datadir") (tmp_dir / "datafile").unlink() - assert tree.isdir("datadir") - assert not tree.isfile("datadir") - assert not tree.isdir("datafile") - assert tree.isfile("datafile") + assert fs.isdir("datadir") + assert not fs.isfile("datadir") + assert not fs.isdir("datafile") + assert fs.isfile("datafile") def test_isdir_mixed(tmp_dir, dvc): @@ -95,9 +95,9 @@ def test_isdir_mixed(tmp_dir, dvc): dvc.add(str(tmp_dir / "dir" / "foo")) - tree = DvcTree(dvc) - assert tree.isdir("dir") - assert not tree.isfile("dir") + fs = DvcFileSystem(dvc) + assert fs.isdir("dir") + assert not fs.isfile("dir") def test_walk(tmp_dir, dvc): @@ -113,7 +113,7 @@ def test_walk(tmp_dir, dvc): ) dvc.add("dir", recursive=True) - tree = DvcTree(dvc) + fs = DvcFileSystem(dvc) expected = [ str(tmp_dir / "dir" / "subdir1"), @@ -126,7 +126,7 @@ def test_walk(tmp_dir, dvc): ] actual = [] - for root, dirs, files in tree.walk("dir"): + for root, dirs, files in fs.walk("dir"): for entry in dirs + files: actual.append(os.path.join(root, entry)) @@ -147,7 +147,7 @@ def test_walk_dir(tmp_dir, dvc): ) dvc.add("dir") - tree = DvcTree(dvc) + fs = DvcFileSystem(dvc) expected = [ str(tmp_dir / "dir" / "subdir1"), @@ -160,7 +160,7 @@ def test_walk_dir(tmp_dir, dvc): ] actual = [] - for root, dirs, files in tree.walk("dir"): + for root, dirs, files in fs.walk("dir"): for entry in dirs + files: actual.append(os.path.join(root, entry)) @@ -173,36 +173,36 @@ def test_walk_onerror(tmp_dir, dvc): raise exc tmp_dir.dvc_gen("foo", "foo") - tree = DvcTree(dvc) + fs = DvcFileSystem(dvc) # path does not exist - for _ in tree.walk("dir"): + for _ in fs.walk("dir"): pass with pytest.raises(OSError): - for _ in tree.walk("dir", onerror=onerror): + for _ in fs.walk("dir", onerror=onerror): pass # path is not a directory - for _ in tree.walk("foo"): + for _ in fs.walk("foo"): pass with pytest.raises(OSError): - for _ in tree.walk("foo", onerror=onerror): + for _ in fs.walk("foo", onerror=onerror): pass def test_isdvc(tmp_dir, dvc): tmp_dir.gen({"foo": "foo", "bar": "bar"}) dvc.add("foo") - tree = DvcTree(dvc) - assert tree.isdvc("foo") - assert not tree.isdvc("bar") + fs = DvcFileSystem(dvc) + assert fs.isdvc("foo") + assert not fs.isdvc("bar") def test_get_hash_file(tmp_dir, dvc): tmp_dir.dvc_gen({"foo": "foo"}) - tree = DvcTree(dvc) + fs = DvcFileSystem(dvc) assert ( - tree.info(PathInfo(tmp_dir) / "foo")["md5"] + fs.info(PathInfo(tmp_dir) / "foo")["md5"] == "acbd18db4cc2f85cedef654fccc4a4d8" ) @@ -213,10 +213,10 @@ def test_get_hash_dir(tmp_dir, dvc, mocker): tmp_dir.dvc_gen( {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} ) - tree = DvcTree(dvc) + fs = DvcFileSystem(dvc) get_file_hash_spy = mocker.spy(dvc_module.oid, "get_file_hash") assert ( - tree.info(PathInfo(tmp_dir) / "dir")["md5"] + fs.info(PathInfo(tmp_dir) / "dir")["md5"] == "8761c4e9acad696bee718615e23e22db.dir" ) assert not get_file_hash_spy.called @@ -226,16 +226,16 @@ def test_get_hash_granular(tmp_dir, dvc): tmp_dir.dvc_gen( {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} ) - tree = DvcTree(dvc) + fs = DvcFileSystem(dvc) subdir = PathInfo(tmp_dir) / "dir" / "subdir" - assert tree.info(subdir).get("md5") is None - assert get_hash(subdir, tree, "md5") == HashInfo( + assert fs.info(subdir).get("md5") is None + assert get_hash(subdir, fs, "md5") == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir", ) assert ( - tree.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc" + fs.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc" ) - assert get_hash(subdir / "data", tree, "md5") == HashInfo( + assert get_hash(subdir / "data", fs, "md5") == HashInfo( "md5", "8d777f385d3dfec8815d20f7496026dc", ) @@ -244,10 +244,10 @@ def test_get_hash_dirty_file(tmp_dir, dvc): tmp_dir.dvc_gen("file", "file") (tmp_dir / "file").write_text("something") - tree = DvcTree(dvc) + fs = DvcFileSystem(dvc) expected = "8c7dd922ad47494fc02c388e12c00eac" - assert tree.info(PathInfo(tmp_dir) / "file").get("md5") == expected - assert get_hash(PathInfo(tmp_dir) / "file", tree, "md5") == HashInfo( + assert fs.info(PathInfo(tmp_dir) / "file").get("md5") == expected + assert get_hash(PathInfo(tmp_dir) / "file", fs, "md5") == HashInfo( "md5", expected ) @@ -256,9 +256,9 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}) (tmp_dir / "dir" / "baz").write_text("baz") - tree = DvcTree(dvc) + fs = DvcFileSystem(dvc) expected = "5ea40360f5b4ec688df672a4db9c17d1.dir" - assert tree.info(PathInfo(tmp_dir) / "dir").get("md5") == expected - assert get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") == HashInfo( + assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") == expected + assert get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") == HashInfo( "md5", expected ) diff --git a/tests/unit/tree/test_hdfs.py b/tests/unit/fs/test_hdfs.py similarity index 94% rename from tests/unit/tree/test_hdfs.py rename to tests/unit/fs/test_hdfs.py index aaa901ba9..0956d38d3 100644 --- a/tests/unit/tree/test_hdfs.py +++ b/tests/unit/fs/test_hdfs.py @@ -1,7 +1,7 @@ import os +from dvc.fs.hdfs import _hadoop_fs_checksum from dvc.path_info import URLInfo -from dvc.tree.hdfs import _hadoop_fs_checksum def test_hadoop_fs_checksum(mocker): diff --git a/tests/unit/tree/test_repo.py b/tests/unit/fs/test_repo.py similarity index 66% rename from tests/unit/tree/test_repo.py rename to tests/unit/fs/test_repo.py index bb84f6500..29667aeda 100644 --- a/tests/unit/tree/test_repo.py +++ b/tests/unit/fs/test_repo.py @@ -4,10 +4,10 @@ from unittest import mock import pytest +from dvc.fs.repo import RepoFileSystem from dvc.hash_info import HashInfo from dvc.oid import get_hash from dvc.path_info import PathInfo -from dvc.tree.repo import RepoTree def test_exists(tmp_dir, dvc): @@ -15,8 +15,8 @@ def test_exists(tmp_dir, dvc): dvc.add("foo") (tmp_dir / "foo").unlink() - tree = RepoTree(dvc) - assert tree.exists("foo") + fs = RepoFileSystem(dvc) + assert fs.exists("foo") def test_open(tmp_dir, dvc): @@ -24,8 +24,8 @@ def test_open(tmp_dir, dvc): dvc.add("foo") (tmp_dir / "foo").unlink() - tree = RepoTree(dvc) - with tree.open(PathInfo(tmp_dir) / "foo", "r") as fobj: + fs = RepoFileSystem(dvc) + with fs.open(PathInfo(tmp_dir) / "foo", "r") as fobj: assert fobj.read() == "foo" @@ -33,8 +33,8 @@ def test_open_dirty_hash(tmp_dir, dvc): tmp_dir.dvc_gen("file", "file") (tmp_dir / "file").write_text("something") - tree = RepoTree(dvc) - with tree.open(PathInfo(tmp_dir) / "file", "r") as fobj: + fs = RepoFileSystem(dvc) + with fs.open(PathInfo(tmp_dir) / "file", "r") as fobj: assert fobj.read() == "something" @@ -42,8 +42,8 @@ def test_open_dirty_no_hash(tmp_dir, dvc): tmp_dir.gen("file", "file") (tmp_dir / "file.dvc").write_text("outs:\n- path: file\n") - tree = RepoTree(dvc) - with tree.open(PathInfo(tmp_dir) / "file", "r") as fobj: + fs = RepoFileSystem(dvc) + with fs.open(PathInfo(tmp_dir) / "file", "r") as fobj: assert fobj.read() == "file" @@ -62,32 +62,32 @@ def test_open_in_history(tmp_dir, scm, dvc): if rev == "workspace": continue - tree = RepoTree(dvc) - with tree.open(PathInfo(tmp_dir) / "foo", "r") as fobj: + fs = RepoFileSystem(dvc) + with fs.open(PathInfo(tmp_dir) / "foo", "r") as fobj: assert fobj.read() == "foo" def test_isdir_isfile(tmp_dir, dvc): tmp_dir.gen({"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}}) - tree = RepoTree(dvc) - assert tree.isdir("datadir") - assert not tree.isfile("datadir") - assert not tree.isdvc("datadir") - assert not tree.isdir("datafile") - assert tree.isfile("datafile") - assert not tree.isdvc("datafile") + fs = RepoFileSystem(dvc) + assert fs.isdir("datadir") + assert not fs.isfile("datadir") + assert not fs.isdvc("datadir") + assert not fs.isdir("datafile") + assert fs.isfile("datafile") + assert not fs.isdvc("datafile") dvc.add(["datadir", "datafile"]) shutil.rmtree(tmp_dir / "datadir") (tmp_dir / "datafile").unlink() - assert tree.isdir("datadir") - assert not tree.isfile("datadir") - assert tree.isdvc("datadir") - assert not tree.isdir("datafile") - assert tree.isfile("datafile") - assert tree.isdvc("datafile") + assert fs.isdir("datadir") + assert not fs.isfile("datadir") + assert fs.isdvc("datadir") + assert not fs.isdir("datafile") + assert fs.isfile("datafile") + assert fs.isdvc("datafile") def test_exists_isdir_isfile_dirty(tmp_dir, dvc): @@ -95,35 +95,35 @@ def test_exists_isdir_isfile_dirty(tmp_dir, dvc): {"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}} ) - tree = RepoTree(dvc) + fs = RepoFileSystem(dvc) shutil.rmtree(tmp_dir / "datadir") (tmp_dir / "datafile").unlink() root = PathInfo(tmp_dir) - assert tree.exists(root / "datafile") - assert tree.exists(root / "datadir") - assert tree.exists(root / "datadir" / "foo") - assert tree.isfile(root / "datafile") - assert not tree.isfile(root / "datadir") - assert tree.isfile(root / "datadir" / "foo") - assert not tree.isdir(root / "datafile") - assert tree.isdir(root / "datadir") - assert not tree.isdir(root / "datadir" / "foo") + assert fs.exists(root / "datafile") + assert fs.exists(root / "datadir") + assert fs.exists(root / "datadir" / "foo") + assert fs.isfile(root / "datafile") + assert not fs.isfile(root / "datadir") + assert fs.isfile(root / "datadir" / "foo") + assert not fs.isdir(root / "datafile") + assert fs.isdir(root / "datadir") + assert not fs.isdir(root / "datadir" / "foo") # NOTE: creating file instead of dir and dir instead of file tmp_dir.gen({"datadir": "data", "datafile": {"foo": "foo", "bar": "bar"}}) - assert tree.exists(root / "datafile") - assert tree.exists(root / "datadir") - assert not tree.exists(root / "datadir" / "foo") - assert tree.exists(root / "datafile" / "foo") - assert not tree.isfile(root / "datafile") - assert tree.isfile(root / "datadir") - assert not tree.isfile(root / "datadir" / "foo") - assert tree.isfile(root / "datafile" / "foo") - assert tree.isdir(root / "datafile") - assert not tree.isdir(root / "datadir") - assert not tree.isdir(root / "datadir" / "foo") - assert not tree.isdir(root / "datafile" / "foo") + assert fs.exists(root / "datafile") + assert fs.exists(root / "datadir") + assert not fs.exists(root / "datadir" / "foo") + assert fs.exists(root / "datafile" / "foo") + assert not fs.isfile(root / "datafile") + assert fs.isfile(root / "datadir") + assert not fs.isfile(root / "datadir" / "foo") + assert fs.isfile(root / "datafile" / "foo") + assert fs.isdir(root / "datafile") + assert not fs.isdir(root / "datadir") + assert not fs.isdir(root / "datadir" / "foo") + assert not fs.isdir(root / "datafile" / "foo") def test_isdir_mixed(tmp_dir, dvc): @@ -131,9 +131,9 @@ def test_isdir_mixed(tmp_dir, dvc): dvc.add(str(tmp_dir / "dir" / "foo")) - tree = RepoTree(dvc) - assert tree.isdir("dir") - assert not tree.isfile("dir") + fs = RepoFileSystem(dvc) + assert fs.isdir("dir") + assert not fs.isfile("dir") @pytest.mark.parametrize( @@ -161,7 +161,7 @@ def test_walk(tmp_dir, dvc, dvcfiles, extra_expected): ) dvc.add(str(tmp_dir / "dir"), recursive=True) tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) - tree = RepoTree(dvc) + fs = RepoFileSystem(dvc) expected = [ PathInfo("dir") / "subdir1", @@ -174,7 +174,7 @@ def test_walk(tmp_dir, dvc, dvcfiles, extra_expected): ] actual = [] - for root, dirs, files in tree.walk("dir", dvcfiles=dvcfiles): + for root, dirs, files in fs.walk("dir", dvcfiles=dvcfiles): for entry in dirs + files: actual.append(os.path.join(root, entry)) @@ -196,7 +196,7 @@ def test_walk_dirty(tmp_dir, dvc): tmp_dir.gen({"dir": {"bar": "bar", "subdir3": {"foo3": "foo3"}}}) (tmp_dir / "dir" / "foo").unlink() - tree = RepoTree(dvc) + fs = RepoFileSystem(dvc) expected = [ PathInfo("dir") / "subdir1", PathInfo("dir") / "subdir2", @@ -209,7 +209,7 @@ def test_walk_dirty(tmp_dir, dvc): ] actual = [] - for root, dirs, files in tree.walk("dir"): + for root, dirs, files in fs.walk("dir"): for entry in dirs + files: actual.append(os.path.join(root, entry)) @@ -224,12 +224,12 @@ def test_walk_dirty_cached_dir(tmp_dir, scm, dvc): ) (tmp_dir / "data" / "foo").unlink() - tree = RepoTree(dvc) + fs = RepoFileSystem(dvc) data = PathInfo(tmp_dir) / "data" actual = [] - for root, dirs, files in tree.walk(data): + for root, dirs, files in fs.walk(data): for entry in dirs + files: actual.append(os.path.join(root, entry)) @@ -248,7 +248,7 @@ def test_walk_mixed_dir(tmp_dir, scm, dvc): ) tmp_dir.scm.commit("add dir") - tree = RepoTree(dvc) + fs = RepoFileSystem(dvc) expected = [ str(PathInfo("dir") / "foo"), @@ -256,7 +256,7 @@ def test_walk_mixed_dir(tmp_dir, scm, dvc): str(PathInfo("dir") / ".gitignore"), ] actual = [] - for root, dirs, files in tree.walk("dir"): + for root, dirs, files in fs.walk("dir"): for entry in dirs + files: actual.append(os.path.join(root, entry)) @@ -269,20 +269,20 @@ def test_walk_onerror(tmp_dir, dvc): raise exc tmp_dir.dvc_gen("foo", "foo") - tree = RepoTree(dvc) + fs = RepoFileSystem(dvc) # path does not exist - for _ in tree.walk("dir"): + for _ in fs.walk("dir"): pass with pytest.raises(OSError): - for _ in tree.walk("dir", onerror=onerror): + for _ in fs.walk("dir", onerror=onerror): pass # path is not a directory - for _ in tree.walk("foo"): + for _ in fs.walk("foo"): pass with pytest.raises(OSError): - for _ in tree.walk("foo", onerror=onerror): + for _ in fs.walk("foo", onerror=onerror): pass @@ -290,12 +290,12 @@ def test_isdvc(tmp_dir, dvc): tmp_dir.gen({"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}}) dvc.add("foo") dvc.add("dir") - tree = RepoTree(dvc) - assert tree.isdvc("foo") - assert not tree.isdvc("bar") - assert tree.isdvc("dir") - assert not tree.isdvc("dir/baz") - assert tree.isdvc("dir/baz", recursive=True) + fs = RepoFileSystem(dvc) + assert fs.isdvc("foo") + assert not fs.isdvc("bar") + assert fs.isdvc("dir") + assert not fs.isdvc("dir/baz") + assert fs.isdvc("dir/baz", recursive=True) def make_subrepo(dir_, scm, config=None): @@ -309,7 +309,7 @@ def make_subrepo(dir_, scm, config=None): def test_subrepos(tmp_dir, scm, dvc): tmp_dir.scm_gen( - {"dir": {"repo.txt": "file to confuse RepoTree"}}, + {"dir": {"repo.txt": "file to confuse RepoFileSystem"}}, commit="dir/repo.txt", ) @@ -324,11 +324,11 @@ def test_subrepos(tmp_dir, scm, dvc): {"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR" ) - dvc.tree._reset() - tree = RepoTree(dvc, subrepos=True) + dvc.fs._reset() + fs = RepoFileSystem(dvc, subrepos=True) - def assert_tree_belongs_to_repo(ret_val): - method = tree._get_repo + def assert_fs_belongs_to_repo(ret_val): + method = fs._get_repo def f(*args, **kwargs): r = method(*args, **kwargs) @@ -338,36 +338,32 @@ def test_subrepos(tmp_dir, scm, dvc): return f with mock.patch.object( - tree, - "_get_repo", - side_effect=assert_tree_belongs_to_repo(subrepo1.dvc), + fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo1.dvc), ): - assert tree.exists(subrepo1 / "foo") is True - assert tree.exists(subrepo1 / "bar") is False + assert fs.exists(subrepo1 / "foo") is True + assert fs.exists(subrepo1 / "bar") is False - assert tree.isfile(subrepo1 / "foo") is True - assert tree.isfile(subrepo1 / "dir1" / "bar") is True - assert tree.isfile(subrepo1 / "dir1") is False + assert fs.isfile(subrepo1 / "foo") is True + assert fs.isfile(subrepo1 / "dir1" / "bar") is True + assert fs.isfile(subrepo1 / "dir1") is False - assert tree.isdir(subrepo1 / "dir1") is True - assert tree.isdir(subrepo1 / "dir1" / "bar") is False - assert tree.isdvc(subrepo1 / "foo") is True + assert fs.isdir(subrepo1 / "dir1") is True + assert fs.isdir(subrepo1 / "dir1" / "bar") is False + assert fs.isdvc(subrepo1 / "foo") is True with mock.patch.object( - tree, - "_get_repo", - side_effect=assert_tree_belongs_to_repo(subrepo2.dvc), + fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo2.dvc), ): - assert tree.exists(subrepo2 / "lorem") is True - assert tree.exists(subrepo2 / "ipsum") is False + assert fs.exists(subrepo2 / "lorem") is True + assert fs.exists(subrepo2 / "ipsum") is False - assert tree.isfile(subrepo2 / "lorem") is True - assert tree.isfile(subrepo2 / "dir2" / "ipsum") is True - assert tree.isfile(subrepo2 / "dir2") is False + assert fs.isfile(subrepo2 / "lorem") is True + assert fs.isfile(subrepo2 / "dir2" / "ipsum") is True + assert fs.isfile(subrepo2 / "dir2") is False - assert tree.isdir(subrepo2 / "dir2") is True - assert tree.isdir(subrepo2 / "dir2" / "ipsum") is False - assert tree.isdvc(subrepo2 / "lorem") is True + assert fs.isdir(subrepo2 / "dir2") is True + assert fs.isdir(subrepo2 / "dir2" / "ipsum") is False + assert fs.isdvc(subrepo2 / "lorem") is True @pytest.mark.parametrize( @@ -389,7 +385,7 @@ def test_subrepos(tmp_dir, scm, dvc): ) def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected): tmp_dir.scm_gen( - {"dir": {"repo.txt": "file to confuse RepoTree"}}, + {"dir": {"repo.txt": "file to confuse RepoFileSystem"}}, commit="dir/repo.txt", ) @@ -405,9 +401,9 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected): {"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR" ) - # using tree that does not have dvcignore - dvc.tree._reset() - tree = RepoTree(dvc, subrepos=True) + # using fs that does not have dvcignore + dvc.fs._reset() + fs = RepoFileSystem(dvc, subrepos=True) expected = [ PathInfo("dir") / "repo", PathInfo("dir") / "repo.txt", @@ -423,22 +419,22 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected): ] actual = [] - for root, dirs, files in tree.walk( - os.path.join(tree.root_dir, "dir"), dvcfiles=dvcfiles + for root, dirs, files in fs.walk( + os.path.join(fs.root_dir, "dir"), dvcfiles=dvcfiles ): for entry in dirs + files: actual.append(os.path.join(root, entry)) expected = [ - os.path.join(tree.root_dir, path) for path in expected + extra_expected + os.path.join(fs.root_dir, path) for path in expected + extra_expected ] assert set(actual) == set(expected) assert len(actual) == len(expected) -def test_repo_tree_no_subrepos(tmp_dir, dvc, scm): +def test_repo_fs_no_subrepos(tmp_dir, dvc, scm): tmp_dir.scm_gen( - {"dir": {"repo.txt": "file to confuse RepoTree"}}, + {"dir": {"repo.txt": "file to confuse RepoFileSystem"}}, commit="dir/repo.txt", ) tmp_dir.dvc_gen({"lorem": "lorem"}, commit="add foo") @@ -448,9 +444,9 @@ def test_repo_tree_no_subrepos(tmp_dir, dvc, scm): subrepo.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO") subrepo.scm_gen({"ipsum": "ipsum"}, commit="BAR") - # using tree that does not have dvcignore - dvc.tree._reset() - tree = RepoTree(dvc, subrepos=False) + # using fs that does not have dvcignore + dvc.fs._reset() + fs = RepoFileSystem(dvc, subrepos=False) expected = [ tmp_dir / ".dvcignore", tmp_dir / ".gitignore", @@ -461,7 +457,7 @@ def test_repo_tree_no_subrepos(tmp_dir, dvc, scm): ] actual = [] - for root, dirs, files in tree.walk(tmp_dir, dvcfiles=True): + for root, dirs, files in fs.walk(tmp_dir, dvcfiles=True): for entry in dirs + files: actual.append(os.path.normpath(os.path.join(root, entry))) @@ -469,44 +465,44 @@ def test_repo_tree_no_subrepos(tmp_dir, dvc, scm): assert set(actual) == set(expected) assert len(actual) == len(expected) - assert tree.isfile(tmp_dir / "lorem") is True - assert tree.isfile(tmp_dir / "dir" / "repo" / "foo") is False - assert tree.isdir(tmp_dir / "dir" / "repo") is False - assert tree.isdir(tmp_dir / "dir") is True + assert fs.isfile(tmp_dir / "lorem") is True + assert fs.isfile(tmp_dir / "dir" / "repo" / "foo") is False + assert fs.isdir(tmp_dir / "dir" / "repo") is False + assert fs.isdir(tmp_dir / "dir") is True - assert tree.isdvc(tmp_dir / "lorem") is True - assert tree.isdvc(tmp_dir / "dir" / "repo" / "dir1") is False + assert fs.isdvc(tmp_dir / "lorem") is True + assert fs.isdvc(tmp_dir / "dir" / "repo" / "dir1") is False - assert tree.exists(tmp_dir / "dir" / "repo.txt") is True - assert tree.exists(tmp_dir / "repo" / "ipsum") is False + assert fs.exists(tmp_dir / "dir" / "repo.txt") is True + assert fs.exists(tmp_dir / "repo" / "ipsum") is False def test_get_hash_cached_file(tmp_dir, dvc, mocker): tmp_dir.dvc_gen({"foo": "foo"}) - tree = RepoTree(dvc) + fs = RepoFileSystem(dvc) expected = "acbd18db4cc2f85cedef654fccc4a4d8" - assert tree.info(PathInfo(tmp_dir) / "foo").get("md5") is None - assert get_hash(PathInfo(tmp_dir) / "foo", tree, "md5") == HashInfo( + assert fs.info(PathInfo(tmp_dir) / "foo").get("md5") is None + assert get_hash(PathInfo(tmp_dir) / "foo", fs, "md5") == HashInfo( "md5", expected, ) (tmp_dir / "foo").unlink() - assert tree.info(PathInfo(tmp_dir) / "foo")["md5"] == expected + assert fs.info(PathInfo(tmp_dir) / "foo")["md5"] == expected def test_get_hash_cached_dir(tmp_dir, dvc, mocker): tmp_dir.dvc_gen( {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} ) - tree = RepoTree(dvc) + fs = RepoFileSystem(dvc) expected = "8761c4e9acad696bee718615e23e22db.dir" - assert tree.info(PathInfo(tmp_dir) / "dir").get("md5") is None - assert get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") == HashInfo( + assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") is None + assert get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") == HashInfo( "md5", "8761c4e9acad696bee718615e23e22db.dir", ) shutil.rmtree(tmp_dir / "dir") - assert tree.info(PathInfo(tmp_dir) / "dir")["md5"] == expected - assert get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") == HashInfo( + assert fs.info(PathInfo(tmp_dir) / "dir")["md5"] == expected + assert get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") == HashInfo( "md5", "8761c4e9acad696bee718615e23e22db.dir", ) @@ -515,19 +511,19 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker): tmp_dir.dvc_gen( {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} ) - tree = RepoTree(dvc) + fs = RepoFileSystem(dvc) subdir = PathInfo(tmp_dir) / "dir" / "subdir" - assert tree.info(subdir).get("md5") is None - assert get_hash(subdir, tree, "md5") == HashInfo( + assert fs.info(subdir).get("md5") is None + assert get_hash(subdir, fs, "md5") == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir", ) - assert tree.info(subdir / "data").get("md5") is None - assert get_hash(subdir / "data", tree, "md5") == HashInfo( + assert fs.info(subdir / "data").get("md5") is None + assert get_hash(subdir / "data", fs, "md5") == HashInfo( "md5", "8d777f385d3dfec8815d20f7496026dc", ) (tmp_dir / "dir" / "subdir" / "data").unlink() assert ( - tree.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc" + fs.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc" ) @@ -543,8 +539,8 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc): ) tmp_dir.scm.commit("add dir") - tree = RepoTree(dvc) - actual = get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") + fs = RepoFileSystem(dvc) + actual = get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") expected = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir") assert actual == expected @@ -553,18 +549,18 @@ def test_get_hash_dirty_file(tmp_dir, dvc): tmp_dir.dvc_gen("file", "file") (tmp_dir / "file").write_text("something") - tree = RepoTree(dvc) - assert tree.info(PathInfo(tmp_dir) / "file").get("md5") is None - actual = get_hash(PathInfo(tmp_dir) / "file", tree, "md5") + fs = RepoFileSystem(dvc) + assert fs.info(PathInfo(tmp_dir) / "file").get("md5") is None + actual = get_hash(PathInfo(tmp_dir) / "file", fs, "md5") expected = HashInfo("md5", "437b930db84b8079c2dd804a71936b5f") assert actual == expected (tmp_dir / "file").unlink() assert ( - tree.info(PathInfo(tmp_dir) / "file")["md5"] + fs.info(PathInfo(tmp_dir) / "file")["md5"] == "8c7dd922ad47494fc02c388e12c00eac" ) - actual = get_hash(PathInfo(tmp_dir) / "file", tree, "md5") + actual = get_hash(PathInfo(tmp_dir) / "file", fs, "md5") expected = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac") assert actual == expected @@ -573,8 +569,8 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}) (tmp_dir / "dir" / "baz").write_text("baz") - tree = RepoTree(dvc) - actual = get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") + fs = RepoFileSystem(dvc) + actual = get_hash(PathInfo(tmp_dir) / "dir", fs, "md5") expected = HashInfo("md5", "ba75a2162ca9c29acecb7957105a0bc2.dir") assert actual == expected assert actual.dir_info.nfiles == 3 @@ -625,7 +621,7 @@ def test_walk_nested_subrepos(tmp_dir, dvc, scm, traverse_subrepos): expected[str(tmp_dir / "subrepo1")].add("subrepo3") actual = {} - tree = RepoTree(dvc, subrepos=traverse_subrepos) - for root, dirs, files in tree.walk(str(tmp_dir)): + fs = RepoFileSystem(dvc, subrepos=traverse_subrepos) + for root, dirs, files in fs.walk(str(tmp_dir)): actual[root] = set(dirs + files) assert expected == actual diff --git a/tests/unit/tree/test_repo_metadata.py b/tests/unit/fs/test_repo_metadata.py similarity index 77% rename from tests/unit/tree/test_repo_metadata.py rename to tests/unit/fs/test_repo_metadata.py index 903e2a360..ad967be76 100644 --- a/tests/unit/tree/test_repo_metadata.py +++ b/tests/unit/fs/test_repo_metadata.py @@ -1,12 +1,12 @@ import pytest +from dvc.fs.repo import RepoFileSystem from dvc.path_info import PathInfo -from dvc.tree.repo import RepoTree -from tests.unit.tree.test_repo import make_subrepo +from tests.unit.fs.test_repo import make_subrepo @pytest.fixture -def repo_tree(tmp_dir, dvc, scm): +def repo_fs(tmp_dir, dvc, scm): fs_structure = { "models": { # mixed dvc + git directory "train.py": "train dot py", @@ -37,14 +37,14 @@ def repo_tree(tmp_dir, dvc, scm): tmp_dir.scm_gen(fs_structure, commit="repo init") tmp_dir.dvc_gen(dvc_structure, commit="use dvc") - yield RepoTree(dvc, subrepos=True) + yield RepoFileSystem(dvc, subrepos=True) -def test_metadata_not_existing(repo_tree): +def test_metadata_not_existing(repo_fs): path = PathInfo("path") / "that" / "does" / "not" / "exist" with pytest.raises(FileNotFoundError): - repo_tree.metadata(path) + repo_fs.metadata(path) @pytest.mark.parametrize( @@ -57,12 +57,12 @@ def test_metadata_not_existing(repo_tree): PathInfo("src") / "utils" / "serve_model.py", ], ) -def test_metadata_git_tracked_file(repo_tree, path): - root = PathInfo(repo_tree.root_dir) - meta = repo_tree.metadata(path) +def test_metadata_git_tracked_file(repo_fs, path): + root = PathInfo(repo_fs.root_dir) + meta = repo_fs.metadata(path) assert meta.path_info == root / path - assert meta.repo.root_dir == repo_tree.root_dir + assert meta.repo.root_dir == repo_fs.root_dir assert not meta.is_output assert not meta.part_of_output assert not meta.contains_outputs @@ -96,12 +96,12 @@ def test_metadata_git_tracked_file(repo_tree, path): ), ], ) -def test_metadata_dvc_tracked_file(repo_tree, path, outs, is_output): - root = PathInfo(repo_tree.root_dir) - meta = repo_tree.metadata(path) +def test_metadata_dvc_tracked_file(repo_fs, path, outs, is_output): + root = PathInfo(repo_fs.root_dir) + meta = repo_fs.metadata(path) assert meta.path_info == root / path - assert meta.repo.root_dir == repo_tree.root_dir + assert meta.repo.root_dir == repo_fs.root_dir assert meta.is_output == is_output assert meta.part_of_output != is_output assert not meta.contains_outputs @@ -114,12 +114,12 @@ def test_metadata_dvc_tracked_file(repo_tree, path, outs, is_output): @pytest.mark.parametrize("path", ["src", "src/utils"]) -def test_metadata_git_only_dirs(repo_tree, path): - root = PathInfo(repo_tree.root_dir) - meta = repo_tree.metadata(path) +def test_metadata_git_only_dirs(repo_fs, path): + root = PathInfo(repo_fs.root_dir) + meta = repo_fs.metadata(path) assert meta.path_info == root / path - assert meta.repo.root_dir == repo_tree.root_dir + assert meta.repo.root_dir == repo_fs.root_dir assert not meta.is_output assert not meta.part_of_output assert not meta.contains_outputs @@ -138,12 +138,12 @@ def test_metadata_git_only_dirs(repo_tree, path): ("models", [PathInfo("models") / "transform.pickle"]), ], ) -def test_metadata_git_dvc_mixed_dirs(repo_tree, path, expected_outs): - root = PathInfo(repo_tree.root_dir) - meta = repo_tree.metadata(root / path) +def test_metadata_git_dvc_mixed_dirs(repo_fs, path, expected_outs): + root = PathInfo(repo_fs.root_dir) + meta = repo_fs.metadata(root / path) assert meta.path_info == root / path - assert meta.repo.root_dir == repo_tree.root_dir + assert meta.repo.root_dir == repo_fs.root_dir assert not meta.is_output assert not meta.part_of_output assert meta.contains_outputs @@ -166,13 +166,13 @@ def test_metadata_git_dvc_mixed_dirs(repo_tree, path, expected_outs): ("data/processed", False), ], ) -def test_metadata_dvc_only_dirs(repo_tree, path, is_output): - data = PathInfo(repo_tree.root_dir) / "data" - root = PathInfo(repo_tree.root_dir) - meta = repo_tree.metadata(root / path) +def test_metadata_dvc_only_dirs(repo_fs, path, is_output): + data = PathInfo(repo_fs.root_dir) / "data" + root = PathInfo(repo_fs.root_dir) + meta = repo_fs.metadata(root / path) assert meta.path_info == root / path - assert meta.repo.root_dir == repo_tree.root_dir + assert meta.repo.root_dir == repo_fs.root_dir assert meta.is_output == is_output assert meta.part_of_output != is_output assert not meta.contains_outputs @@ -184,14 +184,14 @@ def test_metadata_dvc_only_dirs(repo_tree, path, is_output): assert {out.path_info for out in meta.outs} == {data} -def test_metadata_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, repo_tree): +def test_metadata_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, repo_fs): subrepo = tmp_dir / "subrepo" make_subrepo(subrepo, scm) subrepo.scm_gen("foo", "foo", commit="add foo on subrepo") subrepo.dvc_gen("foobar", "foobar", commit="add foobar on subrepo") for path in ["subrepo", "subrepo/foo", "subrepo/foobar"]: - meta = repo_tree.metadata(tmp_dir / path) + meta = repo_fs.metadata(tmp_dir / path) assert meta.repo.root_dir == str( subrepo ), f"repo root didn't match for {path}" @@ -200,5 +200,5 @@ def test_metadata_on_subrepos(make_tmp_dir, tmp_dir, dvc, scm, repo_tree): external_dir = make_tmp_dir("external-output") external_dir.gen("bar", "bar") dvc.add(str(external_dir / "bar"), external=True) - meta = repo_tree.metadata(external_dir / "bar") + meta = repo_fs.metadata(external_dir / "bar") assert meta.repo.root_dir == str(tmp_dir) diff --git a/tests/unit/tree/test_s3.py b/tests/unit/fs/test_s3.py similarity index 70% rename from tests/unit/tree/test_s3.py rename to tests/unit/fs/test_s3.py index caf776f14..568534233 100644 --- a/tests/unit/tree/test_s3.py +++ b/tests/unit/fs/test_s3.py @@ -2,7 +2,7 @@ import pytest from dvc.config import ConfigError from dvc.exceptions import DvcException -from dvc.tree.s3 import S3Tree +from dvc.fs.s3 import S3FileSystem bucket_name = "bucket-name" prefix = "some/prefix" @@ -24,9 +24,9 @@ def grants(): def test_init(dvc): config = {"url": url} - tree = S3Tree(dvc, config) + fs = S3FileSystem(dvc, config) - assert tree.path_info == url + assert fs.path_info == url def test_grants(dvc): @@ -37,17 +37,15 @@ def test_grants(dvc): "grant_write_acp": "id=write-acp-permission-id", "grant_full_control": "id=full-control-permission-id", } - tree = S3Tree(dvc, config) + fs = S3FileSystem(dvc, config) assert ( - tree.extra_args["GrantRead"] + fs.extra_args["GrantRead"] == "id=read-permission-id,id=other-read-permission-id" ) - assert tree.extra_args["GrantReadACP"] == "id=read-acp-permission-id" - assert tree.extra_args["GrantWriteACP"] == "id=write-acp-permission-id" - assert ( - tree.extra_args["GrantFullControl"] == "id=full-control-permission-id" - ) + assert fs.extra_args["GrantReadACP"] == "id=read-acp-permission-id" + assert fs.extra_args["GrantWriteACP"] == "id=write-acp-permission-id" + assert fs.extra_args["GrantFullControl"] == "id=full-control-permission-id" def test_grants_mutually_exclusive_acl_error(dvc, grants): @@ -55,16 +53,16 @@ def test_grants_mutually_exclusive_acl_error(dvc, grants): config = {"url": url, "acl": "public-read", grant_option: grant_value} with pytest.raises(ConfigError): - S3Tree(dvc, config) + S3FileSystem(dvc, config) def test_sse_kms_key_id(dvc): - tree = S3Tree(dvc, {"url": url, "sse_kms_key_id": "key"}) - assert tree.extra_args["SSEKMSKeyId"] == "key" + fs = S3FileSystem(dvc, {"url": url, "sse_kms_key_id": "key"}) + assert fs.extra_args["SSEKMSKeyId"] == "key" def test_key_id_and_secret(dvc): - tree = S3Tree( + fs = S3FileSystem( dvc, { "url": url, @@ -73,44 +71,44 @@ def test_key_id_and_secret(dvc): "session_token": session_token, }, ) - assert tree.access_key_id == key_id - assert tree.secret_access_key == key_secret - assert tree.session_token == session_token + assert fs.access_key_id == key_id + assert fs.secret_access_key == key_secret + assert fs.session_token == session_token def test_get_s3_no_credentials(mocker): from botocore.exceptions import NoCredentialsError - tree = S3Tree(None, {}) + fs = S3FileSystem(None, {}) with pytest.raises(DvcException, match="Unable to find AWS credentials"): - with tree._get_s3(): + with fs._get_s3(): raise NoCredentialsError def test_get_s3_connection_error(mocker): from botocore.exceptions import EndpointConnectionError - tree = S3Tree(None, {}) + fs = S3FileSystem(None, {}) msg = "Unable to connect to 'AWS S3'." with pytest.raises(DvcException, match=msg): - with tree._get_s3(): + with fs._get_s3(): raise EndpointConnectionError(endpoint_url="url") def test_get_s3_connection_error_endpoint(mocker): from botocore.exceptions import EndpointConnectionError - tree = S3Tree(None, {"endpointurl": "https://example.com"}) + fs = S3FileSystem(None, {"endpointurl": "https://example.com"}) msg = "Unable to connect to 'https://example.com'." with pytest.raises(DvcException, match=msg): - with tree._get_s3(): + with fs._get_s3(): raise EndpointConnectionError(endpoint_url="url") def test_get_bucket(): - tree = S3Tree(None, {"url": "s3://mybucket/path"}) + fs = S3FileSystem(None, {"url": "s3://mybucket/path"}) with pytest.raises(DvcException, match="Bucket 'mybucket' does not exist"): - with tree._get_bucket("mybucket") as bucket: + with fs._get_bucket("mybucket") as bucket: raise bucket.meta.client.exceptions.NoSuchBucket({}, None) diff --git a/tests/unit/tree/test_tree.py b/tests/unit/fs/test_tree.py similarity index 67% rename from tests/unit/tree/test_tree.py rename to tests/unit/fs/test_tree.py index 2c4fc3af7..1613367eb 100644 --- a/tests/unit/tree/test_tree.py +++ b/tests/unit/fs/test_tree.py @@ -1,11 +1,11 @@ import pytest from dvc.config import ConfigError +from dvc.fs import get_cloud_fs from dvc.path_info import CloudURLInfo -from dvc.tree import get_cloud_tree -def test_get_cloud_tree(tmp_dir, dvc): +def test_get_cloud_fs(tmp_dir, dvc): tmp_dir.add_remote(name="base", url="s3://bucket/path", default=False) tmp_dir.add_remote(name="first", url="remote://base/first", default=False) tmp_dir.add_remote( @@ -16,12 +16,12 @@ def test_get_cloud_tree(tmp_dir, dvc): first = base / "first" second = first / "second" - assert get_cloud_tree(dvc, name="base").path_info == base - assert get_cloud_tree(dvc, name="first").path_info == first - assert get_cloud_tree(dvc, name="second").path_info == second + assert get_cloud_fs(dvc, name="base").path_info == base + assert get_cloud_fs(dvc, name="first").path_info == first + assert get_cloud_fs(dvc, name="second").path_info == second -def test_get_cloud_tree_validate(tmp_dir, dvc): +def test_get_cloud_fs_validate(tmp_dir, dvc): tmp_dir.add_remote( name="base", url="ssh://example.com/path", default=False ) @@ -36,13 +36,13 @@ def test_get_cloud_tree_validate(tmp_dir, dvc): default=False, ) - assert get_cloud_tree(dvc, name="base").config == { + assert get_cloud_fs(dvc, name="base").config == { "url": "ssh://example.com/path" } - assert get_cloud_tree(dvc, name="first").config == { + assert get_cloud_fs(dvc, name="first").config == { "url": "ssh://example.com/path/first", "type": ["symlink"], } with pytest.raises(ConfigError): - get_cloud_tree(dvc, name="second") + get_cloud_fs(dvc, name="second") diff --git a/tests/unit/output/test_local.py b/tests/unit/output/test_local.py index f04c3b973..49b61de52 100644 --- a/tests/unit/output/test_local.py +++ b/tests/unit/output/test_local.py @@ -20,7 +20,7 @@ class TestLocalOutput(TestDvc): def test_save_missing(self): o = self._get_output() - with patch.object(o.tree, "exists", return_value=False): + with patch.object(o.fs, "exists", return_value=False): with self.assertRaises(o.DoesNotExistError): o.save() diff --git a/tests/unit/output/test_output.py b/tests/unit/output/test_output.py index 62f477cfd..09127e628 100644 --- a/tests/unit/output/test_output.py +++ b/tests/unit/output/test_output.py @@ -75,10 +75,10 @@ def test_get_used_cache(exists, expected_message, mocker, caplog): mocker.patch.object(stage, "addressing", "stage.dvc") mocker.patch.object(stage, "wdir", ".") mocker.patch.object( - stage.repo.tree.dvcignore, "is_ignored", return_value=False, + stage.repo.fs.dvcignore, "is_ignored", return_value=False, ) mocker.patch.object( - stage.repo.tree.dvcignore, + stage.repo.fs.dvcignore, "check_ignore", return_value=_no_match("path"), ) diff --git a/tests/unit/remote/ssh/test_connection.py b/tests/unit/remote/ssh/test_connection.py index 875f57b8c..f77d48abd 100644 --- a/tests/unit/remote/ssh/test_connection.py +++ b/tests/unit/remote/ssh/test_connection.py @@ -6,9 +6,9 @@ import tempfile import pytest +from dvc.fs.ssh.connection import SSHConnection from dvc.info import get_fs_type from dvc.system import System -from dvc.tree.ssh.connection import SSHConnection here = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/unit/remote/ssh/test_pool.py b/tests/unit/remote/ssh/test_pool.py index 42abed76d..c2e0087d0 100644 --- a/tests/unit/remote/ssh/test_pool.py +++ b/tests/unit/remote/ssh/test_pool.py @@ -1,7 +1,7 @@ import pytest -from dvc.tree.pool import get_connection -from dvc.tree.ssh.connection import SSHConnection +from dvc.fs.pool import get_connection +from dvc.fs.ssh.connection import SSHConnection from tests.remotes.ssh import TEST_SSH_KEY_PATH, TEST_SSH_USER diff --git a/tests/unit/remote/ssh/test_ssh.py b/tests/unit/remote/ssh/test_ssh.py index 7c15b00f8..fe865833c 100644 --- a/tests/unit/remote/ssh/test_ssh.py +++ b/tests/unit/remote/ssh/test_ssh.py @@ -5,8 +5,8 @@ import sys import pytest from mock import mock_open, patch +from dvc.fs.ssh import SSHFileSystem from dvc.system import System -from dvc.tree.ssh import SSHTree def test_url(dvc): @@ -19,21 +19,21 @@ def test_url(dvc): url = f"ssh://{user}@{host}:{port}{path}" config = {"url": url} - tree = SSHTree(dvc, config) - assert tree.path_info == url + fs = SSHFileSystem(dvc, config) + assert fs.path_info == url # SCP-like URL ssh://[user@]host.xz:/absolute/path url = f"ssh://{user}@{host}:{path}" config = {"url": url} - tree = SSHTree(dvc, config) - assert tree.path_info == url + fs = SSHFileSystem(dvc, config) + assert fs.path_info == url def test_no_path(dvc): config = {"url": "ssh://127.0.0.1"} - tree = SSHTree(dvc, config) - assert tree.path_info.path == "" + fs = SSHFileSystem(dvc, config) + assert fs.path_info.path == "" mock_ssh_config = """ @@ -66,11 +66,11 @@ else: def test_ssh_host_override_from_config( mock_file, mock_exists, dvc, config, expected_host ): - tree = SSHTree(dvc, config) + fs = SSHFileSystem(dvc, config) - mock_exists.assert_called_with(SSHTree.ssh_config_filename()) - mock_file.assert_called_with(SSHTree.ssh_config_filename()) - assert tree.path_info.host == expected_host + mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename()) + mock_file.assert_called_with(SSHFileSystem.ssh_config_filename()) + assert fs.path_info.host == expected_host @pytest.mark.parametrize( @@ -94,11 +94,11 @@ def test_ssh_host_override_from_config( read_data=mock_ssh_config, ) def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user): - tree = SSHTree(dvc, config) + fs = SSHFileSystem(dvc, config) - mock_exists.assert_called_with(SSHTree.ssh_config_filename()) - mock_file.assert_called_with(SSHTree.ssh_config_filename()) - assert tree.path_info.user == expected_user + mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename()) + mock_file.assert_called_with(SSHFileSystem.ssh_config_filename()) + assert fs.path_info.user == expected_user @pytest.mark.parametrize( @@ -107,7 +107,7 @@ def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user): ({"url": "ssh://example.com:2222"}, 2222), ({"url": "ssh://example.com"}, 1234), ({"url": "ssh://example.com", "port": 4321}, 4321), - ({"url": "ssh://not_in_ssh_config.com"}, SSHTree.DEFAULT_PORT), + ({"url": "ssh://not_in_ssh_config.com"}, SSHFileSystem.DEFAULT_PORT), ({"url": "ssh://not_in_ssh_config.com:2222"}, 2222), ({"url": "ssh://not_in_ssh_config.com:2222", "port": 4321}, 4321), ], @@ -119,11 +119,11 @@ def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user): read_data=mock_ssh_config, ) def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port): - tree = SSHTree(dvc, config) + fs = SSHFileSystem(dvc, config) - mock_exists.assert_called_with(SSHTree.ssh_config_filename()) - mock_file.assert_called_with(SSHTree.ssh_config_filename()) - assert tree.path_info.port == expected_port + mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename()) + mock_file.assert_called_with(SSHFileSystem.ssh_config_filename()) + assert fs.path_info.port == expected_port @pytest.mark.parametrize( @@ -154,11 +154,11 @@ def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port): read_data=mock_ssh_config, ) def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile): - tree = SSHTree(dvc, config) + fs = SSHFileSystem(dvc, config) - mock_exists.assert_called_with(SSHTree.ssh_config_filename()) - mock_file.assert_called_with(SSHTree.ssh_config_filename()) - assert tree.keyfile == expected_keyfile + mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename()) + mock_file.assert_called_with(SSHFileSystem.ssh_config_filename()) + assert fs.keyfile == expected_keyfile @pytest.mark.parametrize( @@ -176,11 +176,11 @@ def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile): read_data=mock_ssh_config, ) def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth): - tree = SSHTree(dvc, config) + fs = SSHFileSystem(dvc, config) - mock_exists.assert_called_with(SSHTree.ssh_config_filename()) - mock_file.assert_called_with(SSHTree.ssh_config_filename()) - assert tree.gss_auth == expected_gss_auth + mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename()) + mock_file.assert_called_with(SSHFileSystem.ssh_config_filename()) + assert fs.gss_auth == expected_gss_auth @pytest.mark.parametrize( @@ -201,20 +201,20 @@ def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth): def test_ssh_allow_agent( mock_file, mock_exists, dvc, config, expected_allow_agent ): - tree = SSHTree(dvc, config) + fs = SSHFileSystem(dvc, config) - mock_exists.assert_called_with(SSHTree.ssh_config_filename()) - mock_file.assert_called_with(SSHTree.ssh_config_filename()) - assert tree.allow_agent == expected_allow_agent + mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename()) + mock_file.assert_called_with(SSHFileSystem.ssh_config_filename()) + assert fs.allow_agent == expected_allow_agent def test_hardlink_optimization(dvc, tmp_dir, ssh): - tree = SSHTree(dvc, ssh.config) + fs = SSHFileSystem(dvc, ssh.config) - from_info = tree.path_info / "empty" - to_info = tree.path_info / "link" + from_info = fs.path_info / "empty" + to_info = fs.path_info / "link" - with tree.open(from_info, "wb"): + with fs.open(from_info, "wb"): pass if os.name == "nt": @@ -222,5 +222,5 @@ def test_hardlink_optimization(dvc, tmp_dir, ssh): else: link_path = to_info.path - tree.hardlink(from_info, to_info) + fs.hardlink(from_info, to_info) assert not System.is_hardlink(link_path) diff --git a/tests/unit/remote/test_azure.py b/tests/unit/remote/test_azure.py index f3b582f57..397377cde 100644 --- a/tests/unit/remote/test_azure.py +++ b/tests/unit/remote/test_azure.py @@ -1,5 +1,5 @@ +from dvc.fs.azure import AzureFileSystem from dvc.path_info import PathInfo -from dvc.tree.azure import AzureTree container_name = "container-name" connection_string = ( @@ -15,27 +15,27 @@ def test_init_env_var(monkeypatch, dvc): monkeypatch.setenv("AZURE_STORAGE_CONNECTION_STRING", connection_string) config = {"url": "azure://"} - tree = AzureTree(dvc, config) - assert tree.path_info == "azure://" + container_name - assert tree._conn_str == connection_string + fs = AzureFileSystem(dvc, config) + assert fs.path_info == "azure://" + container_name + assert fs._conn_str == connection_string def test_init(dvc): prefix = "some/prefix" url = f"azure://{container_name}/{prefix}" config = {"url": url, "connection_string": connection_string} - tree = AzureTree(dvc, config) - assert tree.path_info == url - assert tree._conn_str == connection_string + fs = AzureFileSystem(dvc, config) + assert fs.path_info == url + assert fs._conn_str == connection_string def test_info(tmp_dir, azure): tmp_dir.gen("foo", "foo") - tree = AzureTree(None, azure.config) + fs = AzureFileSystem(None, azure.config) to_info = azure - tree.upload(PathInfo("foo"), to_info) - assert tree.exists(to_info) - hash_ = tree.info(to_info)["etag"] + fs.upload(PathInfo("foo"), to_info) + assert fs.exists(to_info) + hash_ = fs.info(to_info)["etag"] assert isinstance(hash_, str) assert hash_.strip("'").strip('"') == hash_ diff --git a/tests/unit/remote/test_base.py b/tests/unit/remote/test_base.py index c4734acd0..d45dcd2a9 100644 --- a/tests/unit/remote/test_base.py +++ b/tests/unit/remote/test_base.py @@ -4,8 +4,8 @@ import mock import pytest from dvc.cache.base import CloudCache +from dvc.fs.base import BaseFileSystem, RemoteCmdError from dvc.path_info import PathInfo -from dvc.tree.base import BaseTree, RemoteCmdError class _CallableOrNone: @@ -26,19 +26,21 @@ def test_cmd_error(dvc): err = "sed: expression #1, char 2: extra characters after command" with mock.patch.object( - BaseTree, "remove", side_effect=RemoteCmdError("base", cmd, ret, err), + BaseFileSystem, + "remove", + side_effect=RemoteCmdError("base", cmd, ret, err), ): with pytest.raises(RemoteCmdError): - BaseTree(dvc, config).remove("file") + BaseFileSystem(dvc, config).remove("file") @mock.patch.object(CloudCache, "list_hashes_traverse") @mock.patch.object(CloudCache, "list_hashes_exists") def test_hashes_exist(object_exists, traverse, dvc): - cache = CloudCache(BaseTree(dvc, {})) + cache = CloudCache(BaseFileSystem(dvc, {})) # remote does not support traverse - cache.tree.CAN_TRAVERSE = False + cache.fs.CAN_TRAVERSE = False with mock.patch.object( cache, "list_hashes", return_value=list(range(256)) ): @@ -47,7 +49,7 @@ def test_hashes_exist(object_exists, traverse, dvc): object_exists.assert_called_with(hashes, None, None) traverse.assert_not_called() - cache.tree.CAN_TRAVERSE = True + cache.fs.CAN_TRAVERSE = True # large remote, small local object_exists.reset_mock() @@ -61,7 +63,7 @@ def test_hashes_exist(object_exists, traverse, dvc): # before returning all 256 remote hashes max_hashes = math.ceil( cache._max_estimation_size(hashes) - / pow(16, cache.tree.TRAVERSE_PREFIX_LEN) + / pow(16, cache.fs.TRAVERSE_PREFIX_LEN) ) assert max_hashes < 256 object_exists.assert_called_with( @@ -72,7 +74,7 @@ def test_hashes_exist(object_exists, traverse, dvc): # large remote, large local object_exists.reset_mock() traverse.reset_mock() - cache.tree.JOBS = 16 + cache.fs.JOBS = 16 with mock.patch.object( cache, "list_hashes", return_value=list(range(256)) ): @@ -80,7 +82,7 @@ def test_hashes_exist(object_exists, traverse, dvc): cache.hashes_exist(hashes) object_exists.assert_not_called() traverse.assert_called_with( - 256 * pow(16, cache.tree.TRAVERSE_PREFIX_LEN), + 256 * pow(16, cache.fs.TRAVERSE_PREFIX_LEN), set(range(256)), None, None, @@ -94,11 +96,11 @@ def test_hashes_exist(object_exists, traverse, dvc): CloudCache, "_path_to_hash", side_effect=lambda x: x, ) def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc): - cache = CloudCache(BaseTree(dvc, {})) - cache.tree.path_info = PathInfo("foo") + cache = CloudCache(BaseFileSystem(dvc, {})) + cache.fs.path_info = PathInfo("foo") # parallel traverse - size = 256 / cache.tree.JOBS * cache.tree.LIST_OBJECT_PAGE_SIZE + size = 256 / cache.fs.JOBS * cache.fs.LIST_OBJECT_PAGE_SIZE list(cache.list_hashes_traverse(size, {0})) for i in range(1, 16): list_hashes.assert_any_call( @@ -119,8 +121,8 @@ def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc): def test_list_hashes(dvc): - cache = CloudCache(BaseTree(dvc, {})) - cache.tree.path_info = PathInfo("foo") + cache = CloudCache(BaseFileSystem(dvc, {})) + cache.fs.path_info = PathInfo("foo") with mock.patch.object( cache, "_list_paths", return_value=["12/3456", "bar"] @@ -130,20 +132,20 @@ def test_list_hashes(dvc): def test_list_paths(dvc): - cache = CloudCache(BaseTree(dvc, {})) - cache.tree.path_info = PathInfo("foo") + cache = CloudCache(BaseFileSystem(dvc, {})) + cache.fs.path_info = PathInfo("foo") with mock.patch.object( - cache.tree, "walk_files", return_value=[] + cache.fs, "walk_files", return_value=[] ) as walk_mock: for _ in cache._list_paths(): pass - walk_mock.assert_called_with(cache.tree.path_info, prefix=False) + walk_mock.assert_called_with(cache.fs.path_info, prefix=False) for _ in cache._list_paths(prefix="000"): pass walk_mock.assert_called_with( - cache.tree.path_info / "00" / "0", prefix=True + cache.fs.path_info / "00" / "0", prefix=True ) @@ -152,4 +154,4 @@ def test_list_paths(dvc): [(None, False), ("", False), ("3456.dir", True), ("3456", False)], ) def test_is_dir_hash(hash_, result): - assert BaseTree.is_dir_hash(hash_) == result + assert BaseFileSystem.is_dir_hash(hash_) == result diff --git a/tests/unit/remote/test_gdrive.py b/tests/unit/remote/test_gdrive.py index e03c5eac5..b5c5a6933 100644 --- a/tests/unit/remote/test_gdrive.py +++ b/tests/unit/remote/test_gdrive.py @@ -1,6 +1,6 @@ import pytest -from dvc.tree.gdrive import GDriveAuthError, GDriveTree +from dvc.fs.gdrive import GDriveAuthError, GDriveFileSystem USER_CREDS_TOKEN_REFRESH_ERROR = '{"access_token": "", "client_id": "", "client_secret": "", "refresh_token": "", "token_expiry": "", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "", "expires_in": 3600, "scope": "https://www.googleapis.com/auth/drive.appdata https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive", "https://www.googleapis.com/auth/drive.appdata"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": true, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}' # noqa: E501 @@ -15,21 +15,23 @@ class TestRemoteGDrive: } def test_init(self, dvc): - tree = GDriveTree(dvc, self.CONFIG) - assert str(tree.path_info) == self.CONFIG["url"] + fs = GDriveFileSystem(dvc, self.CONFIG) + assert str(fs.path_info) == self.CONFIG["url"] def test_drive(self, dvc, monkeypatch): - tree = GDriveTree(dvc, self.CONFIG) + fs = GDriveFileSystem(dvc, self.CONFIG) monkeypatch.setenv( - GDriveTree.GDRIVE_CREDENTIALS_DATA, USER_CREDS_TOKEN_REFRESH_ERROR + GDriveFileSystem.GDRIVE_CREDENTIALS_DATA, + USER_CREDS_TOKEN_REFRESH_ERROR, ) with pytest.raises(GDriveAuthError): - assert tree._drive + assert fs._drive - monkeypatch.setenv(GDriveTree.GDRIVE_CREDENTIALS_DATA, "") - tree = GDriveTree(dvc, self.CONFIG) + monkeypatch.setenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA, "") + fs = GDriveFileSystem(dvc, self.CONFIG) monkeypatch.setenv( - GDriveTree.GDRIVE_CREDENTIALS_DATA, USER_CREDS_MISSED_KEY_ERROR + GDriveFileSystem.GDRIVE_CREDENTIALS_DATA, + USER_CREDS_MISSED_KEY_ERROR, ) with pytest.raises(GDriveAuthError): - assert tree._drive + assert fs._drive diff --git a/tests/unit/remote/test_gs.py b/tests/unit/remote/test_gs.py index da35d56b2..14fc8f1ff 100644 --- a/tests/unit/remote/test_gs.py +++ b/tests/unit/remote/test_gs.py @@ -2,7 +2,7 @@ import mock import pytest import requests -from dvc.tree.gs import GSTree, dynamic_chunk_size +from dvc.fs.gs import GSFileSystem, dynamic_chunk_size BUCKET = "bucket" PREFIX = "prefix" @@ -17,17 +17,17 @@ CONFIG = { def test_init(dvc): - tree = GSTree(dvc, CONFIG) - assert tree.path_info == URL - assert tree.projectname == PROJECT - assert tree.credentialpath == CREDENTIALPATH + fs = GSFileSystem(dvc, CONFIG) + assert fs.path_info == URL + assert fs.projectname == PROJECT + assert fs.credentialpath == CREDENTIALPATH @mock.patch("google.cloud.storage.Client.from_service_account_json") def test_gs(mock_client, dvc): - tree = GSTree(dvc, CONFIG) - assert tree.credentialpath - assert tree.gs + fs = GSFileSystem(dvc, CONFIG) + assert fs.credentialpath + assert fs.gs mock_client.assert_called_once_with(CREDENTIALPATH) @@ -35,8 +35,8 @@ def test_gs(mock_client, dvc): def test_gs_no_credspath(mock_client, dvc): config = CONFIG.copy() del config["credentialpath"] - tree = GSTree(dvc, config) - assert tree.gs + fs = GSFileSystem(dvc, config) + assert fs.gs mock_client.assert_called_with(PROJECT) diff --git a/tests/unit/remote/test_http.py b/tests/unit/remote/test_http.py index d25b4e95b..830c8e4e2 100644 --- a/tests/unit/remote/test_http.py +++ b/tests/unit/remote/test_http.py @@ -1,14 +1,14 @@ import pytest from dvc.exceptions import HTTPError -from dvc.tree.http import HTTPTree +from dvc.fs.http import HTTPFileSystem def test_download_fails_on_error_code(dvc, http): - tree = HTTPTree(dvc, http.config) + fs = HTTPFileSystem(dvc, http.config) with pytest.raises(HTTPError): - tree._download(http / "missing.txt", "missing.txt") + fs._download(http / "missing.txt", "missing.txt") def test_public_auth_method(dvc): @@ -19,9 +19,9 @@ def test_public_auth_method(dvc): "password": "", } - tree = HTTPTree(dvc, config) + fs = HTTPFileSystem(dvc, config) - assert tree._auth_method() is None + assert fs._auth_method() is None def test_basic_auth_method(dvc): @@ -38,10 +38,10 @@ def test_basic_auth_method(dvc): "password": password, } - tree = HTTPTree(dvc, config) + fs = HTTPFileSystem(dvc, config) - assert tree._auth_method() == auth - assert isinstance(tree._auth_method(), HTTPBasicAuth) + assert fs._auth_method() == auth + assert isinstance(fs._auth_method(), HTTPBasicAuth) def test_digest_auth_method(dvc): @@ -58,10 +58,10 @@ def test_digest_auth_method(dvc): "password": password, } - tree = HTTPTree(dvc, config) + fs = HTTPFileSystem(dvc, config) - assert tree._auth_method() == auth - assert isinstance(tree._auth_method(), HTTPDigestAuth) + assert fs._auth_method() == auth + assert isinstance(fs._auth_method(), HTTPDigestAuth) def test_custom_auth_method(dvc): @@ -75,11 +75,11 @@ def test_custom_auth_method(dvc): "password": password, } - tree = HTTPTree(dvc, config) + fs = HTTPFileSystem(dvc, config) - assert tree._auth_method() is None - assert header in tree.headers - assert tree.headers[header] == password + assert fs._auth_method() is None + assert header in fs.headers + assert fs.headers[header] == password def test_ssl_verify_is_enabled_by_default(dvc): @@ -88,9 +88,9 @@ def test_ssl_verify_is_enabled_by_default(dvc): "path_info": "file.html", } - tree = HTTPTree(dvc, config) + fs = HTTPFileSystem(dvc, config) - assert tree._session.verify is True + assert fs._session.verify is True def test_ssl_verify_disable(dvc): @@ -100,9 +100,9 @@ def test_ssl_verify_disable(dvc): "ssl_verify": False, } - tree = HTTPTree(dvc, config) + fs = HTTPFileSystem(dvc, config) - assert tree._session.verify is False + assert fs._session.verify is False def test_http_method(dvc): @@ -120,11 +120,11 @@ def test_http_method(dvc): "method": "PUT", } - tree = HTTPTree(dvc, config) + fs = HTTPFileSystem(dvc, config) - assert tree._auth_method() == auth - assert tree.method == "PUT" - assert isinstance(tree._auth_method(), HTTPBasicAuth) + assert fs._auth_method() == auth + assert fs.method == "PUT" + assert isinstance(fs._auth_method(), HTTPBasicAuth) def test_exists(mocker): @@ -139,17 +139,17 @@ def test_exists(mocker): # on HEAD request failure. res.raw = io.StringIO("foo") - tree = HTTPTree(None, {}) - mocker.patch.object(tree, "request", return_value=res) + fs = HTTPFileSystem(None, {}) + mocker.patch.object(fs, "request", return_value=res) url = URLInfo("https://example.org/file.txt") res.status_code = 200 - assert tree.exists(url) is True + assert fs.exists(url) is True res.status_code = 404 - assert tree.exists(url) is False + assert fs.exists(url) is False res.status_code = 403 with pytest.raises(HTTPError): - tree.exists(url) + fs.exists(url) diff --git a/tests/unit/remote/test_oss.py b/tests/unit/remote/test_oss.py index 0701b8355..e9fc9ba4c 100644 --- a/tests/unit/remote/test_oss.py +++ b/tests/unit/remote/test_oss.py @@ -1,4 +1,4 @@ -from dvc.tree.oss import OSSTree +from dvc.fs.oss import OSSFileSystem bucket_name = "bucket-name" endpoint = "endpoint" @@ -15,8 +15,8 @@ def test_init(dvc): "oss_key_secret": key_secret, "oss_endpoint": endpoint, } - tree = OSSTree(dvc, config) - assert tree.path_info == url - assert tree.endpoint == endpoint - assert tree.key_id == key_id - assert tree.key_secret == key_secret + fs = OSSFileSystem(dvc, config) + assert fs.path_info == url + assert fs.endpoint == endpoint + assert fs.key_id == key_id + assert fs.key_secret == key_secret diff --git a/tests/unit/remote/test_remote.py b/tests/unit/remote/test_remote.py index ff296dd60..d9b9f7272 100644 --- a/tests/unit/remote/test_remote.py +++ b/tests/unit/remote/test_remote.py @@ -1,8 +1,8 @@ import pytest -from dvc.tree import get_cloud_tree -from dvc.tree.gs import GSTree -from dvc.tree.s3 import S3Tree +from dvc.fs import get_cloud_fs +from dvc.fs.gs import GSFileSystem +from dvc.fs.s3 import S3FileSystem def test_remote_with_hash_jobs(dvc): @@ -12,8 +12,8 @@ def test_remote_with_hash_jobs(dvc): } dvc.config["core"]["checksum_jobs"] = 200 - tree = get_cloud_tree(dvc, name="with_hash_jobs") - assert tree.hash_jobs == 100 + fs = get_cloud_fs(dvc, name="with_hash_jobs") + assert fs.hash_jobs == 100 def test_remote_with_jobs(dvc): @@ -23,32 +23,32 @@ def test_remote_with_jobs(dvc): } dvc.config["core"]["jobs"] = 200 - tree = get_cloud_tree(dvc, name="with_jobs") - assert tree.jobs == 100 + fs = get_cloud_fs(dvc, name="with_jobs") + assert fs.jobs == 100 def test_remote_without_hash_jobs(dvc): dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"} dvc.config["core"]["checksum_jobs"] = 200 - tree = get_cloud_tree(dvc, name="without_hash_jobs") - assert tree.hash_jobs == 200 + fs = get_cloud_fs(dvc, name="without_hash_jobs") + assert fs.hash_jobs == 200 def test_remote_without_hash_jobs_default(dvc): dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"} - tree = get_cloud_tree(dvc, name="without_hash_jobs") - assert tree.hash_jobs == tree.HASH_JOBS + fs = get_cloud_fs(dvc, name="without_hash_jobs") + assert fs.hash_jobs == fs.HASH_JOBS -@pytest.mark.parametrize("tree_cls", [GSTree, S3Tree]) -def test_makedirs_not_create_for_top_level_path(tree_cls, dvc, mocker): - url = f"{tree_cls.scheme}://bucket/" - tree = tree_cls(dvc, {"url": url}) +@pytest.mark.parametrize("fs_cls", [GSFileSystem, S3FileSystem]) +def test_makedirs_not_create_for_top_level_path(fs_cls, dvc, mocker): + url = f"{fs_cls.scheme}://bucket/" + fs = fs_cls(dvc, {"url": url}) mocked_client = mocker.PropertyMock() # we use remote clients with same name as scheme to interact with remote - mocker.patch.object(tree_cls, tree.scheme, mocked_client) + mocker.patch.object(fs_cls, fs.scheme, mocked_client) - tree.makedirs(tree.path_info) + fs.makedirs(fs.path_info) assert not mocked_client.called diff --git a/tests/unit/remote/test_remote_tree.py b/tests/unit/remote/test_remote_tree.py index 319985ec1..5e66edb1a 100644 --- a/tests/unit/remote/test_remote_tree.py +++ b/tests/unit/remote/test_remote_tree.py @@ -2,9 +2,9 @@ import os import pytest +from dvc.fs.s3 import S3FileSystem from dvc.path_info import PathInfo from dvc.remote import get_remote -from dvc.tree.s3 import S3Tree from dvc.utils.fs import walk_files remotes = [pytest.lazy_fixture(fix) for fix in ["gs", "s3"]] @@ -46,7 +46,7 @@ def test_isdir(remote): ] for expected, path in test_cases: - assert remote.tree.isdir(remote.tree.path_info / path) == expected + assert remote.fs.isdir(remote.fs.path_info / path) == expected @pytest.mark.parametrize("remote", remotes, indirect=True) @@ -66,39 +66,37 @@ def test_exists(remote): ] for expected, path in test_cases: - assert remote.tree.exists(remote.tree.path_info / path) == expected + assert remote.fs.exists(remote.fs.path_info / path) == expected @pytest.mark.parametrize("remote", remotes, indirect=True) def test_walk_files(remote): files = [ - remote.tree.path_info / "data/alice", - remote.tree.path_info / "data/alpha", - remote.tree.path_info / "data/subdir-file.txt", - remote.tree.path_info / "data/subdir/1", - remote.tree.path_info / "data/subdir/2", - remote.tree.path_info / "data/subdir/3", - remote.tree.path_info / "data/subdir/empty_file", + remote.fs.path_info / "data/alice", + remote.fs.path_info / "data/alpha", + remote.fs.path_info / "data/subdir-file.txt", + remote.fs.path_info / "data/subdir/1", + remote.fs.path_info / "data/subdir/2", + remote.fs.path_info / "data/subdir/3", + remote.fs.path_info / "data/subdir/empty_file", ] - assert ( - list(remote.tree.walk_files(remote.tree.path_info / "data")) == files - ) + assert list(remote.fs.walk_files(remote.fs.path_info / "data")) == files @pytest.mark.parametrize("remote", [pytest.lazy_fixture("s3")], indirect=True) def test_copy_preserve_etag_across_buckets(remote, dvc): - s3 = remote.tree.s3 + s3 = remote.fs.s3 s3.Bucket("another").create() - another = S3Tree(dvc, {"url": "s3://another", "region": "us-east-1"}) + another = S3FileSystem(dvc, {"url": "s3://another", "region": "us-east-1"}) - from_info = remote.tree.path_info / "foo" + from_info = remote.fs.path_info / "foo" to_info = another.path_info / "foo" - remote.tree.copy(from_info, to_info) + remote.fs.copy(from_info, to_info) - from_hash = remote.tree.info(from_info)["etag"] + from_hash = remote.fs.info(from_info)["etag"] to_hash = another.info(to_info)["etag"] assert from_hash == to_hash @@ -106,13 +104,13 @@ def test_copy_preserve_etag_across_buckets(remote, dvc): @pytest.mark.parametrize("remote", remotes, indirect=True) def test_makedirs(remote): - tree = remote.tree - empty_dir = remote.tree.path_info / "empty_dir" / "" - tree.remove(empty_dir) - assert not tree.exists(empty_dir) - tree.makedirs(empty_dir) - assert tree.exists(empty_dir) - assert tree.isdir(empty_dir) + fs = remote.fs + empty_dir = remote.fs.path_info / "empty_dir" / "" + fs.remove(empty_dir) + assert not fs.exists(empty_dir) + fs.makedirs(empty_dir) + assert fs.exists(empty_dir) + assert fs.isdir(empty_dir) @pytest.mark.parametrize("remote", remotes, indirect=True) @@ -134,14 +132,14 @@ def test_isfile(remote): ] for expected, path in test_cases: - assert remote.tree.isfile(remote.tree.path_info / path) == expected + assert remote.fs.isfile(remote.fs.path_info / path) == expected @pytest.mark.parametrize("remote", remotes, indirect=True) def test_download_dir(remote, tmpdir): path = str(tmpdir / "data") to_info = PathInfo(path) - remote.tree.download(remote.tree.path_info / "data", to_info) + remote.fs.download(remote.fs.path_info / "data", to_info) assert os.path.isdir(path) data_dir = tmpdir / "data" assert len(list(walk_files(path))) == 7 diff --git a/tests/unit/remote/test_webdav.py b/tests/unit/remote/test_webdav.py index 907e812b4..6584ed224 100644 --- a/tests/unit/remote/test_webdav.py +++ b/tests/unit/remote/test_webdav.py @@ -1,6 +1,6 @@ import pytest -from dvc.tree.webdav import WebDAVTree +from dvc.fs.webdav import WebDAVFileSystem # Test configuration url = "webdavs://example.com/public.php/webdav" @@ -12,9 +12,9 @@ password = "password" # Test minimum requiered configuration (url) def test_init(dvc): config = {"url": url} - tree = WebDAVTree(dvc, config) + fs = WebDAVFileSystem(dvc, config) - assert tree.path_info == url + assert fs.path_info == url # Test username from configuration @@ -22,24 +22,24 @@ def test_init(dvc): "config", [{"url": url, "user": user}, {"url": userurl}] ) def test_user(dvc, config): - tree = WebDAVTree(dvc, config) + fs = WebDAVFileSystem(dvc, config) - assert tree.user == user + assert fs.user == user # Test username extraction from url def test_userurl(dvc): config = {"url": userurl} - tree = WebDAVTree(dvc, config) + fs = WebDAVFileSystem(dvc, config) - assert tree.path_info == userurl - assert tree.user == user - assert tree.path_info.user == user + assert fs.path_info == userurl + assert fs.user == user + assert fs.path_info.user == user # test password from config def test_password(dvc): config = {"url": url, "user": user, "password": password} - tree = WebDAVTree(dvc, config) + fs = WebDAVFileSystem(dvc, config) - assert tree.password == password + assert fs.password == password diff --git a/tests/unit/remote/test_webhdfs.py b/tests/unit/remote/test_webhdfs.py index e25431225..3b9c2b7ae 100644 --- a/tests/unit/remote/test_webhdfs.py +++ b/tests/unit/remote/test_webhdfs.py @@ -1,4 +1,4 @@ -from dvc.tree.webhdfs import WebHDFSTree +from dvc.fs.webhdfs import WebHDFSFileSystem user = "test" webhdfs_token = "token" @@ -16,9 +16,9 @@ def test_init(dvc): "user": user, } - tree = WebHDFSTree(dvc, config) - assert tree.path_info == url - assert tree.token == webhdfs_token - assert tree.alias == webhdfs_alias - assert tree.path_info.user == user - assert tree.hdfscli_config == hdfscli_config + fs = WebHDFSFileSystem(dvc, config) + assert fs.path_info == url + assert fs.token == webhdfs_token + assert fs.alias == webhdfs_alias + assert fs.path_info.user == user + assert fs.hdfscli_config == hdfscli_config diff --git a/tests/unit/scm/test_git.py b/tests/unit/scm/test_git.py index e7491a698..8c34b2342 100644 --- a/tests/unit/scm/test_git.py +++ b/tests/unit/scm/test_git.py @@ -42,8 +42,8 @@ def test_walk_with_submodules(tmp_dir, scm, git_dir): files = [] dirs = [] - tree = scm.get_tree("HEAD") - for _, dnames, fnames in tree.walk("."): + fs = scm.get_fs("HEAD") + for _, dnames, fnames in fs.walk("."): dirs.extend(dnames) files.extend(fnames) @@ -59,20 +59,20 @@ def test_walk_onerror(tmp_dir, scm): tmp_dir.scm_gen( {"foo": "foo"}, commit="init", ) - tree = scm.get_tree("HEAD") + fs = scm.get_fs("HEAD") # path does not exist - for _ in tree.walk("dir"): + for _ in fs.walk("dir"): pass with pytest.raises(OSError): - for _ in tree.walk("dir", onerror=onerror): + for _ in fs.walk("dir", onerror=onerror): pass # path is not a directory - for _ in tree.walk("foo"): + for _ in fs.walk("foo"): pass with pytest.raises(OSError): - for _ in tree.walk("foo", onerror=onerror): + for _ in fs.walk("foo", onerror=onerror): pass diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 8ddba3546..4e8164f4b 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -3,7 +3,7 @@ import os import pytest from dvc.config import Config -from dvc.tree.local import LocalTree +from dvc.fs.local import LocalFileSystem @pytest.mark.parametrize( @@ -19,17 +19,17 @@ def test_to_relpath(path, expected): assert Config._to_relpath(os.path.join(".", "config"), path) == expected -def test_get_tree(tmp_dir, scm): +def test_get_fs(tmp_dir, scm): tmp_dir.scm_gen("foo", "foo", commit="add foo") - tree = scm.get_tree("master") - config = Config(tree=tree) + fs = scm.get_fs("master") + config = Config(fs=fs) - assert config.tree == tree - assert config.wtree != tree - assert isinstance(config.wtree, LocalTree) + assert config.fs == fs + assert config.wfs != fs + assert isinstance(config.wfs, LocalFileSystem) - assert config._get_tree("repo") == tree - assert config._get_tree("local") == config.wtree - assert config._get_tree("global") == config.wtree - assert config._get_tree("system") == config.wtree + assert config._get_fs("repo") == fs + assert config._get_fs("local") == config.wfs + assert config._get_fs("global") == config.wfs + assert config._get_fs("system") == config.wfs diff --git a/tests/unit/test_context.py b/tests/unit/test_context.py index eae4dc056..8b6145387 100644 --- a/tests/unit/test_context.py +++ b/tests/unit/test_context.py @@ -4,6 +4,7 @@ from unittest.mock import mock_open import pytest +from dvc.fs.local import LocalFileSystem from dvc.parsing import DEFAULT_PARAMS_FILE from dvc.parsing.context import ( Context, @@ -15,7 +16,6 @@ from dvc.parsing.context import ( Value, recurse_not_a_node, ) -from dvc.tree.local import LocalTree from dvc.utils import relpath from dvc.utils.serialize import dump_yaml, dumps_yaml @@ -218,12 +218,12 @@ def test_overwrite_with_setitem(): def test_load_from(mocker): d = {"x": {"y": {"z": 5}, "lst": [1, 2, 3]}, "foo": "foo"} - tree = mocker.Mock( + fs = mocker.Mock( open=mock_open(read_data=dumps_yaml(d)), **{"exists.return_value": True, "isdir.return_value": False}, ) file = "params.yaml" - c = Context.load_from(tree, file) + c = Context.load_from(fs, file) assert asdict(c["x"].meta) == { "source": file, @@ -288,11 +288,11 @@ def test_track(tmp_dir): ], "dct": {"foo": "foo", "bar": "bar", "baz": "baz"}, } - tree = LocalTree(None, config={}) + fs = LocalFileSystem(None, config={}) path = tmp_dir / "params.yaml" - dump_yaml(path, d, tree) + dump_yaml(path, d, fs) - context = Context.load_from(tree, path) + context = Context.load_from(fs, path) def key_tracked(d, key): assert len(d) == 1 @@ -323,14 +323,14 @@ def test_track_from_multiple_files(tmp_dir): d1 = {"Train": {"us": {"lr": 10}}} d2 = {"Train": {"us": {"layers": 100}}} - tree = LocalTree(None, config={}) + fs = LocalFileSystem(None, config={}) path1 = tmp_dir / "params.yaml" path2 = tmp_dir / "params2.yaml" - dump_yaml(path1, d1, tree) - dump_yaml(path2, d2, tree) + dump_yaml(path1, d1, fs) + dump_yaml(path2, d2, fs) - context = Context.load_from(tree, path1) - c = Context.load_from(tree, path2) + context = Context.load_from(fs, path1) + c = Context.load_from(fs, path2) context.merge_update(c) def key_tracked(d, path, key): @@ -429,7 +429,7 @@ def test_resolve_resolves_boolean_value(): def test_load_from_raises_if_file_not_exist(tmp_dir, dvc): with pytest.raises(ParamsLoadError) as exc_info: - Context.load_from(dvc.tree, tmp_dir / DEFAULT_PARAMS_FILE) + Context.load_from(dvc.fs, tmp_dir / DEFAULT_PARAMS_FILE) assert str(exc_info.value) == "'params.yaml' does not exist" @@ -439,6 +439,6 @@ def test_load_from_raises_if_file_is_directory(tmp_dir, dvc): data_dir.mkdir() with pytest.raises(ParamsLoadError) as exc_info: - Context.load_from(dvc.tree, data_dir) + Context.load_from(dvc.fs, data_dir) assert str(exc_info.value) == "'data' is a directory" diff --git a/tests/unit/test_external_repo.py b/tests/unit/test_external_repo.py index b8be97161..7b14f457f 100644 --- a/tests/unit/test_external_repo.py +++ b/tests/unit/test_external_repo.py @@ -6,7 +6,7 @@ import pytest from dvc.external_repo import external_repo from dvc.objects import save, stage from dvc.path_info import PathInfo -from tests.unit.tree.test_repo import make_subrepo +from tests.unit.fs.test_repo import make_subrepo def test_hook_is_called(tmp_dir, erepo_dir, mocker): @@ -28,9 +28,9 @@ def test_hook_is_called(tmp_dir, erepo_dir, mocker): repo.dvc_gen("bar", "bar", commit=f"dvc add {repo}/bar") with external_repo(str(erepo_dir)) as repo: - spy = mocker.spy(repo.repo_tree, "repo_factory") + spy = mocker.spy(repo.repo_fs, "repo_factory") - list(repo.repo_tree.walk(repo.root_dir)) # drain + list(repo.repo_fs.walk(repo.root_dir)) # drain assert spy.call_count == len(subrepos) paths = [os.path.join(repo.root_dir, path) for path in subrepo_paths] @@ -40,7 +40,7 @@ def test_hook_is_called(tmp_dir, erepo_dir, mocker): path, scm=repo.scm, rev=repo.get_rev(), - repo_factory=repo.repo_tree.repo_factory, + repo_factory=repo.repo_fs.repo_factory, ) for path in paths ], @@ -66,9 +66,9 @@ def test_subrepo_is_constructed_properly( with external_repo( str(tmp_dir), cache_dir=str(cache_dir), cache_types=["symlink"] ) as repo: - spy = mocker.spy(repo.repo_tree, "repo_factory") + spy = mocker.spy(repo.repo_fs, "repo_factory") - list(repo.repo_tree.walk(repo.root_dir)) # drain + list(repo.repo_fs.walk(repo.root_dir)) # drain assert spy.call_count == 1 subrepo = spy.return_value @@ -113,7 +113,7 @@ def test_fetch_external_repo_jobs(tmp_dir, scm, mocker, dvc, local_remote): obj = stage( dvc.cache.local, PathInfo(repo.root_dir) / "dir1", - repo.repo_tree, + repo.repo_fs, follow_subrepos=False, jobs=3, ) diff --git a/tests/unit/test_ignore.py b/tests/unit/test_ignore.py index 3bc0344e7..27c3af9b4 100644 --- a/tests/unit/test_ignore.py +++ b/tests/unit/test_ignore.py @@ -7,9 +7,9 @@ from dvc.ignore import DvcIgnorePatterns def mock_dvcignore(dvcignore_path, patterns): - tree = MagicMock() - with patch.object(tree, "open", mock_open(read_data="\n".join(patterns))): - ignore_patterns = DvcIgnorePatterns.from_files(dvcignore_path, tree) + fs = MagicMock() + with patch.object(fs, "open", mock_open(read_data="\n".join(patterns))): + ignore_patterns = DvcIgnorePatterns.from_files(dvcignore_path, fs) return ignore_patterns diff --git a/tests/unit/utils/test_fs.py b/tests/unit/utils/test_fs.py index 2db42cc09..ba0a53bc1 100644 --- a/tests/unit/utils/test_fs.py +++ b/tests/unit/utils/test_fs.py @@ -6,9 +6,9 @@ import pytest from mock import patch import dvc +from dvc.fs.local import LocalFileSystem from dvc.path_info import PathInfo from dvc.system import System -from dvc.tree.local import LocalTree from dvc.utils import relpath from dvc.utils.fs import ( BasePathNotInCheckedPathException, @@ -28,9 +28,9 @@ from tests.basic_env import TestDir class TestMtimeAndSize(TestDir): def test(self): - tree = LocalTree(None, {"url": self.root_dir}, use_dvcignore=True) - file_time, file_size = get_mtime_and_size(self.DATA, tree) - dir_time, dir_size = get_mtime_and_size(self.DATA_DIR, tree) + fs = LocalFileSystem(None, {"url": self.root_dir}, use_dvcignore=True) + file_time, file_size = get_mtime_and_size(self.DATA, fs) + dir_time, dir_size = get_mtime_and_size(self.DATA_DIR, fs) actual_file_size = os.path.getsize(self.DATA) actual_dir_size = os.path.getsize(self.DATA) + os.path.getsize( @@ -129,15 +129,15 @@ def test_path_object_and_str_are_valid_types_get_mtime_and_size(tmp_dir): tmp_dir.gen( {"dir": {"dir_file": "dir file content"}, "file": "file_content"} ) - tree = LocalTree(None, {"url": os.fspath(tmp_dir)}, use_dvcignore=True) + fs = LocalFileSystem(None, {"url": os.fspath(tmp_dir)}, use_dvcignore=True) - time, size = get_mtime_and_size("dir", tree) - object_time, object_size = get_mtime_and_size(PathInfo("dir"), tree) + time, size = get_mtime_and_size("dir", fs) + object_time, object_size = get_mtime_and_size(PathInfo("dir"), fs) assert time == object_time assert size == object_size - time, size = get_mtime_and_size("file", tree) - object_time, object_size = get_mtime_and_size(PathInfo("file"), tree) + time, size = get_mtime_and_size("file", fs) + object_time, object_size = get_mtime_and_size(PathInfo("file"), fs) assert time == object_time assert size == object_size diff --git a/tests/unit/utils/test_stream.py b/tests/unit/utils/test_stream.py index cd37152c8..d8c608574 100644 --- a/tests/unit/utils/test_stream.py +++ b/tests/unit/utils/test_stream.py @@ -1,4 +1,4 @@ -from dvc.tree.local import LocalTree +from dvc.fs.local import LocalFileSystem from dvc.utils import file_md5 from dvc.utils.stream import HashedStreamReader @@ -11,7 +11,7 @@ def test_hashed_stream_reader(tmp_dir): stream_reader = HashedStreamReader(fobj) assert stream_reader.read(3) == b"foo" - hex_digest = file_md5(foo, LocalTree(None, {})) + hex_digest = file_md5(foo, LocalFileSystem(None, {})) assert stream_reader.is_text_file assert hex_digest == stream_reader.hash_info.value @@ -27,6 +27,6 @@ def test_hashed_stream_reader_as_chunks(tmp_dir): if not chunk: break - hex_digest = file_md5(foo, LocalTree(None, {})) + hex_digest = file_md5(foo, LocalFileSystem(None, {})) assert not stream_reader.is_text_file assert hex_digest == stream_reader.hash_info.value diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index 11b2d8d71..558b6f503 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -3,8 +3,8 @@ import re import pytest +from dvc.fs.local import LocalFileSystem from dvc.path_info import PathInfo -from dvc.tree.local import LocalTree from dvc.utils import ( dict_sha256, file_md5, @@ -87,8 +87,8 @@ def test_fix_env_pyenv(path, orig): def test_file_md5(tmp_dir): tmp_dir.gen("foo", "foo content") - tree = LocalTree(None, {}) - assert file_md5("foo", tree) == file_md5(PathInfo("foo"), tree) + fs = LocalFileSystem(None, {}) + assert file_md5("foo", fs) == file_md5(PathInfo("foo"), fs) def test_tmp_fname():