pull: skip isdir() checks on downloading files (#5696)

* pull: use download_file

* tree: implement filesystem.download_file
This commit is contained in:
Batuhan Taskaya
2021-03-26 15:22:45 +02:00
committed by GitHub
parent 61e470b85e
commit 042b1c7f7d
3 changed files with 7 additions and 4 deletions
+5 -2
View File
@@ -1,6 +1,6 @@
import logging
from concurrent.futures import ThreadPoolExecutor, as_completed
from functools import partial
from functools import partial, partialmethod
from multiprocessing import cpu_count
from typing import Any, ClassVar, Dict, FrozenSet, Optional
@@ -258,6 +258,7 @@ class BaseFileSystem:
name=None,
no_progress_bar=False,
jobs=None,
_only_file=False,
**kwargs,
):
if not hasattr(self, "_download"):
@@ -273,12 +274,14 @@ class BaseFileSystem:
if to_info.scheme != "local":
raise NotImplementedError
if self.isdir(from_info):
if not _only_file and self.isdir(from_info):
return self._download_dir(
from_info, to_info, name, no_progress_bar, jobs, **kwargs,
)
return self._download_file(from_info, to_info, name, no_progress_bar,)
download_file = partialmethod(download, _only_file=True)
def _download_dir(
self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
):
+1 -1
View File
@@ -309,7 +309,7 @@ class Remote:
)
if download:
func = _log_exceptions(self.fs.download, "download")
func = _log_exceptions(self.fs.download_file, "download")
status = STATUS_DELETED
desc = "Downloading"
else:
+1 -1
View File
@@ -52,7 +52,7 @@ def test_cache_reused(erepo_dir, mocker, local_cloud):
erepo_dir.dvc_gen("file", "text", commit="add file")
erepo_dir.dvc.push()
download_spy = mocker.spy(LocalFileSystem, "download")
download_spy = mocker.spy(LocalFileSystem, "download_file")
# Use URL to prevent any fishy optimizations
url = f"file://{erepo_dir}"