mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
pull: skip isdir() checks on downloading files (#5696)
* pull: use download_file * tree: implement filesystem.download_file
This commit is contained in:
+5
-2
@@ -1,6 +1,6 @@
|
|||||||
import logging
|
import logging
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from functools import partial
|
from functools import partial, partialmethod
|
||||||
from multiprocessing import cpu_count
|
from multiprocessing import cpu_count
|
||||||
from typing import Any, ClassVar, Dict, FrozenSet, Optional
|
from typing import Any, ClassVar, Dict, FrozenSet, Optional
|
||||||
|
|
||||||
@@ -258,6 +258,7 @@ class BaseFileSystem:
|
|||||||
name=None,
|
name=None,
|
||||||
no_progress_bar=False,
|
no_progress_bar=False,
|
||||||
jobs=None,
|
jobs=None,
|
||||||
|
_only_file=False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
if not hasattr(self, "_download"):
|
if not hasattr(self, "_download"):
|
||||||
@@ -273,12 +274,14 @@ class BaseFileSystem:
|
|||||||
if to_info.scheme != "local":
|
if to_info.scheme != "local":
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
if self.isdir(from_info):
|
if not _only_file and self.isdir(from_info):
|
||||||
return self._download_dir(
|
return self._download_dir(
|
||||||
from_info, to_info, name, no_progress_bar, jobs, **kwargs,
|
from_info, to_info, name, no_progress_bar, jobs, **kwargs,
|
||||||
)
|
)
|
||||||
return self._download_file(from_info, to_info, name, no_progress_bar,)
|
return self._download_file(from_info, to_info, name, no_progress_bar,)
|
||||||
|
|
||||||
|
download_file = partialmethod(download, _only_file=True)
|
||||||
|
|
||||||
def _download_dir(
|
def _download_dir(
|
||||||
self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
|
self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
|
||||||
):
|
):
|
||||||
|
|||||||
+1
-1
@@ -309,7 +309,7 @@ class Remote:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if download:
|
if download:
|
||||||
func = _log_exceptions(self.fs.download, "download")
|
func = _log_exceptions(self.fs.download_file, "download")
|
||||||
status = STATUS_DELETED
|
status = STATUS_DELETED
|
||||||
desc = "Downloading"
|
desc = "Downloading"
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ def test_cache_reused(erepo_dir, mocker, local_cloud):
|
|||||||
erepo_dir.dvc_gen("file", "text", commit="add file")
|
erepo_dir.dvc_gen("file", "text", commit="add file")
|
||||||
erepo_dir.dvc.push()
|
erepo_dir.dvc.push()
|
||||||
|
|
||||||
download_spy = mocker.spy(LocalFileSystem, "download")
|
download_spy = mocker.spy(LocalFileSystem, "download_file")
|
||||||
|
|
||||||
# Use URL to prevent any fishy optimizations
|
# Use URL to prevent any fishy optimizations
|
||||||
url = f"file://{erepo_dir}"
|
url = f"file://{erepo_dir}"
|
||||||
|
|||||||
Reference in New Issue
Block a user