mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
dvcfs: implement get_file
Currently we use `open`, which is much slower. For example, ``` $ dvc get dvc-bench data/small/dataset ``` goes down from 110sec to 50sec. Discovered as a part of https://github.com/iterative/dvc-bench/issues/419
This commit is contained in:
@@ -361,6 +361,19 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
info["name"] = path
|
||||
return info
|
||||
|
||||
def get_file(self, rpath, lpath, **kwargs): # pylint: disable=arguments-differ
|
||||
key = self._get_key_from_relative(rpath)
|
||||
fs_path = self._from_key(key)
|
||||
try:
|
||||
return self.repo.fs.get_file(fs_path, lpath, **kwargs)
|
||||
except FileNotFoundError:
|
||||
_, dvc_fs, subkey = self._get_subrepo_info(key)
|
||||
if not dvc_fs:
|
||||
raise
|
||||
|
||||
dvc_path = _get_dvc_path(dvc_fs, subkey)
|
||||
return dvc_fs.get_file(dvc_path, lpath, **kwargs)
|
||||
|
||||
|
||||
class DVCFileSystem(FileSystem):
|
||||
protocol = "local"
|
||||
|
||||
+2
-2
@@ -35,7 +35,7 @@ dependencies = [
|
||||
"configobj>=5.0.6",
|
||||
"distro>=1.3",
|
||||
"dpath<3,>=2.0.2",
|
||||
"dvc-data>=0.40.4,<0.41",
|
||||
"dvc-data>=0.41.0,<0.42",
|
||||
"dvc-http",
|
||||
"dvc-render>=0.1.2",
|
||||
"dvc-studio-client>=0.1.1",
|
||||
@@ -57,7 +57,7 @@ dependencies = [
|
||||
"requests>=2.22",
|
||||
"rich>=12",
|
||||
"ruamel.yaml>=0.17.11",
|
||||
"scmrepo>=0.1.11,<1",
|
||||
"scmrepo>=0.1.12,<1",
|
||||
"shortuuid>=0.5",
|
||||
"shtab<2,>=1.3.4",
|
||||
"tabulate>=0.8.7",
|
||||
|
||||
Reference in New Issue
Block a user