dvcfs: implement get_file

Currently we use `open`, which is much slower. For example,

```
$ dvc get dvc-bench data/small/dataset
```

goes down from 110sec to 50sec.

Discovered as a part of https://github.com/iterative/dvc-bench/issues/419
This commit is contained in:
Ruslan Kuprieiev
2023-02-20 11:10:13 +02:00
parent 19038b32bd
commit ea6e01d0e0
2 changed files with 15 additions and 2 deletions
+13
View File
@@ -361,6 +361,19 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
info["name"] = path
return info
def get_file(self, rpath, lpath, **kwargs): # pylint: disable=arguments-differ
key = self._get_key_from_relative(rpath)
fs_path = self._from_key(key)
try:
return self.repo.fs.get_file(fs_path, lpath, **kwargs)
except FileNotFoundError:
_, dvc_fs, subkey = self._get_subrepo_info(key)
if not dvc_fs:
raise
dvc_path = _get_dvc_path(dvc_fs, subkey)
return dvc_fs.get_file(dvc_path, lpath, **kwargs)
class DVCFileSystem(FileSystem):
protocol = "local"
+2 -2
View File
@@ -35,7 +35,7 @@ dependencies = [
"configobj>=5.0.6",
"distro>=1.3",
"dpath<3,>=2.0.2",
"dvc-data>=0.40.4,<0.41",
"dvc-data>=0.41.0,<0.42",
"dvc-http",
"dvc-render>=0.1.2",
"dvc-studio-client>=0.1.1",
@@ -57,7 +57,7 @@ dependencies = [
"requests>=2.22",
"rich>=12",
"ruamel.yaml>=0.17.11",
"scmrepo>=0.1.11,<1",
"scmrepo>=0.1.12,<1",
"shortuuid>=0.5",
"shtab<2,>=1.3.4",
"tabulate>=0.8.7",