Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-11 23:10:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

staging: optimize the number of info() calls (#6281)

Browse Source 
* staging: optimize the number of info() calls

* fs: make info() give details about the object
This commit is contained in:
Batuhan Taskaya
2021-07-07 22:53:43 +03:00
committed by GitHub
parent f90b9cdee1
commit 87b49fd9f9
10 changed files with 39 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/fs/dvc.py
+1 -1
View File
@@ -233,7 +233,7 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
def info(self, path_info):
meta = self.metadata(path_info)
ret = {"type": "dir" if meta.isdir else "file"}
ret = {"type": "directory" if meta.isdir else "file"}
if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info:
hash_info = meta.outs[0].hash_info
ret["size"] = hash_info.size
dvc/fs/gdrive.py
+7 -4
View File
@@ -1,3 +1,4 @@
import errno
import io
import logging
import os
@@ -11,7 +12,7 @@ from urllib.parse import urlparse
from funcy import cached_property, retry, wrap_prop, wrap_with
from funcy.py3 import cat
from dvc.exceptions import DvcException, FileMissingError
from dvc.exceptions import DvcException
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
@@ -521,12 +522,14 @@ class GDriveFileSystem(BaseFileSystem): # pylint:disable=abstract-method
return min(item_ids)
assert not create
raise FileMissingError(path_info, hint)
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), hint or path_info
)
def exists(self, path_info) -> bool:
try:
self._get_item_id(path_info)
except FileMissingError:
except FileNotFoundError:
return False
else:
return True
@@ -616,7 +619,7 @@ class GDriveFileSystem(BaseFileSystem): # pylint:disable=abstract-method
item_id = self._get_item_id(path_info)
gdrive_file = self._drive.CreateFile({"id": item_id})
gdrive_file.FetchMetadata(fields="fileSize")
return {"size": gdrive_file.get("fileSize")}
return {"size": gdrive_file.get("fileSize"), "type": "file"}
def _upload_fobj(self, fobj, to_info, **kwargs):
dirname = to_info.parent
dvc/fs/git.py
+5 -1
View File
@@ -1,5 +1,6 @@
import errno
import os
import stat
from dvc.utils import is_exec, relpath
@@ -106,7 +107,10 @@ class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), path_info
)
return {"size": st.st_size}
return {
"size": st.st_size,
"type": "directory" if stat.S_ISDIR(st.st_mode) else "file",
}
def stat(self, path):
key = self._get_key(path)
dvc/fs/hdfs.py
+14 -2
View File
@@ -1,3 +1,4 @@
import errno
import io
import logging
import os
@@ -220,8 +221,19 @@ class HDFSFileSystem(BaseFileSystem):
def info(self, path_info):
with self.hdfs(path_info) as hdfs:
finfo = hdfs.get_file_info(path_info.path)
return {"size": finfo.size}
from pyarrow.fs import FileType
file_info = hdfs.get_file_info(path_info.path)
if file_info.type is FileType.Directory:
kind = "directory"
elif file_info.type is FileType.File:
kind = "file"
else:
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), path_info
)
return {"size": file_info.size, "type": kind}
def checksum(self, path_info):
return HashInfo(
dvc/fs/http.py
+1 -1
View File
@@ -146,7 +146,7 @@ class HTTPFileSystem(BaseFileSystem): # pylint:disable=abstract-method
resp = self._head(path_info.url)
etag = resp.headers.get("ETag") or resp.headers.get("Content-MD5")
size = self._content_length(resp)
return {"etag": etag, "size": size}
return {"etag": etag, "size": size, "type": "file"}
def _upload_fobj(self, fobj, to_info, **kwargs):
def chunks(fobj):
dvc/fs/local.py
+1 -1
View File
@@ -157,7 +157,7 @@ class LocalFileSystem(BaseFileSystem):
st = os.stat(path_info)
return {
"size": st.st_size,
"type": "dir" if stat.S_ISDIR(st.st_mode) else "file",
"type": "directory" if stat.S_ISDIR(st.st_mode) else "file",
}
def _upload(
dvc/fs/ssh/connection.py
+1 -1
View File
@@ -82,7 +82,7 @@ class SSHConnection:
st = self.sftp.stat(path)
return {
"size": st.st_size,
"type": "dir" if stat.S_ISDIR(st.st_mode) else "file",
"type": "directory" if stat.S_ISDIR(st.st_mode) else "file",
}
def getsize(self, path):
dvc/fs/webhdfs.py
+1 -1
View File
@@ -123,7 +123,7 @@ class WebHDFSFileSystem(BaseFileSystem): # pylint:disable=abstract-method
def info(self, path_info):
st = self.hdfs_client.status(path_info.path)
return {"size": st["length"]}
return {"size": st["length"], "type": "file"}
def checksum(self, path_info):
return HashInfo(
dvc/objects/stage.py
+3 -9
View File
@@ -1,5 +1,3 @@
import errno
import os
from concurrent.futures import ThreadPoolExecutor
from functools import partial
@@ -174,11 +172,7 @@ def stage(odb, path_info, fs, name, upload=False, **kwargs):
isinstance(path_info, str) or path_info.scheme == fs.scheme
)
if not fs.exists(path_info):
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), path_info
)
details = fs.info(path_info)
state = odb.state
# pylint: disable=assignment-from-none
hash_info = state.get(path_info, fs)
@@ -210,12 +204,12 @@ def stage(odb, path_info, fs, name, upload=False, **kwargs):
obj.hash_info.size = hash_info.size
return obj
if fs.isdir(path_info):
if details["type"] == "directory":
obj = _get_tree_obj(path_info, fs, name, odb, state, upload, **kwargs)
else:
_, obj = _get_file_obj(path_info, fs, name, odb, state, upload)
if obj.hash_info and fs.exists(path_info):
if obj.hash_info:
state.save(path_info, fs, obj.hash_info)
return obj
tests/unit/remote/test_http.py
+5 -1
View File
@@ -175,5 +175,9 @@ def test_content_length(mocker, headers, expected_size):
url = URLInfo("https://example.org/file.txt")
assert fs.info(url) == {"etag": None, "size": expected_size}
assert fs.info(url) == {
"etag": None,
"size": expected_size,
"type": "file",
}
assert fs._content_length(res) == expected_size