objects: fix gdrive CI issue (#6338)

* fs: ensure gdrivefs.info returns integer size

* transfer: only verify on pull
This commit is contained in:
Peter Rowlands (변기호)
2021-07-20 18:54:49 +03:00
committed by GitHub
parent d95bd62f8e
commit 12d1e3e981
3 changed files with 15 additions and 5 deletions
+1
View File
@@ -118,6 +118,7 @@ class DataCloud:
jobs=jobs,
src_index=get_index(odb),
cache_odb=self.repo.odb.local,
verify=odb.verify,
)
def status(
+1 -1
View File
@@ -619,7 +619,7 @@ class GDriveFileSystem(BaseFileSystem): # pylint:disable=abstract-method
item_id = self._get_item_id(path_info)
gdrive_file = self._drive.CreateFile({"id": item_id})
gdrive_file.FetchMetadata(fields="fileSize")
return {"size": gdrive_file.get("fileSize"), "type": "file"}
return {"size": int(gdrive_file.get("fileSize")), "type": "file"}
def _upload_fobj(self, fobj, to_info, **kwargs):
dirname = to_info.parent
+13 -4
View File
@@ -41,7 +41,7 @@ def _log_exceptions(func):
return wrapper
def _transfer(src, dest, dir_objs, file_objs, missing, jobs, **kwargs):
def _transfer(src, dest, dir_objs, file_objs, missing, jobs, verify, **kwargs):
from . import save
from .stage import is_memfs_staging
@@ -54,7 +54,14 @@ def _transfer(src, dest, dir_objs, file_objs, missing, jobs, **kwargs):
func = pbar.wrap_fn(func)
with ThreadPoolExecutor(max_workers=jobs) as executor:
processor = partial(
_create_tasks, executor, jobs, func, src, dest, is_staged
_create_tasks,
executor,
jobs,
func,
src,
dest,
is_staged,
verify,
)
processor.save_func = func
_do_transfer(
@@ -69,7 +76,7 @@ def _transfer(src, dest, dir_objs, file_objs, missing, jobs, **kwargs):
return total
def _create_tasks(executor, jobs, func, src, dest, is_staged, objs):
def _create_tasks(executor, jobs, func, src, dest, is_staged, verify, objs):
fails = 0
obj_iter = iter(objs)
@@ -80,7 +87,7 @@ def _create_tasks(executor, jobs, func, src, dest, is_staged, objs):
dest,
_raw_obj(src, obj, is_staged),
move=False,
verify=src.verify,
verify=verify,
)
for obj in itertools.islice(obj_iter, amount)
}
@@ -180,6 +187,7 @@ def transfer(
dest: "ObjectDB",
objs: Iterable["HashFile"],
jobs: Optional[int] = None,
verify: bool = False,
**kwargs,
) -> int:
"""Transfer (copy) the specified objects from one ODB to another.
@@ -214,5 +222,6 @@ def transfer(
files,
status.missing,
jobs,
verify,
**kwargs,
)