From 46fac33ba575e59e69bd993c289846854521c150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Wed, 27 Apr 2022 18:56:40 +0545 Subject: [PATCH] fs: remove unused CHUNK_SIZE var There is no support for chunk_size kwarg in fs.open. Also, `upload_fobj` is now able to figure out proper chunk size from fileobj, so I don't think we need this on _upload_file. --- dvc/data/stage.py | 2 +- dvc/fs/base.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/dvc/data/stage.py b/dvc/data/stage.py index 8a9dbaefb..04e2d1ad1 100644 --- a/dvc/data/stage.py +++ b/dvc/data/stage.py @@ -32,7 +32,7 @@ def _upload_file(from_fs_path, fs, odb, upload_odb, callback=None): fs_path = upload_odb.fs.path tmp_info = fs_path.join(upload_odb.fs_path, tmp_fname()) - with fs.open(from_fs_path, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream: + with fs.open(from_fs_path, mode="rb") as stream: stream = HashedStreamReader(stream) size = fs.size(from_fs_path) with FsspecCallback.as_tqdm_callback( diff --git a/dvc/fs/base.py b/dvc/fs/base.py index cd7102acb..aad6d0b90 100644 --- a/dvc/fs/base.py +++ b/dvc/fs/base.py @@ -64,9 +64,6 @@ class FileSystem: TRAVERSE_THRESHOLD_SIZE = 500000 CAN_TRAVERSE = True - # Needed for some providers, and http open() - CHUNK_SIZE = 64 * 1024 * 1024 # 64 MiB - PARAM_CHECKSUM: ClassVar[Optional[str]] = None def __init__(self, **kwargs):