output: optimize obj collection (#6277)

* output: str only once

Calling `__str__` takes a long time and if we are dealing with a large
dataset, it will result in a very substantial time. For example,
locally, for 700K objects, it took 20sec more.

Related to #6276

* output: don't use os.path.join

Takes around 2sec for 700K objects.

* output: use fs.sep

* output: remove unused sep
This commit is contained in:
Ruslan Kuprieiev
2021-07-03 21:53:55 +03:00
committed by GitHub
parent cfcb7e1294
commit aa255a1102
6 changed files with 11 additions and 3 deletions
+2
View File
@@ -33,6 +33,8 @@ class RemoteMissingDepsError(DvcException):
class BaseFileSystem:
sep = "/"
scheme = "base"
REQUIRES: ClassVar[Dict[str, str]] = {}
PATH_CLS = URLInfo # type: Any
+2
View File
@@ -23,6 +23,8 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
repo: DVC repo.
"""
sep = os.sep
scheme = "local"
PARAM_CHECKSUM = "md5"
+2
View File
@@ -9,6 +9,8 @@ from .base import BaseFileSystem
class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method
"""Proxies the repo file access methods to Git objects"""
sep = os.sep
scheme = "local"
def __init__(self, root_dir, trie):
+2
View File
@@ -14,6 +14,8 @@ logger = logging.getLogger(__name__)
class LocalFileSystem(BaseFileSystem):
sep = os.sep
scheme = Schemes.LOCAL
PATH_CLS = PathInfo
PARAM_CHECKSUM = "md5"
+2
View File
@@ -31,6 +31,8 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
kwargs: Additional keyword arguments passed to the `DvcFileSystem()`.
"""
sep = os.sep
scheme = "local"
PARAM_CHECKSUM = "md5"
+1 -3
View File
@@ -271,8 +271,6 @@ class Output:
IsStageFileError = OutputIsStageFileError # type: Type[DvcException]
IsIgnoredError = OutputIsIgnoredError # type: Type[DvcException]
sep = "/"
def __init__(
self,
stage,
@@ -887,7 +885,7 @@ class Output:
obj.name = str(self)
if isinstance(obj, Tree):
for key, entry_obj in obj:
entry_obj.name = os.path.join(str(self), *key)
entry_obj.name = self.fs.sep.join([obj.name, *key])
def get_used_external(
self, **kwargs