mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
output: optimize obj collection (#6277)
* output: str only once Calling `__str__` takes a long time and if we are dealing with a large dataset, it will result in a very substantial time. For example, locally, for 700K objects, it took 20sec more. Related to #6276 * output: don't use os.path.join Takes around 2sec for 700K objects. * output: use fs.sep * output: remove unused sep
This commit is contained in:
@@ -33,6 +33,8 @@ class RemoteMissingDepsError(DvcException):
|
||||
|
||||
|
||||
class BaseFileSystem:
|
||||
sep = "/"
|
||||
|
||||
scheme = "base"
|
||||
REQUIRES: ClassVar[Dict[str, str]] = {}
|
||||
PATH_CLS = URLInfo # type: Any
|
||||
|
||||
@@ -23,6 +23,8 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
repo: DVC repo.
|
||||
"""
|
||||
|
||||
sep = os.sep
|
||||
|
||||
scheme = "local"
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
|
||||
@@ -9,6 +9,8 @@ from .base import BaseFileSystem
|
||||
class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
"""Proxies the repo file access methods to Git objects"""
|
||||
|
||||
sep = os.sep
|
||||
|
||||
scheme = "local"
|
||||
|
||||
def __init__(self, root_dir, trie):
|
||||
|
||||
@@ -14,6 +14,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LocalFileSystem(BaseFileSystem):
|
||||
sep = os.sep
|
||||
|
||||
scheme = Schemes.LOCAL
|
||||
PATH_CLS = PathInfo
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
@@ -31,6 +31,8 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
kwargs: Additional keyword arguments passed to the `DvcFileSystem()`.
|
||||
"""
|
||||
|
||||
sep = os.sep
|
||||
|
||||
scheme = "local"
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
|
||||
+1
-3
@@ -271,8 +271,6 @@ class Output:
|
||||
IsStageFileError = OutputIsStageFileError # type: Type[DvcException]
|
||||
IsIgnoredError = OutputIsIgnoredError # type: Type[DvcException]
|
||||
|
||||
sep = "/"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
stage,
|
||||
@@ -887,7 +885,7 @@ class Output:
|
||||
obj.name = str(self)
|
||||
if isinstance(obj, Tree):
|
||||
for key, entry_obj in obj:
|
||||
entry_obj.name = os.path.join(str(self), *key)
|
||||
entry_obj.name = self.fs.sep.join([obj.name, *key])
|
||||
|
||||
def get_used_external(
|
||||
self, **kwargs
|
||||
|
||||
Reference in New Issue
Block a user