mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-10 07:18:27 +01:00
mypy: use typesafe cached_property (#8799)
This commit is contained in:
+5
-2
@@ -4,11 +4,14 @@ import os
|
||||
import re
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
from typing import Dict
|
||||
|
||||
from funcy import cached_property, compact, memoize, re_find
|
||||
from funcy import compact, memoize, re_find
|
||||
|
||||
from dvc.exceptions import DvcException, NotDvcRepoError
|
||||
|
||||
from .utils.objects import cached_property
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -116,7 +119,7 @@ class Config(dict):
|
||||
return site_config_dir(cls.APPNAME, cls.APPAUTHOR)
|
||||
|
||||
@cached_property
|
||||
def files(self):
|
||||
def files(self) -> Dict[str, str]:
|
||||
files = {
|
||||
level: os.path.join(self.get_dir(level), self.CONFIG)
|
||||
for level in ("system", "global")
|
||||
|
||||
+2
-3
@@ -3,9 +3,8 @@
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Iterable, Optional
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.config import NoRemoteError, RemoteConfigError
|
||||
from dvc.utils.objects import cached_property
|
||||
from dvc_data.hashfile.db import get_index
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -34,7 +33,7 @@ class Remote:
|
||||
self.fs.version_aware = True
|
||||
|
||||
@cached_property
|
||||
def odb(self):
|
||||
def odb(self) -> "HashFileDB":
|
||||
from dvc_data.hashfile.db import get_odb
|
||||
|
||||
path = self.path
|
||||
|
||||
+20
-10
@@ -1,9 +1,16 @@
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Any, Callable, List, Tuple, TypeVar, Union
|
||||
|
||||
from funcy import cached_property
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
List,
|
||||
Tuple,
|
||||
TypeVar,
|
||||
Union,
|
||||
)
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.parsing.versions import LOCKFILE_VERSION, SCHEMA_KWD
|
||||
@@ -16,11 +23,14 @@ from dvc.stage.exceptions import (
|
||||
from dvc.types import AnyPath
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.collections import apply_diff
|
||||
from dvc.utils.objects import cached_property
|
||||
from dvc.utils.serialize import dump_yaml, modify_yaml
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.repo import Repo
|
||||
|
||||
from .parsing import DataResolver
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_T = TypeVar("_T")
|
||||
|
||||
@@ -291,34 +301,34 @@ class PipelineFile(FileMixin):
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def contents(self):
|
||||
def contents(self) -> Dict[str, Any]:
|
||||
return self._load()[0]
|
||||
|
||||
@cached_property
|
||||
def lockfile_contents(self):
|
||||
def lockfile_contents(self) -> Dict[str, Any]:
|
||||
return self._lockfile.load()
|
||||
|
||||
@cached_property
|
||||
def resolver(self):
|
||||
def resolver(self) -> "DataResolver":
|
||||
from .parsing import DataResolver
|
||||
|
||||
wdir = self.repo.fs.path.parent(self.path)
|
||||
return DataResolver(self.repo, wdir, self.contents)
|
||||
|
||||
@cached_property
|
||||
def stages(self):
|
||||
def stages(self) -> LOADER:
|
||||
return self.LOADER(self, self.contents, self.lockfile_contents)
|
||||
|
||||
@property
|
||||
def metrics(self):
|
||||
def metrics(self) -> List[str]:
|
||||
return self.contents.get("metrics", [])
|
||||
|
||||
@property
|
||||
def plots(self):
|
||||
def plots(self) -> Any:
|
||||
return self.contents.get("plots", {})
|
||||
|
||||
@property
|
||||
def params(self):
|
||||
def params(self) -> List[str]:
|
||||
return self.contents.get("params", [])
|
||||
|
||||
def remove(self, force=False):
|
||||
|
||||
+10
-7
@@ -1,9 +1,8 @@
|
||||
# pylint: disable=unused-import
|
||||
from contextlib import ExitStack
|
||||
from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union
|
||||
|
||||
from funcy import cached_property
|
||||
from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union, cast
|
||||
|
||||
from dvc.utils.objects import cached_property
|
||||
from dvc_objects.fs.callbacks import ( # noqa: F401
|
||||
DEFAULT_CALLBACK,
|
||||
Callback,
|
||||
@@ -11,6 +10,8 @@ from dvc_objects.fs.callbacks import ( # noqa: F401
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from rich.progress import TaskID
|
||||
|
||||
from dvc.ui._rich_progress import RichTransferProgress
|
||||
|
||||
|
||||
@@ -41,7 +42,7 @@ class RichCallback(Callback):
|
||||
super().__init__(size=size, value=value)
|
||||
|
||||
@cached_property
|
||||
def progress(self):
|
||||
def progress(self) -> "RichTransferProgress":
|
||||
from dvc.ui import ui
|
||||
from dvc.ui._rich_progress import RichTransferProgress
|
||||
|
||||
@@ -53,11 +54,13 @@ class RichCallback(Callback):
|
||||
disable=self.disable,
|
||||
console=ui.error_console,
|
||||
)
|
||||
return self._stack.enter_context(progress)
|
||||
return cast(RichTransferProgress, self._stack.enter_context(progress))
|
||||
|
||||
@cached_property
|
||||
def task(self):
|
||||
return self.progress.add_task(**self._task_kwargs)
|
||||
def task(self) -> "TaskID":
|
||||
return self.progress.add_task(
|
||||
**self._task_kwargs # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
+12
-7
@@ -1,13 +1,15 @@
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.utils import as_posix
|
||||
from dvc_data.fs import DataFileSystem as _DataFileSystem
|
||||
from dvc_objects.fs.base import FileSystem
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc_data.fs import DataFileSystem as _DataFileSystem
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -19,9 +21,12 @@ class DataFileSystem(FileSystem):
|
||||
def _prepare_credentials(self, **config):
|
||||
return config
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def fs(self):
|
||||
@functools.cached_property
|
||||
def fs( # pylint: disable=invalid-overridden-method
|
||||
self,
|
||||
) -> "_DataFileSystem":
|
||||
from dvc_data.fs import DataFileSystem as _DataFileSystem
|
||||
|
||||
return _DataFileSystem(**self.fs_args)
|
||||
|
||||
def isdvc(self, path, **kwargs):
|
||||
|
||||
+4
-4
@@ -1,4 +1,5 @@
|
||||
import errno
|
||||
import functools
|
||||
import logging
|
||||
import ntpath
|
||||
import os
|
||||
@@ -8,7 +9,7 @@ from contextlib import suppress
|
||||
from typing import TYPE_CHECKING, Any, Callable, Optional, Tuple, Type, Union
|
||||
|
||||
from fsspec.spec import AbstractFileSystem
|
||||
from funcy import cached_property, wrap_prop, wrap_with
|
||||
from funcy import wrap_with
|
||||
|
||||
from dvc_objects.fs.base import FileSystem
|
||||
from dvc_objects.fs.path import Path
|
||||
@@ -374,9 +375,8 @@ class DVCFileSystem(FileSystem):
|
||||
def _prepare_credentials(self, **config):
|
||||
return config
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def fs(self):
|
||||
@functools.cached_property
|
||||
def fs(self): # pylint: disable=invalid-overridden-method
|
||||
return _DVCFileSystem(**self.fs_args)
|
||||
|
||||
def isdvc(self, path, **kwargs):
|
||||
|
||||
+7
-8
@@ -1,8 +1,6 @@
|
||||
import threading
|
||||
import functools
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from . import FileSystem
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -39,15 +37,16 @@ class GitFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
}
|
||||
)
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def fs(self) -> "FsspecGitFileSystem":
|
||||
@functools.cached_property
|
||||
def fs( # pylint: disable=invalid-overridden-method
|
||||
self,
|
||||
) -> "FsspecGitFileSystem":
|
||||
from scmrepo.fs import GitFileSystem as FsspecGitFileSystem
|
||||
|
||||
return FsspecGitFileSystem(**self.fs_args)
|
||||
|
||||
@cached_property
|
||||
def path(self):
|
||||
@functools.cached_property
|
||||
def path(self): # pylint: disable=invalid-overridden-method
|
||||
return self.fs.path
|
||||
|
||||
@property
|
||||
|
||||
+2
-1
@@ -4,7 +4,7 @@ from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Type
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import cached_property, collecting, project
|
||||
from funcy import collecting, project
|
||||
from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo
|
||||
|
||||
from dvc import prompt
|
||||
@@ -17,6 +17,7 @@ from dvc.exceptions import (
|
||||
MergeError,
|
||||
RemoteCacheRequiredError,
|
||||
)
|
||||
from dvc.utils.objects import cached_property
|
||||
from dvc_data.hashfile import Tree
|
||||
from dvc_data.hashfile import check as ocheck
|
||||
from dvc_data.hashfile import load as oload
|
||||
|
||||
@@ -14,10 +14,11 @@ from typing import (
|
||||
Union,
|
||||
)
|
||||
|
||||
from funcy import cached_property, collecting, first, isa, join, reraise
|
||||
from funcy import collecting, first, isa, join, reraise
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.parsing.interpolate import ParseError
|
||||
from dvc.utils.objects import cached_property
|
||||
|
||||
from .context import (
|
||||
Context,
|
||||
|
||||
+2
-1
@@ -7,9 +7,10 @@ from contextlib import AbstractContextManager
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import List, Optional, TextIO, Union
|
||||
|
||||
from funcy import cached_property
|
||||
from shortuuid import uuid
|
||||
|
||||
from dvc.utils.objects import cached_property
|
||||
|
||||
from .exceptions import TimeoutExpired
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+24
-16
@@ -3,9 +3,7 @@ import os
|
||||
from collections import defaultdict
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
from typing import TYPE_CHECKING, Callable, Iterable, Optional
|
||||
|
||||
from funcy import cached_property
|
||||
from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Union
|
||||
|
||||
from dvc.exceptions import FileMissingError
|
||||
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
|
||||
@@ -13,13 +11,20 @@ from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
|
||||
from dvc.ignore import DvcIgnoreFilter
|
||||
from dvc.utils import env2bool
|
||||
from dvc.utils.fs import path_isin
|
||||
from dvc.utils.objects import cached_property
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.fs import FileSystem
|
||||
from dvc.repo.scm_context import SCMContext
|
||||
from dvc.scm import Base
|
||||
from dvc.fs.data import DataFileSystem
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
from dvc.machine import MachineManager
|
||||
from dvc.scm import Base, Git, NoSCM
|
||||
from dvc.stage import Stage
|
||||
|
||||
from .experiments import Experiments
|
||||
from .index import Index
|
||||
from .scm_context import SCMContext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -237,13 +242,13 @@ class Repo:
|
||||
return self.url or self.root_dir
|
||||
|
||||
@cached_property
|
||||
def index(self):
|
||||
def index(self) -> "Index":
|
||||
from dvc.repo.index import Index
|
||||
|
||||
return Index.from_repo(self)
|
||||
|
||||
def check_graph(
|
||||
self, stages: Iterable["Stage"] = None, callback: Callable = None
|
||||
self, stages: Iterable["Stage"], callback: Callable = None
|
||||
) -> None:
|
||||
if not getattr(self, "_skip_graph_checks", False):
|
||||
new = self.index.update(stages)
|
||||
@@ -267,7 +272,7 @@ class Repo:
|
||||
return external_repo(url, *args, **kwargs)
|
||||
|
||||
@cached_property
|
||||
def scm(self):
|
||||
def scm(self) -> Union["Git", "NoSCM"]:
|
||||
from dvc.scm import SCM, SCMError
|
||||
|
||||
if self._scm:
|
||||
@@ -291,7 +296,6 @@ class Repo:
|
||||
|
||||
@cached_property
|
||||
def dvcignore(self) -> DvcIgnoreFilter:
|
||||
|
||||
return DvcIgnoreFilter(self.fs, self.root_dir)
|
||||
|
||||
def get_rev(self):
|
||||
@@ -306,13 +310,13 @@ class Repo:
|
||||
return self.fs.rev
|
||||
|
||||
@cached_property
|
||||
def experiments(self):
|
||||
def experiments(self) -> "Experiments":
|
||||
from dvc.repo.experiments import Experiments
|
||||
|
||||
return Experiments(self)
|
||||
|
||||
@cached_property
|
||||
def machine(self):
|
||||
def machine(self) -> Optional["MachineManager"]:
|
||||
from dvc.machine import MachineManager
|
||||
|
||||
if self.tmp_dir and (
|
||||
@@ -366,12 +370,14 @@ class Repo:
|
||||
raise NotDvcRepoError(msg)
|
||||
|
||||
@classmethod
|
||||
def find_dvc_dir(cls, root=None):
|
||||
def find_dvc_dir(cls, root=None) -> str:
|
||||
root_dir = cls.find_root(root)
|
||||
return os.path.join(root_dir, cls.DVC_DIR)
|
||||
|
||||
@staticmethod
|
||||
def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
|
||||
def init(
|
||||
root_dir=os.curdir, no_scm=False, force=False, subdir=False
|
||||
) -> "Repo":
|
||||
from dvc.repo.init import init
|
||||
|
||||
return init(
|
||||
@@ -459,7 +465,9 @@ class Repo:
|
||||
return used
|
||||
|
||||
@property
|
||||
def stages(self): # obsolete, only for backward-compatibility
|
||||
def stages(
|
||||
self,
|
||||
) -> List["Stage"]: # obsolete, only for backward-compatibility
|
||||
return self.index.stages
|
||||
|
||||
def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
|
||||
@@ -494,13 +502,13 @@ class Repo:
|
||||
return self.DVC_DIR in path_parts
|
||||
|
||||
@cached_property
|
||||
def datafs(self):
|
||||
def datafs(self) -> "DataFileSystem":
|
||||
from dvc.fs.data import DataFileSystem
|
||||
|
||||
return DataFileSystem(index=self.index.data["repo"])
|
||||
|
||||
@cached_property
|
||||
def dvcfs(self):
|
||||
def dvcfs(self) -> "DVCFileSystem":
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
|
||||
return DVCFileSystem(
|
||||
|
||||
@@ -13,14 +13,12 @@ from typing import (
|
||||
Tuple,
|
||||
TypedDict,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
from dvc.fs.git import GitFileSystem
|
||||
from dvc.ui import ui
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.output import Output
|
||||
from dvc.repo import Repo
|
||||
from dvc.scm import Git, NoSCM
|
||||
from dvc_data.hashfile.db import HashFileDB
|
||||
@@ -175,7 +173,6 @@ def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
|
||||
|
||||
unstaged_diff = defaultdict(list)
|
||||
for out in repo.index.outs:
|
||||
out = cast("Output", out)
|
||||
if not out.use_cache:
|
||||
continue
|
||||
|
||||
@@ -223,7 +220,6 @@ def _diff_head_to_index(
|
||||
staged_diff = defaultdict(list)
|
||||
for rev in repo.brancher(revs=[head]):
|
||||
for out in repo.index.outs:
|
||||
out = cast("Output", out)
|
||||
if not out.use_cache:
|
||||
continue
|
||||
|
||||
|
||||
@@ -4,11 +4,12 @@ import re
|
||||
import time
|
||||
from typing import Dict, Iterable, Optional
|
||||
|
||||
from funcy import cached_property, chain, first
|
||||
from funcy import chain, first
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.ui import ui
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.objects import cached_property
|
||||
|
||||
from .exceptions import (
|
||||
BaselineMismatchError,
|
||||
@@ -62,11 +63,11 @@ class Experiments:
|
||||
return self.repo.scm
|
||||
|
||||
@cached_property
|
||||
def dvc_dir(self):
|
||||
def dvc_dir(self) -> str:
|
||||
return relpath(self.repo.dvc_dir, self.repo.scm.root_dir)
|
||||
|
||||
@cached_property
|
||||
def args_file(self):
|
||||
def args_file(self) -> str:
|
||||
return os.path.join(self.repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE)
|
||||
|
||||
@cached_property
|
||||
@@ -427,8 +428,10 @@ class Experiments:
|
||||
if not name:
|
||||
if rev in self.stash_revs:
|
||||
name = self.stash_revs[rev].name
|
||||
elif rev in self.celery_queue.failed_stash.stash_revs:
|
||||
name = self.celery_queue.failed_stash.stash_revs[rev].name
|
||||
else:
|
||||
failed_stash = self.celery_queue.failed_stash
|
||||
if failed_stash and rev in failed_stash.stash_revs:
|
||||
name = failed_stash.stash_revs[rev].name
|
||||
result[rev] = name
|
||||
return result
|
||||
|
||||
|
||||
@@ -2,9 +2,9 @@ import logging
|
||||
import os
|
||||
from contextlib import ExitStack
|
||||
from tempfile import mkdtemp
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
from typing import TYPE_CHECKING, Optional, Union
|
||||
|
||||
from funcy import cached_property, retry
|
||||
from funcy import retry
|
||||
from scmrepo.exceptions import SCMError as _SCMError
|
||||
from shortuuid import uuid
|
||||
|
||||
@@ -23,6 +23,7 @@ from dvc.repo.experiments.refs import (
|
||||
from dvc.repo.experiments.utils import EXEC_TMP_DIR, get_exp_rwlock
|
||||
from dvc.scm import SCM, GitMergeError
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.objects import cached_property
|
||||
|
||||
from .base import BaseExecutor, TaskStatus
|
||||
|
||||
@@ -31,6 +32,7 @@ if TYPE_CHECKING:
|
||||
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.experiments.stash import ExpStashEntry
|
||||
from dvc.scm import NoSCM
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -46,7 +48,7 @@ class BaseLocalExecutor(BaseExecutor):
|
||||
return f"file://{root_dir}"
|
||||
|
||||
@cached_property
|
||||
def scm(self):
|
||||
def scm(self) -> Union["Git", "NoSCM"]:
|
||||
return SCM(self.root_dir)
|
||||
|
||||
def cleanup(self, infofile: str):
|
||||
|
||||
@@ -81,6 +81,7 @@ class SSHExecutor(BaseExecutor):
|
||||
**kwargs,
|
||||
):
|
||||
machine_name: Optional[str] = kwargs.pop("machine_name", None)
|
||||
assert repo.machine
|
||||
executor = cls._from_stash_entry(
|
||||
repo,
|
||||
entry,
|
||||
|
||||
@@ -17,7 +17,7 @@ from typing import (
|
||||
Union,
|
||||
)
|
||||
|
||||
from funcy import cached_property, retry
|
||||
from funcy import retry
|
||||
|
||||
from dvc.dependency import ParamsDependency
|
||||
from dvc.env import DVC_EXP_BASELINE_REV, DVC_EXP_NAME, DVCLIVE_RESUME
|
||||
@@ -39,6 +39,7 @@ from dvc.repo.experiments.utils import (
|
||||
get_random_exp_name,
|
||||
)
|
||||
from dvc.ui import ui
|
||||
from dvc.utils.objects import cached_property
|
||||
|
||||
from .utils import get_remote_executor_refs
|
||||
|
||||
@@ -131,7 +132,7 @@ class BaseStashQueue(ABC):
|
||||
return os.path.join(self.repo.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR)
|
||||
|
||||
@cached_property
|
||||
def args_file(self):
|
||||
def args_file(self) -> str:
|
||||
return os.path.join(self.repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE)
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -16,7 +16,7 @@ from typing import (
|
||||
)
|
||||
|
||||
from celery.result import AsyncResult
|
||||
from funcy import cached_property, first
|
||||
from funcy import first
|
||||
from kombu.message import Message
|
||||
|
||||
from dvc.daemon import daemonize
|
||||
@@ -26,6 +26,7 @@ from dvc.repo.experiments.executor.base import ExecutorInfo, ExecutorResult
|
||||
from dvc.repo.experiments.refs import CELERY_STASH
|
||||
from dvc.repo.experiments.utils import EXEC_TMP_DIR, get_exp_rwlock
|
||||
from dvc.ui import ui
|
||||
from dvc.utils.objects import cached_property
|
||||
|
||||
from .base import (
|
||||
BaseStashQueue,
|
||||
|
||||
@@ -24,7 +24,9 @@ def remove_tasks( # noqa: C901
|
||||
failed_stash_revs: List["ExpStashEntry"] = []
|
||||
done_entry_set: Set["QueueEntry"] = set()
|
||||
stash_rev_all = celery_queue.stash.stash_revs
|
||||
failed_rev_all = celery_queue.failed_stash.stash_revs
|
||||
failed_rev_all: Dict[str, "ExpStashEntry"] = {}
|
||||
if celery_queue.failed_stash:
|
||||
failed_rev_all = celery_queue.failed_stash.stash_revs
|
||||
for entry in queue_entries:
|
||||
if entry.stash_rev in stash_rev_all:
|
||||
stash_revs[entry.stash_rev] = stash_rev_all[entry.stash_rev]
|
||||
@@ -58,7 +60,8 @@ def remove_tasks( # noqa: C901
|
||||
result.forget()
|
||||
celery_queue.celery.purge(msg.delivery_tag)
|
||||
finally:
|
||||
celery_queue.failed_stash.remove_revs(failed_stash_revs)
|
||||
if celery_queue.failed_stash:
|
||||
celery_queue.failed_stash.remove_revs(failed_stash_revs)
|
||||
|
||||
|
||||
def _get_names(entries: Iterable[Union["QueueEntry", "QueueDoneResult"]]):
|
||||
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Dict, Generator, Optional
|
||||
|
||||
from funcy import cached_property, first
|
||||
from funcy import first
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.repo.experiments.exceptions import ExpQueueEmptyError
|
||||
@@ -15,6 +15,7 @@ from dvc.repo.experiments.executor.base import (
|
||||
)
|
||||
from dvc.repo.experiments.executor.local import TempDirExecutor
|
||||
from dvc.repo.experiments.utils import EXEC_PID_DIR, EXEC_TMP_DIR
|
||||
from dvc.utils.objects import cached_property
|
||||
|
||||
from .base import BaseStashQueue, QueueEntry, QueueGetResult
|
||||
from .utils import fetch_running_exp_from_temp_dir
|
||||
|
||||
@@ -57,7 +57,7 @@ class WorkspaceQueue(BaseStashQueue):
|
||||
return QueueGetResult(entry, executor)
|
||||
|
||||
def iter_queued(self) -> Generator[QueueEntry, None, None]:
|
||||
for rev, entry in self.stash.stash_revs:
|
||||
for rev, entry in self.stash.stash_revs.items():
|
||||
yield QueueEntry(
|
||||
self.repo.root_dir,
|
||||
self.scm.root_dir,
|
||||
|
||||
+1
-1
@@ -16,10 +16,10 @@ from typing import (
|
||||
Union,
|
||||
)
|
||||
|
||||
from funcy import cached_property
|
||||
from funcy.debug import format_time
|
||||
|
||||
from dvc.fs import LocalFileSystem
|
||||
from dvc.utils.objects import cached_property
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from networkx import DiGraph
|
||||
|
||||
@@ -11,7 +11,7 @@ from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
Generator,
|
||||
Iterator,
|
||||
List,
|
||||
Optional,
|
||||
Set,
|
||||
@@ -20,10 +20,11 @@ from typing import (
|
||||
|
||||
import dpath.options
|
||||
import dpath.util
|
||||
from funcy import cached_property, distinct, first, project
|
||||
from funcy import distinct, first, project
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import error_handler, errored_revisions, onerror_collect
|
||||
from dvc.utils.objects import cached_property
|
||||
from dvc.utils.serialize import LOADERS
|
||||
from dvc.utils.threadpool import ThreadPoolExecutor
|
||||
|
||||
@@ -73,7 +74,7 @@ class Plots:
|
||||
onerror: Optional[Callable] = None,
|
||||
props: Optional[Dict] = None,
|
||||
config_files: Optional[Set[str]] = None,
|
||||
) -> Generator[Dict, None, None]:
|
||||
) -> Iterator[Dict]:
|
||||
"""Collects plots definitions and data sources.
|
||||
|
||||
Generator yielding a structure like:
|
||||
@@ -260,9 +261,10 @@ class Plots:
|
||||
dvcfile.dump(out.stage, update_lock=False)
|
||||
|
||||
@cached_property
|
||||
def templates_dir(self):
|
||||
def templates_dir(self) -> Optional[str]:
|
||||
if self.repo.dvc_dir:
|
||||
return os.path.join(self.repo.dvc_dir, "plots")
|
||||
return None
|
||||
|
||||
|
||||
def _is_plot(out: "Output") -> bool:
|
||||
|
||||
@@ -6,7 +6,7 @@ from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Set
|
||||
|
||||
from funcy import cached_property, project
|
||||
from funcy import project
|
||||
|
||||
from dvc import prompt
|
||||
from dvc.exceptions import (
|
||||
@@ -16,6 +16,7 @@ from dvc.exceptions import (
|
||||
MergeError,
|
||||
)
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.objects import cached_property
|
||||
|
||||
from . import params
|
||||
from .decorators import rwlocked
|
||||
@@ -216,11 +217,11 @@ class Stage(params.StageParams):
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def path_in_repo(self):
|
||||
def path_in_repo(self) -> str:
|
||||
return relpath(self.path, self.repo.root_dir)
|
||||
|
||||
@cached_property
|
||||
def relpath(self):
|
||||
def relpath(self) -> str:
|
||||
return relpath(self.path)
|
||||
|
||||
@property
|
||||
|
||||
+2
-5
@@ -4,7 +4,7 @@ import tempfile
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from funcy import cached_property, first
|
||||
from funcy import first
|
||||
|
||||
from dvc import fs
|
||||
from dvc.exceptions import DvcException
|
||||
@@ -61,10 +61,7 @@ def _get_stage_hash(stage):
|
||||
class StageCache:
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
|
||||
@cached_property
|
||||
def cache_dir(self):
|
||||
return os.path.join(self.repo.odb.local.path, "runs")
|
||||
self.cache_dir = os.path.join(self.repo.odb.local.path, "runs")
|
||||
|
||||
def _get_cache_dir(self, key):
|
||||
return os.path.join(self.cache_dir, key[:2], key)
|
||||
|
||||
+4
-2
@@ -2,12 +2,14 @@ import logging
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from itertools import chain
|
||||
from typing import Any, Dict
|
||||
|
||||
from funcy import cached_property, get_in, lcat, once, project
|
||||
from funcy import get_in, lcat, once, project
|
||||
|
||||
from dvc import dependency, output
|
||||
from dvc.parsing import FOREACH_KWD, JOIN, EntryNotFound
|
||||
from dvc.parsing.versions import LOCKFILE_VERSION
|
||||
from dvc.utils.objects import cached_property
|
||||
from dvc_data.hashfile.hash_info import HashInfo
|
||||
from dvc_data.hashfile.meta import Meta
|
||||
|
||||
@@ -40,7 +42,7 @@ class StageLoader(Mapping):
|
||||
self._lockfile_data = lockfile_data.get("stages", {})
|
||||
|
||||
@cached_property
|
||||
def lockfile_data(self):
|
||||
def lockfile_data(self) -> Dict[str, Any]:
|
||||
if not self._lockfile_data:
|
||||
logger.debug("Lockfile for '%s' not found", self.dvcfile.relpath)
|
||||
return self._lockfile_data
|
||||
|
||||
@@ -5,9 +5,8 @@ import posixpath
|
||||
from typing import Callable
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.objects import cached_property
|
||||
|
||||
|
||||
class _BasePath:
|
||||
@@ -169,7 +168,7 @@ class URLInfo(_BasePath):
|
||||
return self.from_parts(*self._base_parts, path=path)
|
||||
|
||||
@cached_property
|
||||
def url(self):
|
||||
def url(self) -> str:
|
||||
return f"{self.scheme}://{self.netloc}{self._spath}"
|
||||
|
||||
def __str__(self):
|
||||
@@ -203,15 +202,17 @@ class URLInfo(_BasePath):
|
||||
return self._spath
|
||||
|
||||
@cached_property
|
||||
def _path(self): # false-positive, pylint: disable=method-hidden
|
||||
def _path( # pylint: disable=method-hidden
|
||||
self,
|
||||
) -> "_URLPathInfo":
|
||||
return _URLPathInfo(self._spath)
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
def name(self) -> str:
|
||||
return self._path.name
|
||||
|
||||
@cached_property
|
||||
def netloc(self):
|
||||
def netloc(self) -> str:
|
||||
netloc = self.host
|
||||
if self.user:
|
||||
netloc = self.user + "@" + netloc
|
||||
@@ -220,7 +221,7 @@ class URLInfo(_BasePath):
|
||||
return netloc
|
||||
|
||||
@property
|
||||
def bucket(self):
|
||||
def bucket(self) -> str:
|
||||
return self.netloc
|
||||
|
||||
@property
|
||||
@@ -321,7 +322,7 @@ class HTTPURLInfo(URLInfo):
|
||||
return self._base_parts + self._path.parts + self._extra_parts
|
||||
|
||||
@cached_property
|
||||
def url(self):
|
||||
def url(self) -> str:
|
||||
return "{}://{}{}{}{}{}".format(
|
||||
self.scheme,
|
||||
self.netloc,
|
||||
@@ -344,7 +345,7 @@ class HTTPURLInfo(URLInfo):
|
||||
|
||||
class WebDAVURLInfo(URLInfo):
|
||||
@cached_property
|
||||
def url(self):
|
||||
def url(self) -> str:
|
||||
return "{}://{}{}".format(
|
||||
self.scheme.replace("webdav", "http"), self.netloc, self._spath
|
||||
)
|
||||
|
||||
+4
-3
@@ -13,9 +13,10 @@ from typing import (
|
||||
Union,
|
||||
)
|
||||
|
||||
from funcy import cached_property
|
||||
from dvc.utils.objects import cached_property
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from rich.console import Console as RichConsole
|
||||
from rich.status import Status
|
||||
from rich.text import Text as RichText
|
||||
|
||||
@@ -206,14 +207,14 @@ class Console:
|
||||
return answer.startswith("y")
|
||||
|
||||
@cached_property
|
||||
def rich_console(self):
|
||||
def rich_console(self) -> "RichConsole":
|
||||
"""rich_console is only set to stdout for now."""
|
||||
from rich import console
|
||||
|
||||
return console.Console()
|
||||
|
||||
@cached_property
|
||||
def error_console(self):
|
||||
def error_console(self) -> "RichConsole":
|
||||
from rich import console
|
||||
|
||||
return console.Console(stderr=True)
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from functools import cached_property
|
||||
else:
|
||||
from funcy import cached_property # noqa: TID251
|
||||
|
||||
__all__ = ["cached_property"]
|
||||
@@ -179,6 +179,10 @@ parametrize-names-type = "csv"
|
||||
[tool.ruff.flake8-unused-arguments]
|
||||
ignore-variadic-names = true
|
||||
|
||||
[tool.ruff.flake8-tidy-imports]
|
||||
[tool.ruff.flake8-tidy-imports.banned-api]
|
||||
"funcy.cached_property".msg = "use `from dvc.utils.objects import cached_property` instead."
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
log_level = "debug"
|
||||
addopts = "-ra --cov-config pyproject.toml"
|
||||
|
||||
Reference in New Issue
Block a user