mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
Misc. imports made lazy: scm,experiments,stage (#5218)
This commit is contained in:
@@ -10,12 +10,10 @@ from typing import Dict, Iterable, Mapping, Optional
|
||||
|
||||
from funcy import cached_property, first
|
||||
|
||||
from dvc.dvcfile import is_lock_file
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.stage.run import CheckpointKilledError
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
from .base import (
|
||||
EXEC_APPLY,
|
||||
@@ -241,6 +239,8 @@ class Experiments:
|
||||
return stash_rev
|
||||
|
||||
def _prune_lockfiles(self):
|
||||
from dvc.dvcfile import is_lock_file
|
||||
|
||||
# NOTE: dirty DVC lock files must be restored to index state to
|
||||
# avoid checking out incorrect persist or checkpoint outs
|
||||
tree = self.scm.get_tree("HEAD")
|
||||
@@ -524,6 +524,8 @@ class Experiments:
|
||||
return result
|
||||
|
||||
def _init_executors(self, to_run):
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
from .executor.local import TempDirExecutor
|
||||
|
||||
executors = {}
|
||||
|
||||
@@ -15,7 +15,6 @@ from dvc.exceptions import GitHookAlreadyExistsError
|
||||
from dvc.scm.base import Base, FileNotInRepoError, RevError
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import path_isin
|
||||
from dvc.utils.serialize import modify_yaml
|
||||
|
||||
from .backend.base import BaseGitBackend, NoGitBackendError
|
||||
from .backend.dulwich import DulwichBackend
|
||||
@@ -230,6 +229,8 @@ class Git(Base):
|
||||
self._install_hook("pre-push")
|
||||
return
|
||||
|
||||
from dvc.utils.serialize import modify_yaml
|
||||
|
||||
config_path = os.path.join(self.root_dir, ".pre-commit-config.yaml")
|
||||
with modify_yaml(config_path) as config:
|
||||
entry = {
|
||||
|
||||
@@ -7,7 +7,6 @@ from typing import Dict, Optional
|
||||
|
||||
from funcy import cached_property, project
|
||||
|
||||
import dvc.dependency as dependency
|
||||
import dvc.prompt as prompt
|
||||
from dvc.exceptions import (
|
||||
CacheLinkError,
|
||||
@@ -15,7 +14,6 @@ from dvc.exceptions import (
|
||||
DvcException,
|
||||
MergeError,
|
||||
)
|
||||
from dvc.output.base import OutputDoesNotExistError
|
||||
from dvc.utils import relpath
|
||||
|
||||
from . import params
|
||||
@@ -241,7 +239,9 @@ class Stage(params.StageParams):
|
||||
if not self.is_import:
|
||||
return False
|
||||
|
||||
return isinstance(self.deps[0], dependency.RepoDependency)
|
||||
from dvc.dependency import RepoDependency
|
||||
|
||||
return isinstance(self.deps[0], RepoDependency)
|
||||
|
||||
@property
|
||||
def is_checkpoint(self):
|
||||
@@ -449,6 +449,8 @@ class Stage(params.StageParams):
|
||||
dep.save()
|
||||
|
||||
def save_outs(self, allow_missing=False):
|
||||
from dvc.output.base import OutputDoesNotExistError
|
||||
|
||||
for out in self.outs:
|
||||
try:
|
||||
out.save()
|
||||
@@ -478,6 +480,8 @@ class Stage(params.StageParams):
|
||||
|
||||
@rwlocked(write=["outs"])
|
||||
def commit(self, allow_missing=False):
|
||||
from dvc.output.base import OutputDoesNotExistError
|
||||
|
||||
link_failures = []
|
||||
for out in self.outs:
|
||||
try:
|
||||
|
||||
+22
-12
@@ -4,14 +4,7 @@ from itertools import product
|
||||
|
||||
from funcy import lsplit, rpartial
|
||||
|
||||
from dvc import dependency, output
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
from ..dependency import ParamsDependency
|
||||
from ..hash_info import HashInfo
|
||||
from ..tree.local import LocalTree
|
||||
from ..tree.s3 import S3Tree
|
||||
from ..utils import dict_md5, format_link, relpath
|
||||
from .exceptions import (
|
||||
MissingDataSource,
|
||||
StageExternalOutputsError,
|
||||
@@ -22,6 +15,8 @@ from .exceptions import (
|
||||
|
||||
|
||||
def check_stage_path(repo, path, is_wdir=False):
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
assert repo is not None
|
||||
|
||||
error_msg = "{wdir_or_path} '{path}' {{}}".format(
|
||||
@@ -42,6 +37,8 @@ def check_stage_path(repo, path, is_wdir=False):
|
||||
|
||||
|
||||
def fill_stage_outputs(stage, **kwargs):
|
||||
from dvc.output import loads_from
|
||||
|
||||
assert not stage.outs
|
||||
|
||||
keys = [
|
||||
@@ -63,7 +60,7 @@ def fill_stage_outputs(stage, **kwargs):
|
||||
)
|
||||
|
||||
for key in keys:
|
||||
stage.outs += output.loads_from(
|
||||
stage.outs += loads_from(
|
||||
stage,
|
||||
kwargs.get(key, []),
|
||||
use_cache="no_cache" not in key,
|
||||
@@ -75,12 +72,12 @@ def fill_stage_outputs(stage, **kwargs):
|
||||
|
||||
|
||||
def _load_live_outputs(stage, live_l=None, live_summary=False):
|
||||
from dvc.output import BaseOutput
|
||||
from dvc.output import BaseOutput, loads_from
|
||||
|
||||
outs = []
|
||||
if live_l:
|
||||
|
||||
outs += output.loads_from(
|
||||
outs += loads_from(
|
||||
stage,
|
||||
[live_l],
|
||||
use_cache=False,
|
||||
@@ -91,15 +88,19 @@ def _load_live_outputs(stage, live_l=None, live_summary=False):
|
||||
|
||||
|
||||
def fill_stage_dependencies(stage, deps=None, erepo=None, params=None):
|
||||
from dvc.dependency import loads_from, loads_params
|
||||
|
||||
assert not stage.deps
|
||||
stage.deps = []
|
||||
stage.deps += dependency.loads_from(stage, deps or [], erepo=erepo)
|
||||
stage.deps += dependency.loads_params(stage, params or [])
|
||||
stage.deps += loads_from(stage, deps or [], erepo=erepo)
|
||||
stage.deps += loads_params(stage, params or [])
|
||||
|
||||
|
||||
def check_no_externals(stage):
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.utils import format_link
|
||||
|
||||
# NOTE: preventing users from accidentally using external outputs. See
|
||||
# https://github.com/iterative/dvc/issues/1545 for more details.
|
||||
|
||||
@@ -154,6 +155,9 @@ def check_missing_outputs(stage):
|
||||
def stage_dump_eq(stage_cls, old_d, new_d):
|
||||
# NOTE: need to remove checksums from old dict in order to compare
|
||||
# it to the new one, since the new one doesn't have checksums yet.
|
||||
from ..tree.local import LocalTree
|
||||
from ..tree.s3 import S3Tree
|
||||
|
||||
old_d.pop(stage_cls.PARAM_MD5, None)
|
||||
new_d.pop(stage_cls.PARAM_MD5, None)
|
||||
outs = old_d.get(stage_cls.PARAM_OUTS, [])
|
||||
@@ -177,6 +181,8 @@ def stage_dump_eq(stage_cls, old_d, new_d):
|
||||
def compute_md5(stage):
|
||||
from dvc.output.base import BaseOutput
|
||||
|
||||
from ..utils import dict_md5
|
||||
|
||||
d = stage.dumpd()
|
||||
|
||||
# Remove md5 and meta, these should not affect stage md5
|
||||
@@ -208,6 +214,8 @@ def compute_md5(stage):
|
||||
|
||||
|
||||
def resolve_wdir(wdir, path):
|
||||
from ..utils import relpath
|
||||
|
||||
rel_wdir = relpath(wdir, os.path.dirname(path))
|
||||
return pathlib.PurePath(rel_wdir).as_posix() if rel_wdir != "." else None
|
||||
|
||||
@@ -238,4 +246,6 @@ def get_dump(stage):
|
||||
|
||||
|
||||
def split_params_deps(stage):
|
||||
from ..dependency import ParamsDependency
|
||||
|
||||
return lsplit(rpartial(isinstance, ParamsDependency), stage.deps)
|
||||
|
||||
Reference in New Issue
Block a user