Misc. imports made lazy: scm,experiments,stage (#5218)

This commit is contained in:
Saugat Pachhai
2021-01-06 17:41:12 +02:00
committed by GitHub
parent 84e9c1b0de
commit 1cc646614f
4 changed files with 35 additions and 18 deletions
+4 -2
View File
@@ -10,12 +10,10 @@ from typing import Dict, Iterable, Mapping, Optional
from funcy import cached_property, first
from dvc.dvcfile import is_lock_file
from dvc.exceptions import DvcException
from dvc.path_info import PathInfo
from dvc.stage.run import CheckpointKilledError
from dvc.utils import relpath
from dvc.utils.fs import makedirs
from .base import (
EXEC_APPLY,
@@ -241,6 +239,8 @@ class Experiments:
return stash_rev
def _prune_lockfiles(self):
from dvc.dvcfile import is_lock_file
# NOTE: dirty DVC lock files must be restored to index state to
# avoid checking out incorrect persist or checkpoint outs
tree = self.scm.get_tree("HEAD")
@@ -524,6 +524,8 @@ class Experiments:
return result
def _init_executors(self, to_run):
from dvc.utils.fs import makedirs
from .executor.local import TempDirExecutor
executors = {}
+2 -1
View File
@@ -15,7 +15,6 @@ from dvc.exceptions import GitHookAlreadyExistsError
from dvc.scm.base import Base, FileNotInRepoError, RevError
from dvc.utils import relpath
from dvc.utils.fs import path_isin
from dvc.utils.serialize import modify_yaml
from .backend.base import BaseGitBackend, NoGitBackendError
from .backend.dulwich import DulwichBackend
@@ -230,6 +229,8 @@ class Git(Base):
self._install_hook("pre-push")
return
from dvc.utils.serialize import modify_yaml
config_path = os.path.join(self.root_dir, ".pre-commit-config.yaml")
with modify_yaml(config_path) as config:
entry = {
+7 -3
View File
@@ -7,7 +7,6 @@ from typing import Dict, Optional
from funcy import cached_property, project
import dvc.dependency as dependency
import dvc.prompt as prompt
from dvc.exceptions import (
CacheLinkError,
@@ -15,7 +14,6 @@ from dvc.exceptions import (
DvcException,
MergeError,
)
from dvc.output.base import OutputDoesNotExistError
from dvc.utils import relpath
from . import params
@@ -241,7 +239,9 @@ class Stage(params.StageParams):
if not self.is_import:
return False
return isinstance(self.deps[0], dependency.RepoDependency)
from dvc.dependency import RepoDependency
return isinstance(self.deps[0], RepoDependency)
@property
def is_checkpoint(self):
@@ -449,6 +449,8 @@ class Stage(params.StageParams):
dep.save()
def save_outs(self, allow_missing=False):
from dvc.output.base import OutputDoesNotExistError
for out in self.outs:
try:
out.save()
@@ -478,6 +480,8 @@ class Stage(params.StageParams):
@rwlocked(write=["outs"])
def commit(self, allow_missing=False):
from dvc.output.base import OutputDoesNotExistError
link_failures = []
for out in self.outs:
try:
+22 -12
View File
@@ -4,14 +4,7 @@ from itertools import product
from funcy import lsplit, rpartial
from dvc import dependency, output
from dvc.utils.fs import path_isin
from ..dependency import ParamsDependency
from ..hash_info import HashInfo
from ..tree.local import LocalTree
from ..tree.s3 import S3Tree
from ..utils import dict_md5, format_link, relpath
from .exceptions import (
MissingDataSource,
StageExternalOutputsError,
@@ -22,6 +15,8 @@ from .exceptions import (
def check_stage_path(repo, path, is_wdir=False):
from dvc.utils.fs import path_isin
assert repo is not None
error_msg = "{wdir_or_path} '{path}' {{}}".format(
@@ -42,6 +37,8 @@ def check_stage_path(repo, path, is_wdir=False):
def fill_stage_outputs(stage, **kwargs):
from dvc.output import loads_from
assert not stage.outs
keys = [
@@ -63,7 +60,7 @@ def fill_stage_outputs(stage, **kwargs):
)
for key in keys:
stage.outs += output.loads_from(
stage.outs += loads_from(
stage,
kwargs.get(key, []),
use_cache="no_cache" not in key,
@@ -75,12 +72,12 @@ def fill_stage_outputs(stage, **kwargs):
def _load_live_outputs(stage, live_l=None, live_summary=False):
from dvc.output import BaseOutput
from dvc.output import BaseOutput, loads_from
outs = []
if live_l:
outs += output.loads_from(
outs += loads_from(
stage,
[live_l],
use_cache=False,
@@ -91,15 +88,19 @@ def _load_live_outputs(stage, live_l=None, live_summary=False):
def fill_stage_dependencies(stage, deps=None, erepo=None, params=None):
from dvc.dependency import loads_from, loads_params
assert not stage.deps
stage.deps = []
stage.deps += dependency.loads_from(stage, deps or [], erepo=erepo)
stage.deps += dependency.loads_params(stage, params or [])
stage.deps += loads_from(stage, deps or [], erepo=erepo)
stage.deps += loads_params(stage, params or [])
def check_no_externals(stage):
from urllib.parse import urlparse
from dvc.utils import format_link
# NOTE: preventing users from accidentally using external outputs. See
# https://github.com/iterative/dvc/issues/1545 for more details.
@@ -154,6 +155,9 @@ def check_missing_outputs(stage):
def stage_dump_eq(stage_cls, old_d, new_d):
# NOTE: need to remove checksums from old dict in order to compare
# it to the new one, since the new one doesn't have checksums yet.
from ..tree.local import LocalTree
from ..tree.s3 import S3Tree
old_d.pop(stage_cls.PARAM_MD5, None)
new_d.pop(stage_cls.PARAM_MD5, None)
outs = old_d.get(stage_cls.PARAM_OUTS, [])
@@ -177,6 +181,8 @@ def stage_dump_eq(stage_cls, old_d, new_d):
def compute_md5(stage):
from dvc.output.base import BaseOutput
from ..utils import dict_md5
d = stage.dumpd()
# Remove md5 and meta, these should not affect stage md5
@@ -208,6 +214,8 @@ def compute_md5(stage):
def resolve_wdir(wdir, path):
from ..utils import relpath
rel_wdir = relpath(wdir, os.path.dirname(path))
return pathlib.PurePath(rel_wdir).as_posix() if rel_wdir != "." else None
@@ -238,4 +246,6 @@ def get_dump(stage):
def split_params_deps(stage):
from ..dependency import ParamsDependency
return lsplit(rpartial(isinstance, ParamsDependency), stage.deps)