Misc. imports lazy on utils,stage.cache and dvc.* (#5228)

dvc/_debug.py

@@ -1,5 +1,8 @@
-import argparse
 from contextlib import ExitStack, contextmanager
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from argparse import Namespace
 
 
 @contextmanager
@@ -56,7 +59,7 @@ def debug():
 
 
 @contextmanager
-def debugtools(args: "argparse.Namespace" = None, **kwargs):
+def debugtools(args: "Namespace" = None, **kwargs):
     kw = vars(args) if args else {}
     kw.update(kwargs)
 
@@ -71,25 +74,18 @@ def debugtools(args: "argparse.Namespace" = None, **kwargs):
 
 
 def add_debugging_flags(parser):
+    from argparse import SUPPRESS
+
     parser.add_argument(
-        "--cprofile",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
+        "--cprofile", action="store_true", default=False, help=SUPPRESS,
     )
-    parser.add_argument("--cprofile-dump", help=argparse.SUPPRESS)
+    parser.add_argument("--cprofile-dump", help=SUPPRESS)
     parser.add_argument(
-        "--pdb", action="store_true", default=False, help=argparse.SUPPRESS,
+        "--pdb", action="store_true", default=False, help=SUPPRESS,
     )
     parser.add_argument(
-        "--instrument",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
+        "--instrument", action="store_true", default=False, help=SUPPRESS,
     )
     parser.add_argument(
-        "--instrument-open",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
+        "--instrument-open", action="store_true", default=False, help=SUPPRESS,
     )

dvc/data_cloud.py

@@ -2,9 +2,6 @@
 
 import logging
 
-from dvc.config import NoRemoteError
-from dvc.remote import get_remote
-
 logger = logging.getLogger(__name__)
 
 
@@ -23,6 +20,8 @@ class DataCloud:
         self.repo = repo
 
     def get_remote(self, name=None, command="<command>"):
+        from dvc.config import NoRemoteError
+
         if not name:
             name = self.repo.config["core"].get("remote")
 
@@ -45,6 +44,8 @@ class DataCloud:
         raise NoRemoteError(error_msg)
 
     def _init_remote(self, name):
+        from dvc.remote import get_remote
+
         return get_remote(self.repo, name=name)
 
     def push(

dvc/rwlock.py

@@ -6,7 +6,6 @@ from contextlib import contextmanager
 from voluptuous import Invalid, Optional, Required, Schema
 
 from .exceptions import DvcException
-from .lock import LockError
 from .utils import relpath
 
 INFO_SCHEMA = {Required("pid"): int, Required("cmd"): str}
@@ -60,6 +59,8 @@ def _infos_to_str(infos):
 
 
 def _check_blockers(lock, info, *, mode, waiters):
+    from .lock import LockError
+
     for path_info in waiters:
         blockers = [
             blocker

dvc/stage/cache.py

@@ -4,17 +4,10 @@ import tempfile
 from contextlib import contextmanager
 
 from funcy import cached_property, first
-from voluptuous import Invalid
 
 from dvc.exceptions import DvcException
 from dvc.path_info import PathInfo
-from dvc.remote.base import _log_exceptions
-from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA
 from dvc.utils import dict_sha256, relpath
-from dvc.utils.serialize import YAMLFileCorruptedError, dump_yaml, load_yaml
-
-from .loader import StageLoader
-from .serialize import to_single_stage_lockfile
 
 logger = logging.getLogger(__name__)
 
@@ -53,6 +46,8 @@ def _can_hash(stage):
 
 
 def _get_stage_hash(stage):
+    from .serialize import to_single_stage_lockfile
+
     assert _can_hash(stage)
     return _get_cache_hash(to_single_stage_lockfile(stage), key=True)
 
@@ -76,6 +71,11 @@ class StageCache:
         return os.path.join(self._get_cache_dir(key), value)
 
     def _load_cache(self, key, value):
+        from voluptuous import Invalid
+
+        from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA
+        from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml
+
         path = self._get_cache_path(key, value)
 
         try:
@@ -105,6 +105,7 @@ class StageCache:
 
     def _create_stage(self, cache, wdir=None):
         from . import PipelineStage, create_stage
+        from .loader import StageLoader
 
         stage = create_stage(
             PipelineStage,
@@ -144,6 +145,8 @@ class StageCache:
                     yield out
 
     def save(self, stage):
+        from .serialize import to_single_stage_lockfile
+
         if not _can_hash(stage):
             return
 
@@ -160,6 +163,9 @@ class StageCache:
         if existing_cache:
             return
 
+        from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA
+        from dvc.utils.serialize import dump_yaml
+
         # sanity check
         COMPILED_LOCK_FILE_STAGE_SCHEMA(cache)
 
@@ -172,6 +178,8 @@ class StageCache:
         self.tree.move(PathInfo(tmp), path)
 
     def restore(self, stage, run_cache=True, pull=False):
+        from .serialize import to_single_stage_lockfile
+
         if not _can_hash(stage):
             raise RunCacheNotFoundError(stage)
 
@@ -224,6 +232,8 @@ class StageCache:
         return ret
 
     def push(self, remote):
+        from dvc.remote.base import _log_exceptions
+
         remote = self.repo.cloud.get_remote(remote)
         return self._transfer(
             _log_exceptions(remote.tree.upload, "upload"),
@@ -232,6 +242,8 @@ class StageCache:
         )
 
     def pull(self, remote):
+        from dvc.remote.base import _log_exceptions
+
         remote = self.repo.cloud.get_remote(remote)
         return self._transfer(
             _log_exceptions(remote.tree.download, "download"),

dvc/state.py

@@ -3,7 +3,6 @@
 import logging
 import os
 import re
-import sqlite3
 from abc import ABC, abstractmethod
 from urllib.parse import urlencode, urlunparse
 
@@ -259,12 +258,14 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
             self.database = _connect_sqlite(self.state_file, {"nolock": 1})
             self.cursor = self.database.cursor()
 
+            from sqlite3 import DatabaseError
+
             # Try loading once to check that the file is indeed a database
             # and reformat it if it is not.
             try:
                 self._prepare_db(empty=empty)
                 return
-            except sqlite3.DatabaseError:
+            except DatabaseError:
                 self.cursor.close()
                 self.database.close()
                 self.database = None
@@ -520,6 +521,8 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
 def _connect_sqlite(filename, options):
     # Connect by URI was added in Python 3.4 and sqlite 3.7.7,
     # we ignore options, which should be fine unless repo is on old NFS/CIFS
+    import sqlite3
+
     if sqlite3.sqlite_version_info < (3, 7, 7):
         return sqlite3.connect(filename)
 

dvc/utils/__init__.py

@@ -12,8 +12,6 @@ import time
 from typing import Optional, Tuple
 
 import colorama
-import nanotime
-from shortuuid import uuid
 
 logger = logging.getLogger(__name__)
 
@@ -230,10 +228,14 @@ def fix_env(env=None):
 
 def tmp_fname(fname):
     """ Temporary name for a partial download """
+    from shortuuid import uuid
+
     return os.fspath(fname) + "." + uuid() + ".tmp"
 
 
 def current_timestamp():
+    import nanotime
+
     return int(nanotime.timestamp(time.time()))
 
 

dvc/utils/fs.py

@@ -5,9 +5,6 @@ import shutil
 import stat
 import sys
 
-import nanotime
-from shortuuid import uuid
-
 from dvc.exceptions import DvcException, FileOwnershipError
 from dvc.system import System
 from dvc.utils import dict_md5
@@ -31,6 +28,7 @@ def get_inode(path):
 
 
 def get_mtime_and_size(path, tree):
+    import nanotime
 
     if tree.isdir(path):
         size = 0
@@ -91,6 +89,7 @@ def move(src, dst, mode=None):
     case src and dst are on different filesystems and actual physical copying
     of data is happening.
     """
+    from shortuuid import uuid
 
     dst = os.path.abspath(dst)
     tmp = f"{dst}.{uuid()}"

dvc/utils/html.py

@@ -1,7 +1,5 @@
 from typing import Dict, List, Optional
 
-import tabulate
-
 PAGE_HTML = """<!DOCTYPE html>
 <html>
 <head>
@@ -27,6 +25,8 @@ class HTML:
         self.elements = []
 
     def with_metrics(self, metrics: Dict[str, Dict]) -> "HTML":
+        import tabulate
+
         header: List[str] = []
         rows: List[List[str]] = []