cli: refactor into a module

By splitting dvc.cli into a module, we'll be able to share cli-related
utilities, split parser-related stuffs into smaller parts, improve
reusability and improve API for command extensions in the future.

dvc/__main__.py

@@ -1,7 +1,7 @@
 """Main entry point for DVC command line tool."""
 import sys
 
-from dvc.main import main
+from dvc.cli import main
 
 if __name__ == "__main__":
     sys.exit(main(sys.argv[1:]))

dvc/cli/__init__.py

@@ -0,0 +1,129 @@
+import logging
+
+# Workaround for CPython bug. See [1] and [2] for more info.
+# [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
+# [2] https://bugs.python.org/issue29288
+
+"".encode("idna")
+
+logger = logging.getLogger("dvc")
+
+
+def parse_args(argv=None):
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        dvc.exceptions.DvcParserError: raised for argument parsing errors.
+    """
+    from .parser import get_main_parser
+
+    parser = get_main_parser()
+    args = parser.parse_args(argv)
+    return args
+
+
+def main(argv=None):  # noqa: C901
+    """Main entry point for dvc CLI.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Returns:
+        int: command's return code.
+    """
+    from dvc._debug import debugtools
+    from dvc.config import ConfigError
+    from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
+    from dvc.logger import FOOTER, disable_other_loggers
+
+    args = None
+    disable_other_loggers()
+
+    outerLogLevel = logger.level
+    try:
+        args = parse_args(argv)
+
+        level = None
+        if args.quiet:
+            level = logging.CRITICAL
+        elif args.verbose == 1:
+            level = logging.DEBUG
+        elif args.verbose > 1:
+            level = logging.TRACE
+
+        if level is not None:
+            logger.setLevel(level)
+
+        logger.trace(args)
+
+        if not args.quiet:
+            from dvc.ui import ui
+
+            ui.enable()
+
+        with debugtools(args):
+            cmd = args.func(args)
+            ret = cmd.do_run()
+    except ConfigError:
+        logger.exception("configuration error")
+        ret = 251
+    except KeyboardInterrupt:
+        logger.exception("interrupted by the user")
+        ret = 252
+    except NotDvcRepoError:
+        logger.exception("")
+        ret = 253
+    except DvcParserError:
+        ret = 254
+    except DvcException:
+        ret = 255
+        logger.exception("")
+    except Exception as exc:  # noqa, pylint: disable=broad-except
+        # pylint: disable=no-member
+        import errno
+
+        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+            from dvc.utils import error_link
+
+            logger.exception(
+                "too many open files, please visit "
+                "{} to see how to handle this "
+                "problem".format(error_link("many-files")),
+                extra={"tb_only": True},
+            )
+        else:
+            from dvc.info import get_dvc_info
+
+            logger.exception("unexpected error")
+
+            dvc_info = get_dvc_info()
+            logger.debug("Version info for developers:\n%s", dvc_info)
+
+            logger.info(FOOTER)
+        ret = 255
+
+    try:
+        from dvc import analytics
+
+        if analytics.is_enabled():
+            analytics.collect_and_send_report(args, ret)
+
+        return ret
+    finally:
+        logger.setLevel(outerLogLevel)
+
+        from dvc.fs.pool import close_pools
+
+        # Closing pools by-hand to prevent weird messages when closing SSH
+        # connections. See https://github.com/iterative/dvc/issues/3248 for
+        # more info.
+        close_pools()
+
+        from dvc.external_repo import clean_repos
+
+        # Remove cached repos in the end of the call, these are anonymous
+        # so won't be reused by any other subsequent run anyway.
+        clean_repos()

dvc/cli/parser.py

@@ -4,8 +4,7 @@ import logging
 import os
 import sys
 
-from ._debug import add_debugging_flags
-from .command import (
+from dvc.command import (
     add,
     cache,
     check_ignore,
@@ -45,8 +44,6 @@ from .command import (
     update,
     version,
 )
-from .command.base import fix_subparsers
-from .exceptions import DvcParserError
 
 logger = logging.getLogger(__name__)
 
@@ -95,6 +92,8 @@ COMMANDS = [
 def _find_parser(parser, cmd_cls):
     defaults = parser._defaults  # pylint: disable=protected-access
     if not cmd_cls or cmd_cls == defaults.get("func"):
+        from dvc.exceptions import DvcParserError
+
         parser.print_help()
         raise DvcParserError()
 
@@ -143,6 +142,8 @@ def get_parent_parser():
     When overwriting `-q` or `-v`, you need to instantiate a new object
     in order to prevent some weird behavior.
     """
+    from dvc._debug import add_debugging_flags
+
     parent_parser = argparse.ArgumentParser(add_help=False)
     add_debugging_flags(parent_parser)
     log_level_group = parent_parser.add_mutually_exclusive_group()
@@ -207,23 +208,11 @@ def get_main_parser():
         help="Use `dvc COMMAND --help` for command-specific help.",
     )
 
+    from dvc.command.base import fix_subparsers
+
     fix_subparsers(subparsers)
 
     for cmd in COMMANDS:
         cmd.add_parser(subparsers, parent_parser)
 
     return parser
-
-
-def parse_args(argv=None):
-    """Parses CLI arguments.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Raises:
-        dvc.exceptions.DvcParserError: raised for argument parsing errors.
-    """
-    parser = get_main_parser()
-    args = parser.parse_args(argv)
-    return args

dvc/command/completion.py

@@ -82,7 +82,7 @@ EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
 
 class CmdCompletion(CmdBaseNoRepo):
     def run(self):
-        from dvc.cli import get_main_parser
+        from dvc.cli.parser import get_main_parser
 
         parser = get_main_parser()
         shell = self.args.shell

dvc/command/data_sync.py

@@ -97,7 +97,7 @@ class CmdDataFetch(CmdDataBase):
 
 
 def shared_parent_parser():
-    from dvc.cli import get_parent_parser
+    from dvc.cli.parser import get_parent_parser
 
     # Parent parser used in pull/push/status
     parent_parser = argparse.ArgumentParser(

dvc/command/git_hook.py

@@ -25,7 +25,7 @@ class CmdHookBase(CmdBaseNoRepo):
 
 class CmdPreCommit(CmdHookBase):
     def _run(self):
-        from dvc.main import main
+        from dvc.cli import main
 
         return main(["status"])
 
@@ -49,14 +49,14 @@ class CmdPostCheckout(CmdHookBase):
         if os.path.isdir(os.path.join(".git", "rebase-merge")):
             return 0
 
-        from dvc.main import main
+        from dvc.cli import main
 
         return main(["checkout"])
 
 
 class CmdPrePush(CmdHookBase):
     def _run(self):
-        from dvc.main import main
+        from dvc.cli import main
 
         return main(["push"])
 

dvc/daemon.py

@@ -42,7 +42,7 @@ def _spawn_windows(cmd, env):
 
 
 def _spawn_posix(cmd, env):
-    from dvc.main import main
+    from dvc.cli import main
 
     # NOTE: using os._exit instead of sys.exit, because dvc built
     # with PyInstaller has trouble with SystemExit exception and throws

dvc/main.py

@@ -1,115 +1,2 @@
-"""Main entry point for dvc CLI."""
-import logging
-
-from dvc._debug import debugtools
-from dvc.cli import parse_args
-from dvc.config import ConfigError
-from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
-from dvc.logger import FOOTER, disable_other_loggers
-
-# Workaround for CPython bug. See [1] and [2] for more info.
-# [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
-# [2] https://bugs.python.org/issue29288
-
-"".encode("idna")
-
-logger = logging.getLogger("dvc")
-
-
-def main(argv=None):  # noqa: C901
-    """Run dvc CLI command.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Returns:
-        int: command's return code.
-    """
-    args = None
-    disable_other_loggers()
-
-    outerLogLevel = logger.level
-    try:
-        args = parse_args(argv)
-
-        level = None
-        if args.quiet:
-            level = logging.CRITICAL
-        elif args.verbose == 1:
-            level = logging.DEBUG
-        elif args.verbose > 1:
-            level = logging.TRACE
-
-        if level is not None:
-            logger.setLevel(level)
-
-        logger.trace(args)
-
-        if not args.quiet:
-            from dvc.ui import ui
-
-            ui.enable()
-
-        with debugtools(args):
-            cmd = args.func(args)
-            ret = cmd.do_run()
-    except ConfigError:
-        logger.exception("configuration error")
-        ret = 251
-    except KeyboardInterrupt:
-        logger.exception("interrupted by the user")
-        ret = 252
-    except NotDvcRepoError:
-        logger.exception("")
-        ret = 253
-    except DvcParserError:
-        ret = 254
-    except DvcException:
-        ret = 255
-        logger.exception("")
-    except Exception as exc:  # noqa, pylint: disable=broad-except
-        # pylint: disable=no-member
-        import errno
-
-        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
-            from dvc.utils import error_link
-
-            logger.exception(
-                "too many open files, please visit "
-                "{} to see how to handle this "
-                "problem".format(error_link("many-files")),
-                extra={"tb_only": True},
-            )
-        else:
-            from dvc.info import get_dvc_info
-
-            logger.exception("unexpected error")
-
-            dvc_info = get_dvc_info()
-            logger.debug("Version info for developers:\n%s", dvc_info)
-
-            logger.info(FOOTER)
-        ret = 255
-
-    try:
-        from dvc import analytics
-
-        if analytics.is_enabled():
-            analytics.collect_and_send_report(args, ret)
-
-        return ret
-    finally:
-        logger.setLevel(outerLogLevel)
-
-        from dvc.fs.pool import close_pools
-
-        # Closing pools by-hand to prevent weird messages when closing SSH
-        # connections. See https://github.com/iterative/dvc/issues/3248 for
-        # more info.
-        close_pools()
-
-        from dvc.external_repo import clean_repos
-
-        # Remove cached repos in the end of the call, these are anonymous
-        # so won't be reused by any other subsequent run anyway.
-        clean_repos()
+# TODO: remove in 3.0, kept for backward compatibility
+from .cli import main  # noqa: F401, pylint: disable=unused-import

dvc/repo/experiments/executor/base.py

@@ -695,7 +695,7 @@ class BaseExecutor(ABC):
         from dvc.logger import disable_other_loggers
 
         # When executor.reproduce is run in a multiprocessing child process,
-        # dvc.main will not be called for that child process so we need to
+        # dvc.cli.main will not be called for that child process so we need to
         # setup logging ourselves
         dvc_logger = logging.getLogger("dvc")
         disable_other_loggers()

setup.cfg

@@ -171,7 +171,7 @@ exclude =
 
 [options.entry_points]
 console_scripts =
-    dvc = dvc.main:main
+    dvc = dvc.cli:main
 
 [flake8]
 ignore=

tests/func/experiments/test_experiments.py

@@ -589,7 +589,7 @@ def test_queue(tmp_dir, scm, dvc, exp_stage, mocker):
 
 
 def test_run_metrics(tmp_dir, scm, dvc, exp_stage, mocker):
-    from dvc.main import main
+    from dvc.cli import main
 
     mocker.patch.object(
         dvc.experiments, "run", return_value={"abc123": "abc123"}

tests/func/experiments/test_init.py

@@ -3,9 +3,9 @@ import os
 
 import pytest
 
+from dvc.cli import main
 from dvc.command.experiments.init import CmdExperimentsInit
 from dvc.exceptions import DvcException
-from dvc.main import main
 from dvc.repo.experiments.init import init
 from dvc.stage.exceptions import DuplicateStageName
 

tests/func/experiments/test_show.py

@@ -5,8 +5,8 @@ from datetime import datetime
 import pytest
 from funcy import first, get_in
 
+from dvc.cli import main
 from dvc.exceptions import InvalidArgumentError
-from dvc.main import main
 from dvc.repo.experiments.base import EXPS_STASH, ExpRefInfo
 from dvc.repo.experiments.executor.base import (
     EXEC_PID_DIR,

tests/func/machine/test_machine_config.py

@@ -4,7 +4,7 @@ import textwrap
 import pytest
 import tpi
 
-from dvc.main import main
+from dvc.cli import main
 from dvc.ui import ui
 from tests.utils import console_width
 

tests/func/machine/test_machine_status.py

@@ -1,4 +1,4 @@
-from dvc.main import main
+from dvc.cli import main
 from dvc.ui import ui
 from tests.utils import console_width
 

tests/func/metrics/test_diff.py

@@ -1,6 +1,6 @@
 import json
 
-from dvc.main import main
+from dvc.cli import main
 
 
 def test_metrics_diff_simple(tmp_dir, scm, dvc, run_copy_metrics):

tests/func/plots/test_show.py

@@ -3,9 +3,9 @@ import os
 import pytest
 from funcy import get_in
 
+from dvc.cli import main
 from dvc.dvcfile import PIPELINE_FILE
 from dvc.exceptions import OverlappingOutputPathsError
-from dvc.main import main
 from dvc.repo import Repo
 from dvc.repo.plots import PlotMetricTypeError
 from dvc.utils import onerror_collect

tests/func/test_add.py

@@ -11,6 +11,7 @@ import colorama
 import pytest
 
 import dvc as dvc_module
+from dvc.cli import main
 from dvc.data.db import ODBManager
 from dvc.dvcfile import DVC_FILE_SUFFIX
 from dvc.exceptions import (
@@ -22,7 +23,6 @@ from dvc.exceptions import (
 )
 from dvc.fs.local import LocalFileSystem
 from dvc.hash_info import HashInfo
-from dvc.main import main
 from dvc.output import (
     OutputAlreadyTrackedError,
     OutputDoesNotExistError,

tests/func/test_analytics.py

@@ -4,7 +4,7 @@ from unittest.mock import call
 import pytest
 
 from dvc.analytics import _scm_in_use, collect_and_send_report
-from dvc.main import main
+from dvc.cli import main
 from dvc.repo import Repo
 from tests.utils import ANY
 

tests/func/test_check_ignore.py

@@ -2,8 +2,8 @@ import os
 
 import pytest
 
+from dvc.cli import main
 from dvc.ignore import DvcIgnore
-from dvc.main import main
 
 
 @pytest.mark.parametrize(

tests/func/test_checkout.py

@@ -9,6 +9,7 @@ from unittest.mock import patch
 
 import pytest
 
+from dvc.cli import main
 from dvc.dvcfile import DVC_FILE_SUFFIX, PIPELINE_FILE, Dvcfile
 from dvc.exceptions import (
     CheckoutError,
@@ -18,7 +19,6 @@ from dvc.exceptions import (
     NoOutputOrStageError,
 )
 from dvc.fs.local import LocalFileSystem
-from dvc.main import main
 from dvc.stage import Stage
 from dvc.stage.exceptions import StageFileDoesNotExistError
 from dvc.system import System

tests/func/test_config.py

@@ -3,8 +3,8 @@ import textwrap
 
 import pytest
 
+from dvc.cli import main
 from dvc.config import Config, ConfigError
-from dvc.main import main
 
 
 def test_config_set(tmp_dir, dvc):

tests/func/test_data_cloud.py

@@ -6,8 +6,8 @@ import pytest
 from flaky.flaky_decorator import flaky
 
 import dvc as dvc_module
+from dvc.cli import main
 from dvc.external_repo import clean_repos
-from dvc.main import main
 from dvc.stage.exceptions import StageNotFound
 from dvc.testing.test_remote import (  # noqa, pylint: disable=unused-import
     TestRemote,

tests/func/test_gc.py

@@ -6,10 +6,10 @@ import configobj
 import pytest
 from git import Repo
 
+from dvc.cli import main
 from dvc.data.db.local import LocalObjectDB
 from dvc.exceptions import CollectCacheError
 from dvc.fs.local import LocalFileSystem
-from dvc.main import main
 from dvc.repo import Repo as DvcRepo
 from dvc.utils.fs import remove
 from tests.basic_env import TestDir, TestDvcGit

tests/func/test_get.py

@@ -3,8 +3,8 @@ import os
 
 import pytest
 
+from dvc.cli import main
 from dvc.data.db import ODBManager
-from dvc.main import main
 from dvc.repo import Repo
 from dvc.repo.get import GetDVCFileError
 from dvc.system import System

tests/func/test_import_url.py

@@ -5,9 +5,9 @@ from uuid import uuid4
 
 import pytest
 
+from dvc.cli import main
 from dvc.dependency.base import DependencyDoesNotExistError
 from dvc.exceptions import InvalidArgumentError
-from dvc.main import main
 from dvc.stage import Stage
 from dvc.testing.test_workspace import TestImport as _TestImport
 from dvc.utils.fs import makedirs

tests/func/test_init.py

@@ -3,9 +3,9 @@ import os
 
 import pytest
 
+from dvc.cli import main
 from dvc.config import Config
 from dvc.exceptions import InitError
-from dvc.main import main
 from dvc.repo import Repo as DvcRepo
 
 

tests/func/test_lock.py

@@ -1,7 +1,7 @@
 import pytest
 
+from dvc.cli import main
 from dvc.lock import Lock, LockError
-from dvc.main import main
 
 
 def test_with(tmp_dir, dvc, mocker):

tests/func/test_merge_driver.py

@@ -2,7 +2,7 @@ import os
 
 import pytest
 
-from dvc.main import main
+from dvc.cli import main
 from dvc.utils.fs import remove
 
 

tests/func/test_move.py

@@ -3,9 +3,9 @@ import textwrap
 
 import pytest
 
+from dvc.cli import main
 from dvc.dvcfile import DVC_FILE_SUFFIX
 from dvc.exceptions import DvcException, MoveNotDataSourceError
-from dvc.main import main
 from dvc.utils.serialize import load_yaml
 from tests.basic_env import TestDvc, TestDvcGit
 from tests.func.test_repro import TestRepro

tests/func/test_odb.py

@@ -4,9 +4,9 @@ import stat
 import configobj
 import pytest
 
+from dvc.cli import main
 from dvc.data.db import ODBManager
 from dvc.hash_info import HashInfo
-from dvc.main import main
 from dvc.objects.errors import ObjectFormatError
 from dvc.utils import relpath
 from tests.basic_env import TestDir, TestDvc

tests/func/test_remote.py

@@ -6,9 +6,9 @@ from unittest.mock import patch
 import configobj
 import pytest
 
+from dvc.cli import main
 from dvc.config import Config
 from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError
-from dvc.main import main
 from dvc.utils.fs import remove
 from tests.basic_env import TestDvc
 

tests/func/test_remove.py

@@ -2,8 +2,8 @@ import os
 
 import pytest
 
+from dvc.cli import main
 from dvc.exceptions import DvcException
-from dvc.main import main
 from dvc.stage.exceptions import StageFileDoesNotExistError
 from dvc.system import System
 from dvc.utils.fs import remove

tests/func/test_repro.py

@@ -7,6 +7,7 @@ from unittest.mock import patch
 
 import pytest
 
+from dvc.cli import main
 from dvc.dvcfile import DVC_FILE, Dvcfile
 from dvc.exceptions import (
     CyclicGraphError,
@@ -14,7 +15,6 @@ from dvc.exceptions import (
     StagePathAsOutputError,
 )
 from dvc.fs.local import LocalFileSystem
-from dvc.main import main
 from dvc.output import Output
 from dvc.stage import Stage
 from dvc.stage.exceptions import StageFileDoesNotExistError

tests/func/test_repro_multistage.py

@@ -5,9 +5,9 @@ from textwrap import dedent
 import pytest
 from funcy import lsplit
 
+from dvc.cli import main
 from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
 from dvc.exceptions import CyclicGraphError, ReproductionError
-from dvc.main import main
 from dvc.stage import PipelineStage
 from tests.func import test_repro
 from tests.utils.scriptify import scriptify

tests/func/test_root.py

@@ -1,4 +1,4 @@
-from dvc.main import main
+from dvc.cli import main
 
 
 def test_root(tmp_dir, dvc, capsys):

tests/func/test_run_single_stage.py

@@ -8,6 +8,7 @@ from unittest import mock
 
 import pytest
 
+from dvc.cli import main
 from dvc.dependency.base import DependencyIsStageFileError
 from dvc.dvcfile import DVC_FILE_SUFFIX
 from dvc.exceptions import (
@@ -18,7 +19,6 @@ from dvc.exceptions import (
     OverlappingOutputPathsError,
     StagePathAsOutputError,
 )
-from dvc.main import main
 from dvc.output import Output, OutputIsStageFileError
 from dvc.repo import Repo as DvcRepo
 from dvc.stage import Stage

tests/func/test_stage.py

@@ -3,9 +3,9 @@ import tempfile
 
 import pytest
 
+from dvc.cli import main
 from dvc.dvcfile import SingleStageFile
 from dvc.fs.local import LocalFileSystem
-from dvc.main import main
 from dvc.output import Output
 from dvc.repo import Repo, lock_repo
 from dvc.stage import PipelineStage, Stage

tests/func/test_status.py

@@ -1,6 +1,6 @@
 import os
 
-from dvc.main import main
+from dvc.cli import main
 
 
 def test_quiet(tmp_dir, dvc, capsys):

tests/func/test_unprotect.py

@@ -1,6 +1,6 @@
 import os
 
-from dvc.main import main
+from dvc.cli import main
 from tests.basic_env import TestDvc
 
 

tests/func/test_version.py

@@ -1,6 +1,6 @@
 import re
 
-from dvc.main import main
+from dvc.cli import main
 from tests.unit.test_info import (
     DVC_VERSION_REGEX,
     PYTHON_VERSION_REGEX,

tests/func/utils/test_strict_yaml.py

@@ -3,7 +3,7 @@ import os
 import pytest
 from ruamel.yaml import __with_libyaml__ as ruamel_clib
 
-from dvc.main import main
+from dvc.cli import main
 from dvc.ui import ui
 
 DUPLICATE_KEYS = """\

tests/unit/command/test_cache.py

@@ -1,7 +1,7 @@
 import os
 import textwrap
 
-from dvc.main import main
+from dvc.cli import main
 
 
 def test_cache_dir_local(tmp_dir, dvc, capsys, caplog):

tests/unit/command/test_config.py

@@ -1,7 +1,8 @@
 import pytest
 
-from dvc.cli import DvcParserError, parse_args
+from dvc.cli import parse_args
 from dvc.command.config import CmdConfig
+from dvc.exceptions import DvcParserError
 
 
 def test_config_formatter():

tests/unit/command/test_git_hook.py

@@ -16,6 +16,6 @@ def test_out_of_repo(tmp_dir, hook, cls, mocker):
     cli_args = parse_args(["git-hook", hook])
     assert cli_args.func == cls
     cmd = cli_args.func(cli_args)
-    mock_main = mocker.patch("dvc.main.main")
+    mock_main = mocker.patch("dvc.cli.main")
     assert cmd.run() == 0
     assert not mock_main.called

tests/unit/test_imports.py

@@ -14,7 +14,7 @@ def test_no_remote_imports():
         "pyarrow",
     }
 
-    code = "import dvc.main, sys; print(' '.join(sys.modules))"
+    code = "import dvc.cli, sys; print(' '.join(sys.modules))"
     res = subprocess.run(
         [sys.executable, "-c", code], stdout=subprocess.PIPE, check=True
     )