From 8b16af965c563dfa60596c4d198db9c1cfc48c9f Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Sun, 13 Sep 2020 09:08:20 +0800 Subject: [PATCH] Print dvc version info for debugging (#4536) * Print dvc version info for debugging fix #4095 1. seperate info from dvc version. 2. print dvc version info when some unexpected error occured. * Psutil NoneType error * Update tests/func/test_version.py Co-authored-by: Ruslan Kuprieiev * Move `package` outside dvc/utils/pkg.py Co-authored-by: Ruslan Kuprieiev --- dvc/command/version.py | 141 +---------------------- dvc/info.py | 136 ++++++++++++++++++++++ dvc/main.py | 4 + tests/func/test_version.py | 63 +--------- tests/unit/remote/ssh/test_connection.py | 5 +- tests/unit/test_info.py | 74 ++++++++++++ 6 files changed, 221 insertions(+), 202 deletions(-) create mode 100644 dvc/info.py create mode 100644 tests/unit/test_info.py diff --git a/dvc/command/version.py b/dvc/command/version.py index aa37106c0..3396e3443 100644 --- a/dvc/command/version.py +++ b/dvc/command/version.py @@ -1,153 +1,18 @@ import argparse -import itertools import logging -import os -import pathlib -import platform -import uuid from dvc.command.base import CmdBaseNoRepo, append_doc_link -from dvc.exceptions import DvcException, NotDvcRepoError -from dvc.scm.base import SCMError -from dvc.system import System -from dvc.utils import error_link -from dvc.utils.pkg import PKG -from dvc.version import __version__ - -try: - import psutil -except ImportError: - psutil = None - +from dvc.info import get_dvc_info logger = logging.getLogger(__name__) class CmdVersion(CmdBaseNoRepo): def run(self): - from dvc.repo import Repo - - package = PKG - if PKG is None: - package = "" - else: - package = f"({PKG})" - - info = [ - f"DVC version: {__version__} {package}", - "---------------------------------", - f"Platform: Python {platform.python_version()} on " - f"{platform.platform()}", - f"Supports: {self.get_supported_remotes()}", - ] - - try: - repo = Repo() - root_directory = repo.root_dir - - # cache_dir might not exist yet (e.g. after `dvc init`), and we - # can't auto-create it, as it might cause issues if the user - # later decides to enable shared cache mode with - # `dvc config cache.shared group`. - if os.path.exists(repo.cache.local.cache_dir): - info.append( - "Cache types: {}".format( - self.get_linktype_support_info(repo) - ) - ) - if psutil: - fs_type = self.get_fs_type(repo.cache.local.cache_dir) - info.append(f"Cache directory: {fs_type}") - else: - info.append("Cache types: " + error_link("no-dvc-cache")) - - except NotDvcRepoError: - root_directory = os.getcwd() - except SCMError: - root_directory = os.getcwd() - info.append("Repo: dvc, git (broken)") - else: - if psutil: - fs_root = self.get_fs_type(os.path.abspath(root_directory)) - info.append(f"Workspace directory: {fs_root}") - - info.append("Repo: {}".format(_get_dvc_repo_info(repo))) - - logger.info("\n".join(info)) + dvc_info = get_dvc_info() + logger.info(dvc_info) return 0 - @staticmethod - def get_fs_type(path): - partition = { - pathlib.Path(part.mountpoint): (part.fstype + " on " + part.device) - for part in psutil.disk_partitions(all=True) - } - - path = pathlib.Path(path) - - for parent in itertools.chain([path], path.parents): - if parent in partition: - return partition[parent] - return ("unknown", "none") - - @staticmethod - def get_linktype_support_info(repo): - links = { - "reflink": (System.reflink, None), - "hardlink": (System.hardlink, System.is_hardlink), - "symlink": (System.symlink, System.is_symlink), - } - - fname = "." + str(uuid.uuid4()) - src = os.path.join(repo.cache.local.cache_dir, fname) - open(src, "w").close() - dst = os.path.join(repo.root_dir, fname) - - cache = [] - - for name, (link, is_link) in links.items(): - try: - link(src, dst) - status = "supported" - if is_link and not is_link(dst): - status = "broken" - os.unlink(dst) - except DvcException: - status = "not supported" - - if status == "supported": - cache.append(name) - os.remove(src) - - return ", ".join(cache) - - @staticmethod - def get_supported_remotes(): - from dvc.tree import TREES - - supported_remotes = [] - for tree_cls in TREES: - if not tree_cls.get_missing_deps(): - supported_remotes.append(tree_cls.scheme) - - if len(supported_remotes) == len(TREES): - return "All remotes" - - if len(supported_remotes) == 1: - return supported_remotes - - return ", ".join(supported_remotes) - - -def _get_dvc_repo_info(repo): - if repo.config.get("core", {}).get("no_scm", False): - return "dvc (no_scm)" - - if repo.root_dir != repo.scm.root_dir: - return "dvc (subdir), git" - - return "dvc, git" - def add_parser(subparsers, parent_parser): VERSION_HELP = ( diff --git a/dvc/info.py b/dvc/info.py new file mode 100644 index 000000000..188d42250 --- /dev/null +++ b/dvc/info.py @@ -0,0 +1,136 @@ +import itertools +import os +import pathlib +import platform +import uuid + +from dvc.exceptions import DvcException, NotDvcRepoError +from dvc.repo import Repo +from dvc.scm.base import SCMError +from dvc.system import System +from dvc.tree import TREES +from dvc.utils import error_link +from dvc.utils.pkg import PKG +from dvc.version import __version__ + +try: + import psutil +except ImportError: + psutil = None + +if PKG is None: + package = "" +else: + package = f"({PKG})" + + +def get_dvc_info(): + info = [ + f"DVC version: {__version__} {package}", + "---------------------------------", + f"Platform: Python {platform.python_version()} on " + f"{platform.platform()}", + f"Supports: {_get_supported_remotes()}", + ] + + try: + repo = Repo() + + # cache_dir might not exist yet (e.g. after `dvc init`), and we + # can't auto-create it, as it might cause issues if the user + # later decides to enable shared cache mode with + # `dvc config cache.shared group`. + if os.path.exists(repo.cache.local.cache_dir): + info.append( + "Cache types: {}".format(_get_linktype_support_info(repo)) + ) + if psutil: + fs_type = get_fs_type(repo.cache.local.cache_dir) + info.append(f"Cache directory: {fs_type}") + else: + info.append("Cache types: " + error_link("no-dvc-cache")) + + except NotDvcRepoError: + pass + except SCMError: + info.append("Repo: dvc, git (broken)") + else: + root_directory = repo.root_dir + if psutil: + fs_root = get_fs_type(os.path.abspath(root_directory)) + info.append(f"Workspace directory: {fs_root}") + info.append("Repo: {}".format(_get_dvc_repo_info(repo))) + return "\n".join(info) + + +def _get_linktype_support_info(repo): + + links = { + "reflink": (System.reflink, None), + "hardlink": (System.hardlink, System.is_hardlink), + "symlink": (System.symlink, System.is_symlink), + } + + fname = "." + str(uuid.uuid4()) + src = os.path.join(repo.cache.local.cache_dir, fname) + open(src, "w").close() + dst = os.path.join(repo.root_dir, fname) + + cache = [] + + for name, (link, is_link) in links.items(): + try: + link(src, dst) + status = "supported" + if is_link and not is_link(dst): + status = "broken" + os.unlink(dst) + except DvcException: + status = "not supported" + + if status == "supported": + cache.append(name) + os.remove(src) + + return ", ".join(cache) + + +def _get_supported_remotes(): + + supported_remotes = [] + for tree_cls in TREES: + if not tree_cls.get_missing_deps(): + supported_remotes.append(tree_cls.scheme) + + if len(supported_remotes) == len(TREES): + return "All remotes" + + if len(supported_remotes) == 1: + return supported_remotes + + return ", ".join(supported_remotes) + + +def get_fs_type(path): + + partition = { + pathlib.Path(part.mountpoint): (part.fstype + " on " + part.device) + for part in psutil.disk_partitions(all=True) + } + + path = pathlib.Path(path) + + for parent in itertools.chain([path], path.parents): + if parent in partition: + return partition[parent] + return ("unknown", "none") + + +def _get_dvc_repo_info(self): + if self.config.get("core", {}).get("no_scm", False): + return "dvc (no_scm)" + + if self.root_dir != self.scm.root_dir: + return "dvc (subdir), git" + + return "dvc, git" diff --git a/dvc/main.py b/dvc/main.py index b74488afb..4bf803005 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -98,8 +98,12 @@ def main(argv=None): # noqa: C901 extra={"tb_only": True}, ) else: + from dvc.info import get_dvc_info + logger.exception("unexpected error") logger.info(FOOTER) + dvc_info = get_dvc_info() + logger.info(dvc_info) ret = 255 try: diff --git a/tests/func/test_version.py b/tests/func/test_version.py index 4c2ad5caa..ebeeaafc5 100644 --- a/tests/func/test_version.py +++ b/tests/func/test_version.py @@ -1,72 +1,13 @@ -import os import re -import shutil -import pytest - -from dvc.command.version import psutil from dvc.main import main -@pytest.mark.parametrize("scm_init", [True, False]) -def test_info_in_repo(scm_init, tmp_dir, caplog): - tmp_dir.init(scm=scm_init, dvc=True) - # Create `.dvc/cache`, that is needed to check supported link types. - os.mkdir(tmp_dir.dvc.cache.local.cache_dir) - +def test_(tmp_dir, dvc, scm, caplog): assert main(["version"]) == 0 assert re.search(r"DVC version: \d+\.\d+\.\d+.*", caplog.text) assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", caplog.text) assert re.search(r"Supports: .*", caplog.text) assert re.search(r"Cache types: .*", caplog.text) - - if scm_init: - assert "Repo: dvc, git" in caplog.text - else: - assert "Repo: dvc (no_scm)" in caplog.text - - -def test_info_in_subdir(tmp_dir, scm, caplog): - dvc_subdir = tmp_dir / "subdir" - dvc_subdir.mkdir() - - with dvc_subdir.chdir(): - dvc_subdir.init(scm=False, dvc=True) - with dvc_subdir.dvc.config.edit() as conf: - del conf["core"]["no_scm"] - - assert main(["version"]) == 0 - - assert "Repo: dvc (subdir), git" in caplog.text - - -def test_info_in_broken_git_repo(tmp_dir, dvc, scm, caplog): - shutil.rmtree(dvc.scm.dir) - assert main(["version"]) == 0 - - assert "Repo: dvc, git (broken)" in caplog.text - - -@pytest.mark.skipif(psutil is None, reason="No psutil.") -def test_fs_info_in_repo(tmp_dir, dvc, caplog): - os.mkdir(dvc.cache.local.cache_dir) - assert main(["version"]) == 0 - - assert re.search(r"Cache directory: .* on .*", caplog.text) - assert re.search(r"Workspace directory: .* on .*", caplog.text) - - -def test_info_outside_of_repo(tmp_dir, caplog): - assert main(["version"]) == 0 - - assert re.search(r"DVC version: \d+\.\d+\.\d+.*", caplog.text) - assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", caplog.text) - assert re.search(r"Supports: .*", caplog.text) - assert not re.search(r"Cache types: .*", caplog.text) - assert "Repo:" not in caplog.text - - -@pytest.mark.skipif(psutil is None, reason="No psutil.") -def test_fs_info_outside_of_repo(tmp_dir, caplog): - assert main(["version"]) == 0 + assert "Repo: dvc, git" in caplog.text diff --git a/tests/unit/remote/ssh/test_connection.py b/tests/unit/remote/ssh/test_connection.py index 10499a231..a84a22924 100644 --- a/tests/unit/remote/ssh/test_connection.py +++ b/tests/unit/remote/ssh/test_connection.py @@ -6,7 +6,7 @@ import tempfile import pytest -from dvc.command.version import CmdVersion +from dvc.info import get_fs_type from dvc.system import System here = os.path.abspath(os.path.dirname(__file__)) @@ -85,8 +85,7 @@ def test_walk(tmp_path, ssh_connection): @pytest.mark.skipif( - CmdVersion.get_fs_type(tempfile.gettempdir())[0] - not in ["xfs", "apfs", "btrfs"], + get_fs_type(tempfile.gettempdir())[0] not in ["xfs", "apfs", "btrfs"], reason="Reflinks only work in specified file systems", ) def test_reflink(tmp_dir, ssh_connection): diff --git a/tests/unit/test_info.py b/tests/unit/test_info.py new file mode 100644 index 000000000..fe5dc2fb0 --- /dev/null +++ b/tests/unit/test_info.py @@ -0,0 +1,74 @@ +import os +import re +import shutil + +import pytest + +from dvc.info import get_dvc_info, psutil + + +@pytest.mark.parametrize("scm_init", [True, False]) +def test_info_in_repo(scm_init, tmp_dir): + tmp_dir.init(scm=scm_init, dvc=True) + # Create `.dvc/cache`, that is needed to check supported link types. + os.mkdir(tmp_dir.dvc.cache.local.cache_dir) + + dvc_info = get_dvc_info() + + assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info) + assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", dvc_info) + assert re.search(r"Supports: .*", dvc_info) + assert re.search(r"Cache types: .*", dvc_info) + + if scm_init: + assert "Repo: dvc, git" in dvc_info + else: + assert "Repo: dvc (no_scm)" in dvc_info + + +def test_info_in_subdir(tmp_dir, scm, caplog): + dvc_subdir = tmp_dir / "subdir" + dvc_subdir.mkdir() + + with dvc_subdir.chdir(): + dvc_subdir.init(scm=False, dvc=True) + with dvc_subdir.dvc.config.edit() as conf: + del conf["core"]["no_scm"] + + dvc_info = get_dvc_info() + + assert "Repo: dvc (subdir), git" in dvc_info + + +def test_info_in_broken_git_repo(tmp_dir, dvc, scm, caplog): + shutil.rmtree(dvc.scm.dir) + dvc_info = get_dvc_info() + + assert "Repo: dvc, git (broken)" in dvc_info + + +@pytest.mark.skipif(psutil is None, reason="No psutil.") +def test_fs_info_in_repo(tmp_dir, dvc, caplog): + os.mkdir(dvc.cache.local.cache_dir) + dvc_info = get_dvc_info() + + assert re.search(r"Cache directory: .* on .*", dvc_info) + assert re.search(r"Workspace directory: .* on .*", dvc_info) + + +def test_info_outside_of_repo(tmp_dir, caplog): + dvc_info = get_dvc_info() + + assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info) + assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", dvc_info) + assert re.search(r"Supports: .*", dvc_info) + assert not re.search(r"Cache types: .*", dvc_info) + assert "Repo:" not in dvc_info + + +@pytest.mark.skipif(psutil is None, reason="No psutil.") +def test_fs_info_outside_of_repo(tmp_dir, caplog): + dvc_info = get_dvc_info() + assert re.search(r"DVC version: \d+\.\d+\.\d+.*", dvc_info) + assert re.search(r"Platform: Python \d\.\d+\.\d+ on .*", dvc_info) + assert re.search(r"Supports: .*", dvc_info)