diff --git a/dvc/analytics.py b/dvc/analytics.py deleted file mode 100644 index 8b39fc31b..000000000 --- a/dvc/analytics.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Collect and send usage analytics""" -from __future__ import unicode_literals - -import errno -import json -import logging -import os - -from dvc import __version__ -from dvc.utils import env2bool -from dvc.utils.compat import str - - -logger = logging.getLogger(__name__) - - -class Analytics(object): - """Class for collecting and sending usage analytics. - - Args: - info (dict): optional existing analytics report. - """ - - URL = "https://analytics.dvc.org" - TIMEOUT_POST = 5 - - USER_ID_FILE = "user_id" - - PARAM_DVC_VERSION = "dvc_version" - PARAM_USER_ID = "user_id" - PARAM_SYSTEM_INFO = "system_info" - - PARAM_OS = "os" - - PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major" - PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor" - PARAM_WINDOWS_VERSION_BUILD = "windows_version_build" - PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack" - - PARAM_MAC_VERSION = "mac_version" - - PARAM_LINUX_DISTRO = "linux_distro" - PARAM_LINUX_DISTRO_VERSION = "linux_distro_version" - PARAM_LINUX_DISTRO_LIKE = "linux_distro_like" - - PARAM_SCM_CLASS = "scm_class" - PARAM_IS_BINARY = "is_binary" - PARAM_CMD_CLASS = "cmd_class" - PARAM_CMD_RETURN_CODE = "cmd_return_code" - - def __init__(self, info=None): - from dvc.config import Config - from dvc.lock import Lock - - if info is None: - info = {} - - self.info = info - - cdir = Config.get_global_config_dir() - try: - os.makedirs(cdir) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - - self.user_id_file = os.path.join(cdir, self.USER_ID_FILE) - self.user_id_file_lock = Lock(self.user_id_file + ".lock") - - @staticmethod - def load(path): - """Loads analytics report from json file specified by path. - - Args: - path (str): path to json file with analytics report. - """ - with open(path, "r") as fobj: - analytics = Analytics(info=json.load(fobj)) - os.unlink(path) - return analytics - - def _write_user_id(self): - import uuid - - with open(self.user_id_file, "w+") as fobj: - user_id = str(uuid.uuid4()) - info = {self.PARAM_USER_ID: user_id} - json.dump(info, fobj) - return user_id - - def _read_user_id(self): - if not os.path.exists(self.user_id_file): - return None - - with open(self.user_id_file, "r") as fobj: - try: - info = json.load(fobj) - except ValueError as exc: - logger.debug("Failed to load user_id: {}".format(exc)) - return None - - return info[self.PARAM_USER_ID] - - def _get_user_id(self): - from dvc.lock import LockError - - try: - with self.user_id_file_lock: - user_id = self._read_user_id() - if user_id is None: - user_id = self._write_user_id() - return user_id - except LockError: - msg = "Failed to acquire '{}'" - logger.debug(msg.format(self.user_id_file_lock.lockfile)) - - def _collect_windows(self): - import sys - - version = sys.getwindowsversion() # pylint: disable=no-member - info = {} - info[self.PARAM_OS] = "windows" - info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major - info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor - info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build - info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack - return info - - def _collect_darwin(self): - import platform - - info = {} - info[self.PARAM_OS] = "mac" - info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0] - return info - - def _collect_linux(self): - import distro - - info = {} - info[self.PARAM_OS] = "linux" - info[self.PARAM_LINUX_DISTRO] = distro.id() - info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version() - info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like() - return info - - def _collect_system_info(self): - import platform - - system = platform.system() - - if system == "Windows": - return self._collect_windows() - - if system == "Darwin": - return self._collect_darwin() - - if system == "Linux": - return self._collect_linux() - - raise NotImplementedError - - def collect(self): - """Collect analytics report.""" - from dvc.scm import SCM - from dvc.utils import is_binary - from dvc.repo import Repo - from dvc.exceptions import NotDvcRepoError - - self.info[self.PARAM_DVC_VERSION] = __version__ - self.info[self.PARAM_IS_BINARY] = is_binary() - self.info[self.PARAM_USER_ID] = self._get_user_id() - - self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info() - - try: - scm = SCM(root_dir=Repo.find_root()) - self.info[self.PARAM_SCM_CLASS] = type(scm).__name__ - except NotDvcRepoError: - pass - - def collect_cmd(self, args, ret): - """Collect analytics info from a CLI command.""" - from dvc.command.daemon import CmdDaemonAnalytics - - assert isinstance(ret, int) or ret is None - - if ret is not None: - self.info[self.PARAM_CMD_RETURN_CODE] = ret - - if args is not None and hasattr(args, "func"): - assert args.func != CmdDaemonAnalytics - self.info[self.PARAM_CMD_CLASS] = args.func.__name__ - - def dump(self): - """Save analytics report to a temporary file. - - Returns: - str: path to the temporary file that contains the analytics report. - """ - import tempfile - - with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: - json.dump(self.info, fobj) - return fobj.name - - @staticmethod - def is_enabled(cmd=None): - from dvc.config import Config, to_bool - from dvc.command.daemon import CmdDaemonBase - - if env2bool("DVC_TEST"): - return False - - if isinstance(cmd, CmdDaemonBase): - return False - - core = Config(validate=False).config.get(Config.SECTION_CORE, {}) - enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true")) - logger.debug( - "Analytics is {}.".format("enabled" if enabled else "disabled") - ) - return enabled - - @staticmethod - def send_cmd(cmd, args, ret): - """Collect and send analytics for CLI command. - - Args: - args (list): parsed args for the CLI command. - ret (int): return value of the CLI command. - """ - from dvc.daemon import daemon - - if not Analytics.is_enabled(cmd): - return - - analytics = Analytics() - analytics.collect_cmd(args, ret) - daemon(["analytics", analytics.dump()]) - - def send(self): - """Collect and send analytics.""" - import requests - - if not self.is_enabled(): - return - - self.collect() - - logger.debug("Sending analytics: {}".format(self.info)) - - try: - requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST) - except requests.exceptions.RequestException as exc: - logger.debug("Failed to send analytics: {}".format(str(exc))) diff --git a/dvc/analytics/__init__.py b/dvc/analytics/__init__.py new file mode 100644 index 000000000..739ef253e --- /dev/null +++ b/dvc/analytics/__init__.py @@ -0,0 +1,64 @@ +import json +import logging +import requests +import subprocess +import tempfile + +from dvc import __version__ +from dvc.analytics import user_id, system_info +from dvc.config import Config, to_bool +from dvc.exceptions import NotDvcRepoError +from dvc.repo import Repo +from dvc.scm import SCM +from dvc.utils import env2bool, is_binary + + +logger = logging.getLogger(__name__) + + +def collect_and_send_report(arguments=None, exit_code=None): + report = { + "cmd_class": arguments.func.__name__, + "cmd_return_code": exit_code, + "dvc_version": __version__, + "is_binary": is_binary(), + "scm_class": scm_in_use(), + "system_info": system_info.collect(), + "user_id": user_id.find_or_create(), + } + + with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: + json.dump(report, fobj) + subprocess.Popen(["python", "-m", "dvc.analytics", fobj.name]) + + +def send(path): + url = "https://analytics.dvc.org" + + with open(path) as fobj: + report = json.load(fobj) + + requests.post(url, json=report, timeout=5) + + +def is_enabled(): + if env2bool("DVC_TEST"): + return False + + enabled = to_bool( + Config(validate=False) + .config.get(Config.SECTION_CORE, {}) + .get(Config.SECTION_CORE_ANALYTICS, "true") + ) + + logger.debug("Analytics is {}enabled.".format("" if enabled else "dis")) + + return enabled + + +def scm_in_use(): + try: + scm = SCM(root_dir=Repo.find_root()) + return type(scm).__name__ + except NotDvcRepoError: + pass diff --git a/dvc/analytics/__main__.py b/dvc/analytics/__main__.py new file mode 100644 index 000000000..13ebfa53b --- /dev/null +++ b/dvc/analytics/__main__.py @@ -0,0 +1,8 @@ +import sys + +from dvc.analytics import send + + +if __name__ == "__main__": + report_fname = sys.argv[1] + send(report_fname) diff --git a/dvc/analytics/system_info.py b/dvc/analytics/system_info.py new file mode 100644 index 000000000..666324cfd --- /dev/null +++ b/dvc/analytics/system_info.py @@ -0,0 +1,29 @@ +import platform +import sys +import distro + + +def collect(): + system = platform.system() + + if system == "Windows": + version = sys.getwindowsversion() + + return { + "os": "windows", + "windows_version_build": version.build, + "windows_version_major": version.major, + "windows_version_minor": version.minor, + "windows_version_service_pack": version.service_pack, + } + + if system == "Darwing": + return {"os": "mac", "mac_version": platform.mac_ver()[0]} + + if system == "Linux": + return { + "os": "linux", + "linux_distro": distro.id(), + "linux_distro_like": distro.like(), + "linux_distro_version": distro.version(), + } diff --git a/dvc/analytics/user_id.py b/dvc/analytics/user_id.py new file mode 100644 index 000000000..4c360cd98 --- /dev/null +++ b/dvc/analytics/user_id.py @@ -0,0 +1,55 @@ +""" +Interact with the user's ID stored under the global config directory. + +The file should contain a JSON with a "user_id" key: + + {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"} + +IDs are generated randomly with UUID. +""" + +import json +import logging +import uuid + +from dvc.config import Config +from dvc.lock import Lock, LockError +from dvc.utils import makedirs + + +logger = logging.getLogger(__name__) + +config_dir = Config.get_global_config_dir() +fname = config_dir / "user_id" + + +def find_or_create(): + lockfile = fname.with_suffix(".lock") + + try: + with Lock(lockfile): + return _load() or _create() + except LockError: + logger.debug("Failed to acquire {lock}".format(lockfile)) + + +def _load(): + if not fname.exists: + return None + + with open(fname) as fobj: + try: + return json.load(fobj).get("user_id") + except json.JSONDecodeError: + pass + + +def _create(): + user_id = str(uuid.uuid4()) + + makedirs(fname.parent, exist_ok=True) + + with open(fname, "w") as fobj: + json.dump({"user_id": user_id}, fobj) + + return user_id diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 2b60bded4..3953ab706 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -22,16 +22,6 @@ class CmdDaemonUpdater(CmdDaemonBase): return 0 -class CmdDaemonAnalytics(CmdDaemonBase): - def run(self): - from dvc.analytics import Analytics - - analytics = Analytics.load(self.args.target) - analytics.send() - - return 0 - - def add_parser(subparsers, parent_parser): DAEMON_HELP = "Service daemon." daemon_parser = subparsers.add_parser( @@ -55,13 +45,3 @@ def add_parser(subparsers, parent_parser): help=DAEMON_UPDATER_HELP, ) daemon_updater_parser.set_defaults(func=CmdDaemonUpdater) - - DAEMON_ANALYTICS_HELP = "Send dvc usage analytics." - daemon_analytics_parser = daemon_subparsers.add_parser( - "analytics", - parents=[parent_parser], - description=DAEMON_ANALYTICS_HELP, - help=DAEMON_ANALYTICS_HELP, - ) - daemon_analytics_parser.add_argument("target", help="Analytics file.") - daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics) diff --git a/dvc/config.py b/dvc/config.py index b883ef259..8e9b11c2b 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -13,8 +13,7 @@ from voluptuous import All, Any, Lower, Range, Coerce, Match from dvc.exceptions import DvcException from dvc.exceptions import NotDvcRepoError -from dvc.utils.compat import open -from dvc.utils.compat import str +from dvc.utils.compat import open, str, pathlib logger = logging.getLogger(__name__) @@ -285,8 +284,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes """ from appdirs import user_config_dir - return user_config_dir( - appname=Config.APPNAME, appauthor=Config.APPAUTHOR + return pathlib.Path( + user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) ) @staticmethod @@ -298,8 +297,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes """ from appdirs import site_config_dir - return site_config_dir( - appname=Config.APPNAME, appauthor=Config.APPAUTHOR + return pathlib.Path( + site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) ) @staticmethod diff --git a/dvc/lock.py b/dvc/lock.py index 147118c0f..7ffe54fc3 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -10,7 +10,7 @@ from funcy.py3 import lkeep from dvc.exceptions import DvcException from dvc.utils import makedirs -from dvc.utils.compat import is_py3 +from dvc.utils.compat import is_py3, str DEFAULT_TIMEOUT = 5 @@ -58,7 +58,7 @@ if is_py3: # [1] https://github.com/iterative/dvc/issues/2582 self._hostname = socket.gethostname() - self._lockfile = lockfile + self._lockfile = str(lockfile) self._lifetime = timedelta(days=365) # Lock for good by default self._separator = flufl.lock.SEP self._set_claimfile() @@ -111,7 +111,7 @@ else: """ def __init__(self, lockfile, tmp_dir=None): - self.lockfile = lockfile + self.lockfile = str(lockfile) self._lock = None @property diff --git a/dvc/main.py b/dvc/main.py index 660f9420d..86e1a9e0d 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import logging -from dvc.analytics import Analytics +from dvc import analytics from dvc.cli import parse_args from dvc.config import ConfigError from dvc.exceptions import DvcParserError @@ -81,6 +81,7 @@ def main(argv=None): if ret != 0: logger.info(FOOTER) - Analytics().send_cmd(cmd, args, ret) + if analytics.is_enabled(): + analytics.collect_and_send_report(args, ret) return ret diff --git a/dvc/repo/init.py b/dvc/repo/init.py index e06307e0a..5df1d0f1f 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -3,7 +3,7 @@ import os import colorama -from dvc.analytics import Analytics +from dvc import analytics from dvc.config import Config from dvc.exceptions import InitError from dvc.repo import Repo @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) def _welcome_message(): - if Analytics.is_enabled(): + if analytics.is_enabled(): logger.info( boxify( "DVC has enabled anonymous aggregate usage analytics.\n" diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py deleted file mode 100644 index b42a11727..000000000 --- a/tests/func/test_analytics.py +++ /dev/null @@ -1,65 +0,0 @@ -import os - -import mock -import requests - -from dvc.analytics import Analytics -from dvc.main import main -from tests.basic_env import TestDir -from tests.basic_env import TestDvc -from tests.basic_env import TestGit - - -def _clean_getenv(key, default=None): - """ - Remove env vars that affect dvc behavior in tests - """ - if key in ["DVC_TEST", "CI"]: - return None - return os.environ.get(key, default) - - -class TestAnalytics(TestDir): - def test(self): - a = Analytics() - a.collect() - self.assertTrue(isinstance(a.info, dict)) - self.assertNotEqual(a.info, {}) - self.assertTrue(a.PARAM_USER_ID in a.info.keys()) - self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys()) - self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {}) - - @mock.patch.object(os, "getenv", new=_clean_getenv) - @mock.patch("requests.post") - def test_send(self, mockpost): - ret = main(["daemon", "analytics", Analytics().dump(), "-v"]) - self.assertEqual(ret, 0) - - self.assertTrue(mockpost.called) - - @mock.patch.object(os, "getenv", new=_clean_getenv) - @mock.patch.object( - requests, "post", side_effect=requests.exceptions.RequestException() - ) - def test_send_failed(self, mockpost): - ret = main(["daemon", "analytics", Analytics().dump(), "-v"]) - self.assertEqual(ret, 0) - - self.assertTrue(mockpost.called) - - -class TestAnalyticsGit(TestAnalytics, TestGit): - pass - - -class TestAnalyticsDvc(TestAnalytics, TestDvc): - @mock.patch("requests.post") - def test_send_disabled(self, mockpost): - ret = main(["config", "core.analytics", "false"]) - self.assertEqual(ret, 0) - - with mock.patch.object(os, "getenv", new=_clean_getenv): - ret = main(["daemon", "analytics", Analytics().dump(), "-v"]) - self.assertEqual(ret, 0) - - self.assertFalse(mockpost.called) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 9e7e2dab1..2fbe134d4 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -1,6 +1,6 @@ import pytest -from dvc.analytics import Analytics +from dvc import analytics @pytest.mark.parametrize( @@ -21,4 +21,4 @@ def test_is_enabled(dvc_repo, config, result, monkeypatch): # reset DVC_TEST env var, which affects `is_enabled()` monkeypatch.delenv("DVC_TEST") - assert result == Analytics.is_enabled() + assert result == analytics.is_enabled()