mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 15:28:28 +01:00
analytics: refactor into a module
This commit is contained in:
@@ -1,256 +0,0 @@
|
||||
"""Collect and send usage analytics"""
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import errno
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc import __version__
|
||||
from dvc.utils import env2bool
|
||||
from dvc.utils.compat import str
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Analytics(object):
|
||||
"""Class for collecting and sending usage analytics.
|
||||
|
||||
Args:
|
||||
info (dict): optional existing analytics report.
|
||||
"""
|
||||
|
||||
URL = "https://analytics.dvc.org"
|
||||
TIMEOUT_POST = 5
|
||||
|
||||
USER_ID_FILE = "user_id"
|
||||
|
||||
PARAM_DVC_VERSION = "dvc_version"
|
||||
PARAM_USER_ID = "user_id"
|
||||
PARAM_SYSTEM_INFO = "system_info"
|
||||
|
||||
PARAM_OS = "os"
|
||||
|
||||
PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major"
|
||||
PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor"
|
||||
PARAM_WINDOWS_VERSION_BUILD = "windows_version_build"
|
||||
PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack"
|
||||
|
||||
PARAM_MAC_VERSION = "mac_version"
|
||||
|
||||
PARAM_LINUX_DISTRO = "linux_distro"
|
||||
PARAM_LINUX_DISTRO_VERSION = "linux_distro_version"
|
||||
PARAM_LINUX_DISTRO_LIKE = "linux_distro_like"
|
||||
|
||||
PARAM_SCM_CLASS = "scm_class"
|
||||
PARAM_IS_BINARY = "is_binary"
|
||||
PARAM_CMD_CLASS = "cmd_class"
|
||||
PARAM_CMD_RETURN_CODE = "cmd_return_code"
|
||||
|
||||
def __init__(self, info=None):
|
||||
from dvc.config import Config
|
||||
from dvc.lock import Lock
|
||||
|
||||
if info is None:
|
||||
info = {}
|
||||
|
||||
self.info = info
|
||||
|
||||
cdir = Config.get_global_config_dir()
|
||||
try:
|
||||
os.makedirs(cdir)
|
||||
except OSError as exc:
|
||||
if exc.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
|
||||
self.user_id_file_lock = Lock(self.user_id_file + ".lock")
|
||||
|
||||
@staticmethod
|
||||
def load(path):
|
||||
"""Loads analytics report from json file specified by path.
|
||||
|
||||
Args:
|
||||
path (str): path to json file with analytics report.
|
||||
"""
|
||||
with open(path, "r") as fobj:
|
||||
analytics = Analytics(info=json.load(fobj))
|
||||
os.unlink(path)
|
||||
return analytics
|
||||
|
||||
def _write_user_id(self):
|
||||
import uuid
|
||||
|
||||
with open(self.user_id_file, "w+") as fobj:
|
||||
user_id = str(uuid.uuid4())
|
||||
info = {self.PARAM_USER_ID: user_id}
|
||||
json.dump(info, fobj)
|
||||
return user_id
|
||||
|
||||
def _read_user_id(self):
|
||||
if not os.path.exists(self.user_id_file):
|
||||
return None
|
||||
|
||||
with open(self.user_id_file, "r") as fobj:
|
||||
try:
|
||||
info = json.load(fobj)
|
||||
except ValueError as exc:
|
||||
logger.debug("Failed to load user_id: {}".format(exc))
|
||||
return None
|
||||
|
||||
return info[self.PARAM_USER_ID]
|
||||
|
||||
def _get_user_id(self):
|
||||
from dvc.lock import LockError
|
||||
|
||||
try:
|
||||
with self.user_id_file_lock:
|
||||
user_id = self._read_user_id()
|
||||
if user_id is None:
|
||||
user_id = self._write_user_id()
|
||||
return user_id
|
||||
except LockError:
|
||||
msg = "Failed to acquire '{}'"
|
||||
logger.debug(msg.format(self.user_id_file_lock.lockfile))
|
||||
|
||||
def _collect_windows(self):
|
||||
import sys
|
||||
|
||||
version = sys.getwindowsversion() # pylint: disable=no-member
|
||||
info = {}
|
||||
info[self.PARAM_OS] = "windows"
|
||||
info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
|
||||
info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
|
||||
info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
|
||||
info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
|
||||
return info
|
||||
|
||||
def _collect_darwin(self):
|
||||
import platform
|
||||
|
||||
info = {}
|
||||
info[self.PARAM_OS] = "mac"
|
||||
info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
|
||||
return info
|
||||
|
||||
def _collect_linux(self):
|
||||
import distro
|
||||
|
||||
info = {}
|
||||
info[self.PARAM_OS] = "linux"
|
||||
info[self.PARAM_LINUX_DISTRO] = distro.id()
|
||||
info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
|
||||
info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
|
||||
return info
|
||||
|
||||
def _collect_system_info(self):
|
||||
import platform
|
||||
|
||||
system = platform.system()
|
||||
|
||||
if system == "Windows":
|
||||
return self._collect_windows()
|
||||
|
||||
if system == "Darwin":
|
||||
return self._collect_darwin()
|
||||
|
||||
if system == "Linux":
|
||||
return self._collect_linux()
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
def collect(self):
|
||||
"""Collect analytics report."""
|
||||
from dvc.scm import SCM
|
||||
from dvc.utils import is_binary
|
||||
from dvc.repo import Repo
|
||||
from dvc.exceptions import NotDvcRepoError
|
||||
|
||||
self.info[self.PARAM_DVC_VERSION] = __version__
|
||||
self.info[self.PARAM_IS_BINARY] = is_binary()
|
||||
self.info[self.PARAM_USER_ID] = self._get_user_id()
|
||||
|
||||
self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
|
||||
|
||||
try:
|
||||
scm = SCM(root_dir=Repo.find_root())
|
||||
self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
|
||||
except NotDvcRepoError:
|
||||
pass
|
||||
|
||||
def collect_cmd(self, args, ret):
|
||||
"""Collect analytics info from a CLI command."""
|
||||
from dvc.command.daemon import CmdDaemonAnalytics
|
||||
|
||||
assert isinstance(ret, int) or ret is None
|
||||
|
||||
if ret is not None:
|
||||
self.info[self.PARAM_CMD_RETURN_CODE] = ret
|
||||
|
||||
if args is not None and hasattr(args, "func"):
|
||||
assert args.func != CmdDaemonAnalytics
|
||||
self.info[self.PARAM_CMD_CLASS] = args.func.__name__
|
||||
|
||||
def dump(self):
|
||||
"""Save analytics report to a temporary file.
|
||||
|
||||
Returns:
|
||||
str: path to the temporary file that contains the analytics report.
|
||||
"""
|
||||
import tempfile
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
|
||||
json.dump(self.info, fobj)
|
||||
return fobj.name
|
||||
|
||||
@staticmethod
|
||||
def is_enabled(cmd=None):
|
||||
from dvc.config import Config, to_bool
|
||||
from dvc.command.daemon import CmdDaemonBase
|
||||
|
||||
if env2bool("DVC_TEST"):
|
||||
return False
|
||||
|
||||
if isinstance(cmd, CmdDaemonBase):
|
||||
return False
|
||||
|
||||
core = Config(validate=False).config.get(Config.SECTION_CORE, {})
|
||||
enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true"))
|
||||
logger.debug(
|
||||
"Analytics is {}.".format("enabled" if enabled else "disabled")
|
||||
)
|
||||
return enabled
|
||||
|
||||
@staticmethod
|
||||
def send_cmd(cmd, args, ret):
|
||||
"""Collect and send analytics for CLI command.
|
||||
|
||||
Args:
|
||||
args (list): parsed args for the CLI command.
|
||||
ret (int): return value of the CLI command.
|
||||
"""
|
||||
from dvc.daemon import daemon
|
||||
|
||||
if not Analytics.is_enabled(cmd):
|
||||
return
|
||||
|
||||
analytics = Analytics()
|
||||
analytics.collect_cmd(args, ret)
|
||||
daemon(["analytics", analytics.dump()])
|
||||
|
||||
def send(self):
|
||||
"""Collect and send analytics."""
|
||||
import requests
|
||||
|
||||
if not self.is_enabled():
|
||||
return
|
||||
|
||||
self.collect()
|
||||
|
||||
logger.debug("Sending analytics: {}".format(self.info))
|
||||
|
||||
try:
|
||||
requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
|
||||
except requests.exceptions.RequestException as exc:
|
||||
logger.debug("Failed to send analytics: {}".format(str(exc)))
|
||||
@@ -0,0 +1,64 @@
|
||||
import json
|
||||
import logging
|
||||
import requests
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from dvc import __version__
|
||||
from dvc.analytics import user_id, system_info
|
||||
from dvc.config import Config, to_bool
|
||||
from dvc.exceptions import NotDvcRepoError
|
||||
from dvc.repo import Repo
|
||||
from dvc.scm import SCM
|
||||
from dvc.utils import env2bool, is_binary
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def collect_and_send_report(arguments=None, exit_code=None):
|
||||
report = {
|
||||
"cmd_class": arguments.func.__name__,
|
||||
"cmd_return_code": exit_code,
|
||||
"dvc_version": __version__,
|
||||
"is_binary": is_binary(),
|
||||
"scm_class": scm_in_use(),
|
||||
"system_info": system_info.collect(),
|
||||
"user_id": user_id.find_or_create(),
|
||||
}
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
|
||||
json.dump(report, fobj)
|
||||
subprocess.Popen(["python", "-m", "dvc.analytics", fobj.name])
|
||||
|
||||
|
||||
def send(path):
|
||||
url = "https://analytics.dvc.org"
|
||||
|
||||
with open(path) as fobj:
|
||||
report = json.load(fobj)
|
||||
|
||||
requests.post(url, json=report, timeout=5)
|
||||
|
||||
|
||||
def is_enabled():
|
||||
if env2bool("DVC_TEST"):
|
||||
return False
|
||||
|
||||
enabled = to_bool(
|
||||
Config(validate=False)
|
||||
.config.get(Config.SECTION_CORE, {})
|
||||
.get(Config.SECTION_CORE_ANALYTICS, "true")
|
||||
)
|
||||
|
||||
logger.debug("Analytics is {}enabled.".format("" if enabled else "dis"))
|
||||
|
||||
return enabled
|
||||
|
||||
|
||||
def scm_in_use():
|
||||
try:
|
||||
scm = SCM(root_dir=Repo.find_root())
|
||||
return type(scm).__name__
|
||||
except NotDvcRepoError:
|
||||
pass
|
||||
@@ -0,0 +1,8 @@
|
||||
import sys
|
||||
|
||||
from dvc.analytics import send
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
report_fname = sys.argv[1]
|
||||
send(report_fname)
|
||||
@@ -0,0 +1,29 @@
|
||||
import platform
|
||||
import sys
|
||||
import distro
|
||||
|
||||
|
||||
def collect():
|
||||
system = platform.system()
|
||||
|
||||
if system == "Windows":
|
||||
version = sys.getwindowsversion()
|
||||
|
||||
return {
|
||||
"os": "windows",
|
||||
"windows_version_build": version.build,
|
||||
"windows_version_major": version.major,
|
||||
"windows_version_minor": version.minor,
|
||||
"windows_version_service_pack": version.service_pack,
|
||||
}
|
||||
|
||||
if system == "Darwing":
|
||||
return {"os": "mac", "mac_version": platform.mac_ver()[0]}
|
||||
|
||||
if system == "Linux":
|
||||
return {
|
||||
"os": "linux",
|
||||
"linux_distro": distro.id(),
|
||||
"linux_distro_like": distro.like(),
|
||||
"linux_distro_version": distro.version(),
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
"""
|
||||
Interact with the user's ID stored under the global config directory.
|
||||
|
||||
The file should contain a JSON with a "user_id" key:
|
||||
|
||||
{"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"}
|
||||
|
||||
IDs are generated randomly with UUID.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.lock import Lock, LockError
|
||||
from dvc.utils import makedirs
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
config_dir = Config.get_global_config_dir()
|
||||
fname = config_dir / "user_id"
|
||||
|
||||
|
||||
def find_or_create():
|
||||
lockfile = fname.with_suffix(".lock")
|
||||
|
||||
try:
|
||||
with Lock(lockfile):
|
||||
return _load() or _create()
|
||||
except LockError:
|
||||
logger.debug("Failed to acquire {lock}".format(lockfile))
|
||||
|
||||
|
||||
def _load():
|
||||
if not fname.exists:
|
||||
return None
|
||||
|
||||
with open(fname) as fobj:
|
||||
try:
|
||||
return json.load(fobj).get("user_id")
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
|
||||
def _create():
|
||||
user_id = str(uuid.uuid4())
|
||||
|
||||
makedirs(fname.parent, exist_ok=True)
|
||||
|
||||
with open(fname, "w") as fobj:
|
||||
json.dump({"user_id": user_id}, fobj)
|
||||
|
||||
return user_id
|
||||
@@ -22,16 +22,6 @@ class CmdDaemonUpdater(CmdDaemonBase):
|
||||
return 0
|
||||
|
||||
|
||||
class CmdDaemonAnalytics(CmdDaemonBase):
|
||||
def run(self):
|
||||
from dvc.analytics import Analytics
|
||||
|
||||
analytics = Analytics.load(self.args.target)
|
||||
analytics.send()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
DAEMON_HELP = "Service daemon."
|
||||
daemon_parser = subparsers.add_parser(
|
||||
@@ -55,13 +45,3 @@ def add_parser(subparsers, parent_parser):
|
||||
help=DAEMON_UPDATER_HELP,
|
||||
)
|
||||
daemon_updater_parser.set_defaults(func=CmdDaemonUpdater)
|
||||
|
||||
DAEMON_ANALYTICS_HELP = "Send dvc usage analytics."
|
||||
daemon_analytics_parser = daemon_subparsers.add_parser(
|
||||
"analytics",
|
||||
parents=[parent_parser],
|
||||
description=DAEMON_ANALYTICS_HELP,
|
||||
help=DAEMON_ANALYTICS_HELP,
|
||||
)
|
||||
daemon_analytics_parser.add_argument("target", help="Analytics file.")
|
||||
daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)
|
||||
|
||||
+5
-6
@@ -13,8 +13,7 @@ from voluptuous import All, Any, Lower, Range, Coerce, Match
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.exceptions import NotDvcRepoError
|
||||
from dvc.utils.compat import open
|
||||
from dvc.utils.compat import str
|
||||
from dvc.utils.compat import open, str, pathlib
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -285,8 +284,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes
|
||||
"""
|
||||
from appdirs import user_config_dir
|
||||
|
||||
return user_config_dir(
|
||||
appname=Config.APPNAME, appauthor=Config.APPAUTHOR
|
||||
return pathlib.Path(
|
||||
user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -298,8 +297,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes
|
||||
"""
|
||||
from appdirs import site_config_dir
|
||||
|
||||
return site_config_dir(
|
||||
appname=Config.APPNAME, appauthor=Config.APPAUTHOR
|
||||
return pathlib.Path(
|
||||
site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
+3
-3
@@ -10,7 +10,7 @@ from funcy.py3 import lkeep
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import makedirs
|
||||
from dvc.utils.compat import is_py3
|
||||
from dvc.utils.compat import is_py3, str
|
||||
|
||||
|
||||
DEFAULT_TIMEOUT = 5
|
||||
@@ -58,7 +58,7 @@ if is_py3:
|
||||
# [1] https://github.com/iterative/dvc/issues/2582
|
||||
self._hostname = socket.gethostname()
|
||||
|
||||
self._lockfile = lockfile
|
||||
self._lockfile = str(lockfile)
|
||||
self._lifetime = timedelta(days=365) # Lock for good by default
|
||||
self._separator = flufl.lock.SEP
|
||||
self._set_claimfile()
|
||||
@@ -111,7 +111,7 @@ else:
|
||||
"""
|
||||
|
||||
def __init__(self, lockfile, tmp_dir=None):
|
||||
self.lockfile = lockfile
|
||||
self.lockfile = str(lockfile)
|
||||
self._lock = None
|
||||
|
||||
@property
|
||||
|
||||
+3
-2
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
|
||||
|
||||
import logging
|
||||
|
||||
from dvc.analytics import Analytics
|
||||
from dvc import analytics
|
||||
from dvc.cli import parse_args
|
||||
from dvc.config import ConfigError
|
||||
from dvc.exceptions import DvcParserError
|
||||
@@ -81,6 +81,7 @@ def main(argv=None):
|
||||
if ret != 0:
|
||||
logger.info(FOOTER)
|
||||
|
||||
Analytics().send_cmd(cmd, args, ret)
|
||||
if analytics.is_enabled():
|
||||
analytics.collect_and_send_report(args, ret)
|
||||
|
||||
return ret
|
||||
|
||||
+2
-2
@@ -3,7 +3,7 @@ import os
|
||||
|
||||
import colorama
|
||||
|
||||
from dvc.analytics import Analytics
|
||||
from dvc import analytics
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import InitError
|
||||
from dvc.repo import Repo
|
||||
@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _welcome_message():
|
||||
if Analytics.is_enabled():
|
||||
if analytics.is_enabled():
|
||||
logger.info(
|
||||
boxify(
|
||||
"DVC has enabled anonymous aggregate usage analytics.\n"
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
import os
|
||||
|
||||
import mock
|
||||
import requests
|
||||
|
||||
from dvc.analytics import Analytics
|
||||
from dvc.main import main
|
||||
from tests.basic_env import TestDir
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.basic_env import TestGit
|
||||
|
||||
|
||||
def _clean_getenv(key, default=None):
|
||||
"""
|
||||
Remove env vars that affect dvc behavior in tests
|
||||
"""
|
||||
if key in ["DVC_TEST", "CI"]:
|
||||
return None
|
||||
return os.environ.get(key, default)
|
||||
|
||||
|
||||
class TestAnalytics(TestDir):
|
||||
def test(self):
|
||||
a = Analytics()
|
||||
a.collect()
|
||||
self.assertTrue(isinstance(a.info, dict))
|
||||
self.assertNotEqual(a.info, {})
|
||||
self.assertTrue(a.PARAM_USER_ID in a.info.keys())
|
||||
self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys())
|
||||
self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {})
|
||||
|
||||
@mock.patch.object(os, "getenv", new=_clean_getenv)
|
||||
@mock.patch("requests.post")
|
||||
def test_send(self, mockpost):
|
||||
ret = main(["daemon", "analytics", Analytics().dump(), "-v"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertTrue(mockpost.called)
|
||||
|
||||
@mock.patch.object(os, "getenv", new=_clean_getenv)
|
||||
@mock.patch.object(
|
||||
requests, "post", side_effect=requests.exceptions.RequestException()
|
||||
)
|
||||
def test_send_failed(self, mockpost):
|
||||
ret = main(["daemon", "analytics", Analytics().dump(), "-v"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertTrue(mockpost.called)
|
||||
|
||||
|
||||
class TestAnalyticsGit(TestAnalytics, TestGit):
|
||||
pass
|
||||
|
||||
|
||||
class TestAnalyticsDvc(TestAnalytics, TestDvc):
|
||||
@mock.patch("requests.post")
|
||||
def test_send_disabled(self, mockpost):
|
||||
ret = main(["config", "core.analytics", "false"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
with mock.patch.object(os, "getenv", new=_clean_getenv):
|
||||
ret = main(["daemon", "analytics", Analytics().dump(), "-v"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertFalse(mockpost.called)
|
||||
@@ -1,6 +1,6 @@
|
||||
import pytest
|
||||
|
||||
from dvc.analytics import Analytics
|
||||
from dvc import analytics
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -21,4 +21,4 @@ def test_is_enabled(dvc_repo, config, result, monkeypatch):
|
||||
# reset DVC_TEST env var, which affects `is_enabled()`
|
||||
monkeypatch.delenv("DVC_TEST")
|
||||
|
||||
assert result == Analytics.is_enabled()
|
||||
assert result == analytics.is_enabled()
|
||||
|
||||
Reference in New Issue
Block a user