analytics: refactor into a module

This commit is contained in:
Mr. Outis
2019-11-20 13:31:44 -06:00
parent 91ac75d023
commit ef1f038c32
12 changed files with 171 additions and 356 deletions
-256
View File
@@ -1,256 +0,0 @@
"""Collect and send usage analytics"""
from __future__ import unicode_literals
import errno
import json
import logging
import os
from dvc import __version__
from dvc.utils import env2bool
from dvc.utils.compat import str
logger = logging.getLogger(__name__)
class Analytics(object):
"""Class for collecting and sending usage analytics.
Args:
info (dict): optional existing analytics report.
"""
URL = "https://analytics.dvc.org"
TIMEOUT_POST = 5
USER_ID_FILE = "user_id"
PARAM_DVC_VERSION = "dvc_version"
PARAM_USER_ID = "user_id"
PARAM_SYSTEM_INFO = "system_info"
PARAM_OS = "os"
PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major"
PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor"
PARAM_WINDOWS_VERSION_BUILD = "windows_version_build"
PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack"
PARAM_MAC_VERSION = "mac_version"
PARAM_LINUX_DISTRO = "linux_distro"
PARAM_LINUX_DISTRO_VERSION = "linux_distro_version"
PARAM_LINUX_DISTRO_LIKE = "linux_distro_like"
PARAM_SCM_CLASS = "scm_class"
PARAM_IS_BINARY = "is_binary"
PARAM_CMD_CLASS = "cmd_class"
PARAM_CMD_RETURN_CODE = "cmd_return_code"
def __init__(self, info=None):
from dvc.config import Config
from dvc.lock import Lock
if info is None:
info = {}
self.info = info
cdir = Config.get_global_config_dir()
try:
os.makedirs(cdir)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
self.user_id_file_lock = Lock(self.user_id_file + ".lock")
@staticmethod
def load(path):
"""Loads analytics report from json file specified by path.
Args:
path (str): path to json file with analytics report.
"""
with open(path, "r") as fobj:
analytics = Analytics(info=json.load(fobj))
os.unlink(path)
return analytics
def _write_user_id(self):
import uuid
with open(self.user_id_file, "w+") as fobj:
user_id = str(uuid.uuid4())
info = {self.PARAM_USER_ID: user_id}
json.dump(info, fobj)
return user_id
def _read_user_id(self):
if not os.path.exists(self.user_id_file):
return None
with open(self.user_id_file, "r") as fobj:
try:
info = json.load(fobj)
except ValueError as exc:
logger.debug("Failed to load user_id: {}".format(exc))
return None
return info[self.PARAM_USER_ID]
def _get_user_id(self):
from dvc.lock import LockError
try:
with self.user_id_file_lock:
user_id = self._read_user_id()
if user_id is None:
user_id = self._write_user_id()
return user_id
except LockError:
msg = "Failed to acquire '{}'"
logger.debug(msg.format(self.user_id_file_lock.lockfile))
def _collect_windows(self):
import sys
version = sys.getwindowsversion() # pylint: disable=no-member
info = {}
info[self.PARAM_OS] = "windows"
info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
return info
def _collect_darwin(self):
import platform
info = {}
info[self.PARAM_OS] = "mac"
info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
return info
def _collect_linux(self):
import distro
info = {}
info[self.PARAM_OS] = "linux"
info[self.PARAM_LINUX_DISTRO] = distro.id()
info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
return info
def _collect_system_info(self):
import platform
system = platform.system()
if system == "Windows":
return self._collect_windows()
if system == "Darwin":
return self._collect_darwin()
if system == "Linux":
return self._collect_linux()
raise NotImplementedError
def collect(self):
"""Collect analytics report."""
from dvc.scm import SCM
from dvc.utils import is_binary
from dvc.repo import Repo
from dvc.exceptions import NotDvcRepoError
self.info[self.PARAM_DVC_VERSION] = __version__
self.info[self.PARAM_IS_BINARY] = is_binary()
self.info[self.PARAM_USER_ID] = self._get_user_id()
self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
try:
scm = SCM(root_dir=Repo.find_root())
self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
except NotDvcRepoError:
pass
def collect_cmd(self, args, ret):
"""Collect analytics info from a CLI command."""
from dvc.command.daemon import CmdDaemonAnalytics
assert isinstance(ret, int) or ret is None
if ret is not None:
self.info[self.PARAM_CMD_RETURN_CODE] = ret
if args is not None and hasattr(args, "func"):
assert args.func != CmdDaemonAnalytics
self.info[self.PARAM_CMD_CLASS] = args.func.__name__
def dump(self):
"""Save analytics report to a temporary file.
Returns:
str: path to the temporary file that contains the analytics report.
"""
import tempfile
with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
json.dump(self.info, fobj)
return fobj.name
@staticmethod
def is_enabled(cmd=None):
from dvc.config import Config, to_bool
from dvc.command.daemon import CmdDaemonBase
if env2bool("DVC_TEST"):
return False
if isinstance(cmd, CmdDaemonBase):
return False
core = Config(validate=False).config.get(Config.SECTION_CORE, {})
enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true"))
logger.debug(
"Analytics is {}.".format("enabled" if enabled else "disabled")
)
return enabled
@staticmethod
def send_cmd(cmd, args, ret):
"""Collect and send analytics for CLI command.
Args:
args (list): parsed args for the CLI command.
ret (int): return value of the CLI command.
"""
from dvc.daemon import daemon
if not Analytics.is_enabled(cmd):
return
analytics = Analytics()
analytics.collect_cmd(args, ret)
daemon(["analytics", analytics.dump()])
def send(self):
"""Collect and send analytics."""
import requests
if not self.is_enabled():
return
self.collect()
logger.debug("Sending analytics: {}".format(self.info))
try:
requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
except requests.exceptions.RequestException as exc:
logger.debug("Failed to send analytics: {}".format(str(exc)))
+64
View File
@@ -0,0 +1,64 @@
import json
import logging
import requests
import subprocess
import tempfile
from dvc import __version__
from dvc.analytics import user_id, system_info
from dvc.config import Config, to_bool
from dvc.exceptions import NotDvcRepoError
from dvc.repo import Repo
from dvc.scm import SCM
from dvc.utils import env2bool, is_binary
logger = logging.getLogger(__name__)
def collect_and_send_report(arguments=None, exit_code=None):
report = {
"cmd_class": arguments.func.__name__,
"cmd_return_code": exit_code,
"dvc_version": __version__,
"is_binary": is_binary(),
"scm_class": scm_in_use(),
"system_info": system_info.collect(),
"user_id": user_id.find_or_create(),
}
with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
json.dump(report, fobj)
subprocess.Popen(["python", "-m", "dvc.analytics", fobj.name])
def send(path):
url = "https://analytics.dvc.org"
with open(path) as fobj:
report = json.load(fobj)
requests.post(url, json=report, timeout=5)
def is_enabled():
if env2bool("DVC_TEST"):
return False
enabled = to_bool(
Config(validate=False)
.config.get(Config.SECTION_CORE, {})
.get(Config.SECTION_CORE_ANALYTICS, "true")
)
logger.debug("Analytics is {}enabled.".format("" if enabled else "dis"))
return enabled
def scm_in_use():
try:
scm = SCM(root_dir=Repo.find_root())
return type(scm).__name__
except NotDvcRepoError:
pass
+8
View File
@@ -0,0 +1,8 @@
import sys
from dvc.analytics import send
if __name__ == "__main__":
report_fname = sys.argv[1]
send(report_fname)
+29
View File
@@ -0,0 +1,29 @@
import platform
import sys
import distro
def collect():
system = platform.system()
if system == "Windows":
version = sys.getwindowsversion()
return {
"os": "windows",
"windows_version_build": version.build,
"windows_version_major": version.major,
"windows_version_minor": version.minor,
"windows_version_service_pack": version.service_pack,
}
if system == "Darwing":
return {"os": "mac", "mac_version": platform.mac_ver()[0]}
if system == "Linux":
return {
"os": "linux",
"linux_distro": distro.id(),
"linux_distro_like": distro.like(),
"linux_distro_version": distro.version(),
}
+55
View File
@@ -0,0 +1,55 @@
"""
Interact with the user's ID stored under the global config directory.
The file should contain a JSON with a "user_id" key:
{"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"}
IDs are generated randomly with UUID.
"""
import json
import logging
import uuid
from dvc.config import Config
from dvc.lock import Lock, LockError
from dvc.utils import makedirs
logger = logging.getLogger(__name__)
config_dir = Config.get_global_config_dir()
fname = config_dir / "user_id"
def find_or_create():
lockfile = fname.with_suffix(".lock")
try:
with Lock(lockfile):
return _load() or _create()
except LockError:
logger.debug("Failed to acquire {lock}".format(lockfile))
def _load():
if not fname.exists:
return None
with open(fname) as fobj:
try:
return json.load(fobj).get("user_id")
except json.JSONDecodeError:
pass
def _create():
user_id = str(uuid.uuid4())
makedirs(fname.parent, exist_ok=True)
with open(fname, "w") as fobj:
json.dump({"user_id": user_id}, fobj)
return user_id
-20
View File
@@ -22,16 +22,6 @@ class CmdDaemonUpdater(CmdDaemonBase):
return 0
class CmdDaemonAnalytics(CmdDaemonBase):
def run(self):
from dvc.analytics import Analytics
analytics = Analytics.load(self.args.target)
analytics.send()
return 0
def add_parser(subparsers, parent_parser):
DAEMON_HELP = "Service daemon."
daemon_parser = subparsers.add_parser(
@@ -55,13 +45,3 @@ def add_parser(subparsers, parent_parser):
help=DAEMON_UPDATER_HELP,
)
daemon_updater_parser.set_defaults(func=CmdDaemonUpdater)
DAEMON_ANALYTICS_HELP = "Send dvc usage analytics."
daemon_analytics_parser = daemon_subparsers.add_parser(
"analytics",
parents=[parent_parser],
description=DAEMON_ANALYTICS_HELP,
help=DAEMON_ANALYTICS_HELP,
)
daemon_analytics_parser.add_argument("target", help="Analytics file.")
daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)
+5 -6
View File
@@ -13,8 +13,7 @@ from voluptuous import All, Any, Lower, Range, Coerce, Match
from dvc.exceptions import DvcException
from dvc.exceptions import NotDvcRepoError
from dvc.utils.compat import open
from dvc.utils.compat import str
from dvc.utils.compat import open, str, pathlib
logger = logging.getLogger(__name__)
@@ -285,8 +284,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes
"""
from appdirs import user_config_dir
return user_config_dir(
appname=Config.APPNAME, appauthor=Config.APPAUTHOR
return pathlib.Path(
user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR)
)
@staticmethod
@@ -298,8 +297,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes
"""
from appdirs import site_config_dir
return site_config_dir(
appname=Config.APPNAME, appauthor=Config.APPAUTHOR
return pathlib.Path(
site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR)
)
@staticmethod
+3 -3
View File
@@ -10,7 +10,7 @@ from funcy.py3 import lkeep
from dvc.exceptions import DvcException
from dvc.utils import makedirs
from dvc.utils.compat import is_py3
from dvc.utils.compat import is_py3, str
DEFAULT_TIMEOUT = 5
@@ -58,7 +58,7 @@ if is_py3:
# [1] https://github.com/iterative/dvc/issues/2582
self._hostname = socket.gethostname()
self._lockfile = lockfile
self._lockfile = str(lockfile)
self._lifetime = timedelta(days=365) # Lock for good by default
self._separator = flufl.lock.SEP
self._set_claimfile()
@@ -111,7 +111,7 @@ else:
"""
def __init__(self, lockfile, tmp_dir=None):
self.lockfile = lockfile
self.lockfile = str(lockfile)
self._lock = None
@property
+3 -2
View File
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
import logging
from dvc.analytics import Analytics
from dvc import analytics
from dvc.cli import parse_args
from dvc.config import ConfigError
from dvc.exceptions import DvcParserError
@@ -81,6 +81,7 @@ def main(argv=None):
if ret != 0:
logger.info(FOOTER)
Analytics().send_cmd(cmd, args, ret)
if analytics.is_enabled():
analytics.collect_and_send_report(args, ret)
return ret
+2 -2
View File
@@ -3,7 +3,7 @@ import os
import colorama
from dvc.analytics import Analytics
from dvc import analytics
from dvc.config import Config
from dvc.exceptions import InitError
from dvc.repo import Repo
@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
def _welcome_message():
if Analytics.is_enabled():
if analytics.is_enabled():
logger.info(
boxify(
"DVC has enabled anonymous aggregate usage analytics.\n"
-65
View File
@@ -1,65 +0,0 @@
import os
import mock
import requests
from dvc.analytics import Analytics
from dvc.main import main
from tests.basic_env import TestDir
from tests.basic_env import TestDvc
from tests.basic_env import TestGit
def _clean_getenv(key, default=None):
"""
Remove env vars that affect dvc behavior in tests
"""
if key in ["DVC_TEST", "CI"]:
return None
return os.environ.get(key, default)
class TestAnalytics(TestDir):
def test(self):
a = Analytics()
a.collect()
self.assertTrue(isinstance(a.info, dict))
self.assertNotEqual(a.info, {})
self.assertTrue(a.PARAM_USER_ID in a.info.keys())
self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys())
self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {})
@mock.patch.object(os, "getenv", new=_clean_getenv)
@mock.patch("requests.post")
def test_send(self, mockpost):
ret = main(["daemon", "analytics", Analytics().dump(), "-v"])
self.assertEqual(ret, 0)
self.assertTrue(mockpost.called)
@mock.patch.object(os, "getenv", new=_clean_getenv)
@mock.patch.object(
requests, "post", side_effect=requests.exceptions.RequestException()
)
def test_send_failed(self, mockpost):
ret = main(["daemon", "analytics", Analytics().dump(), "-v"])
self.assertEqual(ret, 0)
self.assertTrue(mockpost.called)
class TestAnalyticsGit(TestAnalytics, TestGit):
pass
class TestAnalyticsDvc(TestAnalytics, TestDvc):
@mock.patch("requests.post")
def test_send_disabled(self, mockpost):
ret = main(["config", "core.analytics", "false"])
self.assertEqual(ret, 0)
with mock.patch.object(os, "getenv", new=_clean_getenv):
ret = main(["daemon", "analytics", Analytics().dump(), "-v"])
self.assertEqual(ret, 0)
self.assertFalse(mockpost.called)
+2 -2
View File
@@ -1,6 +1,6 @@
import pytest
from dvc.analytics import Analytics
from dvc import analytics
@pytest.mark.parametrize(
@@ -21,4 +21,4 @@ def test_is_enabled(dvc_repo, config, result, monkeypatch):
# reset DVC_TEST env var, which affects `is_enabled()`
monkeypatch.delenv("DVC_TEST")
assert result == Analytics.is_enabled()
assert result == analytics.is_enabled()