From 22c6ad5ba38d46c8d9cf61c703d0d5a23eb311d5 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 22 Nov 2018 01:42:48 +0200 Subject: [PATCH] analytics: send user events Fixes #693 Signed-off-by: Ruslan Kuprieiev --- dvc/analytics.py | 211 ++++++++++++++++++++++++++++++++++++++++ dvc/cli.py | 21 +++- dvc/command/base.py | 20 +--- dvc/command/config.py | 17 +++- dvc/command/daemon.py | 22 ++++- dvc/command/init.py | 5 +- dvc/config.py | 61 ++++++++---- dvc/daemon.py | 4 +- dvc/main.py | 30 +++--- dvc/project.py | 32 +++++- dvc/updater.py | 3 +- tests/test_analytics.py | 62 ++++++++++++ 12 files changed, 425 insertions(+), 63 deletions(-) create mode 100644 dvc/analytics.py create mode 100644 tests/test_analytics.py diff --git a/dvc/analytics.py b/dvc/analytics.py new file mode 100644 index 000000000..06bcb3bef --- /dev/null +++ b/dvc/analytics.py @@ -0,0 +1,211 @@ +import os +import json +import errno + +from dvc import VERSION +from dvc.utils import is_binary +from dvc.lock import Lock, LockError +from dvc.logger import Logger + + +class Analytics(object): + URL = 'https://dddc07bh9f.execute-api.us-east-2.amazonaws.com/production' + TIMEOUT_POST = 10 + + USER_ID_FILE = 'user_id' + + PARAM_DVC_VERSION = 'dvc_version' + PARAM_USER_ID = 'user_id' + PARAM_SYSTEM_INFO = 'system_info' + + PARAM_OS = 'os' + + PARAM_WINDOWS_VERSION_MAJOR = 'windows_version_major' + PARAM_WINDOWS_VERSION_MINOR = 'windows_version_minor' + PARAM_WINDOWS_VERSION_BUILD = 'windows_version_build' + PARAM_WINDOWS_VERSION_SERVICE_PACK = 'windows_version_service_pack' + + PARAM_MAC_VERSION = 'mac_version' + + PARAM_LINUX_DISTRO = 'linux_distro' + PARAM_LINUX_DISTRO_VERSION = 'linux_distro_version' + PARAM_LINUX_DISTRO_LIKE = 'linux_distro_like' + + PARAM_SCM_CLASS = 'scm_class' + PARAM_IS_BINARY = 'is_binary' + PARAM_CMD_CLASS = 'cmd_class' + PARAM_CMD_RETURN_CODE = 'cmd_return_code' + + def __init__(self, info={}): + from dvc.config import Config + + self.info = info + + cdir = Config.get_global_config_dir() + try: + os.makedirs(cdir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + self.user_id_file = os.path.join(cdir, self.USER_ID_FILE) + self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + '.lock') + + @staticmethod + def load(path): + with open(path, 'r') as fobj: + a = Analytics(info=json.load(fobj)) + os.unlink(path) + return a + + def _write_user_id(self): + import uuid + + with open(self.user_id_file, 'w+') as fobj: + user_id = str(uuid.uuid4()) + info = {self.PARAM_USER_ID: user_id} + json.dump(info, fobj) + return user_id + + def _read_user_id(self): + if not os.path.exists(self.user_id_file): + return None + + with open(self.user_id_file, 'r') as fobj: + try: + info = json.load(fobj) + return info[self.PARAM_USER_ID] + except json.JSONDecodeError as exc: + Logger.debug("Failed to load user_id: {}".format(exc)) + return None + + def _get_user_id(self): + try: + with self.user_id_file_lock: + user_id = self._read_user_id() + if user_id is None: + user_id = self._write_user_id() + return user_id + except LockError: + msg = "Failed to acquire '{}'" + Logger.debug(msg.format(self.user_id_file_lock.lock_file)) + + def _collect_windows(self): + import sys + version = sys.getwindowsversion() + info = {} + info[self.PARAM_OS] = 'windows' + info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major + info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor + info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build + info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack + return info + + def _collect_darwin(self): + import platform + info = {} + info[self.PARAM_OS] = 'mac' + info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0] + return info + + def _collect_linux(self): + import distro + info = {} + info[self.PARAM_OS] = 'linux' + info[self.PARAM_LINUX_DISTRO] = distro.id() + info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version() + info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like() + return info + + def _collect_system_info(self): + import platform + system = platform.system() + if system == 'Windows': + return self._collect_windows() + elif system == 'Darwin': + return self._collect_darwin() + elif system == 'Linux': + return self._collect_linux() + else: + raise NotImplementedError + + def _collect(self, args=None, ret=None): + from dvc.scm import SCM + from dvc.project import Project, NotDvcProjectError + from dvc.command.daemon import CmdDaemonAnalytics + + self.info[self.PARAM_DVC_VERSION] = VERSION + self.info[self.PARAM_IS_BINARY] = is_binary() + self.info[self.PARAM_USER_ID] = self._get_user_id() + + self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info() + + try: + scm = SCM(root_dir=Project._find_root()) + self.info[self.PARAM_SCM_CLASS] = type(scm).__name__ + except NotDvcProjectError: + pass + + if ret is not None: + self.info[self.PARAM_CMD_RETURN_CODE] = ret + + if args is not None and hasattr(args, 'func'): + assert args.func != CmdDaemonAnalytics + self.info[self.PARAM_CMD_CLASS] = args.func.__name__ + + def dump(self): + import tempfile + + with tempfile.NamedTemporaryFile(delete=False, mode='w') as fobj: + json.dump(self.info, fobj) + return fobj.name + + @staticmethod + def _is_enabled(cmd): + from dvc.config import Config + from dvc.project import Project, NotDvcProjectError + from dvc.command.daemon import CmdDaemonBase + + if os.getenv('DVC_TEST'): + return False + + if isinstance(cmd, CmdDaemonBase): + return False + + if cmd is None or not hasattr(cmd, 'config'): + try: + dvc_dir = Project._find_dvc_dir() + config = Config(dvc_dir) + assert config is not None + except NotDvcProjectError: + config = Config(validate=False) + assert config is not None + else: + config = cmd.config + assert config is not None + + core = config._config.get(Config.SECTION_CORE, {}) + enabled = core.get(Config.SECTION_CORE_ANALYTICS, True) + Logger.debug("Analytics is {}abled." + .format('en' if enabled else 'dis')) + return enabled + + def send(self, detach=True, cmd=None, args=None, ret=None): + import requests + from dvc.daemon import Daemon + + if not self._is_enabled(cmd): + return + + if detach: # pragma: no cover + Daemon()(['analytics', self.dump()]) + return + + self._collect(args=args, ret=ret) + + Logger.debug("Sending analytics: {}".format(self.info)) + + try: + requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST) + except requests.exceptions.RequestException as exc: + Logger.debug("Failed to send analytics: {}".format(str(exc))) diff --git a/dvc/cli.py b/dvc/cli.py index 9eea136a0..607911670 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -26,7 +26,8 @@ from dvc.command.install import CmdInstall from dvc.command.root import CmdRoot from dvc.command.lock import CmdLock, CmdUnlock from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList -from dvc.command.daemon import CmdDaemonUpdater +from dvc.command.daemon import CmdDaemonUpdater, CmdDaemonAnalytics +from dvc.exceptions import DvcException from dvc.logger import Logger from dvc import VERSION @@ -41,11 +42,16 @@ def _fix_subparsers(subparsers): subparsers.dest = 'cmd' +class DvcParserError(DvcException): + def __init__(self): + super(DvcException, self).__init__("Parser error") + + class DvcParser(argparse.ArgumentParser): def error(self, message): sys.stderr.write('{}{}\n'.format(Logger.error_prefix(), message)) self.print_help() - sys.exit(2) + raise DvcParserError() class VersionAction(argparse.Action): # pragma: no cover @@ -1037,6 +1043,17 @@ def parse_args(argv=None): help=DAEMON_UPDATER_HELP) daemon_updater_parser.set_defaults(func=CmdDaemonUpdater) + DAEMON_ANALYTICS_HELP = 'Send dvc usage analytics.' + daemon_analytics_parser = daemon_subparsers.add_parser( + 'analytics', + parents=[parent_parser], + description=DAEMON_ANALYTICS_HELP, + help=DAEMON_ANALYTICS_HELP) + daemon_analytics_parser.add_argument( + 'target', + help="Analytics file.") + daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics) + args = parser.parse_args(argv) if (issubclass(args.func, CmdRepro) diff --git a/dvc/command/base.py b/dvc/command/base.py index 2a86c058b..8befa169e 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -1,7 +1,4 @@ -import os - from dvc.logger import Logger -from dvc.exceptions import DvcException from dvc.lock import LockError @@ -9,7 +6,8 @@ class CmdBase(object): def __init__(self, args): from dvc.project import Project - self.project = Project(self._find_root()) + self.project = Project() + self.config = self.project.config self.args = args self._set_loglevel(args) @@ -21,20 +19,6 @@ class CmdBase(object): elif args.verbose: Logger.be_verbose() - def _find_root(self): - from dvc.project import Project - - root = os.getcwd() - while True: - dvc_dir = os.path.join(root, Project.DVC_DIR) - if os.path.isdir(dvc_dir): - return root - if os.path.ismount(root): - break - root = os.path.dirname(root) - msg = "Not a dvc repository (checked up to mount point {})" - raise DvcException(msg.format(root)) - def run_cmd(self): try: with self.project.lock: diff --git a/dvc/command/config.py b/dvc/command/config.py index e1e5f86d8..75fb009fd 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -8,18 +8,29 @@ from dvc.exceptions import DvcException class CmdConfig(CmdBase): def __init__(self, args): - from dvc.project import Project + from dvc.project import Project, NotDvcProjectError self.args = args - dvc_dir = os.path.join(self._find_root(), Project.DVC_DIR) - self.config = Config(dvc_dir) + + try: + dvc_dir = os.path.join(Project._find_root(), Project.DVC_DIR) + saved_exc = None + except NotDvcProjectError as exc: + dvc_dir = None + saved_exc = exc + + self.config = Config(dvc_dir, validate=False) if self.args.system: self.configobj = self.config._system_config elif self.args.glob: self.configobj = self.config._global_config elif self.args.local: + if dvc_dir is None: + raise saved_exc self.configobj = self.config._local_config else: + if dvc_dir is None: + raise saved_exc self.configobj = self.config._project_config def run_cmd(self): diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 82ca1bf0c..1e4a02380 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -1,19 +1,35 @@ from dvc.command.base import CmdBase -class CmdDaemonUpdater(CmdBase): +class CmdDaemonBase(CmdBase): def __init__(self, args): - pass + self.args = args + self.config = None + self._set_loglevel(args) def run_cmd(self): return self.run() + +class CmdDaemonUpdater(CmdDaemonBase): def run(self): import os from dvc.project import Project from dvc.updater import Updater - root_dir = self._find_root() + root_dir = Project._find_root() dvc_dir = os.path.join(root_dir, Project.DVC_DIR) updater = Updater(dvc_dir) updater.fetch(detach=False) + + return 0 + + +class CmdDaemonAnalytics(CmdDaemonBase): + def run(self): + from dvc.analytics import Analytics + + analytics = Analytics.load(self.args.target) + analytics.send(detach=False) + + return 0 diff --git a/dvc/command/init.py b/dvc/command/init.py index ea6105dd3..988551b32 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -9,7 +9,10 @@ class CmdInit(object): from dvc.project import Project, InitError try: - Project.init('.', no_scm=self.args.no_scm, force=self.args.force) + self.project = Project.init('.', + no_scm=self.args.no_scm, + force=self.args.force) + self.config = self.project.config except InitError as e: Logger.error('Failed to initiate dvc', e) return 1 diff --git a/dvc/config.py b/dvc/config.py index 670193ac5..c310be2ec 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -68,6 +68,8 @@ class Config(object): SECTION_CORE_REMOTE = 'remote' SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool)) SECTION_CORE_INTERACTIVE = 'interactive' + SECTION_CORE_ANALYTICS = 'analytics' + SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool)) SECTION_CACHE = 'cache' SECTION_CACHE_DIR = 'dir' @@ -106,6 +108,8 @@ class Config(object): Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)), Optional(SECTION_CORE_INTERACTIVE, default=False): SECTION_CORE_INTERACTIVE_SCHEMA, + Optional(SECTION_CORE_ANALYTICS, + default=True): SECTION_CORE_ANALYTICS_SCHEMA, # backward compatibility Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA, @@ -190,28 +194,34 @@ class Config(object): Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA, } - def __init__(self, dvc_dir): - self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) - self.system_config_file = os.path.join(self.system_config_dir, + def __init__(self, dvc_dir=None, validate=True): + self.system_config_file = os.path.join(self.get_system_config_dir(), self.CONFIG) - self.global_config_file = os.path.join(self.global_config_dir, + self.global_config_file = os.path.join(self.get_global_config_dir(), self.CONFIG) - self.config_file = os.path.join(dvc_dir, self.CONFIG) - self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL) - self._load() + if dvc_dir is not None: + self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) + self.config_file = os.path.join(dvc_dir, self.CONFIG) + self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL) + else: + self.dvc_dir = None + self.config_file = None + self.config_local_file = None - @property - def global_config_dir(self): + self.load(validate=validate) + + @staticmethod + def get_global_config_dir(): from appdirs import user_config_dir - return user_config_dir(appname=self.APPNAME, - appauthor=self.APPAUTHOR) + return user_config_dir(appname=Config.APPNAME, + appauthor=Config.APPAUTHOR) - @property - def system_config_dir(self): + @staticmethod + def get_system_config_dir(): from appdirs import site_config_dir - return site_config_dir(appname=self.APPNAME, - appauthor=self.APPAUTHOR) + return site_config_dir(appname=Config.APPNAME, + appauthor=Config.APPAUTHOR) @staticmethod def init(dvc_dir): @@ -222,11 +232,20 @@ class Config(object): def _load(self): self._system_config = configobj.ConfigObj(self.system_config_file) self._global_config = configobj.ConfigObj(self.global_config_file) - self._project_config = configobj.ConfigObj(self.config_file) - self._local_config = configobj.ConfigObj(self.config_local_file) + + if self.config_file is not None: + self._project_config = configobj.ConfigObj(self.config_file) + else: + self._project_config = configobj.ConfigObj() + + if self.config_local_file is not None: + self._local_config = configobj.ConfigObj(self.config_local_file) + else: + self._local_config = configobj.ConfigObj() + self._config = None - def load(self): + def load(self, validate=True): self._load() try: self._config = configobj.ConfigObj(self.system_config_file) @@ -241,7 +260,8 @@ class Config(object): c = self._lower(c) self._config = self._merge(self._config, c) - self._config = Schema(self.SCHEMA).validate(self._config) + if validate: + self._config = Schema(self.SCHEMA).validate(self._config) # NOTE: now converting back to ConfigObj self._config = configobj.ConfigObj(self._config, @@ -271,6 +291,9 @@ class Config(object): self._local_config] for conf in clist: + if conf.filename is None: + continue + try: Logger.debug("Writing '{}'.".format(conf.filename)) dname = os.path.dirname(os.path.abspath(conf.filename)) diff --git a/dvc/daemon.py b/dvc/daemon.py index f0d1775d3..a4d715707 100644 --- a/dvc/daemon.py +++ b/dvc/daemon.py @@ -42,13 +42,13 @@ class Daemon(object): # pragma: no cover close_fds=True, shell=False) - def __call__(self, name): + def __call__(self, args): from dvc.utils import is_binary cmd = [sys.executable] if not is_binary(): cmd += ['-m', 'dvc'] - cmd += ['daemon', name, '-q'] + cmd += ['daemon', '-q'] + args Logger.debug("Trying to spawn '{}'".format(cmd)) diff --git a/dvc/main.py b/dvc/main.py index 1ebc3e625..1a691f7cc 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -1,27 +1,35 @@ from dvc.logger import Logger from dvc.cli import parse_args from dvc.command.base import CmdBase +from dvc.analytics import Analytics +from dvc.cli import DvcParserError +from dvc.project import NotDvcProjectError def main(argv=None): Logger.init() - args = parse_args(argv) - - # Init loglevel early in case we'll run - # into errors before setting it properly - CmdBase._set_loglevel(args) - + args = None + cmd = None try: + args = parse_args(argv) + + # Init loglevel early in case we'll run + # into errors before setting it properly + CmdBase._set_loglevel(args) + cmd = args.func(args) - except Exception as ex: - Logger.error('Initialization error', ex) - return 255 - try: ret = cmd.run_cmd() + except NotDvcProjectError as ex: + Logger.error(str(ex)) + ret = 253 + except DvcParserError: + ret = 254 except Exception as ex: Logger.error('Unexpected error', ex) - return 254 + ret = 255 + + Analytics().send(detach=True, cmd=cmd, args=args, ret=ret) return ret diff --git a/dvc/project.py b/dvc/project.py index 0e261c492..99749a95a 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -10,6 +10,12 @@ class InitError(DvcException): super(InitError, self).__init__(msg) +class NotDvcProjectError(DvcException): + def __init__(self, root): + msg = "Not a dvc repository (checked up to mount point {})" + super(NotDvcProjectError, self).__init__(msg.format(root)) + + class ReproductionError(DvcException): def __init__(self, dvc_file_name, ex): self.path = dvc_file_name @@ -20,7 +26,7 @@ class ReproductionError(DvcException): class Project(object): DVC_DIR = '.dvc' - def __init__(self, root_dir): + def __init__(self, root_dir=None): from dvc.logger import Logger from dvc.config import Config from dvc.state import State @@ -31,11 +37,12 @@ class Project(object): from dvc.updater import Updater from dvc.prompt import Prompt + root_dir = self._find_root(root_dir) + self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.config = Config(self.dvc_dir) - self.config.load() self.scm = SCM(self.root_dir, project=self) self.lock = Lock(self.dvc_dir) @@ -57,6 +64,27 @@ class Project(object): self.updater.check() + @staticmethod + def _find_root(root=None): + if root is None: + root = os.getcwd() + else: + root = os.path.abspath(os.path.realpath(root)) + + while True: + dvc_dir = os.path.join(root, Project.DVC_DIR) + if os.path.isdir(dvc_dir): + return root + if os.path.ismount(root): + break + root = os.path.dirname(root) + raise NotDvcProjectError(root) + + @staticmethod + def _find_dvc_dir(root=None): + root_dir = Project._find_root(root) + return os.path.join(root_dir, Project.DVC_DIR) + def _remind_to_git_add(self): if len(self._files_to_git_add) == 0: return diff --git a/dvc/updater.py b/dvc/updater.py index 5e5cd3505..2e8cff96f 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -68,8 +68,7 @@ class Updater(object): # pragma: no cover from dvc.daemon import Daemon if detach: - d = Daemon() - d('updater') + Daemon()(['updater']) return self._with_lock(self._get_latest_version, 'fetching') diff --git a/tests/test_analytics.py b/tests/test_analytics.py new file mode 100644 index 000000000..4ff9a94cd --- /dev/null +++ b/tests/test_analytics.py @@ -0,0 +1,62 @@ +import os +import mock +import requests +import tempfile + +from dvc.main import main +from dvc.analytics import Analytics +from tests.basic_env import TestDvc, TestGit, TestDir + + +def _clean_getenv(key, default=None): + """ + Remove env vars that affect dvc behavior in tests + """ + if key in ['DVC_TEST', 'CI']: + return None + return os.environ.get(key, default) + + +class TestAnalytics(TestDir): + def test(self): + a = Analytics() + a._collect() + self.assertTrue(isinstance(a.info, dict)) + self.assertNotEqual(a.info, {}) + self.assertTrue(a.PARAM_USER_ID in a.info.keys()) + self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys()) + self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {}) + + @mock.patch.object(os, 'getenv', new=_clean_getenv) + @mock.patch('requests.post') + def test_send(self, mockpost): + ret = main(['daemon', 'analytics', Analytics().dump(), '-v']) + self.assertEqual(ret, 0) + + self.assertTrue(mockpost.called) + + @mock.patch.object(os, 'getenv', new=_clean_getenv) + @mock.patch.object(requests, 'post', + side_effect=requests.exceptions.RequestException()) + def test_send_failed(self, mockpost): + ret = main(['daemon', 'analytics', Analytics().dump(), '-v']) + self.assertEqual(ret, 0) + + self.assertTrue(mockpost.called) + + +class TestAnalyticsGit(TestAnalytics, TestGit): + pass + + +class TestAnalyticsDvc(TestAnalytics, TestDvc): + @mock.patch('requests.post') + def test_send_disabled(self, mockpost): + ret = main(['config', 'core.analytics', 'false']) + self.assertEqual(ret, 0) + + with mock.patch.object(os, 'getenv', new=_clean_getenv): + ret = main(['daemon', 'analytics', Analytics().dump(), '-v']) + self.assertEqual(ret, 0) + + self.assertFalse(mockpost.called)