analytics: send user events

Fixes #693

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/analytics.py

@@ -0,0 +1,211 @@
+import os
+import json
+import errno
+
+from dvc import VERSION
+from dvc.utils import is_binary
+from dvc.lock import Lock, LockError
+from dvc.logger import Logger
+
+
+class Analytics(object):
+    URL = 'https://dddc07bh9f.execute-api.us-east-2.amazonaws.com/production'
+    TIMEOUT_POST = 10
+
+    USER_ID_FILE = 'user_id'
+
+    PARAM_DVC_VERSION = 'dvc_version'
+    PARAM_USER_ID = 'user_id'
+    PARAM_SYSTEM_INFO = 'system_info'
+
+    PARAM_OS = 'os'
+
+    PARAM_WINDOWS_VERSION_MAJOR = 'windows_version_major'
+    PARAM_WINDOWS_VERSION_MINOR = 'windows_version_minor'
+    PARAM_WINDOWS_VERSION_BUILD = 'windows_version_build'
+    PARAM_WINDOWS_VERSION_SERVICE_PACK = 'windows_version_service_pack'
+
+    PARAM_MAC_VERSION = 'mac_version'
+
+    PARAM_LINUX_DISTRO = 'linux_distro'
+    PARAM_LINUX_DISTRO_VERSION = 'linux_distro_version'
+    PARAM_LINUX_DISTRO_LIKE = 'linux_distro_like'
+
+    PARAM_SCM_CLASS = 'scm_class'
+    PARAM_IS_BINARY = 'is_binary'
+    PARAM_CMD_CLASS = 'cmd_class'
+    PARAM_CMD_RETURN_CODE = 'cmd_return_code'
+
+    def __init__(self, info={}):
+        from dvc.config import Config
+
+        self.info = info
+
+        cdir = Config.get_global_config_dir()
+        try:
+            os.makedirs(cdir)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+        self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
+        self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + '.lock')
+
+    @staticmethod
+    def load(path):
+        with open(path, 'r') as fobj:
+            a = Analytics(info=json.load(fobj))
+        os.unlink(path)
+        return a
+
+    def _write_user_id(self):
+        import uuid
+
+        with open(self.user_id_file, 'w+') as fobj:
+            user_id = str(uuid.uuid4())
+            info = {self.PARAM_USER_ID: user_id}
+            json.dump(info, fobj)
+            return user_id
+
+    def _read_user_id(self):
+        if not os.path.exists(self.user_id_file):
+            return None
+
+        with open(self.user_id_file, 'r') as fobj:
+            try:
+                info = json.load(fobj)
+                return info[self.PARAM_USER_ID]
+            except json.JSONDecodeError as exc:
+                Logger.debug("Failed to load user_id: {}".format(exc))
+                return None
+
+    def _get_user_id(self):
+        try:
+            with self.user_id_file_lock:
+                user_id = self._read_user_id()
+                if user_id is None:
+                    user_id = self._write_user_id()
+                return user_id
+        except LockError:
+            msg = "Failed to acquire '{}'"
+            Logger.debug(msg.format(self.user_id_file_lock.lock_file))
+
+    def _collect_windows(self):
+        import sys
+        version = sys.getwindowsversion()
+        info = {}
+        info[self.PARAM_OS] = 'windows'
+        info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
+        info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
+        info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
+        info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
+        return info
+
+    def _collect_darwin(self):
+        import platform
+        info = {}
+        info[self.PARAM_OS] = 'mac'
+        info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
+        return info
+
+    def _collect_linux(self):
+        import distro
+        info = {}
+        info[self.PARAM_OS] = 'linux'
+        info[self.PARAM_LINUX_DISTRO] = distro.id()
+        info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
+        info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
+        return info
+
+    def _collect_system_info(self):
+        import platform
+        system = platform.system()
+        if system == 'Windows':
+            return self._collect_windows()
+        elif system == 'Darwin':
+            return self._collect_darwin()
+        elif system == 'Linux':
+            return self._collect_linux()
+        else:
+            raise NotImplementedError
+
+    def _collect(self, args=None, ret=None):
+        from dvc.scm import SCM
+        from dvc.project import Project, NotDvcProjectError
+        from dvc.command.daemon import CmdDaemonAnalytics
+
+        self.info[self.PARAM_DVC_VERSION] = VERSION
+        self.info[self.PARAM_IS_BINARY] = is_binary()
+        self.info[self.PARAM_USER_ID] = self._get_user_id()
+
+        self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
+
+        try:
+            scm = SCM(root_dir=Project._find_root())
+            self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
+        except NotDvcProjectError:
+            pass
+
+        if ret is not None:
+            self.info[self.PARAM_CMD_RETURN_CODE] = ret
+
+        if args is not None and hasattr(args, 'func'):
+            assert args.func != CmdDaemonAnalytics
+            self.info[self.PARAM_CMD_CLASS] = args.func.__name__
+
+    def dump(self):
+        import tempfile
+
+        with tempfile.NamedTemporaryFile(delete=False, mode='w') as fobj:
+            json.dump(self.info, fobj)
+            return fobj.name
+
+    @staticmethod
+    def _is_enabled(cmd):
+        from dvc.config import Config
+        from dvc.project import Project, NotDvcProjectError
+        from dvc.command.daemon import CmdDaemonBase
+
+        if os.getenv('DVC_TEST'):
+            return False
+
+        if isinstance(cmd, CmdDaemonBase):
+            return False
+
+        if cmd is None or not hasattr(cmd, 'config'):
+            try:
+                dvc_dir = Project._find_dvc_dir()
+                config = Config(dvc_dir)
+                assert config is not None
+            except NotDvcProjectError:
+                config = Config(validate=False)
+                assert config is not None
+        else:
+            config = cmd.config
+            assert config is not None
+
+        core = config._config.get(Config.SECTION_CORE, {})
+        enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
+        Logger.debug("Analytics is {}abled."
+                     .format('en' if enabled else 'dis'))
+        return enabled
+
+    def send(self, detach=True, cmd=None, args=None, ret=None):
+        import requests
+        from dvc.daemon import Daemon
+
+        if not self._is_enabled(cmd):
+            return
+
+        if detach:  # pragma: no cover
+            Daemon()(['analytics', self.dump()])
+            return
+
+        self._collect(args=args, ret=ret)
+
+        Logger.debug("Sending analytics: {}".format(self.info))
+
+        try:
+            requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
+        except requests.exceptions.RequestException as exc:
+            Logger.debug("Failed to send analytics: {}".format(str(exc)))

dvc/cli.py

@@ -26,7 +26,8 @@ from dvc.command.install import CmdInstall
 from dvc.command.root import CmdRoot
 from dvc.command.lock import CmdLock, CmdUnlock
 from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList
-from dvc.command.daemon import CmdDaemonUpdater
+from dvc.command.daemon import CmdDaemonUpdater, CmdDaemonAnalytics
+from dvc.exceptions import DvcException
 from dvc.logger import Logger
 from dvc import VERSION
 
@@ -41,11 +42,16 @@ def _fix_subparsers(subparsers):
         subparsers.dest = 'cmd'
 
 
+class DvcParserError(DvcException):
+    def __init__(self):
+        super(DvcException, self).__init__("Parser error")
+
+
 class DvcParser(argparse.ArgumentParser):
     def error(self, message):
         sys.stderr.write('{}{}\n'.format(Logger.error_prefix(), message))
         self.print_help()
-        sys.exit(2)
+        raise DvcParserError()
 
 
 class VersionAction(argparse.Action):  # pragma: no cover
@@ -1037,6 +1043,17 @@ def parse_args(argv=None):
                         help=DAEMON_UPDATER_HELP)
     daemon_updater_parser.set_defaults(func=CmdDaemonUpdater)
 
+    DAEMON_ANALYTICS_HELP = 'Send dvc usage analytics.'
+    daemon_analytics_parser = daemon_subparsers.add_parser(
+                        'analytics',
+                        parents=[parent_parser],
+                        description=DAEMON_ANALYTICS_HELP,
+                        help=DAEMON_ANALYTICS_HELP)
+    daemon_analytics_parser.add_argument(
+                        'target',
+                        help="Analytics file.")
+    daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)
+
     args = parser.parse_args(argv)
 
     if (issubclass(args.func, CmdRepro)

dvc/command/base.py

@@ -1,7 +1,4 @@
-import os
-
 from dvc.logger import Logger
-from dvc.exceptions import DvcException
 from dvc.lock import LockError
 
 
@@ -9,7 +6,8 @@ class CmdBase(object):
     def __init__(self, args):
         from dvc.project import Project
 
-        self.project = Project(self._find_root())
+        self.project = Project()
+        self.config = self.project.config
         self.args = args
         self._set_loglevel(args)
 
@@ -21,20 +19,6 @@ class CmdBase(object):
         elif args.verbose:
             Logger.be_verbose()
 
-    def _find_root(self):
-        from dvc.project import Project
-
-        root = os.getcwd()
-        while True:
-            dvc_dir = os.path.join(root, Project.DVC_DIR)
-            if os.path.isdir(dvc_dir):
-                return root
-            if os.path.ismount(root):
-                break
-            root = os.path.dirname(root)
-        msg = "Not a dvc repository (checked up to mount point {})"
-        raise DvcException(msg.format(root))
-
     def run_cmd(self):
         try:
             with self.project.lock:

dvc/command/config.py

@@ -8,18 +8,29 @@ from dvc.exceptions import DvcException
 
 class CmdConfig(CmdBase):
     def __init__(self, args):
-        from dvc.project import Project
+        from dvc.project import Project, NotDvcProjectError
 
         self.args = args
-        dvc_dir = os.path.join(self._find_root(), Project.DVC_DIR)
-        self.config = Config(dvc_dir)
+
+        try:
+            dvc_dir = os.path.join(Project._find_root(), Project.DVC_DIR)
+            saved_exc = None
+        except NotDvcProjectError as exc:
+            dvc_dir = None
+            saved_exc = exc
+
+        self.config = Config(dvc_dir, validate=False)
         if self.args.system:
             self.configobj = self.config._system_config
         elif self.args.glob:
             self.configobj = self.config._global_config
         elif self.args.local:
+            if dvc_dir is None:
+                raise saved_exc
             self.configobj = self.config._local_config
         else:
+            if dvc_dir is None:
+                raise saved_exc
             self.configobj = self.config._project_config
 
     def run_cmd(self):

dvc/command/daemon.py

@@ -1,19 +1,35 @@
 from dvc.command.base import CmdBase
 
 
-class CmdDaemonUpdater(CmdBase):
+class CmdDaemonBase(CmdBase):
     def __init__(self, args):
-        pass
+        self.args = args
+        self.config = None
+        self._set_loglevel(args)
 
     def run_cmd(self):
         return self.run()
 
+
+class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
         from dvc.project import Project
         from dvc.updater import Updater
 
-        root_dir = self._find_root()
+        root_dir = Project._find_root()
         dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
         updater = Updater(dvc_dir)
         updater.fetch(detach=False)
+
+        return 0
+
+
+class CmdDaemonAnalytics(CmdDaemonBase):
+    def run(self):
+        from dvc.analytics import Analytics
+
+        analytics = Analytics.load(self.args.target)
+        analytics.send(detach=False)
+
+        return 0

dvc/command/init.py

@@ -9,7 +9,10 @@ class CmdInit(object):
         from dvc.project import Project, InitError
 
         try:
-            Project.init('.', no_scm=self.args.no_scm, force=self.args.force)
+            self.project = Project.init('.',
+                                        no_scm=self.args.no_scm,
+                                        force=self.args.force)
+            self.config = self.project.config
         except InitError as e:
             Logger.error('Failed to initiate dvc', e)
             return 1

dvc/config.py

@@ -68,6 +68,8 @@ class Config(object):
     SECTION_CORE_REMOTE = 'remote'
     SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
     SECTION_CORE_INTERACTIVE = 'interactive'
+    SECTION_CORE_ANALYTICS = 'analytics'
+    SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool))
 
     SECTION_CACHE = 'cache'
     SECTION_CACHE_DIR = 'dir'
@@ -106,6 +108,8 @@ class Config(object):
         Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)),
         Optional(SECTION_CORE_INTERACTIVE,
                  default=False): SECTION_CORE_INTERACTIVE_SCHEMA,
+        Optional(SECTION_CORE_ANALYTICS,
+                 default=True): SECTION_CORE_ANALYTICS_SCHEMA,
 
         # backward compatibility
         Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
@@ -190,28 +194,34 @@ class Config(object):
         Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
     }
 
-    def __init__(self, dvc_dir):
-        self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
-        self.system_config_file = os.path.join(self.system_config_dir,
+    def __init__(self, dvc_dir=None, validate=True):
+        self.system_config_file = os.path.join(self.get_system_config_dir(),
                                                self.CONFIG)
-        self.global_config_file = os.path.join(self.global_config_dir,
+        self.global_config_file = os.path.join(self.get_global_config_dir(),
                                                self.CONFIG)
-        self.config_file = os.path.join(dvc_dir, self.CONFIG)
-        self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
 
-        self._load()
+        if dvc_dir is not None:
+            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
+            self.config_file = os.path.join(dvc_dir, self.CONFIG)
+            self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
+        else:
+            self.dvc_dir = None
+            self.config_file = None
+            self.config_local_file = None
 
-    @property
-    def global_config_dir(self):
+        self.load(validate=validate)
+
+    @staticmethod
+    def get_global_config_dir():
         from appdirs import user_config_dir
-        return user_config_dir(appname=self.APPNAME,
-                               appauthor=self.APPAUTHOR)
+        return user_config_dir(appname=Config.APPNAME,
+                               appauthor=Config.APPAUTHOR)
 
-    @property
-    def system_config_dir(self):
+    @staticmethod
+    def get_system_config_dir():
         from appdirs import site_config_dir
-        return site_config_dir(appname=self.APPNAME,
-                               appauthor=self.APPAUTHOR)
+        return site_config_dir(appname=Config.APPNAME,
+                               appauthor=Config.APPAUTHOR)
 
     @staticmethod
     def init(dvc_dir):
@@ -222,11 +232,20 @@ class Config(object):
     def _load(self):
         self._system_config = configobj.ConfigObj(self.system_config_file)
         self._global_config = configobj.ConfigObj(self.global_config_file)
-        self._project_config = configobj.ConfigObj(self.config_file)
-        self._local_config = configobj.ConfigObj(self.config_local_file)
+
+        if self.config_file is not None:
+            self._project_config = configobj.ConfigObj(self.config_file)
+        else:
+            self._project_config = configobj.ConfigObj()
+
+        if self.config_local_file is not None:
+            self._local_config = configobj.ConfigObj(self.config_local_file)
+        else:
+            self._local_config = configobj.ConfigObj()
+
         self._config = None
 
-    def load(self):
+    def load(self, validate=True):
         self._load()
         try:
             self._config = configobj.ConfigObj(self.system_config_file)
@@ -241,7 +260,8 @@ class Config(object):
                 c = self._lower(c)
                 self._config = self._merge(self._config, c)
 
-            self._config = Schema(self.SCHEMA).validate(self._config)
+            if validate:
+                self._config = Schema(self.SCHEMA).validate(self._config)
 
             # NOTE: now converting back to ConfigObj
             self._config = configobj.ConfigObj(self._config,
@@ -271,6 +291,9 @@ class Config(object):
                      self._local_config]
 
         for conf in clist:
+            if conf.filename is None:
+                continue
+
             try:
                 Logger.debug("Writing '{}'.".format(conf.filename))
                 dname = os.path.dirname(os.path.abspath(conf.filename))

dvc/daemon.py

@@ -42,13 +42,13 @@ class Daemon(object):  # pragma: no cover
               close_fds=True,
               shell=False)
 
-    def __call__(self, name):
+    def __call__(self, args):
         from dvc.utils import is_binary
 
         cmd = [sys.executable]
         if not is_binary():
             cmd += ['-m', 'dvc']
-        cmd += ['daemon', name, '-q']
+        cmd += ['daemon', '-q'] + args
 
         Logger.debug("Trying to spawn '{}'".format(cmd))
 

dvc/main.py

@@ -1,27 +1,35 @@
 from dvc.logger import Logger
 from dvc.cli import parse_args
 from dvc.command.base import CmdBase
+from dvc.analytics import Analytics
+from dvc.cli import DvcParserError
+from dvc.project import NotDvcProjectError
 
 
 def main(argv=None):
     Logger.init()
 
-    args = parse_args(argv)
-
-    # Init loglevel early in case we'll run
-    # into errors before setting it properly
-    CmdBase._set_loglevel(args)
-
+    args = None
+    cmd = None
     try:
+        args = parse_args(argv)
+
+        # Init loglevel early in case we'll run
+        # into errors before setting it properly
+        CmdBase._set_loglevel(args)
+
         cmd = args.func(args)
-    except Exception as ex:
-        Logger.error('Initialization error', ex)
-        return 255
 
-    try:
         ret = cmd.run_cmd()
+    except NotDvcProjectError as ex:
+        Logger.error(str(ex))
+        ret = 253
+    except DvcParserError:
+        ret = 254
     except Exception as ex:
         Logger.error('Unexpected error', ex)
-        return 254
+        ret = 255
+
+    Analytics().send(detach=True, cmd=cmd, args=args, ret=ret)
 
     return ret

dvc/project.py

@@ -10,6 +10,12 @@ class InitError(DvcException):
         super(InitError, self).__init__(msg)
 
 
+class NotDvcProjectError(DvcException):
+    def __init__(self, root):
+        msg = "Not a dvc repository (checked up to mount point {})"
+        super(NotDvcProjectError, self).__init__(msg.format(root))
+
+
 class ReproductionError(DvcException):
     def __init__(self, dvc_file_name, ex):
         self.path = dvc_file_name
@@ -20,7 +26,7 @@ class ReproductionError(DvcException):
 class Project(object):
     DVC_DIR = '.dvc'
 
-    def __init__(self, root_dir):
+    def __init__(self, root_dir=None):
         from dvc.logger import Logger
         from dvc.config import Config
         from dvc.state import State
@@ -31,11 +37,12 @@ class Project(object):
         from dvc.updater import Updater
         from dvc.prompt import Prompt
 
+        root_dir = self._find_root(root_dir)
+
         self.root_dir = os.path.abspath(os.path.realpath(root_dir))
         self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
 
         self.config = Config(self.dvc_dir)
-        self.config.load()
 
         self.scm = SCM(self.root_dir, project=self)
         self.lock = Lock(self.dvc_dir)
@@ -57,6 +64,27 @@ class Project(object):
 
         self.updater.check()
 
+    @staticmethod
+    def _find_root(root=None):
+        if root is None:
+            root = os.getcwd()
+        else:
+            root = os.path.abspath(os.path.realpath(root))
+
+        while True:
+            dvc_dir = os.path.join(root, Project.DVC_DIR)
+            if os.path.isdir(dvc_dir):
+                return root
+            if os.path.ismount(root):
+                break
+            root = os.path.dirname(root)
+        raise NotDvcProjectError(root)
+
+    @staticmethod
+    def _find_dvc_dir(root=None):
+        root_dir = Project._find_root(root)
+        return os.path.join(root_dir, Project.DVC_DIR)
+
     def _remind_to_git_add(self):
         if len(self._files_to_git_add) == 0:
             return

dvc/updater.py

@@ -68,8 +68,7 @@ class Updater(object):  # pragma: no cover
         from dvc.daemon import Daemon
 
         if detach:
-            d = Daemon()
-            d('updater')
+            Daemon()(['updater'])
             return
 
         self._with_lock(self._get_latest_version, 'fetching')

tests/test_analytics.py

@@ -0,0 +1,62 @@
+import os
+import mock
+import requests
+import tempfile
+
+from dvc.main import main
+from dvc.analytics import Analytics
+from tests.basic_env import TestDvc, TestGit, TestDir
+
+
+def _clean_getenv(key, default=None):
+    """
+    Remove env vars that affect dvc behavior in tests
+    """
+    if key in ['DVC_TEST', 'CI']:
+        return None
+    return os.environ.get(key, default)
+
+
+class TestAnalytics(TestDir):
+    def test(self):
+        a = Analytics()
+        a._collect()
+        self.assertTrue(isinstance(a.info, dict))
+        self.assertNotEqual(a.info, {})
+        self.assertTrue(a.PARAM_USER_ID in a.info.keys())
+        self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys())
+        self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {})
+
+    @mock.patch.object(os, 'getenv', new=_clean_getenv)
+    @mock.patch('requests.post')
+    def test_send(self, mockpost):
+        ret = main(['daemon', 'analytics', Analytics().dump(), '-v'])
+        self.assertEqual(ret, 0)
+
+        self.assertTrue(mockpost.called)
+
+    @mock.patch.object(os, 'getenv', new=_clean_getenv)
+    @mock.patch.object(requests, 'post',
+                       side_effect=requests.exceptions.RequestException())
+    def test_send_failed(self, mockpost):
+        ret = main(['daemon', 'analytics', Analytics().dump(), '-v'])
+        self.assertEqual(ret, 0)
+
+        self.assertTrue(mockpost.called)
+
+
+class TestAnalyticsGit(TestAnalytics, TestGit):
+    pass
+
+
+class TestAnalyticsDvc(TestAnalytics, TestDvc):
+    @mock.patch('requests.post')
+    def test_send_disabled(self, mockpost):
+        ret = main(['config', 'core.analytics', 'false'])
+        self.assertEqual(ret, 0)
+
+        with mock.patch.object(os, 'getenv', new=_clean_getenv):
+            ret = main(['daemon', 'analytics', Analytics().dump(), '-v'])
+        self.assertEqual(ret, 0)
+
+        self.assertFalse(mockpost.called)