Merge pull request #1395 from efiop/master

Implement opt-out user analytics

dvc/analytics.py

@@ -0,0 +1,211 @@
+import os
+import json
+import errno
+
+from dvc import VERSION
+from dvc.utils import is_binary
+from dvc.lock import Lock, LockError
+from dvc.logger import Logger
+
+
+class Analytics(object):
+    URL = 'https://dddc07bh9f.execute-api.us-east-2.amazonaws.com/production'
+    TIMEOUT_POST = 10
+
+    USER_ID_FILE = 'user_id'
+
+    PARAM_DVC_VERSION = 'dvc_version'
+    PARAM_USER_ID = 'user_id'
+    PARAM_SYSTEM_INFO = 'system_info'
+
+    PARAM_OS = 'os'
+
+    PARAM_WINDOWS_VERSION_MAJOR = 'windows_version_major'
+    PARAM_WINDOWS_VERSION_MINOR = 'windows_version_minor'
+    PARAM_WINDOWS_VERSION_BUILD = 'windows_version_build'
+    PARAM_WINDOWS_VERSION_SERVICE_PACK = 'windows_version_service_pack'
+
+    PARAM_MAC_VERSION = 'mac_version'
+
+    PARAM_LINUX_DISTRO = 'linux_distro'
+    PARAM_LINUX_DISTRO_VERSION = 'linux_distro_version'
+    PARAM_LINUX_DISTRO_LIKE = 'linux_distro_like'
+
+    PARAM_SCM_CLASS = 'scm_class'
+    PARAM_IS_BINARY = 'is_binary'
+    PARAM_CMD_CLASS = 'cmd_class'
+    PARAM_CMD_RETURN_CODE = 'cmd_return_code'
+
+    def __init__(self, info={}):
+        from dvc.config import Config
+
+        self.info = info
+
+        cdir = Config.get_global_config_dir()
+        try:
+            os.makedirs(cdir)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+        self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
+        self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + '.lock')
+
+    @staticmethod
+    def load(path):
+        with open(path, 'r') as fobj:
+            a = Analytics(info=json.load(fobj))
+        os.unlink(path)
+        return a
+
+    def _write_user_id(self):
+        import uuid
+
+        with open(self.user_id_file, 'w+') as fobj:
+            user_id = str(uuid.uuid4())
+            info = {self.PARAM_USER_ID: user_id}
+            json.dump(info, fobj)
+            return user_id
+
+    def _read_user_id(self):
+        if not os.path.exists(self.user_id_file):
+            return None
+
+        with open(self.user_id_file, 'r') as fobj:
+            try:
+                info = json.load(fobj)
+                return info[self.PARAM_USER_ID]
+            except json.JSONDecodeError as exc:
+                Logger.debug("Failed to load user_id: {}".format(exc))
+                return None
+
+    def _get_user_id(self):
+        try:
+            with self.user_id_file_lock:
+                user_id = self._read_user_id()
+                if user_id is None:
+                    user_id = self._write_user_id()
+                return user_id
+        except LockError:
+            msg = "Failed to acquire '{}'"
+            Logger.debug(msg.format(self.user_id_file_lock.lock_file))
+
+    def _collect_windows(self):
+        import sys
+        version = sys.getwindowsversion()
+        info = {}
+        info[self.PARAM_OS] = 'windows'
+        info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
+        info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
+        info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
+        info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
+        return info
+
+    def _collect_darwin(self):
+        import platform
+        info = {}
+        info[self.PARAM_OS] = 'mac'
+        info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
+        return info
+
+    def _collect_linux(self):
+        import distro
+        info = {}
+        info[self.PARAM_OS] = 'linux'
+        info[self.PARAM_LINUX_DISTRO] = distro.id()
+        info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
+        info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
+        return info
+
+    def _collect_system_info(self):
+        import platform
+        system = platform.system()
+        if system == 'Windows':
+            return self._collect_windows()
+        elif system == 'Darwin':
+            return self._collect_darwin()
+        elif system == 'Linux':
+            return self._collect_linux()
+        else:
+            raise NotImplementedError
+
+    def _collect(self, args=None, ret=None):
+        from dvc.scm import SCM
+        from dvc.project import Project, NotDvcProjectError
+        from dvc.command.daemon import CmdDaemonAnalytics
+
+        self.info[self.PARAM_DVC_VERSION] = VERSION
+        self.info[self.PARAM_IS_BINARY] = is_binary()
+        self.info[self.PARAM_USER_ID] = self._get_user_id()
+
+        self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
+
+        try:
+            scm = SCM(root_dir=Project._find_root())
+            self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
+        except NotDvcProjectError:
+            pass
+
+        if ret is not None:
+            self.info[self.PARAM_CMD_RETURN_CODE] = ret
+
+        if args is not None and hasattr(args, 'func'):
+            assert args.func != CmdDaemonAnalytics
+            self.info[self.PARAM_CMD_CLASS] = args.func.__name__
+
+    def dump(self):
+        import tempfile
+
+        with tempfile.NamedTemporaryFile(delete=False, mode='w') as fobj:
+            json.dump(self.info, fobj)
+            return fobj.name
+
+    @staticmethod
+    def _is_enabled(cmd):
+        from dvc.config import Config
+        from dvc.project import Project, NotDvcProjectError
+        from dvc.command.daemon import CmdDaemonBase
+
+        if os.getenv('DVC_TEST'):
+            return False
+
+        if isinstance(cmd, CmdDaemonBase):
+            return False
+
+        if cmd is None or not hasattr(cmd, 'config'):
+            try:
+                dvc_dir = Project._find_dvc_dir()
+                config = Config(dvc_dir)
+                assert config is not None
+            except NotDvcProjectError:
+                config = Config(validate=False)
+                assert config is not None
+        else:
+            config = cmd.config
+            assert config is not None
+
+        core = config._config.get(Config.SECTION_CORE, {})
+        enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
+        Logger.debug("Analytics is {}abled."
+                     .format('en' if enabled else 'dis'))
+        return enabled
+
+    def send(self, detach=True, cmd=None, args=None, ret=None):
+        import requests
+        from dvc.daemon import Daemon
+
+        if not self._is_enabled(cmd):
+            return
+
+        if detach:  # pragma: no cover
+            Daemon()(['analytics', self.dump()])
+            return
+
+        self._collect(args=args, ret=ret)
+
+        Logger.debug("Sending analytics: {}".format(self.info))
+
+        try:
+            requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
+        except requests.exceptions.RequestException as exc:
+            Logger.debug("Failed to send analytics: {}".format(str(exc)))

dvc/cli.py

@@ -26,7 +26,8 @@ from dvc.command.install import CmdInstall
 from dvc.command.root import CmdRoot
 from dvc.command.lock import CmdLock, CmdUnlock
 from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList
-from dvc.command.daemon import CmdDaemonUpdater
+from dvc.command.daemon import CmdDaemonUpdater, CmdDaemonAnalytics
+from dvc.exceptions import DvcException
 from dvc.logger import Logger
 from dvc import VERSION
 
@@ -41,11 +42,16 @@ def _fix_subparsers(subparsers):
         subparsers.dest = 'cmd'
 
 
+class DvcParserError(DvcException):
+    def __init__(self):
+        super(DvcException, self).__init__("Parser error")
+
+
 class DvcParser(argparse.ArgumentParser):
     def error(self, message):
         sys.stderr.write('{}{}\n'.format(Logger.error_prefix(), message))
         self.print_help()
-        sys.exit(2)
+        raise DvcParserError()
 
 
 class VersionAction(argparse.Action):  # pragma: no cover
@@ -638,6 +644,17 @@ def parse_args(argv=None):
                         nargs='?',
                         default=None,
                         help='Option value.')
+    config_parser.add_argument(
+                        '--global',
+                        dest='glob',
+                        action='store_true',
+                        default=False,
+                        help='Use global config.')
+    config_parser.add_argument(
+                        '--system',
+                        action='store_true',
+                        default=False,
+                        help='Use system config.')
     config_parser.add_argument(
                         '--local',
                         action='store_true',
@@ -672,6 +689,17 @@ def parse_args(argv=None):
     remote_add_parser.add_argument(
                         'url',
                         help='URL.')
+    remote_add_parser.add_argument(
+                        '--global',
+                        dest='glob',
+                        action='store_true',
+                        default=False,
+                        help='Use global config.')
+    remote_add_parser.add_argument(
+                        '--system',
+                        action='store_true',
+                        default=False,
+                        help='Use system config.')
     remote_add_parser.add_argument(
                         '--local',
                         action='store_true',
@@ -694,6 +722,17 @@ def parse_args(argv=None):
     remote_remove_parser.add_argument(
                         'name',
                         help='Name')
+    remote_remove_parser.add_argument(
+                        '--global',
+                        dest='glob',
+                        action='store_true',
+                        default=False,
+                        help='Use global config.')
+    remote_remove_parser.add_argument(
+                        '--system',
+                        action='store_true',
+                        default=False,
+                        help='Use system config.')
     remote_remove_parser.add_argument(
                         '--local',
                         action='store_true',
@@ -723,6 +762,17 @@ def parse_args(argv=None):
                         default=False,
                         action='store_true',
                         help='Unset option.')
+    remote_modify_parser.add_argument(
+                        '--global',
+                        dest='glob',
+                        action='store_true',
+                        default=False,
+                        help='Use global config.')
+    remote_modify_parser.add_argument(
+                        '--system',
+                        action='store_true',
+                        default=False,
+                        help='Use system config.')
     remote_modify_parser.add_argument(
                         '--local',
                         action='store_true',
@@ -736,6 +786,17 @@ def parse_args(argv=None):
                         parents=[parent_parser],
                         description=REMOTE_LIST_HELP,
                         help=REMOTE_LIST_HELP)
+    remote_list_parser.add_argument(
+                        '--global',
+                        dest='glob',
+                        action='store_true',
+                        default=False,
+                        help='Use global config.')
+    remote_list_parser.add_argument(
+                        '--system',
+                        action='store_true',
+                        default=False,
+                        help='Use system config.')
     remote_list_parser.add_argument(
                         '--local',
                         action='store_true',
@@ -982,6 +1043,17 @@ def parse_args(argv=None):
                         help=DAEMON_UPDATER_HELP)
     daemon_updater_parser.set_defaults(func=CmdDaemonUpdater)
 
+    DAEMON_ANALYTICS_HELP = 'Send dvc usage analytics.'
+    daemon_analytics_parser = daemon_subparsers.add_parser(
+                        'analytics',
+                        parents=[parent_parser],
+                        description=DAEMON_ANALYTICS_HELP,
+                        help=DAEMON_ANALYTICS_HELP)
+    daemon_analytics_parser.add_argument(
+                        'target',
+                        help="Analytics file.")
+    daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)
+
     args = parser.parse_args(argv)
 
     if (issubclass(args.func, CmdRepro)

dvc/command/base.py

@@ -1,7 +1,4 @@
-import os
-
 from dvc.logger import Logger
-from dvc.exceptions import DvcException
 from dvc.lock import LockError
 
 
@@ -9,7 +6,8 @@ class CmdBase(object):
     def __init__(self, args):
         from dvc.project import Project
 
-        self.project = Project(self._find_root())
+        self.project = Project()
+        self.config = self.project.config
         self.args = args
         self._set_loglevel(args)
 
@@ -21,20 +19,6 @@ class CmdBase(object):
         elif args.verbose:
             Logger.be_verbose()
 
-    def _find_root(self):
-        from dvc.project import Project
-
-        root = os.getcwd()
-        while True:
-            dvc_dir = os.path.join(root, Project.DVC_DIR)
-            if os.path.isdir(dvc_dir):
-                return root
-            if os.path.ismount(root):
-                break
-            root = os.path.dirname(root)
-        msg = "Not a dvc repository (checked up to mount point {})"
-        raise DvcException(msg.format(root))
-
     def run_cmd(self):
         try:
             with self.project.lock:

dvc/command/config.py

@@ -1,102 +1,79 @@
 import os
-import configobj
 
 from dvc.command.base import CmdBase
 from dvc.logger import Logger
 from dvc.config import Config
+from dvc.exceptions import DvcException
 
 
 class CmdConfig(CmdBase):
     def __init__(self, args):
-        from dvc.project import Project
+        from dvc.project import Project, NotDvcProjectError
 
         self.args = args
-        root_dir = self._find_root()
-        if args.local:
-            config = Config.CONFIG_LOCAL
+
+        try:
+            dvc_dir = os.path.join(Project._find_root(), Project.DVC_DIR)
+            saved_exc = None
+        except NotDvcProjectError as exc:
+            dvc_dir = None
+            saved_exc = exc
+
+        self.config = Config(dvc_dir, validate=False)
+        if self.args.system:
+            self.configobj = self.config._system_config
+        elif self.args.glob:
+            self.configobj = self.config._global_config
+        elif self.args.local:
+            if dvc_dir is None:
+                raise saved_exc
+            self.configobj = self.config._local_config
         else:
-            config = Config.CONFIG
-        self.config_file = os.path.join(root_dir, Project.DVC_DIR, config)
-        # Using configobj because it doesn't
-        # drop comments like configparser does.
-        self.configobj = configobj.ConfigObj(self.config_file)
+            if dvc_dir is None:
+                raise saved_exc
+            self.configobj = self.config._project_config
 
     def run_cmd(self):
         return self.run()
 
-    def _get_key(self, d, name, add=False):
-        for k in d.keys():
-            if k.lower() == name.lower():
-                return k
+    def _unset(self, section, opt=None, configobj=None):
+        if configobj is None:
+            configobj = self.configobj
 
-        if add:
-            d[name] = {}
-            return name
-
-        return None
-
-    def save(self):
         try:
-            self.configobj.write()
-        except Exception as exc:
-            msg = "Failed to write config '{}'".format(self.configobj.filename)
-            Logger.error(msg, exc)
+            self.config.unset(configobj, section, opt)
+            self.config.save(configobj)
+        except DvcException as exc:
+            Logger.error("Failed to unset '{}'".format(self.args.name), exc)
             return 1
         return 0
 
-    def unset(self, section, opt=None):
-        if section not in self.configobj.keys():
-            Logger.error("Section '{}' doesn't exist".format(section))
+    def _show(self, section, opt):
+        try:
+            self.config.show(self.configobj, section, opt)
+        except DvcException as exc:
+            Logger.error("Failed to show '{}'".format(self.args.name), exc)
             return 1
-
-        if opt in self.configobj[section].keys():
-            del self.configobj[section][opt]
-
-        if len(self.configobj[section]) == 0 or opt is None:
-            del self.configobj[section]
-
-        return self.save()
-
-    def show(self):
-        Logger.info(self.configobj[self.section][self.opt])
         return 0
 
-    def set(self, section, opt, value):
-        if section not in self.configobj.keys():
-            self.configobj[section] = {}
-
-        self.configobj[section][opt] = value
-
-        return self.save()
-
-    def check_opt(self):
-        _section, _opt = self.args.name.strip().split('.', 1)
-        add = (self.args.value is not None and self.args.unset is False)
-
-        section = self._get_key(self.configobj, _section, add)
-
-        if not section:
-            Logger.error('Invalid option name {}'.format(_section))
+    def _set(self, section, opt, value):
+        try:
+            self.config.set(self.configobj, section, opt, value)
+            self.config.save(self.configobj)
+        except DvcException as exc:
+            Logger.error("Failed to set '{}.{}' to '{}'".format(section,
+                                                                opt,
+                                                                value),
+                         exc)
             return 1
-
-        opt = self._get_key(self.configobj[section], _opt, add)
-        if not opt:
-            Logger.error('Invalid option value: {}'.format(_opt))
-            return 1
-
-        self.section = section
-        self.opt = opt
-
         return 0
 
     def run(self):
-        if self.check_opt() != 0:
-            return 1
+        section, opt = self.args.name.lower().strip().split('.', 1)
 
         if self.args.unset:
-            return self.unset(self.section, self.opt)
-
-        if self.args.value is None:
-            return self.show()
-
-        return self.set(self.section, self.opt, self.args.value)
+            return self._unset(section, opt)
+        elif self.args.value is None:
+            return self._show(section, opt)
+        else:
+            return self._set(section, opt, self.args.value)

dvc/command/daemon.py

@@ -1,19 +1,35 @@
 from dvc.command.base import CmdBase
 
 
-class CmdDaemonUpdater(CmdBase):
+class CmdDaemonBase(CmdBase):
     def __init__(self, args):
-        pass
+        self.args = args
+        self.config = None
+        self._set_loglevel(args)
 
     def run_cmd(self):
         return self.run()
 
+
+class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
         from dvc.project import Project
         from dvc.updater import Updater
 
-        root_dir = self._find_root()
+        root_dir = Project._find_root()
         dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
         updater = Updater(dvc_dir)
         updater.fetch(detach=False)
+
+        return 0
+
+
+class CmdDaemonAnalytics(CmdDaemonBase):
+    def run(self):
+        from dvc.analytics import Analytics
+
+        analytics = Analytics.load(self.args.target)
+        analytics.send(detach=False)
+
+        return 0

dvc/command/init.py

@@ -9,7 +9,10 @@ class CmdInit(object):
         from dvc.project import Project, InitError
 
         try:
-            Project.init('.', no_scm=self.args.no_scm, force=self.args.force)
+            self.project = Project.init('.',
+                                        no_scm=self.args.no_scm,
+                                        force=self.args.force)
+            self.config = self.project.config
         except InitError as e:
             Logger.error('Failed to initiate dvc', e)
             return 1

dvc/command/remote.py

@@ -1,6 +1,4 @@
-import os
 import re
-import configobj
 
 from dvc.config import Config
 from dvc.command.config import CmdConfig
@@ -10,49 +8,50 @@ from dvc.logger import Logger
 class CmdRemoteAdd(CmdConfig):
     def run(self):
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        ret = self.set(section, Config.SECTION_REMOTE_URL, self.args.url)
+        ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
         if ret != 0:
             return ret
 
         if self.args.default:
             msg = 'Setting \'{}\' as a default remote.'.format(self.args.name)
             Logger.info(msg)
-            ret = self.set(Config.SECTION_CORE,
-                           Config.SECTION_CORE_REMOTE,
-                           self.args.name)
+            ret = self._set(Config.SECTION_CORE,
+                            Config.SECTION_CORE_REMOTE,
+                            self.args.name)
 
         return ret
 
 
 class CmdRemoteRemove(CmdConfig):
-    def _remove_default(self, config_file, remote):
-        path = os.path.join(os.path.dirname(self.config_file),
-                            config_file)
-        config = configobj.ConfigObj(path)
-
+    def _remove_default(self, config):
         core = config.get(Config.SECTION_CORE, None)
         if core is None:
-            return
+            return 0
 
         default = core.get(Config.SECTION_CORE_REMOTE, None)
         if default is None:
-            return
+            return 0
 
-        if default == remote:
-            del config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE]
-            if len(config[Config.SECTION_CORE]) == 0:
-                del config[Config.SECTION_CORE]
-
-        config.write()
+        if default == self.args.name:
+            return self._unset(Config.SECTION_CORE,
+                               opt=Config.SECTION_CORE_REMOTE,
+                               configobj=config)
 
     def run(self):
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        ret = self.unset(section)
+        ret = self._unset(section)
         if ret != 0:
             return ret
 
-        self._remove_default(Config.CONFIG, self.args.name)
-        self._remove_default(Config.CONFIG_LOCAL, self.args.name)
+        for configobj in [self.config._local_config,
+                          self.config._project_config,
+                          self.config._global_config,
+                          self.config._system_config]:
+            self._remove_default(configobj)
+            self.config.save(configobj)
+            if configobj == self.configobj:
+                break
+
         return 0
 
 

dvc/config.py

@@ -2,9 +2,11 @@
 DVC config objects.
 """
 import os
+import errno
 import configobj
 from schema import Schema, Optional, And, Use, Regex
 
+from dvc.logger import Logger
 from dvc.exceptions import DvcException
 
 
@@ -54,6 +56,9 @@ def is_percent(val):
 
 
 class Config(object):
+    APPNAME = 'dvc'
+    APPAUTHOR = 'iterative'
+
     CONFIG = 'config'
     CONFIG_LOCAL = 'config.local'
 
@@ -63,6 +68,8 @@ class Config(object):
     SECTION_CORE_REMOTE = 'remote'
     SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
     SECTION_CORE_INTERACTIVE = 'interactive'
+    SECTION_CORE_ANALYTICS = 'analytics'
+    SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool))
 
     SECTION_CACHE = 'cache'
     SECTION_CACHE_DIR = 'dir'
@@ -101,6 +108,8 @@ class Config(object):
         Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)),
         Optional(SECTION_CORE_INTERACTIVE,
                  default=False): SECTION_CORE_INTERACTIVE_SCHEMA,
+        Optional(SECTION_CORE_ANALYTICS,
+                 default=True): SECTION_CORE_ANALYTICS_SCHEMA,
 
         # backward compatibility
         Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
@@ -185,22 +194,74 @@ class Config(object):
         Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
     }
 
-    def __init__(self, dvc_dir):
-        self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
-        self.config_file = os.path.join(dvc_dir, self.CONFIG)
-        self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
+    def __init__(self, dvc_dir=None, validate=True):
+        self.system_config_file = os.path.join(self.get_system_config_dir(),
+                                               self.CONFIG)
+        self.global_config_file = os.path.join(self.get_global_config_dir(),
+                                               self.CONFIG)
 
+        if dvc_dir is not None:
+            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
+            self.config_file = os.path.join(dvc_dir, self.CONFIG)
+            self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
+        else:
+            self.dvc_dir = None
+            self.config_file = None
+            self.config_local_file = None
+
+        self.load(validate=validate)
+
+    @staticmethod
+    def get_global_config_dir():
+        from appdirs import user_config_dir
+        return user_config_dir(appname=Config.APPNAME,
+                               appauthor=Config.APPAUTHOR)
+
+    @staticmethod
+    def get_system_config_dir():
+        from appdirs import site_config_dir
+        return site_config_dir(appname=Config.APPNAME,
+                               appauthor=Config.APPAUTHOR)
+
+    @staticmethod
+    def init(dvc_dir):
+        config_file = os.path.join(dvc_dir, Config.CONFIG)
+        open(config_file, 'w+').close()
+        return Config(dvc_dir)
+
+    def _load(self):
+        self._system_config = configobj.ConfigObj(self.system_config_file)
+        self._global_config = configobj.ConfigObj(self.global_config_file)
+
+        if self.config_file is not None:
+            self._project_config = configobj.ConfigObj(self.config_file)
+        else:
+            self._project_config = configobj.ConfigObj()
+
+        if self.config_local_file is not None:
+            self._local_config = configobj.ConfigObj(self.config_local_file)
+        else:
+            self._local_config = configobj.ConfigObj()
+
+        self._config = None
+
+    def load(self, validate=True):
+        self._load()
         try:
-            self._config = configobj.ConfigObj(self.config_file)
+            self._config = configobj.ConfigObj(self.system_config_file)
+            user = configobj.ConfigObj(self.global_config_file)
+            config = configobj.ConfigObj(self.config_file)
             local = configobj.ConfigObj(self.config_local_file)
 
             # NOTE: schema doesn't support ConfigObj.Section validation, so we
             # need to convert our config to dict before passing it to
             self._config = self._lower(self._config)
-            local = self._lower(local)
-            self._config = self._merge(self._config, local)
+            for c in [user, config, local]:
+                c = self._lower(c)
+                self._config = self._merge(self._config, c)
 
-            self._config = Schema(self.SCHEMA).validate(self._config)
+            if validate:
+                self._config = Schema(self.SCHEMA).validate(self._config)
 
             # NOTE: now converting back to ConfigObj
             self._config = configobj.ConfigObj(self._config,
@@ -209,6 +270,75 @@ class Config(object):
         except Exception as ex:
             raise ConfigError(ex)
 
+    def _get_key(self, d, name, add=False):
+        for k in d.keys():
+            if k.lower() == name.lower():
+                return k
+
+        if add:
+            d[name] = {}
+            return name
+
+        return None
+
+    def save(self, config=None):
+        if config is not None:
+            clist = [config]
+        else:
+            clist = [self._system_config,
+                     self._global_config,
+                     self._project_config,
+                     self._local_config]
+
+        for conf in clist:
+            if conf.filename is None:
+                continue
+
+            try:
+                Logger.debug("Writing '{}'.".format(conf.filename))
+                dname = os.path.dirname(os.path.abspath(conf.filename))
+                try:
+                    os.makedirs(dname)
+                except OSError as exc:
+                    if exc.errno != errno.EEXIST:
+                        raise
+                conf.write()
+            except Exception as exc:
+                msg = "Failed to write config '{}'".format(conf.filename)
+                raise ConfigError(msg, exc)
+
+    def unset(self, config, section, opt=None):
+        if section not in config.keys():
+            raise ConfigError("Section '{}' doesn't exist".format(section))
+
+        if opt is None:
+            del config[section]
+            return
+
+        if opt not in config[section].keys():
+            raise ConfigError("Option '{}.{}' doesn't exist".format(section,
+                                                                    opt))
+        del config[section][opt]
+
+        if len(config[section]) == 0:
+            del config[section]
+
+    def set(self, config, section, opt, value):
+        if section not in config.keys():
+            config[section] = {}
+
+        config[section][opt] = value
+
+    def show(self, config, section, opt):
+        if section not in config.keys():
+            raise ConfigError("Section '{}' doesn't exist".format(section))
+
+        if opt not in config[section].keys():
+            raise ConfigError("Option '{}.{}' doesn't exist".format(section,
+                                                                    opt))
+
+        Logger.info(config[section][opt])
+
     @staticmethod
     def _merge(first, second):
         res = {}
@@ -229,9 +359,3 @@ class Config(object):
                 new_s[key.lower()] = value
             new_config[s_key.lower()] = new_s
         return new_config
-
-    @staticmethod
-    def init(dvc_dir):
-        config_file = os.path.join(dvc_dir, Config.CONFIG)
-        open(config_file, 'w+').close()
-        return Config(dvc_dir)

dvc/daemon.py

@@ -42,13 +42,13 @@ class Daemon(object):  # pragma: no cover
               close_fds=True,
               shell=False)
 
-    def __call__(self, name):
+    def __call__(self, args):
         from dvc.utils import is_binary
 
         cmd = [sys.executable]
         if not is_binary():
             cmd += ['-m', 'dvc']
-        cmd += ['daemon', name, '-q']
+        cmd += ['daemon', '-q'] + args
 
         Logger.debug("Trying to spawn '{}'".format(cmd))
 

dvc/main.py

@@ -1,27 +1,35 @@
 from dvc.logger import Logger
 from dvc.cli import parse_args
 from dvc.command.base import CmdBase
+from dvc.analytics import Analytics
+from dvc.cli import DvcParserError
+from dvc.project import NotDvcProjectError
 
 
 def main(argv=None):
     Logger.init()
 
-    args = parse_args(argv)
-
-    # Init loglevel early in case we'll run
-    # into errors before setting it properly
-    CmdBase._set_loglevel(args)
-
+    args = None
+    cmd = None
     try:
+        args = parse_args(argv)
+
+        # Init loglevel early in case we'll run
+        # into errors before setting it properly
+        CmdBase._set_loglevel(args)
+
         cmd = args.func(args)
-    except Exception as ex:
-        Logger.error('Initialization error', ex)
-        return 255
 
-    try:
         ret = cmd.run_cmd()
+    except NotDvcProjectError as ex:
+        Logger.error(str(ex))
+        ret = 253
+    except DvcParserError:
+        ret = 254
     except Exception as ex:
         Logger.error('Unexpected error', ex)
-        return 254
+        ret = 255
+
+    Analytics().send(detach=True, cmd=cmd, args=args, ret=ret)
 
     return ret

dvc/project.py

@@ -10,6 +10,12 @@ class InitError(DvcException):
         super(InitError, self).__init__(msg)
 
 
+class NotDvcProjectError(DvcException):
+    def __init__(self, root):
+        msg = "Not a dvc repository (checked up to mount point {})"
+        super(NotDvcProjectError, self).__init__(msg.format(root))
+
+
 class ReproductionError(DvcException):
     def __init__(self, dvc_file_name, ex):
         self.path = dvc_file_name
@@ -20,7 +26,7 @@ class ReproductionError(DvcException):
 class Project(object):
     DVC_DIR = '.dvc'
 
-    def __init__(self, root_dir):
+    def __init__(self, root_dir=None):
         from dvc.logger import Logger
         from dvc.config import Config
         from dvc.state import State
@@ -31,10 +37,13 @@ class Project(object):
         from dvc.updater import Updater
         from dvc.prompt import Prompt
 
+        root_dir = self._find_root(root_dir)
+
         self.root_dir = os.path.abspath(os.path.realpath(root_dir))
         self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
 
         self.config = Config(self.dvc_dir)
+
         self.scm = SCM(self.root_dir, project=self)
         self.lock = Lock(self.dvc_dir)
         # NOTE: storing state and link_state in the repository itself to avoid
@@ -55,6 +64,27 @@ class Project(object):
 
         self.updater.check()
 
+    @staticmethod
+    def _find_root(root=None):
+        if root is None:
+            root = os.getcwd()
+        else:
+            root = os.path.abspath(os.path.realpath(root))
+
+        while True:
+            dvc_dir = os.path.join(root, Project.DVC_DIR)
+            if os.path.isdir(dvc_dir):
+                return root
+            if os.path.ismount(root):
+                break
+            root = os.path.dirname(root)
+        raise NotDvcProjectError(root)
+
+    @staticmethod
+    def _find_dvc_dir(root=None):
+        root_dir = Project._find_root(root)
+        return os.path.join(root_dir, Project.DVC_DIR)
+
     def _remind_to_git_add(self):
         if len(self._files_to_git_add) == 0:
             return

dvc/updater.py

@@ -68,8 +68,7 @@ class Updater(object):  # pragma: no cover
         from dvc.daemon import Daemon
 
         if detach:
-            d = Daemon()
-            d('updater')
+            Daemon()(['updater'])
             return
 
         self._with_lock(self._get_latest_version, 'fetching')

requirements.txt

@@ -24,3 +24,4 @@ grandalf==0.6
 pydot>=1.2.4
 asciimatics>=1.10.0
 distro>=1.3.0
+appdirs>=1.4.3

setup.py

@@ -24,6 +24,7 @@ install_requires = [
     "grandalf==0.6",
     "asciimatics>=1.10.0",
     "distro>=1.3.0",
+    "appdirs>=1.4.3",
 ]
 
 # Extra dependencies for remote integrations

tests/test_analytics.py

@@ -0,0 +1,62 @@
+import os
+import mock
+import requests
+import tempfile
+
+from dvc.main import main
+from dvc.analytics import Analytics
+from tests.basic_env import TestDvc, TestGit, TestDir
+
+
+def _clean_getenv(key, default=None):
+    """
+    Remove env vars that affect dvc behavior in tests
+    """
+    if key in ['DVC_TEST', 'CI']:
+        return None
+    return os.environ.get(key, default)
+
+
+class TestAnalytics(TestDir):
+    def test(self):
+        a = Analytics()
+        a._collect()
+        self.assertTrue(isinstance(a.info, dict))
+        self.assertNotEqual(a.info, {})
+        self.assertTrue(a.PARAM_USER_ID in a.info.keys())
+        self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys())
+        self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {})
+
+    @mock.patch.object(os, 'getenv', new=_clean_getenv)
+    @mock.patch('requests.post')
+    def test_send(self, mockpost):
+        ret = main(['daemon', 'analytics', Analytics().dump(), '-v'])
+        self.assertEqual(ret, 0)
+
+        self.assertTrue(mockpost.called)
+
+    @mock.patch.object(os, 'getenv', new=_clean_getenv)
+    @mock.patch.object(requests, 'post',
+                       side_effect=requests.exceptions.RequestException())
+    def test_send_failed(self, mockpost):
+        ret = main(['daemon', 'analytics', Analytics().dump(), '-v'])
+        self.assertEqual(ret, 0)
+
+        self.assertTrue(mockpost.called)
+
+
+class TestAnalyticsGit(TestAnalytics, TestGit):
+    pass
+
+
+class TestAnalyticsDvc(TestAnalytics, TestDvc):
+    @mock.patch('requests.post')
+    def test_send_disabled(self, mockpost):
+        ret = main(['config', 'core.analytics', 'false'])
+        self.assertEqual(ret, 0)
+
+        with mock.patch.object(os, 'getenv', new=_clean_getenv):
+            ret = main(['daemon', 'analytics', Analytics().dump(), '-v'])
+        self.assertEqual(ret, 0)
+
+        self.assertFalse(mockpost.called)

tests/test_config.py

@@ -89,6 +89,8 @@ class TestConfigCLI(TestDvc):
 
     def test_failed_write(self):
         class A(object):
+            system = False
+            glob = False
             local = False
             name = 'core.remote'
             value = 'myremote'
@@ -98,5 +100,5 @@ class TestConfigCLI(TestDvc):
         cmd = CmdConfig(args)
 
         cmd.configobj.write = None
-        ret = cmd.save()
+        ret = cmd.run()
         self.assertNotEqual(ret, 0)

tests/test_remote.py

@@ -30,6 +30,8 @@ class TestRemote(TestDvc):
 
     def test_failed_write(self):
         class A(object):
+            system = False
+            glob = False
             local = False
             name = 'myremote'
             url = 's3://remote'