From 63544b7e0183eb044460ee2b276ac1667ef1481a Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 16 Nov 2018 16:19:40 +0200 Subject: [PATCH] updater: spawn a background daemon Fix #1304 Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 24 +++++++++++++++ dvc/command/daemon.py | 19 ++++++++++++ dvc/daemon.py | 62 +++++++++++++++++++++++++++++++++++++ dvc/project.py | 1 + dvc/updater.py | 72 ++++++++++++++++++++++++++++--------------- dvc/utils.py | 10 ++++-- 6 files changed, 160 insertions(+), 28 deletions(-) create mode 100644 dvc/command/daemon.py create mode 100644 dvc/daemon.py diff --git a/dvc/cli.py b/dvc/cli.py index a49460f6f..838d2c0b5 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -26,6 +26,7 @@ from dvc.command.install import CmdInstall from dvc.command.root import CmdRoot from dvc.command.lock import CmdLock, CmdUnlock from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList +from dvc.command.daemon import CmdDaemonUpdater from dvc.logger import Logger from dvc import VERSION @@ -942,6 +943,29 @@ def parse_args(argv=None): help=PIPELINE_LIST_HELP) pipeline_list_parser.set_defaults(func=CmdPipelineList) + # Daemon + DAEMON_HELP = 'Service daemon.' + daemon_parser = subparsers.add_parser( + 'daemon', + parents=[parent_parser], + description=DAEMON_HELP, + help=DAEMON_HELP) + + daemon_subparsers = daemon_parser.add_subparsers( + dest='cmd', + help='Use dvc daemon CMD --help ' + 'for command-specific help.') + + _fix_subparsers(daemon_subparsers) + + DAEMON_UPDATER_HELP = 'Fetch latest available version.' + daemon_updater_parser = daemon_subparsers.add_parser( + 'updater', + parents=[parent_parser], + description=DAEMON_UPDATER_HELP, + help=DAEMON_UPDATER_HELP) + daemon_updater_parser.set_defaults(func=CmdDaemonUpdater) + args = parser.parse_args(argv) if (issubclass(args.func, CmdRepro) diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py new file mode 100644 index 000000000..82ca1bf0c --- /dev/null +++ b/dvc/command/daemon.py @@ -0,0 +1,19 @@ +from dvc.command.base import CmdBase + + +class CmdDaemonUpdater(CmdBase): + def __init__(self, args): + pass + + def run_cmd(self): + return self.run() + + def run(self): + import os + from dvc.project import Project + from dvc.updater import Updater + + root_dir = self._find_root() + dvc_dir = os.path.join(root_dir, Project.DVC_DIR) + updater = Updater(dvc_dir) + updater.fetch(detach=False) diff --git a/dvc/daemon.py b/dvc/daemon.py new file mode 100644 index 000000000..231b60eac --- /dev/null +++ b/dvc/daemon.py @@ -0,0 +1,62 @@ +import os +import sys +from subprocess import Popen +from dvc.logger import Logger + + +class Daemon(object): # pragma: no cover + def _spawn_windows(self, cmd): + CREATE_NEW_PROCESS_GROUP = 0x00000200 + DETACHED_PROCESS = 0x00000008 + creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS + Popen(cmd, + close_fds=True, + shell=False, + creationflags=creationflags) + + def _spawn_posix(self, cmd): + try: + pid = os.fork() + if pid > 0: + return + except OSError as exc: + Logger.error("Failed at first fork", exc) + sys.exit(1) + + os.setsid() + os.umask(0) + + try: + pid = os.fork() + if pid > 0: + sys.exit(0) + except OSError as exc: + Logger.error("Failed at second fork", exc) + sys.exit(1) + + sys.stdin.close() + sys.stdout.close() + sys.stderr.close() + + Popen(cmd, + close_fds=True, + shell=False) + + def __call__(self, name): + from dvc.utils import is_binary + + cmd = [sys.executable] + if not is_binary(): + cmd += ['-m', 'dvc'] + cmd += ['daemon', name, '-v'] + + Logger.debug("Trying to spawn '{}'".format(cmd)) + + if os.name == 'nt': + self._spawn_windows(cmd) + elif os.name == 'posix': + self._spawn_posix(cmd) + else: + raise NotImplementedError + + Logger.debug("Spawned '{}'".format(cmd)) diff --git a/dvc/project.py b/dvc/project.py index 797060264..c82ba944f 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -136,6 +136,7 @@ class Project(object): self.lock.lock_file, self.config.config_local_file, self.updater.updater_file, + self.updater.lock.lock_file, ] + self.state.temp_files if self.cache.local.cache_dir.startswith(self.root_dir): diff --git a/dvc/updater.py b/dvc/updater.py index 98748c383..e1df0d138 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -6,6 +6,7 @@ import colorama from dvc import VERSION_BASE from dvc.logger import Logger +from dvc.utils import is_binary class Updater(object): # pragma: no cover @@ -16,40 +17,65 @@ class Updater(object): # pragma: no cover TIMEOUT_GET = 10 def __init__(self, dvc_dir): + from dvc.lock import Lock + self.dvc_dir = dvc_dir self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE) + self.lock = Lock(dvc_dir, self.updater_file + '.lock') self.current = VERSION_BASE + def _is_outdated_file(self): + ctime = os.path.getmtime(self.updater_file) + outdated = (time.time() - ctime >= self.TIMEOUT) + if outdated: + Logger.debug("'{}' is outdated(".format(self.updater_file)) + return outdated + def check(self): if os.getenv('CI'): return - if os.path.isfile(self.updater_file): - ctime = os.path.getctime(self.updater_file) - if time.time() - ctime < self.TIMEOUT: - msg = '{} is not old enough to check for updates' - Logger.debug(msg.format(self.UPDATER_FILE)) - return + with self.lock: + self._check() - os.unlink(self.updater_file) - - Logger.info('Checking for updates...') - - try: - self._get_latest_version() - except Exception as exc: - msg = 'Failed to obtain latest version: {}'.format(str(exc)) - Logger.debug(msg) + def _check(self): + if not os.path.exists(self.updater_file) or self._is_outdated_file(): + self.fetch() return + with open(self.updater_file, 'r') as fobj: + import json + + try: + info = json.load(fobj) + self.latest = info['version'] + except json.decoder.JSONDecodeError as exc: + msg = "'{}' is not a valid json: {}" + Logger.debug(msg.format(self.updater_file, exc)) + self.fetch() + return + if self._is_outdated(): self._notify() + def fetch(self, detach=True): + from dvc.daemon import Daemon + + if detach: + d = Daemon() + d('updater') + return + + with self.lock: + self._get_latest_version() + def _get_latest_version(self): + import json + r = requests.get(self.URL, timeout=self.TIMEOUT_GET) - j = r.json() - self.latest = j['version'] - open(self.updater_file, 'w+').close() + info = r.json() + with open(self.updater_file, 'w+') as fobj: + json.dump(info, fobj) def _is_outdated(self): l_major, l_minor, l_patch = [int(x) for x in self.latest.split('.')] @@ -97,14 +123,10 @@ class Updater(object): # pragma: no cover return instructions[package_manager] - @staticmethod - def _is_binary(): - return getattr(sys, 'frozen', False) - def _get_linux(self): import distro - if not self._is_binary(): + if not is_binary(): return 'pip' package_managers = { @@ -120,7 +142,7 @@ class Updater(object): # pragma: no cover return package_managers.get(distro.id()) def _get_darwin(self): - if not self._is_binary(): + if not is_binary(): if __file__.startswith('/usr/local/Cellar'): return 'formula' else: @@ -136,7 +158,7 @@ class Updater(object): # pragma: no cover return None def _get_windows(self): - return None if self._is_binary() else 'pip' + return None if is_binary() else 'pip' def _get_package_manager(self): import platform diff --git a/dvc/utils.py b/dvc/utils.py index 060d3b62f..7d4660aa6 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -146,6 +146,12 @@ def to_chunks(l, jobs): return [l[x:x+n] for x in range(0, len(l), n)] +# NOTE: Check if we are in a bundle +# https://pythonhosted.org/PyInstaller/runtime-information.html +def is_binary(): + return getattr(sys, 'frozen', False) + + # NOTE: Fix env variables modified by PyInstaller # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html def fix_env(env): @@ -154,9 +160,7 @@ def fix_env(env): else: env = env.copy() - # NOTE: Check if we are in a bundle - # https://pythonhosted.org/PyInstaller/runtime-information.html - if getattr(sys, 'frozen', False): + if is_binary(): lp_key = 'LD_LIBRARY_PATH' lp_orig = env.get(lp_key + '_ORIG', None) if lp_orig is not None: