Merge pull request #1337 from efiop/master

updater: spawn a background daemon
This commit is contained in:
Ruslan Kuprieiev
2018-11-16 22:24:03 +02:00
committed by GitHub
6 changed files with 160 additions and 28 deletions
+24
View File
@@ -26,6 +26,7 @@ from dvc.command.install import CmdInstall
from dvc.command.root import CmdRoot
from dvc.command.lock import CmdLock, CmdUnlock
from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList
from dvc.command.daemon import CmdDaemonUpdater
from dvc.logger import Logger
from dvc import VERSION
@@ -942,6 +943,29 @@ def parse_args(argv=None):
help=PIPELINE_LIST_HELP)
pipeline_list_parser.set_defaults(func=CmdPipelineList)
# Daemon
DAEMON_HELP = 'Service daemon.'
daemon_parser = subparsers.add_parser(
'daemon',
parents=[parent_parser],
description=DAEMON_HELP,
help=DAEMON_HELP)
daemon_subparsers = daemon_parser.add_subparsers(
dest='cmd',
help='Use dvc daemon CMD --help '
'for command-specific help.')
_fix_subparsers(daemon_subparsers)
DAEMON_UPDATER_HELP = 'Fetch latest available version.'
daemon_updater_parser = daemon_subparsers.add_parser(
'updater',
parents=[parent_parser],
description=DAEMON_UPDATER_HELP,
help=DAEMON_UPDATER_HELP)
daemon_updater_parser.set_defaults(func=CmdDaemonUpdater)
args = parser.parse_args(argv)
if (issubclass(args.func, CmdRepro)
+19
View File
@@ -0,0 +1,19 @@
from dvc.command.base import CmdBase
class CmdDaemonUpdater(CmdBase):
def __init__(self, args):
pass
def run_cmd(self):
return self.run()
def run(self):
import os
from dvc.project import Project
from dvc.updater import Updater
root_dir = self._find_root()
dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
updater = Updater(dvc_dir)
updater.fetch(detach=False)
+62
View File
@@ -0,0 +1,62 @@
import os
import sys
from subprocess import Popen
from dvc.logger import Logger
class Daemon(object): # pragma: no cover
def _spawn_windows(self, cmd):
CREATE_NEW_PROCESS_GROUP = 0x00000200
DETACHED_PROCESS = 0x00000008
creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
Popen(cmd,
close_fds=True,
shell=False,
creationflags=creationflags)
def _spawn_posix(self, cmd):
try:
pid = os.fork()
if pid > 0:
return
except OSError as exc:
Logger.error("Failed at first fork", exc)
sys.exit(1)
os.setsid()
os.umask(0)
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError as exc:
Logger.error("Failed at second fork", exc)
sys.exit(1)
sys.stdin.close()
sys.stdout.close()
sys.stderr.close()
Popen(cmd,
close_fds=True,
shell=False)
def __call__(self, name):
from dvc.utils import is_binary
cmd = [sys.executable]
if not is_binary():
cmd += ['-m', 'dvc']
cmd += ['daemon', name, '-v']
Logger.debug("Trying to spawn '{}'".format(cmd))
if os.name == 'nt':
self._spawn_windows(cmd)
elif os.name == 'posix':
self._spawn_posix(cmd)
else:
raise NotImplementedError
Logger.debug("Spawned '{}'".format(cmd))
+1
View File
@@ -136,6 +136,7 @@ class Project(object):
self.lock.lock_file,
self.config.config_local_file,
self.updater.updater_file,
self.updater.lock.lock_file,
] + self.state.temp_files
if self.cache.local.cache_dir.startswith(self.root_dir):
+47 -25
View File
@@ -6,6 +6,7 @@ import colorama
from dvc import VERSION_BASE
from dvc.logger import Logger
from dvc.utils import is_binary
class Updater(object): # pragma: no cover
@@ -16,40 +17,65 @@ class Updater(object): # pragma: no cover
TIMEOUT_GET = 10
def __init__(self, dvc_dir):
from dvc.lock import Lock
self.dvc_dir = dvc_dir
self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
self.lock = Lock(dvc_dir, self.updater_file + '.lock')
self.current = VERSION_BASE
def _is_outdated_file(self):
ctime = os.path.getmtime(self.updater_file)
outdated = (time.time() - ctime >= self.TIMEOUT)
if outdated:
Logger.debug("'{}' is outdated(".format(self.updater_file))
return outdated
def check(self):
if os.getenv('CI'):
return
if os.path.isfile(self.updater_file):
ctime = os.path.getctime(self.updater_file)
if time.time() - ctime < self.TIMEOUT:
msg = '{} is not old enough to check for updates'
Logger.debug(msg.format(self.UPDATER_FILE))
return
with self.lock:
self._check()
os.unlink(self.updater_file)
Logger.info('Checking for updates...')
try:
self._get_latest_version()
except Exception as exc:
msg = 'Failed to obtain latest version: {}'.format(str(exc))
Logger.debug(msg)
def _check(self):
if not os.path.exists(self.updater_file) or self._is_outdated_file():
self.fetch()
return
with open(self.updater_file, 'r') as fobj:
import json
try:
info = json.load(fobj)
self.latest = info['version']
except json.decoder.JSONDecodeError as exc:
msg = "'{}' is not a valid json: {}"
Logger.debug(msg.format(self.updater_file, exc))
self.fetch()
return
if self._is_outdated():
self._notify()
def fetch(self, detach=True):
from dvc.daemon import Daemon
if detach:
d = Daemon()
d('updater')
return
with self.lock:
self._get_latest_version()
def _get_latest_version(self):
import json
r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
j = r.json()
self.latest = j['version']
open(self.updater_file, 'w+').close()
info = r.json()
with open(self.updater_file, 'w+') as fobj:
json.dump(info, fobj)
def _is_outdated(self):
l_major, l_minor, l_patch = [int(x) for x in self.latest.split('.')]
@@ -97,14 +123,10 @@ class Updater(object): # pragma: no cover
return instructions[package_manager]
@staticmethod
def _is_binary():
return getattr(sys, 'frozen', False)
def _get_linux(self):
import distro
if not self._is_binary():
if not is_binary():
return 'pip'
package_managers = {
@@ -120,7 +142,7 @@ class Updater(object): # pragma: no cover
return package_managers.get(distro.id())
def _get_darwin(self):
if not self._is_binary():
if not is_binary():
if __file__.startswith('/usr/local/Cellar'):
return 'formula'
else:
@@ -136,7 +158,7 @@ class Updater(object): # pragma: no cover
return None
def _get_windows(self):
return None if self._is_binary() else 'pip'
return None if is_binary() else 'pip'
def _get_package_manager(self):
import platform
+7 -3
View File
@@ -146,6 +146,12 @@ def to_chunks(l, jobs):
return [l[x:x+n] for x in range(0, len(l), n)]
# NOTE: Check if we are in a bundle
# https://pythonhosted.org/PyInstaller/runtime-information.html
def is_binary():
return getattr(sys, 'frozen', False)
# NOTE: Fix env variables modified by PyInstaller
# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
def fix_env(env):
@@ -154,9 +160,7 @@ def fix_env(env):
else:
env = env.copy()
# NOTE: Check if we are in a bundle
# https://pythonhosted.org/PyInstaller/runtime-information.html
if getattr(sys, 'frozen', False):
if is_binary():
lp_key = 'LD_LIBRARY_PATH'
lp_orig = env.get(lp_key + '_ORIG', None)
if lp_orig is not None: