mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-22 07:18:28 +01:00
Merge pull request #1337 from efiop/master
updater: spawn a background daemon
This commit is contained in:
+24
@@ -26,6 +26,7 @@ from dvc.command.install import CmdInstall
|
||||
from dvc.command.root import CmdRoot
|
||||
from dvc.command.lock import CmdLock, CmdUnlock
|
||||
from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList
|
||||
from dvc.command.daemon import CmdDaemonUpdater
|
||||
from dvc.logger import Logger
|
||||
from dvc import VERSION
|
||||
|
||||
@@ -942,6 +943,29 @@ def parse_args(argv=None):
|
||||
help=PIPELINE_LIST_HELP)
|
||||
pipeline_list_parser.set_defaults(func=CmdPipelineList)
|
||||
|
||||
# Daemon
|
||||
DAEMON_HELP = 'Service daemon.'
|
||||
daemon_parser = subparsers.add_parser(
|
||||
'daemon',
|
||||
parents=[parent_parser],
|
||||
description=DAEMON_HELP,
|
||||
help=DAEMON_HELP)
|
||||
|
||||
daemon_subparsers = daemon_parser.add_subparsers(
|
||||
dest='cmd',
|
||||
help='Use dvc daemon CMD --help '
|
||||
'for command-specific help.')
|
||||
|
||||
_fix_subparsers(daemon_subparsers)
|
||||
|
||||
DAEMON_UPDATER_HELP = 'Fetch latest available version.'
|
||||
daemon_updater_parser = daemon_subparsers.add_parser(
|
||||
'updater',
|
||||
parents=[parent_parser],
|
||||
description=DAEMON_UPDATER_HELP,
|
||||
help=DAEMON_UPDATER_HELP)
|
||||
daemon_updater_parser.set_defaults(func=CmdDaemonUpdater)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if (issubclass(args.func, CmdRepro)
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
from dvc.command.base import CmdBase
|
||||
|
||||
|
||||
class CmdDaemonUpdater(CmdBase):
|
||||
def __init__(self, args):
|
||||
pass
|
||||
|
||||
def run_cmd(self):
|
||||
return self.run()
|
||||
|
||||
def run(self):
|
||||
import os
|
||||
from dvc.project import Project
|
||||
from dvc.updater import Updater
|
||||
|
||||
root_dir = self._find_root()
|
||||
dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
|
||||
updater = Updater(dvc_dir)
|
||||
updater.fetch(detach=False)
|
||||
@@ -0,0 +1,62 @@
|
||||
import os
|
||||
import sys
|
||||
from subprocess import Popen
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
class Daemon(object): # pragma: no cover
|
||||
def _spawn_windows(self, cmd):
|
||||
CREATE_NEW_PROCESS_GROUP = 0x00000200
|
||||
DETACHED_PROCESS = 0x00000008
|
||||
creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
|
||||
Popen(cmd,
|
||||
close_fds=True,
|
||||
shell=False,
|
||||
creationflags=creationflags)
|
||||
|
||||
def _spawn_posix(self, cmd):
|
||||
try:
|
||||
pid = os.fork()
|
||||
if pid > 0:
|
||||
return
|
||||
except OSError as exc:
|
||||
Logger.error("Failed at first fork", exc)
|
||||
sys.exit(1)
|
||||
|
||||
os.setsid()
|
||||
os.umask(0)
|
||||
|
||||
try:
|
||||
pid = os.fork()
|
||||
if pid > 0:
|
||||
sys.exit(0)
|
||||
except OSError as exc:
|
||||
Logger.error("Failed at second fork", exc)
|
||||
sys.exit(1)
|
||||
|
||||
sys.stdin.close()
|
||||
sys.stdout.close()
|
||||
sys.stderr.close()
|
||||
|
||||
Popen(cmd,
|
||||
close_fds=True,
|
||||
shell=False)
|
||||
|
||||
def __call__(self, name):
|
||||
from dvc.utils import is_binary
|
||||
|
||||
cmd = [sys.executable]
|
||||
if not is_binary():
|
||||
cmd += ['-m', 'dvc']
|
||||
cmd += ['daemon', name, '-v']
|
||||
|
||||
Logger.debug("Trying to spawn '{}'".format(cmd))
|
||||
|
||||
if os.name == 'nt':
|
||||
self._spawn_windows(cmd)
|
||||
elif os.name == 'posix':
|
||||
self._spawn_posix(cmd)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
Logger.debug("Spawned '{}'".format(cmd))
|
||||
@@ -136,6 +136,7 @@ class Project(object):
|
||||
self.lock.lock_file,
|
||||
self.config.config_local_file,
|
||||
self.updater.updater_file,
|
||||
self.updater.lock.lock_file,
|
||||
] + self.state.temp_files
|
||||
|
||||
if self.cache.local.cache_dir.startswith(self.root_dir):
|
||||
|
||||
+47
-25
@@ -6,6 +6,7 @@ import colorama
|
||||
|
||||
from dvc import VERSION_BASE
|
||||
from dvc.logger import Logger
|
||||
from dvc.utils import is_binary
|
||||
|
||||
|
||||
class Updater(object): # pragma: no cover
|
||||
@@ -16,40 +17,65 @@ class Updater(object): # pragma: no cover
|
||||
TIMEOUT_GET = 10
|
||||
|
||||
def __init__(self, dvc_dir):
|
||||
from dvc.lock import Lock
|
||||
|
||||
self.dvc_dir = dvc_dir
|
||||
self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
|
||||
self.lock = Lock(dvc_dir, self.updater_file + '.lock')
|
||||
self.current = VERSION_BASE
|
||||
|
||||
def _is_outdated_file(self):
|
||||
ctime = os.path.getmtime(self.updater_file)
|
||||
outdated = (time.time() - ctime >= self.TIMEOUT)
|
||||
if outdated:
|
||||
Logger.debug("'{}' is outdated(".format(self.updater_file))
|
||||
return outdated
|
||||
|
||||
def check(self):
|
||||
if os.getenv('CI'):
|
||||
return
|
||||
|
||||
if os.path.isfile(self.updater_file):
|
||||
ctime = os.path.getctime(self.updater_file)
|
||||
if time.time() - ctime < self.TIMEOUT:
|
||||
msg = '{} is not old enough to check for updates'
|
||||
Logger.debug(msg.format(self.UPDATER_FILE))
|
||||
return
|
||||
with self.lock:
|
||||
self._check()
|
||||
|
||||
os.unlink(self.updater_file)
|
||||
|
||||
Logger.info('Checking for updates...')
|
||||
|
||||
try:
|
||||
self._get_latest_version()
|
||||
except Exception as exc:
|
||||
msg = 'Failed to obtain latest version: {}'.format(str(exc))
|
||||
Logger.debug(msg)
|
||||
def _check(self):
|
||||
if not os.path.exists(self.updater_file) or self._is_outdated_file():
|
||||
self.fetch()
|
||||
return
|
||||
|
||||
with open(self.updater_file, 'r') as fobj:
|
||||
import json
|
||||
|
||||
try:
|
||||
info = json.load(fobj)
|
||||
self.latest = info['version']
|
||||
except json.decoder.JSONDecodeError as exc:
|
||||
msg = "'{}' is not a valid json: {}"
|
||||
Logger.debug(msg.format(self.updater_file, exc))
|
||||
self.fetch()
|
||||
return
|
||||
|
||||
if self._is_outdated():
|
||||
self._notify()
|
||||
|
||||
def fetch(self, detach=True):
|
||||
from dvc.daemon import Daemon
|
||||
|
||||
if detach:
|
||||
d = Daemon()
|
||||
d('updater')
|
||||
return
|
||||
|
||||
with self.lock:
|
||||
self._get_latest_version()
|
||||
|
||||
def _get_latest_version(self):
|
||||
import json
|
||||
|
||||
r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
|
||||
j = r.json()
|
||||
self.latest = j['version']
|
||||
open(self.updater_file, 'w+').close()
|
||||
info = r.json()
|
||||
with open(self.updater_file, 'w+') as fobj:
|
||||
json.dump(info, fobj)
|
||||
|
||||
def _is_outdated(self):
|
||||
l_major, l_minor, l_patch = [int(x) for x in self.latest.split('.')]
|
||||
@@ -97,14 +123,10 @@ class Updater(object): # pragma: no cover
|
||||
|
||||
return instructions[package_manager]
|
||||
|
||||
@staticmethod
|
||||
def _is_binary():
|
||||
return getattr(sys, 'frozen', False)
|
||||
|
||||
def _get_linux(self):
|
||||
import distro
|
||||
|
||||
if not self._is_binary():
|
||||
if not is_binary():
|
||||
return 'pip'
|
||||
|
||||
package_managers = {
|
||||
@@ -120,7 +142,7 @@ class Updater(object): # pragma: no cover
|
||||
return package_managers.get(distro.id())
|
||||
|
||||
def _get_darwin(self):
|
||||
if not self._is_binary():
|
||||
if not is_binary():
|
||||
if __file__.startswith('/usr/local/Cellar'):
|
||||
return 'formula'
|
||||
else:
|
||||
@@ -136,7 +158,7 @@ class Updater(object): # pragma: no cover
|
||||
return None
|
||||
|
||||
def _get_windows(self):
|
||||
return None if self._is_binary() else 'pip'
|
||||
return None if is_binary() else 'pip'
|
||||
|
||||
def _get_package_manager(self):
|
||||
import platform
|
||||
|
||||
+7
-3
@@ -146,6 +146,12 @@ def to_chunks(l, jobs):
|
||||
return [l[x:x+n] for x in range(0, len(l), n)]
|
||||
|
||||
|
||||
# NOTE: Check if we are in a bundle
|
||||
# https://pythonhosted.org/PyInstaller/runtime-information.html
|
||||
def is_binary():
|
||||
return getattr(sys, 'frozen', False)
|
||||
|
||||
|
||||
# NOTE: Fix env variables modified by PyInstaller
|
||||
# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
|
||||
def fix_env(env):
|
||||
@@ -154,9 +160,7 @@ def fix_env(env):
|
||||
else:
|
||||
env = env.copy()
|
||||
|
||||
# NOTE: Check if we are in a bundle
|
||||
# https://pythonhosted.org/PyInstaller/runtime-information.html
|
||||
if getattr(sys, 'frozen', False):
|
||||
if is_binary():
|
||||
lp_key = 'LD_LIBRARY_PATH'
|
||||
lp_orig = env.get(lp_key + '_ORIG', None)
|
||||
if lp_orig is not None:
|
||||
|
||||
Reference in New Issue
Block a user