mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
queue start: give every worker a unique name
Currently, all of our celery workers are called `dvc-exp-worker` and they will write to the same log path. We give every single worker a unique name and seperate they logs if under a debug log level
This commit is contained in:
committed by
Peter Rowlands (변기호)
parent
164efe089f
commit
25b5d1d522
@@ -124,18 +124,21 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
)
|
||||
|
||||
def spawn_worker(self):
|
||||
from shortuuid import uuid
|
||||
|
||||
from dvc_task.proc.process import ManagedProcess
|
||||
|
||||
logger.debug("Spawning exp queue worker")
|
||||
wdir_hash = hashlib.sha256(self.wdir.encode("utf-8")).hexdigest()[:6]
|
||||
node_name = f"dvc-exp-{wdir_hash}-1@localhost"
|
||||
cmd = ["exp", "queue-worker", node_name]
|
||||
name = "dvc-exp-worker"
|
||||
if logger.getEffectiveLevel() < logging.INFO:
|
||||
name = name + str(uuid())
|
||||
if os.name == "nt":
|
||||
daemonize(cmd)
|
||||
else:
|
||||
ManagedProcess.spawn(
|
||||
["dvc"] + cmd, wdir=self.wdir, name="dvc-exp-worker"
|
||||
)
|
||||
ManagedProcess.spawn(["dvc"] + cmd, wdir=self.wdir, name=name)
|
||||
|
||||
def put(self, *args, **kwargs) -> QueueEntry:
|
||||
"""Stash an experiment and add it to the queue."""
|
||||
|
||||
Reference in New Issue
Block a user