From 25b5d1d522f2861b5057c412856eead87ba8f4c8 Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Thu, 2 Jun 2022 15:26:43 +0800 Subject: [PATCH] queue start: give every worker a unique name Currently, all of our celery workers are called `dvc-exp-worker` and they will write to the same log path. We give every single worker a unique name and seperate they logs if under a debug log level --- dvc/repo/experiments/queue/local.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dvc/repo/experiments/queue/local.py b/dvc/repo/experiments/queue/local.py index aac45aa0d..5b118a57e 100644 --- a/dvc/repo/experiments/queue/local.py +++ b/dvc/repo/experiments/queue/local.py @@ -124,18 +124,21 @@ class LocalCeleryQueue(BaseStashQueue): ) def spawn_worker(self): + from shortuuid import uuid + from dvc_task.proc.process import ManagedProcess logger.debug("Spawning exp queue worker") wdir_hash = hashlib.sha256(self.wdir.encode("utf-8")).hexdigest()[:6] node_name = f"dvc-exp-{wdir_hash}-1@localhost" cmd = ["exp", "queue-worker", node_name] + name = "dvc-exp-worker" + if logger.getEffectiveLevel() < logging.INFO: + name = name + str(uuid()) if os.name == "nt": daemonize(cmd) else: - ManagedProcess.spawn( - ["dvc"] + cmd, wdir=self.wdir, name="dvc-exp-worker" - ) + ManagedProcess.spawn(["dvc"] + cmd, wdir=self.wdir, name=name) def put(self, *args, **kwargs) -> QueueEntry: """Stash an experiment and add it to the queue."""