queue start: give every worker a unique name

Currently, all of our celery workers are called `dvc-exp-worker` and
they will write to the same log path.

We give every single worker a unique name and seperate they logs if
under a debug log level
This commit is contained in:
karajan1001
2022-07-12 21:39:49 +09:00
committed by Peter Rowlands (변기호)
parent 164efe089f
commit 25b5d1d522
+6 -3
View File
@@ -124,18 +124,21 @@ class LocalCeleryQueue(BaseStashQueue):
)
def spawn_worker(self):
from shortuuid import uuid
from dvc_task.proc.process import ManagedProcess
logger.debug("Spawning exp queue worker")
wdir_hash = hashlib.sha256(self.wdir.encode("utf-8")).hexdigest()[:6]
node_name = f"dvc-exp-{wdir_hash}-1@localhost"
cmd = ["exp", "queue-worker", node_name]
name = "dvc-exp-worker"
if logger.getEffectiveLevel() < logging.INFO:
name = name + str(uuid())
if os.name == "nt":
daemonize(cmd)
else:
ManagedProcess.spawn(
["dvc"] + cmd, wdir=self.wdir, name="dvc-exp-worker"
)
ManagedProcess.spawn(["dvc"] + cmd, wdir=self.wdir, name=name)
def put(self, *args, **kwargs) -> QueueEntry:
"""Stash an experiment and add it to the queue."""