From patchwork Fri May 24 15:11:55 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 44146 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 18C94C25B74 for ; Fri, 24 May 2024 15:12:12 +0000 (UTC) Received: from mail-wm1-f48.google.com (mail-wm1-f48.google.com [209.85.128.48]) by mx.groups.io with SMTP id smtpd.web11.18332.1716563524037604594 for ; Fri, 24 May 2024 08:12:04 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=e4lMJMX+; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.48, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f48.google.com with SMTP id 5b1f17b1804b1-420180b58c3so68664705e9.2 for ; Fri, 24 May 2024 08:12:03 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1716563522; x=1717168322; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:message-id:date:subject:to :from:from:to:cc:subject:date:message-id:reply-to; bh=TMrCgvPprdokXnS838ol6IhhnYIcvh7LraWq8nbScHI=; b=e4lMJMX+GBvWJjt+N54nHwLjDpn9dh/SnFoymZT0aVuL2rzYskU/Gmy57sW78SWyyo /kwowjIRwDhJfjTj7SynnMYcHNZXnhM5sZOkrm4R6Baun7XF1g622MZtUOGfxX9T5xPd gYYMMQd524CE34NeGng4mzewv0TB3Sjv23lRM= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1716563522; x=1717168322; h=content-transfer-encoding:mime-version:message-id:date:subject:to :from:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=TMrCgvPprdokXnS838ol6IhhnYIcvh7LraWq8nbScHI=; b=eW4+ra0OKEaSB0lu+MBG+cjU9rFtKPvvkR+ePIGrcxTErVv2v47V0oeGRQ1Goz7EaZ wwbak3Ug5DlV9TVvEL3LeTs+kT6DRxY2pdoSgn5fXIGmunPvBVVmx8LTlT80QM3lMkjY 6mt/urlwL/QnIV0ggiLG8S9C0GP3cFoTzJ8ap+wG/1KXisWw5D3eikRwwbggBzzJFcTR XTtHx3Tvr6+DtmL4b4Sichl7dG9pUdyTOGHqeYm8CdzbYRYU/Vngx/NI94QLAaN9k0Fj zV/l1bYun1MGQVQqtEQMNVFKMCz5eaQr2IMiZn9hlTqJfDbwCBVgqMJWTPUlM8ywl1w1 ymtw== X-Gm-Message-State: AOJu0YwhSpC/liVo7RzPypyTQcUvsD7bGKGqpyXt6h9LMs+1vVA57zvT c3GhkHAD2pT0ToMLFaUp6jyhsYEgxqjnE7z37TDw8ODKzJi9Bbuivida5Jbp9QmYMWE0KPNX9Fa D X-Google-Smtp-Source: AGHT+IEl4wtt9Uz+cBZ5LsHj0vDvdfnQdiJe8vXkhMeQ0AufKJPbDsPvNRKQ8CevVo2yH6h8fae55A== X-Received: by 2002:a05:600c:6b1a:b0:416:6af1:27af with SMTP id 5b1f17b1804b1-42108aba785mr21749325e9.35.1716563521819; Fri, 24 May 2024 08:12:01 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:9ca4:3686:8521:57da]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-35579d7db23sm1841759f8f.23.2024.05.24.08.12.01 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 24 May 2024 08:12:01 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 1/6] siggen/runqueue: Store whether the hash was present on the hashserver or not Date: Fri, 24 May 2024 16:11:55 +0100 Message-Id: <20240524151200.2449512-1-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.40.1 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 24 May 2024 15:12:12 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16255 We need to be able to know if a unihash came from the hashequivalence server or not. Add this information to the cache data so we can use it in later code changes. Signed-off-by: Richard Purdie --- lib/bb/runqueue.py | 2 +- lib/bb/siggen.py | 32 ++++++++++++++++---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index 6b43f303d5..6c2034176e 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -2530,7 +2530,7 @@ class RunQueueExecute: for hashtid in torehash: hashequiv_logger.verbose("Task %s unihash changed to %s" % (hashtid, unihash)) self.rqdata.runtaskentries[hashtid].unihash = unihash - bb.parse.siggen.set_unihash(hashtid, unihash) + bb.parse.siggen.set_unihash(hashtid, unihash, True) toprocess.add(hashtid) if torehash: # Need to save after set_unihash above diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py index 03dfda6f3c..e8fe806c36 100644 --- a/lib/bb/siggen.py +++ b/lib/bb/siggen.py @@ -80,7 +80,7 @@ class SignatureGenerator(object): self.runtaskdeps = {} self.file_checksum_values = {} self.taints = {} - self.unitaskhashes = {} + self.hashserv_cache = {} self.tidtopn = {} self.setscenetasks = set() @@ -184,19 +184,19 @@ class SignatureGenerator(object): return def get_taskdata(self): - return (self.runtaskdeps, self.taskhash, self.unihash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.tidtopn, self.setscenetasks) + return (self.runtaskdeps, self.taskhash, self.unihash, self.file_checksum_values, self.taints, self.basehash, self.hashserv_cache, self.tidtopn, self.setscenetasks) def set_taskdata(self, data): - self.runtaskdeps, self.taskhash, self.unihash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.tidtopn, self.setscenetasks = data + self.runtaskdeps, self.taskhash, self.unihash, self.file_checksum_values, self.taints, self.basehash, self.hashserv_cache, self.tidtopn, self.setscenetasks = data def reset(self, data): self.__init__(data) def get_taskhashes(self): - return self.taskhash, self.unihash, self.unitaskhashes, self.tidtopn + return self.taskhash, self.unihash, self.hashserv_cache, self.tidtopn def set_taskhashes(self, hashes): - self.taskhash, self.unihash, self.unitaskhashes, self.tidtopn = hashes + self.taskhash, self.unihash, self.hashserv_cache, self.tidtopn = hashes def save_unitaskhashes(self): return @@ -238,8 +238,8 @@ class SignatureGeneratorBasic(SignatureGenerator): else: self.checksum_cache = None - self.unihash_cache = bb.cache.SimpleCache("3") - self.unitaskhashes = self.unihash_cache.init_cache(data, "bb_unihashes.dat", {}) + self.unihash_cache = bb.cache.SimpleCache("4") + self.hashserv_cache = self.unihash_cache.init_cache(data, "bb_unihashes.dat", {}) self.localdirsexclude = (data.getVar("BB_SIGNATURE_LOCAL_DIRS_EXCLUDE") or "CVS .bzr .git .hg .osc .p4 .repo .svn").split() self.tidtopn = {} @@ -416,7 +416,7 @@ class SignatureGeneratorBasic(SignatureGenerator): bb.fetch2.fetcher_parse_done() def save_unitaskhashes(self): - self.unihash_cache.save(self.unitaskhashes) + self.unihash_cache.save(self.hashserv_cache) def copy_unitaskhashes(self, targetdir): self.unihash_cache.copyfile(targetdir) @@ -630,10 +630,10 @@ class SignatureGeneratorUniHashMixIn(object): return super().get_stampfile_hash(tid) - def set_unihash(self, tid, unihash): + def set_unihash(self, tid, unihash, present): (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid) key = mc + ":" + self.tidtopn[tid] + ":" + taskname - self.unitaskhashes[key] = (self.taskhash[tid], unihash) + self.hashserv_cache[key] = (self.taskhash[tid], unihash, present) self.unihash[tid] = unihash def _get_unihash(self, tid, checkkey=None): @@ -641,11 +641,11 @@ class SignatureGeneratorUniHashMixIn(object): return None (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid) key = mc + ":" + self.tidtopn[tid] + ":" + taskname - if key not in self.unitaskhashes: + if key not in self.hashserv_cache: return None if not checkkey: checkkey = self.taskhash[tid] - (key, unihash) = self.unitaskhashes[key] + (key, unihash, present) = self.hashserv_cache[key] if key != checkkey: return None return unihash @@ -754,12 +754,12 @@ class SignatureGeneratorUniHashMixIn(object): # so it is reported it at debug level 2. If they differ, that # is much more interesting, so it is reported at debug level 1 hashequiv_logger.bbdebug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server)) + self.set_unihash(tid, unihash, True) else: hashequiv_logger.debug2('No reported unihash for %s:%s from %s' % (tid, taskhash, self.server)) unihash = taskhash + self.set_unihash(tid, unihash, False) - - self.set_unihash(tid, unihash) self.unihash[tid] = unihash result[tid] = unihash @@ -835,7 +835,7 @@ class SignatureGeneratorUniHashMixIn(object): if new_unihash != unihash: hashequiv_logger.debug('Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server)) bb.event.fire(bb.runqueue.taskUniHashUpdate(mcfn + ':do_' + task, new_unihash), d) - self.set_unihash(tid, new_unihash) + self.set_unihash(tid, new_unihash, True) d.setVar('BB_UNIHASH', new_unihash) else: hashequiv_logger.debug('Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server)) @@ -875,7 +875,7 @@ class SignatureGeneratorUniHashMixIn(object): hashequiv_logger.verbose('Task %s unihash %s unchanged by server' % (tid, finalunihash)) elif finalunihash == wanted_unihash: hashequiv_logger.verbose('Task %s unihash changed %s -> %s as wanted' % (tid, current_unihash, finalunihash)) - self.set_unihash(tid, finalunihash) + self.set_unihash(tid, finalunihash, True) return True else: # TODO: What to do here? From patchwork Fri May 24 15:11:56 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 44148 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3C7C4C41513 for ; Fri, 24 May 2024 15:12:12 +0000 (UTC) Received: from mail-wr1-f44.google.com (mail-wr1-f44.google.com [209.85.221.44]) by mx.groups.io with SMTP id smtpd.web11.18334.1716563524178123412 for ; Fri, 24 May 2024 08:12:04 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=M/O+y+HW; spf=pass (domain: linuxfoundation.org, ip: 209.85.221.44, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wr1-f44.google.com with SMTP id ffacd0b85a97d-354f14bd80cso2054409f8f.1 for ; Fri, 24 May 2024 08:12:03 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1716563522; x=1717168322; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=oOBHB9UFcpKgyYj46gSdBED+TC6aAiFH7W+dZlvXhz8=; b=M/O+y+HWzq9j2r2wPQfxHPAw2Y3RqqrVEQQVEaRbNgy55l8KL1egKdvRFm03GIJGTZ w7dGk+s0Wsetrpq+hvadtxmP4Q/30SjVv5FGF/aHCrByxuB9Zqwlxi1cQ2/OgyH1awbl RvQDzqOQdgG4hYuAgdsBw1AKaqccvrZDZqvIQ= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1716563522; x=1717168322; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=oOBHB9UFcpKgyYj46gSdBED+TC6aAiFH7W+dZlvXhz8=; b=lG41ZAy/H1gAGMON/gRyAd5tMDaZmSIAMcj7qBiQJRH1Bz23f+kRniEYpsQjHnbG+a 3ojO1NmtAUxgXHE+vmS7UNp7rtXXj9KtUgJnzc8I2erdkUUqZowl8xTUMAEte8HBghrP vVu8Nr/pUi/XS7T2ywimW+JOdZ1Ohb4sIGYgLhYfocuXz4RkKYD/EMOHRWE4QNr3hAaR jNcQ4HNTw9nx2c4mN1RzkorXnZ2bVk55k83z9WiNKOzgC+Tyo0fejuaRRAr0FxEnbiy0 lRfxFa0cSjmbc3dy1WrlieMG/o4gBkZp4o6+R3H61FEoxSkIGOoMT96RmXDmam9jB+w8 RiFQ== X-Gm-Message-State: AOJu0YzXkxXHiTT7TmIg49MviOuj/8c3KXOCn60ZICvUR73lxpJoUwhp xhBl9dxzHO2z8iv+szWHzY73iRLZYnUQVMmhMnVxUMS/+KTAxTvr0eFpbHEgowkIWPHuyb/wOM1 R X-Google-Smtp-Source: AGHT+IHQQFMcV2yk6shQ/byKUkdQHpA5FWXZcvZbR6uQCd5fUivuVJOcdpQLyrWlrfmliSK4eCtq0w== X-Received: by 2002:a05:6000:12cf:b0:355:1e8:4513 with SMTP id ffacd0b85a97d-3552fdfa8ccmr1552624f8f.53.1716563522416; Fri, 24 May 2024 08:12:02 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:9ca4:3686:8521:57da]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-35579d7db23sm1841759f8f.23.2024.05.24.08.12.01 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 24 May 2024 08:12:02 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 2/6] runqueue: Add timing warnings around slow loops Date: Fri, 24 May 2024 16:11:56 +0100 Message-Id: <20240524151200.2449512-2-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <20240524151200.2449512-1-richard.purdie@linuxfoundation.org> References: <20240524151200.2449512-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 24 May 2024 15:12:12 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16256 With hashserve enabled, there are two slow paths/loops, one at initial runqueue generation and also during the rehash process when new outhashes are found. Add timing information at the hashserve log level for when these loops take longer than 30s or 60s overall. This will leave evidence in the logs when things are running particularly slowly. Signed-off-by: Richard Purdie --- lib/bb/runqueue.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index 6c2034176e..b0375fcb36 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -1273,6 +1273,9 @@ class RunQueueData: bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids) + starttime = time.time() + lasttime = starttime + # Iterate over the task list and call into the siggen code dealtwith = set() todeal = set(self.runtaskentries) @@ -1284,6 +1287,14 @@ class RunQueueData: self.prepare_task_hash(tid) bb.event.check_for_interrupts(self.cooker.data) + if time.time() > (lasttime + 30): + lasttime = time.time() + hashequiv_logger.verbose("Initial setup loop progress: %s of %s in %s" % (len(todeal), len(self.runtaskentries), lasttime - starttime)) + + endtime = time.time() + if (endtime-starttime > 60): + hashequiv_logger.verbose("Initial setup loop took: %s" % (endtime-starttime)) + bb.parse.siggen.writeout_file_checksum_cache() #self.dump_data() @@ -2557,6 +2568,9 @@ class RunQueueExecute: elif self.rqdata.runtaskentries[p].depends.isdisjoint(total): next.add(p) + starttime = time.time() + lasttime = starttime + # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled while next: current = next.copy() @@ -2589,6 +2603,14 @@ class RunQueueExecute: total.remove(tid) next.intersection_update(total) + if time.time() > (lasttime + 30): + lasttime = time.time() + hashequiv_logger.verbose("Rehash loop slow progress: %s in %s" % (len(total), lasttime - starttime)) + + endtime = time.time() + if (endtime-starttime > 60): + hashequiv_logger.verbose("Rehash loop took more than 60s: %s" % (endtime-starttime)) + if changed: for mc in self.rq.worker: RunQueue.send_pickled_data(self.rq.worker[mc].process, bb.parse.siggen.get_taskhashes(), "newtaskhashes") From patchwork Fri May 24 15:11:57 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 44147 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4A18AC25B7D for ; Fri, 24 May 2024 15:12:12 +0000 (UTC) Received: from mail-wm1-f50.google.com (mail-wm1-f50.google.com [209.85.128.50]) by mx.groups.io with SMTP id smtpd.web10.18338.1716563524818420007 for ; Fri, 24 May 2024 08:12:05 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=fwHRuSy+; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.50, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f50.google.com with SMTP id 5b1f17b1804b1-4210aa012e5so4989365e9.0 for ; Fri, 24 May 2024 08:12:04 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1716563523; x=1717168323; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=V3ZVElsGXsdf6vPD0TMfEomM31zlR7YLwEb8z9YXX3Q=; b=fwHRuSy+/d2RboEmWPgbLxLIWpBCzGBWCzyojoqqSTue2AZ9gr0oMV6/jA/bTESJN3 +/a9xm4QwpS3XJOpvf2SW4f027+NeSLYUXh69mytPSuqPTyA48uk/xpBepaO4f9ruYyQ mFEp5JFwn/0mjSDuQbBV9taaXlaqprlTaiFSk= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1716563523; x=1717168323; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=V3ZVElsGXsdf6vPD0TMfEomM31zlR7YLwEb8z9YXX3Q=; b=K9DDnNZ99t9f4wnTBL1EuJJR05PkqedE5lIv4QD4Q09GJM2bRnPrPFcCKT8u8WLAwz MfANim7YBnL4PabSarnCqUjlsKSflFvijioKZDbcdzSO4m7kPRrdkVmb6jYhP3JuAPN2 Xgt8ceqVs0mKb+7DqrYjV4b/IS6PpVOJH4EHGVP2COYFgJoHMA/LPaEzeGaDDnR/T38D BFlcEngkKqI8wP42H+RI01x7BMBl4vm20aeNZ4HtDe70q+tuSJDYj2/malQ6hOEEdFM2 Kiczu0bn/xvF99dFbPR2MHo7AnrnuTxd+Y4J94JV1vnt+quARhsdXL9GfR54kdsTiHBF 5gfQ== X-Gm-Message-State: AOJu0YwXdE0+V+VbtYRaiiwbgNaS1+o4Qgs5a2AR0YERfdW3K8A+JQqy sIwX/Uboq6PaqneqvwScoUA4pkteZcXB6qCVns+xj2RvPx6GO+6FLYIjT/mMRrnyE9sG8yzrFBU 0 X-Google-Smtp-Source: AGHT+IG8po/zyI0OccQr5TZbH0+9Ialkh3wKUrMmeXBnvZxi4ig8iWcaPoFnP6UJxEYwokTSwRCB4g== X-Received: by 2002:a5d:6084:0:b0:355:448:f18e with SMTP id ffacd0b85a97d-3552fde0574mr1838909f8f.43.1716563523163; Fri, 24 May 2024 08:12:03 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:9ca4:3686:8521:57da]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-35579d7db23sm1841759f8f.23.2024.05.24.08.12.02 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 24 May 2024 08:12:02 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 3/6] runqueue: Allow rehash loop to exit in case of interrupts Date: Fri, 24 May 2024 16:11:57 +0100 Message-Id: <20240524151200.2449512-3-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <20240524151200.2449512-1-richard.purdie@linuxfoundation.org> References: <20240524151200.2449512-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 24 May 2024 15:12:12 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16257 The initial hash serve loop exits in the case where interrupts are present but probably checks a bit too often. Tweak that and also allow the slow rehash loop to break on interrupt, improving bitbake Ctrl+C response. Signed-off-by: Richard Purdie --- lib/bb/runqueue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index b0375fcb36..586b11da2d 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -1285,7 +1285,8 @@ class RunQueueData: dealtwith.add(tid) todeal.remove(tid) self.prepare_task_hash(tid) - bb.event.check_for_interrupts(self.cooker.data) + + bb.event.check_for_interrupts(self.cooker.data) if time.time() > (lasttime + 30): lasttime = time.time() @@ -2602,6 +2603,7 @@ class RunQueueExecute: next |= self.rqdata.runtaskentries[tid].revdeps total.remove(tid) next.intersection_update(total) + bb.event.check_for_interrupts(self.cooker.data) if time.time() > (lasttime + 30): lasttime = time.time() From patchwork Fri May 24 15:11:58 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 44150 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 75B06C25B7F for ; Fri, 24 May 2024 15:12:12 +0000 (UTC) Received: from mail-wr1-f42.google.com (mail-wr1-f42.google.com [209.85.221.42]) by mx.groups.io with SMTP id smtpd.web10.18339.1716563525636364023 for ; Fri, 24 May 2024 08:12:05 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=CsjBM4X6; spf=pass (domain: linuxfoundation.org, ip: 209.85.221.42, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wr1-f42.google.com with SMTP id ffacd0b85a97d-351b683f2d8so2754258f8f.3 for ; Fri, 24 May 2024 08:12:05 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1716563524; x=1717168324; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=NMW549WOIBzOaANNltcjbXiOT7u1/Hm53BVpCCp1Al4=; b=CsjBM4X65Ehg+Eq0CawcdfnurUgep0EvRRzyau37Fe3j+uXS3+80CznWAGvsqcjajZ ouajk2kNMLsTxx3Ckv2wfmB9OaumpRIibCjpWgTyxitFlOM/WiZl56MPQ8FkDj+oeS7b mCVDgbaiIfeCWWLTVBsp7rOz6J4mYLKE28gEs= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1716563524; x=1717168324; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=NMW549WOIBzOaANNltcjbXiOT7u1/Hm53BVpCCp1Al4=; b=Sh8mjK2T98RvqpPAjgTqQYN1I1i55h9GjMNBtHaAAoYTKGogaH+hDcVc543PU2DxVo aSxDT1kJT2xH9dNQqUGbxX9UhSEFeakvyjCTl4w7FMK7353QHRiX+sAbknVJGML3/80w SbhCG1SuUwIwfuahRGUGTIBOJxs1xtSIAulmuyMvO2b/oekB34SfAEzN533s3CGCx+dS +WyrDf7vthRhb9k6jjAKMJVpoc35APUsjfE4kX6Ld6jHqk/aNRePEy8FY1ziz3FsAm4i KQMKV7mnLF/iKWnv8qIbSEpaPsGBuD/N8Qp4cmnVesENMBBqVw0s5WAI4cXvcTSMbcl7 ykRA== X-Gm-Message-State: AOJu0Yxhp/9vtjAHpX5nZsOGAmVeZIKkGO1pTJseWt8I5XGDGYI0CG4H 3krKtIaLsKOJqqMPJ+nB8vvFOkssjlN3oBxSEef7GqC/ZfJH0Owc7MazmnU4zinEnKmM94jrUvl V X-Google-Smtp-Source: AGHT+IEJLZNl3ikiBpGgP+F2mPeA6aiHEXlCpJzj4JbIHfihoKPRT7itf0hzajYmLX/uB4+xBarFsA== X-Received: by 2002:adf:a44a:0:b0:355:41b:d38e with SMTP id ffacd0b85a97d-35527047b2dmr1610454f8f.36.1716563523929; Fri, 24 May 2024 08:12:03 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:9ca4:3686:8521:57da]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-35579d7db23sm1841759f8f.23.2024.05.24.08.12.03 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 24 May 2024 08:12:03 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 4/6] runqueue: Process unihashes in parallel at init Date: Fri, 24 May 2024 16:11:58 +0100 Message-Id: <20240524151200.2449512-4-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <20240524151200.2449512-1-richard.purdie@linuxfoundation.org> References: <20240524151200.2449512-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 24 May 2024 15:12:12 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16258 Improve the runqueue init code to call unihash queries in parallel since this is faster and more efficient, particularly on slower links with longer round trip times. Signed-off-by: Richard Purdie --- lib/bb/runqueue.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index 586b11da2d..411c47eff9 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -1280,11 +1280,18 @@ class RunQueueData: dealtwith = set() todeal = set(self.runtaskentries) while todeal: + ready = set() for tid in todeal.copy(): if not (self.runtaskentries[tid].depends - dealtwith): - dealtwith.add(tid) - todeal.remove(tid) - self.prepare_task_hash(tid) + self.runtaskentries[tid].taskhash_deps = bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) + # get_taskhash for a given tid *must* be called before get_unihash* below + self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) + ready.add(tid) + unihashes = bb.parse.siggen.get_unihashes(ready) + for tid in ready: + dealtwith.add(tid) + todeal.remove(tid) + self.runtaskentries[tid].unihash = unihashes[tid] bb.event.check_for_interrupts(self.cooker.data) @@ -1301,11 +1308,6 @@ class RunQueueData: #self.dump_data() return len(self.runtaskentries) - def prepare_task_hash(self, tid): - self.runtaskentries[tid].taskhash_deps = bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) - self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) - self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid) - def dump_data(self): """ Dump some debug information on the internal data structures From patchwork Fri May 24 15:11:59 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 44145 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 23CD6C25B7C for ; Fri, 24 May 2024 15:12:12 +0000 (UTC) Received: from mail-lj1-f170.google.com (mail-lj1-f170.google.com [209.85.208.170]) by mx.groups.io with SMTP id smtpd.web11.18335.1716563527056524168 for ; Fri, 24 May 2024 08:12:07 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=P3Y/bBg6; spf=pass (domain: linuxfoundation.org, ip: 209.85.208.170, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-lj1-f170.google.com with SMTP id 38308e7fff4ca-2e73441edf7so57261031fa.1 for ; Fri, 24 May 2024 08:12:06 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1716563525; x=1717168325; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=YuZlwnR7BxBUrMFYytDdPppJ203LnrvSWGEdSStHS1I=; b=P3Y/bBg6+3lX7snQ51xUbaxbx5sNsSA9zV7RnCjbpt93Z0YS1v0EmPAKBIGzbwnyGh DszEgT2E5FaQTeSlL9jpbCarGZPBz0dVzTmdhVFnEeWxeUQOSMhcShJ4m6EaYHnGz9AJ Nu9LJfgjhVJa1Y2p6a+QYSfutQ3lWWkgX8epU= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1716563525; x=1717168325; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=YuZlwnR7BxBUrMFYytDdPppJ203LnrvSWGEdSStHS1I=; b=qz7dJOKgQYf8wE/7K03HJSAzchlbpnatfBV/l8qalLieAcn+AENaHZExV58tt/IU6h XNglj+h/I9D4wKJk0mqhLpN7lPtjTo0Zgh6/9kjoXFhWlMVNyt1X0chwrkABtT1JQwB6 ldXTgislDJVjAtmPscTzDlxbzf0GOh60rzw/IPPpYfQHstZl5fZRdngXtn04te4hBJ3+ o1ZBd7Ha/Tf8m8yuE+L0F/M5HPSuJyrcQG6CS6CNFv89LbjNApuWlbjggA4Wj0XpcO/D lPZFA5vgtagLFBam/fC5YHy+0GZZ0AdoM7Od4wKuWINgF+QziN296czEBuRg+dUzKDlz 2KhQ== X-Gm-Message-State: AOJu0Yzly6JBahU26P+ym2mI1tdXPFfg3YGjzkpkU+yAeOtE3haAZpHY 2wcy0x1y5SZ3SDpHJwvi0KjRWJ3GEN4OzuXbNuhrCEB70e+toN4NwgeZbZF/bhtH+082pJCnHx9 I X-Google-Smtp-Source: AGHT+IFJ/kWfKpZmVNFLtLSs1SMm1eKUS6/WuDuYf5HuW/TXaTeNawrH93UHeLCqN0QZ6ayj5xntvA== X-Received: by 2002:a05:651c:107b:b0:2dc:bf5c:9648 with SMTP id 38308e7fff4ca-2e95b09b355mr18669621fa.19.1716563524631; Fri, 24 May 2024 08:12:04 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:9ca4:3686:8521:57da]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-35579d7db23sm1841759f8f.23.2024.05.24.08.12.04 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 24 May 2024 08:12:04 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 5/6] runqueue: Improve rehash get_unihash parallelism Date: Fri, 24 May 2024 16:11:59 +0100 Message-Id: <20240524151200.2449512-5-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <20240524151200.2449512-1-richard.purdie@linuxfoundation.org> References: <20240524151200.2449512-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 24 May 2024 15:12:12 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16259 Improve the rehash code to query unihashes in parallel since this is more efficient on slower links. Signed-off-by: Richard Purdie --- lib/bb/runqueue.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index 411c47eff9..5626ffcccb 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -2578,13 +2578,21 @@ class RunQueueExecute: while next: current = next.copy() next = set() + ready = {} for tid in current: if self.rqdata.runtaskentries[p].depends and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total): continue + # get_taskhash for a given tid *must* be called before get_unihash* below + ready[tid] = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches) + + unihashes = bb.parse.siggen.get_unihashes(ready.keys()) + + for tid in ready: orighash = self.rqdata.runtaskentries[tid].hash - newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches) + newhash = ready[tid] origuni = self.rqdata.runtaskentries[tid].unihash - newuni = bb.parse.siggen.get_unihash(tid) + newuni = unihashes[tid] + # FIXME, need to check it can come from sstate at all for determinism? remapped = False if newuni == origuni: From patchwork Fri May 24 15:12:00 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 44149 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3D238C25B7A for ; Fri, 24 May 2024 15:12:12 +0000 (UTC) Received: from mail-wm1-f47.google.com (mail-wm1-f47.google.com [209.85.128.47]) by mx.groups.io with SMTP id smtpd.web10.18340.1716563527433556294 for ; Fri, 24 May 2024 08:12:07 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=bHDf4pgf; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.47, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f47.google.com with SMTP id 5b1f17b1804b1-420180b5897so30379305e9.3 for ; Fri, 24 May 2024 08:12:07 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1716563526; x=1717168326; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=s50+QmbbVCh1Kn71NtkSqBhDOjNbtsk6XB7tf6bNMXA=; b=bHDf4pgfh6yWEdC64H20SYAfQ/sabyZpl0BfvAFa48SntA3LuBE5UMqgcTZqy9ZY99 JYr2cn91mZ6UpYN3L/BZ5OfAB4F1bpWf6ayVivPc7rLGz9uta3Lk4+cQ6t1OgUOprTO6 WaWFUk5tZEPkBQJ7tve/79ZrwnRlYL0bLtYq4= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1716563526; x=1717168326; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=s50+QmbbVCh1Kn71NtkSqBhDOjNbtsk6XB7tf6bNMXA=; b=UxawHIe7M2Q0p8yDyw2a91AxL+blnS10ND3dYqdfDrxHLsLlsyalGOAxibH9lb2+mW Miie92x6foSBJt5lmYACF3v5tgCWxFVgwaIXoykURgrgxXVubXraMrYe4IRqPVCgedF7 VM1ESvUAdbpNpxcixp14rPzMhVwfgtVjims7afED4sKPESWoUWl5GjV9q9rW1MZ8c72O jbV7mxQYMugQLaMMOJcQ10vP06mhy6NG10K22skpuMwQ5Mb1frdZ26fhaqaAnC/LfBKt dgLem+vPw2n9hK5at1MO/LiHcrteDxuQ7NaoOmEaWzK5umdRmIOxVkGxYJDF0eXia927 kSxg== X-Gm-Message-State: AOJu0Yz3dhynf+ZA/kbWMi0418P1W9N5azxiR01fQeNnOwBhyFlXTwga 5RmjdCt9eHkAS17CZgAfdFRiRrdwQf/QFMACl70cJqfjMxuxeeLpitFzsvQbYeBrKPYaU8w4pOr G X-Google-Smtp-Source: AGHT+IFWu8UWcn1pSN02UzAUF3wLiuCHzrwk1XsektiB9iCCdHTSK9EsEsB6E90PO1ohoMSW24KAPA== X-Received: by 2002:adf:fc89:0:b0:355:148:ea23 with SMTP id ffacd0b85a97d-3552fdc7ca9mr1676754f8f.56.1716563525492; Fri, 24 May 2024 08:12:05 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:9ca4:3686:8521:57da]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-35579d7db23sm1841759f8f.23.2024.05.24.08.12.04 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 24 May 2024 08:12:05 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 6/6] siggen/runqueue: Allow skipping of rehash if not present on hashserve Date: Fri, 24 May 2024 16:12:00 +0100 Message-Id: <20240524151200.2449512-6-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.40.1 In-Reply-To: <20240524151200.2449512-1-richard.purdie@linuxfoundation.org> References: <20240524151200.2449512-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 24 May 2024 15:12:12 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16260 If a unihash isn't present on the hashserver, it is extremely unlikely any following unihashes would be either. Use this information to skip unihash checks in that scenario where there is low matching to allow the build to proceed more quickly. This significantly improves performance on the low match scenario. Signed-off-by: Richard Purdie --- lib/bb/runqueue.py | 35 ++++++++++++++++++++++++++---- lib/bb/siggen.py | 54 +++++++++++++++++++++++++--------------------- 2 files changed, 61 insertions(+), 28 deletions(-) diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index 5626ffcccb..0fb9edc77e 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -472,6 +472,7 @@ class RunQueueData: self.reset() def reset(self): + self.hashserve_valid = set() self.runtaskentries = {} def runq_depends_names(self, ids): @@ -1287,11 +1288,23 @@ class RunQueueData: # get_taskhash for a given tid *must* be called before get_unihash* below self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) ready.add(tid) - unihashes = bb.parse.siggen.get_unihashes(ready) + + ready2 = set() + for tid in ready: + if self.runtaskentries[tid].depends.issubset(self.hashserve_valid): + ready2.add(tid) + unihashes = bb.parse.siggen.get_unihashes(ready2) for tid in ready: dealtwith.add(tid) todeal.remove(tid) - self.runtaskentries[tid].unihash = unihashes[tid] + if tid in unihashes: + self.runtaskentries[tid].unihash, present = unihashes[tid] + if present: + self.hashserve_valid.add(tid) + else: + taskhash = self.runtaskentries[tid].hash + bb.parse.siggen.set_unihash(tid, taskhash, False) + self.runtaskentries[tid].unihash = taskhash bb.event.check_for_interrupts(self.cooker.data) @@ -2585,13 +2598,25 @@ class RunQueueExecute: # get_taskhash for a given tid *must* be called before get_unihash* below ready[tid] = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches) - unihashes = bb.parse.siggen.get_unihashes(ready.keys()) + toquery = set() + for tid in ready: + if self.rqdata.runtaskentries[tid].depends.issubset(self.rqdata.hashserve_valid): + toquery.add(tid) + + unihashes = bb.parse.siggen.get_unihashes(toquery) for tid in ready: orighash = self.rqdata.runtaskentries[tid].hash newhash = ready[tid] origuni = self.rqdata.runtaskentries[tid].unihash - newuni = unihashes[tid] + + if tid in unihashes: + newuni, present = unihashes[tid] + if present: + self.rqdata.hashserve_valid.add(tid) + else: + bb.parse.siggen.set_unihash(tid, origuni, False) + newuni = origuni # FIXME, need to check it can come from sstate at all for determinism? remapped = False @@ -2623,6 +2648,8 @@ class RunQueueExecute: if (endtime-starttime > 60): hashequiv_logger.verbose("Rehash loop took more than 60s: %s" % (endtime-starttime)) + bb.parse.siggen.save_unitaskhashes() + if changed: for mc in self.rq.worker: RunQueue.send_pickled_data(self.rq.worker[mc].process, bb.parse.siggen.get_taskhashes(), "newtaskhashes") diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py index e8fe806c36..56528ea10a 100644 --- a/lib/bb/siggen.py +++ b/lib/bb/siggen.py @@ -112,16 +112,19 @@ class SignatureGenerator(object): self.datacaches[mc].stamp_extrainfo[mcfn][t] = flag def get_cached_unihash(self, tid): - return None - - def get_unihash(self, tid): - unihash = self.get_cached_unihash(tid) - if unihash: + return None, False + + def get_unihash(self, tid, presenceinfo=False): + unihash, present = self.get_cached_unihash(tid) + if not unihash: + unihash, present = self.taskhash[tid], False + if presenceinfo: + return unihash, present + else: return unihash - return self.taskhash[tid] def get_unihashes(self, tids): - return {tid: self.get_unihash(tid) for tid in tids} + return {tid: self.get_unihash(tid, True) for tid in tids} def prep_taskhash(self, tid, deps, dataCaches): return @@ -624,7 +627,7 @@ class SignatureGeneratorUniHashMixIn(object): # If a unique hash is reported, use it as the stampfile hash. This # ensures that if a task won't be re-run if the taskhash changes, # but it would result in the same output hash - unihash = self._get_unihash(tid) + unihash, _ = self._get_unihash(tid) if unihash is not None: return unihash @@ -638,17 +641,17 @@ class SignatureGeneratorUniHashMixIn(object): def _get_unihash(self, tid, checkkey=None): if tid not in self.tidtopn: - return None + return None, False (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid) key = mc + ":" + self.tidtopn[tid] + ":" + taskname if key not in self.hashserv_cache: - return None + return None, False if not checkkey: checkkey = self.taskhash[tid] (key, unihash, present) = self.hashserv_cache[key] if key != checkkey: - return None - return unihash + return None, False + return unihash, present def get_cached_unihash(self, tid): taskhash = self.taskhash[tid] @@ -656,16 +659,16 @@ class SignatureGeneratorUniHashMixIn(object): # If its not a setscene task we can return if self.setscenetasks and tid not in self.setscenetasks: self.unihash[tid] = None - return taskhash + return taskhash, True # TODO: This cache can grow unbounded. It probably only needs to keep # for each task - unihash = self._get_unihash(tid) + unihash, present = self._get_unihash(tid) if unihash is not None: self.unihash[tid] = unihash - return unihash + return unihash, present - return None + return None, False def _get_method(self, tid): method = self.method @@ -703,8 +706,10 @@ class SignatureGeneratorUniHashMixIn(object): return result - def get_unihash(self, tid): - return self.get_unihashes([tid])[tid] + def get_unihash(self, tid, presenceinfo=False): + if presenceinfo: + return self.get_unihashes([tid])[tid] + return self.get_unihashes([tid])[tid][0] def get_unihashes(self, tids): """ @@ -716,9 +721,9 @@ class SignatureGeneratorUniHashMixIn(object): query_result = {} for tid in tids: - unihash = self.get_cached_unihash(tid) + unihash, present = self.get_cached_unihash(tid) if unihash: - result[tid] = unihash + result[tid] = (unihash, present) else: queries[tid] = (self._get_method(tid), self.taskhash[tid]) @@ -754,14 +759,15 @@ class SignatureGeneratorUniHashMixIn(object): # so it is reported it at debug level 2. If they differ, that # is much more interesting, so it is reported at debug level 1 hashequiv_logger.bbdebug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server)) - self.set_unihash(tid, unihash, True) + present = True else: hashequiv_logger.debug2('No reported unihash for %s:%s from %s' % (tid, taskhash, self.server)) unihash = taskhash - self.set_unihash(tid, unihash, False) + present = False + self.set_unihash(tid, unihash, present) self.unihash[tid] = unihash - result[tid] = unihash + result[tid] = (unihash, present) return result @@ -784,7 +790,7 @@ class SignatureGeneratorUniHashMixIn(object): return # Sanity checks - cache_unihash = self._get_unihash(tid, checkkey=taskhash) + cache_unihash, _ = self._get_unihash(tid, checkkey=taskhash) if cache_unihash is None: bb.fatal('%s not in unihash cache. Please report this error' % key)