Message ID | 20240528204420.2845521-3-richard.purdie@linuxfoundation.org |
---|---|
State | Accepted, archived |
Commit | 721c97a115a7a4bf21955be79391bd6e0099f40e |
Headers | show |
Series | [1/4] runqueue: Add timing warnings around slow loops | expand |
On Tue, May 28, 2024 at 2:44 PM Richard Purdie via lists.openembedded.org <richard.purdie=linuxfoundation.org@lists.openembedded.org> wrote: > > Improve the runqueue init code to call unihash queries in parallel since > this is faster and more efficient, particularly on slower links with longer > round trip times. > > The call to the function from cooker is unneeded since that function calls > prepare() and hence this functionality will already have run, so drop > that obsolete call. > > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> > --- > lib/bb/cooker.py | 1 - > lib/bb/runqueue.py | 18 ++++++++++-------- > 2 files changed, 10 insertions(+), 9 deletions(-) > > diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py > index 076ddaa58d..255b9f199a 100644 > --- a/lib/bb/cooker.py > +++ b/lib/bb/cooker.py > @@ -1459,7 +1459,6 @@ class BBCooker: > > if t in task or getAllTaskSignatures: > try: > - rq.rqdata.prepare_task_hash(tid) > sig.append([pn, t, rq.rqdata.get_task_unihash(tid)]) > except KeyError: > sig.append(self.getTaskSignatures(target, [t])[0]) > diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py > index 84a6f4172c..999868dd72 100644 > --- a/lib/bb/runqueue.py > +++ b/lib/bb/runqueue.py > @@ -1280,11 +1280,18 @@ class RunQueueData: > dealtwith = set() > todeal = set(self.runtaskentries) > while todeal: > + ready = set() > for tid in todeal.copy(): > if not (self.runtaskentries[tid].depends - dealtwith): > - dealtwith.add(tid) > - todeal.remove(tid) > - self.prepare_task_hash(tid) > + self.runtaskentries[tid].taskhash_deps = bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) > + # get_taskhash for a given tid *must* be called before get_unihash* below > + self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) > + ready.add(tid) > + unihashes = bb.parse.siggen.get_unihashes(ready) > + for tid in ready: > + dealtwith.add(tid) > + todeal.remove(tid) > + self.runtaskentries[tid].unihash = unihashes[tid] > > bb.event.check_for_interrupts(self.cooker.data) > > @@ -1301,11 +1308,6 @@ class RunQueueData: > #self.dump_data() > return len(self.runtaskentries) > > - def prepare_task_hash(self, tid): > - self.runtaskentries[tid].taskhash_deps = bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) > - self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) > - self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid) > - > def dump_data(self): > """ > Dump some debug information on the internal data structures Reviewed-By: Joshua Watt <JPEWhacker@gmail.com> > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#16274): https://lists.openembedded.org/g/bitbake-devel/message/16274 > Mute This Topic: https://lists.openembedded.org/mt/106357289/3616693 > Group Owner: bitbake-devel+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [JPEWhacker@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- >
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index 076ddaa58d..255b9f199a 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -1459,7 +1459,6 @@ class BBCooker: if t in task or getAllTaskSignatures: try: - rq.rqdata.prepare_task_hash(tid) sig.append([pn, t, rq.rqdata.get_task_unihash(tid)]) except KeyError: sig.append(self.getTaskSignatures(target, [t])[0]) diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index 84a6f4172c..999868dd72 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -1280,11 +1280,18 @@ class RunQueueData: dealtwith = set() todeal = set(self.runtaskentries) while todeal: + ready = set() for tid in todeal.copy(): if not (self.runtaskentries[tid].depends - dealtwith): - dealtwith.add(tid) - todeal.remove(tid) - self.prepare_task_hash(tid) + self.runtaskentries[tid].taskhash_deps = bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) + # get_taskhash for a given tid *must* be called before get_unihash* below + self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) + ready.add(tid) + unihashes = bb.parse.siggen.get_unihashes(ready) + for tid in ready: + dealtwith.add(tid) + todeal.remove(tid) + self.runtaskentries[tid].unihash = unihashes[tid] bb.event.check_for_interrupts(self.cooker.data) @@ -1301,11 +1308,6 @@ class RunQueueData: #self.dump_data() return len(self.runtaskentries) - def prepare_task_hash(self, tid): - self.runtaskentries[tid].taskhash_deps = bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) - self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) - self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid) - def dump_data(self): """ Dump some debug information on the internal data structures
Improve the runqueue init code to call unihash queries in parallel since this is faster and more efficient, particularly on slower links with longer round trip times. The call to the function from cooker is unneeded since that function calls prepare() and hence this functionality will already have run, so drop that obsolete call. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> --- lib/bb/cooker.py | 1 - lib/bb/runqueue.py | 18 ++++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-)