Message ID | cd06a65773c4be6a51ccba0155cbebe42c2f355e.1722137668.git.liezhi.yang@windriver.com |
---|---|
State | Accepted, archived |
Commit | 0596aa0d5b0e4ed3db11b5bd560f1d3439963a41 |
Headers | show |
Series | [1/1] data_smart: Improve performance for VariableHistory | expand |
On Sat, 2024-07-27 at 20:39 -0700, Robert Yang via lists.openembedded.org wrote: > From: Robert Yang <liezhi.yang@windriver.com> > > Fixed: > - BBMULTICONFIG = "qemux86-64 qemuarm64" and more than 70 layers in BBLAYERS > $ bitbake -p -P > Check profile.log.processed, the record() cost more than 20 seconds, it is less > than 1 second when multiconfig is not enabled, and there would be the following > error when more muticonfigs are enabled: > > Timeout while waiting for a reply from the bitbake server > > Don't change the type of loginfo['detail'] or re-assign it can make record() > back to less than 1 second, this won't affect COW since loginfo is a mutable > type. > > The time mainly affected by two factors: > 1) The number of enabled layers, nearly 1 second added per layer when the > number is larger than 50. > > 2) The global var such as USER_CLASSES, about 1 ~ 2 seconds added per layer > when the layers number is larger than 50. > > Signed-off-by: Robert Yang <liezhi.yang@windriver.com> > --- > bitbake/lib/bb/data_smart.py | 8 +++----- > 1 file changed, 3 insertions(+), 5 deletions(-) > > diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py > index 0128a5bb17..c6049d578e 100644 > --- a/bitbake/lib/bb/data_smart.py > +++ b/bitbake/lib/bb/data_smart.py > @@ -272,12 +272,9 @@ class VariableHistory(object): > return > if 'op' not in loginfo or not loginfo['op']: > loginfo['op'] = 'set' > - if 'detail' in loginfo: > - loginfo['detail'] = str(loginfo['detail']) > if 'variable' not in loginfo or 'file' not in loginfo: > raise ValueError("record() missing variable or file.") > var = loginfo['variable'] > - > if var not in self.variables: > self.variables[var] = [] > if not isinstance(self.variables[var], list): > @@ -336,7 +333,8 @@ class VariableHistory(object): > flag = '[%s] ' % (event['flag']) > else: > flag = '' > - o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) > + o.write("# %s %s:%s%s\n# %s\"%s\"\n" % \ > + (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', str(event['detail'])))) > if len(history) > 1: > o.write("# pre-expansion value:\n") > o.write('# "%s"\n' % (commentVal)) > @@ -390,7 +388,7 @@ class VariableHistory(object): > if isset and event['op'] == 'set?': > continue > isset = True > - items = d.expand(event['detail']).split() > + items = d.expand(str(event['detail'])).split() > for item in items: > # This is a little crude but is belt-and-braces to avoid us > # having to handle every possible operation type specifically > This looks like a good find, thanks for digging into it! I'm a little surprised this causes as much of a problem and was a bit worried about unintended side effects but I didn't find any in testing so it has merged, thanks! Cheers, Richard
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py index 0128a5bb17..c6049d578e 100644 --- a/bitbake/lib/bb/data_smart.py +++ b/bitbake/lib/bb/data_smart.py @@ -272,12 +272,9 @@ class VariableHistory(object): return if 'op' not in loginfo or not loginfo['op']: loginfo['op'] = 'set' - if 'detail' in loginfo: - loginfo['detail'] = str(loginfo['detail']) if 'variable' not in loginfo or 'file' not in loginfo: raise ValueError("record() missing variable or file.") var = loginfo['variable'] - if var not in self.variables: self.variables[var] = [] if not isinstance(self.variables[var], list): @@ -336,7 +333,8 @@ class VariableHistory(object): flag = '[%s] ' % (event['flag']) else: flag = '' - o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) + o.write("# %s %s:%s%s\n# %s\"%s\"\n" % \ + (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', str(event['detail'])))) if len(history) > 1: o.write("# pre-expansion value:\n") o.write('# "%s"\n' % (commentVal)) @@ -390,7 +388,7 @@ class VariableHistory(object): if isset and event['op'] == 'set?': continue isset = True - items = d.expand(event['detail']).split() + items = d.expand(str(event['detail'])).split() for item in items: # This is a little crude but is belt-and-braces to avoid us # having to handle every possible operation type specifically