| Message ID | 20240206121528.2581233-1-richard.purdie@linuxfoundation.org |
|---|---|
| State | New |
| Headers | show |
| Series | lib/bb: Drop BBFILE_PRIORITY support | expand |
Hi Richard, there is a word missing in the commit message: " The functionality had a and place" -> "the functionality had a time and place" Also a typo: "comparibility" -> " compatibility " What is the correct way to set layer priorities after this patch (I'm asking because the BSP I'm using makes use of BBFILE_PRIORITY)? Etienne On Tue, Feb 6, 2024 at 1:15 PM Richard Purdie < richard.purdie@linuxfoundation.org> wrote: > These entries, usually in layer.conf are part of an old, pre-layers > part of bitbake for overlay/collection support. This is not really > used any more and was replaced by the idea of layers. > > In a recent discussion, we asked the question "When do you need to > use proprities?" and the answer was "almost never" given all the other > tools and techniques now available in layers. > > These layer priorities cause users huge amounts of confusion. It is > unclear to people how the "magic values" work if it is unset > and if they are set differently, users don't expect the way it > actually works and the effects it has between layers. > > In discussions I've had, there has been surprisingly strong support > for removing them entirely, even if that does means we'd need to fix > some corner caes. > > I wholly agree and support that idea, so lets remove the support > and take a simplification where we can. The functionality had a > and place but it has passed and we've moved on. > > Developers are about to need to change layer.conf to update the > series comparibility for the next release so it is a good time to > do it. > > This patch removes the core support and replaces it with a warning. > It does not chase down every last reference or function parmeter, > that can follow as time allows. > > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> > --- > lib/bb/cache.py | 1 - > lib/bb/cooker.py | 51 +++++++-------------------------------- > lib/bb/providers.py | 58 +++++++++++++++++++-------------------------- > lib/bb/utils.py | 2 +- > 4 files changed, 34 insertions(+), 78 deletions(-) > > diff --git a/lib/bb/cache.py b/lib/bb/cache.py > index 18d5574a31..64dbffc0f6 100644 > --- a/lib/bb/cache.py > +++ b/lib/bb/cache.py > @@ -818,7 +818,6 @@ class CacheData(object): > # Indirect Cache variables (set elsewhere) > self.ignored_dependencies = [] > self.world_target = set() > - self.bbfile_priority = {} > > def add_from_recipeinfo(self, fn, info_array): > for info in info_array: > diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py > index c5bfef55d6..381379086a 100644 > --- a/lib/bb/cooker.py > +++ b/lib/bb/cooker.py > @@ -1124,26 +1124,15 @@ class BBCooker: > errors = False > self.bbfile_config_priorities = [] > if collections: > - collection_priorities = {} > collection_depends = {} > collection_list = collections.split() > min_prio = 0 > for c in collection_list: > bb.debug(1,'Processing %s in collection list' % (c)) > > - # Get collection priority if defined explicitly > priority = self.data.getVar("BBFILE_PRIORITY_%s" % c) > if priority: > - try: > - prio = int(priority) > - except ValueError: > - parselog.error("invalid value for > BBFILE_PRIORITY_%s: \"%s\"", c, priority) > - errors = True > - if min_prio == 0 or prio < min_prio: > - min_prio = prio > - collection_priorities[c] = prio > - else: > - collection_priorities[c] = None > + bb.warn("Ignoring obsolete BBFILE_PRIORITY_%s" % c) > > # Check dependencies and store information for priority > calculation > deps = self.data.getVar("LAYERDEPENDS_%s" % c) > @@ -1204,22 +1193,9 @@ class BBCooker: > else: > parselog.debug3("Layer '%s' recommends layer > '%s', but this layer is not enabled in your configuration", c, rec) > > - # Recursively work out collection priorities based on > dependencies > - def calc_layer_priority(collection): > - if not collection_priorities[collection]: > - max_depprio = min_prio > - for dep in collection_depends[collection]: > - calc_layer_priority(dep) > - depprio = collection_priorities[dep] > - if depprio > max_depprio: > - max_depprio = depprio > - max_depprio += 1 > - parselog.debug("Calculated priority of layer %s as > %d", collection, max_depprio) > - collection_priorities[collection] = max_depprio > - > - # Calculate all layer priorities using calc_layer_priority > and store in bbfile_config_priorities > + > + # Store all layer priorities in bbfile_config_priorities > for c in collection_list: > - calc_layer_priority(c) > regex = self.data.getVar("BBFILE_PATTERN_%s" % c) > if regex is None: > parselog.error("BBFILE_PATTERN_%s not defined" % c) > @@ -1236,7 +1212,7 @@ class BBCooker: > parselog.error("BBFILE_PATTERN_%s \"%s\" is not a > valid regular expression", c, regex) > errors = True > continue > - self.bbfile_config_priorities.append((c, regex, cre, > collection_priorities[c])) > + self.bbfile_config_priorities.append((c, regex, cre, 0)) > if errors: > # We've already printed the actual error(s) > raise CollectionError("Errors during parsing layer > configuration") > @@ -1358,7 +1334,6 @@ class BBCooker: > # Tweak some variables > item = info_array[0].pn > self.recipecaches[mc].ignored_dependencies = set() > - self.recipecaches[mc].bbfile_priority[fn] = 1 > self.configuration.limited_deps = True > > # Remove external dependencies > @@ -1639,7 +1614,7 @@ class BBCooker: > self.show_appends_with_no_recipes() > self.handlePrefProviders() > for mc in self.multiconfigs: > - self.recipecaches[mc].bbfile_priority = > self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, > self.parser.mcfilelist[mc], self.data) > + > self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, > self.parser.mcfilelist[mc], self.data) > self.state = state.running > > # Send an event listing all stamps reachable after parsing > @@ -1767,9 +1742,9 @@ class CookerCollectFiles(object): > self.bbfile_config_priorities = sorted(priorities, key=lambda > tup: tup[1], reverse=True) > > def calc_bbfile_priority(self, filename): > - for layername, _, regex, pri in self.bbfile_config_priorities: > + for layername, _, regex, _ in self.bbfile_config_priorities: > if regex.match(filename): > - return pri, regex, layername > + return 0, regex, layername > return 0, None, None > > def get_bbfiles(self): > @@ -1801,10 +1776,6 @@ class CookerCollectFiles(object): > > files = (config.getVar( "BBFILES") or "").split() > > - # Sort files by priority > - files.sort( key=lambda fileitem: > self.calc_bbfile_priority(fileitem)[0] ) > - config.setVar("BBFILES_PRIORITIZED", " ".join(files)) > - > if not files: > files = self.get_bbfiles() > > @@ -1926,12 +1897,10 @@ class CookerCollectFiles(object): > return tuple(filelist) > > def collection_priorities(self, pkgfns, fns, d): > - # Return the priorities of the entries in pkgfns > - # Also check that all the regexes in > self.bbfile_config_priorities are used > + # Check that all the regexes in self.bbfile_config_priorities are > used > # (but to do that we need to ensure skipped recipes aren't > counted, nor > # collections in BBFILE_PATTERN_IGNORE_EMPTY) > > - priorities = {} > seen = set() > matched = set() > > @@ -1943,7 +1912,7 @@ class CookerCollectFiles(object): > # Calculate priorities for each file > for p in pkgfns: > realfn, cls, mc = bb.cache.virtualfn2realfn(p) > - priorities[p], regex, _ = self.calc_bbfile_priority(realfn) > + _, regex, _ = self.calc_bbfile_priority(realfn) > if regex in unmatched_regex: > matched_regex.add(regex) > unmatched_regex.remove(regex) > @@ -1985,8 +1954,6 @@ class CookerCollectFiles(object): > collectlog.warning("No bb files in %s matched > BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default', > > collection, pattern)) > > - return priorities > - > class ParsingFailure(Exception): > def __init__(self, realexception, recipe): > self.realexception = realexception > diff --git a/lib/bb/providers.py b/lib/bb/providers.py > index e11a4637d1..f527dc469f 100644 > --- a/lib/bb/providers.py > +++ b/lib/bb/providers.py > @@ -71,21 +71,15 @@ def sortPriorities(pn, dataCache, pkg_pn = None): > files = pkg_pn[pn] > priorities = {} > for f in files: > - priority = dataCache.bbfile_priority[f] > preference = dataCache.pkg_dp[f] > - if priority not in priorities: > - priorities[priority] = {} > - if preference not in priorities[priority]: > - priorities[priority][preference] = [] > - priorities[priority][preference].append(f) > - tmp_pn = [] > - for pri in sorted(priorities): > - tmp_pref = [] > - for pref in sorted(priorities[pri]): > - tmp_pref.extend(priorities[pri][pref]) > - tmp_pn = [tmp_pref] + tmp_pn > - > - return tmp_pn > + if preference not in priorities: > + priorities[preference] = [] > + priorities[preference].append(f) > + tmp_pref = [] > + for pref in sorted(priorities): > + tmp_pref.extend(priorities[pref]) > + > + return tmp_pref > > def versionVariableMatch(cfgData, keyword, pn): > """ > @@ -115,7 +109,7 @@ def preferredVersionMatch(pe, pv, pr, preferred_e, > preferred_v, preferred_r): > return True > return False > > -def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = > None): > +def findPreferredProvider(pn, cfgData, dataCache, pkg_pn, item): > """ > Find the first provider in pkg_pn with REQUIRED_VERSION or > PREFERRED_VERSION set. > """ > @@ -156,15 +150,12 @@ def findPreferredProvider(pn, cfgData, dataCache, > pkg_pn = None, item = None): > preferred_e = None > preferred_r = None > > - for file_set in pkg_pn: > - for f in file_set: > - pe, pv, pr = dataCache.pkg_pepvpr[f] > - if preferredVersionMatch(pe, pv, pr, preferred_e, > preferred_v, preferred_r): > - preferred_file = f > - preferred_ver = (pe, pv, pr) > - break > - if preferred_file: > - break; > + for f in pkg_pn: > + pe, pv, pr = dataCache.pkg_pepvpr[f] > + if preferredVersionMatch(pe, pv, pr, preferred_e, > preferred_v, preferred_r): > + preferred_file = f > + preferred_ver = (pe, pv, pr) > + break > if preferred_r: > pv_str = '%s-%s' % (preferred_v, preferred_r) > else: > @@ -175,14 +166,13 @@ def findPreferredProvider(pn, cfgData, dataCache, > pkg_pn = None, item = None): > if not required: > logger.warning("preferred version %s of %s not > available%s", pv_str, pn, itemstr) > available_vers = [] > - for file_set in pkg_pn: > - for f in file_set: > - pe, pv, pr = dataCache.pkg_pepvpr[f] > - ver_str = pv > - if pe: > - ver_str = "%s:%s" % (pe, ver_str) > - if not ver_str in available_vers: > - available_vers.append(ver_str) > + for f in pkg_pn: > + pe, pv, pr = dataCache.pkg_pepvpr[f] > + ver_str = pv > + if pe: > + ver_str = "%s:%s" % (pe, ver_str) > + if not ver_str in available_vers: > + available_vers.append(ver_str) > if available_vers: > available_vers.sort() > logger.warning("versions of %s available: %s", pn, ' > '.join(available_vers)) > @@ -226,7 +216,7 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = > None, item = None): > # Find the highest priority provider with a REQUIRED_VERSION or > PREFERRED_VERSION set > (preferred_ver, preferred_file, required) = findPreferredProvider(pn, > cfgData, dataCache, sortpkg_pn, item) > # Find the latest version of the highest priority provider > - (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, > sortpkg_pn[0]) > + (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, > sortpkg_pn) > > if not required and preferred_file is None: > preferred_file = latest_f > @@ -272,7 +262,7 @@ def _filterProviders(providers, item, cfgData, > dataCache): > for pn in sorted(sortpkg_pn): > if pn in preferred_versions and preferred_versions[pn][1]: > continue > - preferred_versions[pn] = findLatestProvider(pn, cfgData, > dataCache, sortpkg_pn[pn][0]) > + preferred_versions[pn] = findLatestProvider(pn, cfgData, > dataCache, sortpkg_pn[pn]) > eligible.append(preferred_versions[pn][1]) > > if not eligible: > diff --git a/lib/bb/utils.py b/lib/bb/utils.py > index 068b631c94..3ee3aa8875 100644 > --- a/lib/bb/utils.py > +++ b/lib/bb/utils.py > @@ -1563,7 +1563,7 @@ def get_file_layer(filename, d, collection_res={}): > return match > > result = None > - bbfiles = (d.getVar('BBFILES_PRIORITIZED') or '').split() > + bbfiles = (d.getVar('BBFILES') or '').split() > bbfilesmatch = False > for bbfilesentry in bbfiles: > if fnmatch.fnmatchcase(filename, bbfilesentry): > -- > 2.40.1 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#15828): > https://urldefense.proofpoint.com/v2/url?u=https-3A__lists.openembedded.org_g_bitbake-2Ddevel_message_15828&d=DwIFaQ&c=ncDTmphkJTvjIDPh0hpF_4vCHvabgGkICC2epckfdiw&r=AhkbNonVuMIGRfPx_Qj9TsRih1DULJTKUkSGa66m67E&m=-Bk9X9Hkhx7e8xkBu1GuKqAlibTmXrlpwi04sRozJiDG2hny7t3LoP4wS7J5-J1X&s=_a531uesDcSw3cAizdlUN2ELeqExHwsmX8gPYd9A4M8&e= > Mute This Topic: > https://urldefense.proofpoint.com/v2/url?u=https-3A__lists.openembedded.org_mt_104196430_7048771&d=DwIFaQ&c=ncDTmphkJTvjIDPh0hpF_4vCHvabgGkICC2epckfdiw&r=AhkbNonVuMIGRfPx_Qj9TsRih1DULJTKUkSGa66m67E&m=-Bk9X9Hkhx7e8xkBu1GuKqAlibTmXrlpwi04sRozJiDG2hny7t3LoP4wS7J5-J1X&s=I91ejppj2TYm8gaYh7rkXuF4MY5pjp0ygWa4N1-6_74&e= > Group Owner: bitbake-devel+owner@lists.openembedded.org > Unsubscribe: > https://urldefense.proofpoint.com/v2/url?u=https-3A__lists.openembedded.org_g_bitbake-2Ddevel_unsub&d=DwIFaQ&c=ncDTmphkJTvjIDPh0hpF_4vCHvabgGkICC2epckfdiw&r=AhkbNonVuMIGRfPx_Qj9TsRih1DULJTKUkSGa66m67E&m=-Bk9X9Hkhx7e8xkBu1GuKqAlibTmXrlpwi04sRozJiDG2hny7t3LoP4wS7J5-J1X&s=hlSuw-Ye47nxphecCrzUytKlMvZ1ObQ1WK5hRj_dgb4&e= > [ecordonnier@snap.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
If we're going to nitpick on commit message, then there is also "corner caes" :). On Tue, Feb 6, 2024 at 4:26 PM Etienne Cordonnier via lists.openembedded.org <ecordonnier=snap.com@lists.openembedded.org> wrote: > Hi Richard, there is a word missing in the commit message: > " The functionality had a and place" -> "the functionality had a time and > place" > Also a typo: "comparibility" -> " compatibility " > > What is the correct way to set layer priorities after this patch (I'm > asking because the BSP I'm using makes use of BBFILE_PRIORITY)? > > Etienne > > On Tue, Feb 6, 2024 at 1:15 PM Richard Purdie < > richard.purdie@linuxfoundation.org> wrote: > >> These entries, usually in layer.conf are part of an old, pre-layers >> part of bitbake for overlay/collection support. This is not really >> used any more and was replaced by the idea of layers. >> >> In a recent discussion, we asked the question "When do you need to >> use proprities?" and the answer was "almost never" given all the other >> tools and techniques now available in layers. >> >> These layer priorities cause users huge amounts of confusion. It is >> unclear to people how the "magic values" work if it is unset >> and if they are set differently, users don't expect the way it >> actually works and the effects it has between layers. >> >> In discussions I've had, there has been surprisingly strong support >> for removing them entirely, even if that does means we'd need to fix >> some corner caes. >> >> I wholly agree and support that idea, so lets remove the support >> and take a simplification where we can. The functionality had a >> and place but it has passed and we've moved on. >> >> Developers are about to need to change layer.conf to update the >> series comparibility for the next release so it is a good time to >> do it. >> >> This patch removes the core support and replaces it with a warning. >> It does not chase down every last reference or function parmeter, >> that can follow as time allows. >> >> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> >> --- >> lib/bb/cache.py | 1 - >> lib/bb/cooker.py | 51 +++++++-------------------------------- >> lib/bb/providers.py | 58 +++++++++++++++++++-------------------------- >> lib/bb/utils.py | 2 +- >> 4 files changed, 34 insertions(+), 78 deletions(-) >> >> diff --git a/lib/bb/cache.py b/lib/bb/cache.py >> index 18d5574a31..64dbffc0f6 100644 >> --- a/lib/bb/cache.py >> +++ b/lib/bb/cache.py >> @@ -818,7 +818,6 @@ class CacheData(object): >> # Indirect Cache variables (set elsewhere) >> self.ignored_dependencies = [] >> self.world_target = set() >> - self.bbfile_priority = {} >> >> def add_from_recipeinfo(self, fn, info_array): >> for info in info_array: >> diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py >> index c5bfef55d6..381379086a 100644 >> --- a/lib/bb/cooker.py >> +++ b/lib/bb/cooker.py >> @@ -1124,26 +1124,15 @@ class BBCooker: >> errors = False >> self.bbfile_config_priorities = [] >> if collections: >> - collection_priorities = {} >> collection_depends = {} >> collection_list = collections.split() >> min_prio = 0 >> for c in collection_list: >> bb.debug(1,'Processing %s in collection list' % (c)) >> >> - # Get collection priority if defined explicitly >> priority = self.data.getVar("BBFILE_PRIORITY_%s" % c) >> if priority: >> - try: >> - prio = int(priority) >> - except ValueError: >> - parselog.error("invalid value for >> BBFILE_PRIORITY_%s: \"%s\"", c, priority) >> - errors = True >> - if min_prio == 0 or prio < min_prio: >> - min_prio = prio >> - collection_priorities[c] = prio >> - else: >> - collection_priorities[c] = None >> + bb.warn("Ignoring obsolete BBFILE_PRIORITY_%s" % c) >> >> # Check dependencies and store information for priority >> calculation >> deps = self.data.getVar("LAYERDEPENDS_%s" % c) >> @@ -1204,22 +1193,9 @@ class BBCooker: >> else: >> parselog.debug3("Layer '%s' recommends layer >> '%s', but this layer is not enabled in your configuration", c, rec) >> >> - # Recursively work out collection priorities based on >> dependencies >> - def calc_layer_priority(collection): >> - if not collection_priorities[collection]: >> - max_depprio = min_prio >> - for dep in collection_depends[collection]: >> - calc_layer_priority(dep) >> - depprio = collection_priorities[dep] >> - if depprio > max_depprio: >> - max_depprio = depprio >> - max_depprio += 1 >> - parselog.debug("Calculated priority of layer %s as >> %d", collection, max_depprio) >> - collection_priorities[collection] = max_depprio >> - >> - # Calculate all layer priorities using calc_layer_priority >> and store in bbfile_config_priorities >> + >> + # Store all layer priorities in bbfile_config_priorities >> for c in collection_list: >> - calc_layer_priority(c) >> regex = self.data.getVar("BBFILE_PATTERN_%s" % c) >> if regex is None: >> parselog.error("BBFILE_PATTERN_%s not defined" % c) >> @@ -1236,7 +1212,7 @@ class BBCooker: >> parselog.error("BBFILE_PATTERN_%s \"%s\" is not >> a valid regular expression", c, regex) >> errors = True >> continue >> - self.bbfile_config_priorities.append((c, regex, cre, >> collection_priorities[c])) >> + self.bbfile_config_priorities.append((c, regex, cre, 0)) >> if errors: >> # We've already printed the actual error(s) >> raise CollectionError("Errors during parsing layer >> configuration") >> @@ -1358,7 +1334,6 @@ class BBCooker: >> # Tweak some variables >> item = info_array[0].pn >> self.recipecaches[mc].ignored_dependencies = set() >> - self.recipecaches[mc].bbfile_priority[fn] = 1 >> self.configuration.limited_deps = True >> >> # Remove external dependencies >> @@ -1639,7 +1614,7 @@ class BBCooker: >> self.show_appends_with_no_recipes() >> self.handlePrefProviders() >> for mc in self.multiconfigs: >> - self.recipecaches[mc].bbfile_priority = >> self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, >> self.parser.mcfilelist[mc], self.data) >> + >> self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, >> self.parser.mcfilelist[mc], self.data) >> self.state = state.running >> >> # Send an event listing all stamps reachable after parsing >> @@ -1767,9 +1742,9 @@ class CookerCollectFiles(object): >> self.bbfile_config_priorities = sorted(priorities, key=lambda >> tup: tup[1], reverse=True) >> >> def calc_bbfile_priority(self, filename): >> - for layername, _, regex, pri in self.bbfile_config_priorities: >> + for layername, _, regex, _ in self.bbfile_config_priorities: >> if regex.match(filename): >> - return pri, regex, layername >> + return 0, regex, layername >> return 0, None, None >> >> def get_bbfiles(self): >> @@ -1801,10 +1776,6 @@ class CookerCollectFiles(object): >> >> files = (config.getVar( "BBFILES") or "").split() >> >> - # Sort files by priority >> - files.sort( key=lambda fileitem: >> self.calc_bbfile_priority(fileitem)[0] ) >> - config.setVar("BBFILES_PRIORITIZED", " ".join(files)) >> - >> if not files: >> files = self.get_bbfiles() >> >> @@ -1926,12 +1897,10 @@ class CookerCollectFiles(object): >> return tuple(filelist) >> >> def collection_priorities(self, pkgfns, fns, d): >> - # Return the priorities of the entries in pkgfns >> - # Also check that all the regexes in >> self.bbfile_config_priorities are used >> + # Check that all the regexes in self.bbfile_config_priorities >> are used >> # (but to do that we need to ensure skipped recipes aren't >> counted, nor >> # collections in BBFILE_PATTERN_IGNORE_EMPTY) >> >> - priorities = {} >> seen = set() >> matched = set() >> >> @@ -1943,7 +1912,7 @@ class CookerCollectFiles(object): >> # Calculate priorities for each file >> for p in pkgfns: >> realfn, cls, mc = bb.cache.virtualfn2realfn(p) >> - priorities[p], regex, _ = self.calc_bbfile_priority(realfn) >> + _, regex, _ = self.calc_bbfile_priority(realfn) >> if regex in unmatched_regex: >> matched_regex.add(regex) >> unmatched_regex.remove(regex) >> @@ -1985,8 +1954,6 @@ class CookerCollectFiles(object): >> collectlog.warning("No bb files in %s matched >> BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default', >> >> collection, pattern)) >> >> - return priorities >> - >> class ParsingFailure(Exception): >> def __init__(self, realexception, recipe): >> self.realexception = realexception >> diff --git a/lib/bb/providers.py b/lib/bb/providers.py >> index e11a4637d1..f527dc469f 100644 >> --- a/lib/bb/providers.py >> +++ b/lib/bb/providers.py >> @@ -71,21 +71,15 @@ def sortPriorities(pn, dataCache, pkg_pn = None): >> files = pkg_pn[pn] >> priorities = {} >> for f in files: >> - priority = dataCache.bbfile_priority[f] >> preference = dataCache.pkg_dp[f] >> - if priority not in priorities: >> - priorities[priority] = {} >> - if preference not in priorities[priority]: >> - priorities[priority][preference] = [] >> - priorities[priority][preference].append(f) >> - tmp_pn = [] >> - for pri in sorted(priorities): >> - tmp_pref = [] >> - for pref in sorted(priorities[pri]): >> - tmp_pref.extend(priorities[pri][pref]) >> - tmp_pn = [tmp_pref] + tmp_pn >> - >> - return tmp_pn >> + if preference not in priorities: >> + priorities[preference] = [] >> + priorities[preference].append(f) >> + tmp_pref = [] >> + for pref in sorted(priorities): >> + tmp_pref.extend(priorities[pref]) >> + >> + return tmp_pref >> >> def versionVariableMatch(cfgData, keyword, pn): >> """ >> @@ -115,7 +109,7 @@ def preferredVersionMatch(pe, pv, pr, preferred_e, >> preferred_v, preferred_r): >> return True >> return False >> >> -def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = >> None): >> +def findPreferredProvider(pn, cfgData, dataCache, pkg_pn, item): >> """ >> Find the first provider in pkg_pn with REQUIRED_VERSION or >> PREFERRED_VERSION set. >> """ >> @@ -156,15 +150,12 @@ def findPreferredProvider(pn, cfgData, dataCache, >> pkg_pn = None, item = None): >> preferred_e = None >> preferred_r = None >> >> - for file_set in pkg_pn: >> - for f in file_set: >> - pe, pv, pr = dataCache.pkg_pepvpr[f] >> - if preferredVersionMatch(pe, pv, pr, preferred_e, >> preferred_v, preferred_r): >> - preferred_file = f >> - preferred_ver = (pe, pv, pr) >> - break >> - if preferred_file: >> - break; >> + for f in pkg_pn: >> + pe, pv, pr = dataCache.pkg_pepvpr[f] >> + if preferredVersionMatch(pe, pv, pr, preferred_e, >> preferred_v, preferred_r): >> + preferred_file = f >> + preferred_ver = (pe, pv, pr) >> + break >> if preferred_r: >> pv_str = '%s-%s' % (preferred_v, preferred_r) >> else: >> @@ -175,14 +166,13 @@ def findPreferredProvider(pn, cfgData, dataCache, >> pkg_pn = None, item = None): >> if not required: >> logger.warning("preferred version %s of %s not >> available%s", pv_str, pn, itemstr) >> available_vers = [] >> - for file_set in pkg_pn: >> - for f in file_set: >> - pe, pv, pr = dataCache.pkg_pepvpr[f] >> - ver_str = pv >> - if pe: >> - ver_str = "%s:%s" % (pe, ver_str) >> - if not ver_str in available_vers: >> - available_vers.append(ver_str) >> + for f in pkg_pn: >> + pe, pv, pr = dataCache.pkg_pepvpr[f] >> + ver_str = pv >> + if pe: >> + ver_str = "%s:%s" % (pe, ver_str) >> + if not ver_str in available_vers: >> + available_vers.append(ver_str) >> if available_vers: >> available_vers.sort() >> logger.warning("versions of %s available: %s", pn, ' >> '.join(available_vers)) >> @@ -226,7 +216,7 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = >> None, item = None): >> # Find the highest priority provider with a REQUIRED_VERSION or >> PREFERRED_VERSION set >> (preferred_ver, preferred_file, required) = >> findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item) >> # Find the latest version of the highest priority provider >> - (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, >> sortpkg_pn[0]) >> + (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, >> sortpkg_pn) >> >> if not required and preferred_file is None: >> preferred_file = latest_f >> @@ -272,7 +262,7 @@ def _filterProviders(providers, item, cfgData, >> dataCache): >> for pn in sorted(sortpkg_pn): >> if pn in preferred_versions and preferred_versions[pn][1]: >> continue >> - preferred_versions[pn] = findLatestProvider(pn, cfgData, >> dataCache, sortpkg_pn[pn][0]) >> + preferred_versions[pn] = findLatestProvider(pn, cfgData, >> dataCache, sortpkg_pn[pn]) >> eligible.append(preferred_versions[pn][1]) >> >> if not eligible: >> diff --git a/lib/bb/utils.py b/lib/bb/utils.py >> index 068b631c94..3ee3aa8875 100644 >> --- a/lib/bb/utils.py >> +++ b/lib/bb/utils.py >> @@ -1563,7 +1563,7 @@ def get_file_layer(filename, d, collection_res={}): >> return match >> >> result = None >> - bbfiles = (d.getVar('BBFILES_PRIORITIZED') or '').split() >> + bbfiles = (d.getVar('BBFILES') or '').split() >> bbfilesmatch = False >> for bbfilesentry in bbfiles: >> if fnmatch.fnmatchcase(filename, bbfilesentry): >> -- >> 2.40.1 >> >> >> >> >> > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#15829): > https://lists.openembedded.org/g/bitbake-devel/message/15829 > Mute This Topic: https://lists.openembedded.org/mt/104196430/3617156 > Group Owner: bitbake-devel+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [ > martin.jansa@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
On Tue, 2024-02-06 at 16:26 +0100, Etienne Cordonnier wrote: > Hi Richard, there is a word missing in the commit message: > " The functionality had a and place" -> "the functionality had a time and place" > Also a typo: "comparibility" -> " compatibility " Thanks, I'll tweak those on the branch. > What is the correct way to set layer priorities after this patch (I'm > asking because the BSP I'm using makes use of BBFILE_PRIORITY)? It may set it, but what is it trying to achieve by setting it? As far as I know, the potential issue is needing to set specific PREFERRED_VERSION entries where older recipes need to be used. Cheers, Richard
Is the BBFILE_PRIORITY not controlling the order in which bbappends from different layers gets applied? I thought it was important e.g. when applying kernel configuration files (the last one to be applied override the previous ones). On Tue, Feb 6, 2024 at 4:56 PM Richard Purdie < richard.purdie@linuxfoundation.org> wrote: > On Tue, 2024-02-06 at 16:26 +0100, Etienne Cordonnier wrote: > > Hi Richard, there is a word missing in the commit message: > > " The functionality had a and place" -> "the functionality had a time > and place" > > Also a typo: "comparibility" -> " compatibility " > > Thanks, I'll tweak those on the branch. > > > What is the correct way to set layer priorities after this patch (I'm > > asking because the BSP I'm using makes use of BBFILE_PRIORITY)? > > It may set it, but what is it trying to achieve by setting it? > > As far as I know, the potential issue is needing to set specific > PREFERRED_VERSION entries where older recipes need to be used. > > Cheers, > > Richard >
On 06/02/2024 17:31:03+0100, Etienne Cordonnier via lists.openembedded.org wrote: > Is the BBFILE_PRIORITY not controlling the order in which bbappends from > different layers gets applied? I thought it was important e.g. when > applying kernel configuration files (the last one to be applied override > the previous ones). I'd rather think this is in BBLAYERS list order > > On Tue, Feb 6, 2024 at 4:56 PM Richard Purdie < > richard.purdie@linuxfoundation.org> wrote: > > > On Tue, 2024-02-06 at 16:26 +0100, Etienne Cordonnier wrote: > > > Hi Richard, there is a word missing in the commit message: > > > " The functionality had a and place" -> "the functionality had a time > > and place" > > > Also a typo: "comparibility" -> " compatibility " > > > > Thanks, I'll tweak those on the branch. > > > > > What is the correct way to set layer priorities after this patch (I'm > > > asking because the BSP I'm using makes use of BBFILE_PRIORITY)? > > > > It may set it, but what is it trying to achieve by setting it? > > > > As far as I know, the potential issue is needing to set specific > > PREFERRED_VERSION entries where older recipes need to be used. > > > > Cheers, > > > > Richard > > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#15832): https://lists.openembedded.org/g/bitbake-devel/message/15832 > Mute This Topic: https://lists.openembedded.org/mt/104196430/3617179 > Group Owner: bitbake-devel+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [alexandre.belloni@bootlin.com] > -=-=-=-=-=-=-=-=-=-=-=- >
diff --git a/lib/bb/cache.py b/lib/bb/cache.py index 18d5574a31..64dbffc0f6 100644 --- a/lib/bb/cache.py +++ b/lib/bb/cache.py @@ -818,7 +818,6 @@ class CacheData(object): # Indirect Cache variables (set elsewhere) self.ignored_dependencies = [] self.world_target = set() - self.bbfile_priority = {} def add_from_recipeinfo(self, fn, info_array): for info in info_array: diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index c5bfef55d6..381379086a 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -1124,26 +1124,15 @@ class BBCooker: errors = False self.bbfile_config_priorities = [] if collections: - collection_priorities = {} collection_depends = {} collection_list = collections.split() min_prio = 0 for c in collection_list: bb.debug(1,'Processing %s in collection list' % (c)) - # Get collection priority if defined explicitly priority = self.data.getVar("BBFILE_PRIORITY_%s" % c) if priority: - try: - prio = int(priority) - except ValueError: - parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority) - errors = True - if min_prio == 0 or prio < min_prio: - min_prio = prio - collection_priorities[c] = prio - else: - collection_priorities[c] = None + bb.warn("Ignoring obsolete BBFILE_PRIORITY_%s" % c) # Check dependencies and store information for priority calculation deps = self.data.getVar("LAYERDEPENDS_%s" % c) @@ -1204,22 +1193,9 @@ class BBCooker: else: parselog.debug3("Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec) - # Recursively work out collection priorities based on dependencies - def calc_layer_priority(collection): - if not collection_priorities[collection]: - max_depprio = min_prio - for dep in collection_depends[collection]: - calc_layer_priority(dep) - depprio = collection_priorities[dep] - if depprio > max_depprio: - max_depprio = depprio - max_depprio += 1 - parselog.debug("Calculated priority of layer %s as %d", collection, max_depprio) - collection_priorities[collection] = max_depprio - - # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities + + # Store all layer priorities in bbfile_config_priorities for c in collection_list: - calc_layer_priority(c) regex = self.data.getVar("BBFILE_PATTERN_%s" % c) if regex is None: parselog.error("BBFILE_PATTERN_%s not defined" % c) @@ -1236,7 +1212,7 @@ class BBCooker: parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex) errors = True continue - self.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c])) + self.bbfile_config_priorities.append((c, regex, cre, 0)) if errors: # We've already printed the actual error(s) raise CollectionError("Errors during parsing layer configuration") @@ -1358,7 +1334,6 @@ class BBCooker: # Tweak some variables item = info_array[0].pn self.recipecaches[mc].ignored_dependencies = set() - self.recipecaches[mc].bbfile_priority[fn] = 1 self.configuration.limited_deps = True # Remove external dependencies @@ -1639,7 +1614,7 @@ class BBCooker: self.show_appends_with_no_recipes() self.handlePrefProviders() for mc in self.multiconfigs: - self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data) + self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data) self.state = state.running # Send an event listing all stamps reachable after parsing @@ -1767,9 +1742,9 @@ class CookerCollectFiles(object): self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True) def calc_bbfile_priority(self, filename): - for layername, _, regex, pri in self.bbfile_config_priorities: + for layername, _, regex, _ in self.bbfile_config_priorities: if regex.match(filename): - return pri, regex, layername + return 0, regex, layername return 0, None, None def get_bbfiles(self): @@ -1801,10 +1776,6 @@ class CookerCollectFiles(object): files = (config.getVar( "BBFILES") or "").split() - # Sort files by priority - files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] ) - config.setVar("BBFILES_PRIORITIZED", " ".join(files)) - if not files: files = self.get_bbfiles() @@ -1926,12 +1897,10 @@ class CookerCollectFiles(object): return tuple(filelist) def collection_priorities(self, pkgfns, fns, d): - # Return the priorities of the entries in pkgfns - # Also check that all the regexes in self.bbfile_config_priorities are used + # Check that all the regexes in self.bbfile_config_priorities are used # (but to do that we need to ensure skipped recipes aren't counted, nor # collections in BBFILE_PATTERN_IGNORE_EMPTY) - priorities = {} seen = set() matched = set() @@ -1943,7 +1912,7 @@ class CookerCollectFiles(object): # Calculate priorities for each file for p in pkgfns: realfn, cls, mc = bb.cache.virtualfn2realfn(p) - priorities[p], regex, _ = self.calc_bbfile_priority(realfn) + _, regex, _ = self.calc_bbfile_priority(realfn) if regex in unmatched_regex: matched_regex.add(regex) unmatched_regex.remove(regex) @@ -1985,8 +1954,6 @@ class CookerCollectFiles(object): collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default', collection, pattern)) - return priorities - class ParsingFailure(Exception): def __init__(self, realexception, recipe): self.realexception = realexception diff --git a/lib/bb/providers.py b/lib/bb/providers.py index e11a4637d1..f527dc469f 100644 --- a/lib/bb/providers.py +++ b/lib/bb/providers.py @@ -71,21 +71,15 @@ def sortPriorities(pn, dataCache, pkg_pn = None): files = pkg_pn[pn] priorities = {} for f in files: - priority = dataCache.bbfile_priority[f] preference = dataCache.pkg_dp[f] - if priority not in priorities: - priorities[priority] = {} - if preference not in priorities[priority]: - priorities[priority][preference] = [] - priorities[priority][preference].append(f) - tmp_pn = [] - for pri in sorted(priorities): - tmp_pref = [] - for pref in sorted(priorities[pri]): - tmp_pref.extend(priorities[pri][pref]) - tmp_pn = [tmp_pref] + tmp_pn - - return tmp_pn + if preference not in priorities: + priorities[preference] = [] + priorities[preference].append(f) + tmp_pref = [] + for pref in sorted(priorities): + tmp_pref.extend(priorities[pref]) + + return tmp_pref def versionVariableMatch(cfgData, keyword, pn): """ @@ -115,7 +109,7 @@ def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r): return True return False -def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): +def findPreferredProvider(pn, cfgData, dataCache, pkg_pn, item): """ Find the first provider in pkg_pn with REQUIRED_VERSION or PREFERRED_VERSION set. """ @@ -156,15 +150,12 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): preferred_e = None preferred_r = None - for file_set in pkg_pn: - for f in file_set: - pe, pv, pr = dataCache.pkg_pepvpr[f] - if preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r): - preferred_file = f - preferred_ver = (pe, pv, pr) - break - if preferred_file: - break; + for f in pkg_pn: + pe, pv, pr = dataCache.pkg_pepvpr[f] + if preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r): + preferred_file = f + preferred_ver = (pe, pv, pr) + break if preferred_r: pv_str = '%s-%s' % (preferred_v, preferred_r) else: @@ -175,14 +166,13 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): if not required: logger.warning("preferred version %s of %s not available%s", pv_str, pn, itemstr) available_vers = [] - for file_set in pkg_pn: - for f in file_set: - pe, pv, pr = dataCache.pkg_pepvpr[f] - ver_str = pv - if pe: - ver_str = "%s:%s" % (pe, ver_str) - if not ver_str in available_vers: - available_vers.append(ver_str) + for f in pkg_pn: + pe, pv, pr = dataCache.pkg_pepvpr[f] + ver_str = pv + if pe: + ver_str = "%s:%s" % (pe, ver_str) + if not ver_str in available_vers: + available_vers.append(ver_str) if available_vers: available_vers.sort() logger.warning("versions of %s available: %s", pn, ' '.join(available_vers)) @@ -226,7 +216,7 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): # Find the highest priority provider with a REQUIRED_VERSION or PREFERRED_VERSION set (preferred_ver, preferred_file, required) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item) # Find the latest version of the highest priority provider - (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0]) + (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn) if not required and preferred_file is None: preferred_file = latest_f @@ -272,7 +262,7 @@ def _filterProviders(providers, item, cfgData, dataCache): for pn in sorted(sortpkg_pn): if pn in preferred_versions and preferred_versions[pn][1]: continue - preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0]) + preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn]) eligible.append(preferred_versions[pn][1]) if not eligible: diff --git a/lib/bb/utils.py b/lib/bb/utils.py index 068b631c94..3ee3aa8875 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -1563,7 +1563,7 @@ def get_file_layer(filename, d, collection_res={}): return match result = None - bbfiles = (d.getVar('BBFILES_PRIORITIZED') or '').split() + bbfiles = (d.getVar('BBFILES') or '').split() bbfilesmatch = False for bbfilesentry in bbfiles: if fnmatch.fnmatchcase(filename, bbfilesentry):
These entries, usually in layer.conf are part of an old, pre-layers part of bitbake for overlay/collection support. This is not really used any more and was replaced by the idea of layers. In a recent discussion, we asked the question "When do you need to use proprities?" and the answer was "almost never" given all the other tools and techniques now available in layers. These layer priorities cause users huge amounts of confusion. It is unclear to people how the "magic values" work if it is unset and if they are set differently, users don't expect the way it actually works and the effects it has between layers. In discussions I've had, there has been surprisingly strong support for removing them entirely, even if that does means we'd need to fix some corner caes. I wholly agree and support that idea, so lets remove the support and take a simplification where we can. The functionality had a and place but it has passed and we've moved on. Developers are about to need to change layer.conf to update the series comparibility for the next release so it is a good time to do it. This patch removes the core support and replaces it with a warning. It does not chase down every last reference or function parmeter, that can follow as time allows. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> --- lib/bb/cache.py | 1 - lib/bb/cooker.py | 51 +++++++-------------------------------- lib/bb/providers.py | 58 +++++++++++++++++++-------------------------- lib/bb/utils.py | 2 +- 4 files changed, 34 insertions(+), 78 deletions(-)