| Message ID | 20231019073653.1280730-3-jstephan@baylibre.com |
|---|---|
| State | New |
| Headers | show |
| Series | [v2,1/4] scripts:recipetool:create_buildsys_python: fix license note | expand |
Hello, On 19/10/2023 09:36:52+0200, Julien Stephan wrote: > In order to prepare the support for pyproject.toml (PEP517 [1]) enabled > projects, refactor the code and move setup.py specific code into a > specific class in order to allow sharing the PythonRecipeHandler class > > No functionnal changes expected > I tested with only the first 3 patches and unfortunately, thre were functional changes: https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/5886/steps/14/logs/stdio https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/5935/steps/14/logs/stdio https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/5952/steps/14/logs/stdio https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/5936/steps/14/logs/stdio https://autobuilder.yoctoproject.org/typhoon/#/builders/127/builds/2296/steps/14/logs/stdio 2023-10-19 07:23:07,712 - oe-selftest - INFO - 1: 20/39 149/543 (20.20s) (0 failed) (recipetool.RecipetoolCreateTests.test_recipetool_create_github) 2023-10-19 07:23:07,712 - oe-selftest - INFO - testtools.testresult.real._StringException: Traceback (most recent call last): File "/home/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/selftest/cases/recipetool.py", line 451, in test_recipetool_create_github self.assertTrue(os.path.isfile(recipefile)) File "/usr/lib/python3.11/unittest/case.py", line 715, in assertTrue raise self.failureException(msg) AssertionError: False is not true > [1]: https://peps.python.org/pep-0517/#source-tree > > Signed-off-by: Julien Stephan <jstephan@baylibre.com> > --- > .../lib/recipetool/create_buildsys_python.py | 748 +++++++++--------- > 1 file changed, 385 insertions(+), 363 deletions(-) > > diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py > index 502e1dfbc3d..69f6f5ca511 100644 > --- a/scripts/lib/recipetool/create_buildsys_python.py > +++ b/scripts/lib/recipetool/create_buildsys_python.py > @@ -37,63 +37,8 @@ class PythonRecipeHandler(RecipeHandler): > assume_provided = ['builtins', 'os.path'] > # Assumes that the host python3 builtin_module_names is sane for target too > assume_provided = assume_provided + list(sys.builtin_module_names) > + excluded_fields = [] > > - bbvar_map = { > - 'Name': 'PN', > - 'Version': 'PV', > - 'Home-page': 'HOMEPAGE', > - 'Summary': 'SUMMARY', > - 'Description': 'DESCRIPTION', > - 'License': 'LICENSE', > - 'Requires': 'RDEPENDS:${PN}', > - 'Provides': 'RPROVIDES:${PN}', > - 'Obsoletes': 'RREPLACES:${PN}', > - } > - # PN/PV are already set by recipetool core & desc can be extremely long > - excluded_fields = [ > - 'Description', > - ] > - setup_parse_map = { > - 'Url': 'Home-page', > - 'Classifiers': 'Classifier', > - 'Description': 'Summary', > - } > - setuparg_map = { > - 'Home-page': 'url', > - 'Classifier': 'classifiers', > - 'Summary': 'description', > - 'Description': 'long-description', > - } > - # Values which are lists, used by the setup.py argument based metadata > - # extraction method, to determine how to process the setup.py output. > - setuparg_list_fields = [ > - 'Classifier', > - 'Requires', > - 'Provides', > - 'Obsoletes', > - 'Platform', > - 'Supported-Platform', > - ] > - setuparg_multi_line_values = ['Description'] > - replacements = [ > - ('License', r' +$', ''), > - ('License', r'^ +', ''), > - ('License', r' ', '-'), > - ('License', r'^GNU-', ''), > - ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''), > - ('License', r'^UNKNOWN$', ''), > - > - # Remove currently unhandled version numbers from these variables > - ('Requires', r' *\([^)]*\)', ''), > - ('Provides', r' *\([^)]*\)', ''), > - ('Obsoletes', r' *\([^)]*\)', ''), > - ('Install-requires', r'^([^><= ]+).*', r'\1'), > - ('Extras-require', r'^([^><= ]+).*', r'\1'), > - ('Tests-require', r'^([^><= ]+).*', r'\1'), > - > - # Remove unhandled dependency on particular features (e.g. foo[PDF]) > - ('Install-requires', r'\[[^\]]+\]$', ''), > - ] > > classifier_license_map = { > 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL', > @@ -166,122 +111,34 @@ class PythonRecipeHandler(RecipeHandler): > def __init__(self): > pass > > - def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): > - if 'buildsystem' in handled: > - return False > - > - # Check for non-zero size setup.py files > - setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py']) > - for fn in setupfiles: > - if os.path.getsize(fn): > - break > - else: > - return False > - > - # setup.py is always parsed to get at certain required information, such as > - # distutils vs setuptools > - # > - # If egg info is available, we use it for both its PKG-INFO metadata > - # and for its requires.txt for install_requires. > - # If PKG-INFO is available but no egg info is, we use that for metadata in preference to > - # the parsed setup.py, but use the install_requires info from the > - # parsed setup.py. > - > - setupscript = os.path.join(srctree, 'setup.py') > - try: > - setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript) > - except Exception: > - logger.exception("Failed to parse setup.py") > - setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], [] > - > - egginfo = glob.glob(os.path.join(srctree, '*.egg-info')) > - if egginfo: > - info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO')) > - requires_txt = os.path.join(egginfo[0], 'requires.txt') > - if os.path.exists(requires_txt): > - with codecs.open(requires_txt) as f: > - inst_req = [] > - extras_req = collections.defaultdict(list) > - current_feature = None > - for line in f.readlines(): > - line = line.rstrip() > - if not line: > - continue > - > - if line.startswith('['): > - # PACKAGECONFIG must not contain expressions or whitespace > - line = line.replace(" ", "") > - line = line.replace(':', "") > - line = line.replace('.', "-dot-") > - line = line.replace('"', "") > - line = line.replace('<', "-smaller-") > - line = line.replace('>', "-bigger-") > - line = line.replace('_', "-") > - line = line.replace('(', "") > - line = line.replace(')', "") > - line = line.replace('!', "-not-") > - line = line.replace('=', "-equals-") > - current_feature = line[1:-1] > - elif current_feature: > - extras_req[current_feature].append(line) > - else: > - inst_req.append(line) > - info['Install-requires'] = inst_req > - info['Extras-require'] = extras_req > - elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']): > - info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO')) > - > - if setup_info: > - if 'Install-requires' in setup_info: > - info['Install-requires'] = setup_info['Install-requires'] > - if 'Extras-require' in setup_info: > - info['Extras-require'] = setup_info['Extras-require'] > - else: > - if setup_info: > - info = setup_info > - else: > - info = self.get_setup_args_info(setupscript) > - > - # Grab the license value before applying replacements > - license_str = info.get('License', '').strip() > - > - self.apply_info_replacements(info) > - > - if uses_setuptools: > - classes.append('setuptools3') > - else: > - classes.append('distutils3') > - > - if license_str: > - for i, line in enumerate(lines_before): > - if line.startswith('##LICENSE_PLACEHOLDER##'): > - lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str) > - break > - > - if 'Classifier' in info: > - existing_licenses = info.get('License', '') > - licenses = [] > - for classifier in info['Classifier']: > - if classifier in self.classifier_license_map: > - license = self.classifier_license_map[classifier] > - if license == 'Apache' and 'Apache-2.0' in existing_licenses: > - license = 'Apache-2.0' > - elif license == 'GPL': > - if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: > - license = 'GPL-2.0' > - elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: > - license = 'GPL-3.0' > - elif license == 'LGPL': > - if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: > - license = 'LGPL-2.1' > - elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: > - license = 'LGPL-2.0' > - elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: > - license = 'LGPL-3.0' > - licenses.append(license) > - > - if licenses: > - info['License'] = ' & '.join(licenses) > + def handle_classifier_license(self, classifiers, existing_licenses=""): > + > + licenses = [] > + for classifier in classifiers: > + if classifier in self.classifier_license_map: > + license = self.classifier_license_map[classifier] > + if license == 'Apache' and 'Apache-2.0' in existing_licenses: > + license = 'Apache-2.0' > + elif license == 'GPL': > + if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: > + license = 'GPL-2.0' > + elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: > + license = 'GPL-3.0' > + elif license == 'LGPL': > + if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: > + license = 'LGPL-2.1' > + elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: > + license = 'LGPL-2.0' > + elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: > + license = 'LGPL-3.0' > + licenses.append(license) > + > + if licenses: > + return ' & '.join(licenses) > + > + return None > + > + def map_info_to_bbvar(self, info, extravalues): > > # Map PKG-INFO & setup.py fields to bitbake variables > for field, values in info.items(): > @@ -305,85 +162,220 @@ class PythonRecipeHandler(RecipeHandler): > if bbvar not in extravalues and value: > extravalues[bbvar] = value > > - mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) > - > - extras_req = set() > - if 'Extras-require' in info: > - extras_req = info['Extras-require'] > - if extras_req: > - lines_after.append('# The following configs & dependencies are from setuptools extras_require.') > - lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.') > - lines_after.append('# The upstream names may not correspond exactly to bitbake package names.') > - lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.') > - lines_after.append('#') > - lines_after.append('# Uncomment this line to enable all the optional features.') > - lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req))) > - for feature, feature_reqs in extras_req.items(): > - unmapped_deps.difference_update(feature_reqs) > - > - feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) > - lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) > - > - inst_reqs = set() > - if 'Install-requires' in info: > - if extras_req: > - lines_after.append('') > - inst_reqs = info['Install-requires'] > - if inst_reqs: > - unmapped_deps.difference_update(inst_reqs) > - > - inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) > - lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') > - lines_after.append('# upstream names may not correspond exactly to bitbake package names.') > - lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps))) > + def apply_info_replacements(self, info): > + if not self.replacements: > + return > > - if mapped_deps: > - name = info.get('Name') > - if name and name[0] in mapped_deps: > - # Attempt to avoid self-reference > - mapped_deps.remove(name[0]) > - mapped_deps -= set(self.excluded_pkgdeps) > - if inst_reqs or extras_req: > - lines_after.append('') > - lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') > - lines_after.append('# python sources, and might not be 100% accurate.') > - lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) > + for variable, search, replace in self.replacements: > + if variable not in info: > + continue > > - unmapped_deps -= set(extensions) > - unmapped_deps -= set(self.assume_provided) > - if unmapped_deps: > - if mapped_deps: > - lines_after.append('') > - lines_after.append('# WARNING: We were unable to map the following python package/module') > - lines_after.append('# dependencies to the bitbake packages which include them:') > - lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps)) > + def replace_value(search, replace, value): > + if replace is None: > + if re.search(search, value): > + return None > + else: > + new_value = re.sub(search, replace, value) > + if value != new_value: > + return new_value > + return value > > - handled.append('buildsystem') > + value = info[variable] > + if isinstance(value, str): > + new_value = replace_value(search, replace, value) > + if new_value is None: > + del info[variable] > + elif new_value != value: > + info[variable] = new_value > + elif hasattr(value, 'items'): > + for dkey, dvalue in list(value.items()): > + new_list = [] > + for pos, a_value in enumerate(dvalue): > + new_value = replace_value(search, replace, a_value) > + if new_value is not None and new_value != value: > + new_list.append(new_value) > > - def get_pkginfo(self, pkginfo_fn): > - msg = email.message_from_file(open(pkginfo_fn, 'r')) > - msginfo = {} > - for field in msg.keys(): > - values = msg.get_all(field) > - if len(values) == 1: > - msginfo[field] = values[0] > + if value != new_list: > + value[dkey] = new_list > else: > - msginfo[field] = values > - return msginfo > + new_list = [] > + for pos, a_value in enumerate(value): > + new_value = replace_value(search, replace, a_value) > + if new_value is not None and new_value != value: > + new_list.append(new_value) > > - def parse_setup_py(self, setupscript='./setup.py'): > - with codecs.open(setupscript) as f: > - info, imported_modules, non_literals, extensions = gather_setup_info(f) > + if value != new_list: > + info[variable] = new_list > > - def _map(key): > - key = key.replace('_', '-') > - key = key[0].upper() + key[1:] > - if key in self.setup_parse_map: > - key = self.setup_parse_map[key] > - return key > > - # Naive mapping of setup() arguments to PKG-INFO field names > - for d in [info, non_literals]: > + def scan_python_dependencies(self, paths): > + deps = set() > + try: > + dep_output = self.run_command(['pythondeps', '-d'] + paths) > + except (OSError, subprocess.CalledProcessError): > + pass > + else: > + for line in dep_output.splitlines(): > + line = line.rstrip() > + dep, filename = line.split('\t', 1) > + if filename.endswith('/setup.py'): > + continue > + deps.add(dep) > + > + try: > + provides_output = self.run_command(['pythondeps', '-p'] + paths) > + except (OSError, subprocess.CalledProcessError): > + pass > + else: > + provides_lines = (l.rstrip() for l in provides_output.splitlines()) > + provides = set(l for l in provides_lines if l and l != 'setup') > + deps -= provides > + > + return deps > + > + def parse_pkgdata_for_python_packages(self): > + pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') > + > + ldata = tinfoil.config_data.createCopy() > + bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True) > + python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') > + > + dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') > + python_dirs = [python_sitedir + os.sep, > + os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, > + os.path.dirname(python_sitedir) + os.sep] > + packages = {} > + for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): > + files_info = None > + with open(pkgdatafile, 'r') as f: > + for line in f.readlines(): > + field, value = line.split(': ', 1) > + if field.startswith('FILES_INFO'): > + files_info = ast.literal_eval(value) > + break > + else: > + continue > + > + for fn in files_info: > + for suffix in importlib.machinery.all_suffixes(): > + if fn.endswith(suffix): > + break > + else: > + continue > + > + if fn.startswith(dynload_dir + os.sep): > + if '/.debug/' in fn: > + continue > + base = os.path.basename(fn) > + provided = base.split('.', 1)[0] > + packages[provided] = os.path.basename(pkgdatafile) > + continue > + > + for python_dir in python_dirs: > + if fn.startswith(python_dir): > + relpath = fn[len(python_dir):] > + relstart, _, relremaining = relpath.partition(os.sep) > + if relstart.endswith('.egg'): > + relpath = relremaining > + base, _ = os.path.splitext(relpath) > + > + if '/.debug/' in base: > + continue > + if os.path.basename(base) == '__init__': > + base = os.path.dirname(base) > + base = base.replace(os.sep + os.sep, os.sep) > + provided = base.replace(os.sep, '.') > + packages[provided] = os.path.basename(pkgdatafile) > + return packages > + > + @classmethod > + def run_command(cls, cmd, **popenargs): > + if 'stderr' not in popenargs: > + popenargs['stderr'] = subprocess.STDOUT > + try: > + return subprocess.check_output(cmd, **popenargs).decode('utf-8') > + except OSError as exc: > + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) > + raise > + except subprocess.CalledProcessError as exc: > + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) > + raise > + > +class PythonSetupPyRecipeHandler(PythonRecipeHandler): > + bbvar_map = { > + 'Name': 'PN', > + 'Version': 'PV', > + 'Home-page': 'HOMEPAGE', > + 'Summary': 'SUMMARY', > + 'Description': 'DESCRIPTION', > + 'License': 'LICENSE', > + 'Requires': 'RDEPENDS:${PN}', > + 'Provides': 'RPROVIDES:${PN}', > + 'Obsoletes': 'RREPLACES:${PN}', > + } > + # PN/PV are already set by recipetool core & desc can be extremely long > + excluded_fields = [ > + 'Description', > + ] > + setup_parse_map = { > + 'Url': 'Home-page', > + 'Classifiers': 'Classifier', > + 'Description': 'Summary', > + } > + setuparg_map = { > + 'Home-page': 'url', > + 'Classifier': 'classifiers', > + 'Summary': 'description', > + 'Description': 'long-description', > + } > + # Values which are lists, used by the setup.py argument based metadata > + # extraction method, to determine how to process the setup.py output. > + setuparg_list_fields = [ > + 'Classifier', > + 'Requires', > + 'Provides', > + 'Obsoletes', > + 'Platform', > + 'Supported-Platform', > + ] > + setuparg_multi_line_values = ['Description'] > + > + replacements = [ > + ('License', r' +$', ''), > + ('License', r'^ +', ''), > + ('License', r' ', '-'), > + ('License', r'^GNU-', ''), > + ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''), > + ('License', r'^UNKNOWN$', ''), > + > + # Remove currently unhandled version numbers from these variables > + ('Requires', r' *\([^)]*\)', ''), > + ('Provides', r' *\([^)]*\)', ''), > + ('Obsoletes', r' *\([^)]*\)', ''), > + ('Install-requires', r'^([^><= ]+).*', r'\1'), > + ('Extras-require', r'^([^><= ]+).*', r'\1'), > + ('Tests-require', r'^([^><= ]+).*', r'\1'), > + > + # Remove unhandled dependency on particular features (e.g. foo[PDF]) > + ('Install-requires', r'\[[^\]]+\]$', ''), > + ] > + > + def __init__(self): > + pass > + > + def parse_setup_py(self, setupscript='./setup.py'): > + with codecs.open(setupscript) as f: > + info, imported_modules, non_literals, extensions = gather_setup_info(f) > + > + def _map(key): > + key = key.replace('_', '-') > + key = key[0].upper() + key[1:] > + if key in self.setup_parse_map: > + key = self.setup_parse_map[key] > + return key > + > + # Naive mapping of setup() arguments to PKG-INFO field names > + for d in [info, non_literals]: > for key, value in list(d.items()): > if key is None: > continue > @@ -445,47 +437,16 @@ class PythonRecipeHandler(RecipeHandler): > info[fields[lineno]] = line > return info > > - def apply_info_replacements(self, info): > - for variable, search, replace in self.replacements: > - if variable not in info: > - continue > - > - def replace_value(search, replace, value): > - if replace is None: > - if re.search(search, value): > - return None > - else: > - new_value = re.sub(search, replace, value) > - if value != new_value: > - return new_value > - return value > - > - value = info[variable] > - if isinstance(value, str): > - new_value = replace_value(search, replace, value) > - if new_value is None: > - del info[variable] > - elif new_value != value: > - info[variable] = new_value > - elif hasattr(value, 'items'): > - for dkey, dvalue in list(value.items()): > - new_list = [] > - for pos, a_value in enumerate(dvalue): > - new_value = replace_value(search, replace, a_value) > - if new_value is not None and new_value != value: > - new_list.append(new_value) > - > - if value != new_list: > - value[dkey] = new_list > + def get_pkginfo(self, pkginfo_fn): > + msg = email.message_from_file(open(pkginfo_fn, 'r')) > + msginfo = {} > + for field in msg.keys(): > + values = msg.get_all(field) > + if len(values) == 1: > + msginfo[field] = values[0] > else: > - new_list = [] > - for pos, a_value in enumerate(value): > - new_value = replace_value(search, replace, a_value) > - if new_value is not None and new_value != value: > - new_list.append(new_value) > - > - if value != new_list: > - info[variable] = new_list > + msginfo[field] = values > + return msginfo > > def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals): > if 'Package-dir' in setup_info: > @@ -540,99 +501,160 @@ class PythonRecipeHandler(RecipeHandler): > unmapped_deps.add(dep) > return mapped_deps, unmapped_deps > > - def scan_python_dependencies(self, paths): > - deps = set() > - try: > - dep_output = self.run_command(['pythondeps', '-d'] + paths) > - except (OSError, subprocess.CalledProcessError): > - pass > + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): > + > + if 'buildsystem' in handled: > + return False > + > + # Check for non-zero size setup.py files > + setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py']) > + for fn in setupfiles: > + if os.path.getsize(fn): > + break > else: > - for line in dep_output.splitlines(): > - line = line.rstrip() > - dep, filename = line.split('\t', 1) > - if filename.endswith('/setup.py'): > - continue > - deps.add(dep) > + return False > + > + # setup.py is always parsed to get at certain required information, such as > + # distutils vs setuptools > + # > + # If egg info is available, we use it for both its PKG-INFO metadata > + # and for its requires.txt for install_requires. > + # If PKG-INFO is available but no egg info is, we use that for metadata in preference to > + # the parsed setup.py, but use the install_requires info from the > + # parsed setup.py. > > + setupscript = os.path.join(srctree, 'setup.py') > try: > - provides_output = self.run_command(['pythondeps', '-p'] + paths) > - except (OSError, subprocess.CalledProcessError): > - pass > + setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript) > + except Exception: > + logger.exception("Failed to parse setup.py") > + setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], [] > + > + egginfo = glob.glob(os.path.join(srctree, '*.egg-info')) > + if egginfo: > + info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO')) > + requires_txt = os.path.join(egginfo[0], 'requires.txt') > + if os.path.exists(requires_txt): > + with codecs.open(requires_txt) as f: > + inst_req = [] > + extras_req = collections.defaultdict(list) > + current_feature = None > + for line in f.readlines(): > + line = line.rstrip() > + if not line: > + continue > + > + if line.startswith('['): > + # PACKAGECONFIG must not contain expressions or whitespace > + line = line.replace(" ", "") > + line = line.replace(':', "") > + line = line.replace('.', "-dot-") > + line = line.replace('"', "") > + line = line.replace('<', "-smaller-") > + line = line.replace('>', "-bigger-") > + line = line.replace('_', "-") > + line = line.replace('(', "") > + line = line.replace(')', "") > + line = line.replace('!', "-not-") > + line = line.replace('=', "-equals-") > + current_feature = line[1:-1] > + elif current_feature: > + extras_req[current_feature].append(line) > + else: > + inst_req.append(line) > + info['Install-requires'] = inst_req > + info['Extras-require'] = extras_req > + elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']): > + info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO')) > + > + if setup_info: > + if 'Install-requires' in setup_info: > + info['Install-requires'] = setup_info['Install-requires'] > + if 'Extras-require' in setup_info: > + info['Extras-require'] = setup_info['Extras-require'] > else: > - provides_lines = (l.rstrip() for l in provides_output.splitlines()) > - provides = set(l for l in provides_lines if l and l != 'setup') > - deps -= provides > + if setup_info: > + info = setup_info > + else: > + info = self.get_setup_args_info(setupscript) > > - return deps > + # Grab the license value before applying replacements > + license_str = info.get('License', '').strip() > > - def parse_pkgdata_for_python_packages(self): > - pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') > + self.apply_info_replacements(info) > > - ldata = tinfoil.config_data.createCopy() > - bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True) > - python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') > + if uses_setuptools: > + classes.append('setuptools3') > + else: > + classes.append('distutils3') > > - dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') > - python_dirs = [python_sitedir + os.sep, > - os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, > - os.path.dirname(python_sitedir) + os.sep] > - packages = {} > - for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): > - files_info = None > - with open(pkgdatafile, 'r') as f: > - for line in f.readlines(): > - field, value = line.split(': ', 1) > - if field.startswith('FILES_INFO'): > - files_info = ast.literal_eval(value) > - break > - else: > - continue > + if license_str: > + for i, line in enumerate(lines_before): > + if line.startswith('##LICENSE_PLACEHOLDER##'): > + lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str) > + break > > - for fn in files_info: > - for suffix in importlib.machinery.all_suffixes(): > - if fn.endswith(suffix): > - break > - else: > - continue > + if 'Classifier' in info: > + license = self.handle_classifier_license(info['Classifier'], info.get('License', '')) > + if license: > + info['License'] = license > > - if fn.startswith(dynload_dir + os.sep): > - if '/.debug/' in fn: > - continue > - base = os.path.basename(fn) > - provided = base.split('.', 1)[0] > - packages[provided] = os.path.basename(pkgdatafile) > - continue > + self.map_info_to_bbvar(info, extravalues) > > - for python_dir in python_dirs: > - if fn.startswith(python_dir): > - relpath = fn[len(python_dir):] > - relstart, _, relremaining = relpath.partition(os.sep) > - if relstart.endswith('.egg'): > - relpath = relremaining > - base, _ = os.path.splitext(relpath) > + mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) > > - if '/.debug/' in base: > - continue > - if os.path.basename(base) == '__init__': > - base = os.path.dirname(base) > - base = base.replace(os.sep + os.sep, os.sep) > - provided = base.replace(os.sep, '.') > - packages[provided] = os.path.basename(pkgdatafile) > - return packages > + extras_req = set() > + if 'Extras-require' in info: > + extras_req = info['Extras-require'] > + if extras_req: > + lines_after.append('# The following configs & dependencies are from setuptools extras_require.') > + lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.') > + lines_after.append('# The upstream names may not correspond exactly to bitbake package names.') > + lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.') > + lines_after.append('#') > + lines_after.append('# Uncomment this line to enable all the optional features.') > + lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req))) > + for feature, feature_reqs in extras_req.items(): > + unmapped_deps.difference_update(feature_reqs) > > - @classmethod > - def run_command(cls, cmd, **popenargs): > - if 'stderr' not in popenargs: > - popenargs['stderr'] = subprocess.STDOUT > - try: > - return subprocess.check_output(cmd, **popenargs).decode('utf-8') > - except OSError as exc: > - logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) > - raise > - except subprocess.CalledProcessError as exc: > - logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) > - raise > + feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) > + lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) > > + inst_reqs = set() > + if 'Install-requires' in info: > + if extras_req: > + lines_after.append('') > + inst_reqs = info['Install-requires'] > + if inst_reqs: > + unmapped_deps.difference_update(inst_reqs) > + > + inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) > + lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') > + lines_after.append('# upstream names may not correspond exactly to bitbake package names.') > + lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps))) > + > + if mapped_deps: > + name = info.get('Name') > + if name and name[0] in mapped_deps: > + # Attempt to avoid self-reference > + mapped_deps.remove(name[0]) > + mapped_deps -= set(self.excluded_pkgdeps) > + if inst_reqs or extras_req: > + lines_after.append('') > + lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') > + lines_after.append('# python sources, and might not be 100% accurate.') > + lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) > + > + unmapped_deps -= set(extensions) > + unmapped_deps -= set(self.assume_provided) > + if unmapped_deps: > + if mapped_deps: > + lines_after.append('') > + lines_after.append('# WARNING: We were unable to map the following python package/module') > + lines_after.append('# dependencies to the bitbake packages which include them:') > + lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps)) > + > + handled.append('buildsystem') > > def gather_setup_info(fileobj): > parsed = ast.parse(fileobj.read(), fileobj.name) > @@ -748,4 +770,4 @@ def has_non_literals(value): > > def register_recipe_handlers(handlers): > # We need to make sure this is ahead of the makefile fallback handler > - handlers.append((PythonRecipeHandler(), 70)) > + handlers.append((PythonSetupPyRecipeHandler(), 70)) > -- > 2.42.0 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#189430): https://lists.openembedded.org/g/openembedded-core/message/189430 > Mute This Topic: https://lists.openembedded.org/mt/102055998/3617179 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alexandre.belloni@bootlin.com] > -=-=-=-=-=-=-=-=-=-=-=- >
Le ven. 20 oct. 2023 à 08:01, Alexandre Belloni <alexandre.belloni@bootlin.com> a écrit : > > Hello, > > On 19/10/2023 09:36:52+0200, Julien Stephan wrote: > > In order to prepare the support for pyproject.toml (PEP517 [1]) enabled > > projects, refactor the code and move setup.py specific code into a > > specific class in order to allow sharing the PythonRecipeHandler class > > > > No functionnal changes expected > > > > I tested with only the first 3 patches and unfortunately, thre were > functional changes: > > https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/5886/steps/14/logs/stdio > https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/5935/steps/14/logs/stdio > https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/5952/steps/14/logs/stdio > https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/5936/steps/14/logs/stdio > https://autobuilder.yoctoproject.org/typhoon/#/builders/127/builds/2296/steps/14/logs/stdio > > 2023-10-19 07:23:07,712 - oe-selftest - INFO - 1: 20/39 149/543 (20.20s) (0 failed) (recipetool.RecipetoolCreateTests.test_recipetool_create_github) > 2023-10-19 07:23:07,712 - oe-selftest - INFO - testtools.testresult.real._StringException: Traceback (most recent call last): > File "/home/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/selftest/cases/recipetool.py", line 451, in test_recipetool_create_github > self.assertTrue(os.path.isfile(recipefile)) > File "/usr/lib/python3.11/unittest/case.py", line 715, in assertTrue > raise self.failureException(msg) > AssertionError: False is not true Hi Alexandre, I am sorry, I did run a full self test for devtool but I forgot to run it for recipetool.. I found the issue. This is not this commit, but the one where I now prepend "python3-" for all created recipes. I will update the self tests accordingly :) Cheers Julien > > > [1]: https://peps.python.org/pep-0517/#source-tree > > > > Signed-off-by: Julien Stephan <jstephan@baylibre.com> > > --- > > .../lib/recipetool/create_buildsys_python.py | 748 +++++++++--------- > > 1 file changed, 385 insertions(+), 363 deletions(-) > > > > diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py > > index 502e1dfbc3d..69f6f5ca511 100644 > > --- a/scripts/lib/recipetool/create_buildsys_python.py > > +++ b/scripts/lib/recipetool/create_buildsys_python.py > > @@ -37,63 +37,8 @@ class PythonRecipeHandler(RecipeHandler): > > assume_provided = ['builtins', 'os.path'] > > # Assumes that the host python3 builtin_module_names is sane for target too > > assume_provided = assume_provided + list(sys.builtin_module_names) > > + excluded_fields = [] > > > > - bbvar_map = { > > - 'Name': 'PN', > > - 'Version': 'PV', > > - 'Home-page': 'HOMEPAGE', > > - 'Summary': 'SUMMARY', > > - 'Description': 'DESCRIPTION', > > - 'License': 'LICENSE', > > - 'Requires': 'RDEPENDS:${PN}', > > - 'Provides': 'RPROVIDES:${PN}', > > - 'Obsoletes': 'RREPLACES:${PN}', > > - } > > - # PN/PV are already set by recipetool core & desc can be extremely long > > - excluded_fields = [ > > - 'Description', > > - ] > > - setup_parse_map = { > > - 'Url': 'Home-page', > > - 'Classifiers': 'Classifier', > > - 'Description': 'Summary', > > - } > > - setuparg_map = { > > - 'Home-page': 'url', > > - 'Classifier': 'classifiers', > > - 'Summary': 'description', > > - 'Description': 'long-description', > > - } > > - # Values which are lists, used by the setup.py argument based metadata > > - # extraction method, to determine how to process the setup.py output. > > - setuparg_list_fields = [ > > - 'Classifier', > > - 'Requires', > > - 'Provides', > > - 'Obsoletes', > > - 'Platform', > > - 'Supported-Platform', > > - ] > > - setuparg_multi_line_values = ['Description'] > > - replacements = [ > > - ('License', r' +$', ''), > > - ('License', r'^ +', ''), > > - ('License', r' ', '-'), > > - ('License', r'^GNU-', ''), > > - ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''), > > - ('License', r'^UNKNOWN$', ''), > > - > > - # Remove currently unhandled version numbers from these variables > > - ('Requires', r' *\([^)]*\)', ''), > > - ('Provides', r' *\([^)]*\)', ''), > > - ('Obsoletes', r' *\([^)]*\)', ''), > > - ('Install-requires', r'^([^><= ]+).*', r'\1'), > > - ('Extras-require', r'^([^><= ]+).*', r'\1'), > > - ('Tests-require', r'^([^><= ]+).*', r'\1'), > > - > > - # Remove unhandled dependency on particular features (e.g. foo[PDF]) > > - ('Install-requires', r'\[[^\]]+\]$', ''), > > - ] > > > > classifier_license_map = { > > 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL', > > @@ -166,122 +111,34 @@ class PythonRecipeHandler(RecipeHandler): > > def __init__(self): > > pass > > > > - def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): > > - if 'buildsystem' in handled: > > - return False > > - > > - # Check for non-zero size setup.py files > > - setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py']) > > - for fn in setupfiles: > > - if os.path.getsize(fn): > > - break > > - else: > > - return False > > - > > - # setup.py is always parsed to get at certain required information, such as > > - # distutils vs setuptools > > - # > > - # If egg info is available, we use it for both its PKG-INFO metadata > > - # and for its requires.txt for install_requires. > > - # If PKG-INFO is available but no egg info is, we use that for metadata in preference to > > - # the parsed setup.py, but use the install_requires info from the > > - # parsed setup.py. > > - > > - setupscript = os.path.join(srctree, 'setup.py') > > - try: > > - setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript) > > - except Exception: > > - logger.exception("Failed to parse setup.py") > > - setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], [] > > - > > - egginfo = glob.glob(os.path.join(srctree, '*.egg-info')) > > - if egginfo: > > - info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO')) > > - requires_txt = os.path.join(egginfo[0], 'requires.txt') > > - if os.path.exists(requires_txt): > > - with codecs.open(requires_txt) as f: > > - inst_req = [] > > - extras_req = collections.defaultdict(list) > > - current_feature = None > > - for line in f.readlines(): > > - line = line.rstrip() > > - if not line: > > - continue > > - > > - if line.startswith('['): > > - # PACKAGECONFIG must not contain expressions or whitespace > > - line = line.replace(" ", "") > > - line = line.replace(':', "") > > - line = line.replace('.', "-dot-") > > - line = line.replace('"', "") > > - line = line.replace('<', "-smaller-") > > - line = line.replace('>', "-bigger-") > > - line = line.replace('_', "-") > > - line = line.replace('(', "") > > - line = line.replace(')', "") > > - line = line.replace('!', "-not-") > > - line = line.replace('=', "-equals-") > > - current_feature = line[1:-1] > > - elif current_feature: > > - extras_req[current_feature].append(line) > > - else: > > - inst_req.append(line) > > - info['Install-requires'] = inst_req > > - info['Extras-require'] = extras_req > > - elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']): > > - info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO')) > > - > > - if setup_info: > > - if 'Install-requires' in setup_info: > > - info['Install-requires'] = setup_info['Install-requires'] > > - if 'Extras-require' in setup_info: > > - info['Extras-require'] = setup_info['Extras-require'] > > - else: > > - if setup_info: > > - info = setup_info > > - else: > > - info = self.get_setup_args_info(setupscript) > > - > > - # Grab the license value before applying replacements > > - license_str = info.get('License', '').strip() > > - > > - self.apply_info_replacements(info) > > - > > - if uses_setuptools: > > - classes.append('setuptools3') > > - else: > > - classes.append('distutils3') > > - > > - if license_str: > > - for i, line in enumerate(lines_before): > > - if line.startswith('##LICENSE_PLACEHOLDER##'): > > - lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str) > > - break > > - > > - if 'Classifier' in info: > > - existing_licenses = info.get('License', '') > > - licenses = [] > > - for classifier in info['Classifier']: > > - if classifier in self.classifier_license_map: > > - license = self.classifier_license_map[classifier] > > - if license == 'Apache' and 'Apache-2.0' in existing_licenses: > > - license = 'Apache-2.0' > > - elif license == 'GPL': > > - if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: > > - license = 'GPL-2.0' > > - elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: > > - license = 'GPL-3.0' > > - elif license == 'LGPL': > > - if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: > > - license = 'LGPL-2.1' > > - elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: > > - license = 'LGPL-2.0' > > - elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: > > - license = 'LGPL-3.0' > > - licenses.append(license) > > - > > - if licenses: > > - info['License'] = ' & '.join(licenses) > > + def handle_classifier_license(self, classifiers, existing_licenses=""): > > + > > + licenses = [] > > + for classifier in classifiers: > > + if classifier in self.classifier_license_map: > > + license = self.classifier_license_map[classifier] > > + if license == 'Apache' and 'Apache-2.0' in existing_licenses: > > + license = 'Apache-2.0' > > + elif license == 'GPL': > > + if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: > > + license = 'GPL-2.0' > > + elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: > > + license = 'GPL-3.0' > > + elif license == 'LGPL': > > + if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: > > + license = 'LGPL-2.1' > > + elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: > > + license = 'LGPL-2.0' > > + elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: > > + license = 'LGPL-3.0' > > + licenses.append(license) > > + > > + if licenses: > > + return ' & '.join(licenses) > > + > > + return None > > + > > + def map_info_to_bbvar(self, info, extravalues): > > > > # Map PKG-INFO & setup.py fields to bitbake variables > > for field, values in info.items(): > > @@ -305,85 +162,220 @@ class PythonRecipeHandler(RecipeHandler): > > if bbvar not in extravalues and value: > > extravalues[bbvar] = value > > > > - mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) > > - > > - extras_req = set() > > - if 'Extras-require' in info: > > - extras_req = info['Extras-require'] > > - if extras_req: > > - lines_after.append('# The following configs & dependencies are from setuptools extras_require.') > > - lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.') > > - lines_after.append('# The upstream names may not correspond exactly to bitbake package names.') > > - lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.') > > - lines_after.append('#') > > - lines_after.append('# Uncomment this line to enable all the optional features.') > > - lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req))) > > - for feature, feature_reqs in extras_req.items(): > > - unmapped_deps.difference_update(feature_reqs) > > - > > - feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) > > - lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) > > - > > - inst_reqs = set() > > - if 'Install-requires' in info: > > - if extras_req: > > - lines_after.append('') > > - inst_reqs = info['Install-requires'] > > - if inst_reqs: > > - unmapped_deps.difference_update(inst_reqs) > > - > > - inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) > > - lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') > > - lines_after.append('# upstream names may not correspond exactly to bitbake package names.') > > - lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps))) > > + def apply_info_replacements(self, info): > > + if not self.replacements: > > + return > > > > - if mapped_deps: > > - name = info.get('Name') > > - if name and name[0] in mapped_deps: > > - # Attempt to avoid self-reference > > - mapped_deps.remove(name[0]) > > - mapped_deps -= set(self.excluded_pkgdeps) > > - if inst_reqs or extras_req: > > - lines_after.append('') > > - lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') > > - lines_after.append('# python sources, and might not be 100% accurate.') > > - lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) > > + for variable, search, replace in self.replacements: > > + if variable not in info: > > + continue > > > > - unmapped_deps -= set(extensions) > > - unmapped_deps -= set(self.assume_provided) > > - if unmapped_deps: > > - if mapped_deps: > > - lines_after.append('') > > - lines_after.append('# WARNING: We were unable to map the following python package/module') > > - lines_after.append('# dependencies to the bitbake packages which include them:') > > - lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps)) > > + def replace_value(search, replace, value): > > + if replace is None: > > + if re.search(search, value): > > + return None > > + else: > > + new_value = re.sub(search, replace, value) > > + if value != new_value: > > + return new_value > > + return value > > > > - handled.append('buildsystem') > > + value = info[variable] > > + if isinstance(value, str): > > + new_value = replace_value(search, replace, value) > > + if new_value is None: > > + del info[variable] > > + elif new_value != value: > > + info[variable] = new_value > > + elif hasattr(value, 'items'): > > + for dkey, dvalue in list(value.items()): > > + new_list = [] > > + for pos, a_value in enumerate(dvalue): > > + new_value = replace_value(search, replace, a_value) > > + if new_value is not None and new_value != value: > > + new_list.append(new_value) > > > > - def get_pkginfo(self, pkginfo_fn): > > - msg = email.message_from_file(open(pkginfo_fn, 'r')) > > - msginfo = {} > > - for field in msg.keys(): > > - values = msg.get_all(field) > > - if len(values) == 1: > > - msginfo[field] = values[0] > > + if value != new_list: > > + value[dkey] = new_list > > else: > > - msginfo[field] = values > > - return msginfo > > + new_list = [] > > + for pos, a_value in enumerate(value): > > + new_value = replace_value(search, replace, a_value) > > + if new_value is not None and new_value != value: > > + new_list.append(new_value) > > > > - def parse_setup_py(self, setupscript='./setup.py'): > > - with codecs.open(setupscript) as f: > > - info, imported_modules, non_literals, extensions = gather_setup_info(f) > > + if value != new_list: > > + info[variable] = new_list > > > > - def _map(key): > > - key = key.replace('_', '-') > > - key = key[0].upper() + key[1:] > > - if key in self.setup_parse_map: > > - key = self.setup_parse_map[key] > > - return key > > > > - # Naive mapping of setup() arguments to PKG-INFO field names > > - for d in [info, non_literals]: > > + def scan_python_dependencies(self, paths): > > + deps = set() > > + try: > > + dep_output = self.run_command(['pythondeps', '-d'] + paths) > > + except (OSError, subprocess.CalledProcessError): > > + pass > > + else: > > + for line in dep_output.splitlines(): > > + line = line.rstrip() > > + dep, filename = line.split('\t', 1) > > + if filename.endswith('/setup.py'): > > + continue > > + deps.add(dep) > > + > > + try: > > + provides_output = self.run_command(['pythondeps', '-p'] + paths) > > + except (OSError, subprocess.CalledProcessError): > > + pass > > + else: > > + provides_lines = (l.rstrip() for l in provides_output.splitlines()) > > + provides = set(l for l in provides_lines if l and l != 'setup') > > + deps -= provides > > + > > + return deps > > + > > + def parse_pkgdata_for_python_packages(self): > > + pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') > > + > > + ldata = tinfoil.config_data.createCopy() > > + bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True) > > + python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') > > + > > + dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') > > + python_dirs = [python_sitedir + os.sep, > > + os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, > > + os.path.dirname(python_sitedir) + os.sep] > > + packages = {} > > + for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): > > + files_info = None > > + with open(pkgdatafile, 'r') as f: > > + for line in f.readlines(): > > + field, value = line.split(': ', 1) > > + if field.startswith('FILES_INFO'): > > + files_info = ast.literal_eval(value) > > + break > > + else: > > + continue > > + > > + for fn in files_info: > > + for suffix in importlib.machinery.all_suffixes(): > > + if fn.endswith(suffix): > > + break > > + else: > > + continue > > + > > + if fn.startswith(dynload_dir + os.sep): > > + if '/.debug/' in fn: > > + continue > > + base = os.path.basename(fn) > > + provided = base.split('.', 1)[0] > > + packages[provided] = os.path.basename(pkgdatafile) > > + continue > > + > > + for python_dir in python_dirs: > > + if fn.startswith(python_dir): > > + relpath = fn[len(python_dir):] > > + relstart, _, relremaining = relpath.partition(os.sep) > > + if relstart.endswith('.egg'): > > + relpath = relremaining > > + base, _ = os.path.splitext(relpath) > > + > > + if '/.debug/' in base: > > + continue > > + if os.path.basename(base) == '__init__': > > + base = os.path.dirname(base) > > + base = base.replace(os.sep + os.sep, os.sep) > > + provided = base.replace(os.sep, '.') > > + packages[provided] = os.path.basename(pkgdatafile) > > + return packages > > + > > + @classmethod > > + def run_command(cls, cmd, **popenargs): > > + if 'stderr' not in popenargs: > > + popenargs['stderr'] = subprocess.STDOUT > > + try: > > + return subprocess.check_output(cmd, **popenargs).decode('utf-8') > > + except OSError as exc: > > + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) > > + raise > > + except subprocess.CalledProcessError as exc: > > + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) > > + raise > > + > > +class PythonSetupPyRecipeHandler(PythonRecipeHandler): > > + bbvar_map = { > > + 'Name': 'PN', > > + 'Version': 'PV', > > + 'Home-page': 'HOMEPAGE', > > + 'Summary': 'SUMMARY', > > + 'Description': 'DESCRIPTION', > > + 'License': 'LICENSE', > > + 'Requires': 'RDEPENDS:${PN}', > > + 'Provides': 'RPROVIDES:${PN}', > > + 'Obsoletes': 'RREPLACES:${PN}', > > + } > > + # PN/PV are already set by recipetool core & desc can be extremely long > > + excluded_fields = [ > > + 'Description', > > + ] > > + setup_parse_map = { > > + 'Url': 'Home-page', > > + 'Classifiers': 'Classifier', > > + 'Description': 'Summary', > > + } > > + setuparg_map = { > > + 'Home-page': 'url', > > + 'Classifier': 'classifiers', > > + 'Summary': 'description', > > + 'Description': 'long-description', > > + } > > + # Values which are lists, used by the setup.py argument based metadata > > + # extraction method, to determine how to process the setup.py output. > > + setuparg_list_fields = [ > > + 'Classifier', > > + 'Requires', > > + 'Provides', > > + 'Obsoletes', > > + 'Platform', > > + 'Supported-Platform', > > + ] > > + setuparg_multi_line_values = ['Description'] > > + > > + replacements = [ > > + ('License', r' +$', ''), > > + ('License', r'^ +', ''), > > + ('License', r' ', '-'), > > + ('License', r'^GNU-', ''), > > + ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''), > > + ('License', r'^UNKNOWN$', ''), > > + > > + # Remove currently unhandled version numbers from these variables > > + ('Requires', r' *\([^)]*\)', ''), > > + ('Provides', r' *\([^)]*\)', ''), > > + ('Obsoletes', r' *\([^)]*\)', ''), > > + ('Install-requires', r'^([^><= ]+).*', r'\1'), > > + ('Extras-require', r'^([^><= ]+).*', r'\1'), > > + ('Tests-require', r'^([^><= ]+).*', r'\1'), > > + > > + # Remove unhandled dependency on particular features (e.g. foo[PDF]) > > + ('Install-requires', r'\[[^\]]+\]$', ''), > > + ] > > + > > + def __init__(self): > > + pass > > + > > + def parse_setup_py(self, setupscript='./setup.py'): > > + with codecs.open(setupscript) as f: > > + info, imported_modules, non_literals, extensions = gather_setup_info(f) > > + > > + def _map(key): > > + key = key.replace('_', '-') > > + key = key[0].upper() + key[1:] > > + if key in self.setup_parse_map: > > + key = self.setup_parse_map[key] > > + return key > > + > > + # Naive mapping of setup() arguments to PKG-INFO field names > > + for d in [info, non_literals]: > > for key, value in list(d.items()): > > if key is None: > > continue > > @@ -445,47 +437,16 @@ class PythonRecipeHandler(RecipeHandler): > > info[fields[lineno]] = line > > return info > > > > - def apply_info_replacements(self, info): > > - for variable, search, replace in self.replacements: > > - if variable not in info: > > - continue > > - > > - def replace_value(search, replace, value): > > - if replace is None: > > - if re.search(search, value): > > - return None > > - else: > > - new_value = re.sub(search, replace, value) > > - if value != new_value: > > - return new_value > > - return value > > - > > - value = info[variable] > > - if isinstance(value, str): > > - new_value = replace_value(search, replace, value) > > - if new_value is None: > > - del info[variable] > > - elif new_value != value: > > - info[variable] = new_value > > - elif hasattr(value, 'items'): > > - for dkey, dvalue in list(value.items()): > > - new_list = [] > > - for pos, a_value in enumerate(dvalue): > > - new_value = replace_value(search, replace, a_value) > > - if new_value is not None and new_value != value: > > - new_list.append(new_value) > > - > > - if value != new_list: > > - value[dkey] = new_list > > + def get_pkginfo(self, pkginfo_fn): > > + msg = email.message_from_file(open(pkginfo_fn, 'r')) > > + msginfo = {} > > + for field in msg.keys(): > > + values = msg.get_all(field) > > + if len(values) == 1: > > + msginfo[field] = values[0] > > else: > > - new_list = [] > > - for pos, a_value in enumerate(value): > > - new_value = replace_value(search, replace, a_value) > > - if new_value is not None and new_value != value: > > - new_list.append(new_value) > > - > > - if value != new_list: > > - info[variable] = new_list > > + msginfo[field] = values > > + return msginfo > > > > def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals): > > if 'Package-dir' in setup_info: > > @@ -540,99 +501,160 @@ class PythonRecipeHandler(RecipeHandler): > > unmapped_deps.add(dep) > > return mapped_deps, unmapped_deps > > > > - def scan_python_dependencies(self, paths): > > - deps = set() > > - try: > > - dep_output = self.run_command(['pythondeps', '-d'] + paths) > > - except (OSError, subprocess.CalledProcessError): > > - pass > > + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): > > + > > + if 'buildsystem' in handled: > > + return False > > + > > + # Check for non-zero size setup.py files > > + setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py']) > > + for fn in setupfiles: > > + if os.path.getsize(fn): > > + break > > else: > > - for line in dep_output.splitlines(): > > - line = line.rstrip() > > - dep, filename = line.split('\t', 1) > > - if filename.endswith('/setup.py'): > > - continue > > - deps.add(dep) > > + return False > > + > > + # setup.py is always parsed to get at certain required information, such as > > + # distutils vs setuptools > > + # > > + # If egg info is available, we use it for both its PKG-INFO metadata > > + # and for its requires.txt for install_requires. > > + # If PKG-INFO is available but no egg info is, we use that for metadata in preference to > > + # the parsed setup.py, but use the install_requires info from the > > + # parsed setup.py. > > > > + setupscript = os.path.join(srctree, 'setup.py') > > try: > > - provides_output = self.run_command(['pythondeps', '-p'] + paths) > > - except (OSError, subprocess.CalledProcessError): > > - pass > > + setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript) > > + except Exception: > > + logger.exception("Failed to parse setup.py") > > + setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], [] > > + > > + egginfo = glob.glob(os.path.join(srctree, '*.egg-info')) > > + if egginfo: > > + info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO')) > > + requires_txt = os.path.join(egginfo[0], 'requires.txt') > > + if os.path.exists(requires_txt): > > + with codecs.open(requires_txt) as f: > > + inst_req = [] > > + extras_req = collections.defaultdict(list) > > + current_feature = None > > + for line in f.readlines(): > > + line = line.rstrip() > > + if not line: > > + continue > > + > > + if line.startswith('['): > > + # PACKAGECONFIG must not contain expressions or whitespace > > + line = line.replace(" ", "") > > + line = line.replace(':', "") > > + line = line.replace('.', "-dot-") > > + line = line.replace('"', "") > > + line = line.replace('<', "-smaller-") > > + line = line.replace('>', "-bigger-") > > + line = line.replace('_', "-") > > + line = line.replace('(', "") > > + line = line.replace(')', "") > > + line = line.replace('!', "-not-") > > + line = line.replace('=', "-equals-") > > + current_feature = line[1:-1] > > + elif current_feature: > > + extras_req[current_feature].append(line) > > + else: > > + inst_req.append(line) > > + info['Install-requires'] = inst_req > > + info['Extras-require'] = extras_req > > + elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']): > > + info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO')) > > + > > + if setup_info: > > + if 'Install-requires' in setup_info: > > + info['Install-requires'] = setup_info['Install-requires'] > > + if 'Extras-require' in setup_info: > > + info['Extras-require'] = setup_info['Extras-require'] > > else: > > - provides_lines = (l.rstrip() for l in provides_output.splitlines()) > > - provides = set(l for l in provides_lines if l and l != 'setup') > > - deps -= provides > > + if setup_info: > > + info = setup_info > > + else: > > + info = self.get_setup_args_info(setupscript) > > > > - return deps > > + # Grab the license value before applying replacements > > + license_str = info.get('License', '').strip() > > > > - def parse_pkgdata_for_python_packages(self): > > - pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') > > + self.apply_info_replacements(info) > > > > - ldata = tinfoil.config_data.createCopy() > > - bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True) > > - python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') > > + if uses_setuptools: > > + classes.append('setuptools3') > > + else: > > + classes.append('distutils3') > > > > - dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') > > - python_dirs = [python_sitedir + os.sep, > > - os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, > > - os.path.dirname(python_sitedir) + os.sep] > > - packages = {} > > - for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): > > - files_info = None > > - with open(pkgdatafile, 'r') as f: > > - for line in f.readlines(): > > - field, value = line.split(': ', 1) > > - if field.startswith('FILES_INFO'): > > - files_info = ast.literal_eval(value) > > - break > > - else: > > - continue > > + if license_str: > > + for i, line in enumerate(lines_before): > > + if line.startswith('##LICENSE_PLACEHOLDER##'): > > + lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str) > > + break > > > > - for fn in files_info: > > - for suffix in importlib.machinery.all_suffixes(): > > - if fn.endswith(suffix): > > - break > > - else: > > - continue > > + if 'Classifier' in info: > > + license = self.handle_classifier_license(info['Classifier'], info.get('License', '')) > > + if license: > > + info['License'] = license > > > > - if fn.startswith(dynload_dir + os.sep): > > - if '/.debug/' in fn: > > - continue > > - base = os.path.basename(fn) > > - provided = base.split('.', 1)[0] > > - packages[provided] = os.path.basename(pkgdatafile) > > - continue > > + self.map_info_to_bbvar(info, extravalues) > > > > - for python_dir in python_dirs: > > - if fn.startswith(python_dir): > > - relpath = fn[len(python_dir):] > > - relstart, _, relremaining = relpath.partition(os.sep) > > - if relstart.endswith('.egg'): > > - relpath = relremaining > > - base, _ = os.path.splitext(relpath) > > + mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) > > > > - if '/.debug/' in base: > > - continue > > - if os.path.basename(base) == '__init__': > > - base = os.path.dirname(base) > > - base = base.replace(os.sep + os.sep, os.sep) > > - provided = base.replace(os.sep, '.') > > - packages[provided] = os.path.basename(pkgdatafile) > > - return packages > > + extras_req = set() > > + if 'Extras-require' in info: > > + extras_req = info['Extras-require'] > > + if extras_req: > > + lines_after.append('# The following configs & dependencies are from setuptools extras_require.') > > + lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.') > > + lines_after.append('# The upstream names may not correspond exactly to bitbake package names.') > > + lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.') > > + lines_after.append('#') > > + lines_after.append('# Uncomment this line to enable all the optional features.') > > + lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req))) > > + for feature, feature_reqs in extras_req.items(): > > + unmapped_deps.difference_update(feature_reqs) > > > > - @classmethod > > - def run_command(cls, cmd, **popenargs): > > - if 'stderr' not in popenargs: > > - popenargs['stderr'] = subprocess.STDOUT > > - try: > > - return subprocess.check_output(cmd, **popenargs).decode('utf-8') > > - except OSError as exc: > > - logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) > > - raise > > - except subprocess.CalledProcessError as exc: > > - logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) > > - raise > > + feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) > > + lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) > > > > + inst_reqs = set() > > + if 'Install-requires' in info: > > + if extras_req: > > + lines_after.append('') > > + inst_reqs = info['Install-requires'] > > + if inst_reqs: > > + unmapped_deps.difference_update(inst_reqs) > > + > > + inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) > > + lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') > > + lines_after.append('# upstream names may not correspond exactly to bitbake package names.') > > + lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps))) > > + > > + if mapped_deps: > > + name = info.get('Name') > > + if name and name[0] in mapped_deps: > > + # Attempt to avoid self-reference > > + mapped_deps.remove(name[0]) > > + mapped_deps -= set(self.excluded_pkgdeps) > > + if inst_reqs or extras_req: > > + lines_after.append('') > > + lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') > > + lines_after.append('# python sources, and might not be 100% accurate.') > > + lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) > > + > > + unmapped_deps -= set(extensions) > > + unmapped_deps -= set(self.assume_provided) > > + if unmapped_deps: > > + if mapped_deps: > > + lines_after.append('') > > + lines_after.append('# WARNING: We were unable to map the following python package/module') > > + lines_after.append('# dependencies to the bitbake packages which include them:') > > + lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps)) > > + > > + handled.append('buildsystem') > > > > def gather_setup_info(fileobj): > > parsed = ast.parse(fileobj.read(), fileobj.name) > > @@ -748,4 +770,4 @@ def has_non_literals(value): > > > > def register_recipe_handlers(handlers): > > # We need to make sure this is ahead of the makefile fallback handler > > - handlers.append((PythonRecipeHandler(), 70)) > > + handlers.append((PythonSetupPyRecipeHandler(), 70)) > > -- > > 2.42.0 > > > > > > > -=-=-=-=-=-=-=-=-=-=-=- > > Links: You receive all messages sent to this group. > > View/Reply Online (#189430): https://lists.openembedded.org/g/openembedded-core/message/189430 > > Mute This Topic: https://lists.openembedded.org/mt/102055998/3617179 > > Group Owner: openembedded-core+owner@lists.openembedded.org > > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alexandre.belloni@bootlin.com] > > -=-=-=-=-=-=-=-=-=-=-=- > > > > > -- > Alexandre Belloni, co-owner and COO, Bootlin > Embedded Linux and Kernel engineering > https://bootlin.com
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py index 502e1dfbc3d..69f6f5ca511 100644 --- a/scripts/lib/recipetool/create_buildsys_python.py +++ b/scripts/lib/recipetool/create_buildsys_python.py @@ -37,63 +37,8 @@ class PythonRecipeHandler(RecipeHandler): assume_provided = ['builtins', 'os.path'] # Assumes that the host python3 builtin_module_names is sane for target too assume_provided = assume_provided + list(sys.builtin_module_names) + excluded_fields = [] - bbvar_map = { - 'Name': 'PN', - 'Version': 'PV', - 'Home-page': 'HOMEPAGE', - 'Summary': 'SUMMARY', - 'Description': 'DESCRIPTION', - 'License': 'LICENSE', - 'Requires': 'RDEPENDS:${PN}', - 'Provides': 'RPROVIDES:${PN}', - 'Obsoletes': 'RREPLACES:${PN}', - } - # PN/PV are already set by recipetool core & desc can be extremely long - excluded_fields = [ - 'Description', - ] - setup_parse_map = { - 'Url': 'Home-page', - 'Classifiers': 'Classifier', - 'Description': 'Summary', - } - setuparg_map = { - 'Home-page': 'url', - 'Classifier': 'classifiers', - 'Summary': 'description', - 'Description': 'long-description', - } - # Values which are lists, used by the setup.py argument based metadata - # extraction method, to determine how to process the setup.py output. - setuparg_list_fields = [ - 'Classifier', - 'Requires', - 'Provides', - 'Obsoletes', - 'Platform', - 'Supported-Platform', - ] - setuparg_multi_line_values = ['Description'] - replacements = [ - ('License', r' +$', ''), - ('License', r'^ +', ''), - ('License', r' ', '-'), - ('License', r'^GNU-', ''), - ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''), - ('License', r'^UNKNOWN$', ''), - - # Remove currently unhandled version numbers from these variables - ('Requires', r' *\([^)]*\)', ''), - ('Provides', r' *\([^)]*\)', ''), - ('Obsoletes', r' *\([^)]*\)', ''), - ('Install-requires', r'^([^><= ]+).*', r'\1'), - ('Extras-require', r'^([^><= ]+).*', r'\1'), - ('Tests-require', r'^([^><= ]+).*', r'\1'), - - # Remove unhandled dependency on particular features (e.g. foo[PDF]) - ('Install-requires', r'\[[^\]]+\]$', ''), - ] classifier_license_map = { 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL', @@ -166,122 +111,34 @@ class PythonRecipeHandler(RecipeHandler): def __init__(self): pass - def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): - if 'buildsystem' in handled: - return False - - # Check for non-zero size setup.py files - setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py']) - for fn in setupfiles: - if os.path.getsize(fn): - break - else: - return False - - # setup.py is always parsed to get at certain required information, such as - # distutils vs setuptools - # - # If egg info is available, we use it for both its PKG-INFO metadata - # and for its requires.txt for install_requires. - # If PKG-INFO is available but no egg info is, we use that for metadata in preference to - # the parsed setup.py, but use the install_requires info from the - # parsed setup.py. - - setupscript = os.path.join(srctree, 'setup.py') - try: - setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript) - except Exception: - logger.exception("Failed to parse setup.py") - setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], [] - - egginfo = glob.glob(os.path.join(srctree, '*.egg-info')) - if egginfo: - info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO')) - requires_txt = os.path.join(egginfo[0], 'requires.txt') - if os.path.exists(requires_txt): - with codecs.open(requires_txt) as f: - inst_req = [] - extras_req = collections.defaultdict(list) - current_feature = None - for line in f.readlines(): - line = line.rstrip() - if not line: - continue - - if line.startswith('['): - # PACKAGECONFIG must not contain expressions or whitespace - line = line.replace(" ", "") - line = line.replace(':', "") - line = line.replace('.', "-dot-") - line = line.replace('"', "") - line = line.replace('<', "-smaller-") - line = line.replace('>', "-bigger-") - line = line.replace('_', "-") - line = line.replace('(', "") - line = line.replace(')', "") - line = line.replace('!', "-not-") - line = line.replace('=', "-equals-") - current_feature = line[1:-1] - elif current_feature: - extras_req[current_feature].append(line) - else: - inst_req.append(line) - info['Install-requires'] = inst_req - info['Extras-require'] = extras_req - elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']): - info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO')) - - if setup_info: - if 'Install-requires' in setup_info: - info['Install-requires'] = setup_info['Install-requires'] - if 'Extras-require' in setup_info: - info['Extras-require'] = setup_info['Extras-require'] - else: - if setup_info: - info = setup_info - else: - info = self.get_setup_args_info(setupscript) - - # Grab the license value before applying replacements - license_str = info.get('License', '').strip() - - self.apply_info_replacements(info) - - if uses_setuptools: - classes.append('setuptools3') - else: - classes.append('distutils3') - - if license_str: - for i, line in enumerate(lines_before): - if line.startswith('##LICENSE_PLACEHOLDER##'): - lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str) - break - - if 'Classifier' in info: - existing_licenses = info.get('License', '') - licenses = [] - for classifier in info['Classifier']: - if classifier in self.classifier_license_map: - license = self.classifier_license_map[classifier] - if license == 'Apache' and 'Apache-2.0' in existing_licenses: - license = 'Apache-2.0' - elif license == 'GPL': - if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: - license = 'GPL-2.0' - elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: - license = 'GPL-3.0' - elif license == 'LGPL': - if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: - license = 'LGPL-2.1' - elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: - license = 'LGPL-2.0' - elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: - license = 'LGPL-3.0' - licenses.append(license) - - if licenses: - info['License'] = ' & '.join(licenses) + def handle_classifier_license(self, classifiers, existing_licenses=""): + + licenses = [] + for classifier in classifiers: + if classifier in self.classifier_license_map: + license = self.classifier_license_map[classifier] + if license == 'Apache' and 'Apache-2.0' in existing_licenses: + license = 'Apache-2.0' + elif license == 'GPL': + if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: + license = 'GPL-2.0' + elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: + license = 'GPL-3.0' + elif license == 'LGPL': + if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: + license = 'LGPL-2.1' + elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: + license = 'LGPL-2.0' + elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: + license = 'LGPL-3.0' + licenses.append(license) + + if licenses: + return ' & '.join(licenses) + + return None + + def map_info_to_bbvar(self, info, extravalues): # Map PKG-INFO & setup.py fields to bitbake variables for field, values in info.items(): @@ -305,85 +162,220 @@ class PythonRecipeHandler(RecipeHandler): if bbvar not in extravalues and value: extravalues[bbvar] = value - mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) - - extras_req = set() - if 'Extras-require' in info: - extras_req = info['Extras-require'] - if extras_req: - lines_after.append('# The following configs & dependencies are from setuptools extras_require.') - lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.') - lines_after.append('# The upstream names may not correspond exactly to bitbake package names.') - lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.') - lines_after.append('#') - lines_after.append('# Uncomment this line to enable all the optional features.') - lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req))) - for feature, feature_reqs in extras_req.items(): - unmapped_deps.difference_update(feature_reqs) - - feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) - lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) - - inst_reqs = set() - if 'Install-requires' in info: - if extras_req: - lines_after.append('') - inst_reqs = info['Install-requires'] - if inst_reqs: - unmapped_deps.difference_update(inst_reqs) - - inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) - lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') - lines_after.append('# upstream names may not correspond exactly to bitbake package names.') - lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps))) + def apply_info_replacements(self, info): + if not self.replacements: + return - if mapped_deps: - name = info.get('Name') - if name and name[0] in mapped_deps: - # Attempt to avoid self-reference - mapped_deps.remove(name[0]) - mapped_deps -= set(self.excluded_pkgdeps) - if inst_reqs or extras_req: - lines_after.append('') - lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') - lines_after.append('# python sources, and might not be 100% accurate.') - lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) + for variable, search, replace in self.replacements: + if variable not in info: + continue - unmapped_deps -= set(extensions) - unmapped_deps -= set(self.assume_provided) - if unmapped_deps: - if mapped_deps: - lines_after.append('') - lines_after.append('# WARNING: We were unable to map the following python package/module') - lines_after.append('# dependencies to the bitbake packages which include them:') - lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps)) + def replace_value(search, replace, value): + if replace is None: + if re.search(search, value): + return None + else: + new_value = re.sub(search, replace, value) + if value != new_value: + return new_value + return value - handled.append('buildsystem') + value = info[variable] + if isinstance(value, str): + new_value = replace_value(search, replace, value) + if new_value is None: + del info[variable] + elif new_value != value: + info[variable] = new_value + elif hasattr(value, 'items'): + for dkey, dvalue in list(value.items()): + new_list = [] + for pos, a_value in enumerate(dvalue): + new_value = replace_value(search, replace, a_value) + if new_value is not None and new_value != value: + new_list.append(new_value) - def get_pkginfo(self, pkginfo_fn): - msg = email.message_from_file(open(pkginfo_fn, 'r')) - msginfo = {} - for field in msg.keys(): - values = msg.get_all(field) - if len(values) == 1: - msginfo[field] = values[0] + if value != new_list: + value[dkey] = new_list else: - msginfo[field] = values - return msginfo + new_list = [] + for pos, a_value in enumerate(value): + new_value = replace_value(search, replace, a_value) + if new_value is not None and new_value != value: + new_list.append(new_value) - def parse_setup_py(self, setupscript='./setup.py'): - with codecs.open(setupscript) as f: - info, imported_modules, non_literals, extensions = gather_setup_info(f) + if value != new_list: + info[variable] = new_list - def _map(key): - key = key.replace('_', '-') - key = key[0].upper() + key[1:] - if key in self.setup_parse_map: - key = self.setup_parse_map[key] - return key - # Naive mapping of setup() arguments to PKG-INFO field names - for d in [info, non_literals]: + def scan_python_dependencies(self, paths): + deps = set() + try: + dep_output = self.run_command(['pythondeps', '-d'] + paths) + except (OSError, subprocess.CalledProcessError): + pass + else: + for line in dep_output.splitlines(): + line = line.rstrip() + dep, filename = line.split('\t', 1) + if filename.endswith('/setup.py'): + continue + deps.add(dep) + + try: + provides_output = self.run_command(['pythondeps', '-p'] + paths) + except (OSError, subprocess.CalledProcessError): + pass + else: + provides_lines = (l.rstrip() for l in provides_output.splitlines()) + provides = set(l for l in provides_lines if l and l != 'setup') + deps -= provides + + return deps + + def parse_pkgdata_for_python_packages(self): + pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') + + ldata = tinfoil.config_data.createCopy() + bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True) + python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') + + dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') + python_dirs = [python_sitedir + os.sep, + os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, + os.path.dirname(python_sitedir) + os.sep] + packages = {} + for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): + files_info = None + with open(pkgdatafile, 'r') as f: + for line in f.readlines(): + field, value = line.split(': ', 1) + if field.startswith('FILES_INFO'): + files_info = ast.literal_eval(value) + break + else: + continue + + for fn in files_info: + for suffix in importlib.machinery.all_suffixes(): + if fn.endswith(suffix): + break + else: + continue + + if fn.startswith(dynload_dir + os.sep): + if '/.debug/' in fn: + continue + base = os.path.basename(fn) + provided = base.split('.', 1)[0] + packages[provided] = os.path.basename(pkgdatafile) + continue + + for python_dir in python_dirs: + if fn.startswith(python_dir): + relpath = fn[len(python_dir):] + relstart, _, relremaining = relpath.partition(os.sep) + if relstart.endswith('.egg'): + relpath = relremaining + base, _ = os.path.splitext(relpath) + + if '/.debug/' in base: + continue + if os.path.basename(base) == '__init__': + base = os.path.dirname(base) + base = base.replace(os.sep + os.sep, os.sep) + provided = base.replace(os.sep, '.') + packages[provided] = os.path.basename(pkgdatafile) + return packages + + @classmethod + def run_command(cls, cmd, **popenargs): + if 'stderr' not in popenargs: + popenargs['stderr'] = subprocess.STDOUT + try: + return subprocess.check_output(cmd, **popenargs).decode('utf-8') + except OSError as exc: + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) + raise + except subprocess.CalledProcessError as exc: + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) + raise + +class PythonSetupPyRecipeHandler(PythonRecipeHandler): + bbvar_map = { + 'Name': 'PN', + 'Version': 'PV', + 'Home-page': 'HOMEPAGE', + 'Summary': 'SUMMARY', + 'Description': 'DESCRIPTION', + 'License': 'LICENSE', + 'Requires': 'RDEPENDS:${PN}', + 'Provides': 'RPROVIDES:${PN}', + 'Obsoletes': 'RREPLACES:${PN}', + } + # PN/PV are already set by recipetool core & desc can be extremely long + excluded_fields = [ + 'Description', + ] + setup_parse_map = { + 'Url': 'Home-page', + 'Classifiers': 'Classifier', + 'Description': 'Summary', + } + setuparg_map = { + 'Home-page': 'url', + 'Classifier': 'classifiers', + 'Summary': 'description', + 'Description': 'long-description', + } + # Values which are lists, used by the setup.py argument based metadata + # extraction method, to determine how to process the setup.py output. + setuparg_list_fields = [ + 'Classifier', + 'Requires', + 'Provides', + 'Obsoletes', + 'Platform', + 'Supported-Platform', + ] + setuparg_multi_line_values = ['Description'] + + replacements = [ + ('License', r' +$', ''), + ('License', r'^ +', ''), + ('License', r' ', '-'), + ('License', r'^GNU-', ''), + ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''), + ('License', r'^UNKNOWN$', ''), + + # Remove currently unhandled version numbers from these variables + ('Requires', r' *\([^)]*\)', ''), + ('Provides', r' *\([^)]*\)', ''), + ('Obsoletes', r' *\([^)]*\)', ''), + ('Install-requires', r'^([^><= ]+).*', r'\1'), + ('Extras-require', r'^([^><= ]+).*', r'\1'), + ('Tests-require', r'^([^><= ]+).*', r'\1'), + + # Remove unhandled dependency on particular features (e.g. foo[PDF]) + ('Install-requires', r'\[[^\]]+\]$', ''), + ] + + def __init__(self): + pass + + def parse_setup_py(self, setupscript='./setup.py'): + with codecs.open(setupscript) as f: + info, imported_modules, non_literals, extensions = gather_setup_info(f) + + def _map(key): + key = key.replace('_', '-') + key = key[0].upper() + key[1:] + if key in self.setup_parse_map: + key = self.setup_parse_map[key] + return key + + # Naive mapping of setup() arguments to PKG-INFO field names + for d in [info, non_literals]: for key, value in list(d.items()): if key is None: continue @@ -445,47 +437,16 @@ class PythonRecipeHandler(RecipeHandler): info[fields[lineno]] = line return info - def apply_info_replacements(self, info): - for variable, search, replace in self.replacements: - if variable not in info: - continue - - def replace_value(search, replace, value): - if replace is None: - if re.search(search, value): - return None - else: - new_value = re.sub(search, replace, value) - if value != new_value: - return new_value - return value - - value = info[variable] - if isinstance(value, str): - new_value = replace_value(search, replace, value) - if new_value is None: - del info[variable] - elif new_value != value: - info[variable] = new_value - elif hasattr(value, 'items'): - for dkey, dvalue in list(value.items()): - new_list = [] - for pos, a_value in enumerate(dvalue): - new_value = replace_value(search, replace, a_value) - if new_value is not None and new_value != value: - new_list.append(new_value) - - if value != new_list: - value[dkey] = new_list + def get_pkginfo(self, pkginfo_fn): + msg = email.message_from_file(open(pkginfo_fn, 'r')) + msginfo = {} + for field in msg.keys(): + values = msg.get_all(field) + if len(values) == 1: + msginfo[field] = values[0] else: - new_list = [] - for pos, a_value in enumerate(value): - new_value = replace_value(search, replace, a_value) - if new_value is not None and new_value != value: - new_list.append(new_value) - - if value != new_list: - info[variable] = new_list + msginfo[field] = values + return msginfo def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals): if 'Package-dir' in setup_info: @@ -540,99 +501,160 @@ class PythonRecipeHandler(RecipeHandler): unmapped_deps.add(dep) return mapped_deps, unmapped_deps - def scan_python_dependencies(self, paths): - deps = set() - try: - dep_output = self.run_command(['pythondeps', '-d'] + paths) - except (OSError, subprocess.CalledProcessError): - pass + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + + if 'buildsystem' in handled: + return False + + # Check for non-zero size setup.py files + setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py']) + for fn in setupfiles: + if os.path.getsize(fn): + break else: - for line in dep_output.splitlines(): - line = line.rstrip() - dep, filename = line.split('\t', 1) - if filename.endswith('/setup.py'): - continue - deps.add(dep) + return False + + # setup.py is always parsed to get at certain required information, such as + # distutils vs setuptools + # + # If egg info is available, we use it for both its PKG-INFO metadata + # and for its requires.txt for install_requires. + # If PKG-INFO is available but no egg info is, we use that for metadata in preference to + # the parsed setup.py, but use the install_requires info from the + # parsed setup.py. + setupscript = os.path.join(srctree, 'setup.py') try: - provides_output = self.run_command(['pythondeps', '-p'] + paths) - except (OSError, subprocess.CalledProcessError): - pass + setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript) + except Exception: + logger.exception("Failed to parse setup.py") + setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], [] + + egginfo = glob.glob(os.path.join(srctree, '*.egg-info')) + if egginfo: + info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO')) + requires_txt = os.path.join(egginfo[0], 'requires.txt') + if os.path.exists(requires_txt): + with codecs.open(requires_txt) as f: + inst_req = [] + extras_req = collections.defaultdict(list) + current_feature = None + for line in f.readlines(): + line = line.rstrip() + if not line: + continue + + if line.startswith('['): + # PACKAGECONFIG must not contain expressions or whitespace + line = line.replace(" ", "") + line = line.replace(':', "") + line = line.replace('.', "-dot-") + line = line.replace('"', "") + line = line.replace('<', "-smaller-") + line = line.replace('>', "-bigger-") + line = line.replace('_', "-") + line = line.replace('(', "") + line = line.replace(')', "") + line = line.replace('!', "-not-") + line = line.replace('=', "-equals-") + current_feature = line[1:-1] + elif current_feature: + extras_req[current_feature].append(line) + else: + inst_req.append(line) + info['Install-requires'] = inst_req + info['Extras-require'] = extras_req + elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']): + info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO')) + + if setup_info: + if 'Install-requires' in setup_info: + info['Install-requires'] = setup_info['Install-requires'] + if 'Extras-require' in setup_info: + info['Extras-require'] = setup_info['Extras-require'] else: - provides_lines = (l.rstrip() for l in provides_output.splitlines()) - provides = set(l for l in provides_lines if l and l != 'setup') - deps -= provides + if setup_info: + info = setup_info + else: + info = self.get_setup_args_info(setupscript) - return deps + # Grab the license value before applying replacements + license_str = info.get('License', '').strip() - def parse_pkgdata_for_python_packages(self): - pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') + self.apply_info_replacements(info) - ldata = tinfoil.config_data.createCopy() - bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True) - python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') + if uses_setuptools: + classes.append('setuptools3') + else: + classes.append('distutils3') - dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') - python_dirs = [python_sitedir + os.sep, - os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, - os.path.dirname(python_sitedir) + os.sep] - packages = {} - for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): - files_info = None - with open(pkgdatafile, 'r') as f: - for line in f.readlines(): - field, value = line.split(': ', 1) - if field.startswith('FILES_INFO'): - files_info = ast.literal_eval(value) - break - else: - continue + if license_str: + for i, line in enumerate(lines_before): + if line.startswith('##LICENSE_PLACEHOLDER##'): + lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str) + break - for fn in files_info: - for suffix in importlib.machinery.all_suffixes(): - if fn.endswith(suffix): - break - else: - continue + if 'Classifier' in info: + license = self.handle_classifier_license(info['Classifier'], info.get('License', '')) + if license: + info['License'] = license - if fn.startswith(dynload_dir + os.sep): - if '/.debug/' in fn: - continue - base = os.path.basename(fn) - provided = base.split('.', 1)[0] - packages[provided] = os.path.basename(pkgdatafile) - continue + self.map_info_to_bbvar(info, extravalues) - for python_dir in python_dirs: - if fn.startswith(python_dir): - relpath = fn[len(python_dir):] - relstart, _, relremaining = relpath.partition(os.sep) - if relstart.endswith('.egg'): - relpath = relremaining - base, _ = os.path.splitext(relpath) + mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) - if '/.debug/' in base: - continue - if os.path.basename(base) == '__init__': - base = os.path.dirname(base) - base = base.replace(os.sep + os.sep, os.sep) - provided = base.replace(os.sep, '.') - packages[provided] = os.path.basename(pkgdatafile) - return packages + extras_req = set() + if 'Extras-require' in info: + extras_req = info['Extras-require'] + if extras_req: + lines_after.append('# The following configs & dependencies are from setuptools extras_require.') + lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.') + lines_after.append('# The upstream names may not correspond exactly to bitbake package names.') + lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.') + lines_after.append('#') + lines_after.append('# Uncomment this line to enable all the optional features.') + lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req))) + for feature, feature_reqs in extras_req.items(): + unmapped_deps.difference_update(feature_reqs) - @classmethod - def run_command(cls, cmd, **popenargs): - if 'stderr' not in popenargs: - popenargs['stderr'] = subprocess.STDOUT - try: - return subprocess.check_output(cmd, **popenargs).decode('utf-8') - except OSError as exc: - logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) - raise - except subprocess.CalledProcessError as exc: - logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) - raise + feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) + lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) + inst_reqs = set() + if 'Install-requires' in info: + if extras_req: + lines_after.append('') + inst_reqs = info['Install-requires'] + if inst_reqs: + unmapped_deps.difference_update(inst_reqs) + + inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) + lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') + lines_after.append('# upstream names may not correspond exactly to bitbake package names.') + lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps))) + + if mapped_deps: + name = info.get('Name') + if name and name[0] in mapped_deps: + # Attempt to avoid self-reference + mapped_deps.remove(name[0]) + mapped_deps -= set(self.excluded_pkgdeps) + if inst_reqs or extras_req: + lines_after.append('') + lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') + lines_after.append('# python sources, and might not be 100% accurate.') + lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) + + unmapped_deps -= set(extensions) + unmapped_deps -= set(self.assume_provided) + if unmapped_deps: + if mapped_deps: + lines_after.append('') + lines_after.append('# WARNING: We were unable to map the following python package/module') + lines_after.append('# dependencies to the bitbake packages which include them:') + lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps)) + + handled.append('buildsystem') def gather_setup_info(fileobj): parsed = ast.parse(fileobj.read(), fileobj.name) @@ -748,4 +770,4 @@ def has_non_literals(value): def register_recipe_handlers(handlers): # We need to make sure this is ahead of the makefile fallback handler - handlers.append((PythonRecipeHandler(), 70)) + handlers.append((PythonSetupPyRecipeHandler(), 70))
In order to prepare the support for pyproject.toml (PEP517 [1]) enabled projects, refactor the code and move setup.py specific code into a specific class in order to allow sharing the PythonRecipeHandler class No functionnal changes expected [1]: https://peps.python.org/pep-0517/#source-tree Signed-off-by: Julien Stephan <jstephan@baylibre.com> --- .../lib/recipetool/create_buildsys_python.py | 748 +++++++++--------- 1 file changed, 385 insertions(+), 363 deletions(-)