| Message ID | AM9PR09MB46426836B8FC0FACE562A084A8619@AM9PR09MB4642.eurprd09.prod.outlook.com |
|---|---|
| State | New |
| Headers | show |
| Series | [RFC,1/2] distutils3-legacy: fallback for missing setup.py | expand |
On Wed, Nov 24, 2021 at 12:15 PM Konrad Weihmann <kweihmann@outlook.com> wrote: > add a bbclass to disutils3 that generates a fallback setup.py in case > there is no setup.py available in the source dir, but a setup.cfg. > I’ll check this out later, but I do want to highlight that we will be deprecating distutils bbclasses (and moving them to meta-python for continuity). https://bugzilla.yoctoproject.org/show_bug.cgi?id=14610 We can refactor this series into the refactored setuptools3.bbclass (sans distutils). I will do this if we merge these changes. https://git.yoctoproject.org/poky-contrib/log/?h=timo/nodistutils_14610 I’d like to hear what concerns others have. > Use the mapping provided by > https://setuptools.pypa.io/en/latest/userguide/declarative_config.html > to translate the most essential items to legacy setuptools.setup > dictitonary. > > Signed-off-by: Konrad Weihmann <kweihmann@outlook.com> > --- > meta/classes/distutils3-legacy.bbclass | 112 +++++++++++++++++++++++++ > meta/classes/distutils3.bbclass | 1 + > 2 files changed, 113 insertions(+) > create mode 100644 meta/classes/distutils3-legacy.bbclass > > diff --git a/meta/classes/distutils3-legacy.bbclass > b/meta/classes/distutils3-legacy.bbclass > new file mode 100644 > index 0000000000..266d30138f > --- /dev/null > +++ b/meta/classes/distutils3-legacy.bbclass > @@ -0,0 +1,112 @@ > +# Helper to create a trimmed down setup.py from information found in > +# setup.cfg, in case there is no setup.py shipped with the sources > + > +# this functionality can be safely removed once the pypa community > +# comes up with a safe replacement for the functionality found in > distutils3.bbclass > + > +def distutils_legacy_package_name(d): > + # use pypi name or fall back to BPN > + return d.getVar("PYPI_PACKAGE") or d.getVar('BPN').replace('python-', > '').replace('python3-', '') > + > +DISTUTILS_LEGACY_VERSION ?= "${PV}" > +DISTUTILS_LEGACY_NAME ?= "${@distutils_legacy_package_name(d)}" > + > +python do_create_setup_py_legacy() { > + import os > + > + if os.path.exists(os.path.join(d.getVar("DISTUTILS_SETUP_PATH"), > "setup.py")): > + return > + > + from configparser import ConfigParser, NoOptionError, NoSectionError, > ParsingError > + import re > + > + config = ConfigParser() > + try: > + config.read(os.path.join(d.getVar("DISTUTILS_SETUP_PATH"), > "setup.cfg")) > + except FileNotFoundError: > + return > + > + def _strip(x): > + return re.sub(r"\s|\t|\n", "", x) > + > + def get_section(section): > + try: > + return dict(config.items(section=section)) > + except (NoSectionError, ParsingError): > + return None > + > + def get_option(section, option): > + try: > + return config.get(section=section, option=option) > + except (NoOptionError, NoSectionError, ParsingError): > + return None > + > + def extract_bool(section, option, default): > + _option = get_option(section, option) > + if _option is None: > + return default > + return bool(_strip(_option)) > + > + def extract_str(section, option, default): > + _option = get_option(section, option) > + if _option is None: > + return default > + return _strip(_option) > + > + def extract_dict_vallist(section, default, delim=""): > + _section = get_section(section) > + if _section is None: > + return default > + return {_strip(k): [_strip(x) for x in re.split(delim, v)] if > delim else [ _strip(v) ] for k, v in _section.items()} > + > + def extract_dict(section, default): > + _section = get_section(section) > + if _section is None: > + return default > + return {_strip(k): _strip(v) for k, v in _section.items()} > + > + def extract_list(section, option, default, delim): > + _option = get_option(section, option) > + if _option is None: > + return default > + bb.warn("%s:%s -> %s" % (section, option, _option)) > + _listitems = re.split(delim, _option) if delim else [_option] > + return [_strip(x) for x in _listitems] > + > + def quote(x): > + return '"%s"' % x > + > + _pkginfo = { > + "entry_points": extract_dict_vallist("options.entry_points", {}), > + "include_package_data": extract_bool("options", > "include_package_data", False), > + "name": quote(extract_str("options", "name", > d.getVar("DISTUTILS_LEGACY_NAME"))), > + "package_data": extract_dict_vallist("options.package_data", {}, > r"\s+|,"), > + "packages": extract_list("options", "packages", [], ""), > + "version": quote(d.getVar("DISTUTILS_LEGACY_VERSION")), > + "zip_safe": extract_bool("options", "zip_safe", False), > + "install_requires": extract_list("options", "install_requires", > [], r"\t+|\n+"), > + "python_requires": quote(extract_str("options", > "python_requires", ">0.0")), > + "package_dir": extract_dict("package_dir", {}), > + "py_modules": extract_list("options", "py_modules", [], r"\s+|,"), > + } > + > + # In case packages is using :find module > + # we need to look for top level directories containing a __init__.py > + if _pkginfo["packages"] == ["find:"]: > + # top level search dir can be adjusted by options.packages.find > option > + _path = extract_str("options.packages.find", "where", "") > + _pkginfo["packages"] = set(x.name for x in > os.scandir(os.path.join( > + d.getVar("DISTUTILS_SETUP_PATH"), _path)) if os.path.isdir(x) > and os.path.exists(os.path.join(x, "__init__.py"))) > + > + with open(os.path.join(d.getVar("DISTUTILS_SETUP_PATH"), "setup.py"), > "w") as o: > + o.write("import setuptools\n") > + o.write("setuptools.setup(\n") > + for k, v in _pkginfo.items(): > + o.write("%s = %s,\n" % (str(k), str(v))) > + o.write(")") > + > + > +} > + > +do_create_setup_py_legacy[doc] = "Create a fallback version of legacy > setup.py if not existing" > +addtask do_create_setup_py_legacy before do_configure after do_patch > do_prepare_recipe_sysroot > diff --git a/meta/classes/distutils3.bbclass > b/meta/classes/distutils3.bbclass > index be645d37bd..f26f0d5184 100644 > --- a/meta/classes/distutils3.bbclass > +++ b/meta/classes/distutils3.bbclass > @@ -1,4 +1,5 @@ > inherit distutils3-base > +inherit distutils3-legacy > > B = "${WORKDIR}/build" > distutils_do_configure[cleandirs] = "${B}" > -- > 2.25.1 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#158741): > https://lists.openembedded.org/g/openembedded-core/message/158741 > Mute This Topic: https://lists.openembedded.org/mt/87289428/924729 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ > ticotimo@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
On 24.11.21 22:02, Tim Orling wrote: > > > On Wed, Nov 24, 2021 at 12:15 PM Konrad Weihmann <kweihmann@outlook.com > <mailto:kweihmann@outlook.com>> wrote: > > add a bbclass to disutils3 that generates a fallback setup.py in case > there is no setup.py available in the source dir, but a setup.cfg. > > > I’ll check this out later, but I do want to highlight that we will be > deprecating distutils bbclasses (and moving them to meta-python for > continuity). > > https://bugzilla.yoctoproject.org/show_bug.cgi?id=14610 > <https://bugzilla.yoctoproject.org/show_bug.cgi?id=14610> Not sure if we should support it at all after the rework is done. Most of the python modules that haven't made at least the move to setup.cfg will stop working with py3.12 anyway - putting some pressure on the respective upstream to migrate. So what is the benefit in moving distutils-support to meta-python (which btw bundles poky/meta to openembedded-layer - a move that is discouraged by a couple of projects I came across so far)? I mean the patches currently in the works likely will affect kirkstone+ releases only, right? so current already released branches will remain unaffected by this switch - or is there a plan to even backport this change to other releases? > > > We can refactor this series into the refactored setuptools3.bbclass > (sans distutils). I will do this if we merge these changes. > > https://git.yoctoproject.org/poky-contrib/log/?h=timo/nodistutils_14610 > <https://git.yoctoproject.org/poky-contrib/log/?h=timo/nodistutils_14610> Fine for me - I would be also be fine, if we could use my patch for an intermediate period, till the removal of distutils is done. > > I’d like to hear what concerns others have. > > > Use the mapping provided by > https://setuptools.pypa.io/en/latest/userguide/declarative_config.html > <https://setuptools.pypa.io/en/latest/userguide/declarative_config.html> > to translate the most essential items to legacy setuptools.setup > dictitonary. > > Signed-off-by: Konrad Weihmann <kweihmann@outlook.com > <mailto:kweihmann@outlook.com>> > --- > meta/classes/distutils3-legacy.bbclass | 112 +++++++++++++++++++++++++ > meta/classes/distutils3.bbclass | 1 + > 2 files changed, 113 insertions(+) > create mode 100644 meta/classes/distutils3-legacy.bbclass > > diff --git a/meta/classes/distutils3-legacy.bbclass > b/meta/classes/distutils3-legacy.bbclass > new file mode 100644 > index 0000000000..266d30138f > --- /dev/null > +++ b/meta/classes/distutils3-legacy.bbclass > @@ -0,0 +1,112 @@ > +# Helper to create a trimmed down setup.py from information found in > +# setup.cfg, in case there is no setup.py shipped with the sources > + > +# this functionality can be safely removed once the pypa community > +# comes up with a safe replacement for the functionality found in > distutils3.bbclass > + > +def distutils_legacy_package_name(d): > + # use pypi name or fall back to BPN > + return d.getVar("PYPI_PACKAGE") or > d.getVar('BPN').replace('python-', '').replace('python3-', '') > + > +DISTUTILS_LEGACY_VERSION ?= "${PV}" > +DISTUTILS_LEGACY_NAME ?= "${@distutils_legacy_package_name(d)}" > + > +python do_create_setup_py_legacy() { > + import os > + > + if > os.path.exists(os.path.join(d.getVar("DISTUTILS_SETUP_PATH"), > "setup.py")): > + return > + > + from configparser import ConfigParser, NoOptionError, > NoSectionError, ParsingError > + import re > + > + config = ConfigParser() > + try: > + config.read(os.path.join(d.getVar("DISTUTILS_SETUP_PATH"), > "setup.cfg")) > + except FileNotFoundError: > + return > + > + def _strip(x): > + return re.sub(r"\s|\t|\n", "", x) > + > + def get_section(section): > + try: > + return dict(config.items(section=section)) > + except (NoSectionError, ParsingError): > + return None > + > + def get_option(section, option): > + try: > + return config.get(section=section, option=option) > + except (NoOptionError, NoSectionError, ParsingError): > + return None > + > + def extract_bool(section, option, default): > + _option = get_option(section, option) > + if _option is None: > + return default > + return bool(_strip(_option)) > + > + def extract_str(section, option, default): > + _option = get_option(section, option) > + if _option is None: > + return default > + return _strip(_option) > + > + def extract_dict_vallist(section, default, delim=""): > + _section = get_section(section) > + if _section is None: > + return default > + return {_strip(k): [_strip(x) for x in re.split(delim, v)] > if delim else [ _strip(v) ] for k, v in _section.items()} > + > + def extract_dict(section, default): > + _section = get_section(section) > + if _section is None: > + return default > + return {_strip(k): _strip(v) for k, v in _section.items()} > + > + def extract_list(section, option, default, delim): > + _option = get_option(section, option) > + if _option is None: > + return default > + bb.warn("%s:%s -> %s" % (section, option, _option)) > + _listitems = re.split(delim, _option) if delim else [_option] > + return [_strip(x) for x in _listitems] > + > + def quote(x): > + return '"%s"' % x > + > + _pkginfo = { > + "entry_points": > extract_dict_vallist("options.entry_points", {}), > + "include_package_data": extract_bool("options", > "include_package_data", False), > + "name": quote(extract_str("options", "name", > d.getVar("DISTUTILS_LEGACY_NAME"))), > + "package_data": > extract_dict_vallist("options.package_data", {}, r"\s+|,"), > + "packages": extract_list("options", "packages", [], ""), > + "version": quote(d.getVar("DISTUTILS_LEGACY_VERSION")), > + "zip_safe": extract_bool("options", "zip_safe", False), > + "install_requires": extract_list("options", > "install_requires", [], r"\t+|\n+"), > + "python_requires": quote(extract_str("options", > "python_requires", ">0.0")), > + "package_dir": extract_dict("package_dir", {}), > + "py_modules": extract_list("options", "py_modules", [], > r"\s+|,"), > + } > + > + # In case packages is using :find module > + # we need to look for top level directories containing a > __init__.py > + if _pkginfo["packages"] == ["find:"]: > + # top level search dir can be adjusted by > options.packages.find option > + _path = extract_str("options.packages.find", "where", "") > + _pkginfo["packages"] = set(x.name <http://x.name> for x in > os.scandir(os.path.join( > + d.getVar("DISTUTILS_SETUP_PATH"), _path)) if > os.path.isdir(x) and os.path.exists(os.path.join(x, "__init__.py"))) > + > + with open(os.path.join(d.getVar("DISTUTILS_SETUP_PATH"), > "setup.py"), "w") as o: > + o.write("import setuptools\n") > + o.write("setuptools.setup(\n") > + for k, v in _pkginfo.items(): > + o.write("%s = %s,\n" % (str(k), str(v))) > + o.write(")") > + > + > +} > + > +do_create_setup_py_legacy[doc] = "Create a fallback version of > legacy setup.py if not existing" > +addtask do_create_setup_py_legacy before do_configure after > do_patch do_prepare_recipe_sysroot > diff --git a/meta/classes/distutils3.bbclass > b/meta/classes/distutils3.bbclass > index be645d37bd..f26f0d5184 100644 > --- a/meta/classes/distutils3.bbclass > +++ b/meta/classes/distutils3.bbclass > @@ -1,4 +1,5 @@ > inherit distutils3-base > +inherit distutils3-legacy > > B = "${WORKDIR}/build" > distutils_do_configure[cleandirs] = "${B}" > -- > 2.25.1 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#158741): > https://lists.openembedded.org/g/openembedded-core/message/158741 > <https://lists.openembedded.org/g/openembedded-core/message/158741> > Mute This Topic: https://lists.openembedded.org/mt/87289428/924729 > <https://lists.openembedded.org/mt/87289428/924729> > Group Owner: openembedded-core+owner@lists.openembedded.org > <mailto:openembedded-core%2Bowner@lists.openembedded.org> > Unsubscribe: > https://lists.openembedded.org/g/openembedded-core/unsub > <https://lists.openembedded.org/g/openembedded-core/unsub> > [ticotimo@gmail.com <mailto:ticotimo@gmail.com>] > -=-=-=-=-=-=-=-=-=-=-=- >
diff --git a/meta/classes/distutils3-legacy.bbclass b/meta/classes/distutils3-legacy.bbclass new file mode 100644 index 0000000000..266d30138f --- /dev/null +++ b/meta/classes/distutils3-legacy.bbclass @@ -0,0 +1,112 @@ +# Helper to create a trimmed down setup.py from information found in +# setup.cfg, in case there is no setup.py shipped with the sources + +# this functionality can be safely removed once the pypa community +# comes up with a safe replacement for the functionality found in distutils3.bbclass + +def distutils_legacy_package_name(d): + # use pypi name or fall back to BPN + return d.getVar("PYPI_PACKAGE") or d.getVar('BPN').replace('python-', '').replace('python3-', '') + +DISTUTILS_LEGACY_VERSION ?= "${PV}" +DISTUTILS_LEGACY_NAME ?= "${@distutils_legacy_package_name(d)}" + +python do_create_setup_py_legacy() { + import os + + if os.path.exists(os.path.join(d.getVar("DISTUTILS_SETUP_PATH"), "setup.py")): + return + + from configparser import ConfigParser, NoOptionError, NoSectionError, ParsingError + import re + + config = ConfigParser() + try: + config.read(os.path.join(d.getVar("DISTUTILS_SETUP_PATH"), "setup.cfg")) + except FileNotFoundError: + return + + def _strip(x): + return re.sub(r"\s|\t|\n", "", x) + + def get_section(section): + try: + return dict(config.items(section=section)) + except (NoSectionError, ParsingError): + return None + + def get_option(section, option): + try: + return config.get(section=section, option=option) + except (NoOptionError, NoSectionError, ParsingError): + return None + + def extract_bool(section, option, default): + _option = get_option(section, option) + if _option is None: + return default + return bool(_strip(_option)) + + def extract_str(section, option, default): + _option = get_option(section, option) + if _option is None: + return default + return _strip(_option) + + def extract_dict_vallist(section, default, delim=""): + _section = get_section(section) + if _section is None: + return default + return {_strip(k): [_strip(x) for x in re.split(delim, v)] if delim else [ _strip(v) ] for k, v in _section.items()} + + def extract_dict(section, default): + _section = get_section(section) + if _section is None: + return default + return {_strip(k): _strip(v) for k, v in _section.items()} + + def extract_list(section, option, default, delim): + _option = get_option(section, option) + if _option is None: + return default + bb.warn("%s:%s -> %s" % (section, option, _option)) + _listitems = re.split(delim, _option) if delim else [_option] + return [_strip(x) for x in _listitems] + + def quote(x): + return '"%s"' % x + + _pkginfo = { + "entry_points": extract_dict_vallist("options.entry_points", {}), + "include_package_data": extract_bool("options", "include_package_data", False), + "name": quote(extract_str("options", "name", d.getVar("DISTUTILS_LEGACY_NAME"))), + "package_data": extract_dict_vallist("options.package_data", {}, r"\s+|,"), + "packages": extract_list("options", "packages", [], ""), + "version": quote(d.getVar("DISTUTILS_LEGACY_VERSION")), + "zip_safe": extract_bool("options", "zip_safe", False), + "install_requires": extract_list("options", "install_requires", [], r"\t+|\n+"), + "python_requires": quote(extract_str("options", "python_requires", ">0.0")), + "package_dir": extract_dict("package_dir", {}), + "py_modules": extract_list("options", "py_modules", [], r"\s+|,"), + } + + # In case packages is using :find module + # we need to look for top level directories containing a __init__.py + if _pkginfo["packages"] == ["find:"]: + # top level search dir can be adjusted by options.packages.find option + _path = extract_str("options.packages.find", "where", "") + _pkginfo["packages"] = set(x.name for x in os.scandir(os.path.join( + d.getVar("DISTUTILS_SETUP_PATH"), _path)) if os.path.isdir(x) and os.path.exists(os.path.join(x, "__init__.py"))) + + with open(os.path.join(d.getVar("DISTUTILS_SETUP_PATH"), "setup.py"), "w") as o: + o.write("import setuptools\n") + o.write("setuptools.setup(\n") + for k, v in _pkginfo.items(): + o.write("%s = %s,\n" % (str(k), str(v))) + o.write(")") + + +} + +do_create_setup_py_legacy[doc] = "Create a fallback version of legacy setup.py if not existing" +addtask do_create_setup_py_legacy before do_configure after do_patch do_prepare_recipe_sysroot diff --git a/meta/classes/distutils3.bbclass b/meta/classes/distutils3.bbclass index be645d37bd..f26f0d5184 100644 --- a/meta/classes/distutils3.bbclass +++ b/meta/classes/distutils3.bbclass @@ -1,4 +1,5 @@ inherit distutils3-base +inherit distutils3-legacy B = "${WORKDIR}/build" distutils_do_configure[cleandirs] = "${B}"
add a bbclass to disutils3 that generates a fallback setup.py in case there is no setup.py available in the source dir, but a setup.cfg. Use the mapping provided by https://setuptools.pypa.io/en/latest/userguide/declarative_config.html to translate the most essential items to legacy setuptools.setup dictitonary. Signed-off-by: Konrad Weihmann <kweihmann@outlook.com> --- meta/classes/distutils3-legacy.bbclass | 112 +++++++++++++++++++++++++ meta/classes/distutils3.bbclass | 1 + 2 files changed, 113 insertions(+) create mode 100644 meta/classes/distutils3-legacy.bbclass