| Message ID | 20240501212753.275862-1-gudnimar@noxmedical.com |
|---|---|
| State | New |
| Headers | show |
| Series | python3-manifest: prune python3-core | expand |
On Wed, May 1, 2024 at 2:28 PM Guðni Már Gilbert via lists.openembedded.org <gudnimar=noxmedical.com@lists.openembedded.org> wrote: > Remove file entries which no longer exist in the source. > > I spotted this when I added a custom QA error locally which > raised a warning when a file in FILES list of a package isn't > found in the source. > > This manifest is automatically generated by: https://git.yoctoproject.org/poky/tree/meta/recipes-devtools/python/python3/create_manifest3.py We do not want to be making manual changes to the json file. If there are mistakes due to changes in what is now shipped with Python 3.12.x, we should be fixing in the tooling, not this file. Also, please share your local QA tooling. It does no one any good to have that be siloed. > Signed-off-by: Guðni Már Gilbert <gudnimar@noxmedical.com> > --- > .../python/python3/python3-manifest.json | 23 ------------------- > 1 file changed, 23 deletions(-) > > diff --git a/meta/recipes-devtools/python/python3/python3-manifest.json > b/meta/recipes-devtools/python/python3/python3-manifest.json > index 46092d4004..6e06002783 100644 > --- a/meta/recipes-devtools/python/python3/python3-manifest.json > +++ b/meta/recipes-devtools/python/python3/python3-manifest.json > @@ -219,15 +219,9 @@ > "rdepends": [], > "files": [ > "${bindir}/python${PYTHON_MAJMIN}", > - "${bindir}/python${PYTHON_MAJMIN}.real", > "${bindir}/python3", > "${includedir}/python${PYTHON_MAJMIN}/pyconfig*.h", > - "${libdir}/python${PYTHON_MAJMIN}/UserDict.py", > - "${libdir}/python${PYTHON_MAJMIN}/UserList.py", > - "${libdir}/python${PYTHON_MAJMIN}/UserString.py", > "${libdir}/python${PYTHON_MAJMIN}/__future__.py", > - "${libdir}/python${PYTHON_MAJMIN}/_abcoll.py", > - "${libdir}/python${PYTHON_MAJMIN}/_bootlocale.py", > "${libdir}/python${PYTHON_MAJMIN}/_collections_abc.py", > "${libdir}/python${PYTHON_MAJMIN}/_compression.py", > "${libdir}/python${PYTHON_MAJMIN}/_markupbase.py", > @@ -262,7 +256,6 @@ > "${libdir}/python${PYTHON_MAJMIN}/getopt.py", > "${libdir}/python${PYTHON_MAJMIN}/gettext.py", > "${libdir}/python${PYTHON_MAJMIN}/heapq.py", > - "${libdir}/python${PYTHON_MAJMIN}/imp.py", > "${libdir}/python${PYTHON_MAJMIN}/importlib", > "${libdir}/python${PYTHON_MAJMIN}/importlib/_abc.py", > "${libdir}/python${PYTHON_MAJMIN}/importlib/_bootstrap.py", > @@ -274,10 +267,6 @@ > "${libdir}/python${PYTHON_MAJMIN}/io.py", > "${libdir}/python${PYTHON_MAJMIN}/ipaddress.py", > "${libdir}/python${PYTHON_MAJMIN}/keyword.py", > - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/__pycache__/_struct.*.so", > - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/__pycache__/binascii.*.so", > - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/__pycache__/time.*.so", > - > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/__pycache__/xreadlines.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_bisect.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_bz2.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_csv.*.so", > @@ -286,22 +275,17 @@ > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_opcode.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_posixsubprocess.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_struct.*.so", > - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_typing.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/array.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/binascii.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/grp.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/math.*.so", > - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/parser.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/readline.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/select.*.so", > - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/time.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/unicodedata.*.so", > - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/xreadlines.*.so", > "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/zlib.*.so", > "${libdir}/python${PYTHON_MAJMIN}/linecache.py", > "${libdir}/python${PYTHON_MAJMIN}/locale.py", > "${libdir}/python${PYTHON_MAJMIN}/lzma.py", > - "${libdir}/python${PYTHON_MAJMIN}/new.py", > "${libdir}/python${PYTHON_MAJMIN}/ntpath.py", > "${libdir}/python${PYTHON_MAJMIN}/opcode.py", > "${libdir}/python${PYTHON_MAJMIN}/operator.py", > @@ -323,7 +307,6 @@ > "${libdir}/python${PYTHON_MAJMIN}/shutil.py", > "${libdir}/python${PYTHON_MAJMIN}/signal.py", > "${libdir}/python${PYTHON_MAJMIN}/site.py", > - "${libdir}/python${PYTHON_MAJMIN}/sitecustomize.py", > "${libdir}/python${PYTHON_MAJMIN}/sre_compile.py", > "${libdir}/python${PYTHON_MAJMIN}/sre_constants.py", > "${libdir}/python${PYTHON_MAJMIN}/sre_parse.py", > @@ -331,7 +314,6 @@ > "${libdir}/python${PYTHON_MAJMIN}/stringprep.py", > "${libdir}/python${PYTHON_MAJMIN}/struct.py", > "${libdir}/python${PYTHON_MAJMIN}/subprocess.py", > - "${libdir}/python${PYTHON_MAJMIN}/symbol.py", > "${libdir}/python${PYTHON_MAJMIN}/sysconfig.py", > "${libdir}/python${PYTHON_MAJMIN}/textwrap.py", > "${libdir}/python${PYTHON_MAJMIN}/threading.py", > @@ -349,7 +331,6 @@ > ], > "cached": [ > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/__future__.*.pyc", > - "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_bootlocale.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_collections_abc.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_compression.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_markupbase.*.pyc", > @@ -378,7 +359,6 @@ > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/getopt.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/gettext.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/heapq.*.pyc", > - "${libdir}/python${PYTHON_MAJMIN}/__pycache__/imp.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/inspect.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/io.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/ipaddress.*.pyc", > @@ -395,7 +375,6 @@ > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/pkgutil.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/platform.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/posixpath.*.pyc", > - "${libdir}/python${PYTHON_MAJMIN}/__pycache__/re.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/reprlib.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/rlcompleter.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/runpy.*.pyc", > @@ -410,13 +389,11 @@ > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/stringprep.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/struct.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/subprocess.*.pyc", > - "${libdir}/python${PYTHON_MAJMIN}/__pycache__/symbol.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/sysconfig.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/textwrap.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/threading.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/token.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/tokenize.*.pyc", > - "${libdir}/python${PYTHON_MAJMIN}/__pycache__/traceback.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/types.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/typing.*.pyc", > "${libdir}/python${PYTHON_MAJMIN}/__pycache__/warnings.*.pyc", > -- > 2.34.1 > > > Warning: Copyright Nox Health. This communication is confidential and may > contain privileged information. By the use of email over the Internet or > other communication systems, Nox Health is not waiving either > confidentiality of, or legal privilege in, the content of the email and of > any attachments. If the recipient of this message is not the intended > addressee, please call Nox Health immediately by calling (+1) 888.505.0280 > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#198888): > https://lists.openembedded.org/g/openembedded-core/message/198888 > Mute This Topic: https://lists.openembedded.org/mt/105851698/924729 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ > ticotimo@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
On Wed, May 1, 2024 at 11:27 PM, Tim Orling wrote: > > This manifest is automatically generated by: > https://git.yoctoproject.org/poky/tree/meta/recipes-devtools/python/python3/create_manifest3.py > > > We do not want to be making manual changes to the json file. If there are > mistakes due to changes in what is now shipped with Python 3.12.x, we > should be fixing in the tooling, not this file. > Using the script was my first approach, but I found out that the script doesn't prune the core package and is essentially expecting that the core is already set up correctly in the manifest file. Only items are appended to the core, items are never removed as far as I could see. This could certainly be improved and I made an attempt but could not finish it today. I'd be happy to look into it when I find time. > > Also, please share your local QA tooling. It does no one any good to have > that be siloed > What I have so far is to insert the warning in this line: https://github.com/openembedded/openembedded-core/blob/914128f6bd988cde278e087fb9457a0c70c7e5ec/meta/lib/oe/package.py#L1343 and adding the following two lines: msg = f "FILES variable for package { pkg } contains ' { file } ' but was not found. \n " oe.qa.handle_error( "files-not-found" , msg , d ) before the 'continue' statement. I also added "files-not-found" to WARN_QA in insane.class just like how "files-invalid" is handled
Hi again Tim, update from me regarding the scripts. TLDR: The manifest is out of date, and the python scripts are working as they should. ---- I've convinced myself the python scripts used to generate the manifest *are correct*. However, for anyone reading this, I can confirm the manifest is out of date. It would be nice to run the script on the next Python upgrade since it will remove a lot of dead file entries. I couldn't see where I went wrong in my statement in the previous email I sent. I suspect two things 1) I wasn't looking at the correct "new" json output from the task 2) part to blame is I didn't fully enable all PACKAGECONFIGS, but this seems to mainly affect the tkinter and turtle packages. On the bright side of things I did learn a few things so this wasn't a total waste of time for me :) Just makes things even more fun. Best regards, Gudni
diff --git a/meta/recipes-devtools/python/python3/python3-manifest.json b/meta/recipes-devtools/python/python3/python3-manifest.json index 46092d4004..6e06002783 100644 --- a/meta/recipes-devtools/python/python3/python3-manifest.json +++ b/meta/recipes-devtools/python/python3/python3-manifest.json @@ -219,15 +219,9 @@ "rdepends": [], "files": [ "${bindir}/python${PYTHON_MAJMIN}", - "${bindir}/python${PYTHON_MAJMIN}.real", "${bindir}/python3", "${includedir}/python${PYTHON_MAJMIN}/pyconfig*.h", - "${libdir}/python${PYTHON_MAJMIN}/UserDict.py", - "${libdir}/python${PYTHON_MAJMIN}/UserList.py", - "${libdir}/python${PYTHON_MAJMIN}/UserString.py", "${libdir}/python${PYTHON_MAJMIN}/__future__.py", - "${libdir}/python${PYTHON_MAJMIN}/_abcoll.py", - "${libdir}/python${PYTHON_MAJMIN}/_bootlocale.py", "${libdir}/python${PYTHON_MAJMIN}/_collections_abc.py", "${libdir}/python${PYTHON_MAJMIN}/_compression.py", "${libdir}/python${PYTHON_MAJMIN}/_markupbase.py", @@ -262,7 +256,6 @@ "${libdir}/python${PYTHON_MAJMIN}/getopt.py", "${libdir}/python${PYTHON_MAJMIN}/gettext.py", "${libdir}/python${PYTHON_MAJMIN}/heapq.py", - "${libdir}/python${PYTHON_MAJMIN}/imp.py", "${libdir}/python${PYTHON_MAJMIN}/importlib", "${libdir}/python${PYTHON_MAJMIN}/importlib/_abc.py", "${libdir}/python${PYTHON_MAJMIN}/importlib/_bootstrap.py", @@ -274,10 +267,6 @@ "${libdir}/python${PYTHON_MAJMIN}/io.py", "${libdir}/python${PYTHON_MAJMIN}/ipaddress.py", "${libdir}/python${PYTHON_MAJMIN}/keyword.py", - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/__pycache__/_struct.*.so", - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/__pycache__/binascii.*.so", - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/__pycache__/time.*.so", - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/__pycache__/xreadlines.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_bisect.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_bz2.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_csv.*.so", @@ -286,22 +275,17 @@ "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_opcode.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_posixsubprocess.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_struct.*.so", - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/_typing.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/array.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/binascii.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/grp.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/math.*.so", - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/parser.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/readline.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/select.*.so", - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/time.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/unicodedata.*.so", - "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/xreadlines.*.so", "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/zlib.*.so", "${libdir}/python${PYTHON_MAJMIN}/linecache.py", "${libdir}/python${PYTHON_MAJMIN}/locale.py", "${libdir}/python${PYTHON_MAJMIN}/lzma.py", - "${libdir}/python${PYTHON_MAJMIN}/new.py", "${libdir}/python${PYTHON_MAJMIN}/ntpath.py", "${libdir}/python${PYTHON_MAJMIN}/opcode.py", "${libdir}/python${PYTHON_MAJMIN}/operator.py", @@ -323,7 +307,6 @@ "${libdir}/python${PYTHON_MAJMIN}/shutil.py", "${libdir}/python${PYTHON_MAJMIN}/signal.py", "${libdir}/python${PYTHON_MAJMIN}/site.py", - "${libdir}/python${PYTHON_MAJMIN}/sitecustomize.py", "${libdir}/python${PYTHON_MAJMIN}/sre_compile.py", "${libdir}/python${PYTHON_MAJMIN}/sre_constants.py", "${libdir}/python${PYTHON_MAJMIN}/sre_parse.py", @@ -331,7 +314,6 @@ "${libdir}/python${PYTHON_MAJMIN}/stringprep.py", "${libdir}/python${PYTHON_MAJMIN}/struct.py", "${libdir}/python${PYTHON_MAJMIN}/subprocess.py", - "${libdir}/python${PYTHON_MAJMIN}/symbol.py", "${libdir}/python${PYTHON_MAJMIN}/sysconfig.py", "${libdir}/python${PYTHON_MAJMIN}/textwrap.py", "${libdir}/python${PYTHON_MAJMIN}/threading.py", @@ -349,7 +331,6 @@ ], "cached": [ "${libdir}/python${PYTHON_MAJMIN}/__pycache__/__future__.*.pyc", - "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_bootlocale.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_collections_abc.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_compression.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_markupbase.*.pyc", @@ -378,7 +359,6 @@ "${libdir}/python${PYTHON_MAJMIN}/__pycache__/getopt.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/gettext.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/heapq.*.pyc", - "${libdir}/python${PYTHON_MAJMIN}/__pycache__/imp.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/inspect.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/io.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/ipaddress.*.pyc", @@ -395,7 +375,6 @@ "${libdir}/python${PYTHON_MAJMIN}/__pycache__/pkgutil.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/platform.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/posixpath.*.pyc", - "${libdir}/python${PYTHON_MAJMIN}/__pycache__/re.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/reprlib.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/rlcompleter.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/runpy.*.pyc", @@ -410,13 +389,11 @@ "${libdir}/python${PYTHON_MAJMIN}/__pycache__/stringprep.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/struct.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/subprocess.*.pyc", - "${libdir}/python${PYTHON_MAJMIN}/__pycache__/symbol.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/sysconfig.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/textwrap.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/threading.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/token.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/tokenize.*.pyc", - "${libdir}/python${PYTHON_MAJMIN}/__pycache__/traceback.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/types.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/typing.*.pyc", "${libdir}/python${PYTHON_MAJMIN}/__pycache__/warnings.*.pyc",
Remove file entries which no longer exist in the source. I spotted this when I added a custom QA error locally which raised a warning when a file in FILES list of a package isn't found in the source. Signed-off-by: Guðni Már Gilbert <gudnimar@noxmedical.com> --- .../python/python3/python3-manifest.json | 23 ------------------- 1 file changed, 23 deletions(-)