| Message ID | 20251002155210.782342-1-yoann.congal@smile.fr |
|---|---|
| State | New |
| Headers | show |
| Series | [RFC] bitbake-setup: optionally support JSON5 (e.g. comments) format | expand |
Le jeu. 2 oct. 2025 à 17:52, Yoann Congal <yoann.congal@smile.fr> a écrit : > From: Yoann Congal <yoann.congal@smile.fr> > > One major limitation of JSON files is that comments are not supported. > JSON5 (https://json5.org/) is an extension of the current JSON file > format that support comments (among other evolutions). > > This patch adds the support of the JSON5 file format when reading > configuration files. > > Sadly, no JSON5 support exists in Python stdlib, so we have to install a > module. I chose https://pypi.org/project/json5/ (but > https://pypi.org/project/pyjson5/ may be an alternative). > > I took care of only patching reading of the config file, the internal > JSON files of bitbake-setup are still in classic JSON, which mean that > operations that don't need to read the original configuration still work > without a json5 module. > > I put an example of a JSON5 config in > default-registry/configurations/poky-master_json5.conf.json > This is poky-master.conf.json translated to JSON5 plus some comments. > > To try this: > python3 -m venv .venv; . .venv/bin/activate # if needed > pip install json5 > ./bin/bitbake-setup init > default-registry/configurations/poky-master_json5.conf.json poky_json5 > Addendum: The main need is to have comments in the configuration file. An alternative is to use the YAML format (which supports comments) but is still not supported without installing modules outside of python sdtlib. > > Signed-off-by: Yoann Congal <yoann.congal@smile.fr> > --- > bin/bitbake-setup | 8 ++ > .../poky-master_json5.conf.json | 93 +++++++++++++++++++ > 2 files changed, 101 insertions(+) > create mode 100644 > default-registry/configurations/poky-master_json5.conf.json > > diff --git a/bin/bitbake-setup b/bin/bitbake-setup > index bb466d85..d1c54269 100755 > --- a/bin/bitbake-setup > +++ b/bin/bitbake-setup > @@ -19,6 +19,14 @@ import datetime > import glob > import subprocess > > +try: > + # Try to import the non-standard json5 module to load JSON files > + from json5 import load as json5_load > + json.load = json5_load > + print("Using json5 to load JSON") > +except ImportError: > + pass > + > default_registry = os.path.normpath(os.path.dirname(__file__) + > "/../default-registry") > > bindir = os.path.abspath(os.path.dirname(__file__)) > diff --git a/default-registry/configurations/poky-master_json5.conf.json > b/default-registry/configurations/poky-master_json5.conf.json > new file mode 100644 > index 00000000..cf447e9a > --- /dev/null > +++ b/default-registry/configurations/poky-master_json5.conf.json > @@ -0,0 +1,93 @@ > +{ > + // This file format supports comments > + description: "Poky - The Yocto Project testing distribution > configurations and hardware test platforms", > +// ^ ^ As long as the object keys are compatible with a JS > variable name, quotes are optional: > + // > + sources: { > + bitbake: { > + "git-remote": { // note "git-remote" is not a valid JS > variable name because of the dash > + remotes: { > + origin: { > + uri: "git:// > git.openembedded.org/bitbake;protocol=https", > + }, > + }, > + branch: "master", > + rev: "master", // This file format accepts a comma after > the last element of lists/objects > + }, > + path: "bitbake", > + }, > + "openembedded-core": { > + "git-remote": { > + remotes: { > + origin: { > + uri: "git:// > git.openembedded.org/openembedded-core;protocol=https", > + }, > + }, > + branch: "master", > + rev: "master", > + }, > + path: "openembedded-core", > + }, > + "meta-yocto": { > + "git-remote": { > + remotes: { > + origin: { > + uri: "git:// > git.yoctoproject.org/meta-yocto;protocol=https", > + }, > + }, > + branch: "master", > + rev: "master", > + }, > + path: "meta-yocto", > + }, > + "yocto-docs": { > + "git-remote": { > + remotes: { > + origin: { > + uri: "git:// > git.yoctoproject.org/yocto-docs;protocol=https", > + }, > + }, > + branch: "master", > + rev: "master", > + }, > + path: "yocto-docs", > + }, > + }, > + "bitbake-setup": { > + configurations: [ > + { > + name: "poky_json5", > + description: "Poky - The Yocto Project testing > distribution (from a JSON5 config)", > + "bb-layers": [ > + "openembedded-core/meta", > + "meta-yocto/meta-yocto-bsp", > + "meta-yocto/meta-poky", > + ], > + "oe-fragments": [ > + "core/yocto/sstate-mirror-cdn", > + ], > + "oe-fragments-one-of": { > + machine: { > + description: "Target machines", > + options: [ > + "machine/qemux86-64", > + "machine/qemuarm64", > + "machine/qemuriscv64", > + "machine/genericarm64", > + "machine/genericx86-64", > + ], > + }, > + distro: { > + description: "Distribution configuration > variants", > + options: [ > + "distro/poky", > + "distro/poky-altcfg", > + "distro/poky-tiny", > + ], > + }, > + }, > + }, > + ], > + }, > + version: "1.0", > +} >
On Thu, 2 Oct 2025 at 17:52, Yoann Congal via lists.openembedded.org <yoann.congal=smile.fr@lists.openembedded.org> wrote: > + ], > + }, > + }, > + }, > + ], > + }, I don't know Yoann, these still need to be perfectly balanced and they're still visual clutter (admittedly not as bad as xml was), which is what yaml frees one from. What's the particular need for comments? You said someone else wanted them, what did they say? Alex
Le jeu. 2 oct. 2025 à 18:22, Alexander Kanavin <alex.kanavin@gmail.com> a écrit : > On Thu, 2 Oct 2025 at 17:52, Yoann Congal via lists.openembedded.org > <yoann.congal=smile.fr@lists.openembedded.org> wrote: > > > + ], > > + }, > > + }, > > + }, > > + ], > > + }, > > I don't know Yoann, these still need to be perfectly balanced and > they're still visual clutter (admittedly not as bad as xml was), which > is what yaml frees one from. > Yes, I agree with you on JSON visual clutter. I pushed JSON5 to not stray too far from the python-stdlib-supported default JSON. But, thinking about it, we could use the "install an extra module to read better but incompatible config format" idea to switch to YAML. Should I send another RFC patch to see what that looks like? Another idea I heard was to support a plugin mechanism in bb-setup to let everyone implement their own format but that sounds like a maintenance nightmare. > What's the particular need for comments? You said someone else wanted > them, what did they say? > They said nothing in particular but here is how I would use them (same need as comment in code) : * provide documentation (in the case the "description" field is not enough), * giving the intent of the author, * // TODO: ..., * "temporarily" commented code blocks, * a licence for the file, * a small tutorial on how to use the config (should be in the README next to it though) > > Alex >
On Thu, Oct 2, 2025 at 3:06 PM Yoann Congal via lists.openembedded.org <yoann.congal=smile.fr@lists.openembedded.org> wrote: > > > > Le jeu. 2 oct. 2025 à 18:22, Alexander Kanavin <alex.kanavin@gmail.com> a écrit : >> >> On Thu, 2 Oct 2025 at 17:52, Yoann Congal via lists.openembedded.org >> <yoann.congal=smile.fr@lists.openembedded.org> wrote: >> >> > + ], >> > + }, >> > + }, >> > + }, >> > + ], >> > + }, >> >> I don't know Yoann, these still need to be perfectly balanced and >> they're still visual clutter (admittedly not as bad as xml was), which >> is what yaml frees one from. > > > Yes, I agree with you on JSON visual clutter. > > I pushed JSON5 to not stray too far from the python-stdlib-supported default JSON. > > But, thinking about it, we could use the "install an extra module to read better but incompatible config format" idea to switch to YAML. Should I send another RFC patch to see what that looks like? If we are going with external dependencies, YAML makes more sense IMHO. YAML is a strict superset of JSON, so if you use a YAML parser it can parse either YAML or JSON documents, so all the existing documents would also parse just fine. That said, if the point is to make things interchangeable, it might be annoying to have some documents in YAML and some in JSON, so I'm not convinced this is a good idea on the face of it. > > Another idea I heard was to support a plugin mechanism in bb-setup to let everyone implement their own format but that sounds like a maintenance nightmare. > >> >> What's the particular need for comments? You said someone else wanted >> them, what did they say? > > > They said nothing in particular but here is how I would use them (same need as comment in code) : > * provide documentation (in the case the "description" field is not enough), > * giving the intent of the author, > * // TODO: ..., > * "temporarily" commented code blocks, > * a licence for the file, > * a small tutorial on how to use the config (should be in the README next to it though) > >> >> >> Alex > > > > -- > Yoann Congal > Smile ECS > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#18126): https://lists.openembedded.org/g/bitbake-devel/message/18126 > Mute This Topic: https://lists.openembedded.org/mt/115555001/3616693 > Group Owner: bitbake-devel+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [JPEWhacker@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- >
On Thu, 2025-10-02 at 17:52 +0200, Yoann Congal via lists.openembedded.org wrote: > From: Yoann Congal <yoann.congal@smile.fr> > > One major limitation of JSON files is that comments are not > supported. > JSON5 (https://json5.org/) is an extension of the current JSON file > format that support comments (among other evolutions). > > This patch adds the support of the JSON5 file format when reading > configuration files. > > Sadly, no JSON5 support exists in Python stdlib, so we have to > install a > module. I chose https://pypi.org/project/json5/ (but > https://pypi.org/project/pyjson5/ may be an alternative). > > I took care of only patching reading of the config file, the internal > JSON files of bitbake-setup are still in classic JSON, which mean > that > operations that don't need to read the original configuration still > work > without a json5 module. > > I put an example of a JSON5 config in > default-registry/configurations/poky-master_json5.conf.json > This is poky-master.conf.json translated to JSON5 plus some comments. > > To try this: > python3 -m venv .venv; . .venv/bin/activate # if needed > pip install json5 > ./bin/bitbake-setup init default-registry/configurations/poky- > master_json5.conf.json poky_json5 We could do something horrible like preprocess the json file and remove any lines starting with "//". That works for reading but not for writing. I'm keen to try and keep the external dependencies that bitbake has to a minimum. Why? People don't realise the issues they create for documentation, container environments, things like build-appliance and general ease of use. Yes, you can usually pip install things but it is an extra hurdle for users. The websockets requirement for remote hashequiv is an example of how an obvious/"trivial" dependency can work out to be a pain and a usability issue. Given that, I'd really prefer to stick to what the python standard library supports, which effectively means json at this point. FWIW I personally really dislike yaml files too, for no particularly good reason, I just don't get on with the format. Cheers, Richard
On Fri, Oct 3, 2025 at 3:35 AM Richard Purdie via lists.openembedded.org <richard.purdie=linuxfoundation.org@lists.openembedded.org> wrote: > > On Thu, 2025-10-02 at 17:52 +0200, Yoann Congal via lists.openembedded.org wrote: > > From: Yoann Congal <yoann.congal@smile.fr> > > > > One major limitation of JSON files is that comments are not > > supported. > > JSON5 (https://json5.org/) is an extension of the current JSON file > > format that support comments (among other evolutions). > > > > This patch adds the support of the JSON5 file format when reading > > configuration files. > > > > Sadly, no JSON5 support exists in Python stdlib, so we have to > > install a > > module. I chose https://pypi.org/project/json5/ (but > > https://pypi.org/project/pyjson5/ may be an alternative). > > > > I took care of only patching reading of the config file, the internal > > JSON files of bitbake-setup are still in classic JSON, which mean > > that > > operations that don't need to read the original configuration still > > work > > without a json5 module. > > > > I put an example of a JSON5 config in > > default-registry/configurations/poky-master_json5.conf.json > > This is poky-master.conf.json translated to JSON5 plus some comments. > > > > To try this: > > python3 -m venv .venv; . .venv/bin/activate # if needed > > pip install json5 > > ./bin/bitbake-setup init default-registry/configurations/poky- > > master_json5.conf.json poky_json5 > > We could do something horrible like preprocess the json file and remove > any lines starting with "//". That works for reading but not for > writing. Maybe just a "comment" property that gets ignored then? That's what JSONschema uses (although it does "@comment"), and then it doesn't violate any parsing? e.g. { "comment": "This is a comment", ... } > > I'm keen to try and keep the external dependencies that bitbake has to > a minimum. Why? People don't realise the issues they create for > documentation, container environments, things like build-appliance and > general ease of use. Yes, you can usually pip install things but it is > an extra hurdle for users. The websockets requirement for remote > hashequiv is an example of how an obvious/"trivial" dependency can work > out to be a pain and a usability issue. > > Given that, I'd really prefer to stick to what the python standard > library supports, which effectively means json at this point. > > FWIW I personally really dislike yaml files too, for no particularly > good reason, I just don't get on with the format. > > Cheers, > > Richard > > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#18128): https://lists.openembedded.org/g/bitbake-devel/message/18128 > Mute This Topic: https://lists.openembedded.org/mt/115555001/3616693 > Group Owner: bitbake-devel+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [JPEWhacker@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- >
diff --git a/bin/bitbake-setup b/bin/bitbake-setup index bb466d85..d1c54269 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -19,6 +19,14 @@ import datetime import glob import subprocess +try: + # Try to import the non-standard json5 module to load JSON files + from json5 import load as json5_load + json.load = json5_load + print("Using json5 to load JSON") +except ImportError: + pass + default_registry = os.path.normpath(os.path.dirname(__file__) + "/../default-registry") bindir = os.path.abspath(os.path.dirname(__file__)) diff --git a/default-registry/configurations/poky-master_json5.conf.json b/default-registry/configurations/poky-master_json5.conf.json new file mode 100644 index 00000000..cf447e9a --- /dev/null +++ b/default-registry/configurations/poky-master_json5.conf.json @@ -0,0 +1,93 @@ +{ + // This file format supports comments + description: "Poky - The Yocto Project testing distribution configurations and hardware test platforms", +// ^ ^ As long as the object keys are compatible with a JS variable name, quotes are optional: + // + sources: { + bitbake: { + "git-remote": { // note "git-remote" is not a valid JS variable name because of the dash + remotes: { + origin: { + uri: "git://git.openembedded.org/bitbake;protocol=https", + }, + }, + branch: "master", + rev: "master", // This file format accepts a comma after the last element of lists/objects + }, + path: "bitbake", + }, + "openembedded-core": { + "git-remote": { + remotes: { + origin: { + uri: "git://git.openembedded.org/openembedded-core;protocol=https", + }, + }, + branch: "master", + rev: "master", + }, + path: "openembedded-core", + }, + "meta-yocto": { + "git-remote": { + remotes: { + origin: { + uri: "git://git.yoctoproject.org/meta-yocto;protocol=https", + }, + }, + branch: "master", + rev: "master", + }, + path: "meta-yocto", + }, + "yocto-docs": { + "git-remote": { + remotes: { + origin: { + uri: "git://git.yoctoproject.org/yocto-docs;protocol=https", + }, + }, + branch: "master", + rev: "master", + }, + path: "yocto-docs", + }, + }, + "bitbake-setup": { + configurations: [ + { + name: "poky_json5", + description: "Poky - The Yocto Project testing distribution (from a JSON5 config)", + "bb-layers": [ + "openembedded-core/meta", + "meta-yocto/meta-yocto-bsp", + "meta-yocto/meta-poky", + ], + "oe-fragments": [ + "core/yocto/sstate-mirror-cdn", + ], + "oe-fragments-one-of": { + machine: { + description: "Target machines", + options: [ + "machine/qemux86-64", + "machine/qemuarm64", + "machine/qemuriscv64", + "machine/genericarm64", + "machine/genericx86-64", + ], + }, + distro: { + description: "Distribution configuration variants", + options: [ + "distro/poky", + "distro/poky-altcfg", + "distro/poky-tiny", + ], + }, + }, + }, + ], + }, + version: "1.0", +}