diff mbox series

[1/2] bitbake-setup: support multiple source override files

Message ID 20260529-bb-setup-override-improvements-v1-1-91db62e0149e@toradex.com
State New
Headers show
Series bitbake-setup: improve source-overrides | expand

Commit Message

Ernest Van Hoecke May 29, 2026, 2:51 p.m. UTC
From: Ernest Van Hoecke <ernest.vanhoecke@toradex.com>

The bitbake-setup documentation currently states that:
"The --source-overrides option can be passed multiple times, in which
case the overrides are applied in the order specified in the
command-line."

However, this is not currently true and only the last --source-overrides
argument is actually used.

Allow --source-overrides to be specified more than once and follow the
behaviour set out in the documentation, i.e.: merge override files in
command-line order.

Extend the setup test to exercise two override files through init,
checking that both are applied and that they are applied in order.

Signed-off-by: Ernest Van Hoecke <ernest.vanhoecke@toradex.com>
---
 bin/bitbake-setup     | 17 +++++++-------
 lib/bb/tests/setup.py | 61 ++++++++++++++++++++++++++++++++++-----------------
 2 files changed, 50 insertions(+), 28 deletions(-)

Comments

Alexander Kanavin May 29, 2026, 4:46 p.m. UTC | #1
On Fri, 29 May 2026 at 16:52, Ernest Van Hoecke via
lists.openembedded.org
<ernestvanhoecke=gmail.com@lists.openembedded.org> wrote:
> The bitbake-setup documentation currently states that:
> "The --source-overrides option can be passed multiple times, in which
> case the overrides are applied in the order specified in the
> command-line."
>
> However, this is not currently true and only the last --source-overrides
> argument is actually used.

To be honest, I'd rather fix the documentation, and avoid increasing
code complexity in the lack of clear use case. It's always possible to
merge multiple jsons into one with a simple python script. This would
also keep things consistent with the 'multiple sets of sources in
json' idea, where you'd only pick one of them.

Alex
diff mbox series

Patch

diff --git a/bin/bitbake-setup b/bin/bitbake-setup
index 220540f7f9ca..2e234cccaba9 100755
--- a/bin/bitbake-setup
+++ b/bin/bitbake-setup
@@ -636,21 +636,22 @@  def obtain_config(top_dir, registry, args, source_overrides, d):
     return upstream_config
 
 def obtain_overrides(args):
-    overrides = {'sources':{}}
-    if args.source_overrides:
-        overrides = json.load(open(args.source_overrides))
-        overrides_dir = os.path.dirname(os.path.abspath(args.source_overrides))
+    all_overrides = {'sources':{}}
+    for overrides_file in args.source_overrides or []:
+        overrides = json.load(open(overrides_file))
+        overrides_dir = os.path.dirname(os.path.abspath(overrides_file))
         for s,v in overrides['sources'].items():
             local = v.get('local')
             if local:
                 path = os.path.expanduser(local['path'])
                 if not os.path.isabs(path):
-                    overrides['sources'][s]['local']['path'] = os.path.join(overrides_dir, path)
+                    v['local']['path'] = os.path.join(overrides_dir, path)
+            all_overrides['sources'][s] = v
 
     for local_name, local_path in args.use_local_source:
-        overrides['sources'][local_name] = {'local':{'path':os.path.abspath(os.path.expanduser(local_path))}}
+        all_overrides['sources'][local_name] = {'local':{'path':os.path.abspath(os.path.expanduser(local_path))}}
 
-    return overrides
+    return all_overrides
 
 def configure_vscode(setupdir, layerdir, builddir, init_script):
     """
@@ -1257,7 +1258,7 @@  def main():
     parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a setup from it')
     parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.")
     parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.')
-    parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.')
+    parser_init.add_argument('--source-overrides', action='append', help='Override sources information (repositories/revisions) with values from a local json file.')
     parser_init.add_argument('--setup-dir-name', action='store', help='A custom setup directory name under the top directory.')
     parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.')
     parser_init.add_argument('-L', '--use-local-source', default=[], action='append', nargs=2, metavar=('SOURCE_NAME', 'PATH'),
diff --git a/lib/bb/tests/setup.py b/lib/bb/tests/setup.py
index 638d56d3bb32..d425faa1f6b6 100644
--- a/lib/bb/tests/setup.py
+++ b/lib/bb/tests/setup.py
@@ -180,21 +180,23 @@  print("BBPATH is {{}}".format(os.environ["BBPATH"]))
         self.git('commit -m "Adding {}"'.format(name), cwd=self.registrypath)
         return json.loads(config)
 
-    def add_json_config_to_registry(self, name, rev, branch):
-        sources = """
-        "test-repo": {
-            "git-remote": {
-                "remotes": {
-                    "origin": {
-                        "uri": "file://%s"
-                    }
-                },
-                "branch": "%s",
-                "rev": "%s"
+    def add_json_config_to_registry(self, name, rev, branch, source_names=("test-repo",)):
+        sources = []
+        for source_name in source_names:
+            sources.append("""
+            "%s": {
+                "git-remote": {
+                    "remotes": {
+                        "origin": {
+                            "uri": "file://%s"
+                        }
+                    },
+                    "branch": "%s",
+                    "rev": "%s"
+                }
             }
-        }
-""" % (self.testrepopath, branch, rev)
-        return self._add_json_config_to_registry_helper(name, sources)
+""" % (source_name, self.testrepopath, branch, rev))
+        return self._add_json_config_to_registry_helper(name, ",\n".join(sources))
 
     def add_local_json_config_to_registry(self, name, path):
         sources = """
@@ -442,11 +444,12 @@  print("BBPATH is {{}}".format(os.environ["BBPATH"]))
             sums_after = _conf_chksum(f"{setuppath}/build/conf")
             self.assertEqual(sums_before, sums_after)
 
-        def _check_local_sources(custom_setup_dir):
+        def _check_local_sources(custom_setup_dir, sources=("test-repo",)):
             custom_setup_path = os.path.join(self.tempdir, 'bitbake-builds', custom_setup_dir)
-            custom_layer_path = os.path.join(custom_setup_path, 'layers', 'test-repo')
-            self.assertTrue(os.path.islink(custom_layer_path))
-            self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path))
+            for source in sources:
+                custom_layer_path = os.path.join(custom_setup_path, 'layers', source)
+                self.assertTrue(os.path.islink(custom_layer_path))
+                self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path))
             self.config_is_unchanged(custom_setup_path)
 
         def _check_layer_backups(layer_path, expected_backups):
@@ -525,14 +528,32 @@  print("BBPATH is {{}}".format(os.environ["BBPATH"]))
             "local": {
                 "path": "."
             }
+        },
+        "test-repo-extra": {
+            "local": {
+                "path": "/does/not/exist"
+            }
+        }
+    }
+}"""
+        second_source_override_content = """
+{
+    "sources": {
+        "test-repo-extra": {
+            "local": {
+                "path": "."
+            }
         }
     }
 }"""
         override_filename = 'source-overrides.json'
+        second_override_filename = 'second-source-overrides.json'
         custom_setup_dir = 'special-setup-dir'
+        self.add_json_config_to_registry('test-config-1.conf.json', branch, branch, ("test-repo", "test-repo-extra"))
         self.add_file_to_testrepo(override_filename, source_override_content)
-        out = self.runbbsetup("init --non-interactive --source-overrides {} --setup-dir-name {} test-config-1 gadget".format(os.path.join(self.testrepopath, override_filename), custom_setup_dir))
-        _check_local_sources(custom_setup_dir)
+        self.add_file_to_testrepo(second_override_filename, second_source_override_content)
+        out = self.runbbsetup("init --non-interactive --source-overrides {} --source-overrides {} --setup-dir-name {} test-config-1 gadget".format(os.path.join(self.testrepopath, override_filename), os.path.join(self.testrepopath, second_override_filename), custom_setup_dir))
+        _check_local_sources(custom_setup_dir, ("test-repo", "test-repo-extra"))
 
         # same but use command line options to specify local overrides
         custom_setup_dir = 'special-setup-dir-with-cmdline-overrides'