@@ -1827,10 +1827,37 @@ functionality of the task:
the :term:`BB_NUMBER_THREADS` variable causes ``number_threads`` to
have no effect.
-- ``[postfuncs]``: List of functions to call after the completion of
- the task.
+- ``[postfuncs]``: List of functions to call after the completion of the task.
+ Functions specified may set a ``[postfunc-order]`` flag to specify the order
+ in which they will execute. The ordering is based on the way that python
+ does list indexes. The first group of functions executed are those with a
+ positive or zero ``[postfunc-order]`` flag, in increasing order (i.e. "0"
+ is first, "1" is second, etc.). The next group executed are functions that
+ do not have a ``[postfunc-order]`` flag set, in the relative order they are
+ specified in ``[postfuncs]``. The last group are functions with a negative
+ ``[postfunc-order]`` flag, in increasing order (i.e. "-2" is the second to
+ last, "-1" is the last). If two functions have the same ``[postfunc-order]``
+ value, they will be grouped together, but keep the same relative order they
+ had in ``[postfuncs]``.
+
+- ``[postfunc-order]``: The order this function will be executed in when
+ listed in ``[postfuncs]``.
- ``[prefuncs]``: List of functions to call before the task executes.
+ Functions specified may set a ``[prefunc-order]`` flag to specify the order
+ in which they will execute. The ordering is based on the way that python
+ does list indexes. The first group of functions executed are those with a
+ positive or zero ``[prefunc-order]`` flag, in increasing order (i.e. "0" is
+ first, "1" is second, etc.). The next group executed are functions that do
+ not have a ``[prefunc-order]`` flag set, in the relative order they are
+ specified in ``[prefuncs]``. The last group are functions with a negative
+ ``[prefunc-order]`` flag, in increasing order (i.e. "-2" is the second to
+ last, "-1" is the last). If two functions have the same ``[prefunc-order]``
+ value, they will be grouped together, but keep the same relative order they
+ had in ``[prefuncs]``.
+
+- ``[prefunc-order]``: The order this function will be executed in when
+ listed in ``[prefuncs]``.
- ``[rdepends]``: Controls inter-task runtime dependencies. See the
:term:`RDEPENDS` variable, the
@@ -596,6 +596,54 @@ def _task_data(fn, task, d):
bb.data.expandKeys(localdata)
return localdata
+
+def _sort_funcs(d, funcs, key):
+ """
+ Sorts pre/post functions based on a priority flag of the function. The
+ function can define its order using one of three buckets:
+ 1) A positive or zero priority
+ 2) No priority
+ 3) A negative priority
+
+ When sorting the functions, they are sorted in the following order:
+ 1) Functions with positive or zero priority in ascending order
+ 2) Functions with no priority in the order they were defined (e.g.
+ their relative priority is preserved)
+ 3) Functions with negative priority, in ascending order
+
+ Practically speaking, this implements a similar sorting scheme to python
+ list indexing. "0" will always be the first function exeucted, "1" will be
+ the second (assuming there is a "0"), "-1" will always be the last function
+ executed, and "-2" will be the second to last (assuming there is a "-1").
+
+ Functions with no priority are preserved in their defined order, which keeps
+ this algorithm compatible with simply defining to execution order based on
+ the order in the variable.
+ """
+ # MAX_ORDER_VALUE is arbitrary; it just limits the maximum order priority
+ # that can be assigned to any function
+ MAX_ORDER_VALUE = 10000
+
+ def get_func_key(f):
+ v = d.getVarFlag(f, key, False)
+ if v is None or v == "":
+ return MAX_ORDER_VALUE
+
+ try:
+ v = int(v)
+ except ValueError as e:
+ bb.fatal(f"Invalid value for {prefix}-order varflag on function {f}: {e}")
+
+ if v >= MAX_ORDER_VALUE or v <= -MAX_ORDER_VALUE:
+ bb.fatal(f"Invalid value for {prefix}-order varflag on function {f}: {v} is out of range")
+
+ if v < 0:
+ return (2 * MAX_ORDER_VALUE) + v
+ return v
+
+ return sorted(funcs, key=get_func_key)
+
+
def _exec_task(fn, task, d, quieterr):
"""Execute a BB 'task'
@@ -708,10 +756,10 @@ def _exec_task(fn, task, d, quieterr):
try:
event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata)
- for func in (prefuncs or '').split():
+ for func in _sort_funcs(localdata, (prefuncs or '').split(), "prefunc-order"):
exec_func(func, localdata)
exec_func(task, localdata)
- for func in (postfuncs or '').split():
+ for func in _sort_funcs(localdata, (postfuncs or '').split(), "postfunc-order"):
exec_func(func, localdata)
finally:
# Need to flush and close the logs before sending events where the
new file mode 100644
@@ -0,0 +1,44 @@
+TASKS = ""
+
+python do_build:append() {
+ d.appendVar("TASKS", " build")
+}
+
+python func_1() {
+ d.appendVar("TASKS", " 1")
+}
+
+python func_2() {
+ d.appendVar("TASKS", " 2")
+}
+
+python func_3() {
+ d.appendVar("TASKS", " 3")
+}
+
+python func_4() {
+ d.appendVar("TASKS", " 4")
+}
+
+python func_5() {
+ d.appendVar("TASKS", " 5")
+}
+
+python func_6() {
+ d.appendVar("TASKS", " 6")
+}
+
+python func_7() {
+ d.appendVar("TASKS", " 7")
+}
+
+python func_8() {
+ d.appendVar("TASKS", " 8")
+}
+
+python write_log() {
+ with open(d.getVar("ORDER_LOG"), "w") as f:
+ f.write(d.getVar("TASKS"))
+}
+write_log[postfunc-order] = "-1"
+do_build[postfuncs] += "write_log"
new file mode 100644
@@ -0,0 +1,10 @@
+require funcs.inc
+
+do_build[postfuncs] += "func_8 func_6 func_7 func_4 func_5 func_2 func_3 func_1"
+
+func_1[postfunc-order] = "0"
+func_2[postfunc-order] = "1"
+func_3[postfunc-order] = "1"
+func_6[postfunc-order] = "-3"
+func_7[postfunc-order] = "-3"
+func_8[postfunc-order] = "-2"
new file mode 100644
@@ -0,0 +1,10 @@
+require funcs.inc
+
+do_build[prefuncs] += "func_8 func_6 func_7 func_4 func_5 func_2 func_3 func_1"
+
+func_1[prefunc-order] = "0"
+func_2[prefunc-order] = "1"
+func_3[prefunc-order] = "1"
+func_6[prefunc-order] = "-3"
+func_7[prefunc-order] = "-3"
+func_8[prefunc-order] = "-2"
@@ -12,6 +12,7 @@ import tempfile
import subprocess
import sys
import time
+from contextlib import contextmanager
#
# TODO:
@@ -53,6 +54,14 @@ class RunQueueTests(unittest.TestCase):
os.remove(tasklog)
return tasks
+ @contextmanager
+ def tempdir(self):
+ with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+ try:
+ yield tempdir
+ finally:
+ self.shutdown(tempdir)
+
def test_no_setscenevalid(self):
with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
cmd = ["bitbake", "a1"]
@@ -403,8 +412,23 @@ class RunQueueTests(unittest.TestCase):
self.shutdown(tempdir)
+ def test_postfuncs(self):
+ with self.tempdir() as tempdir:
+ log = f"{tempdir}/postlog.txt"
+ tasks = self.run_bitbakecmd(["bitbake", "postfuncs"], tempdir, extraenv={"ORDER_LOG": log})
+ self.assertTrue(os.path.exists(log), "Log file not written")
+ with open(log, "r") as f:
+ self.assertEqual(f.read(), " build 1 2 3 4 5 6 7 8")
+
+ def test_prefuncs(self):
+ with self.tempdir() as tempdir:
+ log = f"{tempdir}/prelog.txt"
+ tasks = self.run_bitbakecmd(["bitbake", "prefuncs"], tempdir, extraenv={"ORDER_LOG": log})
+ self.assertTrue(os.path.exists(log), "Log file not written")
+ with open(log, "r") as f:
+ self.assertEqual(f.read(), " 1 2 3 4 5 6 7 8 build")
+
def shutdown(self, tempdir):
# Wait for the hashserve socket to disappear else we'll see races with the tempdir cleanup
while (os.path.exists(tempdir + "/hashserve.sock") or os.path.exists(tempdir + "cache/hashserv.db-wal") or os.path.exists(tempdir + "/bitbake.lock")):
time.sleep(0.5)
-
Adds the ability for functions specified in prefuncs and postfuncs to specify the order in which they must execute. This allows specific functions to indicate that they must run first or last (or second, second to last, etc.) without needing all places in the code that manipulate the function list to have to agree on how to order them (a good example of this is sstate.bbclass and buildhistory.bbclass). Ordering is done similar to how python list indices work; 0 is always sorted first, 1 is second, -2 is second to last, -1 is last, etc. In the event of functions having duplicate values, they are grouped together, but keep their same relative ordering. If one or more functions do not have an order assigned, they executed between the positive order functions and the negative order functions, and keep their same relative order (this means that existing behavior is preserved in the absence of any ordering information). Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> --- .../bitbake-user-manual-metadata.rst | 31 ++++++++++- lib/bb/build.py | 52 ++++++++++++++++++- lib/bb/tests/runqueue-tests/recipes/funcs.inc | 44 ++++++++++++++++ .../tests/runqueue-tests/recipes/postfuncs.bb | 10 ++++ .../tests/runqueue-tests/recipes/prefuncs.bb | 10 ++++ lib/bb/tests/runqueue.py | 26 +++++++++- 6 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 lib/bb/tests/runqueue-tests/recipes/funcs.inc create mode 100644 lib/bb/tests/runqueue-tests/recipes/postfuncs.bb create mode 100644 lib/bb/tests/runqueue-tests/recipes/prefuncs.bb