diff mbox series

[v2] parse: Add support for addpylib conf file directive

Message ID 20221207123829.2106891-1-richard.purdie@linuxfoundation.org
State New
Headers show
Series [v2] parse: Add support for addpylib conf file directive | expand

Commit Message

Richard Purdie Dec. 7, 2022, 12:38 p.m. UTC
For many years OE-Core has injected it's own python modules into the
python namespace using an immediate expansion of a variable in base.bbclass.

We really need this to become a first class citizen of the langauge, this new
addpylib directive allows that. Usage is of the form:

addpylib <directory> <namespace>

The namespace is imported and if there is an attribute BBIMPORT, that
list of names is iterated and imported too.

This mirrors what OE-Core has done for a long time with one difference in
implmentation, sys.path is only appended to. This means later imported
namespaces can't overwrite an earlier one and can't overwrite the main
python module space. In practice we've never done that and it isn't
something we should encourage or support.

The new directive is only applied for .conf files and not other filetypes
as it only makes sense in that context. It is also only allowed in the
"base configuration" context of cookerdata since adding it at the recipe
level wouldn't work properly due to the way it changes the global namespace.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

v2: Limit the directive to conf files in the base configuration

 lib/bb/cookerdata.py                 |  2 +-
 lib/bb/parse/__init__.py             |  4 ++--
 lib/bb/parse/ast.py                  | 22 ++++++++++++++++++++++
 lib/bb/parse/parse_py/BBHandler.py   |  4 ++--
 lib/bb/parse/parse_py/ConfHandler.py | 14 +++++++++++---
 5 files changed, 38 insertions(+), 8 deletions(-)
diff mbox series


diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py
index c322ab2ffb..28cb59d83a 100644
--- a/lib/bb/cookerdata.py
+++ b/lib/bb/cookerdata.py
@@ -184,7 +184,7 @@  def catch_parse_error(func):
 def parse_config_file(fn, data, include=True):
-    return bb.parse.handle(fn, data, include)
+    return bb.parse.handle(fn, data, include, baseconfig=True)
 def _inherit(bbclass, data):
diff --git a/lib/bb/parse/__init__.py b/lib/bb/parse/__init__.py
index 347609513b..4cd82f115b 100644
--- a/lib/bb/parse/__init__.py
+++ b/lib/bb/parse/__init__.py
@@ -99,12 +99,12 @@  def supports(fn, data):
             return 1
     return 0
-def handle(fn, data, include = 0):
+def handle(fn, data, include=0, baseconfig=False):
     """Call the handler that is appropriate for this file"""
     for h in handlers:
         if h['supports'](fn, data):
             with data.inchistory.include(fn):
-                return h['handle'](fn, data, include)
+                return h['handle'](fn, data, include, baseconfig)
     raise ParseError("not a BitBake file", fn)
 def init(fn, data):
diff --git a/lib/bb/parse/ast.py b/lib/bb/parse/ast.py
index 9e0a0f5c98..7bfee9ed67 100644
--- a/lib/bb/parse/ast.py
+++ b/lib/bb/parse/ast.py
@@ -9,6 +9,7 @@ 
 # SPDX-License-Identifier: GPL-2.0-only
+import sys
 import bb
 from bb import methodpool
 from bb.parse import logger
@@ -269,6 +270,24 @@  class BBHandlerNode(AstNode):
             data.setVarFlag(h, "handler", 1)
         data.setVar('__BBHANDLERS', bbhands)
+class PyLibNode(AstNode):
+    def __init__(self, filename, lineno, libdir, namespace):
+        AstNode.__init__(self, filename, lineno)
+        self.libdir = libdir
+        self.namespace = namespace
+    def eval(self, data):
+        libdir = data.expand(self.libdir)
+        if libdir not in sys.path:
+            sys.path.append(libdir)
+        try:
+            bb.utils._context[self.namespace] = __import__(self.namespace)
+            toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", [])
+            for i in toimport:
+                bb.utils._context[i.split(".", 1)[0]] = __import__(i)
+        except AttributeError as e:
+            bb.error("Error importing OE modules: %s" % str(e))
 class InheritNode(AstNode):
     def __init__(self, filename, lineno, classes):
         AstNode.__init__(self, filename, lineno)
@@ -320,6 +339,9 @@  def handleDelTask(statements, filename, lineno, m):
 def handleBBHandlers(statements, filename, lineno, m):
     statements.append(BBHandlerNode(filename, lineno, m.group(1)))
+def handlePyLib(statements, filename, lineno, m):
+    statements.append(PyLibNode(filename, lineno, m.group(1), m.group(2)))
 def handleInherit(statements, filename, lineno, m):
     classes = m.group(1)
     statements.append(InheritNode(filename, lineno, classes))
diff --git a/lib/bb/parse/parse_py/BBHandler.py b/lib/bb/parse/parse_py/BBHandler.py
index 584966fea1..4d5b45e1ef 100644
--- a/lib/bb/parse/parse_py/BBHandler.py
+++ b/lib/bb/parse/parse_py/BBHandler.py
@@ -101,7 +101,7 @@  def get_statements(filename, absolute_filename, base_name):
             cached_statements[absolute_filename] = statements
         return statements
-def handle(fn, d, include):
+def handle(fn, d, include, baseconfig=False):
     global __infunc__, __body__, __residue__, __classname__
     __body__ = []
     __infunc__ = []
@@ -265,7 +265,7 @@  def feeder(lineno, s, fn, root, statements, eof=False):
         ast.handleInherit(statements, fn, lineno, m)
-    return ConfHandler.feeder(lineno, s, fn, statements)
+    return ConfHandler.feeder(lineno, s, fn, statements, conffile=False)
 # Add us to the handlers list
 from .. import handlers
diff --git a/lib/bb/parse/parse_py/ConfHandler.py b/lib/bb/parse/parse_py/ConfHandler.py
index 451e68dd66..3076067287 100644
--- a/lib/bb/parse/parse_py/ConfHandler.py
+++ b/lib/bb/parse/parse_py/ConfHandler.py
@@ -46,6 +46,7 @@  __require_regexp__ = re.compile( r"require\s+(.+)" )
 __export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
 __unset_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
 __unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.]+)\]$" )
+__addpylib_regexp__      = re.compile(r"addpylib\s+(.+)\s+(.+)" )
 def init(data):
@@ -107,7 +108,7 @@  def include_single_file(parentfn, fn, lineno, data, error_out):
 # parsing. This turns out to be a hard problem to solve any other way.
 confFilters = []
-def handle(fn, data, include):
+def handle(fn, data, include, baseconfig=False):
     if include == 0:
@@ -144,7 +145,7 @@  def handle(fn, data, include):
             # skip comments
             if s[0] == '#':
-            feeder(lineno, s, abs_fn, statements)
+            feeder(lineno, s, abs_fn, statements, baseconfig=baseconfig)
     # DONE WITH PARSING... time to evaluate
     data.setVar('FILE', abs_fn)
@@ -157,7 +158,9 @@  def handle(fn, data, include):
     return data
-def feeder(lineno, s, fn, statements):
+# baseconfig is set for the bblayers/layer.conf cookerdata config parsing
+# The function is also used by BBHandler, conffile would be False
+def feeder(lineno, s, fn, statements, baseconfig=False, conffile=True):
     m = __config_regexp__.match(s)
     if m:
         groupd = m.groupdict()
@@ -189,6 +192,11 @@  def feeder(lineno, s, fn, statements):
         ast.handleUnsetFlag(statements, fn, lineno, m)
+    m = __addpylib_regexp__.match(s)
+    if baseconfig and conffile and m:
+        ast.handlePyLib(statements, fn, lineno, m)
+        return
     raise ParseError("unparsed line: '%s'" % s, fn, lineno);
 # Add us to the handlers list