diff mbox series

[yocto-autobuilder2] docs: fetch yocto-docs and bitbake using *shared-repos scripts

Message ID 20250103-docs-build-shared-repos-v1-1-7b5705d895a1@bootlin.com
State New
Headers show
Series [yocto-autobuilder2] docs: fetch yocto-docs and bitbake using *shared-repos scripts | expand

Commit Message

Antonin Godard Jan. 3, 2025, 3:24 p.m. UTC 
The docs build was one of the only builder not using the shared-repos
scripts. This prevented some of the repository properties to be
overriden, such as the bitbake/yocto-docs repository URI. Overriding the
bitbake URI is especially useful when using bitbake-contrib instead of
bitbake, to test changes.

Leave the yocto-autobuilder-helper in the codebases (like in the other
builders), but move the yocto-docs/bitbake repositories to NEEDREPOS in
yocto-autobuilder-helper/config.json.

This patch depends on the patch "config.json: add docs build config" on
yocto-autobuilder-helper.

Signed-off-by: Antonin Godard <antonin.godard@bootlin.com>
---
 builders.py   | 36 ++++++++++++++++++++++--------------
 config.py     |  2 +-
 schedulers.py | 11 +----------
 3 files changed, 24 insertions(+), 25 deletions(-)


---
base-commit: adb36c2213d331fd4a4fdcb5f48969b37bf1feb2
change-id: 20250103-docs-build-shared-repos-3cf38b589d73

Best regards,

Comments

Richard Purdie Jan. 6, 2025, 11:53 a.m. UTC | #1 
On Fri, 2025-01-03 at 16:24 +0100, Antonin Godard via lists.yoctoproject.org wrote:
> The docs build was one of the only builder not using the shared-repos
> scripts. This prevented some of the repository properties to be
> overriden, such as the bitbake/yocto-docs repository URI. Overriding the
> bitbake URI is especially useful when using bitbake-contrib instead of
> bitbake, to test changes.
> 
> Leave the yocto-autobuilder-helper in the codebases (like in the other
> builders), but move the yocto-docs/bitbake repositories to NEEDREPOS in
> yocto-autobuilder-helper/config.json.
> 
> This patch depends on the patch "config.json: add docs build config" on
> yocto-autobuilder-helper.
> 
> Signed-off-by: Antonin Godard <antonin.godard@bootlin.com>
> ---
>  builders.py   | 36 ++++++++++++++++++++++--------------
>  config.py     |  2 +-
>  schedulers.py | 11 +----------
>  3 files changed, 24 insertions(+), 25 deletions(-)
> 
> diff --git a/builders.py b/builders.py
> index 88ff328c7ec6ecf0f28da2693cc9c0210e760b35..e19fe5a138b18deec776f19c6b0af507e3021c89 100644
> --- a/builders.py
> +++ b/builders.py
> @@ -389,22 +389,26 @@ def create_doc_builder_factory():
>          mode='incremental',
>          haltOnFailure=True,
>          name='Fetch yocto-autobuilder-helper'))
> -    f.addStep(steps.Git(
> -        repourl=config.repos["yocto-docs"][0],
> -        branch=config.repos["yocto-docs"][1],
> -        codebase='yocto-docs',
> -        workdir=util.Interpolate("%(prop:builddir)s/yocto-docs"),
> -        mode='incremental',
> +    f.addStep(WriteLayerInfo(name='Write main layerinfo.json', haltOnFailure=True))
> +    f.addStep(steps.ShellCommand(
> +        command=[
> +            util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/prepare-shared-repos"),
> +            util.Interpolate("%(prop:builddir)s/layerinfo.json"),
> +            util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
> +            ],
>          haltOnFailure=True,
> -        name='Fetch yocto-docs'))
> -    f.addStep(steps.Git(
> -        repourl=config.repos["bitbake"][0],
> -        branch=config.repos["bitbake"][1],
> -        codebase='bitbake',
> -        workdir=util.Interpolate("%(prop:builddir)s/bitbake"),
> -        mode='incremental',
> +        name="Prepare shared repositories"))
> +    f.addStep(steps.ShellCommand(
> +        command=[
> +            util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/shared-repo-unpack"),
> +            util.Interpolate("%(prop:builddir)s/layerinfo.json"),
> +            util.Interpolate("%(prop:builddir)s"),
> +            util.Property("buildername"),
> +            "-c", util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
> +            "--workername", util.Interpolate("%(prop:workername)s"),
> +            ],
>          haltOnFailure=True,
> -        name='Fetch bitbake'))
> +        name="Unpack shared repositories"))
>      f.addStep(steps.ShellCommand(
>          command=[util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-docs-build"),
>                   util.Interpolate("%(prop:builddir)s"),
> @@ -412,6 +416,10 @@ def create_doc_builder_factory():
>                   util.Interpolate("%(prop:builddir)s/bitbake")],
>          haltOnFailure=True,
>          name="Run documentation Build"))
> +    f.addStep(steps.ShellCommand(
> +                    command=["rm", "-fr", util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir))],
> +                    haltOnFailure=True,
> +                    name="Remove shared repo dir"))
>      return f
>  


I'm torn on this. We create the shared repo dir as there are multiple
builders running one target build and we want them all to match. The
docs build doesn't have this constraint so it doesn't really need to
create a shared repo.

The downsides to that are bypassing the cache mechanism and it seems
missing out on some of the repo config too. I wonder if we should
create a second version of this script which just creates a specific
checkout?

Cheers,

Richard
Richard Purdie Jan. 6, 2025, 1:07 p.m. UTC | #2 
On Fri, 2025-01-03 at 16:24 +0100, Antonin Godard via lists.yoctoproject.org wrote:
> The docs build was one of the only builder not using the shared-repos
> scripts. This prevented some of the repository properties to be
> overriden, such as the bitbake/yocto-docs repository URI. Overriding the
> bitbake URI is especially useful when using bitbake-contrib instead of
> bitbake, to test changes.
> 
> Leave the yocto-autobuilder-helper in the codebases (like in the other
> builders), but move the yocto-docs/bitbake repositories to NEEDREPOS in
> yocto-autobuilder-helper/config.json.
> 
> This patch depends on the patch "config.json: add docs build config" on
> yocto-autobuilder-helper.
> 
> Signed-off-by: Antonin Godard <antonin.godard@bootlin.com>
> ---
>  builders.py   | 36 ++++++++++++++++++++++--------------
>  config.py     |  2 +-
>  schedulers.py | 11 +----------
>  3 files changed, 24 insertions(+), 25 deletions(-)
> 
> diff --git a/builders.py b/builders.py
> index 88ff328c7ec6ecf0f28da2693cc9c0210e760b35..e19fe5a138b18deec776f19c6b0af507e3021c89 100644
> --- a/builders.py
> +++ b/builders.py
> @@ -389,22 +389,26 @@ def create_doc_builder_factory():
>          mode='incremental',
>          haltOnFailure=True,
>          name='Fetch yocto-autobuilder-helper'))
> -    f.addStep(steps.Git(
> -        repourl=config.repos["yocto-docs"][0],

Since I looked at this a bit more, I think this should be:

self.getProperty("repo_yocto-docs") instead of config.repos["yocto-docs"][0]

(see steps/writelayerinfo.py)

> -        branch=config.repos["yocto-docs"][1],

similarly:

self.getProperty("branch_yocto-docs")


then we'd add properties=parent_default_props('docs') to:

schedulers.append(sched.AnyBranchScheduler(name="yocto-docs-changed"

Cheers,

Richard
Antonin Godard Jan. 6, 2025, 1:30 p.m. UTC | #3 
Hi Richard,

On Mon Jan 6, 2025 at 2:07 PM CET, Richard Purdie wrote:
> On Fri, 2025-01-03 at 16:24 +0100, Antonin Godard via lists.yoctoproject.org wrote:
>> The docs build was one of the only builder not using the shared-repos
>> scripts. This prevented some of the repository properties to be
>> overriden, such as the bitbake/yocto-docs repository URI. Overriding the
>> bitbake URI is especially useful when using bitbake-contrib instead of
>> bitbake, to test changes.
>> 
>> Leave the yocto-autobuilder-helper in the codebases (like in the other
>> builders), but move the yocto-docs/bitbake repositories to NEEDREPOS in
>> yocto-autobuilder-helper/config.json.
>> 
>> This patch depends on the patch "config.json: add docs build config" on
>> yocto-autobuilder-helper.
>> 
>> Signed-off-by: Antonin Godard <antonin.godard@bootlin.com>
>> ---
>>  builders.py   | 36 ++++++++++++++++++++++--------------
>>  config.py     |  2 +-
>>  schedulers.py | 11 +----------
>>  3 files changed, 24 insertions(+), 25 deletions(-)
>> 
>> diff --git a/builders.py b/builders.py
>> index 88ff328c7ec6ecf0f28da2693cc9c0210e760b35..e19fe5a138b18deec776f19c6b0af507e3021c89 100644
>> --- a/builders.py
>> +++ b/builders.py
>> @@ -389,22 +389,26 @@ def create_doc_builder_factory():
>>          mode='incremental',
>>          haltOnFailure=True,
>>          name='Fetch yocto-autobuilder-helper'))
>> -    f.addStep(steps.Git(
>> -        repourl=config.repos["yocto-docs"][0],
>
> Since I looked at this a bit more, I think this should be:
>
> self.getProperty("repo_yocto-docs") instead of config.repos["yocto-docs"][0]
>
> (see steps/writelayerinfo.py)
>
>> -        branch=config.repos["yocto-docs"][1],
>
> similarly:
>
> self.getProperty("branch_yocto-docs")
>
>
> then we'd add properties=parent_default_props('docs') to:
>
> schedulers.append(sched.AnyBranchScheduler(name="yocto-docs-changed"

Thanks, this looks easier than using the scripts, I'll try this instead.

Antonin
diff mbox series

Patch

diff --git a/builders.py b/builders.py
index 88ff328c7ec6ecf0f28da2693cc9c0210e760b35..e19fe5a138b18deec776f19c6b0af507e3021c89 100644
--- a/builders.py
+++ b/builders.py
@@ -389,22 +389,26 @@  def create_doc_builder_factory():
         mode='incremental',
         haltOnFailure=True,
         name='Fetch yocto-autobuilder-helper'))
-    f.addStep(steps.Git(
-        repourl=config.repos["yocto-docs"][0],
-        branch=config.repos["yocto-docs"][1],
-        codebase='yocto-docs',
-        workdir=util.Interpolate("%(prop:builddir)s/yocto-docs"),
-        mode='incremental',
+    f.addStep(WriteLayerInfo(name='Write main layerinfo.json', haltOnFailure=True))
+    f.addStep(steps.ShellCommand(
+        command=[
+            util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/prepare-shared-repos"),
+            util.Interpolate("%(prop:builddir)s/layerinfo.json"),
+            util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
+            ],
         haltOnFailure=True,
-        name='Fetch yocto-docs'))
-    f.addStep(steps.Git(
-        repourl=config.repos["bitbake"][0],
-        branch=config.repos["bitbake"][1],
-        codebase='bitbake',
-        workdir=util.Interpolate("%(prop:builddir)s/bitbake"),
-        mode='incremental',
+        name="Prepare shared repositories"))
+    f.addStep(steps.ShellCommand(
+        command=[
+            util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/shared-repo-unpack"),
+            util.Interpolate("%(prop:builddir)s/layerinfo.json"),
+            util.Interpolate("%(prop:builddir)s"),
+            util.Property("buildername"),
+            "-c", util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
+            "--workername", util.Interpolate("%(prop:workername)s"),
+            ],
         haltOnFailure=True,
-        name='Fetch bitbake'))
+        name="Unpack shared repositories"))
     f.addStep(steps.ShellCommand(
         command=[util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-docs-build"),
                  util.Interpolate("%(prop:builddir)s"),
@@ -412,6 +416,10 @@  def create_doc_builder_factory():
                  util.Interpolate("%(prop:builddir)s/bitbake")],
         haltOnFailure=True,
         name="Run documentation Build"))
+    f.addStep(steps.ShellCommand(
+                    command=["rm", "-fr", util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir))],
+                    haltOnFailure=True,
+                    name="Remove shared repo dir"))
     return f
 
 # Only run one docs build at a time
diff --git a/config.py b/config.py
index 94d0405803fb9c916e06518f12d257e978ffc46e..d9203f79006631678f92b2a3991cd0c598116de0 100644
--- a/config.py
+++ b/config.py
@@ -27,7 +27,7 @@  buildertorepos = {
     "checkuri": ["poky"],
     "check-layer": ["poky", "meta-mingw", "meta-gplv2"],
     "check-layer-nightly": ["poky", "meta-agl", "meta-arm", "meta-aws", "meta-intel", "meta-openembedded", "meta-virtualization", "meta-ti", "meta-security", "meta-clang", "meta-exein"],
-    "docs": [],
+    "docs": ["yocto-docs", "bitbake"],
     "reproducible-meta-oe": ["poky", "meta-openembedded"],
     "patchtest": ["poky", "meta-patchtest", "meta-openembedded"],
     "meta-oe-mirror":  ["poky", "meta-openembedded"],
diff --git a/schedulers.py b/schedulers.py
index bbd249a3aec45f60faec6c7f67056050794866a3..15d7948dcf275432a1986ceeee551dec7f6de61b 100644
--- a/schedulers.py
+++ b/schedulers.py
@@ -498,16 +498,7 @@  schedulers.append(parent_scheduler("a-full"))
 schedulers.append(sched.ForceScheduler(
         name="docs",
         builderNames=["docs"],
-        codebases=[util.CodebaseParameter(codebase='yab-helper',
-                                          label="yocto-autobuilder-helper:",
-                                          project=None),
-                   util.CodebaseParameter(codebase='yocto-docs',
-                                          label="yocto-docs:",
-                                          project=None),
-                   util.CodebaseParameter(codebase='bitbake',
-                                          label="bitbake:",
-                                          project=None),
-                   ],
+        codebases=[util.CodebaseParameter(codebase='yab-helper', label="yocto-autobuilder-helper:", project=None)],
         reason=util.StringParameter(
                 name="reason",
                 label="""Reason (please note the reason for triggering the docs build:""",