diff mbox series

[layerindex-web,2/4] rrs/views.py: fetch the recipe list's patch counts in a few queries

Message ID 20261116222410.35784-3-tim.orling@konsulko.com
State New
Headers show
Series Speed up the RRS pages, and test them | expand

Commit Message

Tim Orling Nov. 16, 2026, 10:24 p.m. UTC
_get_recipe_list() ran five queries per recipe, so the recipes and report
pages took about 4 s for OE-Core and 9 s for meta-openembedded, with the
database busy for under 0.5 s of that. Fetch the recipe symbols with
in_bulk() and the patch counts with one annotated query per layer branch,
counting the same recipe as before when several share a pn.

On a copy of the production database the recipe lists of 12 milestones
are identical before and after, and OE-Core 6.2 takes 9 queries and
0.12 s instead of 4880 queries and 3.7 s.

AI-Generated: Claude Opus 5.5
Signed-off-by: Tim Orling <tim.orling@konsulko.com>
---
 rrs/views.py                  | 30 ++++++++++-----
 tests/rrs/test_recipe_list.py | 71 +++++++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+), 9 deletions(-)
 create mode 100644 tests/rrs/test_recipe_list.py
diff mbox series

Patch

diff --git a/rrs/views.py b/rrs/views.py
index b2520ce..b580818 100644
--- a/rrs/views.py
+++ b/rrs/views.py
@@ -17,6 +17,7 @@  from django.shortcuts import get_object_or_404
 from django.views.generic import TemplateView, ListView, DetailView, RedirectView
 from django.urls import resolve, reverse, reverse_lazy
 from django.db import connection
+from django.db.models import Count, Q
 from django.contrib import messages
 
 from layerindex.models import Recipe, StaticBuildDep, Patch
@@ -432,12 +433,29 @@  def _get_recipe_list(milestone):
                 for ma in maintainers_all:
                     maintainers_dict_all[ma['recipesymbol_id']] = ma['name']
 
+        # Fetch the recipe symbols, and the patch counts of their layer index
+        # recipes, in a few queries rather than several per recipe
+        # (AI-generated, Claude Opus 5.5)
+        recipesymbols = RecipeSymbol.objects.in_bulk([recipe['id'] for recipe in recipes])
+        patch_counts = {}
+        if recipesymbols:
+            recipeobjs = Recipe.objects.filter(
+                    layerbranch_id__in={rs.layerbranch_id for rs in recipesymbols.values()},
+                    pn__in={rs.pn for rs in recipesymbols.values()}) \
+                .order_by('id') \
+                .annotate(patches_total=Count('patch'),
+                          patches_pending=Count('patch', filter=Q(patch__status='P')))
+            for recipeobj in recipeobjs:
+                # Keep the first recipe by id for a pn, as .first() did
+                patch_counts.setdefault((recipeobj.layerbranch_id, recipeobj.pn),
+                                        (recipeobj.patches_total, recipeobj.patches_pending))
+
         for recipe in recipes:
             upstream_version = ''
             upstream_status = ''
             no_update_reason = ''
             outdated = ''
-            recipesymbol = RecipeSymbol.objects.filter(id=recipe['id']).first()
+            recipesymbol = recipesymbols.get(recipe['id'])
 
             if recipe_upstream_history:
                 recipe_upstream = recipe_upstream_dict_all.get(recipe['id'])
@@ -479,14 +497,8 @@  def _get_recipe_list(milestone):
             recipe_list_item.upstream_status = upstream_status
             recipe_list_item.upstream_version = upstream_version
             recipe_list_item.outdated = outdated
-            recipeobj = recipesymbol.layerbranch.recipe_set.filter(pn=recipesymbol.pn, layerbranch=recipesymbol.layerbranch).first()
-            if recipeobj:
-                patches = recipeobj.patch_set
-                recipe_list_item.patches_total = patches.count()
-                recipe_list_item.patches_pending = patches.filter(status='P').count()
-            else:
-                recipe_list_item.patches_total = 0
-                recipe_list_item.patches_pending = 0
+            recipe_list_item.patches_total, recipe_list_item.patches_pending = \
+                patch_counts.get((recipesymbol.layerbranch_id, recipesymbol.pn), (0, 0))
             recipe_list_item.maintainer_name = maintainer_name
             recipe_list_item.no_update_reason = no_update_reason
             recipe_list.append(recipe_list_item)
diff --git a/tests/rrs/test_recipe_list.py b/tests/rrs/test_recipe_list.py
new file mode 100644
index 0000000..95b2595
--- /dev/null
+++ b/tests/rrs/test_recipe_list.py
@@ -0,0 +1,71 @@ 
+# layerindex-web - tests for the RRS recipe list
+#
+# Licensed under the MIT license, see COPYING.MIT for details
+#
+# SPDX-License-Identifier: MIT
+#
+# AI-generated, Claude Opus 5.5
+
+# The recipe list shows each recipe's patch counts, and must get them without
+# running queries per recipe.
+
+import datetime
+
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
+
+from layerindex.models import Patch, Recipe
+from rrs.models import RecipeSymbol, RecipeUpgrade
+
+URL = '/rrs/recipes/OE-Core/6.2/M1/'
+
+
+def add_recipe(layerbranch, pn, patch_statuses=(), recipe=True):
+    """A recipe that RRS lists for the milestone, with patches of the given statuses"""
+    symbol = RecipeSymbol.objects.create(layerbranch=layerbranch, pn=pn, summary=pn)
+    date = datetime.datetime(2026, 11, 1)  # USE_TZ is False
+    RecipeUpgrade.objects.create(recipesymbol=symbol, version='1.0', author_date=date,
+                                 commit_date=date, upgrade_type='U')
+    if recipe:
+        obj = Recipe.objects.create(layerbranch=layerbranch, filename=f'{pn}_1.0.bb',
+                                    filepath='recipes', pn=pn, pv='1.0')
+        for i, status in enumerate(patch_statuses):
+            Patch.objects.create(recipe=obj, path=f'{pn}-{i}.patch',
+                                 src_path=f'{pn}-{i}.patch', status=status)
+    return symbol
+
+
+def recipe_list(client):
+    response = client.get(URL)
+    assert response.status_code == 200
+    return {item.name: item for item in response.context['recipe_list']}
+
+
+def test_patch_counts(client, rrs_plan):
+    layerbranch = rrs_plan.layerbranch
+    add_recipe(layerbranch, 'busybox', ['P', 'P', 'A'])
+    add_recipe(layerbranch, 'zlib', [])
+    add_recipe(layerbranch, 'norecipe', recipe=False)
+    # A second recipe with the same pn: the first one by id is counted
+    add_recipe(layerbranch, 'gcc', ['P'])
+    later = Recipe.objects.create(layerbranch=layerbranch, filename='gcc_2.0.bb',
+                                  filepath='recipes', pn='gcc', pv='2.0')
+    Patch.objects.create(recipe=later, path='x.patch', src_path='x.patch', status='P')
+    Patch.objects.create(recipe=later, path='y.patch', src_path='y.patch', status='P')
+
+    counts = {name: (item.patches_total, item.patches_pending)
+              for name, item in recipe_list(client).items()}
+
+    assert counts == {'busybox': (3, 2), 'zlib': (0, 0), 'norecipe': (0, 0), 'gcc': (1, 1)}
+
+
+def test_queries_do_not_grow_with_recipes(client, rrs_plan):
+    add_recipe(rrs_plan.layerbranch, 'recipe-0', ['P'])
+    recipe_list(client)  # fills the per-process caches first
+    with CaptureQueriesContext(connection) as one:
+        assert len(recipe_list(client)) == 1
+    for i in range(1, 6):
+        add_recipe(rrs_plan.layerbranch, f'recipe-{i}', ['P'])
+    with CaptureQueriesContext(connection) as six:
+        assert len(recipe_list(client)) == 6
+    assert len(six) == len(one)