diff mbox series

[layerindex-web] layerindex/restviews.py: stream unpaginated JSON lists

Message ID 20261117192938.22269-1-tim.orling@konsulko.com
State Accepted
Commit d9877fe26b761d9e768afe8e65d9c21eb09bbe31
Headers show
Series [layerindex-web] layerindex/restviews.py: stream unpaginated JSON lists | expand

Commit Message

Tim Orling Nov. 17, 2026, 7:29 p.m. UTC
Most API lists aren't paginated, and clients such as bitbake's
layerindexlib fetch whole tables: /layerindex/api/recipes/ is about
300 MB of JSON. Building that in one piece took a worker to 1.6 GB. Send
JSON lists a chunk of 2000 objects at a time instead, with the same
output; paginated lists and the browsable API are unchanged.

Tests that read an unpaginated list now read the streamed content, and
the API contract tests check the shape of the streamed responses.

[YOCTO #16525]

AI-Generated: Claude Opus 5.5
Signed-off-by: Tim Orling <tim.orling@konsulko.com>
---
 layerindex/restviews.py             | 34 +++++++++++++++
 tests/test_api_contract.py          |  3 ++
 tests/test_published_layers_feed.py |  4 +-
 tests/test_restapi.py               |  9 ++--
 tests/test_restapi_stream.py        | 65 +++++++++++++++++++++++++++++
 5 files changed, 111 insertions(+), 4 deletions(-)
 create mode 100644 tests/test_restapi_stream.py
diff mbox series

Patch

diff --git a/layerindex/restviews.py b/layerindex/restviews.py
index 1b6b926..c55c444 100644
--- a/layerindex/restviews.py
+++ b/layerindex/restviews.py
@@ -7,7 +7,9 @@ 
 # SPDX-License-Identifier: MIT
 
 from layerindex.models import Branch, LayerItem, LayerMaintainer, YPCompatibleVersion, LayerNote, LayerBranch, LayerDependency, Recipe, Machine, Distro, BBClass, Source, Patch, PackageConfig, StaticBuildDep, DynamicBuildDep, RecipeFileDependency, BBAppend, IncFile
+from django.http import StreamingHttpResponse
 from rest_framework import viewsets, serializers, pagination
+from rest_framework.renderers import JSONRenderer
 from layerindex.querysethelper import params_to_queryset, get_search_tuple
 
 class LayerIndexPagination(pagination.PageNumberPagination):
@@ -41,6 +43,38 @@  class ParametricSearchableModelViewSet(viewsets.ReadOnlyModelViewSet):
         (filter_string, search_term, ordering_string) = get_search_tuple(self.request, model)
         return params_to_queryset(model, qs, filter_string, search_term, ordering_string)
 
+    # Objects serialised at a time when streaming a list
+    stream_chunk_size = 2000
+
+    def list(self, request, *args, **kwargs):
+        # An unpaginated list can be a whole table: all the recipes are 300 MB
+        # of JSON, which took a worker to 1.6 GB when built in one piece. Send
+        # JSON a chunk at a time instead, in the same format. Other renderers
+        # (e.g. the browsable API) build the whole response as before.
+        # (AI-generated, Claude Opus 5.5)
+        renderer = request.accepted_renderer
+        if self.paginator is not None or not isinstance(renderer, JSONRenderer):
+            return super().list(request, *args, **kwargs)
+        # Evaluated here, so that an invalid filter is still an error response
+        queryset = self.filter_queryset(self.get_queryset())
+        return StreamingHttpResponse(self.stream_json(queryset, renderer),
+                                     content_type=renderer.media_type)
+
+    def stream_json(self, queryset, renderer):
+        # The objects of each chunk, rendered as a list without its brackets
+        yield b'['
+        separator = b''
+        chunk = []
+        for obj in queryset.iterator(chunk_size=self.stream_chunk_size):
+            chunk.append(obj)
+            if len(chunk) == self.stream_chunk_size:
+                yield separator + renderer.render(self.get_serializer(chunk, many=True).data)[1:-1]
+                separator = b','
+                chunk = []
+        if chunk:
+            yield separator + renderer.render(self.get_serializer(chunk, many=True).data)[1:-1]
+        yield b']'
+
 class BranchSerializer(DynamicFieldsModelSerializer):
     class Meta:
         model = Branch
diff --git a/tests/test_api_contract.py b/tests/test_api_contract.py
index 5de9427..feec1a6 100644
--- a/tests/test_api_contract.py
+++ b/tests/test_api_contract.py
@@ -90,6 +90,9 @@  def api(client, seeded_index):
         response = client.get('/layerindex/api/' + path, params)
         assert response.status_code == 200, path
         assert response['Content-Type'] == 'application/json', path
+        # An unpaginated list is streamed (AI-generated, Claude Opus 5.5)
+        if response.streaming:
+            return json.loads(b''.join(response.streaming_content))
         return response.json()
     return get
 
diff --git a/tests/test_published_layers_feed.py b/tests/test_published_layers_feed.py
index 7554677..26e6664 100644
--- a/tests/test_published_layers_feed.py
+++ b/tests/test_published_layers_feed.py
@@ -1,4 +1,5 @@ 
 import datetime
+import json
 import xml.etree.ElementTree as ET
 from email.utils import parsedate_to_datetime
 from urllib.parse import urlsplit
@@ -290,7 +291,8 @@  def test_published_date_is_not_in_the_rest_api(new_layer, publish, client):
 
     response = client.get('/layerindex/api/layerItems/', {'format': 'json'})
 
-    data = response.json()
+    # The list is streamed (AI-generated, Claude Opus 5.5)
+    data = json.loads(b''.join(response.streaming_content))
     items = data['results'] if isinstance(data, dict) else data
     (item,) = [i for i in items if i['name'] == 'meta-foo']
     assert 'published' not in item
diff --git a/tests/test_restapi.py b/tests/test_restapi.py
index fde1aec..20254c2 100644
--- a/tests/test_restapi.py
+++ b/tests/test_restapi.py
@@ -5,6 +5,7 @@ 
 # SPDX-License-Identifier: MIT
 
 import datetime
+import json
 
 import pytest
 from django.test import TestCase
@@ -24,7 +25,8 @@  class TestRestApiDatetimes(TestCase):
 
         response = self.client.get('/layerindex/api/layerItems/', {'format': 'json'})
         self.assertEqual(response.status_code, 200)
-        data = response.json()
+        # Unpaginated lists are streamed
+        data = json.loads(b''.join(response.streaming_content))
         items = data['results'] if isinstance(data, dict) else data
         item = next(i for i in items if i['name'] == 'meta-test')
         self.assertEqual(item['updated'], '2026-10-18T14:37:05+0000')
@@ -59,7 +61,8 @@  class TestRestApiInvalidParameters(TestCase):
         response = self.client.get('/layerindex/api/layerItems/',
                                    {'filter': 'name:meta-test', 'format': 'json'})
         self.assertEqual(response.status_code, 200)
-        self.assertEqual([i['name'] for i in response.json()], ['meta-test'])
+        self.assertEqual([i['name'] for i in json.loads(b''.join(response.streaming_content))],
+                         ['meta-test'])
 
 
 @pytest.mark.django_db
@@ -76,7 +79,7 @@  class TestRestApiOrderby(TestCase):
         response = self.client.get('/layerindex/api/layerItems/',
                                    {'orderby': orderby, 'format': 'json'})
         self.assertEqual(response.status_code, 200)
-        return [i['name'] for i in response.json()]
+        return [i['name'] for i in json.loads(b''.join(response.streaming_content))]
 
     def test_ascending(self):
         self.assertEqual(self.names('name:+'), ['meta-a', 'meta-b', 'meta-c'])
diff --git a/tests/test_restapi_stream.py b/tests/test_restapi_stream.py
new file mode 100644
index 0000000..48cc36f
--- /dev/null
+++ b/tests/test_restapi_stream.py
@@ -0,0 +1,65 @@ 
+# layerindex-web - tests for streamed REST API lists
+#
+# Licensed under the MIT license, see COPYING.MIT for details
+#
+# SPDX-License-Identifier: MIT
+#
+# AI-generated, Claude Opus 5.5
+
+# An unpaginated list is sent a chunk at a time, so a whole table never has to
+# be built in memory, and the JSON is the same as before. tests/test_api_contract.py
+# checks the shape of the streamed responses.
+
+import json
+from unittest import mock
+
+from layerindex.models import Recipe
+from layerindex.restviews import ParametricSearchableModelViewSet, RecipeSerializer
+
+
+def get_recipes(client, **params):
+    return client.get('/layerindex/api/recipes/', params)
+
+
+def content(response):
+    return b''.join(response.streaming_content)
+
+
+def test_streamed_in_chunks(client, seeded_index):
+    # Chunks of 3 for 4 recipes: the last chunk is short
+    with mock.patch.object(ParametricSearchableModelViewSet, 'stream_chunk_size', 3):
+        response = get_recipes(client, format='json')
+        assert response.status_code == 200
+        assert response.streaming
+        assert response['Content-Type'] == 'application/json'
+        data = json.loads(content(response))
+    expected = RecipeSerializer(Recipe.objects.order_by('id'), many=True).data
+    assert data == json.loads(json.dumps(expected))
+
+
+def test_whole_chunks(client, seeded_index):
+    with mock.patch.object(ParametricSearchableModelViewSet, 'stream_chunk_size', 2):
+        data = json.loads(content(get_recipes(client, format='json')))
+    assert len(data) == Recipe.objects.count() == 4
+
+
+def test_empty_list(client, seeded_index):
+    assert content(get_recipes(client, format='json', filter='pn:nosuchrecipe')) == b'[]'
+
+
+def test_filter_still_applies(client, seeded_index):
+    data = json.loads(content(get_recipes(client, format='json', filter='pn:base-example')))
+    assert [r['pn'] for r in data] == ['base-example']
+
+
+def test_paginated_list_not_streamed(client, seeded_index):
+    response = client.get('/layerindex/api/recipesExtended/', {'format': 'json'})
+    assert response.status_code == 200
+    assert not response.streaming
+    assert response.json()['count'] == 4
+
+
+def test_browsable_api_not_streamed(client, seeded_index):
+    response = client.get('/layerindex/api/recipes/', HTTP_ACCEPT='text/html')
+    assert response.status_code == 200
+    assert not response.streaming