@@ -7,7 +7,9 @@
# SPDX-License-Identifier: MIT
from layerindex.models import Branch, LayerItem, LayerMaintainer, YPCompatibleVersion, LayerNote, LayerBranch, LayerDependency, Recipe, Machine, Distro, BBClass, Source, Patch, PackageConfig, StaticBuildDep, DynamicBuildDep, RecipeFileDependency, BBAppend, IncFile
+from django.http import StreamingHttpResponse
from rest_framework import viewsets, serializers, pagination
+from rest_framework.renderers import JSONRenderer
from layerindex.querysethelper import params_to_queryset, get_search_tuple
class LayerIndexPagination(pagination.PageNumberPagination):
@@ -41,6 +43,38 @@ class ParametricSearchableModelViewSet(viewsets.ReadOnlyModelViewSet):
(filter_string, search_term, ordering_string) = get_search_tuple(self.request, model)
return params_to_queryset(model, qs, filter_string, search_term, ordering_string)
+ # Objects serialised at a time when streaming a list
+ stream_chunk_size = 2000
+
+ def list(self, request, *args, **kwargs):
+ # An unpaginated list can be a whole table: all the recipes are 300 MB
+ # of JSON, which took a worker to 1.6 GB when built in one piece. Send
+ # JSON a chunk at a time instead, in the same format. Other renderers
+ # (e.g. the browsable API) build the whole response as before.
+ # (AI-generated, Claude Opus 5.5)
+ renderer = request.accepted_renderer
+ if self.paginator is not None or not isinstance(renderer, JSONRenderer):
+ return super().list(request, *args, **kwargs)
+ # Evaluated here, so that an invalid filter is still an error response
+ queryset = self.filter_queryset(self.get_queryset())
+ return StreamingHttpResponse(self.stream_json(queryset, renderer),
+ content_type=renderer.media_type)
+
+ def stream_json(self, queryset, renderer):
+ # The objects of each chunk, rendered as a list without its brackets
+ yield b'['
+ separator = b''
+ chunk = []
+ for obj in queryset.iterator(chunk_size=self.stream_chunk_size):
+ chunk.append(obj)
+ if len(chunk) == self.stream_chunk_size:
+ yield separator + renderer.render(self.get_serializer(chunk, many=True).data)[1:-1]
+ separator = b','
+ chunk = []
+ if chunk:
+ yield separator + renderer.render(self.get_serializer(chunk, many=True).data)[1:-1]
+ yield b']'
+
class BranchSerializer(DynamicFieldsModelSerializer):
class Meta:
model = Branch
@@ -90,6 +90,9 @@ def api(client, seeded_index):
response = client.get('/layerindex/api/' + path, params)
assert response.status_code == 200, path
assert response['Content-Type'] == 'application/json', path
+ # An unpaginated list is streamed (AI-generated, Claude Opus 5.5)
+ if response.streaming:
+ return json.loads(b''.join(response.streaming_content))
return response.json()
return get
@@ -1,4 +1,5 @@
import datetime
+import json
import xml.etree.ElementTree as ET
from email.utils import parsedate_to_datetime
from urllib.parse import urlsplit
@@ -290,7 +291,8 @@ def test_published_date_is_not_in_the_rest_api(new_layer, publish, client):
response = client.get('/layerindex/api/layerItems/', {'format': 'json'})
- data = response.json()
+ # The list is streamed (AI-generated, Claude Opus 5.5)
+ data = json.loads(b''.join(response.streaming_content))
items = data['results'] if isinstance(data, dict) else data
(item,) = [i for i in items if i['name'] == 'meta-foo']
assert 'published' not in item
@@ -5,6 +5,7 @@
# SPDX-License-Identifier: MIT
import datetime
+import json
import pytest
from django.test import TestCase
@@ -24,7 +25,8 @@ class TestRestApiDatetimes(TestCase):
response = self.client.get('/layerindex/api/layerItems/', {'format': 'json'})
self.assertEqual(response.status_code, 200)
- data = response.json()
+ # Unpaginated lists are streamed
+ data = json.loads(b''.join(response.streaming_content))
items = data['results'] if isinstance(data, dict) else data
item = next(i for i in items if i['name'] == 'meta-test')
self.assertEqual(item['updated'], '2026-10-18T14:37:05+0000')
@@ -59,7 +61,8 @@ class TestRestApiInvalidParameters(TestCase):
response = self.client.get('/layerindex/api/layerItems/',
{'filter': 'name:meta-test', 'format': 'json'})
self.assertEqual(response.status_code, 200)
- self.assertEqual([i['name'] for i in response.json()], ['meta-test'])
+ self.assertEqual([i['name'] for i in json.loads(b''.join(response.streaming_content))],
+ ['meta-test'])
@pytest.mark.django_db
@@ -76,7 +79,7 @@ class TestRestApiOrderby(TestCase):
response = self.client.get('/layerindex/api/layerItems/',
{'orderby': orderby, 'format': 'json'})
self.assertEqual(response.status_code, 200)
- return [i['name'] for i in response.json()]
+ return [i['name'] for i in json.loads(b''.join(response.streaming_content))]
def test_ascending(self):
self.assertEqual(self.names('name:+'), ['meta-a', 'meta-b', 'meta-c'])
new file mode 100644
@@ -0,0 +1,65 @@
+# layerindex-web - tests for streamed REST API lists
+#
+# Licensed under the MIT license, see COPYING.MIT for details
+#
+# SPDX-License-Identifier: MIT
+#
+# AI-generated, Claude Opus 5.5
+
+# An unpaginated list is sent a chunk at a time, so a whole table never has to
+# be built in memory, and the JSON is the same as before. tests/test_api_contract.py
+# checks the shape of the streamed responses.
+
+import json
+from unittest import mock
+
+from layerindex.models import Recipe
+from layerindex.restviews import ParametricSearchableModelViewSet, RecipeSerializer
+
+
+def get_recipes(client, **params):
+ return client.get('/layerindex/api/recipes/', params)
+
+
+def content(response):
+ return b''.join(response.streaming_content)
+
+
+def test_streamed_in_chunks(client, seeded_index):
+ # Chunks of 3 for 4 recipes: the last chunk is short
+ with mock.patch.object(ParametricSearchableModelViewSet, 'stream_chunk_size', 3):
+ response = get_recipes(client, format='json')
+ assert response.status_code == 200
+ assert response.streaming
+ assert response['Content-Type'] == 'application/json'
+ data = json.loads(content(response))
+ expected = RecipeSerializer(Recipe.objects.order_by('id'), many=True).data
+ assert data == json.loads(json.dumps(expected))
+
+
+def test_whole_chunks(client, seeded_index):
+ with mock.patch.object(ParametricSearchableModelViewSet, 'stream_chunk_size', 2):
+ data = json.loads(content(get_recipes(client, format='json')))
+ assert len(data) == Recipe.objects.count() == 4
+
+
+def test_empty_list(client, seeded_index):
+ assert content(get_recipes(client, format='json', filter='pn:nosuchrecipe')) == b'[]'
+
+
+def test_filter_still_applies(client, seeded_index):
+ data = json.loads(content(get_recipes(client, format='json', filter='pn:base-example')))
+ assert [r['pn'] for r in data] == ['base-example']
+
+
+def test_paginated_list_not_streamed(client, seeded_index):
+ response = client.get('/layerindex/api/recipesExtended/', {'format': 'json'})
+ assert response.status_code == 200
+ assert not response.streaming
+ assert response.json()['count'] == 4
+
+
+def test_browsable_api_not_streamed(client, seeded_index):
+ response = client.get('/layerindex/api/recipes/', HTTP_ACCEPT='text/html')
+ assert response.status_code == 200
+ assert not response.streaming
Most API lists aren't paginated, and clients such as bitbake's layerindexlib fetch whole tables: /layerindex/api/recipes/ is about 300 MB of JSON. Building that in one piece took a worker to 1.6 GB. Send JSON lists a chunk of 2000 objects at a time instead, with the same output; paginated lists and the browsable API are unchanged. Tests that read an unpaginated list now read the streamed content, and the API contract tests check the shape of the streamed responses. [YOCTO #16525] AI-Generated: Claude Opus 5.5 Signed-off-by: Tim Orling <tim.orling@konsulko.com> --- layerindex/restviews.py | 34 +++++++++++++++ tests/test_api_contract.py | 3 ++ tests/test_published_layers_feed.py | 4 +- tests/test_restapi.py | 9 ++-- tests/test_restapi_stream.py | 65 +++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 tests/test_restapi_stream.py