diff mbox series

[toaster,PATCHv2,1/2] toaster: Added feat to import eventlogs files as builds in toaster

Message ID 20231130214235.20330-2-marlon.rodriguez-garcia@savoirfairelinux.com
State New
Headers show
Series New Feature to import eventlogs in toaster | expand

Commit Message

Marlon Rodriguez Garcia Nov. 30, 2023, 9:42 p.m. UTC
New feature added to recreate command line builds that are made when toaster is off.
The feature contains a new button on the base template to access a new template.
A new model was added to register the information on the builds and generate access links
A new form was added to include the option to load specific files
This feature uses the value from the variable BB_DEFAULT_EVENTLOG to read the files created by bitbake

Signed-off-by: Marlon Rodriguez Garcia <marlon.rodriguez-garcia@savoirfairelinux.com>
---
 lib/bb/ui/toasterui.py                        |   2 +-
 .../orm/migrations/0021_eventlogsimports.py   |  22 ++
 lib/toaster/orm/models.py                     |   9 +
 lib/toaster/toastergui/forms.py               |  13 ++
 lib/toaster/toastergui/templates/base.html    |   3 +-
 .../templates/command_line_builds.html        | 171 ++++++++++++++++
 lib/toaster/toastergui/urls.py                |   1 +
 lib/toaster/toastergui/views.py               | 188 +++++++++++++++++-
 8 files changed, 405 insertions(+), 4 deletions(-)
 create mode 100644 lib/toaster/orm/migrations/0021_eventlogsimports.py
 create mode 100644 lib/toaster/toastergui/forms.py
 create mode 100644 lib/toaster/toastergui/templates/command_line_builds.html
diff mbox series

Patch

diff --git a/lib/bb/ui/toasterui.py b/lib/bb/ui/toasterui.py
index ec5bd4f1..6bd21f18 100644
--- a/lib/bb/ui/toasterui.py
+++ b/lib/bb/ui/toasterui.py
@@ -385,7 +385,7 @@  def main(server, eventHandler, params):
                     main.shutdown = 1
 
                 logger.info("ToasterUI build done, brbe: %s", brbe)
-                continue
+                break
 
             if isinstance(event, (bb.command.CommandCompleted,
                                   bb.command.CommandFailed,
diff --git a/lib/toaster/orm/migrations/0021_eventlogsimports.py b/lib/toaster/orm/migrations/0021_eventlogsimports.py
new file mode 100644
index 00000000..328eb575
--- /dev/null
+++ b/lib/toaster/orm/migrations/0021_eventlogsimports.py
@@ -0,0 +1,22 @@ 
+# Generated by Django 4.2.5 on 2023-11-23 18:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0020_models_bigautofield'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='EventLogsImports',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255)),
+                ('imported', models.BooleanField(default=False)),
+                ('build_id', models.IntegerField(blank=True, null=True)),
+            ],
+        ),
+    ]
diff --git a/lib/toaster/orm/models.py b/lib/toaster/orm/models.py
index 1098ad3f..19c96862 100644
--- a/lib/toaster/orm/models.py
+++ b/lib/toaster/orm/models.py
@@ -1868,6 +1868,15 @@  class Distro(models.Model):
     def __unicode__(self):
         return "Distro " + self.name + "(" + self.description + ")"
 
+class EventLogsImports(models.Model):
+    name = models.CharField(max_length=255)
+    imported = models.BooleanField(default=False)
+    build_id = models.IntegerField(blank=True, null=True)
+
+    def __str__(self):
+        return self.name
+
+
 django.db.models.signals.post_save.connect(invalidate_cache)
 django.db.models.signals.post_delete.connect(invalidate_cache)
 django.db.models.signals.m2m_changed.connect(invalidate_cache)
diff --git a/lib/toaster/toastergui/forms.py b/lib/toaster/toastergui/forms.py
new file mode 100644
index 00000000..a87e5391
--- /dev/null
+++ b/lib/toaster/toastergui/forms.py
@@ -0,0 +1,13 @@ 
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013        Intel Corporation
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+from django import forms
+from django.core.validators import FileExtensionValidator
+  
+class LoadFileForm(forms.Form):
+    eventlog_file = forms.FileField(widget=forms.FileInput(attrs={'accept': '.json'}))
diff --git a/lib/toaster/toastergui/templates/base.html b/lib/toaster/toastergui/templates/base.html
index 041448d1..e90be696 100644
--- a/lib/toaster/toastergui/templates/base.html
+++ b/lib/toaster/toastergui/templates/base.html
@@ -132,7 +132,8 @@ 
             {% if project_enable %}
             <a class="btn btn-default navbar-btn navbar-right" id="new-project-button" href="{% url 'newproject' %}">New project</a>
             {% endif %}
-          </div>
+            <a class="btn btn-default navbar-btn navbar-right" id="import_page" style="margin-right: 5px !important" id="import-cmdline-button" href="{% url 'cmdlines' %}">Import command line builds</a>
+            </div>
       </div>
     </nav>
 
diff --git a/lib/toaster/toastergui/templates/command_line_builds.html b/lib/toaster/toastergui/templates/command_line_builds.html
new file mode 100644
index 00000000..5b085e33
--- /dev/null
+++ b/lib/toaster/toastergui/templates/command_line_builds.html
@@ -0,0 +1,171 @@ 
+{% extends "base.html" %}
+{% load projecttags %}
+{% load humanize %}
+
+{% block title %} Import Builds from eventlogs - Toaster {% endblock %}
+
+{% block pagecontent %}
+
+<div class="container-fluid">
+    <div class="row">
+        <div class="col-md-12">
+            <div class="page-header">
+                <div class="row">
+                    <div class="col-md-6">
+                        <h1>Import command line builds</h1>
+                    </div>
+                    {% if import_all %}
+                    <div class="col-md-6">
+                        <button id="import_all" type="button" class="btn btn-primary navbar-btn navbar-right">
+                            <span class="glyphicon glyphicon-upload" style="vertical-align: top;"></span> Import All
+                        </button>
+                    </div> 
+                    {% endif %}    
+                </div>
+            </div>
+            {% if messages %}
+            <div class="row-fluid" id="empty-state-{{table_name}}">
+                {% for message in messages %}
+                <div class="alert alert-danger">{{message}}</div>
+                {%endfor%}
+            </div>
+            {% endif %}
+            <div class="row">
+                <h4 style="margin-left: 15px;"><strong>Import eventlog file</strong></h4>
+                <form method="POST" enctype="multipart/form-data" action="{% url 'cmdlines' %}"> 
+                    {% csrf_token %} 
+                    <div class="col-md-6" style="padding-left: 20px;">
+                        <div class="row">
+                            <input type="hidden" value="{{dir}}" name="dir">
+                            <div class="col-md-3"> {{ form.eventlog_file}}  </div>                            
+                        </div>
+                        <div class="row" style="padding-top: 10px;">
+                            <div class="col-md-6"> 
+                                <button id="file_import" type="submit" disabled="disabled" class="btn btn-default navbar-btn" >
+                                    <span class="glyphicon glyphicon-upload" style="vertical-align: top;"></span> Import
+                                </button>
+                            </div>
+                        </div>
+                    </div>
+                </form>
+            </div>
+
+            <div class="row" style="padding-top: 20px;">
+                <div class="col-md-6 ">
+                    <h4><strong>Import eventlog files from directory</strong>
+                        <a href="#" data-toggle="tooltip" title="{{dir}}">
+                            <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle" viewBox="0 0 16 16" data-toggle="tooltip">
+                                <path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14m0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16"/>
+                                <path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0"/>
+                            </svg>
+                        </a>
+                    </h4>
+                    {% if files %}
+                    <div class="table-responsive">
+                        <table class="table col-md-4 table-bordered table-hover">
+                            <thead>
+                            <tr class="row">
+                                <th scope="col">Name</th>
+                                <th scope="col">Action</th>
+                            </tr>
+                            </thead>
+                            <tbody>
+                                {% for file in files %}
+                                <tr class="row" style="height: 48px;">
+                                    <th scope="row" class="col-md-6" style="vertical-align: middle;">
+                                        <input type="hidden" value="{{file.name}}" name="{{file.name}}">{{file.name}}
+                                    </th>
+                                    <td class="col-md-6 align-middle" style="vertical-align: middle;">
+                                        {% if file.imported == True and file.build_id is not None %}
+                                            <a href="{% url 'builddashboard' file.build_id %}">Build Details</a>
+                                        {%else %}
+                                            <a onclick="_ajax_update('{{file.name}}', false, '{{dir}}')" data-toggle="tooltip" title="Import File">
+                                                <span class="glyphicon glyphicon-upload" style="font-size: 18px;"></span>
+                                            </a>
+                                        {%endif%}
+                                    </td>
+                                </tr>
+                                {% endfor%}
+                            </tbody>
+                        </table>
+                    </div>
+                    {% else %}
+                    <div class="row-fluid" id="empty-state-{{table_name}}">
+                        <div class="alert alert-info">Sorry - no files found</div>
+                    </div>
+                    {%endif%}
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+
+<script>
+
+function _ajax_update(file, all, dir){
+    function getCookie(name) {
+        var cookieValue = null;
+        if (document.cookie && document.cookie !== '') {
+            var cookies = document.cookie.split(';');
+            for (var i = 0; i < cookies.length; i++) {
+                var cookie = jQuery.trim(cookies[i]);
+                // Does this cookie string begin with the name we want?
+                if (cookie.substring(0, name.length + 1) === (name + '=')) {
+                    cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
+                    break;
+                }
+            }
+        }
+    return cookieValue;
+    }
+    var csrftoken = getCookie('csrftoken');
+
+    function csrfSafeMethod(method) {
+        // these HTTP methods do not require CSRF protection
+        return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method));
+    }
+    $.ajaxSetup({
+        beforeSend: function (xhr, settings) {
+            if (!csrfSafeMethod(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken);
+            }
+        }
+    });
+
+    $.ajax({
+        url:'/toastergui/cmdline/',
+        type: "POST",
+        data: {file: file, all: all, dir: dir},
+        success:function(data){
+            window.location = '/toastergui/builds/'
+        },
+        complete:function(data){       
+        },
+        error:function (xhr, textStatus, thrownError){
+            console.log('fail');
+        }
+    });
+}
+
+$('#import_all').on('click', function(){
+    _ajax_update("{{files | safe}}", true, "{{dir | safe}}");
+});
+
+
+$('#import_page').hide();
+
+$(function () {
+  $('[data-toggle="tooltip"]').tooltip()
+})
+
+
+$("#id_eventlog_file").change(function(){
+    console.log($('#file_import'))
+    $('#file_import').prop("disabled", false);
+    $('#file_import').addClass('btn-primary')
+    $('#file_import').removeClass('btn-default')
+})
+
+</script>
+
+{% endblock %}
diff --git a/lib/toaster/toastergui/urls.py b/lib/toaster/toastergui/urls.py
index bc3b0c79..62629494 100644
--- a/lib/toaster/toastergui/urls.py
+++ b/lib/toaster/toastergui/urls.py
@@ -95,6 +95,7 @@  urlpatterns = [
         # project URLs
         url(r'^newproject/$', views.newproject, name='newproject'),
 
+        url(r'^cmdline/$', views.CommandLineBuilds.as_view(), name='cmdlines'),
         url(r'^projects/$',
             tables.ProjectsTable.as_view(template_name="projects-toastertable.html"),
             name='all-projects'),
diff --git a/lib/toaster/toastergui/views.py b/lib/toaster/toastergui/views.py
index cc8517ba..a75fba47 100644
--- a/lib/toaster/toastergui/views.py
+++ b/lib/toaster/toastergui/views.py
@@ -6,24 +6,36 @@ 
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import ast
 import re
+import pickle
+import codecs
+import subprocess
+
+import bb.cooker
+from bb.ui import toasterui
 
 from django.db.models import F, Q, Sum
 from django.db import IntegrityError
-from django.shortcuts import render, redirect, get_object_or_404
+from django.shortcuts import render, redirect, get_object_or_404, HttpResponseRedirect
 from django.utils.http import urlencode
 from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe
 from orm.models import LogMessage, Variable, Package_Dependency, Package
 from orm.models import Task_Dependency, Package_File
 from orm.models import Target_Installed_Package, Target_File
 from orm.models import TargetKernelFile, TargetSDKFile, Target_Image_File
-from orm.models import BitbakeVersion, CustomImageRecipe
+from orm.models import BitbakeVersion, CustomImageRecipe, EventLogsImports
 
 from django.urls import reverse, resolve
+from django.contrib import messages
+
 from django.core.exceptions import ObjectDoesNotExist
+from django.core.files.storage import FileSystemStorage
+from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 from django.http import HttpResponseNotFound, JsonResponse
 from django.utils import timezone
+from django.views.generic import TemplateView
 from datetime import timedelta, datetime
 from toastergui.templatetags.projecttags import json as jsonfilter
 from decimal import Decimal
@@ -32,6 +44,10 @@  import os
 from os.path import dirname
 import mimetypes
 
+from toastergui.forms import LoadFileForm
+
+from collections import namedtuple
+
 import logging
 
 from toastermain.logs import log_view_mixin
@@ -41,6 +57,7 @@  logger = logging.getLogger("toaster")
 # Project creation and managed build enable
 project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
 is_project_specific = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC'))
+import_page = False
 
 class MimeTypeFinder(object):
     # setting this to False enables additional non-standard mimetypes
@@ -1940,3 +1957,170 @@  if True:
         except (ObjectDoesNotExist, IOError):
             return toaster_render(request, "unavailable_artifact.html")
 
+class EventPlayer:
+    """Emulate a connection to a bitbake server."""
+
+    def __init__(self, eventfile, variables):
+        self.eventfile = eventfile
+        self.variables = variables
+        self.eventmask = []
+
+    def waitEvent(self, _timeout):
+        """Read event from the file."""
+        line = self.eventfile.readline().strip()
+        if not line:
+            return
+        try:
+            event_str = json.loads(line)['vars'].encode('utf-8')
+            event = pickle.loads(codecs.decode(event_str, 'base64'))
+            event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
+            if event_name not in self.eventmask:
+                return
+            return event
+        except ValueError as err:
+            print("Failed loading ", line)
+            raise err
+
+    def runCommand(self, command_line):
+        """Emulate running a command on the server."""
+        name = command_line[0]
+
+        if name == "getVariable":
+            var_name = command_line[1]
+            variable = self.variables.get(var_name)
+            if variable:
+                return variable['v'], None
+            return None, "Missing variable %s" % var_name
+
+        elif name == "getAllKeysWithFlags":
+            dump = {}
+            flaglist = command_line[1]
+            for key, val in self.variables.items():
+                try:
+                    if not key.startswith("__"):
+                        dump[key] = {
+                            'v': val['v'],
+                            'history' : val['history'],
+                        }
+                        for flag in flaglist:
+                            dump[key][flag] = val[flag]
+                except Exception as err:
+                    print(err)
+            return (dump, None)
+
+        elif name == 'setEventMask':
+            self.eventmask = command_line[-1]
+            return True, None
+
+        else:
+            raise Exception("Command %s not implemented" % command_line[0])
+
+    def getEventHandle(self):
+        """
+        This method is called by toasterui.
+        The return value is passed to self.runCommand but not used there.
+        """
+        pass
+
+
+class CommandLineBuilds(TemplateView):
+    model = EventLogsImports
+    template_name = 'command_line_builds.html'
+
+    def get_context_data(self, **kwargs):
+        context = super(CommandLineBuilds, self).get_context_data(**kwargs)
+        #get value from BB_DEFAULT_EVENTLOG defined in bitbake.conf
+        eventlog = subprocess.check_output(['bitbake-getvar', 'BB_DEFAULT_EVENTLOG', '--value'])
+        if eventlog:
+            logs_dir = os.path.dirname(eventlog.decode().strip('\n'))
+            files = os.listdir(logs_dir)
+            imported_files = EventLogsImports.objects.all()
+            files_list = []
+
+            # Filter files that end with ".json"
+            event_files = [file for file in files if file.endswith(".json")]
+
+            #build dict for template using db data
+            for event_file in event_files:
+                if imported_files.filter(name=event_file):
+                    files_list.append({
+                        'name': event_file,
+                        'imported': True,
+                        'build_id': imported_files.filter(name=event_file)[0].build_id
+                    })
+                else:
+                    files_list.append({
+                        'name': event_file,
+                        'imported': False,
+                        'build_id': None
+                    })
+                    context['import_all'] = True
+
+            context['files'] = files_list
+            context['dir'] = logs_dir
+        else:
+            context['files'] = []
+            context['dir'] = ''
+
+        context['form'] = LoadFileForm()
+        context['project_enable'] = project_enable
+        return context
+
+    def post(self, request, **kwargs):
+        logs_dir = request.POST.get('dir')
+        all_files =  request.POST.get('all')
+
+        imported_files = EventLogsImports.objects.all()
+        try:
+            if all_files == 'true':
+                files = ast.literal_eval(request.POST.get('file'))
+                for file in files:
+                    if imported_files.filter(name=file.get('name')).exists():
+                        imported_files.filter(name=file.get('name'))[0].imported = True
+                    else:
+                        with open("{}/{}".format(logs_dir, file.get('name'))) as eventfile:
+                            # load variables from the first line
+                            variables = json.loads(eventfile.readline().strip())['allvariables']
+
+                            params = namedtuple('ConfigParams', ['observe_only'])(True)
+                            player = EventPlayer(eventfile, variables)
+
+                            toasterui.main(player, player, params)
+                        event_log_import = EventLogsImports.objects.create(name=file.get('name'), imported=True)
+                        event_log_import.build_id = Build.objects.last().id
+                        event_log_import.save()
+            else:
+                if self.request.FILES.get('eventlog_file'):
+                    file = self.request.FILES['eventlog_file']
+                else:
+                    file = request.POST.get('file')
+
+                if imported_files.filter(name=file).exists():
+                    imported_files.filter(name=file)[0].imported = True
+                else: 
+                    if isinstance(file, InMemoryUploadedFile) or isinstance(file, TemporaryUploadedFile):
+                        variables = json.loads(file.readline().strip())['allvariables']
+                        params = namedtuple('ConfigParams', ['observe_only'])(True)
+                        player = EventPlayer(file, variables)
+                        if not os.path.exists('{}/{}'.format(logs_dir, file.name)):
+                            fs = FileSystemStorage(location=logs_dir)
+                            fs.save(file.name, file)
+                        toasterui.main(player, player, params)
+                    else:
+                        with open("{}/{}".format(logs_dir, file)) as eventfile:
+                            # load variables from the first line
+                            variables = json.loads(eventfile.readline().strip())['allvariables']
+                            params = namedtuple('ConfigParams', ['observe_only'])(True)
+                            player = EventPlayer(eventfile, variables)
+                            toasterui.main(player, player, params)
+                    event_log_import = EventLogsImports.objects.create(name=file, imported=True)
+                    event_log_import.build_id = Build.objects.last().id
+                    event_log_import.save()
+        except json.decoder.JSONDecodeError:
+            messages.add_message(
+                self.request,
+                messages.SUCCESS,
+                "The file content is not in the correct format. Update file content or upload a different file."
+            )
+            return HttpResponseRedirect("/toastergui/cmdline/")
+        return HttpResponseRedirect('/toastergui/builds/')