| Message ID | 20231027003630.1433324-1-alassane.yattara@savoirfairelinux.com |
|---|---|
| State | New |
| Headers | show |
| Series | Toaster: Write logs to BUILDDIR/toaster_logs | expand |
Tested in both local and container environments. On Thu, Oct 26, 2023 at 5:36 PM Alassane Yattara < alassane.yattara@savoirfairelinux.com> wrote: > Fixes "2efb14648 toaster: Monitoring - implement Django logging system" > when > running in a container. > > When running in a container, the previous approach of using BASE_DIR > is not a writable path. Also, we really do not want to be writing logs into > the source tree, as the BASE_DIR was resolving to bitbake/lib/toaster/logs > > Since Toaster is only ever running in an environment where oe-init-buildenv > or similar has been sourced, we should instead write the logs to BUILDDIR. > > Using BUILDDIR to logs make path writable but django-log-viewer does'nt > manage > to write logs using an absolute path as BUILDDIR, where the existing > toaster_ui.log > was already being written. > > Also drop the /logs/ directory, as it has not been created which also > breaks > in a container environment > > To handle the constraints linked to django-log-viewer and /logs/, we've > updated > bitbake/bin/toaster to create a toaster_logs/ directory in BUILDDIR if it > doesn't exist, > when toaster starts up. > > Also manage to set BUILDDIR/toaster_logs/ as default location for toaster > logs. > > Signed-off-by: Alassane Yattara <alassane.yattara@savoirfairelinux.com> > Signed-off-by: Tim Orling <tim.orling@konsulko.com> > Tested-by: Tim Orling <tim.orling@konsulko.com> > --- > bin/toaster | 12 ++++++++++-- > lib/toaster/toastermain/logs.py | 10 +++++----- > lib/toaster/toastermain/settings.py | 4 ++-- > 3 files changed, 17 insertions(+), 9 deletions(-) > > diff --git a/bin/toaster b/bin/toaster > index 558a8195..f002c8c1 100755 > --- a/bin/toaster > +++ b/bin/toaster > @@ -84,7 +84,7 @@ webserverStartAll() > echo "Starting webserver..." > > $MANAGE runserver --noreload "$ADDR_PORT" \ > - </dev/null >>${BUILDDIR}/toaster_web.log 2>&1 \ > + </dev/null >>${TOASTER_LOGS_DIR}/web.log 2>&1 \ > & echo $! >${BUILDDIR}/.toastermain.pid > > sleep 1 > @@ -181,6 +181,14 @@ WEBSERVER=1 > export TOASTER_BUILDSERVER=1 > ADDR_PORT="localhost:8000" > TOASTERDIR=`dirname $BUILDDIR` > +# ${BUILDDIR}/toaster_logs/ became the default location for toaster logs > +# This is needed for implemented django-log-viewer: > https://pypi.org/project/django-log-viewer/ > +# If the directory does not exist, create it. > +TOASTER_LOGS_DIR="${BUILDDIR}/toaster_logs/" > +if [ ! -d $TOASTER_LOGS_DIR ] > +then > + mkdir $TOASTER_LOGS_DIR > +fi > unset CMD > for param in $*; do > case $param in > @@ -299,7 +307,7 @@ case $CMD in > export BITBAKE_UI='toasterui' > if [ $TOASTER_BUILDSERVER -eq 1 ] ; then > $MANAGE runbuilds \ > - </dev/null >>${BUILDDIR}/toaster_runbuilds.log 2>&1 \ > + </dev/null >>${TOASTER_LOGS_DIR}/toaster_runbuilds.log > 2>&1 \ > & echo $! >${BUILDDIR}/.runbuilds.pid > else > echo "Toaster build server not started." > diff --git a/lib/toaster/toastermain/logs.py > b/lib/toaster/toastermain/logs.py > index b4910e44..62d87196 100644 > --- a/lib/toaster/toastermain/logs.py > +++ b/lib/toaster/toastermain/logs.py > @@ -1,13 +1,13 @@ > #!/usr/bin/env python3 > # -*- coding: utf-8 -*- > > +import os > import logging > import json > from pathlib import Path > from django.http import HttpRequest > > -BASE_DIR = Path(__file__).resolve(strict=True).parent.parent > - > +BUILDDIR = Path(os.environ.get('BUILDDIR', '/tmp')) > > def log_api_request(request, response, view, logger_name='api'): > """Helper function for LogAPIMixin""" > @@ -108,7 +108,7 @@ LOGGING_SETTINGS = { > 'file_django': { > 'level': 'INFO', > 'class': 'logging.handlers.TimedRotatingFileHandler', > - 'filename': BASE_DIR / 'logs/django.log', > + 'filename': BUILDDIR / 'toaster_logs/django.log', > 'when': 'D', # interval type > 'interval': 1, # defaults to 1 > 'backupCount': 10, # how many files to keep > @@ -117,7 +117,7 @@ LOGGING_SETTINGS = { > 'file_api': { > 'level': 'INFO', > 'class': 'logging.handlers.TimedRotatingFileHandler', > - 'filename': BASE_DIR / 'logs/api.log', > + 'filename': BUILDDIR / 'toaster_logs/api.log', > 'when': 'D', > 'interval': 1, > 'backupCount': 10, > @@ -126,7 +126,7 @@ LOGGING_SETTINGS = { > 'file_toaster': { > 'level': 'INFO', > 'class': 'logging.handlers.TimedRotatingFileHandler', > - 'filename': BASE_DIR / 'logs/toaster.log', > + 'filename': BUILDDIR / 'toaster_logs/web.log', > 'when': 'D', > 'interval': 1, > 'backupCount': 10, > diff --git a/lib/toaster/toastermain/settings.py > b/lib/toaster/toastermain/settings.py > index b083cf58..5de881c4 100644 > --- a/lib/toaster/toastermain/settings.py > +++ b/lib/toaster/toastermain/settings.py > @@ -316,12 +316,12 @@ for t in os.walk(os.path.dirname(currentdir)): > LOGGING = LOGGING_SETTINGS > > # Build paths inside the project like this: BASE_DIR / 'subdir'. > -BASE_DIR = Path(__file__).resolve(strict=True).parent.parent > +BUILDDIR = os.environ.get("BUILDDIR", "/tmp") > > # LOG VIEWER > # https://pypi.org/project/django-log-viewer/ > LOG_VIEWER_FILES_PATTERN = '*.log*' > -LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs') > +LOG_VIEWER_FILES_DIR = os.path.join(BUILDDIR, "toaster_logs/") > LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page > LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read > LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'] > -- > 2.34.1 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#5899): > https://lists.yoctoproject.org/g/toaster/message/5899 > Mute This Topic: https://lists.yoctoproject.org/mt/102212389/924729 > Group Owner: toaster+owner@lists.yoctoproject.org > Unsubscribe: https://lists.yoctoproject.org/g/toaster/unsub [ > ticotimo@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
Tim, Yes, tested locally and in container ----- Mail original ----- De: "Tim Orling" <ticotimo@gmail.com> À: "Alassane Yattara" <alassane.yattara@savoirfairelinux.com> Cc: "toaster" <toaster@lists.yoctoproject.org>, "Tim Orling" <tim.orling@konsulko.com> Envoyé: Vendredi 27 Octobre 2023 03:28:28 Objet: Re: [Toaster] [PATCH] Toaster: Write logs to BUILDDIR/toaster_logs Tested in both local and container environments. On Thu, Oct 26, 2023 at 5:36 PM Alassane Yattara < alassane.yattara@savoirfairelinux.com> wrote: > Fixes "2efb14648 toaster: Monitoring - implement Django logging system" > when > running in a container. > > When running in a container, the previous approach of using BASE_DIR > is not a writable path. Also, we really do not want to be writing logs into > the source tree, as the BASE_DIR was resolving to bitbake/lib/toaster/logs > > Since Toaster is only ever running in an environment where oe-init-buildenv > or similar has been sourced, we should instead write the logs to BUILDDIR. > > Using BUILDDIR to logs make path writable but django-log-viewer does'nt > manage > to write logs using an absolute path as BUILDDIR, where the existing > toaster_ui.log > was already being written. > > Also drop the /logs/ directory, as it has not been created which also > breaks > in a container environment > > To handle the constraints linked to django-log-viewer and /logs/, we've > updated > bitbake/bin/toaster to create a toaster_logs/ directory in BUILDDIR if it > doesn't exist, > when toaster starts up. > > Also manage to set BUILDDIR/toaster_logs/ as default location for toaster > logs. > > Signed-off-by: Alassane Yattara <alassane.yattara@savoirfairelinux.com> > Signed-off-by: Tim Orling <tim.orling@konsulko.com> > Tested-by: Tim Orling <tim.orling@konsulko.com> > --- > bin/toaster | 12 ++++++++++-- > lib/toaster/toastermain/logs.py | 10 +++++----- > lib/toaster/toastermain/settings.py | 4 ++-- > 3 files changed, 17 insertions(+), 9 deletions(-) > > diff --git a/bin/toaster b/bin/toaster > index 558a8195..f002c8c1 100755 > --- a/bin/toaster > +++ b/bin/toaster > @@ -84,7 +84,7 @@ webserverStartAll() > echo "Starting webserver..." > > $MANAGE runserver --noreload "$ADDR_PORT" \ > - </dev/null >>${BUILDDIR}/toaster_web.log 2>&1 \ > + </dev/null >>${TOASTER_LOGS_DIR}/web.log 2>&1 \ > & echo $! >${BUILDDIR}/.toastermain.pid > > sleep 1 > @@ -181,6 +181,14 @@ WEBSERVER=1 > export TOASTER_BUILDSERVER=1 > ADDR_PORT="localhost:8000" > TOASTERDIR=`dirname $BUILDDIR` > +# ${BUILDDIR}/toaster_logs/ became the default location for toaster logs > +# This is needed for implemented django-log-viewer: > https://pypi.org/project/django-log-viewer/ > +# If the directory does not exist, create it. > +TOASTER_LOGS_DIR="${BUILDDIR}/toaster_logs/" > +if [ ! -d $TOASTER_LOGS_DIR ] > +then > + mkdir $TOASTER_LOGS_DIR > +fi > unset CMD > for param in $*; do > case $param in > @@ -299,7 +307,7 @@ case $CMD in > export BITBAKE_UI='toasterui' > if [ $TOASTER_BUILDSERVER -eq 1 ] ; then > $MANAGE runbuilds \ > - </dev/null >>${BUILDDIR}/toaster_runbuilds.log 2>&1 \ > + </dev/null >>${TOASTER_LOGS_DIR}/toaster_runbuilds.log > 2>&1 \ > & echo $! >${BUILDDIR}/.runbuilds.pid > else > echo "Toaster build server not started." > diff --git a/lib/toaster/toastermain/logs.py > b/lib/toaster/toastermain/logs.py > index b4910e44..62d87196 100644 > --- a/lib/toaster/toastermain/logs.py > +++ b/lib/toaster/toastermain/logs.py > @@ -1,13 +1,13 @@ > #!/usr/bin/env python3 > # -*- coding: utf-8 -*- > > +import os > import logging > import json > from pathlib import Path > from django.http import HttpRequest > > -BASE_DIR = Path(__file__).resolve(strict=True).parent.parent > - > +BUILDDIR = Path(os.environ.get('BUILDDIR', '/tmp')) > > def log_api_request(request, response, view, logger_name='api'): > """Helper function for LogAPIMixin""" > @@ -108,7 +108,7 @@ LOGGING_SETTINGS = { > 'file_django': { > 'level': 'INFO', > 'class': 'logging.handlers.TimedRotatingFileHandler', > - 'filename': BASE_DIR / 'logs/django.log', > + 'filename': BUILDDIR / 'toaster_logs/django.log', > 'when': 'D', # interval type > 'interval': 1, # defaults to 1 > 'backupCount': 10, # how many files to keep > @@ -117,7 +117,7 @@ LOGGING_SETTINGS = { > 'file_api': { > 'level': 'INFO', > 'class': 'logging.handlers.TimedRotatingFileHandler', > - 'filename': BASE_DIR / 'logs/api.log', > + 'filename': BUILDDIR / 'toaster_logs/api.log', > 'when': 'D', > 'interval': 1, > 'backupCount': 10, > @@ -126,7 +126,7 @@ LOGGING_SETTINGS = { > 'file_toaster': { > 'level': 'INFO', > 'class': 'logging.handlers.TimedRotatingFileHandler', > - 'filename': BASE_DIR / 'logs/toaster.log', > + 'filename': BUILDDIR / 'toaster_logs/web.log', > 'when': 'D', > 'interval': 1, > 'backupCount': 10, > diff --git a/lib/toaster/toastermain/settings.py > b/lib/toaster/toastermain/settings.py > index b083cf58..5de881c4 100644 > --- a/lib/toaster/toastermain/settings.py > +++ b/lib/toaster/toastermain/settings.py > @@ -316,12 +316,12 @@ for t in os.walk(os.path.dirname(currentdir)): > LOGGING = LOGGING_SETTINGS > > # Build paths inside the project like this: BASE_DIR / 'subdir'. > -BASE_DIR = Path(__file__).resolve(strict=True).parent.parent > +BUILDDIR = os.environ.get("BUILDDIR", "/tmp") > > # LOG VIEWER > # https://pypi.org/project/django-log-viewer/ > LOG_VIEWER_FILES_PATTERN = '*.log*' > -LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs') > +LOG_VIEWER_FILES_DIR = os.path.join(BUILDDIR, "toaster_logs/") > LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page > LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read > LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'] > -- > 2.34.1 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#5899): > https://lists.yoctoproject.org/g/toaster/message/5899 > Mute This Topic: https://lists.yoctoproject.org/mt/102212389/924729 > Group Owner: toaster+owner@lists.yoctoproject.org > Unsubscribe: https://lists.yoctoproject.org/g/toaster/unsub [ > ticotimo@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
On Thu, 2023-10-26 at 19:28 -0700, Tim Orling wrote: > Tested in both local and container environments. > > On Thu, Oct 26, 2023 at 5:36 PM Alassane Yattara > <alassane.yattara@savoirfairelinux.com> wrote: > > Fixes "2efb14648 toaster: Monitoring - implement Django logging > > system" when > > running in a container. > > > > When running in a container, the previous approach of using > > BASE_DIR > > is not a writable path. Also, we really do not want to be writing > > logs into > > the source tree, as the BASE_DIR was resolving to > > bitbake/lib/toaster/logs > > > > Since Toaster is only ever running in an environment where oe-init- > > buildenv > > or similar has been sourced, we should instead write the logs to > > BUILDDIR. > > > > Using BUILDDIR to logs make path writable but django-log-viewer > > does'nt manage > > to write logs using an absolute path as BUILDDIR, where the > > existing toaster_ui.log > > was already being written. > > > > Also drop the /logs/ directory, as it has not been created which > > also breaks > > in a container environment > > > > To handle the constraints linked to django-log-viewer and /logs/, > > we've updated > > bitbake/bin/toaster to create a toaster_logs/ directory in BUILDDIR > > if it doesn't exist, > > when toaster starts up. > > > > Also manage to set BUILDDIR/toaster_logs/ as default location for > > toaster logs. > > > > Signed-off-by: Alassane Yattara > > <alassane.yattara@savoirfairelinux.com> > > Signed-off-by: Tim Orling <tim.orling@konsulko.com> > > > > Tested-by: Tim Orling <tim.orling@konsulko.com> > Thanks Tim and Alassane, I know that one was a pain to get to the bottom of! I've merged it. Cheers, Richard
On Fri, Oct 27, 2023 at 12:33 AM Richard Purdie < richard.purdie@linuxfoundation.org> wrote: > On Thu, 2023-10-26 at 19:28 -0700, Tim Orling wrote: > > Tested in both local and container environments. > > > > On Thu, Oct 26, 2023 at 5:36 PM Alassane Yattara > > <alassane.yattara@savoirfairelinux.com> wrote: > > > Fixes "2efb14648 toaster: Monitoring - implement Django logging > > > system" when > > > running in a container. > > > > > > When running in a container, the previous approach of using > > > BASE_DIR > > > is not a writable path. Also, we really do not want to be writing > > > logs into > > > the source tree, as the BASE_DIR was resolving to > > > bitbake/lib/toaster/logs > > > > > > Since Toaster is only ever running in an environment where oe-init- > > > buildenv > > > or similar has been sourced, we should instead write the logs to > > > BUILDDIR. > > > > > > Using BUILDDIR to logs make path writable but django-log-viewer > > > does'nt manage > > > to write logs using an absolute path as BUILDDIR, where the > > > existing toaster_ui.log > > > was already being written. > > > > > > Also drop the /logs/ directory, as it has not been created which > > > also breaks > > > in a container environment > > > > > > To handle the constraints linked to django-log-viewer and /logs/, > > > we've updated > > > bitbake/bin/toaster to create a toaster_logs/ directory in BUILDDIR > > > if it doesn't exist, > > > when toaster starts up. > > > > > > Also manage to set BUILDDIR/toaster_logs/ as default location for > > > toaster logs. > > > > > > Signed-off-by: Alassane Yattara > > > <alassane.yattara@savoirfairelinux.com> > > > Signed-off-by: Tim Orling <tim.orling@konsulko.com> > > > > > > > Tested-by: Tim Orling <tim.orling@konsulko.com> > > > > Thanks Tim and Alassane, I know that one was a pain to get to the > bottom of! I've merged it. > Thank you, RP. And Alassane. GitHub actions agree: https://github.com/crops/toaster-container/actions/runs/6674512676 > Cheers, > > Richard > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#5902): > https://lists.yoctoproject.org/g/toaster/message/5902 > Mute This Topic: https://lists.yoctoproject.org/mt/102212389/924729 > Group Owner: toaster+owner@lists.yoctoproject.org > Unsubscribe: https://lists.yoctoproject.org/g/toaster/unsub [ > ticotimo@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
diff --git a/bin/toaster b/bin/toaster index 558a8195..f002c8c1 100755 --- a/bin/toaster +++ b/bin/toaster @@ -84,7 +84,7 @@ webserverStartAll() echo "Starting webserver..." $MANAGE runserver --noreload "$ADDR_PORT" \ - </dev/null >>${BUILDDIR}/toaster_web.log 2>&1 \ + </dev/null >>${TOASTER_LOGS_DIR}/web.log 2>&1 \ & echo $! >${BUILDDIR}/.toastermain.pid sleep 1 @@ -181,6 +181,14 @@ WEBSERVER=1 export TOASTER_BUILDSERVER=1 ADDR_PORT="localhost:8000" TOASTERDIR=`dirname $BUILDDIR` +# ${BUILDDIR}/toaster_logs/ became the default location for toaster logs +# This is needed for implemented django-log-viewer: https://pypi.org/project/django-log-viewer/ +# If the directory does not exist, create it. +TOASTER_LOGS_DIR="${BUILDDIR}/toaster_logs/" +if [ ! -d $TOASTER_LOGS_DIR ] +then + mkdir $TOASTER_LOGS_DIR +fi unset CMD for param in $*; do case $param in @@ -299,7 +307,7 @@ case $CMD in export BITBAKE_UI='toasterui' if [ $TOASTER_BUILDSERVER -eq 1 ] ; then $MANAGE runbuilds \ - </dev/null >>${BUILDDIR}/toaster_runbuilds.log 2>&1 \ + </dev/null >>${TOASTER_LOGS_DIR}/toaster_runbuilds.log 2>&1 \ & echo $! >${BUILDDIR}/.runbuilds.pid else echo "Toaster build server not started." diff --git a/lib/toaster/toastermain/logs.py b/lib/toaster/toastermain/logs.py index b4910e44..62d87196 100644 --- a/lib/toaster/toastermain/logs.py +++ b/lib/toaster/toastermain/logs.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import os import logging import json from pathlib import Path from django.http import HttpRequest -BASE_DIR = Path(__file__).resolve(strict=True).parent.parent - +BUILDDIR = Path(os.environ.get('BUILDDIR', '/tmp')) def log_api_request(request, response, view, logger_name='api'): """Helper function for LogAPIMixin""" @@ -108,7 +108,7 @@ LOGGING_SETTINGS = { 'file_django': { 'level': 'INFO', 'class': 'logging.handlers.TimedRotatingFileHandler', - 'filename': BASE_DIR / 'logs/django.log', + 'filename': BUILDDIR / 'toaster_logs/django.log', 'when': 'D', # interval type 'interval': 1, # defaults to 1 'backupCount': 10, # how many files to keep @@ -117,7 +117,7 @@ LOGGING_SETTINGS = { 'file_api': { 'level': 'INFO', 'class': 'logging.handlers.TimedRotatingFileHandler', - 'filename': BASE_DIR / 'logs/api.log', + 'filename': BUILDDIR / 'toaster_logs/api.log', 'when': 'D', 'interval': 1, 'backupCount': 10, @@ -126,7 +126,7 @@ LOGGING_SETTINGS = { 'file_toaster': { 'level': 'INFO', 'class': 'logging.handlers.TimedRotatingFileHandler', - 'filename': BASE_DIR / 'logs/toaster.log', + 'filename': BUILDDIR / 'toaster_logs/web.log', 'when': 'D', 'interval': 1, 'backupCount': 10, diff --git a/lib/toaster/toastermain/settings.py b/lib/toaster/toastermain/settings.py index b083cf58..5de881c4 100644 --- a/lib/toaster/toastermain/settings.py +++ b/lib/toaster/toastermain/settings.py @@ -316,12 +316,12 @@ for t in os.walk(os.path.dirname(currentdir)): LOGGING = LOGGING_SETTINGS # Build paths inside the project like this: BASE_DIR / 'subdir'. -BASE_DIR = Path(__file__).resolve(strict=True).parent.parent +BUILDDIR = os.environ.get("BUILDDIR", "/tmp") # LOG VIEWER # https://pypi.org/project/django-log-viewer/ LOG_VIEWER_FILES_PATTERN = '*.log*' -LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs') +LOG_VIEWER_FILES_DIR = os.path.join(BUILDDIR, "toaster_logs/") LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL']