From patchwork Thu Nov 17 15:29:26 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 15553 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5B0B2C43219 for ; Thu, 17 Nov 2022 15:29:34 +0000 (UTC) Received: from mail-wr1-f46.google.com (mail-wr1-f46.google.com [209.85.221.46]) by mx.groups.io with SMTP id smtpd.web10.18836.1668698972556497435 for ; Thu, 17 Nov 2022 07:29:32 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=Ofz6fWZ1; spf=pass (domain: linuxfoundation.org, ip: 209.85.221.46, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wr1-f46.google.com with SMTP id g12so4402377wrs.10 for ; Thu, 17 Nov 2022 07:29:32 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:message-id:date:subject:to :from:from:to:cc:subject:date:message-id:reply-to; bh=PX3GxPFqiDcbYTipsTgt+wBqsX/BStwJFOODAIlWUFs=; b=Ofz6fWZ1m2dgU552S4wB3tn6tsJ2a+Y2gYnsoy9zWE1AfbyovUvWDCThag+kP7hoLR SEZhMs/TNK776/ZH5cImZPMz6xm1wieWcBO/y39yjYqK7pekVQb93jyQptiv/vycrSQg 2E35hXWV1dKjEDtC+Mcb0CoWMJgRf0LxGUkRc= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:message-id:date:subject:to :from:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=PX3GxPFqiDcbYTipsTgt+wBqsX/BStwJFOODAIlWUFs=; b=JEGTJkCTp01gv4XrJIZffkjyPaPmj4GfZaQtRua2xdBe5OFLmub+ABL0mzRNJHifrj oJWf6q/5VGagT0ekcW4hvGDgyRbcaVTjayKJAHZYzAuC/ysBQCypFKpNcL9R1nY/Gdr5 nWScnnEOWkjj+pWrG9XxHQpzmWDDjDTAus1PHHwhOcXlqC3OWTUXsg+78nHe5zTPWluR cpt+nyRrvg33pYpNjIyPJVSV/au3JouLL1hCfGfypVRH4LXr/fWtAVhg5+JNpIhKV5Ms vrX1zsrwznMhl/LZxc1FxUfhh9Bn9DjNh/imlCFgK4iAizevlR1czDrm9MMevrQt1ysW JuJQ== X-Gm-Message-State: ANoB5pnP1yoTpfVPCJ9xEGaxJxvzlu6fin1E1YXYAn5dkPcTdTCoCKBQ J+ehKIO6s9y7QsBcpTzU4sx8zQvAX+pTWw== X-Google-Smtp-Source: AA0mqf5REVKAh6MgL0d3RQr+etoQgrI5wgM+R9I1JMCvBq+BBHEPEmddWGdcQE5BuvG6dGthtEGC1A== X-Received: by 2002:adf:e4cf:0:b0:241:a047:d011 with SMTP id v15-20020adfe4cf000000b00241a047d011mr1823370wrm.258.1668698969792; Thu, 17 Nov 2022 07:29:29 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:76d7:bdc9:5165:906c]) by smtp.gmail.com with ESMTPSA id j13-20020a5d604d000000b00236695ff94fsm1254306wrt.34.2022.11.17.07.29.29 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 17 Nov 2022 07:29:29 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 1/3] server: Ensure cooker profiling works Date: Thu, 17 Nov 2022 15:29:26 +0000 Message-Id: <20221117152928.2738113-1-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.34.1 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 17 Nov 2022 15:29:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14088 The previous cleanups meant that when the cooker was started, profiling was always disabled as configuration was sent to the server later and this was too late to profile the main loop. Pass the "profile" option over the server commandline so that we can profile cooker itself again, the setting can now take effect early enough. Signed-off-by: Richard Purdie --- bin/bitbake-server | 7 ++++--- lib/bb/main.py | 2 +- lib/bb/server/process.py | 8 +++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/bin/bitbake-server b/bin/bitbake-server index f53f88b6b0..825e9d56af 100755 --- a/bin/bitbake-server +++ b/bin/bitbake-server @@ -16,7 +16,7 @@ if sys.getfilesystemencoding() != "utf-8": sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.") # Users shouldn't be running this code directly -if len(sys.argv) != 10 or not sys.argv[1].startswith("decafbad"): +if len(sys.argv) != 11 or not sys.argv[1].startswith("decafbad"): print("bitbake-server is meant for internal execution by bitbake itself, please don't use it standalone.") sys.exit(1) @@ -28,7 +28,8 @@ logfile = sys.argv[4] lockname = sys.argv[5] sockname = sys.argv[6] timeout = float(sys.argv[7]) -xmlrpcinterface = (sys.argv[8], int(sys.argv[9])) +profile = bool(int(sys.argv[8])) +xmlrpcinterface = (sys.argv[9], int(sys.argv[10])) if xmlrpcinterface[0] == "None": xmlrpcinterface = (None, xmlrpcinterface[1]) @@ -49,5 +50,5 @@ logger = logging.getLogger("BitBake") handler = bb.event.LogHandler() logger.addHandler(handler) -bb.server.process.execServer(lockfd, readypipeinfd, lockname, sockname, timeout, xmlrpcinterface) +bb.server.process.execServer(lockfd, readypipeinfd, lockname, sockname, timeout, xmlrpcinterface, profile) diff --git a/lib/bb/main.py b/lib/bb/main.py index 93eda3632e..f56c3d46a2 100755 --- a/lib/bb/main.py +++ b/lib/bb/main.py @@ -429,7 +429,7 @@ def setup_bitbake(configParams, extrafeatures=None): logger.info("Starting bitbake server...") # Clear the event queue since we already displayed messages bb.event.ui_queue = [] - server = bb.server.process.BitBakeServer(lock, sockname, featureset, configParams.server_timeout, configParams.xmlrpcinterface) + server = bb.server.process.BitBakeServer(lock, sockname, featureset, configParams.server_timeout, configParams.xmlrpcinterface, configParams.profile) else: logger.info("Reconnecting to bitbake server...") diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 5d02c0b9f5..afd77ac0a5 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -448,13 +448,14 @@ start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f' class BitBakeServer(object): - def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface): + def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface, profile): self.server_timeout = server_timeout self.xmlrpcinterface = xmlrpcinterface self.featureset = featureset self.sockname = sockname self.bitbake_lock = lock + self.profile = profile self.readypipe, self.readypipein = os.pipe() # Place the log in the builddirectory alongside the lock file @@ -518,9 +519,9 @@ class BitBakeServer(object): os.set_inheritable(self.bitbake_lock.fileno(), True) os.set_inheritable(self.readypipein, True) serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server") - os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout or 0), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1])) + os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout or 0), str(int(self.profile)), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1])) -def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface): +def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface, profile): import bb.cookerdata import bb.cooker @@ -549,6 +550,7 @@ def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpc try: featureset = [] cooker = bb.cooker.BBCooker(featureset, server.register_idle_function) + cooker.configuration.profile = profile except bb.BBHandledException: return None writer.send("r") From patchwork Thu Nov 17 15:29:27 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 15554 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 59FADC4332F for ; Thu, 17 Nov 2022 15:29:34 +0000 (UTC) Received: from mail-wr1-f41.google.com (mail-wr1-f41.google.com [209.85.221.41]) by mx.groups.io with SMTP id smtpd.web11.19020.1668698973230291984 for ; Thu, 17 Nov 2022 07:29:33 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=ev4sOw22; spf=pass (domain: linuxfoundation.org, ip: 209.85.221.41, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wr1-f41.google.com with SMTP id g2so774519wrv.6 for ; Thu, 17 Nov 2022 07:29:32 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=jq6A+mNPu9L50A4WdKEIaGdSCb5z4HyjEIZQS/Bws+M=; b=ev4sOw22PT+GvHaL6nxxEX/9Qx1WtZ+cNPhi3bh8ZBPWb1PjZC5ZyAQbbJemcJjb2u YcW8FPuf4R+d9Btx1ecTA1xNAd8WXmVyYwsEIWiopwHf+rGbQAtaByOsYODZt26cpcdq Qd35c972H9RDKO/mgd2gSKqWu9Y1XAHZSTzK8= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=jq6A+mNPu9L50A4WdKEIaGdSCb5z4HyjEIZQS/Bws+M=; b=e2b6cJ8wGbb+o/43U8BS5ntA/G6OA28Cslsygxuud42tHLDu/QnI6YJHMvJDPnuAZk xwXbrXhizb11BZpBXB3T0FFITpQIb3w5m7is/F360X8TkQ668t6+WjT8MGEsb7o/3XWA 0vjj6PlgrD6q6df+9QkNkyAxqFbvZLzdfROmn3Bs7jvj5TZDe3PUv39pY2I4NGfxj72T +73G+QB4/iECt/lMqwiXBpVdfzEkwArEpgpXKSbV3jawsDbynTETyusN86qJlY280z+u g2Z3IVEKFUey9hdv8/A+WWsoe3vmX144H7fawvkpIA6gYUdCWAMz3gEbaRJ5L9OjAjg8 215Q== X-Gm-Message-State: ANoB5pkedgIzbsZSUUqaB3o1IK/Pmt5E/tJTBMxpnV+Bf0epP1ykJZUJ t7bQgCHHUAaBy9T+HRme/wpaCfxP9BNdRw== X-Google-Smtp-Source: AA0mqf4XKcW1e5TXLC8stMLZKrHaTUN1IF2QImmzJd767LVD8lMPJ4ONGk+hCxKUfKgNZz6g9Eknfw== X-Received: by 2002:adf:f206:0:b0:236:698d:9c44 with SMTP id p6-20020adff206000000b00236698d9c44mr1791503wro.571.1668698970714; Thu, 17 Nov 2022 07:29:30 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:76d7:bdc9:5165:906c]) by smtp.gmail.com with ESMTPSA id j13-20020a5d604d000000b00236695ff94fsm1254306wrt.34.2022.11.17.07.29.29 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 17 Nov 2022 07:29:30 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 2/3] worker/runqueue: Reduce initial data transfer in workerdata Date: Thu, 17 Nov 2022 15:29:27 +0000 Message-Id: <20221117152928.2738113-2-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221117152928.2738113-1-richard.purdie@linuxfoundation.org> References: <20221117152928.2738113-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 17 Nov 2022 15:29:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14089 When setting up the worker we were transfering large amounts of data which aren't needed until task execution time. Defer the fakeroot and taskdeps data until they're needed for a specific task. This will duplicate some information when executing different tasks for a given recipe but as is is spread over the build run, it shouldn't be an issue overall. Also take the opportunity to clean up the silly length argument lists that were being passed around at the expense of extra dictionary keys. Signed-off-by: Richard Purdie --- bin/bitbake-worker | 32 +++++++++++++++++++--------- lib/bb/runqueue.py | 52 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/bin/bitbake-worker b/bin/bitbake-worker index d54044f361..3799b170cb 100755 --- a/bin/bitbake-worker +++ b/bin/bitbake-worker @@ -145,7 +145,16 @@ def sigterm_handler(signum, frame): os.killpg(0, signal.SIGTERM) sys.exit() -def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False): +def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask): + + fn = runtask['fn'] + task = runtask['task'] + taskname = runtask['taskname'] + taskhash = runtask['taskhash'] + unihash = runtask['unihash'] + appends = runtask['appends'] + taskdepdata = runtask['taskdepdata'] + quieterrors = runtask['quieterrors'] # We need to setup the environment BEFORE the fork, since # a fork() or exec*() activates PSEUDO... @@ -157,8 +166,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha uid = os.getuid() gid = os.getgid() - - taskdep = workerdata["taskdeps"][fn] + taskdep = runtask['taskdep'] if 'umask' in taskdep and taskname in taskdep['umask']: umask = taskdep['umask'][taskname] elif workerdata["umask"]: @@ -170,24 +178,24 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha except TypeError: pass - dry_run = cfg.dry_run or dry_run_exec + dry_run = cfg.dry_run or runtask['dry_run'] # We can't use the fakeroot environment in a dry run as it possibly hasn't been built if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run: fakeroot = True - envvars = (workerdata["fakerootenv"][fn] or "").split() + envvars = (runtask['fakerootenv'] or "").split() for key, value in (var.split('=') for var in envvars): envbackup[key] = os.environ.get(key) os.environ[key] = value fakeenv[key] = value - fakedirs = (workerdata["fakerootdirs"][fn] or "").split() + fakedirs = (runtask['fakerootdirs'] or "").split() for p in fakedirs: bb.utils.mkdirhier(p) logger.debug2('Running %s:%s under fakeroot, fakedirs: %s' % (fn, taskname, ', '.join(fakedirs))) else: - envvars = (workerdata["fakerootnoenv"][fn] or "").split() + envvars = (runtask['fakerootnoenv'] or "").split() for key, value in (var.split('=') for var in envvars): envbackup[key] = os.environ.get(key) os.environ[key] = value @@ -474,11 +482,15 @@ class BitbakeWorker(object): sys.exit(0) def handle_runtask(self, data): - fn, task, taskname, taskhash, unihash, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data) - workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname)) + runtask = pickle.loads(data) - pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec) + fn = runtask['fn'] + task = runtask['task'] + taskname = runtask['taskname'] + + workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname)) + pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, self.extraconfigdata, runtask) self.build_pids[pid] = task self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout) diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index 437f4a185c..595a58883e 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -1311,10 +1311,6 @@ class RunQueue: workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec, fakerootlogs=fakerootlogs) workerdata = { - "taskdeps" : self.rqdata.dataCaches[mc].task_deps, - "fakerootenv" : self.rqdata.dataCaches[mc].fakerootenv, - "fakerootdirs" : self.rqdata.dataCaches[mc].fakerootdirs, - "fakerootnoenv" : self.rqdata.dataCaches[mc].fakerootnoenv, "sigdata" : bb.parse.siggen.get_taskdata(), "logdefaultlevel" : bb.msg.loggerDefaultLogLevel, "build_verbose_shell" : self.cooker.configuration.build_verbose_shell, @@ -2139,18 +2135,30 @@ class RunQueueExecute: startevent = sceneQueueTaskStarted(task, self.stats, self.rq) bb.event.fire(startevent, self.cfgData) - taskdepdata = self.sq_build_taskdepdata(task) - taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] - taskhash = self.rqdata.get_task_hash(task) - unihash = self.rqdata.get_task_unihash(task) + runtask = { + 'fn' : taskfn, + 'task' : task, + 'taskname' : taskname, + 'taskhash' : self.rqdata.get_task_hash(task), + 'unihash' : self.rqdata.get_task_unihash(task), + 'quieterrors' : True, + 'appends' : self.cooker.collections[mc].get_file_appends(taskfn), + 'taskdepdata' : self.sq_build_taskdepdata(task), + 'dry_run' : False, + 'taskdep': taskdep, + 'fakerootenv' : self.rqdata.dataCaches[mc].fakerootenv[taskfn], + 'fakerootdirs' : self.rqdata.dataCaches[mc].fakerootdirs[taskfn], + 'fakerootnoenv' : self.rqdata.dataCaches[mc].fakerootnoenv[taskfn] + } + if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: if not mc in self.rq.fakeworker: self.rq.start_fakeworker(self, mc) - self.rq.fakeworker[mc].process.stdin.write(b"" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"") + self.rq.fakeworker[mc].process.stdin.write(b"" + pickle.dumps(runtask) + b"") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"") + self.rq.worker[mc].process.stdin.write(b"" + pickle.dumps(runtask) + b"") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) @@ -2220,11 +2228,23 @@ class RunQueueExecute: startevent = runQueueTaskStarted(task, self.stats, self.rq) bb.event.fire(startevent, self.cfgData) - taskdepdata = self.build_taskdepdata(task) - taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] - taskhash = self.rqdata.get_task_hash(task) - unihash = self.rqdata.get_task_unihash(task) + runtask = { + 'fn' : taskfn, + 'task' : task, + 'taskname' : taskname, + 'taskhash' : self.rqdata.get_task_hash(task), + 'unihash' : self.rqdata.get_task_unihash(task), + 'quieterrors' : False, + 'appends' : self.cooker.collections[mc].get_file_appends(taskfn), + 'taskdepdata' : self.build_taskdepdata(task), + 'dry_run' : self.rqdata.setscene_enforce, + 'taskdep': taskdep, + 'fakerootenv' : self.rqdata.dataCaches[mc].fakerootenv[taskfn], + 'fakerootdirs' : self.rqdata.dataCaches[mc].fakerootdirs[taskfn], + 'fakerootnoenv' : self.rqdata.dataCaches[mc].fakerootnoenv[taskfn] + } + if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce): if not mc in self.rq.fakeworker: try: @@ -2234,10 +2254,10 @@ class RunQueueExecute: self.rq.state = runQueueFailed self.stats.taskFailed() return True - self.rq.fakeworker[mc].process.stdin.write(b"" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"") + self.rq.fakeworker[mc].process.stdin.write(b"" + pickle.dumps(runtask) + b"") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"") + self.rq.worker[mc].process.stdin.write(b"" + pickle.dumps(runtask) + b"") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) From patchwork Thu Nov 17 15:29:28 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 15555 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 58794C433FE for ; Thu, 17 Nov 2022 15:29:34 +0000 (UTC) Received: from mail-wr1-f42.google.com (mail-wr1-f42.google.com [209.85.221.42]) by mx.groups.io with SMTP id smtpd.web11.19021.1668698973650117050 for ; Thu, 17 Nov 2022 07:29:33 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=H4tYWGog; spf=pass (domain: linuxfoundation.org, ip: 209.85.221.42, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wr1-f42.google.com with SMTP id bs21so4432633wrb.4 for ; Thu, 17 Nov 2022 07:29:33 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=dDAPxJhxCKkySoQH7dMKP5S43JiG7QFk+vO3o+fR/mE=; b=H4tYWGogYP31G10X6GO1sBVdQ2aU5GjlkDfljKW14lvuFKLXVv2OMp9MePiPezrV9g NBf7+neWbMkA8rUKwWI51rdSgRy7DbBTHPJt4nnWY8n5d1Rf45Cn88n0chVhx0FOJHHV Q/37/uH6kui0/8O3RXjL9jZ1lXi7+Dbkj9nRw= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=dDAPxJhxCKkySoQH7dMKP5S43JiG7QFk+vO3o+fR/mE=; b=kgb828+CpafY0SCGdnpee+gZZQMOXVYwFTIHfFwyYmN0cEPM0AxxBXqfWwhyxHMsaQ 6Lu46gJr5dm3w3Vn1XplratKRevdiz072q7MDl8A4/MKOl1UteVwHW0ISwFerV4TueBh 2Ntbco3nppSNO1bHQgXQuxjmm7YIaH/J69IvH8/K98AIv+RVXLqVtgNRVzMf5pVSnQnJ +ZCynvc3avtoZ0rAJj7HB/SDpydrZp3HwAL4BZnJ4CyZbMgKqO8tNuUYhrqwrkJ0cTGn UtHWOlrres0YBncOsb5LSuQjJv+NVRKZKmHVNH6mHP1z2odx6Bzfm8DWXDbDWw2vbdn1 0Crg== X-Gm-Message-State: ANoB5pkAsNVgakwfg4m+X8A0wm69pPaS6yXNMpN7u55WrflGcRf/IG+a yZf+31mH4SZ3BLFxjWXuCK+tMnA0rXC4zw== X-Google-Smtp-Source: AA0mqf5QO+ITW1DzhGg4jGwr+COAH2rF0jBpfhehcuUB5xbX02vzFZdPonHjz4r7d4EoDDZC62EjaA== X-Received: by 2002:adf:e7cd:0:b0:236:960f:161d with SMTP id e13-20020adfe7cd000000b00236960f161dmr1886273wrn.376.1668698971530; Thu, 17 Nov 2022 07:29:31 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:76d7:bdc9:5165:906c]) by smtp.gmail.com with ESMTPSA id j13-20020a5d604d000000b00236695ff94fsm1254306wrt.34.2022.11.17.07.29.30 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 17 Nov 2022 07:29:31 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 3/3] cache: Drop support for not saving the cache file Date: Thu, 17 Nov 2022 15:29:28 +0000 Message-Id: <20221117152928.2738113-3-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221117152928.2738113-1-richard.purdie@linuxfoundation.org> References: <20221117152928.2738113-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 17 Nov 2022 15:29:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14090 We don't test not using the cache and I'm not aware of anyone using this, it would be hard to with modern bitbake. Drop the conditional code and simply error if CACHE isn't set. Signed-off-by: Richard Purdie --- lib/bb/cache.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/lib/bb/cache.py b/lib/bb/cache.py index 22ed3e80be..cb6cf21727 100644 --- a/lib/bb/cache.py +++ b/lib/bb/cache.py @@ -309,20 +309,12 @@ class Cache(object): self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+') if self.cachedir in [None, '']: - self.has_cache = False - self.logger.info("Not using a cache. " - "Set CACHE = to enable.") - return - - self.has_cache = True + bb.fatal("Please ensure CACHE is set to the cache directory for BitBake to use") def getCacheFile(self, cachefile): return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash) def prepare_cache(self, progress): - if not self.has_cache: - return 0 - loaded = 0 self.cachefile = self.getCacheFile("bb_cache.dat") @@ -361,9 +353,6 @@ class Cache(object): return loaded def cachesize(self): - if not self.has_cache: - return 0 - cachesize = 0 for cache_class in self.caches_array: cachefile = self.getCacheFile(cache_class.cachefile) @@ -472,10 +461,6 @@ class Cache(object): """ if fn not in self.checked: self.cacheValidUpdate(fn, appends) - - # Is cache enabled? - if not self.has_cache: - return False if fn in self.clean: return True return False @@ -485,10 +470,6 @@ class Cache(object): Is the cache valid for fn? Make thorough (slower) checks including timestamps. """ - # Is cache enabled? - if not self.has_cache: - return False - self.checked.add(fn) # File isn't in depends_cache @@ -595,10 +576,6 @@ class Cache(object): Save the cache Called from the parser when complete (or exiting) """ - - if not self.has_cache: - return - if self.cacheclean: self.logger.debug2("Cache is clean, not saving.") return @@ -641,9 +618,6 @@ class Cache(object): if watcher: watcher(info_array[0].file_depends) - if not self.has_cache: - return - if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache: if parsed: self.cacheclean = False