From patchwork Wed Jul 17 18:22:13 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Alexander Kanavin X-Patchwork-Id: 46562 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 05BDCC3DA60 for ; Wed, 17 Jul 2024 18:22:51 +0000 (UTC) Received: from mail-lj1-f180.google.com (mail-lj1-f180.google.com [209.85.208.180]) by mx.groups.io with SMTP id smtpd.web10.320.1721240561767405306 for ; Wed, 17 Jul 2024 11:22:42 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=EMv6eiVC; spf=pass (domain: gmail.com, ip: 209.85.208.180, mailfrom: alex.kanavin@gmail.com) Received: by mail-lj1-f180.google.com with SMTP id 38308e7fff4ca-2ebe40673e8so14701fa.3 for ; Wed, 17 Jul 2024 11:22:41 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1721240559; x=1721845359; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:message-id:date:subject:cc :to:from:from:to:cc:subject:date:message-id:reply-to; bh=vXI9Aog+t6CCsaSLBhYJScPX0YQX6T/lzu75FrjzTRI=; b=EMv6eiVCJQ10qsXeAWNtW8faS0Hy9mO4AN6vIUq5Oainkxv2SjZPIXqBtaWyU8O0nO VCRLXyTBE01ecoLFB3aVhfn/FhaA8L8QFP10iXBJKhbcq4AMiaLsSFk5TYj72ZgCDpQF ygAWIZ85sj2Rn34OV2QX/KAxXurfyXcLYRwI9hhta5fRxoru6G6Yrz5RoPJFahKpKrB6 wpesf9S+9LEawmCYTBMyBZGn32fSr7iMImCGWteLGktpNcLYBJulY/EAj2//WGMC9zXE nW30bNhpVmbyIRl93M3crdw+x5lLnK7N1mThVC4sTmzZ3CBkt0W/HwijvnO5GRLqurCU 2Kwg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1721240559; x=1721845359; h=content-transfer-encoding:mime-version:message-id:date:subject:cc :to:from:x-gm-message-state:from:to:cc:subject:date:message-id :reply-to; bh=vXI9Aog+t6CCsaSLBhYJScPX0YQX6T/lzu75FrjzTRI=; b=lSpYVb1EB4h44UKvm2+lxspBuqiWd6R1f3dh8iBVDQFz6wY1hY4bcRaOCGq2GDN1U5 te9bxszWUFaTr2Pyy/cufzINYbeg0oz+Q99lZLQjZ6V/i7pzGkJpGngH2QEXsRSu0NL+ jASLitc8Ir1r1BSsXZS2ZC20XFrSCorekj2Hiq0B3Ew6fHHjZxLubNgB3ttUxiuBAptg emP+D/yelARMNaV9gC8RoZvGEGYvUaXRaGex6jnZOLKtcbn9ATNh8le9a1Iltjyx2f/Y I9Dlge43MpoR5FbWb/7JZjGuZm0Ig1NbznklOu/i1yrNfHyOTO/ayFh+7G+LEXf+JJi1 6Pew== X-Gm-Message-State: AOJu0YzAG81/tn4wwqX40clcDEmiRAJivItlx7xJHLgWtsEXjR6PtcTb ZShYx9UY7Y0P972NQjiXrC7Vc9D9CgC2B7LPDdZoj9p6YEXbUQYci8nEjw== X-Google-Smtp-Source: AGHT+IH2WFXH1KfQtQSd3NaDQNlBsxDR1DtVvzla6DTTPwigfDobz1gPXBxuYRTSLAjZ8h1komLWBg== X-Received: by 2002:a05:6512:280d:b0:52e:954d:3592 with SMTP id 2adb3069b0e04-52ee5446158mr1629790e87.46.1721240558998; Wed, 17 Jul 2024 11:22:38 -0700 (PDT) Received: from Zen2.lab.linutronix.de. (drugstore.linutronix.de. [80.153.143.164]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-3680db0489csm12421868f8f.104.2024.07.17.11.22.38 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 17 Jul 2024 11:22:38 -0700 (PDT) From: Alexander Kanavin To: openembedded-core@lists.openembedded.org Cc: Alexander Kanavin Subject: [PATCH 1/4] lib/oe/recipeutils: return a dict in get_recipe_upgrade_status() instead of a tuple Date: Wed, 17 Jul 2024 20:22:13 +0200 Message-Id: <20240717182216.1661015-1-alex.kanavin@gmail.com> X-Mailer: git-send-email 2.39.2 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 17 Jul 2024 18:22:51 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/202173 From: Alexander Kanavin Putting various things in a tuple is an anti-pattern of sorts, as the consumers have to unpack it into local variables for readability, or access items directly with indexes, which makes code pretty much unreadable. Signed-off-by: Alexander Kanavin --- meta/lib/oe/recipeutils.py | 2 +- meta/lib/oeqa/selftest/cases/distrodata.py | 4 ++-- scripts/lib/devtool/upgrade.py | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py index 2d69a331132..f9d7dfe253a 100644 --- a/meta/lib/oe/recipeutils.py +++ b/meta/lib/oe/recipeutils.py @@ -1112,7 +1112,7 @@ def _get_recipe_upgrade_status(data): maintainer = data.getVar('RECIPE_MAINTAINER') no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON') - return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason) + return {'pn':pn, 'status':status, 'cur_ver':cur_ver, 'next_ver':next_ver, 'maintainer':maintainer, 'revision':revision, 'no_upgrade_reason':no_upgrade_reason} def get_recipe_upgrade_status(recipes=None): pkgs_list = [] diff --git a/meta/lib/oeqa/selftest/cases/distrodata.py b/meta/lib/oeqa/selftest/cases/distrodata.py index ad952c004b5..b60913dbca4 100644 --- a/meta/lib/oeqa/selftest/cases/distrodata.py +++ b/meta/lib/oeqa/selftest/cases/distrodata.py @@ -22,8 +22,8 @@ class Distrodata(OESelftestTestCase): pkgs = oe.recipeutils.get_recipe_upgrade_status() - regressed_failures = [pkg[0] for pkg in pkgs if pkg[1] == 'UNKNOWN_BROKEN'] - regressed_successes = [pkg[0] for pkg in pkgs if pkg[1] == 'KNOWN_BROKEN'] + regressed_failures = [pkg['pn'] for pkg in pkgs if pkg['status'] == 'UNKNOWN_BROKEN'] + regressed_successes = [pkg['pn'] for pkg in pkgs if pkg['status'] == 'KNOWN_BROKEN'] msg = "" if len(regressed_failures) > 0: msg = msg + """ diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py index 8e13833b51c..10b4f8b5ee5 100644 --- a/scripts/lib/devtool/upgrade.py +++ b/scripts/lib/devtool/upgrade.py @@ -659,13 +659,13 @@ def check_upgrade_status(args, config, basepath, workspace): results = oe.recipeutils.get_recipe_upgrade_status(args.recipe) for result in results: # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason - if args.all or result[1] != 'MATCH': - print("{:25} {:15} {:15} {} {} {}".format( result[0], - result[2], - result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"), - result[4], - result[5] if result[5] != 'N/A' else "", - "cannot be updated due to: %s" %(result[6]) if result[6] else "")) + if args.all or result['status'] != 'MATCH': + print("{:25} {:15} {:15} {} {} {}".format( result['pn'], + result['cur_ver'], + result['status'] if result['status'] != 'UPDATE' else (result['next_ver'] if not result['next_ver'].endswith("new-commits-available") else "new commits"), + result['maintainer'], + result['revision'] if result['revision'] != 'N/A' else "", + "cannot be updated due to: %s" %(result['no_upgrade_reason']) if result['no_upgrade_reason'] else "")) def register_commands(subparsers, context): """Register devtool subcommands from this plugin""" From patchwork Wed Jul 17 18:22:14 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Alexander Kanavin X-Patchwork-Id: 46560 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 9C48BC41513 for ; Wed, 17 Jul 2024 18:22:50 +0000 (UTC) Received: from mail-wr1-f44.google.com (mail-wr1-f44.google.com [209.85.221.44]) by mx.groups.io with SMTP id smtpd.web11.316.1721240561464283615 for ; Wed, 17 Jul 2024 11:22:41 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=dN0eHnnQ; spf=pass (domain: gmail.com, ip: 209.85.221.44, mailfrom: alex.kanavin@gmail.com) Received: by mail-wr1-f44.google.com with SMTP id ffacd0b85a97d-367ab50a07aso14712f8f.0 for ; Wed, 17 Jul 2024 11:22:41 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1721240560; x=1721845360; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=69fcwcZEUKmUl8CGjBbiiCEWPKiT3PFxyEVSfOxcHcU=; b=dN0eHnnQt2/oSeKaj7JJzByPphFwliQw3HmcVH85DNxpcQ8GLMjvUmlyQZdZy5bsU0 hpw7cuWwKQdLRf268NgETJqE6JyRLq5bsjPS1D5FoPcfJ4uR0DAxNKI9pvKhU61hSIRW tgQTqFhX7OS8aWU33KPaNDUUaB3oPHeb3mZwR9rQwqvvaqXsMZxSAghTpp/Z7Si394GM zLK4yECgFYeGba2iaFcX3MjAeq5TXTF8DBeomFkVSuGqsoB2FAJzUUwY0UUrwSTwafZy 0NZ1a6knsbgoco2f9UkdjipIMRS/919P4OHnP88RYjMXgV5p5fupRsKmsfdwlhhR1qds aIwg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1721240560; x=1721845360; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=69fcwcZEUKmUl8CGjBbiiCEWPKiT3PFxyEVSfOxcHcU=; b=siaUdG1t36vcNw17YrDT3CEVvNgOAOk19V7kNv6tIEZ4PcPY6JqB2Sk1iEtBqZvvrB mAN3kEe9BEZPTo9Spb2VO11H9y9klJX54MxdZ7Gu7N1beb2a/8dR3UM8Y9rgwxYdTlA4 DFqteB0DSknA8nWoYHzya2dJ4XAzLo1lEcbXISfzm98WY5c9hrBWexCdXj7WNyHpctDe lO1oQmqovwu+JRalV9zHMAuBbWedffWFGUQcnf7EA33rNpfjMjba0ssbQTuY8hqLziNf U7QnObEmihxV/nzOzVbVXRxgH8SjT0BsywouAbBXXYD1UVGq35I2k9N/0KsM2HjmjFc9 NjFA== X-Gm-Message-State: AOJu0YylzSRPyo5qVuS2hLGqgXCvEIZmH5OZkv9r9mx5jKMClZkyYB3a 0zcGR6YKMqRDBD6sHO+ZdhDpMpWK5m6NzEYexpAMQSVhUzYtmnPPIYTS8g== X-Google-Smtp-Source: AGHT+IFkOuHZC9NtswA91NyhQLVEQ9Cu+aPXqf4D3PceLEtAKPPI2GuSC4eWzvKNAPMH8NTEUbrscQ== X-Received: by 2002:a05:6000:c8e:b0:367:8a87:ada2 with SMTP id ffacd0b85a97d-36831659bfcmr1617435f8f.26.1721240559543; Wed, 17 Jul 2024 11:22:39 -0700 (PDT) Received: from Zen2.lab.linutronix.de. (drugstore.linutronix.de. [80.153.143.164]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-3680db0489csm12421868f8f.104.2024.07.17.11.22.39 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 17 Jul 2024 11:22:39 -0700 (PDT) From: Alexander Kanavin To: openembedded-core@lists.openembedded.org Cc: Alexander Kanavin Subject: [PATCH 2/4] lib/recipeutils: add a function to determine recipes with shared include files Date: Wed, 17 Jul 2024 20:22:14 +0200 Message-Id: <20240717182216.1661015-2-alex.kanavin@gmail.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240717182216.1661015-1-alex.kanavin@gmail.com> References: <20240717182216.1661015-1-alex.kanavin@gmail.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 17 Jul 2024 18:22:50 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/202172 From: Alexander Kanavin This functionality is needed for 'lockstep version upgrades' where several recipes need to be upgraded at the same time to produce a buildable outcome. The function itself obtains BBINCLUDED for each recipe and then massages the data until it takes the form of a list of sets: [{'cmake','cmake-native'}, {'qemu','qemu-native','qemu-system-native'}, ... ] There's also a selftest that checks for the above. Unfortunately this won't detect mutually exclusive recipes like mesa and mesa-gl as they're chosen with PREFERRED_PROVIDER and can't be enabled in the same build at the same time. ('devtool upgrade' will also accept just one of them but not the other) Signed-off-by: Alexander Kanavin --- meta/lib/oe/recipeutils.py | 37 ++++++++++++++++++++++ meta/lib/oeqa/selftest/cases/distrodata.py | 12 +++++++ 2 files changed, 49 insertions(+) diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py index f9d7dfe253a..7586332fe0c 100644 --- a/meta/lib/oe/recipeutils.py +++ b/meta/lib/oe/recipeutils.py @@ -1183,3 +1183,40 @@ def get_recipe_upgrade_status(recipes=None): pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list) return pkgs_list + +def get_common_include_recipes(): + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False) + + recipes = tinfoil.all_recipe_files(variants=False) + + recipeincludes = {} + for fn in recipes: + data = tinfoil.parse_recipe_file(fn) + recipeincludes[fn] = {'bbincluded':data.getVar('BBINCLUDED').split(),'pn':data.getVar('PN')} + return _get_common_include_recipes(recipeincludes) + +def _get_common_include_recipes(recipeincludes_all): + recipeincludes = {} + for fn,data in recipeincludes_all.items(): + bbincluded_filtered = [i for i in data['bbincluded'] if os.path.dirname(i) == os.path.dirname(fn) and i != fn] + if bbincluded_filtered: + recipeincludes[data['pn']] = bbincluded_filtered + + recipeincludes_inverted = {} + for k,v in recipeincludes.items(): + for i in v: + recipeincludes_inverted.setdefault(i,set()).add(k) + + recipeincludes_inverted_filtered = {k:v for k,v in recipeincludes_inverted.items() if len(v) > 1} + + recipes_with_shared_includes = list() + for v in recipeincludes_inverted_filtered.values(): + recipeset = v + for v1 in recipeincludes_inverted_filtered.values(): + if recipeset.intersection(v1): + recipeset.update(v1) + if recipeset not in recipes_with_shared_includes: + recipes_with_shared_includes.append(recipeset) + + return recipes_with_shared_includes diff --git a/meta/lib/oeqa/selftest/cases/distrodata.py b/meta/lib/oeqa/selftest/cases/distrodata.py index b60913dbca4..bc561605220 100644 --- a/meta/lib/oeqa/selftest/cases/distrodata.py +++ b/meta/lib/oeqa/selftest/cases/distrodata.py @@ -115,3 +115,15 @@ The list of oe-core recipes with maintainers is empty. This may indicate that th self.fail(""" Unable to find recipes for the following entries in maintainers.inc: """ + "\n".join(['%s' % i for i in missing_recipes])) + + def test_common_include_recipes(self): + """ + Summary: Test that obtaining recipes that share includes between them returns a sane result + Expected: At least cmake and qemu entries are present in the output + Product: oe-core + Author: Alexander Kanavin + """ + recipes = oe.recipeutils.get_common_include_recipes() + + self.assertIn({'qemu-system-native', 'qemu', 'qemu-native'}, recipes) + self.assertIn({'cmake-native', 'cmake'}, recipes) From patchwork Wed Jul 17 18:22:15 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Alexander Kanavin X-Patchwork-Id: 46561 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 9B5B9C3DA62 for ; Wed, 17 Jul 2024 18:22:50 +0000 (UTC) Received: from mail-wr1-f46.google.com (mail-wr1-f46.google.com [209.85.221.46]) by mx.groups.io with SMTP id smtpd.web11.317.1721240562006271089 for ; Wed, 17 Jul 2024 11:22:42 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=JERQv/Wa; spf=pass (domain: gmail.com, ip: 209.85.221.46, mailfrom: alex.kanavin@gmail.com) Received: by mail-wr1-f46.google.com with SMTP id ffacd0b85a97d-36844375001so14622f8f.0 for ; Wed, 17 Jul 2024 11:22:41 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1721240560; x=1721845360; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=bU06XgkugksKhEWVpJGRNjs8HkpCMYrJK4WwAGkWESk=; b=JERQv/WaGvREaVWv5VMbBgW9Ggf+KhGnwBTZJJMXX8oRPlbwmNf7pWuNDETZkxhRqZ KRzi+dY0qd1EowQA3y27xSri7h3tDy+DrRcSbsLz3ZLKNfNKGsnhhi09VVHyD5B5lG/i e0kUBBWBx+dAhTQYTh9qsMxJCDDhr80TCHUgHyYg4OWwLIUzDfWZYMcgPiFM+KjIdjpS S9Rdvfqn1xVcxSXPqJVXLeaq+ZcZmr/LK2415RgB7uZiLvoYl5k3lQUeng1w69auwY+J SMPGIVZhV1g2+XNENjEZFisgrelTG7K0ZHSDw1oLkN5PWHVJVgNGdc5lhmX8xPQqwhwm 3Heg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1721240560; x=1721845360; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=bU06XgkugksKhEWVpJGRNjs8HkpCMYrJK4WwAGkWESk=; b=Vqx2BBwea+RSDmTj4S85zxE1GfExig+id7gdHYq3cg8nYCJPW8rapj+ba1MgqKVUqe syI/N0+7txy/VjyrEDbZaVcR+dF7ifJnHuzYE+agU/ZJO1YUBh2AeJivr0UOkVr/9QpB WmKUjsrClshRhkSLj7500huxnjNOlrHIMdunnRnMOclIjth2g1qkEldghNGTldAJHgCO QFz1QusBVLx8+63W8m9PzN4P6jWqLc9X0BNetI1mDzuysltPSigxiSzhBocZJpPjoA6n T/02QOOV4XtwO5jz4tTmx19uqJljv82a2F1zznpgM3h7d9z5LL6X0q7yZN+NfB4Jb7nv REEg== X-Gm-Message-State: AOJu0YxlTSZ2IT41vHg0BOjMkUqgMq9CbP+HgruzkxsPKSUR6BO9lPab kGxR6nWpB0ie0pb9hfqh+AVOlRL3Rp79CNl7ijGly89d7+550zcB71aX3A== X-Google-Smtp-Source: AGHT+IHpB1O5iFF2IvCaIcGW5Vum+E5EAG64xw/u5HEta6oa6ojBlQXWVjK5ask/KKIn92Zxa/aACg== X-Received: by 2002:a5d:4843:0:b0:367:4e1a:240e with SMTP id ffacd0b85a97d-3683175d0cemr1736090f8f.50.1721240560093; Wed, 17 Jul 2024 11:22:40 -0700 (PDT) Received: from Zen2.lab.linutronix.de. (drugstore.linutronix.de. [80.153.143.164]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-3680db0489csm12421868f8f.104.2024.07.17.11.22.39 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 17 Jul 2024 11:22:39 -0700 (PDT) From: Alexander Kanavin To: openembedded-core@lists.openembedded.org Cc: Alexander Kanavin Subject: [PATCH 3/4] recipeutils/get_recipe_upgrade_status: group recipes when they need to be upgraded together Date: Wed, 17 Jul 2024 20:22:15 +0200 Message-Id: <20240717182216.1661015-3-alex.kanavin@gmail.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240717182216.1661015-1-alex.kanavin@gmail.com> References: <20240717182216.1661015-1-alex.kanavin@gmail.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 17 Jul 2024 18:22:50 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/202174 From: Alexander Kanavin This will allow 'lockstep upgrades' of such recipes, improving success rates in automated version updating process. devtool check-upgrade-status now prints: These recipes need to be upgraded together { glib-2.0 2.80.2 2.80.4 Anuj Mittal glib-2.0-initial 2.80.2 2.80.4 Anuj Mittal } These recipes need to be upgraded together { util-linux 2.39.3 2.40.2 Chen Qi util-linux-libuuid 2.39.3 2.40.2 Chen Qi } These recipes need to be upgraded together { cmake 3.29.3 3.30.0 Unassigned cmake-native 3.29.3 3.30.0 Unassigned } etc. Signed-off-by: Alexander Kanavin --- meta/lib/oe/recipeutils.py | 23 +++++++++++++++++- meta/lib/oeqa/selftest/cases/distrodata.py | 6 ++--- scripts/lib/devtool/upgrade.py | 28 +++++++++++++++------- 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py index 7586332fe0c..56be75dc9c9 100644 --- a/meta/lib/oe/recipeutils.py +++ b/meta/lib/oe/recipeutils.py @@ -1154,6 +1154,7 @@ def get_recipe_upgrade_status(recipes=None): if not recipes: recipes = tinfoil.all_recipe_files(variants=False) + recipeincludes = {} for fn in recipes: try: if fn.startswith("/"): @@ -1178,11 +1179,13 @@ def get_recipe_upgrade_status(recipes=None): data_copy_list.append(data_copy) + recipeincludes[data.getVar('FILE')] = {'bbincluded':data.getVar('BBINCLUDED').split(),'pn':data.getVar('PN')} + from concurrent.futures import ProcessPoolExecutor with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor: pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list) - return pkgs_list + return _group_recipes(pkgs_list, _get_common_include_recipes(recipeincludes)) def get_common_include_recipes(): with bb.tinfoil.Tinfoil() as tinfoil: @@ -1220,3 +1223,21 @@ def _get_common_include_recipes(recipeincludes_all): recipes_with_shared_includes.append(recipeset) return recipes_with_shared_includes + +def _group_recipes(recipes, groups): + recipedict = {} + for r in recipes: + recipedict[r['pn']] = r + + recipegroups = [] + for g in groups: + recipeset = [] + for r in g: + if r in recipedict.keys(): + recipeset.append(recipedict[r]) + del recipedict[r] + recipegroups.append(recipeset) + + for r in recipedict.values(): + recipegroups.append([r]) + return recipegroups diff --git a/meta/lib/oeqa/selftest/cases/distrodata.py b/meta/lib/oeqa/selftest/cases/distrodata.py index bc561605220..bd375523649 100644 --- a/meta/lib/oeqa/selftest/cases/distrodata.py +++ b/meta/lib/oeqa/selftest/cases/distrodata.py @@ -20,10 +20,10 @@ class Distrodata(OESelftestTestCase): feature = 'LICENSE_FLAGS_ACCEPTED += " commercial"\n' self.write_config(feature) - pkgs = oe.recipeutils.get_recipe_upgrade_status() + pkggroups = oe.recipeutils.get_recipe_upgrade_status() - regressed_failures = [pkg['pn'] for pkg in pkgs if pkg['status'] == 'UNKNOWN_BROKEN'] - regressed_successes = [pkg['pn'] for pkg in pkgs if pkg['status'] == 'KNOWN_BROKEN'] + regressed_failures = [pkg['pn'] for pkgs in pkggroups for pkg in pkgs if pkg['status'] == 'UNKNOWN_BROKEN'] + regressed_successes = [pkg['pn'] for pkgs in pkggroups for pkg in pkgs if pkg['status'] == 'KNOWN_BROKEN'] msg = "" if len(regressed_failures) > 0: msg = msg + """ diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py index 10b4f8b5ee5..4c268af3a7c 100644 --- a/scripts/lib/devtool/upgrade.py +++ b/scripts/lib/devtool/upgrade.py @@ -654,18 +654,28 @@ def latest_version(args, config, basepath, workspace): return 0 def check_upgrade_status(args, config, basepath, workspace): + def _print_status(recipe): + print("{:25} {:15} {:15} {} {} {}".format( recipe['pn'], + recipe['cur_ver'], + recipe['status'] if recipe['status'] != 'UPDATE' else (recipe['next_ver'] if not recipe['next_ver'].endswith("new-commits-available") else "new commits"), + recipe['maintainer'], + recipe['revision'] if recipe['revision'] != 'N/A' else "", + "cannot be updated due to: %s" %(recipe['no_upgrade_reason']) if recipe['no_upgrade_reason'] else "")) if not args.recipe: logger.info("Checking the upstream status for all recipes may take a few minutes") results = oe.recipeutils.get_recipe_upgrade_status(args.recipe) - for result in results: - # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason - if args.all or result['status'] != 'MATCH': - print("{:25} {:15} {:15} {} {} {}".format( result['pn'], - result['cur_ver'], - result['status'] if result['status'] != 'UPDATE' else (result['next_ver'] if not result['next_ver'].endswith("new-commits-available") else "new commits"), - result['maintainer'], - result['revision'] if result['revision'] != 'N/A' else "", - "cannot be updated due to: %s" %(result['no_upgrade_reason']) if result['no_upgrade_reason'] else "")) + for recipegroup in results: + upgrades = [r for r in recipegroup if r['status'] != 'MATCH'] + currents = [r for r in recipegroup if r['status'] == 'MATCH'] + if len(upgrades) > 1: + print("These recipes need to be upgraded together {") + for r in upgrades: + _print_status(r) + if len(upgrades) > 1: + print("}") + for r in currents: + if args.all: + _print_status(r) def register_commands(subparsers, context): """Register devtool subcommands from this plugin""" From patchwork Wed Jul 17 18:22:16 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Alexander Kanavin X-Patchwork-Id: 46563 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id B7DDBC3DA63 for ; Wed, 17 Jul 2024 18:22:50 +0000 (UTC) Received: from mail-wr1-f43.google.com (mail-wr1-f43.google.com [209.85.221.43]) by mx.groups.io with SMTP id smtpd.web10.321.1721240562408722317 for ; Wed, 17 Jul 2024 11:22:42 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=BqGHU/qC; spf=pass (domain: gmail.com, ip: 209.85.221.43, mailfrom: alex.kanavin@gmail.com) Received: by mail-wr1-f43.google.com with SMTP id ffacd0b85a97d-3683329f787so5223f8f.1 for ; Wed, 17 Jul 2024 11:22:42 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1721240561; x=1721845361; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=Fd9F+oHepi4Po6B9jzD0FE/KVTJ11qgndkEoPu+clp0=; b=BqGHU/qCIPaZQ/9vCR3EkVXCdX0PUP0E92oUHH7XURTagqUNjl0WABCWn9kirHNKiQ DA/Lx2ibHQgR/3lGjs7RNAEeG8vUa+xruT6JZzY5CTgC/ud+ekKA3t50lQlqzelbIyHv g9RLjsuHogt9LHNGqRx44UeENE+soBfaAcpP3fzrnRStvutd0wbrUQjAmB22dKW9WRoJ EEUFO8cb89WiNy8BseLyzt3+scjHqrBO1Vnzdul/51JoqgTLQ3jsWL0LakJT2LS5T0v3 Ro8awdlaCuPw+U0CUNPm3nY9Wm39SmqYjAR4Tg+EBs+loVdcbLflomBeSWy+2kafdQdo qHrQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1721240561; x=1721845361; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=Fd9F+oHepi4Po6B9jzD0FE/KVTJ11qgndkEoPu+clp0=; b=HNMpIaU98sFgio+YHJUuqYRPYuoBYWBK4SdG8ZvDGMaFse2l5eGCQeDU0elX6ox6mh rf8yd2pIqmxudSPKC3pYxW+yXWTGQd7+lKS86zU4IIYi28uKjmsDzxfGzifTAPU0E52I fYxCTXJ/Ec678s7Ij04ZJI0SVTpfdY8Fev7ikfGqxPv1KMEttvO96dmdEgeMpULT/zLo 9Zl+Bka3XAJ89gUlvWSAQeB2PGYFOwf2K5lwwJKyKPLY7enFKkgaHotB4NQPPnDYC2bN cLm2nLAALqy7oJ97qw7YgOum/FtvGsc0ifYb6VA6bEWltg5RotrbZtzRXMzItd0Itfut Hk4w== X-Gm-Message-State: AOJu0YwP4Lc3RufQ5WIN9vMtFFGvU+d8jZXrTvd6N6DPKrEs1CUVOxNN qlrcDLeVrPxKAoCE6y/oWE6D/IDh/O4GN/bDzbnB1ZxArRVf92CyxcJfrg== X-Google-Smtp-Source: AGHT+IFg5q8ikwbh2AVRhRRdMUrdM2SKPzfTzwBZa5hUvGDMvOGLnrROaQF4i+eYhc14AsdhflvotQ== X-Received: by 2002:a5d:5f4b:0:b0:368:4492:eca9 with SMTP id ffacd0b85a97d-3684492f176mr748500f8f.20.1721240560766; Wed, 17 Jul 2024 11:22:40 -0700 (PDT) Received: from Zen2.lab.linutronix.de. (drugstore.linutronix.de. [80.153.143.164]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-3680db0489csm12421868f8f.104.2024.07.17.11.22.40 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 17 Jul 2024 11:22:40 -0700 (PDT) From: Alexander Kanavin To: openembedded-core@lists.openembedded.org Cc: Alexander Kanavin Subject: [PATCH 4/4] devtool/upgrade: use PN instead of BPN for naming newly created upgraded recipes Date: Wed, 17 Jul 2024 20:22:16 +0200 Message-Id: <20240717182216.1661015-4-alex.kanavin@gmail.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240717182216.1661015-1-alex.kanavin@gmail.com> References: <20240717182216.1661015-1-alex.kanavin@gmail.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 17 Jul 2024 18:22:50 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/202175 From: Alexander Kanavin BPN isn't correct, as it is set to 'cmake' when 'cmake-native' is being upgraded (or libva for libva-initial etc.) Signed-off-by: Alexander Kanavin --- scripts/lib/devtool/upgrade.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py index 4c268af3a7c..9ec66488516 100644 --- a/scripts/lib/devtool/upgrade.py +++ b/scripts/lib/devtool/upgrade.py @@ -76,19 +76,19 @@ def _rename_recipe_dirs(oldpv, newpv, path): bb.utils.rename(os.path.join(path, oldfile), os.path.join(path, newfile)) -def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path): +def _rename_recipe_file(oldrecipe, pn, oldpv, newpv, path): oldrecipe = os.path.basename(oldrecipe) if oldrecipe.endswith('_%s.bb' % oldpv): - newrecipe = '%s_%s.bb' % (bpn, newpv) + newrecipe = '%s_%s.bb' % (pn, newpv) if oldrecipe != newrecipe: shutil.move(os.path.join(path, oldrecipe), os.path.join(path, newrecipe)) else: newrecipe = oldrecipe return os.path.join(path, newrecipe) -def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path): +def _rename_recipe_files(oldrecipe, pn, oldpv, newpv, path): _rename_recipe_dirs(oldpv, newpv, path) - return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path) + return _rename_recipe_file(oldrecipe, pn, oldpv, newpv, path) def _write_append(rc, srctreebase, srctree, same_dir, no_same_dir, revs, copied, workspace, d): """Writes an append file""" @@ -335,19 +335,19 @@ def _add_license_diff_to_recipe(path, diff): def _create_new_recipe(newpv, checksums, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure): """Creates the new recipe under workspace""" - bpn = rd.getVar('BPN') - path = os.path.join(workspace, 'recipes', bpn) + pn = rd.getVar('PN') + path = os.path.join(workspace, 'recipes', pn) bb.utils.mkdirhier(path) copied, _ = oe.recipeutils.copy_recipe_files(rd, path, all_variants=True) if not copied: - raise DevtoolError('Internal error - no files were copied for recipe %s' % bpn) + raise DevtoolError('Internal error - no files were copied for recipe %s' % pn) logger.debug('Copied %s to %s' % (copied, path)) oldpv = rd.getVar('PV') if not newpv: newpv = oldpv origpath = rd.getVar('FILE') - fullpath = _rename_recipe_files(origpath, bpn, oldpv, newpv, path) + fullpath = _rename_recipe_files(origpath, pn, oldpv, newpv, path) logger.debug('Upgraded %s => %s' % (origpath, fullpath)) newvalues = {} @@ -610,7 +610,7 @@ def upgrade(args, config, basepath, workspace): license_diff = _generate_license_diff(old_licenses, new_licenses) rf, copied = _create_new_recipe(args.version, checksums, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure) except (bb.process.CmdError, DevtoolError) as e: - recipedir = os.path.join(config.workspace_path, 'recipes', rd.getVar('BPN')) + recipedir = os.path.join(config.workspace_path, 'recipes', rd.getVar('PN')) _upgrade_error(e, recipedir, srctree, args.keep_failure) standard._add_md5(config, pn, os.path.dirname(rf))