diff mbox series

[meta-oe] softhsm: fix build with OpenSSL 4.0

Message ID 20260909202539.260304-1-alperyasinak1@gmail.com
State Under Review
Headers show
Series [meta-oe] softhsm: fix build with OpenSSL 4.0 | expand

Commit Message

Alper Ak Sept. 9, 2026, 8:25 p.m. UTC 
do_compile fails with:

  OSSLUtil.cpp:188:46: error: invalid use of incomplete type
  'ASN1_PRINTABLESTRING' {aka 'struct asn1_string_st'}

Backport the upstream fix. It was written for a memory leak, but it
moves to the ASN1_STRING accessors, which is also what OpenSSL 4 needs.

Signed-off-by: Alper Ak <alperyasinak1@gmail.com>
---
 ...n-EdDSA-PrintableString-handling-858.patch | 89 +++++++++++++++++++
 .../recipes-security/softhsm/softhsm_2.7.0.bb |  1 +
 2 files changed, 90 insertions(+)
 create mode 100644 meta-oe/recipes-security/softhsm/softhsm/0003-Fix-memory-leak-in-EdDSA-PrintableString-handling-858.patch
diff mbox series

Patch

diff --git a/meta-oe/recipes-security/softhsm/softhsm/0003-Fix-memory-leak-in-EdDSA-PrintableString-handling-858.patch b/meta-oe/recipes-security/softhsm/softhsm/0003-Fix-memory-leak-in-EdDSA-PrintableString-handling-858.patch
new file mode 100644
index 0000000000..71319ea5e7
--- /dev/null
+++ b/meta-oe/recipes-security/softhsm/softhsm/0003-Fix-memory-leak-in-EdDSA-PrintableString-handling-858.patch
@@ -0,0 +1,89 @@ 
+From: =?UTF-8?q?Ma=C5=82gorzata=20Olsz=C3=B3wka?= <Malgorzata.Olszowka@stunnel.org>
+Date: Sun, 3 May 2026 13:00:07 +0200
+Subject: [PATCH] Fix memory leak in EdDSA PrintableString handling (#858)
+
+ASN1_STRING is opaque in OpenSSL 4, so curve_name->data no longer
+compiles. Use the accessors and free the string on every path.
+
+Upstream-Status: Backport [https://github.com/softhsm/SoftHSMv2/commit/d23ea09d318c03c033420d065f1c64b019cc94ed]
+Signed-off-by: Alper Ak <alperyasinak1@gmail.com>
+---
+ src/lib/crypto/OSSLUtil.cpp | 45 ++++++++++++++++++++++++++-----------
+ 1 file changed, 32 insertions(+), 13 deletions(-)
+
+--- a/src/lib/crypto/OSSLUtil.cpp
++++ b/src/lib/crypto/OSSLUtil.cpp
+@@ -160,12 +160,14 @@
+ // Convert a ByteString to an OpenSSL EVP_PKEY id
+ int OSSL::byteString2oid(const ByteString& byteString)
+ {
+-	ASN1_OBJECT *oid = NULL;
+-	ASN1_PRINTABLESTRING *curve_name = NULL;
++	ASN1_OBJECT *oid;
++	ASN1_PRINTABLESTRING *curve_name;
+ 	const unsigned char *p = byteString.const_byte_str();
+ 	const unsigned char *pp = p;
++	const unsigned char *data;
+ 	long length;
+-	int tag, pclass;
++	int tag, pclass, data_len;
++	int nid = NID_undef;
+ 
+ 	ASN1_get_object(&pp, &length, &tag, &pclass, byteString.size());
+ 	if (pclass == V_ASN1_UNIVERSAL && tag == V_ASN1_OBJECT)
+@@ -178,34 +180,49 @@
+ 			return NID_undef;
+ 		}
+ 
+-		return OBJ_obj2nid(oid);
++		nid = OBJ_obj2nid(oid);
++		ASN1_OBJECT_free(oid);
+ 	}
+ 	else if (pclass == V_ASN1_UNIVERSAL && tag == V_ASN1_PRINTABLESTRING)
+ 	{
+ 		/* The final PKCS#11 3.0 expects curve name encoded as PrintableString */
+ 		curve_name = d2i_ASN1_PRINTABLESTRING(NULL, &p, byteString.size());
+ 
+-		if (strcmp((char *)curve_name->data, "edwards25519") == 0)
++		if (curve_name == NULL)
+ 		{
++			return NID_undef;
++		}
++
++		data = ASN1_STRING_get0_data(curve_name);
++		data_len = ASN1_STRING_length(curve_name);
++
++		if (data_len == 12 && memcmp(data, "edwards25519", data_len) == 0)
++		{
++			ASN1_PRINTABLESTRING_free(curve_name);
+ 			return EVP_PKEY_ED25519;
+ 		}
+ 
+-		if (strcmp((char *)curve_name->data, "curve25519") == 0)
++		if (data_len == 10 && memcmp(data, "curve25519", 10) == 0)
+ 		{
++			ASN1_PRINTABLESTRING_free(curve_name);
+ 			return EVP_PKEY_X25519;
+ 		}
+ 
+-		if (strcmp((char *)curve_name->data, "edwards448") == 0)
++		if (data_len == 10 && memcmp(data, "edwards448", 10) == 0)
+ 		{
++			ASN1_PRINTABLESTRING_free(curve_name);
+ 			return EVP_PKEY_ED448;
+ 		}
+ 
+-		if (strcmp((char *)curve_name->data, "curve448") == 0)
++		if (data_len == 8 && memcmp(data, "curve448", 8) == 0)
+ 		{
++			ASN1_PRINTABLESTRING_free(curve_name);
+ 			return EVP_PKEY_X448;
+ 		}
++
++		ASN1_PRINTABLESTRING_free(curve_name);
+ 	}
+ 
+-	return NID_undef;
++	return nid;
+ }
+ #endif
diff --git a/meta-oe/recipes-security/softhsm/softhsm_2.7.0.bb b/meta-oe/recipes-security/softhsm/softhsm_2.7.0.bb
index 9c4d1251fd..1b2660a2e3 100644
--- a/meta-oe/recipes-security/softhsm/softhsm_2.7.0.bb
+++ b/meta-oe/recipes-security/softhsm/softhsm_2.7.0.bb
@@ -7,6 +7,7 @@  DEPENDS = "sqlite3"
 
 SRC_URI = "git://github.com/softhsm/SoftHSMv2.git;protocol=https;branch=main;tag=${PV} \
            file://0002-Prevent-accessing-of-global-c-objects-once-they-are-.patch \
+           file://0003-Fix-memory-leak-in-EdDSA-PrintableString-handling-858.patch \
 "
 SRCREV = "13e6e86b83748fef74046dbf0c91f664b7acc1c3"