SHAKE-128/256 are not allowed with RSA in FIPS mode

Resolves: rhbz#2144010

0085-FIPS-RSA-disable-shake.patch

@@ -0,0 +1,59 @@
+diff -up openssl-3.0.1/crypto/rsa/rsa_oaep.c.oaep openssl-3.0.1/crypto/rsa/rsa_oaep.c
+--- openssl-3.0.1/crypto/rsa/rsa_oaep.c.oaep	2022-11-14 13:45:05.970402064 +0100
++++ openssl-3.0.1/crypto/rsa/rsa_oaep.c	2022-11-14 13:51:20.725741198 +0100
+@@ -78,8 +78,22 @@ int ossl_rsa_padding_add_PKCS1_OAEP_mgf1
+         return 0;
+ #endif
+     }
++
++#ifdef FIPS_MODULE
++    if (EVP_MD_is_a(md, "SHAKE-128") || EVP_MD_is_a(md, "SHAKE-256")) {
++        ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED);
++        return 0;
++    }
++#endif
+     if (mgf1md == NULL)
+         mgf1md = md;
++
++#ifdef FIPS_MODULE
++    if (EVP_MD_is_a(mgf1md, "SHAKE-128") || EVP_MD_is_a(mgf1md, "SHAKE-256")) {
++        ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED);
++        return 0;
++    }
++#endif
+ 
+     mdlen = EVP_MD_get_size(md);
+     if (mdlen <= 0) {
+diff -up openssl-3.0.1/crypto/rsa/rsa_pss.c.oaep openssl-3.0.1/crypto/rsa/rsa_pss.c
+--- openssl-3.0.1/crypto/rsa/rsa_pss.c.oaep	2022-11-15 14:53:11.103467808 +0100
++++ openssl-3.0.1/crypto/rsa/rsa_pss.c	2022-11-15 15:00:07.233966865 +0100
+@@ -53,6 +53,14 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa,
+     if (mgf1Hash == NULL)
+         mgf1Hash = Hash;
+ 
++#ifdef FIPS_MODULE
++    if (EVP_MD_is_a(Hash, "SHAKE-128") || EVP_MD_is_a(Hash, "SHAKE-256"))
++        goto err;
++
++    if (EVP_MD_is_a(mgf1Hash, "SHAKE-128") || EVP_MD_is_a(mgf1Hash, "SHAKE-256"))
++        goto err;
++#endif
++
+     hLen = EVP_MD_get_size(Hash);
+     if (hLen < 0)
+         goto err;
+@@ -164,6 +172,14 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *
+     if (mgf1Hash == NULL)
+         mgf1Hash = Hash;
+ 
++#ifdef FIPS_MODULE
++    if (EVP_MD_is_a(Hash, "SHAKE-128") || EVP_MD_is_a(Hash, "SHAKE-256"))
++        goto err;
++
++    if (EVP_MD_is_a(mgf1Hash, "SHAKE-128") || EVP_MD_is_a(mgf1Hash, "SHAKE-256"))
++        goto err;
++#endif
++
+     hLen = EVP_MD_get_size(Hash);
+     if (hLen < 0)
+         goto err;

openssl.spec

@@ -29,7 +29,7 @@ print(string.sub(hash, 0, 16))
 Summary: Utilities from the general purpose cryptography library with TLS implementation
 Name: openssl
 Version: 3.0.1
-Release: 43%{?dist}
+Release: 44%{?dist}
 Epoch: 1
 # We have to remove certain patented algorithms from the openssl source
 # tarball with the hobble-openssl script which is included below.
@@ -164,6 +164,8 @@ Patch77: 0077-FIPS-140-3-zeroization.patch
 Patch78: 0078-Add-FIPS-indicator-parameter-to-HKDF.patch
 #https://bugzilla.redhat.com/show_bug.cgi?id=2137723
 Patch79: 0079-CVE-2022-3602.patch
+#https://bugzilla.redhat.com/show_bug.cgi?id=2142121
+Patch85: 0085-FIPS-RSA-disable-shake.patch
 
 License: ASL 2.0
 URL: http://www.openssl.org/
@@ -494,6 +496,10 @@ install -m644 %{SOURCE9} \
 %ldconfig_scriptlets libs
 
 %changelog
+* Mon Nov 14 2022 Dmitry Belyavskiy <dbelyavs@redhat.com> - 1:3.0.1-44
+- SHAKE-128/256 are not allowed with RSA in FIPS mode
+  Resolves: rhbz#2144010
+
 * Tue Nov 01 2022 Dmitry Belyavskiy <dbelyavs@redhat.com> - 1:3.0.1-43
 - CVE-2022-3602: X.509 Email Address Buffer Overflow
 - CVE-2022-3786: X.509 Email Address Buffer Overflow