Disallow SHAKE in OAEP decryption in FIPS mode

This was already blocked for encryption and for both signature creation
and verification in RSASSA-PSS, but RSA-OAEP decryption was missing.

Resolves: rhbz#2142121
Signed-off-by: Clemens Lang <cllang@redhat.com>

0085-FIPS-RSA-disable-shake.patch

@@ -1,7 +1,32 @@
-diff -up openssl-3.0.1/crypto/rsa/rsa_oaep.c.oaep openssl-3.0.1/crypto/rsa/rsa_oaep.c
---- openssl-3.0.1/crypto/rsa/rsa_oaep.c.oaep	2022-11-14 13:45:05.970402064 +0100
-+++ openssl-3.0.1/crypto/rsa/rsa_oaep.c	2022-11-14 13:51:20.725741198 +0100
-@@ -78,8 +78,22 @@ int ossl_rsa_padding_add_PKCS1_OAEP_mgf1
+From 52b347703ba2b98a0efee86c1a483c2f0f9f73d6 Mon Sep 17 00:00:00 2001
+From: Clemens Lang <cllang@redhat.com>
+Date: Wed, 11 Jan 2023 12:52:59 +0100
+Subject: [PATCH] rsa: Disallow SHAKE in OAEP and PSS in FIPS prov
+
+According to FIPS 140-3 IG, section C.C, the SHAKE digest algorithms
+must not be used in higher-level algorithms (such as RSA-OAEP and
+RSASSA-PSS):
+
+"To be used in an approved mode of operation, the SHA-3 hash functions
+may be implemented either as part of an approved higher-level algorithm,
+for example, a digital signature algorithm, or as the standalone
+functions. The SHAKE128 and SHAKE256 extendable-output functions may
+only be used as the standalone algorithms."
+
+Add a check to prevent their use as message digest in PSS signatures and
+as MGF1 hash function in both OAEP and PSS.
+
+Signed-off-by: Clemens Lang <cllang@redhat.com>
+---
+ crypto/rsa/rsa_oaep.c | 28 ++++++++++++++++++++++++++++
+ crypto/rsa/rsa_pss.c  | 16 ++++++++++++++++
+ 2 files changed, 44 insertions(+)
+
+diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c
+index d9be1a4f98..dfe9c9f0e8 100644
+--- a/crypto/rsa/rsa_oaep.c
++++ b/crypto/rsa/rsa_oaep.c
+@@ -73,9 +73,23 @@ int ossl_rsa_padding_add_PKCS1_OAEP_mgf1_ex(OSSL_LIB_CTX *libctx,
          return 0;
  #endif
      }
@@ -14,20 +39,46 @@ diff -up openssl-3.0.1/crypto/rsa/rsa_oaep.c.oaep openssl-3.0.1/crypto/rsa/rsa_o
 +#endif
      if (mgf1md == NULL)
          mgf1md = md;
-+
+ 
 +#ifdef FIPS_MODULE
 +    if (EVP_MD_is_a(mgf1md, "SHAKE-128") || EVP_MD_is_a(mgf1md, "SHAKE-256")) {
 +        ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED);
 +        return 0;
 +    }
 +#endif
- 
++
      mdlen = EVP_MD_get_size(md);
      if (mdlen <= 0) {
-diff -up openssl-3.0.1/crypto/rsa/rsa_pss.c.oaep openssl-3.0.1/crypto/rsa/rsa_pss.c
---- openssl-3.0.1/crypto/rsa/rsa_pss.c.oaep	2022-11-15 14:53:11.103467808 +0100
-+++ openssl-3.0.1/crypto/rsa/rsa_pss.c	2022-11-15 15:00:07.233966865 +0100
-@@ -53,6 +53,14 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa,
+         ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_LENGTH);
+@@ -181,9 +195,23 @@ int RSA_padding_check_PKCS1_OAEP_mgf1(unsigned char *to, int tlen,
+ #endif
+     }
+ 
++#ifdef FIPS_MODULE
++    if (EVP_MD_is_a(md, "SHAKE-128") || EVP_MD_is_a(md, "SHAKE-256")) {
++        ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED);
++        return -1;
++    }
++#endif
++
+     if (mgf1md == NULL)
+         mgf1md = md;
+ 
++#ifdef FIPS_MODULE
++    if (EVP_MD_is_a(mgf1md, "SHAKE-128") || EVP_MD_is_a(mgf1md, "SHAKE-256")) {
++        ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_NOT_ALLOWED);
++        return -1;
++    }
++#endif
++
+     mdlen = EVP_MD_get_size(md);
+ 
+     if (tlen <= 0 || flen <= 0)
+diff --git a/crypto/rsa/rsa_pss.c b/crypto/rsa/rsa_pss.c
+index 33874bfef8..e8681b0351 100644
+--- a/crypto/rsa/rsa_pss.c
++++ b/crypto/rsa/rsa_pss.c
+@@ -53,6 +53,14 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
      if (mgf1Hash == NULL)
          mgf1Hash = Hash;
  
@@ -42,7 +93,7 @@ diff -up openssl-3.0.1/crypto/rsa/rsa_pss.c.oaep openssl-3.0.1/crypto/rsa/rsa_ps
      hLen = EVP_MD_get_size(Hash);
      if (hLen < 0)
          goto err;
-@@ -164,6 +172,14 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *
+@@ -164,6 +172,14 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
      if (mgf1Hash == NULL)
          mgf1Hash = Hash;
  
@@ -57,3 +108,6 @@ diff -up openssl-3.0.1/crypto/rsa/rsa_pss.c.oaep openssl-3.0.1/crypto/rsa/rsa_ps
      hLen = EVP_MD_get_size(Hash);
      if (hLen < 0)
          goto err;
+-- 
+2.39.0
+

openssl.spec

@@ -29,7 +29,7 @@ print(string.sub(hash, 0, 16))
 Summary: Utilities from the general purpose cryptography library with TLS implementation
 Name: openssl
 Version: 3.0.7
-Release: 3%{?dist}
+Release: 4%{?dist}
 Epoch: 1
 # We have to remove certain patented algorithms from the openssl source
 # tarball with the hobble-openssl script which is included below.
@@ -484,6 +484,10 @@ install -m644 %{SOURCE9} \
 %ldconfig_scriptlets libs
 
 %changelog
+* Wed Jan 11 2023 Clemens Lang <cllang@redhat.com> - 1:3.0.7-4
+- Disallow SHAKE in RSA-OAEP decryption in FIPS mode
+  Resolves: rhbz#2142121
+
 * Thu Jan 05 2023 Dmitry Belyavskiy <dbelyavs@redhat.com> - 1:3.0.7-3
 - Refactor OpenSSL fips module MAC verification
   Resolves: rhbz#2157965