qpdf/SOURCES/qpdf-relax.patch

diff -up qpdf-11.7.0/libqpdf/QPDF.cc.relax qpdf-11.7.0/libqpdf/QPDF.cc
--- qpdf-11.7.0/libqpdf/QPDF.cc.relax	2023-12-24 13:36:07.000000000 +0100
+++ qpdf-11.7.0/libqpdf/QPDF.cc	2024-01-08 15:17:27.020951639 +0100
@@ -13,6 +13,10 @@
 #include <sstream>
 #include <vector>
 
+#ifdef USE_CRYPTO_GNUTLS
+# include <gnutls/crypto.h>
+#endif
+
 #include <qpdf/BufferInputSource.hh>
 #include <qpdf/FileInputSource.hh>
 #include <qpdf/OffsetInputSource.hh>
@@ -250,14 +254,26 @@ void
 QPDF::processFile(char const* filename, char const* password)
 {
     auto* fi = new FileInputSource(filename);
+#ifdef USE_CRYPTO_GNUTLS
+    GNUTLS_FIPS140_SET_LAX_MODE();
+#endif
     processInputSource(std::shared_ptr<InputSource>(fi), password);
+#ifdef USE_CRYPTO_GNUTLS
+    GNUTLS_FIPS140_SET_STRICT_MODE();
+#endif
 }
 
 void
 QPDF::processFile(char const* description, FILE* filep, bool close_file, char const* password)
 {
     auto* fi = new FileInputSource(description, filep, close_file);
+#ifdef USE_CRYPTO_GNUTLS
+    GNUTLS_FIPS140_SET_LAX_MODE();
+#endif
     processInputSource(std::shared_ptr<InputSource>(fi), password);
+#ifdef USE_CRYPTO_GNUTLS
+    GNUTLS_FIPS140_SET_STRICT_MODE();
+#endif
 }
 
 void
diff -up qpdf-11.7.0/libqpdf/QPDF_encryption.cc.relax qpdf-11.7.0/libqpdf/QPDF_encryption.cc
--- qpdf-11.7.0/libqpdf/QPDF_encryption.cc.relax	2023-12-24 13:36:07.000000000 +0100
+++ qpdf-11.7.0/libqpdf/QPDF_encryption.cc	2024-01-08 15:19:52.303117277 +0100
@@ -3,6 +3,8 @@
 
 #include <qpdf/assert_debug.h>
 
+#include <qpdf/qpdf-config.h>
+
 #include <qpdf/QPDF.hh>
 
 #include <qpdf/QPDFExc.hh>
@@ -19,6 +21,10 @@
 #include <algorithm>
 #include <cstring>
 
+#ifdef USE_CRYPTO_GNUTLS
+# include <gnutls/crypto.h>
+#endif
+
 static unsigned char const padding_string[] = {
     0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41, 0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08,
     0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80, 0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a};
@@ -349,10 +355,21 @@ QPDF::compute_data_key(
         result += "sAlT";
     }
 
+#ifdef USE_CRYPTO_GNUTLS
+    unsigned oldmode = gnutls_fips140_mode_enabled();
+
+    gnutls_fips140_set_mode(GNUTLS_FIPS140_LAX, GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
     MD5 md5;
     md5.encodeDataIncrementally(result.c_str(), result.length());
     MD5::Digest digest;
     md5.digest(digest);
+
+#ifdef USE_CRYPTO_GNUTLS
+    gnutls_fips140_set_mode(static_cast<gnutls_fips_mode_t>(oldmode), GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
     return {reinterpret_cast<char*>(digest), std::min(result.length(), toS(16))};
 }
 
@@ -976,6 +993,12 @@ QPDF::getKeyForObject(
 void
 QPDF::decryptString(std::string& str, QPDFObjGen const& og)
 {
+#ifdef USE_CRYPTO_GNUTLS
+    unsigned oldmode = gnutls_fips140_mode_enabled();
+
+    gnutls_fips140_set_mode(GNUTLS_FIPS140_LAX, GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
     if (!og.isIndirect()) {
         return;
     }
@@ -1036,6 +1059,10 @@ QPDF::decryptString(std::string& str, QP
     } catch (std::runtime_error& e) {
         throw damagedPDF("error decrypting string for object " + og.unparse() + ": " + e.what());
     }
+
+#ifdef USE_CRYPTO_GNUTLS
+    gnutls_fips140_set_mode(static_cast<gnutls_fips_mode_t>(oldmode), GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
 }
 
 // Prepend a decryption pipeline to 'pipeline'. The decryption pipeline (returned as
@@ -1051,6 +1078,12 @@ QPDF::decryptStream(
     QPDFObjectHandle& stream_dict,
     std::unique_ptr<Pipeline>& decrypt_pipeline)
 {
+#ifdef USE_CRYPTO_GNUTLS
+    unsigned oldmode = gnutls_fips140_mode_enabled();
+
+    gnutls_fips140_set_mode(GNUTLS_FIPS140_LAX, GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
     std::string type;
     if (stream_dict.getKey("/Type").isName()) {
         type = stream_dict.getKey("/Type").getName();
@@ -1151,6 +1184,10 @@ QPDF::decryptStream(
             toI(key.length()));
     }
     pipeline = decrypt_pipeline.get();
+
+#ifdef USE_CRYPTO_GNUTLS
+    gnutls_fips140_set_mode(static_cast<gnutls_fips_mode_t>(oldmode), GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
 }
 
 void
diff -up qpdf-11.7.0/libqpdf/QPDFWriter.cc.relax qpdf-11.7.0/libqpdf/QPDFWriter.cc
--- qpdf-11.7.0/libqpdf/QPDFWriter.cc.relax	2023-12-24 13:36:07.000000000 +0100
+++ qpdf-11.7.0/libqpdf/QPDFWriter.cc	2024-01-08 15:17:27.022951614 +0100
@@ -26,6 +26,10 @@
 #include <cstdlib>
 #include <stdexcept>
 
+#ifdef USE_CRYPTO_GNUTLS
+#include <gnutls/crypto.h>
+#endif
+
 QPDFWriter::ProgressReporter::~ProgressReporter() // NOLINT (modernize-use-equals-default)
 {
     // Must be explicit and not inline -- see QPDF_DLL_CLASS in README-maintainer
@@ -287,6 +291,13 @@ void
 QPDFWriter::setDeterministicID(bool val)
 {
     m->deterministic_id = val;
+
+#ifdef USE_CRYPTO_GNUTLS
+    if (val)
+	GNUTLS_FIPS140_SET_LAX_MODE();
+    else
+	GNUTLS_FIPS140_SET_STRICT_MODE();
+#endif
 }
 
 void
@@ -307,6 +318,13 @@ void
 QPDFWriter::setPreserveEncryption(bool val)
 {
     m->preserve_encryption = val;
+
+#ifdef USE_CRYPTO_GNUTLS
+    if (val)
+	GNUTLS_FIPS140_SET_STRICT_MODE();
+    else
+	GNUTLS_FIPS140_SET_LAX_MODE();
+#endif
 }
 
 void
@@ -1890,11 +1908,21 @@ QPDFWriter::generateID()
             }
         }
 
+#ifdef USE_CRYPTO_GNUTLS
+        unsigned oldmode = gnutls_fips140_mode_enabled();
+
+        gnutls_fips140_set_mode(GNUTLS_FIPS140_LAX, GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
         MD5 m;
         m.encodeString(seed.c_str());
         MD5::Digest digest;
         m.digest(digest);
         result = std::string(reinterpret_cast<char*>(digest), sizeof(MD5::Digest));
+
+#ifdef USE_CRYPTO_GNUTLS
+        gnutls_fips140_set_mode(static_cast<gnutls_fips_mode_t>(oldmode), GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
     }
 
     // If /ID already exists, follow the spec: use the original first word and generate a new second