transmission/openssl3-compat.patch

Description: Compatibility with OpenSSL 3
 We rely on RC4 because of the torrent protocol we're implementing, but this
 is no longer available in the default provider.
Author: Steve Langasek <steve.langasek@ubuntu.com>
Bug-Ubuntu: https://bugs.launchpad.net/bugs/1946215
Last-Update: 2021-12-13
Forwarded: no

Index: transmission-3.00/libtransmission/crypto-utils-openssl.c
===================================================================
--- libtransmission/crypto-utils-openssl.c
+++ libtransmission/crypto-utils-openssl.c
@@ -20,6 +20,9 @@
 #include <openssl/rand.h>
 #include <openssl/ssl.h>
 #include <openssl/x509.h>
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#include <openssl/provider.h>
+#endif
 
 #include "transmission.h"
 #include "crypto-utils.h"
@@ -182,46 +185,86 @@
 
 #endif
 
+typedef struct tr_rc4_ctx {
+    EVP_CIPHER_CTX *cipher_ctx;
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    OSSL_LIB_CTX *lib_ctx;
+#endif
+} tr_rc4_ctx;
+
 tr_rc4_ctx_t tr_rc4_new(void)
 {
-    EVP_CIPHER_CTX* handle = EVP_CIPHER_CTX_new();
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    OSSL_PROVIDER *legacy_provider = NULL;
+    OSSL_PROVIDER *default_provider = NULL;
+#endif
+    const EVP_CIPHER *cipher;
 
-    if (check_result(EVP_CipherInit_ex(handle, EVP_rc4(), NULL, NULL, NULL, -1)))
+    tr_rc4_ctx *handle = malloc(sizeof(tr_rc4_ctx));
+
+    handle->cipher_ctx = EVP_CIPHER_CTX_new();
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    handle->lib_ctx = OSSL_LIB_CTX_new();
+    TR_ASSERT(handle->lib_ctx);
+    legacy_provider = OSSL_PROVIDER_load(handle->lib_ctx, "legacy");
+    TR_ASSERT(legacy_provider);
+    default_provider = OSSL_PROVIDER_load(handle->lib_ctx, "default");
+    TR_ASSERT(default_provider);
+    
+    cipher = EVP_CIPHER_fetch(handle->lib_ctx, "RC4", NULL);
+#else
+    cipher = EVP_rc4();
+#endif
+
+    if (check_result(EVP_CipherInit_ex(handle->cipher_ctx, cipher, NULL, NULL,
+                                       NULL, -1)))
     {
         return handle;
     }
 
-    EVP_CIPHER_CTX_free(handle);
+    EVP_CIPHER_CTX_free(handle->cipher_ctx);
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    OSSL_LIB_CTX_free(handle->lib_ctx);
+#endif
     return NULL;
 }
 
-void tr_rc4_free(tr_rc4_ctx_t handle)
+void tr_rc4_free(tr_rc4_ctx_t h)
 {
-    if (handle == NULL)
+    if (h == NULL)
     {
         return;
     }
 
-    EVP_CIPHER_CTX_free(handle);
+    tr_rc4_ctx *handle = (tr_rc4_ctx *)h;
+
+    EVP_CIPHER_CTX_free(handle->cipher_ctx);
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    OSSL_LIB_CTX_free(handle->lib_ctx);
+#endif
+    free(handle);
 }
 
-void tr_rc4_set_key(tr_rc4_ctx_t handle, uint8_t const* key, size_t key_length)
+void tr_rc4_set_key(tr_rc4_ctx_t h, uint8_t const* key, size_t key_length)
 {
-    TR_ASSERT(handle != NULL);
+    TR_ASSERT(h != NULL);
     TR_ASSERT(key != NULL);
 
-    if (!check_result(EVP_CIPHER_CTX_set_key_length(handle, key_length)))
+    tr_rc4_ctx *handle = (tr_rc4_ctx *)h;
+    if (!check_result(EVP_CIPHER_CTX_set_key_length(handle->cipher_ctx, key_length)))
     {
         return;
     }
 
-    check_result(EVP_CipherInit_ex(handle, NULL, NULL, key, NULL, -1));
+    check_result(EVP_CipherInit_ex(handle->cipher_ctx, NULL, NULL, key, NULL, -1));
 }
 
-void tr_rc4_process(tr_rc4_ctx_t handle, void const* input, void* output, size_t length)
+void tr_rc4_process(tr_rc4_ctx_t h, void const* input, void* output, size_t length)
 {
-    TR_ASSERT(handle != NULL);
+    TR_ASSERT(h != NULL);
 
+    tr_rc4_ctx *handle = (tr_rc4_ctx *)h;
     if (length == 0)
     {
         return;
@@ -232,7 +275,7 @@
 
     int output_length;
 
-    check_result(EVP_CipherUpdate(handle, output, &output_length, input, length));
+    check_result(EVP_CipherUpdate(handle->cipher_ctx, output, &output_length, input, length));
 }
 
 /***