Merge #179333: openssl_3_0: fix apparent x86_64 AVX512 RCE

pkgs/development/libraries/openssl/3.0/rsa-fix-bn_reduce_once_in_place-call-for-rsaz_mod_exp_avx512_x2.patch

@@ -0,0 +1,34 @@
+From 4d8a88c134df634ba610ff8db1eb8478ac5fd345 Mon Sep 17 00:00:00 2001
+From: Xi Ruoyao <xry111@xry111.site>
+Date: Wed, 22 Jun 2022 18:07:05 +0800
+Subject: [PATCH] rsa: fix bn_reduce_once_in_place call for
+ rsaz_mod_exp_avx512_x2
+
+bn_reduce_once_in_place expects the number of BN_ULONG, but factor_size
+is moduli bit size.
+
+Fixes #18625.
+
+Signed-off-by: Xi Ruoyao <xry111@xry111.site>
+
+Reviewed-by: Tomas Mraz <tomas@openssl.org>
+Reviewed-by: Paul Dale <pauli@openssl.org>
+(Merged from https://github.com/openssl/openssl/pull/18626)
+---
+ crypto/bn/rsaz_exp_x2.c | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/crypto/bn/rsaz_exp_x2.c b/crypto/bn/rsaz_exp_x2.c
+index 6b04486e3f56..f979cebd6fb7 100644
+--- a/crypto/bn/rsaz_exp_x2.c
++++ b/crypto/bn/rsaz_exp_x2.c
+@@ -257,6 +257,9 @@ int ossl_rsaz_mod_exp_avx512_x2(BN_ULONG *res1,
+     from_words52(res1, factor_size, rr1_red);
+     from_words52(res2, factor_size, rr2_red);
+ 
++    /* bn_reduce_once_in_place expects number of BN_ULONG, not bit size */
++    factor_size /= sizeof(BN_ULONG) * 8;
++
+     bn_reduce_once_in_place(res1, /*carry=*/0, m1, storage, factor_size);
+     bn_reduce_once_in_place(res2, /*carry=*/0, m2, storage, factor_size);
+ 

pkgs/development/libraries/openssl/default.nix

@@ -210,6 +210,10 @@ in {
       # This patch disables build-time detection.
       ./3.0/openssl-disable-kernel-detection.patch
 
+      # https://guidovranken.com/2022/06/27/notes-on-openssl-remote-memory-corruption/
+      # https://github.com/openssl/openssl/commit/4d8a88c134df634ba610ff8db1eb8478ac5fd345.patch
+      3.0/rsa-fix-bn_reduce_once_in_place-call-for-rsaz_mod_exp_avx512_x2.patch
+
       (if stdenv.hostPlatform.isDarwin
        then ./use-etc-ssl-certs-darwin.patch
        else ./use-etc-ssl-certs.patch)