From 4ee58de2fd274358cec386693dbd04558001b348 Mon Sep 17 00:00:00 2001
From: Bo Yang <atry@meta.com>
Date: Wed, 2 Nov 2022 09:21:50 -0700
Subject: [PATCH] Fix compilation error with OpenSSL 3

Summary:
In OpenSSL 3, `EVP_PKEY_get0_*` functions now return `const` pointers, breaking existing usage depending on mutable pointers returned by OpenSSL 1.x. This diff fix the errors

1. Use `auto` type for return type of `EVP_PKEY_get0_*` functions instead of mutable pointer types.
1. Some `EVP_PKEY_get0_RSA` are replaced with `EVP_PKEY_get1_RSA` in order to get a mutable pointer with the ownership.

See https://www.openssl.org/docs/man3.0/man7/crypto.html#LIBRARY-CONVENTIONS for the `get_0` conventions

Differential Revision: D40942193

fbshipit-source-id: b9a868aec3d332e9de323cb801af52e2692d4686
---
 hphp/runtime/ext/openssl/ext_openssl.cpp | 70 ++++++++++++++++--------
 1 file changed, 47 insertions(+), 23 deletions(-)

diff --git a/hphp/runtime/ext/openssl/ext_openssl.cpp b/hphp/runtime/ext/openssl/ext_openssl.cpp
index c90ea3ace3ebc7..f5c96be42342c0 100644
--- a/hphp/runtime/ext/openssl/ext_openssl.cpp
+++ b/hphp/runtime/ext/openssl/ext_openssl.cpp
@@ -1907,7 +1907,7 @@ Array HHVM_FUNCTION(openssl_pkey_get_details, const Resource& key) {
   case EVP_PKEY_RSA2:
     {
       ktype = OPENSSL_KEYTYPE_RSA;
-      RSA *rsa = EVP_PKEY_get0_RSA(pkey);
+      auto rsa = EVP_PKEY_get0_RSA(pkey);
       assertx(rsa);
       const BIGNUM *n, *e, *d, *p, *q, *dmp1, *dmq1, *iqmp;
       RSA_get0_key(rsa, &n, &e, &d);
@@ -1930,7 +1930,7 @@ Array HHVM_FUNCTION(openssl_pkey_get_details, const Resource& key) {
   case EVP_PKEY_DSA4:
     {
       ktype = OPENSSL_KEYTYPE_DSA;
-      DSA *dsa = EVP_PKEY_get0_DSA(pkey);
+      auto dsa = EVP_PKEY_get0_DSA(pkey);
       assertx(dsa);
       const BIGNUM *p, *q, *g, *pub_key, *priv_key;
       DSA_get0_pqg(dsa, &p, &q, &g);
@@ -1946,7 +1946,7 @@ Array HHVM_FUNCTION(openssl_pkey_get_details, const Resource& key) {
   case EVP_PKEY_DH:
     {
       ktype = OPENSSL_KEYTYPE_DH;
-      DH *dh = EVP_PKEY_get0_DH(pkey);
+      auto dh = EVP_PKEY_get0_DH(pkey);
       assertx(dh);
       const BIGNUM *p, *q, *g, *pub_key, *priv_key;
       DH_get0_pqg(dh, &p, &q, &g);
@@ -2060,11 +2060,17 @@ bool HHVM_FUNCTION(openssl_private_decrypt, const String& data,
   switch (EVP_PKEY_id(pkey)) {
   case EVP_PKEY_RSA:
   case EVP_PKEY_RSA2:
-    cryptedlen = RSA_private_decrypt(data.size(),
-                                     (unsigned char *)data.data(),
-                                     cryptedbuf,
-                                     EVP_PKEY_get0_RSA(pkey),
-                                     padding);
+    {
+      auto rsa = EVP_PKEY_get1_RSA(pkey);
+      SCOPE_EXIT {
+        RSA_free(rsa);
+      };
+      cryptedlen = RSA_private_decrypt(data.size(),
+                                      (unsigned char *)data.data(),
+                                      cryptedbuf,
+                                      rsa,
+                                      padding);
+    }
     if (cryptedlen != -1) {
       successful = 1;
     }
@@ -2100,11 +2106,17 @@ bool HHVM_FUNCTION(openssl_private_encrypt, const String& data,
   switch (EVP_PKEY_id(pkey)) {
   case EVP_PKEY_RSA:
   case EVP_PKEY_RSA2:
-    successful = (RSA_private_encrypt(data.size(),
-                                      (unsigned char *)data.data(),
-                                      cryptedbuf,
-                                      EVP_PKEY_get0_RSA(pkey),
-                                      padding) == cryptedlen);
+    {
+      auto rsa = EVP_PKEY_get1_RSA(pkey);
+      SCOPE_EXIT {
+        RSA_free(rsa);
+      };
+      successful = (RSA_private_encrypt(data.size(),
+                                        (unsigned char *)data.data(),
+                                        cryptedbuf,
+                                        rsa,
+                                        padding) == cryptedlen);
+    }
     break;
   default:
     raise_warning("key type not supported");
@@ -2136,11 +2148,17 @@ bool HHVM_FUNCTION(openssl_public_decrypt, const String& data,
   switch (EVP_PKEY_id(pkey)) {
   case EVP_PKEY_RSA:
   case EVP_PKEY_RSA2:
-    cryptedlen = RSA_public_decrypt(data.size(),
-                                    (unsigned char *)data.data(),
-                                    cryptedbuf,
-                                    EVP_PKEY_get0_RSA(pkey),
-                                    padding);
+    {
+      auto rsa = EVP_PKEY_get1_RSA(pkey);
+      SCOPE_EXIT {
+        RSA_free(rsa);
+      };
+      cryptedlen = RSA_public_decrypt(data.size(),
+                                      (unsigned char *)data.data(),
+                                      cryptedbuf,
+                                      rsa,
+                                      padding);
+    }
     if (cryptedlen != -1) {
       successful = 1;
     }
@@ -2176,11 +2194,17 @@ bool HHVM_FUNCTION(openssl_public_encrypt, const String& data,
   switch (EVP_PKEY_id(pkey)) {
   case EVP_PKEY_RSA:
   case EVP_PKEY_RSA2:
-    successful = (RSA_public_encrypt(data.size(),
-                                     (unsigned char *)data.data(),
-                                     cryptedbuf,
-                                     EVP_PKEY_get0_RSA(pkey),
-                                     padding) == cryptedlen);
+    {
+      auto rsa = EVP_PKEY_get1_RSA(pkey);
+      SCOPE_EXIT {
+        RSA_free(rsa);
+      };
+      successful = (RSA_public_encrypt(data.size(),
+                                      (unsigned char *)data.data(),
+                                      cryptedbuf,
+                                      rsa,
+                                      padding) == cryptedlen);
+    }
     break;
   default:
     raise_warning("key type not supported");