Changeset 73665 in vbox for trunk/src/VBox/Runtime/common/crypto/pkix-signature-rsa.cpp

Timestamp:

Aug 14, 2018 5:49:23 PM (6 years ago)

Author:

vboxsync

Message:

IPRT,SUP,Main: Working on new crypto key handling and rsa signing. bugref:9152

File:

• trunk/src/VBox/Runtime/common/crypto/pkix-signature-rsa.cpp (modified) (11 diffs)

trunk/src/VBox/Runtime/common/crypto/pkix-signature-rsa.cpp

@@ -41 +41 @@
 #include "rsa-internal.h"
 #include "pkix-signature-builtin.h"
+#include "key-internal.h"
 
 
@@ -53 +54 @@
     /** Set if we're signing, clear if verifying.  */
     bool                    fSigning;
-    /** The modulus.  */
-    RTBIGNUM                Modulus;
-    /** The exponent.  */
-    RTBIGNUM                Exponent;
 
     /** Temporary big number for use when signing or verifiying. */
@@ -138 +135 @@
 /** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnInit}  */
 static DECLCALLBACK(int) rtCrPkixSignatureRsa_Init(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, void *pvOpaque,
-                                                   bool fSigning, PCRTASN1BITSTRING pKey, PCRTASN1DYNTYPE pParams)
+                                                   bool fSigning, RTCRKEY hKey, PCRTASN1DYNTYPE pParams)
 {
     RT_NOREF_PV(pDesc); RT_NOREF_PV(pvState); RT_NOREF_PV(pvOpaque);
@@ -145 +142 @@
         return VERR_CR_PKIX_SIGNATURE_TAKES_NO_PARAMETERS;
 
+    RTCRKEYTYPE enmKeyType = RTCrKeyGetType(hKey);
+    if (fSigning)
+        AssertReturn(enmKeyType == RTCRKEYTYPE_RSA_PRIVATE, VERR_CR_PKIX_NOT_RSA_PRIVATE_KEY);
+    else
+        AssertReturn(enmKeyType == RTCRKEYTYPE_RSA_PUBLIC, VERR_CR_PKIX_NOT_RSA_PUBLIC_KEY);
+
     PRTCRPKIXSIGNATURERSA pThis = (PRTCRPKIXSIGNATURERSA)pvState;
     pThis->fSigning = fSigning;
 
-    /*
-     * Decode the key and pick the bits we really need from it.
-     */
-    RTASN1CURSORPRIMARY PrimaryCursor;
-    RTAsn1CursorInitPrimary(&PrimaryCursor, RTASN1BITSTRING_GET_BIT0_PTR(pKey), RTASN1BITSTRING_GET_BYTE_SIZE(pKey),
-                            NULL, &g_RTAsn1DefaultAllocator, RTASN1CURSOR_FLAGS_DER, "rsa");
-    int rc;
-    if (!fSigning)
-    {
-        rc = RTCrRsaPublicKey_DecodeAsn1(&PrimaryCursor.Cursor, 0, &pThis->Scratch.PublicKey, "PublicKey");
-        if (RT_SUCCESS(rc))
-        {
-            rc = RTAsn1Integer_ToBigNum(&pThis->Scratch.PublicKey.Modulus, &pThis->Modulus, 0);
-            if (RT_SUCCESS(rc))
-            {
-                rc = RTAsn1Integer_ToBigNum(&pThis->Scratch.PublicKey.PublicExponent, &pThis->Exponent, 0);
-                if (RT_SUCCESS(rc))
-                {
-                    RTAsn1VtDelete(&pThis->Scratch.PublicKey.SeqCore.Asn1Core);
-                    return VINF_SUCCESS;
-                }
-                RTBigNumDestroy(&pThis->Modulus);
-            }
-            RTAsn1VtDelete(&pThis->Scratch.PublicKey.SeqCore.Asn1Core);
-        }
-    }
-    else
-    {
-        rc = RTCrRsaPrivateKey_DecodeAsn1(&PrimaryCursor.Cursor, 0, &pThis->Scratch.PrivateKey, "PrivateKey");
-        if (RT_SUCCESS(rc))
-        {
-            rc = RTAsn1Integer_ToBigNum(&pThis->Scratch.PrivateKey.Modulus, &pThis->Modulus, RTBIGNUMINIT_F_SENSITIVE);
-            if (RT_SUCCESS(rc))
-            {
-                rc = RTAsn1Integer_ToBigNum(&pThis->Scratch.PrivateKey.PublicExponent, &pThis->Exponent, RTBIGNUMINIT_F_SENSITIVE);
-                if (RT_SUCCESS(rc))
-                {
-                    RTAsn1VtDelete(&pThis->Scratch.PrivateKey.SeqCore.Asn1Core);
-                    return VINF_SUCCESS;
-                }
-                RTBigNumDestroy(&pThis->Modulus);
-            }
-            RTAsn1VtDelete(&pThis->Scratch.PrivateKey.SeqCore.Asn1Core);
-        }
-    }
-    return rc;
+    return VINF_SUCCESS;
 }
 
@@ -213 +171 @@
     RT_NOREF_PV(fSigning); RT_NOREF_PV(pDesc);
     Assert(pThis->fSigning == fSigning);
-
-    RTBigNumDestroy(&pThis->Modulus);
-    RTBigNumDestroy(&pThis->Exponent);
 }
 
@@ -246 +201 @@
      * Figure out which hash and select the associate prebaked DigestInfo.
      */
-    RTDIGESTTYPE const  enmDigest    = RTCrDigestGetType(hDigest);
+    RTDIGESTTYPE const  enmDigest = RTCrDigestGetType(hDigest);
     AssertReturn(enmDigest != RTDIGESTTYPE_INVALID && enmDigest != RTDIGESTTYPE_UNKNOWN, VERR_CR_PKIX_UNKNOWN_DIGEST_TYPE);
     uint8_t const      *pbDigestInfoStart = NULL;
@@ -297 +252 @@
 
 /** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnVerify}  */
-static DECLCALLBACK(int) rtCrPkixSignatureRsa_Verify(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState,
+static DECLCALLBACK(int) rtCrPkixSignatureRsa_Verify(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, RTCRKEY hKey,
                                                      RTCRDIGEST hDigest, void const *pvSignature, size_t cbSignature)
 {
@@ -307 +262 @@
 
     /*
-     * 8.2.2.1 - Length check.
-     */
-    if (cbSignature != RTBigNumByteWidth(&pThis->Modulus))
+     * Get the key bits we need.
+     */
+    Assert(RTCrKeyGetType(hKey) == RTCRKEYTYPE_RSA_PUBLIC);
+    PRTBIGNUM pModulus  = &hKey->u.RsaPublic.Modulus;
+    PRTBIGNUM pExponent = &hKey->u.RsaPublic.Exponent;
+
+    /*
+     * 8.2.2.1 - Length check.  (RFC-3447)
+     */
+    if (cbSignature != RTBigNumByteWidth(pModulus))
         return VERR_CR_PKIX_INVALID_SIGNATURE_LENGTH;
 
@@ -321 +283 @@
         return rc;
     /* b) RSAVP1 - 5.2.2.2: Range check (0 <= s < n). */
-    if (RTBigNumCompare(&pThis->TmpBigNum1, &pThis->Modulus) < 0)
+    if (RTBigNumCompare(&pThis->TmpBigNum1, pModulus) < 0)
     {
         if (RTBigNumCompareWithU64(&pThis->TmpBigNum1, 0) >= 0)
@@ -329 +291 @@
             if (RT_SUCCESS(rc))
             {
-                rc = RTBigNumModExp(&pThis->TmpBigNum2, &pThis->TmpBigNum1, &pThis->Exponent, &pThis->Modulus);
+                rc = RTBigNumModExp(&pThis->TmpBigNum2, &pThis->TmpBigNum1, pExponent, pModulus);
                 if (RT_SUCCESS(rc))
                 {
@@ -387 +349 @@
 
 /** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnSign}  */
-static DECLCALLBACK(int) rtCrPkixSignatureRsa_Sign(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState,
+static DECLCALLBACK(int) rtCrPkixSignatureRsa_Sign(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, RTCRKEY hKey,
                                                    RTCRDIGEST hDigest, void *pvSignature, size_t *pcbSignature)
 {
     PRTCRPKIXSIGNATURERSA pThis = (PRTCRPKIXSIGNATURERSA)pvState;
-    RT_NOREF_PV(pDesc);  RT_NOREF_PV(hDigest); RT_NOREF_PV(pvSignature); RT_NOREF_PV(pcbSignature);
-    Assert(pThis->fSigning); NOREF(pThis);
-    return VERR_NOT_IMPLEMENTED;
+    RT_NOREF_PV(pDesc);
+    Assert(pThis->fSigning);
+
+    /*
+     * Get the key bits we need.
+     */
+    Assert(RTCrKeyGetType(hKey) == RTCRKEYTYPE_RSA_PRIVATE);
+    PRTBIGNUM pModulus  = &hKey->u.RsaPrivate.Modulus;
+    PRTBIGNUM pExponent = &hKey->u.RsaPrivate.PrivateExponent;
+
+    /*
+     * Calc signature length and return if destination buffer isn't big enough.
+     */
+    size_t const cbDst        = *pcbSignature;
+    size_t const cbEncodedMsg = RTBigNumByteWidth(pModulus);
+    *pcbSignature = cbEncodedMsg;
+    if (cbEncodedMsg > sizeof(pThis->Scratch) / 2)
+        return VERR_CR_PKIX_SIGNATURE_TOO_LONG;
+    if (!pvSignature || cbDst < cbEncodedMsg)
+        return VERR_BUFFER_OVERFLOW;
+
+    /*
+     * 8.1.1.1 - EMSA-PSS encoding.  (RFC-3447)
+     */
+    int rc = rtCrPkixSignatureRsa_EmsaPkcs1V15Encode(pThis, hDigest, cbEncodedMsg, false /* fNoDigestInfo */);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * 8.1.1.2 - RSA signature.
+     */
+    /* a) m = OS2IP(EM) -- Convert the encoded message (EM) to integer. */
+    rc = RTBigNumInit(&pThis->TmpBigNum1, RTBIGNUMINIT_F_ENDIAN_BIG | RTBIGNUMINIT_F_UNSIGNED,
+                      pThis->Scratch.abSignature, cbEncodedMsg);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* b) s = RSASP1(K, m = EM) - 5.2.1.1: Range check (0 <= m < n). */
+    if (RTBigNumCompare(&pThis->TmpBigNum1, pModulus) < 0)
+    {
+        /* b) s = RSAVP1(K, m = EM) - 5.2.1.2.a: s = m^d mod n */
+        rc = RTBigNumInitZero(&pThis->TmpBigNum2, 0);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTBigNumModExp(&pThis->TmpBigNum2, &pThis->TmpBigNum1, pExponent, pModulus);
+            if (RT_SUCCESS(rc))
+            {
+                /* c) S = I2OSP(s, k) -- Convert the result to bytes. */
+                rc = RTBigNumToBytesBigEndian(&pThis->TmpBigNum2, pvSignature, cbEncodedMsg);
+                AssertStmt(RT_SUCCESS(rc) || rc != VERR_BUFFER_OVERFLOW, rc = VERR_CR_PKIX_INTERNAL_ERROR);
+            }
+            RTBigNumDestroy(&pThis->TmpBigNum2);
+        }
+    }
+    else
+        rc = VERR_CR_PKIX_SIGNATURE_GE_KEY;
+    RTBigNumDestroy(&pThis->TmpBigNum1);
+    return rc;
 }