VirtualBox

source: vbox/trunk/src/VBox/Runtime/common/crypto/pkix-signature-rsa.cpp@ 73668

Last change on this file since 73668 was 73668, checked in by vboxsync, 6 years ago

IPRT,SUP,Main: Working on new crypto key handling and rsa signing. bugref:9152 [build fix]
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 18.5 KB
Line 
1/* $Id: pkix-signature-rsa.cpp 73668 2018-08-14 17:52:12Z vboxsync $ */
2/** @file
3 * IPRT - Crypto - Public Key Signature Schema Algorithm, RSA Providers.
4 */
5
6/*
7 * Copyright (C) 2006-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*********************************************************************************************************************************
29* Header Files *
30*********************************************************************************************************************************/
31#include "internal/iprt.h"
32#include <iprt/crypto/rsa.h>
33
34#include <iprt/bignum.h>
35#include <iprt/err.h>
36#include <iprt/mem.h>
37#include <iprt/string.h>
38#include <iprt/crypto/digest.h>
39#include <iprt/crypto/pkix.h>
40
41#include "rsa-internal.h"
42#include "pkix-signature-builtin.h"
43#include "key-internal.h"
44
45
46/*********************************************************************************************************************************
47* Structures and Typedefs *
48*********************************************************************************************************************************/
49/**
50 * RSA signature provider instance.
51 */
52typedef struct RTCRPKIXSIGNATURERSA
53{
54 /** Set if we're signing, clear if verifying. */
55 bool fSigning;
56
57 /** Temporary big number for use when signing or verifiying. */
58 RTBIGNUM TmpBigNum1;
59 /** Temporary big number for use when signing or verifiying. */
60 RTBIGNUM TmpBigNum2;
61
62 /** Scratch space for decoding the key. */
63 union
64 {
65 /** Public key. */
66 RTCRRSAPUBLICKEY PublicKey;
67 /** Private key. */
68 RTCRRSAPRIVATEKEY PrivateKey;
69 /** Scratch area where we assemble the signature. */
70 uint8_t abSignature[RTCRRSA_MAX_MODULUS_BITS / 8 * 2];
71 } Scratch;
72} RTCRPKIXSIGNATURERSA;
73/** Pointer to an RSA signature provider instance. */
74typedef RTCRPKIXSIGNATURERSA *PRTCRPKIXSIGNATURERSA;
75
76
77/*********************************************************************************************************************************
78* Global Variables *
79*********************************************************************************************************************************/
80/** @name Pre-encoded DigestInfo DER sequences.
81 * @{ */
82static const uint8_t g_abMd2[] =
83{/* { { 1.2.840.113549.2.2 (MD2), NULL }, hash octet-string } */
84 0x30,0x20, 0x30,0x0c, 0x06,0x08,0x2a,0x86,0x48,0x86,0xf7,0x0d,0x02,0x02, 0x05,0x00, 0x04,0x10
85};
86static const uint8_t g_abMd4[] =
87{/* { { 1.2.840.113549.2.4 (MD4), NULL }, hash octet-string } */
88 0x30,0x20, 0x30,0x0c, 0x06,0x08,0x2a,0x86,0x48,0x86,0xf7,0x0d,0x02,0x04, 0x05,0x00, 0x04,0x10
89};
90static const uint8_t g_abMd5[] =
91{/* { { 1.2.840.113549.2.5 (MD5), NULL }, hash octet-string } */
92 0x30,0x20, 0x30,0x0c, 0x06,0x08,0x2a,0x86,0x48,0x86,0xf7,0x0d,0x02,0x05, 0x05,0x00, 0x04,0x10
93};
94static const uint8_t g_abSha1[] =
95{/* { { 1.3.14.3.2.26 (SHA-1), NULL }, hash octet-string } */
96 0x30,0x21, 0x30,0x09, 0x06,0x05,0x2b,0x0e,0x03,0x02,0x1a, 0x05,0x00, 0x04,0x14
97};
98static const uint8_t g_abSha256[] =
99{/* { { 2.16.840.1.101.3.4.2.1 (SHA-256), NULL }, hash octet-string } */
100 0x30,0x31, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x01, 0x05,0x00, 0x04,0x20
101};
102static const uint8_t g_abSha384[] =
103{/* { { 2.16.840.1.101.3.4.2.2 (SHA-384), NULL }, hash octet-string } */
104 0x30,0x41, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x02, 0x05,0x00, 0x04,0x30
105};
106static const uint8_t g_abSha512[] =
107{/* { { 2.16.840.1.101.3.4.2.3 (SHA-512), NULL }, hash octet-string } */
108 0x30,0x51, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x03, 0x05,0x00, 0x04,0x40
109};
110static const uint8_t g_abSha224[] =
111{/* { { 2.16.840.1.101.3.4.2.4 (SHA-224), NULL }, hash octet-string } */
112 0x30,0x2d, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x04, 0x05,0x00, 0x04,0x1c
113};
114/** @} */
115
116/** Lookup array for the pre-encoded DigestInfo DER sequences. */
117static struct
118{
119 RTDIGESTTYPE enmDigest;
120 const uint8_t *pb;
121 size_t cb;
122} const g_aDigestInfos[] =
123{
124 { RTDIGESTTYPE_MD2, g_abMd2, sizeof(g_abMd2) },
125 { RTDIGESTTYPE_MD4, g_abMd4, sizeof(g_abMd4) },
126 { RTDIGESTTYPE_MD5, g_abMd5, sizeof(g_abMd5) },
127 { RTDIGESTTYPE_SHA1, g_abSha1, sizeof(g_abSha1) },
128 { RTDIGESTTYPE_SHA256, g_abSha256, sizeof(g_abSha256) },
129 { RTDIGESTTYPE_SHA384, g_abSha384, sizeof(g_abSha384) },
130 { RTDIGESTTYPE_SHA512, g_abSha512, sizeof(g_abSha512) },
131 { RTDIGESTTYPE_SHA224, g_abSha224, sizeof(g_abSha224) },
132};
133
134
135/** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnInit} */
136static DECLCALLBACK(int) rtCrPkixSignatureRsa_Init(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, void *pvOpaque,
137 bool fSigning, RTCRKEY hKey, PCRTASN1DYNTYPE pParams)
138{
139 RT_NOREF_PV(pDesc); RT_NOREF_PV(pvState); RT_NOREF_PV(pvOpaque);
140
141 if (pParams)
142 return VERR_CR_PKIX_SIGNATURE_TAKES_NO_PARAMETERS;
143
144 RTCRKEYTYPE enmKeyType = RTCrKeyGetType(hKey);
145 if (fSigning)
146 AssertReturn(enmKeyType == RTCRKEYTYPE_RSA_PRIVATE, VERR_CR_PKIX_NOT_RSA_PRIVATE_KEY);
147 else
148 AssertReturn(enmKeyType == RTCRKEYTYPE_RSA_PUBLIC, VERR_CR_PKIX_NOT_RSA_PUBLIC_KEY);
149
150 PRTCRPKIXSIGNATURERSA pThis = (PRTCRPKIXSIGNATURERSA)pvState;
151 pThis->fSigning = fSigning;
152
153 return VINF_SUCCESS;
154}
155
156
157/** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnReset} */
158static DECLCALLBACK(int) rtCrPkixSignatureRsa_Reset(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, bool fSigning)
159{
160 PRTCRPKIXSIGNATURERSA pThis = (PRTCRPKIXSIGNATURERSA)pvState;
161 RT_NOREF_PV(fSigning); RT_NOREF_PV(pDesc);
162 Assert(pThis->fSigning == fSigning); NOREF(pThis);
163 return VINF_SUCCESS;
164}
165
166
167/** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnDelete} */
168static DECLCALLBACK(void) rtCrPkixSignatureRsa_Delete(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, bool fSigning)
169{
170 PRTCRPKIXSIGNATURERSA pThis = (PRTCRPKIXSIGNATURERSA)pvState;
171 RT_NOREF_PV(fSigning); RT_NOREF_PV(pDesc);
172 Assert(pThis->fSigning == fSigning);
173 NOREF(pThis);
174}
175
176
177/**
178 * Common worker for rtCrPkixSignatureRsa_Verify and
179 * rtCrPkixSignatureRsa_Sign that encodes an EMSA-PKCS1-V1_5 signature in
180 * the scratch area.
181 *
182 * This function is referred to as EMSA-PKCS1-v1_5-ENCODE(M,k) in RFC-3447 and
183 * is described in section 9.2
184 *
185 * @returns IPRT status code.
186 * @param pThis The RSA signature provider instance.
187 * @param hDigest The digest which hash to turn into a signature.
188 * @param cbEncodedMsg The desired encoded message length.
189 * @param fNoDigestInfo If true, skip the DigestInfo and encode the digest
190 * without any prefix like described in v1.5 (RFC-2313)
191 * and observed with RSA+MD5 signed timestamps. If
192 * false, include the prefix like v2.0 (RFC-2437)
193 * describes in step in section 9.2.1
194 * (EMSA-PKCS1-v1_5)
195 */
196static int rtCrPkixSignatureRsa_EmsaPkcs1V15Encode(PRTCRPKIXSIGNATURERSA pThis, RTCRDIGEST hDigest, size_t cbEncodedMsg,
197 bool fNoDigestInfo)
198{
199 AssertReturn(cbEncodedMsg * 2 <= sizeof(pThis->Scratch), VERR_CR_PKIX_INTERNAL_ERROR);
200
201 /*
202 * Figure out which hash and select the associate prebaked DigestInfo.
203 */
204 RTDIGESTTYPE const enmDigest = RTCrDigestGetType(hDigest);
205 AssertReturn(enmDigest != RTDIGESTTYPE_INVALID && enmDigest != RTDIGESTTYPE_UNKNOWN, VERR_CR_PKIX_UNKNOWN_DIGEST_TYPE);
206 uint8_t const *pbDigestInfoStart = NULL;
207 size_t cbDigestInfoStart = 0;
208 for (uint32_t i = 0; i < RT_ELEMENTS(g_aDigestInfos); i++)
209 if (g_aDigestInfos[i].enmDigest == enmDigest)
210 {
211 pbDigestInfoStart = g_aDigestInfos[i].pb;
212 cbDigestInfoStart = g_aDigestInfos[i].cb;
213 break;
214 }
215 if (!pbDigestInfoStart)
216 return VERR_CR_PKIX_UNKNOWN_DIGEST_TYPE;
217
218 /*
219 * Get the hash size and verify that it matches what we've got in the
220 * precooked DigestInfo. ASSUMES less that 256 bytes of hash.
221 */
222 uint32_t const cbHash = RTCrDigestGetHashSize(hDigest);
223 AssertReturn(cbHash > 0 && cbHash < _16K, VERR_OUT_OF_RANGE);
224 AssertReturn(cbHash == pbDigestInfoStart[cbDigestInfoStart - 1], VERR_CR_PKIX_INTERNAL_ERROR);
225
226 if (fNoDigestInfo)
227 cbDigestInfoStart = 0;
228
229 if (cbDigestInfoStart + cbHash + 11 > cbEncodedMsg)
230 return VERR_CR_PKIX_HASH_TOO_LONG_FOR_KEY;
231
232 /*
233 * Encode the message the first part of the scratch area.
234 */
235 uint8_t *pbDst = &pThis->Scratch.abSignature[0];
236 pbDst[0] = 0x00;
237 pbDst[1] = 0x01; /* BT - block type, see RFC-2313. */
238 size_t cbFFs = cbEncodedMsg - cbHash - cbDigestInfoStart - 3;
239 memset(&pbDst[2], 0xff, cbFFs);
240 pbDst += cbFFs + 2;
241 *pbDst++ = 0x00;
242 memcpy(pbDst, pbDigestInfoStart, cbDigestInfoStart);
243 pbDst += cbDigestInfoStart;
244 int rc = RTCrDigestFinal(hDigest, pbDst, cbHash);
245 if (RT_FAILURE(rc))
246 return rc;
247 pbDst += cbHash;
248 Assert((size_t)(pbDst - &pThis->Scratch.abSignature[0]) == cbEncodedMsg);
249 return VINF_SUCCESS;
250}
251
252
253
254/** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnVerify} */
255static DECLCALLBACK(int) rtCrPkixSignatureRsa_Verify(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, RTCRKEY hKey,
256 RTCRDIGEST hDigest, void const *pvSignature, size_t cbSignature)
257{
258 PRTCRPKIXSIGNATURERSA pThis = (PRTCRPKIXSIGNATURERSA)pvState;
259 RT_NOREF_PV(pDesc);
260 Assert(!pThis->fSigning);
261 if (cbSignature > sizeof(pThis->Scratch) / 2)
262 return VERR_CR_PKIX_SIGNATURE_TOO_LONG;
263
264 /*
265 * Get the key bits we need.
266 */
267 Assert(RTCrKeyGetType(hKey) == RTCRKEYTYPE_RSA_PUBLIC);
268 PRTBIGNUM pModulus = &hKey->u.RsaPublic.Modulus;
269 PRTBIGNUM pExponent = &hKey->u.RsaPublic.Exponent;
270
271 /*
272 * 8.2.2.1 - Length check. (RFC-3447)
273 */
274 if (cbSignature != RTBigNumByteWidth(pModulus))
275 return VERR_CR_PKIX_INVALID_SIGNATURE_LENGTH;
276
277 /*
278 * 8.2.2.2 - RSA verification / Decrypt the signature.
279 */
280 /* a) s = OS2IP(S) -- Convert signature to integer. */
281 int rc = RTBigNumInit(&pThis->TmpBigNum1, RTBIGNUMINIT_F_ENDIAN_BIG | RTBIGNUMINIT_F_UNSIGNED,
282 pvSignature, cbSignature);
283 if (RT_FAILURE(rc))
284 return rc;
285 /* b) RSAVP1 - 5.2.2.2: Range check (0 <= s < n). */
286 if (RTBigNumCompare(&pThis->TmpBigNum1, pModulus) < 0)
287 {
288 if (RTBigNumCompareWithU64(&pThis->TmpBigNum1, 0) >= 0)
289 {
290 /* b) RSAVP1 - 5.2.2.3: s^e mod n */
291 rc = RTBigNumInitZero(&pThis->TmpBigNum2, 0);
292 if (RT_SUCCESS(rc))
293 {
294 rc = RTBigNumModExp(&pThis->TmpBigNum2, &pThis->TmpBigNum1, pExponent, pModulus);
295 if (RT_SUCCESS(rc))
296 {
297 /* c) EM' = I2OSP(m, k) -- Convert the result to bytes. */
298 uint32_t cbDecrypted = RTBigNumByteWidth(&pThis->TmpBigNum2) + 1; /* 1 = leading zero byte */
299 if (cbDecrypted <= sizeof(pThis->Scratch) / 2)
300 {
301 uint8_t *pbDecrypted = &pThis->Scratch.abSignature[sizeof(pThis->Scratch) / 2];
302 rc = RTBigNumToBytesBigEndian(&pThis->TmpBigNum2, pbDecrypted, cbDecrypted);
303 if (RT_SUCCESS(rc))
304 {
305 /*
306 * 8.2.2.3 - Build a hopefully identical signature using hDigest.
307 */
308 rc = rtCrPkixSignatureRsa_EmsaPkcs1V15Encode(pThis, hDigest, cbDecrypted, false /* fNoDigestInfo */);
309 if (RT_SUCCESS(rc))
310 {
311 /*
312 * 8.2.2.4 - Compare the two.
313 */
314 if (memcmp(&pThis->Scratch.abSignature[0], pbDecrypted, cbDecrypted) == 0)
315 rc = VINF_SUCCESS;
316 else
317 {
318 /*
319 * Try again without digestinfo. This style signing has been
320 * observed in Vista timestamp counter signatures (Thawte).
321 */
322 rc = rtCrPkixSignatureRsa_EmsaPkcs1V15Encode(pThis, hDigest, cbDecrypted,
323 true /* fNoDigestInfo */);
324 if (RT_SUCCESS(rc))
325 {
326 if (memcmp(&pThis->Scratch.abSignature[0], pbDecrypted, cbDecrypted) == 0)
327 rc = VINF_SUCCESS;
328 else
329 rc = VERR_CR_PKIX_SIGNATURE_MISMATCH;
330 }
331 }
332 }
333 }
334 }
335 else
336 rc = VERR_CR_PKIX_SIGNATURE_TOO_LONG;
337 }
338 RTBigNumDestroy(&pThis->TmpBigNum2);
339 }
340 }
341 else
342 rc = VERR_CR_PKIX_SIGNATURE_NEGATIVE;
343 }
344 else
345 rc = VERR_CR_PKIX_SIGNATURE_GE_KEY;
346 RTBigNumDestroy(&pThis->TmpBigNum1);
347 return rc;
348}
349
350
351/** @impl_interface_method{RTCRPKIXSIGNATUREDESC,pfnSign} */
352static DECLCALLBACK(int) rtCrPkixSignatureRsa_Sign(PCRTCRPKIXSIGNATUREDESC pDesc, void *pvState, RTCRKEY hKey,
353 RTCRDIGEST hDigest, void *pvSignature, size_t *pcbSignature)
354{
355 PRTCRPKIXSIGNATURERSA pThis = (PRTCRPKIXSIGNATURERSA)pvState;
356 RT_NOREF_PV(pDesc);
357 Assert(pThis->fSigning);
358
359 /*
360 * Get the key bits we need.
361 */
362 Assert(RTCrKeyGetType(hKey) == RTCRKEYTYPE_RSA_PRIVATE);
363 PRTBIGNUM pModulus = &hKey->u.RsaPrivate.Modulus;
364 PRTBIGNUM pExponent = &hKey->u.RsaPrivate.PrivateExponent;
365
366 /*
367 * Calc signature length and return if destination buffer isn't big enough.
368 */
369 size_t const cbDst = *pcbSignature;
370 size_t const cbEncodedMsg = RTBigNumByteWidth(pModulus);
371 *pcbSignature = cbEncodedMsg;
372 if (cbEncodedMsg > sizeof(pThis->Scratch) / 2)
373 return VERR_CR_PKIX_SIGNATURE_TOO_LONG;
374 if (!pvSignature || cbDst < cbEncodedMsg)
375 return VERR_BUFFER_OVERFLOW;
376
377 /*
378 * 8.1.1.1 - EMSA-PSS encoding. (RFC-3447)
379 */
380 int rc = rtCrPkixSignatureRsa_EmsaPkcs1V15Encode(pThis, hDigest, cbEncodedMsg, false /* fNoDigestInfo */);
381 if (RT_FAILURE(rc))
382 return rc;
383
384 /*
385 * 8.1.1.2 - RSA signature.
386 */
387 /* a) m = OS2IP(EM) -- Convert the encoded message (EM) to integer. */
388 rc = RTBigNumInit(&pThis->TmpBigNum1, RTBIGNUMINIT_F_ENDIAN_BIG | RTBIGNUMINIT_F_UNSIGNED,
389 pThis->Scratch.abSignature, cbEncodedMsg);
390 if (RT_FAILURE(rc))
391 return rc;
392
393 /* b) s = RSASP1(K, m = EM) - 5.2.1.1: Range check (0 <= m < n). */
394 if (RTBigNumCompare(&pThis->TmpBigNum1, pModulus) < 0)
395 {
396 /* b) s = RSAVP1(K, m = EM) - 5.2.1.2.a: s = m^d mod n */
397 rc = RTBigNumInitZero(&pThis->TmpBigNum2, 0);
398 if (RT_SUCCESS(rc))
399 {
400 rc = RTBigNumModExp(&pThis->TmpBigNum2, &pThis->TmpBigNum1, pExponent, pModulus);
401 if (RT_SUCCESS(rc))
402 {
403 /* c) S = I2OSP(s, k) -- Convert the result to bytes. */
404 rc = RTBigNumToBytesBigEndian(&pThis->TmpBigNum2, pvSignature, cbEncodedMsg);
405 AssertStmt(RT_SUCCESS(rc) || rc != VERR_BUFFER_OVERFLOW, rc = VERR_CR_PKIX_INTERNAL_ERROR);
406 }
407 RTBigNumDestroy(&pThis->TmpBigNum2);
408 }
409 }
410 else
411 rc = VERR_CR_PKIX_SIGNATURE_GE_KEY;
412 RTBigNumDestroy(&pThis->TmpBigNum1);
413 return rc;
414}
415
416
417
418
419/** RSA alias ODIs. */
420static const char * const g_apszHashWithRsaAliases[] =
421{
422 RTCR_PKCS1_MD2_WITH_RSA_OID,
423 RTCR_PKCS1_MD4_WITH_RSA_OID,
424 RTCR_PKCS1_MD5_WITH_RSA_OID,
425 RTCR_PKCS1_SHA1_WITH_RSA_OID,
426 RTCR_PKCS1_SHA256_WITH_RSA_OID,
427 RTCR_PKCS1_SHA384_WITH_RSA_OID,
428 RTCR_PKCS1_SHA512_WITH_RSA_OID,
429 RTCR_PKCS1_SHA224_WITH_RSA_OID,
430 /* Note: Note quite sure about these OIW oddballs. */
431 "1.3.14.3.2.11" /* OIW rsaSignature */,
432 "1.3.14.3.2.14" /* OIW mdc2WithRSASignature */,
433 "1.3.14.3.2.15" /* OIW shaWithRSASignature */,
434 "1.3.14.3.2.24" /* OIW md2WithRSASignature */,
435 "1.3.14.3.2.25" /* OIW md5WithRSASignature */,
436 "1.3.14.3.2.29" /* OIW sha1WithRSASignature */,
437 NULL
438};
439
440
441/** RSA descriptor. */
442DECL_HIDDEN_CONST(RTCRPKIXSIGNATUREDESC const) g_rtCrPkixSigningHashWithRsaDesc =
443{
444 "RSASSA-PKCS1-v1_5",
445 RTCR_PKCS1_RSA_OID,
446 g_apszHashWithRsaAliases,
447 sizeof(RTCRPKIXSIGNATURERSA),
448 0,
449 0,
450 rtCrPkixSignatureRsa_Init,
451 rtCrPkixSignatureRsa_Reset,
452 rtCrPkixSignatureRsa_Delete,
453 rtCrPkixSignatureRsa_Verify,
454 rtCrPkixSignatureRsa_Sign,
455};
456
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette