lib/libcrypto/rsa/rsa_pmeth.c

   1 /* $OpenBSD: rsa_pmeth.c,v 1.20 2017/08/28 17:41:59 jsing Exp $ */
   2 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
   3  * project 2006.
   4  */
   5 /* ====================================================================
   6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  *
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in
  17  *    the documentation and/or other materials provided with the
  18  *    distribution.
  19  *
  20  * 3. All advertising materials mentioning features or use of this
  21  *    software must display the following acknowledgment:
  22  *    "This product includes software developed by the OpenSSL Project
  23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
  24  *
  25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
  26  *    endorse or promote products derived from this software without
  27  *    prior written permission. For written permission, please contact
  28  *    licensing@OpenSSL.org.
  29  *
  30  * 5. Products derived from this software may not be called "OpenSSL"
  31  *    nor may "OpenSSL" appear in their names without prior written
  32  *    permission of the OpenSSL Project.
  33  *
  34  * 6. Redistributions of any form whatsoever must retain the following
  35  *    acknowledgment:
  36  *    "This product includes software developed by the OpenSSL Project
  37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
  38  *
  39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
  40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
  43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  50  * OF THE POSSIBILITY OF SUCH DAMAGE.
  51  * ====================================================================
  52  *
  53  * This product includes cryptographic software written by Eric Young
  54  * (eay@cryptsoft.com).  This product includes software written by Tim
  55  * Hudson (tjh@cryptsoft.com).
  56  *
  57  */
  58
  59 #include <limits.h>
  60 #include <stdio.h>
  61 #include <string.h>
  62
  63 #include <openssl/opensslconf.h>
  64
  65 #include <openssl/asn1t.h>
  66 #include <openssl/bn.h>
  67 #include <openssl/err.h>
  68 #include <openssl/evp.h>
  69 #include <openssl/rsa.h>
  70 #include <openssl/x509.h>
  71
  72
  73 #include "evp_locl.h"
  74 #include "rsa_locl.h"
  75
  76 /* RSA pkey context structure */
  77
  78 typedef struct {
  79         /* Key gen parameters */
  80         int nbits;
  81         BIGNUM *pub_exp;
  82         /* Keygen callback info */
  83         int gentmp[2];
  84         /* RSA padding mode */
  85         int pad_mode;
  86         /* message digest */
  87         const EVP_MD *md;
  88         /* message digest for MGF1 */
  89         const EVP_MD *mgf1md;
  90         /* PSS/OAEP salt length */
  91         int saltlen;
  92         /* Temp buffer */
  93         unsigned char *tbuf;
  94 } RSA_PKEY_CTX;
  95
  96 static int
  97 pkey_rsa_init(EVP_PKEY_CTX *ctx)
  98 {
  99         RSA_PKEY_CTX *rctx;
 100
 101         rctx = malloc(sizeof(RSA_PKEY_CTX));
 102         if (!rctx)
 103                 return 0;
 104         rctx->nbits = 2048;
 105         rctx->pub_exp = NULL;
 106         rctx->pad_mode = RSA_PKCS1_PADDING;
 107         rctx->md = NULL;
 108         rctx->mgf1md = NULL;
 109         rctx->tbuf = NULL;
 110
 111         rctx->saltlen = -2;
 112
 113         ctx->data = rctx;
 114         ctx->keygen_info = rctx->gentmp;
 115         ctx->keygen_info_count = 2;
 116
 117         return 1;
 118 }
 119
 120 static int
 121 pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
 122 {
 123         RSA_PKEY_CTX *dctx, *sctx;
 124
 125         if (!pkey_rsa_init(dst))
 126                 return 0;
 127         sctx = src->data;
 128         dctx = dst->data;
 129         dctx->nbits = sctx->nbits;
 130         if (sctx->pub_exp) {
 131                 dctx->pub_exp = BN_dup(sctx->pub_exp);
 132                 if (!dctx->pub_exp)
 133                         return 0;
 134         }
 135         dctx->pad_mode = sctx->pad_mode;
 136         dctx->md = sctx->md;
 137         return 1;
 138 }
 139
 140 static int
 141 setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
 142 {
 143         if (ctx->tbuf)
 144                 return 1;
 145         ctx->tbuf = malloc(EVP_PKEY_size(pk->pkey));
 146         if (!ctx->tbuf)
 147                 return 0;
 148         return 1;
 149 }
 150
 151 static void
 152 pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
 153 {
 154         RSA_PKEY_CTX *rctx = ctx->data;
 155
 156         if (rctx) {
 157                 BN_free(rctx->pub_exp);
 158                 free(rctx->tbuf);
 159                 free(rctx);
 160         }
 161 }
 162
 163 static int
 164 pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
 165     const unsigned char *tbs, size_t tbslen)
 166 {
 167         int ret;
 168         RSA_PKEY_CTX *rctx = ctx->data;
 169         RSA *rsa = ctx->pkey->pkey.rsa;
 170
 171         if (rctx->md) {
 172                 if (tbslen != (size_t)EVP_MD_size(rctx->md)) {
 173                         RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
 174                         return -1;
 175                 }
 176
 177                 if (rctx->pad_mode == RSA_X931_PADDING) {
 178                         if (!setup_tbuf(rctx, ctx))
 179                                 return -1;
 180                         memcpy(rctx->tbuf, tbs, tbslen);
 181                         rctx->tbuf[tbslen] =
 182                             RSA_X931_hash_id(EVP_MD_type(rctx->md));
 183                         ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf, sig,
 184                             rsa, RSA_X931_PADDING);
 185                 } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
 186                         unsigned int sltmp;
 187
 188                         ret = RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig,
 189                             &sltmp, rsa);
 190                         if (ret <= 0)
 191                                 return ret;
 192                         ret = sltmp;
 193                 } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
 194                         if (!setup_tbuf(rctx, ctx))
 195                                 return -1;
 196                         if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf,
 197                             tbs, rctx->md, rctx->mgf1md, rctx->saltlen))
 198                                 return -1;
 199                         ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
 200                             sig, rsa, RSA_NO_PADDING);
 201                 } else
 202                         return -1;
 203         } else
 204                 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
 205                     rctx->pad_mode);
 206         if (ret < 0)
 207                 return ret;
 208         *siglen = ret;
 209         return 1;
 210 }
 211
 212 static int
 213 pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx, unsigned char *rout, size_t *routlen,
 214     const unsigned char *sig, size_t siglen)
 215 {
 216         int ret;
 217         RSA_PKEY_CTX *rctx = ctx->data;
 218
 219         if (rctx->md) {
 220                 if (rctx->pad_mode == RSA_X931_PADDING) {
 221                         if (!setup_tbuf(rctx, ctx))
 222                                 return -1;
 223                         ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 224                             ctx->pkey->pkey.rsa, RSA_X931_PADDING);
 225                         if (ret < 1)
 226                                 return 0;
 227                         ret--;
 228                         if (rctx->tbuf[ret] !=
 229                                 RSA_X931_hash_id(EVP_MD_type(rctx->md))) {
 230                                 RSAerror(RSA_R_ALGORITHM_MISMATCH);
 231                                 return 0;
 232                         }
 233                         if (ret != EVP_MD_size(rctx->md)) {
 234                                 RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
 235                                 return 0;
 236                         }
 237                         if (rout)
 238                                 memcpy(rout, rctx->tbuf, ret);
 239                 } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
 240                         size_t sltmp;
 241
 242                         ret = int_rsa_verify(EVP_MD_type(rctx->md), NULL, 0,
 243                             rout, &sltmp, sig, siglen, ctx->pkey->pkey.rsa);
 244                         if (ret <= 0)
 245                                 return 0;
 246                         ret = sltmp;
 247                 } else
 248                         return -1;
 249         } else
 250                 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
 251                     rctx->pad_mode);
 252         if (ret < 0)
 253                 return ret;
 254         *routlen = ret;
 255         return 1;
 256 }
 257
 258 static int
 259 pkey_rsa_verify(EVP_PKEY_CTX *ctx, const unsigned char *sig, size_t siglen,
 260     const unsigned char *tbs, size_t tbslen)
 261 {
 262         RSA_PKEY_CTX *rctx = ctx->data;
 263         RSA *rsa = ctx->pkey->pkey.rsa;
 264         size_t rslen;
 265
 266         if (rctx->md) {
 267                 if (rctx->pad_mode == RSA_PKCS1_PADDING)
 268                         return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
 269                             sig, siglen, rsa);
 270                 if (rctx->pad_mode == RSA_X931_PADDING) {
 271                         if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, sig,
 272                             siglen) <= 0)
 273                                 return 0;
 274                 } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
 275                         int ret;
 276
 277                         if (!setup_tbuf(rctx, ctx))
 278                                 return -1;
 279                         ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 280                             rsa, RSA_NO_PADDING);
 281                         if (ret <= 0)
 282                                 return 0;
 283                         ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md,
 284                             rctx->mgf1md, rctx->tbuf, rctx->saltlen);
 285                         if (ret <= 0)
 286                                 return 0;
 287                         return 1;
 288                 } else
 289                         return -1;
 290         } else {
 291                 if (!setup_tbuf(rctx, ctx))
 292                         return -1;
 293                 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa,
 294                     rctx->pad_mode);
 295                 if (rslen == 0)
 296                         return 0;
 297         }
 298
 299         if (rslen != tbslen || memcmp(tbs, rctx->tbuf, rslen))
 300                 return 0;
 301
 302         return 1;
 303 }
 304
 305 static int
 306 pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
 307     const unsigned char *in, size_t inlen)
 308 {
 309         int ret;
 310         RSA_PKEY_CTX *rctx = ctx->data;
 311
 312         ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 313             rctx->pad_mode);
 314         if (ret < 0)
 315                 return ret;
 316         *outlen = ret;
 317         return 1;
 318 }
 319
 320 static int
 321 pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
 322     const unsigned char *in, size_t inlen)
 323 {
 324         int ret;
 325         RSA_PKEY_CTX *rctx = ctx->data;
 326
 327         ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 328             rctx->pad_mode);
 329         if (ret < 0)
 330                 return ret;
 331         *outlen = ret;
 332         return 1;
 333 }
 334
 335 static int
 336 check_padding_md(const EVP_MD *md, int padding)
 337 {
 338         if (!md)
 339                 return 1;
 340
 341         if (padding == RSA_NO_PADDING) {
 342                 RSAerror(RSA_R_INVALID_PADDING_MODE);
 343                 return 0;
 344         }
 345
 346         if (padding == RSA_X931_PADDING) {
 347                 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1) {
 348                         RSAerror(RSA_R_INVALID_X931_DIGEST);
 349                         return 0;
 350                 }
 351                 return 1;
 352         }
 353
 354         return 1;
 355 }
 356
 357 static int
 358 pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 359 {
 360         RSA_PKEY_CTX *rctx = ctx->data;
 361
 362         switch (type) {
 363         case EVP_PKEY_CTRL_RSA_PADDING:
 364                 if (p1 >= RSA_PKCS1_PADDING && p1 <= RSA_PKCS1_PSS_PADDING) {
 365                         if (!check_padding_md(rctx->md, p1))
 366                                 return 0;
 367                         if (p1 == RSA_PKCS1_PSS_PADDING) {
 368                                 if (!(ctx->operation &
 369                                     (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
 370                                         goto bad_pad;
 371                                 if (!rctx->md)
 372                                         rctx->md = EVP_sha1();
 373                         }
 374                         if (p1 == RSA_PKCS1_OAEP_PADDING) {
 375                                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
 376                                         goto bad_pad;
 377                                 if (!rctx->md)
 378                                         rctx->md = EVP_sha1();
 379                         }
 380                         rctx->pad_mode = p1;
 381                         return 1;
 382                 }
 383 bad_pad:
 384                 RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
 385                 return -2;
 386
 387         case EVP_PKEY_CTRL_GET_RSA_PADDING:
 388                 *(int *)p2 = rctx->pad_mode;
 389                 return 1;
 390
 391         case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
 392         case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN:
 393                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
 394                         RSAerror(RSA_R_INVALID_PSS_SALTLEN);
 395                         return -2;
 396                 }
 397                 if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN)
 398                         *(int *)p2 = rctx->saltlen;
 399                 else {
 400                         if (p1 < -2)
 401                                 return -2;
 402                         rctx->saltlen = p1;
 403                 }
 404                 return 1;
 405
 406         case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
 407                 if (p1 < 256) {
 408                         RSAerror(RSA_R_INVALID_KEYBITS);
 409                         return -2;
 410                 }
 411                 rctx->nbits = p1;
 412                 return 1;
 413
 414         case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
 415                 if (!p2)
 416                         return -2;
 417                 rctx->pub_exp = p2;
 418                 return 1;
 419
 420         case EVP_PKEY_CTRL_MD:
 421                 if (!check_padding_md(p2, rctx->pad_mode))
 422                         return 0;
 423                 rctx->md = p2;
 424                 return 1;
 425
 426         case EVP_PKEY_CTRL_RSA_MGF1_MD:
 427         case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
 428                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
 429                         RSAerror(RSA_R_INVALID_MGF1_MD);
 430                         return -2;
 431                 }
 432                 if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD) {
 433                         if (rctx->mgf1md)
 434                                 *(const EVP_MD **)p2 = rctx->mgf1md;
 435                         else
 436                                 *(const EVP_MD **)p2 = rctx->md;
 437                 } else
 438                         rctx->mgf1md = p2;
 439                 return 1;
 440
 441         case EVP_PKEY_CTRL_DIGESTINIT:
 442         case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
 443         case EVP_PKEY_CTRL_PKCS7_DECRYPT:
 444         case EVP_PKEY_CTRL_PKCS7_SIGN:
 445                 return 1;
 446         case EVP_PKEY_CTRL_PEER_KEY:
 447                 RSAerror(RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
 448                 return -2;
 449
 450         default:
 451                 return -2;
 452         }
 453 }
 454
 455 static int
 456 pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx, const char *type, const char *value)
 457 {
 458         long lval;
 459         char *ep;
 460
 461         if (!value) {
 462                 RSAerror(RSA_R_VALUE_MISSING);
 463                 return 0;
 464         }
 465         if (!strcmp(type, "rsa_padding_mode")) {
 466                 int pm;
 467                 if (!strcmp(value, "pkcs1"))
 468                         pm = RSA_PKCS1_PADDING;
 469                 else if (!strcmp(value, "none"))
 470                         pm = RSA_NO_PADDING;
 471                 else if (!strcmp(value, "oeap"))
 472                         pm = RSA_PKCS1_OAEP_PADDING;
 473                 else if (!strcmp(value, "oaep"))
 474                         pm = RSA_PKCS1_OAEP_PADDING;
 475                 else if (!strcmp(value, "x931"))
 476                         pm = RSA_X931_PADDING;
 477                 else if (!strcmp(value, "pss"))
 478                         pm = RSA_PKCS1_PSS_PADDING;
 479                 else {
 480                         RSAerror(RSA_R_UNKNOWN_PADDING_TYPE);
 481                         return -2;
 482                 }
 483                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
 484         }
 485
 486         if (!strcmp(type, "rsa_pss_saltlen")) {
 487                 int saltlen;
 488
 489                 errno = 0;
 490                 lval = strtol(value, &ep, 10);
 491                 if (value[0] == '\0' || *ep != '\0')
 492                         goto not_a_number;
 493                 if ((errno == ERANGE &&
 494                     (lval == LONG_MAX || lval == LONG_MIN)) ||
 495                     (lval > INT_MAX || lval < INT_MIN))
 496                         goto out_of_range;
 497                 saltlen = lval;
 498                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
 499         }
 500
 501         if (!strcmp(type, "rsa_keygen_bits")) {
 502                 int nbits;
 503
 504                 errno = 0;
 505                 lval = strtol(value, &ep, 10);
 506                 if (value[0] == '\0' || *ep != '\0')
 507                         goto not_a_number;
 508                 if ((errno == ERANGE &&
 509                     (lval == LONG_MAX || lval == LONG_MIN)) ||
 510                     (lval > INT_MAX || lval < INT_MIN))
 511                         goto out_of_range;
 512                 nbits = lval;
 513                 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
 514         }
 515
 516         if (!strcmp(type, "rsa_keygen_pubexp")) {
 517                 int ret;
 518                 BIGNUM *pubexp = NULL;
 519
 520                 if (!BN_asc2bn(&pubexp, value))
 521                         return 0;
 522                 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
 523                 if (ret <= 0)
 524                         BN_free(pubexp);
 525                 return ret;
 526         }
 527
 528 not_a_number:
 529 out_of_range:
 530         return -2;
 531 }
 532
 533 static int
 534 pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 535 {
 536         RSA *rsa = NULL;
 537         RSA_PKEY_CTX *rctx = ctx->data;
 538         BN_GENCB *pcb, cb;
 539         int ret;
 540
 541         if (!rctx->pub_exp) {
 542                 rctx->pub_exp = BN_new();
 543                 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
 544                         return 0;
 545         }
 546         rsa = RSA_new();
 547         if (!rsa)
 548                 return 0;
 549         if (ctx->pkey_gencb) {
 550                 pcb = &cb;
 551                 evp_pkey_set_cb_translate(pcb, ctx);
 552         } else
 553                 pcb = NULL;
 554         ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
 555         if (ret > 0)
 556                 EVP_PKEY_assign_RSA(pkey, rsa);
 557         else
 558                 RSA_free(rsa);
 559         return ret;
 560 }
 561
 562 const EVP_PKEY_METHOD rsa_pkey_meth = {
 563         .pkey_id = EVP_PKEY_RSA,
 564         .flags = EVP_PKEY_FLAG_AUTOARGLEN,
 565
 566         .init = pkey_rsa_init,
 567         .copy = pkey_rsa_copy,
 568         .cleanup = pkey_rsa_cleanup,
 569
 570         .keygen = pkey_rsa_keygen,
 571
 572         .sign = pkey_rsa_sign,
 573
 574         .verify = pkey_rsa_verify,
 575
 576         .verify_recover = pkey_rsa_verifyrecover,
 577
 578         .encrypt = pkey_rsa_encrypt,
 579
 580         .decrypt = pkey_rsa_decrypt,
 581
 582         .ctrl = pkey_rsa_ctrl,
 583         .ctrl_str = pkey_rsa_ctrl_str
 584 };