source4/heimdal/lib/hcrypto/rsa.c

   1 /*
   2  * Copyright (c) 2006 - 2008 Kungliga Tekniska Högskolan
   3  * (Royal Institute of Technology, Stockholm, Sweden).
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  *
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * 3. Neither the name of the Institute nor the names of its contributors
  18  *    may be used to endorse or promote products derived from this software
  19  *    without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
  25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 #include <config.h>
  35
  36 #include <stdio.h>
  37 #include <stdlib.h>
  38 #include <krb5-types.h>
  39 #include <rfc2459_asn1.h>
  40
  41 #include <rsa.h>
  42
  43 #include <roken.h>
  44
  45 /**
  46  * @page page_rsa RSA - public-key cryptography
  47  *
  48  * RSA is named by its inventors (Ron Rivest, Adi Shamir, and Leonard
  49  * Adleman) (published in 1977), patented expired in 21 September 2000.
  50  *
  51  * See the library functions here: @ref hcrypto_rsa
  52  */
  53
  54 /**
  55  * Same as RSA_new_method() using NULL as engine.
  56  *
  57  * @return a newly allocated RSA object. Free with RSA_free().
  58  *
  59  * @ingroup hcrypto_rsa
  60  */
  61
  62 RSA *
  63 RSA_new(void)
  64 {
  65     return RSA_new_method(NULL);
  66 }
  67
  68 /**
  69  * Allocate a new RSA object using the engine, if NULL is specified as
  70  * the engine, use the default RSA engine as returned by
  71  * ENGINE_get_default_RSA().
  72  *
  73  * @param engine Specific what ENGINE RSA provider should be used.
  74  *
  75  * @return a newly allocated RSA object. Free with RSA_free().
  76  *
  77  * @ingroup hcrypto_rsa
  78  */
  79
  80 RSA *
  81 RSA_new_method(ENGINE *engine)
  82 {
  83     RSA *rsa;
  84
  85     rsa = calloc(1, sizeof(*rsa));
  86     if (rsa == NULL)
  87         return NULL;
  88
  89     rsa->references = 1;
  90
  91     if (engine) {
  92         ENGINE_up_ref(engine);
  93         rsa->engine = engine;
  94     } else {
  95         rsa->engine = ENGINE_get_default_RSA();
  96     }
  97
  98     if (rsa->engine) {
  99         rsa->meth = ENGINE_get_RSA(rsa->engine);
 100         if (rsa->meth == NULL) {
 101             ENGINE_finish(engine);
 102             free(rsa);
 103             return 0;
 104         }
 105     }
 106
 107     if (rsa->meth == NULL)
 108         rsa->meth = rk_UNCONST(RSA_get_default_method());
 109
 110     (*rsa->meth->init)(rsa);
 111
 112     return rsa;
 113 }
 114
 115 /**
 116  * Free an allocation RSA object.
 117  *
 118  * @param rsa the RSA object to free.
 119  * @ingroup hcrypto_rsa
 120  */
 121
 122 void
 123 RSA_free(RSA *rsa)
 124 {
 125     if (rsa->references <= 0)
 126         abort();
 127
 128     if (--rsa->references > 0)
 129         return;
 130
 131     (*rsa->meth->finish)(rsa);
 132
 133     if (rsa->engine)
 134         ENGINE_finish(rsa->engine);
 135
 136 #define free_if(f) if (f) { BN_free(f); }
 137     free_if(rsa->n);
 138     free_if(rsa->e);
 139     free_if(rsa->d);
 140     free_if(rsa->p);
 141     free_if(rsa->q);
 142     free_if(rsa->dmp1);
 143     free_if(rsa->dmq1);
 144     free_if(rsa->iqmp);
 145 #undef free_if
 146
 147     memset(rsa, 0, sizeof(*rsa));
 148     free(rsa);
 149 }
 150
 151 /**
 152  * Add an extra reference to the RSA object. The object should be free
 153  * with RSA_free() to drop the reference.
 154  *
 155  * @param rsa the object to add reference counting too.
 156  *
 157  * @return the current reference count, can't safely be used except
 158  * for debug printing.
 159  *
 160  * @ingroup hcrypto_rsa
 161  */
 162
 163 int
 164 RSA_up_ref(RSA *rsa)
 165 {
 166     return ++rsa->references;
 167 }
 168
 169 /**
 170  * Return the RSA_METHOD used for this RSA object.
 171  *
 172  * @param rsa the object to get the method from.
 173  *
 174  * @return the method used for this RSA object.
 175  *
 176  * @ingroup hcrypto_rsa
 177  */
 178
 179 const RSA_METHOD *
 180 RSA_get_method(const RSA *rsa)
 181 {
 182     return rsa->meth;
 183 }
 184
 185 /**
 186  * Set a new method for the RSA keypair.
 187  *
 188  * @param rsa rsa parameter.
 189  * @param method the new method for the RSA parameter.
 190  *
 191  * @return 1 on success.
 192  *
 193  * @ingroup hcrypto_rsa
 194  */
 195
 196 int
 197 RSA_set_method(RSA *rsa, const RSA_METHOD *method)
 198 {
 199     (*rsa->meth->finish)(rsa);
 200
 201     if (rsa->engine) {
 202         ENGINE_finish(rsa->engine);
 203         rsa->engine = NULL;
 204     }
 205
 206     rsa->meth = method;
 207     (*rsa->meth->init)(rsa);
 208     return 1;
 209 }
 210
 211 /**
 212  * Set the application data for the RSA object.
 213  *
 214  * @param rsa the rsa object to set the parameter for
 215  * @param arg the data object to store
 216  *
 217  * @return 1 on success.
 218  *
 219  * @ingroup hcrypto_rsa
 220  */
 221
 222 int
 223 RSA_set_app_data(RSA *rsa, void *arg)
 224 {
 225     rsa->ex_data.sk = arg;
 226     return 1;
 227 }
 228
 229 /**
 230  * Get the application data for the RSA object.
 231  *
 232  * @param rsa the rsa object to get the parameter for
 233  *
 234  * @return the data object
 235  *
 236  * @ingroup hcrypto_rsa
 237  */
 238
 239 void *
 240 RSA_get_app_data(RSA *rsa)
 241 {
 242     return rsa->ex_data.sk;
 243 }
 244
 245 int
 246 RSA_check_key(const RSA *key)
 247 {
 248     static const unsigned char inbuf[] = "hello, world!";
 249     RSA *rsa = rk_UNCONST(key);
 250     void *buffer;
 251     int ret;
 252
 253     /*
 254      * XXX I have no clue how to implement this w/o a bignum library.
 255      * Well, when we have a RSA key pair, we can try to encrypt/sign
 256      * and then decrypt/verify.
 257      */
 258
 259     if ((rsa->d == NULL || rsa->n == NULL) &&
 260         (rsa->p == NULL || rsa->q || rsa->dmp1 == NULL || rsa->dmq1 == NULL || rsa->iqmp == NULL))
 261         return 0;
 262
 263     buffer = malloc(RSA_size(rsa));
 264     if (buffer == NULL)
 265         return 0;
 266
 267     ret = RSA_private_encrypt(sizeof(inbuf), inbuf, buffer,
 268                              rsa, RSA_PKCS1_PADDING);
 269     if (ret == -1) {
 270         free(buffer);
 271         return 0;
 272     }
 273
 274     ret = RSA_public_decrypt(ret, buffer, buffer,
 275                               rsa, RSA_PKCS1_PADDING);
 276     if (ret == -1) {
 277         free(buffer);
 278         return 0;
 279     }
 280
 281     if (ret == sizeof(inbuf) && memcmp(buffer, inbuf, sizeof(inbuf)) == 0) {
 282         free(buffer);
 283         return 1;
 284     }
 285     free(buffer);
 286     return 0;
 287 }
 288
 289 int
 290 RSA_size(const RSA *rsa)
 291 {
 292     return BN_num_bytes(rsa->n);
 293 }
 294
 295 #define RSAFUNC(name, body) \
 296 int \
 297 name(int flen,const unsigned char* f, unsigned char* t, RSA* r, int p){\
 298     return body; \
 299 }
 300
 301 RSAFUNC(RSA_public_encrypt, (r)->meth->rsa_pub_enc(flen, f, t, r, p))
 302 RSAFUNC(RSA_public_decrypt, (r)->meth->rsa_pub_dec(flen, f, t, r, p))
 303 RSAFUNC(RSA_private_encrypt, (r)->meth->rsa_priv_enc(flen, f, t, r, p))
 304 RSAFUNC(RSA_private_decrypt, (r)->meth->rsa_priv_dec(flen, f, t, r, p))
 305
 306 /* XXX */
 307 int
 308 RSA_sign(int type, const unsigned char *from, unsigned int flen,
 309          unsigned char *to, unsigned int *tlen, RSA *rsa)
 310 {
 311     return -1;
 312 }
 313
 314 int
 315 RSA_verify(int type, const unsigned char *from, unsigned int flen,
 316            unsigned char *to, unsigned int tlen, RSA *rsa)
 317 {
 318     return -1;
 319 }
 320
 321 /*
 322  * A NULL RSA_METHOD that returns failure for all operations. This is
 323  * used as the default RSA method if we don't have any native
 324  * support.
 325  */
 326
 327 static RSAFUNC(null_rsa_public_encrypt, -1)
 328 static RSAFUNC(null_rsa_public_decrypt, -1)
 329 static RSAFUNC(null_rsa_private_encrypt, -1)
 330 static RSAFUNC(null_rsa_private_decrypt, -1)
 331
 332 /*
 333  *
 334  */
 335
 336 int
 337 RSA_generate_key_ex(RSA *r, int bits, BIGNUM *e, BN_GENCB *cb)
 338 {
 339     if (r->meth->rsa_keygen)
 340         return (*r->meth->rsa_keygen)(r, bits, e, cb);
 341     return 0;
 342 }
 343
 344
 345 /*
 346  *
 347  */
 348
 349 static int
 350 null_rsa_init(RSA *rsa)
 351 {
 352     return 1;
 353 }
 354
 355 static int
 356 null_rsa_finish(RSA *rsa)
 357 {
 358     return 1;
 359 }
 360
 361 static const RSA_METHOD rsa_null_method = {
 362     "hcrypto null RSA",
 363     null_rsa_public_encrypt,
 364     null_rsa_public_decrypt,
 365     null_rsa_private_encrypt,
 366     null_rsa_private_decrypt,
 367     NULL,
 368     NULL,
 369     null_rsa_init,
 370     null_rsa_finish,
 371     0,
 372     NULL,
 373     NULL,
 374     NULL
 375 };
 376
 377 const RSA_METHOD *
 378 RSA_null_method(void)
 379 {
 380     return &rsa_null_method;
 381 }
 382
 383 extern const RSA_METHOD hc_rsa_imath_method;
 384 #ifdef HAVE_GMP
 385 static const RSA_METHOD *default_rsa_method = &hc_rsa_gmp_method;
 386 #else
 387 static const RSA_METHOD *default_rsa_method = &hc_rsa_imath_method;
 388 #endif
 389
 390 const RSA_METHOD *
 391 RSA_get_default_method(void)
 392 {
 393     return default_rsa_method;
 394 }
 395
 396 void
 397 RSA_set_default_method(const RSA_METHOD *meth)
 398 {
 399     default_rsa_method = meth;
 400 }
 401
 402 /*
 403  *
 404  */
 405
 406 static BIGNUM *
 407 heim_int2BN(const heim_integer *i)
 408 {
 409     BIGNUM *bn;
 410
 411     bn = BN_bin2bn(i->data, i->length, NULL);
 412     if (bn)
 413         BN_set_negative(bn, i->negative);
 414     return bn;
 415 }
 416
 417 static int
 418 bn2heim_int(BIGNUM *bn, heim_integer *integer)
 419 {
 420     integer->length = BN_num_bytes(bn);
 421     integer->data = malloc(integer->length);
 422     if (integer->data == NULL) {
 423         integer->length = 0;
 424         return ENOMEM;
 425     }
 426     BN_bn2bin(bn, integer->data);
 427     integer->negative = BN_is_negative(bn);
 428     return 0;
 429 }
 430
 431
 432 RSA *
 433 d2i_RSAPrivateKey(RSA *rsa, const unsigned char **pp, size_t len)
 434 {
 435     RSAPrivateKey data;
 436     RSA *k = rsa;
 437     size_t size;
 438     int ret;
 439
 440     ret = decode_RSAPrivateKey(*pp, len, &data, &size);
 441     if (ret)
 442         return NULL;
 443
 444     *pp += size;
 445
 446     if (k == NULL) {
 447         k = RSA_new();
 448         if (k == NULL) {
 449             free_RSAPrivateKey(&data);
 450             return NULL;
 451         }
 452     }
 453
 454     k->n = heim_int2BN(&data.modulus);
 455     k->e = heim_int2BN(&data.publicExponent);
 456     k->d = heim_int2BN(&data.privateExponent);
 457     k->p = heim_int2BN(&data.prime1);
 458     k->q = heim_int2BN(&data.prime2);
 459     k->dmp1 = heim_int2BN(&data.exponent1);
 460     k->dmq1 = heim_int2BN(&data.exponent2);
 461     k->iqmp = heim_int2BN(&data.coefficient);
 462     free_RSAPrivateKey(&data);
 463
 464     if (k->n == NULL || k->e == NULL || k->d == NULL || k->p == NULL ||
 465         k->q == NULL || k->dmp1 == NULL || k->dmq1 == NULL || k->iqmp == NULL)
 466     {
 467         RSA_free(k);
 468         return NULL;
 469     }
 470
 471     return k;
 472 }
 473
 474 int
 475 i2d_RSAPrivateKey(RSA *rsa, unsigned char **pp)
 476 {
 477     RSAPrivateKey data;
 478     size_t size;
 479     int ret;
 480
 481     if (rsa->n == NULL || rsa->e == NULL || rsa->d == NULL || rsa->p == NULL ||
 482         rsa->q == NULL || rsa->dmp1 == NULL || rsa->dmq1 == NULL ||
 483         rsa->iqmp == NULL)
 484         return -1;
 485
 486     memset(&data, 0, sizeof(data));
 487
 488     ret  = bn2heim_int(rsa->n, &data.modulus);
 489     ret |= bn2heim_int(rsa->e, &data.publicExponent);
 490     ret |= bn2heim_int(rsa->d, &data.privateExponent);
 491     ret |= bn2heim_int(rsa->p, &data.prime1);
 492     ret |= bn2heim_int(rsa->q, &data.prime2);
 493     ret |= bn2heim_int(rsa->dmp1, &data.exponent1);
 494     ret |= bn2heim_int(rsa->dmq1, &data.exponent2);
 495     ret |= bn2heim_int(rsa->iqmp, &data.coefficient);
 496     if (ret) {
 497         free_RSAPrivateKey(&data);
 498         return -1;
 499     }
 500
 501     if (pp == NULL) {
 502         size = length_RSAPrivateKey(&data);
 503         free_RSAPrivateKey(&data);
 504     } else {
 505         void *p;
 506         size_t len;
 507
 508         ASN1_MALLOC_ENCODE(RSAPrivateKey, p, len, &data, &size, ret);
 509         free_RSAPrivateKey(&data);
 510         if (ret)
 511             return -1;
 512         if (len != size)
 513             abort();
 514
 515         memcpy(*pp, p, size);
 516         free(p);
 517
 518         *pp += size;
 519
 520     }
 521     return size;
 522 }
 523
 524 int
 525 i2d_RSAPublicKey(RSA *rsa, unsigned char **pp)
 526 {
 527     RSAPublicKey data;
 528     size_t size;
 529     int ret;
 530
 531     memset(&data, 0, sizeof(data));
 532
 533     if (bn2heim_int(rsa->n, &data.modulus) ||
 534         bn2heim_int(rsa->e, &data.publicExponent))
 535     {
 536         free_RSAPublicKey(&data);
 537         return -1;
 538     }
 539
 540     if (pp == NULL) {
 541         size = length_RSAPublicKey(&data);
 542         free_RSAPublicKey(&data);
 543     } else {
 544         void *p;
 545         size_t len;
 546
 547         ASN1_MALLOC_ENCODE(RSAPublicKey, p, len, &data, &size, ret);
 548         free_RSAPublicKey(&data);
 549         if (ret)
 550             return -1;
 551         if (len != size)
 552             abort();
 553
 554         memcpy(*pp, p, size);
 555         free(p);
 556
 557         *pp += size;
 558     }
 559
 560     return size;
 561 }