[HEIMDAL-646] malloc(0) checks for AIX
[heimdal.git] / lib / hcrypto / dh-imath.c
blob822d5a328bc5cb1af3a9f0c928860689115bca47
1 /*
2 * Copyright (c) 2006 Kungliga Tekniska Högskolan
3 * (Royal Institute of Technology, Stockholm, Sweden).
4 * All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the name of the Institute nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
34 #ifdef HAVE_CONFIG_H
35 #include <config.h>
36 #endif
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <dh.h>
42 #include <roken.h>
44 #include "imath/imath.h"
46 static void
47 BN2mpz(mpz_t *s, const BIGNUM *bn)
49 size_t len;
50 void *p;
52 len = BN_num_bytes(bn);
53 p = malloc(len);
54 BN_bn2bin(bn, p);
55 mp_int_read_unsigned(s, p, len);
56 free(p);
60 static BIGNUM *
61 mpz2BN(mpz_t *s)
63 size_t size;
64 BIGNUM *bn;
65 void *p;
67 size = mp_int_unsigned_len(s);
68 p = malloc(size);
69 if (p == NULL && size != 0)
70 return NULL;
71 mp_int_to_unsigned(s, p, size);
73 bn = BN_bin2bn(p, size, NULL);
74 free(p);
75 return bn;
82 #define DH_NUM_TRIES 10
84 static int
85 dh_generate_key(DH *dh)
87 mpz_t pub, priv_key, g, p;
88 int have_private_key = (dh->priv_key != NULL);
89 int codes, times = 0;
90 mp_result res;
92 if (dh->p == NULL || dh->g == NULL)
93 return 0;
95 while (times++ < DH_NUM_TRIES) {
96 if (!have_private_key) {
97 size_t bits = BN_num_bits(dh->p);
99 if (dh->priv_key)
100 BN_free(dh->priv_key);
102 dh->priv_key = BN_new();
103 if (dh->priv_key == NULL)
104 return 0;
105 if (!BN_rand(dh->priv_key, bits - 1, 0, 0)) {
106 BN_clear_free(dh->priv_key);
107 dh->priv_key = NULL;
108 return 0;
111 if (dh->pub_key)
112 BN_free(dh->pub_key);
114 mp_int_init(&pub);
115 mp_int_init(&priv_key);
116 mp_int_init(&g);
117 mp_int_init(&p);
119 BN2mpz(&priv_key, dh->priv_key);
120 BN2mpz(&g, dh->g);
121 BN2mpz(&p, dh->p);
123 res = mp_int_exptmod(&g, &priv_key, &p, &pub);
125 mp_int_clear(&priv_key);
126 mp_int_clear(&g);
127 mp_int_clear(&p);
128 if (res != MP_OK)
129 continue;
131 dh->pub_key = mpz2BN(&pub);
132 mp_int_clear(&pub);
133 if (dh->pub_key == NULL)
134 return 0;
136 if (DH_check_pubkey(dh, dh->pub_key, &codes) && codes == 0)
137 break;
138 if (have_private_key)
139 return 0;
142 if (times >= DH_NUM_TRIES) {
143 if (!have_private_key && dh->priv_key) {
144 BN_free(dh->priv_key);
145 dh->priv_key = NULL;
147 if (dh->pub_key) {
148 BN_free(dh->pub_key);
149 dh->pub_key = NULL;
151 return 0;
154 return 1;
157 static int
158 dh_compute_key(unsigned char *shared, const BIGNUM * pub, DH *dh)
160 mpz_t s, priv_key, p, peer_pub;
161 size_t size = 0;
162 mp_result res;
164 if (dh->pub_key == NULL || dh->g == NULL || dh->priv_key == NULL)
165 return -1;
167 mp_int_init(&p);
168 BN2mpz(&p, dh->p);
170 mp_int_init(&peer_pub);
171 BN2mpz(&peer_pub, pub);
173 /* check if peers pubkey is reasonable */
174 if (MP_SIGN(&peer_pub) == MP_NEG
175 || mp_int_compare(&peer_pub, &p) >= 0
176 || mp_int_compare_value(&peer_pub, 1) <= 0)
178 mp_int_clear(&p);
179 mp_int_clear(&peer_pub);
180 return -1;
183 mp_int_init(&priv_key);
184 BN2mpz(&priv_key, dh->priv_key);
186 mp_int_init(&s);
188 mp_int_exptmod(&peer_pub, &priv_key, &p, &s);
190 mp_int_clear(&p);
191 mp_int_clear(&peer_pub);
192 mp_int_clear(&priv_key);
194 size = mp_int_unsigned_len(&s);
195 res = mp_int_to_unsigned(&s, shared, size);
196 mp_int_clear(&s);
198 return (res == MP_OK) ? size : -1;
201 static int
202 dh_generate_params(DH *dh, int a, int b, BN_GENCB *callback)
204 /* groups should already be known, we don't care about this */
205 return 0;
208 static int
209 dh_init(DH *dh)
211 return 1;
214 static int
215 dh_finish(DH *dh)
217 return 1;
225 const DH_METHOD _hc_dh_imath_method = {
226 "hcrypto imath DH",
227 dh_generate_key,
228 dh_compute_key,
229 NULL,
230 dh_init,
231 dh_finish,
233 NULL,
234 dh_generate_params
238 * DH implementation using libimath.
240 * @return the DH_METHOD for the DH implementation using libimath.
242 * @ingroup hcrypto_dh
245 const DH_METHOD *
246 DH_imath_method(void)
248 return &_hc_dh_imath_method;