heimdal: Fix CID 745516 Use after free
[Samba.git] / source4 / heimdal / lib / hcrypto / dh-ltm.c
blobf4e03139ff25a122ebfe2206992213d271b92843
1 /*
2 * Copyright (c) 2006 Kungliga Tekniska Högskolan
3 * (Royal Institute of Technology, Stockholm, Sweden).
4 * All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the name of the Institute nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
34 #ifdef HAVE_CONFIG_H
35 #include <config.h>
36 #endif
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <dh.h>
42 #include <roken.h>
44 #include "tommath.h"
46 static void
47 BN2mpz(mp_int *s, const BIGNUM *bn)
49 size_t len;
50 void *p;
52 len = BN_num_bytes(bn);
53 p = malloc(len);
54 BN_bn2bin(bn, p);
55 mp_read_unsigned_bin(s, p, len);
56 free(p);
60 static BIGNUM *
61 mpz2BN(mp_int *s)
63 size_t size;
64 BIGNUM *bn;
65 void *p;
67 size = mp_unsigned_bin_size(s);
68 p = malloc(size);
69 if (p == NULL && size != 0)
70 return NULL;
71 mp_to_unsigned_bin(s, p);
73 bn = BN_bin2bn(p, size, NULL);
74 free(p);
75 return bn;
82 #define DH_NUM_TRIES 10
84 static int
85 ltm_dh_generate_key(DH *dh)
87 mp_int pub, priv_key, g, p;
88 int have_private_key = (dh->priv_key != NULL);
89 int codes, times = 0;
90 int res;
92 if (dh->p == NULL || dh->g == NULL)
93 return 0;
95 while (times++ < DH_NUM_TRIES) {
96 if (!have_private_key) {
97 size_t bits = BN_num_bits(dh->p);
99 if (dh->priv_key)
100 BN_free(dh->priv_key);
102 dh->priv_key = BN_new();
103 if (dh->priv_key == NULL)
104 return 0;
105 if (!BN_rand(dh->priv_key, bits - 1, 0, 0)) {
106 BN_clear_free(dh->priv_key);
107 dh->priv_key = NULL;
108 return 0;
111 if (dh->pub_key) {
112 BN_free(dh->pub_key);
113 dh->pub_key = NULL;
116 mp_init_multi(&pub, &priv_key, &g, &p, NULL);
118 BN2mpz(&priv_key, dh->priv_key);
119 BN2mpz(&g, dh->g);
120 BN2mpz(&p, dh->p);
122 res = mp_exptmod(&g, &priv_key, &p, &pub);
124 mp_clear_multi(&priv_key, &g, &p, NULL);
125 if (res != 0)
126 continue;
128 dh->pub_key = mpz2BN(&pub);
129 mp_clear(&pub);
130 if (dh->pub_key == NULL)
131 return 0;
133 if (DH_check_pubkey(dh, dh->pub_key, &codes) && codes == 0)
134 break;
135 if (have_private_key)
136 return 0;
139 if (times >= DH_NUM_TRIES) {
140 if (!have_private_key && dh->priv_key) {
141 BN_free(dh->priv_key);
142 dh->priv_key = NULL;
144 if (dh->pub_key) {
145 BN_free(dh->pub_key);
146 dh->pub_key = NULL;
148 return 0;
151 return 1;
154 static int
155 ltm_dh_compute_key(unsigned char *shared, const BIGNUM * pub, DH *dh)
157 mp_int s, priv_key, p, peer_pub;
158 int ret;
160 if (dh->pub_key == NULL || dh->g == NULL || dh->priv_key == NULL)
161 return -1;
163 mp_init_multi(&s, &priv_key, &p, &peer_pub, NULL);
164 BN2mpz(&p, dh->p);
165 BN2mpz(&peer_pub, pub);
167 /* check if peers pubkey is reasonable */
168 if (mp_isneg(&peer_pub)
169 || mp_cmp(&peer_pub, &p) >= 0
170 || mp_cmp_d(&peer_pub, 1) <= 0)
172 ret = -1;
173 goto out;
176 BN2mpz(&priv_key, dh->priv_key);
178 ret = mp_exptmod(&peer_pub, &priv_key, &p, &s);
180 if (ret != 0) {
181 ret = -1;
182 goto out;
185 ret = mp_unsigned_bin_size(&s);
186 mp_to_unsigned_bin(&s, shared);
188 out:
189 mp_clear_multi(&s, &priv_key, &p, &peer_pub, NULL);
191 return ret;
194 static int
195 ltm_dh_generate_params(DH *dh, int a, int b, BN_GENCB *callback)
197 /* groups should already be known, we don't care about this */
198 return 0;
201 static int
202 ltm_dh_init(DH *dh)
204 return 1;
207 static int
208 ltm_dh_finish(DH *dh)
210 return 1;
218 const DH_METHOD _hc_dh_ltm_method = {
219 "hcrypto ltm DH",
220 ltm_dh_generate_key,
221 ltm_dh_compute_key,
222 NULL,
223 ltm_dh_init,
224 ltm_dh_finish,
226 NULL,
227 ltm_dh_generate_params
231 * DH implementation using libtommath.
233 * @return the DH_METHOD for the DH implementation using libtommath.
235 * @ingroup hcrypto_dh
238 const DH_METHOD *
239 DH_ltm_method(void)
241 return &_hc_dh_ltm_method;