big svn cleanup
[anytun.git] / src / openvpn / cryptoapi.c
blob275fec98d8365b338b8074d053bcba74ecb88737
1 /*
2 * Copyright (c) 2004 Peter 'Luna' Runestig <peter@runestig.com>
3 * All rights reserved.
5 * Redistribution and use in source and binary forms, with or without modifi-
6 * cation, are permitted provided that the following conditions are met:
8 * o Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
11 * o Redistributions in binary form must reproduce the above copyright no-
12 * tice, this list of conditions and the following disclaimer in the do-
13 * cumentation and/or other materials provided with the distribution.
15 * o The names of the contributors may not be used to endorse or promote
16 * products derived from this software without specific prior written
17 * permission.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LI-
23 * ABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUEN-
24 * TIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEV-
26 * ER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABI-
27 * LITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
28 * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 #include <windows.h>
31 #include <wincrypt.h>
32 #include <stdio.h>
33 #include <ctype.h>
34 #include <assert.h>
35 #include <openssl/ssl.h>
36 #include <openssl/err.h>
38 #ifdef __MINGW32_VERSION
39 /* MinGW w32api is incomplete when it comes to CryptoAPI, as per version 3.1
40 * anyway. This is a hack around that problem. */
41 #define CALG_SSL3_SHAMD5 (ALG_CLASS_HASH | ALG_TYPE_ANY | ALG_SID_SSL3SHAMD5)
42 #define CERT_SYSTEM_STORE_LOCATION_SHIFT 16
43 #define CERT_SYSTEM_STORE_CURRENT_USER_ID 1
44 #define CERT_SYSTEM_STORE_CURRENT_USER (CERT_SYSTEM_STORE_CURRENT_USER_ID << CERT_SYSTEM_STORE_LOCATION_SHIFT)
45 #define CERT_STORE_READONLY_FLAG 0x00008000
46 #define CERT_STORE_OPEN_EXISTING_FLAG 0x00004000
47 #define CRYPT_ACQUIRE_COMPARE_KEY_FLAG 0x00000004
48 static HINSTANCE crypt32dll = NULL;
49 static BOOL WINAPI (*CryptAcquireCertificatePrivateKey) (PCCERT_CONTEXT pCert, DWORD dwFlags,
50 void *pvReserved, HCRYPTPROV *phCryptProv, DWORD *pdwKeySpec, BOOL *pfCallerFreeProv) = NULL;
51 #endif
53 /* Size of an SSL signature: MD5+SHA1 */
54 #define SSL_SIG_LENGTH 36
56 /* try to funnel any Windows/CryptoAPI error messages to OpenSSL ERR_... */
57 #define ERR_LIB_CRYPTOAPI (ERR_LIB_USER + 69) /* 69 is just a number... */
58 #define CRYPTOAPIerr(f) err_put_ms_error(GetLastError(), (f), __FILE__, __LINE__)
59 #define CRYPTOAPI_F_CERT_OPEN_SYSTEM_STORE 100
60 #define CRYPTOAPI_F_CERT_FIND_CERTIFICATE_IN_STORE 101
61 #define CRYPTOAPI_F_CRYPT_ACQUIRE_CERTIFICATE_PRIVATE_KEY 102
62 #define CRYPTOAPI_F_CRYPT_CREATE_HASH 103
63 #define CRYPTOAPI_F_CRYPT_GET_HASH_PARAM 104
64 #define CRYPTOAPI_F_CRYPT_SET_HASH_PARAM 105
65 #define CRYPTOAPI_F_CRYPT_SIGN_HASH 106
66 #define CRYPTOAPI_F_LOAD_LIBRARY 107
67 #define CRYPTOAPI_F_GET_PROC_ADDRESS 108
69 static ERR_STRING_DATA CRYPTOAPI_str_functs[] = {
70 { ERR_PACK(ERR_LIB_CRYPTOAPI, 0, 0), "microsoft cryptoapi"},
71 { ERR_PACK(0, CRYPTOAPI_F_CERT_OPEN_SYSTEM_STORE, 0), "CertOpenSystemStore" },
72 { ERR_PACK(0, CRYPTOAPI_F_CERT_FIND_CERTIFICATE_IN_STORE, 0), "CertFindCertificateInStore" },
73 { ERR_PACK(0, CRYPTOAPI_F_CRYPT_ACQUIRE_CERTIFICATE_PRIVATE_KEY, 0), "CryptAcquireCertificatePrivateKey" },
74 { ERR_PACK(0, CRYPTOAPI_F_CRYPT_CREATE_HASH, 0), "CryptCreateHash" },
75 { ERR_PACK(0, CRYPTOAPI_F_CRYPT_GET_HASH_PARAM, 0), "CryptGetHashParam" },
76 { ERR_PACK(0, CRYPTOAPI_F_CRYPT_SET_HASH_PARAM, 0), "CryptSetHashParam" },
77 { ERR_PACK(0, CRYPTOAPI_F_CRYPT_SIGN_HASH, 0), "CryptSignHash" },
78 { ERR_PACK(0, CRYPTOAPI_F_LOAD_LIBRARY, 0), "LoadLibrary" },
79 { ERR_PACK(0, CRYPTOAPI_F_GET_PROC_ADDRESS, 0), "GetProcAddress" },
80 { 0, NULL }
83 typedef struct _CAPI_DATA {
84 const CERT_CONTEXT *cert_context;
85 HCRYPTPROV crypt_prov;
86 DWORD key_spec;
87 BOOL free_crypt_prov;
88 } CAPI_DATA;
90 static char *ms_error_text(DWORD ms_err)
92 LPVOID lpMsgBuf = NULL;
93 char *rv = NULL;
95 FormatMessage(
96 FORMAT_MESSAGE_ALLOCATE_BUFFER |
97 FORMAT_MESSAGE_FROM_SYSTEM |
98 FORMAT_MESSAGE_IGNORE_INSERTS,
99 NULL, ms_err,
100 MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), /* Default language */
101 (LPTSTR) &lpMsgBuf, 0, NULL);
102 if (lpMsgBuf) {
103 char *p;
104 rv = strdup(lpMsgBuf);
105 LocalFree(lpMsgBuf);
106 /* trim to the left */
107 if (rv)
108 for (p = rv + strlen(rv) - 1; p >= rv; p--) {
109 if (isspace(*p))
110 *p = '\0';
111 else
112 break;
115 return rv;
118 static void err_put_ms_error(DWORD ms_err, int func, const char *file, int line)
120 static int init = 0;
121 # define ERR_MAP_SZ 16
122 static struct {
123 int err;
124 DWORD ms_err; /* I don't think we get more than 16 *different* errors */
125 } err_map[ERR_MAP_SZ]; /* in here, before we give up the whole thing... */
126 int i;
128 if (ms_err == 0)
129 /* 0 is not an error */
130 return;
131 if (!init) {
132 ERR_load_strings(ERR_LIB_CRYPTOAPI, CRYPTOAPI_str_functs);
133 memset(&err_map, 0, sizeof(err_map));
134 init++;
136 /* since MS error codes are 32 bit, and the ones in the ERR_... system is
137 * only 12, we must have a mapping table between them. */
138 for (i = 0; i < ERR_MAP_SZ; i++) {
139 if (err_map[i].ms_err == ms_err) {
140 ERR_PUT_error(ERR_LIB_CRYPTOAPI, func, err_map[i].err, file, line);
141 break;
142 } else if (err_map[i].ms_err == 0 ) {
143 /* end of table, add new entry */
144 ERR_STRING_DATA *esd = calloc(2, sizeof(*esd));
145 if (esd == NULL)
146 break;
147 err_map[i].ms_err = ms_err;
148 err_map[i].err = esd->error = i + 100;
149 esd->string = ms_error_text(ms_err);
150 ERR_load_strings(ERR_LIB_CRYPTOAPI, esd);
151 ERR_PUT_error(ERR_LIB_CRYPTOAPI, func, err_map[i].err, file, line);
152 break;
157 /* encrypt */
158 static int rsa_pub_enc(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding)
160 /* I haven't been able to trigger this one, but I want to know if it happens... */
161 assert(0);
163 return 0;
166 /* verify arbitrary data */
167 static int rsa_pub_dec(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding)
169 /* I haven't been able to trigger this one, but I want to know if it happens... */
170 assert(0);
172 return 0;
175 /* sign arbitrary data */
176 static int rsa_priv_enc(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding)
178 CAPI_DATA *cd = (CAPI_DATA *) rsa->meth->app_data;
179 HCRYPTHASH hash;
180 DWORD hash_size, len, i;
181 unsigned char *buf;
183 if (cd == NULL) {
184 RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT, ERR_R_PASSED_NULL_PARAMETER);
185 return 0;
187 if (padding != RSA_PKCS1_PADDING) {
188 /* AFAICS, CryptSignHash() *always* uses PKCS1 padding. */
189 RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT, RSA_R_UNKNOWN_PADDING_TYPE);
190 return 0;
192 /* Unfortunately, there is no "CryptSign()" function in CryptoAPI, that would
193 * be way to straightforward for M$, I guess... So we have to do it this
194 * tricky way instead, by creating a "Hash", and load the already-made hash
195 * from 'from' into it. */
196 /* For now, we only support NID_md5_sha1 */
197 if (flen != SSL_SIG_LENGTH) {
198 RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT, RSA_R_INVALID_MESSAGE_LENGTH);
199 return 0;
201 if (!CryptCreateHash(cd->crypt_prov, CALG_SSL3_SHAMD5, 0, 0, &hash)) {
202 CRYPTOAPIerr(CRYPTOAPI_F_CRYPT_CREATE_HASH);
203 return 0;
205 len = sizeof(hash_size);
206 if (!CryptGetHashParam(hash, HP_HASHSIZE, (BYTE *) &hash_size, &len, 0)) {
207 CRYPTOAPIerr(CRYPTOAPI_F_CRYPT_GET_HASH_PARAM);
208 CryptDestroyHash(hash);
209 return 0;
211 if ((int) hash_size != flen) {
212 RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT, RSA_R_INVALID_MESSAGE_LENGTH);
213 CryptDestroyHash(hash);
214 return 0;
216 if (!CryptSetHashParam(hash, HP_HASHVAL, (BYTE * ) from, 0)) {
217 CRYPTOAPIerr(CRYPTOAPI_F_CRYPT_SET_HASH_PARAM);
218 CryptDestroyHash(hash);
219 return 0;
222 len = RSA_size(rsa);
223 buf = malloc(len);
224 if (buf == NULL) {
225 RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT, ERR_R_MALLOC_FAILURE);
226 CryptDestroyHash(hash);
227 return 0;
229 if (!CryptSignHash(hash, cd->key_spec, NULL, 0, buf, &len)) {
230 CRYPTOAPIerr(CRYPTOAPI_F_CRYPT_SIGN_HASH);
231 CryptDestroyHash(hash);
232 free(buf);
233 return 0;
235 /* and now, we have to reverse the byte-order in the result from CryptSignHash()... */
236 for (i = 0; i < len; i++)
237 to[i] = buf[len - i - 1];
238 free(buf);
240 CryptDestroyHash(hash);
241 return len;
244 /* decrypt */
245 static int rsa_priv_dec(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding)
247 /* I haven't been able to trigger this one, but I want to know if it happens... */
248 assert(0);
250 return 0;
253 /* called at RSA_new */
254 static int init(RSA *rsa)
257 return 0;
260 /* called at RSA_free */
261 static int finish(RSA *rsa)
263 CAPI_DATA *cd = (CAPI_DATA *) rsa->meth->app_data;
265 if (cd == NULL)
266 return 0;
267 if (cd->crypt_prov && cd->free_crypt_prov)
268 CryptReleaseContext(cd->crypt_prov, 0);
269 if (cd->cert_context)
270 CertFreeCertificateContext(cd->cert_context);
271 free(rsa->meth->app_data);
272 free((char *) rsa->meth);
273 rsa->meth = NULL;
274 return 1;
277 static const CERT_CONTEXT *find_certificate_in_store(const char *cert_prop, HCERTSTORE cert_store)
279 /* Find, and use, the desired certificate from the store. The
280 * 'cert_prop' certificate search string can look like this:
281 * SUBJ:<certificate substring to match>
282 * THUMB:<certificate thumbprint hex value>, e.g.
283 * THUMB:f6 49 24 41 01 b4 fb 44 0c ce f4 36 ae d0 c4 c9 df 7a b6 28
285 const CERT_CONTEXT *rv = NULL;
287 if (!strncmp(cert_prop, "SUBJ:", 5)) {
288 /* skip the tag */
289 cert_prop += 5;
290 rv = CertFindCertificateInStore(cert_store, X509_ASN_ENCODING | PKCS_7_ASN_ENCODING,
291 0, CERT_FIND_SUBJECT_STR_A, cert_prop, NULL);
293 } else if (!strncmp(cert_prop, "THUMB:", 6)) {
294 unsigned char hash[255];
295 char *p;
296 int i, x = 0;
297 CRYPT_HASH_BLOB blob;
299 /* skip the tag */
300 cert_prop += 6;
301 for (p = (char *) cert_prop, i = 0; *p && i < sizeof(hash); i++) {
302 if (*p >= '0' && *p <= '9')
303 x = (*p - '0') << 4;
304 else if (*p >= 'A' && *p <= 'F')
305 x = (*p - 'A' + 10) << 4;
306 else if (*p >= 'a' && *p <= 'f')
307 x = (*p - 'a' + 10) << 4;
308 if (!*++p) /* unexpected end of string */
309 break;
310 if (*p >= '0' && *p <= '9')
311 x += *p - '0';
312 else if (*p >= 'A' && *p <= 'F')
313 x += *p - 'A' + 10;
314 else if (*p >= 'a' && *p <= 'f')
315 x += *p - 'a' + 10;
316 hash[i] = x;
317 /* skip any space(s) between hex numbers */
318 for (p++; *p && *p == ' '; p++);
320 blob.cbData = i;
321 blob.pbData = (unsigned char *) &hash;
322 rv = CertFindCertificateInStore(cert_store, X509_ASN_ENCODING | PKCS_7_ASN_ENCODING,
323 0, CERT_FIND_HASH, &blob, NULL);
327 return rv;
330 int SSL_CTX_use_CryptoAPI_certificate(SSL_CTX *ssl_ctx, const char *cert_prop)
332 HCERTSTORE cs;
333 X509 *cert = NULL;
334 RSA *rsa = NULL, *pub_rsa;
335 CAPI_DATA *cd = calloc(1, sizeof(*cd));
336 RSA_METHOD *my_rsa_method = calloc(1, sizeof(*my_rsa_method));
338 if (cd == NULL || my_rsa_method == NULL) {
339 SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE_FILE, ERR_R_MALLOC_FAILURE);
340 goto err;
342 /* search CURRENT_USER first, then LOCAL_MACHINE */
343 cs = CertOpenStore((LPCSTR) CERT_STORE_PROV_SYSTEM, 0, 0, CERT_SYSTEM_STORE_CURRENT_USER |
344 CERT_STORE_OPEN_EXISTING_FLAG | CERT_STORE_READONLY_FLAG, L"MY");
345 if (cs == NULL) {
346 CRYPTOAPIerr(CRYPTOAPI_F_CERT_OPEN_SYSTEM_STORE);
347 goto err;
349 cd->cert_context = find_certificate_in_store(cert_prop, cs);
350 CertCloseStore(cs, 0);
351 if (!cd->cert_context) {
352 cs = CertOpenStore((LPCSTR) CERT_STORE_PROV_SYSTEM, 0, 0, CERT_SYSTEM_STORE_LOCAL_MACHINE |
353 CERT_STORE_OPEN_EXISTING_FLAG | CERT_STORE_READONLY_FLAG, L"MY");
354 if (cs == NULL) {
355 CRYPTOAPIerr(CRYPTOAPI_F_CERT_OPEN_SYSTEM_STORE);
356 goto err;
358 cd->cert_context = find_certificate_in_store(cert_prop, cs);
359 CertCloseStore(cs, 0);
360 if (cd->cert_context == NULL) {
361 CRYPTOAPIerr(CRYPTOAPI_F_CERT_FIND_CERTIFICATE_IN_STORE);
362 goto err;
366 /* cert_context->pbCertEncoded is the cert X509 DER encoded. */
367 cert = d2i_X509(NULL, (unsigned char **) &cd->cert_context->pbCertEncoded,
368 cd->cert_context->cbCertEncoded);
369 if (cert == NULL) {
370 SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE_FILE, ERR_R_ASN1_LIB);
371 goto err;
374 /* set up stuff to use the private key */
375 #ifdef __MINGW32_VERSION
376 /* MinGW w32api is incomplete when it comes to CryptoAPI, as per version 3.1
377 * anyway. This is a hack around that problem. */
378 if (crypt32dll == NULL) {
379 crypt32dll = LoadLibrary("crypt32");
380 if (crypt32dll == NULL) {
381 CRYPTOAPIerr(CRYPTOAPI_F_LOAD_LIBRARY);
382 goto err;
385 if (CryptAcquireCertificatePrivateKey == NULL) {
386 CryptAcquireCertificatePrivateKey = GetProcAddress(crypt32dll,
387 "CryptAcquireCertificatePrivateKey");
388 if (CryptAcquireCertificatePrivateKey == NULL) {
389 CRYPTOAPIerr(CRYPTOAPI_F_GET_PROC_ADDRESS);
390 goto err;
393 #endif
394 if (!CryptAcquireCertificatePrivateKey(cd->cert_context, CRYPT_ACQUIRE_COMPARE_KEY_FLAG,
395 NULL, &cd->crypt_prov, &cd->key_spec, &cd->free_crypt_prov)) {
396 /* if we don't have a smart card reader here, and we try to access a
397 * smart card certificate, we get:
398 * "Error 1223: The operation was canceled by the user." */
399 CRYPTOAPIerr(CRYPTOAPI_F_CRYPT_ACQUIRE_CERTIFICATE_PRIVATE_KEY);
400 goto err;
402 /* here we don't need to do CryptGetUserKey() or anything; all necessary key
403 * info is in cd->cert_context, and then, in cd->crypt_prov. */
405 my_rsa_method->name = "Microsoft CryptoAPI RSA Method";
406 my_rsa_method->rsa_pub_enc = rsa_pub_enc;
407 my_rsa_method->rsa_pub_dec = rsa_pub_dec;
408 my_rsa_method->rsa_priv_enc = rsa_priv_enc;
409 my_rsa_method->rsa_priv_dec = rsa_priv_dec;
410 /* my_rsa_method->init = init; */
411 my_rsa_method->finish = finish;
412 my_rsa_method->flags = RSA_METHOD_FLAG_NO_CHECK;
413 my_rsa_method->app_data = (char *) cd;
415 rsa = RSA_new();
416 if (rsa == NULL) {
417 SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE_FILE, ERR_R_MALLOC_FAILURE);
418 goto err;
421 /* cert->cert_info->key->pkey is NULL until we call SSL_CTX_use_certificate(),
422 * so we do it here then... */
423 if (!SSL_CTX_use_certificate(ssl_ctx, cert))
424 goto err;
425 /* the public key */
426 pub_rsa = cert->cert_info->key->pkey->pkey.rsa;
427 /* SSL_CTX_use_certificate() increased the reference count in 'cert', so
428 * we decrease it here with X509_free(), or it will never be cleaned up. */
429 X509_free(cert);
430 cert = NULL;
432 /* I'm not sure about what we have to fill in in the RSA, trying out stuff... */
433 /* rsa->n indicates the key size */
434 rsa->n = BN_dup(pub_rsa->n);
435 rsa->flags |= RSA_FLAG_EXT_PKEY;
436 if (!RSA_set_method(rsa, my_rsa_method))
437 goto err;
439 if (!SSL_CTX_use_RSAPrivateKey(ssl_ctx, rsa))
440 goto err;
441 /* SSL_CTX_use_RSAPrivateKey() increased the reference count in 'rsa', so
442 * we decrease it here with RSA_free(), or it will never be cleaned up. */
443 RSA_free(rsa);
444 return 1;
446 err:
447 if (cert)
448 X509_free(cert);
449 if (rsa)
450 RSA_free(rsa);
451 else {
452 if (my_rsa_method)
453 free(my_rsa_method);
454 if (cd) {
455 if (cd->free_crypt_prov && cd->crypt_prov)
456 CryptReleaseContext(cd->crypt_prov, 0);
457 if (cd->cert_context)
458 CertFreeCertificateContext(cd->cert_context);
459 free(cd);
462 return 0;