fix warnings
[heimdal.git] / lib / gssapi / ntlm / crypto.c
blob33d6df73b09c86ccf32939e53ff7b12e0eaccd40
1 /*
2 * Copyright (c) 2006 Kungliga Tekniska Högskolan
3 * (Royal Institute of Technology, Stockholm, Sweden).
4 * All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the name of the Institute nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
34 #include "ntlm.h"
35 struct hx509_certs_data;
36 struct krb5_pk_identity;
37 struct krb5_pk_cert;
38 struct ContentInfo;
39 struct AlgorithmIdentifier;
40 struct _krb5_krb_auth_data;
41 struct krb5_dh_moduli;
42 struct _krb5_key_data;
43 struct _krb5_encryption_type;
44 struct _krb5_key_type;
45 #include "krb5-private.h"
51 static void
52 encode_le_uint32(uint32_t n, unsigned char *p)
54 p[0] = (n >> 0) & 0xFF;
55 p[1] = (n >> 8) & 0xFF;
56 p[2] = (n >> 16) & 0xFF;
57 p[3] = (n >> 24) & 0xFF;
61 static void
62 decode_le_uint32(const void *ptr, uint32_t *n)
64 const unsigned char *p = ptr;
65 *n = (p[0] << 0) | (p[1] << 8) | (p[2] << 16) | (p[3] << 24);
72 const char a2i_signmagic[] =
73 "session key to server-to-client signing key magic constant";
74 const char a2i_sealmagic[] =
75 "session key to server-to-client sealing key magic constant";
76 const char i2a_signmagic[] =
77 "session key to client-to-server signing key magic constant";
78 const char i2a_sealmagic[] =
79 "session key to client-to-server sealing key magic constant";
82 void
83 _gss_ntlm_set_key(struct ntlmv2_key *key, int acceptor, int sealsign,
84 unsigned char *data, size_t len)
86 unsigned char out[16];
87 EVP_MD_CTX *ctx;
88 const char *signmagic;
89 const char *sealmagic;
91 if (acceptor) {
92 signmagic = a2i_signmagic;
93 sealmagic = a2i_sealmagic;
94 } else {
95 signmagic = i2a_signmagic;
96 sealmagic = i2a_sealmagic;
99 key->seq = 0;
101 ctx = EVP_MD_CTX_create();
102 EVP_DigestInit_ex(ctx, EVP_md5(), NULL);
103 EVP_DigestUpdate(ctx, data, len);
104 EVP_DigestUpdate(ctx, signmagic, strlen(signmagic) + 1);
105 EVP_DigestFinal_ex(ctx, key->signkey, NULL);
107 EVP_DigestInit_ex(ctx, EVP_md5(), NULL);
108 EVP_DigestUpdate(ctx, data, len);
109 EVP_DigestUpdate(ctx, sealmagic, strlen(sealmagic) + 1);
110 EVP_DigestFinal_ex(ctx, out, NULL);
111 EVP_MD_CTX_destroy(ctx);
113 RC4_set_key(&key->sealkey, 16, out);
114 if (sealsign)
115 key->signsealkey = &key->sealkey;
122 static OM_uint32
123 v1_sign_message(gss_buffer_t in,
124 RC4_KEY *signkey,
125 uint32_t seq,
126 unsigned char out[16])
128 unsigned char sigature[12];
129 uint32_t crc;
131 _krb5_crc_init_table();
132 crc = _krb5_crc_update(in->value, in->length, 0);
134 encode_le_uint32(0, &sigature[0]);
135 encode_le_uint32(crc, &sigature[4]);
136 encode_le_uint32(seq, &sigature[8]);
138 encode_le_uint32(1, out); /* version */
139 RC4(signkey, sizeof(sigature), sigature, out + 4);
141 if (RAND_bytes(out + 4, 4) != 1)
142 return GSS_S_UNAVAILABLE;
144 return 0;
148 static OM_uint32
149 v2_sign_message(gss_buffer_t in,
150 unsigned char signkey[16],
151 RC4_KEY *sealkey,
152 uint32_t seq,
153 unsigned char out[16])
155 unsigned char hmac[16];
156 unsigned int hmaclen;
157 HMAC_CTX c;
159 HMAC_CTX_init(&c);
160 HMAC_Init_ex(&c, signkey, 16, EVP_md5(), NULL);
162 encode_le_uint32(seq, hmac);
163 HMAC_Update(&c, hmac, 4);
164 HMAC_Update(&c, in->value, in->length);
165 HMAC_Final(&c, hmac, &hmaclen);
166 HMAC_CTX_cleanup(&c);
168 encode_le_uint32(1, &out[0]);
169 if (sealkey)
170 RC4(sealkey, 8, hmac, &out[4]);
171 else
172 memcpy(&out[4], hmac, 8);
174 memset(&out[12], 0, 4);
176 return GSS_S_COMPLETE;
179 static OM_uint32
180 v2_verify_message(gss_buffer_t in,
181 unsigned char signkey[16],
182 RC4_KEY *sealkey,
183 uint32_t seq,
184 const unsigned char checksum[16])
186 OM_uint32 ret;
187 unsigned char out[16];
189 ret = v2_sign_message(in, signkey, sealkey, seq, out);
190 if (ret)
191 return ret;
193 if (memcmp(checksum, out, 16) != 0)
194 return GSS_S_BAD_MIC;
196 return GSS_S_COMPLETE;
199 static OM_uint32
200 v2_seal_message(const gss_buffer_t in,
201 unsigned char signkey[16],
202 uint32_t seq,
203 RC4_KEY *sealkey,
204 gss_buffer_t out)
206 unsigned char *p;
207 OM_uint32 ret;
209 if (in->length + 16 < in->length)
210 return EINVAL;
212 p = malloc(in->length + 16);
213 if (p == NULL)
214 return ENOMEM;
216 RC4(sealkey, in->length, in->value, p);
218 ret = v2_sign_message(in, signkey, sealkey, seq, &p[in->length]);
219 if (ret) {
220 free(p);
221 return ret;
224 out->value = p;
225 out->length = in->length + 16;
227 return 0;
230 static OM_uint32
231 v2_unseal_message(gss_buffer_t in,
232 unsigned char signkey[16],
233 uint32_t seq,
234 RC4_KEY *sealkey,
235 gss_buffer_t out)
237 OM_uint32 ret;
239 if (in->length < 16)
240 return GSS_S_BAD_MIC;
242 out->length = in->length - 16;
243 out->value = malloc(out->length);
244 if (out->value == NULL)
245 return GSS_S_BAD_MIC;
247 RC4(sealkey, out->length, in->value, out->value);
249 ret = v2_verify_message(out, signkey, sealkey, seq,
250 ((const unsigned char *)in->value) + out->length);
251 if (ret) {
252 OM_uint32 junk;
253 gss_release_buffer(&junk, out);
255 return ret;
262 #define CTX_FLAGS_ISSET(_ctx,_flags) \
263 (((_ctx)->flags & (_flags)) == (_flags))
269 OM_uint32 GSSAPI_CALLCONV
270 _gss_ntlm_get_mic
271 (OM_uint32 * minor_status,
272 gss_const_ctx_id_t context_handle,
273 gss_qop_t qop_req,
274 const gss_buffer_t message_buffer,
275 gss_buffer_t message_token
278 ntlm_ctx ctx = (ntlm_ctx)context_handle;
279 OM_uint32 junk;
281 *minor_status = 0;
283 message_token->value = malloc(16);
284 message_token->length = 16;
285 if (message_token->value == NULL) {
286 *minor_status = ENOMEM;
287 return GSS_S_FAILURE;
290 if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SIGN|NTLM_NEG_NTLM2_SESSION)) {
291 OM_uint32 ret;
293 if ((ctx->status & STATUS_SESSIONKEY) == 0) {
294 gss_release_buffer(&junk, message_token);
295 return GSS_S_UNAVAILABLE;
298 ret = v2_sign_message(message_buffer,
299 ctx->u.v2.send.signkey,
300 ctx->u.v2.send.signsealkey,
301 ctx->u.v2.send.seq++,
302 message_token->value);
303 if (ret)
304 gss_release_buffer(&junk, message_token);
305 return ret;
307 } else if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SIGN)) {
308 OM_uint32 ret;
310 if ((ctx->status & STATUS_SESSIONKEY) == 0) {
311 gss_release_buffer(&junk, message_token);
312 return GSS_S_UNAVAILABLE;
315 ret = v1_sign_message(message_buffer,
316 &ctx->u.v1.crypto_send.key,
317 ctx->u.v1.crypto_send.seq++,
318 message_token->value);
319 if (ret)
320 gss_release_buffer(&junk, message_token);
321 return ret;
323 } else if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_ALWAYS_SIGN)) {
324 unsigned char *sigature;
326 sigature = message_token->value;
328 encode_le_uint32(1, &sigature[0]); /* version */
329 encode_le_uint32(0, &sigature[4]);
330 encode_le_uint32(0, &sigature[8]);
331 encode_le_uint32(0, &sigature[12]);
333 return GSS_S_COMPLETE;
335 gss_release_buffer(&junk, message_token);
337 return GSS_S_UNAVAILABLE;
344 OM_uint32 GSSAPI_CALLCONV
345 _gss_ntlm_verify_mic
346 (OM_uint32 * minor_status,
347 gss_const_ctx_id_t context_handle,
348 const gss_buffer_t message_buffer,
349 const gss_buffer_t token_buffer,
350 gss_qop_t * qop_state
353 ntlm_ctx ctx = (ntlm_ctx)context_handle;
355 if (qop_state != NULL)
356 *qop_state = GSS_C_QOP_DEFAULT;
357 *minor_status = 0;
359 if (token_buffer->length != 16)
360 return GSS_S_BAD_MIC;
362 if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SIGN|NTLM_NEG_NTLM2_SESSION)) {
363 OM_uint32 ret;
365 if ((ctx->status & STATUS_SESSIONKEY) == 0)
366 return GSS_S_UNAVAILABLE;
368 ret = v2_verify_message(message_buffer,
369 ctx->u.v2.recv.signkey,
370 ctx->u.v2.recv.signsealkey,
371 ctx->u.v2.recv.seq++,
372 token_buffer->value);
373 if (ret)
374 return ret;
376 return GSS_S_COMPLETE;
377 } else if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SIGN)) {
379 unsigned char sigature[12];
380 uint32_t crc, num;
382 if ((ctx->status & STATUS_SESSIONKEY) == 0)
383 return GSS_S_UNAVAILABLE;
385 decode_le_uint32(token_buffer->value, &num);
386 if (num != 1)
387 return GSS_S_BAD_MIC;
389 RC4(&ctx->u.v1.crypto_recv.key, sizeof(sigature),
390 ((unsigned char *)token_buffer->value) + 4, sigature);
392 _krb5_crc_init_table();
393 crc = _krb5_crc_update(message_buffer->value,
394 message_buffer->length, 0);
395 /* skip first 4 bytes in the encrypted checksum */
396 decode_le_uint32(&sigature[4], &num);
397 if (num != crc)
398 return GSS_S_BAD_MIC;
399 decode_le_uint32(&sigature[8], &num);
400 if (ctx->u.v1.crypto_recv.seq != num)
401 return GSS_S_BAD_MIC;
402 ctx->u.v1.crypto_recv.seq++;
404 return GSS_S_COMPLETE;
405 } else if (ctx->flags & NTLM_NEG_ALWAYS_SIGN) {
406 uint32_t num;
407 unsigned char *p;
409 p = (unsigned char*)(token_buffer->value);
411 decode_le_uint32(&p[0], &num); /* version */
412 if (num != 1) return GSS_S_BAD_MIC;
413 decode_le_uint32(&p[4], &num);
414 if (num != 0) return GSS_S_BAD_MIC;
415 decode_le_uint32(&p[8], &num);
416 if (num != 0) return GSS_S_BAD_MIC;
417 decode_le_uint32(&p[12], &num);
418 if (num != 0) return GSS_S_BAD_MIC;
420 return GSS_S_COMPLETE;
423 return GSS_S_UNAVAILABLE;
430 OM_uint32 GSSAPI_CALLCONV
431 _gss_ntlm_wrap_size_limit (
432 OM_uint32 * minor_status,
433 gss_const_ctx_id_t context_handle,
434 int conf_req_flag,
435 gss_qop_t qop_req,
436 OM_uint32 req_output_size,
437 OM_uint32 * max_input_size
440 ntlm_ctx ctx = (ntlm_ctx)context_handle;
442 *minor_status = 0;
444 if(ctx->flags & NTLM_NEG_SEAL) {
446 if (req_output_size < 16)
447 *max_input_size = 0;
448 else
449 *max_input_size = req_output_size - 16;
451 return GSS_S_COMPLETE;
454 return GSS_S_UNAVAILABLE;
461 OM_uint32 GSSAPI_CALLCONV
462 _gss_ntlm_wrap
463 (OM_uint32 * minor_status,
464 gss_const_ctx_id_t context_handle,
465 int conf_req_flag,
466 gss_qop_t qop_req,
467 const gss_buffer_t input_message_buffer,
468 int * conf_state,
469 gss_buffer_t output_message_buffer
472 ntlm_ctx ctx = (ntlm_ctx)context_handle;
473 OM_uint32 ret;
475 *minor_status = 0;
476 if (conf_state)
477 *conf_state = 0;
478 if (output_message_buffer == GSS_C_NO_BUFFER)
479 return GSS_S_FAILURE;
482 if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SEAL|NTLM_NEG_NTLM2_SESSION)) {
484 return v2_seal_message(input_message_buffer,
485 ctx->u.v2.send.signkey,
486 ctx->u.v2.send.seq++,
487 &ctx->u.v2.send.sealkey,
488 output_message_buffer);
490 } else if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SEAL)) {
491 gss_buffer_desc trailer;
492 OM_uint32 junk;
494 output_message_buffer->length = input_message_buffer->length + 16;
495 output_message_buffer->value = malloc(output_message_buffer->length);
496 if (output_message_buffer->value == NULL) {
497 output_message_buffer->length = 0;
498 return GSS_S_FAILURE;
502 RC4(&ctx->u.v1.crypto_send.key, input_message_buffer->length,
503 input_message_buffer->value, output_message_buffer->value);
505 ret = _gss_ntlm_get_mic(minor_status, context_handle,
506 0, input_message_buffer,
507 &trailer);
508 if (ret) {
509 gss_release_buffer(&junk, output_message_buffer);
510 return ret;
512 if (trailer.length != 16) {
513 gss_release_buffer(&junk, output_message_buffer);
514 gss_release_buffer(&junk, &trailer);
515 return GSS_S_FAILURE;
517 memcpy(((unsigned char *)output_message_buffer->value) +
518 input_message_buffer->length,
519 trailer.value, trailer.length);
520 gss_release_buffer(&junk, &trailer);
522 return GSS_S_COMPLETE;
525 return GSS_S_UNAVAILABLE;
532 OM_uint32 GSSAPI_CALLCONV
533 _gss_ntlm_unwrap
534 (OM_uint32 * minor_status,
535 gss_const_ctx_id_t context_handle,
536 const gss_buffer_t input_message_buffer,
537 gss_buffer_t output_message_buffer,
538 int * conf_state,
539 gss_qop_t * qop_state
542 ntlm_ctx ctx = (ntlm_ctx)context_handle;
543 OM_uint32 ret;
545 *minor_status = 0;
546 output_message_buffer->value = NULL;
547 output_message_buffer->length = 0;
549 if (conf_state)
550 *conf_state = 0;
551 if (qop_state)
552 *qop_state = 0;
554 if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SEAL|NTLM_NEG_NTLM2_SESSION)) {
556 return v2_unseal_message(input_message_buffer,
557 ctx->u.v2.recv.signkey,
558 ctx->u.v2.recv.seq++,
559 &ctx->u.v2.recv.sealkey,
560 output_message_buffer);
562 } else if (CTX_FLAGS_ISSET(ctx, NTLM_NEG_SEAL)) {
564 gss_buffer_desc trailer;
565 OM_uint32 junk;
567 if (input_message_buffer->length < 16)
568 return GSS_S_BAD_MIC;
570 output_message_buffer->length = input_message_buffer->length - 16;
571 output_message_buffer->value = malloc(output_message_buffer->length);
572 if (output_message_buffer->value == NULL) {
573 output_message_buffer->length = 0;
574 return GSS_S_FAILURE;
577 RC4(&ctx->u.v1.crypto_recv.key, output_message_buffer->length,
578 input_message_buffer->value, output_message_buffer->value);
580 trailer.value = ((unsigned char *)input_message_buffer->value) +
581 output_message_buffer->length;
582 trailer.length = 16;
584 ret = _gss_ntlm_verify_mic(minor_status, context_handle,
585 output_message_buffer,
586 &trailer, NULL);
587 if (ret) {
588 gss_release_buffer(&junk, output_message_buffer);
589 return ret;
592 return GSS_S_COMPLETE;
595 return GSS_S_UNAVAILABLE;