Fix documentation for gnutls_dtls_set_mtu()
[gnutls.git] / lib / gnutls_sig.c
blobc576655e1041de14040f0a7e3a4d65237566bc02
1 /*
2 * Copyright (C) 2001-2012 Free Software Foundation, Inc.
4 * Author: Nikos Mavrogiannopoulos
6 * This file is part of GnuTLS.
8 * The GnuTLS is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public License
10 * as published by the Free Software Foundation; either version 3 of
11 * the License, or (at your option) any later version.
13 * This library is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with this program. If not, see <http://www.gnu.org/licenses/>
23 #include <gnutls_int.h>
24 #include <gnutls_errors.h>
25 #include <x509_b64.h>
26 #include <auth/cert.h>
27 #include <algorithms.h>
28 #include <gnutls_datum.h>
29 #include <gnutls_mpi.h>
30 #include <gnutls_global.h>
31 #include <gnutls_pk.h>
32 #include <debug.h>
33 #include <gnutls_buffers.h>
34 #include <gnutls_sig.h>
35 #include <gnutls_kx.h>
36 #include <libtasn1.h>
37 #include <ext/signature.h>
38 #include <gnutls_state.h>
39 #include <x509/common.h>
40 #include <abstract_int.h>
42 static int
43 sign_tls_hash (gnutls_session_t session, gnutls_digest_algorithm_t hash_algo,
44 gnutls_pcert_st* cert, gnutls_privkey_t pkey,
45 const gnutls_datum_t * hash_concat,
46 gnutls_datum_t * signature);
49 /* While this is currently equal to the length of RSA/SHA512
50 * signature, it should also be sufficient for DSS signature and any
51 * other RSA signatures including one with the old MD5/SHA1-combined
52 * format.
54 #define MAX_SIG_SIZE 19 + MAX_HASH_SIZE
56 /* Generates a signature of all the random data and the parameters.
57 * Used in DHE_* ciphersuites.
59 int
60 _gnutls_handshake_sign_data (gnutls_session_t session, gnutls_pcert_st* cert,
61 gnutls_privkey_t pkey, gnutls_datum_t * params,
62 gnutls_datum_t * signature,
63 gnutls_sign_algorithm_t * sign_algo)
65 gnutls_datum_t dconcat;
66 int ret;
67 digest_hd_st td_sha;
68 uint8_t concat[MAX_SIG_SIZE];
69 gnutls_protocol_t ver = gnutls_protocol_get_version (session);
70 gnutls_digest_algorithm_t hash_algo;
72 *sign_algo =
73 _gnutls_session_get_sign_algo (session, cert);
74 if (*sign_algo == GNUTLS_SIGN_UNKNOWN)
76 gnutls_assert ();
77 return GNUTLS_E_UNKNOWN_PK_ALGORITHM;
80 hash_algo = _gnutls_sign_get_hash_algorithm (*sign_algo);
82 _gnutls_handshake_log ("HSK[%p]: signing handshake data: using %s\n",
83 session, gnutls_sign_algorithm_get_name (*sign_algo));
85 ret = _gnutls_hash_init (&td_sha, hash_algo);
86 if (ret < 0)
88 gnutls_assert ();
89 return ret;
92 _gnutls_hash (&td_sha, session->security_parameters.client_random,
93 GNUTLS_RANDOM_SIZE);
94 _gnutls_hash (&td_sha, session->security_parameters.server_random,
95 GNUTLS_RANDOM_SIZE);
96 _gnutls_hash (&td_sha, params->data, params->size);
98 switch (gnutls_pubkey_get_pk_algorithm(cert->pubkey, NULL))
100 case GNUTLS_PK_RSA:
101 if (!_gnutls_version_has_selectable_sighash (ver))
103 digest_hd_st td_md5;
105 ret = _gnutls_hash_init (&td_md5, GNUTLS_MAC_MD5);
106 if (ret < 0)
108 gnutls_assert ();
109 return ret;
112 _gnutls_hash (&td_md5, session->security_parameters.client_random,
113 GNUTLS_RANDOM_SIZE);
114 _gnutls_hash (&td_md5, session->security_parameters.server_random,
115 GNUTLS_RANDOM_SIZE);
116 _gnutls_hash (&td_md5, params->data, params->size);
118 _gnutls_hash_deinit (&td_md5, concat);
119 _gnutls_hash_deinit (&td_sha, &concat[16]);
121 dconcat.data = concat;
122 dconcat.size = 36;
124 else
125 { /* TLS 1.2 way */
127 _gnutls_hash_deinit (&td_sha, concat);
129 dconcat.data = concat;
130 dconcat.size = _gnutls_hash_get_algo_len (hash_algo);
132 break;
133 case GNUTLS_PK_DSA:
134 case GNUTLS_PK_EC:
135 _gnutls_hash_deinit (&td_sha, concat);
137 if (!IS_SHA(hash_algo))
139 gnutls_assert ();
140 return GNUTLS_E_INTERNAL_ERROR;
142 dconcat.data = concat;
143 dconcat.size = _gnutls_hash_get_algo_len (hash_algo);
144 break;
146 default:
147 gnutls_assert ();
148 _gnutls_hash_deinit (&td_sha, NULL);
149 return GNUTLS_E_INTERNAL_ERROR;
152 ret = sign_tls_hash (session, hash_algo, cert, pkey, &dconcat, signature);
153 if (ret < 0)
155 gnutls_assert ();
158 return ret;
162 /* This will create a PKCS1 or DSA signature, as defined in the TLS protocol.
163 * Cert is the certificate of the corresponding private key. It is only checked if
164 * it supports signing.
166 static int
167 sign_tls_hash (gnutls_session_t session, gnutls_digest_algorithm_t hash_algo,
168 gnutls_pcert_st* cert, gnutls_privkey_t pkey,
169 const gnutls_datum_t * hash_concat,
170 gnutls_datum_t * signature)
172 gnutls_protocol_t ver = gnutls_protocol_get_version (session);
173 unsigned int key_usage = 0;
174 /* If our certificate supports signing
177 if (cert != NULL)
179 gnutls_pubkey_get_key_usage(cert->pubkey, &key_usage);
181 if (key_usage != 0)
182 if (!(key_usage & GNUTLS_KEY_DIGITAL_SIGNATURE))
184 gnutls_assert ();
185 return GNUTLS_E_KEY_USAGE_VIOLATION;
188 /* External signing. Deprecated. To be removed. */
189 if (!pkey)
191 int ret;
193 if (!session->internals.sign_func)
194 return gnutls_assert_val(GNUTLS_E_INSUFFICIENT_CREDENTIALS);
196 if (!_gnutls_version_has_selectable_sighash (ver))
197 return (*session->internals.sign_func)
198 (session, session->internals.sign_func_userdata,
199 cert->type, &cert->cert, hash_concat, signature);
200 else
202 gnutls_datum_t digest;
204 ret = _gnutls_set_datum(&digest, hash_concat->data, hash_concat->size);
205 if (ret < 0)
206 return gnutls_assert_val(ret);
208 ret = pk_prepare_hash (gnutls_privkey_get_pk_algorithm(pkey, NULL), hash_algo, &digest);
209 if (ret < 0)
211 gnutls_assert ();
212 goto es_cleanup;
215 ret = (*session->internals.sign_func)
216 (session, session->internals.sign_func_userdata,
217 cert->type, &cert->cert, &digest, signature);
218 es_cleanup:
219 gnutls_free(digest.data);
221 return ret;
226 if (!_gnutls_version_has_selectable_sighash (ver))
227 return _gnutls_privkey_sign_hash (pkey, hash_concat, signature);
228 else
229 return gnutls_privkey_sign_hash (pkey, hash_algo, 0, hash_concat, signature);
232 static int
233 verify_tls_hash (gnutls_protocol_t ver, gnutls_pcert_st* cert,
234 const gnutls_datum_t * hash_concat,
235 gnutls_datum_t * signature, size_t sha1pos,
236 gnutls_sign_algorithm_t sign_algo,
237 gnutls_pk_algorithm_t pk_algo)
239 int ret;
240 gnutls_datum_t vdata;
241 unsigned int key_usage = 0, flags;
243 if (cert == NULL)
245 gnutls_assert ();
246 return GNUTLS_E_CERTIFICATE_ERROR;
249 gnutls_pubkey_get_key_usage(cert->pubkey, &key_usage);
251 /* If the certificate supports signing continue.
253 if (key_usage != 0)
254 if (!(key_usage & GNUTLS_KEY_DIGITAL_SIGNATURE))
256 gnutls_assert ();
257 return GNUTLS_E_KEY_USAGE_VIOLATION;
260 if (pk_algo == GNUTLS_PK_UNKNOWN)
261 pk_algo = gnutls_pubkey_get_pk_algorithm(cert->pubkey, NULL);
262 switch (pk_algo)
264 case GNUTLS_PK_RSA:
266 vdata.data = hash_concat->data;
267 vdata.size = hash_concat->size;
269 /* verify signature */
270 if (!_gnutls_version_has_selectable_sighash (ver))
271 flags = GNUTLS_PUBKEY_VERIFY_FLAG_TLS_RSA;
272 else
273 flags = 0;
274 break;
275 case GNUTLS_PK_DSA:
276 case GNUTLS_PK_EC:
277 vdata.data = &hash_concat->data[sha1pos];
278 vdata.size = hash_concat->size - sha1pos;
280 flags = 0;
282 break;
283 default:
284 gnutls_assert ();
285 return GNUTLS_E_INTERNAL_ERROR;
288 ret = gnutls_pubkey_verify_hash2(cert->pubkey, sign_algo, flags,
289 &vdata, signature);
291 if (ret < 0)
292 return gnutls_assert_val(ret);
295 return 0;
299 /* Generates a signature of all the random data and the parameters.
300 * Used in DHE_* ciphersuites.
303 _gnutls_handshake_verify_data (gnutls_session_t session, gnutls_pcert_st* cert,
304 const gnutls_datum_t * params,
305 gnutls_datum_t * signature,
306 gnutls_sign_algorithm_t sign_algo)
308 gnutls_datum_t dconcat;
309 int ret;
310 digest_hd_st td_md5;
311 digest_hd_st td_sha;
312 uint8_t concat[MAX_SIG_SIZE];
313 gnutls_protocol_t ver = gnutls_protocol_get_version (session);
314 gnutls_digest_algorithm_t hash_algo;
316 if (_gnutls_version_has_selectable_sighash (ver))
318 _gnutls_handshake_log ("HSK[%p]: verify handshake data: using %s\n",
319 session, gnutls_sign_algorithm_get_name (sign_algo));
321 ret = _gnutls_pubkey_compatible_with_sig(cert->pubkey, ver, sign_algo);
322 if (ret < 0)
323 return gnutls_assert_val(ret);
325 ret = _gnutls_session_sign_algo_enabled (session, sign_algo);
326 if (ret < 0)
327 return gnutls_assert_val(ret);
329 hash_algo = _gnutls_sign_get_hash_algorithm (sign_algo);
331 else
333 ret = _gnutls_hash_init (&td_md5, GNUTLS_MAC_MD5);
334 if (ret < 0)
336 gnutls_assert ();
337 return ret;
340 _gnutls_hash (&td_md5, session->security_parameters.client_random,
341 GNUTLS_RANDOM_SIZE);
342 _gnutls_hash (&td_md5, session->security_parameters.server_random,
343 GNUTLS_RANDOM_SIZE);
344 _gnutls_hash (&td_md5, params->data, params->size);
346 hash_algo = GNUTLS_DIG_SHA1;
349 ret = _gnutls_hash_init (&td_sha, hash_algo);
350 if (ret < 0)
352 gnutls_assert ();
353 if (!_gnutls_version_has_selectable_sighash (ver))
354 _gnutls_hash_deinit (&td_md5, NULL);
355 return ret;
358 _gnutls_hash (&td_sha, session->security_parameters.client_random,
359 GNUTLS_RANDOM_SIZE);
360 _gnutls_hash (&td_sha, session->security_parameters.server_random,
361 GNUTLS_RANDOM_SIZE);
362 _gnutls_hash (&td_sha, params->data, params->size);
364 if (!_gnutls_version_has_selectable_sighash (ver))
366 _gnutls_hash_deinit (&td_md5, concat);
367 _gnutls_hash_deinit (&td_sha, &concat[16]);
368 dconcat.data = concat;
369 dconcat.size = 36;
371 else
373 _gnutls_hash_deinit (&td_sha, concat);
375 dconcat.data = concat;
376 dconcat.size = _gnutls_hash_get_algo_len (hash_algo);
379 ret = verify_tls_hash (ver, cert, &dconcat, signature,
380 dconcat.size -
381 _gnutls_hash_get_algo_len (hash_algo),
382 sign_algo,
383 _gnutls_sign_get_pk_algorithm (sign_algo));
384 if (ret < 0)
386 gnutls_assert ();
387 return ret;
390 return ret;
394 /* Client certificate verify calculations
397 /* this is _gnutls_handshake_verify_crt_vrfy for TLS 1.2
399 static int
400 _gnutls_handshake_verify_crt_vrfy12 (gnutls_session_t session,
401 gnutls_pcert_st* cert,
402 gnutls_datum_t * signature,
403 gnutls_sign_algorithm_t sign_algo)
405 int ret;
406 uint8_t concat[MAX_HASH_SIZE];
407 gnutls_datum_t dconcat;
408 gnutls_digest_algorithm_t hash_algo;
409 gnutls_protocol_t ver = gnutls_protocol_get_version (session);
410 gnutls_pk_algorithm_t pk = gnutls_pubkey_get_pk_algorithm(cert->pubkey, NULL);
412 ret = _gnutls_session_sign_algo_enabled(session, sign_algo);
413 if (ret < 0)
414 return gnutls_assert_val(ret);
416 hash_algo = _gnutls_sign_get_hash_algorithm(sign_algo);
418 ret = _gnutls_hash_fast(hash_algo, session->internals.handshake_hash_buffer.data,
419 session->internals.handshake_hash_buffer_prev_len,
420 concat);
421 if (ret < 0)
422 return gnutls_assert_val(ret);
424 dconcat.data = concat;
425 dconcat.size = _gnutls_hash_get_algo_len (hash_algo);
427 ret =
428 verify_tls_hash (ver, cert, &dconcat, signature, 0, sign_algo, pk);
429 if (ret < 0)
431 gnutls_assert ();
432 return ret;
435 return ret;
439 /* Verifies a TLS signature (like the one in the client certificate
440 * verify message).
443 _gnutls_handshake_verify_crt_vrfy (gnutls_session_t session,
444 gnutls_pcert_st *cert,
445 gnutls_datum_t * signature,
446 gnutls_sign_algorithm_t sign_algo)
448 int ret;
449 uint8_t concat[MAX_SIG_SIZE];
450 digest_hd_st td_md5;
451 digest_hd_st td_sha;
452 gnutls_datum_t dconcat;
453 gnutls_protocol_t ver = gnutls_protocol_get_version (session);
455 _gnutls_handshake_log ("HSK[%p]: verify cert vrfy: using %s\n",
456 session, gnutls_sign_algorithm_get_name (sign_algo));
459 if (_gnutls_version_has_selectable_sighash(ver))
460 return _gnutls_handshake_verify_crt_vrfy12 (session, cert, signature,
461 sign_algo);
463 ret =
464 _gnutls_hash_init (&td_md5, GNUTLS_DIG_MD5);
465 if (ret < 0)
467 gnutls_assert ();
468 return ret;
471 ret =
472 _gnutls_hash_init (&td_sha, GNUTLS_DIG_SHA1);
473 if (ret < 0)
475 gnutls_assert ();
476 _gnutls_hash_deinit (&td_md5, NULL);
477 return GNUTLS_E_HASH_FAILED;
480 _gnutls_hash(&td_sha, session->internals.handshake_hash_buffer.data, session->internals.handshake_hash_buffer_prev_len);
481 _gnutls_hash(&td_md5, session->internals.handshake_hash_buffer.data, session->internals.handshake_hash_buffer_prev_len);
483 if (ver == GNUTLS_SSL3)
485 ret = _gnutls_generate_master (session, 1);
486 if (ret < 0)
488 _gnutls_hash_deinit (&td_md5, NULL);
489 _gnutls_hash_deinit (&td_sha, NULL);
490 return gnutls_assert_val(ret);
493 ret = _gnutls_mac_deinit_ssl3_handshake (&td_md5, concat,
494 session->
495 security_parameters.master_secret,
496 GNUTLS_MASTER_SIZE);
497 if (ret < 0)
499 _gnutls_hash_deinit (&td_sha, NULL);
500 return gnutls_assert_val(ret);
503 ret = _gnutls_mac_deinit_ssl3_handshake (&td_sha, &concat[16],
504 session->
505 security_parameters.master_secret,
506 GNUTLS_MASTER_SIZE);
507 if (ret < 0)
509 return gnutls_assert_val(ret);
512 else
514 _gnutls_hash_deinit (&td_md5, concat);
515 _gnutls_hash_deinit (&td_sha, &concat[16]);
518 dconcat.data = concat;
519 dconcat.size = 20 + 16; /* md5+ sha */
521 ret =
522 verify_tls_hash (ver, cert, &dconcat, signature, 16,
523 GNUTLS_SIGN_UNKNOWN,
524 gnutls_pubkey_get_pk_algorithm(cert->pubkey, NULL));
525 if (ret < 0)
527 gnutls_assert ();
528 return ret;
531 return ret;
535 /* the same as _gnutls_handshake_sign_crt_vrfy except that it is made for TLS 1.2
537 static int
538 _gnutls_handshake_sign_crt_vrfy12 (gnutls_session_t session,
539 gnutls_pcert_st* cert, gnutls_privkey_t pkey,
540 gnutls_datum_t * signature)
542 gnutls_datum_t dconcat;
543 int ret;
544 uint8_t concat[MAX_SIG_SIZE];
545 gnutls_sign_algorithm_t sign_algo;
546 gnutls_digest_algorithm_t hash_algo;
548 sign_algo =
549 _gnutls_session_get_sign_algo (session, cert);
550 if (sign_algo == GNUTLS_SIGN_UNKNOWN)
552 gnutls_assert ();
553 return GNUTLS_E_UNKNOWN_PK_ALGORITHM;
556 hash_algo = _gnutls_sign_get_hash_algorithm (sign_algo);
558 _gnutls_debug_log ("sign handshake cert vrfy: picked %s with %s\n",
559 gnutls_sign_algorithm_get_name (sign_algo),
560 gnutls_mac_get_name ((gnutls_mac_algorithm_t)hash_algo));
562 ret = _gnutls_hash_fast (hash_algo, session->internals.handshake_hash_buffer.data,
563 session->internals.handshake_hash_buffer.length,
564 concat);
565 if (ret < 0)
566 return gnutls_assert_val(ret);
568 dconcat.data = concat;
569 dconcat.size = _gnutls_hash_get_algo_len (hash_algo);
571 ret = sign_tls_hash (session, hash_algo, cert, pkey, &dconcat, signature);
572 if (ret < 0)
574 gnutls_assert ();
575 return ret;
578 return sign_algo;
582 /* Generates a signature of all the previous sent packets in the
583 * handshake procedure.
584 * 20040227: now it works for SSL 3.0 as well
585 * 20091031: works for TLS 1.2 too!
587 * For TLS1.x, x<2 returns negative for failure and zero or unspecified for success.
588 * For TLS1.2 returns the signature algorithm used on success, or a negative error code;
591 _gnutls_handshake_sign_crt_vrfy (gnutls_session_t session,
592 gnutls_pcert_st* cert, gnutls_privkey_t pkey,
593 gnutls_datum_t * signature)
595 gnutls_datum_t dconcat;
596 int ret;
597 uint8_t concat[MAX_SIG_SIZE];
598 digest_hd_st td_md5;
599 digest_hd_st td_sha;
600 gnutls_protocol_t ver = gnutls_protocol_get_version (session);
601 gnutls_pk_algorithm_t pk = gnutls_pubkey_get_pk_algorithm(cert->pubkey, NULL);
603 if (_gnutls_version_has_selectable_sighash(ver))
604 return _gnutls_handshake_sign_crt_vrfy12 (session, cert, pkey,
605 signature);
607 ret =
608 _gnutls_hash_init (&td_sha, GNUTLS_DIG_SHA1);
609 if (ret < 0)
611 gnutls_assert ();
612 return ret;
615 _gnutls_hash(&td_sha, session->internals.handshake_hash_buffer.data, session->internals.handshake_hash_buffer.length);
617 if (ver == GNUTLS_SSL3)
619 ret = _gnutls_generate_master (session, 1);
620 if (ret < 0)
622 gnutls_assert ();
623 _gnutls_hash_deinit (&td_sha, NULL);
624 return ret;
627 ret = _gnutls_mac_deinit_ssl3_handshake (&td_sha, &concat[16],
628 session->
629 security_parameters.master_secret,
630 GNUTLS_MASTER_SIZE);
631 if (ret < 0)
632 return gnutls_assert_val(ret);
634 else
635 _gnutls_hash_deinit (&td_sha, &concat[16]);
637 /* ensure 1024 bit DSA keys are used */
638 ret = _gnutls_pubkey_compatible_with_sig(cert->pubkey, ver, GNUTLS_SIGN_UNKNOWN);
639 if (ret < 0)
640 return gnutls_assert_val(ret);
642 switch (pk)
644 case GNUTLS_PK_RSA:
645 ret =
646 _gnutls_hash_init (&td_md5, GNUTLS_DIG_MD5);
647 if (ret < 0)
648 return gnutls_assert_val(ret);
650 _gnutls_hash(&td_md5, session->internals.handshake_hash_buffer.data, session->internals.handshake_hash_buffer.length);
652 if (ver == GNUTLS_SSL3)
654 ret = _gnutls_mac_deinit_ssl3_handshake (&td_md5, concat,
655 session->
656 security_parameters.master_secret,
657 GNUTLS_MASTER_SIZE);
658 if (ret < 0)
659 return gnutls_assert_val(ret);
661 else
662 _gnutls_hash_deinit (&td_md5, concat);
664 dconcat.data = concat;
665 dconcat.size = 36;
666 break;
667 case GNUTLS_PK_DSA:
668 case GNUTLS_PK_EC:
670 dconcat.data = &concat[16];
671 dconcat.size = 20;
672 break;
674 default:
675 return gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR);
677 ret = sign_tls_hash (session, GNUTLS_DIG_NULL, cert, pkey, &dconcat, signature);
678 if (ret < 0)
680 gnutls_assert ();
683 return ret;
687 pk_hash_data (gnutls_pk_algorithm_t pk, gnutls_digest_algorithm_t hash,
688 gnutls_pk_params_st* params,
689 const gnutls_datum_t * data, gnutls_datum_t * digest)
691 int ret;
693 digest->size = _gnutls_hash_get_algo_len (hash);
694 digest->data = gnutls_malloc (digest->size);
695 if (digest->data == NULL)
697 gnutls_assert ();
698 return GNUTLS_E_MEMORY_ERROR;
701 ret = _gnutls_hash_fast (hash, data->data, data->size, digest->data);
702 if (ret < 0)
704 gnutls_assert ();
705 goto cleanup;
708 return 0;
710 cleanup:
711 gnutls_free (digest->data);
712 return ret;
717 * This function will do RSA PKCS #1 1.5 encoding
718 * on the given digest. The given digest must be allocated
719 * and will be freed if replacement is required.
722 pk_prepare_hash (gnutls_pk_algorithm_t pk,
723 gnutls_digest_algorithm_t hash, gnutls_datum_t * digest)
725 int ret;
726 gnutls_datum_t old_digest = { digest->data, digest->size };
728 switch (pk)
730 case GNUTLS_PK_RSA:
731 /* Encode the digest as a DigestInfo
733 if ((ret = encode_ber_digest_info (hash, &old_digest, digest)) != 0)
735 gnutls_assert ();
736 return ret;
739 _gnutls_free_datum (&old_digest);
740 break;
741 case GNUTLS_PK_DSA:
742 case GNUTLS_PK_EC:
743 break;
744 default:
745 gnutls_assert ();
746 return GNUTLS_E_UNIMPLEMENTED_FEATURE;
749 return 0;