s4-rpc_server: support AES decryption in netr_ServerPasswordSet2 server.
[Samba/vl.git] / source4 / heimdal / lib / krb5 / crypto-evp.c
blobe8fb1caf6ae8ad89663af1f1a5fcb51ec36b03d9
1 /*
2 * Copyright (c) 1997 - 2008 Kungliga Tekniska Högskolan
3 * (Royal Institute of Technology, Stockholm, Sweden).
4 * All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the name of the Institute nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
34 #include "krb5_locl.h"
36 void
37 _krb5_evp_schedule(krb5_context context,
38 struct _krb5_key_type *kt,
39 struct _krb5_key_data *kd)
41 struct _krb5_evp_schedule *key = kd->schedule->data;
42 const EVP_CIPHER *c = (*kt->evp)();
44 EVP_CIPHER_CTX_init(&key->ectx);
45 EVP_CIPHER_CTX_init(&key->dctx);
47 EVP_CipherInit_ex(&key->ectx, c, NULL, kd->key->keyvalue.data, NULL, 1);
48 EVP_CipherInit_ex(&key->dctx, c, NULL, kd->key->keyvalue.data, NULL, 0);
51 void
52 _krb5_evp_cleanup(krb5_context context, struct _krb5_key_data *kd)
54 struct _krb5_evp_schedule *key = kd->schedule->data;
55 EVP_CIPHER_CTX_cleanup(&key->ectx);
56 EVP_CIPHER_CTX_cleanup(&key->dctx);
59 krb5_error_code
60 _krb5_evp_encrypt(krb5_context context,
61 struct _krb5_key_data *key,
62 void *data,
63 size_t len,
64 krb5_boolean encryptp,
65 int usage,
66 void *ivec)
68 struct _krb5_evp_schedule *ctx = key->schedule->data;
69 EVP_CIPHER_CTX *c;
70 c = encryptp ? &ctx->ectx : &ctx->dctx;
71 if (ivec == NULL) {
72 /* alloca ? */
73 size_t len2 = EVP_CIPHER_CTX_iv_length(c);
74 void *loiv = malloc(len2);
75 if (loiv == NULL) {
76 krb5_clear_error_message(context);
77 return ENOMEM;
79 memset(loiv, 0, len2);
80 EVP_CipherInit_ex(c, NULL, NULL, NULL, loiv, -1);
81 free(loiv);
82 } else
83 EVP_CipherInit_ex(c, NULL, NULL, NULL, ivec, -1);
84 EVP_Cipher(c, data, data, len);
85 return 0;
88 static const unsigned char zero_ivec[EVP_MAX_BLOCK_LENGTH] = { 0 };
90 krb5_error_code
91 _krb5_evp_encrypt_cts(krb5_context context,
92 struct _krb5_key_data *key,
93 void *data,
94 size_t len,
95 krb5_boolean encryptp,
96 int usage,
97 void *ivec)
99 size_t i, blocksize;
100 struct _krb5_evp_schedule *ctx = key->schedule->data;
101 unsigned char tmp[EVP_MAX_BLOCK_LENGTH], ivec2[EVP_MAX_BLOCK_LENGTH];
102 EVP_CIPHER_CTX *c;
103 unsigned char *p;
105 c = encryptp ? &ctx->ectx : &ctx->dctx;
107 blocksize = EVP_CIPHER_CTX_block_size(c);
109 if (len < blocksize) {
110 krb5_set_error_message(context, EINVAL,
111 "message block too short");
112 return EINVAL;
113 } else if (len == blocksize) {
114 EVP_CipherInit_ex(c, NULL, NULL, NULL, zero_ivec, -1);
115 EVP_Cipher(c, data, data, len);
116 return 0;
119 if (ivec)
120 EVP_CipherInit_ex(c, NULL, NULL, NULL, ivec, -1);
121 else
122 EVP_CipherInit_ex(c, NULL, NULL, NULL, zero_ivec, -1);
124 if (encryptp) {
126 p = data;
127 i = ((len - 1) / blocksize) * blocksize;
128 EVP_Cipher(c, p, p, i);
129 p += i - blocksize;
130 len -= i;
131 memcpy(ivec2, p, blocksize);
133 for (i = 0; i < len; i++)
134 tmp[i] = p[i + blocksize] ^ ivec2[i];
135 for (; i < blocksize; i++)
136 tmp[i] = 0 ^ ivec2[i];
138 EVP_CipherInit_ex(c, NULL, NULL, NULL, zero_ivec, -1);
139 EVP_Cipher(c, p, tmp, blocksize);
141 memcpy(p + blocksize, ivec2, len);
142 if (ivec)
143 memcpy(ivec, p, blocksize);
144 } else {
145 unsigned char tmp2[EVP_MAX_BLOCK_LENGTH], tmp3[EVP_MAX_BLOCK_LENGTH];
147 p = data;
148 if (len > blocksize * 2) {
149 /* remove last two blocks and round up, decrypt this with cbc, then do cts dance */
150 i = ((((len - blocksize * 2) + blocksize - 1) / blocksize) * blocksize);
151 memcpy(ivec2, p + i - blocksize, blocksize);
152 EVP_Cipher(c, p, p, i);
153 p += i;
154 len -= i + blocksize;
155 } else {
156 if (ivec)
157 memcpy(ivec2, ivec, blocksize);
158 else
159 memcpy(ivec2, zero_ivec, blocksize);
160 len -= blocksize;
163 memcpy(tmp, p, blocksize);
164 EVP_CipherInit_ex(c, NULL, NULL, NULL, zero_ivec, -1);
165 EVP_Cipher(c, tmp2, p, blocksize);
167 memcpy(tmp3, p + blocksize, len);
168 memcpy(tmp3 + len, tmp2 + len, blocksize - len); /* xor 0 */
170 for (i = 0; i < len; i++)
171 p[i + blocksize] = tmp2[i] ^ tmp3[i];
173 EVP_CipherInit_ex(c, NULL, NULL, NULL, zero_ivec, -1);
174 EVP_Cipher(c, p, tmp3, blocksize);
176 for (i = 0; i < blocksize; i++)
177 p[i] ^= ivec2[i];
178 if (ivec)
179 memcpy(ivec, tmp, blocksize);
181 return 0;