get rid of some sompiler warnings on IRIX
[Samba.git] / source3 / libsmb / ntlmssp_sign.c
blobff2f97c2e888048a6545e8195d93047c747d383d
1 /*
2 * Unix SMB/CIFS implementation.
3 * Version 3.0
4 * NTLMSSP Signing routines
5 * Copyright (C) Luke Kenneth Casson Leighton 1996-2001
6 * Copyright (C) Andrew Bartlett 2003
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software Foundation,
20 * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
23 #include "includes.h"
25 #define CLI_SIGN "session key to client-to-server signing key magic constant"
26 #define CLI_SEAL "session key to client-to-server sealing key magic constant"
27 #define SRV_SIGN "session key to server-to-client signing key magic constant"
28 #define SRV_SEAL "session key to server-to-client sealing key magic constant"
30 static void NTLMSSPcalc_ap( unsigned char *hash, unsigned char *data, int len)
32 unsigned char index_i = hash[256];
33 unsigned char index_j = hash[257];
34 int ind;
36 for (ind = 0; ind < len; ind++)
38 unsigned char tc;
39 unsigned char t;
41 index_i++;
42 index_j += hash[index_i];
44 tc = hash[index_i];
45 hash[index_i] = hash[index_j];
46 hash[index_j] = tc;
48 t = hash[index_i] + hash[index_j];
49 data[ind] = data[ind] ^ hash[t];
52 hash[256] = index_i;
53 hash[257] = index_j;
56 static void calc_hash(unsigned char *hash, const char *k2, int k2l)
58 unsigned char j = 0;
59 int ind;
61 for (ind = 0; ind < 256; ind++)
63 hash[ind] = (unsigned char)ind;
66 for (ind = 0; ind < 256; ind++)
68 unsigned char tc;
70 j += (hash[ind] + k2[ind%k2l]);
72 tc = hash[ind];
73 hash[ind] = hash[j];
74 hash[j] = tc;
77 hash[256] = 0;
78 hash[257] = 0;
81 static void calc_ntlmv2_hash(unsigned char hash[16], char digest[16],
82 DATA_BLOB session_key,
83 const char *constant)
85 struct MD5Context ctx3;
87 /* NOTE: This code is currently complate fantasy - it's
88 got more in common with reality than the previous code
89 (the LM session key is not the right thing to use) but
90 it still needs work */
92 MD5Init(&ctx3);
93 MD5Update(&ctx3, session_key.data, session_key.length);
94 MD5Update(&ctx3, (const unsigned char *)constant, strlen(constant));
95 MD5Final((unsigned char *)digest, &ctx3);
97 calc_hash(hash, digest, 16);
100 enum ntlmssp_direction {
101 NTLMSSP_SEND,
102 NTLMSSP_RECEIVE
105 static NTSTATUS ntlmssp_make_packet_signature(NTLMSSP_CLIENT_STATE *ntlmssp_state,
106 const uchar *data, size_t length,
107 enum ntlmssp_direction direction,
108 DATA_BLOB *sig)
110 if (ntlmssp_state->neg_flags & NTLMSSP_NEGOTIATE_NTLM2) {
111 HMACMD5Context ctx;
112 char seq_num[4];
113 uchar digest[16];
114 SIVAL(seq_num, 0, ntlmssp_state->ntlmssp_seq_num);
116 hmac_md5_init_limK_to_64((const unsigned char *)(ntlmssp_state->cli_sign_const), 16, &ctx);
117 hmac_md5_update((const unsigned char *)seq_num, 4, &ctx);
118 hmac_md5_update(data, length, &ctx);
119 hmac_md5_final(digest, &ctx);
121 if (!msrpc_gen(sig, "dBd", NTLMSSP_SIGN_VERSION, digest, 8 /* only copy first 8 bytes */
122 , ntlmssp_state->ntlmssp_seq_num)) {
123 return NT_STATUS_NO_MEMORY;
125 switch (direction) {
126 case NTLMSSP_SEND:
127 NTLMSSPcalc_ap(ntlmssp_state->cli_sign_hash, sig->data+4, sig->length-4);
128 break;
129 case NTLMSSP_RECEIVE:
130 NTLMSSPcalc_ap(ntlmssp_state->srv_sign_hash, sig->data+4, sig->length-4);
131 break;
133 } else {
134 uint32 crc;
135 crc = crc32_calc_buffer((const char *)data, length);
136 if (!msrpc_gen(sig, "dddd", NTLMSSP_SIGN_VERSION, 0, crc, ntlmssp_state->ntlmssp_seq_num)) {
137 return NT_STATUS_NO_MEMORY;
140 dump_data_pw("ntlmssp hash:\n", ntlmssp_state->ntlmssp_hash,
141 sizeof(ntlmssp_state->ntlmssp_hash));
142 NTLMSSPcalc_ap(ntlmssp_state->ntlmssp_hash, sig->data+4, sig->length-4);
144 return NT_STATUS_OK;
147 NTSTATUS ntlmssp_client_sign_packet(NTLMSSP_CLIENT_STATE *ntlmssp_state,
148 const uchar *data, size_t length,
149 DATA_BLOB *sig)
151 NTSTATUS nt_status = ntlmssp_make_packet_signature(ntlmssp_state, data, length, NTLMSSP_SEND, sig);
153 /* increment counter on send */
154 ntlmssp_state->ntlmssp_seq_num++;
155 return nt_status;
159 * Check the signature of an incoming packet
160 * @note caller *must* check that the signature is the size it expects
164 NTSTATUS ntlmssp_client_check_packet(NTLMSSP_CLIENT_STATE *ntlmssp_state,
165 const uchar *data, size_t length,
166 const DATA_BLOB *sig)
168 DATA_BLOB local_sig;
169 NTSTATUS nt_status;
171 if (sig->length < 8) {
172 DEBUG(0, ("NTLMSSP packet check failed due to short signature (%u bytes)!\n",
173 sig->length));
176 nt_status = ntlmssp_make_packet_signature(ntlmssp_state, data,
177 length, NTLMSSP_RECEIVE, &local_sig);
179 if (!NT_STATUS_IS_OK(nt_status)) {
180 DEBUG(0, ("NTLMSSP packet check failed with %s\n", nt_errstr(nt_status)));
181 return nt_status;
184 if (memcmp(sig->data+sig->length - 8, local_sig.data+local_sig.length - 8, 8) != 0) {
185 DEBUG(5, ("BAD SIG: wanted signature of\n"));
186 dump_data(5, (const char *)local_sig.data, local_sig.length);
188 DEBUG(5, ("BAD SIG: got signature of\n"));
189 dump_data(5, (const char *)(sig->data), sig->length);
191 DEBUG(0, ("NTLMSSP packet check failed due to invalid signature!\n"));
192 return NT_STATUS_ACCESS_DENIED;
195 /* increment counter on recieive */
196 ntlmssp_state->ntlmssp_seq_num++;
198 return NT_STATUS_OK;
203 * Seal data with the NTLMSSP algorithm
207 NTSTATUS ntlmssp_client_seal_packet(NTLMSSP_CLIENT_STATE *ntlmssp_state,
208 uchar *data, size_t length,
209 DATA_BLOB *sig)
211 DEBUG(10,("ntlmssp_client_seal_data: seal\n"));
212 dump_data_pw("ntlmssp clear data\n", data, length);
213 if (ntlmssp_state->neg_flags & NTLMSSP_NEGOTIATE_NTLM2) {
214 HMACMD5Context ctx;
215 char seq_num[4];
216 uchar digest[16];
217 SIVAL(seq_num, 0, ntlmssp_state->ntlmssp_seq_num);
219 hmac_md5_init_limK_to_64((const unsigned char *)(ntlmssp_state->cli_sign_const), 16, &ctx);
220 hmac_md5_update((const unsigned char *)seq_num, 4, &ctx);
221 hmac_md5_update(data, length, &ctx);
222 hmac_md5_final(digest, &ctx);
224 if (!msrpc_gen(sig, "dBd", NTLMSSP_SIGN_VERSION, digest, 8 /* only copy first 8 bytes */
225 , ntlmssp_state->ntlmssp_seq_num)) {
226 return NT_STATUS_NO_MEMORY;
229 dump_data_pw("ntlmssp client sealing hash:\n",
230 ntlmssp_state->cli_seal_hash,
231 sizeof(ntlmssp_state->cli_seal_hash));
232 NTLMSSPcalc_ap(ntlmssp_state->cli_seal_hash, data, length);
233 dump_data_pw("ntlmssp client signing hash:\n",
234 ntlmssp_state->cli_sign_hash,
235 sizeof(ntlmssp_state->cli_sign_hash));
236 NTLMSSPcalc_ap(ntlmssp_state->cli_sign_hash, sig->data+4, sig->length-4);
237 } else {
238 uint32 crc;
239 crc = crc32_calc_buffer((const char *)data, length);
240 if (!msrpc_gen(sig, "dddd", NTLMSSP_SIGN_VERSION, 0, crc, ntlmssp_state->ntlmssp_seq_num)) {
241 return NT_STATUS_NO_MEMORY;
244 /* The order of these two operations matters - we must first seal the packet,
245 then seal the sequence number - this is becouse the ntlmssp_hash is not
246 constant, but is is rather updated with each iteration */
248 dump_data_pw("ntlmssp hash:\n", ntlmssp_state->ntlmssp_hash,
249 sizeof(ntlmssp_state->ntlmssp_hash));
250 NTLMSSPcalc_ap(ntlmssp_state->ntlmssp_hash, data, length);
252 dump_data_pw("ntlmssp hash:\n", ntlmssp_state->ntlmssp_hash,
253 sizeof(ntlmssp_state->ntlmssp_hash));
254 NTLMSSPcalc_ap(ntlmssp_state->ntlmssp_hash, sig->data+4, sig->length-4);
256 dump_data_pw("ntlmssp sealed data\n", data, length);
258 /* increment counter on send */
259 ntlmssp_state->ntlmssp_seq_num++;
261 return NT_STATUS_OK;
265 * Unseal data with the NTLMSSP algorithm
269 NTSTATUS ntlmssp_client_unseal_packet(NTLMSSP_CLIENT_STATE *ntlmssp_state,
270 uchar *data, size_t length,
271 DATA_BLOB *sig)
273 DEBUG(10,("ntlmssp_client_unseal_data: seal\n"));
274 dump_data_pw("ntlmssp sealed data\n", data, length);
275 if (ntlmssp_state->neg_flags & NTLMSSP_NEGOTIATE_NTLM2) {
276 NTLMSSPcalc_ap(ntlmssp_state->srv_seal_hash, data, length);
277 } else {
278 dump_data_pw("ntlmssp hash:\n", ntlmssp_state->ntlmssp_hash,
279 sizeof(ntlmssp_state->ntlmssp_hash));
280 NTLMSSPcalc_ap(ntlmssp_state->ntlmssp_hash, data, length);
282 dump_data_pw("ntlmssp clear data\n", data, length);
284 return ntlmssp_client_check_packet(ntlmssp_state, data, length, sig);
288 Initialise the state for NTLMSSP signing.
290 NTSTATUS ntlmssp_client_sign_init(NTLMSSP_CLIENT_STATE *ntlmssp_state)
292 unsigned char p24[24];
293 ZERO_STRUCT(p24);
295 DEBUG(3, ("NTLMSSP Sign/Seal - Initialising with flags:\n"));
296 debug_ntlmssp_flags(ntlmssp_state->neg_flags);
298 if (ntlmssp_state->neg_flags & NTLMSSP_NEGOTIATE_NTLM2)
301 calc_ntlmv2_hash(ntlmssp_state->cli_sign_hash,
302 ntlmssp_state->cli_sign_const,
303 ntlmssp_state->session_key, CLI_SIGN);
304 dump_data_pw("NTLMSSP client sign hash:\n",
305 ntlmssp_state->cli_sign_hash,
306 sizeof(ntlmssp_state->cli_sign_hash));
308 calc_ntlmv2_hash(ntlmssp_state->cli_seal_hash,
309 ntlmssp_state->cli_seal_const,
310 ntlmssp_state->session_key, CLI_SEAL);
311 dump_data_pw("NTLMSSP client sesl hash:\n",
312 ntlmssp_state->cli_seal_hash,
313 sizeof(ntlmssp_state->cli_seal_hash));
315 calc_ntlmv2_hash(ntlmssp_state->srv_sign_hash,
316 ntlmssp_state->srv_sign_const,
317 ntlmssp_state->session_key, SRV_SIGN);
318 dump_data_pw("NTLMSSP server sign hash:\n",
319 ntlmssp_state->srv_sign_hash,
320 sizeof(ntlmssp_state->srv_sign_hash));
322 calc_ntlmv2_hash(ntlmssp_state->srv_seal_hash,
323 ntlmssp_state->srv_seal_const,
324 ntlmssp_state->session_key, SRV_SEAL);
325 dump_data_pw("NTLMSSP server seal hash:\n",
326 ntlmssp_state->cli_sign_hash,
327 sizeof(ntlmssp_state->cli_sign_hash));
329 else if (ntlmssp_state->neg_flags & NTLMSSP_NEGOTIATE_LM_KEY) {
330 if (!ntlmssp_state->session_key.data || ntlmssp_state->session_key.length < 8) {
331 /* can't sign or check signatures yet */
332 DEBUG(5, ("NTLMSSP Sign/Seal - cannot use LM KEY yet\n"));
333 return NT_STATUS_UNSUCCESSFUL;
336 DEBUG(5, ("NTLMSSP Sign/Seal - using LM KEY\n"));
338 calc_hash(ntlmssp_state->ntlmssp_hash, (const char *)(ntlmssp_state->session_key.data), 8);
339 dump_data_pw("NTLMSSP hash:\n", ntlmssp_state->ntlmssp_hash,
340 sizeof(ntlmssp_state->ntlmssp_hash));
341 } else {
342 if (!ntlmssp_state->session_key.data || ntlmssp_state->session_key.length < 16) {
343 /* can't sign or check signatures yet */
344 DEBUG(5, ("NTLMSSP Sign/Seal - cannot use NT KEY yet\n"));
345 return NT_STATUS_UNSUCCESSFUL;
348 DEBUG(5, ("NTLMSSP Sign/Seal - using NT KEY\n"));
350 calc_hash(ntlmssp_state->ntlmssp_hash, (const char *)(ntlmssp_state->session_key.data), 16);
351 dump_data_pw("NTLMSSP hash:\n", ntlmssp_state->ntlmssp_hash,
352 sizeof(ntlmssp_state->ntlmssp_hash));
355 ntlmssp_state->ntlmssp_seq_num = 0;
357 return NT_STATUS_OK;