bringing smbfs-1.74 into the main branch
[AROS.git] / workbench / network / smbfs / source_code / crypt.c
blobf1800649e5ca9b1dc774c2d5036c99dd5dfd4ead
1 /*
2 * $Id: crypt.c,v 1.2 2009/07/22 07:52:59 obarthel Exp $
4 * :ts=8
6 * Password encryption routines, lifted from the Samba source code,
7 * "libsmb/smbencrypt.c", "libsmb/smbdes.c" and "lib/md4.c":
9 * Unix SMB/Netbios implementation.
10 * Version 1.9.
12 * SMB parameters and setup
14 * A partial implementation of DES designed for use in the
15 * SMB authentication protocol
17 * An implementation of MD4 designed for use in the SMB authentication protocol
19 * Copyright (C) Andrew Tridgell 1992-1998
20 * Modified by Jeremy Allison 1995.
22 * This program is free software; you can redistribute it and/or modify
23 * it under the terms of the GNU General Public License as published by
24 * the Free Software Foundation; either version 2 of the License, or
25 * (at your option) any later version.
27 * This program is distributed in the hope that it will be useful,
28 * but WITHOUT ANY WARRANTY; without even the implied warranty of
29 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
30 * GNU General Public License for more details.
32 * You should have received a copy of the GNU General Public License
33 * along with this program; if not, write to the Free Software
34 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
37 #include "smbfs.h"
39 /****************************************************************************/
41 #include <smb/smb.h>
43 /****************************************************************************/
45 static void permute (char *out, char *in, const unsigned char * const p, int n);
46 static void left_shift (char *d, int count, int n);
47 static void concat (char *out, char *in1, char *in2, int l1, int l2);
48 static void xor (char *out, char *in1, char *in2, int n);
49 static void dohash (char *out, char *in, char *key, int forw);
50 static void str_to_key (unsigned char *str, unsigned char *key);
51 static void smbhash (unsigned char *out, unsigned char *in, unsigned char *key, int forw);
52 static void E_P16 (unsigned char *p14, unsigned char *p16);
53 static void E_P24 (unsigned char *p21, unsigned char *c8, unsigned char *p24);
54 static int local_wcslen (short *str);
55 static int local_mbstowcs (short *dst, unsigned char *src, int len);
56 static void E_md4hash (unsigned char *passwd, unsigned char *p16);
57 static void smb_owf_encrypt (unsigned char *passwd, unsigned char *c8, unsigned char *p24);
58 static unsigned long F (unsigned long X, unsigned long Y, unsigned long Z);
59 static unsigned long G (unsigned long X, unsigned long Y, unsigned long Z);
60 static unsigned long H (unsigned long X, unsigned long Y, unsigned long Z);
61 static unsigned long lshift (unsigned long x, int s);
62 static void mdfour64 (unsigned long *M);
63 static void copy64 (unsigned long *M, unsigned char *in);
64 static void copy4 (unsigned char *out, unsigned long x);
65 static void mdfour (unsigned char *out, unsigned char *in, int n);
67 /****************************************************************************/
69 /* NOTES:
71 This code makes no attempt to be fast! In fact, it is a very
72 slow implementation
74 This code is NOT a complete DES implementation. It implements only
75 the minimum necessary for SMB authentication, as used by all SMB
76 products (including every copy of Microsoft Windows95 ever sold)
78 In particular, it can only do a unchained forward DES pass. This
79 means it is not possible to use this code for encryption/decryption
80 of data, instead it is only useful as a "hash" algorithm.
82 There is no entry point into this code that allows normal DES operation.
84 I believe this means that this code does not come under ITAR
85 regulations but this is NOT a legal opinion. If you are concerned
86 about the applicability of ITAR regulations to this code then you
87 should confirm it for yourself (and maybe let me know if you come
88 up with a different answer to the one above) */
90 static const unsigned char perm1[56] =
92 57, 49, 41, 33, 25, 17, 9,
93 1, 58, 50, 42, 34, 26, 18,
94 10, 2, 59, 51, 43, 35, 27,
95 19, 11, 3, 60, 52, 44, 36,
96 63, 55, 47, 39, 31, 23, 15,
97 7, 62, 54, 46, 38, 30, 22,
98 14, 6, 61, 53, 45, 37, 29,
99 21, 13, 5, 28, 20, 12, 4
102 static const unsigned char perm2[48] =
104 14, 17, 11, 24, 1, 5,
105 3, 28, 15, 6, 21, 10,
106 23, 19, 12, 4, 26, 8,
107 16, 7, 27, 20, 13, 2,
108 41, 52, 31, 37, 47, 55,
109 30, 40, 51, 45, 33, 48,
110 44, 49, 39, 56, 34, 53,
111 46, 42, 50, 36, 29, 32
114 static const unsigned char perm3[64] =
116 58, 50, 42, 34, 26, 18, 10, 2,
117 60, 52, 44, 36, 28, 20, 12, 4,
118 62, 54, 46, 38, 30, 22, 14, 6,
119 64, 56, 48, 40, 32, 24, 16, 8,
120 57, 49, 41, 33, 25, 17, 9, 1,
121 59, 51, 43, 35, 27, 19, 11, 3,
122 61, 53, 45, 37, 29, 21, 13, 5,
123 63, 55, 47, 39, 31, 23, 15, 7
126 static const unsigned char perm4[48] =
128 32, 1, 2, 3, 4, 5,
129 4, 5, 6, 7, 8, 9,
130 8, 9, 10, 11, 12, 13,
131 12, 13, 14, 15, 16, 17,
132 16, 17, 18, 19, 20, 21,
133 20, 21, 22, 23, 24, 25,
134 24, 25, 26, 27, 28, 29,
135 28, 29, 30, 31, 32, 1
138 static const unsigned char perm5[32] =
140 16, 7, 20, 21,
141 29, 12, 28, 17,
142 1, 15, 23, 26,
143 5, 18, 31, 10,
144 2, 8, 24, 14,
145 32, 27, 3, 9,
146 19, 13, 30, 6,
147 22, 11, 4, 25
150 static const unsigned char perm6[64] =
152 40, 8, 48, 16, 56, 24, 64, 32,
153 39, 7, 47, 15, 55, 23, 63, 31,
154 38, 6, 46, 14, 54, 22, 62, 30,
155 37, 5, 45, 13, 53, 21, 61, 29,
156 36, 4, 44, 12, 52, 20, 60, 28,
157 35, 3, 43, 11, 51, 19, 59, 27,
158 34, 2, 42, 10, 50, 18, 58, 26,
159 33, 1, 41, 9, 49, 17, 57, 25
162 static const unsigned char sc[16] =
164 1, 1, 2, 2,
165 2, 2, 2, 2,
166 1, 2, 2, 2,
167 2, 2, 2, 1
170 static const unsigned char sbox[8][4][16] =
173 {14, 4, 13, 1, 2, 15, 11, 8, 3, 10, 6, 12, 5, 9, 0, 7},
174 {0, 15, 7, 4, 14, 2, 13, 1, 10, 6, 12, 11, 9, 5, 3, 8},
175 {4, 1, 14, 8, 13, 6, 2, 11, 15, 12, 9, 7, 3, 10, 5, 0},
176 {15, 12, 8, 2, 4, 9, 1, 7, 5, 11, 3, 14, 10, 0, 6, 13}
180 {15, 1, 8, 14, 6, 11, 3, 4, 9, 7, 2, 13, 12, 0, 5, 10},
181 {3, 13, 4, 7, 15, 2, 8, 14, 12, 0, 1, 10, 6, 9, 11, 5},
182 {0, 14, 7, 11, 10, 4, 13, 1, 5, 8, 12, 6, 9, 3, 2, 15},
183 {13, 8, 10, 1, 3, 15, 4, 2, 11, 6, 7, 12, 0, 5, 14, 9}
187 {10, 0, 9, 14, 6, 3, 15, 5, 1, 13, 12, 7, 11, 4, 2, 8},
188 {13, 7, 0, 9, 3, 4, 6, 10, 2, 8, 5, 14, 12, 11, 15, 1},
189 {13, 6, 4, 9, 8, 15, 3, 0, 11, 1, 2, 12, 5, 10, 14, 7},
190 {1, 10, 13, 0, 6, 9, 8, 7, 4, 15, 14, 3, 11, 5, 2, 12}
194 {7, 13, 14, 3, 0, 6, 9, 10, 1, 2, 8, 5, 11, 12, 4, 15},
195 {13, 8, 11, 5, 6, 15, 0, 3, 4, 7, 2, 12, 1, 10, 14, 9},
196 {10, 6, 9, 0, 12, 11, 7, 13, 15, 1, 3, 14, 5, 2, 8, 4},
197 {3, 15, 0, 6, 10, 1, 13, 8, 9, 4, 5, 11, 12, 7, 2, 14}
201 {2, 12, 4, 1, 7, 10, 11, 6, 8, 5, 3, 15, 13, 0, 14, 9},
202 {14, 11, 2, 12, 4, 7, 13, 1, 5, 0, 15, 10, 3, 9, 8, 6},
203 {4, 2, 1, 11, 10, 13, 7, 8, 15, 9, 12, 5, 6, 3, 0, 14},
204 {11, 8, 12, 7, 1, 14, 2, 13, 6, 15, 0, 9, 10, 4, 5, 3}
208 {12, 1, 10, 15, 9, 2, 6, 8, 0, 13, 3, 4, 14, 7, 5, 11},
209 {10, 15, 4, 2, 7, 12, 9, 5, 6, 1, 13, 14, 0, 11, 3, 8},
210 {9, 14, 15, 5, 2, 8, 12, 3, 7, 0, 4, 10, 1, 13, 11, 6},
211 {4, 3, 2, 12, 9, 5, 15, 10, 11, 14, 1, 7, 6, 0, 8, 13}
215 {4, 11, 2, 14, 15, 0, 8, 13, 3, 12, 9, 7, 5, 10, 6, 1},
216 {13, 0, 11, 7, 4, 9, 1, 10, 14, 3, 5, 12, 2, 15, 8, 6},
217 {1, 4, 11, 13, 12, 3, 7, 14, 10, 15, 6, 8, 0, 5, 9, 2},
218 {6, 11, 13, 8, 1, 4, 10, 7, 9, 5, 0, 15, 14, 2, 3, 12}
222 {13, 2, 8, 4, 6, 15, 11, 1, 10, 9, 3, 14, 5, 0, 12, 7},
223 {1, 15, 13, 8, 10, 3, 7, 4, 12, 5, 6, 11, 0, 14, 9, 2},
224 {7, 11, 4, 1, 9, 12, 14, 2, 0, 6, 10, 13, 15, 3, 5, 8},
225 {2, 1, 14, 7, 4, 10, 8, 13, 15, 12, 9, 0, 3, 5, 6, 11}
229 static void
230 permute (char *out, char *in, const unsigned char * const p, int n)
232 int i;
234 for (i = 0; i < n; i++)
235 out[i] = in[p[i] - 1];
238 static void
239 left_shift (char *d, int count, int n)
241 char out[64];
242 int i;
244 for (i = 0; i < n; i++)
245 out[i] = d[(i + count) % n];
247 for (i = 0; i < n; i++)
248 d[i] = out[i];
251 static void
252 concat (char *out, char *in1, char *in2, int l1, int l2)
254 while (l1--)
255 (*out++) = (*in1++);
257 while (l2--)
258 (*out++) = (*in2++);
261 static void
262 xor (char *out, char *in1, char *in2, int n)
264 int i;
266 for (i = 0; i < n; i++)
267 out[i] = in1[i] ^ in2[i];
270 static void
271 dohash (char *out, char *in, char *key, int forw)
273 int i, j, k;
274 char pk1[56];
275 char c[28];
276 char d[28];
277 char cd[56];
278 char ki[16][48];
279 char pd1[64];
280 char l[32], r[32];
281 char rl[64];
283 permute (pk1, key, perm1, 56);
285 for (i = 0; i < 28; i++)
286 c[i] = pk1[i];
288 for (i = 0; i < 28; i++)
289 d[i] = pk1[i + 28];
291 for (i = 0; i < 16; i++)
293 left_shift (c, sc[i], 28);
294 left_shift (d, sc[i], 28);
296 concat (cd, c, d, 28, 28);
297 permute (ki[i], cd, perm2, 48);
300 permute (pd1, in, perm3, 64);
302 for (j = 0; j < 32; j++)
304 l[j] = pd1[j];
305 r[j] = pd1[j + 32];
308 for (i = 0; i < 16; i++)
310 char er[48];
311 char erk[48];
312 char b[8][6];
313 char cb[32];
314 char pcb[32];
315 char r2[32];
317 permute (er, r, perm4, 48);
319 xor (erk, er, ki[forw ? i : 15 - i], 48);
321 for (j = 0; j < 8; j++)
323 for (k = 0; k < 6; k++)
324 b[j][k] = erk[j * 6 + k];
327 for (j = 0; j < 8; j++)
329 int m, n;
331 m = (b[j][0] << 1) | b[j][5];
333 n = (b[j][1] << 3) | (b[j][2] << 2) | (b[j][3] << 1) | b[j][4];
335 for (k = 0; k < 4; k++)
336 b[j][k] = (sbox[j][m][n] & (1 << (3 - k))) ? 1 : 0;
339 for (j = 0; j < 8; j++)
341 for (k = 0; k < 4; k++)
342 cb[j * 4 + k] = b[j][k];
345 permute (pcb, cb, perm5, 32);
347 xor (r2, l, pcb, 32);
349 for (j = 0; j < 32; j++)
350 l[j] = r[j];
352 for (j = 0; j < 32; j++)
353 r[j] = r2[j];
356 concat (rl, r, l, 32, 32);
358 permute (out, rl, perm6, 64);
361 static void
362 str_to_key (unsigned char *str, unsigned char *key)
364 int i;
366 key[0] = str[0] >> 1;
367 key[1] = ((str[0] & 0x01) << 6) | (str[1] >> 2);
368 key[2] = ((str[1] & 0x03) << 5) | (str[2] >> 3);
369 key[3] = ((str[2] & 0x07) << 4) | (str[3] >> 4);
370 key[4] = ((str[3] & 0x0F) << 3) | (str[4] >> 5);
371 key[5] = ((str[4] & 0x1F) << 2) | (str[5] >> 6);
372 key[6] = ((str[5] & 0x3F) << 1) | (str[6] >> 7);
373 key[7] = str[6] & 0x7F;
375 for (i = 0; i < 8; i++)
376 key[i] = (key[i] << 1);
379 static void
380 smbhash (unsigned char *out, unsigned char *in, unsigned char *key, int forw)
382 int i;
383 char outb[64];
384 char inb[64];
385 char keyb[64];
386 unsigned char key2[8];
388 str_to_key (key, key2);
390 for (i = 0; i < 64; i++)
392 inb[i] = (in[i / 8] & (1 << (7 - (i % 8)))) ? 1 : 0;
393 keyb[i] = (key2[i / 8] & (1 << (7 - (i % 8)))) ? 1 : 0;
394 outb[i] = 0;
397 dohash (outb, inb, keyb, forw);
399 for (i = 0; i < 8; i++)
400 out[i] = 0;
402 for (i = 0; i < 64; i++)
404 if (outb[i])
405 out[i / 8] |= (1 << (7 - (i % 8)));
409 static void
410 E_P16 (unsigned char *p14, unsigned char *p16)
412 unsigned char sp8[8] =
414 0x4b, 0x47, 0x53, 0x21, 0x40, 0x23, 0x24, 0x25
417 smbhash (p16, sp8, p14, 1);
418 smbhash (p16 + 8, sp8, p14 + 7, 1);
421 static void
422 E_P24 (unsigned char *p21, unsigned char *c8, unsigned char *p24)
424 smbhash (p24, c8, p21, 1);
425 smbhash (p24 + 8, c8, p21 + 7, 1);
426 smbhash (p24 + 16, c8, p21 + 14, 1);
429 /****************************************************************************/
431 /* This implements the X/Open SMB password encryption
432 It takes a password, a 8 byte "crypt key" and puts 24 bytes of
433 encrypted password into p24 */
434 void
435 smb_encrypt (unsigned char *passwd, unsigned char *c8, unsigned char *p24)
437 unsigned char p14[15], p21[21];
438 int len;
440 memset (p21, '\0', sizeof(p21));
441 memset (p14, '\0', sizeof(p14));
443 len = strlen((const char *)passwd);
444 if(len > (int)sizeof(p14)-1)
445 len = sizeof(p14)-1;
447 memcpy (p14, passwd, len);
449 StringToUpper ((char *) p14);
450 E_P16 (p14, p21);
452 smb_owf_encrypt (p21, c8, p24);
455 /****************************************************************************/
457 /* Routines for Windows NT MD4 Hash functions. */
458 static int
459 local_wcslen (short *str)
461 int len = 0;
463 while ((*str++) != 0)
464 len++;
466 return len;
469 /* Convert a string into an NT UNICODE string.
470 Note that regardless of processor type
471 this must be in intel (little-endian)
472 format. */
473 static int
474 local_mbstowcs (short *dst, unsigned char *src, int len)
476 int i;
477 short val;
479 for (i = 0; i < len; i++)
481 val = (*src++);
482 WSET(dst,0,val);
483 dst++;
485 if (val == 0)
486 break;
489 return i;
492 /* Creates the MD4 Hash of the users password in NT UNICODE. */
493 static void
494 E_md4hash (unsigned char *passwd, unsigned char *p16)
496 short wpwd[129];
497 int len;
499 /* Password cannot be longer than 128 characters */
500 len = strlen ((char *) passwd);
501 if (len > 128)
502 len = 128;
504 /* Password must be converted to NT unicode */
505 local_mbstowcs (wpwd, passwd, len);
506 wpwd[len] = 0; /* Ensure string is null terminated */
508 /* Calculate length in bytes */
509 len = local_wcslen (wpwd) * sizeof (short);
511 mdfour (p16, (unsigned char *) wpwd, len);
514 /* Does the des encryption from the NT or LM MD4 hash. */
515 static void
516 smb_owf_encrypt (unsigned char *passwd, unsigned char *c8, unsigned char *p24)
518 unsigned char p21[21];
520 memset (p21, '\0', sizeof(p21));
522 memcpy (p21, passwd, 16);
523 E_P24 (p21, c8, p24);
526 /****************************************************************************/
528 /* Does the NT MD4 hash then des encryption. */
529 void
530 smb_nt_encrypt (unsigned char *passwd, unsigned char *c8, unsigned char *p24)
532 unsigned char p21[21];
534 memset (p21, '\0', sizeof(p21));
536 E_md4hash (passwd, p21);
537 smb_owf_encrypt (p21, c8, p24);
540 /****************************************************************************/
542 static unsigned long A, B, C, D;
544 static unsigned long
545 F(unsigned long X, unsigned long Y, unsigned long Z)
547 return (X & Y) | ((~X) & Z);
550 static unsigned long
551 G(unsigned long X, unsigned long Y, unsigned long Z)
553 return (X & Y) | (X & Z) | (Y & Z);
556 static unsigned long
557 H(unsigned long X, unsigned long Y, unsigned long Z)
559 return X ^ Y ^ Z;
562 static unsigned long
563 lshift (unsigned long x, int s)
565 return (x << s) | (x >> (32 - s));
568 #define ROUND1(a,b,c,d,k,s) a = lshift(a + F(b,c,d) + X[k], s)
569 #define ROUND2(a,b,c,d,k,s) a = lshift(a + G(b,c,d) + X[k] + 0x5A827999UL,s)
570 #define ROUND3(a,b,c,d,k,s) a = lshift(a + H(b,c,d) + X[k] + 0x6ED9EBA1UL,s)
572 /* this applies md4 to 64 byte chunks */
573 static void
574 mdfour64 (unsigned long *M)
576 unsigned long AA, BB, CC, DD;
577 unsigned long X[16];
579 memcpy(X,M,sizeof(X));
581 AA = A;
582 BB = B;
583 CC = C;
584 DD = D;
586 ROUND1 (A, B, C, D, 0, 3);
587 ROUND1 (D, A, B, C, 1, 7);
588 ROUND1 (C, D, A, B, 2, 11);
589 ROUND1 (B, C, D, A, 3, 19);
590 ROUND1 (A, B, C, D, 4, 3);
591 ROUND1 (D, A, B, C, 5, 7);
592 ROUND1 (C, D, A, B, 6, 11);
593 ROUND1 (B, C, D, A, 7, 19);
594 ROUND1 (A, B, C, D, 8, 3);
595 ROUND1 (D, A, B, C, 9, 7);
596 ROUND1 (C, D, A, B, 10, 11);
597 ROUND1 (B, C, D, A, 11, 19);
598 ROUND1 (A, B, C, D, 12, 3);
599 ROUND1 (D, A, B, C, 13, 7);
600 ROUND1 (C, D, A, B, 14, 11);
601 ROUND1 (B, C, D, A, 15, 19);
603 ROUND2 (A, B, C, D, 0, 3);
604 ROUND2 (D, A, B, C, 4, 5);
605 ROUND2 (C, D, A, B, 8, 9);
606 ROUND2 (B, C, D, A, 12, 13);
607 ROUND2 (A, B, C, D, 1, 3);
608 ROUND2 (D, A, B, C, 5, 5);
609 ROUND2 (C, D, A, B, 9, 9);
610 ROUND2 (B, C, D, A, 13, 13);
611 ROUND2 (A, B, C, D, 2, 3);
612 ROUND2 (D, A, B, C, 6, 5);
613 ROUND2 (C, D, A, B, 10, 9);
614 ROUND2 (B, C, D, A, 14, 13);
615 ROUND2 (A, B, C, D, 3, 3);
616 ROUND2 (D, A, B, C, 7, 5);
617 ROUND2 (C, D, A, B, 11, 9);
618 ROUND2 (B, C, D, A, 15, 13);
620 ROUND3 (A, B, C, D, 0, 3);
621 ROUND3 (D, A, B, C, 8, 9);
622 ROUND3 (C, D, A, B, 4, 11);
623 ROUND3 (B, C, D, A, 12, 15);
624 ROUND3 (A, B, C, D, 2, 3);
625 ROUND3 (D, A, B, C, 10, 9);
626 ROUND3 (C, D, A, B, 6, 11);
627 ROUND3 (B, C, D, A, 14, 15);
628 ROUND3 (A, B, C, D, 1, 3);
629 ROUND3 (D, A, B, C, 9, 9);
630 ROUND3 (C, D, A, B, 5, 11);
631 ROUND3 (B, C, D, A, 13, 15);
632 ROUND3 (A, B, C, D, 3, 3);
633 ROUND3 (D, A, B, C, 11, 9);
634 ROUND3 (C, D, A, B, 7, 11);
635 ROUND3 (B, C, D, A, 15, 15);
637 A += AA;
638 B += BB;
639 C += CC;
640 D += DD;
643 static void
644 copy64 (unsigned long *M, unsigned char *in)
646 int i;
648 for (i = 0; i < 16; i++)
650 M[i] = (in[i * 4 + 3] << 24) | (in[i * 4 + 2] << 16) |
651 (in[i * 4 + 1] << 8) | (in[i * 4 + 0] << 0);
655 static void
656 copy4 (unsigned char *out, unsigned long x)
658 out[0] = x & 0xFF;
659 out[1] = (x >> 8) & 0xFF;
660 out[2] = (x >> 16) & 0xFF;
661 out[3] = (x >> 24) & 0xFF;
664 /* produce a md4 message digest from data of length n bytes */
665 static void
666 mdfour (unsigned char *out, unsigned char *in, int n)
668 unsigned char buf[128];
669 unsigned long M[16];
670 unsigned long b = n * 8;
672 A = 0x67452301;
673 B = 0xefcdab89;
674 C = 0x98badcfe;
675 D = 0x10325476;
677 while (n > 64)
679 copy64 (M, in);
680 mdfour64 (M);
682 in += 64;
683 n -= 64;
686 memset (buf, 0, sizeof(buf));
687 memcpy (buf, in, n);
688 buf[n] = 0x80;
690 if (n <= 55)
692 copy4 (buf + 56, b);
693 copy64 (M, buf);
694 mdfour64 (M);
696 else
698 copy4 (buf + 120, b);
699 copy64 (M, buf);
700 mdfour64 (M);
701 copy64 (M, buf + 64);
702 mdfour64 (M);
705 memset (buf, 0, sizeof(buf));
706 copy64 (M, buf);
708 copy4 (out, A);
709 copy4 (out + 4, B);
710 copy4 (out + 8, C);
711 copy4 (out + 12, D);