libtommath: Fix possible integer overflow CVE-2023-36328
[heimdal.git] / lib / hcrypto / rc4.c
blobcc7882d298b49be63c8a8fff6716518653187ddd
1 /*
2 * Copyright (c) 2004 Kungliga Tekniska Högskolan
3 * (Royal Institute of Technology, Stockholm, Sweden).
4 * All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the name of the Institute nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
34 /* implemented from description in draft-kaukonen-cipher-arcfour-03.txt */
36 #include <config.h>
37 #include <roken.h>
39 #include <rc4.h>
41 #define SWAP(k,x,y) \
42 { unsigned int _t; \
43 _t = k->state[x]; \
44 k->state[x] = k->state[y]; \
45 k->state[y] = _t; \
48 void
49 RC4_set_key(RC4_KEY *key, const int len, const unsigned char *data)
51 int i, j;
53 for (i = 0; i < 256; i++)
54 key->state[i] = i;
55 for (i = 0, j = 0; i < 256; i++) {
56 j = (j + key->state[i] + data[i % len]) % 256;
57 SWAP(key, i, j);
59 key->x = key->y = 0;
62 void
63 RC4(RC4_KEY *key, const int len, const unsigned char *in, unsigned char *out)
65 int i, t;
66 unsigned x, y;
68 x = key->x;
69 y = key->y;
70 for (i = 0; i < len; i++) {
71 x = (x + 1) % 256;
72 y = (y + key->state[x]) % 256;
73 SWAP(key, x, y);
74 t = (key->state[x] + key->state[y]) % 256;
75 *out++ = key->state[t] ^ *in++;
77 key->x = x;
78 key->y = y;