1 /* crc32c.c -- compute CRC-32C using the Intel crc32 instruction
2 * Copyright (C) 2013 Mark Adler
3 * Version 1.1 1 Aug 2013 Mark Adler
7 This software is provided 'as-is', without any express or implied
8 warranty. In no event will the author be held liable for any damages
9 arising from the use of this software.
11 Permission is granted to anyone to use this software for any purpose,
12 including commercial applications, and to alter it and redistribute it
13 freely, subject to the following restrictions:
15 1. The origin of this software must not be misrepresented; you must not
16 claim that you wrote the original software. If you use this software
17 in a product, an acknowledgment in the product documentation would be
18 appreciated but is not required.
19 2. Altered source versions must be plainly marked as such, and must not be
20 misrepresented as being the original software.
21 3. This notice may not be removed or altered from any source distribution.
24 madler@alumni.caltech.edu
27 /* Use hardware CRC instruction on Intel SSE 4.2 processors. This computes a
28 CRC-32C, *not* the CRC-32 used by Ethernet and zip, gzip, etc. A software
29 version is provided as a fall-back, as well as for speed comparisons. */
32 1.0 10 Feb 2013 First version
33 1.1 1 Aug 2013 Correct comments on why three crc instructions in parallel
42 /* CRC-32C (iSCSI) polynomial in reversed bit order. */
43 #define POLY 0x82f63b78
45 /* Table for a quadword-at-a-time software crc. */
46 static pthread_once_t crc32c_once_sw
= PTHREAD_ONCE_INIT
;
47 static uint32_t crc32c_table
[8][256];
49 /* Construct table for software CRC-32C calculation. */
50 static void crc32c_init_sw(void)
54 for (n
= 0; n
< 256; n
++) {
56 crc
= crc
& 1 ? (crc
>> 1) ^ POLY
: crc
>> 1;
57 crc
= crc
& 1 ? (crc
>> 1) ^ POLY
: crc
>> 1;
58 crc
= crc
& 1 ? (crc
>> 1) ^ POLY
: crc
>> 1;
59 crc
= crc
& 1 ? (crc
>> 1) ^ POLY
: crc
>> 1;
60 crc
= crc
& 1 ? (crc
>> 1) ^ POLY
: crc
>> 1;
61 crc
= crc
& 1 ? (crc
>> 1) ^ POLY
: crc
>> 1;
62 crc
= crc
& 1 ? (crc
>> 1) ^ POLY
: crc
>> 1;
63 crc
= crc
& 1 ? (crc
>> 1) ^ POLY
: crc
>> 1;
64 crc32c_table
[0][n
] = crc
;
66 for (n
= 0; n
< 256; n
++) {
67 crc
= crc32c_table
[0][n
];
68 for (k
= 1; k
< 8; k
++) {
69 crc
= crc32c_table
[0][crc
& 0xff] ^ (crc
>> 8);
70 crc32c_table
[k
][n
] = crc
;
75 /* Table-driven software version as a fall-back. This is about 15 times slower
76 than using the hardware instructions. This assumes little-endian integers,
77 as is the case on Intel processors that the assembler code here is for. */
78 static uint32_t crc32c_sw(uint32_t crci
, const void *buf
, size_t len
)
80 const unsigned char *next
= buf
;
84 pthread_once(&crc32c_once_sw
, crc32c_init_sw
);
85 crc
= crci
^ 0xffffffff;
86 while (len
&& ((uintptr_t)next
& 7) != 0) {
87 crc
= crc32c_table
[0][(crc
^ *next
++) & 0xff] ^ (crc
>> 8);
91 crc
^= *(uint64_t *)next
;
92 crc
= crc32c_table
[7][crc
& 0xff] ^
93 crc32c_table
[6][(crc
>> 8) & 0xff] ^
94 crc32c_table
[5][(crc
>> 16) & 0xff] ^
95 crc32c_table
[4][(crc
>> 24) & 0xff] ^
96 crc32c_table
[3][(crc
>> 32) & 0xff] ^
97 crc32c_table
[2][(crc
>> 40) & 0xff] ^
98 crc32c_table
[1][(crc
>> 48) & 0xff] ^
99 crc32c_table
[0][crc
>> 56];
104 crc
= crc32c_table
[0][(crc
^ *next
++) & 0xff] ^ (crc
>> 8);
107 return (uint32_t)crc
^ 0xffffffff;
110 /* Multiply a matrix times a vector over the Galois field of two elements,
111 GF(2). Each element is a bit in an unsigned integer. mat must have at
112 least as many entries as the power of two for most significant one bit in
114 static inline uint32_t gf2_matrix_times(uint32_t *mat
, uint32_t vec
)
128 /* Multiply a matrix by itself over GF(2). Both mat and square must have 32
130 static inline void gf2_matrix_square(uint32_t *square
, uint32_t *mat
)
134 for (n
= 0; n
< 32; n
++)
135 square
[n
] = gf2_matrix_times(mat
, mat
[n
]);
138 /* Construct an operator to apply len zeros to a crc. len must be a power of
139 two. If len is not a power of two, then the result is the same as for the
140 largest power of two less than len. The result for len == 0 is the same as
141 for len == 1. A version of this routine could be easily written for any
142 len, but that is not needed for this application. */
143 static void crc32c_zeros_op(uint32_t *even
, size_t len
)
147 uint32_t odd
[32]; /* odd-power-of-two zeros operator */
149 /* put operator for one zero bit in odd */
150 odd
[0] = POLY
; /* CRC-32C polynomial */
152 for (n
= 1; n
< 32; n
++) {
157 /* put operator for two zero bits in even */
158 gf2_matrix_square(even
, odd
);
160 /* put operator for four zero bits in odd */
161 gf2_matrix_square(odd
, even
);
163 /* first square will put the operator for one zero byte (eight zero bits),
164 in even -- next square puts operator for two zero bytes in odd, and so
165 on, until len has been rotated down to zero */
167 gf2_matrix_square(even
, odd
);
171 gf2_matrix_square(odd
, even
);
175 /* answer ended up in odd -- copy to even */
176 for (n
= 0; n
< 32; n
++)
180 /* Take a length and build four lookup tables for applying the zeros operator
181 for that length, byte-by-byte on the operand. */
182 static void crc32c_zeros(uint32_t zeros
[][256], size_t len
)
187 crc32c_zeros_op(op
, len
);
188 for (n
= 0; n
< 256; n
++) {
189 zeros
[0][n
] = gf2_matrix_times(op
, n
);
190 zeros
[1][n
] = gf2_matrix_times(op
, n
<< 8);
191 zeros
[2][n
] = gf2_matrix_times(op
, n
<< 16);
192 zeros
[3][n
] = gf2_matrix_times(op
, n
<< 24);
196 /* Apply the zeros operator table to crc. */
197 static inline uint32_t crc32c_shift(uint32_t zeros
[][256], uint32_t crc
)
199 return zeros
[0][crc
& 0xff] ^ zeros
[1][(crc
>> 8) & 0xff] ^
200 zeros
[2][(crc
>> 16) & 0xff] ^ zeros
[3][crc
>> 24];
203 /* Block sizes for three-way parallel crc computation. LONG and SHORT must
204 both be powers of two. The associated string constants must be set
205 accordingly, for use in constructing the assembler instructions. */
207 #define LONGx1 "8192"
208 #define LONGx2 "16384"
210 #define SHORTx1 "256"
211 #define SHORTx2 "512"
213 /* Tables for hardware crc that shift a crc by LONG and SHORT zeros. */
214 static pthread_once_t crc32c_once_hw
= PTHREAD_ONCE_INIT
;
215 static uint32_t crc32c_long
[4][256];
216 static uint32_t crc32c_short
[4][256];
218 /* Initialize tables for shifting crcs. */
219 static void crc32c_init_hw(void)
221 crc32c_zeros(crc32c_long
, LONG
);
222 crc32c_zeros(crc32c_short
, SHORT
);
225 /* Compute CRC-32C using the Intel hardware instruction. */
226 static uint32_t crc32c_hw(uint32_t crc
, const void *buf
, size_t len
)
228 const unsigned char *next
= buf
;
229 const unsigned char *end
;
230 uint64_t crc0
, crc1
, crc2
; /* need to be 64 bits for crc32q */
232 /* populate shift tables the first time through */
233 pthread_once(&crc32c_once_hw
, crc32c_init_hw
);
235 /* pre-process the crc */
236 crc0
= crc
^ 0xffffffff;
238 /* compute the crc for up to seven leading bytes to bring the data pointer
239 to an eight-byte boundary */
240 while (len
&& ((uintptr_t)next
& 7) != 0) {
241 __asm__("crc32b\t" "(%1), %0"
243 : "r"(next
), "0"(crc0
));
248 /* compute the crc on sets of LONG*3 bytes, executing three independent crc
249 instructions, each on LONG bytes -- this is optimized for the Nehalem,
250 Westmere, Sandy Bridge, and Ivy Bridge architectures, which have a
251 throughput of one crc per cycle, but a latency of three cycles */
252 while (len
>= LONG
*3) {
257 __asm__("crc32q\t" "(%3), %0\n\t"
258 "crc32q\t" LONGx1
"(%3), %1\n\t"
259 "crc32q\t" LONGx2
"(%3), %2"
260 : "=r"(crc0
), "=r"(crc1
), "=r"(crc2
)
261 : "r"(next
), "0"(crc0
), "1"(crc1
), "2"(crc2
));
263 } while (next
< end
);
264 crc0
= crc32c_shift(crc32c_long
, crc0
) ^ crc1
;
265 crc0
= crc32c_shift(crc32c_long
, crc0
) ^ crc2
;
270 /* do the same thing, but now on SHORT*3 blocks for the remaining data less
271 than a LONG*3 block */
272 while (len
>= SHORT
*3) {
277 __asm__("crc32q\t" "(%3), %0\n\t"
278 "crc32q\t" SHORTx1
"(%3), %1\n\t"
279 "crc32q\t" SHORTx2
"(%3), %2"
280 : "=r"(crc0
), "=r"(crc1
), "=r"(crc2
)
281 : "r"(next
), "0"(crc0
), "1"(crc1
), "2"(crc2
));
283 } while (next
< end
);
284 crc0
= crc32c_shift(crc32c_short
, crc0
) ^ crc1
;
285 crc0
= crc32c_shift(crc32c_short
, crc0
) ^ crc2
;
290 /* compute the crc on the remaining eight-byte units less than a SHORT*3
292 end
= next
+ (len
- (len
& 7));
294 __asm__("crc32q\t" "(%1), %0"
296 : "r"(next
), "0"(crc0
));
301 /* compute the crc for up to seven trailing bytes */
303 __asm__("crc32b\t" "(%1), %0"
305 : "r"(next
), "0"(crc0
));
310 /* return a post-processed crc */
311 return (uint32_t)crc0
^ 0xffffffff;
314 /* Check for SSE 4.2. SSE 4.2 was first supported in Nehalem processors
315 introduced in November, 2008. This does not check for the existence of the
316 cpuid instruction itself, which was introduced on the 486SL in 1992, so this
317 will fail on earlier x86 processors. cpuid works on all Pentium and later
319 #define SSE42(have) \
327 (have) = (ecx >> 20) & 1; \
330 /* Compute a CRC-32C. If the crc32 instruction is available, use the hardware
331 version. Otherwise, use the software version. */
332 uint32_t crc32c(uint32_t crc
, const void *buf
, size_t len
)
337 return sse42
? crc32c_hw(crc
, buf
, len
) : crc32c_sw(crc
, buf
, len
);
342 #define SIZE (262144*3)
345 int main(int argc
, char **argv
)
356 fputs("out of memory", stderr
);
359 while ((got
= read(0, buf
, SIZE
)) > 0) {
362 n
= (size_t)got
- off
;
365 crc
= argc
> 1 ? crc32c_sw(crc
, buf
+ off
, n
) :
366 crc32c(crc
, buf
+ off
, n
);
368 } while (off
< (size_t)got
);
372 fputs("read error\n", stderr
);
375 printf("%08x\n", crc
);