2 * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
3 * instructions. This file contains glue code.
5 * Copyright (c) 2009 Intel Corp.
6 * Author: Huang Ying <ying.huang@intel.com>
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License version 2 as published
10 * by the Free Software Foundation.
13 #include <linux/err.h>
14 #include <linux/module.h>
15 #include <linux/init.h>
16 #include <linux/kernel.h>
17 #include <linux/crypto.h>
18 #include <crypto/algapi.h>
19 #include <crypto/cryptd.h>
20 #include <crypto/gf128mul.h>
21 #include <crypto/internal/hash.h>
23 #include <asm/cpu_device_id.h>
25 #define GHASH_BLOCK_SIZE 16
26 #define GHASH_DIGEST_SIZE 16
28 void clmul_ghash_mul(char *dst
, const be128
*shash
);
30 void clmul_ghash_update(char *dst
, const char *src
, unsigned int srclen
,
33 void clmul_ghash_setkey(be128
*shash
, const u8
*key
);
35 struct ghash_async_ctx
{
36 struct cryptd_ahash
*cryptd_tfm
;
43 struct ghash_desc_ctx
{
44 u8 buffer
[GHASH_BLOCK_SIZE
];
48 static int ghash_init(struct shash_desc
*desc
)
50 struct ghash_desc_ctx
*dctx
= shash_desc_ctx(desc
);
52 memset(dctx
, 0, sizeof(*dctx
));
57 static int ghash_setkey(struct crypto_shash
*tfm
,
58 const u8
*key
, unsigned int keylen
)
60 struct ghash_ctx
*ctx
= crypto_shash_ctx(tfm
);
62 if (keylen
!= GHASH_BLOCK_SIZE
) {
63 crypto_shash_set_flags(tfm
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
67 clmul_ghash_setkey(&ctx
->shash
, key
);
72 static int ghash_update(struct shash_desc
*desc
,
73 const u8
*src
, unsigned int srclen
)
75 struct ghash_desc_ctx
*dctx
= shash_desc_ctx(desc
);
76 struct ghash_ctx
*ctx
= crypto_shash_ctx(desc
->tfm
);
77 u8
*dst
= dctx
->buffer
;
81 int n
= min(srclen
, dctx
->bytes
);
82 u8
*pos
= dst
+ (GHASH_BLOCK_SIZE
- dctx
->bytes
);
91 clmul_ghash_mul(dst
, &ctx
->shash
);
94 clmul_ghash_update(dst
, src
, srclen
, &ctx
->shash
);
98 src
+= srclen
- (srclen
& 0xf);
100 dctx
->bytes
= GHASH_BLOCK_SIZE
- srclen
;
108 static void ghash_flush(struct ghash_ctx
*ctx
, struct ghash_desc_ctx
*dctx
)
110 u8
*dst
= dctx
->buffer
;
113 u8
*tmp
= dst
+ (GHASH_BLOCK_SIZE
- dctx
->bytes
);
115 while (dctx
->bytes
--)
119 clmul_ghash_mul(dst
, &ctx
->shash
);
126 static int ghash_final(struct shash_desc
*desc
, u8
*dst
)
128 struct ghash_desc_ctx
*dctx
= shash_desc_ctx(desc
);
129 struct ghash_ctx
*ctx
= crypto_shash_ctx(desc
->tfm
);
130 u8
*buf
= dctx
->buffer
;
132 ghash_flush(ctx
, dctx
);
133 memcpy(dst
, buf
, GHASH_BLOCK_SIZE
);
138 static struct shash_alg ghash_alg
= {
139 .digestsize
= GHASH_DIGEST_SIZE
,
141 .update
= ghash_update
,
142 .final
= ghash_final
,
143 .setkey
= ghash_setkey
,
144 .descsize
= sizeof(struct ghash_desc_ctx
),
146 .cra_name
= "__ghash",
147 .cra_driver_name
= "__ghash-pclmulqdqni",
149 .cra_flags
= CRYPTO_ALG_TYPE_SHASH
,
150 .cra_blocksize
= GHASH_BLOCK_SIZE
,
151 .cra_ctxsize
= sizeof(struct ghash_ctx
),
152 .cra_module
= THIS_MODULE
,
153 .cra_list
= LIST_HEAD_INIT(ghash_alg
.base
.cra_list
),
157 static int ghash_async_init(struct ahash_request
*req
)
159 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
160 struct ghash_async_ctx
*ctx
= crypto_ahash_ctx(tfm
);
161 struct ahash_request
*cryptd_req
= ahash_request_ctx(req
);
162 struct cryptd_ahash
*cryptd_tfm
= ctx
->cryptd_tfm
;
164 if (!irq_fpu_usable()) {
165 memcpy(cryptd_req
, req
, sizeof(*req
));
166 ahash_request_set_tfm(cryptd_req
, &cryptd_tfm
->base
);
167 return crypto_ahash_init(cryptd_req
);
169 struct shash_desc
*desc
= cryptd_shash_desc(cryptd_req
);
170 struct crypto_shash
*child
= cryptd_ahash_child(cryptd_tfm
);
173 desc
->flags
= req
->base
.flags
;
174 return crypto_shash_init(desc
);
178 static int ghash_async_update(struct ahash_request
*req
)
180 struct ahash_request
*cryptd_req
= ahash_request_ctx(req
);
182 if (!irq_fpu_usable()) {
183 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
184 struct ghash_async_ctx
*ctx
= crypto_ahash_ctx(tfm
);
185 struct cryptd_ahash
*cryptd_tfm
= ctx
->cryptd_tfm
;
187 memcpy(cryptd_req
, req
, sizeof(*req
));
188 ahash_request_set_tfm(cryptd_req
, &cryptd_tfm
->base
);
189 return crypto_ahash_update(cryptd_req
);
191 struct shash_desc
*desc
= cryptd_shash_desc(cryptd_req
);
192 return shash_ahash_update(req
, desc
);
196 static int ghash_async_final(struct ahash_request
*req
)
198 struct ahash_request
*cryptd_req
= ahash_request_ctx(req
);
200 if (!irq_fpu_usable()) {
201 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
202 struct ghash_async_ctx
*ctx
= crypto_ahash_ctx(tfm
);
203 struct cryptd_ahash
*cryptd_tfm
= ctx
->cryptd_tfm
;
205 memcpy(cryptd_req
, req
, sizeof(*req
));
206 ahash_request_set_tfm(cryptd_req
, &cryptd_tfm
->base
);
207 return crypto_ahash_final(cryptd_req
);
209 struct shash_desc
*desc
= cryptd_shash_desc(cryptd_req
);
210 return crypto_shash_final(desc
, req
->result
);
214 static int ghash_async_digest(struct ahash_request
*req
)
216 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
217 struct ghash_async_ctx
*ctx
= crypto_ahash_ctx(tfm
);
218 struct ahash_request
*cryptd_req
= ahash_request_ctx(req
);
219 struct cryptd_ahash
*cryptd_tfm
= ctx
->cryptd_tfm
;
221 if (!irq_fpu_usable()) {
222 memcpy(cryptd_req
, req
, sizeof(*req
));
223 ahash_request_set_tfm(cryptd_req
, &cryptd_tfm
->base
);
224 return crypto_ahash_digest(cryptd_req
);
226 struct shash_desc
*desc
= cryptd_shash_desc(cryptd_req
);
227 struct crypto_shash
*child
= cryptd_ahash_child(cryptd_tfm
);
230 desc
->flags
= req
->base
.flags
;
231 return shash_ahash_digest(req
, desc
);
235 static int ghash_async_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
238 struct ghash_async_ctx
*ctx
= crypto_ahash_ctx(tfm
);
239 struct crypto_ahash
*child
= &ctx
->cryptd_tfm
->base
;
242 crypto_ahash_clear_flags(child
, CRYPTO_TFM_REQ_MASK
);
243 crypto_ahash_set_flags(child
, crypto_ahash_get_flags(tfm
)
244 & CRYPTO_TFM_REQ_MASK
);
245 err
= crypto_ahash_setkey(child
, key
, keylen
);
246 crypto_ahash_set_flags(tfm
, crypto_ahash_get_flags(child
)
247 & CRYPTO_TFM_RES_MASK
);
252 static int ghash_async_init_tfm(struct crypto_tfm
*tfm
)
254 struct cryptd_ahash
*cryptd_tfm
;
255 struct ghash_async_ctx
*ctx
= crypto_tfm_ctx(tfm
);
257 cryptd_tfm
= cryptd_alloc_ahash("__ghash-pclmulqdqni", 0, 0);
258 if (IS_ERR(cryptd_tfm
))
259 return PTR_ERR(cryptd_tfm
);
260 ctx
->cryptd_tfm
= cryptd_tfm
;
261 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
262 sizeof(struct ahash_request
) +
263 crypto_ahash_reqsize(&cryptd_tfm
->base
));
268 static void ghash_async_exit_tfm(struct crypto_tfm
*tfm
)
270 struct ghash_async_ctx
*ctx
= crypto_tfm_ctx(tfm
);
272 cryptd_free_ahash(ctx
->cryptd_tfm
);
275 static struct ahash_alg ghash_async_alg
= {
276 .init
= ghash_async_init
,
277 .update
= ghash_async_update
,
278 .final
= ghash_async_final
,
279 .setkey
= ghash_async_setkey
,
280 .digest
= ghash_async_digest
,
282 .digestsize
= GHASH_DIGEST_SIZE
,
285 .cra_driver_name
= "ghash-clmulni",
287 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
| CRYPTO_ALG_ASYNC
,
288 .cra_blocksize
= GHASH_BLOCK_SIZE
,
289 .cra_type
= &crypto_ahash_type
,
290 .cra_module
= THIS_MODULE
,
291 .cra_list
= LIST_HEAD_INIT(ghash_async_alg
.halg
.base
.cra_list
),
292 .cra_init
= ghash_async_init_tfm
,
293 .cra_exit
= ghash_async_exit_tfm
,
298 static const struct x86_cpu_id pcmul_cpu_id
[] = {
299 X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ
), /* Pickle-Mickle-Duck */
302 MODULE_DEVICE_TABLE(x86cpu
, pcmul_cpu_id
);
304 static int __init
ghash_pclmulqdqni_mod_init(void)
308 if (!x86_match_cpu(pcmul_cpu_id
))
311 err
= crypto_register_shash(&ghash_alg
);
314 err
= crypto_register_ahash(&ghash_async_alg
);
321 crypto_unregister_shash(&ghash_alg
);
326 static void __exit
ghash_pclmulqdqni_mod_exit(void)
328 crypto_unregister_ahash(&ghash_async_alg
);
329 crypto_unregister_shash(&ghash_alg
);
332 module_init(ghash_pclmulqdqni_mod_init
);
333 module_exit(ghash_pclmulqdqni_mod_exit
);
335 MODULE_LICENSE("GPL");
336 MODULE_DESCRIPTION("GHASH Message Digest Algorithm, "
337 "acclerated by PCLMULQDQ-NI");
338 MODULE_ALIAS("ghash");