?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
crypto: caam - replace sec4_sg pointer with array
[linux-2.6/btrfs-unstable.git] / drivers / crypto / caam / caamhash.c
blobaaaa3724e1f899d39c33b7d9860109b5335aff11
1 /*
2 * caam - Freescale FSL CAAM support for ahash functions of crypto API
3 *
4 * Copyright 2011 Freescale Semiconductor, Inc.
5 *
6 * Based on caamalg.c crypto API driver.
7 *
8 * relationship of digest job descriptor or first job descriptor after init to
9 * shared descriptors:
10 *
11 * --------------- ---------------
12 * | JobDesc #1 |-------------------->| ShareDesc |
13 * | *(packet 1) | | (hashKey) |
14 * --------------- | (operation) |
15 * ---------------
16 *
17 * relationship of subsequent job descriptors to shared descriptors:
18 *
19 * --------------- ---------------
20 * | JobDesc #2 |-------------------->| ShareDesc |
21 * | *(packet 2) | |------------->| (hashKey) |
22 * --------------- | |-------->| (operation) |
23 * . | | | (load ctx2) |
24 * . | | ---------------
25 * --------------- | |
26 * | JobDesc #3 |------| |
27 * | *(packet 3) | |
28 * --------------- |
29 * . |
30 * . |
31 * --------------- |
32 * | JobDesc #4 |------------
33 * | *(packet 4) |
34 * ---------------
35 *
36 * The SharedDesc never changes for a connection unless rekeyed, but
37 * each packet will likely be in a different place. So all we need
38 * to know to process the packet is where the input is, where the
39 * output goes, and what context we want to process with. Context is
40 * in the SharedDesc, packet references in the JobDesc.
41 *
42 * So, a job desc looks like:
43 *
44 * ---------------------
45 * | Header |
46 * | ShareDesc Pointer |
47 * | SEQ_OUT_PTR |
48 * | (output buffer) |
49 * | (output length) |
50 * | SEQ_IN_PTR |
51 * | (input buffer) |
52 * | (input length) |
53 * ---------------------
54 */
55
56 #include "compat.h"
57
58 #include "regs.h"
59 #include "intern.h"
60 #include "desc_constr.h"
61 #include "jr.h"
62 #include "error.h"
63 #include "sg_sw_sec4.h"
64 #include "key_gen.h"
65
66 #define CAAM_CRA_PRIORITY 3000
67
68 /* max hash key is max split key size */
69 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2)
70
71 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
72 #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE
73
74 /* length of descriptors text */
75 #define DESC_AHASH_BASE (4 * CAAM_CMD_SZ)
76 #define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ)
77 #define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
78 #define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
79 #define DESC_AHASH_FINUP_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
80 #define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
81
82 #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \
83 CAAM_MAX_HASH_KEY_SIZE)
84 #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
85
86 /* caam context sizes for hashes: running digest + 8 */
87 #define HASH_MSG_LEN 8
88 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
89
90 #ifdef DEBUG
91 /* for print_hex_dumps with line references */
92 #define debug(format, arg...) printk(format, arg)
93 #else
94 #define debug(format, arg...)
95 #endif
96
97
98 static struct list_head hash_list;
99
100 /* ahash per-session context */
101 struct caam_hash_ctx {
102 u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
103 u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
104 u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
105 u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
106 u32 sh_desc_finup[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
107 dma_addr_t sh_desc_update_dma ____cacheline_aligned;
108 dma_addr_t sh_desc_update_first_dma;
109 dma_addr_t sh_desc_fin_dma;
110 dma_addr_t sh_desc_digest_dma;
111 dma_addr_t sh_desc_finup_dma;
112 struct device *jrdev;
113 u32 alg_type;
114 u32 alg_op;
115 u8 key[CAAM_MAX_HASH_KEY_SIZE];
116 dma_addr_t key_dma;
117 int ctx_len;
118 unsigned int split_key_len;
119 unsigned int split_key_pad_len;
120 };
121
122 /* ahash state */
123 struct caam_hash_state {
124 dma_addr_t buf_dma;
125 dma_addr_t ctx_dma;
126 u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
127 int buflen_0;
128 u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
129 int buflen_1;
130 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
131 int (*update)(struct ahash_request *req);
132 int (*final)(struct ahash_request *req);
133 int (*finup)(struct ahash_request *req);
134 int current_buf;
135 };
136
137 struct caam_export_state {
138 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
139 u8 caam_ctx[MAX_CTX_LEN];
140 int buflen;
141 int (*update)(struct ahash_request *req);
142 int (*final)(struct ahash_request *req);
143 int (*finup)(struct ahash_request *req);
144 };
145
146 /* Common job descriptor seq in/out ptr routines */
147
148 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */
149 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
150 struct caam_hash_state *state,
151 int ctx_len)
152 {
153 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx,
154 ctx_len, DMA_FROM_DEVICE);
155 if (dma_mapping_error(jrdev, state->ctx_dma)) {
156 dev_err(jrdev, "unable to map ctx\n");
157 return -ENOMEM;
158 }
159
160 append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0);
161
162 return 0;
163 }
164
165 /* Map req->result, and append seq_out_ptr command that points to it */
166 static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev,
167 u8 *result, int digestsize)
168 {
169 dma_addr_t dst_dma;
170
171 dst_dma = dma_map_single(jrdev, result, digestsize, DMA_FROM_DEVICE);
172 append_seq_out_ptr(desc, dst_dma, digestsize, 0);
173
174 return dst_dma;
175 }
176
177 /* Map current buffer in state and put it in link table */
178 static inline dma_addr_t buf_map_to_sec4_sg(struct device *jrdev,
179 struct sec4_sg_entry *sec4_sg,
180 u8 *buf, int buflen)
181 {
182 dma_addr_t buf_dma;
183
184 buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE);
185 dma_to_sec4_sg_one(sec4_sg, buf_dma, buflen, 0);
186
187 return buf_dma;
188 }
189
190 /* Map req->src and put it in link table */
191 static inline void src_map_to_sec4_sg(struct device *jrdev,
192 struct scatterlist *src, int src_nents,
193 struct sec4_sg_entry *sec4_sg)
194 {
195 dma_map_sg(jrdev, src, src_nents, DMA_TO_DEVICE);
196 sg_to_sec4_sg_last(src, src_nents, sec4_sg, 0);
197 }
198
199 /*
200 * Only put buffer in link table if it contains data, which is possible,
201 * since a buffer has previously been used, and needs to be unmapped,
202 */
203 static inline dma_addr_t
204 try_buf_map_to_sec4_sg(struct device *jrdev, struct sec4_sg_entry *sec4_sg,
205 u8 *buf, dma_addr_t buf_dma, int buflen,
206 int last_buflen)
207 {
208 if (buf_dma && !dma_mapping_error(jrdev, buf_dma))
209 dma_unmap_single(jrdev, buf_dma, last_buflen, DMA_TO_DEVICE);
210 if (buflen)
211 buf_dma = buf_map_to_sec4_sg(jrdev, sec4_sg, buf, buflen);
212 else
213 buf_dma = 0;
214
215 return buf_dma;
216 }
217
218 /* Map state->caam_ctx, and add it to link table */
219 static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev,
220 struct caam_hash_state *state, int ctx_len,
221 struct sec4_sg_entry *sec4_sg, u32 flag)
222 {
223 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);
224 if (dma_mapping_error(jrdev, state->ctx_dma)) {
225 dev_err(jrdev, "unable to map ctx\n");
226 return -ENOMEM;
227 }
228
229 dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0);
230
231 return 0;
232 }
233
234 /* Common shared descriptor commands */
235 static inline void append_key_ahash(u32 *desc, struct caam_hash_ctx *ctx)
236 {
237 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
238 ctx->split_key_len, CLASS_2 |
239 KEY_DEST_MDHA_SPLIT | KEY_ENC);
240 }
241
242 /* Append key if it has been set */
243 static inline void init_sh_desc_key_ahash(u32 *desc, struct caam_hash_ctx *ctx)
244 {
245 u32 *key_jump_cmd;
246
247 init_sh_desc(desc, HDR_SHARE_SERIAL);
248
249 if (ctx->split_key_len) {
250 /* Skip if already shared */
251 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
252 JUMP_COND_SHRD);
253
254 append_key_ahash(desc, ctx);
255
256 set_jump_tgt_here(desc, key_jump_cmd);
257 }
258
259 /* Propagate errors from shared to job descriptor */
260 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
261 }
262
263 /*
264 * For ahash read data from seqin following state->caam_ctx,
265 * and write resulting class2 context to seqout, which may be state->caam_ctx
266 * or req->result
267 */
268 static inline void ahash_append_load_str(u32 *desc, int digestsize)
269 {
270 /* Calculate remaining bytes to read */
271 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
272
273 /* Read remaining bytes */
274 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 |
275 FIFOLD_TYPE_MSG | KEY_VLF);
276
277 /* Store class2 context bytes */
278 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
279 LDST_SRCDST_BYTE_CONTEXT);
280 }
281
282 /*
283 * For ahash update, final and finup, import context, read and write to seqout
284 */
285 static inline void ahash_ctx_data_to_out(u32 *desc, u32 op, u32 state,
286 int digestsize,
287 struct caam_hash_ctx *ctx)
288 {
289 init_sh_desc_key_ahash(desc, ctx);
290
291 /* Import context from software */
292 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
293 LDST_CLASS_2_CCB | ctx->ctx_len);
294
295 /* Class 2 operation */
296 append_operation(desc, op | state | OP_ALG_ENCRYPT);
297
298 /*
299 * Load from buf and/or src and write to req->result or state->context
300 */
301 ahash_append_load_str(desc, digestsize);
302 }
303
304 /* For ahash firsts and digest, read and write to seqout */
305 static inline void ahash_data_to_out(u32 *desc, u32 op, u32 state,
306 int digestsize, struct caam_hash_ctx *ctx)
307 {
308 init_sh_desc_key_ahash(desc, ctx);
309
310 /* Class 2 operation */
311 append_operation(desc, op | state | OP_ALG_ENCRYPT);
312
313 /*
314 * Load from buf and/or src and write to req->result or state->context
315 */
316 ahash_append_load_str(desc, digestsize);
317 }
318
319 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
320 {
321 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
322 int digestsize = crypto_ahash_digestsize(ahash);
323 struct device *jrdev = ctx->jrdev;
324 u32 have_key = 0;
325 u32 *desc;
326
327 if (ctx->split_key_len)
328 have_key = OP_ALG_AAI_HMAC_PRECOMP;
329
330 /* ahash_update shared descriptor */
331 desc = ctx->sh_desc_update;
332
333 init_sh_desc(desc, HDR_SHARE_SERIAL);
334
335 /* Import context from software */
336 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
337 LDST_CLASS_2_CCB | ctx->ctx_len);
338
339 /* Class 2 operation */
340 append_operation(desc, ctx->alg_type | OP_ALG_AS_UPDATE |
341 OP_ALG_ENCRYPT);
342
343 /* Load data and write to result or context */
344 ahash_append_load_str(desc, ctx->ctx_len);
345
346 ctx->sh_desc_update_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
347 DMA_TO_DEVICE);
348 if (dma_mapping_error(jrdev, ctx->sh_desc_update_dma)) {
349 dev_err(jrdev, "unable to map shared descriptor\n");
350 return -ENOMEM;
351 }
352 #ifdef DEBUG
353 print_hex_dump(KERN_ERR,
354 "ahash update shdesc@"__stringify(__LINE__)": ",
355 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
356 #endif
357
358 /* ahash_update_first shared descriptor */
359 desc = ctx->sh_desc_update_first;
360
361 ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INIT,
362 ctx->ctx_len, ctx);
363
364 ctx->sh_desc_update_first_dma = dma_map_single(jrdev, desc,
365 desc_bytes(desc),
366 DMA_TO_DEVICE);
367 if (dma_mapping_error(jrdev, ctx->sh_desc_update_first_dma)) {
368 dev_err(jrdev, "unable to map shared descriptor\n");
369 return -ENOMEM;
370 }
371 #ifdef DEBUG
372 print_hex_dump(KERN_ERR,
373 "ahash update first shdesc@"__stringify(__LINE__)": ",
374 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
375 #endif
376
377 /* ahash_final shared descriptor */
378 desc = ctx->sh_desc_fin;
379
380 ahash_ctx_data_to_out(desc, have_key | ctx->alg_type,
381 OP_ALG_AS_FINALIZE, digestsize, ctx);
382
383 ctx->sh_desc_fin_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
384 DMA_TO_DEVICE);
385 if (dma_mapping_error(jrdev, ctx->sh_desc_fin_dma)) {
386 dev_err(jrdev, "unable to map shared descriptor\n");
387 return -ENOMEM;
388 }
389 #ifdef DEBUG
390 print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
391 DUMP_PREFIX_ADDRESS, 16, 4, desc,
392 desc_bytes(desc), 1);
393 #endif
394
395 /* ahash_finup shared descriptor */
396 desc = ctx->sh_desc_finup;
397
398 ahash_ctx_data_to_out(desc, have_key | ctx->alg_type,
399 OP_ALG_AS_FINALIZE, digestsize, ctx);
400
401 ctx->sh_desc_finup_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
402 DMA_TO_DEVICE);
403 if (dma_mapping_error(jrdev, ctx->sh_desc_finup_dma)) {
404 dev_err(jrdev, "unable to map shared descriptor\n");
405 return -ENOMEM;
406 }
407 #ifdef DEBUG
408 print_hex_dump(KERN_ERR, "ahash finup shdesc@"__stringify(__LINE__)": ",
409 DUMP_PREFIX_ADDRESS, 16, 4, desc,
410 desc_bytes(desc), 1);
411 #endif
412
413 /* ahash_digest shared descriptor */
414 desc = ctx->sh_desc_digest;
415
416 ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INITFINAL,
417 digestsize, ctx);
418
419 ctx->sh_desc_digest_dma = dma_map_single(jrdev, desc,
420 desc_bytes(desc),
421 DMA_TO_DEVICE);
422 if (dma_mapping_error(jrdev, ctx->sh_desc_digest_dma)) {
423 dev_err(jrdev, "unable to map shared descriptor\n");
424 return -ENOMEM;
425 }
426 #ifdef DEBUG
427 print_hex_dump(KERN_ERR,
428 "ahash digest shdesc@"__stringify(__LINE__)": ",
429 DUMP_PREFIX_ADDRESS, 16, 4, desc,
430 desc_bytes(desc), 1);
431 #endif
432
433 return 0;
434 }
435
436 static int gen_split_hash_key(struct caam_hash_ctx *ctx, const u8 *key_in,
437 u32 keylen)
438 {
439 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
440 ctx->split_key_pad_len, key_in, keylen,
441 ctx->alg_op);
442 }
443
444 /* Digest hash size if it is too large */
445 static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
446 u32 *keylen, u8 *key_out, u32 digestsize)
447 {
448 struct device *jrdev = ctx->jrdev;
449 u32 *desc;
450 struct split_key_result result;
451 dma_addr_t src_dma, dst_dma;
452 int ret = 0;
453
454 desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
455 if (!desc) {
456 dev_err(jrdev, "unable to allocate key input memory\n");
457 return -ENOMEM;
458 }
459
460 init_job_desc(desc, 0);
461
462 src_dma = dma_map_single(jrdev, (void *)key_in, *keylen,
463 DMA_TO_DEVICE);
464 if (dma_mapping_error(jrdev, src_dma)) {
465 dev_err(jrdev, "unable to map key input memory\n");
466 kfree(desc);
467 return -ENOMEM;
468 }
469 dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize,
470 DMA_FROM_DEVICE);
471 if (dma_mapping_error(jrdev, dst_dma)) {
472 dev_err(jrdev, "unable to map key output memory\n");
473 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
474 kfree(desc);
475 return -ENOMEM;
476 }
477
478 /* Job descriptor to perform unkeyed hash on key_in */
479 append_operation(desc, ctx->alg_type | OP_ALG_ENCRYPT |
480 OP_ALG_AS_INITFINAL);
481 append_seq_in_ptr(desc, src_dma, *keylen, 0);
482 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
483 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
484 append_seq_out_ptr(desc, dst_dma, digestsize, 0);
485 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
486 LDST_SRCDST_BYTE_CONTEXT);
487
488 #ifdef DEBUG
489 print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
490 DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1);
491 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
492 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
493 #endif
494
495 result.err = 0;
496 init_completion(&result.completion);
497
498 ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
499 if (!ret) {
500 /* in progress */
501 wait_for_completion_interruptible(&result.completion);
502 ret = result.err;
503 #ifdef DEBUG
504 print_hex_dump(KERN_ERR,
505 "digested key@"__stringify(__LINE__)": ",
506 DUMP_PREFIX_ADDRESS, 16, 4, key_in,
507 digestsize, 1);
508 #endif
509 }
510 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
511 dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE);
512
513 *keylen = digestsize;
514
515 kfree(desc);
516
517 return ret;
518 }
519
520 static int ahash_setkey(struct crypto_ahash *ahash,
521 const u8 *key, unsigned int keylen)
522 {
523 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
524 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
525 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
526 struct device *jrdev = ctx->jrdev;
527 int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
528 int digestsize = crypto_ahash_digestsize(ahash);
529 int ret = 0;
530 u8 *hashed_key = NULL;
531
532 #ifdef DEBUG
533 printk(KERN_ERR "keylen %d\n", keylen);
534 #endif
535
536 if (keylen > blocksize) {
537 hashed_key = kmalloc(sizeof(u8) * digestsize, GFP_KERNEL |
538 GFP_DMA);
539 if (!hashed_key)
540 return -ENOMEM;
541 ret = hash_digest_key(ctx, key, &keylen, hashed_key,
542 digestsize);
543 if (ret)
544 goto badkey;
545 key = hashed_key;
546 }
547
548 /* Pick class 2 key length from algorithm submask */
549 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
550 OP_ALG_ALGSEL_SHIFT] * 2;
551 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
552
553 #ifdef DEBUG
554 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
555 ctx->split_key_len, ctx->split_key_pad_len);
556 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
557 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
558 #endif
559
560 ret = gen_split_hash_key(ctx, key, keylen);
561 if (ret)
562 goto badkey;
563
564 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len,
565 DMA_TO_DEVICE);
566 if (dma_mapping_error(jrdev, ctx->key_dma)) {
567 dev_err(jrdev, "unable to map key i/o memory\n");
568 ret = -ENOMEM;
569 goto map_err;
570 }
571 #ifdef DEBUG
572 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
573 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
574 ctx->split_key_pad_len, 1);
575 #endif
576
577 ret = ahash_set_sh_desc(ahash);
578 if (ret) {
579 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len,
580 DMA_TO_DEVICE);
581 }
582
583 map_err:
584 kfree(hashed_key);
585 return ret;
586 badkey:
587 kfree(hashed_key);
588 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
589 return -EINVAL;
590 }
591
592 /*
593 * ahash_edesc - s/w-extended ahash descriptor
594 * @dst_dma: physical mapped address of req->result
595 * @sec4_sg_dma: physical mapped address of h/w link table
596 * @src_nents: number of segments in input scatterlist
597 * @sec4_sg_bytes: length of dma mapped sec4_sg space
598 * @hw_desc: the h/w job descriptor followed by any referenced link tables
599 * @sec4_sg: h/w link table
600 */
601 struct ahash_edesc {
602 dma_addr_t dst_dma;
603 dma_addr_t sec4_sg_dma;
604 int src_nents;
605 int sec4_sg_bytes;
606 u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned;
607 struct sec4_sg_entry sec4_sg[0];
608 };
609
610 static inline void ahash_unmap(struct device *dev,
611 struct ahash_edesc *edesc,
612 struct ahash_request *req, int dst_len)
613 {
614 if (edesc->src_nents)
615 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
616 if (edesc->dst_dma)
617 dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);
618
619 if (edesc->sec4_sg_bytes)
620 dma_unmap_single(dev, edesc->sec4_sg_dma,
621 edesc->sec4_sg_bytes, DMA_TO_DEVICE);
622 }
623
624 static inline void ahash_unmap_ctx(struct device *dev,
625 struct ahash_edesc *edesc,
626 struct ahash_request *req, int dst_len, u32 flag)
627 {
628 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
629 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
630 struct caam_hash_state *state = ahash_request_ctx(req);
631
632 if (state->ctx_dma)
633 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag);
634 ahash_unmap(dev, edesc, req, dst_len);
635 }
636
637 static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
638 void *context)
639 {
640 struct ahash_request *req = context;
641 struct ahash_edesc *edesc;
642 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
643 int digestsize = crypto_ahash_digestsize(ahash);
644 #ifdef DEBUG
645 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
646 struct caam_hash_state *state = ahash_request_ctx(req);
647
648 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
649 #endif
650
651 edesc = (struct ahash_edesc *)((char *)desc -
652 offsetof(struct ahash_edesc, hw_desc));
653 if (err)
654 caam_jr_strstatus(jrdev, err);
655
656 ahash_unmap(jrdev, edesc, req, digestsize);
657 kfree(edesc);
658
659 #ifdef DEBUG
660 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
661 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
662 ctx->ctx_len, 1);
663 if (req->result)
664 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
665 DUMP_PREFIX_ADDRESS, 16, 4, req->result,
666 digestsize, 1);
667 #endif
668
669 req->base.complete(&req->base, err);
670 }
671
672 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
673 void *context)
674 {
675 struct ahash_request *req = context;
676 struct ahash_edesc *edesc;
677 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
678 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
679 #ifdef DEBUG
680 struct caam_hash_state *state = ahash_request_ctx(req);
681 int digestsize = crypto_ahash_digestsize(ahash);
682
683 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
684 #endif
685
686 edesc = (struct ahash_edesc *)((char *)desc -
687 offsetof(struct ahash_edesc, hw_desc));
688 if (err)
689 caam_jr_strstatus(jrdev, err);
690
691 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
692 kfree(edesc);
693
694 #ifdef DEBUG
695 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
696 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
697 ctx->ctx_len, 1);
698 if (req->result)
699 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
700 DUMP_PREFIX_ADDRESS, 16, 4, req->result,
701 digestsize, 1);
702 #endif
703
704 req->base.complete(&req->base, err);
705 }
706
707 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
708 void *context)
709 {
710 struct ahash_request *req = context;
711 struct ahash_edesc *edesc;
712 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
713 int digestsize = crypto_ahash_digestsize(ahash);
714 #ifdef DEBUG
715 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
716 struct caam_hash_state *state = ahash_request_ctx(req);
717
718 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
719 #endif
720
721 edesc = (struct ahash_edesc *)((char *)desc -
722 offsetof(struct ahash_edesc, hw_desc));
723 if (err)
724 caam_jr_strstatus(jrdev, err);
725
726 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE);
727 kfree(edesc);
728
729 #ifdef DEBUG
730 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
731 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
732 ctx->ctx_len, 1);
733 if (req->result)
734 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
735 DUMP_PREFIX_ADDRESS, 16, 4, req->result,
736 digestsize, 1);
737 #endif
738
739 req->base.complete(&req->base, err);
740 }
741
742 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
743 void *context)
744 {
745 struct ahash_request *req = context;
746 struct ahash_edesc *edesc;
747 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
748 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
749 #ifdef DEBUG
750 struct caam_hash_state *state = ahash_request_ctx(req);
751 int digestsize = crypto_ahash_digestsize(ahash);
752
753 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
754 #endif
755
756 edesc = (struct ahash_edesc *)((char *)desc -
757 offsetof(struct ahash_edesc, hw_desc));
758 if (err)
759 caam_jr_strstatus(jrdev, err);
760
761 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
762 kfree(edesc);
763
764 #ifdef DEBUG
765 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
766 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
767 ctx->ctx_len, 1);
768 if (req->result)
769 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
770 DUMP_PREFIX_ADDRESS, 16, 4, req->result,
771 digestsize, 1);
772 #endif
773
774 req->base.complete(&req->base, err);
775 }
776
777 /* submit update job descriptor */
778 static int ahash_update_ctx(struct ahash_request *req)
779 {
780 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
781 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
782 struct caam_hash_state *state = ahash_request_ctx(req);
783 struct device *jrdev = ctx->jrdev;
784 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
785 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
786 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
787 int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0;
788 u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1;
789 int *next_buflen = state->current_buf ? &state->buflen_0 :
790 &state->buflen_1, last_buflen;
791 int in_len = *buflen + req->nbytes, to_hash;
792 u32 *sh_desc = ctx->sh_desc_update, *desc;
793 dma_addr_t ptr = ctx->sh_desc_update_dma;
794 int src_nents, sec4_sg_bytes, sec4_sg_src_index;
795 struct ahash_edesc *edesc;
796 int ret = 0;
797 int sh_len;
798
799 last_buflen = *next_buflen;
800 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
801 to_hash = in_len - *next_buflen;
802
803 if (to_hash) {
804 src_nents = sg_nents_for_len(req->src,
805 req->nbytes - (*next_buflen));
806 if (src_nents < 0) {
807 dev_err(jrdev, "Invalid number of src SG.\n");
808 return src_nents;
809 }
810 sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
811 sec4_sg_bytes = (sec4_sg_src_index + src_nents) *
812 sizeof(struct sec4_sg_entry);
813
814 /*
815 * allocate space for base edesc and hw desc commands,
816 * link tables
817 */
818 edesc = kzalloc(sizeof(*edesc) + sec4_sg_bytes,
819 GFP_DMA | flags);
820 if (!edesc) {
821 dev_err(jrdev,
822 "could not allocate extended descriptor\n");
823 return -ENOMEM;
824 }
825
826 edesc->src_nents = src_nents;
827 edesc->sec4_sg_bytes = sec4_sg_bytes;
828
829 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
830 edesc->sec4_sg, DMA_BIDIRECTIONAL);
831 if (ret)
832 return ret;
833
834 state->buf_dma = try_buf_map_to_sec4_sg(jrdev,
835 edesc->sec4_sg + 1,
836 buf, state->buf_dma,
837 *buflen, last_buflen);
838
839 if (src_nents) {
840 src_map_to_sec4_sg(jrdev, req->src, src_nents,
841 edesc->sec4_sg + sec4_sg_src_index);
842 if (*next_buflen)
843 scatterwalk_map_and_copy(next_buf, req->src,
844 to_hash - *buflen,
845 *next_buflen, 0);
846 } else {
847 (edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
848 cpu_to_caam32(SEC4_SG_LEN_FIN);
849 }
850
851 state->current_buf = !state->current_buf;
852
853 sh_len = desc_len(sh_desc);
854 desc = edesc->hw_desc;
855 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER |
856 HDR_REVERSE);
857
858 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
859 sec4_sg_bytes,
860 DMA_TO_DEVICE);
861 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
862 dev_err(jrdev, "unable to map S/G table\n");
863 return -ENOMEM;
864 }
865
866 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
867 to_hash, LDST_SGF);
868
869 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
870
871 #ifdef DEBUG
872 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
873 DUMP_PREFIX_ADDRESS, 16, 4, desc,
874 desc_bytes(desc), 1);
875 #endif
876
877 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req);
878 if (!ret) {
879 ret = -EINPROGRESS;
880 } else {
881 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len,
882 DMA_BIDIRECTIONAL);
883 kfree(edesc);
884 }
885 } else if (*next_buflen) {
886 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
887 req->nbytes, 0);
888 *buflen = *next_buflen;
889 *next_buflen = last_buflen;
890 }
891 #ifdef DEBUG
892 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
893 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
894 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
895 DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
896 *next_buflen, 1);
897 #endif
898
899 return ret;
900 }
901
902 static int ahash_final_ctx(struct ahash_request *req)
903 {
904 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
905 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
906 struct caam_hash_state *state = ahash_request_ctx(req);
907 struct device *jrdev = ctx->jrdev;
908 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
909 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
910 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
911 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
912 int last_buflen = state->current_buf ? state->buflen_0 :
913 state->buflen_1;
914 u32 *sh_desc = ctx->sh_desc_fin, *desc;
915 dma_addr_t ptr = ctx->sh_desc_fin_dma;
916 int sec4_sg_bytes, sec4_sg_src_index;
917 int digestsize = crypto_ahash_digestsize(ahash);
918 struct ahash_edesc *edesc;
919 int ret = 0;
920 int sh_len;
921
922 sec4_sg_src_index = 1 + (buflen ? 1 : 0);
923 sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry);
924
925 /* allocate space for base edesc and hw desc commands, link tables */
926 edesc = kzalloc(sizeof(*edesc) + sec4_sg_bytes, GFP_DMA | flags);
927 if (!edesc) {
928 dev_err(jrdev, "could not allocate extended descriptor\n");
929 return -ENOMEM;
930 }
931
932 sh_len = desc_len(sh_desc);
933 desc = edesc->hw_desc;
934 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
935
936 edesc->sec4_sg_bytes = sec4_sg_bytes;
937 edesc->src_nents = 0;
938
939 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
940 edesc->sec4_sg, DMA_TO_DEVICE);
941 if (ret)
942 return ret;
943
944 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,
945 buf, state->buf_dma, buflen,
946 last_buflen);
947 (edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
948 cpu_to_caam32(SEC4_SG_LEN_FIN);
949
950 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
951 sec4_sg_bytes, DMA_TO_DEVICE);
952 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
953 dev_err(jrdev, "unable to map S/G table\n");
954 return -ENOMEM;
955 }
956
957 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,
958 LDST_SGF);
959
960 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
961 digestsize);
962 if (dma_mapping_error(jrdev, edesc->dst_dma)) {
963 dev_err(jrdev, "unable to map dst\n");
964 return -ENOMEM;
965 }
966
967 #ifdef DEBUG
968 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
969 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
970 #endif
971
972 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
973 if (!ret) {
974 ret = -EINPROGRESS;
975 } else {
976 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
977 kfree(edesc);
978 }
979
980 return ret;
981 }
982
983 static int ahash_finup_ctx(struct ahash_request *req)
984 {
985 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
986 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
987 struct caam_hash_state *state = ahash_request_ctx(req);
988 struct device *jrdev = ctx->jrdev;
989 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
990 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
991 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
992 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
993 int last_buflen = state->current_buf ? state->buflen_0 :
994 state->buflen_1;
995 u32 *sh_desc = ctx->sh_desc_finup, *desc;
996 dma_addr_t ptr = ctx->sh_desc_finup_dma;
997 int sec4_sg_bytes, sec4_sg_src_index;
998 int src_nents;
999 int digestsize = crypto_ahash_digestsize(ahash);
1000 struct ahash_edesc *edesc;
1001 int ret = 0;
1002 int sh_len;
1003
1004 src_nents = sg_nents_for_len(req->src, req->nbytes);
1005 if (src_nents < 0) {
1006 dev_err(jrdev, "Invalid number of src SG.\n");
1007 return src_nents;
1008 }
1009 sec4_sg_src_index = 1 + (buflen ? 1 : 0);
1010 sec4_sg_bytes = (sec4_sg_src_index + src_nents) *
1011 sizeof(struct sec4_sg_entry);
1012
1013 /* allocate space for base edesc and hw desc commands, link tables */
1014 edesc = kzalloc(sizeof(*edesc) + sec4_sg_bytes, GFP_DMA | flags);
1015 if (!edesc) {
1016 dev_err(jrdev, "could not allocate extended descriptor\n");
1017 return -ENOMEM;
1018 }
1019
1020 sh_len = desc_len(sh_desc);
1021 desc = edesc->hw_desc;
1022 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
1023
1024 edesc->src_nents = src_nents;
1025 edesc->sec4_sg_bytes = sec4_sg_bytes;
1026
1027 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
1028 edesc->sec4_sg, DMA_TO_DEVICE);
1029 if (ret)
1030 return ret;
1031
1032 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,
1033 buf, state->buf_dma, buflen,
1034 last_buflen);
1035
1036 src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg +
1037 sec4_sg_src_index);
1038
1039 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1040 sec4_sg_bytes, DMA_TO_DEVICE);
1041 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1042 dev_err(jrdev, "unable to map S/G table\n");
1043 return -ENOMEM;
1044 }
1045
1046 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
1047 buflen + req->nbytes, LDST_SGF);
1048
1049 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1050 digestsize);
1051 if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1052 dev_err(jrdev, "unable to map dst\n");
1053 return -ENOMEM;
1054 }
1055
1056 #ifdef DEBUG
1057 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1058 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1059 #endif
1060
1061 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
1062 if (!ret) {
1063 ret = -EINPROGRESS;
1064 } else {
1065 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
1066 kfree(edesc);
1067 }
1068
1069 return ret;
1070 }
1071
1072 static int ahash_digest(struct ahash_request *req)
1073 {
1074 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1075 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1076 struct device *jrdev = ctx->jrdev;
1077 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1078 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1079 u32 *sh_desc = ctx->sh_desc_digest, *desc;
1080 dma_addr_t ptr = ctx->sh_desc_digest_dma;
1081 int digestsize = crypto_ahash_digestsize(ahash);
1082 int src_nents, sec4_sg_bytes;
1083 dma_addr_t src_dma;
1084 struct ahash_edesc *edesc;
1085 int ret = 0;
1086 u32 options;
1087 int sh_len;
1088
1089 src_nents = sg_nents_for_len(req->src, req->nbytes);
1090 if (src_nents < 0) {
1091 dev_err(jrdev, "Invalid number of src SG.\n");
1092 return src_nents;
1093 }
1094 dma_map_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1095 if (src_nents > 1)
1096 sec4_sg_bytes = src_nents * sizeof(struct sec4_sg_entry);
1097 else
1098 sec4_sg_bytes = 0;
1099
1100 /* allocate space for base edesc and hw desc commands, link tables */
1101 edesc = kzalloc(sizeof(*edesc) + sec4_sg_bytes, GFP_DMA | flags);
1102 if (!edesc) {
1103 dev_err(jrdev, "could not allocate extended descriptor\n");
1104 return -ENOMEM;
1105 }
1106
1107 edesc->sec4_sg_bytes = sec4_sg_bytes;
1108 edesc->src_nents = src_nents;
1109
1110 sh_len = desc_len(sh_desc);
1111 desc = edesc->hw_desc;
1112 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
1113
1114 if (src_nents > 1) {
1115 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
1116 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1117 sec4_sg_bytes, DMA_TO_DEVICE);
1118 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1119 dev_err(jrdev, "unable to map S/G table\n");
1120 return -ENOMEM;
1121 }
1122 src_dma = edesc->sec4_sg_dma;
1123 options = LDST_SGF;
1124 } else {
1125 src_dma = sg_dma_address(req->src);
1126 options = 0;
1127 }
1128 append_seq_in_ptr(desc, src_dma, req->nbytes, options);
1129
1130 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1131 digestsize);
1132 if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1133 dev_err(jrdev, "unable to map dst\n");
1134 return -ENOMEM;
1135 }
1136
1137 #ifdef DEBUG
1138 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1139 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1140 #endif
1141
1142 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1143 if (!ret) {
1144 ret = -EINPROGRESS;
1145 } else {
1146 ahash_unmap(jrdev, edesc, req, digestsize);
1147 kfree(edesc);
1148 }
1149
1150 return ret;
1151 }
1152
1153 /* submit ahash final if it the first job descriptor */
1154 static int ahash_final_no_ctx(struct ahash_request *req)
1155 {
1156 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1157 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1158 struct caam_hash_state *state = ahash_request_ctx(req);
1159 struct device *jrdev = ctx->jrdev;
1160 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1161 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1162 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
1163 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
1164 u32 *sh_desc = ctx->sh_desc_digest, *desc;
1165 dma_addr_t ptr = ctx->sh_desc_digest_dma;
1166 int digestsize = crypto_ahash_digestsize(ahash);
1167 struct ahash_edesc *edesc;
1168 int ret = 0;
1169 int sh_len;
1170
1171 /* allocate space for base edesc and hw desc commands, link tables */
1172 edesc = kzalloc(sizeof(*edesc), GFP_DMA | flags);
1173 if (!edesc) {
1174 dev_err(jrdev, "could not allocate extended descriptor\n");
1175 return -ENOMEM;
1176 }
1177
1178 edesc->sec4_sg_bytes = 0;
1179 sh_len = desc_len(sh_desc);
1180 desc = edesc->hw_desc;
1181 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
1182
1183 state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE);
1184 if (dma_mapping_error(jrdev, state->buf_dma)) {
1185 dev_err(jrdev, "unable to map src\n");
1186 return -ENOMEM;
1187 }
1188
1189 append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
1190
1191 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1192 digestsize);
1193 if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1194 dev_err(jrdev, "unable to map dst\n");
1195 return -ENOMEM;
1196 }
1197 edesc->src_nents = 0;
1198
1199 #ifdef DEBUG
1200 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1201 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1202 #endif
1203
1204 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1205 if (!ret) {
1206 ret = -EINPROGRESS;
1207 } else {
1208 ahash_unmap(jrdev, edesc, req, digestsize);
1209 kfree(edesc);
1210 }
1211
1212 return ret;
1213 }
1214
1215 /* submit ahash update if it the first job descriptor after update */
1216 static int ahash_update_no_ctx(struct ahash_request *req)
1217 {
1218 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1219 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1220 struct caam_hash_state *state = ahash_request_ctx(req);
1221 struct device *jrdev = ctx->jrdev;
1222 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1223 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1224 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
1225 int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0;
1226 u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1;
1227 int *next_buflen = state->current_buf ? &state->buflen_0 :
1228 &state->buflen_1;
1229 int in_len = *buflen + req->nbytes, to_hash;
1230 int sec4_sg_bytes, src_nents;
1231 struct ahash_edesc *edesc;
1232 u32 *desc, *sh_desc = ctx->sh_desc_update_first;
1233 dma_addr_t ptr = ctx->sh_desc_update_first_dma;
1234 int ret = 0;
1235 int sh_len;
1236
1237 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
1238 to_hash = in_len - *next_buflen;
1239
1240 if (to_hash) {
1241 src_nents = sg_nents_for_len(req->src,
1242 req->nbytes - *next_buflen);
1243 if (src_nents < 0) {
1244 dev_err(jrdev, "Invalid number of src SG.\n");
1245 return src_nents;
1246 }
1247 sec4_sg_bytes = (1 + src_nents) *
1248 sizeof(struct sec4_sg_entry);
1249
1250 /*
1251 * allocate space for base edesc and hw desc commands,
1252 * link tables
1253 */
1254 edesc = kzalloc(sizeof(*edesc) + sec4_sg_bytes,
1255 GFP_DMA | flags);
1256 if (!edesc) {
1257 dev_err(jrdev,
1258 "could not allocate extended descriptor\n");
1259 return -ENOMEM;
1260 }
1261
1262 edesc->src_nents = src_nents;
1263 edesc->sec4_sg_bytes = sec4_sg_bytes;
1264 edesc->dst_dma = 0;
1265
1266 state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg,
1267 buf, *buflen);
1268 src_map_to_sec4_sg(jrdev, req->src, src_nents,
1269 edesc->sec4_sg + 1);
1270 if (*next_buflen) {
1271 scatterwalk_map_and_copy(next_buf, req->src,
1272 to_hash - *buflen,
1273 *next_buflen, 0);
1274 }
1275
1276 state->current_buf = !state->current_buf;
1277
1278 sh_len = desc_len(sh_desc);
1279 desc = edesc->hw_desc;
1280 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER |
1281 HDR_REVERSE);
1282
1283 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1284 sec4_sg_bytes,
1285 DMA_TO_DEVICE);
1286 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1287 dev_err(jrdev, "unable to map S/G table\n");
1288 return -ENOMEM;
1289 }
1290
1291 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);
1292
1293 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1294 if (ret)
1295 return ret;
1296
1297 #ifdef DEBUG
1298 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1299 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1300 desc_bytes(desc), 1);
1301 #endif
1302
1303 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1304 if (!ret) {
1305 ret = -EINPROGRESS;
1306 state->update = ahash_update_ctx;
1307 state->finup = ahash_finup_ctx;
1308 state->final = ahash_final_ctx;
1309 } else {
1310 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len,
1311 DMA_TO_DEVICE);
1312 kfree(edesc);
1313 }
1314 } else if (*next_buflen) {
1315 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
1316 req->nbytes, 0);
1317 *buflen = *next_buflen;
1318 *next_buflen = 0;
1319 }
1320 #ifdef DEBUG
1321 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
1322 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
1323 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
1324 DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
1325 *next_buflen, 1);
1326 #endif
1327
1328 return ret;
1329 }
1330
1331 /* submit ahash finup if it the first job descriptor after update */
1332 static int ahash_finup_no_ctx(struct ahash_request *req)
1333 {
1334 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1335 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1336 struct caam_hash_state *state = ahash_request_ctx(req);
1337 struct device *jrdev = ctx->jrdev;
1338 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1339 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1340 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
1341 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
1342 int last_buflen = state->current_buf ? state->buflen_0 :
1343 state->buflen_1;
1344 u32 *sh_desc = ctx->sh_desc_digest, *desc;
1345 dma_addr_t ptr = ctx->sh_desc_digest_dma;
1346 int sec4_sg_bytes, sec4_sg_src_index, src_nents;
1347 int digestsize = crypto_ahash_digestsize(ahash);
1348 struct ahash_edesc *edesc;
1349 int sh_len;
1350 int ret = 0;
1351
1352 src_nents = sg_nents_for_len(req->src, req->nbytes);
1353 if (src_nents < 0) {
1354 dev_err(jrdev, "Invalid number of src SG.\n");
1355 return src_nents;
1356 }
1357 sec4_sg_src_index = 2;
1358 sec4_sg_bytes = (sec4_sg_src_index + src_nents) *
1359 sizeof(struct sec4_sg_entry);
1360
1361 /* allocate space for base edesc and hw desc commands, link tables */
1362 edesc = kzalloc(sizeof(*edesc) + sec4_sg_bytes, GFP_DMA | flags);
1363 if (!edesc) {
1364 dev_err(jrdev, "could not allocate extended descriptor\n");
1365 return -ENOMEM;
1366 }
1367
1368 sh_len = desc_len(sh_desc);
1369 desc = edesc->hw_desc;
1370 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
1371
1372 edesc->src_nents = src_nents;
1373 edesc->sec4_sg_bytes = sec4_sg_bytes;
1374
1375 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf,
1376 state->buf_dma, buflen,
1377 last_buflen);
1378
1379 src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + 1);
1380
1381 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1382 sec4_sg_bytes, DMA_TO_DEVICE);
1383 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1384 dev_err(jrdev, "unable to map S/G table\n");
1385 return -ENOMEM;
1386 }
1387
1388 append_seq_in_ptr(desc, edesc->sec4_sg_dma, buflen +
1389 req->nbytes, LDST_SGF);
1390
1391 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1392 digestsize);
1393 if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1394 dev_err(jrdev, "unable to map dst\n");
1395 return -ENOMEM;
1396 }
1397
1398 #ifdef DEBUG
1399 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1400 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1401 #endif
1402
1403 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1404 if (!ret) {
1405 ret = -EINPROGRESS;
1406 } else {
1407 ahash_unmap(jrdev, edesc, req, digestsize);
1408 kfree(edesc);
1409 }
1410
1411 return ret;
1412 }
1413
1414 /* submit first update job descriptor after init */
1415 static int ahash_update_first(struct ahash_request *req)
1416 {
1417 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1418 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1419 struct caam_hash_state *state = ahash_request_ctx(req);
1420 struct device *jrdev = ctx->jrdev;
1421 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1422 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1423 u8 *next_buf = state->current_buf ? state->buf_1 : state->buf_0;
1424 int *next_buflen = state->current_buf ?
1425 &state->buflen_1 : &state->buflen_0;
1426 int to_hash;
1427 u32 *sh_desc = ctx->sh_desc_update_first, *desc;
1428 dma_addr_t ptr = ctx->sh_desc_update_first_dma;
1429 int sec4_sg_bytes, src_nents;
1430 dma_addr_t src_dma;
1431 u32 options;
1432 struct ahash_edesc *edesc;
1433 int ret = 0;
1434 int sh_len;
1435
1436 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
1437 1);
1438 to_hash = req->nbytes - *next_buflen;
1439
1440 if (to_hash) {
1441 src_nents = sg_nents_for_len(req->src,
1442 req->nbytes - *next_buflen);
1443 if (src_nents < 0) {
1444 dev_err(jrdev, "Invalid number of src SG.\n");
1445 return src_nents;
1446 }
1447 dma_map_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1448 if (src_nents > 1)
1449 sec4_sg_bytes = src_nents *
1450 sizeof(struct sec4_sg_entry);
1451 else
1452 sec4_sg_bytes = 0;
1453
1454 /*
1455 * allocate space for base edesc and hw desc commands,
1456 * link tables
1457 */
1458 edesc = kzalloc(sizeof(*edesc) + sec4_sg_bytes,
1459 GFP_DMA | flags);
1460 if (!edesc) {
1461 dev_err(jrdev,
1462 "could not allocate extended descriptor\n");
1463 return -ENOMEM;
1464 }
1465
1466 edesc->src_nents = src_nents;
1467 edesc->sec4_sg_bytes = sec4_sg_bytes;
1468 edesc->dst_dma = 0;
1469
1470 if (src_nents > 1) {
1471 sg_to_sec4_sg_last(req->src, src_nents,
1472 edesc->sec4_sg, 0);
1473 edesc->sec4_sg_dma = dma_map_single(jrdev,
1474 edesc->sec4_sg,
1475 sec4_sg_bytes,
1476 DMA_TO_DEVICE);
1477 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1478 dev_err(jrdev, "unable to map S/G table\n");
1479 return -ENOMEM;
1480 }
1481 src_dma = edesc->sec4_sg_dma;
1482 options = LDST_SGF;
1483 } else {
1484 src_dma = sg_dma_address(req->src);
1485 options = 0;
1486 }
1487
1488 if (*next_buflen)
1489 scatterwalk_map_and_copy(next_buf, req->src, to_hash,
1490 *next_buflen, 0);
1491
1492 sh_len = desc_len(sh_desc);
1493 desc = edesc->hw_desc;
1494 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER |
1495 HDR_REVERSE);
1496
1497 append_seq_in_ptr(desc, src_dma, to_hash, options);
1498
1499 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1500 if (ret)
1501 return ret;
1502
1503 #ifdef DEBUG
1504 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1505 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1506 desc_bytes(desc), 1);
1507 #endif
1508
1509 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst,
1510 req);
1511 if (!ret) {
1512 ret = -EINPROGRESS;
1513 state->update = ahash_update_ctx;
1514 state->finup = ahash_finup_ctx;
1515 state->final = ahash_final_ctx;
1516 } else {
1517 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len,
1518 DMA_TO_DEVICE);
1519 kfree(edesc);
1520 }
1521 } else if (*next_buflen) {
1522 state->update = ahash_update_no_ctx;
1523 state->finup = ahash_finup_no_ctx;
1524 state->final = ahash_final_no_ctx;
1525 scatterwalk_map_and_copy(next_buf, req->src, 0,
1526 req->nbytes, 0);
1527 }
1528 #ifdef DEBUG
1529 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
1530 DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
1531 *next_buflen, 1);
1532 #endif
1533
1534 return ret;
1535 }
1536
1537 static int ahash_finup_first(struct ahash_request *req)
1538 {
1539 return ahash_digest(req);
1540 }
1541
1542 static int ahash_init(struct ahash_request *req)
1543 {
1544 struct caam_hash_state *state = ahash_request_ctx(req);
1545
1546 state->update = ahash_update_first;
1547 state->finup = ahash_finup_first;
1548 state->final = ahash_final_no_ctx;
1549
1550 state->current_buf = 0;
1551 state->buf_dma = 0;
1552 state->buflen_0 = 0;
1553 state->buflen_1 = 0;
1554
1555 return 0;
1556 }
1557
1558 static int ahash_update(struct ahash_request *req)
1559 {
1560 struct caam_hash_state *state = ahash_request_ctx(req);
1561
1562 return state->update(req);
1563 }
1564
1565 static int ahash_finup(struct ahash_request *req)
1566 {
1567 struct caam_hash_state *state = ahash_request_ctx(req);
1568
1569 return state->finup(req);
1570 }
1571
1572 static int ahash_final(struct ahash_request *req)
1573 {
1574 struct caam_hash_state *state = ahash_request_ctx(req);
1575
1576 return state->final(req);
1577 }
1578
1579 static int ahash_export(struct ahash_request *req, void *out)
1580 {
1581 struct caam_hash_state *state = ahash_request_ctx(req);
1582 struct caam_export_state *export = out;
1583 int len;
1584 u8 *buf;
1585
1586 if (state->current_buf) {
1587 buf = state->buf_1;
1588 len = state->buflen_1;
1589 } else {
1590 buf = state->buf_0;
1591 len = state->buflen_0;
1592 }
1593
1594 memcpy(export->buf, buf, len);
1595 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
1596 export->buflen = len;
1597 export->update = state->update;
1598 export->final = state->final;
1599 export->finup = state->finup;
1600
1601 return 0;
1602 }
1603
1604 static int ahash_import(struct ahash_request *req, const void *in)
1605 {
1606 struct caam_hash_state *state = ahash_request_ctx(req);
1607 const struct caam_export_state *export = in;
1608
1609 memset(state, 0, sizeof(*state));
1610 memcpy(state->buf_0, export->buf, export->buflen);
1611 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
1612 state->buflen_0 = export->buflen;
1613 state->update = export->update;
1614 state->final = export->final;
1615 state->finup = export->finup;
1616
1617 return 0;
1618 }
1619
1620 struct caam_hash_template {
1621 char name[CRYPTO_MAX_ALG_NAME];
1622 char driver_name[CRYPTO_MAX_ALG_NAME];
1623 char hmac_name[CRYPTO_MAX_ALG_NAME];
1624 char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
1625 unsigned int blocksize;
1626 struct ahash_alg template_ahash;
1627 u32 alg_type;
1628 u32 alg_op;
1629 };
1630
1631 /* ahash descriptors */
1632 static struct caam_hash_template driver_hash[] = {
1633 {
1634 .name = "sha1",
1635 .driver_name = "sha1-caam",
1636 .hmac_name = "hmac(sha1)",
1637 .hmac_driver_name = "hmac-sha1-caam",
1638 .blocksize = SHA1_BLOCK_SIZE,
1639 .template_ahash = {
1640 .init = ahash_init,
1641 .update = ahash_update,
1642 .final = ahash_final,
1643 .finup = ahash_finup,
1644 .digest = ahash_digest,
1645 .export = ahash_export,
1646 .import = ahash_import,
1647 .setkey = ahash_setkey,
1648 .halg = {
1649 .digestsize = SHA1_DIGEST_SIZE,
1650 .statesize = sizeof(struct caam_export_state),
1651 },
1652 },
1653 .alg_type = OP_ALG_ALGSEL_SHA1,
1654 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
1655 }, {
1656 .name = "sha224",
1657 .driver_name = "sha224-caam",
1658 .hmac_name = "hmac(sha224)",
1659 .hmac_driver_name = "hmac-sha224-caam",
1660 .blocksize = SHA224_BLOCK_SIZE,
1661 .template_ahash = {
1662 .init = ahash_init,
1663 .update = ahash_update,
1664 .final = ahash_final,
1665 .finup = ahash_finup,
1666 .digest = ahash_digest,
1667 .export = ahash_export,
1668 .import = ahash_import,
1669 .setkey = ahash_setkey,
1670 .halg = {
1671 .digestsize = SHA224_DIGEST_SIZE,
1672 .statesize = sizeof(struct caam_export_state),
1673 },
1674 },
1675 .alg_type = OP_ALG_ALGSEL_SHA224,
1676 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
1677 }, {
1678 .name = "sha256",
1679 .driver_name = "sha256-caam",
1680 .hmac_name = "hmac(sha256)",
1681 .hmac_driver_name = "hmac-sha256-caam",
1682 .blocksize = SHA256_BLOCK_SIZE,
1683 .template_ahash = {
1684 .init = ahash_init,
1685 .update = ahash_update,
1686 .final = ahash_final,
1687 .finup = ahash_finup,
1688 .digest = ahash_digest,
1689 .export = ahash_export,
1690 .import = ahash_import,
1691 .setkey = ahash_setkey,
1692 .halg = {
1693 .digestsize = SHA256_DIGEST_SIZE,
1694 .statesize = sizeof(struct caam_export_state),
1695 },
1696 },
1697 .alg_type = OP_ALG_ALGSEL_SHA256,
1698 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
1699 }, {
1700 .name = "sha384",
1701 .driver_name = "sha384-caam",
1702 .hmac_name = "hmac(sha384)",
1703 .hmac_driver_name = "hmac-sha384-caam",
1704 .blocksize = SHA384_BLOCK_SIZE,
1705 .template_ahash = {
1706 .init = ahash_init,
1707 .update = ahash_update,
1708 .final = ahash_final,
1709 .finup = ahash_finup,
1710 .digest = ahash_digest,
1711 .export = ahash_export,
1712 .import = ahash_import,
1713 .setkey = ahash_setkey,
1714 .halg = {
1715 .digestsize = SHA384_DIGEST_SIZE,
1716 .statesize = sizeof(struct caam_export_state),
1717 },
1718 },
1719 .alg_type = OP_ALG_ALGSEL_SHA384,
1720 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
1721 }, {
1722 .name = "sha512",
1723 .driver_name = "sha512-caam",
1724 .hmac_name = "hmac(sha512)",
1725 .hmac_driver_name = "hmac-sha512-caam",
1726 .blocksize = SHA512_BLOCK_SIZE,
1727 .template_ahash = {
1728 .init = ahash_init,
1729 .update = ahash_update,
1730 .final = ahash_final,
1731 .finup = ahash_finup,
1732 .digest = ahash_digest,
1733 .export = ahash_export,
1734 .import = ahash_import,
1735 .setkey = ahash_setkey,
1736 .halg = {
1737 .digestsize = SHA512_DIGEST_SIZE,
1738 .statesize = sizeof(struct caam_export_state),
1739 },
1740 },
1741 .alg_type = OP_ALG_ALGSEL_SHA512,
1742 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
1743 }, {
1744 .name = "md5",
1745 .driver_name = "md5-caam",
1746 .hmac_name = "hmac(md5)",
1747 .hmac_driver_name = "hmac-md5-caam",
1748 .blocksize = MD5_BLOCK_WORDS * 4,
1749 .template_ahash = {
1750 .init = ahash_init,
1751 .update = ahash_update,
1752 .final = ahash_final,
1753 .finup = ahash_finup,
1754 .digest = ahash_digest,
1755 .export = ahash_export,
1756 .import = ahash_import,
1757 .setkey = ahash_setkey,
1758 .halg = {
1759 .digestsize = MD5_DIGEST_SIZE,
1760 .statesize = sizeof(struct caam_export_state),
1761 },
1762 },
1763 .alg_type = OP_ALG_ALGSEL_MD5,
1764 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
1765 },
1766 };
1767
1768 struct caam_hash_alg {
1769 struct list_head entry;
1770 int alg_type;
1771 int alg_op;
1772 struct ahash_alg ahash_alg;
1773 };
1774
1775 static int caam_hash_cra_init(struct crypto_tfm *tfm)
1776 {
1777 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
1778 struct crypto_alg *base = tfm->__crt_alg;
1779 struct hash_alg_common *halg =
1780 container_of(base, struct hash_alg_common, base);
1781 struct ahash_alg *alg =
1782 container_of(halg, struct ahash_alg, halg);
1783 struct caam_hash_alg *caam_hash =
1784 container_of(alg, struct caam_hash_alg, ahash_alg);
1785 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1786 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
1787 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
1788 HASH_MSG_LEN + SHA1_DIGEST_SIZE,
1789 HASH_MSG_LEN + 32,
1790 HASH_MSG_LEN + SHA256_DIGEST_SIZE,
1791 HASH_MSG_LEN + 64,
1792 HASH_MSG_LEN + SHA512_DIGEST_SIZE };
1793 int ret = 0;
1794
1795 /*
1796 * Get a Job ring from Job Ring driver to ensure in-order
1797 * crypto request processing per tfm
1798 */
1799 ctx->jrdev = caam_jr_alloc();
1800 if (IS_ERR(ctx->jrdev)) {
1801 pr_err("Job Ring Device allocation for transform failed\n");
1802 return PTR_ERR(ctx->jrdev);
1803 }
1804 /* copy descriptor header template value */
1805 ctx->alg_type = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
1806 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_hash->alg_op;
1807
1808 ctx->ctx_len = runninglen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
1809 OP_ALG_ALGSEL_SHIFT];
1810
1811 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1812 sizeof(struct caam_hash_state));
1813
1814 ret = ahash_set_sh_desc(ahash);
1815
1816 return ret;
1817 }
1818
1819 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
1820 {
1821 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1822
1823 if (ctx->sh_desc_update_dma &&
1824 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_dma))
1825 dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_dma,
1826 desc_bytes(ctx->sh_desc_update),
1827 DMA_TO_DEVICE);
1828 if (ctx->sh_desc_update_first_dma &&
1829 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_first_dma))
1830 dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_first_dma,
1831 desc_bytes(ctx->sh_desc_update_first),
1832 DMA_TO_DEVICE);
1833 if (ctx->sh_desc_fin_dma &&
1834 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_fin_dma))
1835 dma_unmap_single(ctx->jrdev, ctx->sh_desc_fin_dma,
1836 desc_bytes(ctx->sh_desc_fin), DMA_TO_DEVICE);
1837 if (ctx->sh_desc_digest_dma &&
1838 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_digest_dma))
1839 dma_unmap_single(ctx->jrdev, ctx->sh_desc_digest_dma,
1840 desc_bytes(ctx->sh_desc_digest),
1841 DMA_TO_DEVICE);
1842 if (ctx->sh_desc_finup_dma &&
1843 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_finup_dma))
1844 dma_unmap_single(ctx->jrdev, ctx->sh_desc_finup_dma,
1845 desc_bytes(ctx->sh_desc_finup), DMA_TO_DEVICE);
1846
1847 caam_jr_free(ctx->jrdev);
1848 }
1849
1850 static void __exit caam_algapi_hash_exit(void)
1851 {
1852 struct caam_hash_alg *t_alg, *n;
1853
1854 if (!hash_list.next)
1855 return;
1856
1857 list_for_each_entry_safe(t_alg, n, &hash_list, entry) {
1858 crypto_unregister_ahash(&t_alg->ahash_alg);
1859 list_del(&t_alg->entry);
1860 kfree(t_alg);
1861 }
1862 }
1863
1864 static struct caam_hash_alg *
1865 caam_hash_alloc(struct caam_hash_template *template,
1866 bool keyed)
1867 {
1868 struct caam_hash_alg *t_alg;
1869 struct ahash_alg *halg;
1870 struct crypto_alg *alg;
1871
1872 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
1873 if (!t_alg) {
1874 pr_err("failed to allocate t_alg\n");
1875 return ERR_PTR(-ENOMEM);
1876 }
1877
1878 t_alg->ahash_alg = template->template_ahash;
1879 halg = &t_alg->ahash_alg;
1880 alg = &halg->halg.base;
1881
1882 if (keyed) {
1883 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1884 template->hmac_name);
1885 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1886 template->hmac_driver_name);
1887 } else {
1888 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1889 template->name);
1890 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1891 template->driver_name);
1892 t_alg->ahash_alg.setkey = NULL;
1893 }
1894 alg->cra_module = THIS_MODULE;
1895 alg->cra_init = caam_hash_cra_init;
1896 alg->cra_exit = caam_hash_cra_exit;
1897 alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
1898 alg->cra_priority = CAAM_CRA_PRIORITY;
1899 alg->cra_blocksize = template->blocksize;
1900 alg->cra_alignmask = 0;
1901 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH;
1902 alg->cra_type = &crypto_ahash_type;
1903
1904 t_alg->alg_type = template->alg_type;
1905 t_alg->alg_op = template->alg_op;
1906
1907 return t_alg;
1908 }
1909
1910 static int __init caam_algapi_hash_init(void)
1911 {
1912 struct device_node *dev_node;
1913 struct platform_device *pdev;
1914 struct device *ctrldev;
1915 int i = 0, err = 0;
1916 struct caam_drv_private *priv;
1917 unsigned int md_limit = SHA512_DIGEST_SIZE;
1918 u32 cha_inst, cha_vid;
1919
1920 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
1921 if (!dev_node) {
1922 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
1923 if (!dev_node)
1924 return -ENODEV;
1925 }
1926
1927 pdev = of_find_device_by_node(dev_node);
1928 if (!pdev) {
1929 of_node_put(dev_node);
1930 return -ENODEV;
1931 }
1932
1933 ctrldev = &pdev->dev;
1934 priv = dev_get_drvdata(ctrldev);
1935 of_node_put(dev_node);
1936
1937 /*
1938 * If priv is NULL, it's probably because the caam driver wasn't
1939 * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
1940 */
1941 if (!priv)
1942 return -ENODEV;
1943
1944 /*
1945 * Register crypto algorithms the device supports. First, identify
1946 * presence and attributes of MD block.
1947 */
1948 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
1949 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
1950
1951 /*
1952 * Skip registration of any hashing algorithms if MD block
1953 * is not present.
1954 */
1955 if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT))
1956 return -ENODEV;
1957
1958 /* Limit digest size based on LP256 */
1959 if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)
1960 md_limit = SHA256_DIGEST_SIZE;
1961
1962 INIT_LIST_HEAD(&hash_list);
1963
1964 /* register crypto algorithms the device supports */
1965 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
1966 struct caam_hash_alg *t_alg;
1967 struct caam_hash_template *alg = driver_hash + i;
1968
1969 /* If MD size is not supported by device, skip registration */
1970 if (alg->template_ahash.halg.digestsize > md_limit)
1971 continue;
1972
1973 /* register hmac version */
1974 t_alg = caam_hash_alloc(alg, true);
1975 if (IS_ERR(t_alg)) {
1976 err = PTR_ERR(t_alg);
1977 pr_warn("%s alg allocation failed\n", alg->driver_name);
1978 continue;
1979 }
1980
1981 err = crypto_register_ahash(&t_alg->ahash_alg);
1982 if (err) {
1983 pr_warn("%s alg registration failed: %d\n",
1984 t_alg->ahash_alg.halg.base.cra_driver_name,
1985 err);
1986 kfree(t_alg);
1987 } else
1988 list_add_tail(&t_alg->entry, &hash_list);
1989
1990 /* register unkeyed version */
1991 t_alg = caam_hash_alloc(alg, false);
1992 if (IS_ERR(t_alg)) {
1993 err = PTR_ERR(t_alg);
1994 pr_warn("%s alg allocation failed\n", alg->driver_name);
1995 continue;
1996 }
1997
1998 err = crypto_register_ahash(&t_alg->ahash_alg);
1999 if (err) {
2000 pr_warn("%s alg registration failed: %d\n",
2001 t_alg->ahash_alg.halg.base.cra_driver_name,
2002 err);
2003 kfree(t_alg);
2004 } else
2005 list_add_tail(&t_alg->entry, &hash_list);
2006 }
2007
2008 return err;
2009 }
2010
2011 module_init(caam_algapi_hash_init);
2012 module_exit(caam_algapi_hash_exit);
2013
2014 MODULE_LICENSE("GPL");
2015 MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API");
2016 MODULE_AUTHOR("Freescale Semiconductor - NMG");
(deprecated) Btrfs devel tree