2 * Hash algorithms supported by the CESA: MD5, SHA1 and SHA256.
4 * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
5 * Author: Arnaud Ebalard <arno@natisbad.org>
7 * This work is based on an initial version written by
8 * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc >
10 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of the GNU General Public License version 2 as published
12 * by the Free Software Foundation.
15 #include <crypto/md5.h>
16 #include <crypto/sha.h>
20 struct mv_cesa_ahash_dma_iter
{
21 struct mv_cesa_dma_iter base
;
22 struct mv_cesa_sg_dma_iter src
;
26 mv_cesa_ahash_req_iter_init(struct mv_cesa_ahash_dma_iter
*iter
,
27 struct ahash_request
*req
)
29 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
30 unsigned int len
= req
->nbytes
;
33 len
= (len
+ creq
->cache_ptr
) & ~CESA_HASH_BLOCK_SIZE_MSK
;
35 mv_cesa_req_dma_iter_init(&iter
->base
, len
);
36 mv_cesa_sg_dma_iter_init(&iter
->src
, req
->src
, DMA_TO_DEVICE
);
37 iter
->src
.op_offset
= creq
->cache_ptr
;
41 mv_cesa_ahash_req_iter_next_op(struct mv_cesa_ahash_dma_iter
*iter
)
43 iter
->src
.op_offset
= 0;
45 return mv_cesa_req_dma_iter_next_op(&iter
->base
);
48 static inline int mv_cesa_ahash_dma_alloc_cache(struct mv_cesa_ahash_req
*creq
,
51 struct mv_cesa_ahash_dma_req
*dreq
= &creq
->req
.dma
;
53 creq
->cache
= dma_pool_alloc(cesa_dev
->dma
->cache_pool
, flags
,
61 static inline int mv_cesa_ahash_std_alloc_cache(struct mv_cesa_ahash_req
*creq
,
64 creq
->cache
= kzalloc(CESA_MAX_HASH_BLOCK_SIZE
, flags
);
71 static int mv_cesa_ahash_alloc_cache(struct ahash_request
*req
)
73 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
74 gfp_t flags
= (req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
) ?
75 GFP_KERNEL
: GFP_ATOMIC
;
81 if (creq
->req
.base
.type
== CESA_DMA_REQ
)
82 ret
= mv_cesa_ahash_dma_alloc_cache(creq
, flags
);
84 ret
= mv_cesa_ahash_std_alloc_cache(creq
, flags
);
89 static inline void mv_cesa_ahash_dma_free_cache(struct mv_cesa_ahash_req
*creq
)
91 dma_pool_free(cesa_dev
->dma
->cache_pool
, creq
->cache
,
92 creq
->req
.dma
.cache_dma
);
95 static inline void mv_cesa_ahash_std_free_cache(struct mv_cesa_ahash_req
*creq
)
100 static void mv_cesa_ahash_free_cache(struct mv_cesa_ahash_req
*creq
)
105 if (creq
->req
.base
.type
== CESA_DMA_REQ
)
106 mv_cesa_ahash_dma_free_cache(creq
);
108 mv_cesa_ahash_std_free_cache(creq
);
113 static int mv_cesa_ahash_dma_alloc_padding(struct mv_cesa_ahash_dma_req
*req
,
119 req
->padding
= dma_pool_alloc(cesa_dev
->dma
->padding_pool
, flags
,
127 static void mv_cesa_ahash_dma_free_padding(struct mv_cesa_ahash_dma_req
*req
)
132 dma_pool_free(cesa_dev
->dma
->padding_pool
, req
->padding
,
137 static inline void mv_cesa_ahash_dma_last_cleanup(struct ahash_request
*req
)
139 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
141 mv_cesa_ahash_dma_free_padding(&creq
->req
.dma
);
144 static inline void mv_cesa_ahash_dma_cleanup(struct ahash_request
*req
)
146 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
148 dma_unmap_sg(cesa_dev
->dev
, req
->src
, creq
->src_nents
, DMA_TO_DEVICE
);
149 mv_cesa_dma_cleanup(&creq
->req
.dma
.base
);
152 static inline void mv_cesa_ahash_cleanup(struct ahash_request
*req
)
154 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
156 if (creq
->req
.base
.type
== CESA_DMA_REQ
)
157 mv_cesa_ahash_dma_cleanup(req
);
160 static void mv_cesa_ahash_last_cleanup(struct ahash_request
*req
)
162 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
164 mv_cesa_ahash_free_cache(creq
);
166 if (creq
->req
.base
.type
== CESA_DMA_REQ
)
167 mv_cesa_ahash_dma_last_cleanup(req
);
170 static int mv_cesa_ahash_pad_len(struct mv_cesa_ahash_req
*creq
)
172 unsigned int index
, padlen
;
174 index
= creq
->len
& CESA_HASH_BLOCK_SIZE_MSK
;
175 padlen
= (index
< 56) ? (56 - index
) : (64 + 56 - index
);
180 static int mv_cesa_ahash_pad_req(struct mv_cesa_ahash_req
*creq
, u8
*buf
)
182 unsigned int index
, padlen
;
185 /* Pad out to 56 mod 64 */
186 index
= creq
->len
& CESA_HASH_BLOCK_SIZE_MSK
;
187 padlen
= mv_cesa_ahash_pad_len(creq
);
188 memset(buf
+ 1, 0, padlen
- 1);
191 __le64 bits
= cpu_to_le64(creq
->len
<< 3);
192 memcpy(buf
+ padlen
, &bits
, sizeof(bits
));
194 __be64 bits
= cpu_to_be64(creq
->len
<< 3);
195 memcpy(buf
+ padlen
, &bits
, sizeof(bits
));
201 static void mv_cesa_ahash_std_step(struct ahash_request
*req
)
203 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
204 struct mv_cesa_ahash_std_req
*sreq
= &creq
->req
.std
;
205 struct mv_cesa_engine
*engine
= sreq
->base
.engine
;
206 struct mv_cesa_op_ctx
*op
;
207 unsigned int new_cache_ptr
= 0;
212 memcpy(engine
->sram
+ CESA_SA_DATA_SRAM_OFFSET
, creq
->cache
,
215 len
= min_t(size_t, req
->nbytes
+ creq
->cache_ptr
- sreq
->offset
,
216 CESA_SA_SRAM_PAYLOAD_SIZE
);
218 if (!creq
->last_req
) {
219 new_cache_ptr
= len
& CESA_HASH_BLOCK_SIZE_MSK
;
220 len
&= ~CESA_HASH_BLOCK_SIZE_MSK
;
223 if (len
- creq
->cache_ptr
)
224 sreq
->offset
+= sg_pcopy_to_buffer(req
->src
, creq
->src_nents
,
226 CESA_SA_DATA_SRAM_OFFSET
+
228 len
- creq
->cache_ptr
,
233 frag_mode
= mv_cesa_get_op_cfg(op
) & CESA_SA_DESC_CFG_FRAG_MSK
;
235 if (creq
->last_req
&& sreq
->offset
== req
->nbytes
&&
236 creq
->len
<= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX
) {
237 if (frag_mode
== CESA_SA_DESC_CFG_FIRST_FRAG
)
238 frag_mode
= CESA_SA_DESC_CFG_NOT_FRAG
;
239 else if (frag_mode
== CESA_SA_DESC_CFG_MID_FRAG
)
240 frag_mode
= CESA_SA_DESC_CFG_LAST_FRAG
;
243 if (frag_mode
== CESA_SA_DESC_CFG_NOT_FRAG
||
244 frag_mode
== CESA_SA_DESC_CFG_LAST_FRAG
) {
246 creq
->len
<= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX
) {
247 mv_cesa_set_mac_op_total_len(op
, creq
->len
);
249 int trailerlen
= mv_cesa_ahash_pad_len(creq
) + 8;
251 if (len
+ trailerlen
> CESA_SA_SRAM_PAYLOAD_SIZE
) {
252 len
&= CESA_HASH_BLOCK_SIZE_MSK
;
253 new_cache_ptr
= 64 - trailerlen
;
256 CESA_SA_DATA_SRAM_OFFSET
+ len
,
259 len
+= mv_cesa_ahash_pad_req(creq
,
261 CESA_SA_DATA_SRAM_OFFSET
);
264 if (frag_mode
== CESA_SA_DESC_CFG_LAST_FRAG
)
265 frag_mode
= CESA_SA_DESC_CFG_MID_FRAG
;
267 frag_mode
= CESA_SA_DESC_CFG_FIRST_FRAG
;
271 mv_cesa_set_mac_op_frag_len(op
, len
);
272 mv_cesa_update_op_cfg(op
, frag_mode
, CESA_SA_DESC_CFG_FRAG_MSK
);
274 /* FIXME: only update enc_len field */
275 memcpy(engine
->sram
, op
, sizeof(*op
));
277 if (frag_mode
== CESA_SA_DESC_CFG_FIRST_FRAG
)
278 mv_cesa_update_op_cfg(op
, CESA_SA_DESC_CFG_MID_FRAG
,
279 CESA_SA_DESC_CFG_FRAG_MSK
);
281 creq
->cache_ptr
= new_cache_ptr
;
283 mv_cesa_set_int_mask(engine
, CESA_SA_INT_ACCEL0_DONE
);
284 writel(CESA_SA_CFG_PARA_DIS
, engine
->regs
+ CESA_SA_CFG
);
285 writel(CESA_SA_CMD_EN_CESA_SA_ACCL0
, engine
->regs
+ CESA_SA_CMD
);
288 static int mv_cesa_ahash_std_process(struct ahash_request
*req
, u32 status
)
290 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
291 struct mv_cesa_ahash_std_req
*sreq
= &creq
->req
.std
;
293 if (sreq
->offset
< (req
->nbytes
- creq
->cache_ptr
))
299 static inline void mv_cesa_ahash_dma_prepare(struct ahash_request
*req
)
301 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
302 struct mv_cesa_tdma_req
*dreq
= &creq
->req
.dma
.base
;
304 mv_cesa_dma_prepare(dreq
, dreq
->base
.engine
);
307 static void mv_cesa_ahash_std_prepare(struct ahash_request
*req
)
309 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
310 struct mv_cesa_ahash_std_req
*sreq
= &creq
->req
.std
;
311 struct mv_cesa_engine
*engine
= sreq
->base
.engine
;
314 mv_cesa_adjust_op(engine
, &creq
->op_tmpl
);
315 memcpy(engine
->sram
, &creq
->op_tmpl
, sizeof(creq
->op_tmpl
));
318 static void mv_cesa_ahash_step(struct crypto_async_request
*req
)
320 struct ahash_request
*ahashreq
= ahash_request_cast(req
);
321 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(ahashreq
);
323 if (creq
->req
.base
.type
== CESA_DMA_REQ
)
324 mv_cesa_dma_step(&creq
->req
.dma
.base
);
326 mv_cesa_ahash_std_step(ahashreq
);
329 static int mv_cesa_ahash_process(struct crypto_async_request
*req
, u32 status
)
331 struct ahash_request
*ahashreq
= ahash_request_cast(req
);
332 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(ahashreq
);
333 struct mv_cesa_engine
*engine
= creq
->req
.base
.engine
;
334 unsigned int digsize
;
337 if (creq
->req
.base
.type
== CESA_DMA_REQ
)
338 ret
= mv_cesa_dma_process(&creq
->req
.dma
.base
, status
);
340 ret
= mv_cesa_ahash_std_process(ahashreq
, status
);
342 if (ret
== -EINPROGRESS
)
345 digsize
= crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq
));
346 for (i
= 0; i
< digsize
/ 4; i
++)
347 creq
->state
[i
] = readl(engine
->regs
+ CESA_IVDIG(i
));
350 sg_pcopy_to_buffer(ahashreq
->src
, creq
->src_nents
,
353 ahashreq
->nbytes
- creq
->cache_ptr
);
355 if (creq
->last_req
) {
357 * Hardware's MD5 digest is in little endian format, but
358 * SHA in big endian format
361 __le32
*result
= (void *)ahashreq
->result
;
363 for (i
= 0; i
< digsize
/ 4; i
++)
364 result
[i
] = cpu_to_le32(creq
->state
[i
]);
366 __be32
*result
= (void *)ahashreq
->result
;
368 for (i
= 0; i
< digsize
/ 4; i
++)
369 result
[i
] = cpu_to_be32(creq
->state
[i
]);
376 static void mv_cesa_ahash_prepare(struct crypto_async_request
*req
,
377 struct mv_cesa_engine
*engine
)
379 struct ahash_request
*ahashreq
= ahash_request_cast(req
);
380 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(ahashreq
);
381 unsigned int digsize
;
384 creq
->req
.base
.engine
= engine
;
386 if (creq
->req
.base
.type
== CESA_DMA_REQ
)
387 mv_cesa_ahash_dma_prepare(ahashreq
);
389 mv_cesa_ahash_std_prepare(ahashreq
);
391 digsize
= crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq
));
392 for (i
= 0; i
< digsize
/ 4; i
++)
393 writel(creq
->state
[i
],
394 engine
->regs
+ CESA_IVDIG(i
));
397 static void mv_cesa_ahash_req_cleanup(struct crypto_async_request
*req
)
399 struct ahash_request
*ahashreq
= ahash_request_cast(req
);
400 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(ahashreq
);
403 mv_cesa_ahash_last_cleanup(ahashreq
);
405 mv_cesa_ahash_cleanup(ahashreq
);
408 static const struct mv_cesa_req_ops mv_cesa_ahash_req_ops
= {
409 .step
= mv_cesa_ahash_step
,
410 .process
= mv_cesa_ahash_process
,
411 .prepare
= mv_cesa_ahash_prepare
,
412 .cleanup
= mv_cesa_ahash_req_cleanup
,
415 static int mv_cesa_ahash_init(struct ahash_request
*req
,
416 struct mv_cesa_op_ctx
*tmpl
, bool algo_le
)
418 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
420 memset(creq
, 0, sizeof(*creq
));
421 mv_cesa_update_op_cfg(tmpl
,
422 CESA_SA_DESC_CFG_OP_MAC_ONLY
|
423 CESA_SA_DESC_CFG_FIRST_FRAG
,
424 CESA_SA_DESC_CFG_OP_MSK
|
425 CESA_SA_DESC_CFG_FRAG_MSK
);
426 mv_cesa_set_mac_op_total_len(tmpl
, 0);
427 mv_cesa_set_mac_op_frag_len(tmpl
, 0);
428 creq
->op_tmpl
= *tmpl
;
430 creq
->algo_le
= algo_le
;
435 static inline int mv_cesa_ahash_cra_init(struct crypto_tfm
*tfm
)
437 struct mv_cesa_hash_ctx
*ctx
= crypto_tfm_ctx(tfm
);
439 ctx
->base
.ops
= &mv_cesa_ahash_req_ops
;
441 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
442 sizeof(struct mv_cesa_ahash_req
));
446 static int mv_cesa_ahash_cache_req(struct ahash_request
*req
, bool *cached
)
448 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
451 if (((creq
->cache_ptr
+ req
->nbytes
) & CESA_HASH_BLOCK_SIZE_MSK
) &&
453 ret
= mv_cesa_ahash_alloc_cache(req
);
458 if (creq
->cache_ptr
+ req
->nbytes
< 64 && !creq
->last_req
) {
464 sg_pcopy_to_buffer(req
->src
, creq
->src_nents
,
465 creq
->cache
+ creq
->cache_ptr
,
468 creq
->cache_ptr
+= req
->nbytes
;
474 static struct mv_cesa_op_ctx
*
475 mv_cesa_dma_add_frag(struct mv_cesa_tdma_chain
*chain
,
476 struct mv_cesa_op_ctx
*tmpl
, unsigned int frag_len
,
479 struct mv_cesa_op_ctx
*op
;
482 op
= mv_cesa_dma_add_op(chain
, tmpl
, false, flags
);
486 /* Set the operation block fragment length. */
487 mv_cesa_set_mac_op_frag_len(op
, frag_len
);
489 /* Append dummy desc to launch operation */
490 ret
= mv_cesa_dma_add_dummy_launch(chain
, flags
);
497 static struct mv_cesa_op_ctx
*
498 mv_cesa_ahash_dma_add_cache(struct mv_cesa_tdma_chain
*chain
,
499 struct mv_cesa_ahash_dma_iter
*dma_iter
,
500 struct mv_cesa_ahash_req
*creq
,
503 struct mv_cesa_ahash_dma_req
*ahashdreq
= &creq
->req
.dma
;
504 struct mv_cesa_op_ctx
*op
= NULL
;
507 if (!creq
->cache_ptr
)
510 ret
= mv_cesa_dma_add_data_transfer(chain
,
511 CESA_SA_DATA_SRAM_OFFSET
,
512 ahashdreq
->cache_dma
,
514 CESA_TDMA_DST_IN_SRAM
,
519 if (!dma_iter
->base
.op_len
)
520 op
= mv_cesa_dma_add_frag(chain
, &creq
->op_tmpl
,
521 creq
->cache_ptr
, flags
);
526 static struct mv_cesa_op_ctx
*
527 mv_cesa_ahash_dma_add_data(struct mv_cesa_tdma_chain
*chain
,
528 struct mv_cesa_ahash_dma_iter
*dma_iter
,
529 struct mv_cesa_ahash_req
*creq
,
532 struct mv_cesa_op_ctx
*op
;
535 /* Add input transfers */
536 ret
= mv_cesa_dma_add_op_transfers(chain
, &dma_iter
->base
,
537 &dma_iter
->src
, flags
);
541 op
= mv_cesa_dma_add_frag(chain
, &creq
->op_tmpl
, dma_iter
->base
.op_len
,
546 if (mv_cesa_mac_op_is_first_frag(&creq
->op_tmpl
))
547 mv_cesa_update_op_cfg(&creq
->op_tmpl
,
548 CESA_SA_DESC_CFG_MID_FRAG
,
549 CESA_SA_DESC_CFG_FRAG_MSK
);
554 static struct mv_cesa_op_ctx
*
555 mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain
*chain
,
556 struct mv_cesa_ahash_dma_iter
*dma_iter
,
557 struct mv_cesa_ahash_req
*creq
,
558 struct mv_cesa_op_ctx
*op
,
561 struct mv_cesa_ahash_dma_req
*ahashdreq
= &creq
->req
.dma
;
562 unsigned int len
, trailerlen
, padoff
= 0;
568 if (op
&& creq
->len
<= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX
) {
569 u32 frag
= CESA_SA_DESC_CFG_NOT_FRAG
;
571 if (!mv_cesa_mac_op_is_first_frag(op
))
572 frag
= CESA_SA_DESC_CFG_LAST_FRAG
;
574 mv_cesa_update_op_cfg(op
, frag
, CESA_SA_DESC_CFG_FRAG_MSK
);
579 ret
= mv_cesa_ahash_dma_alloc_padding(ahashdreq
, flags
);
583 trailerlen
= mv_cesa_ahash_pad_req(creq
, ahashdreq
->padding
);
586 len
= min(CESA_SA_SRAM_PAYLOAD_SIZE
- dma_iter
->base
.op_len
,
589 ret
= mv_cesa_dma_add_data_transfer(chain
,
590 CESA_SA_DATA_SRAM_OFFSET
+
591 dma_iter
->base
.op_len
,
592 ahashdreq
->padding_dma
,
593 len
, CESA_TDMA_DST_IN_SRAM
,
598 mv_cesa_update_op_cfg(op
, CESA_SA_DESC_CFG_MID_FRAG
,
599 CESA_SA_DESC_CFG_FRAG_MSK
);
600 mv_cesa_set_mac_op_frag_len(op
,
601 dma_iter
->base
.op_len
+ len
);
606 if (padoff
>= trailerlen
)
609 if (!mv_cesa_mac_op_is_first_frag(&creq
->op_tmpl
))
610 mv_cesa_update_op_cfg(&creq
->op_tmpl
,
611 CESA_SA_DESC_CFG_MID_FRAG
,
612 CESA_SA_DESC_CFG_FRAG_MSK
);
614 ret
= mv_cesa_dma_add_data_transfer(chain
,
615 CESA_SA_DATA_SRAM_OFFSET
,
616 ahashdreq
->padding_dma
+
619 CESA_TDMA_DST_IN_SRAM
,
624 return mv_cesa_dma_add_frag(chain
, &creq
->op_tmpl
, trailerlen
- padoff
,
628 static int mv_cesa_ahash_dma_req_init(struct ahash_request
*req
)
630 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
631 gfp_t flags
= (req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
) ?
632 GFP_KERNEL
: GFP_ATOMIC
;
633 struct mv_cesa_ahash_dma_req
*ahashdreq
= &creq
->req
.dma
;
634 struct mv_cesa_tdma_req
*dreq
= &ahashdreq
->base
;
635 struct mv_cesa_tdma_chain chain
;
636 struct mv_cesa_ahash_dma_iter iter
;
637 struct mv_cesa_op_ctx
*op
= NULL
;
640 dreq
->chain
.first
= NULL
;
641 dreq
->chain
.last
= NULL
;
643 if (creq
->src_nents
) {
644 ret
= dma_map_sg(cesa_dev
->dev
, req
->src
, creq
->src_nents
,
652 mv_cesa_tdma_desc_iter_init(&chain
);
653 mv_cesa_ahash_req_iter_init(&iter
, req
);
655 op
= mv_cesa_ahash_dma_add_cache(&chain
, &iter
,
663 if (!iter
.base
.op_len
)
666 op
= mv_cesa_ahash_dma_add_data(&chain
, &iter
,
672 } while (mv_cesa_ahash_req_iter_next_op(&iter
));
674 op
= mv_cesa_ahash_dma_last_req(&chain
, &iter
, creq
, op
, flags
);
681 /* Add dummy desc to wait for crypto operation end */
682 ret
= mv_cesa_dma_add_dummy_end(&chain
, flags
);
688 creq
->cache_ptr
= req
->nbytes
+ creq
->cache_ptr
-
698 mv_cesa_dma_cleanup(dreq
);
699 dma_unmap_sg(cesa_dev
->dev
, req
->src
, creq
->src_nents
, DMA_TO_DEVICE
);
702 mv_cesa_ahash_last_cleanup(req
);
707 static int mv_cesa_ahash_req_init(struct ahash_request
*req
, bool *cached
)
709 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
712 if (cesa_dev
->caps
->has_tdma
)
713 creq
->req
.base
.type
= CESA_DMA_REQ
;
715 creq
->req
.base
.type
= CESA_STD_REQ
;
717 creq
->src_nents
= sg_nents_for_len(req
->src
, req
->nbytes
);
719 ret
= mv_cesa_ahash_cache_req(req
, cached
);
726 if (creq
->req
.base
.type
== CESA_DMA_REQ
)
727 ret
= mv_cesa_ahash_dma_req_init(req
);
732 static int mv_cesa_ahash_update(struct ahash_request
*req
)
734 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
738 creq
->len
+= req
->nbytes
;
739 ret
= mv_cesa_ahash_req_init(req
, &cached
);
746 ret
= mv_cesa_queue_req(&req
->base
);
747 if (ret
&& ret
!= -EINPROGRESS
) {
748 mv_cesa_ahash_cleanup(req
);
755 static int mv_cesa_ahash_final(struct ahash_request
*req
)
757 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
758 struct mv_cesa_op_ctx
*tmpl
= &creq
->op_tmpl
;
762 mv_cesa_set_mac_op_total_len(tmpl
, creq
->len
);
763 creq
->last_req
= true;
766 ret
= mv_cesa_ahash_req_init(req
, &cached
);
773 ret
= mv_cesa_queue_req(&req
->base
);
774 if (ret
&& ret
!= -EINPROGRESS
)
775 mv_cesa_ahash_cleanup(req
);
780 static int mv_cesa_ahash_finup(struct ahash_request
*req
)
782 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
783 struct mv_cesa_op_ctx
*tmpl
= &creq
->op_tmpl
;
787 creq
->len
+= req
->nbytes
;
788 mv_cesa_set_mac_op_total_len(tmpl
, creq
->len
);
789 creq
->last_req
= true;
791 ret
= mv_cesa_ahash_req_init(req
, &cached
);
798 ret
= mv_cesa_queue_req(&req
->base
);
799 if (ret
&& ret
!= -EINPROGRESS
)
800 mv_cesa_ahash_cleanup(req
);
805 static int mv_cesa_ahash_export(struct ahash_request
*req
, void *hash
,
806 u64
*len
, void *cache
)
808 struct crypto_ahash
*ahash
= crypto_ahash_reqtfm(req
);
809 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
810 unsigned int digsize
= crypto_ahash_digestsize(ahash
);
811 unsigned int blocksize
;
813 blocksize
= crypto_ahash_blocksize(ahash
);
816 memcpy(hash
, creq
->state
, digsize
);
817 memset(cache
, 0, blocksize
);
819 memcpy(cache
, creq
->cache
, creq
->cache_ptr
);
824 static int mv_cesa_ahash_import(struct ahash_request
*req
, const void *hash
,
825 u64 len
, const void *cache
)
827 struct crypto_ahash
*ahash
= crypto_ahash_reqtfm(req
);
828 struct mv_cesa_ahash_req
*creq
= ahash_request_ctx(req
);
829 unsigned int digsize
= crypto_ahash_digestsize(ahash
);
830 unsigned int blocksize
;
831 unsigned int cache_ptr
;
834 ret
= crypto_ahash_init(req
);
838 blocksize
= crypto_ahash_blocksize(ahash
);
839 if (len
>= blocksize
)
840 mv_cesa_update_op_cfg(&creq
->op_tmpl
,
841 CESA_SA_DESC_CFG_MID_FRAG
,
842 CESA_SA_DESC_CFG_FRAG_MSK
);
845 memcpy(creq
->state
, hash
, digsize
);
848 cache_ptr
= do_div(len
, blocksize
);
852 ret
= mv_cesa_ahash_alloc_cache(req
);
856 memcpy(creq
->cache
, cache
, cache_ptr
);
857 creq
->cache_ptr
= cache_ptr
;
862 static int mv_cesa_md5_init(struct ahash_request
*req
)
864 struct mv_cesa_op_ctx tmpl
= { };
866 mv_cesa_set_op_cfg(&tmpl
, CESA_SA_DESC_CFG_MACM_MD5
);
868 mv_cesa_ahash_init(req
, &tmpl
, true);
873 static int mv_cesa_md5_export(struct ahash_request
*req
, void *out
)
875 struct md5_state
*out_state
= out
;
877 return mv_cesa_ahash_export(req
, out_state
->hash
,
878 &out_state
->byte_count
, out_state
->block
);
881 static int mv_cesa_md5_import(struct ahash_request
*req
, const void *in
)
883 const struct md5_state
*in_state
= in
;
885 return mv_cesa_ahash_import(req
, in_state
->hash
, in_state
->byte_count
,
889 static int mv_cesa_md5_digest(struct ahash_request
*req
)
893 ret
= mv_cesa_md5_init(req
);
897 return mv_cesa_ahash_finup(req
);
900 struct ahash_alg mv_md5_alg
= {
901 .init
= mv_cesa_md5_init
,
902 .update
= mv_cesa_ahash_update
,
903 .final
= mv_cesa_ahash_final
,
904 .finup
= mv_cesa_ahash_finup
,
905 .digest
= mv_cesa_md5_digest
,
906 .export
= mv_cesa_md5_export
,
907 .import
= mv_cesa_md5_import
,
909 .digestsize
= MD5_DIGEST_SIZE
,
910 .statesize
= sizeof(struct md5_state
),
913 .cra_driver_name
= "mv-md5",
915 .cra_flags
= CRYPTO_ALG_ASYNC
|
916 CRYPTO_ALG_KERN_DRIVER_ONLY
,
917 .cra_blocksize
= MD5_HMAC_BLOCK_SIZE
,
918 .cra_ctxsize
= sizeof(struct mv_cesa_hash_ctx
),
919 .cra_init
= mv_cesa_ahash_cra_init
,
920 .cra_module
= THIS_MODULE
,
925 static int mv_cesa_sha1_init(struct ahash_request
*req
)
927 struct mv_cesa_op_ctx tmpl
= { };
929 mv_cesa_set_op_cfg(&tmpl
, CESA_SA_DESC_CFG_MACM_SHA1
);
931 mv_cesa_ahash_init(req
, &tmpl
, false);
936 static int mv_cesa_sha1_export(struct ahash_request
*req
, void *out
)
938 struct sha1_state
*out_state
= out
;
940 return mv_cesa_ahash_export(req
, out_state
->state
, &out_state
->count
,
944 static int mv_cesa_sha1_import(struct ahash_request
*req
, const void *in
)
946 const struct sha1_state
*in_state
= in
;
948 return mv_cesa_ahash_import(req
, in_state
->state
, in_state
->count
,
952 static int mv_cesa_sha1_digest(struct ahash_request
*req
)
956 ret
= mv_cesa_sha1_init(req
);
960 return mv_cesa_ahash_finup(req
);
963 struct ahash_alg mv_sha1_alg
= {
964 .init
= mv_cesa_sha1_init
,
965 .update
= mv_cesa_ahash_update
,
966 .final
= mv_cesa_ahash_final
,
967 .finup
= mv_cesa_ahash_finup
,
968 .digest
= mv_cesa_sha1_digest
,
969 .export
= mv_cesa_sha1_export
,
970 .import
= mv_cesa_sha1_import
,
972 .digestsize
= SHA1_DIGEST_SIZE
,
973 .statesize
= sizeof(struct sha1_state
),
976 .cra_driver_name
= "mv-sha1",
978 .cra_flags
= CRYPTO_ALG_ASYNC
|
979 CRYPTO_ALG_KERN_DRIVER_ONLY
,
980 .cra_blocksize
= SHA1_BLOCK_SIZE
,
981 .cra_ctxsize
= sizeof(struct mv_cesa_hash_ctx
),
982 .cra_init
= mv_cesa_ahash_cra_init
,
983 .cra_module
= THIS_MODULE
,
988 static int mv_cesa_sha256_init(struct ahash_request
*req
)
990 struct mv_cesa_op_ctx tmpl
= { };
992 mv_cesa_set_op_cfg(&tmpl
, CESA_SA_DESC_CFG_MACM_SHA256
);
994 mv_cesa_ahash_init(req
, &tmpl
, false);
999 static int mv_cesa_sha256_digest(struct ahash_request
*req
)
1003 ret
= mv_cesa_sha256_init(req
);
1007 return mv_cesa_ahash_finup(req
);
1010 static int mv_cesa_sha256_export(struct ahash_request
*req
, void *out
)
1012 struct sha256_state
*out_state
= out
;
1014 return mv_cesa_ahash_export(req
, out_state
->state
, &out_state
->count
,
1018 static int mv_cesa_sha256_import(struct ahash_request
*req
, const void *in
)
1020 const struct sha256_state
*in_state
= in
;
1022 return mv_cesa_ahash_import(req
, in_state
->state
, in_state
->count
,
1026 struct ahash_alg mv_sha256_alg
= {
1027 .init
= mv_cesa_sha256_init
,
1028 .update
= mv_cesa_ahash_update
,
1029 .final
= mv_cesa_ahash_final
,
1030 .finup
= mv_cesa_ahash_finup
,
1031 .digest
= mv_cesa_sha256_digest
,
1032 .export
= mv_cesa_sha256_export
,
1033 .import
= mv_cesa_sha256_import
,
1035 .digestsize
= SHA256_DIGEST_SIZE
,
1036 .statesize
= sizeof(struct sha256_state
),
1038 .cra_name
= "sha256",
1039 .cra_driver_name
= "mv-sha256",
1040 .cra_priority
= 300,
1041 .cra_flags
= CRYPTO_ALG_ASYNC
|
1042 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1043 .cra_blocksize
= SHA256_BLOCK_SIZE
,
1044 .cra_ctxsize
= sizeof(struct mv_cesa_hash_ctx
),
1045 .cra_init
= mv_cesa_ahash_cra_init
,
1046 .cra_module
= THIS_MODULE
,
1051 struct mv_cesa_ahash_result
{
1052 struct completion completion
;
1056 static void mv_cesa_hmac_ahash_complete(struct crypto_async_request
*req
,
1059 struct mv_cesa_ahash_result
*result
= req
->data
;
1061 if (error
== -EINPROGRESS
)
1064 result
->error
= error
;
1065 complete(&result
->completion
);
1068 static int mv_cesa_ahmac_iv_state_init(struct ahash_request
*req
, u8
*pad
,
1069 void *state
, unsigned int blocksize
)
1071 struct mv_cesa_ahash_result result
;
1072 struct scatterlist sg
;
1075 ahash_request_set_callback(req
, CRYPTO_TFM_REQ_MAY_BACKLOG
,
1076 mv_cesa_hmac_ahash_complete
, &result
);
1077 sg_init_one(&sg
, pad
, blocksize
);
1078 ahash_request_set_crypt(req
, &sg
, pad
, blocksize
);
1079 init_completion(&result
.completion
);
1081 ret
= crypto_ahash_init(req
);
1085 ret
= crypto_ahash_update(req
);
1086 if (ret
&& ret
!= -EINPROGRESS
)
1089 wait_for_completion_interruptible(&result
.completion
);
1091 return result
.error
;
1093 ret
= crypto_ahash_export(req
, state
);
1100 static int mv_cesa_ahmac_pad_init(struct ahash_request
*req
,
1101 const u8
*key
, unsigned int keylen
,
1103 unsigned int blocksize
)
1105 struct mv_cesa_ahash_result result
;
1106 struct scatterlist sg
;
1110 if (keylen
<= blocksize
) {
1111 memcpy(ipad
, key
, keylen
);
1113 u8
*keydup
= kmemdup(key
, keylen
, GFP_KERNEL
);
1118 ahash_request_set_callback(req
, CRYPTO_TFM_REQ_MAY_BACKLOG
,
1119 mv_cesa_hmac_ahash_complete
,
1121 sg_init_one(&sg
, keydup
, keylen
);
1122 ahash_request_set_crypt(req
, &sg
, ipad
, keylen
);
1123 init_completion(&result
.completion
);
1125 ret
= crypto_ahash_digest(req
);
1126 if (ret
== -EINPROGRESS
) {
1127 wait_for_completion_interruptible(&result
.completion
);
1131 /* Set the memory region to 0 to avoid any leak. */
1132 memset(keydup
, 0, keylen
);
1138 keylen
= crypto_ahash_digestsize(crypto_ahash_reqtfm(req
));
1141 memset(ipad
+ keylen
, 0, blocksize
- keylen
);
1142 memcpy(opad
, ipad
, blocksize
);
1144 for (i
= 0; i
< blocksize
; i
++) {
1152 static int mv_cesa_ahmac_setkey(const char *hash_alg_name
,
1153 const u8
*key
, unsigned int keylen
,
1154 void *istate
, void *ostate
)
1156 struct ahash_request
*req
;
1157 struct crypto_ahash
*tfm
;
1158 unsigned int blocksize
;
1163 tfm
= crypto_alloc_ahash(hash_alg_name
, CRYPTO_ALG_TYPE_AHASH
,
1164 CRYPTO_ALG_TYPE_AHASH_MASK
);
1166 return PTR_ERR(tfm
);
1168 req
= ahash_request_alloc(tfm
, GFP_KERNEL
);
1174 crypto_ahash_clear_flags(tfm
, ~0);
1176 blocksize
= crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm
));
1178 ipad
= kzalloc(2 * blocksize
, GFP_KERNEL
);
1184 opad
= ipad
+ blocksize
;
1186 ret
= mv_cesa_ahmac_pad_init(req
, key
, keylen
, ipad
, opad
, blocksize
);
1190 ret
= mv_cesa_ahmac_iv_state_init(req
, ipad
, istate
, blocksize
);
1194 ret
= mv_cesa_ahmac_iv_state_init(req
, opad
, ostate
, blocksize
);
1199 ahash_request_free(req
);
1201 crypto_free_ahash(tfm
);
1206 static int mv_cesa_ahmac_cra_init(struct crypto_tfm
*tfm
)
1208 struct mv_cesa_hmac_ctx
*ctx
= crypto_tfm_ctx(tfm
);
1210 ctx
->base
.ops
= &mv_cesa_ahash_req_ops
;
1212 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
1213 sizeof(struct mv_cesa_ahash_req
));
1217 static int mv_cesa_ahmac_md5_init(struct ahash_request
*req
)
1219 struct mv_cesa_hmac_ctx
*ctx
= crypto_tfm_ctx(req
->base
.tfm
);
1220 struct mv_cesa_op_ctx tmpl
= { };
1222 mv_cesa_set_op_cfg(&tmpl
, CESA_SA_DESC_CFG_MACM_HMAC_MD5
);
1223 memcpy(tmpl
.ctx
.hash
.iv
, ctx
->iv
, sizeof(ctx
->iv
));
1225 mv_cesa_ahash_init(req
, &tmpl
, true);
1230 static int mv_cesa_ahmac_md5_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
1231 unsigned int keylen
)
1233 struct mv_cesa_hmac_ctx
*ctx
= crypto_tfm_ctx(crypto_ahash_tfm(tfm
));
1234 struct md5_state istate
, ostate
;
1237 ret
= mv_cesa_ahmac_setkey("mv-md5", key
, keylen
, &istate
, &ostate
);
1241 for (i
= 0; i
< ARRAY_SIZE(istate
.hash
); i
++)
1242 ctx
->iv
[i
] = be32_to_cpu(istate
.hash
[i
]);
1244 for (i
= 0; i
< ARRAY_SIZE(ostate
.hash
); i
++)
1245 ctx
->iv
[i
+ 8] = be32_to_cpu(ostate
.hash
[i
]);
1250 static int mv_cesa_ahmac_md5_digest(struct ahash_request
*req
)
1254 ret
= mv_cesa_ahmac_md5_init(req
);
1258 return mv_cesa_ahash_finup(req
);
1261 struct ahash_alg mv_ahmac_md5_alg
= {
1262 .init
= mv_cesa_ahmac_md5_init
,
1263 .update
= mv_cesa_ahash_update
,
1264 .final
= mv_cesa_ahash_final
,
1265 .finup
= mv_cesa_ahash_finup
,
1266 .digest
= mv_cesa_ahmac_md5_digest
,
1267 .setkey
= mv_cesa_ahmac_md5_setkey
,
1268 .export
= mv_cesa_md5_export
,
1269 .import
= mv_cesa_md5_import
,
1271 .digestsize
= MD5_DIGEST_SIZE
,
1272 .statesize
= sizeof(struct md5_state
),
1274 .cra_name
= "hmac(md5)",
1275 .cra_driver_name
= "mv-hmac-md5",
1276 .cra_priority
= 300,
1277 .cra_flags
= CRYPTO_ALG_ASYNC
|
1278 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1279 .cra_blocksize
= MD5_HMAC_BLOCK_SIZE
,
1280 .cra_ctxsize
= sizeof(struct mv_cesa_hmac_ctx
),
1281 .cra_init
= mv_cesa_ahmac_cra_init
,
1282 .cra_module
= THIS_MODULE
,
1287 static int mv_cesa_ahmac_sha1_init(struct ahash_request
*req
)
1289 struct mv_cesa_hmac_ctx
*ctx
= crypto_tfm_ctx(req
->base
.tfm
);
1290 struct mv_cesa_op_ctx tmpl
= { };
1292 mv_cesa_set_op_cfg(&tmpl
, CESA_SA_DESC_CFG_MACM_HMAC_SHA1
);
1293 memcpy(tmpl
.ctx
.hash
.iv
, ctx
->iv
, sizeof(ctx
->iv
));
1295 mv_cesa_ahash_init(req
, &tmpl
, false);
1300 static int mv_cesa_ahmac_sha1_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
1301 unsigned int keylen
)
1303 struct mv_cesa_hmac_ctx
*ctx
= crypto_tfm_ctx(crypto_ahash_tfm(tfm
));
1304 struct sha1_state istate
, ostate
;
1307 ret
= mv_cesa_ahmac_setkey("mv-sha1", key
, keylen
, &istate
, &ostate
);
1311 for (i
= 0; i
< ARRAY_SIZE(istate
.state
); i
++)
1312 ctx
->iv
[i
] = be32_to_cpu(istate
.state
[i
]);
1314 for (i
= 0; i
< ARRAY_SIZE(ostate
.state
); i
++)
1315 ctx
->iv
[i
+ 8] = be32_to_cpu(ostate
.state
[i
]);
1320 static int mv_cesa_ahmac_sha1_digest(struct ahash_request
*req
)
1324 ret
= mv_cesa_ahmac_sha1_init(req
);
1328 return mv_cesa_ahash_finup(req
);
1331 struct ahash_alg mv_ahmac_sha1_alg
= {
1332 .init
= mv_cesa_ahmac_sha1_init
,
1333 .update
= mv_cesa_ahash_update
,
1334 .final
= mv_cesa_ahash_final
,
1335 .finup
= mv_cesa_ahash_finup
,
1336 .digest
= mv_cesa_ahmac_sha1_digest
,
1337 .setkey
= mv_cesa_ahmac_sha1_setkey
,
1338 .export
= mv_cesa_sha1_export
,
1339 .import
= mv_cesa_sha1_import
,
1341 .digestsize
= SHA1_DIGEST_SIZE
,
1342 .statesize
= sizeof(struct sha1_state
),
1344 .cra_name
= "hmac(sha1)",
1345 .cra_driver_name
= "mv-hmac-sha1",
1346 .cra_priority
= 300,
1347 .cra_flags
= CRYPTO_ALG_ASYNC
|
1348 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1349 .cra_blocksize
= SHA1_BLOCK_SIZE
,
1350 .cra_ctxsize
= sizeof(struct mv_cesa_hmac_ctx
),
1351 .cra_init
= mv_cesa_ahmac_cra_init
,
1352 .cra_module
= THIS_MODULE
,
1357 static int mv_cesa_ahmac_sha256_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
1358 unsigned int keylen
)
1360 struct mv_cesa_hmac_ctx
*ctx
= crypto_tfm_ctx(crypto_ahash_tfm(tfm
));
1361 struct sha256_state istate
, ostate
;
1364 ret
= mv_cesa_ahmac_setkey("mv-sha256", key
, keylen
, &istate
, &ostate
);
1368 for (i
= 0; i
< ARRAY_SIZE(istate
.state
); i
++)
1369 ctx
->iv
[i
] = be32_to_cpu(istate
.state
[i
]);
1371 for (i
= 0; i
< ARRAY_SIZE(ostate
.state
); i
++)
1372 ctx
->iv
[i
+ 8] = be32_to_cpu(ostate
.state
[i
]);
1377 static int mv_cesa_ahmac_sha256_init(struct ahash_request
*req
)
1379 struct mv_cesa_hmac_ctx
*ctx
= crypto_tfm_ctx(req
->base
.tfm
);
1380 struct mv_cesa_op_ctx tmpl
= { };
1382 mv_cesa_set_op_cfg(&tmpl
, CESA_SA_DESC_CFG_MACM_HMAC_SHA256
);
1383 memcpy(tmpl
.ctx
.hash
.iv
, ctx
->iv
, sizeof(ctx
->iv
));
1385 mv_cesa_ahash_init(req
, &tmpl
, false);
1390 static int mv_cesa_ahmac_sha256_digest(struct ahash_request
*req
)
1394 ret
= mv_cesa_ahmac_sha256_init(req
);
1398 return mv_cesa_ahash_finup(req
);
1401 struct ahash_alg mv_ahmac_sha256_alg
= {
1402 .init
= mv_cesa_ahmac_sha256_init
,
1403 .update
= mv_cesa_ahash_update
,
1404 .final
= mv_cesa_ahash_final
,
1405 .finup
= mv_cesa_ahash_finup
,
1406 .digest
= mv_cesa_ahmac_sha256_digest
,
1407 .setkey
= mv_cesa_ahmac_sha256_setkey
,
1408 .export
= mv_cesa_sha256_export
,
1409 .import
= mv_cesa_sha256_import
,
1411 .digestsize
= SHA256_DIGEST_SIZE
,
1412 .statesize
= sizeof(struct sha256_state
),
1414 .cra_name
= "hmac(sha256)",
1415 .cra_driver_name
= "mv-hmac-sha256",
1416 .cra_priority
= 300,
1417 .cra_flags
= CRYPTO_ALG_ASYNC
|
1418 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1419 .cra_blocksize
= SHA256_BLOCK_SIZE
,
1420 .cra_ctxsize
= sizeof(struct mv_cesa_hmac_ctx
),
1421 .cra_init
= mv_cesa_ahmac_cra_init
,
1422 .cra_module
= THIS_MODULE
,