4 * Support for OMAP SHA1/MD5 HW acceleration.
6 * Copyright (c) 2010 Nokia Corporation
7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as published
11 * by the Free Software Foundation.
13 * Some ideas are from old omap-sha1-md5.c driver.
16 #define pr_fmt(fmt) "%s: " fmt, __func__
18 #include <linux/err.h>
19 #include <linux/device.h>
20 #include <linux/module.h>
21 #include <linux/init.h>
22 #include <linux/errno.h>
23 #include <linux/interrupt.h>
24 #include <linux/kernel.h>
25 #include <linux/clk.h>
26 #include <linux/irq.h>
28 #include <linux/platform_device.h>
29 #include <linux/scatterlist.h>
30 #include <linux/dma-mapping.h>
31 #include <linux/delay.h>
32 #include <linux/crypto.h>
33 #include <linux/cryptohash.h>
34 #include <crypto/scatterwalk.h>
35 #include <crypto/algapi.h>
36 #include <crypto/sha.h>
37 #include <crypto/hash.h>
38 #include <crypto/internal/hash.h>
42 #include <mach/irqs.h>
44 #define SHA_REG_DIGEST(x) (0x00 + ((x) * 0x04))
45 #define SHA_REG_DIN(x) (0x1C + ((x) * 0x04))
47 #define SHA1_MD5_BLOCK_SIZE SHA1_BLOCK_SIZE
48 #define MD5_DIGEST_SIZE 16
50 #define SHA_REG_DIGCNT 0x14
52 #define SHA_REG_CTRL 0x18
53 #define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5)
54 #define SHA_REG_CTRL_CLOSE_HASH (1 << 4)
55 #define SHA_REG_CTRL_ALGO_CONST (1 << 3)
56 #define SHA_REG_CTRL_ALGO (1 << 2)
57 #define SHA_REG_CTRL_INPUT_READY (1 << 1)
58 #define SHA_REG_CTRL_OUTPUT_READY (1 << 0)
60 #define SHA_REG_REV 0x5C
61 #define SHA_REG_REV_MAJOR 0xF0
62 #define SHA_REG_REV_MINOR 0x0F
64 #define SHA_REG_MASK 0x60
65 #define SHA_REG_MASK_DMA_EN (1 << 3)
66 #define SHA_REG_MASK_IT_EN (1 << 2)
67 #define SHA_REG_MASK_SOFTRESET (1 << 1)
68 #define SHA_REG_AUTOIDLE (1 << 0)
70 #define SHA_REG_SYSSTATUS 0x64
71 #define SHA_REG_SYSSTATUS_RESETDONE (1 << 0)
73 #define DEFAULT_TIMEOUT_INTERVAL HZ
75 #define FLAGS_FINUP 0x0002
76 #define FLAGS_FINAL 0x0004
77 #define FLAGS_SG 0x0008
78 #define FLAGS_SHA1 0x0010
79 #define FLAGS_DMA_ACTIVE 0x0020
80 #define FLAGS_OUTPUT_READY 0x0040
81 #define FLAGS_CLEAN 0x0080
82 #define FLAGS_INIT 0x0100
83 #define FLAGS_CPU 0x0200
84 #define FLAGS_HMAC 0x0400
85 #define FLAGS_ERROR 0x0800
86 #define FLAGS_BUSY 0x1000
91 #define OMAP_ALIGN_MASK (sizeof(u32)-1)
92 #define OMAP_ALIGNED __attribute__((aligned(sizeof(u32))))
94 #define BUFLEN PAGE_SIZE
98 struct omap_sham_reqctx
{
99 struct omap_sham_dev
*dd
;
103 u8 digest
[SHA1_DIGEST_SIZE
] OMAP_ALIGNED
;
110 struct scatterlist
*sg
;
111 unsigned int offset
; /* offset in current sg */
112 unsigned int total
; /* total request */
114 u8 buffer
[0] OMAP_ALIGNED
;
117 struct omap_sham_hmac_ctx
{
118 struct crypto_shash
*shash
;
119 u8 ipad
[SHA1_MD5_BLOCK_SIZE
];
120 u8 opad
[SHA1_MD5_BLOCK_SIZE
];
123 struct omap_sham_ctx
{
124 struct omap_sham_dev
*dd
;
129 struct crypto_shash
*fallback
;
131 struct omap_sham_hmac_ctx base
[0];
134 #define OMAP_SHAM_QUEUE_LENGTH 1
136 struct omap_sham_dev
{
137 struct list_head list
;
138 unsigned long phys_base
;
140 void __iomem
*io_base
;
147 struct tasklet_struct done_task
;
148 struct tasklet_struct queue_task
;
151 struct crypto_queue queue
;
152 struct ahash_request
*req
;
155 struct omap_sham_drv
{
156 struct list_head dev_list
;
161 static struct omap_sham_drv sham
= {
162 .dev_list
= LIST_HEAD_INIT(sham
.dev_list
),
163 .lock
= __SPIN_LOCK_UNLOCKED(sham
.lock
),
166 static inline u32
omap_sham_read(struct omap_sham_dev
*dd
, u32 offset
)
168 return __raw_readl(dd
->io_base
+ offset
);
171 static inline void omap_sham_write(struct omap_sham_dev
*dd
,
172 u32 offset
, u32 value
)
174 __raw_writel(value
, dd
->io_base
+ offset
);
177 static inline void omap_sham_write_mask(struct omap_sham_dev
*dd
, u32 address
,
182 val
= omap_sham_read(dd
, address
);
185 omap_sham_write(dd
, address
, val
);
188 static inline int omap_sham_wait(struct omap_sham_dev
*dd
, u32 offset
, u32 bit
)
190 unsigned long timeout
= jiffies
+ DEFAULT_TIMEOUT_INTERVAL
;
192 while (!(omap_sham_read(dd
, offset
) & bit
)) {
193 if (time_is_before_jiffies(timeout
))
200 static void omap_sham_copy_hash(struct ahash_request
*req
, int out
)
202 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
203 u32
*hash
= (u32
*)ctx
->digest
;
206 /* MD5 is almost unused. So copy sha1 size to reduce code */
207 for (i
= 0; i
< SHA1_DIGEST_SIZE
/ sizeof(u32
); i
++) {
209 hash
[i
] = omap_sham_read(ctx
->dd
,
212 omap_sham_write(ctx
->dd
,
213 SHA_REG_DIGEST(i
), hash
[i
]);
217 static void omap_sham_copy_ready_hash(struct ahash_request
*req
)
219 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
220 u32
*in
= (u32
*)ctx
->digest
;
221 u32
*hash
= (u32
*)req
->result
;
227 if (likely(ctx
->flags
& FLAGS_SHA1
)) {
228 /* SHA1 results are in big endian */
229 for (i
= 0; i
< SHA1_DIGEST_SIZE
/ sizeof(u32
); i
++)
230 hash
[i
] = be32_to_cpu(in
[i
]);
232 /* MD5 results are in little endian */
233 for (i
= 0; i
< MD5_DIGEST_SIZE
/ sizeof(u32
); i
++)
234 hash
[i
] = le32_to_cpu(in
[i
]);
238 static int omap_sham_hw_init(struct omap_sham_dev
*dd
)
240 clk_enable(dd
->iclk
);
242 if (!(dd
->flags
& FLAGS_INIT
)) {
243 omap_sham_write_mask(dd
, SHA_REG_MASK
,
244 SHA_REG_MASK_SOFTRESET
, SHA_REG_MASK_SOFTRESET
);
246 if (omap_sham_wait(dd
, SHA_REG_SYSSTATUS
,
247 SHA_REG_SYSSTATUS_RESETDONE
))
250 dd
->flags
|= FLAGS_INIT
;
257 static void omap_sham_write_ctrl(struct omap_sham_dev
*dd
, size_t length
,
260 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
261 u32 val
= length
<< 5, mask
;
263 if (likely(ctx
->digcnt
))
264 omap_sham_write(dd
, SHA_REG_DIGCNT
, ctx
->digcnt
);
266 omap_sham_write_mask(dd
, SHA_REG_MASK
,
267 SHA_REG_MASK_IT_EN
| (dma
? SHA_REG_MASK_DMA_EN
: 0),
268 SHA_REG_MASK_IT_EN
| SHA_REG_MASK_DMA_EN
);
270 * Setting ALGO_CONST only for the first iteration
271 * and CLOSE_HASH only for the last one.
273 if (ctx
->flags
& FLAGS_SHA1
)
274 val
|= SHA_REG_CTRL_ALGO
;
276 val
|= SHA_REG_CTRL_ALGO_CONST
;
278 val
|= SHA_REG_CTRL_CLOSE_HASH
;
280 mask
= SHA_REG_CTRL_ALGO_CONST
| SHA_REG_CTRL_CLOSE_HASH
|
281 SHA_REG_CTRL_ALGO
| SHA_REG_CTRL_LENGTH
;
283 omap_sham_write_mask(dd
, SHA_REG_CTRL
, val
, mask
);
286 static int omap_sham_xmit_cpu(struct omap_sham_dev
*dd
, const u8
*buf
,
287 size_t length
, int final
)
289 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
291 const u32
*buffer
= (const u32
*)buf
;
293 dev_dbg(dd
->dev
, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
294 ctx
->digcnt
, length
, final
);
296 omap_sham_write_ctrl(dd
, length
, final
, 0);
298 /* should be non-zero before next lines to disable clocks later */
299 ctx
->digcnt
+= length
;
301 if (omap_sham_wait(dd
, SHA_REG_CTRL
, SHA_REG_CTRL_INPUT_READY
))
305 ctx
->flags
|= FLAGS_FINAL
; /* catch last interrupt */
307 len32
= DIV_ROUND_UP(length
, sizeof(u32
));
309 for (count
= 0; count
< len32
; count
++)
310 omap_sham_write(dd
, SHA_REG_DIN(count
), buffer
[count
]);
315 static int omap_sham_xmit_dma(struct omap_sham_dev
*dd
, dma_addr_t dma_addr
,
316 size_t length
, int final
)
318 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
321 dev_dbg(dd
->dev
, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
322 ctx
->digcnt
, length
, final
);
324 len32
= DIV_ROUND_UP(length
, sizeof(u32
));
326 omap_set_dma_transfer_params(dd
->dma_lch
, OMAP_DMA_DATA_TYPE_S32
, len32
,
327 1, OMAP_DMA_SYNC_PACKET
, dd
->dma
,
328 OMAP_DMA_DST_SYNC_PREFETCH
);
330 omap_set_dma_src_params(dd
->dma_lch
, 0, OMAP_DMA_AMODE_POST_INC
,
333 omap_sham_write_ctrl(dd
, length
, final
, 1);
335 ctx
->digcnt
+= length
;
338 ctx
->flags
|= FLAGS_FINAL
; /* catch last interrupt */
340 dd
->flags
|= FLAGS_DMA_ACTIVE
;
342 omap_start_dma(dd
->dma_lch
);
347 static size_t omap_sham_append_buffer(struct omap_sham_reqctx
*ctx
,
348 const u8
*data
, size_t length
)
350 size_t count
= min(length
, ctx
->buflen
- ctx
->bufcnt
);
352 count
= min(count
, ctx
->total
);
355 memcpy(ctx
->buffer
+ ctx
->bufcnt
, data
, count
);
356 ctx
->bufcnt
+= count
;
361 static size_t omap_sham_append_sg(struct omap_sham_reqctx
*ctx
)
366 count
= omap_sham_append_buffer(ctx
,
367 sg_virt(ctx
->sg
) + ctx
->offset
,
368 ctx
->sg
->length
- ctx
->offset
);
371 ctx
->offset
+= count
;
373 if (ctx
->offset
== ctx
->sg
->length
) {
374 ctx
->sg
= sg_next(ctx
->sg
);
385 static int omap_sham_xmit_dma_map(struct omap_sham_dev
*dd
,
386 struct omap_sham_reqctx
*ctx
,
387 size_t length
, int final
)
389 ctx
->dma_addr
= dma_map_single(dd
->dev
, ctx
->buffer
, ctx
->buflen
,
391 if (dma_mapping_error(dd
->dev
, ctx
->dma_addr
)) {
392 dev_err(dd
->dev
, "dma %u bytes error\n", ctx
->buflen
);
396 ctx
->flags
&= ~FLAGS_SG
;
398 /* next call does not fail... so no unmap in the case of error */
399 return omap_sham_xmit_dma(dd
, ctx
->dma_addr
, length
, final
);
402 static int omap_sham_update_dma_slow(struct omap_sham_dev
*dd
)
404 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
408 omap_sham_append_sg(ctx
);
410 final
= (ctx
->flags
& FLAGS_FINUP
) && !ctx
->total
;
412 dev_dbg(dd
->dev
, "slow: bufcnt: %u, digcnt: %d, final: %d\n",
413 ctx
->bufcnt
, ctx
->digcnt
, final
);
415 if (final
|| (ctx
->bufcnt
== ctx
->buflen
&& ctx
->total
)) {
418 return omap_sham_xmit_dma_map(dd
, ctx
, count
, final
);
424 /* Start address alignment */
425 #define SG_AA(sg) (IS_ALIGNED(sg->offset, sizeof(u32)))
426 /* SHA1 block size alignment */
427 #define SG_SA(sg) (IS_ALIGNED(sg->length, SHA1_MD5_BLOCK_SIZE))
429 static int omap_sham_update_dma_start(struct omap_sham_dev
*dd
)
431 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
432 unsigned int length
, final
, tail
;
433 struct scatterlist
*sg
;
438 if (ctx
->bufcnt
|| ctx
->offset
)
439 return omap_sham_update_dma_slow(dd
);
441 dev_dbg(dd
->dev
, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
442 ctx
->digcnt
, ctx
->bufcnt
, ctx
->total
);
447 return omap_sham_update_dma_slow(dd
);
449 if (!sg_is_last(sg
) && !SG_SA(sg
))
450 /* size is not SHA1_BLOCK_SIZE aligned */
451 return omap_sham_update_dma_slow(dd
);
453 length
= min(ctx
->total
, sg
->length
);
455 if (sg_is_last(sg
)) {
456 if (!(ctx
->flags
& FLAGS_FINUP
)) {
457 /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */
458 tail
= length
& (SHA1_MD5_BLOCK_SIZE
- 1);
459 /* without finup() we need one block to close hash */
461 tail
= SHA1_MD5_BLOCK_SIZE
;
466 if (!dma_map_sg(dd
->dev
, ctx
->sg
, 1, DMA_TO_DEVICE
)) {
467 dev_err(dd
->dev
, "dma_map_sg error\n");
471 ctx
->flags
|= FLAGS_SG
;
473 ctx
->total
-= length
;
474 ctx
->offset
= length
; /* offset where to start slow */
476 final
= (ctx
->flags
& FLAGS_FINUP
) && !ctx
->total
;
478 /* next call does not fail... so no unmap in the case of error */
479 return omap_sham_xmit_dma(dd
, sg_dma_address(ctx
->sg
), length
, final
);
482 static int omap_sham_update_cpu(struct omap_sham_dev
*dd
)
484 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
487 omap_sham_append_sg(ctx
);
488 bufcnt
= ctx
->bufcnt
;
491 return omap_sham_xmit_cpu(dd
, ctx
->buffer
, bufcnt
, 1);
494 static int omap_sham_update_dma_stop(struct omap_sham_dev
*dd
)
496 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
498 omap_stop_dma(dd
->dma_lch
);
499 if (ctx
->flags
& FLAGS_SG
) {
500 dma_unmap_sg(dd
->dev
, ctx
->sg
, 1, DMA_TO_DEVICE
);
501 if (ctx
->sg
->length
== ctx
->offset
) {
502 ctx
->sg
= sg_next(ctx
->sg
);
507 dma_unmap_single(dd
->dev
, ctx
->dma_addr
, ctx
->buflen
,
514 static void omap_sham_cleanup(struct ahash_request
*req
)
516 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
517 struct omap_sham_dev
*dd
= ctx
->dd
;
520 spin_lock_irqsave(&dd
->lock
, flags
);
521 if (ctx
->flags
& FLAGS_CLEAN
) {
522 spin_unlock_irqrestore(&dd
->lock
, flags
);
525 ctx
->flags
|= FLAGS_CLEAN
;
526 spin_unlock_irqrestore(&dd
->lock
, flags
);
529 omap_sham_copy_ready_hash(req
);
531 dev_dbg(dd
->dev
, "digcnt: %d, bufcnt: %d\n", ctx
->digcnt
, ctx
->bufcnt
);
534 static int omap_sham_init(struct ahash_request
*req
)
536 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
537 struct omap_sham_ctx
*tctx
= crypto_ahash_ctx(tfm
);
538 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
539 struct omap_sham_dev
*dd
= NULL
, *tmp
;
541 spin_lock_bh(&sham
.lock
);
543 list_for_each_entry(tmp
, &sham
.dev_list
, list
) {
551 spin_unlock_bh(&sham
.lock
);
557 dev_dbg(dd
->dev
, "init: digest size: %d\n",
558 crypto_ahash_digestsize(tfm
));
560 if (crypto_ahash_digestsize(tfm
) == SHA1_DIGEST_SIZE
)
561 ctx
->flags
|= FLAGS_SHA1
;
565 ctx
->buflen
= BUFLEN
;
567 if (tctx
->flags
& FLAGS_HMAC
) {
568 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
570 memcpy(ctx
->buffer
, bctx
->ipad
, SHA1_MD5_BLOCK_SIZE
);
571 ctx
->bufcnt
= SHA1_MD5_BLOCK_SIZE
;
572 ctx
->flags
|= FLAGS_HMAC
;
579 static int omap_sham_update_req(struct omap_sham_dev
*dd
)
581 struct ahash_request
*req
= dd
->req
;
582 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
585 dev_dbg(dd
->dev
, "update_req: total: %u, digcnt: %d, finup: %d\n",
586 ctx
->total
, ctx
->digcnt
, (ctx
->flags
& FLAGS_FINUP
) != 0);
588 if (ctx
->flags
& FLAGS_CPU
)
589 err
= omap_sham_update_cpu(dd
);
591 err
= omap_sham_update_dma_start(dd
);
593 /* wait for dma completion before can take more data */
594 dev_dbg(dd
->dev
, "update: err: %d, digcnt: %d\n", err
, ctx
->digcnt
);
599 static int omap_sham_final_req(struct omap_sham_dev
*dd
)
601 struct ahash_request
*req
= dd
->req
;
602 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
603 int err
= 0, use_dma
= 1;
605 if (ctx
->bufcnt
<= 64)
606 /* faster to handle last block with cpu */
610 err
= omap_sham_xmit_dma_map(dd
, ctx
, ctx
->bufcnt
, 1);
612 err
= omap_sham_xmit_cpu(dd
, ctx
->buffer
, ctx
->bufcnt
, 1);
616 dev_dbg(dd
->dev
, "final_req: err: %d\n", err
);
621 static int omap_sham_finish_req_hmac(struct ahash_request
*req
)
623 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
624 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(req
->base
.tfm
);
625 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
626 int bs
= crypto_shash_blocksize(bctx
->shash
);
627 int ds
= crypto_shash_digestsize(bctx
->shash
);
629 struct shash_desc shash
;
630 char ctx
[crypto_shash_descsize(bctx
->shash
)];
633 desc
.shash
.tfm
= bctx
->shash
;
634 desc
.shash
.flags
= 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */
636 return crypto_shash_init(&desc
.shash
) ?:
637 crypto_shash_update(&desc
.shash
, bctx
->opad
, bs
) ?:
638 crypto_shash_finup(&desc
.shash
, ctx
->digest
, ds
, ctx
->digest
);
641 static void omap_sham_finish_req(struct ahash_request
*req
, int err
)
643 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
644 struct omap_sham_dev
*dd
= ctx
->dd
;
647 omap_sham_copy_hash(ctx
->dd
->req
, 1);
648 if (ctx
->flags
& FLAGS_HMAC
)
649 err
= omap_sham_finish_req_hmac(req
);
651 ctx
->flags
|= FLAGS_ERROR
;
654 if ((ctx
->flags
& FLAGS_FINAL
) || err
)
655 omap_sham_cleanup(req
);
657 clk_disable(dd
->iclk
);
658 dd
->flags
&= ~FLAGS_BUSY
;
660 if (req
->base
.complete
)
661 req
->base
.complete(&req
->base
, err
);
664 static int omap_sham_handle_queue(struct omap_sham_dev
*dd
,
665 struct ahash_request
*req
)
667 struct crypto_async_request
*async_req
, *backlog
= 0;
668 struct omap_sham_reqctx
*ctx
;
669 struct ahash_request
*prev_req
;
671 int err
= 0, ret
= 0;
673 spin_lock_irqsave(&dd
->lock
, flags
);
675 ret
= ahash_enqueue_request(&dd
->queue
, req
);
676 if (dd
->flags
& FLAGS_BUSY
) {
677 spin_unlock_irqrestore(&dd
->lock
, flags
);
680 async_req
= crypto_dequeue_request(&dd
->queue
);
682 dd
->flags
|= FLAGS_BUSY
;
683 backlog
= crypto_get_backlog(&dd
->queue
);
685 spin_unlock_irqrestore(&dd
->lock
, flags
);
691 backlog
->complete(backlog
, -EINPROGRESS
);
693 req
= ahash_request_cast(async_req
);
698 ctx
= ahash_request_ctx(req
);
700 dev_dbg(dd
->dev
, "handling new req, op: %lu, nbytes: %d\n",
701 ctx
->op
, req
->nbytes
);
704 err
= omap_sham_hw_init(dd
);
708 omap_set_dma_dest_params(dd
->dma_lch
, 0,
709 OMAP_DMA_AMODE_CONSTANT
,
710 dd
->phys_base
+ SHA_REG_DIN(0), 0, 16);
712 omap_set_dma_dest_burst_mode(dd
->dma_lch
,
713 OMAP_DMA_DATA_BURST_16
);
715 omap_set_dma_src_burst_mode(dd
->dma_lch
,
716 OMAP_DMA_DATA_BURST_4
);
719 /* request has changed - restore hash */
720 omap_sham_copy_hash(req
, 0);
722 if (ctx
->op
== OP_UPDATE
) {
723 err
= omap_sham_update_req(dd
);
724 if (err
!= -EINPROGRESS
&& (ctx
->flags
& FLAGS_FINUP
))
725 /* no final() after finup() */
726 err
= omap_sham_final_req(dd
);
727 } else if (ctx
->op
== OP_FINAL
) {
728 err
= omap_sham_final_req(dd
);
731 if (err
!= -EINPROGRESS
) {
732 /* done_task will not finish it, so do it here */
733 omap_sham_finish_req(req
, err
);
734 tasklet_schedule(&dd
->queue_task
);
737 dev_dbg(dd
->dev
, "exit, err: %d\n", err
);
742 static int omap_sham_enqueue(struct ahash_request
*req
, unsigned int op
)
744 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
745 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(req
->base
.tfm
);
746 struct omap_sham_dev
*dd
= tctx
->dd
;
750 return omap_sham_handle_queue(dd
, req
);
753 static int omap_sham_update(struct ahash_request
*req
)
755 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
760 ctx
->total
= req
->nbytes
;
764 if (ctx
->flags
& FLAGS_FINUP
) {
765 if ((ctx
->digcnt
+ ctx
->bufcnt
+ ctx
->total
) < 9) {
767 * OMAP HW accel works only with buffers >= 9
768 * will switch to bypass in final()
769 * final has the same request and data
771 omap_sham_append_sg(ctx
);
773 } else if (ctx
->bufcnt
+ ctx
->total
<= SHA1_MD5_BLOCK_SIZE
) {
775 * faster to use CPU for short transfers
777 ctx
->flags
|= FLAGS_CPU
;
779 } else if (ctx
->bufcnt
+ ctx
->total
< ctx
->buflen
) {
780 omap_sham_append_sg(ctx
);
784 return omap_sham_enqueue(req
, OP_UPDATE
);
787 static int omap_sham_shash_digest(struct crypto_shash
*shash
, u32 flags
,
788 const u8
*data
, unsigned int len
, u8
*out
)
791 struct shash_desc shash
;
792 char ctx
[crypto_shash_descsize(shash
)];
795 desc
.shash
.tfm
= shash
;
796 desc
.shash
.flags
= flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
798 return crypto_shash_digest(&desc
.shash
, data
, len
, out
);
801 static int omap_sham_final_shash(struct ahash_request
*req
)
803 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(req
->base
.tfm
);
804 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
806 return omap_sham_shash_digest(tctx
->fallback
, req
->base
.flags
,
807 ctx
->buffer
, ctx
->bufcnt
, req
->result
);
810 static int omap_sham_final(struct ahash_request
*req
)
812 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
815 ctx
->flags
|= FLAGS_FINUP
;
817 if (!(ctx
->flags
& FLAGS_ERROR
)) {
818 /* OMAP HW accel works only with buffers >= 9 */
819 /* HMAC is always >= 9 because of ipad */
820 if ((ctx
->digcnt
+ ctx
->bufcnt
) < 9)
821 err
= omap_sham_final_shash(req
);
822 else if (ctx
->bufcnt
)
823 return omap_sham_enqueue(req
, OP_FINAL
);
826 omap_sham_cleanup(req
);
831 static int omap_sham_finup(struct ahash_request
*req
)
833 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
836 ctx
->flags
|= FLAGS_FINUP
;
838 err1
= omap_sham_update(req
);
839 if (err1
== -EINPROGRESS
)
842 * final() has to be always called to cleanup resources
843 * even if udpate() failed, except EINPROGRESS
845 err2
= omap_sham_final(req
);
850 static int omap_sham_digest(struct ahash_request
*req
)
852 return omap_sham_init(req
) ?: omap_sham_finup(req
);
855 static int omap_sham_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
858 struct omap_sham_ctx
*tctx
= crypto_ahash_ctx(tfm
);
859 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
860 int bs
= crypto_shash_blocksize(bctx
->shash
);
861 int ds
= crypto_shash_digestsize(bctx
->shash
);
863 err
= crypto_shash_setkey(tctx
->fallback
, key
, keylen
);
868 err
= omap_sham_shash_digest(bctx
->shash
,
869 crypto_shash_get_flags(bctx
->shash
),
870 key
, keylen
, bctx
->ipad
);
875 memcpy(bctx
->ipad
, key
, keylen
);
878 memset(bctx
->ipad
+ keylen
, 0, bs
- keylen
);
879 memcpy(bctx
->opad
, bctx
->ipad
, bs
);
881 for (i
= 0; i
< bs
; i
++) {
882 bctx
->ipad
[i
] ^= 0x36;
883 bctx
->opad
[i
] ^= 0x5c;
889 static int omap_sham_cra_init_alg(struct crypto_tfm
*tfm
, const char *alg_base
)
891 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(tfm
);
892 const char *alg_name
= crypto_tfm_alg_name(tfm
);
896 /* Allocate a fallback and abort if it failed. */
897 tctx
->fallback
= crypto_alloc_shash(alg_name
, 0,
898 CRYPTO_ALG_NEED_FALLBACK
);
899 if (IS_ERR(tctx
->fallback
)) {
900 pr_err("omap-sham: fallback driver '%s' "
901 "could not be loaded.\n", alg_name
);
902 return PTR_ERR(tctx
->fallback
);
905 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
906 sizeof(struct omap_sham_reqctx
) + BUFLEN
);
909 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
910 tctx
->flags
|= FLAGS_HMAC
;
911 bctx
->shash
= crypto_alloc_shash(alg_base
, 0,
912 CRYPTO_ALG_NEED_FALLBACK
);
913 if (IS_ERR(bctx
->shash
)) {
914 pr_err("omap-sham: base driver '%s' "
915 "could not be loaded.\n", alg_base
);
916 crypto_free_shash(tctx
->fallback
);
917 return PTR_ERR(bctx
->shash
);
925 static int omap_sham_cra_init(struct crypto_tfm
*tfm
)
927 return omap_sham_cra_init_alg(tfm
, NULL
);
930 static int omap_sham_cra_sha1_init(struct crypto_tfm
*tfm
)
932 return omap_sham_cra_init_alg(tfm
, "sha1");
935 static int omap_sham_cra_md5_init(struct crypto_tfm
*tfm
)
937 return omap_sham_cra_init_alg(tfm
, "md5");
940 static void omap_sham_cra_exit(struct crypto_tfm
*tfm
)
942 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(tfm
);
944 crypto_free_shash(tctx
->fallback
);
945 tctx
->fallback
= NULL
;
947 if (tctx
->flags
& FLAGS_HMAC
) {
948 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
949 crypto_free_shash(bctx
->shash
);
953 static struct ahash_alg algs
[] = {
955 .init
= omap_sham_init
,
956 .update
= omap_sham_update
,
957 .final
= omap_sham_final
,
958 .finup
= omap_sham_finup
,
959 .digest
= omap_sham_digest
,
960 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
963 .cra_driver_name
= "omap-sha1",
965 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
967 CRYPTO_ALG_NEED_FALLBACK
,
968 .cra_blocksize
= SHA1_BLOCK_SIZE
,
969 .cra_ctxsize
= sizeof(struct omap_sham_ctx
),
971 .cra_module
= THIS_MODULE
,
972 .cra_init
= omap_sham_cra_init
,
973 .cra_exit
= omap_sham_cra_exit
,
977 .init
= omap_sham_init
,
978 .update
= omap_sham_update
,
979 .final
= omap_sham_final
,
980 .finup
= omap_sham_finup
,
981 .digest
= omap_sham_digest
,
982 .halg
.digestsize
= MD5_DIGEST_SIZE
,
985 .cra_driver_name
= "omap-md5",
987 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
989 CRYPTO_ALG_NEED_FALLBACK
,
990 .cra_blocksize
= SHA1_BLOCK_SIZE
,
991 .cra_ctxsize
= sizeof(struct omap_sham_ctx
),
992 .cra_alignmask
= OMAP_ALIGN_MASK
,
993 .cra_module
= THIS_MODULE
,
994 .cra_init
= omap_sham_cra_init
,
995 .cra_exit
= omap_sham_cra_exit
,
999 .init
= omap_sham_init
,
1000 .update
= omap_sham_update
,
1001 .final
= omap_sham_final
,
1002 .finup
= omap_sham_finup
,
1003 .digest
= omap_sham_digest
,
1004 .setkey
= omap_sham_setkey
,
1005 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
1007 .cra_name
= "hmac(sha1)",
1008 .cra_driver_name
= "omap-hmac-sha1",
1009 .cra_priority
= 100,
1010 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
1012 CRYPTO_ALG_NEED_FALLBACK
,
1013 .cra_blocksize
= SHA1_BLOCK_SIZE
,
1014 .cra_ctxsize
= sizeof(struct omap_sham_ctx
) +
1015 sizeof(struct omap_sham_hmac_ctx
),
1016 .cra_alignmask
= OMAP_ALIGN_MASK
,
1017 .cra_module
= THIS_MODULE
,
1018 .cra_init
= omap_sham_cra_sha1_init
,
1019 .cra_exit
= omap_sham_cra_exit
,
1023 .init
= omap_sham_init
,
1024 .update
= omap_sham_update
,
1025 .final
= omap_sham_final
,
1026 .finup
= omap_sham_finup
,
1027 .digest
= omap_sham_digest
,
1028 .setkey
= omap_sham_setkey
,
1029 .halg
.digestsize
= MD5_DIGEST_SIZE
,
1031 .cra_name
= "hmac(md5)",
1032 .cra_driver_name
= "omap-hmac-md5",
1033 .cra_priority
= 100,
1034 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
1036 CRYPTO_ALG_NEED_FALLBACK
,
1037 .cra_blocksize
= SHA1_BLOCK_SIZE
,
1038 .cra_ctxsize
= sizeof(struct omap_sham_ctx
) +
1039 sizeof(struct omap_sham_hmac_ctx
),
1040 .cra_alignmask
= OMAP_ALIGN_MASK
,
1041 .cra_module
= THIS_MODULE
,
1042 .cra_init
= omap_sham_cra_md5_init
,
1043 .cra_exit
= omap_sham_cra_exit
,
1048 static void omap_sham_done_task(unsigned long data
)
1050 struct omap_sham_dev
*dd
= (struct omap_sham_dev
*)data
;
1051 struct ahash_request
*req
= dd
->req
;
1052 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
1053 int ready
= 0, err
= 0;
1055 if (ctx
->flags
& FLAGS_OUTPUT_READY
) {
1056 ctx
->flags
&= ~FLAGS_OUTPUT_READY
;
1060 if (dd
->flags
& FLAGS_DMA_ACTIVE
) {
1061 dd
->flags
&= ~FLAGS_DMA_ACTIVE
;
1062 omap_sham_update_dma_stop(dd
);
1064 err
= omap_sham_update_dma_start(dd
);
1067 err
= dd
->err
? : err
;
1069 if (err
!= -EINPROGRESS
&& (ready
|| err
)) {
1070 dev_dbg(dd
->dev
, "update done: err: %d\n", err
);
1071 /* finish curent request */
1072 omap_sham_finish_req(req
, err
);
1073 /* start new request */
1074 omap_sham_handle_queue(dd
, NULL
);
1078 static void omap_sham_queue_task(unsigned long data
)
1080 struct omap_sham_dev
*dd
= (struct omap_sham_dev
*)data
;
1082 omap_sham_handle_queue(dd
, NULL
);
1085 static irqreturn_t
omap_sham_irq(int irq
, void *dev_id
)
1087 struct omap_sham_dev
*dd
= dev_id
;
1088 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
1091 dev_err(dd
->dev
, "unknown interrupt.\n");
1095 if (unlikely(ctx
->flags
& FLAGS_FINAL
))
1096 /* final -> allow device to go to power-saving mode */
1097 omap_sham_write_mask(dd
, SHA_REG_CTRL
, 0, SHA_REG_CTRL_LENGTH
);
1099 omap_sham_write_mask(dd
, SHA_REG_CTRL
, SHA_REG_CTRL_OUTPUT_READY
,
1100 SHA_REG_CTRL_OUTPUT_READY
);
1101 omap_sham_read(dd
, SHA_REG_CTRL
);
1103 ctx
->flags
|= FLAGS_OUTPUT_READY
;
1105 tasklet_schedule(&dd
->done_task
);
1110 static void omap_sham_dma_callback(int lch
, u16 ch_status
, void *data
)
1112 struct omap_sham_dev
*dd
= data
;
1114 if (ch_status
!= OMAP_DMA_BLOCK_IRQ
) {
1115 pr_err("omap-sham DMA error status: 0x%hx\n", ch_status
);
1117 dd
->flags
&= ~FLAGS_INIT
; /* request to re-initialize */
1120 tasklet_schedule(&dd
->done_task
);
1123 static int omap_sham_dma_init(struct omap_sham_dev
*dd
)
1129 err
= omap_request_dma(dd
->dma
, dev_name(dd
->dev
),
1130 omap_sham_dma_callback
, dd
, &dd
->dma_lch
);
1132 dev_err(dd
->dev
, "Unable to request DMA channel\n");
1139 static void omap_sham_dma_cleanup(struct omap_sham_dev
*dd
)
1141 if (dd
->dma_lch
>= 0) {
1142 omap_free_dma(dd
->dma_lch
);
1147 static int __devinit
omap_sham_probe(struct platform_device
*pdev
)
1149 struct omap_sham_dev
*dd
;
1150 struct device
*dev
= &pdev
->dev
;
1151 struct resource
*res
;
1154 dd
= kzalloc(sizeof(struct omap_sham_dev
), GFP_KERNEL
);
1156 dev_err(dev
, "unable to alloc data struct.\n");
1161 platform_set_drvdata(pdev
, dd
);
1163 INIT_LIST_HEAD(&dd
->list
);
1164 spin_lock_init(&dd
->lock
);
1165 tasklet_init(&dd
->done_task
, omap_sham_done_task
, (unsigned long)dd
);
1166 tasklet_init(&dd
->queue_task
, omap_sham_queue_task
, (unsigned long)dd
);
1167 crypto_init_queue(&dd
->queue
, OMAP_SHAM_QUEUE_LENGTH
);
1171 /* Get the base address */
1172 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1174 dev_err(dev
, "no MEM resource info\n");
1178 dd
->phys_base
= res
->start
;
1181 res
= platform_get_resource(pdev
, IORESOURCE_DMA
, 0);
1183 dev_err(dev
, "no DMA resource info\n");
1187 dd
->dma
= res
->start
;
1190 dd
->irq
= platform_get_irq(pdev
, 0);
1192 dev_err(dev
, "no IRQ resource info\n");
1197 err
= request_irq(dd
->irq
, omap_sham_irq
,
1198 IRQF_TRIGGER_LOW
, dev_name(dev
), dd
);
1200 dev_err(dev
, "unable to request irq.\n");
1204 err
= omap_sham_dma_init(dd
);
1208 /* Initializing the clock */
1209 dd
->iclk
= clk_get(dev
, "ick");
1211 dev_err(dev
, "clock intialization failed.\n");
1216 dd
->io_base
= ioremap(dd
->phys_base
, SZ_4K
);
1218 dev_err(dev
, "can't ioremap\n");
1223 clk_enable(dd
->iclk
);
1224 dev_info(dev
, "hw accel on OMAP rev %u.%u\n",
1225 (omap_sham_read(dd
, SHA_REG_REV
) & SHA_REG_REV_MAJOR
) >> 4,
1226 omap_sham_read(dd
, SHA_REG_REV
) & SHA_REG_REV_MINOR
);
1227 clk_disable(dd
->iclk
);
1229 spin_lock(&sham
.lock
);
1230 list_add_tail(&dd
->list
, &sham
.dev_list
);
1231 spin_unlock(&sham
.lock
);
1233 for (i
= 0; i
< ARRAY_SIZE(algs
); i
++) {
1234 err
= crypto_register_ahash(&algs
[i
]);
1242 for (j
= 0; j
< i
; j
++)
1243 crypto_unregister_ahash(&algs
[j
]);
1244 iounmap(dd
->io_base
);
1248 omap_sham_dma_cleanup(dd
);
1251 free_irq(dd
->irq
, dd
);
1256 dev_err(dev
, "initialization failed.\n");
1261 static int __devexit
omap_sham_remove(struct platform_device
*pdev
)
1263 static struct omap_sham_dev
*dd
;
1266 dd
= platform_get_drvdata(pdev
);
1269 spin_lock(&sham
.lock
);
1270 list_del(&dd
->list
);
1271 spin_unlock(&sham
.lock
);
1272 for (i
= 0; i
< ARRAY_SIZE(algs
); i
++)
1273 crypto_unregister_ahash(&algs
[i
]);
1274 tasklet_kill(&dd
->done_task
);
1275 tasklet_kill(&dd
->queue_task
);
1276 iounmap(dd
->io_base
);
1278 omap_sham_dma_cleanup(dd
);
1280 free_irq(dd
->irq
, dd
);
1287 static struct platform_driver omap_sham_driver
= {
1288 .probe
= omap_sham_probe
,
1289 .remove
= omap_sham_remove
,
1291 .name
= "omap-sham",
1292 .owner
= THIS_MODULE
,
1296 static int __init
omap_sham_mod_init(void)
1298 pr_info("loading %s driver\n", "omap-sham");
1300 if (!cpu_class_is_omap2() ||
1301 omap_type() != OMAP2_DEVICE_TYPE_SEC
) {
1302 pr_err("Unsupported cpu\n");
1306 return platform_driver_register(&omap_sham_driver
);
1309 static void __exit
omap_sham_mod_exit(void)
1311 platform_driver_unregister(&omap_sham_driver
);
1314 module_init(omap_sham_mod_init
);
1315 module_exit(omap_sham_mod_exit
);
1317 MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support.");
1318 MODULE_LICENSE("GPL v2");
1319 MODULE_AUTHOR("Dmitry Kasatkin");