4 * Support for OMAP SHA1/MD5 HW acceleration.
6 * Copyright (c) 2010 Nokia Corporation
7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as published
11 * by the Free Software Foundation.
13 * Some ideas are from old omap-sha1-md5.c driver.
16 #define pr_fmt(fmt) "%s: " fmt, __func__
18 #include <linux/err.h>
19 #include <linux/device.h>
20 #include <linux/module.h>
21 #include <linux/init.h>
22 #include <linux/errno.h>
23 #include <linux/interrupt.h>
24 #include <linux/kernel.h>
25 #include <linux/clk.h>
26 #include <linux/irq.h>
28 #include <linux/platform_device.h>
29 #include <linux/scatterlist.h>
30 #include <linux/dma-mapping.h>
31 #include <linux/delay.h>
32 #include <linux/crypto.h>
33 #include <linux/cryptohash.h>
34 #include <crypto/scatterwalk.h>
35 #include <crypto/algapi.h>
36 #include <crypto/sha.h>
37 #include <crypto/hash.h>
38 #include <crypto/internal/hash.h>
42 #include <mach/irqs.h>
44 #define SHA_REG_DIGEST(x) (0x00 + ((x) * 0x04))
45 #define SHA_REG_DIN(x) (0x1C + ((x) * 0x04))
47 #define SHA1_MD5_BLOCK_SIZE SHA1_BLOCK_SIZE
48 #define MD5_DIGEST_SIZE 16
50 #define SHA_REG_DIGCNT 0x14
52 #define SHA_REG_CTRL 0x18
53 #define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5)
54 #define SHA_REG_CTRL_CLOSE_HASH (1 << 4)
55 #define SHA_REG_CTRL_ALGO_CONST (1 << 3)
56 #define SHA_REG_CTRL_ALGO (1 << 2)
57 #define SHA_REG_CTRL_INPUT_READY (1 << 1)
58 #define SHA_REG_CTRL_OUTPUT_READY (1 << 0)
60 #define SHA_REG_REV 0x5C
61 #define SHA_REG_REV_MAJOR 0xF0
62 #define SHA_REG_REV_MINOR 0x0F
64 #define SHA_REG_MASK 0x60
65 #define SHA_REG_MASK_DMA_EN (1 << 3)
66 #define SHA_REG_MASK_IT_EN (1 << 2)
67 #define SHA_REG_MASK_SOFTRESET (1 << 1)
68 #define SHA_REG_AUTOIDLE (1 << 0)
70 #define SHA_REG_SYSSTATUS 0x64
71 #define SHA_REG_SYSSTATUS_RESETDONE (1 << 0)
73 #define DEFAULT_TIMEOUT_INTERVAL HZ
75 /* mostly device flags */
78 #define FLAGS_DMA_ACTIVE 2
79 #define FLAGS_OUTPUT_READY 3
82 #define FLAGS_DMA_READY 6
84 #define FLAGS_FINUP 16
88 #define FLAGS_ERROR 20
93 #define OMAP_ALIGN_MASK (sizeof(u32)-1)
94 #define OMAP_ALIGNED __attribute__((aligned(sizeof(u32))))
96 #define BUFLEN PAGE_SIZE
100 struct omap_sham_reqctx
{
101 struct omap_sham_dev
*dd
;
105 u8 digest
[SHA1_DIGEST_SIZE
] OMAP_ALIGNED
;
112 struct scatterlist
*sg
;
113 unsigned int offset
; /* offset in current sg */
114 unsigned int total
; /* total request */
116 u8 buffer
[0] OMAP_ALIGNED
;
119 struct omap_sham_hmac_ctx
{
120 struct crypto_shash
*shash
;
121 u8 ipad
[SHA1_MD5_BLOCK_SIZE
];
122 u8 opad
[SHA1_MD5_BLOCK_SIZE
];
125 struct omap_sham_ctx
{
126 struct omap_sham_dev
*dd
;
131 struct crypto_shash
*fallback
;
133 struct omap_sham_hmac_ctx base
[0];
136 #define OMAP_SHAM_QUEUE_LENGTH 1
138 struct omap_sham_dev
{
139 struct list_head list
;
140 unsigned long phys_base
;
142 void __iomem
*io_base
;
149 struct tasklet_struct done_task
;
152 struct crypto_queue queue
;
153 struct ahash_request
*req
;
156 struct omap_sham_drv
{
157 struct list_head dev_list
;
162 static struct omap_sham_drv sham
= {
163 .dev_list
= LIST_HEAD_INIT(sham
.dev_list
),
164 .lock
= __SPIN_LOCK_UNLOCKED(sham
.lock
),
167 static inline u32
omap_sham_read(struct omap_sham_dev
*dd
, u32 offset
)
169 return __raw_readl(dd
->io_base
+ offset
);
172 static inline void omap_sham_write(struct omap_sham_dev
*dd
,
173 u32 offset
, u32 value
)
175 __raw_writel(value
, dd
->io_base
+ offset
);
178 static inline void omap_sham_write_mask(struct omap_sham_dev
*dd
, u32 address
,
183 val
= omap_sham_read(dd
, address
);
186 omap_sham_write(dd
, address
, val
);
189 static inline int omap_sham_wait(struct omap_sham_dev
*dd
, u32 offset
, u32 bit
)
191 unsigned long timeout
= jiffies
+ DEFAULT_TIMEOUT_INTERVAL
;
193 while (!(omap_sham_read(dd
, offset
) & bit
)) {
194 if (time_is_before_jiffies(timeout
))
201 static void omap_sham_copy_hash(struct ahash_request
*req
, int out
)
203 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
204 u32
*hash
= (u32
*)ctx
->digest
;
207 /* MD5 is almost unused. So copy sha1 size to reduce code */
208 for (i
= 0; i
< SHA1_DIGEST_SIZE
/ sizeof(u32
); i
++) {
210 hash
[i
] = omap_sham_read(ctx
->dd
,
213 omap_sham_write(ctx
->dd
,
214 SHA_REG_DIGEST(i
), hash
[i
]);
218 static void omap_sham_copy_ready_hash(struct ahash_request
*req
)
220 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
221 u32
*in
= (u32
*)ctx
->digest
;
222 u32
*hash
= (u32
*)req
->result
;
228 if (likely(ctx
->flags
& BIT(FLAGS_SHA1
))) {
229 /* SHA1 results are in big endian */
230 for (i
= 0; i
< SHA1_DIGEST_SIZE
/ sizeof(u32
); i
++)
231 hash
[i
] = be32_to_cpu(in
[i
]);
233 /* MD5 results are in little endian */
234 for (i
= 0; i
< MD5_DIGEST_SIZE
/ sizeof(u32
); i
++)
235 hash
[i
] = le32_to_cpu(in
[i
]);
239 static int omap_sham_hw_init(struct omap_sham_dev
*dd
)
241 clk_enable(dd
->iclk
);
243 if (!test_bit(FLAGS_INIT
, &dd
->flags
)) {
244 omap_sham_write_mask(dd
, SHA_REG_MASK
,
245 SHA_REG_MASK_SOFTRESET
, SHA_REG_MASK_SOFTRESET
);
247 if (omap_sham_wait(dd
, SHA_REG_SYSSTATUS
,
248 SHA_REG_SYSSTATUS_RESETDONE
))
251 set_bit(FLAGS_INIT
, &dd
->flags
);
258 static void omap_sham_write_ctrl(struct omap_sham_dev
*dd
, size_t length
,
261 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
262 u32 val
= length
<< 5, mask
;
264 if (likely(ctx
->digcnt
))
265 omap_sham_write(dd
, SHA_REG_DIGCNT
, ctx
->digcnt
);
267 omap_sham_write_mask(dd
, SHA_REG_MASK
,
268 SHA_REG_MASK_IT_EN
| (dma
? SHA_REG_MASK_DMA_EN
: 0),
269 SHA_REG_MASK_IT_EN
| SHA_REG_MASK_DMA_EN
);
271 * Setting ALGO_CONST only for the first iteration
272 * and CLOSE_HASH only for the last one.
274 if (ctx
->flags
& BIT(FLAGS_SHA1
))
275 val
|= SHA_REG_CTRL_ALGO
;
277 val
|= SHA_REG_CTRL_ALGO_CONST
;
279 val
|= SHA_REG_CTRL_CLOSE_HASH
;
281 mask
= SHA_REG_CTRL_ALGO_CONST
| SHA_REG_CTRL_CLOSE_HASH
|
282 SHA_REG_CTRL_ALGO
| SHA_REG_CTRL_LENGTH
;
284 omap_sham_write_mask(dd
, SHA_REG_CTRL
, val
, mask
);
287 static int omap_sham_xmit_cpu(struct omap_sham_dev
*dd
, const u8
*buf
,
288 size_t length
, int final
)
290 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
292 const u32
*buffer
= (const u32
*)buf
;
294 dev_dbg(dd
->dev
, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
295 ctx
->digcnt
, length
, final
);
297 omap_sham_write_ctrl(dd
, length
, final
, 0);
299 /* should be non-zero before next lines to disable clocks later */
300 ctx
->digcnt
+= length
;
302 if (omap_sham_wait(dd
, SHA_REG_CTRL
, SHA_REG_CTRL_INPUT_READY
))
306 set_bit(FLAGS_FINAL
, &dd
->flags
); /* catch last interrupt */
308 set_bit(FLAGS_CPU
, &dd
->flags
);
310 len32
= DIV_ROUND_UP(length
, sizeof(u32
));
312 for (count
= 0; count
< len32
; count
++)
313 omap_sham_write(dd
, SHA_REG_DIN(count
), buffer
[count
]);
318 static int omap_sham_xmit_dma(struct omap_sham_dev
*dd
, dma_addr_t dma_addr
,
319 size_t length
, int final
)
321 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
324 dev_dbg(dd
->dev
, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
325 ctx
->digcnt
, length
, final
);
327 len32
= DIV_ROUND_UP(length
, sizeof(u32
));
329 omap_set_dma_transfer_params(dd
->dma_lch
, OMAP_DMA_DATA_TYPE_S32
, len32
,
330 1, OMAP_DMA_SYNC_PACKET
, dd
->dma
,
331 OMAP_DMA_DST_SYNC_PREFETCH
);
333 omap_set_dma_src_params(dd
->dma_lch
, 0, OMAP_DMA_AMODE_POST_INC
,
336 omap_sham_write_ctrl(dd
, length
, final
, 1);
338 ctx
->digcnt
+= length
;
341 set_bit(FLAGS_FINAL
, &dd
->flags
); /* catch last interrupt */
343 set_bit(FLAGS_DMA_ACTIVE
, &dd
->flags
);
345 omap_start_dma(dd
->dma_lch
);
350 static size_t omap_sham_append_buffer(struct omap_sham_reqctx
*ctx
,
351 const u8
*data
, size_t length
)
353 size_t count
= min(length
, ctx
->buflen
- ctx
->bufcnt
);
355 count
= min(count
, ctx
->total
);
358 memcpy(ctx
->buffer
+ ctx
->bufcnt
, data
, count
);
359 ctx
->bufcnt
+= count
;
364 static size_t omap_sham_append_sg(struct omap_sham_reqctx
*ctx
)
369 count
= omap_sham_append_buffer(ctx
,
370 sg_virt(ctx
->sg
) + ctx
->offset
,
371 ctx
->sg
->length
- ctx
->offset
);
374 ctx
->offset
+= count
;
376 if (ctx
->offset
== ctx
->sg
->length
) {
377 ctx
->sg
= sg_next(ctx
->sg
);
388 static int omap_sham_xmit_dma_map(struct omap_sham_dev
*dd
,
389 struct omap_sham_reqctx
*ctx
,
390 size_t length
, int final
)
392 ctx
->dma_addr
= dma_map_single(dd
->dev
, ctx
->buffer
, ctx
->buflen
,
394 if (dma_mapping_error(dd
->dev
, ctx
->dma_addr
)) {
395 dev_err(dd
->dev
, "dma %u bytes error\n", ctx
->buflen
);
399 ctx
->flags
&= ~BIT(FLAGS_SG
);
401 /* next call does not fail... so no unmap in the case of error */
402 return omap_sham_xmit_dma(dd
, ctx
->dma_addr
, length
, final
);
405 static int omap_sham_update_dma_slow(struct omap_sham_dev
*dd
)
407 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
411 omap_sham_append_sg(ctx
);
413 final
= (ctx
->flags
& BIT(FLAGS_FINUP
)) && !ctx
->total
;
415 dev_dbg(dd
->dev
, "slow: bufcnt: %u, digcnt: %d, final: %d\n",
416 ctx
->bufcnt
, ctx
->digcnt
, final
);
418 if (final
|| (ctx
->bufcnt
== ctx
->buflen
&& ctx
->total
)) {
421 return omap_sham_xmit_dma_map(dd
, ctx
, count
, final
);
427 /* Start address alignment */
428 #define SG_AA(sg) (IS_ALIGNED(sg->offset, sizeof(u32)))
429 /* SHA1 block size alignment */
430 #define SG_SA(sg) (IS_ALIGNED(sg->length, SHA1_MD5_BLOCK_SIZE))
432 static int omap_sham_update_dma_start(struct omap_sham_dev
*dd
)
434 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
435 unsigned int length
, final
, tail
;
436 struct scatterlist
*sg
;
441 if (ctx
->bufcnt
|| ctx
->offset
)
442 return omap_sham_update_dma_slow(dd
);
444 dev_dbg(dd
->dev
, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
445 ctx
->digcnt
, ctx
->bufcnt
, ctx
->total
);
450 return omap_sham_update_dma_slow(dd
);
452 if (!sg_is_last(sg
) && !SG_SA(sg
))
453 /* size is not SHA1_BLOCK_SIZE aligned */
454 return omap_sham_update_dma_slow(dd
);
456 length
= min(ctx
->total
, sg
->length
);
458 if (sg_is_last(sg
)) {
459 if (!(ctx
->flags
& BIT(FLAGS_FINUP
))) {
460 /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */
461 tail
= length
& (SHA1_MD5_BLOCK_SIZE
- 1);
462 /* without finup() we need one block to close hash */
464 tail
= SHA1_MD5_BLOCK_SIZE
;
469 if (!dma_map_sg(dd
->dev
, ctx
->sg
, 1, DMA_TO_DEVICE
)) {
470 dev_err(dd
->dev
, "dma_map_sg error\n");
474 ctx
->flags
|= BIT(FLAGS_SG
);
476 ctx
->total
-= length
;
477 ctx
->offset
= length
; /* offset where to start slow */
479 final
= (ctx
->flags
& BIT(FLAGS_FINUP
)) && !ctx
->total
;
481 /* next call does not fail... so no unmap in the case of error */
482 return omap_sham_xmit_dma(dd
, sg_dma_address(ctx
->sg
), length
, final
);
485 static int omap_sham_update_cpu(struct omap_sham_dev
*dd
)
487 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
490 omap_sham_append_sg(ctx
);
491 bufcnt
= ctx
->bufcnt
;
494 return omap_sham_xmit_cpu(dd
, ctx
->buffer
, bufcnt
, 1);
497 static int omap_sham_update_dma_stop(struct omap_sham_dev
*dd
)
499 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
501 omap_stop_dma(dd
->dma_lch
);
502 if (ctx
->flags
& BIT(FLAGS_SG
)) {
503 dma_unmap_sg(dd
->dev
, ctx
->sg
, 1, DMA_TO_DEVICE
);
504 if (ctx
->sg
->length
== ctx
->offset
) {
505 ctx
->sg
= sg_next(ctx
->sg
);
510 dma_unmap_single(dd
->dev
, ctx
->dma_addr
, ctx
->buflen
,
517 static int omap_sham_init(struct ahash_request
*req
)
519 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
520 struct omap_sham_ctx
*tctx
= crypto_ahash_ctx(tfm
);
521 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
522 struct omap_sham_dev
*dd
= NULL
, *tmp
;
524 spin_lock_bh(&sham
.lock
);
526 list_for_each_entry(tmp
, &sham
.dev_list
, list
) {
534 spin_unlock_bh(&sham
.lock
);
540 dev_dbg(dd
->dev
, "init: digest size: %d\n",
541 crypto_ahash_digestsize(tfm
));
543 if (crypto_ahash_digestsize(tfm
) == SHA1_DIGEST_SIZE
)
544 ctx
->flags
|= BIT(FLAGS_SHA1
);
548 ctx
->buflen
= BUFLEN
;
550 if (tctx
->flags
& BIT(FLAGS_HMAC
)) {
551 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
553 memcpy(ctx
->buffer
, bctx
->ipad
, SHA1_MD5_BLOCK_SIZE
);
554 ctx
->bufcnt
= SHA1_MD5_BLOCK_SIZE
;
555 ctx
->flags
|= BIT(FLAGS_HMAC
);
562 static int omap_sham_update_req(struct omap_sham_dev
*dd
)
564 struct ahash_request
*req
= dd
->req
;
565 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
568 dev_dbg(dd
->dev
, "update_req: total: %u, digcnt: %d, finup: %d\n",
569 ctx
->total
, ctx
->digcnt
, (ctx
->flags
& BIT(FLAGS_FINUP
)) != 0);
571 if (ctx
->flags
& BIT(FLAGS_CPU
))
572 err
= omap_sham_update_cpu(dd
);
574 err
= omap_sham_update_dma_start(dd
);
576 /* wait for dma completion before can take more data */
577 dev_dbg(dd
->dev
, "update: err: %d, digcnt: %d\n", err
, ctx
->digcnt
);
582 static int omap_sham_final_req(struct omap_sham_dev
*dd
)
584 struct ahash_request
*req
= dd
->req
;
585 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
586 int err
= 0, use_dma
= 1;
588 if (ctx
->bufcnt
<= 64)
589 /* faster to handle last block with cpu */
593 err
= omap_sham_xmit_dma_map(dd
, ctx
, ctx
->bufcnt
, 1);
595 err
= omap_sham_xmit_cpu(dd
, ctx
->buffer
, ctx
->bufcnt
, 1);
599 dev_dbg(dd
->dev
, "final_req: err: %d\n", err
);
604 static int omap_sham_finish_hmac(struct ahash_request
*req
)
606 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(req
->base
.tfm
);
607 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
608 int bs
= crypto_shash_blocksize(bctx
->shash
);
609 int ds
= crypto_shash_digestsize(bctx
->shash
);
611 struct shash_desc shash
;
612 char ctx
[crypto_shash_descsize(bctx
->shash
)];
615 desc
.shash
.tfm
= bctx
->shash
;
616 desc
.shash
.flags
= 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */
618 return crypto_shash_init(&desc
.shash
) ?:
619 crypto_shash_update(&desc
.shash
, bctx
->opad
, bs
) ?:
620 crypto_shash_finup(&desc
.shash
, req
->result
, ds
, req
->result
);
623 static int omap_sham_finish(struct ahash_request
*req
)
625 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
626 struct omap_sham_dev
*dd
= ctx
->dd
;
630 omap_sham_copy_ready_hash(req
);
631 if (ctx
->flags
& BIT(FLAGS_HMAC
))
632 err
= omap_sham_finish_hmac(req
);
635 dev_dbg(dd
->dev
, "digcnt: %d, bufcnt: %d\n", ctx
->digcnt
, ctx
->bufcnt
);
640 static void omap_sham_finish_req(struct ahash_request
*req
, int err
)
642 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
643 struct omap_sham_dev
*dd
= ctx
->dd
;
646 omap_sham_copy_hash(req
, 1);
647 if (test_bit(FLAGS_FINAL
, &dd
->flags
))
648 err
= omap_sham_finish(req
);
650 ctx
->flags
|= BIT(FLAGS_ERROR
);
653 clk_disable(dd
->iclk
);
654 dd
->flags
&= ~BIT(FLAGS_BUSY
);
656 if (req
->base
.complete
)
657 req
->base
.complete(&req
->base
, err
);
659 /* handle new request */
660 tasklet_schedule(&dd
->done_task
);
663 static int omap_sham_handle_queue(struct omap_sham_dev
*dd
,
664 struct ahash_request
*req
)
666 struct crypto_async_request
*async_req
, *backlog
;
667 struct omap_sham_reqctx
*ctx
;
669 int err
= 0, ret
= 0;
671 spin_lock_irqsave(&dd
->lock
, flags
);
673 ret
= ahash_enqueue_request(&dd
->queue
, req
);
674 if (test_bit(FLAGS_BUSY
, &dd
->flags
)) {
675 spin_unlock_irqrestore(&dd
->lock
, flags
);
678 backlog
= crypto_get_backlog(&dd
->queue
);
679 async_req
= crypto_dequeue_request(&dd
->queue
);
681 set_bit(FLAGS_BUSY
, &dd
->flags
);
682 spin_unlock_irqrestore(&dd
->lock
, flags
);
688 backlog
->complete(backlog
, -EINPROGRESS
);
690 req
= ahash_request_cast(async_req
);
692 ctx
= ahash_request_ctx(req
);
694 dev_dbg(dd
->dev
, "handling new req, op: %lu, nbytes: %d\n",
695 ctx
->op
, req
->nbytes
);
697 err
= omap_sham_hw_init(dd
);
701 omap_set_dma_dest_params(dd
->dma_lch
, 0,
702 OMAP_DMA_AMODE_CONSTANT
,
703 dd
->phys_base
+ SHA_REG_DIN(0), 0, 16);
705 omap_set_dma_dest_burst_mode(dd
->dma_lch
,
706 OMAP_DMA_DATA_BURST_16
);
708 omap_set_dma_src_burst_mode(dd
->dma_lch
,
709 OMAP_DMA_DATA_BURST_4
);
712 /* request has changed - restore hash */
713 omap_sham_copy_hash(req
, 0);
715 if (ctx
->op
== OP_UPDATE
) {
716 err
= omap_sham_update_req(dd
);
717 if (err
!= -EINPROGRESS
&& (ctx
->flags
& BIT(FLAGS_FINUP
)))
718 /* no final() after finup() */
719 err
= omap_sham_final_req(dd
);
720 } else if (ctx
->op
== OP_FINAL
) {
721 err
= omap_sham_final_req(dd
);
724 if (err
!= -EINPROGRESS
)
725 /* done_task will not finish it, so do it here */
726 omap_sham_finish_req(req
, err
);
728 dev_dbg(dd
->dev
, "exit, err: %d\n", err
);
733 static int omap_sham_enqueue(struct ahash_request
*req
, unsigned int op
)
735 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
736 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(req
->base
.tfm
);
737 struct omap_sham_dev
*dd
= tctx
->dd
;
741 return omap_sham_handle_queue(dd
, req
);
744 static int omap_sham_update(struct ahash_request
*req
)
746 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
751 ctx
->total
= req
->nbytes
;
755 if (ctx
->flags
& BIT(FLAGS_FINUP
)) {
756 if ((ctx
->digcnt
+ ctx
->bufcnt
+ ctx
->total
) < 9) {
758 * OMAP HW accel works only with buffers >= 9
759 * will switch to bypass in final()
760 * final has the same request and data
762 omap_sham_append_sg(ctx
);
764 } else if (ctx
->bufcnt
+ ctx
->total
<= SHA1_MD5_BLOCK_SIZE
) {
766 * faster to use CPU for short transfers
768 ctx
->flags
|= BIT(FLAGS_CPU
);
770 } else if (ctx
->bufcnt
+ ctx
->total
< ctx
->buflen
) {
771 omap_sham_append_sg(ctx
);
775 return omap_sham_enqueue(req
, OP_UPDATE
);
778 static int omap_sham_shash_digest(struct crypto_shash
*shash
, u32 flags
,
779 const u8
*data
, unsigned int len
, u8
*out
)
782 struct shash_desc shash
;
783 char ctx
[crypto_shash_descsize(shash
)];
786 desc
.shash
.tfm
= shash
;
787 desc
.shash
.flags
= flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
789 return crypto_shash_digest(&desc
.shash
, data
, len
, out
);
792 static int omap_sham_final_shash(struct ahash_request
*req
)
794 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(req
->base
.tfm
);
795 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
797 return omap_sham_shash_digest(tctx
->fallback
, req
->base
.flags
,
798 ctx
->buffer
, ctx
->bufcnt
, req
->result
);
801 static int omap_sham_final(struct ahash_request
*req
)
803 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
805 ctx
->flags
|= BIT(FLAGS_FINUP
);
807 if (ctx
->flags
& BIT(FLAGS_ERROR
))
808 return 0; /* uncompleted hash is not needed */
810 /* OMAP HW accel works only with buffers >= 9 */
811 /* HMAC is always >= 9 because ipad == block size */
812 if ((ctx
->digcnt
+ ctx
->bufcnt
) < 9)
813 return omap_sham_final_shash(req
);
814 else if (ctx
->bufcnt
)
815 return omap_sham_enqueue(req
, OP_FINAL
);
817 /* copy ready hash (+ finalize hmac) */
818 return omap_sham_finish(req
);
821 static int omap_sham_finup(struct ahash_request
*req
)
823 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
826 ctx
->flags
|= BIT(FLAGS_FINUP
);
828 err1
= omap_sham_update(req
);
829 if (err1
== -EINPROGRESS
|| err1
== -EBUSY
)
832 * final() has to be always called to cleanup resources
833 * even if udpate() failed, except EINPROGRESS
835 err2
= omap_sham_final(req
);
840 static int omap_sham_digest(struct ahash_request
*req
)
842 return omap_sham_init(req
) ?: omap_sham_finup(req
);
845 static int omap_sham_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
848 struct omap_sham_ctx
*tctx
= crypto_ahash_ctx(tfm
);
849 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
850 int bs
= crypto_shash_blocksize(bctx
->shash
);
851 int ds
= crypto_shash_digestsize(bctx
->shash
);
853 err
= crypto_shash_setkey(tctx
->fallback
, key
, keylen
);
858 err
= omap_sham_shash_digest(bctx
->shash
,
859 crypto_shash_get_flags(bctx
->shash
),
860 key
, keylen
, bctx
->ipad
);
865 memcpy(bctx
->ipad
, key
, keylen
);
868 memset(bctx
->ipad
+ keylen
, 0, bs
- keylen
);
869 memcpy(bctx
->opad
, bctx
->ipad
, bs
);
871 for (i
= 0; i
< bs
; i
++) {
872 bctx
->ipad
[i
] ^= 0x36;
873 bctx
->opad
[i
] ^= 0x5c;
879 static int omap_sham_cra_init_alg(struct crypto_tfm
*tfm
, const char *alg_base
)
881 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(tfm
);
882 const char *alg_name
= crypto_tfm_alg_name(tfm
);
884 /* Allocate a fallback and abort if it failed. */
885 tctx
->fallback
= crypto_alloc_shash(alg_name
, 0,
886 CRYPTO_ALG_NEED_FALLBACK
);
887 if (IS_ERR(tctx
->fallback
)) {
888 pr_err("omap-sham: fallback driver '%s' "
889 "could not be loaded.\n", alg_name
);
890 return PTR_ERR(tctx
->fallback
);
893 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
894 sizeof(struct omap_sham_reqctx
) + BUFLEN
);
897 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
898 tctx
->flags
|= BIT(FLAGS_HMAC
);
899 bctx
->shash
= crypto_alloc_shash(alg_base
, 0,
900 CRYPTO_ALG_NEED_FALLBACK
);
901 if (IS_ERR(bctx
->shash
)) {
902 pr_err("omap-sham: base driver '%s' "
903 "could not be loaded.\n", alg_base
);
904 crypto_free_shash(tctx
->fallback
);
905 return PTR_ERR(bctx
->shash
);
913 static int omap_sham_cra_init(struct crypto_tfm
*tfm
)
915 return omap_sham_cra_init_alg(tfm
, NULL
);
918 static int omap_sham_cra_sha1_init(struct crypto_tfm
*tfm
)
920 return omap_sham_cra_init_alg(tfm
, "sha1");
923 static int omap_sham_cra_md5_init(struct crypto_tfm
*tfm
)
925 return omap_sham_cra_init_alg(tfm
, "md5");
928 static void omap_sham_cra_exit(struct crypto_tfm
*tfm
)
930 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(tfm
);
932 crypto_free_shash(tctx
->fallback
);
933 tctx
->fallback
= NULL
;
935 if (tctx
->flags
& BIT(FLAGS_HMAC
)) {
936 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
937 crypto_free_shash(bctx
->shash
);
941 static struct ahash_alg algs
[] = {
943 .init
= omap_sham_init
,
944 .update
= omap_sham_update
,
945 .final
= omap_sham_final
,
946 .finup
= omap_sham_finup
,
947 .digest
= omap_sham_digest
,
948 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
951 .cra_driver_name
= "omap-sha1",
953 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
955 CRYPTO_ALG_NEED_FALLBACK
,
956 .cra_blocksize
= SHA1_BLOCK_SIZE
,
957 .cra_ctxsize
= sizeof(struct omap_sham_ctx
),
959 .cra_module
= THIS_MODULE
,
960 .cra_init
= omap_sham_cra_init
,
961 .cra_exit
= omap_sham_cra_exit
,
965 .init
= omap_sham_init
,
966 .update
= omap_sham_update
,
967 .final
= omap_sham_final
,
968 .finup
= omap_sham_finup
,
969 .digest
= omap_sham_digest
,
970 .halg
.digestsize
= MD5_DIGEST_SIZE
,
973 .cra_driver_name
= "omap-md5",
975 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
977 CRYPTO_ALG_NEED_FALLBACK
,
978 .cra_blocksize
= SHA1_BLOCK_SIZE
,
979 .cra_ctxsize
= sizeof(struct omap_sham_ctx
),
980 .cra_alignmask
= OMAP_ALIGN_MASK
,
981 .cra_module
= THIS_MODULE
,
982 .cra_init
= omap_sham_cra_init
,
983 .cra_exit
= omap_sham_cra_exit
,
987 .init
= omap_sham_init
,
988 .update
= omap_sham_update
,
989 .final
= omap_sham_final
,
990 .finup
= omap_sham_finup
,
991 .digest
= omap_sham_digest
,
992 .setkey
= omap_sham_setkey
,
993 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
995 .cra_name
= "hmac(sha1)",
996 .cra_driver_name
= "omap-hmac-sha1",
998 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
1000 CRYPTO_ALG_NEED_FALLBACK
,
1001 .cra_blocksize
= SHA1_BLOCK_SIZE
,
1002 .cra_ctxsize
= sizeof(struct omap_sham_ctx
) +
1003 sizeof(struct omap_sham_hmac_ctx
),
1004 .cra_alignmask
= OMAP_ALIGN_MASK
,
1005 .cra_module
= THIS_MODULE
,
1006 .cra_init
= omap_sham_cra_sha1_init
,
1007 .cra_exit
= omap_sham_cra_exit
,
1011 .init
= omap_sham_init
,
1012 .update
= omap_sham_update
,
1013 .final
= omap_sham_final
,
1014 .finup
= omap_sham_finup
,
1015 .digest
= omap_sham_digest
,
1016 .setkey
= omap_sham_setkey
,
1017 .halg
.digestsize
= MD5_DIGEST_SIZE
,
1019 .cra_name
= "hmac(md5)",
1020 .cra_driver_name
= "omap-hmac-md5",
1021 .cra_priority
= 100,
1022 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
1024 CRYPTO_ALG_NEED_FALLBACK
,
1025 .cra_blocksize
= SHA1_BLOCK_SIZE
,
1026 .cra_ctxsize
= sizeof(struct omap_sham_ctx
) +
1027 sizeof(struct omap_sham_hmac_ctx
),
1028 .cra_alignmask
= OMAP_ALIGN_MASK
,
1029 .cra_module
= THIS_MODULE
,
1030 .cra_init
= omap_sham_cra_md5_init
,
1031 .cra_exit
= omap_sham_cra_exit
,
1036 static void omap_sham_done_task(unsigned long data
)
1038 struct omap_sham_dev
*dd
= (struct omap_sham_dev
*)data
;
1041 if (!test_bit(FLAGS_BUSY
, &dd
->flags
)) {
1042 omap_sham_handle_queue(dd
, NULL
);
1046 if (test_bit(FLAGS_CPU
, &dd
->flags
)) {
1047 if (test_and_clear_bit(FLAGS_OUTPUT_READY
, &dd
->flags
))
1049 } else if (test_bit(FLAGS_DMA_READY
, &dd
->flags
)) {
1050 if (test_and_clear_bit(FLAGS_DMA_ACTIVE
, &dd
->flags
)) {
1051 omap_sham_update_dma_stop(dd
);
1057 if (test_and_clear_bit(FLAGS_OUTPUT_READY
, &dd
->flags
)) {
1058 /* hash or semi-hash ready */
1059 clear_bit(FLAGS_DMA_READY
, &dd
->flags
);
1060 err
= omap_sham_update_dma_start(dd
);
1061 if (err
!= -EINPROGRESS
)
1069 dev_dbg(dd
->dev
, "update done: err: %d\n", err
);
1070 /* finish curent request */
1071 omap_sham_finish_req(dd
->req
, err
);
1074 static irqreturn_t
omap_sham_irq(int irq
, void *dev_id
)
1076 struct omap_sham_dev
*dd
= dev_id
;
1078 if (unlikely(test_bit(FLAGS_FINAL
, &dd
->flags
)))
1079 /* final -> allow device to go to power-saving mode */
1080 omap_sham_write_mask(dd
, SHA_REG_CTRL
, 0, SHA_REG_CTRL_LENGTH
);
1082 omap_sham_write_mask(dd
, SHA_REG_CTRL
, SHA_REG_CTRL_OUTPUT_READY
,
1083 SHA_REG_CTRL_OUTPUT_READY
);
1084 omap_sham_read(dd
, SHA_REG_CTRL
);
1086 set_bit(FLAGS_OUTPUT_READY
, &dd
->flags
);
1087 tasklet_schedule(&dd
->done_task
);
1092 static void omap_sham_dma_callback(int lch
, u16 ch_status
, void *data
)
1094 struct omap_sham_dev
*dd
= data
;
1096 if (ch_status
!= OMAP_DMA_BLOCK_IRQ
) {
1097 pr_err("omap-sham DMA error status: 0x%hx\n", ch_status
);
1099 clear_bit(FLAGS_INIT
, &dd
->flags
);/* request to re-initialize */
1102 set_bit(FLAGS_DMA_READY
, &dd
->flags
);
1103 tasklet_schedule(&dd
->done_task
);
1106 static int omap_sham_dma_init(struct omap_sham_dev
*dd
)
1112 err
= omap_request_dma(dd
->dma
, dev_name(dd
->dev
),
1113 omap_sham_dma_callback
, dd
, &dd
->dma_lch
);
1115 dev_err(dd
->dev
, "Unable to request DMA channel\n");
1122 static void omap_sham_dma_cleanup(struct omap_sham_dev
*dd
)
1124 if (dd
->dma_lch
>= 0) {
1125 omap_free_dma(dd
->dma_lch
);
1130 static int __devinit
omap_sham_probe(struct platform_device
*pdev
)
1132 struct omap_sham_dev
*dd
;
1133 struct device
*dev
= &pdev
->dev
;
1134 struct resource
*res
;
1137 dd
= kzalloc(sizeof(struct omap_sham_dev
), GFP_KERNEL
);
1139 dev_err(dev
, "unable to alloc data struct.\n");
1144 platform_set_drvdata(pdev
, dd
);
1146 INIT_LIST_HEAD(&dd
->list
);
1147 spin_lock_init(&dd
->lock
);
1148 tasklet_init(&dd
->done_task
, omap_sham_done_task
, (unsigned long)dd
);
1149 crypto_init_queue(&dd
->queue
, OMAP_SHAM_QUEUE_LENGTH
);
1153 /* Get the base address */
1154 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1156 dev_err(dev
, "no MEM resource info\n");
1160 dd
->phys_base
= res
->start
;
1163 res
= platform_get_resource(pdev
, IORESOURCE_DMA
, 0);
1165 dev_err(dev
, "no DMA resource info\n");
1169 dd
->dma
= res
->start
;
1172 dd
->irq
= platform_get_irq(pdev
, 0);
1174 dev_err(dev
, "no IRQ resource info\n");
1179 err
= request_irq(dd
->irq
, omap_sham_irq
,
1180 IRQF_TRIGGER_LOW
, dev_name(dev
), dd
);
1182 dev_err(dev
, "unable to request irq.\n");
1186 err
= omap_sham_dma_init(dd
);
1190 /* Initializing the clock */
1191 dd
->iclk
= clk_get(dev
, "ick");
1192 if (IS_ERR(dd
->iclk
)) {
1193 dev_err(dev
, "clock intialization failed.\n");
1194 err
= PTR_ERR(dd
->iclk
);
1198 dd
->io_base
= ioremap(dd
->phys_base
, SZ_4K
);
1200 dev_err(dev
, "can't ioremap\n");
1205 clk_enable(dd
->iclk
);
1206 dev_info(dev
, "hw accel on OMAP rev %u.%u\n",
1207 (omap_sham_read(dd
, SHA_REG_REV
) & SHA_REG_REV_MAJOR
) >> 4,
1208 omap_sham_read(dd
, SHA_REG_REV
) & SHA_REG_REV_MINOR
);
1209 clk_disable(dd
->iclk
);
1211 spin_lock(&sham
.lock
);
1212 list_add_tail(&dd
->list
, &sham
.dev_list
);
1213 spin_unlock(&sham
.lock
);
1215 for (i
= 0; i
< ARRAY_SIZE(algs
); i
++) {
1216 err
= crypto_register_ahash(&algs
[i
]);
1224 for (j
= 0; j
< i
; j
++)
1225 crypto_unregister_ahash(&algs
[j
]);
1226 iounmap(dd
->io_base
);
1230 omap_sham_dma_cleanup(dd
);
1233 free_irq(dd
->irq
, dd
);
1238 dev_err(dev
, "initialization failed.\n");
1243 static int __devexit
omap_sham_remove(struct platform_device
*pdev
)
1245 static struct omap_sham_dev
*dd
;
1248 dd
= platform_get_drvdata(pdev
);
1251 spin_lock(&sham
.lock
);
1252 list_del(&dd
->list
);
1253 spin_unlock(&sham
.lock
);
1254 for (i
= 0; i
< ARRAY_SIZE(algs
); i
++)
1255 crypto_unregister_ahash(&algs
[i
]);
1256 tasklet_kill(&dd
->done_task
);
1257 iounmap(dd
->io_base
);
1259 omap_sham_dma_cleanup(dd
);
1261 free_irq(dd
->irq
, dd
);
1268 static struct platform_driver omap_sham_driver
= {
1269 .probe
= omap_sham_probe
,
1270 .remove
= omap_sham_remove
,
1272 .name
= "omap-sham",
1273 .owner
= THIS_MODULE
,
1277 static int __init
omap_sham_mod_init(void)
1279 pr_info("loading %s driver\n", "omap-sham");
1281 if (!cpu_class_is_omap2() ||
1282 (omap_type() != OMAP2_DEVICE_TYPE_SEC
&&
1283 omap_type() != OMAP2_DEVICE_TYPE_EMU
)) {
1284 pr_err("Unsupported cpu\n");
1288 return platform_driver_register(&omap_sham_driver
);
1291 static void __exit
omap_sham_mod_exit(void)
1293 platform_driver_unregister(&omap_sham_driver
);
1296 module_init(omap_sham_mod_init
);
1297 module_exit(omap_sham_mod_exit
);
1299 MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support.");
1300 MODULE_LICENSE("GPL v2");
1301 MODULE_AUTHOR("Dmitry Kasatkin");