4 * Support for OMAP SHA1/MD5 HW acceleration.
6 * Copyright (c) 2010 Nokia Corporation
7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as published
11 * by the Free Software Foundation.
13 * Some ideas are from old omap-sha1-md5.c driver.
16 #define pr_fmt(fmt) "%s: " fmt, __func__
18 #include <linux/err.h>
19 #include <linux/device.h>
20 #include <linux/module.h>
21 #include <linux/init.h>
22 #include <linux/errno.h>
23 #include <linux/interrupt.h>
24 #include <linux/kernel.h>
25 #include <linux/clk.h>
26 #include <linux/irq.h>
28 #include <linux/platform_device.h>
29 #include <linux/scatterlist.h>
30 #include <linux/dma-mapping.h>
31 #include <linux/delay.h>
32 #include <linux/crypto.h>
33 #include <linux/cryptohash.h>
34 #include <crypto/scatterwalk.h>
35 #include <crypto/algapi.h>
36 #include <crypto/sha.h>
37 #include <crypto/hash.h>
38 #include <crypto/internal/hash.h>
42 #include <mach/irqs.h>
44 #define SHA_REG_DIGEST(x) (0x00 + ((x) * 0x04))
45 #define SHA_REG_DIN(x) (0x1C + ((x) * 0x04))
47 #define SHA1_MD5_BLOCK_SIZE SHA1_BLOCK_SIZE
48 #define MD5_DIGEST_SIZE 16
50 #define SHA_REG_DIGCNT 0x14
52 #define SHA_REG_CTRL 0x18
53 #define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5)
54 #define SHA_REG_CTRL_CLOSE_HASH (1 << 4)
55 #define SHA_REG_CTRL_ALGO_CONST (1 << 3)
56 #define SHA_REG_CTRL_ALGO (1 << 2)
57 #define SHA_REG_CTRL_INPUT_READY (1 << 1)
58 #define SHA_REG_CTRL_OUTPUT_READY (1 << 0)
60 #define SHA_REG_REV 0x5C
61 #define SHA_REG_REV_MAJOR 0xF0
62 #define SHA_REG_REV_MINOR 0x0F
64 #define SHA_REG_MASK 0x60
65 #define SHA_REG_MASK_DMA_EN (1 << 3)
66 #define SHA_REG_MASK_IT_EN (1 << 2)
67 #define SHA_REG_MASK_SOFTRESET (1 << 1)
68 #define SHA_REG_AUTOIDLE (1 << 0)
70 #define SHA_REG_SYSSTATUS 0x64
71 #define SHA_REG_SYSSTATUS_RESETDONE (1 << 0)
73 #define DEFAULT_TIMEOUT_INTERVAL HZ
75 #define FLAGS_FIRST 0x0001
76 #define FLAGS_FINUP 0x0002
77 #define FLAGS_FINAL 0x0004
78 #define FLAGS_FAST 0x0008
79 #define FLAGS_SHA1 0x0010
80 #define FLAGS_DMA_ACTIVE 0x0020
81 #define FLAGS_OUTPUT_READY 0x0040
82 #define FLAGS_CLEAN 0x0080
83 #define FLAGS_INIT 0x0100
84 #define FLAGS_CPU 0x0200
85 #define FLAGS_HMAC 0x0400
86 #define FLAGS_ERROR 0x0800
87 #define FLAGS_BUSY 0x1000
94 struct omap_sham_reqctx
{
95 struct omap_sham_dev
*dd
;
99 u8 digest
[SHA1_DIGEST_SIZE
];
107 struct scatterlist
*sg
;
108 unsigned int offset
; /* offset in current sg */
109 unsigned int total
; /* total request */
112 struct omap_sham_hmac_ctx
{
113 struct crypto_shash
*shash
;
114 u8 ipad
[SHA1_MD5_BLOCK_SIZE
];
115 u8 opad
[SHA1_MD5_BLOCK_SIZE
];
118 struct omap_sham_ctx
{
119 struct omap_sham_dev
*dd
;
124 struct crypto_shash
*fallback
;
126 struct omap_sham_hmac_ctx base
[0];
129 #define OMAP_SHAM_QUEUE_LENGTH 1
131 struct omap_sham_dev
{
132 struct list_head list
;
133 unsigned long phys_base
;
135 void __iomem
*io_base
;
142 struct tasklet_struct done_task
;
143 struct tasklet_struct queue_task
;
146 struct crypto_queue queue
;
147 struct ahash_request
*req
;
150 struct omap_sham_drv
{
151 struct list_head dev_list
;
156 static struct omap_sham_drv sham
= {
157 .dev_list
= LIST_HEAD_INIT(sham
.dev_list
),
158 .lock
= __SPIN_LOCK_UNLOCKED(sham
.lock
),
161 static inline u32
omap_sham_read(struct omap_sham_dev
*dd
, u32 offset
)
163 return __raw_readl(dd
->io_base
+ offset
);
166 static inline void omap_sham_write(struct omap_sham_dev
*dd
,
167 u32 offset
, u32 value
)
169 __raw_writel(value
, dd
->io_base
+ offset
);
172 static inline void omap_sham_write_mask(struct omap_sham_dev
*dd
, u32 address
,
177 val
= omap_sham_read(dd
, address
);
180 omap_sham_write(dd
, address
, val
);
183 static inline int omap_sham_wait(struct omap_sham_dev
*dd
, u32 offset
, u32 bit
)
185 unsigned long timeout
= jiffies
+ DEFAULT_TIMEOUT_INTERVAL
;
187 while (!(omap_sham_read(dd
, offset
) & bit
)) {
188 if (time_is_before_jiffies(timeout
))
195 static void omap_sham_copy_hash(struct ahash_request
*req
, int out
)
197 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
198 u32
*hash
= (u32
*)ctx
->digest
;
201 if (likely(ctx
->flags
& FLAGS_SHA1
)) {
202 /* SHA1 results are in big endian */
203 for (i
= 0; i
< SHA1_DIGEST_SIZE
/ sizeof(u32
); i
++)
205 hash
[i
] = be32_to_cpu(omap_sham_read(ctx
->dd
,
208 omap_sham_write(ctx
->dd
, SHA_REG_DIGEST(i
),
209 cpu_to_be32(hash
[i
]));
211 /* MD5 results are in little endian */
212 for (i
= 0; i
< MD5_DIGEST_SIZE
/ sizeof(u32
); i
++)
214 hash
[i
] = le32_to_cpu(omap_sham_read(ctx
->dd
,
217 omap_sham_write(ctx
->dd
, SHA_REG_DIGEST(i
),
218 cpu_to_le32(hash
[i
]));
222 static int omap_sham_write_ctrl(struct omap_sham_dev
*dd
, size_t length
,
225 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
226 u32 val
= length
<< 5, mask
;
228 if (unlikely(!ctx
->digcnt
)) {
230 clk_enable(dd
->iclk
);
232 if (!(dd
->flags
& FLAGS_INIT
)) {
233 omap_sham_write_mask(dd
, SHA_REG_MASK
,
234 SHA_REG_MASK_SOFTRESET
, SHA_REG_MASK_SOFTRESET
);
236 if (omap_sham_wait(dd
, SHA_REG_SYSSTATUS
,
237 SHA_REG_SYSSTATUS_RESETDONE
)) {
238 clk_disable(dd
->iclk
);
241 dd
->flags
|= FLAGS_INIT
;
245 omap_sham_write(dd
, SHA_REG_DIGCNT
, ctx
->digcnt
);
248 omap_sham_write_mask(dd
, SHA_REG_MASK
,
249 SHA_REG_MASK_IT_EN
| (dma
? SHA_REG_MASK_DMA_EN
: 0),
250 SHA_REG_MASK_IT_EN
| SHA_REG_MASK_DMA_EN
);
252 * Setting ALGO_CONST only for the first iteration
253 * and CLOSE_HASH only for the last one.
255 if (ctx
->flags
& FLAGS_SHA1
)
256 val
|= SHA_REG_CTRL_ALGO
;
258 val
|= SHA_REG_CTRL_ALGO_CONST
;
260 val
|= SHA_REG_CTRL_CLOSE_HASH
;
262 mask
= SHA_REG_CTRL_ALGO_CONST
| SHA_REG_CTRL_CLOSE_HASH
|
263 SHA_REG_CTRL_ALGO
| SHA_REG_CTRL_LENGTH
;
265 omap_sham_write_mask(dd
, SHA_REG_CTRL
, val
, mask
);
270 static int omap_sham_xmit_cpu(struct omap_sham_dev
*dd
, const u8
*buf
,
271 size_t length
, int final
)
273 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
274 int err
, count
, len32
;
275 const u32
*buffer
= (const u32
*)buf
;
277 dev_dbg(dd
->dev
, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
278 ctx
->digcnt
, length
, final
);
280 err
= omap_sham_write_ctrl(dd
, length
, final
, 0);
284 /* should be non-zero before next lines to disable clocks later */
285 ctx
->digcnt
+= length
;
287 if (omap_sham_wait(dd
, SHA_REG_CTRL
, SHA_REG_CTRL_INPUT_READY
))
291 ctx
->flags
|= FLAGS_FINAL
; /* catch last interrupt */
293 len32
= DIV_ROUND_UP(length
, sizeof(u32
));
295 for (count
= 0; count
< len32
; count
++)
296 omap_sham_write(dd
, SHA_REG_DIN(count
), buffer
[count
]);
301 static int omap_sham_xmit_dma(struct omap_sham_dev
*dd
, dma_addr_t dma_addr
,
302 size_t length
, int final
)
304 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
307 dev_dbg(dd
->dev
, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
308 ctx
->digcnt
, length
, final
);
309 /* flush cache entries related to our page */
310 if (dma_addr
== ctx
->dma_addr
)
311 dma_sync_single_for_device(dd
->dev
, dma_addr
, length
,
314 len32
= DIV_ROUND_UP(length
, sizeof(u32
));
316 omap_set_dma_transfer_params(dd
->dma_lch
, OMAP_DMA_DATA_TYPE_S32
, len32
,
317 1, OMAP_DMA_SYNC_PACKET
, dd
->dma
,
318 OMAP_DMA_DST_SYNC_PREFETCH
);
320 omap_set_dma_src_params(dd
->dma_lch
, 0, OMAP_DMA_AMODE_POST_INC
,
323 omap_set_dma_dest_params(dd
->dma_lch
, 0,
324 OMAP_DMA_AMODE_CONSTANT
,
325 dd
->phys_base
+ SHA_REG_DIN(0), 0, 16);
327 omap_set_dma_dest_burst_mode(dd
->dma_lch
,
328 OMAP_DMA_DATA_BURST_16
);
330 omap_set_dma_src_burst_mode(dd
->dma_lch
,
331 OMAP_DMA_DATA_BURST_4
);
333 err
= omap_sham_write_ctrl(dd
, length
, final
, 1);
337 ctx
->digcnt
+= length
;
340 ctx
->flags
|= FLAGS_FINAL
; /* catch last interrupt */
342 dd
->flags
|= FLAGS_DMA_ACTIVE
;
344 omap_start_dma(dd
->dma_lch
);
349 static size_t omap_sham_append_buffer(struct omap_sham_reqctx
*ctx
,
350 const u8
*data
, size_t length
)
352 size_t count
= min(length
, ctx
->buflen
- ctx
->bufcnt
);
354 count
= min(count
, ctx
->total
);
357 memcpy(ctx
->buffer
+ ctx
->bufcnt
, data
, count
);
358 ctx
->bufcnt
+= count
;
363 static size_t omap_sham_append_sg(struct omap_sham_reqctx
*ctx
)
368 count
= omap_sham_append_buffer(ctx
,
369 sg_virt(ctx
->sg
) + ctx
->offset
,
370 ctx
->sg
->length
- ctx
->offset
);
373 ctx
->offset
+= count
;
375 if (ctx
->offset
== ctx
->sg
->length
) {
376 ctx
->sg
= sg_next(ctx
->sg
);
387 static int omap_sham_update_dma_slow(struct omap_sham_dev
*dd
)
389 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
396 omap_sham_append_sg(ctx
);
398 final
= (ctx
->flags
& FLAGS_FINUP
) && !ctx
->total
;
400 dev_dbg(dd
->dev
, "slow: bufcnt: %u, digcnt: %d, final: %d\n",
401 ctx
->bufcnt
, ctx
->digcnt
, final
);
403 if (final
|| (ctx
->bufcnt
== ctx
->buflen
&& ctx
->total
)) {
406 return omap_sham_xmit_dma(dd
, ctx
->dma_addr
, count
, final
);
412 static int omap_sham_update_dma_fast(struct omap_sham_dev
*dd
)
414 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
418 ctx
->flags
|= FLAGS_FAST
;
420 length
= min(ctx
->total
, sg_dma_len(ctx
->sg
));
423 if (!dma_map_sg(dd
->dev
, ctx
->sg
, 1, DMA_TO_DEVICE
)) {
424 dev_err(dd
->dev
, "dma_map_sg error\n");
428 ctx
->total
-= length
;
430 err
= omap_sham_xmit_dma(dd
, sg_dma_address(ctx
->sg
), length
, 1);
431 if (err
!= -EINPROGRESS
)
432 dma_unmap_sg(dd
->dev
, ctx
->sg
, 1, DMA_TO_DEVICE
);
437 static int omap_sham_update_cpu(struct omap_sham_dev
*dd
)
439 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
442 omap_sham_append_sg(ctx
);
443 bufcnt
= ctx
->bufcnt
;
446 return omap_sham_xmit_cpu(dd
, ctx
->buffer
, bufcnt
, 1);
449 static int omap_sham_update_dma_stop(struct omap_sham_dev
*dd
)
451 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
453 omap_stop_dma(dd
->dma_lch
);
454 if (ctx
->flags
& FLAGS_FAST
)
455 dma_unmap_sg(dd
->dev
, ctx
->sg
, 1, DMA_TO_DEVICE
);
460 static void omap_sham_cleanup(struct ahash_request
*req
)
462 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
463 struct omap_sham_dev
*dd
= ctx
->dd
;
466 spin_lock_irqsave(&dd
->lock
, flags
);
467 if (ctx
->flags
& FLAGS_CLEAN
) {
468 spin_unlock_irqrestore(&dd
->lock
, flags
);
471 ctx
->flags
|= FLAGS_CLEAN
;
472 spin_unlock_irqrestore(&dd
->lock
, flags
);
475 clk_disable(dd
->iclk
);
476 memcpy(req
->result
, ctx
->digest
, (ctx
->flags
& FLAGS_SHA1
) ?
477 SHA1_DIGEST_SIZE
: MD5_DIGEST_SIZE
);
481 dma_unmap_single(dd
->dev
, ctx
->dma_addr
, ctx
->buflen
,
485 free_page((unsigned long)ctx
->buffer
);
487 dev_dbg(dd
->dev
, "digcnt: %d, bufcnt: %d\n", ctx
->digcnt
, ctx
->bufcnt
);
490 static int omap_sham_init(struct ahash_request
*req
)
492 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
493 struct omap_sham_ctx
*tctx
= crypto_ahash_ctx(tfm
);
494 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
495 struct omap_sham_dev
*dd
= NULL
, *tmp
;
497 spin_lock_bh(&sham
.lock
);
499 list_for_each_entry(tmp
, &sham
.dev_list
, list
) {
507 spin_unlock_bh(&sham
.lock
);
513 ctx
->flags
|= FLAGS_FIRST
;
515 dev_dbg(dd
->dev
, "init: digest size: %d\n",
516 crypto_ahash_digestsize(tfm
));
518 if (crypto_ahash_digestsize(tfm
) == SHA1_DIGEST_SIZE
)
519 ctx
->flags
|= FLAGS_SHA1
;
524 ctx
->buflen
= PAGE_SIZE
;
525 ctx
->buffer
= (void *)__get_free_page(
526 (req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
) ?
527 GFP_KERNEL
: GFP_ATOMIC
);
531 ctx
->dma_addr
= dma_map_single(dd
->dev
, ctx
->buffer
, ctx
->buflen
,
533 if (dma_mapping_error(dd
->dev
, ctx
->dma_addr
)) {
534 dev_err(dd
->dev
, "dma %u bytes error\n", ctx
->buflen
);
535 free_page((unsigned long)ctx
->buffer
);
539 if (tctx
->flags
& FLAGS_HMAC
) {
540 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
542 memcpy(ctx
->buffer
, bctx
->ipad
, SHA1_MD5_BLOCK_SIZE
);
543 ctx
->bufcnt
= SHA1_MD5_BLOCK_SIZE
;
544 ctx
->flags
|= FLAGS_HMAC
;
551 static int omap_sham_update_req(struct omap_sham_dev
*dd
)
553 struct ahash_request
*req
= dd
->req
;
554 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
557 dev_dbg(dd
->dev
, "update_req: total: %u, digcnt: %d, finup: %d\n",
558 ctx
->total
, ctx
->digcnt
, (ctx
->flags
& FLAGS_FINUP
) != 0);
560 if (ctx
->flags
& FLAGS_CPU
)
561 err
= omap_sham_update_cpu(dd
);
562 else if (ctx
->flags
& FLAGS_FAST
)
563 err
= omap_sham_update_dma_fast(dd
);
565 err
= omap_sham_update_dma_slow(dd
);
567 /* wait for dma completion before can take more data */
568 dev_dbg(dd
->dev
, "update: err: %d, digcnt: %d\n", err
, ctx
->digcnt
);
573 static int omap_sham_final_req(struct omap_sham_dev
*dd
)
575 struct ahash_request
*req
= dd
->req
;
576 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
577 int err
= 0, use_dma
= 1;
579 if (ctx
->bufcnt
<= 64)
580 /* faster to handle last block with cpu */
584 err
= omap_sham_xmit_dma(dd
, ctx
->dma_addr
, ctx
->bufcnt
, 1);
586 err
= omap_sham_xmit_cpu(dd
, ctx
->buffer
, ctx
->bufcnt
, 1);
590 dev_dbg(dd
->dev
, "final_req: err: %d\n", err
);
595 static int omap_sham_finish_req_hmac(struct ahash_request
*req
)
597 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
598 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(req
->base
.tfm
);
599 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
600 int bs
= crypto_shash_blocksize(bctx
->shash
);
601 int ds
= crypto_shash_digestsize(bctx
->shash
);
603 struct shash_desc shash
;
604 char ctx
[crypto_shash_descsize(bctx
->shash
)];
607 desc
.shash
.tfm
= bctx
->shash
;
608 desc
.shash
.flags
= 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */
610 return crypto_shash_init(&desc
.shash
) ?:
611 crypto_shash_update(&desc
.shash
, bctx
->opad
, bs
) ?:
612 crypto_shash_finup(&desc
.shash
, ctx
->digest
, ds
, ctx
->digest
);
615 static void omap_sham_finish_req(struct ahash_request
*req
, int err
)
617 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
620 omap_sham_copy_hash(ctx
->dd
->req
, 1);
621 if (ctx
->flags
& FLAGS_HMAC
)
622 err
= omap_sham_finish_req_hmac(req
);
624 ctx
->flags
|= FLAGS_ERROR
;
627 if ((ctx
->flags
& FLAGS_FINAL
) || err
)
628 omap_sham_cleanup(req
);
630 ctx
->dd
->flags
&= ~FLAGS_BUSY
;
632 if (req
->base
.complete
)
633 req
->base
.complete(&req
->base
, err
);
636 static int omap_sham_handle_queue(struct omap_sham_dev
*dd
,
637 struct ahash_request
*req
)
639 struct crypto_async_request
*async_req
, *backlog
;
640 struct omap_sham_reqctx
*ctx
;
641 struct ahash_request
*prev_req
;
643 int err
= 0, ret
= 0;
645 spin_lock_irqsave(&dd
->lock
, flags
);
647 ret
= ahash_enqueue_request(&dd
->queue
, req
);
648 if (dd
->flags
& FLAGS_BUSY
) {
649 spin_unlock_irqrestore(&dd
->lock
, flags
);
652 async_req
= crypto_dequeue_request(&dd
->queue
);
654 dd
->flags
|= FLAGS_BUSY
;
655 backlog
= crypto_get_backlog(&dd
->queue
);
657 spin_unlock_irqrestore(&dd
->lock
, flags
);
663 backlog
->complete(backlog
, -EINPROGRESS
);
665 req
= ahash_request_cast(async_req
);
670 ctx
= ahash_request_ctx(req
);
672 dev_dbg(dd
->dev
, "handling new req, op: %lu, nbytes: %d\n",
673 ctx
->op
, req
->nbytes
);
675 if (req
!= prev_req
&& ctx
->digcnt
)
676 /* request has changed - restore hash */
677 omap_sham_copy_hash(req
, 0);
679 if (ctx
->op
== OP_UPDATE
) {
680 err
= omap_sham_update_req(dd
);
681 if (err
!= -EINPROGRESS
&& (ctx
->flags
& FLAGS_FINUP
))
682 /* no final() after finup() */
683 err
= omap_sham_final_req(dd
);
684 } else if (ctx
->op
== OP_FINAL
) {
685 err
= omap_sham_final_req(dd
);
688 if (err
!= -EINPROGRESS
) {
689 /* done_task will not finish it, so do it here */
690 omap_sham_finish_req(req
, err
);
691 tasklet_schedule(&dd
->queue_task
);
694 dev_dbg(dd
->dev
, "exit, err: %d\n", err
);
699 static int omap_sham_enqueue(struct ahash_request
*req
, unsigned int op
)
701 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
702 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(req
->base
.tfm
);
703 struct omap_sham_dev
*dd
= tctx
->dd
;
707 return omap_sham_handle_queue(dd
, req
);
710 static int omap_sham_update(struct ahash_request
*req
)
712 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
717 ctx
->total
= req
->nbytes
;
721 if (ctx
->flags
& FLAGS_FINUP
) {
722 if ((ctx
->digcnt
+ ctx
->bufcnt
+ ctx
->total
) < 9) {
724 * OMAP HW accel works only with buffers >= 9
725 * will switch to bypass in final()
726 * final has the same request and data
728 omap_sham_append_sg(ctx
);
730 } else if (ctx
->bufcnt
+ ctx
->total
<= 64) {
731 ctx
->flags
|= FLAGS_CPU
;
732 } else if (!ctx
->bufcnt
&& sg_is_last(ctx
->sg
)) {
733 /* may be can use faster functions */
734 int aligned
= IS_ALIGNED((u32
)ctx
->sg
->offset
,
737 if (aligned
&& (ctx
->flags
& FLAGS_FIRST
))
738 /* digest: first and final */
739 ctx
->flags
|= FLAGS_FAST
;
741 ctx
->flags
&= ~FLAGS_FIRST
;
743 } else if (ctx
->bufcnt
+ ctx
->total
<= ctx
->buflen
) {
744 /* if not finaup -> not fast */
745 omap_sham_append_sg(ctx
);
749 return omap_sham_enqueue(req
, OP_UPDATE
);
752 static int omap_sham_shash_digest(struct crypto_shash
*shash
, u32 flags
,
753 const u8
*data
, unsigned int len
, u8
*out
)
756 struct shash_desc shash
;
757 char ctx
[crypto_shash_descsize(shash
)];
760 desc
.shash
.tfm
= shash
;
761 desc
.shash
.flags
= flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
763 return crypto_shash_digest(&desc
.shash
, data
, len
, out
);
766 static int omap_sham_final_shash(struct ahash_request
*req
)
768 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(req
->base
.tfm
);
769 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
771 return omap_sham_shash_digest(tctx
->fallback
, req
->base
.flags
,
772 ctx
->buffer
, ctx
->bufcnt
, req
->result
);
775 static int omap_sham_final(struct ahash_request
*req
)
777 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
780 ctx
->flags
|= FLAGS_FINUP
;
782 if (!(ctx
->flags
& FLAGS_ERROR
)) {
783 /* OMAP HW accel works only with buffers >= 9 */
784 /* HMAC is always >= 9 because of ipad */
785 if ((ctx
->digcnt
+ ctx
->bufcnt
) < 9)
786 err
= omap_sham_final_shash(req
);
787 else if (ctx
->bufcnt
)
788 return omap_sham_enqueue(req
, OP_FINAL
);
791 omap_sham_cleanup(req
);
796 static int omap_sham_finup(struct ahash_request
*req
)
798 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
801 ctx
->flags
|= FLAGS_FINUP
;
803 err1
= omap_sham_update(req
);
804 if (err1
== -EINPROGRESS
)
807 * final() has to be always called to cleanup resources
808 * even if udpate() failed, except EINPROGRESS
810 err2
= omap_sham_final(req
);
815 static int omap_sham_digest(struct ahash_request
*req
)
817 return omap_sham_init(req
) ?: omap_sham_finup(req
);
820 static int omap_sham_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
823 struct omap_sham_ctx
*tctx
= crypto_ahash_ctx(tfm
);
824 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
825 int bs
= crypto_shash_blocksize(bctx
->shash
);
826 int ds
= crypto_shash_digestsize(bctx
->shash
);
828 err
= crypto_shash_setkey(tctx
->fallback
, key
, keylen
);
833 err
= omap_sham_shash_digest(bctx
->shash
,
834 crypto_shash_get_flags(bctx
->shash
),
835 key
, keylen
, bctx
->ipad
);
840 memcpy(bctx
->ipad
, key
, keylen
);
843 memset(bctx
->ipad
+ keylen
, 0, bs
- keylen
);
844 memcpy(bctx
->opad
, bctx
->ipad
, bs
);
846 for (i
= 0; i
< bs
; i
++) {
847 bctx
->ipad
[i
] ^= 0x36;
848 bctx
->opad
[i
] ^= 0x5c;
854 static int omap_sham_cra_init_alg(struct crypto_tfm
*tfm
, const char *alg_base
)
856 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(tfm
);
857 const char *alg_name
= crypto_tfm_alg_name(tfm
);
861 /* Allocate a fallback and abort if it failed. */
862 tctx
->fallback
= crypto_alloc_shash(alg_name
, 0,
863 CRYPTO_ALG_NEED_FALLBACK
);
864 if (IS_ERR(tctx
->fallback
)) {
865 pr_err("omap-sham: fallback driver '%s' "
866 "could not be loaded.\n", alg_name
);
867 return PTR_ERR(tctx
->fallback
);
870 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
871 sizeof(struct omap_sham_reqctx
));
874 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
875 tctx
->flags
|= FLAGS_HMAC
;
876 bctx
->shash
= crypto_alloc_shash(alg_base
, 0,
877 CRYPTO_ALG_NEED_FALLBACK
);
878 if (IS_ERR(bctx
->shash
)) {
879 pr_err("omap-sham: base driver '%s' "
880 "could not be loaded.\n", alg_base
);
881 crypto_free_shash(tctx
->fallback
);
882 return PTR_ERR(bctx
->shash
);
890 static int omap_sham_cra_init(struct crypto_tfm
*tfm
)
892 return omap_sham_cra_init_alg(tfm
, NULL
);
895 static int omap_sham_cra_sha1_init(struct crypto_tfm
*tfm
)
897 return omap_sham_cra_init_alg(tfm
, "sha1");
900 static int omap_sham_cra_md5_init(struct crypto_tfm
*tfm
)
902 return omap_sham_cra_init_alg(tfm
, "md5");
905 static void omap_sham_cra_exit(struct crypto_tfm
*tfm
)
907 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(tfm
);
909 crypto_free_shash(tctx
->fallback
);
910 tctx
->fallback
= NULL
;
912 if (tctx
->flags
& FLAGS_HMAC
) {
913 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
914 crypto_free_shash(bctx
->shash
);
918 static struct ahash_alg algs
[] = {
920 .init
= omap_sham_init
,
921 .update
= omap_sham_update
,
922 .final
= omap_sham_final
,
923 .finup
= omap_sham_finup
,
924 .digest
= omap_sham_digest
,
925 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
928 .cra_driver_name
= "omap-sha1",
930 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
932 CRYPTO_ALG_NEED_FALLBACK
,
933 .cra_blocksize
= SHA1_BLOCK_SIZE
,
934 .cra_ctxsize
= sizeof(struct omap_sham_ctx
),
936 .cra_module
= THIS_MODULE
,
937 .cra_init
= omap_sham_cra_init
,
938 .cra_exit
= omap_sham_cra_exit
,
942 .init
= omap_sham_init
,
943 .update
= omap_sham_update
,
944 .final
= omap_sham_final
,
945 .finup
= omap_sham_finup
,
946 .digest
= omap_sham_digest
,
947 .halg
.digestsize
= MD5_DIGEST_SIZE
,
950 .cra_driver_name
= "omap-md5",
952 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
954 CRYPTO_ALG_NEED_FALLBACK
,
955 .cra_blocksize
= SHA1_BLOCK_SIZE
,
956 .cra_ctxsize
= sizeof(struct omap_sham_ctx
),
958 .cra_module
= THIS_MODULE
,
959 .cra_init
= omap_sham_cra_init
,
960 .cra_exit
= omap_sham_cra_exit
,
964 .init
= omap_sham_init
,
965 .update
= omap_sham_update
,
966 .final
= omap_sham_final
,
967 .finup
= omap_sham_finup
,
968 .digest
= omap_sham_digest
,
969 .setkey
= omap_sham_setkey
,
970 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
972 .cra_name
= "hmac(sha1)",
973 .cra_driver_name
= "omap-hmac-sha1",
975 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
977 CRYPTO_ALG_NEED_FALLBACK
,
978 .cra_blocksize
= SHA1_BLOCK_SIZE
,
979 .cra_ctxsize
= sizeof(struct omap_sham_ctx
) +
980 sizeof(struct omap_sham_hmac_ctx
),
982 .cra_module
= THIS_MODULE
,
983 .cra_init
= omap_sham_cra_sha1_init
,
984 .cra_exit
= omap_sham_cra_exit
,
988 .init
= omap_sham_init
,
989 .update
= omap_sham_update
,
990 .final
= omap_sham_final
,
991 .finup
= omap_sham_finup
,
992 .digest
= omap_sham_digest
,
993 .setkey
= omap_sham_setkey
,
994 .halg
.digestsize
= MD5_DIGEST_SIZE
,
996 .cra_name
= "hmac(md5)",
997 .cra_driver_name
= "omap-hmac-md5",
999 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
1001 CRYPTO_ALG_NEED_FALLBACK
,
1002 .cra_blocksize
= SHA1_BLOCK_SIZE
,
1003 .cra_ctxsize
= sizeof(struct omap_sham_ctx
) +
1004 sizeof(struct omap_sham_hmac_ctx
),
1006 .cra_module
= THIS_MODULE
,
1007 .cra_init
= omap_sham_cra_md5_init
,
1008 .cra_exit
= omap_sham_cra_exit
,
1013 static void omap_sham_done_task(unsigned long data
)
1015 struct omap_sham_dev
*dd
= (struct omap_sham_dev
*)data
;
1016 struct ahash_request
*req
= dd
->req
;
1017 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
1018 int ready
= 0, err
= 0;
1020 if (ctx
->flags
& FLAGS_OUTPUT_READY
) {
1021 ctx
->flags
&= ~FLAGS_OUTPUT_READY
;
1025 if (dd
->flags
& FLAGS_DMA_ACTIVE
) {
1026 dd
->flags
&= ~FLAGS_DMA_ACTIVE
;
1027 omap_sham_update_dma_stop(dd
);
1029 err
= omap_sham_update_dma_slow(dd
);
1032 err
= dd
->err
? : err
;
1034 if (err
!= -EINPROGRESS
&& (ready
|| err
)) {
1035 dev_dbg(dd
->dev
, "update done: err: %d\n", err
);
1036 /* finish curent request */
1037 omap_sham_finish_req(req
, err
);
1038 /* start new request */
1039 omap_sham_handle_queue(dd
, NULL
);
1043 static void omap_sham_queue_task(unsigned long data
)
1045 struct omap_sham_dev
*dd
= (struct omap_sham_dev
*)data
;
1047 omap_sham_handle_queue(dd
, NULL
);
1050 static irqreturn_t
omap_sham_irq(int irq
, void *dev_id
)
1052 struct omap_sham_dev
*dd
= dev_id
;
1053 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
1056 dev_err(dd
->dev
, "unknown interrupt.\n");
1060 if (unlikely(ctx
->flags
& FLAGS_FINAL
))
1061 /* final -> allow device to go to power-saving mode */
1062 omap_sham_write_mask(dd
, SHA_REG_CTRL
, 0, SHA_REG_CTRL_LENGTH
);
1064 omap_sham_write_mask(dd
, SHA_REG_CTRL
, SHA_REG_CTRL_OUTPUT_READY
,
1065 SHA_REG_CTRL_OUTPUT_READY
);
1066 omap_sham_read(dd
, SHA_REG_CTRL
);
1068 ctx
->flags
|= FLAGS_OUTPUT_READY
;
1070 tasklet_schedule(&dd
->done_task
);
1075 static void omap_sham_dma_callback(int lch
, u16 ch_status
, void *data
)
1077 struct omap_sham_dev
*dd
= data
;
1079 if (ch_status
!= OMAP_DMA_BLOCK_IRQ
) {
1080 pr_err("omap-sham DMA error status: 0x%hx\n", ch_status
);
1082 dd
->flags
&= ~FLAGS_INIT
; /* request to re-initialize */
1085 tasklet_schedule(&dd
->done_task
);
1088 static int omap_sham_dma_init(struct omap_sham_dev
*dd
)
1094 err
= omap_request_dma(dd
->dma
, dev_name(dd
->dev
),
1095 omap_sham_dma_callback
, dd
, &dd
->dma_lch
);
1097 dev_err(dd
->dev
, "Unable to request DMA channel\n");
1104 static void omap_sham_dma_cleanup(struct omap_sham_dev
*dd
)
1106 if (dd
->dma_lch
>= 0) {
1107 omap_free_dma(dd
->dma_lch
);
1112 static int __devinit
omap_sham_probe(struct platform_device
*pdev
)
1114 struct omap_sham_dev
*dd
;
1115 struct device
*dev
= &pdev
->dev
;
1116 struct resource
*res
;
1119 dd
= kzalloc(sizeof(struct omap_sham_dev
), GFP_KERNEL
);
1121 dev_err(dev
, "unable to alloc data struct.\n");
1126 platform_set_drvdata(pdev
, dd
);
1128 INIT_LIST_HEAD(&dd
->list
);
1129 spin_lock_init(&dd
->lock
);
1130 tasklet_init(&dd
->done_task
, omap_sham_done_task
, (unsigned long)dd
);
1131 tasklet_init(&dd
->queue_task
, omap_sham_queue_task
, (unsigned long)dd
);
1132 crypto_init_queue(&dd
->queue
, OMAP_SHAM_QUEUE_LENGTH
);
1136 /* Get the base address */
1137 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1139 dev_err(dev
, "no MEM resource info\n");
1143 dd
->phys_base
= res
->start
;
1146 res
= platform_get_resource(pdev
, IORESOURCE_DMA
, 0);
1148 dev_err(dev
, "no DMA resource info\n");
1152 dd
->dma
= res
->start
;
1155 dd
->irq
= platform_get_irq(pdev
, 0);
1157 dev_err(dev
, "no IRQ resource info\n");
1162 err
= request_irq(dd
->irq
, omap_sham_irq
,
1163 IRQF_TRIGGER_LOW
, dev_name(dev
), dd
);
1165 dev_err(dev
, "unable to request irq.\n");
1169 err
= omap_sham_dma_init(dd
);
1173 /* Initializing the clock */
1174 dd
->iclk
= clk_get(dev
, "ick");
1176 dev_err(dev
, "clock intialization failed.\n");
1181 dd
->io_base
= ioremap(dd
->phys_base
, SZ_4K
);
1183 dev_err(dev
, "can't ioremap\n");
1188 clk_enable(dd
->iclk
);
1189 dev_info(dev
, "hw accel on OMAP rev %u.%u\n",
1190 (omap_sham_read(dd
, SHA_REG_REV
) & SHA_REG_REV_MAJOR
) >> 4,
1191 omap_sham_read(dd
, SHA_REG_REV
) & SHA_REG_REV_MINOR
);
1192 clk_disable(dd
->iclk
);
1194 spin_lock(&sham
.lock
);
1195 list_add_tail(&dd
->list
, &sham
.dev_list
);
1196 spin_unlock(&sham
.lock
);
1198 for (i
= 0; i
< ARRAY_SIZE(algs
); i
++) {
1199 err
= crypto_register_ahash(&algs
[i
]);
1207 for (j
= 0; j
< i
; j
++)
1208 crypto_unregister_ahash(&algs
[j
]);
1209 iounmap(dd
->io_base
);
1213 omap_sham_dma_cleanup(dd
);
1216 free_irq(dd
->irq
, dd
);
1221 dev_err(dev
, "initialization failed.\n");
1226 static int __devexit
omap_sham_remove(struct platform_device
*pdev
)
1228 static struct omap_sham_dev
*dd
;
1231 dd
= platform_get_drvdata(pdev
);
1234 spin_lock(&sham
.lock
);
1235 list_del(&dd
->list
);
1236 spin_unlock(&sham
.lock
);
1237 for (i
= 0; i
< ARRAY_SIZE(algs
); i
++)
1238 crypto_unregister_ahash(&algs
[i
]);
1239 tasklet_kill(&dd
->done_task
);
1240 tasklet_kill(&dd
->queue_task
);
1241 iounmap(dd
->io_base
);
1243 omap_sham_dma_cleanup(dd
);
1245 free_irq(dd
->irq
, dd
);
1252 static struct platform_driver omap_sham_driver
= {
1253 .probe
= omap_sham_probe
,
1254 .remove
= omap_sham_remove
,
1256 .name
= "omap-sham",
1257 .owner
= THIS_MODULE
,
1261 static int __init
omap_sham_mod_init(void)
1263 pr_info("loading %s driver\n", "omap-sham");
1265 if (!cpu_class_is_omap2() ||
1266 omap_type() != OMAP2_DEVICE_TYPE_SEC
) {
1267 pr_err("Unsupported cpu\n");
1271 return platform_driver_register(&omap_sham_driver
);
1274 static void __exit
omap_sham_mod_exit(void)
1276 platform_driver_unregister(&omap_sham_driver
);
1279 module_init(omap_sham_mod_init
);
1280 module_exit(omap_sham_mod_exit
);
1282 MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support.");
1283 MODULE_LICENSE("GPL v2");
1284 MODULE_AUTHOR("Dmitry Kasatkin");