2 * talitos - Freescale Integrated Security Engine (SEC) device driver
4 * Copyright (c) 2008-2010 Freescale Semiconductor, Inc.
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/mod_devicetable.h>
31 #include <linux/device.h>
32 #include <linux/interrupt.h>
33 #include <linux/crypto.h>
34 #include <linux/hw_random.h>
35 #include <linux/of_platform.h>
36 #include <linux/dma-mapping.h>
38 #include <linux/spinlock.h>
39 #include <linux/rtnetlink.h>
40 #include <linux/slab.h>
42 #include <crypto/algapi.h>
43 #include <crypto/aes.h>
44 #include <crypto/des.h>
45 #include <crypto/sha.h>
46 #include <crypto/md5.h>
47 #include <crypto/aead.h>
48 #include <crypto/authenc.h>
49 #include <crypto/skcipher.h>
50 #include <crypto/hash.h>
51 #include <crypto/internal/hash.h>
52 #include <crypto/scatterwalk.h>
56 #define TALITOS_TIMEOUT 100000
57 #define TALITOS_MAX_DATA_LEN 65535
59 #define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f)
60 #define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
61 #define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
63 /* descriptor pointer entry */
65 __be16 len
; /* length */
66 u8 j_extent
; /* jump to sg link table and/or extent */
67 u8 eptr
; /* extended address */
68 __be32 ptr
; /* address */
71 static const struct talitos_ptr zero_entry
= {
80 __be32 hdr
; /* header high bits */
81 __be32 hdr_lo
; /* header low bits */
82 struct talitos_ptr ptr
[7]; /* ptr/len pair array */
86 * talitos_request - descriptor submission request
87 * @desc: descriptor pointer (kernel virtual)
88 * @dma_desc: descriptor's physical bus address
89 * @callback: whom to call when descriptor processing is done
90 * @context: caller context (optional)
92 struct talitos_request
{
93 struct talitos_desc
*desc
;
95 void (*callback
) (struct device
*dev
, struct talitos_desc
*desc
,
96 void *context
, int error
);
100 /* per-channel fifo management */
101 struct talitos_channel
{
103 struct talitos_request
*fifo
;
105 /* number of requests pending in channel h/w fifo */
106 atomic_t submit_count ____cacheline_aligned
;
108 /* request submission (head) lock */
109 spinlock_t head_lock ____cacheline_aligned
;
110 /* index to next free descriptor request */
113 /* request release (tail) lock */
114 spinlock_t tail_lock ____cacheline_aligned
;
115 /* index to next in-progress/done descriptor request */
119 struct talitos_private
{
121 struct platform_device
*ofdev
;
125 /* SEC version geometry (from device tree node) */
126 unsigned int num_channels
;
127 unsigned int chfifo_len
;
128 unsigned int exec_units
;
129 unsigned int desc_types
;
131 /* SEC Compatibility info */
132 unsigned long features
;
135 * length of the request fifo
136 * fifo_len is chfifo_len rounded up to next power of 2
137 * so we can use bitwise ops to wrap
139 unsigned int fifo_len
;
141 struct talitos_channel
*chan
;
143 /* next channel to be assigned next incoming descriptor */
144 atomic_t last_chan ____cacheline_aligned
;
146 /* request callback tasklet */
147 struct tasklet_struct done_task
;
149 /* list of registered algorithms */
150 struct list_head alg_list
;
157 #define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
158 #define TALITOS_FTR_HW_AUTH_CHECK 0x00000002
159 #define TALITOS_FTR_SHA224_HWINIT 0x00000004
161 static void to_talitos_ptr(struct talitos_ptr
*talitos_ptr
, dma_addr_t dma_addr
)
163 talitos_ptr
->ptr
= cpu_to_be32(lower_32_bits(dma_addr
));
164 talitos_ptr
->eptr
= upper_32_bits(dma_addr
);
168 * map virtual single (contiguous) pointer to h/w descriptor pointer
170 static void map_single_talitos_ptr(struct device
*dev
,
171 struct talitos_ptr
*talitos_ptr
,
172 unsigned short len
, void *data
,
173 unsigned char extent
,
174 enum dma_data_direction dir
)
176 dma_addr_t dma_addr
= dma_map_single(dev
, data
, len
, dir
);
178 talitos_ptr
->len
= cpu_to_be16(len
);
179 to_talitos_ptr(talitos_ptr
, dma_addr
);
180 talitos_ptr
->j_extent
= extent
;
184 * unmap bus single (contiguous) h/w descriptor pointer
186 static void unmap_single_talitos_ptr(struct device
*dev
,
187 struct talitos_ptr
*talitos_ptr
,
188 enum dma_data_direction dir
)
190 dma_unmap_single(dev
, be32_to_cpu(talitos_ptr
->ptr
),
191 be16_to_cpu(talitos_ptr
->len
), dir
);
194 static int reset_channel(struct device
*dev
, int ch
)
196 struct talitos_private
*priv
= dev_get_drvdata(dev
);
197 unsigned int timeout
= TALITOS_TIMEOUT
;
199 setbits32(priv
->reg
+ TALITOS_CCCR(ch
), TALITOS_CCCR_RESET
);
201 while ((in_be32(priv
->reg
+ TALITOS_CCCR(ch
)) & TALITOS_CCCR_RESET
)
206 dev_err(dev
, "failed to reset channel %d\n", ch
);
210 /* set 36-bit addressing, done writeback enable and done IRQ enable */
211 setbits32(priv
->reg
+ TALITOS_CCCR_LO(ch
), TALITOS_CCCR_LO_EAE
|
212 TALITOS_CCCR_LO_CDWE
| TALITOS_CCCR_LO_CDIE
);
214 /* and ICCR writeback, if available */
215 if (priv
->features
& TALITOS_FTR_HW_AUTH_CHECK
)
216 setbits32(priv
->reg
+ TALITOS_CCCR_LO(ch
),
217 TALITOS_CCCR_LO_IWSE
);
222 static int reset_device(struct device
*dev
)
224 struct talitos_private
*priv
= dev_get_drvdata(dev
);
225 unsigned int timeout
= TALITOS_TIMEOUT
;
227 setbits32(priv
->reg
+ TALITOS_MCR
, TALITOS_MCR_SWR
);
229 while ((in_be32(priv
->reg
+ TALITOS_MCR
) & TALITOS_MCR_SWR
)
234 dev_err(dev
, "failed to reset device\n");
242 * Reset and initialize the device
244 static int init_device(struct device
*dev
)
246 struct talitos_private
*priv
= dev_get_drvdata(dev
);
251 * errata documentation: warning: certain SEC interrupts
252 * are not fully cleared by writing the MCR:SWR bit,
253 * set bit twice to completely reset
255 err
= reset_device(dev
);
259 err
= reset_device(dev
);
264 for (ch
= 0; ch
< priv
->num_channels
; ch
++) {
265 err
= reset_channel(dev
, ch
);
270 /* enable channel done and error interrupts */
271 setbits32(priv
->reg
+ TALITOS_IMR
, TALITOS_IMR_INIT
);
272 setbits32(priv
->reg
+ TALITOS_IMR_LO
, TALITOS_IMR_LO_INIT
);
274 /* disable integrity check error interrupts (use writeback instead) */
275 if (priv
->features
& TALITOS_FTR_HW_AUTH_CHECK
)
276 setbits32(priv
->reg
+ TALITOS_MDEUICR_LO
,
277 TALITOS_MDEUICR_LO_ICE
);
283 * talitos_submit - submits a descriptor to the device for processing
284 * @dev: the SEC device to be used
285 * @desc: the descriptor to be processed by the device
286 * @callback: whom to call when processing is complete
287 * @context: a handle for use by caller (optional)
289 * desc must contain valid dma-mapped (bus physical) address pointers.
290 * callback must check err and feedback in descriptor header
291 * for device processing status.
293 static int talitos_submit(struct device
*dev
, struct talitos_desc
*desc
,
294 void (*callback
)(struct device
*dev
,
295 struct talitos_desc
*desc
,
296 void *context
, int error
),
299 struct talitos_private
*priv
= dev_get_drvdata(dev
);
300 struct talitos_request
*request
;
301 unsigned long flags
, ch
;
304 /* select done notification */
305 desc
->hdr
|= DESC_HDR_DONE_NOTIFY
;
307 /* emulate SEC's round-robin channel fifo polling scheme */
308 ch
= atomic_inc_return(&priv
->last_chan
) & (priv
->num_channels
- 1);
310 spin_lock_irqsave(&priv
->chan
[ch
].head_lock
, flags
);
312 if (!atomic_inc_not_zero(&priv
->chan
[ch
].submit_count
)) {
313 /* h/w fifo is full */
314 spin_unlock_irqrestore(&priv
->chan
[ch
].head_lock
, flags
);
318 head
= priv
->chan
[ch
].head
;
319 request
= &priv
->chan
[ch
].fifo
[head
];
321 /* map descriptor and save caller data */
322 request
->dma_desc
= dma_map_single(dev
, desc
, sizeof(*desc
),
324 request
->callback
= callback
;
325 request
->context
= context
;
327 /* increment fifo head */
328 priv
->chan
[ch
].head
= (priv
->chan
[ch
].head
+ 1) & (priv
->fifo_len
- 1);
331 request
->desc
= desc
;
335 out_be32(priv
->reg
+ TALITOS_FF(ch
), upper_32_bits(request
->dma_desc
));
336 out_be32(priv
->reg
+ TALITOS_FF_LO(ch
),
337 lower_32_bits(request
->dma_desc
));
339 spin_unlock_irqrestore(&priv
->chan
[ch
].head_lock
, flags
);
345 * process what was done, notify callback of error if not
347 static void flush_channel(struct device
*dev
, int ch
, int error
, int reset_ch
)
349 struct talitos_private
*priv
= dev_get_drvdata(dev
);
350 struct talitos_request
*request
, saved_req
;
354 spin_lock_irqsave(&priv
->chan
[ch
].tail_lock
, flags
);
356 tail
= priv
->chan
[ch
].tail
;
357 while (priv
->chan
[ch
].fifo
[tail
].desc
) {
358 request
= &priv
->chan
[ch
].fifo
[tail
];
360 /* descriptors with their done bits set don't get the error */
362 if ((request
->desc
->hdr
& DESC_HDR_DONE
) == DESC_HDR_DONE
)
370 dma_unmap_single(dev
, request
->dma_desc
,
371 sizeof(struct talitos_desc
),
374 /* copy entries so we can call callback outside lock */
375 saved_req
.desc
= request
->desc
;
376 saved_req
.callback
= request
->callback
;
377 saved_req
.context
= request
->context
;
379 /* release request entry in fifo */
381 request
->desc
= NULL
;
383 /* increment fifo tail */
384 priv
->chan
[ch
].tail
= (tail
+ 1) & (priv
->fifo_len
- 1);
386 spin_unlock_irqrestore(&priv
->chan
[ch
].tail_lock
, flags
);
388 atomic_dec(&priv
->chan
[ch
].submit_count
);
390 saved_req
.callback(dev
, saved_req
.desc
, saved_req
.context
,
392 /* channel may resume processing in single desc error case */
393 if (error
&& !reset_ch
&& status
== error
)
395 spin_lock_irqsave(&priv
->chan
[ch
].tail_lock
, flags
);
396 tail
= priv
->chan
[ch
].tail
;
399 spin_unlock_irqrestore(&priv
->chan
[ch
].tail_lock
, flags
);
403 * process completed requests for channels that have done status
405 static void talitos_done(unsigned long data
)
407 struct device
*dev
= (struct device
*)data
;
408 struct talitos_private
*priv
= dev_get_drvdata(dev
);
411 for (ch
= 0; ch
< priv
->num_channels
; ch
++)
412 flush_channel(dev
, ch
, 0, 0);
414 /* At this point, all completed channels have been processed.
415 * Unmask done interrupts for channels completed later on.
417 setbits32(priv
->reg
+ TALITOS_IMR
, TALITOS_IMR_INIT
);
418 setbits32(priv
->reg
+ TALITOS_IMR_LO
, TALITOS_IMR_LO_INIT
);
422 * locate current (offending) descriptor
424 static struct talitos_desc
*current_desc(struct device
*dev
, int ch
)
426 struct talitos_private
*priv
= dev_get_drvdata(dev
);
427 int tail
= priv
->chan
[ch
].tail
;
430 cur_desc
= in_be32(priv
->reg
+ TALITOS_CDPR_LO(ch
));
432 while (priv
->chan
[ch
].fifo
[tail
].dma_desc
!= cur_desc
) {
433 tail
= (tail
+ 1) & (priv
->fifo_len
- 1);
434 if (tail
== priv
->chan
[ch
].tail
) {
435 dev_err(dev
, "couldn't locate current descriptor\n");
440 return priv
->chan
[ch
].fifo
[tail
].desc
;
444 * user diagnostics; report root cause of error based on execution unit status
446 static void report_eu_error(struct device
*dev
, int ch
,
447 struct talitos_desc
*desc
)
449 struct talitos_private
*priv
= dev_get_drvdata(dev
);
452 switch (desc
->hdr
& DESC_HDR_SEL0_MASK
) {
453 case DESC_HDR_SEL0_AFEU
:
454 dev_err(dev
, "AFEUISR 0x%08x_%08x\n",
455 in_be32(priv
->reg
+ TALITOS_AFEUISR
),
456 in_be32(priv
->reg
+ TALITOS_AFEUISR_LO
));
458 case DESC_HDR_SEL0_DEU
:
459 dev_err(dev
, "DEUISR 0x%08x_%08x\n",
460 in_be32(priv
->reg
+ TALITOS_DEUISR
),
461 in_be32(priv
->reg
+ TALITOS_DEUISR_LO
));
463 case DESC_HDR_SEL0_MDEUA
:
464 case DESC_HDR_SEL0_MDEUB
:
465 dev_err(dev
, "MDEUISR 0x%08x_%08x\n",
466 in_be32(priv
->reg
+ TALITOS_MDEUISR
),
467 in_be32(priv
->reg
+ TALITOS_MDEUISR_LO
));
469 case DESC_HDR_SEL0_RNG
:
470 dev_err(dev
, "RNGUISR 0x%08x_%08x\n",
471 in_be32(priv
->reg
+ TALITOS_RNGUISR
),
472 in_be32(priv
->reg
+ TALITOS_RNGUISR_LO
));
474 case DESC_HDR_SEL0_PKEU
:
475 dev_err(dev
, "PKEUISR 0x%08x_%08x\n",
476 in_be32(priv
->reg
+ TALITOS_PKEUISR
),
477 in_be32(priv
->reg
+ TALITOS_PKEUISR_LO
));
479 case DESC_HDR_SEL0_AESU
:
480 dev_err(dev
, "AESUISR 0x%08x_%08x\n",
481 in_be32(priv
->reg
+ TALITOS_AESUISR
),
482 in_be32(priv
->reg
+ TALITOS_AESUISR_LO
));
484 case DESC_HDR_SEL0_CRCU
:
485 dev_err(dev
, "CRCUISR 0x%08x_%08x\n",
486 in_be32(priv
->reg
+ TALITOS_CRCUISR
),
487 in_be32(priv
->reg
+ TALITOS_CRCUISR_LO
));
489 case DESC_HDR_SEL0_KEU
:
490 dev_err(dev
, "KEUISR 0x%08x_%08x\n",
491 in_be32(priv
->reg
+ TALITOS_KEUISR
),
492 in_be32(priv
->reg
+ TALITOS_KEUISR_LO
));
496 switch (desc
->hdr
& DESC_HDR_SEL1_MASK
) {
497 case DESC_HDR_SEL1_MDEUA
:
498 case DESC_HDR_SEL1_MDEUB
:
499 dev_err(dev
, "MDEUISR 0x%08x_%08x\n",
500 in_be32(priv
->reg
+ TALITOS_MDEUISR
),
501 in_be32(priv
->reg
+ TALITOS_MDEUISR_LO
));
503 case DESC_HDR_SEL1_CRCU
:
504 dev_err(dev
, "CRCUISR 0x%08x_%08x\n",
505 in_be32(priv
->reg
+ TALITOS_CRCUISR
),
506 in_be32(priv
->reg
+ TALITOS_CRCUISR_LO
));
510 for (i
= 0; i
< 8; i
++)
511 dev_err(dev
, "DESCBUF 0x%08x_%08x\n",
512 in_be32(priv
->reg
+ TALITOS_DESCBUF(ch
) + 8*i
),
513 in_be32(priv
->reg
+ TALITOS_DESCBUF_LO(ch
) + 8*i
));
517 * recover from error interrupts
519 static void talitos_error(unsigned long data
, u32 isr
, u32 isr_lo
)
521 struct device
*dev
= (struct device
*)data
;
522 struct talitos_private
*priv
= dev_get_drvdata(dev
);
523 unsigned int timeout
= TALITOS_TIMEOUT
;
524 int ch
, error
, reset_dev
= 0, reset_ch
= 0;
527 for (ch
= 0; ch
< priv
->num_channels
; ch
++) {
528 /* skip channels without errors */
529 if (!(isr
& (1 << (ch
* 2 + 1))))
534 v
= in_be32(priv
->reg
+ TALITOS_CCPSR(ch
));
535 v_lo
= in_be32(priv
->reg
+ TALITOS_CCPSR_LO(ch
));
537 if (v_lo
& TALITOS_CCPSR_LO_DOF
) {
538 dev_err(dev
, "double fetch fifo overflow error\n");
542 if (v_lo
& TALITOS_CCPSR_LO_SOF
) {
543 /* h/w dropped descriptor */
544 dev_err(dev
, "single fetch fifo overflow error\n");
547 if (v_lo
& TALITOS_CCPSR_LO_MDTE
)
548 dev_err(dev
, "master data transfer error\n");
549 if (v_lo
& TALITOS_CCPSR_LO_SGDLZ
)
550 dev_err(dev
, "s/g data length zero error\n");
551 if (v_lo
& TALITOS_CCPSR_LO_FPZ
)
552 dev_err(dev
, "fetch pointer zero error\n");
553 if (v_lo
& TALITOS_CCPSR_LO_IDH
)
554 dev_err(dev
, "illegal descriptor header error\n");
555 if (v_lo
& TALITOS_CCPSR_LO_IEU
)
556 dev_err(dev
, "invalid execution unit error\n");
557 if (v_lo
& TALITOS_CCPSR_LO_EU
)
558 report_eu_error(dev
, ch
, current_desc(dev
, ch
));
559 if (v_lo
& TALITOS_CCPSR_LO_GB
)
560 dev_err(dev
, "gather boundary error\n");
561 if (v_lo
& TALITOS_CCPSR_LO_GRL
)
562 dev_err(dev
, "gather return/length error\n");
563 if (v_lo
& TALITOS_CCPSR_LO_SB
)
564 dev_err(dev
, "scatter boundary error\n");
565 if (v_lo
& TALITOS_CCPSR_LO_SRL
)
566 dev_err(dev
, "scatter return/length error\n");
568 flush_channel(dev
, ch
, error
, reset_ch
);
571 reset_channel(dev
, ch
);
573 setbits32(priv
->reg
+ TALITOS_CCCR(ch
),
575 setbits32(priv
->reg
+ TALITOS_CCCR_LO(ch
), 0);
576 while ((in_be32(priv
->reg
+ TALITOS_CCCR(ch
)) &
577 TALITOS_CCCR_CONT
) && --timeout
)
580 dev_err(dev
, "failed to restart channel %d\n",
586 if (reset_dev
|| isr
& ~TALITOS_ISR_CHERR
|| isr_lo
) {
587 dev_err(dev
, "done overflow, internal time out, or rngu error: "
588 "ISR 0x%08x_%08x\n", isr
, isr_lo
);
590 /* purge request queues */
591 for (ch
= 0; ch
< priv
->num_channels
; ch
++)
592 flush_channel(dev
, ch
, -EIO
, 1);
594 /* reset and reinitialize the device */
599 static irqreturn_t
talitos_interrupt(int irq
, void *data
)
601 struct device
*dev
= data
;
602 struct talitos_private
*priv
= dev_get_drvdata(dev
);
605 isr
= in_be32(priv
->reg
+ TALITOS_ISR
);
606 isr_lo
= in_be32(priv
->reg
+ TALITOS_ISR_LO
);
607 /* Acknowledge interrupt */
608 out_be32(priv
->reg
+ TALITOS_ICR
, isr
);
609 out_be32(priv
->reg
+ TALITOS_ICR_LO
, isr_lo
);
611 if (unlikely((isr
& ~TALITOS_ISR_CHDONE
) || isr_lo
))
612 talitos_error((unsigned long)data
, isr
, isr_lo
);
614 if (likely(isr
& TALITOS_ISR_CHDONE
)) {
615 /* mask further done interrupts. */
616 clrbits32(priv
->reg
+ TALITOS_IMR
, TALITOS_IMR_DONE
);
617 /* done_task will unmask done interrupts at exit */
618 tasklet_schedule(&priv
->done_task
);
621 return (isr
|| isr_lo
) ? IRQ_HANDLED
: IRQ_NONE
;
627 static int talitos_rng_data_present(struct hwrng
*rng
, int wait
)
629 struct device
*dev
= (struct device
*)rng
->priv
;
630 struct talitos_private
*priv
= dev_get_drvdata(dev
);
634 for (i
= 0; i
< 20; i
++) {
635 ofl
= in_be32(priv
->reg
+ TALITOS_RNGUSR_LO
) &
636 TALITOS_RNGUSR_LO_OFL
;
645 static int talitos_rng_data_read(struct hwrng
*rng
, u32
*data
)
647 struct device
*dev
= (struct device
*)rng
->priv
;
648 struct talitos_private
*priv
= dev_get_drvdata(dev
);
650 /* rng fifo requires 64-bit accesses */
651 *data
= in_be32(priv
->reg
+ TALITOS_RNGU_FIFO
);
652 *data
= in_be32(priv
->reg
+ TALITOS_RNGU_FIFO_LO
);
657 static int talitos_rng_init(struct hwrng
*rng
)
659 struct device
*dev
= (struct device
*)rng
->priv
;
660 struct talitos_private
*priv
= dev_get_drvdata(dev
);
661 unsigned int timeout
= TALITOS_TIMEOUT
;
663 setbits32(priv
->reg
+ TALITOS_RNGURCR_LO
, TALITOS_RNGURCR_LO_SR
);
664 while (!(in_be32(priv
->reg
+ TALITOS_RNGUSR_LO
) & TALITOS_RNGUSR_LO_RD
)
668 dev_err(dev
, "failed to reset rng hw\n");
672 /* start generating */
673 setbits32(priv
->reg
+ TALITOS_RNGUDSR_LO
, 0);
678 static int talitos_register_rng(struct device
*dev
)
680 struct talitos_private
*priv
= dev_get_drvdata(dev
);
682 priv
->rng
.name
= dev_driver_string(dev
),
683 priv
->rng
.init
= talitos_rng_init
,
684 priv
->rng
.data_present
= talitos_rng_data_present
,
685 priv
->rng
.data_read
= talitos_rng_data_read
,
686 priv
->rng
.priv
= (unsigned long)dev
;
688 return hwrng_register(&priv
->rng
);
691 static void talitos_unregister_rng(struct device
*dev
)
693 struct talitos_private
*priv
= dev_get_drvdata(dev
);
695 hwrng_unregister(&priv
->rng
);
701 #define TALITOS_CRA_PRIORITY 3000
702 #define TALITOS_MAX_KEY_SIZE 64
703 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
705 #define MD5_BLOCK_SIZE 64
709 __be32 desc_hdr_template
;
710 u8 key
[TALITOS_MAX_KEY_SIZE
];
711 u8 iv
[TALITOS_MAX_IV_LENGTH
];
713 unsigned int enckeylen
;
714 unsigned int authkeylen
;
715 unsigned int authsize
;
718 #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
719 #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
721 struct talitos_ahash_req_ctx
{
722 u32 hw_context
[TALITOS_MDEU_MAX_CONTEXT_SIZE
/ sizeof(u32
)];
723 unsigned int hw_context_size
;
724 u8 buf
[HASH_MAX_BLOCK_SIZE
];
725 u8 bufnext
[HASH_MAX_BLOCK_SIZE
];
729 unsigned int to_hash_later
;
731 struct scatterlist bufsl
[2];
732 struct scatterlist
*psrc
;
735 static int aead_setauthsize(struct crypto_aead
*authenc
,
736 unsigned int authsize
)
738 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
740 ctx
->authsize
= authsize
;
745 static int aead_setkey(struct crypto_aead
*authenc
,
746 const u8
*key
, unsigned int keylen
)
748 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
749 struct rtattr
*rta
= (void *)key
;
750 struct crypto_authenc_key_param
*param
;
751 unsigned int authkeylen
;
752 unsigned int enckeylen
;
754 if (!RTA_OK(rta
, keylen
))
757 if (rta
->rta_type
!= CRYPTO_AUTHENC_KEYA_PARAM
)
760 if (RTA_PAYLOAD(rta
) < sizeof(*param
))
763 param
= RTA_DATA(rta
);
764 enckeylen
= be32_to_cpu(param
->enckeylen
);
766 key
+= RTA_ALIGN(rta
->rta_len
);
767 keylen
-= RTA_ALIGN(rta
->rta_len
);
769 if (keylen
< enckeylen
)
772 authkeylen
= keylen
- enckeylen
;
774 if (keylen
> TALITOS_MAX_KEY_SIZE
)
777 memcpy(&ctx
->key
, key
, keylen
);
779 ctx
->keylen
= keylen
;
780 ctx
->enckeylen
= enckeylen
;
781 ctx
->authkeylen
= authkeylen
;
786 crypto_aead_set_flags(authenc
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
791 * talitos_edesc - s/w-extended descriptor
792 * @src_nents: number of segments in input scatterlist
793 * @dst_nents: number of segments in output scatterlist
794 * @dma_len: length of dma mapped link_tbl space
795 * @dma_link_tbl: bus physical address of link_tbl
796 * @desc: h/w descriptor
797 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
799 * if decrypting (with authcheck), or either one of src_nents or dst_nents
800 * is greater than 1, an integrity check value is concatenated to the end
803 struct talitos_edesc
{
809 dma_addr_t dma_link_tbl
;
810 struct talitos_desc desc
;
811 struct talitos_ptr link_tbl
[0];
814 static int talitos_map_sg(struct device
*dev
, struct scatterlist
*sg
,
815 unsigned int nents
, enum dma_data_direction dir
,
818 if (unlikely(chained
))
820 dma_map_sg(dev
, sg
, 1, dir
);
821 sg
= scatterwalk_sg_next(sg
);
824 dma_map_sg(dev
, sg
, nents
, dir
);
828 static void talitos_unmap_sg_chain(struct device
*dev
, struct scatterlist
*sg
,
829 enum dma_data_direction dir
)
832 dma_unmap_sg(dev
, sg
, 1, dir
);
833 sg
= scatterwalk_sg_next(sg
);
837 static void talitos_sg_unmap(struct device
*dev
,
838 struct talitos_edesc
*edesc
,
839 struct scatterlist
*src
,
840 struct scatterlist
*dst
)
842 unsigned int src_nents
= edesc
->src_nents
? : 1;
843 unsigned int dst_nents
= edesc
->dst_nents
? : 1;
846 if (edesc
->src_is_chained
)
847 talitos_unmap_sg_chain(dev
, src
, DMA_TO_DEVICE
);
849 dma_unmap_sg(dev
, src
, src_nents
, DMA_TO_DEVICE
);
852 if (edesc
->dst_is_chained
)
853 talitos_unmap_sg_chain(dev
, dst
,
856 dma_unmap_sg(dev
, dst
, dst_nents
,
860 if (edesc
->src_is_chained
)
861 talitos_unmap_sg_chain(dev
, src
, DMA_BIDIRECTIONAL
);
863 dma_unmap_sg(dev
, src
, src_nents
, DMA_BIDIRECTIONAL
);
866 static void ipsec_esp_unmap(struct device
*dev
,
867 struct talitos_edesc
*edesc
,
868 struct aead_request
*areq
)
870 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[6], DMA_FROM_DEVICE
);
871 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[3], DMA_TO_DEVICE
);
872 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[2], DMA_TO_DEVICE
);
873 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[0], DMA_TO_DEVICE
);
875 dma_unmap_sg(dev
, areq
->assoc
, 1, DMA_TO_DEVICE
);
877 talitos_sg_unmap(dev
, edesc
, areq
->src
, areq
->dst
);
880 dma_unmap_single(dev
, edesc
->dma_link_tbl
, edesc
->dma_len
,
885 * ipsec_esp descriptor callbacks
887 static void ipsec_esp_encrypt_done(struct device
*dev
,
888 struct talitos_desc
*desc
, void *context
,
891 struct aead_request
*areq
= context
;
892 struct crypto_aead
*authenc
= crypto_aead_reqtfm(areq
);
893 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
894 struct talitos_edesc
*edesc
;
895 struct scatterlist
*sg
;
898 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
900 ipsec_esp_unmap(dev
, edesc
, areq
);
902 /* copy the generated ICV to dst */
903 if (edesc
->dma_len
) {
904 icvdata
= &edesc
->link_tbl
[edesc
->src_nents
+
905 edesc
->dst_nents
+ 2];
906 sg
= sg_last(areq
->dst
, edesc
->dst_nents
);
907 memcpy((char *)sg_virt(sg
) + sg
->length
- ctx
->authsize
,
908 icvdata
, ctx
->authsize
);
913 aead_request_complete(areq
, err
);
916 static void ipsec_esp_decrypt_swauth_done(struct device
*dev
,
917 struct talitos_desc
*desc
,
918 void *context
, int err
)
920 struct aead_request
*req
= context
;
921 struct crypto_aead
*authenc
= crypto_aead_reqtfm(req
);
922 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
923 struct talitos_edesc
*edesc
;
924 struct scatterlist
*sg
;
927 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
929 ipsec_esp_unmap(dev
, edesc
, req
);
934 icvdata
= &edesc
->link_tbl
[edesc
->src_nents
+
935 edesc
->dst_nents
+ 2];
937 icvdata
= &edesc
->link_tbl
[0];
939 sg
= sg_last(req
->dst
, edesc
->dst_nents
? : 1);
940 err
= memcmp(icvdata
, (char *)sg_virt(sg
) + sg
->length
-
941 ctx
->authsize
, ctx
->authsize
) ? -EBADMSG
: 0;
946 aead_request_complete(req
, err
);
949 static void ipsec_esp_decrypt_hwauth_done(struct device
*dev
,
950 struct talitos_desc
*desc
,
951 void *context
, int err
)
953 struct aead_request
*req
= context
;
954 struct talitos_edesc
*edesc
;
956 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
958 ipsec_esp_unmap(dev
, edesc
, req
);
960 /* check ICV auth status */
961 if (!err
&& ((desc
->hdr_lo
& DESC_HDR_LO_ICCR1_MASK
) !=
962 DESC_HDR_LO_ICCR1_PASS
))
967 aead_request_complete(req
, err
);
971 * convert scatterlist to SEC h/w link table format
972 * stop at cryptlen bytes
974 static int sg_to_link_tbl(struct scatterlist
*sg
, int sg_count
,
975 int cryptlen
, struct talitos_ptr
*link_tbl_ptr
)
980 to_talitos_ptr(link_tbl_ptr
, sg_dma_address(sg
));
981 link_tbl_ptr
->len
= cpu_to_be16(sg_dma_len(sg
));
982 link_tbl_ptr
->j_extent
= 0;
984 cryptlen
-= sg_dma_len(sg
);
985 sg
= scatterwalk_sg_next(sg
);
988 /* adjust (decrease) last one (or two) entry's len to cryptlen */
990 while (be16_to_cpu(link_tbl_ptr
->len
) <= (-cryptlen
)) {
991 /* Empty this entry, and move to previous one */
992 cryptlen
+= be16_to_cpu(link_tbl_ptr
->len
);
993 link_tbl_ptr
->len
= 0;
997 link_tbl_ptr
->len
= cpu_to_be16(be16_to_cpu(link_tbl_ptr
->len
)
1000 /* tag end of link table */
1001 link_tbl_ptr
->j_extent
= DESC_PTR_LNKTBL_RETURN
;
1007 * fill in and submit ipsec_esp descriptor
1009 static int ipsec_esp(struct talitos_edesc
*edesc
, struct aead_request
*areq
,
1011 void (*callback
) (struct device
*dev
,
1012 struct talitos_desc
*desc
,
1013 void *context
, int error
))
1015 struct crypto_aead
*aead
= crypto_aead_reqtfm(areq
);
1016 struct talitos_ctx
*ctx
= crypto_aead_ctx(aead
);
1017 struct device
*dev
= ctx
->dev
;
1018 struct talitos_desc
*desc
= &edesc
->desc
;
1019 unsigned int cryptlen
= areq
->cryptlen
;
1020 unsigned int authsize
= ctx
->authsize
;
1021 unsigned int ivsize
= crypto_aead_ivsize(aead
);
1023 int sg_link_tbl_len
;
1026 map_single_talitos_ptr(dev
, &desc
->ptr
[0], ctx
->authkeylen
, &ctx
->key
,
1029 map_single_talitos_ptr(dev
, &desc
->ptr
[1], areq
->assoclen
+ ivsize
,
1030 sg_virt(areq
->assoc
), 0, DMA_TO_DEVICE
);
1032 map_single_talitos_ptr(dev
, &desc
->ptr
[2], ivsize
, giv
?: areq
->iv
, 0,
1036 map_single_talitos_ptr(dev
, &desc
->ptr
[3], ctx
->enckeylen
,
1037 (char *)&ctx
->key
+ ctx
->authkeylen
, 0,
1042 * map and adjust cipher len to aead request cryptlen.
1043 * extent is bytes of HMAC postpended to ciphertext,
1044 * typically 12 for ipsec
1046 desc
->ptr
[4].len
= cpu_to_be16(cryptlen
);
1047 desc
->ptr
[4].j_extent
= authsize
;
1049 sg_count
= talitos_map_sg(dev
, areq
->src
, edesc
->src_nents
? : 1,
1050 (areq
->src
== areq
->dst
) ? DMA_BIDIRECTIONAL
1052 edesc
->src_is_chained
);
1054 if (sg_count
== 1) {
1055 to_talitos_ptr(&desc
->ptr
[4], sg_dma_address(areq
->src
));
1057 sg_link_tbl_len
= cryptlen
;
1059 if (edesc
->desc
.hdr
& DESC_HDR_MODE1_MDEU_CICV
)
1060 sg_link_tbl_len
= cryptlen
+ authsize
;
1062 sg_count
= sg_to_link_tbl(areq
->src
, sg_count
, sg_link_tbl_len
,
1063 &edesc
->link_tbl
[0]);
1065 desc
->ptr
[4].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1066 to_talitos_ptr(&desc
->ptr
[4], edesc
->dma_link_tbl
);
1067 dma_sync_single_for_device(dev
, edesc
->dma_link_tbl
,
1071 /* Only one segment now, so no link tbl needed */
1072 to_talitos_ptr(&desc
->ptr
[4],
1073 sg_dma_address(areq
->src
));
1078 desc
->ptr
[5].len
= cpu_to_be16(cryptlen
);
1079 desc
->ptr
[5].j_extent
= authsize
;
1081 if (areq
->src
!= areq
->dst
)
1082 sg_count
= talitos_map_sg(dev
, areq
->dst
,
1083 edesc
->dst_nents
? : 1,
1085 edesc
->dst_is_chained
);
1087 if (sg_count
== 1) {
1088 to_talitos_ptr(&desc
->ptr
[5], sg_dma_address(areq
->dst
));
1090 struct talitos_ptr
*link_tbl_ptr
=
1091 &edesc
->link_tbl
[edesc
->src_nents
+ 1];
1093 to_talitos_ptr(&desc
->ptr
[5], edesc
->dma_link_tbl
+
1094 (edesc
->src_nents
+ 1) *
1095 sizeof(struct talitos_ptr
));
1096 sg_count
= sg_to_link_tbl(areq
->dst
, sg_count
, cryptlen
,
1099 /* Add an entry to the link table for ICV data */
1100 link_tbl_ptr
+= sg_count
- 1;
1101 link_tbl_ptr
->j_extent
= 0;
1104 link_tbl_ptr
->j_extent
= DESC_PTR_LNKTBL_RETURN
;
1105 link_tbl_ptr
->len
= cpu_to_be16(authsize
);
1107 /* icv data follows link tables */
1108 to_talitos_ptr(link_tbl_ptr
, edesc
->dma_link_tbl
+
1109 (edesc
->src_nents
+ edesc
->dst_nents
+ 2) *
1110 sizeof(struct talitos_ptr
));
1111 desc
->ptr
[5].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1112 dma_sync_single_for_device(ctx
->dev
, edesc
->dma_link_tbl
,
1113 edesc
->dma_len
, DMA_BIDIRECTIONAL
);
1117 map_single_talitos_ptr(dev
, &desc
->ptr
[6], ivsize
, ctx
->iv
, 0,
1120 ret
= talitos_submit(dev
, desc
, callback
, areq
);
1121 if (ret
!= -EINPROGRESS
) {
1122 ipsec_esp_unmap(dev
, edesc
, areq
);
1129 * derive number of elements in scatterlist
1131 static int sg_count(struct scatterlist
*sg_list
, int nbytes
, int *chained
)
1133 struct scatterlist
*sg
= sg_list
;
1137 while (nbytes
> 0) {
1139 nbytes
-= sg
->length
;
1140 if (!sg_is_last(sg
) && (sg
+ 1)->length
== 0)
1142 sg
= scatterwalk_sg_next(sg
);
1149 * sg_copy_end_to_buffer - Copy end data from SG list to a linear buffer
1151 * @nents: Number of SG entries
1152 * @buf: Where to copy to
1153 * @buflen: The number of bytes to copy
1154 * @skip: The number of bytes to skip before copying.
1155 * Note: skip + buflen should equal SG total size.
1157 * Returns the number of copied bytes.
1160 static size_t sg_copy_end_to_buffer(struct scatterlist
*sgl
, unsigned int nents
,
1161 void *buf
, size_t buflen
, unsigned int skip
)
1163 unsigned int offset
= 0;
1164 unsigned int boffset
= 0;
1165 struct sg_mapping_iter miter
;
1166 unsigned long flags
;
1167 unsigned int sg_flags
= SG_MITER_ATOMIC
;
1168 size_t total_buffer
= buflen
+ skip
;
1170 sg_flags
|= SG_MITER_FROM_SG
;
1172 sg_miter_start(&miter
, sgl
, nents
, sg_flags
);
1174 local_irq_save(flags
);
1176 while (sg_miter_next(&miter
) && offset
< total_buffer
) {
1178 unsigned int ignore
;
1180 if ((offset
+ miter
.length
) > skip
) {
1181 if (offset
< skip
) {
1182 /* Copy part of this segment */
1183 ignore
= skip
- offset
;
1184 len
= miter
.length
- ignore
;
1185 if (boffset
+ len
> buflen
)
1186 len
= buflen
- boffset
;
1187 memcpy(buf
+ boffset
, miter
.addr
+ ignore
, len
);
1189 /* Copy all of this segment (up to buflen) */
1191 if (boffset
+ len
> buflen
)
1192 len
= buflen
- boffset
;
1193 memcpy(buf
+ boffset
, miter
.addr
, len
);
1197 offset
+= miter
.length
;
1200 sg_miter_stop(&miter
);
1202 local_irq_restore(flags
);
1207 * allocate and map the extended descriptor
1209 static struct talitos_edesc
*talitos_edesc_alloc(struct device
*dev
,
1210 struct scatterlist
*src
,
1211 struct scatterlist
*dst
,
1213 unsigned int cryptlen
,
1214 unsigned int authsize
,
1218 struct talitos_edesc
*edesc
;
1219 int src_nents
, dst_nents
, alloc_len
, dma_len
;
1220 int src_chained
, dst_chained
= 0;
1221 gfp_t flags
= cryptoflags
& CRYPTO_TFM_REQ_MAY_SLEEP
? GFP_KERNEL
:
1224 if (cryptlen
+ authsize
> TALITOS_MAX_DATA_LEN
) {
1225 dev_err(dev
, "length exceeds h/w max limit\n");
1226 return ERR_PTR(-EINVAL
);
1229 src_nents
= sg_count(src
, cryptlen
+ authsize
, &src_chained
);
1230 src_nents
= (src_nents
== 1) ? 0 : src_nents
;
1236 dst_nents
= src_nents
;
1238 dst_nents
= sg_count(dst
, cryptlen
+ authsize
,
1240 dst_nents
= (dst_nents
== 1) ? 0 : dst_nents
;
1245 * allocate space for base edesc plus the link tables,
1246 * allowing for two separate entries for ICV and generated ICV (+ 2),
1247 * and the ICV data itself
1249 alloc_len
= sizeof(struct talitos_edesc
);
1250 if (src_nents
|| dst_nents
) {
1251 dma_len
= (src_nents
+ dst_nents
+ 2) *
1252 sizeof(struct talitos_ptr
) + authsize
;
1253 alloc_len
+= dma_len
;
1256 alloc_len
+= icv_stashing
? authsize
: 0;
1259 edesc
= kmalloc(alloc_len
, GFP_DMA
| flags
);
1261 dev_err(dev
, "could not allocate edescriptor\n");
1262 return ERR_PTR(-ENOMEM
);
1265 edesc
->src_nents
= src_nents
;
1266 edesc
->dst_nents
= dst_nents
;
1267 edesc
->src_is_chained
= src_chained
;
1268 edesc
->dst_is_chained
= dst_chained
;
1269 edesc
->dma_len
= dma_len
;
1271 edesc
->dma_link_tbl
= dma_map_single(dev
, &edesc
->link_tbl
[0],
1278 static struct talitos_edesc
*aead_edesc_alloc(struct aead_request
*areq
,
1281 struct crypto_aead
*authenc
= crypto_aead_reqtfm(areq
);
1282 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1284 return talitos_edesc_alloc(ctx
->dev
, areq
->src
, areq
->dst
, 0,
1285 areq
->cryptlen
, ctx
->authsize
, icv_stashing
,
1289 static int aead_encrypt(struct aead_request
*req
)
1291 struct crypto_aead
*authenc
= crypto_aead_reqtfm(req
);
1292 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1293 struct talitos_edesc
*edesc
;
1295 /* allocate extended descriptor */
1296 edesc
= aead_edesc_alloc(req
, 0);
1298 return PTR_ERR(edesc
);
1301 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_MODE0_ENCRYPT
;
1303 return ipsec_esp(edesc
, req
, NULL
, 0, ipsec_esp_encrypt_done
);
1306 static int aead_decrypt(struct aead_request
*req
)
1308 struct crypto_aead
*authenc
= crypto_aead_reqtfm(req
);
1309 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1310 unsigned int authsize
= ctx
->authsize
;
1311 struct talitos_private
*priv
= dev_get_drvdata(ctx
->dev
);
1312 struct talitos_edesc
*edesc
;
1313 struct scatterlist
*sg
;
1316 req
->cryptlen
-= authsize
;
1318 /* allocate extended descriptor */
1319 edesc
= aead_edesc_alloc(req
, 1);
1321 return PTR_ERR(edesc
);
1323 if ((priv
->features
& TALITOS_FTR_HW_AUTH_CHECK
) &&
1324 ((!edesc
->src_nents
&& !edesc
->dst_nents
) ||
1325 priv
->features
& TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT
)) {
1327 /* decrypt and check the ICV */
1328 edesc
->desc
.hdr
= ctx
->desc_hdr_template
|
1329 DESC_HDR_DIR_INBOUND
|
1330 DESC_HDR_MODE1_MDEU_CICV
;
1332 /* reset integrity check result bits */
1333 edesc
->desc
.hdr_lo
= 0;
1335 return ipsec_esp(edesc
, req
, NULL
, 0,
1336 ipsec_esp_decrypt_hwauth_done
);
1340 /* Have to check the ICV with software */
1341 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_DIR_INBOUND
;
1343 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1345 icvdata
= &edesc
->link_tbl
[edesc
->src_nents
+
1346 edesc
->dst_nents
+ 2];
1348 icvdata
= &edesc
->link_tbl
[0];
1350 sg
= sg_last(req
->src
, edesc
->src_nents
? : 1);
1352 memcpy(icvdata
, (char *)sg_virt(sg
) + sg
->length
- ctx
->authsize
,
1355 return ipsec_esp(edesc
, req
, NULL
, 0, ipsec_esp_decrypt_swauth_done
);
1358 static int aead_givencrypt(struct aead_givcrypt_request
*req
)
1360 struct aead_request
*areq
= &req
->areq
;
1361 struct crypto_aead
*authenc
= crypto_aead_reqtfm(areq
);
1362 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1363 struct talitos_edesc
*edesc
;
1365 /* allocate extended descriptor */
1366 edesc
= aead_edesc_alloc(areq
, 0);
1368 return PTR_ERR(edesc
);
1371 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_MODE0_ENCRYPT
;
1373 memcpy(req
->giv
, ctx
->iv
, crypto_aead_ivsize(authenc
));
1374 /* avoid consecutive packets going out with same IV */
1375 *(__be64
*)req
->giv
^= cpu_to_be64(req
->seq
);
1377 return ipsec_esp(edesc
, areq
, req
->giv
, req
->seq
,
1378 ipsec_esp_encrypt_done
);
1381 static int ablkcipher_setkey(struct crypto_ablkcipher
*cipher
,
1382 const u8
*key
, unsigned int keylen
)
1384 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1385 struct ablkcipher_alg
*alg
= crypto_ablkcipher_alg(cipher
);
1387 if (keylen
> TALITOS_MAX_KEY_SIZE
)
1390 if (keylen
< alg
->min_keysize
|| keylen
> alg
->max_keysize
)
1393 memcpy(&ctx
->key
, key
, keylen
);
1394 ctx
->keylen
= keylen
;
1399 crypto_ablkcipher_set_flags(cipher
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
1403 static void common_nonsnoop_unmap(struct device
*dev
,
1404 struct talitos_edesc
*edesc
,
1405 struct ablkcipher_request
*areq
)
1407 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[5], DMA_FROM_DEVICE
);
1408 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[2], DMA_TO_DEVICE
);
1409 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[1], DMA_TO_DEVICE
);
1411 talitos_sg_unmap(dev
, edesc
, areq
->src
, areq
->dst
);
1414 dma_unmap_single(dev
, edesc
->dma_link_tbl
, edesc
->dma_len
,
1418 static void ablkcipher_done(struct device
*dev
,
1419 struct talitos_desc
*desc
, void *context
,
1422 struct ablkcipher_request
*areq
= context
;
1423 struct talitos_edesc
*edesc
;
1425 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
1427 common_nonsnoop_unmap(dev
, edesc
, areq
);
1431 areq
->base
.complete(&areq
->base
, err
);
1434 static int common_nonsnoop(struct talitos_edesc
*edesc
,
1435 struct ablkcipher_request
*areq
,
1437 void (*callback
) (struct device
*dev
,
1438 struct talitos_desc
*desc
,
1439 void *context
, int error
))
1441 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1442 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1443 struct device
*dev
= ctx
->dev
;
1444 struct talitos_desc
*desc
= &edesc
->desc
;
1445 unsigned int cryptlen
= areq
->nbytes
;
1446 unsigned int ivsize
;
1449 /* first DWORD empty */
1450 desc
->ptr
[0].len
= 0;
1451 to_talitos_ptr(&desc
->ptr
[0], 0);
1452 desc
->ptr
[0].j_extent
= 0;
1455 ivsize
= crypto_ablkcipher_ivsize(cipher
);
1456 map_single_talitos_ptr(dev
, &desc
->ptr
[1], ivsize
, giv
?: areq
->info
, 0,
1460 map_single_talitos_ptr(dev
, &desc
->ptr
[2], ctx
->keylen
,
1461 (char *)&ctx
->key
, 0, DMA_TO_DEVICE
);
1466 desc
->ptr
[3].len
= cpu_to_be16(cryptlen
);
1467 desc
->ptr
[3].j_extent
= 0;
1469 sg_count
= talitos_map_sg(dev
, areq
->src
, edesc
->src_nents
? : 1,
1470 (areq
->src
== areq
->dst
) ? DMA_BIDIRECTIONAL
1472 edesc
->src_is_chained
);
1474 if (sg_count
== 1) {
1475 to_talitos_ptr(&desc
->ptr
[3], sg_dma_address(areq
->src
));
1477 sg_count
= sg_to_link_tbl(areq
->src
, sg_count
, cryptlen
,
1478 &edesc
->link_tbl
[0]);
1480 to_talitos_ptr(&desc
->ptr
[3], edesc
->dma_link_tbl
);
1481 desc
->ptr
[3].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1482 dma_sync_single_for_device(dev
, edesc
->dma_link_tbl
,
1486 /* Only one segment now, so no link tbl needed */
1487 to_talitos_ptr(&desc
->ptr
[3],
1488 sg_dma_address(areq
->src
));
1493 desc
->ptr
[4].len
= cpu_to_be16(cryptlen
);
1494 desc
->ptr
[4].j_extent
= 0;
1496 if (areq
->src
!= areq
->dst
)
1497 sg_count
= talitos_map_sg(dev
, areq
->dst
,
1498 edesc
->dst_nents
? : 1,
1500 edesc
->dst_is_chained
);
1502 if (sg_count
== 1) {
1503 to_talitos_ptr(&desc
->ptr
[4], sg_dma_address(areq
->dst
));
1505 struct talitos_ptr
*link_tbl_ptr
=
1506 &edesc
->link_tbl
[edesc
->src_nents
+ 1];
1508 to_talitos_ptr(&desc
->ptr
[4], edesc
->dma_link_tbl
+
1509 (edesc
->src_nents
+ 1) *
1510 sizeof(struct talitos_ptr
));
1511 desc
->ptr
[4].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1512 sg_count
= sg_to_link_tbl(areq
->dst
, sg_count
, cryptlen
,
1514 dma_sync_single_for_device(ctx
->dev
, edesc
->dma_link_tbl
,
1515 edesc
->dma_len
, DMA_BIDIRECTIONAL
);
1519 map_single_talitos_ptr(dev
, &desc
->ptr
[5], ivsize
, ctx
->iv
, 0,
1522 /* last DWORD empty */
1523 desc
->ptr
[6].len
= 0;
1524 to_talitos_ptr(&desc
->ptr
[6], 0);
1525 desc
->ptr
[6].j_extent
= 0;
1527 ret
= talitos_submit(dev
, desc
, callback
, areq
);
1528 if (ret
!= -EINPROGRESS
) {
1529 common_nonsnoop_unmap(dev
, edesc
, areq
);
1535 static struct talitos_edesc
*ablkcipher_edesc_alloc(struct ablkcipher_request
*
1538 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1539 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1541 return talitos_edesc_alloc(ctx
->dev
, areq
->src
, areq
->dst
, 0,
1542 areq
->nbytes
, 0, 0, areq
->base
.flags
);
1545 static int ablkcipher_encrypt(struct ablkcipher_request
*areq
)
1547 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1548 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1549 struct talitos_edesc
*edesc
;
1551 /* allocate extended descriptor */
1552 edesc
= ablkcipher_edesc_alloc(areq
);
1554 return PTR_ERR(edesc
);
1557 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_MODE0_ENCRYPT
;
1559 return common_nonsnoop(edesc
, areq
, NULL
, ablkcipher_done
);
1562 static int ablkcipher_decrypt(struct ablkcipher_request
*areq
)
1564 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1565 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1566 struct talitos_edesc
*edesc
;
1568 /* allocate extended descriptor */
1569 edesc
= ablkcipher_edesc_alloc(areq
);
1571 return PTR_ERR(edesc
);
1573 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_DIR_INBOUND
;
1575 return common_nonsnoop(edesc
, areq
, NULL
, ablkcipher_done
);
1578 static void common_nonsnoop_hash_unmap(struct device
*dev
,
1579 struct talitos_edesc
*edesc
,
1580 struct ahash_request
*areq
)
1582 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1584 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[5], DMA_FROM_DEVICE
);
1586 /* When using hashctx-in, must unmap it. */
1587 if (edesc
->desc
.ptr
[1].len
)
1588 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[1],
1591 if (edesc
->desc
.ptr
[2].len
)
1592 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[2],
1595 talitos_sg_unmap(dev
, edesc
, req_ctx
->psrc
, NULL
);
1598 dma_unmap_single(dev
, edesc
->dma_link_tbl
, edesc
->dma_len
,
1603 static void ahash_done(struct device
*dev
,
1604 struct talitos_desc
*desc
, void *context
,
1607 struct ahash_request
*areq
= context
;
1608 struct talitos_edesc
*edesc
=
1609 container_of(desc
, struct talitos_edesc
, desc
);
1610 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1612 if (!req_ctx
->last
&& req_ctx
->to_hash_later
) {
1613 /* Position any partial block for next update/final/finup */
1614 memcpy(req_ctx
->buf
, req_ctx
->bufnext
, req_ctx
->to_hash_later
);
1615 req_ctx
->nbuf
= req_ctx
->to_hash_later
;
1617 common_nonsnoop_hash_unmap(dev
, edesc
, areq
);
1621 areq
->base
.complete(&areq
->base
, err
);
1624 static int common_nonsnoop_hash(struct talitos_edesc
*edesc
,
1625 struct ahash_request
*areq
, unsigned int length
,
1626 void (*callback
) (struct device
*dev
,
1627 struct talitos_desc
*desc
,
1628 void *context
, int error
))
1630 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
1631 struct talitos_ctx
*ctx
= crypto_ahash_ctx(tfm
);
1632 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1633 struct device
*dev
= ctx
->dev
;
1634 struct talitos_desc
*desc
= &edesc
->desc
;
1637 /* first DWORD empty */
1638 desc
->ptr
[0] = zero_entry
;
1640 /* hash context in */
1641 if (!req_ctx
->first
|| req_ctx
->swinit
) {
1642 map_single_talitos_ptr(dev
, &desc
->ptr
[1],
1643 req_ctx
->hw_context_size
,
1644 (char *)req_ctx
->hw_context
, 0,
1646 req_ctx
->swinit
= 0;
1648 desc
->ptr
[1] = zero_entry
;
1649 /* Indicate next op is not the first. */
1655 map_single_talitos_ptr(dev
, &desc
->ptr
[2], ctx
->keylen
,
1656 (char *)&ctx
->key
, 0, DMA_TO_DEVICE
);
1658 desc
->ptr
[2] = zero_entry
;
1663 desc
->ptr
[3].len
= cpu_to_be16(length
);
1664 desc
->ptr
[3].j_extent
= 0;
1666 sg_count
= talitos_map_sg(dev
, req_ctx
->psrc
,
1667 edesc
->src_nents
? : 1,
1669 edesc
->src_is_chained
);
1671 if (sg_count
== 1) {
1672 to_talitos_ptr(&desc
->ptr
[3], sg_dma_address(req_ctx
->psrc
));
1674 sg_count
= sg_to_link_tbl(req_ctx
->psrc
, sg_count
, length
,
1675 &edesc
->link_tbl
[0]);
1677 desc
->ptr
[3].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1678 to_talitos_ptr(&desc
->ptr
[3], edesc
->dma_link_tbl
);
1679 dma_sync_single_for_device(ctx
->dev
,
1680 edesc
->dma_link_tbl
,
1684 /* Only one segment now, so no link tbl needed */
1685 to_talitos_ptr(&desc
->ptr
[3],
1686 sg_dma_address(req_ctx
->psrc
));
1690 /* fifth DWORD empty */
1691 desc
->ptr
[4] = zero_entry
;
1693 /* hash/HMAC out -or- hash context out */
1695 map_single_talitos_ptr(dev
, &desc
->ptr
[5],
1696 crypto_ahash_digestsize(tfm
),
1697 areq
->result
, 0, DMA_FROM_DEVICE
);
1699 map_single_talitos_ptr(dev
, &desc
->ptr
[5],
1700 req_ctx
->hw_context_size
,
1701 req_ctx
->hw_context
, 0, DMA_FROM_DEVICE
);
1703 /* last DWORD empty */
1704 desc
->ptr
[6] = zero_entry
;
1706 ret
= talitos_submit(dev
, desc
, callback
, areq
);
1707 if (ret
!= -EINPROGRESS
) {
1708 common_nonsnoop_hash_unmap(dev
, edesc
, areq
);
1714 static struct talitos_edesc
*ahash_edesc_alloc(struct ahash_request
*areq
,
1715 unsigned int nbytes
)
1717 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
1718 struct talitos_ctx
*ctx
= crypto_ahash_ctx(tfm
);
1719 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1721 return talitos_edesc_alloc(ctx
->dev
, req_ctx
->psrc
, NULL
, 1,
1722 nbytes
, 0, 0, areq
->base
.flags
);
1725 static int ahash_init(struct ahash_request
*areq
)
1727 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
1728 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1730 /* Initialize the context */
1732 req_ctx
->first
= 1; /* first indicates h/w must init its context */
1733 req_ctx
->swinit
= 0; /* assume h/w init of context */
1734 req_ctx
->hw_context_size
=
1735 (crypto_ahash_digestsize(tfm
) <= SHA256_DIGEST_SIZE
)
1736 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1737 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
;
1743 * on h/w without explicit sha224 support, we initialize h/w context
1744 * manually with sha224 constants, and tell it to run sha256.
1746 static int ahash_init_sha224_swinit(struct ahash_request
*areq
)
1748 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1751 req_ctx
->swinit
= 1;/* prevent h/w initting context with sha256 values*/
1753 req_ctx
->hw_context
[0] = SHA224_H0
;
1754 req_ctx
->hw_context
[1] = SHA224_H1
;
1755 req_ctx
->hw_context
[2] = SHA224_H2
;
1756 req_ctx
->hw_context
[3] = SHA224_H3
;
1757 req_ctx
->hw_context
[4] = SHA224_H4
;
1758 req_ctx
->hw_context
[5] = SHA224_H5
;
1759 req_ctx
->hw_context
[6] = SHA224_H6
;
1760 req_ctx
->hw_context
[7] = SHA224_H7
;
1762 /* init 64-bit count */
1763 req_ctx
->hw_context
[8] = 0;
1764 req_ctx
->hw_context
[9] = 0;
1769 static int ahash_process_req(struct ahash_request
*areq
, unsigned int nbytes
)
1771 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
1772 struct talitos_ctx
*ctx
= crypto_ahash_ctx(tfm
);
1773 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1774 struct talitos_edesc
*edesc
;
1775 unsigned int blocksize
=
1776 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm
));
1777 unsigned int nbytes_to_hash
;
1778 unsigned int to_hash_later
;
1782 if (!req_ctx
->last
&& (nbytes
+ req_ctx
->nbuf
<= blocksize
)) {
1783 /* Buffer up to one whole block */
1784 sg_copy_to_buffer(areq
->src
,
1785 sg_count(areq
->src
, nbytes
, &chained
),
1786 req_ctx
->buf
+ req_ctx
->nbuf
, nbytes
);
1787 req_ctx
->nbuf
+= nbytes
;
1791 /* At least (blocksize + 1) bytes are available to hash */
1792 nbytes_to_hash
= nbytes
+ req_ctx
->nbuf
;
1793 to_hash_later
= nbytes_to_hash
& (blocksize
- 1);
1797 else if (to_hash_later
)
1798 /* There is a partial block. Hash the full block(s) now */
1799 nbytes_to_hash
-= to_hash_later
;
1801 /* Keep one block buffered */
1802 nbytes_to_hash
-= blocksize
;
1803 to_hash_later
= blocksize
;
1806 /* Chain in any previously buffered data */
1807 if (req_ctx
->nbuf
) {
1808 nsg
= (req_ctx
->nbuf
< nbytes_to_hash
) ? 2 : 1;
1809 sg_init_table(req_ctx
->bufsl
, nsg
);
1810 sg_set_buf(req_ctx
->bufsl
, req_ctx
->buf
, req_ctx
->nbuf
);
1812 scatterwalk_sg_chain(req_ctx
->bufsl
, 2, areq
->src
);
1813 req_ctx
->psrc
= req_ctx
->bufsl
;
1815 req_ctx
->psrc
= areq
->src
;
1817 if (to_hash_later
) {
1818 int nents
= sg_count(areq
->src
, nbytes
, &chained
);
1819 sg_copy_end_to_buffer(areq
->src
, nents
,
1822 nbytes
- to_hash_later
);
1824 req_ctx
->to_hash_later
= to_hash_later
;
1826 /* Allocate extended descriptor */
1827 edesc
= ahash_edesc_alloc(areq
, nbytes_to_hash
);
1829 return PTR_ERR(edesc
);
1831 edesc
->desc
.hdr
= ctx
->desc_hdr_template
;
1833 /* On last one, request SEC to pad; otherwise continue */
1835 edesc
->desc
.hdr
|= DESC_HDR_MODE0_MDEU_PAD
;
1837 edesc
->desc
.hdr
|= DESC_HDR_MODE0_MDEU_CONT
;
1839 /* request SEC to INIT hash. */
1840 if (req_ctx
->first
&& !req_ctx
->swinit
)
1841 edesc
->desc
.hdr
|= DESC_HDR_MODE0_MDEU_INIT
;
1843 /* When the tfm context has a keylen, it's an HMAC.
1844 * A first or last (ie. not middle) descriptor must request HMAC.
1846 if (ctx
->keylen
&& (req_ctx
->first
|| req_ctx
->last
))
1847 edesc
->desc
.hdr
|= DESC_HDR_MODE0_MDEU_HMAC
;
1849 return common_nonsnoop_hash(edesc
, areq
, nbytes_to_hash
,
1853 static int ahash_update(struct ahash_request
*areq
)
1855 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1859 return ahash_process_req(areq
, areq
->nbytes
);
1862 static int ahash_final(struct ahash_request
*areq
)
1864 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1868 return ahash_process_req(areq
, 0);
1871 static int ahash_finup(struct ahash_request
*areq
)
1873 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1877 return ahash_process_req(areq
, areq
->nbytes
);
1880 static int ahash_digest(struct ahash_request
*areq
)
1882 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1883 struct crypto_ahash
*ahash
= crypto_ahash_reqtfm(areq
);
1888 return ahash_process_req(areq
, areq
->nbytes
);
1891 struct talitos_alg_template
{
1894 struct crypto_alg crypto
;
1895 struct ahash_alg hash
;
1897 __be32 desc_hdr_template
;
1900 static struct talitos_alg_template driver_algs
[] = {
1901 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
1902 { .type
= CRYPTO_ALG_TYPE_AEAD
,
1904 .cra_name
= "authenc(hmac(sha1),cbc(aes))",
1905 .cra_driver_name
= "authenc-hmac-sha1-cbc-aes-talitos",
1906 .cra_blocksize
= AES_BLOCK_SIZE
,
1907 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
1908 .cra_type
= &crypto_aead_type
,
1910 .setkey
= aead_setkey
,
1911 .setauthsize
= aead_setauthsize
,
1912 .encrypt
= aead_encrypt
,
1913 .decrypt
= aead_decrypt
,
1914 .givencrypt
= aead_givencrypt
,
1915 .geniv
= "<built-in>",
1916 .ivsize
= AES_BLOCK_SIZE
,
1917 .maxauthsize
= SHA1_DIGEST_SIZE
,
1920 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
1921 DESC_HDR_SEL0_AESU
|
1922 DESC_HDR_MODE0_AESU_CBC
|
1923 DESC_HDR_SEL1_MDEUA
|
1924 DESC_HDR_MODE1_MDEU_INIT
|
1925 DESC_HDR_MODE1_MDEU_PAD
|
1926 DESC_HDR_MODE1_MDEU_SHA1_HMAC
,
1928 { .type
= CRYPTO_ALG_TYPE_AEAD
,
1930 .cra_name
= "authenc(hmac(sha1),cbc(des3_ede))",
1931 .cra_driver_name
= "authenc-hmac-sha1-cbc-3des-talitos",
1932 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1933 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
1934 .cra_type
= &crypto_aead_type
,
1936 .setkey
= aead_setkey
,
1937 .setauthsize
= aead_setauthsize
,
1938 .encrypt
= aead_encrypt
,
1939 .decrypt
= aead_decrypt
,
1940 .givencrypt
= aead_givencrypt
,
1941 .geniv
= "<built-in>",
1942 .ivsize
= DES3_EDE_BLOCK_SIZE
,
1943 .maxauthsize
= SHA1_DIGEST_SIZE
,
1946 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
1948 DESC_HDR_MODE0_DEU_CBC
|
1949 DESC_HDR_MODE0_DEU_3DES
|
1950 DESC_HDR_SEL1_MDEUA
|
1951 DESC_HDR_MODE1_MDEU_INIT
|
1952 DESC_HDR_MODE1_MDEU_PAD
|
1953 DESC_HDR_MODE1_MDEU_SHA1_HMAC
,
1955 { .type
= CRYPTO_ALG_TYPE_AEAD
,
1957 .cra_name
= "authenc(hmac(sha256),cbc(aes))",
1958 .cra_driver_name
= "authenc-hmac-sha256-cbc-aes-talitos",
1959 .cra_blocksize
= AES_BLOCK_SIZE
,
1960 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
1961 .cra_type
= &crypto_aead_type
,
1963 .setkey
= aead_setkey
,
1964 .setauthsize
= aead_setauthsize
,
1965 .encrypt
= aead_encrypt
,
1966 .decrypt
= aead_decrypt
,
1967 .givencrypt
= aead_givencrypt
,
1968 .geniv
= "<built-in>",
1969 .ivsize
= AES_BLOCK_SIZE
,
1970 .maxauthsize
= SHA256_DIGEST_SIZE
,
1973 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
1974 DESC_HDR_SEL0_AESU
|
1975 DESC_HDR_MODE0_AESU_CBC
|
1976 DESC_HDR_SEL1_MDEUA
|
1977 DESC_HDR_MODE1_MDEU_INIT
|
1978 DESC_HDR_MODE1_MDEU_PAD
|
1979 DESC_HDR_MODE1_MDEU_SHA256_HMAC
,
1981 { .type
= CRYPTO_ALG_TYPE_AEAD
,
1983 .cra_name
= "authenc(hmac(sha256),cbc(des3_ede))",
1984 .cra_driver_name
= "authenc-hmac-sha256-cbc-3des-talitos",
1985 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1986 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
1987 .cra_type
= &crypto_aead_type
,
1989 .setkey
= aead_setkey
,
1990 .setauthsize
= aead_setauthsize
,
1991 .encrypt
= aead_encrypt
,
1992 .decrypt
= aead_decrypt
,
1993 .givencrypt
= aead_givencrypt
,
1994 .geniv
= "<built-in>",
1995 .ivsize
= DES3_EDE_BLOCK_SIZE
,
1996 .maxauthsize
= SHA256_DIGEST_SIZE
,
1999 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2001 DESC_HDR_MODE0_DEU_CBC
|
2002 DESC_HDR_MODE0_DEU_3DES
|
2003 DESC_HDR_SEL1_MDEUA
|
2004 DESC_HDR_MODE1_MDEU_INIT
|
2005 DESC_HDR_MODE1_MDEU_PAD
|
2006 DESC_HDR_MODE1_MDEU_SHA256_HMAC
,
2008 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2010 .cra_name
= "authenc(hmac(md5),cbc(aes))",
2011 .cra_driver_name
= "authenc-hmac-md5-cbc-aes-talitos",
2012 .cra_blocksize
= AES_BLOCK_SIZE
,
2013 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2014 .cra_type
= &crypto_aead_type
,
2016 .setkey
= aead_setkey
,
2017 .setauthsize
= aead_setauthsize
,
2018 .encrypt
= aead_encrypt
,
2019 .decrypt
= aead_decrypt
,
2020 .givencrypt
= aead_givencrypt
,
2021 .geniv
= "<built-in>",
2022 .ivsize
= AES_BLOCK_SIZE
,
2023 .maxauthsize
= MD5_DIGEST_SIZE
,
2026 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2027 DESC_HDR_SEL0_AESU
|
2028 DESC_HDR_MODE0_AESU_CBC
|
2029 DESC_HDR_SEL1_MDEUA
|
2030 DESC_HDR_MODE1_MDEU_INIT
|
2031 DESC_HDR_MODE1_MDEU_PAD
|
2032 DESC_HDR_MODE1_MDEU_MD5_HMAC
,
2034 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2036 .cra_name
= "authenc(hmac(md5),cbc(des3_ede))",
2037 .cra_driver_name
= "authenc-hmac-md5-cbc-3des-talitos",
2038 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2039 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2040 .cra_type
= &crypto_aead_type
,
2042 .setkey
= aead_setkey
,
2043 .setauthsize
= aead_setauthsize
,
2044 .encrypt
= aead_encrypt
,
2045 .decrypt
= aead_decrypt
,
2046 .givencrypt
= aead_givencrypt
,
2047 .geniv
= "<built-in>",
2048 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2049 .maxauthsize
= MD5_DIGEST_SIZE
,
2052 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2054 DESC_HDR_MODE0_DEU_CBC
|
2055 DESC_HDR_MODE0_DEU_3DES
|
2056 DESC_HDR_SEL1_MDEUA
|
2057 DESC_HDR_MODE1_MDEU_INIT
|
2058 DESC_HDR_MODE1_MDEU_PAD
|
2059 DESC_HDR_MODE1_MDEU_MD5_HMAC
,
2061 /* ABLKCIPHER algorithms. */
2062 { .type
= CRYPTO_ALG_TYPE_ABLKCIPHER
,
2064 .cra_name
= "cbc(aes)",
2065 .cra_driver_name
= "cbc-aes-talitos",
2066 .cra_blocksize
= AES_BLOCK_SIZE
,
2067 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
2069 .cra_type
= &crypto_ablkcipher_type
,
2071 .setkey
= ablkcipher_setkey
,
2072 .encrypt
= ablkcipher_encrypt
,
2073 .decrypt
= ablkcipher_decrypt
,
2075 .min_keysize
= AES_MIN_KEY_SIZE
,
2076 .max_keysize
= AES_MAX_KEY_SIZE
,
2077 .ivsize
= AES_BLOCK_SIZE
,
2080 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2081 DESC_HDR_SEL0_AESU
|
2082 DESC_HDR_MODE0_AESU_CBC
,
2084 { .type
= CRYPTO_ALG_TYPE_ABLKCIPHER
,
2086 .cra_name
= "cbc(des3_ede)",
2087 .cra_driver_name
= "cbc-3des-talitos",
2088 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2089 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
2091 .cra_type
= &crypto_ablkcipher_type
,
2093 .setkey
= ablkcipher_setkey
,
2094 .encrypt
= ablkcipher_encrypt
,
2095 .decrypt
= ablkcipher_decrypt
,
2097 .min_keysize
= DES3_EDE_KEY_SIZE
,
2098 .max_keysize
= DES3_EDE_KEY_SIZE
,
2099 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2102 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2104 DESC_HDR_MODE0_DEU_CBC
|
2105 DESC_HDR_MODE0_DEU_3DES
,
2107 /* AHASH algorithms. */
2108 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2111 .update
= ahash_update
,
2112 .final
= ahash_final
,
2113 .finup
= ahash_finup
,
2114 .digest
= ahash_digest
,
2115 .halg
.digestsize
= MD5_DIGEST_SIZE
,
2118 .cra_driver_name
= "md5-talitos",
2119 .cra_blocksize
= MD5_BLOCK_SIZE
,
2120 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2122 .cra_type
= &crypto_ahash_type
2125 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2126 DESC_HDR_SEL0_MDEUA
|
2127 DESC_HDR_MODE0_MDEU_MD5
,
2129 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2132 .update
= ahash_update
,
2133 .final
= ahash_final
,
2134 .finup
= ahash_finup
,
2135 .digest
= ahash_digest
,
2136 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
2139 .cra_driver_name
= "sha1-talitos",
2140 .cra_blocksize
= SHA1_BLOCK_SIZE
,
2141 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2143 .cra_type
= &crypto_ahash_type
2146 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2147 DESC_HDR_SEL0_MDEUA
|
2148 DESC_HDR_MODE0_MDEU_SHA1
,
2150 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2153 .update
= ahash_update
,
2154 .final
= ahash_final
,
2155 .finup
= ahash_finup
,
2156 .digest
= ahash_digest
,
2157 .halg
.digestsize
= SHA224_DIGEST_SIZE
,
2159 .cra_name
= "sha224",
2160 .cra_driver_name
= "sha224-talitos",
2161 .cra_blocksize
= SHA224_BLOCK_SIZE
,
2162 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2164 .cra_type
= &crypto_ahash_type
2167 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2168 DESC_HDR_SEL0_MDEUA
|
2169 DESC_HDR_MODE0_MDEU_SHA224
,
2171 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2174 .update
= ahash_update
,
2175 .final
= ahash_final
,
2176 .finup
= ahash_finup
,
2177 .digest
= ahash_digest
,
2178 .halg
.digestsize
= SHA256_DIGEST_SIZE
,
2180 .cra_name
= "sha256",
2181 .cra_driver_name
= "sha256-talitos",
2182 .cra_blocksize
= SHA256_BLOCK_SIZE
,
2183 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2185 .cra_type
= &crypto_ahash_type
2188 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2189 DESC_HDR_SEL0_MDEUA
|
2190 DESC_HDR_MODE0_MDEU_SHA256
,
2192 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2195 .update
= ahash_update
,
2196 .final
= ahash_final
,
2197 .finup
= ahash_finup
,
2198 .digest
= ahash_digest
,
2199 .halg
.digestsize
= SHA384_DIGEST_SIZE
,
2201 .cra_name
= "sha384",
2202 .cra_driver_name
= "sha384-talitos",
2203 .cra_blocksize
= SHA384_BLOCK_SIZE
,
2204 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2206 .cra_type
= &crypto_ahash_type
2209 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2210 DESC_HDR_SEL0_MDEUB
|
2211 DESC_HDR_MODE0_MDEUB_SHA384
,
2213 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2216 .update
= ahash_update
,
2217 .final
= ahash_final
,
2218 .finup
= ahash_finup
,
2219 .digest
= ahash_digest
,
2220 .halg
.digestsize
= SHA512_DIGEST_SIZE
,
2222 .cra_name
= "sha512",
2223 .cra_driver_name
= "sha512-talitos",
2224 .cra_blocksize
= SHA512_BLOCK_SIZE
,
2225 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2227 .cra_type
= &crypto_ahash_type
2230 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2231 DESC_HDR_SEL0_MDEUB
|
2232 DESC_HDR_MODE0_MDEUB_SHA512
,
2236 struct talitos_crypto_alg
{
2237 struct list_head entry
;
2239 struct talitos_alg_template algt
;
2242 static int talitos_cra_init(struct crypto_tfm
*tfm
)
2244 struct crypto_alg
*alg
= tfm
->__crt_alg
;
2245 struct talitos_crypto_alg
*talitos_alg
;
2246 struct talitos_ctx
*ctx
= crypto_tfm_ctx(tfm
);
2248 if ((alg
->cra_flags
& CRYPTO_ALG_TYPE_MASK
) == CRYPTO_ALG_TYPE_AHASH
)
2249 talitos_alg
= container_of(__crypto_ahash_alg(alg
),
2250 struct talitos_crypto_alg
,
2253 talitos_alg
= container_of(alg
, struct talitos_crypto_alg
,
2256 /* update context with ptr to dev */
2257 ctx
->dev
= talitos_alg
->dev
;
2259 /* copy descriptor header template value */
2260 ctx
->desc_hdr_template
= talitos_alg
->algt
.desc_hdr_template
;
2265 static int talitos_cra_init_aead(struct crypto_tfm
*tfm
)
2267 struct talitos_ctx
*ctx
= crypto_tfm_ctx(tfm
);
2269 talitos_cra_init(tfm
);
2271 /* random first IV */
2272 get_random_bytes(ctx
->iv
, TALITOS_MAX_IV_LENGTH
);
2277 static int talitos_cra_init_ahash(struct crypto_tfm
*tfm
)
2279 struct talitos_ctx
*ctx
= crypto_tfm_ctx(tfm
);
2281 talitos_cra_init(tfm
);
2284 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
2285 sizeof(struct talitos_ahash_req_ctx
));
2291 * given the alg's descriptor header template, determine whether descriptor
2292 * type and primary/secondary execution units required match the hw
2293 * capabilities description provided in the device tree node.
2295 static int hw_supports(struct device
*dev
, __be32 desc_hdr_template
)
2297 struct talitos_private
*priv
= dev_get_drvdata(dev
);
2300 ret
= (1 << DESC_TYPE(desc_hdr_template
) & priv
->desc_types
) &&
2301 (1 << PRIMARY_EU(desc_hdr_template
) & priv
->exec_units
);
2303 if (SECONDARY_EU(desc_hdr_template
))
2304 ret
= ret
&& (1 << SECONDARY_EU(desc_hdr_template
)
2305 & priv
->exec_units
);
2310 static int talitos_remove(struct platform_device
*ofdev
)
2312 struct device
*dev
= &ofdev
->dev
;
2313 struct talitos_private
*priv
= dev_get_drvdata(dev
);
2314 struct talitos_crypto_alg
*t_alg
, *n
;
2317 list_for_each_entry_safe(t_alg
, n
, &priv
->alg_list
, entry
) {
2318 switch (t_alg
->algt
.type
) {
2319 case CRYPTO_ALG_TYPE_ABLKCIPHER
:
2320 case CRYPTO_ALG_TYPE_AEAD
:
2321 crypto_unregister_alg(&t_alg
->algt
.alg
.crypto
);
2323 case CRYPTO_ALG_TYPE_AHASH
:
2324 crypto_unregister_ahash(&t_alg
->algt
.alg
.hash
);
2327 list_del(&t_alg
->entry
);
2331 if (hw_supports(dev
, DESC_HDR_SEL0_RNG
))
2332 talitos_unregister_rng(dev
);
2334 for (i
= 0; i
< priv
->num_channels
; i
++)
2335 kfree(priv
->chan
[i
].fifo
);
2339 if (priv
->irq
!= NO_IRQ
) {
2340 free_irq(priv
->irq
, dev
);
2341 irq_dispose_mapping(priv
->irq
);
2344 tasklet_kill(&priv
->done_task
);
2348 dev_set_drvdata(dev
, NULL
);
2355 static struct talitos_crypto_alg
*talitos_alg_alloc(struct device
*dev
,
2356 struct talitos_alg_template
2359 struct talitos_private
*priv
= dev_get_drvdata(dev
);
2360 struct talitos_crypto_alg
*t_alg
;
2361 struct crypto_alg
*alg
;
2363 t_alg
= kzalloc(sizeof(struct talitos_crypto_alg
), GFP_KERNEL
);
2365 return ERR_PTR(-ENOMEM
);
2367 t_alg
->algt
= *template;
2369 switch (t_alg
->algt
.type
) {
2370 case CRYPTO_ALG_TYPE_ABLKCIPHER
:
2371 alg
= &t_alg
->algt
.alg
.crypto
;
2372 alg
->cra_init
= talitos_cra_init
;
2374 case CRYPTO_ALG_TYPE_AEAD
:
2375 alg
= &t_alg
->algt
.alg
.crypto
;
2376 alg
->cra_init
= talitos_cra_init_aead
;
2378 case CRYPTO_ALG_TYPE_AHASH
:
2379 alg
= &t_alg
->algt
.alg
.hash
.halg
.base
;
2380 alg
->cra_init
= talitos_cra_init_ahash
;
2381 if (!(priv
->features
& TALITOS_FTR_SHA224_HWINIT
) &&
2382 !strcmp(alg
->cra_name
, "sha224")) {
2383 t_alg
->algt
.alg
.hash
.init
= ahash_init_sha224_swinit
;
2384 t_alg
->algt
.desc_hdr_template
=
2385 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2386 DESC_HDR_SEL0_MDEUA
|
2387 DESC_HDR_MODE0_MDEU_SHA256
;
2391 dev_err(dev
, "unknown algorithm type %d\n", t_alg
->algt
.type
);
2392 return ERR_PTR(-EINVAL
);
2395 alg
->cra_module
= THIS_MODULE
;
2396 alg
->cra_priority
= TALITOS_CRA_PRIORITY
;
2397 alg
->cra_alignmask
= 0;
2398 alg
->cra_ctxsize
= sizeof(struct talitos_ctx
);
2405 static int talitos_probe(struct platform_device
*ofdev
,
2406 const struct of_device_id
*match
)
2408 struct device
*dev
= &ofdev
->dev
;
2409 struct device_node
*np
= ofdev
->dev
.of_node
;
2410 struct talitos_private
*priv
;
2411 const unsigned int *prop
;
2414 priv
= kzalloc(sizeof(struct talitos_private
), GFP_KERNEL
);
2418 dev_set_drvdata(dev
, priv
);
2420 priv
->ofdev
= ofdev
;
2422 tasklet_init(&priv
->done_task
, talitos_done
, (unsigned long)dev
);
2424 INIT_LIST_HEAD(&priv
->alg_list
);
2426 priv
->irq
= irq_of_parse_and_map(np
, 0);
2428 if (priv
->irq
== NO_IRQ
) {
2429 dev_err(dev
, "failed to map irq\n");
2434 /* get the irq line */
2435 err
= request_irq(priv
->irq
, talitos_interrupt
, 0,
2436 dev_driver_string(dev
), dev
);
2438 dev_err(dev
, "failed to request irq %d\n", priv
->irq
);
2439 irq_dispose_mapping(priv
->irq
);
2444 priv
->reg
= of_iomap(np
, 0);
2446 dev_err(dev
, "failed to of_iomap\n");
2451 /* get SEC version capabilities from device tree */
2452 prop
= of_get_property(np
, "fsl,num-channels", NULL
);
2454 priv
->num_channels
= *prop
;
2456 prop
= of_get_property(np
, "fsl,channel-fifo-len", NULL
);
2458 priv
->chfifo_len
= *prop
;
2460 prop
= of_get_property(np
, "fsl,exec-units-mask", NULL
);
2462 priv
->exec_units
= *prop
;
2464 prop
= of_get_property(np
, "fsl,descriptor-types-mask", NULL
);
2466 priv
->desc_types
= *prop
;
2468 if (!is_power_of_2(priv
->num_channels
) || !priv
->chfifo_len
||
2469 !priv
->exec_units
|| !priv
->desc_types
) {
2470 dev_err(dev
, "invalid property data in device tree node\n");
2475 if (of_device_is_compatible(np
, "fsl,sec3.0"))
2476 priv
->features
|= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT
;
2478 if (of_device_is_compatible(np
, "fsl,sec2.1"))
2479 priv
->features
|= TALITOS_FTR_HW_AUTH_CHECK
|
2480 TALITOS_FTR_SHA224_HWINIT
;
2482 priv
->chan
= kzalloc(sizeof(struct talitos_channel
) *
2483 priv
->num_channels
, GFP_KERNEL
);
2485 dev_err(dev
, "failed to allocate channel management space\n");
2490 for (i
= 0; i
< priv
->num_channels
; i
++) {
2491 spin_lock_init(&priv
->chan
[i
].head_lock
);
2492 spin_lock_init(&priv
->chan
[i
].tail_lock
);
2495 priv
->fifo_len
= roundup_pow_of_two(priv
->chfifo_len
);
2497 for (i
= 0; i
< priv
->num_channels
; i
++) {
2498 priv
->chan
[i
].fifo
= kzalloc(sizeof(struct talitos_request
) *
2499 priv
->fifo_len
, GFP_KERNEL
);
2500 if (!priv
->chan
[i
].fifo
) {
2501 dev_err(dev
, "failed to allocate request fifo %d\n", i
);
2507 for (i
= 0; i
< priv
->num_channels
; i
++)
2508 atomic_set(&priv
->chan
[i
].submit_count
,
2509 -(priv
->chfifo_len
- 1));
2511 dma_set_mask(dev
, DMA_BIT_MASK(36));
2513 /* reset and initialize the h/w */
2514 err
= init_device(dev
);
2516 dev_err(dev
, "failed to initialize device\n");
2520 /* register the RNG, if available */
2521 if (hw_supports(dev
, DESC_HDR_SEL0_RNG
)) {
2522 err
= talitos_register_rng(dev
);
2524 dev_err(dev
, "failed to register hwrng: %d\n", err
);
2527 dev_info(dev
, "hwrng\n");
2530 /* register crypto algorithms the device supports */
2531 for (i
= 0; i
< ARRAY_SIZE(driver_algs
); i
++) {
2532 if (hw_supports(dev
, driver_algs
[i
].desc_hdr_template
)) {
2533 struct talitos_crypto_alg
*t_alg
;
2536 t_alg
= talitos_alg_alloc(dev
, &driver_algs
[i
]);
2537 if (IS_ERR(t_alg
)) {
2538 err
= PTR_ERR(t_alg
);
2542 switch (t_alg
->algt
.type
) {
2543 case CRYPTO_ALG_TYPE_ABLKCIPHER
:
2544 case CRYPTO_ALG_TYPE_AEAD
:
2545 err
= crypto_register_alg(
2546 &t_alg
->algt
.alg
.crypto
);
2547 name
= t_alg
->algt
.alg
.crypto
.cra_driver_name
;
2549 case CRYPTO_ALG_TYPE_AHASH
:
2550 err
= crypto_register_ahash(
2551 &t_alg
->algt
.alg
.hash
);
2553 t_alg
->algt
.alg
.hash
.halg
.base
.cra_driver_name
;
2557 dev_err(dev
, "%s alg registration failed\n",
2561 list_add_tail(&t_alg
->entry
, &priv
->alg_list
);
2562 dev_info(dev
, "%s\n", name
);
2570 talitos_remove(ofdev
);
2575 static const struct of_device_id talitos_match
[] = {
2577 .compatible
= "fsl,sec2.0",
2581 MODULE_DEVICE_TABLE(of
, talitos_match
);
2583 static struct of_platform_driver talitos_driver
= {
2586 .owner
= THIS_MODULE
,
2587 .of_match_table
= talitos_match
,
2589 .probe
= talitos_probe
,
2590 .remove
= talitos_remove
,
2593 static int __init
talitos_init(void)
2595 return of_register_platform_driver(&talitos_driver
);
2597 module_init(talitos_init
);
2599 static void __exit
talitos_exit(void)
2601 of_unregister_platform_driver(&talitos_driver
);
2603 module_exit(talitos_exit
);
2605 MODULE_LICENSE("GPL");
2606 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
2607 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");