?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
MIPS: perf: fix deadlock
[linux-stable.git] / drivers / crypto / talitos.c
blob0bba6a19d36a28e3b1c83acb12d7ecf0ad52501a
1 /*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
4 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
5 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/mod_devicetable.h>
31 #include <linux/device.h>
32 #include <linux/interrupt.h>
33 #include <linux/crypto.h>
34 #include <linux/hw_random.h>
35 #include <linux/of_address.h>
36 #include <linux/of_irq.h>
37 #include <linux/of_platform.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/io.h>
40 #include <linux/spinlock.h>
41 #include <linux/rtnetlink.h>
42 #include <linux/slab.h>
43
44 #include <crypto/algapi.h>
45 #include <crypto/aes.h>
46 #include <crypto/des.h>
47 #include <crypto/sha.h>
48 #include <crypto/md5.h>
49 #include <crypto/internal/aead.h>
50 #include <crypto/authenc.h>
51 #include <crypto/skcipher.h>
52 #include <crypto/hash.h>
53 #include <crypto/internal/hash.h>
54 #include <crypto/scatterwalk.h>
55
56 #include "talitos.h"
57
58 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
59 bool is_sec1)
60 {
61 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
62 if (!is_sec1)
63 ptr->eptr = upper_32_bits(dma_addr);
64 }
65
66 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
67 struct talitos_ptr *src_ptr, bool is_sec1)
68 {
69 dst_ptr->ptr = src_ptr->ptr;
70 if (!is_sec1)
71 dst_ptr->eptr = src_ptr->eptr;
72 }
73
74 static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len,
75 bool is_sec1)
76 {
77 if (is_sec1) {
78 ptr->res = 0;
79 ptr->len1 = cpu_to_be16(len);
80 } else {
81 ptr->len = cpu_to_be16(len);
82 }
83 }
84
85 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
86 bool is_sec1)
87 {
88 if (is_sec1)
89 return be16_to_cpu(ptr->len1);
90 else
91 return be16_to_cpu(ptr->len);
92 }
93
94 static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
95 bool is_sec1)
96 {
97 if (!is_sec1)
98 ptr->j_extent = val;
99 }
100
101 static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
102 {
103 if (!is_sec1)
104 ptr->j_extent |= val;
105 }
106
107 /*
108 * map virtual single (contiguous) pointer to h/w descriptor pointer
109 */
110 static void map_single_talitos_ptr(struct device *dev,
111 struct talitos_ptr *ptr,
112 unsigned int len, void *data,
113 enum dma_data_direction dir)
114 {
115 dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
116 struct talitos_private *priv = dev_get_drvdata(dev);
117 bool is_sec1 = has_ftr_sec1(priv);
118
119 to_talitos_ptr_len(ptr, len, is_sec1);
120 to_talitos_ptr(ptr, dma_addr, is_sec1);
121 to_talitos_ptr_ext_set(ptr, 0, is_sec1);
122 }
123
124 /*
125 * unmap bus single (contiguous) h/w descriptor pointer
126 */
127 static void unmap_single_talitos_ptr(struct device *dev,
128 struct talitos_ptr *ptr,
129 enum dma_data_direction dir)
130 {
131 struct talitos_private *priv = dev_get_drvdata(dev);
132 bool is_sec1 = has_ftr_sec1(priv);
133
134 dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
135 from_talitos_ptr_len(ptr, is_sec1), dir);
136 }
137
138 static int reset_channel(struct device *dev, int ch)
139 {
140 struct talitos_private *priv = dev_get_drvdata(dev);
141 unsigned int timeout = TALITOS_TIMEOUT;
142 bool is_sec1 = has_ftr_sec1(priv);
143
144 if (is_sec1) {
145 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
146 TALITOS1_CCCR_LO_RESET);
147
148 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
149 TALITOS1_CCCR_LO_RESET) && --timeout)
150 cpu_relax();
151 } else {
152 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
153 TALITOS2_CCCR_RESET);
154
155 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
156 TALITOS2_CCCR_RESET) && --timeout)
157 cpu_relax();
158 }
159
160 if (timeout == 0) {
161 dev_err(dev, "failed to reset channel %d\n", ch);
162 return -EIO;
163 }
164
165 /* set 36-bit addressing, done writeback enable and done IRQ enable */
166 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
167 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
168
169 /* and ICCR writeback, if available */
170 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
171 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
172 TALITOS_CCCR_LO_IWSE);
173
174 return 0;
175 }
176
177 static int reset_device(struct device *dev)
178 {
179 struct talitos_private *priv = dev_get_drvdata(dev);
180 unsigned int timeout = TALITOS_TIMEOUT;
181 bool is_sec1 = has_ftr_sec1(priv);
182 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
183
184 setbits32(priv->reg + TALITOS_MCR, mcr);
185
186 while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
187 && --timeout)
188 cpu_relax();
189
190 if (priv->irq[1]) {
191 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
192 setbits32(priv->reg + TALITOS_MCR, mcr);
193 }
194
195 if (timeout == 0) {
196 dev_err(dev, "failed to reset device\n");
197 return -EIO;
198 }
199
200 return 0;
201 }
202
203 /*
204 * Reset and initialize the device
205 */
206 static int init_device(struct device *dev)
207 {
208 struct talitos_private *priv = dev_get_drvdata(dev);
209 int ch, err;
210 bool is_sec1 = has_ftr_sec1(priv);
211
212 /*
213 * Master reset
214 * errata documentation: warning: certain SEC interrupts
215 * are not fully cleared by writing the MCR:SWR bit,
216 * set bit twice to completely reset
217 */
218 err = reset_device(dev);
219 if (err)
220 return err;
221
222 err = reset_device(dev);
223 if (err)
224 return err;
225
226 /* reset channels */
227 for (ch = 0; ch < priv->num_channels; ch++) {
228 err = reset_channel(dev, ch);
229 if (err)
230 return err;
231 }
232
233 /* enable channel done and error interrupts */
234 if (is_sec1) {
235 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
236 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
237 /* disable parity error check in DEU (erroneous? test vect.) */
238 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
239 } else {
240 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
241 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
242 }
243
244 /* disable integrity check error interrupts (use writeback instead) */
245 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
246 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
247 TALITOS_MDEUICR_LO_ICE);
248
249 return 0;
250 }
251
252 /**
253 * talitos_submit - submits a descriptor to the device for processing
254 * @dev: the SEC device to be used
255 * @ch: the SEC device channel to be used
256 * @desc: the descriptor to be processed by the device
257 * @callback: whom to call when processing is complete
258 * @context: a handle for use by caller (optional)
259 *
260 * desc must contain valid dma-mapped (bus physical) address pointers.
261 * callback must check err and feedback in descriptor header
262 * for device processing status.
263 */
264 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
265 void (*callback)(struct device *dev,
266 struct talitos_desc *desc,
267 void *context, int error),
268 void *context)
269 {
270 struct talitos_private *priv = dev_get_drvdata(dev);
271 struct talitos_request *request;
272 unsigned long flags;
273 int head;
274 bool is_sec1 = has_ftr_sec1(priv);
275
276 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
277
278 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
279 /* h/w fifo is full */
280 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
281 return -EAGAIN;
282 }
283
284 head = priv->chan[ch].head;
285 request = &priv->chan[ch].fifo[head];
286
287 /* map descriptor and save caller data */
288 if (is_sec1) {
289 desc->hdr1 = desc->hdr;
290 desc->next_desc = 0;
291 request->dma_desc = dma_map_single(dev, &desc->hdr1,
292 TALITOS_DESC_SIZE,
293 DMA_BIDIRECTIONAL);
294 } else {
295 request->dma_desc = dma_map_single(dev, desc,
296 TALITOS_DESC_SIZE,
297 DMA_BIDIRECTIONAL);
298 }
299 request->callback = callback;
300 request->context = context;
301
302 /* increment fifo head */
303 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
304
305 smp_wmb();
306 request->desc = desc;
307
308 /* GO! */
309 wmb();
310 out_be32(priv->chan[ch].reg + TALITOS_FF,
311 upper_32_bits(request->dma_desc));
312 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
313 lower_32_bits(request->dma_desc));
314
315 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
316
317 return -EINPROGRESS;
318 }
319 EXPORT_SYMBOL(talitos_submit);
320
321 /*
322 * process what was done, notify callback of error if not
323 */
324 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
325 {
326 struct talitos_private *priv = dev_get_drvdata(dev);
327 struct talitos_request *request, saved_req;
328 unsigned long flags;
329 int tail, status;
330 bool is_sec1 = has_ftr_sec1(priv);
331
332 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
333
334 tail = priv->chan[ch].tail;
335 while (priv->chan[ch].fifo[tail].desc) {
336 __be32 hdr;
337
338 request = &priv->chan[ch].fifo[tail];
339
340 /* descriptors with their done bits set don't get the error */
341 rmb();
342 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
343
344 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
345 status = 0;
346 else
347 if (!error)
348 break;
349 else
350 status = error;
351
352 dma_unmap_single(dev, request->dma_desc,
353 TALITOS_DESC_SIZE,
354 DMA_BIDIRECTIONAL);
355
356 /* copy entries so we can call callback outside lock */
357 saved_req.desc = request->desc;
358 saved_req.callback = request->callback;
359 saved_req.context = request->context;
360
361 /* release request entry in fifo */
362 smp_wmb();
363 request->desc = NULL;
364
365 /* increment fifo tail */
366 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
367
368 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
369
370 atomic_dec(&priv->chan[ch].submit_count);
371
372 saved_req.callback(dev, saved_req.desc, saved_req.context,
373 status);
374 /* channel may resume processing in single desc error case */
375 if (error && !reset_ch && status == error)
376 return;
377 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
378 tail = priv->chan[ch].tail;
379 }
380
381 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
382 }
383
384 /*
385 * process completed requests for channels that have done status
386 */
387 #define DEF_TALITOS1_DONE(name, ch_done_mask) \
388 static void talitos1_done_##name(unsigned long data) \
389 { \
390 struct device *dev = (struct device *)data; \
391 struct talitos_private *priv = dev_get_drvdata(dev); \
392 unsigned long flags; \
393 \
394 if (ch_done_mask & 0x10000000) \
395 flush_channel(dev, 0, 0, 0); \
396 if (priv->num_channels == 1) \
397 goto out; \
398 if (ch_done_mask & 0x40000000) \
399 flush_channel(dev, 1, 0, 0); \
400 if (ch_done_mask & 0x00010000) \
401 flush_channel(dev, 2, 0, 0); \
402 if (ch_done_mask & 0x00040000) \
403 flush_channel(dev, 3, 0, 0); \
404 \
405 out: \
406 /* At this point, all completed channels have been processed */ \
407 /* Unmask done interrupts for channels completed later on. */ \
408 spin_lock_irqsave(&priv->reg_lock, flags); \
409 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
410 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
411 spin_unlock_irqrestore(&priv->reg_lock, flags); \
412 }
413
414 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
415
416 #define DEF_TALITOS2_DONE(name, ch_done_mask) \
417 static void talitos2_done_##name(unsigned long data) \
418 { \
419 struct device *dev = (struct device *)data; \
420 struct talitos_private *priv = dev_get_drvdata(dev); \
421 unsigned long flags; \
422 \
423 if (ch_done_mask & 1) \
424 flush_channel(dev, 0, 0, 0); \
425 if (priv->num_channels == 1) \
426 goto out; \
427 if (ch_done_mask & (1 << 2)) \
428 flush_channel(dev, 1, 0, 0); \
429 if (ch_done_mask & (1 << 4)) \
430 flush_channel(dev, 2, 0, 0); \
431 if (ch_done_mask & (1 << 6)) \
432 flush_channel(dev, 3, 0, 0); \
433 \
434 out: \
435 /* At this point, all completed channels have been processed */ \
436 /* Unmask done interrupts for channels completed later on. */ \
437 spin_lock_irqsave(&priv->reg_lock, flags); \
438 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
439 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
440 spin_unlock_irqrestore(&priv->reg_lock, flags); \
441 }
442
443 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
444 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
445 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
446
447 /*
448 * locate current (offending) descriptor
449 */
450 static u32 current_desc_hdr(struct device *dev, int ch)
451 {
452 struct talitos_private *priv = dev_get_drvdata(dev);
453 int tail, iter;
454 dma_addr_t cur_desc;
455
456 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
457 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
458
459 if (!cur_desc) {
460 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
461 return 0;
462 }
463
464 tail = priv->chan[ch].tail;
465
466 iter = tail;
467 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
468 iter = (iter + 1) & (priv->fifo_len - 1);
469 if (iter == tail) {
470 dev_err(dev, "couldn't locate current descriptor\n");
471 return 0;
472 }
473 }
474
475 return priv->chan[ch].fifo[iter].desc->hdr;
476 }
477
478 /*
479 * user diagnostics; report root cause of error based on execution unit status
480 */
481 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
482 {
483 struct talitos_private *priv = dev_get_drvdata(dev);
484 int i;
485
486 if (!desc_hdr)
487 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
488
489 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
490 case DESC_HDR_SEL0_AFEU:
491 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
492 in_be32(priv->reg_afeu + TALITOS_EUISR),
493 in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
494 break;
495 case DESC_HDR_SEL0_DEU:
496 dev_err(dev, "DEUISR 0x%08x_%08x\n",
497 in_be32(priv->reg_deu + TALITOS_EUISR),
498 in_be32(priv->reg_deu + TALITOS_EUISR_LO));
499 break;
500 case DESC_HDR_SEL0_MDEUA:
501 case DESC_HDR_SEL0_MDEUB:
502 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
503 in_be32(priv->reg_mdeu + TALITOS_EUISR),
504 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
505 break;
506 case DESC_HDR_SEL0_RNG:
507 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
508 in_be32(priv->reg_rngu + TALITOS_ISR),
509 in_be32(priv->reg_rngu + TALITOS_ISR_LO));
510 break;
511 case DESC_HDR_SEL0_PKEU:
512 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
513 in_be32(priv->reg_pkeu + TALITOS_EUISR),
514 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
515 break;
516 case DESC_HDR_SEL0_AESU:
517 dev_err(dev, "AESUISR 0x%08x_%08x\n",
518 in_be32(priv->reg_aesu + TALITOS_EUISR),
519 in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
520 break;
521 case DESC_HDR_SEL0_CRCU:
522 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
523 in_be32(priv->reg_crcu + TALITOS_EUISR),
524 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
525 break;
526 case DESC_HDR_SEL0_KEU:
527 dev_err(dev, "KEUISR 0x%08x_%08x\n",
528 in_be32(priv->reg_pkeu + TALITOS_EUISR),
529 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
530 break;
531 }
532
533 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
534 case DESC_HDR_SEL1_MDEUA:
535 case DESC_HDR_SEL1_MDEUB:
536 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
537 in_be32(priv->reg_mdeu + TALITOS_EUISR),
538 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
539 break;
540 case DESC_HDR_SEL1_CRCU:
541 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
542 in_be32(priv->reg_crcu + TALITOS_EUISR),
543 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
544 break;
545 }
546
547 for (i = 0; i < 8; i++)
548 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
549 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
550 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
551 }
552
553 /*
554 * recover from error interrupts
555 */
556 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
557 {
558 struct talitos_private *priv = dev_get_drvdata(dev);
559 unsigned int timeout = TALITOS_TIMEOUT;
560 int ch, error, reset_dev = 0;
561 u32 v_lo;
562 bool is_sec1 = has_ftr_sec1(priv);
563 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
564
565 for (ch = 0; ch < priv->num_channels; ch++) {
566 /* skip channels without errors */
567 if (is_sec1) {
568 /* bits 29, 31, 17, 19 */
569 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
570 continue;
571 } else {
572 if (!(isr & (1 << (ch * 2 + 1))))
573 continue;
574 }
575
576 error = -EINVAL;
577
578 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
579
580 if (v_lo & TALITOS_CCPSR_LO_DOF) {
581 dev_err(dev, "double fetch fifo overflow error\n");
582 error = -EAGAIN;
583 reset_ch = 1;
584 }
585 if (v_lo & TALITOS_CCPSR_LO_SOF) {
586 /* h/w dropped descriptor */
587 dev_err(dev, "single fetch fifo overflow error\n");
588 error = -EAGAIN;
589 }
590 if (v_lo & TALITOS_CCPSR_LO_MDTE)
591 dev_err(dev, "master data transfer error\n");
592 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
593 dev_err(dev, is_sec1 ? "pointer not complete error\n"
594 : "s/g data length zero error\n");
595 if (v_lo & TALITOS_CCPSR_LO_FPZ)
596 dev_err(dev, is_sec1 ? "parity error\n"
597 : "fetch pointer zero error\n");
598 if (v_lo & TALITOS_CCPSR_LO_IDH)
599 dev_err(dev, "illegal descriptor header error\n");
600 if (v_lo & TALITOS_CCPSR_LO_IEU)
601 dev_err(dev, is_sec1 ? "static assignment error\n"
602 : "invalid exec unit error\n");
603 if (v_lo & TALITOS_CCPSR_LO_EU)
604 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
605 if (!is_sec1) {
606 if (v_lo & TALITOS_CCPSR_LO_GB)
607 dev_err(dev, "gather boundary error\n");
608 if (v_lo & TALITOS_CCPSR_LO_GRL)
609 dev_err(dev, "gather return/length error\n");
610 if (v_lo & TALITOS_CCPSR_LO_SB)
611 dev_err(dev, "scatter boundary error\n");
612 if (v_lo & TALITOS_CCPSR_LO_SRL)
613 dev_err(dev, "scatter return/length error\n");
614 }
615
616 flush_channel(dev, ch, error, reset_ch);
617
618 if (reset_ch) {
619 reset_channel(dev, ch);
620 } else {
621 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
622 TALITOS2_CCCR_CONT);
623 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
624 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
625 TALITOS2_CCCR_CONT) && --timeout)
626 cpu_relax();
627 if (timeout == 0) {
628 dev_err(dev, "failed to restart channel %d\n",
629 ch);
630 reset_dev = 1;
631 }
632 }
633 }
634 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
635 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
636 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
637 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
638 isr, isr_lo);
639 else
640 dev_err(dev, "done overflow, internal time out, or "
641 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
642
643 /* purge request queues */
644 for (ch = 0; ch < priv->num_channels; ch++)
645 flush_channel(dev, ch, -EIO, 1);
646
647 /* reset and reinitialize the device */
648 init_device(dev);
649 }
650 }
651
652 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
653 static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
654 { \
655 struct device *dev = data; \
656 struct talitos_private *priv = dev_get_drvdata(dev); \
657 u32 isr, isr_lo; \
658 unsigned long flags; \
659 \
660 spin_lock_irqsave(&priv->reg_lock, flags); \
661 isr = in_be32(priv->reg + TALITOS_ISR); \
662 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
663 /* Acknowledge interrupt */ \
664 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
665 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
666 \
667 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
668 spin_unlock_irqrestore(&priv->reg_lock, flags); \
669 talitos_error(dev, isr & ch_err_mask, isr_lo); \
670 } \
671 else { \
672 if (likely(isr & ch_done_mask)) { \
673 /* mask further done interrupts. */ \
674 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
675 /* done_task will unmask done interrupts at exit */ \
676 tasklet_schedule(&priv->done_task[tlet]); \
677 } \
678 spin_unlock_irqrestore(&priv->reg_lock, flags); \
679 } \
680 \
681 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
682 IRQ_NONE; \
683 }
684
685 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
686
687 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
688 static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
689 { \
690 struct device *dev = data; \
691 struct talitos_private *priv = dev_get_drvdata(dev); \
692 u32 isr, isr_lo; \
693 unsigned long flags; \
694 \
695 spin_lock_irqsave(&priv->reg_lock, flags); \
696 isr = in_be32(priv->reg + TALITOS_ISR); \
697 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
698 /* Acknowledge interrupt */ \
699 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
700 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
701 \
702 if (unlikely(isr & ch_err_mask || isr_lo)) { \
703 spin_unlock_irqrestore(&priv->reg_lock, flags); \
704 talitos_error(dev, isr & ch_err_mask, isr_lo); \
705 } \
706 else { \
707 if (likely(isr & ch_done_mask)) { \
708 /* mask further done interrupts. */ \
709 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
710 /* done_task will unmask done interrupts at exit */ \
711 tasklet_schedule(&priv->done_task[tlet]); \
712 } \
713 spin_unlock_irqrestore(&priv->reg_lock, flags); \
714 } \
715 \
716 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
717 IRQ_NONE; \
718 }
719
720 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
721 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
722 0)
723 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
724 1)
725
726 /*
727 * hwrng
728 */
729 static int talitos_rng_data_present(struct hwrng *rng, int wait)
730 {
731 struct device *dev = (struct device *)rng->priv;
732 struct talitos_private *priv = dev_get_drvdata(dev);
733 u32 ofl;
734 int i;
735
736 for (i = 0; i < 20; i++) {
737 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
738 TALITOS_RNGUSR_LO_OFL;
739 if (ofl || !wait)
740 break;
741 udelay(10);
742 }
743
744 return !!ofl;
745 }
746
747 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
748 {
749 struct device *dev = (struct device *)rng->priv;
750 struct talitos_private *priv = dev_get_drvdata(dev);
751
752 /* rng fifo requires 64-bit accesses */
753 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
754 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
755
756 return sizeof(u32);
757 }
758
759 static int talitos_rng_init(struct hwrng *rng)
760 {
761 struct device *dev = (struct device *)rng->priv;
762 struct talitos_private *priv = dev_get_drvdata(dev);
763 unsigned int timeout = TALITOS_TIMEOUT;
764
765 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
766 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
767 & TALITOS_RNGUSR_LO_RD)
768 && --timeout)
769 cpu_relax();
770 if (timeout == 0) {
771 dev_err(dev, "failed to reset rng hw\n");
772 return -ENODEV;
773 }
774
775 /* start generating */
776 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
777
778 return 0;
779 }
780
781 static int talitos_register_rng(struct device *dev)
782 {
783 struct talitos_private *priv = dev_get_drvdata(dev);
784 int err;
785
786 priv->rng.name = dev_driver_string(dev),
787 priv->rng.init = talitos_rng_init,
788 priv->rng.data_present = talitos_rng_data_present,
789 priv->rng.data_read = talitos_rng_data_read,
790 priv->rng.priv = (unsigned long)dev;
791
792 err = hwrng_register(&priv->rng);
793 if (!err)
794 priv->rng_registered = true;
795
796 return err;
797 }
798
799 static void talitos_unregister_rng(struct device *dev)
800 {
801 struct talitos_private *priv = dev_get_drvdata(dev);
802
803 if (!priv->rng_registered)
804 return;
805
806 hwrng_unregister(&priv->rng);
807 priv->rng_registered = false;
808 }
809
810 /*
811 * crypto alg
812 */
813 #define TALITOS_CRA_PRIORITY 3000
814 /*
815 * Defines a priority for doing AEAD with descriptors type
816 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
817 */
818 #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1)
819 #define TALITOS_MAX_KEY_SIZE 96
820 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
821
822 struct talitos_ctx {
823 struct device *dev;
824 int ch;
825 __be32 desc_hdr_template;
826 u8 key[TALITOS_MAX_KEY_SIZE];
827 u8 iv[TALITOS_MAX_IV_LENGTH];
828 unsigned int keylen;
829 unsigned int enckeylen;
830 unsigned int authkeylen;
831 };
832
833 #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
834 #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
835
836 struct talitos_ahash_req_ctx {
837 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
838 unsigned int hw_context_size;
839 u8 buf[HASH_MAX_BLOCK_SIZE];
840 u8 bufnext[HASH_MAX_BLOCK_SIZE];
841 unsigned int swinit;
842 unsigned int first;
843 unsigned int last;
844 unsigned int to_hash_later;
845 unsigned int nbuf;
846 struct scatterlist bufsl[2];
847 struct scatterlist *psrc;
848 };
849
850 struct talitos_export_state {
851 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
852 u8 buf[HASH_MAX_BLOCK_SIZE];
853 unsigned int swinit;
854 unsigned int first;
855 unsigned int last;
856 unsigned int to_hash_later;
857 unsigned int nbuf;
858 };
859
860 static int aead_setkey(struct crypto_aead *authenc,
861 const u8 *key, unsigned int keylen)
862 {
863 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
864 struct crypto_authenc_keys keys;
865
866 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
867 goto badkey;
868
869 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
870 goto badkey;
871
872 memcpy(ctx->key, keys.authkey, keys.authkeylen);
873 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
874
875 ctx->keylen = keys.authkeylen + keys.enckeylen;
876 ctx->enckeylen = keys.enckeylen;
877 ctx->authkeylen = keys.authkeylen;
878
879 return 0;
880
881 badkey:
882 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
883 return -EINVAL;
884 }
885
886 /*
887 * talitos_edesc - s/w-extended descriptor
888 * @src_nents: number of segments in input scatterlist
889 * @dst_nents: number of segments in output scatterlist
890 * @icv_ool: whether ICV is out-of-line
891 * @iv_dma: dma address of iv for checking continuity and link table
892 * @dma_len: length of dma mapped link_tbl space
893 * @dma_link_tbl: bus physical address of link_tbl/buf
894 * @desc: h/w descriptor
895 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
896 * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
897 *
898 * if decrypting (with authcheck), or either one of src_nents or dst_nents
899 * is greater than 1, an integrity check value is concatenated to the end
900 * of link_tbl data
901 */
902 struct talitos_edesc {
903 int src_nents;
904 int dst_nents;
905 bool icv_ool;
906 dma_addr_t iv_dma;
907 int dma_len;
908 dma_addr_t dma_link_tbl;
909 struct talitos_desc desc;
910 union {
911 struct talitos_ptr link_tbl[0];
912 u8 buf[0];
913 };
914 };
915
916 static void talitos_sg_unmap(struct device *dev,
917 struct talitos_edesc *edesc,
918 struct scatterlist *src,
919 struct scatterlist *dst,
920 unsigned int len, unsigned int offset)
921 {
922 struct talitos_private *priv = dev_get_drvdata(dev);
923 bool is_sec1 = has_ftr_sec1(priv);
924 unsigned int src_nents = edesc->src_nents ? : 1;
925 unsigned int dst_nents = edesc->dst_nents ? : 1;
926
927 if (is_sec1 && dst && dst_nents > 1) {
928 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
929 len, DMA_FROM_DEVICE);
930 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
931 offset);
932 }
933 if (src != dst) {
934 if (src_nents == 1 || !is_sec1)
935 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
936
937 if (dst && (dst_nents == 1 || !is_sec1))
938 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
939 } else if (src_nents == 1 || !is_sec1) {
940 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
941 }
942 }
943
944 static void ipsec_esp_unmap(struct device *dev,
945 struct talitos_edesc *edesc,
946 struct aead_request *areq)
947 {
948 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
949 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
950 unsigned int ivsize = crypto_aead_ivsize(aead);
951
952 if (edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)
953 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
954 DMA_FROM_DEVICE);
955 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
956 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
957 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
958
959 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen,
960 areq->assoclen);
961
962 if (edesc->dma_len)
963 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
964 DMA_BIDIRECTIONAL);
965
966 if (!(edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)) {
967 unsigned int dst_nents = edesc->dst_nents ? : 1;
968
969 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
970 areq->assoclen + areq->cryptlen - ivsize);
971 }
972 }
973
974 /*
975 * ipsec_esp descriptor callbacks
976 */
977 static void ipsec_esp_encrypt_done(struct device *dev,
978 struct talitos_desc *desc, void *context,
979 int err)
980 {
981 struct talitos_private *priv = dev_get_drvdata(dev);
982 bool is_sec1 = has_ftr_sec1(priv);
983 struct aead_request *areq = context;
984 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
985 unsigned int authsize = crypto_aead_authsize(authenc);
986 struct talitos_edesc *edesc;
987 struct scatterlist *sg;
988 void *icvdata;
989
990 edesc = container_of(desc, struct talitos_edesc, desc);
991
992 ipsec_esp_unmap(dev, edesc, areq);
993
994 /* copy the generated ICV to dst */
995 if (edesc->icv_ool) {
996 if (is_sec1)
997 icvdata = edesc->buf + areq->assoclen + areq->cryptlen;
998 else
999 icvdata = &edesc->link_tbl[edesc->src_nents +
1000 edesc->dst_nents + 2];
1001 sg = sg_last(areq->dst, edesc->dst_nents);
1002 memcpy((char *)sg_virt(sg) + sg->length - authsize,
1003 icvdata, authsize);
1004 }
1005
1006 kfree(edesc);
1007
1008 aead_request_complete(areq, err);
1009 }
1010
1011 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
1012 struct talitos_desc *desc,
1013 void *context, int err)
1014 {
1015 struct aead_request *req = context;
1016 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1017 unsigned int authsize = crypto_aead_authsize(authenc);
1018 struct talitos_edesc *edesc;
1019 struct scatterlist *sg;
1020 char *oicv, *icv;
1021 struct talitos_private *priv = dev_get_drvdata(dev);
1022 bool is_sec1 = has_ftr_sec1(priv);
1023
1024 edesc = container_of(desc, struct talitos_edesc, desc);
1025
1026 ipsec_esp_unmap(dev, edesc, req);
1027
1028 if (!err) {
1029 /* auth check */
1030 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
1031 icv = (char *)sg_virt(sg) + sg->length - authsize;
1032
1033 if (edesc->dma_len) {
1034 if (is_sec1)
1035 oicv = (char *)&edesc->dma_link_tbl +
1036 req->assoclen + req->cryptlen;
1037 else
1038 oicv = (char *)
1039 &edesc->link_tbl[edesc->src_nents +
1040 edesc->dst_nents + 2];
1041 if (edesc->icv_ool)
1042 icv = oicv + authsize;
1043 } else
1044 oicv = (char *)&edesc->link_tbl[0];
1045
1046 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
1047 }
1048
1049 kfree(edesc);
1050
1051 aead_request_complete(req, err);
1052 }
1053
1054 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
1055 struct talitos_desc *desc,
1056 void *context, int err)
1057 {
1058 struct aead_request *req = context;
1059 struct talitos_edesc *edesc;
1060
1061 edesc = container_of(desc, struct talitos_edesc, desc);
1062
1063 ipsec_esp_unmap(dev, edesc, req);
1064
1065 /* check ICV auth status */
1066 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1067 DESC_HDR_LO_ICCR1_PASS))
1068 err = -EBADMSG;
1069
1070 kfree(edesc);
1071
1072 aead_request_complete(req, err);
1073 }
1074
1075 /*
1076 * convert scatterlist to SEC h/w link table format
1077 * stop at cryptlen bytes
1078 */
1079 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1080 unsigned int offset, int cryptlen,
1081 struct talitos_ptr *link_tbl_ptr)
1082 {
1083 int n_sg = sg_count;
1084 int count = 0;
1085
1086 while (cryptlen && sg && n_sg--) {
1087 unsigned int len = sg_dma_len(sg);
1088
1089 if (offset >= len) {
1090 offset -= len;
1091 goto next;
1092 }
1093
1094 len -= offset;
1095
1096 if (len > cryptlen)
1097 len = cryptlen;
1098
1099 to_talitos_ptr(link_tbl_ptr + count,
1100 sg_dma_address(sg) + offset, 0);
1101 to_talitos_ptr_len(link_tbl_ptr + count, len, 0);
1102 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
1103 count++;
1104 cryptlen -= len;
1105 offset = 0;
1106
1107 next:
1108 sg = sg_next(sg);
1109 }
1110
1111 /* tag end of link table */
1112 if (count > 0)
1113 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1114 DESC_PTR_LNKTBL_RETURN, 0);
1115
1116 return count;
1117 }
1118
1119 int talitos_sg_map(struct device *dev, struct scatterlist *src,
1120 unsigned int len, struct talitos_edesc *edesc,
1121 struct talitos_ptr *ptr,
1122 int sg_count, unsigned int offset, int tbl_off)
1123 {
1124 struct talitos_private *priv = dev_get_drvdata(dev);
1125 bool is_sec1 = has_ftr_sec1(priv);
1126
1127 to_talitos_ptr_len(ptr, len, is_sec1);
1128 to_talitos_ptr_ext_set(ptr, 0, is_sec1);
1129
1130 if (sg_count == 1) {
1131 to_talitos_ptr(ptr, sg_dma_address(src) + offset, is_sec1);
1132 return sg_count;
1133 }
1134 if (is_sec1) {
1135 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, is_sec1);
1136 return sg_count;
1137 }
1138 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len,
1139 &edesc->link_tbl[tbl_off]);
1140 if (sg_count == 1) {
1141 /* Only one segment now, so no link tbl needed*/
1142 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1143 return sg_count;
1144 }
1145 to_talitos_ptr(ptr, edesc->dma_link_tbl +
1146 tbl_off * sizeof(struct talitos_ptr), is_sec1);
1147 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1148
1149 return sg_count;
1150 }
1151
1152 /*
1153 * fill in and submit ipsec_esp descriptor
1154 */
1155 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1156 void (*callback)(struct device *dev,
1157 struct talitos_desc *desc,
1158 void *context, int error))
1159 {
1160 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1161 unsigned int authsize = crypto_aead_authsize(aead);
1162 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1163 struct device *dev = ctx->dev;
1164 struct talitos_desc *desc = &edesc->desc;
1165 unsigned int cryptlen = areq->cryptlen;
1166 unsigned int ivsize = crypto_aead_ivsize(aead);
1167 int tbl_off = 0;
1168 int sg_count, ret;
1169 int sg_link_tbl_len;
1170 bool sync_needed = false;
1171 struct talitos_private *priv = dev_get_drvdata(dev);
1172 bool is_sec1 = has_ftr_sec1(priv);
1173
1174 /* hmac key */
1175 map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1176 DMA_TO_DEVICE);
1177
1178 sg_count = edesc->src_nents ?: 1;
1179 if (is_sec1 && sg_count > 1)
1180 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1181 areq->assoclen + cryptlen);
1182 else
1183 sg_count = dma_map_sg(dev, areq->src, sg_count,
1184 (areq->src == areq->dst) ?
1185 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1186
1187 /* hmac data */
1188 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1189 &desc->ptr[1], sg_count, 0, tbl_off);
1190
1191 if (ret > 1) {
1192 tbl_off += ret;
1193 sync_needed = true;
1194 }
1195
1196 /* cipher iv */
1197 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1198 to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, is_sec1);
1199 to_talitos_ptr_len(&desc->ptr[2], ivsize, is_sec1);
1200 to_talitos_ptr_ext_set(&desc->ptr[2], 0, is_sec1);
1201 } else {
1202 to_talitos_ptr(&desc->ptr[3], edesc->iv_dma, is_sec1);
1203 to_talitos_ptr_len(&desc->ptr[3], ivsize, is_sec1);
1204 to_talitos_ptr_ext_set(&desc->ptr[3], 0, is_sec1);
1205 }
1206
1207 /* cipher key */
1208 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1209 map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1210 (char *)&ctx->key + ctx->authkeylen,
1211 DMA_TO_DEVICE);
1212 else
1213 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->enckeylen,
1214 (char *)&ctx->key + ctx->authkeylen,
1215 DMA_TO_DEVICE);
1216
1217 /*
1218 * cipher in
1219 * map and adjust cipher len to aead request cryptlen.
1220 * extent is bytes of HMAC postpended to ciphertext,
1221 * typically 12 for ipsec
1222 */
1223 to_talitos_ptr_len(&desc->ptr[4], cryptlen, is_sec1);
1224 to_talitos_ptr_ext_set(&desc->ptr[4], 0, is_sec1);
1225
1226 sg_link_tbl_len = cryptlen;
1227
1228 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1229 to_talitos_ptr_ext_set(&desc->ptr[4], authsize, is_sec1);
1230
1231 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1232 sg_link_tbl_len += authsize;
1233 }
1234
1235 sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1236 &desc->ptr[4], sg_count, areq->assoclen,
1237 tbl_off);
1238
1239 if (sg_count > 1) {
1240 tbl_off += sg_count;
1241 sync_needed = true;
1242 }
1243
1244 /* cipher out */
1245 if (areq->src != areq->dst) {
1246 sg_count = edesc->dst_nents ? : 1;
1247 if (!is_sec1 || sg_count == 1)
1248 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1249 }
1250
1251 sg_count = talitos_sg_map(dev, areq->dst, cryptlen, edesc,
1252 &desc->ptr[5], sg_count, areq->assoclen,
1253 tbl_off);
1254
1255 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1256 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
1257
1258 if (sg_count > 1) {
1259 edesc->icv_ool = true;
1260 sync_needed = true;
1261
1262 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1263 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1264 int offset = (edesc->src_nents + edesc->dst_nents + 2) *
1265 sizeof(struct talitos_ptr) + authsize;
1266
1267 /* Add an entry to the link table for ICV data */
1268 tbl_ptr += sg_count - 1;
1269 to_talitos_ptr_ext_set(tbl_ptr, 0, is_sec1);
1270 tbl_ptr++;
1271 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN,
1272 is_sec1);
1273 to_talitos_ptr_len(tbl_ptr, authsize, is_sec1);
1274
1275 /* icv data follows link tables */
1276 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset,
1277 is_sec1);
1278 }
1279 } else {
1280 edesc->icv_ool = false;
1281 }
1282
1283 /* ICV data */
1284 if (!(desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)) {
1285 to_talitos_ptr_len(&desc->ptr[6], authsize, is_sec1);
1286 to_talitos_ptr(&desc->ptr[6], edesc->dma_link_tbl +
1287 areq->assoclen + cryptlen, is_sec1);
1288 }
1289
1290 /* iv out */
1291 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1292 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1293 DMA_FROM_DEVICE);
1294
1295 if (sync_needed)
1296 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1297 edesc->dma_len,
1298 DMA_BIDIRECTIONAL);
1299
1300 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1301 if (ret != -EINPROGRESS) {
1302 ipsec_esp_unmap(dev, edesc, areq);
1303 kfree(edesc);
1304 }
1305 return ret;
1306 }
1307
1308 /*
1309 * allocate and map the extended descriptor
1310 */
1311 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1312 struct scatterlist *src,
1313 struct scatterlist *dst,
1314 u8 *iv,
1315 unsigned int assoclen,
1316 unsigned int cryptlen,
1317 unsigned int authsize,
1318 unsigned int ivsize,
1319 int icv_stashing,
1320 u32 cryptoflags,
1321 bool encrypt)
1322 {
1323 struct talitos_edesc *edesc;
1324 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
1325 dma_addr_t iv_dma = 0;
1326 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1327 GFP_ATOMIC;
1328 struct talitos_private *priv = dev_get_drvdata(dev);
1329 bool is_sec1 = has_ftr_sec1(priv);
1330 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1331 void *err;
1332
1333 if (cryptlen + authsize > max_len) {
1334 dev_err(dev, "length exceeds h/w max limit\n");
1335 return ERR_PTR(-EINVAL);
1336 }
1337
1338 if (ivsize)
1339 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1340
1341 if (!dst || dst == src) {
1342 src_len = assoclen + cryptlen + authsize;
1343 src_nents = sg_nents_for_len(src, src_len);
1344 if (src_nents < 0) {
1345 dev_err(dev, "Invalid number of src SG.\n");
1346 err = ERR_PTR(-EINVAL);
1347 goto error_sg;
1348 }
1349 src_nents = (src_nents == 1) ? 0 : src_nents;
1350 dst_nents = dst ? src_nents : 0;
1351 dst_len = 0;
1352 } else { /* dst && dst != src*/
1353 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1354 src_nents = sg_nents_for_len(src, src_len);
1355 if (src_nents < 0) {
1356 dev_err(dev, "Invalid number of src SG.\n");
1357 err = ERR_PTR(-EINVAL);
1358 goto error_sg;
1359 }
1360 src_nents = (src_nents == 1) ? 0 : src_nents;
1361 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1362 dst_nents = sg_nents_for_len(dst, dst_len);
1363 if (dst_nents < 0) {
1364 dev_err(dev, "Invalid number of dst SG.\n");
1365 err = ERR_PTR(-EINVAL);
1366 goto error_sg;
1367 }
1368 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1369 }
1370
1371 /*
1372 * allocate space for base edesc plus the link tables,
1373 * allowing for two separate entries for AD and generated ICV (+ 2),
1374 * and space for two sets of ICVs (stashed and generated)
1375 */
1376 alloc_len = sizeof(struct talitos_edesc);
1377 if (src_nents || dst_nents) {
1378 if (is_sec1)
1379 dma_len = (src_nents ? src_len : 0) +
1380 (dst_nents ? dst_len : 0);
1381 else
1382 dma_len = (src_nents + dst_nents + 2) *
1383 sizeof(struct talitos_ptr) + authsize * 2;
1384 alloc_len += dma_len;
1385 } else {
1386 dma_len = 0;
1387 alloc_len += icv_stashing ? authsize : 0;
1388 }
1389
1390 edesc = kmalloc(alloc_len, GFP_DMA | flags);
1391 if (!edesc) {
1392 dev_err(dev, "could not allocate edescriptor\n");
1393 err = ERR_PTR(-ENOMEM);
1394 goto error_sg;
1395 }
1396
1397 edesc->src_nents = src_nents;
1398 edesc->dst_nents = dst_nents;
1399 edesc->iv_dma = iv_dma;
1400 edesc->dma_len = dma_len;
1401 if (dma_len)
1402 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1403 edesc->dma_len,
1404 DMA_BIDIRECTIONAL);
1405
1406 return edesc;
1407 error_sg:
1408 if (iv_dma)
1409 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1410 return err;
1411 }
1412
1413 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1414 int icv_stashing, bool encrypt)
1415 {
1416 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1417 unsigned int authsize = crypto_aead_authsize(authenc);
1418 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1419 unsigned int ivsize = crypto_aead_ivsize(authenc);
1420
1421 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1422 iv, areq->assoclen, areq->cryptlen,
1423 authsize, ivsize, icv_stashing,
1424 areq->base.flags, encrypt);
1425 }
1426
1427 static int aead_encrypt(struct aead_request *req)
1428 {
1429 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1430 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1431 struct talitos_edesc *edesc;
1432
1433 /* allocate extended descriptor */
1434 edesc = aead_edesc_alloc(req, req->iv, 0, true);
1435 if (IS_ERR(edesc))
1436 return PTR_ERR(edesc);
1437
1438 /* set encrypt */
1439 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1440
1441 return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
1442 }
1443
1444 static int aead_decrypt(struct aead_request *req)
1445 {
1446 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1447 unsigned int authsize = crypto_aead_authsize(authenc);
1448 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1449 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1450 struct talitos_edesc *edesc;
1451 struct scatterlist *sg;
1452 void *icvdata;
1453
1454 req->cryptlen -= authsize;
1455
1456 /* allocate extended descriptor */
1457 edesc = aead_edesc_alloc(req, req->iv, 1, false);
1458 if (IS_ERR(edesc))
1459 return PTR_ERR(edesc);
1460
1461 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1462 ((!edesc->src_nents && !edesc->dst_nents) ||
1463 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1464
1465 /* decrypt and check the ICV */
1466 edesc->desc.hdr = ctx->desc_hdr_template |
1467 DESC_HDR_DIR_INBOUND |
1468 DESC_HDR_MODE1_MDEU_CICV;
1469
1470 /* reset integrity check result bits */
1471 edesc->desc.hdr_lo = 0;
1472
1473 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
1474 }
1475
1476 /* Have to check the ICV with software */
1477 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1478
1479 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1480 if (edesc->dma_len)
1481 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1482 edesc->dst_nents + 2];
1483 else
1484 icvdata = &edesc->link_tbl[0];
1485
1486 sg = sg_last(req->src, edesc->src_nents ? : 1);
1487
1488 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
1489
1490 return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
1491 }
1492
1493 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1494 const u8 *key, unsigned int keylen)
1495 {
1496 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1497
1498 memcpy(&ctx->key, key, keylen);
1499 ctx->keylen = keylen;
1500
1501 return 0;
1502 }
1503
1504 static void common_nonsnoop_unmap(struct device *dev,
1505 struct talitos_edesc *edesc,
1506 struct ablkcipher_request *areq)
1507 {
1508 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1509
1510 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0);
1511 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1512 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1513
1514 if (edesc->dma_len)
1515 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1516 DMA_BIDIRECTIONAL);
1517 }
1518
1519 static void ablkcipher_done(struct device *dev,
1520 struct talitos_desc *desc, void *context,
1521 int err)
1522 {
1523 struct ablkcipher_request *areq = context;
1524 struct talitos_edesc *edesc;
1525
1526 edesc = container_of(desc, struct talitos_edesc, desc);
1527
1528 common_nonsnoop_unmap(dev, edesc, areq);
1529
1530 kfree(edesc);
1531
1532 areq->base.complete(&areq->base, err);
1533 }
1534
1535 static int common_nonsnoop(struct talitos_edesc *edesc,
1536 struct ablkcipher_request *areq,
1537 void (*callback) (struct device *dev,
1538 struct talitos_desc *desc,
1539 void *context, int error))
1540 {
1541 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1542 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1543 struct device *dev = ctx->dev;
1544 struct talitos_desc *desc = &edesc->desc;
1545 unsigned int cryptlen = areq->nbytes;
1546 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1547 int sg_count, ret;
1548 bool sync_needed = false;
1549 struct talitos_private *priv = dev_get_drvdata(dev);
1550 bool is_sec1 = has_ftr_sec1(priv);
1551
1552 /* first DWORD empty */
1553 desc->ptr[0] = zero_entry;
1554
1555 /* cipher iv */
1556 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1);
1557 to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1);
1558 to_talitos_ptr_ext_set(&desc->ptr[1], 0, is_sec1);
1559
1560 /* cipher key */
1561 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1562 (char *)&ctx->key, DMA_TO_DEVICE);
1563
1564 sg_count = edesc->src_nents ?: 1;
1565 if (is_sec1 && sg_count > 1)
1566 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1567 cryptlen);
1568 else
1569 sg_count = dma_map_sg(dev, areq->src, sg_count,
1570 (areq->src == areq->dst) ?
1571 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1572 /*
1573 * cipher in
1574 */
1575 sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1576 &desc->ptr[3], sg_count, 0, 0);
1577 if (sg_count > 1)
1578 sync_needed = true;
1579
1580 /* cipher out */
1581 if (areq->src != areq->dst) {
1582 sg_count = edesc->dst_nents ? : 1;
1583 if (!is_sec1 || sg_count == 1)
1584 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1585 }
1586
1587 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1588 sg_count, 0, (edesc->src_nents + 1));
1589 if (ret > 1)
1590 sync_needed = true;
1591
1592 /* iv out */
1593 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1594 DMA_FROM_DEVICE);
1595
1596 /* last DWORD empty */
1597 desc->ptr[6] = zero_entry;
1598
1599 if (sync_needed)
1600 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1601 edesc->dma_len, DMA_BIDIRECTIONAL);
1602
1603 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1604 if (ret != -EINPROGRESS) {
1605 common_nonsnoop_unmap(dev, edesc, areq);
1606 kfree(edesc);
1607 }
1608 return ret;
1609 }
1610
1611 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1612 areq, bool encrypt)
1613 {
1614 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1615 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1616 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1617
1618 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1619 areq->info, 0, areq->nbytes, 0, ivsize, 0,
1620 areq->base.flags, encrypt);
1621 }
1622
1623 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1624 {
1625 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1626 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1627 struct talitos_edesc *edesc;
1628
1629 /* allocate extended descriptor */
1630 edesc = ablkcipher_edesc_alloc(areq, true);
1631 if (IS_ERR(edesc))
1632 return PTR_ERR(edesc);
1633
1634 /* set encrypt */
1635 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1636
1637 return common_nonsnoop(edesc, areq, ablkcipher_done);
1638 }
1639
1640 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1641 {
1642 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1643 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1644 struct talitos_edesc *edesc;
1645
1646 /* allocate extended descriptor */
1647 edesc = ablkcipher_edesc_alloc(areq, false);
1648 if (IS_ERR(edesc))
1649 return PTR_ERR(edesc);
1650
1651 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1652
1653 return common_nonsnoop(edesc, areq, ablkcipher_done);
1654 }
1655
1656 static void common_nonsnoop_hash_unmap(struct device *dev,
1657 struct talitos_edesc *edesc,
1658 struct ahash_request *areq)
1659 {
1660 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1661 struct talitos_private *priv = dev_get_drvdata(dev);
1662 bool is_sec1 = has_ftr_sec1(priv);
1663
1664 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1665
1666 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
1667
1668 /* When using hashctx-in, must unmap it. */
1669 if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1670 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1671 DMA_TO_DEVICE);
1672
1673 if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
1674 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1675 DMA_TO_DEVICE);
1676
1677 if (edesc->dma_len)
1678 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1679 DMA_BIDIRECTIONAL);
1680
1681 }
1682
1683 static void ahash_done(struct device *dev,
1684 struct talitos_desc *desc, void *context,
1685 int err)
1686 {
1687 struct ahash_request *areq = context;
1688 struct talitos_edesc *edesc =
1689 container_of(desc, struct talitos_edesc, desc);
1690 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1691
1692 if (!req_ctx->last && req_ctx->to_hash_later) {
1693 /* Position any partial block for next update/final/finup */
1694 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
1695 req_ctx->nbuf = req_ctx->to_hash_later;
1696 }
1697 common_nonsnoop_hash_unmap(dev, edesc, areq);
1698
1699 kfree(edesc);
1700
1701 areq->base.complete(&areq->base, err);
1702 }
1703
1704 /*
1705 * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1706 * ourself and submit a padded block
1707 */
1708 void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1709 struct talitos_edesc *edesc,
1710 struct talitos_ptr *ptr)
1711 {
1712 static u8 padded_hash[64] = {
1713 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1714 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1715 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1716 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1717 };
1718
1719 pr_err_once("Bug in SEC1, padding ourself\n");
1720 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1721 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1722 (char *)padded_hash, DMA_TO_DEVICE);
1723 }
1724
1725 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1726 struct ahash_request *areq, unsigned int length,
1727 void (*callback) (struct device *dev,
1728 struct talitos_desc *desc,
1729 void *context, int error))
1730 {
1731 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1732 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1733 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1734 struct device *dev = ctx->dev;
1735 struct talitos_desc *desc = &edesc->desc;
1736 int ret;
1737 bool sync_needed = false;
1738 struct talitos_private *priv = dev_get_drvdata(dev);
1739 bool is_sec1 = has_ftr_sec1(priv);
1740 int sg_count;
1741
1742 /* first DWORD empty */
1743 desc->ptr[0] = zero_entry;
1744
1745 /* hash context in */
1746 if (!req_ctx->first || req_ctx->swinit) {
1747 map_single_talitos_ptr(dev, &desc->ptr[1],
1748 req_ctx->hw_context_size,
1749 (char *)req_ctx->hw_context,
1750 DMA_TO_DEVICE);
1751 req_ctx->swinit = 0;
1752 } else {
1753 desc->ptr[1] = zero_entry;
1754 /* Indicate next op is not the first. */
1755 req_ctx->first = 0;
1756 }
1757
1758 /* HMAC key */
1759 if (ctx->keylen)
1760 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1761 (char *)&ctx->key, DMA_TO_DEVICE);
1762 else
1763 desc->ptr[2] = zero_entry;
1764
1765 sg_count = edesc->src_nents ?: 1;
1766 if (is_sec1 && sg_count > 1)
1767 sg_copy_to_buffer(areq->src, sg_count, edesc->buf, length);
1768 else
1769 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1770 DMA_TO_DEVICE);
1771 /*
1772 * data in
1773 */
1774 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1775 &desc->ptr[3], sg_count, 0, 0);
1776 if (sg_count > 1)
1777 sync_needed = true;
1778
1779 /* fifth DWORD empty */
1780 desc->ptr[4] = zero_entry;
1781
1782 /* hash/HMAC out -or- hash context out */
1783 if (req_ctx->last)
1784 map_single_talitos_ptr(dev, &desc->ptr[5],
1785 crypto_ahash_digestsize(tfm),
1786 areq->result, DMA_FROM_DEVICE);
1787 else
1788 map_single_talitos_ptr(dev, &desc->ptr[5],
1789 req_ctx->hw_context_size,
1790 req_ctx->hw_context, DMA_FROM_DEVICE);
1791
1792 /* last DWORD empty */
1793 desc->ptr[6] = zero_entry;
1794
1795 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1796 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1797
1798 if (sync_needed)
1799 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1800 edesc->dma_len, DMA_BIDIRECTIONAL);
1801
1802 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1803 if (ret != -EINPROGRESS) {
1804 common_nonsnoop_hash_unmap(dev, edesc, areq);
1805 kfree(edesc);
1806 }
1807 return ret;
1808 }
1809
1810 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1811 unsigned int nbytes)
1812 {
1813 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1814 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1815 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1816
1817 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
1818 nbytes, 0, 0, 0, areq->base.flags, false);
1819 }
1820
1821 static int ahash_init(struct ahash_request *areq)
1822 {
1823 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1824 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1825
1826 /* Initialize the context */
1827 req_ctx->nbuf = 0;
1828 req_ctx->first = 1; /* first indicates h/w must init its context */
1829 req_ctx->swinit = 0; /* assume h/w init of context */
1830 req_ctx->hw_context_size =
1831 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1832 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1833 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1834
1835 return 0;
1836 }
1837
1838 /*
1839 * on h/w without explicit sha224 support, we initialize h/w context
1840 * manually with sha224 constants, and tell it to run sha256.
1841 */
1842 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1843 {
1844 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1845
1846 ahash_init(areq);
1847 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1848
1849 req_ctx->hw_context[0] = SHA224_H0;
1850 req_ctx->hw_context[1] = SHA224_H1;
1851 req_ctx->hw_context[2] = SHA224_H2;
1852 req_ctx->hw_context[3] = SHA224_H3;
1853 req_ctx->hw_context[4] = SHA224_H4;
1854 req_ctx->hw_context[5] = SHA224_H5;
1855 req_ctx->hw_context[6] = SHA224_H6;
1856 req_ctx->hw_context[7] = SHA224_H7;
1857
1858 /* init 64-bit count */
1859 req_ctx->hw_context[8] = 0;
1860 req_ctx->hw_context[9] = 0;
1861
1862 return 0;
1863 }
1864
1865 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1866 {
1867 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1868 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1869 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1870 struct talitos_edesc *edesc;
1871 unsigned int blocksize =
1872 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1873 unsigned int nbytes_to_hash;
1874 unsigned int to_hash_later;
1875 unsigned int nsg;
1876 int nents;
1877
1878 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1879 /* Buffer up to one whole block */
1880 nents = sg_nents_for_len(areq->src, nbytes);
1881 if (nents < 0) {
1882 dev_err(ctx->dev, "Invalid number of src SG.\n");
1883 return nents;
1884 }
1885 sg_copy_to_buffer(areq->src, nents,
1886 req_ctx->buf + req_ctx->nbuf, nbytes);
1887 req_ctx->nbuf += nbytes;
1888 return 0;
1889 }
1890
1891 /* At least (blocksize + 1) bytes are available to hash */
1892 nbytes_to_hash = nbytes + req_ctx->nbuf;
1893 to_hash_later = nbytes_to_hash & (blocksize - 1);
1894
1895 if (req_ctx->last)
1896 to_hash_later = 0;
1897 else if (to_hash_later)
1898 /* There is a partial block. Hash the full block(s) now */
1899 nbytes_to_hash -= to_hash_later;
1900 else {
1901 /* Keep one block buffered */
1902 nbytes_to_hash -= blocksize;
1903 to_hash_later = blocksize;
1904 }
1905
1906 /* Chain in any previously buffered data */
1907 if (req_ctx->nbuf) {
1908 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1909 sg_init_table(req_ctx->bufsl, nsg);
1910 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1911 if (nsg > 1)
1912 sg_chain(req_ctx->bufsl, 2, areq->src);
1913 req_ctx->psrc = req_ctx->bufsl;
1914 } else
1915 req_ctx->psrc = areq->src;
1916
1917 if (to_hash_later) {
1918 nents = sg_nents_for_len(areq->src, nbytes);
1919 if (nents < 0) {
1920 dev_err(ctx->dev, "Invalid number of src SG.\n");
1921 return nents;
1922 }
1923 sg_pcopy_to_buffer(areq->src, nents,
1924 req_ctx->bufnext,
1925 to_hash_later,
1926 nbytes - to_hash_later);
1927 }
1928 req_ctx->to_hash_later = to_hash_later;
1929
1930 /* Allocate extended descriptor */
1931 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1932 if (IS_ERR(edesc))
1933 return PTR_ERR(edesc);
1934
1935 edesc->desc.hdr = ctx->desc_hdr_template;
1936
1937 /* On last one, request SEC to pad; otherwise continue */
1938 if (req_ctx->last)
1939 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1940 else
1941 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1942
1943 /* request SEC to INIT hash. */
1944 if (req_ctx->first && !req_ctx->swinit)
1945 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1946
1947 /* When the tfm context has a keylen, it's an HMAC.
1948 * A first or last (ie. not middle) descriptor must request HMAC.
1949 */
1950 if (ctx->keylen && (req_ctx->first || req_ctx->last))
1951 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1952
1953 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1954 ahash_done);
1955 }
1956
1957 static int ahash_update(struct ahash_request *areq)
1958 {
1959 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1960
1961 req_ctx->last = 0;
1962
1963 return ahash_process_req(areq, areq->nbytes);
1964 }
1965
1966 static int ahash_final(struct ahash_request *areq)
1967 {
1968 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1969
1970 req_ctx->last = 1;
1971
1972 return ahash_process_req(areq, 0);
1973 }
1974
1975 static int ahash_finup(struct ahash_request *areq)
1976 {
1977 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1978
1979 req_ctx->last = 1;
1980
1981 return ahash_process_req(areq, areq->nbytes);
1982 }
1983
1984 static int ahash_digest(struct ahash_request *areq)
1985 {
1986 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1987 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
1988
1989 ahash->init(areq);
1990 req_ctx->last = 1;
1991
1992 return ahash_process_req(areq, areq->nbytes);
1993 }
1994
1995 static int ahash_export(struct ahash_request *areq, void *out)
1996 {
1997 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1998 struct talitos_export_state *export = out;
1999
2000 memcpy(export->hw_context, req_ctx->hw_context,
2001 req_ctx->hw_context_size);
2002 memcpy(export->buf, req_ctx->buf, req_ctx->nbuf);
2003 export->swinit = req_ctx->swinit;
2004 export->first = req_ctx->first;
2005 export->last = req_ctx->last;
2006 export->to_hash_later = req_ctx->to_hash_later;
2007 export->nbuf = req_ctx->nbuf;
2008
2009 return 0;
2010 }
2011
2012 static int ahash_import(struct ahash_request *areq, const void *in)
2013 {
2014 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2015 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2016 const struct talitos_export_state *export = in;
2017
2018 memset(req_ctx, 0, sizeof(*req_ctx));
2019 req_ctx->hw_context_size =
2020 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2021 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2022 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2023 memcpy(req_ctx->hw_context, export->hw_context,
2024 req_ctx->hw_context_size);
2025 memcpy(req_ctx->buf, export->buf, export->nbuf);
2026 req_ctx->swinit = export->swinit;
2027 req_ctx->first = export->first;
2028 req_ctx->last = export->last;
2029 req_ctx->to_hash_later = export->to_hash_later;
2030 req_ctx->nbuf = export->nbuf;
2031
2032 return 0;
2033 }
2034
2035 struct keyhash_result {
2036 struct completion completion;
2037 int err;
2038 };
2039
2040 static void keyhash_complete(struct crypto_async_request *req, int err)
2041 {
2042 struct keyhash_result *res = req->data;
2043
2044 if (err == -EINPROGRESS)
2045 return;
2046
2047 res->err = err;
2048 complete(&res->completion);
2049 }
2050
2051 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2052 u8 *hash)
2053 {
2054 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2055
2056 struct scatterlist sg[1];
2057 struct ahash_request *req;
2058 struct keyhash_result hresult;
2059 int ret;
2060
2061 init_completion(&hresult.completion);
2062
2063 req = ahash_request_alloc(tfm, GFP_KERNEL);
2064 if (!req)
2065 return -ENOMEM;
2066
2067 /* Keep tfm keylen == 0 during hash of the long key */
2068 ctx->keylen = 0;
2069 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2070 keyhash_complete, &hresult);
2071
2072 sg_init_one(&sg[0], key, keylen);
2073
2074 ahash_request_set_crypt(req, sg, hash, keylen);
2075 ret = crypto_ahash_digest(req);
2076 switch (ret) {
2077 case 0:
2078 break;
2079 case -EINPROGRESS:
2080 case -EBUSY:
2081 ret = wait_for_completion_interruptible(
2082 &hresult.completion);
2083 if (!ret)
2084 ret = hresult.err;
2085 break;
2086 default:
2087 break;
2088 }
2089 ahash_request_free(req);
2090
2091 return ret;
2092 }
2093
2094 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2095 unsigned int keylen)
2096 {
2097 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2098 unsigned int blocksize =
2099 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2100 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2101 unsigned int keysize = keylen;
2102 u8 hash[SHA512_DIGEST_SIZE];
2103 int ret;
2104
2105 if (keylen <= blocksize)
2106 memcpy(ctx->key, key, keysize);
2107 else {
2108 /* Must get the hash of the long key */
2109 ret = keyhash(tfm, key, keylen, hash);
2110
2111 if (ret) {
2112 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2113 return -EINVAL;
2114 }
2115
2116 keysize = digestsize;
2117 memcpy(ctx->key, hash, digestsize);
2118 }
2119
2120 ctx->keylen = keysize;
2121
2122 return 0;
2123 }
2124
2125
2126 struct talitos_alg_template {
2127 u32 type;
2128 u32 priority;
2129 union {
2130 struct crypto_alg crypto;
2131 struct ahash_alg hash;
2132 struct aead_alg aead;
2133 } alg;
2134 __be32 desc_hdr_template;
2135 };
2136
2137 static struct talitos_alg_template driver_algs[] = {
2138 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
2139 { .type = CRYPTO_ALG_TYPE_AEAD,
2140 .alg.aead = {
2141 .base = {
2142 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2143 .cra_driver_name = "authenc-hmac-sha1-"
2144 "cbc-aes-talitos",
2145 .cra_blocksize = AES_BLOCK_SIZE,
2146 .cra_flags = CRYPTO_ALG_ASYNC,
2147 },
2148 .ivsize = AES_BLOCK_SIZE,
2149 .maxauthsize = SHA1_DIGEST_SIZE,
2150 },
2151 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2152 DESC_HDR_SEL0_AESU |
2153 DESC_HDR_MODE0_AESU_CBC |
2154 DESC_HDR_SEL1_MDEUA |
2155 DESC_HDR_MODE1_MDEU_INIT |
2156 DESC_HDR_MODE1_MDEU_PAD |
2157 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2158 },
2159 { .type = CRYPTO_ALG_TYPE_AEAD,
2160 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2161 .alg.aead = {
2162 .base = {
2163 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2164 .cra_driver_name = "authenc-hmac-sha1-"
2165 "cbc-aes-talitos",
2166 .cra_blocksize = AES_BLOCK_SIZE,
2167 .cra_flags = CRYPTO_ALG_ASYNC,
2168 },
2169 .ivsize = AES_BLOCK_SIZE,
2170 .maxauthsize = SHA1_DIGEST_SIZE,
2171 },
2172 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2173 DESC_HDR_SEL0_AESU |
2174 DESC_HDR_MODE0_AESU_CBC |
2175 DESC_HDR_SEL1_MDEUA |
2176 DESC_HDR_MODE1_MDEU_INIT |
2177 DESC_HDR_MODE1_MDEU_PAD |
2178 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2179 },
2180 { .type = CRYPTO_ALG_TYPE_AEAD,
2181 .alg.aead = {
2182 .base = {
2183 .cra_name = "authenc(hmac(sha1),"
2184 "cbc(des3_ede))",
2185 .cra_driver_name = "authenc-hmac-sha1-"
2186 "cbc-3des-talitos",
2187 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2188 .cra_flags = CRYPTO_ALG_ASYNC,
2189 },
2190 .ivsize = DES3_EDE_BLOCK_SIZE,
2191 .maxauthsize = SHA1_DIGEST_SIZE,
2192 },
2193 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2194 DESC_HDR_SEL0_DEU |
2195 DESC_HDR_MODE0_DEU_CBC |
2196 DESC_HDR_MODE0_DEU_3DES |
2197 DESC_HDR_SEL1_MDEUA |
2198 DESC_HDR_MODE1_MDEU_INIT |
2199 DESC_HDR_MODE1_MDEU_PAD |
2200 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2201 },
2202 { .type = CRYPTO_ALG_TYPE_AEAD,
2203 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2204 .alg.aead = {
2205 .base = {
2206 .cra_name = "authenc(hmac(sha1),"
2207 "cbc(des3_ede))",
2208 .cra_driver_name = "authenc-hmac-sha1-"
2209 "cbc-3des-talitos",
2210 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2211 .cra_flags = CRYPTO_ALG_ASYNC,
2212 },
2213 .ivsize = DES3_EDE_BLOCK_SIZE,
2214 .maxauthsize = SHA1_DIGEST_SIZE,
2215 },
2216 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2217 DESC_HDR_SEL0_DEU |
2218 DESC_HDR_MODE0_DEU_CBC |
2219 DESC_HDR_MODE0_DEU_3DES |
2220 DESC_HDR_SEL1_MDEUA |
2221 DESC_HDR_MODE1_MDEU_INIT |
2222 DESC_HDR_MODE1_MDEU_PAD |
2223 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2224 },
2225 { .type = CRYPTO_ALG_TYPE_AEAD,
2226 .alg.aead = {
2227 .base = {
2228 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2229 .cra_driver_name = "authenc-hmac-sha224-"
2230 "cbc-aes-talitos",
2231 .cra_blocksize = AES_BLOCK_SIZE,
2232 .cra_flags = CRYPTO_ALG_ASYNC,
2233 },
2234 .ivsize = AES_BLOCK_SIZE,
2235 .maxauthsize = SHA224_DIGEST_SIZE,
2236 },
2237 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2238 DESC_HDR_SEL0_AESU |
2239 DESC_HDR_MODE0_AESU_CBC |
2240 DESC_HDR_SEL1_MDEUA |
2241 DESC_HDR_MODE1_MDEU_INIT |
2242 DESC_HDR_MODE1_MDEU_PAD |
2243 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2244 },
2245 { .type = CRYPTO_ALG_TYPE_AEAD,
2246 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2247 .alg.aead = {
2248 .base = {
2249 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2250 .cra_driver_name = "authenc-hmac-sha224-"
2251 "cbc-aes-talitos",
2252 .cra_blocksize = AES_BLOCK_SIZE,
2253 .cra_flags = CRYPTO_ALG_ASYNC,
2254 },
2255 .ivsize = AES_BLOCK_SIZE,
2256 .maxauthsize = SHA224_DIGEST_SIZE,
2257 },
2258 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2259 DESC_HDR_SEL0_AESU |
2260 DESC_HDR_MODE0_AESU_CBC |
2261 DESC_HDR_SEL1_MDEUA |
2262 DESC_HDR_MODE1_MDEU_INIT |
2263 DESC_HDR_MODE1_MDEU_PAD |
2264 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2265 },
2266 { .type = CRYPTO_ALG_TYPE_AEAD,
2267 .alg.aead = {
2268 .base = {
2269 .cra_name = "authenc(hmac(sha224),"
2270 "cbc(des3_ede))",
2271 .cra_driver_name = "authenc-hmac-sha224-"
2272 "cbc-3des-talitos",
2273 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2274 .cra_flags = CRYPTO_ALG_ASYNC,
2275 },
2276 .ivsize = DES3_EDE_BLOCK_SIZE,
2277 .maxauthsize = SHA224_DIGEST_SIZE,
2278 },
2279 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2280 DESC_HDR_SEL0_DEU |
2281 DESC_HDR_MODE0_DEU_CBC |
2282 DESC_HDR_MODE0_DEU_3DES |
2283 DESC_HDR_SEL1_MDEUA |
2284 DESC_HDR_MODE1_MDEU_INIT |
2285 DESC_HDR_MODE1_MDEU_PAD |
2286 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2287 },
2288 { .type = CRYPTO_ALG_TYPE_AEAD,
2289 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2290 .alg.aead = {
2291 .base = {
2292 .cra_name = "authenc(hmac(sha224),"
2293 "cbc(des3_ede))",
2294 .cra_driver_name = "authenc-hmac-sha224-"
2295 "cbc-3des-talitos",
2296 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2297 .cra_flags = CRYPTO_ALG_ASYNC,
2298 },
2299 .ivsize = DES3_EDE_BLOCK_SIZE,
2300 .maxauthsize = SHA224_DIGEST_SIZE,
2301 },
2302 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2303 DESC_HDR_SEL0_DEU |
2304 DESC_HDR_MODE0_DEU_CBC |
2305 DESC_HDR_MODE0_DEU_3DES |
2306 DESC_HDR_SEL1_MDEUA |
2307 DESC_HDR_MODE1_MDEU_INIT |
2308 DESC_HDR_MODE1_MDEU_PAD |
2309 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2310 },
2311 { .type = CRYPTO_ALG_TYPE_AEAD,
2312 .alg.aead = {
2313 .base = {
2314 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2315 .cra_driver_name = "authenc-hmac-sha256-"
2316 "cbc-aes-talitos",
2317 .cra_blocksize = AES_BLOCK_SIZE,
2318 .cra_flags = CRYPTO_ALG_ASYNC,
2319 },
2320 .ivsize = AES_BLOCK_SIZE,
2321 .maxauthsize = SHA256_DIGEST_SIZE,
2322 },
2323 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2324 DESC_HDR_SEL0_AESU |
2325 DESC_HDR_MODE0_AESU_CBC |
2326 DESC_HDR_SEL1_MDEUA |
2327 DESC_HDR_MODE1_MDEU_INIT |
2328 DESC_HDR_MODE1_MDEU_PAD |
2329 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2330 },
2331 { .type = CRYPTO_ALG_TYPE_AEAD,
2332 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2333 .alg.aead = {
2334 .base = {
2335 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2336 .cra_driver_name = "authenc-hmac-sha256-"
2337 "cbc-aes-talitos",
2338 .cra_blocksize = AES_BLOCK_SIZE,
2339 .cra_flags = CRYPTO_ALG_ASYNC,
2340 },
2341 .ivsize = AES_BLOCK_SIZE,
2342 .maxauthsize = SHA256_DIGEST_SIZE,
2343 },
2344 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2345 DESC_HDR_SEL0_AESU |
2346 DESC_HDR_MODE0_AESU_CBC |
2347 DESC_HDR_SEL1_MDEUA |
2348 DESC_HDR_MODE1_MDEU_INIT |
2349 DESC_HDR_MODE1_MDEU_PAD |
2350 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2351 },
2352 { .type = CRYPTO_ALG_TYPE_AEAD,
2353 .alg.aead = {
2354 .base = {
2355 .cra_name = "authenc(hmac(sha256),"
2356 "cbc(des3_ede))",
2357 .cra_driver_name = "authenc-hmac-sha256-"
2358 "cbc-3des-talitos",
2359 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2360 .cra_flags = CRYPTO_ALG_ASYNC,
2361 },
2362 .ivsize = DES3_EDE_BLOCK_SIZE,
2363 .maxauthsize = SHA256_DIGEST_SIZE,
2364 },
2365 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2366 DESC_HDR_SEL0_DEU |
2367 DESC_HDR_MODE0_DEU_CBC |
2368 DESC_HDR_MODE0_DEU_3DES |
2369 DESC_HDR_SEL1_MDEUA |
2370 DESC_HDR_MODE1_MDEU_INIT |
2371 DESC_HDR_MODE1_MDEU_PAD |
2372 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2373 },
2374 { .type = CRYPTO_ALG_TYPE_AEAD,
2375 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2376 .alg.aead = {
2377 .base = {
2378 .cra_name = "authenc(hmac(sha256),"
2379 "cbc(des3_ede))",
2380 .cra_driver_name = "authenc-hmac-sha256-"
2381 "cbc-3des-talitos",
2382 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2383 .cra_flags = CRYPTO_ALG_ASYNC,
2384 },
2385 .ivsize = DES3_EDE_BLOCK_SIZE,
2386 .maxauthsize = SHA256_DIGEST_SIZE,
2387 },
2388 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2389 DESC_HDR_SEL0_DEU |
2390 DESC_HDR_MODE0_DEU_CBC |
2391 DESC_HDR_MODE0_DEU_3DES |
2392 DESC_HDR_SEL1_MDEUA |
2393 DESC_HDR_MODE1_MDEU_INIT |
2394 DESC_HDR_MODE1_MDEU_PAD |
2395 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2396 },
2397 { .type = CRYPTO_ALG_TYPE_AEAD,
2398 .alg.aead = {
2399 .base = {
2400 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2401 .cra_driver_name = "authenc-hmac-sha384-"
2402 "cbc-aes-talitos",
2403 .cra_blocksize = AES_BLOCK_SIZE,
2404 .cra_flags = CRYPTO_ALG_ASYNC,
2405 },
2406 .ivsize = AES_BLOCK_SIZE,
2407 .maxauthsize = SHA384_DIGEST_SIZE,
2408 },
2409 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2410 DESC_HDR_SEL0_AESU |
2411 DESC_HDR_MODE0_AESU_CBC |
2412 DESC_HDR_SEL1_MDEUB |
2413 DESC_HDR_MODE1_MDEU_INIT |
2414 DESC_HDR_MODE1_MDEU_PAD |
2415 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2416 },
2417 { .type = CRYPTO_ALG_TYPE_AEAD,
2418 .alg.aead = {
2419 .base = {
2420 .cra_name = "authenc(hmac(sha384),"
2421 "cbc(des3_ede))",
2422 .cra_driver_name = "authenc-hmac-sha384-"
2423 "cbc-3des-talitos",
2424 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2425 .cra_flags = CRYPTO_ALG_ASYNC,
2426 },
2427 .ivsize = DES3_EDE_BLOCK_SIZE,
2428 .maxauthsize = SHA384_DIGEST_SIZE,
2429 },
2430 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2431 DESC_HDR_SEL0_DEU |
2432 DESC_HDR_MODE0_DEU_CBC |
2433 DESC_HDR_MODE0_DEU_3DES |
2434 DESC_HDR_SEL1_MDEUB |
2435 DESC_HDR_MODE1_MDEU_INIT |
2436 DESC_HDR_MODE1_MDEU_PAD |
2437 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2438 },
2439 { .type = CRYPTO_ALG_TYPE_AEAD,
2440 .alg.aead = {
2441 .base = {
2442 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2443 .cra_driver_name = "authenc-hmac-sha512-"
2444 "cbc-aes-talitos",
2445 .cra_blocksize = AES_BLOCK_SIZE,
2446 .cra_flags = CRYPTO_ALG_ASYNC,
2447 },
2448 .ivsize = AES_BLOCK_SIZE,
2449 .maxauthsize = SHA512_DIGEST_SIZE,
2450 },
2451 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2452 DESC_HDR_SEL0_AESU |
2453 DESC_HDR_MODE0_AESU_CBC |
2454 DESC_HDR_SEL1_MDEUB |
2455 DESC_HDR_MODE1_MDEU_INIT |
2456 DESC_HDR_MODE1_MDEU_PAD |
2457 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2458 },
2459 { .type = CRYPTO_ALG_TYPE_AEAD,
2460 .alg.aead = {
2461 .base = {
2462 .cra_name = "authenc(hmac(sha512),"
2463 "cbc(des3_ede))",
2464 .cra_driver_name = "authenc-hmac-sha512-"
2465 "cbc-3des-talitos",
2466 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2467 .cra_flags = CRYPTO_ALG_ASYNC,
2468 },
2469 .ivsize = DES3_EDE_BLOCK_SIZE,
2470 .maxauthsize = SHA512_DIGEST_SIZE,
2471 },
2472 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2473 DESC_HDR_SEL0_DEU |
2474 DESC_HDR_MODE0_DEU_CBC |
2475 DESC_HDR_MODE0_DEU_3DES |
2476 DESC_HDR_SEL1_MDEUB |
2477 DESC_HDR_MODE1_MDEU_INIT |
2478 DESC_HDR_MODE1_MDEU_PAD |
2479 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2480 },
2481 { .type = CRYPTO_ALG_TYPE_AEAD,
2482 .alg.aead = {
2483 .base = {
2484 .cra_name = "authenc(hmac(md5),cbc(aes))",
2485 .cra_driver_name = "authenc-hmac-md5-"
2486 "cbc-aes-talitos",
2487 .cra_blocksize = AES_BLOCK_SIZE,
2488 .cra_flags = CRYPTO_ALG_ASYNC,
2489 },
2490 .ivsize = AES_BLOCK_SIZE,
2491 .maxauthsize = MD5_DIGEST_SIZE,
2492 },
2493 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2494 DESC_HDR_SEL0_AESU |
2495 DESC_HDR_MODE0_AESU_CBC |
2496 DESC_HDR_SEL1_MDEUA |
2497 DESC_HDR_MODE1_MDEU_INIT |
2498 DESC_HDR_MODE1_MDEU_PAD |
2499 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2500 },
2501 { .type = CRYPTO_ALG_TYPE_AEAD,
2502 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2503 .alg.aead = {
2504 .base = {
2505 .cra_name = "authenc(hmac(md5),cbc(aes))",
2506 .cra_driver_name = "authenc-hmac-md5-"
2507 "cbc-aes-talitos",
2508 .cra_blocksize = AES_BLOCK_SIZE,
2509 .cra_flags = CRYPTO_ALG_ASYNC,
2510 },
2511 .ivsize = AES_BLOCK_SIZE,
2512 .maxauthsize = MD5_DIGEST_SIZE,
2513 },
2514 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2515 DESC_HDR_SEL0_AESU |
2516 DESC_HDR_MODE0_AESU_CBC |
2517 DESC_HDR_SEL1_MDEUA |
2518 DESC_HDR_MODE1_MDEU_INIT |
2519 DESC_HDR_MODE1_MDEU_PAD |
2520 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2521 },
2522 { .type = CRYPTO_ALG_TYPE_AEAD,
2523 .alg.aead = {
2524 .base = {
2525 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2526 .cra_driver_name = "authenc-hmac-md5-"
2527 "cbc-3des-talitos",
2528 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2529 .cra_flags = CRYPTO_ALG_ASYNC,
2530 },
2531 .ivsize = DES3_EDE_BLOCK_SIZE,
2532 .maxauthsize = MD5_DIGEST_SIZE,
2533 },
2534 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2535 DESC_HDR_SEL0_DEU |
2536 DESC_HDR_MODE0_DEU_CBC |
2537 DESC_HDR_MODE0_DEU_3DES |
2538 DESC_HDR_SEL1_MDEUA |
2539 DESC_HDR_MODE1_MDEU_INIT |
2540 DESC_HDR_MODE1_MDEU_PAD |
2541 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2542 },
2543 { .type = CRYPTO_ALG_TYPE_AEAD,
2544 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2545 .alg.aead = {
2546 .base = {
2547 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2548 .cra_driver_name = "authenc-hmac-md5-"
2549 "cbc-3des-talitos",
2550 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2551 .cra_flags = CRYPTO_ALG_ASYNC,
2552 },
2553 .ivsize = DES3_EDE_BLOCK_SIZE,
2554 .maxauthsize = MD5_DIGEST_SIZE,
2555 },
2556 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2557 DESC_HDR_SEL0_DEU |
2558 DESC_HDR_MODE0_DEU_CBC |
2559 DESC_HDR_MODE0_DEU_3DES |
2560 DESC_HDR_SEL1_MDEUA |
2561 DESC_HDR_MODE1_MDEU_INIT |
2562 DESC_HDR_MODE1_MDEU_PAD |
2563 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2564 },
2565 /* ABLKCIPHER algorithms. */
2566 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2567 .alg.crypto = {
2568 .cra_name = "ecb(aes)",
2569 .cra_driver_name = "ecb-aes-talitos",
2570 .cra_blocksize = AES_BLOCK_SIZE,
2571 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2572 CRYPTO_ALG_ASYNC,
2573 .cra_ablkcipher = {
2574 .min_keysize = AES_MIN_KEY_SIZE,
2575 .max_keysize = AES_MAX_KEY_SIZE,
2576 .ivsize = AES_BLOCK_SIZE,
2577 }
2578 },
2579 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2580 DESC_HDR_SEL0_AESU,
2581 },
2582 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2583 .alg.crypto = {
2584 .cra_name = "cbc(aes)",
2585 .cra_driver_name = "cbc-aes-talitos",
2586 .cra_blocksize = AES_BLOCK_SIZE,
2587 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2588 CRYPTO_ALG_ASYNC,
2589 .cra_ablkcipher = {
2590 .min_keysize = AES_MIN_KEY_SIZE,
2591 .max_keysize = AES_MAX_KEY_SIZE,
2592 .ivsize = AES_BLOCK_SIZE,
2593 }
2594 },
2595 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2596 DESC_HDR_SEL0_AESU |
2597 DESC_HDR_MODE0_AESU_CBC,
2598 },
2599 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2600 .alg.crypto = {
2601 .cra_name = "ctr(aes)",
2602 .cra_driver_name = "ctr-aes-talitos",
2603 .cra_blocksize = AES_BLOCK_SIZE,
2604 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2605 CRYPTO_ALG_ASYNC,
2606 .cra_ablkcipher = {
2607 .min_keysize = AES_MIN_KEY_SIZE,
2608 .max_keysize = AES_MAX_KEY_SIZE,
2609 .ivsize = AES_BLOCK_SIZE,
2610 }
2611 },
2612 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2613 DESC_HDR_SEL0_AESU |
2614 DESC_HDR_MODE0_AESU_CTR,
2615 },
2616 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2617 .alg.crypto = {
2618 .cra_name = "ecb(des)",
2619 .cra_driver_name = "ecb-des-talitos",
2620 .cra_blocksize = DES_BLOCK_SIZE,
2621 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2622 CRYPTO_ALG_ASYNC,
2623 .cra_ablkcipher = {
2624 .min_keysize = DES_KEY_SIZE,
2625 .max_keysize = DES_KEY_SIZE,
2626 .ivsize = DES_BLOCK_SIZE,
2627 }
2628 },
2629 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2630 DESC_HDR_SEL0_DEU,
2631 },
2632 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2633 .alg.crypto = {
2634 .cra_name = "cbc(des)",
2635 .cra_driver_name = "cbc-des-talitos",
2636 .cra_blocksize = DES_BLOCK_SIZE,
2637 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2638 CRYPTO_ALG_ASYNC,
2639 .cra_ablkcipher = {
2640 .min_keysize = DES_KEY_SIZE,
2641 .max_keysize = DES_KEY_SIZE,
2642 .ivsize = DES_BLOCK_SIZE,
2643 }
2644 },
2645 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2646 DESC_HDR_SEL0_DEU |
2647 DESC_HDR_MODE0_DEU_CBC,
2648 },
2649 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2650 .alg.crypto = {
2651 .cra_name = "ecb(des3_ede)",
2652 .cra_driver_name = "ecb-3des-talitos",
2653 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2654 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2655 CRYPTO_ALG_ASYNC,
2656 .cra_ablkcipher = {
2657 .min_keysize = DES3_EDE_KEY_SIZE,
2658 .max_keysize = DES3_EDE_KEY_SIZE,
2659 .ivsize = DES3_EDE_BLOCK_SIZE,
2660 }
2661 },
2662 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2663 DESC_HDR_SEL0_DEU |
2664 DESC_HDR_MODE0_DEU_3DES,
2665 },
2666 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2667 .alg.crypto = {
2668 .cra_name = "cbc(des3_ede)",
2669 .cra_driver_name = "cbc-3des-talitos",
2670 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2671 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2672 CRYPTO_ALG_ASYNC,
2673 .cra_ablkcipher = {
2674 .min_keysize = DES3_EDE_KEY_SIZE,
2675 .max_keysize = DES3_EDE_KEY_SIZE,
2676 .ivsize = DES3_EDE_BLOCK_SIZE,
2677 }
2678 },
2679 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2680 DESC_HDR_SEL0_DEU |
2681 DESC_HDR_MODE0_DEU_CBC |
2682 DESC_HDR_MODE0_DEU_3DES,
2683 },
2684 /* AHASH algorithms. */
2685 { .type = CRYPTO_ALG_TYPE_AHASH,
2686 .alg.hash = {
2687 .halg.digestsize = MD5_DIGEST_SIZE,
2688 .halg.statesize = sizeof(struct talitos_export_state),
2689 .halg.base = {
2690 .cra_name = "md5",
2691 .cra_driver_name = "md5-talitos",
2692 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2693 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2694 CRYPTO_ALG_ASYNC,
2695 }
2696 },
2697 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2698 DESC_HDR_SEL0_MDEUA |
2699 DESC_HDR_MODE0_MDEU_MD5,
2700 },
2701 { .type = CRYPTO_ALG_TYPE_AHASH,
2702 .alg.hash = {
2703 .halg.digestsize = SHA1_DIGEST_SIZE,
2704 .halg.statesize = sizeof(struct talitos_export_state),
2705 .halg.base = {
2706 .cra_name = "sha1",
2707 .cra_driver_name = "sha1-talitos",
2708 .cra_blocksize = SHA1_BLOCK_SIZE,
2709 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2710 CRYPTO_ALG_ASYNC,
2711 }
2712 },
2713 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2714 DESC_HDR_SEL0_MDEUA |
2715 DESC_HDR_MODE0_MDEU_SHA1,
2716 },
2717 { .type = CRYPTO_ALG_TYPE_AHASH,
2718 .alg.hash = {
2719 .halg.digestsize = SHA224_DIGEST_SIZE,
2720 .halg.statesize = sizeof(struct talitos_export_state),
2721 .halg.base = {
2722 .cra_name = "sha224",
2723 .cra_driver_name = "sha224-talitos",
2724 .cra_blocksize = SHA224_BLOCK_SIZE,
2725 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2726 CRYPTO_ALG_ASYNC,
2727 }
2728 },
2729 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2730 DESC_HDR_SEL0_MDEUA |
2731 DESC_HDR_MODE0_MDEU_SHA224,
2732 },
2733 { .type = CRYPTO_ALG_TYPE_AHASH,
2734 .alg.hash = {
2735 .halg.digestsize = SHA256_DIGEST_SIZE,
2736 .halg.statesize = sizeof(struct talitos_export_state),
2737 .halg.base = {
2738 .cra_name = "sha256",
2739 .cra_driver_name = "sha256-talitos",
2740 .cra_blocksize = SHA256_BLOCK_SIZE,
2741 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2742 CRYPTO_ALG_ASYNC,
2743 }
2744 },
2745 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2746 DESC_HDR_SEL0_MDEUA |
2747 DESC_HDR_MODE0_MDEU_SHA256,
2748 },
2749 { .type = CRYPTO_ALG_TYPE_AHASH,
2750 .alg.hash = {
2751 .halg.digestsize = SHA384_DIGEST_SIZE,
2752 .halg.statesize = sizeof(struct talitos_export_state),
2753 .halg.base = {
2754 .cra_name = "sha384",
2755 .cra_driver_name = "sha384-talitos",
2756 .cra_blocksize = SHA384_BLOCK_SIZE,
2757 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2758 CRYPTO_ALG_ASYNC,
2759 }
2760 },
2761 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2762 DESC_HDR_SEL0_MDEUB |
2763 DESC_HDR_MODE0_MDEUB_SHA384,
2764 },
2765 { .type = CRYPTO_ALG_TYPE_AHASH,
2766 .alg.hash = {
2767 .halg.digestsize = SHA512_DIGEST_SIZE,
2768 .halg.statesize = sizeof(struct talitos_export_state),
2769 .halg.base = {
2770 .cra_name = "sha512",
2771 .cra_driver_name = "sha512-talitos",
2772 .cra_blocksize = SHA512_BLOCK_SIZE,
2773 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2774 CRYPTO_ALG_ASYNC,
2775 }
2776 },
2777 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2778 DESC_HDR_SEL0_MDEUB |
2779 DESC_HDR_MODE0_MDEUB_SHA512,
2780 },
2781 { .type = CRYPTO_ALG_TYPE_AHASH,
2782 .alg.hash = {
2783 .halg.digestsize = MD5_DIGEST_SIZE,
2784 .halg.statesize = sizeof(struct talitos_export_state),
2785 .halg.base = {
2786 .cra_name = "hmac(md5)",
2787 .cra_driver_name = "hmac-md5-talitos",
2788 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2789 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2790 CRYPTO_ALG_ASYNC,
2791 }
2792 },
2793 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2794 DESC_HDR_SEL0_MDEUA |
2795 DESC_HDR_MODE0_MDEU_MD5,
2796 },
2797 { .type = CRYPTO_ALG_TYPE_AHASH,
2798 .alg.hash = {
2799 .halg.digestsize = SHA1_DIGEST_SIZE,
2800 .halg.statesize = sizeof(struct talitos_export_state),
2801 .halg.base = {
2802 .cra_name = "hmac(sha1)",
2803 .cra_driver_name = "hmac-sha1-talitos",
2804 .cra_blocksize = SHA1_BLOCK_SIZE,
2805 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2806 CRYPTO_ALG_ASYNC,
2807 }
2808 },
2809 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2810 DESC_HDR_SEL0_MDEUA |
2811 DESC_HDR_MODE0_MDEU_SHA1,
2812 },
2813 { .type = CRYPTO_ALG_TYPE_AHASH,
2814 .alg.hash = {
2815 .halg.digestsize = SHA224_DIGEST_SIZE,
2816 .halg.statesize = sizeof(struct talitos_export_state),
2817 .halg.base = {
2818 .cra_name = "hmac(sha224)",
2819 .cra_driver_name = "hmac-sha224-talitos",
2820 .cra_blocksize = SHA224_BLOCK_SIZE,
2821 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2822 CRYPTO_ALG_ASYNC,
2823 }
2824 },
2825 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2826 DESC_HDR_SEL0_MDEUA |
2827 DESC_HDR_MODE0_MDEU_SHA224,
2828 },
2829 { .type = CRYPTO_ALG_TYPE_AHASH,
2830 .alg.hash = {
2831 .halg.digestsize = SHA256_DIGEST_SIZE,
2832 .halg.statesize = sizeof(struct talitos_export_state),
2833 .halg.base = {
2834 .cra_name = "hmac(sha256)",
2835 .cra_driver_name = "hmac-sha256-talitos",
2836 .cra_blocksize = SHA256_BLOCK_SIZE,
2837 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2838 CRYPTO_ALG_ASYNC,
2839 }
2840 },
2841 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2842 DESC_HDR_SEL0_MDEUA |
2843 DESC_HDR_MODE0_MDEU_SHA256,
2844 },
2845 { .type = CRYPTO_ALG_TYPE_AHASH,
2846 .alg.hash = {
2847 .halg.digestsize = SHA384_DIGEST_SIZE,
2848 .halg.statesize = sizeof(struct talitos_export_state),
2849 .halg.base = {
2850 .cra_name = "hmac(sha384)",
2851 .cra_driver_name = "hmac-sha384-talitos",
2852 .cra_blocksize = SHA384_BLOCK_SIZE,
2853 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2854 CRYPTO_ALG_ASYNC,
2855 }
2856 },
2857 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2858 DESC_HDR_SEL0_MDEUB |
2859 DESC_HDR_MODE0_MDEUB_SHA384,
2860 },
2861 { .type = CRYPTO_ALG_TYPE_AHASH,
2862 .alg.hash = {
2863 .halg.digestsize = SHA512_DIGEST_SIZE,
2864 .halg.statesize = sizeof(struct talitos_export_state),
2865 .halg.base = {
2866 .cra_name = "hmac(sha512)",
2867 .cra_driver_name = "hmac-sha512-talitos",
2868 .cra_blocksize = SHA512_BLOCK_SIZE,
2869 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2870 CRYPTO_ALG_ASYNC,
2871 }
2872 },
2873 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2874 DESC_HDR_SEL0_MDEUB |
2875 DESC_HDR_MODE0_MDEUB_SHA512,
2876 }
2877 };
2878
2879 struct talitos_crypto_alg {
2880 struct list_head entry;
2881 struct device *dev;
2882 struct talitos_alg_template algt;
2883 };
2884
2885 static int talitos_init_common(struct talitos_ctx *ctx,
2886 struct talitos_crypto_alg *talitos_alg)
2887 {
2888 struct talitos_private *priv;
2889
2890 /* update context with ptr to dev */
2891 ctx->dev = talitos_alg->dev;
2892
2893 /* assign SEC channel to tfm in round-robin fashion */
2894 priv = dev_get_drvdata(ctx->dev);
2895 ctx->ch = atomic_inc_return(&priv->last_chan) &
2896 (priv->num_channels - 1);
2897
2898 /* copy descriptor header template value */
2899 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2900
2901 /* select done notification */
2902 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2903
2904 return 0;
2905 }
2906
2907 static int talitos_cra_init(struct crypto_tfm *tfm)
2908 {
2909 struct crypto_alg *alg = tfm->__crt_alg;
2910 struct talitos_crypto_alg *talitos_alg;
2911 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2912
2913 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2914 talitos_alg = container_of(__crypto_ahash_alg(alg),
2915 struct talitos_crypto_alg,
2916 algt.alg.hash);
2917 else
2918 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2919 algt.alg.crypto);
2920
2921 return talitos_init_common(ctx, talitos_alg);
2922 }
2923
2924 static int talitos_cra_init_aead(struct crypto_aead *tfm)
2925 {
2926 struct aead_alg *alg = crypto_aead_alg(tfm);
2927 struct talitos_crypto_alg *talitos_alg;
2928 struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
2929
2930 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2931 algt.alg.aead);
2932
2933 return talitos_init_common(ctx, talitos_alg);
2934 }
2935
2936 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2937 {
2938 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2939
2940 talitos_cra_init(tfm);
2941
2942 ctx->keylen = 0;
2943 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2944 sizeof(struct talitos_ahash_req_ctx));
2945
2946 return 0;
2947 }
2948
2949 /*
2950 * given the alg's descriptor header template, determine whether descriptor
2951 * type and primary/secondary execution units required match the hw
2952 * capabilities description provided in the device tree node.
2953 */
2954 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2955 {
2956 struct talitos_private *priv = dev_get_drvdata(dev);
2957 int ret;
2958
2959 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2960 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2961
2962 if (SECONDARY_EU(desc_hdr_template))
2963 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2964 & priv->exec_units);
2965
2966 return ret;
2967 }
2968
2969 static int talitos_remove(struct platform_device *ofdev)
2970 {
2971 struct device *dev = &ofdev->dev;
2972 struct talitos_private *priv = dev_get_drvdata(dev);
2973 struct talitos_crypto_alg *t_alg, *n;
2974 int i;
2975
2976 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
2977 switch (t_alg->algt.type) {
2978 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2979 break;
2980 case CRYPTO_ALG_TYPE_AEAD:
2981 crypto_unregister_aead(&t_alg->algt.alg.aead);
2982 case CRYPTO_ALG_TYPE_AHASH:
2983 crypto_unregister_ahash(&t_alg->algt.alg.hash);
2984 break;
2985 }
2986 list_del(&t_alg->entry);
2987 kfree(t_alg);
2988 }
2989
2990 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2991 talitos_unregister_rng(dev);
2992
2993 for (i = 0; priv->chan && i < priv->num_channels; i++)
2994 kfree(priv->chan[i].fifo);
2995
2996 kfree(priv->chan);
2997
2998 for (i = 0; i < 2; i++)
2999 if (priv->irq[i]) {
3000 free_irq(priv->irq[i], dev);
3001 irq_dispose_mapping(priv->irq[i]);
3002 }
3003
3004 tasklet_kill(&priv->done_task[0]);
3005 if (priv->irq[1])
3006 tasklet_kill(&priv->done_task[1]);
3007
3008 iounmap(priv->reg);
3009
3010 kfree(priv);
3011
3012 return 0;
3013 }
3014
3015 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3016 struct talitos_alg_template
3017 *template)
3018 {
3019 struct talitos_private *priv = dev_get_drvdata(dev);
3020 struct talitos_crypto_alg *t_alg;
3021 struct crypto_alg *alg;
3022
3023 t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
3024 if (!t_alg)
3025 return ERR_PTR(-ENOMEM);
3026
3027 t_alg->algt = *template;
3028
3029 switch (t_alg->algt.type) {
3030 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3031 alg = &t_alg->algt.alg.crypto;
3032 alg->cra_init = talitos_cra_init;
3033 alg->cra_type = &crypto_ablkcipher_type;
3034 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
3035 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
3036 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
3037 alg->cra_ablkcipher.geniv = "eseqiv";
3038 break;
3039 case CRYPTO_ALG_TYPE_AEAD:
3040 alg = &t_alg->algt.alg.aead.base;
3041 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3042 t_alg->algt.alg.aead.setkey = aead_setkey;
3043 t_alg->algt.alg.aead.encrypt = aead_encrypt;
3044 t_alg->algt.alg.aead.decrypt = aead_decrypt;
3045 break;
3046 case CRYPTO_ALG_TYPE_AHASH:
3047 alg = &t_alg->algt.alg.hash.halg.base;
3048 alg->cra_init = talitos_cra_init_ahash;
3049 alg->cra_type = &crypto_ahash_type;
3050 t_alg->algt.alg.hash.init = ahash_init;
3051 t_alg->algt.alg.hash.update = ahash_update;
3052 t_alg->algt.alg.hash.final = ahash_final;
3053 t_alg->algt.alg.hash.finup = ahash_finup;
3054 t_alg->algt.alg.hash.digest = ahash_digest;
3055 t_alg->algt.alg.hash.setkey = ahash_setkey;
3056 t_alg->algt.alg.hash.import = ahash_import;
3057 t_alg->algt.alg.hash.export = ahash_export;
3058
3059 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
3060 !strncmp(alg->cra_name, "hmac", 4)) {
3061 kfree(t_alg);
3062 return ERR_PTR(-ENOTSUPP);
3063 }
3064 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3065 (!strcmp(alg->cra_name, "sha224") ||
3066 !strcmp(alg->cra_name, "hmac(sha224)"))) {
3067 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3068 t_alg->algt.desc_hdr_template =
3069 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3070 DESC_HDR_SEL0_MDEUA |
3071 DESC_HDR_MODE0_MDEU_SHA256;
3072 }
3073 break;
3074 default:
3075 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
3076 kfree(t_alg);
3077 return ERR_PTR(-EINVAL);
3078 }
3079
3080 alg->cra_module = THIS_MODULE;
3081 if (t_alg->algt.priority)
3082 alg->cra_priority = t_alg->algt.priority;
3083 else
3084 alg->cra_priority = TALITOS_CRA_PRIORITY;
3085 alg->cra_alignmask = 0;
3086 alg->cra_ctxsize = sizeof(struct talitos_ctx);
3087 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
3088
3089 t_alg->dev = dev;
3090
3091 return t_alg;
3092 }
3093
3094 static int talitos_probe_irq(struct platform_device *ofdev)
3095 {
3096 struct device *dev = &ofdev->dev;
3097 struct device_node *np = ofdev->dev.of_node;
3098 struct talitos_private *priv = dev_get_drvdata(dev);
3099 int err;
3100 bool is_sec1 = has_ftr_sec1(priv);
3101
3102 priv->irq[0] = irq_of_parse_and_map(np, 0);
3103 if (!priv->irq[0]) {
3104 dev_err(dev, "failed to map irq\n");
3105 return -EINVAL;
3106 }
3107 if (is_sec1) {
3108 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3109 dev_driver_string(dev), dev);
3110 goto primary_out;
3111 }
3112
3113 priv->irq[1] = irq_of_parse_and_map(np, 1);
3114
3115 /* get the primary irq line */
3116 if (!priv->irq[1]) {
3117 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
3118 dev_driver_string(dev), dev);
3119 goto primary_out;
3120 }
3121
3122 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
3123 dev_driver_string(dev), dev);
3124 if (err)
3125 goto primary_out;
3126
3127 /* get the secondary irq line */
3128 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
3129 dev_driver_string(dev), dev);
3130 if (err) {
3131 dev_err(dev, "failed to request secondary irq\n");
3132 irq_dispose_mapping(priv->irq[1]);
3133 priv->irq[1] = 0;
3134 }
3135
3136 return err;
3137
3138 primary_out:
3139 if (err) {
3140 dev_err(dev, "failed to request primary irq\n");
3141 irq_dispose_mapping(priv->irq[0]);
3142 priv->irq[0] = 0;
3143 }
3144
3145 return err;
3146 }
3147
3148 static int talitos_probe(struct platform_device *ofdev)
3149 {
3150 struct device *dev = &ofdev->dev;
3151 struct device_node *np = ofdev->dev.of_node;
3152 struct talitos_private *priv;
3153 const unsigned int *prop;
3154 int i, err;
3155 int stride;
3156
3157 priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
3158 if (!priv)
3159 return -ENOMEM;
3160
3161 INIT_LIST_HEAD(&priv->alg_list);
3162
3163 dev_set_drvdata(dev, priv);
3164
3165 priv->ofdev = ofdev;
3166
3167 spin_lock_init(&priv->reg_lock);
3168
3169 priv->reg = of_iomap(np, 0);
3170 if (!priv->reg) {
3171 dev_err(dev, "failed to of_iomap\n");
3172 err = -ENOMEM;
3173 goto err_out;
3174 }
3175
3176 /* get SEC version capabilities from device tree */
3177 prop = of_get_property(np, "fsl,num-channels", NULL);
3178 if (prop)
3179 priv->num_channels = *prop;
3180
3181 prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
3182 if (prop)
3183 priv->chfifo_len = *prop;
3184
3185 prop = of_get_property(np, "fsl,exec-units-mask", NULL);
3186 if (prop)
3187 priv->exec_units = *prop;
3188
3189 prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
3190 if (prop)
3191 priv->desc_types = *prop;
3192
3193 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3194 !priv->exec_units || !priv->desc_types) {
3195 dev_err(dev, "invalid property data in device tree node\n");
3196 err = -EINVAL;
3197 goto err_out;
3198 }
3199
3200 if (of_device_is_compatible(np, "fsl,sec3.0"))
3201 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3202
3203 if (of_device_is_compatible(np, "fsl,sec2.1"))
3204 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
3205 TALITOS_FTR_SHA224_HWINIT |
3206 TALITOS_FTR_HMAC_OK;
3207
3208 if (of_device_is_compatible(np, "fsl,sec1.0"))
3209 priv->features |= TALITOS_FTR_SEC1;
3210
3211 if (of_device_is_compatible(np, "fsl,sec1.2")) {
3212 priv->reg_deu = priv->reg + TALITOS12_DEU;
3213 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3214 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3215 stride = TALITOS1_CH_STRIDE;
3216 } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3217 priv->reg_deu = priv->reg + TALITOS10_DEU;
3218 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3219 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3220 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3221 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3222 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3223 stride = TALITOS1_CH_STRIDE;
3224 } else {
3225 priv->reg_deu = priv->reg + TALITOS2_DEU;
3226 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3227 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3228 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3229 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3230 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3231 priv->reg_keu = priv->reg + TALITOS2_KEU;
3232 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3233 stride = TALITOS2_CH_STRIDE;
3234 }
3235
3236 err = talitos_probe_irq(ofdev);
3237 if (err)
3238 goto err_out;
3239
3240 if (of_device_is_compatible(np, "fsl,sec1.0")) {
3241 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3242 (unsigned long)dev);
3243 } else {
3244 if (!priv->irq[1]) {
3245 tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3246 (unsigned long)dev);
3247 } else {
3248 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3249 (unsigned long)dev);
3250 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3251 (unsigned long)dev);
3252 }
3253 }
3254
3255 priv->chan = kzalloc(sizeof(struct talitos_channel) *
3256 priv->num_channels, GFP_KERNEL);
3257 if (!priv->chan) {
3258 dev_err(dev, "failed to allocate channel management space\n");
3259 err = -ENOMEM;
3260 goto err_out;
3261 }
3262
3263 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3264
3265 for (i = 0; i < priv->num_channels; i++) {
3266 priv->chan[i].reg = priv->reg + stride * (i + 1);
3267 if (!priv->irq[1] || !(i & 1))
3268 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3269
3270 spin_lock_init(&priv->chan[i].head_lock);
3271 spin_lock_init(&priv->chan[i].tail_lock);
3272
3273 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
3274 priv->fifo_len, GFP_KERNEL);
3275 if (!priv->chan[i].fifo) {
3276 dev_err(dev, "failed to allocate request fifo %d\n", i);
3277 err = -ENOMEM;
3278 goto err_out;
3279 }
3280
3281 atomic_set(&priv->chan[i].submit_count,
3282 -(priv->chfifo_len - 1));
3283 }
3284
3285 dma_set_mask(dev, DMA_BIT_MASK(36));
3286
3287 /* reset and initialize the h/w */
3288 err = init_device(dev);
3289 if (err) {
3290 dev_err(dev, "failed to initialize device\n");
3291 goto err_out;
3292 }
3293
3294 /* register the RNG, if available */
3295 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3296 err = talitos_register_rng(dev);
3297 if (err) {
3298 dev_err(dev, "failed to register hwrng: %d\n", err);
3299 goto err_out;
3300 } else
3301 dev_info(dev, "hwrng\n");
3302 }
3303
3304 /* register crypto algorithms the device supports */
3305 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3306 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3307 struct talitos_crypto_alg *t_alg;
3308 struct crypto_alg *alg = NULL;
3309
3310 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3311 if (IS_ERR(t_alg)) {
3312 err = PTR_ERR(t_alg);
3313 if (err == -ENOTSUPP)
3314 continue;
3315 goto err_out;
3316 }
3317
3318 switch (t_alg->algt.type) {
3319 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3320 err = crypto_register_alg(
3321 &t_alg->algt.alg.crypto);
3322 alg = &t_alg->algt.alg.crypto;
3323 break;
3324
3325 case CRYPTO_ALG_TYPE_AEAD:
3326 err = crypto_register_aead(
3327 &t_alg->algt.alg.aead);
3328 alg = &t_alg->algt.alg.aead.base;
3329 break;
3330
3331 case CRYPTO_ALG_TYPE_AHASH:
3332 err = crypto_register_ahash(
3333 &t_alg->algt.alg.hash);
3334 alg = &t_alg->algt.alg.hash.halg.base;
3335 break;
3336 }
3337 if (err) {
3338 dev_err(dev, "%s alg registration failed\n",
3339 alg->cra_driver_name);
3340 kfree(t_alg);
3341 } else
3342 list_add_tail(&t_alg->entry, &priv->alg_list);
3343 }
3344 }
3345 if (!list_empty(&priv->alg_list))
3346 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3347 (char *)of_get_property(np, "compatible", NULL));
3348
3349 return 0;
3350
3351 err_out:
3352 talitos_remove(ofdev);
3353
3354 return err;
3355 }
3356
3357 static const struct of_device_id talitos_match[] = {
3358 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
3359 {
3360 .compatible = "fsl,sec1.0",
3361 },
3362 #endif
3363 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
3364 {
3365 .compatible = "fsl,sec2.0",
3366 },
3367 #endif
3368 {},
3369 };
3370 MODULE_DEVICE_TABLE(of, talitos_match);
3371
3372 static struct platform_driver talitos_driver = {
3373 .driver = {
3374 .name = "talitos",
3375 .of_match_table = talitos_match,
3376 },
3377 .probe = talitos_probe,
3378 .remove = talitos_remove,
3379 };
3380
3381 module_platform_driver(talitos_driver);
3382
3383 MODULE_LICENSE("GPL");
3384 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3385 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
Linux kernel stable tree