?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
GUI: Fix Tomato RAF theme for all builds. Compilation typo.
[tomato.git] / release / src-rt-6.x.4708 / linux / linux-2.6.36 / arch / cris / arch-v32 / drivers / cryptocop.c
blob2db96c26b6ab16c23b8fa16834505a20c2be6e4a
1 /*
2 * Stream co-processor driver for the ETRAX FS
3 *
4 * Copyright (C) 2003-2007 Axis Communications AB
5 */
6
7 #include <linux/init.h>
8 #include <linux/sched.h>
9 #include <linux/module.h>
10 #include <linux/slab.h>
11 #include <linux/string.h>
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/spinlock.h>
15 #include <linux/stddef.h>
16
17 #include <asm/uaccess.h>
18 #include <asm/io.h>
19 #include <asm/atomic.h>
20
21 #include <linux/list.h>
22 #include <linux/interrupt.h>
23
24 #include <asm/signal.h>
25 #include <asm/irq.h>
26
27 #include <dma.h>
28 #include <hwregs/dma.h>
29 #include <hwregs/reg_map.h>
30 #include <hwregs/reg_rdwr.h>
31 #include <hwregs/intr_vect_defs.h>
32
33 #include <hwregs/strcop.h>
34 #include <hwregs/strcop_defs.h>
35 #include <cryptocop.h>
36
37 #ifdef CONFIG_ETRAXFS
38 #define IN_DMA 9
39 #define OUT_DMA 8
40 #define IN_DMA_INST regi_dma9
41 #define OUT_DMA_INST regi_dma8
42 #define DMA_IRQ DMA9_INTR_VECT
43 #else
44 #define IN_DMA 3
45 #define OUT_DMA 2
46 #define IN_DMA_INST regi_dma3
47 #define OUT_DMA_INST regi_dma2
48 #define DMA_IRQ DMA3_INTR_VECT
49 #endif
50
51 #define DESCR_ALLOC_PAD (31)
52
53 struct cryptocop_dma_desc {
54 char *free_buf; /* If non-null will be kfreed in free_cdesc() */
55 dma_descr_data *dma_descr;
56
57 unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
58
59 unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
60 struct cryptocop_dma_desc *next;
61 };
62
63
64 struct cryptocop_int_operation{
65 void *alloc_ptr;
66 cryptocop_session_id sid;
67
68 dma_descr_context ctx_out;
69 dma_descr_context ctx_in;
70
71 /* DMA descriptors allocated by driver. */
72 struct cryptocop_dma_desc *cdesc_out;
73 struct cryptocop_dma_desc *cdesc_in;
74
75 /* Strcop config to use. */
76 cryptocop_3des_mode tdes_mode;
77 cryptocop_csum_type csum_mode;
78
79 /* DMA descrs provided by consumer. */
80 dma_descr_data *ddesc_out;
81 dma_descr_data *ddesc_in;
82 };
83
84
85 struct cryptocop_tfrm_ctx {
86 cryptocop_tfrm_id tid;
87 unsigned int blocklength;
88
89 unsigned int start_ix;
90
91 struct cryptocop_tfrm_cfg *tcfg;
92 struct cryptocop_transform_ctx *tctx;
93
94 unsigned char previous_src;
95 unsigned char current_src;
96
97 /* Values to use in metadata out. */
98 unsigned char hash_conf;
99 unsigned char hash_mode;
100 unsigned char ciph_conf;
101 unsigned char cbcmode;
102 unsigned char decrypt;
103
104 unsigned int requires_padding:1;
105 unsigned int strict_block_length:1;
106 unsigned int active:1;
107 unsigned int done:1;
108 size_t consumed;
109 size_t produced;
110
111 /* Pad (input) descriptors to put in the DMA out list when the transform
112 * output is put on the DMA in list. */
113 struct cryptocop_dma_desc *pad_descs;
114
115 struct cryptocop_tfrm_ctx *prev_src;
116 struct cryptocop_tfrm_ctx *curr_src;
117
118 /* Mapping to HW. */
119 unsigned char unit_no;
120 };
121
122
123 struct cryptocop_private{
124 cryptocop_session_id sid;
125 struct cryptocop_private *next;
126 };
127
128 /* Session list. */
129
130 struct cryptocop_transform_ctx{
131 struct cryptocop_transform_init init;
132 unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
133 unsigned int dec_key_set:1;
134
135 struct cryptocop_transform_ctx *next;
136 };
137
138
139 struct cryptocop_session{
140 cryptocop_session_id sid;
141
142 struct cryptocop_transform_ctx *tfrm_ctx;
143
144 struct cryptocop_session *next;
145 };
146
147 /* Priority levels for jobs sent to the cryptocop. Checksum operations from
148 kernel have highest priority since TCPIP stack processing must not
149 be a bottleneck. */
150 typedef enum {
151 cryptocop_prio_kernel_csum = 0,
152 cryptocop_prio_kernel = 1,
153 cryptocop_prio_user = 2,
154 cryptocop_prio_no_prios = 3
155 } cryptocop_queue_priority;
156
157 struct cryptocop_prio_queue{
158 struct list_head jobs;
159 cryptocop_queue_priority prio;
160 };
161
162 struct cryptocop_prio_job{
163 struct list_head node;
164 cryptocop_queue_priority prio;
165
166 struct cryptocop_operation *oper;
167 struct cryptocop_int_operation *iop;
168 };
169
170 struct ioctl_job_cb_ctx {
171 unsigned int processed:1;
172 };
173
174
175 static struct cryptocop_session *cryptocop_sessions = NULL;
176 spinlock_t cryptocop_sessions_lock;
177
178 /* Next Session ID to assign. */
179 static cryptocop_session_id next_sid = 1;
180
181 /* Pad for checksum. */
182 static const char csum_zero_pad[1] = {0x00};
183
184 /* Trash buffer for mem2mem operations. */
185 #define MEM2MEM_DISCARD_BUF_LENGTH (512)
186 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
187
188 /* Descriptor pool. */
189 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE (100)
190 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
191 static struct cryptocop_dma_desc *descr_pool_free_list;
192 static int descr_pool_no_free;
193 static spinlock_t descr_pool_lock;
194
195 /* Lock to stop cryptocop to start processing of a new operation. The holder
196 of this lock MUST call cryptocop_start_job() after it is unlocked. */
197 spinlock_t cryptocop_process_lock;
198
199 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
200 static spinlock_t cryptocop_job_queue_lock;
201 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
202 static spinlock_t running_job_lock;
203
204 /* The interrupt handler appends completed jobs to this list. The scehduled
205 * tasklet removes them upon sending the response to the crypto consumer. */
206 static struct list_head cryptocop_completed_jobs;
207 static spinlock_t cryptocop_completed_jobs_lock;
208
209 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
210
211
212 /** Local functions. **/
213
214 static int cryptocop_open(struct inode *, struct file *);
215
216 static int cryptocop_release(struct inode *, struct file *);
217
218 static long cryptocop_ioctl(struct file *file,
219 unsigned int cmd, unsigned long arg);
220
221 static void cryptocop_start_job(void);
222
223 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
224 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
225
226 static int cryptocop_job_queue_init(void);
227 static void cryptocop_job_queue_close(void);
228
229 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
230
231 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
232
233 static int transform_ok(struct cryptocop_transform_init *tinit);
234
235 static struct cryptocop_session *get_session(cryptocop_session_id sid);
236
237 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
238
239 static void delete_internal_operation(struct cryptocop_int_operation *iop);
240
241 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength);
242
243 static int init_stream_coprocessor(void);
244
245 static void __exit exit_stream_coprocessor(void);
246
247 /*#define LDEBUG*/
248 #ifdef LDEBUG
249 #define DEBUG(s) s
250 #define DEBUG_API(s) s
251 static void print_cryptocop_operation(struct cryptocop_operation *cop);
252 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
253 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
254 static void print_lock_status(void);
255 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
256 #define assert(s) do{if (!(s)) panic(#s);} while(0);
257 #else
258 #define DEBUG(s)
259 #define DEBUG_API(s)
260 #define assert(s)
261 #endif
262
263
264 /* Transform constants. */
265 #define DES_BLOCK_LENGTH (8)
266 #define AES_BLOCK_LENGTH (16)
267 #define MD5_BLOCK_LENGTH (64)
268 #define SHA1_BLOCK_LENGTH (64)
269 #define CSUM_BLOCK_LENGTH (2)
270 #define MD5_STATE_LENGTH (16)
271 #define SHA1_STATE_LENGTH (20)
272
273 /* The device number. */
274 #define CRYPTOCOP_MAJOR (254)
275 #define CRYPTOCOP_MINOR (0)
276
277
278
279 const struct file_operations cryptocop_fops = {
280 .owner = THIS_MODULE,
281 .open = cryptocop_open,
282 .release = cryptocop_release,
283 .unlocked_ioctl = cryptocop_ioctl
284 };
285
286
287 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
288 {
289 DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
290 kfree(cdesc->free_buf);
291
292 if (cdesc->from_pool) {
293 unsigned long int flags;
294 spin_lock_irqsave(&descr_pool_lock, flags);
295 cdesc->next = descr_pool_free_list;
296 descr_pool_free_list = cdesc;
297 ++descr_pool_no_free;
298 spin_unlock_irqrestore(&descr_pool_lock, flags);
299 } else {
300 kfree(cdesc);
301 }
302 }
303
304
305 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
306 {
307 int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
308 struct cryptocop_dma_desc *cdesc;
309
310 if (use_pool) {
311 unsigned long int flags;
312 spin_lock_irqsave(&descr_pool_lock, flags);
313 if (!descr_pool_free_list) {
314 spin_unlock_irqrestore(&descr_pool_lock, flags);
315 DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
316 return NULL;
317 }
318 cdesc = descr_pool_free_list;
319 descr_pool_free_list = descr_pool_free_list->next;
320 --descr_pool_no_free;
321 spin_unlock_irqrestore(&descr_pool_lock, flags);
322 cdesc->from_pool = 1;
323 } else {
324 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
325 if (!cdesc) {
326 DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
327 return NULL;
328 }
329 cdesc->from_pool = 0;
330 }
331 cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
332
333 cdesc->next = NULL;
334
335 cdesc->free_buf = NULL;
336 cdesc->dma_descr->out_eop = 0;
337 cdesc->dma_descr->in_eop = 0;
338 cdesc->dma_descr->intr = 0;
339 cdesc->dma_descr->eol = 0;
340 cdesc->dma_descr->wait = 0;
341 cdesc->dma_descr->buf = NULL;
342 cdesc->dma_descr->after = NULL;
343
344 DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
345 return cdesc;
346 }
347
348
349 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
350 {
351 DEBUG(printk("setup_descr_chain: entering\n"));
352 while (cd) {
353 if (cd->next) {
354 cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
355 } else {
356 cd->dma_descr->next = NULL;
357 }
358 cd = cd->next;
359 }
360 DEBUG(printk("setup_descr_chain: exit\n"));
361 }
362
363
364 /* Create a pad descriptor for the transform.
365 * Return -1 for error, 0 if pad created. */
366 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
367 {
368 struct cryptocop_dma_desc *cdesc = NULL;
369 int error = 0;
370 struct strcop_meta_out mo = {
371 .ciphsel = src_none,
372 .hashsel = src_none,
373 .csumsel = src_none
374 };
375 char *pad;
376 size_t plen;
377
378 DEBUG(printk("create_pad_descriptor: start.\n"));
379 /* Setup pad descriptor. */
380
381 DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
382 cdesc = alloc_cdesc(alloc_flag);
383 if (!cdesc){
384 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
385 goto error_cleanup;
386 }
387 switch (tc->unit_no) {
388 case src_md5:
389 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
390 if (error){
391 DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
392 goto error_cleanup;
393 }
394 cdesc->free_buf = pad;
395 mo.hashsel = src_dma;
396 mo.hashconf = tc->hash_conf;
397 mo.hashmode = tc->hash_mode;
398 break;
399 case src_sha1:
400 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
401 if (error){
402 DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
403 goto error_cleanup;
404 }
405 cdesc->free_buf = pad;
406 mo.hashsel = src_dma;
407 mo.hashconf = tc->hash_conf;
408 mo.hashmode = tc->hash_mode;
409 break;
410 case src_csum:
411 if (tc->consumed % tc->blocklength){
412 pad = (char*)csum_zero_pad;
413 plen = 1;
414 } else {
415 pad = (char*)cdesc; /* Use any pointer. */
416 plen = 0;
417 }
418 mo.csumsel = src_dma;
419 break;
420 }
421 cdesc->dma_descr->wait = 1;
422 cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed. EOP is ok here since the padded unit is the only one active. */
423 cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
424 cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
425
426 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
427 *pad_desc = cdesc;
428
429 return 0;
430
431 error_cleanup:
432 if (cdesc) free_cdesc(cdesc);
433 return -1;
434 }
435
436
437 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
438 {
439 struct cryptocop_dma_desc *key_desc = alloc_cdesc(alloc_flag);
440 struct strcop_meta_out mo = {0};
441
442 DEBUG(printk("setup_key_dl_desc\n"));
443
444 if (!key_desc) {
445 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
446 return -ENOMEM;
447 }
448
449 /* Download key. */
450 if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
451 /* Precook the AES decrypt key. */
452 if (!tc->tctx->dec_key_set){
453 get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
454 tc->tctx->dec_key_set = 1;
455 }
456 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
457 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
458 } else {
459 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
460 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
461 }
462 /* Setup metadata. */
463 mo.dlkey = 1;
464 switch (tc->tctx->init.keylen) {
465 case 64:
466 mo.decrypt = 0;
467 mo.hashmode = 0;
468 break;
469 case 128:
470 mo.decrypt = 0;
471 mo.hashmode = 1;
472 break;
473 case 192:
474 mo.decrypt = 1;
475 mo.hashmode = 0;
476 break;
477 case 256:
478 mo.decrypt = 1;
479 mo.hashmode = 1;
480 break;
481 default:
482 break;
483 }
484 mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
485 key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
486
487 key_desc->dma_descr->out_eop = 1;
488 key_desc->dma_descr->wait = 1;
489 key_desc->dma_descr->intr = 0;
490
491 *kd = key_desc;
492 return 0;
493 }
494
495 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
496 {
497 struct cryptocop_dma_desc *iv_desc = alloc_cdesc(alloc_flag);
498 struct strcop_meta_out mo = {0};
499
500 DEBUG(printk("setup_cipher_iv_desc\n"));
501
502 if (!iv_desc) {
503 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
504 return -ENOMEM;
505 }
506 /* Download IV. */
507 iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
508 iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
509
510 /* Setup metadata. */
511 mo.hashsel = mo.csumsel = src_none;
512 mo.ciphsel = src_dma;
513 mo.ciphconf = tc->ciph_conf;
514 mo.cbcmode = tc->cbcmode;
515
516 iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
517
518 iv_desc->dma_descr->out_eop = 0;
519 iv_desc->dma_descr->wait = 1;
520 iv_desc->dma_descr->intr = 0;
521
522 *id = iv_desc;
523 return 0;
524 }
525
526 /* Map the ouput length of the transform to operation output starting on the inject index. */
527 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
528 {
529 int err = 0;
530 struct cryptocop_dma_desc head = {0};
531 struct cryptocop_dma_desc *outdesc = &head;
532 size_t iov_offset = 0;
533 size_t out_ix = 0;
534 int outiov_ix = 0;
535 struct strcop_meta_in mi = {0};
536
537 size_t out_length = tc->produced;
538 int rem_length;
539 int dlength;
540
541 assert(out_length != 0);
542 if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
543 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
544 return -EINVAL;
545 }
546 /* Traverse the out iovec until the result inject index is reached. */
547 while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
548 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
549 outiov_ix++;
550 }
551 if (outiov_ix >= operation->tfrm_op.outcount){
552 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
553 return -EINVAL;
554 }
555 iov_offset = tc->tcfg->inject_ix - out_ix;
556 mi.dmasel = tc->unit_no;
557
558 /* Setup the output descriptors. */
559 while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
560 outdesc->next = alloc_cdesc(alloc_flag);
561 if (!outdesc->next) {
562 DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
563 err = -ENOMEM;
564 goto error_cleanup;
565 }
566 outdesc = outdesc->next;
567 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
568 dlength = (out_length < rem_length) ? out_length : rem_length;
569
570 DEBUG(printk("create_input_descriptors:\n"
571 "outiov_ix=%d, rem_length=%d, dlength=%d\n"
572 "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
573 "outcount=%d, outiov_ix=%d\n",
574 outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
575
576 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
577 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
578 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
579
580 out_length -= dlength;
581 iov_offset += dlength;
582 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
583 iov_offset = 0;
584 ++outiov_ix;
585 }
586 }
587 if (out_length > 0){
588 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
589 err = -EINVAL;
590 goto error_cleanup;
591 }
592 /* Set sync in last descriptor. */
593 mi.sync = 1;
594 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
595
596 *id = head.next;
597 return 0;
598
599 error_cleanup:
600 while (head.next) {
601 outdesc = head.next->next;
602 free_cdesc(head.next);
603 head.next = outdesc;
604 }
605 return err;
606 }
607
608
609 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
610 {
611 while (desc_len != 0) {
612 struct cryptocop_dma_desc *cdesc;
613 int rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
614 int dlength = (desc_len < rem_length) ? desc_len : rem_length;
615
616 cdesc = alloc_cdesc(alloc_flag);
617 if (!cdesc) {
618 DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
619 return -ENOMEM;
620 }
621 (*current_out_cdesc)->next = cdesc;
622 (*current_out_cdesc) = cdesc;
623
624 cdesc->free_buf = NULL;
625
626 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
627 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
628
629 assert(desc_len >= dlength);
630 desc_len -= dlength;
631 *iniov_offset += dlength;
632 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
633 *iniov_offset = 0;
634 ++(*iniov_ix);
635 if (*iniov_ix > operation->tfrm_op.incount) {
636 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
637 return -EINVAL;
638 }
639 }
640 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
641 } /* while (desc_len != 0) */
642 /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
643 (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
644
645 return 0;
646 }
647
648
649 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
650 {
651 DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
652 if (tc->tcfg) {
653 int failed = 0;
654 struct cryptocop_dma_desc *idescs = NULL;
655 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
656 if (tc->pad_descs) {
657 DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
658 while (tc->pad_descs) {
659 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
660 (*current_out_cdesc)->next = tc->pad_descs;
661 tc->pad_descs = tc->pad_descs->next;
662 (*current_out_cdesc) = (*current_out_cdesc)->next;
663 }
664 }
665
666 /* Setup and append output descriptors to DMA in list. */
667 if (tc->unit_no == src_dma){
668 /* mem2mem. Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
669 struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
670 unsigned int start_ix = tc->start_ix;
671 while (start_ix){
672 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
673 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
674 if (!(*current_in_cdesc)->next){
675 DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
676 return -ENOMEM;
677 }
678 (*current_in_cdesc) = (*current_in_cdesc)->next;
679 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
680 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
681 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
682 start_ix -= desclen;
683 }
684 mi.sync = 1;
685 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
686 }
687
688 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
689 if (failed){
690 DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
691 return failed;
692 }
693 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
694 while (idescs) {
695 DEBUG(printk("append descriptor 0x%p\n", idescs));
696 (*current_in_cdesc)->next = idescs;
697 idescs = idescs->next;
698 (*current_in_cdesc) = (*current_in_cdesc)->next;
699 }
700 }
701 return 0;
702 }
703
704
705
706 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
707 {
708 struct cryptocop_session *sess;
709 struct cryptocop_transform_ctx *tctx;
710
711 struct cryptocop_tfrm_ctx digest_ctx = {
712 .previous_src = src_none,
713 .current_src = src_none,
714 .start_ix = 0,
715 .requires_padding = 1,
716 .strict_block_length = 0,
717 .hash_conf = 0,
718 .hash_mode = 0,
719 .ciph_conf = 0,
720 .cbcmode = 0,
721 .decrypt = 0,
722 .consumed = 0,
723 .produced = 0,
724 .pad_descs = NULL,
725 .active = 0,
726 .done = 0,
727 .prev_src = NULL,
728 .curr_src = NULL,
729 .tcfg = NULL};
730 struct cryptocop_tfrm_ctx cipher_ctx = {
731 .previous_src = src_none,
732 .current_src = src_none,
733 .start_ix = 0,
734 .requires_padding = 0,
735 .strict_block_length = 1,
736 .hash_conf = 0,
737 .hash_mode = 0,
738 .ciph_conf = 0,
739 .cbcmode = 0,
740 .decrypt = 0,
741 .consumed = 0,
742 .produced = 0,
743 .pad_descs = NULL,
744 .active = 0,
745 .done = 0,
746 .prev_src = NULL,
747 .curr_src = NULL,
748 .tcfg = NULL};
749 struct cryptocop_tfrm_ctx csum_ctx = {
750 .previous_src = src_none,
751 .current_src = src_none,
752 .start_ix = 0,
753 .blocklength = 2,
754 .requires_padding = 1,
755 .strict_block_length = 0,
756 .hash_conf = 0,
757 .hash_mode = 0,
758 .ciph_conf = 0,
759 .cbcmode = 0,
760 .decrypt = 0,
761 .consumed = 0,
762 .produced = 0,
763 .pad_descs = NULL,
764 .active = 0,
765 .done = 0,
766 .tcfg = NULL,
767 .prev_src = NULL,
768 .curr_src = NULL,
769 .unit_no = src_csum};
770 struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
771
772 unsigned int indata_ix = 0;
773
774 /* iovec accounting. */
775 int iniov_ix = 0;
776 int iniov_offset = 0;
777
778 /* Operation descriptor cfg traversal pointer. */
779 struct cryptocop_desc *odsc;
780
781 int failed = 0;
782 /* List heads for allocated descriptors. */
783 struct cryptocop_dma_desc out_cdesc_head = {0};
784 struct cryptocop_dma_desc in_cdesc_head = {0};
785
786 struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
787 struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
788
789 struct cryptocop_tfrm_ctx *output_tc = NULL;
790 void *iop_alloc_ptr;
791
792 assert(operation != NULL);
793 assert(int_op != NULL);
794
795 DEBUG(printk("cryptocop_setup_dma_list: start\n"));
796 DEBUG(print_cryptocop_operation(operation));
797
798 sess = get_session(operation->sid);
799 if (!sess) {
800 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
801 failed = -EINVAL;
802 goto error_cleanup;
803 }
804 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
805 if (!iop_alloc_ptr) {
806 DEBUG_API(printk("cryptocop_setup_dma_list: kmalloc cryptocop_int_operation\n"));
807 failed = -ENOMEM;
808 goto error_cleanup;
809 }
810 (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
811 DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
812 (*int_op)->alloc_ptr = iop_alloc_ptr;
813 DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
814
815 (*int_op)->sid = operation->sid;
816 (*int_op)->cdesc_out = NULL;
817 (*int_op)->cdesc_in = NULL;
818 (*int_op)->tdes_mode = cryptocop_3des_ede;
819 (*int_op)->csum_mode = cryptocop_csum_le;
820 (*int_op)->ddesc_out = NULL;
821 (*int_op)->ddesc_in = NULL;
822
823 /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
824 if (!tcfg) {
825 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
826 failed = -EINVAL;
827 goto error_cleanup;
828 }
829 while (tcfg) {
830 tctx = get_transform_ctx(sess, tcfg->tid);
831 if (!tctx) {
832 DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
833 failed = -EINVAL;
834 goto error_cleanup;
835 }
836 if (tcfg->inject_ix > operation->tfrm_op.outlen){
837 DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
838 failed = -EINVAL;
839 goto error_cleanup;
840 }
841 switch (tctx->init.alg){
842 case cryptocop_alg_mem2mem:
843 if (cipher_ctx.tcfg != NULL){
844 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
845 failed = -EINVAL;
846 goto error_cleanup;
847 }
848 /* mem2mem is handled as a NULL cipher. */
849 cipher_ctx.cbcmode = 0;
850 cipher_ctx.decrypt = 0;
851 cipher_ctx.blocklength = 1;
852 cipher_ctx.ciph_conf = 0;
853 cipher_ctx.unit_no = src_dma;
854 cipher_ctx.tcfg = tcfg;
855 cipher_ctx.tctx = tctx;
856 break;
857 case cryptocop_alg_des:
858 case cryptocop_alg_3des:
859 case cryptocop_alg_aes:
860 /* cipher */
861 if (cipher_ctx.tcfg != NULL){
862 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
863 failed = -EINVAL;
864 goto error_cleanup;
865 }
866 cipher_ctx.tcfg = tcfg;
867 cipher_ctx.tctx = tctx;
868 if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
869 cipher_ctx.decrypt = 1;
870 }
871 switch (tctx->init.cipher_mode) {
872 case cryptocop_cipher_mode_ecb:
873 cipher_ctx.cbcmode = 0;
874 break;
875 case cryptocop_cipher_mode_cbc:
876 cipher_ctx.cbcmode = 1;
877 break;
878 default:
879 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
880 failed = -EINVAL;
881 goto error_cleanup;
882 }
883 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
884 switch (tctx->init.alg){
885 case cryptocop_alg_des:
886 cipher_ctx.ciph_conf = 0;
887 cipher_ctx.unit_no = src_des;
888 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
889 break;
890 case cryptocop_alg_3des:
891 cipher_ctx.ciph_conf = 1;
892 cipher_ctx.unit_no = src_des;
893 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
894 break;
895 case cryptocop_alg_aes:
896 cipher_ctx.ciph_conf = 2;
897 cipher_ctx.unit_no = src_aes;
898 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
899 break;
900 default:
901 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
902 }
903 (*int_op)->tdes_mode = tctx->init.tdes_mode;
904 break;
905 case cryptocop_alg_md5:
906 case cryptocop_alg_sha1:
907 /* digest */
908 if (digest_ctx.tcfg != NULL){
909 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
910 failed = -EINVAL;
911 goto error_cleanup;
912 }
913 digest_ctx.tcfg = tcfg;
914 digest_ctx.tctx = tctx;
915 digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
916 switch (tctx->init.alg){
917 case cryptocop_alg_md5:
918 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
919 digest_ctx.unit_no = src_md5;
920 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
921 break;
922 case cryptocop_alg_sha1:
923 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
924 digest_ctx.unit_no = src_sha1;
925 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
926 break;
927 default:
928 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
929 }
930 break;
931 case cryptocop_alg_csum:
932 /* digest */
933 if (csum_ctx.tcfg != NULL){
934 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
935 failed = -EINVAL;
936 goto error_cleanup;
937 }
938 (*int_op)->csum_mode = tctx->init.csum_mode;
939 csum_ctx.tcfg = tcfg;
940 csum_ctx.tctx = tctx;
941 break;
942 default:
943 /* no algorithm. */
944 DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
945 failed = -EINVAL;
946 goto error_cleanup;
947 }
948 tcfg = tcfg->next;
949 }
950 /* Download key if a cipher is used. */
951 if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
952 struct cryptocop_dma_desc *key_desc = NULL;
953
954 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
955 if (failed) {
956 DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
957 goto error_cleanup;
958 }
959 current_out_cdesc->next = key_desc;
960 current_out_cdesc = key_desc;
961 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
962
963 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
964 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
965 struct cryptocop_dma_desc *iv_desc = NULL;
966
967 DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
968
969 failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
970 if (failed) {
971 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
972 goto error_cleanup;
973 }
974 current_out_cdesc->next = iv_desc;
975 current_out_cdesc = iv_desc;
976 indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
977 }
978 }
979
980 /* Process descriptors. */
981 odsc = operation->tfrm_op.desc;
982 while (odsc) {
983 struct cryptocop_desc_cfg *dcfg = odsc->cfg;
984 struct strcop_meta_out meta_out = {0};
985 size_t desc_len = odsc->length;
986 int active_count, eop_needed_count;
987
988 output_tc = NULL;
989
990 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
991
992 while (dcfg) {
993 struct cryptocop_tfrm_ctx *tc = NULL;
994
995 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
996 /* Get the local context for the transform and mark it as the output unit if it produces output. */
997 if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
998 tc = &digest_ctx;
999 } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1000 tc = &cipher_ctx;
1001 } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1002 tc = &csum_ctx;
1003 }
1004 if (!tc) {
1005 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1006 failed = -EINVAL;
1007 goto error_cleanup;
1008 }
1009 if (tc->done) {
1010 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1011 failed = -EINVAL;
1012 goto error_cleanup;
1013 }
1014 if (!tc->active) {
1015 tc->start_ix = indata_ix;
1016 tc->active = 1;
1017 }
1018
1019 tc->previous_src = tc->current_src;
1020 tc->prev_src = tc->curr_src;
1021 /* Map source unit id to DMA source config. */
1022 switch (dcfg->src){
1023 case cryptocop_source_dma:
1024 tc->current_src = src_dma;
1025 break;
1026 case cryptocop_source_des:
1027 tc->current_src = src_des;
1028 break;
1029 case cryptocop_source_3des:
1030 tc->current_src = src_des;
1031 break;
1032 case cryptocop_source_aes:
1033 tc->current_src = src_aes;
1034 break;
1035 case cryptocop_source_md5:
1036 case cryptocop_source_sha1:
1037 case cryptocop_source_csum:
1038 case cryptocop_source_none:
1039 default:
1040 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1041 */
1042 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1043 failed = -EINVAL;
1044 goto error_cleanup;
1045 }
1046 if (tc->current_src != src_dma) {
1047 /* Find the unit we are sourcing from. */
1048 if (digest_ctx.unit_no == tc->current_src){
1049 tc->curr_src = &digest_ctx;
1050 } else if (cipher_ctx.unit_no == tc->current_src){
1051 tc->curr_src = &cipher_ctx;
1052 } else if (csum_ctx.unit_no == tc->current_src){
1053 tc->curr_src = &csum_ctx;
1054 }
1055 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1056 DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1057 failed = -EINVAL;
1058 goto error_cleanup;
1059 }
1060 } else {
1061 tc->curr_src = NULL;
1062 }
1063
1064 /* Detect source switch. */
1065 DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1066 if (tc->active && (tc->current_src != tc->previous_src)) {
1067 /* Only allow source switch when both the old source unit and the new one have
1068 * no pending data to process (i.e. the consumed length must be a multiple of the
1069 * transform blocklength). */
1070 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1071 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1072 ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1073 {
1074 DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1075 failed = -EINVAL;
1076 goto error_cleanup;
1077 }
1078 }
1079 /* Detect unit deactivation. */
1080 if (dcfg->last) {
1081 /* Length check of this is handled below. */
1082 tc->done = 1;
1083 }
1084 dcfg = dcfg->next;
1085 } /* while (dcfg) */
1086 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1087
1088 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1089 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1090 failed = -EINVAL;
1091 goto error_cleanup;
1092 }
1093 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1094 DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1095 failed = -EINVAL;
1096 goto error_cleanup;
1097 }
1098 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1099 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1100 failed = -EINVAL;
1101 goto error_cleanup;
1102 }
1103
1104 /* Update consumed and produced lengths.
1105
1106 The consumed length accounting here is actually cheating. If a unit source from DMA (or any
1107 other unit that process data in blocks of one octet) it is correct, but if it source from a
1108 block processing unit, i.e. a cipher, it will be temporarily incorrect at some times. However
1109 since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1110 unit has processed an exact multiple of its block length the end result will be correct.
1111 Beware that if the source change restriction change this code will need to be (much) reworked.
1112 */
1113 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1114
1115 if (csum_ctx.active) {
1116 csum_ctx.consumed += desc_len;
1117 if (csum_ctx.done) {
1118 csum_ctx.produced = 2;
1119 }
1120 DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1121 }
1122 if (digest_ctx.active) {
1123 digest_ctx.consumed += desc_len;
1124 if (digest_ctx.done) {
1125 if (digest_ctx.unit_no == src_md5) {
1126 digest_ctx.produced = MD5_STATE_LENGTH;
1127 } else {
1128 digest_ctx.produced = SHA1_STATE_LENGTH;
1129 }
1130 }
1131 DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1132 }
1133 if (cipher_ctx.active) {
1134 /* Ciphers are allowed only to source from DMA out. That is filtered above. */
1135 assert(cipher_ctx.current_src == src_dma);
1136 cipher_ctx.consumed += desc_len;
1137 cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1138 if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1139 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1140 }
1141 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1142 }
1143
1144 /* Setup the DMA out descriptors. */
1145 /* Configure the metadata. */
1146 active_count = 0;
1147 eop_needed_count = 0;
1148 if (cipher_ctx.active) {
1149 ++active_count;
1150 if (cipher_ctx.unit_no == src_dma){
1151 /* mem2mem */
1152 meta_out.ciphsel = src_none;
1153 } else {
1154 meta_out.ciphsel = cipher_ctx.current_src;
1155 }
1156 meta_out.ciphconf = cipher_ctx.ciph_conf;
1157 meta_out.cbcmode = cipher_ctx.cbcmode;
1158 meta_out.decrypt = cipher_ctx.decrypt;
1159 DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1160 if (cipher_ctx.done) ++eop_needed_count;
1161 } else {
1162 meta_out.ciphsel = src_none;
1163 }
1164
1165 if (digest_ctx.active) {
1166 ++active_count;
1167 meta_out.hashsel = digest_ctx.current_src;
1168 meta_out.hashconf = digest_ctx.hash_conf;
1169 meta_out.hashmode = 0; /* Explicit mode is not used here. */
1170 DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1171 if (digest_ctx.done) {
1172 assert(digest_ctx.pad_descs == NULL);
1173 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1174 if (failed) {
1175 DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1176 goto error_cleanup;
1177 }
1178 }
1179 } else {
1180 meta_out.hashsel = src_none;
1181 }
1182
1183 if (csum_ctx.active) {
1184 ++active_count;
1185 meta_out.csumsel = csum_ctx.current_src;
1186 if (csum_ctx.done) {
1187 assert(csum_ctx.pad_descs == NULL);
1188 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1189 if (failed) {
1190 DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1191 goto error_cleanup;
1192 }
1193 }
1194 } else {
1195 meta_out.csumsel = src_none;
1196 }
1197 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1198 /* Setup DMA out descriptors for the indata. */
1199 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1200 if (failed) {
1201 DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1202 goto error_cleanup;
1203 }
1204 /* Setup out EOP. If there are active units that are not done here they cannot get an EOP
1205 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1206 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1207 */
1208 assert(active_count >= eop_needed_count);
1209 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1210 if (eop_needed_count) {
1211 /* This means that the bulk operation (cipeher/m2m) is terminated. */
1212 if (active_count > 1) {
1213 /* Use zero length EOP descriptor. */
1214 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1215 struct strcop_meta_out ed_mo = {0};
1216 if (!ed) {
1217 DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1218 failed = -ENOMEM;
1219 goto error_cleanup;
1220 }
1221
1222 assert(cipher_ctx.active && cipher_ctx.done);
1223
1224 if (cipher_ctx.unit_no == src_dma){
1225 /* mem2mem */
1226 ed_mo.ciphsel = src_none;
1227 } else {
1228 ed_mo.ciphsel = cipher_ctx.current_src;
1229 }
1230 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1231 ed_mo.cbcmode = cipher_ctx.cbcmode;
1232 ed_mo.decrypt = cipher_ctx.decrypt;
1233
1234 ed->free_buf = NULL;
1235 ed->dma_descr->wait = 1;
1236 ed->dma_descr->out_eop = 1;
1237
1238 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1239 ed->dma_descr->after = ed->dma_descr->buf;
1240 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1241 current_out_cdesc->next = ed;
1242 current_out_cdesc = ed;
1243 } else {
1244 /* Set EOP in the current out descriptor since the only active module is
1245 * the one needing the EOP. */
1246
1247 current_out_cdesc->dma_descr->out_eop = 1;
1248 }
1249 }
1250
1251 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1252 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1253 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1254 indata_ix += odsc->length;
1255 odsc = odsc->next;
1256 } /* while (odsc) */ /* Process descriptors. */
1257 DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1258 if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1259 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1260 failed = -EINVAL;
1261 goto error_cleanup;
1262 }
1263 if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1264 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1265 failed = -EINVAL;
1266 goto error_cleanup;
1267 }
1268 if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1269 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1270 failed = -EINVAL;
1271 goto error_cleanup;
1272 }
1273
1274 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1275 if (failed){
1276 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1277 goto error_cleanup;
1278 }
1279 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1280 if (failed){
1281 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1282 goto error_cleanup;
1283 }
1284 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1285 if (failed){
1286 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1287 goto error_cleanup;
1288 }
1289
1290 DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1291 (*int_op)->cdesc_out = out_cdesc_head.next;
1292 (*int_op)->cdesc_in = in_cdesc_head.next;
1293 DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1294
1295 setup_descr_chain(out_cdesc_head.next);
1296 setup_descr_chain(in_cdesc_head.next);
1297
1298 /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1299 * last DMA out descriptor for EOL.
1300 */
1301 current_in_cdesc->dma_descr->intr = 1;
1302 current_in_cdesc->dma_descr->eol = 1;
1303 current_out_cdesc->dma_descr->eol = 1;
1304
1305 /* Setup DMA contexts. */
1306 (*int_op)->ctx_out.next = NULL;
1307 (*int_op)->ctx_out.eol = 1;
1308 (*int_op)->ctx_out.intr = 0;
1309 (*int_op)->ctx_out.store_mode = 0;
1310 (*int_op)->ctx_out.en = 0;
1311 (*int_op)->ctx_out.dis = 0;
1312 (*int_op)->ctx_out.md0 = 0;
1313 (*int_op)->ctx_out.md1 = 0;
1314 (*int_op)->ctx_out.md2 = 0;
1315 (*int_op)->ctx_out.md3 = 0;
1316 (*int_op)->ctx_out.md4 = 0;
1317 (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1318 (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1319
1320 (*int_op)->ctx_in.next = NULL;
1321 (*int_op)->ctx_in.eol = 1;
1322 (*int_op)->ctx_in.intr = 0;
1323 (*int_op)->ctx_in.store_mode = 0;
1324 (*int_op)->ctx_in.en = 0;
1325 (*int_op)->ctx_in.dis = 0;
1326 (*int_op)->ctx_in.md0 = 0;
1327 (*int_op)->ctx_in.md1 = 0;
1328 (*int_op)->ctx_in.md2 = 0;
1329 (*int_op)->ctx_in.md3 = 0;
1330 (*int_op)->ctx_in.md4 = 0;
1331
1332 (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1333 (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1334
1335 DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1336 return 0;
1337
1338 error_cleanup:
1339 {
1340 /* Free all allocated resources. */
1341 struct cryptocop_dma_desc *tmp_cdesc;
1342 while (digest_ctx.pad_descs){
1343 tmp_cdesc = digest_ctx.pad_descs->next;
1344 free_cdesc(digest_ctx.pad_descs);
1345 digest_ctx.pad_descs = tmp_cdesc;
1346 }
1347 while (csum_ctx.pad_descs){
1348 tmp_cdesc = csum_ctx.pad_descs->next;
1349 free_cdesc(csum_ctx.pad_descs);
1350 csum_ctx.pad_descs = tmp_cdesc;
1351 }
1352 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1353
1354 if (*int_op != NULL) delete_internal_operation(*int_op);
1355 }
1356 DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1357 return failed;
1358 }
1359
1360
1361 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1362 {
1363 void *ptr = iop->alloc_ptr;
1364 struct cryptocop_dma_desc *cd = iop->cdesc_out;
1365 struct cryptocop_dma_desc *next;
1366
1367 DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1368
1369 while (cd) {
1370 next = cd->next;
1371 free_cdesc(cd);
1372 cd = next;
1373 }
1374 cd = iop->cdesc_in;
1375 while (cd) {
1376 next = cd->next;
1377 free_cdesc(cd);
1378 cd = next;
1379 }
1380 kfree(ptr);
1381 }
1382
1383 #define MD5_MIN_PAD_LENGTH (9)
1384 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1385
1386 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1387 {
1388 size_t padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1389 unsigned char *p;
1390 int i;
1391 unsigned long long int bit_length = hashed_length << 3;
1392
1393 if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1394
1395 p = kmalloc(padlen, alloc_flag);
1396 if (!p) return -ENOMEM;
1397
1398 *p = 0x80;
1399 memset(p+1, 0, padlen - 1);
1400
1401 DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1402
1403 i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1404 while (bit_length != 0){
1405 p[i++] = bit_length % 0x100;
1406 bit_length >>= 8;
1407 }
1408
1409 *pad = (char*)p;
1410 *pad_length = padlen;
1411
1412 return 0;
1413 }
1414
1415 #define SHA1_MIN_PAD_LENGTH (9)
1416 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1417
1418 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1419 {
1420 size_t padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1421 unsigned char *p;
1422 int i;
1423 unsigned long long int bit_length = hashed_length << 3;
1424
1425 if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1426
1427 p = kmalloc(padlen, alloc_flag);
1428 if (!p) return -ENOMEM;
1429
1430 *p = 0x80;
1431 memset(p+1, 0, padlen - 1);
1432
1433 DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1434
1435 i = padlen - 1;
1436 while (bit_length != 0){
1437 p[i--] = bit_length % 0x100;
1438 bit_length >>= 8;
1439 }
1440
1441 *pad = (char*)p;
1442 *pad_length = padlen;
1443
1444 return 0;
1445 }
1446
1447
1448 static int transform_ok(struct cryptocop_transform_init *tinit)
1449 {
1450 switch (tinit->alg){
1451 case cryptocop_alg_csum:
1452 switch (tinit->csum_mode){
1453 case cryptocop_csum_le:
1454 case cryptocop_csum_be:
1455 break;
1456 default:
1457 DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1458 return -EINVAL;
1459 }
1460 case cryptocop_alg_mem2mem:
1461 case cryptocop_alg_md5:
1462 case cryptocop_alg_sha1:
1463 if (tinit->keylen != 0) {
1464 DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1465 return -EINVAL; /* This check is a bit strict. */
1466 }
1467 break;
1468 case cryptocop_alg_des:
1469 if (tinit->keylen != 64) {
1470 DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1471 return -EINVAL;
1472 }
1473 break;
1474 case cryptocop_alg_3des:
1475 if (tinit->keylen != 192) {
1476 DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1477 return -EINVAL;
1478 }
1479 break;
1480 case cryptocop_alg_aes:
1481 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1482 DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1483 return -EINVAL;
1484 }
1485 break;
1486 case cryptocop_no_alg:
1487 default:
1488 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1489 return -EINVAL;
1490 }
1491
1492 switch (tinit->alg){
1493 case cryptocop_alg_des:
1494 case cryptocop_alg_3des:
1495 case cryptocop_alg_aes:
1496 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1497 default:
1498 break;
1499 }
1500 return 0;
1501 }
1502
1503
1504 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1505 {
1506 struct cryptocop_session *sess;
1507 struct cryptocop_transform_init *tfrm_in = tinit;
1508 struct cryptocop_transform_init *tmp_in;
1509 int no_tfrms = 0;
1510 int i;
1511 unsigned long int flags;
1512
1513 init_stream_coprocessor(); /* For safety if we are called early */
1514
1515 while (tfrm_in){
1516 int err;
1517 ++no_tfrms;
1518 if ((err = transform_ok(tfrm_in))) {
1519 DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1520 return err;
1521 }
1522 tfrm_in = tfrm_in->next;
1523 }
1524 if (0 == no_tfrms) {
1525 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1526 return -EINVAL;
1527 }
1528
1529 sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1530 if (!sess){
1531 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1532 return -ENOMEM;
1533 }
1534
1535 sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1536 if (!sess->tfrm_ctx) {
1537 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1538 kfree(sess);
1539 return -ENOMEM;
1540 }
1541
1542 tfrm_in = tinit;
1543 for (i = 0; i < no_tfrms; i++){
1544 tmp_in = tfrm_in->next;
1545 while (tmp_in){
1546 if (tmp_in->tid == tfrm_in->tid) {
1547 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1548 kfree(sess->tfrm_ctx);
1549 kfree(sess);
1550 return -EINVAL;
1551 }
1552 tmp_in = tmp_in->next;
1553 }
1554 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1555 sess->tfrm_ctx[i].dec_key_set = 0;
1556 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1557
1558 tfrm_in = tfrm_in->next;
1559 }
1560 sess->tfrm_ctx[i-1].next = NULL;
1561
1562 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1563 sess->sid = next_sid;
1564 next_sid++;
1565 /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1566 * OTOH 2^64 is a really large number of session. */
1567 if (next_sid == 0) next_sid = 1;
1568
1569 /* Prepend to session list. */
1570 sess->next = cryptocop_sessions;
1571 cryptocop_sessions = sess;
1572 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1573 *sid = sess->sid;
1574 return 0;
1575 }
1576
1577
1578 int cryptocop_free_session(cryptocop_session_id sid)
1579 {
1580 struct cryptocop_transform_ctx *tc;
1581 struct cryptocop_session *sess = NULL;
1582 struct cryptocop_session *psess = NULL;
1583 unsigned long int flags;
1584 int i;
1585 LIST_HEAD(remove_list);
1586 struct list_head *node, *tmp;
1587 struct cryptocop_prio_job *pj;
1588
1589 DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1590
1591 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1592 sess = cryptocop_sessions;
1593 while (sess && sess->sid != sid){
1594 psess = sess;
1595 sess = sess->next;
1596 }
1597 if (sess){
1598 if (psess){
1599 psess->next = sess->next;
1600 } else {
1601 cryptocop_sessions = sess->next;
1602 }
1603 }
1604 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1605
1606 if (!sess) return -EINVAL;
1607
1608 /* Remove queued jobs. */
1609 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1610
1611 for (i = 0; i < cryptocop_prio_no_prios; i++){
1612 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1613 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1614 pj = list_entry(node, struct cryptocop_prio_job, node);
1615 if (pj->oper->sid == sid) {
1616 list_move_tail(node, &remove_list);
1617 }
1618 }
1619 }
1620 }
1621 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1622
1623 list_for_each_safe(node, tmp, &remove_list) {
1624 list_del(node);
1625 pj = list_entry(node, struct cryptocop_prio_job, node);
1626 pj->oper->operation_status = -EAGAIN; /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1627 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1628 pj->oper->cb(pj->oper, pj->oper->cb_data);
1629 delete_internal_operation(pj->iop);
1630 kfree(pj);
1631 }
1632
1633 tc = sess->tfrm_ctx;
1634 /* Erase keying data. */
1635 while (tc){
1636 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1637 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1638 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1639 tc = tc->next;
1640 }
1641 kfree(sess->tfrm_ctx);
1642 kfree(sess);
1643
1644 return 0;
1645 }
1646
1647 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1648 {
1649 struct cryptocop_session *sess;
1650 unsigned long int flags;
1651
1652 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1653 sess = cryptocop_sessions;
1654 while (sess && (sess->sid != sid)){
1655 sess = sess->next;
1656 }
1657 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1658
1659 return sess;
1660 }
1661
1662 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1663 {
1664 struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1665
1666 DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1667 assert(sess != NULL);
1668 while (tc && tc->init.tid != tid){
1669 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1670 tc = tc->next;
1671 }
1672 DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1673 return tc;
1674 }
1675
1676
1677
1678 /* The AES s-transform matrix (s-box). */
1679 static const u8 aes_sbox[256] = {
1680 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, 171, 118,
1681 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, 156, 164, 114, 192,
1682 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, 229, 241, 113, 216, 49, 21,
1683 4, 199, 35, 195, 24, 150, 5, 154, 7, 18, 128, 226, 235, 39, 178, 117,
1684 9, 131, 44, 26, 27, 110, 90, 160, 82, 59, 214, 179, 41, 227, 47, 132,
1685 83, 209, 0, 237, 32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207,
1686 208, 239, 170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168,
1687 81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243, 210,
1688 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100, 93, 25, 115,
1689 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184, 20, 222, 94, 11, 219,
1690 224, 50, 58, 10, 73, 6, 36, 92, 194, 211, 172, 98, 145, 149, 228, 121,
1691 231, 200, 55, 109, 141, 213, 78, 169, 108, 86, 244, 234, 101, 122, 174, 8,
1692 186, 120, 37, 46, 28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138,
1693 112, 62, 181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158,
1694 225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223,
1695 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84, 187, 22
1696 };
1697
1698 /* AES has a 32 bit word round constants for each round in the
1699 * key schedule. round_constant[i] is really Rcon[i+1] in FIPS187.
1700 */
1701 static u32 round_constant[11] = {
1702 0x01000000, 0x02000000, 0x04000000, 0x08000000,
1703 0x10000000, 0x20000000, 0x40000000, 0x80000000,
1704 0x1B000000, 0x36000000, 0x6C000000
1705 };
1706
1707 /* Apply the s-box to each of the four occtets in w. */
1708 static u32 aes_ks_subword(const u32 w)
1709 {
1710 u8 bytes[4];
1711
1712 *(u32*)(&bytes[0]) = w;
1713 bytes[0] = aes_sbox[bytes[0]];
1714 bytes[1] = aes_sbox[bytes[1]];
1715 bytes[2] = aes_sbox[bytes[2]];
1716 bytes[3] = aes_sbox[bytes[3]];
1717 return *(u32*)(&bytes[0]);
1718 }
1719
1720 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1721 * (Note that AES words are 32 bit long)
1722 *
1723 * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1724 * word temp
1725 * i = 0
1726 * while (i < Nk) {
1727 * w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1728 * i = i + 1
1729 * }
1730 * i = Nk
1731 *
1732 * while (i < (Nb * (Nr + 1))) {
1733 * temp = w[i - 1]
1734 * if ((i mod Nk) == 0) {
1735 * temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1736 * }
1737 * else if ((Nk > 6) && ((i mod Nk) == 4)) {
1738 * temp = SubWord(temp)
1739 * }
1740 * w[i] = w[i - Nk] xor temp
1741 * }
1742 * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1743 * SubWord(t) applies the AES s-box individually to each octet
1744 * in a 32 bit word.
1745 *
1746 * For AES Nk can have the values 4, 6, and 8 (corresponding to
1747 * values for Nr of 10, 12, and 14). Nb is always 4.
1748 *
1749 * To construct w[i], w[i - 1] and w[i - Nk] must be
1750 * available. Consequently we must keep a state of the last Nk words
1751 * to be able to create the last round keys.
1752 */
1753 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength)
1754 {
1755 u32 temp;
1756 u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1757 u8 w_last_ix;
1758 int i;
1759 u8 nr, nk;
1760
1761 switch (keylength){
1762 case 128:
1763 nk = 4;
1764 nr = 10;
1765 break;
1766 case 192:
1767 nk = 6;
1768 nr = 12;
1769 break;
1770 case 256:
1771 nk = 8;
1772 nr = 14;
1773 break;
1774 default:
1775 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1776 };
1777
1778 /* Need to do host byte order correction here since key is byte oriented and the
1779 * kx algorithm is word (u32) oriented. */
1780 for (i = 0; i < nk; i+=1) {
1781 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1782 }
1783
1784 i = (int)nk;
1785 w_last_ix = i - 1;
1786 while (i < (4 * (nr + 2))) {
1787 temp = w_ring[w_last_ix];
1788 if (!(i % nk)) {
1789 /* RotWord(temp) */
1790 temp = (temp << 8) | (temp >> 24);
1791 temp = aes_ks_subword(temp);
1792 temp ^= round_constant[i/nk - 1];
1793 } else if ((nk > 6) && ((i % nk) == 4)) {
1794 temp = aes_ks_subword(temp);
1795 }
1796 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1797 temp ^= w_ring[w_last_ix];
1798 w_ring[w_last_ix] = temp;
1799
1800 /* We need the round keys for round Nr+1 and Nr+2 (round key
1801 * Nr+2 is the round key beyond the last one used when
1802 * encrypting). Rounds are numbered starting from 0, Nr=10
1803 * implies 11 rounds are used in encryption/decryption.
1804 */
1805 if (i >= (4 * nr)) {
1806 /* Need to do host byte order correction here, the key
1807 * is byte oriented. */
1808 *(u32*)dec_key = cpu_to_be32(temp);
1809 dec_key += 4;
1810 }
1811 ++i;
1812 }
1813 }
1814
1815
1816 /**** Job/operation management. ****/
1817
1818 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1819 {
1820 return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1821 }
1822
1823 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1824 {
1825 return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1826 }
1827
1828 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1829 {
1830 return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1831 }
1832
1833 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1834 {
1835 int ret;
1836 struct cryptocop_prio_job *pj = NULL;
1837 unsigned long int flags;
1838
1839 DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1840
1841 if (!operation || !operation->cb){
1842 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1843 return -EINVAL;
1844 }
1845
1846 if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1847 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1848 return ret;
1849 }
1850 assert(pj != NULL);
1851
1852 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1853 list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1854 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1855
1856 /* Make sure a job is running */
1857 cryptocop_start_job();
1858 return 0;
1859 }
1860
1861 static void cryptocop_do_tasklet(unsigned long unused);
1862 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1863
1864 static void cryptocop_do_tasklet(unsigned long unused)
1865 {
1866 struct list_head *node;
1867 struct cryptocop_prio_job *pj = NULL;
1868 unsigned long flags;
1869
1870 DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1871
1872 do {
1873 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1874 if (!list_empty(&cryptocop_completed_jobs)){
1875 node = cryptocop_completed_jobs.next;
1876 list_del(node);
1877 pj = list_entry(node, struct cryptocop_prio_job, node);
1878 } else {
1879 pj = NULL;
1880 }
1881 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1882 if (pj) {
1883 assert(pj->oper != NULL);
1884
1885 /* Notify consumer of operation completeness. */
1886 DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1887
1888 pj->oper->operation_status = 0; /* Job is completed. */
1889 pj->oper->cb(pj->oper, pj->oper->cb_data);
1890 delete_internal_operation(pj->iop);
1891 kfree(pj);
1892 }
1893 } while (pj != NULL);
1894
1895 DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1896 }
1897
1898 static irqreturn_t
1899 dma_done_interrupt(int irq, void *dev_id)
1900 {
1901 struct cryptocop_prio_job *done_job;
1902 reg_dma_rw_ack_intr ack_intr = {
1903 .data = 1,
1904 };
1905
1906 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1907
1908 DEBUG(printk("cryptocop DMA done\n"));
1909
1910 spin_lock(&running_job_lock);
1911 if (cryptocop_running_job == NULL){
1912 printk("stream co-processor got interrupt when not busy\n");
1913 spin_unlock(&running_job_lock);
1914 return IRQ_HANDLED;
1915 }
1916 done_job = cryptocop_running_job;
1917 cryptocop_running_job = NULL;
1918 spin_unlock(&running_job_lock);
1919
1920 /* Start processing a job. */
1921 if (!spin_trylock(&cryptocop_process_lock)){
1922 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1923 } else {
1924 cryptocop_start_job();
1925 spin_unlock(&cryptocop_process_lock);
1926 }
1927
1928 done_job->oper->operation_status = 0; /* Job is completed. */
1929 if (done_job->oper->fast_callback){
1930 /* This operation wants callback from interrupt. */
1931 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1932 delete_internal_operation(done_job->iop);
1933 kfree(done_job);
1934 } else {
1935 spin_lock(&cryptocop_completed_jobs_lock);
1936 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1937 spin_unlock(&cryptocop_completed_jobs_lock);
1938 tasklet_schedule(&cryptocop_tasklet);
1939 }
1940
1941 DEBUG(printk("cryptocop leave irq handler\n"));
1942 return IRQ_HANDLED;
1943 }
1944
1945
1946 /* Setup interrupts and DMA channels. */
1947 static int init_cryptocop(void)
1948 {
1949 unsigned long flags;
1950 reg_dma_rw_cfg dma_cfg = {.en = 1};
1951 reg_dma_rw_intr_mask intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1952 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
1953 reg_strcop_rw_cfg strcop_cfg = {
1954 .ipend = regk_strcop_little,
1955 .td1 = regk_strcop_e,
1956 .td2 = regk_strcop_d,
1957 .td3 = regk_strcop_e,
1958 .ignore_sync = 0,
1959 .en = 1
1960 };
1961
1962 if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1963 "stream co-processor DMA", NULL))
1964 panic("request_irq stream co-processor irq dma9");
1965
1966 (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1967 0, dma_strp);
1968 (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969 0, dma_strp);
1970
1971 local_irq_save(flags);
1972
1973 /* Reset and enable the cryptocop. */
1974 strcop_cfg.en = 0;
1975 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1976 strcop_cfg.en = 1;
1977 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978
1979 /* Enable DMAs. */
1980 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1981 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1982
1983 /* Set up wordsize = 4 for DMAs. */
1984 DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1985 DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1986
1987 /* Enable interrupts. */
1988 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1989
1990 /* Clear intr ack. */
1991 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1992
1993 local_irq_restore(flags);
1994
1995 return 0;
1996 }
1997
1998 /* Free used cryptocop hw resources (interrupt and DMA channels). */
1999 static void release_cryptocop(void)
2000 {
2001 unsigned long flags;
2002 reg_dma_rw_cfg dma_cfg = {.en = 0};
2003 reg_dma_rw_intr_mask intr_mask_in = {0};
2004 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
2005
2006 local_irq_save(flags);
2007
2008 /* Clear intr ack. */
2009 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2010
2011 /* Disable DMAs. */
2012 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2013 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2014
2015 /* Disable interrupts. */
2016 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2017
2018 local_irq_restore(flags);
2019
2020 free_irq(DMA_IRQ, NULL);
2021
2022 (void)crisv32_free_dma(OUT_DMA);
2023 (void)crisv32_free_dma(IN_DMA);
2024 }
2025
2026
2027 /* Init job queue. */
2028 static int cryptocop_job_queue_init(void)
2029 {
2030 int i;
2031
2032 INIT_LIST_HEAD(&cryptocop_completed_jobs);
2033
2034 for (i = 0; i < cryptocop_prio_no_prios; i++){
2035 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2036 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2037 }
2038 return 0;
2039 }
2040
2041
2042 static void cryptocop_job_queue_close(void)
2043 {
2044 struct list_head *node, *tmp;
2045 struct cryptocop_prio_job *pj = NULL;
2046 unsigned long int process_flags, flags;
2047 int i;
2048
2049
2050 /* Stop strcop from getting an operation to process while we are closing the
2051 module. */
2052 spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2053
2054 /* Empty the job queue. */
2055 for (i = 0; i < cryptocop_prio_no_prios; i++){
2056 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2057 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2058 pj = list_entry(node, struct cryptocop_prio_job, node);
2059 list_del(node);
2060
2061 /* Call callback to notify consumer of job removal. */
2062 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2063 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2064 pj->oper->cb(pj->oper, pj->oper->cb_data);
2065
2066 delete_internal_operation(pj->iop);
2067 kfree(pj);
2068 }
2069 }
2070 }
2071 spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2072
2073 /* Remove the running job, if any. */
2074 spin_lock_irqsave(&running_job_lock, flags);
2075 if (cryptocop_running_job){
2076 reg_strcop_rw_cfg rw_cfg;
2077 reg_dma_rw_cfg dma_out_cfg, dma_in_cfg;
2078
2079 /* Stop DMA. */
2080 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2081 dma_out_cfg.en = regk_dma_no;
2082 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2083
2084 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2085 dma_in_cfg.en = regk_dma_no;
2086 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2087
2088 /* Disble the cryptocop. */
2089 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2090 rw_cfg.en = 0;
2091 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2092
2093 pj = cryptocop_running_job;
2094 cryptocop_running_job = NULL;
2095
2096 /* Call callback to notify consumer of job removal. */
2097 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2098 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2099 pj->oper->cb(pj->oper, pj->oper->cb_data);
2100
2101 delete_internal_operation(pj->iop);
2102 kfree(pj);
2103 }
2104 spin_unlock_irqrestore(&running_job_lock, flags);
2105
2106 /* Remove completed jobs, if any. */
2107 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2108
2109 list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2110 pj = list_entry(node, struct cryptocop_prio_job, node);
2111 list_del(node);
2112 /* Call callback to notify consumer of job removal. */
2113 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2114 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2115 pj->oper->cb(pj->oper, pj->oper->cb_data);
2116
2117 delete_internal_operation(pj->iop);
2118 kfree(pj);
2119 }
2120 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2121 }
2122
2123
2124 static void cryptocop_start_job(void)
2125 {
2126 int i;
2127 struct cryptocop_prio_job *pj;
2128 unsigned long int flags;
2129 unsigned long int running_job_flags;
2130 reg_strcop_rw_cfg rw_cfg = {.en = 1, .ignore_sync = 0};
2131
2132 DEBUG(printk("cryptocop_start_job: entering\n"));
2133
2134 spin_lock_irqsave(&running_job_lock, running_job_flags);
2135 if (cryptocop_running_job != NULL){
2136 /* Already running. */
2137 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2138 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2139 return;
2140 }
2141 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2142
2143 /* Check the queues in priority order. */
2144 for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2145 if (i == cryptocop_prio_no_prios) {
2146 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2147 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2148 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2149 return; /* No jobs to run */
2150 }
2151 DEBUG(printk("starting job for prio %d\n", i));
2152
2153 /* TODO: Do not starve lower priority jobs. Let in a lower
2154 * prio job for every N-th processed higher prio job or some
2155 * other scheduling policy. This could reasonably be
2156 * tweakable since the optimal balance would depend on the
2157 * type of load on the system. */
2158
2159 /* Pull the DMA lists from the job and start the DMA client. */
2160 pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2161 list_del(&pj->node);
2162 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2163 cryptocop_running_job = pj;
2164
2165 /* Set config register (3DES and CSUM modes). */
2166 switch (pj->iop->tdes_mode){
2167 case cryptocop_3des_eee:
2168 rw_cfg.td1 = regk_strcop_e;
2169 rw_cfg.td2 = regk_strcop_e;
2170 rw_cfg.td3 = regk_strcop_e;
2171 break;
2172 case cryptocop_3des_eed:
2173 rw_cfg.td1 = regk_strcop_e;
2174 rw_cfg.td2 = regk_strcop_e;
2175 rw_cfg.td3 = regk_strcop_d;
2176 break;
2177 case cryptocop_3des_ede:
2178 rw_cfg.td1 = regk_strcop_e;
2179 rw_cfg.td2 = regk_strcop_d;
2180 rw_cfg.td3 = regk_strcop_e;
2181 break;
2182 case cryptocop_3des_edd:
2183 rw_cfg.td1 = regk_strcop_e;
2184 rw_cfg.td2 = regk_strcop_d;
2185 rw_cfg.td3 = regk_strcop_d;
2186 break;
2187 case cryptocop_3des_dee:
2188 rw_cfg.td1 = regk_strcop_d;
2189 rw_cfg.td2 = regk_strcop_e;
2190 rw_cfg.td3 = regk_strcop_e;
2191 break;
2192 case cryptocop_3des_ded:
2193 rw_cfg.td1 = regk_strcop_d;
2194 rw_cfg.td2 = regk_strcop_e;
2195 rw_cfg.td3 = regk_strcop_d;
2196 break;
2197 case cryptocop_3des_dde:
2198 rw_cfg.td1 = regk_strcop_d;
2199 rw_cfg.td2 = regk_strcop_d;
2200 rw_cfg.td3 = regk_strcop_e;
2201 break;
2202 case cryptocop_3des_ddd:
2203 rw_cfg.td1 = regk_strcop_d;
2204 rw_cfg.td2 = regk_strcop_d;
2205 rw_cfg.td3 = regk_strcop_d;
2206 break;
2207 default:
2208 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2209 }
2210 switch (pj->iop->csum_mode){
2211 case cryptocop_csum_le:
2212 rw_cfg.ipend = regk_strcop_little;
2213 break;
2214 case cryptocop_csum_be:
2215 rw_cfg.ipend = regk_strcop_big;
2216 break;
2217 default:
2218 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2219 }
2220 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2221
2222 DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2223 "ctx_in: 0x%p, phys: 0x%p\n"
2224 "ctx_out: 0x%p, phys: 0x%p\n",
2225 pj,
2226 &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2227 &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2228
2229 /* Start input DMA. */
2230 flush_dma_context(&pj->iop->ctx_in);
2231 DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2232
2233 /* Start output DMA. */
2234 DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2235
2236 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2237 DEBUG(printk("cryptocop_start_job: exiting\n"));
2238 }
2239
2240
2241 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2242 {
2243 int err;
2244 int alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2245 void *iop_alloc_ptr = NULL;
2246
2247 *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2248 if (!*pj) return -ENOMEM;
2249
2250 DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2251
2252 (*pj)->oper = operation;
2253 DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n", (*pj)->oper->cb, (*pj)->oper->cb_data));
2254
2255 if (operation->use_dmalists) {
2256 DEBUG(print_user_dma_lists(&operation->list_op));
2257 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2258 DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2259 kfree(*pj);
2260 return -EINVAL;
2261 }
2262 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2263 if (!iop_alloc_ptr) {
2264 DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2265 kfree(*pj);
2266 return -ENOMEM;
2267 }
2268 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2269 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2270 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2271 (*pj)->iop->sid = operation->sid;
2272 (*pj)->iop->cdesc_out = NULL;
2273 (*pj)->iop->cdesc_in = NULL;
2274 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2275 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2276 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2277 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2278
2279 /* Setup DMA contexts. */
2280 (*pj)->iop->ctx_out.next = NULL;
2281 (*pj)->iop->ctx_out.eol = 1;
2282 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2283 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2284
2285 (*pj)->iop->ctx_in.next = NULL;
2286 (*pj)->iop->ctx_in.eol = 1;
2287 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2288 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2289 } else {
2290 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2291 DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2292 kfree(*pj);
2293 return err;
2294 }
2295 }
2296 DEBUG(print_dma_descriptors((*pj)->iop));
2297
2298 DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2299
2300 return 0;
2301 }
2302
2303 static int cryptocop_open(struct inode *inode, struct file *filp)
2304 {
2305 int p = iminor(inode);
2306
2307 if (p != CRYPTOCOP_MINOR) return -EINVAL;
2308
2309 filp->private_data = NULL;
2310 return 0;
2311 }
2312
2313
2314 static int cryptocop_release(struct inode *inode, struct file *filp)
2315 {
2316 struct cryptocop_private *dev = filp->private_data;
2317 struct cryptocop_private *dev_next;
2318
2319 while (dev){
2320 dev_next = dev->next;
2321 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2322 (void)cryptocop_free_session(dev->sid);
2323 }
2324 kfree(dev);
2325 dev = dev_next;
2326 }
2327
2328 return 0;
2329 }
2330
2331
2332 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2333 unsigned int cmd, unsigned long arg)
2334 {
2335 struct cryptocop_private *dev = filp->private_data;
2336 struct cryptocop_private *prev_dev = NULL;
2337 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2338 struct strcop_session_op sop;
2339 int err;
2340
2341 DEBUG(printk("cryptocop_ioctl_close_session\n"));
2342
2343 if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2344 return -EFAULT;
2345 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2346 if (err) return -EFAULT;
2347
2348 while (dev && (dev->sid != sop.ses_id)) {
2349 prev_dev = dev;
2350 dev = dev->next;
2351 }
2352 if (dev){
2353 if (prev_dev){
2354 prev_dev->next = dev->next;
2355 } else {
2356 filp->private_data = dev->next;
2357 }
2358 err = cryptocop_free_session(dev->sid);
2359 if (err) return -EFAULT;
2360 } else {
2361 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2362 return -EINVAL;
2363 }
2364 return 0;
2365 }
2366
2367
2368 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2369 {
2370 struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2371
2372 DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2373
2374 jc->processed = 1;
2375 wake_up(&cryptocop_ioc_process_wq);
2376 }
2377
2378
2379 #define CRYPTOCOP_IOCTL_CIPHER_TID (1)
2380 #define CRYPTOCOP_IOCTL_DIGEST_TID (2)
2381 #define CRYPTOCOP_IOCTL_CSUM_TID (3)
2382
2383 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2384 {
2385 size_t ch_ix = 0;
2386
2387 if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2388 if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2389 if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2390
2391 DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2392 return ch_ix;
2393 }
2394
2395
2396 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2397 {
2398 size_t ch_ix = INT_MAX;
2399 size_t tmp_ix = 0;
2400
2401 if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2402 if (crp_op->cipher_start > ix) {
2403 ch_ix = crp_op->cipher_start;
2404 } else {
2405 ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2406 }
2407 }
2408 if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2409 if (crp_op->digest_start > ix) {
2410 tmp_ix = crp_op->digest_start;
2411 } else {
2412 tmp_ix = crp_op->digest_start + crp_op->digest_len;
2413 }
2414 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2415 }
2416 if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2417 if (crp_op->csum_start > ix) {
2418 tmp_ix = crp_op->csum_start;
2419 } else {
2420 tmp_ix = crp_op->csum_start + crp_op->csum_len;
2421 }
2422 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2423 }
2424 if (ch_ix == INT_MAX) ch_ix = ix;
2425 DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2426 return ch_ix;
2427 }
2428
2429
2430 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2431 * Return -1 for ok, 0 for fail. */
2432 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2433 {
2434 int tmplen;
2435
2436 assert(iov != NULL);
2437 assert(iovix != NULL);
2438 assert(pages != NULL);
2439 assert(pageix != NULL);
2440 assert(pageoffset != NULL);
2441
2442 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2443
2444 while (map_length > 0){
2445 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2446 if (*iovix >= iovlen){
2447 DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2448 return 0;
2449 }
2450 if (*pageix >= nopages){
2451 DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2452 return 0;
2453 }
2454 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2455 tmplen = PAGE_SIZE - *pageoffset;
2456 if (tmplen < map_length){
2457 (*pageoffset) = 0;
2458 (*pageix)++;
2459 } else {
2460 tmplen = map_length;
2461 (*pageoffset) += map_length;
2462 }
2463 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2464 iov[*iovix].iov_len = tmplen;
2465 map_length -= tmplen;
2466 (*iovix)++;
2467 }
2468 DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2469 return -1;
2470 }
2471
2472
2473
2474 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2475 {
2476 int i;
2477 struct cryptocop_private *dev = filp->private_data;
2478 struct strcop_crypto_op *crp_oper = (struct strcop_crypto_op *)arg;
2479 struct strcop_crypto_op oper = {0};
2480 int err = 0;
2481 struct cryptocop_operation *cop = NULL;
2482
2483 struct ioctl_job_cb_ctx *jc = NULL;
2484
2485 struct page **inpages = NULL;
2486 struct page **outpages = NULL;
2487 int noinpages = 0;
2488 int nooutpages = 0;
2489
2490 struct cryptocop_desc descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2491 * can get connected/disconnected on different places in the indata. */
2492 struct cryptocop_desc_cfg dcfgs[5*3];
2493 int desc_ix = 0;
2494 int dcfg_ix = 0;
2495 struct cryptocop_tfrm_cfg ciph_tcfg = {0};
2496 struct cryptocop_tfrm_cfg digest_tcfg = {0};
2497 struct cryptocop_tfrm_cfg csum_tcfg = {0};
2498
2499 unsigned char *digest_result = NULL;
2500 int digest_length = 0;
2501 int cblocklen = 0;
2502 unsigned char csum_result[CSUM_BLOCK_LENGTH];
2503 struct cryptocop_session *sess;
2504
2505 int iovlen = 0;
2506 int iovix = 0;
2507 int pageix = 0;
2508 int pageoffset = 0;
2509
2510 size_t prev_ix = 0;
2511 size_t next_ix;
2512
2513 int cipher_active, digest_active, csum_active;
2514 int end_digest, end_csum;
2515 int digest_done = 0;
2516 int cipher_done = 0;
2517 int csum_done = 0;
2518
2519 DEBUG(printk("cryptocop_ioctl_process\n"));
2520
2521 if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2522 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2523 return -EFAULT;
2524 }
2525 if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2526 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2527 return -EFAULT;
2528 }
2529 DEBUG(print_strcop_crypto_op(&oper));
2530
2531 while (dev && dev->sid != oper.ses_id) dev = dev->next;
2532 if (!dev){
2533 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2534 return -EINVAL;
2535 }
2536
2537 /* Check buffers. */
2538 if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2539 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2540 return -EINVAL;
2541 }
2542
2543 if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2544 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2545 return -EFAULT;
2546 }
2547 if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2548 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2549 return -EFAULT;
2550 }
2551
2552 cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2553 if (!cop) {
2554 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2555 return -ENOMEM;
2556 }
2557 jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2558 if (!jc) {
2559 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2560 err = -ENOMEM;
2561 goto error_cleanup;
2562 }
2563 jc->processed = 0;
2564
2565 cop->cb_data = jc;
2566 cop->cb = ioctl_process_job_callback;
2567 cop->operation_status = 0;
2568 cop->use_dmalists = 0;
2569 cop->in_interrupt = 0;
2570 cop->fast_callback = 0;
2571 cop->tfrm_op.tfrm_cfg = NULL;
2572 cop->tfrm_op.desc = NULL;
2573 cop->tfrm_op.indata = NULL;
2574 cop->tfrm_op.incount = 0;
2575 cop->tfrm_op.inlen = 0;
2576 cop->tfrm_op.outdata = NULL;
2577 cop->tfrm_op.outcount = 0;
2578 cop->tfrm_op.outlen = 0;
2579
2580 sess = get_session(oper.ses_id);
2581 if (!sess){
2582 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2583 kfree(cop);
2584 kfree(jc);
2585 return -EINVAL;
2586 }
2587
2588 if (oper.do_cipher) {
2589 unsigned int cipher_outlen = 0;
2590 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2591 if (!tc) {
2592 DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2593 err = -EINVAL;
2594 goto error_cleanup;
2595 }
2596 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2597 ciph_tcfg.inject_ix = 0;
2598 ciph_tcfg.flags = 0;
2599 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2600 DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2601 kfree(cop);
2602 kfree(jc);
2603 return -EINVAL;
2604 }
2605 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2606 if (oper.cipher_len % cblocklen) {
2607 kfree(cop);
2608 kfree(jc);
2609 DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2610 return -EINVAL;
2611 }
2612 cipher_outlen = oper.cipher_len;
2613 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2614 if (oper.cipher_explicit) {
2615 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2616 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2617 } else {
2618 cipher_outlen = oper.cipher_len - cblocklen;
2619 }
2620 } else {
2621 if (oper.cipher_explicit){
2622 kfree(cop);
2623 kfree(jc);
2624 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2625 return -EINVAL;
2626 }
2627 }
2628 if (oper.cipher_outlen != cipher_outlen) {
2629 kfree(cop);
2630 kfree(jc);
2631 DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2632 return -EINVAL;
2633 }
2634
2635 if (oper.decrypt){
2636 ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2637 } else {
2638 ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2639 }
2640 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2641 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2642 }
2643 if (oper.do_digest){
2644 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2645 if (!tc) {
2646 DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2647 err = -EINVAL;
2648 goto error_cleanup;
2649 }
2650 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2651 digest_result = kmalloc(digest_length, GFP_KERNEL);
2652 if (!digest_result) {
2653 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2654 err = -EINVAL;
2655 goto error_cleanup;
2656 }
2657 DEBUG(memset(digest_result, 0xff, digest_length));
2658
2659 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2660 digest_tcfg.inject_ix = 0;
2661 ciph_tcfg.inject_ix += digest_length;
2662 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2663 DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2664 err = -EINVAL;
2665 goto error_cleanup;
2666 }
2667
2668 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2669 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2670 }
2671 if (oper.do_csum){
2672 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2673 csum_tcfg.inject_ix = digest_length;
2674 ciph_tcfg.inject_ix += 2;
2675
2676 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2677 DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2678 kfree(cop);
2679 kfree(jc);
2680 return -EINVAL;
2681 }
2682
2683 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2684 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2685 }
2686
2687 prev_ix = first_cfg_change_ix(&oper);
2688 if (prev_ix > oper.inlen) {
2689 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2690 nooutpages = noinpages = 0;
2691 err = -EINVAL;
2692 goto error_cleanup;
2693 }
2694 DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2695
2696 /* Map user pages for in and out data of the operation. */
2697 noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2698 DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2699 inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2700 if (!inpages){
2701 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2702 nooutpages = noinpages = 0;
2703 err = -ENOMEM;
2704 goto error_cleanup;
2705 }
2706 if (oper.do_cipher){
2707 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2708 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2709 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2710 if (!outpages){
2711 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2712 nooutpages = noinpages = 0;
2713 err = -ENOMEM;
2714 goto error_cleanup;
2715 }
2716 }
2717
2718 /* Acquire the mm page semaphore. */
2719 down_read(&current->mm->mmap_sem);
2720
2721 err = get_user_pages(current,
2722 current->mm,
2723 (unsigned long int)(oper.indata + prev_ix),
2724 noinpages,
2725 0, /* read access only for in data */
2726 0, /* no force */
2727 inpages,
2728 NULL);
2729
2730 if (err < 0) {
2731 up_read(&current->mm->mmap_sem);
2732 nooutpages = noinpages = 0;
2733 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2734 goto error_cleanup;
2735 }
2736 noinpages = err;
2737 if (oper.do_cipher){
2738 err = get_user_pages(current,
2739 current->mm,
2740 (unsigned long int)oper.cipher_outdata,
2741 nooutpages,
2742 1, /* write access for out data */
2743 0, /* no force */
2744 outpages,
2745 NULL);
2746 up_read(&current->mm->mmap_sem);
2747 if (err < 0) {
2748 nooutpages = 0;
2749 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2750 goto error_cleanup;
2751 }
2752 nooutpages = err;
2753 } else {
2754 up_read(&current->mm->mmap_sem);
2755 }
2756
2757 /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2758 * csum output and splits when units are (dis-)connected. */
2759 cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2760 cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2761 if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2762 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2763 err = -ENOMEM;
2764 goto error_cleanup;
2765 }
2766
2767 cop->tfrm_op.inlen = oper.inlen - prev_ix;
2768 cop->tfrm_op.outlen = 0;
2769 if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2770 if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2771 if (oper.do_csum) cop->tfrm_op.outlen += 2;
2772
2773 /* Setup the in iovecs. */
2774 cop->tfrm_op.incount = noinpages;
2775 if (noinpages > 1){
2776 size_t tmplen = cop->tfrm_op.inlen;
2777
2778 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2779 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2780 tmplen -= cop->tfrm_op.indata[0].iov_len;
2781 for (i = 1; i<noinpages; i++){
2782 cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2783 cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2784 tmplen -= PAGE_SIZE;
2785 }
2786 } else {
2787 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2788 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2789 }
2790
2791 iovlen = nooutpages + 6;
2792 pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2793
2794 next_ix = next_cfg_change_ix(&oper, prev_ix);
2795 if (prev_ix == next_ix){
2796 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2797 err = -EINVAL; /* This should be impossible barring bugs. */
2798 goto error_cleanup;
2799 }
2800 while (prev_ix != next_ix){
2801 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2802 descs[desc_ix].cfg = NULL;
2803 descs[desc_ix].length = next_ix - prev_ix;
2804
2805 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2806 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2807 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2808 cipher_active = 1;
2809
2810 if (next_ix == (oper.cipher_start + oper.cipher_len)){
2811 cipher_done = 1;
2812 dcfgs[dcfg_ix].last = 1;
2813 } else {
2814 dcfgs[dcfg_ix].last = 0;
2815 }
2816 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2817 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2818 ++dcfg_ix;
2819 }
2820 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2821 digest_active = 1;
2822 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2823 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2824 if (next_ix == (oper.digest_start + oper.digest_len)){
2825 assert(!digest_done);
2826 digest_done = 1;
2827 dcfgs[dcfg_ix].last = 1;
2828 } else {
2829 dcfgs[dcfg_ix].last = 0;
2830 }
2831 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2832 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2833 ++dcfg_ix;
2834 }
2835 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2836 csum_active = 1;
2837 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2838 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2839 if (next_ix == (oper.csum_start + oper.csum_len)){
2840 csum_done = 1;
2841 dcfgs[dcfg_ix].last = 1;
2842 } else {
2843 dcfgs[dcfg_ix].last = 0;
2844 }
2845 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2846 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2847 ++dcfg_ix;
2848 }
2849 if (!descs[desc_ix].cfg){
2850 DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2851 err = -EINVAL;
2852 goto error_cleanup;
2853 }
2854 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2855 ++desc_ix;
2856 prev_ix = next_ix;
2857 next_ix = next_cfg_change_ix(&oper, prev_ix);
2858 }
2859 if (desc_ix > 0){
2860 descs[desc_ix-1].next = NULL;
2861 } else {
2862 descs[0].next = NULL;
2863 }
2864 if (oper.do_digest) {
2865 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2866 /* Add outdata iovec, length == <length of type of digest> */
2867 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2868 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2869 ++iovix;
2870 }
2871 if (oper.do_csum) {
2872 /* Add outdata iovec, length == 2, the length of csum. */
2873 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2874 /* Add outdata iovec, length == <length of type of digest> */
2875 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2876 cop->tfrm_op.outdata[iovix].iov_len = 2;
2877 ++iovix;
2878 }
2879 if (oper.do_cipher) {
2880 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2881 DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2882 err = -ENOSYS; /* This should be impossible barring bugs. */
2883 goto error_cleanup;
2884 }
2885 }
2886 DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2887 cop->tfrm_op.outcount = iovix;
2888 assert(iovix <= (nooutpages + 6));
2889
2890 cop->sid = oper.ses_id;
2891 cop->tfrm_op.desc = &descs[0];
2892
2893 DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2894
2895 if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2896 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2897 err = -EINVAL;
2898 goto error_cleanup;
2899 }
2900
2901 DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2902
2903 wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2904 DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2905 if (!jc->processed){
2906 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2907 err = -EIO;
2908 goto error_cleanup;
2909 }
2910
2911 /* Job process done. Cipher output should already be correct in job so no post processing of outdata. */
2912 DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2913 if (cop->operation_status == 0){
2914 if (oper.do_digest){
2915 DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2916 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2917 if (0 != err){
2918 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2919 err = -EFAULT;
2920 goto error_cleanup;
2921 }
2922 }
2923 if (oper.do_csum){
2924 DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2925 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2926 if (0 != err){
2927 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2928 err = -EFAULT;
2929 goto error_cleanup;
2930 }
2931 }
2932 err = 0;
2933 } else {
2934 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2935 err = cop->operation_status;
2936 }
2937
2938 error_cleanup:
2939 /* Release page caches. */
2940 for (i = 0; i < noinpages; i++){
2941 put_page(inpages[i]);
2942 }
2943 for (i = 0; i < nooutpages; i++){
2944 int spdl_err;
2945 /* Mark output pages dirty. */
2946 spdl_err = set_page_dirty_lock(outpages[i]);
2947 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2948 }
2949 for (i = 0; i < nooutpages; i++){
2950 put_page(outpages[i]);
2951 }
2952
2953 kfree(digest_result);
2954 kfree(inpages);
2955 kfree(outpages);
2956 if (cop){
2957 kfree(cop->tfrm_op.indata);
2958 kfree(cop->tfrm_op.outdata);
2959 kfree(cop);
2960 }
2961 kfree(jc);
2962
2963 DEBUG(print_lock_status());
2964
2965 return err;
2966 }
2967
2968
2969 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2970 {
2971 cryptocop_session_id sid;
2972 int err;
2973 struct cryptocop_private *dev;
2974 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2975 struct strcop_session_op sop;
2976 struct cryptocop_transform_init *tis = NULL;
2977 struct cryptocop_transform_init ti_cipher = {0};
2978 struct cryptocop_transform_init ti_digest = {0};
2979 struct cryptocop_transform_init ti_csum = {0};
2980
2981 if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2982 return -EFAULT;
2983 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2984 if (err) return -EFAULT;
2985 if (sop.cipher != cryptocop_cipher_none) {
2986 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2987 }
2988 DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2989
2990 DEBUG(printk("\tcipher:%d\n"
2991 "\tcipher_mode:%d\n"
2992 "\tdigest:%d\n"
2993 "\tcsum:%d\n",
2994 (int)sop.cipher,
2995 (int)sop.cmode,
2996 (int)sop.digest,
2997 (int)sop.csum));
2998
2999 if (sop.cipher != cryptocop_cipher_none){
3000 /* Init the cipher. */
3001 switch (sop.cipher){
3002 case cryptocop_cipher_des:
3003 ti_cipher.alg = cryptocop_alg_des;
3004 break;
3005 case cryptocop_cipher_3des:
3006 ti_cipher.alg = cryptocop_alg_3des;
3007 break;
3008 case cryptocop_cipher_aes:
3009 ti_cipher.alg = cryptocop_alg_aes;
3010 break;
3011 default:
3012 DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3013 return -EINVAL;
3014 };
3015 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3016 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3017 ti_cipher.keylen = sop.keylen;
3018 switch (sop.cmode){
3019 case cryptocop_cipher_mode_cbc:
3020 case cryptocop_cipher_mode_ecb:
3021 ti_cipher.cipher_mode = sop.cmode;
3022 break;
3023 default:
3024 DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3025 return -EINVAL;
3026 }
3027 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3028 switch (sop.des3_mode){
3029 case cryptocop_3des_eee:
3030 case cryptocop_3des_eed:
3031 case cryptocop_3des_ede:
3032 case cryptocop_3des_edd:
3033 case cryptocop_3des_dee:
3034 case cryptocop_3des_ded:
3035 case cryptocop_3des_dde:
3036 case cryptocop_3des_ddd:
3037 ti_cipher.tdes_mode = sop.des3_mode;
3038 break;
3039 default:
3040 DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3041 return -EINVAL;
3042 }
3043 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3044 ti_cipher.next = tis;
3045 tis = &ti_cipher;
3046 } /* if (sop.cipher != cryptocop_cipher_none) */
3047 if (sop.digest != cryptocop_digest_none){
3048 DEBUG(printk("setting digest transform\n"));
3049 switch (sop.digest){
3050 case cryptocop_digest_md5:
3051 ti_digest.alg = cryptocop_alg_md5;
3052 break;
3053 case cryptocop_digest_sha1:
3054 ti_digest.alg = cryptocop_alg_sha1;
3055 break;
3056 default:
3057 DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3058 return -EINVAL;
3059 }
3060 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3061 ti_digest.next = tis;
3062 tis = &ti_digest;
3063 } /* if (sop.digest != cryptocop_digest_none) */
3064 if (sop.csum != cryptocop_csum_none){
3065 DEBUG(printk("setting csum transform\n"));
3066 switch (sop.csum){
3067 case cryptocop_csum_le:
3068 case cryptocop_csum_be:
3069 ti_csum.csum_mode = sop.csum;
3070 break;
3071 default:
3072 DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3073 return -EINVAL;
3074 }
3075 ti_csum.alg = cryptocop_alg_csum;
3076 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3077 ti_csum.next = tis;
3078 tis = &ti_csum;
3079 } /* (sop.csum != cryptocop_csum_none) */
3080 dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3081 if (!dev){
3082 DEBUG_API(printk("create session, alloc dev\n"));
3083 return -ENOMEM;
3084 }
3085
3086 err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3087 DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3088
3089 if (err) {
3090 kfree(dev);
3091 return err;
3092 }
3093 sess_op->ses_id = sid;
3094 dev->sid = sid;
3095 dev->next = filp->private_data;
3096 filp->private_data = dev;
3097
3098 return 0;
3099 }
3100
3101 static long cryptocop_ioctl_unlocked(struct inode *inode,
3102 struct file *filp, unsigned int cmd, unsigned long arg)
3103 {
3104 int err = 0;
3105 if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3106 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3107 return -ENOTTY;
3108 }
3109 if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3110 return -ENOTTY;
3111 }
3112 /* Access check of the argument. Some commands, e.g. create session and process op,
3113 needs additional checks. Those are handled in the command handling functions. */
3114 if (_IOC_DIR(cmd) & _IOC_READ)
3115 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3116 else if (_IOC_DIR(cmd) & _IOC_WRITE)
3117 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3118 if (err) return -EFAULT;
3119
3120 switch (cmd) {
3121 case CRYPTOCOP_IO_CREATE_SESSION:
3122 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3123 case CRYPTOCOP_IO_CLOSE_SESSION:
3124 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3125 case CRYPTOCOP_IO_PROCESS_OP:
3126 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3127 default:
3128 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3129 return -ENOTTY;
3130 }
3131 return 0;
3132 }
3133
3134 static long
3135 cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3136 {
3137 struct inode *inode = file->f_path.dentry->d_inode;
3138 long ret;
3139
3140 lock_kernel();
3141 ret = cryptocop_ioctl_unlocked(inode, filp, cmd, arg);
3142 unlock_kernel();
3143
3144 return ret;
3145 }
3146
3147
3148 #ifdef LDEBUG
3149 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3150 {
3151 struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3152 struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3153 int i;
3154
3155 printk("print_dma_descriptors start\n");
3156
3157 printk("iop:\n");
3158 printk("\tsid: 0x%lld\n", iop->sid);
3159
3160 printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3161 printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3162 printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3163 printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3164
3165 printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3166 printk("\tnext: 0x%p\n"
3167 "\tsaved_data: 0x%p\n"
3168 "\tsaved_data_buf: 0x%p\n",
3169 iop->ctx_out.next,
3170 iop->ctx_out.saved_data,
3171 iop->ctx_out.saved_data_buf);
3172
3173 printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3174 printk("\tnext: 0x%p\n"
3175 "\tsaved_data: 0x%p\n"
3176 "\tsaved_data_buf: 0x%p\n",
3177 iop->ctx_in.next,
3178 iop->ctx_in.saved_data,
3179 iop->ctx_in.saved_data_buf);
3180
3181 i = 0;
3182 while (cdesc_out) {
3183 dma_descr_data *td;
3184 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3185 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3186 td = cdesc_out->dma_descr;
3187 printk("\n\tbuf: 0x%p\n"
3188 "\tafter: 0x%p\n"
3189 "\tmd: 0x%04x\n"
3190 "\tnext: 0x%p\n",
3191 td->buf,
3192 td->after,
3193 td->md,
3194 td->next);
3195 printk("flags:\n"
3196 "\twait:\t%d\n"
3197 "\teol:\t%d\n"
3198 "\touteop:\t%d\n"
3199 "\tineop:\t%d\n"
3200 "\tintr:\t%d\n",
3201 td->wait,
3202 td->eol,
3203 td->out_eop,
3204 td->in_eop,
3205 td->intr);
3206 cdesc_out = cdesc_out->next;
3207 i++;
3208 }
3209 i = 0;
3210 while (cdesc_in) {
3211 dma_descr_data *td;
3212 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3213 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3214 td = cdesc_in->dma_descr;
3215 printk("\n\tbuf: 0x%p\n"
3216 "\tafter: 0x%p\n"
3217 "\tmd: 0x%04x\n"
3218 "\tnext: 0x%p\n",
3219 td->buf,
3220 td->after,
3221 td->md,
3222 td->next);
3223 printk("flags:\n"
3224 "\twait:\t%d\n"
3225 "\teol:\t%d\n"
3226 "\touteop:\t%d\n"
3227 "\tineop:\t%d\n"
3228 "\tintr:\t%d\n",
3229 td->wait,
3230 td->eol,
3231 td->out_eop,
3232 td->in_eop,
3233 td->intr);
3234 cdesc_in = cdesc_in->next;
3235 i++;
3236 }
3237
3238 printk("print_dma_descriptors end\n");
3239 }
3240
3241
3242 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3243 {
3244 printk("print_strcop_crypto_op, 0x%p\n", cop);
3245
3246 /* Indata. */
3247 printk("indata=0x%p\n"
3248 "inlen=%d\n"
3249 "do_cipher=%d\n"
3250 "decrypt=%d\n"
3251 "cipher_explicit=%d\n"
3252 "cipher_start=%d\n"
3253 "cipher_len=%d\n"
3254 "outdata=0x%p\n"
3255 "outlen=%d\n",
3256 cop->indata,
3257 cop->inlen,
3258 cop->do_cipher,
3259 cop->decrypt,
3260 cop->cipher_explicit,
3261 cop->cipher_start,
3262 cop->cipher_len,
3263 cop->cipher_outdata,
3264 cop->cipher_outlen);
3265
3266 printk("do_digest=%d\n"
3267 "digest_start=%d\n"
3268 "digest_len=%d\n",
3269 cop->do_digest,
3270 cop->digest_start,
3271 cop->digest_len);
3272
3273 printk("do_csum=%d\n"
3274 "csum_start=%d\n"
3275 "csum_len=%d\n",
3276 cop->do_csum,
3277 cop->csum_start,
3278 cop->csum_len);
3279 }
3280
3281 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3282 {
3283 struct cryptocop_desc *d;
3284 struct cryptocop_tfrm_cfg *tc;
3285 struct cryptocop_desc_cfg *dc;
3286 int i;
3287
3288 printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3289 printk("sid: %lld\n", cop->sid);
3290 printk("operation_status=%d\n"
3291 "use_dmalists=%d\n"
3292 "in_interrupt=%d\n"
3293 "fast_callback=%d\n",
3294 cop->operation_status,
3295 cop->use_dmalists,
3296 cop->in_interrupt,
3297 cop->fast_callback);
3298
3299 if (cop->use_dmalists){
3300 print_user_dma_lists(&cop->list_op);
3301 } else {
3302 printk("cop->tfrm_op\n"
3303 "tfrm_cfg=0x%p\n"
3304 "desc=0x%p\n"
3305 "indata=0x%p\n"
3306 "incount=%d\n"
3307 "inlen=%d\n"
3308 "outdata=0x%p\n"
3309 "outcount=%d\n"
3310 "outlen=%d\n\n",
3311 cop->tfrm_op.tfrm_cfg,
3312 cop->tfrm_op.desc,
3313 cop->tfrm_op.indata,
3314 cop->tfrm_op.incount,
3315 cop->tfrm_op.inlen,
3316 cop->tfrm_op.outdata,
3317 cop->tfrm_op.outcount,
3318 cop->tfrm_op.outlen);
3319
3320 tc = cop->tfrm_op.tfrm_cfg;
3321 while (tc){
3322 printk("tfrm_cfg, 0x%p\n"
3323 "tid=%d\n"
3324 "flags=%d\n"
3325 "inject_ix=%d\n"
3326 "next=0x%p\n",
3327 tc,
3328 tc->tid,
3329 tc->flags,
3330 tc->inject_ix,
3331 tc->next);
3332 tc = tc->next;
3333 }
3334 d = cop->tfrm_op.desc;
3335 while (d){
3336 printk("\n======================desc, 0x%p\n"
3337 "length=%d\n"
3338 "cfg=0x%p\n"
3339 "next=0x%p\n",
3340 d,
3341 d->length,
3342 d->cfg,
3343 d->next);
3344 dc = d->cfg;
3345 while (dc){
3346 printk("=========desc_cfg, 0x%p\n"
3347 "tid=%d\n"
3348 "src=%d\n"
3349 "last=%d\n"
3350 "next=0x%p\n",
3351 dc,
3352 dc->tid,
3353 dc->src,
3354 dc->last,
3355 dc->next);
3356 dc = dc->next;
3357 }
3358 d = d->next;
3359 }
3360 printk("\n====iniov\n");
3361 for (i = 0; i < cop->tfrm_op.incount; i++){
3362 printk("indata[%d]\n"
3363 "base=0x%p\n"
3364 "len=%d\n",
3365 i,
3366 cop->tfrm_op.indata[i].iov_base,
3367 cop->tfrm_op.indata[i].iov_len);
3368 }
3369 printk("\n====outiov\n");
3370 for (i = 0; i < cop->tfrm_op.outcount; i++){
3371 printk("outdata[%d]\n"
3372 "base=0x%p\n"
3373 "len=%d\n",
3374 i,
3375 cop->tfrm_op.outdata[i].iov_base,
3376 cop->tfrm_op.outdata[i].iov_len);
3377 }
3378 }
3379 printk("------------end print_cryptocop_operation\n");
3380 }
3381
3382
3383 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3384 {
3385 dma_descr_data *dd;
3386 int i;
3387
3388 printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3389
3390 printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3391 printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3392
3393 printk("##############outlist\n");
3394 dd = phys_to_virt((unsigned long int)dma_op->outlist);
3395 i = 0;
3396 while (dd != NULL) {
3397 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3398 printk("\n\tbuf: 0x%p\n"
3399 "\tafter: 0x%p\n"
3400 "\tmd: 0x%04x\n"
3401 "\tnext: 0x%p\n",
3402 dd->buf,
3403 dd->after,
3404 dd->md,
3405 dd->next);
3406 printk("flags:\n"
3407 "\twait:\t%d\n"
3408 "\teol:\t%d\n"
3409 "\touteop:\t%d\n"
3410 "\tineop:\t%d\n"
3411 "\tintr:\t%d\n",
3412 dd->wait,
3413 dd->eol,
3414 dd->out_eop,
3415 dd->in_eop,
3416 dd->intr);
3417 if (dd->eol)
3418 dd = NULL;
3419 else
3420 dd = phys_to_virt((unsigned long int)dd->next);
3421 ++i;
3422 }
3423
3424 printk("##############inlist\n");
3425 dd = phys_to_virt((unsigned long int)dma_op->inlist);
3426 i = 0;
3427 while (dd != NULL) {
3428 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3429 printk("\n\tbuf: 0x%p\n"
3430 "\tafter: 0x%p\n"
3431 "\tmd: 0x%04x\n"
3432 "\tnext: 0x%p\n",
3433 dd->buf,
3434 dd->after,
3435 dd->md,
3436 dd->next);
3437 printk("flags:\n"
3438 "\twait:\t%d\n"
3439 "\teol:\t%d\n"
3440 "\touteop:\t%d\n"
3441 "\tineop:\t%d\n"
3442 "\tintr:\t%d\n",
3443 dd->wait,
3444 dd->eol,
3445 dd->out_eop,
3446 dd->in_eop,
3447 dd->intr);
3448 if (dd->eol)
3449 dd = NULL;
3450 else
3451 dd = phys_to_virt((unsigned long int)dd->next);
3452 ++i;
3453 }
3454 }
3455
3456
3457 static void print_lock_status(void)
3458 {
3459 printk("**********************print_lock_status\n");
3460 printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3461 printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3462 printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3463 printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3464 printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3465 printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3466 }
3467 #endif /* LDEBUG */
3468
3469
3470 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3471
3472 static int init_stream_coprocessor(void)
3473 {
3474 int err;
3475 int i;
3476 static int initialized = 0;
3477
3478 if (initialized)
3479 return 0;
3480
3481 initialized = 1;
3482
3483 printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3484
3485 err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3486 if (err < 0) {
3487 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3488 return err;
3489 }
3490
3491 err = init_cryptocop();
3492 if (err) {
3493 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3494 return err;
3495 }
3496 err = cryptocop_job_queue_init();
3497 if (err) {
3498 release_cryptocop();
3499 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3500 return err;
3501 }
3502 /* Init the descriptor pool. */
3503 for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3504 descr_pool[i].from_pool = 1;
3505 descr_pool[i].next = &descr_pool[i + 1];
3506 }
3507 descr_pool[i].from_pool = 1;
3508 descr_pool[i].next = NULL;
3509 descr_pool_free_list = &descr_pool[0];
3510 descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3511
3512 spin_lock_init(&cryptocop_completed_jobs_lock);
3513 spin_lock_init(&cryptocop_job_queue_lock);
3514 spin_lock_init(&descr_pool_lock);
3515 spin_lock_init(&cryptocop_sessions_lock);
3516 spin_lock_init(&running_job_lock);
3517 spin_lock_init(&cryptocop_process_lock);
3518
3519 cryptocop_sessions = NULL;
3520 next_sid = 1;
3521
3522 cryptocop_running_job = NULL;
3523
3524 printk("stream co-processor: init done.\n");
3525 return 0;
3526 }
3527
3528 static void __exit exit_stream_coprocessor(void)
3529 {
3530 release_cryptocop();
3531 cryptocop_job_queue_close();
3532 }
3533
3534 module_init(init_stream_coprocessor);
3535 module_exit(exit_stream_coprocessor);
Tomato firmware and modifications.