?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
cris: autoconvert trivial BKL users
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / arch / cris / arch-v32 / drivers / cryptocop.c
blob0973d5a2780facd8110d434815da2502a8ff0bc1
1 /*
2 * Stream co-processor driver for the ETRAX FS
3 *
4 * Copyright (C) 2003-2007 Axis Communications AB
5 */
6
7 #include <linux/init.h>
8 #include <linux/sched.h>
9 #include <linux/module.h>
10 #include <linux/slab.h>
11 #include <linux/string.h>
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/spinlock.h>
15 #include <linux/stddef.h>
16
17 #include <asm/uaccess.h>
18 #include <asm/io.h>
19 #include <asm/atomic.h>
20
21 #include <linux/list.h>
22 #include <linux/interrupt.h>
23
24 #include <asm/signal.h>
25 #include <asm/irq.h>
26
27 #include <dma.h>
28 #include <hwregs/dma.h>
29 #include <hwregs/reg_map.h>
30 #include <hwregs/reg_rdwr.h>
31 #include <hwregs/intr_vect_defs.h>
32
33 #include <hwregs/strcop.h>
34 #include <hwregs/strcop_defs.h>
35 #include <cryptocop.h>
36
37 #ifdef CONFIG_ETRAXFS
38 #define IN_DMA 9
39 #define OUT_DMA 8
40 #define IN_DMA_INST regi_dma9
41 #define OUT_DMA_INST regi_dma8
42 #define DMA_IRQ DMA9_INTR_VECT
43 #else
44 #define IN_DMA 3
45 #define OUT_DMA 2
46 #define IN_DMA_INST regi_dma3
47 #define OUT_DMA_INST regi_dma2
48 #define DMA_IRQ DMA3_INTR_VECT
49 #endif
50
51 #define DESCR_ALLOC_PAD (31)
52
53 struct cryptocop_dma_desc {
54 char *free_buf; /* If non-null will be kfreed in free_cdesc() */
55 dma_descr_data *dma_descr;
56
57 unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
58
59 unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
60 struct cryptocop_dma_desc *next;
61 };
62
63
64 struct cryptocop_int_operation{
65 void *alloc_ptr;
66 cryptocop_session_id sid;
67
68 dma_descr_context ctx_out;
69 dma_descr_context ctx_in;
70
71 /* DMA descriptors allocated by driver. */
72 struct cryptocop_dma_desc *cdesc_out;
73 struct cryptocop_dma_desc *cdesc_in;
74
75 /* Strcop config to use. */
76 cryptocop_3des_mode tdes_mode;
77 cryptocop_csum_type csum_mode;
78
79 /* DMA descrs provided by consumer. */
80 dma_descr_data *ddesc_out;
81 dma_descr_data *ddesc_in;
82 };
83
84
85 struct cryptocop_tfrm_ctx {
86 cryptocop_tfrm_id tid;
87 unsigned int blocklength;
88
89 unsigned int start_ix;
90
91 struct cryptocop_tfrm_cfg *tcfg;
92 struct cryptocop_transform_ctx *tctx;
93
94 unsigned char previous_src;
95 unsigned char current_src;
96
97 /* Values to use in metadata out. */
98 unsigned char hash_conf;
99 unsigned char hash_mode;
100 unsigned char ciph_conf;
101 unsigned char cbcmode;
102 unsigned char decrypt;
103
104 unsigned int requires_padding:1;
105 unsigned int strict_block_length:1;
106 unsigned int active:1;
107 unsigned int done:1;
108 size_t consumed;
109 size_t produced;
110
111 /* Pad (input) descriptors to put in the DMA out list when the transform
112 * output is put on the DMA in list. */
113 struct cryptocop_dma_desc *pad_descs;
114
115 struct cryptocop_tfrm_ctx *prev_src;
116 struct cryptocop_tfrm_ctx *curr_src;
117
118 /* Mapping to HW. */
119 unsigned char unit_no;
120 };
121
122
123 struct cryptocop_private{
124 cryptocop_session_id sid;
125 struct cryptocop_private *next;
126 };
127
128 /* Session list. */
129
130 struct cryptocop_transform_ctx{
131 struct cryptocop_transform_init init;
132 unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
133 unsigned int dec_key_set:1;
134
135 struct cryptocop_transform_ctx *next;
136 };
137
138
139 struct cryptocop_session{
140 cryptocop_session_id sid;
141
142 struct cryptocop_transform_ctx *tfrm_ctx;
143
144 struct cryptocop_session *next;
145 };
146
147 /* Priority levels for jobs sent to the cryptocop. Checksum operations from
148 kernel have highest priority since TCPIP stack processing must not
149 be a bottleneck. */
150 typedef enum {
151 cryptocop_prio_kernel_csum = 0,
152 cryptocop_prio_kernel = 1,
153 cryptocop_prio_user = 2,
154 cryptocop_prio_no_prios = 3
155 } cryptocop_queue_priority;
156
157 struct cryptocop_prio_queue{
158 struct list_head jobs;
159 cryptocop_queue_priority prio;
160 };
161
162 struct cryptocop_prio_job{
163 struct list_head node;
164 cryptocop_queue_priority prio;
165
166 struct cryptocop_operation *oper;
167 struct cryptocop_int_operation *iop;
168 };
169
170 struct ioctl_job_cb_ctx {
171 unsigned int processed:1;
172 };
173
174
175 static struct cryptocop_session *cryptocop_sessions = NULL;
176 spinlock_t cryptocop_sessions_lock;
177
178 /* Next Session ID to assign. */
179 static cryptocop_session_id next_sid = 1;
180
181 /* Pad for checksum. */
182 static const char csum_zero_pad[1] = {0x00};
183
184 /* Trash buffer for mem2mem operations. */
185 #define MEM2MEM_DISCARD_BUF_LENGTH (512)
186 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
187
188 /* Descriptor pool. */
189 /* FIXME Tweak this value. */
190 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE (100)
191 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
192 static struct cryptocop_dma_desc *descr_pool_free_list;
193 static int descr_pool_no_free;
194 static spinlock_t descr_pool_lock;
195
196 /* Lock to stop cryptocop to start processing of a new operation. The holder
197 of this lock MUST call cryptocop_start_job() after it is unlocked. */
198 spinlock_t cryptocop_process_lock;
199
200 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
201 static spinlock_t cryptocop_job_queue_lock;
202 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
203 static spinlock_t running_job_lock;
204
205 /* The interrupt handler appends completed jobs to this list. The scehduled
206 * tasklet removes them upon sending the response to the crypto consumer. */
207 static struct list_head cryptocop_completed_jobs;
208 static spinlock_t cryptocop_completed_jobs_lock;
209
210 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
211
212
213 /** Local functions. **/
214
215 static int cryptocop_open(struct inode *, struct file *);
216
217 static int cryptocop_release(struct inode *, struct file *);
218
219 static long cryptocop_ioctl(struct file *file,
220 unsigned int cmd, unsigned long arg);
221
222 static void cryptocop_start_job(void);
223
224 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
225 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
226
227 static int cryptocop_job_queue_init(void);
228 static void cryptocop_job_queue_close(void);
229
230 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
231
232 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
233
234 static int transform_ok(struct cryptocop_transform_init *tinit);
235
236 static struct cryptocop_session *get_session(cryptocop_session_id sid);
237
238 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
239
240 static void delete_internal_operation(struct cryptocop_int_operation *iop);
241
242 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength);
243
244 static int init_stream_coprocessor(void);
245
246 static void __exit exit_stream_coprocessor(void);
247
248 /*#define LDEBUG*/
249 #ifdef LDEBUG
250 #define DEBUG(s) s
251 #define DEBUG_API(s) s
252 static void print_cryptocop_operation(struct cryptocop_operation *cop);
253 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
254 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
255 static void print_lock_status(void);
256 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
257 #define assert(s) do{if (!(s)) panic(#s);} while(0);
258 #else
259 #define DEBUG(s)
260 #define DEBUG_API(s)
261 #define assert(s)
262 #endif
263
264
265 /* Transform constants. */
266 #define DES_BLOCK_LENGTH (8)
267 #define AES_BLOCK_LENGTH (16)
268 #define MD5_BLOCK_LENGTH (64)
269 #define SHA1_BLOCK_LENGTH (64)
270 #define CSUM_BLOCK_LENGTH (2)
271 #define MD5_STATE_LENGTH (16)
272 #define SHA1_STATE_LENGTH (20)
273
274 /* The device number. */
275 #define CRYPTOCOP_MAJOR (254)
276 #define CRYPTOCOP_MINOR (0)
277
278
279
280 const struct file_operations cryptocop_fops = {
281 .owner = THIS_MODULE,
282 .open = cryptocop_open,
283 .release = cryptocop_release,
284 .unlocked_ioctl = cryptocop_ioctl
285 };
286
287
288 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
289 {
290 DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
291 kfree(cdesc->free_buf);
292
293 if (cdesc->from_pool) {
294 unsigned long int flags;
295 spin_lock_irqsave(&descr_pool_lock, flags);
296 cdesc->next = descr_pool_free_list;
297 descr_pool_free_list = cdesc;
298 ++descr_pool_no_free;
299 spin_unlock_irqrestore(&descr_pool_lock, flags);
300 } else {
301 kfree(cdesc);
302 }
303 }
304
305
306 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
307 {
308 int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
309 struct cryptocop_dma_desc *cdesc;
310
311 if (use_pool) {
312 unsigned long int flags;
313 spin_lock_irqsave(&descr_pool_lock, flags);
314 if (!descr_pool_free_list) {
315 spin_unlock_irqrestore(&descr_pool_lock, flags);
316 DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
317 return NULL;
318 }
319 cdesc = descr_pool_free_list;
320 descr_pool_free_list = descr_pool_free_list->next;
321 --descr_pool_no_free;
322 spin_unlock_irqrestore(&descr_pool_lock, flags);
323 cdesc->from_pool = 1;
324 } else {
325 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
326 if (!cdesc) {
327 DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
328 return NULL;
329 }
330 cdesc->from_pool = 0;
331 }
332 cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
333
334 cdesc->next = NULL;
335
336 cdesc->free_buf = NULL;
337 cdesc->dma_descr->out_eop = 0;
338 cdesc->dma_descr->in_eop = 0;
339 cdesc->dma_descr->intr = 0;
340 cdesc->dma_descr->eol = 0;
341 cdesc->dma_descr->wait = 0;
342 cdesc->dma_descr->buf = NULL;
343 cdesc->dma_descr->after = NULL;
344
345 DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
346 return cdesc;
347 }
348
349
350 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
351 {
352 DEBUG(printk("setup_descr_chain: entering\n"));
353 while (cd) {
354 if (cd->next) {
355 cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
356 } else {
357 cd->dma_descr->next = NULL;
358 }
359 cd = cd->next;
360 }
361 DEBUG(printk("setup_descr_chain: exit\n"));
362 }
363
364
365 /* Create a pad descriptor for the transform.
366 * Return -1 for error, 0 if pad created. */
367 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
368 {
369 struct cryptocop_dma_desc *cdesc = NULL;
370 int error = 0;
371 struct strcop_meta_out mo = {
372 .ciphsel = src_none,
373 .hashsel = src_none,
374 .csumsel = src_none
375 };
376 char *pad;
377 size_t plen;
378
379 DEBUG(printk("create_pad_descriptor: start.\n"));
380 /* Setup pad descriptor. */
381
382 DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
383 cdesc = alloc_cdesc(alloc_flag);
384 if (!cdesc){
385 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
386 goto error_cleanup;
387 }
388 switch (tc->unit_no) {
389 case src_md5:
390 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
391 if (error){
392 DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
393 goto error_cleanup;
394 }
395 cdesc->free_buf = pad;
396 mo.hashsel = src_dma;
397 mo.hashconf = tc->hash_conf;
398 mo.hashmode = tc->hash_mode;
399 break;
400 case src_sha1:
401 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
402 if (error){
403 DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
404 goto error_cleanup;
405 }
406 cdesc->free_buf = pad;
407 mo.hashsel = src_dma;
408 mo.hashconf = tc->hash_conf;
409 mo.hashmode = tc->hash_mode;
410 break;
411 case src_csum:
412 if (tc->consumed % tc->blocklength){
413 pad = (char*)csum_zero_pad;
414 plen = 1;
415 } else {
416 pad = (char*)cdesc; /* Use any pointer. */
417 plen = 0;
418 }
419 mo.csumsel = src_dma;
420 break;
421 }
422 cdesc->dma_descr->wait = 1;
423 cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed. EOP is ok here since the padded unit is the only one active. */
424 cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
425 cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
426
427 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
428 *pad_desc = cdesc;
429
430 return 0;
431
432 error_cleanup:
433 if (cdesc) free_cdesc(cdesc);
434 return -1;
435 }
436
437
438 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
439 {
440 struct cryptocop_dma_desc *key_desc = alloc_cdesc(alloc_flag);
441 struct strcop_meta_out mo = {0};
442
443 DEBUG(printk("setup_key_dl_desc\n"));
444
445 if (!key_desc) {
446 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
447 return -ENOMEM;
448 }
449
450 /* Download key. */
451 if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
452 /* Precook the AES decrypt key. */
453 if (!tc->tctx->dec_key_set){
454 get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
455 tc->tctx->dec_key_set = 1;
456 }
457 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
458 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
459 } else {
460 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
461 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
462 }
463 /* Setup metadata. */
464 mo.dlkey = 1;
465 switch (tc->tctx->init.keylen) {
466 case 64:
467 mo.decrypt = 0;
468 mo.hashmode = 0;
469 break;
470 case 128:
471 mo.decrypt = 0;
472 mo.hashmode = 1;
473 break;
474 case 192:
475 mo.decrypt = 1;
476 mo.hashmode = 0;
477 break;
478 case 256:
479 mo.decrypt = 1;
480 mo.hashmode = 1;
481 break;
482 default:
483 break;
484 }
485 mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
486 key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
487
488 key_desc->dma_descr->out_eop = 1;
489 key_desc->dma_descr->wait = 1;
490 key_desc->dma_descr->intr = 0;
491
492 *kd = key_desc;
493 return 0;
494 }
495
496 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
497 {
498 struct cryptocop_dma_desc *iv_desc = alloc_cdesc(alloc_flag);
499 struct strcop_meta_out mo = {0};
500
501 DEBUG(printk("setup_cipher_iv_desc\n"));
502
503 if (!iv_desc) {
504 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
505 return -ENOMEM;
506 }
507 /* Download IV. */
508 iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
509 iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
510
511 /* Setup metadata. */
512 mo.hashsel = mo.csumsel = src_none;
513 mo.ciphsel = src_dma;
514 mo.ciphconf = tc->ciph_conf;
515 mo.cbcmode = tc->cbcmode;
516
517 iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
518
519 iv_desc->dma_descr->out_eop = 0;
520 iv_desc->dma_descr->wait = 1;
521 iv_desc->dma_descr->intr = 0;
522
523 *id = iv_desc;
524 return 0;
525 }
526
527 /* Map the ouput length of the transform to operation output starting on the inject index. */
528 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
529 {
530 int err = 0;
531 struct cryptocop_dma_desc head = {0};
532 struct cryptocop_dma_desc *outdesc = &head;
533 size_t iov_offset = 0;
534 size_t out_ix = 0;
535 int outiov_ix = 0;
536 struct strcop_meta_in mi = {0};
537
538 size_t out_length = tc->produced;
539 int rem_length;
540 int dlength;
541
542 assert(out_length != 0);
543 if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
544 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
545 return -EINVAL;
546 }
547 /* Traverse the out iovec until the result inject index is reached. */
548 while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
549 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
550 outiov_ix++;
551 }
552 if (outiov_ix >= operation->tfrm_op.outcount){
553 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
554 return -EINVAL;
555 }
556 iov_offset = tc->tcfg->inject_ix - out_ix;
557 mi.dmasel = tc->unit_no;
558
559 /* Setup the output descriptors. */
560 while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
561 outdesc->next = alloc_cdesc(alloc_flag);
562 if (!outdesc->next) {
563 DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
564 err = -ENOMEM;
565 goto error_cleanup;
566 }
567 outdesc = outdesc->next;
568 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
569 dlength = (out_length < rem_length) ? out_length : rem_length;
570
571 DEBUG(printk("create_input_descriptors:\n"
572 "outiov_ix=%d, rem_length=%d, dlength=%d\n"
573 "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
574 "outcount=%d, outiov_ix=%d\n",
575 outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
576
577 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
578 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
579 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
580
581 out_length -= dlength;
582 iov_offset += dlength;
583 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
584 iov_offset = 0;
585 ++outiov_ix;
586 }
587 }
588 if (out_length > 0){
589 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
590 err = -EINVAL;
591 goto error_cleanup;
592 }
593 /* Set sync in last descriptor. */
594 mi.sync = 1;
595 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
596
597 *id = head.next;
598 return 0;
599
600 error_cleanup:
601 while (head.next) {
602 outdesc = head.next->next;
603 free_cdesc(head.next);
604 head.next = outdesc;
605 }
606 return err;
607 }
608
609
610 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
611 {
612 while (desc_len != 0) {
613 struct cryptocop_dma_desc *cdesc;
614 int rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
615 int dlength = (desc_len < rem_length) ? desc_len : rem_length;
616
617 cdesc = alloc_cdesc(alloc_flag);
618 if (!cdesc) {
619 DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
620 return -ENOMEM;
621 }
622 (*current_out_cdesc)->next = cdesc;
623 (*current_out_cdesc) = cdesc;
624
625 cdesc->free_buf = NULL;
626
627 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
628 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
629
630 assert(desc_len >= dlength);
631 desc_len -= dlength;
632 *iniov_offset += dlength;
633 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
634 *iniov_offset = 0;
635 ++(*iniov_ix);
636 if (*iniov_ix > operation->tfrm_op.incount) {
637 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
638 return -EINVAL;
639 }
640 }
641 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
642 } /* while (desc_len != 0) */
643 /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
644 (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
645
646 return 0;
647 }
648
649
650 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
651 {
652 DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
653 if (tc->tcfg) {
654 int failed = 0;
655 struct cryptocop_dma_desc *idescs = NULL;
656 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
657 if (tc->pad_descs) {
658 DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
659 while (tc->pad_descs) {
660 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
661 (*current_out_cdesc)->next = tc->pad_descs;
662 tc->pad_descs = tc->pad_descs->next;
663 (*current_out_cdesc) = (*current_out_cdesc)->next;
664 }
665 }
666
667 /* Setup and append output descriptors to DMA in list. */
668 if (tc->unit_no == src_dma){
669 /* mem2mem. Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
670 struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
671 unsigned int start_ix = tc->start_ix;
672 while (start_ix){
673 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
674 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
675 if (!(*current_in_cdesc)->next){
676 DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
677 return -ENOMEM;
678 }
679 (*current_in_cdesc) = (*current_in_cdesc)->next;
680 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
681 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
682 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
683 start_ix -= desclen;
684 }
685 mi.sync = 1;
686 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
687 }
688
689 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
690 if (failed){
691 DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
692 return failed;
693 }
694 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
695 while (idescs) {
696 DEBUG(printk("append descriptor 0x%p\n", idescs));
697 (*current_in_cdesc)->next = idescs;
698 idescs = idescs->next;
699 (*current_in_cdesc) = (*current_in_cdesc)->next;
700 }
701 }
702 return 0;
703 }
704
705
706
707 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
708 {
709 struct cryptocop_session *sess;
710 struct cryptocop_transform_ctx *tctx;
711
712 struct cryptocop_tfrm_ctx digest_ctx = {
713 .previous_src = src_none,
714 .current_src = src_none,
715 .start_ix = 0,
716 .requires_padding = 1,
717 .strict_block_length = 0,
718 .hash_conf = 0,
719 .hash_mode = 0,
720 .ciph_conf = 0,
721 .cbcmode = 0,
722 .decrypt = 0,
723 .consumed = 0,
724 .produced = 0,
725 .pad_descs = NULL,
726 .active = 0,
727 .done = 0,
728 .prev_src = NULL,
729 .curr_src = NULL,
730 .tcfg = NULL};
731 struct cryptocop_tfrm_ctx cipher_ctx = {
732 .previous_src = src_none,
733 .current_src = src_none,
734 .start_ix = 0,
735 .requires_padding = 0,
736 .strict_block_length = 1,
737 .hash_conf = 0,
738 .hash_mode = 0,
739 .ciph_conf = 0,
740 .cbcmode = 0,
741 .decrypt = 0,
742 .consumed = 0,
743 .produced = 0,
744 .pad_descs = NULL,
745 .active = 0,
746 .done = 0,
747 .prev_src = NULL,
748 .curr_src = NULL,
749 .tcfg = NULL};
750 struct cryptocop_tfrm_ctx csum_ctx = {
751 .previous_src = src_none,
752 .current_src = src_none,
753 .start_ix = 0,
754 .blocklength = 2,
755 .requires_padding = 1,
756 .strict_block_length = 0,
757 .hash_conf = 0,
758 .hash_mode = 0,
759 .ciph_conf = 0,
760 .cbcmode = 0,
761 .decrypt = 0,
762 .consumed = 0,
763 .produced = 0,
764 .pad_descs = NULL,
765 .active = 0,
766 .done = 0,
767 .tcfg = NULL,
768 .prev_src = NULL,
769 .curr_src = NULL,
770 .unit_no = src_csum};
771 struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
772
773 unsigned int indata_ix = 0;
774
775 /* iovec accounting. */
776 int iniov_ix = 0;
777 int iniov_offset = 0;
778
779 /* Operation descriptor cfg traversal pointer. */
780 struct cryptocop_desc *odsc;
781
782 int failed = 0;
783 /* List heads for allocated descriptors. */
784 struct cryptocop_dma_desc out_cdesc_head = {0};
785 struct cryptocop_dma_desc in_cdesc_head = {0};
786
787 struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
788 struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
789
790 struct cryptocop_tfrm_ctx *output_tc = NULL;
791 void *iop_alloc_ptr;
792
793 assert(operation != NULL);
794 assert(int_op != NULL);
795
796 DEBUG(printk("cryptocop_setup_dma_list: start\n"));
797 DEBUG(print_cryptocop_operation(operation));
798
799 sess = get_session(operation->sid);
800 if (!sess) {
801 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
802 failed = -EINVAL;
803 goto error_cleanup;
804 }
805 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
806 if (!iop_alloc_ptr) {
807 DEBUG_API(printk("cryptocop_setup_dma_list: kmalloc cryptocop_int_operation\n"));
808 failed = -ENOMEM;
809 goto error_cleanup;
810 }
811 (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
812 DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
813 (*int_op)->alloc_ptr = iop_alloc_ptr;
814 DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
815
816 (*int_op)->sid = operation->sid;
817 (*int_op)->cdesc_out = NULL;
818 (*int_op)->cdesc_in = NULL;
819 (*int_op)->tdes_mode = cryptocop_3des_ede;
820 (*int_op)->csum_mode = cryptocop_csum_le;
821 (*int_op)->ddesc_out = NULL;
822 (*int_op)->ddesc_in = NULL;
823
824 /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
825 if (!tcfg) {
826 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
827 failed = -EINVAL;
828 goto error_cleanup;
829 }
830 while (tcfg) {
831 tctx = get_transform_ctx(sess, tcfg->tid);
832 if (!tctx) {
833 DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
834 failed = -EINVAL;
835 goto error_cleanup;
836 }
837 if (tcfg->inject_ix > operation->tfrm_op.outlen){
838 DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
839 failed = -EINVAL;
840 goto error_cleanup;
841 }
842 switch (tctx->init.alg){
843 case cryptocop_alg_mem2mem:
844 if (cipher_ctx.tcfg != NULL){
845 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
846 failed = -EINVAL;
847 goto error_cleanup;
848 }
849 /* mem2mem is handled as a NULL cipher. */
850 cipher_ctx.cbcmode = 0;
851 cipher_ctx.decrypt = 0;
852 cipher_ctx.blocklength = 1;
853 cipher_ctx.ciph_conf = 0;
854 cipher_ctx.unit_no = src_dma;
855 cipher_ctx.tcfg = tcfg;
856 cipher_ctx.tctx = tctx;
857 break;
858 case cryptocop_alg_des:
859 case cryptocop_alg_3des:
860 case cryptocop_alg_aes:
861 /* cipher */
862 if (cipher_ctx.tcfg != NULL){
863 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
864 failed = -EINVAL;
865 goto error_cleanup;
866 }
867 cipher_ctx.tcfg = tcfg;
868 cipher_ctx.tctx = tctx;
869 if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
870 cipher_ctx.decrypt = 1;
871 }
872 switch (tctx->init.cipher_mode) {
873 case cryptocop_cipher_mode_ecb:
874 cipher_ctx.cbcmode = 0;
875 break;
876 case cryptocop_cipher_mode_cbc:
877 cipher_ctx.cbcmode = 1;
878 break;
879 default:
880 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
881 failed = -EINVAL;
882 goto error_cleanup;
883 }
884 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
885 switch (tctx->init.alg){
886 case cryptocop_alg_des:
887 cipher_ctx.ciph_conf = 0;
888 cipher_ctx.unit_no = src_des;
889 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
890 break;
891 case cryptocop_alg_3des:
892 cipher_ctx.ciph_conf = 1;
893 cipher_ctx.unit_no = src_des;
894 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
895 break;
896 case cryptocop_alg_aes:
897 cipher_ctx.ciph_conf = 2;
898 cipher_ctx.unit_no = src_aes;
899 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
900 break;
901 default:
902 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
903 }
904 (*int_op)->tdes_mode = tctx->init.tdes_mode;
905 break;
906 case cryptocop_alg_md5:
907 case cryptocop_alg_sha1:
908 /* digest */
909 if (digest_ctx.tcfg != NULL){
910 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
911 failed = -EINVAL;
912 goto error_cleanup;
913 }
914 digest_ctx.tcfg = tcfg;
915 digest_ctx.tctx = tctx;
916 digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
917 switch (tctx->init.alg){
918 case cryptocop_alg_md5:
919 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
920 digest_ctx.unit_no = src_md5;
921 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
922 break;
923 case cryptocop_alg_sha1:
924 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
925 digest_ctx.unit_no = src_sha1;
926 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
927 break;
928 default:
929 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
930 }
931 break;
932 case cryptocop_alg_csum:
933 /* digest */
934 if (csum_ctx.tcfg != NULL){
935 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
936 failed = -EINVAL;
937 goto error_cleanup;
938 }
939 (*int_op)->csum_mode = tctx->init.csum_mode;
940 csum_ctx.tcfg = tcfg;
941 csum_ctx.tctx = tctx;
942 break;
943 default:
944 /* no algorithm. */
945 DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
946 failed = -EINVAL;
947 goto error_cleanup;
948 }
949 tcfg = tcfg->next;
950 }
951 /* Download key if a cipher is used. */
952 if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
953 struct cryptocop_dma_desc *key_desc = NULL;
954
955 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
956 if (failed) {
957 DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
958 goto error_cleanup;
959 }
960 current_out_cdesc->next = key_desc;
961 current_out_cdesc = key_desc;
962 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
963
964 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
965 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
966 struct cryptocop_dma_desc *iv_desc = NULL;
967
968 DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
969
970 failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
971 if (failed) {
972 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
973 goto error_cleanup;
974 }
975 current_out_cdesc->next = iv_desc;
976 current_out_cdesc = iv_desc;
977 indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
978 }
979 }
980
981 /* Process descriptors. */
982 odsc = operation->tfrm_op.desc;
983 while (odsc) {
984 struct cryptocop_desc_cfg *dcfg = odsc->cfg;
985 struct strcop_meta_out meta_out = {0};
986 size_t desc_len = odsc->length;
987 int active_count, eop_needed_count;
988
989 output_tc = NULL;
990
991 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
992
993 while (dcfg) {
994 struct cryptocop_tfrm_ctx *tc = NULL;
995
996 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
997 /* Get the local context for the transform and mark it as the output unit if it produces output. */
998 if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
999 tc = &digest_ctx;
1000 } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1001 tc = &cipher_ctx;
1002 } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1003 tc = &csum_ctx;
1004 }
1005 if (!tc) {
1006 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1007 failed = -EINVAL;
1008 goto error_cleanup;
1009 }
1010 if (tc->done) {
1011 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1012 failed = -EINVAL;
1013 goto error_cleanup;
1014 }
1015 if (!tc->active) {
1016 tc->start_ix = indata_ix;
1017 tc->active = 1;
1018 }
1019
1020 tc->previous_src = tc->current_src;
1021 tc->prev_src = tc->curr_src;
1022 /* Map source unit id to DMA source config. */
1023 switch (dcfg->src){
1024 case cryptocop_source_dma:
1025 tc->current_src = src_dma;
1026 break;
1027 case cryptocop_source_des:
1028 tc->current_src = src_des;
1029 break;
1030 case cryptocop_source_3des:
1031 tc->current_src = src_des;
1032 break;
1033 case cryptocop_source_aes:
1034 tc->current_src = src_aes;
1035 break;
1036 case cryptocop_source_md5:
1037 case cryptocop_source_sha1:
1038 case cryptocop_source_csum:
1039 case cryptocop_source_none:
1040 default:
1041 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1042 */
1043 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1044 failed = -EINVAL;
1045 goto error_cleanup;
1046 }
1047 if (tc->current_src != src_dma) {
1048 /* Find the unit we are sourcing from. */
1049 if (digest_ctx.unit_no == tc->current_src){
1050 tc->curr_src = &digest_ctx;
1051 } else if (cipher_ctx.unit_no == tc->current_src){
1052 tc->curr_src = &cipher_ctx;
1053 } else if (csum_ctx.unit_no == tc->current_src){
1054 tc->curr_src = &csum_ctx;
1055 }
1056 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1057 DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1058 failed = -EINVAL;
1059 goto error_cleanup;
1060 }
1061 } else {
1062 tc->curr_src = NULL;
1063 }
1064
1065 /* Detect source switch. */
1066 DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1067 if (tc->active && (tc->current_src != tc->previous_src)) {
1068 /* Only allow source switch when both the old source unit and the new one have
1069 * no pending data to process (i.e. the consumed length must be a multiple of the
1070 * transform blocklength). */
1071 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1072 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1073 ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1074 {
1075 DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1076 failed = -EINVAL;
1077 goto error_cleanup;
1078 }
1079 }
1080 /* Detect unit deactivation. */
1081 if (dcfg->last) {
1082 /* Length check of this is handled below. */
1083 tc->done = 1;
1084 }
1085 dcfg = dcfg->next;
1086 } /* while (dcfg) */
1087 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1088
1089 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1090 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1091 failed = -EINVAL;
1092 goto error_cleanup;
1093 }
1094 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1095 DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1096 failed = -EINVAL;
1097 goto error_cleanup;
1098 }
1099 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1100 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1101 failed = -EINVAL;
1102 goto error_cleanup;
1103 }
1104
1105 /* Update consumed and produced lengths.
1106
1107 The consumed length accounting here is actually cheating. If a unit source from DMA (or any
1108 other unit that process data in blocks of one octet) it is correct, but if it source from a
1109 block processing unit, i.e. a cipher, it will be temporarily incorrect at some times. However
1110 since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1111 unit has processed an exact multiple of its block length the end result will be correct.
1112 Beware that if the source change restriction change this code will need to be (much) reworked.
1113 */
1114 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1115
1116 if (csum_ctx.active) {
1117 csum_ctx.consumed += desc_len;
1118 if (csum_ctx.done) {
1119 csum_ctx.produced = 2;
1120 }
1121 DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1122 }
1123 if (digest_ctx.active) {
1124 digest_ctx.consumed += desc_len;
1125 if (digest_ctx.done) {
1126 if (digest_ctx.unit_no == src_md5) {
1127 digest_ctx.produced = MD5_STATE_LENGTH;
1128 } else {
1129 digest_ctx.produced = SHA1_STATE_LENGTH;
1130 }
1131 }
1132 DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1133 }
1134 if (cipher_ctx.active) {
1135 /* Ciphers are allowed only to source from DMA out. That is filtered above. */
1136 assert(cipher_ctx.current_src == src_dma);
1137 cipher_ctx.consumed += desc_len;
1138 cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1139 if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1140 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1141 }
1142 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1143 }
1144
1145 /* Setup the DMA out descriptors. */
1146 /* Configure the metadata. */
1147 active_count = 0;
1148 eop_needed_count = 0;
1149 if (cipher_ctx.active) {
1150 ++active_count;
1151 if (cipher_ctx.unit_no == src_dma){
1152 /* mem2mem */
1153 meta_out.ciphsel = src_none;
1154 } else {
1155 meta_out.ciphsel = cipher_ctx.current_src;
1156 }
1157 meta_out.ciphconf = cipher_ctx.ciph_conf;
1158 meta_out.cbcmode = cipher_ctx.cbcmode;
1159 meta_out.decrypt = cipher_ctx.decrypt;
1160 DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1161 if (cipher_ctx.done) ++eop_needed_count;
1162 } else {
1163 meta_out.ciphsel = src_none;
1164 }
1165
1166 if (digest_ctx.active) {
1167 ++active_count;
1168 meta_out.hashsel = digest_ctx.current_src;
1169 meta_out.hashconf = digest_ctx.hash_conf;
1170 meta_out.hashmode = 0; /* Explicit mode is not used here. */
1171 DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1172 if (digest_ctx.done) {
1173 assert(digest_ctx.pad_descs == NULL);
1174 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1175 if (failed) {
1176 DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1177 goto error_cleanup;
1178 }
1179 }
1180 } else {
1181 meta_out.hashsel = src_none;
1182 }
1183
1184 if (csum_ctx.active) {
1185 ++active_count;
1186 meta_out.csumsel = csum_ctx.current_src;
1187 if (csum_ctx.done) {
1188 assert(csum_ctx.pad_descs == NULL);
1189 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1190 if (failed) {
1191 DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1192 goto error_cleanup;
1193 }
1194 }
1195 } else {
1196 meta_out.csumsel = src_none;
1197 }
1198 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1199 /* Setup DMA out descriptors for the indata. */
1200 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1201 if (failed) {
1202 DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1203 goto error_cleanup;
1204 }
1205 /* Setup out EOP. If there are active units that are not done here they cannot get an EOP
1206 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1207 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1208 */
1209 assert(active_count >= eop_needed_count);
1210 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1211 if (eop_needed_count) {
1212 /* This means that the bulk operation (cipeher/m2m) is terminated. */
1213 if (active_count > 1) {
1214 /* Use zero length EOP descriptor. */
1215 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1216 struct strcop_meta_out ed_mo = {0};
1217 if (!ed) {
1218 DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1219 failed = -ENOMEM;
1220 goto error_cleanup;
1221 }
1222
1223 assert(cipher_ctx.active && cipher_ctx.done);
1224
1225 if (cipher_ctx.unit_no == src_dma){
1226 /* mem2mem */
1227 ed_mo.ciphsel = src_none;
1228 } else {
1229 ed_mo.ciphsel = cipher_ctx.current_src;
1230 }
1231 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1232 ed_mo.cbcmode = cipher_ctx.cbcmode;
1233 ed_mo.decrypt = cipher_ctx.decrypt;
1234
1235 ed->free_buf = NULL;
1236 ed->dma_descr->wait = 1;
1237 ed->dma_descr->out_eop = 1;
1238
1239 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1240 ed->dma_descr->after = ed->dma_descr->buf;
1241 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1242 current_out_cdesc->next = ed;
1243 current_out_cdesc = ed;
1244 } else {
1245 /* Set EOP in the current out descriptor since the only active module is
1246 * the one needing the EOP. */
1247
1248 current_out_cdesc->dma_descr->out_eop = 1;
1249 }
1250 }
1251
1252 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1253 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1254 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1255 indata_ix += odsc->length;
1256 odsc = odsc->next;
1257 } /* while (odsc) */ /* Process descriptors. */
1258 DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1259 if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1260 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1261 failed = -EINVAL;
1262 goto error_cleanup;
1263 }
1264 if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1265 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1266 failed = -EINVAL;
1267 goto error_cleanup;
1268 }
1269 if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1270 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1271 failed = -EINVAL;
1272 goto error_cleanup;
1273 }
1274
1275 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1276 if (failed){
1277 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1278 goto error_cleanup;
1279 }
1280 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1281 if (failed){
1282 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1283 goto error_cleanup;
1284 }
1285 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1286 if (failed){
1287 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1288 goto error_cleanup;
1289 }
1290
1291 DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1292 (*int_op)->cdesc_out = out_cdesc_head.next;
1293 (*int_op)->cdesc_in = in_cdesc_head.next;
1294 DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1295
1296 setup_descr_chain(out_cdesc_head.next);
1297 setup_descr_chain(in_cdesc_head.next);
1298
1299 /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1300 * last DMA out descriptor for EOL.
1301 */
1302 current_in_cdesc->dma_descr->intr = 1;
1303 current_in_cdesc->dma_descr->eol = 1;
1304 current_out_cdesc->dma_descr->eol = 1;
1305
1306 /* Setup DMA contexts. */
1307 (*int_op)->ctx_out.next = NULL;
1308 (*int_op)->ctx_out.eol = 1;
1309 (*int_op)->ctx_out.intr = 0;
1310 (*int_op)->ctx_out.store_mode = 0;
1311 (*int_op)->ctx_out.en = 0;
1312 (*int_op)->ctx_out.dis = 0;
1313 (*int_op)->ctx_out.md0 = 0;
1314 (*int_op)->ctx_out.md1 = 0;
1315 (*int_op)->ctx_out.md2 = 0;
1316 (*int_op)->ctx_out.md3 = 0;
1317 (*int_op)->ctx_out.md4 = 0;
1318 (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1319 (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1320
1321 (*int_op)->ctx_in.next = NULL;
1322 (*int_op)->ctx_in.eol = 1;
1323 (*int_op)->ctx_in.intr = 0;
1324 (*int_op)->ctx_in.store_mode = 0;
1325 (*int_op)->ctx_in.en = 0;
1326 (*int_op)->ctx_in.dis = 0;
1327 (*int_op)->ctx_in.md0 = 0;
1328 (*int_op)->ctx_in.md1 = 0;
1329 (*int_op)->ctx_in.md2 = 0;
1330 (*int_op)->ctx_in.md3 = 0;
1331 (*int_op)->ctx_in.md4 = 0;
1332
1333 (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1334 (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1335
1336 DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1337 return 0;
1338
1339 error_cleanup:
1340 {
1341 /* Free all allocated resources. */
1342 struct cryptocop_dma_desc *tmp_cdesc;
1343 while (digest_ctx.pad_descs){
1344 tmp_cdesc = digest_ctx.pad_descs->next;
1345 free_cdesc(digest_ctx.pad_descs);
1346 digest_ctx.pad_descs = tmp_cdesc;
1347 }
1348 while (csum_ctx.pad_descs){
1349 tmp_cdesc = csum_ctx.pad_descs->next;
1350 free_cdesc(csum_ctx.pad_descs);
1351 csum_ctx.pad_descs = tmp_cdesc;
1352 }
1353 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1354
1355 if (*int_op != NULL) delete_internal_operation(*int_op);
1356 }
1357 DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1358 return failed;
1359 }
1360
1361
1362 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1363 {
1364 void *ptr = iop->alloc_ptr;
1365 struct cryptocop_dma_desc *cd = iop->cdesc_out;
1366 struct cryptocop_dma_desc *next;
1367
1368 DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1369
1370 while (cd) {
1371 next = cd->next;
1372 free_cdesc(cd);
1373 cd = next;
1374 }
1375 cd = iop->cdesc_in;
1376 while (cd) {
1377 next = cd->next;
1378 free_cdesc(cd);
1379 cd = next;
1380 }
1381 kfree(ptr);
1382 }
1383
1384 #define MD5_MIN_PAD_LENGTH (9)
1385 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1386
1387 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1388 {
1389 size_t padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1390 unsigned char *p;
1391 int i;
1392 unsigned long long int bit_length = hashed_length << 3;
1393
1394 if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1395
1396 p = kmalloc(padlen, alloc_flag);
1397 if (!p) return -ENOMEM;
1398
1399 *p = 0x80;
1400 memset(p+1, 0, padlen - 1);
1401
1402 DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1403
1404 i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1405 while (bit_length != 0){
1406 p[i++] = bit_length % 0x100;
1407 bit_length >>= 8;
1408 }
1409
1410 *pad = (char*)p;
1411 *pad_length = padlen;
1412
1413 return 0;
1414 }
1415
1416 #define SHA1_MIN_PAD_LENGTH (9)
1417 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1418
1419 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1420 {
1421 size_t padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1422 unsigned char *p;
1423 int i;
1424 unsigned long long int bit_length = hashed_length << 3;
1425
1426 if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1427
1428 p = kmalloc(padlen, alloc_flag);
1429 if (!p) return -ENOMEM;
1430
1431 *p = 0x80;
1432 memset(p+1, 0, padlen - 1);
1433
1434 DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1435
1436 i = padlen - 1;
1437 while (bit_length != 0){
1438 p[i--] = bit_length % 0x100;
1439 bit_length >>= 8;
1440 }
1441
1442 *pad = (char*)p;
1443 *pad_length = padlen;
1444
1445 return 0;
1446 }
1447
1448
1449 static int transform_ok(struct cryptocop_transform_init *tinit)
1450 {
1451 switch (tinit->alg){
1452 case cryptocop_alg_csum:
1453 switch (tinit->csum_mode){
1454 case cryptocop_csum_le:
1455 case cryptocop_csum_be:
1456 break;
1457 default:
1458 DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1459 return -EINVAL;
1460 }
1461 case cryptocop_alg_mem2mem:
1462 case cryptocop_alg_md5:
1463 case cryptocop_alg_sha1:
1464 if (tinit->keylen != 0) {
1465 DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1466 return -EINVAL; /* This check is a bit strict. */
1467 }
1468 break;
1469 case cryptocop_alg_des:
1470 if (tinit->keylen != 64) {
1471 DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1472 return -EINVAL;
1473 }
1474 break;
1475 case cryptocop_alg_3des:
1476 if (tinit->keylen != 192) {
1477 DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1478 return -EINVAL;
1479 }
1480 break;
1481 case cryptocop_alg_aes:
1482 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1483 DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1484 return -EINVAL;
1485 }
1486 break;
1487 case cryptocop_no_alg:
1488 default:
1489 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1490 return -EINVAL;
1491 }
1492
1493 switch (tinit->alg){
1494 case cryptocop_alg_des:
1495 case cryptocop_alg_3des:
1496 case cryptocop_alg_aes:
1497 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1498 default:
1499 break;
1500 }
1501 return 0;
1502 }
1503
1504
1505 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1506 {
1507 struct cryptocop_session *sess;
1508 struct cryptocop_transform_init *tfrm_in = tinit;
1509 struct cryptocop_transform_init *tmp_in;
1510 int no_tfrms = 0;
1511 int i;
1512 unsigned long int flags;
1513
1514 init_stream_coprocessor(); /* For safety if we are called early */
1515
1516 while (tfrm_in){
1517 int err;
1518 ++no_tfrms;
1519 if ((err = transform_ok(tfrm_in))) {
1520 DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1521 return err;
1522 }
1523 tfrm_in = tfrm_in->next;
1524 }
1525 if (0 == no_tfrms) {
1526 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1527 return -EINVAL;
1528 }
1529
1530 sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1531 if (!sess){
1532 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1533 return -ENOMEM;
1534 }
1535
1536 sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1537 if (!sess->tfrm_ctx) {
1538 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1539 kfree(sess);
1540 return -ENOMEM;
1541 }
1542
1543 tfrm_in = tinit;
1544 for (i = 0; i < no_tfrms; i++){
1545 tmp_in = tfrm_in->next;
1546 while (tmp_in){
1547 if (tmp_in->tid == tfrm_in->tid) {
1548 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1549 kfree(sess->tfrm_ctx);
1550 kfree(sess);
1551 return -EINVAL;
1552 }
1553 tmp_in = tmp_in->next;
1554 }
1555 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1556 sess->tfrm_ctx[i].dec_key_set = 0;
1557 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1558
1559 tfrm_in = tfrm_in->next;
1560 }
1561 sess->tfrm_ctx[i-1].next = NULL;
1562
1563 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1564 sess->sid = next_sid;
1565 next_sid++;
1566 /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1567 * OTOH 2^64 is a really large number of session. */
1568 if (next_sid == 0) next_sid = 1;
1569
1570 /* Prepend to session list. */
1571 sess->next = cryptocop_sessions;
1572 cryptocop_sessions = sess;
1573 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1574 *sid = sess->sid;
1575 return 0;
1576 }
1577
1578
1579 int cryptocop_free_session(cryptocop_session_id sid)
1580 {
1581 struct cryptocop_transform_ctx *tc;
1582 struct cryptocop_session *sess = NULL;
1583 struct cryptocop_session *psess = NULL;
1584 unsigned long int flags;
1585 int i;
1586 LIST_HEAD(remove_list);
1587 struct list_head *node, *tmp;
1588 struct cryptocop_prio_job *pj;
1589
1590 DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1591
1592 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1593 sess = cryptocop_sessions;
1594 while (sess && sess->sid != sid){
1595 psess = sess;
1596 sess = sess->next;
1597 }
1598 if (sess){
1599 if (psess){
1600 psess->next = sess->next;
1601 } else {
1602 cryptocop_sessions = sess->next;
1603 }
1604 }
1605 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1606
1607 if (!sess) return -EINVAL;
1608
1609 /* Remove queued jobs. */
1610 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1611
1612 for (i = 0; i < cryptocop_prio_no_prios; i++){
1613 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1614 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1615 pj = list_entry(node, struct cryptocop_prio_job, node);
1616 if (pj->oper->sid == sid) {
1617 list_move_tail(node, &remove_list);
1618 }
1619 }
1620 }
1621 }
1622 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1623
1624 list_for_each_safe(node, tmp, &remove_list) {
1625 list_del(node);
1626 pj = list_entry(node, struct cryptocop_prio_job, node);
1627 pj->oper->operation_status = -EAGAIN; /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1628 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1629 pj->oper->cb(pj->oper, pj->oper->cb_data);
1630 delete_internal_operation(pj->iop);
1631 kfree(pj);
1632 }
1633
1634 tc = sess->tfrm_ctx;
1635 /* Erase keying data. */
1636 while (tc){
1637 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1638 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1639 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1640 tc = tc->next;
1641 }
1642 kfree(sess->tfrm_ctx);
1643 kfree(sess);
1644
1645 return 0;
1646 }
1647
1648 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1649 {
1650 struct cryptocop_session *sess;
1651 unsigned long int flags;
1652
1653 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1654 sess = cryptocop_sessions;
1655 while (sess && (sess->sid != sid)){
1656 sess = sess->next;
1657 }
1658 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1659
1660 return sess;
1661 }
1662
1663 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1664 {
1665 struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1666
1667 DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1668 assert(sess != NULL);
1669 while (tc && tc->init.tid != tid){
1670 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1671 tc = tc->next;
1672 }
1673 DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1674 return tc;
1675 }
1676
1677
1678
1679 /* The AES s-transform matrix (s-box). */
1680 static const u8 aes_sbox[256] = {
1681 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, 171, 118,
1682 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, 156, 164, 114, 192,
1683 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, 229, 241, 113, 216, 49, 21,
1684 4, 199, 35, 195, 24, 150, 5, 154, 7, 18, 128, 226, 235, 39, 178, 117,
1685 9, 131, 44, 26, 27, 110, 90, 160, 82, 59, 214, 179, 41, 227, 47, 132,
1686 83, 209, 0, 237, 32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207,
1687 208, 239, 170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168,
1688 81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243, 210,
1689 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100, 93, 25, 115,
1690 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184, 20, 222, 94, 11, 219,
1691 224, 50, 58, 10, 73, 6, 36, 92, 194, 211, 172, 98, 145, 149, 228, 121,
1692 231, 200, 55, 109, 141, 213, 78, 169, 108, 86, 244, 234, 101, 122, 174, 8,
1693 186, 120, 37, 46, 28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138,
1694 112, 62, 181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158,
1695 225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223,
1696 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84, 187, 22
1697 };
1698
1699 /* AES has a 32 bit word round constants for each round in the
1700 * key schedule. round_constant[i] is really Rcon[i+1] in FIPS187.
1701 */
1702 static u32 round_constant[11] = {
1703 0x01000000, 0x02000000, 0x04000000, 0x08000000,
1704 0x10000000, 0x20000000, 0x40000000, 0x80000000,
1705 0x1B000000, 0x36000000, 0x6C000000
1706 };
1707
1708 /* Apply the s-box to each of the four occtets in w. */
1709 static u32 aes_ks_subword(const u32 w)
1710 {
1711 u8 bytes[4];
1712
1713 *(u32*)(&bytes[0]) = w;
1714 bytes[0] = aes_sbox[bytes[0]];
1715 bytes[1] = aes_sbox[bytes[1]];
1716 bytes[2] = aes_sbox[bytes[2]];
1717 bytes[3] = aes_sbox[bytes[3]];
1718 return *(u32*)(&bytes[0]);
1719 }
1720
1721 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1722 * (Note that AES words are 32 bit long)
1723 *
1724 * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1725 * word temp
1726 * i = 0
1727 * while (i < Nk) {
1728 * w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1729 * i = i + 1
1730 * }
1731 * i = Nk
1732 *
1733 * while (i < (Nb * (Nr + 1))) {
1734 * temp = w[i - 1]
1735 * if ((i mod Nk) == 0) {
1736 * temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1737 * }
1738 * else if ((Nk > 6) && ((i mod Nk) == 4)) {
1739 * temp = SubWord(temp)
1740 * }
1741 * w[i] = w[i - Nk] xor temp
1742 * }
1743 * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1744 * SubWord(t) applies the AES s-box individually to each octet
1745 * in a 32 bit word.
1746 *
1747 * For AES Nk can have the values 4, 6, and 8 (corresponding to
1748 * values for Nr of 10, 12, and 14). Nb is always 4.
1749 *
1750 * To construct w[i], w[i - 1] and w[i - Nk] must be
1751 * available. Consequently we must keep a state of the last Nk words
1752 * to be able to create the last round keys.
1753 */
1754 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength)
1755 {
1756 u32 temp;
1757 u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1758 u8 w_last_ix;
1759 int i;
1760 u8 nr, nk;
1761
1762 switch (keylength){
1763 case 128:
1764 nk = 4;
1765 nr = 10;
1766 break;
1767 case 192:
1768 nk = 6;
1769 nr = 12;
1770 break;
1771 case 256:
1772 nk = 8;
1773 nr = 14;
1774 break;
1775 default:
1776 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1777 };
1778
1779 /* Need to do host byte order correction here since key is byte oriented and the
1780 * kx algorithm is word (u32) oriented. */
1781 for (i = 0; i < nk; i+=1) {
1782 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1783 }
1784
1785 i = (int)nk;
1786 w_last_ix = i - 1;
1787 while (i < (4 * (nr + 2))) {
1788 temp = w_ring[w_last_ix];
1789 if (!(i % nk)) {
1790 /* RotWord(temp) */
1791 temp = (temp << 8) | (temp >> 24);
1792 temp = aes_ks_subword(temp);
1793 temp ^= round_constant[i/nk - 1];
1794 } else if ((nk > 6) && ((i % nk) == 4)) {
1795 temp = aes_ks_subword(temp);
1796 }
1797 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1798 temp ^= w_ring[w_last_ix];
1799 w_ring[w_last_ix] = temp;
1800
1801 /* We need the round keys for round Nr+1 and Nr+2 (round key
1802 * Nr+2 is the round key beyond the last one used when
1803 * encrypting). Rounds are numbered starting from 0, Nr=10
1804 * implies 11 rounds are used in encryption/decryption.
1805 */
1806 if (i >= (4 * nr)) {
1807 /* Need to do host byte order correction here, the key
1808 * is byte oriented. */
1809 *(u32*)dec_key = cpu_to_be32(temp);
1810 dec_key += 4;
1811 }
1812 ++i;
1813 }
1814 }
1815
1816
1817 /**** Job/operation management. ****/
1818
1819 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1820 {
1821 return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1822 }
1823
1824 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1825 {
1826 return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1827 }
1828
1829 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1830 {
1831 return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1832 }
1833
1834 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1835 {
1836 int ret;
1837 struct cryptocop_prio_job *pj = NULL;
1838 unsigned long int flags;
1839
1840 DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1841
1842 if (!operation || !operation->cb){
1843 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1844 return -EINVAL;
1845 }
1846
1847 if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1848 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1849 return ret;
1850 }
1851 assert(pj != NULL);
1852
1853 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1854 list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1855 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1856
1857 /* Make sure a job is running */
1858 cryptocop_start_job();
1859 return 0;
1860 }
1861
1862 static void cryptocop_do_tasklet(unsigned long unused);
1863 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1864
1865 static void cryptocop_do_tasklet(unsigned long unused)
1866 {
1867 struct list_head *node;
1868 struct cryptocop_prio_job *pj = NULL;
1869 unsigned long flags;
1870
1871 DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1872
1873 do {
1874 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1875 if (!list_empty(&cryptocop_completed_jobs)){
1876 node = cryptocop_completed_jobs.next;
1877 list_del(node);
1878 pj = list_entry(node, struct cryptocop_prio_job, node);
1879 } else {
1880 pj = NULL;
1881 }
1882 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1883 if (pj) {
1884 assert(pj->oper != NULL);
1885
1886 /* Notify consumer of operation completeness. */
1887 DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1888
1889 pj->oper->operation_status = 0; /* Job is completed. */
1890 pj->oper->cb(pj->oper, pj->oper->cb_data);
1891 delete_internal_operation(pj->iop);
1892 kfree(pj);
1893 }
1894 } while (pj != NULL);
1895
1896 DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1897 }
1898
1899 static irqreturn_t
1900 dma_done_interrupt(int irq, void *dev_id)
1901 {
1902 struct cryptocop_prio_job *done_job;
1903 reg_dma_rw_ack_intr ack_intr = {
1904 .data = 1,
1905 };
1906
1907 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1908
1909 DEBUG(printk("cryptocop DMA done\n"));
1910
1911 spin_lock(&running_job_lock);
1912 if (cryptocop_running_job == NULL){
1913 printk("stream co-processor got interrupt when not busy\n");
1914 spin_unlock(&running_job_lock);
1915 return IRQ_HANDLED;
1916 }
1917 done_job = cryptocop_running_job;
1918 cryptocop_running_job = NULL;
1919 spin_unlock(&running_job_lock);
1920
1921 /* Start processing a job. */
1922 if (!spin_trylock(&cryptocop_process_lock)){
1923 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1924 } else {
1925 cryptocop_start_job();
1926 spin_unlock(&cryptocop_process_lock);
1927 }
1928
1929 done_job->oper->operation_status = 0; /* Job is completed. */
1930 if (done_job->oper->fast_callback){
1931 /* This operation wants callback from interrupt. */
1932 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1933 delete_internal_operation(done_job->iop);
1934 kfree(done_job);
1935 } else {
1936 spin_lock(&cryptocop_completed_jobs_lock);
1937 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1938 spin_unlock(&cryptocop_completed_jobs_lock);
1939 tasklet_schedule(&cryptocop_tasklet);
1940 }
1941
1942 DEBUG(printk("cryptocop leave irq handler\n"));
1943 return IRQ_HANDLED;
1944 }
1945
1946
1947 /* Setup interrupts and DMA channels. */
1948 static int init_cryptocop(void)
1949 {
1950 unsigned long flags;
1951 reg_dma_rw_cfg dma_cfg = {.en = 1};
1952 reg_dma_rw_intr_mask intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1953 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
1954 reg_strcop_rw_cfg strcop_cfg = {
1955 .ipend = regk_strcop_little,
1956 .td1 = regk_strcop_e,
1957 .td2 = regk_strcop_d,
1958 .td3 = regk_strcop_e,
1959 .ignore_sync = 0,
1960 .en = 1
1961 };
1962
1963 if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1964 "stream co-processor DMA", NULL))
1965 panic("request_irq stream co-processor irq dma9");
1966
1967 (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1968 0, dma_strp);
1969 (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1970 0, dma_strp);
1971
1972 local_irq_save(flags);
1973
1974 /* Reset and enable the cryptocop. */
1975 strcop_cfg.en = 0;
1976 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1977 strcop_cfg.en = 1;
1978 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1979
1980 /* Enable DMAs. */
1981 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1982 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1983
1984 /* Set up wordsize = 4 for DMAs. */
1985 DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1986 DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1987
1988 /* Enable interrupts. */
1989 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1990
1991 /* Clear intr ack. */
1992 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1993
1994 local_irq_restore(flags);
1995
1996 return 0;
1997 }
1998
1999 /* Free used cryptocop hw resources (interrupt and DMA channels). */
2000 static void release_cryptocop(void)
2001 {
2002 unsigned long flags;
2003 reg_dma_rw_cfg dma_cfg = {.en = 0};
2004 reg_dma_rw_intr_mask intr_mask_in = {0};
2005 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
2006
2007 local_irq_save(flags);
2008
2009 /* Clear intr ack. */
2010 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2011
2012 /* Disable DMAs. */
2013 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2014 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2015
2016 /* Disable interrupts. */
2017 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2018
2019 local_irq_restore(flags);
2020
2021 free_irq(DMA_IRQ, NULL);
2022
2023 (void)crisv32_free_dma(OUT_DMA);
2024 (void)crisv32_free_dma(IN_DMA);
2025 }
2026
2027
2028 /* Init job queue. */
2029 static int cryptocop_job_queue_init(void)
2030 {
2031 int i;
2032
2033 INIT_LIST_HEAD(&cryptocop_completed_jobs);
2034
2035 for (i = 0; i < cryptocop_prio_no_prios; i++){
2036 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2037 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2038 }
2039 return 0;
2040 }
2041
2042
2043 static void cryptocop_job_queue_close(void)
2044 {
2045 struct list_head *node, *tmp;
2046 struct cryptocop_prio_job *pj = NULL;
2047 unsigned long int process_flags, flags;
2048 int i;
2049
2050 /* FIXME: This is as yet untested code. */
2051
2052 /* Stop strcop from getting an operation to process while we are closing the
2053 module. */
2054 spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2055
2056 /* Empty the job queue. */
2057 for (i = 0; i < cryptocop_prio_no_prios; i++){
2058 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2059 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2060 pj = list_entry(node, struct cryptocop_prio_job, node);
2061 list_del(node);
2062
2063 /* Call callback to notify consumer of job removal. */
2064 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2065 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2066 pj->oper->cb(pj->oper, pj->oper->cb_data);
2067
2068 delete_internal_operation(pj->iop);
2069 kfree(pj);
2070 }
2071 }
2072 }
2073 spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2074
2075 /* Remove the running job, if any. */
2076 spin_lock_irqsave(&running_job_lock, flags);
2077 if (cryptocop_running_job){
2078 reg_strcop_rw_cfg rw_cfg;
2079 reg_dma_rw_cfg dma_out_cfg, dma_in_cfg;
2080
2081 /* Stop DMA. */
2082 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2083 dma_out_cfg.en = regk_dma_no;
2084 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2085
2086 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2087 dma_in_cfg.en = regk_dma_no;
2088 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2089
2090 /* Disble the cryptocop. */
2091 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2092 rw_cfg.en = 0;
2093 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2094
2095 pj = cryptocop_running_job;
2096 cryptocop_running_job = NULL;
2097
2098 /* Call callback to notify consumer of job removal. */
2099 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2100 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2101 pj->oper->cb(pj->oper, pj->oper->cb_data);
2102
2103 delete_internal_operation(pj->iop);
2104 kfree(pj);
2105 }
2106 spin_unlock_irqrestore(&running_job_lock, flags);
2107
2108 /* Remove completed jobs, if any. */
2109 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2110
2111 list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2112 pj = list_entry(node, struct cryptocop_prio_job, node);
2113 list_del(node);
2114 /* Call callback to notify consumer of job removal. */
2115 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2116 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2117 pj->oper->cb(pj->oper, pj->oper->cb_data);
2118
2119 delete_internal_operation(pj->iop);
2120 kfree(pj);
2121 }
2122 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2123 }
2124
2125
2126 static void cryptocop_start_job(void)
2127 {
2128 int i;
2129 struct cryptocop_prio_job *pj;
2130 unsigned long int flags;
2131 unsigned long int running_job_flags;
2132 reg_strcop_rw_cfg rw_cfg = {.en = 1, .ignore_sync = 0};
2133
2134 DEBUG(printk("cryptocop_start_job: entering\n"));
2135
2136 spin_lock_irqsave(&running_job_lock, running_job_flags);
2137 if (cryptocop_running_job != NULL){
2138 /* Already running. */
2139 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2140 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2141 return;
2142 }
2143 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2144
2145 /* Check the queues in priority order. */
2146 for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2147 if (i == cryptocop_prio_no_prios) {
2148 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2149 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2150 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2151 return; /* No jobs to run */
2152 }
2153 DEBUG(printk("starting job for prio %d\n", i));
2154
2155 /* TODO: Do not starve lower priority jobs. Let in a lower
2156 * prio job for every N-th processed higher prio job or some
2157 * other scheduling policy. This could reasonably be
2158 * tweakable since the optimal balance would depend on the
2159 * type of load on the system. */
2160
2161 /* Pull the DMA lists from the job and start the DMA client. */
2162 pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2163 list_del(&pj->node);
2164 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2165 cryptocop_running_job = pj;
2166
2167 /* Set config register (3DES and CSUM modes). */
2168 switch (pj->iop->tdes_mode){
2169 case cryptocop_3des_eee:
2170 rw_cfg.td1 = regk_strcop_e;
2171 rw_cfg.td2 = regk_strcop_e;
2172 rw_cfg.td3 = regk_strcop_e;
2173 break;
2174 case cryptocop_3des_eed:
2175 rw_cfg.td1 = regk_strcop_e;
2176 rw_cfg.td2 = regk_strcop_e;
2177 rw_cfg.td3 = regk_strcop_d;
2178 break;
2179 case cryptocop_3des_ede:
2180 rw_cfg.td1 = regk_strcop_e;
2181 rw_cfg.td2 = regk_strcop_d;
2182 rw_cfg.td3 = regk_strcop_e;
2183 break;
2184 case cryptocop_3des_edd:
2185 rw_cfg.td1 = regk_strcop_e;
2186 rw_cfg.td2 = regk_strcop_d;
2187 rw_cfg.td3 = regk_strcop_d;
2188 break;
2189 case cryptocop_3des_dee:
2190 rw_cfg.td1 = regk_strcop_d;
2191 rw_cfg.td2 = regk_strcop_e;
2192 rw_cfg.td3 = regk_strcop_e;
2193 break;
2194 case cryptocop_3des_ded:
2195 rw_cfg.td1 = regk_strcop_d;
2196 rw_cfg.td2 = regk_strcop_e;
2197 rw_cfg.td3 = regk_strcop_d;
2198 break;
2199 case cryptocop_3des_dde:
2200 rw_cfg.td1 = regk_strcop_d;
2201 rw_cfg.td2 = regk_strcop_d;
2202 rw_cfg.td3 = regk_strcop_e;
2203 break;
2204 case cryptocop_3des_ddd:
2205 rw_cfg.td1 = regk_strcop_d;
2206 rw_cfg.td2 = regk_strcop_d;
2207 rw_cfg.td3 = regk_strcop_d;
2208 break;
2209 default:
2210 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2211 }
2212 switch (pj->iop->csum_mode){
2213 case cryptocop_csum_le:
2214 rw_cfg.ipend = regk_strcop_little;
2215 break;
2216 case cryptocop_csum_be:
2217 rw_cfg.ipend = regk_strcop_big;
2218 break;
2219 default:
2220 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2221 }
2222 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2223
2224 DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2225 "ctx_in: 0x%p, phys: 0x%p\n"
2226 "ctx_out: 0x%p, phys: 0x%p\n",
2227 pj,
2228 &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2229 &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2230
2231 /* Start input DMA. */
2232 flush_dma_context(&pj->iop->ctx_in);
2233 DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2234
2235 /* Start output DMA. */
2236 DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2237
2238 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2239 DEBUG(printk("cryptocop_start_job: exiting\n"));
2240 }
2241
2242
2243 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2244 {
2245 int err;
2246 int alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2247 void *iop_alloc_ptr = NULL;
2248
2249 *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2250 if (!*pj) return -ENOMEM;
2251
2252 DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2253
2254 (*pj)->oper = operation;
2255 DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n", (*pj)->oper->cb, (*pj)->oper->cb_data));
2256
2257 if (operation->use_dmalists) {
2258 DEBUG(print_user_dma_lists(&operation->list_op));
2259 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2260 DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2261 kfree(*pj);
2262 return -EINVAL;
2263 }
2264 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2265 if (!iop_alloc_ptr) {
2266 DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2267 kfree(*pj);
2268 return -ENOMEM;
2269 }
2270 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2271 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2272 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2273 (*pj)->iop->sid = operation->sid;
2274 (*pj)->iop->cdesc_out = NULL;
2275 (*pj)->iop->cdesc_in = NULL;
2276 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2277 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2278 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2279 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2280
2281 /* Setup DMA contexts. */
2282 (*pj)->iop->ctx_out.next = NULL;
2283 (*pj)->iop->ctx_out.eol = 1;
2284 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2285 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2286
2287 (*pj)->iop->ctx_in.next = NULL;
2288 (*pj)->iop->ctx_in.eol = 1;
2289 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2290 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2291 } else {
2292 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2293 DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2294 kfree(*pj);
2295 return err;
2296 }
2297 }
2298 DEBUG(print_dma_descriptors((*pj)->iop));
2299
2300 DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2301
2302 return 0;
2303 }
2304
2305 static int cryptocop_open(struct inode *inode, struct file *filp)
2306 {
2307 int p = iminor(inode);
2308
2309 if (p != CRYPTOCOP_MINOR) return -EINVAL;
2310
2311 filp->private_data = NULL;
2312 return 0;
2313 }
2314
2315
2316 static int cryptocop_release(struct inode *inode, struct file *filp)
2317 {
2318 struct cryptocop_private *dev = filp->private_data;
2319 struct cryptocop_private *dev_next;
2320
2321 while (dev){
2322 dev_next = dev->next;
2323 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2324 (void)cryptocop_free_session(dev->sid);
2325 }
2326 kfree(dev);
2327 dev = dev_next;
2328 }
2329
2330 return 0;
2331 }
2332
2333
2334 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2335 unsigned int cmd, unsigned long arg)
2336 {
2337 struct cryptocop_private *dev = filp->private_data;
2338 struct cryptocop_private *prev_dev = NULL;
2339 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2340 struct strcop_session_op sop;
2341 int err;
2342
2343 DEBUG(printk("cryptocop_ioctl_close_session\n"));
2344
2345 if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2346 return -EFAULT;
2347 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2348 if (err) return -EFAULT;
2349
2350 while (dev && (dev->sid != sop.ses_id)) {
2351 prev_dev = dev;
2352 dev = dev->next;
2353 }
2354 if (dev){
2355 if (prev_dev){
2356 prev_dev->next = dev->next;
2357 } else {
2358 filp->private_data = dev->next;
2359 }
2360 err = cryptocop_free_session(dev->sid);
2361 if (err) return -EFAULT;
2362 } else {
2363 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2364 return -EINVAL;
2365 }
2366 return 0;
2367 }
2368
2369
2370 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2371 {
2372 struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2373
2374 DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2375
2376 jc->processed = 1;
2377 wake_up(&cryptocop_ioc_process_wq);
2378 }
2379
2380
2381 #define CRYPTOCOP_IOCTL_CIPHER_TID (1)
2382 #define CRYPTOCOP_IOCTL_DIGEST_TID (2)
2383 #define CRYPTOCOP_IOCTL_CSUM_TID (3)
2384
2385 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2386 {
2387 size_t ch_ix = 0;
2388
2389 if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2390 if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2391 if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2392
2393 DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2394 return ch_ix;
2395 }
2396
2397
2398 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2399 {
2400 size_t ch_ix = INT_MAX;
2401 size_t tmp_ix = 0;
2402
2403 if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2404 if (crp_op->cipher_start > ix) {
2405 ch_ix = crp_op->cipher_start;
2406 } else {
2407 ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2408 }
2409 }
2410 if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2411 if (crp_op->digest_start > ix) {
2412 tmp_ix = crp_op->digest_start;
2413 } else {
2414 tmp_ix = crp_op->digest_start + crp_op->digest_len;
2415 }
2416 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2417 }
2418 if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2419 if (crp_op->csum_start > ix) {
2420 tmp_ix = crp_op->csum_start;
2421 } else {
2422 tmp_ix = crp_op->csum_start + crp_op->csum_len;
2423 }
2424 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2425 }
2426 if (ch_ix == INT_MAX) ch_ix = ix;
2427 DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2428 return ch_ix;
2429 }
2430
2431
2432 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2433 * Return -1 for ok, 0 for fail. */
2434 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2435 {
2436 int tmplen;
2437
2438 assert(iov != NULL);
2439 assert(iovix != NULL);
2440 assert(pages != NULL);
2441 assert(pageix != NULL);
2442 assert(pageoffset != NULL);
2443
2444 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2445
2446 while (map_length > 0){
2447 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2448 if (*iovix >= iovlen){
2449 DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2450 return 0;
2451 }
2452 if (*pageix >= nopages){
2453 DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2454 return 0;
2455 }
2456 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2457 tmplen = PAGE_SIZE - *pageoffset;
2458 if (tmplen < map_length){
2459 (*pageoffset) = 0;
2460 (*pageix)++;
2461 } else {
2462 tmplen = map_length;
2463 (*pageoffset) += map_length;
2464 }
2465 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2466 iov[*iovix].iov_len = tmplen;
2467 map_length -= tmplen;
2468 (*iovix)++;
2469 }
2470 DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2471 return -1;
2472 }
2473
2474
2475
2476 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2477 {
2478 int i;
2479 struct cryptocop_private *dev = filp->private_data;
2480 struct strcop_crypto_op *crp_oper = (struct strcop_crypto_op *)arg;
2481 struct strcop_crypto_op oper = {0};
2482 int err = 0;
2483 struct cryptocop_operation *cop = NULL;
2484
2485 struct ioctl_job_cb_ctx *jc = NULL;
2486
2487 struct page **inpages = NULL;
2488 struct page **outpages = NULL;
2489 int noinpages = 0;
2490 int nooutpages = 0;
2491
2492 struct cryptocop_desc descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2493 * can get connected/disconnected on different places in the indata. */
2494 struct cryptocop_desc_cfg dcfgs[5*3];
2495 int desc_ix = 0;
2496 int dcfg_ix = 0;
2497 struct cryptocop_tfrm_cfg ciph_tcfg = {0};
2498 struct cryptocop_tfrm_cfg digest_tcfg = {0};
2499 struct cryptocop_tfrm_cfg csum_tcfg = {0};
2500
2501 unsigned char *digest_result = NULL;
2502 int digest_length = 0;
2503 int cblocklen = 0;
2504 unsigned char csum_result[CSUM_BLOCK_LENGTH];
2505 struct cryptocop_session *sess;
2506
2507 int iovlen = 0;
2508 int iovix = 0;
2509 int pageix = 0;
2510 int pageoffset = 0;
2511
2512 size_t prev_ix = 0;
2513 size_t next_ix;
2514
2515 int cipher_active, digest_active, csum_active;
2516 int end_digest, end_csum;
2517 int digest_done = 0;
2518 int cipher_done = 0;
2519 int csum_done = 0;
2520
2521 DEBUG(printk("cryptocop_ioctl_process\n"));
2522
2523 if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2524 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2525 return -EFAULT;
2526 }
2527 if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2528 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2529 return -EFAULT;
2530 }
2531 DEBUG(print_strcop_crypto_op(&oper));
2532
2533 while (dev && dev->sid != oper.ses_id) dev = dev->next;
2534 if (!dev){
2535 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2536 return -EINVAL;
2537 }
2538
2539 /* Check buffers. */
2540 if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2541 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2542 return -EINVAL;
2543 }
2544
2545 if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2546 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2547 return -EFAULT;
2548 }
2549 if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2550 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2551 return -EFAULT;
2552 }
2553
2554 cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2555 if (!cop) {
2556 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2557 return -ENOMEM;
2558 }
2559 jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2560 if (!jc) {
2561 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2562 err = -ENOMEM;
2563 goto error_cleanup;
2564 }
2565 jc->processed = 0;
2566
2567 cop->cb_data = jc;
2568 cop->cb = ioctl_process_job_callback;
2569 cop->operation_status = 0;
2570 cop->use_dmalists = 0;
2571 cop->in_interrupt = 0;
2572 cop->fast_callback = 0;
2573 cop->tfrm_op.tfrm_cfg = NULL;
2574 cop->tfrm_op.desc = NULL;
2575 cop->tfrm_op.indata = NULL;
2576 cop->tfrm_op.incount = 0;
2577 cop->tfrm_op.inlen = 0;
2578 cop->tfrm_op.outdata = NULL;
2579 cop->tfrm_op.outcount = 0;
2580 cop->tfrm_op.outlen = 0;
2581
2582 sess = get_session(oper.ses_id);
2583 if (!sess){
2584 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2585 kfree(cop);
2586 kfree(jc);
2587 return -EINVAL;
2588 }
2589
2590 if (oper.do_cipher) {
2591 unsigned int cipher_outlen = 0;
2592 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2593 if (!tc) {
2594 DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2595 err = -EINVAL;
2596 goto error_cleanup;
2597 }
2598 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2599 ciph_tcfg.inject_ix = 0;
2600 ciph_tcfg.flags = 0;
2601 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2602 DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2603 kfree(cop);
2604 kfree(jc);
2605 return -EINVAL;
2606 }
2607 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2608 if (oper.cipher_len % cblocklen) {
2609 kfree(cop);
2610 kfree(jc);
2611 DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2612 return -EINVAL;
2613 }
2614 cipher_outlen = oper.cipher_len;
2615 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2616 if (oper.cipher_explicit) {
2617 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2618 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2619 } else {
2620 cipher_outlen = oper.cipher_len - cblocklen;
2621 }
2622 } else {
2623 if (oper.cipher_explicit){
2624 kfree(cop);
2625 kfree(jc);
2626 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2627 return -EINVAL;
2628 }
2629 }
2630 if (oper.cipher_outlen != cipher_outlen) {
2631 kfree(cop);
2632 kfree(jc);
2633 DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2634 return -EINVAL;
2635 }
2636
2637 if (oper.decrypt){
2638 ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2639 } else {
2640 ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2641 }
2642 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2643 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2644 }
2645 if (oper.do_digest){
2646 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2647 if (!tc) {
2648 DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2649 err = -EINVAL;
2650 goto error_cleanup;
2651 }
2652 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2653 digest_result = kmalloc(digest_length, GFP_KERNEL);
2654 if (!digest_result) {
2655 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2656 err = -EINVAL;
2657 goto error_cleanup;
2658 }
2659 DEBUG(memset(digest_result, 0xff, digest_length));
2660
2661 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2662 digest_tcfg.inject_ix = 0;
2663 ciph_tcfg.inject_ix += digest_length;
2664 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2665 DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2666 err = -EINVAL;
2667 goto error_cleanup;
2668 }
2669
2670 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2671 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2672 }
2673 if (oper.do_csum){
2674 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2675 csum_tcfg.inject_ix = digest_length;
2676 ciph_tcfg.inject_ix += 2;
2677
2678 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2679 DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2680 kfree(cop);
2681 kfree(jc);
2682 return -EINVAL;
2683 }
2684
2685 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2686 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2687 }
2688
2689 prev_ix = first_cfg_change_ix(&oper);
2690 if (prev_ix > oper.inlen) {
2691 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2692 nooutpages = noinpages = 0;
2693 err = -EINVAL;
2694 goto error_cleanup;
2695 }
2696 DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2697
2698 /* Map user pages for in and out data of the operation. */
2699 noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2700 DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2701 inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2702 if (!inpages){
2703 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2704 nooutpages = noinpages = 0;
2705 err = -ENOMEM;
2706 goto error_cleanup;
2707 }
2708 if (oper.do_cipher){
2709 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2710 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2711 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2712 if (!outpages){
2713 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2714 nooutpages = noinpages = 0;
2715 err = -ENOMEM;
2716 goto error_cleanup;
2717 }
2718 }
2719
2720 /* Acquire the mm page semaphore. */
2721 down_read(&current->mm->mmap_sem);
2722
2723 err = get_user_pages(current,
2724 current->mm,
2725 (unsigned long int)(oper.indata + prev_ix),
2726 noinpages,
2727 0, /* read access only for in data */
2728 0, /* no force */
2729 inpages,
2730 NULL);
2731
2732 if (err < 0) {
2733 up_read(&current->mm->mmap_sem);
2734 nooutpages = noinpages = 0;
2735 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2736 goto error_cleanup;
2737 }
2738 noinpages = err;
2739 if (oper.do_cipher){
2740 err = get_user_pages(current,
2741 current->mm,
2742 (unsigned long int)oper.cipher_outdata,
2743 nooutpages,
2744 1, /* write access for out data */
2745 0, /* no force */
2746 outpages,
2747 NULL);
2748 up_read(&current->mm->mmap_sem);
2749 if (err < 0) {
2750 nooutpages = 0;
2751 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2752 goto error_cleanup;
2753 }
2754 nooutpages = err;
2755 } else {
2756 up_read(&current->mm->mmap_sem);
2757 }
2758
2759 /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2760 * csum output and splits when units are (dis-)connected. */
2761 cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2762 cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2763 if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2764 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2765 err = -ENOMEM;
2766 goto error_cleanup;
2767 }
2768
2769 cop->tfrm_op.inlen = oper.inlen - prev_ix;
2770 cop->tfrm_op.outlen = 0;
2771 if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2772 if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2773 if (oper.do_csum) cop->tfrm_op.outlen += 2;
2774
2775 /* Setup the in iovecs. */
2776 cop->tfrm_op.incount = noinpages;
2777 if (noinpages > 1){
2778 size_t tmplen = cop->tfrm_op.inlen;
2779
2780 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2781 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2782 tmplen -= cop->tfrm_op.indata[0].iov_len;
2783 for (i = 1; i<noinpages; i++){
2784 cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2785 cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2786 tmplen -= PAGE_SIZE;
2787 }
2788 } else {
2789 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2790 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2791 }
2792
2793 iovlen = nooutpages + 6;
2794 pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2795
2796 next_ix = next_cfg_change_ix(&oper, prev_ix);
2797 if (prev_ix == next_ix){
2798 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2799 err = -EINVAL; /* This should be impossible barring bugs. */
2800 goto error_cleanup;
2801 }
2802 while (prev_ix != next_ix){
2803 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2804 descs[desc_ix].cfg = NULL;
2805 descs[desc_ix].length = next_ix - prev_ix;
2806
2807 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2808 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2809 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2810 cipher_active = 1;
2811
2812 if (next_ix == (oper.cipher_start + oper.cipher_len)){
2813 cipher_done = 1;
2814 dcfgs[dcfg_ix].last = 1;
2815 } else {
2816 dcfgs[dcfg_ix].last = 0;
2817 }
2818 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2819 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2820 ++dcfg_ix;
2821 }
2822 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2823 digest_active = 1;
2824 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2825 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2826 if (next_ix == (oper.digest_start + oper.digest_len)){
2827 assert(!digest_done);
2828 digest_done = 1;
2829 dcfgs[dcfg_ix].last = 1;
2830 } else {
2831 dcfgs[dcfg_ix].last = 0;
2832 }
2833 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2834 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2835 ++dcfg_ix;
2836 }
2837 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2838 csum_active = 1;
2839 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2840 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2841 if (next_ix == (oper.csum_start + oper.csum_len)){
2842 csum_done = 1;
2843 dcfgs[dcfg_ix].last = 1;
2844 } else {
2845 dcfgs[dcfg_ix].last = 0;
2846 }
2847 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2848 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2849 ++dcfg_ix;
2850 }
2851 if (!descs[desc_ix].cfg){
2852 DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2853 err = -EINVAL;
2854 goto error_cleanup;
2855 }
2856 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2857 ++desc_ix;
2858 prev_ix = next_ix;
2859 next_ix = next_cfg_change_ix(&oper, prev_ix);
2860 }
2861 if (desc_ix > 0){
2862 descs[desc_ix-1].next = NULL;
2863 } else {
2864 descs[0].next = NULL;
2865 }
2866 if (oper.do_digest) {
2867 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2868 /* Add outdata iovec, length == <length of type of digest> */
2869 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2870 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2871 ++iovix;
2872 }
2873 if (oper.do_csum) {
2874 /* Add outdata iovec, length == 2, the length of csum. */
2875 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2876 /* Add outdata iovec, length == <length of type of digest> */
2877 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2878 cop->tfrm_op.outdata[iovix].iov_len = 2;
2879 ++iovix;
2880 }
2881 if (oper.do_cipher) {
2882 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2883 DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2884 err = -ENOSYS; /* This should be impossible barring bugs. */
2885 goto error_cleanup;
2886 }
2887 }
2888 DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2889 cop->tfrm_op.outcount = iovix;
2890 assert(iovix <= (nooutpages + 6));
2891
2892 cop->sid = oper.ses_id;
2893 cop->tfrm_op.desc = &descs[0];
2894
2895 DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2896
2897 if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2898 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2899 err = -EINVAL;
2900 goto error_cleanup;
2901 }
2902
2903 DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2904
2905 wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2906 DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2907 if (!jc->processed){
2908 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2909 err = -EIO;
2910 goto error_cleanup;
2911 }
2912
2913 /* Job process done. Cipher output should already be correct in job so no post processing of outdata. */
2914 DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2915 if (cop->operation_status == 0){
2916 if (oper.do_digest){
2917 DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2918 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2919 if (0 != err){
2920 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2921 err = -EFAULT;
2922 goto error_cleanup;
2923 }
2924 }
2925 if (oper.do_csum){
2926 DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2927 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2928 if (0 != err){
2929 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2930 err = -EFAULT;
2931 goto error_cleanup;
2932 }
2933 }
2934 err = 0;
2935 } else {
2936 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2937 err = cop->operation_status;
2938 }
2939
2940 error_cleanup:
2941 /* Release page caches. */
2942 for (i = 0; i < noinpages; i++){
2943 put_page(inpages[i]);
2944 }
2945 for (i = 0; i < nooutpages; i++){
2946 int spdl_err;
2947 /* Mark output pages dirty. */
2948 spdl_err = set_page_dirty_lock(outpages[i]);
2949 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2950 }
2951 for (i = 0; i < nooutpages; i++){
2952 put_page(outpages[i]);
2953 }
2954
2955 kfree(digest_result);
2956 kfree(inpages);
2957 kfree(outpages);
2958 if (cop){
2959 kfree(cop->tfrm_op.indata);
2960 kfree(cop->tfrm_op.outdata);
2961 kfree(cop);
2962 }
2963 kfree(jc);
2964
2965 DEBUG(print_lock_status());
2966
2967 return err;
2968 }
2969
2970
2971 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2972 {
2973 cryptocop_session_id sid;
2974 int err;
2975 struct cryptocop_private *dev;
2976 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2977 struct strcop_session_op sop;
2978 struct cryptocop_transform_init *tis = NULL;
2979 struct cryptocop_transform_init ti_cipher = {0};
2980 struct cryptocop_transform_init ti_digest = {0};
2981 struct cryptocop_transform_init ti_csum = {0};
2982
2983 if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2984 return -EFAULT;
2985 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2986 if (err) return -EFAULT;
2987 if (sop.cipher != cryptocop_cipher_none) {
2988 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2989 }
2990 DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2991
2992 DEBUG(printk("\tcipher:%d\n"
2993 "\tcipher_mode:%d\n"
2994 "\tdigest:%d\n"
2995 "\tcsum:%d\n",
2996 (int)sop.cipher,
2997 (int)sop.cmode,
2998 (int)sop.digest,
2999 (int)sop.csum));
3000
3001 if (sop.cipher != cryptocop_cipher_none){
3002 /* Init the cipher. */
3003 switch (sop.cipher){
3004 case cryptocop_cipher_des:
3005 ti_cipher.alg = cryptocop_alg_des;
3006 break;
3007 case cryptocop_cipher_3des:
3008 ti_cipher.alg = cryptocop_alg_3des;
3009 break;
3010 case cryptocop_cipher_aes:
3011 ti_cipher.alg = cryptocop_alg_aes;
3012 break;
3013 default:
3014 DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3015 return -EINVAL;
3016 };
3017 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3018 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3019 ti_cipher.keylen = sop.keylen;
3020 switch (sop.cmode){
3021 case cryptocop_cipher_mode_cbc:
3022 case cryptocop_cipher_mode_ecb:
3023 ti_cipher.cipher_mode = sop.cmode;
3024 break;
3025 default:
3026 DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3027 return -EINVAL;
3028 }
3029 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3030 switch (sop.des3_mode){
3031 case cryptocop_3des_eee:
3032 case cryptocop_3des_eed:
3033 case cryptocop_3des_ede:
3034 case cryptocop_3des_edd:
3035 case cryptocop_3des_dee:
3036 case cryptocop_3des_ded:
3037 case cryptocop_3des_dde:
3038 case cryptocop_3des_ddd:
3039 ti_cipher.tdes_mode = sop.des3_mode;
3040 break;
3041 default:
3042 DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3043 return -EINVAL;
3044 }
3045 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3046 ti_cipher.next = tis;
3047 tis = &ti_cipher;
3048 } /* if (sop.cipher != cryptocop_cipher_none) */
3049 if (sop.digest != cryptocop_digest_none){
3050 DEBUG(printk("setting digest transform\n"));
3051 switch (sop.digest){
3052 case cryptocop_digest_md5:
3053 ti_digest.alg = cryptocop_alg_md5;
3054 break;
3055 case cryptocop_digest_sha1:
3056 ti_digest.alg = cryptocop_alg_sha1;
3057 break;
3058 default:
3059 DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3060 return -EINVAL;
3061 }
3062 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3063 ti_digest.next = tis;
3064 tis = &ti_digest;
3065 } /* if (sop.digest != cryptocop_digest_none) */
3066 if (sop.csum != cryptocop_csum_none){
3067 DEBUG(printk("setting csum transform\n"));
3068 switch (sop.csum){
3069 case cryptocop_csum_le:
3070 case cryptocop_csum_be:
3071 ti_csum.csum_mode = sop.csum;
3072 break;
3073 default:
3074 DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3075 return -EINVAL;
3076 }
3077 ti_csum.alg = cryptocop_alg_csum;
3078 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3079 ti_csum.next = tis;
3080 tis = &ti_csum;
3081 } /* (sop.csum != cryptocop_csum_none) */
3082 dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3083 if (!dev){
3084 DEBUG_API(printk("create session, alloc dev\n"));
3085 return -ENOMEM;
3086 }
3087
3088 err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3089 DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3090
3091 if (err) {
3092 kfree(dev);
3093 return err;
3094 }
3095 sess_op->ses_id = sid;
3096 dev->sid = sid;
3097 dev->next = filp->private_data;
3098 filp->private_data = dev;
3099
3100 return 0;
3101 }
3102
3103 static long cryptocop_ioctl_unlocked(struct inode *inode,
3104 struct file *filp, unsigned int cmd, unsigned long arg)
3105 {
3106 int err = 0;
3107 if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3108 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3109 return -ENOTTY;
3110 }
3111 if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3112 return -ENOTTY;
3113 }
3114 /* Access check of the argument. Some commands, e.g. create session and process op,
3115 needs additional checks. Those are handled in the command handling functions. */
3116 if (_IOC_DIR(cmd) & _IOC_READ)
3117 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3118 else if (_IOC_DIR(cmd) & _IOC_WRITE)
3119 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3120 if (err) return -EFAULT;
3121
3122 switch (cmd) {
3123 case CRYPTOCOP_IO_CREATE_SESSION:
3124 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3125 case CRYPTOCOP_IO_CLOSE_SESSION:
3126 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3127 case CRYPTOCOP_IO_PROCESS_OP:
3128 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3129 default:
3130 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3131 return -ENOTTY;
3132 }
3133 return 0;
3134 }
3135
3136 static long
3137 cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3138 {
3139 struct inode *inode = file->f_path.dentry->d_inode;
3140 long ret;
3141
3142 mutex_lock(&cryptocop_mutex);
3143 ret = cryptocop_ioctl_unlocked(inode, filp, cmd, arg);
3144 mutex_unlock(&cryptocop_mutex);
3145
3146 return ret;
3147 }
3148
3149
3150 #ifdef LDEBUG
3151 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3152 {
3153 struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3154 struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3155 int i;
3156
3157 printk("print_dma_descriptors start\n");
3158
3159 printk("iop:\n");
3160 printk("\tsid: 0x%lld\n", iop->sid);
3161
3162 printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3163 printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3164 printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3165 printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3166
3167 printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3168 printk("\tnext: 0x%p\n"
3169 "\tsaved_data: 0x%p\n"
3170 "\tsaved_data_buf: 0x%p\n",
3171 iop->ctx_out.next,
3172 iop->ctx_out.saved_data,
3173 iop->ctx_out.saved_data_buf);
3174
3175 printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3176 printk("\tnext: 0x%p\n"
3177 "\tsaved_data: 0x%p\n"
3178 "\tsaved_data_buf: 0x%p\n",
3179 iop->ctx_in.next,
3180 iop->ctx_in.saved_data,
3181 iop->ctx_in.saved_data_buf);
3182
3183 i = 0;
3184 while (cdesc_out) {
3185 dma_descr_data *td;
3186 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3187 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3188 td = cdesc_out->dma_descr;
3189 printk("\n\tbuf: 0x%p\n"
3190 "\tafter: 0x%p\n"
3191 "\tmd: 0x%04x\n"
3192 "\tnext: 0x%p\n",
3193 td->buf,
3194 td->after,
3195 td->md,
3196 td->next);
3197 printk("flags:\n"
3198 "\twait:\t%d\n"
3199 "\teol:\t%d\n"
3200 "\touteop:\t%d\n"
3201 "\tineop:\t%d\n"
3202 "\tintr:\t%d\n",
3203 td->wait,
3204 td->eol,
3205 td->out_eop,
3206 td->in_eop,
3207 td->intr);
3208 cdesc_out = cdesc_out->next;
3209 i++;
3210 }
3211 i = 0;
3212 while (cdesc_in) {
3213 dma_descr_data *td;
3214 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3215 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3216 td = cdesc_in->dma_descr;
3217 printk("\n\tbuf: 0x%p\n"
3218 "\tafter: 0x%p\n"
3219 "\tmd: 0x%04x\n"
3220 "\tnext: 0x%p\n",
3221 td->buf,
3222 td->after,
3223 td->md,
3224 td->next);
3225 printk("flags:\n"
3226 "\twait:\t%d\n"
3227 "\teol:\t%d\n"
3228 "\touteop:\t%d\n"
3229 "\tineop:\t%d\n"
3230 "\tintr:\t%d\n",
3231 td->wait,
3232 td->eol,
3233 td->out_eop,
3234 td->in_eop,
3235 td->intr);
3236 cdesc_in = cdesc_in->next;
3237 i++;
3238 }
3239
3240 printk("print_dma_descriptors end\n");
3241 }
3242
3243
3244 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3245 {
3246 printk("print_strcop_crypto_op, 0x%p\n", cop);
3247
3248 /* Indata. */
3249 printk("indata=0x%p\n"
3250 "inlen=%d\n"
3251 "do_cipher=%d\n"
3252 "decrypt=%d\n"
3253 "cipher_explicit=%d\n"
3254 "cipher_start=%d\n"
3255 "cipher_len=%d\n"
3256 "outdata=0x%p\n"
3257 "outlen=%d\n",
3258 cop->indata,
3259 cop->inlen,
3260 cop->do_cipher,
3261 cop->decrypt,
3262 cop->cipher_explicit,
3263 cop->cipher_start,
3264 cop->cipher_len,
3265 cop->cipher_outdata,
3266 cop->cipher_outlen);
3267
3268 printk("do_digest=%d\n"
3269 "digest_start=%d\n"
3270 "digest_len=%d\n",
3271 cop->do_digest,
3272 cop->digest_start,
3273 cop->digest_len);
3274
3275 printk("do_csum=%d\n"
3276 "csum_start=%d\n"
3277 "csum_len=%d\n",
3278 cop->do_csum,
3279 cop->csum_start,
3280 cop->csum_len);
3281 }
3282
3283 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3284 {
3285 struct cryptocop_desc *d;
3286 struct cryptocop_tfrm_cfg *tc;
3287 struct cryptocop_desc_cfg *dc;
3288 int i;
3289
3290 printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3291 printk("sid: %lld\n", cop->sid);
3292 printk("operation_status=%d\n"
3293 "use_dmalists=%d\n"
3294 "in_interrupt=%d\n"
3295 "fast_callback=%d\n",
3296 cop->operation_status,
3297 cop->use_dmalists,
3298 cop->in_interrupt,
3299 cop->fast_callback);
3300
3301 if (cop->use_dmalists){
3302 print_user_dma_lists(&cop->list_op);
3303 } else {
3304 printk("cop->tfrm_op\n"
3305 "tfrm_cfg=0x%p\n"
3306 "desc=0x%p\n"
3307 "indata=0x%p\n"
3308 "incount=%d\n"
3309 "inlen=%d\n"
3310 "outdata=0x%p\n"
3311 "outcount=%d\n"
3312 "outlen=%d\n\n",
3313 cop->tfrm_op.tfrm_cfg,
3314 cop->tfrm_op.desc,
3315 cop->tfrm_op.indata,
3316 cop->tfrm_op.incount,
3317 cop->tfrm_op.inlen,
3318 cop->tfrm_op.outdata,
3319 cop->tfrm_op.outcount,
3320 cop->tfrm_op.outlen);
3321
3322 tc = cop->tfrm_op.tfrm_cfg;
3323 while (tc){
3324 printk("tfrm_cfg, 0x%p\n"
3325 "tid=%d\n"
3326 "flags=%d\n"
3327 "inject_ix=%d\n"
3328 "next=0x%p\n",
3329 tc,
3330 tc->tid,
3331 tc->flags,
3332 tc->inject_ix,
3333 tc->next);
3334 tc = tc->next;
3335 }
3336 d = cop->tfrm_op.desc;
3337 while (d){
3338 printk("\n======================desc, 0x%p\n"
3339 "length=%d\n"
3340 "cfg=0x%p\n"
3341 "next=0x%p\n",
3342 d,
3343 d->length,
3344 d->cfg,
3345 d->next);
3346 dc = d->cfg;
3347 while (dc){
3348 printk("=========desc_cfg, 0x%p\n"
3349 "tid=%d\n"
3350 "src=%d\n"
3351 "last=%d\n"
3352 "next=0x%p\n",
3353 dc,
3354 dc->tid,
3355 dc->src,
3356 dc->last,
3357 dc->next);
3358 dc = dc->next;
3359 }
3360 d = d->next;
3361 }
3362 printk("\n====iniov\n");
3363 for (i = 0; i < cop->tfrm_op.incount; i++){
3364 printk("indata[%d]\n"
3365 "base=0x%p\n"
3366 "len=%d\n",
3367 i,
3368 cop->tfrm_op.indata[i].iov_base,
3369 cop->tfrm_op.indata[i].iov_len);
3370 }
3371 printk("\n====outiov\n");
3372 for (i = 0; i < cop->tfrm_op.outcount; i++){
3373 printk("outdata[%d]\n"
3374 "base=0x%p\n"
3375 "len=%d\n",
3376 i,
3377 cop->tfrm_op.outdata[i].iov_base,
3378 cop->tfrm_op.outdata[i].iov_len);
3379 }
3380 }
3381 printk("------------end print_cryptocop_operation\n");
3382 }
3383
3384
3385 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3386 {
3387 dma_descr_data *dd;
3388 int i;
3389
3390 printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3391
3392 printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3393 printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3394
3395 printk("##############outlist\n");
3396 dd = phys_to_virt((unsigned long int)dma_op->outlist);
3397 i = 0;
3398 while (dd != NULL) {
3399 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3400 printk("\n\tbuf: 0x%p\n"
3401 "\tafter: 0x%p\n"
3402 "\tmd: 0x%04x\n"
3403 "\tnext: 0x%p\n",
3404 dd->buf,
3405 dd->after,
3406 dd->md,
3407 dd->next);
3408 printk("flags:\n"
3409 "\twait:\t%d\n"
3410 "\teol:\t%d\n"
3411 "\touteop:\t%d\n"
3412 "\tineop:\t%d\n"
3413 "\tintr:\t%d\n",
3414 dd->wait,
3415 dd->eol,
3416 dd->out_eop,
3417 dd->in_eop,
3418 dd->intr);
3419 if (dd->eol)
3420 dd = NULL;
3421 else
3422 dd = phys_to_virt((unsigned long int)dd->next);
3423 ++i;
3424 }
3425
3426 printk("##############inlist\n");
3427 dd = phys_to_virt((unsigned long int)dma_op->inlist);
3428 i = 0;
3429 while (dd != NULL) {
3430 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3431 printk("\n\tbuf: 0x%p\n"
3432 "\tafter: 0x%p\n"
3433 "\tmd: 0x%04x\n"
3434 "\tnext: 0x%p\n",
3435 dd->buf,
3436 dd->after,
3437 dd->md,
3438 dd->next);
3439 printk("flags:\n"
3440 "\twait:\t%d\n"
3441 "\teol:\t%d\n"
3442 "\touteop:\t%d\n"
3443 "\tineop:\t%d\n"
3444 "\tintr:\t%d\n",
3445 dd->wait,
3446 dd->eol,
3447 dd->out_eop,
3448 dd->in_eop,
3449 dd->intr);
3450 if (dd->eol)
3451 dd = NULL;
3452 else
3453 dd = phys_to_virt((unsigned long int)dd->next);
3454 ++i;
3455 }
3456 }
3457
3458
3459 static void print_lock_status(void)
3460 {
3461 printk("**********************print_lock_status\n");
3462 printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3463 printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3464 printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3465 printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3466 printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3467 printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3468 }
3469 #endif /* LDEBUG */
3470
3471
3472 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3473
3474 static int init_stream_coprocessor(void)
3475 {
3476 int err;
3477 int i;
3478 static int initialized = 0;
3479
3480 if (initialized)
3481 return 0;
3482
3483 initialized = 1;
3484
3485 printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3486
3487 err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3488 if (err < 0) {
3489 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3490 return err;
3491 }
3492
3493 err = init_cryptocop();
3494 if (err) {
3495 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3496 return err;
3497 }
3498 err = cryptocop_job_queue_init();
3499 if (err) {
3500 release_cryptocop();
3501 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3502 return err;
3503 }
3504 /* Init the descriptor pool. */
3505 for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3506 descr_pool[i].from_pool = 1;
3507 descr_pool[i].next = &descr_pool[i + 1];
3508 }
3509 descr_pool[i].from_pool = 1;
3510 descr_pool[i].next = NULL;
3511 descr_pool_free_list = &descr_pool[0];
3512 descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3513
3514 spin_lock_init(&cryptocop_completed_jobs_lock);
3515 spin_lock_init(&cryptocop_job_queue_lock);
3516 spin_lock_init(&descr_pool_lock);
3517 spin_lock_init(&cryptocop_sessions_lock);
3518 spin_lock_init(&running_job_lock);
3519 spin_lock_init(&cryptocop_process_lock);
3520
3521 cryptocop_sessions = NULL;
3522 next_sid = 1;
3523
3524 cryptocop_running_job = NULL;
3525
3526 printk("stream co-processor: init done.\n");
3527 return 0;
3528 }
3529
3530 static void __exit exit_stream_coprocessor(void)
3531 {
3532 release_cryptocop();
3533 cryptocop_job_queue_close();
3534 }
3535
3536 module_init(init_stream_coprocessor);
3537 module_exit(exit_stream_coprocessor);
3538
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree