?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
udf: fix udf_debug macro
[linux-2.6.git] / arch / cris / arch-v32 / drivers / cryptocop.c
blobe8914d4016966385e6c380c0471d371c05006c69
1 /* $Id: cryptocop.c,v 1.13 2005/04/21 17:27:55 henriken Exp $
2 *
3 * Stream co-processor driver for the ETRAX FS
4 *
5 * Copyright (C) 2003-2005 Axis Communications AB
6 */
7
8 #include <linux/init.h>
9 #include <linux/sched.h>
10 #include <linux/module.h>
11 #include <linux/slab.h>
12 #include <linux/string.h>
13 #include <linux/fs.h>
14 #include <linux/mm.h>
15 #include <linux/spinlock.h>
16 #include <linux/stddef.h>
17
18 #include <asm/uaccess.h>
19 #include <asm/io.h>
20 #include <asm/atomic.h>
21
22 #include <linux/list.h>
23 #include <linux/interrupt.h>
24
25 #include <asm/signal.h>
26 #include <asm/irq.h>
27
28 #include <asm/arch/dma.h>
29 #include <asm/arch/hwregs/dma.h>
30 #include <asm/arch/hwregs/reg_map.h>
31 #include <asm/arch/hwregs/reg_rdwr.h>
32 #include <asm/arch/hwregs/intr_vect_defs.h>
33
34 #include <asm/arch/hwregs/strcop.h>
35 #include <asm/arch/hwregs/strcop_defs.h>
36 #include <asm/arch/cryptocop.h>
37
38
39
40 #define DESCR_ALLOC_PAD (31)
41
42 struct cryptocop_dma_desc {
43 char *free_buf; /* If non-null will be kfreed in free_cdesc() */
44 dma_descr_data *dma_descr;
45
46 unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
47
48 unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
49 struct cryptocop_dma_desc *next;
50 };
51
52
53 struct cryptocop_int_operation{
54 void *alloc_ptr;
55 cryptocop_session_id sid;
56
57 dma_descr_context ctx_out;
58 dma_descr_context ctx_in;
59
60 /* DMA descriptors allocated by driver. */
61 struct cryptocop_dma_desc *cdesc_out;
62 struct cryptocop_dma_desc *cdesc_in;
63
64 /* Strcop config to use. */
65 cryptocop_3des_mode tdes_mode;
66 cryptocop_csum_type csum_mode;
67
68 /* DMA descrs provided by consumer. */
69 dma_descr_data *ddesc_out;
70 dma_descr_data *ddesc_in;
71 };
72
73
74 struct cryptocop_tfrm_ctx {
75 cryptocop_tfrm_id tid;
76 unsigned int blocklength;
77
78 unsigned int start_ix;
79
80 struct cryptocop_tfrm_cfg *tcfg;
81 struct cryptocop_transform_ctx *tctx;
82
83 unsigned char previous_src;
84 unsigned char current_src;
85
86 /* Values to use in metadata out. */
87 unsigned char hash_conf;
88 unsigned char hash_mode;
89 unsigned char ciph_conf;
90 unsigned char cbcmode;
91 unsigned char decrypt;
92
93 unsigned int requires_padding:1;
94 unsigned int strict_block_length:1;
95 unsigned int active:1;
96 unsigned int done:1;
97 size_t consumed;
98 size_t produced;
99
100 /* Pad (input) descriptors to put in the DMA out list when the transform
101 * output is put on the DMA in list. */
102 struct cryptocop_dma_desc *pad_descs;
103
104 struct cryptocop_tfrm_ctx *prev_src;
105 struct cryptocop_tfrm_ctx *curr_src;
106
107 /* Mapping to HW. */
108 unsigned char unit_no;
109 };
110
111
112 struct cryptocop_private{
113 cryptocop_session_id sid;
114 struct cryptocop_private *next;
115 };
116
117 /* Session list. */
118
119 struct cryptocop_transform_ctx{
120 struct cryptocop_transform_init init;
121 unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
122 unsigned int dec_key_set:1;
123
124 struct cryptocop_transform_ctx *next;
125 };
126
127
128 struct cryptocop_session{
129 cryptocop_session_id sid;
130
131 struct cryptocop_transform_ctx *tfrm_ctx;
132
133 struct cryptocop_session *next;
134 };
135
136 /* Priority levels for jobs sent to the cryptocop. Checksum operations from
137 kernel have highest priority since TCPIP stack processing must not
138 be a bottleneck. */
139 typedef enum {
140 cryptocop_prio_kernel_csum = 0,
141 cryptocop_prio_kernel = 1,
142 cryptocop_prio_user = 2,
143 cryptocop_prio_no_prios = 3
144 } cryptocop_queue_priority;
145
146 struct cryptocop_prio_queue{
147 struct list_head jobs;
148 cryptocop_queue_priority prio;
149 };
150
151 struct cryptocop_prio_job{
152 struct list_head node;
153 cryptocop_queue_priority prio;
154
155 struct cryptocop_operation *oper;
156 struct cryptocop_int_operation *iop;
157 };
158
159 struct ioctl_job_cb_ctx {
160 unsigned int processed:1;
161 };
162
163
164 static struct cryptocop_session *cryptocop_sessions = NULL;
165 spinlock_t cryptocop_sessions_lock;
166
167 /* Next Session ID to assign. */
168 static cryptocop_session_id next_sid = 1;
169
170 /* Pad for checksum. */
171 static const char csum_zero_pad[1] = {0x00};
172
173 /* Trash buffer for mem2mem operations. */
174 #define MEM2MEM_DISCARD_BUF_LENGTH (512)
175 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
176
177 /* Descriptor pool. */
178 /* FIXME Tweak this value. */
179 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE (100)
180 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
181 static struct cryptocop_dma_desc *descr_pool_free_list;
182 static int descr_pool_no_free;
183 static spinlock_t descr_pool_lock;
184
185 /* Lock to stop cryptocop to start processing of a new operation. The holder
186 of this lock MUST call cryptocop_start_job() after it is unlocked. */
187 spinlock_t cryptocop_process_lock;
188
189 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
190 static spinlock_t cryptocop_job_queue_lock;
191 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
192 static spinlock_t running_job_lock;
193
194 /* The interrupt handler appends completed jobs to this list. The scehduled
195 * tasklet removes them upon sending the response to the crypto consumer. */
196 static struct list_head cryptocop_completed_jobs;
197 static spinlock_t cryptocop_completed_jobs_lock;
198
199 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
200
201
202 /** Local functions. **/
203
204 static int cryptocop_open(struct inode *, struct file *);
205
206 static int cryptocop_release(struct inode *, struct file *);
207
208 static int cryptocop_ioctl(struct inode *inode, struct file *file,
209 unsigned int cmd, unsigned long arg);
210
211 static void cryptocop_start_job(void);
212
213 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
214 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
215
216 static int cryptocop_job_queue_init(void);
217 static void cryptocop_job_queue_close(void);
218
219 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
220
221 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
222
223 static int transform_ok(struct cryptocop_transform_init *tinit);
224
225 static struct cryptocop_session *get_session(cryptocop_session_id sid);
226
227 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
228
229 static void delete_internal_operation(struct cryptocop_int_operation *iop);
230
231 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength);
232
233 static int init_stream_coprocessor(void);
234
235 static void __exit exit_stream_coprocessor(void);
236
237 /*#define LDEBUG*/
238 #ifdef LDEBUG
239 #define DEBUG(s) s
240 #define DEBUG_API(s) s
241 static void print_cryptocop_operation(struct cryptocop_operation *cop);
242 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
243 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
244 static void print_lock_status(void);
245 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
246 #define assert(s) do{if (!(s)) panic(#s);} while(0);
247 #else
248 #define DEBUG(s)
249 #define DEBUG_API(s)
250 #define assert(s)
251 #endif
252
253
254 /* Transform constants. */
255 #define DES_BLOCK_LENGTH (8)
256 #define AES_BLOCK_LENGTH (16)
257 #define MD5_BLOCK_LENGTH (64)
258 #define SHA1_BLOCK_LENGTH (64)
259 #define CSUM_BLOCK_LENGTH (2)
260 #define MD5_STATE_LENGTH (16)
261 #define SHA1_STATE_LENGTH (20)
262
263 /* The device number. */
264 #define CRYPTOCOP_MAJOR (254)
265 #define CRYPTOCOP_MINOR (0)
266
267
268
269 const struct file_operations cryptocop_fops = {
270 .owner = THIS_MODULE,
271 .open = cryptocop_open,
272 .release = cryptocop_release,
273 .ioctl = cryptocop_ioctl
274 };
275
276
277 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
278 {
279 DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
280 kfree(cdesc->free_buf);
281
282 if (cdesc->from_pool) {
283 unsigned long int flags;
284 spin_lock_irqsave(&descr_pool_lock, flags);
285 cdesc->next = descr_pool_free_list;
286 descr_pool_free_list = cdesc;
287 ++descr_pool_no_free;
288 spin_unlock_irqrestore(&descr_pool_lock, flags);
289 } else {
290 kfree(cdesc);
291 }
292 }
293
294
295 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
296 {
297 int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
298 struct cryptocop_dma_desc *cdesc;
299
300 if (use_pool) {
301 unsigned long int flags;
302 spin_lock_irqsave(&descr_pool_lock, flags);
303 if (!descr_pool_free_list) {
304 spin_unlock_irqrestore(&descr_pool_lock, flags);
305 DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
306 return NULL;
307 }
308 cdesc = descr_pool_free_list;
309 descr_pool_free_list = descr_pool_free_list->next;
310 --descr_pool_no_free;
311 spin_unlock_irqrestore(&descr_pool_lock, flags);
312 cdesc->from_pool = 1;
313 } else {
314 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
315 if (!cdesc) {
316 DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
317 return NULL;
318 }
319 cdesc->from_pool = 0;
320 }
321 cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
322
323 cdesc->next = NULL;
324
325 cdesc->free_buf = NULL;
326 cdesc->dma_descr->out_eop = 0;
327 cdesc->dma_descr->in_eop = 0;
328 cdesc->dma_descr->intr = 0;
329 cdesc->dma_descr->eol = 0;
330 cdesc->dma_descr->wait = 0;
331 cdesc->dma_descr->buf = NULL;
332 cdesc->dma_descr->after = NULL;
333
334 DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
335 return cdesc;
336 }
337
338
339 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
340 {
341 DEBUG(printk("setup_descr_chain: entering\n"));
342 while (cd) {
343 if (cd->next) {
344 cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
345 } else {
346 cd->dma_descr->next = NULL;
347 }
348 cd = cd->next;
349 }
350 DEBUG(printk("setup_descr_chain: exit\n"));
351 }
352
353
354 /* Create a pad descriptor for the transform.
355 * Return -1 for error, 0 if pad created. */
356 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
357 {
358 struct cryptocop_dma_desc *cdesc = NULL;
359 int error = 0;
360 struct strcop_meta_out mo = {
361 .ciphsel = src_none,
362 .hashsel = src_none,
363 .csumsel = src_none
364 };
365 char *pad;
366 size_t plen;
367
368 DEBUG(printk("create_pad_descriptor: start.\n"));
369 /* Setup pad descriptor. */
370
371 DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
372 cdesc = alloc_cdesc(alloc_flag);
373 if (!cdesc){
374 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
375 goto error_cleanup;
376 }
377 switch (tc->unit_no) {
378 case src_md5:
379 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
380 if (error){
381 DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
382 goto error_cleanup;
383 }
384 cdesc->free_buf = pad;
385 mo.hashsel = src_dma;
386 mo.hashconf = tc->hash_conf;
387 mo.hashmode = tc->hash_mode;
388 break;
389 case src_sha1:
390 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
391 if (error){
392 DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
393 goto error_cleanup;
394 }
395 cdesc->free_buf = pad;
396 mo.hashsel = src_dma;
397 mo.hashconf = tc->hash_conf;
398 mo.hashmode = tc->hash_mode;
399 break;
400 case src_csum:
401 if (tc->consumed % tc->blocklength){
402 pad = (char*)csum_zero_pad;
403 plen = 1;
404 } else {
405 pad = (char*)cdesc; /* Use any pointer. */
406 plen = 0;
407 }
408 mo.csumsel = src_dma;
409 break;
410 }
411 cdesc->dma_descr->wait = 1;
412 cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed. EOP is ok here since the padded unit is the only one active. */
413 cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
414 cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
415
416 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
417 *pad_desc = cdesc;
418
419 return 0;
420
421 error_cleanup:
422 if (cdesc) free_cdesc(cdesc);
423 return -1;
424 }
425
426
427 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
428 {
429 struct cryptocop_dma_desc *key_desc = alloc_cdesc(alloc_flag);
430 struct strcop_meta_out mo = {0};
431
432 DEBUG(printk("setup_key_dl_desc\n"));
433
434 if (!key_desc) {
435 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
436 return -ENOMEM;
437 }
438
439 /* Download key. */
440 if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
441 /* Precook the AES decrypt key. */
442 if (!tc->tctx->dec_key_set){
443 get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
444 tc->tctx->dec_key_set = 1;
445 }
446 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
447 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
448 } else {
449 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
450 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
451 }
452 /* Setup metadata. */
453 mo.dlkey = 1;
454 switch (tc->tctx->init.keylen) {
455 case 64:
456 mo.decrypt = 0;
457 mo.hashmode = 0;
458 break;
459 case 128:
460 mo.decrypt = 0;
461 mo.hashmode = 1;
462 break;
463 case 192:
464 mo.decrypt = 1;
465 mo.hashmode = 0;
466 break;
467 case 256:
468 mo.decrypt = 1;
469 mo.hashmode = 1;
470 break;
471 default:
472 break;
473 }
474 mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
475 key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
476
477 key_desc->dma_descr->out_eop = 1;
478 key_desc->dma_descr->wait = 1;
479 key_desc->dma_descr->intr = 0;
480
481 *kd = key_desc;
482 return 0;
483 }
484
485 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
486 {
487 struct cryptocop_dma_desc *iv_desc = alloc_cdesc(alloc_flag);
488 struct strcop_meta_out mo = {0};
489
490 DEBUG(printk("setup_cipher_iv_desc\n"));
491
492 if (!iv_desc) {
493 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
494 return -ENOMEM;
495 }
496 /* Download IV. */
497 iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
498 iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
499
500 /* Setup metadata. */
501 mo.hashsel = mo.csumsel = src_none;
502 mo.ciphsel = src_dma;
503 mo.ciphconf = tc->ciph_conf;
504 mo.cbcmode = tc->cbcmode;
505
506 iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
507
508 iv_desc->dma_descr->out_eop = 0;
509 iv_desc->dma_descr->wait = 1;
510 iv_desc->dma_descr->intr = 0;
511
512 *id = iv_desc;
513 return 0;
514 }
515
516 /* Map the ouput length of the transform to operation output starting on the inject index. */
517 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
518 {
519 int err = 0;
520 struct cryptocop_dma_desc head = {0};
521 struct cryptocop_dma_desc *outdesc = &head;
522 size_t iov_offset = 0;
523 size_t out_ix = 0;
524 int outiov_ix = 0;
525 struct strcop_meta_in mi = {0};
526
527 size_t out_length = tc->produced;
528 int rem_length;
529 int dlength;
530
531 assert(out_length != 0);
532 if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
533 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
534 return -EINVAL;
535 }
536 /* Traverse the out iovec until the result inject index is reached. */
537 while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
538 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
539 outiov_ix++;
540 }
541 if (outiov_ix >= operation->tfrm_op.outcount){
542 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
543 return -EINVAL;
544 }
545 iov_offset = tc->tcfg->inject_ix - out_ix;
546 mi.dmasel = tc->unit_no;
547
548 /* Setup the output descriptors. */
549 while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
550 outdesc->next = alloc_cdesc(alloc_flag);
551 if (!outdesc->next) {
552 DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
553 err = -ENOMEM;
554 goto error_cleanup;
555 }
556 outdesc = outdesc->next;
557 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
558 dlength = (out_length < rem_length) ? out_length : rem_length;
559
560 DEBUG(printk("create_input_descriptors:\n"
561 "outiov_ix=%d, rem_length=%d, dlength=%d\n"
562 "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
563 "outcount=%d, outiov_ix=%d\n",
564 outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
565
566 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
567 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
568 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
569
570 out_length -= dlength;
571 iov_offset += dlength;
572 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
573 iov_offset = 0;
574 ++outiov_ix;
575 }
576 }
577 if (out_length > 0){
578 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
579 err = -EINVAL;
580 goto error_cleanup;
581 }
582 /* Set sync in last descriptor. */
583 mi.sync = 1;
584 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
585
586 *id = head.next;
587 return 0;
588
589 error_cleanup:
590 while (head.next) {
591 outdesc = head.next->next;
592 free_cdesc(head.next);
593 head.next = outdesc;
594 }
595 return err;
596 }
597
598
599 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
600 {
601 while (desc_len != 0) {
602 struct cryptocop_dma_desc *cdesc;
603 int rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
604 int dlength = (desc_len < rem_length) ? desc_len : rem_length;
605
606 cdesc = alloc_cdesc(alloc_flag);
607 if (!cdesc) {
608 DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
609 return -ENOMEM;
610 }
611 (*current_out_cdesc)->next = cdesc;
612 (*current_out_cdesc) = cdesc;
613
614 cdesc->free_buf = NULL;
615
616 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
617 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
618
619 desc_len -= dlength;
620 *iniov_offset += dlength;
621 assert(desc_len >= 0);
622 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
623 *iniov_offset = 0;
624 ++(*iniov_ix);
625 if (*iniov_ix > operation->tfrm_op.incount) {
626 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
627 return -EINVAL;
628 }
629 }
630 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
631 } /* while (desc_len != 0) */
632 /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
633 (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
634
635 return 0;
636 }
637
638
639 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
640 {
641 DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
642 if (tc->tcfg) {
643 int failed = 0;
644 struct cryptocop_dma_desc *idescs = NULL;
645 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
646 if (tc->pad_descs) {
647 DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
648 while (tc->pad_descs) {
649 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
650 (*current_out_cdesc)->next = tc->pad_descs;
651 tc->pad_descs = tc->pad_descs->next;
652 (*current_out_cdesc) = (*current_out_cdesc)->next;
653 }
654 }
655
656 /* Setup and append output descriptors to DMA in list. */
657 if (tc->unit_no == src_dma){
658 /* mem2mem. Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
659 struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
660 unsigned int start_ix = tc->start_ix;
661 while (start_ix){
662 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
663 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
664 if (!(*current_in_cdesc)->next){
665 DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
666 return -ENOMEM;
667 }
668 (*current_in_cdesc) = (*current_in_cdesc)->next;
669 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
670 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
671 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
672 start_ix -= desclen;
673 }
674 mi.sync = 1;
675 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
676 }
677
678 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
679 if (failed){
680 DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
681 return failed;
682 }
683 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
684 while (idescs) {
685 DEBUG(printk("append descriptor 0x%p\n", idescs));
686 (*current_in_cdesc)->next = idescs;
687 idescs = idescs->next;
688 (*current_in_cdesc) = (*current_in_cdesc)->next;
689 }
690 }
691 return 0;
692 }
693
694
695
696 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
697 {
698 struct cryptocop_session *sess;
699 struct cryptocop_transform_ctx *tctx;
700
701 struct cryptocop_tfrm_ctx digest_ctx = {
702 .previous_src = src_none,
703 .current_src = src_none,
704 .start_ix = 0,
705 .requires_padding = 1,
706 .strict_block_length = 0,
707 .hash_conf = 0,
708 .hash_mode = 0,
709 .ciph_conf = 0,
710 .cbcmode = 0,
711 .decrypt = 0,
712 .consumed = 0,
713 .produced = 0,
714 .pad_descs = NULL,
715 .active = 0,
716 .done = 0,
717 .prev_src = NULL,
718 .curr_src = NULL,
719 .tcfg = NULL};
720 struct cryptocop_tfrm_ctx cipher_ctx = {
721 .previous_src = src_none,
722 .current_src = src_none,
723 .start_ix = 0,
724 .requires_padding = 0,
725 .strict_block_length = 1,
726 .hash_conf = 0,
727 .hash_mode = 0,
728 .ciph_conf = 0,
729 .cbcmode = 0,
730 .decrypt = 0,
731 .consumed = 0,
732 .produced = 0,
733 .pad_descs = NULL,
734 .active = 0,
735 .done = 0,
736 .prev_src = NULL,
737 .curr_src = NULL,
738 .tcfg = NULL};
739 struct cryptocop_tfrm_ctx csum_ctx = {
740 .previous_src = src_none,
741 .current_src = src_none,
742 .start_ix = 0,
743 .blocklength = 2,
744 .requires_padding = 1,
745 .strict_block_length = 0,
746 .hash_conf = 0,
747 .hash_mode = 0,
748 .ciph_conf = 0,
749 .cbcmode = 0,
750 .decrypt = 0,
751 .consumed = 0,
752 .produced = 0,
753 .pad_descs = NULL,
754 .active = 0,
755 .done = 0,
756 .tcfg = NULL,
757 .prev_src = NULL,
758 .curr_src = NULL,
759 .unit_no = src_csum};
760 struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
761
762 unsigned int indata_ix = 0;
763
764 /* iovec accounting. */
765 int iniov_ix = 0;
766 int iniov_offset = 0;
767
768 /* Operation descriptor cfg traversal pointer. */
769 struct cryptocop_desc *odsc;
770
771 int failed = 0;
772 /* List heads for allocated descriptors. */
773 struct cryptocop_dma_desc out_cdesc_head = {0};
774 struct cryptocop_dma_desc in_cdesc_head = {0};
775
776 struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
777 struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
778
779 struct cryptocop_tfrm_ctx *output_tc = NULL;
780 void *iop_alloc_ptr;
781
782 assert(operation != NULL);
783 assert(int_op != NULL);
784
785 DEBUG(printk("cryptocop_setup_dma_list: start\n"));
786 DEBUG(print_cryptocop_operation(operation));
787
788 sess = get_session(operation->sid);
789 if (!sess) {
790 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
791 failed = -EINVAL;
792 goto error_cleanup;
793 }
794 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
795 if (!iop_alloc_ptr) {
796 DEBUG_API(printk("cryptocop_setup_dma_list: kmalloc cryptocop_int_operation\n"));
797 failed = -ENOMEM;
798 goto error_cleanup;
799 }
800 (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
801 DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
802 (*int_op)->alloc_ptr = iop_alloc_ptr;
803 DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
804
805 (*int_op)->sid = operation->sid;
806 (*int_op)->cdesc_out = NULL;
807 (*int_op)->cdesc_in = NULL;
808 (*int_op)->tdes_mode = cryptocop_3des_ede;
809 (*int_op)->csum_mode = cryptocop_csum_le;
810 (*int_op)->ddesc_out = NULL;
811 (*int_op)->ddesc_in = NULL;
812
813 /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
814 if (!tcfg) {
815 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
816 failed = -EINVAL;
817 goto error_cleanup;
818 }
819 while (tcfg) {
820 tctx = get_transform_ctx(sess, tcfg->tid);
821 if (!tctx) {
822 DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
823 failed = -EINVAL;
824 goto error_cleanup;
825 }
826 if (tcfg->inject_ix > operation->tfrm_op.outlen){
827 DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
828 failed = -EINVAL;
829 goto error_cleanup;
830 }
831 switch (tctx->init.alg){
832 case cryptocop_alg_mem2mem:
833 if (cipher_ctx.tcfg != NULL){
834 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
835 failed = -EINVAL;
836 goto error_cleanup;
837 }
838 /* mem2mem is handled as a NULL cipher. */
839 cipher_ctx.cbcmode = 0;
840 cipher_ctx.decrypt = 0;
841 cipher_ctx.blocklength = 1;
842 cipher_ctx.ciph_conf = 0;
843 cipher_ctx.unit_no = src_dma;
844 cipher_ctx.tcfg = tcfg;
845 cipher_ctx.tctx = tctx;
846 break;
847 case cryptocop_alg_des:
848 case cryptocop_alg_3des:
849 case cryptocop_alg_aes:
850 /* cipher */
851 if (cipher_ctx.tcfg != NULL){
852 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
853 failed = -EINVAL;
854 goto error_cleanup;
855 }
856 cipher_ctx.tcfg = tcfg;
857 cipher_ctx.tctx = tctx;
858 if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
859 cipher_ctx.decrypt = 1;
860 }
861 switch (tctx->init.cipher_mode) {
862 case cryptocop_cipher_mode_ecb:
863 cipher_ctx.cbcmode = 0;
864 break;
865 case cryptocop_cipher_mode_cbc:
866 cipher_ctx.cbcmode = 1;
867 break;
868 default:
869 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
870 failed = -EINVAL;
871 goto error_cleanup;
872 }
873 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
874 switch (tctx->init.alg){
875 case cryptocop_alg_des:
876 cipher_ctx.ciph_conf = 0;
877 cipher_ctx.unit_no = src_des;
878 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
879 break;
880 case cryptocop_alg_3des:
881 cipher_ctx.ciph_conf = 1;
882 cipher_ctx.unit_no = src_des;
883 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
884 break;
885 case cryptocop_alg_aes:
886 cipher_ctx.ciph_conf = 2;
887 cipher_ctx.unit_no = src_aes;
888 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
889 break;
890 default:
891 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
892 }
893 (*int_op)->tdes_mode = tctx->init.tdes_mode;
894 break;
895 case cryptocop_alg_md5:
896 case cryptocop_alg_sha1:
897 /* digest */
898 if (digest_ctx.tcfg != NULL){
899 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
900 failed = -EINVAL;
901 goto error_cleanup;
902 }
903 digest_ctx.tcfg = tcfg;
904 digest_ctx.tctx = tctx;
905 digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
906 switch (tctx->init.alg){
907 case cryptocop_alg_md5:
908 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
909 digest_ctx.unit_no = src_md5;
910 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
911 break;
912 case cryptocop_alg_sha1:
913 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
914 digest_ctx.unit_no = src_sha1;
915 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
916 break;
917 default:
918 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
919 }
920 break;
921 case cryptocop_alg_csum:
922 /* digest */
923 if (csum_ctx.tcfg != NULL){
924 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
925 failed = -EINVAL;
926 goto error_cleanup;
927 }
928 (*int_op)->csum_mode = tctx->init.csum_mode;
929 csum_ctx.tcfg = tcfg;
930 csum_ctx.tctx = tctx;
931 break;
932 default:
933 /* no algorithm. */
934 DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
935 failed = -EINVAL;
936 goto error_cleanup;
937 }
938 tcfg = tcfg->next;
939 }
940 /* Download key if a cipher is used. */
941 if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
942 struct cryptocop_dma_desc *key_desc = NULL;
943
944 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
945 if (failed) {
946 DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
947 goto error_cleanup;
948 }
949 current_out_cdesc->next = key_desc;
950 current_out_cdesc = key_desc;
951 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
952
953 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
954 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
955 struct cryptocop_dma_desc *iv_desc = NULL;
956
957 DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
958
959 failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
960 if (failed) {
961 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
962 goto error_cleanup;
963 }
964 current_out_cdesc->next = iv_desc;
965 current_out_cdesc = iv_desc;
966 indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
967 }
968 }
969
970 /* Process descriptors. */
971 odsc = operation->tfrm_op.desc;
972 while (odsc) {
973 struct cryptocop_desc_cfg *dcfg = odsc->cfg;
974 struct strcop_meta_out meta_out = {0};
975 size_t desc_len = odsc->length;
976 int active_count, eop_needed_count;
977
978 output_tc = NULL;
979
980 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
981
982 while (dcfg) {
983 struct cryptocop_tfrm_ctx *tc = NULL;
984
985 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
986 /* Get the local context for the transform and mark it as the output unit if it produces output. */
987 if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
988 tc = &digest_ctx;
989 } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
990 tc = &cipher_ctx;
991 } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
992 tc = &csum_ctx;
993 }
994 if (!tc) {
995 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
996 failed = -EINVAL;
997 goto error_cleanup;
998 }
999 if (tc->done) {
1000 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1001 failed = -EINVAL;
1002 goto error_cleanup;
1003 }
1004 if (!tc->active) {
1005 tc->start_ix = indata_ix;
1006 tc->active = 1;
1007 }
1008
1009 tc->previous_src = tc->current_src;
1010 tc->prev_src = tc->curr_src;
1011 /* Map source unit id to DMA source config. */
1012 switch (dcfg->src){
1013 case cryptocop_source_dma:
1014 tc->current_src = src_dma;
1015 break;
1016 case cryptocop_source_des:
1017 tc->current_src = src_des;
1018 break;
1019 case cryptocop_source_3des:
1020 tc->current_src = src_des;
1021 break;
1022 case cryptocop_source_aes:
1023 tc->current_src = src_aes;
1024 break;
1025 case cryptocop_source_md5:
1026 case cryptocop_source_sha1:
1027 case cryptocop_source_csum:
1028 case cryptocop_source_none:
1029 default:
1030 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1031 */
1032 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1033 failed = -EINVAL;
1034 goto error_cleanup;
1035 }
1036 if (tc->current_src != src_dma) {
1037 /* Find the unit we are sourcing from. */
1038 if (digest_ctx.unit_no == tc->current_src){
1039 tc->curr_src = &digest_ctx;
1040 } else if (cipher_ctx.unit_no == tc->current_src){
1041 tc->curr_src = &cipher_ctx;
1042 } else if (csum_ctx.unit_no == tc->current_src){
1043 tc->curr_src = &csum_ctx;
1044 }
1045 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1046 DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1047 failed = -EINVAL;
1048 goto error_cleanup;
1049 }
1050 } else {
1051 tc->curr_src = NULL;
1052 }
1053
1054 /* Detect source switch. */
1055 DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1056 if (tc->active && (tc->current_src != tc->previous_src)) {
1057 /* Only allow source switch when both the old source unit and the new one have
1058 * no pending data to process (i.e. the consumed length must be a multiple of the
1059 * transform blocklength). */
1060 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1061 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1062 ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1063 {
1064 DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1065 failed = -EINVAL;
1066 goto error_cleanup;
1067 }
1068 }
1069 /* Detect unit deactivation. */
1070 if (dcfg->last) {
1071 /* Length check of this is handled below. */
1072 tc->done = 1;
1073 }
1074 dcfg = dcfg->next;
1075 } /* while (dcfg) */
1076 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1077
1078 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1079 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1080 failed = -EINVAL;
1081 goto error_cleanup;
1082 }
1083 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1084 DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1085 failed = -EINVAL;
1086 goto error_cleanup;
1087 }
1088 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1089 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1090 failed = -EINVAL;
1091 goto error_cleanup;
1092 }
1093
1094 /* Update consumed and produced lengths.
1095
1096 The consumed length accounting here is actually cheating. If a unit source from DMA (or any
1097 other unit that process data in blocks of one octet) it is correct, but if it source from a
1098 block processing unit, i.e. a cipher, it will be temporarily incorrect at some times. However
1099 since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1100 unit has processed an exact multiple of its block length the end result will be correct.
1101 Beware that if the source change restriction change this code will need to be (much) reworked.
1102 */
1103 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1104
1105 if (csum_ctx.active) {
1106 csum_ctx.consumed += desc_len;
1107 if (csum_ctx.done) {
1108 csum_ctx.produced = 2;
1109 }
1110 DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1111 }
1112 if (digest_ctx.active) {
1113 digest_ctx.consumed += desc_len;
1114 if (digest_ctx.done) {
1115 if (digest_ctx.unit_no == src_md5) {
1116 digest_ctx.produced = MD5_STATE_LENGTH;
1117 } else {
1118 digest_ctx.produced = SHA1_STATE_LENGTH;
1119 }
1120 }
1121 DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1122 }
1123 if (cipher_ctx.active) {
1124 /* Ciphers are allowed only to source from DMA out. That is filtered above. */
1125 assert(cipher_ctx.current_src == src_dma);
1126 cipher_ctx.consumed += desc_len;
1127 cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1128 if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1129 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1130 }
1131 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1132 }
1133
1134 /* Setup the DMA out descriptors. */
1135 /* Configure the metadata. */
1136 active_count = 0;
1137 eop_needed_count = 0;
1138 if (cipher_ctx.active) {
1139 ++active_count;
1140 if (cipher_ctx.unit_no == src_dma){
1141 /* mem2mem */
1142 meta_out.ciphsel = src_none;
1143 } else {
1144 meta_out.ciphsel = cipher_ctx.current_src;
1145 }
1146 meta_out.ciphconf = cipher_ctx.ciph_conf;
1147 meta_out.cbcmode = cipher_ctx.cbcmode;
1148 meta_out.decrypt = cipher_ctx.decrypt;
1149 DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1150 if (cipher_ctx.done) ++eop_needed_count;
1151 } else {
1152 meta_out.ciphsel = src_none;
1153 }
1154
1155 if (digest_ctx.active) {
1156 ++active_count;
1157 meta_out.hashsel = digest_ctx.current_src;
1158 meta_out.hashconf = digest_ctx.hash_conf;
1159 meta_out.hashmode = 0; /* Explicit mode is not used here. */
1160 DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1161 if (digest_ctx.done) {
1162 assert(digest_ctx.pad_descs == NULL);
1163 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1164 if (failed) {
1165 DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1166 goto error_cleanup;
1167 }
1168 }
1169 } else {
1170 meta_out.hashsel = src_none;
1171 }
1172
1173 if (csum_ctx.active) {
1174 ++active_count;
1175 meta_out.csumsel = csum_ctx.current_src;
1176 if (csum_ctx.done) {
1177 assert(csum_ctx.pad_descs == NULL);
1178 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1179 if (failed) {
1180 DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1181 goto error_cleanup;
1182 }
1183 }
1184 } else {
1185 meta_out.csumsel = src_none;
1186 }
1187 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1188 /* Setup DMA out descriptors for the indata. */
1189 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1190 if (failed) {
1191 DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1192 goto error_cleanup;
1193 }
1194 /* Setup out EOP. If there are active units that are not done here they cannot get an EOP
1195 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1196 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1197 */
1198 assert(active_count >= eop_needed_count);
1199 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1200 if (eop_needed_count) {
1201 /* This means that the bulk operation (cipeher/m2m) is terminated. */
1202 if (active_count > 1) {
1203 /* Use zero length EOP descriptor. */
1204 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1205 struct strcop_meta_out ed_mo = {0};
1206 if (!ed) {
1207 DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1208 failed = -ENOMEM;
1209 goto error_cleanup;
1210 }
1211
1212 assert(cipher_ctx.active && cipher_ctx.done);
1213
1214 if (cipher_ctx.unit_no == src_dma){
1215 /* mem2mem */
1216 ed_mo.ciphsel = src_none;
1217 } else {
1218 ed_mo.ciphsel = cipher_ctx.current_src;
1219 }
1220 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1221 ed_mo.cbcmode = cipher_ctx.cbcmode;
1222 ed_mo.decrypt = cipher_ctx.decrypt;
1223
1224 ed->free_buf = NULL;
1225 ed->dma_descr->wait = 1;
1226 ed->dma_descr->out_eop = 1;
1227
1228 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1229 ed->dma_descr->after = ed->dma_descr->buf;
1230 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1231 current_out_cdesc->next = ed;
1232 current_out_cdesc = ed;
1233 } else {
1234 /* Set EOP in the current out descriptor since the only active module is
1235 * the one needing the EOP. */
1236
1237 current_out_cdesc->dma_descr->out_eop = 1;
1238 }
1239 }
1240
1241 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1242 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1243 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1244 indata_ix += odsc->length;
1245 odsc = odsc->next;
1246 } /* while (odsc) */ /* Process descriptors. */
1247 DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1248 if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1249 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1250 failed = -EINVAL;
1251 goto error_cleanup;
1252 }
1253 if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1254 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1255 failed = -EINVAL;
1256 goto error_cleanup;
1257 }
1258 if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1259 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1260 failed = -EINVAL;
1261 goto error_cleanup;
1262 }
1263
1264 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1265 if (failed){
1266 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1267 goto error_cleanup;
1268 }
1269 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1270 if (failed){
1271 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1272 goto error_cleanup;
1273 }
1274 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1275 if (failed){
1276 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1277 goto error_cleanup;
1278 }
1279
1280 DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1281 (*int_op)->cdesc_out = out_cdesc_head.next;
1282 (*int_op)->cdesc_in = in_cdesc_head.next;
1283 DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1284
1285 setup_descr_chain(out_cdesc_head.next);
1286 setup_descr_chain(in_cdesc_head.next);
1287
1288 /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1289 * last DMA out descriptor for EOL.
1290 */
1291 current_in_cdesc->dma_descr->intr = 1;
1292 current_in_cdesc->dma_descr->eol = 1;
1293 current_out_cdesc->dma_descr->eol = 1;
1294
1295 /* Setup DMA contexts. */
1296 (*int_op)->ctx_out.next = NULL;
1297 (*int_op)->ctx_out.eol = 1;
1298 (*int_op)->ctx_out.intr = 0;
1299 (*int_op)->ctx_out.store_mode = 0;
1300 (*int_op)->ctx_out.en = 0;
1301 (*int_op)->ctx_out.dis = 0;
1302 (*int_op)->ctx_out.md0 = 0;
1303 (*int_op)->ctx_out.md1 = 0;
1304 (*int_op)->ctx_out.md2 = 0;
1305 (*int_op)->ctx_out.md3 = 0;
1306 (*int_op)->ctx_out.md4 = 0;
1307 (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1308 (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1309
1310 (*int_op)->ctx_in.next = NULL;
1311 (*int_op)->ctx_in.eol = 1;
1312 (*int_op)->ctx_in.intr = 0;
1313 (*int_op)->ctx_in.store_mode = 0;
1314 (*int_op)->ctx_in.en = 0;
1315 (*int_op)->ctx_in.dis = 0;
1316 (*int_op)->ctx_in.md0 = 0;
1317 (*int_op)->ctx_in.md1 = 0;
1318 (*int_op)->ctx_in.md2 = 0;
1319 (*int_op)->ctx_in.md3 = 0;
1320 (*int_op)->ctx_in.md4 = 0;
1321
1322 (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1323 (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1324
1325 DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1326 return 0;
1327
1328 error_cleanup:
1329 {
1330 /* Free all allocated resources. */
1331 struct cryptocop_dma_desc *tmp_cdesc;
1332 while (digest_ctx.pad_descs){
1333 tmp_cdesc = digest_ctx.pad_descs->next;
1334 free_cdesc(digest_ctx.pad_descs);
1335 digest_ctx.pad_descs = tmp_cdesc;
1336 }
1337 while (csum_ctx.pad_descs){
1338 tmp_cdesc = csum_ctx.pad_descs->next;
1339 free_cdesc(csum_ctx.pad_descs);
1340 csum_ctx.pad_descs = tmp_cdesc;
1341 }
1342 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1343
1344 if (*int_op != NULL) delete_internal_operation(*int_op);
1345 }
1346 DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1347 return failed;
1348 }
1349
1350
1351 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1352 {
1353 void *ptr = iop->alloc_ptr;
1354 struct cryptocop_dma_desc *cd = iop->cdesc_out;
1355 struct cryptocop_dma_desc *next;
1356
1357 DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1358
1359 while (cd) {
1360 next = cd->next;
1361 free_cdesc(cd);
1362 cd = next;
1363 }
1364 cd = iop->cdesc_in;
1365 while (cd) {
1366 next = cd->next;
1367 free_cdesc(cd);
1368 cd = next;
1369 }
1370 kfree(ptr);
1371 }
1372
1373 #define MD5_MIN_PAD_LENGTH (9)
1374 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1375
1376 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1377 {
1378 size_t padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1379 unsigned char *p;
1380 int i;
1381 unsigned long long int bit_length = hashed_length << 3;
1382
1383 if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1384
1385 p = kmalloc(padlen, alloc_flag);
1386 if (!pad) return -ENOMEM;
1387
1388 *p = 0x80;
1389 memset(p+1, 0, padlen - 1);
1390
1391 DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1392
1393 i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1394 while (bit_length != 0){
1395 p[i++] = bit_length % 0x100;
1396 bit_length >>= 8;
1397 }
1398
1399 *pad = (char*)p;
1400 *pad_length = padlen;
1401
1402 return 0;
1403 }
1404
1405 #define SHA1_MIN_PAD_LENGTH (9)
1406 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1407
1408 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1409 {
1410 size_t padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1411 unsigned char *p;
1412 int i;
1413 unsigned long long int bit_length = hashed_length << 3;
1414
1415 if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1416
1417 p = kmalloc(padlen, alloc_flag);
1418 if (!pad) return -ENOMEM;
1419
1420 *p = 0x80;
1421 memset(p+1, 0, padlen - 1);
1422
1423 DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1424
1425 i = padlen - 1;
1426 while (bit_length != 0){
1427 p[i--] = bit_length % 0x100;
1428 bit_length >>= 8;
1429 }
1430
1431 *pad = (char*)p;
1432 *pad_length = padlen;
1433
1434 return 0;
1435 }
1436
1437
1438 static int transform_ok(struct cryptocop_transform_init *tinit)
1439 {
1440 switch (tinit->alg){
1441 case cryptocop_alg_csum:
1442 switch (tinit->csum_mode){
1443 case cryptocop_csum_le:
1444 case cryptocop_csum_be:
1445 break;
1446 default:
1447 DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1448 return -EINVAL;
1449 }
1450 case cryptocop_alg_mem2mem:
1451 case cryptocop_alg_md5:
1452 case cryptocop_alg_sha1:
1453 if (tinit->keylen != 0) {
1454 DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1455 return -EINVAL; /* This check is a bit strict. */
1456 }
1457 break;
1458 case cryptocop_alg_des:
1459 if (tinit->keylen != 64) {
1460 DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1461 return -EINVAL;
1462 }
1463 break;
1464 case cryptocop_alg_3des:
1465 if (tinit->keylen != 192) {
1466 DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1467 return -EINVAL;
1468 }
1469 break;
1470 case cryptocop_alg_aes:
1471 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1472 DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1473 return -EINVAL;
1474 }
1475 break;
1476 case cryptocop_no_alg:
1477 default:
1478 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1479 return -EINVAL;
1480 }
1481
1482 switch (tinit->alg){
1483 case cryptocop_alg_des:
1484 case cryptocop_alg_3des:
1485 case cryptocop_alg_aes:
1486 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1487 default:
1488 break;
1489 }
1490 return 0;
1491 }
1492
1493
1494 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1495 {
1496 struct cryptocop_session *sess;
1497 struct cryptocop_transform_init *tfrm_in = tinit;
1498 struct cryptocop_transform_init *tmp_in;
1499 int no_tfrms = 0;
1500 int i;
1501 unsigned long int flags;
1502
1503 init_stream_coprocessor(); /* For safety if we are called early */
1504
1505 while (tfrm_in){
1506 int err;
1507 ++no_tfrms;
1508 if ((err = transform_ok(tfrm_in))) {
1509 DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1510 return err;
1511 }
1512 tfrm_in = tfrm_in->next;
1513 }
1514 if (0 == no_tfrms) {
1515 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1516 return -EINVAL;
1517 }
1518
1519 sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1520 if (!sess){
1521 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1522 return -ENOMEM;
1523 }
1524
1525 sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1526 if (!sess->tfrm_ctx) {
1527 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1528 kfree(sess);
1529 return -ENOMEM;
1530 }
1531
1532 tfrm_in = tinit;
1533 for (i = 0; i < no_tfrms; i++){
1534 tmp_in = tfrm_in->next;
1535 while (tmp_in){
1536 if (tmp_in->tid == tfrm_in->tid) {
1537 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1538 kfree(sess->tfrm_ctx);
1539 kfree(sess);
1540 return -EINVAL;
1541 }
1542 tmp_in = tmp_in->next;
1543 }
1544 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1545 sess->tfrm_ctx[i].dec_key_set = 0;
1546 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1547
1548 tfrm_in = tfrm_in->next;
1549 }
1550 sess->tfrm_ctx[i-1].next = NULL;
1551
1552 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1553 sess->sid = next_sid;
1554 next_sid++;
1555 /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1556 * OTOH 2^64 is a really large number of session. */
1557 if (next_sid == 0) next_sid = 1;
1558
1559 /* Prepend to session list. */
1560 sess->next = cryptocop_sessions;
1561 cryptocop_sessions = sess;
1562 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1563 *sid = sess->sid;
1564 return 0;
1565 }
1566
1567
1568 int cryptocop_free_session(cryptocop_session_id sid)
1569 {
1570 struct cryptocop_transform_ctx *tc;
1571 struct cryptocop_session *sess = NULL;
1572 struct cryptocop_session *psess = NULL;
1573 unsigned long int flags;
1574 int i;
1575 LIST_HEAD(remove_list);
1576 struct list_head *node, *tmp;
1577 struct cryptocop_prio_job *pj;
1578
1579 DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1580
1581 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1582 sess = cryptocop_sessions;
1583 while (sess && sess->sid != sid){
1584 psess = sess;
1585 sess = sess->next;
1586 }
1587 if (sess){
1588 if (psess){
1589 psess->next = sess->next;
1590 } else {
1591 cryptocop_sessions = sess->next;
1592 }
1593 }
1594 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1595
1596 if (!sess) return -EINVAL;
1597
1598 /* Remove queued jobs. */
1599 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1600
1601 for (i = 0; i < cryptocop_prio_no_prios; i++){
1602 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1603 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1604 pj = list_entry(node, struct cryptocop_prio_job, node);
1605 if (pj->oper->sid == sid) {
1606 list_move_tail(node, &remove_list);
1607 }
1608 }
1609 }
1610 }
1611 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1612
1613 list_for_each_safe(node, tmp, &remove_list) {
1614 list_del(node);
1615 pj = list_entry(node, struct cryptocop_prio_job, node);
1616 pj->oper->operation_status = -EAGAIN; /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1617 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1618 pj->oper->cb(pj->oper, pj->oper->cb_data);
1619 delete_internal_operation(pj->iop);
1620 kfree(pj);
1621 }
1622
1623 tc = sess->tfrm_ctx;
1624 /* Erase keying data. */
1625 while (tc){
1626 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1627 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1628 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1629 tc = tc->next;
1630 }
1631 kfree(sess->tfrm_ctx);
1632 kfree(sess);
1633
1634 return 0;
1635 }
1636
1637 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1638 {
1639 struct cryptocop_session *sess;
1640 unsigned long int flags;
1641
1642 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1643 sess = cryptocop_sessions;
1644 while (sess && (sess->sid != sid)){
1645 sess = sess->next;
1646 }
1647 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1648
1649 return sess;
1650 }
1651
1652 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1653 {
1654 struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1655
1656 DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1657 assert(sess != NULL);
1658 while (tc && tc->init.tid != tid){
1659 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1660 tc = tc->next;
1661 }
1662 DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1663 return tc;
1664 }
1665
1666
1667
1668 /* The AES s-transform matrix (s-box). */
1669 static const u8 aes_sbox[256] = {
1670 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, 171, 118,
1671 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, 156, 164, 114, 192,
1672 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, 229, 241, 113, 216, 49, 21,
1673 4, 199, 35, 195, 24, 150, 5, 154, 7, 18, 128, 226, 235, 39, 178, 117,
1674 9, 131, 44, 26, 27, 110, 90, 160, 82, 59, 214, 179, 41, 227, 47, 132,
1675 83, 209, 0, 237, 32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207,
1676 208, 239, 170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168,
1677 81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243, 210,
1678 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100, 93, 25, 115,
1679 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184, 20, 222, 94, 11, 219,
1680 224, 50, 58, 10, 73, 6, 36, 92, 194, 211, 172, 98, 145, 149, 228, 121,
1681 231, 200, 55, 109, 141, 213, 78, 169, 108, 86, 244, 234, 101, 122, 174, 8,
1682 186, 120, 37, 46, 28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138,
1683 112, 62, 181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158,
1684 225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223,
1685 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84, 187, 22
1686 };
1687
1688 /* AES has a 32 bit word round constants for each round in the
1689 * key schedule. round_constant[i] is really Rcon[i+1] in FIPS187.
1690 */
1691 static u32 round_constant[11] = {
1692 0x01000000, 0x02000000, 0x04000000, 0x08000000,
1693 0x10000000, 0x20000000, 0x40000000, 0x80000000,
1694 0x1B000000, 0x36000000, 0x6C000000
1695 };
1696
1697 /* Apply the s-box to each of the four occtets in w. */
1698 static u32 aes_ks_subword(const u32 w)
1699 {
1700 u8 bytes[4];
1701
1702 *(u32*)(&bytes[0]) = w;
1703 bytes[0] = aes_sbox[bytes[0]];
1704 bytes[1] = aes_sbox[bytes[1]];
1705 bytes[2] = aes_sbox[bytes[2]];
1706 bytes[3] = aes_sbox[bytes[3]];
1707 return *(u32*)(&bytes[0]);
1708 }
1709
1710 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1711 * (Note that AES words are 32 bit long)
1712 *
1713 * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1714 * word temp
1715 * i = 0
1716 * while (i < Nk) {
1717 * w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1718 * i = i + 1
1719 * }
1720 * i = Nk
1721 *
1722 * while (i < (Nb * (Nr + 1))) {
1723 * temp = w[i - 1]
1724 * if ((i mod Nk) == 0) {
1725 * temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1726 * }
1727 * else if ((Nk > 6) && ((i mod Nk) == 4)) {
1728 * temp = SubWord(temp)
1729 * }
1730 * w[i] = w[i - Nk] xor temp
1731 * }
1732 * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1733 * SubWord(t) applies the AES s-box individually to each octet
1734 * in a 32 bit word.
1735 *
1736 * For AES Nk can have the values 4, 6, and 8 (corresponding to
1737 * values for Nr of 10, 12, and 14). Nb is always 4.
1738 *
1739 * To construct w[i], w[i - 1] and w[i - Nk] must be
1740 * available. Consequently we must keep a state of the last Nk words
1741 * to be able to create the last round keys.
1742 */
1743 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength)
1744 {
1745 u32 temp;
1746 u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1747 u8 w_last_ix;
1748 int i;
1749 u8 nr, nk;
1750
1751 switch (keylength){
1752 case 128:
1753 nk = 4;
1754 nr = 10;
1755 break;
1756 case 192:
1757 nk = 6;
1758 nr = 12;
1759 break;
1760 case 256:
1761 nk = 8;
1762 nr = 14;
1763 break;
1764 default:
1765 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1766 };
1767
1768 /* Need to do host byte order correction here since key is byte oriented and the
1769 * kx algorithm is word (u32) oriented. */
1770 for (i = 0; i < nk; i+=1) {
1771 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1772 }
1773
1774 i = (int)nk;
1775 w_last_ix = i - 1;
1776 while (i < (4 * (nr + 2))) {
1777 temp = w_ring[w_last_ix];
1778 if (!(i % nk)) {
1779 /* RotWord(temp) */
1780 temp = (temp << 8) | (temp >> 24);
1781 temp = aes_ks_subword(temp);
1782 temp ^= round_constant[i/nk - 1];
1783 } else if ((nk > 6) && ((i % nk) == 4)) {
1784 temp = aes_ks_subword(temp);
1785 }
1786 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1787 temp ^= w_ring[w_last_ix];
1788 w_ring[w_last_ix] = temp;
1789
1790 /* We need the round keys for round Nr+1 and Nr+2 (round key
1791 * Nr+2 is the round key beyond the last one used when
1792 * encrypting). Rounds are numbered starting from 0, Nr=10
1793 * implies 11 rounds are used in encryption/decryption.
1794 */
1795 if (i >= (4 * nr)) {
1796 /* Need to do host byte order correction here, the key
1797 * is byte oriented. */
1798 *(u32*)dec_key = cpu_to_be32(temp);
1799 dec_key += 4;
1800 }
1801 ++i;
1802 }
1803 }
1804
1805
1806 /**** Job/operation management. ****/
1807
1808 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1809 {
1810 return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1811 }
1812
1813 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1814 {
1815 return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1816 }
1817
1818 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1819 {
1820 return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1821 }
1822
1823 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1824 {
1825 int ret;
1826 struct cryptocop_prio_job *pj = NULL;
1827 unsigned long int flags;
1828
1829 DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1830
1831 if (!operation || !operation->cb){
1832 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1833 return -EINVAL;
1834 }
1835
1836 if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1837 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1838 return ret;
1839 }
1840 assert(pj != NULL);
1841
1842 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1843 list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1844 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1845
1846 /* Make sure a job is running */
1847 cryptocop_start_job();
1848 return 0;
1849 }
1850
1851 static void cryptocop_do_tasklet(unsigned long unused);
1852 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1853
1854 static void cryptocop_do_tasklet(unsigned long unused)
1855 {
1856 struct list_head *node;
1857 struct cryptocop_prio_job *pj = NULL;
1858 unsigned long flags;
1859
1860 DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1861
1862 do {
1863 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1864 if (!list_empty(&cryptocop_completed_jobs)){
1865 node = cryptocop_completed_jobs.next;
1866 list_del(node);
1867 pj = list_entry(node, struct cryptocop_prio_job, node);
1868 } else {
1869 pj = NULL;
1870 }
1871 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1872 if (pj) {
1873 assert(pj->oper != NULL);
1874
1875 /* Notify consumer of operation completeness. */
1876 DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1877
1878 pj->oper->operation_status = 0; /* Job is completed. */
1879 pj->oper->cb(pj->oper, pj->oper->cb_data);
1880 delete_internal_operation(pj->iop);
1881 kfree(pj);
1882 }
1883 } while (pj != NULL);
1884
1885 DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1886 }
1887
1888 static irqreturn_t
1889 dma_done_interrupt(int irq, void *dev_id, struct pt_regs * regs)
1890 {
1891 struct cryptocop_prio_job *done_job;
1892 reg_dma_rw_ack_intr ack_intr = {
1893 .data = 1,
1894 };
1895
1896 REG_WR (dma, regi_dma9, rw_ack_intr, ack_intr);
1897
1898 DEBUG(printk("cryptocop DMA done\n"));
1899
1900 spin_lock(&running_job_lock);
1901 if (cryptocop_running_job == NULL){
1902 printk("stream co-processor got interrupt when not busy\n");
1903 spin_unlock(&running_job_lock);
1904 return IRQ_HANDLED;
1905 }
1906 done_job = cryptocop_running_job;
1907 cryptocop_running_job = NULL;
1908 spin_unlock(&running_job_lock);
1909
1910 /* Start processing a job. */
1911 if (!spin_trylock(&cryptocop_process_lock)){
1912 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1913 } else {
1914 cryptocop_start_job();
1915 spin_unlock(&cryptocop_process_lock);
1916 }
1917
1918 done_job->oper->operation_status = 0; /* Job is completed. */
1919 if (done_job->oper->fast_callback){
1920 /* This operation wants callback from interrupt. */
1921 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1922 delete_internal_operation(done_job->iop);
1923 kfree(done_job);
1924 } else {
1925 spin_lock(&cryptocop_completed_jobs_lock);
1926 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1927 spin_unlock(&cryptocop_completed_jobs_lock);
1928 tasklet_schedule(&cryptocop_tasklet);
1929 }
1930
1931 DEBUG(printk("cryptocop leave irq handler\n"));
1932 return IRQ_HANDLED;
1933 }
1934
1935
1936 /* Setup interrupts and DMA channels. */
1937 static int init_cryptocop(void)
1938 {
1939 unsigned long flags;
1940 reg_intr_vect_rw_mask intr_mask;
1941 reg_dma_rw_cfg dma_cfg = {.en = 1};
1942 reg_dma_rw_intr_mask intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1943 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
1944 reg_strcop_rw_cfg strcop_cfg = {
1945 .ipend = regk_strcop_little,
1946 .td1 = regk_strcop_e,
1947 .td2 = regk_strcop_d,
1948 .td3 = regk_strcop_e,
1949 .ignore_sync = 0,
1950 .en = 1
1951 };
1952
1953 if (request_irq(DMA9_INTR_VECT, dma_done_interrupt, 0, "stream co-processor DMA", NULL)) panic("request_irq stream co-processor irq dma9");
1954
1955 (void)crisv32_request_dma(8, "strcop", DMA_PANIC_ON_ERROR, 0, dma_strp);
1956 (void)crisv32_request_dma(9, "strcop", DMA_PANIC_ON_ERROR, 0, dma_strp);
1957
1958 local_irq_save(flags);
1959
1960 /* Reset and enable the cryptocop. */
1961 strcop_cfg.en = 0;
1962 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1963 strcop_cfg.en = 1;
1964 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1965
1966 /* Enable DMA9 interrupt */
1967 intr_mask = REG_RD(intr_vect, regi_irq, rw_mask);
1968 intr_mask.dma9 = 1;
1969 REG_WR(intr_vect, regi_irq, rw_mask, intr_mask);
1970
1971 /* Enable DMAs. */
1972 REG_WR(dma, regi_dma9, rw_cfg, dma_cfg); /* input DMA */
1973 REG_WR(dma, regi_dma8, rw_cfg, dma_cfg); /* output DMA */
1974
1975 /* Set up wordsize = 4 for DMAs. */
1976 DMA_WR_CMD (regi_dma8, regk_dma_set_w_size4);
1977 DMA_WR_CMD (regi_dma9, regk_dma_set_w_size4);
1978
1979 /* Enable interrupts. */
1980 REG_WR(dma, regi_dma9, rw_intr_mask, intr_mask_in);
1981
1982 /* Clear intr ack. */
1983 REG_WR(dma, regi_dma9, rw_ack_intr, ack_intr);
1984
1985 local_irq_restore(flags);
1986
1987 return 0;
1988 }
1989
1990 /* Free used cryptocop hw resources (interrupt and DMA channels). */
1991 static void release_cryptocop(void)
1992 {
1993 unsigned long flags;
1994 reg_intr_vect_rw_mask intr_mask;
1995 reg_dma_rw_cfg dma_cfg = {.en = 0};
1996 reg_dma_rw_intr_mask intr_mask_in = {0};
1997 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
1998
1999 local_irq_save(flags);
2000
2001 /* Clear intr ack. */
2002 REG_WR(dma, regi_dma9, rw_ack_intr, ack_intr);
2003
2004 /* Disable DMA9 interrupt */
2005 intr_mask = REG_RD(intr_vect, regi_irq, rw_mask);
2006 intr_mask.dma9 = 0;
2007 REG_WR(intr_vect, regi_irq, rw_mask, intr_mask);
2008
2009 /* Disable DMAs. */
2010 REG_WR(dma, regi_dma9, rw_cfg, dma_cfg); /* input DMA */
2011 REG_WR(dma, regi_dma8, rw_cfg, dma_cfg); /* output DMA */
2012
2013 /* Disable interrupts. */
2014 REG_WR(dma, regi_dma9, rw_intr_mask, intr_mask_in);
2015
2016 local_irq_restore(flags);
2017
2018 free_irq(DMA9_INTR_VECT, NULL);
2019
2020 (void)crisv32_free_dma(8);
2021 (void)crisv32_free_dma(9);
2022 }
2023
2024
2025 /* Init job queue. */
2026 static int cryptocop_job_queue_init(void)
2027 {
2028 int i;
2029
2030 INIT_LIST_HEAD(&cryptocop_completed_jobs);
2031
2032 for (i = 0; i < cryptocop_prio_no_prios; i++){
2033 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2034 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2035 }
2036 return 0;
2037 }
2038
2039
2040 static void cryptocop_job_queue_close(void)
2041 {
2042 struct list_head *node, *tmp;
2043 struct cryptocop_prio_job *pj = NULL;
2044 unsigned long int process_flags, flags;
2045 int i;
2046
2047 /* FIXME: This is as yet untested code. */
2048
2049 /* Stop strcop from getting an operation to process while we are closing the
2050 module. */
2051 spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2052
2053 /* Empty the job queue. */
2054 for (i = 0; i < cryptocop_prio_no_prios; i++){
2055 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2056 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2057 pj = list_entry(node, struct cryptocop_prio_job, node);
2058 list_del(node);
2059
2060 /* Call callback to notify consumer of job removal. */
2061 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2062 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2063 pj->oper->cb(pj->oper, pj->oper->cb_data);
2064
2065 delete_internal_operation(pj->iop);
2066 kfree(pj);
2067 }
2068 }
2069 }
2070 spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2071
2072 /* Remove the running job, if any. */
2073 spin_lock_irqsave(&running_job_lock, flags);
2074 if (cryptocop_running_job){
2075 reg_strcop_rw_cfg rw_cfg;
2076 reg_dma_rw_cfg dma_out_cfg, dma_in_cfg;
2077
2078 /* Stop DMA. */
2079 dma_out_cfg = REG_RD(dma, regi_dma8, rw_cfg);
2080 dma_out_cfg.en = regk_dma_no;
2081 REG_WR(dma, regi_dma8, rw_cfg, dma_out_cfg);
2082
2083 dma_in_cfg = REG_RD(dma, regi_dma9, rw_cfg);
2084 dma_in_cfg.en = regk_dma_no;
2085 REG_WR(dma, regi_dma9, rw_cfg, dma_in_cfg);
2086
2087 /* Disble the cryptocop. */
2088 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2089 rw_cfg.en = 0;
2090 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2091
2092 pj = cryptocop_running_job;
2093 cryptocop_running_job = NULL;
2094
2095 /* Call callback to notify consumer of job removal. */
2096 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2097 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2098 pj->oper->cb(pj->oper, pj->oper->cb_data);
2099
2100 delete_internal_operation(pj->iop);
2101 kfree(pj);
2102 }
2103 spin_unlock_irqrestore(&running_job_lock, flags);
2104
2105 /* Remove completed jobs, if any. */
2106 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2107
2108 list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2109 pj = list_entry(node, struct cryptocop_prio_job, node);
2110 list_del(node);
2111 /* Call callback to notify consumer of job removal. */
2112 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2113 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2114 pj->oper->cb(pj->oper, pj->oper->cb_data);
2115
2116 delete_internal_operation(pj->iop);
2117 kfree(pj);
2118 }
2119 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2120 }
2121
2122
2123 static void cryptocop_start_job(void)
2124 {
2125 int i;
2126 struct cryptocop_prio_job *pj;
2127 unsigned long int flags;
2128 unsigned long int running_job_flags;
2129 reg_strcop_rw_cfg rw_cfg = {.en = 1, .ignore_sync = 0};
2130
2131 DEBUG(printk("cryptocop_start_job: entering\n"));
2132
2133 spin_lock_irqsave(&running_job_lock, running_job_flags);
2134 if (cryptocop_running_job != NULL){
2135 /* Already running. */
2136 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2137 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2138 return;
2139 }
2140 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2141
2142 /* Check the queues in priority order. */
2143 for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2144 if (i == cryptocop_prio_no_prios) {
2145 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2146 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2147 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2148 return; /* No jobs to run */
2149 }
2150 DEBUG(printk("starting job for prio %d\n", i));
2151
2152 /* TODO: Do not starve lower priority jobs. Let in a lower
2153 * prio job for every N-th processed higher prio job or some
2154 * other scheduling policy. This could reasonably be
2155 * tweakable since the optimal balance would depend on the
2156 * type of load on the system. */
2157
2158 /* Pull the DMA lists from the job and start the DMA client. */
2159 pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2160 list_del(&pj->node);
2161 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2162 cryptocop_running_job = pj;
2163
2164 /* Set config register (3DES and CSUM modes). */
2165 switch (pj->iop->tdes_mode){
2166 case cryptocop_3des_eee:
2167 rw_cfg.td1 = regk_strcop_e;
2168 rw_cfg.td2 = regk_strcop_e;
2169 rw_cfg.td3 = regk_strcop_e;
2170 break;
2171 case cryptocop_3des_eed:
2172 rw_cfg.td1 = regk_strcop_e;
2173 rw_cfg.td2 = regk_strcop_e;
2174 rw_cfg.td3 = regk_strcop_d;
2175 break;
2176 case cryptocop_3des_ede:
2177 rw_cfg.td1 = regk_strcop_e;
2178 rw_cfg.td2 = regk_strcop_d;
2179 rw_cfg.td3 = regk_strcop_e;
2180 break;
2181 case cryptocop_3des_edd:
2182 rw_cfg.td1 = regk_strcop_e;
2183 rw_cfg.td2 = regk_strcop_d;
2184 rw_cfg.td3 = regk_strcop_d;
2185 break;
2186 case cryptocop_3des_dee:
2187 rw_cfg.td1 = regk_strcop_d;
2188 rw_cfg.td2 = regk_strcop_e;
2189 rw_cfg.td3 = regk_strcop_e;
2190 break;
2191 case cryptocop_3des_ded:
2192 rw_cfg.td1 = regk_strcop_d;
2193 rw_cfg.td2 = regk_strcop_e;
2194 rw_cfg.td3 = regk_strcop_d;
2195 break;
2196 case cryptocop_3des_dde:
2197 rw_cfg.td1 = regk_strcop_d;
2198 rw_cfg.td2 = regk_strcop_d;
2199 rw_cfg.td3 = regk_strcop_e;
2200 break;
2201 case cryptocop_3des_ddd:
2202 rw_cfg.td1 = regk_strcop_d;
2203 rw_cfg.td2 = regk_strcop_d;
2204 rw_cfg.td3 = regk_strcop_d;
2205 break;
2206 default:
2207 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2208 }
2209 switch (pj->iop->csum_mode){
2210 case cryptocop_csum_le:
2211 rw_cfg.ipend = regk_strcop_little;
2212 break;
2213 case cryptocop_csum_be:
2214 rw_cfg.ipend = regk_strcop_big;
2215 break;
2216 default:
2217 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2218 }
2219 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2220
2221 DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2222 "ctx_in: 0x%p, phys: 0x%p\n"
2223 "ctx_out: 0x%p, phys: 0x%p\n",
2224 pj,
2225 &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2226 &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2227
2228 /* Start input DMA. */
2229 DMA_START_CONTEXT(regi_dma9, virt_to_phys(&pj->iop->ctx_in));
2230
2231 /* Start output DMA. */
2232 DMA_START_CONTEXT(regi_dma8, virt_to_phys(&pj->iop->ctx_out));
2233
2234 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2235 DEBUG(printk("cryptocop_start_job: exiting\n"));
2236 }
2237
2238
2239 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2240 {
2241 int err;
2242 int alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2243 void *iop_alloc_ptr = NULL;
2244
2245 *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2246 if (!*pj) return -ENOMEM;
2247
2248 DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2249
2250 (*pj)->oper = operation;
2251 DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n", (*pj)->oper->cb, (*pj)->oper->cb_data));
2252
2253 if (operation->use_dmalists) {
2254 DEBUG(print_user_dma_lists(&operation->list_op));
2255 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2256 DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2257 kfree(*pj);
2258 return -EINVAL;
2259 }
2260 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2261 if (!iop_alloc_ptr) {
2262 DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2263 kfree(*pj);
2264 return -ENOMEM;
2265 }
2266 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2267 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2268 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2269 (*pj)->iop->sid = operation->sid;
2270 (*pj)->iop->cdesc_out = NULL;
2271 (*pj)->iop->cdesc_in = NULL;
2272 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2273 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2274 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2275 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2276
2277 /* Setup DMA contexts. */
2278 (*pj)->iop->ctx_out.next = NULL;
2279 (*pj)->iop->ctx_out.eol = 1;
2280 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2281 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2282
2283 (*pj)->iop->ctx_in.next = NULL;
2284 (*pj)->iop->ctx_in.eol = 1;
2285 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2286 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2287 } else {
2288 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2289 DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2290 kfree(*pj);
2291 return err;
2292 }
2293 }
2294 DEBUG(print_dma_descriptors((*pj)->iop));
2295
2296 DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2297
2298 return 0;
2299 }
2300
2301
2302 static int cryptocop_open(struct inode *inode, struct file *filp)
2303 {
2304 int p = iminor(inode);
2305
2306 if (p != CRYPTOCOP_MINOR) return -EINVAL;
2307
2308 filp->private_data = NULL;
2309 return 0;
2310 }
2311
2312
2313 static int cryptocop_release(struct inode *inode, struct file *filp)
2314 {
2315 struct cryptocop_private *dev = filp->private_data;
2316 struct cryptocop_private *dev_next;
2317
2318 while (dev){
2319 dev_next = dev->next;
2320 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2321 (void)cryptocop_free_session(dev->sid);
2322 }
2323 kfree(dev);
2324 dev = dev_next;
2325 }
2326
2327 return 0;
2328 }
2329
2330
2331 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2332 unsigned int cmd, unsigned long arg)
2333 {
2334 struct cryptocop_private *dev = filp->private_data;
2335 struct cryptocop_private *prev_dev = NULL;
2336 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2337 struct strcop_session_op sop;
2338 int err;
2339
2340 DEBUG(printk("cryptocop_ioctl_close_session\n"));
2341
2342 if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2343 return -EFAULT;
2344 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2345 if (err) return -EFAULT;
2346
2347 while (dev && (dev->sid != sop.ses_id)) {
2348 prev_dev = dev;
2349 dev = dev->next;
2350 }
2351 if (dev){
2352 if (prev_dev){
2353 prev_dev->next = dev->next;
2354 } else {
2355 filp->private_data = dev->next;
2356 }
2357 err = cryptocop_free_session(dev->sid);
2358 if (err) return -EFAULT;
2359 } else {
2360 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2361 return -EINVAL;
2362 }
2363 return 0;
2364 }
2365
2366
2367 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2368 {
2369 struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2370
2371 DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2372
2373 jc->processed = 1;
2374 wake_up(&cryptocop_ioc_process_wq);
2375 }
2376
2377
2378 #define CRYPTOCOP_IOCTL_CIPHER_TID (1)
2379 #define CRYPTOCOP_IOCTL_DIGEST_TID (2)
2380 #define CRYPTOCOP_IOCTL_CSUM_TID (3)
2381
2382 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2383 {
2384 size_t ch_ix = 0;
2385
2386 if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2387 if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2388 if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2389
2390 DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2391 return ch_ix;
2392 }
2393
2394
2395 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2396 {
2397 size_t ch_ix = INT_MAX;
2398 size_t tmp_ix = 0;
2399
2400 if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2401 if (crp_op->cipher_start > ix) {
2402 ch_ix = crp_op->cipher_start;
2403 } else {
2404 ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2405 }
2406 }
2407 if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2408 if (crp_op->digest_start > ix) {
2409 tmp_ix = crp_op->digest_start;
2410 } else {
2411 tmp_ix = crp_op->digest_start + crp_op->digest_len;
2412 }
2413 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2414 }
2415 if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2416 if (crp_op->csum_start > ix) {
2417 tmp_ix = crp_op->csum_start;
2418 } else {
2419 tmp_ix = crp_op->csum_start + crp_op->csum_len;
2420 }
2421 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2422 }
2423 if (ch_ix == INT_MAX) ch_ix = ix;
2424 DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2425 return ch_ix;
2426 }
2427
2428
2429 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2430 * Return -1 for ok, 0 for fail. */
2431 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2432 {
2433 int tmplen;
2434
2435 assert(iov != NULL);
2436 assert(iovix != NULL);
2437 assert(pages != NULL);
2438 assert(pageix != NULL);
2439 assert(pageoffset != NULL);
2440
2441 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2442
2443 while (map_length > 0){
2444 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2445 if (*iovix >= iovlen){
2446 DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2447 return 0;
2448 }
2449 if (*pageix >= nopages){
2450 DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2451 return 0;
2452 }
2453 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2454 tmplen = PAGE_SIZE - *pageoffset;
2455 if (tmplen < map_length){
2456 (*pageoffset) = 0;
2457 (*pageix)++;
2458 } else {
2459 tmplen = map_length;
2460 (*pageoffset) += map_length;
2461 }
2462 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2463 iov[*iovix].iov_len = tmplen;
2464 map_length -= tmplen;
2465 (*iovix)++;
2466 }
2467 DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2468 return -1;
2469 }
2470
2471
2472
2473 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2474 {
2475 int i;
2476 struct cryptocop_private *dev = filp->private_data;
2477 struct strcop_crypto_op *crp_oper = (struct strcop_crypto_op *)arg;
2478 struct strcop_crypto_op oper = {0};
2479 int err = 0;
2480 struct cryptocop_operation *cop = NULL;
2481
2482 struct ioctl_job_cb_ctx *jc = NULL;
2483
2484 struct page **inpages = NULL;
2485 struct page **outpages = NULL;
2486 int noinpages = 0;
2487 int nooutpages = 0;
2488
2489 struct cryptocop_desc descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2490 * can get connected/disconnected on different places in the indata. */
2491 struct cryptocop_desc_cfg dcfgs[5*3];
2492 int desc_ix = 0;
2493 int dcfg_ix = 0;
2494 struct cryptocop_tfrm_cfg ciph_tcfg = {0};
2495 struct cryptocop_tfrm_cfg digest_tcfg = {0};
2496 struct cryptocop_tfrm_cfg csum_tcfg = {0};
2497
2498 unsigned char *digest_result = NULL;
2499 int digest_length = 0;
2500 int cblocklen = 0;
2501 unsigned char csum_result[CSUM_BLOCK_LENGTH];
2502 struct cryptocop_session *sess;
2503
2504 int iovlen = 0;
2505 int iovix = 0;
2506 int pageix = 0;
2507 int pageoffset = 0;
2508
2509 size_t prev_ix = 0;
2510 size_t next_ix;
2511
2512 int cipher_active, digest_active, csum_active;
2513 int end_digest, end_csum;
2514 int digest_done = 0;
2515 int cipher_done = 0;
2516 int csum_done = 0;
2517
2518 DEBUG(printk("cryptocop_ioctl_process\n"));
2519
2520 if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2521 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2522 return -EFAULT;
2523 }
2524 if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2525 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2526 return -EFAULT;
2527 }
2528 DEBUG(print_strcop_crypto_op(&oper));
2529
2530 while (dev && dev->sid != oper.ses_id) dev = dev->next;
2531 if (!dev){
2532 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2533 return -EINVAL;
2534 }
2535
2536 /* Check buffers. */
2537 if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2538 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2539 return -EINVAL;
2540 }
2541
2542 if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2543 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2544 return -EFAULT;
2545 }
2546 if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2547 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2548 return -EFAULT;
2549 }
2550
2551 cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2552 if (!cop) {
2553 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2554 return -ENOMEM;
2555 }
2556 jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2557 if (!jc) {
2558 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2559 err = -ENOMEM;
2560 goto error_cleanup;
2561 }
2562 jc->processed = 0;
2563
2564 cop->cb_data = jc;
2565 cop->cb = ioctl_process_job_callback;
2566 cop->operation_status = 0;
2567 cop->use_dmalists = 0;
2568 cop->in_interrupt = 0;
2569 cop->fast_callback = 0;
2570 cop->tfrm_op.tfrm_cfg = NULL;
2571 cop->tfrm_op.desc = NULL;
2572 cop->tfrm_op.indata = NULL;
2573 cop->tfrm_op.incount = 0;
2574 cop->tfrm_op.inlen = 0;
2575 cop->tfrm_op.outdata = NULL;
2576 cop->tfrm_op.outcount = 0;
2577 cop->tfrm_op.outlen = 0;
2578
2579 sess = get_session(oper.ses_id);
2580 if (!sess){
2581 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2582 kfree(cop);
2583 kfree(jc);
2584 return -EINVAL;
2585 }
2586
2587 if (oper.do_cipher) {
2588 unsigned int cipher_outlen = 0;
2589 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2590 if (!tc) {
2591 DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2592 err = -EINVAL;
2593 goto error_cleanup;
2594 }
2595 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2596 ciph_tcfg.inject_ix = 0;
2597 ciph_tcfg.flags = 0;
2598 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2599 DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2600 kfree(cop);
2601 kfree(jc);
2602 return -EINVAL;
2603 }
2604 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2605 if (oper.cipher_len % cblocklen) {
2606 kfree(cop);
2607 kfree(jc);
2608 DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2609 return -EINVAL;
2610 }
2611 cipher_outlen = oper.cipher_len;
2612 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2613 if (oper.cipher_explicit) {
2614 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2615 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2616 } else {
2617 cipher_outlen = oper.cipher_len - cblocklen;
2618 }
2619 } else {
2620 if (oper.cipher_explicit){
2621 kfree(cop);
2622 kfree(jc);
2623 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2624 return -EINVAL;
2625 }
2626 }
2627 if (oper.cipher_outlen != cipher_outlen) {
2628 kfree(cop);
2629 kfree(jc);
2630 DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2631 return -EINVAL;
2632 }
2633
2634 if (oper.decrypt){
2635 ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2636 } else {
2637 ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2638 }
2639 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2640 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2641 }
2642 if (oper.do_digest){
2643 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2644 if (!tc) {
2645 DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2646 err = -EINVAL;
2647 goto error_cleanup;
2648 }
2649 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2650 digest_result = kmalloc(digest_length, GFP_KERNEL);
2651 if (!digest_result) {
2652 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2653 err = -EINVAL;
2654 goto error_cleanup;
2655 }
2656 DEBUG(memset(digest_result, 0xff, digest_length));
2657
2658 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2659 digest_tcfg.inject_ix = 0;
2660 ciph_tcfg.inject_ix += digest_length;
2661 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2662 DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2663 err = -EINVAL;
2664 goto error_cleanup;
2665 }
2666
2667 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2668 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2669 }
2670 if (oper.do_csum){
2671 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2672 csum_tcfg.inject_ix = digest_length;
2673 ciph_tcfg.inject_ix += 2;
2674
2675 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2676 DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2677 kfree(cop);
2678 kfree(jc);
2679 return -EINVAL;
2680 }
2681
2682 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2683 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2684 }
2685
2686 prev_ix = first_cfg_change_ix(&oper);
2687 if (prev_ix > oper.inlen) {
2688 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2689 nooutpages = noinpages = 0;
2690 err = -EINVAL;
2691 goto error_cleanup;
2692 }
2693 DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2694
2695 /* Map user pages for in and out data of the operation. */
2696 noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2697 DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2698 inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2699 if (!inpages){
2700 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2701 nooutpages = noinpages = 0;
2702 err = -ENOMEM;
2703 goto error_cleanup;
2704 }
2705 if (oper.do_cipher){
2706 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2707 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2708 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2709 if (!outpages){
2710 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2711 nooutpages = noinpages = 0;
2712 err = -ENOMEM;
2713 goto error_cleanup;
2714 }
2715 }
2716
2717 /* Acquire the mm page semaphore. */
2718 down_read(&current->mm->mmap_sem);
2719
2720 err = get_user_pages(current,
2721 current->mm,
2722 (unsigned long int)(oper.indata + prev_ix),
2723 noinpages,
2724 0, /* read access only for in data */
2725 0, /* no force */
2726 inpages,
2727 NULL);
2728
2729 if (err < 0) {
2730 up_read(&current->mm->mmap_sem);
2731 nooutpages = noinpages = 0;
2732 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2733 goto error_cleanup;
2734 }
2735 noinpages = err;
2736 if (oper.do_cipher){
2737 err = get_user_pages(current,
2738 current->mm,
2739 (unsigned long int)oper.cipher_outdata,
2740 nooutpages,
2741 1, /* write access for out data */
2742 0, /* no force */
2743 outpages,
2744 NULL);
2745 up_read(&current->mm->mmap_sem);
2746 if (err < 0) {
2747 nooutpages = 0;
2748 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2749 goto error_cleanup;
2750 }
2751 nooutpages = err;
2752 } else {
2753 up_read(&current->mm->mmap_sem);
2754 }
2755
2756 /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2757 * csum output and splits when units are (dis-)connected. */
2758 cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2759 cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2760 if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2761 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2762 err = -ENOMEM;
2763 goto error_cleanup;
2764 }
2765
2766 cop->tfrm_op.inlen = oper.inlen - prev_ix;
2767 cop->tfrm_op.outlen = 0;
2768 if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2769 if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2770 if (oper.do_csum) cop->tfrm_op.outlen += 2;
2771
2772 /* Setup the in iovecs. */
2773 cop->tfrm_op.incount = noinpages;
2774 if (noinpages > 1){
2775 size_t tmplen = cop->tfrm_op.inlen;
2776
2777 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2778 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2779 tmplen -= cop->tfrm_op.indata[0].iov_len;
2780 for (i = 1; i<noinpages; i++){
2781 cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2782 cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2783 tmplen -= PAGE_SIZE;
2784 }
2785 } else {
2786 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2787 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2788 }
2789
2790 iovlen = nooutpages + 6;
2791 pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2792
2793 next_ix = next_cfg_change_ix(&oper, prev_ix);
2794 if (prev_ix == next_ix){
2795 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2796 err = -EINVAL; /* This should be impossible barring bugs. */
2797 goto error_cleanup;
2798 }
2799 while (prev_ix != next_ix){
2800 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2801 descs[desc_ix].cfg = NULL;
2802 descs[desc_ix].length = next_ix - prev_ix;
2803
2804 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2805 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2806 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2807 cipher_active = 1;
2808
2809 if (next_ix == (oper.cipher_start + oper.cipher_len)){
2810 cipher_done = 1;
2811 dcfgs[dcfg_ix].last = 1;
2812 } else {
2813 dcfgs[dcfg_ix].last = 0;
2814 }
2815 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2816 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2817 ++dcfg_ix;
2818 }
2819 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2820 digest_active = 1;
2821 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2822 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2823 if (next_ix == (oper.digest_start + oper.digest_len)){
2824 assert(!digest_done);
2825 digest_done = 1;
2826 dcfgs[dcfg_ix].last = 1;
2827 } else {
2828 dcfgs[dcfg_ix].last = 0;
2829 }
2830 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2831 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2832 ++dcfg_ix;
2833 }
2834 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2835 csum_active = 1;
2836 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2837 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2838 if (next_ix == (oper.csum_start + oper.csum_len)){
2839 csum_done = 1;
2840 dcfgs[dcfg_ix].last = 1;
2841 } else {
2842 dcfgs[dcfg_ix].last = 0;
2843 }
2844 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2845 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2846 ++dcfg_ix;
2847 }
2848 if (!descs[desc_ix].cfg){
2849 DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2850 err = -EINVAL;
2851 goto error_cleanup;
2852 }
2853 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2854 ++desc_ix;
2855 prev_ix = next_ix;
2856 next_ix = next_cfg_change_ix(&oper, prev_ix);
2857 }
2858 if (desc_ix > 0){
2859 descs[desc_ix-1].next = NULL;
2860 } else {
2861 descs[0].next = NULL;
2862 }
2863 if (oper.do_digest) {
2864 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2865 /* Add outdata iovec, length == <length of type of digest> */
2866 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2867 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2868 ++iovix;
2869 }
2870 if (oper.do_csum) {
2871 /* Add outdata iovec, length == 2, the length of csum. */
2872 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2873 /* Add outdata iovec, length == <length of type of digest> */
2874 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2875 cop->tfrm_op.outdata[iovix].iov_len = 2;
2876 ++iovix;
2877 }
2878 if (oper.do_cipher) {
2879 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2880 DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2881 err = -ENOSYS; /* This should be impossible barring bugs. */
2882 goto error_cleanup;
2883 }
2884 }
2885 DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2886 cop->tfrm_op.outcount = iovix;
2887 assert(iovix <= (nooutpages + 6));
2888
2889 cop->sid = oper.ses_id;
2890 cop->tfrm_op.desc = &descs[0];
2891
2892 DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2893
2894 if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2895 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2896 err = -EINVAL;
2897 goto error_cleanup;
2898 }
2899
2900 DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2901
2902 wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2903 DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2904 if (!jc->processed){
2905 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2906 err = -EIO;
2907 goto error_cleanup;
2908 }
2909
2910 /* Job process done. Cipher output should already be correct in job so no post processing of outdata. */
2911 DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2912 if (cop->operation_status == 0){
2913 if (oper.do_digest){
2914 DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2915 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2916 if (0 != err){
2917 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2918 err = -EFAULT;
2919 goto error_cleanup;
2920 }
2921 }
2922 if (oper.do_csum){
2923 DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2924 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2925 if (0 != err){
2926 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2927 err = -EFAULT;
2928 goto error_cleanup;
2929 }
2930 }
2931 err = 0;
2932 } else {
2933 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2934 err = cop->operation_status;
2935 }
2936
2937 error_cleanup:
2938 /* Release page caches. */
2939 for (i = 0; i < noinpages; i++){
2940 put_page(inpages[i]);
2941 }
2942 for (i = 0; i < nooutpages; i++){
2943 int spdl_err;
2944 /* Mark output pages dirty. */
2945 spdl_err = set_page_dirty_lock(outpages[i]);
2946 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2947 }
2948 for (i = 0; i < nooutpages; i++){
2949 put_page(outpages[i]);
2950 }
2951
2952 kfree(digest_result);
2953 kfree(inpages);
2954 kfree(outpages);
2955 if (cop){
2956 kfree(cop->tfrm_op.indata);
2957 kfree(cop->tfrm_op.outdata);
2958 kfree(cop);
2959 }
2960 kfree(jc);
2961
2962 DEBUG(print_lock_status());
2963
2964 return err;
2965 }
2966
2967
2968 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2969 {
2970 cryptocop_session_id sid;
2971 int err;
2972 struct cryptocop_private *dev;
2973 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2974 struct strcop_session_op sop;
2975 struct cryptocop_transform_init *tis = NULL;
2976 struct cryptocop_transform_init ti_cipher = {0};
2977 struct cryptocop_transform_init ti_digest = {0};
2978 struct cryptocop_transform_init ti_csum = {0};
2979
2980 if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2981 return -EFAULT;
2982 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2983 if (err) return -EFAULT;
2984 if (sop.cipher != cryptocop_cipher_none) {
2985 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2986 }
2987 DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2988
2989 DEBUG(printk("\tcipher:%d\n"
2990 "\tcipher_mode:%d\n"
2991 "\tdigest:%d\n"
2992 "\tcsum:%d\n",
2993 (int)sop.cipher,
2994 (int)sop.cmode,
2995 (int)sop.digest,
2996 (int)sop.csum));
2997
2998 if (sop.cipher != cryptocop_cipher_none){
2999 /* Init the cipher. */
3000 switch (sop.cipher){
3001 case cryptocop_cipher_des:
3002 ti_cipher.alg = cryptocop_alg_des;
3003 break;
3004 case cryptocop_cipher_3des:
3005 ti_cipher.alg = cryptocop_alg_3des;
3006 break;
3007 case cryptocop_cipher_aes:
3008 ti_cipher.alg = cryptocop_alg_aes;
3009 break;
3010 default:
3011 DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3012 return -EINVAL;
3013 };
3014 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3015 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3016 ti_cipher.keylen = sop.keylen;
3017 switch (sop.cmode){
3018 case cryptocop_cipher_mode_cbc:
3019 case cryptocop_cipher_mode_ecb:
3020 ti_cipher.cipher_mode = sop.cmode;
3021 break;
3022 default:
3023 DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3024 return -EINVAL;
3025 }
3026 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3027 switch (sop.des3_mode){
3028 case cryptocop_3des_eee:
3029 case cryptocop_3des_eed:
3030 case cryptocop_3des_ede:
3031 case cryptocop_3des_edd:
3032 case cryptocop_3des_dee:
3033 case cryptocop_3des_ded:
3034 case cryptocop_3des_dde:
3035 case cryptocop_3des_ddd:
3036 ti_cipher.tdes_mode = sop.des3_mode;
3037 break;
3038 default:
3039 DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3040 return -EINVAL;
3041 }
3042 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3043 ti_cipher.next = tis;
3044 tis = &ti_cipher;
3045 } /* if (sop.cipher != cryptocop_cipher_none) */
3046 if (sop.digest != cryptocop_digest_none){
3047 DEBUG(printk("setting digest transform\n"));
3048 switch (sop.digest){
3049 case cryptocop_digest_md5:
3050 ti_digest.alg = cryptocop_alg_md5;
3051 break;
3052 case cryptocop_digest_sha1:
3053 ti_digest.alg = cryptocop_alg_sha1;
3054 break;
3055 default:
3056 DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3057 return -EINVAL;
3058 }
3059 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3060 ti_digest.next = tis;
3061 tis = &ti_digest;
3062 } /* if (sop.digest != cryptocop_digest_none) */
3063 if (sop.csum != cryptocop_csum_none){
3064 DEBUG(printk("setting csum transform\n"));
3065 switch (sop.csum){
3066 case cryptocop_csum_le:
3067 case cryptocop_csum_be:
3068 ti_csum.csum_mode = sop.csum;
3069 break;
3070 default:
3071 DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3072 return -EINVAL;
3073 }
3074 ti_csum.alg = cryptocop_alg_csum;
3075 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3076 ti_csum.next = tis;
3077 tis = &ti_csum;
3078 } /* (sop.csum != cryptocop_csum_none) */
3079 dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3080 if (!dev){
3081 DEBUG_API(printk("create session, alloc dev\n"));
3082 return -ENOMEM;
3083 }
3084
3085 err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3086 DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3087
3088 if (err) {
3089 kfree(dev);
3090 return err;
3091 }
3092 sess_op->ses_id = sid;
3093 dev->sid = sid;
3094 dev->next = filp->private_data;
3095 filp->private_data = dev;
3096
3097 return 0;
3098 }
3099
3100 static int cryptocop_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
3101 {
3102 int err = 0;
3103 if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3104 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3105 return -ENOTTY;
3106 }
3107 if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3108 return -ENOTTY;
3109 }
3110 /* Access check of the argument. Some commands, e.g. create session and process op,
3111 needs additional checks. Those are handled in the command handling functions. */
3112 if (_IOC_DIR(cmd) & _IOC_READ)
3113 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3114 else if (_IOC_DIR(cmd) & _IOC_WRITE)
3115 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3116 if (err) return -EFAULT;
3117
3118 switch (cmd) {
3119 case CRYPTOCOP_IO_CREATE_SESSION:
3120 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3121 case CRYPTOCOP_IO_CLOSE_SESSION:
3122 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3123 case CRYPTOCOP_IO_PROCESS_OP:
3124 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3125 default:
3126 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3127 return -ENOTTY;
3128 }
3129 return 0;
3130 }
3131
3132
3133 #ifdef LDEBUG
3134 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3135 {
3136 struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3137 struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3138 int i;
3139
3140 printk("print_dma_descriptors start\n");
3141
3142 printk("iop:\n");
3143 printk("\tsid: 0x%lld\n", iop->sid);
3144
3145 printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3146 printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3147 printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3148 printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3149
3150 printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3151 printk("\tnext: 0x%p\n"
3152 "\tsaved_data: 0x%p\n"
3153 "\tsaved_data_buf: 0x%p\n",
3154 iop->ctx_out.next,
3155 iop->ctx_out.saved_data,
3156 iop->ctx_out.saved_data_buf);
3157
3158 printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3159 printk("\tnext: 0x%p\n"
3160 "\tsaved_data: 0x%p\n"
3161 "\tsaved_data_buf: 0x%p\n",
3162 iop->ctx_in.next,
3163 iop->ctx_in.saved_data,
3164 iop->ctx_in.saved_data_buf);
3165
3166 i = 0;
3167 while (cdesc_out) {
3168 dma_descr_data *td;
3169 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3170 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3171 td = cdesc_out->dma_descr;
3172 printk("\n\tbuf: 0x%p\n"
3173 "\tafter: 0x%p\n"
3174 "\tmd: 0x%04x\n"
3175 "\tnext: 0x%p\n",
3176 td->buf,
3177 td->after,
3178 td->md,
3179 td->next);
3180 printk("flags:\n"
3181 "\twait:\t%d\n"
3182 "\teol:\t%d\n"
3183 "\touteop:\t%d\n"
3184 "\tineop:\t%d\n"
3185 "\tintr:\t%d\n",
3186 td->wait,
3187 td->eol,
3188 td->out_eop,
3189 td->in_eop,
3190 td->intr);
3191 cdesc_out = cdesc_out->next;
3192 i++;
3193 }
3194 i = 0;
3195 while (cdesc_in) {
3196 dma_descr_data *td;
3197 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3198 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3199 td = cdesc_in->dma_descr;
3200 printk("\n\tbuf: 0x%p\n"
3201 "\tafter: 0x%p\n"
3202 "\tmd: 0x%04x\n"
3203 "\tnext: 0x%p\n",
3204 td->buf,
3205 td->after,
3206 td->md,
3207 td->next);
3208 printk("flags:\n"
3209 "\twait:\t%d\n"
3210 "\teol:\t%d\n"
3211 "\touteop:\t%d\n"
3212 "\tineop:\t%d\n"
3213 "\tintr:\t%d\n",
3214 td->wait,
3215 td->eol,
3216 td->out_eop,
3217 td->in_eop,
3218 td->intr);
3219 cdesc_in = cdesc_in->next;
3220 i++;
3221 }
3222
3223 printk("print_dma_descriptors end\n");
3224 }
3225
3226
3227 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3228 {
3229 printk("print_strcop_crypto_op, 0x%p\n", cop);
3230
3231 /* Indata. */
3232 printk("indata=0x%p\n"
3233 "inlen=%d\n"
3234 "do_cipher=%d\n"
3235 "decrypt=%d\n"
3236 "cipher_explicit=%d\n"
3237 "cipher_start=%d\n"
3238 "cipher_len=%d\n"
3239 "outdata=0x%p\n"
3240 "outlen=%d\n",
3241 cop->indata,
3242 cop->inlen,
3243 cop->do_cipher,
3244 cop->decrypt,
3245 cop->cipher_explicit,
3246 cop->cipher_start,
3247 cop->cipher_len,
3248 cop->cipher_outdata,
3249 cop->cipher_outlen);
3250
3251 printk("do_digest=%d\n"
3252 "digest_start=%d\n"
3253 "digest_len=%d\n",
3254 cop->do_digest,
3255 cop->digest_start,
3256 cop->digest_len);
3257
3258 printk("do_csum=%d\n"
3259 "csum_start=%d\n"
3260 "csum_len=%d\n",
3261 cop->do_csum,
3262 cop->csum_start,
3263 cop->csum_len);
3264 }
3265
3266 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3267 {
3268 struct cryptocop_desc *d;
3269 struct cryptocop_tfrm_cfg *tc;
3270 struct cryptocop_desc_cfg *dc;
3271 int i;
3272
3273 printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3274 printk("sid: %lld\n", cop->sid);
3275 printk("operation_status=%d\n"
3276 "use_dmalists=%d\n"
3277 "in_interrupt=%d\n"
3278 "fast_callback=%d\n",
3279 cop->operation_status,
3280 cop->use_dmalists,
3281 cop->in_interrupt,
3282 cop->fast_callback);
3283
3284 if (cop->use_dmalists){
3285 print_user_dma_lists(&cop->list_op);
3286 } else {
3287 printk("cop->tfrm_op\n"
3288 "tfrm_cfg=0x%p\n"
3289 "desc=0x%p\n"
3290 "indata=0x%p\n"
3291 "incount=%d\n"
3292 "inlen=%d\n"
3293 "outdata=0x%p\n"
3294 "outcount=%d\n"
3295 "outlen=%d\n\n",
3296 cop->tfrm_op.tfrm_cfg,
3297 cop->tfrm_op.desc,
3298 cop->tfrm_op.indata,
3299 cop->tfrm_op.incount,
3300 cop->tfrm_op.inlen,
3301 cop->tfrm_op.outdata,
3302 cop->tfrm_op.outcount,
3303 cop->tfrm_op.outlen);
3304
3305 tc = cop->tfrm_op.tfrm_cfg;
3306 while (tc){
3307 printk("tfrm_cfg, 0x%p\n"
3308 "tid=%d\n"
3309 "flags=%d\n"
3310 "inject_ix=%d\n"
3311 "next=0x%p\n",
3312 tc,
3313 tc->tid,
3314 tc->flags,
3315 tc->inject_ix,
3316 tc->next);
3317 tc = tc->next;
3318 }
3319 d = cop->tfrm_op.desc;
3320 while (d){
3321 printk("\n======================desc, 0x%p\n"
3322 "length=%d\n"
3323 "cfg=0x%p\n"
3324 "next=0x%p\n",
3325 d,
3326 d->length,
3327 d->cfg,
3328 d->next);
3329 dc = d->cfg;
3330 while (dc){
3331 printk("=========desc_cfg, 0x%p\n"
3332 "tid=%d\n"
3333 "src=%d\n"
3334 "last=%d\n"
3335 "next=0x%p\n",
3336 dc,
3337 dc->tid,
3338 dc->src,
3339 dc->last,
3340 dc->next);
3341 dc = dc->next;
3342 }
3343 d = d->next;
3344 }
3345 printk("\n====iniov\n");
3346 for (i = 0; i < cop->tfrm_op.incount; i++){
3347 printk("indata[%d]\n"
3348 "base=0x%p\n"
3349 "len=%d\n",
3350 i,
3351 cop->tfrm_op.indata[i].iov_base,
3352 cop->tfrm_op.indata[i].iov_len);
3353 }
3354 printk("\n====outiov\n");
3355 for (i = 0; i < cop->tfrm_op.outcount; i++){
3356 printk("outdata[%d]\n"
3357 "base=0x%p\n"
3358 "len=%d\n",
3359 i,
3360 cop->tfrm_op.outdata[i].iov_base,
3361 cop->tfrm_op.outdata[i].iov_len);
3362 }
3363 }
3364 printk("------------end print_cryptocop_operation\n");
3365 }
3366
3367
3368 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3369 {
3370 dma_descr_data *dd;
3371 int i;
3372
3373 printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3374
3375 printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3376 printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3377
3378 printk("##############outlist\n");
3379 dd = phys_to_virt((unsigned long int)dma_op->outlist);
3380 i = 0;
3381 while (dd != NULL) {
3382 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3383 printk("\n\tbuf: 0x%p\n"
3384 "\tafter: 0x%p\n"
3385 "\tmd: 0x%04x\n"
3386 "\tnext: 0x%p\n",
3387 dd->buf,
3388 dd->after,
3389 dd->md,
3390 dd->next);
3391 printk("flags:\n"
3392 "\twait:\t%d\n"
3393 "\teol:\t%d\n"
3394 "\touteop:\t%d\n"
3395 "\tineop:\t%d\n"
3396 "\tintr:\t%d\n",
3397 dd->wait,
3398 dd->eol,
3399 dd->out_eop,
3400 dd->in_eop,
3401 dd->intr);
3402 if (dd->eol)
3403 dd = NULL;
3404 else
3405 dd = phys_to_virt((unsigned long int)dd->next);
3406 ++i;
3407 }
3408
3409 printk("##############inlist\n");
3410 dd = phys_to_virt((unsigned long int)dma_op->inlist);
3411 i = 0;
3412 while (dd != NULL) {
3413 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3414 printk("\n\tbuf: 0x%p\n"
3415 "\tafter: 0x%p\n"
3416 "\tmd: 0x%04x\n"
3417 "\tnext: 0x%p\n",
3418 dd->buf,
3419 dd->after,
3420 dd->md,
3421 dd->next);
3422 printk("flags:\n"
3423 "\twait:\t%d\n"
3424 "\teol:\t%d\n"
3425 "\touteop:\t%d\n"
3426 "\tineop:\t%d\n"
3427 "\tintr:\t%d\n",
3428 dd->wait,
3429 dd->eol,
3430 dd->out_eop,
3431 dd->in_eop,
3432 dd->intr);
3433 if (dd->eol)
3434 dd = NULL;
3435 else
3436 dd = phys_to_virt((unsigned long int)dd->next);
3437 ++i;
3438 }
3439 }
3440
3441
3442 static void print_lock_status(void)
3443 {
3444 printk("**********************print_lock_status\n");
3445 printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3446 printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3447 printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3448 printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3449 printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3450 printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3451 }
3452 #endif /* LDEBUG */
3453
3454
3455 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3456
3457 static int init_stream_coprocessor(void)
3458 {
3459 int err;
3460 int i;
3461 static int initialized = 0;
3462
3463 if (initialized)
3464 return 0;
3465
3466 initialized = 1;
3467
3468 printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3469
3470 err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3471 if (err < 0) {
3472 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3473 return err;
3474 }
3475
3476 err = init_cryptocop();
3477 if (err) {
3478 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3479 return err;
3480 }
3481 err = cryptocop_job_queue_init();
3482 if (err) {
3483 release_cryptocop();
3484 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3485 return err;
3486 }
3487 /* Init the descriptor pool. */
3488 for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3489 descr_pool[i].from_pool = 1;
3490 descr_pool[i].next = &descr_pool[i + 1];
3491 }
3492 descr_pool[i].from_pool = 1;
3493 descr_pool[i].next = NULL;
3494 descr_pool_free_list = &descr_pool[0];
3495 descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3496
3497 spin_lock_init(&cryptocop_completed_jobs_lock);
3498 spin_lock_init(&cryptocop_job_queue_lock);
3499 spin_lock_init(&descr_pool_lock);
3500 spin_lock_init(&cryptocop_sessions_lock);
3501 spin_lock_init(&running_job_lock);
3502 spin_lock_init(&cryptocop_process_lock);
3503
3504 cryptocop_sessions = NULL;
3505 next_sid = 1;
3506
3507 cryptocop_running_job = NULL;
3508
3509 printk("stream co-processor: init done.\n");
3510 return 0;
3511 }
3512
3513 static void __exit exit_stream_coprocessor(void)
3514 {
3515 release_cryptocop();
3516 cryptocop_job_queue_close();
3517 }
3518
3519 module_init(init_stream_coprocessor);
3520 module_exit(exit_stream_coprocessor);
3521
Linus' kernel tree