?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
add some unlikely() hints to get_userbuf()
[cryptodev-linux.git] / cryptodev_main.c
blobd2c78c114efe8ba970e5b83d9393f7101c2cc9aa
1 /*
2 * Driver for /dev/crypto device (aka CryptoDev)
3 *
4 * Copyright (c) 2004 Michal Ludvig <mludvig@logix.net.nz>, SuSE Labs
5 * Copyright (c) 2009,2010 Nikos Mavrogiannopoulos <nmav@gnutls.org>
6 *
7 * This file is part of linux cryptodev.
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License
11 * as published by the Free Software Foundation; either version 2
12 * of the License, or (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc.,
22 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
23 */
24
25 /*
26 * Device /dev/crypto provides an interface for
27 * accessing kernel CryptoAPI algorithms (ciphers,
28 * hashes) from userspace programs.
29 *
30 * /dev/crypto interface was originally introduced in
31 * OpenBSD and this module attempts to keep the API.
32 *
33 */
34
35 #include <linux/crypto.h>
36 #include <linux/mm.h>
37 #include <linux/highmem.h>
38 #include <linux/ioctl.h>
39 #include <linux/random.h>
40 #include <linux/syscalls.h>
41 #include <linux/pagemap.h>
42 #include <linux/uaccess.h>
43 #include "cryptodev.h"
44 #include <linux/scatterlist.h>
45 #include "cryptodev_int.h"
46 #include "version.h"
47
48 MODULE_AUTHOR("Nikos Mavrogiannopoulos <nmav@gnutls.org>");
49 MODULE_DESCRIPTION("CryptoDev driver");
50 MODULE_LICENSE("GPL");
51
52 /* ====== Compile-time config ====== */
53
54 #define CRYPTODEV_STATS
55
56 /* ====== Module parameters ====== */
57
58 int cryptodev_verbosity;
59 module_param(cryptodev_verbosity, int, 0644);
60 MODULE_PARM_DESC(cryptodev_verbosity, "0: normal, 1: verbose, 2: debug");
61
62 #ifdef CRYPTODEV_STATS
63 static int enable_stats;
64 module_param(enable_stats, int, 0644);
65 MODULE_PARM_DESC(enable_stats, "collect statictics about cryptodev usage");
66 #endif
67
68 /* ====== CryptoAPI ====== */
69 struct fcrypt {
70 struct list_head list;
71 struct mutex sem;
72 };
73
74 struct crypt_priv {
75 struct fcrypt fcrypt;
76 };
77
78 #define FILL_SG(sg, ptr, len) \
79 do { \
80 (sg)->page = virt_to_page(ptr); \
81 (sg)->offset = offset_in_page(ptr); \
82 (sg)->length = len; \
83 (sg)->dma_address = 0; \
84 } while (0)
85
86 struct csession {
87 struct list_head entry;
88 struct mutex sem;
89 struct cipher_data cdata;
90 struct hash_data hdata;
91 uint32_t sid;
92 #ifdef CRYPTODEV_STATS
93 #if !((COP_ENCRYPT < 2) && (COP_DECRYPT < 2))
94 #error Struct csession.stat uses COP_{ENCRYPT,DECRYPT} as indices. Do something!
95 #endif
96 unsigned long long stat[2];
97 size_t stat_max_size, stat_count;
98 #endif
99 int array_size;
100 struct page **pages;
101 struct scatterlist *sg;
102 };
103
104 /* Prepare session for future use. */
105 static int
106 crypto_create_session(struct fcrypt *fcr, struct session_op *sop)
107 {
108 struct csession *ses_new = NULL, *ses_ptr;
109 int ret = 0;
110 const char *alg_name = NULL;
111 const char *hash_name = NULL;
112 int hmac_mode = 1;
113
114 /* Does the request make sense? */
115 if (unlikely(!sop->cipher && !sop->mac)) {
116 dprintk(1, KERN_DEBUG, "Both 'cipher' and 'mac' unset.\n");
117 return -EINVAL;
118 }
119
120 switch (sop->cipher) {
121 case 0:
122 break;
123 case CRYPTO_DES_CBC:
124 alg_name = "cbc(des)";
125 break;
126 case CRYPTO_3DES_CBC:
127 alg_name = "cbc(des3_ede)";
128 break;
129 case CRYPTO_BLF_CBC:
130 alg_name = "cbc(blowfish)";
131 break;
132 case CRYPTO_AES_CBC:
133 alg_name = "cbc(aes)";
134 break;
135 case CRYPTO_CAMELLIA_CBC:
136 alg_name = "cbc(camelia)";
137 break;
138 case CRYPTO_AES_CTR:
139 alg_name = "ctr(aes)";
140 break;
141 case CRYPTO_NULL:
142 alg_name = "ecb(cipher_null)";
143 break;
144 default:
145 dprintk(1, KERN_DEBUG, "%s: bad cipher: %d\n", __func__,
146 sop->cipher);
147 return -EINVAL;
148 }
149
150 switch (sop->mac) {
151 case 0:
152 break;
153 case CRYPTO_MD5_HMAC:
154 hash_name = "hmac(md5)";
155 break;
156 case CRYPTO_RIPEMD160_HMAC:
157 hash_name = "hmac(rmd160)";
158 break;
159 case CRYPTO_SHA1_HMAC:
160 hash_name = "hmac(sha1)";
161 break;
162 case CRYPTO_SHA2_256_HMAC:
163 hash_name = "hmac(sha256)";
164 break;
165 case CRYPTO_SHA2_384_HMAC:
166 hash_name = "hmac(sha384)";
167 break;
168 case CRYPTO_SHA2_512_HMAC:
169 hash_name = "hmac(sha512)";
170 break;
171
172 /* non-hmac cases */
173 case CRYPTO_MD5:
174 hash_name = "md5";
175 hmac_mode = 0;
176 break;
177 case CRYPTO_RIPEMD160:
178 hash_name = "rmd160";
179 hmac_mode = 0;
180 break;
181 case CRYPTO_SHA1:
182 hash_name = "sha1";
183 hmac_mode = 0;
184 break;
185 case CRYPTO_SHA2_256:
186 hash_name = "sha256";
187 hmac_mode = 0;
188 break;
189 case CRYPTO_SHA2_384:
190 hash_name = "sha384";
191 hmac_mode = 0;
192 break;
193 case CRYPTO_SHA2_512:
194 hash_name = "sha512";
195 hmac_mode = 0;
196 break;
197
198 default:
199 dprintk(1, KERN_DEBUG, "%s: bad mac: %d\n", __func__,
200 sop->mac);
201 return -EINVAL;
202 }
203
204 /* Create a session and put it to the list. */
205 ses_new = kzalloc(sizeof(*ses_new), GFP_KERNEL);
206 if (!ses_new)
207 return -ENOMEM;
208
209 /* Set-up crypto transform. */
210 if (alg_name) {
211 uint8_t keyp[CRYPTO_CIPHER_MAX_KEY_LEN];
212
213 if (unlikely(sop->keylen > CRYPTO_CIPHER_MAX_KEY_LEN)) {
214 dprintk(1, KERN_DEBUG,
215 "Setting key failed for %s-%zu.\n",
216 alg_name, (size_t)sop->keylen*8);
217 ret = -EINVAL;
218 goto error_cipher;
219 }
220
221 if (unlikely(copy_from_user(keyp, sop->key, sop->keylen))) {
222 ret = -EFAULT;
223 goto error_cipher;
224 }
225
226 ret = cryptodev_cipher_init(&ses_new->cdata, alg_name, keyp,
227 sop->keylen);
228 if (ret < 0) {
229 dprintk(1, KERN_DEBUG,
230 "%s: Failed to load cipher for %s\n",
231 __func__, alg_name);
232 ret = -EINVAL;
233 goto error_cipher;
234 }
235 }
236
237 if (hash_name) {
238 uint8_t keyp[CRYPTO_HMAC_MAX_KEY_LEN];
239
240 if (unlikely(sop->mackeylen > CRYPTO_HMAC_MAX_KEY_LEN)) {
241 dprintk(1, KERN_DEBUG,
242 "Setting key failed for %s-%zu.\n",
243 alg_name, (size_t)sop->mackeylen*8);
244 ret = -EINVAL;
245 goto error_hash;
246 }
247
248 if (unlikely(copy_from_user(keyp, sop->mackey,
249 sop->mackeylen))) {
250 ret = -EFAULT;
251 goto error_hash;
252 }
253
254 ret = cryptodev_hash_init(&ses_new->hdata, hash_name, hmac_mode,
255 keyp, sop->mackeylen);
256 if (ret != 0) {
257 dprintk(1, KERN_DEBUG,
258 "%s: Failed to load hash for %s\n",
259 __func__, hash_name);
260 ret = -EINVAL;
261 goto error_hash;
262 }
263 }
264
265 ses_new->array_size = DEFAULT_PREALLOC_PAGES;
266 dprintk(2, KERN_DEBUG, "%s: preallocating for %d user pages\n",
267 __func__, ses_new->array_size);
268 ses_new->pages = kzalloc(ses_new->array_size *
269 sizeof(struct page *), GFP_KERNEL);
270 ses_new->sg = kzalloc(ses_new->array_size *
271 sizeof(struct scatterlist), GFP_KERNEL);
272 if (ses_new->sg == NULL || ses_new->pages == NULL) {
273 dprintk(0, KERN_DEBUG, "Memory error\n");
274 ret = -ENOMEM;
275 goto error_hash;
276 }
277
278 /* put the new session to the list */
279 get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
280 mutex_init(&ses_new->sem);
281
282 mutex_lock(&fcr->sem);
283 restart:
284 list_for_each_entry(ses_ptr, &fcr->list, entry) {
285 /* Check for duplicate SID */
286 if (unlikely(ses_new->sid == ses_ptr->sid)) {
287 get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
288 /* Unless we have a broken RNG this
289 shouldn't loop forever... ;-) */
290 goto restart;
291 }
292 }
293
294 list_add(&ses_new->entry, &fcr->list);
295 mutex_unlock(&fcr->sem);
296
297 /* Fill in some values for the user. */
298 sop->ses = ses_new->sid;
299
300 return 0;
301
302 error_hash:
303 cryptodev_cipher_deinit(&ses_new->cdata);
304 kfree(ses_new->sg);
305 kfree(ses_new->pages);
306 error_cipher:
307 kfree(ses_new);
308
309 return ret;
310
311 }
312
313 /* Everything that needs to be done when remowing a session. */
314 static inline void
315 crypto_destroy_session(struct csession *ses_ptr)
316 {
317 if (!mutex_trylock(&ses_ptr->sem)) {
318 dprintk(2, KERN_DEBUG, "Waiting for semaphore of sid=0x%08X\n",
319 ses_ptr->sid);
320 mutex_lock(&ses_ptr->sem);
321 }
322 dprintk(2, KERN_DEBUG, "Removed session 0x%08X\n", ses_ptr->sid);
323 #if defined(CRYPTODEV_STATS)
324 if (enable_stats)
325 dprintk(2, KERN_DEBUG,
326 "Usage in Bytes: enc=%llu, dec=%llu, \
327 max=%zu, avg=%lu, cnt=%zu\n",
328 ses_ptr->stat[COP_ENCRYPT], ses_ptr->stat[COP_DECRYPT],
329 ses_ptr->stat_max_size, ses_ptr->stat_count > 0
330 ? ((unsigned long)(ses_ptr->stat[COP_ENCRYPT]+
331 ses_ptr->stat[COP_DECRYPT]) /
332 ses_ptr->stat_count) : 0,
333 ses_ptr->stat_count);
334 #endif
335 cryptodev_cipher_deinit(&ses_ptr->cdata);
336 cryptodev_hash_deinit(&ses_ptr->hdata);
337 dprintk(2, KERN_DEBUG, "%s: freeing space for %d user pages\n",
338 __func__, ses_ptr->array_size);
339 kfree(ses_ptr->pages);
340 kfree(ses_ptr->sg);
341 mutex_unlock(&ses_ptr->sem);
342 kfree(ses_ptr);
343 }
344
345 /* Look up a session by ID and remove. */
346 static int
347 crypto_finish_session(struct fcrypt *fcr, uint32_t sid)
348 {
349 struct csession *tmp, *ses_ptr;
350 struct list_head *head;
351 int ret = 0;
352
353 mutex_lock(&fcr->sem);
354 head = &fcr->list;
355 list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
356 if (ses_ptr->sid == sid) {
357 list_del(&ses_ptr->entry);
358 crypto_destroy_session(ses_ptr);
359 break;
360 }
361 }
362
363 if (unlikely(!ses_ptr)) {
364 dprintk(1, KERN_ERR, "Session with sid=0x%08X not found!\n",
365 sid);
366 ret = -ENOENT;
367 }
368 mutex_unlock(&fcr->sem);
369
370 return ret;
371 }
372
373 /* Remove all sessions when closing the file */
374 static int
375 crypto_finish_all_sessions(struct fcrypt *fcr)
376 {
377 struct csession *tmp, *ses_ptr;
378 struct list_head *head;
379
380 mutex_lock(&fcr->sem);
381
382 head = &fcr->list;
383 list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
384 list_del(&ses_ptr->entry);
385 crypto_destroy_session(ses_ptr);
386 }
387 mutex_unlock(&fcr->sem);
388
389 return 0;
390 }
391
392 /* Look up session by session ID. The returned session is locked. */
393 static struct csession *
394 crypto_get_session_by_sid(struct fcrypt *fcr, uint32_t sid)
395 {
396 struct csession *ses_ptr;
397
398 mutex_lock(&fcr->sem);
399 list_for_each_entry(ses_ptr, &fcr->list, entry) {
400 if (ses_ptr->sid == sid) {
401 mutex_lock(&ses_ptr->sem);
402 break;
403 }
404 }
405 mutex_unlock(&fcr->sem);
406
407 return ses_ptr;
408 }
409
410 static int
411 hash_n_crypt(struct csession *ses_ptr, struct crypt_op *cop,
412 struct scatterlist *src_sg, struct scatterlist *dst_sg,
413 uint32_t len)
414 {
415 int ret;
416
417 /* Always hash before encryption and after decryption. Maybe
418 * we should introduce a flag to switch... TBD later on.
419 */
420 if (cop->op == COP_ENCRYPT) {
421 if (ses_ptr->hdata.init != 0) {
422 ret = cryptodev_hash_update(&ses_ptr->hdata,
423 src_sg, len);
424 if (unlikely(ret))
425 goto out_err;
426 }
427 if (ses_ptr->cdata.init != 0) {
428 ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
429 src_sg, dst_sg, len);
430
431 if (unlikely(ret))
432 goto out_err;
433 }
434 } else {
435 if (ses_ptr->cdata.init != 0) {
436 ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
437 src_sg, dst_sg, len);
438
439 if (unlikely(ret))
440 goto out_err;
441 }
442
443 if (ses_ptr->hdata.init != 0) {
444 ret = cryptodev_hash_update(&ses_ptr->hdata,
445 dst_sg, len);
446 if (unlikely(ret))
447 goto out_err;
448 }
449 }
450 return 0;
451 out_err:
452 dprintk(0, KERN_ERR, "CryptoAPI failure: %d\n", ret);
453 return ret;
454 }
455
456
457 /* This is the main crypto function - feed it with plaintext
458 and get a ciphertext (or vice versa :-) */
459 static int
460 __crypto_run_std(struct csession *ses_ptr, struct crypt_op *cop)
461 {
462 char *data;
463 char __user *src, *dst;
464 struct scatterlist sg;
465 size_t nbytes, bufsize;
466 int ret = 0;
467
468 nbytes = cop->len;
469 data = (char *)__get_free_page(GFP_KERNEL);
470
471 if (unlikely(!data))
472 return -ENOMEM;
473
474 bufsize = PAGE_SIZE < nbytes ? PAGE_SIZE : nbytes;
475
476 src = cop->src;
477 dst = cop->dst;
478
479 while (nbytes > 0) {
480 size_t current_len = nbytes > bufsize ? bufsize : nbytes;
481
482 if (unlikely(copy_from_user(data, src, current_len))) {
483 ret = -EFAULT;
484 break;
485 }
486
487 sg_init_one(&sg, data, current_len);
488
489 ret = hash_n_crypt(ses_ptr, cop, &sg, &sg, current_len);
490
491 if (unlikely(ret))
492 break;
493
494 if (ses_ptr->cdata.init != 0) {
495 if (unlikely(copy_to_user(dst, data, current_len))) {
496 ret = -EFAULT;
497 break;
498 }
499 }
500
501 dst += current_len;
502 nbytes -= current_len;
503 src += current_len;
504 }
505
506 free_page((unsigned long)data);
507 return ret;
508 }
509
510 void release_user_pages(struct page **pg, int pagecount)
511 {
512 while (pagecount--) {
513 if (!PageReserved(pg[pagecount]))
514 SetPageDirty(pg[pagecount]);
515 page_cache_release(pg[pagecount]);
516 }
517 }
518
519 /* offset of buf in it's first page */
520 #define PAGEOFFSET(buf) ((unsigned long)buf & ~PAGE_MASK)
521
522 /* fetch the pages addr resides in into pg and initialise sg with them */
523 int __get_userbuf(uint8_t __user *addr, uint32_t len, int write,
524 int pgcount, struct page **pg, struct scatterlist *sg)
525 {
526 int ret, pglen, i = 0;
527 struct scatterlist *sgp;
528
529 down_write(&current->mm->mmap_sem);
530 ret = get_user_pages(current, current->mm,
531 (unsigned long)addr, pgcount, write, 0, pg, NULL);
532 up_write(&current->mm->mmap_sem);
533 if (ret != pgcount)
534 return -EINVAL;
535
536 sg_init_table(sg, pgcount);
537
538 pglen = min((ptrdiff_t)(PAGE_SIZE - PAGEOFFSET(addr)), (ptrdiff_t)len);
539 sg_set_page(sg, pg[i++], pglen, PAGEOFFSET(addr));
540
541 len -= pglen;
542 for (sgp = sg_next(sg); len; sgp = sg_next(sgp)) {
543 pglen = min((uint32_t)PAGE_SIZE, len);
544 sg_set_page(sgp, pg[i++], pglen, 0);
545 len -= pglen;
546 }
547 sg_mark_end(sg_last(sg, pgcount));
548 return 0;
549 }
550
551 /* make cop->src and cop->dst available in scatterlists */
552 static int get_userbuf(struct csession *ses, struct kernel_crypt_op *kcop,
553 struct scatterlist **src_sg, struct scatterlist **dst_sg,
554 int *tot_pages)
555 {
556 int src_pagecount, dst_pagecount = 0, pagecount, write_src = 1;
557 struct crypt_op *cop = &kcop->cop;
558
559 if (cop->src == NULL)
560 return -EINVAL;
561
562 src_pagecount = PAGECOUNT(cop->src, cop->len);
563 if (!ses->cdata.init) { /* hashing only */
564 write_src = 0;
565 } else if (cop->src != cop->dst) { /* non-in-situ transformation */
566 if (cop->dst == NULL)
567 return -EINVAL;
568
569 dst_pagecount = PAGECOUNT(cop->dst, cop->len);
570 write_src = 0;
571 }
572 (*tot_pages) = pagecount = src_pagecount + dst_pagecount;
573
574 if (pagecount > ses->array_size) {
575 struct scatterlist *sg;
576 struct page **pages;
577 int array_size;
578
579 for (array_size = ses->array_size; array_size < pagecount;
580 array_size *= 2)
581 ;
582
583 dprintk(2, KERN_DEBUG, "%s: reallocating to %d elements\n",
584 __func__, array_size);
585 pages = krealloc(ses->pages, array_size * sizeof(struct page *),
586 GFP_KERNEL);
587 if (unlikely(!pages))
588 return -ENOMEM;
589 ses->pages = pages;
590 sg = krealloc(ses->sg, array_size * sizeof(struct scatterlist),
591 GFP_KERNEL);
592 if (unlikely(!sg))
593 return -ENOMEM;
594 ses->sg = sg;
595 ses->array_size = array_size;
596 }
597
598 if (__get_userbuf(cop->src, cop->len, write_src,
599 src_pagecount, ses->pages, ses->sg)) {
600 dprintk(1, KERN_ERR,
601 "failed to get user pages for data input\n");
602 return -EINVAL;
603 }
604 (*src_sg) = (*dst_sg) = ses->sg;
605
606 if (dst_pagecount) {
607 (*dst_sg) = ses->sg + src_pagecount;
608
609 if (__get_userbuf(cop->dst, cop->len, 1, dst_pagecount,
610 ses->pages + src_pagecount, *dst_sg)) {
611 dprintk(1, KERN_ERR,
612 "failed to get user pages for data output\n");
613 release_user_pages(ses->pages, src_pagecount);
614 return -EINVAL;
615 }
616 }
617 return 0;
618 }
619
620 /* This is the main crypto function - zero-copy edition */
621 static int
622 __crypto_run_zc(struct csession *ses_ptr, struct kernel_crypt_op *kcop)
623 {
624 struct scatterlist *src_sg, *dst_sg;
625 struct crypt_op *cop = &kcop->cop;
626 int ret = 0, pagecount;
627
628 ret = get_userbuf(ses_ptr, kcop, &src_sg, &dst_sg, &pagecount);
629 if (unlikely(ret)) {
630 dprintk(1, KERN_ERR, "Error getting user pages. \
631 Falling back to non zero copy.\n");
632 return __crypto_run_std(ses_ptr, cop);
633 }
634
635 ret = hash_n_crypt(ses_ptr, cop, src_sg, dst_sg, cop->len);
636
637 release_user_pages(ses_ptr->pages, pagecount);
638 return ret;
639 }
640
641 static int crypto_run(struct fcrypt *fcr, struct kernel_crypt_op *kcop)
642 {
643 struct csession *ses_ptr;
644 struct crypt_op *cop = &kcop->cop;
645 int ret;
646
647 if (unlikely(cop->op != COP_ENCRYPT && cop->op != COP_DECRYPT)) {
648 dprintk(1, KERN_DEBUG, "invalid operation op=%u\n", cop->op);
649 return -EINVAL;
650 }
651
652 /* this also enters ses_ptr->sem */
653 ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
654 if (unlikely(!ses_ptr)) {
655 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
656 return -EINVAL;
657 }
658
659 if (ses_ptr->hdata.init != 0 && !(cop->flags & COP_FLAG_UPDATE) &&
660 !(cop->flags & COP_FLAG_FINAL)) {
661 ret = cryptodev_hash_reset(&ses_ptr->hdata);
662 if (unlikely(ret)) {
663 dprintk(1, KERN_ERR,
664 "error in cryptodev_hash_reset()\n");
665 goto out_unlock;
666 }
667 }
668
669 if (ses_ptr->cdata.init != 0) {
670 int blocksize = ses_ptr->cdata.blocksize;
671
672 if (unlikely(cop->len % blocksize)) {
673 dprintk(1, KERN_ERR,
674 "data size (%u) isn't a multiple \
675 of block size (%u)\n",
676 cop->len, blocksize);
677 ret = -EINVAL;
678 goto out_unlock;
679 }
680
681 cryptodev_cipher_set_iv(&ses_ptr->cdata, kcop->iv,
682 min(ses_ptr->cdata.ivsize, kcop->ivlen));
683 }
684
685 if (cop->len != 0) {
686 ret = __crypto_run_zc(ses_ptr, kcop);
687 if (unlikely(ret))
688 goto out_unlock;
689 }
690
691 if (ses_ptr->hdata.init != 0 &&
692 ((cop->flags & COP_FLAG_FINAL) ||
693 (!(cop->flags & COP_FLAG_UPDATE) || cop->len == 0))) {
694
695 ret = cryptodev_hash_final(&ses_ptr->hdata, kcop->hash_output);
696 if (unlikely(ret)) {
697 dprintk(0, KERN_ERR, "CryptoAPI failure: %d\n", ret);
698 goto out_unlock;
699 }
700 }
701
702 #if defined(CRYPTODEV_STATS)
703 if (enable_stats) {
704 /* this is safe - we check cop->op at the function entry */
705 ses_ptr->stat[cop->op] += cop->len;
706 if (ses_ptr->stat_max_size < cop->len)
707 ses_ptr->stat_max_size = cop->len;
708 ses_ptr->stat_count++;
709 }
710 #endif
711
712 out_unlock:
713 mutex_unlock(&ses_ptr->sem);
714 return ret;
715 }
716
717 /* ====== /dev/crypto ====== */
718
719 static int
720 cryptodev_open(struct inode *inode, struct file *filp)
721 {
722 struct crypt_priv *pcr;
723
724 pcr = kmalloc(sizeof(*pcr), GFP_KERNEL);
725 if (!pcr)
726 return -ENOMEM;
727
728 memset(pcr, 0, sizeof(*pcr));
729 mutex_init(&pcr->fcrypt.sem);
730 INIT_LIST_HEAD(&pcr->fcrypt.list);
731
732 filp->private_data = pcr;
733 return 0;
734 }
735
736 static int
737 cryptodev_release(struct inode *inode, struct file *filp)
738 {
739 struct crypt_priv *pcr = filp->private_data;
740
741 if (pcr) {
742 crypto_finish_all_sessions(&pcr->fcrypt);
743 kfree(pcr);
744 filp->private_data = NULL;
745 }
746
747 return 0;
748 }
749
750 static int
751 clonefd(struct file *filp)
752 {
753 int ret;
754 ret = get_unused_fd();
755 if (ret >= 0) {
756 get_file(filp);
757 fd_install(ret, filp);
758 }
759
760 return ret;
761 }
762
763 /* this function has to be called from process context */
764 static int fill_kcop_from_cop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
765 {
766 struct crypt_op *cop = &kcop->cop;
767 struct csession *ses_ptr;
768 int rc;
769
770 /* this also enters ses_ptr->sem */
771 ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
772 if (unlikely(!ses_ptr)) {
773 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
774 return -EINVAL;
775 }
776 kcop->ivlen = cop->iv ? ses_ptr->cdata.ivsize : 0;
777 kcop->digestsize = ses_ptr->hdata.digestsize;
778 mutex_unlock(&ses_ptr->sem);
779
780 if (unlikely(rc = copy_from_user(kcop->iv, cop->iv, kcop->ivlen))) {
781 dprintk(1, KERN_ERR,
782 "error copying IV (%d bytes), copy_from_user returned %d for address %lx\n",
783 kcop->ivlen, rc, (unsigned long)cop->iv);
784 return -EFAULT;
785 }
786
787 return 0;
788 }
789
790 static int kcop_from_user(struct kernel_crypt_op *kcop,
791 struct fcrypt *fcr, void __user *arg)
792 {
793 if (unlikely(copy_from_user(&kcop->cop, arg, sizeof(kcop->cop))))
794 return -EFAULT;
795
796 return fill_kcop_from_cop(kcop, fcr);
797 }
798
799 static long
800 cryptodev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg_)
801 {
802 void __user *arg = (void __user *)arg_;
803 int __user *p = arg;
804 struct session_op sop;
805 struct kernel_crypt_op kcop;
806 struct crypt_priv *pcr = filp->private_data;
807 struct fcrypt *fcr;
808 uint32_t ses;
809 int ret, fd;
810
811 if (unlikely(!pcr))
812 BUG();
813
814 fcr = &pcr->fcrypt;
815
816 switch (cmd) {
817 case CIOCASYMFEAT:
818 return put_user(0, p);
819 case CRIOGET:
820 fd = clonefd(filp);
821 ret = put_user(fd, p);
822 if (unlikely(ret)) {
823 sys_close(fd);
824 return ret;
825 }
826 return ret;
827 case CIOCGSESSION:
828 if (unlikely(copy_from_user(&sop, arg, sizeof(sop))))
829 return -EFAULT;
830
831 ret = crypto_create_session(fcr, &sop);
832 if (unlikely(ret))
833 return ret;
834 ret = copy_to_user(arg, &sop, sizeof(sop));
835 if (unlikely(ret)) {
836 crypto_finish_session(fcr, sop.ses);
837 return -EFAULT;
838 }
839 return ret;
840 case CIOCFSESSION:
841 ret = get_user(ses, (uint32_t __user *)arg);
842 if (unlikely(ret))
843 return ret;
844 ret = crypto_finish_session(fcr, ses);
845 return ret;
846 case CIOCCRYPT:
847 if (unlikely(ret = kcop_from_user(&kcop, fcr, arg)))
848 return ret;
849
850 ret = crypto_run(fcr, &kcop);
851 if (unlikely(ret))
852 return ret;
853 ret = copy_to_user(kcop.cop.mac, kcop.hash_output, kcop.digestsize);
854 if (unlikely(ret))
855 return ret;
856 if (unlikely(copy_to_user(arg, &kcop.cop, sizeof(kcop.cop))))
857 return -EFAULT;
858 return 0;
859
860 default:
861 return -EINVAL;
862 }
863 }
864
865 /* compatibility code for 32bit userlands */
866 #ifdef CONFIG_COMPAT
867
868 static inline void
869 compat_to_session_op(struct compat_session_op *compat, struct session_op *sop)
870 {
871 sop->cipher = compat->cipher;
872 sop->mac = compat->mac;
873 sop->keylen = compat->keylen;
874
875 sop->key = compat_ptr(compat->key);
876 sop->mackeylen = compat->mackeylen;
877 sop->mackey = compat_ptr(compat->mackey);
878 sop->ses = compat->ses;
879 }
880
881 static inline void
882 session_op_to_compat(struct session_op *sop, struct compat_session_op *compat)
883 {
884 compat->cipher = sop->cipher;
885 compat->mac = sop->mac;
886 compat->keylen = sop->keylen;
887
888 compat->key = ptr_to_compat(sop->key);
889 compat->mackeylen = sop->mackeylen;
890 compat->mackey = ptr_to_compat(sop->mackey);
891 compat->ses = sop->ses;
892 }
893
894 static inline void
895 compat_to_crypt_op(struct compat_crypt_op *compat, struct crypt_op *cop)
896 {
897 cop->ses = compat->ses;
898 cop->op = compat->op;
899 cop->flags = compat->flags;
900 cop->len = compat->len;
901
902 cop->src = compat_ptr(compat->src);
903 cop->dst = compat_ptr(compat->dst);
904 cop->mac = compat_ptr(compat->mac);
905 cop->iv = compat_ptr(compat->iv);
906 }
907
908 static inline void
909 crypt_op_to_compat(struct crypt_op *cop, struct compat_crypt_op *compat)
910 {
911 compat->ses = cop->ses;
912 compat->op = cop->op;
913 compat->flags = cop->flags;
914 compat->len = cop->len;
915
916 compat->src = ptr_to_compat(cop->src);
917 compat->dst = ptr_to_compat(cop->dst);
918 compat->mac = ptr_to_compat(cop->mac);
919 compat->iv = ptr_to_compat(cop->iv);
920 }
921
922 static int compat_kcop_from_user(struct kernel_crypt_op *kcop,
923 struct fcrypt *fcr, void __user *arg)
924 {
925 struct compat_crypt_op compat_cop;
926
927 if (unlikely(copy_from_user(&compat_cop, arg, sizeof(compat_cop))))
928 return -EFAULT;
929 compat_to_crypt_op(&compat_cop, &kcop->cop);
930
931 return fill_kcop_from_cop(kcop, fcr);
932 }
933
934 static long
935 cryptodev_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg_)
936 {
937 void __user *arg = (void __user *)arg_;
938 struct fcrypt *fcr = file->private_data;
939 struct session_op sop;
940 struct compat_session_op compat_sop;
941 struct kernel_crypt_op kcop;
942 struct compat_crypt_op compat_cop;
943 int ret;
944
945 if (unlikely(!fcr))
946 BUG();
947
948 switch (cmd) {
949 case CIOCASYMFEAT:
950 case CRIOGET:
951 case CIOCFSESSION:
952 return cryptodev_ioctl(file, cmd, arg_);
953
954 case COMPAT_CIOCGSESSION:
955 if (unlikely(copy_from_user(&compat_sop, arg,
956 sizeof(compat_sop))))
957 return -EFAULT;
958 compat_to_session_op(&compat_sop, &sop);
959
960 ret = crypto_create_session(fcr, &sop);
961 if (unlikely(ret))
962 return ret;
963
964 session_op_to_compat(&sop, &compat_sop);
965 ret = copy_to_user(arg, &compat_sop, sizeof(compat_sop));
966 if (unlikely(ret)) {
967 crypto_finish_session(fcr, sop.ses);
968 return -EFAULT;
969 }
970 return ret;
971
972 case COMPAT_CIOCCRYPT:
973 ret = compat_kcop_from_user(&kcop, fcr, arg);
974 if (unlikely(ret))
975 return ret;
976
977 ret = crypto_run(fcr, &kcop);
978 if (unlikely(ret))
979 return ret;
980
981 ret = copy_to_user(kcop.cop.mac,
982 kcop.hash_output, kcop.digestsize);
983 if (unlikely(ret))
984 return ret;
985
986 crypt_op_to_compat(&kcop.cop, &compat_cop);
987 if (unlikely(copy_to_user(arg, &compat_cop,
988 sizeof(compat_cop))))
989 return -EFAULT;
990 return 0;
991
992 default:
993 return -EINVAL;
994 }
995 }
996
997 #endif /* CONFIG_COMPAT */
998
999 static const struct file_operations cryptodev_fops = {
1000 .owner = THIS_MODULE,
1001 .open = cryptodev_open,
1002 .release = cryptodev_release,
1003 .unlocked_ioctl = cryptodev_ioctl,
1004 #ifdef CONFIG_COMPAT
1005 .compat_ioctl = cryptodev_compat_ioctl,
1006 #endif /* CONFIG_COMPAT */
1007 };
1008
1009 static struct miscdevice cryptodev = {
1010 .minor = MISC_DYNAMIC_MINOR,
1011 .name = "crypto",
1012 .fops = &cryptodev_fops,
1013 };
1014
1015 static int __init
1016 cryptodev_register(void)
1017 {
1018 int rc;
1019
1020 rc = misc_register(&cryptodev);
1021 if (unlikely(rc)) {
1022 printk(KERN_ERR PFX "registration of /dev/crypto failed\n");
1023 return rc;
1024 }
1025
1026 return 0;
1027 }
1028
1029 static void __exit
1030 cryptodev_deregister(void)
1031 {
1032 misc_deregister(&cryptodev);
1033 }
1034
1035 /* ====== Module init/exit ====== */
1036 static int __init init_cryptodev(void)
1037 {
1038 int rc;
1039
1040 rc = cryptodev_register();
1041 if (unlikely(rc))
1042 return rc;
1043
1044 printk(KERN_INFO PFX "driver %s loaded.\n", VERSION);
1045
1046 return 0;
1047 }
1048
1049 static void __exit exit_cryptodev(void)
1050 {
1051 cryptodev_deregister();
1052 printk(KERN_INFO PFX "driver unloaded.\n");
1053 }
1054
1055 module_init(init_cryptodev);
1056 module_exit(exit_cryptodev);
1057
Cryptodev for linux