ioctl.c

   1 /*
   2  * Driver for /dev/crypto device (aka CryptoDev)
   3  *
   4  * Copyright (c) 2004 Michal Ludvig <mludvig@logix.net.nz>, SuSE Labs
   5  * Copyright (c) 2009,2010 Nikos Mavrogiannopoulos <nmav@gnutls.org>
   6  *
   7  * This file is part of linux cryptodev.
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License
  11  * as published by the Free Software Foundation; either version 2
  12  * of the License, or (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc.,
  22  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  23  */
  24
  25 /*
  26  * Device /dev/crypto provides an interface for
  27  * accessing kernel CryptoAPI algorithms (ciphers,
  28  * hashes) from userspace programs.
  29  *
  30  * /dev/crypto interface was originally introduced in
  31  * OpenBSD and this module attempts to keep the API.
  32  *
  33  */
  34
  35 #include <crypto/hash.h>
  36 #include <linux/crypto.h>
  37 #include <linux/mm.h>
  38 #include <linux/highmem.h>
  39 #include <linux/ioctl.h>
  40 #include <linux/random.h>
  41 #include <linux/syscalls.h>
  42 #include <linux/pagemap.h>
  43 #include <linux/poll.h>
  44 #include <linux/uaccess.h>
  45 #include <crypto/cryptodev.h>
  46 #include <linux/scatterlist.h>
  47 #include "cryptodev_int.h"
  48 #include "version.h"
  49
  50 MODULE_AUTHOR("Nikos Mavrogiannopoulos <nmav@gnutls.org>");
  51 MODULE_DESCRIPTION("CryptoDev driver");
  52 MODULE_LICENSE("GPL");
  53
  54 /* ====== Compile-time config ====== */
  55
  56 /* Default (pre-allocated) and maximum size of the job queue.
  57  * These are free, pending and done items all together. */
  58 #define DEF_COP_RINGSIZE 16
  59 #define MAX_COP_RINGSIZE 64
  60
  61 /* ====== Module parameters ====== */
  62
  63 int cryptodev_verbosity;
  64 module_param(cryptodev_verbosity, int, 0644);
  65 MODULE_PARM_DESC(cryptodev_verbosity, "0: normal, 1: verbose, 2: debug");
  66
  67 /* ====== CryptoAPI ====== */
  68 struct todo_list_item {
  69         struct list_head __hook;
  70         struct kernel_crypt_op kcop;
  71         int result;
  72 };
  73
  74 struct locked_list {
  75         struct list_head list;
  76         struct mutex lock;
  77 };
  78
  79 struct crypt_priv {
  80         struct fcrypt fcrypt;
  81         struct locked_list free, todo, done;
  82         int itemcount;
  83         struct work_struct cryptask;
  84         wait_queue_head_t user_waiter;
  85 };
  86
  87 #define FILL_SG(sg, ptr, len)                                   \
  88         do {                                                    \
  89                 (sg)->page = virt_to_page(ptr);                 \
  90                 (sg)->offset = offset_in_page(ptr);             \
  91                 (sg)->length = len;                             \
  92                 (sg)->dma_address = 0;                          \
  93         } while (0)
  94
  95 /* cryptodev's own workqueue, keeps crypto tasks from disturbing the force */
  96 static struct workqueue_struct *cryptodev_wq;
  97
  98 /* Prepare session for future use. */
  99 static int
 100 crypto_create_session(struct fcrypt *fcr, struct session_op *sop)
 101 {
 102         struct csession *ses_new = NULL, *ses_ptr;
 103         int ret = 0;
 104         const char *alg_name = NULL;
 105         const char *hash_name = NULL;
 106         int hmac_mode = 1, stream = 0;
 107
 108         /* Does the request make sense? */
 109         if (unlikely(!sop->cipher && !sop->mac)) {
 110                 dprintk(1, KERN_DEBUG, "Both 'cipher' and 'mac' unset.\n");
 111                 return -EINVAL;
 112         }
 113
 114         switch (sop->cipher) {
 115         case 0:
 116                 break;
 117         case CRYPTO_DES_CBC:
 118                 alg_name = "cbc(des)";
 119                 break;
 120         case CRYPTO_3DES_CBC:
 121                 alg_name = "cbc(des3_ede)";
 122                 break;
 123         case CRYPTO_BLF_CBC:
 124                 alg_name = "cbc(blowfish)";
 125                 break;
 126         case CRYPTO_AES_CBC:
 127                 alg_name = "cbc(aes)";
 128                 break;
 129         case CRYPTO_AES_ECB:
 130                 alg_name = "ecb(aes)";
 131                 break;
 132         case CRYPTO_CAMELLIA_CBC:
 133                 alg_name = "cbc(camelia)";
 134                 break;
 135         case CRYPTO_AES_CTR:
 136                 alg_name = "ctr(aes)";
 137                 stream = 1;
 138                 break;
 139         case CRYPTO_NULL:
 140                 alg_name = "ecb(cipher_null)";
 141                 stream = 1;
 142                 break;
 143         default:
 144                 dprintk(1, KERN_DEBUG, "%s: bad cipher: %d\n", __func__,
 145                         sop->cipher);
 146                 return -EINVAL;
 147         }
 148
 149         switch (sop->mac) {
 150         case 0:
 151                 break;
 152         case CRYPTO_MD5_HMAC:
 153                 hash_name = "hmac(md5)";
 154                 break;
 155         case CRYPTO_RIPEMD160_HMAC:
 156                 hash_name = "hmac(rmd160)";
 157                 break;
 158         case CRYPTO_SHA1_HMAC:
 159                 hash_name = "hmac(sha1)";
 160                 break;
 161         case CRYPTO_SHA2_256_HMAC:
 162                 hash_name = "hmac(sha256)";
 163                 break;
 164         case CRYPTO_SHA2_384_HMAC:
 165                 hash_name = "hmac(sha384)";
 166                 break;
 167         case CRYPTO_SHA2_512_HMAC:
 168                 hash_name = "hmac(sha512)";
 169                 break;
 170
 171         /* non-hmac cases */
 172         case CRYPTO_MD5:
 173                 hash_name = "md5";
 174                 hmac_mode = 0;
 175                 break;
 176         case CRYPTO_RIPEMD160:
 177                 hash_name = "rmd160";
 178                 hmac_mode = 0;
 179                 break;
 180         case CRYPTO_SHA1:
 181                 hash_name = "sha1";
 182                 hmac_mode = 0;
 183                 break;
 184         case CRYPTO_SHA2_256:
 185                 hash_name = "sha256";
 186                 hmac_mode = 0;
 187                 break;
 188         case CRYPTO_SHA2_384:
 189                 hash_name = "sha384";
 190                 hmac_mode = 0;
 191                 break;
 192         case CRYPTO_SHA2_512:
 193                 hash_name = "sha512";
 194                 hmac_mode = 0;
 195                 break;
 196
 197         default:
 198                 dprintk(1, KERN_DEBUG, "%s: bad mac: %d\n", __func__,
 199                         sop->mac);
 200                 return -EINVAL;
 201         }
 202
 203         /* Create a session and put it to the list. */
 204         ses_new = kzalloc(sizeof(*ses_new), GFP_KERNEL);
 205         if (!ses_new)
 206                 return -ENOMEM;
 207
 208         /* Set-up crypto transform. */
 209         if (alg_name) {
 210                 uint8_t keyp[CRYPTO_CIPHER_MAX_KEY_LEN];
 211
 212                 if (unlikely(sop->keylen > CRYPTO_CIPHER_MAX_KEY_LEN)) {
 213                         dprintk(1, KERN_DEBUG,
 214                                 "Setting key failed for %s-%zu.\n",
 215                                 alg_name, (size_t)sop->keylen*8);
 216                         ret = -EINVAL;
 217                         goto error_cipher;
 218                 }
 219
 220                 if (unlikely(copy_from_user(keyp, sop->key, sop->keylen))) {
 221                         ret = -EFAULT;
 222                         goto error_cipher;
 223                 }
 224
 225                 ret = cryptodev_cipher_init(&ses_new->cdata, alg_name, keyp,
 226                                                 sop->keylen, stream);
 227                 if (ret < 0) {
 228                         dprintk(1, KERN_DEBUG,
 229                                 "%s: Failed to load cipher for %s\n",
 230                                 __func__, alg_name);
 231                         ret = -EINVAL;
 232                         goto error_cipher;
 233                 }
 234         }
 235
 236         if (hash_name) {
 237                 uint8_t keyp[CRYPTO_HMAC_MAX_KEY_LEN];
 238
 239                 if (unlikely(sop->mackeylen > CRYPTO_HMAC_MAX_KEY_LEN)) {
 240                         dprintk(1, KERN_DEBUG,
 241                                 "Setting key failed for %s-%zu.\n",
 242                                 alg_name, (size_t)sop->mackeylen*8);
 243                         ret = -EINVAL;
 244                         goto error_hash;
 245                 }
 246
 247                 if (sop->mackey && unlikely(copy_from_user(keyp, sop->mackey,
 248                                             sop->mackeylen))) {
 249                         ret = -EFAULT;
 250                         goto error_hash;
 251                 }
 252
 253                 ret = cryptodev_hash_init(&ses_new->hdata, hash_name, hmac_mode,
 254                                                         keyp, sop->mackeylen);
 255                 if (ret != 0) {
 256                         dprintk(1, KERN_DEBUG,
 257                         "%s: Failed to load hash for %s\n",
 258                         __func__, hash_name);
 259                         ret = -EINVAL;
 260                         goto error_hash;
 261                 }
 262         }
 263
 264         ses_new->alignmask = max(ses_new->cdata.alignmask,
 265                                                   ses_new->hdata.alignmask);
 266         dprintk(2, KERN_DEBUG, "%s: got alignmask %d\n", __func__, ses_new->alignmask);
 267
 268         ses_new->array_size = DEFAULT_PREALLOC_PAGES;
 269         dprintk(2, KERN_DEBUG, "%s: preallocating for %d user pages\n",
 270                         __func__, ses_new->array_size);
 271         ses_new->pages = kzalloc(ses_new->array_size *
 272                         sizeof(struct page *), GFP_KERNEL);
 273         ses_new->sg = kzalloc(ses_new->array_size *
 274                         sizeof(struct scatterlist), GFP_KERNEL);
 275         if (ses_new->sg == NULL || ses_new->pages == NULL) {
 276                 dprintk(0, KERN_DEBUG, "Memory error\n");
 277                 ret = -ENOMEM;
 278                 goto error_hash;
 279         }
 280
 281         /* put the new session to the list */
 282         get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
 283         mutex_init(&ses_new->sem);
 284
 285         mutex_lock(&fcr->sem);
 286 restart:
 287         list_for_each_entry(ses_ptr, &fcr->list, entry) {
 288                 /* Check for duplicate SID */
 289                 if (unlikely(ses_new->sid == ses_ptr->sid)) {
 290                         get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
 291                         /* Unless we have a broken RNG this
 292                            shouldn't loop forever... ;-) */
 293                         goto restart;
 294                 }
 295         }
 296
 297         list_add(&ses_new->entry, &fcr->list);
 298         mutex_unlock(&fcr->sem);
 299
 300         /* Fill in some values for the user. */
 301         sop->ses = ses_new->sid;
 302
 303         return 0;
 304
 305 error_hash:
 306         cryptodev_cipher_deinit(&ses_new->cdata);
 307         kfree(ses_new->sg);
 308         kfree(ses_new->pages);
 309 error_cipher:
 310         kfree(ses_new);
 311
 312         return ret;
 313
 314 }
 315
 316 /* Everything that needs to be done when remowing a session. */
 317 static inline void
 318 crypto_destroy_session(struct csession *ses_ptr)
 319 {
 320         if (!mutex_trylock(&ses_ptr->sem)) {
 321                 dprintk(2, KERN_DEBUG, "Waiting for semaphore of sid=0x%08X\n",
 322                         ses_ptr->sid);
 323                 mutex_lock(&ses_ptr->sem);
 324         }
 325         dprintk(2, KERN_DEBUG, "Removed session 0x%08X\n", ses_ptr->sid);
 326         cryptodev_cipher_deinit(&ses_ptr->cdata);
 327         cryptodev_hash_deinit(&ses_ptr->hdata);
 328         dprintk(2, KERN_DEBUG, "%s: freeing space for %d user pages\n",
 329                         __func__, ses_ptr->array_size);
 330         kfree(ses_ptr->pages);
 331         kfree(ses_ptr->sg);
 332         mutex_unlock(&ses_ptr->sem);
 333         kfree(ses_ptr);
 334 }
 335
 336 /* Look up a session by ID and remove. */
 337 static int
 338 crypto_finish_session(struct fcrypt *fcr, uint32_t sid)
 339 {
 340         struct csession *tmp, *ses_ptr;
 341         struct list_head *head;
 342         int ret = 0;
 343
 344         mutex_lock(&fcr->sem);
 345         head = &fcr->list;
 346         list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
 347                 if (ses_ptr->sid == sid) {
 348                         list_del(&ses_ptr->entry);
 349                         crypto_destroy_session(ses_ptr);
 350                         break;
 351                 }
 352         }
 353
 354         if (unlikely(!ses_ptr)) {
 355                 dprintk(1, KERN_ERR, "Session with sid=0x%08X not found!\n",
 356                         sid);
 357                 ret = -ENOENT;
 358         }
 359         mutex_unlock(&fcr->sem);
 360
 361         return ret;
 362 }
 363
 364 /* Remove all sessions when closing the file */
 365 static int
 366 crypto_finish_all_sessions(struct fcrypt *fcr)
 367 {
 368         struct csession *tmp, *ses_ptr;
 369         struct list_head *head;
 370
 371         mutex_lock(&fcr->sem);
 372
 373         head = &fcr->list;
 374         list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
 375                 list_del(&ses_ptr->entry);
 376                 crypto_destroy_session(ses_ptr);
 377         }
 378         mutex_unlock(&fcr->sem);
 379
 380         return 0;
 381 }
 382
 383 /* Look up session by session ID. The returned session is locked. */
 384 struct csession *
 385 crypto_get_session_by_sid(struct fcrypt *fcr, uint32_t sid)
 386 {
 387         struct csession *ses_ptr, *retval = 0;
 388
 389         mutex_lock(&fcr->sem);
 390         list_for_each_entry(ses_ptr, &fcr->list, entry) {
 391                 if (ses_ptr->sid == sid) {
 392                         mutex_lock(&ses_ptr->sem);
 393                         retval = ses_ptr;
 394                         break;
 395                 }
 396         }
 397         mutex_unlock(&fcr->sem);
 398
 399         return retval;
 400 }
 401
 402 void release_user_pages(struct page **pg, int pagecount)
 403 {
 404         while (pagecount--) {
 405                 if (!PageReserved(pg[pagecount]))
 406                         SetPageDirty(pg[pagecount]);
 407                 page_cache_release(pg[pagecount]);
 408         }
 409 }
 410
 411 /* offset of buf in it's first page */
 412 #define PAGEOFFSET(buf) ((unsigned long)buf & ~PAGE_MASK)
 413
 414 /* fetch the pages addr resides in into pg and initialise sg with them */
 415 int __get_userbuf(uint8_t __user *addr, uint32_t len, int write,
 416                 int pgcount, struct page **pg, struct scatterlist *sg,
 417                 struct task_struct *task, struct mm_struct *mm)
 418 {
 419         int ret, pglen, i = 0;
 420         struct scatterlist *sgp;
 421
 422         down_write(&mm->mmap_sem);
 423         ret = get_user_pages(task, mm,
 424                         (unsigned long)addr, pgcount, write, 0, pg, NULL);
 425         up_write(&mm->mmap_sem);
 426         if (ret != pgcount)
 427                 return -EINVAL;
 428
 429         sg_init_table(sg, pgcount);
 430
 431         pglen = min((ptrdiff_t)(PAGE_SIZE - PAGEOFFSET(addr)), (ptrdiff_t)len);
 432         sg_set_page(sg, pg[i++], pglen, PAGEOFFSET(addr));
 433
 434         len -= pglen;
 435         for (sgp = sg_next(sg); len; sgp = sg_next(sgp)) {
 436                 pglen = min((uint32_t)PAGE_SIZE, len);
 437                 sg_set_page(sgp, pg[i++], pglen, 0);
 438                 len -= pglen;
 439         }
 440         sg_mark_end(sg_last(sg, pgcount));
 441         return 0;
 442 }
 443
 444 int adjust_sg_array(struct csession * ses, int pagecount)
 445 {
 446 struct scatterlist *sg;
 447 struct page **pages;
 448 int array_size;
 449
 450         for (array_size = ses->array_size; array_size < pagecount;
 451              array_size *= 2)
 452                 ;
 453
 454         dprintk(2, KERN_DEBUG, "%s: reallocating to %d elements\n",
 455                         __func__, array_size);
 456         pages = krealloc(ses->pages, array_size * sizeof(struct page *),
 457                          GFP_KERNEL);
 458         if (unlikely(!pages))
 459                 return -ENOMEM;
 460         ses->pages = pages;
 461         sg = krealloc(ses->sg, array_size * sizeof(struct scatterlist),
 462                       GFP_KERNEL);
 463         if (unlikely(!sg))
 464                 return -ENOMEM;
 465         ses->sg = sg;
 466         ses->array_size = array_size;
 467
 468         return 0;
 469 }
 470
 471
 472 static void cryptask_routine(struct work_struct *work)
 473 {
 474         struct crypt_priv *pcr = container_of(work, struct crypt_priv, cryptask);
 475         struct todo_list_item *item;
 476         LIST_HEAD(tmp);
 477
 478         /* fetch all pending jobs into the temporary list */
 479         mutex_lock(&pcr->todo.lock);
 480         list_cut_position(&tmp, &pcr->todo.list, pcr->todo.list.prev);
 481         mutex_unlock(&pcr->todo.lock);
 482
 483         /* handle each job locklessly */
 484         list_for_each_entry(item, &tmp, __hook) {
 485                 item->result = crypto_run(&pcr->fcrypt, &item->kcop);
 486                 if (unlikely(item->result))
 487                         dprintk(0, KERN_ERR, "%s: crypto_run() failed: %d\n",
 488                                         __func__, item->result);
 489         }
 490
 491         /* push all handled jobs to the done list at once */
 492         mutex_lock(&pcr->done.lock);
 493         list_splice_tail(&tmp, &pcr->done.list);
 494         mutex_unlock(&pcr->done.lock);
 495
 496         /* wake for POLLIN */
 497         wake_up_interruptible(&pcr->user_waiter);
 498 }
 499
 500 /* ====== /dev/crypto ====== */
 501
 502 static int
 503 cryptodev_open(struct inode *inode, struct file *filp)
 504 {
 505         struct todo_list_item *tmp;
 506         struct crypt_priv *pcr;
 507         int i;
 508
 509         pcr = kmalloc(sizeof(*pcr), GFP_KERNEL);
 510         if (!pcr)
 511                 return -ENOMEM;
 512
 513         memset(pcr, 0, sizeof(*pcr));
 514         mutex_init(&pcr->fcrypt.sem);
 515         INIT_LIST_HEAD(&pcr->fcrypt.list);
 516
 517         INIT_LIST_HEAD(&pcr->free.list);
 518         INIT_LIST_HEAD(&pcr->todo.list);
 519         INIT_LIST_HEAD(&pcr->done.list);
 520         INIT_WORK(&pcr->cryptask, cryptask_routine);
 521         mutex_init(&pcr->free.lock);
 522         mutex_init(&pcr->todo.lock);
 523         mutex_init(&pcr->done.lock);
 524         init_waitqueue_head(&pcr->user_waiter);
 525
 526         for (i = 0; i < DEF_COP_RINGSIZE; i++) {
 527                 tmp = kzalloc(sizeof(struct todo_list_item), GFP_KERNEL);
 528                 pcr->itemcount++;
 529                 dprintk(2, KERN_DEBUG, "%s: allocated new item at %lx\n",
 530                                 __func__, (unsigned long)tmp);
 531                 list_add(&tmp->__hook, &pcr->free.list);
 532         }
 533
 534         filp->private_data = pcr;
 535         dprintk(2, KERN_DEBUG,
 536                 "Cryptodev handle initialised, %d elements in queue\n",
 537                 DEF_COP_RINGSIZE);
 538         return 0;
 539 }
 540
 541 static int
 542 cryptodev_release(struct inode *inode, struct file *filp)
 543 {
 544         struct crypt_priv *pcr = filp->private_data;
 545         struct todo_list_item *item, *item_safe;
 546         int items_freed = 0;
 547
 548         if (!pcr)
 549                 return 0;
 550
 551         cancel_work_sync(&pcr->cryptask);
 552
 553         mutex_destroy(&pcr->todo.lock);
 554         mutex_destroy(&pcr->done.lock);
 555         mutex_destroy(&pcr->free.lock);
 556
 557         list_splice_tail(&pcr->todo.list, &pcr->free.list);
 558         list_splice_tail(&pcr->done.list, &pcr->free.list);
 559
 560         list_for_each_entry_safe(item, item_safe, &pcr->free.list, __hook) {
 561                 dprintk(2, KERN_DEBUG, "%s: freeing item at %lx\n",
 562                                 __func__, (unsigned long)item);
 563                 list_del(&item->__hook);
 564                 kfree(item);
 565                 items_freed++;
 566
 567         }
 568         if (items_freed != pcr->itemcount) {
 569                 dprintk(0, KERN_ERR,
 570                         "%s: freed %d items, but %d should exist!\n",
 571                         __func__, items_freed, pcr->itemcount);
 572         }
 573
 574         crypto_finish_all_sessions(&pcr->fcrypt);
 575         kfree(pcr);
 576         filp->private_data = NULL;
 577
 578         dprintk(2, KERN_DEBUG,
 579                 "Cryptodev handle deinitialised, %d elements freed\n",
 580                 items_freed);
 581         return 0;
 582 }
 583
 584 static int
 585 clonefd(struct file *filp)
 586 {
 587         int ret;
 588         ret = get_unused_fd();
 589         if (ret >= 0) {
 590                         get_file(filp);
 591                         fd_install(ret, filp);
 592         }
 593
 594         return ret;
 595 }
 596
 597 /* enqueue a job for asynchronous completion
 598  *
 599  * returns:
 600  * -EBUSY when there are no free queue slots left
 601  *        (and the number of slots has reached it MAX_COP_RINGSIZE)
 602  * -EFAULT when there was a memory allocation error
 603  * 0 on success */
 604 static int crypto_async_run(struct crypt_priv *pcr, struct kernel_crypt_op *kcop)
 605 {
 606         struct todo_list_item *item = NULL;
 607
 608         mutex_lock(&pcr->free.lock);
 609         if (likely(!list_empty(&pcr->free.list))) {
 610                 item = list_first_entry(&pcr->free.list,
 611                                 struct todo_list_item, __hook);
 612                 list_del(&item->__hook);
 613         } else if (pcr->itemcount < MAX_COP_RINGSIZE) {
 614                 pcr->itemcount++;
 615         } else {
 616                 mutex_unlock(&pcr->free.lock);
 617                 return -EBUSY;
 618         }
 619         mutex_unlock(&pcr->free.lock);
 620
 621         if (unlikely(!item)) {
 622                 item = kzalloc(sizeof(struct todo_list_item), GFP_KERNEL);
 623                 if (unlikely(!item))
 624                         return -EFAULT;
 625                 dprintk(1, KERN_INFO, "%s: increased item count to %d\n",
 626                                 __func__, pcr->itemcount);
 627         }
 628
 629         memcpy(&item->kcop, kcop, sizeof(struct kernel_crypt_op));
 630
 631         mutex_lock(&pcr->todo.lock);
 632         list_add_tail(&item->__hook, &pcr->todo.list);
 633         mutex_unlock(&pcr->todo.lock);
 634
 635         queue_work(cryptodev_wq, &pcr->cryptask);
 636         return 0;
 637 }
 638
 639 /* get the first completed job from the "done" queue
 640  *
 641  * returns:
 642  * -EBUSY if no completed jobs are ready (yet)
 643  * the return value of crypto_run() otherwise */
 644 static int crypto_async_fetch(struct crypt_priv *pcr,
 645                 struct kernel_crypt_op *kcop)
 646 {
 647         struct todo_list_item *item;
 648         int retval;
 649
 650         mutex_lock(&pcr->done.lock);
 651         if (list_empty(&pcr->done.list)) {
 652                 mutex_unlock(&pcr->done.lock);
 653                 return -EBUSY;
 654         }
 655         item = list_first_entry(&pcr->done.list, struct todo_list_item, __hook);
 656         list_del(&item->__hook);
 657         mutex_unlock(&pcr->done.lock);
 658
 659         memcpy(kcop, &item->kcop, sizeof(struct kernel_crypt_op));
 660         retval = item->result;
 661
 662         mutex_lock(&pcr->free.lock);
 663         list_add_tail(&item->__hook, &pcr->free.list);
 664         mutex_unlock(&pcr->free.lock);
 665
 666         /* wake for POLLOUT */
 667         wake_up_interruptible(&pcr->user_waiter);
 668
 669         return retval;
 670 }
 671
 672 /* this function has to be called from process context */
 673 static int fill_kcop_from_cop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
 674 {
 675         struct crypt_op *cop = &kcop->cop;
 676         struct csession *ses_ptr;
 677         int rc;
 678
 679         /* this also enters ses_ptr->sem */
 680         ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
 681         if (unlikely(!ses_ptr)) {
 682                 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
 683                 return -EINVAL;
 684         }
 685         kcop->ivlen = cop->iv ? ses_ptr->cdata.ivsize : 0;
 686         kcop->digestsize = 0; /* will be updated during operation */
 687
 688         crypto_put_session(ses_ptr);
 689
 690         kcop->task = current;
 691         kcop->mm = current->mm;
 692
 693         if (cop->iv) {
 694                 rc = copy_from_user(kcop->iv, cop->iv, kcop->ivlen);
 695                 if (unlikely(rc)) {
 696                         dprintk(1, KERN_ERR,
 697                                 "error copying IV (%d bytes), copy_from_user returned %d for address %lx\n",
 698                                 kcop->ivlen, rc, (unsigned long)cop->iv);
 699                         return -EFAULT;
 700                 }
 701         }
 702
 703         return 0;
 704 }
 705
 706 /* this function has to be called from process context */
 707 static int fill_cop_from_kcop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
 708 {
 709         int ret;
 710
 711         if (kcop->digestsize) {
 712                 ret = copy_to_user(kcop->cop.mac,
 713                                 kcop->hash_output, kcop->digestsize);
 714                 if (unlikely(ret))
 715                         return -EFAULT;
 716         }
 717         if (kcop->ivlen && kcop->cop.flags & COP_FLAG_WRITE_IV) {
 718                 ret = copy_to_user(kcop->cop.iv,
 719                                 kcop->iv, kcop->ivlen);
 720                 if (unlikely(ret))
 721                         return -EFAULT;
 722         }
 723         return 0;
 724 }
 725
 726 static int kcop_from_user(struct kernel_crypt_op *kcop,
 727                         struct fcrypt *fcr, void __user *arg)
 728 {
 729         if (unlikely(copy_from_user(&kcop->cop, arg, sizeof(kcop->cop))))
 730                 return -EFAULT;
 731
 732         return fill_kcop_from_cop(kcop, fcr);
 733 }
 734
 735 static int kcop_to_user(struct kernel_crypt_op *kcop,
 736                         struct fcrypt *fcr, void __user *arg)
 737 {
 738         int ret;
 739
 740         ret = fill_cop_from_kcop(kcop, fcr);
 741         if (unlikely(ret))
 742                 return ret;
 743
 744         if (unlikely(copy_to_user(arg, &kcop->cop, sizeof(kcop->cop))))
 745                 return -EFAULT;
 746         return 0;
 747 }
 748
 749 static inline void tfm_info_to_alg_info(struct alg_info *dst, struct crypto_tfm *tfm)
 750 {
 751         snprintf(dst->cra_name, CRYPTODEV_MAX_ALG_NAME,
 752                         "%s", crypto_tfm_alg_name(tfm));
 753         snprintf(dst->cra_driver_name, CRYPTODEV_MAX_ALG_NAME,
 754                         "%s", crypto_tfm_alg_driver_name(tfm));
 755 }
 756
 757 static int get_session_info(struct fcrypt *fcr, struct session_info_op *siop)
 758 {
 759         struct csession *ses_ptr;
 760
 761         /* this also enters ses_ptr->sem */
 762         ses_ptr = crypto_get_session_by_sid(fcr, siop->ses);
 763         if (unlikely(!ses_ptr)) {
 764                 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", siop->ses);
 765                 return -EINVAL;
 766         }
 767
 768         if (ses_ptr->cdata.init) {
 769                 tfm_info_to_alg_info(&siop->cipher_info,
 770                                 crypto_ablkcipher_tfm(ses_ptr->cdata.async.s));
 771         }
 772         if (ses_ptr->hdata.init) {
 773                 tfm_info_to_alg_info(&siop->hash_info,
 774                                 crypto_ahash_tfm(ses_ptr->hdata.async.s));
 775         }
 776
 777         siop->alignmask = ses_ptr->alignmask;
 778
 779         crypto_put_session(ses_ptr);
 780         return 0;
 781 }
 782
 783 static long
 784 cryptodev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg_)
 785 {
 786         void __user *arg = (void __user *)arg_;
 787         int __user *p = arg;
 788         struct session_op sop;
 789         struct kernel_crypt_op kcop;
 790         struct kernel_crypt_auth_op kcaop;
 791         struct crypt_priv *pcr = filp->private_data;
 792         struct fcrypt *fcr;
 793         struct session_info_op siop;
 794         uint32_t ses;
 795         int ret, fd;
 796
 797         if (unlikely(!pcr))
 798                 BUG();
 799
 800         fcr = &pcr->fcrypt;
 801
 802         switch (cmd) {
 803         case CIOCASYMFEAT:
 804                 return put_user(0, p);
 805         case CRIOGET:
 806                 fd = clonefd(filp);
 807                 ret = put_user(fd, p);
 808                 if (unlikely(ret)) {
 809                         sys_close(fd);
 810                         return ret;
 811                 }
 812                 return ret;
 813         case CIOCGSESSION:
 814                 if (unlikely(copy_from_user(&sop, arg, sizeof(sop))))
 815                         return -EFAULT;
 816
 817                 ret = crypto_create_session(fcr, &sop);
 818                 if (unlikely(ret))
 819                         return ret;
 820                 ret = copy_to_user(arg, &sop, sizeof(sop));
 821                 if (unlikely(ret)) {
 822                         crypto_finish_session(fcr, sop.ses);
 823                         return -EFAULT;
 824                 }
 825                 return ret;
 826         case CIOCFSESSION:
 827                 ret = get_user(ses, (uint32_t __user *)arg);
 828                 if (unlikely(ret))
 829                         return ret;
 830                 ret = crypto_finish_session(fcr, ses);
 831                 return ret;
 832         case CIOCGSESSINFO:
 833                 if (unlikely(copy_from_user(&siop, arg, sizeof(siop))))
 834                         return -EFAULT;
 835
 836                 ret = get_session_info(fcr, &siop);
 837                 if (unlikely(ret))
 838                         return ret;
 839                 return copy_to_user(arg, &siop, sizeof(siop));
 840         case CIOCCRYPT:
 841                 if (unlikely(ret = kcop_from_user(&kcop, fcr, arg))) {
 842                         dprintk(1, KERN_WARNING, "Error copying from user");
 843                         return ret;
 844                 }
 845
 846                 ret = crypto_run(fcr, &kcop);
 847                 if (unlikely(ret)) {
 848                         dprintk(1, KERN_WARNING, "Error in crypto_run");
 849                         return ret;
 850                 }
 851
 852                 return kcop_to_user(&kcop, fcr, arg);
 853         case CIOCAUTHCRYPT:
 854                 if (unlikely(ret = kcaop_from_user(&kcaop, fcr, arg))) {
 855                         dprintk(1, KERN_WARNING, "Error copying from user");
 856                         return ret;
 857                 }
 858
 859                 ret = crypto_auth_run(fcr, &kcaop);
 860                 if (unlikely(ret)) {
 861                         dprintk(1, KERN_WARNING, "Error in crypto_auth_run");
 862                         return ret;
 863                 }
 864                 return kcaop_to_user(&kcaop, fcr, arg);
 865         case CIOCASYNCCRYPT:
 866                 if (unlikely(ret = kcop_from_user(&kcop, fcr, arg)))
 867                         return ret;
 868
 869                 return crypto_async_run(pcr, &kcop);
 870         case CIOCASYNCFETCH:
 871                 ret = crypto_async_fetch(pcr, &kcop);
 872                 if (unlikely(ret))
 873                         return ret;
 874
 875                 return kcop_to_user(&kcop, fcr, arg);
 876         default:
 877                 return -EINVAL;
 878         }
 879 }
 880
 881 /* compatibility code for 32bit userlands */
 882 #ifdef CONFIG_COMPAT
 883
 884 static inline void
 885 compat_to_session_op(struct compat_session_op *compat, struct session_op *sop)
 886 {
 887         sop->cipher = compat->cipher;
 888         sop->mac = compat->mac;
 889         sop->keylen = compat->keylen;
 890
 891         sop->key       = compat_ptr(compat->key);
 892         sop->mackeylen = compat->mackeylen;
 893         sop->mackey    = compat_ptr(compat->mackey);
 894         sop->ses       = compat->ses;
 895 }
 896
 897 static inline void
 898 session_op_to_compat(struct session_op *sop, struct compat_session_op *compat)
 899 {
 900         compat->cipher = sop->cipher;
 901         compat->mac = sop->mac;
 902         compat->keylen = sop->keylen;
 903
 904         compat->key       = ptr_to_compat(sop->key);
 905         compat->mackeylen = sop->mackeylen;
 906         compat->mackey    = ptr_to_compat(sop->mackey);
 907         compat->ses       = sop->ses;
 908 }
 909
 910 static inline void
 911 compat_to_crypt_op(struct compat_crypt_op *compat, struct crypt_op *cop)
 912 {
 913         cop->ses = compat->ses;
 914         cop->op = compat->op;
 915         cop->flags = compat->flags;
 916         cop->len = compat->len;
 917
 918         cop->src = compat_ptr(compat->src);
 919         cop->dst = compat_ptr(compat->dst);
 920         cop->mac = compat_ptr(compat->mac);
 921         cop->iv  = compat_ptr(compat->iv);
 922 }
 923
 924 static inline void
 925 crypt_op_to_compat(struct crypt_op *cop, struct compat_crypt_op *compat)
 926 {
 927         compat->ses = cop->ses;
 928         compat->op = cop->op;
 929         compat->flags = cop->flags;
 930         compat->len = cop->len;
 931
 932         compat->src = ptr_to_compat(cop->src);
 933         compat->dst = ptr_to_compat(cop->dst);
 934         compat->mac = ptr_to_compat(cop->mac);
 935         compat->iv  = ptr_to_compat(cop->iv);
 936 }
 937
 938 static int compat_kcop_from_user(struct kernel_crypt_op *kcop,
 939                                  struct fcrypt *fcr, void __user *arg)
 940 {
 941         struct compat_crypt_op compat_cop;
 942
 943         if (unlikely(copy_from_user(&compat_cop, arg, sizeof(compat_cop))))
 944                 return -EFAULT;
 945         compat_to_crypt_op(&compat_cop, &kcop->cop);
 946
 947         return fill_kcop_from_cop(kcop, fcr);
 948 }
 949
 950 static int compat_kcop_to_user(struct kernel_crypt_op *kcop,
 951                                  struct fcrypt *fcr, void __user *arg)
 952 {
 953         int ret;
 954         struct compat_crypt_op compat_cop;
 955
 956         ret = fill_cop_from_kcop(kcop, fcr);
 957         if (unlikely(ret)) {
 958                 dprintk(1, KERN_WARNING, "Error in fill_cop_from_kcop");
 959                 return ret;
 960         }
 961         crypt_op_to_compat(&kcop->cop, &compat_cop);
 962
 963         if (unlikely(copy_to_user(arg, &compat_cop, sizeof(compat_cop)))) {
 964                 dprintk(1, KERN_WARNING, "Error copying to user");
 965                 return -EFAULT;
 966         }
 967         return 0;
 968 }
 969
 970 static long
 971 cryptodev_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg_)
 972 {
 973         void __user *arg = (void __user *)arg_;
 974         struct crypt_priv *pcr = file->private_data;
 975         struct fcrypt *fcr;
 976         struct session_op sop;
 977         struct compat_session_op compat_sop;
 978         struct kernel_crypt_op kcop;
 979         int ret;
 980
 981         if (unlikely(!pcr))
 982                 BUG();
 983
 984         fcr = &pcr->fcrypt;
 985
 986         switch (cmd) {
 987         case CIOCASYMFEAT:
 988         case CRIOGET:
 989         case CIOCFSESSION:
 990         case CIOCGSESSINFO:
 991                 return cryptodev_ioctl(file, cmd, arg_);
 992
 993         case COMPAT_CIOCGSESSION:
 994                 if (unlikely(copy_from_user(&compat_sop, arg,
 995                                             sizeof(compat_sop))))
 996                         return -EFAULT;
 997                 compat_to_session_op(&compat_sop, &sop);
 998
 999                 ret = crypto_create_session(fcr, &sop);
1000                 if (unlikely(ret))
1001                         return ret;
1002
1003                 session_op_to_compat(&sop, &compat_sop);
1004                 ret = copy_to_user(arg, &compat_sop, sizeof(compat_sop));
1005                 if (unlikely(ret)) {
1006                         crypto_finish_session(fcr, sop.ses);
1007                         return -EFAULT;
1008                 }
1009                 return ret;
1010
1011         case COMPAT_CIOCCRYPT:
1012                 ret = compat_kcop_from_user(&kcop, fcr, arg);
1013                 if (unlikely(ret))
1014                         return ret;
1015
1016                 ret = crypto_run(fcr, &kcop);
1017                 if (unlikely(ret))
1018                         return ret;
1019
1020                 return compat_kcop_to_user(&kcop, fcr, arg);
1021         case COMPAT_CIOCASYNCCRYPT:
1022                 if (unlikely(ret = compat_kcop_from_user(&kcop, fcr, arg)))
1023                         return ret;
1024
1025                 return crypto_async_run(pcr, &kcop);
1026         case COMPAT_CIOCASYNCFETCH:
1027                 ret = crypto_async_fetch(pcr, &kcop);
1028                 if (unlikely(ret))
1029                         return ret;
1030
1031                 return compat_kcop_to_user(&kcop, fcr, arg);
1032
1033         default:
1034                 return -EINVAL;
1035         }
1036 }
1037
1038 #endif /* CONFIG_COMPAT */
1039
1040 static unsigned int cryptodev_poll(struct file *file, poll_table *wait)
1041 {
1042         struct crypt_priv *pcr = file->private_data;
1043         int ret = 0;
1044
1045         poll_wait(file, &pcr->user_waiter, wait);
1046
1047         if (!list_empty_careful(&pcr->done.list))
1048                 ret |= POLLIN | POLLRDNORM;
1049         if (!list_empty_careful(&pcr->free.list) || pcr->itemcount < MAX_COP_RINGSIZE)
1050                 ret |= POLLOUT | POLLWRNORM;
1051
1052         return ret;
1053 }
1054
1055 static const struct file_operations cryptodev_fops = {
1056         .owner = THIS_MODULE,
1057         .open = cryptodev_open,
1058         .release = cryptodev_release,
1059         .unlocked_ioctl = cryptodev_ioctl,
1060 #ifdef CONFIG_COMPAT
1061         .compat_ioctl = cryptodev_compat_ioctl,
1062 #endif /* CONFIG_COMPAT */
1063         .poll = cryptodev_poll,
1064 };
1065
1066 static struct miscdevice cryptodev = {
1067         .minor = MISC_DYNAMIC_MINOR,
1068         .name = "crypto",
1069         .fops = &cryptodev_fops,
1070 };
1071
1072 static int __init
1073 cryptodev_register(void)
1074 {
1075         int rc;
1076
1077         rc = misc_register(&cryptodev);
1078         if (unlikely(rc)) {
1079                 printk(KERN_ERR PFX "registration of /dev/crypto failed\n");
1080                 return rc;
1081         }
1082
1083         return 0;
1084 }
1085
1086 static void __exit
1087 cryptodev_deregister(void)
1088 {
1089         misc_deregister(&cryptodev);
1090 }
1091
1092 /* ====== Module init/exit ====== */
1093 static int __init init_cryptodev(void)
1094 {
1095         int rc;
1096
1097         cryptodev_wq = create_workqueue("cryptodev_queue");
1098         if (unlikely(!cryptodev_wq)) {
1099                 printk(KERN_ERR PFX "failed to allocate the cryptodev workqueue\n");
1100                 return -EFAULT;
1101         }
1102
1103         rc = cryptodev_register();
1104         if (unlikely(rc)) {
1105                 destroy_workqueue(cryptodev_wq);
1106                 return rc;
1107         }
1108
1109         printk(KERN_INFO PFX "driver %s loaded.\n", VERSION);
1110
1111         return 0;
1112 }
1113
1114 static void __exit exit_cryptodev(void)
1115 {
1116         flush_workqueue(cryptodev_wq);
1117         destroy_workqueue(cryptodev_wq);
1118
1119         cryptodev_deregister();
1120         printk(KERN_INFO PFX "driver unloaded.\n");
1121 }
1122
1123 module_init(init_cryptodev);
1124 module_exit(exit_cryptodev);
1125