ioctl.c

   1 /*
   2  * Driver for /dev/crypto device (aka CryptoDev)
   3  *
   4  * Copyright (c) 2004 Michal Ludvig <mludvig@logix.net.nz>, SuSE Labs
   5  * Copyright (c) 2009,2010 Nikos Mavrogiannopoulos <nmav@gnutls.org>
   6  *
   7  * This file is part of linux cryptodev.
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License
  11  * as published by the Free Software Foundation; either version 2
  12  * of the License, or (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc.,
  22  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  23  */
  24
  25 /*
  26  * Device /dev/crypto provides an interface for
  27  * accessing kernel CryptoAPI algorithms (ciphers,
  28  * hashes) from userspace programs.
  29  *
  30  * /dev/crypto interface was originally introduced in
  31  * OpenBSD and this module attempts to keep the API.
  32  *
  33  */
  34
  35 #include <crypto/hash.h>
  36 #include <linux/crypto.h>
  37 #include <linux/mm.h>
  38 #include <linux/highmem.h>
  39 #include <linux/ioctl.h>
  40 #include <linux/random.h>
  41 #include <linux/syscalls.h>
  42 #include <linux/pagemap.h>
  43 #include <linux/poll.h>
  44 #include <linux/uaccess.h>
  45 #include <crypto/cryptodev.h>
  46 #include <linux/scatterlist.h>
  47 #include "cryptodev_int.h"
  48 #include "zc.h"
  49 #include "version.h"
  50
  51 MODULE_AUTHOR("Nikos Mavrogiannopoulos <nmav@gnutls.org>");
  52 MODULE_DESCRIPTION("CryptoDev driver");
  53 MODULE_LICENSE("GPL");
  54
  55 /* ====== Compile-time config ====== */
  56
  57 /* Default (pre-allocated) and maximum size of the job queue.
  58  * These are free, pending and done items all together. */
  59 #define DEF_COP_RINGSIZE 16
  60 #define MAX_COP_RINGSIZE 64
  61
  62 /* ====== Module parameters ====== */
  63
  64 int cryptodev_verbosity;
  65 module_param(cryptodev_verbosity, int, 0644);
  66 MODULE_PARM_DESC(cryptodev_verbosity, "0: normal, 1: verbose, 2: debug");
  67
  68 /* ====== CryptoAPI ====== */
  69 struct todo_list_item {
  70         struct list_head __hook;
  71         struct kernel_crypt_op kcop;
  72         int result;
  73 };
  74
  75 struct locked_list {
  76         struct list_head list;
  77         struct mutex lock;
  78 };
  79
  80 struct crypt_priv {
  81         struct fcrypt fcrypt;
  82         struct locked_list free, todo, done;
  83         int itemcount;
  84         struct work_struct cryptask;
  85         wait_queue_head_t user_waiter;
  86 };
  87
  88 #define FILL_SG(sg, ptr, len)                                   \
  89         do {                                                    \
  90                 (sg)->page = virt_to_page(ptr);                 \
  91                 (sg)->offset = offset_in_page(ptr);             \
  92                 (sg)->length = len;                             \
  93                 (sg)->dma_address = 0;                          \
  94         } while (0)
  95
  96 /* cryptodev's own workqueue, keeps crypto tasks from disturbing the force */
  97 static struct workqueue_struct *cryptodev_wq;
  98
  99 /* Prepare session for future use. */
 100 static int
 101 crypto_create_session(struct fcrypt *fcr, struct session_op *sop)
 102 {
 103         struct csession *ses_new = NULL, *ses_ptr;
 104         int ret = 0;
 105         const char *alg_name = NULL;
 106         const char *hash_name = NULL;
 107         int hmac_mode = 1, stream = 0, aead = 0;
 108
 109         /* Does the request make sense? */
 110         if (unlikely(!sop->cipher && !sop->mac)) {
 111                 dprintk(1, KERN_DEBUG, "Both 'cipher' and 'mac' unset.\n");
 112                 return -EINVAL;
 113         }
 114
 115         switch (sop->cipher) {
 116         case 0:
 117                 break;
 118         case CRYPTO_DES_CBC:
 119                 alg_name = "cbc(des)";
 120                 break;
 121         case CRYPTO_3DES_CBC:
 122                 alg_name = "cbc(des3_ede)";
 123                 break;
 124         case CRYPTO_BLF_CBC:
 125                 alg_name = "cbc(blowfish)";
 126                 break;
 127         case CRYPTO_AES_CBC:
 128                 alg_name = "cbc(aes)";
 129                 break;
 130         case CRYPTO_AES_ECB:
 131                 alg_name = "ecb(aes)";
 132                 break;
 133         case CRYPTO_CAMELLIA_CBC:
 134                 alg_name = "cbc(camelia)";
 135                 break;
 136         case CRYPTO_AES_CTR:
 137                 alg_name = "ctr(aes)";
 138                 stream = 1;
 139                 break;
 140         case CRYPTO_AES_GCM:
 141                 alg_name = "gcm(aes)";
 142                 stream = 1;
 143                 aead = 1;
 144                 break;
 145         case CRYPTO_NULL:
 146                 alg_name = "ecb(cipher_null)";
 147                 stream = 1;
 148                 break;
 149         default:
 150                 dprintk(1, KERN_DEBUG, "%s: bad cipher: %d\n", __func__,
 151                         sop->cipher);
 152                 return -EINVAL;
 153         }
 154
 155         switch (sop->mac) {
 156         case 0:
 157                 break;
 158         case CRYPTO_MD5_HMAC:
 159                 hash_name = "hmac(md5)";
 160                 break;
 161         case CRYPTO_RIPEMD160_HMAC:
 162                 hash_name = "hmac(rmd160)";
 163                 break;
 164         case CRYPTO_SHA1_HMAC:
 165                 hash_name = "hmac(sha1)";
 166                 break;
 167         case CRYPTO_SHA2_256_HMAC:
 168                 hash_name = "hmac(sha256)";
 169                 break;
 170         case CRYPTO_SHA2_384_HMAC:
 171                 hash_name = "hmac(sha384)";
 172                 break;
 173         case CRYPTO_SHA2_512_HMAC:
 174                 hash_name = "hmac(sha512)";
 175                 break;
 176
 177         /* non-hmac cases */
 178         case CRYPTO_MD5:
 179                 hash_name = "md5";
 180                 hmac_mode = 0;
 181                 break;
 182         case CRYPTO_RIPEMD160:
 183                 hash_name = "rmd160";
 184                 hmac_mode = 0;
 185                 break;
 186         case CRYPTO_SHA1:
 187                 hash_name = "sha1";
 188                 hmac_mode = 0;
 189                 break;
 190         case CRYPTO_SHA2_256:
 191                 hash_name = "sha256";
 192                 hmac_mode = 0;
 193                 break;
 194         case CRYPTO_SHA2_384:
 195                 hash_name = "sha384";
 196                 hmac_mode = 0;
 197                 break;
 198         case CRYPTO_SHA2_512:
 199                 hash_name = "sha512";
 200                 hmac_mode = 0;
 201                 break;
 202         default:
 203                 dprintk(1, KERN_DEBUG, "%s: bad mac: %d\n", __func__,
 204                         sop->mac);
 205                 return -EINVAL;
 206         }
 207
 208         /* Create a session and put it to the list. */
 209         ses_new = kzalloc(sizeof(*ses_new), GFP_KERNEL);
 210         if (!ses_new)
 211                 return -ENOMEM;
 212
 213         /* Set-up crypto transform. */
 214         if (alg_name) {
 215                 uint8_t keyp[CRYPTO_CIPHER_MAX_KEY_LEN];
 216
 217                 if (unlikely(sop->keylen > CRYPTO_CIPHER_MAX_KEY_LEN)) {
 218                         dprintk(1, KERN_DEBUG,
 219                                 "Setting key failed for %s-%zu.\n",
 220                                 alg_name, (size_t)sop->keylen*8);
 221                         ret = -EINVAL;
 222                         goto error_cipher;
 223                 }
 224
 225                 if (unlikely(copy_from_user(keyp, sop->key, sop->keylen))) {
 226                         ret = -EFAULT;
 227                         goto error_cipher;
 228                 }
 229
 230                 ret = cryptodev_cipher_init(&ses_new->cdata, alg_name, keyp,
 231                                                 sop->keylen, stream, aead);
 232                 if (ret < 0) {
 233                         dprintk(1, KERN_DEBUG,
 234                                 "%s: Failed to load cipher for %s\n",
 235                                 __func__, alg_name);
 236                         ret = -EINVAL;
 237                         goto error_cipher;
 238                 }
 239         }
 240
 241         if (hash_name && aead == 0) {
 242                 uint8_t keyp[CRYPTO_HMAC_MAX_KEY_LEN];
 243
 244                 if (unlikely(sop->mackeylen > CRYPTO_HMAC_MAX_KEY_LEN)) {
 245                         dprintk(1, KERN_DEBUG,
 246                                 "Setting key failed for %s-%zu.\n",
 247                                 alg_name, (size_t)sop->mackeylen*8);
 248                         ret = -EINVAL;
 249                         goto error_hash;
 250                 }
 251
 252                 if (sop->mackey && unlikely(copy_from_user(keyp, sop->mackey,
 253                                             sop->mackeylen))) {
 254                         ret = -EFAULT;
 255                         goto error_hash;
 256                 }
 257
 258                 ret = cryptodev_hash_init(&ses_new->hdata, hash_name, hmac_mode,
 259                                                         keyp, sop->mackeylen);
 260                 if (ret != 0) {
 261                         dprintk(1, KERN_DEBUG,
 262                         "%s: Failed to load hash for %s\n",
 263                         __func__, hash_name);
 264                         ret = -EINVAL;
 265                         goto error_hash;
 266                 }
 267         }
 268
 269         ses_new->alignmask = max(ses_new->cdata.alignmask,
 270                                                   ses_new->hdata.alignmask);
 271         dprintk(2, KERN_DEBUG, "%s: got alignmask %d\n", __func__, ses_new->alignmask);
 272
 273         ses_new->array_size = DEFAULT_PREALLOC_PAGES;
 274         dprintk(2, KERN_DEBUG, "%s: preallocating for %d user pages\n",
 275                         __func__, ses_new->array_size);
 276         ses_new->pages = kzalloc(ses_new->array_size *
 277                         sizeof(struct page *), GFP_KERNEL);
 278         ses_new->sg = kzalloc(ses_new->array_size *
 279                         sizeof(struct scatterlist), GFP_KERNEL);
 280         if (ses_new->sg == NULL || ses_new->pages == NULL) {
 281                 dprintk(0, KERN_DEBUG, "Memory error\n");
 282                 ret = -ENOMEM;
 283                 goto error_hash;
 284         }
 285
 286         /* put the new session to the list */
 287         get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
 288         mutex_init(&ses_new->sem);
 289
 290         mutex_lock(&fcr->sem);
 291 restart:
 292         list_for_each_entry(ses_ptr, &fcr->list, entry) {
 293                 /* Check for duplicate SID */
 294                 if (unlikely(ses_new->sid == ses_ptr->sid)) {
 295                         get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
 296                         /* Unless we have a broken RNG this
 297                            shouldn't loop forever... ;-) */
 298                         goto restart;
 299                 }
 300         }
 301
 302         list_add(&ses_new->entry, &fcr->list);
 303         mutex_unlock(&fcr->sem);
 304
 305         /* Fill in some values for the user. */
 306         sop->ses = ses_new->sid;
 307
 308         return 0;
 309
 310 error_hash:
 311         cryptodev_cipher_deinit(&ses_new->cdata);
 312         kfree(ses_new->sg);
 313         kfree(ses_new->pages);
 314 error_cipher:
 315         kfree(ses_new);
 316
 317         return ret;
 318
 319 }
 320
 321 /* Everything that needs to be done when remowing a session. */
 322 static inline void
 323 crypto_destroy_session(struct csession *ses_ptr)
 324 {
 325         if (!mutex_trylock(&ses_ptr->sem)) {
 326                 dprintk(2, KERN_DEBUG, "Waiting for semaphore of sid=0x%08X\n",
 327                         ses_ptr->sid);
 328                 mutex_lock(&ses_ptr->sem);
 329         }
 330         dprintk(2, KERN_DEBUG, "Removed session 0x%08X\n", ses_ptr->sid);
 331         cryptodev_cipher_deinit(&ses_ptr->cdata);
 332         cryptodev_hash_deinit(&ses_ptr->hdata);
 333         dprintk(2, KERN_DEBUG, "%s: freeing space for %d user pages\n",
 334                         __func__, ses_ptr->array_size);
 335         kfree(ses_ptr->pages);
 336         kfree(ses_ptr->sg);
 337         mutex_unlock(&ses_ptr->sem);
 338         kfree(ses_ptr);
 339 }
 340
 341 /* Look up a session by ID and remove. */
 342 static int
 343 crypto_finish_session(struct fcrypt *fcr, uint32_t sid)
 344 {
 345         struct csession *tmp, *ses_ptr;
 346         struct list_head *head;
 347         int ret = 0;
 348
 349         mutex_lock(&fcr->sem);
 350         head = &fcr->list;
 351         list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
 352                 if (ses_ptr->sid == sid) {
 353                         list_del(&ses_ptr->entry);
 354                         crypto_destroy_session(ses_ptr);
 355                         break;
 356                 }
 357         }
 358
 359         if (unlikely(!ses_ptr)) {
 360                 dprintk(1, KERN_ERR, "Session with sid=0x%08X not found!\n",
 361                         sid);
 362                 ret = -ENOENT;
 363         }
 364         mutex_unlock(&fcr->sem);
 365
 366         return ret;
 367 }
 368
 369 /* Remove all sessions when closing the file */
 370 static int
 371 crypto_finish_all_sessions(struct fcrypt *fcr)
 372 {
 373         struct csession *tmp, *ses_ptr;
 374         struct list_head *head;
 375
 376         mutex_lock(&fcr->sem);
 377
 378         head = &fcr->list;
 379         list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
 380                 list_del(&ses_ptr->entry);
 381                 crypto_destroy_session(ses_ptr);
 382         }
 383         mutex_unlock(&fcr->sem);
 384
 385         return 0;
 386 }
 387
 388 /* Look up session by session ID. The returned session is locked. */
 389 struct csession *
 390 crypto_get_session_by_sid(struct fcrypt *fcr, uint32_t sid)
 391 {
 392         struct csession *ses_ptr, *retval = 0;
 393
 394         mutex_lock(&fcr->sem);
 395         list_for_each_entry(ses_ptr, &fcr->list, entry) {
 396                 if (ses_ptr->sid == sid) {
 397                         mutex_lock(&ses_ptr->sem);
 398                         retval = ses_ptr;
 399                         break;
 400                 }
 401         }
 402         mutex_unlock(&fcr->sem);
 403
 404         return retval;
 405 }
 406
 407 static void cryptask_routine(struct work_struct *work)
 408 {
 409         struct crypt_priv *pcr = container_of(work, struct crypt_priv, cryptask);
 410         struct todo_list_item *item;
 411         LIST_HEAD(tmp);
 412
 413         /* fetch all pending jobs into the temporary list */
 414         mutex_lock(&pcr->todo.lock);
 415         list_cut_position(&tmp, &pcr->todo.list, pcr->todo.list.prev);
 416         mutex_unlock(&pcr->todo.lock);
 417
 418         /* handle each job locklessly */
 419         list_for_each_entry(item, &tmp, __hook) {
 420                 item->result = crypto_run(&pcr->fcrypt, &item->kcop);
 421                 if (unlikely(item->result))
 422                         dprintk(0, KERN_ERR, "%s: crypto_run() failed: %d\n",
 423                                         __func__, item->result);
 424         }
 425
 426         /* push all handled jobs to the done list at once */
 427         mutex_lock(&pcr->done.lock);
 428         list_splice_tail(&tmp, &pcr->done.list);
 429         mutex_unlock(&pcr->done.lock);
 430
 431         /* wake for POLLIN */
 432         wake_up_interruptible(&pcr->user_waiter);
 433 }
 434
 435 /* ====== /dev/crypto ====== */
 436
 437 static int
 438 cryptodev_open(struct inode *inode, struct file *filp)
 439 {
 440         struct todo_list_item *tmp;
 441         struct crypt_priv *pcr;
 442         int i;
 443
 444         pcr = kmalloc(sizeof(*pcr), GFP_KERNEL);
 445         if (!pcr)
 446                 return -ENOMEM;
 447
 448         memset(pcr, 0, sizeof(*pcr));
 449         mutex_init(&pcr->fcrypt.sem);
 450         INIT_LIST_HEAD(&pcr->fcrypt.list);
 451
 452         INIT_LIST_HEAD(&pcr->free.list);
 453         INIT_LIST_HEAD(&pcr->todo.list);
 454         INIT_LIST_HEAD(&pcr->done.list);
 455         INIT_WORK(&pcr->cryptask, cryptask_routine);
 456         mutex_init(&pcr->free.lock);
 457         mutex_init(&pcr->todo.lock);
 458         mutex_init(&pcr->done.lock);
 459         init_waitqueue_head(&pcr->user_waiter);
 460
 461         for (i = 0; i < DEF_COP_RINGSIZE; i++) {
 462                 tmp = kzalloc(sizeof(struct todo_list_item), GFP_KERNEL);
 463                 pcr->itemcount++;
 464                 dprintk(2, KERN_DEBUG, "%s: allocated new item at %lx\n",
 465                                 __func__, (unsigned long)tmp);
 466                 list_add(&tmp->__hook, &pcr->free.list);
 467         }
 468
 469         filp->private_data = pcr;
 470         dprintk(2, KERN_DEBUG,
 471                 "Cryptodev handle initialised, %d elements in queue\n",
 472                 DEF_COP_RINGSIZE);
 473         return 0;
 474 }
 475
 476 static int
 477 cryptodev_release(struct inode *inode, struct file *filp)
 478 {
 479         struct crypt_priv *pcr = filp->private_data;
 480         struct todo_list_item *item, *item_safe;
 481         int items_freed = 0;
 482
 483         if (!pcr)
 484                 return 0;
 485
 486         cancel_work_sync(&pcr->cryptask);
 487
 488         mutex_destroy(&pcr->todo.lock);
 489         mutex_destroy(&pcr->done.lock);
 490         mutex_destroy(&pcr->free.lock);
 491
 492         list_splice_tail(&pcr->todo.list, &pcr->free.list);
 493         list_splice_tail(&pcr->done.list, &pcr->free.list);
 494
 495         list_for_each_entry_safe(item, item_safe, &pcr->free.list, __hook) {
 496                 dprintk(2, KERN_DEBUG, "%s: freeing item at %lx\n",
 497                                 __func__, (unsigned long)item);
 498                 list_del(&item->__hook);
 499                 kfree(item);
 500                 items_freed++;
 501
 502         }
 503         if (items_freed != pcr->itemcount) {
 504                 dprintk(0, KERN_ERR,
 505                         "%s: freed %d items, but %d should exist!\n",
 506                         __func__, items_freed, pcr->itemcount);
 507         }
 508
 509         crypto_finish_all_sessions(&pcr->fcrypt);
 510         kfree(pcr);
 511         filp->private_data = NULL;
 512
 513         dprintk(2, KERN_DEBUG,
 514                 "Cryptodev handle deinitialised, %d elements freed\n",
 515                 items_freed);
 516         return 0;
 517 }
 518
 519 static int
 520 clonefd(struct file *filp)
 521 {
 522         int ret;
 523         ret = get_unused_fd();
 524         if (ret >= 0) {
 525                         get_file(filp);
 526                         fd_install(ret, filp);
 527         }
 528
 529         return ret;
 530 }
 531
 532 /* enqueue a job for asynchronous completion
 533  *
 534  * returns:
 535  * -EBUSY when there are no free queue slots left
 536  *        (and the number of slots has reached it MAX_COP_RINGSIZE)
 537  * -EFAULT when there was a memory allocation error
 538  * 0 on success */
 539 static int crypto_async_run(struct crypt_priv *pcr, struct kernel_crypt_op *kcop)
 540 {
 541         struct todo_list_item *item = NULL;
 542
 543         mutex_lock(&pcr->free.lock);
 544         if (likely(!list_empty(&pcr->free.list))) {
 545                 item = list_first_entry(&pcr->free.list,
 546                                 struct todo_list_item, __hook);
 547                 list_del(&item->__hook);
 548         } else if (pcr->itemcount < MAX_COP_RINGSIZE) {
 549                 pcr->itemcount++;
 550         } else {
 551                 mutex_unlock(&pcr->free.lock);
 552                 return -EBUSY;
 553         }
 554         mutex_unlock(&pcr->free.lock);
 555
 556         if (unlikely(!item)) {
 557                 item = kzalloc(sizeof(struct todo_list_item), GFP_KERNEL);
 558                 if (unlikely(!item))
 559                         return -EFAULT;
 560                 dprintk(1, KERN_INFO, "%s: increased item count to %d\n",
 561                                 __func__, pcr->itemcount);
 562         }
 563
 564         memcpy(&item->kcop, kcop, sizeof(struct kernel_crypt_op));
 565
 566         mutex_lock(&pcr->todo.lock);
 567         list_add_tail(&item->__hook, &pcr->todo.list);
 568         mutex_unlock(&pcr->todo.lock);
 569
 570         queue_work(cryptodev_wq, &pcr->cryptask);
 571         return 0;
 572 }
 573
 574 /* get the first completed job from the "done" queue
 575  *
 576  * returns:
 577  * -EBUSY if no completed jobs are ready (yet)
 578  * the return value of crypto_run() otherwise */
 579 static int crypto_async_fetch(struct crypt_priv *pcr,
 580                 struct kernel_crypt_op *kcop)
 581 {
 582         struct todo_list_item *item;
 583         int retval;
 584
 585         mutex_lock(&pcr->done.lock);
 586         if (list_empty(&pcr->done.list)) {
 587                 mutex_unlock(&pcr->done.lock);
 588                 return -EBUSY;
 589         }
 590         item = list_first_entry(&pcr->done.list, struct todo_list_item, __hook);
 591         list_del(&item->__hook);
 592         mutex_unlock(&pcr->done.lock);
 593
 594         memcpy(kcop, &item->kcop, sizeof(struct kernel_crypt_op));
 595         retval = item->result;
 596
 597         mutex_lock(&pcr->free.lock);
 598         list_add_tail(&item->__hook, &pcr->free.list);
 599         mutex_unlock(&pcr->free.lock);
 600
 601         /* wake for POLLOUT */
 602         wake_up_interruptible(&pcr->user_waiter);
 603
 604         return retval;
 605 }
 606
 607 /* this function has to be called from process context */
 608 static int fill_kcop_from_cop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
 609 {
 610         struct crypt_op *cop = &kcop->cop;
 611         struct csession *ses_ptr;
 612         int rc;
 613
 614         /* this also enters ses_ptr->sem */
 615         ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
 616         if (unlikely(!ses_ptr)) {
 617                 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
 618                 return -EINVAL;
 619         }
 620         kcop->ivlen = cop->iv ? ses_ptr->cdata.ivsize : 0;
 621         kcop->digestsize = 0; /* will be updated during operation */
 622
 623         crypto_put_session(ses_ptr);
 624
 625         kcop->task = current;
 626         kcop->mm = current->mm;
 627
 628         if (cop->iv) {
 629                 rc = copy_from_user(kcop->iv, cop->iv, kcop->ivlen);
 630                 if (unlikely(rc)) {
 631                         dprintk(1, KERN_ERR,
 632                                 "error copying IV (%d bytes), copy_from_user returned %d for address %lx\n",
 633                                 kcop->ivlen, rc, (unsigned long)cop->iv);
 634                         return -EFAULT;
 635                 }
 636         }
 637
 638         return 0;
 639 }
 640
 641 /* this function has to be called from process context */
 642 static int fill_cop_from_kcop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
 643 {
 644         int ret;
 645
 646         if (kcop->digestsize) {
 647                 ret = copy_to_user(kcop->cop.mac,
 648                                 kcop->hash_output, kcop->digestsize);
 649                 if (unlikely(ret))
 650                         return -EFAULT;
 651         }
 652         if (kcop->ivlen && kcop->cop.flags & COP_FLAG_WRITE_IV) {
 653                 ret = copy_to_user(kcop->cop.iv,
 654                                 kcop->iv, kcop->ivlen);
 655                 if (unlikely(ret))
 656                         return -EFAULT;
 657         }
 658         return 0;
 659 }
 660
 661 static int kcop_from_user(struct kernel_crypt_op *kcop,
 662                         struct fcrypt *fcr, void __user *arg)
 663 {
 664         if (unlikely(copy_from_user(&kcop->cop, arg, sizeof(kcop->cop))))
 665                 return -EFAULT;
 666
 667         return fill_kcop_from_cop(kcop, fcr);
 668 }
 669
 670 static int kcop_to_user(struct kernel_crypt_op *kcop,
 671                         struct fcrypt *fcr, void __user *arg)
 672 {
 673         int ret;
 674
 675         ret = fill_cop_from_kcop(kcop, fcr);
 676         if (unlikely(ret))
 677                 return ret;
 678
 679         if (unlikely(copy_to_user(arg, &kcop->cop, sizeof(kcop->cop))))
 680                 return -EFAULT;
 681         return 0;
 682 }
 683
 684 static inline void tfm_info_to_alg_info(struct alg_info *dst, struct crypto_tfm *tfm)
 685 {
 686         snprintf(dst->cra_name, CRYPTODEV_MAX_ALG_NAME,
 687                         "%s", crypto_tfm_alg_name(tfm));
 688         snprintf(dst->cra_driver_name, CRYPTODEV_MAX_ALG_NAME,
 689                         "%s", crypto_tfm_alg_driver_name(tfm));
 690 }
 691
 692 static int get_session_info(struct fcrypt *fcr, struct session_info_op *siop)
 693 {
 694         struct csession *ses_ptr;
 695
 696         /* this also enters ses_ptr->sem */
 697         ses_ptr = crypto_get_session_by_sid(fcr, siop->ses);
 698         if (unlikely(!ses_ptr)) {
 699                 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", siop->ses);
 700                 return -EINVAL;
 701         }
 702
 703         if (ses_ptr->cdata.init) {
 704                 tfm_info_to_alg_info(&siop->cipher_info,
 705                                 crypto_ablkcipher_tfm(ses_ptr->cdata.async.s));
 706         }
 707         if (ses_ptr->hdata.init) {
 708                 tfm_info_to_alg_info(&siop->hash_info,
 709                                 crypto_ahash_tfm(ses_ptr->hdata.async.s));
 710         }
 711
 712         siop->alignmask = ses_ptr->alignmask;
 713
 714         crypto_put_session(ses_ptr);
 715         return 0;
 716 }
 717
 718 static long
 719 cryptodev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg_)
 720 {
 721         void __user *arg = (void __user *)arg_;
 722         int __user *p = arg;
 723         struct session_op sop;
 724         struct kernel_crypt_op kcop;
 725         struct kernel_crypt_auth_op kcaop;
 726         struct crypt_priv *pcr = filp->private_data;
 727         struct fcrypt *fcr;
 728         struct session_info_op siop;
 729         uint32_t ses;
 730         int ret, fd;
 731
 732         if (unlikely(!pcr))
 733                 BUG();
 734
 735         fcr = &pcr->fcrypt;
 736
 737         switch (cmd) {
 738         case CIOCASYMFEAT:
 739                 return put_user(0, p);
 740         case CRIOGET:
 741                 fd = clonefd(filp);
 742                 ret = put_user(fd, p);
 743                 if (unlikely(ret)) {
 744                         sys_close(fd);
 745                         return ret;
 746                 }
 747                 return ret;
 748         case CIOCGSESSION:
 749                 if (unlikely(copy_from_user(&sop, arg, sizeof(sop))))
 750                         return -EFAULT;
 751
 752                 ret = crypto_create_session(fcr, &sop);
 753                 if (unlikely(ret))
 754                         return ret;
 755                 ret = copy_to_user(arg, &sop, sizeof(sop));
 756                 if (unlikely(ret)) {
 757                         crypto_finish_session(fcr, sop.ses);
 758                         return -EFAULT;
 759                 }
 760                 return ret;
 761         case CIOCFSESSION:
 762                 ret = get_user(ses, (uint32_t __user *)arg);
 763                 if (unlikely(ret))
 764                         return ret;
 765                 ret = crypto_finish_session(fcr, ses);
 766                 return ret;
 767         case CIOCGSESSINFO:
 768                 if (unlikely(copy_from_user(&siop, arg, sizeof(siop))))
 769                         return -EFAULT;
 770
 771                 ret = get_session_info(fcr, &siop);
 772                 if (unlikely(ret))
 773                         return ret;
 774                 return copy_to_user(arg, &siop, sizeof(siop));
 775         case CIOCCRYPT:
 776                 if (unlikely(ret = kcop_from_user(&kcop, fcr, arg))) {
 777                         dprintk(1, KERN_WARNING, "Error copying from user");
 778                         return ret;
 779                 }
 780
 781                 ret = crypto_run(fcr, &kcop);
 782                 if (unlikely(ret)) {
 783                         dprintk(1, KERN_WARNING, "Error in crypto_run");
 784                         return ret;
 785                 }
 786
 787                 return kcop_to_user(&kcop, fcr, arg);
 788         case CIOCAUTHCRYPT:
 789                 if (unlikely(ret = kcaop_from_user(&kcaop, fcr, arg))) {
 790                         dprintk(1, KERN_WARNING, "Error copying from user");
 791                         return ret;
 792                 }
 793
 794                 ret = crypto_auth_run(fcr, &kcaop);
 795                 if (unlikely(ret)) {
 796                         dprintk(1, KERN_WARNING, "Error in crypto_auth_run");
 797                         return ret;
 798                 }
 799                 return kcaop_to_user(&kcaop, fcr, arg);
 800         case CIOCASYNCCRYPT:
 801                 if (unlikely(ret = kcop_from_user(&kcop, fcr, arg)))
 802                         return ret;
 803
 804                 return crypto_async_run(pcr, &kcop);
 805         case CIOCASYNCFETCH:
 806                 ret = crypto_async_fetch(pcr, &kcop);
 807                 if (unlikely(ret))
 808                         return ret;
 809
 810                 return kcop_to_user(&kcop, fcr, arg);
 811         default:
 812                 return -EINVAL;
 813         }
 814 }
 815
 816 /* compatibility code for 32bit userlands */
 817 #ifdef CONFIG_COMPAT
 818
 819 static inline void
 820 compat_to_session_op(struct compat_session_op *compat, struct session_op *sop)
 821 {
 822         sop->cipher = compat->cipher;
 823         sop->mac = compat->mac;
 824         sop->keylen = compat->keylen;
 825
 826         sop->key       = compat_ptr(compat->key);
 827         sop->mackeylen = compat->mackeylen;
 828         sop->mackey    = compat_ptr(compat->mackey);
 829         sop->ses       = compat->ses;
 830 }
 831
 832 static inline void
 833 session_op_to_compat(struct session_op *sop, struct compat_session_op *compat)
 834 {
 835         compat->cipher = sop->cipher;
 836         compat->mac = sop->mac;
 837         compat->keylen = sop->keylen;
 838
 839         compat->key       = ptr_to_compat(sop->key);
 840         compat->mackeylen = sop->mackeylen;
 841         compat->mackey    = ptr_to_compat(sop->mackey);
 842         compat->ses       = sop->ses;
 843 }
 844
 845 static inline void
 846 compat_to_crypt_op(struct compat_crypt_op *compat, struct crypt_op *cop)
 847 {
 848         cop->ses = compat->ses;
 849         cop->op = compat->op;
 850         cop->flags = compat->flags;
 851         cop->len = compat->len;
 852
 853         cop->src = compat_ptr(compat->src);
 854         cop->dst = compat_ptr(compat->dst);
 855         cop->mac = compat_ptr(compat->mac);
 856         cop->iv  = compat_ptr(compat->iv);
 857 }
 858
 859 static inline void
 860 crypt_op_to_compat(struct crypt_op *cop, struct compat_crypt_op *compat)
 861 {
 862         compat->ses = cop->ses;
 863         compat->op = cop->op;
 864         compat->flags = cop->flags;
 865         compat->len = cop->len;
 866
 867         compat->src = ptr_to_compat(cop->src);
 868         compat->dst = ptr_to_compat(cop->dst);
 869         compat->mac = ptr_to_compat(cop->mac);
 870         compat->iv  = ptr_to_compat(cop->iv);
 871 }
 872
 873 static int compat_kcop_from_user(struct kernel_crypt_op *kcop,
 874                                  struct fcrypt *fcr, void __user *arg)
 875 {
 876         struct compat_crypt_op compat_cop;
 877
 878         if (unlikely(copy_from_user(&compat_cop, arg, sizeof(compat_cop))))
 879                 return -EFAULT;
 880         compat_to_crypt_op(&compat_cop, &kcop->cop);
 881
 882         return fill_kcop_from_cop(kcop, fcr);
 883 }
 884
 885 static int compat_kcop_to_user(struct kernel_crypt_op *kcop,
 886                                  struct fcrypt *fcr, void __user *arg)
 887 {
 888         int ret;
 889         struct compat_crypt_op compat_cop;
 890
 891         ret = fill_cop_from_kcop(kcop, fcr);
 892         if (unlikely(ret)) {
 893                 dprintk(1, KERN_WARNING, "Error in fill_cop_from_kcop");
 894                 return ret;
 895         }
 896         crypt_op_to_compat(&kcop->cop, &compat_cop);
 897
 898         if (unlikely(copy_to_user(arg, &compat_cop, sizeof(compat_cop)))) {
 899                 dprintk(1, KERN_WARNING, "Error copying to user");
 900                 return -EFAULT;
 901         }
 902         return 0;
 903 }
 904
 905 static long
 906 cryptodev_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg_)
 907 {
 908         void __user *arg = (void __user *)arg_;
 909         struct crypt_priv *pcr = file->private_data;
 910         struct fcrypt *fcr;
 911         struct session_op sop;
 912         struct compat_session_op compat_sop;
 913         struct kernel_crypt_op kcop;
 914         int ret;
 915
 916         if (unlikely(!pcr))
 917                 BUG();
 918
 919         fcr = &pcr->fcrypt;
 920
 921         switch (cmd) {
 922         case CIOCASYMFEAT:
 923         case CRIOGET:
 924         case CIOCFSESSION:
 925         case CIOCGSESSINFO:
 926                 return cryptodev_ioctl(file, cmd, arg_);
 927
 928         case COMPAT_CIOCGSESSION:
 929                 if (unlikely(copy_from_user(&compat_sop, arg,
 930                                             sizeof(compat_sop))))
 931                         return -EFAULT;
 932                 compat_to_session_op(&compat_sop, &sop);
 933
 934                 ret = crypto_create_session(fcr, &sop);
 935                 if (unlikely(ret))
 936                         return ret;
 937
 938                 session_op_to_compat(&sop, &compat_sop);
 939                 ret = copy_to_user(arg, &compat_sop, sizeof(compat_sop));
 940                 if (unlikely(ret)) {
 941                         crypto_finish_session(fcr, sop.ses);
 942                         return -EFAULT;
 943                 }
 944                 return ret;
 945
 946         case COMPAT_CIOCCRYPT:
 947                 ret = compat_kcop_from_user(&kcop, fcr, arg);
 948                 if (unlikely(ret))
 949                         return ret;
 950
 951                 ret = crypto_run(fcr, &kcop);
 952                 if (unlikely(ret))
 953                         return ret;
 954
 955                 return compat_kcop_to_user(&kcop, fcr, arg);
 956         case COMPAT_CIOCASYNCCRYPT:
 957                 if (unlikely(ret = compat_kcop_from_user(&kcop, fcr, arg)))
 958                         return ret;
 959
 960                 return crypto_async_run(pcr, &kcop);
 961         case COMPAT_CIOCASYNCFETCH:
 962                 ret = crypto_async_fetch(pcr, &kcop);
 963                 if (unlikely(ret))
 964                         return ret;
 965
 966                 return compat_kcop_to_user(&kcop, fcr, arg);
 967
 968         default:
 969                 return -EINVAL;
 970         }
 971 }
 972
 973 #endif /* CONFIG_COMPAT */
 974
 975 static unsigned int cryptodev_poll(struct file *file, poll_table *wait)
 976 {
 977         struct crypt_priv *pcr = file->private_data;
 978         int ret = 0;
 979
 980         poll_wait(file, &pcr->user_waiter, wait);
 981
 982         if (!list_empty_careful(&pcr->done.list))
 983                 ret |= POLLIN | POLLRDNORM;
 984         if (!list_empty_careful(&pcr->free.list) || pcr->itemcount < MAX_COP_RINGSIZE)
 985                 ret |= POLLOUT | POLLWRNORM;
 986
 987         return ret;
 988 }
 989
 990 static const struct file_operations cryptodev_fops = {
 991         .owner = THIS_MODULE,
 992         .open = cryptodev_open,
 993         .release = cryptodev_release,
 994         .unlocked_ioctl = cryptodev_ioctl,
 995 #ifdef CONFIG_COMPAT
 996         .compat_ioctl = cryptodev_compat_ioctl,
 997 #endif /* CONFIG_COMPAT */
 998         .poll = cryptodev_poll,
 999 };
1000
1001 static struct miscdevice cryptodev = {
1002         .minor = MISC_DYNAMIC_MINOR,
1003         .name = "crypto",
1004         .fops = &cryptodev_fops,
1005 };
1006
1007 static int __init
1008 cryptodev_register(void)
1009 {
1010         int rc;
1011
1012         rc = misc_register(&cryptodev);
1013         if (unlikely(rc)) {
1014                 printk(KERN_ERR PFX "registration of /dev/crypto failed\n");
1015                 return rc;
1016         }
1017
1018         return 0;
1019 }
1020
1021 static void __exit
1022 cryptodev_deregister(void)
1023 {
1024         misc_deregister(&cryptodev);
1025 }
1026
1027 /* ====== Module init/exit ====== */
1028 static int __init init_cryptodev(void)
1029 {
1030         int rc;
1031
1032         cryptodev_wq = create_workqueue("cryptodev_queue");
1033         if (unlikely(!cryptodev_wq)) {
1034                 printk(KERN_ERR PFX "failed to allocate the cryptodev workqueue\n");
1035                 return -EFAULT;
1036         }
1037
1038         rc = cryptodev_register();
1039         if (unlikely(rc)) {
1040                 destroy_workqueue(cryptodev_wq);
1041                 return rc;
1042         }
1043
1044         printk(KERN_INFO PFX "driver %s loaded.\n", VERSION);
1045
1046         return 0;
1047 }
1048
1049 static void __exit exit_cryptodev(void)
1050 {
1051         flush_workqueue(cryptodev_wq);
1052         destroy_workqueue(cryptodev_wq);
1053
1054         cryptodev_deregister();
1055         printk(KERN_INFO PFX "driver unloaded.\n");
1056 }
1057
1058 module_init(init_cryptodev);
1059 module_exit(exit_cryptodev);
1060