drivers/md/dm-crypt.c

   1 /*
   2  * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
   3  * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
   4  *
   5  * This file is released under the GPL.
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/kernel.h>
  11 #include <linux/bio.h>
  12 #include <linux/blkdev.h>
  13 #include <linux/mempool.h>
  14 #include <linux/slab.h>
  15 #include <linux/crypto.h>
  16 #include <linux/workqueue.h>
  17 #include <asm/atomic.h>
  18 #include <linux/scatterlist.h>
  19 #include <asm/page.h>
  20
  21 #include "dm.h"
  22
  23 #define PFX     "crypt: "
  24
  25 /*
  26  * per bio private data
  27  */
  28 struct crypt_io {
  29         struct dm_target *target;
  30         struct bio *bio;
  31         struct bio *first_clone;
  32         struct work_struct work;
  33         atomic_t pending;
  34         int error;
  35 };
  36
  37 /*
  38  * context holding the current state of a multi-part conversion
  39  */
  40 struct convert_context {
  41         struct bio *bio_in;
  42         struct bio *bio_out;
  43         unsigned int offset_in;
  44         unsigned int offset_out;
  45         unsigned int idx_in;
  46         unsigned int idx_out;
  47         sector_t sector;
  48         int write;
  49 };
  50
  51 struct crypt_config;
  52
  53 struct crypt_iv_operations {
  54         int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
  55                    const char *opts);
  56         void (*dtr)(struct crypt_config *cc);
  57         const char *(*status)(struct crypt_config *cc);
  58         int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector);
  59 };
  60
  61 /*
  62  * Crypt: maps a linear range of a block device
  63  * and encrypts / decrypts at the same time.
  64  */
  65 struct crypt_config {
  66         struct dm_dev *dev;
  67         sector_t start;
  68
  69         /*
  70          * pool for per bio private data and
  71          * for encryption buffer pages
  72          */
  73         mempool_t *io_pool;
  74         mempool_t *page_pool;
  75
  76         /*
  77          * crypto related data
  78          */
  79         struct crypt_iv_operations *iv_gen_ops;
  80         char *iv_mode;
  81         void *iv_gen_private;
  82         sector_t iv_offset;
  83         unsigned int iv_size;
  84
  85         struct crypto_tfm *tfm;
  86         unsigned int key_size;
  87         u8 key[0];
  88 };
  89
  90 #define MIN_IOS        256
  91 #define MIN_POOL_PAGES 32
  92 #define MIN_BIO_PAGES  8
  93
  94 static kmem_cache_t *_crypt_io_pool;
  95
  96 /*
  97  * Mempool alloc and free functions for the page
  98  */
  99 static void *mempool_alloc_page(gfp_t gfp_mask, void *data)
 100 {
 101         return alloc_page(gfp_mask);
 102 }
 103
 104 static void mempool_free_page(void *page, void *data)
 105 {
 106         __free_page(page);
 107 }
 108
 109
 110 /*
 111  * Different IV generation algorithms:
 112  *
 113  * plain: the initial vector is the 32-bit low-endian version of the sector
 114  *        number, padded with zeros if neccessary.
 115  *
 116  * ess_iv: "encrypted sector|salt initial vector", the sector number is
 117  *         encrypted with the bulk cipher using a salt as key. The salt
 118  *         should be derived from the bulk cipher's key via hashing.
 119  *
 120  * plumb: unimplemented, see:
 121  * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
 122  */
 123
 124 static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
 125 {
 126         memset(iv, 0, cc->iv_size);
 127         *(u32 *)iv = cpu_to_le32(sector & 0xffffffff);
 128
 129         return 0;
 130 }
 131
 132 static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
 133                               const char *opts)
 134 {
 135         struct crypto_tfm *essiv_tfm;
 136         struct crypto_tfm *hash_tfm;
 137         struct scatterlist sg;
 138         unsigned int saltsize;
 139         u8 *salt;
 140
 141         if (opts == NULL) {
 142                 ti->error = PFX "Digest algorithm missing for ESSIV mode";
 143                 return -EINVAL;
 144         }
 145
 146         /* Hash the cipher key with the given hash algorithm */
 147         hash_tfm = crypto_alloc_tfm(opts, CRYPTO_TFM_REQ_MAY_SLEEP);
 148         if (hash_tfm == NULL) {
 149                 ti->error = PFX "Error initializing ESSIV hash";
 150                 return -EINVAL;
 151         }
 152
 153         if (crypto_tfm_alg_type(hash_tfm) != CRYPTO_ALG_TYPE_DIGEST) {
 154                 ti->error = PFX "Expected digest algorithm for ESSIV hash";
 155                 crypto_free_tfm(hash_tfm);
 156                 return -EINVAL;
 157         }
 158
 159         saltsize = crypto_tfm_alg_digestsize(hash_tfm);
 160         salt = kmalloc(saltsize, GFP_KERNEL);
 161         if (salt == NULL) {
 162                 ti->error = PFX "Error kmallocing salt storage in ESSIV";
 163                 crypto_free_tfm(hash_tfm);
 164                 return -ENOMEM;
 165         }
 166
 167         sg_set_buf(&sg, cc->key, cc->key_size);
 168         crypto_digest_digest(hash_tfm, &sg, 1, salt);
 169         crypto_free_tfm(hash_tfm);
 170
 171         /* Setup the essiv_tfm with the given salt */
 172         essiv_tfm = crypto_alloc_tfm(crypto_tfm_alg_name(cc->tfm),
 173                                      CRYPTO_TFM_MODE_ECB |
 174                                      CRYPTO_TFM_REQ_MAY_SLEEP);
 175         if (essiv_tfm == NULL) {
 176                 ti->error = PFX "Error allocating crypto tfm for ESSIV";
 177                 kfree(salt);
 178                 return -EINVAL;
 179         }
 180         if (crypto_tfm_alg_blocksize(essiv_tfm)
 181             != crypto_tfm_alg_ivsize(cc->tfm)) {
 182                 ti->error = PFX "Block size of ESSIV cipher does "
 183                                 "not match IV size of block cipher";
 184                 crypto_free_tfm(essiv_tfm);
 185                 kfree(salt);
 186                 return -EINVAL;
 187         }
 188         if (crypto_cipher_setkey(essiv_tfm, salt, saltsize) < 0) {
 189                 ti->error = PFX "Failed to set key for ESSIV cipher";
 190                 crypto_free_tfm(essiv_tfm);
 191                 kfree(salt);
 192                 return -EINVAL;
 193         }
 194         kfree(salt);
 195
 196         cc->iv_gen_private = (void *)essiv_tfm;
 197         return 0;
 198 }
 199
 200 static void crypt_iv_essiv_dtr(struct crypt_config *cc)
 201 {
 202         crypto_free_tfm((struct crypto_tfm *)cc->iv_gen_private);
 203         cc->iv_gen_private = NULL;
 204 }
 205
 206 static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
 207 {
 208         struct scatterlist sg;
 209
 210         memset(iv, 0, cc->iv_size);
 211         *(u64 *)iv = cpu_to_le64(sector);
 212
 213         sg_set_buf(&sg, iv, cc->iv_size);
 214         crypto_cipher_encrypt((struct crypto_tfm *)cc->iv_gen_private,
 215                               &sg, &sg, cc->iv_size);
 216
 217         return 0;
 218 }
 219
 220 static struct crypt_iv_operations crypt_iv_plain_ops = {
 221         .generator = crypt_iv_plain_gen
 222 };
 223
 224 static struct crypt_iv_operations crypt_iv_essiv_ops = {
 225         .ctr       = crypt_iv_essiv_ctr,
 226         .dtr       = crypt_iv_essiv_dtr,
 227         .generator = crypt_iv_essiv_gen
 228 };
 229
 230
 231 static inline int
 232 crypt_convert_scatterlist(struct crypt_config *cc, struct scatterlist *out,
 233                           struct scatterlist *in, unsigned int length,
 234                           int write, sector_t sector)
 235 {
 236         u8 iv[cc->iv_size];
 237         int r;
 238
 239         if (cc->iv_gen_ops) {
 240                 r = cc->iv_gen_ops->generator(cc, iv, sector);
 241                 if (r < 0)
 242                         return r;
 243
 244                 if (write)
 245                         r = crypto_cipher_encrypt_iv(cc->tfm, out, in, length, iv);
 246                 else
 247                         r = crypto_cipher_decrypt_iv(cc->tfm, out, in, length, iv);
 248         } else {
 249                 if (write)
 250                         r = crypto_cipher_encrypt(cc->tfm, out, in, length);
 251                 else
 252                         r = crypto_cipher_decrypt(cc->tfm, out, in, length);
 253         }
 254
 255         return r;
 256 }
 257
 258 static void
 259 crypt_convert_init(struct crypt_config *cc, struct convert_context *ctx,
 260                    struct bio *bio_out, struct bio *bio_in,
 261                    sector_t sector, int write)
 262 {
 263         ctx->bio_in = bio_in;
 264         ctx->bio_out = bio_out;
 265         ctx->offset_in = 0;
 266         ctx->offset_out = 0;
 267         ctx->idx_in = bio_in ? bio_in->bi_idx : 0;
 268         ctx->idx_out = bio_out ? bio_out->bi_idx : 0;
 269         ctx->sector = sector + cc->iv_offset;
 270         ctx->write = write;
 271 }
 272
 273 /*
 274  * Encrypt / decrypt data from one bio to another one (can be the same one)
 275  */
 276 static int crypt_convert(struct crypt_config *cc,
 277                          struct convert_context *ctx)
 278 {
 279         int r = 0;
 280
 281         while(ctx->idx_in < ctx->bio_in->bi_vcnt &&
 282               ctx->idx_out < ctx->bio_out->bi_vcnt) {
 283                 struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in);
 284                 struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out);
 285                 struct scatterlist sg_in = {
 286                         .page = bv_in->bv_page,
 287                         .offset = bv_in->bv_offset + ctx->offset_in,
 288                         .length = 1 << SECTOR_SHIFT
 289                 };
 290                 struct scatterlist sg_out = {
 291                         .page = bv_out->bv_page,
 292                         .offset = bv_out->bv_offset + ctx->offset_out,
 293                         .length = 1 << SECTOR_SHIFT
 294                 };
 295
 296                 ctx->offset_in += sg_in.length;
 297                 if (ctx->offset_in >= bv_in->bv_len) {
 298                         ctx->offset_in = 0;
 299                         ctx->idx_in++;
 300                 }
 301
 302                 ctx->offset_out += sg_out.length;
 303                 if (ctx->offset_out >= bv_out->bv_len) {
 304                         ctx->offset_out = 0;
 305                         ctx->idx_out++;
 306                 }
 307
 308                 r = crypt_convert_scatterlist(cc, &sg_out, &sg_in, sg_in.length,
 309                                               ctx->write, ctx->sector);
 310                 if (r < 0)
 311                         break;
 312
 313                 ctx->sector++;
 314         }
 315
 316         return r;
 317 }
 318
 319 /*
 320  * Generate a new unfragmented bio with the given size
 321  * This should never violate the device limitations
 322  * May return a smaller bio when running out of pages
 323  */
 324 static struct bio *
 325 crypt_alloc_buffer(struct crypt_config *cc, unsigned int size,
 326                    struct bio *base_bio, unsigned int *bio_vec_idx)
 327 {
 328         struct bio *bio;
 329         unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
 330         gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM;
 331         unsigned int i;
 332
 333         /*
 334          * Use __GFP_NOMEMALLOC to tell the VM to act less aggressively and
 335          * to fail earlier.  This is not necessary but increases throughput.
 336          * FIXME: Is this really intelligent?
 337          */
 338         if (base_bio)
 339                 bio = bio_clone(base_bio, GFP_NOIO|__GFP_NOMEMALLOC);
 340         else
 341                 bio = bio_alloc(GFP_NOIO|__GFP_NOMEMALLOC, nr_iovecs);
 342         if (!bio)
 343                 return NULL;
 344
 345         /* if the last bio was not complete, continue where that one ended */
 346         bio->bi_idx = *bio_vec_idx;
 347         bio->bi_vcnt = *bio_vec_idx;
 348         bio->bi_size = 0;
 349         bio->bi_flags &= ~(1 << BIO_SEG_VALID);
 350
 351         /* bio->bi_idx pages have already been allocated */
 352         size -= bio->bi_idx * PAGE_SIZE;
 353
 354         for(i = bio->bi_idx; i < nr_iovecs; i++) {
 355                 struct bio_vec *bv = bio_iovec_idx(bio, i);
 356
 357                 bv->bv_page = mempool_alloc(cc->page_pool, gfp_mask);
 358                 if (!bv->bv_page)
 359                         break;
 360
 361                 /*
 362                  * if additional pages cannot be allocated without waiting,
 363                  * return a partially allocated bio, the caller will then try
 364                  * to allocate additional bios while submitting this partial bio
 365                  */
 366                 if ((i - bio->bi_idx) == (MIN_BIO_PAGES - 1))
 367                         gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT;
 368
 369                 bv->bv_offset = 0;
 370                 if (size > PAGE_SIZE)
 371                         bv->bv_len = PAGE_SIZE;
 372                 else
 373                         bv->bv_len = size;
 374
 375                 bio->bi_size += bv->bv_len;
 376                 bio->bi_vcnt++;
 377                 size -= bv->bv_len;
 378         }
 379
 380         if (!bio->bi_size) {
 381                 bio_put(bio);
 382                 return NULL;
 383         }
 384
 385         /*
 386          * Remember the last bio_vec allocated to be able
 387          * to correctly continue after the splitting.
 388          */
 389         *bio_vec_idx = bio->bi_vcnt;
 390
 391         return bio;
 392 }
 393
 394 static void crypt_free_buffer_pages(struct crypt_config *cc,
 395                                     struct bio *bio, unsigned int bytes)
 396 {
 397         unsigned int i, start, end;
 398         struct bio_vec *bv;
 399
 400         /*
 401          * This is ugly, but Jens Axboe thinks that using bi_idx in the
 402          * endio function is too dangerous at the moment, so I calculate the
 403          * correct position using bi_vcnt and bi_size.
 404          * The bv_offset and bv_len fields might already be modified but we
 405          * know that we always allocated whole pages.
 406          * A fix to the bi_idx issue in the kernel is in the works, so
 407          * we will hopefully be able to revert to the cleaner solution soon.
 408          */
 409         i = bio->bi_vcnt - 1;
 410         bv = bio_iovec_idx(bio, i);
 411         end = (i << PAGE_SHIFT) + (bv->bv_offset + bv->bv_len) - bio->bi_size;
 412         start = end - bytes;
 413
 414         start >>= PAGE_SHIFT;
 415         if (!bio->bi_size)
 416                 end = bio->bi_vcnt;
 417         else
 418                 end >>= PAGE_SHIFT;
 419
 420         for(i = start; i < end; i++) {
 421                 bv = bio_iovec_idx(bio, i);
 422                 BUG_ON(!bv->bv_page);
 423                 mempool_free(bv->bv_page, cc->page_pool);
 424                 bv->bv_page = NULL;
 425         }
 426 }
 427
 428 /*
 429  * One of the bios was finished. Check for completion of
 430  * the whole request and correctly clean up the buffer.
 431  */
 432 static void dec_pending(struct crypt_io *io, int error)
 433 {
 434         struct crypt_config *cc = (struct crypt_config *) io->target->private;
 435
 436         if (error < 0)
 437                 io->error = error;
 438
 439         if (!atomic_dec_and_test(&io->pending))
 440                 return;
 441
 442         if (io->first_clone)
 443                 bio_put(io->first_clone);
 444
 445         bio_endio(io->bio, io->bio->bi_size, io->error);
 446
 447         mempool_free(io, cc->io_pool);
 448 }
 449
 450 /*
 451  * kcryptd:
 452  *
 453  * Needed because it would be very unwise to do decryption in an
 454  * interrupt context, so bios returning from read requests get
 455  * queued here.
 456  */
 457 static struct workqueue_struct *_kcryptd_workqueue;
 458
 459 static void kcryptd_do_work(void *data)
 460 {
 461         struct crypt_io *io = (struct crypt_io *) data;
 462         struct crypt_config *cc = (struct crypt_config *) io->target->private;
 463         struct convert_context ctx;
 464         int r;
 465
 466         crypt_convert_init(cc, &ctx, io->bio, io->bio,
 467                            io->bio->bi_sector - io->target->begin, 0);
 468         r = crypt_convert(cc, &ctx);
 469
 470         dec_pending(io, r);
 471 }
 472
 473 static void kcryptd_queue_io(struct crypt_io *io)
 474 {
 475         INIT_WORK(&io->work, kcryptd_do_work, io);
 476         queue_work(_kcryptd_workqueue, &io->work);
 477 }
 478
 479 /*
 480  * Decode key from its hex representation
 481  */
 482 static int crypt_decode_key(u8 *key, char *hex, unsigned int size)
 483 {
 484         char buffer[3];
 485         char *endp;
 486         unsigned int i;
 487
 488         buffer[2] = '\0';
 489
 490         for(i = 0; i < size; i++) {
 491                 buffer[0] = *hex++;
 492                 buffer[1] = *hex++;
 493
 494                 key[i] = (u8)simple_strtoul(buffer, &endp, 16);
 495
 496                 if (endp != &buffer[2])
 497                         return -EINVAL;
 498         }
 499
 500         if (*hex != '\0')
 501                 return -EINVAL;
 502
 503         return 0;
 504 }
 505
 506 /*
 507  * Encode key into its hex representation
 508  */
 509 static void crypt_encode_key(char *hex, u8 *key, unsigned int size)
 510 {
 511         unsigned int i;
 512
 513         for(i = 0; i < size; i++) {
 514                 sprintf(hex, "%02x", *key);
 515                 hex += 2;
 516                 key++;
 517         }
 518 }
 519
 520 /*
 521  * Construct an encryption mapping:
 522  * <cipher> <key> <iv_offset> <dev_path> <start>
 523  */
 524 static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 525 {
 526         struct crypt_config *cc;
 527         struct crypto_tfm *tfm;
 528         char *tmp;
 529         char *cipher;
 530         char *chainmode;
 531         char *ivmode;
 532         char *ivopts;
 533         unsigned int crypto_flags;
 534         unsigned int key_size;
 535
 536         if (argc != 5) {
 537                 ti->error = PFX "Not enough arguments";
 538                 return -EINVAL;
 539         }
 540
 541         tmp = argv[0];
 542         cipher = strsep(&tmp, "-");
 543         chainmode = strsep(&tmp, "-");
 544         ivopts = strsep(&tmp, "-");
 545         ivmode = strsep(&ivopts, ":");
 546
 547         if (tmp)
 548                 DMWARN(PFX "Unexpected additional cipher options");
 549
 550         key_size = strlen(argv[1]) >> 1;
 551
 552         cc = kmalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL);
 553         if (cc == NULL) {
 554                 ti->error =
 555                         PFX "Cannot allocate transparent encryption context";
 556                 return -ENOMEM;
 557         }
 558
 559         cc->key_size = key_size;
 560         if ((!key_size && strcmp(argv[1], "-") != 0) ||
 561             (key_size && crypt_decode_key(cc->key, argv[1], key_size) < 0)) {
 562                 ti->error = PFX "Error decoding key";
 563                 goto bad1;
 564         }
 565
 566         /* Compatiblity mode for old dm-crypt cipher strings */
 567         if (!chainmode || (strcmp(chainmode, "plain") == 0 && !ivmode)) {
 568                 chainmode = "cbc";
 569                 ivmode = "plain";
 570         }
 571
 572         /* Choose crypto_flags according to chainmode */
 573         if (strcmp(chainmode, "cbc") == 0)
 574                 crypto_flags = CRYPTO_TFM_MODE_CBC;
 575         else if (strcmp(chainmode, "ecb") == 0)
 576                 crypto_flags = CRYPTO_TFM_MODE_ECB;
 577         else {
 578                 ti->error = PFX "Unknown chaining mode";
 579                 goto bad1;
 580         }
 581
 582         if (crypto_flags != CRYPTO_TFM_MODE_ECB && !ivmode) {
 583                 ti->error = PFX "This chaining mode requires an IV mechanism";
 584                 goto bad1;
 585         }
 586
 587         tfm = crypto_alloc_tfm(cipher, crypto_flags | CRYPTO_TFM_REQ_MAY_SLEEP);
 588         if (!tfm) {
 589                 ti->error = PFX "Error allocating crypto tfm";
 590                 goto bad1;
 591         }
 592         if (crypto_tfm_alg_type(tfm) != CRYPTO_ALG_TYPE_CIPHER) {
 593                 ti->error = PFX "Expected cipher algorithm";
 594                 goto bad2;
 595         }
 596
 597         cc->tfm = tfm;
 598
 599         /*
 600          * Choose ivmode. Valid modes: "plain", "essiv:<esshash>".
 601          * See comments at iv code
 602          */
 603
 604         if (ivmode == NULL)
 605                 cc->iv_gen_ops = NULL;
 606         else if (strcmp(ivmode, "plain") == 0)
 607                 cc->iv_gen_ops = &crypt_iv_plain_ops;
 608         else if (strcmp(ivmode, "essiv") == 0)
 609                 cc->iv_gen_ops = &crypt_iv_essiv_ops;
 610         else {
 611                 ti->error = PFX "Invalid IV mode";
 612                 goto bad2;
 613         }
 614
 615         if (cc->iv_gen_ops && cc->iv_gen_ops->ctr &&
 616             cc->iv_gen_ops->ctr(cc, ti, ivopts) < 0)
 617                 goto bad2;
 618
 619         if (tfm->crt_cipher.cit_decrypt_iv && tfm->crt_cipher.cit_encrypt_iv)
 620                 /* at least a 64 bit sector number should fit in our buffer */
 621                 cc->iv_size = max(crypto_tfm_alg_ivsize(tfm),
 622                                   (unsigned int)(sizeof(u64) / sizeof(u8)));
 623         else {
 624                 cc->iv_size = 0;
 625                 if (cc->iv_gen_ops) {
 626                         DMWARN(PFX "Selected cipher does not support IVs");
 627                         if (cc->iv_gen_ops->dtr)
 628                                 cc->iv_gen_ops->dtr(cc);
 629                         cc->iv_gen_ops = NULL;
 630                 }
 631         }
 632
 633         cc->io_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
 634                                      mempool_free_slab, _crypt_io_pool);
 635         if (!cc->io_pool) {
 636                 ti->error = PFX "Cannot allocate crypt io mempool";
 637                 goto bad3;
 638         }
 639
 640         cc->page_pool = mempool_create(MIN_POOL_PAGES, mempool_alloc_page,
 641                                        mempool_free_page, NULL);
 642         if (!cc->page_pool) {
 643                 ti->error = PFX "Cannot allocate page mempool";
 644                 goto bad4;
 645         }
 646
 647         if (tfm->crt_cipher.cit_setkey(tfm, cc->key, key_size) < 0) {
 648                 ti->error = PFX "Error setting key";
 649                 goto bad5;
 650         }
 651
 652         if (sscanf(argv[2], SECTOR_FORMAT, &cc->iv_offset) != 1) {
 653                 ti->error = PFX "Invalid iv_offset sector";
 654                 goto bad5;
 655         }
 656
 657         if (sscanf(argv[4], SECTOR_FORMAT, &cc->start) != 1) {
 658                 ti->error = PFX "Invalid device sector";
 659                 goto bad5;
 660         }
 661
 662         if (dm_get_device(ti, argv[3], cc->start, ti->len,
 663                           dm_table_get_mode(ti->table), &cc->dev)) {
 664                 ti->error = PFX "Device lookup failed";
 665                 goto bad5;
 666         }
 667
 668         if (ivmode && cc->iv_gen_ops) {
 669                 if (ivopts)
 670                         *(ivopts - 1) = ':';
 671                 cc->iv_mode = kmalloc(strlen(ivmode) + 1, GFP_KERNEL);
 672                 if (!cc->iv_mode) {
 673                         ti->error = PFX "Error kmallocing iv_mode string";
 674                         goto bad5;
 675                 }
 676                 strcpy(cc->iv_mode, ivmode);
 677         } else
 678                 cc->iv_mode = NULL;
 679
 680         ti->private = cc;
 681         return 0;
 682
 683 bad5:
 684         mempool_destroy(cc->page_pool);
 685 bad4:
 686         mempool_destroy(cc->io_pool);
 687 bad3:
 688         if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
 689                 cc->iv_gen_ops->dtr(cc);
 690 bad2:
 691         crypto_free_tfm(tfm);
 692 bad1:
 693         /* Must zero key material before freeing */
 694         memset(cc, 0, sizeof(*cc) + cc->key_size * sizeof(u8));
 695         kfree(cc);
 696         return -EINVAL;
 697 }
 698
 699 static void crypt_dtr(struct dm_target *ti)
 700 {
 701         struct crypt_config *cc = (struct crypt_config *) ti->private;
 702
 703         mempool_destroy(cc->page_pool);
 704         mempool_destroy(cc->io_pool);
 705
 706         kfree(cc->iv_mode);
 707         if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
 708                 cc->iv_gen_ops->dtr(cc);
 709         crypto_free_tfm(cc->tfm);
 710         dm_put_device(ti, cc->dev);
 711
 712         /* Must zero key material before freeing */
 713         memset(cc, 0, sizeof(*cc) + cc->key_size * sizeof(u8));
 714         kfree(cc);
 715 }
 716
 717 static int crypt_endio(struct bio *bio, unsigned int done, int error)
 718 {
 719         struct crypt_io *io = (struct crypt_io *) bio->bi_private;
 720         struct crypt_config *cc = (struct crypt_config *) io->target->private;
 721
 722         if (bio_data_dir(bio) == WRITE) {
 723                 /*
 724                  * free the processed pages, even if
 725                  * it's only a partially completed write
 726                  */
 727                 crypt_free_buffer_pages(cc, bio, done);
 728         }
 729
 730         if (bio->bi_size)
 731                 return 1;
 732
 733         bio_put(bio);
 734
 735         /*
 736          * successful reads are decrypted by the worker thread
 737          */
 738         if ((bio_data_dir(bio) == READ)
 739             && bio_flagged(bio, BIO_UPTODATE)) {
 740                 kcryptd_queue_io(io);
 741                 return 0;
 742         }
 743
 744         dec_pending(io, error);
 745         return error;
 746 }
 747
 748 static inline struct bio *
 749 crypt_clone(struct crypt_config *cc, struct crypt_io *io, struct bio *bio,
 750             sector_t sector, unsigned int *bvec_idx,
 751             struct convert_context *ctx)
 752 {
 753         struct bio *clone;
 754
 755         if (bio_data_dir(bio) == WRITE) {
 756                 clone = crypt_alloc_buffer(cc, bio->bi_size,
 757                                  io->first_clone, bvec_idx);
 758                 if (clone) {
 759                         ctx->bio_out = clone;
 760                         if (crypt_convert(cc, ctx) < 0) {
 761                                 crypt_free_buffer_pages(cc, clone,
 762                                                         clone->bi_size);
 763                                 bio_put(clone);
 764                                 return NULL;
 765                         }
 766                 }
 767         } else {
 768                 /*
 769                  * The block layer might modify the bvec array, so always
 770                  * copy the required bvecs because we need the original
 771                  * one in order to decrypt the whole bio data *afterwards*.
 772                  */
 773                 clone = bio_alloc(GFP_NOIO, bio_segments(bio));
 774                 if (clone) {
 775                         clone->bi_idx = 0;
 776                         clone->bi_vcnt = bio_segments(bio);
 777                         clone->bi_size = bio->bi_size;
 778                         memcpy(clone->bi_io_vec, bio_iovec(bio),
 779                                sizeof(struct bio_vec) * clone->bi_vcnt);
 780                 }
 781         }
 782
 783         if (!clone)
 784                 return NULL;
 785
 786         clone->bi_private = io;
 787         clone->bi_end_io = crypt_endio;
 788         clone->bi_bdev = cc->dev->bdev;
 789         clone->bi_sector = cc->start + sector;
 790         clone->bi_rw = bio->bi_rw;
 791
 792         return clone;
 793 }
 794
 795 static int crypt_map(struct dm_target *ti, struct bio *bio,
 796                      union map_info *map_context)
 797 {
 798         struct crypt_config *cc = (struct crypt_config *) ti->private;
 799         struct crypt_io *io = mempool_alloc(cc->io_pool, GFP_NOIO);
 800         struct convert_context ctx;
 801         struct bio *clone;
 802         unsigned int remaining = bio->bi_size;
 803         sector_t sector = bio->bi_sector - ti->begin;
 804         unsigned int bvec_idx = 0;
 805
 806         io->target = ti;
 807         io->bio = bio;
 808         io->first_clone = NULL;
 809         io->error = 0;
 810         atomic_set(&io->pending, 1); /* hold a reference */
 811
 812         if (bio_data_dir(bio) == WRITE)
 813                 crypt_convert_init(cc, &ctx, NULL, bio, sector, 1);
 814
 815         /*
 816          * The allocated buffers can be smaller than the whole bio,
 817          * so repeat the whole process until all the data can be handled.
 818          */
 819         while (remaining) {
 820                 clone = crypt_clone(cc, io, bio, sector, &bvec_idx, &ctx);
 821                 if (!clone)
 822                         goto cleanup;
 823
 824                 if (!io->first_clone) {
 825                         /*
 826                          * hold a reference to the first clone, because it
 827                          * holds the bio_vec array and that can't be freed
 828                          * before all other clones are released
 829                          */
 830                         bio_get(clone);
 831                         io->first_clone = clone;
 832                 }
 833                 atomic_inc(&io->pending);
 834
 835                 remaining -= clone->bi_size;
 836                 sector += bio_sectors(clone);
 837
 838                 generic_make_request(clone);
 839
 840                 /* out of memory -> run queues */
 841                 if (remaining)
 842                         blk_congestion_wait(bio_data_dir(clone), HZ/100);
 843         }
 844
 845         /* drop reference, clones could have returned before we reach this */
 846         dec_pending(io, 0);
 847         return 0;
 848
 849 cleanup:
 850         if (io->first_clone) {
 851                 dec_pending(io, -ENOMEM);
 852                 return 0;
 853         }
 854
 855         /* if no bio has been dispatched yet, we can directly return the error */
 856         mempool_free(io, cc->io_pool);
 857         return -ENOMEM;
 858 }
 859
 860 static int crypt_status(struct dm_target *ti, status_type_t type,
 861                         char *result, unsigned int maxlen)
 862 {
 863         struct crypt_config *cc = (struct crypt_config *) ti->private;
 864         const char *cipher;
 865         const char *chainmode = NULL;
 866         unsigned int sz = 0;
 867
 868         switch (type) {
 869         case STATUSTYPE_INFO:
 870                 result[0] = '\0';
 871                 break;
 872
 873         case STATUSTYPE_TABLE:
 874                 cipher = crypto_tfm_alg_name(cc->tfm);
 875
 876                 switch(cc->tfm->crt_cipher.cit_mode) {
 877                 case CRYPTO_TFM_MODE_CBC:
 878                         chainmode = "cbc";
 879                         break;
 880                 case CRYPTO_TFM_MODE_ECB:
 881                         chainmode = "ecb";
 882                         break;
 883                 default:
 884                         BUG();
 885                 }
 886
 887                 if (cc->iv_mode)
 888                         DMEMIT("%s-%s-%s ", cipher, chainmode, cc->iv_mode);
 889                 else
 890                         DMEMIT("%s-%s ", cipher, chainmode);
 891
 892                 if (cc->key_size > 0) {
 893                         if ((maxlen - sz) < ((cc->key_size << 1) + 1))
 894                                 return -ENOMEM;
 895
 896                         crypt_encode_key(result + sz, cc->key, cc->key_size);
 897                         sz += cc->key_size << 1;
 898                 } else {
 899                         if (sz >= maxlen)
 900                                 return -ENOMEM;
 901                         result[sz++] = '-';
 902                 }
 903
 904                 DMEMIT(" " SECTOR_FORMAT " %s " SECTOR_FORMAT,
 905                        cc->iv_offset, cc->dev->name, cc->start);
 906                 break;
 907         }
 908         return 0;
 909 }
 910
 911 static struct target_type crypt_target = {
 912         .name   = "crypt",
 913         .version= {1, 1, 0},
 914         .module = THIS_MODULE,
 915         .ctr    = crypt_ctr,
 916         .dtr    = crypt_dtr,
 917         .map    = crypt_map,
 918         .status = crypt_status,
 919 };
 920
 921 static int __init dm_crypt_init(void)
 922 {
 923         int r;
 924
 925         _crypt_io_pool = kmem_cache_create("dm-crypt_io",
 926                                            sizeof(struct crypt_io),
 927                                            0, 0, NULL, NULL);
 928         if (!_crypt_io_pool)
 929                 return -ENOMEM;
 930
 931         _kcryptd_workqueue = create_workqueue("kcryptd");
 932         if (!_kcryptd_workqueue) {
 933                 r = -ENOMEM;
 934                 DMERR(PFX "couldn't create kcryptd");
 935                 goto bad1;
 936         }
 937
 938         r = dm_register_target(&crypt_target);
 939         if (r < 0) {
 940                 DMERR(PFX "register failed %d", r);
 941                 goto bad2;
 942         }
 943
 944         return 0;
 945
 946 bad2:
 947         destroy_workqueue(_kcryptd_workqueue);
 948 bad1:
 949         kmem_cache_destroy(_crypt_io_pool);
 950         return r;
 951 }
 952
 953 static void __exit dm_crypt_exit(void)
 954 {
 955         int r = dm_unregister_target(&crypt_target);
 956
 957         if (r < 0)
 958                 DMERR(PFX "unregister failed %d", r);
 959
 960         destroy_workqueue(_kcryptd_workqueue);
 961         kmem_cache_destroy(_crypt_io_pool);
 962 }
 963
 964 module_init(dm_crypt_init);
 965 module_exit(dm_crypt_exit);
 966
 967 MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
 968 MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
 969 MODULE_LICENSE("GPL");