2 * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
3 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
4 * Copyright (C) 2006-2009 Red Hat, Inc. All rights reserved.
6 * This file is released under the GPL.
9 #include <linux/completion.h>
10 #include <linux/err.h>
11 #include <linux/module.h>
12 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/bio.h>
15 #include <linux/blkdev.h>
16 #include <linux/mempool.h>
17 #include <linux/slab.h>
18 #include <linux/crypto.h>
19 #include <linux/workqueue.h>
20 #include <linux/backing-dev.h>
21 #include <linux/percpu.h>
22 #include <asm/atomic.h>
23 #include <linux/scatterlist.h>
25 #include <asm/unaligned.h>
27 #include <linux/device-mapper.h>
29 #define DM_MSG_PREFIX "crypt"
30 #define MESG_STR(x) x, sizeof(x)
33 * context holding the current state of a multi-part conversion
35 struct convert_context
{
36 struct completion restart
;
39 unsigned int offset_in
;
40 unsigned int offset_out
;
48 * per bio private data
51 struct dm_target
*target
;
53 struct work_struct work
;
55 struct convert_context ctx
;
60 struct dm_crypt_io
*base_io
;
63 struct dm_crypt_request
{
64 struct convert_context
*ctx
;
65 struct scatterlist sg_in
;
66 struct scatterlist sg_out
;
72 struct crypt_iv_operations
{
73 int (*ctr
)(struct crypt_config
*cc
, struct dm_target
*ti
,
75 void (*dtr
)(struct crypt_config
*cc
);
76 int (*init
)(struct crypt_config
*cc
);
77 int (*wipe
)(struct crypt_config
*cc
);
78 int (*generator
)(struct crypt_config
*cc
, u8
*iv
,
79 struct dm_crypt_request
*dmreq
);
80 int (*post
)(struct crypt_config
*cc
, u8
*iv
,
81 struct dm_crypt_request
*dmreq
);
84 struct iv_essiv_private
{
85 struct crypto_hash
*hash_tfm
;
89 struct iv_benbi_private
{
94 * Crypt: maps a linear range of a block device
95 * and encrypts / decrypts at the same time.
97 enum flags
{ DM_CRYPT_SUSPENDED
, DM_CRYPT_KEY_VALID
};
100 * Duplicated per-CPU state for cipher.
103 struct ablkcipher_request
*req
;
104 struct crypto_ablkcipher
*tfm
;
106 /* ESSIV: struct crypto_cipher *essiv_tfm */
111 * The fields in here must be read only after initialization,
112 * changing state should be in crypt_cpu.
114 struct crypt_config
{
119 * pool for per bio private data, crypto requests and
120 * encryption requeusts/buffer pages
124 mempool_t
*page_pool
;
127 struct workqueue_struct
*io_queue
;
128 struct workqueue_struct
*crypt_queue
;
133 struct crypt_iv_operations
*iv_gen_ops
;
135 struct iv_essiv_private essiv
;
136 struct iv_benbi_private benbi
;
139 unsigned int iv_size
;
142 * Duplicated per cpu state. Access through
143 * per_cpu_ptr() only.
145 struct crypt_cpu __percpu
*cpu
;
148 * Layout of each crypto request:
150 * struct ablkcipher_request
153 * struct dm_crypt_request
157 * The padding is added so that dm_crypt_request and the IV are
160 unsigned int dmreq_start
;
163 unsigned int key_size
;
168 #define MIN_POOL_PAGES 32
169 #define MIN_BIO_PAGES 8
171 static struct kmem_cache
*_crypt_io_pool
;
173 static void clone_init(struct dm_crypt_io
*, struct bio
*);
174 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
);
175 static u8
*iv_of_dmreq(struct crypt_config
*cc
, struct dm_crypt_request
*dmreq
);
177 static struct crypt_cpu
*this_crypt_config(struct crypt_config
*cc
)
179 return this_cpu_ptr(cc
->cpu
);
183 * Use this to access cipher attributes that are the same for each CPU.
185 static struct crypto_ablkcipher
*any_tfm(struct crypt_config
*cc
)
187 return __this_cpu_ptr(cc
->cpu
)->tfm
;
191 * Different IV generation algorithms:
193 * plain: the initial vector is the 32-bit little-endian version of the sector
194 * number, padded with zeros if necessary.
196 * plain64: the initial vector is the 64-bit little-endian version of the sector
197 * number, padded with zeros if necessary.
199 * essiv: "encrypted sector|salt initial vector", the sector number is
200 * encrypted with the bulk cipher using a salt as key. The salt
201 * should be derived from the bulk cipher's key via hashing.
203 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
204 * (needed for LRW-32-AES and possible other narrow block modes)
206 * null: the initial vector is always zero. Provides compatibility with
207 * obsolete loop_fish2 devices. Do not use for new devices.
209 * plumb: unimplemented, see:
210 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
213 static int crypt_iv_plain_gen(struct crypt_config
*cc
, u8
*iv
,
214 struct dm_crypt_request
*dmreq
)
216 memset(iv
, 0, cc
->iv_size
);
217 *(u32
*)iv
= cpu_to_le32(dmreq
->iv_sector
& 0xffffffff);
222 static int crypt_iv_plain64_gen(struct crypt_config
*cc
, u8
*iv
,
223 struct dm_crypt_request
*dmreq
)
225 memset(iv
, 0, cc
->iv_size
);
226 *(u64
*)iv
= cpu_to_le64(dmreq
->iv_sector
);
231 /* Initialise ESSIV - compute salt but no local memory allocations */
232 static int crypt_iv_essiv_init(struct crypt_config
*cc
)
234 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
235 struct hash_desc desc
;
236 struct scatterlist sg
;
237 struct crypto_cipher
*essiv_tfm
;
240 sg_init_one(&sg
, cc
->key
, cc
->key_size
);
241 desc
.tfm
= essiv
->hash_tfm
;
242 desc
.flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
244 err
= crypto_hash_digest(&desc
, &sg
, cc
->key_size
, essiv
->salt
);
248 for_each_possible_cpu(cpu
) {
249 essiv_tfm
= per_cpu_ptr(cc
->cpu
, cpu
)->iv_private
,
251 err
= crypto_cipher_setkey(essiv_tfm
, essiv
->salt
,
252 crypto_hash_digestsize(essiv
->hash_tfm
));
260 /* Wipe salt and reset key derived from volume key */
261 static int crypt_iv_essiv_wipe(struct crypt_config
*cc
)
263 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
264 unsigned salt_size
= crypto_hash_digestsize(essiv
->hash_tfm
);
265 struct crypto_cipher
*essiv_tfm
;
268 memset(essiv
->salt
, 0, salt_size
);
270 for_each_possible_cpu(cpu
) {
271 essiv_tfm
= per_cpu_ptr(cc
->cpu
, cpu
)->iv_private
;
272 r
= crypto_cipher_setkey(essiv_tfm
, essiv
->salt
, salt_size
);
280 /* Set up per cpu cipher state */
281 static struct crypto_cipher
*setup_essiv_cpu(struct crypt_config
*cc
,
282 struct dm_target
*ti
,
283 u8
*salt
, unsigned saltsize
)
285 struct crypto_cipher
*essiv_tfm
;
288 /* Setup the essiv_tfm with the given salt */
289 essiv_tfm
= crypto_alloc_cipher(cc
->cipher
, 0, CRYPTO_ALG_ASYNC
);
290 if (IS_ERR(essiv_tfm
)) {
291 ti
->error
= "Error allocating crypto tfm for ESSIV";
295 if (crypto_cipher_blocksize(essiv_tfm
) !=
296 crypto_ablkcipher_ivsize(any_tfm(cc
))) {
297 ti
->error
= "Block size of ESSIV cipher does "
298 "not match IV size of block cipher";
299 crypto_free_cipher(essiv_tfm
);
300 return ERR_PTR(-EINVAL
);
303 err
= crypto_cipher_setkey(essiv_tfm
, salt
, saltsize
);
305 ti
->error
= "Failed to set key for ESSIV cipher";
306 crypto_free_cipher(essiv_tfm
);
313 static void crypt_iv_essiv_dtr(struct crypt_config
*cc
)
316 struct crypt_cpu
*cpu_cc
;
317 struct crypto_cipher
*essiv_tfm
;
318 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
320 crypto_free_hash(essiv
->hash_tfm
);
321 essiv
->hash_tfm
= NULL
;
326 for_each_possible_cpu(cpu
) {
327 cpu_cc
= per_cpu_ptr(cc
->cpu
, cpu
);
328 essiv_tfm
= cpu_cc
->iv_private
;
331 crypto_free_cipher(essiv_tfm
);
333 cpu_cc
->iv_private
= NULL
;
337 static int crypt_iv_essiv_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
340 struct crypto_cipher
*essiv_tfm
= NULL
;
341 struct crypto_hash
*hash_tfm
= NULL
;
346 ti
->error
= "Digest algorithm missing for ESSIV mode";
350 /* Allocate hash algorithm */
351 hash_tfm
= crypto_alloc_hash(opts
, 0, CRYPTO_ALG_ASYNC
);
352 if (IS_ERR(hash_tfm
)) {
353 ti
->error
= "Error initializing ESSIV hash";
354 err
= PTR_ERR(hash_tfm
);
358 salt
= kzalloc(crypto_hash_digestsize(hash_tfm
), GFP_KERNEL
);
360 ti
->error
= "Error kmallocing salt storage in ESSIV";
365 cc
->iv_gen_private
.essiv
.salt
= salt
;
366 cc
->iv_gen_private
.essiv
.hash_tfm
= hash_tfm
;
368 for_each_possible_cpu(cpu
) {
369 essiv_tfm
= setup_essiv_cpu(cc
, ti
, salt
,
370 crypto_hash_digestsize(hash_tfm
));
371 if (IS_ERR(essiv_tfm
)) {
372 crypt_iv_essiv_dtr(cc
);
373 return PTR_ERR(essiv_tfm
);
375 per_cpu_ptr(cc
->cpu
, cpu
)->iv_private
= essiv_tfm
;
381 if (hash_tfm
&& !IS_ERR(hash_tfm
))
382 crypto_free_hash(hash_tfm
);
387 static int crypt_iv_essiv_gen(struct crypt_config
*cc
, u8
*iv
,
388 struct dm_crypt_request
*dmreq
)
390 struct crypto_cipher
*essiv_tfm
= this_crypt_config(cc
)->iv_private
;
392 memset(iv
, 0, cc
->iv_size
);
393 *(u64
*)iv
= cpu_to_le64(dmreq
->iv_sector
);
394 crypto_cipher_encrypt_one(essiv_tfm
, iv
, iv
);
399 static int crypt_iv_benbi_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
402 unsigned bs
= crypto_ablkcipher_blocksize(any_tfm(cc
));
405 /* we need to calculate how far we must shift the sector count
406 * to get the cipher block count, we use this shift in _gen */
408 if (1 << log
!= bs
) {
409 ti
->error
= "cypher blocksize is not a power of 2";
414 ti
->error
= "cypher blocksize is > 512";
418 cc
->iv_gen_private
.benbi
.shift
= 9 - log
;
423 static void crypt_iv_benbi_dtr(struct crypt_config
*cc
)
427 static int crypt_iv_benbi_gen(struct crypt_config
*cc
, u8
*iv
,
428 struct dm_crypt_request
*dmreq
)
432 memset(iv
, 0, cc
->iv_size
- sizeof(u64
)); /* rest is cleared below */
434 val
= cpu_to_be64(((u64
)dmreq
->iv_sector
<< cc
->iv_gen_private
.benbi
.shift
) + 1);
435 put_unaligned(val
, (__be64
*)(iv
+ cc
->iv_size
- sizeof(u64
)));
440 static int crypt_iv_null_gen(struct crypt_config
*cc
, u8
*iv
,
441 struct dm_crypt_request
*dmreq
)
443 memset(iv
, 0, cc
->iv_size
);
448 static struct crypt_iv_operations crypt_iv_plain_ops
= {
449 .generator
= crypt_iv_plain_gen
452 static struct crypt_iv_operations crypt_iv_plain64_ops
= {
453 .generator
= crypt_iv_plain64_gen
456 static struct crypt_iv_operations crypt_iv_essiv_ops
= {
457 .ctr
= crypt_iv_essiv_ctr
,
458 .dtr
= crypt_iv_essiv_dtr
,
459 .init
= crypt_iv_essiv_init
,
460 .wipe
= crypt_iv_essiv_wipe
,
461 .generator
= crypt_iv_essiv_gen
464 static struct crypt_iv_operations crypt_iv_benbi_ops
= {
465 .ctr
= crypt_iv_benbi_ctr
,
466 .dtr
= crypt_iv_benbi_dtr
,
467 .generator
= crypt_iv_benbi_gen
470 static struct crypt_iv_operations crypt_iv_null_ops
= {
471 .generator
= crypt_iv_null_gen
474 static void crypt_convert_init(struct crypt_config
*cc
,
475 struct convert_context
*ctx
,
476 struct bio
*bio_out
, struct bio
*bio_in
,
479 ctx
->bio_in
= bio_in
;
480 ctx
->bio_out
= bio_out
;
483 ctx
->idx_in
= bio_in
? bio_in
->bi_idx
: 0;
484 ctx
->idx_out
= bio_out
? bio_out
->bi_idx
: 0;
485 ctx
->sector
= sector
+ cc
->iv_offset
;
486 init_completion(&ctx
->restart
);
489 static struct dm_crypt_request
*dmreq_of_req(struct crypt_config
*cc
,
490 struct ablkcipher_request
*req
)
492 return (struct dm_crypt_request
*)((char *)req
+ cc
->dmreq_start
);
495 static struct ablkcipher_request
*req_of_dmreq(struct crypt_config
*cc
,
496 struct dm_crypt_request
*dmreq
)
498 return (struct ablkcipher_request
*)((char *)dmreq
- cc
->dmreq_start
);
501 static u8
*iv_of_dmreq(struct crypt_config
*cc
,
502 struct dm_crypt_request
*dmreq
)
504 return (u8
*)ALIGN((unsigned long)(dmreq
+ 1),
505 crypto_ablkcipher_alignmask(any_tfm(cc
)) + 1);
508 static int crypt_convert_block(struct crypt_config
*cc
,
509 struct convert_context
*ctx
,
510 struct ablkcipher_request
*req
)
512 struct bio_vec
*bv_in
= bio_iovec_idx(ctx
->bio_in
, ctx
->idx_in
);
513 struct bio_vec
*bv_out
= bio_iovec_idx(ctx
->bio_out
, ctx
->idx_out
);
514 struct dm_crypt_request
*dmreq
;
518 dmreq
= dmreq_of_req(cc
, req
);
519 iv
= iv_of_dmreq(cc
, dmreq
);
521 dmreq
->iv_sector
= ctx
->sector
;
523 sg_init_table(&dmreq
->sg_in
, 1);
524 sg_set_page(&dmreq
->sg_in
, bv_in
->bv_page
, 1 << SECTOR_SHIFT
,
525 bv_in
->bv_offset
+ ctx
->offset_in
);
527 sg_init_table(&dmreq
->sg_out
, 1);
528 sg_set_page(&dmreq
->sg_out
, bv_out
->bv_page
, 1 << SECTOR_SHIFT
,
529 bv_out
->bv_offset
+ ctx
->offset_out
);
531 ctx
->offset_in
+= 1 << SECTOR_SHIFT
;
532 if (ctx
->offset_in
>= bv_in
->bv_len
) {
537 ctx
->offset_out
+= 1 << SECTOR_SHIFT
;
538 if (ctx
->offset_out
>= bv_out
->bv_len
) {
543 if (cc
->iv_gen_ops
) {
544 r
= cc
->iv_gen_ops
->generator(cc
, iv
, dmreq
);
549 ablkcipher_request_set_crypt(req
, &dmreq
->sg_in
, &dmreq
->sg_out
,
550 1 << SECTOR_SHIFT
, iv
);
552 if (bio_data_dir(ctx
->bio_in
) == WRITE
)
553 r
= crypto_ablkcipher_encrypt(req
);
555 r
= crypto_ablkcipher_decrypt(req
);
557 if (!r
&& cc
->iv_gen_ops
&& cc
->iv_gen_ops
->post
)
558 r
= cc
->iv_gen_ops
->post(cc
, iv
, dmreq
);
563 static void kcryptd_async_done(struct crypto_async_request
*async_req
,
566 static void crypt_alloc_req(struct crypt_config
*cc
,
567 struct convert_context
*ctx
)
569 struct crypt_cpu
*this_cc
= this_crypt_config(cc
);
572 this_cc
->req
= mempool_alloc(cc
->req_pool
, GFP_NOIO
);
574 ablkcipher_request_set_tfm(this_cc
->req
, this_cc
->tfm
);
575 ablkcipher_request_set_callback(this_cc
->req
,
576 CRYPTO_TFM_REQ_MAY_BACKLOG
| CRYPTO_TFM_REQ_MAY_SLEEP
,
577 kcryptd_async_done
, dmreq_of_req(cc
, this_cc
->req
));
581 * Encrypt / decrypt data from one bio to another one (can be the same one)
583 static int crypt_convert(struct crypt_config
*cc
,
584 struct convert_context
*ctx
)
586 struct crypt_cpu
*this_cc
= this_crypt_config(cc
);
589 atomic_set(&ctx
->pending
, 1);
591 while(ctx
->idx_in
< ctx
->bio_in
->bi_vcnt
&&
592 ctx
->idx_out
< ctx
->bio_out
->bi_vcnt
) {
594 crypt_alloc_req(cc
, ctx
);
596 atomic_inc(&ctx
->pending
);
598 r
= crypt_convert_block(cc
, ctx
, this_cc
->req
);
603 wait_for_completion(&ctx
->restart
);
604 INIT_COMPLETION(ctx
->restart
);
613 atomic_dec(&ctx
->pending
);
620 atomic_dec(&ctx
->pending
);
628 static void dm_crypt_bio_destructor(struct bio
*bio
)
630 struct dm_crypt_io
*io
= bio
->bi_private
;
631 struct crypt_config
*cc
= io
->target
->private;
633 bio_free(bio
, cc
->bs
);
637 * Generate a new unfragmented bio with the given size
638 * This should never violate the device limitations
639 * May return a smaller bio when running out of pages, indicated by
640 * *out_of_pages set to 1.
642 static struct bio
*crypt_alloc_buffer(struct dm_crypt_io
*io
, unsigned size
,
643 unsigned *out_of_pages
)
645 struct crypt_config
*cc
= io
->target
->private;
647 unsigned int nr_iovecs
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
648 gfp_t gfp_mask
= GFP_NOIO
| __GFP_HIGHMEM
;
652 clone
= bio_alloc_bioset(GFP_NOIO
, nr_iovecs
, cc
->bs
);
656 clone_init(io
, clone
);
659 for (i
= 0; i
< nr_iovecs
; i
++) {
660 page
= mempool_alloc(cc
->page_pool
, gfp_mask
);
667 * if additional pages cannot be allocated without waiting,
668 * return a partially allocated bio, the caller will then try
669 * to allocate additional bios while submitting this partial bio
671 if (i
== (MIN_BIO_PAGES
- 1))
672 gfp_mask
= (gfp_mask
| __GFP_NOWARN
) & ~__GFP_WAIT
;
674 len
= (size
> PAGE_SIZE
) ? PAGE_SIZE
: size
;
676 if (!bio_add_page(clone
, page
, len
, 0)) {
677 mempool_free(page
, cc
->page_pool
);
684 if (!clone
->bi_size
) {
692 static void crypt_free_buffer_pages(struct crypt_config
*cc
, struct bio
*clone
)
697 for (i
= 0; i
< clone
->bi_vcnt
; i
++) {
698 bv
= bio_iovec_idx(clone
, i
);
699 BUG_ON(!bv
->bv_page
);
700 mempool_free(bv
->bv_page
, cc
->page_pool
);
705 static struct dm_crypt_io
*crypt_io_alloc(struct dm_target
*ti
,
706 struct bio
*bio
, sector_t sector
)
708 struct crypt_config
*cc
= ti
->private;
709 struct dm_crypt_io
*io
;
711 io
= mempool_alloc(cc
->io_pool
, GFP_NOIO
);
717 atomic_set(&io
->pending
, 0);
722 static void crypt_inc_pending(struct dm_crypt_io
*io
)
724 atomic_inc(&io
->pending
);
728 * One of the bios was finished. Check for completion of
729 * the whole request and correctly clean up the buffer.
730 * If base_io is set, wait for the last fragment to complete.
732 static void crypt_dec_pending(struct dm_crypt_io
*io
)
734 struct crypt_config
*cc
= io
->target
->private;
735 struct bio
*base_bio
= io
->base_bio
;
736 struct dm_crypt_io
*base_io
= io
->base_io
;
737 int error
= io
->error
;
739 if (!atomic_dec_and_test(&io
->pending
))
742 mempool_free(io
, cc
->io_pool
);
744 if (likely(!base_io
))
745 bio_endio(base_bio
, error
);
747 if (error
&& !base_io
->error
)
748 base_io
->error
= error
;
749 crypt_dec_pending(base_io
);
754 * kcryptd/kcryptd_io:
756 * Needed because it would be very unwise to do decryption in an
759 * kcryptd performs the actual encryption or decryption.
761 * kcryptd_io performs the IO submission.
763 * They must be separated as otherwise the final stages could be
764 * starved by new requests which can block in the first stages due
765 * to memory allocation.
767 * The work is done per CPU global for all dm-crypt instances.
768 * They should not depend on each other and do not block.
770 static void crypt_endio(struct bio
*clone
, int error
)
772 struct dm_crypt_io
*io
= clone
->bi_private
;
773 struct crypt_config
*cc
= io
->target
->private;
774 unsigned rw
= bio_data_dir(clone
);
776 if (unlikely(!bio_flagged(clone
, BIO_UPTODATE
) && !error
))
780 * free the processed pages
783 crypt_free_buffer_pages(cc
, clone
);
787 if (rw
== READ
&& !error
) {
788 kcryptd_queue_crypt(io
);
795 crypt_dec_pending(io
);
798 static void clone_init(struct dm_crypt_io
*io
, struct bio
*clone
)
800 struct crypt_config
*cc
= io
->target
->private;
802 clone
->bi_private
= io
;
803 clone
->bi_end_io
= crypt_endio
;
804 clone
->bi_bdev
= cc
->dev
->bdev
;
805 clone
->bi_rw
= io
->base_bio
->bi_rw
;
806 clone
->bi_destructor
= dm_crypt_bio_destructor
;
809 static void kcryptd_unplug(struct crypt_config
*cc
)
811 blk_unplug(bdev_get_queue(cc
->dev
->bdev
));
814 static int kcryptd_io_read(struct dm_crypt_io
*io
, gfp_t gfp
)
816 struct crypt_config
*cc
= io
->target
->private;
817 struct bio
*base_bio
= io
->base_bio
;
821 * The block layer might modify the bvec array, so always
822 * copy the required bvecs because we need the original
823 * one in order to decrypt the whole bio data *afterwards*.
825 clone
= bio_alloc_bioset(gfp
, bio_segments(base_bio
), cc
->bs
);
831 crypt_inc_pending(io
);
833 clone_init(io
, clone
);
835 clone
->bi_vcnt
= bio_segments(base_bio
);
836 clone
->bi_size
= base_bio
->bi_size
;
837 clone
->bi_sector
= cc
->start
+ io
->sector
;
838 memcpy(clone
->bi_io_vec
, bio_iovec(base_bio
),
839 sizeof(struct bio_vec
) * clone
->bi_vcnt
);
841 generic_make_request(clone
);
845 static void kcryptd_io_write(struct dm_crypt_io
*io
)
847 struct bio
*clone
= io
->ctx
.bio_out
;
848 generic_make_request(clone
);
851 static void kcryptd_io(struct work_struct
*work
)
853 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
855 if (bio_data_dir(io
->base_bio
) == READ
) {
856 crypt_inc_pending(io
);
857 if (kcryptd_io_read(io
, GFP_NOIO
))
859 crypt_dec_pending(io
);
861 kcryptd_io_write(io
);
864 static void kcryptd_queue_io(struct dm_crypt_io
*io
)
866 struct crypt_config
*cc
= io
->target
->private;
868 INIT_WORK(&io
->work
, kcryptd_io
);
869 queue_work(cc
->io_queue
, &io
->work
);
872 static void kcryptd_crypt_write_io_submit(struct dm_crypt_io
*io
,
873 int error
, int async
)
875 struct bio
*clone
= io
->ctx
.bio_out
;
876 struct crypt_config
*cc
= io
->target
->private;
878 if (unlikely(error
< 0)) {
879 crypt_free_buffer_pages(cc
, clone
);
882 crypt_dec_pending(io
);
886 /* crypt_convert should have filled the clone bio */
887 BUG_ON(io
->ctx
.idx_out
< clone
->bi_vcnt
);
889 clone
->bi_sector
= cc
->start
+ io
->sector
;
892 kcryptd_queue_io(io
);
894 generic_make_request(clone
);
897 static void kcryptd_crypt_write_convert(struct dm_crypt_io
*io
)
899 struct crypt_config
*cc
= io
->target
->private;
901 struct dm_crypt_io
*new_io
;
903 unsigned out_of_pages
= 0;
904 unsigned remaining
= io
->base_bio
->bi_size
;
905 sector_t sector
= io
->sector
;
909 * Prevent io from disappearing until this function completes.
911 crypt_inc_pending(io
);
912 crypt_convert_init(cc
, &io
->ctx
, NULL
, io
->base_bio
, sector
);
915 * The allocated buffers can be smaller than the whole bio,
916 * so repeat the whole process until all the data can be handled.
919 clone
= crypt_alloc_buffer(io
, remaining
, &out_of_pages
);
920 if (unlikely(!clone
)) {
925 io
->ctx
.bio_out
= clone
;
928 remaining
-= clone
->bi_size
;
929 sector
+= bio_sectors(clone
);
931 crypt_inc_pending(io
);
932 r
= crypt_convert(cc
, &io
->ctx
);
933 crypt_finished
= atomic_dec_and_test(&io
->ctx
.pending
);
935 /* Encryption was already finished, submit io now */
936 if (crypt_finished
) {
937 kcryptd_crypt_write_io_submit(io
, r
, 0);
940 * If there was an error, do not try next fragments.
941 * For async, error is processed in async handler.
950 * Out of memory -> run queues
951 * But don't wait if split was due to the io size restriction
953 if (unlikely(out_of_pages
))
954 congestion_wait(BLK_RW_ASYNC
, HZ
/100);
957 * With async crypto it is unsafe to share the crypto context
958 * between fragments, so switch to a new dm_crypt_io structure.
960 if (unlikely(!crypt_finished
&& remaining
)) {
961 new_io
= crypt_io_alloc(io
->target
, io
->base_bio
,
963 crypt_inc_pending(new_io
);
964 crypt_convert_init(cc
, &new_io
->ctx
, NULL
,
965 io
->base_bio
, sector
);
966 new_io
->ctx
.idx_in
= io
->ctx
.idx_in
;
967 new_io
->ctx
.offset_in
= io
->ctx
.offset_in
;
970 * Fragments after the first use the base_io
974 new_io
->base_io
= io
;
976 new_io
->base_io
= io
->base_io
;
977 crypt_inc_pending(io
->base_io
);
978 crypt_dec_pending(io
);
985 crypt_dec_pending(io
);
988 static void kcryptd_crypt_read_done(struct dm_crypt_io
*io
, int error
)
990 if (unlikely(error
< 0))
993 crypt_dec_pending(io
);
996 static void kcryptd_crypt_read_convert(struct dm_crypt_io
*io
)
998 struct crypt_config
*cc
= io
->target
->private;
1001 crypt_inc_pending(io
);
1003 crypt_convert_init(cc
, &io
->ctx
, io
->base_bio
, io
->base_bio
,
1006 r
= crypt_convert(cc
, &io
->ctx
);
1008 if (atomic_dec_and_test(&io
->ctx
.pending
))
1009 kcryptd_crypt_read_done(io
, r
);
1011 crypt_dec_pending(io
);
1014 static void kcryptd_async_done(struct crypto_async_request
*async_req
,
1017 struct dm_crypt_request
*dmreq
= async_req
->data
;
1018 struct convert_context
*ctx
= dmreq
->ctx
;
1019 struct dm_crypt_io
*io
= container_of(ctx
, struct dm_crypt_io
, ctx
);
1020 struct crypt_config
*cc
= io
->target
->private;
1022 if (error
== -EINPROGRESS
) {
1023 complete(&ctx
->restart
);
1027 if (!error
&& cc
->iv_gen_ops
&& cc
->iv_gen_ops
->post
)
1028 error
= cc
->iv_gen_ops
->post(cc
, iv_of_dmreq(cc
, dmreq
), dmreq
);
1030 mempool_free(req_of_dmreq(cc
, dmreq
), cc
->req_pool
);
1032 if (!atomic_dec_and_test(&ctx
->pending
))
1035 if (bio_data_dir(io
->base_bio
) == READ
)
1036 kcryptd_crypt_read_done(io
, error
);
1038 kcryptd_crypt_write_io_submit(io
, error
, 1);
1041 static void kcryptd_crypt(struct work_struct
*work
)
1043 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
1045 if (bio_data_dir(io
->base_bio
) == READ
)
1046 kcryptd_crypt_read_convert(io
);
1048 kcryptd_crypt_write_convert(io
);
1051 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
)
1053 struct crypt_config
*cc
= io
->target
->private;
1055 INIT_WORK(&io
->work
, kcryptd_crypt
);
1056 queue_work(cc
->crypt_queue
, &io
->work
);
1060 * Decode key from its hex representation
1062 static int crypt_decode_key(u8
*key
, char *hex
, unsigned int size
)
1070 for (i
= 0; i
< size
; i
++) {
1074 key
[i
] = (u8
)simple_strtoul(buffer
, &endp
, 16);
1076 if (endp
!= &buffer
[2])
1087 * Encode key into its hex representation
1089 static void crypt_encode_key(char *hex
, u8
*key
, unsigned int size
)
1093 for (i
= 0; i
< size
; i
++) {
1094 sprintf(hex
, "%02x", *key
);
1100 static int crypt_setkey_allcpus(struct crypt_config
*cc
)
1102 int cpu
, err
= 0, r
;
1104 for_each_possible_cpu(cpu
) {
1105 r
= crypto_ablkcipher_setkey(per_cpu_ptr(cc
->cpu
, cpu
)->tfm
,
1106 cc
->key
, cc
->key_size
);
1114 static int crypt_set_key(struct crypt_config
*cc
, char *key
)
1116 /* The key size may not be changed. */
1117 if (cc
->key_size
!= (strlen(key
) >> 1))
1120 /* Hyphen (which gives a key_size of zero) means there is no key. */
1121 if (!cc
->key_size
&& strcmp(key
, "-"))
1124 if (cc
->key_size
&& crypt_decode_key(cc
->key
, key
, cc
->key_size
) < 0)
1127 set_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
1129 return crypt_setkey_allcpus(cc
);
1132 static int crypt_wipe_key(struct crypt_config
*cc
)
1134 clear_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
1135 memset(&cc
->key
, 0, cc
->key_size
* sizeof(u8
));
1137 return crypt_setkey_allcpus(cc
);
1140 static void crypt_dtr(struct dm_target
*ti
)
1142 struct crypt_config
*cc
= ti
->private;
1143 struct crypt_cpu
*cpu_cc
;
1152 destroy_workqueue(cc
->io_queue
);
1153 if (cc
->crypt_queue
)
1154 destroy_workqueue(cc
->crypt_queue
);
1157 for_each_possible_cpu(cpu
) {
1158 cpu_cc
= per_cpu_ptr(cc
->cpu
, cpu
);
1160 mempool_free(cpu_cc
->req
, cc
->req_pool
);
1162 crypto_free_ablkcipher(cpu_cc
->tfm
);
1166 bioset_free(cc
->bs
);
1169 mempool_destroy(cc
->page_pool
);
1171 mempool_destroy(cc
->req_pool
);
1173 mempool_destroy(cc
->io_pool
);
1175 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->dtr
)
1176 cc
->iv_gen_ops
->dtr(cc
);
1179 dm_put_device(ti
, cc
->dev
);
1182 free_percpu(cc
->cpu
);
1185 kzfree(cc
->cipher_string
);
1187 /* Must zero key material before freeing */
1191 static int crypt_ctr_cipher(struct dm_target
*ti
,
1192 char *cipher_in
, char *key
)
1194 struct crypt_config
*cc
= ti
->private;
1195 struct crypto_ablkcipher
*tfm
;
1196 char *tmp
, *cipher
, *chainmode
, *ivmode
, *ivopts
;
1197 char *cipher_api
= NULL
;
1198 int cpu
, ret
= -EINVAL
;
1200 /* Convert to crypto api definition? */
1201 if (strchr(cipher_in
, '(')) {
1202 ti
->error
= "Bad cipher specification";
1206 cc
->cipher_string
= kstrdup(cipher_in
, GFP_KERNEL
);
1207 if (!cc
->cipher_string
)
1211 * Legacy dm-crypt cipher specification
1212 * cipher-mode-iv:ivopts
1215 cipher
= strsep(&tmp
, "-");
1217 cc
->cipher
= kstrdup(cipher
, GFP_KERNEL
);
1221 chainmode
= strsep(&tmp
, "-");
1222 ivopts
= strsep(&tmp
, "-");
1223 ivmode
= strsep(&ivopts
, ":");
1226 DMWARN("Ignoring unexpected additional cipher options");
1228 cc
->cpu
= alloc_percpu(struct crypt_cpu
);
1230 ti
->error
= "Cannot allocate per cpu state";
1235 * For compatibility with the original dm-crypt mapping format, if
1236 * only the cipher name is supplied, use cbc-plain.
1238 if (!chainmode
|| (!strcmp(chainmode
, "plain") && !ivmode
)) {
1243 if (strcmp(chainmode
, "ecb") && !ivmode
) {
1244 ti
->error
= "IV mechanism required";
1248 cipher_api
= kmalloc(CRYPTO_MAX_ALG_NAME
, GFP_KERNEL
);
1252 ret
= snprintf(cipher_api
, CRYPTO_MAX_ALG_NAME
,
1253 "%s(%s)", chainmode
, cipher
);
1259 /* Allocate cipher */
1260 for_each_possible_cpu(cpu
) {
1261 tfm
= crypto_alloc_ablkcipher(cipher_api
, 0, 0);
1264 ti
->error
= "Error allocating crypto tfm";
1267 per_cpu_ptr(cc
->cpu
, cpu
)->tfm
= tfm
;
1270 /* Initialize and set key */
1271 ret
= crypt_set_key(cc
, key
);
1273 ti
->error
= "Error decoding and setting key";
1278 cc
->iv_size
= crypto_ablkcipher_ivsize(any_tfm(cc
));
1280 /* at least a 64 bit sector number should fit in our buffer */
1281 cc
->iv_size
= max(cc
->iv_size
,
1282 (unsigned int)(sizeof(u64
) / sizeof(u8
)));
1284 DMWARN("Selected cipher does not support IVs");
1288 /* Choose ivmode, see comments at iv code. */
1290 cc
->iv_gen_ops
= NULL
;
1291 else if (strcmp(ivmode
, "plain") == 0)
1292 cc
->iv_gen_ops
= &crypt_iv_plain_ops
;
1293 else if (strcmp(ivmode
, "plain64") == 0)
1294 cc
->iv_gen_ops
= &crypt_iv_plain64_ops
;
1295 else if (strcmp(ivmode
, "essiv") == 0)
1296 cc
->iv_gen_ops
= &crypt_iv_essiv_ops
;
1297 else if (strcmp(ivmode
, "benbi") == 0)
1298 cc
->iv_gen_ops
= &crypt_iv_benbi_ops
;
1299 else if (strcmp(ivmode
, "null") == 0)
1300 cc
->iv_gen_ops
= &crypt_iv_null_ops
;
1303 ti
->error
= "Invalid IV mode";
1308 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->ctr
) {
1309 ret
= cc
->iv_gen_ops
->ctr(cc
, ti
, ivopts
);
1311 ti
->error
= "Error creating IV";
1316 /* Initialize IV (set keys for ESSIV etc) */
1317 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->init
) {
1318 ret
= cc
->iv_gen_ops
->init(cc
);
1320 ti
->error
= "Error initialising IV";
1331 ti
->error
= "Cannot allocate cipher strings";
1336 * Construct an encryption mapping:
1337 * <cipher> <key> <iv_offset> <dev_path> <start>
1339 static int crypt_ctr(struct dm_target
*ti
, unsigned int argc
, char **argv
)
1341 struct crypt_config
*cc
;
1342 unsigned int key_size
;
1343 unsigned long long tmpll
;
1347 ti
->error
= "Not enough arguments";
1351 key_size
= strlen(argv
[1]) >> 1;
1353 cc
= kzalloc(sizeof(*cc
) + key_size
* sizeof(u8
), GFP_KERNEL
);
1355 ti
->error
= "Cannot allocate encryption context";
1358 cc
->key_size
= key_size
;
1361 ret
= crypt_ctr_cipher(ti
, argv
[0], argv
[1]);
1366 cc
->io_pool
= mempool_create_slab_pool(MIN_IOS
, _crypt_io_pool
);
1368 ti
->error
= "Cannot allocate crypt io mempool";
1372 cc
->dmreq_start
= sizeof(struct ablkcipher_request
);
1373 cc
->dmreq_start
+= crypto_ablkcipher_reqsize(any_tfm(cc
));
1374 cc
->dmreq_start
= ALIGN(cc
->dmreq_start
, crypto_tfm_ctx_alignment());
1375 cc
->dmreq_start
+= crypto_ablkcipher_alignmask(any_tfm(cc
)) &
1376 ~(crypto_tfm_ctx_alignment() - 1);
1378 cc
->req_pool
= mempool_create_kmalloc_pool(MIN_IOS
, cc
->dmreq_start
+
1379 sizeof(struct dm_crypt_request
) + cc
->iv_size
);
1380 if (!cc
->req_pool
) {
1381 ti
->error
= "Cannot allocate crypt request mempool";
1385 cc
->page_pool
= mempool_create_page_pool(MIN_POOL_PAGES
, 0);
1386 if (!cc
->page_pool
) {
1387 ti
->error
= "Cannot allocate page mempool";
1391 cc
->bs
= bioset_create(MIN_IOS
, 0);
1393 ti
->error
= "Cannot allocate crypt bioset";
1398 if (sscanf(argv
[2], "%llu", &tmpll
) != 1) {
1399 ti
->error
= "Invalid iv_offset sector";
1402 cc
->iv_offset
= tmpll
;
1404 if (dm_get_device(ti
, argv
[3], dm_table_get_mode(ti
->table
), &cc
->dev
)) {
1405 ti
->error
= "Device lookup failed";
1409 if (sscanf(argv
[4], "%llu", &tmpll
) != 1) {
1410 ti
->error
= "Invalid device sector";
1416 cc
->io_queue
= alloc_workqueue("kcryptd_io",
1420 if (!cc
->io_queue
) {
1421 ti
->error
= "Couldn't create kcryptd io queue";
1425 cc
->crypt_queue
= alloc_workqueue("kcryptd",
1430 if (!cc
->crypt_queue
) {
1431 ti
->error
= "Couldn't create kcryptd queue";
1435 ti
->num_flush_requests
= 1;
1443 static int crypt_map(struct dm_target
*ti
, struct bio
*bio
,
1444 union map_info
*map_context
)
1446 struct dm_crypt_io
*io
;
1447 struct crypt_config
*cc
;
1449 if (bio
->bi_rw
& REQ_FLUSH
) {
1451 bio
->bi_bdev
= cc
->dev
->bdev
;
1452 return DM_MAPIO_REMAPPED
;
1455 io
= crypt_io_alloc(ti
, bio
, dm_target_offset(ti
, bio
->bi_sector
));
1457 if (bio_data_dir(io
->base_bio
) == READ
) {
1458 if (kcryptd_io_read(io
, GFP_NOWAIT
))
1459 kcryptd_queue_io(io
);
1461 kcryptd_queue_crypt(io
);
1463 return DM_MAPIO_SUBMITTED
;
1466 static int crypt_status(struct dm_target
*ti
, status_type_t type
,
1467 char *result
, unsigned int maxlen
)
1469 struct crypt_config
*cc
= ti
->private;
1470 unsigned int sz
= 0;
1473 case STATUSTYPE_INFO
:
1477 case STATUSTYPE_TABLE
:
1478 DMEMIT("%s ", cc
->cipher_string
);
1480 if (cc
->key_size
> 0) {
1481 if ((maxlen
- sz
) < ((cc
->key_size
<< 1) + 1))
1484 crypt_encode_key(result
+ sz
, cc
->key
, cc
->key_size
);
1485 sz
+= cc
->key_size
<< 1;
1492 DMEMIT(" %llu %s %llu", (unsigned long long)cc
->iv_offset
,
1493 cc
->dev
->name
, (unsigned long long)cc
->start
);
1499 static void crypt_postsuspend(struct dm_target
*ti
)
1501 struct crypt_config
*cc
= ti
->private;
1503 set_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
1506 static int crypt_preresume(struct dm_target
*ti
)
1508 struct crypt_config
*cc
= ti
->private;
1510 if (!test_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
)) {
1511 DMERR("aborting resume - crypt key is not set.");
1518 static void crypt_resume(struct dm_target
*ti
)
1520 struct crypt_config
*cc
= ti
->private;
1522 clear_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
1525 /* Message interface
1529 static int crypt_message(struct dm_target
*ti
, unsigned argc
, char **argv
)
1531 struct crypt_config
*cc
= ti
->private;
1537 if (!strnicmp(argv
[0], MESG_STR("key"))) {
1538 if (!test_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
)) {
1539 DMWARN("not suspended during key manipulation.");
1542 if (argc
== 3 && !strnicmp(argv
[1], MESG_STR("set"))) {
1543 ret
= crypt_set_key(cc
, argv
[2]);
1546 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->init
)
1547 ret
= cc
->iv_gen_ops
->init(cc
);
1550 if (argc
== 2 && !strnicmp(argv
[1], MESG_STR("wipe"))) {
1551 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->wipe
) {
1552 ret
= cc
->iv_gen_ops
->wipe(cc
);
1556 return crypt_wipe_key(cc
);
1561 DMWARN("unrecognised message received.");
1565 static int crypt_merge(struct dm_target
*ti
, struct bvec_merge_data
*bvm
,
1566 struct bio_vec
*biovec
, int max_size
)
1568 struct crypt_config
*cc
= ti
->private;
1569 struct request_queue
*q
= bdev_get_queue(cc
->dev
->bdev
);
1571 if (!q
->merge_bvec_fn
)
1574 bvm
->bi_bdev
= cc
->dev
->bdev
;
1575 bvm
->bi_sector
= cc
->start
+ dm_target_offset(ti
, bvm
->bi_sector
);
1577 return min(max_size
, q
->merge_bvec_fn(q
, bvm
, biovec
));
1580 static int crypt_iterate_devices(struct dm_target
*ti
,
1581 iterate_devices_callout_fn fn
, void *data
)
1583 struct crypt_config
*cc
= ti
->private;
1585 return fn(ti
, cc
->dev
, cc
->start
, ti
->len
, data
);
1588 static struct target_type crypt_target
= {
1590 .version
= {1, 9, 0},
1591 .module
= THIS_MODULE
,
1595 .status
= crypt_status
,
1596 .postsuspend
= crypt_postsuspend
,
1597 .preresume
= crypt_preresume
,
1598 .resume
= crypt_resume
,
1599 .message
= crypt_message
,
1600 .merge
= crypt_merge
,
1601 .iterate_devices
= crypt_iterate_devices
,
1604 static int __init
dm_crypt_init(void)
1608 _crypt_io_pool
= KMEM_CACHE(dm_crypt_io
, 0);
1609 if (!_crypt_io_pool
)
1612 r
= dm_register_target(&crypt_target
);
1614 DMERR("register failed %d", r
);
1615 kmem_cache_destroy(_crypt_io_pool
);
1621 static void __exit
dm_crypt_exit(void)
1623 dm_unregister_target(&crypt_target
);
1624 kmem_cache_destroy(_crypt_io_pool
);
1627 module_init(dm_crypt_init
);
1628 module_exit(dm_crypt_exit
);
1630 MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
1631 MODULE_DESCRIPTION(DM_NAME
" target for transparent encryption / decryption");
1632 MODULE_LICENSE("GPL");