2 * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
3 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
4 * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved.
6 * This file is released under the GPL.
9 #include <linux/completion.h>
10 #include <linux/err.h>
11 #include <linux/module.h>
12 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/bio.h>
15 #include <linux/blkdev.h>
16 #include <linux/mempool.h>
17 #include <linux/slab.h>
18 #include <linux/crypto.h>
19 #include <linux/workqueue.h>
20 #include <linux/backing-dev.h>
21 #include <asm/atomic.h>
22 #include <linux/scatterlist.h>
24 #include <asm/unaligned.h>
26 #include <linux/device-mapper.h>
28 #define DM_MSG_PREFIX "crypt"
29 #define MESG_STR(x) x, sizeof(x)
32 * context holding the current state of a multi-part conversion
34 struct convert_context
{
35 struct completion restart
;
38 unsigned int offset_in
;
39 unsigned int offset_out
;
47 * per bio private data
50 struct dm_target
*target
;
52 struct work_struct work
;
54 struct convert_context ctx
;
59 struct dm_crypt_io
*base_io
;
62 struct dm_crypt_request
{
63 struct convert_context
*ctx
;
64 struct scatterlist sg_in
;
65 struct scatterlist sg_out
;
70 struct crypt_iv_operations
{
71 int (*ctr
)(struct crypt_config
*cc
, struct dm_target
*ti
,
73 void (*dtr
)(struct crypt_config
*cc
);
74 int (*generator
)(struct crypt_config
*cc
, u8
*iv
, sector_t sector
);
77 struct iv_essiv_private
{
78 struct crypto_cipher
*tfm
;
81 struct iv_benbi_private
{
86 * Crypt: maps a linear range of a block device
87 * and encrypts / decrypts at the same time.
89 enum flags
{ DM_CRYPT_SUSPENDED
, DM_CRYPT_KEY_VALID
};
95 * pool for per bio private data, crypto requests and
96 * encryption requeusts/buffer pages
100 mempool_t
*page_pool
;
103 struct workqueue_struct
*io_queue
;
104 struct workqueue_struct
*crypt_queue
;
107 * crypto related data
109 struct crypt_iv_operations
*iv_gen_ops
;
112 struct iv_essiv_private essiv
;
113 struct iv_benbi_private benbi
;
116 unsigned int iv_size
;
119 * Layout of each crypto request:
121 * struct ablkcipher_request
124 * struct dm_crypt_request
128 * The padding is added so that dm_crypt_request and the IV are
131 unsigned int dmreq_start
;
132 struct ablkcipher_request
*req
;
134 char cipher
[CRYPTO_MAX_ALG_NAME
];
135 char chainmode
[CRYPTO_MAX_ALG_NAME
];
136 struct crypto_ablkcipher
*tfm
;
138 unsigned int key_size
;
143 #define MIN_POOL_PAGES 32
144 #define MIN_BIO_PAGES 8
146 static struct kmem_cache
*_crypt_io_pool
;
148 static void clone_init(struct dm_crypt_io
*, struct bio
*);
149 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
);
152 * Different IV generation algorithms:
154 * plain: the initial vector is the 32-bit little-endian version of the sector
155 * number, padded with zeros if necessary.
157 * essiv: "encrypted sector|salt initial vector", the sector number is
158 * encrypted with the bulk cipher using a salt as key. The salt
159 * should be derived from the bulk cipher's key via hashing.
161 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
162 * (needed for LRW-32-AES and possible other narrow block modes)
164 * null: the initial vector is always zero. Provides compatibility with
165 * obsolete loop_fish2 devices. Do not use for new devices.
167 * plumb: unimplemented, see:
168 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
171 static int crypt_iv_plain_gen(struct crypt_config
*cc
, u8
*iv
, sector_t sector
)
173 memset(iv
, 0, cc
->iv_size
);
174 *(u32
*)iv
= cpu_to_le32(sector
& 0xffffffff);
179 static void crypt_iv_essiv_dtr(struct crypt_config
*cc
)
181 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
183 crypto_free_cipher(essiv
->tfm
);
187 static int crypt_iv_essiv_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
190 struct crypto_cipher
*essiv_tfm
= NULL
;
191 struct crypto_hash
*hash_tfm
= NULL
;
192 struct hash_desc desc
;
193 struct scatterlist sg
;
194 unsigned int saltsize
;
199 ti
->error
= "Digest algorithm missing for ESSIV mode";
203 /* Hash the cipher key with the given hash algorithm */
204 hash_tfm
= crypto_alloc_hash(opts
, 0, CRYPTO_ALG_ASYNC
);
205 if (IS_ERR(hash_tfm
)) {
206 ti
->error
= "Error initializing ESSIV hash";
207 err
= PTR_ERR(hash_tfm
);
211 saltsize
= crypto_hash_digestsize(hash_tfm
);
212 salt
= kzalloc(saltsize
, GFP_KERNEL
);
214 ti
->error
= "Error kmallocing salt storage in ESSIV";
219 sg_init_one(&sg
, cc
->key
, cc
->key_size
);
221 desc
.flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
222 err
= crypto_hash_digest(&desc
, &sg
, cc
->key_size
, salt
);
223 crypto_free_hash(hash_tfm
);
227 ti
->error
= "Error calculating hash in ESSIV";
231 /* Setup the essiv_tfm with the given salt */
232 essiv_tfm
= crypto_alloc_cipher(cc
->cipher
, 0, CRYPTO_ALG_ASYNC
);
233 if (IS_ERR(essiv_tfm
)) {
234 ti
->error
= "Error allocating crypto tfm for ESSIV";
235 err
= PTR_ERR(essiv_tfm
);
238 if (crypto_cipher_blocksize(essiv_tfm
) !=
239 crypto_ablkcipher_ivsize(cc
->tfm
)) {
240 ti
->error
= "Block size of ESSIV cipher does "
241 "not match IV size of block cipher";
245 err
= crypto_cipher_setkey(essiv_tfm
, salt
, saltsize
);
247 ti
->error
= "Failed to set key for ESSIV cipher";
252 cc
->iv_gen_private
.essiv
.tfm
= essiv_tfm
;
256 if (essiv_tfm
&& !IS_ERR(essiv_tfm
))
257 crypto_free_cipher(essiv_tfm
);
258 if (hash_tfm
&& !IS_ERR(hash_tfm
))
259 crypto_free_hash(hash_tfm
);
264 static int crypt_iv_essiv_gen(struct crypt_config
*cc
, u8
*iv
, sector_t sector
)
266 memset(iv
, 0, cc
->iv_size
);
267 *(u64
*)iv
= cpu_to_le64(sector
);
268 crypto_cipher_encrypt_one(cc
->iv_gen_private
.essiv
.tfm
, iv
, iv
);
272 static int crypt_iv_benbi_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
275 unsigned bs
= crypto_ablkcipher_blocksize(cc
->tfm
);
278 /* we need to calculate how far we must shift the sector count
279 * to get the cipher block count, we use this shift in _gen */
281 if (1 << log
!= bs
) {
282 ti
->error
= "cypher blocksize is not a power of 2";
287 ti
->error
= "cypher blocksize is > 512";
291 cc
->iv_gen_private
.benbi
.shift
= 9 - log
;
296 static void crypt_iv_benbi_dtr(struct crypt_config
*cc
)
300 static int crypt_iv_benbi_gen(struct crypt_config
*cc
, u8
*iv
, sector_t sector
)
304 memset(iv
, 0, cc
->iv_size
- sizeof(u64
)); /* rest is cleared below */
306 val
= cpu_to_be64(((u64
)sector
<< cc
->iv_gen_private
.benbi
.shift
) + 1);
307 put_unaligned(val
, (__be64
*)(iv
+ cc
->iv_size
- sizeof(u64
)));
312 static int crypt_iv_null_gen(struct crypt_config
*cc
, u8
*iv
, sector_t sector
)
314 memset(iv
, 0, cc
->iv_size
);
319 static struct crypt_iv_operations crypt_iv_plain_ops
= {
320 .generator
= crypt_iv_plain_gen
323 static struct crypt_iv_operations crypt_iv_essiv_ops
= {
324 .ctr
= crypt_iv_essiv_ctr
,
325 .dtr
= crypt_iv_essiv_dtr
,
326 .generator
= crypt_iv_essiv_gen
329 static struct crypt_iv_operations crypt_iv_benbi_ops
= {
330 .ctr
= crypt_iv_benbi_ctr
,
331 .dtr
= crypt_iv_benbi_dtr
,
332 .generator
= crypt_iv_benbi_gen
335 static struct crypt_iv_operations crypt_iv_null_ops
= {
336 .generator
= crypt_iv_null_gen
339 static void crypt_convert_init(struct crypt_config
*cc
,
340 struct convert_context
*ctx
,
341 struct bio
*bio_out
, struct bio
*bio_in
,
344 ctx
->bio_in
= bio_in
;
345 ctx
->bio_out
= bio_out
;
348 ctx
->idx_in
= bio_in
? bio_in
->bi_idx
: 0;
349 ctx
->idx_out
= bio_out
? bio_out
->bi_idx
: 0;
350 ctx
->sector
= sector
+ cc
->iv_offset
;
351 init_completion(&ctx
->restart
);
354 static struct dm_crypt_request
*dmreq_of_req(struct crypt_config
*cc
,
355 struct ablkcipher_request
*req
)
357 return (struct dm_crypt_request
*)((char *)req
+ cc
->dmreq_start
);
360 static struct ablkcipher_request
*req_of_dmreq(struct crypt_config
*cc
,
361 struct dm_crypt_request
*dmreq
)
363 return (struct ablkcipher_request
*)((char *)dmreq
- cc
->dmreq_start
);
366 static int crypt_convert_block(struct crypt_config
*cc
,
367 struct convert_context
*ctx
,
368 struct ablkcipher_request
*req
)
370 struct bio_vec
*bv_in
= bio_iovec_idx(ctx
->bio_in
, ctx
->idx_in
);
371 struct bio_vec
*bv_out
= bio_iovec_idx(ctx
->bio_out
, ctx
->idx_out
);
372 struct dm_crypt_request
*dmreq
;
376 dmreq
= dmreq_of_req(cc
, req
);
377 iv
= (u8
*)ALIGN((unsigned long)(dmreq
+ 1),
378 crypto_ablkcipher_alignmask(cc
->tfm
) + 1);
381 sg_init_table(&dmreq
->sg_in
, 1);
382 sg_set_page(&dmreq
->sg_in
, bv_in
->bv_page
, 1 << SECTOR_SHIFT
,
383 bv_in
->bv_offset
+ ctx
->offset_in
);
385 sg_init_table(&dmreq
->sg_out
, 1);
386 sg_set_page(&dmreq
->sg_out
, bv_out
->bv_page
, 1 << SECTOR_SHIFT
,
387 bv_out
->bv_offset
+ ctx
->offset_out
);
389 ctx
->offset_in
+= 1 << SECTOR_SHIFT
;
390 if (ctx
->offset_in
>= bv_in
->bv_len
) {
395 ctx
->offset_out
+= 1 << SECTOR_SHIFT
;
396 if (ctx
->offset_out
>= bv_out
->bv_len
) {
401 if (cc
->iv_gen_ops
) {
402 r
= cc
->iv_gen_ops
->generator(cc
, iv
, ctx
->sector
);
407 ablkcipher_request_set_crypt(req
, &dmreq
->sg_in
, &dmreq
->sg_out
,
408 1 << SECTOR_SHIFT
, iv
);
410 if (bio_data_dir(ctx
->bio_in
) == WRITE
)
411 r
= crypto_ablkcipher_encrypt(req
);
413 r
= crypto_ablkcipher_decrypt(req
);
418 static void kcryptd_async_done(struct crypto_async_request
*async_req
,
420 static void crypt_alloc_req(struct crypt_config
*cc
,
421 struct convert_context
*ctx
)
424 cc
->req
= mempool_alloc(cc
->req_pool
, GFP_NOIO
);
425 ablkcipher_request_set_tfm(cc
->req
, cc
->tfm
);
426 ablkcipher_request_set_callback(cc
->req
, CRYPTO_TFM_REQ_MAY_BACKLOG
|
427 CRYPTO_TFM_REQ_MAY_SLEEP
,
429 dmreq_of_req(cc
, cc
->req
));
433 * Encrypt / decrypt data from one bio to another one (can be the same one)
435 static int crypt_convert(struct crypt_config
*cc
,
436 struct convert_context
*ctx
)
440 atomic_set(&ctx
->pending
, 1);
442 while(ctx
->idx_in
< ctx
->bio_in
->bi_vcnt
&&
443 ctx
->idx_out
< ctx
->bio_out
->bi_vcnt
) {
445 crypt_alloc_req(cc
, ctx
);
447 atomic_inc(&ctx
->pending
);
449 r
= crypt_convert_block(cc
, ctx
, cc
->req
);
454 wait_for_completion(&ctx
->restart
);
455 INIT_COMPLETION(ctx
->restart
);
464 atomic_dec(&ctx
->pending
);
471 atomic_dec(&ctx
->pending
);
479 static void dm_crypt_bio_destructor(struct bio
*bio
)
481 struct dm_crypt_io
*io
= bio
->bi_private
;
482 struct crypt_config
*cc
= io
->target
->private;
484 bio_free(bio
, cc
->bs
);
488 * Generate a new unfragmented bio with the given size
489 * This should never violate the device limitations
490 * May return a smaller bio when running out of pages, indicated by
491 * *out_of_pages set to 1.
493 static struct bio
*crypt_alloc_buffer(struct dm_crypt_io
*io
, unsigned size
,
494 unsigned *out_of_pages
)
496 struct crypt_config
*cc
= io
->target
->private;
498 unsigned int nr_iovecs
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
499 gfp_t gfp_mask
= GFP_NOIO
| __GFP_HIGHMEM
;
503 clone
= bio_alloc_bioset(GFP_NOIO
, nr_iovecs
, cc
->bs
);
507 clone_init(io
, clone
);
510 for (i
= 0; i
< nr_iovecs
; i
++) {
511 page
= mempool_alloc(cc
->page_pool
, gfp_mask
);
518 * if additional pages cannot be allocated without waiting,
519 * return a partially allocated bio, the caller will then try
520 * to allocate additional bios while submitting this partial bio
522 if (i
== (MIN_BIO_PAGES
- 1))
523 gfp_mask
= (gfp_mask
| __GFP_NOWARN
) & ~__GFP_WAIT
;
525 len
= (size
> PAGE_SIZE
) ? PAGE_SIZE
: size
;
527 if (!bio_add_page(clone
, page
, len
, 0)) {
528 mempool_free(page
, cc
->page_pool
);
535 if (!clone
->bi_size
) {
543 static void crypt_free_buffer_pages(struct crypt_config
*cc
, struct bio
*clone
)
548 for (i
= 0; i
< clone
->bi_vcnt
; i
++) {
549 bv
= bio_iovec_idx(clone
, i
);
550 BUG_ON(!bv
->bv_page
);
551 mempool_free(bv
->bv_page
, cc
->page_pool
);
556 static struct dm_crypt_io
*crypt_io_alloc(struct dm_target
*ti
,
557 struct bio
*bio
, sector_t sector
)
559 struct crypt_config
*cc
= ti
->private;
560 struct dm_crypt_io
*io
;
562 io
= mempool_alloc(cc
->io_pool
, GFP_NOIO
);
568 atomic_set(&io
->pending
, 0);
573 static void crypt_inc_pending(struct dm_crypt_io
*io
)
575 atomic_inc(&io
->pending
);
579 * One of the bios was finished. Check for completion of
580 * the whole request and correctly clean up the buffer.
581 * If base_io is set, wait for the last fragment to complete.
583 static void crypt_dec_pending(struct dm_crypt_io
*io
)
585 struct crypt_config
*cc
= io
->target
->private;
586 struct bio
*base_bio
= io
->base_bio
;
587 struct dm_crypt_io
*base_io
= io
->base_io
;
588 int error
= io
->error
;
590 if (!atomic_dec_and_test(&io
->pending
))
593 mempool_free(io
, cc
->io_pool
);
595 if (likely(!base_io
))
596 bio_endio(base_bio
, error
);
598 if (error
&& !base_io
->error
)
599 base_io
->error
= error
;
600 crypt_dec_pending(base_io
);
605 * kcryptd/kcryptd_io:
607 * Needed because it would be very unwise to do decryption in an
610 * kcryptd performs the actual encryption or decryption.
612 * kcryptd_io performs the IO submission.
614 * They must be separated as otherwise the final stages could be
615 * starved by new requests which can block in the first stages due
616 * to memory allocation.
618 static void crypt_endio(struct bio
*clone
, int error
)
620 struct dm_crypt_io
*io
= clone
->bi_private
;
621 struct crypt_config
*cc
= io
->target
->private;
622 unsigned rw
= bio_data_dir(clone
);
624 if (unlikely(!bio_flagged(clone
, BIO_UPTODATE
) && !error
))
628 * free the processed pages
631 crypt_free_buffer_pages(cc
, clone
);
635 if (rw
== READ
&& !error
) {
636 kcryptd_queue_crypt(io
);
643 crypt_dec_pending(io
);
646 static void clone_init(struct dm_crypt_io
*io
, struct bio
*clone
)
648 struct crypt_config
*cc
= io
->target
->private;
650 clone
->bi_private
= io
;
651 clone
->bi_end_io
= crypt_endio
;
652 clone
->bi_bdev
= cc
->dev
->bdev
;
653 clone
->bi_rw
= io
->base_bio
->bi_rw
;
654 clone
->bi_destructor
= dm_crypt_bio_destructor
;
657 static void kcryptd_io_read(struct dm_crypt_io
*io
)
659 struct crypt_config
*cc
= io
->target
->private;
660 struct bio
*base_bio
= io
->base_bio
;
663 crypt_inc_pending(io
);
666 * The block layer might modify the bvec array, so always
667 * copy the required bvecs because we need the original
668 * one in order to decrypt the whole bio data *afterwards*.
670 clone
= bio_alloc_bioset(GFP_NOIO
, bio_segments(base_bio
), cc
->bs
);
671 if (unlikely(!clone
)) {
673 crypt_dec_pending(io
);
677 clone_init(io
, clone
);
679 clone
->bi_vcnt
= bio_segments(base_bio
);
680 clone
->bi_size
= base_bio
->bi_size
;
681 clone
->bi_sector
= cc
->start
+ io
->sector
;
682 memcpy(clone
->bi_io_vec
, bio_iovec(base_bio
),
683 sizeof(struct bio_vec
) * clone
->bi_vcnt
);
685 generic_make_request(clone
);
688 static void kcryptd_io_write(struct dm_crypt_io
*io
)
690 struct bio
*clone
= io
->ctx
.bio_out
;
691 generic_make_request(clone
);
694 static void kcryptd_io(struct work_struct
*work
)
696 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
698 if (bio_data_dir(io
->base_bio
) == READ
)
701 kcryptd_io_write(io
);
704 static void kcryptd_queue_io(struct dm_crypt_io
*io
)
706 struct crypt_config
*cc
= io
->target
->private;
708 INIT_WORK(&io
->work
, kcryptd_io
);
709 queue_work(cc
->io_queue
, &io
->work
);
712 static void kcryptd_crypt_write_io_submit(struct dm_crypt_io
*io
,
713 int error
, int async
)
715 struct bio
*clone
= io
->ctx
.bio_out
;
716 struct crypt_config
*cc
= io
->target
->private;
718 if (unlikely(error
< 0)) {
719 crypt_free_buffer_pages(cc
, clone
);
722 crypt_dec_pending(io
);
726 /* crypt_convert should have filled the clone bio */
727 BUG_ON(io
->ctx
.idx_out
< clone
->bi_vcnt
);
729 clone
->bi_sector
= cc
->start
+ io
->sector
;
732 kcryptd_queue_io(io
);
734 generic_make_request(clone
);
737 static void kcryptd_crypt_write_convert(struct dm_crypt_io
*io
)
739 struct crypt_config
*cc
= io
->target
->private;
741 struct dm_crypt_io
*new_io
;
743 unsigned out_of_pages
= 0;
744 unsigned remaining
= io
->base_bio
->bi_size
;
745 sector_t sector
= io
->sector
;
749 * Prevent io from disappearing until this function completes.
751 crypt_inc_pending(io
);
752 crypt_convert_init(cc
, &io
->ctx
, NULL
, io
->base_bio
, sector
);
755 * The allocated buffers can be smaller than the whole bio,
756 * so repeat the whole process until all the data can be handled.
759 clone
= crypt_alloc_buffer(io
, remaining
, &out_of_pages
);
760 if (unlikely(!clone
)) {
765 io
->ctx
.bio_out
= clone
;
768 remaining
-= clone
->bi_size
;
769 sector
+= bio_sectors(clone
);
771 crypt_inc_pending(io
);
772 r
= crypt_convert(cc
, &io
->ctx
);
773 crypt_finished
= atomic_dec_and_test(&io
->ctx
.pending
);
775 /* Encryption was already finished, submit io now */
776 if (crypt_finished
) {
777 kcryptd_crypt_write_io_submit(io
, r
, 0);
780 * If there was an error, do not try next fragments.
781 * For async, error is processed in async handler.
790 * Out of memory -> run queues
791 * But don't wait if split was due to the io size restriction
793 if (unlikely(out_of_pages
))
794 congestion_wait(BLK_RW_ASYNC
, HZ
/100);
797 * With async crypto it is unsafe to share the crypto context
798 * between fragments, so switch to a new dm_crypt_io structure.
800 if (unlikely(!crypt_finished
&& remaining
)) {
801 new_io
= crypt_io_alloc(io
->target
, io
->base_bio
,
803 crypt_inc_pending(new_io
);
804 crypt_convert_init(cc
, &new_io
->ctx
, NULL
,
805 io
->base_bio
, sector
);
806 new_io
->ctx
.idx_in
= io
->ctx
.idx_in
;
807 new_io
->ctx
.offset_in
= io
->ctx
.offset_in
;
810 * Fragments after the first use the base_io
814 new_io
->base_io
= io
;
816 new_io
->base_io
= io
->base_io
;
817 crypt_inc_pending(io
->base_io
);
818 crypt_dec_pending(io
);
825 crypt_dec_pending(io
);
828 static void kcryptd_crypt_read_done(struct dm_crypt_io
*io
, int error
)
830 if (unlikely(error
< 0))
833 crypt_dec_pending(io
);
836 static void kcryptd_crypt_read_convert(struct dm_crypt_io
*io
)
838 struct crypt_config
*cc
= io
->target
->private;
841 crypt_inc_pending(io
);
843 crypt_convert_init(cc
, &io
->ctx
, io
->base_bio
, io
->base_bio
,
846 r
= crypt_convert(cc
, &io
->ctx
);
848 if (atomic_dec_and_test(&io
->ctx
.pending
))
849 kcryptd_crypt_read_done(io
, r
);
851 crypt_dec_pending(io
);
854 static void kcryptd_async_done(struct crypto_async_request
*async_req
,
857 struct dm_crypt_request
*dmreq
= async_req
->data
;
858 struct convert_context
*ctx
= dmreq
->ctx
;
859 struct dm_crypt_io
*io
= container_of(ctx
, struct dm_crypt_io
, ctx
);
860 struct crypt_config
*cc
= io
->target
->private;
862 if (error
== -EINPROGRESS
) {
863 complete(&ctx
->restart
);
867 mempool_free(req_of_dmreq(cc
, dmreq
), cc
->req_pool
);
869 if (!atomic_dec_and_test(&ctx
->pending
))
872 if (bio_data_dir(io
->base_bio
) == READ
)
873 kcryptd_crypt_read_done(io
, error
);
875 kcryptd_crypt_write_io_submit(io
, error
, 1);
878 static void kcryptd_crypt(struct work_struct
*work
)
880 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
882 if (bio_data_dir(io
->base_bio
) == READ
)
883 kcryptd_crypt_read_convert(io
);
885 kcryptd_crypt_write_convert(io
);
888 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
)
890 struct crypt_config
*cc
= io
->target
->private;
892 INIT_WORK(&io
->work
, kcryptd_crypt
);
893 queue_work(cc
->crypt_queue
, &io
->work
);
897 * Decode key from its hex representation
899 static int crypt_decode_key(u8
*key
, char *hex
, unsigned int size
)
907 for (i
= 0; i
< size
; i
++) {
911 key
[i
] = (u8
)simple_strtoul(buffer
, &endp
, 16);
913 if (endp
!= &buffer
[2])
924 * Encode key into its hex representation
926 static void crypt_encode_key(char *hex
, u8
*key
, unsigned int size
)
930 for (i
= 0; i
< size
; i
++) {
931 sprintf(hex
, "%02x", *key
);
937 static int crypt_set_key(struct crypt_config
*cc
, char *key
)
939 unsigned key_size
= strlen(key
) >> 1;
941 if (cc
->key_size
&& cc
->key_size
!= key_size
)
944 cc
->key_size
= key_size
; /* initial settings */
946 if ((!key_size
&& strcmp(key
, "-")) ||
947 (key_size
&& crypt_decode_key(cc
->key
, key
, key_size
) < 0))
950 set_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
952 return crypto_ablkcipher_setkey(cc
->tfm
, cc
->key
, cc
->key_size
);
955 static int crypt_wipe_key(struct crypt_config
*cc
)
957 clear_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
958 memset(&cc
->key
, 0, cc
->key_size
* sizeof(u8
));
959 return crypto_ablkcipher_setkey(cc
->tfm
, cc
->key
, cc
->key_size
);
963 * Construct an encryption mapping:
964 * <cipher> <key> <iv_offset> <dev_path> <start>
966 static int crypt_ctr(struct dm_target
*ti
, unsigned int argc
, char **argv
)
968 struct crypt_config
*cc
;
969 struct crypto_ablkcipher
*tfm
;
975 unsigned int key_size
;
976 unsigned long long tmpll
;
979 ti
->error
= "Not enough arguments";
984 cipher
= strsep(&tmp
, "-");
985 chainmode
= strsep(&tmp
, "-");
986 ivopts
= strsep(&tmp
, "-");
987 ivmode
= strsep(&ivopts
, ":");
990 DMWARN("Unexpected additional cipher options");
992 key_size
= strlen(argv
[1]) >> 1;
994 cc
= kzalloc(sizeof(*cc
) + key_size
* sizeof(u8
), GFP_KERNEL
);
997 "Cannot allocate transparent encryption context";
1001 /* Compatibility mode for old dm-crypt cipher strings */
1002 if (!chainmode
|| (strcmp(chainmode
, "plain") == 0 && !ivmode
)) {
1007 if (strcmp(chainmode
, "ecb") && !ivmode
) {
1008 ti
->error
= "This chaining mode requires an IV mechanism";
1012 if (snprintf(cc
->cipher
, CRYPTO_MAX_ALG_NAME
, "%s(%s)",
1013 chainmode
, cipher
) >= CRYPTO_MAX_ALG_NAME
) {
1014 ti
->error
= "Chain mode + cipher name is too long";
1018 tfm
= crypto_alloc_ablkcipher(cc
->cipher
, 0, 0);
1020 ti
->error
= "Error allocating crypto tfm";
1024 strcpy(cc
->cipher
, cipher
);
1025 strcpy(cc
->chainmode
, chainmode
);
1028 if (crypt_set_key(cc
, argv
[1]) < 0) {
1029 ti
->error
= "Error decoding and setting key";
1034 * Choose ivmode. Valid modes: "plain", "essiv:<esshash>", "benbi".
1035 * See comments at iv code
1039 cc
->iv_gen_ops
= NULL
;
1040 else if (strcmp(ivmode
, "plain") == 0)
1041 cc
->iv_gen_ops
= &crypt_iv_plain_ops
;
1042 else if (strcmp(ivmode
, "essiv") == 0)
1043 cc
->iv_gen_ops
= &crypt_iv_essiv_ops
;
1044 else if (strcmp(ivmode
, "benbi") == 0)
1045 cc
->iv_gen_ops
= &crypt_iv_benbi_ops
;
1046 else if (strcmp(ivmode
, "null") == 0)
1047 cc
->iv_gen_ops
= &crypt_iv_null_ops
;
1049 ti
->error
= "Invalid IV mode";
1053 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->ctr
&&
1054 cc
->iv_gen_ops
->ctr(cc
, ti
, ivopts
) < 0)
1057 cc
->iv_size
= crypto_ablkcipher_ivsize(tfm
);
1059 /* at least a 64 bit sector number should fit in our buffer */
1060 cc
->iv_size
= max(cc
->iv_size
,
1061 (unsigned int)(sizeof(u64
) / sizeof(u8
)));
1063 if (cc
->iv_gen_ops
) {
1064 DMWARN("Selected cipher does not support IVs");
1065 if (cc
->iv_gen_ops
->dtr
)
1066 cc
->iv_gen_ops
->dtr(cc
);
1067 cc
->iv_gen_ops
= NULL
;
1071 cc
->io_pool
= mempool_create_slab_pool(MIN_IOS
, _crypt_io_pool
);
1073 ti
->error
= "Cannot allocate crypt io mempool";
1077 cc
->dmreq_start
= sizeof(struct ablkcipher_request
);
1078 cc
->dmreq_start
+= crypto_ablkcipher_reqsize(tfm
);
1079 cc
->dmreq_start
= ALIGN(cc
->dmreq_start
, crypto_tfm_ctx_alignment());
1080 cc
->dmreq_start
+= crypto_ablkcipher_alignmask(tfm
) &
1081 ~(crypto_tfm_ctx_alignment() - 1);
1083 cc
->req_pool
= mempool_create_kmalloc_pool(MIN_IOS
, cc
->dmreq_start
+
1084 sizeof(struct dm_crypt_request
) + cc
->iv_size
);
1085 if (!cc
->req_pool
) {
1086 ti
->error
= "Cannot allocate crypt request mempool";
1091 cc
->page_pool
= mempool_create_page_pool(MIN_POOL_PAGES
, 0);
1092 if (!cc
->page_pool
) {
1093 ti
->error
= "Cannot allocate page mempool";
1097 cc
->bs
= bioset_create(MIN_IOS
, 0);
1099 ti
->error
= "Cannot allocate crypt bioset";
1103 if (sscanf(argv
[2], "%llu", &tmpll
) != 1) {
1104 ti
->error
= "Invalid iv_offset sector";
1107 cc
->iv_offset
= tmpll
;
1109 if (sscanf(argv
[4], "%llu", &tmpll
) != 1) {
1110 ti
->error
= "Invalid device sector";
1115 if (dm_get_device(ti
, argv
[3], cc
->start
, ti
->len
,
1116 dm_table_get_mode(ti
->table
), &cc
->dev
)) {
1117 ti
->error
= "Device lookup failed";
1121 if (ivmode
&& cc
->iv_gen_ops
) {
1123 *(ivopts
- 1) = ':';
1124 cc
->iv_mode
= kmalloc(strlen(ivmode
) + 1, GFP_KERNEL
);
1126 ti
->error
= "Error kmallocing iv_mode string";
1127 goto bad_ivmode_string
;
1129 strcpy(cc
->iv_mode
, ivmode
);
1133 cc
->io_queue
= create_singlethread_workqueue("kcryptd_io");
1134 if (!cc
->io_queue
) {
1135 ti
->error
= "Couldn't create kcryptd io queue";
1139 cc
->crypt_queue
= create_singlethread_workqueue("kcryptd");
1140 if (!cc
->crypt_queue
) {
1141 ti
->error
= "Couldn't create kcryptd queue";
1142 goto bad_crypt_queue
;
1145 ti
->num_flush_requests
= 1;
1150 destroy_workqueue(cc
->io_queue
);
1154 dm_put_device(ti
, cc
->dev
);
1156 bioset_free(cc
->bs
);
1158 mempool_destroy(cc
->page_pool
);
1160 mempool_destroy(cc
->req_pool
);
1162 mempool_destroy(cc
->io_pool
);
1164 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->dtr
)
1165 cc
->iv_gen_ops
->dtr(cc
);
1167 crypto_free_ablkcipher(tfm
);
1169 /* Must zero key material before freeing */
1174 static void crypt_dtr(struct dm_target
*ti
)
1176 struct crypt_config
*cc
= (struct crypt_config
*) ti
->private;
1178 destroy_workqueue(cc
->io_queue
);
1179 destroy_workqueue(cc
->crypt_queue
);
1182 mempool_free(cc
->req
, cc
->req_pool
);
1184 bioset_free(cc
->bs
);
1185 mempool_destroy(cc
->page_pool
);
1186 mempool_destroy(cc
->req_pool
);
1187 mempool_destroy(cc
->io_pool
);
1190 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->dtr
)
1191 cc
->iv_gen_ops
->dtr(cc
);
1192 crypto_free_ablkcipher(cc
->tfm
);
1193 dm_put_device(ti
, cc
->dev
);
1195 /* Must zero key material before freeing */
1199 static int crypt_map(struct dm_target
*ti
, struct bio
*bio
,
1200 union map_info
*map_context
)
1202 struct dm_crypt_io
*io
;
1203 struct crypt_config
*cc
;
1205 if (unlikely(bio_empty_barrier(bio
))) {
1207 bio
->bi_bdev
= cc
->dev
->bdev
;
1208 return DM_MAPIO_REMAPPED
;
1211 io
= crypt_io_alloc(ti
, bio
, bio
->bi_sector
- ti
->begin
);
1213 if (bio_data_dir(io
->base_bio
) == READ
)
1214 kcryptd_queue_io(io
);
1216 kcryptd_queue_crypt(io
);
1218 return DM_MAPIO_SUBMITTED
;
1221 static int crypt_status(struct dm_target
*ti
, status_type_t type
,
1222 char *result
, unsigned int maxlen
)
1224 struct crypt_config
*cc
= (struct crypt_config
*) ti
->private;
1225 unsigned int sz
= 0;
1228 case STATUSTYPE_INFO
:
1232 case STATUSTYPE_TABLE
:
1234 DMEMIT("%s-%s-%s ", cc
->cipher
, cc
->chainmode
,
1237 DMEMIT("%s-%s ", cc
->cipher
, cc
->chainmode
);
1239 if (cc
->key_size
> 0) {
1240 if ((maxlen
- sz
) < ((cc
->key_size
<< 1) + 1))
1243 crypt_encode_key(result
+ sz
, cc
->key
, cc
->key_size
);
1244 sz
+= cc
->key_size
<< 1;
1251 DMEMIT(" %llu %s %llu", (unsigned long long)cc
->iv_offset
,
1252 cc
->dev
->name
, (unsigned long long)cc
->start
);
1258 static void crypt_postsuspend(struct dm_target
*ti
)
1260 struct crypt_config
*cc
= ti
->private;
1262 set_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
1265 static int crypt_preresume(struct dm_target
*ti
)
1267 struct crypt_config
*cc
= ti
->private;
1269 if (!test_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
)) {
1270 DMERR("aborting resume - crypt key is not set.");
1277 static void crypt_resume(struct dm_target
*ti
)
1279 struct crypt_config
*cc
= ti
->private;
1281 clear_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
1284 /* Message interface
1288 static int crypt_message(struct dm_target
*ti
, unsigned argc
, char **argv
)
1290 struct crypt_config
*cc
= ti
->private;
1295 if (!strnicmp(argv
[0], MESG_STR("key"))) {
1296 if (!test_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
)) {
1297 DMWARN("not suspended during key manipulation.");
1300 if (argc
== 3 && !strnicmp(argv
[1], MESG_STR("set")))
1301 return crypt_set_key(cc
, argv
[2]);
1302 if (argc
== 2 && !strnicmp(argv
[1], MESG_STR("wipe")))
1303 return crypt_wipe_key(cc
);
1307 DMWARN("unrecognised message received.");
1311 static int crypt_merge(struct dm_target
*ti
, struct bvec_merge_data
*bvm
,
1312 struct bio_vec
*biovec
, int max_size
)
1314 struct crypt_config
*cc
= ti
->private;
1315 struct request_queue
*q
= bdev_get_queue(cc
->dev
->bdev
);
1317 if (!q
->merge_bvec_fn
)
1320 bvm
->bi_bdev
= cc
->dev
->bdev
;
1321 bvm
->bi_sector
= cc
->start
+ bvm
->bi_sector
- ti
->begin
;
1323 return min(max_size
, q
->merge_bvec_fn(q
, bvm
, biovec
));
1326 static int crypt_iterate_devices(struct dm_target
*ti
,
1327 iterate_devices_callout_fn fn
, void *data
)
1329 struct crypt_config
*cc
= ti
->private;
1331 return fn(ti
, cc
->dev
, cc
->start
, ti
->len
, data
);
1334 static struct target_type crypt_target
= {
1336 .version
= {1, 7, 0},
1337 .module
= THIS_MODULE
,
1341 .status
= crypt_status
,
1342 .postsuspend
= crypt_postsuspend
,
1343 .preresume
= crypt_preresume
,
1344 .resume
= crypt_resume
,
1345 .message
= crypt_message
,
1346 .merge
= crypt_merge
,
1347 .iterate_devices
= crypt_iterate_devices
,
1350 static int __init
dm_crypt_init(void)
1354 _crypt_io_pool
= KMEM_CACHE(dm_crypt_io
, 0);
1355 if (!_crypt_io_pool
)
1358 r
= dm_register_target(&crypt_target
);
1360 DMERR("register failed %d", r
);
1361 kmem_cache_destroy(_crypt_io_pool
);
1367 static void __exit
dm_crypt_exit(void)
1369 dm_unregister_target(&crypt_target
);
1370 kmem_cache_destroy(_crypt_io_pool
);
1373 module_init(dm_crypt_init
);
1374 module_exit(dm_crypt_exit
);
1376 MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
1377 MODULE_DESCRIPTION(DM_NAME
" target for transparent encryption / decryption");
1378 MODULE_LICENSE("GPL");