2 * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
3 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
4 * Copyright (C) 2006-2009 Red Hat, Inc. All rights reserved.
6 * This file is released under the GPL.
9 #include <linux/completion.h>
10 #include <linux/err.h>
11 #include <linux/module.h>
12 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/bio.h>
15 #include <linux/blkdev.h>
16 #include <linux/mempool.h>
17 #include <linux/slab.h>
18 #include <linux/crypto.h>
19 #include <linux/workqueue.h>
20 #include <linux/backing-dev.h>
21 #include <linux/percpu.h>
22 #include <linux/atomic.h>
23 #include <linux/scatterlist.h>
25 #include <asm/unaligned.h>
26 #include <crypto/hash.h>
27 #include <crypto/md5.h>
28 #include <crypto/algapi.h>
30 #include <linux/device-mapper.h>
32 #define DM_MSG_PREFIX "crypt"
35 * context holding the current state of a multi-part conversion
37 struct convert_context
{
38 struct completion restart
;
41 unsigned int offset_in
;
42 unsigned int offset_out
;
50 * per bio private data
53 struct crypt_config
*cc
;
55 struct work_struct work
;
57 struct convert_context ctx
;
62 struct dm_crypt_io
*base_io
;
65 struct dm_crypt_request
{
66 struct convert_context
*ctx
;
67 struct scatterlist sg_in
;
68 struct scatterlist sg_out
;
74 struct crypt_iv_operations
{
75 int (*ctr
)(struct crypt_config
*cc
, struct dm_target
*ti
,
77 void (*dtr
)(struct crypt_config
*cc
);
78 int (*init
)(struct crypt_config
*cc
);
79 int (*wipe
)(struct crypt_config
*cc
);
80 int (*generator
)(struct crypt_config
*cc
, u8
*iv
,
81 struct dm_crypt_request
*dmreq
);
82 int (*post
)(struct crypt_config
*cc
, u8
*iv
,
83 struct dm_crypt_request
*dmreq
);
86 struct iv_essiv_private
{
87 struct crypto_hash
*hash_tfm
;
91 struct iv_benbi_private
{
95 #define LMK_SEED_SIZE 64 /* hash + 0 */
96 struct iv_lmk_private
{
97 struct crypto_shash
*hash_tfm
;
102 * Crypt: maps a linear range of a block device
103 * and encrypts / decrypts at the same time.
105 enum flags
{ DM_CRYPT_SUSPENDED
, DM_CRYPT_KEY_VALID
};
108 * Duplicated per-CPU state for cipher.
111 struct ablkcipher_request
*req
;
115 * The fields in here must be read only after initialization,
116 * changing state should be in crypt_cpu.
118 struct crypt_config
{
123 * pool for per bio private data, crypto requests and
124 * encryption requeusts/buffer pages
128 mempool_t
*page_pool
;
131 struct workqueue_struct
*io_queue
;
132 struct workqueue_struct
*crypt_queue
;
137 struct crypt_iv_operations
*iv_gen_ops
;
139 struct iv_essiv_private essiv
;
140 struct iv_benbi_private benbi
;
141 struct iv_lmk_private lmk
;
144 unsigned int iv_size
;
147 * Duplicated per cpu state. Access through
148 * per_cpu_ptr() only.
150 struct crypt_cpu __percpu
*cpu
;
152 /* ESSIV: struct crypto_cipher *essiv_tfm */
154 struct crypto_ablkcipher
**tfms
;
158 * Layout of each crypto request:
160 * struct ablkcipher_request
163 * struct dm_crypt_request
167 * The padding is added so that dm_crypt_request and the IV are
170 unsigned int dmreq_start
;
173 unsigned int key_size
;
174 unsigned int key_parts
;
179 #define MIN_POOL_PAGES 32
181 static struct kmem_cache
*_crypt_io_pool
;
183 static void clone_init(struct dm_crypt_io
*, struct bio
*);
184 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
);
185 static u8
*iv_of_dmreq(struct crypt_config
*cc
, struct dm_crypt_request
*dmreq
);
187 static struct crypt_cpu
*this_crypt_config(struct crypt_config
*cc
)
189 return this_cpu_ptr(cc
->cpu
);
193 * Use this to access cipher attributes that are the same for each CPU.
195 static struct crypto_ablkcipher
*any_tfm(struct crypt_config
*cc
)
201 * Different IV generation algorithms:
203 * plain: the initial vector is the 32-bit little-endian version of the sector
204 * number, padded with zeros if necessary.
206 * plain64: the initial vector is the 64-bit little-endian version of the sector
207 * number, padded with zeros if necessary.
209 * essiv: "encrypted sector|salt initial vector", the sector number is
210 * encrypted with the bulk cipher using a salt as key. The salt
211 * should be derived from the bulk cipher's key via hashing.
213 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
214 * (needed for LRW-32-AES and possible other narrow block modes)
216 * null: the initial vector is always zero. Provides compatibility with
217 * obsolete loop_fish2 devices. Do not use for new devices.
219 * lmk: Compatible implementation of the block chaining mode used
220 * by the Loop-AES block device encryption system
221 * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/
222 * It operates on full 512 byte sectors and uses CBC
223 * with an IV derived from the sector number, the data and
224 * optionally extra IV seed.
225 * This means that after decryption the first block
226 * of sector must be tweaked according to decrypted data.
227 * Loop-AES can use three encryption schemes:
228 * version 1: is plain aes-cbc mode
229 * version 2: uses 64 multikey scheme with lmk IV generator
230 * version 3: the same as version 2 with additional IV seed
231 * (it uses 65 keys, last key is used as IV seed)
233 * plumb: unimplemented, see:
234 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
237 static int crypt_iv_plain_gen(struct crypt_config
*cc
, u8
*iv
,
238 struct dm_crypt_request
*dmreq
)
240 memset(iv
, 0, cc
->iv_size
);
241 *(__le32
*)iv
= cpu_to_le32(dmreq
->iv_sector
& 0xffffffff);
246 static int crypt_iv_plain64_gen(struct crypt_config
*cc
, u8
*iv
,
247 struct dm_crypt_request
*dmreq
)
249 memset(iv
, 0, cc
->iv_size
);
250 *(__le64
*)iv
= cpu_to_le64(dmreq
->iv_sector
);
255 /* Initialise ESSIV - compute salt but no local memory allocations */
256 static int crypt_iv_essiv_init(struct crypt_config
*cc
)
258 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
259 struct hash_desc desc
;
260 struct scatterlist sg
;
261 struct crypto_cipher
*essiv_tfm
;
264 sg_init_one(&sg
, cc
->key
, cc
->key_size
);
265 desc
.tfm
= essiv
->hash_tfm
;
266 desc
.flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
268 err
= crypto_hash_digest(&desc
, &sg
, cc
->key_size
, essiv
->salt
);
272 essiv_tfm
= cc
->iv_private
;
274 err
= crypto_cipher_setkey(essiv_tfm
, essiv
->salt
,
275 crypto_hash_digestsize(essiv
->hash_tfm
));
282 /* Wipe salt and reset key derived from volume key */
283 static int crypt_iv_essiv_wipe(struct crypt_config
*cc
)
285 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
286 unsigned salt_size
= crypto_hash_digestsize(essiv
->hash_tfm
);
287 struct crypto_cipher
*essiv_tfm
;
290 memset(essiv
->salt
, 0, salt_size
);
292 essiv_tfm
= cc
->iv_private
;
293 r
= crypto_cipher_setkey(essiv_tfm
, essiv
->salt
, salt_size
);
300 /* Set up per cpu cipher state */
301 static struct crypto_cipher
*setup_essiv_cpu(struct crypt_config
*cc
,
302 struct dm_target
*ti
,
303 u8
*salt
, unsigned saltsize
)
305 struct crypto_cipher
*essiv_tfm
;
308 /* Setup the essiv_tfm with the given salt */
309 essiv_tfm
= crypto_alloc_cipher(cc
->cipher
, 0, CRYPTO_ALG_ASYNC
);
310 if (IS_ERR(essiv_tfm
)) {
311 ti
->error
= "Error allocating crypto tfm for ESSIV";
315 if (crypto_cipher_blocksize(essiv_tfm
) !=
316 crypto_ablkcipher_ivsize(any_tfm(cc
))) {
317 ti
->error
= "Block size of ESSIV cipher does "
318 "not match IV size of block cipher";
319 crypto_free_cipher(essiv_tfm
);
320 return ERR_PTR(-EINVAL
);
323 err
= crypto_cipher_setkey(essiv_tfm
, salt
, saltsize
);
325 ti
->error
= "Failed to set key for ESSIV cipher";
326 crypto_free_cipher(essiv_tfm
);
333 static void crypt_iv_essiv_dtr(struct crypt_config
*cc
)
335 struct crypto_cipher
*essiv_tfm
;
336 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
338 crypto_free_hash(essiv
->hash_tfm
);
339 essiv
->hash_tfm
= NULL
;
344 essiv_tfm
= cc
->iv_private
;
347 crypto_free_cipher(essiv_tfm
);
349 cc
->iv_private
= NULL
;
352 static int crypt_iv_essiv_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
355 struct crypto_cipher
*essiv_tfm
= NULL
;
356 struct crypto_hash
*hash_tfm
= NULL
;
361 ti
->error
= "Digest algorithm missing for ESSIV mode";
365 /* Allocate hash algorithm */
366 hash_tfm
= crypto_alloc_hash(opts
, 0, CRYPTO_ALG_ASYNC
);
367 if (IS_ERR(hash_tfm
)) {
368 ti
->error
= "Error initializing ESSIV hash";
369 err
= PTR_ERR(hash_tfm
);
373 salt
= kzalloc(crypto_hash_digestsize(hash_tfm
), GFP_KERNEL
);
375 ti
->error
= "Error kmallocing salt storage in ESSIV";
380 cc
->iv_gen_private
.essiv
.salt
= salt
;
381 cc
->iv_gen_private
.essiv
.hash_tfm
= hash_tfm
;
383 essiv_tfm
= setup_essiv_cpu(cc
, ti
, salt
,
384 crypto_hash_digestsize(hash_tfm
));
385 if (IS_ERR(essiv_tfm
)) {
386 crypt_iv_essiv_dtr(cc
);
387 return PTR_ERR(essiv_tfm
);
389 cc
->iv_private
= essiv_tfm
;
394 if (hash_tfm
&& !IS_ERR(hash_tfm
))
395 crypto_free_hash(hash_tfm
);
400 static int crypt_iv_essiv_gen(struct crypt_config
*cc
, u8
*iv
,
401 struct dm_crypt_request
*dmreq
)
403 struct crypto_cipher
*essiv_tfm
= cc
->iv_private
;
405 memset(iv
, 0, cc
->iv_size
);
406 *(__le64
*)iv
= cpu_to_le64(dmreq
->iv_sector
);
407 crypto_cipher_encrypt_one(essiv_tfm
, iv
, iv
);
412 static int crypt_iv_benbi_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
415 unsigned bs
= crypto_ablkcipher_blocksize(any_tfm(cc
));
418 /* we need to calculate how far we must shift the sector count
419 * to get the cipher block count, we use this shift in _gen */
421 if (1 << log
!= bs
) {
422 ti
->error
= "cypher blocksize is not a power of 2";
427 ti
->error
= "cypher blocksize is > 512";
431 cc
->iv_gen_private
.benbi
.shift
= 9 - log
;
436 static void crypt_iv_benbi_dtr(struct crypt_config
*cc
)
440 static int crypt_iv_benbi_gen(struct crypt_config
*cc
, u8
*iv
,
441 struct dm_crypt_request
*dmreq
)
445 memset(iv
, 0, cc
->iv_size
- sizeof(u64
)); /* rest is cleared below */
447 val
= cpu_to_be64(((u64
)dmreq
->iv_sector
<< cc
->iv_gen_private
.benbi
.shift
) + 1);
448 put_unaligned(val
, (__be64
*)(iv
+ cc
->iv_size
- sizeof(u64
)));
453 static int crypt_iv_null_gen(struct crypt_config
*cc
, u8
*iv
,
454 struct dm_crypt_request
*dmreq
)
456 memset(iv
, 0, cc
->iv_size
);
461 static void crypt_iv_lmk_dtr(struct crypt_config
*cc
)
463 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
465 if (lmk
->hash_tfm
&& !IS_ERR(lmk
->hash_tfm
))
466 crypto_free_shash(lmk
->hash_tfm
);
467 lmk
->hash_tfm
= NULL
;
473 static int crypt_iv_lmk_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
476 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
478 lmk
->hash_tfm
= crypto_alloc_shash("md5", 0, 0);
479 if (IS_ERR(lmk
->hash_tfm
)) {
480 ti
->error
= "Error initializing LMK hash";
481 return PTR_ERR(lmk
->hash_tfm
);
484 /* No seed in LMK version 2 */
485 if (cc
->key_parts
== cc
->tfms_count
) {
490 lmk
->seed
= kzalloc(LMK_SEED_SIZE
, GFP_KERNEL
);
492 crypt_iv_lmk_dtr(cc
);
493 ti
->error
= "Error kmallocing seed storage in LMK";
500 static int crypt_iv_lmk_init(struct crypt_config
*cc
)
502 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
503 int subkey_size
= cc
->key_size
/ cc
->key_parts
;
505 /* LMK seed is on the position of LMK_KEYS + 1 key */
507 memcpy(lmk
->seed
, cc
->key
+ (cc
->tfms_count
* subkey_size
),
508 crypto_shash_digestsize(lmk
->hash_tfm
));
513 static int crypt_iv_lmk_wipe(struct crypt_config
*cc
)
515 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
518 memset(lmk
->seed
, 0, LMK_SEED_SIZE
);
523 static int crypt_iv_lmk_one(struct crypt_config
*cc
, u8
*iv
,
524 struct dm_crypt_request
*dmreq
,
527 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
529 struct shash_desc desc
;
530 char ctx
[crypto_shash_descsize(lmk
->hash_tfm
)];
532 struct md5_state md5state
;
536 sdesc
.desc
.tfm
= lmk
->hash_tfm
;
537 sdesc
.desc
.flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
539 r
= crypto_shash_init(&sdesc
.desc
);
544 r
= crypto_shash_update(&sdesc
.desc
, lmk
->seed
, LMK_SEED_SIZE
);
549 /* Sector is always 512B, block size 16, add data of blocks 1-31 */
550 r
= crypto_shash_update(&sdesc
.desc
, data
+ 16, 16 * 31);
554 /* Sector is cropped to 56 bits here */
555 buf
[0] = cpu_to_le32(dmreq
->iv_sector
& 0xFFFFFFFF);
556 buf
[1] = cpu_to_le32((((u64
)dmreq
->iv_sector
>> 32) & 0x00FFFFFF) | 0x80000000);
557 buf
[2] = cpu_to_le32(4024);
559 r
= crypto_shash_update(&sdesc
.desc
, (u8
*)buf
, sizeof(buf
));
563 /* No MD5 padding here */
564 r
= crypto_shash_export(&sdesc
.desc
, &md5state
);
568 for (i
= 0; i
< MD5_HASH_WORDS
; i
++)
569 __cpu_to_le32s(&md5state
.hash
[i
]);
570 memcpy(iv
, &md5state
.hash
, cc
->iv_size
);
575 static int crypt_iv_lmk_gen(struct crypt_config
*cc
, u8
*iv
,
576 struct dm_crypt_request
*dmreq
)
581 if (bio_data_dir(dmreq
->ctx
->bio_in
) == WRITE
) {
582 src
= kmap_atomic(sg_page(&dmreq
->sg_in
));
583 r
= crypt_iv_lmk_one(cc
, iv
, dmreq
, src
+ dmreq
->sg_in
.offset
);
586 memset(iv
, 0, cc
->iv_size
);
591 static int crypt_iv_lmk_post(struct crypt_config
*cc
, u8
*iv
,
592 struct dm_crypt_request
*dmreq
)
597 if (bio_data_dir(dmreq
->ctx
->bio_in
) == WRITE
)
600 dst
= kmap_atomic(sg_page(&dmreq
->sg_out
));
601 r
= crypt_iv_lmk_one(cc
, iv
, dmreq
, dst
+ dmreq
->sg_out
.offset
);
603 /* Tweak the first block of plaintext sector */
605 crypto_xor(dst
+ dmreq
->sg_out
.offset
, iv
, cc
->iv_size
);
611 static struct crypt_iv_operations crypt_iv_plain_ops
= {
612 .generator
= crypt_iv_plain_gen
615 static struct crypt_iv_operations crypt_iv_plain64_ops
= {
616 .generator
= crypt_iv_plain64_gen
619 static struct crypt_iv_operations crypt_iv_essiv_ops
= {
620 .ctr
= crypt_iv_essiv_ctr
,
621 .dtr
= crypt_iv_essiv_dtr
,
622 .init
= crypt_iv_essiv_init
,
623 .wipe
= crypt_iv_essiv_wipe
,
624 .generator
= crypt_iv_essiv_gen
627 static struct crypt_iv_operations crypt_iv_benbi_ops
= {
628 .ctr
= crypt_iv_benbi_ctr
,
629 .dtr
= crypt_iv_benbi_dtr
,
630 .generator
= crypt_iv_benbi_gen
633 static struct crypt_iv_operations crypt_iv_null_ops
= {
634 .generator
= crypt_iv_null_gen
637 static struct crypt_iv_operations crypt_iv_lmk_ops
= {
638 .ctr
= crypt_iv_lmk_ctr
,
639 .dtr
= crypt_iv_lmk_dtr
,
640 .init
= crypt_iv_lmk_init
,
641 .wipe
= crypt_iv_lmk_wipe
,
642 .generator
= crypt_iv_lmk_gen
,
643 .post
= crypt_iv_lmk_post
646 static void crypt_convert_init(struct crypt_config
*cc
,
647 struct convert_context
*ctx
,
648 struct bio
*bio_out
, struct bio
*bio_in
,
651 ctx
->bio_in
= bio_in
;
652 ctx
->bio_out
= bio_out
;
655 ctx
->idx_in
= bio_in
? bio_in
->bi_idx
: 0;
656 ctx
->idx_out
= bio_out
? bio_out
->bi_idx
: 0;
657 ctx
->cc_sector
= sector
+ cc
->iv_offset
;
658 init_completion(&ctx
->restart
);
661 static struct dm_crypt_request
*dmreq_of_req(struct crypt_config
*cc
,
662 struct ablkcipher_request
*req
)
664 return (struct dm_crypt_request
*)((char *)req
+ cc
->dmreq_start
);
667 static struct ablkcipher_request
*req_of_dmreq(struct crypt_config
*cc
,
668 struct dm_crypt_request
*dmreq
)
670 return (struct ablkcipher_request
*)((char *)dmreq
- cc
->dmreq_start
);
673 static u8
*iv_of_dmreq(struct crypt_config
*cc
,
674 struct dm_crypt_request
*dmreq
)
676 return (u8
*)ALIGN((unsigned long)(dmreq
+ 1),
677 crypto_ablkcipher_alignmask(any_tfm(cc
)) + 1);
680 static int crypt_convert_block(struct crypt_config
*cc
,
681 struct convert_context
*ctx
,
682 struct ablkcipher_request
*req
)
684 struct bio_vec
*bv_in
= bio_iovec_idx(ctx
->bio_in
, ctx
->idx_in
);
685 struct bio_vec
*bv_out
= bio_iovec_idx(ctx
->bio_out
, ctx
->idx_out
);
686 struct dm_crypt_request
*dmreq
;
690 dmreq
= dmreq_of_req(cc
, req
);
691 iv
= iv_of_dmreq(cc
, dmreq
);
693 dmreq
->iv_sector
= ctx
->cc_sector
;
695 sg_init_table(&dmreq
->sg_in
, 1);
696 sg_set_page(&dmreq
->sg_in
, bv_in
->bv_page
, 1 << SECTOR_SHIFT
,
697 bv_in
->bv_offset
+ ctx
->offset_in
);
699 sg_init_table(&dmreq
->sg_out
, 1);
700 sg_set_page(&dmreq
->sg_out
, bv_out
->bv_page
, 1 << SECTOR_SHIFT
,
701 bv_out
->bv_offset
+ ctx
->offset_out
);
703 ctx
->offset_in
+= 1 << SECTOR_SHIFT
;
704 if (ctx
->offset_in
>= bv_in
->bv_len
) {
709 ctx
->offset_out
+= 1 << SECTOR_SHIFT
;
710 if (ctx
->offset_out
>= bv_out
->bv_len
) {
715 if (cc
->iv_gen_ops
) {
716 r
= cc
->iv_gen_ops
->generator(cc
, iv
, dmreq
);
721 ablkcipher_request_set_crypt(req
, &dmreq
->sg_in
, &dmreq
->sg_out
,
722 1 << SECTOR_SHIFT
, iv
);
724 if (bio_data_dir(ctx
->bio_in
) == WRITE
)
725 r
= crypto_ablkcipher_encrypt(req
);
727 r
= crypto_ablkcipher_decrypt(req
);
729 if (!r
&& cc
->iv_gen_ops
&& cc
->iv_gen_ops
->post
)
730 r
= cc
->iv_gen_ops
->post(cc
, iv
, dmreq
);
735 static void kcryptd_async_done(struct crypto_async_request
*async_req
,
738 static void crypt_alloc_req(struct crypt_config
*cc
,
739 struct convert_context
*ctx
)
741 struct crypt_cpu
*this_cc
= this_crypt_config(cc
);
742 unsigned key_index
= ctx
->cc_sector
& (cc
->tfms_count
- 1);
745 this_cc
->req
= mempool_alloc(cc
->req_pool
, GFP_NOIO
);
747 ablkcipher_request_set_tfm(this_cc
->req
, cc
->tfms
[key_index
]);
748 ablkcipher_request_set_callback(this_cc
->req
,
749 CRYPTO_TFM_REQ_MAY_BACKLOG
| CRYPTO_TFM_REQ_MAY_SLEEP
,
750 kcryptd_async_done
, dmreq_of_req(cc
, this_cc
->req
));
754 * Encrypt / decrypt data from one bio to another one (can be the same one)
756 static int crypt_convert(struct crypt_config
*cc
,
757 struct convert_context
*ctx
)
759 struct crypt_cpu
*this_cc
= this_crypt_config(cc
);
762 atomic_set(&ctx
->cc_pending
, 1);
764 while(ctx
->idx_in
< ctx
->bio_in
->bi_vcnt
&&
765 ctx
->idx_out
< ctx
->bio_out
->bi_vcnt
) {
767 crypt_alloc_req(cc
, ctx
);
769 atomic_inc(&ctx
->cc_pending
);
771 r
= crypt_convert_block(cc
, ctx
, this_cc
->req
);
776 wait_for_completion(&ctx
->restart
);
777 INIT_COMPLETION(ctx
->restart
);
786 atomic_dec(&ctx
->cc_pending
);
793 atomic_dec(&ctx
->cc_pending
);
802 * Generate a new unfragmented bio with the given size
803 * This should never violate the device limitations
804 * May return a smaller bio when running out of pages, indicated by
805 * *out_of_pages set to 1.
807 static struct bio
*crypt_alloc_buffer(struct dm_crypt_io
*io
, unsigned size
,
808 unsigned *out_of_pages
)
810 struct crypt_config
*cc
= io
->cc
;
812 unsigned int nr_iovecs
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
813 gfp_t gfp_mask
= GFP_NOIO
| __GFP_HIGHMEM
;
817 clone
= bio_alloc_bioset(GFP_NOIO
, nr_iovecs
, cc
->bs
);
821 clone_init(io
, clone
);
824 for (i
= 0; i
< nr_iovecs
; i
++) {
825 page
= mempool_alloc(cc
->page_pool
, gfp_mask
);
832 * If additional pages cannot be allocated without waiting,
833 * return a partially-allocated bio. The caller will then try
834 * to allocate more bios while submitting this partial bio.
836 gfp_mask
= (gfp_mask
| __GFP_NOWARN
) & ~__GFP_WAIT
;
838 len
= (size
> PAGE_SIZE
) ? PAGE_SIZE
: size
;
840 if (!bio_add_page(clone
, page
, len
, 0)) {
841 mempool_free(page
, cc
->page_pool
);
848 if (!clone
->bi_size
) {
856 static void crypt_free_buffer_pages(struct crypt_config
*cc
, struct bio
*clone
)
861 for (i
= 0; i
< clone
->bi_vcnt
; i
++) {
862 bv
= bio_iovec_idx(clone
, i
);
863 BUG_ON(!bv
->bv_page
);
864 mempool_free(bv
->bv_page
, cc
->page_pool
);
869 static struct dm_crypt_io
*crypt_io_alloc(struct crypt_config
*cc
,
870 struct bio
*bio
, sector_t sector
)
872 struct dm_crypt_io
*io
;
874 io
= mempool_alloc(cc
->io_pool
, GFP_NOIO
);
880 atomic_set(&io
->io_pending
, 0);
885 static void crypt_inc_pending(struct dm_crypt_io
*io
)
887 atomic_inc(&io
->io_pending
);
891 * One of the bios was finished. Check for completion of
892 * the whole request and correctly clean up the buffer.
893 * If base_io is set, wait for the last fragment to complete.
895 static void crypt_dec_pending(struct dm_crypt_io
*io
)
897 struct crypt_config
*cc
= io
->cc
;
898 struct bio
*base_bio
= io
->base_bio
;
899 struct dm_crypt_io
*base_io
= io
->base_io
;
900 int error
= io
->error
;
902 if (!atomic_dec_and_test(&io
->io_pending
))
905 mempool_free(io
, cc
->io_pool
);
907 if (likely(!base_io
))
908 bio_endio(base_bio
, error
);
910 if (error
&& !base_io
->error
)
911 base_io
->error
= error
;
912 crypt_dec_pending(base_io
);
917 * kcryptd/kcryptd_io:
919 * Needed because it would be very unwise to do decryption in an
922 * kcryptd performs the actual encryption or decryption.
924 * kcryptd_io performs the IO submission.
926 * They must be separated as otherwise the final stages could be
927 * starved by new requests which can block in the first stages due
928 * to memory allocation.
930 * The work is done per CPU global for all dm-crypt instances.
931 * They should not depend on each other and do not block.
933 static void crypt_endio(struct bio
*clone
, int error
)
935 struct dm_crypt_io
*io
= clone
->bi_private
;
936 struct crypt_config
*cc
= io
->cc
;
937 unsigned rw
= bio_data_dir(clone
);
939 if (unlikely(!bio_flagged(clone
, BIO_UPTODATE
) && !error
))
943 * free the processed pages
946 crypt_free_buffer_pages(cc
, clone
);
950 if (rw
== READ
&& !error
) {
951 kcryptd_queue_crypt(io
);
958 crypt_dec_pending(io
);
961 static void clone_init(struct dm_crypt_io
*io
, struct bio
*clone
)
963 struct crypt_config
*cc
= io
->cc
;
965 clone
->bi_private
= io
;
966 clone
->bi_end_io
= crypt_endio
;
967 clone
->bi_bdev
= cc
->dev
->bdev
;
968 clone
->bi_rw
= io
->base_bio
->bi_rw
;
971 static int kcryptd_io_read(struct dm_crypt_io
*io
, gfp_t gfp
)
973 struct crypt_config
*cc
= io
->cc
;
974 struct bio
*base_bio
= io
->base_bio
;
978 * The block layer might modify the bvec array, so always
979 * copy the required bvecs because we need the original
980 * one in order to decrypt the whole bio data *afterwards*.
982 clone
= bio_clone_bioset(base_bio
, gfp
, cc
->bs
);
986 crypt_inc_pending(io
);
988 clone_init(io
, clone
);
989 clone
->bi_sector
= cc
->start
+ io
->sector
;
991 generic_make_request(clone
);
995 static void kcryptd_io_write(struct dm_crypt_io
*io
)
997 struct bio
*clone
= io
->ctx
.bio_out
;
998 generic_make_request(clone
);
1001 static void kcryptd_io(struct work_struct
*work
)
1003 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
1005 if (bio_data_dir(io
->base_bio
) == READ
) {
1006 crypt_inc_pending(io
);
1007 if (kcryptd_io_read(io
, GFP_NOIO
))
1008 io
->error
= -ENOMEM
;
1009 crypt_dec_pending(io
);
1011 kcryptd_io_write(io
);
1014 static void kcryptd_queue_io(struct dm_crypt_io
*io
)
1016 struct crypt_config
*cc
= io
->cc
;
1018 INIT_WORK(&io
->work
, kcryptd_io
);
1019 queue_work(cc
->io_queue
, &io
->work
);
1022 static void kcryptd_crypt_write_io_submit(struct dm_crypt_io
*io
, int async
)
1024 struct bio
*clone
= io
->ctx
.bio_out
;
1025 struct crypt_config
*cc
= io
->cc
;
1027 if (unlikely(io
->error
< 0)) {
1028 crypt_free_buffer_pages(cc
, clone
);
1030 crypt_dec_pending(io
);
1034 /* crypt_convert should have filled the clone bio */
1035 BUG_ON(io
->ctx
.idx_out
< clone
->bi_vcnt
);
1037 clone
->bi_sector
= cc
->start
+ io
->sector
;
1040 kcryptd_queue_io(io
);
1042 generic_make_request(clone
);
1045 static void kcryptd_crypt_write_convert(struct dm_crypt_io
*io
)
1047 struct crypt_config
*cc
= io
->cc
;
1049 struct dm_crypt_io
*new_io
;
1051 unsigned out_of_pages
= 0;
1052 unsigned remaining
= io
->base_bio
->bi_size
;
1053 sector_t sector
= io
->sector
;
1057 * Prevent io from disappearing until this function completes.
1059 crypt_inc_pending(io
);
1060 crypt_convert_init(cc
, &io
->ctx
, NULL
, io
->base_bio
, sector
);
1063 * The allocated buffers can be smaller than the whole bio,
1064 * so repeat the whole process until all the data can be handled.
1067 clone
= crypt_alloc_buffer(io
, remaining
, &out_of_pages
);
1068 if (unlikely(!clone
)) {
1069 io
->error
= -ENOMEM
;
1073 io
->ctx
.bio_out
= clone
;
1074 io
->ctx
.idx_out
= 0;
1076 remaining
-= clone
->bi_size
;
1077 sector
+= bio_sectors(clone
);
1079 crypt_inc_pending(io
);
1081 r
= crypt_convert(cc
, &io
->ctx
);
1085 crypt_finished
= atomic_dec_and_test(&io
->ctx
.cc_pending
);
1087 /* Encryption was already finished, submit io now */
1088 if (crypt_finished
) {
1089 kcryptd_crypt_write_io_submit(io
, 0);
1092 * If there was an error, do not try next fragments.
1093 * For async, error is processed in async handler.
1095 if (unlikely(r
< 0))
1098 io
->sector
= sector
;
1102 * Out of memory -> run queues
1103 * But don't wait if split was due to the io size restriction
1105 if (unlikely(out_of_pages
))
1106 congestion_wait(BLK_RW_ASYNC
, HZ
/100);
1109 * With async crypto it is unsafe to share the crypto context
1110 * between fragments, so switch to a new dm_crypt_io structure.
1112 if (unlikely(!crypt_finished
&& remaining
)) {
1113 new_io
= crypt_io_alloc(io
->cc
, io
->base_bio
,
1115 crypt_inc_pending(new_io
);
1116 crypt_convert_init(cc
, &new_io
->ctx
, NULL
,
1117 io
->base_bio
, sector
);
1118 new_io
->ctx
.idx_in
= io
->ctx
.idx_in
;
1119 new_io
->ctx
.offset_in
= io
->ctx
.offset_in
;
1122 * Fragments after the first use the base_io
1126 new_io
->base_io
= io
;
1128 new_io
->base_io
= io
->base_io
;
1129 crypt_inc_pending(io
->base_io
);
1130 crypt_dec_pending(io
);
1137 crypt_dec_pending(io
);
1140 static void kcryptd_crypt_read_done(struct dm_crypt_io
*io
)
1142 crypt_dec_pending(io
);
1145 static void kcryptd_crypt_read_convert(struct dm_crypt_io
*io
)
1147 struct crypt_config
*cc
= io
->cc
;
1150 crypt_inc_pending(io
);
1152 crypt_convert_init(cc
, &io
->ctx
, io
->base_bio
, io
->base_bio
,
1155 r
= crypt_convert(cc
, &io
->ctx
);
1159 if (atomic_dec_and_test(&io
->ctx
.cc_pending
))
1160 kcryptd_crypt_read_done(io
);
1162 crypt_dec_pending(io
);
1165 static void kcryptd_async_done(struct crypto_async_request
*async_req
,
1168 struct dm_crypt_request
*dmreq
= async_req
->data
;
1169 struct convert_context
*ctx
= dmreq
->ctx
;
1170 struct dm_crypt_io
*io
= container_of(ctx
, struct dm_crypt_io
, ctx
);
1171 struct crypt_config
*cc
= io
->cc
;
1173 if (error
== -EINPROGRESS
) {
1174 complete(&ctx
->restart
);
1178 if (!error
&& cc
->iv_gen_ops
&& cc
->iv_gen_ops
->post
)
1179 error
= cc
->iv_gen_ops
->post(cc
, iv_of_dmreq(cc
, dmreq
), dmreq
);
1184 mempool_free(req_of_dmreq(cc
, dmreq
), cc
->req_pool
);
1186 if (!atomic_dec_and_test(&ctx
->cc_pending
))
1189 if (bio_data_dir(io
->base_bio
) == READ
)
1190 kcryptd_crypt_read_done(io
);
1192 kcryptd_crypt_write_io_submit(io
, 1);
1195 static void kcryptd_crypt(struct work_struct
*work
)
1197 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
1199 if (bio_data_dir(io
->base_bio
) == READ
)
1200 kcryptd_crypt_read_convert(io
);
1202 kcryptd_crypt_write_convert(io
);
1205 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
)
1207 struct crypt_config
*cc
= io
->cc
;
1209 INIT_WORK(&io
->work
, kcryptd_crypt
);
1210 queue_work(cc
->crypt_queue
, &io
->work
);
1214 * Decode key from its hex representation
1216 static int crypt_decode_key(u8
*key
, char *hex
, unsigned int size
)
1223 for (i
= 0; i
< size
; i
++) {
1227 if (kstrtou8(buffer
, 16, &key
[i
]))
1238 * Encode key into its hex representation
1240 static void crypt_encode_key(char *hex
, u8
*key
, unsigned int size
)
1244 for (i
= 0; i
< size
; i
++) {
1245 sprintf(hex
, "%02x", *key
);
1251 static void crypt_free_tfms(struct crypt_config
*cc
)
1258 for (i
= 0; i
< cc
->tfms_count
; i
++)
1259 if (cc
->tfms
[i
] && !IS_ERR(cc
->tfms
[i
])) {
1260 crypto_free_ablkcipher(cc
->tfms
[i
]);
1268 static int crypt_alloc_tfms(struct crypt_config
*cc
, char *ciphermode
)
1273 cc
->tfms
= kmalloc(cc
->tfms_count
* sizeof(struct crypto_ablkcipher
*),
1278 for (i
= 0; i
< cc
->tfms_count
; i
++) {
1279 cc
->tfms
[i
] = crypto_alloc_ablkcipher(ciphermode
, 0, 0);
1280 if (IS_ERR(cc
->tfms
[i
])) {
1281 err
= PTR_ERR(cc
->tfms
[i
]);
1282 crypt_free_tfms(cc
);
1290 static int crypt_setkey_allcpus(struct crypt_config
*cc
)
1292 unsigned subkey_size
= cc
->key_size
>> ilog2(cc
->tfms_count
);
1295 for (i
= 0; i
< cc
->tfms_count
; i
++) {
1296 r
= crypto_ablkcipher_setkey(cc
->tfms
[i
],
1297 cc
->key
+ (i
* subkey_size
),
1306 static int crypt_set_key(struct crypt_config
*cc
, char *key
)
1309 int key_string_len
= strlen(key
);
1311 /* The key size may not be changed. */
1312 if (cc
->key_size
!= (key_string_len
>> 1))
1315 /* Hyphen (which gives a key_size of zero) means there is no key. */
1316 if (!cc
->key_size
&& strcmp(key
, "-"))
1319 if (cc
->key_size
&& crypt_decode_key(cc
->key
, key
, cc
->key_size
) < 0)
1322 set_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
1324 r
= crypt_setkey_allcpus(cc
);
1327 /* Hex key string not needed after here, so wipe it. */
1328 memset(key
, '0', key_string_len
);
1333 static int crypt_wipe_key(struct crypt_config
*cc
)
1335 clear_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
1336 memset(&cc
->key
, 0, cc
->key_size
* sizeof(u8
));
1338 return crypt_setkey_allcpus(cc
);
1341 static void crypt_dtr(struct dm_target
*ti
)
1343 struct crypt_config
*cc
= ti
->private;
1344 struct crypt_cpu
*cpu_cc
;
1353 destroy_workqueue(cc
->io_queue
);
1354 if (cc
->crypt_queue
)
1355 destroy_workqueue(cc
->crypt_queue
);
1358 for_each_possible_cpu(cpu
) {
1359 cpu_cc
= per_cpu_ptr(cc
->cpu
, cpu
);
1361 mempool_free(cpu_cc
->req
, cc
->req_pool
);
1364 crypt_free_tfms(cc
);
1367 bioset_free(cc
->bs
);
1370 mempool_destroy(cc
->page_pool
);
1372 mempool_destroy(cc
->req_pool
);
1374 mempool_destroy(cc
->io_pool
);
1376 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->dtr
)
1377 cc
->iv_gen_ops
->dtr(cc
);
1380 dm_put_device(ti
, cc
->dev
);
1383 free_percpu(cc
->cpu
);
1386 kzfree(cc
->cipher_string
);
1388 /* Must zero key material before freeing */
1392 static int crypt_ctr_cipher(struct dm_target
*ti
,
1393 char *cipher_in
, char *key
)
1395 struct crypt_config
*cc
= ti
->private;
1396 char *tmp
, *cipher
, *chainmode
, *ivmode
, *ivopts
, *keycount
;
1397 char *cipher_api
= NULL
;
1401 /* Convert to crypto api definition? */
1402 if (strchr(cipher_in
, '(')) {
1403 ti
->error
= "Bad cipher specification";
1407 cc
->cipher_string
= kstrdup(cipher_in
, GFP_KERNEL
);
1408 if (!cc
->cipher_string
)
1412 * Legacy dm-crypt cipher specification
1413 * cipher[:keycount]-mode-iv:ivopts
1416 keycount
= strsep(&tmp
, "-");
1417 cipher
= strsep(&keycount
, ":");
1421 else if (sscanf(keycount
, "%u%c", &cc
->tfms_count
, &dummy
) != 1 ||
1422 !is_power_of_2(cc
->tfms_count
)) {
1423 ti
->error
= "Bad cipher key count specification";
1426 cc
->key_parts
= cc
->tfms_count
;
1428 cc
->cipher
= kstrdup(cipher
, GFP_KERNEL
);
1432 chainmode
= strsep(&tmp
, "-");
1433 ivopts
= strsep(&tmp
, "-");
1434 ivmode
= strsep(&ivopts
, ":");
1437 DMWARN("Ignoring unexpected additional cipher options");
1439 cc
->cpu
= __alloc_percpu(sizeof(*(cc
->cpu
)),
1440 __alignof__(struct crypt_cpu
));
1442 ti
->error
= "Cannot allocate per cpu state";
1447 * For compatibility with the original dm-crypt mapping format, if
1448 * only the cipher name is supplied, use cbc-plain.
1450 if (!chainmode
|| (!strcmp(chainmode
, "plain") && !ivmode
)) {
1455 if (strcmp(chainmode
, "ecb") && !ivmode
) {
1456 ti
->error
= "IV mechanism required";
1460 cipher_api
= kmalloc(CRYPTO_MAX_ALG_NAME
, GFP_KERNEL
);
1464 ret
= snprintf(cipher_api
, CRYPTO_MAX_ALG_NAME
,
1465 "%s(%s)", chainmode
, cipher
);
1471 /* Allocate cipher */
1472 ret
= crypt_alloc_tfms(cc
, cipher_api
);
1474 ti
->error
= "Error allocating crypto tfm";
1478 /* Initialize and set key */
1479 ret
= crypt_set_key(cc
, key
);
1481 ti
->error
= "Error decoding and setting key";
1486 cc
->iv_size
= crypto_ablkcipher_ivsize(any_tfm(cc
));
1488 /* at least a 64 bit sector number should fit in our buffer */
1489 cc
->iv_size
= max(cc
->iv_size
,
1490 (unsigned int)(sizeof(u64
) / sizeof(u8
)));
1492 DMWARN("Selected cipher does not support IVs");
1496 /* Choose ivmode, see comments at iv code. */
1498 cc
->iv_gen_ops
= NULL
;
1499 else if (strcmp(ivmode
, "plain") == 0)
1500 cc
->iv_gen_ops
= &crypt_iv_plain_ops
;
1501 else if (strcmp(ivmode
, "plain64") == 0)
1502 cc
->iv_gen_ops
= &crypt_iv_plain64_ops
;
1503 else if (strcmp(ivmode
, "essiv") == 0)
1504 cc
->iv_gen_ops
= &crypt_iv_essiv_ops
;
1505 else if (strcmp(ivmode
, "benbi") == 0)
1506 cc
->iv_gen_ops
= &crypt_iv_benbi_ops
;
1507 else if (strcmp(ivmode
, "null") == 0)
1508 cc
->iv_gen_ops
= &crypt_iv_null_ops
;
1509 else if (strcmp(ivmode
, "lmk") == 0) {
1510 cc
->iv_gen_ops
= &crypt_iv_lmk_ops
;
1511 /* Version 2 and 3 is recognised according
1512 * to length of provided multi-key string.
1513 * If present (version 3), last key is used as IV seed.
1515 if (cc
->key_size
% cc
->key_parts
)
1519 ti
->error
= "Invalid IV mode";
1524 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->ctr
) {
1525 ret
= cc
->iv_gen_ops
->ctr(cc
, ti
, ivopts
);
1527 ti
->error
= "Error creating IV";
1532 /* Initialize IV (set keys for ESSIV etc) */
1533 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->init
) {
1534 ret
= cc
->iv_gen_ops
->init(cc
);
1536 ti
->error
= "Error initialising IV";
1547 ti
->error
= "Cannot allocate cipher strings";
1552 * Construct an encryption mapping:
1553 * <cipher> <key> <iv_offset> <dev_path> <start>
1555 static int crypt_ctr(struct dm_target
*ti
, unsigned int argc
, char **argv
)
1557 struct crypt_config
*cc
;
1558 unsigned int key_size
, opt_params
;
1559 unsigned long long tmpll
;
1561 struct dm_arg_set as
;
1562 const char *opt_string
;
1565 static struct dm_arg _args
[] = {
1566 {0, 1, "Invalid number of feature args"},
1570 ti
->error
= "Not enough arguments";
1574 key_size
= strlen(argv
[1]) >> 1;
1576 cc
= kzalloc(sizeof(*cc
) + key_size
* sizeof(u8
), GFP_KERNEL
);
1578 ti
->error
= "Cannot allocate encryption context";
1581 cc
->key_size
= key_size
;
1584 ret
= crypt_ctr_cipher(ti
, argv
[0], argv
[1]);
1589 cc
->io_pool
= mempool_create_slab_pool(MIN_IOS
, _crypt_io_pool
);
1591 ti
->error
= "Cannot allocate crypt io mempool";
1595 cc
->dmreq_start
= sizeof(struct ablkcipher_request
);
1596 cc
->dmreq_start
+= crypto_ablkcipher_reqsize(any_tfm(cc
));
1597 cc
->dmreq_start
= ALIGN(cc
->dmreq_start
, crypto_tfm_ctx_alignment());
1598 cc
->dmreq_start
+= crypto_ablkcipher_alignmask(any_tfm(cc
)) &
1599 ~(crypto_tfm_ctx_alignment() - 1);
1601 cc
->req_pool
= mempool_create_kmalloc_pool(MIN_IOS
, cc
->dmreq_start
+
1602 sizeof(struct dm_crypt_request
) + cc
->iv_size
);
1603 if (!cc
->req_pool
) {
1604 ti
->error
= "Cannot allocate crypt request mempool";
1608 cc
->page_pool
= mempool_create_page_pool(MIN_POOL_PAGES
, 0);
1609 if (!cc
->page_pool
) {
1610 ti
->error
= "Cannot allocate page mempool";
1614 cc
->bs
= bioset_create(MIN_IOS
, 0);
1616 ti
->error
= "Cannot allocate crypt bioset";
1621 if (sscanf(argv
[2], "%llu%c", &tmpll
, &dummy
) != 1) {
1622 ti
->error
= "Invalid iv_offset sector";
1625 cc
->iv_offset
= tmpll
;
1627 if (dm_get_device(ti
, argv
[3], dm_table_get_mode(ti
->table
), &cc
->dev
)) {
1628 ti
->error
= "Device lookup failed";
1632 if (sscanf(argv
[4], "%llu%c", &tmpll
, &dummy
) != 1) {
1633 ti
->error
= "Invalid device sector";
1641 /* Optional parameters */
1646 ret
= dm_read_arg_group(_args
, &as
, &opt_params
, &ti
->error
);
1650 opt_string
= dm_shift_arg(&as
);
1652 if (opt_params
== 1 && opt_string
&&
1653 !strcasecmp(opt_string
, "allow_discards"))
1654 ti
->num_discard_requests
= 1;
1655 else if (opt_params
) {
1657 ti
->error
= "Invalid feature arguments";
1663 cc
->io_queue
= alloc_workqueue("kcryptd_io",
1667 if (!cc
->io_queue
) {
1668 ti
->error
= "Couldn't create kcryptd io queue";
1672 cc
->crypt_queue
= alloc_workqueue("kcryptd",
1677 if (!cc
->crypt_queue
) {
1678 ti
->error
= "Couldn't create kcryptd queue";
1682 ti
->num_flush_requests
= 1;
1683 ti
->discard_zeroes_data_unsupported
= true;
1692 static int crypt_map(struct dm_target
*ti
, struct bio
*bio
,
1693 union map_info
*map_context
)
1695 struct dm_crypt_io
*io
;
1696 struct crypt_config
*cc
= ti
->private;
1699 * If bio is REQ_FLUSH or REQ_DISCARD, just bypass crypt queues.
1700 * - for REQ_FLUSH device-mapper core ensures that no IO is in-flight
1701 * - for REQ_DISCARD caller must use flush if IO ordering matters
1703 if (unlikely(bio
->bi_rw
& (REQ_FLUSH
| REQ_DISCARD
))) {
1704 bio
->bi_bdev
= cc
->dev
->bdev
;
1705 if (bio_sectors(bio
))
1706 bio
->bi_sector
= cc
->start
+ dm_target_offset(ti
, bio
->bi_sector
);
1707 return DM_MAPIO_REMAPPED
;
1710 io
= crypt_io_alloc(cc
, bio
, dm_target_offset(ti
, bio
->bi_sector
));
1712 if (bio_data_dir(io
->base_bio
) == READ
) {
1713 if (kcryptd_io_read(io
, GFP_NOWAIT
))
1714 kcryptd_queue_io(io
);
1716 kcryptd_queue_crypt(io
);
1718 return DM_MAPIO_SUBMITTED
;
1721 static int crypt_status(struct dm_target
*ti
, status_type_t type
,
1722 unsigned status_flags
, char *result
, unsigned maxlen
)
1724 struct crypt_config
*cc
= ti
->private;
1725 unsigned int sz
= 0;
1728 case STATUSTYPE_INFO
:
1732 case STATUSTYPE_TABLE
:
1733 DMEMIT("%s ", cc
->cipher_string
);
1735 if (cc
->key_size
> 0) {
1736 if ((maxlen
- sz
) < ((cc
->key_size
<< 1) + 1))
1739 crypt_encode_key(result
+ sz
, cc
->key
, cc
->key_size
);
1740 sz
+= cc
->key_size
<< 1;
1747 DMEMIT(" %llu %s %llu", (unsigned long long)cc
->iv_offset
,
1748 cc
->dev
->name
, (unsigned long long)cc
->start
);
1750 if (ti
->num_discard_requests
)
1751 DMEMIT(" 1 allow_discards");
1758 static void crypt_postsuspend(struct dm_target
*ti
)
1760 struct crypt_config
*cc
= ti
->private;
1762 set_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
1765 static int crypt_preresume(struct dm_target
*ti
)
1767 struct crypt_config
*cc
= ti
->private;
1769 if (!test_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
)) {
1770 DMERR("aborting resume - crypt key is not set.");
1777 static void crypt_resume(struct dm_target
*ti
)
1779 struct crypt_config
*cc
= ti
->private;
1781 clear_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
1784 /* Message interface
1788 static int crypt_message(struct dm_target
*ti
, unsigned argc
, char **argv
)
1790 struct crypt_config
*cc
= ti
->private;
1796 if (!strcasecmp(argv
[0], "key")) {
1797 if (!test_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
)) {
1798 DMWARN("not suspended during key manipulation.");
1801 if (argc
== 3 && !strcasecmp(argv
[1], "set")) {
1802 ret
= crypt_set_key(cc
, argv
[2]);
1805 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->init
)
1806 ret
= cc
->iv_gen_ops
->init(cc
);
1809 if (argc
== 2 && !strcasecmp(argv
[1], "wipe")) {
1810 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->wipe
) {
1811 ret
= cc
->iv_gen_ops
->wipe(cc
);
1815 return crypt_wipe_key(cc
);
1820 DMWARN("unrecognised message received.");
1824 static int crypt_merge(struct dm_target
*ti
, struct bvec_merge_data
*bvm
,
1825 struct bio_vec
*biovec
, int max_size
)
1827 struct crypt_config
*cc
= ti
->private;
1828 struct request_queue
*q
= bdev_get_queue(cc
->dev
->bdev
);
1830 if (!q
->merge_bvec_fn
)
1833 bvm
->bi_bdev
= cc
->dev
->bdev
;
1834 bvm
->bi_sector
= cc
->start
+ dm_target_offset(ti
, bvm
->bi_sector
);
1836 return min(max_size
, q
->merge_bvec_fn(q
, bvm
, biovec
));
1839 static int crypt_iterate_devices(struct dm_target
*ti
,
1840 iterate_devices_callout_fn fn
, void *data
)
1842 struct crypt_config
*cc
= ti
->private;
1844 return fn(ti
, cc
->dev
, cc
->start
, ti
->len
, data
);
1847 static struct target_type crypt_target
= {
1849 .version
= {1, 11, 0},
1850 .module
= THIS_MODULE
,
1854 .status
= crypt_status
,
1855 .postsuspend
= crypt_postsuspend
,
1856 .preresume
= crypt_preresume
,
1857 .resume
= crypt_resume
,
1858 .message
= crypt_message
,
1859 .merge
= crypt_merge
,
1860 .iterate_devices
= crypt_iterate_devices
,
1863 static int __init
dm_crypt_init(void)
1867 _crypt_io_pool
= KMEM_CACHE(dm_crypt_io
, 0);
1868 if (!_crypt_io_pool
)
1871 r
= dm_register_target(&crypt_target
);
1873 DMERR("register failed %d", r
);
1874 kmem_cache_destroy(_crypt_io_pool
);
1880 static void __exit
dm_crypt_exit(void)
1882 dm_unregister_target(&crypt_target
);
1883 kmem_cache_destroy(_crypt_io_pool
);
1886 module_init(dm_crypt_init
);
1887 module_exit(dm_crypt_exit
);
1889 MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
1890 MODULE_DESCRIPTION(DM_NAME
" target for transparent encryption / decryption");
1891 MODULE_LICENSE("GPL");