add patch use-swap-in-memswap
[ext4-patch-queue.git] / crypto-use-per-inode-tfm
blob66da8c40c559b0ae3020d063201eab02833e2681
1 ext4 crypto: use per-inode tfm structure
3 As suggested by Herbert Xu, we shouldn't allocate a new tfm each time
4 we read or write a page.  Instead we can use a single tfm hanging off
5 the inode's crypt_info structure for all of our encryption needs for
6 that inode, since the tfm can be used by multiple crypto requests in
7 parallel.
9 Also use cmpxchg() to avoid races that could result in crypt_info
10 structure getting doubly allocated or doubly freed.
12 Signed-off-by: Theodore Ts'o <tytso@mit.edu>
13 diff --git a/fs/ext4/crypto.c b/fs/ext4/crypto.c
14 index 28a0e4bd..c3a9b08 100644
15 --- a/fs/ext4/crypto.c
16 +++ b/fs/ext4/crypto.c
17 @@ -80,8 +80,6 @@ void ext4_release_crypto_ctx(struct ext4_crypto_ctx *ctx)
18         ctx->w.bounce_page = NULL;
19         ctx->w.control_page = NULL;
20         if (ctx->flags & EXT4_CTX_REQUIRES_FREE_ENCRYPT_FL) {
21 -               if (ctx->tfm)
22 -                       crypto_free_tfm(ctx->tfm);
23                 kmem_cache_free(ext4_crypto_ctx_cachep, ctx);
24         } else {
25                 spin_lock_irqsave(&ext4_crypto_ctx_lock, flags);
26 @@ -136,36 +134,6 @@ struct ext4_crypto_ctx *ext4_get_crypto_ctx(struct inode *inode)
27         }
28         ctx->flags &= ~EXT4_WRITE_PATH_FL;
30 -       /* Allocate a new Crypto API context if we don't already have
31 -        * one or if it isn't the right mode. */
32 -       if (ctx->tfm && (ctx->mode != ci->ci_data_mode)) {
33 -               crypto_free_tfm(ctx->tfm);
34 -               ctx->tfm = NULL;
35 -               ctx->mode = EXT4_ENCRYPTION_MODE_INVALID;
36 -       }
37 -       if (!ctx->tfm) {
38 -               switch (ci->ci_data_mode) {
39 -               case EXT4_ENCRYPTION_MODE_AES_256_XTS:
40 -                       ctx->tfm = crypto_ablkcipher_tfm(
41 -                               crypto_alloc_ablkcipher("xts(aes)", 0, 0));
42 -                       break;
43 -               case EXT4_ENCRYPTION_MODE_AES_256_GCM:
44 -                       /* TODO(mhalcrow): AEAD w/ gcm(aes);
45 -                        * crypto_aead_setauthsize() */
46 -                       ctx->tfm = ERR_PTR(-ENOTSUPP);
47 -                       break;
48 -               default:
49 -                       BUG();
50 -               }
51 -               if (IS_ERR_OR_NULL(ctx->tfm)) {
52 -                       res = PTR_ERR(ctx->tfm);
53 -                       ctx->tfm = NULL;
54 -                       goto out;
55 -               }
56 -               ctx->mode = ci->ci_data_mode;
57 -       }
58 -       BUG_ON(ci->ci_size != ext4_encryption_key_size(ci->ci_data_mode));
60  out:
61         if (res) {
62                 if (!IS_ERR_OR_NULL(ctx))
63 @@ -185,11 +153,8 @@ void ext4_exit_crypto(void)
64  {
65         struct ext4_crypto_ctx *pos, *n;
67 -       list_for_each_entry_safe(pos, n, &ext4_free_crypto_ctxs, free_list) {
68 -               if (pos->tfm)
69 -                       crypto_free_tfm(pos->tfm);
70 +       list_for_each_entry_safe(pos, n, &ext4_free_crypto_ctxs, free_list)
71                 kmem_cache_free(ext4_crypto_ctx_cachep, pos);
72 -       }
73         INIT_LIST_HEAD(&ext4_free_crypto_ctxs);
74         if (ext4_bounce_page_pool)
75                 mempool_destroy(ext4_bounce_page_pool);
76 @@ -303,32 +268,11 @@ static int ext4_page_crypto(struct ext4_crypto_ctx *ctx,
77         struct ablkcipher_request *req = NULL;
78         DECLARE_EXT4_COMPLETION_RESULT(ecr);
79         struct scatterlist dst, src;
80 -       struct ext4_inode_info *ei = EXT4_I(inode);
81 -       struct crypto_ablkcipher *atfm = __crypto_ablkcipher_cast(ctx->tfm);
82 +       struct ext4_crypt_info *ci = EXT4_I(inode)->i_crypt_info;
83 +       struct crypto_ablkcipher *tfm = ci->ci_ctfm;
84         int res = 0;
86 -       BUG_ON(!ctx->tfm);
87 -       BUG_ON(ctx->mode != ei->i_crypt_info->ci_data_mode);
89 -       if (ctx->mode != EXT4_ENCRYPTION_MODE_AES_256_XTS) {
90 -               printk_ratelimited(KERN_ERR
91 -                                  "%s: unsupported crypto algorithm: %d\n",
92 -                                  __func__, ctx->mode);
93 -               return -ENOTSUPP;
94 -       }
96 -       crypto_ablkcipher_clear_flags(atfm, ~0);
97 -       crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_REQ_WEAK_KEY);
99 -       res = crypto_ablkcipher_setkey(atfm, ei->i_crypt_info->ci_raw,
100 -                                      ei->i_crypt_info->ci_size);
101 -       if (res) {
102 -               printk_ratelimited(KERN_ERR
103 -                                  "%s: crypto_ablkcipher_setkey() failed\n",
104 -                                  __func__);
105 -               return res;
106 -       }
107 -       req = ablkcipher_request_alloc(atfm, GFP_NOFS);
108 +       req = ablkcipher_request_alloc(tfm, GFP_NOFS);
109         if (!req) {
110                 printk_ratelimited(KERN_ERR
111                                    "%s: crypto_request_alloc() failed\n",
112 diff --git a/fs/ext4/crypto_fname.c b/fs/ext4/crypto_fname.c
113 index e63dd29..29a2dc9 100644
114 --- a/fs/ext4/crypto_fname.c
115 +++ b/fs/ext4/crypto_fname.c
116 @@ -252,52 +252,6 @@ static int digest_decode(const char *src, int len, char *dst)
117         return cp - dst;
120 -int ext4_setup_fname_crypto(struct inode *inode)
122 -       struct ext4_inode_info *ei = EXT4_I(inode);
123 -       struct ext4_crypt_info *ci = ei->i_crypt_info;
124 -       struct crypto_ablkcipher *ctfm;
125 -       int res;
127 -       /* Check if the crypto policy is set on the inode */
128 -       res = ext4_encrypted_inode(inode);
129 -       if (res == 0)
130 -               return 0;
132 -       res = ext4_get_encryption_info(inode);
133 -       if (res < 0)
134 -               return res;
135 -       ci = ei->i_crypt_info;
137 -       if (!ci || ci->ci_ctfm)
138 -               return 0;
140 -       if (ci->ci_filename_mode != EXT4_ENCRYPTION_MODE_AES_256_CTS) {
141 -               printk_once(KERN_WARNING "ext4: unsupported key mode %d\n",
142 -                           ci->ci_filename_mode);
143 -               return -ENOKEY;
144 -       }
146 -       ctfm = crypto_alloc_ablkcipher("cts(cbc(aes))", 0, 0);
147 -       if (!ctfm || IS_ERR(ctfm)) {
148 -               res = ctfm ? PTR_ERR(ctfm) : -ENOMEM;
149 -               printk(KERN_DEBUG "%s: error (%d) allocating crypto tfm\n",
150 -                      __func__, res);
151 -               return res;
152 -       }
153 -       crypto_ablkcipher_clear_flags(ctfm, ~0);
154 -       crypto_tfm_set_flags(crypto_ablkcipher_tfm(ctfm),
155 -                            CRYPTO_TFM_REQ_WEAK_KEY);
157 -       res = crypto_ablkcipher_setkey(ctfm, ci->ci_raw, ci->ci_size);
158 -       if (res) {
159 -               crypto_free_ablkcipher(ctfm);
160 -               return -EIO;
161 -       }
162 -       ci->ci_ctfm = ctfm;
163 -       return 0;
166  /**
167   * ext4_fname_crypto_round_up() -
168   *
169 @@ -449,7 +403,7 @@ int ext4_fname_setup_filename(struct inode *dir, const struct qstr *iname,
170                 fname->disk_name.len = iname->len;
171                 goto out;
172         }
173 -       ret = ext4_setup_fname_crypto(dir);
174 +       ret = ext4_get_encryption_info(dir);
175         if (ret)
176                 return ret;
177         ci = EXT4_I(dir)->i_crypt_info;
178 diff --git a/fs/ext4/crypto_key.c b/fs/ext4/crypto_key.c
179 index 858d7d6..442d24e 100644
180 --- a/fs/ext4/crypto_key.c
181 +++ b/fs/ext4/crypto_key.c
182 @@ -84,20 +84,32 @@ out:
183         return res;
186 -void ext4_free_encryption_info(struct inode *inode)
187 +void ext4_free_crypt_info(struct ext4_crypt_info *ci)
189 -       struct ext4_inode_info *ei = EXT4_I(inode);
190 -       struct ext4_crypt_info *ci = ei->i_crypt_info;
192         if (!ci)
193                 return;
195         if (ci->ci_keyring_key)
196                 key_put(ci->ci_keyring_key);
197         crypto_free_ablkcipher(ci->ci_ctfm);
198 -       memzero_explicit(&ci->ci_raw, sizeof(ci->ci_raw));
199         kmem_cache_free(ext4_crypt_info_cachep, ci);
200 -       ei->i_crypt_info = NULL;
203 +void ext4_free_encryption_info(struct inode *inode,
204 +                              struct ext4_crypt_info *ci)
206 +       struct ext4_inode_info *ei = EXT4_I(inode);
207 +       struct ext4_crypt_info *prev;
209 +       if (ci == NULL)
210 +               ci = ACCESS_ONCE(ei->i_crypt_info);
211 +       if (ci == NULL)
212 +               return;
213 +       prev = cmpxchg(&ei->i_crypt_info, ci, NULL);
214 +       if (prev != ci)
215 +               return;
217 +       ext4_free_crypt_info(ci);
220  int _ext4_get_encryption_info(struct inode *inode)
221 @@ -111,6 +123,10 @@ int _ext4_get_encryption_info(struct inode *inode)
222         struct ext4_encryption_context ctx;
223         struct user_key_payload *ukp;
224         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
225 +       struct crypto_ablkcipher *ctfm;
226 +       const char *cipher_str;
227 +       char raw_key[EXT4_MAX_KEY_SIZE];
228 +       char mode;
229         int res;
231         if (!ext4_read_workqueue) {
232 @@ -119,11 +135,14 @@ int _ext4_get_encryption_info(struct inode *inode)
233                         return res;
234         }
236 -       if (ei->i_crypt_info) {
237 -               if (!ei->i_crypt_info->ci_keyring_key ||
238 -                   key_validate(ei->i_crypt_info->ci_keyring_key) == 0)
239 +retry:
240 +       crypt_info = ACCESS_ONCE(ei->i_crypt_info);
241 +       if (crypt_info) {
242 +               if (!crypt_info->ci_keyring_key ||
243 +                   key_validate(crypt_info->ci_keyring_key) == 0)
244                         return 0;
245 -               ext4_free_encryption_info(inode);
246 +               ext4_free_encryption_info(inode, crypt_info);
247 +               goto retry;
248         }
250         res = ext4_xattr_get(inode, EXT4_XATTR_INDEX_ENCRYPTION,
251 @@ -144,26 +163,37 @@ int _ext4_get_encryption_info(struct inode *inode)
252         if (!crypt_info)
253                 return -ENOMEM;
255 -       ei->i_crypt_policy_flags = ctx.flags;
256         crypt_info->ci_flags = ctx.flags;
257         crypt_info->ci_data_mode = ctx.contents_encryption_mode;
258         crypt_info->ci_filename_mode = ctx.filenames_encryption_mode;
259         crypt_info->ci_ctfm = NULL;
260 +       crypt_info->ci_keyring_key = NULL;
261         memcpy(crypt_info->ci_master_key, ctx.master_key_descriptor,
262                sizeof(crypt_info->ci_master_key));
263         if (S_ISREG(inode->i_mode))
264 -               crypt_info->ci_size =
265 -                       ext4_encryption_key_size(crypt_info->ci_data_mode);
266 +               mode = crypt_info->ci_data_mode;
267         else if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
268 -               crypt_info->ci_size =
269 -                       ext4_encryption_key_size(crypt_info->ci_filename_mode);
270 +               mode = crypt_info->ci_filename_mode;
271         else
272                 BUG();
273 -       BUG_ON(!crypt_info->ci_size);
274 -       if (DUMMY_ENCRYPTION_ENABLED(sbi)) {
275 -               memset(crypt_info->ci_raw, 0x42, EXT4_AES_256_XTS_KEY_SIZE);
276 +       switch (mode) {
277 +       case EXT4_ENCRYPTION_MODE_AES_256_XTS:
278 +               cipher_str = "xts(aes)";
279 +               break;
280 +       case EXT4_ENCRYPTION_MODE_AES_256_CTS:
281 +               cipher_str = "cts(cbc(aes))";
282 +               break;
283 +       default:
284 +               printk_once(KERN_WARNING
285 +                           "ext4: unsupported key mode %d (ino %u)\n",
286 +                           mode, (unsigned) inode->i_ino);
287 +               res = -ENOKEY;
288                 goto out;
289         }
290 +       if (DUMMY_ENCRYPTION_ENABLED(sbi)) {
291 +               memset(raw_key, 0x42, EXT4_AES_256_XTS_KEY_SIZE);
292 +               goto got_key;
293 +       }
294         memcpy(full_key_descriptor, EXT4_KEY_DESC_PREFIX,
295                EXT4_KEY_DESC_PREFIX_SIZE);
296         sprintf(full_key_descriptor + EXT4_KEY_DESC_PREFIX_SIZE,
297 @@ -177,6 +207,7 @@ int _ext4_get_encryption_info(struct inode *inode)
298                 keyring_key = NULL;
299                 goto out;
300         }
301 +       crypt_info->ci_keyring_key = keyring_key;
302         BUG_ON(keyring_key->type != &key_type_logon);
303         ukp = ((struct user_key_payload *)keyring_key->payload.data);
304         if (ukp->datalen != sizeof(struct ext4_encryption_key)) {
305 @@ -188,19 +219,36 @@ int _ext4_get_encryption_info(struct inode *inode)
306                      EXT4_KEY_DERIVATION_NONCE_SIZE);
307         BUG_ON(master_key->size != EXT4_AES_256_XTS_KEY_SIZE);
308         res = ext4_derive_key_aes(ctx.nonce, master_key->raw,
309 -                                 crypt_info->ci_raw);
310 -out:
311 -       if (res < 0) {
312 -               if (res == -ENOKEY)
313 -                       res = 0;
314 -               kmem_cache_free(ext4_crypt_info_cachep, crypt_info);
315 -       } else {
316 -               ei->i_crypt_info = crypt_info;
317 -               crypt_info->ci_keyring_key = keyring_key;
318 -               keyring_key = NULL;
319 +                                 raw_key);
320 +got_key:
321 +       ctfm = crypto_alloc_ablkcipher(cipher_str, 0, 0);
322 +       if (!ctfm || IS_ERR(ctfm)) {
323 +               res = ctfm ? PTR_ERR(ctfm) : -ENOMEM;
324 +               printk(KERN_DEBUG
325 +                      "%s: error %d (inode %u) allocating crypto tfm\n",
326 +                      __func__, res, (unsigned) inode->i_ino);
327 +               goto out;
328 +       }
329 +       crypt_info->ci_ctfm = ctfm;
330 +       crypto_ablkcipher_clear_flags(ctfm, ~0);
331 +       crypto_tfm_set_flags(crypto_ablkcipher_tfm(ctfm),
332 +                            CRYPTO_TFM_REQ_WEAK_KEY);
333 +       res = crypto_ablkcipher_setkey(ctfm, raw_key,
334 +                                      ext4_encryption_key_size(mode));
335 +       if (res)
336 +               goto out;
337 +       memzero_explicit(raw_key, sizeof(raw_key));
338 +       if (cmpxchg(&ei->i_crypt_info, NULL, crypt_info) != NULL) {
339 +               ext4_free_crypt_info(crypt_info);
340 +               goto retry;
341         }
342 -       if (keyring_key)
343 -               key_put(keyring_key);
344 +       return 0;
346 +out:
347 +       if (res == -ENOKEY)
348 +               res = 0;
349 +       ext4_free_crypt_info(crypt_info);
350 +       memzero_explicit(raw_key, sizeof(raw_key));
351         return res;
354 diff --git a/fs/ext4/dir.c b/fs/ext4/dir.c
355 index 28cb94f..e11e6ae 100644
356 --- a/fs/ext4/dir.c
357 +++ b/fs/ext4/dir.c
358 @@ -133,9 +133,6 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx)
359                         return err;
360         }
362 -       err = ext4_setup_fname_crypto(inode);
363 -       if (err)
364 -               return err;
365         if (ext4_encrypted_inode(inode)) {
366                 err = ext4_fname_crypto_alloc_buffer(inode, EXT4_NAME_LEN,
367                                                      &fname_crypto_str);
368 diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
369 index 23e33fb..7435ff2 100644
370 --- a/fs/ext4/ext4.h
371 +++ b/fs/ext4/ext4.h
372 @@ -911,7 +911,6 @@ struct ext4_inode_info {
374         /* on-disk additional length */
375         __u16 i_extra_isize;
376 -       char i_crypt_policy_flags;
378         /* Indicate the inline data space. */
379         u16 i_inline_off;
380 @@ -2105,7 +2104,6 @@ int ext4_fname_usr_to_disk(struct inode *inode,
381                            const struct qstr *iname,
382                            struct ext4_str *oname);
383  #ifdef CONFIG_EXT4_FS_ENCRYPTION
384 -int ext4_setup_fname_crypto(struct inode *inode);
385  void ext4_fname_crypto_free_buffer(struct ext4_str *crypto_str);
386  int ext4_fname_setup_filename(struct inode *dir, const struct qstr *iname,
387                               int lookup, struct ext4_filename *fname);
388 @@ -2131,7 +2129,8 @@ static inline void ext4_fname_free_filename(struct ext4_filename *fname) { }
391  /* crypto_key.c */
392 -void ext4_free_encryption_info(struct inode *inode);
393 +void ext4_free_crypt_info(struct ext4_crypt_info *ci);
394 +void ext4_free_encryption_info(struct inode *inode, struct ext4_crypt_info *ci);
395  int _ext4_get_encryption_info(struct inode *inode);
397  #ifdef CONFIG_EXT4_FS_ENCRYPTION
398 diff --git a/fs/ext4/ext4_crypto.h b/fs/ext4/ext4_crypto.h
399 index c5258f2..34e0d24 100644
400 --- a/fs/ext4/ext4_crypto.h
401 +++ b/fs/ext4/ext4_crypto.h
402 @@ -74,13 +74,11 @@ struct ext4_encryption_key {
403  } __attribute__((__packed__));
405  struct ext4_crypt_info {
406 -       unsigned char   ci_size;
407         char            ci_data_mode;
408         char            ci_filename_mode;
409         char            ci_flags;
410         struct crypto_ablkcipher *ci_ctfm;
411         struct key      *ci_keyring_key;
412 -       char            ci_raw[EXT4_MAX_KEY_SIZE];
413         char            ci_master_key[EXT4_KEY_DESCRIPTOR_SIZE];
414  };
416 @@ -89,7 +87,6 @@ struct ext4_crypt_info {
417  #define EXT4_WRITE_PATH_FL                           0x00000004
419  struct ext4_crypto_ctx {
420 -       struct crypto_tfm *tfm;         /* Crypto API context */
421         union {
422                 struct {
423                         struct page *bounce_page;       /* Ciphertext page */
424 diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c
425 index 9bed99f..6ab50f8 100644
426 --- a/fs/ext4/namei.c
427 +++ b/fs/ext4/namei.c
428 @@ -607,11 +607,12 @@ static struct stats dx_show_leaf(struct inode *dir,
429                                 char *name;
430                                 struct ext4_str fname_crypto_str
431                                         = {.name = NULL, .len = 0};
432 -                               int res;
433 +                               int res = 0;
435                                 name  = de->name;
436                                 len = de->name_len;
437 -                               res = ext4_setup_fname_crypto(dir);
438 +                               if (ext4_encrypted_inode(inode))
439 +                                       res = ext4_get_encryption_info(dir);
440                                 if (res) {
441                                         printk(KERN_WARNING "Error setting up"
442                                                " fname crypto: %d\n", res);
443 @@ -953,12 +954,12 @@ static int htree_dirblock_to_tree(struct file *dir_file,
444                                            EXT4_DIR_REC_LEN(0));
445  #ifdef CONFIG_EXT4_FS_ENCRYPTION
446         /* Check if the directory is encrypted */
447 -       err = ext4_setup_fname_crypto(dir);
448 -       if (err) {
449 -               brelse(bh);
450 -               return err;
451 -       }
452         if (ext4_encrypted_inode(dir)) {
453 +               err = ext4_get_encryption_info(dir);
454 +               if (err < 0) {
455 +                       brelse(bh);
456 +                       return err;
457 +               }
458                 err = ext4_fname_crypto_alloc_buffer(dir, EXT4_NAME_LEN,
459                                                      &fname_crypto_str);
460                 if (err < 0) {
461 @@ -3108,7 +3109,7 @@ static int ext4_symlink(struct inode *dir,
462                 err = ext4_inherit_context(dir, inode);
463                 if (err)
464                         goto err_drop_inode;
465 -               err = ext4_setup_fname_crypto(inode);
466 +               err = ext4_get_encryption_info(inode);
467                 if (err)
468                         goto err_drop_inode;
469                 istr.name = (const unsigned char *) symname;
470 diff --git a/fs/ext4/super.c b/fs/ext4/super.c
471 index b0bd1c1..56bfc2f 100644
472 --- a/fs/ext4/super.c
473 +++ b/fs/ext4/super.c
474 @@ -959,7 +959,7 @@ void ext4_clear_inode(struct inode *inode)
475         }
476  #ifdef CONFIG_EXT4_FS_ENCRYPTION
477         if (EXT4_I(inode)->i_crypt_info)
478 -               ext4_free_encryption_info(inode);
479 +               ext4_free_encryption_info(inode, EXT4_I(inode)->i_crypt_info);
480  #endif
483 diff --git a/fs/ext4/symlink.c b/fs/ext4/symlink.c
484 index 3287088..68e915a 100644
485 --- a/fs/ext4/symlink.c
486 +++ b/fs/ext4/symlink.c
487 @@ -37,7 +37,7 @@ static void *ext4_follow_link(struct dentry *dentry, struct nameidata *nd)
488         if (!ext4_encrypted_inode(inode))
489                 return page_follow_link_light(dentry, nd);
491 -       res = ext4_setup_fname_crypto(inode);
492 +       res = ext4_get_encryption_info(inode);
493         if (res)
494                 return ERR_PTR(res);