| blob | 124e9d5370572024cc72339fe2a15acaeb9dc87d |
1 ext4: support block size != page size
3 From: Michael Halcrow <mhalcrow@google.com>
5 After enough time hemming and hawing over taking additional
6 dependencies on buffer head, I decided to go ahead and just do it to
7 see if it looks like a viable approach.
9 This patch built a kernel with block size == 1024. It's not
10 particularly efficient, in that it allocates a struct bio for every
11 buffer head. With some additional complexity, I probably could
12 coalesce several buffer heads for one page into a single bio
13 completion.
15 Ted, if you think this is a reasonable thing to do, please let me
16 know. You can also let me know if you think it's an unreasonable thing
17 to do.
19 Signed-off-by: Michael Halcrow <mhalcrow@google.com>
20 Signed-off-by: Theodore Ts'o <tytso@mit.edu>
21 ---
22 fs/buffer.c | 23 +++++---
23 fs/ext4/crypto.c | 112 +++++++++++++++++++------------------
24 fs/ext4/ext4.h | 10 +++-
25 fs/ext4/ext4_crypto.h | 15 +++--
26 fs/ext4/inode.c | 150 ++++++++++++++++++++++++++++++++++----------------
27 fs/ext4/page-io.c | 141 ++++++++++++++++++++++++++++++-----------------
28 6 files changed, 286 insertions(+), 165 deletions(-)
30 diff --git a/fs/buffer.c b/fs/buffer.c
31 index f2dcb6b..2506de5 100644
32 --- a/fs/buffer.c
33 +++ b/fs/buffer.c
34 @@ -289,6 +289,22 @@ void end_buffer_async_read(struct buffer_head *bh, int uptodate)
35 page = bh->b_page;
36 if (uptodate) {
37 set_buffer_uptodate(bh);
38 + if (bh->b_private) {
39 + /*
40 + * Encryption involves a different buffer head tracking
41 + * mechanism. There's an atomic counter with a shared
42 + * crypto context for each page. When that hits 0, then
43 + * the callback sets the page uptodate and unlocks the
44 + * page.
45 + */
46 + struct bio *bio = (struct bio *)bh->b_private;
47 +
48 + BUG_ON(!bio->bi_cb);
49 + bio->bi_cb(bio, 0);
50 + clear_buffer_async_read(bh);
51 + unlock_buffer(bh);
52 + goto out;
53 + }
54 } else {
55 clear_buffer_uptodate(bh);
56 buffer_io_error(bh, ", async page read");
57 @@ -318,13 +334,6 @@ void end_buffer_async_read(struct buffer_head *bh, int uptodate)
58 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
59 local_irq_restore(flags);
61 - if (bh->b_private) {
62 - struct bio *bio = (struct bio *)bh->b_private;
63 - BUG_ON(!bio->bi_cb);
64 - if (!bio->bi_cb(bio, !(page_uptodate && !PageError(page))))
65 - goto out;
66 - }
67 -
68 /*
69 * If none of the buffers had errors and they are all
70 * uptodate then we can set the page uptodate.
71 diff --git a/fs/ext4/crypto.c b/fs/ext4/crypto.c
72 index d655a64..6150574 100644
73 --- a/fs/ext4/crypto.c
74 +++ b/fs/ext4/crypto.c
75 @@ -14,6 +14,7 @@
76 * derivation code implements an HKDF (see RFC 5869).
77 */
79 +#include <crypto/aes.h>
80 #include <crypto/hash.h>
81 #include <crypto/sha.h>
82 #include <keys/user-type.h>
83 @@ -167,6 +168,8 @@ struct ext4_crypto_ctx *ext4_get_crypto_ctx(
84 }
85 BUG_ON(key->size != ext4_encryption_key_size(key->mode));
87 + mutex_init(&ctx->tfm_mutex);
88 +
89 /* There shouldn't be a bounce page attached to the crypto
90 * context at this point. */
91 BUG_ON(ctx->bounce_page);
92 @@ -283,22 +286,25 @@ fail:
93 }
95 /**
96 - * ext4_xts_tweak_for_page() - Generates an XTS tweak for a page
97 + * ext4_xts_tweak() - Generates an XTS tweak
98 * @xts_tweak: Buffer into which this writes the XTS tweak.
99 * @page: The page for which this generates a tweak.
100 + * @offset: The offset within the page.
101 *
102 - * Generates an XTS tweak value for the given page.
103 + * Generates an XTS tweak value for the given page and offset within the page.
104 */
105 -static void ext4_xts_tweak_for_page(u8 xts_tweak[EXT4_XTS_TWEAK_SIZE],
106 - const struct page *page)
107 +static void ext4_xts_tweak(u8 xts_tweak[EXT4_XTS_TWEAK_SIZE],
108 + const struct page *page, size_t offset)
109 {
110 + size_t logical_offset = (page->index << PAGE_CACHE_SHIFT) + offset;
111 +
112 /* Only do this for XTS tweak values. For other modes (CBC,
113 * GCM, etc.), you most like will need to do something
114 * different. */
115 - BUILD_BUG_ON(EXT4_XTS_TWEAK_SIZE < sizeof(page->index));
116 - memcpy(xts_tweak, &page->index, sizeof(page->index));
117 - memset(&xts_tweak[sizeof(page->index)], 0,
118 - EXT4_XTS_TWEAK_SIZE - sizeof(page->index));
119 + BUILD_BUG_ON(EXT4_XTS_TWEAK_SIZE < sizeof(offset));
120 + memcpy(xts_tweak, &logical_offset, sizeof(logical_offset));
121 + memset(&xts_tweak[sizeof(logical_offset)], 0,
122 + EXT4_XTS_TWEAK_SIZE - sizeof(logical_offset));
123 }
125 /**
126 @@ -345,9 +351,9 @@ static void ext4_crypt_complete(struct crypto_async_request *req, int res)
127 * @plaintext_page: Plaintext page that acts as a control page.
128 * @ctx: Encryption context for the pages.
129 */
130 -static void ext4_prep_pages_for_write(struct page *ciphertext_page,
131 - struct page *plaintext_page,
132 - struct ext4_crypto_ctx *ctx)
133 +void ext4_prep_pages_for_write(struct page *ciphertext_page,
134 + struct page *plaintext_page,
135 + struct ext4_crypto_ctx *ctx)
136 {
137 SetPageDirty(ciphertext_page);
138 SetPagePrivate(ciphertext_page);
139 @@ -360,17 +366,18 @@ static void ext4_prep_pages_for_write(struct page *ciphertext_page,
140 * ext4_xts_encrypt() - Encrypts a page using AES-256-XTS
141 * @ctx: The encryption context.
142 * @plaintext_page: The page to encrypt. Must be locked.
143 + * @offset: The offset within the page to encrypt.
144 + * @size: The size of the region to encrypt.
145 *
146 - * Allocates a ciphertext page and encrypts plaintext_page into it using the ctx
147 - * encryption context. Uses AES-256-XTS.
148 + * Encrypts plaintext_page into the ctx bounce page using the ctx encryption
149 + * context. Uses AES-256-XTS.
150 *
151 * Called on the page write path.
152 *
153 - * Return: An allocated page with the encrypted content on success. Else, an
154 - * error value or NULL.
155 + * Return: Zero on success; non-zero otherwise.
156 */
157 -static struct page *ext4_xts_encrypt(struct ext4_crypto_ctx *ctx,
158 - struct page *plaintext_page)
159 +int ext4_xts_encrypt(struct ext4_crypto_ctx *ctx, struct page *plaintext_page,
160 + size_t offset, size_t size)
161 {
162 struct page *ciphertext_page = ctx->bounce_page;
163 u8 xts_tweak[EXT4_XTS_TWEAK_SIZE];
164 @@ -384,6 +391,10 @@ static struct page *ext4_xts_encrypt(struct ext4_crypto_ctx *ctx,
165 BUG_ON(!ciphertext_page);
166 BUG_ON(!ctx->tfm);
167 BUG_ON(ei->i_encryption_key.mode != EXT4_ENCRYPTION_MODE_AES_256_XTS);
168 + BUG_ON(offset % AES_BLOCK_SIZE != 0);
169 + BUG_ON(size % AES_BLOCK_SIZE != 0);
170 + BUG_ON(offset + size > PAGE_CACHE_SIZE);
171 +
172 crypto_ablkcipher_clear_flags(atfm, ~0);
173 crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_REQ_WEAK_KEY);
175 @@ -392,24 +403,22 @@ static struct page *ext4_xts_encrypt(struct ext4_crypto_ctx *ctx,
176 * single key can encrypt, we directly use the inode master key */
177 res = crypto_ablkcipher_setkey(atfm, ei->i_encryption_key.raw,
178 ei->i_encryption_key.size);
179 + if (IS_ERR_VALUE(res))
180 + goto out;
181 req = ablkcipher_request_alloc(atfm, GFP_NOFS);
182 if (!req) {
183 - printk_ratelimited(KERN_ERR
184 - "%s: crypto_request_alloc() failed\n",
185 - __func__);
186 - ciphertext_page = ERR_PTR(-ENOMEM);
187 + res = -ENOMEM;
188 goto out;
189 }
190 ablkcipher_request_set_callback(
191 req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
192 ext4_crypt_complete, &ecr);
193 - ext4_xts_tweak_for_page(xts_tweak, plaintext_page);
194 + ext4_xts_tweak(xts_tweak, plaintext_page, offset);
195 sg_init_table(&dst, 1);
196 - sg_set_page(&dst, ciphertext_page, PAGE_CACHE_SIZE, 0);
197 + sg_set_page(&dst, ciphertext_page, size, offset);
198 sg_init_table(&src, 1);
199 - sg_set_page(&src, plaintext_page, PAGE_CACHE_SIZE, 0);
200 - ablkcipher_request_set_crypt(req, &src, &dst, PAGE_CACHE_SIZE,
201 - xts_tweak);
202 + sg_set_page(&src, plaintext_page, size, offset);
203 + ablkcipher_request_set_crypt(req, &src, &dst, size, xts_tweak);
204 res = crypto_ablkcipher_encrypt(req);
205 if (res == -EINPROGRESS || res == -EBUSY) {
206 BUG_ON(req->base.data != &ecr);
207 @@ -417,40 +426,33 @@ static struct page *ext4_xts_encrypt(struct ext4_crypto_ctx *ctx,
208 res = ecr.res;
209 }
210 ablkcipher_request_free(req);
211 - if (res) {
212 - printk_ratelimited(
213 - KERN_ERR
214 - "%s: crypto_ablkcipher_encrypt() returned %d\n",
215 - __func__, res);
216 - ciphertext_page = ERR_PTR(res);
217 - goto out;
218 - }
219 out:
220 - return ciphertext_page;
221 + return res;
222 }
224 /**
225 * ext4_encrypt() - Encrypts a page
226 * @ctx: The encryption context.
227 * @plaintext_page: The page to encrypt. Must be locked.
228 + * @offset: The offset within the page to encrypt.
229 + * @size: The size of the region to encrypt.
230 *
231 * Allocates a ciphertext page and encrypts plaintext_page into it using the ctx
232 * encryption context.
233 *
234 * Called on the page write path.
235 *
236 - * Return: An allocated page with the encrypted content on success. Else, an
237 - * error value or NULL.
238 + * Return: Zero on success; non-zero otherwise.
239 */
240 -struct page *ext4_encrypt(struct ext4_crypto_ctx *ctx,
241 - struct page *plaintext_page)
242 +int ext4_encrypt(struct ext4_crypto_ctx *ctx, struct page *plaintext_page,
243 + size_t offset, size_t size)
244 {
245 - struct page *ciphertext_page = NULL;
246 + int res = -EINVAL;
248 BUG_ON(!PageLocked(plaintext_page));
249 switch (ctx->mode) {
250 case EXT4_ENCRYPTION_MODE_AES_256_XTS:
251 - ciphertext_page = ext4_xts_encrypt(ctx, plaintext_page);
252 + res = ext4_xts_encrypt(ctx, plaintext_page, offset, size);
253 break;
254 case EXT4_ENCRYPTION_MODE_AES_256_GCM:
255 /* TODO(mhalcrow): We'll need buffers for the
256 @@ -458,24 +460,25 @@ struct page *ext4_encrypt(struct ext4_crypto_ctx *ctx,
257 * ones below */
258 case EXT4_ENCRYPTION_MODE_HMAC_SHA1:
259 case EXT4_ENCRYPTION_MODE_AES_256_XTS_RANDOM_IV_HMAC_SHA1:
260 - ciphertext_page = ERR_PTR(-ENOTSUPP);
261 + res = -ENOTSUPP;
262 break;
263 default:
264 BUG();
265 }
266 - if (!IS_ERR_OR_NULL(ciphertext_page))
267 - ext4_prep_pages_for_write(ciphertext_page, plaintext_page, ctx);
268 - return ciphertext_page;
269 + return res;
270 }
272 /**
273 * ext4_xts_decrypt() - Decrypts a page using AES-256-XTS
274 - * @ctx: The encryption context.
275 - * @page: The page to decrypt. Must be locked.
276 + * @ctx: The encryption context.
277 + * @page: The page to decrypt. Must be locked.
278 + * @offset: The offset within the page to decrypt.
279 + * @size: The size of the region to decrypt.
280 *
281 * Return: Zero on success, non-zero otherwise.
282 */
283 -static int ext4_xts_decrypt(struct ext4_crypto_ctx *ctx, struct page *page)
284 +static int ext4_xts_decrypt(struct ext4_crypto_ctx *ctx, struct page *page,
285 + size_t offset, size_t size)
286 {
287 u8 xts_tweak[EXT4_XTS_TWEAK_SIZE];
288 struct ablkcipher_request *req = NULL;
289 @@ -503,10 +506,10 @@ static int ext4_xts_decrypt(struct ext4_crypto_ctx *ctx, struct page *page)
290 ablkcipher_request_set_callback(
291 req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
292 ext4_crypt_complete, &ecr);
293 - ext4_xts_tweak_for_page(xts_tweak, page);
294 + ext4_xts_tweak(xts_tweak, page, offset);
295 sg_init_table(&sg, 1);
296 - sg_set_page(&sg, page, PAGE_CACHE_SIZE, 0);
297 - ablkcipher_request_set_crypt(req, &sg, &sg, PAGE_CACHE_SIZE, xts_tweak);
298 + sg_set_page(&sg, page, size, offset);
299 + ablkcipher_request_set_crypt(req, &sg, &sg, size, xts_tweak);
300 res = crypto_ablkcipher_decrypt(req);
301 if (res == -EINPROGRESS || res == -EBUSY) {
302 BUG_ON(req->base.data != &ecr);
303 @@ -524,6 +527,8 @@ out:
304 * ext4_decrypt() - Decrypts a page in-place
305 * @ctx: The encryption context.
306 * @page: The page to decrypt. Must be locked.
307 + * @offset: The offset within the page to decrypt.
308 + * @size: The size of the region to decrypt.
309 *
310 * Decrypts page in-place using the ctx encryption context.
311 *
312 @@ -531,14 +536,15 @@ out:
313 *
314 * Return: Zero on success, non-zero otherwise.
315 */
316 -int ext4_decrypt(struct ext4_crypto_ctx *ctx, struct page *page)
317 +int ext4_decrypt(struct ext4_crypto_ctx *ctx, struct page *page, size_t offset,
318 + size_t size)
319 {
320 int res = 0;
322 BUG_ON(!PageLocked(page));
323 switch (ctx->mode) {
324 case EXT4_ENCRYPTION_MODE_AES_256_XTS:
325 - res = ext4_xts_decrypt(ctx, page);
326 + res = ext4_xts_decrypt(ctx, page, offset, size);
327 break;
328 case EXT4_ENCRYPTION_MODE_AES_256_GCM:
329 case EXT4_ENCRYPTION_MODE_HMAC_SHA1:
330 diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
331 index 292a3a3..4ec0f62 100644
332 --- a/fs/ext4/ext4.h
333 +++ b/fs/ext4/ext4.h
334 @@ -2837,11 +2837,15 @@ struct ext4_crypto_ctx *ext4_get_crypto_ctx(
335 bool with_page, const struct ext4_encryption_key *key);
336 void ext4_release_crypto_ctx(struct ext4_crypto_ctx *ctx);
337 void set_bh_to_page(struct buffer_head *head, struct page *page);
338 -struct page *ext4_encrypt(struct ext4_crypto_ctx *ctx,
339 - struct page *plaintext_page);
340 -int ext4_decrypt(struct ext4_crypto_ctx *ctx, struct page *page);
341 +int ext4_encrypt(struct ext4_crypto_ctx *ctx, struct page *plaintext_page,
342 + size_t offset, size_t size);
343 +int ext4_decrypt(struct ext4_crypto_ctx *ctx, struct page *page, size_t offset,
344 + size_t size);
345 int ext4_get_crypto_key(const struct file *file);
346 int ext4_set_crypto_key(struct dentry *dentry);
347 +void ext4_prep_pages_for_write(struct page *ciphertext_page,
348 + struct page *plaintext_page,
349 + struct ext4_crypto_ctx *ctx);
350 static inline bool ext4_is_encryption_enabled(struct ext4_inode_info *ei)
351 {
352 return ei->i_encryption_key.mode != EXT4_ENCRYPTION_MODE_INVALID;
353 diff --git a/fs/ext4/ext4_crypto.h b/fs/ext4/ext4_crypto.h
354 index 6cb5ba9..e747b51 100644
355 --- a/fs/ext4/ext4_crypto.h
356 +++ b/fs/ext4/ext4_crypto.h
357 @@ -32,12 +32,6 @@
358 #define EXT4_MAX_IV_SIZE AES_BLOCK_SIZE
359 #define EXT4_MAX_AUTH_SIZE EXT4_AES_256_GCM_AUTH_SIZE
361 -/* The metadata directory is only necessary only for the sibling file
362 - * directory under the mount root, which will be replaced by per-block
363 - * metadata when it's ready. */
364 -#define EXT4_METADATA_DIRECTORY_NAME ".ext4_crypt_data"
365 -#define EXT4_METADATA_DIRECTORY_NAME_SIZE 16
366 -
367 /**
368 * Packet format:
369 * 4 bytes: Size of packet (inclusive of these 4 bytes)
370 @@ -142,6 +136,7 @@ struct ext4_encryption_wrapper_desc {
372 struct ext4_crypto_ctx {
373 struct crypto_tfm *tfm; /* Crypto API context */
374 + struct mutex tfm_mutex;
375 struct page *bounce_page; /* Ciphertext page on write path */
376 struct page *control_page; /* Original page on write path */
377 struct bio *bio; /* The bio for this context */
378 @@ -150,6 +145,14 @@ struct ext4_crypto_ctx {
379 int flags; /* Flags */
380 enum ext4_encryption_mode mode; /* Encryption mode for tfm */
381 atomic_t dbg_refcnt; /* TODO(mhalcrow): Remove for release */
382 + atomic_t refcnt; /* Number of references to this ctx */
383 +};
384 +
385 +struct ext4_segment_ctx {
386 + struct ext4_crypto_ctx *ctx;
387 + struct work_struct work; /* Work queue for read complete path */
388 + size_t offset;
389 + size_t size;
390 };
392 static inline int ext4_encryption_key_size(enum ext4_encryption_mode mode)
393 diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
394 index bcf8c2e..9f4abc9 100644
395 --- a/fs/ext4/inode.c
396 +++ b/fs/ext4/inode.c
397 @@ -784,6 +784,7 @@ struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
398 struct buffer_head *bh;
399 struct ext4_inode_info *ei = EXT4_I(inode);
400 struct ext4_crypto_ctx *ctx;
401 + size_t offset_in_page;
403 bh = ext4_getblk(handle, inode, block, create);
404 if (IS_ERR(bh))
405 @@ -795,7 +796,11 @@ struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
406 if (buffer_uptodate(bh)) {
407 if (ext4_is_encryption_enabled(ei)) {
408 ctx = ext4_get_crypto_ctx(false, &ei->i_encryption_key);
409 - WARN_ON_ONCE(ext4_decrypt(ctx, bh->b_page));
410 + offset_in_page = ((block <<
411 + inode->i_sb->s_blocksize_bits) %
412 + PAGE_CACHE_SIZE);
413 + WARN_ON_ONCE(ext4_decrypt(ctx, bh->b_page,
414 + offset_in_page, bh->b_size));
415 ext4_release_crypto_ctx(ctx);
416 }
417 return bh;
418 @@ -898,7 +903,7 @@ static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
419 int err = 0;
420 unsigned blocksize, bbits;
421 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
422 - bool decrypt = false;
423 + size_t offset_in_page;
424 struct ext4_crypto_ctx *ctx;
426 BUG_ON(!PageLocked(page));
427 @@ -953,7 +958,22 @@ static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
428 (block_start < from || block_end > to)) {
429 ll_rw_block(READ, 1, &bh);
430 *wait_bh++=bh;
431 - decrypt = ext4_is_encryption_enabled(ei);
432 + if (ext4_is_encryption_enabled(ei)) {
433 + ctx = ext4_get_crypto_ctx(
434 + false, &ei->i_encryption_key);
435 + if (!ctx) {
436 + err = -ENOMEM;
437 + goto out;
438 + }
439 + offset_in_page = ((block_start * blocksize) %
440 + PAGE_CACHE_SIZE);
441 + err = ext4_decrypt(ctx, page, offset_in_page,
442 + blocksize);
443 + ext4_release_crypto_ctx(ctx);
444 + if (err)
445 + break;
446 +
447 + }
448 }
449 }
450 /*
451 @@ -966,14 +986,6 @@ static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
452 }
453 if (unlikely(err)) {
454 page_zero_new_buffers(page, from, to);
455 - } else if (decrypt) {
456 - ctx = ext4_get_crypto_ctx(false, &ei->i_encryption_key);
457 - if (!ctx) {
458 - err = -ENOMEM;
459 - goto out;
460 - }
461 - err = ext4_decrypt(ctx, page);
462 - ext4_release_crypto_ctx(ctx);
463 }
464 out:
465 return err;
466 @@ -2914,31 +2926,46 @@ static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
467 return generic_block_bmap(mapping, block, ext4_get_block);
468 }
470 -static void ext4_completion_work(struct work_struct *work)
471 +static void ext4_segment_completion_work(struct work_struct *work)
472 {
473 - struct ext4_crypto_ctx *ctx =
474 - container_of(work, struct ext4_crypto_ctx, work);
475 + struct ext4_segment_ctx *segment_ctx =
476 + container_of(work, struct ext4_segment_ctx, work);
477 + struct ext4_crypto_ctx *ctx = segment_ctx->ctx;
478 struct page *page = ctx->control_page;
479 - WARN_ON_ONCE(ext4_decrypt(ctx, page));
480 - ext4_release_crypto_ctx(ctx);
481 - SetPageUptodate(page);
482 - unlock_page(page);
483 +
484 + mutex_lock(&ctx->tfm_mutex);
485 + WARN_ON_ONCE(ext4_decrypt(ctx, page, segment_ctx->offset,
486 + segment_ctx->size));
487 + mutex_unlock(&ctx->tfm_mutex);
488 + if (atomic_dec_and_test(&ctx->refcnt)) {
489 + SetPageUptodate(page);
490 + unlock_page(page);
491 + ext4_release_crypto_ctx(ctx);
492 + }
493 + kfree(segment_ctx);
494 }
496 -static int ext4_complete_cb(struct bio *bio, int res)
497 +static int ext4_complete_segment_cb(struct bio *bio, int res)
498 {
499 - struct ext4_crypto_ctx *ctx = bio->bi_cb_ctx;
500 - struct page *page = ctx->control_page;
501 + struct ext4_segment_ctx *segment_ctx = bio->bi_cb_ctx;
502 +
503 if (res) {
504 - ext4_release_crypto_ctx(ctx);
505 - unlock_page(page);
506 + if (atomic_dec_and_test(&segment_ctx->ctx->refcnt))
507 + ext4_release_crypto_ctx(segment_ctx->ctx);
508 + kfree(segment_ctx);
509 return res;
510 }
511 - INIT_WORK(&ctx->work, ext4_completion_work);
512 - queue_work(mpage_read_workqueue, &ctx->work);
513 + BUG_ON(!segment_ctx);
514 + INIT_WORK(&segment_ctx->work, ext4_segment_completion_work);
515 + queue_work(mpage_read_workqueue, &segment_ctx->work);
516 return 0;
517 }
519 +struct bh_to_submit {
520 + struct buffer_head *bh;
521 + size_t offset;
522 +};
523 +
524 static int ext4_read_full_page(struct page *page)
525 {
526 struct inode *inode = page->mapping->host;
527 @@ -2948,10 +2975,12 @@ static int ext4_read_full_page(struct page *page)
528 sector_t iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
529 sector_t lblock = (i_size_read(inode)+blocksize-1) >> bbits;
530 struct buffer_head *bh = head;
531 - struct buffer_head *arr[MAX_BUF_PER_PAGE];
532 + struct bh_to_submit arr[MAX_BUF_PER_PAGE];
533 + struct ext4_crypto_ctx *ctx = NULL;
534 int nr = 0;
535 int i = 0;
536 int fully_mapped = 1;
537 + size_t offset;
539 do {
540 if (buffer_uptodate(bh))
541 @@ -2980,7 +3009,8 @@ static int ext4_read_full_page(struct page *page)
542 if (buffer_uptodate(bh))
543 continue;
544 }
545 - arr[nr++] = bh;
546 + arr[nr].bh = bh;
547 + arr[nr++].offset = i * bh->b_size;
548 } while (i++, iblock++, (bh = bh->b_this_page) != head);
550 if (fully_mapped)
551 @@ -2997,15 +3027,9 @@ static int ext4_read_full_page(struct page *page)
552 return 0;
553 }
555 - /*
556 - * Encryption requires blocksize is page size, so we should never see
557 - * more than one buffer head per page.
558 - */
559 - BUG_ON(nr != 1);
560 -
561 /* Stage two: lock the buffers */
562 for (i = 0; i < nr; i++) {
563 - bh = arr[i];
564 + bh = arr[i].bh;
565 lock_buffer(bh);
566 mark_buffer_async_read(bh);
567 }
568 @@ -3015,18 +3039,48 @@ static int ext4_read_full_page(struct page *page)
569 * inside the buffer lock in case another process reading
570 * the underlying blockdev brought it uptodate (the sct fix).
571 */
572 - for (i = 0; i < nr; i++) {
573 - bh = arr[i];
574 - if (buffer_uptodate(bh))
575 + for (i = 0; i < nr; i++, offset += head->b_size) {
576 + bh = arr[i].bh;
577 + offset = arr[i].offset;
578 + if (buffer_uptodate(bh)) {
579 end_buffer_async_read(bh, 1);
580 - else {
581 + } else {
582 struct ext4_inode_info *ei = EXT4_I(inode);
583 - struct ext4_crypto_ctx *ctx = ext4_get_crypto_ctx(
584 - false, &ei->i_encryption_key);
585 - atomic_inc(&ctx->dbg_refcnt);
586 - ctx->control_page = page;
587 - if (submit_bh_cb(READ, bh, ext4_complete_cb, ctx))
588 - ext4_release_crypto_ctx(ctx);
589 + struct ext4_segment_ctx *segment_ctx;
590 +
591 + if (!ctx) {
592 + ctx = ext4_get_crypto_ctx(
593 + false, &ei->i_encryption_key);
594 + if (!ctx)
595 + break;
596 + atomic_inc(&ctx->dbg_refcnt);
597 + ctx->control_page = page;
598 +
599 + /* Keep ref until all segments are submitted */
600 + atomic_inc(&ctx->refcnt);
601 + }
602 + segment_ctx = kzalloc(sizeof(struct ext4_segment_ctx),
603 + GFP_NOFS);
604 + if (!segment_ctx)
605 + continue;
606 + segment_ctx->offset = offset;
607 + segment_ctx->size = head->b_size;
608 + segment_ctx->ctx = ctx;
609 + atomic_inc(&ctx->refcnt);
610 + if (submit_bh_cb(READ, bh,
611 + ext4_complete_segment_cb,
612 + segment_ctx)) {
613 + atomic_dec(&ctx->refcnt);
614 + kfree(segment_ctx);
615 + continue;
616 + }
617 + }
618 + }
619 + if (ctx) {
620 + if (atomic_dec_and_test(&ctx->refcnt)) {
621 + SetPageUptodate(page);
622 + unlock_page(page);
623 + ext4_release_crypto_ctx(ctx);
624 }
625 }
626 return 0;
627 @@ -3477,6 +3531,7 @@ static int ext4_block_zero_page_range(handle_t *handle,
628 struct buffer_head *bh;
629 struct page *page;
630 struct ext4_crypto_ctx *ctx;
631 + size_t offset_in_page;
632 int err = 0;
634 page = find_or_create_page(mapping, from >> PAGE_CACHE_SHIFT,
635 @@ -3533,9 +3588,12 @@ static int ext4_block_zero_page_range(handle_t *handle,
636 if (!buffer_uptodate(bh))
637 goto unlock;
638 if (ext4_is_encryption_enabled(ei)) {
639 - BUG_ON(blocksize != PAGE_CACHE_SIZE);
640 ctx = ext4_get_crypto_ctx(false, &ei->i_encryption_key);
641 - WARN_ON_ONCE(ext4_decrypt(ctx, page));
642 + offset_in_page = ((iblock <<
643 + inode->i_sb->s_blocksize_bits) %
644 + PAGE_CACHE_SIZE);
645 + WARN_ON_ONCE(ext4_decrypt(ctx, page, offset_in_page,
646 + bh->b_size));
647 ext4_release_crypto_ctx(ctx);
648 }
649 }
650 diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c
651 index b68d178..8109eba08 100644
652 --- a/fs/ext4/page-io.c
653 +++ b/fs/ext4/page-io.c
654 @@ -434,57 +434,26 @@ static void ext4_abort_bio_write(struct page *page,
655 } while (bh != head);
656 }
658 -static int io_encrypt_submit_page(struct ext4_io_submit *io, struct page *page)
659 -{
660 - struct page *data_page = NULL;
661 - struct ext4_crypto_ctx *ctx = NULL;
662 - struct inode *inode = page->mapping->host;
663 - struct ext4_inode_info *ei = EXT4_I(inode);
664 - struct buffer_head *bh;
665 - int res = 0;
666 -
667 - ctx = ext4_get_crypto_ctx(true, &ei->i_encryption_key);
668 - if (IS_ERR(ctx))
669 - return PTR_ERR(ctx);
670 -
671 - bh = page_buffers(page);
672 - data_page = ext4_encrypt(ctx, page);
673 - if (IS_ERR(data_page)) {
674 - ext4_release_crypto_ctx(ctx);
675 - res = PTR_ERR(data_page);
676 - printk_ratelimited(KERN_ERR "%s: ext4_encrypt() returned %d\n",
677 - __func__, res);
678 - goto out;
679 - }
680 - lock_page(data_page);
681 - res = io_submit_add_bh(io, inode, bh);
682 - if (res)
683 - ext4_restore_control_page(data_page);
684 -out:
685 - return res;
686 -}
687 -
688 -static int ext4_bio_write_buffers(struct ext4_io_submit *io,
689 - struct page *page,
690 - int len,
691 - struct writeback_control *wbc)
692 +/**
693 + * ext4_prep_bh() - Prepare and mark buffers to submit
694 + * @io:
695 + * @page:
696 + *
697 + * We have to mark all buffers in the page before submitting so that
698 + * end_page_writeback() cannot be called from ext4_bio_end_io() when IO on the
699 + * first buffer finishes and we are still working on submitting the second
700 + * buffer.
701 + */
702 +static void ext4_prep_bh(struct ext4_io_submit *io, struct page *page, int len)
703 {
704 - struct inode *inode = page->mapping->host;
705 - struct ext4_inode_info *ei = EXT4_I(inode);
706 unsigned block_start;
707 struct buffer_head *bh, *head;
708 - int ret = 0;
709 - int nr_submitted = 0;
711 - /*
712 - * In the first loop we prepare and mark buffers to submit. We have to
713 - * mark all buffers in the page before submitting so that
714 - * end_page_writeback() cannot be called from ext4_bio_end_io() when IO
715 - * on the first buffer finishes and we are still working on submitting
716 - * the second buffer.
717 - */
718 + size_t offset = 0;
719 +
720 bh = head = page_buffers(page);
721 do {
722 + offset += head->b_size;
723 block_start = bh_offset(bh);
724 if (block_start >= len) {
725 clear_buffer_dirty(bh);
726 @@ -505,17 +474,79 @@ static int ext4_bio_write_buffers(struct ext4_io_submit *io,
727 }
728 set_buffer_async_write(bh);
729 } while ((bh = bh->b_this_page) != head);
730 +}
731 +
732 +static struct ext4_crypto_ctx *ext4_encrypt_bh(struct ext4_inode_info *ei,
733 + struct page *page)
734 +{
735 + struct ext4_crypto_ctx *ctx = NULL;
736 + size_t offset, next_offset = 0;
737 + struct buffer_head *bh, *head;
738 + int ret;
739 +
740 + bh = head = page_buffers(page);
741 + do {
742 + offset = next_offset;
743 + next_offset += head->b_size;
744 + if (!buffer_async_write(bh))
745 + continue;
746 + if (!ctx) {
747 + ctx = ext4_get_crypto_ctx(true, &ei->i_encryption_key);
748 + if (IS_ERR(ctx))
749 + goto out;
750 + }
751 + ret = ext4_encrypt(ctx, page, offset, head->b_size);
752 + if (ret)
753 + goto out;
754 + } while ((bh = bh->b_this_page) != head);
755 +out:
756 + if (ret) {
757 + if (ctx)
758 + ext4_release_crypto_ctx(ctx);
759 + ctx = ERR_PTR(ret);
760 + }
761 + return ctx;
762 +}
763 +
764 +struct inode *ext4_inode;
765 +EXPORT_SYMBOL(ext4_inode);
766 +
767 +static int ext4_bio_write_buffers(struct ext4_io_submit *io,
768 + struct page *page, int len,
769 + struct writeback_control *wbc)
770 +{
771 + struct inode *inode = page->mapping->host;
772 + struct ext4_inode_info *ei = EXT4_I(inode);
773 + struct buffer_head *bh, *head;
774 + struct ext4_crypto_ctx *ctx = NULL;
775 + bool pages_prepped = false;
776 + int ret = 0;
777 + int nr_submitted = 0;
778 +
779 + /* Mark buffer heads for write */
780 + ext4_prep_bh(io, page, len);
781 +
782 + /* Encrypt the buffers if encryption is enabled */
783 + if (ext4_is_encryption_enabled(ei)) {
784 + ctx = ext4_encrypt_bh(ei, page);
785 + if (IS_ERR(ctx)) {
786 + ret = PTR_ERR(ctx);
787 + ctx = NULL;
788 + goto out;
789 + }
790 + }
792 /* Now submit buffers to write */
793 bh = head = page_buffers(page);
794 do {
795 if (!buffer_async_write(bh))
796 continue;
797 - if (ext4_is_encryption_enabled(ei)) {
798 - ret = io_encrypt_submit_page(io, page);
799 - } else {
800 - ret = io_submit_add_bh(io, inode, bh);
801 + if (ctx && !pages_prepped) {
802 + ext4_prep_pages_for_write(ctx->bounce_page, page, ctx);
803 + /* ctx->bounce_page now owns its ctx */
804 + pages_prepped = true;
805 }
806 + ret = io_submit_add_bh(io, inode, bh);
807 if (ret) {
808 /*
809 * We only get here on ENOMEM. Not much else
810 @@ -528,15 +559,25 @@ static int ext4_bio_write_buffers(struct ext4_io_submit *io,
811 clear_buffer_dirty(bh);
812 } while ((bh = bh->b_this_page) != head);
814 +out:
815 /* Error stopped previous loop? Clean up buffers... */
816 if (ret) {
817 printk_ratelimited(KERN_ERR "%s: ret = %d\n", __func__, ret);
818 + if (ctx) {
819 + if (pages_prepped)
820 + ext4_restore_control_page(ctx->bounce_page);
821 + else
822 + ext4_release_crypto_ctx(ctx);
823 + }
824 ext4_abort_bio_write(page, wbc);
825 }
826 unlock_page(page);
827 /* Nothing submitted - we have to end page writeback */
828 - if (!nr_submitted)
829 + if (!nr_submitted) {
830 + if (ctx)
831 + ext4_release_crypto_ctx(ctx);
832 end_page_writeback(page);
833 + }
834 return ret;
835 }
837 --
838 2.2.0.rc0.207.ga3a616c