1 ext4: implement the EXT4 encryption write path
3 From: Michael Halcrow <mhalcrow@google.com>
5 Signed-off-by: Michael Halcrow <mhalcrow@google.com>
6 Signed-off-by: Theodore Ts'o <tytso@mit.edu>
10 fs/ext4/namei.c | 11 +++-
11 fs/ext4/page-io.c | 180 +++++++++++++++++++++++++++++++++++++++++++-----------
12 4 files changed, 161 insertions(+), 40 deletions(-)
14 diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
15 index ef89b61..ca27a7f 100644
18 @@ -2807,6 +2807,11 @@ struct page *ext4_encrypt(struct ext4_crypto_ctx *ctx,
19 int ext4_decrypt(struct ext4_crypto_ctx *ctx, struct page *page);
20 int ext4_get_crypto_key(const struct file *file);
21 int ext4_set_crypto_key(struct dentry *dentry);
22 +static inline bool ext4_is_encryption_enabled(struct ext4_inode_info *ei)
24 + return ei->i_encryption_key.mode != EXT4_ENCRYPTION_MODE_INVALID;
29 * Disable DIO read nolock optimization, so new dioreaders will be forced
30 diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
31 index 8a06473..42385c9 100644
34 @@ -2431,6 +2431,7 @@ static int ext4_writepages(struct address_space *mapping,
35 handle_t *handle = NULL;
36 struct mpage_da_data mpd;
37 struct inode *inode = mapping->host;
38 + struct ext4_inode_info *ei = EXT4_I(inode);
39 int needed_blocks, rsv_blocks = 0, ret = 0;
40 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
42 @@ -2447,7 +2448,7 @@ static int ext4_writepages(struct address_space *mapping,
43 if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
46 - if (ext4_should_journal_data(inode)) {
47 + if (ext4_should_journal_data(inode) || ext4_is_encryption_enabled(ei)) {
50 blk_start_plug(&plug);
51 @@ -3097,6 +3098,7 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
53 struct file *file = iocb->ki_filp;
54 struct inode *inode = file->f_mapping->host;
55 + struct ext4_inode_info *ei = EXT4_I(inode);
57 size_t count = iov_iter_count(iter);
59 @@ -3173,6 +3175,7 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
60 get_block_func = ext4_get_block_write;
61 dio_flags = DIO_LOCKING;
63 + BUG_ON(ext4_is_encryption_enabled(ei));
64 ret = __blockdev_direct_IO(rw, iocb, inode,
65 inode->i_sb->s_bdev, iter,
67 diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c
68 index 3520ab8..a8ed5d6 100644
71 @@ -2238,6 +2238,7 @@ static int ext4_create(struct inode *dir, struct dentry *dentry, umode_t mode,
75 + struct ext4_sb_info *sbi = EXT4_SB(dir->i_sb);
76 int err, credits, retries = 0;
78 dquot_initialize(dir);
79 @@ -2254,8 +2255,14 @@ retry:
80 inode->i_fop = &ext4_file_operations;
82 err = ext4_add_nondir(handle, dentry, inode);
83 - if (!err && IS_DIRSYNC(dir))
84 - ext4_handle_sync(handle);
86 + if (sbi->s_default_encryption_mode !=
87 + EXT4_ENCRYPTION_MODE_INVALID) {
88 + ext4_set_crypto_key(dentry);
90 + if (IS_DIRSYNC(dir))
91 + ext4_handle_sync(handle);
95 ext4_journal_stop(handle);
96 diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c
97 index b24a254..872503d 100644
98 --- a/fs/ext4/page-io.c
99 +++ b/fs/ext4/page-io.c
100 @@ -61,6 +61,25 @@ static void buffer_io_error(struct buffer_head *bh)
101 (unsigned long long)bh->b_blocknr);
104 +static void ext4_restore_control_page(struct page *data_page)
106 + struct page *control_page = NULL;
107 + struct ext4_crypto_ctx *ctx = NULL;
109 + BUG_ON(!PagePrivate(data_page));
110 + ctx = (struct ext4_crypto_ctx *)page_private(data_page);
112 + control_page = ctx->control_page;
113 + BUG_ON(!control_page);
114 + BUG_ON(!page_buffers(control_page));
115 + set_bh_to_page(page_buffers(control_page), control_page);
116 + set_page_private(data_page, (unsigned long)NULL);
117 + ClearPagePrivate(data_page);
118 + BUG_ON(!PageLocked(data_page));
119 + unlock_page(data_page);
120 + ext4_release_crypto_ctx(ctx);
123 static void ext4_finish_bio(struct bio *bio)
126 @@ -69,6 +88,8 @@ static void ext4_finish_bio(struct bio *bio)
128 bio_for_each_segment_all(bvec, bio, i) {
129 struct page *page = bvec->bv_page;
130 + struct page *data_page = NULL;
131 + struct ext4_crypto_ctx *ctx = NULL;
132 struct buffer_head *bh, *head;
133 unsigned bio_start = bvec->bv_offset;
134 unsigned bio_end = bio_start + bvec->bv_len;
135 @@ -78,6 +99,22 @@ static void ext4_finish_bio(struct bio *bio)
139 + if (!page->mapping) {
140 + /* The bounce data pages are unmapped. */
142 + BUG_ON(!PagePrivate(data_page));
143 + ctx = (struct ext4_crypto_ctx *)page_private(data_page);
145 + page = ctx->control_page;
148 + /* TODO(mhalcrow): Remove this else{} for release */
149 + struct inode *inode = page->mapping->host;
150 + struct ext4_inode_info *ei = EXT4_I(inode);
152 + BUG_ON(ext4_is_encryption_enabled(ei));
157 set_bit(AS_EIO, &page->mapping->flags);
158 @@ -102,8 +139,11 @@ static void ext4_finish_bio(struct bio *bio)
159 } while ((bh = bh->b_this_page) != head);
160 bit_spin_unlock(BH_Uptodate_Lock, &head->b_state);
161 local_irq_restore(flags);
165 + ext4_restore_control_page(data_page);
166 end_page_writeback(page);
171 @@ -398,40 +438,29 @@ submit_and_retry:
175 -int ext4_bio_write_page(struct ext4_io_submit *io,
178 - struct writeback_control *wbc,
181 +static void ext4_abort_bio_write(struct page *page,
182 + struct writeback_control *wbc) {
183 + struct buffer_head *bh, *head;
185 + redirty_page_for_writepage(wbc, page);
186 + bh = head = page_buffers(page);
188 + clear_buffer_async_write(bh);
189 + bh = bh->b_this_page;
190 + } while (bh != head);
193 +static int ext4_bio_write_buffers(struct ext4_io_submit *io,
195 + struct page *data_page,
197 + struct writeback_control *wbc) {
198 struct inode *inode = page->mapping->host;
199 - unsigned block_start, blocksize;
200 + unsigned block_start;
201 struct buffer_head *bh, *head;
203 int nr_submitted = 0;
205 - blocksize = 1 << inode->i_blkbits;
207 - BUG_ON(!PageLocked(page));
208 - BUG_ON(PageWriteback(page));
211 - set_page_writeback_keepwrite(page);
213 - set_page_writeback(page);
214 - ClearPageError(page);
217 - * Comments copied from block_write_full_page:
219 - * The page straddles i_size. It must be zeroed out on each and every
220 - * writepage invocation because it may be mmapped. "A file is mapped
221 - * in multiples of the page size. For a file that is not a multiple of
222 - * the page size, the remaining memory is zeroed when mapped, and
223 - * writes to that region are not written out to the file."
225 - if (len < PAGE_CACHE_SIZE)
226 - zero_user_segment(page, len, PAGE_CACHE_SIZE);
228 * In the first loop we prepare and mark buffers to submit. We have to
229 * mark all buffers in the page before submitting so that
230 @@ -449,7 +478,12 @@ int ext4_bio_write_page(struct ext4_io_submit *io,
232 if (!buffer_dirty(bh) || buffer_delay(bh) ||
233 !buffer_mapped(bh) || buffer_unwritten(bh)) {
234 - /* A hole? We can safely clear the dirty bit */
235 + /* A hole? We can safely clear the dirty bit,
236 + * so long as we're not encrypting */
238 + BUG_ON(!buffer_dirty(bh));
239 + BUG_ON(!buffer_mapped(bh));
241 if (!buffer_mapped(bh))
242 clear_buffer_dirty(bh);
244 @@ -475,7 +509,6 @@ int ext4_bio_write_page(struct ext4_io_submit *io,
245 * we can do but mark the page as dirty, and
246 * better luck next time.
248 - redirty_page_for_writepage(wbc, page);
252 @@ -484,14 +517,87 @@ int ext4_bio_write_page(struct ext4_io_submit *io,
254 /* Error stopped previous loop? Clean up buffers... */
257 - clear_buffer_async_write(bh);
258 - bh = bh->b_this_page;
259 - } while (bh != head);
260 + printk_ratelimited(KERN_ERR "%s: ret = [%d]\n", __func__, ret);
261 + ext4_abort_bio_write(page, wbc);
264 /* Nothing submitted - we have to end page writeback */
266 + if (!nr_submitted) {
268 + ext4_restore_control_page(data_page);
269 end_page_writeback(page);
274 +static int ext4_bio_encrypt_and_write(struct ext4_io_submit *io,
275 + struct page *control_page,
276 + struct writeback_control *wbc) {
277 + struct page *data_page = NULL;
278 + struct ext4_crypto_ctx *ctx = NULL;
279 + struct inode *inode = control_page->mapping->host;
280 + struct ext4_inode_info *ei = EXT4_I(inode);
283 + BUG_ON(!ext4_is_encryption_enabled(ei));
284 + ctx = ext4_get_crypto_ctx(true, &ei->i_encryption_key);
286 + res = PTR_ERR(ctx);
289 + data_page = ext4_encrypt(ctx, control_page);
290 + if (IS_ERR(data_page)) {
291 + res = PTR_ERR(data_page);
292 + printk_ratelimited(KERN_ERR
293 + "%s: ext4_encrypt() returned %d\n",
295 + goto free_ctx_and_fail;
297 + BUG_ON(PageLocked(data_page));
298 + lock_page(data_page);
299 + return ext4_bio_write_buffers(io, control_page, data_page,
300 + PAGE_CACHE_SIZE, wbc);
302 + ext4_release_crypto_ctx(ctx);
304 + ext4_abort_bio_write(control_page, wbc);
305 + end_page_writeback(control_page);
309 +int ext4_bio_write_page(struct ext4_io_submit *io,
312 + struct writeback_control *wbc,
315 + struct ext4_inode_info *ei = EXT4_I(page->mapping->host);
318 + BUG_ON(!PageLocked(page));
319 + BUG_ON(PageWriteback(page));
321 + set_page_writeback_keepwrite(page);
323 + set_page_writeback(page);
324 + ClearPageError(page);
327 + * Comments copied from block_write_full_page_endio:
329 + * The page straddles i_size. It must be zeroed out on each and every
330 + * writepage invocation because it may be mmapped. "A file is mapped
331 + * in multiples of the page size. For a file that is not a multiple of
332 + * the page size, the remaining memory is zeroed when mapped, and
333 + * writes to that region are not written out to the file."
335 + if (len < PAGE_CACHE_SIZE)
336 + zero_user_segment(page, len, PAGE_CACHE_SIZE);
338 + if (ext4_is_encryption_enabled(ei))
339 + ret = ext4_bio_encrypt_and_write(io, page, wbc);
341 + ret = ext4_bio_write_buffers(io, page, NULL, len, wbc);
346 2.1.0.rc2.206.gedb03e5