| blob | 771873d1d1a8eee057f370c0d2a54c5dbff31c2e |
1 ext4: implement the EXT4 encryption write path
3 From: Michael Halcrow <mhalcrow@google.com>
5 Signed-off-by: Michael Halcrow <mhalcrow@google.com>
6 Signed-off-by: Theodore Ts'o <tytso@mit.edu>
7 ---
8 fs/ext4/ext4.h | 5 ++
9 fs/ext4/inode.c | 5 +-
10 fs/ext4/namei.c | 11 +++-
11 fs/ext4/page-io.c | 180 +++++++++++++++++++++++++++++++++++++++++++-----------
12 4 files changed, 161 insertions(+), 40 deletions(-)
14 diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
15 index ef89b61..ca27a7f 100644
16 --- a/fs/ext4/ext4.h
17 +++ b/fs/ext4/ext4.h
18 @@ -2807,6 +2807,11 @@ struct page *ext4_encrypt(struct ext4_crypto_ctx *ctx,
19 int ext4_decrypt(struct ext4_crypto_ctx *ctx, struct page *page);
20 int ext4_get_crypto_key(const struct file *file);
21 int ext4_set_crypto_key(struct dentry *dentry);
22 +static inline bool ext4_is_encryption_enabled(struct ext4_inode_info *ei)
23 +{
24 + return ei->i_encryption_key.mode != EXT4_ENCRYPTION_MODE_INVALID;
25 +}
26 +
28 /*
29 * Disable DIO read nolock optimization, so new dioreaders will be forced
30 diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
31 index 8a06473..42385c9 100644
32 --- a/fs/ext4/inode.c
33 +++ b/fs/ext4/inode.c
34 @@ -2431,6 +2431,7 @@ static int ext4_writepages(struct address_space *mapping,
35 handle_t *handle = NULL;
36 struct mpage_da_data mpd;
37 struct inode *inode = mapping->host;
38 + struct ext4_inode_info *ei = EXT4_I(inode);
39 int needed_blocks, rsv_blocks = 0, ret = 0;
40 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
41 bool done;
42 @@ -2447,7 +2448,7 @@ static int ext4_writepages(struct address_space *mapping,
43 if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
44 goto out_writepages;
46 - if (ext4_should_journal_data(inode)) {
47 + if (ext4_should_journal_data(inode) || ext4_is_encryption_enabled(ei)) {
48 struct blk_plug plug;
50 blk_start_plug(&plug);
51 @@ -3097,6 +3098,7 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
52 {
53 struct file *file = iocb->ki_filp;
54 struct inode *inode = file->f_mapping->host;
55 + struct ext4_inode_info *ei = EXT4_I(inode);
56 ssize_t ret;
57 size_t count = iov_iter_count(iter);
58 int overwrite = 0;
59 @@ -3173,6 +3175,7 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
60 get_block_func = ext4_get_block_write;
61 dio_flags = DIO_LOCKING;
62 }
63 + BUG_ON(ext4_is_encryption_enabled(ei));
64 ret = __blockdev_direct_IO(rw, iocb, inode,
65 inode->i_sb->s_bdev, iter,
66 offset,
67 diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c
68 index 3520ab8..a8ed5d6 100644
69 --- a/fs/ext4/namei.c
70 +++ b/fs/ext4/namei.c
71 @@ -2238,6 +2238,7 @@ static int ext4_create(struct inode *dir, struct dentry *dentry, umode_t mode,
72 {
73 handle_t *handle;
74 struct inode *inode;
75 + struct ext4_sb_info *sbi = EXT4_SB(dir->i_sb);
76 int err, credits, retries = 0;
78 dquot_initialize(dir);
79 @@ -2254,8 +2255,14 @@ retry:
80 inode->i_fop = &ext4_file_operations;
81 ext4_set_aops(inode);
82 err = ext4_add_nondir(handle, dentry, inode);
83 - if (!err && IS_DIRSYNC(dir))
84 - ext4_handle_sync(handle);
85 + if (!err) {
86 + if (sbi->s_default_encryption_mode !=
87 + EXT4_ENCRYPTION_MODE_INVALID) {
88 + ext4_set_crypto_key(dentry);
89 + }
90 + if (IS_DIRSYNC(dir))
91 + ext4_handle_sync(handle);
92 + }
93 }
94 if (handle)
95 ext4_journal_stop(handle);
96 diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c
97 index b24a254..872503d 100644
98 --- a/fs/ext4/page-io.c
99 +++ b/fs/ext4/page-io.c
100 @@ -61,6 +61,25 @@ static void buffer_io_error(struct buffer_head *bh)
101 (unsigned long long)bh->b_blocknr);
102 }
104 +static void ext4_restore_control_page(struct page *data_page)
105 +{
106 + struct page *control_page = NULL;
107 + struct ext4_crypto_ctx *ctx = NULL;
108 +
109 + BUG_ON(!PagePrivate(data_page));
110 + ctx = (struct ext4_crypto_ctx *)page_private(data_page);
111 + BUG_ON(!ctx);
112 + control_page = ctx->control_page;
113 + BUG_ON(!control_page);
114 + BUG_ON(!page_buffers(control_page));
115 + set_bh_to_page(page_buffers(control_page), control_page);
116 + set_page_private(data_page, (unsigned long)NULL);
117 + ClearPagePrivate(data_page);
118 + BUG_ON(!PageLocked(data_page));
119 + unlock_page(data_page);
120 + ext4_release_crypto_ctx(ctx);
121 +}
122 +
123 static void ext4_finish_bio(struct bio *bio)
124 {
125 int i;
126 @@ -69,6 +88,8 @@ static void ext4_finish_bio(struct bio *bio)
128 bio_for_each_segment_all(bvec, bio, i) {
129 struct page *page = bvec->bv_page;
130 + struct page *data_page = NULL;
131 + struct ext4_crypto_ctx *ctx = NULL;
132 struct buffer_head *bh, *head;
133 unsigned bio_start = bvec->bv_offset;
134 unsigned bio_end = bio_start + bvec->bv_len;
135 @@ -78,6 +99,22 @@ static void ext4_finish_bio(struct bio *bio)
136 if (!page)
137 continue;
139 + if (!page->mapping) {
140 + /* The bounce data pages are unmapped. */
141 + data_page = page;
142 + BUG_ON(!PagePrivate(data_page));
143 + ctx = (struct ext4_crypto_ctx *)page_private(data_page);
144 + BUG_ON(!ctx);
145 + page = ctx->control_page;
146 + BUG_ON(!page);
147 + } else {
148 + /* TODO(mhalcrow): Remove this else{} for release */
149 + struct inode *inode = page->mapping->host;
150 + struct ext4_inode_info *ei = EXT4_I(inode);
151 +
152 + BUG_ON(ext4_is_encryption_enabled(ei));
153 + }
154 +
155 if (error) {
156 SetPageError(page);
157 set_bit(AS_EIO, &page->mapping->flags);
158 @@ -102,8 +139,11 @@ static void ext4_finish_bio(struct bio *bio)
159 } while ((bh = bh->b_this_page) != head);
160 bit_spin_unlock(BH_Uptodate_Lock, &head->b_state);
161 local_irq_restore(flags);
162 - if (!under_io)
163 + if (!under_io) {
164 + if (ctx)
165 + ext4_restore_control_page(data_page);
166 end_page_writeback(page);
167 + }
168 }
169 }
171 @@ -398,40 +438,29 @@ submit_and_retry:
172 return 0;
173 }
175 -int ext4_bio_write_page(struct ext4_io_submit *io,
176 - struct page *page,
177 - int len,
178 - struct writeback_control *wbc,
179 - bool keep_towrite)
180 -{
181 +static void ext4_abort_bio_write(struct page *page,
182 + struct writeback_control *wbc) {
183 + struct buffer_head *bh, *head;
184 +
185 + redirty_page_for_writepage(wbc, page);
186 + bh = head = page_buffers(page);
187 + do {
188 + clear_buffer_async_write(bh);
189 + bh = bh->b_this_page;
190 + } while (bh != head);
191 +}
192 +
193 +static int ext4_bio_write_buffers(struct ext4_io_submit *io,
194 + struct page *page,
195 + struct page *data_page,
196 + int len,
197 + struct writeback_control *wbc) {
198 struct inode *inode = page->mapping->host;
199 - unsigned block_start, blocksize;
200 + unsigned block_start;
201 struct buffer_head *bh, *head;
202 int ret = 0;
203 int nr_submitted = 0;
205 - blocksize = 1 << inode->i_blkbits;
206 -
207 - BUG_ON(!PageLocked(page));
208 - BUG_ON(PageWriteback(page));
209 -
210 - if (keep_towrite)
211 - set_page_writeback_keepwrite(page);
212 - else
213 - set_page_writeback(page);
214 - ClearPageError(page);
215 -
216 - /*
217 - * Comments copied from block_write_full_page:
218 - *
219 - * The page straddles i_size. It must be zeroed out on each and every
220 - * writepage invocation because it may be mmapped. "A file is mapped
221 - * in multiples of the page size. For a file that is not a multiple of
222 - * the page size, the remaining memory is zeroed when mapped, and
223 - * writes to that region are not written out to the file."
224 - */
225 - if (len < PAGE_CACHE_SIZE)
226 - zero_user_segment(page, len, PAGE_CACHE_SIZE);
227 /*
228 * In the first loop we prepare and mark buffers to submit. We have to
229 * mark all buffers in the page before submitting so that
230 @@ -449,7 +478,12 @@ int ext4_bio_write_page(struct ext4_io_submit *io,
231 }
232 if (!buffer_dirty(bh) || buffer_delay(bh) ||
233 !buffer_mapped(bh) || buffer_unwritten(bh)) {
234 - /* A hole? We can safely clear the dirty bit */
235 + /* A hole? We can safely clear the dirty bit,
236 + * so long as we're not encrypting */
237 + if (data_page) {
238 + BUG_ON(!buffer_dirty(bh));
239 + BUG_ON(!buffer_mapped(bh));
240 + }
241 if (!buffer_mapped(bh))
242 clear_buffer_dirty(bh);
243 if (io->io_bio)
244 @@ -475,7 +509,6 @@ int ext4_bio_write_page(struct ext4_io_submit *io,
245 * we can do but mark the page as dirty, and
246 * better luck next time.
247 */
248 - redirty_page_for_writepage(wbc, page);
249 break;
250 }
251 nr_submitted++;
252 @@ -484,14 +517,87 @@ int ext4_bio_write_page(struct ext4_io_submit *io,
254 /* Error stopped previous loop? Clean up buffers... */
255 if (ret) {
256 - do {
257 - clear_buffer_async_write(bh);
258 - bh = bh->b_this_page;
259 - } while (bh != head);
260 + printk_ratelimited(KERN_ERR "%s: ret = [%d]\n", __func__, ret);
261 + ext4_abort_bio_write(page, wbc);
262 }
263 unlock_page(page);
264 /* Nothing submitted - we have to end page writeback */
265 - if (!nr_submitted)
266 + if (!nr_submitted) {
267 + if (data_page)
268 + ext4_restore_control_page(data_page);
269 end_page_writeback(page);
270 + }
271 + return ret;
272 +}
273 +
274 +static int ext4_bio_encrypt_and_write(struct ext4_io_submit *io,
275 + struct page *control_page,
276 + struct writeback_control *wbc) {
277 + struct page *data_page = NULL;
278 + struct ext4_crypto_ctx *ctx = NULL;
279 + struct inode *inode = control_page->mapping->host;
280 + struct ext4_inode_info *ei = EXT4_I(inode);
281 + int res = 0;
282 +
283 + BUG_ON(!ext4_is_encryption_enabled(ei));
284 + ctx = ext4_get_crypto_ctx(true, &ei->i_encryption_key);
285 + if (IS_ERR(ctx)) {
286 + res = PTR_ERR(ctx);
287 + goto fail;
288 + }
289 + data_page = ext4_encrypt(ctx, control_page);
290 + if (IS_ERR(data_page)) {
291 + res = PTR_ERR(data_page);
292 + printk_ratelimited(KERN_ERR
293 + "%s: ext4_encrypt() returned %d\n",
294 + __func__, res);
295 + goto free_ctx_and_fail;
296 + }
297 + BUG_ON(PageLocked(data_page));
298 + lock_page(data_page);
299 + return ext4_bio_write_buffers(io, control_page, data_page,
300 + PAGE_CACHE_SIZE, wbc);
301 +free_ctx_and_fail:
302 + ext4_release_crypto_ctx(ctx);
303 +fail:
304 + ext4_abort_bio_write(control_page, wbc);
305 + end_page_writeback(control_page);
306 + return res;
307 +}
308 +
309 +int ext4_bio_write_page(struct ext4_io_submit *io,
310 + struct page *page,
311 + int len,
312 + struct writeback_control *wbc,
313 + bool keep_towrite)
314 +{
315 + struct ext4_inode_info *ei = EXT4_I(page->mapping->host);
316 + int ret = 0;
317 +
318 + BUG_ON(!PageLocked(page));
319 + BUG_ON(PageWriteback(page));
320 + if (keep_towrite)
321 + set_page_writeback_keepwrite(page);
322 + else
323 + set_page_writeback(page);
324 + ClearPageError(page);
325 +
326 + /*
327 + * Comments copied from block_write_full_page_endio:
328 + *
329 + * The page straddles i_size. It must be zeroed out on each and every
330 + * writepage invocation because it may be mmapped. "A file is mapped
331 + * in multiples of the page size. For a file that is not a multiple of
332 + * the page size, the remaining memory is zeroed when mapped, and
333 + * writes to that region are not written out to the file."
334 + */
335 + if (len < PAGE_CACHE_SIZE)
336 + zero_user_segment(page, len, PAGE_CACHE_SIZE);
337 +
338 + if (ext4_is_encryption_enabled(ei))
339 + ret = ext4_bio_encrypt_and_write(io, page, wbc);
340 + else
341 + ret = ext4_bio_write_buffers(io, page, NULL, len, wbc);
342 + unlock_page(page);
343 return ret;
344 }
345 --
346 2.1.0.rc2.206.gedb03e5