| blob | bf677807703923c2d4dd99c4b64f0a98fc661c19 |
1 ext4 crypto: implement the ext4 encryption write path
3 From: Michael Halcrow <mhalcrow@google.com>
5 Pulls block_write_begin() into fs/ext4/inode.c because it might need
6 to do a low-level read of the existing data, in which case we need to
7 decrypt it.
9 Signed-off-by: Michael Halcrow <mhalcrow@google.com>
10 Signed-off-by: Ildar Muslukhov <ildarm@google.com>
11 Signed-off-by: Theodore Ts'o <tytso@mit.edu>
12 ---
13 fs/ext4/extents.c | 17 +++++++++
14 fs/ext4/ialloc.c | 5 +++
15 fs/ext4/inode.c | 112 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-
16 fs/ext4/page-io.c | 45 ++++++++++++++++++++---
17 4 files changed, 173 insertions(+), 6 deletions(-)
19 diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c
20 index 74580ea..973816b 100644
21 --- a/fs/ext4/extents.c
22 +++ b/fs/ext4/extents.c
23 @@ -3122,6 +3122,9 @@ static int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex)
24 ee_len = ext4_ext_get_actual_len(ex);
25 ee_pblock = ext4_ext_pblock(ex);
27 + if (ext4_encrypted_inode(inode))
28 + return ext4_encrypted_zeroout(inode, ex);
29 +
30 ret = sb_issue_zeroout(inode->i_sb, ee_pblock, ee_len, GFP_NOFS);
31 if (ret > 0)
32 ret = 0;
33 @@ -4898,6 +4901,20 @@ long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
34 ext4_lblk_t lblk;
35 unsigned int blkbits = inode->i_blkbits;
37 + /*
38 + * Encrypted inodes can't handle collapse range or insert
39 + * range since we would need to re-encrypt blocks with a
40 + * different IV or XTS tweak (which are based on the logical
41 + * block number).
42 + *
43 + * XXX It's not clear why zero range isn't working, but we'll
44 + * leave it disabled for encrypted inodes for now. This is a
45 + * bug we should fix....
46 + */
47 + if (ext4_encrypted_inode(inode) &&
48 + (mode & (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE)))
49 + return -EOPNOTSUPP;
50 +
51 /* Return error if mode is not supported */
52 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
53 FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE))
54 diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c
55 index 6ab6f63..247737e 100644
56 --- a/fs/ext4/ialloc.c
57 +++ b/fs/ext4/ialloc.c
58 @@ -996,6 +996,11 @@ got:
59 ei->i_block_group = group;
60 ei->i_last_alloc_group = ~0;
62 + /* If the directory encrypted, then we should encrypt the inode. */
63 + if ((S_ISDIR(mode) || S_ISREG(mode) || S_ISLNK(mode)) &&
64 + ext4_encrypted_inode(dir))
65 + ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT);
66 +
67 ext4_set_inode_flags(inode);
68 if (IS_DIRSYNC(inode))
69 ext4_handle_sync(handle);
70 diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
71 index cd30091..7c4527e 100644
72 --- a/fs/ext4/inode.c
73 +++ b/fs/ext4/inode.c
74 @@ -886,6 +886,95 @@ int do_journal_get_write_access(handle_t *handle,
76 static int ext4_get_block_write_nolock(struct inode *inode, sector_t iblock,
77 struct buffer_head *bh_result, int create);
78 +
79 +#ifdef CONFIG_EXT4_FS_ENCRYPTION
80 +static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
81 + get_block_t *get_block)
82 +{
83 + unsigned from = pos & (PAGE_CACHE_SIZE - 1);
84 + unsigned to = from + len;
85 + struct inode *inode = page->mapping->host;
86 + unsigned block_start, block_end;
87 + sector_t block;
88 + int err = 0;
89 + unsigned blocksize = inode->i_sb->s_blocksize;
90 + unsigned bbits;
91 + struct buffer_head *bh, *head, *wait[2], **wait_bh = wait;
92 + bool decrypt = false;
93 +
94 + BUG_ON(!PageLocked(page));
95 + BUG_ON(from > PAGE_CACHE_SIZE);
96 + BUG_ON(to > PAGE_CACHE_SIZE);
97 + BUG_ON(from > to);
98 +
99 + if (!page_has_buffers(page))
100 + create_empty_buffers(page, blocksize, 0);
101 + head = page_buffers(page);
102 + bbits = ilog2(blocksize);
103 + block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
104 +
105 + for (bh = head, block_start = 0; bh != head || !block_start;
106 + block++, block_start = block_end, bh = bh->b_this_page) {
107 + block_end = block_start + blocksize;
108 + if (block_end <= from || block_start >= to) {
109 + if (PageUptodate(page)) {
110 + if (!buffer_uptodate(bh))
111 + set_buffer_uptodate(bh);
112 + }
113 + continue;
114 + }
115 + if (buffer_new(bh))
116 + clear_buffer_new(bh);
117 + if (!buffer_mapped(bh)) {
118 + WARN_ON(bh->b_size != blocksize);
119 + err = get_block(inode, block, bh, 1);
120 + if (err)
121 + break;
122 + if (buffer_new(bh)) {
123 + unmap_underlying_metadata(bh->b_bdev,
124 + bh->b_blocknr);
125 + if (PageUptodate(page)) {
126 + clear_buffer_new(bh);
127 + set_buffer_uptodate(bh);
128 + mark_buffer_dirty(bh);
129 + continue;
130 + }
131 + if (block_end > to || block_start < from)
132 + zero_user_segments(page, to, block_end,
133 + block_start, from);
134 + continue;
135 + }
136 + }
137 + if (PageUptodate(page)) {
138 + if (!buffer_uptodate(bh))
139 + set_buffer_uptodate(bh);
140 + continue;
141 + }
142 + if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
143 + !buffer_unwritten(bh) &&
144 + (block_start < from || block_end > to)) {
145 + ll_rw_block(READ, 1, &bh);
146 + *wait_bh++ = bh;
147 + decrypt = ext4_encrypted_inode(inode) &&
148 + S_ISREG(inode->i_mode);
149 + }
150 + }
151 + /*
152 + * If we issued read requests, let them complete.
153 + */
154 + while (wait_bh > wait) {
155 + wait_on_buffer(*--wait_bh);
156 + if (!buffer_uptodate(*wait_bh))
157 + err = -EIO;
158 + }
159 + if (unlikely(err))
160 + page_zero_new_buffers(page, from, to);
161 + else if (decrypt)
162 + err = ext4_decrypt_one(inode, page);
163 + return err;
164 +}
165 +#endif
166 +
167 static int ext4_write_begin(struct file *file, struct address_space *mapping,
168 loff_t pos, unsigned len, unsigned flags,
169 struct page **pagep, void **fsdata)
170 @@ -948,11 +1037,19 @@ retry_journal:
171 /* In case writeback began while the page was unlocked */
172 wait_for_stable_page(page);
174 +#ifdef CONFIG_EXT4_FS_ENCRYPTION
175 + if (ext4_should_dioread_nolock(inode))
176 + ret = ext4_block_write_begin(page, pos, len,
177 + ext4_get_block_write);
178 + else
179 + ret = ext4_block_write_begin(page, pos, len,
180 + ext4_get_block);
181 +#else
182 if (ext4_should_dioread_nolock(inode))
183 ret = __block_write_begin(page, pos, len, ext4_get_block_write);
184 else
185 ret = __block_write_begin(page, pos, len, ext4_get_block);
186 -
187 +#endif
188 if (!ret && ext4_should_journal_data(inode)) {
189 ret = ext4_walk_page_buffers(handle, page_buffers(page),
190 from, to, NULL,
191 @@ -2574,7 +2671,12 @@ retry_journal:
192 /* In case writeback began while the page was unlocked */
193 wait_for_stable_page(page);
195 +#ifdef CONFIG_EXT4_FS_ENCRYPTION
196 + ret = ext4_block_write_begin(page, pos, len,
197 + ext4_da_get_block_prep);
198 +#else
199 ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep);
200 +#endif
201 if (ret < 0) {
202 unlock_page(page);
203 ext4_journal_stop(handle);
204 @@ -3032,6 +3134,9 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
205 get_block_func = ext4_get_block_write;
206 dio_flags = DIO_LOCKING;
207 }
208 +#ifdef CONFIG_EXT4_FS_ENCRYPTION
209 + BUG_ON(ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode));
210 +#endif
211 if (IS_DAX(inode))
212 ret = dax_do_io(rw, iocb, inode, iter, offset, get_block_func,
213 ext4_end_io_dio, dio_flags);
214 @@ -3096,6 +3201,11 @@ static ssize_t ext4_direct_IO(int rw, struct kiocb *iocb,
215 size_t count = iov_iter_count(iter);
216 ssize_t ret;
218 +#ifdef CONFIG_EXT4_FS_ENCRYPTION
219 + if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode))
220 + return 0;
221 +#endif
222 +
223 /*
224 * If we are doing data journalling we don't support O_DIRECT
225 */
226 diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c
227 index 5687e47..51a5f12 100644
228 --- a/fs/ext4/page-io.c
229 +++ b/fs/ext4/page-io.c
230 @@ -67,6 +67,10 @@ static void ext4_finish_bio(struct bio *bio)
232 bio_for_each_segment_all(bvec, bio, i) {
233 struct page *page = bvec->bv_page;
234 +#ifdef CONFIG_EXT4_FS_ENCRYPTION
235 + struct page *data_page = NULL;
236 + struct ext4_crypto_ctx *ctx = NULL;
237 +#endif
238 struct buffer_head *bh, *head;
239 unsigned bio_start = bvec->bv_offset;
240 unsigned bio_end = bio_start + bvec->bv_len;
241 @@ -76,6 +80,15 @@ static void ext4_finish_bio(struct bio *bio)
242 if (!page)
243 continue;
245 +#ifdef CONFIG_EXT4_FS_ENCRYPTION
246 + if (!page->mapping) {
247 + /* The bounce data pages are unmapped. */
248 + data_page = page;
249 + ctx = (struct ext4_crypto_ctx *)page_private(data_page);
250 + page = ctx->control_page;
251 + }
252 +#endif
253 +
254 if (error) {
255 SetPageError(page);
256 set_bit(AS_EIO, &page->mapping->flags);
257 @@ -100,8 +113,13 @@ static void ext4_finish_bio(struct bio *bio)
258 } while ((bh = bh->b_this_page) != head);
259 bit_spin_unlock(BH_Uptodate_Lock, &head->b_state);
260 local_irq_restore(flags);
261 - if (!under_io)
262 + if (!under_io) {
263 +#ifdef CONFIG_EXT4_FS_ENCRYPTION
264 + if (ctx)
265 + ext4_restore_control_page(data_page);
266 +#endif
267 end_page_writeback(page);
268 + }
269 }
270 }
272 @@ -376,6 +394,7 @@ static int io_submit_init_bio(struct ext4_io_submit *io,
274 static int io_submit_add_bh(struct ext4_io_submit *io,
275 struct inode *inode,
276 + struct page *page,
277 struct buffer_head *bh)
278 {
279 int ret;
280 @@ -389,7 +408,7 @@ submit_and_retry:
281 if (ret)
282 return ret;
283 }
284 - ret = bio_add_page(io->io_bio, bh->b_page, bh->b_size, bh_offset(bh));
285 + ret = bio_add_page(io->io_bio, page, bh->b_size, bh_offset(bh));
286 if (ret != bh->b_size)
287 goto submit_and_retry;
288 io->io_next_block++;
289 @@ -402,6 +421,7 @@ int ext4_bio_write_page(struct ext4_io_submit *io,
290 struct writeback_control *wbc,
291 bool keep_towrite)
292 {
293 + struct page *data_page = NULL;
294 struct inode *inode = page->mapping->host;
295 unsigned block_start, blocksize;
296 struct buffer_head *bh, *head;
297 @@ -461,19 +481,29 @@ int ext4_bio_write_page(struct ext4_io_submit *io,
298 set_buffer_async_write(bh);
299 } while ((bh = bh->b_this_page) != head);
301 - /* Now submit buffers to write */
302 bh = head = page_buffers(page);
303 +
304 + if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode)) {
305 + data_page = ext4_encrypt(inode, page);
306 + if (IS_ERR(data_page)) {
307 + ret = PTR_ERR(data_page);
308 + data_page = NULL;
309 + goto out;
310 + }
311 + }
312 +
313 + /* Now submit buffers to write */
314 do {
315 if (!buffer_async_write(bh))
316 continue;
317 - ret = io_submit_add_bh(io, inode, bh);
318 + ret = io_submit_add_bh(io, inode,
319 + data_page ? data_page : page, bh);
320 if (ret) {
321 /*
322 * We only get here on ENOMEM. Not much else
323 * we can do but mark the page as dirty, and
324 * better luck next time.
325 */
326 - redirty_page_for_writepage(wbc, page);
327 break;
328 }
329 nr_submitted++;
330 @@ -482,6 +512,11 @@ int ext4_bio_write_page(struct ext4_io_submit *io,
332 /* Error stopped previous loop? Clean up buffers... */
333 if (ret) {
334 + out:
335 + if (data_page)
336 + ext4_restore_control_page(data_page);
337 + printk_ratelimited(KERN_ERR "%s: ret = %d\n", __func__, ret);
338 + redirty_page_for_writepage(wbc, page);
339 do {
340 clear_buffer_async_write(bh);
341 bh = bh->b_this_page;