| blob | 1eae106858a40938ada794054a432dd631229711 |
1 ext4 crypto: add ext4_mpage_readpages()
3 This takes code from fs/mpage.c and optimizes it for ext4. Its
4 primary reason is to allow us to more easily add encryption to ext4's
5 read path in an efficient manner.
7 Signed-off-by: Theodore Ts'o <tytso@mit.edu>
8 ---
9 fs/ext4/Makefile | 2 +-
10 fs/ext4/ext4.h | 4 ++
11 fs/ext4/inode.c | 4 +-
12 fs/ext4/readpage.c | 264 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
13 4 files changed, 271 insertions(+), 3 deletions(-)
15 diff --git a/fs/ext4/Makefile b/fs/ext4/Makefile
16 index 0310fec..cd6f50f 100644
17 --- a/fs/ext4/Makefile
18 +++ b/fs/ext4/Makefile
19 @@ -8,7 +8,7 @@ ext4-y := balloc.o bitmap.o dir.o file.o fsync.o ialloc.o inode.o page-io.o \
20 ioctl.o namei.o super.o symlink.o hash.o resize.o extents.o \
21 ext4_jbd2.o migrate.o mballoc.o block_validity.o move_extent.o \
22 mmp.o indirect.o extents_status.o xattr.o xattr_user.o \
23 - xattr_trusted.o inline.o
24 + xattr_trusted.o inline.o readpage.o
26 ext4-$(CONFIG_EXT4_FS_POSIX_ACL) += acl.o
27 ext4-$(CONFIG_EXT4_FS_SECURITY) += xattr_security.o
28 diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
29 index a75fba6..06e8add 100644
30 --- a/fs/ext4/ext4.h
31 +++ b/fs/ext4/ext4.h
32 @@ -2683,6 +2683,10 @@ static inline void ext4_set_de_type(struct super_block *sb,
33 de->file_type = ext4_type_by_mode[(mode & S_IFMT)>>S_SHIFT];
34 }
36 +/* readpages.c */
37 +extern int ext4_mpage_readpages(struct address_space *mapping,
38 + struct list_head *pages, struct page *page,
39 + unsigned nr_pages);
41 /* symlink.c */
42 extern const struct inode_operations ext4_symlink_inode_operations;
43 diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
44 index 5653fa4..a68cacc 100644
45 --- a/fs/ext4/inode.c
46 +++ b/fs/ext4/inode.c
47 @@ -2798,7 +2798,7 @@ static int ext4_readpage(struct file *file, struct page *page)
48 ret = ext4_readpage_inline(inode, page);
50 if (ret == -EAGAIN)
51 - return mpage_readpage(page, ext4_get_block);
52 + return ext4_mpage_readpages(page->mapping, NULL, page, 1);
54 return ret;
55 }
56 @@ -2813,7 +2813,7 @@ ext4_readpages(struct file *file, struct address_space *mapping,
57 if (ext4_has_inline_data(inode))
58 return 0;
60 - return mpage_readpages(mapping, pages, nr_pages, ext4_get_block);
61 + return ext4_mpage_readpages(mapping, pages, NULL, nr_pages);
62 }
64 static void ext4_invalidatepage(struct page *page, unsigned int offset,
65 diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c
66 new file mode 100644
67 index 0000000..fff9fe6
68 --- /dev/null
69 +++ b/fs/ext4/readpage.c
70 @@ -0,0 +1,264 @@
71 +/*
72 + * linux/fs/ext4/readpage.c
73 + *
74 + * Copyright (C) 2002, Linus Torvalds.
75 + * Copyright (C) 2015, Google, Inc.
76 + *
77 + * This was originally taken from fs/mpage.c
78 + *
79 + * The intent is the ext4_mpage_readpages() function here is intended
80 + * to replace mpage_readpages() in the general case, not just for
81 + * encrypted files. It has some limitations (see below), where it
82 + * will fall back to read_block_full_page(), but these limitations
83 + * should only be hit when page_size != block_size.
84 + *
85 + * This will allow us to attach a callback function to support ext4
86 + * encryption.
87 + *
88 + * If anything unusual happens, such as:
89 + *
90 + * - encountering a page which has buffers
91 + * - encountering a page which has a non-hole after a hole
92 + * - encountering a page with non-contiguous blocks
93 + *
94 + * then this code just gives up and calls the buffer_head-based read function.
95 + * It does handle a page which has holes at the end - that is a common case:
96 + * the end-of-file on blocksize < PAGE_CACHE_SIZE setups.
97 + *
98 + */
99 +
100 +#include <linux/kernel.h>
101 +#include <linux/export.h>
102 +#include <linux/mm.h>
103 +#include <linux/kdev_t.h>
104 +#include <linux/gfp.h>
105 +#include <linux/bio.h>
106 +#include <linux/fs.h>
107 +#include <linux/buffer_head.h>
108 +#include <linux/blkdev.h>
109 +#include <linux/highmem.h>
110 +#include <linux/prefetch.h>
111 +#include <linux/mpage.h>
112 +#include <linux/writeback.h>
113 +#include <linux/backing-dev.h>
114 +#include <linux/pagevec.h>
115 +#include <linux/cleancache.h>
116 +
117 +#include "ext4.h"
118 +
119 +/*
120 + * I/O completion handler for multipage BIOs.
121 + *
122 + * The mpage code never puts partial pages into a BIO (except for end-of-file).
123 + * If a page does not map to a contiguous run of blocks then it simply falls
124 + * back to block_read_full_page().
125 + *
126 + * Why is this? If a page's completion depends on a number of different BIOs
127 + * which can complete in any order (or at the same time) then determining the
128 + * status of that page is hard. See end_buffer_async_read() for the details.
129 + * There is no point in duplicating all that complexity.
130 + */
131 +static void mpage_end_io(struct bio *bio, int err)
132 +{
133 + struct bio_vec *bv;
134 + int i;
135 +
136 + bio_for_each_segment_all(bv, bio, i) {
137 + struct page *page = bv->bv_page;
138 +
139 + if (!err) {
140 + SetPageUptodate(page);
141 + } else {
142 + ClearPageUptodate(page);
143 + SetPageError(page);
144 + }
145 + unlock_page(page);
146 + }
147 +
148 + bio_put(bio);
149 +}
150 +
151 +int ext4_mpage_readpages(struct address_space *mapping,
152 + struct list_head *pages, struct page *page,
153 + unsigned nr_pages)
154 +{
155 + struct bio *bio = NULL;
156 + unsigned page_idx;
157 + sector_t last_block_in_bio = 0;
158 +
159 + struct inode *inode = mapping->host;
160 + const unsigned blkbits = inode->i_blkbits;
161 + const unsigned blocks_per_page = PAGE_CACHE_SIZE >> blkbits;
162 + const unsigned blocksize = 1 << blkbits;
163 + sector_t block_in_file;
164 + sector_t last_block;
165 + sector_t last_block_in_file;
166 + sector_t blocks[MAX_BUF_PER_PAGE];
167 + unsigned page_block;
168 + struct block_device *bdev = inode->i_sb->s_bdev;
169 + int length;
170 + unsigned relative_block = 0;
171 + struct ext4_map_blocks map;
172 +
173 + map.m_pblk = 0;
174 + map.m_lblk = 0;
175 + map.m_len = 0;
176 + map.m_flags = 0;
177 +
178 + for (page_idx = 0; nr_pages; page_idx++, nr_pages--) {
179 + int fully_mapped = 1;
180 + unsigned first_hole = blocks_per_page;
181 +
182 + prefetchw(&page->flags);
183 + if (pages) {
184 + page = list_entry(pages->prev, struct page, lru);
185 + list_del(&page->lru);
186 + if (add_to_page_cache_lru(page, mapping,
187 + page->index, GFP_KERNEL))
188 + goto next_page;
189 + }
190 +
191 + if (page_has_buffers(page))
192 + goto confused;
193 +
194 + block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
195 + last_block = block_in_file + nr_pages * blocks_per_page;
196 + last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
197 + if (last_block > last_block_in_file)
198 + last_block = last_block_in_file;
199 + page_block = 0;
200 +
201 + /*
202 + * Map blocks using the previous result first.
203 + */
204 + if ((map.m_flags & EXT4_MAP_MAPPED) &&
205 + block_in_file > map.m_lblk &&
206 + block_in_file < (map.m_lblk + map.m_len)) {
207 + unsigned map_offset = block_in_file - map.m_lblk;
208 + unsigned last = map.m_len - map_offset;
209 +
210 + for (relative_block = 0; ; relative_block++) {
211 + if (relative_block == last) {
212 + /* needed? */
213 + map.m_flags &= ~EXT4_MAP_MAPPED;
214 + break;
215 + }
216 + if (page_block == blocks_per_page)
217 + break;
218 + blocks[page_block] = map.m_pblk + map_offset +
219 + relative_block;
220 + page_block++;
221 + block_in_file++;
222 + }
223 + }
224 +
225 + /*
226 + * Then do more ext4_map_blocks() calls until we are
227 + * done with this page.
228 + */
229 + while (page_block < blocks_per_page) {
230 + if (block_in_file < last_block) {
231 + map.m_lblk = block_in_file;
232 + map.m_len = last_block - block_in_file;
233 +
234 + if (ext4_map_blocks(NULL, inode, &map, 0) < 0) {
235 + set_error_page:
236 + SetPageError(page);
237 + zero_user_segment(page, 0,
238 + PAGE_CACHE_SIZE);
239 + unlock_page(page);
240 + goto next_page;
241 + }
242 + }
243 + if ((map.m_flags & EXT4_MAP_MAPPED) == 0) {
244 + fully_mapped = 0;
245 + if (first_hole == blocks_per_page)
246 + first_hole = page_block;
247 + page_block++;
248 + block_in_file++;
249 + continue;
250 + }
251 + if (first_hole != blocks_per_page)
252 + goto confused; /* hole -> non-hole */
253 +
254 + /* Contiguous blocks? */
255 + if (page_block && blocks[page_block-1] != map.m_pblk-1)
256 + goto confused;
257 + for (relative_block = 0; ; relative_block++) {
258 + if (relative_block == map.m_len) {
259 + /* needed? */
260 + map.m_flags &= ~EXT4_MAP_MAPPED;
261 + break;
262 + } else if (page_block == blocks_per_page)
263 + break;
264 + blocks[page_block] = map.m_pblk+relative_block;
265 + page_block++;
266 + block_in_file++;
267 + }
268 + }
269 + if (first_hole != blocks_per_page) {
270 + zero_user_segment(page, first_hole << blkbits,
271 + PAGE_CACHE_SIZE);
272 + if (first_hole == 0) {
273 + SetPageUptodate(page);
274 + unlock_page(page);
275 + goto next_page;
276 + }
277 + } else if (fully_mapped) {
278 + SetPageMappedToDisk(page);
279 + }
280 + if (fully_mapped && blocks_per_page == 1 &&
281 + !PageUptodate(page) && cleancache_get_page(page) == 0) {
282 + SetPageUptodate(page);
283 + goto confused;
284 + }
285 +
286 + /*
287 + * This page will go to BIO. Do we need to send this
288 + * BIO off first?
289 + */
290 + if (bio && (last_block_in_bio != blocks[0] - 1)) {
291 + submit_and_realloc:
292 + submit_bio(READ, bio);
293 + bio = NULL;
294 + }
295 + if (bio == NULL) {
296 + bio = bio_alloc(GFP_KERNEL,
297 + min_t(int, nr_pages, bio_get_nr_vecs(bdev)));
298 + if (!bio)
299 + goto set_error_page;
300 + bio->bi_bdev = bdev;
301 + bio->bi_iter.bi_sector = blocks[0] << (blkbits - 9);
302 + bio->bi_end_io = mpage_end_io;
303 + }
304 +
305 + length = first_hole << blkbits;
306 + if (bio_add_page(bio, page, length, 0) < length)
307 + goto submit_and_realloc;
308 +
309 + if (((map.m_flags & EXT4_MAP_BOUNDARY) &&
310 + (relative_block == map.m_len)) ||
311 + (first_hole != blocks_per_page)) {
312 + submit_bio(READ, bio);
313 + bio = NULL;
314 + } else
315 + last_block_in_bio = blocks[blocks_per_page - 1];
316 + goto next_page;
317 + confused:
318 + if (bio) {
319 + submit_bio(READ, bio);
320 + bio = NULL;
321 + }
322 + if (!PageUptodate(page))
323 + block_read_full_page(page, ext4_get_block);
324 + else
325 + unlock_page(page);
326 + next_page:
327 + if (pages)
328 + page_cache_release(page);
329 + }
330 + BUG_ON(pages && !list_empty(pages));
331 + if (bio)
332 + submit_bio(READ, bio);
333 + return 0;
334 +}