| blob | 37fc325c50254fa10ed02280d6695479d97abe3f |
1 ext4: Add multi block allocator for ext4
3 From: Alex Tomas <alex@clusterfs.com>
5 Signed-off-by: Alex Tomas <alex@clusterfs.com>
6 Signed-off-by: Andreas Dilger <adilger@clusterfs.com>
7 Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
8 Signed-off-by: Eric Sandeen <sandeen@redhat.com>
9 Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
11 diff --git a/fs/ext4/Makefile b/fs/ext4/Makefile
12 index d5fd80b..ac6fa8c 100644
13 --- a/fs/ext4/Makefile
14 +++ b/fs/ext4/Makefile
15 @@ -6,7 +6,7 @@ obj-$(CONFIG_EXT4DEV_FS) += ext4dev.o
17 ext4dev-y := balloc.o bitmap.o dir.o file.o fsync.o ialloc.o inode.o \
18 ioctl.o namei.o super.o symlink.o hash.o resize.o extents.o \
19 - ext4_jbd2.o migrate.o
20 + ext4_jbd2.o migrate.o mballoc.o
22 ext4dev-$(CONFIG_EXT4DEV_FS_XATTR) += xattr.o xattr_user.o xattr_trusted.o
23 ext4dev-$(CONFIG_EXT4DEV_FS_POSIX_ACL) += acl.o
24 diff --git a/fs/ext4/balloc.c b/fs/ext4/balloc.c
25 index 54d3da7..643046b 100644
26 --- a/fs/ext4/balloc.c
27 +++ b/fs/ext4/balloc.c
28 @@ -577,6 +577,8 @@ void ext4_discard_reservation(struct inode *inode)
29 struct ext4_reserve_window_node *rsv;
30 spinlock_t *rsv_lock = &EXT4_SB(inode->i_sb)->s_rsv_window_lock;
32 + ext4_mb_discard_inode_preallocations(inode);
33 +
34 if (!block_i)
35 return;
37 @@ -785,19 +787,29 @@ error_return:
38 * @inode: inode
39 * @block: start physical block to free
40 * @count: number of blocks to count
41 + * @metadata: Are these metadata blocks
42 */
43 void ext4_free_blocks(handle_t *handle, struct inode *inode,
44 - ext4_fsblk_t block, unsigned long count)
45 + ext4_fsblk_t block, unsigned long count,
46 + int metadata)
47 {
48 struct super_block * sb;
49 unsigned long dquot_freed_blocks;
51 + /* this isn't the right place to decide whether block is metadata
52 + * inode.c/extents.c knows better, but for safety ... */
53 + if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode) ||
54 + ext4_should_journal_data(inode))
55 + metadata = 1;
56 +
57 sb = inode->i_sb;
58 - if (!sb) {
59 - printk ("ext4_free_blocks: nonexistent device");
60 - return;
61 - }
62 - ext4_free_blocks_sb(handle, sb, block, count, &dquot_freed_blocks);
63 +
64 + if (!test_opt(sb, MBALLOC) || !EXT4_SB(sb)->s_group_info)
65 + ext4_free_blocks_sb(handle, sb, block, count,
66 + &dquot_freed_blocks);
67 + else
68 + ext4_mb_free_blocks(handle, inode, block, count,
69 + metadata, &dquot_freed_blocks);
70 if (dquot_freed_blocks)
71 DQUOT_FREE_BLOCK(inode, dquot_freed_blocks);
72 return;
73 @@ -1576,7 +1588,7 @@ int ext4_should_retry_alloc(struct super_block *sb, int *retries)
74 }
76 /**
77 - * ext4_new_blocks() -- core block(s) allocation function
78 + * ext4_new_blocks_old() -- core block(s) allocation function
79 * @handle: handle to this transaction
80 * @inode: file inode
81 * @goal: given target block(filesystem wide)
82 @@ -1589,7 +1601,7 @@ int ext4_should_retry_alloc(struct super_block *sb, int *retries)
83 * any specific goal block.
84 *
85 */
86 -ext4_fsblk_t ext4_new_blocks(handle_t *handle, struct inode *inode,
87 +ext4_fsblk_t ext4_new_blocks_old(handle_t *handle, struct inode *inode,
88 ext4_fsblk_t goal, unsigned long *count, int *errp)
89 {
90 struct buffer_head *bitmap_bh = NULL;
91 @@ -1849,13 +1861,46 @@ out:
92 }
94 ext4_fsblk_t ext4_new_block(handle_t *handle, struct inode *inode,
95 - ext4_fsblk_t goal, int *errp)
96 + ext4_fsblk_t goal, int *errp)
97 +{
98 + struct ext4_allocation_request ar;
99 + ext4_fsblk_t ret;
100 +
101 + if (!test_opt(inode->i_sb, MBALLOC)) {
102 + unsigned long count = 1;
103 + ret = ext4_new_blocks_old(handle, inode, goal, &count, errp);
104 + return ret;
105 + }
106 +
107 + memset(&ar, 0, sizeof(ar));
108 + ar.inode = inode;
109 + ar.goal = goal;
110 + ar.len = 1;
111 + ret = ext4_mb_new_blocks(handle, &ar, errp);
112 + return ret;
113 +}
114 +
115 +ext4_fsblk_t ext4_new_blocks(handle_t *handle, struct inode *inode,
116 + ext4_fsblk_t goal, unsigned long *count, int *errp)
117 {
118 - unsigned long count = 1;
119 + struct ext4_allocation_request ar;
120 + ext4_fsblk_t ret;
122 - return ext4_new_blocks(handle, inode, goal, &count, errp);
123 + if (!test_opt(inode->i_sb, MBALLOC)) {
124 + ret = ext4_new_blocks_old(handle, inode, goal, count, errp);
125 + return ret;
126 + }
127 +
128 + memset(&ar, 0, sizeof(ar));
129 + ar.inode = inode;
130 + ar.goal = goal;
131 + ar.len = *count;
132 + ret = ext4_mb_new_blocks(handle, &ar, errp);
133 + *count = ar.len;
134 + return ret;
135 }
137 +
138 /**
139 * ext4_count_free_blocks() -- count filesystem free blocks
140 * @sb: superblock
141 diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c
142 index a60227c..8cf5545 100644
143 --- a/fs/ext4/extents.c
144 +++ b/fs/ext4/extents.c
145 @@ -853,7 +853,7 @@ cleanup:
146 for (i = 0; i < depth; i++) {
147 if (!ablocks[i])
148 continue;
149 - ext4_free_blocks(handle, inode, ablocks[i], 1);
150 + ext4_free_blocks(handle, inode, ablocks[i], 1, 1);
151 }
152 }
153 kfree(ablocks);
154 @@ -1698,7 +1698,7 @@ static int ext4_ext_rm_idx(handle_t *handle, struct inode *inode,
155 ext_debug("index is empty, remove it, free block %llu\n", leaf);
156 bh = sb_find_get_block(inode->i_sb, leaf);
157 ext4_forget(handle, 1, inode, bh, leaf);
158 - ext4_free_blocks(handle, inode, leaf, 1);
159 + ext4_free_blocks(handle, inode, leaf, 1, 1);
160 return err;
161 }
163 @@ -1759,8 +1759,10 @@ static int ext4_remove_blocks(handle_t *handle, struct inode *inode,
164 {
165 struct buffer_head *bh;
166 unsigned short ee_len = ext4_ext_get_actual_len(ex);
167 - int i;
168 + int i, metadata = 0;
170 + if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
171 + metadata = 1;
172 #ifdef EXTENTS_STATS
173 {
174 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
175 @@ -1789,7 +1791,7 @@ static int ext4_remove_blocks(handle_t *handle, struct inode *inode,
176 bh = sb_find_get_block(inode->i_sb, start + i);
177 ext4_forget(handle, 0, inode, bh, start + i);
178 }
179 - ext4_free_blocks(handle, inode, start, num);
180 + ext4_free_blocks(handle, inode, start, num, metadata);
181 } else if (from == le32_to_cpu(ex->ee_block)
182 && to <= le32_to_cpu(ex->ee_block) + ee_len - 1) {
183 printk(KERN_INFO "strange request: removal %u-%u from %u:%u\n",
184 @@ -2287,6 +2289,7 @@ int ext4_ext_get_blocks(handle_t *handle, struct inode *inode,
185 ext4_fsblk_t goal, newblock;
186 int err = 0, depth, ret;
187 unsigned long allocated = 0;
188 + struct ext4_allocation_request ar;
190 __clear_bit(BH_New, &bh_result->b_state);
191 ext_debug("blocks %u/%lu requested for inode %u\n",
192 @@ -2397,8 +2400,15 @@ int ext4_ext_get_blocks(handle_t *handle, struct inode *inode,
193 if (S_ISREG(inode->i_mode) && (!EXT4_I(inode)->i_block_alloc_info))
194 ext4_init_block_alloc_info(inode);
196 - /* allocate new block */
197 - goal = ext4_ext_find_goal(inode, path, iblock);
198 + /* find neighbour allocated blocks */
199 + ar.lleft = iblock;
200 + err = ext4_ext_search_left(inode, path, &ar.lleft, &ar.pleft);
201 + if (err)
202 + goto out2;
203 + ar.lright = iblock;
204 + err = ext4_ext_search_right(inode, path, &ar.lright, &ar.pright);
205 + if (err)
206 + goto out2;
208 /*
209 * See if request is beyond maximum number of blocks we can have in
210 @@ -2421,7 +2431,18 @@ int ext4_ext_get_blocks(handle_t *handle, struct inode *inode,
211 allocated = le16_to_cpu(newex.ee_len);
212 else
213 allocated = max_blocks;
214 - newblock = ext4_new_blocks(handle, inode, goal, &allocated, &err);
215 +
216 + /* allocate new block */
217 + ar.inode = inode;
218 + ar.goal = ext4_ext_find_goal(inode, path, iblock);
219 + ar.logical = iblock;
220 + ar.len = allocated;
221 + if (S_ISREG(inode->i_mode))
222 + ar.flags = EXT4_MB_HINT_DATA;
223 + else
224 + /* disable in-core preallocation for non-regular files */
225 + ar.flags = 0;
226 + newblock = ext4_mb_new_blocks(handle, &ar, &err);
227 if (!newblock)
228 goto out2;
229 ext_debug("allocate new block: goal %llu, found %llu/%lu\n",
230 @@ -2429,14 +2450,17 @@ int ext4_ext_get_blocks(handle_t *handle, struct inode *inode,
232 /* try to insert new extent into found leaf and return */
233 ext4_ext_store_pblock(&newex, newblock);
234 - newex.ee_len = cpu_to_le16(allocated);
235 + newex.ee_len = cpu_to_le16(ar.len);
236 if (create == EXT4_CREATE_UNINITIALIZED_EXT) /* Mark uninitialized */
237 ext4_ext_mark_uninitialized(&newex);
238 err = ext4_ext_insert_extent(handle, inode, path, &newex);
239 if (err) {
240 /* free data blocks we just allocated */
241 + /* not a good idea to call discard here directly,
242 + * but otherwise we'd need to call it every free() */
243 + ext4_mb_discard_inode_preallocations(inode);
244 ext4_free_blocks(handle, inode, ext_pblock(&newex),
245 - le16_to_cpu(newex.ee_len));
246 + le16_to_cpu(newex.ee_len), 0);
247 goto out2;
248 }
250 @@ -2445,6 +2469,7 @@ int ext4_ext_get_blocks(handle_t *handle, struct inode *inode,
252 /* previous routine could use block we allocated */
253 newblock = ext_pblock(&newex);
254 + allocated = le16_to_cpu(newex.ee_len);
255 outnew:
256 __set_bit(BH_New, &bh_result->b_state);
258 @@ -2496,6 +2521,8 @@ void ext4_ext_truncate(struct inode * inode, struct page *page)
259 down_write(&EXT4_I(inode)->i_data_sem);
260 ext4_ext_invalidate_cache(inode);
262 + ext4_mb_discard_inode_preallocations(inode);
263 +
264 /*
265 * TODO: optimization is possible here.
266 * Probably we need not scan at all,
267 diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
268 index 3c013e5..2947800 100644
269 --- a/fs/ext4/inode.c
270 +++ b/fs/ext4/inode.c
271 @@ -551,7 +551,7 @@ static int ext4_alloc_blocks(handle_t *handle, struct inode *inode,
272 return ret;
273 failed_out:
274 for (i = 0; i <index; i++)
275 - ext4_free_blocks(handle, inode, new_blocks[i], 1);
276 + ext4_free_blocks(handle, inode, new_blocks[i], 1, 0);
277 return ret;
278 }
280 @@ -650,9 +650,9 @@ failed:
281 ext4_journal_forget(handle, branch[i].bh);
282 }
283 for (i = 0; i <indirect_blks; i++)
284 - ext4_free_blocks(handle, inode, new_blocks[i], 1);
285 + ext4_free_blocks(handle, inode, new_blocks[i], 1, 0);
287 - ext4_free_blocks(handle, inode, new_blocks[i], num);
288 + ext4_free_blocks(handle, inode, new_blocks[i], num, 0);
290 return err;
291 }
292 @@ -749,9 +749,10 @@ err_out:
293 for (i = 1; i <= num; i++) {
294 BUFFER_TRACE(where[i].bh, "call jbd2_journal_forget");
295 ext4_journal_forget(handle, where[i].bh);
296 - ext4_free_blocks(handle,inode,le32_to_cpu(where[i-1].key),1);
297 + ext4_free_blocks(handle, inode,
298 + le32_to_cpu(where[i-1].key), 1, 0);
299 }
300 - ext4_free_blocks(handle, inode, le32_to_cpu(where[num].key), blks);
301 + ext4_free_blocks(handle, inode, le32_to_cpu(where[num].key), blks, 0);
303 return err;
304 }
305 @@ -2051,7 +2052,7 @@ static void ext4_clear_blocks(handle_t *handle, struct inode *inode,
306 }
307 }
309 - ext4_free_blocks(handle, inode, block_to_free, count);
310 + ext4_free_blocks(handle, inode, block_to_free, count, 0);
311 }
313 /**
314 @@ -2224,7 +2225,7 @@ static void ext4_free_branches(handle_t *handle, struct inode *inode,
315 ext4_journal_test_restart(handle, inode);
316 }
318 - ext4_free_blocks(handle, inode, nr, 1);
319 + ext4_free_blocks(handle, inode, nr, 1, 1);
321 if (parent_bh) {
322 /*
323 diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c
324 new file mode 100644
325 index 0000000..aacbbb2
326 --- /dev/null
327 +++ b/fs/ext4/mballoc.c
328 @@ -0,0 +1,4551 @@
329 +/*
330 + * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
331 + * Written by Alex Tomas <alex@clusterfs.com>
332 + *
333 + * This program is free software; you can redistribute it and/or modify
334 + * it under the terms of the GNU General Public License version 2 as
335 + * published by the Free Software Foundation.
336 + *
337 + * This program is distributed in the hope that it will be useful,
338 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
339 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
340 + * GNU General Public License for more details.
341 + *
342 + * You should have received a copy of the GNU General Public Licens
343 + * along with this program; if not, write to the Free Software
344 + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
345 + */
346 +
347 +
348 +/*
349 + * mballoc.c contains the multiblocks allocation routines
350 + */
351 +
352 +#include <linux/time.h>
353 +#include <linux/fs.h>
354 +#include <linux/namei.h>
355 +#include <linux/ext4_jbd2.h>
356 +#include <linux/ext4_fs.h>
357 +#include <linux/quotaops.h>
358 +#include <linux/buffer_head.h>
359 +#include <linux/module.h>
360 +#include <linux/swap.h>
361 +#include <linux/proc_fs.h>
362 +#include <linux/pagemap.h>
363 +#include <linux/seq_file.h>
364 +#include <linux/version.h>
365 +#include "group.h"
366 +
367 +/*
368 + * MUSTDO:
369 + * - test ext4_ext_search_left() and ext4_ext_search_right()
370 + * - search for metadata in few groups
371 + *
372 + * TODO v4:
373 + * - normalization should take into account whether file is still open
374 + * - discard preallocations if no free space left (policy?)
375 + * - don't normalize tails
376 + * - quota
377 + * - reservation for superuser
378 + *
379 + * TODO v3:
380 + * - bitmap read-ahead (proposed by Oleg Drokin aka green)
381 + * - track min/max extents in each group for better group selection
382 + * - mb_mark_used() may allocate chunk right after splitting buddy
383 + * - tree of groups sorted by number of free blocks
384 + * - error handling
385 + */
386 +
387 +/*
388 + * The allocation request involve request for multiple number of blocks
389 + * near to the goal(block) value specified.
390 + *
391 + * During initialization phase of the allocator we decide to use the group
392 + * preallocation or inode preallocation depending on the size file. The
393 + * size of the file could be the resulting file size we would have after
394 + * allocation or the current file size which ever is larger. If the size is
395 + * less that sbi->s_mb_stream_request we select the group
396 + * preallocation. The default value of s_mb_stream_request is 16
397 + * blocks. This can also be tuned via
398 + * /proc/fs/ext4/<partition>/stream_req. The value is represented in terms
399 + * of number of blocks.
400 + *
401 + * The main motivation for having small file use group preallocation is to
402 + * ensure that we have small file closer in the disk.
403 + *
404 + * First stage the allocator looks at the inode prealloc list
405 + * ext4_inode_info->i_prealloc_list contain list of prealloc spaces for
406 + * this particular inode. The inode prealloc space is represented as:
407 + *
408 + * pa_lstart -> the logical start block for this prealloc space
409 + * pa_pstart -> the physical start block for this prealloc space
410 + * pa_len -> lenght for this prealloc space
411 + * pa_free -> free space available in this prealloc space
412 + *
413 + * The inode preallocation space is used looking at the _logical_ start
414 + * block. If only the logical file block falls within the range of prealloc
415 + * space we will consume the particular prealloc space. This make sure that
416 + * that the we have contiguous physical blocks representing the file blocks
417 + *
418 + * The important thing to be noted in case of inode prealloc space is that
419 + * we don't modify the values associated to inode prealloc space except
420 + * pa_free.
421 + *
422 + * If we are not able to find blocks in the inode prealloc space and if we
423 + * have the group allocation flag set then we look at the locality group
424 + * prealloc space. These are per CPU prealloc list repreasented as
425 + *
426 + * ext4_sb_info.s_locality_groups[smp_processor_id()]
427 + *
428 + * The reason for having a per cpu locality group is to reduce the contention
429 + * between CPUs. It is possible to get scheduled at this point.
430 + *
431 + * The locality group prealloc space is used looking at whether we have
432 + * enough free space (pa_free) withing the prealloc space.
433 + *
434 + * If we can't allocate blocks via inode prealloc or/and locality group
435 + * prealloc then we look at the buddy cache. The buddy cache is represented
436 + * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets
437 + * mapped to the buddy and bitmap information regarding different
438 + * groups. The buddy information is attached to buddy cache inode so that
439 + * we can access them through the page cache. The information regarding
440 + * each group is loaded via ext4_mb_load_buddy. The information involve
441 + * block bitmap and buddy information. The information are stored in the
442 + * inode as:
443 + *
444 + * { page }
445 + * [ group 0 buddy][ group 0 bitmap] [group 1][ group 1]...
446 + *
447 + *
448 + * one block each for bitmap and buddy information. So for each group we
449 + * take up 2 blocks. A page can contain blocks_per_page (PAGE_CACHE_SIZE /
450 + * blocksize) blocks. So it can have information regarding groups_per_page
451 + * which is blocks_per_page/2
452 + *
453 + * The buddy cache inode is not stored on disk. The inode is thrown
454 + * away when the filesystem is unmounted.
455 + *
456 + * We look for count number of blocks in the buddy cache. If we were able
457 + * to locate that many free blocks we return with additional information
458 + * regarding rest of the contiguous physical block available
459 + *
460 + * Before allocating blocks via buddy cache we normalize the request
461 + * blocks. This ensure we ask for more blocks that we needed. The extra
462 + * blocks that we get after allocation is added to the respective prealloc
463 + * list. In case of inode preallocation we follow a list of heuristics
464 + * based on file size. This can be found in ext4_mb_normalize_request. If
465 + * we are doing a group prealloc we try to normalize the request to
466 + * sbi->s_mb_group_prealloc. Default value of s_mb_group_prealloc is set to
467 + * 512 blocks. This can be tuned via
468 + * /proc/fs/ext4/<partition/group_prealloc. The value is represented in
469 + * terms of number of blocks. If we have mounted the file system with -O
470 + * stripe=<value> option the group prealloc request is normalized to the
471 + * stripe value (sbi->s_stripe)
472 + *
473 + * The regular allocator(using the buddy cache) support few tunables.
474 + *
475 + * /proc/fs/ext4/<partition>/min_to_scan
476 + * /proc/fs/ext4/<partition>/max_to_scan
477 + * /proc/fs/ext4/<partition>/order2_req
478 + *
479 + * The regular allocator use buddy scan only if the request len is power of
480 + * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The
481 + * value of s_mb_order2_reqs can be tuned via
482 + * /proc/fs/ext4/<partition>/order2_req. If the request len is equal to
483 + * stripe size (sbi->s_stripe), we try to search for contigous block in
484 + * stripe size. This should result in better allocation on RAID setup. If
485 + * not we search in the specific group using bitmap for best extents. The
486 + * tunable min_to_scan and max_to_scan controll the behaviour here.
487 + * min_to_scan indicate how long the mballoc __must__ look for a best
488 + * extent and max_to_scanindicate how long the mballoc __can__ look for a
489 + * best extent in the found extents. Searching for the blocks starts with
490 + * the group specified as the goal value in allocation context via
491 + * ac_g_ex. Each group is first checked based on the criteria whether it
492 + * can used for allocation. ext4_mb_good_group explains how the groups are
493 + * checked.
494 + *
495 + * Both the prealloc space are getting populated as above. So for the first
496 + * request we will hit the buddy cache which will result in this prealloc
497 + * space getting filled. The prealloc space is then later used for the
498 + * subsequent request.
499 + */
500 +
501 +/*
502 + * mballoc operates on the following data:
503 + * - on-disk bitmap
504 + * - in-core buddy (actually includes buddy and bitmap)
505 + * - preallocation descriptors (PAs)
506 + *
507 + * there are two types of preallocations:
508 + * - inode
509 + * assiged to specific inode and can be used for this inode only.
510 + * it describes part of inode's space preallocated to specific
511 + * physical blocks. any block from that preallocated can be used
512 + * independent. the descriptor just tracks number of blocks left
513 + * unused. so, before taking some block from descriptor, one must
514 + * make sure corresponded logical block isn't allocated yet. this
515 + * also means that freeing any block within descriptor's range
516 + * must discard all preallocated blocks.
517 + * - locality group
518 + * assigned to specific locality group which does not translate to
519 + * permanent set of inodes: inode can join and leave group. space
520 + * from this type of preallocation can be used for any inode. thus
521 + * it's consumed from the beginning to the end.
522 + *
523 + * relation between them can be expressed as:
524 + * in-core buddy = on-disk bitmap + preallocation descriptors
525 + *
526 + * this mean blocks mballoc considers used are:
527 + * - allocated blocks (persistent)
528 + * - preallocated blocks (non-persistent)
529 + *
530 + * consistency in mballoc world means that at any time a block is either
531 + * free or used in ALL structures. notice: "any time" should not be read
532 + * literally -- time is discrete and delimited by locks.
533 + *
534 + * to keep it simple, we don't use block numbers, instead we count number of
535 + * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA.
536 + *
537 + * all operations can be expressed as:
538 + * - init buddy: buddy = on-disk + PAs
539 + * - new PA: buddy += N; PA = N
540 + * - use inode PA: on-disk += N; PA -= N
541 + * - discard inode PA buddy -= on-disk - PA; PA = 0
542 + * - use locality group PA on-disk += N; PA -= N
543 + * - discard locality group PA buddy -= PA; PA = 0
544 + * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap
545 + * is used in real operation because we can't know actual used
546 + * bits from PA, only from on-disk bitmap
547 + *
548 + * if we follow this strict logic, then all operations above should be atomic.
549 + * given some of them can block, we'd have to use something like semaphores
550 + * killing performance on high-end SMP hardware. let's try to relax it using
551 + * the following knowledge:
552 + * 1) if buddy is referenced, it's already initialized
553 + * 2) while block is used in buddy and the buddy is referenced,
554 + * nobody can re-allocate that block
555 + * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has
556 + * bit set and PA claims same block, it's OK. IOW, one can set bit in
557 + * on-disk bitmap if buddy has same bit set or/and PA covers corresponded
558 + * block
559 + *
560 + * so, now we're building a concurrency table:
561 + * - init buddy vs.
562 + * - new PA
563 + * blocks for PA are allocated in the buddy, buddy must be referenced
564 + * until PA is linked to allocation group to avoid concurrent buddy init
565 + * - use inode PA
566 + * we need to make sure that either on-disk bitmap or PA has uptodate data
567 + * given (3) we care that PA-=N operation doesn't interfere with init
568 + * - discard inode PA
569 + * the simplest way would be to have buddy initialized by the discard
570 + * - use locality group PA
571 + * again PA-=N must be serialized with init
572 + * - discard locality group PA
573 + * the simplest way would be to have buddy initialized by the discard
574 + * - new PA vs.
575 + * - use inode PA
576 + * i_data_sem serializes them
577 + * - discard inode PA
578 + * discard process must wait until PA isn't used by another process
579 + * - use locality group PA
580 + * some mutex should serialize them
581 + * - discard locality group PA
582 + * discard process must wait until PA isn't used by another process
583 + * - use inode PA
584 + * - use inode PA
585 + * i_data_sem or another mutex should serializes them
586 + * - discard inode PA
587 + * discard process must wait until PA isn't used by another process
588 + * - use locality group PA
589 + * nothing wrong here -- they're different PAs covering different blocks
590 + * - discard locality group PA
591 + * discard process must wait until PA isn't used by another process
592 + *
593 + * now we're ready to make few consequences:
594 + * - PA is referenced and while it is no discard is possible
595 + * - PA is referenced until block isn't marked in on-disk bitmap
596 + * - PA changes only after on-disk bitmap
597 + * - discard must not compete with init. either init is done before
598 + * any discard or they're serialized somehow
599 + * - buddy init as sum of on-disk bitmap and PAs is done atomically
600 + *
601 + * a special case when we've used PA to emptiness. no need to modify buddy
602 + * in this case, but we should care about concurrent init
603 + *
604 + */
605 +
606 + /*
607 + * Logic in few words:
608 + *
609 + * - allocation:
610 + * load group
611 + * find blocks
612 + * mark bits in on-disk bitmap
613 + * release group
614 + *
615 + * - use preallocation:
616 + * find proper PA (per-inode or group)
617 + * load group
618 + * mark bits in on-disk bitmap
619 + * release group
620 + * release PA
621 + *
622 + * - free:
623 + * load group
624 + * mark bits in on-disk bitmap
625 + * release group
626 + *
627 + * - discard preallocations in group:
628 + * mark PAs deleted
629 + * move them onto local list
630 + * load on-disk bitmap
631 + * load group
632 + * remove PA from object (inode or locality group)
633 + * mark free blocks in-core
634 + *
635 + * - discard inode's preallocations:
636 + */
637 +
638 +/*
639 + * Locking rules
640 + *
641 + * Locks:
642 + * - bitlock on a group (group)
643 + * - object (inode/locality) (object)
644 + * - per-pa lock (pa)
645 + *
646 + * Paths:
647 + * - new pa
648 + * object
649 + * group
650 + *
651 + * - find and use pa:
652 + * pa
653 + *
654 + * - release consumed pa:
655 + * pa
656 + * group
657 + * object
658 + *
659 + * - generate in-core bitmap:
660 + * group
661 + * pa
662 + *
663 + * - discard all for given object (inode, locality group):
664 + * object
665 + * pa
666 + * group
667 + *
668 + * - discard all for given group:
669 + * group
670 + * pa
671 + * group
672 + * object
673 + *
674 + */
675 +
676 +/*
677 + * with AGGRESSIVE_CHECK allocator runs consistency checks over
678 + * structures. these checks slow things down a lot
679 + */
680 +#define AGGRESSIVE_CHECK__
681 +
682 +/*
683 + * with DOUBLE_CHECK defined mballoc creates persistent in-core
684 + * bitmaps, maintains and uses them to check for double allocations
685 + */
686 +#define DOUBLE_CHECK__
687 +
688 +/*
689 + */
690 +#define MB_DEBUG__
691 +#ifdef MB_DEBUG
692 +#define mb_debug(fmt, a...) printk(fmt, ##a)
693 +#else
694 +#define mb_debug(fmt, a...)
695 +#endif
696 +
697 +/*
698 + * with EXT4_MB_HISTORY mballoc stores last N allocations in memory
699 + * and you can monitor it in /proc/fs/ext4/<dev>/mb_history
700 + */
701 +#define EXT4_MB_HISTORY
702 +#define EXT4_MB_HISTORY_ALLOC 1 /* allocation */
703 +#define EXT4_MB_HISTORY_PREALLOC 2 /* preallocated blocks used */
704 +#define EXT4_MB_HISTORY_DISCARD 4 /* preallocation discarded */
705 +#define EXT4_MB_HISTORY_FREE 8 /* free */
706 +
707 +#define EXT4_MB_HISTORY_DEFAULT (EXT4_MB_HISTORY_ALLOC | \
708 + EXT4_MB_HISTORY_PREALLOC)
709 +
710 +/*
711 + * How long mballoc can look for a best extent (in found extents)
712 + */
713 +#define MB_DEFAULT_MAX_TO_SCAN 200
714 +
715 +/*
716 + * How long mballoc must look for a best extent
717 + */
718 +#define MB_DEFAULT_MIN_TO_SCAN 10
719 +
720 +/*
721 + * How many groups mballoc will scan looking for the best chunk
722 + */
723 +#define MB_DEFAULT_MAX_GROUPS_TO_SCAN 5
724 +
725 +/*
726 + * with 'ext4_mb_stats' allocator will collect stats that will be
727 + * shown at umount. The collecting costs though!
728 + */
729 +#define MB_DEFAULT_STATS 1
730 +
731 +/*
732 + * files smaller than MB_DEFAULT_STREAM_THRESHOLD are served
733 + * by the stream allocator, which purpose is to pack requests
734 + * as close each to other as possible to produce smooth I/O traffic
735 + * We use locality group prealloc space for stream request.
736 + * We can tune the same via /proc/fs/ext4/<parition>/stream_req
737 + */
738 +#define MB_DEFAULT_STREAM_THRESHOLD 16 /* 64K */
739 +
740 +/*
741 + * for which requests use 2^N search using buddies
742 + */
743 +#define MB_DEFAULT_ORDER2_REQS 2
744 +
745 +/*
746 + * default group prealloc size 512 blocks
747 + */
748 +#define MB_DEFAULT_GROUP_PREALLOC 512
749 +
750 +static struct kmem_cache *ext4_pspace_cachep;
751 +
752 +#ifdef EXT4_BB_MAX_BLOCKS
753 +#undef EXT4_BB_MAX_BLOCKS
754 +#endif
755 +#define EXT4_BB_MAX_BLOCKS 30
756 +
757 +struct ext4_free_metadata {
758 + ext4_group_t group;
759 + unsigned short num;
760 + ext4_grpblk_t blocks[EXT4_BB_MAX_BLOCKS];
761 + struct list_head list;
762 +};
763 +
764 +struct ext4_group_info {
765 + unsigned long bb_state;
766 + unsigned long bb_tid;
767 + struct ext4_free_metadata *bb_md_cur;
768 + unsigned short bb_first_free;
769 + unsigned short bb_free;
770 + unsigned short bb_fragments;
771 + struct list_head bb_prealloc_list;
772 +#ifdef DOUBLE_CHECK
773 + void *bb_bitmap;
774 +#endif
775 + unsigned short bb_counters[];
776 +};
777 +
778 +#define EXT4_GROUP_INFO_NEED_INIT_BIT 0
779 +#define EXT4_GROUP_INFO_LOCKED_BIT 1
780 +
781 +#define EXT4_MB_GRP_NEED_INIT(grp) \
782 + (test_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &((grp)->bb_state)))
783 +
784 +
785 +struct ext4_prealloc_space {
786 + struct list_head pa_inode_list;
787 + struct list_head pa_group_list;
788 + union {
789 + struct list_head pa_tmp_list;
790 + struct rcu_head pa_rcu;
791 + } u;
792 + spinlock_t pa_lock;
793 + atomic_t pa_count;
794 + unsigned pa_deleted;
795 + ext4_fsblk_t pa_pstart; /* phys. block */
796 + ext4_lblk_t pa_lstart; /* log. block */
797 + unsigned short pa_len; /* len of preallocated chunk */
798 + unsigned short pa_free; /* how many blocks are free */
799 + unsigned short pa_linear; /* consumed in one direction
800 + * strictly, for grp prealloc */
801 + spinlock_t *pa_obj_lock;
802 + struct inode *pa_inode; /* hack, for history only */
803 +};
804 +
805 +
806 +struct ext4_free_extent {
807 + ext4_lblk_t fe_logical;
808 + ext4_grpblk_t fe_start;
809 + ext4_group_t fe_group;
810 + int fe_len;
811 +};
812 +
813 +/*
814 + * Locality group:
815 + * we try to group all related changes together
816 + * so that writeback can flush/allocate them together as well
817 + */
818 +struct ext4_locality_group {
819 + /* for allocator */
820 + struct semaphore lg_sem; /* to serialize allocates */
821 + struct list_head lg_prealloc_list;/* list of preallocations */
822 + spinlock_t lg_prealloc_lock;
823 +};
824 +
825 +struct ext4_allocation_context {
826 + struct inode *ac_inode;
827 + struct super_block *ac_sb;
828 +
829 + /* original request */
830 + struct ext4_free_extent ac_o_ex;
831 +
832 + /* goal request (after normalization) */
833 + struct ext4_free_extent ac_g_ex;
834 +
835 + /* the best found extent */
836 + struct ext4_free_extent ac_b_ex;
837 +
838 + /* copy of the bext found extent taken before preallocation efforts */
839 + struct ext4_free_extent ac_f_ex;
840 +
841 + /* number of iterations done. we have to track to limit searching */
842 + unsigned long ac_ex_scanned;
843 + __u16 ac_groups_scanned;
844 + __u16 ac_found;
845 + __u16 ac_tail;
846 + __u16 ac_buddy;
847 + __u16 ac_flags; /* allocation hints */
848 + __u8 ac_status;
849 + __u8 ac_criteria;
850 + __u8 ac_repeats;
851 + __u8 ac_2order; /* if request is to allocate 2^N blocks and
852 + * N > 0, the field stores N, otherwise 0 */
853 + __u8 ac_op; /* operation, for history only */
854 + struct page *ac_bitmap_page;
855 + struct page *ac_buddy_page;
856 + struct ext4_prealloc_space *ac_pa;
857 + struct ext4_locality_group *ac_lg;
858 +};
859 +
860 +#define AC_STATUS_CONTINUE 1
861 +#define AC_STATUS_FOUND 2
862 +#define AC_STATUS_BREAK 3
863 +
864 +struct ext4_mb_history {
865 + struct ext4_free_extent orig; /* orig allocation */
866 + struct ext4_free_extent goal; /* goal allocation */
867 + struct ext4_free_extent result; /* result allocation */
868 + unsigned pid;
869 + unsigned ino;
870 + __u16 found; /* how many extents have been found */
871 + __u16 groups; /* how many groups have been scanned */
872 + __u16 tail; /* what tail broke some buddy */
873 + __u16 buddy; /* buddy the tail ^^^ broke */
874 + __u16 flags;
875 + __u8 cr:3; /* which phase the result extent was found at */
876 + __u8 op:4;
877 + __u8 merged:1;
878 +};
879 +
880 +struct ext4_buddy {
881 + struct page *bd_buddy_page;
882 + void *bd_buddy;
883 + struct page *bd_bitmap_page;
884 + void *bd_bitmap;
885 + struct ext4_group_info *bd_info;
886 + struct super_block *bd_sb;
887 + __u16 bd_blkbits;
888 + ext4_group_t bd_group;
889 +};
890 +#define EXT4_MB_BITMAP(e4b) ((e4b)->bd_bitmap)
891 +#define EXT4_MB_BUDDY(e4b) ((e4b)->bd_buddy)
892 +
893 +#ifndef EXT4_MB_HISTORY
894 +#define ext4_mb_store_history(ac)
895 +#else
896 +static void ext4_mb_store_history(struct ext4_allocation_context *ac);
897 +#endif
898 +
899 +#define in_range(b, first, len) ((b) >= (first) && (b) <= (first) + (len) - 1)
900 +
901 +static struct proc_dir_entry *proc_root_ext4;
902 +struct buffer_head *read_block_bitmap(struct super_block *, ext4_group_t);
903 +ext4_fsblk_t ext4_new_blocks_old(handle_t *handle, struct inode *inode,
904 + ext4_fsblk_t goal, unsigned long *count, int *errp);
905 +
906 +static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap,
907 + ext4_group_t group);
908 +static void ext4_mb_poll_new_transaction(struct super_block *, handle_t *);
909 +static void ext4_mb_free_committed_blocks(struct super_block *);
910 +static void ext4_mb_return_to_preallocation(struct inode *inode,
911 + struct ext4_buddy *e4b, sector_t block,
912 + int count);
913 +static void ext4_mb_put_pa(struct ext4_allocation_context *,
914 + struct super_block *, struct ext4_prealloc_space *pa);
915 +static int ext4_mb_init_per_dev_proc(struct super_block *sb);
916 +static int ext4_mb_destroy_per_dev_proc(struct super_block *sb);
917 +
918 +
919 +static inline void ext4_lock_group(struct super_block *sb, ext4_group_t group)
920 +{
921 + struct ext4_group_info *grinfo = ext4_get_group_info(sb, group);
922 +
923 + bit_spin_lock(EXT4_GROUP_INFO_LOCKED_BIT, &(grinfo->bb_state));
924 +}
925 +
926 +static inline void ext4_unlock_group(struct super_block *sb,
927 + ext4_group_t group)
928 +{
929 + struct ext4_group_info *grinfo = ext4_get_group_info(sb, group);
930 +
931 + bit_spin_unlock(EXT4_GROUP_INFO_LOCKED_BIT, &(grinfo->bb_state));
932 +}
933 +
934 +static inline int ext4_is_group_locked(struct super_block *sb,
935 + ext4_group_t group)
936 +{
937 + struct ext4_group_info *grinfo = ext4_get_group_info(sb, group);
938 +
939 + return bit_spin_is_locked(EXT4_GROUP_INFO_LOCKED_BIT,
940 + &(grinfo->bb_state));
941 +}
942 +
943 +static ext4_fsblk_t ext4_grp_offs_to_block(struct super_block *sb,
944 + struct ext4_free_extent *fex)
945 +{
946 + ext4_fsblk_t block;
947 +
948 + block = (ext4_fsblk_t) fex->fe_group * EXT4_BLOCKS_PER_GROUP(sb)
949 + + fex->fe_start
950 + + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);
951 + return block;
952 +}
953 +
954 +#if BITS_PER_LONG == 64
955 +#define mb_correct_addr_and_bit(bit, addr) \
956 +{ \
957 + bit += ((unsigned long) addr & 7UL) << 3; \
958 + addr = (void *) ((unsigned long) addr & ~7UL); \
959 +}
960 +#elif BITS_PER_LONG == 32
961 +#define mb_correct_addr_and_bit(bit, addr) \
962 +{ \
963 + bit += ((unsigned long) addr & 3UL) << 3; \
964 + addr = (void *) ((unsigned long) addr & ~3UL); \
965 +}
966 +#else
967 +#error "how many bits you are?!"
968 +#endif
969 +
970 +static inline int mb_test_bit(int bit, void *addr)
971 +{
972 + mb_correct_addr_and_bit(bit, addr);
973 + return ext4_test_bit(bit, addr);
974 +}
975 +
976 +static inline void mb_set_bit(int bit, void *addr)
977 +{
978 + mb_correct_addr_and_bit(bit, addr);
979 + ext4_set_bit(bit, addr);
980 +}
981 +
982 +static inline void mb_set_bit_atomic(spinlock_t *lock, int bit, void *addr)
983 +{
984 + mb_correct_addr_and_bit(bit, addr);
985 + ext4_set_bit_atomic(lock, bit, addr);
986 +}
987 +
988 +static inline void mb_clear_bit(int bit, void *addr)
989 +{
990 + mb_correct_addr_and_bit(bit, addr);
991 + ext4_clear_bit(bit, addr);
992 +}
993 +
994 +static inline void mb_clear_bit_atomic(spinlock_t *lock, int bit, void *addr)
995 +{
996 + mb_correct_addr_and_bit(bit, addr);
997 + ext4_clear_bit_atomic(lock, bit, addr);
998 +}
999 +
1000 +static inline void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max)
1001 +{
1002 + char *bb;
1003 +
1004 + /* FIXME!! is this needed */
1005 + BUG_ON(EXT4_MB_BITMAP(e4b) == EXT4_MB_BUDDY(e4b));
1006 + BUG_ON(max == NULL);
1007 +
1008 + if (order > e4b->bd_blkbits + 1) {
1009 + *max = 0;
1010 + return NULL;
1011 + }
1012 +
1013 + /* at order 0 we see each particular block */
1014 + *max = 1 << (e4b->bd_blkbits + 3);
1015 + if (order == 0)
1016 + return EXT4_MB_BITMAP(e4b);
1017 +
1018 + bb = EXT4_MB_BUDDY(e4b) + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order];
1019 + *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order];
1020 +
1021 + return bb;
1022 +}
1023 +
1024 +#ifdef DOUBLE_CHECK
1025 +static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b,
1026 + int first, int count)
1027 +{
1028 + int i;
1029 + struct super_block *sb = e4b->bd_sb;
1030 +
1031 + if (unlikely(e4b->bd_info->bb_bitmap == NULL))
1032 + return;
1033 + BUG_ON(!ext4_is_group_locked(sb, e4b->bd_group));
1034 + for (i = 0; i < count; i++) {
1035 + if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) {
1036 + ext4_fsblk_t blocknr;
1037 + blocknr = e4b->bd_group * EXT4_BLOCKS_PER_GROUP(sb);
1038 + blocknr += first + i;
1039 + blocknr +=
1040 + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);
1041 +
1042 + ext4_error(sb, __FUNCTION__, "double-free of inode"
1043 + " %lu's block %llu(bit %u in group %lu)\n",
1044 + inode ? inode->i_ino : 0, blocknr,
1045 + first + i, e4b->bd_group);
1046 + }
1047 + mb_clear_bit(first + i, e4b->bd_info->bb_bitmap);
1048 + }
1049 +}
1050 +
1051 +static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count)
1052 +{
1053 + int i;
1054 +
1055 + if (unlikely(e4b->bd_info->bb_bitmap == NULL))
1056 + return;
1057 + BUG_ON(!ext4_is_group_locked(e4b->bd_sb, e4b->bd_group));
1058 + for (i = 0; i < count; i++) {
1059 + BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap));
1060 + mb_set_bit(first + i, e4b->bd_info->bb_bitmap);
1061 + }
1062 +}
1063 +
1064 +static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap)
1065 +{
1066 + if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) {
1067 + unsigned char *b1, *b2;
1068 + int i;
1069 + b1 = (unsigned char *) e4b->bd_info->bb_bitmap;
1070 + b2 = (unsigned char *) bitmap;
1071 + for (i = 0; i < e4b->bd_sb->s_blocksize; i++) {
1072 + if (b1[i] != b2[i]) {
1073 + printk("corruption in group %lu at byte %u(%u):"
1074 + " %x in copy != %x on disk/prealloc\n",
1075 + e4b->bd_group, i, i * 8, b1[i], b2[i]);
1076 + BUG();
1077 + }
1078 + }
1079 + }
1080 +}
1081 +
1082 +#else
1083 +#define mb_free_blocks_double(a, b, c, d)
1084 +#define mb_mark_used_double(a, b, c)
1085 +#define mb_cmp_bitmaps(a, b)
1086 +#endif
1087 +
1088 +#ifdef AGGRESSIVE_CHECK
1089 +
1090 +#define MB_CHECK_ASSERT(assert) \
1091 +do { \
1092 + if (!(assert)) { \
1093 + printk(KERN_EMERG \
1094 + "Assertion failure in %s() at %s:%d: \"%s\"\n", \
1095 + function, file, line, # assert); \
1096 + BUG(); \
1097 + } \
1098 +} while (0)
1099 +
1100 +static int __mb_check_buddy(struct ext4_buddy *e4b, char *file,
1101 + const char *function, int line)
1102 +{
1103 + struct super_block *sb = e4b->bd_sb;
1104 + int order = e4b->bd_blkbits + 1;
1105 + int max;
1106 + int max2;
1107 + int i;
1108 + int j;
1109 + int k;
1110 + int count;
1111 + struct ext4_group_info *grp;
1112 + int fragments = 0;
1113 + int fstart;
1114 + struct list_head *cur;
1115 + void *buddy;
1116 + void *buddy2;
1117 +
1118 + if (!test_opt(sb, MBALLOC))
1119 + return 0;
1120 +
1121 + {
1122 + static int mb_check_counter;
1123 + if (mb_check_counter++ % 100 != 0)
1124 + return 0;
1125 + }
1126 +
1127 + while (order > 1) {
1128 + buddy = mb_find_buddy(e4b, order, &max);
1129 + MB_CHECK_ASSERT(buddy);
1130 + buddy2 = mb_find_buddy(e4b, order - 1, &max2);
1131 + MB_CHECK_ASSERT(buddy2);
1132 + MB_CHECK_ASSERT(buddy != buddy2);
1133 + MB_CHECK_ASSERT(max * 2 == max2);
1134 +
1135 + count = 0;
1136 + for (i = 0; i < max; i++) {
1137 +
1138 + if (mb_test_bit(i, buddy)) {
1139 + /* only single bit in buddy2 may be 1 */
1140 + if (!mb_test_bit(i << 1, buddy2)) {
1141 + MB_CHECK_ASSERT(
1142 + mb_test_bit((i<<1)+1, buddy2));
1143 + } else if (!mb_test_bit((i << 1) + 1, buddy2)) {
1144 + MB_CHECK_ASSERT(
1145 + mb_test_bit(i << 1, buddy2));
1146 + }
1147 + continue;
1148 + }
1149 +
1150 + /* both bits in buddy2 must be 0 */
1151 + MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2));
1152 + MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2));
1153 +
1154 + for (j = 0; j < (1 << order); j++) {
1155 + k = (i * (1 << order)) + j;
1156 + MB_CHECK_ASSERT(
1157 + !mb_test_bit(k, EXT4_MB_BITMAP(e4b)));
1158 + }
1159 + count++;
1160 + }
1161 + MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count);
1162 + order--;
1163 + }
1164 +
1165 + fstart = -1;
1166 + buddy = mb_find_buddy(e4b, 0, &max);
1167 + for (i = 0; i < max; i++) {
1168 + if (!mb_test_bit(i, buddy)) {
1169 + MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free);
1170 + if (fstart == -1) {
1171 + fragments++;
1172 + fstart = i;
1173 + }
1174 + continue;
1175 + }
1176 + fstart = -1;
1177 + /* check used bits only */
1178 + for (j = 0; j < e4b->bd_blkbits + 1; j++) {
1179 + buddy2 = mb_find_buddy(e4b, j, &max2);
1180 + k = i >> j;
1181 + MB_CHECK_ASSERT(k < max2);
1182 + MB_CHECK_ASSERT(mb_test_bit(k, buddy2));
1183 + }
1184 + }
1185 + MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info));
1186 + MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments);
1187 +
1188 + grp = ext4_get_group_info(sb, e4b->bd_group);
1189 + buddy = mb_find_buddy(e4b, 0, &max);
1190 + list_for_each(cur, &grp->bb_prealloc_list) {
1191 + ext4_group_t groupnr;
1192 + struct ext4_prealloc_space *pa;
1193 + pa = list_entry(cur, struct ext4_prealloc_space, group_list);
1194 + ext4_get_group_no_and_offset(sb, pa->pstart, &groupnr, &k);
1195 + MB_CHECK_ASSERT(groupnr == e4b->bd_group);
1196 + for (i = 0; i < pa->len; i++)
1197 + MB_CHECK_ASSERT(mb_test_bit(k + i, buddy));
1198 + }
1199 + return 0;
1200 +}
1201 +#undef MB_CHECK_ASSERT
1202 +#define mb_check_buddy(e4b) __mb_check_buddy(e4b, \
1203 + __FILE__, __FUNCTION__, __LINE__)
1204 +#else
1205 +#define mb_check_buddy(e4b)
1206 +#endif
1207 +
1208 +/* find most significant bit */
1209 +static int fmsb(unsigned short word)
1210 +{
1211 + int order;
1212 +
1213 + if (word > 255) {
1214 + order = 7;
1215 + word >>= 8;
1216 + } else {
1217 + order = -1;
1218 + }
1219 +
1220 + do {
1221 + order++;
1222 + word >>= 1;
1223 + } while (word != 0);
1224 +
1225 + return order;
1226 +}
1227 +
1228 +/* FIXME!! need more doc */
1229 +static void ext4_mb_mark_free_simple(struct super_block *sb,
1230 + void *buddy, unsigned first, int len,
1231 + struct ext4_group_info *grp)
1232 +{
1233 + struct ext4_sb_info *sbi = EXT4_SB(sb);
1234 + unsigned short min;
1235 + unsigned short max;
1236 + unsigned short chunk;
1237 + unsigned short border;
1238 +
1239 + BUG_ON(len >= EXT4_BLOCKS_PER_GROUP(sb));
1240 +
1241 + border = 2 << sb->s_blocksize_bits;
1242 +
1243 + while (len > 0) {
1244 + /* find how many blocks can be covered since this position */
1245 + max = ffs(first | border) - 1;
1246 +
1247 + /* find how many blocks of power 2 we need to mark */
1248 + min = fmsb(len);
1249 +
1250 + if (max < min)
1251 + min = max;
1252 + chunk = 1 << min;
1253 +
1254 + /* mark multiblock chunks only */
1255 + grp->bb_counters[min]++;
1256 + if (min > 0)
1257 + mb_clear_bit(first >> min,
1258 + buddy + sbi->s_mb_offsets[min]);
1259 +
1260 + len -= chunk;
1261 + first += chunk;
1262 + }
1263 +}
1264 +
1265 +static void ext4_mb_generate_buddy(struct super_block *sb,
1266 + void *buddy, void *bitmap, ext4_group_t group)
1267 +{
1268 + struct ext4_group_info *grp = ext4_get_group_info(sb, group);
1269 + unsigned short max = EXT4_BLOCKS_PER_GROUP(sb);
1270 + unsigned short i = 0;
1271 + unsigned short first;
1272 + unsigned short len;
1273 + unsigned free = 0;
1274 + unsigned fragments = 0;
1275 + unsigned long long period = get_cycles();
1276 +
1277 + /* initialize buddy from bitmap which is aggregation
1278 + * of on-disk bitmap and preallocations */
1279 + i = ext4_find_next_zero_bit(bitmap, max, 0);
1280 + grp->bb_first_free = i;
1281 + while (i < max) {
1282 + fragments++;
1283 + first = i;
1284 + i = ext4_find_next_bit(bitmap, max, i);
1285 + len = i - first;
1286 + free += len;
1287 + if (len > 1)
1288 + ext4_mb_mark_free_simple(sb, buddy, first, len, grp);
1289 + else
1290 + grp->bb_counters[0]++;
1291 + if (i < max)
1292 + i = ext4_find_next_zero_bit(bitmap, max, i);
1293 + }
1294 + grp->bb_fragments = fragments;
1295 +
1296 + if (free != grp->bb_free) {
1297 + printk(KERN_DEBUG
1298 + "EXT4-fs: group %lu: %u blocks in bitmap, %u in gd\n",
1299 + group, free, grp->bb_free);
1300 + grp->bb_free = free;
1301 + }
1302 +
1303 + clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state));
1304 +
1305 + period = get_cycles() - period;
1306 + spin_lock(&EXT4_SB(sb)->s_bal_lock);
1307 + EXT4_SB(sb)->s_mb_buddies_generated++;
1308 + EXT4_SB(sb)->s_mb_generation_time += period;
1309 + spin_unlock(&EXT4_SB(sb)->s_bal_lock);
1310 +}
1311 +
1312 +/* The buddy information is attached the buddy cache inode
1313 + * for convenience. The information regarding each group
1314 + * is loaded via ext4_mb_load_buddy. The information involve
1315 + * block bitmap and buddy information. The information are
1316 + * stored in the inode as
1317 + *
1318 + * { page }
1319 + * [ group 0 buddy][ group 0 bitmap] [group 1][ group 1]...
1320 + *
1321 + *
1322 + * one block each for bitmap and buddy information.
1323 + * So for each group we take up 2 blocks. A page can
1324 + * contain blocks_per_page (PAGE_CACHE_SIZE / blocksize) blocks.
1325 + * So it can have information regarding groups_per_page which
1326 + * is blocks_per_page/2
1327 + */
1328 +
1329 +static int ext4_mb_init_cache(struct page *page, char *incore)
1330 +{
1331 + int blocksize;
1332 + int blocks_per_page;
1333 + int groups_per_page;
1334 + int err = 0;
1335 + int i;
1336 + ext4_group_t first_group;
1337 + int first_block;
1338 + struct super_block *sb;
1339 + struct buffer_head *bhs;
1340 + struct buffer_head **bh;
1341 + struct inode *inode;
1342 + char *data;
1343 + char *bitmap;
1344 +
1345 + mb_debug("init page %lu\n", page->index);
1346 +
1347 + inode = page->mapping->host;
1348 + sb = inode->i_sb;
1349 + blocksize = 1 << inode->i_blkbits;
1350 + blocks_per_page = PAGE_CACHE_SIZE / blocksize;
1351 +
1352 + groups_per_page = blocks_per_page >> 1;
1353 + if (groups_per_page == 0)
1354 + groups_per_page = 1;
1355 +
1356 + /* allocate buffer_heads to read bitmaps */
1357 + if (groups_per_page > 1) {
1358 + err = -ENOMEM;
1359 + i = sizeof(struct buffer_head *) * groups_per_page;
1360 + bh = kmalloc(i, GFP_NOFS);
1361 + if (bh == NULL)
1362 + goto out;
1363 + memset(bh, 0, i);
1364 + } else
1365 + bh = &bhs;
1366 +
1367 + first_group = page->index * blocks_per_page / 2;
1368 +
1369 + /* read all groups the page covers into the cache */
1370 + for (i = 0; i < groups_per_page; i++) {
1371 + struct ext4_group_desc *desc;
1372 +
1373 + if (first_group + i >= EXT4_SB(sb)->s_groups_count)
1374 + break;
1375 +
1376 + err = -EIO;
1377 + desc = ext4_get_group_desc(sb, first_group + i, NULL);
1378 + if (desc == NULL)
1379 + goto out;
1380 +
1381 + err = -ENOMEM;
1382 + bh[i] = sb_getblk(sb, ext4_block_bitmap(sb, desc));
1383 + if (bh[i] == NULL)
1384 + goto out;
1385 +
1386 + if (buffer_uptodate(bh[i]))
1387 + continue;
1388 +
1389 + lock_buffer(bh[i]);
1390 + if (buffer_uptodate(bh[i])) {
1391 + unlock_buffer(bh[i]);
1392 + continue;
1393 + }
1394 +
1395 + if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) {
1396 + ext4_init_block_bitmap(sb, bh[i],
1397 + first_group + i, desc);
1398 + set_buffer_uptodate(bh[i]);
1399 + unlock_buffer(bh[i]);
1400 + continue;
1401 + }
1402 + get_bh(bh[i]);
1403 + bh[i]->b_end_io = end_buffer_read_sync;
1404 + submit_bh(READ, bh[i]);
1405 + mb_debug("read bitmap for group %lu\n", first_group + i);
1406 + }
1407 +
1408 + /* wait for I/O completion */
1409 + for (i = 0; i < groups_per_page && bh[i]; i++)
1410 + wait_on_buffer(bh[i]);
1411 +
1412 + err = -EIO;
1413 + for (i = 0; i < groups_per_page && bh[i]; i++)
1414 + if (!buffer_uptodate(bh[i]))
1415 + goto out;
1416 +
1417 + first_block = page->index * blocks_per_page;
1418 + for (i = 0; i < blocks_per_page; i++) {
1419 + int group;
1420 + struct ext4_group_info *grinfo;
1421 +
1422 + group = (first_block + i) >> 1;
1423 + if (group >= EXT4_SB(sb)->s_groups_count)
1424 + break;
1425 +
1426 + /*
1427 + * data carry information regarding this
1428 + * particular group in the format specified
1429 + * above
1430 + *
1431 + */
1432 + data = page_address(page) + (i * blocksize);
1433 + bitmap = bh[group - first_group]->b_data;
1434 +
1435 + /*
1436 + * We place the buddy block and bitmap block
1437 + * close together
1438 + */
1439 + if ((first_block + i) & 1) {
1440 + /* this is block of buddy */
1441 + BUG_ON(incore == NULL);
1442 + mb_debug("put buddy for group %u in page %lu/%x\n",
1443 + group, page->index, i * blocksize);
1444 + memset(data, 0xff, blocksize);
1445 + grinfo = ext4_get_group_info(sb, group);
1446 + grinfo->bb_fragments = 0;
1447 + memset(grinfo->bb_counters, 0,
1448 + sizeof(unsigned short)*(sb->s_blocksize_bits+2));
1449 + /*
1450 + * incore got set to the group block bitmap below
1451 + */
1452 + ext4_mb_generate_buddy(sb, data, incore, group);
1453 + incore = NULL;
1454 + } else {
1455 + /* this is block of bitmap */
1456 + BUG_ON(incore != NULL);
1457 + mb_debug("put bitmap for group %u in page %lu/%x\n",
1458 + group, page->index, i * blocksize);
1459 +
1460 + /* see comments in ext4_mb_put_pa() */
1461 + ext4_lock_group(sb, group);
1462 + memcpy(data, bitmap, blocksize);
1463 +
1464 + /* mark all preallocated blks used in in-core bitmap */
1465 + ext4_mb_generate_from_pa(sb, data, group);
1466 + ext4_unlock_group(sb, group);
1467 +
1468 + /* set incore so that the buddy information can be
1469 + * generated using this
1470 + */
1471 + incore = data;
1472 + }
1473 + }
1474 + SetPageUptodate(page);
1475 +
1476 +out:
1477 + if (bh) {
1478 + for (i = 0; i < groups_per_page && bh[i]; i++)
1479 + brelse(bh[i]);
1480 + if (bh != &bhs)
1481 + kfree(bh);
1482 + }
1483 + return err;
1484 +}
1485 +
1486 +static int ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group,
1487 + struct ext4_buddy *e4b)
1488 +{
1489 + struct ext4_sb_info *sbi = EXT4_SB(sb);
1490 + struct inode *inode = sbi->s_buddy_cache;
1491 + int blocks_per_page;
1492 + int block;
1493 + int pnum;
1494 + int poff;
1495 + struct page *page;
1496 +
1497 + mb_debug("load group %lu\n", group);
1498 +
1499 + blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
1500 +
1501 + e4b->bd_blkbits = sb->s_blocksize_bits;
1502 + e4b->bd_info = ext4_get_group_info(sb, group);
1503 + e4b->bd_sb = sb;
1504 + e4b->bd_group = group;
1505 + e4b->bd_buddy_page = NULL;
1506 + e4b->bd_bitmap_page = NULL;
1507 +
1508 + /*
1509 + * the buddy cache inode stores the block bitmap
1510 + * and buddy information in consecutive blocks.
1511 + * So for each group we need two blocks.
1512 + */
1513 + block = group * 2;
1514 + pnum = block / blocks_per_page;
1515 + poff = block % blocks_per_page;
1516 +
1517 + /* we could use find_or_create_page(), but it locks page
1518 + * what we'd like to avoid in fast path ... */
1519 + page = find_get_page(inode->i_mapping, pnum);
1520 + if (page == NULL || !PageUptodate(page)) {
1521 + if (page)
1522 + page_cache_release(page);
1523 + page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
1524 + if (page) {
1525 + BUG_ON(page->mapping != inode->i_mapping);
1526 + if (!PageUptodate(page)) {
1527 + ext4_mb_init_cache(page, NULL);
1528 + mb_cmp_bitmaps(e4b, page_address(page) +
1529 + (poff * sb->s_blocksize));
1530 + }
1531 + unlock_page(page);
1532 + }
1533 + }
1534 + if (page == NULL || !PageUptodate(page))
1535 + goto err;
1536 + e4b->bd_bitmap_page = page;
1537 + e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);
1538 + mark_page_accessed(page);
1539 +
1540 + block++;
1541 + pnum = block / blocks_per_page;
1542 + poff = block % blocks_per_page;
1543 +
1544 + page = find_get_page(inode->i_mapping, pnum);
1545 + if (page == NULL || !PageUptodate(page)) {
1546 + if (page)
1547 + page_cache_release(page);
1548 + page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
1549 + if (page) {
1550 + BUG_ON(page->mapping != inode->i_mapping);
1551 + if (!PageUptodate(page))
1552 + ext4_mb_init_cache(page, e4b->bd_bitmap);
1553 +
1554 + unlock_page(page);
1555 + }
1556 + }
1557 + if (page == NULL || !PageUptodate(page))
1558 + goto err;
1559 + e4b->bd_buddy_page = page;
1560 + e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize);
1561 + mark_page_accessed(page);
1562 +
1563 + BUG_ON(e4b->bd_bitmap_page == NULL);
1564 + BUG_ON(e4b->bd_buddy_page == NULL);
1565 +
1566 + return 0;
1567 +
1568 +err:
1569 + if (e4b->bd_bitmap_page)
1570 + page_cache_release(e4b->bd_bitmap_page);
1571 + if (e4b->bd_buddy_page)
1572 + page_cache_release(e4b->bd_buddy_page);
1573 + e4b->bd_buddy = NULL;
1574 + e4b->bd_bitmap = NULL;
1575 + return -EIO;
1576 +}
1577 +
1578 +static void ext4_mb_release_desc(struct ext4_buddy *e4b)
1579 +{
1580 + if (e4b->bd_bitmap_page)
1581 + page_cache_release(e4b->bd_bitmap_page);
1582 + if (e4b->bd_buddy_page)
1583 + page_cache_release(e4b->bd_buddy_page);
1584 +}
1585 +
1586 +
1587 +static int mb_find_order_for_block(struct ext4_buddy *e4b, int block)
1588 +{
1589 + int order = 1;
1590 + void *bb;
1591 +
1592 + BUG_ON(EXT4_MB_BITMAP(e4b) == EXT4_MB_BUDDY(e4b));
1593 + BUG_ON(block >= (1 << (e4b->bd_blkbits + 3)));
1594 +
1595 + bb = EXT4_MB_BUDDY(e4b);
1596 + while (order <= e4b->bd_blkbits + 1) {
1597 + block = block >> 1;
1598 + if (!mb_test_bit(block, bb)) {
1599 + /* this block is part of buddy of order 'order' */
1600 + return order;
1601 + }
1602 + bb += 1 << (e4b->bd_blkbits - order);
1603 + order++;
1604 + }
1605 + return 0;
1606 +}
1607 +
1608 +static void mb_clear_bits(spinlock_t *lock, void *bm, int cur, int len)
1609 +{
1610 + __u32 *addr;
1611 +
1612 + len = cur + len;
1613 + while (cur < len) {
1614 + if ((cur & 31) == 0 && (len - cur) >= 32) {
1615 + /* fast path: clear whole word at once */
1616 + addr = bm + (cur >> 3);
1617 + *addr = 0;
1618 + cur += 32;
1619 + continue;
1620 + }
1621 + mb_clear_bit_atomic(lock, cur, bm);
1622 + cur++;
1623 + }
1624 +}
1625 +
1626 +static void mb_set_bits(spinlock_t *lock, void *bm, int cur, int len)
1627 +{
1628 + __u32 *addr;
1629 +
1630 + len = cur + len;
1631 + while (cur < len) {
1632 + if ((cur & 31) == 0 && (len - cur) >= 32) {
1633 + /* fast path: clear whole word at once */
1634 + addr = bm + (cur >> 3);
1635 + *addr = 0xffffffff;
1636 + cur += 32;
1637 + continue;
1638 + }
1639 + mb_set_bit_atomic(lock, cur, bm);
1640 + cur++;
1641 + }
1642 +}
1643 +
1644 +static int mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b,
1645 + int first, int count)
1646 +{
1647 + int block = 0;
1648 + int max = 0;
1649 + int order;
1650 + void *buddy;
1651 + void *buddy2;
1652 + struct super_block *sb = e4b->bd_sb;
1653 +
1654 + BUG_ON(first + count > (sb->s_blocksize << 3));
1655 + BUG_ON(!ext4_is_group_locked(sb, e4b->bd_group));
1656 + mb_check_buddy(e4b);
1657 + mb_free_blocks_double(inode, e4b, first, count);
1658 +
1659 + e4b->bd_info->bb_free += count;
1660 + if (first < e4b->bd_info->bb_first_free)
1661 + e4b->bd_info->bb_first_free = first;
1662 +
1663 + /* let's maintain fragments counter */
1664 + if (first != 0)
1665 + block = !mb_test_bit(first - 1, EXT4_MB_BITMAP(e4b));
1666 + if (first + count < EXT4_SB(sb)->s_mb_maxs[0])
1667 + max = !mb_test_bit(first + count, EXT4_MB_BITMAP(e4b));
1668 + if (block && max)
1669 + e4b->bd_info->bb_fragments--;
1670 + else if (!block && !max)
1671 + e4b->bd_info->bb_fragments++;
1672 +
1673 + /* let's maintain buddy itself */
1674 + while (count-- > 0) {
1675 + block = first++;
1676 + order = 0;
1677 +
1678 + if (!mb_test_bit(block, EXT4_MB_BITMAP(e4b))) {
1679 + ext4_fsblk_t blocknr;
1680 + blocknr = e4b->bd_group * EXT4_BLOCKS_PER_GROUP(sb);
1681 + blocknr += block;
1682 + blocknr +=
1683 + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);
1684 +
1685 + ext4_error(sb, __FUNCTION__, "double-free of inode"
1686 + " %lu's block %llu(bit %u in group %lu)\n",
1687 + inode ? inode->i_ino : 0, blocknr, block,
1688 + e4b->bd_group);
1689 + }
1690 + mb_clear_bit(block, EXT4_MB_BITMAP(e4b));
1691 + e4b->bd_info->bb_counters[order]++;
1692 +
1693 + /* start of the buddy */
1694 + buddy = mb_find_buddy(e4b, order, &max);
1695 +
1696 + do {
1697 + block &= ~1UL;
1698 + if (mb_test_bit(block, buddy) ||
1699 + mb_test_bit(block + 1, buddy))
1700 + break;
1701 +
1702 + /* both the buddies are free, try to coalesce them */
1703 + buddy2 = mb_find_buddy(e4b, order + 1, &max);
1704 +
1705 + if (!buddy2)
1706 + break;
1707 +
1708 + if (order > 0) {
1709 + /* for special purposes, we don't set
1710 + * free bits in bitmap */
1711 + mb_set_bit(block, buddy);
1712 + mb_set_bit(block + 1, buddy);
1713 + }
1714 + e4b->bd_info->bb_counters[order]--;
1715 + e4b->bd_info->bb_counters[order]--;
1716 +
1717 + block = block >> 1;
1718 + order++;
1719 + e4b->bd_info->bb_counters[order]++;
1720 +
1721 + mb_clear_bit(block, buddy2);
1722 + buddy = buddy2;
1723 + } while (1);
1724 + }
1725 + mb_check_buddy(e4b);
1726 +
1727 + return 0;
1728 +}
1729 +
1730 +static int mb_find_extent(struct ext4_buddy *e4b, int order, int block,
1731 + int needed, struct ext4_free_extent *ex)
1732 +{
1733 + int next = block;
1734 + int max;
1735 + int ord;
1736 + void *buddy;
1737 +
1738 + BUG_ON(!ext4_is_group_locked(e4b->bd_sb, e4b->bd_group));
1739 + BUG_ON(ex == NULL);
1740 +
1741 + buddy = mb_find_buddy(e4b, order, &max);
1742 + BUG_ON(buddy == NULL);
1743 + BUG_ON(block >= max);
1744 + if (mb_test_bit(block, buddy)) {
1745 + ex->fe_len = 0;
1746 + ex->fe_start = 0;
1747 + ex->fe_group = 0;
1748 + return 0;
1749 + }
1750 +
1751 + /* FIXME dorp order completely ? */
1752 + if (likely(order == 0)) {
1753 + /* find actual order */
1754 + order = mb_find_order_for_block(e4b, block);
1755 + block = block >> order;
1756 + }
1757 +
1758 + ex->fe_len = 1 << order;
1759 + ex->fe_start = block << order;
1760 + ex->fe_group = e4b->bd_group;
1761 +
1762 + /* calc difference from given start */
1763 + next = next - ex->fe_start;
1764 + ex->fe_len -= next;
1765 + ex->fe_start += next;
1766 +
1767 + while (needed > ex->fe_len &&
1768 + (buddy = mb_find_buddy(e4b, order, &max))) {
1769 +
1770 + if (block + 1 >= max)
1771 + break;
1772 +
1773 + next = (block + 1) * (1 << order);
1774 + if (mb_test_bit(next, EXT4_MB_BITMAP(e4b)))
1775 + break;
1776 +
1777 + ord = mb_find_order_for_block(e4b, next);
1778 +
1779 + order = ord;
1780 + block = next >> order;
1781 + ex->fe_len += 1 << order;
1782 + }
1783 +
1784 + BUG_ON(ex->fe_start + ex->fe_len > (1 << (e4b->bd_blkbits + 3)));
1785 + return ex->fe_len;
1786 +}
1787 +
1788 +static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex)
1789 +{
1790 + int ord;
1791 + int mlen = 0;
1792 + int max = 0;
1793 + int cur;
1794 + int start = ex->fe_start;
1795 + int len = ex->fe_len;
1796 + unsigned ret = 0;
1797 + int len0 = len;
1798 + void *buddy;
1799 +
1800 + BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3));
1801 + BUG_ON(e4b->bd_group != ex->fe_group);
1802 + BUG_ON(!ext4_is_group_locked(e4b->bd_sb, e4b->bd_group));
1803 + mb_check_buddy(e4b);
1804 + mb_mark_used_double(e4b, start, len);
1805 +
1806 + e4b->bd_info->bb_free -= len;
1807 + if (e4b->bd_info->bb_first_free == start)
1808 + e4b->bd_info->bb_first_free += len;
1809 +
1810 + /* let's maintain fragments counter */
1811 + if (start != 0)
1812 + mlen = !mb_test_bit(start - 1, EXT4_MB_BITMAP(e4b));
1813 + if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0])
1814 + max = !mb_test_bit(start + len, EXT4_MB_BITMAP(e4b));
1815 + if (mlen && max)
1816 + e4b->bd_info->bb_fragments++;
1817 + else if (!mlen && !max)
1818 + e4b->bd_info->bb_fragments--;
1819 +
1820 + /* let's maintain buddy itself */
1821 + while (len) {
1822 + ord = mb_find_order_for_block(e4b, start);
1823 +
1824 + if (((start >> ord) << ord) == start && len >= (1 << ord)) {
1825 + /* the whole chunk may be allocated at once! */
1826 + mlen = 1 << ord;
1827 + buddy = mb_find_buddy(e4b, ord, &max);
1828 + BUG_ON((start >> ord) >= max);
1829 + mb_set_bit(start >> ord, buddy);
1830 + e4b->bd_info->bb_counters[ord]--;
1831 + start += mlen;
1832 + len -= mlen;
1833 + BUG_ON(len < 0);
1834 + continue;
1835 + }
1836 +
1837 + /* store for history */
1838 + if (ret == 0)
1839 + ret = len | (ord << 16);
1840 +
1841 + /* we have to split large buddy */
1842 + BUG_ON(ord <= 0);
1843 + buddy = mb_find_buddy(e4b, ord, &max);
1844 + mb_set_bit(start >> ord, buddy);
1845 + e4b->bd_info->bb_counters[ord]--;
1846 +
1847 + ord--;
1848 + cur = (start >> ord) & ~1U;
1849 + buddy = mb_find_buddy(e4b, ord, &max);
1850 + mb_clear_bit(cur, buddy);
1851 + mb_clear_bit(cur + 1, buddy);
1852 + e4b->bd_info->bb_counters[ord]++;
1853 + e4b->bd_info->bb_counters[ord]++;
1854 + }
1855 +
1856 + mb_set_bits(sb_bgl_lock(EXT4_SB(e4b->bd_sb), ex->fe_group),
1857 + EXT4_MB_BITMAP(e4b), ex->fe_start, len0);
1858 + mb_check_buddy(e4b);
1859 +
1860 + return ret;
1861 +}
1862 +
1863 +/*
1864 + * Must be called under group lock!
1865 + */
1866 +static void ext4_mb_use_best_found(struct ext4_allocation_context *ac,
1867 + struct ext4_buddy *e4b)
1868 +{
1869 + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
1870 + int ret;
1871 +
1872 + BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);
1873 + BUG_ON(ac->ac_status == AC_STATUS_FOUND);
1874 +
1875 + ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len);
1876 + ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical;
1877 + ret = mb_mark_used(e4b, &ac->ac_b_ex);
1878 +
1879 + /* preallocation can change ac_b_ex, thus we store actually
1880 + * allocated blocks for history */
1881 + ac->ac_f_ex = ac->ac_b_ex;
1882 +
1883 + ac->ac_status = AC_STATUS_FOUND;
1884 + ac->ac_tail = ret & 0xffff;
1885 + ac->ac_buddy = ret >> 16;
1886 +
1887 + /* XXXXXXX: SUCH A HORRIBLE **CK */
1888 + /*FIXME!! Why ? */
1889 + ac->ac_bitmap_page = e4b->bd_bitmap_page;
1890 + get_page(ac->ac_bitmap_page);
1891 + ac->ac_buddy_page = e4b->bd_buddy_page;
1892 + get_page(ac->ac_buddy_page);
1893 +
1894 + /* store last allocated for subsequent stream allocation */
1895 + if ((ac->ac_flags & EXT4_MB_HINT_DATA)) {
1896 + spin_lock(&sbi->s_md_lock);
1897 + sbi->s_mb_last_group = ac->ac_f_ex.fe_group;
1898 + sbi->s_mb_last_start = ac->ac_f_ex.fe_start;
1899 + spin_unlock(&sbi->s_md_lock);
1900 + }
1901 +}
1902 +
1903 +/*
1904 + * regular allocator, for general purposes allocation
1905 + */
1906 +
1907 +static void ext4_mb_check_limits(struct ext4_allocation_context *ac,
1908 + struct ext4_buddy *e4b,
1909 + int finish_group)
1910 +{
1911 + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
1912 + struct ext4_free_extent *bex = &ac->ac_b_ex;
1913 + struct ext4_free_extent *gex = &ac->ac_g_ex;
1914 + struct ext4_free_extent ex;
1915 + int max;
1916 +
1917 + /*
1918 + * We don't want to scan for a whole year
1919 + */
1920 + if (ac->ac_found > sbi->s_mb_max_to_scan &&
1921 + !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
1922 + ac->ac_status = AC_STATUS_BREAK;
1923 + return;
1924 + }
1925 +
1926 + /*
1927 + * Haven't found good chunk so far, let's continue
1928 + */
1929 + if (bex->fe_len < gex->fe_len)
1930 + return;
1931 +
1932 + if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan)
1933 + && bex->fe_group == e4b->bd_group) {
1934 + /* recheck chunk's availability - we don't know
1935 + * when it was found (within this lock-unlock
1936 + * period or not) */
1937 + max = mb_find_extent(e4b, 0, bex->fe_start, gex->fe_len, &ex);
1938 + if (max >= gex->fe_len) {
1939 + ext4_mb_use_best_found(ac, e4b);
1940 + return;
1941 + }
1942 + }
1943 +}
1944 +
1945 +/*
1946 + * The routine checks whether found extent is good enough. If it is,
1947 + * then the extent gets marked used and flag is set to the context
1948 + * to stop scanning. Otherwise, the extent is compared with the
1949 + * previous found extent and if new one is better, then it's stored
1950 + * in the context. Later, the best found extent will be used, if
1951 + * mballoc can't find good enough extent.
1952 + *
1953 + * FIXME: real allocation policy is to be designed yet!
1954 + */
1955 +static void ext4_mb_measure_extent(struct ext4_allocation_context *ac,
1956 + struct ext4_free_extent *ex,
1957 + struct ext4_buddy *e4b)
1958 +{
1959 + struct ext4_free_extent *bex = &ac->ac_b_ex;
1960 + struct ext4_free_extent *gex = &ac->ac_g_ex;
1961 +
1962 + BUG_ON(ex->fe_len <= 0);
1963 + BUG_ON(ex->fe_len >= EXT4_BLOCKS_PER_GROUP(ac->ac_sb));
1964 + BUG_ON(ex->fe_start >= EXT4_BLOCKS_PER_GROUP(ac->ac_sb));
1965 + BUG_ON(ac->ac_status != AC_STATUS_CONTINUE);
1966 +
1967 + ac->ac_found++;
1968 +
1969 + /*
1970 + * The special case - take what you catch first
1971 + */
1972 + if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
1973 + *bex = *ex;
1974 + ext4_mb_use_best_found(ac, e4b);
1975 + return;
1976 + }
1977 +
1978 + /*
1979 + * Let's check whether the chuck is good enough
1980 + */
1981 + if (ex->fe_len == gex->fe_len) {
1982 + *bex = *ex;
1983 + ext4_mb_use_best_found(ac, e4b);
1984 + return;
1985 + }
1986 +
1987 + /*
1988 + * If this is first found extent, just store it in the context
1989 + */
1990 + if (bex->fe_len == 0) {
1991 + *bex = *ex;
1992 + return;
1993 + }
1994 +
1995 + /*
1996 + * If new found extent is better, store it in the context
1997 + */
1998 + if (bex->fe_len < gex->fe_len) {
1999 + /* if the request isn't satisfied, any found extent
2000 + * larger than previous best one is better */
2001 + if (ex->fe_len > bex->fe_len)
2002 + *bex = *ex;
2003 + } else if (ex->fe_len > gex->fe_len) {
2004 + /* if the request is satisfied, then we try to find
2005 + * an extent that still satisfy the request, but is
2006 + * smaller than previous one */
2007 + if (ex->fe_len < bex->fe_len)
2008 + *bex = *ex;
2009 + }
2010 +
2011 + ext4_mb_check_limits(ac, e4b, 0);
2012 +}
2013 +
2014 +static int ext4_mb_try_best_found(struct ext4_allocation_context *ac,
2015 + struct ext4_buddy *e4b)
2016 +{
2017 + struct ext4_free_extent ex = ac->ac_b_ex;
2018 + ext4_group_t group = ex.fe_group;
2019 + int max;
2020 + int err;
2021 +
2022 + BUG_ON(ex.fe_len <= 0);
2023 + err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
2024 + if (err)
2025 + return err;
2026 +
2027 + ext4_lock_group(ac->ac_sb, group);
2028 + max = mb_find_extent(e4b, 0, ex.fe_start, ex.fe_len, &ex);
2029 +
2030 + if (max > 0) {
2031 + ac->ac_b_ex = ex;
2032 + ext4_mb_use_best_found(ac, e4b);
2033 + }
2034 +
2035 + ext4_unlock_group(ac->ac_sb, group);
2036 + ext4_mb_release_desc(e4b);
2037 +
2038 + return 0;
2039 +}
2040 +
2041 +static int ext4_mb_find_by_goal(struct ext4_allocation_context *ac,
2042 + struct ext4_buddy *e4b)
2043 +{
2044 + ext4_group_t group = ac->ac_g_ex.fe_group;
2045 + int max;
2046 + int err;
2047 + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
2048 + struct ext4_super_block *es = sbi->s_es;
2049 + struct ext4_free_extent ex;
2050 +
2051 + if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL))
2052 + return 0;
2053 +
2054 + err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
2055 + if (err)
2056 + return err;
2057 +
2058 + ext4_lock_group(ac->ac_sb, group);
2059 + max = mb_find_extent(e4b, 0, ac->ac_g_ex.fe_start,
2060 + ac->ac_g_ex.fe_len, &ex);
2061 +
2062 + if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) {
2063 + ext4_fsblk_t start;
2064 +
2065 + start = (e4b->bd_group * EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) +
2066 + ex.fe_start + le32_to_cpu(es->s_first_data_block);
2067 + /* use do_div to get remainder (would be 64-bit modulo) */
2068 + if (do_div(start, sbi->s_stripe) == 0) {
2069 + ac->ac_found++;
2070 + ac->ac_b_ex = ex;
2071 + ext4_mb_use_best_found(ac, e4b);
2072 + }
2073 + } else if (max >= ac->ac_g_ex.fe_len) {
2074 + BUG_ON(ex.fe_len <= 0);
2075 + BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);
2076 + BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);
2077 + ac->ac_found++;
2078 + ac->ac_b_ex = ex;
2079 + ext4_mb_use_best_found(ac, e4b);
2080 + } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) {
2081 + /* Sometimes, caller may want to merge even small
2082 + * number of blocks to an existing extent */
2083 + BUG_ON(ex.fe_len <= 0);
2084 + BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);
2085 + BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);
2086 + ac->ac_found++;
2087 + ac->ac_b_ex = ex;
2088 + ext4_mb_use_best_found(ac, e4b);
2089 + }
2090 + ext4_unlock_group(ac->ac_sb, group);
2091 + ext4_mb_release_desc(e4b);
2092 +
2093 + return 0;
2094 +}
2095 +
2096 +/*
2097 + * The routine scans buddy structures (not bitmap!) from given order
2098 + * to max order and tries to find big enough chunk to satisfy the req
2099 + */
2100 +static void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac,
2101 + struct ext4_buddy *e4b)
2102 +{
2103 + struct super_block *sb = ac->ac_sb;
2104 + struct ext4_group_info *grp = e4b->bd_info;
2105 + void *buddy;
2106 + int i;
2107 + int k;
2108 + int max;
2109 +
2110 + BUG_ON(ac->ac_2order <= 0);
2111 + for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) {
2112 + if (grp->bb_counters[i] == 0)
2113 + continue;
2114 +
2115 + buddy = mb_find_buddy(e4b, i, &max);
2116 + BUG_ON(buddy == NULL);
2117 +
2118 + k = ext4_find_next_zero_bit(buddy, max, 0);
2119 + BUG_ON(k >= max);
2120 +
2121 + ac->ac_found++;
2122 +
2123 + ac->ac_b_ex.fe_len = 1 << i;
2124 + ac->ac_b_ex.fe_start = k << i;
2125 + ac->ac_b_ex.fe_group = e4b->bd_group;
2126 +
2127 + ext4_mb_use_best_found(ac, e4b);
2128 +
2129 + BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len);
2130 +
2131 + if (EXT4_SB(sb)->s_mb_stats)
2132 + atomic_inc(&EXT4_SB(sb)->s_bal_2orders);
2133 +
2134 + break;
2135 + }
2136 +}
2137 +
2138 +/*
2139 + * The routine scans the group and measures all found extents.
2140 + * In order to optimize scanning, caller must pass number of
2141 + * free blocks in the group, so the routine can know upper limit.
2142 + */
2143 +static void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac,
2144 + struct ext4_buddy *e4b)
2145 +{
2146 + struct super_block *sb = ac->ac_sb;
2147 + void *bitmap = EXT4_MB_BITMAP(e4b);
2148 + struct ext4_free_extent ex;
2149 + int i;
2150 + int free;
2151 +
2152 + free = e4b->bd_info->bb_free;
2153 + BUG_ON(free <= 0);
2154 +
2155 + i = e4b->bd_info->bb_first_free;
2156 +
2157 + while (free && ac->ac_status == AC_STATUS_CONTINUE) {
2158 + i = ext4_find_next_zero_bit(bitmap,
2159 + EXT4_BLOCKS_PER_GROUP(sb), i);
2160 + if (i >= EXT4_BLOCKS_PER_GROUP(sb)) {
2161 + BUG_ON(free != 0);
2162 + break;
2163 + }
2164 +
2165 + mb_find_extent(e4b, 0, i, ac->ac_g_ex.fe_len, &ex);
2166 + BUG_ON(ex.fe_len <= 0);
2167 + BUG_ON(free < ex.fe_len);
2168 +
2169 + ext4_mb_measure_extent(ac, &ex, e4b);
2170 +
2171 + i += ex.fe_len;
2172 + free -= ex.fe_len;
2173 + }
2174 +
2175 + ext4_mb_check_limits(ac, e4b, 1);
2176 +}
2177 +
2178 +/*
2179 + * This is a special case for storages like raid5
2180 + * we try to find stripe-aligned chunks for stripe-size requests
2181 + * XXX should do so at least for multiples of stripe size as well
2182 + */
2183 +static void ext4_mb_scan_aligned(struct ext4_allocation_context *ac,
2184 + struct ext4_buddy *e4b)
2185 +{
2186 + struct super_block *sb = ac->ac_sb;
2187 + struct ext4_sb_info *sbi = EXT4_SB(sb);
2188 + void *bitmap = EXT4_MB_BITMAP(e4b);
2189 + struct ext4_free_extent ex;
2190 + ext4_fsblk_t first_group_block;
2191 + ext4_fsblk_t a;
2192 + ext4_grpblk_t i;
2193 + int max;
2194 +
2195 + BUG_ON(sbi->s_stripe == 0);
2196 +
2197 + /* find first stripe-aligned block in group */
2198 + first_group_block = e4b->bd_group * EXT4_BLOCKS_PER_GROUP(sb)
2199 + + le32_to_cpu(sbi->s_es->s_first_data_block);
2200 + a = first_group_block + sbi->s_stripe - 1;
2201 + do_div(a, sbi->s_stripe);
2202 + i = (a * sbi->s_stripe) - first_group_block;
2203 +
2204 + while (i < EXT4_BLOCKS_PER_GROUP(sb)) {
2205 + if (!mb_test_bit(i, bitmap)) {
2206 + max = mb_find_extent(e4b, 0, i, sbi->s_stripe, &ex);
2207 + if (max >= sbi->s_stripe) {
2208 + ac->ac_found++;
2209 + ac->ac_b_ex = ex;
2210 + ext4_mb_use_best_found(ac, e4b);
2211 + break;
2212 + }
2213 + }
2214 + i += sbi->s_stripe;
2215 + }
2216 +}
2217 +
2218 +static int ext4_mb_good_group(struct ext4_allocation_context *ac,
2219 + ext4_group_t group, int cr)
2220 +{
2221 + unsigned free, fragments;
2222 + unsigned i, bits;
2223 + struct ext4_group_desc *desc;
2224 + struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);
2225 +
2226 + BUG_ON(cr < 0 || cr >= 4);
2227 + BUG_ON(EXT4_MB_GRP_NEED_INIT(grp));
2228 +
2229 + free = grp->bb_free;
2230 + fragments = grp->bb_fragments;
2231 + if (free == 0)
2232 + return 0;
2233 + if (fragments == 0)
2234 + return 0;
2235 +
2236 + switch (cr) {
2237 + case 0:
2238 + BUG_ON(ac->ac_2order == 0);
2239 + /* If this group is uninitialized, skip it initially */
2240 + desc = ext4_get_group_desc(ac->ac_sb, group, NULL);
2241 + if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))
2242 + return 0;
2243 +
2244 + bits = ac->ac_sb->s_blocksize_bits + 1;
2245 + for (i = ac->ac_2order; i <= bits; i++)
2246 + if (grp->bb_counters[i] > 0)
2247 + return 1;
2248 + break;
2249 + case 1:
2250 + if ((free / fragments) >= ac->ac_g_ex.fe_len)
2251 + return 1;
2252 + break;
2253 + case 2:
2254 + if (free >= ac->ac_g_ex.fe_len)
2255 + return 1;
2256 + break;
2257 + case 3:
2258 + return 1;
2259 + default:
2260 + BUG();
2261 + }
2262 +
2263 + return 0;
2264 +}
2265 +
2266 +static int ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
2267 +{
2268 + ext4_group_t group;
2269 + ext4_group_t i;
2270 + int cr;
2271 + int err = 0;
2272 + int bsbits;
2273 + struct ext4_sb_info *sbi;
2274 + struct super_block *sb;
2275 + struct ext4_buddy e4b;
2276 + loff_t size, isize;
2277 +
2278 + sb = ac->ac_sb;
2279 + sbi = EXT4_SB(sb);
2280 + BUG_ON(ac->ac_status == AC_STATUS_FOUND);
2281 +
2282 + /* first, try the goal */
2283 + err = ext4_mb_find_by_goal(ac, &e4b);
2284 + if (err || ac->ac_status == AC_STATUS_FOUND)
2285 + goto out;
2286 +
2287 + if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
2288 + goto out;
2289 +
2290 + /*
2291 + * ac->ac2_order is set only if the fe_len is a power of 2
2292 + * if ac2_order is set we also set criteria to 0 so that we
2293 + * try exact allocation using buddy.
2294 + */
2295 + i = fls(ac->ac_g_ex.fe_len);
2296 + ac->ac_2order = 0;
2297 + /*
2298 + * We search using buddy data only if the order of the request
2299 + * is greater than equal to the sbi_s_mb_order2_reqs
2300 + * You can tune it via /proc/fs/ext4/<partition>/order2_req
2301 + */
2302 + if (i >= sbi->s_mb_order2_reqs) {
2303 + /*
2304 + * This should tell if fe_len is exactly power of 2
2305 + */
2306 + if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0)
2307 + ac->ac_2order = i - 1;
2308 + }
2309 +
2310 + bsbits = ac->ac_sb->s_blocksize_bits;
2311 + /* if stream allocation is enabled, use global goal */
2312 + size = ac->ac_o_ex.fe_logical + ac->ac_o_ex.fe_len;
2313 + isize = i_size_read(ac->ac_inode) >> bsbits;
2314 + if (size < isize)
2315 + size = isize;
2316 +
2317 + if (size < sbi->s_mb_stream_request &&
2318 + (ac->ac_flags & EXT4_MB_HINT_DATA)) {
2319 + /* TBD: may be hot point */
2320 + spin_lock(&sbi->s_md_lock);
2321 + ac->ac_g_ex.fe_group = sbi->s_mb_last_group;
2322 + ac->ac_g_ex.fe_start = sbi->s_mb_last_start;
2323 + spin_unlock(&sbi->s_md_lock);
2324 + }
2325 +
2326 + /* searching for the right group start from the goal value specified */
2327 + group = ac->ac_g_ex.fe_group;
2328 +
2329 + /* Let's just scan groups to find more-less suitable blocks */
2330 + cr = ac->ac_2order ? 0 : 1;
2331 + /*
2332 + * cr == 0 try to get exact allocation,
2333 + * cr == 3 try to get anything
2334 + */
2335 +repeat:
2336 + for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) {
2337 + ac->ac_criteria = cr;
2338 + for (i = 0; i < EXT4_SB(sb)->s_groups_count; group++, i++) {
2339 + struct ext4_group_info *grp;
2340 + struct ext4_group_desc *desc;
2341 +
2342 + if (group == EXT4_SB(sb)->s_groups_count)
2343 + group = 0;
2344 +
2345 + /* quick check to skip empty groups */
2346 + grp = ext4_get_group_info(ac->ac_sb, group);
2347 + if (grp->bb_free == 0)
2348 + continue;
2349 +
2350 + /*
2351 + * if the group is already init we check whether it is
2352 + * a good group and if not we don't load the buddy
2353 + */
2354 + if (EXT4_MB_GRP_NEED_INIT(grp)) {
2355 + /*
2356 + * we need full data about the group
2357 + * to make a good selection
2358 + */
2359 + err = ext4_mb_load_buddy(sb, group, &e4b);
2360 + if (err)
2361 + goto out;
2362 + ext4_mb_release_desc(&e4b);
2363 + }
2364 +
2365 + /*
2366 + * If the particular group doesn't satisfy our
2367 + * criteria we continue with the next group
2368 + */
2369 + if (!ext4_mb_good_group(ac, group, cr))
2370 + continue;
2371 +
2372 + err = ext4_mb_load_buddy(sb, group, &e4b);
2373 + if (err)
2374 + goto out;
2375 +
2376 + ext4_lock_group(sb, group);
2377 + if (!ext4_mb_good_group(ac, group, cr)) {
2378 + /* someone did allocation from this group */
2379 + ext4_unlock_group(sb, group);
2380 + ext4_mb_release_desc(&e4b);
2381 + continue;
2382 + }
2383 +
2384 + ac->ac_groups_scanned++;
2385 + desc = ext4_get_group_desc(sb, group, NULL);
2386 + if (cr == 0 || (desc->bg_flags &
2387 + cpu_to_le16(EXT4_BG_BLOCK_UNINIT) &&
2388 + ac->ac_2order != 0))
2389 + ext4_mb_simple_scan_group(ac, &e4b);
2390 + else if (cr == 1 &&
2391 + ac->ac_g_ex.fe_len == sbi->s_stripe)
2392 + ext4_mb_scan_aligned(ac, &e4b);
2393 + else
2394 + ext4_mb_complex_scan_group(ac, &e4b);
2395 +
2396 + ext4_unlock_group(sb, group);
2397 + ext4_mb_release_desc(&e4b);
2398 +
2399 + if (ac->ac_status != AC_STATUS_CONTINUE)
2400 + break;
2401 + }
2402 + }
2403 +
2404 + if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND &&
2405 + !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
2406 + /*
2407 + * We've been searching too long. Let's try to allocate
2408 + * the best chunk we've found so far
2409 + */
2410 +
2411 + ext4_mb_try_best_found(ac, &e4b);
2412 + if (ac->ac_status != AC_STATUS_FOUND) {
2413 + /*
2414 + * Someone more lucky has already allocated it.
2415 + * The only thing we can do is just take first
2416 + * found block(s)
2417 + printk(KERN_DEBUG "EXT4-fs: someone won our chunk\n");
2418 + */
2419 + ac->ac_b_ex.fe_group = 0;
2420 + ac->ac_b_ex.fe_start = 0;
2421 + ac->ac_b_ex.fe_len = 0;
2422 + ac->ac_status = AC_STATUS_CONTINUE;
2423 + ac->ac_flags |= EXT4_MB_HINT_FIRST;
2424 + cr = 3;
2425 + atomic_inc(&sbi->s_mb_lost_chunks);
2426 + goto repeat;
2427 + }
2428 + }
2429 +out:
2430 + return err;
2431 +}
2432 +
2433 +#ifdef EXT4_MB_HISTORY
2434 +struct ext4_mb_proc_session {
2435 + struct ext4_mb_history *history;
2436 + struct super_block *sb;
2437 + int start;
2438 + int max;
2439 +};
2440 +
2441 +static void *ext4_mb_history_skip_empty(struct ext4_mb_proc_session *s,
2442 + struct ext4_mb_history *hs,
2443 + int first)
2444 +{
2445 + if (hs == s->history + s->max)
2446 + hs = s->history;
2447 + if (!first && hs == s->history + s->start)
2448 + return NULL;
2449 + while (hs->orig.fe_len == 0) {
2450 + hs++;
2451 + if (hs == s->history + s->max)
2452 + hs = s->history;
2453 + if (hs == s->history + s->start)
2454 + return NULL;
2455 + }
2456 + return hs;
2457 +}
2458 +
2459 +static void *ext4_mb_seq_history_start(struct seq_file *seq, loff_t *pos)
2460 +{
2461 + struct ext4_mb_proc_session *s = seq->private;
2462 + struct ext4_mb_history *hs;
2463 + int l = *pos;
2464 +
2465 + if (l == 0)
2466 + return SEQ_START_TOKEN;
2467 + hs = ext4_mb_history_skip_empty(s, s->history + s->start, 1);
2468 + if (!hs)
2469 + return NULL;
2470 + while (--l && (hs = ext4_mb_history_skip_empty(s, ++hs, 0)) != NULL);
2471 + return hs;
2472 +}
2473 +
2474 +static void *ext4_mb_seq_history_next(struct seq_file *seq, void *v,
2475 + loff_t *pos)
2476 +{
2477 + struct ext4_mb_proc_session *s = seq->private;
2478 + struct ext4_mb_history *hs = v;
2479 +
2480 + ++*pos;
2481 + if (v == SEQ_START_TOKEN)
2482 + return ext4_mb_history_skip_empty(s, s->history + s->start, 1);
2483 + else
2484 + return ext4_mb_history_skip_empty(s, ++hs, 0);
2485 +}
2486 +
2487 +static int ext4_mb_seq_history_show(struct seq_file *seq, void *v)
2488 +{
2489 + char buf[25], buf2[25], buf3[25], *fmt;
2490 + struct ext4_mb_history *hs = v;
2491 +
2492 + if (v == SEQ_START_TOKEN) {
2493 + seq_printf(seq, "%-5s %-8s %-23s %-23s %-23s %-5s "
2494 + "%-5s %-2s %-5s %-5s %-5s %-6s\n",
2495 + "pid", "inode", "original", "goal", "result", "found",
2496 + "grps", "cr", "flags", "merge", "tail", "broken");
2497 + return 0;
2498 + }
2499 +
2500 + if (hs->op == EXT4_MB_HISTORY_ALLOC) {
2501 + fmt = "%-5u %-8u %-23s %-23s %-23s %-5u %-5u %-2u "
2502 + "%-5u %-5s %-5u %-6u\n";
2503 + sprintf(buf2, "%lu/%d/%u@%u", hs->result.fe_group,
2504 + hs->result.fe_start, hs->result.fe_len,
2505 + hs->result.fe_logical);
2506 + sprintf(buf, "%lu/%d/%u@%u", hs->orig.fe_group,
2507 + hs->orig.fe_start, hs->orig.fe_len,
2508 + hs->orig.fe_logical);
2509 + sprintf(buf3, "%lu/%d/%u@%u", hs->goal.fe_group,
2510 + hs->goal.fe_start, hs->goal.fe_len,
2511 + hs->goal.fe_logical);
2512 + seq_printf(seq, fmt, hs->pid, hs->ino, buf, buf3, buf2,
2513 + hs->found, hs->groups, hs->cr, hs->flags,
2514 + hs->merged ? "M" : "", hs->tail,
2515 + hs->buddy ? 1 << hs->buddy : 0);
2516 + } else if (hs->op == EXT4_MB_HISTORY_PREALLOC) {
2517 + fmt = "%-5u %-8u %-23s %-23s %-23s\n";
2518 + sprintf(buf2, "%lu/%d/%u@%u", hs->result.fe_group,
2519 + hs->result.fe_start, hs->result.fe_len,
2520 + hs->result.fe_logical);
2521 + sprintf(buf, "%lu/%d/%u@%u", hs->orig.fe_group,
2522 + hs->orig.fe_start, hs->orig.fe_len,
2523 + hs->orig.fe_logical);
2524 + seq_printf(seq, fmt, hs->pid, hs->ino, buf, "", buf2);
2525 + } else if (hs->op == EXT4_MB_HISTORY_DISCARD) {
2526 + sprintf(buf2, "%lu/%d/%u", hs->result.fe_group,
2527 + hs->result.fe_start, hs->result.fe_len);
2528 + seq_printf(seq, "%-5u %-8u %-23s discard\n",
2529 + hs->pid, hs->ino, buf2);
2530 + } else if (hs->op == EXT4_MB_HISTORY_FREE) {
2531 + sprintf(buf2, "%lu/%d/%u", hs->result.fe_group,
2532 + hs->result.fe_start, hs->result.fe_len);
2533 + seq_printf(seq, "%-5u %-8u %-23s free\n",
2534 + hs->pid, hs->ino, buf2);
2535 + }
2536 + return 0;
2537 +}
2538 +
2539 +static void ext4_mb_seq_history_stop(struct seq_file *seq, void *v)
2540 +{
2541 +}
2542 +
2543 +static struct seq_operations ext4_mb_seq_history_ops = {
2544 + .start = ext4_mb_seq_history_start,
2545 + .next = ext4_mb_seq_history_next,
2546 + .stop = ext4_mb_seq_history_stop,
2547 + .show = ext4_mb_seq_history_show,
2548 +};
2549 +
2550 +static int ext4_mb_seq_history_open(struct inode *inode, struct file *file)
2551 +{
2552 + struct super_block *sb = PDE(inode)->data;
2553 + struct ext4_sb_info *sbi = EXT4_SB(sb);
2554 + struct ext4_mb_proc_session *s;
2555 + int rc;
2556 + int size;
2557 +
2558 + s = kmalloc(sizeof(*s), GFP_KERNEL);
2559 + if (s == NULL)
2560 + return -ENOMEM;
2561 + s->sb = sb;
2562 + size = sizeof(struct ext4_mb_history) * sbi->s_mb_history_max;
2563 + s->history = kmalloc(size, GFP_KERNEL);
2564 + if (s->history == NULL) {
2565 + kfree(s);
2566 + return -ENOMEM;
2567 + }
2568 +
2569 + spin_lock(&sbi->s_mb_history_lock);
2570 + memcpy(s->history, sbi->s_mb_history, size);
2571 + s->max = sbi->s_mb_history_max;
2572 + s->start = sbi->s_mb_history_cur % s->max;
2573 + spin_unlock(&sbi->s_mb_history_lock);
2574 +
2575 + rc = seq_open(file, &ext4_mb_seq_history_ops);
2576 + if (rc == 0) {
2577 + struct seq_file *m = (struct seq_file *)file->private_data;
2578 + m->private = s;
2579 + } else {
2580 + kfree(s->history);
2581 + kfree(s);
2582 + }
2583 + return rc;
2584 +
2585 +}
2586 +
2587 +static int ext4_mb_seq_history_release(struct inode *inode, struct file *file)
2588 +{
2589 + struct seq_file *seq = (struct seq_file *)file->private_data;
2590 + struct ext4_mb_proc_session *s = seq->private;
2591 + kfree(s->history);
2592 + kfree(s);
2593 + return seq_release(inode, file);
2594 +}
2595 +
2596 +static ssize_t ext4_mb_seq_history_write(struct file *file,
2597 + const char __user *buffer,
2598 + size_t count, loff_t *ppos)
2599 +{
2600 + struct seq_file *seq = (struct seq_file *)file->private_data;
2601 + struct ext4_mb_proc_session *s = seq->private;
2602 + struct super_block *sb = s->sb;
2603 + char str[32];
2604 + int value;
2605 +
2606 + if (count >= sizeof(str)) {
2607 + printk(KERN_ERR "EXT4-fs: %s string too long, max %u bytes\n",
2608 + "mb_history", (int)sizeof(str));
2609 + return -EOVERFLOW;
2610 + }
2611 +
2612 + if (copy_from_user(str, buffer, count))
2613 + return -EFAULT;
2614 +
2615 + value = simple_strtol(str, NULL, 0);
2616 + if (value < 0)
2617 + return -ERANGE;
2618 + EXT4_SB(sb)->s_mb_history_filter = value;
2619 +
2620 + return count;
2621 +}
2622 +
2623 +static struct file_operations ext4_mb_seq_history_fops = {
2624 + .owner = THIS_MODULE,
2625 + .open = ext4_mb_seq_history_open,
2626 + .read = seq_read,
2627 + .write = ext4_mb_seq_history_write,
2628 + .llseek = seq_lseek,
2629 + .release = ext4_mb_seq_history_release,
2630 +};
2631 +
2632 +static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos)
2633 +{
2634 + struct super_block *sb = seq->private;
2635 + struct ext4_sb_info *sbi = EXT4_SB(sb);
2636 + ext4_group_t group;
2637 +
2638 + if (*pos < 0 || *pos >= sbi->s_groups_count)
2639 + return NULL;
2640 +
2641 + group = *pos + 1;
2642 + return (void *) group;
2643 +}
2644 +
2645 +static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos)
2646 +{
2647 + struct super_block *sb = seq->private;
2648 + struct ext4_sb_info *sbi = EXT4_SB(sb);
2649 + ext4_group_t group;
2650 +
2651 + ++*pos;
2652 + if (*pos < 0 || *pos >= sbi->s_groups_count)
2653 + return NULL;
2654 + group = *pos + 1;
2655 + return (void *) group;;
2656 +}
2657 +
2658 +static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v)
2659 +{
2660 + struct super_block *sb = seq->private;
2661 + long group = (long) v;
2662 + int i;
2663 + int err;
2664 + struct ext4_buddy e4b;
2665 + struct sg {
2666 + struct ext4_group_info info;
2667 + unsigned short counters[16];
2668 + } sg;
2669 +
2670 + group--;
2671 + if (group == 0)
2672 + seq_printf(seq, "#%-5s: %-5s %-5s %-5s "
2673 + "[ %-5s %-5s %-5s %-5s %-5s %-5s %-5s "
2674 + "%-5s %-5s %-5s %-5s %-5s %-5s %-5s ]\n",
2675 + "group", "free", "frags", "first",
2676 + "2^0", "2^1", "2^2", "2^3", "2^4", "2^5", "2^6",
2677 + "2^7", "2^8", "2^9", "2^10", "2^11", "2^12", "2^13");
2678 +
2679 + i = (sb->s_blocksize_bits + 2) * sizeof(sg.info.bb_counters[0]) +
2680 + sizeof(struct ext4_group_info);
2681 + err = ext4_mb_load_buddy(sb, group, &e4b);
2682 + if (err) {
2683 + seq_printf(seq, "#%-5lu: I/O error\n", group);
2684 + return 0;
2685 + }
2686 + ext4_lock_group(sb, group);
2687 + memcpy(&sg, ext4_get_group_info(sb, group), i);
2688 + ext4_unlock_group(sb, group);
2689 + ext4_mb_release_desc(&e4b);
2690 +
2691 + seq_printf(seq, "#%-5lu: %-5u %-5u %-5u [", group, sg.info.bb_free,
2692 + sg.info.bb_fragments, sg.info.bb_first_free);
2693 + for (i = 0; i <= 13; i++)
2694 + seq_printf(seq, " %-5u", i <= sb->s_blocksize_bits + 1 ?
2695 + sg.info.bb_counters[i] : 0);
2696 + seq_printf(seq, " ]\n");
2697 +
2698 + return 0;
2699 +}
2700 +
2701 +static void ext4_mb_seq_groups_stop(struct seq_file *seq, void *v)
2702 +{
2703 +}
2704 +
2705 +static struct seq_operations ext4_mb_seq_groups_ops = {
2706 + .start = ext4_mb_seq_groups_start,
2707 + .next = ext4_mb_seq_groups_next,
2708 + .stop = ext4_mb_seq_groups_stop,
2709 + .show = ext4_mb_seq_groups_show,
2710 +};
2711 +
2712 +static int ext4_mb_seq_groups_open(struct inode *inode, struct file *file)
2713 +{
2714 + struct super_block *sb = PDE(inode)->data;
2715 + int rc;
2716 +
2717 + rc = seq_open(file, &ext4_mb_seq_groups_ops);
2718 + if (rc == 0) {
2719 + struct seq_file *m = (struct seq_file *)file->private_data;
2720 + m->private = sb;
2721 + }
2722 + return rc;
2723 +
2724 +}
2725 +
2726 +static struct file_operations ext4_mb_seq_groups_fops = {
2727 + .owner = THIS_MODULE,
2728 + .open = ext4_mb_seq_groups_open,
2729 + .read = seq_read,
2730 + .llseek = seq_lseek,
2731 + .release = seq_release,
2732 +};
2733 +
2734 +static void ext4_mb_history_release(struct super_block *sb)
2735 +{
2736 + struct ext4_sb_info *sbi = EXT4_SB(sb);
2737 +
2738 + remove_proc_entry("mb_groups", sbi->s_mb_proc);
2739 + remove_proc_entry("mb_history", sbi->s_mb_proc);
2740 +
2741 + kfree(sbi->s_mb_history);
2742 +}
2743 +
2744 +static void ext4_mb_history_init(struct super_block *sb)
2745 +{
2746 + struct ext4_sb_info *sbi = EXT4_SB(sb);
2747 + int i;
2748 +
2749 + if (sbi->s_mb_proc != NULL) {
2750 + struct proc_dir_entry *p;
2751 + p = create_proc_entry("mb_history", S_IRUGO, sbi->s_mb_proc);
2752 + if (p) {
2753 + p->proc_fops = &ext4_mb_seq_history_fops;
2754 + p->data = sb;
2755 + }
2756 + p = create_proc_entry("mb_groups", S_IRUGO, sbi->s_mb_proc);
2757 + if (p) {
2758 + p->proc_fops = &ext4_mb_seq_groups_fops;
2759 + p->data = sb;
2760 + }
2761 + }
2762 +
2763 + sbi->s_mb_history_max = 1000;
2764 + sbi->s_mb_history_cur = 0;
2765 + spin_lock_init(&sbi->s_mb_history_lock);
2766 + i = sbi->s_mb_history_max * sizeof(struct ext4_mb_history);
2767 + sbi->s_mb_history = kmalloc(i, GFP_KERNEL);
2768 + if (likely(sbi->s_mb_history != NULL))
2769 + memset(sbi->s_mb_history, 0, i);
2770 + /* if we can't allocate history, then we simple won't use it */
2771 +}
2772 +
2773 +static void ext4_mb_store_history(struct ext4_allocation_context *ac)
2774 +{
2775 + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
2776 + struct ext4_mb_history h;
2777 +
2778 + if (unlikely(sbi->s_mb_history == NULL))
2779 + return;
2780 +
2781 + if (!(ac->ac_op & sbi->s_mb_history_filter))
2782 + return;
2783 +
2784 + h.op = ac->ac_op;
2785 + h.pid = current->pid;
2786 + h.ino = ac->ac_inode ? ac->ac_inode->i_ino : 0;
2787 + h.orig = ac->ac_o_ex;
2788 + h.result = ac->ac_b_ex;
2789 + h.flags = ac->ac_flags;
2790 + h.found = ac->ac_found;
2791 + h.groups = ac->ac_groups_scanned;
2792 + h.cr = ac->ac_criteria;
2793 + h.tail = ac->ac_tail;
2794 + h.buddy = ac->ac_buddy;
2795 + h.merged = 0;
2796 + if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) {
2797 + if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&
2798 + ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)
2799 + h.merged = 1;
2800 + h.goal = ac->ac_g_ex;
2801 + h.result = ac->ac_f_ex;
2802 + }
2803 +
2804 + spin_lock(&sbi->s_mb_history_lock);
2805 + memcpy(sbi->s_mb_history + sbi->s_mb_history_cur, &h, sizeof(h));
2806 + if (++sbi->s_mb_history_cur >= sbi->s_mb_history_max)
2807 + sbi->s_mb_history_cur = 0;
2808 + spin_unlock(&sbi->s_mb_history_lock);
2809 +}
2810 +
2811 +#else
2812 +#define ext4_mb_history_release(sb)
2813 +#define ext4_mb_history_init(sb)
2814 +#endif
2815 +
2816 +static int ext4_mb_init_backend(struct super_block *sb)
2817 +{
2818 + ext4_group_t i;
2819 + int j, len, metalen;
2820 + struct ext4_sb_info *sbi = EXT4_SB(sb);
2821 + int num_meta_group_infos =
2822 + (sbi->s_groups_count + EXT4_DESC_PER_BLOCK(sb) - 1) >>
2823 + EXT4_DESC_PER_BLOCK_BITS(sb);
2824 + struct ext4_group_info **meta_group_info;
2825 +
2826 + /* An 8TB filesystem with 64-bit pointers requires a 4096 byte
2827 + * kmalloc. A 128kb malloc should suffice for a 256TB filesystem.
2828 + * So a two level scheme suffices for now. */
2829 + sbi->s_group_info = kmalloc(sizeof(*sbi->s_group_info) *
2830 + num_meta_group_infos, GFP_KERNEL);
2831 + if (sbi->s_group_info == NULL) {
2832 + printk(KERN_ERR "EXT4-fs: can't allocate buddy meta group\n");
2833 + return -ENOMEM;
2834 + }
2835 + sbi->s_buddy_cache = new_inode(sb);
2836 + if (sbi->s_buddy_cache == NULL) {
2837 + printk(KERN_ERR "EXT4-fs: can't get new inode\n");
2838 + goto err_freesgi;
2839 + }
2840 + EXT4_I(sbi->s_buddy_cache)->i_disksize = 0;
2841 +
2842 + metalen = sizeof(*meta_group_info) << EXT4_DESC_PER_BLOCK_BITS(sb);
2843 + for (i = 0; i < num_meta_group_infos; i++) {
2844 + if ((i + 1) == num_meta_group_infos)
2845 + metalen = sizeof(*meta_group_info) *
2846 + (sbi->s_groups_count -
2847 + (i << EXT4_DESC_PER_BLOCK_BITS(sb)));
2848 + meta_group_info = kmalloc(metalen, GFP_KERNEL);
2849 + if (meta_group_info == NULL) {
2850 + printk(KERN_ERR "EXT4-fs: can't allocate mem for a "
2851 + "buddy group\n");
2852 + goto err_freemeta;
2853 + }
2854 + sbi->s_group_info[i] = meta_group_info;
2855 + }
2856 +
2857 + /*
2858 + * calculate needed size. if change bb_counters size,
2859 + * don't forget about ext4_mb_generate_buddy()
2860 + */
2861 + len = sizeof(struct ext4_group_info);
2862 + len += sizeof(unsigned short) * (sb->s_blocksize_bits + 2);
2863 + for (i = 0; i < sbi->s_groups_count; i++) {
2864 + struct ext4_group_desc *desc;
2865 +
2866 + meta_group_info =
2867 + sbi->s_group_info[i >> EXT4_DESC_PER_BLOCK_BITS(sb)];
2868 + j = i & (EXT4_DESC_PER_BLOCK(sb) - 1);
2869 +
2870 + meta_group_info[j] = kzalloc(len, GFP_KERNEL);
2871 + if (meta_group_info[j] == NULL) {
2872 + printk(KERN_ERR "EXT4-fs: can't allocate buddy mem\n");
2873 + i--;
2874 + goto err_freebuddy;
2875 + }
2876 + desc = ext4_get_group_desc(sb, i, NULL);
2877 + if (desc == NULL) {
2878 + printk(KERN_ERR
2879 + "EXT4-fs: can't read descriptor %lu\n", i);
2880 + goto err_freebuddy;
2881 + }
2882 + memset(meta_group_info[j], 0, len);
2883 + set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT,
2884 + &(meta_group_info[j]->bb_state));
2885 +
2886 + /*
2887 + * initialize bb_free to be able to skip
2888 + * empty groups without initialization
2889 + */
2890 + if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) {
2891 + meta_group_info[j]->bb_free =
2892 + ext4_free_blocks_after_init(sb, i, desc);
2893 + } else {
2894 + meta_group_info[j]->bb_free =
2895 + le16_to_cpu(desc->bg_free_blocks_count);
2896 + }
2897 +
2898 + INIT_LIST_HEAD(&meta_group_info[j]->bb_prealloc_list);
2899 +
2900 +#ifdef DOUBLE_CHECK
2901 + {
2902 + struct buffer_head *bh;
2903 + meta_group_info[j]->bb_bitmap =
2904 + kmalloc(sb->s_blocksize, GFP_KERNEL);
2905 + BUG_ON(meta_group_info[j]->bb_bitmap == NULL);
2906 + bh = read_block_bitmap(sb, i);
2907 + BUG_ON(bh == NULL);
2908 + memcpy(meta_group_info[j]->bb_bitmap, bh->b_data,
2909 + sb->s_blocksize);
2910 + brelse(bh);
2911 + }
2912 +#endif
2913 +
2914 + }
2915 +
2916 + return 0;
2917 +
2918 +err_freebuddy:
2919 + while (i >= 0) {
2920 + kfree(ext4_get_group_info(sb, i));
2921 + i--;
2922 + }
2923 + i = num_meta_group_infos;
2924 +err_freemeta:
2925 + while (--i >= 0)
2926 + kfree(sbi->s_group_info[i]);
2927 + iput(sbi->s_buddy_cache);
2928 +err_freesgi:
2929 + kfree(sbi->s_group_info);
2930 + return -ENOMEM;
2931 +}
2932 +
2933 +int ext4_mb_init(struct super_block *sb, int needs_recovery)
2934 +{
2935 + struct ext4_sb_info *sbi = EXT4_SB(sb);
2936 + unsigned i;
2937 + unsigned offset;
2938 + unsigned max;
2939 +
2940 + if (!test_opt(sb, MBALLOC))
2941 + return 0;
2942 +
2943 + i = (sb->s_blocksize_bits + 2) * sizeof(unsigned short);
2944 +
2945 + sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL);
2946 + if (sbi->s_mb_offsets == NULL) {
2947 + clear_opt(sbi->s_mount_opt, MBALLOC);
2948 + return -ENOMEM;
2949 + }
2950 + sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL);
2951 + if (sbi->s_mb_maxs == NULL) {
2952 + clear_opt(sbi->s_mount_opt, MBALLOC);
2953 + kfree(sbi->s_mb_maxs);
2954 + return -ENOMEM;
2955 + }
2956 +
2957 + /* order 0 is regular bitmap */
2958 + sbi->s_mb_maxs[0] = sb->s_blocksize << 3;
2959 + sbi->s_mb_offsets[0] = 0;
2960 +
2961 + i = 1;
2962 + offset = 0;
2963 + max = sb->s_blocksize << 2;
2964 + do {
2965 + sbi->s_mb_offsets[i] = offset;
2966 + sbi->s_mb_maxs[i] = max;
2967 + offset += 1 << (sb->s_blocksize_bits - i);
2968 + max = max >> 1;
2969 + i++;
2970 + } while (i <= sb->s_blocksize_bits + 1);
2971 +
2972 + /* init file for buddy data */
2973 + i = ext4_mb_init_backend(sb);
2974 + if (i) {
2975 + clear_opt(sbi->s_mount_opt, MBALLOC);
2976 + kfree(sbi->s_mb_offsets);
2977 + kfree(sbi->s_mb_maxs);
2978 + return i;
2979 + }
2980 +
2981 + spin_lock_init(&sbi->s_md_lock);
2982 + INIT_LIST_HEAD(&sbi->s_active_transaction);
2983 + INIT_LIST_HEAD(&sbi->s_closed_transaction);
2984 + INIT_LIST_HEAD(&sbi->s_committed_transaction);
2985 + spin_lock_init(&sbi->s_bal_lock);
2986 +
2987 + sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN;
2988 + sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN;
2989 + sbi->s_mb_stats = MB_DEFAULT_STATS;
2990 + sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD;
2991 + sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS;
2992 + sbi->s_mb_history_filter = EXT4_MB_HISTORY_DEFAULT;
2993 + sbi->s_mb_group_prealloc = MB_DEFAULT_GROUP_PREALLOC;
2994 +
2995 + i = sizeof(struct ext4_locality_group) * NR_CPUS;
2996 + sbi->s_locality_groups = kmalloc(i, GFP_NOFS);
2997 + if (sbi->s_locality_groups == NULL) {
2998 + clear_opt(sbi->s_mount_opt, MBALLOC);
2999 + kfree(sbi->s_mb_offsets);
3000 + kfree(sbi->s_mb_maxs);
3001 + return -ENOMEM;
3002 + }
3003 + for (i = 0; i < NR_CPUS; i++) {
3004 + struct ext4_locality_group *lg;
3005 + lg = &sbi->s_locality_groups[i];
3006 + sema_init(&lg->lg_sem, 1);
3007 + INIT_LIST_HEAD(&lg->lg_prealloc_list);
3008 + spin_lock_init(&lg->lg_prealloc_lock);
3009 + }
3010 +
3011 + ext4_mb_init_per_dev_proc(sb);
3012 + ext4_mb_history_init(sb);
3013 +
3014 + printk("EXT4-fs: mballoc enabled\n");
3015 + return 0;
3016 +}
3017 +
3018 +static void ext4_mb_cleanup_pa(struct ext4_group_info *grp)
3019 +{
3020 + struct ext4_prealloc_space *pa;
3021 + struct list_head *cur, *tmp;
3022 + int count = 0;
3023 +
3024 + list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) {
3025 + pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list);
3026 + list_del_rcu(&pa->pa_group_list);
3027 + count++;
3028 + kfree(pa);
3029 + }
3030 + if (count)
3031 + mb_debug("mballoc: %u PAs left\n", count);
3032 +
3033 +}
3034 +
3035 +int ext4_mb_release(struct super_block *sb)
3036 +{
3037 + ext4_group_t i;
3038 + int num_meta_group_infos;
3039 + struct ext4_group_info *grinfo;
3040 + struct ext4_sb_info *sbi = EXT4_SB(sb);
3041 +
3042 + if (!test_opt(sb, MBALLOC))
3043 + return 0;
3044 +
3045 + /* release freed, non-committed blocks */
3046 + spin_lock(&sbi->s_md_lock);
3047 + list_splice_init(&sbi->s_closed_transaction,
3048 + &sbi->s_committed_transaction);
3049 + list_splice_init(&sbi->s_active_transaction,
3050 + &sbi->s_committed_transaction);
3051 + spin_unlock(&sbi->s_md_lock);
3052 + ext4_mb_free_committed_blocks(sb);
3053 +
3054 + if (sbi->s_group_info) {
3055 + for (i = 0; i < sbi->s_groups_count; i++) {
3056 + grinfo = ext4_get_group_info(sb, i);
3057 +#ifdef DOUBLE_CHECK
3058 + kfree(grinfo->bb_bitmap);
3059 +#endif
3060 + ext4_mb_cleanup_pa(grinfo);
3061 + kfree(grinfo);
3062 + }
3063 + num_meta_group_infos = (sbi->s_groups_count +
3064 + EXT4_DESC_PER_BLOCK(sb) - 1) >>
3065 + EXT4_DESC_PER_BLOCK_BITS(sb);
3066 + for (i = 0; i < num_meta_group_infos; i++)
3067 + kfree(sbi->s_group_info[i]);
3068 + kfree(sbi->s_group_info);
3069 + }
3070 + kfree(sbi->s_mb_offsets);
3071 + kfree(sbi->s_mb_maxs);
3072 + if (sbi->s_buddy_cache)
3073 + iput(sbi->s_buddy_cache);
3074 + if (sbi->s_mb_stats) {
3075 + printk(KERN_INFO
3076 + "EXT4-fs: mballoc: %u blocks %u reqs (%u success)\n",
3077 + atomic_read(&sbi->s_bal_allocated),
3078 + atomic_read(&sbi->s_bal_reqs),
3079 + atomic_read(&sbi->s_bal_success));
3080 + printk(KERN_INFO
3081 + "EXT4-fs: mballoc: %u extents scanned, %u goal hits, "
3082 + "%u 2^N hits, %u breaks, %u lost\n",
3083 + atomic_read(&sbi->s_bal_ex_scanned),
3084 + atomic_read(&sbi->s_bal_goals),
3085 + atomic_read(&sbi->s_bal_2orders),
3086 + atomic_read(&sbi->s_bal_breaks),
3087 + atomic_read(&sbi->s_mb_lost_chunks));
3088 + printk(KERN_INFO
3089 + "EXT4-fs: mballoc: %lu generated and it took %Lu\n",
3090 + sbi->s_mb_buddies_generated++,
3091 + sbi->s_mb_generation_time);
3092 + printk(KERN_INFO
3093 + "EXT4-fs: mballoc: %u preallocated, %u discarded\n",
3094 + atomic_read(&sbi->s_mb_preallocated),
3095 + atomic_read(&sbi->s_mb_discarded));
3096 + }
3097 +
3098 + kfree(sbi->s_locality_groups);
3099 +
3100 + ext4_mb_history_release(sb);
3101 + ext4_mb_destroy_per_dev_proc(sb);
3102 +
3103 + return 0;
3104 +}
3105 +
3106 +static void ext4_mb_free_committed_blocks(struct super_block *sb)
3107 +{
3108 + struct ext4_sb_info *sbi = EXT4_SB(sb);
3109 + int err;
3110 + int i;
3111 + int count = 0;
3112 + int count2 = 0;
3113 + struct ext4_free_metadata *md;
3114 + struct ext4_buddy e4b;
3115 +
3116 + if (list_empty(&sbi->s_committed_transaction))
3117 + return;
3118 +
3119 + /* there is committed blocks to be freed yet */
3120 + do {
3121 + /* get next array of blocks */
3122 + md = NULL;
3123 + spin_lock(&sbi->s_md_lock);
3124 + if (!list_empty(&sbi->s_committed_transaction)) {
3125 + md = list_entry(sbi->s_committed_transaction.next,
3126 + struct ext4_free_metadata, list);
3127 + list_del(&md->list);
3128 + }
3129 + spin_unlock(&sbi->s_md_lock);
3130 +
3131 + if (md == NULL)
3132 + break;
3133 +
3134 + mb_debug("gonna free %u blocks in group %lu (0x%p):",
3135 + md->num, md->group, md);
3136 +
3137 + err = ext4_mb_load_buddy(sb, md->group, &e4b);
3138 + /* we expect to find existing buddy because it's pinned */
3139 + BUG_ON(err != 0);
3140 +
3141 + /* there are blocks to put in buddy to make them really free */
3142 + count += md->num;
3143 + count2++;
3144 + ext4_lock_group(sb, md->group);
3145 + for (i = 0; i < md->num; i++) {
3146 + mb_debug(" %u", md->blocks[i]);
3147 + err = mb_free_blocks(NULL, &e4b, md->blocks[i], 1);
3148 + BUG_ON(err != 0);
3149 + }
3150 + mb_debug("\n");
3151 + ext4_unlock_group(sb, md->group);
3152 +
3153 + /* balance refcounts from ext4_mb_free_metadata() */
3154 + page_cache_release(e4b.bd_buddy_page);
3155 + page_cache_release(e4b.bd_bitmap_page);
3156 +
3157 + kfree(md);
3158 + ext4_mb_release_desc(&e4b);
3159 +
3160 + } while (md);
3161 +
3162 + mb_debug("freed %u blocks in %u structures\n", count, count2);
3163 +}
3164 +
3165 +#define EXT4_ROOT "ext4"
3166 +#define EXT4_MB_STATS_NAME "stats"
3167 +#define EXT4_MB_MAX_TO_SCAN_NAME "max_to_scan"
3168 +#define EXT4_MB_MIN_TO_SCAN_NAME "min_to_scan"
3169 +#define EXT4_MB_ORDER2_REQ "order2_req"
3170 +#define EXT4_MB_STREAM_REQ "stream_req"
3171 +#define EXT4_MB_GROUP_PREALLOC "group_prealloc"
3172 +
3173 +
3174 +
3175 +#define MB_PROC_VALUE_READ(name) \
3176 +static int ext4_mb_read_##name(char *page, char **start, \
3177 + off_t off, int count, int *eof, void *data) \
3178 +{ \
3179 + struct ext4_sb_info *sbi = data; \
3180 + int len; \
3181 + *eof = 1; \
3182 + if (off != 0) \
3183 + return 0; \
3184 + len = sprintf(page, "%ld\n", sbi->s_mb_##name); \
3185 + *start = page; \
3186 + return len; \
3187 +}
3188 +
3189 +#define MB_PROC_VALUE_WRITE(name) \
3190 +static int ext4_mb_write_##name(struct file *file, \
3191 + const char __user *buf, unsigned long cnt, void *data) \
3192 +{ \
3193 + struct ext4_sb_info *sbi = data; \
3194 + char str[32]; \
3195 + long value; \
3196 + if (cnt >= sizeof(str)) \
3197 + return -EINVAL; \
3198 + if (copy_from_user(str, buf, cnt)) \
3199 + return -EFAULT; \
3200 + value = simple_strtol(str, NULL, 0); \
3201 + if (value <= 0) \
3202 + return -ERANGE; \
3203 + sbi->s_mb_##name = value; \
3204 + return cnt; \
3205 +}
3206 +
3207 +MB_PROC_VALUE_READ(stats);
3208 +MB_PROC_VALUE_WRITE(stats);
3209 +MB_PROC_VALUE_READ(max_to_scan);
3210 +MB_PROC_VALUE_WRITE(max_to_scan);
3211 +MB_PROC_VALUE_READ(min_to_scan);
3212 +MB_PROC_VALUE_WRITE(min_to_scan);
3213 +MB_PROC_VALUE_READ(order2_reqs);
3214 +MB_PROC_VALUE_WRITE(order2_reqs);
3215 +MB_PROC_VALUE_READ(stream_request);
3216 +MB_PROC_VALUE_WRITE(stream_request);
3217 +MB_PROC_VALUE_READ(group_prealloc);
3218 +MB_PROC_VALUE_WRITE(group_prealloc);
3219 +
3220 +#define MB_PROC_HANDLER(name, var) \
3221 +do { \
3222 + proc = create_proc_entry(name, mode, sbi->s_mb_proc); \
3223 + if (proc == NULL) { \
3224 + printk(KERN_ERR "EXT4-fs: can't to create %s\n", name); \
3225 + goto err_out; \
3226 + } \
3227 + proc->data = sbi; \
3228 + proc->read_proc = ext4_mb_read_##var ; \
3229 + proc->write_proc = ext4_mb_write_##var; \
3230 +} while (0)
3231 +
3232 +static int ext4_mb_init_per_dev_proc(struct super_block *sb)
3233 +{
3234 + mode_t mode = S_IFREG | S_IRUGO | S_IWUSR;
3235 + struct ext4_sb_info *sbi = EXT4_SB(sb);
3236 + struct proc_dir_entry *proc;
3237 + char devname[64];
3238 +
3239 + snprintf(devname, sizeof(devname) - 1, "%s",
3240 + bdevname(sb->s_bdev, devname));
3241 + sbi->s_mb_proc = proc_mkdir(devname, proc_root_ext4);
3242 +
3243 + MB_PROC_HANDLER(EXT4_MB_STATS_NAME, stats);
3244 + MB_PROC_HANDLER(EXT4_MB_MAX_TO_SCAN_NAME, max_to_scan);
3245 + MB_PROC_HANDLER(EXT4_MB_MIN_TO_SCAN_NAME, min_to_scan);
3246 + MB_PROC_HANDLER(EXT4_MB_ORDER2_REQ, order2_reqs);
3247 + MB_PROC_HANDLER(EXT4_MB_STREAM_REQ, stream_request);
3248 + MB_PROC_HANDLER(EXT4_MB_GROUP_PREALLOC, group_prealloc);
3249 +
3250 + return 0;
3251 +
3252 +err_out:
3253 + printk(KERN_ERR "EXT4-fs: Unable to create %s\n", devname);
3254 + remove_proc_entry(EXT4_MB_GROUP_PREALLOC, sbi->s_mb_proc);
3255 + remove_proc_entry(EXT4_MB_STREAM_REQ, sbi->s_mb_proc);
3256 + remove_proc_entry(EXT4_MB_ORDER2_REQ, sbi->s_mb_proc);
3257 + remove_proc_entry(EXT4_MB_MIN_TO_SCAN_NAME, sbi->s_mb_proc);
3258 + remove_proc_entry(EXT4_MB_MAX_TO_SCAN_NAME, sbi->s_mb_proc);
3259 + remove_proc_entry(EXT4_MB_STATS_NAME, sbi->s_mb_proc);
3260 + remove_proc_entry(devname, proc_root_ext4);
3261 + sbi->s_mb_proc = NULL;
3262 +
3263 + return -ENOMEM;
3264 +}
3265 +
3266 +static int ext4_mb_destroy_per_dev_proc(struct super_block *sb)
3267 +{
3268 + struct ext4_sb_info *sbi = EXT4_SB(sb);
3269 + char devname[64];
3270 +
3271 + if (sbi->s_mb_proc == NULL)
3272 + return -EINVAL;
3273 +
3274 + snprintf(devname, sizeof(devname) - 1, "%s",
3275 + bdevname(sb->s_bdev, devname));
3276 + remove_proc_entry(EXT4_MB_GROUP_PREALLOC, sbi->s_mb_proc);
3277 + remove_proc_entry(EXT4_MB_STREAM_REQ, sbi->s_mb_proc);
3278 + remove_proc_entry(EXT4_MB_ORDER2_REQ, sbi->s_mb_proc);
3279 + remove_proc_entry(EXT4_MB_MIN_TO_SCAN_NAME, sbi->s_mb_proc);
3280 + remove_proc_entry(EXT4_MB_MAX_TO_SCAN_NAME, sbi->s_mb_proc);
3281 + remove_proc_entry(EXT4_MB_STATS_NAME, sbi->s_mb_proc);
3282 + remove_proc_entry(devname, proc_root_ext4);
3283 +
3284 + return 0;
3285 +}
3286 +
3287 +int __init init_ext4_mballoc(void)
3288 +{
3289 + ext4_pspace_cachep =
3290 + kmem_cache_create("ext4_prealloc_space",
3291 + sizeof(struct ext4_prealloc_space),
3292 + 0, SLAB_RECLAIM_ACCOUNT, NULL);
3293 + if (ext4_pspace_cachep == NULL)
3294 + return -ENOMEM;
3295 +
3296 +#ifdef CONFIG_PROC_FS
3297 + proc_root_ext4 = proc_mkdir(EXT4_ROOT, proc_root_fs);
3298 + if (proc_root_ext4 == NULL)
3299 + printk(KERN_ERR "EXT4-fs: Unable to create %s\n", EXT4_ROOT);
3300 +#endif
3301 +
3302 + return 0;
3303 +}
3304 +
3305 +void exit_ext4_mballoc(void)
3306 +{
3307 + /* XXX: synchronize_rcu(); */
3308 + kmem_cache_destroy(ext4_pspace_cachep);
3309 +#ifdef CONFIG_PROC_FS
3310 + remove_proc_entry(EXT4_ROOT, proc_root_fs);
3311 +#endif
3312 +}
3313 +
3314 +
3315 +/*
3316 + * Check quota and mark choosed space (ac->ac_b_ex) non-free in bitmaps
3317 + * Returns 0 if success or error code
3318 + */
3319 +static int ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,
3320 + handle_t *handle)
3321 +{
3322 + struct buffer_head *bitmap_bh = NULL;
3323 + struct ext4_super_block *es;
3324 + struct ext4_group_desc *gdp;
3325 + struct buffer_head *gdp_bh;
3326 + struct ext4_sb_info *sbi;
3327 + struct super_block *sb;
3328 + ext4_fsblk_t block;
3329 + int err;
3330 +
3331 + BUG_ON(ac->ac_status != AC_STATUS_FOUND);
3332 + BUG_ON(ac->ac_b_ex.fe_len <= 0);
3333 +
3334 + sb = ac->ac_sb;
3335 + sbi = EXT4_SB(sb);
3336 + es = sbi->s_es;
3337 +
3338 + ext4_debug("using block group %lu(%d)\n", ac->ac_b_ex.fe_group,
3339 + gdp->bg_free_blocks_count);
3340 +
3341 + err = -EIO;
3342 + bitmap_bh = read_block_bitmap(sb, ac->ac_b_ex.fe_group);
3343 + if (!bitmap_bh)
3344 + goto out_err;
3345 +
3346 + err = ext4_journal_get_write_access(handle, bitmap_bh);
3347 + if (err)
3348 + goto out_err;
3349 +
3350 + err = -EIO;
3351 + gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh);
3352 + if (!gdp)
3353 + goto out_err;
3354 +
3355 + err = ext4_journal_get_write_access(handle, gdp_bh);
3356 + if (err)
3357 + goto out_err;
3358 +
3359 + block = ac->ac_b_ex.fe_group * EXT4_BLOCKS_PER_GROUP(sb)
3360 + + ac->ac_b_ex.fe_start
3361 + + le32_to_cpu(es->s_first_data_block);
3362 +
3363 + if (block == ext4_block_bitmap(sb, gdp) ||
3364 + block == ext4_inode_bitmap(sb, gdp) ||
3365 + in_range(block, ext4_inode_table(sb, gdp),
3366 + EXT4_SB(sb)->s_itb_per_group)) {
3367 +
3368 + ext4_error(sb, __FUNCTION__,
3369 + "Allocating block in system zone - block = %llu",
3370 + block);
3371 + }
3372 +#ifdef AGGRESSIVE_CHECK
3373 + {
3374 + int i;
3375 + for (i = 0; i < ac->ac_b_ex.fe_len; i++) {
3376 + BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i,
3377 + bitmap_bh->b_data));
3378 + }
3379 + }
3380 +#endif
3381 + mb_set_bits(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group), bitmap_bh->b_data,
3382 + ac->ac_b_ex.fe_start, ac->ac_b_ex.fe_len);
3383 +
3384 + spin_lock(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group));
3385 + if (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) {
3386 + gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);
3387 + gdp->bg_free_blocks_count =
3388 + cpu_to_le16(ext4_free_blocks_after_init(sb,
3389 + ac->ac_b_ex.fe_group,
3390 + gdp));
3391 + }
3392 + gdp->bg_free_blocks_count =
3393 + cpu_to_le16(le16_to_cpu(gdp->bg_free_blocks_count)
3394 + - ac->ac_b_ex.fe_len);
3395 + gdp->bg_checksum = ext4_group_desc_csum(sbi, ac->ac_b_ex.fe_group, gdp);
3396 + spin_unlock(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group));
3397 + percpu_counter_sub(&sbi->s_freeblocks_counter, ac->ac_b_ex.fe_len);
3398 +
3399 + err = ext4_journal_dirty_metadata(handle, bitmap_bh);
3400 + if (err)
3401 + goto out_err;
3402 + err = ext4_journal_dirty_metadata(handle, gdp_bh);
3403 +
3404 +out_err:
3405 + sb->s_dirt = 1;
3406 + brelse(bitmap_bh);
3407 + return err;
3408 +}
3409 +
3410 +/*
3411 + * here we normalize request for locality group
3412 + * Group request are normalized to s_strip size if we set the same via mount
3413 + * option. If not we set it to s_mb_group_prealloc which can be configured via
3414 + * /proc/fs/ext4/<partition>/group_prealloc
3415 + *
3416 + * XXX: should we try to preallocate more than the group has now?
3417 + */
3418 +static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac)
3419 +{
3420 + struct super_block *sb = ac->ac_sb;
3421 + struct ext4_locality_group *lg = ac->ac_lg;
3422 +
3423 + BUG_ON(lg == NULL);
3424 + if (EXT4_SB(sb)->s_stripe)
3425 + ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_stripe;
3426 + else
3427 + ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc;
3428 + mb_debug("#%u: goal %lu blocks for locality group\n",
3429 + current->pid, ac->ac_g_ex.fe_len);
3430 +}
3431 +
3432 +/*
3433 + * Normalization means making request better in terms of
3434 + * size and alignment
3435 + */
3436 +static void ext4_mb_normalize_request(struct ext4_allocation_context *ac,
3437 + struct ext4_allocation_request *ar)
3438 +{
3439 + int bsbits, max;
3440 + ext4_lblk_t end;
3441 + struct list_head *cur;
3442 + loff_t size, orig_size, start_off;
3443 + ext4_lblk_t start, orig_start;
3444 + struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
3445 +
3446 + /* do normalize only data requests, metadata requests
3447 + do not need preallocation */
3448 + if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
3449 + return;
3450 +
3451 + /* sometime caller may want exact blocks */
3452 + if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
3453 + return;
3454 +
3455 + /* caller may indicate that preallocation isn't
3456 + * required (it's a tail, for example) */
3457 + if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC)
3458 + return;
3459 +
3460 + if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) {
3461 + ext4_mb_normalize_group_request(ac);
3462 + return ;
3463 + }
3464 +
3465 + bsbits = ac->ac_sb->s_blocksize_bits;
3466 +
3467 + /* first, let's learn actual file size
3468 + * given current request is allocated */
3469 + size = ac->ac_o_ex.fe_logical + ac->ac_o_ex.fe_len;
3470 + size = size << bsbits;
3471 + if (size < i_size_read(ac->ac_inode))
3472 + size = i_size_read(ac->ac_inode);
3473 +
3474 + /* max available blocks in a free group */
3475 + max = EXT4_BLOCKS_PER_GROUP(ac->ac_sb) - 1 - 1 -
3476 + EXT4_SB(ac->ac_sb)->s_itb_per_group;
3477 +
3478 +#define NRL_CHECK_SIZE(req, size, max,bits) \
3479 + (req <= (size) || max <= ((size) >> bits))
3480 +
3481 + /* first, try to predict filesize */
3482 + /* XXX: should this table be tunable? */
3483 + start_off = 0;
3484 + if (size <= 16 * 1024) {
3485 + size = 16 * 1024;
3486 + } else if (size <= 32 * 1024) {
3487 + size = 32 * 1024;
3488 + } else if (size <= 64 * 1024) {
3489 + size = 64 * 1024;
3490 + } else if (size <= 128 * 1024) {
3491 + size = 128 * 1024;
3492 + } else if (size <= 256 * 1024) {
3493 + size = 256 * 1024;
3494 + } else if (size <= 512 * 1024) {
3495 + size = 512 * 1024;
3496 + } else if (size <= 1024 * 1024) {
3497 + size = 1024 * 1024;
3498 + } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, bsbits)) {
3499 + start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
3500 + (20 - bsbits)) << 20;
3501 + size = 1024 * 1024;
3502 + } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, bsbits)) {
3503 + start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
3504 + (22 - bsbits)) << 22;
3505 + size = 4 * 1024 * 1024;
3506 + } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len,
3507 + (8<<20)>>bsbits, max, bsbits)) {
3508 + start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
3509 + (23 - bsbits)) << 23;
3510 + size = 8 * 1024 * 1024;
3511 + } else {
3512 + start_off = (loff_t)ac->ac_o_ex.fe_logical << bsbits;
3513 + size = ac->ac_o_ex.fe_len << bsbits;
3514 + }
3515 + orig_size = size = size >> bsbits;
3516 + orig_start = start = start_off >> bsbits;
3517 +
3518 + /* don't cover already allocated blocks in selected range */
3519 + if (ar->pleft && start <= ar->lleft) {
3520 + size -= ar->lleft + 1 - start;
3521 + start = ar->lleft + 1;
3522 + }
3523 + if (ar->pright && start + size - 1 >= ar->lright)
3524 + size -= start + size - ar->lright;
3525 +
3526 + end = start + size;
3527 +
3528 + /* check we don't cross already preallocated blocks */
3529 + rcu_read_lock();
3530 + list_for_each_rcu(cur, &ei->i_prealloc_list) {
3531 + struct ext4_prealloc_space *pa;
3532 + unsigned long pa_end;
3533 +
3534 + pa = list_entry(cur, struct ext4_prealloc_space, pa_inode_list);
3535 +
3536 + if (pa->pa_deleted)
3537 + continue;
3538 + spin_lock(&pa->pa_lock);
3539 + if (pa->pa_deleted) {
3540 + spin_unlock(&pa->pa_lock);
3541 + continue;
3542 + }
3543 +
3544 + pa_end = pa->pa_lstart + pa->pa_len;
3545 +
3546 + /* PA must not overlap original request */
3547 + BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end ||
3548 + ac->ac_o_ex.fe_logical < pa->pa_lstart));
3549 +
3550 + /* skip PA normalized request doesn't overlap with */
3551 + if (pa->pa_lstart >= end) {
3552 + spin_unlock(&pa->pa_lock);
3553 + continue;
3554 + }
3555 + if (pa_end <= start) {
3556 + spin_unlock(&pa->pa_lock);
3557 + continue;
3558 + }
3559 + BUG_ON(pa->pa_lstart <= start && pa_end >= end);
3560 +
3561 + if (pa_end <= ac->ac_o_ex.fe_logical) {
3562 + BUG_ON(pa_end < start);
3563 + start = pa_end;
3564 + }
3565 +
3566 + if (pa->pa_lstart > ac->ac_o_ex.fe_logical) {
3567 + BUG_ON(pa->pa_lstart > end);
3568 + end = pa->pa_lstart;
3569 + }
3570 + spin_unlock(&pa->pa_lock);
3571 + }
3572 + rcu_read_unlock();
3573 + size = end - start;
3574 +
3575 + /* XXX: extra loop to check we really don't overlap preallocations */
3576 + rcu_read_lock();
3577 + list_for_each_rcu(cur, &ei->i_prealloc_list) {
3578 + struct ext4_prealloc_space *pa;
3579 + unsigned long pa_end;
3580 + pa = list_entry(cur, struct ext4_prealloc_space, pa_inode_list);
3581 + spin_lock(&pa->pa_lock);
3582 + if (pa->pa_deleted == 0) {
3583 + pa_end = pa->pa_lstart + pa->pa_len;
3584 + BUG_ON(!(start >= pa_end || end <= pa->pa_lstart));
3585 + }
3586 + spin_unlock(&pa->pa_lock);
3587 + }
3588 + rcu_read_unlock();
3589 +
3590 + if (start + size <= ac->ac_o_ex.fe_logical &&
3591 + start > ac->ac_o_ex.fe_logical) {
3592 + printk(KERN_ERR "start %lu, size %lu, fe_logical %lu\n",
3593 + (unsigned long) start, (unsigned long) size,
3594 + (unsigned long) ac->ac_o_ex.fe_logical);
3595 + }
3596 + BUG_ON(start + size <= ac->ac_o_ex.fe_logical &&
3597 + start > ac->ac_o_ex.fe_logical);
3598 + BUG_ON(size <= 0 || size >= EXT4_BLOCKS_PER_GROUP(ac->ac_sb));
3599 +
3600 + /* now prepare goal request */
3601 +
3602 + /* XXX: is it better to align blocks WRT to logical
3603 + * placement or satisfy big request as is */
3604 + ac->ac_g_ex.fe_logical = start;
3605 + ac->ac_g_ex.fe_len = size;
3606 +
3607 + /* define goal start in order to merge */
3608 + if (ar->pright && (ar->lright == (start + size))) {
3609 + /* merge to the right */
3610 + ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size,
3611 + &ac->ac_f_ex.fe_group,
3612 + &ac->ac_f_ex.fe_start);
3613 + ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;
3614 + }
3615 + if (ar->pleft && (ar->lleft + 1 == start)) {
3616 + /* merge to the left */
3617 + ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1,
3618 + &ac->ac_f_ex.fe_group,
3619 + &ac->ac_f_ex.fe_start);
3620 + ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;
3621 + }
3622 +
3623 + mb_debug("goal: %u(was %u) blocks at %u\n", (unsigned) size,
3624 + (unsigned) orig_size, (unsigned) start);
3625 +}
3626 +
3627 +static void ext4_mb_collect_stats(struct ext4_allocation_context *ac)
3628 +{
3629 + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
3630 +
3631 + if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) {
3632 + atomic_inc(&sbi->s_bal_reqs);
3633 + atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);
3634 + if (ac->ac_o_ex.fe_len >= ac->ac_g_ex.fe_len)
3635 + atomic_inc(&sbi->s_bal_success);
3636 + atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);
3637 + if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&
3638 + ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)
3639 + atomic_inc(&sbi->s_bal_goals);
3640 + if (ac->ac_found > sbi->s_mb_max_to_scan)
3641 + atomic_inc(&sbi->s_bal_breaks);
3642 + }
3643 +
3644 + ext4_mb_store_history(ac);
3645 +}
3646 +
3647 +/*
3648 + * use blocks preallocated to inode
3649 + */
3650 +static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac,
3651 + struct ext4_prealloc_space *pa)
3652 +{
3653 + ext4_fsblk_t start;
3654 + ext4_fsblk_t end;
3655 + int len;
3656 +
3657 + /* found preallocated blocks, use them */
3658 + start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart);
3659 + end = min(pa->pa_pstart + pa->pa_len, start + ac->ac_o_ex.fe_len);
3660 + len = end - start;
3661 + ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group,
3662 + &ac->ac_b_ex.fe_start);
3663 + ac->ac_b_ex.fe_len = len;
3664 + ac->ac_status = AC_STATUS_FOUND;
3665 + ac->ac_pa = pa;
3666 +
3667 + BUG_ON(start < pa->pa_pstart);
3668 + BUG_ON(start + len > pa->pa_pstart + pa->pa_len);
3669 + BUG_ON(pa->pa_free < len);
3670 + pa->pa_free -= len;
3671 +
3672 + mb_debug("use %llu/%lu from inode pa %p\n", start, len, pa);
3673 +}
3674 +
3675 +/*
3676 + * use blocks preallocated to locality group
3677 + */
3678 +static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac,
3679 + struct ext4_prealloc_space *pa)
3680 +{
3681 + unsigned len = ac->ac_o_ex.fe_len;
3682 +
3683 + ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart,
3684 + &ac->ac_b_ex.fe_group,
3685 + &ac->ac_b_ex.fe_start);
3686 + ac->ac_b_ex.fe_len = len;
3687 + ac->ac_status = AC_STATUS_FOUND;
3688 + ac->ac_pa = pa;
3689 +
3690 + /* we don't correct pa_pstart or pa_plen here to avoid
3691 + * possible race when tte group is being loaded concurrently
3692 + * instead we correct pa later, after blocks are marked
3693 + * in on-disk bitmap -- see ext4_mb_release_context() */
3694 + /*
3695 + * FIXME!! but the other CPUs can look at this particular
3696 + * pa and think that it have enought free blocks if we
3697 + * don't update pa_free here right ?
3698 + */
3699 + mb_debug("use %u/%u from group pa %p\n", pa->pa_lstart-len, len, pa);
3700 +}
3701 +
3702 +/*
3703 + * search goal blocks in preallocated space
3704 + */
3705 +static int ext4_mb_use_preallocated(struct ext4_allocation_context *ac)
3706 +{
3707 + struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
3708 + struct ext4_locality_group *lg;
3709 + struct ext4_prealloc_space *pa;
3710 + struct list_head *cur;
3711 +
3712 + /* only data can be preallocated */
3713 + if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
3714 + return 0;
3715 +
3716 + /* first, try per-file preallocation */
3717 + rcu_read_lock();
3718 + list_for_each_rcu(cur, &ei->i_prealloc_list) {
3719 + pa = list_entry(cur, struct ext4_prealloc_space, pa_inode_list);
3720 +
3721 + /* all fields in this condition don't change,
3722 + * so we can skip locking for them */
3723 + if (ac->ac_o_ex.fe_logical < pa->pa_lstart ||
3724 + ac->ac_o_ex.fe_logical >= pa->pa_lstart + pa->pa_len)
3725 + continue;
3726 +
3727 + /* found preallocated blocks, use them */
3728 + spin_lock(&pa->pa_lock);
3729 + if (pa->pa_deleted == 0 && pa->pa_free) {
3730 + atomic_inc(&pa->pa_count);
3731 + ext4_mb_use_inode_pa(ac, pa);
3732 + spin_unlock(&pa->pa_lock);
3733 + ac->ac_criteria = 10;
3734 + rcu_read_unlock();
3735 + return 1;
3736 + }
3737 + spin_unlock(&pa->pa_lock);
3738 + }
3739 + rcu_read_unlock();
3740 +
3741 + /* can we use group allocation? */
3742 + if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC))
3743 + return 0;
3744 +
3745 + /* inode may have no locality group for some reason */
3746 + lg = ac->ac_lg;
3747 + if (lg == NULL)
3748 + return 0;
3749 +
3750 + rcu_read_lock();
3751 + list_for_each_rcu(cur, &lg->lg_prealloc_list) {
3752 + pa = list_entry(cur, struct ext4_prealloc_space, pa_inode_list);
3753 + spin_lock(&pa->pa_lock);
3754 + if (pa->pa_deleted == 0 && pa->pa_free >= ac->ac_o_ex.fe_len) {
3755 + atomic_inc(&pa->pa_count);
3756 + ext4_mb_use_group_pa(ac, pa);
3757 + spin_unlock(&pa->pa_lock);
3758 + ac->ac_criteria = 20;
3759 + rcu_read_unlock();
3760 + return 1;
3761 + }
3762 + spin_unlock(&pa->pa_lock);
3763 + }
3764 + rcu_read_unlock();
3765 +
3766 + return 0;
3767 +}
3768 +
3769 +/*
3770 + * the function goes through all preallocation in this group and marks them
3771 + * used in in-core bitmap. buddy must be generated from this bitmap
3772 + */
3773 +static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap,
3774 + ext4_group_t group)
3775 +{
3776 + struct ext4_group_info *grp = ext4_get_group_info(sb, group);
3777 + struct ext4_prealloc_space *pa;
3778 + struct list_head *cur;
3779 + ext4_group_t groupnr;
3780 + ext4_grpblk_t start;
3781 + int preallocated = 0;
3782 + int count = 0;
3783 + int len;
3784 +
3785 + /* all form of preallocation discards first load group,
3786 + * so the only competing code is preallocation use.
3787 + * we don't need any locking here
3788 + * notice we do NOT ignore preallocations with pa_deleted
3789 + * otherwise we could leave used blocks available for
3790 + * allocation in buddy when concurrent ext4_mb_put_pa()
3791 + * is dropping preallocation
3792 + */
3793 + list_for_each_rcu(cur, &grp->bb_prealloc_list) {
3794 + pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list);
3795 + spin_lock(&pa->pa_lock);
3796 + ext4_get_group_no_and_offset(sb, pa->pa_pstart,
3797 + &groupnr, &start);
3798 + len = pa->pa_len;
3799 + spin_unlock(&pa->pa_lock);
3800 + if (unlikely(len == 0))
3801 + continue;
3802 + BUG_ON(groupnr != group);
3803 + mb_set_bits(sb_bgl_lock(EXT4_SB(sb), group),
3804 + bitmap, start, len);
3805 + preallocated += len;
3806 + count++;
3807 + }
3808 + mb_debug("prellocated %u for group %lu\n", preallocated, group);
3809 +}
3810 +
3811 +static void ext4_mb_pa_callback(struct rcu_head *head)
3812 +{
3813 + struct ext4_prealloc_space *pa;
3814 + pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu);
3815 + kmem_cache_free(ext4_pspace_cachep, pa);
3816 +}
3817 +#define mb_call_rcu(__pa) call_rcu(&(__pa)->u.pa_rcu, ext4_mb_pa_callback)
3818 +
3819 +/*
3820 + * drops a reference to preallocated space descriptor
3821 + * if this was the last reference and the space is consumed
3822 + */
3823 +static void ext4_mb_put_pa(struct ext4_allocation_context *ac,
3824 + struct super_block *sb, struct ext4_prealloc_space *pa)
3825 +{
3826 + unsigned long grp;
3827 +
3828 + if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0)
3829 + return;
3830 +
3831 + /* in this short window concurrent discard can set pa_deleted */
3832 + spin_lock(&pa->pa_lock);
3833 + if (pa->pa_deleted == 1) {
3834 + spin_unlock(&pa->pa_lock);
3835 + return;
3836 + }
3837 +
3838 + pa->pa_deleted = 1;
3839 + spin_unlock(&pa->pa_lock);
3840 +
3841 + /* -1 is to protect from crossing allocation group */
3842 + ext4_get_group_no_and_offset(sb, pa->pa_pstart - 1, &grp, NULL);
3843 +
3844 + /*
3845 + * possible race:
3846 + *
3847 + * P1 (buddy init) P2 (regular allocation)
3848 + * find block B in PA
3849 + * copy on-disk bitmap to buddy
3850 + * mark B in on-disk bitmap
3851 + * drop PA from group
3852 + * mark all PAs in buddy
3853 + *
3854 + * thus, P1 initializes buddy with B available. to prevent this
3855 + * we make "copy" and "mark all PAs" atomic and serialize "drop PA"
3856 + * against that pair
3857 + */
3858 + ext4_lock_group(sb, grp);
3859 + list_del_rcu(&pa->pa_group_list);
3860 + ext4_unlock_group(sb, grp);
3861 +
3862 + spin_lock(pa->pa_obj_lock);
3863 + list_del_rcu(&pa->pa_inode_list);
3864 + spin_unlock(pa->pa_obj_lock);
3865 +
3866 + mb_call_rcu(pa);
3867 +}
3868 +
3869 +/*
3870 + * creates new preallocated space for given inode
3871 + */
3872 +static int ext4_mb_new_inode_pa(struct ext4_allocation_context *ac)
3873 +{
3874 + struct super_block *sb = ac->ac_sb;
3875 + struct ext4_prealloc_space *pa;
3876 + struct ext4_group_info *grp;
3877 + struct ext4_inode_info *ei;
3878 +
3879 + /* preallocate only when found space is larger then requested */
3880 + BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);
3881 + BUG_ON(ac->ac_status != AC_STATUS_FOUND);
3882 + BUG_ON(!S_ISREG(ac->ac_inode->i_mode));
3883 +
3884 + pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS);
3885 + if (pa == NULL)
3886 + return -ENOMEM;
3887 +
3888 + if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) {
3889 + int winl;
3890 + int wins;
3891 + int win;
3892 + int offs;
3893 +
3894 + /* we can't allocate as much as normalizer wants.
3895 + * so, found space must get proper lstart
3896 + * to cover original request */
3897 + BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical);
3898 + BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len);
3899 +
3900 + /* we're limited by original request in that
3901 + * logical block must be covered any way
3902 + * winl is window we can move our chunk within */
3903 + winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical;
3904 +
3905 + /* also, we should cover whole original request */
3906 + wins = ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len;
3907 +
3908 + /* the smallest one defines real window */
3909 + win = min(winl, wins);
3910 +
3911 + offs = ac->ac_o_ex.fe_logical % ac->ac_b_ex.fe_len;
3912 + if (offs && offs < win)
3913 + win = offs;
3914 +
3915 + ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical - win;
3916 + BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical);
3917 + BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len);
3918 + }
3919 +
3920 + /* preallocation can change ac_b_ex, thus we store actually
3921 + * allocated blocks for history */
3922 + ac->ac_f_ex = ac->ac_b_ex;
3923 +
3924 + pa->pa_lstart = ac->ac_b_ex.fe_logical;
3925 + pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
3926 + pa->pa_len = ac->ac_b_ex.fe_len;
3927 + pa->pa_free = pa->pa_len;
3928 + atomic_set(&pa->pa_count, 1);
3929 + spin_lock_init(&pa->pa_lock);
3930 + pa->pa_deleted = 0;
3931 + pa->pa_linear = 0;
3932 +
3933 + mb_debug("new inode pa %p: %llu/%u for %u\n", pa,
3934 + pa->pa_pstart, pa->pa_len, pa->pa_lstart);
3935 +
3936 + ext4_mb_use_inode_pa(ac, pa);
3937 + atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);
3938 +
3939 + ei = EXT4_I(ac->ac_inode);
3940 + grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
3941 +
3942 + pa->pa_obj_lock = &ei->i_prealloc_lock;
3943 + pa->pa_inode = ac->ac_inode;
3944 +
3945 + ext4_lock_group(sb, ac->ac_b_ex.fe_group);
3946 + list_add_rcu(&pa->pa_group_list, &grp->bb_prealloc_list);
3947 + ext4_unlock_group(sb, ac->ac_b_ex.fe_group);
3948 +
3949 + spin_lock(pa->pa_obj_lock);
3950 + list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list);
3951 + spin_unlock(pa->pa_obj_lock);
3952 +
3953 + return 0;
3954 +}
3955 +
3956 +/*
3957 + * creates new preallocated space for locality group inodes belongs to
3958 + */
3959 +static int ext4_mb_new_group_pa(struct ext4_allocation_context *ac)
3960 +{
3961 + struct super_block *sb = ac->ac_sb;
3962 + struct ext4_locality_group *lg;
3963 + struct ext4_prealloc_space *pa;
3964 + struct ext4_group_info *grp;
3965 +
3966 + /* preallocate only when found space is larger then requested */
3967 + BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);
3968 + BUG_ON(ac->ac_status != AC_STATUS_FOUND);
3969 + BUG_ON(!S_ISREG(ac->ac_inode->i_mode));
3970 +
3971 + BUG_ON(ext4_pspace_cachep == NULL);
3972 + pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS);
3973 + if (pa == NULL)
3974 + return -ENOMEM;
3975 +
3976 + /* preallocation can change ac_b_ex, thus we store actually
3977 + * allocated blocks for history */
3978 + ac->ac_f_ex = ac->ac_b_ex;
3979 +
3980 + pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
3981 + pa->pa_lstart = pa->pa_pstart;
3982 + pa->pa_len = ac->ac_b_ex.fe_len;
3983 + pa->pa_free = pa->pa_len;
3984 + atomic_set(&pa->pa_count, 1);
3985 + spin_lock_init(&pa->pa_lock);
3986 + pa->pa_deleted = 0;
3987 + pa->pa_linear = 1;
3988 +
3989 + mb_debug("new group pa %p: %llu/%u for %u\n", pa,
3990 + pa->pa_pstart, pa->pa_len, pa->pa_lstart);
3991 +
3992 + ext4_mb_use_group_pa(ac, pa);
3993 + atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);
3994 +
3995 + grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
3996 + lg = ac->ac_lg;
3997 + BUG_ON(lg == NULL);
3998 +
3999 + pa->pa_obj_lock = &lg->lg_prealloc_lock;
4000 + pa->pa_inode = NULL;
4001 +
4002 + ext4_lock_group(sb, ac->ac_b_ex.fe_group);
4003 + list_add_rcu(&pa->pa_group_list, &grp->bb_prealloc_list);
4004 + ext4_unlock_group(sb, ac->ac_b_ex.fe_group);
4005 +
4006 + spin_lock(pa->pa_obj_lock);
4007 + list_add_tail_rcu(&pa->pa_inode_list, &lg->lg_prealloc_list);
4008 + spin_unlock(pa->pa_obj_lock);
4009 +
4010 + return 0;
4011 +}
4012 +
4013 +static int ext4_mb_new_preallocation(struct ext4_allocation_context *ac)
4014 +{
4015 + int err;
4016 +
4017 + if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)
4018 + err = ext4_mb_new_group_pa(ac);
4019 + else
4020 + err = ext4_mb_new_inode_pa(ac);
4021 + return err;
4022 +}
4023 +
4024 +/*
4025 + * finds all unused blocks in on-disk bitmap, frees them in
4026 + * in-core bitmap and buddy.
4027 + * @pa must be unlinked from inode and group lists, so that
4028 + * nobody else can find/use it.
4029 + * the caller MUST hold group/inode locks.
4030 + * TODO: optimize the case when there are no in-core structures yet
4031 + */
4032 +static int ext4_mb_release_inode_pa(struct ext4_buddy *e4b,
4033 + struct buffer_head *bitmap_bh,
4034 + struct ext4_prealloc_space *pa)
4035 +{
4036 + struct ext4_allocation_context ac;
4037 + struct super_block *sb = e4b->bd_sb;
4038 + struct ext4_sb_info *sbi = EXT4_SB(sb);
4039 + unsigned long end;
4040 + unsigned long next;
4041 + ext4_group_t group;
4042 + ext4_grpblk_t bit;
4043 + sector_t start;
4044 + int err = 0;
4045 + int free = 0;
4046 +
4047 + BUG_ON(pa->pa_deleted == 0);
4048 + ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);
4049 + BUG_ON(group != e4b->bd_group && pa->pa_len != 0);
4050 + end = bit + pa->pa_len;
4051 +
4052 + ac.ac_sb = sb;
4053 + ac.ac_inode = pa->pa_inode;
4054 + ac.ac_op = EXT4_MB_HISTORY_DISCARD;
4055 +
4056 + while (bit < end) {
4057 + bit = ext4_find_next_zero_bit(bitmap_bh->b_data, end, bit);
4058 + if (bit >= end)
4059 + break;
4060 + next = ext4_find_next_bit(bitmap_bh->b_data, end, bit);
4061 + if (next > end)
4062 + next = end;
4063 + start = group * EXT4_BLOCKS_PER_GROUP(sb) + bit +
4064 + le32_to_cpu(sbi->s_es->s_first_data_block);
4065 + mb_debug(" free preallocated %u/%u in group %u\n",
4066 + (unsigned) start, (unsigned) next - bit,
4067 + (unsigned) group);
4068 + free += next - bit;
4069 +
4070 + ac.ac_b_ex.fe_group = group;
4071 + ac.ac_b_ex.fe_start = bit;
4072 + ac.ac_b_ex.fe_len = next - bit;
4073 + ac.ac_b_ex.fe_logical = 0;
4074 + ext4_mb_store_history(&ac);
4075 +
4076 + mb_free_blocks(pa->pa_inode, e4b, bit, next - bit);
4077 + bit = next + 1;
4078 + }
4079 + if (free != pa->pa_free) {
4080 + printk(KERN_ERR "pa %p: logic %lu, phys. %lu, len %lu\n",
4081 + pa, (unsigned long) pa->pa_lstart,
4082 + (unsigned long) pa->pa_pstart,
4083 + (unsigned long) pa->pa_len);
4084 + printk(KERN_ERR "free %u, pa_free %u\n", free, pa->pa_free);
4085 + }
4086 + BUG_ON(free != pa->pa_free);
4087 + atomic_add(free, &sbi->s_mb_discarded);
4088 +
4089 + return err;
4090 +}
4091 +
4092 +static int ext4_mb_release_group_pa(struct ext4_buddy *e4b,
4093 + struct ext4_prealloc_space *pa)
4094 +{
4095 + struct ext4_allocation_context ac;
4096 + struct super_block *sb = e4b->bd_sb;
4097 + ext4_group_t group;
4098 + ext4_grpblk_t bit;
4099 +
4100 + ac.ac_op = EXT4_MB_HISTORY_DISCARD;
4101 +
4102 + BUG_ON(pa->pa_deleted == 0);
4103 + ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);
4104 + BUG_ON(group != e4b->bd_group && pa->pa_len != 0);
4105 + mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len);
4106 + atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded);
4107 +
4108 + ac.ac_sb = sb;
4109 + ac.ac_inode = NULL;
4110 + ac.ac_b_ex.fe_group = group;
4111 + ac.ac_b_ex.fe_start = bit;
4112 + ac.ac_b_ex.fe_len = pa->pa_len;
4113 + ac.ac_b_ex.fe_logical = 0;
4114 + ext4_mb_store_history(&ac);
4115 +
4116 + return 0;
4117 +}
4118 +
4119 +/*
4120 + * releases all preallocations in given group
4121 + *
4122 + * first, we need to decide discard policy:
4123 + * - when do we discard
4124 + * 1) ENOSPC
4125 + * - how many do we discard
4126 + * 1) how many requested
4127 + */
4128 +static int ext4_mb_discard_group_preallocations(struct super_block *sb,
4129 + ext4_group_t group, int needed)
4130 +{
4131 + struct ext4_group_info *grp = ext4_get_group_info(sb, group);
4132 + struct buffer_head *bitmap_bh = NULL;
4133 + struct ext4_prealloc_space *pa, *tmp;
4134 + struct list_head list;
4135 + struct ext4_buddy e4b;
4136 + int err;
4137 + int busy = 0;
4138 + int free = 0;
4139 +
4140 + mb_debug("discard preallocation for group %lu\n", group);
4141 +
4142 + if (list_empty(&grp->bb_prealloc_list))
4143 + return 0;
4144 +
4145 + bitmap_bh = read_block_bitmap(sb, group);
4146 + if (bitmap_bh == NULL) {
4147 + /* error handling here */
4148 + ext4_mb_release_desc(&e4b);
4149 + BUG_ON(bitmap_bh == NULL);
4150 + }
4151 +
4152 + err = ext4_mb_load_buddy(sb, group, &e4b);
4153 + BUG_ON(err != 0); /* error handling here */
4154 +
4155 + if (needed == 0)
4156 + needed = EXT4_BLOCKS_PER_GROUP(sb) + 1;
4157 +
4158 + grp = ext4_get_group_info(sb, group);
4159 + INIT_LIST_HEAD(&list);
4160 +
4161 +repeat:
4162 + ext4_lock_group(sb, group);
4163 + list_for_each_entry_safe(pa, tmp,
4164 + &grp->bb_prealloc_list, pa_group_list) {
4165 + spin_lock(&pa->pa_lock);
4166 + if (atomic_read(&pa->pa_count)) {
4167 + spin_unlock(&pa->pa_lock);
4168 + busy = 1;
4169 + continue;
4170 + }
4171 + if (pa->pa_deleted) {
4172 + spin_unlock(&pa->pa_lock);
4173 + continue;
4174 + }
4175 +
4176 + /* seems this one can be freed ... */
4177 + pa->pa_deleted = 1;
4178 +
4179 + /* we can trust pa_free ... */
4180 + free += pa->pa_free;
4181 +
4182 + spin_unlock(&pa->pa_lock);
4183 +
4184 + list_del_rcu(&pa->pa_group_list);
4185 + list_add(&pa->u.pa_tmp_list, &list);
4186 + }
4187 +
4188 + /* if we still need more blocks and some PAs were used, try again */
4189 + if (free < needed && busy) {
4190 + busy = 0;
4191 + ext4_unlock_group(sb, group);
4192 + /*
4193 + * Yield the CPU here so that we don't get soft lockup
4194 + * in non preempt case.
4195 + */
4196 + yield();
4197 + goto repeat;
4198 + }
4199 +
4200 + /* found anything to free? */
4201 + if (list_empty(&list)) {
4202 + BUG_ON(free != 0);
4203 + goto out;
4204 + }
4205 +
4206 + /* now free all selected PAs */
4207 + list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) {
4208 +
4209 + /* remove from object (inode or locality group) */
4210 + spin_lock(pa->pa_obj_lock);
4211 + list_del_rcu(&pa->pa_inode_list);
4212 + spin_unlock(pa->pa_obj_lock);
4213 +
4214 + if (pa->pa_linear)
4215 + ext4_mb_release_group_pa(&e4b, pa);
4216 + else
4217 + ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa);
4218 +
4219 + list_del(&pa->u.pa_tmp_list);
4220 + mb_call_rcu(pa);
4221 + }
4222 +
4223 +out:
4224 + ext4_unlock_group(sb, group);
4225 + ext4_mb_release_desc(&e4b);
4226 + brelse(bitmap_bh);
4227 + return free;
4228 +}
4229 +
4230 +/*
4231 + * releases all non-used preallocated blocks for given inode
4232 + *
4233 + * It's important to discard preallocations under i_data_sem
4234 + * We don't want another block to be served from the prealloc
4235 + * space when we are discarding the inode prealloc space.
4236 + *
4237 + * FIXME!! Make sure it is valid at all the call sites
4238 + */
4239 +void ext4_mb_discard_inode_preallocations(struct inode *inode)
4240 +{
4241 + struct ext4_inode_info *ei = EXT4_I(inode);
4242 + struct super_block *sb = inode->i_sb;
4243 + struct buffer_head *bitmap_bh = NULL;
4244 + struct ext4_prealloc_space *pa, *tmp;
4245 + ext4_group_t group = 0;
4246 + struct list_head list;
4247 + struct ext4_buddy e4b;
4248 + int err;
4249 +
4250 + if (!test_opt(sb, MBALLOC) || !S_ISREG(inode->i_mode)) {
4251 + /*BUG_ON(!list_empty(&ei->i_prealloc_list));*/
4252 + return;
4253 + }
4254 +
4255 + mb_debug("discard preallocation for inode %lu\n", inode->i_ino);
4256 +
4257 + INIT_LIST_HEAD(&list);
4258 +
4259 +repeat:
4260 + /* first, collect all pa's in the inode */
4261 + spin_lock(&ei->i_prealloc_lock);
4262 + while (!list_empty(&ei->i_prealloc_list)) {
4263 + pa = list_entry(ei->i_prealloc_list.next,
4264 + struct ext4_prealloc_space, pa_inode_list);
4265 + BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock);
4266 + spin_lock(&pa->pa_lock);
4267 + if (atomic_read(&pa->pa_count)) {
4268 + /* this shouldn't happen often - nobody should
4269 + * use preallocation while we're discarding it */
4270 + spin_unlock(&pa->pa_lock);
4271 + spin_unlock(&ei->i_prealloc_lock);
4272 + printk(KERN_ERR "uh-oh! used pa while discarding\n");
4273 + dump_stack();
4274 + current->state = TASK_UNINTERRUPTIBLE;
4275 + schedule_timeout(HZ);
4276 + goto repeat;
4277 +
4278 + }
4279 + if (pa->pa_deleted == 0) {
4280 + pa->pa_deleted = 1;
4281 + spin_unlock(&pa->pa_lock);
4282 + list_del_rcu(&pa->pa_inode_list);
4283 + list_add(&pa->u.pa_tmp_list, &list);
4284 + continue;
4285 + }
4286 +
4287 + /* someone is deleting pa right now */
4288 + spin_unlock(&pa->pa_lock);
4289 + spin_unlock(&ei->i_prealloc_lock);
4290 +
4291 + /* we have to wait here because pa_deleted
4292 + * doesn't mean pa is already unlinked from
4293 + * the list. as we might be called from
4294 + * ->clear_inode() the inode will get freed
4295 + * and concurrent thread which is unlinking
4296 + * pa from inode's list may access already
4297 + * freed memory, bad-bad-bad */
4298 +
4299 + /* XXX: if this happens too often, we can
4300 + * add a flag to force wait only in case
4301 + * of ->clear_inode(), but not in case of
4302 + * regular truncate */
4303 + current->state = TASK_UNINTERRUPTIBLE;
4304 + schedule_timeout(HZ);
4305 + goto repeat;
4306 + }
4307 + spin_unlock(&ei->i_prealloc_lock);
4308 +
4309 + list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) {
4310 + BUG_ON(pa->pa_linear != 0);
4311 + ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, NULL);
4312 +
4313 + err = ext4_mb_load_buddy(sb, group, &e4b);
4314 + BUG_ON(err != 0); /* error handling here */
4315 +
4316 + bitmap_bh = read_block_bitmap(sb, group);
4317 + if (bitmap_bh == NULL) {
4318 + /* error handling here */
4319 + ext4_mb_release_desc(&e4b);
4320 + BUG_ON(bitmap_bh == NULL);
4321 + }
4322 +
4323 + ext4_lock_group(sb, group);
4324 + list_del_rcu(&pa->pa_group_list);
4325 + ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa);
4326 + ext4_unlock_group(sb, group);
4327 +
4328 + ext4_mb_release_desc(&e4b);
4329 + brelse(bitmap_bh);
4330 +
4331 + list_del(&pa->u.pa_tmp_list);
4332 + mb_call_rcu(pa);
4333 + }
4334 +}
4335 +
4336 +/*
4337 + * finds all preallocated spaces and return blocks being freed to them
4338 + * if preallocated space becomes full (no block is used from the space)
4339 + * then the function frees space in buddy
4340 + * XXX: at the moment, truncate (which is the only way to free blocks)
4341 + * discards all preallocations
4342 + */
4343 +static void ext4_mb_return_to_preallocation(struct inode *inode,
4344 + struct ext4_buddy *e4b,
4345 + sector_t block, int count)
4346 +{
4347 + BUG_ON(!list_empty(&EXT4_I(inode)->i_prealloc_list));
4348 +}
4349 +#ifdef MB_DEBUG
4350 +static void ext4_mb_show_ac(struct ext4_allocation_context *ac)
4351 +{
4352 + struct super_block *sb = ac->ac_sb;
4353 + ext4_group_t i;
4354 +
4355 + printk(KERN_ERR "EXT4-fs: Can't allocate:"
4356 + " Allocation context details:\n");
4357 + printk(KERN_ERR "EXT4-fs: status %d flags %d\n",
4358 + ac->ac_status, ac->ac_flags);
4359 + printk(KERN_ERR "EXT4-fs: orig %lu/%lu/%lu@%lu, goal %lu/%lu/%lu@%lu, "
4360 + "best %lu/%lu/%lu@%lu cr %d\n",
4361 + (unsigned long)ac->ac_o_ex.fe_group,
4362 + (unsigned long)ac->ac_o_ex.fe_start,
4363 + (unsigned long)ac->ac_o_ex.fe_len,
4364 + (unsigned long)ac->ac_o_ex.fe_logical,
4365 + (unsigned long)ac->ac_g_ex.fe_group,
4366 + (unsigned long)ac->ac_g_ex.fe_start,
4367 + (unsigned long)ac->ac_g_ex.fe_len,
4368 + (unsigned long)ac->ac_g_ex.fe_logical,
4369 + (unsigned long)ac->ac_b_ex.fe_group,
4370 + (unsigned long)ac->ac_b_ex.fe_start,
4371 + (unsigned long)ac->ac_b_ex.fe_len,
4372 + (unsigned long)ac->ac_b_ex.fe_logical,
4373 + (int)ac->ac_criteria);
4374 + printk(KERN_ERR "EXT4-fs: %lu scanned, %d found\n", ac->ac_ex_scanned,
4375 + ac->ac_found);
4376 + printk(KERN_ERR "EXT4-fs: groups: \n");
4377 + for (i = 0; i < EXT4_SB(sb)->s_groups_count; i++) {
4378 + struct ext4_group_info *grp = ext4_get_group_info(sb, i);
4379 + struct ext4_prealloc_space *pa;
4380 + ext4_grpblk_t start;
4381 + struct list_head *cur;
4382 + list_for_each_rcu(cur, &grp->bb_prealloc_list) {
4383 + pa = list_entry(cur, struct ext4_prealloc_space,
4384 + pa_group_list);
4385 + spin_lock(&pa->pa_lock);
4386 + ext4_get_group_no_and_offset(sb, pa->pa_pstart,
4387 + NULL, &start);
4388 + spin_unlock(&pa->pa_lock);
4389 + printk(KERN_ERR "PA:%lu:%d:%u \n", i,
4390 + start, pa->pa_len);
4391 + }
4392 +
4393 + if (grp->bb_free == 0)
4394 + continue;
4395 + printk(KERN_ERR "%lu: %d/%d \n",
4396 + i, grp->bb_free, grp->bb_fragments);
4397 + }
4398 + printk(KERN_ERR "\n");
4399 +}
4400 +#else
4401 +#define ext4_mb_show_ac(x)
4402 +#endif
4403 +
4404 +/*
4405 + * We use locality group preallocation for small size file. The size of the
4406 + * file is determined by the current size or the resulting size after
4407 + * allocation which ever is larger
4408 + *
4409 + * One can tune this size via /proc/fs/ext4/<partition>/stream_req
4410 + */
4411 +static void ext4_mb_group_or_file(struct ext4_allocation_context *ac)
4412 +{
4413 + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
4414 + int bsbits = ac->ac_sb->s_blocksize_bits;
4415 + loff_t size, isize;
4416 +
4417 + if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
4418 + return;
4419 +
4420 + size = ac->ac_o_ex.fe_logical + ac->ac_o_ex.fe_len;
4421 + isize = i_size_read(ac->ac_inode) >> bsbits;
4422 + if (size < isize)
4423 + size = isize;
4424 +
4425 + /* don't use group allocation for large files */
4426 + if (size >= sbi->s_mb_stream_request)
4427 + return;
4428 +
4429 + if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
4430 + return;
4431 +
4432 + BUG_ON(ac->ac_lg != NULL);
4433 + ac->ac_lg = &sbi->s_locality_groups[get_cpu()];
4434 + put_cpu();
4435 +
4436 + /* we're going to use group allocation */
4437 + ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC;
4438 +
4439 + /* serialize all allocations in the group */
4440 + down(&ac->ac_lg->lg_sem);
4441 +}
4442 +
4443 +static int ext4_mb_initialize_context(struct ext4_allocation_context *ac,
4444 + struct ext4_allocation_request *ar)
4445 +{
4446 + struct super_block *sb = ar->inode->i_sb;
4447 + struct ext4_sb_info *sbi = EXT4_SB(sb);
4448 + struct ext4_super_block *es = sbi->s_es;
4449 + ext4_group_t group;
4450 + unsigned long len;
4451 + unsigned long goal;
4452 + ext4_grpblk_t block;
4453 +
4454 + /* we can't allocate > group size */
4455 + len = ar->len;
4456 +
4457 + /* just a dirty hack to filter too big requests */
4458 + if (len >= EXT4_BLOCKS_PER_GROUP(sb) - 10)
4459 + len = EXT4_BLOCKS_PER_GROUP(sb) - 10;
4460 +
4461 + /* start searching from the goal */
4462 + goal = ar->goal;
4463 + if (goal < le32_to_cpu(es->s_first_data_block) ||
4464 + goal >= ext4_blocks_count(es))
4465 + goal = le32_to_cpu(es->s_first_data_block);
4466 + ext4_get_group_no_and_offset(sb, goal, &group, &block);
4467 +
4468 + /* set up allocation goals */
4469 + ac->ac_b_ex.fe_logical = ar->logical;
4470 + ac->ac_b_ex.fe_group = 0;
4471 + ac->ac_b_ex.fe_start = 0;
4472 + ac->ac_b_ex.fe_len = 0;
4473 + ac->ac_status = AC_STATUS_CONTINUE;
4474 + ac->ac_groups_scanned = 0;
4475 + ac->ac_ex_scanned = 0;
4476 + ac->ac_found = 0;
4477 + ac->ac_sb = sb;
4478 + ac->ac_inode = ar->inode;
4479 + ac->ac_o_ex.fe_logical = ar->logical;
4480 + ac->ac_o_ex.fe_group = group;
4481 + ac->ac_o_ex.fe_start = block;
4482 + ac->ac_o_ex.fe_len = len;
4483 + ac->ac_g_ex.fe_logical = ar->logical;
4484 + ac->ac_g_ex.fe_group = group;
4485 + ac->ac_g_ex.fe_start = block;
4486 + ac->ac_g_ex.fe_len = len;
4487 + ac->ac_f_ex.fe_len = 0;
4488 + ac->ac_flags = ar->flags;
4489 + ac->ac_2order = 0;
4490 + ac->ac_criteria = 0;
4491 + ac->ac_pa = NULL;
4492 + ac->ac_bitmap_page = NULL;
4493 + ac->ac_buddy_page = NULL;
4494 + ac->ac_lg = NULL;
4495 +
4496 + /* we have to define context: we'll we work with a file or
4497 + * locality group. this is a policy, actually */
4498 + ext4_mb_group_or_file(ac);
4499 +
4500 + mb_debug("init ac: %u blocks @ %u, goal %u, flags %x, 2^%d, "
4501 + "left: %u/%u, right %u/%u to %swritable\n",
4502 + (unsigned) ar->len, (unsigned) ar->logical,
4503 + (unsigned) ar->goal, ac->ac_flags, ac->ac_2order,
4504 + (unsigned) ar->lleft, (unsigned) ar->pleft,
4505 + (unsigned) ar->lright, (unsigned) ar->pright,
4506 + atomic_read(&ar->inode->i_writecount) ? "" : "non-");
4507 + return 0;
4508 +
4509 +}
4510 +
4511 +/*
4512 + * release all resource we used in allocation
4513 + */
4514 +static int ext4_mb_release_context(struct ext4_allocation_context *ac)
4515 +{
4516 + if (ac->ac_pa) {
4517 + if (ac->ac_pa->pa_linear) {
4518 + /* see comment in ext4_mb_use_group_pa() */
4519 + spin_lock(&ac->ac_pa->pa_lock);
4520 + ac->ac_pa->pa_pstart += ac->ac_b_ex.fe_len;
4521 + ac->ac_pa->pa_lstart += ac->ac_b_ex.fe_len;
4522 + ac->ac_pa->pa_free -= ac->ac_b_ex.fe_len;
4523 + ac->ac_pa->pa_len -= ac->ac_b_ex.fe_len;
4524 + spin_unlock(&ac->ac_pa->pa_lock);
4525 + }
4526 + ext4_mb_put_pa(ac, ac->ac_sb, ac->ac_pa);
4527 + }
4528 + if (ac->ac_bitmap_page)
4529 + page_cache_release(ac->ac_bitmap_page);
4530 + if (ac->ac_buddy_page)
4531 + page_cache_release(ac->ac_buddy_page);
4532 + if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)
4533 + up(&ac->ac_lg->lg_sem);
4534 + ext4_mb_collect_stats(ac);
4535 + return 0;
4536 +}
4537 +
4538 +static int ext4_mb_discard_preallocations(struct super_block *sb, int needed)
4539 +{
4540 + ext4_group_t i;
4541 + int ret;
4542 + int freed = 0;
4543 +
4544 + for (i = 0; i < EXT4_SB(sb)->s_groups_count && needed > 0; i++) {
4545 + ret = ext4_mb_discard_group_preallocations(sb, i, needed);
4546 + freed += ret;
4547 + needed -= ret;
4548 + }
4549 +
4550 + return freed;
4551 +}
4552 +
4553 +/*
4554 + * Main entry point into mballoc to allocate blocks
4555 + * it tries to use preallocation first, then falls back
4556 + * to usual allocation
4557 + */
4558 +ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle,
4559 + struct ext4_allocation_request *ar, int *errp)
4560 +{
4561 + struct ext4_allocation_context ac;
4562 + struct ext4_sb_info *sbi;
4563 + struct super_block *sb;
4564 + ext4_fsblk_t block = 0;
4565 + int freed;
4566 + int inquota;
4567 +
4568 + sb = ar->inode->i_sb;
4569 + sbi = EXT4_SB(sb);
4570 +
4571 + if (!test_opt(sb, MBALLOC)) {
4572 + block = ext4_new_blocks_old(handle, ar->inode, ar->goal,
4573 + &(ar->len), errp);
4574 + return block;
4575 + }
4576 +
4577 + while (ar->len && DQUOT_ALLOC_BLOCK(ar->inode, ar->len)) {
4578 + ar->flags |= EXT4_MB_HINT_NOPREALLOC;
4579 + ar->len--;
4580 + }
4581 + if (ar->len == 0) {
4582 + *errp = -EDQUOT;
4583 + return 0;
4584 + }
4585 + inquota = ar->len;
4586 +
4587 + ext4_mb_poll_new_transaction(sb, handle);
4588 +
4589 + *errp = ext4_mb_initialize_context(&ac, ar);
4590 + if (*errp) {
4591 + ar->len = 0;
4592 + goto out;
4593 + }
4594 +
4595 + ac.ac_op = EXT4_MB_HISTORY_PREALLOC;
4596 + if (!ext4_mb_use_preallocated(&ac)) {
4597 +
4598 + ac.ac_op = EXT4_MB_HISTORY_ALLOC;
4599 + ext4_mb_normalize_request(&ac, ar);
4600 +
4601 +repeat:
4602 + /* allocate space in core */
4603 + ext4_mb_regular_allocator(&ac);
4604 +
4605 + /* as we've just preallocated more space than
4606 + * user requested orinally, we store allocated
4607 + * space in a special descriptor */
4608 + if (ac.ac_status == AC_STATUS_FOUND &&
4609 + ac.ac_o_ex.fe_len < ac.ac_b_ex.fe_len)
4610 + ext4_mb_new_preallocation(&ac);
4611 + }
4612 +
4613 + if (likely(ac.ac_status == AC_STATUS_FOUND)) {
4614 + ext4_mb_mark_diskspace_used(&ac, handle);
4615 + *errp = 0;
4616 + block = ext4_grp_offs_to_block(sb, &ac.ac_b_ex);
4617 + ar->len = ac.ac_b_ex.fe_len;
4618 + } else {
4619 + freed = ext4_mb_discard_preallocations(sb, ac.ac_o_ex.fe_len);
4620 + if (freed)
4621 + goto repeat;
4622 + *errp = -ENOSPC;
4623 + ac.ac_b_ex.fe_len = 0;
4624 + ar->len = 0;
4625 + ext4_mb_show_ac(&ac);
4626 + }
4627 +
4628 + ext4_mb_release_context(&ac);
4629 +
4630 +out:
4631 + if (ar->len < inquota)
4632 + DQUOT_FREE_BLOCK(ar->inode, inquota - ar->len);
4633 +
4634 + return block;
4635 +}
4636 +static void ext4_mb_poll_new_transaction(struct super_block *sb,
4637 + handle_t *handle)
4638 +{
4639 + struct ext4_sb_info *sbi = EXT4_SB(sb);
4640 +
4641 + if (sbi->s_last_transaction == handle->h_transaction->t_tid)
4642 + return;
4643 +
4644 + /* new transaction! time to close last one and free blocks for
4645 + * committed transaction. we know that only transaction can be
4646 + * active, so previos transaction can be being logged and we
4647 + * know that transaction before previous is known to be already
4648 + * logged. this means that now we may free blocks freed in all
4649 + * transactions before previous one. hope I'm clear enough ... */
4650 +
4651 + spin_lock(&sbi->s_md_lock);
4652 + if (sbi->s_last_transaction != handle->h_transaction->t_tid) {
4653 + mb_debug("new transaction %lu, old %lu\n",
4654 + (unsigned long) handle->h_transaction->t_tid,
4655 + (unsigned long) sbi->s_last_transaction);
4656 + list_splice_init(&sbi->s_closed_transaction,
4657 + &sbi->s_committed_transaction);
4658 + list_splice_init(&sbi->s_active_transaction,
4659 + &sbi->s_closed_transaction);
4660 + sbi->s_last_transaction = handle->h_transaction->t_tid;
4661 + }
4662 + spin_unlock(&sbi->s_md_lock);
4663 +
4664 + ext4_mb_free_committed_blocks(sb);
4665 +}
4666 +
4667 +static int ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b,
4668 + ext4_group_t group, ext4_grpblk_t block, int count)
4669 +{
4670 + struct ext4_group_info *db = e4b->bd_info;
4671 + struct super_block *sb = e4b->bd_sb;
4672 + struct ext4_sb_info *sbi = EXT4_SB(sb);
4673 + struct ext4_free_metadata *md;
4674 + int i;
4675 +
4676 + BUG_ON(e4b->bd_bitmap_page == NULL);
4677 + BUG_ON(e4b->bd_buddy_page == NULL);
4678 +
4679 + ext4_lock_group(sb, group);
4680 + for (i = 0; i < count; i++) {
4681 + md = db->bb_md_cur;
4682 + if (md && db->bb_tid != handle->h_transaction->t_tid) {
4683 + db->bb_md_cur = NULL;
4684 + md = NULL;
4685 + }
4686 +
4687 + if (md == NULL) {
4688 + ext4_unlock_group(sb, group);
4689 + md = kmalloc(sizeof(*md), GFP_KERNEL);
4690 + if (md == NULL)
4691 + return -ENOMEM;
4692 + md->num = 0;
4693 + md->group = group;
4694 +
4695 + ext4_lock_group(sb, group);
4696 + if (db->bb_md_cur == NULL) {
4697 + spin_lock(&sbi->s_md_lock);
4698 + list_add(&md->list, &sbi->s_active_transaction);
4699 + spin_unlock(&sbi->s_md_lock);
4700 + /* protect buddy cache from being freed,
4701 + * otherwise we'll refresh it from
4702 + * on-disk bitmap and lose not-yet-available
4703 + * blocks */
4704 + page_cache_get(e4b->bd_buddy_page);
4705 + page_cache_get(e4b->bd_bitmap_page);
4706 + db->bb_md_cur = md;
4707 + db->bb_tid = handle->h_transaction->t_tid;
4708 + mb_debug("new md 0x%p for group %lu\n",
4709 + md, md->group);
4710 + } else {
4711 + kfree(md);
4712 + md = db->bb_md_cur;
4713 + }
4714 + }
4715 +
4716 + BUG_ON(md->num >= EXT4_BB_MAX_BLOCKS);
4717 + md->blocks[md->num] = block + i;
4718 + md->num++;
4719 + if (md->num == EXT4_BB_MAX_BLOCKS) {
4720 + /* no more space, put full container on a sb's list */
4721 + db->bb_md_cur = NULL;
4722 + }
4723 + }
4724 + ext4_unlock_group(sb, group);
4725 + return 0;
4726 +}
4727 +
4728 +/*
4729 + * Main entry point into mballoc to free blocks
4730 + */
4731 +void ext4_mb_free_blocks(handle_t *handle, struct inode *inode,
4732 + unsigned long block, unsigned long count,
4733 + int metadata, unsigned long *freed)
4734 +{
4735 + struct buffer_head *bitmap_bh = NULL;
4736 + struct super_block *sb = inode->i_sb;
4737 + struct ext4_allocation_context ac;
4738 + struct ext4_group_desc *gdp;
4739 + struct ext4_super_block *es;
4740 + unsigned long overflow;
4741 + ext4_grpblk_t bit;
4742 + struct buffer_head *gd_bh;
4743 + ext4_group_t block_group;
4744 + struct ext4_sb_info *sbi;
4745 + struct ext4_buddy e4b;
4746 + int err = 0;
4747 + int ret;
4748 +
4749 + *freed = 0;
4750 +
4751 + ext4_mb_poll_new_transaction(sb, handle);
4752 +
4753 + sbi = EXT4_SB(sb);
4754 + es = EXT4_SB(sb)->s_es;
4755 + if (block < le32_to_cpu(es->s_first_data_block) ||
4756 + block + count < block ||
4757 + block + count > ext4_blocks_count(es)) {
4758 + ext4_error(sb, __FUNCTION__,
4759 + "Freeing blocks not in datazone - "
4760 + "block = %lu, count = %lu", block, count);
4761 + goto error_return;
4762 + }
4763 +
4764 + ext4_debug("freeing block %lu\n", block);
4765 +
4766 + ac.ac_op = EXT4_MB_HISTORY_FREE;
4767 + ac.ac_inode = inode;
4768 + ac.ac_sb = sb;
4769 +
4770 +do_more:
4771 + overflow = 0;
4772 + ext4_get_group_no_and_offset(sb, block, &block_group, &bit);
4773 +
4774 + /*
4775 + * Check to see if we are freeing blocks across a group
4776 + * boundary.
4777 + */
4778 + if (bit + count > EXT4_BLOCKS_PER_GROUP(sb)) {
4779 + overflow = bit + count - EXT4_BLOCKS_PER_GROUP(sb);
4780 + count -= overflow;
4781 + }
4782 + brelse(bitmap_bh);
4783 + bitmap_bh = read_block_bitmap(sb, block_group);
4784 + if (!bitmap_bh)
4785 + goto error_return;
4786 + gdp = ext4_get_group_desc(sb, block_group, &gd_bh);
4787 + if (!gdp)
4788 + goto error_return;
4789 +
4790 + if (in_range(ext4_block_bitmap(sb, gdp), block, count) ||
4791 + in_range(ext4_inode_bitmap(sb, gdp), block, count) ||
4792 + in_range(block, ext4_inode_table(sb, gdp),
4793 + EXT4_SB(sb)->s_itb_per_group) ||
4794 + in_range(block + count - 1, ext4_inode_table(sb, gdp),
4795 + EXT4_SB(sb)->s_itb_per_group)) {
4796 +
4797 + ext4_error(sb, __FUNCTION__,
4798 + "Freeing blocks in system zone - "
4799 + "Block = %lu, count = %lu", block, count);
4800 + }
4801 +
4802 + BUFFER_TRACE(bitmap_bh, "getting write access");
4803 + err = ext4_journal_get_write_access(handle, bitmap_bh);
4804 + if (err)
4805 + goto error_return;
4806 +
4807 + /*
4808 + * We are about to modify some metadata. Call the journal APIs
4809 + * to unshare ->b_data if a currently-committing transaction is
4810 + * using it
4811 + */
4812 + BUFFER_TRACE(gd_bh, "get_write_access");
4813 + err = ext4_journal_get_write_access(handle, gd_bh);
4814 + if (err)
4815 + goto error_return;
4816 +
4817 + err = ext4_mb_load_buddy(sb, block_group, &e4b);
4818 + if (err)
4819 + goto error_return;
4820 +
4821 +#ifdef AGGRESSIVE_CHECK
4822 + {
4823 + int i;
4824 + for (i = 0; i < count; i++)
4825 + BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data));
4826 + }
4827 +#endif
4828 + mb_clear_bits(sb_bgl_lock(sbi, block_group), bitmap_bh->b_data,
4829 + bit, count);
4830 +
4831 + /* We dirtied the bitmap block */
4832 + BUFFER_TRACE(bitmap_bh, "dirtied bitmap block");
4833 + err = ext4_journal_dirty_metadata(handle, bitmap_bh);
4834 +
4835 + ac.ac_b_ex.fe_group = block_group;
4836 + ac.ac_b_ex.fe_start = bit;
4837 + ac.ac_b_ex.fe_len = count;
4838 + ext4_mb_store_history(&ac);
4839 +
4840 + if (metadata) {
4841 + /* blocks being freed are metadata. these blocks shouldn't
4842 + * be used until this transaction is committed */
4843 + ext4_mb_free_metadata(handle, &e4b, block_group, bit, count);
4844 + } else {
4845 + ext4_lock_group(sb, block_group);
4846 + err = mb_free_blocks(inode, &e4b, bit, count);
4847 + ext4_mb_return_to_preallocation(inode, &e4b, block, count);
4848 + ext4_unlock_group(sb, block_group);
4849 + BUG_ON(err != 0);
4850 + }
4851 +
4852 + spin_lock(sb_bgl_lock(sbi, block_group));
4853 + gdp->bg_free_blocks_count =
4854 + cpu_to_le16(le16_to_cpu(gdp->bg_free_blocks_count) + count);
4855 + gdp->bg_checksum = ext4_group_desc_csum(sbi, block_group, gdp);
4856 + spin_unlock(sb_bgl_lock(sbi, block_group));
4857 + percpu_counter_add(&sbi->s_freeblocks_counter, count);
4858 +
4859 + ext4_mb_release_desc(&e4b);
4860 +
4861 + *freed += count;
4862 +
4863 + /* And the group descriptor block */
4864 + BUFFER_TRACE(gd_bh, "dirtied group descriptor block");
4865 + ret = ext4_journal_dirty_metadata(handle, gd_bh);
4866 + if (!err)
4867 + err = ret;
4868 +
4869 + if (overflow && !err) {
4870 + block += count;
4871 + count = overflow;
4872 + goto do_more;
4873 + }
4874 + sb->s_dirt = 1;
4875 +error_return:
4876 + brelse(bitmap_bh);
4877 + ext4_std_error(sb, err);
4878 + return;
4879 +}
4880 diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c
4881 index 7203d3d..6b40f55 100644
4882 --- a/fs/ext4/migrate.c
4883 +++ b/fs/ext4/migrate.c
4884 @@ -253,11 +253,11 @@ static int free_dind_blocks(handle_t *handle,
4885 for (i = 0; i < max_entries; i++) {
4886 if (tmp_idata[i]) {
4887 ext4_free_blocks(handle, inode,
4888 - le32_to_cpu(tmp_idata[i]), 1);
4889 + le32_to_cpu(tmp_idata[i]), 1, 1);
4890 }
4891 }
4892 brelse(bh);
4893 - ext4_free_blocks(handle, inode, le32_to_cpu(i_data), 1);
4894 + ext4_free_blocks(handle, inode, le32_to_cpu(i_data), 1, 1);
4896 return 0;
4898 @@ -289,7 +289,7 @@ static int free_tind_blocks(handle_t *handle,
4899 }
4900 }
4901 brelse(bh);
4902 - ext4_free_blocks(handle, inode, le32_to_cpu(i_data), 1);
4903 + ext4_free_blocks(handle, inode, le32_to_cpu(i_data), 1, 1);
4905 return 0;
4907 @@ -304,7 +304,7 @@ static int free_ind_block(handle_t *handle, struct inode *inode)
4908 if (ei->i_data[EXT4_IND_BLOCK]) {
4910 ext4_free_blocks(handle, inode,
4911 - le32_to_cpu(ei->i_data[EXT4_IND_BLOCK]), 1);
4912 + le32_to_cpu(ei->i_data[EXT4_IND_BLOCK]), 1, 1);
4914 }
4916 @@ -402,7 +402,7 @@ static int free_ext_idx(handle_t *handle, struct inode *inode,
4917 if (eh->eh_depth == 0) {
4919 brelse(bh);
4920 - ext4_free_blocks(handle, inode, block, 1);
4921 + ext4_free_blocks(handle, inode, block, 1, 1);
4923 } else {
4925 diff --git a/fs/ext4/super.c b/fs/ext4/super.c
4926 index 64fc7f1..136d095 100644
4927 --- a/fs/ext4/super.c
4928 +++ b/fs/ext4/super.c
4929 @@ -503,6 +503,7 @@ static void ext4_put_super (struct super_block * sb)
4930 struct ext4_super_block *es = sbi->s_es;
4931 int i;
4933 + ext4_mb_release(sb);
4934 ext4_ext_release(sb);
4935 ext4_xattr_put_super(sb);
4936 jbd2_journal_destroy(sbi->s_journal);
4937 @@ -569,6 +570,8 @@ static struct inode *ext4_alloc_inode(struct super_block *sb)
4938 ei->i_block_alloc_info = NULL;
4939 ei->vfs_inode.i_version = 1;
4940 memset(&ei->i_cached_extent, 0, sizeof(struct ext4_ext_cache));
4941 + INIT_LIST_HEAD(&ei->i_prealloc_list);
4942 + spin_lock_init(&ei->i_prealloc_lock);
4943 return &ei->vfs_inode;
4944 }
4946 @@ -881,6 +884,7 @@ enum {
4947 Opt_jqfmt_vfsold, Opt_jqfmt_vfsv0, Opt_quota, Opt_noquota,
4948 Opt_ignore, Opt_barrier, Opt_err, Opt_resize, Opt_usrquota,
4949 Opt_grpquota, Opt_extents, Opt_noextents, Opt_i_version,
4950 + Opt_mballoc, Opt_nomballoc, Opt_stripe,
4951 };
4953 static match_table_t tokens = {
4954 @@ -935,6 +939,9 @@ static match_table_t tokens = {
4955 {Opt_extents, "extents"},
4956 {Opt_noextents, "noextents"},
4957 {Opt_i_version, "i_version"},
4958 + {Opt_mballoc, "mballoc"},
4959 + {Opt_nomballoc, "nomballoc"},
4960 + {Opt_stripe, "stripe=%u"},
4961 {Opt_err, NULL},
4962 {Opt_resize, "resize"},
4963 };
4964 @@ -1284,6 +1291,19 @@ clear_qf_name:
4965 set_opt(sbi->s_mount_opt, I_VERSION);
4966 sb->s_flags |= MS_I_VERSION;
4967 break;
4968 + case Opt_mballoc:
4969 + set_opt(sbi->s_mount_opt, MBALLOC);
4970 + break;
4971 + case Opt_nomballoc:
4972 + clear_opt(sbi->s_mount_opt, MBALLOC);
4973 + break;
4974 + case Opt_stripe:
4975 + if (match_int(&args[0], &option))
4976 + return 0;
4977 + if (option < 0)
4978 + return 0;
4979 + sbi->s_stripe = option;
4980 + break;
4981 default:
4982 printk (KERN_ERR
4983 "EXT4-fs: Unrecognized mount option \"%s\" "
4984 @@ -1742,6 +1762,33 @@ static ext4_fsblk_t descriptor_loc(struct super_block *sb,
4985 return (has_super + ext4_group_first_block_no(sb, bg));
4986 }
4988 +/**
4989 + * ext4_get_stripe_size: Get the stripe size.
4990 + * @sbi: In memory super block info
4991 + *
4992 + * If we have specified it via mount option, then
4993 + * use the mount option value. If the value specified at mount time is
4994 + * greater than the blocks per group use the super block value.
4995 + * If the super block value is greater than blocks per group return 0.
4996 + * Allocator needs it be less than blocks per group.
4997 + *
4998 + */
4999 +static unsigned long ext4_get_stripe_size(struct ext4_sb_info *sbi)
5000 +{
5001 + unsigned long stride = le16_to_cpu(sbi->s_es->s_raid_stride);
5002 + unsigned long stripe_width =
5003 + le32_to_cpu(sbi->s_es->s_raid_stripe_width);
5004 +
5005 + if (sbi->s_stripe && sbi->s_stripe <= sbi->s_blocks_per_group) {
5006 + return sbi->s_stripe;
5007 + } else if (stripe_width <= sbi->s_blocks_per_group) {
5008 + return stripe_width;
5009 + } else if (stride <= sbi->s_blocks_per_group) {
5010 + return stride;
5011 + }
5012 +
5013 + return 0;
5014 +}
5016 static int ext4_fill_super (struct super_block *sb, void *data, int silent)
5017 __releases(kernel_sem)
5018 @@ -2091,6 +2138,8 @@ static int ext4_fill_super (struct super_block *sb, void *data, int silent)
5019 sbi->s_rsv_window_head.rsv_goal_size = 0;
5020 ext4_rsv_window_add(sb, &sbi->s_rsv_window_head);
5022 + sbi->s_stripe = ext4_get_stripe_size(sbi);
5023 +
5024 /*
5025 * set up enough so that it can read an inode
5026 */
5027 @@ -2250,6 +2299,7 @@ static int ext4_fill_super (struct super_block *sb, void *data, int silent)
5028 "writeback");
5030 ext4_ext_init(sb);
5031 + ext4_mb_init(sb, needs_recovery);
5033 lock_kernel();
5034 return 0;
5035 @@ -3232,9 +3282,15 @@ static struct file_system_type ext4dev_fs_type = {
5037 static int __init init_ext4_fs(void)
5038 {
5039 - int err = init_ext4_xattr();
5040 + int err;
5041 +
5042 + err = init_ext4_mballoc();
5043 if (err)
5044 return err;
5045 +
5046 + err = init_ext4_xattr();
5047 + if (err)
5048 + goto out2;
5049 err = init_inodecache();
5050 if (err)
5051 goto out1;
5052 @@ -3246,6 +3302,8 @@ out:
5053 destroy_inodecache();
5054 out1:
5055 exit_ext4_xattr();
5056 +out2:
5057 + exit_ext4_mballoc();
5058 return err;
5059 }
5061 @@ -3254,6 +3312,7 @@ static void __exit exit_ext4_fs(void)
5062 unregister_filesystem(&ext4dev_fs_type);
5063 destroy_inodecache();
5064 exit_ext4_xattr();
5065 + exit_ext4_mballoc();
5066 }
5068 MODULE_AUTHOR("Remy Card, Stephen Tweedie, Andrew Morton, Andreas Dilger, Theodore Ts'o and others");
5069 diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c
5070 index 8638730..d796213 100644
5071 --- a/fs/ext4/xattr.c
5072 +++ b/fs/ext4/xattr.c
5073 @@ -480,7 +480,7 @@ ext4_xattr_release_block(handle_t *handle, struct inode *inode,
5074 ea_bdebug(bh, "refcount now=0; freeing");
5075 if (ce)
5076 mb_cache_entry_free(ce);
5077 - ext4_free_blocks(handle, inode, bh->b_blocknr, 1);
5078 + ext4_free_blocks(handle, inode, bh->b_blocknr, 1, 1);
5079 get_bh(bh);
5080 ext4_forget(handle, 1, inode, bh, bh->b_blocknr);
5081 } else {
5082 @@ -821,7 +821,7 @@ inserted:
5083 new_bh = sb_getblk(sb, block);
5084 if (!new_bh) {
5085 getblk_failed:
5086 - ext4_free_blocks(handle, inode, block, 1);
5087 + ext4_free_blocks(handle, inode, block, 1, 1);
5088 error = -EIO;
5089 goto cleanup;
5090 }
5091 diff --git a/include/linux/ext4_fs.h b/include/linux/ext4_fs.h
5092 index d0b7ca9..1852313 100644
5093 --- a/include/linux/ext4_fs.h
5094 +++ b/include/linux/ext4_fs.h
5095 @@ -20,6 +20,8 @@
5096 #include <linux/blkdev.h>
5097 #include <linux/magic.h>
5099 +#include <linux/ext4_fs_i.h>
5100 +
5101 /*
5102 * The second extended filesystem constants/structures
5103 */
5104 @@ -51,6 +53,50 @@
5105 #define ext4_debug(f, a...) do {} while (0)
5106 #endif
5108 +#define EXT4_MULTIBLOCK_ALLOCATOR 1
5109 +
5110 +/* prefer goal again. length */
5111 +#define EXT4_MB_HINT_MERGE 1
5112 +/* blocks already reserved */
5113 +#define EXT4_MB_HINT_RESERVED 2
5114 +/* metadata is being allocated */
5115 +#define EXT4_MB_HINT_METADATA 4
5116 +/* first blocks in the file */
5117 +#define EXT4_MB_HINT_FIRST 8
5118 +/* search for the best chunk */
5119 +#define EXT4_MB_HINT_BEST 16
5120 +/* data is being allocated */
5121 +#define EXT4_MB_HINT_DATA 32
5122 +/* don't preallocate (for tails) */
5123 +#define EXT4_MB_HINT_NOPREALLOC 64
5124 +/* allocate for locality group */
5125 +#define EXT4_MB_HINT_GROUP_ALLOC 128
5126 +/* allocate goal blocks or none */
5127 +#define EXT4_MB_HINT_GOAL_ONLY 256
5128 +/* goal is meaningful */
5129 +#define EXT4_MB_HINT_TRY_GOAL 512
5130 +
5131 +struct ext4_allocation_request {
5132 + /* target inode for block we're allocating */
5133 + struct inode *inode;
5134 + /* logical block in target inode */
5135 + ext4_lblk_t logical;
5136 + /* phys. target (a hint) */
5137 + ext4_fsblk_t goal;
5138 + /* the closest logical allocated block to the left */
5139 + ext4_lblk_t lleft;
5140 + /* phys. block for ^^^ */
5141 + ext4_fsblk_t pleft;
5142 + /* the closest logical allocated block to the right */
5143 + ext4_lblk_t lright;
5144 + /* phys. block for ^^^ */
5145 + ext4_fsblk_t pright;
5146 + /* how many blocks we want to allocate */
5147 + unsigned long len;
5148 + /* flags. see above EXT4_MB_HINT_* */
5149 + unsigned long flags;
5150 +};
5151 +
5152 /*
5153 * Special inodes numbers
5154 */
5155 @@ -474,6 +520,7 @@ do { \
5156 #define EXT4_MOUNT_JOURNAL_CHECKSUM 0x800000 /* Journal checksums */
5157 #define EXT4_MOUNT_JOURNAL_ASYNC_COMMIT 0x1000000 /* Journal Async Commit */
5158 #define EXT4_MOUNT_I_VERSION 0x2000000 /* i_version support */
5159 +#define EXT4_MOUNT_MBALLOC 0x4000000 /* Buddy allocation support */
5160 /* Compatibility, for having both ext2_fs.h and ext4_fs.h included at once */
5161 #ifndef _LINUX_EXT2_FS_H
5162 #define clear_opt(o, opt) o &= ~EXT4_MOUNT_##opt
5163 @@ -912,7 +959,7 @@ extern ext4_fsblk_t ext4_new_blocks (handle_t *handle, struct inode *inode,
5164 extern ext4_fsblk_t ext4_new_blocks_old(handle_t *handle, struct inode *inode,
5165 ext4_fsblk_t goal, unsigned long *count, int *errp);
5166 extern void ext4_free_blocks (handle_t *handle, struct inode *inode,
5167 - ext4_fsblk_t block, unsigned long count);
5168 + ext4_fsblk_t block, unsigned long count, int metadata);
5169 extern void ext4_free_blocks_sb (handle_t *handle, struct super_block *sb,
5170 ext4_fsblk_t block, unsigned long count,
5171 unsigned long *pdquot_freed_blocks);
5172 @@ -950,6 +997,20 @@ extern unsigned long ext4_count_dirs (struct super_block *);
5173 extern void ext4_check_inodes_bitmap (struct super_block *);
5174 extern unsigned long ext4_count_free (struct buffer_head *, unsigned);
5176 +/* mballoc.c */
5177 +extern long ext4_mb_stats;
5178 +extern long ext4_mb_max_to_scan;
5179 +extern int ext4_mb_init(struct super_block *, int);
5180 +extern int ext4_mb_release(struct super_block *);
5181 +extern ext4_fsblk_t ext4_mb_new_blocks(handle_t *,
5182 + struct ext4_allocation_request *, int *);
5183 +extern int ext4_mb_reserve_blocks(struct super_block *, int);
5184 +extern void ext4_mb_discard_inode_preallocations(struct inode *);
5185 +extern int __init init_ext4_mballoc(void);
5186 +extern void exit_ext4_mballoc(void);
5187 +extern void ext4_mb_free_blocks(handle_t *, struct inode *,
5188 + unsigned long, unsigned long, int, unsigned long *);
5189 +
5191 /* inode.c */
5192 int ext4_forget(handle_t *handle, int is_metadata, struct inode *inode,
5193 @@ -1080,6 +1141,19 @@ static inline void ext4_isize_set(struct ext4_inode *raw_inode, loff_t i_size)
5194 raw_inode->i_size_high = cpu_to_le32(i_size >> 32);
5195 }
5197 +static inline
5198 +struct ext4_group_info *ext4_get_group_info(struct super_block *sb,
5199 + ext4_group_t group)
5200 +{
5201 + struct ext4_group_info ***grp_info;
5202 + long indexv, indexh;
5203 + grp_info = EXT4_SB(sb)->s_group_info;
5204 + indexv = group >> (EXT4_DESC_PER_BLOCK_BITS(sb));
5205 + indexh = group & ((EXT4_DESC_PER_BLOCK(sb)) - 1);
5206 + return grp_info[indexv][indexh];
5207 +}
5208 +
5209 +
5210 #define ext4_std_error(sb, errno) \
5211 do { \
5212 if ((errno)) \
5213 diff --git a/include/linux/ext4_fs_i.h b/include/linux/ext4_fs_i.h
5214 index 4377d24..d5508d3 100644
5215 --- a/include/linux/ext4_fs_i.h
5216 +++ b/include/linux/ext4_fs_i.h
5217 @@ -158,6 +158,10 @@ struct ext4_inode_info {
5218 * struct timespec i_{a,c,m}time in the generic inode.
5219 */
5220 struct timespec i_crtime;
5221 +
5222 + /* mballoc */
5223 + struct list_head i_prealloc_list;
5224 + spinlock_t i_prealloc_lock;
5225 };
5227 #endif /* _LINUX_EXT4_FS_I */
5228 diff --git a/include/linux/ext4_fs_sb.h b/include/linux/ext4_fs_sb.h
5229 index 38a47ec..abaae2c 100644
5230 --- a/include/linux/ext4_fs_sb.h
5231 +++ b/include/linux/ext4_fs_sb.h
5232 @@ -91,6 +91,58 @@ struct ext4_sb_info {
5233 unsigned long s_ext_blocks;
5234 unsigned long s_ext_extents;
5235 #endif
5236 +
5237 + /* for buddy allocator */
5238 + struct ext4_group_info ***s_group_info;
5239 + struct inode *s_buddy_cache;
5240 + long s_blocks_reserved;
5241 + spinlock_t s_reserve_lock;
5242 + struct list_head s_active_transaction;
5243 + struct list_head s_closed_transaction;
5244 + struct list_head s_committed_transaction;
5245 + spinlock_t s_md_lock;
5246 + tid_t s_last_transaction;
5247 + unsigned short *s_mb_offsets, *s_mb_maxs;
5248 +
5249 + /* tunables */
5250 + unsigned long s_stripe;
5251 + unsigned long s_mb_stream_request;
5252 + unsigned long s_mb_max_to_scan;
5253 + unsigned long s_mb_min_to_scan;
5254 + unsigned long s_mb_stats;
5255 + unsigned long s_mb_order2_reqs;
5256 + unsigned long s_mb_group_prealloc;
5257 + /* where last allocation was done - for stream allocation */
5258 + unsigned long s_mb_last_group;
5259 + unsigned long s_mb_last_start;
5260 +
5261 + /* history to debug policy */
5262 + struct ext4_mb_history *s_mb_history;
5263 + int s_mb_history_cur;
5264 + int s_mb_history_max;
5265 + int s_mb_history_num;
5266 + struct proc_dir_entry *s_mb_proc;
5267 + spinlock_t s_mb_history_lock;
5268 + int s_mb_history_filter;
5269 +
5270 + /* stats for buddy allocator */
5271 + spinlock_t s_mb_pa_lock;
5272 + atomic_t s_bal_reqs; /* number of reqs with len > 1 */
5273 + atomic_t s_bal_success; /* we found long enough chunks */
5274 + atomic_t s_bal_allocated; /* in blocks */
5275 + atomic_t s_bal_ex_scanned; /* total extents scanned */
5276 + atomic_t s_bal_goals; /* goal hits */
5277 + atomic_t s_bal_breaks; /* too long searches */
5278 + atomic_t s_bal_2orders; /* 2^order hits */
5279 + spinlock_t s_bal_lock;
5280 + unsigned long s_mb_buddies_generated;
5281 + unsigned long long s_mb_generation_time;
5282 + atomic_t s_mb_lost_chunks;
5283 + atomic_t s_mb_preallocated;
5284 + atomic_t s_mb_discarded;
5285 +
5286 + /* locality groups */
5287 + struct ext4_locality_group *s_locality_groups;
5288 };
5290 #endif /* _LINUX_EXT4_FS_SB */