| blob | b6c5ede53557ce5285c75f04db41a2ac3aab45f9 |
1 ext4: send parallel discards on commit completions
3 From: Daeho Jeong <daeho.jeong@samsung.com>
5 Now, when we mount ext4 filesystem with '-o discard' option, we have to
6 issue all the discard commands for the blocks to be deallocated and
7 wait for the completion of the commands on the commit complete phase.
8 Because this procedure might involve a lot of sequential combinations of
9 issuing discard commands and waiting for that, the delay of this
10 procedure might be too much long, even to 17.0s in our test,
11 and it results in long commit delay and fsync() performance degradation.
13 To reduce this kind of delay, instead of adding callback for each
14 extent and handling all of them in a sequential manner on commit phase,
15 we instead add a separate list of extents to free to the superblock and
16 then process this list at once after transaction commits so that
17 we can issue all the discard commands in a parallel manner like XFS
18 filesystem.
20 Finally, we could enhance the discard command handling performance.
21 The result was such that 17.0s delay of a single commit in the worst
22 case has been enhanced to 4.8s.
24 Signed-off-by: Daeho Jeong <daeho.jeong@samsung.com>
25 Signed-off-by: Theodore Ts'o <tytso@mit.edu>
26 Tested-by: Hobin Woo <hobin.woo@samsung.com>
27 Tested-by: Kitae Lee <kitae87.lee@samsung.com>
28 Reviewed-by: Jan Kara <jack@suse.cz>
29 ---
30 fs/ext4/ext4.h | 3 ++
31 fs/ext4/mballoc.c | 141 +++++++++++++++++++++++++++++++++++-------------------
32 fs/ext4/mballoc.h | 6 +--
33 fs/ext4/super.c | 3 ++
34 4 files changed, 100 insertions(+), 53 deletions(-)
36 diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
37 index fb69ee2..7dce2cf 100644
38 --- a/fs/ext4/ext4.h
39 +++ b/fs/ext4/ext4.h
40 @@ -1444,6 +1444,8 @@ struct ext4_sb_info {
41 unsigned int *s_mb_maxs;
42 unsigned int s_group_info_size;
43 unsigned int s_mb_free_pending;
44 + struct list_head s_freed_data_list; /* List of blocks to be freed
45 + after commit completed */
47 /* tunables */
48 unsigned long s_stripe;
49 @@ -2434,6 +2436,7 @@ extern int ext4_mb_add_groupinfo(struct super_block *sb,
50 extern int ext4_group_add_blocks(handle_t *handle, struct super_block *sb,
51 ext4_fsblk_t block, unsigned long count);
52 extern int ext4_trim_fs(struct super_block *, struct fstrim_range *);
53 +extern void ext4_process_freed_data(struct super_block *sb, tid_t commit_tid);
55 /* inode.c */
56 int ext4_inode_is_fast_symlink(struct inode *inode);
57 diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c
58 index 354dc1a..ab70b69 100644
59 --- a/fs/ext4/mballoc.c
60 +++ b/fs/ext4/mballoc.c
61 @@ -367,8 +367,6 @@ static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap,
62 ext4_group_t group);
63 static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap,
64 ext4_group_t group);
65 -static void ext4_free_data_callback(struct super_block *sb,
66 - struct ext4_journal_cb_entry *jce, int rc);
68 static inline void *mb_correct_addr_and_bit(int *bit, void *addr)
69 {
70 @@ -2639,6 +2637,7 @@ int ext4_mb_init(struct super_block *sb)
71 spin_lock_init(&sbi->s_md_lock);
72 spin_lock_init(&sbi->s_bal_lock);
73 sbi->s_mb_free_pending = 0;
74 + INIT_LIST_HEAD(&sbi->s_freed_data_list);
76 sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN;
77 sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN;
78 @@ -2782,7 +2781,8 @@ int ext4_mb_release(struct super_block *sb)
79 }
81 static inline int ext4_issue_discard(struct super_block *sb,
82 - ext4_group_t block_group, ext4_grpblk_t cluster, int count)
83 + ext4_group_t block_group, ext4_grpblk_t cluster, int count,
84 + struct bio **biop)
85 {
86 ext4_fsblk_t discard_block;
88 @@ -2791,18 +2791,18 @@ static inline int ext4_issue_discard(struct super_block *sb,
89 count = EXT4_C2B(EXT4_SB(sb), count);
90 trace_ext4_discard_blocks(sb,
91 (unsigned long long) discard_block, count);
92 - return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0);
93 + if (biop) {
94 + return __blkdev_issue_discard(sb->s_bdev,
95 + (sector_t)discard_block << (sb->s_blocksize_bits - 9),
96 + (sector_t)count << (sb->s_blocksize_bits - 9),
97 + GFP_NOFS, 0, biop);
98 + } else
99 + return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0);
100 }
102 -/*
103 - * This function is called by the jbd2 layer once the commit has finished,
104 - * so we know we can free the blocks that were released with that commit.
105 - */
106 -static void ext4_free_data_callback(struct super_block *sb,
107 - struct ext4_journal_cb_entry *jce,
108 - int rc)
109 +static void ext4_free_data_in_buddy(struct super_block *sb,
110 + struct ext4_free_data *entry)
111 {
112 - struct ext4_free_data *entry = (struct ext4_free_data *)jce;
113 struct ext4_buddy e4b;
114 struct ext4_group_info *db;
115 int err, count = 0, count2 = 0;
116 @@ -2810,18 +2810,6 @@ static void ext4_free_data_callback(struct super_block *sb,
117 mb_debug(1, "gonna free %u blocks in group %u (0x%p):",
118 entry->efd_count, entry->efd_group, entry);
120 - if (test_opt(sb, DISCARD)) {
121 - err = ext4_issue_discard(sb, entry->efd_group,
122 - entry->efd_start_cluster,
123 - entry->efd_count);
124 - if (err && err != -EOPNOTSUPP)
125 - ext4_msg(sb, KERN_WARNING, "discard request in"
126 - " group:%d block:%d count:%d failed"
127 - " with %d", entry->efd_group,
128 - entry->efd_start_cluster,
129 - entry->efd_count, err);
130 - }
131 -
132 err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b);
133 /* we expect to find existing buddy because it's pinned */
134 BUG_ON(err != 0);
135 @@ -2862,6 +2850,56 @@ static void ext4_free_data_callback(struct super_block *sb,
136 mb_debug(1, "freed %u blocks in %u structures\n", count, count2);
137 }
139 +/*
140 + * This function is called by the jbd2 layer once the commit has finished,
141 + * so we know we can free the blocks that were released with that commit.
142 + */
143 +void ext4_process_freed_data(struct super_block *sb, tid_t commit_tid)
144 +{
145 + struct ext4_sb_info *sbi = EXT4_SB(sb);
146 + struct ext4_free_data *entry, *tmp;
147 + struct bio *discard_bio = NULL;
148 + struct list_head freed_data_list;
149 + struct list_head *cut_pos = NULL;
150 + int err;
151 +
152 + INIT_LIST_HEAD(&freed_data_list);
153 +
154 + spin_lock(&sbi->s_md_lock);
155 + list_for_each_entry(entry, &sbi->s_freed_data_list, efd_list) {
156 + if (entry->efd_tid != commit_tid)
157 + break;
158 + cut_pos = &entry->efd_list;
159 + }
160 + if (cut_pos)
161 + list_cut_position(&freed_data_list, &sbi->s_freed_data_list,
162 + cut_pos);
163 + spin_unlock(&sbi->s_md_lock);
164 +
165 + if (test_opt(sb, DISCARD)) {
166 + list_for_each_entry(entry, &freed_data_list, efd_list) {
167 + err = ext4_issue_discard(sb, entry->efd_group,
168 + entry->efd_start_cluster,
169 + entry->efd_count,
170 + &discard_bio);
171 + if (err && err != -EOPNOTSUPP) {
172 + ext4_msg(sb, KERN_WARNING, "discard request in"
173 + " group:%d block:%d count:%d failed"
174 + " with %d", entry->efd_group,
175 + entry->efd_start_cluster,
176 + entry->efd_count, err);
177 + } else if (err == -EOPNOTSUPP)
178 + break;
179 + }
180 +
181 + if (discard_bio)
182 + submit_bio_wait(discard_bio);
183 + }
184 +
185 + list_for_each_entry_safe(entry, tmp, &freed_data_list, efd_list)
186 + ext4_free_data_in_buddy(sb, entry);
187 +}
188 +
189 int __init ext4_init_mballoc(void)
190 {
191 ext4_pspace_cachep = KMEM_CACHE(ext4_prealloc_space,
192 @@ -4578,14 +4616,28 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle,
193 * are contiguous, AND the extents were freed by the same transaction,
194 * AND the blocks are associated with the same group.
195 */
196 -static int can_merge(struct ext4_free_data *entry1,
197 - struct ext4_free_data *entry2)
198 +static void ext4_try_merge_freed_extent(struct ext4_sb_info *sbi,
199 + struct ext4_free_data *entry,
200 + struct ext4_free_data *new_entry,
201 + struct rb_root *entry_rb_root)
202 {
203 - if ((entry1->efd_tid == entry2->efd_tid) &&
204 - (entry1->efd_group == entry2->efd_group) &&
205 - ((entry1->efd_start_cluster + entry1->efd_count) == entry2->efd_start_cluster))
206 - return 1;
207 - return 0;
208 + if ((entry->efd_tid != new_entry->efd_tid) ||
209 + (entry->efd_group != new_entry->efd_group))
210 + return;
211 + if (entry->efd_start_cluster + entry->efd_count ==
212 + new_entry->efd_start_cluster) {
213 + new_entry->efd_start_cluster = entry->efd_start_cluster;
214 + new_entry->efd_count += entry->efd_count;
215 + } else if (new_entry->efd_start_cluster + new_entry->efd_count ==
216 + entry->efd_start_cluster) {
217 + new_entry->efd_count += entry->efd_count;
218 + } else
219 + return;
220 + spin_lock(&sbi->s_md_lock);
221 + list_del(&entry->efd_list);
222 + spin_unlock(&sbi->s_md_lock);
223 + rb_erase(&entry->efd_node, entry_rb_root);
224 + kmem_cache_free(ext4_free_data_cachep, entry);
225 }
227 static noinline_for_stack int
228 @@ -4641,29 +4693,19 @@ static int can_merge(struct ext4_free_data *entry1,
229 node = rb_prev(new_node);
230 if (node) {
231 entry = rb_entry(node, struct ext4_free_data, efd_node);
232 - if (can_merge(entry, new_entry) &&
233 - ext4_journal_callback_try_del(handle, &entry->efd_jce)) {
234 - new_entry->efd_start_cluster = entry->efd_start_cluster;
235 - new_entry->efd_count += entry->efd_count;
236 - rb_erase(node, &(db->bb_free_root));
237 - kmem_cache_free(ext4_free_data_cachep, entry);
238 - }
239 + ext4_try_merge_freed_extent(sbi, entry, new_entry,
240 + &(db->bb_free_root));
241 }
243 node = rb_next(new_node);
244 if (node) {
245 entry = rb_entry(node, struct ext4_free_data, efd_node);
246 - if (can_merge(new_entry, entry) &&
247 - ext4_journal_callback_try_del(handle, &entry->efd_jce)) {
248 - new_entry->efd_count += entry->efd_count;
249 - rb_erase(node, &(db->bb_free_root));
250 - kmem_cache_free(ext4_free_data_cachep, entry);
251 - }
252 + ext4_try_merge_freed_extent(sbi, entry, new_entry,
253 + &(db->bb_free_root));
254 }
255 - /* Add the extent to transaction's private list */
256 - new_entry->efd_jce.jce_func = ext4_free_data_callback;
257 +
258 spin_lock(&sbi->s_md_lock);
259 - _ext4_journal_callback_add(handle, &new_entry->efd_jce);
260 + list_add_tail(&new_entry->efd_list, &sbi->s_freed_data_list);
261 sbi->s_mb_free_pending += clusters;
262 spin_unlock(&sbi->s_md_lock);
263 return 0;
264 @@ -4866,7 +4908,8 @@ void ext4_free_blocks(handle_t *handle, struct inode *inode,
265 * them with group lock_held
266 */
267 if (test_opt(sb, DISCARD)) {
268 - err = ext4_issue_discard(sb, block_group, bit, count);
269 + err = ext4_issue_discard(sb, block_group, bit, count,
270 + NULL);
271 if (err && err != -EOPNOTSUPP)
272 ext4_msg(sb, KERN_WARNING, "discard request in"
273 " group:%d block:%d count:%lu failed"
274 @@ -5089,7 +5132,7 @@ static int ext4_trim_extent(struct super_block *sb, int start, int count,
275 */
276 mb_mark_used(e4b, &ex);
277 ext4_unlock_group(sb, group);
278 - ret = ext4_issue_discard(sb, group, start, count);
279 + ret = ext4_issue_discard(sb, group, start, count, NULL);
280 ext4_lock_group(sb, group);
281 mb_free_blocks(NULL, e4b, start, ex.fe_len);
282 return ret;
283 diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h
284 index 1aba469..694204d 100644
285 --- a/fs/ext4/mballoc.h
286 +++ b/fs/ext4/mballoc.h
287 @@ -78,10 +78,8 @@
290 struct ext4_free_data {
291 - /* MUST be the first member */
292 - struct ext4_journal_cb_entry efd_jce;
293 -
294 - /* ext4_free_data private data starts from here */
295 + /* this links the free block information from sb_info */
296 + struct list_head efd_list;
298 /* this links the free block information from group_info */
299 struct rb_node efd_node;
300 diff --git a/fs/ext4/super.c b/fs/ext4/super.c
301 index a9448db..bf95387 100644
302 --- a/fs/ext4/super.c
303 +++ b/fs/ext4/super.c
304 @@ -371,6 +371,9 @@ static void ext4_journal_commit_callback(journal_t *journal, transaction_t *txn)
305 struct ext4_journal_cb_entry *jce;
307 BUG_ON(txn->t_state == T_FINISHED);
308 +
309 + ext4_process_freed_data(sb, txn->t_tid);
310 +
311 spin_lock(&sbi->s_md_lock);
312 while (!list_empty(&txn->t_private_list)) {
313 jce = list_entry(txn->t_private_list.next,
314 --
315 1.9.1