?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Btrfs: account for missing devices in RAID allocation profiles
[linux-2.6/x86.git] / fs / btrfs / extent-tree.c
blob4be231e0d2bd62c3114dfb20c31a740dc50021e7
1 /*
2 * Copyright (C) 2007 Oracle. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18 #include <linux/sched.h>
19 #include <linux/pagemap.h>
20 #include <linux/writeback.h>
21 #include <linux/blkdev.h>
22 #include <linux/sort.h>
23 #include <linux/rcupdate.h>
24 #include <linux/kthread.h>
25 #include <linux/slab.h>
26 #include "compat.h"
27 #include "hash.h"
28 #include "ctree.h"
29 #include "disk-io.h"
30 #include "print-tree.h"
31 #include "transaction.h"
32 #include "volumes.h"
33 #include "locking.h"
34 #include "free-space-cache.h"
35
36 static int update_block_group(struct btrfs_trans_handle *trans,
37 struct btrfs_root *root,
38 u64 bytenr, u64 num_bytes, int alloc);
39 static int update_reserved_bytes(struct btrfs_block_group_cache *cache,
40 u64 num_bytes, int reserve, int sinfo);
41 static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
42 struct btrfs_root *root,
43 u64 bytenr, u64 num_bytes, u64 parent,
44 u64 root_objectid, u64 owner_objectid,
45 u64 owner_offset, int refs_to_drop,
46 struct btrfs_delayed_extent_op *extra_op);
47 static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
48 struct extent_buffer *leaf,
49 struct btrfs_extent_item *ei);
50 static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
51 struct btrfs_root *root,
52 u64 parent, u64 root_objectid,
53 u64 flags, u64 owner, u64 offset,
54 struct btrfs_key *ins, int ref_mod);
55 static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
56 struct btrfs_root *root,
57 u64 parent, u64 root_objectid,
58 u64 flags, struct btrfs_disk_key *key,
59 int level, struct btrfs_key *ins);
60 static int do_chunk_alloc(struct btrfs_trans_handle *trans,
61 struct btrfs_root *extent_root, u64 alloc_bytes,
62 u64 flags, int force);
63 static int find_next_key(struct btrfs_path *path, int level,
64 struct btrfs_key *key);
65 static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
66 int dump_block_groups);
67
68 static noinline int
69 block_group_cache_done(struct btrfs_block_group_cache *cache)
70 {
71 smp_mb();
72 return cache->cached == BTRFS_CACHE_FINISHED;
73 }
74
75 static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
76 {
77 return (cache->flags & bits) == bits;
78 }
79
80 void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
81 {
82 atomic_inc(&cache->count);
83 }
84
85 void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
86 {
87 if (atomic_dec_and_test(&cache->count)) {
88 WARN_ON(cache->pinned > 0);
89 WARN_ON(cache->reserved > 0);
90 WARN_ON(cache->reserved_pinned > 0);
91 kfree(cache);
92 }
93 }
94
95 /*
96 * this adds the block group to the fs_info rb tree for the block group
97 * cache
98 */
99 static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
100 struct btrfs_block_group_cache *block_group)
101 {
102 struct rb_node **p;
103 struct rb_node *parent = NULL;
104 struct btrfs_block_group_cache *cache;
105
106 spin_lock(&info->block_group_cache_lock);
107 p = &info->block_group_cache_tree.rb_node;
108
109 while (*p) {
110 parent = *p;
111 cache = rb_entry(parent, struct btrfs_block_group_cache,
112 cache_node);
113 if (block_group->key.objectid < cache->key.objectid) {
114 p = &(*p)->rb_left;
115 } else if (block_group->key.objectid > cache->key.objectid) {
116 p = &(*p)->rb_right;
117 } else {
118 spin_unlock(&info->block_group_cache_lock);
119 return -EEXIST;
120 }
121 }
122
123 rb_link_node(&block_group->cache_node, parent, p);
124 rb_insert_color(&block_group->cache_node,
125 &info->block_group_cache_tree);
126 spin_unlock(&info->block_group_cache_lock);
127
128 return 0;
129 }
130
131 /*
132 * This will return the block group at or after bytenr if contains is 0, else
133 * it will return the block group that contains the bytenr
134 */
135 static struct btrfs_block_group_cache *
136 block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
137 int contains)
138 {
139 struct btrfs_block_group_cache *cache, *ret = NULL;
140 struct rb_node *n;
141 u64 end, start;
142
143 spin_lock(&info->block_group_cache_lock);
144 n = info->block_group_cache_tree.rb_node;
145
146 while (n) {
147 cache = rb_entry(n, struct btrfs_block_group_cache,
148 cache_node);
149 end = cache->key.objectid + cache->key.offset - 1;
150 start = cache->key.objectid;
151
152 if (bytenr < start) {
153 if (!contains && (!ret || start < ret->key.objectid))
154 ret = cache;
155 n = n->rb_left;
156 } else if (bytenr > start) {
157 if (contains && bytenr <= end) {
158 ret = cache;
159 break;
160 }
161 n = n->rb_right;
162 } else {
163 ret = cache;
164 break;
165 }
166 }
167 if (ret)
168 btrfs_get_block_group(ret);
169 spin_unlock(&info->block_group_cache_lock);
170
171 return ret;
172 }
173
174 static int add_excluded_extent(struct btrfs_root *root,
175 u64 start, u64 num_bytes)
176 {
177 u64 end = start + num_bytes - 1;
178 set_extent_bits(&root->fs_info->freed_extents[0],
179 start, end, EXTENT_UPTODATE, GFP_NOFS);
180 set_extent_bits(&root->fs_info->freed_extents[1],
181 start, end, EXTENT_UPTODATE, GFP_NOFS);
182 return 0;
183 }
184
185 static void free_excluded_extents(struct btrfs_root *root,
186 struct btrfs_block_group_cache *cache)
187 {
188 u64 start, end;
189
190 start = cache->key.objectid;
191 end = start + cache->key.offset - 1;
192
193 clear_extent_bits(&root->fs_info->freed_extents[0],
194 start, end, EXTENT_UPTODATE, GFP_NOFS);
195 clear_extent_bits(&root->fs_info->freed_extents[1],
196 start, end, EXTENT_UPTODATE, GFP_NOFS);
197 }
198
199 static int exclude_super_stripes(struct btrfs_root *root,
200 struct btrfs_block_group_cache *cache)
201 {
202 u64 bytenr;
203 u64 *logical;
204 int stripe_len;
205 int i, nr, ret;
206
207 if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
208 stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
209 cache->bytes_super += stripe_len;
210 ret = add_excluded_extent(root, cache->key.objectid,
211 stripe_len);
212 BUG_ON(ret);
213 }
214
215 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
216 bytenr = btrfs_sb_offset(i);
217 ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
218 cache->key.objectid, bytenr,
219 0, &logical, &nr, &stripe_len);
220 BUG_ON(ret);
221
222 while (nr--) {
223 cache->bytes_super += stripe_len;
224 ret = add_excluded_extent(root, logical[nr],
225 stripe_len);
226 BUG_ON(ret);
227 }
228
229 kfree(logical);
230 }
231 return 0;
232 }
233
234 static struct btrfs_caching_control *
235 get_caching_control(struct btrfs_block_group_cache *cache)
236 {
237 struct btrfs_caching_control *ctl;
238
239 spin_lock(&cache->lock);
240 if (cache->cached != BTRFS_CACHE_STARTED) {
241 spin_unlock(&cache->lock);
242 return NULL;
243 }
244
245 /* We're loading it the fast way, so we don't have a caching_ctl. */
246 if (!cache->caching_ctl) {
247 spin_unlock(&cache->lock);
248 return NULL;
249 }
250
251 ctl = cache->caching_ctl;
252 atomic_inc(&ctl->count);
253 spin_unlock(&cache->lock);
254 return ctl;
255 }
256
257 static void put_caching_control(struct btrfs_caching_control *ctl)
258 {
259 if (atomic_dec_and_test(&ctl->count))
260 kfree(ctl);
261 }
262
263 /*
264 * this is only called by cache_block_group, since we could have freed extents
265 * we need to check the pinned_extents for any extents that can't be used yet
266 * since their free space will be released as soon as the transaction commits.
267 */
268 static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
269 struct btrfs_fs_info *info, u64 start, u64 end)
270 {
271 u64 extent_start, extent_end, size, total_added = 0;
272 int ret;
273
274 while (start < end) {
275 ret = find_first_extent_bit(info->pinned_extents, start,
276 &extent_start, &extent_end,
277 EXTENT_DIRTY | EXTENT_UPTODATE);
278 if (ret)
279 break;
280
281 if (extent_start <= start) {
282 start = extent_end + 1;
283 } else if (extent_start > start && extent_start < end) {
284 size = extent_start - start;
285 total_added += size;
286 ret = btrfs_add_free_space(block_group, start,
287 size);
288 BUG_ON(ret);
289 start = extent_end + 1;
290 } else {
291 break;
292 }
293 }
294
295 if (start < end) {
296 size = end - start;
297 total_added += size;
298 ret = btrfs_add_free_space(block_group, start, size);
299 BUG_ON(ret);
300 }
301
302 return total_added;
303 }
304
305 static int caching_kthread(void *data)
306 {
307 struct btrfs_block_group_cache *block_group = data;
308 struct btrfs_fs_info *fs_info = block_group->fs_info;
309 struct btrfs_caching_control *caching_ctl = block_group->caching_ctl;
310 struct btrfs_root *extent_root = fs_info->extent_root;
311 struct btrfs_path *path;
312 struct extent_buffer *leaf;
313 struct btrfs_key key;
314 u64 total_found = 0;
315 u64 last = 0;
316 u32 nritems;
317 int ret = 0;
318
319 path = btrfs_alloc_path();
320 if (!path)
321 return -ENOMEM;
322
323 exclude_super_stripes(extent_root, block_group);
324 spin_lock(&block_group->space_info->lock);
325 block_group->space_info->bytes_readonly += block_group->bytes_super;
326 spin_unlock(&block_group->space_info->lock);
327
328 last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
329
330 /*
331 * We don't want to deadlock with somebody trying to allocate a new
332 * extent for the extent root while also trying to search the extent
333 * root to add free space. So we skip locking and search the commit
334 * root, since its read-only
335 */
336 path->skip_locking = 1;
337 path->search_commit_root = 1;
338 path->reada = 2;
339
340 key.objectid = last;
341 key.offset = 0;
342 key.type = BTRFS_EXTENT_ITEM_KEY;
343 again:
344 mutex_lock(&caching_ctl->mutex);
345 /* need to make sure the commit_root doesn't disappear */
346 down_read(&fs_info->extent_commit_sem);
347
348 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
349 if (ret < 0)
350 goto err;
351
352 leaf = path->nodes[0];
353 nritems = btrfs_header_nritems(leaf);
354
355 while (1) {
356 smp_mb();
357 if (fs_info->closing > 1) {
358 last = (u64)-1;
359 break;
360 }
361
362 if (path->slots[0] < nritems) {
363 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
364 } else {
365 ret = find_next_key(path, 0, &key);
366 if (ret)
367 break;
368
369 caching_ctl->progress = last;
370 btrfs_release_path(extent_root, path);
371 up_read(&fs_info->extent_commit_sem);
372 mutex_unlock(&caching_ctl->mutex);
373 if (btrfs_transaction_in_commit(fs_info))
374 schedule_timeout(1);
375 else
376 cond_resched();
377 goto again;
378 }
379
380 if (key.objectid < block_group->key.objectid) {
381 path->slots[0]++;
382 continue;
383 }
384
385 if (key.objectid >= block_group->key.objectid +
386 block_group->key.offset)
387 break;
388
389 if (key.type == BTRFS_EXTENT_ITEM_KEY) {
390 total_found += add_new_free_space(block_group,
391 fs_info, last,
392 key.objectid);
393 last = key.objectid + key.offset;
394
395 if (total_found > (1024 * 1024 * 2)) {
396 total_found = 0;
397 wake_up(&caching_ctl->wait);
398 }
399 }
400 path->slots[0]++;
401 }
402 ret = 0;
403
404 total_found += add_new_free_space(block_group, fs_info, last,
405 block_group->key.objectid +
406 block_group->key.offset);
407 caching_ctl->progress = (u64)-1;
408
409 spin_lock(&block_group->lock);
410 block_group->caching_ctl = NULL;
411 block_group->cached = BTRFS_CACHE_FINISHED;
412 spin_unlock(&block_group->lock);
413
414 err:
415 btrfs_free_path(path);
416 up_read(&fs_info->extent_commit_sem);
417
418 free_excluded_extents(extent_root, block_group);
419
420 mutex_unlock(&caching_ctl->mutex);
421 wake_up(&caching_ctl->wait);
422
423 put_caching_control(caching_ctl);
424 atomic_dec(&block_group->space_info->caching_threads);
425 btrfs_put_block_group(block_group);
426
427 return 0;
428 }
429
430 static int cache_block_group(struct btrfs_block_group_cache *cache,
431 struct btrfs_trans_handle *trans,
432 struct btrfs_root *root,
433 int load_cache_only)
434 {
435 struct btrfs_fs_info *fs_info = cache->fs_info;
436 struct btrfs_caching_control *caching_ctl;
437 struct task_struct *tsk;
438 int ret = 0;
439
440 smp_mb();
441 if (cache->cached != BTRFS_CACHE_NO)
442 return 0;
443
444 /*
445 * We can't do the read from on-disk cache during a commit since we need
446 * to have the normal tree locking. Also if we are currently trying to
447 * allocate blocks for the tree root we can't do the fast caching since
448 * we likely hold important locks.
449 */
450 if (!trans->transaction->in_commit &&
451 (root && root != root->fs_info->tree_root)) {
452 spin_lock(&cache->lock);
453 if (cache->cached != BTRFS_CACHE_NO) {
454 spin_unlock(&cache->lock);
455 return 0;
456 }
457 cache->cached = BTRFS_CACHE_STARTED;
458 spin_unlock(&cache->lock);
459
460 ret = load_free_space_cache(fs_info, cache);
461
462 spin_lock(&cache->lock);
463 if (ret == 1) {
464 cache->cached = BTRFS_CACHE_FINISHED;
465 cache->last_byte_to_unpin = (u64)-1;
466 } else {
467 cache->cached = BTRFS_CACHE_NO;
468 }
469 spin_unlock(&cache->lock);
470 if (ret == 1)
471 return 0;
472 }
473
474 if (load_cache_only)
475 return 0;
476
477 caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_KERNEL);
478 BUG_ON(!caching_ctl);
479
480 INIT_LIST_HEAD(&caching_ctl->list);
481 mutex_init(&caching_ctl->mutex);
482 init_waitqueue_head(&caching_ctl->wait);
483 caching_ctl->block_group = cache;
484 caching_ctl->progress = cache->key.objectid;
485 /* one for caching kthread, one for caching block group list */
486 atomic_set(&caching_ctl->count, 2);
487
488 spin_lock(&cache->lock);
489 if (cache->cached != BTRFS_CACHE_NO) {
490 spin_unlock(&cache->lock);
491 kfree(caching_ctl);
492 return 0;
493 }
494 cache->caching_ctl = caching_ctl;
495 cache->cached = BTRFS_CACHE_STARTED;
496 spin_unlock(&cache->lock);
497
498 down_write(&fs_info->extent_commit_sem);
499 list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
500 up_write(&fs_info->extent_commit_sem);
501
502 atomic_inc(&cache->space_info->caching_threads);
503 btrfs_get_block_group(cache);
504
505 tsk = kthread_run(caching_kthread, cache, "btrfs-cache-%llu\n",
506 cache->key.objectid);
507 if (IS_ERR(tsk)) {
508 ret = PTR_ERR(tsk);
509 printk(KERN_ERR "error running thread %d\n", ret);
510 BUG();
511 }
512
513 return ret;
514 }
515
516 /*
517 * return the block group that starts at or after bytenr
518 */
519 static struct btrfs_block_group_cache *
520 btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
521 {
522 struct btrfs_block_group_cache *cache;
523
524 cache = block_group_cache_tree_search(info, bytenr, 0);
525
526 return cache;
527 }
528
529 /*
530 * return the block group that contains the given bytenr
531 */
532 struct btrfs_block_group_cache *btrfs_lookup_block_group(
533 struct btrfs_fs_info *info,
534 u64 bytenr)
535 {
536 struct btrfs_block_group_cache *cache;
537
538 cache = block_group_cache_tree_search(info, bytenr, 1);
539
540 return cache;
541 }
542
543 static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
544 u64 flags)
545 {
546 struct list_head *head = &info->space_info;
547 struct btrfs_space_info *found;
548
549 flags &= BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_SYSTEM |
550 BTRFS_BLOCK_GROUP_METADATA;
551
552 rcu_read_lock();
553 list_for_each_entry_rcu(found, head, list) {
554 if (found->flags & flags) {
555 rcu_read_unlock();
556 return found;
557 }
558 }
559 rcu_read_unlock();
560 return NULL;
561 }
562
563 /*
564 * after adding space to the filesystem, we need to clear the full flags
565 * on all the space infos.
566 */
567 void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
568 {
569 struct list_head *head = &info->space_info;
570 struct btrfs_space_info *found;
571
572 rcu_read_lock();
573 list_for_each_entry_rcu(found, head, list)
574 found->full = 0;
575 rcu_read_unlock();
576 }
577
578 static u64 div_factor(u64 num, int factor)
579 {
580 if (factor == 10)
581 return num;
582 num *= factor;
583 do_div(num, 10);
584 return num;
585 }
586
587 static u64 div_factor_fine(u64 num, int factor)
588 {
589 if (factor == 100)
590 return num;
591 num *= factor;
592 do_div(num, 100);
593 return num;
594 }
595
596 u64 btrfs_find_block_group(struct btrfs_root *root,
597 u64 search_start, u64 search_hint, int owner)
598 {
599 struct btrfs_block_group_cache *cache;
600 u64 used;
601 u64 last = max(search_hint, search_start);
602 u64 group_start = 0;
603 int full_search = 0;
604 int factor = 9;
605 int wrapped = 0;
606 again:
607 while (1) {
608 cache = btrfs_lookup_first_block_group(root->fs_info, last);
609 if (!cache)
610 break;
611
612 spin_lock(&cache->lock);
613 last = cache->key.objectid + cache->key.offset;
614 used = btrfs_block_group_used(&cache->item);
615
616 if ((full_search || !cache->ro) &&
617 block_group_bits(cache, BTRFS_BLOCK_GROUP_METADATA)) {
618 if (used + cache->pinned + cache->reserved <
619 div_factor(cache->key.offset, factor)) {
620 group_start = cache->key.objectid;
621 spin_unlock(&cache->lock);
622 btrfs_put_block_group(cache);
623 goto found;
624 }
625 }
626 spin_unlock(&cache->lock);
627 btrfs_put_block_group(cache);
628 cond_resched();
629 }
630 if (!wrapped) {
631 last = search_start;
632 wrapped = 1;
633 goto again;
634 }
635 if (!full_search && factor < 10) {
636 last = search_start;
637 full_search = 1;
638 factor = 10;
639 goto again;
640 }
641 found:
642 return group_start;
643 }
644
645 /* simple helper to search for an existing extent at a given offset */
646 int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
647 {
648 int ret;
649 struct btrfs_key key;
650 struct btrfs_path *path;
651
652 path = btrfs_alloc_path();
653 BUG_ON(!path);
654 key.objectid = start;
655 key.offset = len;
656 btrfs_set_key_type(&key, BTRFS_EXTENT_ITEM_KEY);
657 ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
658 0, 0);
659 btrfs_free_path(path);
660 return ret;
661 }
662
663 /*
664 * helper function to lookup reference count and flags of extent.
665 *
666 * the head node for delayed ref is used to store the sum of all the
667 * reference count modifications queued up in the rbtree. the head
668 * node may also store the extent flags to set. This way you can check
669 * to see what the reference count and extent flags would be if all of
670 * the delayed refs are not processed.
671 */
672 int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
673 struct btrfs_root *root, u64 bytenr,
674 u64 num_bytes, u64 *refs, u64 *flags)
675 {
676 struct btrfs_delayed_ref_head *head;
677 struct btrfs_delayed_ref_root *delayed_refs;
678 struct btrfs_path *path;
679 struct btrfs_extent_item *ei;
680 struct extent_buffer *leaf;
681 struct btrfs_key key;
682 u32 item_size;
683 u64 num_refs;
684 u64 extent_flags;
685 int ret;
686
687 path = btrfs_alloc_path();
688 if (!path)
689 return -ENOMEM;
690
691 key.objectid = bytenr;
692 key.type = BTRFS_EXTENT_ITEM_KEY;
693 key.offset = num_bytes;
694 if (!trans) {
695 path->skip_locking = 1;
696 path->search_commit_root = 1;
697 }
698 again:
699 ret = btrfs_search_slot(trans, root->fs_info->extent_root,
700 &key, path, 0, 0);
701 if (ret < 0)
702 goto out_free;
703
704 if (ret == 0) {
705 leaf = path->nodes[0];
706 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
707 if (item_size >= sizeof(*ei)) {
708 ei = btrfs_item_ptr(leaf, path->slots[0],
709 struct btrfs_extent_item);
710 num_refs = btrfs_extent_refs(leaf, ei);
711 extent_flags = btrfs_extent_flags(leaf, ei);
712 } else {
713 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
714 struct btrfs_extent_item_v0 *ei0;
715 BUG_ON(item_size != sizeof(*ei0));
716 ei0 = btrfs_item_ptr(leaf, path->slots[0],
717 struct btrfs_extent_item_v0);
718 num_refs = btrfs_extent_refs_v0(leaf, ei0);
719 /* FIXME: this isn't correct for data */
720 extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
721 #else
722 BUG();
723 #endif
724 }
725 BUG_ON(num_refs == 0);
726 } else {
727 num_refs = 0;
728 extent_flags = 0;
729 ret = 0;
730 }
731
732 if (!trans)
733 goto out;
734
735 delayed_refs = &trans->transaction->delayed_refs;
736 spin_lock(&delayed_refs->lock);
737 head = btrfs_find_delayed_ref_head(trans, bytenr);
738 if (head) {
739 if (!mutex_trylock(&head->mutex)) {
740 atomic_inc(&head->node.refs);
741 spin_unlock(&delayed_refs->lock);
742
743 btrfs_release_path(root->fs_info->extent_root, path);
744
745 mutex_lock(&head->mutex);
746 mutex_unlock(&head->mutex);
747 btrfs_put_delayed_ref(&head->node);
748 goto again;
749 }
750 if (head->extent_op && head->extent_op->update_flags)
751 extent_flags |= head->extent_op->flags_to_set;
752 else
753 BUG_ON(num_refs == 0);
754
755 num_refs += head->node.ref_mod;
756 mutex_unlock(&head->mutex);
757 }
758 spin_unlock(&delayed_refs->lock);
759 out:
760 WARN_ON(num_refs == 0);
761 if (refs)
762 *refs = num_refs;
763 if (flags)
764 *flags = extent_flags;
765 out_free:
766 btrfs_free_path(path);
767 return ret;
768 }
769
770 /*
771 * Back reference rules. Back refs have three main goals:
772 *
773 * 1) differentiate between all holders of references to an extent so that
774 * when a reference is dropped we can make sure it was a valid reference
775 * before freeing the extent.
776 *
777 * 2) Provide enough information to quickly find the holders of an extent
778 * if we notice a given block is corrupted or bad.
779 *
780 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
781 * maintenance. This is actually the same as #2, but with a slightly
782 * different use case.
783 *
784 * There are two kinds of back refs. The implicit back refs is optimized
785 * for pointers in non-shared tree blocks. For a given pointer in a block,
786 * back refs of this kind provide information about the block's owner tree
787 * and the pointer's key. These information allow us to find the block by
788 * b-tree searching. The full back refs is for pointers in tree blocks not
789 * referenced by their owner trees. The location of tree block is recorded
790 * in the back refs. Actually the full back refs is generic, and can be
791 * used in all cases the implicit back refs is used. The major shortcoming
792 * of the full back refs is its overhead. Every time a tree block gets
793 * COWed, we have to update back refs entry for all pointers in it.
794 *
795 * For a newly allocated tree block, we use implicit back refs for
796 * pointers in it. This means most tree related operations only involve
797 * implicit back refs. For a tree block created in old transaction, the
798 * only way to drop a reference to it is COW it. So we can detect the
799 * event that tree block loses its owner tree's reference and do the
800 * back refs conversion.
801 *
802 * When a tree block is COW'd through a tree, there are four cases:
803 *
804 * The reference count of the block is one and the tree is the block's
805 * owner tree. Nothing to do in this case.
806 *
807 * The reference count of the block is one and the tree is not the
808 * block's owner tree. In this case, full back refs is used for pointers
809 * in the block. Remove these full back refs, add implicit back refs for
810 * every pointers in the new block.
811 *
812 * The reference count of the block is greater than one and the tree is
813 * the block's owner tree. In this case, implicit back refs is used for
814 * pointers in the block. Add full back refs for every pointers in the
815 * block, increase lower level extents' reference counts. The original
816 * implicit back refs are entailed to the new block.
817 *
818 * The reference count of the block is greater than one and the tree is
819 * not the block's owner tree. Add implicit back refs for every pointer in
820 * the new block, increase lower level extents' reference count.
821 *
822 * Back Reference Key composing:
823 *
824 * The key objectid corresponds to the first byte in the extent,
825 * The key type is used to differentiate between types of back refs.
826 * There are different meanings of the key offset for different types
827 * of back refs.
828 *
829 * File extents can be referenced by:
830 *
831 * - multiple snapshots, subvolumes, or different generations in one subvol
832 * - different files inside a single subvolume
833 * - different offsets inside a file (bookend extents in file.c)
834 *
835 * The extent ref structure for the implicit back refs has fields for:
836 *
837 * - Objectid of the subvolume root
838 * - objectid of the file holding the reference
839 * - original offset in the file
840 * - how many bookend extents
841 *
842 * The key offset for the implicit back refs is hash of the first
843 * three fields.
844 *
845 * The extent ref structure for the full back refs has field for:
846 *
847 * - number of pointers in the tree leaf
848 *
849 * The key offset for the implicit back refs is the first byte of
850 * the tree leaf
851 *
852 * When a file extent is allocated, The implicit back refs is used.
853 * the fields are filled in:
854 *
855 * (root_key.objectid, inode objectid, offset in file, 1)
856 *
857 * When a file extent is removed file truncation, we find the
858 * corresponding implicit back refs and check the following fields:
859 *
860 * (btrfs_header_owner(leaf), inode objectid, offset in file)
861 *
862 * Btree extents can be referenced by:
863 *
864 * - Different subvolumes
865 *
866 * Both the implicit back refs and the full back refs for tree blocks
867 * only consist of key. The key offset for the implicit back refs is
868 * objectid of block's owner tree. The key offset for the full back refs
869 * is the first byte of parent block.
870 *
871 * When implicit back refs is used, information about the lowest key and
872 * level of the tree block are required. These information are stored in
873 * tree block info structure.
874 */
875
876 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
877 static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
878 struct btrfs_root *root,
879 struct btrfs_path *path,
880 u64 owner, u32 extra_size)
881 {
882 struct btrfs_extent_item *item;
883 struct btrfs_extent_item_v0 *ei0;
884 struct btrfs_extent_ref_v0 *ref0;
885 struct btrfs_tree_block_info *bi;
886 struct extent_buffer *leaf;
887 struct btrfs_key key;
888 struct btrfs_key found_key;
889 u32 new_size = sizeof(*item);
890 u64 refs;
891 int ret;
892
893 leaf = path->nodes[0];
894 BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));
895
896 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
897 ei0 = btrfs_item_ptr(leaf, path->slots[0],
898 struct btrfs_extent_item_v0);
899 refs = btrfs_extent_refs_v0(leaf, ei0);
900
901 if (owner == (u64)-1) {
902 while (1) {
903 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
904 ret = btrfs_next_leaf(root, path);
905 if (ret < 0)
906 return ret;
907 BUG_ON(ret > 0);
908 leaf = path->nodes[0];
909 }
910 btrfs_item_key_to_cpu(leaf, &found_key,
911 path->slots[0]);
912 BUG_ON(key.objectid != found_key.objectid);
913 if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
914 path->slots[0]++;
915 continue;
916 }
917 ref0 = btrfs_item_ptr(leaf, path->slots[0],
918 struct btrfs_extent_ref_v0);
919 owner = btrfs_ref_objectid_v0(leaf, ref0);
920 break;
921 }
922 }
923 btrfs_release_path(root, path);
924
925 if (owner < BTRFS_FIRST_FREE_OBJECTID)
926 new_size += sizeof(*bi);
927
928 new_size -= sizeof(*ei0);
929 ret = btrfs_search_slot(trans, root, &key, path,
930 new_size + extra_size, 1);
931 if (ret < 0)
932 return ret;
933 BUG_ON(ret);
934
935 ret = btrfs_extend_item(trans, root, path, new_size);
936 BUG_ON(ret);
937
938 leaf = path->nodes[0];
939 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
940 btrfs_set_extent_refs(leaf, item, refs);
941 /* FIXME: get real generation */
942 btrfs_set_extent_generation(leaf, item, 0);
943 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
944 btrfs_set_extent_flags(leaf, item,
945 BTRFS_EXTENT_FLAG_TREE_BLOCK |
946 BTRFS_BLOCK_FLAG_FULL_BACKREF);
947 bi = (struct btrfs_tree_block_info *)(item + 1);
948 /* FIXME: get first key of the block */
949 memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
950 btrfs_set_tree_block_level(leaf, bi, (int)owner);
951 } else {
952 btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
953 }
954 btrfs_mark_buffer_dirty(leaf);
955 return 0;
956 }
957 #endif
958
959 static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
960 {
961 u32 high_crc = ~(u32)0;
962 u32 low_crc = ~(u32)0;
963 __le64 lenum;
964
965 lenum = cpu_to_le64(root_objectid);
966 high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
967 lenum = cpu_to_le64(owner);
968 low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
969 lenum = cpu_to_le64(offset);
970 low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
971
972 return ((u64)high_crc << 31) ^ (u64)low_crc;
973 }
974
975 static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
976 struct btrfs_extent_data_ref *ref)
977 {
978 return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
979 btrfs_extent_data_ref_objectid(leaf, ref),
980 btrfs_extent_data_ref_offset(leaf, ref));
981 }
982
983 static int match_extent_data_ref(struct extent_buffer *leaf,
984 struct btrfs_extent_data_ref *ref,
985 u64 root_objectid, u64 owner, u64 offset)
986 {
987 if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
988 btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
989 btrfs_extent_data_ref_offset(leaf, ref) != offset)
990 return 0;
991 return 1;
992 }
993
994 static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
995 struct btrfs_root *root,
996 struct btrfs_path *path,
997 u64 bytenr, u64 parent,
998 u64 root_objectid,
999 u64 owner, u64 offset)
1000 {
1001 struct btrfs_key key;
1002 struct btrfs_extent_data_ref *ref;
1003 struct extent_buffer *leaf;
1004 u32 nritems;
1005 int ret;
1006 int recow;
1007 int err = -ENOENT;
1008
1009 key.objectid = bytenr;
1010 if (parent) {
1011 key.type = BTRFS_SHARED_DATA_REF_KEY;
1012 key.offset = parent;
1013 } else {
1014 key.type = BTRFS_EXTENT_DATA_REF_KEY;
1015 key.offset = hash_extent_data_ref(root_objectid,
1016 owner, offset);
1017 }
1018 again:
1019 recow = 0;
1020 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1021 if (ret < 0) {
1022 err = ret;
1023 goto fail;
1024 }
1025
1026 if (parent) {
1027 if (!ret)
1028 return 0;
1029 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1030 key.type = BTRFS_EXTENT_REF_V0_KEY;
1031 btrfs_release_path(root, path);
1032 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1033 if (ret < 0) {
1034 err = ret;
1035 goto fail;
1036 }
1037 if (!ret)
1038 return 0;
1039 #endif
1040 goto fail;
1041 }
1042
1043 leaf = path->nodes[0];
1044 nritems = btrfs_header_nritems(leaf);
1045 while (1) {
1046 if (path->slots[0] >= nritems) {
1047 ret = btrfs_next_leaf(root, path);
1048 if (ret < 0)
1049 err = ret;
1050 if (ret)
1051 goto fail;
1052
1053 leaf = path->nodes[0];
1054 nritems = btrfs_header_nritems(leaf);
1055 recow = 1;
1056 }
1057
1058 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1059 if (key.objectid != bytenr ||
1060 key.type != BTRFS_EXTENT_DATA_REF_KEY)
1061 goto fail;
1062
1063 ref = btrfs_item_ptr(leaf, path->slots[0],
1064 struct btrfs_extent_data_ref);
1065
1066 if (match_extent_data_ref(leaf, ref, root_objectid,
1067 owner, offset)) {
1068 if (recow) {
1069 btrfs_release_path(root, path);
1070 goto again;
1071 }
1072 err = 0;
1073 break;
1074 }
1075 path->slots[0]++;
1076 }
1077 fail:
1078 return err;
1079 }
1080
1081 static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
1082 struct btrfs_root *root,
1083 struct btrfs_path *path,
1084 u64 bytenr, u64 parent,
1085 u64 root_objectid, u64 owner,
1086 u64 offset, int refs_to_add)
1087 {
1088 struct btrfs_key key;
1089 struct extent_buffer *leaf;
1090 u32 size;
1091 u32 num_refs;
1092 int ret;
1093
1094 key.objectid = bytenr;
1095 if (parent) {
1096 key.type = BTRFS_SHARED_DATA_REF_KEY;
1097 key.offset = parent;
1098 size = sizeof(struct btrfs_shared_data_ref);
1099 } else {
1100 key.type = BTRFS_EXTENT_DATA_REF_KEY;
1101 key.offset = hash_extent_data_ref(root_objectid,
1102 owner, offset);
1103 size = sizeof(struct btrfs_extent_data_ref);
1104 }
1105
1106 ret = btrfs_insert_empty_item(trans, root, path, &key, size);
1107 if (ret && ret != -EEXIST)
1108 goto fail;
1109
1110 leaf = path->nodes[0];
1111 if (parent) {
1112 struct btrfs_shared_data_ref *ref;
1113 ref = btrfs_item_ptr(leaf, path->slots[0],
1114 struct btrfs_shared_data_ref);
1115 if (ret == 0) {
1116 btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
1117 } else {
1118 num_refs = btrfs_shared_data_ref_count(leaf, ref);
1119 num_refs += refs_to_add;
1120 btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
1121 }
1122 } else {
1123 struct btrfs_extent_data_ref *ref;
1124 while (ret == -EEXIST) {
1125 ref = btrfs_item_ptr(leaf, path->slots[0],
1126 struct btrfs_extent_data_ref);
1127 if (match_extent_data_ref(leaf, ref, root_objectid,
1128 owner, offset))
1129 break;
1130 btrfs_release_path(root, path);
1131 key.offset++;
1132 ret = btrfs_insert_empty_item(trans, root, path, &key,
1133 size);
1134 if (ret && ret != -EEXIST)
1135 goto fail;
1136
1137 leaf = path->nodes[0];
1138 }
1139 ref = btrfs_item_ptr(leaf, path->slots[0],
1140 struct btrfs_extent_data_ref);
1141 if (ret == 0) {
1142 btrfs_set_extent_data_ref_root(leaf, ref,
1143 root_objectid);
1144 btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
1145 btrfs_set_extent_data_ref_offset(leaf, ref, offset);
1146 btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
1147 } else {
1148 num_refs = btrfs_extent_data_ref_count(leaf, ref);
1149 num_refs += refs_to_add;
1150 btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
1151 }
1152 }
1153 btrfs_mark_buffer_dirty(leaf);
1154 ret = 0;
1155 fail:
1156 btrfs_release_path(root, path);
1157 return ret;
1158 }
1159
1160 static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
1161 struct btrfs_root *root,
1162 struct btrfs_path *path,
1163 int refs_to_drop)
1164 {
1165 struct btrfs_key key;
1166 struct btrfs_extent_data_ref *ref1 = NULL;
1167 struct btrfs_shared_data_ref *ref2 = NULL;
1168 struct extent_buffer *leaf;
1169 u32 num_refs = 0;
1170 int ret = 0;
1171
1172 leaf = path->nodes[0];
1173 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1174
1175 if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
1176 ref1 = btrfs_item_ptr(leaf, path->slots[0],
1177 struct btrfs_extent_data_ref);
1178 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1179 } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
1180 ref2 = btrfs_item_ptr(leaf, path->slots[0],
1181 struct btrfs_shared_data_ref);
1182 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1183 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1184 } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
1185 struct btrfs_extent_ref_v0 *ref0;
1186 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1187 struct btrfs_extent_ref_v0);
1188 num_refs = btrfs_ref_count_v0(leaf, ref0);
1189 #endif
1190 } else {
1191 BUG();
1192 }
1193
1194 BUG_ON(num_refs < refs_to_drop);
1195 num_refs -= refs_to_drop;
1196
1197 if (num_refs == 0) {
1198 ret = btrfs_del_item(trans, root, path);
1199 } else {
1200 if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
1201 btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
1202 else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
1203 btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
1204 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1205 else {
1206 struct btrfs_extent_ref_v0 *ref0;
1207 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1208 struct btrfs_extent_ref_v0);
1209 btrfs_set_ref_count_v0(leaf, ref0, num_refs);
1210 }
1211 #endif
1212 btrfs_mark_buffer_dirty(leaf);
1213 }
1214 return ret;
1215 }
1216
1217 static noinline u32 extent_data_ref_count(struct btrfs_root *root,
1218 struct btrfs_path *path,
1219 struct btrfs_extent_inline_ref *iref)
1220 {
1221 struct btrfs_key key;
1222 struct extent_buffer *leaf;
1223 struct btrfs_extent_data_ref *ref1;
1224 struct btrfs_shared_data_ref *ref2;
1225 u32 num_refs = 0;
1226
1227 leaf = path->nodes[0];
1228 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1229 if (iref) {
1230 if (btrfs_extent_inline_ref_type(leaf, iref) ==
1231 BTRFS_EXTENT_DATA_REF_KEY) {
1232 ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
1233 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1234 } else {
1235 ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
1236 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1237 }
1238 } else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
1239 ref1 = btrfs_item_ptr(leaf, path->slots[0],
1240 struct btrfs_extent_data_ref);
1241 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1242 } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
1243 ref2 = btrfs_item_ptr(leaf, path->slots[0],
1244 struct btrfs_shared_data_ref);
1245 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1246 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1247 } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
1248 struct btrfs_extent_ref_v0 *ref0;
1249 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1250 struct btrfs_extent_ref_v0);
1251 num_refs = btrfs_ref_count_v0(leaf, ref0);
1252 #endif
1253 } else {
1254 WARN_ON(1);
1255 }
1256 return num_refs;
1257 }
1258
1259 static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
1260 struct btrfs_root *root,
1261 struct btrfs_path *path,
1262 u64 bytenr, u64 parent,
1263 u64 root_objectid)
1264 {
1265 struct btrfs_key key;
1266 int ret;
1267
1268 key.objectid = bytenr;
1269 if (parent) {
1270 key.type = BTRFS_SHARED_BLOCK_REF_KEY;
1271 key.offset = parent;
1272 } else {
1273 key.type = BTRFS_TREE_BLOCK_REF_KEY;
1274 key.offset = root_objectid;
1275 }
1276
1277 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1278 if (ret > 0)
1279 ret = -ENOENT;
1280 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1281 if (ret == -ENOENT && parent) {
1282 btrfs_release_path(root, path);
1283 key.type = BTRFS_EXTENT_REF_V0_KEY;
1284 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1285 if (ret > 0)
1286 ret = -ENOENT;
1287 }
1288 #endif
1289 return ret;
1290 }
1291
1292 static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
1293 struct btrfs_root *root,
1294 struct btrfs_path *path,
1295 u64 bytenr, u64 parent,
1296 u64 root_objectid)
1297 {
1298 struct btrfs_key key;
1299 int ret;
1300
1301 key.objectid = bytenr;
1302 if (parent) {
1303 key.type = BTRFS_SHARED_BLOCK_REF_KEY;
1304 key.offset = parent;
1305 } else {
1306 key.type = BTRFS_TREE_BLOCK_REF_KEY;
1307 key.offset = root_objectid;
1308 }
1309
1310 ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
1311 btrfs_release_path(root, path);
1312 return ret;
1313 }
1314
1315 static inline int extent_ref_type(u64 parent, u64 owner)
1316 {
1317 int type;
1318 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1319 if (parent > 0)
1320 type = BTRFS_SHARED_BLOCK_REF_KEY;
1321 else
1322 type = BTRFS_TREE_BLOCK_REF_KEY;
1323 } else {
1324 if (parent > 0)
1325 type = BTRFS_SHARED_DATA_REF_KEY;
1326 else
1327 type = BTRFS_EXTENT_DATA_REF_KEY;
1328 }
1329 return type;
1330 }
1331
1332 static int find_next_key(struct btrfs_path *path, int level,
1333 struct btrfs_key *key)
1334
1335 {
1336 for (; level < BTRFS_MAX_LEVEL; level++) {
1337 if (!path->nodes[level])
1338 break;
1339 if (path->slots[level] + 1 >=
1340 btrfs_header_nritems(path->nodes[level]))
1341 continue;
1342 if (level == 0)
1343 btrfs_item_key_to_cpu(path->nodes[level], key,
1344 path->slots[level] + 1);
1345 else
1346 btrfs_node_key_to_cpu(path->nodes[level], key,
1347 path->slots[level] + 1);
1348 return 0;
1349 }
1350 return 1;
1351 }
1352
1353 /*
1354 * look for inline back ref. if back ref is found, *ref_ret is set
1355 * to the address of inline back ref, and 0 is returned.
1356 *
1357 * if back ref isn't found, *ref_ret is set to the address where it
1358 * should be inserted, and -ENOENT is returned.
1359 *
1360 * if insert is true and there are too many inline back refs, the path
1361 * points to the extent item, and -EAGAIN is returned.
1362 *
1363 * NOTE: inline back refs are ordered in the same way that back ref
1364 * items in the tree are ordered.
1365 */
1366 static noinline_for_stack
1367 int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
1368 struct btrfs_root *root,
1369 struct btrfs_path *path,
1370 struct btrfs_extent_inline_ref **ref_ret,
1371 u64 bytenr, u64 num_bytes,
1372 u64 parent, u64 root_objectid,
1373 u64 owner, u64 offset, int insert)
1374 {
1375 struct btrfs_key key;
1376 struct extent_buffer *leaf;
1377 struct btrfs_extent_item *ei;
1378 struct btrfs_extent_inline_ref *iref;
1379 u64 flags;
1380 u64 item_size;
1381 unsigned long ptr;
1382 unsigned long end;
1383 int extra_size;
1384 int type;
1385 int want;
1386 int ret;
1387 int err = 0;
1388
1389 key.objectid = bytenr;
1390 key.type = BTRFS_EXTENT_ITEM_KEY;
1391 key.offset = num_bytes;
1392
1393 want = extent_ref_type(parent, owner);
1394 if (insert) {
1395 extra_size = btrfs_extent_inline_ref_size(want);
1396 path->keep_locks = 1;
1397 } else
1398 extra_size = -1;
1399 ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1400 if (ret < 0) {
1401 err = ret;
1402 goto out;
1403 }
1404 BUG_ON(ret);
1405
1406 leaf = path->nodes[0];
1407 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1408 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1409 if (item_size < sizeof(*ei)) {
1410 if (!insert) {
1411 err = -ENOENT;
1412 goto out;
1413 }
1414 ret = convert_extent_item_v0(trans, root, path, owner,
1415 extra_size);
1416 if (ret < 0) {
1417 err = ret;
1418 goto out;
1419 }
1420 leaf = path->nodes[0];
1421 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1422 }
1423 #endif
1424 BUG_ON(item_size < sizeof(*ei));
1425
1426 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1427 flags = btrfs_extent_flags(leaf, ei);
1428
1429 ptr = (unsigned long)(ei + 1);
1430 end = (unsigned long)ei + item_size;
1431
1432 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
1433 ptr += sizeof(struct btrfs_tree_block_info);
1434 BUG_ON(ptr > end);
1435 } else {
1436 BUG_ON(!(flags & BTRFS_EXTENT_FLAG_DATA));
1437 }
1438
1439 err = -ENOENT;
1440 while (1) {
1441 if (ptr >= end) {
1442 WARN_ON(ptr > end);
1443 break;
1444 }
1445 iref = (struct btrfs_extent_inline_ref *)ptr;
1446 type = btrfs_extent_inline_ref_type(leaf, iref);
1447 if (want < type)
1448 break;
1449 if (want > type) {
1450 ptr += btrfs_extent_inline_ref_size(type);
1451 continue;
1452 }
1453
1454 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1455 struct btrfs_extent_data_ref *dref;
1456 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1457 if (match_extent_data_ref(leaf, dref, root_objectid,
1458 owner, offset)) {
1459 err = 0;
1460 break;
1461 }
1462 if (hash_extent_data_ref_item(leaf, dref) <
1463 hash_extent_data_ref(root_objectid, owner, offset))
1464 break;
1465 } else {
1466 u64 ref_offset;
1467 ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
1468 if (parent > 0) {
1469 if (parent == ref_offset) {
1470 err = 0;
1471 break;
1472 }
1473 if (ref_offset < parent)
1474 break;
1475 } else {
1476 if (root_objectid == ref_offset) {
1477 err = 0;
1478 break;
1479 }
1480 if (ref_offset < root_objectid)
1481 break;
1482 }
1483 }
1484 ptr += btrfs_extent_inline_ref_size(type);
1485 }
1486 if (err == -ENOENT && insert) {
1487 if (item_size + extra_size >=
1488 BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
1489 err = -EAGAIN;
1490 goto out;
1491 }
1492 /*
1493 * To add new inline back ref, we have to make sure
1494 * there is no corresponding back ref item.
1495 * For simplicity, we just do not add new inline back
1496 * ref if there is any kind of item for this block
1497 */
1498 if (find_next_key(path, 0, &key) == 0 &&
1499 key.objectid == bytenr &&
1500 key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1501 err = -EAGAIN;
1502 goto out;
1503 }
1504 }
1505 *ref_ret = (struct btrfs_extent_inline_ref *)ptr;
1506 out:
1507 if (insert) {
1508 path->keep_locks = 0;
1509 btrfs_unlock_up_safe(path, 1);
1510 }
1511 return err;
1512 }
1513
1514 /*
1515 * helper to add new inline back ref
1516 */
1517 static noinline_for_stack
1518 int setup_inline_extent_backref(struct btrfs_trans_handle *trans,
1519 struct btrfs_root *root,
1520 struct btrfs_path *path,
1521 struct btrfs_extent_inline_ref *iref,
1522 u64 parent, u64 root_objectid,
1523 u64 owner, u64 offset, int refs_to_add,
1524 struct btrfs_delayed_extent_op *extent_op)
1525 {
1526 struct extent_buffer *leaf;
1527 struct btrfs_extent_item *ei;
1528 unsigned long ptr;
1529 unsigned long end;
1530 unsigned long item_offset;
1531 u64 refs;
1532 int size;
1533 int type;
1534 int ret;
1535
1536 leaf = path->nodes[0];
1537 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1538 item_offset = (unsigned long)iref - (unsigned long)ei;
1539
1540 type = extent_ref_type(parent, owner);
1541 size = btrfs_extent_inline_ref_size(type);
1542
1543 ret = btrfs_extend_item(trans, root, path, size);
1544 BUG_ON(ret);
1545
1546 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1547 refs = btrfs_extent_refs(leaf, ei);
1548 refs += refs_to_add;
1549 btrfs_set_extent_refs(leaf, ei, refs);
1550 if (extent_op)
1551 __run_delayed_extent_op(extent_op, leaf, ei);
1552
1553 ptr = (unsigned long)ei + item_offset;
1554 end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
1555 if (ptr < end - size)
1556 memmove_extent_buffer(leaf, ptr + size, ptr,
1557 end - size - ptr);
1558
1559 iref = (struct btrfs_extent_inline_ref *)ptr;
1560 btrfs_set_extent_inline_ref_type(leaf, iref, type);
1561 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1562 struct btrfs_extent_data_ref *dref;
1563 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1564 btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
1565 btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
1566 btrfs_set_extent_data_ref_offset(leaf, dref, offset);
1567 btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
1568 } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
1569 struct btrfs_shared_data_ref *sref;
1570 sref = (struct btrfs_shared_data_ref *)(iref + 1);
1571 btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
1572 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
1573 } else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
1574 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
1575 } else {
1576 btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
1577 }
1578 btrfs_mark_buffer_dirty(leaf);
1579 return 0;
1580 }
1581
1582 static int lookup_extent_backref(struct btrfs_trans_handle *trans,
1583 struct btrfs_root *root,
1584 struct btrfs_path *path,
1585 struct btrfs_extent_inline_ref **ref_ret,
1586 u64 bytenr, u64 num_bytes, u64 parent,
1587 u64 root_objectid, u64 owner, u64 offset)
1588 {
1589 int ret;
1590
1591 ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
1592 bytenr, num_bytes, parent,
1593 root_objectid, owner, offset, 0);
1594 if (ret != -ENOENT)
1595 return ret;
1596
1597 btrfs_release_path(root, path);
1598 *ref_ret = NULL;
1599
1600 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1601 ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
1602 root_objectid);
1603 } else {
1604 ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
1605 root_objectid, owner, offset);
1606 }
1607 return ret;
1608 }
1609
1610 /*
1611 * helper to update/remove inline back ref
1612 */
1613 static noinline_for_stack
1614 int update_inline_extent_backref(struct btrfs_trans_handle *trans,
1615 struct btrfs_root *root,
1616 struct btrfs_path *path,
1617 struct btrfs_extent_inline_ref *iref,
1618 int refs_to_mod,
1619 struct btrfs_delayed_extent_op *extent_op)
1620 {
1621 struct extent_buffer *leaf;
1622 struct btrfs_extent_item *ei;
1623 struct btrfs_extent_data_ref *dref = NULL;
1624 struct btrfs_shared_data_ref *sref = NULL;
1625 unsigned long ptr;
1626 unsigned long end;
1627 u32 item_size;
1628 int size;
1629 int type;
1630 int ret;
1631 u64 refs;
1632
1633 leaf = path->nodes[0];
1634 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1635 refs = btrfs_extent_refs(leaf, ei);
1636 WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
1637 refs += refs_to_mod;
1638 btrfs_set_extent_refs(leaf, ei, refs);
1639 if (extent_op)
1640 __run_delayed_extent_op(extent_op, leaf, ei);
1641
1642 type = btrfs_extent_inline_ref_type(leaf, iref);
1643
1644 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1645 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1646 refs = btrfs_extent_data_ref_count(leaf, dref);
1647 } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
1648 sref = (struct btrfs_shared_data_ref *)(iref + 1);
1649 refs = btrfs_shared_data_ref_count(leaf, sref);
1650 } else {
1651 refs = 1;
1652 BUG_ON(refs_to_mod != -1);
1653 }
1654
1655 BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
1656 refs += refs_to_mod;
1657
1658 if (refs > 0) {
1659 if (type == BTRFS_EXTENT_DATA_REF_KEY)
1660 btrfs_set_extent_data_ref_count(leaf, dref, refs);
1661 else
1662 btrfs_set_shared_data_ref_count(leaf, sref, refs);
1663 } else {
1664 size = btrfs_extent_inline_ref_size(type);
1665 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1666 ptr = (unsigned long)iref;
1667 end = (unsigned long)ei + item_size;
1668 if (ptr + size < end)
1669 memmove_extent_buffer(leaf, ptr, ptr + size,
1670 end - ptr - size);
1671 item_size -= size;
1672 ret = btrfs_truncate_item(trans, root, path, item_size, 1);
1673 BUG_ON(ret);
1674 }
1675 btrfs_mark_buffer_dirty(leaf);
1676 return 0;
1677 }
1678
1679 static noinline_for_stack
1680 int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
1681 struct btrfs_root *root,
1682 struct btrfs_path *path,
1683 u64 bytenr, u64 num_bytes, u64 parent,
1684 u64 root_objectid, u64 owner,
1685 u64 offset, int refs_to_add,
1686 struct btrfs_delayed_extent_op *extent_op)
1687 {
1688 struct btrfs_extent_inline_ref *iref;
1689 int ret;
1690
1691 ret = lookup_inline_extent_backref(trans, root, path, &iref,
1692 bytenr, num_bytes, parent,
1693 root_objectid, owner, offset, 1);
1694 if (ret == 0) {
1695 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
1696 ret = update_inline_extent_backref(trans, root, path, iref,
1697 refs_to_add, extent_op);
1698 } else if (ret == -ENOENT) {
1699 ret = setup_inline_extent_backref(trans, root, path, iref,
1700 parent, root_objectid,
1701 owner, offset, refs_to_add,
1702 extent_op);
1703 }
1704 return ret;
1705 }
1706
1707 static int insert_extent_backref(struct btrfs_trans_handle *trans,
1708 struct btrfs_root *root,
1709 struct btrfs_path *path,
1710 u64 bytenr, u64 parent, u64 root_objectid,
1711 u64 owner, u64 offset, int refs_to_add)
1712 {
1713 int ret;
1714 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1715 BUG_ON(refs_to_add != 1);
1716 ret = insert_tree_block_ref(trans, root, path, bytenr,
1717 parent, root_objectid);
1718 } else {
1719 ret = insert_extent_data_ref(trans, root, path, bytenr,
1720 parent, root_objectid,
1721 owner, offset, refs_to_add);
1722 }
1723 return ret;
1724 }
1725
1726 static int remove_extent_backref(struct btrfs_trans_handle *trans,
1727 struct btrfs_root *root,
1728 struct btrfs_path *path,
1729 struct btrfs_extent_inline_ref *iref,
1730 int refs_to_drop, int is_data)
1731 {
1732 int ret;
1733
1734 BUG_ON(!is_data && refs_to_drop != 1);
1735 if (iref) {
1736 ret = update_inline_extent_backref(trans, root, path, iref,
1737 -refs_to_drop, NULL);
1738 } else if (is_data) {
1739 ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
1740 } else {
1741 ret = btrfs_del_item(trans, root, path);
1742 }
1743 return ret;
1744 }
1745
1746 static void btrfs_issue_discard(struct block_device *bdev,
1747 u64 start, u64 len)
1748 {
1749 blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_KERNEL,
1750 BLKDEV_IFL_WAIT | BLKDEV_IFL_BARRIER);
1751 }
1752
1753 static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
1754 u64 num_bytes)
1755 {
1756 int ret;
1757 u64 map_length = num_bytes;
1758 struct btrfs_multi_bio *multi = NULL;
1759
1760 if (!btrfs_test_opt(root, DISCARD))
1761 return 0;
1762
1763 /* Tell the block device(s) that the sectors can be discarded */
1764 ret = btrfs_map_block(&root->fs_info->mapping_tree, READ,
1765 bytenr, &map_length, &multi, 0);
1766 if (!ret) {
1767 struct btrfs_bio_stripe *stripe = multi->stripes;
1768 int i;
1769
1770 if (map_length > num_bytes)
1771 map_length = num_bytes;
1772
1773 for (i = 0; i < multi->num_stripes; i++, stripe++) {
1774 btrfs_issue_discard(stripe->dev->bdev,
1775 stripe->physical,
1776 map_length);
1777 }
1778 kfree(multi);
1779 }
1780
1781 return ret;
1782 }
1783
1784 int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
1785 struct btrfs_root *root,
1786 u64 bytenr, u64 num_bytes, u64 parent,
1787 u64 root_objectid, u64 owner, u64 offset)
1788 {
1789 int ret;
1790 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
1791 root_objectid == BTRFS_TREE_LOG_OBJECTID);
1792
1793 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1794 ret = btrfs_add_delayed_tree_ref(trans, bytenr, num_bytes,
1795 parent, root_objectid, (int)owner,
1796 BTRFS_ADD_DELAYED_REF, NULL);
1797 } else {
1798 ret = btrfs_add_delayed_data_ref(trans, bytenr, num_bytes,
1799 parent, root_objectid, owner, offset,
1800 BTRFS_ADD_DELAYED_REF, NULL);
1801 }
1802 return ret;
1803 }
1804
1805 static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
1806 struct btrfs_root *root,
1807 u64 bytenr, u64 num_bytes,
1808 u64 parent, u64 root_objectid,
1809 u64 owner, u64 offset, int refs_to_add,
1810 struct btrfs_delayed_extent_op *extent_op)
1811 {
1812 struct btrfs_path *path;
1813 struct extent_buffer *leaf;
1814 struct btrfs_extent_item *item;
1815 u64 refs;
1816 int ret;
1817 int err = 0;
1818
1819 path = btrfs_alloc_path();
1820 if (!path)
1821 return -ENOMEM;
1822
1823 path->reada = 1;
1824 path->leave_spinning = 1;
1825 /* this will setup the path even if it fails to insert the back ref */
1826 ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
1827 path, bytenr, num_bytes, parent,
1828 root_objectid, owner, offset,
1829 refs_to_add, extent_op);
1830 if (ret == 0)
1831 goto out;
1832
1833 if (ret != -EAGAIN) {
1834 err = ret;
1835 goto out;
1836 }
1837
1838 leaf = path->nodes[0];
1839 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1840 refs = btrfs_extent_refs(leaf, item);
1841 btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
1842 if (extent_op)
1843 __run_delayed_extent_op(extent_op, leaf, item);
1844
1845 btrfs_mark_buffer_dirty(leaf);
1846 btrfs_release_path(root->fs_info->extent_root, path);
1847
1848 path->reada = 1;
1849 path->leave_spinning = 1;
1850
1851 /* now insert the actual backref */
1852 ret = insert_extent_backref(trans, root->fs_info->extent_root,
1853 path, bytenr, parent, root_objectid,
1854 owner, offset, refs_to_add);
1855 BUG_ON(ret);
1856 out:
1857 btrfs_free_path(path);
1858 return err;
1859 }
1860
1861 static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
1862 struct btrfs_root *root,
1863 struct btrfs_delayed_ref_node *node,
1864 struct btrfs_delayed_extent_op *extent_op,
1865 int insert_reserved)
1866 {
1867 int ret = 0;
1868 struct btrfs_delayed_data_ref *ref;
1869 struct btrfs_key ins;
1870 u64 parent = 0;
1871 u64 ref_root = 0;
1872 u64 flags = 0;
1873
1874 ins.objectid = node->bytenr;
1875 ins.offset = node->num_bytes;
1876 ins.type = BTRFS_EXTENT_ITEM_KEY;
1877
1878 ref = btrfs_delayed_node_to_data_ref(node);
1879 if (node->type == BTRFS_SHARED_DATA_REF_KEY)
1880 parent = ref->parent;
1881 else
1882 ref_root = ref->root;
1883
1884 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
1885 if (extent_op) {
1886 BUG_ON(extent_op->update_key);
1887 flags |= extent_op->flags_to_set;
1888 }
1889 ret = alloc_reserved_file_extent(trans, root,
1890 parent, ref_root, flags,
1891 ref->objectid, ref->offset,
1892 &ins, node->ref_mod);
1893 } else if (node->action == BTRFS_ADD_DELAYED_REF) {
1894 ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
1895 node->num_bytes, parent,
1896 ref_root, ref->objectid,
1897 ref->offset, node->ref_mod,
1898 extent_op);
1899 } else if (node->action == BTRFS_DROP_DELAYED_REF) {
1900 ret = __btrfs_free_extent(trans, root, node->bytenr,
1901 node->num_bytes, parent,
1902 ref_root, ref->objectid,
1903 ref->offset, node->ref_mod,
1904 extent_op);
1905 } else {
1906 BUG();
1907 }
1908 return ret;
1909 }
1910
1911 static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
1912 struct extent_buffer *leaf,
1913 struct btrfs_extent_item *ei)
1914 {
1915 u64 flags = btrfs_extent_flags(leaf, ei);
1916 if (extent_op->update_flags) {
1917 flags |= extent_op->flags_to_set;
1918 btrfs_set_extent_flags(leaf, ei, flags);
1919 }
1920
1921 if (extent_op->update_key) {
1922 struct btrfs_tree_block_info *bi;
1923 BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
1924 bi = (struct btrfs_tree_block_info *)(ei + 1);
1925 btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
1926 }
1927 }
1928
1929 static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
1930 struct btrfs_root *root,
1931 struct btrfs_delayed_ref_node *node,
1932 struct btrfs_delayed_extent_op *extent_op)
1933 {
1934 struct btrfs_key key;
1935 struct btrfs_path *path;
1936 struct btrfs_extent_item *ei;
1937 struct extent_buffer *leaf;
1938 u32 item_size;
1939 int ret;
1940 int err = 0;
1941
1942 path = btrfs_alloc_path();
1943 if (!path)
1944 return -ENOMEM;
1945
1946 key.objectid = node->bytenr;
1947 key.type = BTRFS_EXTENT_ITEM_KEY;
1948 key.offset = node->num_bytes;
1949
1950 path->reada = 1;
1951 path->leave_spinning = 1;
1952 ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
1953 path, 0, 1);
1954 if (ret < 0) {
1955 err = ret;
1956 goto out;
1957 }
1958 if (ret > 0) {
1959 err = -EIO;
1960 goto out;
1961 }
1962
1963 leaf = path->nodes[0];
1964 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1965 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1966 if (item_size < sizeof(*ei)) {
1967 ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
1968 path, (u64)-1, 0);
1969 if (ret < 0) {
1970 err = ret;
1971 goto out;
1972 }
1973 leaf = path->nodes[0];
1974 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1975 }
1976 #endif
1977 BUG_ON(item_size < sizeof(*ei));
1978 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1979 __run_delayed_extent_op(extent_op, leaf, ei);
1980
1981 btrfs_mark_buffer_dirty(leaf);
1982 out:
1983 btrfs_free_path(path);
1984 return err;
1985 }
1986
1987 static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
1988 struct btrfs_root *root,
1989 struct btrfs_delayed_ref_node *node,
1990 struct btrfs_delayed_extent_op *extent_op,
1991 int insert_reserved)
1992 {
1993 int ret = 0;
1994 struct btrfs_delayed_tree_ref *ref;
1995 struct btrfs_key ins;
1996 u64 parent = 0;
1997 u64 ref_root = 0;
1998
1999 ins.objectid = node->bytenr;
2000 ins.offset = node->num_bytes;
2001 ins.type = BTRFS_EXTENT_ITEM_KEY;
2002
2003 ref = btrfs_delayed_node_to_tree_ref(node);
2004 if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
2005 parent = ref->parent;
2006 else
2007 ref_root = ref->root;
2008
2009 BUG_ON(node->ref_mod != 1);
2010 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
2011 BUG_ON(!extent_op || !extent_op->update_flags ||
2012 !extent_op->update_key);
2013 ret = alloc_reserved_tree_block(trans, root,
2014 parent, ref_root,
2015 extent_op->flags_to_set,
2016 &extent_op->key,
2017 ref->level, &ins);
2018 } else if (node->action == BTRFS_ADD_DELAYED_REF) {
2019 ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
2020 node->num_bytes, parent, ref_root,
2021 ref->level, 0, 1, extent_op);
2022 } else if (node->action == BTRFS_DROP_DELAYED_REF) {
2023 ret = __btrfs_free_extent(trans, root, node->bytenr,
2024 node->num_bytes, parent, ref_root,
2025 ref->level, 0, 1, extent_op);
2026 } else {
2027 BUG();
2028 }
2029 return ret;
2030 }
2031
2032 /* helper function to actually process a single delayed ref entry */
2033 static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
2034 struct btrfs_root *root,
2035 struct btrfs_delayed_ref_node *node,
2036 struct btrfs_delayed_extent_op *extent_op,
2037 int insert_reserved)
2038 {
2039 int ret;
2040 if (btrfs_delayed_ref_is_head(node)) {
2041 struct btrfs_delayed_ref_head *head;
2042 /*
2043 * we've hit the end of the chain and we were supposed
2044 * to insert this extent into the tree. But, it got
2045 * deleted before we ever needed to insert it, so all
2046 * we have to do is clean up the accounting
2047 */
2048 BUG_ON(extent_op);
2049 head = btrfs_delayed_node_to_head(node);
2050 if (insert_reserved) {
2051 btrfs_pin_extent(root, node->bytenr,
2052 node->num_bytes, 1);
2053 if (head->is_data) {
2054 ret = btrfs_del_csums(trans, root,
2055 node->bytenr,
2056 node->num_bytes);
2057 BUG_ON(ret);
2058 }
2059 }
2060 mutex_unlock(&head->mutex);
2061 return 0;
2062 }
2063
2064 if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
2065 node->type == BTRFS_SHARED_BLOCK_REF_KEY)
2066 ret = run_delayed_tree_ref(trans, root, node, extent_op,
2067 insert_reserved);
2068 else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
2069 node->type == BTRFS_SHARED_DATA_REF_KEY)
2070 ret = run_delayed_data_ref(trans, root, node, extent_op,
2071 insert_reserved);
2072 else
2073 BUG();
2074 return ret;
2075 }
2076
2077 static noinline struct btrfs_delayed_ref_node *
2078 select_delayed_ref(struct btrfs_delayed_ref_head *head)
2079 {
2080 struct rb_node *node;
2081 struct btrfs_delayed_ref_node *ref;
2082 int action = BTRFS_ADD_DELAYED_REF;
2083 again:
2084 /*
2085 * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
2086 * this prevents ref count from going down to zero when
2087 * there still are pending delayed ref.
2088 */
2089 node = rb_prev(&head->node.rb_node);
2090 while (1) {
2091 if (!node)
2092 break;
2093 ref = rb_entry(node, struct btrfs_delayed_ref_node,
2094 rb_node);
2095 if (ref->bytenr != head->node.bytenr)
2096 break;
2097 if (ref->action == action)
2098 return ref;
2099 node = rb_prev(node);
2100 }
2101 if (action == BTRFS_ADD_DELAYED_REF) {
2102 action = BTRFS_DROP_DELAYED_REF;
2103 goto again;
2104 }
2105 return NULL;
2106 }
2107
2108 static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
2109 struct btrfs_root *root,
2110 struct list_head *cluster)
2111 {
2112 struct btrfs_delayed_ref_root *delayed_refs;
2113 struct btrfs_delayed_ref_node *ref;
2114 struct btrfs_delayed_ref_head *locked_ref = NULL;
2115 struct btrfs_delayed_extent_op *extent_op;
2116 int ret;
2117 int count = 0;
2118 int must_insert_reserved = 0;
2119
2120 delayed_refs = &trans->transaction->delayed_refs;
2121 while (1) {
2122 if (!locked_ref) {
2123 /* pick a new head ref from the cluster list */
2124 if (list_empty(cluster))
2125 break;
2126
2127 locked_ref = list_entry(cluster->next,
2128 struct btrfs_delayed_ref_head, cluster);
2129
2130 /* grab the lock that says we are going to process
2131 * all the refs for this head */
2132 ret = btrfs_delayed_ref_lock(trans, locked_ref);
2133
2134 /*
2135 * we may have dropped the spin lock to get the head
2136 * mutex lock, and that might have given someone else
2137 * time to free the head. If that's true, it has been
2138 * removed from our list and we can move on.
2139 */
2140 if (ret == -EAGAIN) {
2141 locked_ref = NULL;
2142 count++;
2143 continue;
2144 }
2145 }
2146
2147 /*
2148 * record the must insert reserved flag before we
2149 * drop the spin lock.
2150 */
2151 must_insert_reserved = locked_ref->must_insert_reserved;
2152 locked_ref->must_insert_reserved = 0;
2153
2154 extent_op = locked_ref->extent_op;
2155 locked_ref->extent_op = NULL;
2156
2157 /*
2158 * locked_ref is the head node, so we have to go one
2159 * node back for any delayed ref updates
2160 */
2161 ref = select_delayed_ref(locked_ref);
2162 if (!ref) {
2163 /* All delayed refs have been processed, Go ahead
2164 * and send the head node to run_one_delayed_ref,
2165 * so that any accounting fixes can happen
2166 */
2167 ref = &locked_ref->node;
2168
2169 if (extent_op && must_insert_reserved) {
2170 kfree(extent_op);
2171 extent_op = NULL;
2172 }
2173
2174 if (extent_op) {
2175 spin_unlock(&delayed_refs->lock);
2176
2177 ret = run_delayed_extent_op(trans, root,
2178 ref, extent_op);
2179 BUG_ON(ret);
2180 kfree(extent_op);
2181
2182 cond_resched();
2183 spin_lock(&delayed_refs->lock);
2184 continue;
2185 }
2186
2187 list_del_init(&locked_ref->cluster);
2188 locked_ref = NULL;
2189 }
2190
2191 ref->in_tree = 0;
2192 rb_erase(&ref->rb_node, &delayed_refs->root);
2193 delayed_refs->num_entries--;
2194
2195 spin_unlock(&delayed_refs->lock);
2196
2197 ret = run_one_delayed_ref(trans, root, ref, extent_op,
2198 must_insert_reserved);
2199 BUG_ON(ret);
2200
2201 btrfs_put_delayed_ref(ref);
2202 kfree(extent_op);
2203 count++;
2204
2205 cond_resched();
2206 spin_lock(&delayed_refs->lock);
2207 }
2208 return count;
2209 }
2210
2211 /*
2212 * this starts processing the delayed reference count updates and
2213 * extent insertions we have queued up so far. count can be
2214 * 0, which means to process everything in the tree at the start
2215 * of the run (but not newly added entries), or it can be some target
2216 * number you'd like to process.
2217 */
2218 int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
2219 struct btrfs_root *root, unsigned long count)
2220 {
2221 struct rb_node *node;
2222 struct btrfs_delayed_ref_root *delayed_refs;
2223 struct btrfs_delayed_ref_node *ref;
2224 struct list_head cluster;
2225 int ret;
2226 int run_all = count == (unsigned long)-1;
2227 int run_most = 0;
2228
2229 if (root == root->fs_info->extent_root)
2230 root = root->fs_info->tree_root;
2231
2232 delayed_refs = &trans->transaction->delayed_refs;
2233 INIT_LIST_HEAD(&cluster);
2234 again:
2235 spin_lock(&delayed_refs->lock);
2236 if (count == 0) {
2237 count = delayed_refs->num_entries * 2;
2238 run_most = 1;
2239 }
2240 while (1) {
2241 if (!(run_all || run_most) &&
2242 delayed_refs->num_heads_ready < 64)
2243 break;
2244
2245 /*
2246 * go find something we can process in the rbtree. We start at
2247 * the beginning of the tree, and then build a cluster
2248 * of refs to process starting at the first one we are able to
2249 * lock
2250 */
2251 ret = btrfs_find_ref_cluster(trans, &cluster,
2252 delayed_refs->run_delayed_start);
2253 if (ret)
2254 break;
2255
2256 ret = run_clustered_refs(trans, root, &cluster);
2257 BUG_ON(ret < 0);
2258
2259 count -= min_t(unsigned long, ret, count);
2260
2261 if (count == 0)
2262 break;
2263 }
2264
2265 if (run_all) {
2266 node = rb_first(&delayed_refs->root);
2267 if (!node)
2268 goto out;
2269 count = (unsigned long)-1;
2270
2271 while (node) {
2272 ref = rb_entry(node, struct btrfs_delayed_ref_node,
2273 rb_node);
2274 if (btrfs_delayed_ref_is_head(ref)) {
2275 struct btrfs_delayed_ref_head *head;
2276
2277 head = btrfs_delayed_node_to_head(ref);
2278 atomic_inc(&ref->refs);
2279
2280 spin_unlock(&delayed_refs->lock);
2281 mutex_lock(&head->mutex);
2282 mutex_unlock(&head->mutex);
2283
2284 btrfs_put_delayed_ref(ref);
2285 cond_resched();
2286 goto again;
2287 }
2288 node = rb_next(node);
2289 }
2290 spin_unlock(&delayed_refs->lock);
2291 schedule_timeout(1);
2292 goto again;
2293 }
2294 out:
2295 spin_unlock(&delayed_refs->lock);
2296 return 0;
2297 }
2298
2299 int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
2300 struct btrfs_root *root,
2301 u64 bytenr, u64 num_bytes, u64 flags,
2302 int is_data)
2303 {
2304 struct btrfs_delayed_extent_op *extent_op;
2305 int ret;
2306
2307 extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
2308 if (!extent_op)
2309 return -ENOMEM;
2310
2311 extent_op->flags_to_set = flags;
2312 extent_op->update_flags = 1;
2313 extent_op->update_key = 0;
2314 extent_op->is_data = is_data ? 1 : 0;
2315
2316 ret = btrfs_add_delayed_extent_op(trans, bytenr, num_bytes, extent_op);
2317 if (ret)
2318 kfree(extent_op);
2319 return ret;
2320 }
2321
2322 static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
2323 struct btrfs_root *root,
2324 struct btrfs_path *path,
2325 u64 objectid, u64 offset, u64 bytenr)
2326 {
2327 struct btrfs_delayed_ref_head *head;
2328 struct btrfs_delayed_ref_node *ref;
2329 struct btrfs_delayed_data_ref *data_ref;
2330 struct btrfs_delayed_ref_root *delayed_refs;
2331 struct rb_node *node;
2332 int ret = 0;
2333
2334 ret = -ENOENT;
2335 delayed_refs = &trans->transaction->delayed_refs;
2336 spin_lock(&delayed_refs->lock);
2337 head = btrfs_find_delayed_ref_head(trans, bytenr);
2338 if (!head)
2339 goto out;
2340
2341 if (!mutex_trylock(&head->mutex)) {
2342 atomic_inc(&head->node.refs);
2343 spin_unlock(&delayed_refs->lock);
2344
2345 btrfs_release_path(root->fs_info->extent_root, path);
2346
2347 mutex_lock(&head->mutex);
2348 mutex_unlock(&head->mutex);
2349 btrfs_put_delayed_ref(&head->node);
2350 return -EAGAIN;
2351 }
2352
2353 node = rb_prev(&head->node.rb_node);
2354 if (!node)
2355 goto out_unlock;
2356
2357 ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
2358
2359 if (ref->bytenr != bytenr)
2360 goto out_unlock;
2361
2362 ret = 1;
2363 if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
2364 goto out_unlock;
2365
2366 data_ref = btrfs_delayed_node_to_data_ref(ref);
2367
2368 node = rb_prev(node);
2369 if (node) {
2370 ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
2371 if (ref->bytenr == bytenr)
2372 goto out_unlock;
2373 }
2374
2375 if (data_ref->root != root->root_key.objectid ||
2376 data_ref->objectid != objectid || data_ref->offset != offset)
2377 goto out_unlock;
2378
2379 ret = 0;
2380 out_unlock:
2381 mutex_unlock(&head->mutex);
2382 out:
2383 spin_unlock(&delayed_refs->lock);
2384 return ret;
2385 }
2386
2387 static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
2388 struct btrfs_root *root,
2389 struct btrfs_path *path,
2390 u64 objectid, u64 offset, u64 bytenr)
2391 {
2392 struct btrfs_root *extent_root = root->fs_info->extent_root;
2393 struct extent_buffer *leaf;
2394 struct btrfs_extent_data_ref *ref;
2395 struct btrfs_extent_inline_ref *iref;
2396 struct btrfs_extent_item *ei;
2397 struct btrfs_key key;
2398 u32 item_size;
2399 int ret;
2400
2401 key.objectid = bytenr;
2402 key.offset = (u64)-1;
2403 key.type = BTRFS_EXTENT_ITEM_KEY;
2404
2405 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
2406 if (ret < 0)
2407 goto out;
2408 BUG_ON(ret == 0);
2409
2410 ret = -ENOENT;
2411 if (path->slots[0] == 0)
2412 goto out;
2413
2414 path->slots[0]--;
2415 leaf = path->nodes[0];
2416 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2417
2418 if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
2419 goto out;
2420
2421 ret = 1;
2422 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
2423 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
2424 if (item_size < sizeof(*ei)) {
2425 WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
2426 goto out;
2427 }
2428 #endif
2429 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2430
2431 if (item_size != sizeof(*ei) +
2432 btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
2433 goto out;
2434
2435 if (btrfs_extent_generation(leaf, ei) <=
2436 btrfs_root_last_snapshot(&root->root_item))
2437 goto out;
2438
2439 iref = (struct btrfs_extent_inline_ref *)(ei + 1);
2440 if (btrfs_extent_inline_ref_type(leaf, iref) !=
2441 BTRFS_EXTENT_DATA_REF_KEY)
2442 goto out;
2443
2444 ref = (struct btrfs_extent_data_ref *)(&iref->offset);
2445 if (btrfs_extent_refs(leaf, ei) !=
2446 btrfs_extent_data_ref_count(leaf, ref) ||
2447 btrfs_extent_data_ref_root(leaf, ref) !=
2448 root->root_key.objectid ||
2449 btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
2450 btrfs_extent_data_ref_offset(leaf, ref) != offset)
2451 goto out;
2452
2453 ret = 0;
2454 out:
2455 return ret;
2456 }
2457
2458 int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
2459 struct btrfs_root *root,
2460 u64 objectid, u64 offset, u64 bytenr)
2461 {
2462 struct btrfs_path *path;
2463 int ret;
2464 int ret2;
2465
2466 path = btrfs_alloc_path();
2467 if (!path)
2468 return -ENOENT;
2469
2470 do {
2471 ret = check_committed_ref(trans, root, path, objectid,
2472 offset, bytenr);
2473 if (ret && ret != -ENOENT)
2474 goto out;
2475
2476 ret2 = check_delayed_ref(trans, root, path, objectid,
2477 offset, bytenr);
2478 } while (ret2 == -EAGAIN);
2479
2480 if (ret2 && ret2 != -ENOENT) {
2481 ret = ret2;
2482 goto out;
2483 }
2484
2485 if (ret != -ENOENT || ret2 != -ENOENT)
2486 ret = 0;
2487 out:
2488 btrfs_free_path(path);
2489 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
2490 WARN_ON(ret > 0);
2491 return ret;
2492 }
2493
2494 #if 0
2495 int btrfs_cache_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
2496 struct extent_buffer *buf, u32 nr_extents)
2497 {
2498 struct btrfs_key key;
2499 struct btrfs_file_extent_item *fi;
2500 u64 root_gen;
2501 u32 nritems;
2502 int i;
2503 int level;
2504 int ret = 0;
2505 int shared = 0;
2506
2507 if (!root->ref_cows)
2508 return 0;
2509
2510 if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
2511 shared = 0;
2512 root_gen = root->root_key.offset;
2513 } else {
2514 shared = 1;
2515 root_gen = trans->transid - 1;
2516 }
2517
2518 level = btrfs_header_level(buf);
2519 nritems = btrfs_header_nritems(buf);
2520
2521 if (level == 0) {
2522 struct btrfs_leaf_ref *ref;
2523 struct btrfs_extent_info *info;
2524
2525 ref = btrfs_alloc_leaf_ref(root, nr_extents);
2526 if (!ref) {
2527 ret = -ENOMEM;
2528 goto out;
2529 }
2530
2531 ref->root_gen = root_gen;
2532 ref->bytenr = buf->start;
2533 ref->owner = btrfs_header_owner(buf);
2534 ref->generation = btrfs_header_generation(buf);
2535 ref->nritems = nr_extents;
2536 info = ref->extents;
2537
2538 for (i = 0; nr_extents > 0 && i < nritems; i++) {
2539 u64 disk_bytenr;
2540 btrfs_item_key_to_cpu(buf, &key, i);
2541 if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
2542 continue;
2543 fi = btrfs_item_ptr(buf, i,
2544 struct btrfs_file_extent_item);
2545 if (btrfs_file_extent_type(buf, fi) ==
2546 BTRFS_FILE_EXTENT_INLINE)
2547 continue;
2548 disk_bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
2549 if (disk_bytenr == 0)
2550 continue;
2551
2552 info->bytenr = disk_bytenr;
2553 info->num_bytes =
2554 btrfs_file_extent_disk_num_bytes(buf, fi);
2555 info->objectid = key.objectid;
2556 info->offset = key.offset;
2557 info++;
2558 }
2559
2560 ret = btrfs_add_leaf_ref(root, ref, shared);
2561 if (ret == -EEXIST && shared) {
2562 struct btrfs_leaf_ref *old;
2563 old = btrfs_lookup_leaf_ref(root, ref->bytenr);
2564 BUG_ON(!old);
2565 btrfs_remove_leaf_ref(root, old);
2566 btrfs_free_leaf_ref(root, old);
2567 ret = btrfs_add_leaf_ref(root, ref, shared);
2568 }
2569 WARN_ON(ret);
2570 btrfs_free_leaf_ref(root, ref);
2571 }
2572 out:
2573 return ret;
2574 }
2575
2576 /* when a block goes through cow, we update the reference counts of
2577 * everything that block points to. The internal pointers of the block
2578 * can be in just about any order, and it is likely to have clusters of
2579 * things that are close together and clusters of things that are not.
2580 *
2581 * To help reduce the seeks that come with updating all of these reference
2582 * counts, sort them by byte number before actual updates are done.
2583 *
2584 * struct refsort is used to match byte number to slot in the btree block.
2585 * we sort based on the byte number and then use the slot to actually
2586 * find the item.
2587 *
2588 * struct refsort is smaller than strcut btrfs_item and smaller than
2589 * struct btrfs_key_ptr. Since we're currently limited to the page size
2590 * for a btree block, there's no way for a kmalloc of refsorts for a
2591 * single node to be bigger than a page.
2592 */
2593 struct refsort {
2594 u64 bytenr;
2595 u32 slot;
2596 };
2597
2598 /*
2599 * for passing into sort()
2600 */
2601 static int refsort_cmp(const void *a_void, const void *b_void)
2602 {
2603 const struct refsort *a = a_void;
2604 const struct refsort *b = b_void;
2605
2606 if (a->bytenr < b->bytenr)
2607 return -1;
2608 if (a->bytenr > b->bytenr)
2609 return 1;
2610 return 0;
2611 }
2612 #endif
2613
2614 static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
2615 struct btrfs_root *root,
2616 struct extent_buffer *buf,
2617 int full_backref, int inc)
2618 {
2619 u64 bytenr;
2620 u64 num_bytes;
2621 u64 parent;
2622 u64 ref_root;
2623 u32 nritems;
2624 struct btrfs_key key;
2625 struct btrfs_file_extent_item *fi;
2626 int i;
2627 int level;
2628 int ret = 0;
2629 int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
2630 u64, u64, u64, u64, u64, u64);
2631
2632 ref_root = btrfs_header_owner(buf);
2633 nritems = btrfs_header_nritems(buf);
2634 level = btrfs_header_level(buf);
2635
2636 if (!root->ref_cows && level == 0)
2637 return 0;
2638
2639 if (inc)
2640 process_func = btrfs_inc_extent_ref;
2641 else
2642 process_func = btrfs_free_extent;
2643
2644 if (full_backref)
2645 parent = buf->start;
2646 else
2647 parent = 0;
2648
2649 for (i = 0; i < nritems; i++) {
2650 if (level == 0) {
2651 btrfs_item_key_to_cpu(buf, &key, i);
2652 if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
2653 continue;
2654 fi = btrfs_item_ptr(buf, i,
2655 struct btrfs_file_extent_item);
2656 if (btrfs_file_extent_type(buf, fi) ==
2657 BTRFS_FILE_EXTENT_INLINE)
2658 continue;
2659 bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
2660 if (bytenr == 0)
2661 continue;
2662
2663 num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
2664 key.offset -= btrfs_file_extent_offset(buf, fi);
2665 ret = process_func(trans, root, bytenr, num_bytes,
2666 parent, ref_root, key.objectid,
2667 key.offset);
2668 if (ret)
2669 goto fail;
2670 } else {
2671 bytenr = btrfs_node_blockptr(buf, i);
2672 num_bytes = btrfs_level_size(root, level - 1);
2673 ret = process_func(trans, root, bytenr, num_bytes,
2674 parent, ref_root, level - 1, 0);
2675 if (ret)
2676 goto fail;
2677 }
2678 }
2679 return 0;
2680 fail:
2681 BUG();
2682 return ret;
2683 }
2684
2685 int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
2686 struct extent_buffer *buf, int full_backref)
2687 {
2688 return __btrfs_mod_ref(trans, root, buf, full_backref, 1);
2689 }
2690
2691 int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
2692 struct extent_buffer *buf, int full_backref)
2693 {
2694 return __btrfs_mod_ref(trans, root, buf, full_backref, 0);
2695 }
2696
2697 static int write_one_cache_group(struct btrfs_trans_handle *trans,
2698 struct btrfs_root *root,
2699 struct btrfs_path *path,
2700 struct btrfs_block_group_cache *cache)
2701 {
2702 int ret;
2703 struct btrfs_root *extent_root = root->fs_info->extent_root;
2704 unsigned long bi;
2705 struct extent_buffer *leaf;
2706
2707 ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
2708 if (ret < 0)
2709 goto fail;
2710 BUG_ON(ret);
2711
2712 leaf = path->nodes[0];
2713 bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
2714 write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
2715 btrfs_mark_buffer_dirty(leaf);
2716 btrfs_release_path(extent_root, path);
2717 fail:
2718 if (ret)
2719 return ret;
2720 return 0;
2721
2722 }
2723
2724 static struct btrfs_block_group_cache *
2725 next_block_group(struct btrfs_root *root,
2726 struct btrfs_block_group_cache *cache)
2727 {
2728 struct rb_node *node;
2729 spin_lock(&root->fs_info->block_group_cache_lock);
2730 node = rb_next(&cache->cache_node);
2731 btrfs_put_block_group(cache);
2732 if (node) {
2733 cache = rb_entry(node, struct btrfs_block_group_cache,
2734 cache_node);
2735 btrfs_get_block_group(cache);
2736 } else
2737 cache = NULL;
2738 spin_unlock(&root->fs_info->block_group_cache_lock);
2739 return cache;
2740 }
2741
2742 static int cache_save_setup(struct btrfs_block_group_cache *block_group,
2743 struct btrfs_trans_handle *trans,
2744 struct btrfs_path *path)
2745 {
2746 struct btrfs_root *root = block_group->fs_info->tree_root;
2747 struct inode *inode = NULL;
2748 u64 alloc_hint = 0;
2749 int dcs = BTRFS_DC_ERROR;
2750 int num_pages = 0;
2751 int retries = 0;
2752 int ret = 0;
2753
2754 /*
2755 * If this block group is smaller than 100 megs don't bother caching the
2756 * block group.
2757 */
2758 if (block_group->key.offset < (100 * 1024 * 1024)) {
2759 spin_lock(&block_group->lock);
2760 block_group->disk_cache_state = BTRFS_DC_WRITTEN;
2761 spin_unlock(&block_group->lock);
2762 return 0;
2763 }
2764
2765 again:
2766 inode = lookup_free_space_inode(root, block_group, path);
2767 if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
2768 ret = PTR_ERR(inode);
2769 btrfs_release_path(root, path);
2770 goto out;
2771 }
2772
2773 if (IS_ERR(inode)) {
2774 BUG_ON(retries);
2775 retries++;
2776
2777 if (block_group->ro)
2778 goto out_free;
2779
2780 ret = create_free_space_inode(root, trans, block_group, path);
2781 if (ret)
2782 goto out_free;
2783 goto again;
2784 }
2785
2786 /*
2787 * We want to set the generation to 0, that way if anything goes wrong
2788 * from here on out we know not to trust this cache when we load up next
2789 * time.
2790 */
2791 BTRFS_I(inode)->generation = 0;
2792 ret = btrfs_update_inode(trans, root, inode);
2793 WARN_ON(ret);
2794
2795 if (i_size_read(inode) > 0) {
2796 ret = btrfs_truncate_free_space_cache(root, trans, path,
2797 inode);
2798 if (ret)
2799 goto out_put;
2800 }
2801
2802 spin_lock(&block_group->lock);
2803 if (block_group->cached != BTRFS_CACHE_FINISHED) {
2804 /* We're not cached, don't bother trying to write stuff out */
2805 dcs = BTRFS_DC_WRITTEN;
2806 spin_unlock(&block_group->lock);
2807 goto out_put;
2808 }
2809 spin_unlock(&block_group->lock);
2810
2811 num_pages = (int)div64_u64(block_group->key.offset, 1024 * 1024 * 1024);
2812 if (!num_pages)
2813 num_pages = 1;
2814
2815 /*
2816 * Just to make absolutely sure we have enough space, we're going to
2817 * preallocate 12 pages worth of space for each block group. In
2818 * practice we ought to use at most 8, but we need extra space so we can
2819 * add our header and have a terminator between the extents and the
2820 * bitmaps.
2821 */
2822 num_pages *= 16;
2823 num_pages *= PAGE_CACHE_SIZE;
2824
2825 ret = btrfs_check_data_free_space(inode, num_pages);
2826 if (ret)
2827 goto out_put;
2828
2829 ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
2830 num_pages, num_pages,
2831 &alloc_hint);
2832 if (!ret)
2833 dcs = BTRFS_DC_SETUP;
2834 btrfs_free_reserved_data_space(inode, num_pages);
2835 out_put:
2836 iput(inode);
2837 out_free:
2838 btrfs_release_path(root, path);
2839 out:
2840 spin_lock(&block_group->lock);
2841 block_group->disk_cache_state = dcs;
2842 spin_unlock(&block_group->lock);
2843
2844 return ret;
2845 }
2846
2847 int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
2848 struct btrfs_root *root)
2849 {
2850 struct btrfs_block_group_cache *cache;
2851 int err = 0;
2852 struct btrfs_path *path;
2853 u64 last = 0;
2854
2855 path = btrfs_alloc_path();
2856 if (!path)
2857 return -ENOMEM;
2858
2859 again:
2860 while (1) {
2861 cache = btrfs_lookup_first_block_group(root->fs_info, last);
2862 while (cache) {
2863 if (cache->disk_cache_state == BTRFS_DC_CLEAR)
2864 break;
2865 cache = next_block_group(root, cache);
2866 }
2867 if (!cache) {
2868 if (last == 0)
2869 break;
2870 last = 0;
2871 continue;
2872 }
2873 err = cache_save_setup(cache, trans, path);
2874 last = cache->key.objectid + cache->key.offset;
2875 btrfs_put_block_group(cache);
2876 }
2877
2878 while (1) {
2879 if (last == 0) {
2880 err = btrfs_run_delayed_refs(trans, root,
2881 (unsigned long)-1);
2882 BUG_ON(err);
2883 }
2884
2885 cache = btrfs_lookup_first_block_group(root->fs_info, last);
2886 while (cache) {
2887 if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
2888 btrfs_put_block_group(cache);
2889 goto again;
2890 }
2891
2892 if (cache->dirty)
2893 break;
2894 cache = next_block_group(root, cache);
2895 }
2896 if (!cache) {
2897 if (last == 0)
2898 break;
2899 last = 0;
2900 continue;
2901 }
2902
2903 if (cache->disk_cache_state == BTRFS_DC_SETUP)
2904 cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
2905 cache->dirty = 0;
2906 last = cache->key.objectid + cache->key.offset;
2907
2908 err = write_one_cache_group(trans, root, path, cache);
2909 BUG_ON(err);
2910 btrfs_put_block_group(cache);
2911 }
2912
2913 while (1) {
2914 /*
2915 * I don't think this is needed since we're just marking our
2916 * preallocated extent as written, but just in case it can't
2917 * hurt.
2918 */
2919 if (last == 0) {
2920 err = btrfs_run_delayed_refs(trans, root,
2921 (unsigned long)-1);
2922 BUG_ON(err);
2923 }
2924
2925 cache = btrfs_lookup_first_block_group(root->fs_info, last);
2926 while (cache) {
2927 /*
2928 * Really this shouldn't happen, but it could if we
2929 * couldn't write the entire preallocated extent and
2930 * splitting the extent resulted in a new block.
2931 */
2932 if (cache->dirty) {
2933 btrfs_put_block_group(cache);
2934 goto again;
2935 }
2936 if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
2937 break;
2938 cache = next_block_group(root, cache);
2939 }
2940 if (!cache) {
2941 if (last == 0)
2942 break;
2943 last = 0;
2944 continue;
2945 }
2946
2947 btrfs_write_out_cache(root, trans, cache, path);
2948
2949 /*
2950 * If we didn't have an error then the cache state is still
2951 * NEED_WRITE, so we can set it to WRITTEN.
2952 */
2953 if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
2954 cache->disk_cache_state = BTRFS_DC_WRITTEN;
2955 last = cache->key.objectid + cache->key.offset;
2956 btrfs_put_block_group(cache);
2957 }
2958
2959 btrfs_free_path(path);
2960 return 0;
2961 }
2962
2963 int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
2964 {
2965 struct btrfs_block_group_cache *block_group;
2966 int readonly = 0;
2967
2968 block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
2969 if (!block_group || block_group->ro)
2970 readonly = 1;
2971 if (block_group)
2972 btrfs_put_block_group(block_group);
2973 return readonly;
2974 }
2975
2976 static int update_space_info(struct btrfs_fs_info *info, u64 flags,
2977 u64 total_bytes, u64 bytes_used,
2978 struct btrfs_space_info **space_info)
2979 {
2980 struct btrfs_space_info *found;
2981 int i;
2982 int factor;
2983
2984 if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
2985 BTRFS_BLOCK_GROUP_RAID10))
2986 factor = 2;
2987 else
2988 factor = 1;
2989
2990 found = __find_space_info(info, flags);
2991 if (found) {
2992 spin_lock(&found->lock);
2993 found->total_bytes += total_bytes;
2994 found->disk_total += total_bytes * factor;
2995 found->bytes_used += bytes_used;
2996 found->disk_used += bytes_used * factor;
2997 found->full = 0;
2998 spin_unlock(&found->lock);
2999 *space_info = found;
3000 return 0;
3001 }
3002 found = kzalloc(sizeof(*found), GFP_NOFS);
3003 if (!found)
3004 return -ENOMEM;
3005
3006 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
3007 INIT_LIST_HEAD(&found->block_groups[i]);
3008 init_rwsem(&found->groups_sem);
3009 spin_lock_init(&found->lock);
3010 found->flags = flags & (BTRFS_BLOCK_GROUP_DATA |
3011 BTRFS_BLOCK_GROUP_SYSTEM |
3012 BTRFS_BLOCK_GROUP_METADATA);
3013 found->total_bytes = total_bytes;
3014 found->disk_total = total_bytes * factor;
3015 found->bytes_used = bytes_used;
3016 found->disk_used = bytes_used * factor;
3017 found->bytes_pinned = 0;
3018 found->bytes_reserved = 0;
3019 found->bytes_readonly = 0;
3020 found->bytes_may_use = 0;
3021 found->full = 0;
3022 found->force_alloc = 0;
3023 *space_info = found;
3024 list_add_rcu(&found->list, &info->space_info);
3025 atomic_set(&found->caching_threads, 0);
3026 return 0;
3027 }
3028
3029 static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
3030 {
3031 u64 extra_flags = flags & (BTRFS_BLOCK_GROUP_RAID0 |
3032 BTRFS_BLOCK_GROUP_RAID1 |
3033 BTRFS_BLOCK_GROUP_RAID10 |
3034 BTRFS_BLOCK_GROUP_DUP);
3035 if (extra_flags) {
3036 if (flags & BTRFS_BLOCK_GROUP_DATA)
3037 fs_info->avail_data_alloc_bits |= extra_flags;
3038 if (flags & BTRFS_BLOCK_GROUP_METADATA)
3039 fs_info->avail_metadata_alloc_bits |= extra_flags;
3040 if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
3041 fs_info->avail_system_alloc_bits |= extra_flags;
3042 }
3043 }
3044
3045 u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
3046 {
3047 /*
3048 * we add in the count of missing devices because we want
3049 * to make sure that any RAID levels on a degraded FS
3050 * continue to be honored.
3051 */
3052 u64 num_devices = root->fs_info->fs_devices->rw_devices +
3053 root->fs_info->fs_devices->missing_devices;
3054
3055 if (num_devices == 1)
3056 flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0);
3057 if (num_devices < 4)
3058 flags &= ~BTRFS_BLOCK_GROUP_RAID10;
3059
3060 if ((flags & BTRFS_BLOCK_GROUP_DUP) &&
3061 (flags & (BTRFS_BLOCK_GROUP_RAID1 |
3062 BTRFS_BLOCK_GROUP_RAID10))) {
3063 flags &= ~BTRFS_BLOCK_GROUP_DUP;
3064 }
3065
3066 if ((flags & BTRFS_BLOCK_GROUP_RAID1) &&
3067 (flags & BTRFS_BLOCK_GROUP_RAID10)) {
3068 flags &= ~BTRFS_BLOCK_GROUP_RAID1;
3069 }
3070
3071 if ((flags & BTRFS_BLOCK_GROUP_RAID0) &&
3072 ((flags & BTRFS_BLOCK_GROUP_RAID1) |
3073 (flags & BTRFS_BLOCK_GROUP_RAID10) |
3074 (flags & BTRFS_BLOCK_GROUP_DUP)))
3075 flags &= ~BTRFS_BLOCK_GROUP_RAID0;
3076 return flags;
3077 }
3078
3079 static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
3080 {
3081 if (flags & BTRFS_BLOCK_GROUP_DATA)
3082 flags |= root->fs_info->avail_data_alloc_bits &
3083 root->fs_info->data_alloc_profile;
3084 else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
3085 flags |= root->fs_info->avail_system_alloc_bits &
3086 root->fs_info->system_alloc_profile;
3087 else if (flags & BTRFS_BLOCK_GROUP_METADATA)
3088 flags |= root->fs_info->avail_metadata_alloc_bits &
3089 root->fs_info->metadata_alloc_profile;
3090 return btrfs_reduce_alloc_profile(root, flags);
3091 }
3092
3093 static u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
3094 {
3095 u64 flags;
3096
3097 if (data)
3098 flags = BTRFS_BLOCK_GROUP_DATA;
3099 else if (root == root->fs_info->chunk_root)
3100 flags = BTRFS_BLOCK_GROUP_SYSTEM;
3101 else
3102 flags = BTRFS_BLOCK_GROUP_METADATA;
3103
3104 return get_alloc_profile(root, flags);
3105 }
3106
3107 void btrfs_set_inode_space_info(struct btrfs_root *root, struct inode *inode)
3108 {
3109 BTRFS_I(inode)->space_info = __find_space_info(root->fs_info,
3110 BTRFS_BLOCK_GROUP_DATA);
3111 }
3112
3113 /*
3114 * This will check the space that the inode allocates from to make sure we have
3115 * enough space for bytes.
3116 */
3117 int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
3118 {
3119 struct btrfs_space_info *data_sinfo;
3120 struct btrfs_root *root = BTRFS_I(inode)->root;
3121 u64 used;
3122 int ret = 0, committed = 0, alloc_chunk = 1;
3123
3124 /* make sure bytes are sectorsize aligned */
3125 bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
3126
3127 if (root == root->fs_info->tree_root) {
3128 alloc_chunk = 0;
3129 committed = 1;
3130 }
3131
3132 data_sinfo = BTRFS_I(inode)->space_info;
3133 if (!data_sinfo)
3134 goto alloc;
3135
3136 again:
3137 /* make sure we have enough space to handle the data first */
3138 spin_lock(&data_sinfo->lock);
3139 used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
3140 data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
3141 data_sinfo->bytes_may_use;
3142
3143 if (used + bytes > data_sinfo->total_bytes) {
3144 struct btrfs_trans_handle *trans;
3145
3146 /*
3147 * if we don't have enough free bytes in this space then we need
3148 * to alloc a new chunk.
3149 */
3150 if (!data_sinfo->full && alloc_chunk) {
3151 u64 alloc_target;
3152
3153 data_sinfo->force_alloc = 1;
3154 spin_unlock(&data_sinfo->lock);
3155 alloc:
3156 alloc_target = btrfs_get_alloc_profile(root, 1);
3157 trans = btrfs_join_transaction(root, 1);
3158 if (IS_ERR(trans))
3159 return PTR_ERR(trans);
3160
3161 ret = do_chunk_alloc(trans, root->fs_info->extent_root,
3162 bytes + 2 * 1024 * 1024,
3163 alloc_target, 0);
3164 btrfs_end_transaction(trans, root);
3165 if (ret < 0)
3166 return ret;
3167
3168 if (!data_sinfo) {
3169 btrfs_set_inode_space_info(root, inode);
3170 data_sinfo = BTRFS_I(inode)->space_info;
3171 }
3172 goto again;
3173 }
3174 spin_unlock(&data_sinfo->lock);
3175
3176 /* commit the current transaction and try again */
3177 if (!committed && !root->fs_info->open_ioctl_trans) {
3178 committed = 1;
3179 trans = btrfs_join_transaction(root, 1);
3180 if (IS_ERR(trans))
3181 return PTR_ERR(trans);
3182 ret = btrfs_commit_transaction(trans, root);
3183 if (ret)
3184 return ret;
3185 goto again;
3186 }
3187
3188 #if 0 /* I hope we never need this code again, just in case */
3189 printk(KERN_ERR "no space left, need %llu, %llu bytes_used, "
3190 "%llu bytes_reserved, " "%llu bytes_pinned, "
3191 "%llu bytes_readonly, %llu may use %llu total\n",
3192 (unsigned long long)bytes,
3193 (unsigned long long)data_sinfo->bytes_used,
3194 (unsigned long long)data_sinfo->bytes_reserved,
3195 (unsigned long long)data_sinfo->bytes_pinned,
3196 (unsigned long long)data_sinfo->bytes_readonly,
3197 (unsigned long long)data_sinfo->bytes_may_use,
3198 (unsigned long long)data_sinfo->total_bytes);
3199 #endif
3200 return -ENOSPC;
3201 }
3202 data_sinfo->bytes_may_use += bytes;
3203 BTRFS_I(inode)->reserved_bytes += bytes;
3204 spin_unlock(&data_sinfo->lock);
3205
3206 return 0;
3207 }
3208
3209 /*
3210 * called when we are clearing an delalloc extent from the
3211 * inode's io_tree or there was an error for whatever reason
3212 * after calling btrfs_check_data_free_space
3213 */
3214 void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
3215 {
3216 struct btrfs_root *root = BTRFS_I(inode)->root;
3217 struct btrfs_space_info *data_sinfo;
3218
3219 /* make sure bytes are sectorsize aligned */
3220 bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
3221
3222 data_sinfo = BTRFS_I(inode)->space_info;
3223 spin_lock(&data_sinfo->lock);
3224 data_sinfo->bytes_may_use -= bytes;
3225 BTRFS_I(inode)->reserved_bytes -= bytes;
3226 spin_unlock(&data_sinfo->lock);
3227 }
3228
3229 static void force_metadata_allocation(struct btrfs_fs_info *info)
3230 {
3231 struct list_head *head = &info->space_info;
3232 struct btrfs_space_info *found;
3233
3234 rcu_read_lock();
3235 list_for_each_entry_rcu(found, head, list) {
3236 if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3237 found->force_alloc = 1;
3238 }
3239 rcu_read_unlock();
3240 }
3241
3242 static int should_alloc_chunk(struct btrfs_root *root,
3243 struct btrfs_space_info *sinfo, u64 alloc_bytes)
3244 {
3245 u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
3246 u64 thresh;
3247
3248 if (sinfo->bytes_used + sinfo->bytes_reserved +
3249 alloc_bytes + 256 * 1024 * 1024 < num_bytes)
3250 return 0;
3251
3252 if (sinfo->bytes_used + sinfo->bytes_reserved +
3253 alloc_bytes < div_factor(num_bytes, 8))
3254 return 0;
3255
3256 thresh = btrfs_super_total_bytes(&root->fs_info->super_copy);
3257 thresh = max_t(u64, 256 * 1024 * 1024, div_factor_fine(thresh, 5));
3258
3259 if (num_bytes > thresh && sinfo->bytes_used < div_factor(num_bytes, 3))
3260 return 0;
3261
3262 return 1;
3263 }
3264
3265 static int do_chunk_alloc(struct btrfs_trans_handle *trans,
3266 struct btrfs_root *extent_root, u64 alloc_bytes,
3267 u64 flags, int force)
3268 {
3269 struct btrfs_space_info *space_info;
3270 struct btrfs_fs_info *fs_info = extent_root->fs_info;
3271 int ret = 0;
3272
3273 mutex_lock(&fs_info->chunk_mutex);
3274
3275 flags = btrfs_reduce_alloc_profile(extent_root, flags);
3276
3277 space_info = __find_space_info(extent_root->fs_info, flags);
3278 if (!space_info) {
3279 ret = update_space_info(extent_root->fs_info, flags,
3280 0, 0, &space_info);
3281 BUG_ON(ret);
3282 }
3283 BUG_ON(!space_info);
3284
3285 spin_lock(&space_info->lock);
3286 if (space_info->force_alloc)
3287 force = 1;
3288 if (space_info->full) {
3289 spin_unlock(&space_info->lock);
3290 goto out;
3291 }
3292
3293 if (!force && !should_alloc_chunk(extent_root, space_info,
3294 alloc_bytes)) {
3295 spin_unlock(&space_info->lock);
3296 goto out;
3297 }
3298 spin_unlock(&space_info->lock);
3299
3300 /*
3301 * If we have mixed data/metadata chunks we want to make sure we keep
3302 * allocating mixed chunks instead of individual chunks.
3303 */
3304 if (btrfs_mixed_space_info(space_info))
3305 flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);
3306
3307 /*
3308 * if we're doing a data chunk, go ahead and make sure that
3309 * we keep a reasonable number of metadata chunks allocated in the
3310 * FS as well.
3311 */
3312 if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3313 fs_info->data_chunk_allocations++;
3314 if (!(fs_info->data_chunk_allocations %
3315 fs_info->metadata_ratio))
3316 force_metadata_allocation(fs_info);
3317 }
3318
3319 ret = btrfs_alloc_chunk(trans, extent_root, flags);
3320 spin_lock(&space_info->lock);
3321 if (ret)
3322 space_info->full = 1;
3323 else
3324 ret = 1;
3325 space_info->force_alloc = 0;
3326 spin_unlock(&space_info->lock);
3327 out:
3328 mutex_unlock(&extent_root->fs_info->chunk_mutex);
3329 return ret;
3330 }
3331
3332 /*
3333 * shrink metadata reservation for delalloc
3334 */
3335 static int shrink_delalloc(struct btrfs_trans_handle *trans,
3336 struct btrfs_root *root, u64 to_reclaim, int sync)
3337 {
3338 struct btrfs_block_rsv *block_rsv;
3339 struct btrfs_space_info *space_info;
3340 u64 reserved;
3341 u64 max_reclaim;
3342 u64 reclaimed = 0;
3343 int pause = 1;
3344 int nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
3345
3346 block_rsv = &root->fs_info->delalloc_block_rsv;
3347 space_info = block_rsv->space_info;
3348
3349 smp_mb();
3350 reserved = space_info->bytes_reserved;
3351
3352 if (reserved == 0)
3353 return 0;
3354
3355 max_reclaim = min(reserved, to_reclaim);
3356
3357 while (1) {
3358 /* have the flusher threads jump in and do some IO */
3359 smp_mb();
3360 nr_pages = min_t(unsigned long, nr_pages,
3361 root->fs_info->delalloc_bytes >> PAGE_CACHE_SHIFT);
3362 writeback_inodes_sb_nr_if_idle(root->fs_info->sb, nr_pages);
3363
3364 spin_lock(&space_info->lock);
3365 if (reserved > space_info->bytes_reserved)
3366 reclaimed += reserved - space_info->bytes_reserved;
3367 reserved = space_info->bytes_reserved;
3368 spin_unlock(&space_info->lock);
3369
3370 if (reserved == 0 || reclaimed >= max_reclaim)
3371 break;
3372
3373 if (trans && trans->transaction->blocked)
3374 return -EAGAIN;
3375
3376 __set_current_state(TASK_INTERRUPTIBLE);
3377 schedule_timeout(pause);
3378 pause <<= 1;
3379 if (pause > HZ / 10)
3380 pause = HZ / 10;
3381
3382 }
3383 return reclaimed >= to_reclaim;
3384 }
3385
3386 /*
3387 * Retries tells us how many times we've called reserve_metadata_bytes. The
3388 * idea is if this is the first call (retries == 0) then we will add to our
3389 * reserved count if we can't make the allocation in order to hold our place
3390 * while we go and try and free up space. That way for retries > 1 we don't try
3391 * and add space, we just check to see if the amount of unused space is >= the
3392 * total space, meaning that our reservation is valid.
3393 *
3394 * However if we don't intend to retry this reservation, pass -1 as retries so
3395 * that it short circuits this logic.
3396 */
3397 static int reserve_metadata_bytes(struct btrfs_trans_handle *trans,
3398 struct btrfs_root *root,
3399 struct btrfs_block_rsv *block_rsv,
3400 u64 orig_bytes, int flush)
3401 {
3402 struct btrfs_space_info *space_info = block_rsv->space_info;
3403 u64 unused;
3404 u64 num_bytes = orig_bytes;
3405 int retries = 0;
3406 int ret = 0;
3407 bool reserved = false;
3408 bool committed = false;
3409
3410 again:
3411 ret = -ENOSPC;
3412 if (reserved)
3413 num_bytes = 0;
3414
3415 spin_lock(&space_info->lock);
3416 unused = space_info->bytes_used + space_info->bytes_reserved +
3417 space_info->bytes_pinned + space_info->bytes_readonly +
3418 space_info->bytes_may_use;
3419
3420 /*
3421 * The idea here is that we've not already over-reserved the block group
3422 * then we can go ahead and save our reservation first and then start
3423 * flushing if we need to. Otherwise if we've already overcommitted
3424 * lets start flushing stuff first and then come back and try to make
3425 * our reservation.
3426 */
3427 if (unused <= space_info->total_bytes) {
3428 unused = space_info->total_bytes - unused;
3429 if (unused >= num_bytes) {
3430 if (!reserved)
3431 space_info->bytes_reserved += orig_bytes;
3432 ret = 0;
3433 } else {
3434 /*
3435 * Ok set num_bytes to orig_bytes since we aren't
3436 * overocmmitted, this way we only try and reclaim what
3437 * we need.
3438 */
3439 num_bytes = orig_bytes;
3440 }
3441 } else {
3442 /*
3443 * Ok we're over committed, set num_bytes to the overcommitted
3444 * amount plus the amount of bytes that we need for this
3445 * reservation.
3446 */
3447 num_bytes = unused - space_info->total_bytes +
3448 (orig_bytes * (retries + 1));
3449 }
3450
3451 /*
3452 * Couldn't make our reservation, save our place so while we're trying
3453 * to reclaim space we can actually use it instead of somebody else
3454 * stealing it from us.
3455 */
3456 if (ret && !reserved) {
3457 space_info->bytes_reserved += orig_bytes;
3458 reserved = true;
3459 }
3460
3461 spin_unlock(&space_info->lock);
3462
3463 if (!ret)
3464 return 0;
3465
3466 if (!flush)
3467 goto out;
3468
3469 /*
3470 * We do synchronous shrinking since we don't actually unreserve
3471 * metadata until after the IO is completed.
3472 */
3473 ret = shrink_delalloc(trans, root, num_bytes, 1);
3474 if (ret > 0)
3475 return 0;
3476 else if (ret < 0)
3477 goto out;
3478
3479 /*
3480 * So if we were overcommitted it's possible that somebody else flushed
3481 * out enough space and we simply didn't have enough space to reclaim,
3482 * so go back around and try again.
3483 */
3484 if (retries < 2) {
3485 retries++;
3486 goto again;
3487 }
3488
3489 spin_lock(&space_info->lock);
3490 /*
3491 * Not enough space to be reclaimed, don't bother committing the
3492 * transaction.
3493 */
3494 if (space_info->bytes_pinned < orig_bytes)
3495 ret = -ENOSPC;
3496 spin_unlock(&space_info->lock);
3497 if (ret)
3498 goto out;
3499
3500 ret = -EAGAIN;
3501 if (trans || committed)
3502 goto out;
3503
3504 ret = -ENOSPC;
3505 trans = btrfs_join_transaction(root, 1);
3506 if (IS_ERR(trans))
3507 goto out;
3508 ret = btrfs_commit_transaction(trans, root);
3509 if (!ret) {
3510 trans = NULL;
3511 committed = true;
3512 goto again;
3513 }
3514
3515 out:
3516 if (reserved) {
3517 spin_lock(&space_info->lock);
3518 space_info->bytes_reserved -= orig_bytes;
3519 spin_unlock(&space_info->lock);
3520 }
3521
3522 return ret;
3523 }
3524
3525 static struct btrfs_block_rsv *get_block_rsv(struct btrfs_trans_handle *trans,
3526 struct btrfs_root *root)
3527 {
3528 struct btrfs_block_rsv *block_rsv;
3529 if (root->ref_cows)
3530 block_rsv = trans->block_rsv;
3531 else
3532 block_rsv = root->block_rsv;
3533
3534 if (!block_rsv)
3535 block_rsv = &root->fs_info->empty_block_rsv;
3536
3537 return block_rsv;
3538 }
3539
3540 static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
3541 u64 num_bytes)
3542 {
3543 int ret = -ENOSPC;
3544 spin_lock(&block_rsv->lock);
3545 if (block_rsv->reserved >= num_bytes) {
3546 block_rsv->reserved -= num_bytes;
3547 if (block_rsv->reserved < block_rsv->size)
3548 block_rsv->full = 0;
3549 ret = 0;
3550 }
3551 spin_unlock(&block_rsv->lock);
3552 return ret;
3553 }
3554
3555 static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
3556 u64 num_bytes, int update_size)
3557 {
3558 spin_lock(&block_rsv->lock);
3559 block_rsv->reserved += num_bytes;
3560 if (update_size)
3561 block_rsv->size += num_bytes;
3562 else if (block_rsv->reserved >= block_rsv->size)
3563 block_rsv->full = 1;
3564 spin_unlock(&block_rsv->lock);
3565 }
3566
3567 void block_rsv_release_bytes(struct btrfs_block_rsv *block_rsv,
3568 struct btrfs_block_rsv *dest, u64 num_bytes)
3569 {
3570 struct btrfs_space_info *space_info = block_rsv->space_info;
3571
3572 spin_lock(&block_rsv->lock);
3573 if (num_bytes == (u64)-1)
3574 num_bytes = block_rsv->size;
3575 block_rsv->size -= num_bytes;
3576 if (block_rsv->reserved >= block_rsv->size) {
3577 num_bytes = block_rsv->reserved - block_rsv->size;
3578 block_rsv->reserved = block_rsv->size;
3579 block_rsv->full = 1;
3580 } else {
3581 num_bytes = 0;
3582 }
3583 spin_unlock(&block_rsv->lock);
3584
3585 if (num_bytes > 0) {
3586 if (dest) {
3587 block_rsv_add_bytes(dest, num_bytes, 0);
3588 } else {
3589 spin_lock(&space_info->lock);
3590 space_info->bytes_reserved -= num_bytes;
3591 spin_unlock(&space_info->lock);
3592 }
3593 }
3594 }
3595
3596 static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
3597 struct btrfs_block_rsv *dst, u64 num_bytes)
3598 {
3599 int ret;
3600
3601 ret = block_rsv_use_bytes(src, num_bytes);
3602 if (ret)
3603 return ret;
3604
3605 block_rsv_add_bytes(dst, num_bytes, 1);
3606 return 0;
3607 }
3608
3609 void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv)
3610 {
3611 memset(rsv, 0, sizeof(*rsv));
3612 spin_lock_init(&rsv->lock);
3613 atomic_set(&rsv->usage, 1);
3614 rsv->priority = 6;
3615 INIT_LIST_HEAD(&rsv->list);
3616 }
3617
3618 struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root)
3619 {
3620 struct btrfs_block_rsv *block_rsv;
3621 struct btrfs_fs_info *fs_info = root->fs_info;
3622
3623 block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
3624 if (!block_rsv)
3625 return NULL;
3626
3627 btrfs_init_block_rsv(block_rsv);
3628 block_rsv->space_info = __find_space_info(fs_info,
3629 BTRFS_BLOCK_GROUP_METADATA);
3630 return block_rsv;
3631 }
3632
3633 void btrfs_free_block_rsv(struct btrfs_root *root,
3634 struct btrfs_block_rsv *rsv)
3635 {
3636 if (rsv && atomic_dec_and_test(&rsv->usage)) {
3637 btrfs_block_rsv_release(root, rsv, (u64)-1);
3638 if (!rsv->durable)
3639 kfree(rsv);
3640 }
3641 }
3642
3643 /*
3644 * make the block_rsv struct be able to capture freed space.
3645 * the captured space will re-add to the the block_rsv struct
3646 * after transaction commit
3647 */
3648 void btrfs_add_durable_block_rsv(struct btrfs_fs_info *fs_info,
3649 struct btrfs_block_rsv *block_rsv)
3650 {
3651 block_rsv->durable = 1;
3652 mutex_lock(&fs_info->durable_block_rsv_mutex);
3653 list_add_tail(&block_rsv->list, &fs_info->durable_block_rsv_list);
3654 mutex_unlock(&fs_info->durable_block_rsv_mutex);
3655 }
3656
3657 int btrfs_block_rsv_add(struct btrfs_trans_handle *trans,
3658 struct btrfs_root *root,
3659 struct btrfs_block_rsv *block_rsv,
3660 u64 num_bytes)
3661 {
3662 int ret;
3663
3664 if (num_bytes == 0)
3665 return 0;
3666
3667 ret = reserve_metadata_bytes(trans, root, block_rsv, num_bytes, 1);
3668 if (!ret) {
3669 block_rsv_add_bytes(block_rsv, num_bytes, 1);
3670 return 0;
3671 }
3672
3673 return ret;
3674 }
3675
3676 int btrfs_block_rsv_check(struct btrfs_trans_handle *trans,
3677 struct btrfs_root *root,
3678 struct btrfs_block_rsv *block_rsv,
3679 u64 min_reserved, int min_factor)
3680 {
3681 u64 num_bytes = 0;
3682 int commit_trans = 0;
3683 int ret = -ENOSPC;
3684
3685 if (!block_rsv)
3686 return 0;
3687
3688 spin_lock(&block_rsv->lock);
3689 if (min_factor > 0)
3690 num_bytes = div_factor(block_rsv->size, min_factor);
3691 if (min_reserved > num_bytes)
3692 num_bytes = min_reserved;
3693
3694 if (block_rsv->reserved >= num_bytes) {
3695 ret = 0;
3696 } else {
3697 num_bytes -= block_rsv->reserved;
3698 if (block_rsv->durable &&
3699 block_rsv->freed[0] + block_rsv->freed[1] >= num_bytes)
3700 commit_trans = 1;
3701 }
3702 spin_unlock(&block_rsv->lock);
3703 if (!ret)
3704 return 0;
3705
3706 if (block_rsv->refill_used) {
3707 ret = reserve_metadata_bytes(trans, root, block_rsv,
3708 num_bytes, 0);
3709 if (!ret) {
3710 block_rsv_add_bytes(block_rsv, num_bytes, 0);
3711 return 0;
3712 }
3713 }
3714
3715 if (commit_trans) {
3716 if (trans)
3717 return -EAGAIN;
3718
3719 trans = btrfs_join_transaction(root, 1);
3720 BUG_ON(IS_ERR(trans));
3721 ret = btrfs_commit_transaction(trans, root);
3722 return 0;
3723 }
3724
3725 WARN_ON(1);
3726 printk(KERN_INFO"block_rsv size %llu reserved %llu freed %llu %llu\n",
3727 block_rsv->size, block_rsv->reserved,
3728 block_rsv->freed[0], block_rsv->freed[1]);
3729
3730 return -ENOSPC;
3731 }
3732
3733 int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
3734 struct btrfs_block_rsv *dst_rsv,
3735 u64 num_bytes)
3736 {
3737 return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
3738 }
3739
3740 void btrfs_block_rsv_release(struct btrfs_root *root,
3741 struct btrfs_block_rsv *block_rsv,
3742 u64 num_bytes)
3743 {
3744 struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
3745 if (global_rsv->full || global_rsv == block_rsv ||
3746 block_rsv->space_info != global_rsv->space_info)
3747 global_rsv = NULL;
3748 block_rsv_release_bytes(block_rsv, global_rsv, num_bytes);
3749 }
3750
3751 /*
3752 * helper to calculate size of global block reservation.
3753 * the desired value is sum of space used by extent tree,
3754 * checksum tree and root tree
3755 */
3756 static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
3757 {
3758 struct btrfs_space_info *sinfo;
3759 u64 num_bytes;
3760 u64 meta_used;
3761 u64 data_used;
3762 int csum_size = btrfs_super_csum_size(&fs_info->super_copy);
3763 #if 0
3764 /*
3765 * per tree used space accounting can be inaccuracy, so we
3766 * can't rely on it.
3767 */
3768 spin_lock(&fs_info->extent_root->accounting_lock);
3769 num_bytes = btrfs_root_used(&fs_info->extent_root->root_item);
3770 spin_unlock(&fs_info->extent_root->accounting_lock);
3771
3772 spin_lock(&fs_info->csum_root->accounting_lock);
3773 num_bytes += btrfs_root_used(&fs_info->csum_root->root_item);
3774 spin_unlock(&fs_info->csum_root->accounting_lock);
3775
3776 spin_lock(&fs_info->tree_root->accounting_lock);
3777 num_bytes += btrfs_root_used(&fs_info->tree_root->root_item);
3778 spin_unlock(&fs_info->tree_root->accounting_lock);
3779 #endif
3780 sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
3781 spin_lock(&sinfo->lock);
3782 data_used = sinfo->bytes_used;
3783 spin_unlock(&sinfo->lock);
3784
3785 sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
3786 spin_lock(&sinfo->lock);
3787 if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
3788 data_used = 0;
3789 meta_used = sinfo->bytes_used;
3790 spin_unlock(&sinfo->lock);
3791
3792 num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
3793 csum_size * 2;
3794 num_bytes += div64_u64(data_used + meta_used, 50);
3795
3796 if (num_bytes * 3 > meta_used)
3797 num_bytes = div64_u64(meta_used, 3);
3798
3799 return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
3800 }
3801
3802 static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
3803 {
3804 struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
3805 struct btrfs_space_info *sinfo = block_rsv->space_info;
3806 u64 num_bytes;
3807
3808 num_bytes = calc_global_metadata_size(fs_info);
3809
3810 spin_lock(&block_rsv->lock);
3811 spin_lock(&sinfo->lock);
3812
3813 block_rsv->size = num_bytes;
3814
3815 num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
3816 sinfo->bytes_reserved + sinfo->bytes_readonly +
3817 sinfo->bytes_may_use;
3818
3819 if (sinfo->total_bytes > num_bytes) {
3820 num_bytes = sinfo->total_bytes - num_bytes;
3821 block_rsv->reserved += num_bytes;
3822 sinfo->bytes_reserved += num_bytes;
3823 }
3824
3825 if (block_rsv->reserved >= block_rsv->size) {
3826 num_bytes = block_rsv->reserved - block_rsv->size;
3827 sinfo->bytes_reserved -= num_bytes;
3828 block_rsv->reserved = block_rsv->size;
3829 block_rsv->full = 1;
3830 }
3831 #if 0
3832 printk(KERN_INFO"global block rsv size %llu reserved %llu\n",
3833 block_rsv->size, block_rsv->reserved);
3834 #endif
3835 spin_unlock(&sinfo->lock);
3836 spin_unlock(&block_rsv->lock);
3837 }
3838
3839 static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
3840 {
3841 struct btrfs_space_info *space_info;
3842
3843 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
3844 fs_info->chunk_block_rsv.space_info = space_info;
3845 fs_info->chunk_block_rsv.priority = 10;
3846
3847 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
3848 fs_info->global_block_rsv.space_info = space_info;
3849 fs_info->global_block_rsv.priority = 10;
3850 fs_info->global_block_rsv.refill_used = 1;
3851 fs_info->delalloc_block_rsv.space_info = space_info;
3852 fs_info->trans_block_rsv.space_info = space_info;
3853 fs_info->empty_block_rsv.space_info = space_info;
3854 fs_info->empty_block_rsv.priority = 10;
3855
3856 fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
3857 fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
3858 fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
3859 fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
3860 fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
3861
3862 btrfs_add_durable_block_rsv(fs_info, &fs_info->global_block_rsv);
3863
3864 btrfs_add_durable_block_rsv(fs_info, &fs_info->delalloc_block_rsv);
3865
3866 update_global_block_rsv(fs_info);
3867 }
3868
3869 static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
3870 {
3871 block_rsv_release_bytes(&fs_info->global_block_rsv, NULL, (u64)-1);
3872 WARN_ON(fs_info->delalloc_block_rsv.size > 0);
3873 WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
3874 WARN_ON(fs_info->trans_block_rsv.size > 0);
3875 WARN_ON(fs_info->trans_block_rsv.reserved > 0);
3876 WARN_ON(fs_info->chunk_block_rsv.size > 0);
3877 WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
3878 }
3879
3880 static u64 calc_trans_metadata_size(struct btrfs_root *root, int num_items)
3881 {
3882 return (root->leafsize + root->nodesize * (BTRFS_MAX_LEVEL - 1)) *
3883 3 * num_items;
3884 }
3885
3886 int btrfs_trans_reserve_metadata(struct btrfs_trans_handle *trans,
3887 struct btrfs_root *root,
3888 int num_items)
3889 {
3890 u64 num_bytes;
3891 int ret;
3892
3893 if (num_items == 0 || root->fs_info->chunk_root == root)
3894 return 0;
3895
3896 num_bytes = calc_trans_metadata_size(root, num_items);
3897 ret = btrfs_block_rsv_add(trans, root, &root->fs_info->trans_block_rsv,
3898 num_bytes);
3899 if (!ret) {
3900 trans->bytes_reserved += num_bytes;
3901 trans->block_rsv = &root->fs_info->trans_block_rsv;
3902 }
3903 return ret;
3904 }
3905
3906 void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
3907 struct btrfs_root *root)
3908 {
3909 if (!trans->bytes_reserved)
3910 return;
3911
3912 BUG_ON(trans->block_rsv != &root->fs_info->trans_block_rsv);
3913 btrfs_block_rsv_release(root, trans->block_rsv,
3914 trans->bytes_reserved);
3915 trans->bytes_reserved = 0;
3916 }
3917
3918 int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
3919 struct inode *inode)
3920 {
3921 struct btrfs_root *root = BTRFS_I(inode)->root;
3922 struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
3923 struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;
3924
3925 /*
3926 * one for deleting orphan item, one for updating inode and
3927 * two for calling btrfs_truncate_inode_items.
3928 *
3929 * btrfs_truncate_inode_items is a delete operation, it frees
3930 * more space than it uses in most cases. So two units of
3931 * metadata space should be enough for calling it many times.
3932 * If all of the metadata space is used, we can commit
3933 * transaction and use space it freed.
3934 */
3935 u64 num_bytes = calc_trans_metadata_size(root, 4);
3936 return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
3937 }
3938
3939 void btrfs_orphan_release_metadata(struct inode *inode)
3940 {
3941 struct btrfs_root *root = BTRFS_I(inode)->root;
3942 u64 num_bytes = calc_trans_metadata_size(root, 4);
3943 btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
3944 }
3945
3946 int btrfs_snap_reserve_metadata(struct btrfs_trans_handle *trans,
3947 struct btrfs_pending_snapshot *pending)
3948 {
3949 struct btrfs_root *root = pending->root;
3950 struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
3951 struct btrfs_block_rsv *dst_rsv = &pending->block_rsv;
3952 /*
3953 * two for root back/forward refs, two for directory entries
3954 * and one for root of the snapshot.
3955 */
3956 u64 num_bytes = calc_trans_metadata_size(root, 5);
3957 dst_rsv->space_info = src_rsv->space_info;
3958 return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
3959 }
3960
3961 static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes)
3962 {
3963 return num_bytes >>= 3;
3964 }
3965
3966 int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
3967 {
3968 struct btrfs_root *root = BTRFS_I(inode)->root;
3969 struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
3970 u64 to_reserve;
3971 int nr_extents;
3972 int ret;
3973
3974 if (btrfs_transaction_in_commit(root->fs_info))
3975 schedule_timeout(1);
3976
3977 num_bytes = ALIGN(num_bytes, root->sectorsize);
3978
3979 spin_lock(&BTRFS_I(inode)->accounting_lock);
3980 nr_extents = atomic_read(&BTRFS_I(inode)->outstanding_extents) + 1;
3981 if (nr_extents > BTRFS_I(inode)->reserved_extents) {
3982 nr_extents -= BTRFS_I(inode)->reserved_extents;
3983 to_reserve = calc_trans_metadata_size(root, nr_extents);
3984 } else {
3985 nr_extents = 0;
3986 to_reserve = 0;
3987 }
3988 spin_unlock(&BTRFS_I(inode)->accounting_lock);
3989
3990 to_reserve += calc_csum_metadata_size(inode, num_bytes);
3991 ret = reserve_metadata_bytes(NULL, root, block_rsv, to_reserve, 1);
3992 if (ret)
3993 return ret;
3994
3995 spin_lock(&BTRFS_I(inode)->accounting_lock);
3996 BTRFS_I(inode)->reserved_extents += nr_extents;
3997 atomic_inc(&BTRFS_I(inode)->outstanding_extents);
3998 spin_unlock(&BTRFS_I(inode)->accounting_lock);
3999
4000 block_rsv_add_bytes(block_rsv, to_reserve, 1);
4001
4002 if (block_rsv->size > 512 * 1024 * 1024)
4003 shrink_delalloc(NULL, root, to_reserve, 0);
4004
4005 return 0;
4006 }
4007
4008 void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
4009 {
4010 struct btrfs_root *root = BTRFS_I(inode)->root;
4011 u64 to_free;
4012 int nr_extents;
4013
4014 num_bytes = ALIGN(num_bytes, root->sectorsize);
4015 atomic_dec(&BTRFS_I(inode)->outstanding_extents);
4016
4017 spin_lock(&BTRFS_I(inode)->accounting_lock);
4018 nr_extents = atomic_read(&BTRFS_I(inode)->outstanding_extents);
4019 if (nr_extents < BTRFS_I(inode)->reserved_extents) {
4020 nr_extents = BTRFS_I(inode)->reserved_extents - nr_extents;
4021 BTRFS_I(inode)->reserved_extents -= nr_extents;
4022 } else {
4023 nr_extents = 0;
4024 }
4025 spin_unlock(&BTRFS_I(inode)->accounting_lock);
4026
4027 to_free = calc_csum_metadata_size(inode, num_bytes);
4028 if (nr_extents > 0)
4029 to_free += calc_trans_metadata_size(root, nr_extents);
4030
4031 btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
4032 to_free);
4033 }
4034
4035 int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
4036 {
4037 int ret;
4038
4039 ret = btrfs_check_data_free_space(inode, num_bytes);
4040 if (ret)
4041 return ret;
4042
4043 ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
4044 if (ret) {
4045 btrfs_free_reserved_data_space(inode, num_bytes);
4046 return ret;
4047 }
4048
4049 return 0;
4050 }
4051
4052 void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
4053 {
4054 btrfs_delalloc_release_metadata(inode, num_bytes);
4055 btrfs_free_reserved_data_space(inode, num_bytes);
4056 }
4057
4058 static int update_block_group(struct btrfs_trans_handle *trans,
4059 struct btrfs_root *root,
4060 u64 bytenr, u64 num_bytes, int alloc)
4061 {
4062 struct btrfs_block_group_cache *cache = NULL;
4063 struct btrfs_fs_info *info = root->fs_info;
4064 u64 total = num_bytes;
4065 u64 old_val;
4066 u64 byte_in_group;
4067 int factor;
4068
4069 /* block accounting for super block */
4070 spin_lock(&info->delalloc_lock);
4071 old_val = btrfs_super_bytes_used(&info->super_copy);
4072 if (alloc)
4073 old_val += num_bytes;
4074 else
4075 old_val -= num_bytes;
4076 btrfs_set_super_bytes_used(&info->super_copy, old_val);
4077 spin_unlock(&info->delalloc_lock);
4078
4079 while (total) {
4080 cache = btrfs_lookup_block_group(info, bytenr);
4081 if (!cache)
4082 return -1;
4083 if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
4084 BTRFS_BLOCK_GROUP_RAID1 |
4085 BTRFS_BLOCK_GROUP_RAID10))
4086 factor = 2;
4087 else
4088 factor = 1;
4089 /*
4090 * If this block group has free space cache written out, we
4091 * need to make sure to load it if we are removing space. This
4092 * is because we need the unpinning stage to actually add the
4093 * space back to the block group, otherwise we will leak space.
4094 */
4095 if (!alloc && cache->cached == BTRFS_CACHE_NO)
4096 cache_block_group(cache, trans, NULL, 1);
4097
4098 byte_in_group = bytenr - cache->key.objectid;
4099 WARN_ON(byte_in_group > cache->key.offset);
4100
4101 spin_lock(&cache->space_info->lock);
4102 spin_lock(&cache->lock);
4103
4104 if (btrfs_super_cache_generation(&info->super_copy) != 0 &&
4105 cache->disk_cache_state < BTRFS_DC_CLEAR)
4106 cache->disk_cache_state = BTRFS_DC_CLEAR;
4107
4108 cache->dirty = 1;
4109 old_val = btrfs_block_group_used(&cache->item);
4110 num_bytes = min(total, cache->key.offset - byte_in_group);
4111 if (alloc) {
4112 old_val += num_bytes;
4113 btrfs_set_block_group_used(&cache->item, old_val);
4114 cache->reserved -= num_bytes;
4115 cache->space_info->bytes_reserved -= num_bytes;
4116 cache->space_info->bytes_used += num_bytes;
4117 cache->space_info->disk_used += num_bytes * factor;
4118 spin_unlock(&cache->lock);
4119 spin_unlock(&cache->space_info->lock);
4120 } else {
4121 old_val -= num_bytes;
4122 btrfs_set_block_group_used(&cache->item, old_val);
4123 cache->pinned += num_bytes;
4124 cache->space_info->bytes_pinned += num_bytes;
4125 cache->space_info->bytes_used -= num_bytes;
4126 cache->space_info->disk_used -= num_bytes * factor;
4127 spin_unlock(&cache->lock);
4128 spin_unlock(&cache->space_info->lock);
4129
4130 set_extent_dirty(info->pinned_extents,
4131 bytenr, bytenr + num_bytes - 1,
4132 GFP_NOFS | __GFP_NOFAIL);
4133 }
4134 btrfs_put_block_group(cache);
4135 total -= num_bytes;
4136 bytenr += num_bytes;
4137 }
4138 return 0;
4139 }
4140
4141 static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
4142 {
4143 struct btrfs_block_group_cache *cache;
4144 u64 bytenr;
4145
4146 cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
4147 if (!cache)
4148 return 0;
4149
4150 bytenr = cache->key.objectid;
4151 btrfs_put_block_group(cache);
4152
4153 return bytenr;
4154 }
4155
4156 static int pin_down_extent(struct btrfs_root *root,
4157 struct btrfs_block_group_cache *cache,
4158 u64 bytenr, u64 num_bytes, int reserved)
4159 {
4160 spin_lock(&cache->space_info->lock);
4161 spin_lock(&cache->lock);
4162 cache->pinned += num_bytes;
4163 cache->space_info->bytes_pinned += num_bytes;
4164 if (reserved) {
4165 cache->reserved -= num_bytes;
4166 cache->space_info->bytes_reserved -= num_bytes;
4167 }
4168 spin_unlock(&cache->lock);
4169 spin_unlock(&cache->space_info->lock);
4170
4171 set_extent_dirty(root->fs_info->pinned_extents, bytenr,
4172 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
4173 return 0;
4174 }
4175
4176 /*
4177 * this function must be called within transaction
4178 */
4179 int btrfs_pin_extent(struct btrfs_root *root,
4180 u64 bytenr, u64 num_bytes, int reserved)
4181 {
4182 struct btrfs_block_group_cache *cache;
4183
4184 cache = btrfs_lookup_block_group(root->fs_info, bytenr);
4185 BUG_ON(!cache);
4186
4187 pin_down_extent(root, cache, bytenr, num_bytes, reserved);
4188
4189 btrfs_put_block_group(cache);
4190 return 0;
4191 }
4192
4193 /*
4194 * update size of reserved extents. this function may return -EAGAIN
4195 * if 'reserve' is true or 'sinfo' is false.
4196 */
4197 static int update_reserved_bytes(struct btrfs_block_group_cache *cache,
4198 u64 num_bytes, int reserve, int sinfo)
4199 {
4200 int ret = 0;
4201 if (sinfo) {
4202 struct btrfs_space_info *space_info = cache->space_info;
4203 spin_lock(&space_info->lock);
4204 spin_lock(&cache->lock);
4205 if (reserve) {
4206 if (cache->ro) {
4207 ret = -EAGAIN;
4208 } else {
4209 cache->reserved += num_bytes;
4210 space_info->bytes_reserved += num_bytes;
4211 }
4212 } else {
4213 if (cache->ro)
4214 space_info->bytes_readonly += num_bytes;
4215 cache->reserved -= num_bytes;
4216 space_info->bytes_reserved -= num_bytes;
4217 }
4218 spin_unlock(&cache->lock);
4219 spin_unlock(&space_info->lock);
4220 } else {
4221 spin_lock(&cache->lock);
4222 if (cache->ro) {
4223 ret = -EAGAIN;
4224 } else {
4225 if (reserve)
4226 cache->reserved += num_bytes;
4227 else
4228 cache->reserved -= num_bytes;
4229 }
4230 spin_unlock(&cache->lock);
4231 }
4232 return ret;
4233 }
4234
4235 int btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
4236 struct btrfs_root *root)
4237 {
4238 struct btrfs_fs_info *fs_info = root->fs_info;
4239 struct btrfs_caching_control *next;
4240 struct btrfs_caching_control *caching_ctl;
4241 struct btrfs_block_group_cache *cache;
4242
4243 down_write(&fs_info->extent_commit_sem);
4244
4245 list_for_each_entry_safe(caching_ctl, next,
4246 &fs_info->caching_block_groups, list) {
4247 cache = caching_ctl->block_group;
4248 if (block_group_cache_done(cache)) {
4249 cache->last_byte_to_unpin = (u64)-1;
4250 list_del_init(&caching_ctl->list);
4251 put_caching_control(caching_ctl);
4252 } else {
4253 cache->last_byte_to_unpin = caching_ctl->progress;
4254 }
4255 }
4256
4257 if (fs_info->pinned_extents == &fs_info->freed_extents[0])
4258 fs_info->pinned_extents = &fs_info->freed_extents[1];
4259 else
4260 fs_info->pinned_extents = &fs_info->freed_extents[0];
4261
4262 up_write(&fs_info->extent_commit_sem);
4263
4264 update_global_block_rsv(fs_info);
4265 return 0;
4266 }
4267
4268 static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
4269 {
4270 struct btrfs_fs_info *fs_info = root->fs_info;
4271 struct btrfs_block_group_cache *cache = NULL;
4272 u64 len;
4273
4274 while (start <= end) {
4275 if (!cache ||
4276 start >= cache->key.objectid + cache->key.offset) {
4277 if (cache)
4278 btrfs_put_block_group(cache);
4279 cache = btrfs_lookup_block_group(fs_info, start);
4280 BUG_ON(!cache);
4281 }
4282
4283 len = cache->key.objectid + cache->key.offset - start;
4284 len = min(len, end + 1 - start);
4285
4286 if (start < cache->last_byte_to_unpin) {
4287 len = min(len, cache->last_byte_to_unpin - start);
4288 btrfs_add_free_space(cache, start, len);
4289 }
4290
4291 start += len;
4292
4293 spin_lock(&cache->space_info->lock);
4294 spin_lock(&cache->lock);
4295 cache->pinned -= len;
4296 cache->space_info->bytes_pinned -= len;
4297 if (cache->ro) {
4298 cache->space_info->bytes_readonly += len;
4299 } else if (cache->reserved_pinned > 0) {
4300 len = min(len, cache->reserved_pinned);
4301 cache->reserved_pinned -= len;
4302 cache->space_info->bytes_reserved += len;
4303 }
4304 spin_unlock(&cache->lock);
4305 spin_unlock(&cache->space_info->lock);
4306 }
4307
4308 if (cache)
4309 btrfs_put_block_group(cache);
4310 return 0;
4311 }
4312
4313 int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
4314 struct btrfs_root *root)
4315 {
4316 struct btrfs_fs_info *fs_info = root->fs_info;
4317 struct extent_io_tree *unpin;
4318 struct btrfs_block_rsv *block_rsv;
4319 struct btrfs_block_rsv *next_rsv;
4320 u64 start;
4321 u64 end;
4322 int idx;
4323 int ret;
4324
4325 if (fs_info->pinned_extents == &fs_info->freed_extents[0])
4326 unpin = &fs_info->freed_extents[1];
4327 else
4328 unpin = &fs_info->freed_extents[0];
4329
4330 while (1) {
4331 ret = find_first_extent_bit(unpin, 0, &start, &end,
4332 EXTENT_DIRTY);
4333 if (ret)
4334 break;
4335
4336 ret = btrfs_discard_extent(root, start, end + 1 - start);
4337
4338 clear_extent_dirty(unpin, start, end, GFP_NOFS);
4339 unpin_extent_range(root, start, end);
4340 cond_resched();
4341 }
4342
4343 mutex_lock(&fs_info->durable_block_rsv_mutex);
4344 list_for_each_entry_safe(block_rsv, next_rsv,
4345 &fs_info->durable_block_rsv_list, list) {
4346
4347 idx = trans->transid & 0x1;
4348 if (block_rsv->freed[idx] > 0) {
4349 block_rsv_add_bytes(block_rsv,
4350 block_rsv->freed[idx], 0);
4351 block_rsv->freed[idx] = 0;
4352 }
4353 if (atomic_read(&block_rsv->usage) == 0) {
4354 btrfs_block_rsv_release(root, block_rsv, (u64)-1);
4355
4356 if (block_rsv->freed[0] == 0 &&
4357 block_rsv->freed[1] == 0) {
4358 list_del_init(&block_rsv->list);
4359 kfree(block_rsv);
4360 }
4361 } else {
4362 btrfs_block_rsv_release(root, block_rsv, 0);
4363 }
4364 }
4365 mutex_unlock(&fs_info->durable_block_rsv_mutex);
4366
4367 return 0;
4368 }
4369
4370 static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
4371 struct btrfs_root *root,
4372 u64 bytenr, u64 num_bytes, u64 parent,
4373 u64 root_objectid, u64 owner_objectid,
4374 u64 owner_offset, int refs_to_drop,
4375 struct btrfs_delayed_extent_op *extent_op)
4376 {
4377 struct btrfs_key key;
4378 struct btrfs_path *path;
4379 struct btrfs_fs_info *info = root->fs_info;
4380 struct btrfs_root *extent_root = info->extent_root;
4381 struct extent_buffer *leaf;
4382 struct btrfs_extent_item *ei;
4383 struct btrfs_extent_inline_ref *iref;
4384 int ret;
4385 int is_data;
4386 int extent_slot = 0;
4387 int found_extent = 0;
4388 int num_to_del = 1;
4389 u32 item_size;
4390 u64 refs;
4391
4392 path = btrfs_alloc_path();
4393 if (!path)
4394 return -ENOMEM;
4395
4396 path->reada = 1;
4397 path->leave_spinning = 1;
4398
4399 is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
4400 BUG_ON(!is_data && refs_to_drop != 1);
4401
4402 ret = lookup_extent_backref(trans, extent_root, path, &iref,
4403 bytenr, num_bytes, parent,
4404 root_objectid, owner_objectid,
4405 owner_offset);
4406 if (ret == 0) {
4407 extent_slot = path->slots[0];
4408 while (extent_slot >= 0) {
4409 btrfs_item_key_to_cpu(path->nodes[0], &key,
4410 extent_slot);
4411 if (key.objectid != bytenr)
4412 break;
4413 if (key.type == BTRFS_EXTENT_ITEM_KEY &&
4414 key.offset == num_bytes) {
4415 found_extent = 1;
4416 break;
4417 }
4418 if (path->slots[0] - extent_slot > 5)
4419 break;
4420 extent_slot--;
4421 }
4422 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
4423 item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
4424 if (found_extent && item_size < sizeof(*ei))
4425 found_extent = 0;
4426 #endif
4427 if (!found_extent) {
4428 BUG_ON(iref);
4429 ret = remove_extent_backref(trans, extent_root, path,
4430 NULL, refs_to_drop,
4431 is_data);
4432 BUG_ON(ret);
4433 btrfs_release_path(extent_root, path);
4434 path->leave_spinning = 1;
4435
4436 key.objectid = bytenr;
4437 key.type = BTRFS_EXTENT_ITEM_KEY;
4438 key.offset = num_bytes;
4439
4440 ret = btrfs_search_slot(trans, extent_root,
4441 &key, path, -1, 1);
4442 if (ret) {
4443 printk(KERN_ERR "umm, got %d back from search"
4444 ", was looking for %llu\n", ret,
4445 (unsigned long long)bytenr);
4446 btrfs_print_leaf(extent_root, path->nodes[0]);
4447 }
4448 BUG_ON(ret);
4449 extent_slot = path->slots[0];
4450 }
4451 } else {
4452 btrfs_print_leaf(extent_root, path->nodes[0]);
4453 WARN_ON(1);
4454 printk(KERN_ERR "btrfs unable to find ref byte nr %llu "
4455 "parent %llu root %llu owner %llu offset %llu\n",
4456 (unsigned long long)bytenr,
4457 (unsigned long long)parent,
4458 (unsigned long long)root_objectid,
4459 (unsigned long long)owner_objectid,
4460 (unsigned long long)owner_offset);
4461 }
4462
4463 leaf = path->nodes[0];
4464 item_size = btrfs_item_size_nr(leaf, extent_slot);
4465 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
4466 if (item_size < sizeof(*ei)) {
4467 BUG_ON(found_extent || extent_slot != path->slots[0]);
4468 ret = convert_extent_item_v0(trans, extent_root, path,
4469 owner_objectid, 0);
4470 BUG_ON(ret < 0);
4471
4472 btrfs_release_path(extent_root, path);
4473 path->leave_spinning = 1;
4474
4475 key.objectid = bytenr;
4476 key.type = BTRFS_EXTENT_ITEM_KEY;
4477 key.offset = num_bytes;
4478
4479 ret = btrfs_search_slot(trans, extent_root, &key, path,
4480 -1, 1);
4481 if (ret) {
4482 printk(KERN_ERR "umm, got %d back from search"
4483 ", was looking for %llu\n", ret,
4484 (unsigned long long)bytenr);
4485 btrfs_print_leaf(extent_root, path->nodes[0]);
4486 }
4487 BUG_ON(ret);
4488 extent_slot = path->slots[0];
4489 leaf = path->nodes[0];
4490 item_size = btrfs_item_size_nr(leaf, extent_slot);
4491 }
4492 #endif
4493 BUG_ON(item_size < sizeof(*ei));
4494 ei = btrfs_item_ptr(leaf, extent_slot,
4495 struct btrfs_extent_item);
4496 if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID) {
4497 struct btrfs_tree_block_info *bi;
4498 BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
4499 bi = (struct btrfs_tree_block_info *)(ei + 1);
4500 WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
4501 }
4502
4503 refs = btrfs_extent_refs(leaf, ei);
4504 BUG_ON(refs < refs_to_drop);
4505 refs -= refs_to_drop;
4506
4507 if (refs > 0) {
4508 if (extent_op)
4509 __run_delayed_extent_op(extent_op, leaf, ei);
4510 /*
4511 * In the case of inline back ref, reference count will
4512 * be updated by remove_extent_backref
4513 */
4514 if (iref) {
4515 BUG_ON(!found_extent);
4516 } else {
4517 btrfs_set_extent_refs(leaf, ei, refs);
4518 btrfs_mark_buffer_dirty(leaf);
4519 }
4520 if (found_extent) {
4521 ret = remove_extent_backref(trans, extent_root, path,
4522 iref, refs_to_drop,
4523 is_data);
4524 BUG_ON(ret);
4525 }
4526 } else {
4527 if (found_extent) {
4528 BUG_ON(is_data && refs_to_drop !=
4529 extent_data_ref_count(root, path, iref));
4530 if (iref) {
4531 BUG_ON(path->slots[0] != extent_slot);
4532 } else {
4533 BUG_ON(path->slots[0] != extent_slot + 1);
4534 path->slots[0] = extent_slot;
4535 num_to_del = 2;
4536 }
4537 }
4538
4539 ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
4540 num_to_del);
4541 BUG_ON(ret);
4542 btrfs_release_path(extent_root, path);
4543
4544 if (is_data) {
4545 ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
4546 BUG_ON(ret);
4547 } else {
4548 invalidate_mapping_pages(info->btree_inode->i_mapping,
4549 bytenr >> PAGE_CACHE_SHIFT,
4550 (bytenr + num_bytes - 1) >> PAGE_CACHE_SHIFT);
4551 }
4552
4553 ret = update_block_group(trans, root, bytenr, num_bytes, 0);
4554 BUG_ON(ret);
4555 }
4556 btrfs_free_path(path);
4557 return ret;
4558 }
4559
4560 /*
4561 * when we free an block, it is possible (and likely) that we free the last
4562 * delayed ref for that extent as well. This searches the delayed ref tree for
4563 * a given extent, and if there are no other delayed refs to be processed, it
4564 * removes it from the tree.
4565 */
4566 static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
4567 struct btrfs_root *root, u64 bytenr)
4568 {
4569 struct btrfs_delayed_ref_head *head;
4570 struct btrfs_delayed_ref_root *delayed_refs;
4571 struct btrfs_delayed_ref_node *ref;
4572 struct rb_node *node;
4573 int ret = 0;
4574
4575 delayed_refs = &trans->transaction->delayed_refs;
4576 spin_lock(&delayed_refs->lock);
4577 head = btrfs_find_delayed_ref_head(trans, bytenr);
4578 if (!head)
4579 goto out;
4580
4581 node = rb_prev(&head->node.rb_node);
4582 if (!node)
4583 goto out;
4584
4585 ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
4586
4587 /* there are still entries for this ref, we can't drop it */
4588 if (ref->bytenr == bytenr)
4589 goto out;
4590
4591 if (head->extent_op) {
4592 if (!head->must_insert_reserved)
4593 goto out;
4594 kfree(head->extent_op);
4595 head->extent_op = NULL;
4596 }
4597
4598 /*
4599 * waiting for the lock here would deadlock. If someone else has it
4600 * locked they are already in the process of dropping it anyway
4601 */
4602 if (!mutex_trylock(&head->mutex))
4603 goto out;
4604
4605 /*
4606 * at this point we have a head with no other entries. Go
4607 * ahead and process it.
4608 */
4609 head->node.in_tree = 0;
4610 rb_erase(&head->node.rb_node, &delayed_refs->root);
4611
4612 delayed_refs->num_entries--;
4613
4614 /*
4615 * we don't take a ref on the node because we're removing it from the
4616 * tree, so we just steal the ref the tree was holding.
4617 */
4618 delayed_refs->num_heads--;
4619 if (list_empty(&head->cluster))
4620 delayed_refs->num_heads_ready--;
4621
4622 list_del_init(&head->cluster);
4623 spin_unlock(&delayed_refs->lock);
4624
4625 BUG_ON(head->extent_op);
4626 if (head->must_insert_reserved)
4627 ret = 1;
4628
4629 mutex_unlock(&head->mutex);
4630 btrfs_put_delayed_ref(&head->node);
4631 return ret;
4632 out:
4633 spin_unlock(&delayed_refs->lock);
4634 return 0;
4635 }
4636
4637 void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
4638 struct btrfs_root *root,
4639 struct extent_buffer *buf,
4640 u64 parent, int last_ref)
4641 {
4642 struct btrfs_block_rsv *block_rsv;
4643 struct btrfs_block_group_cache *cache = NULL;
4644 int ret;
4645
4646 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
4647 ret = btrfs_add_delayed_tree_ref(trans, buf->start, buf->len,
4648 parent, root->root_key.objectid,
4649 btrfs_header_level(buf),
4650 BTRFS_DROP_DELAYED_REF, NULL);
4651 BUG_ON(ret);
4652 }
4653
4654 if (!last_ref)
4655 return;
4656
4657 block_rsv = get_block_rsv(trans, root);
4658 cache = btrfs_lookup_block_group(root->fs_info, buf->start);
4659 if (block_rsv->space_info != cache->space_info)
4660 goto out;
4661
4662 if (btrfs_header_generation(buf) == trans->transid) {
4663 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
4664 ret = check_ref_cleanup(trans, root, buf->start);
4665 if (!ret)
4666 goto pin;
4667 }
4668
4669 if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
4670 pin_down_extent(root, cache, buf->start, buf->len, 1);
4671 goto pin;
4672 }
4673
4674 WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));
4675
4676 btrfs_add_free_space(cache, buf->start, buf->len);
4677 ret = update_reserved_bytes(cache, buf->len, 0, 0);
4678 if (ret == -EAGAIN) {
4679 /* block group became read-only */
4680 update_reserved_bytes(cache, buf->len, 0, 1);
4681 goto out;
4682 }
4683
4684 ret = 1;
4685 spin_lock(&block_rsv->lock);
4686 if (block_rsv->reserved < block_rsv->size) {
4687 block_rsv->reserved += buf->len;
4688 ret = 0;
4689 }
4690 spin_unlock(&block_rsv->lock);
4691
4692 if (ret) {
4693 spin_lock(&cache->space_info->lock);
4694 cache->space_info->bytes_reserved -= buf->len;
4695 spin_unlock(&cache->space_info->lock);
4696 }
4697 goto out;
4698 }
4699 pin:
4700 if (block_rsv->durable && !cache->ro) {
4701 ret = 0;
4702 spin_lock(&cache->lock);
4703 if (!cache->ro) {
4704 cache->reserved_pinned += buf->len;
4705 ret = 1;
4706 }
4707 spin_unlock(&cache->lock);
4708
4709 if (ret) {
4710 spin_lock(&block_rsv->lock);
4711 block_rsv->freed[trans->transid & 0x1] += buf->len;
4712 spin_unlock(&block_rsv->lock);
4713 }
4714 }
4715 out:
4716 btrfs_put_block_group(cache);
4717 }
4718
4719 int btrfs_free_extent(struct btrfs_trans_handle *trans,
4720 struct btrfs_root *root,
4721 u64 bytenr, u64 num_bytes, u64 parent,
4722 u64 root_objectid, u64 owner, u64 offset)
4723 {
4724 int ret;
4725
4726 /*
4727 * tree log blocks never actually go into the extent allocation
4728 * tree, just update pinning info and exit early.
4729 */
4730 if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
4731 WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
4732 /* unlocks the pinned mutex */
4733 btrfs_pin_extent(root, bytenr, num_bytes, 1);
4734 ret = 0;
4735 } else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
4736 ret = btrfs_add_delayed_tree_ref(trans, bytenr, num_bytes,
4737 parent, root_objectid, (int)owner,
4738 BTRFS_DROP_DELAYED_REF, NULL);
4739 BUG_ON(ret);
4740 } else {
4741 ret = btrfs_add_delayed_data_ref(trans, bytenr, num_bytes,
4742 parent, root_objectid, owner,
4743 offset, BTRFS_DROP_DELAYED_REF, NULL);
4744 BUG_ON(ret);
4745 }
4746 return ret;
4747 }
4748
4749 static u64 stripe_align(struct btrfs_root *root, u64 val)
4750 {
4751 u64 mask = ((u64)root->stripesize - 1);
4752 u64 ret = (val + mask) & ~mask;
4753 return ret;
4754 }
4755
4756 /*
4757 * when we wait for progress in the block group caching, its because
4758 * our allocation attempt failed at least once. So, we must sleep
4759 * and let some progress happen before we try again.
4760 *
4761 * This function will sleep at least once waiting for new free space to
4762 * show up, and then it will check the block group free space numbers
4763 * for our min num_bytes. Another option is to have it go ahead
4764 * and look in the rbtree for a free extent of a given size, but this
4765 * is a good start.
4766 */
4767 static noinline int
4768 wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
4769 u64 num_bytes)
4770 {
4771 struct btrfs_caching_control *caching_ctl;
4772 DEFINE_WAIT(wait);
4773
4774 caching_ctl = get_caching_control(cache);
4775 if (!caching_ctl)
4776 return 0;
4777
4778 wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
4779 (cache->free_space >= num_bytes));
4780
4781 put_caching_control(caching_ctl);
4782 return 0;
4783 }
4784
4785 static noinline int
4786 wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
4787 {
4788 struct btrfs_caching_control *caching_ctl;
4789 DEFINE_WAIT(wait);
4790
4791 caching_ctl = get_caching_control(cache);
4792 if (!caching_ctl)
4793 return 0;
4794
4795 wait_event(caching_ctl->wait, block_group_cache_done(cache));
4796
4797 put_caching_control(caching_ctl);
4798 return 0;
4799 }
4800
4801 static int get_block_group_index(struct btrfs_block_group_cache *cache)
4802 {
4803 int index;
4804 if (cache->flags & BTRFS_BLOCK_GROUP_RAID10)
4805 index = 0;
4806 else if (cache->flags & BTRFS_BLOCK_GROUP_RAID1)
4807 index = 1;
4808 else if (cache->flags & BTRFS_BLOCK_GROUP_DUP)
4809 index = 2;
4810 else if (cache->flags & BTRFS_BLOCK_GROUP_RAID0)
4811 index = 3;
4812 else
4813 index = 4;
4814 return index;
4815 }
4816
4817 enum btrfs_loop_type {
4818 LOOP_FIND_IDEAL = 0,
4819 LOOP_CACHING_NOWAIT = 1,
4820 LOOP_CACHING_WAIT = 2,
4821 LOOP_ALLOC_CHUNK = 3,
4822 LOOP_NO_EMPTY_SIZE = 4,
4823 };
4824
4825 /*
4826 * walks the btree of allocated extents and find a hole of a given size.
4827 * The key ins is changed to record the hole:
4828 * ins->objectid == block start
4829 * ins->flags = BTRFS_EXTENT_ITEM_KEY
4830 * ins->offset == number of blocks
4831 * Any available blocks before search_start are skipped.
4832 */
4833 static noinline int find_free_extent(struct btrfs_trans_handle *trans,
4834 struct btrfs_root *orig_root,
4835 u64 num_bytes, u64 empty_size,
4836 u64 search_start, u64 search_end,
4837 u64 hint_byte, struct btrfs_key *ins,
4838 int data)
4839 {
4840 int ret = 0;
4841 struct btrfs_root *root = orig_root->fs_info->extent_root;
4842 struct btrfs_free_cluster *last_ptr = NULL;
4843 struct btrfs_block_group_cache *block_group = NULL;
4844 int empty_cluster = 2 * 1024 * 1024;
4845 int allowed_chunk_alloc = 0;
4846 int done_chunk_alloc = 0;
4847 struct btrfs_space_info *space_info;
4848 int last_ptr_loop = 0;
4849 int loop = 0;
4850 int index = 0;
4851 bool found_uncached_bg = false;
4852 bool failed_cluster_refill = false;
4853 bool failed_alloc = false;
4854 bool use_cluster = true;
4855 u64 ideal_cache_percent = 0;
4856 u64 ideal_cache_offset = 0;
4857
4858 WARN_ON(num_bytes < root->sectorsize);
4859 btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
4860 ins->objectid = 0;
4861 ins->offset = 0;
4862
4863 space_info = __find_space_info(root->fs_info, data);
4864 if (!space_info) {
4865 printk(KERN_ERR "No space info for %d\n", data);
4866 return -ENOSPC;
4867 }
4868
4869 /*
4870 * If the space info is for both data and metadata it means we have a
4871 * small filesystem and we can't use the clustering stuff.
4872 */
4873 if (btrfs_mixed_space_info(space_info))
4874 use_cluster = false;
4875
4876 if (orig_root->ref_cows || empty_size)
4877 allowed_chunk_alloc = 1;
4878
4879 if (data & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
4880 last_ptr = &root->fs_info->meta_alloc_cluster;
4881 if (!btrfs_test_opt(root, SSD))
4882 empty_cluster = 64 * 1024;
4883 }
4884
4885 if ((data & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
4886 btrfs_test_opt(root, SSD)) {
4887 last_ptr = &root->fs_info->data_alloc_cluster;
4888 }
4889
4890 if (last_ptr) {
4891 spin_lock(&last_ptr->lock);
4892 if (last_ptr->block_group)
4893 hint_byte = last_ptr->window_start;
4894 spin_unlock(&last_ptr->lock);
4895 }
4896
4897 search_start = max(search_start, first_logical_byte(root, 0));
4898 search_start = max(search_start, hint_byte);
4899
4900 if (!last_ptr)
4901 empty_cluster = 0;
4902
4903 if (search_start == hint_byte) {
4904 ideal_cache:
4905 block_group = btrfs_lookup_block_group(root->fs_info,
4906 search_start);
4907 /*
4908 * we don't want to use the block group if it doesn't match our
4909 * allocation bits, or if its not cached.
4910 *
4911 * However if we are re-searching with an ideal block group
4912 * picked out then we don't care that the block group is cached.
4913 */
4914 if (block_group && block_group_bits(block_group, data) &&
4915 (block_group->cached != BTRFS_CACHE_NO ||
4916 search_start == ideal_cache_offset)) {
4917 down_read(&space_info->groups_sem);
4918 if (list_empty(&block_group->list) ||
4919 block_group->ro) {
4920 /*
4921 * someone is removing this block group,
4922 * we can't jump into the have_block_group
4923 * target because our list pointers are not
4924 * valid
4925 */
4926 btrfs_put_block_group(block_group);
4927 up_read(&space_info->groups_sem);
4928 } else {
4929 index = get_block_group_index(block_group);
4930 goto have_block_group;
4931 }
4932 } else if (block_group) {
4933 btrfs_put_block_group(block_group);
4934 }
4935 }
4936 search:
4937 down_read(&space_info->groups_sem);
4938 list_for_each_entry(block_group, &space_info->block_groups[index],
4939 list) {
4940 u64 offset;
4941 int cached;
4942
4943 btrfs_get_block_group(block_group);
4944 search_start = block_group->key.objectid;
4945
4946 have_block_group:
4947 if (unlikely(block_group->cached == BTRFS_CACHE_NO)) {
4948 u64 free_percent;
4949
4950 ret = cache_block_group(block_group, trans,
4951 orig_root, 1);
4952 if (block_group->cached == BTRFS_CACHE_FINISHED)
4953 goto have_block_group;
4954
4955 free_percent = btrfs_block_group_used(&block_group->item);
4956 free_percent *= 100;
4957 free_percent = div64_u64(free_percent,
4958 block_group->key.offset);
4959 free_percent = 100 - free_percent;
4960 if (free_percent > ideal_cache_percent &&
4961 likely(!block_group->ro)) {
4962 ideal_cache_offset = block_group->key.objectid;
4963 ideal_cache_percent = free_percent;
4964 }
4965
4966 /*
4967 * We only want to start kthread caching if we are at
4968 * the point where we will wait for caching to make
4969 * progress, or if our ideal search is over and we've
4970 * found somebody to start caching.
4971 */
4972 if (loop > LOOP_CACHING_NOWAIT ||
4973 (loop > LOOP_FIND_IDEAL &&
4974 atomic_read(&space_info->caching_threads) < 2)) {
4975 ret = cache_block_group(block_group, trans,
4976 orig_root, 0);
4977 BUG_ON(ret);
4978 }
4979 found_uncached_bg = true;
4980
4981 /*
4982 * If loop is set for cached only, try the next block
4983 * group.
4984 */
4985 if (loop == LOOP_FIND_IDEAL)
4986 goto loop;
4987 }
4988
4989 cached = block_group_cache_done(block_group);
4990 if (unlikely(!cached))
4991 found_uncached_bg = true;
4992
4993 if (unlikely(block_group->ro))
4994 goto loop;
4995
4996 /*
4997 * Ok we want to try and use the cluster allocator, so lets look
4998 * there, unless we are on LOOP_NO_EMPTY_SIZE, since we will
4999 * have tried the cluster allocator plenty of times at this
5000 * point and not have found anything, so we are likely way too
5001 * fragmented for the clustering stuff to find anything, so lets
5002 * just skip it and let the allocator find whatever block it can
5003 * find
5004 */
5005 if (last_ptr && loop < LOOP_NO_EMPTY_SIZE) {
5006 /*
5007 * the refill lock keeps out other
5008 * people trying to start a new cluster
5009 */
5010 spin_lock(&last_ptr->refill_lock);
5011 if (last_ptr->block_group &&
5012 (last_ptr->block_group->ro ||
5013 !block_group_bits(last_ptr->block_group, data))) {
5014 offset = 0;
5015 goto refill_cluster;
5016 }
5017
5018 offset = btrfs_alloc_from_cluster(block_group, last_ptr,
5019 num_bytes, search_start);
5020 if (offset) {
5021 /* we have a block, we're done */
5022 spin_unlock(&last_ptr->refill_lock);
5023 goto checks;
5024 }
5025
5026 spin_lock(&last_ptr->lock);
5027 /*
5028 * whoops, this cluster doesn't actually point to
5029 * this block group. Get a ref on the block
5030 * group is does point to and try again
5031 */
5032 if (!last_ptr_loop && last_ptr->block_group &&
5033 last_ptr->block_group != block_group) {
5034
5035 btrfs_put_block_group(block_group);
5036 block_group = last_ptr->block_group;
5037 btrfs_get_block_group(block_group);
5038 spin_unlock(&last_ptr->lock);
5039 spin_unlock(&last_ptr->refill_lock);
5040
5041 last_ptr_loop = 1;
5042 search_start = block_group->key.objectid;
5043 /*
5044 * we know this block group is properly
5045 * in the list because
5046 * btrfs_remove_block_group, drops the
5047 * cluster before it removes the block
5048 * group from the list
5049 */
5050 goto have_block_group;
5051 }
5052 spin_unlock(&last_ptr->lock);
5053 refill_cluster:
5054 /*
5055 * this cluster didn't work out, free it and
5056 * start over
5057 */
5058 btrfs_return_cluster_to_free_space(NULL, last_ptr);
5059
5060 last_ptr_loop = 0;
5061
5062 /* allocate a cluster in this block group */
5063 ret = btrfs_find_space_cluster(trans, root,
5064 block_group, last_ptr,
5065 offset, num_bytes,
5066 empty_cluster + empty_size);
5067 if (ret == 0) {
5068 /*
5069 * now pull our allocation out of this
5070 * cluster
5071 */
5072 offset = btrfs_alloc_from_cluster(block_group,
5073 last_ptr, num_bytes,
5074 search_start);
5075 if (offset) {
5076 /* we found one, proceed */
5077 spin_unlock(&last_ptr->refill_lock);
5078 goto checks;
5079 }
5080 } else if (!cached && loop > LOOP_CACHING_NOWAIT
5081 && !failed_cluster_refill) {
5082 spin_unlock(&last_ptr->refill_lock);
5083
5084 failed_cluster_refill = true;
5085 wait_block_group_cache_progress(block_group,
5086 num_bytes + empty_cluster + empty_size);
5087 goto have_block_group;
5088 }
5089
5090 /*
5091 * at this point we either didn't find a cluster
5092 * or we weren't able to allocate a block from our
5093 * cluster. Free the cluster we've been trying
5094 * to use, and go to the next block group
5095 */
5096 btrfs_return_cluster_to_free_space(NULL, last_ptr);
5097 spin_unlock(&last_ptr->refill_lock);
5098 goto loop;
5099 }
5100
5101 offset = btrfs_find_space_for_alloc(block_group, search_start,
5102 num_bytes, empty_size);
5103 /*
5104 * If we didn't find a chunk, and we haven't failed on this
5105 * block group before, and this block group is in the middle of
5106 * caching and we are ok with waiting, then go ahead and wait
5107 * for progress to be made, and set failed_alloc to true.
5108 *
5109 * If failed_alloc is true then we've already waited on this
5110 * block group once and should move on to the next block group.
5111 */
5112 if (!offset && !failed_alloc && !cached &&
5113 loop > LOOP_CACHING_NOWAIT) {
5114 wait_block_group_cache_progress(block_group,
5115 num_bytes + empty_size);
5116 failed_alloc = true;
5117 goto have_block_group;
5118 } else if (!offset) {
5119 goto loop;
5120 }
5121 checks:
5122 search_start = stripe_align(root, offset);
5123 /* move on to the next group */
5124 if (search_start + num_bytes >= search_end) {
5125 btrfs_add_free_space(block_group, offset, num_bytes);
5126 goto loop;
5127 }
5128
5129 /* move on to the next group */
5130 if (search_start + num_bytes >
5131 block_group->key.objectid + block_group->key.offset) {
5132 btrfs_add_free_space(block_group, offset, num_bytes);
5133 goto loop;
5134 }
5135
5136 ins->objectid = search_start;
5137 ins->offset = num_bytes;
5138
5139 if (offset < search_start)
5140 btrfs_add_free_space(block_group, offset,
5141 search_start - offset);
5142 BUG_ON(offset > search_start);
5143
5144 ret = update_reserved_bytes(block_group, num_bytes, 1,
5145 (data & BTRFS_BLOCK_GROUP_DATA));
5146 if (ret == -EAGAIN) {
5147 btrfs_add_free_space(block_group, offset, num_bytes);
5148 goto loop;
5149 }
5150
5151 /* we are all good, lets return */
5152 ins->objectid = search_start;
5153 ins->offset = num_bytes;
5154
5155 if (offset < search_start)
5156 btrfs_add_free_space(block_group, offset,
5157 search_start - offset);
5158 BUG_ON(offset > search_start);
5159 break;
5160 loop:
5161 failed_cluster_refill = false;
5162 failed_alloc = false;
5163 BUG_ON(index != get_block_group_index(block_group));
5164 btrfs_put_block_group(block_group);
5165 }
5166 up_read(&space_info->groups_sem);
5167
5168 if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
5169 goto search;
5170
5171 /* LOOP_FIND_IDEAL, only search caching/cached bg's, and don't wait for
5172 * for them to make caching progress. Also
5173 * determine the best possible bg to cache
5174 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
5175 * caching kthreads as we move along
5176 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
5177 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
5178 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
5179 * again
5180 */
5181 if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE &&
5182 (found_uncached_bg || empty_size || empty_cluster ||
5183 allowed_chunk_alloc)) {
5184 index = 0;
5185 if (loop == LOOP_FIND_IDEAL && found_uncached_bg) {
5186 found_uncached_bg = false;
5187 loop++;
5188 if (!ideal_cache_percent &&
5189 atomic_read(&space_info->caching_threads))
5190 goto search;
5191
5192 /*
5193 * 1 of the following 2 things have happened so far
5194 *
5195 * 1) We found an ideal block group for caching that
5196 * is mostly full and will cache quickly, so we might
5197 * as well wait for it.
5198 *
5199 * 2) We searched for cached only and we didn't find
5200 * anything, and we didn't start any caching kthreads
5201 * either, so chances are we will loop through and
5202 * start a couple caching kthreads, and then come back
5203 * around and just wait for them. This will be slower
5204 * because we will have 2 caching kthreads reading at
5205 * the same time when we could have just started one
5206 * and waited for it to get far enough to give us an
5207 * allocation, so go ahead and go to the wait caching
5208 * loop.
5209 */
5210 loop = LOOP_CACHING_WAIT;
5211 search_start = ideal_cache_offset;
5212 ideal_cache_percent = 0;
5213 goto ideal_cache;
5214 } else if (loop == LOOP_FIND_IDEAL) {
5215 /*
5216 * Didn't find a uncached bg, wait on anything we find
5217 * next.
5218 */
5219 loop = LOOP_CACHING_WAIT;
5220 goto search;
5221 }
5222
5223 if (loop < LOOP_CACHING_WAIT) {
5224 loop++;
5225 goto search;
5226 }
5227
5228 if (loop == LOOP_ALLOC_CHUNK) {
5229 empty_size = 0;
5230 empty_cluster = 0;
5231 }
5232
5233 if (allowed_chunk_alloc) {
5234 ret = do_chunk_alloc(trans, root, num_bytes +
5235 2 * 1024 * 1024, data, 1);
5236 allowed_chunk_alloc = 0;
5237 done_chunk_alloc = 1;
5238 } else if (!done_chunk_alloc) {
5239 space_info->force_alloc = 1;
5240 }
5241
5242 if (loop < LOOP_NO_EMPTY_SIZE) {
5243 loop++;
5244 goto search;
5245 }
5246 ret = -ENOSPC;
5247 } else if (!ins->objectid) {
5248 ret = -ENOSPC;
5249 }
5250
5251 /* we found what we needed */
5252 if (ins->objectid) {
5253 if (!(data & BTRFS_BLOCK_GROUP_DATA))
5254 trans->block_group = block_group->key.objectid;
5255
5256 btrfs_put_block_group(block_group);
5257 ret = 0;
5258 }
5259
5260 return ret;
5261 }
5262
5263 static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
5264 int dump_block_groups)
5265 {
5266 struct btrfs_block_group_cache *cache;
5267 int index = 0;
5268
5269 spin_lock(&info->lock);
5270 printk(KERN_INFO "space_info has %llu free, is %sfull\n",
5271 (unsigned long long)(info->total_bytes - info->bytes_used -
5272 info->bytes_pinned - info->bytes_reserved -
5273 info->bytes_readonly),
5274 (info->full) ? "" : "not ");
5275 printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
5276 "reserved=%llu, may_use=%llu, readonly=%llu\n",
5277 (unsigned long long)info->total_bytes,
5278 (unsigned long long)info->bytes_used,
5279 (unsigned long long)info->bytes_pinned,
5280 (unsigned long long)info->bytes_reserved,
5281 (unsigned long long)info->bytes_may_use,
5282 (unsigned long long)info->bytes_readonly);
5283 spin_unlock(&info->lock);
5284
5285 if (!dump_block_groups)
5286 return;
5287
5288 down_read(&info->groups_sem);
5289 again:
5290 list_for_each_entry(cache, &info->block_groups[index], list) {
5291 spin_lock(&cache->lock);
5292 printk(KERN_INFO "block group %llu has %llu bytes, %llu used "
5293 "%llu pinned %llu reserved\n",
5294 (unsigned long long)cache->key.objectid,
5295 (unsigned long long)cache->key.offset,
5296 (unsigned long long)btrfs_block_group_used(&cache->item),
5297 (unsigned long long)cache->pinned,
5298 (unsigned long long)cache->reserved);
5299 btrfs_dump_free_space(cache, bytes);
5300 spin_unlock(&cache->lock);
5301 }
5302 if (++index < BTRFS_NR_RAID_TYPES)
5303 goto again;
5304 up_read(&info->groups_sem);
5305 }
5306
5307 int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
5308 struct btrfs_root *root,
5309 u64 num_bytes, u64 min_alloc_size,
5310 u64 empty_size, u64 hint_byte,
5311 u64 search_end, struct btrfs_key *ins,
5312 u64 data)
5313 {
5314 int ret;
5315 u64 search_start = 0;
5316
5317 data = btrfs_get_alloc_profile(root, data);
5318 again:
5319 /*
5320 * the only place that sets empty_size is btrfs_realloc_node, which
5321 * is not called recursively on allocations
5322 */
5323 if (empty_size || root->ref_cows)
5324 ret = do_chunk_alloc(trans, root->fs_info->extent_root,
5325 num_bytes + 2 * 1024 * 1024, data, 0);
5326
5327 WARN_ON(num_bytes < root->sectorsize);
5328 ret = find_free_extent(trans, root, num_bytes, empty_size,
5329 search_start, search_end, hint_byte,
5330 ins, data);
5331
5332 if (ret == -ENOSPC && num_bytes > min_alloc_size) {
5333 num_bytes = num_bytes >> 1;
5334 num_bytes = num_bytes & ~(root->sectorsize - 1);
5335 num_bytes = max(num_bytes, min_alloc_size);
5336 do_chunk_alloc(trans, root->fs_info->extent_root,
5337 num_bytes, data, 1);
5338 goto again;
5339 }
5340 if (ret == -ENOSPC) {
5341 struct btrfs_space_info *sinfo;
5342
5343 sinfo = __find_space_info(root->fs_info, data);
5344 printk(KERN_ERR "btrfs allocation failed flags %llu, "
5345 "wanted %llu\n", (unsigned long long)data,
5346 (unsigned long long)num_bytes);
5347 dump_space_info(sinfo, num_bytes, 1);
5348 }
5349
5350 return ret;
5351 }
5352
5353 int btrfs_free_reserved_extent(struct btrfs_root *root, u64 start, u64 len)
5354 {
5355 struct btrfs_block_group_cache *cache;
5356 int ret = 0;
5357
5358 cache = btrfs_lookup_block_group(root->fs_info, start);
5359 if (!cache) {
5360 printk(KERN_ERR "Unable to find block group for %llu\n",
5361 (unsigned long long)start);
5362 return -ENOSPC;
5363 }
5364
5365 ret = btrfs_discard_extent(root, start, len);
5366
5367 btrfs_add_free_space(cache, start, len);
5368 update_reserved_bytes(cache, len, 0, 1);
5369 btrfs_put_block_group(cache);
5370
5371 return ret;
5372 }
5373
5374 static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
5375 struct btrfs_root *root,
5376 u64 parent, u64 root_objectid,
5377 u64 flags, u64 owner, u64 offset,
5378 struct btrfs_key *ins, int ref_mod)
5379 {
5380 int ret;
5381 struct btrfs_fs_info *fs_info = root->fs_info;
5382 struct btrfs_extent_item *extent_item;
5383 struct btrfs_extent_inline_ref *iref;
5384 struct btrfs_path *path;
5385 struct extent_buffer *leaf;
5386 int type;
5387 u32 size;
5388
5389 if (parent > 0)
5390 type = BTRFS_SHARED_DATA_REF_KEY;
5391 else
5392 type = BTRFS_EXTENT_DATA_REF_KEY;
5393
5394 size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
5395
5396 path = btrfs_alloc_path();
5397 BUG_ON(!path);
5398
5399 path->leave_spinning = 1;
5400 ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
5401 ins, size);
5402 BUG_ON(ret);
5403
5404 leaf = path->nodes[0];
5405 extent_item = btrfs_item_ptr(leaf, path->slots[0],
5406 struct btrfs_extent_item);
5407 btrfs_set_extent_refs(leaf, extent_item, ref_mod);
5408 btrfs_set_extent_generation(leaf, extent_item, trans->transid);
5409 btrfs_set_extent_flags(leaf, extent_item,
5410 flags | BTRFS_EXTENT_FLAG_DATA);
5411
5412 iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
5413 btrfs_set_extent_inline_ref_type(leaf, iref, type);
5414 if (parent > 0) {
5415 struct btrfs_shared_data_ref *ref;
5416 ref = (struct btrfs_shared_data_ref *)(iref + 1);
5417 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
5418 btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
5419 } else {
5420 struct btrfs_extent_data_ref *ref;
5421 ref = (struct btrfs_extent_data_ref *)(&iref->offset);
5422 btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
5423 btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
5424 btrfs_set_extent_data_ref_offset(leaf, ref, offset);
5425 btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
5426 }
5427
5428 btrfs_mark_buffer_dirty(path->nodes[0]);
5429 btrfs_free_path(path);
5430
5431 ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
5432 if (ret) {
5433 printk(KERN_ERR "btrfs update block group failed for %llu "
5434 "%llu\n", (unsigned long long)ins->objectid,
5435 (unsigned long long)ins->offset);
5436 BUG();
5437 }
5438 return ret;
5439 }
5440
5441 static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
5442 struct btrfs_root *root,
5443 u64 parent, u64 root_objectid,
5444 u64 flags, struct btrfs_disk_key *key,
5445 int level, struct btrfs_key *ins)
5446 {
5447 int ret;
5448 struct btrfs_fs_info *fs_info = root->fs_info;
5449 struct btrfs_extent_item *extent_item;
5450 struct btrfs_tree_block_info *block_info;
5451 struct btrfs_extent_inline_ref *iref;
5452 struct btrfs_path *path;
5453 struct extent_buffer *leaf;
5454 u32 size = sizeof(*extent_item) + sizeof(*block_info) + sizeof(*iref);
5455
5456 path = btrfs_alloc_path();
5457 BUG_ON(!path);
5458
5459 path->leave_spinning = 1;
5460 ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
5461 ins, size);
5462 BUG_ON(ret);
5463
5464 leaf = path->nodes[0];
5465 extent_item = btrfs_item_ptr(leaf, path->slots[0],
5466 struct btrfs_extent_item);
5467 btrfs_set_extent_refs(leaf, extent_item, 1);
5468 btrfs_set_extent_generation(leaf, extent_item, trans->transid);
5469 btrfs_set_extent_flags(leaf, extent_item,
5470 flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
5471 block_info = (struct btrfs_tree_block_info *)(extent_item + 1);
5472
5473 btrfs_set_tree_block_key(leaf, block_info, key);
5474 btrfs_set_tree_block_level(leaf, block_info, level);
5475
5476 iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
5477 if (parent > 0) {
5478 BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
5479 btrfs_set_extent_inline_ref_type(leaf, iref,
5480 BTRFS_SHARED_BLOCK_REF_KEY);
5481 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
5482 } else {
5483 btrfs_set_extent_inline_ref_type(leaf, iref,
5484 BTRFS_TREE_BLOCK_REF_KEY);
5485 btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
5486 }
5487
5488 btrfs_mark_buffer_dirty(leaf);
5489 btrfs_free_path(path);
5490
5491 ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
5492 if (ret) {
5493 printk(KERN_ERR "btrfs update block group failed for %llu "
5494 "%llu\n", (unsigned long long)ins->objectid,
5495 (unsigned long long)ins->offset);
5496 BUG();
5497 }
5498 return ret;
5499 }
5500
5501 int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
5502 struct btrfs_root *root,
5503 u64 root_objectid, u64 owner,
5504 u64 offset, struct btrfs_key *ins)
5505 {
5506 int ret;
5507
5508 BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);
5509
5510 ret = btrfs_add_delayed_data_ref(trans, ins->objectid, ins->offset,
5511 0, root_objectid, owner, offset,
5512 BTRFS_ADD_DELAYED_EXTENT, NULL);
5513 return ret;
5514 }
5515
5516 /*
5517 * this is used by the tree logging recovery code. It records that
5518 * an extent has been allocated and makes sure to clear the free
5519 * space cache bits as well
5520 */
5521 int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
5522 struct btrfs_root *root,
5523 u64 root_objectid, u64 owner, u64 offset,
5524 struct btrfs_key *ins)
5525 {
5526 int ret;
5527 struct btrfs_block_group_cache *block_group;
5528 struct btrfs_caching_control *caching_ctl;
5529 u64 start = ins->objectid;
5530 u64 num_bytes = ins->offset;
5531
5532 block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
5533 cache_block_group(block_group, trans, NULL, 0);
5534 caching_ctl = get_caching_control(block_group);
5535
5536 if (!caching_ctl) {
5537 BUG_ON(!block_group_cache_done(block_group));
5538 ret = btrfs_remove_free_space(block_group, start, num_bytes);
5539 BUG_ON(ret);
5540 } else {
5541 mutex_lock(&caching_ctl->mutex);
5542
5543 if (start >= caching_ctl->progress) {
5544 ret = add_excluded_extent(root, start, num_bytes);
5545 BUG_ON(ret);
5546 } else if (start + num_bytes <= caching_ctl->progress) {
5547 ret = btrfs_remove_free_space(block_group,
5548 start, num_bytes);
5549 BUG_ON(ret);
5550 } else {
5551 num_bytes = caching_ctl->progress - start;
5552 ret = btrfs_remove_free_space(block_group,
5553 start, num_bytes);
5554 BUG_ON(ret);
5555
5556 start = caching_ctl->progress;
5557 num_bytes = ins->objectid + ins->offset -
5558 caching_ctl->progress;
5559 ret = add_excluded_extent(root, start, num_bytes);
5560 BUG_ON(ret);
5561 }
5562
5563 mutex_unlock(&caching_ctl->mutex);
5564 put_caching_control(caching_ctl);
5565 }
5566
5567 ret = update_reserved_bytes(block_group, ins->offset, 1, 1);
5568 BUG_ON(ret);
5569 btrfs_put_block_group(block_group);
5570 ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
5571 0, owner, offset, ins, 1);
5572 return ret;
5573 }
5574
5575 struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
5576 struct btrfs_root *root,
5577 u64 bytenr, u32 blocksize,
5578 int level)
5579 {
5580 struct extent_buffer *buf;
5581
5582 buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
5583 if (!buf)
5584 return ERR_PTR(-ENOMEM);
5585 btrfs_set_header_generation(buf, trans->transid);
5586 btrfs_set_buffer_lockdep_class(buf, level);
5587 btrfs_tree_lock(buf);
5588 clean_tree_block(trans, root, buf);
5589
5590 btrfs_set_lock_blocking(buf);
5591 btrfs_set_buffer_uptodate(buf);
5592
5593 if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
5594 /*
5595 * we allow two log transactions at a time, use different
5596 * EXENT bit to differentiate dirty pages.
5597 */
5598 if (root->log_transid % 2 == 0)
5599 set_extent_dirty(&root->dirty_log_pages, buf->start,
5600 buf->start + buf->len - 1, GFP_NOFS);
5601 else
5602 set_extent_new(&root->dirty_log_pages, buf->start,
5603 buf->start + buf->len - 1, GFP_NOFS);
5604 } else {
5605 set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
5606 buf->start + buf->len - 1, GFP_NOFS);
5607 }
5608 trans->blocks_used++;
5609 /* this returns a buffer locked for blocking */
5610 return buf;
5611 }
5612
5613 static struct btrfs_block_rsv *
5614 use_block_rsv(struct btrfs_trans_handle *trans,
5615 struct btrfs_root *root, u32 blocksize)
5616 {
5617 struct btrfs_block_rsv *block_rsv;
5618 int ret;
5619
5620 block_rsv = get_block_rsv(trans, root);
5621
5622 if (block_rsv->size == 0) {
5623 ret = reserve_metadata_bytes(trans, root, block_rsv,
5624 blocksize, 0);
5625 if (ret)
5626 return ERR_PTR(ret);
5627 return block_rsv;
5628 }
5629
5630 ret = block_rsv_use_bytes(block_rsv, blocksize);
5631 if (!ret)
5632 return block_rsv;
5633
5634 return ERR_PTR(-ENOSPC);
5635 }
5636
5637 static void unuse_block_rsv(struct btrfs_block_rsv *block_rsv, u32 blocksize)
5638 {
5639 block_rsv_add_bytes(block_rsv, blocksize, 0);
5640 block_rsv_release_bytes(block_rsv, NULL, 0);
5641 }
5642
5643 /*
5644 * finds a free extent and does all the dirty work required for allocation
5645 * returns the key for the extent through ins, and a tree buffer for
5646 * the first block of the extent through buf.
5647 *
5648 * returns the tree buffer or NULL.
5649 */
5650 struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
5651 struct btrfs_root *root, u32 blocksize,
5652 u64 parent, u64 root_objectid,
5653 struct btrfs_disk_key *key, int level,
5654 u64 hint, u64 empty_size)
5655 {
5656 struct btrfs_key ins;
5657 struct btrfs_block_rsv *block_rsv;
5658 struct extent_buffer *buf;
5659 u64 flags = 0;
5660 int ret;
5661
5662
5663 block_rsv = use_block_rsv(trans, root, blocksize);
5664 if (IS_ERR(block_rsv))
5665 return ERR_CAST(block_rsv);
5666
5667 ret = btrfs_reserve_extent(trans, root, blocksize, blocksize,
5668 empty_size, hint, (u64)-1, &ins, 0);
5669 if (ret) {
5670 unuse_block_rsv(block_rsv, blocksize);
5671 return ERR_PTR(ret);
5672 }
5673
5674 buf = btrfs_init_new_buffer(trans, root, ins.objectid,
5675 blocksize, level);
5676 BUG_ON(IS_ERR(buf));
5677
5678 if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
5679 if (parent == 0)
5680 parent = ins.objectid;
5681 flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
5682 } else
5683 BUG_ON(parent > 0);
5684
5685 if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
5686 struct btrfs_delayed_extent_op *extent_op;
5687 extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
5688 BUG_ON(!extent_op);
5689 if (key)
5690 memcpy(&extent_op->key, key, sizeof(extent_op->key));
5691 else
5692 memset(&extent_op->key, 0, sizeof(extent_op->key));
5693 extent_op->flags_to_set = flags;
5694 extent_op->update_key = 1;
5695 extent_op->update_flags = 1;
5696 extent_op->is_data = 0;
5697
5698 ret = btrfs_add_delayed_tree_ref(trans, ins.objectid,
5699 ins.offset, parent, root_objectid,
5700 level, BTRFS_ADD_DELAYED_EXTENT,
5701 extent_op);
5702 BUG_ON(ret);
5703 }
5704 return buf;
5705 }
5706
5707 struct walk_control {
5708 u64 refs[BTRFS_MAX_LEVEL];
5709 u64 flags[BTRFS_MAX_LEVEL];
5710 struct btrfs_key update_progress;
5711 int stage;
5712 int level;
5713 int shared_level;
5714 int update_ref;
5715 int keep_locks;
5716 int reada_slot;
5717 int reada_count;
5718 };
5719
5720 #define DROP_REFERENCE 1
5721 #define UPDATE_BACKREF 2
5722
5723 static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
5724 struct btrfs_root *root,
5725 struct walk_control *wc,
5726 struct btrfs_path *path)
5727 {
5728 u64 bytenr;
5729 u64 generation;
5730 u64 refs;
5731 u64 flags;
5732 u32 nritems;
5733 u32 blocksize;
5734 struct btrfs_key key;
5735 struct extent_buffer *eb;
5736 int ret;
5737 int slot;
5738 int nread = 0;
5739
5740 if (path->slots[wc->level] < wc->reada_slot) {
5741 wc->reada_count = wc->reada_count * 2 / 3;
5742 wc->reada_count = max(wc->reada_count, 2);
5743 } else {
5744 wc->reada_count = wc->reada_count * 3 / 2;
5745 wc->reada_count = min_t(int, wc->reada_count,
5746 BTRFS_NODEPTRS_PER_BLOCK(root));
5747 }
5748
5749 eb = path->nodes[wc->level];
5750 nritems = btrfs_header_nritems(eb);
5751 blocksize = btrfs_level_size(root, wc->level - 1);
5752
5753 for (slot = path->slots[wc->level]; slot < nritems; slot++) {
5754 if (nread >= wc->reada_count)
5755 break;
5756
5757 cond_resched();
5758 bytenr = btrfs_node_blockptr(eb, slot);
5759 generation = btrfs_node_ptr_generation(eb, slot);
5760
5761 if (slot == path->slots[wc->level])
5762 goto reada;
5763
5764 if (wc->stage == UPDATE_BACKREF &&
5765 generation <= root->root_key.offset)
5766 continue;
5767
5768 /* We don't lock the tree block, it's OK to be racy here */
5769 ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
5770 &refs, &flags);
5771 BUG_ON(ret);
5772 BUG_ON(refs == 0);
5773
5774 if (wc->stage == DROP_REFERENCE) {
5775 if (refs == 1)
5776 goto reada;
5777
5778 if (wc->level == 1 &&
5779 (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
5780 continue;
5781 if (!wc->update_ref ||
5782 generation <= root->root_key.offset)
5783 continue;
5784 btrfs_node_key_to_cpu(eb, &key, slot);
5785 ret = btrfs_comp_cpu_keys(&key,
5786 &wc->update_progress);
5787 if (ret < 0)
5788 continue;
5789 } else {
5790 if (wc->level == 1 &&
5791 (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
5792 continue;
5793 }
5794 reada:
5795 ret = readahead_tree_block(root, bytenr, blocksize,
5796 generation);
5797 if (ret)
5798 break;
5799 nread++;
5800 }
5801 wc->reada_slot = slot;
5802 }
5803
5804 /*
5805 * hepler to process tree block while walking down the tree.
5806 *
5807 * when wc->stage == UPDATE_BACKREF, this function updates
5808 * back refs for pointers in the block.
5809 *
5810 * NOTE: return value 1 means we should stop walking down.
5811 */
5812 static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
5813 struct btrfs_root *root,
5814 struct btrfs_path *path,
5815 struct walk_control *wc, int lookup_info)
5816 {
5817 int level = wc->level;
5818 struct extent_buffer *eb = path->nodes[level];
5819 u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
5820 int ret;
5821
5822 if (wc->stage == UPDATE_BACKREF &&
5823 btrfs_header_owner(eb) != root->root_key.objectid)
5824 return 1;
5825
5826 /*
5827 * when reference count of tree block is 1, it won't increase
5828 * again. once full backref flag is set, we never clear it.
5829 */
5830 if (lookup_info &&
5831 ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
5832 (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
5833 BUG_ON(!path->locks[level]);
5834 ret = btrfs_lookup_extent_info(trans, root,
5835 eb->start, eb->len,
5836 &wc->refs[level],
5837 &wc->flags[level]);
5838 BUG_ON(ret);
5839 BUG_ON(wc->refs[level] == 0);
5840 }
5841
5842 if (wc->stage == DROP_REFERENCE) {
5843 if (wc->refs[level] > 1)
5844 return 1;
5845
5846 if (path->locks[level] && !wc->keep_locks) {
5847 btrfs_tree_unlock(eb);
5848 path->locks[level] = 0;
5849 }
5850 return 0;
5851 }
5852
5853 /* wc->stage == UPDATE_BACKREF */
5854 if (!(wc->flags[level] & flag)) {
5855 BUG_ON(!path->locks[level]);
5856 ret = btrfs_inc_ref(trans, root, eb, 1);
5857 BUG_ON(ret);
5858 ret = btrfs_dec_ref(trans, root, eb, 0);
5859 BUG_ON(ret);
5860 ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
5861 eb->len, flag, 0);
5862 BUG_ON(ret);
5863 wc->flags[level] |= flag;
5864 }
5865
5866 /*
5867 * the block is shared by multiple trees, so it's not good to
5868 * keep the tree lock
5869 */
5870 if (path->locks[level] && level > 0) {
5871 btrfs_tree_unlock(eb);
5872 path->locks[level] = 0;
5873 }
5874 return 0;
5875 }
5876
5877 /*
5878 * hepler to process tree block pointer.
5879 *
5880 * when wc->stage == DROP_REFERENCE, this function checks
5881 * reference count of the block pointed to. if the block
5882 * is shared and we need update back refs for the subtree
5883 * rooted at the block, this function changes wc->stage to
5884 * UPDATE_BACKREF. if the block is shared and there is no
5885 * need to update back, this function drops the reference
5886 * to the block.
5887 *
5888 * NOTE: return value 1 means we should stop walking down.
5889 */
5890 static noinline int do_walk_down(struct btrfs_trans_handle *trans,
5891 struct btrfs_root *root,
5892 struct btrfs_path *path,
5893 struct walk_control *wc, int *lookup_info)
5894 {
5895 u64 bytenr;
5896 u64 generation;
5897 u64 parent;
5898 u32 blocksize;
5899 struct btrfs_key key;
5900 struct extent_buffer *next;
5901 int level = wc->level;
5902 int reada = 0;
5903 int ret = 0;
5904
5905 generation = btrfs_node_ptr_generation(path->nodes[level],
5906 path->slots[level]);
5907 /*
5908 * if the lower level block was created before the snapshot
5909 * was created, we know there is no need to update back refs
5910 * for the subtree
5911 */
5912 if (wc->stage == UPDATE_BACKREF &&
5913 generation <= root->root_key.offset) {
5914 *lookup_info = 1;
5915 return 1;
5916 }
5917
5918 bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
5919 blocksize = btrfs_level_size(root, level - 1);
5920
5921 next = btrfs_find_tree_block(root, bytenr, blocksize);
5922 if (!next) {
5923 next = btrfs_find_create_tree_block(root, bytenr, blocksize);
5924 if (!next)
5925 return -ENOMEM;
5926 reada = 1;
5927 }
5928 btrfs_tree_lock(next);
5929 btrfs_set_lock_blocking(next);
5930
5931 ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
5932 &wc->refs[level - 1],
5933 &wc->flags[level - 1]);
5934 BUG_ON(ret);
5935 BUG_ON(wc->refs[level - 1] == 0);
5936 *lookup_info = 0;
5937
5938 if (wc->stage == DROP_REFERENCE) {
5939 if (wc->refs[level - 1] > 1) {
5940 if (level == 1 &&
5941 (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
5942 goto skip;
5943
5944 if (!wc->update_ref ||
5945 generation <= root->root_key.offset)
5946 goto skip;
5947
5948 btrfs_node_key_to_cpu(path->nodes[level], &key,
5949 path->slots[level]);
5950 ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
5951 if (ret < 0)
5952 goto skip;
5953
5954 wc->stage = UPDATE_BACKREF;
5955 wc->shared_level = level - 1;
5956 }
5957 } else {
5958 if (level == 1 &&
5959 (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
5960 goto skip;
5961 }
5962
5963 if (!btrfs_buffer_uptodate(next, generation)) {
5964 btrfs_tree_unlock(next);
5965 free_extent_buffer(next);
5966 next = NULL;
5967 *lookup_info = 1;
5968 }
5969
5970 if (!next) {
5971 if (reada && level == 1)
5972 reada_walk_down(trans, root, wc, path);
5973 next = read_tree_block(root, bytenr, blocksize, generation);
5974 btrfs_tree_lock(next);
5975 btrfs_set_lock_blocking(next);
5976 }
5977
5978 level--;
5979 BUG_ON(level != btrfs_header_level(next));
5980 path->nodes[level] = next;
5981 path->slots[level] = 0;
5982 path->locks[level] = 1;
5983 wc->level = level;
5984 if (wc->level == 1)
5985 wc->reada_slot = 0;
5986 return 0;
5987 skip:
5988 wc->refs[level - 1] = 0;
5989 wc->flags[level - 1] = 0;
5990 if (wc->stage == DROP_REFERENCE) {
5991 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
5992 parent = path->nodes[level]->start;
5993 } else {
5994 BUG_ON(root->root_key.objectid !=
5995 btrfs_header_owner(path->nodes[level]));
5996 parent = 0;
5997 }
5998
5999 ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
6000 root->root_key.objectid, level - 1, 0);
6001 BUG_ON(ret);
6002 }
6003 btrfs_tree_unlock(next);
6004 free_extent_buffer(next);
6005 *lookup_info = 1;
6006 return 1;
6007 }
6008
6009 /*
6010 * hepler to process tree block while walking up the tree.
6011 *
6012 * when wc->stage == DROP_REFERENCE, this function drops
6013 * reference count on the block.
6014 *
6015 * when wc->stage == UPDATE_BACKREF, this function changes
6016 * wc->stage back to DROP_REFERENCE if we changed wc->stage
6017 * to UPDATE_BACKREF previously while processing the block.
6018 *
6019 * NOTE: return value 1 means we should stop walking up.
6020 */
6021 static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
6022 struct btrfs_root *root,
6023 struct btrfs_path *path,
6024 struct walk_control *wc)
6025 {
6026 int ret;
6027 int level = wc->level;
6028 struct extent_buffer *eb = path->nodes[level];
6029 u64 parent = 0;
6030
6031 if (wc->stage == UPDATE_BACKREF) {
6032 BUG_ON(wc->shared_level < level);
6033 if (level < wc->shared_level)
6034 goto out;
6035
6036 ret = find_next_key(path, level + 1, &wc->update_progress);
6037 if (ret > 0)
6038 wc->update_ref = 0;
6039
6040 wc->stage = DROP_REFERENCE;
6041 wc->shared_level = -1;
6042 path->slots[level] = 0;
6043
6044 /*
6045 * check reference count again if the block isn't locked.
6046 * we should start walking down the tree again if reference
6047 * count is one.
6048 */
6049 if (!path->locks[level]) {
6050 BUG_ON(level == 0);
6051 btrfs_tree_lock(eb);
6052 btrfs_set_lock_blocking(eb);
6053 path->locks[level] = 1;
6054
6055 ret = btrfs_lookup_extent_info(trans, root,
6056 eb->start, eb->len,
6057 &wc->refs[level],
6058 &wc->flags[level]);
6059 BUG_ON(ret);
6060 BUG_ON(wc->refs[level] == 0);
6061 if (wc->refs[level] == 1) {
6062 btrfs_tree_unlock(eb);
6063 path->locks[level] = 0;
6064 return 1;
6065 }
6066 }
6067 }
6068
6069 /* wc->stage == DROP_REFERENCE */
6070 BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
6071
6072 if (wc->refs[level] == 1) {
6073 if (level == 0) {
6074 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
6075 ret = btrfs_dec_ref(trans, root, eb, 1);
6076 else
6077 ret = btrfs_dec_ref(trans, root, eb, 0);
6078 BUG_ON(ret);
6079 }
6080 /* make block locked assertion in clean_tree_block happy */
6081 if (!path->locks[level] &&
6082 btrfs_header_generation(eb) == trans->transid) {
6083 btrfs_tree_lock(eb);
6084 btrfs_set_lock_blocking(eb);
6085 path->locks[level] = 1;
6086 }
6087 clean_tree_block(trans, root, eb);
6088 }
6089
6090 if (eb == root->node) {
6091 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
6092 parent = eb->start;
6093 else
6094 BUG_ON(root->root_key.objectid !=
6095 btrfs_header_owner(eb));
6096 } else {
6097 if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
6098 parent = path->nodes[level + 1]->start;
6099 else
6100 BUG_ON(root->root_key.objectid !=
6101 btrfs_header_owner(path->nodes[level + 1]));
6102 }
6103
6104 btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
6105 out:
6106 wc->refs[level] = 0;
6107 wc->flags[level] = 0;
6108 return 0;
6109 }
6110
6111 static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
6112 struct btrfs_root *root,
6113 struct btrfs_path *path,
6114 struct walk_control *wc)
6115 {
6116 int level = wc->level;
6117 int lookup_info = 1;
6118 int ret;
6119
6120 while (level >= 0) {
6121 ret = walk_down_proc(trans, root, path, wc, lookup_info);
6122 if (ret > 0)
6123 break;
6124
6125 if (level == 0)
6126 break;
6127
6128 if (path->slots[level] >=
6129 btrfs_header_nritems(path->nodes[level]))
6130 break;
6131
6132 ret = do_walk_down(trans, root, path, wc, &lookup_info);
6133 if (ret > 0) {
6134 path->slots[level]++;
6135 continue;
6136 } else if (ret < 0)
6137 return ret;
6138 level = wc->level;
6139 }
6140 return 0;
6141 }
6142
6143 static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
6144 struct btrfs_root *root,
6145 struct btrfs_path *path,
6146 struct walk_control *wc, int max_level)
6147 {
6148 int level = wc->level;
6149 int ret;
6150
6151 path->slots[level] = btrfs_header_nritems(path->nodes[level]);
6152 while (level < max_level && path->nodes[level]) {
6153 wc->level = level;
6154 if (path->slots[level] + 1 <
6155 btrfs_header_nritems(path->nodes[level])) {
6156 path->slots[level]++;
6157 return 0;
6158 } else {
6159 ret = walk_up_proc(trans, root, path, wc);
6160 if (ret > 0)
6161 return 0;
6162
6163 if (path->locks[level]) {
6164 btrfs_tree_unlock(path->nodes[level]);
6165 path->locks[level] = 0;
6166 }
6167 free_extent_buffer(path->nodes[level]);
6168 path->nodes[level] = NULL;
6169 level++;
6170 }
6171 }
6172 return 1;
6173 }
6174
6175 /*
6176 * drop a subvolume tree.
6177 *
6178 * this function traverses the tree freeing any blocks that only
6179 * referenced by the tree.
6180 *
6181 * when a shared tree block is found. this function decreases its
6182 * reference count by one. if update_ref is true, this function
6183 * also make sure backrefs for the shared block and all lower level
6184 * blocks are properly updated.
6185 */
6186 int btrfs_drop_snapshot(struct btrfs_root *root,
6187 struct btrfs_block_rsv *block_rsv, int update_ref)
6188 {
6189 struct btrfs_path *path;
6190 struct btrfs_trans_handle *trans;
6191 struct btrfs_root *tree_root = root->fs_info->tree_root;
6192 struct btrfs_root_item *root_item = &root->root_item;
6193 struct walk_control *wc;
6194 struct btrfs_key key;
6195 int err = 0;
6196 int ret;
6197 int level;
6198
6199 path = btrfs_alloc_path();
6200 BUG_ON(!path);
6201
6202 wc = kzalloc(sizeof(*wc), GFP_NOFS);
6203 BUG_ON(!wc);
6204
6205 trans = btrfs_start_transaction(tree_root, 0);
6206 if (block_rsv)
6207 trans->block_rsv = block_rsv;
6208
6209 if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
6210 level = btrfs_header_level(root->node);
6211 path->nodes[level] = btrfs_lock_root_node(root);
6212 btrfs_set_lock_blocking(path->nodes[level]);
6213 path->slots[level] = 0;
6214 path->locks[level] = 1;
6215 memset(&wc->update_progress, 0,
6216 sizeof(wc->update_progress));
6217 } else {
6218 btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
6219 memcpy(&wc->update_progress, &key,
6220 sizeof(wc->update_progress));
6221
6222 level = root_item->drop_level;
6223 BUG_ON(level == 0);
6224 path->lowest_level = level;
6225 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6226 path->lowest_level = 0;
6227 if (ret < 0) {
6228 err = ret;
6229 goto out;
6230 }
6231 WARN_ON(ret > 0);
6232
6233 /*
6234 * unlock our path, this is safe because only this
6235 * function is allowed to delete this snapshot
6236 */
6237 btrfs_unlock_up_safe(path, 0);
6238
6239 level = btrfs_header_level(root->node);
6240 while (1) {
6241 btrfs_tree_lock(path->nodes[level]);
6242 btrfs_set_lock_blocking(path->nodes[level]);
6243
6244 ret = btrfs_lookup_extent_info(trans, root,
6245 path->nodes[level]->start,
6246 path->nodes[level]->len,
6247 &wc->refs[level],
6248 &wc->flags[level]);
6249 BUG_ON(ret);
6250 BUG_ON(wc->refs[level] == 0);
6251
6252 if (level == root_item->drop_level)
6253 break;
6254
6255 btrfs_tree_unlock(path->nodes[level]);
6256 WARN_ON(wc->refs[level] != 1);
6257 level--;
6258 }
6259 }
6260
6261 wc->level = level;
6262 wc->shared_level = -1;
6263 wc->stage = DROP_REFERENCE;
6264 wc->update_ref = update_ref;
6265 wc->keep_locks = 0;
6266 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
6267
6268 while (1) {
6269 ret = walk_down_tree(trans, root, path, wc);
6270 if (ret < 0) {
6271 err = ret;
6272 break;
6273 }
6274
6275 ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
6276 if (ret < 0) {
6277 err = ret;
6278 break;
6279 }
6280
6281 if (ret > 0) {
6282 BUG_ON(wc->stage != DROP_REFERENCE);
6283 break;
6284 }
6285
6286 if (wc->stage == DROP_REFERENCE) {
6287 level = wc->level;
6288 btrfs_node_key(path->nodes[level],
6289 &root_item->drop_progress,
6290 path->slots[level]);
6291 root_item->drop_level = level;
6292 }
6293
6294 BUG_ON(wc->level == 0);
6295 if (btrfs_should_end_transaction(trans, tree_root)) {
6296 ret = btrfs_update_root(trans, tree_root,
6297 &root->root_key,
6298 root_item);
6299 BUG_ON(ret);
6300
6301 btrfs_end_transaction_throttle(trans, tree_root);
6302 trans = btrfs_start_transaction(tree_root, 0);
6303 if (block_rsv)
6304 trans->block_rsv = block_rsv;
6305 }
6306 }
6307 btrfs_release_path(root, path);
6308 BUG_ON(err);
6309
6310 ret = btrfs_del_root(trans, tree_root, &root->root_key);
6311 BUG_ON(ret);
6312
6313 if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
6314 ret = btrfs_find_last_root(tree_root, root->root_key.objectid,
6315 NULL, NULL);
6316 BUG_ON(ret < 0);
6317 if (ret > 0) {
6318 /* if we fail to delete the orphan item this time
6319 * around, it'll get picked up the next time.
6320 *
6321 * The most common failure here is just -ENOENT.
6322 */
6323 btrfs_del_orphan_item(trans, tree_root,
6324 root->root_key.objectid);
6325 }
6326 }
6327
6328 if (root->in_radix) {
6329 btrfs_free_fs_root(tree_root->fs_info, root);
6330 } else {
6331 free_extent_buffer(root->node);
6332 free_extent_buffer(root->commit_root);
6333 kfree(root);
6334 }
6335 out:
6336 btrfs_end_transaction_throttle(trans, tree_root);
6337 kfree(wc);
6338 btrfs_free_path(path);
6339 return err;
6340 }
6341
6342 /*
6343 * drop subtree rooted at tree block 'node'.
6344 *
6345 * NOTE: this function will unlock and release tree block 'node'
6346 */
6347 int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
6348 struct btrfs_root *root,
6349 struct extent_buffer *node,
6350 struct extent_buffer *parent)
6351 {
6352 struct btrfs_path *path;
6353 struct walk_control *wc;
6354 int level;
6355 int parent_level;
6356 int ret = 0;
6357 int wret;
6358
6359 BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);
6360
6361 path = btrfs_alloc_path();
6362 BUG_ON(!path);
6363
6364 wc = kzalloc(sizeof(*wc), GFP_NOFS);
6365 BUG_ON(!wc);
6366
6367 btrfs_assert_tree_locked(parent);
6368 parent_level = btrfs_header_level(parent);
6369 extent_buffer_get(parent);
6370 path->nodes[parent_level] = parent;
6371 path->slots[parent_level] = btrfs_header_nritems(parent);
6372
6373 btrfs_assert_tree_locked(node);
6374 level = btrfs_header_level(node);
6375 path->nodes[level] = node;
6376 path->slots[level] = 0;
6377 path->locks[level] = 1;
6378
6379 wc->refs[parent_level] = 1;
6380 wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
6381 wc->level = level;
6382 wc->shared_level = -1;
6383 wc->stage = DROP_REFERENCE;
6384 wc->update_ref = 0;
6385 wc->keep_locks = 1;
6386 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
6387
6388 while (1) {
6389 wret = walk_down_tree(trans, root, path, wc);
6390 if (wret < 0) {
6391 ret = wret;
6392 break;
6393 }
6394
6395 wret = walk_up_tree(trans, root, path, wc, parent_level);
6396 if (wret < 0)
6397 ret = wret;
6398 if (wret != 0)
6399 break;
6400 }
6401
6402 kfree(wc);
6403 btrfs_free_path(path);
6404 return ret;
6405 }
6406
6407 #if 0
6408 static unsigned long calc_ra(unsigned long start, unsigned long last,
6409 unsigned long nr)
6410 {
6411 return min(last, start + nr - 1);
6412 }
6413
6414 static noinline int relocate_inode_pages(struct inode *inode, u64 start,
6415 u64 len)
6416 {
6417 u64 page_start;
6418 u64 page_end;
6419 unsigned long first_index;
6420 unsigned long last_index;
6421 unsigned long i;
6422 struct page *page;
6423 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
6424 struct file_ra_state *ra;
6425 struct btrfs_ordered_extent *ordered;
6426 unsigned int total_read = 0;
6427 unsigned int total_dirty = 0;
6428 int ret = 0;
6429
6430 ra = kzalloc(sizeof(*ra), GFP_NOFS);
6431
6432 mutex_lock(&inode->i_mutex);
6433 first_index = start >> PAGE_CACHE_SHIFT;
6434 last_index = (start + len - 1) >> PAGE_CACHE_SHIFT;
6435
6436 /* make sure the dirty trick played by the caller work */
6437 ret = invalidate_inode_pages2_range(inode->i_mapping,
6438 first_index, last_index);
6439 if (ret)
6440 goto out_unlock;
6441
6442 file_ra_state_init(ra, inode->i_mapping);
6443
6444 for (i = first_index ; i <= last_index; i++) {
6445 if (total_read % ra->ra_pages == 0) {
6446 btrfs_force_ra(inode->i_mapping, ra, NULL, i,
6447 calc_ra(i, last_index, ra->ra_pages));
6448 }
6449 total_read++;
6450 again:
6451 if (((u64)i << PAGE_CACHE_SHIFT) > i_size_read(inode))
6452 BUG_ON(1);
6453 page = grab_cache_page(inode->i_mapping, i);
6454 if (!page) {
6455 ret = -ENOMEM;
6456 goto out_unlock;
6457 }
6458 if (!PageUptodate(page)) {
6459 btrfs_readpage(NULL, page);
6460 lock_page(page);
6461 if (!PageUptodate(page)) {
6462 unlock_page(page);
6463 page_cache_release(page);
6464 ret = -EIO;
6465 goto out_unlock;
6466 }
6467 }
6468 wait_on_page_writeback(page);
6469
6470 page_start = (u64)page->index << PAGE_CACHE_SHIFT;
6471 page_end = page_start + PAGE_CACHE_SIZE - 1;
6472 lock_extent(io_tree, page_start, page_end, GFP_NOFS);
6473
6474 ordered = btrfs_lookup_ordered_extent(inode, page_start);
6475 if (ordered) {
6476 unlock_extent(io_tree, page_start, page_end, GFP_NOFS);
6477 unlock_page(page);
6478 page_cache_release(page);
6479 btrfs_start_ordered_extent(inode, ordered, 1);
6480 btrfs_put_ordered_extent(ordered);
6481 goto again;
6482 }
6483 set_page_extent_mapped(page);
6484
6485 if (i == first_index)
6486 set_extent_bits(io_tree, page_start, page_end,
6487 EXTENT_BOUNDARY, GFP_NOFS);
6488 btrfs_set_extent_delalloc(inode, page_start, page_end);
6489
6490 set_page_dirty(page);
6491 total_dirty++;
6492
6493 unlock_extent(io_tree, page_start, page_end, GFP_NOFS);
6494 unlock_page(page);
6495 page_cache_release(page);
6496 }
6497
6498 out_unlock:
6499 kfree(ra);
6500 mutex_unlock(&inode->i_mutex);
6501 balance_dirty_pages_ratelimited_nr(inode->i_mapping, total_dirty);
6502 return ret;
6503 }
6504
6505 static noinline int relocate_data_extent(struct inode *reloc_inode,
6506 struct btrfs_key *extent_key,
6507 u64 offset)
6508 {
6509 struct btrfs_root *root = BTRFS_I(reloc_inode)->root;
6510 struct extent_map_tree *em_tree = &BTRFS_I(reloc_inode)->extent_tree;
6511 struct extent_map *em;
6512 u64 start = extent_key->objectid - offset;
6513 u64 end = start + extent_key->offset - 1;
6514
6515 em = alloc_extent_map(GFP_NOFS);
6516 BUG_ON(!em || IS_ERR(em));
6517
6518 em->start = start;
6519 em->len = extent_key->offset;
6520 em->block_len = extent_key->offset;
6521 em->block_start = extent_key->objectid;
6522 em->bdev = root->fs_info->fs_devices->latest_bdev;
6523 set_bit(EXTENT_FLAG_PINNED, &em->flags);
6524
6525 /* setup extent map to cheat btrfs_readpage */
6526 lock_extent(&BTRFS_I(reloc_inode)->io_tree, start, end, GFP_NOFS);
6527 while (1) {
6528 int ret;
6529 write_lock(&em_tree->lock);
6530 ret = add_extent_mapping(em_tree, em);
6531 write_unlock(&em_tree->lock);
6532 if (ret != -EEXIST) {
6533 free_extent_map(em);
6534 break;
6535 }
6536 btrfs_drop_extent_cache(reloc_inode, start, end, 0);
6537 }
6538 unlock_extent(&BTRFS_I(reloc_inode)->io_tree, start, end, GFP_NOFS);
6539
6540 return relocate_inode_pages(reloc_inode, start, extent_key->offset);
6541 }
6542
6543 struct btrfs_ref_path {
6544 u64 extent_start;
6545 u64 nodes[BTRFS_MAX_LEVEL];
6546 u64 root_objectid;
6547 u64 root_generation;
6548 u64 owner_objectid;
6549 u32 num_refs;
6550 int lowest_level;
6551 int current_level;
6552 int shared_level;
6553
6554 struct btrfs_key node_keys[BTRFS_MAX_LEVEL];
6555 u64 new_nodes[BTRFS_MAX_LEVEL];
6556 };
6557
6558 struct disk_extent {
6559 u64 ram_bytes;
6560 u64 disk_bytenr;
6561 u64 disk_num_bytes;
6562 u64 offset;
6563 u64 num_bytes;
6564 u8 compression;
6565 u8 encryption;
6566 u16 other_encoding;
6567 };
6568
6569 static int is_cowonly_root(u64 root_objectid)
6570 {
6571 if (root_objectid == BTRFS_ROOT_TREE_OBJECTID ||
6572 root_objectid == BTRFS_EXTENT_TREE_OBJECTID ||
6573 root_objectid == BTRFS_CHUNK_TREE_OBJECTID ||
6574 root_objectid == BTRFS_DEV_TREE_OBJECTID ||
6575 root_objectid == BTRFS_TREE_LOG_OBJECTID ||
6576 root_objectid == BTRFS_CSUM_TREE_OBJECTID)
6577 return 1;
6578 return 0;
6579 }
6580
6581 static noinline int __next_ref_path(struct btrfs_trans_handle *trans,
6582 struct btrfs_root *extent_root,
6583 struct btrfs_ref_path *ref_path,
6584 int first_time)
6585 {
6586 struct extent_buffer *leaf;
6587 struct btrfs_path *path;
6588 struct btrfs_extent_ref *ref;
6589 struct btrfs_key key;
6590 struct btrfs_key found_key;
6591 u64 bytenr;
6592 u32 nritems;
6593 int level;
6594 int ret = 1;
6595
6596 path = btrfs_alloc_path();
6597 if (!path)
6598 return -ENOMEM;
6599
6600 if (first_time) {
6601 ref_path->lowest_level = -1;
6602 ref_path->current_level = -1;
6603 ref_path->shared_level = -1;
6604 goto walk_up;
6605 }
6606 walk_down:
6607 level = ref_path->current_level - 1;
6608 while (level >= -1) {
6609 u64 parent;
6610 if (level < ref_path->lowest_level)
6611 break;
6612
6613 if (level >= 0)
6614 bytenr = ref_path->nodes[level];
6615 else
6616 bytenr = ref_path->extent_start;
6617 BUG_ON(bytenr == 0);
6618
6619 parent = ref_path->nodes[level + 1];
6620 ref_path->nodes[level + 1] = 0;
6621 ref_path->current_level = level;
6622 BUG_ON(parent == 0);
6623
6624 key.objectid = bytenr;
6625 key.offset = parent + 1;
6626 key.type = BTRFS_EXTENT_REF_KEY;
6627
6628 ret = btrfs_search_slot(trans, extent_root, &key, path, 0, 0);
6629 if (ret < 0)
6630 goto out;
6631 BUG_ON(ret == 0);
6632
6633 leaf = path->nodes[0];
6634 nritems = btrfs_header_nritems(leaf);
6635 if (path->slots[0] >= nritems) {
6636 ret = btrfs_next_leaf(extent_root, path);
6637 if (ret < 0)
6638 goto out;
6639 if (ret > 0)
6640 goto next;
6641 leaf = path->nodes[0];
6642 }
6643
6644 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6645 if (found_key.objectid == bytenr &&
6646 found_key.type == BTRFS_EXTENT_REF_KEY) {
6647 if (level < ref_path->shared_level)
6648 ref_path->shared_level = level;
6649 goto found;
6650 }
6651 next:
6652 level--;
6653 btrfs_release_path(extent_root, path);
6654 cond_resched();
6655 }
6656 /* reached lowest level */
6657 ret = 1;
6658 goto out;
6659 walk_up:
6660 level = ref_path->current_level;
6661 while (level < BTRFS_MAX_LEVEL - 1) {
6662 u64 ref_objectid;
6663
6664 if (level >= 0)
6665 bytenr = ref_path->nodes[level];
6666 else
6667 bytenr = ref_path->extent_start;
6668
6669 BUG_ON(bytenr == 0);
6670
6671 key.objectid = bytenr;
6672 key.offset = 0;
6673 key.type = BTRFS_EXTENT_REF_KEY;
6674
6675 ret = btrfs_search_slot(trans, extent_root, &key, path, 0, 0);
6676 if (ret < 0)
6677 goto out;
6678
6679 leaf = path->nodes[0];
6680 nritems = btrfs_header_nritems(leaf);
6681 if (path->slots[0] >= nritems) {
6682 ret = btrfs_next_leaf(extent_root, path);
6683 if (ret < 0)
6684 goto out;
6685 if (ret > 0) {
6686 /* the extent was freed by someone */
6687 if (ref_path->lowest_level == level)
6688 goto out;
6689 btrfs_release_path(extent_root, path);
6690 goto walk_down;
6691 }
6692 leaf = path->nodes[0];
6693 }
6694
6695 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6696 if (found_key.objectid != bytenr ||
6697 found_key.type != BTRFS_EXTENT_REF_KEY) {
6698 /* the extent was freed by someone */
6699 if (ref_path->lowest_level == level) {
6700 ret = 1;
6701 goto out;
6702 }
6703 btrfs_release_path(extent_root, path);
6704 goto walk_down;
6705 }
6706 found:
6707 ref = btrfs_item_ptr(leaf, path->slots[0],
6708 struct btrfs_extent_ref);
6709 ref_objectid = btrfs_ref_objectid(leaf, ref);
6710 if (ref_objectid < BTRFS_FIRST_FREE_OBJECTID) {
6711 if (first_time) {
6712 level = (int)ref_objectid;
6713 BUG_ON(level >= BTRFS_MAX_LEVEL);
6714 ref_path->lowest_level = level;
6715 ref_path->current_level = level;
6716 ref_path->nodes[level] = bytenr;
6717 } else {
6718 WARN_ON(ref_objectid != level);
6719 }
6720 } else {
6721 WARN_ON(level != -1);
6722 }
6723 first_time = 0;
6724
6725 if (ref_path->lowest_level == level) {
6726 ref_path->owner_objectid = ref_objectid;
6727 ref_path->num_refs = btrfs_ref_num_refs(leaf, ref);
6728 }
6729
6730 /*
6731 * the block is tree root or the block isn't in reference
6732 * counted tree.
6733 */
6734 if (found_key.objectid == found_key.offset ||
6735 is_cowonly_root(btrfs_ref_root(leaf, ref))) {
6736 ref_path->root_objectid = btrfs_ref_root(leaf, ref);
6737 ref_path->root_generation =
6738 btrfs_ref_generation(leaf, ref);
6739 if (level < 0) {
6740 /* special reference from the tree log */
6741 ref_path->nodes[0] = found_key.offset;
6742 ref_path->current_level = 0;
6743 }
6744 ret = 0;
6745 goto out;
6746 }
6747
6748 level++;
6749 BUG_ON(ref_path->nodes[level] != 0);
6750 ref_path->nodes[level] = found_key.offset;
6751 ref_path->current_level = level;
6752
6753 /*
6754 * the reference was created in the running transaction,
6755 * no need to continue walking up.
6756 */
6757 if (btrfs_ref_generation(leaf, ref) == trans->transid) {
6758 ref_path->root_objectid = btrfs_ref_root(leaf, ref);
6759 ref_path->root_generation =
6760 btrfs_ref_generation(leaf, ref);
6761 ret = 0;
6762 goto out;
6763 }
6764
6765 btrfs_release_path(extent_root, path);
6766 cond_resched();
6767 }
6768 /* reached max tree level, but no tree root found. */
6769 BUG();
6770 out:
6771 btrfs_free_path(path);
6772 return ret;
6773 }
6774
6775 static int btrfs_first_ref_path(struct btrfs_trans_handle *trans,
6776 struct btrfs_root *extent_root,
6777 struct btrfs_ref_path *ref_path,
6778 u64 extent_start)
6779 {
6780 memset(ref_path, 0, sizeof(*ref_path));
6781 ref_path->extent_start = extent_start;
6782
6783 return __next_ref_path(trans, extent_root, ref_path, 1);
6784 }
6785
6786 static int btrfs_next_ref_path(struct btrfs_trans_handle *trans,
6787 struct btrfs_root *extent_root,
6788 struct btrfs_ref_path *ref_path)
6789 {
6790 return __next_ref_path(trans, extent_root, ref_path, 0);
6791 }
6792
6793 static noinline int get_new_locations(struct inode *reloc_inode,
6794 struct btrfs_key *extent_key,
6795 u64 offset, int no_fragment,
6796 struct disk_extent **extents,
6797 int *nr_extents)
6798 {
6799 struct btrfs_root *root = BTRFS_I(reloc_inode)->root;
6800 struct btrfs_path *path;
6801 struct btrfs_file_extent_item *fi;
6802 struct extent_buffer *leaf;
6803 struct disk_extent *exts = *extents;
6804 struct btrfs_key found_key;
6805 u64 cur_pos;
6806 u64 last_byte;
6807 u32 nritems;
6808 int nr = 0;
6809 int max = *nr_extents;
6810 int ret;
6811
6812 WARN_ON(!no_fragment && *extents);
6813 if (!exts) {
6814 max = 1;
6815 exts = kmalloc(sizeof(*exts) * max, GFP_NOFS);
6816 if (!exts)
6817 return -ENOMEM;
6818 }
6819
6820 path = btrfs_alloc_path();
6821 BUG_ON(!path);
6822
6823 cur_pos = extent_key->objectid - offset;
6824 last_byte = extent_key->objectid + extent_key->offset;
6825 ret = btrfs_lookup_file_extent(NULL, root, path, reloc_inode->i_ino,
6826 cur_pos, 0);
6827 if (ret < 0)
6828 goto out;
6829 if (ret > 0) {
6830 ret = -ENOENT;
6831 goto out;
6832 }
6833
6834 while (1) {
6835 leaf = path->nodes[0];
6836 nritems = btrfs_header_nritems(leaf);
6837 if (path->slots[0] >= nritems) {
6838 ret = btrfs_next_leaf(root, path);
6839 if (ret < 0)
6840 goto out;
6841 if (ret > 0)
6842 break;
6843 leaf = path->nodes[0];
6844 }
6845
6846 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6847 if (found_key.offset != cur_pos ||
6848 found_key.type != BTRFS_EXTENT_DATA_KEY ||
6849 found_key.objectid != reloc_inode->i_ino)
6850 break;
6851
6852 fi = btrfs_item_ptr(leaf, path->slots[0],
6853 struct btrfs_file_extent_item);
6854 if (btrfs_file_extent_type(leaf, fi) !=
6855 BTRFS_FILE_EXTENT_REG ||
6856 btrfs_file_extent_disk_bytenr(leaf, fi) == 0)
6857 break;
6858
6859 if (nr == max) {
6860 struct disk_extent *old = exts;
6861 max *= 2;
6862 exts = kzalloc(sizeof(*exts) * max, GFP_NOFS);
6863 memcpy(exts, old, sizeof(*exts) * nr);
6864 if (old != *extents)
6865 kfree(old);
6866 }
6867
6868 exts[nr].disk_bytenr =
6869 btrfs_file_extent_disk_bytenr(leaf, fi);
6870 exts[nr].disk_num_bytes =
6871 btrfs_file_extent_disk_num_bytes(leaf, fi);
6872 exts[nr].offset = btrfs_file_extent_offset(leaf, fi);
6873 exts[nr].num_bytes = btrfs_file_extent_num_bytes(leaf, fi);
6874 exts[nr].ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
6875 exts[nr].compression = btrfs_file_extent_compression(leaf, fi);
6876 exts[nr].encryption = btrfs_file_extent_encryption(leaf, fi);
6877 exts[nr].other_encoding = btrfs_file_extent_other_encoding(leaf,
6878 fi);
6879 BUG_ON(exts[nr].offset > 0);
6880 BUG_ON(exts[nr].compression || exts[nr].encryption);
6881 BUG_ON(exts[nr].num_bytes != exts[nr].disk_num_bytes);
6882
6883 cur_pos += exts[nr].num_bytes;
6884 nr++;
6885
6886 if (cur_pos + offset >= last_byte)
6887 break;
6888
6889 if (no_fragment) {
6890 ret = 1;
6891 goto out;
6892 }
6893 path->slots[0]++;
6894 }
6895
6896 BUG_ON(cur_pos + offset > last_byte);
6897 if (cur_pos + offset < last_byte) {
6898 ret = -ENOENT;
6899 goto out;
6900 }
6901 ret = 0;
6902 out:
6903 btrfs_free_path(path);
6904 if (ret) {
6905 if (exts != *extents)
6906 kfree(exts);
6907 } else {
6908 *extents = exts;
6909 *nr_extents = nr;
6910 }
6911 return ret;
6912 }
6913
6914 static noinline int replace_one_extent(struct btrfs_trans_handle *trans,
6915 struct btrfs_root *root,
6916 struct btrfs_path *path,
6917 struct btrfs_key *extent_key,
6918 struct btrfs_key *leaf_key,
6919 struct btrfs_ref_path *ref_path,
6920 struct disk_extent *new_extents,
6921 int nr_extents)
6922 {
6923 struct extent_buffer *leaf;
6924 struct btrfs_file_extent_item *fi;
6925 struct inode *inode = NULL;
6926 struct btrfs_key key;
6927 u64 lock_start = 0;
6928 u64 lock_end = 0;
6929 u64 num_bytes;
6930 u64 ext_offset;
6931 u64 search_end = (u64)-1;
6932 u32 nritems;
6933 int nr_scaned = 0;
6934 int extent_locked = 0;
6935 int extent_type;
6936 int ret;
6937
6938 memcpy(&key, leaf_key, sizeof(key));
6939 if (ref_path->owner_objectid != BTRFS_MULTIPLE_OBJECTIDS) {
6940 if (key.objectid < ref_path->owner_objectid ||
6941 (key.objectid == ref_path->owner_objectid &&
6942 key.type < BTRFS_EXTENT_DATA_KEY)) {
6943 key.objectid = ref_path->owner_objectid;
6944 key.type = BTRFS_EXTENT_DATA_KEY;
6945 key.offset = 0;
6946 }
6947 }
6948
6949 while (1) {
6950 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
6951 if (ret < 0)
6952 goto out;
6953
6954 leaf = path->nodes[0];
6955 nritems = btrfs_header_nritems(leaf);
6956 next:
6957 if (extent_locked && ret > 0) {
6958 /*
6959 * the file extent item was modified by someone
6960 * before the extent got locked.
6961 */
6962 unlock_extent(&BTRFS_I(inode)->io_tree, lock_start,
6963 lock_end, GFP_NOFS);
6964 extent_locked = 0;
6965 }
6966
6967 if (path->slots[0] >= nritems) {
6968 if (++nr_scaned > 2)
6969 break;
6970
6971 BUG_ON(extent_locked);
6972 ret = btrfs_next_leaf(root, path);
6973 if (ret < 0)
6974 goto out;
6975 if (ret > 0)
6976 break;
6977 leaf = path->nodes[0];
6978 nritems = btrfs_header_nritems(leaf);
6979 }
6980
6981 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
6982
6983 if (ref_path->owner_objectid != BTRFS_MULTIPLE_OBJECTIDS) {
6984 if ((key.objectid > ref_path->owner_objectid) ||
6985 (key.objectid == ref_path->owner_objectid &&
6986 key.type > BTRFS_EXTENT_DATA_KEY) ||
6987 key.offset >= search_end)
6988 break;
6989 }
6990
6991 if (inode && key.objectid != inode->i_ino) {
6992 BUG_ON(extent_locked);
6993 btrfs_release_path(root, path);
6994 mutex_unlock(&inode->i_mutex);
6995 iput(inode);
6996 inode = NULL;
6997 continue;
6998 }
6999
7000 if (key.type != BTRFS_EXTENT_DATA_KEY) {
7001 path->slots[0]++;
7002 ret = 1;
7003 goto next;
7004 }
7005 fi = btrfs_item_ptr(leaf, path->slots[0],
7006 struct btrfs_file_extent_item);
7007 extent_type = btrfs_file_extent_type(leaf, fi);
7008 if ((extent_type != BTRFS_FILE_EXTENT_REG &&
7009 extent_type != BTRFS_FILE_EXTENT_PREALLOC) ||
7010 (btrfs_file_extent_disk_bytenr(leaf, fi) !=
7011 extent_key->objectid)) {
7012 path->slots[0]++;
7013 ret = 1;
7014 goto next;
7015 }
7016
7017 num_bytes = btrfs_file_extent_num_bytes(leaf, fi);
7018 ext_offset = btrfs_file_extent_offset(leaf, fi);
7019
7020 if (search_end == (u64)-1) {
7021 search_end = key.offset - ext_offset +
7022 btrfs_file_extent_ram_bytes(leaf, fi);
7023 }
7024
7025 if (!extent_locked) {
7026 lock_start = key.offset;
7027 lock_end = lock_start + num_bytes - 1;
7028 } else {
7029 if (lock_start > key.offset ||
7030 lock_end + 1 < key.offset + num_bytes) {
7031 unlock_extent(&BTRFS_I(inode)->io_tree,
7032 lock_start, lock_end, GFP_NOFS);
7033 extent_locked = 0;
7034 }
7035 }
7036
7037 if (!inode) {
7038 btrfs_release_path(root, path);
7039
7040 inode = btrfs_iget_locked(root->fs_info->sb,
7041 key.objectid, root);
7042 if (inode->i_state & I_NEW) {
7043 BTRFS_I(inode)->root = root;
7044 BTRFS_I(inode)->location.objectid =
7045 key.objectid;
7046 BTRFS_I(inode)->location.type =
7047 BTRFS_INODE_ITEM_KEY;
7048 BTRFS_I(inode)->location.offset = 0;
7049 btrfs_read_locked_inode(inode);
7050 unlock_new_inode(inode);
7051 }
7052 /*
7053 * some code call btrfs_commit_transaction while
7054 * holding the i_mutex, so we can't use mutex_lock
7055 * here.
7056 */
7057 if (is_bad_inode(inode) ||
7058 !mutex_trylock(&inode->i_mutex)) {
7059 iput(inode);
7060 inode = NULL;
7061 key.offset = (u64)-1;
7062 goto skip;
7063 }
7064 }
7065
7066 if (!extent_locked) {
7067 struct btrfs_ordered_extent *ordered;
7068
7069 btrfs_release_path(root, path);
7070
7071 lock_extent(&BTRFS_I(inode)->io_tree, lock_start,
7072 lock_end, GFP_NOFS);
7073 ordered = btrfs_lookup_first_ordered_extent(inode,
7074 lock_end);
7075 if (ordered &&
7076 ordered->file_offset <= lock_end &&
7077 ordered->file_offset + ordered->len > lock_start) {
7078 unlock_extent(&BTRFS_I(inode)->io_tree,
7079 lock_start, lock_end, GFP_NOFS);
7080 btrfs_start_ordered_extent(inode, ordered, 1);
7081 btrfs_put_ordered_extent(ordered);
7082 key.offset += num_bytes;
7083 goto skip;
7084 }
7085 if (ordered)
7086 btrfs_put_ordered_extent(ordered);
7087
7088 extent_locked = 1;
7089 continue;
7090 }
7091
7092 if (nr_extents == 1) {
7093 /* update extent pointer in place */
7094 btrfs_set_file_extent_disk_bytenr(leaf, fi,
7095 new_extents[0].disk_bytenr);
7096 btrfs_set_file_extent_disk_num_bytes(leaf, fi,
7097 new_extents[0].disk_num_bytes);
7098 btrfs_mark_buffer_dirty(leaf);
7099
7100 btrfs_drop_extent_cache(inode, key.offset,
7101 key.offset + num_bytes - 1, 0);
7102
7103 ret = btrfs_inc_extent_ref(trans, root,
7104 new_extents[0].disk_bytenr,
7105 new_extents[0].disk_num_bytes,
7106 leaf->start,
7107 root->root_key.objectid,
7108 trans->transid,
7109 key.objectid);
7110 BUG_ON(ret);
7111
7112 ret = btrfs_free_extent(trans, root,
7113 extent_key->objectid,
7114 extent_key->offset,
7115 leaf->start,
7116 btrfs_header_owner(leaf),
7117 btrfs_header_generation(leaf),
7118 key.objectid, 0);
7119 BUG_ON(ret);
7120
7121 btrfs_release_path(root, path);
7122 key.offset += num_bytes;
7123 } else {
7124 BUG_ON(1);
7125 #if 0
7126 u64 alloc_hint;
7127 u64 extent_len;
7128 int i;
7129 /*
7130 * drop old extent pointer at first, then insert the
7131 * new pointers one bye one
7132 */
7133 btrfs_release_path(root, path);
7134 ret = btrfs_drop_extents(trans, root, inode, key.offset,
7135 key.offset + num_bytes,
7136 key.offset, &alloc_hint);
7137 BUG_ON(ret);
7138
7139 for (i = 0; i < nr_extents; i++) {
7140 if (ext_offset >= new_extents[i].num_bytes) {
7141 ext_offset -= new_extents[i].num_bytes;
7142 continue;
7143 }
7144 extent_len = min(new_extents[i].num_bytes -
7145 ext_offset, num_bytes);
7146
7147 ret = btrfs_insert_empty_item(trans, root,
7148 path, &key,
7149 sizeof(*fi));
7150 BUG_ON(ret);
7151
7152 leaf = path->nodes[0];
7153 fi = btrfs_item_ptr(leaf, path->slots[0],
7154 struct btrfs_file_extent_item);
7155 btrfs_set_file_extent_generation(leaf, fi,
7156 trans->transid);
7157 btrfs_set_file_extent_type(leaf, fi,
7158 BTRFS_FILE_EXTENT_REG);
7159 btrfs_set_file_extent_disk_bytenr(leaf, fi,
7160 new_extents[i].disk_bytenr);
7161 btrfs_set_file_extent_disk_num_bytes(leaf, fi,
7162 new_extents[i].disk_num_bytes);
7163 btrfs_set_file_extent_ram_bytes(leaf, fi,
7164 new_extents[i].ram_bytes);
7165
7166 btrfs_set_file_extent_compression(leaf, fi,
7167 new_extents[i].compression);
7168 btrfs_set_file_extent_encryption(leaf, fi,
7169 new_extents[i].encryption);
7170 btrfs_set_file_extent_other_encoding(leaf, fi,
7171 new_extents[i].other_encoding);
7172
7173 btrfs_set_file_extent_num_bytes(leaf, fi,
7174 extent_len);
7175 ext_offset += new_extents[i].offset;
7176 btrfs_set_file_extent_offset(leaf, fi,
7177 ext_offset);
7178 btrfs_mark_buffer_dirty(leaf);
7179
7180 btrfs_drop_extent_cache(inode, key.offset,
7181 key.offset + extent_len - 1, 0);
7182
7183 ret = btrfs_inc_extent_ref(trans, root,
7184 new_extents[i].disk_bytenr,
7185 new_extents[i].disk_num_bytes,
7186 leaf->start,
7187 root->root_key.objectid,
7188 trans->transid, key.objectid);
7189 BUG_ON(ret);
7190 btrfs_release_path(root, path);
7191
7192 inode_add_bytes(inode, extent_len);
7193
7194 ext_offset = 0;
7195 num_bytes -= extent_len;
7196 key.offset += extent_len;
7197
7198 if (num_bytes == 0)
7199 break;
7200 }
7201 BUG_ON(i >= nr_extents);
7202 #endif
7203 }
7204
7205 if (extent_locked) {
7206 unlock_extent(&BTRFS_I(inode)->io_tree, lock_start,
7207 lock_end, GFP_NOFS);
7208 extent_locked = 0;
7209 }
7210 skip:
7211 if (ref_path->owner_objectid != BTRFS_MULTIPLE_OBJECTIDS &&
7212 key.offset >= search_end)
7213 break;
7214
7215 cond_resched();
7216 }
7217 ret = 0;
7218 out:
7219 btrfs_release_path(root, path);
7220 if (inode) {
7221 mutex_unlock(&inode->i_mutex);
7222 if (extent_locked) {
7223 unlock_extent(&BTRFS_I(inode)->io_tree, lock_start,
7224 lock_end, GFP_NOFS);
7225 }
7226 iput(inode);
7227 }
7228 return ret;
7229 }
7230
7231 int btrfs_reloc_tree_cache_ref(struct btrfs_trans_handle *trans,
7232 struct btrfs_root *root,
7233 struct extent_buffer *buf, u64 orig_start)
7234 {
7235 int level;
7236 int ret;
7237
7238 BUG_ON(btrfs_header_generation(buf) != trans->transid);
7239 BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);
7240
7241 level = btrfs_header_level(buf);
7242 if (level == 0) {
7243 struct btrfs_leaf_ref *ref;
7244 struct btrfs_leaf_ref *orig_ref;
7245
7246 orig_ref = btrfs_lookup_leaf_ref(root, orig_start);
7247 if (!orig_ref)
7248 return -ENOENT;
7249
7250 ref = btrfs_alloc_leaf_ref(root, orig_ref->nritems);
7251 if (!ref) {
7252 btrfs_free_leaf_ref(root, orig_ref);
7253 return -ENOMEM;
7254 }
7255
7256 ref->nritems = orig_ref->nritems;
7257 memcpy(ref->extents, orig_ref->extents,
7258 sizeof(ref->extents[0]) * ref->nritems);
7259
7260 btrfs_free_leaf_ref(root, orig_ref);
7261
7262 ref->root_gen = trans->transid;
7263 ref->bytenr = buf->start;
7264 ref->owner = btrfs_header_owner(buf);
7265 ref->generation = btrfs_header_generation(buf);
7266
7267 ret = btrfs_add_leaf_ref(root, ref, 0);
7268 WARN_ON(ret);
7269 btrfs_free_leaf_ref(root, ref);
7270 }
7271 return 0;
7272 }
7273
7274 static noinline int invalidate_extent_cache(struct btrfs_root *root,
7275 struct extent_buffer *leaf,
7276 struct btrfs_block_group_cache *group,
7277 struct btrfs_root *target_root)
7278 {
7279 struct btrfs_key key;
7280 struct inode *inode = NULL;
7281 struct btrfs_file_extent_item *fi;
7282 struct extent_state *cached_state = NULL;
7283 u64 num_bytes;
7284 u64 skip_objectid = 0;
7285 u32 nritems;
7286 u32 i;
7287
7288 nritems = btrfs_header_nritems(leaf);
7289 for (i = 0; i < nritems; i++) {
7290 btrfs_item_key_to_cpu(leaf, &key, i);
7291 if (key.objectid == skip_objectid ||
7292 key.type != BTRFS_EXTENT_DATA_KEY)
7293 continue;
7294 fi = btrfs_item_ptr(leaf, i, struct btrfs_file_extent_item);
7295 if (btrfs_file_extent_type(leaf, fi) ==
7296 BTRFS_FILE_EXTENT_INLINE)
7297 continue;
7298 if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0)
7299 continue;
7300 if (!inode || inode->i_ino != key.objectid) {
7301 iput(inode);
7302 inode = btrfs_ilookup(target_root->fs_info->sb,
7303 key.objectid, target_root, 1);
7304 }
7305 if (!inode) {
7306 skip_objectid = key.objectid;
7307 continue;
7308 }
7309 num_bytes = btrfs_file_extent_num_bytes(leaf, fi);
7310
7311 lock_extent_bits(&BTRFS_I(inode)->io_tree, key.offset,
7312 key.offset + num_bytes - 1, 0, &cached_state,
7313 GFP_NOFS);
7314 btrfs_drop_extent_cache(inode, key.offset,
7315 key.offset + num_bytes - 1, 1);
7316 unlock_extent_cached(&BTRFS_I(inode)->io_tree, key.offset,
7317 key.offset + num_bytes - 1, &cached_state,
7318 GFP_NOFS);
7319 cond_resched();
7320 }
7321 iput(inode);
7322 return 0;
7323 }
7324
7325 static noinline int replace_extents_in_leaf(struct btrfs_trans_handle *trans,
7326 struct btrfs_root *root,
7327 struct extent_buffer *leaf,
7328 struct btrfs_block_group_cache *group,
7329 struct inode *reloc_inode)
7330 {
7331 struct btrfs_key key;
7332 struct btrfs_key extent_key;
7333 struct btrfs_file_extent_item *fi;
7334 struct btrfs_leaf_ref *ref;
7335 struct disk_extent *new_extent;
7336 u64 bytenr;
7337 u64 num_bytes;
7338 u32 nritems;
7339 u32 i;
7340 int ext_index;
7341 int nr_extent;
7342 int ret;
7343
7344 new_extent = kmalloc(sizeof(*new_extent), GFP_NOFS);
7345 BUG_ON(!new_extent);
7346
7347 ref = btrfs_lookup_leaf_ref(root, leaf->start);
7348 BUG_ON(!ref);
7349
7350 ext_index = -1;
7351 nritems = btrfs_header_nritems(leaf);
7352 for (i = 0; i < nritems; i++) {
7353 btrfs_item_key_to_cpu(leaf, &key, i);
7354 if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
7355 continue;
7356 fi = btrfs_item_ptr(leaf, i, struct btrfs_file_extent_item);
7357 if (btrfs_file_extent_type(leaf, fi) ==
7358 BTRFS_FILE_EXTENT_INLINE)
7359 continue;
7360 bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
7361 num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
7362 if (bytenr == 0)
7363 continue;
7364
7365 ext_index++;
7366 if (bytenr >= group->key.objectid + group->key.offset ||
7367 bytenr + num_bytes <= group->key.objectid)
7368 continue;
7369
7370 extent_key.objectid = bytenr;
7371 extent_key.offset = num_bytes;
7372 extent_key.type = BTRFS_EXTENT_ITEM_KEY;
7373 nr_extent = 1;
7374 ret = get_new_locations(reloc_inode, &extent_key,
7375 group->key.objectid, 1,
7376 &new_extent, &nr_extent);
7377 if (ret > 0)
7378 continue;
7379 BUG_ON(ret < 0);
7380
7381 BUG_ON(ref->extents[ext_index].bytenr != bytenr);
7382 BUG_ON(ref->extents[ext_index].num_bytes != num_bytes);
7383 ref->extents[ext_index].bytenr = new_extent->disk_bytenr;
7384 ref->extents[ext_index].num_bytes = new_extent->disk_num_bytes;
7385
7386 btrfs_set_file_extent_disk_bytenr(leaf, fi,
7387 new_extent->disk_bytenr);
7388 btrfs_set_file_extent_disk_num_bytes(leaf, fi,
7389 new_extent->disk_num_bytes);
7390 btrfs_mark_buffer_dirty(leaf);
7391
7392 ret = btrfs_inc_extent_ref(trans, root,
7393 new_extent->disk_bytenr,
7394 new_extent->disk_num_bytes,
7395 leaf->start,
7396 root->root_key.objectid,
7397 trans->transid, key.objectid);
7398 BUG_ON(ret);
7399
7400 ret = btrfs_free_extent(trans, root,
7401 bytenr, num_bytes, leaf->start,
7402 btrfs_header_owner(leaf),
7403 btrfs_header_generation(leaf),
7404 key.objectid, 0);
7405 BUG_ON(ret);
7406 cond_resched();
7407 }
7408 kfree(new_extent);
7409 BUG_ON(ext_index + 1 != ref->nritems);
7410 btrfs_free_leaf_ref(root, ref);
7411 return 0;
7412 }
7413
7414 int btrfs_free_reloc_root(struct btrfs_trans_handle *trans,
7415 struct btrfs_root *root)
7416 {
7417 struct btrfs_root *reloc_root;
7418 int ret;
7419
7420 if (root->reloc_root) {
7421 reloc_root = root->reloc_root;
7422 root->reloc_root = NULL;
7423 list_add(&reloc_root->dead_list,
7424 &root->fs_info->dead_reloc_roots);
7425
7426 btrfs_set_root_bytenr(&reloc_root->root_item,
7427 reloc_root->node->start);
7428 btrfs_set_root_level(&root->root_item,
7429 btrfs_header_level(reloc_root->node));
7430 memset(&reloc_root->root_item.drop_progress, 0,
7431 sizeof(struct btrfs_disk_key));
7432 reloc_root->root_item.drop_level = 0;
7433
7434 ret = btrfs_update_root(trans, root->fs_info->tree_root,
7435 &reloc_root->root_key,
7436 &reloc_root->root_item);
7437 BUG_ON(ret);
7438 }
7439 return 0;
7440 }
7441
7442 int btrfs_drop_dead_reloc_roots(struct btrfs_root *root)
7443 {
7444 struct btrfs_trans_handle *trans;
7445 struct btrfs_root *reloc_root;
7446 struct btrfs_root *prev_root = NULL;
7447 struct list_head dead_roots;
7448 int ret;
7449 unsigned long nr;
7450
7451 INIT_LIST_HEAD(&dead_roots);
7452 list_splice_init(&root->fs_info->dead_reloc_roots, &dead_roots);
7453
7454 while (!list_empty(&dead_roots)) {
7455 reloc_root = list_entry(dead_roots.prev,
7456 struct btrfs_root, dead_list);
7457 list_del_init(&reloc_root->dead_list);
7458
7459 BUG_ON(reloc_root->commit_root != NULL);
7460 while (1) {
7461 trans = btrfs_join_transaction(root, 1);
7462 BUG_ON(!trans);
7463
7464 mutex_lock(&root->fs_info->drop_mutex);
7465 ret = btrfs_drop_snapshot(trans, reloc_root);
7466 if (ret != -EAGAIN)
7467 break;
7468 mutex_unlock(&root->fs_info->drop_mutex);
7469
7470 nr = trans->blocks_used;
7471 ret = btrfs_end_transaction(trans, root);
7472 BUG_ON(ret);
7473 btrfs_btree_balance_dirty(root, nr);
7474 }
7475
7476 free_extent_buffer(reloc_root->node);
7477
7478 ret = btrfs_del_root(trans, root->fs_info->tree_root,
7479 &reloc_root->root_key);
7480 BUG_ON(ret);
7481 mutex_unlock(&root->fs_info->drop_mutex);
7482
7483 nr = trans->blocks_used;
7484 ret = btrfs_end_transaction(trans, root);
7485 BUG_ON(ret);
7486 btrfs_btree_balance_dirty(root, nr);
7487
7488 kfree(prev_root);
7489 prev_root = reloc_root;
7490 }
7491 if (prev_root) {
7492 btrfs_remove_leaf_refs(prev_root, (u64)-1, 0);
7493 kfree(prev_root);
7494 }
7495 return 0;
7496 }
7497
7498 int btrfs_add_dead_reloc_root(struct btrfs_root *root)
7499 {
7500 list_add(&root->dead_list, &root->fs_info->dead_reloc_roots);
7501 return 0;
7502 }
7503
7504 int btrfs_cleanup_reloc_trees(struct btrfs_root *root)
7505 {
7506 struct btrfs_root *reloc_root;
7507 struct btrfs_trans_handle *trans;
7508 struct btrfs_key location;
7509 int found;
7510 int ret;
7511
7512 mutex_lock(&root->fs_info->tree_reloc_mutex);
7513 ret = btrfs_find_dead_roots(root, BTRFS_TREE_RELOC_OBJECTID, NULL);
7514 BUG_ON(ret);
7515 found = !list_empty(&root->fs_info->dead_reloc_roots);
7516 mutex_unlock(&root->fs_info->tree_reloc_mutex);
7517
7518 if (found) {
7519 trans = btrfs_start_transaction(root, 1);
7520 BUG_ON(!trans);
7521 ret = btrfs_commit_transaction(trans, root);
7522 BUG_ON(ret);
7523 }
7524
7525 location.objectid = BTRFS_DATA_RELOC_TREE_OBJECTID;
7526 location.offset = (u64)-1;
7527 location.type = BTRFS_ROOT_ITEM_KEY;
7528
7529 reloc_root = btrfs_read_fs_root_no_name(root->fs_info, &location);
7530 BUG_ON(!reloc_root);
7531 btrfs_orphan_cleanup(reloc_root);
7532 return 0;
7533 }
7534
7535 static noinline int init_reloc_tree(struct btrfs_trans_handle *trans,
7536 struct btrfs_root *root)
7537 {
7538 struct btrfs_root *reloc_root;
7539 struct extent_buffer *eb;
7540 struct btrfs_root_item *root_item;
7541 struct btrfs_key root_key;
7542 int ret;
7543
7544 BUG_ON(!root->ref_cows);
7545 if (root->reloc_root)
7546 return 0;
7547
7548 root_item = kmalloc(sizeof(*root_item), GFP_NOFS);
7549 BUG_ON(!root_item);
7550
7551 ret = btrfs_copy_root(trans, root, root->commit_root,
7552 &eb, BTRFS_TREE_RELOC_OBJECTID);
7553 BUG_ON(ret);
7554
7555 root_key.objectid = BTRFS_TREE_RELOC_OBJECTID;
7556 root_key.offset = root->root_key.objectid;
7557 root_key.type = BTRFS_ROOT_ITEM_KEY;
7558
7559 memcpy(root_item, &root->root_item, sizeof(root_item));
7560 btrfs_set_root_refs(root_item, 0);
7561 btrfs_set_root_bytenr(root_item, eb->start);
7562 btrfs_set_root_level(root_item, btrfs_header_level(eb));
7563 btrfs_set_root_generation(root_item, trans->transid);
7564
7565 btrfs_tree_unlock(eb);
7566 free_extent_buffer(eb);
7567
7568 ret = btrfs_insert_root(trans, root->fs_info->tree_root,
7569 &root_key, root_item);
7570 BUG_ON(ret);
7571 kfree(root_item);
7572
7573 reloc_root = btrfs_read_fs_root_no_radix(root->fs_info->tree_root,
7574 &root_key);
7575 BUG_ON(!reloc_root);
7576 reloc_root->last_trans = trans->transid;
7577 reloc_root->commit_root = NULL;
7578 reloc_root->ref_tree = &root->fs_info->reloc_ref_tree;
7579
7580 root->reloc_root = reloc_root;
7581 return 0;
7582 }
7583
7584 /*
7585 * Core function of space balance.
7586 *
7587 * The idea is using reloc trees to relocate tree blocks in reference
7588 * counted roots. There is one reloc tree for each subvol, and all
7589 * reloc trees share same root key objectid. Reloc trees are snapshots
7590 * of the latest committed roots of subvols (root->commit_root).
7591 *
7592 * To relocate a tree block referenced by a subvol, there are two steps.
7593 * COW the block through subvol's reloc tree, then update block pointer
7594 * in the subvol to point to the new block. Since all reloc trees share
7595 * same root key objectid, doing special handing for tree blocks owned
7596 * by them is easy. Once a tree block has been COWed in one reloc tree,
7597 * we can use the resulting new block directly when the same block is
7598 * required to COW again through other reloc trees. By this way, relocated
7599 * tree blocks are shared between reloc trees, so they are also shared
7600 * between subvols.
7601 */
7602 static noinline int relocate_one_path(struct btrfs_trans_handle *trans,
7603 struct btrfs_root *root,
7604 struct btrfs_path *path,
7605 struct btrfs_key *first_key,
7606 struct btrfs_ref_path *ref_path,
7607 struct btrfs_block_group_cache *group,
7608 struct inode *reloc_inode)
7609 {
7610 struct btrfs_root *reloc_root;
7611 struct extent_buffer *eb = NULL;
7612 struct btrfs_key *keys;
7613 u64 *nodes;
7614 int level;
7615 int shared_level;
7616 int lowest_level = 0;
7617 int ret;
7618
7619 if (ref_path->owner_objectid < BTRFS_FIRST_FREE_OBJECTID)
7620 lowest_level = ref_path->owner_objectid;
7621
7622 if (!root->ref_cows) {
7623 path->lowest_level = lowest_level;
7624 ret = btrfs_search_slot(trans, root, first_key, path, 0, 1);
7625 BUG_ON(ret < 0);
7626 path->lowest_level = 0;
7627 btrfs_release_path(root, path);
7628 return 0;
7629 }
7630
7631 mutex_lock(&root->fs_info->tree_reloc_mutex);
7632 ret = init_reloc_tree(trans, root);
7633 BUG_ON(ret);
7634 reloc_root = root->reloc_root;
7635
7636 shared_level = ref_path->shared_level;
7637 ref_path->shared_level = BTRFS_MAX_LEVEL - 1;
7638
7639 keys = ref_path->node_keys;
7640 nodes = ref_path->new_nodes;
7641 memset(&keys[shared_level + 1], 0,
7642 sizeof(*keys) * (BTRFS_MAX_LEVEL - shared_level - 1));
7643 memset(&nodes[shared_level + 1], 0,
7644 sizeof(*nodes) * (BTRFS_MAX_LEVEL - shared_level - 1));
7645
7646 if (nodes[lowest_level] == 0) {
7647 path->lowest_level = lowest_level;
7648 ret = btrfs_search_slot(trans, reloc_root, first_key, path,
7649 0, 1);
7650 BUG_ON(ret);
7651 for (level = lowest_level; level < BTRFS_MAX_LEVEL; level++) {
7652 eb = path->nodes[level];
7653 if (!eb || eb == reloc_root->node)
7654 break;
7655 nodes[level] = eb->start;
7656 if (level == 0)
7657 btrfs_item_key_to_cpu(eb, &keys[level], 0);
7658 else
7659 btrfs_node_key_to_cpu(eb, &keys[level], 0);
7660 }
7661 if (nodes[0] &&
7662 ref_path->owner_objectid >= BTRFS_FIRST_FREE_OBJECTID) {
7663 eb = path->nodes[0];
7664 ret = replace_extents_in_leaf(trans, reloc_root, eb,
7665 group, reloc_inode);
7666 BUG_ON(ret);
7667 }
7668 btrfs_release_path(reloc_root, path);
7669 } else {
7670 ret = btrfs_merge_path(trans, reloc_root, keys, nodes,
7671 lowest_level);
7672 BUG_ON(ret);
7673 }
7674
7675 /*
7676 * replace tree blocks in the fs tree with tree blocks in
7677 * the reloc tree.
7678 */
7679 ret = btrfs_merge_path(trans, root, keys, nodes, lowest_level);
7680 BUG_ON(ret < 0);
7681
7682 if (ref_path->owner_objectid >= BTRFS_FIRST_FREE_OBJECTID) {
7683 ret = btrfs_search_slot(trans, reloc_root, first_key, path,
7684 0, 0);
7685 BUG_ON(ret);
7686 extent_buffer_get(path->nodes[0]);
7687 eb = path->nodes[0];
7688 btrfs_release_path(reloc_root, path);
7689 ret = invalidate_extent_cache(reloc_root, eb, group, root);
7690 BUG_ON(ret);
7691 free_extent_buffer(eb);
7692 }
7693
7694 mutex_unlock(&root->fs_info->tree_reloc_mutex);
7695 path->lowest_level = 0;
7696 return 0;
7697 }
7698
7699 static noinline int relocate_tree_block(struct btrfs_trans_handle *trans,
7700 struct btrfs_root *root,
7701 struct btrfs_path *path,
7702 struct btrfs_key *first_key,
7703 struct btrfs_ref_path *ref_path)
7704 {
7705 int ret;
7706
7707 ret = relocate_one_path(trans, root, path, first_key,
7708 ref_path, NULL, NULL);
7709 BUG_ON(ret);
7710
7711 return 0;
7712 }
7713
7714 static noinline int del_extent_zero(struct btrfs_trans_handle *trans,
7715 struct btrfs_root *extent_root,
7716 struct btrfs_path *path,
7717 struct btrfs_key *extent_key)
7718 {
7719 int ret;
7720
7721 ret = btrfs_search_slot(trans, extent_root, extent_key, path, -1, 1);
7722 if (ret)
7723 goto out;
7724 ret = btrfs_del_item(trans, extent_root, path);
7725 out:
7726 btrfs_release_path(extent_root, path);
7727 return ret;
7728 }
7729
7730 static noinline struct btrfs_root *read_ref_root(struct btrfs_fs_info *fs_info,
7731 struct btrfs_ref_path *ref_path)
7732 {
7733 struct btrfs_key root_key;
7734
7735 root_key.objectid = ref_path->root_objectid;
7736 root_key.type = BTRFS_ROOT_ITEM_KEY;
7737 if (is_cowonly_root(ref_path->root_objectid))
7738 root_key.offset = 0;
7739 else
7740 root_key.offset = (u64)-1;
7741
7742 return btrfs_read_fs_root_no_name(fs_info, &root_key);
7743 }
7744
7745 static noinline int relocate_one_extent(struct btrfs_root *extent_root,
7746 struct btrfs_path *path,
7747 struct btrfs_key *extent_key,
7748 struct btrfs_block_group_cache *group,
7749 struct inode *reloc_inode, int pass)
7750 {
7751 struct btrfs_trans_handle *trans;
7752 struct btrfs_root *found_root;
7753 struct btrfs_ref_path *ref_path = NULL;
7754 struct disk_extent *new_extents = NULL;
7755 int nr_extents = 0;
7756 int loops;
7757 int ret;
7758 int level;
7759 struct btrfs_key first_key;
7760 u64 prev_block = 0;
7761
7762
7763 trans = btrfs_start_transaction(extent_root, 1);
7764 BUG_ON(!trans);
7765
7766 if (extent_key->objectid == 0) {
7767 ret = del_extent_zero(trans, extent_root, path, extent_key);
7768 goto out;
7769 }
7770
7771 ref_path = kmalloc(sizeof(*ref_path), GFP_NOFS);
7772 if (!ref_path) {
7773 ret = -ENOMEM;
7774 goto out;
7775 }
7776
7777 for (loops = 0; ; loops++) {
7778 if (loops == 0) {
7779 ret = btrfs_first_ref_path(trans, extent_root, ref_path,
7780 extent_key->objectid);
7781 } else {
7782 ret = btrfs_next_ref_path(trans, extent_root, ref_path);
7783 }
7784 if (ret < 0)
7785 goto out;
7786 if (ret > 0)
7787 break;
7788
7789 if (ref_path->root_objectid == BTRFS_TREE_LOG_OBJECTID ||
7790 ref_path->root_objectid == BTRFS_TREE_RELOC_OBJECTID)
7791 continue;
7792
7793 found_root = read_ref_root(extent_root->fs_info, ref_path);
7794 BUG_ON(!found_root);
7795 /*
7796 * for reference counted tree, only process reference paths
7797 * rooted at the latest committed root.
7798 */
7799 if (found_root->ref_cows &&
7800 ref_path->root_generation != found_root->root_key.offset)
7801 continue;
7802
7803 if (ref_path->owner_objectid >= BTRFS_FIRST_FREE_OBJECTID) {
7804 if (pass == 0) {
7805 /*
7806 * copy data extents to new locations
7807 */
7808 u64 group_start = group->key.objectid;
7809 ret = relocate_data_extent(reloc_inode,
7810 extent_key,
7811 group_start);
7812 if (ret < 0)
7813 goto out;
7814 break;
7815 }
7816 level = 0;
7817 } else {
7818 level = ref_path->owner_objectid;
7819 }
7820
7821 if (prev_block != ref_path->nodes[level]) {
7822 struct extent_buffer *eb;
7823 u64 block_start = ref_path->nodes[level];
7824 u64 block_size = btrfs_level_size(found_root, level);
7825
7826 eb = read_tree_block(found_root, block_start,
7827 block_size, 0);
7828 btrfs_tree_lock(eb);
7829 BUG_ON(level != btrfs_header_level(eb));
7830
7831 if (level == 0)
7832 btrfs_item_key_to_cpu(eb, &first_key, 0);
7833 else
7834 btrfs_node_key_to_cpu(eb, &first_key, 0);
7835
7836 btrfs_tree_unlock(eb);
7837 free_extent_buffer(eb);
7838 prev_block = block_start;
7839 }
7840
7841 mutex_lock(&extent_root->fs_info->trans_mutex);
7842 btrfs_record_root_in_trans(found_root);
7843 mutex_unlock(&extent_root->fs_info->trans_mutex);
7844 if (ref_path->owner_objectid >= BTRFS_FIRST_FREE_OBJECTID) {
7845 /*
7846 * try to update data extent references while
7847 * keeping metadata shared between snapshots.
7848 */
7849 if (pass == 1) {
7850 ret = relocate_one_path(trans, found_root,
7851 path, &first_key, ref_path,
7852 group, reloc_inode);
7853 if (ret < 0)
7854 goto out;
7855 continue;
7856 }
7857 /*
7858 * use fallback method to process the remaining
7859 * references.
7860 */
7861 if (!new_extents) {
7862 u64 group_start = group->key.objectid;
7863 new_extents = kmalloc(sizeof(*new_extents),
7864 GFP_NOFS);
7865 nr_extents = 1;
7866 ret = get_new_locations(reloc_inode,
7867 extent_key,
7868 group_start, 1,
7869 &new_extents,
7870 &nr_extents);
7871 if (ret)
7872 goto out;
7873 }
7874 ret = replace_one_extent(trans, found_root,
7875 path, extent_key,
7876 &first_key, ref_path,
7877 new_extents, nr_extents);
7878 } else {
7879 ret = relocate_tree_block(trans, found_root, path,
7880 &first_key, ref_path);
7881 }
7882 if (ret < 0)
7883 goto out;
7884 }
7885 ret = 0;
7886 out:
7887 btrfs_end_transaction(trans, extent_root);
7888 kfree(new_extents);
7889 kfree(ref_path);
7890 return ret;
7891 }
7892 #endif
7893
7894 static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
7895 {
7896 u64 num_devices;
7897 u64 stripped = BTRFS_BLOCK_GROUP_RAID0 |
7898 BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;
7899
7900 /*
7901 * we add in the count of missing devices because we want
7902 * to make sure that any RAID levels on a degraded FS
7903 * continue to be honored.
7904 */
7905 num_devices = root->fs_info->fs_devices->rw_devices +
7906 root->fs_info->fs_devices->missing_devices;
7907
7908 if (num_devices == 1) {
7909 stripped |= BTRFS_BLOCK_GROUP_DUP;
7910 stripped = flags & ~stripped;
7911
7912 /* turn raid0 into single device chunks */
7913 if (flags & BTRFS_BLOCK_GROUP_RAID0)
7914 return stripped;
7915
7916 /* turn mirroring into duplication */
7917 if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
7918 BTRFS_BLOCK_GROUP_RAID10))
7919 return stripped | BTRFS_BLOCK_GROUP_DUP;
7920 return flags;
7921 } else {
7922 /* they already had raid on here, just return */
7923 if (flags & stripped)
7924 return flags;
7925
7926 stripped |= BTRFS_BLOCK_GROUP_DUP;
7927 stripped = flags & ~stripped;
7928
7929 /* switch duplicated blocks with raid1 */
7930 if (flags & BTRFS_BLOCK_GROUP_DUP)
7931 return stripped | BTRFS_BLOCK_GROUP_RAID1;
7932
7933 /* turn single device chunks into raid0 */
7934 return stripped | BTRFS_BLOCK_GROUP_RAID0;
7935 }
7936 return flags;
7937 }
7938
7939 static int set_block_group_ro(struct btrfs_block_group_cache *cache)
7940 {
7941 struct btrfs_space_info *sinfo = cache->space_info;
7942 u64 num_bytes;
7943 int ret = -ENOSPC;
7944
7945 if (cache->ro)
7946 return 0;
7947
7948 spin_lock(&sinfo->lock);
7949 spin_lock(&cache->lock);
7950 num_bytes = cache->key.offset - cache->reserved - cache->pinned -
7951 cache->bytes_super - btrfs_block_group_used(&cache->item);
7952
7953 if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
7954 sinfo->bytes_may_use + sinfo->bytes_readonly +
7955 cache->reserved_pinned + num_bytes < sinfo->total_bytes) {
7956 sinfo->bytes_readonly += num_bytes;
7957 sinfo->bytes_reserved += cache->reserved_pinned;
7958 cache->reserved_pinned = 0;
7959 cache->ro = 1;
7960 ret = 0;
7961 }
7962 spin_unlock(&cache->lock);
7963 spin_unlock(&sinfo->lock);
7964 return ret;
7965 }
7966
7967 int btrfs_set_block_group_ro(struct btrfs_root *root,
7968 struct btrfs_block_group_cache *cache)
7969
7970 {
7971 struct btrfs_trans_handle *trans;
7972 u64 alloc_flags;
7973 int ret;
7974
7975 BUG_ON(cache->ro);
7976
7977 trans = btrfs_join_transaction(root, 1);
7978 BUG_ON(IS_ERR(trans));
7979
7980 alloc_flags = update_block_group_flags(root, cache->flags);
7981 if (alloc_flags != cache->flags)
7982 do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags, 1);
7983
7984 ret = set_block_group_ro(cache);
7985 if (!ret)
7986 goto out;
7987 alloc_flags = get_alloc_profile(root, cache->space_info->flags);
7988 ret = do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags, 1);
7989 if (ret < 0)
7990 goto out;
7991 ret = set_block_group_ro(cache);
7992 out:
7993 btrfs_end_transaction(trans, root);
7994 return ret;
7995 }
7996
7997 int btrfs_set_block_group_rw(struct btrfs_root *root,
7998 struct btrfs_block_group_cache *cache)
7999 {
8000 struct btrfs_space_info *sinfo = cache->space_info;
8001 u64 num_bytes;
8002
8003 BUG_ON(!cache->ro);
8004
8005 spin_lock(&sinfo->lock);
8006 spin_lock(&cache->lock);
8007 num_bytes = cache->key.offset - cache->reserved - cache->pinned -
8008 cache->bytes_super - btrfs_block_group_used(&cache->item);
8009 sinfo->bytes_readonly -= num_bytes;
8010 cache->ro = 0;
8011 spin_unlock(&cache->lock);
8012 spin_unlock(&sinfo->lock);
8013 return 0;
8014 }
8015
8016 /*
8017 * checks to see if its even possible to relocate this block group.
8018 *
8019 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
8020 * ok to go ahead and try.
8021 */
8022 int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
8023 {
8024 struct btrfs_block_group_cache *block_group;
8025 struct btrfs_space_info *space_info;
8026 struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
8027 struct btrfs_device *device;
8028 int full = 0;
8029 int ret = 0;
8030
8031 block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
8032
8033 /* odd, couldn't find the block group, leave it alone */
8034 if (!block_group)
8035 return -1;
8036
8037 /* no bytes used, we're good */
8038 if (!btrfs_block_group_used(&block_group->item))
8039 goto out;
8040
8041 space_info = block_group->space_info;
8042 spin_lock(&space_info->lock);
8043
8044 full = space_info->full;
8045
8046 /*
8047 * if this is the last block group we have in this space, we can't
8048 * relocate it unless we're able to allocate a new chunk below.
8049 *
8050 * Otherwise, we need to make sure we have room in the space to handle
8051 * all of the extents from this block group. If we can, we're good
8052 */
8053 if ((space_info->total_bytes != block_group->key.offset) &&
8054 (space_info->bytes_used + space_info->bytes_reserved +
8055 space_info->bytes_pinned + space_info->bytes_readonly +
8056 btrfs_block_group_used(&block_group->item) <
8057 space_info->total_bytes)) {
8058 spin_unlock(&space_info->lock);
8059 goto out;
8060 }
8061 spin_unlock(&space_info->lock);
8062
8063 /*
8064 * ok we don't have enough space, but maybe we have free space on our
8065 * devices to allocate new chunks for relocation, so loop through our
8066 * alloc devices and guess if we have enough space. However, if we
8067 * were marked as full, then we know there aren't enough chunks, and we
8068 * can just return.
8069 */
8070 ret = -1;
8071 if (full)
8072 goto out;
8073
8074 mutex_lock(&root->fs_info->chunk_mutex);
8075 list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
8076 u64 min_free = btrfs_block_group_used(&block_group->item);
8077 u64 dev_offset, max_avail;
8078
8079 /*
8080 * check to make sure we can actually find a chunk with enough
8081 * space to fit our block group in.
8082 */
8083 if (device->total_bytes > device->bytes_used + min_free) {
8084 ret = find_free_dev_extent(NULL, device, min_free,
8085 &dev_offset, &max_avail);
8086 if (!ret)
8087 break;
8088 ret = -1;
8089 }
8090 }
8091 mutex_unlock(&root->fs_info->chunk_mutex);
8092 out:
8093 btrfs_put_block_group(block_group);
8094 return ret;
8095 }
8096
8097 static int find_first_block_group(struct btrfs_root *root,
8098 struct btrfs_path *path, struct btrfs_key *key)
8099 {
8100 int ret = 0;
8101 struct btrfs_key found_key;
8102 struct extent_buffer *leaf;
8103 int slot;
8104
8105 ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
8106 if (ret < 0)
8107 goto out;
8108
8109 while (1) {
8110 slot = path->slots[0];
8111 leaf = path->nodes[0];
8112 if (slot >= btrfs_header_nritems(leaf)) {
8113 ret = btrfs_next_leaf(root, path);
8114 if (ret == 0)
8115 continue;
8116 if (ret < 0)
8117 goto out;
8118 break;
8119 }
8120 btrfs_item_key_to_cpu(leaf, &found_key, slot);
8121
8122 if (found_key.objectid >= key->objectid &&
8123 found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
8124 ret = 0;
8125 goto out;
8126 }
8127 path->slots[0]++;
8128 }
8129 out:
8130 return ret;
8131 }
8132
8133 void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
8134 {
8135 struct btrfs_block_group_cache *block_group;
8136 u64 last = 0;
8137
8138 while (1) {
8139 struct inode *inode;
8140
8141 block_group = btrfs_lookup_first_block_group(info, last);
8142 while (block_group) {
8143 spin_lock(&block_group->lock);
8144 if (block_group->iref)
8145 break;
8146 spin_unlock(&block_group->lock);
8147 block_group = next_block_group(info->tree_root,
8148 block_group);
8149 }
8150 if (!block_group) {
8151 if (last == 0)
8152 break;
8153 last = 0;
8154 continue;
8155 }
8156
8157 inode = block_group->inode;
8158 block_group->iref = 0;
8159 block_group->inode = NULL;
8160 spin_unlock(&block_group->lock);
8161 iput(inode);
8162 last = block_group->key.objectid + block_group->key.offset;
8163 btrfs_put_block_group(block_group);
8164 }
8165 }
8166
8167 int btrfs_free_block_groups(struct btrfs_fs_info *info)
8168 {
8169 struct btrfs_block_group_cache *block_group;
8170 struct btrfs_space_info *space_info;
8171 struct btrfs_caching_control *caching_ctl;
8172 struct rb_node *n;
8173
8174 down_write(&info->extent_commit_sem);
8175 while (!list_empty(&info->caching_block_groups)) {
8176 caching_ctl = list_entry(info->caching_block_groups.next,
8177 struct btrfs_caching_control, list);
8178 list_del(&caching_ctl->list);
8179 put_caching_control(caching_ctl);
8180 }
8181 up_write(&info->extent_commit_sem);
8182
8183 spin_lock(&info->block_group_cache_lock);
8184 while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
8185 block_group = rb_entry(n, struct btrfs_block_group_cache,
8186 cache_node);
8187 rb_erase(&block_group->cache_node,
8188 &info->block_group_cache_tree);
8189 spin_unlock(&info->block_group_cache_lock);
8190
8191 down_write(&block_group->space_info->groups_sem);
8192 list_del(&block_group->list);
8193 up_write(&block_group->space_info->groups_sem);
8194
8195 if (block_group->cached == BTRFS_CACHE_STARTED)
8196 wait_block_group_cache_done(block_group);
8197
8198 btrfs_remove_free_space_cache(block_group);
8199 btrfs_put_block_group(block_group);
8200
8201 spin_lock(&info->block_group_cache_lock);
8202 }
8203 spin_unlock(&info->block_group_cache_lock);
8204
8205 /* now that all the block groups are freed, go through and
8206 * free all the space_info structs. This is only called during
8207 * the final stages of unmount, and so we know nobody is
8208 * using them. We call synchronize_rcu() once before we start,
8209 * just to be on the safe side.
8210 */
8211 synchronize_rcu();
8212
8213 release_global_block_rsv(info);
8214
8215 while(!list_empty(&info->space_info)) {
8216 space_info = list_entry(info->space_info.next,
8217 struct btrfs_space_info,
8218 list);
8219 if (space_info->bytes_pinned > 0 ||
8220 space_info->bytes_reserved > 0) {
8221 WARN_ON(1);
8222 dump_space_info(space_info, 0, 0);
8223 }
8224 list_del(&space_info->list);
8225 kfree(space_info);
8226 }
8227 return 0;
8228 }
8229
8230 static void __link_block_group(struct btrfs_space_info *space_info,
8231 struct btrfs_block_group_cache *cache)
8232 {
8233 int index = get_block_group_index(cache);
8234
8235 down_write(&space_info->groups_sem);
8236 list_add_tail(&cache->list, &space_info->block_groups[index]);
8237 up_write(&space_info->groups_sem);
8238 }
8239
8240 int btrfs_read_block_groups(struct btrfs_root *root)
8241 {
8242 struct btrfs_path *path;
8243 int ret;
8244 struct btrfs_block_group_cache *cache;
8245 struct btrfs_fs_info *info = root->fs_info;
8246 struct btrfs_space_info *space_info;
8247 struct btrfs_key key;
8248 struct btrfs_key found_key;
8249 struct extent_buffer *leaf;
8250 int need_clear = 0;
8251 u64 cache_gen;
8252
8253 root = info->extent_root;
8254 key.objectid = 0;
8255 key.offset = 0;
8256 btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
8257 path = btrfs_alloc_path();
8258 if (!path)
8259 return -ENOMEM;
8260
8261 cache_gen = btrfs_super_cache_generation(&root->fs_info->super_copy);
8262 if (cache_gen != 0 &&
8263 btrfs_super_generation(&root->fs_info->super_copy) != cache_gen)
8264 need_clear = 1;
8265 if (btrfs_test_opt(root, CLEAR_CACHE))
8266 need_clear = 1;
8267 if (!btrfs_test_opt(root, SPACE_CACHE) && cache_gen)
8268 printk(KERN_INFO "btrfs: disk space caching is enabled\n");
8269
8270 while (1) {
8271 ret = find_first_block_group(root, path, &key);
8272 if (ret > 0)
8273 break;
8274 if (ret != 0)
8275 goto error;
8276
8277 leaf = path->nodes[0];
8278 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
8279 cache = kzalloc(sizeof(*cache), GFP_NOFS);
8280 if (!cache) {
8281 ret = -ENOMEM;
8282 goto error;
8283 }
8284
8285 atomic_set(&cache->count, 1);
8286 spin_lock_init(&cache->lock);
8287 spin_lock_init(&cache->tree_lock);
8288 cache->fs_info = info;
8289 INIT_LIST_HEAD(&cache->list);
8290 INIT_LIST_HEAD(&cache->cluster_list);
8291
8292 if (need_clear)
8293 cache->disk_cache_state = BTRFS_DC_CLEAR;
8294
8295 /*
8296 * we only want to have 32k of ram per block group for keeping
8297 * track of free space, and if we pass 1/2 of that we want to
8298 * start converting things over to using bitmaps
8299 */
8300 cache->extents_thresh = ((1024 * 32) / 2) /
8301 sizeof(struct btrfs_free_space);
8302
8303 read_extent_buffer(leaf, &cache->item,
8304 btrfs_item_ptr_offset(leaf, path->slots[0]),
8305 sizeof(cache->item));
8306 memcpy(&cache->key, &found_key, sizeof(found_key));
8307
8308 key.objectid = found_key.objectid + found_key.offset;
8309 btrfs_release_path(root, path);
8310 cache->flags = btrfs_block_group_flags(&cache->item);
8311 cache->sectorsize = root->sectorsize;
8312
8313 /*
8314 * check for two cases, either we are full, and therefore
8315 * don't need to bother with the caching work since we won't
8316 * find any space, or we are empty, and we can just add all
8317 * the space in and be done with it. This saves us _alot_ of
8318 * time, particularly in the full case.
8319 */
8320 if (found_key.offset == btrfs_block_group_used(&cache->item)) {
8321 exclude_super_stripes(root, cache);
8322 cache->last_byte_to_unpin = (u64)-1;
8323 cache->cached = BTRFS_CACHE_FINISHED;
8324 free_excluded_extents(root, cache);
8325 } else if (btrfs_block_group_used(&cache->item) == 0) {
8326 exclude_super_stripes(root, cache);
8327 cache->last_byte_to_unpin = (u64)-1;
8328 cache->cached = BTRFS_CACHE_FINISHED;
8329 add_new_free_space(cache, root->fs_info,
8330 found_key.objectid,
8331 found_key.objectid +
8332 found_key.offset);
8333 free_excluded_extents(root, cache);
8334 }
8335
8336 ret = update_space_info(info, cache->flags, found_key.offset,
8337 btrfs_block_group_used(&cache->item),
8338 &space_info);
8339 BUG_ON(ret);
8340 cache->space_info = space_info;
8341 spin_lock(&cache->space_info->lock);
8342 cache->space_info->bytes_readonly += cache->bytes_super;
8343 spin_unlock(&cache->space_info->lock);
8344
8345 __link_block_group(space_info, cache);
8346
8347 ret = btrfs_add_block_group_cache(root->fs_info, cache);
8348 BUG_ON(ret);
8349
8350 set_avail_alloc_bits(root->fs_info, cache->flags);
8351 if (btrfs_chunk_readonly(root, cache->key.objectid))
8352 set_block_group_ro(cache);
8353 }
8354
8355 list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
8356 if (!(get_alloc_profile(root, space_info->flags) &
8357 (BTRFS_BLOCK_GROUP_RAID10 |
8358 BTRFS_BLOCK_GROUP_RAID1 |
8359 BTRFS_BLOCK_GROUP_DUP)))
8360 continue;
8361 /*
8362 * avoid allocating from un-mirrored block group if there are
8363 * mirrored block groups.
8364 */
8365 list_for_each_entry(cache, &space_info->block_groups[3], list)
8366 set_block_group_ro(cache);
8367 list_for_each_entry(cache, &space_info->block_groups[4], list)
8368 set_block_group_ro(cache);
8369 }
8370
8371 init_global_block_rsv(info);
8372 ret = 0;
8373 error:
8374 btrfs_free_path(path);
8375 return ret;
8376 }
8377
8378 int btrfs_make_block_group(struct btrfs_trans_handle *trans,
8379 struct btrfs_root *root, u64 bytes_used,
8380 u64 type, u64 chunk_objectid, u64 chunk_offset,
8381 u64 size)
8382 {
8383 int ret;
8384 struct btrfs_root *extent_root;
8385 struct btrfs_block_group_cache *cache;
8386
8387 extent_root = root->fs_info->extent_root;
8388
8389 root->fs_info->last_trans_log_full_commit = trans->transid;
8390
8391 cache = kzalloc(sizeof(*cache), GFP_NOFS);
8392 if (!cache)
8393 return -ENOMEM;
8394
8395 cache->key.objectid = chunk_offset;
8396 cache->key.offset = size;
8397 cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
8398 cache->sectorsize = root->sectorsize;
8399 cache->fs_info = root->fs_info;
8400
8401 /*
8402 * we only want to have 32k of ram per block group for keeping track
8403 * of free space, and if we pass 1/2 of that we want to start
8404 * converting things over to using bitmaps
8405 */
8406 cache->extents_thresh = ((1024 * 32) / 2) /
8407 sizeof(struct btrfs_free_space);
8408 atomic_set(&cache->count, 1);
8409 spin_lock_init(&cache->lock);
8410 spin_lock_init(&cache->tree_lock);
8411 INIT_LIST_HEAD(&cache->list);
8412 INIT_LIST_HEAD(&cache->cluster_list);
8413
8414 btrfs_set_block_group_used(&cache->item, bytes_used);
8415 btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
8416 cache->flags = type;
8417 btrfs_set_block_group_flags(&cache->item, type);
8418
8419 cache->last_byte_to_unpin = (u64)-1;
8420 cache->cached = BTRFS_CACHE_FINISHED;
8421 exclude_super_stripes(root, cache);
8422
8423 add_new_free_space(cache, root->fs_info, chunk_offset,
8424 chunk_offset + size);
8425
8426 free_excluded_extents(root, cache);
8427
8428 ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
8429 &cache->space_info);
8430 BUG_ON(ret);
8431
8432 spin_lock(&cache->space_info->lock);
8433 cache->space_info->bytes_readonly += cache->bytes_super;
8434 spin_unlock(&cache->space_info->lock);
8435
8436 __link_block_group(cache->space_info, cache);
8437
8438 ret = btrfs_add_block_group_cache(root->fs_info, cache);
8439 BUG_ON(ret);
8440
8441 ret = btrfs_insert_item(trans, extent_root, &cache->key, &cache->item,
8442 sizeof(cache->item));
8443 BUG_ON(ret);
8444
8445 set_avail_alloc_bits(extent_root->fs_info, type);
8446
8447 return 0;
8448 }
8449
8450 int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
8451 struct btrfs_root *root, u64 group_start)
8452 {
8453 struct btrfs_path *path;
8454 struct btrfs_block_group_cache *block_group;
8455 struct btrfs_free_cluster *cluster;
8456 struct btrfs_root *tree_root = root->fs_info->tree_root;
8457 struct btrfs_key key;
8458 struct inode *inode;
8459 int ret;
8460 int factor;
8461
8462 root = root->fs_info->extent_root;
8463
8464 block_group = btrfs_lookup_block_group(root->fs_info, group_start);
8465 BUG_ON(!block_group);
8466 BUG_ON(!block_group->ro);
8467
8468 memcpy(&key, &block_group->key, sizeof(key));
8469 if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
8470 BTRFS_BLOCK_GROUP_RAID1 |
8471 BTRFS_BLOCK_GROUP_RAID10))
8472 factor = 2;
8473 else
8474 factor = 1;
8475
8476 /* make sure this block group isn't part of an allocation cluster */
8477 cluster = &root->fs_info->data_alloc_cluster;
8478 spin_lock(&cluster->refill_lock);
8479 btrfs_return_cluster_to_free_space(block_group, cluster);
8480 spin_unlock(&cluster->refill_lock);
8481
8482 /*
8483 * make sure this block group isn't part of a metadata
8484 * allocation cluster
8485 */
8486 cluster = &root->fs_info->meta_alloc_cluster;
8487 spin_lock(&cluster->refill_lock);
8488 btrfs_return_cluster_to_free_space(block_group, cluster);
8489 spin_unlock(&cluster->refill_lock);
8490
8491 path = btrfs_alloc_path();
8492 BUG_ON(!path);
8493
8494 inode = lookup_free_space_inode(root, block_group, path);
8495 if (!IS_ERR(inode)) {
8496 btrfs_orphan_add(trans, inode);
8497 clear_nlink(inode);
8498 /* One for the block groups ref */
8499 spin_lock(&block_group->lock);
8500 if (block_group->iref) {
8501 block_group->iref = 0;
8502 block_group->inode = NULL;
8503 spin_unlock(&block_group->lock);
8504 iput(inode);
8505 } else {
8506 spin_unlock(&block_group->lock);
8507 }
8508 /* One for our lookup ref */
8509 iput(inode);
8510 }
8511
8512 key.objectid = BTRFS_FREE_SPACE_OBJECTID;
8513 key.offset = block_group->key.objectid;
8514 key.type = 0;
8515
8516 ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
8517 if (ret < 0)
8518 goto out;
8519 if (ret > 0)
8520 btrfs_release_path(tree_root, path);
8521 if (ret == 0) {
8522 ret = btrfs_del_item(trans, tree_root, path);
8523 if (ret)
8524 goto out;
8525 btrfs_release_path(tree_root, path);
8526 }
8527
8528 spin_lock(&root->fs_info->block_group_cache_lock);
8529 rb_erase(&block_group->cache_node,
8530 &root->fs_info->block_group_cache_tree);
8531 spin_unlock(&root->fs_info->block_group_cache_lock);
8532
8533 down_write(&block_group->space_info->groups_sem);
8534 /*
8535 * we must use list_del_init so people can check to see if they
8536 * are still on the list after taking the semaphore
8537 */
8538 list_del_init(&block_group->list);
8539 up_write(&block_group->space_info->groups_sem);
8540
8541 if (block_group->cached == BTRFS_CACHE_STARTED)
8542 wait_block_group_cache_done(block_group);
8543
8544 btrfs_remove_free_space_cache(block_group);
8545
8546 spin_lock(&block_group->space_info->lock);
8547 block_group->space_info->total_bytes -= block_group->key.offset;
8548 block_group->space_info->bytes_readonly -= block_group->key.offset;
8549 block_group->space_info->disk_total -= block_group->key.offset * factor;
8550 spin_unlock(&block_group->space_info->lock);
8551
8552 memcpy(&key, &block_group->key, sizeof(key));
8553
8554 btrfs_clear_space_info_full(root->fs_info);
8555
8556 btrfs_put_block_group(block_group);
8557 btrfs_put_block_group(block_group);
8558
8559 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
8560 if (ret > 0)
8561 ret = -EIO;
8562 if (ret < 0)
8563 goto out;
8564
8565 ret = btrfs_del_item(trans, root, path);
8566 out:
8567 btrfs_free_path(path);
8568 return ret;
8569 }
x86 "subsystem" repository