2 * Copyright (C) 2011 STRATO. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
19 #include <linux/sched.h>
20 #include <linux/pagemap.h>
21 #include <linux/writeback.h>
22 #include <linux/blkdev.h>
23 #include <linux/rbtree.h>
24 #include <linux/slab.h>
25 #include <linux/workqueue.h>
29 #include "transaction.h"
30 #include "dev-replace.h"
35 * This is the implementation for the generic read ahead framework.
37 * To trigger a readahead, btrfs_reada_add must be called. It will start
38 * a read ahead for the given range [start, end) on tree root. The returned
39 * handle can either be used to wait on the readahead to finish
40 * (btrfs_reada_wait), or to send it to the background (btrfs_reada_detach).
42 * The read ahead works as follows:
43 * On btrfs_reada_add, the root of the tree is inserted into a radix_tree.
44 * reada_start_machine will then search for extents to prefetch and trigger
45 * some reads. When a read finishes for a node, all contained node/leaf
46 * pointers that lie in the given range will also be enqueued. The reads will
47 * be triggered in sequential order, thus giving a big win over a naive
48 * enumeration. It will also make use of multi-device layouts. Each disk
49 * will have its on read pointer and all disks will by utilized in parallel.
50 * Also will no two disks read both sides of a mirror simultaneously, as this
51 * would waste seeking capacity. Instead both disks will read different parts
53 * Any number of readaheads can be started in parallel. The read order will be
54 * determined globally, i.e. 2 parallel readaheads will normally finish faster
55 * than the 2 started one after another.
58 #define MAX_IN_FLIGHT 6
61 struct list_head list
;
62 struct reada_control
*rc
;
70 struct list_head extctl
;
73 struct reada_zone
*zones
[BTRFS_MAX_MIRRORS
];
82 struct list_head list
;
85 struct btrfs_device
*device
;
86 struct btrfs_device
*devs
[BTRFS_MAX_MIRRORS
]; /* full list, incl
92 struct reada_machine_work
{
93 struct btrfs_work work
;
94 struct btrfs_fs_info
*fs_info
;
97 static void reada_extent_put(struct btrfs_fs_info
*, struct reada_extent
*);
98 static void reada_control_release(struct kref
*kref
);
99 static void reada_zone_release(struct kref
*kref
);
100 static void reada_start_machine(struct btrfs_fs_info
*fs_info
);
101 static void __reada_start_machine(struct btrfs_fs_info
*fs_info
);
103 static int reada_add_block(struct reada_control
*rc
, u64 logical
,
104 struct btrfs_key
*top
, u64 generation
);
107 /* in case of err, eb might be NULL */
108 static void __readahead_hook(struct btrfs_fs_info
*fs_info
,
109 struct reada_extent
*re
, struct extent_buffer
*eb
,
116 struct list_head list
;
118 spin_lock(&re
->lock
);
120 * just take the full list from the extent. afterwards we
121 * don't need the lock anymore
123 list_replace_init(&re
->extctl
, &list
);
125 spin_unlock(&re
->lock
);
128 * this is the error case, the extent buffer has not been
129 * read correctly. We won't access anything from it and
130 * just cleanup our data structures. Effectively this will
131 * cut the branch below this node from read ahead.
137 * FIXME: currently we just set nritems to 0 if this is a leaf,
138 * effectively ignoring the content. In a next step we could
139 * trigger more readahead depending from the content, e.g.
140 * fetch the checksums for the extents in the leaf.
142 if (!btrfs_header_level(eb
))
145 nritems
= btrfs_header_nritems(eb
);
146 generation
= btrfs_header_generation(eb
);
147 for (i
= 0; i
< nritems
; i
++) {
148 struct reada_extctl
*rec
;
150 struct btrfs_key key
;
151 struct btrfs_key next_key
;
153 btrfs_node_key_to_cpu(eb
, &key
, i
);
155 btrfs_node_key_to_cpu(eb
, &next_key
, i
+ 1);
158 bytenr
= btrfs_node_blockptr(eb
, i
);
159 n_gen
= btrfs_node_ptr_generation(eb
, i
);
161 list_for_each_entry(rec
, &list
, list
) {
162 struct reada_control
*rc
= rec
->rc
;
165 * if the generation doesn't match, just ignore this
166 * extctl. This will probably cut off a branch from
167 * prefetch. Alternatively one could start a new (sub-)
168 * prefetch for this branch, starting again from root.
169 * FIXME: move the generation check out of this loop
172 if (rec
->generation
!= generation
) {
174 "generation mismatch for (%llu,%d,%llu) %llu != %llu",
175 key
.objectid
, key
.type
, key
.offset
,
176 rec
->generation
, generation
);
179 if (rec
->generation
== generation
&&
180 btrfs_comp_cpu_keys(&key
, &rc
->key_end
) < 0 &&
181 btrfs_comp_cpu_keys(&next_key
, &rc
->key_start
) > 0)
182 reada_add_block(rc
, bytenr
, &next_key
, n_gen
);
188 * free extctl records
190 while (!list_empty(&list
)) {
191 struct reada_control
*rc
;
192 struct reada_extctl
*rec
;
194 rec
= list_first_entry(&list
, struct reada_extctl
, list
);
195 list_del(&rec
->list
);
199 kref_get(&rc
->refcnt
);
200 if (atomic_dec_and_test(&rc
->elems
)) {
201 kref_put(&rc
->refcnt
, reada_control_release
);
204 kref_put(&rc
->refcnt
, reada_control_release
);
206 reada_extent_put(fs_info
, re
); /* one ref for each entry */
213 * start is passed separately in case eb in NULL, which may be the case with
216 int btree_readahead_hook(struct btrfs_fs_info
*fs_info
,
217 struct extent_buffer
*eb
, u64 start
, int err
)
220 struct reada_extent
*re
;
223 spin_lock(&fs_info
->reada_lock
);
224 re
= radix_tree_lookup(&fs_info
->reada_tree
,
225 start
>> PAGE_SHIFT
);
228 spin_unlock(&fs_info
->reada_lock
);
234 __readahead_hook(fs_info
, re
, eb
, start
, err
);
235 reada_extent_put(fs_info
, re
); /* our ref */
238 reada_start_machine(fs_info
);
242 static struct reada_zone
*reada_find_zone(struct btrfs_fs_info
*fs_info
,
243 struct btrfs_device
*dev
, u64 logical
,
244 struct btrfs_bio
*bbio
)
247 struct reada_zone
*zone
;
248 struct btrfs_block_group_cache
*cache
= NULL
;
254 spin_lock(&fs_info
->reada_lock
);
255 ret
= radix_tree_gang_lookup(&dev
->reada_zones
, (void **)&zone
,
256 logical
>> PAGE_SHIFT
, 1);
257 if (ret
== 1 && logical
>= zone
->start
&& logical
<= zone
->end
) {
258 kref_get(&zone
->refcnt
);
259 spin_unlock(&fs_info
->reada_lock
);
263 spin_unlock(&fs_info
->reada_lock
);
265 cache
= btrfs_lookup_block_group(fs_info
, logical
);
269 start
= cache
->key
.objectid
;
270 end
= start
+ cache
->key
.offset
- 1;
271 btrfs_put_block_group(cache
);
273 zone
= kzalloc(sizeof(*zone
), GFP_KERNEL
);
279 INIT_LIST_HEAD(&zone
->list
);
280 spin_lock_init(&zone
->lock
);
282 kref_init(&zone
->refcnt
);
284 zone
->device
= dev
; /* our device always sits at index 0 */
285 for (i
= 0; i
< bbio
->num_stripes
; ++i
) {
286 /* bounds have already been checked */
287 zone
->devs
[i
] = bbio
->stripes
[i
].dev
;
289 zone
->ndevs
= bbio
->num_stripes
;
291 spin_lock(&fs_info
->reada_lock
);
292 ret
= radix_tree_insert(&dev
->reada_zones
,
293 (unsigned long)(zone
->end
>> PAGE_SHIFT
),
296 if (ret
== -EEXIST
) {
298 ret
= radix_tree_gang_lookup(&dev
->reada_zones
, (void **)&zone
,
299 logical
>> PAGE_SHIFT
, 1);
300 if (ret
== 1 && logical
>= zone
->start
&& logical
<= zone
->end
)
301 kref_get(&zone
->refcnt
);
305 spin_unlock(&fs_info
->reada_lock
);
310 static struct reada_extent
*reada_find_extent(struct btrfs_root
*root
,
312 struct btrfs_key
*top
)
315 struct reada_extent
*re
= NULL
;
316 struct reada_extent
*re_exist
= NULL
;
317 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
318 struct btrfs_bio
*bbio
= NULL
;
319 struct btrfs_device
*dev
;
320 struct btrfs_device
*prev_dev
;
325 unsigned long index
= logical
>> PAGE_SHIFT
;
326 int dev_replace_is_ongoing
;
329 spin_lock(&fs_info
->reada_lock
);
330 re
= radix_tree_lookup(&fs_info
->reada_tree
, index
);
333 spin_unlock(&fs_info
->reada_lock
);
338 re
= kzalloc(sizeof(*re
), GFP_KERNEL
);
342 blocksize
= root
->nodesize
;
343 re
->logical
= logical
;
345 INIT_LIST_HEAD(&re
->extctl
);
346 spin_lock_init(&re
->lock
);
353 ret
= btrfs_map_block(fs_info
, BTRFS_MAP_GET_READ_MIRRORS
, logical
,
355 if (ret
|| !bbio
|| length
< blocksize
)
358 if (bbio
->num_stripes
> BTRFS_MAX_MIRRORS
) {
359 btrfs_err(root
->fs_info
,
360 "readahead: more than %d copies not supported",
365 real_stripes
= bbio
->num_stripes
- bbio
->num_tgtdevs
;
366 for (nzones
= 0; nzones
< real_stripes
; ++nzones
) {
367 struct reada_zone
*zone
;
369 dev
= bbio
->stripes
[nzones
].dev
;
371 /* cannot read ahead on missing device. */
375 zone
= reada_find_zone(fs_info
, dev
, logical
, bbio
);
379 re
->zones
[re
->nzones
++] = zone
;
380 spin_lock(&zone
->lock
);
382 kref_get(&zone
->refcnt
);
384 spin_unlock(&zone
->lock
);
385 spin_lock(&fs_info
->reada_lock
);
386 kref_put(&zone
->refcnt
, reada_zone_release
);
387 spin_unlock(&fs_info
->reada_lock
);
389 if (re
->nzones
== 0) {
390 /* not a single zone found, error and out */
394 /* insert extent in reada_tree + all per-device trees, all or nothing */
395 btrfs_dev_replace_lock(&fs_info
->dev_replace
, 0);
396 spin_lock(&fs_info
->reada_lock
);
397 ret
= radix_tree_insert(&fs_info
->reada_tree
, index
, re
);
398 if (ret
== -EEXIST
) {
399 re_exist
= radix_tree_lookup(&fs_info
->reada_tree
, index
);
402 spin_unlock(&fs_info
->reada_lock
);
403 btrfs_dev_replace_unlock(&fs_info
->dev_replace
, 0);
407 spin_unlock(&fs_info
->reada_lock
);
408 btrfs_dev_replace_unlock(&fs_info
->dev_replace
, 0);
412 dev_replace_is_ongoing
= btrfs_dev_replace_is_ongoing(
413 &fs_info
->dev_replace
);
414 for (nzones
= 0; nzones
< re
->nzones
; ++nzones
) {
415 dev
= re
->zones
[nzones
]->device
;
417 if (dev
== prev_dev
) {
419 * in case of DUP, just add the first zone. As both
420 * are on the same device, there's nothing to gain
422 * Also, it wouldn't work, as the tree is per device
423 * and adding would fail with EEXIST
430 if (dev_replace_is_ongoing
&&
431 dev
== fs_info
->dev_replace
.tgtdev
) {
433 * as this device is selected for reading only as
434 * a last resort, skip it for read ahead.
439 ret
= radix_tree_insert(&dev
->reada_extents
, index
, re
);
441 while (--nzones
>= 0) {
442 dev
= re
->zones
[nzones
]->device
;
444 /* ignore whether the entry was inserted */
445 radix_tree_delete(&dev
->reada_extents
, index
);
447 BUG_ON(fs_info
== NULL
);
448 radix_tree_delete(&fs_info
->reada_tree
, index
);
449 spin_unlock(&fs_info
->reada_lock
);
450 btrfs_dev_replace_unlock(&fs_info
->dev_replace
, 0);
455 spin_unlock(&fs_info
->reada_lock
);
456 btrfs_dev_replace_unlock(&fs_info
->dev_replace
, 0);
461 btrfs_put_bbio(bbio
);
465 for (nzones
= 0; nzones
< re
->nzones
; ++nzones
) {
466 struct reada_zone
*zone
;
468 zone
= re
->zones
[nzones
];
469 kref_get(&zone
->refcnt
);
470 spin_lock(&zone
->lock
);
472 if (zone
->elems
== 0) {
474 * no fs_info->reada_lock needed, as this can't be
477 kref_put(&zone
->refcnt
, reada_zone_release
);
479 spin_unlock(&zone
->lock
);
481 spin_lock(&fs_info
->reada_lock
);
482 kref_put(&zone
->refcnt
, reada_zone_release
);
483 spin_unlock(&fs_info
->reada_lock
);
485 btrfs_put_bbio(bbio
);
490 static void reada_extent_put(struct btrfs_fs_info
*fs_info
,
491 struct reada_extent
*re
)
494 unsigned long index
= re
->logical
>> PAGE_SHIFT
;
496 spin_lock(&fs_info
->reada_lock
);
498 spin_unlock(&fs_info
->reada_lock
);
502 radix_tree_delete(&fs_info
->reada_tree
, index
);
503 for (i
= 0; i
< re
->nzones
; ++i
) {
504 struct reada_zone
*zone
= re
->zones
[i
];
506 radix_tree_delete(&zone
->device
->reada_extents
, index
);
509 spin_unlock(&fs_info
->reada_lock
);
511 for (i
= 0; i
< re
->nzones
; ++i
) {
512 struct reada_zone
*zone
= re
->zones
[i
];
514 kref_get(&zone
->refcnt
);
515 spin_lock(&zone
->lock
);
517 if (zone
->elems
== 0) {
518 /* no fs_info->reada_lock needed, as this can't be
520 kref_put(&zone
->refcnt
, reada_zone_release
);
522 spin_unlock(&zone
->lock
);
524 spin_lock(&fs_info
->reada_lock
);
525 kref_put(&zone
->refcnt
, reada_zone_release
);
526 spin_unlock(&fs_info
->reada_lock
);
532 static void reada_zone_release(struct kref
*kref
)
534 struct reada_zone
*zone
= container_of(kref
, struct reada_zone
, refcnt
);
536 radix_tree_delete(&zone
->device
->reada_zones
,
537 zone
->end
>> PAGE_SHIFT
);
542 static void reada_control_release(struct kref
*kref
)
544 struct reada_control
*rc
= container_of(kref
, struct reada_control
,
550 static int reada_add_block(struct reada_control
*rc
, u64 logical
,
551 struct btrfs_key
*top
, u64 generation
)
553 struct btrfs_root
*root
= rc
->root
;
554 struct reada_extent
*re
;
555 struct reada_extctl
*rec
;
557 re
= reada_find_extent(root
, logical
, top
); /* takes one ref */
561 rec
= kzalloc(sizeof(*rec
), GFP_KERNEL
);
563 reada_extent_put(root
->fs_info
, re
);
568 rec
->generation
= generation
;
569 atomic_inc(&rc
->elems
);
571 spin_lock(&re
->lock
);
572 list_add_tail(&rec
->list
, &re
->extctl
);
573 spin_unlock(&re
->lock
);
575 /* leave the ref on the extent */
581 * called with fs_info->reada_lock held
583 static void reada_peer_zones_set_lock(struct reada_zone
*zone
, int lock
)
586 unsigned long index
= zone
->end
>> PAGE_SHIFT
;
588 for (i
= 0; i
< zone
->ndevs
; ++i
) {
589 struct reada_zone
*peer
;
590 peer
= radix_tree_lookup(&zone
->devs
[i
]->reada_zones
, index
);
591 if (peer
&& peer
->device
!= zone
->device
)
597 * called with fs_info->reada_lock held
599 static int reada_pick_zone(struct btrfs_device
*dev
)
601 struct reada_zone
*top_zone
= NULL
;
602 struct reada_zone
*top_locked_zone
= NULL
;
604 u64 top_locked_elems
= 0;
605 unsigned long index
= 0;
608 if (dev
->reada_curr_zone
) {
609 reada_peer_zones_set_lock(dev
->reada_curr_zone
, 0);
610 kref_put(&dev
->reada_curr_zone
->refcnt
, reada_zone_release
);
611 dev
->reada_curr_zone
= NULL
;
613 /* pick the zone with the most elements */
615 struct reada_zone
*zone
;
617 ret
= radix_tree_gang_lookup(&dev
->reada_zones
,
618 (void **)&zone
, index
, 1);
621 index
= (zone
->end
>> PAGE_SHIFT
) + 1;
623 if (zone
->elems
> top_locked_elems
) {
624 top_locked_elems
= zone
->elems
;
625 top_locked_zone
= zone
;
628 if (zone
->elems
> top_elems
) {
629 top_elems
= zone
->elems
;
635 dev
->reada_curr_zone
= top_zone
;
636 else if (top_locked_zone
)
637 dev
->reada_curr_zone
= top_locked_zone
;
641 dev
->reada_next
= dev
->reada_curr_zone
->start
;
642 kref_get(&dev
->reada_curr_zone
->refcnt
);
643 reada_peer_zones_set_lock(dev
->reada_curr_zone
, 1);
648 static int reada_start_machine_dev(struct btrfs_fs_info
*fs_info
,
649 struct btrfs_device
*dev
)
651 struct reada_extent
*re
= NULL
;
653 struct extent_buffer
*eb
= NULL
;
658 spin_lock(&fs_info
->reada_lock
);
659 if (dev
->reada_curr_zone
== NULL
) {
660 ret
= reada_pick_zone(dev
);
662 spin_unlock(&fs_info
->reada_lock
);
667 * FIXME currently we issue the reads one extent at a time. If we have
668 * a contiguous block of extents, we could also coagulate them or use
669 * plugging to speed things up
671 ret
= radix_tree_gang_lookup(&dev
->reada_extents
, (void **)&re
,
672 dev
->reada_next
>> PAGE_SHIFT
, 1);
673 if (ret
== 0 || re
->logical
> dev
->reada_curr_zone
->end
) {
674 ret
= reada_pick_zone(dev
);
676 spin_unlock(&fs_info
->reada_lock
);
680 ret
= radix_tree_gang_lookup(&dev
->reada_extents
, (void **)&re
,
681 dev
->reada_next
>> PAGE_SHIFT
, 1);
684 spin_unlock(&fs_info
->reada_lock
);
687 dev
->reada_next
= re
->logical
+ fs_info
->tree_root
->nodesize
;
690 spin_unlock(&fs_info
->reada_lock
);
692 spin_lock(&re
->lock
);
693 if (re
->scheduled
|| list_empty(&re
->extctl
)) {
694 spin_unlock(&re
->lock
);
695 reada_extent_put(fs_info
, re
);
699 spin_unlock(&re
->lock
);
704 for (i
= 0; i
< re
->nzones
; ++i
) {
705 if (re
->zones
[i
]->device
== dev
) {
710 logical
= re
->logical
;
712 atomic_inc(&dev
->reada_in_flight
);
713 ret
= reada_tree_block_flagged(fs_info
->extent_root
, logical
,
716 __readahead_hook(fs_info
, re
, NULL
, logical
, ret
);
718 __readahead_hook(fs_info
, re
, eb
, eb
->start
, ret
);
721 free_extent_buffer(eb
);
723 atomic_dec(&dev
->reada_in_flight
);
724 reada_extent_put(fs_info
, re
);
730 static void reada_start_machine_worker(struct btrfs_work
*work
)
732 struct reada_machine_work
*rmw
;
733 struct btrfs_fs_info
*fs_info
;
736 rmw
= container_of(work
, struct reada_machine_work
, work
);
737 fs_info
= rmw
->fs_info
;
741 old_ioprio
= IOPRIO_PRIO_VALUE(task_nice_ioclass(current
),
742 task_nice_ioprio(current
));
743 set_task_ioprio(current
, BTRFS_IOPRIO_READA
);
744 __reada_start_machine(fs_info
);
745 set_task_ioprio(current
, old_ioprio
);
747 atomic_dec(&fs_info
->reada_works_cnt
);
750 static void __reada_start_machine(struct btrfs_fs_info
*fs_info
)
752 struct btrfs_device
*device
;
753 struct btrfs_fs_devices
*fs_devices
= fs_info
->fs_devices
;
760 mutex_lock(&fs_devices
->device_list_mutex
);
761 list_for_each_entry(device
, &fs_devices
->devices
, dev_list
) {
762 if (atomic_read(&device
->reada_in_flight
) <
764 enqueued
+= reada_start_machine_dev(fs_info
,
767 mutex_unlock(&fs_devices
->device_list_mutex
);
769 } while (enqueued
&& total
< 10000);
775 * If everything is already in the cache, this is effectively single
776 * threaded. To a) not hold the caller for too long and b) to utilize
777 * more cores, we broke the loop above after 10000 iterations and now
778 * enqueue to workers to finish it. This will distribute the load to
781 for (i
= 0; i
< 2; ++i
) {
782 reada_start_machine(fs_info
);
783 if (atomic_read(&fs_info
->reada_works_cnt
) >
784 BTRFS_MAX_MIRRORS
* 2)
789 static void reada_start_machine(struct btrfs_fs_info
*fs_info
)
791 struct reada_machine_work
*rmw
;
793 rmw
= kzalloc(sizeof(*rmw
), GFP_KERNEL
);
795 /* FIXME we cannot handle this properly right now */
798 btrfs_init_work(&rmw
->work
, btrfs_readahead_helper
,
799 reada_start_machine_worker
, NULL
, NULL
);
800 rmw
->fs_info
= fs_info
;
802 btrfs_queue_work(fs_info
->readahead_workers
, &rmw
->work
);
803 atomic_inc(&fs_info
->reada_works_cnt
);
807 static void dump_devs(struct btrfs_fs_info
*fs_info
, int all
)
809 struct btrfs_device
*device
;
810 struct btrfs_fs_devices
*fs_devices
= fs_info
->fs_devices
;
817 spin_lock(&fs_info
->reada_lock
);
818 list_for_each_entry(device
, &fs_devices
->devices
, dev_list
) {
819 btrfs_debug(fs_info
, "dev %lld has %d in flight", device
->devid
,
820 atomic_read(&device
->reada_in_flight
));
823 struct reada_zone
*zone
;
824 ret
= radix_tree_gang_lookup(&device
->reada_zones
,
825 (void **)&zone
, index
, 1);
828 pr_debug(" zone %llu-%llu elems %llu locked %d devs",
829 zone
->start
, zone
->end
, zone
->elems
,
831 for (j
= 0; j
< zone
->ndevs
; ++j
) {
833 zone
->devs
[j
]->devid
);
835 if (device
->reada_curr_zone
== zone
)
836 pr_cont(" curr off %llu",
837 device
->reada_next
- zone
->start
);
839 index
= (zone
->end
>> PAGE_SHIFT
) + 1;
844 struct reada_extent
*re
= NULL
;
846 ret
= radix_tree_gang_lookup(&device
->reada_extents
,
847 (void **)&re
, index
, 1);
850 pr_debug(" re: logical %llu size %u empty %d scheduled %d",
851 re
->logical
, fs_info
->tree_root
->nodesize
,
852 list_empty(&re
->extctl
), re
->scheduled
);
854 for (i
= 0; i
< re
->nzones
; ++i
) {
855 pr_cont(" zone %llu-%llu devs",
858 for (j
= 0; j
< re
->zones
[i
]->ndevs
; ++j
) {
860 re
->zones
[i
]->devs
[j
]->devid
);
864 index
= (re
->logical
>> PAGE_SHIFT
) + 1;
873 struct reada_extent
*re
= NULL
;
875 ret
= radix_tree_gang_lookup(&fs_info
->reada_tree
, (void **)&re
,
879 if (!re
->scheduled
) {
880 index
= (re
->logical
>> PAGE_SHIFT
) + 1;
883 pr_debug("re: logical %llu size %u list empty %d scheduled %d",
884 re
->logical
, fs_info
->tree_root
->nodesize
,
885 list_empty(&re
->extctl
), re
->scheduled
);
886 for (i
= 0; i
< re
->nzones
; ++i
) {
887 pr_cont(" zone %llu-%llu devs",
890 for (j
= 0; j
< re
->zones
[i
]->ndevs
; ++j
) {
892 re
->zones
[i
]->devs
[j
]->devid
);
896 index
= (re
->logical
>> PAGE_SHIFT
) + 1;
898 spin_unlock(&fs_info
->reada_lock
);
905 struct reada_control
*btrfs_reada_add(struct btrfs_root
*root
,
906 struct btrfs_key
*key_start
, struct btrfs_key
*key_end
)
908 struct reada_control
*rc
;
912 struct extent_buffer
*node
;
913 static struct btrfs_key max_key
= {
919 rc
= kzalloc(sizeof(*rc
), GFP_KERNEL
);
921 return ERR_PTR(-ENOMEM
);
924 rc
->key_start
= *key_start
;
925 rc
->key_end
= *key_end
;
926 atomic_set(&rc
->elems
, 0);
927 init_waitqueue_head(&rc
->wait
);
928 kref_init(&rc
->refcnt
);
929 kref_get(&rc
->refcnt
); /* one ref for having elements */
931 node
= btrfs_root_node(root
);
933 generation
= btrfs_header_generation(node
);
934 free_extent_buffer(node
);
936 ret
= reada_add_block(rc
, start
, &max_key
, generation
);
942 reada_start_machine(root
->fs_info
);
948 int btrfs_reada_wait(void *handle
)
950 struct reada_control
*rc
= handle
;
951 struct btrfs_fs_info
*fs_info
= rc
->root
->fs_info
;
953 while (atomic_read(&rc
->elems
)) {
954 if (!atomic_read(&fs_info
->reada_works_cnt
))
955 reada_start_machine(fs_info
);
956 wait_event_timeout(rc
->wait
, atomic_read(&rc
->elems
) == 0,
958 dump_devs(rc
->root
->fs_info
,
959 atomic_read(&rc
->elems
) < 10 ? 1 : 0);
962 dump_devs(rc
->root
->fs_info
, atomic_read(&rc
->elems
) < 10 ? 1 : 0);
964 kref_put(&rc
->refcnt
, reada_control_release
);
969 int btrfs_reada_wait(void *handle
)
971 struct reada_control
*rc
= handle
;
972 struct btrfs_fs_info
*fs_info
= rc
->root
->fs_info
;
974 while (atomic_read(&rc
->elems
)) {
975 if (!atomic_read(&fs_info
->reada_works_cnt
))
976 reada_start_machine(fs_info
);
977 wait_event_timeout(rc
->wait
, atomic_read(&rc
->elems
) == 0,
981 kref_put(&rc
->refcnt
, reada_control_release
);
987 void btrfs_reada_detach(void *handle
)
989 struct reada_control
*rc
= handle
;
991 kref_put(&rc
->refcnt
, reada_control_release
);