2 * Copyright (C) 2011 STRATO. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
19 #include <linux/blkdev.h>
20 #include <linux/ratelimit.h>
24 #include "ordered-data.h"
25 #include "transaction.h"
29 * This is only the first step towards a full-features scrub. It reads all
30 * extent and super block and verifies the checksums. In case a bad checksum
31 * is found or the extent cannot be read, good data will be written back if
34 * Future enhancements:
35 * - To enhance the performance, better read-ahead strategies for the
36 * extent-tree can be employed.
37 * - In case an unrepairable extent is encountered, track which files are
38 * affected and report them
39 * - In case of a read error on files with nodatasum, map the file and read
40 * the extent to trigger a writeback of the good copy
41 * - track and record media errors, throw out bad devices
42 * - add a mode to also read unallocated space
43 * - make the prefetch cancellable
49 static void scrub_bio_end_io(struct bio
*bio
, int err
);
50 static void scrub_checksum(struct btrfs_work
*work
);
51 static int scrub_checksum_data(struct scrub_dev
*sdev
,
52 struct scrub_page
*spag
, void *buffer
);
53 static int scrub_checksum_tree_block(struct scrub_dev
*sdev
,
54 struct scrub_page
*spag
, u64 logical
,
56 static int scrub_checksum_super(struct scrub_bio
*sbio
, void *buffer
);
57 static int scrub_fixup_check(struct scrub_bio
*sbio
, int ix
);
58 static void scrub_fixup_end_io(struct bio
*bio
, int err
);
59 static int scrub_fixup_io(int rw
, struct block_device
*bdev
, sector_t sector
,
61 static void scrub_fixup(struct scrub_bio
*sbio
, int ix
);
63 #define SCRUB_PAGES_PER_BIO 16 /* 64k per bio */
64 #define SCRUB_BIOS_PER_DEV 16 /* 1 MB per device in flight */
67 u64 flags
; /* extent flags */
71 u8 csum
[BTRFS_CSUM_SIZE
];
76 struct scrub_dev
*sdev
;
81 struct scrub_page spag
[SCRUB_PAGES_PER_BIO
];
84 struct btrfs_work work
;
88 struct scrub_bio
*bios
[SCRUB_BIOS_PER_DEV
];
89 struct btrfs_device
*dev
;
95 wait_queue_head_t list_wait
;
97 struct list_head csum_list
;
103 struct btrfs_scrub_progress stat
;
104 spinlock_t stat_lock
;
107 struct scrub_fixup_nodatasum
{
108 struct scrub_dev
*sdev
;
110 struct btrfs_root
*root
;
111 struct btrfs_work work
;
115 struct scrub_warning
{
116 struct btrfs_path
*path
;
117 u64 extent_item_size
;
123 struct btrfs_device
*dev
;
128 static void scrub_free_csums(struct scrub_dev
*sdev
)
130 while (!list_empty(&sdev
->csum_list
)) {
131 struct btrfs_ordered_sum
*sum
;
132 sum
= list_first_entry(&sdev
->csum_list
,
133 struct btrfs_ordered_sum
, list
);
134 list_del(&sum
->list
);
139 static void scrub_free_bio(struct bio
*bio
)
142 struct page
*last_page
= NULL
;
147 for (i
= 0; i
< bio
->bi_vcnt
; ++i
) {
148 if (bio
->bi_io_vec
[i
].bv_page
== last_page
)
150 last_page
= bio
->bi_io_vec
[i
].bv_page
;
151 __free_page(last_page
);
156 static noinline_for_stack
void scrub_free_dev(struct scrub_dev
*sdev
)
163 for (i
= 0; i
< SCRUB_BIOS_PER_DEV
; ++i
) {
164 struct scrub_bio
*sbio
= sdev
->bios
[i
];
169 scrub_free_bio(sbio
->bio
);
173 scrub_free_csums(sdev
);
177 static noinline_for_stack
178 struct scrub_dev
*scrub_setup_dev(struct btrfs_device
*dev
)
180 struct scrub_dev
*sdev
;
182 struct btrfs_fs_info
*fs_info
= dev
->dev_root
->fs_info
;
184 sdev
= kzalloc(sizeof(*sdev
), GFP_NOFS
);
188 for (i
= 0; i
< SCRUB_BIOS_PER_DEV
; ++i
) {
189 struct scrub_bio
*sbio
;
191 sbio
= kzalloc(sizeof(*sbio
), GFP_NOFS
);
194 sdev
->bios
[i
] = sbio
;
199 sbio
->work
.func
= scrub_checksum
;
201 if (i
!= SCRUB_BIOS_PER_DEV
-1)
202 sdev
->bios
[i
]->next_free
= i
+ 1;
204 sdev
->bios
[i
]->next_free
= -1;
206 sdev
->first_free
= 0;
208 atomic_set(&sdev
->in_flight
, 0);
209 atomic_set(&sdev
->fixup_cnt
, 0);
210 atomic_set(&sdev
->cancel_req
, 0);
211 sdev
->csum_size
= btrfs_super_csum_size(&fs_info
->super_copy
);
212 INIT_LIST_HEAD(&sdev
->csum_list
);
214 spin_lock_init(&sdev
->list_lock
);
215 spin_lock_init(&sdev
->stat_lock
);
216 init_waitqueue_head(&sdev
->list_wait
);
220 scrub_free_dev(sdev
);
221 return ERR_PTR(-ENOMEM
);
224 static int scrub_print_warning_inode(u64 inum
, u64 offset
, u64 root
, void *ctx
)
230 struct extent_buffer
*eb
;
231 struct btrfs_inode_item
*inode_item
;
232 struct scrub_warning
*swarn
= ctx
;
233 struct btrfs_fs_info
*fs_info
= swarn
->dev
->dev_root
->fs_info
;
234 struct inode_fs_paths
*ipath
= NULL
;
235 struct btrfs_root
*local_root
;
236 struct btrfs_key root_key
;
238 root_key
.objectid
= root
;
239 root_key
.type
= BTRFS_ROOT_ITEM_KEY
;
240 root_key
.offset
= (u64
)-1;
241 local_root
= btrfs_read_fs_root_no_name(fs_info
, &root_key
);
242 if (IS_ERR(local_root
)) {
243 ret
= PTR_ERR(local_root
);
247 ret
= inode_item_info(inum
, 0, local_root
, swarn
->path
);
249 btrfs_release_path(swarn
->path
);
253 eb
= swarn
->path
->nodes
[0];
254 inode_item
= btrfs_item_ptr(eb
, swarn
->path
->slots
[0],
255 struct btrfs_inode_item
);
256 isize
= btrfs_inode_size(eb
, inode_item
);
257 nlink
= btrfs_inode_nlink(eb
, inode_item
);
258 btrfs_release_path(swarn
->path
);
260 ipath
= init_ipath(4096, local_root
, swarn
->path
);
261 ret
= paths_from_inode(inum
, ipath
);
267 * we deliberately ignore the bit ipath might have been too small to
268 * hold all of the paths here
270 for (i
= 0; i
< ipath
->fspath
->elem_cnt
; ++i
)
271 printk(KERN_WARNING
"btrfs: %s at logical %llu on dev "
272 "%s, sector %llu, root %llu, inode %llu, offset %llu, "
273 "length %llu, links %u (path: %s)\n", swarn
->errstr
,
274 swarn
->logical
, swarn
->dev
->name
,
275 (unsigned long long)swarn
->sector
, root
, inum
, offset
,
276 min(isize
- offset
, (u64
)PAGE_SIZE
), nlink
,
277 ipath
->fspath
->str
[i
]);
283 printk(KERN_WARNING
"btrfs: %s at logical %llu on dev "
284 "%s, sector %llu, root %llu, inode %llu, offset %llu: path "
285 "resolving failed with ret=%d\n", swarn
->errstr
,
286 swarn
->logical
, swarn
->dev
->name
,
287 (unsigned long long)swarn
->sector
, root
, inum
, offset
, ret
);
293 static void scrub_print_warning(const char *errstr
, struct scrub_bio
*sbio
,
296 struct btrfs_device
*dev
= sbio
->sdev
->dev
;
297 struct btrfs_fs_info
*fs_info
= dev
->dev_root
->fs_info
;
298 struct btrfs_path
*path
;
299 struct btrfs_key found_key
;
300 struct extent_buffer
*eb
;
301 struct btrfs_extent_item
*ei
;
302 struct scrub_warning swarn
;
307 unsigned long ptr
= 0;
308 const int bufsize
= 4096;
311 path
= btrfs_alloc_path();
313 swarn
.scratch_buf
= kmalloc(bufsize
, GFP_NOFS
);
314 swarn
.msg_buf
= kmalloc(bufsize
, GFP_NOFS
);
315 swarn
.sector
= (sbio
->physical
+ ix
* PAGE_SIZE
) >> 9;
316 swarn
.logical
= sbio
->logical
+ ix
* PAGE_SIZE
;
317 swarn
.errstr
= errstr
;
319 swarn
.msg_bufsize
= bufsize
;
320 swarn
.scratch_bufsize
= bufsize
;
322 if (!path
|| !swarn
.scratch_buf
|| !swarn
.msg_buf
)
325 ret
= extent_from_logical(fs_info
, swarn
.logical
, path
, &found_key
);
329 extent_offset
= swarn
.logical
- found_key
.objectid
;
330 swarn
.extent_item_size
= found_key
.offset
;
333 ei
= btrfs_item_ptr(eb
, path
->slots
[0], struct btrfs_extent_item
);
334 item_size
= btrfs_item_size_nr(eb
, path
->slots
[0]);
336 if (ret
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) {
338 ret
= tree_backref_for_extent(&ptr
, eb
, ei
, item_size
,
339 &ref_root
, &ref_level
);
340 printk(KERN_WARNING
"%s at logical %llu on dev %s, "
341 "sector %llu: metadata %s (level %d) in tree "
342 "%llu\n", errstr
, swarn
.logical
, dev
->name
,
343 (unsigned long long)swarn
.sector
,
344 ref_level
? "node" : "leaf",
345 ret
< 0 ? -1 : ref_level
,
346 ret
< 0 ? -1 : ref_root
);
350 iterate_extent_inodes(fs_info
, path
, found_key
.objectid
,
352 scrub_print_warning_inode
, &swarn
);
356 btrfs_free_path(path
);
357 kfree(swarn
.scratch_buf
);
358 kfree(swarn
.msg_buf
);
361 static int scrub_fixup_readpage(u64 inum
, u64 offset
, u64 root
, void *ctx
)
365 struct scrub_fixup_nodatasum
*fixup
= ctx
;
368 struct btrfs_key key
;
370 u64 end
= offset
+ PAGE_SIZE
- 1;
371 struct btrfs_root
*local_root
;
374 key
.type
= BTRFS_ROOT_ITEM_KEY
;
375 key
.offset
= (u64
)-1;
376 local_root
= btrfs_read_fs_root_no_name(fixup
->root
->fs_info
, &key
);
377 if (IS_ERR(local_root
))
378 return PTR_ERR(local_root
);
380 key
.type
= BTRFS_INODE_ITEM_KEY
;
383 inode
= btrfs_iget(fixup
->root
->fs_info
->sb
, &key
, local_root
, NULL
);
385 return PTR_ERR(inode
);
387 ret
= set_extent_bit(&BTRFS_I(inode
)->io_tree
, offset
, end
,
388 EXTENT_DAMAGED
, 0, NULL
, NULL
, GFP_NOFS
);
390 /* set_extent_bit should either succeed or give proper error */
393 return ret
< 0 ? ret
: -EFAULT
;
395 index
= offset
>> PAGE_CACHE_SHIFT
;
397 page
= find_or_create_page(inode
->i_mapping
, index
, GFP_NOFS
);
401 ret
= extent_read_full_page(&BTRFS_I(inode
)->io_tree
, page
,
402 btrfs_get_extent
, fixup
->mirror_num
);
403 wait_on_page_locked(page
);
404 corrected
= !test_range_bit(&BTRFS_I(inode
)->io_tree
, offset
, end
,
405 EXTENT_DAMAGED
, 0, NULL
);
408 WARN_ON(!PageUptodate(page
));
410 clear_extent_bit(&BTRFS_I(inode
)->io_tree
, offset
, end
,
411 EXTENT_DAMAGED
, 0, 0, NULL
, GFP_NOFS
);
419 if (ret
== 0 && corrected
) {
421 * we only need to call readpage for one of the inodes belonging
422 * to this extent. so make iterate_extent_inodes stop
430 static void scrub_fixup_nodatasum(struct btrfs_work
*work
)
433 struct scrub_fixup_nodatasum
*fixup
;
434 struct scrub_dev
*sdev
;
435 struct btrfs_trans_handle
*trans
= NULL
;
436 struct btrfs_fs_info
*fs_info
;
437 struct btrfs_path
*path
;
438 int uncorrectable
= 0;
440 fixup
= container_of(work
, struct scrub_fixup_nodatasum
, work
);
442 fs_info
= fixup
->root
->fs_info
;
444 path
= btrfs_alloc_path();
446 spin_lock(&sdev
->stat_lock
);
447 ++sdev
->stat
.malloc_errors
;
448 spin_unlock(&sdev
->stat_lock
);
453 trans
= btrfs_join_transaction(fixup
->root
);
460 * the idea is to trigger a regular read through the standard path. we
461 * read a page from the (failed) logical address by specifying the
462 * corresponding copynum of the failed sector. thus, that readpage is
464 * that is the point where on-the-fly error correction will kick in
465 * (once it's finished) and rewrite the failed sector if a good copy
468 ret
= iterate_inodes_from_logical(fixup
->logical
, fixup
->root
->fs_info
,
469 path
, scrub_fixup_readpage
,
477 spin_lock(&sdev
->stat_lock
);
478 ++sdev
->stat
.corrected_errors
;
479 spin_unlock(&sdev
->stat_lock
);
482 if (trans
&& !IS_ERR(trans
))
483 btrfs_end_transaction(trans
, fixup
->root
);
485 spin_lock(&sdev
->stat_lock
);
486 ++sdev
->stat
.uncorrectable_errors
;
487 spin_unlock(&sdev
->stat_lock
);
488 printk_ratelimited(KERN_ERR
"btrfs: unable to fixup "
489 "(nodatasum) error at logical %llu\n",
493 btrfs_free_path(path
);
496 /* see caller why we're pretending to be paused in the scrub counters */
497 mutex_lock(&fs_info
->scrub_lock
);
498 atomic_dec(&fs_info
->scrubs_running
);
499 atomic_dec(&fs_info
->scrubs_paused
);
500 mutex_unlock(&fs_info
->scrub_lock
);
501 atomic_dec(&sdev
->fixup_cnt
);
502 wake_up(&fs_info
->scrub_pause_wait
);
503 wake_up(&sdev
->list_wait
);
507 * scrub_recheck_error gets called when either verification of the page
508 * failed or the bio failed to read, e.g. with EIO. In the latter case,
509 * recheck_error gets called for every page in the bio, even though only
512 static int scrub_recheck_error(struct scrub_bio
*sbio
, int ix
)
514 struct scrub_dev
*sdev
= sbio
->sdev
;
515 u64 sector
= (sbio
->physical
+ ix
* PAGE_SIZE
) >> 9;
516 static DEFINE_RATELIMIT_STATE(_rs
, DEFAULT_RATELIMIT_INTERVAL
,
517 DEFAULT_RATELIMIT_BURST
);
520 if (scrub_fixup_io(READ
, sbio
->sdev
->dev
->bdev
, sector
,
521 sbio
->bio
->bi_io_vec
[ix
].bv_page
) == 0) {
522 if (scrub_fixup_check(sbio
, ix
) == 0)
525 if (__ratelimit(&_rs
))
526 scrub_print_warning("i/o error", sbio
, ix
);
528 if (__ratelimit(&_rs
))
529 scrub_print_warning("checksum error", sbio
, ix
);
532 spin_lock(&sdev
->stat_lock
);
533 ++sdev
->stat
.read_errors
;
534 spin_unlock(&sdev
->stat_lock
);
536 scrub_fixup(sbio
, ix
);
540 static int scrub_fixup_check(struct scrub_bio
*sbio
, int ix
)
545 u64 flags
= sbio
->spag
[ix
].flags
;
547 page
= sbio
->bio
->bi_io_vec
[ix
].bv_page
;
548 buffer
= kmap_atomic(page
, KM_USER0
);
549 if (flags
& BTRFS_EXTENT_FLAG_DATA
) {
550 ret
= scrub_checksum_data(sbio
->sdev
,
551 sbio
->spag
+ ix
, buffer
);
552 } else if (flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) {
553 ret
= scrub_checksum_tree_block(sbio
->sdev
,
555 sbio
->logical
+ ix
* PAGE_SIZE
,
560 kunmap_atomic(buffer
, KM_USER0
);
565 static void scrub_fixup_end_io(struct bio
*bio
, int err
)
567 complete((struct completion
*)bio
->bi_private
);
570 static void scrub_fixup(struct scrub_bio
*sbio
, int ix
)
572 struct scrub_dev
*sdev
= sbio
->sdev
;
573 struct btrfs_fs_info
*fs_info
= sdev
->dev
->dev_root
->fs_info
;
574 struct btrfs_mapping_tree
*map_tree
= &fs_info
->mapping_tree
;
575 struct btrfs_bio
*bbio
= NULL
;
576 struct scrub_fixup_nodatasum
*fixup
;
577 u64 logical
= sbio
->logical
+ ix
* PAGE_SIZE
;
581 DECLARE_COMPLETION_ONSTACK(complete
);
583 if ((sbio
->spag
[ix
].flags
& BTRFS_EXTENT_FLAG_DATA
) &&
584 (sbio
->spag
[ix
].have_csum
== 0)) {
585 fixup
= kzalloc(sizeof(*fixup
), GFP_NOFS
);
589 fixup
->logical
= logical
;
590 fixup
->root
= fs_info
->extent_root
;
591 fixup
->mirror_num
= sbio
->spag
[ix
].mirror_num
;
593 * increment scrubs_running to prevent cancel requests from
594 * completing as long as a fixup worker is running. we must also
595 * increment scrubs_paused to prevent deadlocking on pause
596 * requests used for transactions commits (as the worker uses a
597 * transaction context). it is safe to regard the fixup worker
598 * as paused for all matters practical. effectively, we only
599 * avoid cancellation requests from completing.
601 mutex_lock(&fs_info
->scrub_lock
);
602 atomic_inc(&fs_info
->scrubs_running
);
603 atomic_inc(&fs_info
->scrubs_paused
);
604 mutex_unlock(&fs_info
->scrub_lock
);
605 atomic_inc(&sdev
->fixup_cnt
);
606 fixup
->work
.func
= scrub_fixup_nodatasum
;
607 btrfs_queue_worker(&fs_info
->scrub_workers
, &fixup
->work
);
612 ret
= btrfs_map_block(map_tree
, REQ_WRITE
, logical
, &length
,
614 if (ret
|| !bbio
|| length
< PAGE_SIZE
) {
616 "scrub_fixup: btrfs_map_block failed us for %llu\n",
617 (unsigned long long)logical
);
622 if (bbio
->num_stripes
== 1)
623 /* there aren't any replicas */
627 * first find a good copy
629 for (i
= 0; i
< bbio
->num_stripes
; ++i
) {
630 if (i
+ 1 == sbio
->spag
[ix
].mirror_num
)
633 if (scrub_fixup_io(READ
, bbio
->stripes
[i
].dev
->bdev
,
634 bbio
->stripes
[i
].physical
>> 9,
635 sbio
->bio
->bi_io_vec
[ix
].bv_page
)) {
636 /* I/O-error, this is not a good copy */
640 if (scrub_fixup_check(sbio
, ix
) == 0)
643 if (i
== bbio
->num_stripes
)
646 if (!sdev
->readonly
) {
648 * bi_io_vec[ix].bv_page now contains good data, write it back
650 if (scrub_fixup_io(WRITE
, sdev
->dev
->bdev
,
651 (sbio
->physical
+ ix
* PAGE_SIZE
) >> 9,
652 sbio
->bio
->bi_io_vec
[ix
].bv_page
)) {
653 /* I/O-error, writeback failed, give up */
659 spin_lock(&sdev
->stat_lock
);
660 ++sdev
->stat
.corrected_errors
;
661 spin_unlock(&sdev
->stat_lock
);
663 printk_ratelimited(KERN_ERR
"btrfs: fixed up error at logical %llu\n",
664 (unsigned long long)logical
);
669 spin_lock(&sdev
->stat_lock
);
670 ++sdev
->stat
.uncorrectable_errors
;
671 spin_unlock(&sdev
->stat_lock
);
673 printk_ratelimited(KERN_ERR
"btrfs: unable to fixup (regular) error at "
674 "logical %llu\n", (unsigned long long)logical
);
677 static int scrub_fixup_io(int rw
, struct block_device
*bdev
, sector_t sector
,
680 struct bio
*bio
= NULL
;
682 DECLARE_COMPLETION_ONSTACK(complete
);
684 bio
= bio_alloc(GFP_NOFS
, 1);
686 bio
->bi_sector
= sector
;
687 bio_add_page(bio
, page
, PAGE_SIZE
, 0);
688 bio
->bi_end_io
= scrub_fixup_end_io
;
689 bio
->bi_private
= &complete
;
692 /* this will also unplug the queue */
693 wait_for_completion(&complete
);
695 ret
= !test_bit(BIO_UPTODATE
, &bio
->bi_flags
);
700 static void scrub_bio_end_io(struct bio
*bio
, int err
)
702 struct scrub_bio
*sbio
= bio
->bi_private
;
703 struct scrub_dev
*sdev
= sbio
->sdev
;
704 struct btrfs_fs_info
*fs_info
= sdev
->dev
->dev_root
->fs_info
;
709 btrfs_queue_worker(&fs_info
->scrub_workers
, &sbio
->work
);
712 static void scrub_checksum(struct btrfs_work
*work
)
714 struct scrub_bio
*sbio
= container_of(work
, struct scrub_bio
, work
);
715 struct scrub_dev
*sdev
= sbio
->sdev
;
725 for (i
= 0; i
< sbio
->count
; ++i
)
726 ret
|= scrub_recheck_error(sbio
, i
);
728 spin_lock(&sdev
->stat_lock
);
729 ++sdev
->stat
.unverified_errors
;
730 spin_unlock(&sdev
->stat_lock
);
733 sbio
->bio
->bi_flags
&= ~(BIO_POOL_MASK
- 1);
734 sbio
->bio
->bi_flags
|= 1 << BIO_UPTODATE
;
735 sbio
->bio
->bi_phys_segments
= 0;
736 sbio
->bio
->bi_idx
= 0;
738 for (i
= 0; i
< sbio
->count
; i
++) {
740 bi
= &sbio
->bio
->bi_io_vec
[i
];
742 bi
->bv_len
= PAGE_SIZE
;
746 for (i
= 0; i
< sbio
->count
; ++i
) {
747 page
= sbio
->bio
->bi_io_vec
[i
].bv_page
;
748 buffer
= kmap_atomic(page
, KM_USER0
);
749 flags
= sbio
->spag
[i
].flags
;
750 logical
= sbio
->logical
+ i
* PAGE_SIZE
;
752 if (flags
& BTRFS_EXTENT_FLAG_DATA
) {
753 ret
= scrub_checksum_data(sdev
, sbio
->spag
+ i
, buffer
);
754 } else if (flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) {
755 ret
= scrub_checksum_tree_block(sdev
, sbio
->spag
+ i
,
757 } else if (flags
& BTRFS_EXTENT_FLAG_SUPER
) {
759 (void)scrub_checksum_super(sbio
, buffer
);
763 kunmap_atomic(buffer
, KM_USER0
);
765 ret
= scrub_recheck_error(sbio
, i
);
767 spin_lock(&sdev
->stat_lock
);
768 ++sdev
->stat
.unverified_errors
;
769 spin_unlock(&sdev
->stat_lock
);
775 scrub_free_bio(sbio
->bio
);
777 spin_lock(&sdev
->list_lock
);
778 sbio
->next_free
= sdev
->first_free
;
779 sdev
->first_free
= sbio
->index
;
780 spin_unlock(&sdev
->list_lock
);
781 atomic_dec(&sdev
->in_flight
);
782 wake_up(&sdev
->list_wait
);
785 static int scrub_checksum_data(struct scrub_dev
*sdev
,
786 struct scrub_page
*spag
, void *buffer
)
788 u8 csum
[BTRFS_CSUM_SIZE
];
791 struct btrfs_root
*root
= sdev
->dev
->dev_root
;
793 if (!spag
->have_csum
)
796 crc
= btrfs_csum_data(root
, buffer
, crc
, PAGE_SIZE
);
797 btrfs_csum_final(crc
, csum
);
798 if (memcmp(csum
, spag
->csum
, sdev
->csum_size
))
801 spin_lock(&sdev
->stat_lock
);
802 ++sdev
->stat
.data_extents_scrubbed
;
803 sdev
->stat
.data_bytes_scrubbed
+= PAGE_SIZE
;
805 ++sdev
->stat
.csum_errors
;
806 spin_unlock(&sdev
->stat_lock
);
811 static int scrub_checksum_tree_block(struct scrub_dev
*sdev
,
812 struct scrub_page
*spag
, u64 logical
,
815 struct btrfs_header
*h
;
816 struct btrfs_root
*root
= sdev
->dev
->dev_root
;
817 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
818 u8 csum
[BTRFS_CSUM_SIZE
];
824 * we don't use the getter functions here, as we
825 * a) don't have an extent buffer and
826 * b) the page is already kmapped
828 h
= (struct btrfs_header
*)buffer
;
830 if (logical
!= le64_to_cpu(h
->bytenr
))
833 if (spag
->generation
!= le64_to_cpu(h
->generation
))
836 if (memcmp(h
->fsid
, fs_info
->fsid
, BTRFS_UUID_SIZE
))
839 if (memcmp(h
->chunk_tree_uuid
, fs_info
->chunk_tree_uuid
,
843 crc
= btrfs_csum_data(root
, buffer
+ BTRFS_CSUM_SIZE
, crc
,
844 PAGE_SIZE
- BTRFS_CSUM_SIZE
);
845 btrfs_csum_final(crc
, csum
);
846 if (memcmp(csum
, h
->csum
, sdev
->csum_size
))
849 spin_lock(&sdev
->stat_lock
);
850 ++sdev
->stat
.tree_extents_scrubbed
;
851 sdev
->stat
.tree_bytes_scrubbed
+= PAGE_SIZE
;
853 ++sdev
->stat
.csum_errors
;
855 ++sdev
->stat
.verify_errors
;
856 spin_unlock(&sdev
->stat_lock
);
858 return fail
|| crc_fail
;
861 static int scrub_checksum_super(struct scrub_bio
*sbio
, void *buffer
)
863 struct btrfs_super_block
*s
;
865 struct scrub_dev
*sdev
= sbio
->sdev
;
866 struct btrfs_root
*root
= sdev
->dev
->dev_root
;
867 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
868 u8 csum
[BTRFS_CSUM_SIZE
];
872 s
= (struct btrfs_super_block
*)buffer
;
873 logical
= sbio
->logical
;
875 if (logical
!= le64_to_cpu(s
->bytenr
))
878 if (sbio
->spag
[0].generation
!= le64_to_cpu(s
->generation
))
881 if (memcmp(s
->fsid
, fs_info
->fsid
, BTRFS_UUID_SIZE
))
884 crc
= btrfs_csum_data(root
, buffer
+ BTRFS_CSUM_SIZE
, crc
,
885 PAGE_SIZE
- BTRFS_CSUM_SIZE
);
886 btrfs_csum_final(crc
, csum
);
887 if (memcmp(csum
, s
->csum
, sbio
->sdev
->csum_size
))
892 * if we find an error in a super block, we just report it.
893 * They will get written with the next transaction commit
896 spin_lock(&sdev
->stat_lock
);
897 ++sdev
->stat
.super_errors
;
898 spin_unlock(&sdev
->stat_lock
);
904 static int scrub_submit(struct scrub_dev
*sdev
)
906 struct scrub_bio
*sbio
;
910 if (sdev
->curr
== -1)
913 sbio
= sdev
->bios
[sdev
->curr
];
915 bio
= bio_alloc(GFP_NOFS
, sbio
->count
);
919 bio
->bi_private
= sbio
;
920 bio
->bi_end_io
= scrub_bio_end_io
;
921 bio
->bi_bdev
= sdev
->dev
->bdev
;
922 bio
->bi_sector
= sbio
->physical
>> 9;
924 for (i
= 0; i
< sbio
->count
; ++i
) {
928 page
= alloc_page(GFP_NOFS
);
932 ret
= bio_add_page(bio
, page
, PAGE_SIZE
, 0);
941 atomic_inc(&sdev
->in_flight
);
943 submit_bio(READ
, bio
);
953 static int scrub_page(struct scrub_dev
*sdev
, u64 logical
, u64 len
,
954 u64 physical
, u64 flags
, u64 gen
, int mirror_num
,
957 struct scrub_bio
*sbio
;
961 * grab a fresh bio or wait for one to become available
963 while (sdev
->curr
== -1) {
964 spin_lock(&sdev
->list_lock
);
965 sdev
->curr
= sdev
->first_free
;
966 if (sdev
->curr
!= -1) {
967 sdev
->first_free
= sdev
->bios
[sdev
->curr
]->next_free
;
968 sdev
->bios
[sdev
->curr
]->next_free
= -1;
969 sdev
->bios
[sdev
->curr
]->count
= 0;
970 spin_unlock(&sdev
->list_lock
);
972 spin_unlock(&sdev
->list_lock
);
973 wait_event(sdev
->list_wait
, sdev
->first_free
!= -1);
976 sbio
= sdev
->bios
[sdev
->curr
];
977 if (sbio
->count
== 0) {
978 sbio
->physical
= physical
;
979 sbio
->logical
= logical
;
980 } else if (sbio
->physical
+ sbio
->count
* PAGE_SIZE
!= physical
||
981 sbio
->logical
+ sbio
->count
* PAGE_SIZE
!= logical
) {
984 ret
= scrub_submit(sdev
);
989 sbio
->spag
[sbio
->count
].flags
= flags
;
990 sbio
->spag
[sbio
->count
].generation
= gen
;
991 sbio
->spag
[sbio
->count
].have_csum
= 0;
992 sbio
->spag
[sbio
->count
].mirror_num
= mirror_num
;
994 sbio
->spag
[sbio
->count
].have_csum
= 1;
995 memcpy(sbio
->spag
[sbio
->count
].csum
, csum
, sdev
->csum_size
);
998 if (sbio
->count
== SCRUB_PAGES_PER_BIO
|| force
) {
1001 ret
= scrub_submit(sdev
);
1009 static int scrub_find_csum(struct scrub_dev
*sdev
, u64 logical
, u64 len
,
1012 struct btrfs_ordered_sum
*sum
= NULL
;
1015 unsigned long num_sectors
;
1016 u32 sectorsize
= sdev
->dev
->dev_root
->sectorsize
;
1018 while (!list_empty(&sdev
->csum_list
)) {
1019 sum
= list_first_entry(&sdev
->csum_list
,
1020 struct btrfs_ordered_sum
, list
);
1021 if (sum
->bytenr
> logical
)
1023 if (sum
->bytenr
+ sum
->len
> logical
)
1026 ++sdev
->stat
.csum_discards
;
1027 list_del(&sum
->list
);
1034 num_sectors
= sum
->len
/ sectorsize
;
1035 for (i
= 0; i
< num_sectors
; ++i
) {
1036 if (sum
->sums
[i
].bytenr
== logical
) {
1037 memcpy(csum
, &sum
->sums
[i
].sum
, sdev
->csum_size
);
1042 if (ret
&& i
== num_sectors
- 1) {
1043 list_del(&sum
->list
);
1049 /* scrub extent tries to collect up to 64 kB for each bio */
1050 static int scrub_extent(struct scrub_dev
*sdev
, u64 logical
, u64 len
,
1051 u64 physical
, u64 flags
, u64 gen
, int mirror_num
)
1054 u8 csum
[BTRFS_CSUM_SIZE
];
1057 u64 l
= min_t(u64
, len
, PAGE_SIZE
);
1060 if (flags
& BTRFS_EXTENT_FLAG_DATA
) {
1061 /* push csums to sbio */
1062 have_csum
= scrub_find_csum(sdev
, logical
, l
, csum
);
1064 ++sdev
->stat
.no_csum
;
1066 ret
= scrub_page(sdev
, logical
, l
, physical
, flags
, gen
,
1067 mirror_num
, have_csum
? csum
: NULL
, 0);
1077 static noinline_for_stack
int scrub_stripe(struct scrub_dev
*sdev
,
1078 struct map_lookup
*map
, int num
, u64 base
, u64 length
)
1080 struct btrfs_path
*path
;
1081 struct btrfs_fs_info
*fs_info
= sdev
->dev
->dev_root
->fs_info
;
1082 struct btrfs_root
*root
= fs_info
->extent_root
;
1083 struct btrfs_root
*csum_root
= fs_info
->csum_root
;
1084 struct btrfs_extent_item
*extent
;
1085 struct blk_plug plug
;
1092 struct extent_buffer
*l
;
1093 struct btrfs_key key
;
1099 u64 increment
= map
->stripe_len
;
1104 do_div(nstripes
, map
->stripe_len
);
1105 if (map
->type
& BTRFS_BLOCK_GROUP_RAID0
) {
1106 offset
= map
->stripe_len
* num
;
1107 increment
= map
->stripe_len
* map
->num_stripes
;
1109 } else if (map
->type
& BTRFS_BLOCK_GROUP_RAID10
) {
1110 int factor
= map
->num_stripes
/ map
->sub_stripes
;
1111 offset
= map
->stripe_len
* (num
/ map
->sub_stripes
);
1112 increment
= map
->stripe_len
* factor
;
1113 mirror_num
= num
% map
->sub_stripes
+ 1;
1114 } else if (map
->type
& BTRFS_BLOCK_GROUP_RAID1
) {
1115 increment
= map
->stripe_len
;
1116 mirror_num
= num
% map
->num_stripes
+ 1;
1117 } else if (map
->type
& BTRFS_BLOCK_GROUP_DUP
) {
1118 increment
= map
->stripe_len
;
1119 mirror_num
= num
% map
->num_stripes
+ 1;
1121 increment
= map
->stripe_len
;
1125 path
= btrfs_alloc_path();
1130 path
->search_commit_root
= 1;
1131 path
->skip_locking
= 1;
1134 * find all extents for each stripe and just read them to get
1135 * them into the page cache
1136 * FIXME: we can do better. build a more intelligent prefetching
1138 logical
= base
+ offset
;
1139 physical
= map
->stripes
[num
].physical
;
1141 for (i
= 0; i
< nstripes
; ++i
) {
1142 key
.objectid
= logical
;
1143 key
.type
= BTRFS_EXTENT_ITEM_KEY
;
1144 key
.offset
= (u64
)0;
1146 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
1151 * we might miss half an extent here, but that doesn't matter,
1152 * as it's only the prefetch
1156 slot
= path
->slots
[0];
1157 if (slot
>= btrfs_header_nritems(l
)) {
1158 ret
= btrfs_next_leaf(root
, path
);
1166 btrfs_item_key_to_cpu(l
, &key
, slot
);
1168 if (key
.objectid
>= logical
+ map
->stripe_len
)
1173 btrfs_release_path(path
);
1174 logical
+= increment
;
1175 physical
+= map
->stripe_len
;
1180 * collect all data csums for the stripe to avoid seeking during
1181 * the scrub. This might currently (crc32) end up to be about 1MB
1184 blk_start_plug(&plug
);
1186 logical
= base
+ offset
+ start_stripe
* increment
;
1187 for (i
= start_stripe
; i
< nstripes
; ++i
) {
1188 ret
= btrfs_lookup_csums_range(csum_root
, logical
,
1189 logical
+ map
->stripe_len
- 1,
1190 &sdev
->csum_list
, 1);
1194 logical
+= increment
;
1198 * now find all extents for each stripe and scrub them
1200 logical
= base
+ offset
+ start_stripe
* increment
;
1201 physical
= map
->stripes
[num
].physical
+ start_stripe
* map
->stripe_len
;
1203 for (i
= start_stripe
; i
< nstripes
; ++i
) {
1207 if (atomic_read(&fs_info
->scrub_cancel_req
) ||
1208 atomic_read(&sdev
->cancel_req
)) {
1213 * check to see if we have to pause
1215 if (atomic_read(&fs_info
->scrub_pause_req
)) {
1216 /* push queued extents */
1218 wait_event(sdev
->list_wait
,
1219 atomic_read(&sdev
->in_flight
) == 0);
1220 atomic_inc(&fs_info
->scrubs_paused
);
1221 wake_up(&fs_info
->scrub_pause_wait
);
1222 mutex_lock(&fs_info
->scrub_lock
);
1223 while (atomic_read(&fs_info
->scrub_pause_req
)) {
1224 mutex_unlock(&fs_info
->scrub_lock
);
1225 wait_event(fs_info
->scrub_pause_wait
,
1226 atomic_read(&fs_info
->scrub_pause_req
) == 0);
1227 mutex_lock(&fs_info
->scrub_lock
);
1229 atomic_dec(&fs_info
->scrubs_paused
);
1230 mutex_unlock(&fs_info
->scrub_lock
);
1231 wake_up(&fs_info
->scrub_pause_wait
);
1232 scrub_free_csums(sdev
);
1237 key
.objectid
= logical
;
1238 key
.type
= BTRFS_EXTENT_ITEM_KEY
;
1239 key
.offset
= (u64
)0;
1241 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
1245 ret
= btrfs_previous_item(root
, path
, 0,
1246 BTRFS_EXTENT_ITEM_KEY
);
1250 /* there's no smaller item, so stick with the
1252 btrfs_release_path(path
);
1253 ret
= btrfs_search_slot(NULL
, root
, &key
,
1262 slot
= path
->slots
[0];
1263 if (slot
>= btrfs_header_nritems(l
)) {
1264 ret
= btrfs_next_leaf(root
, path
);
1272 btrfs_item_key_to_cpu(l
, &key
, slot
);
1274 if (key
.objectid
+ key
.offset
<= logical
)
1277 if (key
.objectid
>= logical
+ map
->stripe_len
)
1280 if (btrfs_key_type(&key
) != BTRFS_EXTENT_ITEM_KEY
)
1283 extent
= btrfs_item_ptr(l
, slot
,
1284 struct btrfs_extent_item
);
1285 flags
= btrfs_extent_flags(l
, extent
);
1286 generation
= btrfs_extent_generation(l
, extent
);
1288 if (key
.objectid
< logical
&&
1289 (flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
)) {
1291 "btrfs scrub: tree block %llu spanning "
1292 "stripes, ignored. logical=%llu\n",
1293 (unsigned long long)key
.objectid
,
1294 (unsigned long long)logical
);
1299 * trim extent to this stripe
1301 if (key
.objectid
< logical
) {
1302 key
.offset
-= logical
- key
.objectid
;
1303 key
.objectid
= logical
;
1305 if (key
.objectid
+ key
.offset
>
1306 logical
+ map
->stripe_len
) {
1307 key
.offset
= logical
+ map
->stripe_len
-
1311 ret
= scrub_extent(sdev
, key
.objectid
, key
.offset
,
1312 key
.objectid
- logical
+ physical
,
1313 flags
, generation
, mirror_num
);
1320 btrfs_release_path(path
);
1321 logical
+= increment
;
1322 physical
+= map
->stripe_len
;
1323 spin_lock(&sdev
->stat_lock
);
1324 sdev
->stat
.last_physical
= physical
;
1325 spin_unlock(&sdev
->stat_lock
);
1327 /* push queued extents */
1331 blk_finish_plug(&plug
);
1333 btrfs_free_path(path
);
1334 return ret
< 0 ? ret
: 0;
1337 static noinline_for_stack
int scrub_chunk(struct scrub_dev
*sdev
,
1338 u64 chunk_tree
, u64 chunk_objectid
, u64 chunk_offset
, u64 length
)
1340 struct btrfs_mapping_tree
*map_tree
=
1341 &sdev
->dev
->dev_root
->fs_info
->mapping_tree
;
1342 struct map_lookup
*map
;
1343 struct extent_map
*em
;
1347 read_lock(&map_tree
->map_tree
.lock
);
1348 em
= lookup_extent_mapping(&map_tree
->map_tree
, chunk_offset
, 1);
1349 read_unlock(&map_tree
->map_tree
.lock
);
1354 map
= (struct map_lookup
*)em
->bdev
;
1355 if (em
->start
!= chunk_offset
)
1358 if (em
->len
< length
)
1361 for (i
= 0; i
< map
->num_stripes
; ++i
) {
1362 if (map
->stripes
[i
].dev
== sdev
->dev
) {
1363 ret
= scrub_stripe(sdev
, map
, i
, chunk_offset
, length
);
1369 free_extent_map(em
);
1374 static noinline_for_stack
1375 int scrub_enumerate_chunks(struct scrub_dev
*sdev
, u64 start
, u64 end
)
1377 struct btrfs_dev_extent
*dev_extent
= NULL
;
1378 struct btrfs_path
*path
;
1379 struct btrfs_root
*root
= sdev
->dev
->dev_root
;
1380 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
1387 struct extent_buffer
*l
;
1388 struct btrfs_key key
;
1389 struct btrfs_key found_key
;
1390 struct btrfs_block_group_cache
*cache
;
1392 path
= btrfs_alloc_path();
1397 path
->search_commit_root
= 1;
1398 path
->skip_locking
= 1;
1400 key
.objectid
= sdev
->dev
->devid
;
1402 key
.type
= BTRFS_DEV_EXTENT_KEY
;
1406 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
1410 if (path
->slots
[0] >=
1411 btrfs_header_nritems(path
->nodes
[0])) {
1412 ret
= btrfs_next_leaf(root
, path
);
1419 slot
= path
->slots
[0];
1421 btrfs_item_key_to_cpu(l
, &found_key
, slot
);
1423 if (found_key
.objectid
!= sdev
->dev
->devid
)
1426 if (btrfs_key_type(&found_key
) != BTRFS_DEV_EXTENT_KEY
)
1429 if (found_key
.offset
>= end
)
1432 if (found_key
.offset
< key
.offset
)
1435 dev_extent
= btrfs_item_ptr(l
, slot
, struct btrfs_dev_extent
);
1436 length
= btrfs_dev_extent_length(l
, dev_extent
);
1438 if (found_key
.offset
+ length
<= start
) {
1439 key
.offset
= found_key
.offset
+ length
;
1440 btrfs_release_path(path
);
1444 chunk_tree
= btrfs_dev_extent_chunk_tree(l
, dev_extent
);
1445 chunk_objectid
= btrfs_dev_extent_chunk_objectid(l
, dev_extent
);
1446 chunk_offset
= btrfs_dev_extent_chunk_offset(l
, dev_extent
);
1449 * get a reference on the corresponding block group to prevent
1450 * the chunk from going away while we scrub it
1452 cache
= btrfs_lookup_block_group(fs_info
, chunk_offset
);
1457 ret
= scrub_chunk(sdev
, chunk_tree
, chunk_objectid
,
1458 chunk_offset
, length
);
1459 btrfs_put_block_group(cache
);
1463 key
.offset
= found_key
.offset
+ length
;
1464 btrfs_release_path(path
);
1467 btrfs_free_path(path
);
1470 * ret can still be 1 from search_slot or next_leaf,
1471 * that's not an error
1473 return ret
< 0 ? ret
: 0;
1476 static noinline_for_stack
int scrub_supers(struct scrub_dev
*sdev
)
1482 struct btrfs_device
*device
= sdev
->dev
;
1483 struct btrfs_root
*root
= device
->dev_root
;
1485 gen
= root
->fs_info
->last_trans_committed
;
1487 for (i
= 0; i
< BTRFS_SUPER_MIRROR_MAX
; i
++) {
1488 bytenr
= btrfs_sb_offset(i
);
1489 if (bytenr
+ BTRFS_SUPER_INFO_SIZE
>= device
->total_bytes
)
1492 ret
= scrub_page(sdev
, bytenr
, PAGE_SIZE
, bytenr
,
1493 BTRFS_EXTENT_FLAG_SUPER
, gen
, i
, NULL
, 1);
1497 wait_event(sdev
->list_wait
, atomic_read(&sdev
->in_flight
) == 0);
1503 * get a reference count on fs_info->scrub_workers. start worker if necessary
1505 static noinline_for_stack
int scrub_workers_get(struct btrfs_root
*root
)
1507 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
1509 mutex_lock(&fs_info
->scrub_lock
);
1510 if (fs_info
->scrub_workers_refcnt
== 0) {
1511 btrfs_init_workers(&fs_info
->scrub_workers
, "scrub",
1512 fs_info
->thread_pool_size
, &fs_info
->generic_worker
);
1513 fs_info
->scrub_workers
.idle_thresh
= 4;
1514 btrfs_start_workers(&fs_info
->scrub_workers
, 1);
1516 ++fs_info
->scrub_workers_refcnt
;
1517 mutex_unlock(&fs_info
->scrub_lock
);
1522 static noinline_for_stack
void scrub_workers_put(struct btrfs_root
*root
)
1524 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
1526 mutex_lock(&fs_info
->scrub_lock
);
1527 if (--fs_info
->scrub_workers_refcnt
== 0)
1528 btrfs_stop_workers(&fs_info
->scrub_workers
);
1529 WARN_ON(fs_info
->scrub_workers_refcnt
< 0);
1530 mutex_unlock(&fs_info
->scrub_lock
);
1534 int btrfs_scrub_dev(struct btrfs_root
*root
, u64 devid
, u64 start
, u64 end
,
1535 struct btrfs_scrub_progress
*progress
, int readonly
)
1537 struct scrub_dev
*sdev
;
1538 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
1540 struct btrfs_device
*dev
;
1542 if (btrfs_fs_closing(root
->fs_info
))
1546 * check some assumptions
1548 if (root
->sectorsize
!= PAGE_SIZE
||
1549 root
->sectorsize
!= root
->leafsize
||
1550 root
->sectorsize
!= root
->nodesize
) {
1551 printk(KERN_ERR
"btrfs_scrub: size assumptions fail\n");
1555 ret
= scrub_workers_get(root
);
1559 mutex_lock(&root
->fs_info
->fs_devices
->device_list_mutex
);
1560 dev
= btrfs_find_device(root
, devid
, NULL
, NULL
);
1561 if (!dev
|| dev
->missing
) {
1562 mutex_unlock(&root
->fs_info
->fs_devices
->device_list_mutex
);
1563 scrub_workers_put(root
);
1566 mutex_lock(&fs_info
->scrub_lock
);
1568 if (!dev
->in_fs_metadata
) {
1569 mutex_unlock(&fs_info
->scrub_lock
);
1570 mutex_unlock(&root
->fs_info
->fs_devices
->device_list_mutex
);
1571 scrub_workers_put(root
);
1575 if (dev
->scrub_device
) {
1576 mutex_unlock(&fs_info
->scrub_lock
);
1577 mutex_unlock(&root
->fs_info
->fs_devices
->device_list_mutex
);
1578 scrub_workers_put(root
);
1579 return -EINPROGRESS
;
1581 sdev
= scrub_setup_dev(dev
);
1583 mutex_unlock(&fs_info
->scrub_lock
);
1584 mutex_unlock(&root
->fs_info
->fs_devices
->device_list_mutex
);
1585 scrub_workers_put(root
);
1586 return PTR_ERR(sdev
);
1588 sdev
->readonly
= readonly
;
1589 dev
->scrub_device
= sdev
;
1591 atomic_inc(&fs_info
->scrubs_running
);
1592 mutex_unlock(&fs_info
->scrub_lock
);
1593 mutex_unlock(&root
->fs_info
->fs_devices
->device_list_mutex
);
1595 down_read(&fs_info
->scrub_super_lock
);
1596 ret
= scrub_supers(sdev
);
1597 up_read(&fs_info
->scrub_super_lock
);
1600 ret
= scrub_enumerate_chunks(sdev
, start
, end
);
1602 wait_event(sdev
->list_wait
, atomic_read(&sdev
->in_flight
) == 0);
1603 atomic_dec(&fs_info
->scrubs_running
);
1604 wake_up(&fs_info
->scrub_pause_wait
);
1606 wait_event(sdev
->list_wait
, atomic_read(&sdev
->fixup_cnt
) == 0);
1609 memcpy(progress
, &sdev
->stat
, sizeof(*progress
));
1611 mutex_lock(&fs_info
->scrub_lock
);
1612 dev
->scrub_device
= NULL
;
1613 mutex_unlock(&fs_info
->scrub_lock
);
1615 scrub_free_dev(sdev
);
1616 scrub_workers_put(root
);
1621 int btrfs_scrub_pause(struct btrfs_root
*root
)
1623 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
1625 mutex_lock(&fs_info
->scrub_lock
);
1626 atomic_inc(&fs_info
->scrub_pause_req
);
1627 while (atomic_read(&fs_info
->scrubs_paused
) !=
1628 atomic_read(&fs_info
->scrubs_running
)) {
1629 mutex_unlock(&fs_info
->scrub_lock
);
1630 wait_event(fs_info
->scrub_pause_wait
,
1631 atomic_read(&fs_info
->scrubs_paused
) ==
1632 atomic_read(&fs_info
->scrubs_running
));
1633 mutex_lock(&fs_info
->scrub_lock
);
1635 mutex_unlock(&fs_info
->scrub_lock
);
1640 int btrfs_scrub_continue(struct btrfs_root
*root
)
1642 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
1644 atomic_dec(&fs_info
->scrub_pause_req
);
1645 wake_up(&fs_info
->scrub_pause_wait
);
1649 int btrfs_scrub_pause_super(struct btrfs_root
*root
)
1651 down_write(&root
->fs_info
->scrub_super_lock
);
1655 int btrfs_scrub_continue_super(struct btrfs_root
*root
)
1657 up_write(&root
->fs_info
->scrub_super_lock
);
1661 int btrfs_scrub_cancel(struct btrfs_root
*root
)
1663 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
1665 mutex_lock(&fs_info
->scrub_lock
);
1666 if (!atomic_read(&fs_info
->scrubs_running
)) {
1667 mutex_unlock(&fs_info
->scrub_lock
);
1671 atomic_inc(&fs_info
->scrub_cancel_req
);
1672 while (atomic_read(&fs_info
->scrubs_running
)) {
1673 mutex_unlock(&fs_info
->scrub_lock
);
1674 wait_event(fs_info
->scrub_pause_wait
,
1675 atomic_read(&fs_info
->scrubs_running
) == 0);
1676 mutex_lock(&fs_info
->scrub_lock
);
1678 atomic_dec(&fs_info
->scrub_cancel_req
);
1679 mutex_unlock(&fs_info
->scrub_lock
);
1684 int btrfs_scrub_cancel_dev(struct btrfs_root
*root
, struct btrfs_device
*dev
)
1686 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
1687 struct scrub_dev
*sdev
;
1689 mutex_lock(&fs_info
->scrub_lock
);
1690 sdev
= dev
->scrub_device
;
1692 mutex_unlock(&fs_info
->scrub_lock
);
1695 atomic_inc(&sdev
->cancel_req
);
1696 while (dev
->scrub_device
) {
1697 mutex_unlock(&fs_info
->scrub_lock
);
1698 wait_event(fs_info
->scrub_pause_wait
,
1699 dev
->scrub_device
== NULL
);
1700 mutex_lock(&fs_info
->scrub_lock
);
1702 mutex_unlock(&fs_info
->scrub_lock
);
1706 int btrfs_scrub_cancel_devid(struct btrfs_root
*root
, u64 devid
)
1708 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
1709 struct btrfs_device
*dev
;
1713 * we have to hold the device_list_mutex here so the device
1714 * does not go away in cancel_dev. FIXME: find a better solution
1716 mutex_lock(&fs_info
->fs_devices
->device_list_mutex
);
1717 dev
= btrfs_find_device(root
, devid
, NULL
, NULL
);
1719 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
1722 ret
= btrfs_scrub_cancel_dev(root
, dev
);
1723 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
1728 int btrfs_scrub_progress(struct btrfs_root
*root
, u64 devid
,
1729 struct btrfs_scrub_progress
*progress
)
1731 struct btrfs_device
*dev
;
1732 struct scrub_dev
*sdev
= NULL
;
1734 mutex_lock(&root
->fs_info
->fs_devices
->device_list_mutex
);
1735 dev
= btrfs_find_device(root
, devid
, NULL
, NULL
);
1737 sdev
= dev
->scrub_device
;
1739 memcpy(progress
, &sdev
->stat
, sizeof(*progress
));
1740 mutex_unlock(&root
->fs_info
->fs_devices
->device_list_mutex
);
1742 return dev
? (sdev
? 0 : -ENOTCONN
) : -ENODEV
;