2 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
4 * bitmap_create - sets up the bitmap structure
5 * bitmap_destroy - destroys the bitmap structure
7 * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
8 * - added disk storage for bitmap
9 * - changes to allow various bitmap chunk sizes
15 * flush after percent set rather than just time based. (maybe both).
16 * wait if count gets too high, wake when it drops to half.
17 * allow bitmap to be mirrored with superblock (before or after...)
18 * allow hot-add to re-instate a current device.
19 * allow hot-add of bitmap after quiessing device
22 #include <linux/module.h>
23 #include <linux/errno.h>
24 #include <linux/slab.h>
25 #include <linux/init.h>
26 #include <linux/config.h>
27 #include <linux/timer.h>
28 #include <linux/sched.h>
29 #include <linux/list.h>
30 #include <linux/file.h>
31 #include <linux/mount.h>
32 #include <linux/buffer_head.h>
33 #include <linux/raid/md.h>
34 #include <linux/raid/bitmap.h>
41 /* these are for debugging purposes only! */
43 /* define one and only one of these */
44 #define INJECT_FAULTS_1 0 /* cause bitmap_alloc_page to fail always */
45 #define INJECT_FAULTS_2 0 /* cause bitmap file to be kicked when first bit set*/
46 #define INJECT_FAULTS_3 0 /* treat bitmap file as kicked at init time */
47 #define INJECT_FAULTS_4 0 /* undef */
48 #define INJECT_FAULTS_5 0 /* undef */
49 #define INJECT_FAULTS_6 0
51 /* if these are defined, the driver will fail! debug only */
52 #define INJECT_FATAL_FAULT_1 0 /* fail kmalloc, causing bitmap_create to fail */
53 #define INJECT_FATAL_FAULT_2 0 /* undef */
54 #define INJECT_FATAL_FAULT_3 0 /* undef */
57 //#define DPRINTK PRINTK /* set this NULL to avoid verbose debug output */
58 #define DPRINTK(x...) do { } while(0)
62 # define PRINTK(x...) printk(KERN_DEBUG x)
68 static inline char * bmname(struct bitmap
*bitmap
)
70 return bitmap
->mddev
? mdname(bitmap
->mddev
) : "mdX";
75 * test if the bitmap is active
77 int bitmap_active(struct bitmap
*bitmap
)
84 spin_lock_irqsave(&bitmap
->lock
, flags
);
85 res
= bitmap
->flags
& BITMAP_ACTIVE
;
86 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
90 #define WRITE_POOL_SIZE 256
93 * just a placeholder - calls kmalloc for bitmap pages
95 static unsigned char *bitmap_alloc_page(struct bitmap
*bitmap
)
99 #ifdef INJECT_FAULTS_1
102 page
= kmalloc(PAGE_SIZE
, GFP_NOIO
);
105 printk("%s: bitmap_alloc_page FAILED\n", bmname(bitmap
));
107 PRINTK("%s: bitmap_alloc_page: allocated page at %p\n",
108 bmname(bitmap
), page
);
113 * for now just a placeholder -- just calls kfree for bitmap pages
115 static void bitmap_free_page(struct bitmap
*bitmap
, unsigned char *page
)
117 PRINTK("%s: bitmap_free_page: free page %p\n", bmname(bitmap
), page
);
122 * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
124 * 1) check to see if this page is allocated, if it's not then try to alloc
125 * 2) if the alloc fails, set the page's hijacked flag so we'll use the
126 * page pointer directly as a counter
128 * if we find our page, we increment the page's refcount so that it stays
129 * allocated while we're using it
131 static int bitmap_checkpage(struct bitmap
*bitmap
, unsigned long page
, int create
)
133 unsigned char *mappage
;
135 if (page
>= bitmap
->pages
) {
137 "%s: invalid bitmap page request: %lu (> %lu)\n",
138 bmname(bitmap
), page
, bitmap
->pages
-1);
143 if (bitmap
->bp
[page
].hijacked
) /* it's hijacked, don't try to alloc */
146 if (bitmap
->bp
[page
].map
) /* page is already allocated, just return */
152 spin_unlock_irq(&bitmap
->lock
);
154 /* this page has not been allocated yet */
156 if ((mappage
= bitmap_alloc_page(bitmap
)) == NULL
) {
157 PRINTK("%s: bitmap map page allocation failed, hijacking\n",
159 /* failed - set the hijacked flag so that we can use the
160 * pointer as a counter */
161 spin_lock_irq(&bitmap
->lock
);
162 if (!bitmap
->bp
[page
].map
)
163 bitmap
->bp
[page
].hijacked
= 1;
169 spin_lock_irq(&bitmap
->lock
);
171 /* recheck the page */
173 if (bitmap
->bp
[page
].map
|| bitmap
->bp
[page
].hijacked
) {
174 /* somebody beat us to getting the page */
175 bitmap_free_page(bitmap
, mappage
);
179 /* no page was in place and we have one, so install it */
181 memset(mappage
, 0, PAGE_SIZE
);
182 bitmap
->bp
[page
].map
= mappage
;
183 bitmap
->missing_pages
--;
189 /* if page is completely empty, put it back on the free list, or dealloc it */
190 /* if page was hijacked, unmark the flag so it might get alloced next time */
191 /* Note: lock should be held when calling this */
192 static void bitmap_checkfree(struct bitmap
*bitmap
, unsigned long page
)
196 if (bitmap
->bp
[page
].count
) /* page is still busy */
199 /* page is no longer in use, it can be released */
201 if (bitmap
->bp
[page
].hijacked
) { /* page was hijacked, undo this now */
202 bitmap
->bp
[page
].hijacked
= 0;
203 bitmap
->bp
[page
].map
= NULL
;
207 /* normal case, free the page */
210 /* actually ... let's not. We will probably need the page again exactly when
211 * memory is tight and we are flusing to disk
215 ptr
= bitmap
->bp
[page
].map
;
216 bitmap
->bp
[page
].map
= NULL
;
217 bitmap
->missing_pages
++;
218 bitmap_free_page(bitmap
, ptr
);
225 * bitmap file handling - read and write the bitmap file and its superblock
228 /* copy the pathname of a file to a buffer */
229 char *file_path(struct file
*file
, char *buf
, int count
)
240 buf
= d_path(d
, v
, buf
, count
);
242 return IS_ERR(buf
) ? NULL
: buf
;
246 * basic page I/O operations
249 /* IO operations when bitmap is stored near all superblocks */
250 static struct page
*read_sb_page(mddev_t
*mddev
, long offset
, unsigned long index
)
252 /* choose a good rdev and read the page from there */
255 struct list_head
*tmp
;
256 struct page
*page
= alloc_page(GFP_KERNEL
);
260 return ERR_PTR(-ENOMEM
);
262 ITERATE_RDEV(mddev
, rdev
, tmp
) {
263 if (! test_bit(In_sync
, &rdev
->flags
)
264 || test_bit(Faulty
, &rdev
->flags
))
267 target
= (rdev
->sb_offset
<< 1) + offset
+ index
* (PAGE_SIZE
/512);
269 if (sync_page_io(rdev
->bdev
, target
, PAGE_SIZE
, page
, READ
)) {
274 return ERR_PTR(-EIO
);
278 static int write_sb_page(mddev_t
*mddev
, long offset
, struct page
*page
, int wait
)
281 struct list_head
*tmp
;
283 ITERATE_RDEV(mddev
, rdev
, tmp
)
284 if (test_bit(In_sync
, &rdev
->flags
)
285 && !test_bit(Faulty
, &rdev
->flags
))
286 md_super_write(mddev
, rdev
,
287 (rdev
->sb_offset
<<1) + offset
288 + page
->index
* (PAGE_SIZE
/512),
293 md_super_wait(mddev
);
298 * write out a page to a file
300 static int write_page(struct bitmap
*bitmap
, struct page
*page
, int wait
)
304 if (bitmap
->file
== NULL
)
305 return write_sb_page(bitmap
->mddev
, bitmap
->offset
, page
, wait
);
307 flush_dcache_page(page
); /* make sure visible to anyone reading the file */
312 if (TestSetPageLocked(page
))
313 return -EAGAIN
; /* already locked */
314 if (PageWriteback(page
)) {
320 ret
= page
->mapping
->a_ops
->prepare_write(bitmap
->file
, page
, 0, PAGE_SIZE
);
322 ret
= page
->mapping
->a_ops
->commit_write(bitmap
->file
, page
, 0,
329 set_page_dirty(page
); /* force it to be written out */
332 /* add to list to be waited for */
333 struct page_list
*item
= mempool_alloc(bitmap
->write_pool
, GFP_NOIO
);
336 spin_lock(&bitmap
->write_lock
);
337 list_add(&item
->list
, &bitmap
->complete_pages
);
338 spin_unlock(&bitmap
->write_lock
);
340 return write_one_page(page
, wait
);
343 /* read a page from a file, pinning it into cache, and return bytes_read */
344 static struct page
*read_page(struct file
*file
, unsigned long index
,
345 unsigned long *bytes_read
)
347 struct inode
*inode
= file
->f_mapping
->host
;
348 struct page
*page
= NULL
;
349 loff_t isize
= i_size_read(inode
);
350 unsigned long end_index
= isize
>> PAGE_SHIFT
;
352 PRINTK("read bitmap file (%dB @ %Lu)\n", (int)PAGE_SIZE
,
353 (unsigned long long)index
<< PAGE_SHIFT
);
355 page
= read_cache_page(inode
->i_mapping
, index
,
356 (filler_t
*)inode
->i_mapping
->a_ops
->readpage
, file
);
359 wait_on_page_locked(page
);
360 if (!PageUptodate(page
) || PageError(page
)) {
362 page
= ERR_PTR(-EIO
);
366 if (index
> end_index
) /* we have read beyond EOF */
368 else if (index
== end_index
) /* possible short read */
369 *bytes_read
= isize
& ~PAGE_MASK
;
371 *bytes_read
= PAGE_SIZE
; /* got a full page */
374 printk(KERN_ALERT
"md: bitmap read error: (%dB @ %Lu): %ld\n",
376 (unsigned long long)index
<< PAGE_SHIFT
,
382 * bitmap file superblock operations
385 /* update the event counter and sync the superblock to disk */
386 int bitmap_update_sb(struct bitmap
*bitmap
)
391 if (!bitmap
|| !bitmap
->mddev
) /* no bitmap for this array */
393 spin_lock_irqsave(&bitmap
->lock
, flags
);
394 if (!bitmap
->sb_page
) { /* no superblock */
395 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
398 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
399 sb
= (bitmap_super_t
*)kmap_atomic(bitmap
->sb_page
, KM_USER0
);
400 sb
->events
= cpu_to_le64(bitmap
->mddev
->events
);
401 if (!bitmap
->mddev
->degraded
)
402 sb
->events_cleared
= cpu_to_le64(bitmap
->mddev
->events
);
403 kunmap_atomic(sb
, KM_USER0
);
404 return write_page(bitmap
, bitmap
->sb_page
, 1);
407 /* print out the bitmap file superblock */
408 void bitmap_print_sb(struct bitmap
*bitmap
)
412 if (!bitmap
|| !bitmap
->sb_page
)
414 sb
= (bitmap_super_t
*)kmap_atomic(bitmap
->sb_page
, KM_USER0
);
415 printk(KERN_DEBUG
"%s: bitmap file superblock:\n", bmname(bitmap
));
416 printk(KERN_DEBUG
" magic: %08x\n", le32_to_cpu(sb
->magic
));
417 printk(KERN_DEBUG
" version: %d\n", le32_to_cpu(sb
->version
));
418 printk(KERN_DEBUG
" uuid: %08x.%08x.%08x.%08x\n",
419 *(__u32
*)(sb
->uuid
+0),
420 *(__u32
*)(sb
->uuid
+4),
421 *(__u32
*)(sb
->uuid
+8),
422 *(__u32
*)(sb
->uuid
+12));
423 printk(KERN_DEBUG
" events: %llu\n",
424 (unsigned long long) le64_to_cpu(sb
->events
));
425 printk(KERN_DEBUG
"events cleared: %llu\n",
426 (unsigned long long) le64_to_cpu(sb
->events_cleared
));
427 printk(KERN_DEBUG
" state: %08x\n", le32_to_cpu(sb
->state
));
428 printk(KERN_DEBUG
" chunksize: %d B\n", le32_to_cpu(sb
->chunksize
));
429 printk(KERN_DEBUG
" daemon sleep: %ds\n", le32_to_cpu(sb
->daemon_sleep
));
430 printk(KERN_DEBUG
" sync size: %llu KB\n",
431 (unsigned long long)le64_to_cpu(sb
->sync_size
)/2);
432 printk(KERN_DEBUG
"max write behind: %d\n", le32_to_cpu(sb
->write_behind
));
433 kunmap_atomic(sb
, KM_USER0
);
436 /* read the superblock from the bitmap file and initialize some bitmap fields */
437 static int bitmap_read_sb(struct bitmap
*bitmap
)
441 unsigned long chunksize
, daemon_sleep
, write_behind
;
442 unsigned long bytes_read
;
443 unsigned long long events
;
446 /* page 0 is the superblock, read it... */
448 bitmap
->sb_page
= read_page(bitmap
->file
, 0, &bytes_read
);
450 bitmap
->sb_page
= read_sb_page(bitmap
->mddev
, bitmap
->offset
, 0);
451 bytes_read
= PAGE_SIZE
;
453 if (IS_ERR(bitmap
->sb_page
)) {
454 err
= PTR_ERR(bitmap
->sb_page
);
455 bitmap
->sb_page
= NULL
;
459 sb
= (bitmap_super_t
*)kmap_atomic(bitmap
->sb_page
, KM_USER0
);
461 if (bytes_read
< sizeof(*sb
)) { /* short read */
462 printk(KERN_INFO
"%s: bitmap file superblock truncated\n",
468 chunksize
= le32_to_cpu(sb
->chunksize
);
469 daemon_sleep
= le32_to_cpu(sb
->daemon_sleep
);
470 write_behind
= le32_to_cpu(sb
->write_behind
);
472 /* verify that the bitmap-specific fields are valid */
473 if (sb
->magic
!= cpu_to_le32(BITMAP_MAGIC
))
474 reason
= "bad magic";
475 else if (le32_to_cpu(sb
->version
) < BITMAP_MAJOR_LO
||
476 le32_to_cpu(sb
->version
) > BITMAP_MAJOR_HI
)
477 reason
= "unrecognized superblock version";
478 else if (chunksize
< PAGE_SIZE
)
479 reason
= "bitmap chunksize too small";
480 else if ((1 << ffz(~chunksize
)) != chunksize
)
481 reason
= "bitmap chunksize not a power of 2";
482 else if (daemon_sleep
< 1 || daemon_sleep
> MAX_SCHEDULE_TIMEOUT
/ HZ
)
483 reason
= "daemon sleep period out of range";
484 else if (write_behind
> COUNTER_MAX
)
485 reason
= "write-behind limit out of range (0 - 16383)";
487 printk(KERN_INFO
"%s: invalid bitmap file superblock: %s\n",
488 bmname(bitmap
), reason
);
492 /* keep the array size field of the bitmap superblock up to date */
493 sb
->sync_size
= cpu_to_le64(bitmap
->mddev
->resync_max_sectors
);
495 if (!bitmap
->mddev
->persistent
)
499 * if we have a persistent array superblock, compare the
500 * bitmap's UUID and event counter to the mddev's
502 if (memcmp(sb
->uuid
, bitmap
->mddev
->uuid
, 16)) {
503 printk(KERN_INFO
"%s: bitmap superblock UUID mismatch\n",
507 events
= le64_to_cpu(sb
->events
);
508 if (events
< bitmap
->mddev
->events
) {
509 printk(KERN_INFO
"%s: bitmap file is out of date (%llu < %llu) "
510 "-- forcing full recovery\n", bmname(bitmap
), events
,
511 (unsigned long long) bitmap
->mddev
->events
);
512 sb
->state
|= BITMAP_STALE
;
515 /* assign fields using values from superblock */
516 bitmap
->chunksize
= chunksize
;
517 bitmap
->daemon_sleep
= daemon_sleep
;
518 bitmap
->daemon_lastrun
= jiffies
;
519 bitmap
->max_write_behind
= write_behind
;
520 bitmap
->flags
|= sb
->state
;
521 if (le32_to_cpu(sb
->version
) == BITMAP_MAJOR_HOSTENDIAN
)
522 bitmap
->flags
|= BITMAP_HOSTENDIAN
;
523 bitmap
->events_cleared
= le64_to_cpu(sb
->events_cleared
);
524 if (sb
->state
& BITMAP_STALE
)
525 bitmap
->events_cleared
= bitmap
->mddev
->events
;
528 kunmap_atomic(sb
, KM_USER0
);
530 bitmap_print_sb(bitmap
);
534 enum bitmap_mask_op
{
539 /* record the state of the bitmap in the superblock */
540 static void bitmap_mask_state(struct bitmap
*bitmap
, enum bitmap_state bits
,
541 enum bitmap_mask_op op
)
546 spin_lock_irqsave(&bitmap
->lock
, flags
);
547 if (!bitmap
->sb_page
) { /* can't set the state */
548 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
551 get_page(bitmap
->sb_page
);
552 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
553 sb
= (bitmap_super_t
*)kmap_atomic(bitmap
->sb_page
, KM_USER0
);
555 case MASK_SET
: sb
->state
|= bits
;
557 case MASK_UNSET
: sb
->state
&= ~bits
;
561 kunmap_atomic(sb
, KM_USER0
);
562 put_page(bitmap
->sb_page
);
566 * general bitmap file operations
569 /* calculate the index of the page that contains this bit */
570 static inline unsigned long file_page_index(unsigned long chunk
)
572 return CHUNK_BIT_OFFSET(chunk
) >> PAGE_BIT_SHIFT
;
575 /* calculate the (bit) offset of this bit within a page */
576 static inline unsigned long file_page_offset(unsigned long chunk
)
578 return CHUNK_BIT_OFFSET(chunk
) & (PAGE_BITS
- 1);
582 * return a pointer to the page in the filemap that contains the given bit
584 * this lookup is complicated by the fact that the bitmap sb might be exactly
585 * 1 page (e.g., x86) or less than 1 page -- so the bitmap might start on page
588 static inline struct page
*filemap_get_page(struct bitmap
*bitmap
,
591 return bitmap
->filemap
[file_page_index(chunk
) - file_page_index(0)];
595 static void bitmap_file_unmap(struct bitmap
*bitmap
)
597 struct page
**map
, *sb_page
;
602 spin_lock_irqsave(&bitmap
->lock
, flags
);
603 map
= bitmap
->filemap
;
604 bitmap
->filemap
= NULL
;
605 attr
= bitmap
->filemap_attr
;
606 bitmap
->filemap_attr
= NULL
;
607 pages
= bitmap
->file_pages
;
608 bitmap
->file_pages
= 0;
609 sb_page
= bitmap
->sb_page
;
610 bitmap
->sb_page
= NULL
;
611 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
614 if (map
[pages
]->index
!= 0) /* 0 is sb_page, release it below */
615 put_page(map
[pages
]);
619 safe_put_page(sb_page
);
622 /* dequeue the next item in a page list -- don't call from irq context */
623 static struct page_list
*dequeue_page(struct bitmap
*bitmap
)
625 struct page_list
*item
= NULL
;
626 struct list_head
*head
= &bitmap
->complete_pages
;
628 spin_lock(&bitmap
->write_lock
);
629 if (list_empty(head
))
631 item
= list_entry(head
->prev
, struct page_list
, list
);
632 list_del(head
->prev
);
634 spin_unlock(&bitmap
->write_lock
);
638 static void drain_write_queues(struct bitmap
*bitmap
)
640 struct page_list
*item
;
642 while ((item
= dequeue_page(bitmap
))) {
643 /* don't bother to wait */
644 put_page(item
->page
);
645 mempool_free(item
, bitmap
->write_pool
);
649 static void bitmap_file_put(struct bitmap
*bitmap
)
655 spin_lock_irqsave(&bitmap
->lock
, flags
);
658 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
660 drain_write_queues(bitmap
);
662 bitmap_file_unmap(bitmap
);
665 inode
= file
->f_mapping
->host
;
666 spin_lock(&inode
->i_lock
);
667 atomic_set(&inode
->i_writecount
, 1); /* allow writes again */
668 spin_unlock(&inode
->i_lock
);
675 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
676 * then it is no longer reliable, so we stop using it and we mark the file
677 * as failed in the superblock
679 static void bitmap_file_kick(struct bitmap
*bitmap
)
681 char *path
, *ptr
= NULL
;
683 bitmap_mask_state(bitmap
, BITMAP_STALE
, MASK_SET
);
684 bitmap_update_sb(bitmap
);
687 path
= kmalloc(PAGE_SIZE
, GFP_KERNEL
);
689 ptr
= file_path(bitmap
->file
, path
, PAGE_SIZE
);
691 printk(KERN_ALERT
"%s: kicking failed bitmap file %s from array!\n",
692 bmname(bitmap
), ptr
? ptr
: "");
697 bitmap_file_put(bitmap
);
702 enum bitmap_page_attr
{
703 BITMAP_PAGE_DIRTY
= 0, // there are set bits that need to be synced
704 BITMAP_PAGE_CLEAN
= 1, // there are bits that might need to be cleared
705 BITMAP_PAGE_NEEDWRITE
=2, // there are cleared bits that need to be synced
708 static inline void set_page_attr(struct bitmap
*bitmap
, struct page
*page
,
709 enum bitmap_page_attr attr
)
711 __set_bit((page
->index
<<2) + attr
, bitmap
->filemap_attr
);
714 static inline void clear_page_attr(struct bitmap
*bitmap
, struct page
*page
,
715 enum bitmap_page_attr attr
)
717 __clear_bit((page
->index
<<2) + attr
, bitmap
->filemap_attr
);
720 static inline unsigned long test_page_attr(struct bitmap
*bitmap
, struct page
*page
,
721 enum bitmap_page_attr attr
)
723 return test_bit((page
->index
<<2) + attr
, bitmap
->filemap_attr
);
727 * bitmap_file_set_bit -- called before performing a write to the md device
728 * to set (and eventually sync) a particular bit in the bitmap file
730 * we set the bit immediately, then we record the page number so that
731 * when an unplug occurs, we can flush the dirty pages out to disk
733 static void bitmap_file_set_bit(struct bitmap
*bitmap
, sector_t block
)
738 unsigned long chunk
= block
>> CHUNK_BLOCK_SHIFT(bitmap
);
740 if (!bitmap
->filemap
) {
744 page
= filemap_get_page(bitmap
, chunk
);
745 bit
= file_page_offset(chunk
);
748 /* make sure the page stays cached until it gets written out */
749 if (! test_page_attr(bitmap
, page
, BITMAP_PAGE_DIRTY
))
753 kaddr
= kmap_atomic(page
, KM_USER0
);
754 if (bitmap
->flags
& BITMAP_HOSTENDIAN
)
757 ext2_set_bit(bit
, kaddr
);
758 kunmap_atomic(kaddr
, KM_USER0
);
759 PRINTK("set file bit %lu page %lu\n", bit
, page
->index
);
761 /* record page number so it gets flushed to disk when unplug occurs */
762 set_page_attr(bitmap
, page
, BITMAP_PAGE_DIRTY
);
766 static void bitmap_writeback(struct bitmap
*bitmap
);
768 /* this gets called when the md device is ready to unplug its underlying
769 * (slave) device queues -- before we let any writes go down, we need to
770 * sync the dirty pages of the bitmap file to disk */
771 int bitmap_unplug(struct bitmap
*bitmap
)
773 unsigned long i
, flags
;
774 int dirty
, need_write
;
782 /* look at each page to see if there are any set bits that need to be
783 * flushed out to disk */
784 for (i
= 0; i
< bitmap
->file_pages
; i
++) {
785 spin_lock_irqsave(&bitmap
->lock
, flags
);
786 if (!bitmap
->filemap
) {
787 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
790 page
= bitmap
->filemap
[i
];
791 dirty
= test_page_attr(bitmap
, page
, BITMAP_PAGE_DIRTY
);
792 need_write
= test_page_attr(bitmap
, page
, BITMAP_PAGE_NEEDWRITE
);
793 clear_page_attr(bitmap
, page
, BITMAP_PAGE_DIRTY
);
794 clear_page_attr(bitmap
, page
, BITMAP_PAGE_NEEDWRITE
);
797 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
799 if (dirty
| need_write
) {
800 err
= write_page(bitmap
, page
, 0);
801 if (err
== -EAGAIN
) {
803 err
= write_page(bitmap
, page
, 1);
811 if (wait
) { /* if any writes were performed, we need to wait on them */
813 bitmap_writeback(bitmap
);
815 md_super_wait(bitmap
->mddev
);
820 static void bitmap_set_memory_bits(struct bitmap
*bitmap
, sector_t offset
, int needed
);
821 /* * bitmap_init_from_disk -- called at bitmap_create time to initialize
822 * the in-memory bitmap from the on-disk bitmap -- also, sets up the
823 * memory mapping of the bitmap file
825 * if there's no bitmap file, or if the bitmap file had been
826 * previously kicked from the array, we mark all the bits as
827 * 1's in order to cause a full resync.
829 * We ignore all bits for sectors that end earlier than 'start'.
830 * This is used when reading an out-of-date bitmap...
832 static int bitmap_init_from_disk(struct bitmap
*bitmap
, sector_t start
)
834 unsigned long i
, chunks
, index
, oldindex
, bit
;
835 struct page
*page
= NULL
, *oldpage
= NULL
;
836 unsigned long num_pages
, bit_cnt
= 0;
838 unsigned long bytes
, offset
, dummy
;
843 chunks
= bitmap
->chunks
;
846 BUG_ON(!file
&& !bitmap
->offset
);
848 #ifdef INJECT_FAULTS_3
851 outofdate
= bitmap
->flags
& BITMAP_STALE
;
854 printk(KERN_INFO
"%s: bitmap file is out of date, doing full "
855 "recovery\n", bmname(bitmap
));
857 bytes
= (chunks
+ 7) / 8;
859 num_pages
= (bytes
+ sizeof(bitmap_super_t
) + PAGE_SIZE
- 1) / PAGE_SIZE
;
861 if (file
&& i_size_read(file
->f_mapping
->host
) < bytes
+ sizeof(bitmap_super_t
)) {
862 printk(KERN_INFO
"%s: bitmap file too short %lu < %lu\n",
864 (unsigned long) i_size_read(file
->f_mapping
->host
),
865 bytes
+ sizeof(bitmap_super_t
));
871 bitmap
->filemap
= kmalloc(sizeof(struct page
*) * num_pages
, GFP_KERNEL
);
872 if (!bitmap
->filemap
)
875 /* We need 4 bits per page, rounded up to a multiple of sizeof(unsigned long) */
876 bitmap
->filemap_attr
= kzalloc(
877 (((num_pages
*4/8)+sizeof(unsigned long)-1)
878 /sizeof(unsigned long))
879 *sizeof(unsigned long),
881 if (!bitmap
->filemap_attr
)
886 for (i
= 0; i
< chunks
; i
++) {
888 index
= file_page_index(i
);
889 bit
= file_page_offset(i
);
890 if (index
!= oldindex
) { /* this is a new page, read it in */
891 /* unmap the old page, we're done with it */
894 * if we're here then the superblock page
895 * contains some bits (PAGE_SIZE != sizeof sb)
896 * we've already read it in, so just use it
898 page
= bitmap
->sb_page
;
899 offset
= sizeof(bitmap_super_t
);
901 page
= read_page(file
, index
, &dummy
);
904 page
= read_sb_page(bitmap
->mddev
, bitmap
->offset
, index
);
907 if (IS_ERR(page
)) { /* read error */
917 * if bitmap is out of date, dirty the
918 * whole page and write it out
920 paddr
= kmap_atomic(page
, KM_USER0
);
921 memset(paddr
+ offset
, 0xff,
923 kunmap_atomic(paddr
, KM_USER0
);
924 ret
= write_page(bitmap
, page
, 1);
926 /* release, page not in filemap yet */
932 bitmap
->filemap
[bitmap
->file_pages
++] = page
;
934 paddr
= kmap_atomic(page
, KM_USER0
);
935 if (bitmap
->flags
& BITMAP_HOSTENDIAN
)
936 b
= test_bit(bit
, paddr
);
938 b
= ext2_test_bit(bit
, paddr
);
939 kunmap_atomic(paddr
, KM_USER0
);
941 /* if the disk bit is set, set the memory bit */
942 bitmap_set_memory_bits(bitmap
, i
<< CHUNK_BLOCK_SHIFT(bitmap
),
943 ((i
+1) << (CHUNK_BLOCK_SHIFT(bitmap
)) >= start
)
946 set_page_attr(bitmap
, page
, BITMAP_PAGE_CLEAN
);
950 /* everything went OK */
952 bitmap_mask_state(bitmap
, BITMAP_STALE
, MASK_UNSET
);
954 if (bit_cnt
) { /* Kick recovery if any bits were set */
955 set_bit(MD_RECOVERY_NEEDED
, &bitmap
->mddev
->recovery
);
956 md_wakeup_thread(bitmap
->mddev
->thread
);
960 printk(KERN_INFO
"%s: bitmap initialized from disk: "
961 "read %lu/%lu pages, set %lu bits, status: %d\n",
962 bmname(bitmap
), bitmap
->file_pages
, num_pages
, bit_cnt
, ret
);
967 void bitmap_write_all(struct bitmap
*bitmap
)
969 /* We don't actually write all bitmap blocks here,
970 * just flag them as needing to be written
974 for (i
=0; i
< bitmap
->file_pages
; i
++)
975 set_page_attr(bitmap
, bitmap
->filemap
[i
],
976 BITMAP_PAGE_NEEDWRITE
);
980 static void bitmap_count_page(struct bitmap
*bitmap
, sector_t offset
, int inc
)
982 sector_t chunk
= offset
>> CHUNK_BLOCK_SHIFT(bitmap
);
983 unsigned long page
= chunk
>> PAGE_COUNTER_SHIFT
;
984 bitmap
->bp
[page
].count
+= inc
;
986 if (page == 0) printk("count page 0, offset %llu: %d gives %d\n",
987 (unsigned long long)offset, inc, bitmap->bp[page].count);
989 bitmap_checkfree(bitmap
, page
);
991 static bitmap_counter_t
*bitmap_get_counter(struct bitmap
*bitmap
,
992 sector_t offset
, int *blocks
,
996 * bitmap daemon -- periodically wakes up to clean bits and flush pages
1000 int bitmap_daemon_work(struct bitmap
*bitmap
)
1003 unsigned long flags
;
1004 struct page
*page
= NULL
, *lastpage
= NULL
;
1011 if (time_before(jiffies
, bitmap
->daemon_lastrun
+ bitmap
->daemon_sleep
*HZ
))
1013 bitmap
->daemon_lastrun
= jiffies
;
1015 for (j
= 0; j
< bitmap
->chunks
; j
++) {
1016 bitmap_counter_t
*bmc
;
1017 spin_lock_irqsave(&bitmap
->lock
, flags
);
1018 if (!bitmap
->filemap
) {
1019 /* error or shutdown */
1020 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1024 page
= filemap_get_page(bitmap
, j
);
1026 if (page
!= lastpage
) {
1027 /* skip this page unless it's marked as needing cleaning */
1028 if (!test_page_attr(bitmap
, page
, BITMAP_PAGE_CLEAN
)) {
1029 int need_write
= test_page_attr(bitmap
, page
,
1030 BITMAP_PAGE_NEEDWRITE
);
1033 clear_page_attr(bitmap
, page
, BITMAP_PAGE_NEEDWRITE
);
1035 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1037 switch (write_page(bitmap
, page
, 0)) {
1039 set_page_attr(bitmap
, page
, BITMAP_PAGE_NEEDWRITE
);
1044 bitmap_file_kick(bitmap
);
1051 /* grab the new page, sync and release the old */
1053 if (lastpage
!= NULL
) {
1054 if (test_page_attr(bitmap
, lastpage
, BITMAP_PAGE_NEEDWRITE
)) {
1055 clear_page_attr(bitmap
, lastpage
, BITMAP_PAGE_NEEDWRITE
);
1056 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1057 err
= write_page(bitmap
, lastpage
, 0);
1058 if (err
== -EAGAIN
) {
1060 set_page_attr(bitmap
, lastpage
, BITMAP_PAGE_NEEDWRITE
);
1063 set_page_attr(bitmap
, lastpage
, BITMAP_PAGE_NEEDWRITE
);
1064 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1068 bitmap_file_kick(bitmap
);
1070 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1073 printk("bitmap clean at page %lu\n", j);
1075 spin_lock_irqsave(&bitmap
->lock
, flags
);
1076 clear_page_attr(bitmap
, page
, BITMAP_PAGE_CLEAN
);
1078 bmc
= bitmap_get_counter(bitmap
, j
<< CHUNK_BLOCK_SHIFT(bitmap
),
1082 if (j < 100) printk("bitmap: j=%lu, *bmc = 0x%x\n", j, *bmc);
1085 *bmc
=1; /* maybe clear the bit next time */
1086 set_page_attr(bitmap
, page
, BITMAP_PAGE_CLEAN
);
1087 } else if (*bmc
== 1) {
1088 /* we can clear the bit */
1090 bitmap_count_page(bitmap
, j
<< CHUNK_BLOCK_SHIFT(bitmap
),
1094 paddr
= kmap_atomic(page
, KM_USER0
);
1095 if (bitmap
->flags
& BITMAP_HOSTENDIAN
)
1096 clear_bit(file_page_offset(j
), paddr
);
1098 ext2_clear_bit(file_page_offset(j
), paddr
);
1099 kunmap_atomic(paddr
, KM_USER0
);
1102 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1105 /* now sync the final page */
1106 if (lastpage
!= NULL
) {
1107 spin_lock_irqsave(&bitmap
->lock
, flags
);
1108 if (test_page_attr(bitmap
, lastpage
, BITMAP_PAGE_NEEDWRITE
)) {
1109 clear_page_attr(bitmap
, lastpage
, BITMAP_PAGE_NEEDWRITE
);
1110 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1111 err
= write_page(bitmap
, lastpage
, 0);
1112 if (err
== -EAGAIN
) {
1113 set_page_attr(bitmap
, lastpage
, BITMAP_PAGE_NEEDWRITE
);
1117 set_page_attr(bitmap
, lastpage
, BITMAP_PAGE_NEEDWRITE
);
1118 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1127 static void bitmap_writeback(struct bitmap
*bitmap
)
1130 struct page_list
*item
;
1133 PRINTK("%s: bitmap writeback daemon woke up...\n", bmname(bitmap
));
1134 /* wait on bitmap page writebacks */
1135 while ((item
= dequeue_page(bitmap
))) {
1137 mempool_free(item
, bitmap
->write_pool
);
1138 PRINTK("wait on page writeback: %p\n", page
);
1139 wait_on_page_writeback(page
);
1140 PRINTK("finished page writeback: %p\n", page
);
1142 err
= PageError(page
);
1145 printk(KERN_WARNING
"%s: bitmap file writeback "
1146 "failed (page %lu): %d\n",
1147 bmname(bitmap
), page
->index
, err
);
1148 bitmap_file_kick(bitmap
);
1154 static bitmap_counter_t
*bitmap_get_counter(struct bitmap
*bitmap
,
1155 sector_t offset
, int *blocks
,
1158 /* If 'create', we might release the lock and reclaim it.
1159 * The lock must have been taken with interrupts enabled.
1160 * If !create, we don't release the lock.
1162 sector_t chunk
= offset
>> CHUNK_BLOCK_SHIFT(bitmap
);
1163 unsigned long page
= chunk
>> PAGE_COUNTER_SHIFT
;
1164 unsigned long pageoff
= (chunk
& PAGE_COUNTER_MASK
) << COUNTER_BYTE_SHIFT
;
1167 if (bitmap_checkpage(bitmap
, page
, create
) < 0) {
1168 csize
= ((sector_t
)1) << (CHUNK_BLOCK_SHIFT(bitmap
));
1169 *blocks
= csize
- (offset
& (csize
- 1));
1172 /* now locked ... */
1174 if (bitmap
->bp
[page
].hijacked
) { /* hijacked pointer */
1175 /* should we use the first or second counter field
1176 * of the hijacked pointer? */
1177 int hi
= (pageoff
> PAGE_COUNTER_MASK
);
1178 csize
= ((sector_t
)1) << (CHUNK_BLOCK_SHIFT(bitmap
) +
1179 PAGE_COUNTER_SHIFT
- 1);
1180 *blocks
= csize
- (offset
& (csize
- 1));
1181 return &((bitmap_counter_t
*)
1182 &bitmap
->bp
[page
].map
)[hi
];
1183 } else { /* page is allocated */
1184 csize
= ((sector_t
)1) << (CHUNK_BLOCK_SHIFT(bitmap
));
1185 *blocks
= csize
- (offset
& (csize
- 1));
1186 return (bitmap_counter_t
*)
1187 &(bitmap
->bp
[page
].map
[pageoff
]);
1191 int bitmap_startwrite(struct bitmap
*bitmap
, sector_t offset
, unsigned long sectors
, int behind
)
1193 if (!bitmap
) return 0;
1196 atomic_inc(&bitmap
->behind_writes
);
1197 PRINTK(KERN_DEBUG
"inc write-behind count %d/%d\n",
1198 atomic_read(&bitmap
->behind_writes
), bitmap
->max_write_behind
);
1203 bitmap_counter_t
*bmc
;
1205 spin_lock_irq(&bitmap
->lock
);
1206 bmc
= bitmap_get_counter(bitmap
, offset
, &blocks
, 1);
1208 spin_unlock_irq(&bitmap
->lock
);
1214 bitmap_file_set_bit(bitmap
, offset
);
1215 bitmap_count_page(bitmap
,offset
, 1);
1216 blk_plug_device(bitmap
->mddev
->queue
);
1221 BUG_ON((*bmc
& COUNTER_MAX
) == COUNTER_MAX
);
1224 spin_unlock_irq(&bitmap
->lock
);
1227 if (sectors
> blocks
)
1234 void bitmap_endwrite(struct bitmap
*bitmap
, sector_t offset
, unsigned long sectors
,
1235 int success
, int behind
)
1237 if (!bitmap
) return;
1239 atomic_dec(&bitmap
->behind_writes
);
1240 PRINTK(KERN_DEBUG
"dec write-behind count %d/%d\n",
1241 atomic_read(&bitmap
->behind_writes
), bitmap
->max_write_behind
);
1246 unsigned long flags
;
1247 bitmap_counter_t
*bmc
;
1249 spin_lock_irqsave(&bitmap
->lock
, flags
);
1250 bmc
= bitmap_get_counter(bitmap
, offset
, &blocks
, 0);
1252 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1256 if (!success
&& ! (*bmc
& NEEDED_MASK
))
1257 *bmc
|= NEEDED_MASK
;
1261 set_page_attr(bitmap
,
1262 filemap_get_page(bitmap
, offset
>> CHUNK_BLOCK_SHIFT(bitmap
)),
1265 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1267 if (sectors
> blocks
)
1273 int bitmap_start_sync(struct bitmap
*bitmap
, sector_t offset
, int *blocks
,
1276 bitmap_counter_t
*bmc
;
1278 if (bitmap
== NULL
) {/* FIXME or bitmap set as 'failed' */
1280 return 1; /* always resync if no bitmap */
1282 spin_lock_irq(&bitmap
->lock
);
1283 bmc
= bitmap_get_counter(bitmap
, offset
, blocks
, 0);
1289 else if (NEEDED(*bmc
)) {
1291 if (!degraded
) { /* don't set/clear bits if degraded */
1292 *bmc
|= RESYNC_MASK
;
1293 *bmc
&= ~NEEDED_MASK
;
1297 spin_unlock_irq(&bitmap
->lock
);
1301 void bitmap_end_sync(struct bitmap
*bitmap
, sector_t offset
, int *blocks
, int aborted
)
1303 bitmap_counter_t
*bmc
;
1304 unsigned long flags
;
1306 if (offset == 0) printk("bitmap_end_sync 0 (%d)\n", aborted);
1307 */ if (bitmap
== NULL
) {
1311 spin_lock_irqsave(&bitmap
->lock
, flags
);
1312 bmc
= bitmap_get_counter(bitmap
, offset
, blocks
, 0);
1317 if (offset == 0) printk("bitmap_end sync found 0x%x, blocks %d\n", *bmc, *blocks);
1320 *bmc
&= ~RESYNC_MASK
;
1322 if (!NEEDED(*bmc
) && aborted
)
1323 *bmc
|= NEEDED_MASK
;
1326 set_page_attr(bitmap
,
1327 filemap_get_page(bitmap
, offset
>> CHUNK_BLOCK_SHIFT(bitmap
)),
1333 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1336 void bitmap_close_sync(struct bitmap
*bitmap
)
1338 /* Sync has finished, and any bitmap chunks that weren't synced
1339 * properly have been aborted. It remains to us to clear the
1340 * RESYNC bit wherever it is still on
1342 sector_t sector
= 0;
1344 if (!bitmap
) return;
1345 while (sector
< bitmap
->mddev
->resync_max_sectors
) {
1346 bitmap_end_sync(bitmap
, sector
, &blocks
, 0);
1348 if (sector < 500) printk("bitmap_close_sync: sec %llu blks %d\n",
1349 (unsigned long long)sector, blocks);
1350 */ sector
+= blocks
;
1354 static void bitmap_set_memory_bits(struct bitmap
*bitmap
, sector_t offset
, int needed
)
1356 /* For each chunk covered by any of these sectors, set the
1357 * counter to 1 and set resync_needed. They should all
1358 * be 0 at this point
1362 bitmap_counter_t
*bmc
;
1363 spin_lock_irq(&bitmap
->lock
);
1364 bmc
= bitmap_get_counter(bitmap
, offset
, &secs
, 1);
1366 spin_unlock_irq(&bitmap
->lock
);
1371 *bmc
= 1 | (needed
?NEEDED_MASK
:0);
1372 bitmap_count_page(bitmap
, offset
, 1);
1373 page
= filemap_get_page(bitmap
, offset
>> CHUNK_BLOCK_SHIFT(bitmap
));
1374 set_page_attr(bitmap
, page
, BITMAP_PAGE_CLEAN
);
1376 spin_unlock_irq(&bitmap
->lock
);
1381 * flush out any pending updates
1383 void bitmap_flush(mddev_t
*mddev
)
1385 struct bitmap
*bitmap
= mddev
->bitmap
;
1388 if (!bitmap
) /* there was no bitmap */
1391 /* run the daemon_work three time to ensure everything is flushed
1394 sleep
= bitmap
->daemon_sleep
;
1395 bitmap
->daemon_sleep
= 0;
1396 bitmap_daemon_work(bitmap
);
1397 bitmap_daemon_work(bitmap
);
1398 bitmap_daemon_work(bitmap
);
1399 bitmap
->daemon_sleep
= sleep
;
1400 bitmap_update_sb(bitmap
);
1404 * free memory that was allocated
1406 static void bitmap_free(struct bitmap
*bitmap
)
1408 unsigned long k
, pages
;
1409 struct bitmap_page
*bp
;
1411 if (!bitmap
) /* there was no bitmap */
1414 /* release the bitmap file and kill the daemon */
1415 bitmap_file_put(bitmap
);
1418 pages
= bitmap
->pages
;
1420 /* free all allocated memory */
1422 mempool_destroy(bitmap
->write_pool
);
1424 if (bp
) /* deallocate the page memory */
1425 for (k
= 0; k
< pages
; k
++)
1426 if (bp
[k
].map
&& !bp
[k
].hijacked
)
1431 void bitmap_destroy(mddev_t
*mddev
)
1433 struct bitmap
*bitmap
= mddev
->bitmap
;
1435 if (!bitmap
) /* there was no bitmap */
1438 mddev
->bitmap
= NULL
; /* disconnect from the md device */
1440 mddev
->thread
->timeout
= MAX_SCHEDULE_TIMEOUT
;
1442 bitmap_free(bitmap
);
1446 * initialize the bitmap structure
1447 * if this returns an error, bitmap_destroy must be called to do clean up
1449 int bitmap_create(mddev_t
*mddev
)
1451 struct bitmap
*bitmap
;
1452 unsigned long blocks
= mddev
->resync_max_sectors
;
1453 unsigned long chunks
;
1454 unsigned long pages
;
1455 struct file
*file
= mddev
->bitmap_file
;
1459 BUG_ON(sizeof(bitmap_super_t
) != 256);
1461 if (!file
&& !mddev
->bitmap_offset
) /* bitmap disabled, nothing to do */
1464 BUG_ON(file
&& mddev
->bitmap_offset
);
1466 bitmap
= kzalloc(sizeof(*bitmap
), GFP_KERNEL
);
1470 spin_lock_init(&bitmap
->lock
);
1471 bitmap
->mddev
= mddev
;
1473 spin_lock_init(&bitmap
->write_lock
);
1474 INIT_LIST_HEAD(&bitmap
->complete_pages
);
1475 bitmap
->write_pool
= mempool_create_kmalloc_pool(WRITE_POOL_SIZE
,
1476 sizeof(struct page_list
));
1478 if (!bitmap
->write_pool
)
1481 bitmap
->file
= file
;
1482 bitmap
->offset
= mddev
->bitmap_offset
;
1483 if (file
) get_file(file
);
1484 /* read superblock from bitmap file (this sets bitmap->chunksize) */
1485 err
= bitmap_read_sb(bitmap
);
1489 bitmap
->chunkshift
= find_first_bit(&bitmap
->chunksize
,
1490 sizeof(bitmap
->chunksize
));
1492 /* now that chunksize and chunkshift are set, we can use these macros */
1493 chunks
= (blocks
+ CHUNK_BLOCK_RATIO(bitmap
) - 1) /
1494 CHUNK_BLOCK_RATIO(bitmap
);
1495 pages
= (chunks
+ PAGE_COUNTER_RATIO
- 1) / PAGE_COUNTER_RATIO
;
1499 bitmap
->chunks
= chunks
;
1500 bitmap
->pages
= pages
;
1501 bitmap
->missing_pages
= pages
;
1502 bitmap
->counter_bits
= COUNTER_BITS
;
1504 bitmap
->syncchunk
= ~0UL;
1506 #ifdef INJECT_FATAL_FAULT_1
1509 bitmap
->bp
= kzalloc(pages
* sizeof(*bitmap
->bp
), GFP_KERNEL
);
1515 bitmap
->flags
|= BITMAP_ACTIVE
;
1517 /* now that we have some pages available, initialize the in-memory
1518 * bitmap from the on-disk bitmap */
1520 if (mddev
->degraded
== 0
1521 || bitmap
->events_cleared
== mddev
->events
)
1522 /* no need to keep dirty bits to optimise a re-add of a missing device */
1523 start
= mddev
->recovery_cp
;
1524 err
= bitmap_init_from_disk(bitmap
, start
);
1529 printk(KERN_INFO
"created bitmap (%lu pages) for device %s\n",
1530 pages
, bmname(bitmap
));
1532 mddev
->bitmap
= bitmap
;
1534 mddev
->thread
->timeout
= bitmap
->daemon_sleep
* HZ
;
1536 return bitmap_update_sb(bitmap
);
1539 bitmap_free(bitmap
);
1543 /* the bitmap API -- for raid personalities */
1544 EXPORT_SYMBOL(bitmap_startwrite
);
1545 EXPORT_SYMBOL(bitmap_endwrite
);
1546 EXPORT_SYMBOL(bitmap_start_sync
);
1547 EXPORT_SYMBOL(bitmap_end_sync
);
1548 EXPORT_SYMBOL(bitmap_unplug
);
1549 EXPORT_SYMBOL(bitmap_close_sync
);
1550 EXPORT_SYMBOL(bitmap_daemon_work
);