?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[PATCH] md: fix --re-add for raid1 and raid6
[linux-2.6/zen-sources.git] / drivers / md / raid1.c
blob3066c587b5391202dda91f8e4a9bdc977a4b335a
1 /*
2 * raid1.c : Multiple Devices driver for Linux
3 *
4 * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat
5 *
6 * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman
7 *
8 * RAID-1 management functions.
9 *
10 * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000
11 *
12 * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk>
13 * Various fixes by Neil Brown <neilb@cse.unsw.edu.au>
14 *
15 * Changes by Peter T. Breuer <ptb@it.uc3m.es> 31/1/2003 to support
16 * bitmapped intelligence in resync:
17 *
18 * - bitmap marked during normal i/o
19 * - bitmap used to skip nondirty blocks during sync
20 *
21 * Additions to bitmap code, (C) 2003-2004 Paul Clements, SteelEye Technology:
22 * - persistent bitmap code
23 *
24 * This program is free software; you can redistribute it and/or modify
25 * it under the terms of the GNU General Public License as published by
26 * the Free Software Foundation; either version 2, or (at your option)
27 * any later version.
28 *
29 * You should have received a copy of the GNU General Public License
30 * (for example /usr/src/linux/COPYING); if not, write to the Free
31 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
32 */
33
34 #include "dm-bio-list.h"
35 #include <linux/raid/raid1.h>
36 #include <linux/raid/bitmap.h>
37
38 #define DEBUG 0
39 #if DEBUG
40 #define PRINTK(x...) printk(x)
41 #else
42 #define PRINTK(x...)
43 #endif
44
45 /*
46 * Number of guaranteed r1bios in case of extreme VM load:
47 */
48 #define NR_RAID1_BIOS 256
49
50 static mdk_personality_t raid1_personality;
51
52 static void unplug_slaves(mddev_t *mddev);
53
54
55 static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data)
56 {
57 struct pool_info *pi = data;
58 r1bio_t *r1_bio;
59 int size = offsetof(r1bio_t, bios[pi->raid_disks]);
60
61 /* allocate a r1bio with room for raid_disks entries in the bios array */
62 r1_bio = kmalloc(size, gfp_flags);
63 if (r1_bio)
64 memset(r1_bio, 0, size);
65 else
66 unplug_slaves(pi->mddev);
67
68 return r1_bio;
69 }
70
71 static void r1bio_pool_free(void *r1_bio, void *data)
72 {
73 kfree(r1_bio);
74 }
75
76 #define RESYNC_BLOCK_SIZE (64*1024)
77 //#define RESYNC_BLOCK_SIZE PAGE_SIZE
78 #define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
79 #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
80 #define RESYNC_WINDOW (2048*1024)
81
82 static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data)
83 {
84 struct pool_info *pi = data;
85 struct page *page;
86 r1bio_t *r1_bio;
87 struct bio *bio;
88 int i, j;
89
90 r1_bio = r1bio_pool_alloc(gfp_flags, pi);
91 if (!r1_bio) {
92 unplug_slaves(pi->mddev);
93 return NULL;
94 }
95
96 /*
97 * Allocate bios : 1 for reading, n-1 for writing
98 */
99 for (j = pi->raid_disks ; j-- ; ) {
100 bio = bio_alloc(gfp_flags, RESYNC_PAGES);
101 if (!bio)
102 goto out_free_bio;
103 r1_bio->bios[j] = bio;
104 }
105 /*
106 * Allocate RESYNC_PAGES data pages and attach them to
107 * the first bio;
108 */
109 bio = r1_bio->bios[0];
110 for (i = 0; i < RESYNC_PAGES; i++) {
111 page = alloc_page(gfp_flags);
112 if (unlikely(!page))
113 goto out_free_pages;
114
115 bio->bi_io_vec[i].bv_page = page;
116 }
117
118 r1_bio->master_bio = NULL;
119
120 return r1_bio;
121
122 out_free_pages:
123 for ( ; i > 0 ; i--)
124 __free_page(bio->bi_io_vec[i-1].bv_page);
125 out_free_bio:
126 while ( ++j < pi->raid_disks )
127 bio_put(r1_bio->bios[j]);
128 r1bio_pool_free(r1_bio, data);
129 return NULL;
130 }
131
132 static void r1buf_pool_free(void *__r1_bio, void *data)
133 {
134 struct pool_info *pi = data;
135 int i;
136 r1bio_t *r1bio = __r1_bio;
137 struct bio *bio = r1bio->bios[0];
138
139 for (i = 0; i < RESYNC_PAGES; i++) {
140 __free_page(bio->bi_io_vec[i].bv_page);
141 bio->bi_io_vec[i].bv_page = NULL;
142 }
143 for (i=0 ; i < pi->raid_disks; i++)
144 bio_put(r1bio->bios[i]);
145
146 r1bio_pool_free(r1bio, data);
147 }
148
149 static void put_all_bios(conf_t *conf, r1bio_t *r1_bio)
150 {
151 int i;
152
153 for (i = 0; i < conf->raid_disks; i++) {
154 struct bio **bio = r1_bio->bios + i;
155 if (*bio)
156 bio_put(*bio);
157 *bio = NULL;
158 }
159 }
160
161 static inline void free_r1bio(r1bio_t *r1_bio)
162 {
163 unsigned long flags;
164
165 conf_t *conf = mddev_to_conf(r1_bio->mddev);
166
167 /*
168 * Wake up any possible resync thread that waits for the device
169 * to go idle.
170 */
171 spin_lock_irqsave(&conf->resync_lock, flags);
172 if (!--conf->nr_pending) {
173 wake_up(&conf->wait_idle);
174 wake_up(&conf->wait_resume);
175 }
176 spin_unlock_irqrestore(&conf->resync_lock, flags);
177
178 put_all_bios(conf, r1_bio);
179 mempool_free(r1_bio, conf->r1bio_pool);
180 }
181
182 static inline void put_buf(r1bio_t *r1_bio)
183 {
184 conf_t *conf = mddev_to_conf(r1_bio->mddev);
185 unsigned long flags;
186
187 mempool_free(r1_bio, conf->r1buf_pool);
188
189 spin_lock_irqsave(&conf->resync_lock, flags);
190 if (!conf->barrier)
191 BUG();
192 --conf->barrier;
193 wake_up(&conf->wait_resume);
194 wake_up(&conf->wait_idle);
195
196 if (!--conf->nr_pending) {
197 wake_up(&conf->wait_idle);
198 wake_up(&conf->wait_resume);
199 }
200 spin_unlock_irqrestore(&conf->resync_lock, flags);
201 }
202
203 static void reschedule_retry(r1bio_t *r1_bio)
204 {
205 unsigned long flags;
206 mddev_t *mddev = r1_bio->mddev;
207 conf_t *conf = mddev_to_conf(mddev);
208
209 spin_lock_irqsave(&conf->device_lock, flags);
210 list_add(&r1_bio->retry_list, &conf->retry_list);
211 spin_unlock_irqrestore(&conf->device_lock, flags);
212
213 md_wakeup_thread(mddev->thread);
214 }
215
216 /*
217 * raid_end_bio_io() is called when we have finished servicing a mirrored
218 * operation and are ready to return a success/failure code to the buffer
219 * cache layer.
220 */
221 static void raid_end_bio_io(r1bio_t *r1_bio)
222 {
223 struct bio *bio = r1_bio->master_bio;
224
225 /* if nobody has done the final endio yet, do it now */
226 if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {
227 PRINTK(KERN_DEBUG "raid1: sync end %s on sectors %llu-%llu\n",
228 (bio_data_dir(bio) == WRITE) ? "write" : "read",
229 (unsigned long long) bio->bi_sector,
230 (unsigned long long) bio->bi_sector +
231 (bio->bi_size >> 9) - 1);
232
233 bio_endio(bio, bio->bi_size,
234 test_bit(R1BIO_Uptodate, &r1_bio->state) ? 0 : -EIO);
235 }
236 free_r1bio(r1_bio);
237 }
238
239 /*
240 * Update disk head position estimator based on IRQ completion info.
241 */
242 static inline void update_head_pos(int disk, r1bio_t *r1_bio)
243 {
244 conf_t *conf = mddev_to_conf(r1_bio->mddev);
245
246 conf->mirrors[disk].head_position =
247 r1_bio->sector + (r1_bio->sectors);
248 }
249
250 static int raid1_end_read_request(struct bio *bio, unsigned int bytes_done, int error)
251 {
252 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
253 r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
254 int mirror;
255 conf_t *conf = mddev_to_conf(r1_bio->mddev);
256
257 if (bio->bi_size)
258 return 1;
259
260 mirror = r1_bio->read_disk;
261 /*
262 * this branch is our 'one mirror IO has finished' event handler:
263 */
264 if (!uptodate)
265 md_error(r1_bio->mddev, conf->mirrors[mirror].rdev);
266 else
267 /*
268 * Set R1BIO_Uptodate in our master bio, so that
269 * we will return a good error code for to the higher
270 * levels even if IO on some other mirrored buffer fails.
271 *
272 * The 'master' represents the composite IO operation to
273 * user-side. So if something waits for IO, then it will
274 * wait for the 'master' bio.
275 */
276 set_bit(R1BIO_Uptodate, &r1_bio->state);
277
278 update_head_pos(mirror, r1_bio);
279
280 /*
281 * we have only one bio on the read side
282 */
283 if (uptodate)
284 raid_end_bio_io(r1_bio);
285 else {
286 /*
287 * oops, read error:
288 */
289 char b[BDEVNAME_SIZE];
290 if (printk_ratelimit())
291 printk(KERN_ERR "raid1: %s: rescheduling sector %llu\n",
292 bdevname(conf->mirrors[mirror].rdev->bdev,b), (unsigned long long)r1_bio->sector);
293 reschedule_retry(r1_bio);
294 }
295
296 rdev_dec_pending(conf->mirrors[mirror].rdev, conf->mddev);
297 return 0;
298 }
299
300 static int raid1_end_write_request(struct bio *bio, unsigned int bytes_done, int error)
301 {
302 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
303 r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
304 int mirror, behind = test_bit(R1BIO_BehindIO, &r1_bio->state);
305 conf_t *conf = mddev_to_conf(r1_bio->mddev);
306
307 if (bio->bi_size)
308 return 1;
309
310 for (mirror = 0; mirror < conf->raid_disks; mirror++)
311 if (r1_bio->bios[mirror] == bio)
312 break;
313
314 if (error == -ENOTSUPP && test_bit(R1BIO_Barrier, &r1_bio->state)) {
315 set_bit(BarriersNotsupp, &conf->mirrors[mirror].rdev->flags);
316 set_bit(R1BIO_BarrierRetry, &r1_bio->state);
317 r1_bio->mddev->barriers_work = 0;
318 } else {
319 /*
320 * this branch is our 'one mirror IO has finished' event handler:
321 */
322 r1_bio->bios[mirror] = NULL;
323 bio_put(bio);
324 if (!uptodate) {
325 md_error(r1_bio->mddev, conf->mirrors[mirror].rdev);
326 /* an I/O failed, we can't clear the bitmap */
327 set_bit(R1BIO_Degraded, &r1_bio->state);
328 } else
329 /*
330 * Set R1BIO_Uptodate in our master bio, so that
331 * we will return a good error code for to the higher
332 * levels even if IO on some other mirrored buffer fails.
333 *
334 * The 'master' represents the composite IO operation to
335 * user-side. So if something waits for IO, then it will
336 * wait for the 'master' bio.
337 */
338 set_bit(R1BIO_Uptodate, &r1_bio->state);
339
340 update_head_pos(mirror, r1_bio);
341
342 if (behind) {
343 if (test_bit(WriteMostly, &conf->mirrors[mirror].rdev->flags))
344 atomic_dec(&r1_bio->behind_remaining);
345
346 /* In behind mode, we ACK the master bio once the I/O has safely
347 * reached all non-writemostly disks. Setting the Returned bit
348 * ensures that this gets done only once -- we don't ever want to
349 * return -EIO here, instead we'll wait */
350
351 if (atomic_read(&r1_bio->behind_remaining) >= (atomic_read(&r1_bio->remaining)-1) &&
352 test_bit(R1BIO_Uptodate, &r1_bio->state)) {
353 /* Maybe we can return now */
354 if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {
355 struct bio *mbio = r1_bio->master_bio;
356 PRINTK(KERN_DEBUG "raid1: behind end write sectors %llu-%llu\n",
357 (unsigned long long) mbio->bi_sector,
358 (unsigned long long) mbio->bi_sector +
359 (mbio->bi_size >> 9) - 1);
360 bio_endio(mbio, mbio->bi_size, 0);
361 }
362 }
363 }
364 }
365 /*
366 *
367 * Let's see if all mirrored write operations have finished
368 * already.
369 */
370 if (atomic_dec_and_test(&r1_bio->remaining)) {
371 if (test_bit(R1BIO_BarrierRetry, &r1_bio->state)) {
372 reschedule_retry(r1_bio);
373 /* Don't dec_pending yet, we want to hold
374 * the reference over the retry
375 */
376 return 0;
377 }
378 if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
379 /* free extra copy of the data pages */
380 /* FIXME bio has been freed!!! */
381 int i = bio->bi_vcnt;
382 while (i--)
383 __free_page(bio->bi_io_vec[i].bv_page);
384 }
385 /* clear the bitmap if all writes complete successfully */
386 bitmap_endwrite(r1_bio->mddev->bitmap, r1_bio->sector,
387 r1_bio->sectors,
388 !test_bit(R1BIO_Degraded, &r1_bio->state),
389 behind);
390 md_write_end(r1_bio->mddev);
391 raid_end_bio_io(r1_bio);
392 }
393
394 rdev_dec_pending(conf->mirrors[mirror].rdev, conf->mddev);
395 return 0;
396 }
397
398
399 /*
400 * This routine returns the disk from which the requested read should
401 * be done. There is a per-array 'next expected sequential IO' sector
402 * number - if this matches on the next IO then we use the last disk.
403 * There is also a per-disk 'last know head position' sector that is
404 * maintained from IRQ contexts, both the normal and the resync IO
405 * completion handlers update this position correctly. If there is no
406 * perfect sequential match then we pick the disk whose head is closest.
407 *
408 * If there are 2 mirrors in the same 2 devices, performance degrades
409 * because position is mirror, not device based.
410 *
411 * The rdev for the device selected will have nr_pending incremented.
412 */
413 static int read_balance(conf_t *conf, r1bio_t *r1_bio)
414 {
415 const unsigned long this_sector = r1_bio->sector;
416 int new_disk = conf->last_used, disk = new_disk;
417 int wonly_disk = -1;
418 const int sectors = r1_bio->sectors;
419 sector_t new_distance, current_distance;
420 mdk_rdev_t *rdev;
421
422 rcu_read_lock();
423 /*
424 * Check if we can balance. We can balance on the whole
425 * device if no resync is going on, or below the resync window.
426 * We take the first readable disk when above the resync window.
427 */
428 retry:
429 if (conf->mddev->recovery_cp < MaxSector &&
430 (this_sector + sectors >= conf->next_resync)) {
431 /* Choose the first operation device, for consistancy */
432 new_disk = 0;
433
434 for (rdev = rcu_dereference(conf->mirrors[new_disk].rdev);
435 !rdev || !test_bit(In_sync, &rdev->flags)
436 || test_bit(WriteMostly, &rdev->flags);
437 rdev = rcu_dereference(conf->mirrors[++new_disk].rdev)) {
438
439 if (rdev && test_bit(In_sync, &rdev->flags))
440 wonly_disk = new_disk;
441
442 if (new_disk == conf->raid_disks - 1) {
443 new_disk = wonly_disk;
444 break;
445 }
446 }
447 goto rb_out;
448 }
449
450
451 /* make sure the disk is operational */
452 for (rdev = rcu_dereference(conf->mirrors[new_disk].rdev);
453 !rdev || !test_bit(In_sync, &rdev->flags) ||
454 test_bit(WriteMostly, &rdev->flags);
455 rdev = rcu_dereference(conf->mirrors[new_disk].rdev)) {
456
457 if (rdev && test_bit(In_sync, &rdev->flags))
458 wonly_disk = new_disk;
459
460 if (new_disk <= 0)
461 new_disk = conf->raid_disks;
462 new_disk--;
463 if (new_disk == disk) {
464 new_disk = wonly_disk;
465 break;
466 }
467 }
468
469 if (new_disk < 0)
470 goto rb_out;
471
472 disk = new_disk;
473 /* now disk == new_disk == starting point for search */
474
475 /*
476 * Don't change to another disk for sequential reads:
477 */
478 if (conf->next_seq_sect == this_sector)
479 goto rb_out;
480 if (this_sector == conf->mirrors[new_disk].head_position)
481 goto rb_out;
482
483 current_distance = abs(this_sector - conf->mirrors[disk].head_position);
484
485 /* Find the disk whose head is closest */
486
487 do {
488 if (disk <= 0)
489 disk = conf->raid_disks;
490 disk--;
491
492 rdev = rcu_dereference(conf->mirrors[disk].rdev);
493
494 if (!rdev ||
495 !test_bit(In_sync, &rdev->flags) ||
496 test_bit(WriteMostly, &rdev->flags))
497 continue;
498
499 if (!atomic_read(&rdev->nr_pending)) {
500 new_disk = disk;
501 break;
502 }
503 new_distance = abs(this_sector - conf->mirrors[disk].head_position);
504 if (new_distance < current_distance) {
505 current_distance = new_distance;
506 new_disk = disk;
507 }
508 } while (disk != conf->last_used);
509
510 rb_out:
511
512
513 if (new_disk >= 0) {
514 rdev = rcu_dereference(conf->mirrors[new_disk].rdev);
515 if (!rdev)
516 goto retry;
517 atomic_inc(&rdev->nr_pending);
518 if (!test_bit(In_sync, &rdev->flags)) {
519 /* cannot risk returning a device that failed
520 * before we inc'ed nr_pending
521 */
522 atomic_dec(&rdev->nr_pending);
523 goto retry;
524 }
525 conf->next_seq_sect = this_sector + sectors;
526 conf->last_used = new_disk;
527 }
528 rcu_read_unlock();
529
530 return new_disk;
531 }
532
533 static void unplug_slaves(mddev_t *mddev)
534 {
535 conf_t *conf = mddev_to_conf(mddev);
536 int i;
537
538 rcu_read_lock();
539 for (i=0; i<mddev->raid_disks; i++) {
540 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
541 if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) {
542 request_queue_t *r_queue = bdev_get_queue(rdev->bdev);
543
544 atomic_inc(&rdev->nr_pending);
545 rcu_read_unlock();
546
547 if (r_queue->unplug_fn)
548 r_queue->unplug_fn(r_queue);
549
550 rdev_dec_pending(rdev, mddev);
551 rcu_read_lock();
552 }
553 }
554 rcu_read_unlock();
555 }
556
557 static void raid1_unplug(request_queue_t *q)
558 {
559 mddev_t *mddev = q->queuedata;
560
561 unplug_slaves(mddev);
562 md_wakeup_thread(mddev->thread);
563 }
564
565 static int raid1_issue_flush(request_queue_t *q, struct gendisk *disk,
566 sector_t *error_sector)
567 {
568 mddev_t *mddev = q->queuedata;
569 conf_t *conf = mddev_to_conf(mddev);
570 int i, ret = 0;
571
572 rcu_read_lock();
573 for (i=0; i<mddev->raid_disks && ret == 0; i++) {
574 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
575 if (rdev && !test_bit(Faulty, &rdev->flags)) {
576 struct block_device *bdev = rdev->bdev;
577 request_queue_t *r_queue = bdev_get_queue(bdev);
578
579 if (!r_queue->issue_flush_fn)
580 ret = -EOPNOTSUPP;
581 else {
582 atomic_inc(&rdev->nr_pending);
583 rcu_read_unlock();
584 ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk,
585 error_sector);
586 rdev_dec_pending(rdev, mddev);
587 rcu_read_lock();
588 }
589 }
590 }
591 rcu_read_unlock();
592 return ret;
593 }
594
595 /*
596 * Throttle resync depth, so that we can both get proper overlapping of
597 * requests, but are still able to handle normal requests quickly.
598 */
599 #define RESYNC_DEPTH 32
600
601 static void device_barrier(conf_t *conf, sector_t sect)
602 {
603 spin_lock_irq(&conf->resync_lock);
604 wait_event_lock_irq(conf->wait_idle, !waitqueue_active(&conf->wait_resume),
605 conf->resync_lock, raid1_unplug(conf->mddev->queue));
606
607 if (!conf->barrier++) {
608 wait_event_lock_irq(conf->wait_idle, !conf->nr_pending,
609 conf->resync_lock, raid1_unplug(conf->mddev->queue));
610 if (conf->nr_pending)
611 BUG();
612 }
613 wait_event_lock_irq(conf->wait_resume, conf->barrier < RESYNC_DEPTH,
614 conf->resync_lock, raid1_unplug(conf->mddev->queue));
615 conf->next_resync = sect;
616 spin_unlock_irq(&conf->resync_lock);
617 }
618
619 /* duplicate the data pages for behind I/O */
620 static struct page **alloc_behind_pages(struct bio *bio)
621 {
622 int i;
623 struct bio_vec *bvec;
624 struct page **pages = kmalloc(bio->bi_vcnt * sizeof(struct page *),
625 GFP_NOIO);
626 if (unlikely(!pages))
627 goto do_sync_io;
628
629 memset(pages, 0, bio->bi_vcnt * sizeof(struct page *));
630
631 bio_for_each_segment(bvec, bio, i) {
632 pages[i] = alloc_page(GFP_NOIO);
633 if (unlikely(!pages[i]))
634 goto do_sync_io;
635 memcpy(kmap(pages[i]) + bvec->bv_offset,
636 kmap(bvec->bv_page) + bvec->bv_offset, bvec->bv_len);
637 kunmap(pages[i]);
638 kunmap(bvec->bv_page);
639 }
640
641 return pages;
642
643 do_sync_io:
644 if (pages)
645 for (i = 0; i < bio->bi_vcnt && pages[i]; i++)
646 __free_page(pages[i]);
647 kfree(pages);
648 PRINTK("%dB behind alloc failed, doing sync I/O\n", bio->bi_size);
649 return NULL;
650 }
651
652 static int make_request(request_queue_t *q, struct bio * bio)
653 {
654 mddev_t *mddev = q->queuedata;
655 conf_t *conf = mddev_to_conf(mddev);
656 mirror_info_t *mirror;
657 r1bio_t *r1_bio;
658 struct bio *read_bio;
659 int i, targets = 0, disks;
660 mdk_rdev_t *rdev;
661 struct bitmap *bitmap = mddev->bitmap;
662 unsigned long flags;
663 struct bio_list bl;
664 struct page **behind_pages = NULL;
665 const int rw = bio_data_dir(bio);
666 int do_barriers;
667
668 if (unlikely(!mddev->barriers_work && bio_barrier(bio))) {
669 bio_endio(bio, bio->bi_size, -EOPNOTSUPP);
670 return 0;
671 }
672
673 /*
674 * Register the new request and wait if the reconstruction
675 * thread has put up a bar for new requests.
676 * Continue immediately if no resync is active currently.
677 */
678 md_write_start(mddev, bio); /* wait on superblock update early */
679
680 spin_lock_irq(&conf->resync_lock);
681 wait_event_lock_irq(conf->wait_resume, !conf->barrier, conf->resync_lock, );
682 conf->nr_pending++;
683 spin_unlock_irq(&conf->resync_lock);
684
685 disk_stat_inc(mddev->gendisk, ios[rw]);
686 disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
687
688 /*
689 * make_request() can abort the operation when READA is being
690 * used and no empty request is available.
691 *
692 */
693 r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);
694
695 r1_bio->master_bio = bio;
696 r1_bio->sectors = bio->bi_size >> 9;
697 r1_bio->state = 0;
698 r1_bio->mddev = mddev;
699 r1_bio->sector = bio->bi_sector;
700
701 if (rw == READ) {
702 /*
703 * read balancing logic:
704 */
705 int rdisk = read_balance(conf, r1_bio);
706
707 if (rdisk < 0) {
708 /* couldn't find anywhere to read from */
709 raid_end_bio_io(r1_bio);
710 return 0;
711 }
712 mirror = conf->mirrors + rdisk;
713
714 r1_bio->read_disk = rdisk;
715
716 read_bio = bio_clone(bio, GFP_NOIO);
717
718 r1_bio->bios[rdisk] = read_bio;
719
720 read_bio->bi_sector = r1_bio->sector + mirror->rdev->data_offset;
721 read_bio->bi_bdev = mirror->rdev->bdev;
722 read_bio->bi_end_io = raid1_end_read_request;
723 read_bio->bi_rw = READ;
724 read_bio->bi_private = r1_bio;
725
726 generic_make_request(read_bio);
727 return 0;
728 }
729
730 /*
731 * WRITE:
732 */
733 /* first select target devices under spinlock and
734 * inc refcount on their rdev. Record them by setting
735 * bios[x] to bio
736 */
737 disks = conf->raid_disks;
738 #if 0
739 { static int first=1;
740 if (first) printk("First Write sector %llu disks %d\n",
741 (unsigned long long)r1_bio->sector, disks);
742 first = 0;
743 }
744 #endif
745 rcu_read_lock();
746 for (i = 0; i < disks; i++) {
747 if ((rdev=rcu_dereference(conf->mirrors[i].rdev)) != NULL &&
748 !test_bit(Faulty, &rdev->flags)) {
749 atomic_inc(&rdev->nr_pending);
750 if (test_bit(Faulty, &rdev->flags)) {
751 atomic_dec(&rdev->nr_pending);
752 r1_bio->bios[i] = NULL;
753 } else
754 r1_bio->bios[i] = bio;
755 targets++;
756 } else
757 r1_bio->bios[i] = NULL;
758 }
759 rcu_read_unlock();
760
761 BUG_ON(targets == 0); /* we never fail the last device */
762
763 if (targets < conf->raid_disks) {
764 /* array is degraded, we will not clear the bitmap
765 * on I/O completion (see raid1_end_write_request) */
766 set_bit(R1BIO_Degraded, &r1_bio->state);
767 }
768
769 /* do behind I/O ? */
770 if (bitmap &&
771 atomic_read(&bitmap->behind_writes) < bitmap->max_write_behind &&
772 (behind_pages = alloc_behind_pages(bio)) != NULL)
773 set_bit(R1BIO_BehindIO, &r1_bio->state);
774
775 atomic_set(&r1_bio->remaining, 0);
776 atomic_set(&r1_bio->behind_remaining, 0);
777
778 do_barriers = bio->bi_rw & BIO_RW_BARRIER;
779 if (do_barriers)
780 set_bit(R1BIO_Barrier, &r1_bio->state);
781
782 bio_list_init(&bl);
783 for (i = 0; i < disks; i++) {
784 struct bio *mbio;
785 if (!r1_bio->bios[i])
786 continue;
787
788 mbio = bio_clone(bio, GFP_NOIO);
789 r1_bio->bios[i] = mbio;
790
791 mbio->bi_sector = r1_bio->sector + conf->mirrors[i].rdev->data_offset;
792 mbio->bi_bdev = conf->mirrors[i].rdev->bdev;
793 mbio->bi_end_io = raid1_end_write_request;
794 mbio->bi_rw = WRITE | do_barriers;
795 mbio->bi_private = r1_bio;
796
797 if (behind_pages) {
798 struct bio_vec *bvec;
799 int j;
800
801 /* Yes, I really want the '__' version so that
802 * we clear any unused pointer in the io_vec, rather
803 * than leave them unchanged. This is important
804 * because when we come to free the pages, we won't
805 * know the originial bi_idx, so we just free
806 * them all
807 */
808 __bio_for_each_segment(bvec, mbio, j, 0)
809 bvec->bv_page = behind_pages[j];
810 if (test_bit(WriteMostly, &conf->mirrors[i].rdev->flags))
811 atomic_inc(&r1_bio->behind_remaining);
812 }
813
814 atomic_inc(&r1_bio->remaining);
815
816 bio_list_add(&bl, mbio);
817 }
818 kfree(behind_pages); /* the behind pages are attached to the bios now */
819
820 bitmap_startwrite(bitmap, bio->bi_sector, r1_bio->sectors,
821 test_bit(R1BIO_BehindIO, &r1_bio->state));
822 spin_lock_irqsave(&conf->device_lock, flags);
823 bio_list_merge(&conf->pending_bio_list, &bl);
824 bio_list_init(&bl);
825
826 blk_plug_device(mddev->queue);
827 spin_unlock_irqrestore(&conf->device_lock, flags);
828
829 #if 0
830 while ((bio = bio_list_pop(&bl)) != NULL)
831 generic_make_request(bio);
832 #endif
833
834 return 0;
835 }
836
837 static void status(struct seq_file *seq, mddev_t *mddev)
838 {
839 conf_t *conf = mddev_to_conf(mddev);
840 int i;
841
842 seq_printf(seq, " [%d/%d] [", conf->raid_disks,
843 conf->working_disks);
844 for (i = 0; i < conf->raid_disks; i++)
845 seq_printf(seq, "%s",
846 conf->mirrors[i].rdev &&
847 test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_");
848 seq_printf(seq, "]");
849 }
850
851
852 static void error(mddev_t *mddev, mdk_rdev_t *rdev)
853 {
854 char b[BDEVNAME_SIZE];
855 conf_t *conf = mddev_to_conf(mddev);
856
857 /*
858 * If it is not operational, then we have already marked it as dead
859 * else if it is the last working disks, ignore the error, let the
860 * next level up know.
861 * else mark the drive as failed
862 */
863 if (test_bit(In_sync, &rdev->flags)
864 && conf->working_disks == 1)
865 /*
866 * Don't fail the drive, act as though we were just a
867 * normal single drive
868 */
869 return;
870 if (test_bit(In_sync, &rdev->flags)) {
871 mddev->degraded++;
872 conf->working_disks--;
873 /*
874 * if recovery is running, make sure it aborts.
875 */
876 set_bit(MD_RECOVERY_ERR, &mddev->recovery);
877 }
878 clear_bit(In_sync, &rdev->flags);
879 set_bit(Faulty, &rdev->flags);
880 mddev->sb_dirty = 1;
881 printk(KERN_ALERT "raid1: Disk failure on %s, disabling device. \n"
882 " Operation continuing on %d devices\n",
883 bdevname(rdev->bdev,b), conf->working_disks);
884 }
885
886 static void print_conf(conf_t *conf)
887 {
888 int i;
889 mirror_info_t *tmp;
890
891 printk("RAID1 conf printout:\n");
892 if (!conf) {
893 printk("(!conf)\n");
894 return;
895 }
896 printk(" --- wd:%d rd:%d\n", conf->working_disks,
897 conf->raid_disks);
898
899 for (i = 0; i < conf->raid_disks; i++) {
900 char b[BDEVNAME_SIZE];
901 tmp = conf->mirrors + i;
902 if (tmp->rdev)
903 printk(" disk %d, wo:%d, o:%d, dev:%s\n",
904 i, !test_bit(In_sync, &tmp->rdev->flags), !test_bit(Faulty, &tmp->rdev->flags),
905 bdevname(tmp->rdev->bdev,b));
906 }
907 }
908
909 static void close_sync(conf_t *conf)
910 {
911 spin_lock_irq(&conf->resync_lock);
912 wait_event_lock_irq(conf->wait_resume, !conf->barrier,
913 conf->resync_lock, raid1_unplug(conf->mddev->queue));
914 spin_unlock_irq(&conf->resync_lock);
915
916 if (conf->barrier) BUG();
917 if (waitqueue_active(&conf->wait_idle)) BUG();
918
919 mempool_destroy(conf->r1buf_pool);
920 conf->r1buf_pool = NULL;
921 }
922
923 static int raid1_spare_active(mddev_t *mddev)
924 {
925 int i;
926 conf_t *conf = mddev->private;
927 mirror_info_t *tmp;
928
929 /*
930 * Find all failed disks within the RAID1 configuration
931 * and mark them readable
932 */
933 for (i = 0; i < conf->raid_disks; i++) {
934 tmp = conf->mirrors + i;
935 if (tmp->rdev
936 && !test_bit(Faulty, &tmp->rdev->flags)
937 && !test_bit(In_sync, &tmp->rdev->flags)) {
938 conf->working_disks++;
939 mddev->degraded--;
940 set_bit(In_sync, &tmp->rdev->flags);
941 }
942 }
943
944 print_conf(conf);
945 return 0;
946 }
947
948
949 static int raid1_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
950 {
951 conf_t *conf = mddev->private;
952 int found = 0;
953 int mirror = 0;
954 mirror_info_t *p;
955
956 for (mirror=0; mirror < mddev->raid_disks; mirror++)
957 if ( !(p=conf->mirrors+mirror)->rdev) {
958
959 blk_queue_stack_limits(mddev->queue,
960 rdev->bdev->bd_disk->queue);
961 /* as we don't honour merge_bvec_fn, we must never risk
962 * violating it, so limit ->max_sector to one PAGE, as
963 * a one page request is never in violation.
964 */
965 if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
966 mddev->queue->max_sectors > (PAGE_SIZE>>9))
967 blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
968
969 p->head_position = 0;
970 rdev->raid_disk = mirror;
971 found = 1;
972 /* As all devices are equivalent, we don't need a full recovery
973 * if this was recently any drive of the array
974 */
975 if (rdev->saved_raid_disk < 0)
976 conf->fullsync = 1;
977 rcu_assign_pointer(p->rdev, rdev);
978 break;
979 }
980
981 print_conf(conf);
982 return found;
983 }
984
985 static int raid1_remove_disk(mddev_t *mddev, int number)
986 {
987 conf_t *conf = mddev->private;
988 int err = 0;
989 mdk_rdev_t *rdev;
990 mirror_info_t *p = conf->mirrors+ number;
991
992 print_conf(conf);
993 rdev = p->rdev;
994 if (rdev) {
995 if (test_bit(In_sync, &rdev->flags) ||
996 atomic_read(&rdev->nr_pending)) {
997 err = -EBUSY;
998 goto abort;
999 }
1000 p->rdev = NULL;
1001 synchronize_rcu();
1002 if (atomic_read(&rdev->nr_pending)) {
1003 /* lost the race, try later */
1004 err = -EBUSY;
1005 p->rdev = rdev;
1006 }
1007 }
1008 abort:
1009
1010 print_conf(conf);
1011 return err;
1012 }
1013
1014
1015 static int end_sync_read(struct bio *bio, unsigned int bytes_done, int error)
1016 {
1017 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1018 r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
1019 conf_t *conf = mddev_to_conf(r1_bio->mddev);
1020
1021 if (bio->bi_size)
1022 return 1;
1023
1024 if (r1_bio->bios[r1_bio->read_disk] != bio)
1025 BUG();
1026 update_head_pos(r1_bio->read_disk, r1_bio);
1027 /*
1028 * we have read a block, now it needs to be re-written,
1029 * or re-read if the read failed.
1030 * We don't do much here, just schedule handling by raid1d
1031 */
1032 if (!uptodate) {
1033 md_error(r1_bio->mddev,
1034 conf->mirrors[r1_bio->read_disk].rdev);
1035 } else
1036 set_bit(R1BIO_Uptodate, &r1_bio->state);
1037 rdev_dec_pending(conf->mirrors[r1_bio->read_disk].rdev, conf->mddev);
1038 reschedule_retry(r1_bio);
1039 return 0;
1040 }
1041
1042 static int end_sync_write(struct bio *bio, unsigned int bytes_done, int error)
1043 {
1044 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1045 r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
1046 mddev_t *mddev = r1_bio->mddev;
1047 conf_t *conf = mddev_to_conf(mddev);
1048 int i;
1049 int mirror=0;
1050
1051 if (bio->bi_size)
1052 return 1;
1053
1054 for (i = 0; i < conf->raid_disks; i++)
1055 if (r1_bio->bios[i] == bio) {
1056 mirror = i;
1057 break;
1058 }
1059 if (!uptodate)
1060 md_error(mddev, conf->mirrors[mirror].rdev);
1061
1062 update_head_pos(mirror, r1_bio);
1063
1064 if (atomic_dec_and_test(&r1_bio->remaining)) {
1065 md_done_sync(mddev, r1_bio->sectors, uptodate);
1066 put_buf(r1_bio);
1067 }
1068 rdev_dec_pending(conf->mirrors[mirror].rdev, mddev);
1069 return 0;
1070 }
1071
1072 static void sync_request_write(mddev_t *mddev, r1bio_t *r1_bio)
1073 {
1074 conf_t *conf = mddev_to_conf(mddev);
1075 int i;
1076 int disks = conf->raid_disks;
1077 struct bio *bio, *wbio;
1078
1079 bio = r1_bio->bios[r1_bio->read_disk];
1080
1081 /*
1082 if (r1_bio->sector == 0) printk("First sync write startss\n");
1083 */
1084 /*
1085 * schedule writes
1086 */
1087 if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) {
1088 /*
1089 * There is no point trying a read-for-reconstruct as
1090 * reconstruct is about to be aborted
1091 */
1092 char b[BDEVNAME_SIZE];
1093 printk(KERN_ALERT "raid1: %s: unrecoverable I/O read error"
1094 " for block %llu\n",
1095 bdevname(bio->bi_bdev,b),
1096 (unsigned long long)r1_bio->sector);
1097 md_done_sync(mddev, r1_bio->sectors, 0);
1098 put_buf(r1_bio);
1099 return;
1100 }
1101
1102 atomic_set(&r1_bio->remaining, 1);
1103 for (i = 0; i < disks ; i++) {
1104 wbio = r1_bio->bios[i];
1105 if (wbio->bi_end_io != end_sync_write)
1106 continue;
1107
1108 atomic_inc(&conf->mirrors[i].rdev->nr_pending);
1109 atomic_inc(&r1_bio->remaining);
1110 md_sync_acct(conf->mirrors[i].rdev->bdev, wbio->bi_size >> 9);
1111
1112 generic_make_request(wbio);
1113 }
1114
1115 if (atomic_dec_and_test(&r1_bio->remaining)) {
1116 /* if we're here, all write(s) have completed, so clean up */
1117 md_done_sync(mddev, r1_bio->sectors, 1);
1118 put_buf(r1_bio);
1119 }
1120 }
1121
1122 /*
1123 * This is a kernel thread which:
1124 *
1125 * 1. Retries failed read operations on working mirrors.
1126 * 2. Updates the raid superblock when problems encounter.
1127 * 3. Performs writes following reads for array syncronising.
1128 */
1129
1130 static void raid1d(mddev_t *mddev)
1131 {
1132 r1bio_t *r1_bio;
1133 struct bio *bio;
1134 unsigned long flags;
1135 conf_t *conf = mddev_to_conf(mddev);
1136 struct list_head *head = &conf->retry_list;
1137 int unplug=0;
1138 mdk_rdev_t *rdev;
1139
1140 md_check_recovery(mddev);
1141
1142 for (;;) {
1143 char b[BDEVNAME_SIZE];
1144 spin_lock_irqsave(&conf->device_lock, flags);
1145
1146 if (conf->pending_bio_list.head) {
1147 bio = bio_list_get(&conf->pending_bio_list);
1148 blk_remove_plug(mddev->queue);
1149 spin_unlock_irqrestore(&conf->device_lock, flags);
1150 /* flush any pending bitmap writes to disk before proceeding w/ I/O */
1151 if (bitmap_unplug(mddev->bitmap) != 0)
1152 printk("%s: bitmap file write failed!\n", mdname(mddev));
1153
1154 while (bio) { /* submit pending writes */
1155 struct bio *next = bio->bi_next;
1156 bio->bi_next = NULL;
1157 generic_make_request(bio);
1158 bio = next;
1159 }
1160 unplug = 1;
1161
1162 continue;
1163 }
1164
1165 if (list_empty(head))
1166 break;
1167 r1_bio = list_entry(head->prev, r1bio_t, retry_list);
1168 list_del(head->prev);
1169 spin_unlock_irqrestore(&conf->device_lock, flags);
1170
1171 mddev = r1_bio->mddev;
1172 conf = mddev_to_conf(mddev);
1173 if (test_bit(R1BIO_IsSync, &r1_bio->state)) {
1174 sync_request_write(mddev, r1_bio);
1175 unplug = 1;
1176 } else if (test_bit(R1BIO_BarrierRetry, &r1_bio->state)) {
1177 /* some requests in the r1bio were BIO_RW_BARRIER
1178 * requests which failed with -ENOTSUPP. Hohumm..
1179 * Better resubmit without the barrier.
1180 * We know which devices to resubmit for, because
1181 * all others have had their bios[] entry cleared.
1182 */
1183 int i;
1184 clear_bit(R1BIO_BarrierRetry, &r1_bio->state);
1185 clear_bit(R1BIO_Barrier, &r1_bio->state);
1186 for (i=0; i < conf->raid_disks; i++)
1187 if (r1_bio->bios[i]) {
1188 struct bio_vec *bvec;
1189 int j;
1190
1191 bio = bio_clone(r1_bio->master_bio, GFP_NOIO);
1192 /* copy pages from the failed bio, as
1193 * this might be a write-behind device */
1194 __bio_for_each_segment(bvec, bio, j, 0)
1195 bvec->bv_page = bio_iovec_idx(r1_bio->bios[i], j)->bv_page;
1196 bio_put(r1_bio->bios[i]);
1197 bio->bi_sector = r1_bio->sector +
1198 conf->mirrors[i].rdev->data_offset;
1199 bio->bi_bdev = conf->mirrors[i].rdev->bdev;
1200 bio->bi_end_io = raid1_end_write_request;
1201 bio->bi_rw = WRITE;
1202 bio->bi_private = r1_bio;
1203 r1_bio->bios[i] = bio;
1204 generic_make_request(bio);
1205 }
1206 } else {
1207 int disk;
1208 bio = r1_bio->bios[r1_bio->read_disk];
1209 if ((disk=read_balance(conf, r1_bio)) == -1) {
1210 printk(KERN_ALERT "raid1: %s: unrecoverable I/O"
1211 " read error for block %llu\n",
1212 bdevname(bio->bi_bdev,b),
1213 (unsigned long long)r1_bio->sector);
1214 raid_end_bio_io(r1_bio);
1215 } else {
1216 r1_bio->bios[r1_bio->read_disk] = NULL;
1217 r1_bio->read_disk = disk;
1218 bio_put(bio);
1219 bio = bio_clone(r1_bio->master_bio, GFP_NOIO);
1220 r1_bio->bios[r1_bio->read_disk] = bio;
1221 rdev = conf->mirrors[disk].rdev;
1222 if (printk_ratelimit())
1223 printk(KERN_ERR "raid1: %s: redirecting sector %llu to"
1224 " another mirror\n",
1225 bdevname(rdev->bdev,b),
1226 (unsigned long long)r1_bio->sector);
1227 bio->bi_sector = r1_bio->sector + rdev->data_offset;
1228 bio->bi_bdev = rdev->bdev;
1229 bio->bi_end_io = raid1_end_read_request;
1230 bio->bi_rw = READ;
1231 bio->bi_private = r1_bio;
1232 unplug = 1;
1233 generic_make_request(bio);
1234 }
1235 }
1236 }
1237 spin_unlock_irqrestore(&conf->device_lock, flags);
1238 if (unplug)
1239 unplug_slaves(mddev);
1240 }
1241
1242
1243 static int init_resync(conf_t *conf)
1244 {
1245 int buffs;
1246
1247 buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
1248 if (conf->r1buf_pool)
1249 BUG();
1250 conf->r1buf_pool = mempool_create(buffs, r1buf_pool_alloc, r1buf_pool_free,
1251 conf->poolinfo);
1252 if (!conf->r1buf_pool)
1253 return -ENOMEM;
1254 conf->next_resync = 0;
1255 return 0;
1256 }
1257
1258 /*
1259 * perform a "sync" on one "block"
1260 *
1261 * We need to make sure that no normal I/O request - particularly write
1262 * requests - conflict with active sync requests.
1263 *
1264 * This is achieved by tracking pending requests and a 'barrier' concept
1265 * that can be installed to exclude normal IO requests.
1266 */
1267
1268 static sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster)
1269 {
1270 conf_t *conf = mddev_to_conf(mddev);
1271 mirror_info_t *mirror;
1272 r1bio_t *r1_bio;
1273 struct bio *bio;
1274 sector_t max_sector, nr_sectors;
1275 int disk;
1276 int i;
1277 int wonly;
1278 int write_targets = 0;
1279 int sync_blocks;
1280 int still_degraded = 0;
1281
1282 if (!conf->r1buf_pool)
1283 {
1284 /*
1285 printk("sync start - bitmap %p\n", mddev->bitmap);
1286 */
1287 if (init_resync(conf))
1288 return 0;
1289 }
1290
1291 max_sector = mddev->size << 1;
1292 if (sector_nr >= max_sector) {
1293 /* If we aborted, we need to abort the
1294 * sync on the 'current' bitmap chunk (there will
1295 * only be one in raid1 resync.
1296 * We can find the current addess in mddev->curr_resync
1297 */
1298 if (mddev->curr_resync < max_sector) /* aborted */
1299 bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
1300 &sync_blocks, 1);
1301 else /* completed sync */
1302 conf->fullsync = 0;
1303
1304 bitmap_close_sync(mddev->bitmap);
1305 close_sync(conf);
1306 return 0;
1307 }
1308
1309 /* before building a request, check if we can skip these blocks..
1310 * This call the bitmap_start_sync doesn't actually record anything
1311 */
1312 if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
1313 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
1314 /* We can skip this block, and probably several more */
1315 *skipped = 1;
1316 return sync_blocks;
1317 }
1318 /*
1319 * If there is non-resync activity waiting for us then
1320 * put in a delay to throttle resync.
1321 */
1322 if (!go_faster && waitqueue_active(&conf->wait_resume))
1323 msleep_interruptible(1000);
1324 device_barrier(conf, sector_nr + RESYNC_SECTORS);
1325
1326 /*
1327 * If reconstructing, and >1 working disc,
1328 * could dedicate one to rebuild and others to
1329 * service read requests ..
1330 */
1331 disk = conf->last_used;
1332 /* make sure disk is operational */
1333 wonly = disk;
1334 while (conf->mirrors[disk].rdev == NULL ||
1335 !test_bit(In_sync, &conf->mirrors[disk].rdev->flags) ||
1336 test_bit(WriteMostly, &conf->mirrors[disk].rdev->flags)
1337 ) {
1338 if (conf->mirrors[disk].rdev &&
1339 test_bit(In_sync, &conf->mirrors[disk].rdev->flags))
1340 wonly = disk;
1341 if (disk <= 0)
1342 disk = conf->raid_disks;
1343 disk--;
1344 if (disk == conf->last_used) {
1345 disk = wonly;
1346 break;
1347 }
1348 }
1349 conf->last_used = disk;
1350 atomic_inc(&conf->mirrors[disk].rdev->nr_pending);
1351
1352
1353 mirror = conf->mirrors + disk;
1354
1355 r1_bio = mempool_alloc(conf->r1buf_pool, GFP_NOIO);
1356
1357 spin_lock_irq(&conf->resync_lock);
1358 conf->nr_pending++;
1359 spin_unlock_irq(&conf->resync_lock);
1360
1361 r1_bio->mddev = mddev;
1362 r1_bio->sector = sector_nr;
1363 r1_bio->state = 0;
1364 set_bit(R1BIO_IsSync, &r1_bio->state);
1365 r1_bio->read_disk = disk;
1366
1367 for (i=0; i < conf->raid_disks; i++) {
1368 bio = r1_bio->bios[i];
1369
1370 /* take from bio_init */
1371 bio->bi_next = NULL;
1372 bio->bi_flags |= 1 << BIO_UPTODATE;
1373 bio->bi_rw = 0;
1374 bio->bi_vcnt = 0;
1375 bio->bi_idx = 0;
1376 bio->bi_phys_segments = 0;
1377 bio->bi_hw_segments = 0;
1378 bio->bi_size = 0;
1379 bio->bi_end_io = NULL;
1380 bio->bi_private = NULL;
1381
1382 if (i == disk) {
1383 bio->bi_rw = READ;
1384 bio->bi_end_io = end_sync_read;
1385 } else if (conf->mirrors[i].rdev == NULL ||
1386 test_bit(Faulty, &conf->mirrors[i].rdev->flags)) {
1387 still_degraded = 1;
1388 continue;
1389 } else if (!test_bit(In_sync, &conf->mirrors[i].rdev->flags) ||
1390 sector_nr + RESYNC_SECTORS > mddev->recovery_cp ||
1391 test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
1392 bio->bi_rw = WRITE;
1393 bio->bi_end_io = end_sync_write;
1394 write_targets ++;
1395 } else
1396 /* no need to read or write here */
1397 continue;
1398 bio->bi_sector = sector_nr + conf->mirrors[i].rdev->data_offset;
1399 bio->bi_bdev = conf->mirrors[i].rdev->bdev;
1400 bio->bi_private = r1_bio;
1401 }
1402
1403 if (write_targets == 0) {
1404 /* There is nowhere to write, so all non-sync
1405 * drives must be failed - so we are finished
1406 */
1407 sector_t rv = max_sector - sector_nr;
1408 *skipped = 1;
1409 put_buf(r1_bio);
1410 rdev_dec_pending(conf->mirrors[disk].rdev, mddev);
1411 return rv;
1412 }
1413
1414 nr_sectors = 0;
1415 sync_blocks = 0;
1416 do {
1417 struct page *page;
1418 int len = PAGE_SIZE;
1419 if (sector_nr + (len>>9) > max_sector)
1420 len = (max_sector - sector_nr) << 9;
1421 if (len == 0)
1422 break;
1423 if (sync_blocks == 0) {
1424 if (!bitmap_start_sync(mddev->bitmap, sector_nr,
1425 &sync_blocks, still_degraded) &&
1426 !conf->fullsync &&
1427 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
1428 break;
1429 if (sync_blocks < (PAGE_SIZE>>9))
1430 BUG();
1431 if (len > (sync_blocks<<9))
1432 len = sync_blocks<<9;
1433 }
1434
1435 for (i=0 ; i < conf->raid_disks; i++) {
1436 bio = r1_bio->bios[i];
1437 if (bio->bi_end_io) {
1438 page = r1_bio->bios[0]->bi_io_vec[bio->bi_vcnt].bv_page;
1439 if (bio_add_page(bio, page, len, 0) == 0) {
1440 /* stop here */
1441 r1_bio->bios[0]->bi_io_vec[bio->bi_vcnt].bv_page = page;
1442 while (i > 0) {
1443 i--;
1444 bio = r1_bio->bios[i];
1445 if (bio->bi_end_io==NULL)
1446 continue;
1447 /* remove last page from this bio */
1448 bio->bi_vcnt--;
1449 bio->bi_size -= len;
1450 bio->bi_flags &= ~(1<< BIO_SEG_VALID);
1451 }
1452 goto bio_full;
1453 }
1454 }
1455 }
1456 nr_sectors += len>>9;
1457 sector_nr += len>>9;
1458 sync_blocks -= (len>>9);
1459 } while (r1_bio->bios[disk]->bi_vcnt < RESYNC_PAGES);
1460 bio_full:
1461 bio = r1_bio->bios[disk];
1462 r1_bio->sectors = nr_sectors;
1463
1464 md_sync_acct(mirror->rdev->bdev, nr_sectors);
1465
1466 generic_make_request(bio);
1467
1468 return nr_sectors;
1469 }
1470
1471 static int run(mddev_t *mddev)
1472 {
1473 conf_t *conf;
1474 int i, j, disk_idx;
1475 mirror_info_t *disk;
1476 mdk_rdev_t *rdev;
1477 struct list_head *tmp;
1478
1479 if (mddev->level != 1) {
1480 printk("raid1: %s: raid level not set to mirroring (%d)\n",
1481 mdname(mddev), mddev->level);
1482 goto out;
1483 }
1484 /*
1485 * copy the already verified devices into our private RAID1
1486 * bookkeeping area. [whatever we allocate in run(),
1487 * should be freed in stop()]
1488 */
1489 conf = kmalloc(sizeof(conf_t), GFP_KERNEL);
1490 mddev->private = conf;
1491 if (!conf)
1492 goto out_no_mem;
1493
1494 memset(conf, 0, sizeof(*conf));
1495 conf->mirrors = kmalloc(sizeof(struct mirror_info)*mddev->raid_disks,
1496 GFP_KERNEL);
1497 if (!conf->mirrors)
1498 goto out_no_mem;
1499
1500 memset(conf->mirrors, 0, sizeof(struct mirror_info)*mddev->raid_disks);
1501
1502 conf->poolinfo = kmalloc(sizeof(*conf->poolinfo), GFP_KERNEL);
1503 if (!conf->poolinfo)
1504 goto out_no_mem;
1505 conf->poolinfo->mddev = mddev;
1506 conf->poolinfo->raid_disks = mddev->raid_disks;
1507 conf->r1bio_pool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
1508 r1bio_pool_free,
1509 conf->poolinfo);
1510 if (!conf->r1bio_pool)
1511 goto out_no_mem;
1512
1513 ITERATE_RDEV(mddev, rdev, tmp) {
1514 disk_idx = rdev->raid_disk;
1515 if (disk_idx >= mddev->raid_disks
1516 || disk_idx < 0)
1517 continue;
1518 disk = conf->mirrors + disk_idx;
1519
1520 disk->rdev = rdev;
1521
1522 blk_queue_stack_limits(mddev->queue,
1523 rdev->bdev->bd_disk->queue);
1524 /* as we don't honour merge_bvec_fn, we must never risk
1525 * violating it, so limit ->max_sector to one PAGE, as
1526 * a one page request is never in violation.
1527 */
1528 if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
1529 mddev->queue->max_sectors > (PAGE_SIZE>>9))
1530 blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
1531
1532 disk->head_position = 0;
1533 if (!test_bit(Faulty, &rdev->flags) && test_bit(In_sync, &rdev->flags))
1534 conf->working_disks++;
1535 }
1536 conf->raid_disks = mddev->raid_disks;
1537 conf->mddev = mddev;
1538 spin_lock_init(&conf->device_lock);
1539 INIT_LIST_HEAD(&conf->retry_list);
1540 if (conf->working_disks == 1)
1541 mddev->recovery_cp = MaxSector;
1542
1543 spin_lock_init(&conf->resync_lock);
1544 init_waitqueue_head(&conf->wait_idle);
1545 init_waitqueue_head(&conf->wait_resume);
1546
1547 bio_list_init(&conf->pending_bio_list);
1548 bio_list_init(&conf->flushing_bio_list);
1549
1550 if (!conf->working_disks) {
1551 printk(KERN_ERR "raid1: no operational mirrors for %s\n",
1552 mdname(mddev));
1553 goto out_free_conf;
1554 }
1555
1556 mddev->degraded = 0;
1557 for (i = 0; i < conf->raid_disks; i++) {
1558
1559 disk = conf->mirrors + i;
1560
1561 if (!disk->rdev) {
1562 disk->head_position = 0;
1563 mddev->degraded++;
1564 }
1565 }
1566
1567 /*
1568 * find the first working one and use it as a starting point
1569 * to read balancing.
1570 */
1571 for (j = 0; j < conf->raid_disks &&
1572 (!conf->mirrors[j].rdev ||
1573 !test_bit(In_sync, &conf->mirrors[j].rdev->flags)) ; j++)
1574 /* nothing */;
1575 conf->last_used = j;
1576
1577
1578 mddev->thread = md_register_thread(raid1d, mddev, "%s_raid1");
1579 if (!mddev->thread) {
1580 printk(KERN_ERR
1581 "raid1: couldn't allocate thread for %s\n",
1582 mdname(mddev));
1583 goto out_free_conf;
1584 }
1585 if (mddev->bitmap) mddev->thread->timeout = mddev->bitmap->daemon_sleep * HZ;
1586
1587 printk(KERN_INFO
1588 "raid1: raid set %s active with %d out of %d mirrors\n",
1589 mdname(mddev), mddev->raid_disks - mddev->degraded,
1590 mddev->raid_disks);
1591 /*
1592 * Ok, everything is just fine now
1593 */
1594 mddev->array_size = mddev->size;
1595
1596 mddev->queue->unplug_fn = raid1_unplug;
1597 mddev->queue->issue_flush_fn = raid1_issue_flush;
1598
1599 return 0;
1600
1601 out_no_mem:
1602 printk(KERN_ERR "raid1: couldn't allocate memory for %s\n",
1603 mdname(mddev));
1604
1605 out_free_conf:
1606 if (conf) {
1607 if (conf->r1bio_pool)
1608 mempool_destroy(conf->r1bio_pool);
1609 kfree(conf->mirrors);
1610 kfree(conf->poolinfo);
1611 kfree(conf);
1612 mddev->private = NULL;
1613 }
1614 out:
1615 return -EIO;
1616 }
1617
1618 static int stop(mddev_t *mddev)
1619 {
1620 conf_t *conf = mddev_to_conf(mddev);
1621 struct bitmap *bitmap = mddev->bitmap;
1622 int behind_wait = 0;
1623
1624 /* wait for behind writes to complete */
1625 while (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
1626 behind_wait++;
1627 printk(KERN_INFO "raid1: behind writes in progress on device %s, waiting to stop (%d)\n", mdname(mddev), behind_wait);
1628 set_current_state(TASK_UNINTERRUPTIBLE);
1629 schedule_timeout(HZ); /* wait a second */
1630 /* need to kick something here to make sure I/O goes? */
1631 }
1632
1633 md_unregister_thread(mddev->thread);
1634 mddev->thread = NULL;
1635 blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
1636 if (conf->r1bio_pool)
1637 mempool_destroy(conf->r1bio_pool);
1638 kfree(conf->mirrors);
1639 kfree(conf->poolinfo);
1640 kfree(conf);
1641 mddev->private = NULL;
1642 return 0;
1643 }
1644
1645 static int raid1_resize(mddev_t *mddev, sector_t sectors)
1646 {
1647 /* no resync is happening, and there is enough space
1648 * on all devices, so we can resize.
1649 * We need to make sure resync covers any new space.
1650 * If the array is shrinking we should possibly wait until
1651 * any io in the removed space completes, but it hardly seems
1652 * worth it.
1653 */
1654 mddev->array_size = sectors>>1;
1655 set_capacity(mddev->gendisk, mddev->array_size << 1);
1656 mddev->changed = 1;
1657 if (mddev->array_size > mddev->size && mddev->recovery_cp == MaxSector) {
1658 mddev->recovery_cp = mddev->size << 1;
1659 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
1660 }
1661 mddev->size = mddev->array_size;
1662 mddev->resync_max_sectors = sectors;
1663 return 0;
1664 }
1665
1666 static int raid1_reshape(mddev_t *mddev, int raid_disks)
1667 {
1668 /* We need to:
1669 * 1/ resize the r1bio_pool
1670 * 2/ resize conf->mirrors
1671 *
1672 * We allocate a new r1bio_pool if we can.
1673 * Then raise a device barrier and wait until all IO stops.
1674 * Then resize conf->mirrors and swap in the new r1bio pool.
1675 *
1676 * At the same time, we "pack" the devices so that all the missing
1677 * devices have the higher raid_disk numbers.
1678 */
1679 mempool_t *newpool, *oldpool;
1680 struct pool_info *newpoolinfo;
1681 mirror_info_t *newmirrors;
1682 conf_t *conf = mddev_to_conf(mddev);
1683 int cnt;
1684
1685 int d, d2;
1686
1687 if (raid_disks < conf->raid_disks) {
1688 cnt=0;
1689 for (d= 0; d < conf->raid_disks; d++)
1690 if (conf->mirrors[d].rdev)
1691 cnt++;
1692 if (cnt > raid_disks)
1693 return -EBUSY;
1694 }
1695
1696 newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL);
1697 if (!newpoolinfo)
1698 return -ENOMEM;
1699 newpoolinfo->mddev = mddev;
1700 newpoolinfo->raid_disks = raid_disks;
1701
1702 newpool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
1703 r1bio_pool_free, newpoolinfo);
1704 if (!newpool) {
1705 kfree(newpoolinfo);
1706 return -ENOMEM;
1707 }
1708 newmirrors = kmalloc(sizeof(struct mirror_info) * raid_disks, GFP_KERNEL);
1709 if (!newmirrors) {
1710 kfree(newpoolinfo);
1711 mempool_destroy(newpool);
1712 return -ENOMEM;
1713 }
1714 memset(newmirrors, 0, sizeof(struct mirror_info)*raid_disks);
1715
1716 spin_lock_irq(&conf->resync_lock);
1717 conf->barrier++;
1718 wait_event_lock_irq(conf->wait_idle, !conf->nr_pending,
1719 conf->resync_lock, raid1_unplug(mddev->queue));
1720 spin_unlock_irq(&conf->resync_lock);
1721
1722 /* ok, everything is stopped */
1723 oldpool = conf->r1bio_pool;
1724 conf->r1bio_pool = newpool;
1725
1726 for (d=d2=0; d < conf->raid_disks; d++)
1727 if (conf->mirrors[d].rdev) {
1728 conf->mirrors[d].rdev->raid_disk = d2;
1729 newmirrors[d2++].rdev = conf->mirrors[d].rdev;
1730 }
1731 kfree(conf->mirrors);
1732 conf->mirrors = newmirrors;
1733 kfree(conf->poolinfo);
1734 conf->poolinfo = newpoolinfo;
1735
1736 mddev->degraded += (raid_disks - conf->raid_disks);
1737 conf->raid_disks = mddev->raid_disks = raid_disks;
1738
1739 conf->last_used = 0; /* just make sure it is in-range */
1740 spin_lock_irq(&conf->resync_lock);
1741 conf->barrier--;
1742 spin_unlock_irq(&conf->resync_lock);
1743 wake_up(&conf->wait_resume);
1744 wake_up(&conf->wait_idle);
1745
1746
1747 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
1748 md_wakeup_thread(mddev->thread);
1749
1750 mempool_destroy(oldpool);
1751 return 0;
1752 }
1753
1754 static void raid1_quiesce(mddev_t *mddev, int state)
1755 {
1756 conf_t *conf = mddev_to_conf(mddev);
1757
1758 switch(state) {
1759 case 1:
1760 spin_lock_irq(&conf->resync_lock);
1761 conf->barrier++;
1762 wait_event_lock_irq(conf->wait_idle, !conf->nr_pending,
1763 conf->resync_lock, raid1_unplug(mddev->queue));
1764 spin_unlock_irq(&conf->resync_lock);
1765 break;
1766 case 0:
1767 spin_lock_irq(&conf->resync_lock);
1768 conf->barrier--;
1769 spin_unlock_irq(&conf->resync_lock);
1770 wake_up(&conf->wait_resume);
1771 wake_up(&conf->wait_idle);
1772 break;
1773 }
1774 if (mddev->thread) {
1775 if (mddev->bitmap)
1776 mddev->thread->timeout = mddev->bitmap->daemon_sleep * HZ;
1777 else
1778 mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1779 md_wakeup_thread(mddev->thread);
1780 }
1781 }
1782
1783
1784 static mdk_personality_t raid1_personality =
1785 {
1786 .name = "raid1",
1787 .owner = THIS_MODULE,
1788 .make_request = make_request,
1789 .run = run,
1790 .stop = stop,
1791 .status = status,
1792 .error_handler = error,
1793 .hot_add_disk = raid1_add_disk,
1794 .hot_remove_disk= raid1_remove_disk,
1795 .spare_active = raid1_spare_active,
1796 .sync_request = sync_request,
1797 .resize = raid1_resize,
1798 .reshape = raid1_reshape,
1799 .quiesce = raid1_quiesce,
1800 };
1801
1802 static int __init raid_init(void)
1803 {
1804 return register_md_personality(RAID1, &raid1_personality);
1805 }
1806
1807 static void raid_exit(void)
1808 {
1809 unregister_md_personality(RAID1);
1810 }
1811
1812 module_init(raid_init);
1813 module_exit(raid_exit);
1814 MODULE_LICENSE("GPL");
1815 MODULE_ALIAS("md-personality-3"); /* RAID1 */
Stable & featurefull patchset based on Linus's git branch