1 // SPDX-License-Identifier: GPL-2.0
3 * Main bcache entry point - handle a read or a write request and decide what to
4 * do with it; the make_request functions are called by the block layer.
6 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
7 * Copyright 2012 Google, Inc.
14 #include "writeback.h"
16 #include <linux/module.h>
17 #include <linux/hash.h>
18 #include <linux/random.h>
19 #include <linux/backing-dev.h>
21 #include <trace/events/bcache.h>
23 #define CUTOFF_CACHE_ADD 95
24 #define CUTOFF_CACHE_READA 90
26 struct kmem_cache
*bch_search_cache
;
28 static void bch_data_insert_start(struct closure
*);
30 static unsigned cache_mode(struct cached_dev
*dc
, struct bio
*bio
)
32 return BDEV_CACHE_MODE(&dc
->sb
);
35 static bool verify(struct cached_dev
*dc
, struct bio
*bio
)
40 static void bio_csum(struct bio
*bio
, struct bkey
*k
)
43 struct bvec_iter iter
;
46 bio_for_each_segment(bv
, bio
, iter
) {
47 void *d
= kmap(bv
.bv_page
) + bv
.bv_offset
;
48 csum
= bch_crc64_update(csum
, d
, bv
.bv_len
);
52 k
->ptr
[KEY_PTRS(k
)] = csum
& (~0ULL >> 1);
55 /* Insert data into cache */
57 static void bch_data_insert_keys(struct closure
*cl
)
59 struct data_insert_op
*op
= container_of(cl
, struct data_insert_op
, cl
);
60 atomic_t
*journal_ref
= NULL
;
61 struct bkey
*replace_key
= op
->replace
? &op
->replace_key
: NULL
;
65 * If we're looping, might already be waiting on
66 * another journal write - can't wait on more than one journal write at
69 * XXX: this looks wrong
72 while (atomic_read(&s
->cl
.remaining
) & CLOSURE_WAITING
)
77 journal_ref
= bch_journal(op
->c
, &op
->insert_keys
,
78 op
->flush_journal
? cl
: NULL
);
80 ret
= bch_btree_insert(op
->c
, &op
->insert_keys
,
81 journal_ref
, replace_key
);
83 op
->replace_collision
= true;
85 op
->status
= BLK_STS_RESOURCE
;
86 op
->insert_data_done
= true;
90 atomic_dec_bug(journal_ref
);
92 if (!op
->insert_data_done
) {
93 continue_at(cl
, bch_data_insert_start
, op
->wq
);
97 bch_keylist_free(&op
->insert_keys
);
101 static int bch_keylist_realloc(struct keylist
*l
, unsigned u64s
,
104 size_t oldsize
= bch_keylist_nkeys(l
);
105 size_t newsize
= oldsize
+ u64s
;
108 * The journalling code doesn't handle the case where the keys to insert
109 * is bigger than an empty write: If we just return -ENOMEM here,
110 * bio_insert() and bio_invalidate() will insert the keys created so far
111 * and finish the rest when the keylist is empty.
113 if (newsize
* sizeof(uint64_t) > block_bytes(c
) - sizeof(struct jset
))
116 return __bch_keylist_realloc(l
, u64s
);
119 static void bch_data_invalidate(struct closure
*cl
)
121 struct data_insert_op
*op
= container_of(cl
, struct data_insert_op
, cl
);
122 struct bio
*bio
= op
->bio
;
124 pr_debug("invalidating %i sectors from %llu",
125 bio_sectors(bio
), (uint64_t) bio
->bi_iter
.bi_sector
);
127 while (bio_sectors(bio
)) {
128 unsigned sectors
= min(bio_sectors(bio
),
129 1U << (KEY_SIZE_BITS
- 1));
131 if (bch_keylist_realloc(&op
->insert_keys
, 2, op
->c
))
134 bio
->bi_iter
.bi_sector
+= sectors
;
135 bio
->bi_iter
.bi_size
-= sectors
<< 9;
137 bch_keylist_add(&op
->insert_keys
,
138 &KEY(op
->inode
, bio
->bi_iter
.bi_sector
, sectors
));
141 op
->insert_data_done
= true;
144 continue_at(cl
, bch_data_insert_keys
, op
->wq
);
147 static void bch_data_insert_error(struct closure
*cl
)
149 struct data_insert_op
*op
= container_of(cl
, struct data_insert_op
, cl
);
152 * Our data write just errored, which means we've got a bunch of keys to
153 * insert that point to data that wasn't succesfully written.
155 * We don't have to insert those keys but we still have to invalidate
156 * that region of the cache - so, if we just strip off all the pointers
157 * from the keys we'll accomplish just that.
160 struct bkey
*src
= op
->insert_keys
.keys
, *dst
= op
->insert_keys
.keys
;
162 while (src
!= op
->insert_keys
.top
) {
163 struct bkey
*n
= bkey_next(src
);
165 SET_KEY_PTRS(src
, 0);
166 memmove(dst
, src
, bkey_bytes(src
));
168 dst
= bkey_next(dst
);
172 op
->insert_keys
.top
= dst
;
174 bch_data_insert_keys(cl
);
177 static void bch_data_insert_endio(struct bio
*bio
)
179 struct closure
*cl
= bio
->bi_private
;
180 struct data_insert_op
*op
= container_of(cl
, struct data_insert_op
, cl
);
182 if (bio
->bi_status
) {
183 /* TODO: We could try to recover from this. */
185 op
->status
= bio
->bi_status
;
186 else if (!op
->replace
)
187 set_closure_fn(cl
, bch_data_insert_error
, op
->wq
);
189 set_closure_fn(cl
, NULL
, NULL
);
192 bch_bbio_endio(op
->c
, bio
, bio
->bi_status
, "writing data to cache");
195 static void bch_data_insert_start(struct closure
*cl
)
197 struct data_insert_op
*op
= container_of(cl
, struct data_insert_op
, cl
);
198 struct bio
*bio
= op
->bio
, *n
;
201 return bch_data_invalidate(cl
);
203 if (atomic_sub_return(bio_sectors(bio
), &op
->c
->sectors_to_gc
) < 0)
207 * Journal writes are marked REQ_PREFLUSH; if the original write was a
208 * flush, it'll wait on the journal write.
210 bio
->bi_opf
&= ~(REQ_PREFLUSH
|REQ_FUA
);
215 struct bio_set
*split
= op
->c
->bio_split
;
217 /* 1 for the device pointer and 1 for the chksum */
218 if (bch_keylist_realloc(&op
->insert_keys
,
219 3 + (op
->csum
? 1 : 0),
221 continue_at(cl
, bch_data_insert_keys
, op
->wq
);
225 k
= op
->insert_keys
.top
;
227 SET_KEY_INODE(k
, op
->inode
);
228 SET_KEY_OFFSET(k
, bio
->bi_iter
.bi_sector
);
230 if (!bch_alloc_sectors(op
->c
, k
, bio_sectors(bio
),
231 op
->write_point
, op
->write_prio
,
235 n
= bio_next_split(bio
, KEY_SIZE(k
), GFP_NOIO
, split
);
237 n
->bi_end_io
= bch_data_insert_endio
;
241 SET_KEY_DIRTY(k
, true);
243 for (i
= 0; i
< KEY_PTRS(k
); i
++)
244 SET_GC_MARK(PTR_BUCKET(op
->c
, k
, i
),
248 SET_KEY_CSUM(k
, op
->csum
);
252 trace_bcache_cache_insert(k
);
253 bch_keylist_push(&op
->insert_keys
);
255 bio_set_op_attrs(n
, REQ_OP_WRITE
, 0);
256 bch_submit_bbio(n
, op
->c
, k
, 0);
259 op
->insert_data_done
= true;
260 continue_at(cl
, bch_data_insert_keys
, op
->wq
);
263 /* bch_alloc_sectors() blocks if s->writeback = true */
264 BUG_ON(op
->writeback
);
267 * But if it's not a writeback write we'd rather just bail out if
268 * there aren't any buckets ready to write to - it might take awhile and
269 * we might be starving btree writes for gc or something.
274 * Writethrough write: We can't complete the write until we've
275 * updated the index. But we don't want to delay the write while
276 * we wait for buckets to be freed up, so just invalidate the
280 return bch_data_invalidate(cl
);
283 * From a cache miss, we can just insert the keys for the data
284 * we have written or bail out if we didn't do anything.
286 op
->insert_data_done
= true;
289 if (!bch_keylist_empty(&op
->insert_keys
))
290 continue_at(cl
, bch_data_insert_keys
, op
->wq
);
297 * bch_data_insert - stick some data in the cache
299 * This is the starting point for any data to end up in a cache device; it could
300 * be from a normal write, or a writeback write, or a write to a flash only
301 * volume - it's also used by the moving garbage collector to compact data in
302 * mostly empty buckets.
304 * It first writes the data to the cache, creating a list of keys to be inserted
305 * (if the data had to be fragmented there will be multiple keys); after the
306 * data is written it calls bch_journal, and after the keys have been added to
307 * the next journal write they're inserted into the btree.
309 * It inserts the data in s->cache_bio; bi_sector is used for the key offset,
310 * and op->inode is used for the key inode.
312 * If s->bypass is true, instead of inserting the data it invalidates the
313 * region of the cache represented by s->cache_bio and op->inode.
315 void bch_data_insert(struct closure
*cl
)
317 struct data_insert_op
*op
= container_of(cl
, struct data_insert_op
, cl
);
319 trace_bcache_write(op
->c
, op
->inode
, op
->bio
,
320 op
->writeback
, op
->bypass
);
322 bch_keylist_init(&op
->insert_keys
);
324 bch_data_insert_start(cl
);
329 unsigned bch_get_congested(struct cache_set
*c
)
334 if (!c
->congested_read_threshold_us
&&
335 !c
->congested_write_threshold_us
)
338 i
= (local_clock_us() - c
->congested_last_us
) / 1024;
342 i
+= atomic_read(&c
->congested
);
349 i
= fract_exp_two(i
, 6);
351 rand
= get_random_int();
352 i
-= bitmap_weight(&rand
, BITS_PER_LONG
);
354 return i
> 0 ? i
: 1;
357 static void add_sequential(struct task_struct
*t
)
359 ewma_add(t
->sequential_io_avg
,
360 t
->sequential_io
, 8, 0);
362 t
->sequential_io
= 0;
365 static struct hlist_head
*iohash(struct cached_dev
*dc
, uint64_t k
)
367 return &dc
->io_hash
[hash_64(k
, RECENT_IO_BITS
)];
370 static bool check_should_bypass(struct cached_dev
*dc
, struct bio
*bio
)
372 struct cache_set
*c
= dc
->disk
.c
;
373 unsigned mode
= cache_mode(dc
, bio
);
374 unsigned sectors
, congested
= bch_get_congested(c
);
375 struct task_struct
*task
= current
;
378 if (test_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
) ||
379 c
->gc_stats
.in_use
> CUTOFF_CACHE_ADD
||
380 (bio_op(bio
) == REQ_OP_DISCARD
))
383 if (mode
== CACHE_MODE_NONE
||
384 (mode
== CACHE_MODE_WRITEAROUND
&&
385 op_is_write(bio_op(bio
))))
388 if (bio
->bi_iter
.bi_sector
& (c
->sb
.block_size
- 1) ||
389 bio_sectors(bio
) & (c
->sb
.block_size
- 1)) {
390 pr_debug("skipping unaligned io");
394 if (bypass_torture_test(dc
)) {
395 if ((get_random_int() & 3) == 3)
401 if (!congested
&& !dc
->sequential_cutoff
)
404 spin_lock(&dc
->io_lock
);
406 hlist_for_each_entry(i
, iohash(dc
, bio
->bi_iter
.bi_sector
), hash
)
407 if (i
->last
== bio
->bi_iter
.bi_sector
&&
408 time_before(jiffies
, i
->jiffies
))
411 i
= list_first_entry(&dc
->io_lru
, struct io
, lru
);
413 add_sequential(task
);
416 if (i
->sequential
+ bio
->bi_iter
.bi_size
> i
->sequential
)
417 i
->sequential
+= bio
->bi_iter
.bi_size
;
419 i
->last
= bio_end_sector(bio
);
420 i
->jiffies
= jiffies
+ msecs_to_jiffies(5000);
421 task
->sequential_io
= i
->sequential
;
424 hlist_add_head(&i
->hash
, iohash(dc
, i
->last
));
425 list_move_tail(&i
->lru
, &dc
->io_lru
);
427 spin_unlock(&dc
->io_lock
);
429 sectors
= max(task
->sequential_io
,
430 task
->sequential_io_avg
) >> 9;
432 if (dc
->sequential_cutoff
&&
433 sectors
>= dc
->sequential_cutoff
>> 9) {
434 trace_bcache_bypass_sequential(bio
);
438 if (congested
&& sectors
>= congested
) {
439 trace_bcache_bypass_congested(bio
);
444 bch_rescale_priorities(c
, bio_sectors(bio
));
447 bch_mark_sectors_bypassed(c
, dc
, bio_sectors(bio
));
454 /* Stack frame for bio_complete */
458 struct bio
*orig_bio
;
459 struct bio
*cache_miss
;
460 struct bcache_device
*d
;
462 unsigned insert_bio_sectors
;
463 unsigned recoverable
:1;
465 unsigned read_dirty_data
:1;
467 unsigned long start_time
;
470 struct data_insert_op iop
;
473 static void bch_cache_read_endio(struct bio
*bio
)
475 struct bbio
*b
= container_of(bio
, struct bbio
, bio
);
476 struct closure
*cl
= bio
->bi_private
;
477 struct search
*s
= container_of(cl
, struct search
, cl
);
480 * If the bucket was reused while our bio was in flight, we might have
481 * read the wrong data. Set s->error but not error so it doesn't get
482 * counted against the cache device, but we'll still reread the data
483 * from the backing device.
487 s
->iop
.status
= bio
->bi_status
;
488 else if (!KEY_DIRTY(&b
->key
) &&
489 ptr_stale(s
->iop
.c
, &b
->key
, 0)) {
490 atomic_long_inc(&s
->iop
.c
->cache_read_races
);
491 s
->iop
.status
= BLK_STS_IOERR
;
494 bch_bbio_endio(s
->iop
.c
, bio
, bio
->bi_status
, "reading from cache");
498 * Read from a single key, handling the initial cache miss if the key starts in
499 * the middle of the bio
501 static int cache_lookup_fn(struct btree_op
*op
, struct btree
*b
, struct bkey
*k
)
503 struct search
*s
= container_of(op
, struct search
, op
);
504 struct bio
*n
, *bio
= &s
->bio
.bio
;
505 struct bkey
*bio_key
;
508 if (bkey_cmp(k
, &KEY(s
->iop
.inode
, bio
->bi_iter
.bi_sector
, 0)) <= 0)
511 if (KEY_INODE(k
) != s
->iop
.inode
||
512 KEY_START(k
) > bio
->bi_iter
.bi_sector
) {
513 unsigned bio_sectors
= bio_sectors(bio
);
514 unsigned sectors
= KEY_INODE(k
) == s
->iop
.inode
515 ? min_t(uint64_t, INT_MAX
,
516 KEY_START(k
) - bio
->bi_iter
.bi_sector
)
519 int ret
= s
->d
->cache_miss(b
, s
, bio
, sectors
);
520 if (ret
!= MAP_CONTINUE
)
523 /* if this was a complete miss we shouldn't get here */
524 BUG_ON(bio_sectors
<= sectors
);
530 /* XXX: figure out best pointer - for multiple cache devices */
533 PTR_BUCKET(b
->c
, k
, ptr
)->prio
= INITIAL_PRIO
;
536 s
->read_dirty_data
= true;
538 n
= bio_next_split(bio
, min_t(uint64_t, INT_MAX
,
539 KEY_OFFSET(k
) - bio
->bi_iter
.bi_sector
),
540 GFP_NOIO
, s
->d
->bio_split
);
542 bio_key
= &container_of(n
, struct bbio
, bio
)->key
;
543 bch_bkey_copy_single_ptr(bio_key
, k
, ptr
);
545 bch_cut_front(&KEY(s
->iop
.inode
, n
->bi_iter
.bi_sector
, 0), bio_key
);
546 bch_cut_back(&KEY(s
->iop
.inode
, bio_end_sector(n
), 0), bio_key
);
548 n
->bi_end_io
= bch_cache_read_endio
;
549 n
->bi_private
= &s
->cl
;
552 * The bucket we're reading from might be reused while our bio
553 * is in flight, and we could then end up reading the wrong
556 * We guard against this by checking (in cache_read_endio()) if
557 * the pointer is stale again; if so, we treat it as an error
558 * and reread from the backing device (but we don't pass that
559 * error up anywhere).
562 __bch_submit_bbio(n
, b
->c
);
563 return n
== bio
? MAP_DONE
: MAP_CONTINUE
;
566 static void cache_lookup(struct closure
*cl
)
568 struct search
*s
= container_of(cl
, struct search
, iop
.cl
);
569 struct bio
*bio
= &s
->bio
.bio
;
572 bch_btree_op_init(&s
->op
, -1);
574 ret
= bch_btree_map_keys(&s
->op
, s
->iop
.c
,
575 &KEY(s
->iop
.inode
, bio
->bi_iter
.bi_sector
, 0),
576 cache_lookup_fn
, MAP_END_KEY
);
577 if (ret
== -EAGAIN
) {
578 continue_at(cl
, cache_lookup
, bcache_wq
);
585 /* Common code for the make_request functions */
587 static void request_endio(struct bio
*bio
)
589 struct closure
*cl
= bio
->bi_private
;
591 if (bio
->bi_status
) {
592 struct search
*s
= container_of(cl
, struct search
, cl
);
593 s
->iop
.status
= bio
->bi_status
;
594 /* Only cache read errors are recoverable */
595 s
->recoverable
= false;
602 static void bio_complete(struct search
*s
)
605 struct request_queue
*q
= s
->orig_bio
->bi_disk
->queue
;
606 generic_end_io_acct(q
, bio_data_dir(s
->orig_bio
),
607 &s
->d
->disk
->part0
, s
->start_time
);
609 trace_bcache_request_end(s
->d
, s
->orig_bio
);
610 s
->orig_bio
->bi_status
= s
->iop
.status
;
611 bio_endio(s
->orig_bio
);
616 static void do_bio_hook(struct search
*s
, struct bio
*orig_bio
)
618 struct bio
*bio
= &s
->bio
.bio
;
620 bio_init(bio
, NULL
, 0);
621 __bio_clone_fast(bio
, orig_bio
);
622 bio
->bi_end_io
= request_endio
;
623 bio
->bi_private
= &s
->cl
;
628 static void search_free(struct closure
*cl
)
630 struct search
*s
= container_of(cl
, struct search
, cl
);
636 closure_debug_destroy(cl
);
637 mempool_free(s
, s
->d
->c
->search
);
640 static inline struct search
*search_alloc(struct bio
*bio
,
641 struct bcache_device
*d
)
645 s
= mempool_alloc(d
->c
->search
, GFP_NOIO
);
647 closure_init(&s
->cl
, NULL
);
651 s
->cache_miss
= NULL
;
654 s
->write
= op_is_write(bio_op(bio
));
655 s
->read_dirty_data
= 0;
656 s
->start_time
= jiffies
;
660 s
->iop
.inode
= d
->id
;
661 s
->iop
.write_point
= hash_long((unsigned long) current
, 16);
662 s
->iop
.write_prio
= 0;
665 s
->iop
.flush_journal
= op_is_flush(bio
->bi_opf
);
666 s
->iop
.wq
= bcache_wq
;
673 static void cached_dev_bio_complete(struct closure
*cl
)
675 struct search
*s
= container_of(cl
, struct search
, cl
);
676 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
684 static void cached_dev_cache_miss_done(struct closure
*cl
)
686 struct search
*s
= container_of(cl
, struct search
, cl
);
688 if (s
->iop
.replace_collision
)
689 bch_mark_cache_miss_collision(s
->iop
.c
, s
->d
);
692 bio_free_pages(s
->iop
.bio
);
694 cached_dev_bio_complete(cl
);
697 static void cached_dev_read_error(struct closure
*cl
)
699 struct search
*s
= container_of(cl
, struct search
, cl
);
700 struct bio
*bio
= &s
->bio
.bio
;
702 if (s
->recoverable
) {
703 /* Retry from the backing device: */
704 trace_bcache_read_retry(s
->orig_bio
);
707 do_bio_hook(s
, s
->orig_bio
);
709 /* XXX: invalidate cache */
711 closure_bio_submit(bio
, cl
);
714 continue_at(cl
, cached_dev_cache_miss_done
, NULL
);
717 static void cached_dev_read_done(struct closure
*cl
)
719 struct search
*s
= container_of(cl
, struct search
, cl
);
720 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
723 * We had a cache miss; cache_bio now contains data ready to be inserted
726 * First, we copy the data we just read from cache_bio's bounce buffers
727 * to the buffers the original bio pointed to:
731 bio_reset(s
->iop
.bio
);
732 s
->iop
.bio
->bi_iter
.bi_sector
= s
->cache_miss
->bi_iter
.bi_sector
;
733 bio_copy_dev(s
->iop
.bio
, s
->cache_miss
);
734 s
->iop
.bio
->bi_iter
.bi_size
= s
->insert_bio_sectors
<< 9;
735 bch_bio_map(s
->iop
.bio
, NULL
);
737 bio_copy_data(s
->cache_miss
, s
->iop
.bio
);
739 bio_put(s
->cache_miss
);
740 s
->cache_miss
= NULL
;
743 if (verify(dc
, &s
->bio
.bio
) && s
->recoverable
&& !s
->read_dirty_data
)
744 bch_data_verify(dc
, s
->orig_bio
);
749 !test_bit(CACHE_SET_STOPPING
, &s
->iop
.c
->flags
)) {
750 BUG_ON(!s
->iop
.replace
);
751 closure_call(&s
->iop
.cl
, bch_data_insert
, NULL
, cl
);
754 continue_at(cl
, cached_dev_cache_miss_done
, NULL
);
757 static void cached_dev_read_done_bh(struct closure
*cl
)
759 struct search
*s
= container_of(cl
, struct search
, cl
);
760 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
762 bch_mark_cache_accounting(s
->iop
.c
, s
->d
,
763 !s
->cache_miss
, s
->iop
.bypass
);
764 trace_bcache_read(s
->orig_bio
, !s
->cache_miss
, s
->iop
.bypass
);
767 continue_at_nobarrier(cl
, cached_dev_read_error
, bcache_wq
);
768 else if (s
->iop
.bio
|| verify(dc
, &s
->bio
.bio
))
769 continue_at_nobarrier(cl
, cached_dev_read_done
, bcache_wq
);
771 continue_at_nobarrier(cl
, cached_dev_bio_complete
, NULL
);
774 static int cached_dev_cache_miss(struct btree
*b
, struct search
*s
,
775 struct bio
*bio
, unsigned sectors
)
777 int ret
= MAP_CONTINUE
;
779 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
780 struct bio
*miss
, *cache_bio
;
782 if (s
->cache_miss
|| s
->iop
.bypass
) {
783 miss
= bio_next_split(bio
, sectors
, GFP_NOIO
, s
->d
->bio_split
);
784 ret
= miss
== bio
? MAP_DONE
: MAP_CONTINUE
;
788 if (!(bio
->bi_opf
& REQ_RAHEAD
) &&
789 !(bio
->bi_opf
& REQ_META
) &&
790 s
->iop
.c
->gc_stats
.in_use
< CUTOFF_CACHE_READA
)
791 reada
= min_t(sector_t
, dc
->readahead
>> 9,
792 get_capacity(bio
->bi_disk
) - bio_end_sector(bio
));
794 s
->insert_bio_sectors
= min(sectors
, bio_sectors(bio
) + reada
);
796 s
->iop
.replace_key
= KEY(s
->iop
.inode
,
797 bio
->bi_iter
.bi_sector
+ s
->insert_bio_sectors
,
798 s
->insert_bio_sectors
);
800 ret
= bch_btree_insert_check_key(b
, &s
->op
, &s
->iop
.replace_key
);
804 s
->iop
.replace
= true;
806 miss
= bio_next_split(bio
, sectors
, GFP_NOIO
, s
->d
->bio_split
);
808 /* btree_search_recurse()'s btree iterator is no good anymore */
809 ret
= miss
== bio
? MAP_DONE
: -EINTR
;
811 cache_bio
= bio_alloc_bioset(GFP_NOWAIT
,
812 DIV_ROUND_UP(s
->insert_bio_sectors
, PAGE_SECTORS
),
817 cache_bio
->bi_iter
.bi_sector
= miss
->bi_iter
.bi_sector
;
818 bio_copy_dev(cache_bio
, miss
);
819 cache_bio
->bi_iter
.bi_size
= s
->insert_bio_sectors
<< 9;
821 cache_bio
->bi_end_io
= request_endio
;
822 cache_bio
->bi_private
= &s
->cl
;
824 bch_bio_map(cache_bio
, NULL
);
825 if (bio_alloc_pages(cache_bio
, __GFP_NOWARN
|GFP_NOIO
))
829 bch_mark_cache_readahead(s
->iop
.c
, s
->d
);
831 s
->cache_miss
= miss
;
832 s
->iop
.bio
= cache_bio
;
834 closure_bio_submit(cache_bio
, &s
->cl
);
840 miss
->bi_end_io
= request_endio
;
841 miss
->bi_private
= &s
->cl
;
842 closure_bio_submit(miss
, &s
->cl
);
846 static void cached_dev_read(struct cached_dev
*dc
, struct search
*s
)
848 struct closure
*cl
= &s
->cl
;
850 closure_call(&s
->iop
.cl
, cache_lookup
, NULL
, cl
);
851 continue_at(cl
, cached_dev_read_done_bh
, NULL
);
856 static void cached_dev_write_complete(struct closure
*cl
)
858 struct search
*s
= container_of(cl
, struct search
, cl
);
859 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
861 up_read_non_owner(&dc
->writeback_lock
);
862 cached_dev_bio_complete(cl
);
865 static void cached_dev_write(struct cached_dev
*dc
, struct search
*s
)
867 struct closure
*cl
= &s
->cl
;
868 struct bio
*bio
= &s
->bio
.bio
;
869 struct bkey start
= KEY(dc
->disk
.id
, bio
->bi_iter
.bi_sector
, 0);
870 struct bkey end
= KEY(dc
->disk
.id
, bio_end_sector(bio
), 0);
872 bch_keybuf_check_overlapping(&s
->iop
.c
->moving_gc_keys
, &start
, &end
);
874 down_read_non_owner(&dc
->writeback_lock
);
875 if (bch_keybuf_check_overlapping(&dc
->writeback_keys
, &start
, &end
)) {
877 * We overlap with some dirty data undergoing background
878 * writeback, force this write to writeback
880 s
->iop
.bypass
= false;
881 s
->iop
.writeback
= true;
885 * Discards aren't _required_ to do anything, so skipping if
886 * check_overlapping returned true is ok
888 * But check_overlapping drops dirty keys for which io hasn't started,
889 * so we still want to call it.
891 if (bio_op(bio
) == REQ_OP_DISCARD
)
892 s
->iop
.bypass
= true;
894 if (should_writeback(dc
, s
->orig_bio
,
897 s
->iop
.bypass
= false;
898 s
->iop
.writeback
= true;
902 s
->iop
.bio
= s
->orig_bio
;
905 if ((bio_op(bio
) != REQ_OP_DISCARD
) ||
906 blk_queue_discard(bdev_get_queue(dc
->bdev
)))
907 closure_bio_submit(bio
, cl
);
908 } else if (s
->iop
.writeback
) {
909 bch_writeback_add(dc
);
912 if (bio
->bi_opf
& REQ_PREFLUSH
) {
913 /* Also need to send a flush to the backing device */
914 struct bio
*flush
= bio_alloc_bioset(GFP_NOIO
, 0,
917 bio_copy_dev(flush
, bio
);
918 flush
->bi_end_io
= request_endio
;
919 flush
->bi_private
= cl
;
920 flush
->bi_opf
= REQ_OP_WRITE
| REQ_PREFLUSH
;
922 closure_bio_submit(flush
, cl
);
925 s
->iop
.bio
= bio_clone_fast(bio
, GFP_NOIO
, dc
->disk
.bio_split
);
927 closure_bio_submit(bio
, cl
);
930 closure_call(&s
->iop
.cl
, bch_data_insert
, NULL
, cl
);
931 continue_at(cl
, cached_dev_write_complete
, NULL
);
934 static void cached_dev_nodata(struct closure
*cl
)
936 struct search
*s
= container_of(cl
, struct search
, cl
);
937 struct bio
*bio
= &s
->bio
.bio
;
939 if (s
->iop
.flush_journal
)
940 bch_journal_meta(s
->iop
.c
, cl
);
942 /* If it's a flush, we send the flush to the backing device too */
943 closure_bio_submit(bio
, cl
);
945 continue_at(cl
, cached_dev_bio_complete
, NULL
);
948 /* Cached devices - read & write stuff */
950 static blk_qc_t
cached_dev_make_request(struct request_queue
*q
,
954 struct bcache_device
*d
= bio
->bi_disk
->private_data
;
955 struct cached_dev
*dc
= container_of(d
, struct cached_dev
, disk
);
956 int rw
= bio_data_dir(bio
);
958 generic_start_io_acct(q
, rw
, bio_sectors(bio
), &d
->disk
->part0
);
960 bio_set_dev(bio
, dc
->bdev
);
961 bio
->bi_iter
.bi_sector
+= dc
->sb
.data_offset
;
963 if (cached_dev_get(dc
)) {
964 s
= search_alloc(bio
, d
);
965 trace_bcache_request_start(s
->d
, bio
);
967 if (!bio
->bi_iter
.bi_size
) {
969 * can't call bch_journal_meta from under
970 * generic_make_request
972 continue_at_nobarrier(&s
->cl
,
976 s
->iop
.bypass
= check_should_bypass(dc
, bio
);
979 cached_dev_write(dc
, s
);
981 cached_dev_read(dc
, s
);
984 if ((bio_op(bio
) == REQ_OP_DISCARD
) &&
985 !blk_queue_discard(bdev_get_queue(dc
->bdev
)))
988 generic_make_request(bio
);
991 return BLK_QC_T_NONE
;
994 static int cached_dev_ioctl(struct bcache_device
*d
, fmode_t mode
,
995 unsigned int cmd
, unsigned long arg
)
997 struct cached_dev
*dc
= container_of(d
, struct cached_dev
, disk
);
998 return __blkdev_driver_ioctl(dc
->bdev
, mode
, cmd
, arg
);
1001 static int cached_dev_congested(void *data
, int bits
)
1003 struct bcache_device
*d
= data
;
1004 struct cached_dev
*dc
= container_of(d
, struct cached_dev
, disk
);
1005 struct request_queue
*q
= bdev_get_queue(dc
->bdev
);
1008 if (bdi_congested(q
->backing_dev_info
, bits
))
1011 if (cached_dev_get(dc
)) {
1015 for_each_cache(ca
, d
->c
, i
) {
1016 q
= bdev_get_queue(ca
->bdev
);
1017 ret
|= bdi_congested(q
->backing_dev_info
, bits
);
1026 void bch_cached_dev_request_init(struct cached_dev
*dc
)
1028 struct gendisk
*g
= dc
->disk
.disk
;
1030 g
->queue
->make_request_fn
= cached_dev_make_request
;
1031 g
->queue
->backing_dev_info
->congested_fn
= cached_dev_congested
;
1032 dc
->disk
.cache_miss
= cached_dev_cache_miss
;
1033 dc
->disk
.ioctl
= cached_dev_ioctl
;
1036 /* Flash backed devices */
1038 static int flash_dev_cache_miss(struct btree
*b
, struct search
*s
,
1039 struct bio
*bio
, unsigned sectors
)
1041 unsigned bytes
= min(sectors
, bio_sectors(bio
)) << 9;
1043 swap(bio
->bi_iter
.bi_size
, bytes
);
1045 swap(bio
->bi_iter
.bi_size
, bytes
);
1047 bio_advance(bio
, bytes
);
1049 if (!bio
->bi_iter
.bi_size
)
1052 return MAP_CONTINUE
;
1055 static void flash_dev_nodata(struct closure
*cl
)
1057 struct search
*s
= container_of(cl
, struct search
, cl
);
1059 if (s
->iop
.flush_journal
)
1060 bch_journal_meta(s
->iop
.c
, cl
);
1062 continue_at(cl
, search_free
, NULL
);
1065 static blk_qc_t
flash_dev_make_request(struct request_queue
*q
,
1070 struct bcache_device
*d
= bio
->bi_disk
->private_data
;
1071 int rw
= bio_data_dir(bio
);
1073 generic_start_io_acct(q
, rw
, bio_sectors(bio
), &d
->disk
->part0
);
1075 s
= search_alloc(bio
, d
);
1079 trace_bcache_request_start(s
->d
, bio
);
1081 if (!bio
->bi_iter
.bi_size
) {
1083 * can't call bch_journal_meta from under
1084 * generic_make_request
1086 continue_at_nobarrier(&s
->cl
,
1089 return BLK_QC_T_NONE
;
1091 bch_keybuf_check_overlapping(&s
->iop
.c
->moving_gc_keys
,
1092 &KEY(d
->id
, bio
->bi_iter
.bi_sector
, 0),
1093 &KEY(d
->id
, bio_end_sector(bio
), 0));
1095 s
->iop
.bypass
= (bio_op(bio
) == REQ_OP_DISCARD
) != 0;
1096 s
->iop
.writeback
= true;
1099 closure_call(&s
->iop
.cl
, bch_data_insert
, NULL
, cl
);
1101 closure_call(&s
->iop
.cl
, cache_lookup
, NULL
, cl
);
1104 continue_at(cl
, search_free
, NULL
);
1105 return BLK_QC_T_NONE
;
1108 static int flash_dev_ioctl(struct bcache_device
*d
, fmode_t mode
,
1109 unsigned int cmd
, unsigned long arg
)
1114 static int flash_dev_congested(void *data
, int bits
)
1116 struct bcache_device
*d
= data
;
1117 struct request_queue
*q
;
1122 for_each_cache(ca
, d
->c
, i
) {
1123 q
= bdev_get_queue(ca
->bdev
);
1124 ret
|= bdi_congested(q
->backing_dev_info
, bits
);
1130 void bch_flash_dev_request_init(struct bcache_device
*d
)
1132 struct gendisk
*g
= d
->disk
;
1134 g
->queue
->make_request_fn
= flash_dev_make_request
;
1135 g
->queue
->backing_dev_info
->congested_fn
= flash_dev_congested
;
1136 d
->cache_miss
= flash_dev_cache_miss
;
1137 d
->ioctl
= flash_dev_ioctl
;
1140 void bch_request_exit(void)
1142 if (bch_search_cache
)
1143 kmem_cache_destroy(bch_search_cache
);
1146 int __init
bch_request_init(void)
1148 bch_search_cache
= KMEM_CACHE(search
, 0);
1149 if (!bch_search_cache
)