2 * linux/drivers/block/ll_rw_blk.c
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 1994, Karl Keyte: Added support for disk statistics
6 * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
7 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
8 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au> - July2000
9 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
13 * This handles all read/write requests to block devices
15 #include <linux/config.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18 #include <linux/backing-dev.h>
19 #include <linux/bio.h>
20 #include <linux/blkdev.h>
21 #include <linux/highmem.h>
23 #include <linux/kernel_stat.h>
24 #include <linux/string.h>
25 #include <linux/init.h>
26 #include <linux/bootmem.h> /* for max_pfn/max_low_pfn */
27 #include <linux/completion.h>
28 #include <linux/slab.h>
29 #include <linux/swap.h>
30 #include <linux/writeback.h>
35 #include <scsi/scsi_cmnd.h>
37 static void blk_unplug_work(void *data
);
38 static void blk_unplug_timeout(unsigned long data
);
41 * For the allocated request tables
43 static kmem_cache_t
*request_cachep
;
46 * For queue allocation
48 static kmem_cache_t
*requestq_cachep
;
51 * For io context allocations
53 static kmem_cache_t
*iocontext_cachep
;
55 static wait_queue_head_t congestion_wqh
[2] = {
56 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh
[0]),
57 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh
[1])
61 * Controlling structure to kblockd
63 static struct workqueue_struct
*kblockd_workqueue
;
65 unsigned long blk_max_low_pfn
, blk_max_pfn
;
67 EXPORT_SYMBOL(blk_max_low_pfn
);
68 EXPORT_SYMBOL(blk_max_pfn
);
70 /* Amount of time in which a process may batch requests */
71 #define BLK_BATCH_TIME (HZ/50UL)
73 /* Number of requests a "batching" process may submit */
74 #define BLK_BATCH_REQ 32
77 * Return the threshold (number of used requests) at which the queue is
78 * considered to be congested. It include a little hysteresis to keep the
79 * context switch rate down.
81 static inline int queue_congestion_on_threshold(struct request_queue
*q
)
83 return q
->nr_congestion_on
;
87 * The threshold at which a queue is considered to be uncongested
89 static inline int queue_congestion_off_threshold(struct request_queue
*q
)
91 return q
->nr_congestion_off
;
94 static void blk_queue_congestion_threshold(struct request_queue
*q
)
98 nr
= q
->nr_requests
- (q
->nr_requests
/ 8) + 1;
99 if (nr
> q
->nr_requests
)
101 q
->nr_congestion_on
= nr
;
103 nr
= q
->nr_requests
- (q
->nr_requests
/ 8) - 1;
106 q
->nr_congestion_off
= nr
;
110 * A queue has just exitted congestion. Note this in the global counter of
111 * congested queues, and wake up anyone who was waiting for requests to be
114 static void clear_queue_congested(request_queue_t
*q
, int rw
)
117 wait_queue_head_t
*wqh
= &congestion_wqh
[rw
];
119 bit
= (rw
== WRITE
) ? BDI_write_congested
: BDI_read_congested
;
120 clear_bit(bit
, &q
->backing_dev_info
.state
);
121 smp_mb__after_clear_bit();
122 if (waitqueue_active(wqh
))
127 * A queue has just entered congestion. Flag that in the queue's VM-visible
128 * state flags and increment the global gounter of congested queues.
130 static void set_queue_congested(request_queue_t
*q
, int rw
)
134 bit
= (rw
== WRITE
) ? BDI_write_congested
: BDI_read_congested
;
135 set_bit(bit
, &q
->backing_dev_info
.state
);
139 * blk_get_backing_dev_info - get the address of a queue's backing_dev_info
142 * Locates the passed device's request queue and returns the address of its
145 * Will return NULL if the request queue cannot be located.
147 struct backing_dev_info
*blk_get_backing_dev_info(struct block_device
*bdev
)
149 struct backing_dev_info
*ret
= NULL
;
150 request_queue_t
*q
= bdev_get_queue(bdev
);
153 ret
= &q
->backing_dev_info
;
157 EXPORT_SYMBOL(blk_get_backing_dev_info
);
159 void blk_queue_activity_fn(request_queue_t
*q
, activity_fn
*fn
, void *data
)
162 q
->activity_data
= data
;
165 EXPORT_SYMBOL(blk_queue_activity_fn
);
168 * blk_queue_prep_rq - set a prepare_request function for queue
170 * @pfn: prepare_request function
172 * It's possible for a queue to register a prepare_request callback which
173 * is invoked before the request is handed to the request_fn. The goal of
174 * the function is to prepare a request for I/O, it can be used to build a
175 * cdb from the request data for instance.
178 void blk_queue_prep_rq(request_queue_t
*q
, prep_rq_fn
*pfn
)
183 EXPORT_SYMBOL(blk_queue_prep_rq
);
186 * blk_queue_merge_bvec - set a merge_bvec function for queue
188 * @mbfn: merge_bvec_fn
190 * Usually queues have static limitations on the max sectors or segments that
191 * we can put in a request. Stacking drivers may have some settings that
192 * are dynamic, and thus we have to query the queue whether it is ok to
193 * add a new bio_vec to a bio at a given offset or not. If the block device
194 * has such limitations, it needs to register a merge_bvec_fn to control
195 * the size of bio's sent to it. Note that a block device *must* allow a
196 * single page to be added to an empty bio. The block device driver may want
197 * to use the bio_split() function to deal with these bio's. By default
198 * no merge_bvec_fn is defined for a queue, and only the fixed limits are
201 void blk_queue_merge_bvec(request_queue_t
*q
, merge_bvec_fn
*mbfn
)
203 q
->merge_bvec_fn
= mbfn
;
206 EXPORT_SYMBOL(blk_queue_merge_bvec
);
209 * blk_queue_make_request - define an alternate make_request function for a device
210 * @q: the request queue for the device to be affected
211 * @mfn: the alternate make_request function
214 * The normal way for &struct bios to be passed to a device
215 * driver is for them to be collected into requests on a request
216 * queue, and then to allow the device driver to select requests
217 * off that queue when it is ready. This works well for many block
218 * devices. However some block devices (typically virtual devices
219 * such as md or lvm) do not benefit from the processing on the
220 * request queue, and are served best by having the requests passed
221 * directly to them. This can be achieved by providing a function
222 * to blk_queue_make_request().
225 * The driver that does this *must* be able to deal appropriately
226 * with buffers in "highmemory". This can be accomplished by either calling
227 * __bio_kmap_atomic() to get a temporary kernel mapping, or by calling
228 * blk_queue_bounce() to create a buffer in normal memory.
230 void blk_queue_make_request(request_queue_t
* q
, make_request_fn
* mfn
)
235 q
->nr_requests
= BLKDEV_MAX_RQ
;
236 q
->max_phys_segments
= MAX_PHYS_SEGMENTS
;
237 q
->max_hw_segments
= MAX_HW_SEGMENTS
;
238 q
->make_request_fn
= mfn
;
239 q
->backing_dev_info
.ra_pages
= (VM_MAX_READAHEAD
* 1024) / PAGE_CACHE_SIZE
;
240 q
->backing_dev_info
.state
= 0;
241 q
->backing_dev_info
.memory_backed
= 0;
242 blk_queue_max_sectors(q
, MAX_SECTORS
);
243 blk_queue_hardsect_size(q
, 512);
244 blk_queue_dma_alignment(q
, 511);
245 blk_queue_congestion_threshold(q
);
247 q
->unplug_thresh
= 4; /* hmm */
248 q
->unplug_delay
= (3 * HZ
) / 1000; /* 3 milliseconds */
249 if (q
->unplug_delay
== 0)
252 INIT_WORK(&q
->unplug_work
, blk_unplug_work
, q
);
254 q
->unplug_timer
.function
= blk_unplug_timeout
;
255 q
->unplug_timer
.data
= (unsigned long)q
;
258 * by default assume old behaviour and bounce for any highmem page
260 blk_queue_bounce_limit(q
, BLK_BOUNCE_HIGH
);
262 blk_queue_activity_fn(q
, NULL
, NULL
);
265 EXPORT_SYMBOL(blk_queue_make_request
);
268 * blk_queue_ordered - does this queue support ordered writes
269 * @q: the request queue
273 * For journalled file systems, doing ordered writes on a commit
274 * block instead of explicitly doing wait_on_buffer (which is bad
275 * for performance) can be a big win. Block drivers supporting this
276 * feature should call this function and indicate so.
279 void blk_queue_ordered(request_queue_t
*q
, int flag
)
282 set_bit(QUEUE_FLAG_ORDERED
, &q
->queue_flags
);
284 clear_bit(QUEUE_FLAG_ORDERED
, &q
->queue_flags
);
287 EXPORT_SYMBOL(blk_queue_ordered
);
290 * blk_queue_issue_flush_fn - set function for issuing a flush
291 * @q: the request queue
292 * @iff: the function to be called issuing the flush
295 * If a driver supports issuing a flush command, the support is notified
296 * to the block layer by defining it through this call.
299 void blk_queue_issue_flush_fn(request_queue_t
*q
, issue_flush_fn
*iff
)
301 q
->issue_flush_fn
= iff
;
304 EXPORT_SYMBOL(blk_queue_issue_flush_fn
);
307 * blk_queue_bounce_limit - set bounce buffer limit for queue
308 * @q: the request queue for the device
309 * @dma_addr: bus address limit
312 * Different hardware can have different requirements as to what pages
313 * it can do I/O directly to. A low level driver can call
314 * blk_queue_bounce_limit to have lower memory pages allocated as bounce
315 * buffers for doing I/O to pages residing above @page. By default
316 * the block layer sets this to the highest numbered "low" memory page.
318 void blk_queue_bounce_limit(request_queue_t
*q
, u64 dma_addr
)
320 unsigned long bounce_pfn
= dma_addr
>> PAGE_SHIFT
;
323 * set appropriate bounce gfp mask -- unfortunately we don't have a
324 * full 4GB zone, so we have to resort to low memory for any bounces.
325 * ISA has its own < 16MB zone.
327 if (bounce_pfn
< blk_max_low_pfn
) {
328 BUG_ON(dma_addr
< BLK_BOUNCE_ISA
);
329 init_emergency_isa_pool();
330 q
->bounce_gfp
= GFP_NOIO
| GFP_DMA
;
332 q
->bounce_gfp
= GFP_NOIO
;
334 q
->bounce_pfn
= bounce_pfn
;
337 EXPORT_SYMBOL(blk_queue_bounce_limit
);
340 * blk_queue_max_sectors - set max sectors for a request for this queue
341 * @q: the request queue for the device
342 * @max_sectors: max sectors in the usual 512b unit
345 * Enables a low level driver to set an upper limit on the size of
348 void blk_queue_max_sectors(request_queue_t
*q
, unsigned short max_sectors
)
350 if ((max_sectors
<< 9) < PAGE_CACHE_SIZE
) {
351 max_sectors
= 1 << (PAGE_CACHE_SHIFT
- 9);
352 printk("%s: set to minimum %d\n", __FUNCTION__
, max_sectors
);
355 q
->max_sectors
= q
->max_hw_sectors
= max_sectors
;
358 EXPORT_SYMBOL(blk_queue_max_sectors
);
361 * blk_queue_max_phys_segments - set max phys segments for a request for this queue
362 * @q: the request queue for the device
363 * @max_segments: max number of segments
366 * Enables a low level driver to set an upper limit on the number of
367 * physical data segments in a request. This would be the largest sized
368 * scatter list the driver could handle.
370 void blk_queue_max_phys_segments(request_queue_t
*q
, unsigned short max_segments
)
374 printk("%s: set to minimum %d\n", __FUNCTION__
, max_segments
);
377 q
->max_phys_segments
= max_segments
;
380 EXPORT_SYMBOL(blk_queue_max_phys_segments
);
383 * blk_queue_max_hw_segments - set max hw segments for a request for this queue
384 * @q: the request queue for the device
385 * @max_segments: max number of segments
388 * Enables a low level driver to set an upper limit on the number of
389 * hw data segments in a request. This would be the largest number of
390 * address/length pairs the host adapter can actually give as once
393 void blk_queue_max_hw_segments(request_queue_t
*q
, unsigned short max_segments
)
397 printk("%s: set to minimum %d\n", __FUNCTION__
, max_segments
);
400 q
->max_hw_segments
= max_segments
;
403 EXPORT_SYMBOL(blk_queue_max_hw_segments
);
406 * blk_queue_max_segment_size - set max segment size for blk_rq_map_sg
407 * @q: the request queue for the device
408 * @max_size: max size of segment in bytes
411 * Enables a low level driver to set an upper limit on the size of a
414 void blk_queue_max_segment_size(request_queue_t
*q
, unsigned int max_size
)
416 if (max_size
< PAGE_CACHE_SIZE
) {
417 max_size
= PAGE_CACHE_SIZE
;
418 printk("%s: set to minimum %d\n", __FUNCTION__
, max_size
);
421 q
->max_segment_size
= max_size
;
424 EXPORT_SYMBOL(blk_queue_max_segment_size
);
427 * blk_queue_hardsect_size - set hardware sector size for the queue
428 * @q: the request queue for the device
429 * @size: the hardware sector size, in bytes
432 * This should typically be set to the lowest possible sector size
433 * that the hardware can operate on (possible without reverting to
434 * even internal read-modify-write operations). Usually the default
435 * of 512 covers most hardware.
437 void blk_queue_hardsect_size(request_queue_t
*q
, unsigned short size
)
439 q
->hardsect_size
= size
;
442 EXPORT_SYMBOL(blk_queue_hardsect_size
);
445 * Returns the minimum that is _not_ zero, unless both are zero.
447 #define min_not_zero(l, r) (l == 0) ? r : ((r == 0) ? l : min(l, r))
450 * blk_queue_stack_limits - inherit underlying queue limits for stacked drivers
451 * @t: the stacking driver (top)
452 * @b: the underlying device (bottom)
454 void blk_queue_stack_limits(request_queue_t
*t
, request_queue_t
*b
)
456 /* zero is "infinity" */
457 t
->max_sectors
= t
->max_hw_sectors
=
458 min_not_zero(t
->max_sectors
,b
->max_sectors
);
460 t
->max_phys_segments
= min(t
->max_phys_segments
,b
->max_phys_segments
);
461 t
->max_hw_segments
= min(t
->max_hw_segments
,b
->max_hw_segments
);
462 t
->max_segment_size
= min(t
->max_segment_size
,b
->max_segment_size
);
463 t
->hardsect_size
= max(t
->hardsect_size
,b
->hardsect_size
);
466 EXPORT_SYMBOL(blk_queue_stack_limits
);
469 * blk_queue_segment_boundary - set boundary rules for segment merging
470 * @q: the request queue for the device
471 * @mask: the memory boundary mask
473 void blk_queue_segment_boundary(request_queue_t
*q
, unsigned long mask
)
475 if (mask
< PAGE_CACHE_SIZE
- 1) {
476 mask
= PAGE_CACHE_SIZE
- 1;
477 printk("%s: set to minimum %lx\n", __FUNCTION__
, mask
);
480 q
->seg_boundary_mask
= mask
;
483 EXPORT_SYMBOL(blk_queue_segment_boundary
);
486 * blk_queue_dma_alignment - set dma length and memory alignment
487 * @q: the request queue for the device
488 * @mask: alignment mask
491 * set required memory and length aligment for direct dma transactions.
492 * this is used when buiding direct io requests for the queue.
495 void blk_queue_dma_alignment(request_queue_t
*q
, int mask
)
497 q
->dma_alignment
= mask
;
500 EXPORT_SYMBOL(blk_queue_dma_alignment
);
503 * blk_queue_find_tag - find a request by its tag and queue
505 * @q: The request queue for the device
506 * @tag: The tag of the request
509 * Should be used when a device returns a tag and you want to match
512 * no locks need be held.
514 struct request
*blk_queue_find_tag(request_queue_t
*q
, int tag
)
516 struct blk_queue_tag
*bqt
= q
->queue_tags
;
518 if (unlikely(bqt
== NULL
|| tag
>= bqt
->real_max_depth
))
521 return bqt
->tag_index
[tag
];
524 EXPORT_SYMBOL(blk_queue_find_tag
);
527 * __blk_queue_free_tags - release tag maintenance info
528 * @q: the request queue for the device
531 * blk_cleanup_queue() will take care of calling this function, if tagging
532 * has been used. So there's no need to call this directly.
534 static void __blk_queue_free_tags(request_queue_t
*q
)
536 struct blk_queue_tag
*bqt
= q
->queue_tags
;
541 if (atomic_dec_and_test(&bqt
->refcnt
)) {
543 BUG_ON(!list_empty(&bqt
->busy_list
));
545 kfree(bqt
->tag_index
);
546 bqt
->tag_index
= NULL
;
554 q
->queue_tags
= NULL
;
555 q
->queue_flags
&= ~(1 << QUEUE_FLAG_QUEUED
);
559 * blk_queue_free_tags - release tag maintenance info
560 * @q: the request queue for the device
563 * This is used to disabled tagged queuing to a device, yet leave
566 void blk_queue_free_tags(request_queue_t
*q
)
568 clear_bit(QUEUE_FLAG_QUEUED
, &q
->queue_flags
);
571 EXPORT_SYMBOL(blk_queue_free_tags
);
574 init_tag_map(request_queue_t
*q
, struct blk_queue_tag
*tags
, int depth
)
577 struct request
**tag_index
;
578 unsigned long *tag_map
;
580 if (depth
> q
->nr_requests
* 2) {
581 depth
= q
->nr_requests
* 2;
582 printk(KERN_ERR
"%s: adjusted depth to %d\n",
583 __FUNCTION__
, depth
);
586 tag_index
= kmalloc(depth
* sizeof(struct request
*), GFP_ATOMIC
);
590 bits
= (depth
/ BLK_TAGS_PER_LONG
) + 1;
591 tag_map
= kmalloc(bits
* sizeof(unsigned long), GFP_ATOMIC
);
595 memset(tag_index
, 0, depth
* sizeof(struct request
*));
596 memset(tag_map
, 0, bits
* sizeof(unsigned long));
597 tags
->max_depth
= depth
;
598 tags
->real_max_depth
= bits
* BITS_PER_LONG
;
599 tags
->tag_index
= tag_index
;
600 tags
->tag_map
= tag_map
;
603 * set the upper bits if the depth isn't a multiple of the word size
605 for (i
= depth
; i
< bits
* BLK_TAGS_PER_LONG
; i
++)
606 __set_bit(i
, tag_map
);
615 * blk_queue_init_tags - initialize the queue tag info
616 * @q: the request queue for the device
617 * @depth: the maximum queue depth supported
619 int blk_queue_init_tags(request_queue_t
*q
, int depth
,
620 struct blk_queue_tag
*tags
)
624 BUG_ON(tags
&& q
->queue_tags
&& tags
!= q
->queue_tags
);
626 if (!tags
&& !q
->queue_tags
) {
627 tags
= kmalloc(sizeof(struct blk_queue_tag
), GFP_ATOMIC
);
631 if (init_tag_map(q
, tags
, depth
))
634 INIT_LIST_HEAD(&tags
->busy_list
);
636 atomic_set(&tags
->refcnt
, 1);
637 } else if (q
->queue_tags
) {
638 if ((rc
= blk_queue_resize_tags(q
, depth
)))
640 set_bit(QUEUE_FLAG_QUEUED
, &q
->queue_flags
);
643 atomic_inc(&tags
->refcnt
);
646 * assign it, all done
648 q
->queue_tags
= tags
;
649 q
->queue_flags
|= (1 << QUEUE_FLAG_QUEUED
);
656 EXPORT_SYMBOL(blk_queue_init_tags
);
659 * blk_queue_resize_tags - change the queueing depth
660 * @q: the request queue for the device
661 * @new_depth: the new max command queueing depth
664 * Must be called with the queue lock held.
666 int blk_queue_resize_tags(request_queue_t
*q
, int new_depth
)
668 struct blk_queue_tag
*bqt
= q
->queue_tags
;
669 struct request
**tag_index
;
670 unsigned long *tag_map
;
677 * don't bother sizing down
679 if (new_depth
<= bqt
->real_max_depth
) {
680 bqt
->max_depth
= new_depth
;
685 * save the old state info, so we can copy it back
687 tag_index
= bqt
->tag_index
;
688 tag_map
= bqt
->tag_map
;
689 max_depth
= bqt
->real_max_depth
;
691 if (init_tag_map(q
, bqt
, new_depth
))
694 memcpy(bqt
->tag_index
, tag_index
, max_depth
* sizeof(struct request
*));
695 bits
= max_depth
/ BLK_TAGS_PER_LONG
;
696 memcpy(bqt
->tag_map
, tag_map
, bits
* sizeof(unsigned long));
703 EXPORT_SYMBOL(blk_queue_resize_tags
);
706 * blk_queue_end_tag - end tag operations for a request
707 * @q: the request queue for the device
708 * @rq: the request that has completed
711 * Typically called when end_that_request_first() returns 0, meaning
712 * all transfers have been done for a request. It's important to call
713 * this function before end_that_request_last(), as that will put the
714 * request back on the free list thus corrupting the internal tag list.
717 * queue lock must be held.
719 void blk_queue_end_tag(request_queue_t
*q
, struct request
*rq
)
721 struct blk_queue_tag
*bqt
= q
->queue_tags
;
726 if (unlikely(tag
>= bqt
->real_max_depth
))
729 if (unlikely(!__test_and_clear_bit(tag
, bqt
->tag_map
))) {
730 printk("attempt to clear non-busy tag (%d)\n", tag
);
734 list_del_init(&rq
->queuelist
);
735 rq
->flags
&= ~REQ_QUEUED
;
738 if (unlikely(bqt
->tag_index
[tag
] == NULL
))
739 printk("tag %d is missing\n", tag
);
741 bqt
->tag_index
[tag
] = NULL
;
745 EXPORT_SYMBOL(blk_queue_end_tag
);
748 * blk_queue_start_tag - find a free tag and assign it
749 * @q: the request queue for the device
750 * @rq: the block request that needs tagging
753 * This can either be used as a stand-alone helper, or possibly be
754 * assigned as the queue &prep_rq_fn (in which case &struct request
755 * automagically gets a tag assigned). Note that this function
756 * assumes that any type of request can be queued! if this is not
757 * true for your device, you must check the request type before
758 * calling this function. The request will also be removed from
759 * the request queue, so it's the drivers responsibility to readd
760 * it if it should need to be restarted for some reason.
763 * queue lock must be held.
765 int blk_queue_start_tag(request_queue_t
*q
, struct request
*rq
)
767 struct blk_queue_tag
*bqt
= q
->queue_tags
;
768 unsigned long *map
= bqt
->tag_map
;
771 if (unlikely((rq
->flags
& REQ_QUEUED
))) {
773 "request %p for device [%s] already tagged %d",
774 rq
, rq
->rq_disk
? rq
->rq_disk
->disk_name
: "?", rq
->tag
);
778 for (map
= bqt
->tag_map
; *map
== -1UL; map
++) {
779 tag
+= BLK_TAGS_PER_LONG
;
781 if (tag
>= bqt
->max_depth
)
786 __set_bit(tag
, bqt
->tag_map
);
788 rq
->flags
|= REQ_QUEUED
;
790 bqt
->tag_index
[tag
] = rq
;
791 blkdev_dequeue_request(rq
);
792 list_add(&rq
->queuelist
, &bqt
->busy_list
);
797 EXPORT_SYMBOL(blk_queue_start_tag
);
800 * blk_queue_invalidate_tags - invalidate all pending tags
801 * @q: the request queue for the device
804 * Hardware conditions may dictate a need to stop all pending requests.
805 * In this case, we will safely clear the block side of the tag queue and
806 * readd all requests to the request queue in the right order.
809 * queue lock must be held.
811 void blk_queue_invalidate_tags(request_queue_t
*q
)
813 struct blk_queue_tag
*bqt
= q
->queue_tags
;
814 struct list_head
*tmp
, *n
;
817 list_for_each_safe(tmp
, n
, &bqt
->busy_list
) {
818 rq
= list_entry_rq(tmp
);
821 printk("bad tag found on list\n");
822 list_del_init(&rq
->queuelist
);
823 rq
->flags
&= ~REQ_QUEUED
;
825 blk_queue_end_tag(q
, rq
);
827 rq
->flags
&= ~REQ_STARTED
;
828 __elv_add_request(q
, rq
, ELEVATOR_INSERT_BACK
, 0);
832 EXPORT_SYMBOL(blk_queue_invalidate_tags
);
834 static char *rq_flags
[] = {
852 "REQ_DRIVE_TASKFILE",
859 void blk_dump_rq_flags(struct request
*rq
, char *msg
)
863 printk("%s: dev %s: flags = ", msg
,
864 rq
->rq_disk
? rq
->rq_disk
->disk_name
: "?");
867 if (rq
->flags
& (1 << bit
))
868 printk("%s ", rq_flags
[bit
]);
870 } while (bit
< __REQ_NR_BITS
);
872 printk("\nsector %llu, nr/cnr %lu/%u\n", (unsigned long long)rq
->sector
,
874 rq
->current_nr_sectors
);
875 printk("bio %p, biotail %p, buffer %p, data %p, len %u\n", rq
->bio
, rq
->biotail
, rq
->buffer
, rq
->data
, rq
->data_len
);
877 if (rq
->flags
& (REQ_BLOCK_PC
| REQ_PC
)) {
879 for (bit
= 0; bit
< sizeof(rq
->cmd
); bit
++)
880 printk("%02x ", rq
->cmd
[bit
]);
885 EXPORT_SYMBOL(blk_dump_rq_flags
);
887 void blk_recount_segments(request_queue_t
*q
, struct bio
*bio
)
889 struct bio_vec
*bv
, *bvprv
= NULL
;
890 int i
, nr_phys_segs
, nr_hw_segs
, seg_size
, hw_seg_size
, cluster
;
891 int high
, highprv
= 1;
893 if (unlikely(!bio
->bi_io_vec
))
896 cluster
= q
->queue_flags
& (1 << QUEUE_FLAG_CLUSTER
);
897 hw_seg_size
= seg_size
= nr_phys_segs
= nr_hw_segs
= 0;
898 bio_for_each_segment(bv
, bio
, i
) {
900 * the trick here is making sure that a high page is never
901 * considered part of another segment, since that might
902 * change with the bounce page.
904 high
= page_to_pfn(bv
->bv_page
) >= q
->bounce_pfn
;
908 if (seg_size
+ bv
->bv_len
> q
->max_segment_size
)
910 if (!BIOVEC_PHYS_MERGEABLE(bvprv
, bv
))
912 if (!BIOVEC_SEG_BOUNDARY(q
, bvprv
, bv
))
914 if (BIOVEC_VIRT_OVERSIZE(hw_seg_size
+ bv
->bv_len
))
917 seg_size
+= bv
->bv_len
;
918 hw_seg_size
+= bv
->bv_len
;
923 if (BIOVEC_VIRT_MERGEABLE(bvprv
, bv
) &&
924 !BIOVEC_VIRT_OVERSIZE(hw_seg_size
+ bv
->bv_len
)) {
925 hw_seg_size
+= bv
->bv_len
;
928 if (hw_seg_size
> bio
->bi_hw_front_size
)
929 bio
->bi_hw_front_size
= hw_seg_size
;
930 hw_seg_size
= BIOVEC_VIRT_START_SIZE(bv
) + bv
->bv_len
;
936 seg_size
= bv
->bv_len
;
939 if (hw_seg_size
> bio
->bi_hw_back_size
)
940 bio
->bi_hw_back_size
= hw_seg_size
;
941 if (nr_hw_segs
== 1 && hw_seg_size
> bio
->bi_hw_front_size
)
942 bio
->bi_hw_front_size
= hw_seg_size
;
943 bio
->bi_phys_segments
= nr_phys_segs
;
944 bio
->bi_hw_segments
= nr_hw_segs
;
945 bio
->bi_flags
|= (1 << BIO_SEG_VALID
);
949 int blk_phys_contig_segment(request_queue_t
*q
, struct bio
*bio
,
952 if (!(q
->queue_flags
& (1 << QUEUE_FLAG_CLUSTER
)))
955 if (!BIOVEC_PHYS_MERGEABLE(__BVEC_END(bio
), __BVEC_START(nxt
)))
957 if (bio
->bi_size
+ nxt
->bi_size
> q
->max_segment_size
)
961 * bio and nxt are contigous in memory, check if the queue allows
962 * these two to be merged into one
964 if (BIO_SEG_BOUNDARY(q
, bio
, nxt
))
970 EXPORT_SYMBOL(blk_phys_contig_segment
);
972 int blk_hw_contig_segment(request_queue_t
*q
, struct bio
*bio
,
975 if (unlikely(!bio_flagged(bio
, BIO_SEG_VALID
)))
976 blk_recount_segments(q
, bio
);
977 if (unlikely(!bio_flagged(nxt
, BIO_SEG_VALID
)))
978 blk_recount_segments(q
, nxt
);
979 if (!BIOVEC_VIRT_MERGEABLE(__BVEC_END(bio
), __BVEC_START(nxt
)) ||
980 BIOVEC_VIRT_OVERSIZE(bio
->bi_hw_front_size
+ bio
->bi_hw_back_size
))
982 if (bio
->bi_size
+ nxt
->bi_size
> q
->max_segment_size
)
988 EXPORT_SYMBOL(blk_hw_contig_segment
);
991 * map a request to scatterlist, return number of sg entries setup. Caller
992 * must make sure sg can hold rq->nr_phys_segments entries
994 int blk_rq_map_sg(request_queue_t
*q
, struct request
*rq
, struct scatterlist
*sg
)
996 struct bio_vec
*bvec
, *bvprv
;
998 int nsegs
, i
, cluster
;
1001 cluster
= q
->queue_flags
& (1 << QUEUE_FLAG_CLUSTER
);
1004 * for each bio in rq
1007 rq_for_each_bio(bio
, rq
) {
1009 * for each segment in bio
1011 bio_for_each_segment(bvec
, bio
, i
) {
1012 int nbytes
= bvec
->bv_len
;
1014 if (bvprv
&& cluster
) {
1015 if (sg
[nsegs
- 1].length
+ nbytes
> q
->max_segment_size
)
1018 if (!BIOVEC_PHYS_MERGEABLE(bvprv
, bvec
))
1020 if (!BIOVEC_SEG_BOUNDARY(q
, bvprv
, bvec
))
1023 sg
[nsegs
- 1].length
+= nbytes
;
1026 memset(&sg
[nsegs
],0,sizeof(struct scatterlist
));
1027 sg
[nsegs
].page
= bvec
->bv_page
;
1028 sg
[nsegs
].length
= nbytes
;
1029 sg
[nsegs
].offset
= bvec
->bv_offset
;
1034 } /* segments in bio */
1040 EXPORT_SYMBOL(blk_rq_map_sg
);
1043 * the standard queue merge functions, can be overridden with device
1044 * specific ones if so desired
1047 static inline int ll_new_mergeable(request_queue_t
*q
,
1048 struct request
*req
,
1051 int nr_phys_segs
= bio_phys_segments(q
, bio
);
1053 if (req
->nr_phys_segments
+ nr_phys_segs
> q
->max_phys_segments
) {
1054 req
->flags
|= REQ_NOMERGE
;
1055 if (req
== q
->last_merge
)
1056 q
->last_merge
= NULL
;
1061 * A hw segment is just getting larger, bump just the phys
1064 req
->nr_phys_segments
+= nr_phys_segs
;
1068 static inline int ll_new_hw_segment(request_queue_t
*q
,
1069 struct request
*req
,
1072 int nr_hw_segs
= bio_hw_segments(q
, bio
);
1073 int nr_phys_segs
= bio_phys_segments(q
, bio
);
1075 if (req
->nr_hw_segments
+ nr_hw_segs
> q
->max_hw_segments
1076 || req
->nr_phys_segments
+ nr_phys_segs
> q
->max_phys_segments
) {
1077 req
->flags
|= REQ_NOMERGE
;
1078 if (req
== q
->last_merge
)
1079 q
->last_merge
= NULL
;
1084 * This will form the start of a new hw segment. Bump both
1087 req
->nr_hw_segments
+= nr_hw_segs
;
1088 req
->nr_phys_segments
+= nr_phys_segs
;
1092 static int ll_back_merge_fn(request_queue_t
*q
, struct request
*req
,
1097 if (req
->nr_sectors
+ bio_sectors(bio
) > q
->max_sectors
) {
1098 req
->flags
|= REQ_NOMERGE
;
1099 if (req
== q
->last_merge
)
1100 q
->last_merge
= NULL
;
1103 if (unlikely(!bio_flagged(req
->biotail
, BIO_SEG_VALID
)))
1104 blk_recount_segments(q
, req
->biotail
);
1105 if (unlikely(!bio_flagged(bio
, BIO_SEG_VALID
)))
1106 blk_recount_segments(q
, bio
);
1107 len
= req
->biotail
->bi_hw_back_size
+ bio
->bi_hw_front_size
;
1108 if (BIOVEC_VIRT_MERGEABLE(__BVEC_END(req
->biotail
), __BVEC_START(bio
)) &&
1109 !BIOVEC_VIRT_OVERSIZE(len
)) {
1110 int mergeable
= ll_new_mergeable(q
, req
, bio
);
1113 if (req
->nr_hw_segments
== 1)
1114 req
->bio
->bi_hw_front_size
= len
;
1115 if (bio
->bi_hw_segments
== 1)
1116 bio
->bi_hw_back_size
= len
;
1121 return ll_new_hw_segment(q
, req
, bio
);
1124 static int ll_front_merge_fn(request_queue_t
*q
, struct request
*req
,
1129 if (req
->nr_sectors
+ bio_sectors(bio
) > q
->max_sectors
) {
1130 req
->flags
|= REQ_NOMERGE
;
1131 if (req
== q
->last_merge
)
1132 q
->last_merge
= NULL
;
1135 len
= bio
->bi_hw_back_size
+ req
->bio
->bi_hw_front_size
;
1136 if (unlikely(!bio_flagged(bio
, BIO_SEG_VALID
)))
1137 blk_recount_segments(q
, bio
);
1138 if (unlikely(!bio_flagged(req
->bio
, BIO_SEG_VALID
)))
1139 blk_recount_segments(q
, req
->bio
);
1140 if (BIOVEC_VIRT_MERGEABLE(__BVEC_END(bio
), __BVEC_START(req
->bio
)) &&
1141 !BIOVEC_VIRT_OVERSIZE(len
)) {
1142 int mergeable
= ll_new_mergeable(q
, req
, bio
);
1145 if (bio
->bi_hw_segments
== 1)
1146 bio
->bi_hw_front_size
= len
;
1147 if (req
->nr_hw_segments
== 1)
1148 req
->biotail
->bi_hw_back_size
= len
;
1153 return ll_new_hw_segment(q
, req
, bio
);
1156 static int ll_merge_requests_fn(request_queue_t
*q
, struct request
*req
,
1157 struct request
*next
)
1159 int total_phys_segments
= req
->nr_phys_segments
+next
->nr_phys_segments
;
1160 int total_hw_segments
= req
->nr_hw_segments
+ next
->nr_hw_segments
;
1163 * First check if the either of the requests are re-queued
1164 * requests. Can't merge them if they are.
1166 if (req
->special
|| next
->special
)
1170 * Will it become to large?
1172 if ((req
->nr_sectors
+ next
->nr_sectors
) > q
->max_sectors
)
1175 total_phys_segments
= req
->nr_phys_segments
+ next
->nr_phys_segments
;
1176 if (blk_phys_contig_segment(q
, req
->biotail
, next
->bio
))
1177 total_phys_segments
--;
1179 if (total_phys_segments
> q
->max_phys_segments
)
1182 total_hw_segments
= req
->nr_hw_segments
+ next
->nr_hw_segments
;
1183 if (blk_hw_contig_segment(q
, req
->biotail
, next
->bio
)) {
1184 int len
= req
->biotail
->bi_hw_back_size
+ next
->bio
->bi_hw_front_size
;
1186 * propagate the combined length to the end of the requests
1188 if (req
->nr_hw_segments
== 1)
1189 req
->bio
->bi_hw_front_size
= len
;
1190 if (next
->nr_hw_segments
== 1)
1191 next
->biotail
->bi_hw_back_size
= len
;
1192 total_hw_segments
--;
1195 if (total_hw_segments
> q
->max_hw_segments
)
1198 /* Merge is OK... */
1199 req
->nr_phys_segments
= total_phys_segments
;
1200 req
->nr_hw_segments
= total_hw_segments
;
1205 * "plug" the device if there are no outstanding requests: this will
1206 * force the transfer to start only after we have put all the requests
1209 * This is called with interrupts off and no requests on the queue and
1210 * with the queue lock held.
1212 void blk_plug_device(request_queue_t
*q
)
1214 WARN_ON(!irqs_disabled());
1217 * don't plug a stopped queue, it must be paired with blk_start_queue()
1218 * which will restart the queueing
1220 if (test_bit(QUEUE_FLAG_STOPPED
, &q
->queue_flags
))
1223 if (!test_and_set_bit(QUEUE_FLAG_PLUGGED
, &q
->queue_flags
))
1224 mod_timer(&q
->unplug_timer
, jiffies
+ q
->unplug_delay
);
1227 EXPORT_SYMBOL(blk_plug_device
);
1230 * remove the queue from the plugged list, if present. called with
1231 * queue lock held and interrupts disabled.
1233 int blk_remove_plug(request_queue_t
*q
)
1235 WARN_ON(!irqs_disabled());
1237 if (!test_and_clear_bit(QUEUE_FLAG_PLUGGED
, &q
->queue_flags
))
1240 del_timer(&q
->unplug_timer
);
1244 EXPORT_SYMBOL(blk_remove_plug
);
1247 * remove the plug and let it rip..
1249 void __generic_unplug_device(request_queue_t
*q
)
1251 if (test_bit(QUEUE_FLAG_STOPPED
, &q
->queue_flags
))
1254 if (!blk_remove_plug(q
))
1258 * was plugged, fire request_fn if queue has stuff to do
1260 if (elv_next_request(q
))
1263 EXPORT_SYMBOL(__generic_unplug_device
);
1266 * generic_unplug_device - fire a request queue
1267 * @q: The &request_queue_t in question
1270 * Linux uses plugging to build bigger requests queues before letting
1271 * the device have at them. If a queue is plugged, the I/O scheduler
1272 * is still adding and merging requests on the queue. Once the queue
1273 * gets unplugged, the request_fn defined for the queue is invoked and
1274 * transfers started.
1276 void generic_unplug_device(request_queue_t
*q
)
1278 spin_lock_irq(q
->queue_lock
);
1279 __generic_unplug_device(q
);
1280 spin_unlock_irq(q
->queue_lock
);
1282 EXPORT_SYMBOL(generic_unplug_device
);
1284 static void blk_backing_dev_unplug(struct backing_dev_info
*bdi
,
1287 request_queue_t
*q
= bdi
->unplug_io_data
;
1290 * devices don't necessarily have an ->unplug_fn defined
1296 static void blk_unplug_work(void *data
)
1298 request_queue_t
*q
= data
;
1303 static void blk_unplug_timeout(unsigned long data
)
1305 request_queue_t
*q
= (request_queue_t
*)data
;
1307 kblockd_schedule_work(&q
->unplug_work
);
1311 * blk_start_queue - restart a previously stopped queue
1312 * @q: The &request_queue_t in question
1315 * blk_start_queue() will clear the stop flag on the queue, and call
1316 * the request_fn for the queue if it was in a stopped state when
1317 * entered. Also see blk_stop_queue(). Queue lock must be held.
1319 void blk_start_queue(request_queue_t
*q
)
1321 clear_bit(QUEUE_FLAG_STOPPED
, &q
->queue_flags
);
1324 * one level of recursion is ok and is much faster than kicking
1325 * the unplug handling
1327 if (!test_and_set_bit(QUEUE_FLAG_REENTER
, &q
->queue_flags
)) {
1329 clear_bit(QUEUE_FLAG_REENTER
, &q
->queue_flags
);
1332 kblockd_schedule_work(&q
->unplug_work
);
1336 EXPORT_SYMBOL(blk_start_queue
);
1339 * blk_stop_queue - stop a queue
1340 * @q: The &request_queue_t in question
1343 * The Linux block layer assumes that a block driver will consume all
1344 * entries on the request queue when the request_fn strategy is called.
1345 * Often this will not happen, because of hardware limitations (queue
1346 * depth settings). If a device driver gets a 'queue full' response,
1347 * or if it simply chooses not to queue more I/O at one point, it can
1348 * call this function to prevent the request_fn from being called until
1349 * the driver has signalled it's ready to go again. This happens by calling
1350 * blk_start_queue() to restart queue operations. Queue lock must be held.
1352 void blk_stop_queue(request_queue_t
*q
)
1355 set_bit(QUEUE_FLAG_STOPPED
, &q
->queue_flags
);
1358 EXPORT_SYMBOL(blk_stop_queue
);
1361 * blk_run_queue - run a single device queue
1362 * @q: The queue to run
1364 void blk_run_queue(struct request_queue
*q
)
1366 unsigned long flags
;
1368 spin_lock_irqsave(q
->queue_lock
, flags
);
1371 spin_unlock_irqrestore(q
->queue_lock
, flags
);
1374 EXPORT_SYMBOL(blk_run_queue
);
1377 * blk_cleanup_queue: - release a &request_queue_t when it is no longer needed
1378 * @q: the request queue to be released
1381 * blk_cleanup_queue is the pair to blk_init_queue() or
1382 * blk_queue_make_request(). It should be called when a request queue is
1383 * being released; typically when a block device is being de-registered.
1384 * Currently, its primary task it to free all the &struct request
1385 * structures that were allocated to the queue and the queue itself.
1388 * Hopefully the low level driver will have finished any
1389 * outstanding requests first...
1391 void blk_cleanup_queue(request_queue_t
* q
)
1393 struct request_list
*rl
= &q
->rq
;
1395 if (!atomic_dec_and_test(&q
->refcnt
))
1400 del_timer_sync(&q
->unplug_timer
);
1404 mempool_destroy(rl
->rq_pool
);
1407 __blk_queue_free_tags(q
);
1409 kmem_cache_free(requestq_cachep
, q
);
1412 EXPORT_SYMBOL(blk_cleanup_queue
);
1414 static int blk_init_free_list(request_queue_t
*q
)
1416 struct request_list
*rl
= &q
->rq
;
1418 rl
->count
[READ
] = rl
->count
[WRITE
] = 0;
1419 init_waitqueue_head(&rl
->wait
[READ
]);
1420 init_waitqueue_head(&rl
->wait
[WRITE
]);
1422 rl
->rq_pool
= mempool_create(BLKDEV_MIN_RQ
, mempool_alloc_slab
, mempool_free_slab
, request_cachep
);
1430 static int __make_request(request_queue_t
*, struct bio
*);
1432 static elevator_t
*chosen_elevator
=
1433 #if defined(CONFIG_IOSCHED_AS)
1435 #elif defined(CONFIG_IOSCHED_DEADLINE)
1437 #elif defined(CONFIG_IOSCHED_CFQ)
1439 #elif defined(CONFIG_IOSCHED_NOOP)
1443 #error "You must have at least 1 I/O scheduler selected"
1446 #if defined(CONFIG_IOSCHED_AS) || defined(CONFIG_IOSCHED_DEADLINE) || defined (CONFIG_IOSCHED_NOOP)
1447 static int __init
elevator_setup(char *str
)
1449 #ifdef CONFIG_IOSCHED_DEADLINE
1450 if (!strcmp(str
, "deadline"))
1451 chosen_elevator
= &iosched_deadline
;
1453 #ifdef CONFIG_IOSCHED_AS
1454 if (!strcmp(str
, "as"))
1455 chosen_elevator
= &iosched_as
;
1457 #ifdef CONFIG_IOSCHED_CFQ
1458 if (!strcmp(str
, "cfq"))
1459 chosen_elevator
= &iosched_cfq
;
1461 #ifdef CONFIG_IOSCHED_NOOP
1462 if (!strcmp(str
, "noop"))
1463 chosen_elevator
= &elevator_noop
;
1468 __setup("elevator=", elevator_setup
);
1469 #endif /* CONFIG_IOSCHED_AS || CONFIG_IOSCHED_DEADLINE || CONFIG_IOSCHED_NOOP */
1471 request_queue_t
*blk_alloc_queue(int gfp_mask
)
1473 request_queue_t
*q
= kmem_cache_alloc(requestq_cachep
, gfp_mask
);
1478 memset(q
, 0, sizeof(*q
));
1479 init_timer(&q
->unplug_timer
);
1480 atomic_set(&q
->refcnt
, 1);
1482 q
->backing_dev_info
.unplug_io_fn
= blk_backing_dev_unplug
;
1483 q
->backing_dev_info
.unplug_io_data
= q
;
1488 EXPORT_SYMBOL(blk_alloc_queue
);
1491 * blk_init_queue - prepare a request queue for use with a block device
1492 * @rfn: The function to be called to process requests that have been
1493 * placed on the queue.
1494 * @lock: Request queue spin lock
1497 * If a block device wishes to use the standard request handling procedures,
1498 * which sorts requests and coalesces adjacent requests, then it must
1499 * call blk_init_queue(). The function @rfn will be called when there
1500 * are requests on the queue that need to be processed. If the device
1501 * supports plugging, then @rfn may not be called immediately when requests
1502 * are available on the queue, but may be called at some time later instead.
1503 * Plugged queues are generally unplugged when a buffer belonging to one
1504 * of the requests on the queue is needed, or due to memory pressure.
1506 * @rfn is not required, or even expected, to remove all requests off the
1507 * queue, but only as many as it can handle at a time. If it does leave
1508 * requests on the queue, it is responsible for arranging that the requests
1509 * get dealt with eventually.
1511 * The queue spin lock must be held while manipulating the requests on the
1514 * Function returns a pointer to the initialized request queue, or NULL if
1515 * it didn't succeed.
1518 * blk_init_queue() must be paired with a blk_cleanup_queue() call
1519 * when the block device is deactivated (such as at module unload).
1521 request_queue_t
*blk_init_queue(request_fn_proc
*rfn
, spinlock_t
*lock
)
1526 q
= blk_alloc_queue(GFP_KERNEL
);
1530 if (blk_init_free_list(q
))
1535 printk("Using %s io scheduler\n", chosen_elevator
->elevator_name
);
1538 q
->request_fn
= rfn
;
1539 q
->back_merge_fn
= ll_back_merge_fn
;
1540 q
->front_merge_fn
= ll_front_merge_fn
;
1541 q
->merge_requests_fn
= ll_merge_requests_fn
;
1542 q
->prep_rq_fn
= NULL
;
1543 q
->unplug_fn
= generic_unplug_device
;
1544 q
->queue_flags
= (1 << QUEUE_FLAG_CLUSTER
);
1545 q
->queue_lock
= lock
;
1547 blk_queue_segment_boundary(q
, 0xffffffff);
1549 blk_queue_make_request(q
, __make_request
);
1550 blk_queue_max_segment_size(q
, MAX_SEGMENT_SIZE
);
1552 blk_queue_max_hw_segments(q
, MAX_HW_SEGMENTS
);
1553 blk_queue_max_phys_segments(q
, MAX_PHYS_SEGMENTS
);
1558 if (!elevator_init(q
, chosen_elevator
))
1561 blk_cleanup_queue(q
);
1563 kmem_cache_free(requestq_cachep
, q
);
1567 EXPORT_SYMBOL(blk_init_queue
);
1569 int blk_get_queue(request_queue_t
*q
)
1571 if (!test_bit(QUEUE_FLAG_DEAD
, &q
->queue_flags
)) {
1572 atomic_inc(&q
->refcnt
);
1579 EXPORT_SYMBOL(blk_get_queue
);
1581 static inline void blk_free_request(request_queue_t
*q
, struct request
*rq
)
1583 elv_put_request(q
, rq
);
1584 mempool_free(rq
, q
->rq
.rq_pool
);
1587 static inline struct request
*blk_alloc_request(request_queue_t
*q
,int gfp_mask
)
1589 struct request
*rq
= mempool_alloc(q
->rq
.rq_pool
, gfp_mask
);
1594 if (!elv_set_request(q
, rq
, gfp_mask
))
1597 mempool_free(rq
, q
->rq
.rq_pool
);
1602 * ioc_batching returns true if the ioc is a valid batching request and
1603 * should be given priority access to a request.
1605 static inline int ioc_batching(struct io_context
*ioc
)
1611 * Make sure the process is able to allocate at least 1 request
1612 * even if the batch times out, otherwise we could theoretically
1615 return ioc
->nr_batch_requests
== BLK_BATCH_REQ
||
1616 (ioc
->nr_batch_requests
> 0
1617 && time_before(jiffies
, ioc
->last_waited
+ BLK_BATCH_TIME
));
1621 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
1622 * will cause the process to be a "batcher" on all queues in the system. This
1623 * is the behaviour we want though - once it gets a wakeup it should be given
1626 void ioc_set_batching(struct io_context
*ioc
)
1628 if (!ioc
|| ioc_batching(ioc
))
1631 ioc
->nr_batch_requests
= BLK_BATCH_REQ
;
1632 ioc
->last_waited
= jiffies
;
1636 * A request has just been released. Account for it, update the full and
1637 * congestion status, wake up any waiters. Called under q->queue_lock.
1639 static void freed_request(request_queue_t
*q
, int rw
)
1641 struct request_list
*rl
= &q
->rq
;
1644 if (rl
->count
[rw
] < queue_congestion_off_threshold(q
))
1645 clear_queue_congested(q
, rw
);
1646 if (rl
->count
[rw
]+1 <= q
->nr_requests
) {
1647 if (waitqueue_active(&rl
->wait
[rw
]))
1648 wake_up(&rl
->wait
[rw
]);
1649 if (!waitqueue_active(&rl
->wait
[rw
]))
1650 blk_clear_queue_full(q
, rw
);
1654 #define blkdev_free_rq(list) list_entry((list)->next, struct request, queuelist)
1656 * Get a free request, queue_lock must not be held
1658 static struct request
*get_request(request_queue_t
*q
, int rw
, int gfp_mask
)
1660 struct request
*rq
= NULL
;
1661 struct request_list
*rl
= &q
->rq
;
1662 struct io_context
*ioc
= get_io_context(gfp_mask
);
1664 spin_lock_irq(q
->queue_lock
);
1665 if (rl
->count
[rw
]+1 >= q
->nr_requests
) {
1667 * The queue will fill after this allocation, so set it as
1668 * full, and mark this process as "batching". This process
1669 * will be allowed to complete a batch of requests, others
1672 if (!blk_queue_full(q
, rw
)) {
1673 ioc_set_batching(ioc
);
1674 blk_set_queue_full(q
, rw
);
1678 if (blk_queue_full(q
, rw
)
1679 && !ioc_batching(ioc
) && !elv_may_queue(q
, rw
)) {
1681 * The queue is full and the allocating process is not a
1682 * "batcher", and not exempted by the IO scheduler
1684 spin_unlock_irq(q
->queue_lock
);
1689 if (rl
->count
[rw
] >= queue_congestion_on_threshold(q
))
1690 set_queue_congested(q
, rw
);
1691 spin_unlock_irq(q
->queue_lock
);
1693 rq
= blk_alloc_request(q
, gfp_mask
);
1696 * Allocation failed presumably due to memory. Undo anything
1697 * we might have messed up.
1699 * Allocating task should really be put onto the front of the
1700 * wait queue, but this is pretty rare.
1702 spin_lock_irq(q
->queue_lock
);
1703 freed_request(q
, rw
);
1704 spin_unlock_irq(q
->queue_lock
);
1708 if (ioc_batching(ioc
))
1709 ioc
->nr_batch_requests
--;
1711 INIT_LIST_HEAD(&rq
->queuelist
);
1714 * first three bits are identical in rq->flags and bio->bi_rw,
1715 * see bio.h and blkdev.h
1720 rq
->rq_status
= RQ_ACTIVE
;
1721 rq
->bio
= rq
->biotail
= NULL
;
1733 put_io_context(ioc
);
1738 * No available requests for this queue, unplug the device and wait for some
1739 * requests to become available.
1741 static struct request
*get_request_wait(request_queue_t
*q
, int rw
)
1746 generic_unplug_device(q
);
1748 struct request_list
*rl
= &q
->rq
;
1750 prepare_to_wait_exclusive(&rl
->wait
[rw
], &wait
,
1751 TASK_UNINTERRUPTIBLE
);
1753 rq
= get_request(q
, rw
, GFP_NOIO
);
1756 struct io_context
*ioc
;
1761 * After sleeping, we become a "batching" process and
1762 * will be able to allocate at least one request, and
1763 * up to a big batch of them for a small period time.
1764 * See ioc_batching, ioc_set_batching
1766 ioc
= get_io_context(GFP_NOIO
);
1767 ioc_set_batching(ioc
);
1768 put_io_context(ioc
);
1770 finish_wait(&rl
->wait
[rw
], &wait
);
1776 struct request
*blk_get_request(request_queue_t
*q
, int rw
, int gfp_mask
)
1780 BUG_ON(rw
!= READ
&& rw
!= WRITE
);
1782 if (gfp_mask
& __GFP_WAIT
)
1783 rq
= get_request_wait(q
, rw
);
1785 rq
= get_request(q
, rw
, gfp_mask
);
1790 EXPORT_SYMBOL(blk_get_request
);
1793 * blk_requeue_request - put a request back on queue
1794 * @q: request queue where request should be inserted
1795 * @rq: request to be inserted
1798 * Drivers often keep queueing requests until the hardware cannot accept
1799 * more, when that condition happens we need to put the request back
1800 * on the queue. Must be called with queue lock held.
1802 void blk_requeue_request(request_queue_t
*q
, struct request
*rq
)
1804 if (blk_rq_tagged(rq
))
1805 blk_queue_end_tag(q
, rq
);
1807 elv_requeue_request(q
, rq
);
1810 EXPORT_SYMBOL(blk_requeue_request
);
1813 * blk_insert_request - insert a special request in to a request queue
1814 * @q: request queue where request should be inserted
1815 * @rq: request to be inserted
1816 * @at_head: insert request at head or tail of queue
1817 * @data: private data
1818 * @reinsert: true if request it a reinsertion of previously processed one
1821 * Many block devices need to execute commands asynchronously, so they don't
1822 * block the whole kernel from preemption during request execution. This is
1823 * accomplished normally by inserting aritficial requests tagged as
1824 * REQ_SPECIAL in to the corresponding request queue, and letting them be
1825 * scheduled for actual execution by the request queue.
1827 * We have the option of inserting the head or the tail of the queue.
1828 * Typically we use the tail for new ioctls and so forth. We use the head
1829 * of the queue for things like a QUEUE_FULL message from a device, or a
1830 * host that is unable to accept a particular command.
1832 void blk_insert_request(request_queue_t
*q
, struct request
*rq
,
1833 int at_head
, void *data
, int reinsert
)
1835 unsigned long flags
;
1838 * tell I/O scheduler that this isn't a regular read/write (ie it
1839 * must not attempt merges on this) and that it acts as a soft
1842 rq
->flags
|= REQ_SPECIAL
| REQ_SOFTBARRIER
;
1846 spin_lock_irqsave(q
->queue_lock
, flags
);
1849 * If command is tagged, release the tag
1852 blk_requeue_request(q
, rq
);
1854 int where
= ELEVATOR_INSERT_BACK
;
1857 where
= ELEVATOR_INSERT_FRONT
;
1859 if (blk_rq_tagged(rq
))
1860 blk_queue_end_tag(q
, rq
);
1862 drive_stat_acct(rq
, rq
->nr_sectors
, 1);
1863 __elv_add_request(q
, rq
, where
, 0);
1865 if (blk_queue_plugged(q
))
1866 __generic_unplug_device(q
);
1869 spin_unlock_irqrestore(q
->queue_lock
, flags
);
1872 EXPORT_SYMBOL(blk_insert_request
);
1875 * blk_rq_map_user - map user data to a request, for REQ_BLOCK_PC usage
1876 * @q: request queue where request should be inserted
1877 * @rw: READ or WRITE data
1878 * @ubuf: the user buffer
1879 * @len: length of user data
1882 * Data will be mapped directly for zero copy io, if possible. Otherwise
1883 * a kernel bounce buffer is used.
1885 * A matching blk_rq_unmap_user() must be issued at the end of io, while
1886 * still in process context.
1888 * Note: The mapped bio may need to be bounced through blk_queue_bounce()
1889 * before being submitted to the device, as pages mapped may be out of
1890 * reach. It's the callers responsibility to make sure this happens. The
1891 * original bio must be passed back in to blk_rq_unmap_user() for proper
1894 struct request
*blk_rq_map_user(request_queue_t
*q
, int rw
, void __user
*ubuf
,
1897 unsigned long uaddr
;
1901 if (len
> (q
->max_sectors
<< 9))
1902 return ERR_PTR(-EINVAL
);
1903 if ((!len
&& ubuf
) || (len
&& !ubuf
))
1904 return ERR_PTR(-EINVAL
);
1906 rq
= blk_get_request(q
, rw
, __GFP_WAIT
);
1908 return ERR_PTR(-ENOMEM
);
1911 * if alignment requirement is satisfied, map in user pages for
1912 * direct dma. else, set up kernel bounce buffers
1914 uaddr
= (unsigned long) ubuf
;
1915 if (!(uaddr
& queue_dma_alignment(q
)) && !(len
& queue_dma_alignment(q
)))
1916 bio
= bio_map_user(q
, NULL
, uaddr
, len
, rw
== READ
);
1918 bio
= bio_copy_user(q
, uaddr
, len
, rw
== READ
);
1921 rq
->bio
= rq
->biotail
= bio
;
1922 blk_rq_bio_prep(q
, rq
, bio
);
1924 rq
->buffer
= rq
->data
= NULL
;
1930 * bio is the err-ptr
1932 blk_put_request(rq
);
1933 return (struct request
*) bio
;
1936 EXPORT_SYMBOL(blk_rq_map_user
);
1939 * blk_rq_unmap_user - unmap a request with user data
1940 * @rq: request to be unmapped
1941 * @ubuf: user buffer
1942 * @ulen: length of user buffer
1945 * Unmap a request previously mapped by blk_rq_map_user().
1947 int blk_rq_unmap_user(struct request
*rq
, struct bio
*bio
, unsigned int ulen
)
1952 if (bio_flagged(bio
, BIO_USER_MAPPED
))
1953 bio_unmap_user(bio
);
1955 ret
= bio_uncopy_user(bio
);
1958 blk_put_request(rq
);
1962 EXPORT_SYMBOL(blk_rq_unmap_user
);
1965 * blk_execute_rq - insert a request into queue for execution
1966 * @q: queue to insert the request in
1967 * @bd_disk: matching gendisk
1968 * @rq: request to insert
1971 * Insert a fully prepared request at the back of the io scheduler queue
1974 int blk_execute_rq(request_queue_t
*q
, struct gendisk
*bd_disk
,
1977 DECLARE_COMPLETION(wait
);
1978 char sense
[SCSI_SENSE_BUFFERSIZE
];
1981 rq
->rq_disk
= bd_disk
;
1984 * we need an extra reference to the request, so we can look at
1985 * it after io completion
1990 memset(sense
, 0, sizeof(sense
));
1995 rq
->flags
|= REQ_NOMERGE
;
1997 rq
->waiting
= &wait
;
1998 elv_add_request(q
, rq
, ELEVATOR_INSERT_BACK
, 1);
1999 generic_unplug_device(q
);
2000 wait_for_completion(rq
->waiting
);
2009 EXPORT_SYMBOL(blk_execute_rq
);
2012 * blkdev_issue_flush - queue a flush
2013 * @bdev: blockdev to issue flush for
2014 * @error_sector: error sector
2017 * Issue a flush for the block device in question. Caller can supply
2018 * room for storing the error offset in case of a flush error, if they
2019 * wish to. Caller must run wait_for_completion() on its own.
2021 int blkdev_issue_flush(struct block_device
*bdev
, sector_t
*error_sector
)
2025 if (bdev
->bd_disk
== NULL
)
2028 q
= bdev_get_queue(bdev
);
2031 if (!q
->issue_flush_fn
)
2034 return q
->issue_flush_fn(q
, bdev
->bd_disk
, error_sector
);
2037 EXPORT_SYMBOL(blkdev_issue_flush
);
2040 * blkdev_scsi_issue_flush_fn - issue flush for SCSI devices
2043 * @error_sector: error offset
2046 * Devices understanding the SCSI command set, can use this function as
2047 * a helper for issuing a cache flush. Note: driver is required to store
2048 * the error offset (in case of error flushing) in ->sector of struct
2051 int blkdev_scsi_issue_flush_fn(request_queue_t
*q
, struct gendisk
*disk
,
2052 sector_t
*error_sector
)
2054 struct request
*rq
= blk_get_request(q
, WRITE
, __GFP_WAIT
);
2057 rq
->flags
|= REQ_BLOCK_PC
| REQ_SOFTBARRIER
;
2059 memset(rq
->cmd
, 0, sizeof(rq
->cmd
));
2064 rq
->timeout
= 60 * HZ
;
2066 ret
= blk_execute_rq(q
, disk
, rq
);
2068 if (ret
&& error_sector
)
2069 *error_sector
= rq
->sector
;
2071 blk_put_request(rq
);
2075 EXPORT_SYMBOL(blkdev_scsi_issue_flush_fn
);
2077 void drive_stat_acct(struct request
*rq
, int nr_sectors
, int new_io
)
2079 int rw
= rq_data_dir(rq
);
2081 if (!blk_fs_request(rq
) || !rq
->rq_disk
)
2085 disk_stat_add(rq
->rq_disk
, read_sectors
, nr_sectors
);
2087 disk_stat_inc(rq
->rq_disk
, read_merges
);
2088 } else if (rw
== WRITE
) {
2089 disk_stat_add(rq
->rq_disk
, write_sectors
, nr_sectors
);
2091 disk_stat_inc(rq
->rq_disk
, write_merges
);
2094 disk_round_stats(rq
->rq_disk
);
2095 rq
->rq_disk
->in_flight
++;
2100 * add-request adds a request to the linked list.
2101 * queue lock is held and interrupts disabled, as we muck with the
2102 * request queue list.
2104 static inline void add_request(request_queue_t
* q
, struct request
* req
)
2106 drive_stat_acct(req
, req
->nr_sectors
, 1);
2109 q
->activity_fn(q
->activity_data
, rq_data_dir(req
));
2112 * elevator indicated where it wants this request to be
2113 * inserted at elevator_merge time
2115 __elv_add_request(q
, req
, ELEVATOR_INSERT_SORT
, 0);
2119 * disk_round_stats() - Round off the performance stats on a struct
2122 * The average IO queue length and utilisation statistics are maintained
2123 * by observing the current state of the queue length and the amount of
2124 * time it has been in this state for.
2126 * Normally, that accounting is done on IO completion, but that can result
2127 * in more than a second's worth of IO being accounted for within any one
2128 * second, leading to >100% utilisation. To deal with that, we call this
2129 * function to do a round-off before returning the results when reading
2130 * /proc/diskstats. This accounts immediately for all queue usage up to
2131 * the current jiffies and restarts the counters again.
2133 void disk_round_stats(struct gendisk
*disk
)
2135 unsigned long now
= jiffies
;
2137 disk_stat_add(disk
, time_in_queue
,
2138 disk
->in_flight
* (now
- disk
->stamp
));
2141 if (disk
->in_flight
)
2142 disk_stat_add(disk
, io_ticks
, (now
- disk
->stamp_idle
));
2143 disk
->stamp_idle
= now
;
2147 * queue lock must be held
2149 void __blk_put_request(request_queue_t
*q
, struct request
*req
)
2151 struct request_list
*rl
= req
->rl
;
2155 if (unlikely(--req
->ref_count
))
2158 req
->rq_status
= RQ_INACTIVE
;
2163 * Request may not have originated from ll_rw_blk. if not,
2164 * it didn't come out of our reserved rq pools
2167 int rw
= rq_data_dir(req
);
2169 elv_completed_request(q
, req
);
2171 BUG_ON(!list_empty(&req
->queuelist
));
2173 blk_free_request(q
, req
);
2174 freed_request(q
, rw
);
2178 void blk_put_request(struct request
*req
)
2181 * if req->rl isn't set, this request didnt originate from the
2182 * block layer, so it's safe to just disregard it
2185 unsigned long flags
;
2186 request_queue_t
*q
= req
->q
;
2188 spin_lock_irqsave(q
->queue_lock
, flags
);
2189 __blk_put_request(q
, req
);
2190 spin_unlock_irqrestore(q
->queue_lock
, flags
);
2194 EXPORT_SYMBOL(blk_put_request
);
2197 * blk_congestion_wait - wait for a queue to become uncongested
2198 * @rw: READ or WRITE
2199 * @timeout: timeout in jiffies
2201 * Waits for up to @timeout jiffies for a queue (any queue) to exit congestion.
2202 * If no queues are congested then just wait for the next request to be
2205 long blk_congestion_wait(int rw
, long timeout
)
2209 wait_queue_head_t
*wqh
= &congestion_wqh
[rw
];
2211 prepare_to_wait(wqh
, &wait
, TASK_UNINTERRUPTIBLE
);
2212 ret
= io_schedule_timeout(timeout
);
2213 finish_wait(wqh
, &wait
);
2217 EXPORT_SYMBOL(blk_congestion_wait
);
2220 * Has to be called with the request spinlock acquired
2222 static int attempt_merge(request_queue_t
*q
, struct request
*req
,
2223 struct request
*next
)
2225 if (!rq_mergeable(req
) || !rq_mergeable(next
))
2231 if (req
->sector
+ req
->nr_sectors
!= next
->sector
)
2234 if (rq_data_dir(req
) != rq_data_dir(next
)
2235 || req
->rq_disk
!= next
->rq_disk
2236 || next
->waiting
|| next
->special
)
2240 * If we are allowed to merge, then append bio list
2241 * from next to rq and release next. merge_requests_fn
2242 * will have updated segment counts, update sector
2245 if (!q
->merge_requests_fn(q
, req
, next
))
2249 * At this point we have either done a back merge
2250 * or front merge. We need the smaller start_time of
2251 * the merged requests to be the current request
2252 * for accounting purposes.
2254 if (time_after(req
->start_time
, next
->start_time
))
2255 req
->start_time
= next
->start_time
;
2257 req
->biotail
->bi_next
= next
->bio
;
2258 req
->biotail
= next
->biotail
;
2260 req
->nr_sectors
= req
->hard_nr_sectors
+= next
->hard_nr_sectors
;
2262 elv_merge_requests(q
, req
, next
);
2265 disk_round_stats(req
->rq_disk
);
2266 req
->rq_disk
->in_flight
--;
2269 __blk_put_request(q
, next
);
2273 static inline int attempt_back_merge(request_queue_t
*q
, struct request
*rq
)
2275 struct request
*next
= elv_latter_request(q
, rq
);
2278 return attempt_merge(q
, rq
, next
);
2283 static inline int attempt_front_merge(request_queue_t
*q
, struct request
*rq
)
2285 struct request
*prev
= elv_former_request(q
, rq
);
2288 return attempt_merge(q
, prev
, rq
);
2294 * blk_attempt_remerge - attempt to remerge active head with next request
2295 * @q: The &request_queue_t belonging to the device
2296 * @rq: The head request (usually)
2299 * For head-active devices, the queue can easily be unplugged so quickly
2300 * that proper merging is not done on the front request. This may hurt
2301 * performance greatly for some devices. The block layer cannot safely
2302 * do merging on that first request for these queues, but the driver can
2303 * call this function and make it happen any way. Only the driver knows
2304 * when it is safe to do so.
2306 void blk_attempt_remerge(request_queue_t
*q
, struct request
*rq
)
2308 unsigned long flags
;
2310 spin_lock_irqsave(q
->queue_lock
, flags
);
2311 attempt_back_merge(q
, rq
);
2312 spin_unlock_irqrestore(q
->queue_lock
, flags
);
2315 EXPORT_SYMBOL(blk_attempt_remerge
);
2318 * Non-locking blk_attempt_remerge variant.
2320 void __blk_attempt_remerge(request_queue_t
*q
, struct request
*rq
)
2322 attempt_back_merge(q
, rq
);
2325 EXPORT_SYMBOL(__blk_attempt_remerge
);
2327 static int __make_request(request_queue_t
*q
, struct bio
*bio
)
2329 struct request
*req
, *freereq
= NULL
;
2330 int el_ret
, rw
, nr_sectors
, cur_nr_sectors
, barrier
, err
;
2333 sector
= bio
->bi_sector
;
2334 nr_sectors
= bio_sectors(bio
);
2335 cur_nr_sectors
= bio_cur_sectors(bio
);
2337 rw
= bio_data_dir(bio
);
2340 * low level driver can indicate that it wants pages above a
2341 * certain limit bounced to low memory (ie for highmem, or even
2342 * ISA dma in theory)
2344 blk_queue_bounce(q
, &bio
);
2346 spin_lock_prefetch(q
->queue_lock
);
2348 barrier
= bio_barrier(bio
);
2349 if (barrier
&& !(q
->queue_flags
& (1 << QUEUE_FLAG_ORDERED
))) {
2355 spin_lock_irq(q
->queue_lock
);
2357 if (elv_queue_empty(q
)) {
2364 el_ret
= elv_merge(q
, &req
, bio
);
2366 case ELEVATOR_BACK_MERGE
:
2367 BUG_ON(!rq_mergeable(req
));
2369 if (!q
->back_merge_fn(q
, req
, bio
))
2372 req
->biotail
->bi_next
= bio
;
2374 req
->nr_sectors
= req
->hard_nr_sectors
+= nr_sectors
;
2375 drive_stat_acct(req
, nr_sectors
, 0);
2376 if (!attempt_back_merge(q
, req
))
2377 elv_merged_request(q
, req
);
2380 case ELEVATOR_FRONT_MERGE
:
2381 BUG_ON(!rq_mergeable(req
));
2383 if (!q
->front_merge_fn(q
, req
, bio
))
2386 bio
->bi_next
= req
->bio
;
2387 req
->cbio
= req
->bio
= bio
;
2388 req
->nr_cbio_segments
= bio_segments(bio
);
2389 req
->nr_cbio_sectors
= bio_sectors(bio
);
2392 * may not be valid. if the low level driver said
2393 * it didn't need a bounce buffer then it better
2394 * not touch req->buffer either...
2396 req
->buffer
= bio_data(bio
);
2397 req
->current_nr_sectors
= cur_nr_sectors
;
2398 req
->hard_cur_sectors
= cur_nr_sectors
;
2399 req
->sector
= req
->hard_sector
= sector
;
2400 req
->nr_sectors
= req
->hard_nr_sectors
+= nr_sectors
;
2401 drive_stat_acct(req
, nr_sectors
, 0);
2402 if (!attempt_front_merge(q
, req
))
2403 elv_merged_request(q
, req
);
2407 * elevator says don't/can't merge. get new request
2409 case ELEVATOR_NO_MERGE
:
2413 printk("elevator returned crap (%d)\n", el_ret
);
2418 * Grab a free request from the freelist - if that is empty, check
2419 * if we are doing read ahead and abort instead of blocking for
2427 spin_unlock_irq(q
->queue_lock
);
2428 if ((freereq
= get_request(q
, rw
, GFP_ATOMIC
)) == NULL
) {
2433 if (bio_rw_ahead(bio
))
2436 freereq
= get_request_wait(q
, rw
);
2441 req
->flags
|= REQ_CMD
;
2444 * inherit FAILFAST from bio (for read-ahead, and explicit FAILFAST)
2446 if (bio_rw_ahead(bio
) || bio_failfast(bio
))
2447 req
->flags
|= REQ_FAILFAST
;
2450 * REQ_BARRIER implies no merging, but lets make it explicit
2453 req
->flags
|= (REQ_HARDBARRIER
| REQ_NOMERGE
);
2456 req
->hard_sector
= req
->sector
= sector
;
2457 req
->hard_nr_sectors
= req
->nr_sectors
= nr_sectors
;
2458 req
->current_nr_sectors
= req
->hard_cur_sectors
= cur_nr_sectors
;
2459 req
->nr_phys_segments
= bio_phys_segments(q
, bio
);
2460 req
->nr_hw_segments
= bio_hw_segments(q
, bio
);
2461 req
->nr_cbio_segments
= bio_segments(bio
);
2462 req
->nr_cbio_sectors
= bio_sectors(bio
);
2463 req
->buffer
= bio_data(bio
); /* see ->buffer comment above */
2464 req
->waiting
= NULL
;
2465 req
->cbio
= req
->bio
= req
->biotail
= bio
;
2466 req
->rq_disk
= bio
->bi_bdev
->bd_disk
;
2467 req
->start_time
= jiffies
;
2469 add_request(q
, req
);
2472 __blk_put_request(q
, freereq
);
2474 __generic_unplug_device(q
);
2476 spin_unlock_irq(q
->queue_lock
);
2480 bio_endio(bio
, nr_sectors
<< 9, err
);
2485 * If bio->bi_dev is a partition, remap the location
2487 static inline void blk_partition_remap(struct bio
*bio
)
2489 struct block_device
*bdev
= bio
->bi_bdev
;
2491 if (bdev
!= bdev
->bd_contains
) {
2492 struct hd_struct
*p
= bdev
->bd_part
;
2494 switch (bio
->bi_rw
) {
2496 p
->read_sectors
+= bio_sectors(bio
);
2500 p
->write_sectors
+= bio_sectors(bio
);
2504 bio
->bi_sector
+= p
->start_sect
;
2505 bio
->bi_bdev
= bdev
->bd_contains
;
2510 * generic_make_request: hand a buffer to its device driver for I/O
2511 * @bio: The bio describing the location in memory and on the device.
2513 * generic_make_request() is used to make I/O requests of block
2514 * devices. It is passed a &struct bio, which describes the I/O that needs
2517 * generic_make_request() does not return any status. The
2518 * success/failure status of the request, along with notification of
2519 * completion, is delivered asynchronously through the bio->bi_end_io
2520 * function described (one day) else where.
2522 * The caller of generic_make_request must make sure that bi_io_vec
2523 * are set to describe the memory buffer, and that bi_dev and bi_sector are
2524 * set to describe the device address, and the
2525 * bi_end_io and optionally bi_private are set to describe how
2526 * completion notification should be signaled.
2528 * generic_make_request and the drivers it calls may use bi_next if this
2529 * bio happens to be merged with someone else, and may change bi_dev and
2530 * bi_sector for remaps as it sees fit. So the values of these fields
2531 * should NOT be depended on after the call to generic_make_request.
2533 void generic_make_request(struct bio
*bio
)
2537 int ret
, nr_sectors
= bio_sectors(bio
);
2540 /* Test device or partition size, when known. */
2541 maxsector
= bio
->bi_bdev
->bd_inode
->i_size
>> 9;
2543 sector_t sector
= bio
->bi_sector
;
2545 if (maxsector
< nr_sectors
||
2546 maxsector
- nr_sectors
< sector
) {
2547 char b
[BDEVNAME_SIZE
];
2548 /* This may well happen - the kernel calls
2549 * bread() without checking the size of the
2550 * device, e.g., when mounting a device. */
2552 "attempt to access beyond end of device\n");
2553 printk(KERN_INFO
"%s: rw=%ld, want=%Lu, limit=%Lu\n",
2554 bdevname(bio
->bi_bdev
, b
),
2556 (unsigned long long) sector
+ nr_sectors
,
2557 (long long) maxsector
);
2559 set_bit(BIO_EOF
, &bio
->bi_flags
);
2565 * Resolve the mapping until finished. (drivers are
2566 * still free to implement/resolve their own stacking
2567 * by explicitly returning 0)
2569 * NOTE: we don't repeat the blk_size check for each new device.
2570 * Stacking drivers are expected to know what they are doing.
2573 char b
[BDEVNAME_SIZE
];
2575 q
= bdev_get_queue(bio
->bi_bdev
);
2578 "generic_make_request: Trying to access "
2579 "nonexistent block-device %s (%Lu)\n",
2580 bdevname(bio
->bi_bdev
, b
),
2581 (long long) bio
->bi_sector
);
2583 bio_endio(bio
, bio
->bi_size
, -EIO
);
2587 if (unlikely(bio_sectors(bio
) > q
->max_hw_sectors
)) {
2588 printk("bio too big device %s (%u > %u)\n",
2589 bdevname(bio
->bi_bdev
, b
),
2595 if (test_bit(QUEUE_FLAG_DEAD
, &q
->queue_flags
))
2599 * If this device has partitions, remap block n
2600 * of partition p to block n+start(p) of the disk.
2602 blk_partition_remap(bio
);
2604 ret
= q
->make_request_fn(q
, bio
);
2608 EXPORT_SYMBOL(generic_make_request
);
2611 * submit_bio: submit a bio to the block device layer for I/O
2612 * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
2613 * @bio: The &struct bio which describes the I/O
2615 * submit_bio() is very similar in purpose to generic_make_request(), and
2616 * uses that function to do most of the work. Both are fairly rough
2617 * interfaces, @bio must be presetup and ready for I/O.
2620 void submit_bio(int rw
, struct bio
*bio
)
2622 int count
= bio_sectors(bio
);
2624 BIO_BUG_ON(!bio
->bi_size
);
2625 BIO_BUG_ON(!bio
->bi_io_vec
);
2628 mod_page_state(pgpgout
, count
);
2630 mod_page_state(pgpgin
, count
);
2632 if (unlikely(block_dump
)) {
2633 char b
[BDEVNAME_SIZE
];
2634 printk(KERN_DEBUG
"%s(%d): %s block %Lu on %s\n",
2635 current
->comm
, current
->pid
,
2636 (rw
& WRITE
) ? "WRITE" : "READ",
2637 (unsigned long long)bio
->bi_sector
,
2638 bdevname(bio
->bi_bdev
,b
));
2641 generic_make_request(bio
);
2644 EXPORT_SYMBOL(submit_bio
);
2647 * blk_rq_next_segment
2648 * @rq: the request being processed
2651 * Points to the next segment in the request if the current segment
2652 * is complete. Leaves things unchanged if this segment is not over
2653 * or if no more segments are left in this request.
2655 * Meant to be used for bio traversal during I/O submission
2656 * Does not affect any I/O completions or update completion state
2657 * in the request, and does not modify any bio fields.
2659 * Decrementing rq->nr_sectors, rq->current_nr_sectors and
2660 * rq->nr_cbio_sectors as data is transferred is the caller's
2661 * responsibility and should be done before calling this routine.
2663 void blk_rq_next_segment(struct request
*rq
)
2665 if (rq
->current_nr_sectors
> 0)
2668 if (rq
->nr_cbio_sectors
> 0) {
2669 --rq
->nr_cbio_segments
;
2670 rq
->current_nr_sectors
= blk_rq_vec(rq
)->bv_len
>> 9;
2672 if ((rq
->cbio
= rq
->cbio
->bi_next
)) {
2673 rq
->nr_cbio_segments
= bio_segments(rq
->cbio
);
2674 rq
->nr_cbio_sectors
= bio_sectors(rq
->cbio
);
2675 rq
->current_nr_sectors
= bio_cur_sectors(rq
->cbio
);
2679 /* remember the size of this segment before we start I/O */
2680 rq
->hard_cur_sectors
= rq
->current_nr_sectors
;
2684 * process_that_request_first - process partial request submission
2685 * @req: the request being processed
2686 * @nr_sectors: number of sectors I/O has been submitted on
2689 * May be used for processing bio's while submitting I/O without
2690 * signalling completion. Fails if more data is requested than is
2691 * available in the request in which case it doesn't advance any
2694 * Assumes a request is correctly set up. No sanity checks.
2697 * 0 - no more data left to submit (not processed)
2698 * 1 - data available to submit for this request (processed)
2700 int process_that_request_first(struct request
*req
, unsigned int nr_sectors
)
2704 if (req
->nr_sectors
< nr_sectors
)
2707 req
->nr_sectors
-= nr_sectors
;
2708 req
->sector
+= nr_sectors
;
2709 while (nr_sectors
) {
2710 nsect
= min_t(unsigned, req
->current_nr_sectors
, nr_sectors
);
2711 req
->current_nr_sectors
-= nsect
;
2712 nr_sectors
-= nsect
;
2714 req
->nr_cbio_sectors
-= nsect
;
2715 blk_rq_next_segment(req
);
2721 EXPORT_SYMBOL(process_that_request_first
);
2723 void blk_recalc_rq_segments(struct request
*rq
)
2725 struct bio
*bio
, *prevbio
= NULL
;
2726 int nr_phys_segs
, nr_hw_segs
;
2731 nr_phys_segs
= nr_hw_segs
= 0;
2732 rq_for_each_bio(bio
, rq
) {
2733 /* Force bio hw/phys segs to be recalculated. */
2734 bio
->bi_flags
&= ~(1 << BIO_SEG_VALID
);
2736 nr_phys_segs
+= bio_phys_segments(rq
->q
, bio
);
2737 nr_hw_segs
+= bio_hw_segments(rq
->q
, bio
);
2739 if (blk_phys_contig_segment(rq
->q
, prevbio
, bio
))
2741 if (blk_hw_contig_segment(rq
->q
, prevbio
, bio
))
2747 rq
->nr_phys_segments
= nr_phys_segs
;
2748 rq
->nr_hw_segments
= nr_hw_segs
;
2751 void blk_recalc_rq_sectors(struct request
*rq
, int nsect
)
2753 if (blk_fs_request(rq
)) {
2754 rq
->hard_sector
+= nsect
;
2755 rq
->hard_nr_sectors
-= nsect
;
2758 * Move the I/O submission pointers ahead if required,
2759 * i.e. for drivers not aware of rq->cbio.
2761 if ((rq
->nr_sectors
>= rq
->hard_nr_sectors
) &&
2762 (rq
->sector
<= rq
->hard_sector
)) {
2763 rq
->sector
= rq
->hard_sector
;
2764 rq
->nr_sectors
= rq
->hard_nr_sectors
;
2765 rq
->hard_cur_sectors
= bio_cur_sectors(rq
->bio
);
2766 rq
->current_nr_sectors
= rq
->hard_cur_sectors
;
2767 rq
->nr_cbio_segments
= bio_segments(rq
->bio
);
2768 rq
->nr_cbio_sectors
= bio_sectors(rq
->bio
);
2769 rq
->buffer
= bio_data(rq
->bio
);
2775 * if total number of sectors is less than the first segment
2776 * size, something has gone terribly wrong
2778 if (rq
->nr_sectors
< rq
->current_nr_sectors
) {
2779 printk("blk: request botched\n");
2780 rq
->nr_sectors
= rq
->current_nr_sectors
;
2785 static int __end_that_request_first(struct request
*req
, int uptodate
,
2788 int total_bytes
, bio_nbytes
, error
, next_idx
= 0;
2792 * extend uptodate bool to allow < 0 value to be direct io error
2795 if (end_io_error(uptodate
))
2796 error
= !uptodate
? -EIO
: uptodate
;
2799 * for a REQ_BLOCK_PC request, we want to carry any eventual
2800 * sense key with us all the way through
2802 if (!blk_pc_request(req
))
2806 if (blk_fs_request(req
) && !(req
->flags
& REQ_QUIET
))
2807 printk("end_request: I/O error, dev %s, sector %llu\n",
2808 req
->rq_disk
? req
->rq_disk
->disk_name
: "?",
2809 (unsigned long long)req
->sector
);
2812 total_bytes
= bio_nbytes
= 0;
2813 while ((bio
= req
->bio
) != NULL
) {
2816 if (nr_bytes
>= bio
->bi_size
) {
2817 req
->bio
= bio
->bi_next
;
2818 nbytes
= bio
->bi_size
;
2819 bio_endio(bio
, nbytes
, error
);
2823 int idx
= bio
->bi_idx
+ next_idx
;
2825 if (unlikely(bio
->bi_idx
>= bio
->bi_vcnt
)) {
2826 blk_dump_rq_flags(req
, "__end_that");
2827 printk("%s: bio idx %d >= vcnt %d\n",
2829 bio
->bi_idx
, bio
->bi_vcnt
);
2833 nbytes
= bio_iovec_idx(bio
, idx
)->bv_len
;
2834 BIO_BUG_ON(nbytes
> bio
->bi_size
);
2837 * not a complete bvec done
2839 if (unlikely(nbytes
> nr_bytes
)) {
2840 bio_nbytes
+= nr_bytes
;
2841 total_bytes
+= nr_bytes
;
2846 * advance to the next vector
2849 bio_nbytes
+= nbytes
;
2852 total_bytes
+= nbytes
;
2855 if ((bio
= req
->bio
)) {
2857 * end more in this run, or just return 'not-done'
2859 if (unlikely(nr_bytes
<= 0))
2871 * if the request wasn't completed, update state
2874 bio_endio(bio
, bio_nbytes
, error
);
2875 bio
->bi_idx
+= next_idx
;
2876 bio_iovec(bio
)->bv_offset
+= nr_bytes
;
2877 bio_iovec(bio
)->bv_len
-= nr_bytes
;
2880 blk_recalc_rq_sectors(req
, total_bytes
>> 9);
2881 blk_recalc_rq_segments(req
);
2886 * end_that_request_first - end I/O on a request
2887 * @req: the request being processed
2888 * @uptodate: 1 for success, 0 for I/O error, < 0 for specific error
2889 * @nr_sectors: number of sectors to end I/O on
2892 * Ends I/O on a number of sectors attached to @req, and sets it up
2893 * for the next range of segments (if any) in the cluster.
2896 * 0 - we are done with this request, call end_that_request_last()
2897 * 1 - still buffers pending for this request
2899 int end_that_request_first(struct request
*req
, int uptodate
, int nr_sectors
)
2901 return __end_that_request_first(req
, uptodate
, nr_sectors
<< 9);
2904 EXPORT_SYMBOL(end_that_request_first
);
2907 * end_that_request_chunk - end I/O on a request
2908 * @req: the request being processed
2909 * @uptodate: 1 for success, 0 for I/O error, < 0 for specific error
2910 * @nr_bytes: number of bytes to complete
2913 * Ends I/O on a number of bytes attached to @req, and sets it up
2914 * for the next range of segments (if any). Like end_that_request_first(),
2915 * but deals with bytes instead of sectors.
2918 * 0 - we are done with this request, call end_that_request_last()
2919 * 1 - still buffers pending for this request
2921 int end_that_request_chunk(struct request
*req
, int uptodate
, int nr_bytes
)
2923 return __end_that_request_first(req
, uptodate
, nr_bytes
);
2926 EXPORT_SYMBOL(end_that_request_chunk
);
2929 * queue lock must be held
2931 void end_that_request_last(struct request
*req
)
2933 struct gendisk
*disk
= req
->rq_disk
;
2934 struct completion
*waiting
= req
->waiting
;
2936 if (unlikely(laptop_mode
) && blk_fs_request(req
))
2937 laptop_io_completion();
2939 if (disk
&& blk_fs_request(req
)) {
2940 unsigned long duration
= jiffies
- req
->start_time
;
2941 switch (rq_data_dir(req
)) {
2943 disk_stat_inc(disk
, writes
);
2944 disk_stat_add(disk
, write_ticks
, duration
);
2947 disk_stat_inc(disk
, reads
);
2948 disk_stat_add(disk
, read_ticks
, duration
);
2951 disk_round_stats(disk
);
2954 __blk_put_request(req
->q
, req
);
2955 /* Do this LAST! The structure may be freed immediately afterwards */
2960 EXPORT_SYMBOL(end_that_request_last
);
2962 void end_request(struct request
*req
, int uptodate
)
2964 if (!end_that_request_first(req
, uptodate
, req
->hard_cur_sectors
)) {
2965 add_disk_randomness(req
->rq_disk
);
2966 blkdev_dequeue_request(req
);
2967 end_that_request_last(req
);
2971 EXPORT_SYMBOL(end_request
);
2973 void blk_rq_bio_prep(request_queue_t
*q
, struct request
*rq
, struct bio
*bio
)
2975 /* first three bits are identical in rq->flags and bio->bi_rw */
2976 rq
->flags
|= (bio
->bi_rw
& 7);
2978 rq
->nr_phys_segments
= bio_phys_segments(q
, bio
);
2979 rq
->nr_hw_segments
= bio_hw_segments(q
, bio
);
2980 rq
->current_nr_sectors
= bio_cur_sectors(bio
);
2981 rq
->hard_cur_sectors
= rq
->current_nr_sectors
;
2982 rq
->hard_nr_sectors
= rq
->nr_sectors
= bio_sectors(bio
);
2983 rq
->nr_cbio_segments
= bio_segments(bio
);
2984 rq
->nr_cbio_sectors
= bio_sectors(bio
);
2985 rq
->buffer
= bio_data(bio
);
2987 rq
->cbio
= rq
->bio
= rq
->biotail
= bio
;
2990 EXPORT_SYMBOL(blk_rq_bio_prep
);
2992 void blk_rq_prep_restart(struct request
*rq
)
2996 bio
= rq
->cbio
= rq
->bio
;
2998 rq
->nr_cbio_segments
= bio_segments(bio
);
2999 rq
->nr_cbio_sectors
= bio_sectors(bio
);
3000 rq
->hard_cur_sectors
= bio_cur_sectors(bio
);
3001 rq
->buffer
= bio_data(bio
);
3003 rq
->sector
= rq
->hard_sector
;
3004 rq
->nr_sectors
= rq
->hard_nr_sectors
;
3005 rq
->current_nr_sectors
= rq
->hard_cur_sectors
;
3008 EXPORT_SYMBOL(blk_rq_prep_restart
);
3010 int kblockd_schedule_work(struct work_struct
*work
)
3012 return queue_work(kblockd_workqueue
, work
);
3015 EXPORT_SYMBOL(kblockd_schedule_work
);
3017 void kblockd_flush(void)
3019 flush_workqueue(kblockd_workqueue
);
3022 int __init
blk_dev_init(void)
3024 kblockd_workqueue
= create_workqueue("kblockd");
3025 if (!kblockd_workqueue
)
3026 panic("Failed to create kblockd\n");
3028 request_cachep
= kmem_cache_create("blkdev_requests",
3029 sizeof(struct request
), 0, SLAB_PANIC
, NULL
, NULL
);
3031 requestq_cachep
= kmem_cache_create("blkdev_queue",
3032 sizeof(request_queue_t
), 0, SLAB_PANIC
, NULL
, NULL
);
3034 iocontext_cachep
= kmem_cache_create("blkdev_ioc",
3035 sizeof(struct io_context
), 0, SLAB_PANIC
, NULL
, NULL
);
3037 blk_max_low_pfn
= max_low_pfn
;
3038 blk_max_pfn
= max_pfn
;
3043 * IO Context helper functions
3045 void put_io_context(struct io_context
*ioc
)
3050 BUG_ON(atomic_read(&ioc
->refcount
) == 0);
3052 if (atomic_dec_and_test(&ioc
->refcount
)) {
3053 if (ioc
->aic
&& ioc
->aic
->dtor
)
3054 ioc
->aic
->dtor(ioc
->aic
);
3055 kmem_cache_free(iocontext_cachep
, ioc
);
3059 /* Called by the exitting task */
3060 void exit_io_context(void)
3062 unsigned long flags
;
3063 struct io_context
*ioc
;
3065 local_irq_save(flags
);
3066 ioc
= current
->io_context
;
3068 if (ioc
->aic
&& ioc
->aic
->exit
)
3069 ioc
->aic
->exit(ioc
->aic
);
3070 put_io_context(ioc
);
3071 current
->io_context
= NULL
;
3074 local_irq_restore(flags
);
3078 * If the current task has no IO context then create one and initialise it.
3079 * If it does have a context, take a ref on it.
3081 * This is always called in the context of the task which submitted the I/O.
3082 * But weird things happen, so we disable local interrupts to ensure exclusive
3083 * access to *current.
3085 struct io_context
*get_io_context(int gfp_flags
)
3087 struct task_struct
*tsk
= current
;
3088 unsigned long flags
;
3089 struct io_context
*ret
;
3091 local_irq_save(flags
);
3092 ret
= tsk
->io_context
;
3094 ret
= kmem_cache_alloc(iocontext_cachep
, GFP_ATOMIC
);
3096 atomic_set(&ret
->refcount
, 1);
3097 ret
->pid
= tsk
->pid
;
3098 ret
->last_waited
= jiffies
; /* doesn't matter... */
3099 ret
->nr_batch_requests
= 0; /* because this is 0 */
3101 tsk
->io_context
= ret
;
3105 atomic_inc(&ret
->refcount
);
3106 local_irq_restore(flags
);
3110 void copy_io_context(struct io_context
**pdst
, struct io_context
**psrc
)
3112 struct io_context
*src
= *psrc
;
3113 struct io_context
*dst
= *pdst
;
3116 BUG_ON(atomic_read(&src
->refcount
) == 0);
3117 atomic_inc(&src
->refcount
);
3118 put_io_context(dst
);
3123 void swap_io_context(struct io_context
**ioc1
, struct io_context
**ioc2
)
3125 struct io_context
*temp
;
3135 struct queue_sysfs_entry
{
3136 struct attribute attr
;
3137 ssize_t (*show
)(struct request_queue
*, char *);
3138 ssize_t (*store
)(struct request_queue
*, const char *, size_t);
3142 queue_var_show(unsigned int var
, char *page
)
3144 return sprintf(page
, "%d\n", var
);
3148 queue_var_store(unsigned long *var
, const char *page
, size_t count
)
3150 char *p
= (char *) page
;
3152 *var
= simple_strtoul(p
, &p
, 10);
3156 static ssize_t
queue_requests_show(struct request_queue
*q
, char *page
)
3158 return queue_var_show(q
->nr_requests
, (page
));
3162 queue_requests_store(struct request_queue
*q
, const char *page
, size_t count
)
3164 struct request_list
*rl
= &q
->rq
;
3166 int ret
= queue_var_store(&q
->nr_requests
, page
, count
);
3167 if (q
->nr_requests
< BLKDEV_MIN_RQ
)
3168 q
->nr_requests
= BLKDEV_MIN_RQ
;
3169 blk_queue_congestion_threshold(q
);
3171 if (rl
->count
[READ
] >= queue_congestion_on_threshold(q
))
3172 set_queue_congested(q
, READ
);
3173 else if (rl
->count
[READ
] < queue_congestion_off_threshold(q
))
3174 clear_queue_congested(q
, READ
);
3176 if (rl
->count
[WRITE
] >= queue_congestion_on_threshold(q
))
3177 set_queue_congested(q
, WRITE
);
3178 else if (rl
->count
[WRITE
] < queue_congestion_off_threshold(q
))
3179 clear_queue_congested(q
, WRITE
);
3181 if (rl
->count
[READ
] >= q
->nr_requests
) {
3182 blk_set_queue_full(q
, READ
);
3183 } else if (rl
->count
[READ
]+1 <= q
->nr_requests
) {
3184 blk_clear_queue_full(q
, READ
);
3185 wake_up(&rl
->wait
[READ
]);
3188 if (rl
->count
[WRITE
] >= q
->nr_requests
) {
3189 blk_set_queue_full(q
, WRITE
);
3190 } else if (rl
->count
[WRITE
]+1 <= q
->nr_requests
) {
3191 blk_clear_queue_full(q
, WRITE
);
3192 wake_up(&rl
->wait
[WRITE
]);
3197 static ssize_t
queue_ra_show(struct request_queue
*q
, char *page
)
3199 int ra_kb
= q
->backing_dev_info
.ra_pages
<< (PAGE_CACHE_SHIFT
- 10);
3201 return queue_var_show(ra_kb
, (page
));
3205 queue_ra_store(struct request_queue
*q
, const char *page
, size_t count
)
3207 unsigned long ra_kb
;
3208 ssize_t ret
= queue_var_store(&ra_kb
, page
, count
);
3210 spin_lock_irq(q
->queue_lock
);
3211 if (ra_kb
> (q
->max_sectors
>> 1))
3212 ra_kb
= (q
->max_sectors
>> 1);
3214 q
->backing_dev_info
.ra_pages
= ra_kb
>> (PAGE_CACHE_SHIFT
- 10);
3215 spin_unlock_irq(q
->queue_lock
);
3220 static ssize_t
queue_max_sectors_show(struct request_queue
*q
, char *page
)
3222 int max_sectors_kb
= q
->max_sectors
>> 1;
3224 return queue_var_show(max_sectors_kb
, (page
));
3228 queue_max_sectors_store(struct request_queue
*q
, const char *page
, size_t count
)
3230 unsigned long max_sectors_kb
,
3231 max_hw_sectors_kb
= q
->max_hw_sectors
>> 1,
3232 page_kb
= 1 << (PAGE_CACHE_SHIFT
- 10);
3233 ssize_t ret
= queue_var_store(&max_sectors_kb
, page
, count
);
3236 if (max_sectors_kb
> max_hw_sectors_kb
|| max_sectors_kb
< page_kb
)
3239 * Take the queue lock to update the readahead and max_sectors
3240 * values synchronously:
3242 spin_lock_irq(q
->queue_lock
);
3244 * Trim readahead window as well, if necessary:
3246 ra_kb
= q
->backing_dev_info
.ra_pages
<< (PAGE_CACHE_SHIFT
- 10);
3247 if (ra_kb
> max_sectors_kb
)
3248 q
->backing_dev_info
.ra_pages
=
3249 max_sectors_kb
>> (PAGE_CACHE_SHIFT
- 10);
3251 q
->max_sectors
= max_sectors_kb
<< 1;
3252 spin_unlock_irq(q
->queue_lock
);
3257 static ssize_t
queue_max_hw_sectors_show(struct request_queue
*q
, char *page
)
3259 int max_hw_sectors_kb
= q
->max_hw_sectors
>> 1;
3261 return queue_var_show(max_hw_sectors_kb
, (page
));
3265 static struct queue_sysfs_entry queue_requests_entry
= {
3266 .attr
= {.name
= "nr_requests", .mode
= S_IRUGO
| S_IWUSR
},
3267 .show
= queue_requests_show
,
3268 .store
= queue_requests_store
,
3271 static struct queue_sysfs_entry queue_ra_entry
= {
3272 .attr
= {.name
= "read_ahead_kb", .mode
= S_IRUGO
| S_IWUSR
},
3273 .show
= queue_ra_show
,
3274 .store
= queue_ra_store
,
3277 static struct queue_sysfs_entry queue_max_sectors_entry
= {
3278 .attr
= {.name
= "max_sectors_kb", .mode
= S_IRUGO
| S_IWUSR
},
3279 .show
= queue_max_sectors_show
,
3280 .store
= queue_max_sectors_store
,
3283 static struct queue_sysfs_entry queue_max_hw_sectors_entry
= {
3284 .attr
= {.name
= "max_hw_sectors_kb", .mode
= S_IRUGO
},
3285 .show
= queue_max_hw_sectors_show
,
3288 static struct attribute
*default_attrs
[] = {
3289 &queue_requests_entry
.attr
,
3290 &queue_ra_entry
.attr
,
3291 &queue_max_hw_sectors_entry
.attr
,
3292 &queue_max_sectors_entry
.attr
,
3296 #define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)
3299 queue_attr_show(struct kobject
*kobj
, struct attribute
*attr
, char *page
)
3301 struct queue_sysfs_entry
*entry
= to_queue(attr
);
3302 struct request_queue
*q
;
3304 q
= container_of(kobj
, struct request_queue
, kobj
);
3308 return entry
->show(q
, page
);
3312 queue_attr_store(struct kobject
*kobj
, struct attribute
*attr
,
3313 const char *page
, size_t length
)
3315 struct queue_sysfs_entry
*entry
= to_queue(attr
);
3316 struct request_queue
*q
;
3318 q
= container_of(kobj
, struct request_queue
, kobj
);
3322 return entry
->store(q
, page
, length
);
3325 static struct sysfs_ops queue_sysfs_ops
= {
3326 .show
= queue_attr_show
,
3327 .store
= queue_attr_store
,
3330 struct kobj_type queue_ktype
= {
3331 .sysfs_ops
= &queue_sysfs_ops
,
3332 .default_attrs
= default_attrs
,
3335 int blk_register_queue(struct gendisk
*disk
)
3339 request_queue_t
*q
= disk
->queue
;
3341 if (!q
|| !q
->request_fn
)
3344 q
->kobj
.parent
= kobject_get(&disk
->kobj
);
3345 if (!q
->kobj
.parent
)
3348 snprintf(q
->kobj
.name
, KOBJ_NAME_LEN
, "%s", "queue");
3349 q
->kobj
.ktype
= &queue_ktype
;
3351 ret
= kobject_register(&q
->kobj
);
3355 ret
= elv_register_queue(q
);
3357 kobject_unregister(&q
->kobj
);
3364 void blk_unregister_queue(struct gendisk
*disk
)
3366 request_queue_t
*q
= disk
->queue
;
3368 if (q
&& q
->request_fn
) {
3369 elv_unregister_queue(q
);
3371 kobject_unregister(&q
->kobj
);
3372 kobject_put(&disk
->kobj
);