2 * Functions related to tagged command queuing
4 #include <linux/kernel.h>
5 #include <linux/module.h>
7 #include <linux/blkdev.h>
12 * blk_queue_find_tag - find a request by its tag and queue
13 * @q: The request queue for the device
14 * @tag: The tag of the request
17 * Should be used when a device returns a tag and you want to match
20 * no locks need be held.
22 struct request
*blk_queue_find_tag(struct request_queue
*q
, int tag
)
24 return blk_map_queue_find_tag(q
->queue_tags
, tag
);
26 EXPORT_SYMBOL(blk_queue_find_tag
);
29 * __blk_free_tags - release a given set of tag maintenance info
30 * @bqt: the tag map to free
32 * Tries to free the specified @bqt. Returns true if it was
33 * actually freed and false if there are still references using it
35 static int __blk_free_tags(struct blk_queue_tag
*bqt
)
39 retval
= atomic_dec_and_test(&bqt
->refcnt
);
41 BUG_ON(find_first_bit(bqt
->tag_map
, bqt
->max_depth
) <
44 kfree(bqt
->tag_index
);
45 bqt
->tag_index
= NULL
;
57 * __blk_queue_free_tags - release tag maintenance info
58 * @q: the request queue for the device
61 * blk_cleanup_queue() will take care of calling this function, if tagging
62 * has been used. So there's no need to call this directly.
64 void __blk_queue_free_tags(struct request_queue
*q
)
66 struct blk_queue_tag
*bqt
= q
->queue_tags
;
74 queue_flag_clear_unlocked(QUEUE_FLAG_QUEUED
, q
);
78 * blk_free_tags - release a given set of tag maintenance info
79 * @bqt: the tag map to free
81 * For externally managed @bqt frees the map. Callers of this
82 * function must guarantee to have released all the queues that
83 * might have been using this tag map.
85 void blk_free_tags(struct blk_queue_tag
*bqt
)
87 if (unlikely(!__blk_free_tags(bqt
)))
90 EXPORT_SYMBOL(blk_free_tags
);
93 * blk_queue_free_tags - release tag maintenance info
94 * @q: the request queue for the device
97 * This is used to disable tagged queuing to a device, yet leave
100 void blk_queue_free_tags(struct request_queue
*q
)
102 queue_flag_clear_unlocked(QUEUE_FLAG_QUEUED
, q
);
104 EXPORT_SYMBOL(blk_queue_free_tags
);
107 init_tag_map(struct request_queue
*q
, struct blk_queue_tag
*tags
, int depth
)
109 struct request
**tag_index
;
110 unsigned long *tag_map
;
113 if (q
&& depth
> q
->nr_requests
* 2) {
114 depth
= q
->nr_requests
* 2;
115 printk(KERN_ERR
"%s: adjusted depth to %d\n",
119 tag_index
= kzalloc(depth
* sizeof(struct request
*), GFP_ATOMIC
);
123 nr_ulongs
= ALIGN(depth
, BITS_PER_LONG
) / BITS_PER_LONG
;
124 tag_map
= kzalloc(nr_ulongs
* sizeof(unsigned long), GFP_ATOMIC
);
128 tags
->real_max_depth
= depth
;
129 tags
->max_depth
= depth
;
130 tags
->tag_index
= tag_index
;
131 tags
->tag_map
= tag_map
;
139 static struct blk_queue_tag
*__blk_queue_init_tags(struct request_queue
*q
,
142 struct blk_queue_tag
*tags
;
144 tags
= kmalloc(sizeof(struct blk_queue_tag
), GFP_ATOMIC
);
148 if (init_tag_map(q
, tags
, depth
))
151 atomic_set(&tags
->refcnt
, 1);
159 * blk_init_tags - initialize the tag info for an external tag map
160 * @depth: the maximum queue depth supported
162 struct blk_queue_tag
*blk_init_tags(int depth
)
164 return __blk_queue_init_tags(NULL
, depth
);
166 EXPORT_SYMBOL(blk_init_tags
);
169 * blk_queue_init_tags - initialize the queue tag info
170 * @q: the request queue for the device
171 * @depth: the maximum queue depth supported
172 * @tags: the tag to use
174 * Queue lock must be held here if the function is called to resize an
177 int blk_queue_init_tags(struct request_queue
*q
, int depth
,
178 struct blk_queue_tag
*tags
)
182 BUG_ON(tags
&& q
->queue_tags
&& tags
!= q
->queue_tags
);
184 if (!tags
&& !q
->queue_tags
) {
185 tags
= __blk_queue_init_tags(q
, depth
);
189 } else if (q
->queue_tags
) {
190 rc
= blk_queue_resize_tags(q
, depth
);
193 queue_flag_set(QUEUE_FLAG_QUEUED
, q
);
196 atomic_inc(&tags
->refcnt
);
199 * assign it, all done
201 q
->queue_tags
= tags
;
202 queue_flag_set_unlocked(QUEUE_FLAG_QUEUED
, q
);
203 INIT_LIST_HEAD(&q
->tag_busy_list
);
209 EXPORT_SYMBOL(blk_queue_init_tags
);
212 * blk_queue_resize_tags - change the queueing depth
213 * @q: the request queue for the device
214 * @new_depth: the new max command queueing depth
217 * Must be called with the queue lock held.
219 int blk_queue_resize_tags(struct request_queue
*q
, int new_depth
)
221 struct blk_queue_tag
*bqt
= q
->queue_tags
;
222 struct request
**tag_index
;
223 unsigned long *tag_map
;
224 int max_depth
, nr_ulongs
;
230 * if we already have large enough real_max_depth. just
231 * adjust max_depth. *NOTE* as requests with tag value
232 * between new_depth and real_max_depth can be in-flight, tag
233 * map can not be shrunk blindly here.
235 if (new_depth
<= bqt
->real_max_depth
) {
236 bqt
->max_depth
= new_depth
;
241 * Currently cannot replace a shared tag map with a new
242 * one, so error out if this is the case
244 if (atomic_read(&bqt
->refcnt
) != 1)
248 * save the old state info, so we can copy it back
250 tag_index
= bqt
->tag_index
;
251 tag_map
= bqt
->tag_map
;
252 max_depth
= bqt
->real_max_depth
;
254 if (init_tag_map(q
, bqt
, new_depth
))
257 memcpy(bqt
->tag_index
, tag_index
, max_depth
* sizeof(struct request
*));
258 nr_ulongs
= ALIGN(max_depth
, BITS_PER_LONG
) / BITS_PER_LONG
;
259 memcpy(bqt
->tag_map
, tag_map
, nr_ulongs
* sizeof(unsigned long));
265 EXPORT_SYMBOL(blk_queue_resize_tags
);
268 * blk_queue_end_tag - end tag operations for a request
269 * @q: the request queue for the device
270 * @rq: the request that has completed
273 * Typically called when end_that_request_first() returns %0, meaning
274 * all transfers have been done for a request. It's important to call
275 * this function before end_that_request_last(), as that will put the
276 * request back on the free list thus corrupting the internal tag list.
279 * queue lock must be held.
281 void blk_queue_end_tag(struct request_queue
*q
, struct request
*rq
)
283 struct blk_queue_tag
*bqt
= q
->queue_tags
;
288 if (unlikely(tag
>= bqt
->real_max_depth
))
290 * This can happen after tag depth has been reduced.
291 * FIXME: how about a warning or info message here?
295 list_del_init(&rq
->queuelist
);
296 rq
->cmd_flags
&= ~REQ_QUEUED
;
299 if (unlikely(bqt
->tag_index
[tag
] == NULL
))
300 printk(KERN_ERR
"%s: tag %d is missing\n",
303 bqt
->tag_index
[tag
] = NULL
;
305 if (unlikely(!test_bit(tag
, bqt
->tag_map
))) {
306 printk(KERN_ERR
"%s: attempt to clear non-busy tag (%d)\n",
311 * The tag_map bit acts as a lock for tag_index[bit], so we need
312 * unlock memory barrier semantics.
314 clear_bit_unlock(tag
, bqt
->tag_map
);
316 EXPORT_SYMBOL(blk_queue_end_tag
);
319 * blk_queue_start_tag - find a free tag and assign it
320 * @q: the request queue for the device
321 * @rq: the block request that needs tagging
324 * This can either be used as a stand-alone helper, or possibly be
325 * assigned as the queue &prep_rq_fn (in which case &struct request
326 * automagically gets a tag assigned). Note that this function
327 * assumes that any type of request can be queued! if this is not
328 * true for your device, you must check the request type before
329 * calling this function. The request will also be removed from
330 * the request queue, so it's the drivers responsibility to readd
331 * it if it should need to be restarted for some reason.
334 * queue lock must be held.
336 int blk_queue_start_tag(struct request_queue
*q
, struct request
*rq
)
338 struct blk_queue_tag
*bqt
= q
->queue_tags
;
339 unsigned max_depth
, offset
;
342 if (unlikely((rq
->cmd_flags
& REQ_QUEUED
))) {
344 "%s: request %p for device [%s] already tagged %d",
346 rq
->rq_disk
? rq
->rq_disk
->disk_name
: "?", rq
->tag
);
351 * Protect against shared tag maps, as we may not have exclusive
352 * access to the tag map.
354 * We reserve a few tags just for sync IO, since we don't want
355 * to starve sync IO on behalf of flooding async IO.
357 max_depth
= bqt
->max_depth
;
361 offset
= max_depth
>> 2;
364 tag
= find_next_zero_bit(bqt
->tag_map
, max_depth
, offset
);
365 if (tag
>= max_depth
)
368 } while (test_and_set_bit_lock(tag
, bqt
->tag_map
));
370 * We need lock ordering semantics given by test_and_set_bit_lock.
371 * See blk_queue_end_tag for details.
374 rq
->cmd_flags
|= REQ_QUEUED
;
376 bqt
->tag_index
[tag
] = rq
;
377 blkdev_dequeue_request(rq
);
378 list_add(&rq
->queuelist
, &q
->tag_busy_list
);
381 EXPORT_SYMBOL(blk_queue_start_tag
);
384 * blk_queue_invalidate_tags - invalidate all pending tags
385 * @q: the request queue for the device
388 * Hardware conditions may dictate a need to stop all pending requests.
389 * In this case, we will safely clear the block side of the tag queue and
390 * readd all requests to the request queue in the right order.
393 * queue lock must be held.
395 void blk_queue_invalidate_tags(struct request_queue
*q
)
397 struct list_head
*tmp
, *n
;
399 list_for_each_safe(tmp
, n
, &q
->tag_busy_list
)
400 blk_requeue_request(q
, list_entry_rq(tmp
));
402 EXPORT_SYMBOL(blk_queue_invalidate_tags
);