?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
blkcg: use double locking instead of RCU for blkg synchronization
[linux-2.6.git] / block / blk-cgroup.c
blobe9e3b038c702ee981a1cd36fb7c5ded0c29d38d1
1 /*
2 * Common Block IO controller cgroup interface
3 *
4 * Based on ideas and code from CFQ, CFS and BFQ:
5 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
6 *
7 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
8 * Paolo Valente <paolo.valente@unimore.it>
9 *
10 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
11 * Nauman Rafique <nauman@google.com>
12 */
13 #include <linux/ioprio.h>
14 #include <linux/seq_file.h>
15 #include <linux/kdev_t.h>
16 #include <linux/module.h>
17 #include <linux/err.h>
18 #include <linux/blkdev.h>
19 #include <linux/slab.h>
20 #include <linux/genhd.h>
21 #include <linux/delay.h>
22 #include "blk-cgroup.h"
23 #include "blk.h"
24
25 #define MAX_KEY_LEN 100
26
27 static DEFINE_SPINLOCK(blkio_list_lock);
28 static LIST_HEAD(blkio_list);
29
30 static DEFINE_MUTEX(all_q_mutex);
31 static LIST_HEAD(all_q_list);
32
33 struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT };
34 EXPORT_SYMBOL_GPL(blkio_root_cgroup);
35
36 static struct blkio_policy_type *blkio_policy[BLKIO_NR_POLICIES];
37
38 static struct cgroup_subsys_state *blkiocg_create(struct cgroup_subsys *,
39 struct cgroup *);
40 static int blkiocg_can_attach(struct cgroup_subsys *, struct cgroup *,
41 struct cgroup_taskset *);
42 static void blkiocg_attach(struct cgroup_subsys *, struct cgroup *,
43 struct cgroup_taskset *);
44 static int blkiocg_pre_destroy(struct cgroup_subsys *, struct cgroup *);
45 static void blkiocg_destroy(struct cgroup_subsys *, struct cgroup *);
46 static int blkiocg_populate(struct cgroup_subsys *, struct cgroup *);
47
48 /* for encoding cft->private value on file */
49 #define BLKIOFILE_PRIVATE(x, val) (((x) << 16) | (val))
50 /* What policy owns the file, proportional or throttle */
51 #define BLKIOFILE_POLICY(val) (((val) >> 16) & 0xffff)
52 #define BLKIOFILE_ATTR(val) ((val) & 0xffff)
53
54 struct cgroup_subsys blkio_subsys = {
55 .name = "blkio",
56 .create = blkiocg_create,
57 .can_attach = blkiocg_can_attach,
58 .attach = blkiocg_attach,
59 .pre_destroy = blkiocg_pre_destroy,
60 .destroy = blkiocg_destroy,
61 .populate = blkiocg_populate,
62 .subsys_id = blkio_subsys_id,
63 .module = THIS_MODULE,
64 };
65 EXPORT_SYMBOL_GPL(blkio_subsys);
66
67 struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup)
68 {
69 return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
70 struct blkio_cgroup, css);
71 }
72 EXPORT_SYMBOL_GPL(cgroup_to_blkio_cgroup);
73
74 struct blkio_cgroup *task_blkio_cgroup(struct task_struct *tsk)
75 {
76 return container_of(task_subsys_state(tsk, blkio_subsys_id),
77 struct blkio_cgroup, css);
78 }
79 EXPORT_SYMBOL_GPL(task_blkio_cgroup);
80
81 static inline void blkio_update_group_weight(struct blkio_group *blkg,
82 int plid, unsigned int weight)
83 {
84 struct blkio_policy_type *blkiop;
85
86 list_for_each_entry(blkiop, &blkio_list, list) {
87 /* If this policy does not own the blkg, do not send updates */
88 if (blkiop->plid != plid)
89 continue;
90 if (blkiop->ops.blkio_update_group_weight_fn)
91 blkiop->ops.blkio_update_group_weight_fn(blkg->q,
92 blkg, weight);
93 }
94 }
95
96 static inline void blkio_update_group_bps(struct blkio_group *blkg, int plid,
97 u64 bps, int fileid)
98 {
99 struct blkio_policy_type *blkiop;
100
101 list_for_each_entry(blkiop, &blkio_list, list) {
102
103 /* If this policy does not own the blkg, do not send updates */
104 if (blkiop->plid != plid)
105 continue;
106
107 if (fileid == BLKIO_THROTL_read_bps_device
108 && blkiop->ops.blkio_update_group_read_bps_fn)
109 blkiop->ops.blkio_update_group_read_bps_fn(blkg->q,
110 blkg, bps);
111
112 if (fileid == BLKIO_THROTL_write_bps_device
113 && blkiop->ops.blkio_update_group_write_bps_fn)
114 blkiop->ops.blkio_update_group_write_bps_fn(blkg->q,
115 blkg, bps);
116 }
117 }
118
119 static inline void blkio_update_group_iops(struct blkio_group *blkg,
120 int plid, unsigned int iops,
121 int fileid)
122 {
123 struct blkio_policy_type *blkiop;
124
125 list_for_each_entry(blkiop, &blkio_list, list) {
126
127 /* If this policy does not own the blkg, do not send updates */
128 if (blkiop->plid != plid)
129 continue;
130
131 if (fileid == BLKIO_THROTL_read_iops_device
132 && blkiop->ops.blkio_update_group_read_iops_fn)
133 blkiop->ops.blkio_update_group_read_iops_fn(blkg->q,
134 blkg, iops);
135
136 if (fileid == BLKIO_THROTL_write_iops_device
137 && blkiop->ops.blkio_update_group_write_iops_fn)
138 blkiop->ops.blkio_update_group_write_iops_fn(blkg->q,
139 blkg,iops);
140 }
141 }
142
143 /*
144 * Add to the appropriate stat variable depending on the request type.
145 * This should be called with the blkg->stats_lock held.
146 */
147 static void blkio_add_stat(uint64_t *stat, uint64_t add, bool direction,
148 bool sync)
149 {
150 if (direction)
151 stat[BLKIO_STAT_WRITE] += add;
152 else
153 stat[BLKIO_STAT_READ] += add;
154 if (sync)
155 stat[BLKIO_STAT_SYNC] += add;
156 else
157 stat[BLKIO_STAT_ASYNC] += add;
158 }
159
160 /*
161 * Decrements the appropriate stat variable if non-zero depending on the
162 * request type. Panics on value being zero.
163 * This should be called with the blkg->stats_lock held.
164 */
165 static void blkio_check_and_dec_stat(uint64_t *stat, bool direction, bool sync)
166 {
167 if (direction) {
168 BUG_ON(stat[BLKIO_STAT_WRITE] == 0);
169 stat[BLKIO_STAT_WRITE]--;
170 } else {
171 BUG_ON(stat[BLKIO_STAT_READ] == 0);
172 stat[BLKIO_STAT_READ]--;
173 }
174 if (sync) {
175 BUG_ON(stat[BLKIO_STAT_SYNC] == 0);
176 stat[BLKIO_STAT_SYNC]--;
177 } else {
178 BUG_ON(stat[BLKIO_STAT_ASYNC] == 0);
179 stat[BLKIO_STAT_ASYNC]--;
180 }
181 }
182
183 #ifdef CONFIG_DEBUG_BLK_CGROUP
184 /* This should be called with the blkg->stats_lock held. */
185 static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
186 struct blkio_policy_type *pol,
187 struct blkio_group *curr_blkg)
188 {
189 struct blkg_policy_data *pd = blkg->pd[pol->plid];
190
191 if (blkio_blkg_waiting(&pd->stats))
192 return;
193 if (blkg == curr_blkg)
194 return;
195 pd->stats.start_group_wait_time = sched_clock();
196 blkio_mark_blkg_waiting(&pd->stats);
197 }
198
199 /* This should be called with the blkg->stats_lock held. */
200 static void blkio_update_group_wait_time(struct blkio_group_stats *stats)
201 {
202 unsigned long long now;
203
204 if (!blkio_blkg_waiting(stats))
205 return;
206
207 now = sched_clock();
208 if (time_after64(now, stats->start_group_wait_time))
209 stats->group_wait_time += now - stats->start_group_wait_time;
210 blkio_clear_blkg_waiting(stats);
211 }
212
213 /* This should be called with the blkg->stats_lock held. */
214 static void blkio_end_empty_time(struct blkio_group_stats *stats)
215 {
216 unsigned long long now;
217
218 if (!blkio_blkg_empty(stats))
219 return;
220
221 now = sched_clock();
222 if (time_after64(now, stats->start_empty_time))
223 stats->empty_time += now - stats->start_empty_time;
224 blkio_clear_blkg_empty(stats);
225 }
226
227 void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
228 struct blkio_policy_type *pol)
229 {
230 struct blkg_policy_data *pd = blkg->pd[pol->plid];
231 unsigned long flags;
232
233 spin_lock_irqsave(&blkg->stats_lock, flags);
234 BUG_ON(blkio_blkg_idling(&pd->stats));
235 pd->stats.start_idle_time = sched_clock();
236 blkio_mark_blkg_idling(&pd->stats);
237 spin_unlock_irqrestore(&blkg->stats_lock, flags);
238 }
239 EXPORT_SYMBOL_GPL(blkiocg_update_set_idle_time_stats);
240
241 void blkiocg_update_idle_time_stats(struct blkio_group *blkg,
242 struct blkio_policy_type *pol)
243 {
244 struct blkg_policy_data *pd = blkg->pd[pol->plid];
245 unsigned long flags;
246 unsigned long long now;
247 struct blkio_group_stats *stats;
248
249 spin_lock_irqsave(&blkg->stats_lock, flags);
250 stats = &pd->stats;
251 if (blkio_blkg_idling(stats)) {
252 now = sched_clock();
253 if (time_after64(now, stats->start_idle_time))
254 stats->idle_time += now - stats->start_idle_time;
255 blkio_clear_blkg_idling(stats);
256 }
257 spin_unlock_irqrestore(&blkg->stats_lock, flags);
258 }
259 EXPORT_SYMBOL_GPL(blkiocg_update_idle_time_stats);
260
261 void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
262 struct blkio_policy_type *pol)
263 {
264 struct blkg_policy_data *pd = blkg->pd[pol->plid];
265 unsigned long flags;
266 struct blkio_group_stats *stats;
267
268 spin_lock_irqsave(&blkg->stats_lock, flags);
269 stats = &pd->stats;
270 stats->avg_queue_size_sum +=
271 stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] +
272 stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE];
273 stats->avg_queue_size_samples++;
274 blkio_update_group_wait_time(stats);
275 spin_unlock_irqrestore(&blkg->stats_lock, flags);
276 }
277 EXPORT_SYMBOL_GPL(blkiocg_update_avg_queue_size_stats);
278
279 void blkiocg_set_start_empty_time(struct blkio_group *blkg,
280 struct blkio_policy_type *pol)
281 {
282 struct blkg_policy_data *pd = blkg->pd[pol->plid];
283 unsigned long flags;
284 struct blkio_group_stats *stats;
285
286 spin_lock_irqsave(&blkg->stats_lock, flags);
287 stats = &pd->stats;
288
289 if (stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] ||
290 stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE]) {
291 spin_unlock_irqrestore(&blkg->stats_lock, flags);
292 return;
293 }
294
295 /*
296 * group is already marked empty. This can happen if cfqq got new
297 * request in parent group and moved to this group while being added
298 * to service tree. Just ignore the event and move on.
299 */
300 if(blkio_blkg_empty(stats)) {
301 spin_unlock_irqrestore(&blkg->stats_lock, flags);
302 return;
303 }
304
305 stats->start_empty_time = sched_clock();
306 blkio_mark_blkg_empty(stats);
307 spin_unlock_irqrestore(&blkg->stats_lock, flags);
308 }
309 EXPORT_SYMBOL_GPL(blkiocg_set_start_empty_time);
310
311 void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
312 struct blkio_policy_type *pol,
313 unsigned long dequeue)
314 {
315 struct blkg_policy_data *pd = blkg->pd[pol->plid];
316
317 pd->stats.dequeue += dequeue;
318 }
319 EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats);
320 #else
321 static inline void blkio_set_start_group_wait_time(struct blkio_group *blkg,
322 struct blkio_policy_type *pol,
323 struct blkio_group *curr_blkg) { }
324 static inline void blkio_end_empty_time(struct blkio_group_stats *stats) { }
325 #endif
326
327 void blkiocg_update_io_add_stats(struct blkio_group *blkg,
328 struct blkio_policy_type *pol,
329 struct blkio_group *curr_blkg, bool direction,
330 bool sync)
331 {
332 struct blkg_policy_data *pd = blkg->pd[pol->plid];
333 unsigned long flags;
334
335 spin_lock_irqsave(&blkg->stats_lock, flags);
336 blkio_add_stat(pd->stats.stat_arr[BLKIO_STAT_QUEUED], 1, direction,
337 sync);
338 blkio_end_empty_time(&pd->stats);
339 blkio_set_start_group_wait_time(blkg, pol, curr_blkg);
340 spin_unlock_irqrestore(&blkg->stats_lock, flags);
341 }
342 EXPORT_SYMBOL_GPL(blkiocg_update_io_add_stats);
343
344 void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
345 struct blkio_policy_type *pol,
346 bool direction, bool sync)
347 {
348 struct blkg_policy_data *pd = blkg->pd[pol->plid];
349 unsigned long flags;
350
351 spin_lock_irqsave(&blkg->stats_lock, flags);
352 blkio_check_and_dec_stat(pd->stats.stat_arr[BLKIO_STAT_QUEUED],
353 direction, sync);
354 spin_unlock_irqrestore(&blkg->stats_lock, flags);
355 }
356 EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats);
357
358 void blkiocg_update_timeslice_used(struct blkio_group *blkg,
359 struct blkio_policy_type *pol,
360 unsigned long time,
361 unsigned long unaccounted_time)
362 {
363 struct blkg_policy_data *pd = blkg->pd[pol->plid];
364 unsigned long flags;
365
366 spin_lock_irqsave(&blkg->stats_lock, flags);
367 pd->stats.time += time;
368 #ifdef CONFIG_DEBUG_BLK_CGROUP
369 pd->stats.unaccounted_time += unaccounted_time;
370 #endif
371 spin_unlock_irqrestore(&blkg->stats_lock, flags);
372 }
373 EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used);
374
375 /*
376 * should be called under rcu read lock or queue lock to make sure blkg pointer
377 * is valid.
378 */
379 void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
380 struct blkio_policy_type *pol,
381 uint64_t bytes, bool direction, bool sync)
382 {
383 struct blkg_policy_data *pd = blkg->pd[pol->plid];
384 struct blkio_group_stats_cpu *stats_cpu;
385 unsigned long flags;
386
387 /*
388 * Disabling interrupts to provide mutual exclusion between two
389 * writes on same cpu. It probably is not needed for 64bit. Not
390 * optimizing that case yet.
391 */
392 local_irq_save(flags);
393
394 stats_cpu = this_cpu_ptr(pd->stats_cpu);
395
396 u64_stats_update_begin(&stats_cpu->syncp);
397 stats_cpu->sectors += bytes >> 9;
398 blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICED],
399 1, direction, sync);
400 blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICE_BYTES],
401 bytes, direction, sync);
402 u64_stats_update_end(&stats_cpu->syncp);
403 local_irq_restore(flags);
404 }
405 EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats);
406
407 void blkiocg_update_completion_stats(struct blkio_group *blkg,
408 struct blkio_policy_type *pol,
409 uint64_t start_time,
410 uint64_t io_start_time, bool direction,
411 bool sync)
412 {
413 struct blkg_policy_data *pd = blkg->pd[pol->plid];
414 struct blkio_group_stats *stats;
415 unsigned long flags;
416 unsigned long long now = sched_clock();
417
418 spin_lock_irqsave(&blkg->stats_lock, flags);
419 stats = &pd->stats;
420 if (time_after64(now, io_start_time))
421 blkio_add_stat(stats->stat_arr[BLKIO_STAT_SERVICE_TIME],
422 now - io_start_time, direction, sync);
423 if (time_after64(io_start_time, start_time))
424 blkio_add_stat(stats->stat_arr[BLKIO_STAT_WAIT_TIME],
425 io_start_time - start_time, direction, sync);
426 spin_unlock_irqrestore(&blkg->stats_lock, flags);
427 }
428 EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats);
429
430 /* Merged stats are per cpu. */
431 void blkiocg_update_io_merged_stats(struct blkio_group *blkg,
432 struct blkio_policy_type *pol,
433 bool direction, bool sync)
434 {
435 struct blkg_policy_data *pd = blkg->pd[pol->plid];
436 struct blkio_group_stats_cpu *stats_cpu;
437 unsigned long flags;
438
439 /*
440 * Disabling interrupts to provide mutual exclusion between two
441 * writes on same cpu. It probably is not needed for 64bit. Not
442 * optimizing that case yet.
443 */
444 local_irq_save(flags);
445
446 stats_cpu = this_cpu_ptr(pd->stats_cpu);
447
448 u64_stats_update_begin(&stats_cpu->syncp);
449 blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_MERGED], 1,
450 direction, sync);
451 u64_stats_update_end(&stats_cpu->syncp);
452 local_irq_restore(flags);
453 }
454 EXPORT_SYMBOL_GPL(blkiocg_update_io_merged_stats);
455
456 /**
457 * blkg_free - free a blkg
458 * @blkg: blkg to free
459 *
460 * Free @blkg which may be partially allocated.
461 */
462 static void blkg_free(struct blkio_group *blkg)
463 {
464 int i;
465
466 if (!blkg)
467 return;
468
469 for (i = 0; i < BLKIO_NR_POLICIES; i++) {
470 struct blkg_policy_data *pd = blkg->pd[i];
471
472 if (pd) {
473 free_percpu(pd->stats_cpu);
474 kfree(pd);
475 }
476 }
477
478 kfree(blkg);
479 }
480
481 /**
482 * blkg_alloc - allocate a blkg
483 * @blkcg: block cgroup the new blkg is associated with
484 * @q: request_queue the new blkg is associated with
485 *
486 * Allocate a new blkg assocating @blkcg and @q.
487 *
488 * FIXME: Should be called with queue locked but currently isn't due to
489 * percpu stat breakage.
490 */
491 static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg,
492 struct request_queue *q)
493 {
494 struct blkio_group *blkg;
495 int i;
496
497 /* alloc and init base part */
498 blkg = kzalloc_node(sizeof(*blkg), GFP_ATOMIC, q->node);
499 if (!blkg)
500 return NULL;
501
502 spin_lock_init(&blkg->stats_lock);
503 rcu_assign_pointer(blkg->q, q);
504 INIT_LIST_HEAD(&blkg->q_node);
505 blkg->blkcg = blkcg;
506 blkg->refcnt = 1;
507 cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path));
508
509 for (i = 0; i < BLKIO_NR_POLICIES; i++) {
510 struct blkio_policy_type *pol = blkio_policy[i];
511 struct blkg_policy_data *pd;
512
513 if (!pol)
514 continue;
515
516 /* alloc per-policy data and attach it to blkg */
517 pd = kzalloc_node(sizeof(*pd) + pol->pdata_size, GFP_ATOMIC,
518 q->node);
519 if (!pd) {
520 blkg_free(blkg);
521 return NULL;
522 }
523
524 blkg->pd[i] = pd;
525 pd->blkg = blkg;
526
527 /* broken, read comment in the callsite */
528 pd->stats_cpu = alloc_percpu(struct blkio_group_stats_cpu);
529 if (!pd->stats_cpu) {
530 blkg_free(blkg);
531 return NULL;
532 }
533 }
534
535 /* invoke per-policy init */
536 for (i = 0; i < BLKIO_NR_POLICIES; i++) {
537 struct blkio_policy_type *pol = blkio_policy[i];
538
539 if (pol)
540 pol->ops.blkio_init_group_fn(blkg);
541 }
542
543 return blkg;
544 }
545
546 struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg,
547 struct request_queue *q,
548 enum blkio_policy_id plid,
549 bool for_root)
550 __releases(q->queue_lock) __acquires(q->queue_lock)
551 {
552 struct blkio_group *blkg, *new_blkg;
553
554 WARN_ON_ONCE(!rcu_read_lock_held());
555 lockdep_assert_held(q->queue_lock);
556
557 /*
558 * This could be the first entry point of blkcg implementation and
559 * we shouldn't allow anything to go through for a bypassing queue.
560 * The following can be removed if blkg lookup is guaranteed to
561 * fail on a bypassing queue.
562 */
563 if (unlikely(blk_queue_bypass(q)) && !for_root)
564 return ERR_PTR(blk_queue_dead(q) ? -EINVAL : -EBUSY);
565
566 blkg = blkg_lookup(blkcg, q);
567 if (blkg)
568 return blkg;
569
570 /* blkg holds a reference to blkcg */
571 if (!css_tryget(&blkcg->css))
572 return ERR_PTR(-EINVAL);
573
574 /*
575 * Allocate and initialize.
576 *
577 * FIXME: The following is broken. Percpu memory allocation
578 * requires %GFP_KERNEL context and can't be performed from IO
579 * path. Allocation here should inherently be atomic and the
580 * following lock dancing can be removed once the broken percpu
581 * allocation is fixed.
582 */
583 spin_unlock_irq(q->queue_lock);
584 rcu_read_unlock();
585
586 new_blkg = blkg_alloc(blkcg, q);
587
588 rcu_read_lock();
589 spin_lock_irq(q->queue_lock);
590
591 /* did bypass get turned on inbetween? */
592 if (unlikely(blk_queue_bypass(q)) && !for_root) {
593 blkg = ERR_PTR(blk_queue_dead(q) ? -EINVAL : -EBUSY);
594 goto out;
595 }
596
597 /* did someone beat us to it? */
598 blkg = blkg_lookup(blkcg, q);
599 if (unlikely(blkg))
600 goto out;
601
602 /* did alloc fail? */
603 if (unlikely(!new_blkg)) {
604 blkg = ERR_PTR(-ENOMEM);
605 goto out;
606 }
607
608 /* insert */
609 spin_lock(&blkcg->lock);
610 swap(blkg, new_blkg);
611
612 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
613 list_add(&blkg->q_node, &q->blkg_list);
614 q->nr_blkgs++;
615
616 spin_unlock(&blkcg->lock);
617 out:
618 blkg_free(new_blkg);
619 return blkg;
620 }
621 EXPORT_SYMBOL_GPL(blkg_lookup_create);
622
623 /* called under rcu_read_lock(). */
624 struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg,
625 struct request_queue *q)
626 {
627 struct blkio_group *blkg;
628 struct hlist_node *n;
629
630 hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node)
631 if (blkg->q == q)
632 return blkg;
633 return NULL;
634 }
635 EXPORT_SYMBOL_GPL(blkg_lookup);
636
637 static void blkg_destroy(struct blkio_group *blkg)
638 {
639 struct request_queue *q = blkg->q;
640 struct blkio_cgroup *blkcg = blkg->blkcg;
641
642 lockdep_assert_held(q->queue_lock);
643 lockdep_assert_held(&blkcg->lock);
644
645 /* Something wrong if we are trying to remove same group twice */
646 WARN_ON_ONCE(list_empty(&blkg->q_node));
647 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
648 list_del_init(&blkg->q_node);
649 hlist_del_init_rcu(&blkg->blkcg_node);
650
651 WARN_ON_ONCE(q->nr_blkgs <= 0);
652 q->nr_blkgs--;
653
654 /*
655 * Put the reference taken at the time of creation so that when all
656 * queues are gone, group can be destroyed.
657 */
658 blkg_put(blkg);
659 }
660
661 /*
662 * XXX: This updates blkg policy data in-place for root blkg, which is
663 * necessary across elevator switch and policy registration as root blkgs
664 * aren't shot down. This broken and racy implementation is temporary.
665 * Eventually, blkg shoot down will be replaced by proper in-place update.
666 */
667 void update_root_blkg_pd(struct request_queue *q, enum blkio_policy_id plid)
668 {
669 struct blkio_policy_type *pol = blkio_policy[plid];
670 struct blkio_group *blkg = blkg_lookup(&blkio_root_cgroup, q);
671 struct blkg_policy_data *pd;
672
673 if (!blkg)
674 return;
675
676 kfree(blkg->pd[plid]);
677 blkg->pd[plid] = NULL;
678
679 if (!pol)
680 return;
681
682 pd = kzalloc(sizeof(*pd) + pol->pdata_size, GFP_KERNEL);
683 WARN_ON_ONCE(!pd);
684
685 pd->stats_cpu = alloc_percpu(struct blkio_group_stats_cpu);
686 WARN_ON_ONCE(!pd->stats_cpu);
687
688 blkg->pd[plid] = pd;
689 pd->blkg = blkg;
690 pol->ops.blkio_init_group_fn(blkg);
691 }
692 EXPORT_SYMBOL_GPL(update_root_blkg_pd);
693
694 /**
695 * blkg_destroy_all - destroy all blkgs associated with a request_queue
696 * @q: request_queue of interest
697 * @destroy_root: whether to destroy root blkg or not
698 *
699 * Destroy blkgs associated with @q. If @destroy_root is %true, all are
700 * destroyed; otherwise, root blkg is left alone.
701 */
702 void blkg_destroy_all(struct request_queue *q, bool destroy_root)
703 {
704 struct blkio_group *blkg, *n;
705
706 spin_lock_irq(q->queue_lock);
707
708 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
709 struct blkio_cgroup *blkcg = blkg->blkcg;
710
711 /* skip root? */
712 if (!destroy_root && blkg->blkcg == &blkio_root_cgroup)
713 continue;
714
715 spin_lock(&blkcg->lock);
716 blkg_destroy(blkg);
717 spin_unlock(&blkcg->lock);
718 }
719
720 spin_unlock_irq(q->queue_lock);
721 }
722 EXPORT_SYMBOL_GPL(blkg_destroy_all);
723
724 static void blkg_rcu_free(struct rcu_head *rcu_head)
725 {
726 blkg_free(container_of(rcu_head, struct blkio_group, rcu_head));
727 }
728
729 void __blkg_release(struct blkio_group *blkg)
730 {
731 /* release the extra blkcg reference this blkg has been holding */
732 css_put(&blkg->blkcg->css);
733
734 /*
735 * A group is freed in rcu manner. But having an rcu lock does not
736 * mean that one can access all the fields of blkg and assume these
737 * are valid. For example, don't try to follow throtl_data and
738 * request queue links.
739 *
740 * Having a reference to blkg under an rcu allows acess to only
741 * values local to groups like group stats and group rate limits
742 */
743 call_rcu(&blkg->rcu_head, blkg_rcu_free);
744 }
745 EXPORT_SYMBOL_GPL(__blkg_release);
746
747 static void blkio_reset_stats_cpu(struct blkio_group *blkg, int plid)
748 {
749 struct blkg_policy_data *pd = blkg->pd[plid];
750 struct blkio_group_stats_cpu *stats_cpu;
751 int i, j, k;
752 /*
753 * Note: On 64 bit arch this should not be an issue. This has the
754 * possibility of returning some inconsistent value on 32bit arch
755 * as 64bit update on 32bit is non atomic. Taking care of this
756 * corner case makes code very complicated, like sending IPIs to
757 * cpus, taking care of stats of offline cpus etc.
758 *
759 * reset stats is anyway more of a debug feature and this sounds a
760 * corner case. So I am not complicating the code yet until and
761 * unless this becomes a real issue.
762 */
763 for_each_possible_cpu(i) {
764 stats_cpu = per_cpu_ptr(pd->stats_cpu, i);
765 stats_cpu->sectors = 0;
766 for(j = 0; j < BLKIO_STAT_CPU_NR; j++)
767 for (k = 0; k < BLKIO_STAT_TOTAL; k++)
768 stats_cpu->stat_arr_cpu[j][k] = 0;
769 }
770 }
771
772 static int
773 blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)
774 {
775 struct blkio_cgroup *blkcg;
776 struct blkio_group *blkg;
777 struct blkio_group_stats *stats;
778 struct hlist_node *n;
779 uint64_t queued[BLKIO_STAT_TOTAL];
780 int i;
781 #ifdef CONFIG_DEBUG_BLK_CGROUP
782 bool idling, waiting, empty;
783 unsigned long long now = sched_clock();
784 #endif
785
786 blkcg = cgroup_to_blkio_cgroup(cgroup);
787 spin_lock(&blkio_list_lock);
788 spin_lock_irq(&blkcg->lock);
789 hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
790 struct blkio_policy_type *pol;
791
792 list_for_each_entry(pol, &blkio_list, list) {
793 struct blkg_policy_data *pd = blkg->pd[pol->plid];
794
795 spin_lock(&blkg->stats_lock);
796 stats = &pd->stats;
797 #ifdef CONFIG_DEBUG_BLK_CGROUP
798 idling = blkio_blkg_idling(stats);
799 waiting = blkio_blkg_waiting(stats);
800 empty = blkio_blkg_empty(stats);
801 #endif
802 for (i = 0; i < BLKIO_STAT_TOTAL; i++)
803 queued[i] = stats->stat_arr[BLKIO_STAT_QUEUED][i];
804 memset(stats, 0, sizeof(struct blkio_group_stats));
805 for (i = 0; i < BLKIO_STAT_TOTAL; i++)
806 stats->stat_arr[BLKIO_STAT_QUEUED][i] = queued[i];
807 #ifdef CONFIG_DEBUG_BLK_CGROUP
808 if (idling) {
809 blkio_mark_blkg_idling(stats);
810 stats->start_idle_time = now;
811 }
812 if (waiting) {
813 blkio_mark_blkg_waiting(stats);
814 stats->start_group_wait_time = now;
815 }
816 if (empty) {
817 blkio_mark_blkg_empty(stats);
818 stats->start_empty_time = now;
819 }
820 #endif
821 spin_unlock(&blkg->stats_lock);
822
823 /* Reset Per cpu stats which don't take blkg->stats_lock */
824 blkio_reset_stats_cpu(blkg, pol->plid);
825 }
826 }
827
828 spin_unlock_irq(&blkcg->lock);
829 spin_unlock(&blkio_list_lock);
830 return 0;
831 }
832
833 static void blkio_get_key_name(enum stat_sub_type type, const char *dname,
834 char *str, int chars_left, bool diskname_only)
835 {
836 snprintf(str, chars_left, "%s", dname);
837 chars_left -= strlen(str);
838 if (chars_left <= 0) {
839 printk(KERN_WARNING
840 "Possibly incorrect cgroup stat display format");
841 return;
842 }
843 if (diskname_only)
844 return;
845 switch (type) {
846 case BLKIO_STAT_READ:
847 strlcat(str, " Read", chars_left);
848 break;
849 case BLKIO_STAT_WRITE:
850 strlcat(str, " Write", chars_left);
851 break;
852 case BLKIO_STAT_SYNC:
853 strlcat(str, " Sync", chars_left);
854 break;
855 case BLKIO_STAT_ASYNC:
856 strlcat(str, " Async", chars_left);
857 break;
858 case BLKIO_STAT_TOTAL:
859 strlcat(str, " Total", chars_left);
860 break;
861 default:
862 strlcat(str, " Invalid", chars_left);
863 }
864 }
865
866 static uint64_t blkio_fill_stat(char *str, int chars_left, uint64_t val,
867 struct cgroup_map_cb *cb, const char *dname)
868 {
869 blkio_get_key_name(0, dname, str, chars_left, true);
870 cb->fill(cb, str, val);
871 return val;
872 }
873
874
875 static uint64_t blkio_read_stat_cpu(struct blkio_group *blkg, int plid,
876 enum stat_type_cpu type, enum stat_sub_type sub_type)
877 {
878 struct blkg_policy_data *pd = blkg->pd[plid];
879 int cpu;
880 struct blkio_group_stats_cpu *stats_cpu;
881 u64 val = 0, tval;
882
883 for_each_possible_cpu(cpu) {
884 unsigned int start;
885 stats_cpu = per_cpu_ptr(pd->stats_cpu, cpu);
886
887 do {
888 start = u64_stats_fetch_begin(&stats_cpu->syncp);
889 if (type == BLKIO_STAT_CPU_SECTORS)
890 tval = stats_cpu->sectors;
891 else
892 tval = stats_cpu->stat_arr_cpu[type][sub_type];
893 } while(u64_stats_fetch_retry(&stats_cpu->syncp, start));
894
895 val += tval;
896 }
897
898 return val;
899 }
900
901 static uint64_t blkio_get_stat_cpu(struct blkio_group *blkg, int plid,
902 struct cgroup_map_cb *cb, const char *dname,
903 enum stat_type_cpu type)
904 {
905 uint64_t disk_total, val;
906 char key_str[MAX_KEY_LEN];
907 enum stat_sub_type sub_type;
908
909 if (type == BLKIO_STAT_CPU_SECTORS) {
910 val = blkio_read_stat_cpu(blkg, plid, type, 0);
911 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, val, cb,
912 dname);
913 }
914
915 for (sub_type = BLKIO_STAT_READ; sub_type < BLKIO_STAT_TOTAL;
916 sub_type++) {
917 blkio_get_key_name(sub_type, dname, key_str, MAX_KEY_LEN,
918 false);
919 val = blkio_read_stat_cpu(blkg, plid, type, sub_type);
920 cb->fill(cb, key_str, val);
921 }
922
923 disk_total = blkio_read_stat_cpu(blkg, plid, type, BLKIO_STAT_READ) +
924 blkio_read_stat_cpu(blkg, plid, type, BLKIO_STAT_WRITE);
925
926 blkio_get_key_name(BLKIO_STAT_TOTAL, dname, key_str, MAX_KEY_LEN,
927 false);
928 cb->fill(cb, key_str, disk_total);
929 return disk_total;
930 }
931
932 /* This should be called with blkg->stats_lock held */
933 static uint64_t blkio_get_stat(struct blkio_group *blkg, int plid,
934 struct cgroup_map_cb *cb, const char *dname,
935 enum stat_type type)
936 {
937 struct blkg_policy_data *pd = blkg->pd[plid];
938 uint64_t disk_total;
939 char key_str[MAX_KEY_LEN];
940 enum stat_sub_type sub_type;
941
942 if (type == BLKIO_STAT_TIME)
943 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
944 pd->stats.time, cb, dname);
945 #ifdef CONFIG_DEBUG_BLK_CGROUP
946 if (type == BLKIO_STAT_UNACCOUNTED_TIME)
947 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
948 pd->stats.unaccounted_time, cb, dname);
949 if (type == BLKIO_STAT_AVG_QUEUE_SIZE) {
950 uint64_t sum = pd->stats.avg_queue_size_sum;
951 uint64_t samples = pd->stats.avg_queue_size_samples;
952 if (samples)
953 do_div(sum, samples);
954 else
955 sum = 0;
956 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
957 sum, cb, dname);
958 }
959 if (type == BLKIO_STAT_GROUP_WAIT_TIME)
960 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
961 pd->stats.group_wait_time, cb, dname);
962 if (type == BLKIO_STAT_IDLE_TIME)
963 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
964 pd->stats.idle_time, cb, dname);
965 if (type == BLKIO_STAT_EMPTY_TIME)
966 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
967 pd->stats.empty_time, cb, dname);
968 if (type == BLKIO_STAT_DEQUEUE)
969 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
970 pd->stats.dequeue, cb, dname);
971 #endif
972
973 for (sub_type = BLKIO_STAT_READ; sub_type < BLKIO_STAT_TOTAL;
974 sub_type++) {
975 blkio_get_key_name(sub_type, dname, key_str, MAX_KEY_LEN,
976 false);
977 cb->fill(cb, key_str, pd->stats.stat_arr[type][sub_type]);
978 }
979 disk_total = pd->stats.stat_arr[type][BLKIO_STAT_READ] +
980 pd->stats.stat_arr[type][BLKIO_STAT_WRITE];
981 blkio_get_key_name(BLKIO_STAT_TOTAL, dname, key_str, MAX_KEY_LEN,
982 false);
983 cb->fill(cb, key_str, disk_total);
984 return disk_total;
985 }
986
987 static int blkio_policy_parse_and_set(char *buf, enum blkio_policy_id plid,
988 int fileid, struct blkio_cgroup *blkcg)
989 {
990 struct gendisk *disk = NULL;
991 struct blkio_group *blkg = NULL;
992 struct blkg_policy_data *pd;
993 char *s[4], *p, *major_s = NULL, *minor_s = NULL;
994 unsigned long major, minor;
995 int i = 0, ret = -EINVAL;
996 int part;
997 dev_t dev;
998 u64 temp;
999
1000 memset(s, 0, sizeof(s));
1001
1002 while ((p = strsep(&buf, " ")) != NULL) {
1003 if (!*p)
1004 continue;
1005
1006 s[i++] = p;
1007
1008 /* Prevent from inputing too many things */
1009 if (i == 3)
1010 break;
1011 }
1012
1013 if (i != 2)
1014 goto out;
1015
1016 p = strsep(&s[0], ":");
1017 if (p != NULL)
1018 major_s = p;
1019 else
1020 goto out;
1021
1022 minor_s = s[0];
1023 if (!minor_s)
1024 goto out;
1025
1026 if (strict_strtoul(major_s, 10, &major))
1027 goto out;
1028
1029 if (strict_strtoul(minor_s, 10, &minor))
1030 goto out;
1031
1032 dev = MKDEV(major, minor);
1033
1034 if (strict_strtoull(s[1], 10, &temp))
1035 goto out;
1036
1037 disk = get_gendisk(dev, &part);
1038 if (!disk || part)
1039 goto out;
1040
1041 rcu_read_lock();
1042
1043 spin_lock_irq(disk->queue->queue_lock);
1044 blkg = blkg_lookup_create(blkcg, disk->queue, plid, false);
1045 spin_unlock_irq(disk->queue->queue_lock);
1046
1047 if (IS_ERR(blkg)) {
1048 ret = PTR_ERR(blkg);
1049 goto out_unlock;
1050 }
1051
1052 pd = blkg->pd[plid];
1053
1054 switch (plid) {
1055 case BLKIO_POLICY_PROP:
1056 if ((temp < BLKIO_WEIGHT_MIN && temp > 0) ||
1057 temp > BLKIO_WEIGHT_MAX)
1058 goto out_unlock;
1059
1060 pd->conf.weight = temp;
1061 blkio_update_group_weight(blkg, plid, temp ?: blkcg->weight);
1062 break;
1063 case BLKIO_POLICY_THROTL:
1064 switch(fileid) {
1065 case BLKIO_THROTL_read_bps_device:
1066 pd->conf.bps[READ] = temp;
1067 blkio_update_group_bps(blkg, plid, temp ?: -1, fileid);
1068 break;
1069 case BLKIO_THROTL_write_bps_device:
1070 pd->conf.bps[WRITE] = temp;
1071 blkio_update_group_bps(blkg, plid, temp ?: -1, fileid);
1072 break;
1073 case BLKIO_THROTL_read_iops_device:
1074 if (temp > THROTL_IOPS_MAX)
1075 goto out_unlock;
1076 pd->conf.iops[READ] = temp;
1077 blkio_update_group_iops(blkg, plid, temp ?: -1, fileid);
1078 break;
1079 case BLKIO_THROTL_write_iops_device:
1080 if (temp > THROTL_IOPS_MAX)
1081 goto out_unlock;
1082 pd->conf.iops[WRITE] = temp;
1083 blkio_update_group_iops(blkg, plid, temp ?: -1, fileid);
1084 break;
1085 }
1086 break;
1087 default:
1088 BUG();
1089 }
1090 ret = 0;
1091 out_unlock:
1092 rcu_read_unlock();
1093 out:
1094 put_disk(disk);
1095
1096 /*
1097 * If queue was bypassing, we should retry. Do so after a short
1098 * msleep(). It isn't strictly necessary but queue can be
1099 * bypassing for some time and it's always nice to avoid busy
1100 * looping.
1101 */
1102 if (ret == -EBUSY) {
1103 msleep(10);
1104 return restart_syscall();
1105 }
1106 return ret;
1107 }
1108
1109 static int blkiocg_file_write(struct cgroup *cgrp, struct cftype *cft,
1110 const char *buffer)
1111 {
1112 int ret = 0;
1113 char *buf;
1114 struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
1115 enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1116 int fileid = BLKIOFILE_ATTR(cft->private);
1117
1118 buf = kstrdup(buffer, GFP_KERNEL);
1119 if (!buf)
1120 return -ENOMEM;
1121
1122 ret = blkio_policy_parse_and_set(buf, plid, fileid, blkcg);
1123 kfree(buf);
1124 return ret;
1125 }
1126
1127 static const char *blkg_dev_name(struct blkio_group *blkg)
1128 {
1129 /* some drivers (floppy) instantiate a queue w/o disk registered */
1130 if (blkg->q->backing_dev_info.dev)
1131 return dev_name(blkg->q->backing_dev_info.dev);
1132 return NULL;
1133 }
1134
1135 static void blkio_print_group_conf(struct cftype *cft, struct blkio_group *blkg,
1136 struct seq_file *m)
1137 {
1138 int plid = BLKIOFILE_POLICY(cft->private);
1139 int fileid = BLKIOFILE_ATTR(cft->private);
1140 struct blkg_policy_data *pd = blkg->pd[plid];
1141 const char *dname = blkg_dev_name(blkg);
1142 int rw = WRITE;
1143
1144 if (!dname)
1145 return;
1146
1147 switch (plid) {
1148 case BLKIO_POLICY_PROP:
1149 if (pd->conf.weight)
1150 seq_printf(m, "%s\t%u\n",
1151 dname, pd->conf.weight);
1152 break;
1153 case BLKIO_POLICY_THROTL:
1154 switch (fileid) {
1155 case BLKIO_THROTL_read_bps_device:
1156 rw = READ;
1157 case BLKIO_THROTL_write_bps_device:
1158 if (pd->conf.bps[rw])
1159 seq_printf(m, "%s\t%llu\n",
1160 dname, pd->conf.bps[rw]);
1161 break;
1162 case BLKIO_THROTL_read_iops_device:
1163 rw = READ;
1164 case BLKIO_THROTL_write_iops_device:
1165 if (pd->conf.iops[rw])
1166 seq_printf(m, "%s\t%u\n",
1167 dname, pd->conf.iops[rw]);
1168 break;
1169 }
1170 break;
1171 default:
1172 BUG();
1173 }
1174 }
1175
1176 /* cgroup files which read their data from policy nodes end up here */
1177 static void blkio_read_conf(struct cftype *cft, struct blkio_cgroup *blkcg,
1178 struct seq_file *m)
1179 {
1180 struct blkio_group *blkg;
1181 struct hlist_node *n;
1182
1183 spin_lock_irq(&blkcg->lock);
1184 hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node)
1185 blkio_print_group_conf(cft, blkg, m);
1186 spin_unlock_irq(&blkcg->lock);
1187 }
1188
1189 static int blkiocg_file_read(struct cgroup *cgrp, struct cftype *cft,
1190 struct seq_file *m)
1191 {
1192 struct blkio_cgroup *blkcg;
1193 enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1194 int name = BLKIOFILE_ATTR(cft->private);
1195
1196 blkcg = cgroup_to_blkio_cgroup(cgrp);
1197
1198 switch(plid) {
1199 case BLKIO_POLICY_PROP:
1200 switch(name) {
1201 case BLKIO_PROP_weight_device:
1202 blkio_read_conf(cft, blkcg, m);
1203 return 0;
1204 default:
1205 BUG();
1206 }
1207 break;
1208 case BLKIO_POLICY_THROTL:
1209 switch(name){
1210 case BLKIO_THROTL_read_bps_device:
1211 case BLKIO_THROTL_write_bps_device:
1212 case BLKIO_THROTL_read_iops_device:
1213 case BLKIO_THROTL_write_iops_device:
1214 blkio_read_conf(cft, blkcg, m);
1215 return 0;
1216 default:
1217 BUG();
1218 }
1219 break;
1220 default:
1221 BUG();
1222 }
1223
1224 return 0;
1225 }
1226
1227 static int blkio_read_blkg_stats(struct blkio_cgroup *blkcg,
1228 struct cftype *cft, struct cgroup_map_cb *cb,
1229 enum stat_type type, bool show_total, bool pcpu)
1230 {
1231 struct blkio_group *blkg;
1232 struct hlist_node *n;
1233 uint64_t cgroup_total = 0;
1234
1235 rcu_read_lock();
1236 hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node) {
1237 const char *dname = blkg_dev_name(blkg);
1238 int plid = BLKIOFILE_POLICY(cft->private);
1239
1240 if (!dname)
1241 continue;
1242 if (pcpu) {
1243 cgroup_total += blkio_get_stat_cpu(blkg, plid,
1244 cb, dname, type);
1245 } else {
1246 spin_lock_irq(&blkg->stats_lock);
1247 cgroup_total += blkio_get_stat(blkg, plid,
1248 cb, dname, type);
1249 spin_unlock_irq(&blkg->stats_lock);
1250 }
1251 }
1252 if (show_total)
1253 cb->fill(cb, "Total", cgroup_total);
1254 rcu_read_unlock();
1255 return 0;
1256 }
1257
1258 /* All map kind of cgroup file get serviced by this function */
1259 static int blkiocg_file_read_map(struct cgroup *cgrp, struct cftype *cft,
1260 struct cgroup_map_cb *cb)
1261 {
1262 struct blkio_cgroup *blkcg;
1263 enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1264 int name = BLKIOFILE_ATTR(cft->private);
1265
1266 blkcg = cgroup_to_blkio_cgroup(cgrp);
1267
1268 switch(plid) {
1269 case BLKIO_POLICY_PROP:
1270 switch(name) {
1271 case BLKIO_PROP_time:
1272 return blkio_read_blkg_stats(blkcg, cft, cb,
1273 BLKIO_STAT_TIME, 0, 0);
1274 case BLKIO_PROP_sectors:
1275 return blkio_read_blkg_stats(blkcg, cft, cb,
1276 BLKIO_STAT_CPU_SECTORS, 0, 1);
1277 case BLKIO_PROP_io_service_bytes:
1278 return blkio_read_blkg_stats(blkcg, cft, cb,
1279 BLKIO_STAT_CPU_SERVICE_BYTES, 1, 1);
1280 case BLKIO_PROP_io_serviced:
1281 return blkio_read_blkg_stats(blkcg, cft, cb,
1282 BLKIO_STAT_CPU_SERVICED, 1, 1);
1283 case BLKIO_PROP_io_service_time:
1284 return blkio_read_blkg_stats(blkcg, cft, cb,
1285 BLKIO_STAT_SERVICE_TIME, 1, 0);
1286 case BLKIO_PROP_io_wait_time:
1287 return blkio_read_blkg_stats(blkcg, cft, cb,
1288 BLKIO_STAT_WAIT_TIME, 1, 0);
1289 case BLKIO_PROP_io_merged:
1290 return blkio_read_blkg_stats(blkcg, cft, cb,
1291 BLKIO_STAT_CPU_MERGED, 1, 1);
1292 case BLKIO_PROP_io_queued:
1293 return blkio_read_blkg_stats(blkcg, cft, cb,
1294 BLKIO_STAT_QUEUED, 1, 0);
1295 #ifdef CONFIG_DEBUG_BLK_CGROUP
1296 case BLKIO_PROP_unaccounted_time:
1297 return blkio_read_blkg_stats(blkcg, cft, cb,
1298 BLKIO_STAT_UNACCOUNTED_TIME, 0, 0);
1299 case BLKIO_PROP_dequeue:
1300 return blkio_read_blkg_stats(blkcg, cft, cb,
1301 BLKIO_STAT_DEQUEUE, 0, 0);
1302 case BLKIO_PROP_avg_queue_size:
1303 return blkio_read_blkg_stats(blkcg, cft, cb,
1304 BLKIO_STAT_AVG_QUEUE_SIZE, 0, 0);
1305 case BLKIO_PROP_group_wait_time:
1306 return blkio_read_blkg_stats(blkcg, cft, cb,
1307 BLKIO_STAT_GROUP_WAIT_TIME, 0, 0);
1308 case BLKIO_PROP_idle_time:
1309 return blkio_read_blkg_stats(blkcg, cft, cb,
1310 BLKIO_STAT_IDLE_TIME, 0, 0);
1311 case BLKIO_PROP_empty_time:
1312 return blkio_read_blkg_stats(blkcg, cft, cb,
1313 BLKIO_STAT_EMPTY_TIME, 0, 0);
1314 #endif
1315 default:
1316 BUG();
1317 }
1318 break;
1319 case BLKIO_POLICY_THROTL:
1320 switch(name){
1321 case BLKIO_THROTL_io_service_bytes:
1322 return blkio_read_blkg_stats(blkcg, cft, cb,
1323 BLKIO_STAT_CPU_SERVICE_BYTES, 1, 1);
1324 case BLKIO_THROTL_io_serviced:
1325 return blkio_read_blkg_stats(blkcg, cft, cb,
1326 BLKIO_STAT_CPU_SERVICED, 1, 1);
1327 default:
1328 BUG();
1329 }
1330 break;
1331 default:
1332 BUG();
1333 }
1334
1335 return 0;
1336 }
1337
1338 static int blkio_weight_write(struct blkio_cgroup *blkcg, int plid, u64 val)
1339 {
1340 struct blkio_group *blkg;
1341 struct hlist_node *n;
1342
1343 if (val < BLKIO_WEIGHT_MIN || val > BLKIO_WEIGHT_MAX)
1344 return -EINVAL;
1345
1346 spin_lock(&blkio_list_lock);
1347 spin_lock_irq(&blkcg->lock);
1348 blkcg->weight = (unsigned int)val;
1349
1350 hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
1351 struct blkg_policy_data *pd = blkg->pd[plid];
1352
1353 if (!pd->conf.weight)
1354 blkio_update_group_weight(blkg, plid, blkcg->weight);
1355 }
1356
1357 spin_unlock_irq(&blkcg->lock);
1358 spin_unlock(&blkio_list_lock);
1359 return 0;
1360 }
1361
1362 static u64 blkiocg_file_read_u64 (struct cgroup *cgrp, struct cftype *cft) {
1363 struct blkio_cgroup *blkcg;
1364 enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1365 int name = BLKIOFILE_ATTR(cft->private);
1366
1367 blkcg = cgroup_to_blkio_cgroup(cgrp);
1368
1369 switch(plid) {
1370 case BLKIO_POLICY_PROP:
1371 switch(name) {
1372 case BLKIO_PROP_weight:
1373 return (u64)blkcg->weight;
1374 }
1375 break;
1376 default:
1377 BUG();
1378 }
1379 return 0;
1380 }
1381
1382 static int
1383 blkiocg_file_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
1384 {
1385 struct blkio_cgroup *blkcg;
1386 enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1387 int name = BLKIOFILE_ATTR(cft->private);
1388
1389 blkcg = cgroup_to_blkio_cgroup(cgrp);
1390
1391 switch(plid) {
1392 case BLKIO_POLICY_PROP:
1393 switch(name) {
1394 case BLKIO_PROP_weight:
1395 return blkio_weight_write(blkcg, plid, val);
1396 }
1397 break;
1398 default:
1399 BUG();
1400 }
1401
1402 return 0;
1403 }
1404
1405 struct cftype blkio_files[] = {
1406 {
1407 .name = "weight_device",
1408 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1409 BLKIO_PROP_weight_device),
1410 .read_seq_string = blkiocg_file_read,
1411 .write_string = blkiocg_file_write,
1412 .max_write_len = 256,
1413 },
1414 {
1415 .name = "weight",
1416 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1417 BLKIO_PROP_weight),
1418 .read_u64 = blkiocg_file_read_u64,
1419 .write_u64 = blkiocg_file_write_u64,
1420 },
1421 {
1422 .name = "time",
1423 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1424 BLKIO_PROP_time),
1425 .read_map = blkiocg_file_read_map,
1426 },
1427 {
1428 .name = "sectors",
1429 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1430 BLKIO_PROP_sectors),
1431 .read_map = blkiocg_file_read_map,
1432 },
1433 {
1434 .name = "io_service_bytes",
1435 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1436 BLKIO_PROP_io_service_bytes),
1437 .read_map = blkiocg_file_read_map,
1438 },
1439 {
1440 .name = "io_serviced",
1441 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1442 BLKIO_PROP_io_serviced),
1443 .read_map = blkiocg_file_read_map,
1444 },
1445 {
1446 .name = "io_service_time",
1447 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1448 BLKIO_PROP_io_service_time),
1449 .read_map = blkiocg_file_read_map,
1450 },
1451 {
1452 .name = "io_wait_time",
1453 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1454 BLKIO_PROP_io_wait_time),
1455 .read_map = blkiocg_file_read_map,
1456 },
1457 {
1458 .name = "io_merged",
1459 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1460 BLKIO_PROP_io_merged),
1461 .read_map = blkiocg_file_read_map,
1462 },
1463 {
1464 .name = "io_queued",
1465 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1466 BLKIO_PROP_io_queued),
1467 .read_map = blkiocg_file_read_map,
1468 },
1469 {
1470 .name = "reset_stats",
1471 .write_u64 = blkiocg_reset_stats,
1472 },
1473 #ifdef CONFIG_BLK_DEV_THROTTLING
1474 {
1475 .name = "throttle.read_bps_device",
1476 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1477 BLKIO_THROTL_read_bps_device),
1478 .read_seq_string = blkiocg_file_read,
1479 .write_string = blkiocg_file_write,
1480 .max_write_len = 256,
1481 },
1482
1483 {
1484 .name = "throttle.write_bps_device",
1485 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1486 BLKIO_THROTL_write_bps_device),
1487 .read_seq_string = blkiocg_file_read,
1488 .write_string = blkiocg_file_write,
1489 .max_write_len = 256,
1490 },
1491
1492 {
1493 .name = "throttle.read_iops_device",
1494 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1495 BLKIO_THROTL_read_iops_device),
1496 .read_seq_string = blkiocg_file_read,
1497 .write_string = blkiocg_file_write,
1498 .max_write_len = 256,
1499 },
1500
1501 {
1502 .name = "throttle.write_iops_device",
1503 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1504 BLKIO_THROTL_write_iops_device),
1505 .read_seq_string = blkiocg_file_read,
1506 .write_string = blkiocg_file_write,
1507 .max_write_len = 256,
1508 },
1509 {
1510 .name = "throttle.io_service_bytes",
1511 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1512 BLKIO_THROTL_io_service_bytes),
1513 .read_map = blkiocg_file_read_map,
1514 },
1515 {
1516 .name = "throttle.io_serviced",
1517 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1518 BLKIO_THROTL_io_serviced),
1519 .read_map = blkiocg_file_read_map,
1520 },
1521 #endif /* CONFIG_BLK_DEV_THROTTLING */
1522
1523 #ifdef CONFIG_DEBUG_BLK_CGROUP
1524 {
1525 .name = "avg_queue_size",
1526 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1527 BLKIO_PROP_avg_queue_size),
1528 .read_map = blkiocg_file_read_map,
1529 },
1530 {
1531 .name = "group_wait_time",
1532 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1533 BLKIO_PROP_group_wait_time),
1534 .read_map = blkiocg_file_read_map,
1535 },
1536 {
1537 .name = "idle_time",
1538 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1539 BLKIO_PROP_idle_time),
1540 .read_map = blkiocg_file_read_map,
1541 },
1542 {
1543 .name = "empty_time",
1544 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1545 BLKIO_PROP_empty_time),
1546 .read_map = blkiocg_file_read_map,
1547 },
1548 {
1549 .name = "dequeue",
1550 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1551 BLKIO_PROP_dequeue),
1552 .read_map = blkiocg_file_read_map,
1553 },
1554 {
1555 .name = "unaccounted_time",
1556 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1557 BLKIO_PROP_unaccounted_time),
1558 .read_map = blkiocg_file_read_map,
1559 },
1560 #endif
1561 };
1562
1563 static int blkiocg_populate(struct cgroup_subsys *subsys, struct cgroup *cgroup)
1564 {
1565 return cgroup_add_files(cgroup, subsys, blkio_files,
1566 ARRAY_SIZE(blkio_files));
1567 }
1568
1569 /**
1570 * blkiocg_pre_destroy - cgroup pre_destroy callback
1571 * @subsys: cgroup subsys
1572 * @cgroup: cgroup of interest
1573 *
1574 * This function is called when @cgroup is about to go away and responsible
1575 * for shooting down all blkgs associated with @cgroup. blkgs should be
1576 * removed while holding both q and blkcg locks. As blkcg lock is nested
1577 * inside q lock, this function performs reverse double lock dancing.
1578 *
1579 * This is the blkcg counterpart of ioc_release_fn().
1580 */
1581 static int blkiocg_pre_destroy(struct cgroup_subsys *subsys,
1582 struct cgroup *cgroup)
1583 {
1584 struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
1585
1586 rcu_read_lock();
1587 spin_lock_irq(&blkcg->lock);
1588
1589 while (!hlist_empty(&blkcg->blkg_list)) {
1590 struct blkio_group *blkg = hlist_entry(blkcg->blkg_list.first,
1591 struct blkio_group, blkcg_node);
1592 struct request_queue *q = rcu_dereference(blkg->q);
1593
1594 if (spin_trylock(q->queue_lock)) {
1595 blkg_destroy(blkg);
1596 spin_unlock(q->queue_lock);
1597 } else {
1598 spin_unlock_irq(&blkcg->lock);
1599 rcu_read_unlock();
1600 cpu_relax();
1601 rcu_read_lock();
1602 spin_lock(&blkcg->lock);
1603 }
1604 }
1605
1606 spin_unlock_irq(&blkcg->lock);
1607 rcu_read_unlock();
1608 return 0;
1609 }
1610
1611 static void blkiocg_destroy(struct cgroup_subsys *subsys, struct cgroup *cgroup)
1612 {
1613 struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
1614
1615 if (blkcg != &blkio_root_cgroup)
1616 kfree(blkcg);
1617 }
1618
1619 static struct cgroup_subsys_state *
1620 blkiocg_create(struct cgroup_subsys *subsys, struct cgroup *cgroup)
1621 {
1622 struct blkio_cgroup *blkcg;
1623 struct cgroup *parent = cgroup->parent;
1624
1625 if (!parent) {
1626 blkcg = &blkio_root_cgroup;
1627 goto done;
1628 }
1629
1630 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
1631 if (!blkcg)
1632 return ERR_PTR(-ENOMEM);
1633
1634 blkcg->weight = BLKIO_WEIGHT_DEFAULT;
1635 done:
1636 spin_lock_init(&blkcg->lock);
1637 INIT_HLIST_HEAD(&blkcg->blkg_list);
1638
1639 return &blkcg->css;
1640 }
1641
1642 /**
1643 * blkcg_init_queue - initialize blkcg part of request queue
1644 * @q: request_queue to initialize
1645 *
1646 * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
1647 * part of new request_queue @q.
1648 *
1649 * RETURNS:
1650 * 0 on success, -errno on failure.
1651 */
1652 int blkcg_init_queue(struct request_queue *q)
1653 {
1654 int ret;
1655
1656 might_sleep();
1657
1658 ret = blk_throtl_init(q);
1659 if (ret)
1660 return ret;
1661
1662 mutex_lock(&all_q_mutex);
1663 INIT_LIST_HEAD(&q->all_q_node);
1664 list_add_tail(&q->all_q_node, &all_q_list);
1665 mutex_unlock(&all_q_mutex);
1666
1667 return 0;
1668 }
1669
1670 /**
1671 * blkcg_drain_queue - drain blkcg part of request_queue
1672 * @q: request_queue to drain
1673 *
1674 * Called from blk_drain_queue(). Responsible for draining blkcg part.
1675 */
1676 void blkcg_drain_queue(struct request_queue *q)
1677 {
1678 lockdep_assert_held(q->queue_lock);
1679
1680 blk_throtl_drain(q);
1681 }
1682
1683 /**
1684 * blkcg_exit_queue - exit and release blkcg part of request_queue
1685 * @q: request_queue being released
1686 *
1687 * Called from blk_release_queue(). Responsible for exiting blkcg part.
1688 */
1689 void blkcg_exit_queue(struct request_queue *q)
1690 {
1691 mutex_lock(&all_q_mutex);
1692 list_del_init(&q->all_q_node);
1693 mutex_unlock(&all_q_mutex);
1694
1695 blkg_destroy_all(q, true);
1696
1697 blk_throtl_exit(q);
1698 }
1699
1700 /*
1701 * We cannot support shared io contexts, as we have no mean to support
1702 * two tasks with the same ioc in two different groups without major rework
1703 * of the main cic data structures. For now we allow a task to change
1704 * its cgroup only if it's the only owner of its ioc.
1705 */
1706 static int blkiocg_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
1707 struct cgroup_taskset *tset)
1708 {
1709 struct task_struct *task;
1710 struct io_context *ioc;
1711 int ret = 0;
1712
1713 /* task_lock() is needed to avoid races with exit_io_context() */
1714 cgroup_taskset_for_each(task, cgrp, tset) {
1715 task_lock(task);
1716 ioc = task->io_context;
1717 if (ioc && atomic_read(&ioc->nr_tasks) > 1)
1718 ret = -EINVAL;
1719 task_unlock(task);
1720 if (ret)
1721 break;
1722 }
1723 return ret;
1724 }
1725
1726 static void blkiocg_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
1727 struct cgroup_taskset *tset)
1728 {
1729 struct task_struct *task;
1730 struct io_context *ioc;
1731
1732 cgroup_taskset_for_each(task, cgrp, tset) {
1733 /* we don't lose anything even if ioc allocation fails */
1734 ioc = get_task_io_context(task, GFP_ATOMIC, NUMA_NO_NODE);
1735 if (ioc) {
1736 ioc_cgroup_changed(ioc);
1737 put_io_context(ioc);
1738 }
1739 }
1740 }
1741
1742 static void blkcg_bypass_start(void)
1743 __acquires(&all_q_mutex)
1744 {
1745 struct request_queue *q;
1746
1747 mutex_lock(&all_q_mutex);
1748
1749 list_for_each_entry(q, &all_q_list, all_q_node) {
1750 blk_queue_bypass_start(q);
1751 blkg_destroy_all(q, false);
1752 }
1753 }
1754
1755 static void blkcg_bypass_end(void)
1756 __releases(&all_q_mutex)
1757 {
1758 struct request_queue *q;
1759
1760 list_for_each_entry(q, &all_q_list, all_q_node)
1761 blk_queue_bypass_end(q);
1762
1763 mutex_unlock(&all_q_mutex);
1764 }
1765
1766 void blkio_policy_register(struct blkio_policy_type *blkiop)
1767 {
1768 struct request_queue *q;
1769
1770 blkcg_bypass_start();
1771 spin_lock(&blkio_list_lock);
1772
1773 BUG_ON(blkio_policy[blkiop->plid]);
1774 blkio_policy[blkiop->plid] = blkiop;
1775 list_add_tail(&blkiop->list, &blkio_list);
1776
1777 spin_unlock(&blkio_list_lock);
1778 list_for_each_entry(q, &all_q_list, all_q_node)
1779 update_root_blkg_pd(q, blkiop->plid);
1780 blkcg_bypass_end();
1781 }
1782 EXPORT_SYMBOL_GPL(blkio_policy_register);
1783
1784 void blkio_policy_unregister(struct blkio_policy_type *blkiop)
1785 {
1786 struct request_queue *q;
1787
1788 blkcg_bypass_start();
1789 spin_lock(&blkio_list_lock);
1790
1791 BUG_ON(blkio_policy[blkiop->plid] != blkiop);
1792 blkio_policy[blkiop->plid] = NULL;
1793 list_del_init(&blkiop->list);
1794
1795 spin_unlock(&blkio_list_lock);
1796 list_for_each_entry(q, &all_q_list, all_q_node)
1797 update_root_blkg_pd(q, blkiop->plid);
1798 blkcg_bypass_end();
1799 }
1800 EXPORT_SYMBOL_GPL(blkio_policy_unregister);
Linus' kernel tree