2 * Common Block IO controller cgroup interface
4 * Based on ideas and code from CFQ, CFS and BFQ:
5 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
7 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
8 * Paolo Valente <paolo.valente@unimore.it>
10 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
11 * Nauman Rafique <nauman@google.com>
13 #include <linux/ioprio.h>
14 #include <linux/seq_file.h>
15 #include <linux/kdev_t.h>
16 #include <linux/module.h>
17 #include <linux/err.h>
18 #include <linux/blkdev.h>
19 #include <linux/slab.h>
20 #include <linux/genhd.h>
21 #include <linux/delay.h>
22 #include "blk-cgroup.h"
25 #define MAX_KEY_LEN 100
27 static DEFINE_SPINLOCK(blkio_list_lock
);
28 static LIST_HEAD(blkio_list
);
30 static DEFINE_MUTEX(all_q_mutex
);
31 static LIST_HEAD(all_q_list
);
33 struct blkio_cgroup blkio_root_cgroup
= { .weight
= 2*BLKIO_WEIGHT_DEFAULT
};
34 EXPORT_SYMBOL_GPL(blkio_root_cgroup
);
36 static struct blkio_policy_type
*blkio_policy
[BLKIO_NR_POLICIES
];
38 static struct cgroup_subsys_state
*blkiocg_create(struct cgroup_subsys
*,
40 static int blkiocg_can_attach(struct cgroup_subsys
*, struct cgroup
*,
41 struct cgroup_taskset
*);
42 static void blkiocg_attach(struct cgroup_subsys
*, struct cgroup
*,
43 struct cgroup_taskset
*);
44 static int blkiocg_pre_destroy(struct cgroup_subsys
*, struct cgroup
*);
45 static void blkiocg_destroy(struct cgroup_subsys
*, struct cgroup
*);
46 static int blkiocg_populate(struct cgroup_subsys
*, struct cgroup
*);
48 /* for encoding cft->private value on file */
49 #define BLKIOFILE_PRIVATE(x, val) (((x) << 16) | (val))
50 /* What policy owns the file, proportional or throttle */
51 #define BLKIOFILE_POLICY(val) (((val) >> 16) & 0xffff)
52 #define BLKIOFILE_ATTR(val) ((val) & 0xffff)
54 struct cgroup_subsys blkio_subsys
= {
56 .create
= blkiocg_create
,
57 .can_attach
= blkiocg_can_attach
,
58 .attach
= blkiocg_attach
,
59 .pre_destroy
= blkiocg_pre_destroy
,
60 .destroy
= blkiocg_destroy
,
61 .populate
= blkiocg_populate
,
62 .subsys_id
= blkio_subsys_id
,
63 .module
= THIS_MODULE
,
65 EXPORT_SYMBOL_GPL(blkio_subsys
);
67 struct blkio_cgroup
*cgroup_to_blkio_cgroup(struct cgroup
*cgroup
)
69 return container_of(cgroup_subsys_state(cgroup
, blkio_subsys_id
),
70 struct blkio_cgroup
, css
);
72 EXPORT_SYMBOL_GPL(cgroup_to_blkio_cgroup
);
74 struct blkio_cgroup
*task_blkio_cgroup(struct task_struct
*tsk
)
76 return container_of(task_subsys_state(tsk
, blkio_subsys_id
),
77 struct blkio_cgroup
, css
);
79 EXPORT_SYMBOL_GPL(task_blkio_cgroup
);
81 static inline void blkio_update_group_weight(struct blkio_group
*blkg
,
82 int plid
, unsigned int weight
)
84 struct blkio_policy_type
*blkiop
;
86 list_for_each_entry(blkiop
, &blkio_list
, list
) {
87 /* If this policy does not own the blkg, do not send updates */
88 if (blkiop
->plid
!= plid
)
90 if (blkiop
->ops
.blkio_update_group_weight_fn
)
91 blkiop
->ops
.blkio_update_group_weight_fn(blkg
->q
,
96 static inline void blkio_update_group_bps(struct blkio_group
*blkg
, int plid
,
99 struct blkio_policy_type
*blkiop
;
101 list_for_each_entry(blkiop
, &blkio_list
, list
) {
103 /* If this policy does not own the blkg, do not send updates */
104 if (blkiop
->plid
!= plid
)
107 if (fileid
== BLKIO_THROTL_read_bps_device
108 && blkiop
->ops
.blkio_update_group_read_bps_fn
)
109 blkiop
->ops
.blkio_update_group_read_bps_fn(blkg
->q
,
112 if (fileid
== BLKIO_THROTL_write_bps_device
113 && blkiop
->ops
.blkio_update_group_write_bps_fn
)
114 blkiop
->ops
.blkio_update_group_write_bps_fn(blkg
->q
,
119 static inline void blkio_update_group_iops(struct blkio_group
*blkg
,
120 int plid
, unsigned int iops
,
123 struct blkio_policy_type
*blkiop
;
125 list_for_each_entry(blkiop
, &blkio_list
, list
) {
127 /* If this policy does not own the blkg, do not send updates */
128 if (blkiop
->plid
!= plid
)
131 if (fileid
== BLKIO_THROTL_read_iops_device
132 && blkiop
->ops
.blkio_update_group_read_iops_fn
)
133 blkiop
->ops
.blkio_update_group_read_iops_fn(blkg
->q
,
136 if (fileid
== BLKIO_THROTL_write_iops_device
137 && blkiop
->ops
.blkio_update_group_write_iops_fn
)
138 blkiop
->ops
.blkio_update_group_write_iops_fn(blkg
->q
,
144 * Add to the appropriate stat variable depending on the request type.
145 * This should be called with the blkg->stats_lock held.
147 static void blkio_add_stat(uint64_t *stat
, uint64_t add
, bool direction
,
151 stat
[BLKIO_STAT_WRITE
] += add
;
153 stat
[BLKIO_STAT_READ
] += add
;
155 stat
[BLKIO_STAT_SYNC
] += add
;
157 stat
[BLKIO_STAT_ASYNC
] += add
;
161 * Decrements the appropriate stat variable if non-zero depending on the
162 * request type. Panics on value being zero.
163 * This should be called with the blkg->stats_lock held.
165 static void blkio_check_and_dec_stat(uint64_t *stat
, bool direction
, bool sync
)
168 BUG_ON(stat
[BLKIO_STAT_WRITE
] == 0);
169 stat
[BLKIO_STAT_WRITE
]--;
171 BUG_ON(stat
[BLKIO_STAT_READ
] == 0);
172 stat
[BLKIO_STAT_READ
]--;
175 BUG_ON(stat
[BLKIO_STAT_SYNC
] == 0);
176 stat
[BLKIO_STAT_SYNC
]--;
178 BUG_ON(stat
[BLKIO_STAT_ASYNC
] == 0);
179 stat
[BLKIO_STAT_ASYNC
]--;
183 #ifdef CONFIG_DEBUG_BLK_CGROUP
184 /* This should be called with the blkg->stats_lock held. */
185 static void blkio_set_start_group_wait_time(struct blkio_group
*blkg
,
186 struct blkio_policy_type
*pol
,
187 struct blkio_group
*curr_blkg
)
189 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
191 if (blkio_blkg_waiting(&pd
->stats
))
193 if (blkg
== curr_blkg
)
195 pd
->stats
.start_group_wait_time
= sched_clock();
196 blkio_mark_blkg_waiting(&pd
->stats
);
199 /* This should be called with the blkg->stats_lock held. */
200 static void blkio_update_group_wait_time(struct blkio_group_stats
*stats
)
202 unsigned long long now
;
204 if (!blkio_blkg_waiting(stats
))
208 if (time_after64(now
, stats
->start_group_wait_time
))
209 stats
->group_wait_time
+= now
- stats
->start_group_wait_time
;
210 blkio_clear_blkg_waiting(stats
);
213 /* This should be called with the blkg->stats_lock held. */
214 static void blkio_end_empty_time(struct blkio_group_stats
*stats
)
216 unsigned long long now
;
218 if (!blkio_blkg_empty(stats
))
222 if (time_after64(now
, stats
->start_empty_time
))
223 stats
->empty_time
+= now
- stats
->start_empty_time
;
224 blkio_clear_blkg_empty(stats
);
227 void blkiocg_update_set_idle_time_stats(struct blkio_group
*blkg
,
228 struct blkio_policy_type
*pol
)
230 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
233 spin_lock_irqsave(&blkg
->stats_lock
, flags
);
234 BUG_ON(blkio_blkg_idling(&pd
->stats
));
235 pd
->stats
.start_idle_time
= sched_clock();
236 blkio_mark_blkg_idling(&pd
->stats
);
237 spin_unlock_irqrestore(&blkg
->stats_lock
, flags
);
239 EXPORT_SYMBOL_GPL(blkiocg_update_set_idle_time_stats
);
241 void blkiocg_update_idle_time_stats(struct blkio_group
*blkg
,
242 struct blkio_policy_type
*pol
)
244 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
246 unsigned long long now
;
247 struct blkio_group_stats
*stats
;
249 spin_lock_irqsave(&blkg
->stats_lock
, flags
);
251 if (blkio_blkg_idling(stats
)) {
253 if (time_after64(now
, stats
->start_idle_time
))
254 stats
->idle_time
+= now
- stats
->start_idle_time
;
255 blkio_clear_blkg_idling(stats
);
257 spin_unlock_irqrestore(&blkg
->stats_lock
, flags
);
259 EXPORT_SYMBOL_GPL(blkiocg_update_idle_time_stats
);
261 void blkiocg_update_avg_queue_size_stats(struct blkio_group
*blkg
,
262 struct blkio_policy_type
*pol
)
264 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
266 struct blkio_group_stats
*stats
;
268 spin_lock_irqsave(&blkg
->stats_lock
, flags
);
270 stats
->avg_queue_size_sum
+=
271 stats
->stat_arr
[BLKIO_STAT_QUEUED
][BLKIO_STAT_READ
] +
272 stats
->stat_arr
[BLKIO_STAT_QUEUED
][BLKIO_STAT_WRITE
];
273 stats
->avg_queue_size_samples
++;
274 blkio_update_group_wait_time(stats
);
275 spin_unlock_irqrestore(&blkg
->stats_lock
, flags
);
277 EXPORT_SYMBOL_GPL(blkiocg_update_avg_queue_size_stats
);
279 void blkiocg_set_start_empty_time(struct blkio_group
*blkg
,
280 struct blkio_policy_type
*pol
)
282 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
284 struct blkio_group_stats
*stats
;
286 spin_lock_irqsave(&blkg
->stats_lock
, flags
);
289 if (stats
->stat_arr
[BLKIO_STAT_QUEUED
][BLKIO_STAT_READ
] ||
290 stats
->stat_arr
[BLKIO_STAT_QUEUED
][BLKIO_STAT_WRITE
]) {
291 spin_unlock_irqrestore(&blkg
->stats_lock
, flags
);
296 * group is already marked empty. This can happen if cfqq got new
297 * request in parent group and moved to this group while being added
298 * to service tree. Just ignore the event and move on.
300 if(blkio_blkg_empty(stats
)) {
301 spin_unlock_irqrestore(&blkg
->stats_lock
, flags
);
305 stats
->start_empty_time
= sched_clock();
306 blkio_mark_blkg_empty(stats
);
307 spin_unlock_irqrestore(&blkg
->stats_lock
, flags
);
309 EXPORT_SYMBOL_GPL(blkiocg_set_start_empty_time
);
311 void blkiocg_update_dequeue_stats(struct blkio_group
*blkg
,
312 struct blkio_policy_type
*pol
,
313 unsigned long dequeue
)
315 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
317 pd
->stats
.dequeue
+= dequeue
;
319 EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats
);
321 static inline void blkio_set_start_group_wait_time(struct blkio_group
*blkg
,
322 struct blkio_policy_type
*pol
,
323 struct blkio_group
*curr_blkg
) { }
324 static inline void blkio_end_empty_time(struct blkio_group_stats
*stats
) { }
327 void blkiocg_update_io_add_stats(struct blkio_group
*blkg
,
328 struct blkio_policy_type
*pol
,
329 struct blkio_group
*curr_blkg
, bool direction
,
332 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
335 spin_lock_irqsave(&blkg
->stats_lock
, flags
);
336 blkio_add_stat(pd
->stats
.stat_arr
[BLKIO_STAT_QUEUED
], 1, direction
,
338 blkio_end_empty_time(&pd
->stats
);
339 blkio_set_start_group_wait_time(blkg
, pol
, curr_blkg
);
340 spin_unlock_irqrestore(&blkg
->stats_lock
, flags
);
342 EXPORT_SYMBOL_GPL(blkiocg_update_io_add_stats
);
344 void blkiocg_update_io_remove_stats(struct blkio_group
*blkg
,
345 struct blkio_policy_type
*pol
,
346 bool direction
, bool sync
)
348 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
351 spin_lock_irqsave(&blkg
->stats_lock
, flags
);
352 blkio_check_and_dec_stat(pd
->stats
.stat_arr
[BLKIO_STAT_QUEUED
],
354 spin_unlock_irqrestore(&blkg
->stats_lock
, flags
);
356 EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats
);
358 void blkiocg_update_timeslice_used(struct blkio_group
*blkg
,
359 struct blkio_policy_type
*pol
,
361 unsigned long unaccounted_time
)
363 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
366 spin_lock_irqsave(&blkg
->stats_lock
, flags
);
367 pd
->stats
.time
+= time
;
368 #ifdef CONFIG_DEBUG_BLK_CGROUP
369 pd
->stats
.unaccounted_time
+= unaccounted_time
;
371 spin_unlock_irqrestore(&blkg
->stats_lock
, flags
);
373 EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used
);
376 * should be called under rcu read lock or queue lock to make sure blkg pointer
379 void blkiocg_update_dispatch_stats(struct blkio_group
*blkg
,
380 struct blkio_policy_type
*pol
,
381 uint64_t bytes
, bool direction
, bool sync
)
383 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
384 struct blkio_group_stats_cpu
*stats_cpu
;
388 * Disabling interrupts to provide mutual exclusion between two
389 * writes on same cpu. It probably is not needed for 64bit. Not
390 * optimizing that case yet.
392 local_irq_save(flags
);
394 stats_cpu
= this_cpu_ptr(pd
->stats_cpu
);
396 u64_stats_update_begin(&stats_cpu
->syncp
);
397 stats_cpu
->sectors
+= bytes
>> 9;
398 blkio_add_stat(stats_cpu
->stat_arr_cpu
[BLKIO_STAT_CPU_SERVICED
],
400 blkio_add_stat(stats_cpu
->stat_arr_cpu
[BLKIO_STAT_CPU_SERVICE_BYTES
],
401 bytes
, direction
, sync
);
402 u64_stats_update_end(&stats_cpu
->syncp
);
403 local_irq_restore(flags
);
405 EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats
);
407 void blkiocg_update_completion_stats(struct blkio_group
*blkg
,
408 struct blkio_policy_type
*pol
,
410 uint64_t io_start_time
, bool direction
,
413 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
414 struct blkio_group_stats
*stats
;
416 unsigned long long now
= sched_clock();
418 spin_lock_irqsave(&blkg
->stats_lock
, flags
);
420 if (time_after64(now
, io_start_time
))
421 blkio_add_stat(stats
->stat_arr
[BLKIO_STAT_SERVICE_TIME
],
422 now
- io_start_time
, direction
, sync
);
423 if (time_after64(io_start_time
, start_time
))
424 blkio_add_stat(stats
->stat_arr
[BLKIO_STAT_WAIT_TIME
],
425 io_start_time
- start_time
, direction
, sync
);
426 spin_unlock_irqrestore(&blkg
->stats_lock
, flags
);
428 EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats
);
430 /* Merged stats are per cpu. */
431 void blkiocg_update_io_merged_stats(struct blkio_group
*blkg
,
432 struct blkio_policy_type
*pol
,
433 bool direction
, bool sync
)
435 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
436 struct blkio_group_stats_cpu
*stats_cpu
;
440 * Disabling interrupts to provide mutual exclusion between two
441 * writes on same cpu. It probably is not needed for 64bit. Not
442 * optimizing that case yet.
444 local_irq_save(flags
);
446 stats_cpu
= this_cpu_ptr(pd
->stats_cpu
);
448 u64_stats_update_begin(&stats_cpu
->syncp
);
449 blkio_add_stat(stats_cpu
->stat_arr_cpu
[BLKIO_STAT_CPU_MERGED
], 1,
451 u64_stats_update_end(&stats_cpu
->syncp
);
452 local_irq_restore(flags
);
454 EXPORT_SYMBOL_GPL(blkiocg_update_io_merged_stats
);
457 * blkg_free - free a blkg
458 * @blkg: blkg to free
460 * Free @blkg which may be partially allocated.
462 static void blkg_free(struct blkio_group
*blkg
)
469 for (i
= 0; i
< BLKIO_NR_POLICIES
; i
++) {
470 struct blkg_policy_data
*pd
= blkg
->pd
[i
];
473 free_percpu(pd
->stats_cpu
);
482 * blkg_alloc - allocate a blkg
483 * @blkcg: block cgroup the new blkg is associated with
484 * @q: request_queue the new blkg is associated with
486 * Allocate a new blkg assocating @blkcg and @q.
488 * FIXME: Should be called with queue locked but currently isn't due to
489 * percpu stat breakage.
491 static struct blkio_group
*blkg_alloc(struct blkio_cgroup
*blkcg
,
492 struct request_queue
*q
)
494 struct blkio_group
*blkg
;
497 /* alloc and init base part */
498 blkg
= kzalloc_node(sizeof(*blkg
), GFP_ATOMIC
, q
->node
);
502 spin_lock_init(&blkg
->stats_lock
);
503 rcu_assign_pointer(blkg
->q
, q
);
504 INIT_LIST_HEAD(&blkg
->q_node
);
507 cgroup_path(blkcg
->css
.cgroup
, blkg
->path
, sizeof(blkg
->path
));
509 for (i
= 0; i
< BLKIO_NR_POLICIES
; i
++) {
510 struct blkio_policy_type
*pol
= blkio_policy
[i
];
511 struct blkg_policy_data
*pd
;
516 /* alloc per-policy data and attach it to blkg */
517 pd
= kzalloc_node(sizeof(*pd
) + pol
->pdata_size
, GFP_ATOMIC
,
527 /* broken, read comment in the callsite */
528 pd
->stats_cpu
= alloc_percpu(struct blkio_group_stats_cpu
);
529 if (!pd
->stats_cpu
) {
535 /* invoke per-policy init */
536 for (i
= 0; i
< BLKIO_NR_POLICIES
; i
++) {
537 struct blkio_policy_type
*pol
= blkio_policy
[i
];
540 pol
->ops
.blkio_init_group_fn(blkg
);
546 struct blkio_group
*blkg_lookup_create(struct blkio_cgroup
*blkcg
,
547 struct request_queue
*q
,
548 enum blkio_policy_id plid
,
550 __releases(q
->queue_lock
) __acquires(q
->queue_lock
)
552 struct blkio_group
*blkg
, *new_blkg
;
554 WARN_ON_ONCE(!rcu_read_lock_held());
555 lockdep_assert_held(q
->queue_lock
);
558 * This could be the first entry point of blkcg implementation and
559 * we shouldn't allow anything to go through for a bypassing queue.
560 * The following can be removed if blkg lookup is guaranteed to
561 * fail on a bypassing queue.
563 if (unlikely(blk_queue_bypass(q
)) && !for_root
)
564 return ERR_PTR(blk_queue_dead(q
) ? -EINVAL
: -EBUSY
);
566 blkg
= blkg_lookup(blkcg
, q
);
570 /* blkg holds a reference to blkcg */
571 if (!css_tryget(&blkcg
->css
))
572 return ERR_PTR(-EINVAL
);
575 * Allocate and initialize.
577 * FIXME: The following is broken. Percpu memory allocation
578 * requires %GFP_KERNEL context and can't be performed from IO
579 * path. Allocation here should inherently be atomic and the
580 * following lock dancing can be removed once the broken percpu
581 * allocation is fixed.
583 spin_unlock_irq(q
->queue_lock
);
586 new_blkg
= blkg_alloc(blkcg
, q
);
589 spin_lock_irq(q
->queue_lock
);
591 /* did bypass get turned on inbetween? */
592 if (unlikely(blk_queue_bypass(q
)) && !for_root
) {
593 blkg
= ERR_PTR(blk_queue_dead(q
) ? -EINVAL
: -EBUSY
);
597 /* did someone beat us to it? */
598 blkg
= blkg_lookup(blkcg
, q
);
602 /* did alloc fail? */
603 if (unlikely(!new_blkg
)) {
604 blkg
= ERR_PTR(-ENOMEM
);
609 spin_lock(&blkcg
->lock
);
610 swap(blkg
, new_blkg
);
612 hlist_add_head_rcu(&blkg
->blkcg_node
, &blkcg
->blkg_list
);
613 list_add(&blkg
->q_node
, &q
->blkg_list
);
616 spin_unlock(&blkcg
->lock
);
621 EXPORT_SYMBOL_GPL(blkg_lookup_create
);
623 /* called under rcu_read_lock(). */
624 struct blkio_group
*blkg_lookup(struct blkio_cgroup
*blkcg
,
625 struct request_queue
*q
)
627 struct blkio_group
*blkg
;
628 struct hlist_node
*n
;
630 hlist_for_each_entry_rcu(blkg
, n
, &blkcg
->blkg_list
, blkcg_node
)
635 EXPORT_SYMBOL_GPL(blkg_lookup
);
637 static void blkg_destroy(struct blkio_group
*blkg
)
639 struct request_queue
*q
= blkg
->q
;
640 struct blkio_cgroup
*blkcg
= blkg
->blkcg
;
642 lockdep_assert_held(q
->queue_lock
);
643 lockdep_assert_held(&blkcg
->lock
);
645 /* Something wrong if we are trying to remove same group twice */
646 WARN_ON_ONCE(list_empty(&blkg
->q_node
));
647 WARN_ON_ONCE(hlist_unhashed(&blkg
->blkcg_node
));
648 list_del_init(&blkg
->q_node
);
649 hlist_del_init_rcu(&blkg
->blkcg_node
);
651 WARN_ON_ONCE(q
->nr_blkgs
<= 0);
655 * Put the reference taken at the time of creation so that when all
656 * queues are gone, group can be destroyed.
662 * XXX: This updates blkg policy data in-place for root blkg, which is
663 * necessary across elevator switch and policy registration as root blkgs
664 * aren't shot down. This broken and racy implementation is temporary.
665 * Eventually, blkg shoot down will be replaced by proper in-place update.
667 void update_root_blkg_pd(struct request_queue
*q
, enum blkio_policy_id plid
)
669 struct blkio_policy_type
*pol
= blkio_policy
[plid
];
670 struct blkio_group
*blkg
= blkg_lookup(&blkio_root_cgroup
, q
);
671 struct blkg_policy_data
*pd
;
676 kfree(blkg
->pd
[plid
]);
677 blkg
->pd
[plid
] = NULL
;
682 pd
= kzalloc(sizeof(*pd
) + pol
->pdata_size
, GFP_KERNEL
);
685 pd
->stats_cpu
= alloc_percpu(struct blkio_group_stats_cpu
);
686 WARN_ON_ONCE(!pd
->stats_cpu
);
690 pol
->ops
.blkio_init_group_fn(blkg
);
692 EXPORT_SYMBOL_GPL(update_root_blkg_pd
);
695 * blkg_destroy_all - destroy all blkgs associated with a request_queue
696 * @q: request_queue of interest
697 * @destroy_root: whether to destroy root blkg or not
699 * Destroy blkgs associated with @q. If @destroy_root is %true, all are
700 * destroyed; otherwise, root blkg is left alone.
702 void blkg_destroy_all(struct request_queue
*q
, bool destroy_root
)
704 struct blkio_group
*blkg
, *n
;
706 spin_lock_irq(q
->queue_lock
);
708 list_for_each_entry_safe(blkg
, n
, &q
->blkg_list
, q_node
) {
709 struct blkio_cgroup
*blkcg
= blkg
->blkcg
;
712 if (!destroy_root
&& blkg
->blkcg
== &blkio_root_cgroup
)
715 spin_lock(&blkcg
->lock
);
717 spin_unlock(&blkcg
->lock
);
720 spin_unlock_irq(q
->queue_lock
);
722 EXPORT_SYMBOL_GPL(blkg_destroy_all
);
724 static void blkg_rcu_free(struct rcu_head
*rcu_head
)
726 blkg_free(container_of(rcu_head
, struct blkio_group
, rcu_head
));
729 void __blkg_release(struct blkio_group
*blkg
)
731 /* release the extra blkcg reference this blkg has been holding */
732 css_put(&blkg
->blkcg
->css
);
735 * A group is freed in rcu manner. But having an rcu lock does not
736 * mean that one can access all the fields of blkg and assume these
737 * are valid. For example, don't try to follow throtl_data and
738 * request queue links.
740 * Having a reference to blkg under an rcu allows acess to only
741 * values local to groups like group stats and group rate limits
743 call_rcu(&blkg
->rcu_head
, blkg_rcu_free
);
745 EXPORT_SYMBOL_GPL(__blkg_release
);
747 static void blkio_reset_stats_cpu(struct blkio_group
*blkg
, int plid
)
749 struct blkg_policy_data
*pd
= blkg
->pd
[plid
];
750 struct blkio_group_stats_cpu
*stats_cpu
;
753 * Note: On 64 bit arch this should not be an issue. This has the
754 * possibility of returning some inconsistent value on 32bit arch
755 * as 64bit update on 32bit is non atomic. Taking care of this
756 * corner case makes code very complicated, like sending IPIs to
757 * cpus, taking care of stats of offline cpus etc.
759 * reset stats is anyway more of a debug feature and this sounds a
760 * corner case. So I am not complicating the code yet until and
761 * unless this becomes a real issue.
763 for_each_possible_cpu(i
) {
764 stats_cpu
= per_cpu_ptr(pd
->stats_cpu
, i
);
765 stats_cpu
->sectors
= 0;
766 for(j
= 0; j
< BLKIO_STAT_CPU_NR
; j
++)
767 for (k
= 0; k
< BLKIO_STAT_TOTAL
; k
++)
768 stats_cpu
->stat_arr_cpu
[j
][k
] = 0;
773 blkiocg_reset_stats(struct cgroup
*cgroup
, struct cftype
*cftype
, u64 val
)
775 struct blkio_cgroup
*blkcg
;
776 struct blkio_group
*blkg
;
777 struct blkio_group_stats
*stats
;
778 struct hlist_node
*n
;
779 uint64_t queued
[BLKIO_STAT_TOTAL
];
781 #ifdef CONFIG_DEBUG_BLK_CGROUP
782 bool idling
, waiting
, empty
;
783 unsigned long long now
= sched_clock();
786 blkcg
= cgroup_to_blkio_cgroup(cgroup
);
787 spin_lock(&blkio_list_lock
);
788 spin_lock_irq(&blkcg
->lock
);
789 hlist_for_each_entry(blkg
, n
, &blkcg
->blkg_list
, blkcg_node
) {
790 struct blkio_policy_type
*pol
;
792 list_for_each_entry(pol
, &blkio_list
, list
) {
793 struct blkg_policy_data
*pd
= blkg
->pd
[pol
->plid
];
795 spin_lock(&blkg
->stats_lock
);
797 #ifdef CONFIG_DEBUG_BLK_CGROUP
798 idling
= blkio_blkg_idling(stats
);
799 waiting
= blkio_blkg_waiting(stats
);
800 empty
= blkio_blkg_empty(stats
);
802 for (i
= 0; i
< BLKIO_STAT_TOTAL
; i
++)
803 queued
[i
] = stats
->stat_arr
[BLKIO_STAT_QUEUED
][i
];
804 memset(stats
, 0, sizeof(struct blkio_group_stats
));
805 for (i
= 0; i
< BLKIO_STAT_TOTAL
; i
++)
806 stats
->stat_arr
[BLKIO_STAT_QUEUED
][i
] = queued
[i
];
807 #ifdef CONFIG_DEBUG_BLK_CGROUP
809 blkio_mark_blkg_idling(stats
);
810 stats
->start_idle_time
= now
;
813 blkio_mark_blkg_waiting(stats
);
814 stats
->start_group_wait_time
= now
;
817 blkio_mark_blkg_empty(stats
);
818 stats
->start_empty_time
= now
;
821 spin_unlock(&blkg
->stats_lock
);
823 /* Reset Per cpu stats which don't take blkg->stats_lock */
824 blkio_reset_stats_cpu(blkg
, pol
->plid
);
828 spin_unlock_irq(&blkcg
->lock
);
829 spin_unlock(&blkio_list_lock
);
833 static void blkio_get_key_name(enum stat_sub_type type
, const char *dname
,
834 char *str
, int chars_left
, bool diskname_only
)
836 snprintf(str
, chars_left
, "%s", dname
);
837 chars_left
-= strlen(str
);
838 if (chars_left
<= 0) {
840 "Possibly incorrect cgroup stat display format");
846 case BLKIO_STAT_READ
:
847 strlcat(str
, " Read", chars_left
);
849 case BLKIO_STAT_WRITE
:
850 strlcat(str
, " Write", chars_left
);
852 case BLKIO_STAT_SYNC
:
853 strlcat(str
, " Sync", chars_left
);
855 case BLKIO_STAT_ASYNC
:
856 strlcat(str
, " Async", chars_left
);
858 case BLKIO_STAT_TOTAL
:
859 strlcat(str
, " Total", chars_left
);
862 strlcat(str
, " Invalid", chars_left
);
866 static uint64_t blkio_fill_stat(char *str
, int chars_left
, uint64_t val
,
867 struct cgroup_map_cb
*cb
, const char *dname
)
869 blkio_get_key_name(0, dname
, str
, chars_left
, true);
870 cb
->fill(cb
, str
, val
);
875 static uint64_t blkio_read_stat_cpu(struct blkio_group
*blkg
, int plid
,
876 enum stat_type_cpu type
, enum stat_sub_type sub_type
)
878 struct blkg_policy_data
*pd
= blkg
->pd
[plid
];
880 struct blkio_group_stats_cpu
*stats_cpu
;
883 for_each_possible_cpu(cpu
) {
885 stats_cpu
= per_cpu_ptr(pd
->stats_cpu
, cpu
);
888 start
= u64_stats_fetch_begin(&stats_cpu
->syncp
);
889 if (type
== BLKIO_STAT_CPU_SECTORS
)
890 tval
= stats_cpu
->sectors
;
892 tval
= stats_cpu
->stat_arr_cpu
[type
][sub_type
];
893 } while(u64_stats_fetch_retry(&stats_cpu
->syncp
, start
));
901 static uint64_t blkio_get_stat_cpu(struct blkio_group
*blkg
, int plid
,
902 struct cgroup_map_cb
*cb
, const char *dname
,
903 enum stat_type_cpu type
)
905 uint64_t disk_total
, val
;
906 char key_str
[MAX_KEY_LEN
];
907 enum stat_sub_type sub_type
;
909 if (type
== BLKIO_STAT_CPU_SECTORS
) {
910 val
= blkio_read_stat_cpu(blkg
, plid
, type
, 0);
911 return blkio_fill_stat(key_str
, MAX_KEY_LEN
- 1, val
, cb
,
915 for (sub_type
= BLKIO_STAT_READ
; sub_type
< BLKIO_STAT_TOTAL
;
917 blkio_get_key_name(sub_type
, dname
, key_str
, MAX_KEY_LEN
,
919 val
= blkio_read_stat_cpu(blkg
, plid
, type
, sub_type
);
920 cb
->fill(cb
, key_str
, val
);
923 disk_total
= blkio_read_stat_cpu(blkg
, plid
, type
, BLKIO_STAT_READ
) +
924 blkio_read_stat_cpu(blkg
, plid
, type
, BLKIO_STAT_WRITE
);
926 blkio_get_key_name(BLKIO_STAT_TOTAL
, dname
, key_str
, MAX_KEY_LEN
,
928 cb
->fill(cb
, key_str
, disk_total
);
932 /* This should be called with blkg->stats_lock held */
933 static uint64_t blkio_get_stat(struct blkio_group
*blkg
, int plid
,
934 struct cgroup_map_cb
*cb
, const char *dname
,
937 struct blkg_policy_data
*pd
= blkg
->pd
[plid
];
939 char key_str
[MAX_KEY_LEN
];
940 enum stat_sub_type sub_type
;
942 if (type
== BLKIO_STAT_TIME
)
943 return blkio_fill_stat(key_str
, MAX_KEY_LEN
- 1,
944 pd
->stats
.time
, cb
, dname
);
945 #ifdef CONFIG_DEBUG_BLK_CGROUP
946 if (type
== BLKIO_STAT_UNACCOUNTED_TIME
)
947 return blkio_fill_stat(key_str
, MAX_KEY_LEN
- 1,
948 pd
->stats
.unaccounted_time
, cb
, dname
);
949 if (type
== BLKIO_STAT_AVG_QUEUE_SIZE
) {
950 uint64_t sum
= pd
->stats
.avg_queue_size_sum
;
951 uint64_t samples
= pd
->stats
.avg_queue_size_samples
;
953 do_div(sum
, samples
);
956 return blkio_fill_stat(key_str
, MAX_KEY_LEN
- 1,
959 if (type
== BLKIO_STAT_GROUP_WAIT_TIME
)
960 return blkio_fill_stat(key_str
, MAX_KEY_LEN
- 1,
961 pd
->stats
.group_wait_time
, cb
, dname
);
962 if (type
== BLKIO_STAT_IDLE_TIME
)
963 return blkio_fill_stat(key_str
, MAX_KEY_LEN
- 1,
964 pd
->stats
.idle_time
, cb
, dname
);
965 if (type
== BLKIO_STAT_EMPTY_TIME
)
966 return blkio_fill_stat(key_str
, MAX_KEY_LEN
- 1,
967 pd
->stats
.empty_time
, cb
, dname
);
968 if (type
== BLKIO_STAT_DEQUEUE
)
969 return blkio_fill_stat(key_str
, MAX_KEY_LEN
- 1,
970 pd
->stats
.dequeue
, cb
, dname
);
973 for (sub_type
= BLKIO_STAT_READ
; sub_type
< BLKIO_STAT_TOTAL
;
975 blkio_get_key_name(sub_type
, dname
, key_str
, MAX_KEY_LEN
,
977 cb
->fill(cb
, key_str
, pd
->stats
.stat_arr
[type
][sub_type
]);
979 disk_total
= pd
->stats
.stat_arr
[type
][BLKIO_STAT_READ
] +
980 pd
->stats
.stat_arr
[type
][BLKIO_STAT_WRITE
];
981 blkio_get_key_name(BLKIO_STAT_TOTAL
, dname
, key_str
, MAX_KEY_LEN
,
983 cb
->fill(cb
, key_str
, disk_total
);
987 static int blkio_policy_parse_and_set(char *buf
, enum blkio_policy_id plid
,
988 int fileid
, struct blkio_cgroup
*blkcg
)
990 struct gendisk
*disk
= NULL
;
991 struct blkio_group
*blkg
= NULL
;
992 struct blkg_policy_data
*pd
;
993 char *s
[4], *p
, *major_s
= NULL
, *minor_s
= NULL
;
994 unsigned long major
, minor
;
995 int i
= 0, ret
= -EINVAL
;
1000 memset(s
, 0, sizeof(s
));
1002 while ((p
= strsep(&buf
, " ")) != NULL
) {
1008 /* Prevent from inputing too many things */
1016 p
= strsep(&s
[0], ":");
1026 if (strict_strtoul(major_s
, 10, &major
))
1029 if (strict_strtoul(minor_s
, 10, &minor
))
1032 dev
= MKDEV(major
, minor
);
1034 if (strict_strtoull(s
[1], 10, &temp
))
1037 disk
= get_gendisk(dev
, &part
);
1043 spin_lock_irq(disk
->queue
->queue_lock
);
1044 blkg
= blkg_lookup_create(blkcg
, disk
->queue
, plid
, false);
1045 spin_unlock_irq(disk
->queue
->queue_lock
);
1048 ret
= PTR_ERR(blkg
);
1052 pd
= blkg
->pd
[plid
];
1055 case BLKIO_POLICY_PROP
:
1056 if ((temp
< BLKIO_WEIGHT_MIN
&& temp
> 0) ||
1057 temp
> BLKIO_WEIGHT_MAX
)
1060 pd
->conf
.weight
= temp
;
1061 blkio_update_group_weight(blkg
, plid
, temp
?: blkcg
->weight
);
1063 case BLKIO_POLICY_THROTL
:
1065 case BLKIO_THROTL_read_bps_device
:
1066 pd
->conf
.bps
[READ
] = temp
;
1067 blkio_update_group_bps(blkg
, plid
, temp
?: -1, fileid
);
1069 case BLKIO_THROTL_write_bps_device
:
1070 pd
->conf
.bps
[WRITE
] = temp
;
1071 blkio_update_group_bps(blkg
, plid
, temp
?: -1, fileid
);
1073 case BLKIO_THROTL_read_iops_device
:
1074 if (temp
> THROTL_IOPS_MAX
)
1076 pd
->conf
.iops
[READ
] = temp
;
1077 blkio_update_group_iops(blkg
, plid
, temp
?: -1, fileid
);
1079 case BLKIO_THROTL_write_iops_device
:
1080 if (temp
> THROTL_IOPS_MAX
)
1082 pd
->conf
.iops
[WRITE
] = temp
;
1083 blkio_update_group_iops(blkg
, plid
, temp
?: -1, fileid
);
1097 * If queue was bypassing, we should retry. Do so after a short
1098 * msleep(). It isn't strictly necessary but queue can be
1099 * bypassing for some time and it's always nice to avoid busy
1102 if (ret
== -EBUSY
) {
1104 return restart_syscall();
1109 static int blkiocg_file_write(struct cgroup
*cgrp
, struct cftype
*cft
,
1114 struct blkio_cgroup
*blkcg
= cgroup_to_blkio_cgroup(cgrp
);
1115 enum blkio_policy_id plid
= BLKIOFILE_POLICY(cft
->private);
1116 int fileid
= BLKIOFILE_ATTR(cft
->private);
1118 buf
= kstrdup(buffer
, GFP_KERNEL
);
1122 ret
= blkio_policy_parse_and_set(buf
, plid
, fileid
, blkcg
);
1127 static const char *blkg_dev_name(struct blkio_group
*blkg
)
1129 /* some drivers (floppy) instantiate a queue w/o disk registered */
1130 if (blkg
->q
->backing_dev_info
.dev
)
1131 return dev_name(blkg
->q
->backing_dev_info
.dev
);
1135 static void blkio_print_group_conf(struct cftype
*cft
, struct blkio_group
*blkg
,
1138 int plid
= BLKIOFILE_POLICY(cft
->private);
1139 int fileid
= BLKIOFILE_ATTR(cft
->private);
1140 struct blkg_policy_data
*pd
= blkg
->pd
[plid
];
1141 const char *dname
= blkg_dev_name(blkg
);
1148 case BLKIO_POLICY_PROP
:
1149 if (pd
->conf
.weight
)
1150 seq_printf(m
, "%s\t%u\n",
1151 dname
, pd
->conf
.weight
);
1153 case BLKIO_POLICY_THROTL
:
1155 case BLKIO_THROTL_read_bps_device
:
1157 case BLKIO_THROTL_write_bps_device
:
1158 if (pd
->conf
.bps
[rw
])
1159 seq_printf(m
, "%s\t%llu\n",
1160 dname
, pd
->conf
.bps
[rw
]);
1162 case BLKIO_THROTL_read_iops_device
:
1164 case BLKIO_THROTL_write_iops_device
:
1165 if (pd
->conf
.iops
[rw
])
1166 seq_printf(m
, "%s\t%u\n",
1167 dname
, pd
->conf
.iops
[rw
]);
1176 /* cgroup files which read their data from policy nodes end up here */
1177 static void blkio_read_conf(struct cftype
*cft
, struct blkio_cgroup
*blkcg
,
1180 struct blkio_group
*blkg
;
1181 struct hlist_node
*n
;
1183 spin_lock_irq(&blkcg
->lock
);
1184 hlist_for_each_entry(blkg
, n
, &blkcg
->blkg_list
, blkcg_node
)
1185 blkio_print_group_conf(cft
, blkg
, m
);
1186 spin_unlock_irq(&blkcg
->lock
);
1189 static int blkiocg_file_read(struct cgroup
*cgrp
, struct cftype
*cft
,
1192 struct blkio_cgroup
*blkcg
;
1193 enum blkio_policy_id plid
= BLKIOFILE_POLICY(cft
->private);
1194 int name
= BLKIOFILE_ATTR(cft
->private);
1196 blkcg
= cgroup_to_blkio_cgroup(cgrp
);
1199 case BLKIO_POLICY_PROP
:
1201 case BLKIO_PROP_weight_device
:
1202 blkio_read_conf(cft
, blkcg
, m
);
1208 case BLKIO_POLICY_THROTL
:
1210 case BLKIO_THROTL_read_bps_device
:
1211 case BLKIO_THROTL_write_bps_device
:
1212 case BLKIO_THROTL_read_iops_device
:
1213 case BLKIO_THROTL_write_iops_device
:
1214 blkio_read_conf(cft
, blkcg
, m
);
1227 static int blkio_read_blkg_stats(struct blkio_cgroup
*blkcg
,
1228 struct cftype
*cft
, struct cgroup_map_cb
*cb
,
1229 enum stat_type type
, bool show_total
, bool pcpu
)
1231 struct blkio_group
*blkg
;
1232 struct hlist_node
*n
;
1233 uint64_t cgroup_total
= 0;
1236 hlist_for_each_entry_rcu(blkg
, n
, &blkcg
->blkg_list
, blkcg_node
) {
1237 const char *dname
= blkg_dev_name(blkg
);
1238 int plid
= BLKIOFILE_POLICY(cft
->private);
1243 cgroup_total
+= blkio_get_stat_cpu(blkg
, plid
,
1246 spin_lock_irq(&blkg
->stats_lock
);
1247 cgroup_total
+= blkio_get_stat(blkg
, plid
,
1249 spin_unlock_irq(&blkg
->stats_lock
);
1253 cb
->fill(cb
, "Total", cgroup_total
);
1258 /* All map kind of cgroup file get serviced by this function */
1259 static int blkiocg_file_read_map(struct cgroup
*cgrp
, struct cftype
*cft
,
1260 struct cgroup_map_cb
*cb
)
1262 struct blkio_cgroup
*blkcg
;
1263 enum blkio_policy_id plid
= BLKIOFILE_POLICY(cft
->private);
1264 int name
= BLKIOFILE_ATTR(cft
->private);
1266 blkcg
= cgroup_to_blkio_cgroup(cgrp
);
1269 case BLKIO_POLICY_PROP
:
1271 case BLKIO_PROP_time
:
1272 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1273 BLKIO_STAT_TIME
, 0, 0);
1274 case BLKIO_PROP_sectors
:
1275 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1276 BLKIO_STAT_CPU_SECTORS
, 0, 1);
1277 case BLKIO_PROP_io_service_bytes
:
1278 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1279 BLKIO_STAT_CPU_SERVICE_BYTES
, 1, 1);
1280 case BLKIO_PROP_io_serviced
:
1281 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1282 BLKIO_STAT_CPU_SERVICED
, 1, 1);
1283 case BLKIO_PROP_io_service_time
:
1284 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1285 BLKIO_STAT_SERVICE_TIME
, 1, 0);
1286 case BLKIO_PROP_io_wait_time
:
1287 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1288 BLKIO_STAT_WAIT_TIME
, 1, 0);
1289 case BLKIO_PROP_io_merged
:
1290 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1291 BLKIO_STAT_CPU_MERGED
, 1, 1);
1292 case BLKIO_PROP_io_queued
:
1293 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1294 BLKIO_STAT_QUEUED
, 1, 0);
1295 #ifdef CONFIG_DEBUG_BLK_CGROUP
1296 case BLKIO_PROP_unaccounted_time
:
1297 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1298 BLKIO_STAT_UNACCOUNTED_TIME
, 0, 0);
1299 case BLKIO_PROP_dequeue
:
1300 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1301 BLKIO_STAT_DEQUEUE
, 0, 0);
1302 case BLKIO_PROP_avg_queue_size
:
1303 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1304 BLKIO_STAT_AVG_QUEUE_SIZE
, 0, 0);
1305 case BLKIO_PROP_group_wait_time
:
1306 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1307 BLKIO_STAT_GROUP_WAIT_TIME
, 0, 0);
1308 case BLKIO_PROP_idle_time
:
1309 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1310 BLKIO_STAT_IDLE_TIME
, 0, 0);
1311 case BLKIO_PROP_empty_time
:
1312 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1313 BLKIO_STAT_EMPTY_TIME
, 0, 0);
1319 case BLKIO_POLICY_THROTL
:
1321 case BLKIO_THROTL_io_service_bytes
:
1322 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1323 BLKIO_STAT_CPU_SERVICE_BYTES
, 1, 1);
1324 case BLKIO_THROTL_io_serviced
:
1325 return blkio_read_blkg_stats(blkcg
, cft
, cb
,
1326 BLKIO_STAT_CPU_SERVICED
, 1, 1);
1338 static int blkio_weight_write(struct blkio_cgroup
*blkcg
, int plid
, u64 val
)
1340 struct blkio_group
*blkg
;
1341 struct hlist_node
*n
;
1343 if (val
< BLKIO_WEIGHT_MIN
|| val
> BLKIO_WEIGHT_MAX
)
1346 spin_lock(&blkio_list_lock
);
1347 spin_lock_irq(&blkcg
->lock
);
1348 blkcg
->weight
= (unsigned int)val
;
1350 hlist_for_each_entry(blkg
, n
, &blkcg
->blkg_list
, blkcg_node
) {
1351 struct blkg_policy_data
*pd
= blkg
->pd
[plid
];
1353 if (!pd
->conf
.weight
)
1354 blkio_update_group_weight(blkg
, plid
, blkcg
->weight
);
1357 spin_unlock_irq(&blkcg
->lock
);
1358 spin_unlock(&blkio_list_lock
);
1362 static u64
blkiocg_file_read_u64 (struct cgroup
*cgrp
, struct cftype
*cft
) {
1363 struct blkio_cgroup
*blkcg
;
1364 enum blkio_policy_id plid
= BLKIOFILE_POLICY(cft
->private);
1365 int name
= BLKIOFILE_ATTR(cft
->private);
1367 blkcg
= cgroup_to_blkio_cgroup(cgrp
);
1370 case BLKIO_POLICY_PROP
:
1372 case BLKIO_PROP_weight
:
1373 return (u64
)blkcg
->weight
;
1383 blkiocg_file_write_u64(struct cgroup
*cgrp
, struct cftype
*cft
, u64 val
)
1385 struct blkio_cgroup
*blkcg
;
1386 enum blkio_policy_id plid
= BLKIOFILE_POLICY(cft
->private);
1387 int name
= BLKIOFILE_ATTR(cft
->private);
1389 blkcg
= cgroup_to_blkio_cgroup(cgrp
);
1392 case BLKIO_POLICY_PROP
:
1394 case BLKIO_PROP_weight
:
1395 return blkio_weight_write(blkcg
, plid
, val
);
1405 struct cftype blkio_files
[] = {
1407 .name
= "weight_device",
1408 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1409 BLKIO_PROP_weight_device
),
1410 .read_seq_string
= blkiocg_file_read
,
1411 .write_string
= blkiocg_file_write
,
1412 .max_write_len
= 256,
1416 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1418 .read_u64
= blkiocg_file_read_u64
,
1419 .write_u64
= blkiocg_file_write_u64
,
1423 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1425 .read_map
= blkiocg_file_read_map
,
1429 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1430 BLKIO_PROP_sectors
),
1431 .read_map
= blkiocg_file_read_map
,
1434 .name
= "io_service_bytes",
1435 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1436 BLKIO_PROP_io_service_bytes
),
1437 .read_map
= blkiocg_file_read_map
,
1440 .name
= "io_serviced",
1441 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1442 BLKIO_PROP_io_serviced
),
1443 .read_map
= blkiocg_file_read_map
,
1446 .name
= "io_service_time",
1447 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1448 BLKIO_PROP_io_service_time
),
1449 .read_map
= blkiocg_file_read_map
,
1452 .name
= "io_wait_time",
1453 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1454 BLKIO_PROP_io_wait_time
),
1455 .read_map
= blkiocg_file_read_map
,
1458 .name
= "io_merged",
1459 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1460 BLKIO_PROP_io_merged
),
1461 .read_map
= blkiocg_file_read_map
,
1464 .name
= "io_queued",
1465 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1466 BLKIO_PROP_io_queued
),
1467 .read_map
= blkiocg_file_read_map
,
1470 .name
= "reset_stats",
1471 .write_u64
= blkiocg_reset_stats
,
1473 #ifdef CONFIG_BLK_DEV_THROTTLING
1475 .name
= "throttle.read_bps_device",
1476 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL
,
1477 BLKIO_THROTL_read_bps_device
),
1478 .read_seq_string
= blkiocg_file_read
,
1479 .write_string
= blkiocg_file_write
,
1480 .max_write_len
= 256,
1484 .name
= "throttle.write_bps_device",
1485 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL
,
1486 BLKIO_THROTL_write_bps_device
),
1487 .read_seq_string
= blkiocg_file_read
,
1488 .write_string
= blkiocg_file_write
,
1489 .max_write_len
= 256,
1493 .name
= "throttle.read_iops_device",
1494 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL
,
1495 BLKIO_THROTL_read_iops_device
),
1496 .read_seq_string
= blkiocg_file_read
,
1497 .write_string
= blkiocg_file_write
,
1498 .max_write_len
= 256,
1502 .name
= "throttle.write_iops_device",
1503 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL
,
1504 BLKIO_THROTL_write_iops_device
),
1505 .read_seq_string
= blkiocg_file_read
,
1506 .write_string
= blkiocg_file_write
,
1507 .max_write_len
= 256,
1510 .name
= "throttle.io_service_bytes",
1511 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL
,
1512 BLKIO_THROTL_io_service_bytes
),
1513 .read_map
= blkiocg_file_read_map
,
1516 .name
= "throttle.io_serviced",
1517 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL
,
1518 BLKIO_THROTL_io_serviced
),
1519 .read_map
= blkiocg_file_read_map
,
1521 #endif /* CONFIG_BLK_DEV_THROTTLING */
1523 #ifdef CONFIG_DEBUG_BLK_CGROUP
1525 .name
= "avg_queue_size",
1526 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1527 BLKIO_PROP_avg_queue_size
),
1528 .read_map
= blkiocg_file_read_map
,
1531 .name
= "group_wait_time",
1532 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1533 BLKIO_PROP_group_wait_time
),
1534 .read_map
= blkiocg_file_read_map
,
1537 .name
= "idle_time",
1538 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1539 BLKIO_PROP_idle_time
),
1540 .read_map
= blkiocg_file_read_map
,
1543 .name
= "empty_time",
1544 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1545 BLKIO_PROP_empty_time
),
1546 .read_map
= blkiocg_file_read_map
,
1550 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1551 BLKIO_PROP_dequeue
),
1552 .read_map
= blkiocg_file_read_map
,
1555 .name
= "unaccounted_time",
1556 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP
,
1557 BLKIO_PROP_unaccounted_time
),
1558 .read_map
= blkiocg_file_read_map
,
1563 static int blkiocg_populate(struct cgroup_subsys
*subsys
, struct cgroup
*cgroup
)
1565 return cgroup_add_files(cgroup
, subsys
, blkio_files
,
1566 ARRAY_SIZE(blkio_files
));
1570 * blkiocg_pre_destroy - cgroup pre_destroy callback
1571 * @subsys: cgroup subsys
1572 * @cgroup: cgroup of interest
1574 * This function is called when @cgroup is about to go away and responsible
1575 * for shooting down all blkgs associated with @cgroup. blkgs should be
1576 * removed while holding both q and blkcg locks. As blkcg lock is nested
1577 * inside q lock, this function performs reverse double lock dancing.
1579 * This is the blkcg counterpart of ioc_release_fn().
1581 static int blkiocg_pre_destroy(struct cgroup_subsys
*subsys
,
1582 struct cgroup
*cgroup
)
1584 struct blkio_cgroup
*blkcg
= cgroup_to_blkio_cgroup(cgroup
);
1587 spin_lock_irq(&blkcg
->lock
);
1589 while (!hlist_empty(&blkcg
->blkg_list
)) {
1590 struct blkio_group
*blkg
= hlist_entry(blkcg
->blkg_list
.first
,
1591 struct blkio_group
, blkcg_node
);
1592 struct request_queue
*q
= rcu_dereference(blkg
->q
);
1594 if (spin_trylock(q
->queue_lock
)) {
1596 spin_unlock(q
->queue_lock
);
1598 spin_unlock_irq(&blkcg
->lock
);
1602 spin_lock(&blkcg
->lock
);
1606 spin_unlock_irq(&blkcg
->lock
);
1611 static void blkiocg_destroy(struct cgroup_subsys
*subsys
, struct cgroup
*cgroup
)
1613 struct blkio_cgroup
*blkcg
= cgroup_to_blkio_cgroup(cgroup
);
1615 if (blkcg
!= &blkio_root_cgroup
)
1619 static struct cgroup_subsys_state
*
1620 blkiocg_create(struct cgroup_subsys
*subsys
, struct cgroup
*cgroup
)
1622 struct blkio_cgroup
*blkcg
;
1623 struct cgroup
*parent
= cgroup
->parent
;
1626 blkcg
= &blkio_root_cgroup
;
1630 blkcg
= kzalloc(sizeof(*blkcg
), GFP_KERNEL
);
1632 return ERR_PTR(-ENOMEM
);
1634 blkcg
->weight
= BLKIO_WEIGHT_DEFAULT
;
1636 spin_lock_init(&blkcg
->lock
);
1637 INIT_HLIST_HEAD(&blkcg
->blkg_list
);
1643 * blkcg_init_queue - initialize blkcg part of request queue
1644 * @q: request_queue to initialize
1646 * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
1647 * part of new request_queue @q.
1650 * 0 on success, -errno on failure.
1652 int blkcg_init_queue(struct request_queue
*q
)
1658 ret
= blk_throtl_init(q
);
1662 mutex_lock(&all_q_mutex
);
1663 INIT_LIST_HEAD(&q
->all_q_node
);
1664 list_add_tail(&q
->all_q_node
, &all_q_list
);
1665 mutex_unlock(&all_q_mutex
);
1671 * blkcg_drain_queue - drain blkcg part of request_queue
1672 * @q: request_queue to drain
1674 * Called from blk_drain_queue(). Responsible for draining blkcg part.
1676 void blkcg_drain_queue(struct request_queue
*q
)
1678 lockdep_assert_held(q
->queue_lock
);
1680 blk_throtl_drain(q
);
1684 * blkcg_exit_queue - exit and release blkcg part of request_queue
1685 * @q: request_queue being released
1687 * Called from blk_release_queue(). Responsible for exiting blkcg part.
1689 void blkcg_exit_queue(struct request_queue
*q
)
1691 mutex_lock(&all_q_mutex
);
1692 list_del_init(&q
->all_q_node
);
1693 mutex_unlock(&all_q_mutex
);
1695 blkg_destroy_all(q
, true);
1701 * We cannot support shared io contexts, as we have no mean to support
1702 * two tasks with the same ioc in two different groups without major rework
1703 * of the main cic data structures. For now we allow a task to change
1704 * its cgroup only if it's the only owner of its ioc.
1706 static int blkiocg_can_attach(struct cgroup_subsys
*ss
, struct cgroup
*cgrp
,
1707 struct cgroup_taskset
*tset
)
1709 struct task_struct
*task
;
1710 struct io_context
*ioc
;
1713 /* task_lock() is needed to avoid races with exit_io_context() */
1714 cgroup_taskset_for_each(task
, cgrp
, tset
) {
1716 ioc
= task
->io_context
;
1717 if (ioc
&& atomic_read(&ioc
->nr_tasks
) > 1)
1726 static void blkiocg_attach(struct cgroup_subsys
*ss
, struct cgroup
*cgrp
,
1727 struct cgroup_taskset
*tset
)
1729 struct task_struct
*task
;
1730 struct io_context
*ioc
;
1732 cgroup_taskset_for_each(task
, cgrp
, tset
) {
1733 /* we don't lose anything even if ioc allocation fails */
1734 ioc
= get_task_io_context(task
, GFP_ATOMIC
, NUMA_NO_NODE
);
1736 ioc_cgroup_changed(ioc
);
1737 put_io_context(ioc
);
1742 static void blkcg_bypass_start(void)
1743 __acquires(&all_q_mutex
)
1745 struct request_queue
*q
;
1747 mutex_lock(&all_q_mutex
);
1749 list_for_each_entry(q
, &all_q_list
, all_q_node
) {
1750 blk_queue_bypass_start(q
);
1751 blkg_destroy_all(q
, false);
1755 static void blkcg_bypass_end(void)
1756 __releases(&all_q_mutex
)
1758 struct request_queue
*q
;
1760 list_for_each_entry(q
, &all_q_list
, all_q_node
)
1761 blk_queue_bypass_end(q
);
1763 mutex_unlock(&all_q_mutex
);
1766 void blkio_policy_register(struct blkio_policy_type
*blkiop
)
1768 struct request_queue
*q
;
1770 blkcg_bypass_start();
1771 spin_lock(&blkio_list_lock
);
1773 BUG_ON(blkio_policy
[blkiop
->plid
]);
1774 blkio_policy
[blkiop
->plid
] = blkiop
;
1775 list_add_tail(&blkiop
->list
, &blkio_list
);
1777 spin_unlock(&blkio_list_lock
);
1778 list_for_each_entry(q
, &all_q_list
, all_q_node
)
1779 update_root_blkg_pd(q
, blkiop
->plid
);
1782 EXPORT_SYMBOL_GPL(blkio_policy_register
);
1784 void blkio_policy_unregister(struct blkio_policy_type
*blkiop
)
1786 struct request_queue
*q
;
1788 blkcg_bypass_start();
1789 spin_lock(&blkio_list_lock
);
1791 BUG_ON(blkio_policy
[blkiop
->plid
] != blkiop
);
1792 blkio_policy
[blkiop
->plid
] = NULL
;
1793 list_del_init(&blkiop
->list
);
1795 spin_unlock(&blkio_list_lock
);
1796 list_for_each_entry(q
, &all_q_list
, all_q_node
)
1797 update_root_blkg_pd(q
, blkiop
->plid
);
1800 EXPORT_SYMBOL_GPL(blkio_policy_unregister
);