2 * Fast batching percpu counters.
5 #include <linux/percpu_counter.h>
6 #include <linux/notifier.h>
7 #include <linux/mutex.h>
8 #include <linux/init.h>
10 #include <linux/module.h>
12 #ifdef CONFIG_HOTPLUG_CPU
13 static LIST_HEAD(percpu_counters
);
14 static DEFINE_MUTEX(percpu_counters_lock
);
17 void percpu_counter_set(struct percpu_counter
*fbc
, s64 amount
)
21 spin_lock(&fbc
->lock
);
22 for_each_possible_cpu(cpu
) {
23 s32
*pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
27 spin_unlock(&fbc
->lock
);
29 EXPORT_SYMBOL(percpu_counter_set
);
31 void __percpu_counter_add(struct percpu_counter
*fbc
, s64 amount
, s32 batch
)
37 pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
38 count
= *pcount
+ amount
;
39 if (count
>= batch
|| count
<= -batch
) {
40 spin_lock(&fbc
->lock
);
43 spin_unlock(&fbc
->lock
);
49 EXPORT_SYMBOL(__percpu_counter_add
);
52 * Add up all the per-cpu counts, return the result. This is a more accurate
53 * but much slower version of percpu_counter_read_positive()
55 s64
__percpu_counter_sum(struct percpu_counter
*fbc
)
60 spin_lock(&fbc
->lock
);
62 for_each_online_cpu(cpu
) {
63 s32
*pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
69 spin_unlock(&fbc
->lock
);
72 EXPORT_SYMBOL(__percpu_counter_sum
);
74 static struct lock_class_key percpu_counter_irqsafe
;
76 int percpu_counter_init(struct percpu_counter
*fbc
, s64 amount
)
78 spin_lock_init(&fbc
->lock
);
80 fbc
->counters
= alloc_percpu(s32
);
83 #ifdef CONFIG_HOTPLUG_CPU
84 INIT_LIST_HEAD(&fbc
->list
);
85 mutex_lock(&percpu_counters_lock
);
86 list_add(&fbc
->list
, &percpu_counters
);
87 mutex_unlock(&percpu_counters_lock
);
91 EXPORT_SYMBOL(percpu_counter_init
);
93 int percpu_counter_init_irq(struct percpu_counter
*fbc
, s64 amount
)
97 err
= percpu_counter_init(fbc
, amount
);
99 lockdep_set_class(&fbc
->lock
, &percpu_counter_irqsafe
);
103 void percpu_counter_destroy(struct percpu_counter
*fbc
)
108 free_percpu(fbc
->counters
);
109 fbc
->counters
= NULL
;
110 #ifdef CONFIG_HOTPLUG_CPU
111 mutex_lock(&percpu_counters_lock
);
112 list_del(&fbc
->list
);
113 mutex_unlock(&percpu_counters_lock
);
116 EXPORT_SYMBOL(percpu_counter_destroy
);
118 #ifdef CONFIG_HOTPLUG_CPU
119 static int __cpuinit
percpu_counter_hotcpu_callback(struct notifier_block
*nb
,
120 unsigned long action
, void *hcpu
)
123 struct percpu_counter
*fbc
;
125 if (action
!= CPU_DEAD
)
128 cpu
= (unsigned long)hcpu
;
129 mutex_lock(&percpu_counters_lock
);
130 list_for_each_entry(fbc
, &percpu_counters
, list
) {
134 spin_lock_irqsave(&fbc
->lock
, flags
);
135 pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
136 fbc
->count
+= *pcount
;
138 spin_unlock_irqrestore(&fbc
->lock
, flags
);
140 mutex_unlock(&percpu_counters_lock
);
144 static int __init
percpu_counter_startup(void)
146 hotcpu_notifier(percpu_counter_hotcpu_callback
, 0);
149 module_init(percpu_counter_startup
);