percpu_counters-make-fbc-count-read-atomic-on-32-b.patch

   1 percpu_counters: make fbc->count read atomic on 32 bit architecture
   2
   3 From: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
   4
   5 fbc->count is of type s64. The change was introduced by
   6 0216bfcffe424a5473daa4da47440881b36c1f4 which changed the type
   7 from long to s64. Moving to s64 also means on 32 bit architectures
   8 we can get wrong values on fbc->count. Since fbc->count is read
   9 more frequently and updated rarely use seqlocks. This should
  10 reduce the impact of locking in the read path for 32bit arch.
  11
  12 Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
  13 CC: Peter Zijlstra <a.p.zijlstra@chello.nl>
  14 CC: Andrew Morton <akpm@linux-foundation.org>
  15 CC: linux-kernel@vger.kernel.org
  16 ---
  17  include/linux/percpu_counter.h |   18 ++++++++++++++----
  18  lib/percpu_counter.c           |   35 +++++++++++++++++++++++++----------
  19  2 files changed, 39 insertions(+), 14 deletions(-)
  20
  21 diff --git a/include/linux/percpu_counter.h b/include/linux/percpu_counter.h
  22 index 9007ccd..e657ca2 100644
  23 --- a/include/linux/percpu_counter.h
  24 +++ b/include/linux/percpu_counter.h
  25 @@ -6,7 +6,7 @@
  26   * WARNING: these things are HUGE.  4 kbytes per counter on 32-way P4.
  27   */
  28
  29 -#include <linux/spinlock.h>
  30 +#include <linux/seqlock.h>
  31  #include <linux/smp.h>
  32  #include <linux/list.h>
  33  #include <linux/threads.h>
  34 @@ -16,7 +16,7 @@
  35  #ifdef CONFIG_SMP
  36
  37  struct percpu_counter {
  38 -       spinlock_t lock;
  39 +       seqlock_t lock;
  40         s64 count;
  41  #ifdef CONFIG_HOTPLUG_CPU
  42         struct list_head list;  /* All percpu_counters are on a list */
  43 @@ -53,10 +53,20 @@ static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
  44         return __percpu_counter_sum(fbc);
  45  }
  46
  47 -static inline s64 percpu_counter_read(struct percpu_counter *fbc)
  48 +#if BITS_PER_LONG == 64
  49 +static inline s64 fbc_count(struct percpu_counter *fbc)
  50  {
  51         return fbc->count;
  52  }
  53 +#else
  54 +/* doesn't have atomic 64 bit operation */
  55 +extern s64 fbc_count(struct percpu_counter *fbc);
  56 +#endif
  57 +
  58 +static inline s64 percpu_counter_read(struct percpu_counter *fbc)
  59 +{
  60 +       return fbc_count(fbc);
  61 +}
  62
  63  /*
  64   * It is possible for the percpu_counter_read() to return a small negative
  65 @@ -65,7 +75,7 @@ static inline s64 percpu_counter_read(struct percpu_counter *fbc)
  66   */
  67  static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
  68  {
  69 -       s64 ret = fbc->count;
  70 +       s64 ret = fbc_count(fbc);
  71
  72         barrier();              /* Prevent reloads of fbc->count */
  73         if (ret >= 0)
  74 diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
  75 index a866389..9f4a03c 100644
  76 --- a/lib/percpu_counter.c
  77 +++ b/lib/percpu_counter.c
  78 @@ -18,13 +18,13 @@ void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
  79  {
  80         int cpu;
  81
  82 -       spin_lock(&fbc->lock);
  83 +       write_seqlock(&fbc->lock);
  84         for_each_possible_cpu(cpu) {
  85                 s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
  86                 *pcount = 0;
  87         }
  88         fbc->count = amount;
  89 -       spin_unlock(&fbc->lock);
  90 +       write_sequnlock(&fbc->lock);
  91  }
  92  EXPORT_SYMBOL(percpu_counter_set);
  93
  94 @@ -37,10 +37,10 @@ void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
  95         pcount = per_cpu_ptr(fbc->counters, cpu);
  96         count = *pcount + amount;
  97         if (count >= batch || count <= -batch) {
  98 -               spin_lock(&fbc->lock);
  99 +               write_seqlock(&fbc->lock);
 100                 fbc->count += count;
 101                 *pcount = 0;
 102 -               spin_unlock(&fbc->lock);
 103 +               write_sequnlock(&fbc->lock);
 104         } else {
 105                 *pcount = count;
 106         }
 107 @@ -57,7 +57,7 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc)
 108         s64 ret;
 109         int cpu;
 110
 111 -       spin_lock(&fbc->lock);
 112 +       write_seqlock(&fbc->lock);
 113         ret = fbc->count;
 114         for_each_online_cpu(cpu) {
 115                 s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
 116 @@ -66,16 +66,31 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc)
 117         }
 118         fbc->count = ret;
 119
 120 -       spin_unlock(&fbc->lock);
 121 +       write_sequnlock(&fbc->lock);
 122         return ret;
 123  }
 124  EXPORT_SYMBOL(__percpu_counter_sum);
 125
 126 +#if BITS_PER_LONG != 64
 127 +/* doesn't have atomic 64 bit operation */
 128 +s64 fbc_count(struct percpu_counter *fbc)
 129 +{
 130 +       s64 ret;
 131 +       unsigned seq;
 132 +       do {
 133 +               seq = read_seqbegin(&fbc->lock);
 134 +               ret = fbc->count;
 135 +       } while (read_seqretry(&fbc->lock, seq));
 136 +       return ret;
 137 +}
 138 +EXPORT_SYMBOL(fbc_count);
 139 +#endif
 140 +
 141  static struct lock_class_key percpu_counter_irqsafe;
 142
 143  int percpu_counter_init(struct percpu_counter *fbc, s64 amount)
 144  {
 145 -       spin_lock_init(&fbc->lock);
 146 +       seqlock_init(&fbc->lock);
 147         fbc->count = amount;
 148         fbc->counters = alloc_percpu(s32);
 149         if (!fbc->counters)
 150 @@ -95,7 +110,7 @@ int percpu_counter_init_irq(struct percpu_counter *fbc, s64 amount)
 151
 152         err = percpu_counter_init(fbc, amount);
 153         if (!err)
 154 -               lockdep_set_class(&fbc->lock, &percpu_counter_irqsafe);
 155 +               lockdep_set_class(&fbc->lock.lock, &percpu_counter_irqsafe);
 156         return err;
 157  }
 158
 159 @@ -130,11 +145,11 @@ static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb,
 160                 s32 *pcount;
 161                 unsigned long flags;
 162
 163 -               spin_lock_irqsave(&fbc->lock, flags);
 164 +               write_seqlock_irqsave(&fbc->lock, flags);
 165                 pcount = per_cpu_ptr(fbc->counters, cpu);
 166                 fbc->count += *pcount;
 167                 *pcount = 0;
 168 -               spin_unlock_irqrestore(&fbc->lock, flags);
 169 +               write_sequnlock_irqrestore(&fbc->lock, flags);
 170         }
 171         mutex_unlock(&percpu_counters_lock);
 172         return NOTIFY_OK;