kernel/2.6.29.6-aldebaran-rt/include/linux/percpu.h

   1 #ifndef __LINUX_PERCPU_H
   2 #define __LINUX_PERCPU_H
   3
   4 #include <linux/preempt.h>
   5 #include <linux/slab.h> /* For kmalloc() */
   6 #include <linux/smp.h>
   7 #include <linux/cpumask.h>
   8 #include <linux/pfn.h>
   9
  10 #include <asm/percpu.h>
  11
  12 #ifndef PER_CPU_BASE_SECTION
  13 #ifdef CONFIG_SMP
  14 #define PER_CPU_BASE_SECTION ".data.percpu"
  15 #else
  16 #define PER_CPU_BASE_SECTION ".data"
  17 #endif
  18 #endif
  19
  20 #ifdef CONFIG_SMP
  21
  22 #ifdef MODULE
  23 #define PER_CPU_SHARED_ALIGNED_SECTION ""
  24 #else
  25 #define PER_CPU_SHARED_ALIGNED_SECTION ".shared_aligned"
  26 #endif
  27 #define PER_CPU_FIRST_SECTION ".first"
  28
  29 #else
  30
  31 #define PER_CPU_SHARED_ALIGNED_SECTION ""
  32 #define PER_CPU_FIRST_SECTION ""
  33
  34 #endif
  35
  36 #define DEFINE_PER_CPU_SECTION(type, name, section)                     \
  37         __attribute__((__section__(PER_CPU_BASE_SECTION section)))      \
  38         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  39
  40 #define DEFINE_PER_CPU_SPINLOCK(name, section)                          \
  41         __attribute__((__section__(PER_CPU_BASE_SECTION section)))      \
  42         PER_CPU_ATTRIBUTES __DEFINE_SPINLOCK(per_cpu__lock_##name##_locked);
  43
  44 #define DEFINE_PER_CPU(type, name)                                      \
  45         DEFINE_PER_CPU_SECTION(type, name, "")
  46
  47 #define DEFINE_PER_CPU_LOCKED(type, name)                               \
  48         DEFINE_PER_CPU_SPINLOCK(name, "")                               \
  49         DEFINE_PER_CPU_SECTION(type, name##_locked, "")
  50
  51 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                       \
  52         DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
  53         ____cacheline_aligned_in_smp
  54
  55 #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)                         \
  56         DEFINE_PER_CPU_SECTION(type, name, ".page_aligned")
  57
  58 #define DEFINE_PER_CPU_FIRST(type, name)                                \
  59         DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
  60
  61 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
  62 #define EXPORT_PER_CPU_LOCKED_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var##_locked)
  63 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
  64 #define EXPORT_PER_CPU_LOCKED_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var##_locked)
  65
  66 /* enough to cover all DEFINE_PER_CPUs in modules */
  67 #ifdef CONFIG_MODULES
  68 #define PERCPU_MODULE_RESERVE           (8 << 10)
  69 #else
  70 #define PERCPU_MODULE_RESERVE           0
  71 #endif
  72
  73 #ifndef PERCPU_ENOUGH_ROOM
  74 #define PERCPU_ENOUGH_ROOM                                              \
  75         (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) +      \
  76          PERCPU_MODULE_RESERVE)
  77 #endif
  78
  79 /*
  80  * Must be an lvalue. Since @var must be a simple identifier,
  81  * we force a syntax error here if it isn't.
  82  */
  83 #define get_cpu_var(var) (*({                           \
  84         extern int simple_identifier_##var(void);       \
  85         preempt_disable();                              \
  86         &__get_cpu_var(var); }))
  87 #define put_cpu_var(var) preempt_enable()
  88
  89 /*
  90  * Per-CPU data structures with an additional lock - useful for
  91  * PREEMPT_RT code that wants to reschedule but also wants
  92  * per-CPU data structures.
  93  *
  94  * 'cpu' gets updated with the CPU the task is currently executing on.
  95  *
  96  * NOTE: on normal !PREEMPT_RT kernels these per-CPU variables
  97  * are the same as the normal per-CPU variables, so there no
  98  * runtime overhead.
  99  */
 100 #ifdef CONFIG_PREEMPT_RT
 101 #define get_cpu_var_locked(var, cpuptr)                 \
 102 (*({                                                    \
 103         spinlock_t *__lock;                             \
 104         int __cpu;                                      \
 105                                                         \
 106 again:                                                  \
 107         __cpu = raw_smp_processor_id();                 \
 108         __lock = &__get_cpu_lock(var, __cpu);           \
 109         spin_lock(__lock);                              \
 110         if (!cpu_online(__cpu)) {                       \
 111                 spin_unlock(__lock);                    \
 112                 goto again;                             \
 113         }                                               \
 114         *(cpuptr) = __cpu;                              \
 115         &__get_cpu_var_locked(var, __cpu);              \
 116 }))
 117 #else
 118 #define get_cpu_var_locked(var, cpuptr)                 \
 119 (*({                                                    \
 120         int __cpu;                                      \
 121                                                         \
 122         preempt_disable();                              \
 123         __cpu = smp_processor_id();                     \
 124         spin_lock(&__get_cpu_lock(var, __cpu));         \
 125         preempt_enable();                               \
 126         *(cpuptr) = __cpu;                              \
 127         &__get_cpu_var_locked(var, __cpu);              \
 128 }))
 129 #endif
 130
 131 #define put_cpu_var_locked(var, cpu) \
 132          do { (void)cpu; spin_unlock(&__get_cpu_lock(var, cpu)); } while (0)
 133
 134 #ifdef CONFIG_SMP
 135
 136 #ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
 137
 138 /* minimum unit size, also is the maximum supported allocation size */
 139 #define PCPU_MIN_UNIT_SIZE              PFN_ALIGN(64 << 10)
 140
 141 /*
 142  * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy
 143  * back on the first chunk for dynamic percpu allocation if arch is
 144  * manually allocating and mapping it for faster access (as a part of
 145  * large page mapping for example).
 146  *
 147  * The following values give between one and two pages of free space
 148  * after typical minimal boot (2-way SMP, single disk and NIC) with
 149  * both defconfig and a distro config on x86_64 and 32.  More
 150  * intelligent way to determine this would be nice.
 151  */
 152 #if BITS_PER_LONG > 32
 153 #define PERCPU_DYNAMIC_RESERVE          (20 << 10)
 154 #else
 155 #define PERCPU_DYNAMIC_RESERVE          (12 << 10)
 156 #endif
 157
 158 extern void *pcpu_base_addr;
 159
 160 typedef struct page * (*pcpu_get_page_fn_t)(unsigned int cpu, int pageno);
 161 typedef void (*pcpu_populate_pte_fn_t)(unsigned long addr);
 162
 163 extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
 164                                 size_t static_size, size_t reserved_size,
 165                                 ssize_t dyn_size, ssize_t unit_size,
 166                                 void *base_addr,
 167                                 pcpu_populate_pte_fn_t populate_pte_fn);
 168
 169 extern ssize_t __init pcpu_embed_first_chunk(
 170                                 size_t static_size, size_t reserved_size,
 171                                 ssize_t dyn_size, ssize_t unit_size);
 172
 173 /*
 174  * Use this to get to a cpu's version of the per-cpu object
 175  * dynamically allocated. Non-atomic access to the current CPU's
 176  * version should probably be combined with get_cpu()/put_cpu().
 177  */
 178 #define per_cpu_ptr(ptr, cpu)   SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
 179
 180 extern void *__alloc_reserved_percpu(size_t size, size_t align);
 181
 182 #else /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
 183
 184 struct percpu_data {
 185         void *ptrs[1];
 186 };
 187
 188 #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
 189
 190 #define per_cpu_ptr(ptr, cpu)                                           \
 191 ({                                                                      \
 192         struct percpu_data *__p = __percpu_disguise(ptr);               \
 193         (__typeof__(ptr))__p->ptrs[(cpu)];                              \
 194 })
 195
 196 #endif /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
 197
 198 extern void *__alloc_percpu(size_t size, size_t align);
 199 extern void free_percpu(void *__pdata);
 200
 201 #else /* CONFIG_SMP */
 202
 203 #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
 204
 205 static inline void *__alloc_percpu(size_t size, size_t align)
 206 {
 207         /*
 208          * Can't easily make larger alignment work with kmalloc.  WARN
 209          * on it.  Larger alignment should only be used for module
 210          * percpu sections on SMP for which this path isn't used.
 211          */
 212         WARN_ON_ONCE(align > SMP_CACHE_BYTES);
 213         return kzalloc(size, GFP_KERNEL);
 214 }
 215
 216 static inline void free_percpu(void *p)
 217 {
 218         kfree(p);
 219 }
 220
 221 #endif /* CONFIG_SMP */
 222
 223 #define alloc_percpu(type)      (type *)__alloc_percpu(sizeof(type), \
 224                                                        __alignof__(type))
 225
 226 #endif /* __LINUX_PERCPU_H */