include/linux/percpu.h

   1 #ifndef __LINUX_PERCPU_H
   2 #define __LINUX_PERCPU_H
   3
   4 #include <linux/preempt.h>
   5 #include <linux/slab.h> /* For kmalloc() */
   6 #include <linux/smp.h>
   7 #include <linux/cpumask.h>
   8
   9 #include <asm/percpu.h>
  10
  11 #ifdef CONFIG_SMP
  12 #define DEFINE_PER_CPU(type, name)                                      \
  13         __attribute__((__section__(".data.percpu")))                    \
  14         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  15
  16 #ifdef MODULE
  17 #define SHARED_ALIGNED_SECTION ".data.percpu"
  18 #else
  19 #define SHARED_ALIGNED_SECTION ".data.percpu.shared_aligned"
  20 #endif
  21
  22 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                       \
  23         __attribute__((__section__(SHARED_ALIGNED_SECTION)))            \
  24         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name             \
  25         ____cacheline_aligned_in_smp
  26
  27 #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)                 \
  28         __attribute__((__section__(".data.percpu.page_aligned")))       \
  29         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  30 #else
  31 #define DEFINE_PER_CPU(type, name)                                      \
  32         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  33
  34 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                     \
  35         DEFINE_PER_CPU(type, name)
  36
  37 #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)               \
  38         DEFINE_PER_CPU(type, name)
  39 #endif
  40
  41 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
  42 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
  43
  44 /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
  45 #ifndef PERCPU_ENOUGH_ROOM
  46 #ifdef CONFIG_MODULES
  47 #define PERCPU_MODULE_RESERVE   8192
  48 #else
  49 #define PERCPU_MODULE_RESERVE   0
  50 #endif
  51
  52 #define PERCPU_ENOUGH_ROOM                                              \
  53         (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE)
  54 #endif  /* PERCPU_ENOUGH_ROOM */
  55
  56 /*
  57  * Must be an lvalue. Since @var must be a simple identifier,
  58  * we force a syntax error here if it isn't.
  59  */
  60 #define get_cpu_var(var) (*({                           \
  61         extern int simple_identifier_##var(void);       \
  62         preempt_disable();                              \
  63         &__get_cpu_var(var); }))
  64 #define put_cpu_var(var) preempt_enable()
  65
  66 #ifdef CONFIG_SMP
  67
  68 struct percpu_data {
  69         void *ptrs[1];
  70 };
  71
  72 #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
  73 /*
  74  * Use this to get to a cpu's version of the per-cpu object dynamically
  75  * allocated. Non-atomic access to the current CPU's version should
  76  * probably be combined with get_cpu()/put_cpu().
  77  */
  78 #define percpu_ptr(ptr, cpu)                              \
  79 ({                                                        \
  80         struct percpu_data *__p = __percpu_disguise(ptr); \
  81         (__typeof__(ptr))__p->ptrs[(cpu)];                \
  82 })
  83
  84 extern void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu);
  85 extern void percpu_depopulate(void *__pdata, int cpu);
  86 extern int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
  87                                   cpumask_t *mask);
  88 extern void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask);
  89 extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask);
  90 extern void percpu_free(void *__pdata);
  91
  92 #else /* CONFIG_SMP */
  93
  94 #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
  95
  96 static inline void percpu_depopulate(void *__pdata, int cpu)
  97 {
  98 }
  99
 100 static inline void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask)
 101 {
 102 }
 103
 104 static inline void *percpu_populate(void *__pdata, size_t size, gfp_t gfp,
 105                                     int cpu)
 106 {
 107         return percpu_ptr(__pdata, cpu);
 108 }
 109
 110 static inline int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
 111                                          cpumask_t *mask)
 112 {
 113         return 0;
 114 }
 115
 116 static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
 117 {
 118         return kzalloc(size, gfp);
 119 }
 120
 121 static inline void percpu_free(void *__pdata)
 122 {
 123         kfree(__pdata);
 124 }
 125
 126 #endif /* CONFIG_SMP */
 127
 128 #define percpu_populate_mask(__pdata, size, gfp, mask) \
 129         __percpu_populate_mask((__pdata), (size), (gfp), &(mask))
 130 #define percpu_depopulate_mask(__pdata, mask) \
 131         __percpu_depopulate_mask((__pdata), &(mask))
 132 #define percpu_alloc_mask(size, gfp, mask) \
 133         __percpu_alloc_mask((size), (gfp), &(mask))
 134
 135 #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map)
 136
 137 /* (legacy) interface for use without CPU hotplug handling */
 138
 139 #define __alloc_percpu(size)    percpu_alloc_mask((size), GFP_KERNEL, \
 140                                                   cpu_possible_map)
 141 #define alloc_percpu(type)      (type *)__alloc_percpu(sizeof(type))
 142 #define free_percpu(ptr)        percpu_free((ptr))
 143 #define per_cpu_ptr(ptr, cpu)   percpu_ptr((ptr), (cpu))
 144
 145 #endif /* __LINUX_PERCPU_H */