include/linux/percpu.h

   1 #ifndef __LINUX_PERCPU_H
   2 #define __LINUX_PERCPU_H
   3
   4 #include <linux/preempt.h>
   5 #include <linux/slab.h> /* For kmalloc() */
   6 #include <linux/smp.h>
   7 #include <linux/string.h> /* For memset() */
   8 #include <linux/cpumask.h>
   9
  10 #include <asm/percpu.h>
  11
  12 /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
  13 #ifndef PERCPU_ENOUGH_ROOM
  14 #ifdef CONFIG_MODULES
  15 #define PERCPU_MODULE_RESERVE   8192
  16 #else
  17 #define PERCPU_MODULE_RESERVE   0
  18 #endif
  19
  20 #define PERCPU_ENOUGH_ROOM                                              \
  21         (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE)
  22 #endif  /* PERCPU_ENOUGH_ROOM */
  23
  24 /*
  25  * Must be an lvalue. Since @var must be a simple identifier,
  26  * we force a syntax error here if it isn't.
  27  */
  28 #define get_cpu_var(var) (*({                           \
  29         extern int simple_identifier_##var(void);       \
  30         preempt_disable();                              \
  31         &__get_cpu_var(var); }))
  32 #define put_cpu_var(var) preempt_enable()
  33
  34 #ifdef CONFIG_SMP
  35
  36 struct percpu_data {
  37         void *ptrs[NR_CPUS];
  38 };
  39
  40 #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
  41 /*
  42  * Use this to get to a cpu's version of the per-cpu object dynamically
  43  * allocated. Non-atomic access to the current CPU's version should
  44  * probably be combined with get_cpu()/put_cpu().
  45  */
  46 #define percpu_ptr(ptr, cpu)                              \
  47 ({                                                        \
  48         struct percpu_data *__p = __percpu_disguise(ptr); \
  49         (__typeof__(ptr))__p->ptrs[(cpu)];                \
  50 })
  51
  52 extern void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu);
  53 extern void percpu_depopulate(void *__pdata, int cpu);
  54 extern int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
  55                                   cpumask_t *mask);
  56 extern void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask);
  57 extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask);
  58 extern void percpu_free(void *__pdata);
  59
  60 #else /* CONFIG_SMP */
  61
  62 #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
  63
  64 static inline void percpu_depopulate(void *__pdata, int cpu)
  65 {
  66 }
  67
  68 static inline void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask)
  69 {
  70 }
  71
  72 static inline void *percpu_populate(void *__pdata, size_t size, gfp_t gfp,
  73                                     int cpu)
  74 {
  75         return percpu_ptr(__pdata, cpu);
  76 }
  77
  78 static inline int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
  79                                          cpumask_t *mask)
  80 {
  81         return 0;
  82 }
  83
  84 static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
  85 {
  86         return kzalloc(size, gfp);
  87 }
  88
  89 static inline void percpu_free(void *__pdata)
  90 {
  91         kfree(__pdata);
  92 }
  93
  94 #endif /* CONFIG_SMP */
  95
  96 #define percpu_populate_mask(__pdata, size, gfp, mask) \
  97         __percpu_populate_mask((__pdata), (size), (gfp), &(mask))
  98 #define percpu_depopulate_mask(__pdata, mask) \
  99         __percpu_depopulate_mask((__pdata), &(mask))
 100 #define percpu_alloc_mask(size, gfp, mask) \
 101         __percpu_alloc_mask((size), (gfp), &(mask))
 102
 103 #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map)
 104
 105 /* (legacy) interface for use without CPU hotplug handling */
 106
 107 #define __alloc_percpu(size)    percpu_alloc_mask((size), GFP_KERNEL, \
 108                                                   cpu_possible_map)
 109 #define alloc_percpu(type)      (type *)__alloc_percpu(sizeof(type))
 110 #define free_percpu(ptr)        percpu_free((ptr))
 111 #define per_cpu_ptr(ptr, cpu)   percpu_ptr((ptr), (cpu))
 112
 113 #endif /* __LINUX_PERCPU_H */