mm/percpu-internal.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _MM_PERCPU_INTERNAL_H
   3 #define _MM_PERCPU_INTERNAL_H
   4
   5 #include <linux/types.h>
   6 #include <linux/percpu.h>
   7
   8 /*
   9  * pcpu_block_md is the metadata block struct.
  10  * Each chunk's bitmap is split into a number of full blocks.
  11  * All units are in terms of bits.
  12  */
  13 struct pcpu_block_md {
  14         int                     contig_hint;    /* contig hint for block */
  15         int                     contig_hint_start; /* block relative starting
  16                                                       position of the contig hint */
  17         int                     left_free;      /* size of free space along
  18                                                    the left side of the block */
  19         int                     right_free;     /* size of free space along
  20                                                    the right side of the block */
  21         int                     first_free;     /* block position of first free */
  22 };
  23
  24 struct pcpu_chunk {
  25 #ifdef CONFIG_PERCPU_STATS
  26         int                     nr_alloc;       /* # of allocations */
  27         size_t                  max_alloc_size; /* largest allocation size */
  28 #endif
  29
  30         struct list_head        list;           /* linked to pcpu_slot lists */
  31         int                     free_bytes;     /* free bytes in the chunk */
  32         int                     contig_bits;    /* max contiguous size hint */
  33         int                     contig_bits_start; /* contig_bits starting
  34                                                       offset */
  35         void                    *base_addr;     /* base address of this chunk */
  36
  37         unsigned long           *alloc_map;     /* allocation map */
  38         unsigned long           *bound_map;     /* boundary map */
  39         struct pcpu_block_md    *md_blocks;     /* metadata blocks */
  40
  41         void                    *data;          /* chunk data */
  42         int                     first_bit;      /* no free below this */
  43         bool                    immutable;      /* no [de]population allowed */
  44         int                     start_offset;   /* the overlap with the previous
  45                                                    region to have a page aligned
  46                                                    base_addr */
  47         int                     end_offset;     /* additional area required to
  48                                                    have the region end page
  49                                                    aligned */
  50
  51         int                     nr_pages;       /* # of pages served by this chunk */
  52         int                     nr_populated;   /* # of populated pages */
  53         int                     nr_empty_pop_pages; /* # of empty populated pages */
  54         unsigned long           populated[];    /* populated bitmap */
  55 };
  56
  57 extern spinlock_t pcpu_lock;
  58
  59 extern struct list_head *pcpu_slot;
  60 extern int pcpu_nr_slots;
  61 extern int pcpu_nr_empty_pop_pages;
  62
  63 extern struct pcpu_chunk *pcpu_first_chunk;
  64 extern struct pcpu_chunk *pcpu_reserved_chunk;
  65
  66 /**
  67  * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks
  68  * @chunk: chunk of interest
  69  *
  70  * This conversion is from the number of physical pages that the chunk
  71  * serves to the number of bitmap blocks used.
  72  */
  73 static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk)
  74 {
  75         return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE;
  76 }
  77
  78 /**
  79  * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap
  80  * @pages: number of physical pages
  81  *
  82  * This conversion is from physical pages to the number of bits
  83  * required in the bitmap.
  84  */
  85 static inline int pcpu_nr_pages_to_map_bits(int pages)
  86 {
  87         return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;
  88 }
  89
  90 /**
  91  * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap
  92  * @chunk: chunk of interest
  93  *
  94  * This conversion is from the number of physical pages that the chunk
  95  * serves to the number of bits in the bitmap.
  96  */
  97 static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk)
  98 {
  99         return pcpu_nr_pages_to_map_bits(chunk->nr_pages);
 100 }
 101
 102 #ifdef CONFIG_PERCPU_STATS
 103
 104 #include <linux/spinlock.h>
 105
 106 struct percpu_stats {
 107         u64 nr_alloc;           /* lifetime # of allocations */
 108         u64 nr_dealloc;         /* lifetime # of deallocations */
 109         u64 nr_cur_alloc;       /* current # of allocations */
 110         u64 nr_max_alloc;       /* max # of live allocations */
 111         u32 nr_chunks;          /* current # of live chunks */
 112         u32 nr_max_chunks;      /* max # of live chunks */
 113         size_t min_alloc_size;  /* min allocaiton size */
 114         size_t max_alloc_size;  /* max allocation size */
 115 };
 116
 117 extern struct percpu_stats pcpu_stats;
 118 extern struct pcpu_alloc_info pcpu_stats_ai;
 119
 120 /*
 121  * For debug purposes. We don't care about the flexible array.
 122  */
 123 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
 124 {
 125         memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info));
 126
 127         /* initialize min_alloc_size to unit_size */
 128         pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size;
 129 }
 130
 131 /*
 132  * pcpu_stats_area_alloc - increment area allocation stats
 133  * @chunk: the location of the area being allocated
 134  * @size: size of area to allocate in bytes
 135  *
 136  * CONTEXT:
 137  * pcpu_lock.
 138  */
 139 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
 140 {
 141         lockdep_assert_held(&pcpu_lock);
 142
 143         pcpu_stats.nr_alloc++;
 144         pcpu_stats.nr_cur_alloc++;
 145         pcpu_stats.nr_max_alloc =
 146                 max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc);
 147         pcpu_stats.min_alloc_size =
 148                 min(pcpu_stats.min_alloc_size, size);
 149         pcpu_stats.max_alloc_size =
 150                 max(pcpu_stats.max_alloc_size, size);
 151
 152         chunk->nr_alloc++;
 153         chunk->max_alloc_size = max(chunk->max_alloc_size, size);
 154 }
 155
 156 /*
 157  * pcpu_stats_area_dealloc - decrement allocation stats
 158  * @chunk: the location of the area being deallocated
 159  *
 160  * CONTEXT:
 161  * pcpu_lock.
 162  */
 163 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
 164 {
 165         lockdep_assert_held(&pcpu_lock);
 166
 167         pcpu_stats.nr_dealloc++;
 168         pcpu_stats.nr_cur_alloc--;
 169
 170         chunk->nr_alloc--;
 171 }
 172
 173 /*
 174  * pcpu_stats_chunk_alloc - increment chunk stats
 175  */
 176 static inline void pcpu_stats_chunk_alloc(void)
 177 {
 178         unsigned long flags;
 179         spin_lock_irqsave(&pcpu_lock, flags);
 180
 181         pcpu_stats.nr_chunks++;
 182         pcpu_stats.nr_max_chunks =
 183                 max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks);
 184
 185         spin_unlock_irqrestore(&pcpu_lock, flags);
 186 }
 187
 188 /*
 189  * pcpu_stats_chunk_dealloc - decrement chunk stats
 190  */
 191 static inline void pcpu_stats_chunk_dealloc(void)
 192 {
 193         unsigned long flags;
 194         spin_lock_irqsave(&pcpu_lock, flags);
 195
 196         pcpu_stats.nr_chunks--;
 197
 198         spin_unlock_irqrestore(&pcpu_lock, flags);
 199 }
 200
 201 #else
 202
 203 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
 204 {
 205 }
 206
 207 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
 208 {
 209 }
 210
 211 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
 212 {
 213 }
 214
 215 static inline void pcpu_stats_chunk_alloc(void)
 216 {
 217 }
 218
 219 static inline void pcpu_stats_chunk_dealloc(void)
 220 {
 221 }
 222
 223 #endif /* !CONFIG_PERCPU_STATS */
 224
 225 #endif