mm/swap_cgroup.c

   1 #include <linux/swap_cgroup.h>
   2 #include <linux/vmalloc.h>
   3 #include <linux/mm.h>
   4
   5 #include <linux/swapops.h> /* depends on mm.h include */
   6
   7 static DEFINE_MUTEX(swap_cgroup_mutex);
   8 struct swap_cgroup_ctrl {
   9         struct page **map;
  10         unsigned long length;
  11         spinlock_t      lock;
  12 };
  13
  14 static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
  15
  16 struct swap_cgroup {
  17         unsigned short          id;
  18 };
  19 #define SC_PER_PAGE     (PAGE_SIZE/sizeof(struct swap_cgroup))
  20
  21 /*
  22  * SwapCgroup implements "lookup" and "exchange" operations.
  23  * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
  24  * against SwapCache. At swap_free(), this is accessed directly from swap.
  25  *
  26  * This means,
  27  *  - we have no race in "exchange" when we're accessed via SwapCache because
  28  *    SwapCache(and its swp_entry) is under lock.
  29  *  - When called via swap_free(), there is no user of this entry and no race.
  30  * Then, we don't need lock around "exchange".
  31  *
  32  * TODO: we can push these buffers out to HIGHMEM.
  33  */
  34
  35 /*
  36  * allocate buffer for swap_cgroup.
  37  */
  38 static int swap_cgroup_prepare(int type)
  39 {
  40         struct page *page;
  41         struct swap_cgroup_ctrl *ctrl;
  42         unsigned long idx, max;
  43
  44         ctrl = &swap_cgroup_ctrl[type];
  45
  46         for (idx = 0; idx < ctrl->length; idx++) {
  47                 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  48                 if (!page)
  49                         goto not_enough_page;
  50                 ctrl->map[idx] = page;
  51         }
  52         return 0;
  53 not_enough_page:
  54         max = idx;
  55         for (idx = 0; idx < max; idx++)
  56                 __free_page(ctrl->map[idx]);
  57
  58         return -ENOMEM;
  59 }
  60
  61 static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
  62                                         struct swap_cgroup_ctrl **ctrlp)
  63 {
  64         pgoff_t offset = swp_offset(ent);
  65         struct swap_cgroup_ctrl *ctrl;
  66         struct page *mappage;
  67         struct swap_cgroup *sc;
  68
  69         ctrl = &swap_cgroup_ctrl[swp_type(ent)];
  70         if (ctrlp)
  71                 *ctrlp = ctrl;
  72
  73         mappage = ctrl->map[offset / SC_PER_PAGE];
  74         sc = page_address(mappage);
  75         return sc + offset % SC_PER_PAGE;
  76 }
  77
  78 /**
  79  * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
  80  * @ent: swap entry to be cmpxchged
  81  * @old: old id
  82  * @new: new id
  83  *
  84  * Returns old id at success, 0 at failure.
  85  * (There is no mem_cgroup using 0 as its id)
  86  */
  87 unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
  88                                         unsigned short old, unsigned short new)
  89 {
  90         struct swap_cgroup_ctrl *ctrl;
  91         struct swap_cgroup *sc;
  92         unsigned long flags;
  93         unsigned short retval;
  94
  95         sc = lookup_swap_cgroup(ent, &ctrl);
  96
  97         spin_lock_irqsave(&ctrl->lock, flags);
  98         retval = sc->id;
  99         if (retval == old)
 100                 sc->id = new;
 101         else
 102                 retval = 0;
 103         spin_unlock_irqrestore(&ctrl->lock, flags);
 104         return retval;
 105 }
 106
 107 /**
 108  * swap_cgroup_record - record mem_cgroup for this swp_entry.
 109  * @ent: swap entry to be recorded into
 110  * @id: mem_cgroup to be recorded
 111  *
 112  * Returns old value at success, 0 at failure.
 113  * (Of course, old value can be 0.)
 114  */
 115 unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
 116 {
 117         struct swap_cgroup_ctrl *ctrl;
 118         struct swap_cgroup *sc;
 119         unsigned short old;
 120         unsigned long flags;
 121
 122         sc = lookup_swap_cgroup(ent, &ctrl);
 123
 124         spin_lock_irqsave(&ctrl->lock, flags);
 125         old = sc->id;
 126         sc->id = id;
 127         spin_unlock_irqrestore(&ctrl->lock, flags);
 128
 129         return old;
 130 }
 131
 132 /**
 133  * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
 134  * @ent: swap entry to be looked up.
 135  *
 136  * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
 137  */
 138 unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
 139 {
 140         return lookup_swap_cgroup(ent, NULL)->id;
 141 }
 142
 143 int swap_cgroup_swapon(int type, unsigned long max_pages)
 144 {
 145         void *array;
 146         unsigned long array_size;
 147         unsigned long length;
 148         struct swap_cgroup_ctrl *ctrl;
 149
 150         if (!do_swap_account)
 151                 return 0;
 152
 153         length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
 154         array_size = length * sizeof(void *);
 155
 156         array = vzalloc(array_size);
 157         if (!array)
 158                 goto nomem;
 159
 160         ctrl = &swap_cgroup_ctrl[type];
 161         mutex_lock(&swap_cgroup_mutex);
 162         ctrl->length = length;
 163         ctrl->map = array;
 164         spin_lock_init(&ctrl->lock);
 165         if (swap_cgroup_prepare(type)) {
 166                 /* memory shortage */
 167                 ctrl->map = NULL;
 168                 ctrl->length = 0;
 169                 mutex_unlock(&swap_cgroup_mutex);
 170                 vfree(array);
 171                 goto nomem;
 172         }
 173         mutex_unlock(&swap_cgroup_mutex);
 174
 175         return 0;
 176 nomem:
 177         pr_info("couldn't allocate enough memory for swap_cgroup\n");
 178         pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
 179         return -ENOMEM;
 180 }
 181
 182 void swap_cgroup_swapoff(int type)
 183 {
 184         struct page **map;
 185         unsigned long i, length;
 186         struct swap_cgroup_ctrl *ctrl;
 187
 188         if (!do_swap_account)
 189                 return;
 190
 191         mutex_lock(&swap_cgroup_mutex);
 192         ctrl = &swap_cgroup_ctrl[type];
 193         map = ctrl->map;
 194         length = ctrl->length;
 195         ctrl->map = NULL;
 196         ctrl->length = 0;
 197         mutex_unlock(&swap_cgroup_mutex);
 198
 199         if (map) {
 200                 for (i = 0; i < length; i++) {
 201                         struct page *page = map[i];
 202                         if (page)
 203                                 __free_page(page);
 204                         if (!(i % SWAP_CLUSTER_MAX))
 205                                 cond_resched();
 206                 }
 207                 vfree(map);
 208         }
 209 }