mm/page_cgroup.c

   1 #include <linux/mm.h>
   2 #include <linux/mmzone.h>
   3 #include <linux/bootmem.h>
   4 #include <linux/bit_spinlock.h>
   5 #include <linux/page_cgroup.h>
   6 #include <linux/hash.h>
   7 #include <linux/slab.h>
   8 #include <linux/memory.h>
   9 #include <linux/vmalloc.h>
  10 #include <linux/cgroup.h>
  11
  12 static void __meminit
  13 __init_page_cgroup(struct page_cgroup *pc, unsigned long pfn)
  14 {
  15         pc->flags = 0;
  16         pc->mem_cgroup = NULL;
  17         pc->page = pfn_to_page(pfn);
  18 }
  19 static unsigned long total_usage;
  20
  21 #if !defined(CONFIG_SPARSEMEM)
  22
  23
  24 void __init pgdat_page_cgroup_init(struct pglist_data *pgdat)
  25 {
  26         pgdat->node_page_cgroup = NULL;
  27 }
  28
  29 struct page_cgroup *lookup_page_cgroup(struct page *page)
  30 {
  31         unsigned long pfn = page_to_pfn(page);
  32         unsigned long offset;
  33         struct page_cgroup *base;
  34
  35         base = NODE_DATA(page_to_nid(page))->node_page_cgroup;
  36         if (unlikely(!base))
  37                 return NULL;
  38
  39         offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn;
  40         return base + offset;
  41 }
  42
  43 static int __init alloc_node_page_cgroup(int nid)
  44 {
  45         struct page_cgroup *base, *pc;
  46         unsigned long table_size;
  47         unsigned long start_pfn, nr_pages, index;
  48
  49         start_pfn = NODE_DATA(nid)->node_start_pfn;
  50         nr_pages = NODE_DATA(nid)->node_spanned_pages;
  51
  52         table_size = sizeof(struct page_cgroup) * nr_pages;
  53
  54         base = __alloc_bootmem_node_nopanic(NODE_DATA(nid),
  55                         table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
  56         if (!base)
  57                 return -ENOMEM;
  58         for (index = 0; index < nr_pages; index++) {
  59                 pc = base + index;
  60                 __init_page_cgroup(pc, start_pfn + index);
  61         }
  62         NODE_DATA(nid)->node_page_cgroup = base;
  63         total_usage += table_size;
  64         return 0;
  65 }
  66
  67 void __init page_cgroup_init(void)
  68 {
  69
  70         int nid, fail;
  71
  72         if (mem_cgroup_subsys.disabled)
  73                 return;
  74
  75         for_each_online_node(nid)  {
  76                 fail = alloc_node_page_cgroup(nid);
  77                 if (fail)
  78                         goto fail;
  79         }
  80         printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
  81         printk(KERN_INFO "please try cgroup_disable=memory option if you"
  82         " don't want\n");
  83         return;
  84 fail:
  85         printk(KERN_CRIT "allocation of page_cgroup was failed.\n");
  86         printk(KERN_CRIT "please try cgroup_disable=memory boot option\n");
  87         panic("Out of memory");
  88 }
  89
  90 #else /* CONFIG_FLAT_NODE_MEM_MAP */
  91
  92 struct page_cgroup *lookup_page_cgroup(struct page *page)
  93 {
  94         unsigned long pfn = page_to_pfn(page);
  95         struct mem_section *section = __pfn_to_section(pfn);
  96
  97         return section->page_cgroup + pfn;
  98 }
  99
 100 int __meminit init_section_page_cgroup(unsigned long pfn)
 101 {
 102         struct mem_section *section;
 103         struct page_cgroup *base, *pc;
 104         unsigned long table_size;
 105         int nid, index;
 106
 107         section = __pfn_to_section(pfn);
 108
 109         if (section->page_cgroup)
 110                 return 0;
 111
 112         nid = page_to_nid(pfn_to_page(pfn));
 113
 114         table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
 115         if (slab_is_available()) {
 116                 base = kmalloc_node(table_size, GFP_KERNEL, nid);
 117                 if (!base)
 118                         base = vmalloc_node(table_size, nid);
 119         } else {
 120                 base = __alloc_bootmem_node_nopanic(NODE_DATA(nid), table_size,
 121                                 PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
 122         }
 123
 124         if (!base) {
 125                 printk(KERN_ERR "page cgroup allocation failure\n");
 126                 return -ENOMEM;
 127         }
 128
 129         for (index = 0; index < PAGES_PER_SECTION; index++) {
 130                 pc = base + index;
 131                 __init_page_cgroup(pc, pfn + index);
 132         }
 133
 134         section = __pfn_to_section(pfn);
 135         section->page_cgroup = base - pfn;
 136         total_usage += table_size;
 137         return 0;
 138 }
 139 #ifdef CONFIG_MEMORY_HOTPLUG
 140 void __free_page_cgroup(unsigned long pfn)
 141 {
 142         struct mem_section *ms;
 143         struct page_cgroup *base;
 144
 145         ms = __pfn_to_section(pfn);
 146         if (!ms || !ms->page_cgroup)
 147                 return;
 148         base = ms->page_cgroup + pfn;
 149         if (is_vmalloc_addr(base)) {
 150                 vfree(base);
 151                 ms->page_cgroup = NULL;
 152         } else {
 153                 struct page *page = virt_to_page(base);
 154                 if (!PageReserved(page)) { /* Is bootmem ? */
 155                         kfree(base);
 156                         ms->page_cgroup = NULL;
 157                 }
 158         }
 159 }
 160
 161 int online_page_cgroup(unsigned long start_pfn,
 162                         unsigned long nr_pages,
 163                         int nid)
 164 {
 165         unsigned long start, end, pfn;
 166         int fail = 0;
 167
 168         start = start_pfn & ~(PAGES_PER_SECTION - 1);
 169         end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);
 170
 171         for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) {
 172                 if (!pfn_present(pfn))
 173                         continue;
 174                 fail = init_section_page_cgroup(pfn);
 175         }
 176         if (!fail)
 177                 return 0;
 178
 179         /* rollback */
 180         for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
 181                 __free_page_cgroup(pfn);
 182
 183         return -ENOMEM;
 184 }
 185
 186 int offline_page_cgroup(unsigned long start_pfn,
 187                 unsigned long nr_pages, int nid)
 188 {
 189         unsigned long start, end, pfn;
 190
 191         start = start_pfn & ~(PAGES_PER_SECTION - 1);
 192         end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);
 193
 194         for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
 195                 __free_page_cgroup(pfn);
 196         return 0;
 197
 198 }
 199
 200 static int page_cgroup_callback(struct notifier_block *self,
 201                                unsigned long action, void *arg)
 202 {
 203         struct memory_notify *mn = arg;
 204         int ret = 0;
 205         switch (action) {
 206         case MEM_GOING_ONLINE:
 207                 ret = online_page_cgroup(mn->start_pfn,
 208                                    mn->nr_pages, mn->status_change_nid);
 209                 break;
 210         case MEM_CANCEL_ONLINE:
 211         case MEM_OFFLINE:
 212                 offline_page_cgroup(mn->start_pfn,
 213                                 mn->nr_pages, mn->status_change_nid);
 214                 break;
 215         case MEM_GOING_OFFLINE:
 216                 break;
 217         case MEM_ONLINE:
 218         case MEM_CANCEL_OFFLINE:
 219                 break;
 220         }
 221         ret = notifier_from_errno(ret);
 222         return ret;
 223 }
 224
 225 #endif
 226
 227 void __init page_cgroup_init(void)
 228 {
 229         unsigned long pfn;
 230         int fail = 0;
 231
 232         if (mem_cgroup_subsys.disabled)
 233                 return;
 234
 235         for (pfn = 0; !fail && pfn < max_pfn; pfn += PAGES_PER_SECTION) {
 236                 if (!pfn_present(pfn))
 237                         continue;
 238                 fail = init_section_page_cgroup(pfn);
 239         }
 240         if (fail) {
 241                 printk(KERN_CRIT "try cgroup_disable=memory boot option\n");
 242                 panic("Out of memory");
 243         } else {
 244                 hotplug_memory_notifier(page_cgroup_callback, 0);
 245         }
 246         printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
 247         printk(KERN_INFO "please try cgroup_disable=memory option if you don't"
 248         " want\n");
 249 }
 250
 251 void __init pgdat_page_cgroup_init(struct pglist_data *pgdat)
 252 {
 253         return;
 254 }
 255
 256 #endif