mm/memcontrol.c

   1 /* memcontrol.c - Memory Controller
   2  *
   3  * Copyright IBM Corporation, 2007
   4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
   5  *
   6  * Copyright 2007 OpenVZ SWsoft Inc
   7  * Author: Pavel Emelianov <xemul@openvz.org>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  */
  19
  20 #include <linux/res_counter.h>
  21 #include <linux/memcontrol.h>
  22 #include <linux/cgroup.h>
  23 #include <linux/mm.h>
  24 #include <linux/smp.h>
  25 #include <linux/page-flags.h>
  26 #include <linux/backing-dev.h>
  27 #include <linux/bit_spinlock.h>
  28 #include <linux/rcupdate.h>
  29 #include <linux/swap.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/fs.h>
  32 #include <linux/seq_file.h>
  33
  34 #include <asm/uaccess.h>
  35
  36 struct cgroup_subsys mem_cgroup_subsys;
  37 static const int MEM_CGROUP_RECLAIM_RETRIES = 5;
  38
  39 /*
  40  * Statistics for memory cgroup.
  41  */
  42 enum mem_cgroup_stat_index {
  43         /*
  44          * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
  45          */
  46         MEM_CGROUP_STAT_CACHE,     /* # of pages charged as cache */
  47         MEM_CGROUP_STAT_RSS,       /* # of pages charged as rss */
  48
  49         MEM_CGROUP_STAT_NSTATS,
  50 };
  51
  52 struct mem_cgroup_stat_cpu {
  53         s64 count[MEM_CGROUP_STAT_NSTATS];
  54 } ____cacheline_aligned_in_smp;
  55
  56 struct mem_cgroup_stat {
  57         struct mem_cgroup_stat_cpu cpustat[NR_CPUS];
  58 };
  59
  60 /*
  61  * For accounting under irq disable, no need for increment preempt count.
  62  */
  63 static void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat *stat,
  64                 enum mem_cgroup_stat_index idx, int val)
  65 {
  66         int cpu = smp_processor_id();
  67         stat->cpustat[cpu].count[idx] += val;
  68 }
  69
  70 static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
  71                 enum mem_cgroup_stat_index idx)
  72 {
  73         int cpu;
  74         s64 ret = 0;
  75         for_each_possible_cpu(cpu)
  76                 ret += stat->cpustat[cpu].count[idx];
  77         return ret;
  78 }
  79
  80 /*
  81  * per-zone information in memory controller.
  82  */
  83
  84 enum mem_cgroup_zstat_index {
  85         MEM_CGROUP_ZSTAT_ACTIVE,
  86         MEM_CGROUP_ZSTAT_INACTIVE,
  87
  88         NR_MEM_CGROUP_ZSTAT,
  89 };
  90
  91 struct mem_cgroup_per_zone {
  92         /*
  93          * spin_lock to protect the per cgroup LRU
  94          */
  95         spinlock_t              lru_lock;
  96         struct list_head        active_list;
  97         struct list_head        inactive_list;
  98         unsigned long count[NR_MEM_CGROUP_ZSTAT];
  99 };
 100 /* Macro for accessing counter */
 101 #define MEM_CGROUP_ZSTAT(mz, idx)       ((mz)->count[(idx)])
 102
 103 struct mem_cgroup_per_node {
 104         struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
 105 };
 106
 107 struct mem_cgroup_lru_info {
 108         struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
 109 };
 110
 111 /*
 112  * The memory controller data structure. The memory controller controls both
 113  * page cache and RSS per cgroup. We would eventually like to provide
 114  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 115  * to help the administrator determine what knobs to tune.
 116  *
 117  * TODO: Add a water mark for the memory controller. Reclaim will begin when
 118  * we hit the water mark. May be even add a low water mark, such that
 119  * no reclaim occurs from a cgroup at it's low water mark, this is
 120  * a feature that will be implemented much later in the future.
 121  */
 122 struct mem_cgroup {
 123         struct cgroup_subsys_state css;
 124         /*
 125          * the counter to account for memory usage
 126          */
 127         struct res_counter res;
 128         /*
 129          * Per cgroup active and inactive list, similar to the
 130          * per zone LRU lists.
 131          */
 132         struct mem_cgroup_lru_info info;
 133
 134         int     prev_priority;  /* for recording reclaim priority */
 135         /*
 136          * statistics.
 137          */
 138         struct mem_cgroup_stat stat;
 139 };
 140 <<<<<<< HEAD:mm/memcontrol.c
 141 =======
 142 static struct mem_cgroup init_mem_cgroup;
 143 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 144
 145 /*
 146  * We use the lower bit of the page->page_cgroup pointer as a bit spin
 147 <<<<<<< HEAD:mm/memcontrol.c
 148  * lock. We need to ensure that page->page_cgroup is atleast two
 149  * byte aligned (based on comments from Nick Piggin)
 150 =======
 151  * lock.  We need to ensure that page->page_cgroup is at least two
 152  * byte aligned (based on comments from Nick Piggin).  But since
 153  * bit_spin_lock doesn't actually set that lock bit in a non-debug
 154  * uniprocessor kernel, we should avoid setting it here too.
 155 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 156  */
 157 #define PAGE_CGROUP_LOCK_BIT    0x0
 158 <<<<<<< HEAD:mm/memcontrol.c
 159 #define PAGE_CGROUP_LOCK                (1 << PAGE_CGROUP_LOCK_BIT)
 160 =======
 161 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 162 #define PAGE_CGROUP_LOCK        (1 << PAGE_CGROUP_LOCK_BIT)
 163 #else
 164 #define PAGE_CGROUP_LOCK        0x0
 165 #endif
 166 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 167
 168 /*
 169  * A page_cgroup page is associated with every page descriptor. The
 170  * page_cgroup helps us identify information about the cgroup
 171  */
 172 struct page_cgroup {
 173         struct list_head lru;           /* per cgroup LRU list */
 174         struct page *page;
 175         struct mem_cgroup *mem_cgroup;
 176 <<<<<<< HEAD:mm/memcontrol.c
 177         atomic_t ref_cnt;               /* Helpful when pages move b/w  */
 178                                         /* mapped and cached states     */
 179         int      flags;
 180 =======
 181         int ref_cnt;                    /* cached, mapped, migrating */
 182         int flags;
 183 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 184 };
 185 #define PAGE_CGROUP_FLAG_CACHE  (0x1)   /* charged as cache */
 186 #define PAGE_CGROUP_FLAG_ACTIVE (0x2)   /* page is active in this cgroup */
 187
 188 <<<<<<< HEAD:mm/memcontrol.c
 189 static inline int page_cgroup_nid(struct page_cgroup *pc)
 190 =======
 191 static int page_cgroup_nid(struct page_cgroup *pc)
 192 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 193 {
 194         return page_to_nid(pc->page);
 195 }
 196
 197 <<<<<<< HEAD:mm/memcontrol.c
 198 static inline enum zone_type page_cgroup_zid(struct page_cgroup *pc)
 199 =======
 200 static enum zone_type page_cgroup_zid(struct page_cgroup *pc)
 201 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 202 {
 203         return page_zonenum(pc->page);
 204 }
 205
 206 <<<<<<< HEAD:mm/memcontrol.c
 207 enum {
 208         MEM_CGROUP_TYPE_UNSPEC = 0,
 209         MEM_CGROUP_TYPE_MAPPED,
 210         MEM_CGROUP_TYPE_CACHED,
 211         MEM_CGROUP_TYPE_ALL,
 212         MEM_CGROUP_TYPE_MAX,
 213 };
 214
 215 =======
 216 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 217 enum charge_type {
 218         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
 219         MEM_CGROUP_CHARGE_TYPE_MAPPED,
 220 };
 221
 222 <<<<<<< HEAD:mm/memcontrol.c
 223
 224 =======
 225 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 226 /*
 227  * Always modified under lru lock. Then, not necessary to preempt_disable()
 228  */
 229 static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, int flags,
 230                                         bool charge)
 231 {
 232         int val = (charge)? 1 : -1;
 233         struct mem_cgroup_stat *stat = &mem->stat;
 234 <<<<<<< HEAD:mm/memcontrol.c
 235         VM_BUG_ON(!irqs_disabled());
 236 =======
 237 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 238
 239 <<<<<<< HEAD:mm/memcontrol.c
 240 =======
 241         VM_BUG_ON(!irqs_disabled());
 242 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 243         if (flags & PAGE_CGROUP_FLAG_CACHE)
 244 <<<<<<< HEAD:mm/memcontrol.c
 245                 __mem_cgroup_stat_add_safe(stat,
 246                                         MEM_CGROUP_STAT_CACHE, val);
 247 =======
 248                 __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_CACHE, val);
 249 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 250         else
 251                 __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_RSS, val);
 252 }
 253
 254 <<<<<<< HEAD:mm/memcontrol.c
 255 static inline struct mem_cgroup_per_zone *
 256 =======
 257 static struct mem_cgroup_per_zone *
 258 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 259 mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
 260 {
 261 <<<<<<< HEAD:mm/memcontrol.c
 262         BUG_ON(!mem->info.nodeinfo[nid]);
 263 =======
 264 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 265         return &mem->info.nodeinfo[nid]->zoneinfo[zid];
 266 }
 267
 268 <<<<<<< HEAD:mm/memcontrol.c
 269 static inline struct mem_cgroup_per_zone *
 270 =======
 271 static struct mem_cgroup_per_zone *
 272 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 273 page_cgroup_zoneinfo(struct page_cgroup *pc)
 274 {
 275         struct mem_cgroup *mem = pc->mem_cgroup;
 276         int nid = page_cgroup_nid(pc);
 277         int zid = page_cgroup_zid(pc);
 278
 279         return mem_cgroup_zoneinfo(mem, nid, zid);
 280 }
 281
 282 static unsigned long mem_cgroup_get_all_zonestat(struct mem_cgroup *mem,
 283                                         enum mem_cgroup_zstat_index idx)
 284 {
 285         int nid, zid;
 286         struct mem_cgroup_per_zone *mz;
 287         u64 total = 0;
 288
 289         for_each_online_node(nid)
 290                 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
 291                         mz = mem_cgroup_zoneinfo(mem, nid, zid);
 292                         total += MEM_CGROUP_ZSTAT(mz, idx);
 293                 }
 294         return total;
 295 }
 296
 297 <<<<<<< HEAD:mm/memcontrol.c
 298 static struct mem_cgroup init_mem_cgroup;
 299
 300 static inline
 301 struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
 302 =======
 303 static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
 304 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 305 {
 306         return container_of(cgroup_subsys_state(cont,
 307                                 mem_cgroup_subsys_id), struct mem_cgroup,
 308                                 css);
 309 }
 310
 311 <<<<<<< HEAD:mm/memcontrol.c
 312 static inline
 313 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
 314 =======
 315 static struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
 316 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 317 {
 318         return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
 319                                 struct mem_cgroup, css);
 320 }
 321
 322 void mm_init_cgroup(struct mm_struct *mm, struct task_struct *p)
 323 {
 324         struct mem_cgroup *mem;
 325
 326         mem = mem_cgroup_from_task(p);
 327         css_get(&mem->css);
 328         mm->mem_cgroup = mem;
 329 }
 330
 331 void mm_free_cgroup(struct mm_struct *mm)
 332 {
 333         css_put(&mm->mem_cgroup->css);
 334 }
 335
 336 static inline int page_cgroup_locked(struct page *page)
 337 {
 338 <<<<<<< HEAD:mm/memcontrol.c
 339         return bit_spin_is_locked(PAGE_CGROUP_LOCK_BIT,
 340                                         &page->page_cgroup);
 341 =======
 342         return bit_spin_is_locked(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
 343 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 344 }
 345
 346 <<<<<<< HEAD:mm/memcontrol.c
 347 void page_assign_page_cgroup(struct page *page, struct page_cgroup *pc)
 348 =======
 349 static void page_assign_page_cgroup(struct page *page, struct page_cgroup *pc)
 350 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 351 {
 352 <<<<<<< HEAD:mm/memcontrol.c
 353         int locked;
 354
 355         /*
 356          * While resetting the page_cgroup we might not hold the
 357          * page_cgroup lock. free_hot_cold_page() is an example
 358          * of such a scenario
 359          */
 360         if (pc)
 361                 VM_BUG_ON(!page_cgroup_locked(page));
 362         locked = (page->page_cgroup & PAGE_CGROUP_LOCK);
 363         page->page_cgroup = ((unsigned long)pc | locked);
 364 =======
 365         VM_BUG_ON(!page_cgroup_locked(page));
 366         page->page_cgroup = ((unsigned long)pc | PAGE_CGROUP_LOCK);
 367 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 368 }
 369
 370 struct page_cgroup *page_get_page_cgroup(struct page *page)
 371 {
 372 <<<<<<< HEAD:mm/memcontrol.c
 373         return (struct page_cgroup *)
 374                 (page->page_cgroup & ~PAGE_CGROUP_LOCK);
 375 =======
 376         return (struct page_cgroup *) (page->page_cgroup & ~PAGE_CGROUP_LOCK);
 377 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 378 }
 379
 380 <<<<<<< HEAD:mm/memcontrol.c
 381 static void __always_inline lock_page_cgroup(struct page *page)
 382 =======
 383 static void lock_page_cgroup(struct page *page)
 384 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 385 {
 386         bit_spin_lock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
 387 <<<<<<< HEAD:mm/memcontrol.c
 388         VM_BUG_ON(!page_cgroup_locked(page));
 389 }
 390
 391 static void __always_inline unlock_page_cgroup(struct page *page)
 392 {
 393         bit_spin_unlock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
 394 =======
 395 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 396 }
 397
 398 <<<<<<< HEAD:mm/memcontrol.c
 399 /*
 400  * Tie new page_cgroup to struct page under lock_page_cgroup()
 401  * This can fail if the page has been tied to a page_cgroup.
 402  * If success, returns 0.
 403  */
 404 static int page_cgroup_assign_new_page_cgroup(struct page *page,
 405                                                 struct page_cgroup *pc)
 406 =======
 407 static int try_lock_page_cgroup(struct page *page)
 408 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 409 {
 410 <<<<<<< HEAD:mm/memcontrol.c
 411         int ret = 0;
 412
 413         lock_page_cgroup(page);
 414         if (!page_get_page_cgroup(page))
 415                 page_assign_page_cgroup(page, pc);
 416         else /* A page is tied to other pc. */
 417                 ret = 1;
 418         unlock_page_cgroup(page);
 419         return ret;
 420 =======
 421         return bit_spin_trylock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
 422 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 423 }
 424
 425 <<<<<<< HEAD:mm/memcontrol.c
 426 /*
 427  * Clear page->page_cgroup member under lock_page_cgroup().
 428  * If given "pc" value is different from one page->page_cgroup,
 429  * page->cgroup is not cleared.
 430  * Returns a value of page->page_cgroup at lock taken.
 431  * A can can detect failure of clearing by following
 432  *  clear_page_cgroup(page, pc) == pc
 433  */
 434
 435 static struct page_cgroup *clear_page_cgroup(struct page *page,
 436                                                 struct page_cgroup *pc)
 437 =======
 438 static void unlock_page_cgroup(struct page *page)
 439 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 440 {
 441 <<<<<<< HEAD:mm/memcontrol.c
 442         struct page_cgroup *ret;
 443         /* lock and clear */
 444         lock_page_cgroup(page);
 445         ret = page_get_page_cgroup(page);
 446         if (likely(ret == pc))
 447                 page_assign_page_cgroup(page, NULL);
 448         unlock_page_cgroup(page);
 449         return ret;
 450 =======
 451         bit_spin_unlock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
 452 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 453 }
 454
 455 static void __mem_cgroup_remove_list(struct page_cgroup *pc)
 456 {
 457         int from = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
 458         struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);
 459
 460         if (from)
 461                 MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) -= 1;
 462         else
 463                 MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) -= 1;
 464
 465         mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, false);
 466         list_del_init(&pc->lru);
 467 }
 468
 469 static void __mem_cgroup_add_list(struct page_cgroup *pc)
 470 {
 471         int to = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
 472         struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);
 473
 474         if (!to) {
 475                 MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) += 1;
 476                 list_add(&pc->lru, &mz->inactive_list);
 477         } else {
 478                 MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) += 1;
 479                 list_add(&pc->lru, &mz->active_list);
 480         }
 481         mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, true);
 482 }
 483
 484 static void __mem_cgroup_move_lists(struct page_cgroup *pc, bool active)
 485 {
 486         int from = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
 487         struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);
 488
 489         if (from)
 490                 MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) -= 1;
 491         else
 492                 MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) -= 1;
 493
 494         if (active) {
 495                 MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) += 1;
 496                 pc->flags |= PAGE_CGROUP_FLAG_ACTIVE;
 497                 list_move(&pc->lru, &mz->active_list);
 498         } else {
 499                 MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) += 1;
 500                 pc->flags &= ~PAGE_CGROUP_FLAG_ACTIVE;
 501                 list_move(&pc->lru, &mz->inactive_list);
 502         }
 503 }
 504
 505 int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
 506 {
 507         int ret;
 508
 509         task_lock(task);
 510 <<<<<<< HEAD:mm/memcontrol.c
 511         ret = task->mm && vm_match_cgroup(task->mm, mem);
 512 =======
 513         ret = task->mm && mm_match_cgroup(task->mm, mem);
 514 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 515         task_unlock(task);
 516         return ret;
 517 }
 518
 519 /*
 520  * This routine assumes that the appropriate zone's lru lock is already held
 521  */
 522 <<<<<<< HEAD:mm/memcontrol.c
 523 void mem_cgroup_move_lists(struct page_cgroup *pc, bool active)
 524 =======
 525 void mem_cgroup_move_lists(struct page *page, bool active)
 526 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 527 {
 528 <<<<<<< HEAD:mm/memcontrol.c
 529 =======
 530         struct page_cgroup *pc;
 531 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 532         struct mem_cgroup_per_zone *mz;
 533         unsigned long flags;
 534
 535 <<<<<<< HEAD:mm/memcontrol.c
 536         if (!pc)
 537 =======
 538         /*
 539          * We cannot lock_page_cgroup while holding zone's lru_lock,
 540          * because other holders of lock_page_cgroup can be interrupted
 541          * with an attempt to rotate_reclaimable_page.  But we cannot
 542          * safely get to page_cgroup without it, so just try_lock it:
 543          * mem_cgroup_isolate_pages allows for page left on wrong list.
 544          */
 545         if (!try_lock_page_cgroup(page))
 546 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 547                 return;
 548
 549 <<<<<<< HEAD:mm/memcontrol.c
 550         mz = page_cgroup_zoneinfo(pc);
 551         spin_lock_irqsave(&mz->lru_lock, flags);
 552         __mem_cgroup_move_lists(pc, active);
 553         spin_unlock_irqrestore(&mz->lru_lock, flags);
 554 =======
 555         pc = page_get_page_cgroup(page);
 556         if (pc) {
 557                 mz = page_cgroup_zoneinfo(pc);
 558                 spin_lock_irqsave(&mz->lru_lock, flags);
 559                 __mem_cgroup_move_lists(pc, active);
 560                 spin_unlock_irqrestore(&mz->lru_lock, flags);
 561         }
 562         unlock_page_cgroup(page);
 563 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 564 }
 565
 566 /*
 567  * Calculate mapped_ratio under memory controller. This will be used in
 568  * vmscan.c for deteremining we have to reclaim mapped pages.
 569  */
 570 int mem_cgroup_calc_mapped_ratio(struct mem_cgroup *mem)
 571 {
 572         long total, rss;
 573
 574         /*
 575          * usage is recorded in bytes. But, here, we assume the number of
 576          * physical pages can be represented by "long" on any arch.
 577          */
 578         total = (long) (mem->res.usage >> PAGE_SHIFT) + 1L;
 579         rss = (long)mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS);
 580         return (int)((rss * 100L) / total);
 581 }
 582 <<<<<<< HEAD:mm/memcontrol.c
 583 =======
 584
 585 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 586 /*
 587  * This function is called from vmscan.c. In page reclaiming loop. balance
 588  * between active and inactive list is calculated. For memory controller
 589  * page reclaiming, we should use using mem_cgroup's imbalance rather than
 590  * zone's global lru imbalance.
 591  */
 592 long mem_cgroup_reclaim_imbalance(struct mem_cgroup *mem)
 593 {
 594         unsigned long active, inactive;
 595         /* active and inactive are the number of pages. 'long' is ok.*/
 596         active = mem_cgroup_get_all_zonestat(mem, MEM_CGROUP_ZSTAT_ACTIVE);
 597         inactive = mem_cgroup_get_all_zonestat(mem, MEM_CGROUP_ZSTAT_INACTIVE);
 598         return (long) (active / (inactive + 1));
 599 }
 600
 601 /*
 602  * prev_priority control...this will be used in memory reclaim path.
 603  */
 604 int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem)
 605 {
 606         return mem->prev_priority;
 607 }
 608
 609 void mem_cgroup_note_reclaim_priority(struct mem_cgroup *mem, int priority)
 610 {
 611         if (priority < mem->prev_priority)
 612                 mem->prev_priority = priority;
 613 }
 614
 615 void mem_cgroup_record_reclaim_priority(struct mem_cgroup *mem, int priority)
 616 {
 617         mem->prev_priority = priority;
 618 }
 619
 620 /*
 621  * Calculate # of pages to be scanned in this priority/zone.
 622  * See also vmscan.c
 623  *
 624  * priority starts from "DEF_PRIORITY" and decremented in each loop.
 625  * (see include/linux/mmzone.h)
 626  */
 627
 628 long mem_cgroup_calc_reclaim_active(struct mem_cgroup *mem,
 629                                    struct zone *zone, int priority)
 630 {
 631         long nr_active;
 632         int nid = zone->zone_pgdat->node_id;
 633         int zid = zone_idx(zone);
 634         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);
 635
 636         nr_active = MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE);
 637         return (nr_active >> priority);
 638 }
 639
 640 long mem_cgroup_calc_reclaim_inactive(struct mem_cgroup *mem,
 641                                         struct zone *zone, int priority)
 642 {
 643         long nr_inactive;
 644         int nid = zone->zone_pgdat->node_id;
 645         int zid = zone_idx(zone);
 646         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);
 647
 648         nr_inactive = MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE);
 649 <<<<<<< HEAD:mm/memcontrol.c
 650
 651 =======
 652 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 653         return (nr_inactive >> priority);
 654 }
 655
 656 unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
 657                                         struct list_head *dst,
 658                                         unsigned long *scanned, int order,
 659                                         int mode, struct zone *z,
 660                                         struct mem_cgroup *mem_cont,
 661                                         int active)
 662 {
 663         unsigned long nr_taken = 0;
 664         struct page *page;
 665         unsigned long scan;
 666         LIST_HEAD(pc_list);
 667         struct list_head *src;
 668         struct page_cgroup *pc, *tmp;
 669         int nid = z->zone_pgdat->node_id;
 670         int zid = zone_idx(z);
 671         struct mem_cgroup_per_zone *mz;
 672
 673         mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
 674         if (active)
 675                 src = &mz->active_list;
 676         else
 677                 src = &mz->inactive_list;
 678
 679
 680         spin_lock(&mz->lru_lock);
 681         scan = 0;
 682         list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
 683                 if (scan >= nr_to_scan)
 684                         break;
 685                 page = pc->page;
 686 <<<<<<< HEAD:mm/memcontrol.c
 687                 VM_BUG_ON(!pc);
 688 =======
 689 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 690
 691                 if (unlikely(!PageLRU(page)))
 692                         continue;
 693
 694                 if (PageActive(page) && !active) {
 695                         __mem_cgroup_move_lists(pc, true);
 696                         continue;
 697                 }
 698                 if (!PageActive(page) && active) {
 699                         __mem_cgroup_move_lists(pc, false);
 700                         continue;
 701                 }
 702
 703                 scan++;
 704                 list_move(&pc->lru, &pc_list);
 705
 706                 if (__isolate_lru_page(page, mode) == 0) {
 707                         list_move(&page->lru, dst);
 708                         nr_taken++;
 709                 }
 710         }
 711
 712         list_splice(&pc_list, src);
 713         spin_unlock(&mz->lru_lock);
 714
 715         *scanned = scan;
 716         return nr_taken;
 717 }
 718
 719 /*
 720  * Charge the memory controller for page usage.
 721  * Return
 722  * 0 if the charge was successful
 723  * < 0 if the cgroup is over its limit
 724  */
 725 static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
 726                                 gfp_t gfp_mask, enum charge_type ctype)
 727 {
 728         struct mem_cgroup *mem;
 729         struct page_cgroup *pc;
 730         unsigned long flags;
 731         unsigned long nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
 732         struct mem_cgroup_per_zone *mz;
 733
 734         /*
 735          * Should page_cgroup's go to their own slab?
 736          * One could optimize the performance of the charging routine
 737          * by saving a bit in the page_flags and using it as a lock
 738          * to see if the cgroup page already has a page_cgroup associated
 739          * with it
 740          */
 741 retry:
 742 <<<<<<< HEAD:mm/memcontrol.c
 743         if (page) {
 744                 lock_page_cgroup(page);
 745                 pc = page_get_page_cgroup(page);
 746                 /*
 747                  * The page_cgroup exists and
 748                  * the page has already been accounted.
 749                  */
 750                 if (pc) {
 751                         if (unlikely(!atomic_inc_not_zero(&pc->ref_cnt))) {
 752                                 /* this page is under being uncharged ? */
 753                                 unlock_page_cgroup(page);
 754                                 cpu_relax();
 755                                 goto retry;
 756                         } else {
 757                                 unlock_page_cgroup(page);
 758                                 goto done;
 759                         }
 760                 }
 761 =======
 762         lock_page_cgroup(page);
 763         pc = page_get_page_cgroup(page);
 764         /*
 765          * The page_cgroup exists and
 766          * the page has already been accounted.
 767          */
 768         if (pc) {
 769                 VM_BUG_ON(pc->page != page);
 770                 VM_BUG_ON(pc->ref_cnt <= 0);
 771
 772                 pc->ref_cnt++;
 773 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 774                 unlock_page_cgroup(page);
 775 <<<<<<< HEAD:mm/memcontrol.c
 776 =======
 777                 goto done;
 778 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 779         }
 780 <<<<<<< HEAD:mm/memcontrol.c
 781 =======
 782         unlock_page_cgroup(page);
 783 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 784
 785         pc = kzalloc(sizeof(struct page_cgroup), gfp_mask);
 786         if (pc == NULL)
 787                 goto err;
 788
 789         /*
 790          * We always charge the cgroup the mm_struct belongs to.
 791          * The mm_struct's mem_cgroup changes on task migration if the
 792          * thread group leader migrates. It's possible that mm is not
 793          * set, if so charge the init_mm (happens for pagecache usage).
 794          */
 795         if (!mm)
 796                 mm = &init_mm;
 797
 798         rcu_read_lock();
 799         mem = rcu_dereference(mm->mem_cgroup);
 800         /*
 801 <<<<<<< HEAD:mm/memcontrol.c
 802          * For every charge from the cgroup, increment reference
 803          * count
 804 =======
 805          * For every charge from the cgroup, increment reference count
 806 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 807          */
 808         css_get(&mem->css);
 809         rcu_read_unlock();
 810
 811 <<<<<<< HEAD:mm/memcontrol.c
 812         /*
 813          * If we created the page_cgroup, we should free it on exceeding
 814          * the cgroup limit.
 815          */
 816 =======
 817 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 818         while (res_counter_charge(&mem->res, PAGE_SIZE)) {
 819                 if (!(gfp_mask & __GFP_WAIT))
 820                         goto out;
 821
 822                 if (try_to_free_mem_cgroup_pages(mem, gfp_mask))
 823                         continue;
 824
 825                 /*
 826 <<<<<<< HEAD:mm/memcontrol.c
 827                  * try_to_free_mem_cgroup_pages() might not give us a full
 828                  * picture of reclaim. Some pages are reclaimed and might be
 829                  * moved to swap cache or just unmapped from the cgroup.
 830                  * Check the limit again to see if the reclaim reduced the
 831                  * current usage of the cgroup before giving up
 832                  */
 833 =======
 834                  * try_to_free_mem_cgroup_pages() might not give us a full
 835                  * picture of reclaim. Some pages are reclaimed and might be
 836                  * moved to swap cache or just unmapped from the cgroup.
 837                  * Check the limit again to see if the reclaim reduced the
 838                  * current usage of the cgroup before giving up
 839                  */
 840 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 841                 if (res_counter_check_under_limit(&mem->res))
 842                         continue;
 843
 844                 if (!nr_retries--) {
 845                         mem_cgroup_out_of_memory(mem, gfp_mask);
 846                         goto out;
 847                 }
 848                 congestion_wait(WRITE, HZ/10);
 849         }
 850
 851 <<<<<<< HEAD:mm/memcontrol.c
 852         atomic_set(&pc->ref_cnt, 1);
 853 =======
 854         pc->ref_cnt = 1;
 855 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 856         pc->mem_cgroup = mem;
 857         pc->page = page;
 858         pc->flags = PAGE_CGROUP_FLAG_ACTIVE;
 859         if (ctype == MEM_CGROUP_CHARGE_TYPE_CACHE)
 860                 pc->flags |= PAGE_CGROUP_FLAG_CACHE;
 861
 862 <<<<<<< HEAD:mm/memcontrol.c
 863         if (!page || page_cgroup_assign_new_page_cgroup(page, pc)) {
 864 =======
 865         lock_page_cgroup(page);
 866         if (page_get_page_cgroup(page)) {
 867                 unlock_page_cgroup(page);
 868 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 869                 /*
 870                  * Another charge has been added to this page already.
 871                  * We take lock_page_cgroup(page) again and read
 872                  * page->cgroup, increment refcnt.... just retry is OK.
 873                  */
 874                 res_counter_uncharge(&mem->res, PAGE_SIZE);
 875                 css_put(&mem->css);
 876                 kfree(pc);
 877 <<<<<<< HEAD:mm/memcontrol.c
 878                 if (!page)
 879                         goto done;
 880 =======
 881 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 882                 goto retry;
 883         }
 884 <<<<<<< HEAD:mm/memcontrol.c
 885 =======
 886         page_assign_page_cgroup(page, pc);
 887 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 888
 889         mz = page_cgroup_zoneinfo(pc);
 890         spin_lock_irqsave(&mz->lru_lock, flags);
 891 <<<<<<< HEAD:mm/memcontrol.c
 892         /* Update statistics vector */
 893 =======
 894 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 895         __mem_cgroup_add_list(pc);
 896         spin_unlock_irqrestore(&mz->lru_lock, flags);
 897
 898 <<<<<<< HEAD:mm/memcontrol.c
 899 =======
 900         unlock_page_cgroup(page);
 901 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 902 done:
 903         return 0;
 904 out:
 905         css_put(&mem->css);
 906         kfree(pc);
 907 err:
 908         return -ENOMEM;
 909 }
 910
 911 <<<<<<< HEAD:mm/memcontrol.c
 912 int mem_cgroup_charge(struct page *page, struct mm_struct *mm,
 913                         gfp_t gfp_mask)
 914 =======
 915 int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
 916 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 917 {
 918         return mem_cgroup_charge_common(page, mm, gfp_mask,
 919 <<<<<<< HEAD:mm/memcontrol.c
 920                         MEM_CGROUP_CHARGE_TYPE_MAPPED);
 921 =======
 922                                 MEM_CGROUP_CHARGE_TYPE_MAPPED);
 923 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 924 }
 925
 926 <<<<<<< HEAD:mm/memcontrol.c
 927 /*
 928  * See if the cached pages should be charged at all?
 929  */
 930 =======
 931 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 932 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
 933                                 gfp_t gfp_mask)
 934 {
 935 <<<<<<< HEAD:mm/memcontrol.c
 936         int ret = 0;
 937 =======
 938 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 939         if (!mm)
 940                 mm = &init_mm;
 941 <<<<<<< HEAD:mm/memcontrol.c
 942
 943         ret = mem_cgroup_charge_common(page, mm, gfp_mask,
 944 =======
 945         return mem_cgroup_charge_common(page, mm, gfp_mask,
 946 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 947                                 MEM_CGROUP_CHARGE_TYPE_CACHE);
 948 <<<<<<< HEAD:mm/memcontrol.c
 949         return ret;
 950 =======
 951 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 952 }
 953
 954 /*
 955  * Uncharging is always a welcome operation, we never complain, simply
 956 <<<<<<< HEAD:mm/memcontrol.c
 957  * uncharge. This routine should be called with lock_page_cgroup held
 958 =======
 959  * uncharge.
 960 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 961  */
 962 <<<<<<< HEAD:mm/memcontrol.c
 963 void mem_cgroup_uncharge(struct page_cgroup *pc)
 964 =======
 965 void mem_cgroup_uncharge_page(struct page *page)
 966 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 967 {
 968 <<<<<<< HEAD:mm/memcontrol.c
 969 =======
 970         struct page_cgroup *pc;
 971 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 972         struct mem_cgroup *mem;
 973         struct mem_cgroup_per_zone *mz;
 974 <<<<<<< HEAD:mm/memcontrol.c
 975         struct page *page;
 976 =======
 977 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 978         unsigned long flags;
 979
 980         /*
 981          * Check if our page_cgroup is valid
 982          */
 983 <<<<<<< HEAD:mm/memcontrol.c
 984 =======
 985         lock_page_cgroup(page);
 986         pc = page_get_page_cgroup(page);
 987 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 988         if (!pc)
 989 <<<<<<< HEAD:mm/memcontrol.c
 990                 return;
 991 =======
 992                 goto unlock;
 993 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
 994
 995 <<<<<<< HEAD:mm/memcontrol.c
 996         if (atomic_dec_and_test(&pc->ref_cnt)) {
 997                 page = pc->page;
 998 =======
 999         VM_BUG_ON(pc->page != page);
1000         VM_BUG_ON(pc->ref_cnt <= 0);
1001
1002         if (--(pc->ref_cnt) == 0) {
1003 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1004                 mz = page_cgroup_zoneinfo(pc);
1005 <<<<<<< HEAD:mm/memcontrol.c
1006                 /*
1007                  * get page->cgroup and clear it under lock.
1008                  * force_empty can drop page->cgroup without checking refcnt.
1009                  */
1010 =======
1011                 spin_lock_irqsave(&mz->lru_lock, flags);
1012                 __mem_cgroup_remove_list(pc);
1013                 spin_unlock_irqrestore(&mz->lru_lock, flags);
1014
1015                 page_assign_page_cgroup(page, NULL);
1016 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1017                 unlock_page_cgroup(page);
1018 <<<<<<< HEAD:mm/memcontrol.c
1019                 if (clear_page_cgroup(page, pc) == pc) {
1020                         mem = pc->mem_cgroup;
1021                         css_put(&mem->css);
1022                         res_counter_uncharge(&mem->res, PAGE_SIZE);
1023                         spin_lock_irqsave(&mz->lru_lock, flags);
1024                         __mem_cgroup_remove_list(pc);
1025                         spin_unlock_irqrestore(&mz->lru_lock, flags);
1026                         kfree(pc);
1027                 }
1028                 lock_page_cgroup(page);
1029 =======
1030
1031                 mem = pc->mem_cgroup;
1032                 res_counter_uncharge(&mem->res, PAGE_SIZE);
1033                 css_put(&mem->css);
1034
1035                 kfree(pc);
1036                 return;
1037 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1038         }
1039 <<<<<<< HEAD:mm/memcontrol.c
1040 }
1041 =======
1042 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1043
1044 <<<<<<< HEAD:mm/memcontrol.c
1045 void mem_cgroup_uncharge_page(struct page *page)
1046 {
1047         lock_page_cgroup(page);
1048         mem_cgroup_uncharge(page_get_page_cgroup(page));
1049 =======
1050 unlock:
1051 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1052         unlock_page_cgroup(page);
1053 }
1054
1055 /*
1056  * Returns non-zero if a page (under migration) has valid page_cgroup member.
1057  * Refcnt of page_cgroup is incremented.
1058  */
1059 <<<<<<< HEAD:mm/memcontrol.c
1060
1061 =======
1062 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1063 int mem_cgroup_prepare_migration(struct page *page)
1064 {
1065         struct page_cgroup *pc;
1066 <<<<<<< HEAD:mm/memcontrol.c
1067         int ret = 0;
1068 =======
1069
1070 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1071         lock_page_cgroup(page);
1072         pc = page_get_page_cgroup(page);
1073 <<<<<<< HEAD:mm/memcontrol.c
1074         if (pc && atomic_inc_not_zero(&pc->ref_cnt))
1075                 ret = 1;
1076 =======
1077         if (pc)
1078                 pc->ref_cnt++;
1079 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1080         unlock_page_cgroup(page);
1081 <<<<<<< HEAD:mm/memcontrol.c
1082         return ret;
1083 =======
1084         return pc != NULL;
1085 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1086 }
1087
1088 void mem_cgroup_end_migration(struct page *page)
1089 {
1090 <<<<<<< HEAD:mm/memcontrol.c
1091         struct page_cgroup *pc;
1092
1093         lock_page_cgroup(page);
1094         pc = page_get_page_cgroup(page);
1095         mem_cgroup_uncharge(pc);
1096         unlock_page_cgroup(page);
1097 =======
1098         mem_cgroup_uncharge_page(page);
1099 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1100 }
1101 <<<<<<< HEAD:mm/memcontrol.c
1102 =======
1103
1104 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1105 /*
1106 <<<<<<< HEAD:mm/memcontrol.c
1107  * We know both *page* and *newpage* are now not-on-LRU and Pg_locked.
1108 =======
1109  * We know both *page* and *newpage* are now not-on-LRU and PG_locked.
1110 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1111  * And no race with uncharge() routines because page_cgroup for *page*
1112  * has extra one reference by mem_cgroup_prepare_migration.
1113  */
1114 <<<<<<< HEAD:mm/memcontrol.c
1115
1116 =======
1117 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1118 void mem_cgroup_page_migration(struct page *page, struct page *newpage)
1119 {
1120         struct page_cgroup *pc;
1121 <<<<<<< HEAD:mm/memcontrol.c
1122         struct mem_cgroup *mem;
1123         unsigned long flags;
1124 =======
1125 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1126         struct mem_cgroup_per_zone *mz;
1127 <<<<<<< HEAD:mm/memcontrol.c
1128 retry:
1129 =======
1130         unsigned long flags;
1131
1132         lock_page_cgroup(page);
1133 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1134         pc = page_get_page_cgroup(page);
1135 <<<<<<< HEAD:mm/memcontrol.c
1136         if (!pc)
1137 =======
1138         if (!pc) {
1139                 unlock_page_cgroup(page);
1140 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1141                 return;
1142 <<<<<<< HEAD:mm/memcontrol.c
1143         mem = pc->mem_cgroup;
1144 =======
1145         }
1146
1147 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1148         mz = page_cgroup_zoneinfo(pc);
1149 <<<<<<< HEAD:mm/memcontrol.c
1150         if (clear_page_cgroup(page, pc) != pc)
1151                 goto retry;
1152 =======
1153 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1154         spin_lock_irqsave(&mz->lru_lock, flags);
1155 <<<<<<< HEAD:mm/memcontrol.c
1156
1157 =======
1158 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1159         __mem_cgroup_remove_list(pc);
1160         spin_unlock_irqrestore(&mz->lru_lock, flags);
1161
1162 <<<<<<< HEAD:mm/memcontrol.c
1163 =======
1164         page_assign_page_cgroup(page, NULL);
1165         unlock_page_cgroup(page);
1166
1167 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1168         pc->page = newpage;
1169         lock_page_cgroup(newpage);
1170         page_assign_page_cgroup(newpage, pc);
1171 <<<<<<< HEAD:mm/memcontrol.c
1172         unlock_page_cgroup(newpage);
1173 =======
1174 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1175
1176         mz = page_cgroup_zoneinfo(pc);
1177         spin_lock_irqsave(&mz->lru_lock, flags);
1178         __mem_cgroup_add_list(pc);
1179         spin_unlock_irqrestore(&mz->lru_lock, flags);
1180 <<<<<<< HEAD:mm/memcontrol.c
1181         return;
1182 =======
1183
1184         unlock_page_cgroup(newpage);
1185 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1186 }
1187
1188 /*
1189  * This routine traverse page_cgroup in given list and drop them all.
1190  * This routine ignores page_cgroup->ref_cnt.
1191  * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
1192  */
1193 #define FORCE_UNCHARGE_BATCH    (128)
1194 <<<<<<< HEAD:mm/memcontrol.c
1195 static void
1196 mem_cgroup_force_empty_list(struct mem_cgroup *mem,
1197 =======
1198 static void mem_cgroup_force_empty_list(struct mem_cgroup *mem,
1199 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1200                             struct mem_cgroup_per_zone *mz,
1201                             int active)
1202 {
1203         struct page_cgroup *pc;
1204         struct page *page;
1205 <<<<<<< HEAD:mm/memcontrol.c
1206         int count;
1207 =======
1208         int count = FORCE_UNCHARGE_BATCH;
1209 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1210         unsigned long flags;
1211         struct list_head *list;
1212
1213         if (active)
1214                 list = &mz->active_list;
1215         else
1216                 list = &mz->inactive_list;
1217
1218 <<<<<<< HEAD:mm/memcontrol.c
1219         if (list_empty(list))
1220                 return;
1221 retry:
1222         count = FORCE_UNCHARGE_BATCH;
1223 =======
1224 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1225         spin_lock_irqsave(&mz->lru_lock, flags);
1226 <<<<<<< HEAD:mm/memcontrol.c
1227
1228         while (--count && !list_empty(list)) {
1229 =======
1230         while (!list_empty(list)) {
1231 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1232                 pc = list_entry(list->prev, struct page_cgroup, lru);
1233                 page = pc->page;
1234 <<<<<<< HEAD:mm/memcontrol.c
1235                 /* Avoid race with charge */
1236                 atomic_set(&pc->ref_cnt, 0);
1237                 if (clear_page_cgroup(page, pc) == pc) {
1238                         css_put(&mem->css);
1239                         res_counter_uncharge(&mem->res, PAGE_SIZE);
1240                         __mem_cgroup_remove_list(pc);
1241                         kfree(pc);
1242                 } else  /* being uncharged ? ...do relax */
1243                         break;
1244 =======
1245                 get_page(page);
1246                 spin_unlock_irqrestore(&mz->lru_lock, flags);
1247                 mem_cgroup_uncharge_page(page);
1248                 put_page(page);
1249                 if (--count <= 0) {
1250                         count = FORCE_UNCHARGE_BATCH;
1251                         cond_resched();
1252                 }
1253                 spin_lock_irqsave(&mz->lru_lock, flags);
1254 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1255         }
1256         spin_unlock_irqrestore(&mz->lru_lock, flags);
1257 <<<<<<< HEAD:mm/memcontrol.c
1258         if (!list_empty(list)) {
1259                 cond_resched();
1260                 goto retry;
1261         }
1262         return;
1263 =======
1264 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1265 }
1266
1267 /*
1268  * make mem_cgroup's charge to be 0 if there is no task.
1269  * This enables deleting this mem_cgroup.
1270  */
1271 <<<<<<< HEAD:mm/memcontrol.c
1272
1273 int mem_cgroup_force_empty(struct mem_cgroup *mem)
1274 =======
1275 static int mem_cgroup_force_empty(struct mem_cgroup *mem)
1276 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1277 {
1278         int ret = -EBUSY;
1279         int node, zid;
1280 <<<<<<< HEAD:mm/memcontrol.c
1281 =======
1282
1283 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1284         css_get(&mem->css);
1285         /*
1286          * page reclaim code (kswapd etc..) will move pages between
1287 <<<<<<< HEAD:mm/memcontrol.c
1288 `        * active_list <-> inactive_list while we don't take a lock.
1289 =======
1290          * active_list <-> inactive_list while we don't take a lock.
1291 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1292          * So, we have to do loop here until all lists are empty.
1293          */
1294         while (mem->res.usage > 0) {
1295                 if (atomic_read(&mem->css.cgroup->count) > 0)
1296                         goto out;
1297                 for_each_node_state(node, N_POSSIBLE)
1298                         for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1299                                 struct mem_cgroup_per_zone *mz;
1300                                 mz = mem_cgroup_zoneinfo(mem, node, zid);
1301                                 /* drop all page_cgroup in active_list */
1302                                 mem_cgroup_force_empty_list(mem, mz, 1);
1303                                 /* drop all page_cgroup in inactive_list */
1304                                 mem_cgroup_force_empty_list(mem, mz, 0);
1305                         }
1306         }
1307         ret = 0;
1308 out:
1309         css_put(&mem->css);
1310         return ret;
1311 }
1312
1313 <<<<<<< HEAD:mm/memcontrol.c
1314
1315
1316 int mem_cgroup_write_strategy(char *buf, unsigned long long *tmp)
1317 =======
1318 static int mem_cgroup_write_strategy(char *buf, unsigned long long *tmp)
1319 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1320 {
1321         *tmp = memparse(buf, &buf);
1322         if (*buf != '\0')
1323                 return -EINVAL;
1324
1325         /*
1326          * Round up the value to the closest page size
1327          */
1328         *tmp = ((*tmp + PAGE_SIZE - 1) >> PAGE_SHIFT) << PAGE_SHIFT;
1329         return 0;
1330 }
1331
1332 static ssize_t mem_cgroup_read(struct cgroup *cont,
1333                         struct cftype *cft, struct file *file,
1334                         char __user *userbuf, size_t nbytes, loff_t *ppos)
1335 {
1336         return res_counter_read(&mem_cgroup_from_cont(cont)->res,
1337                                 cft->private, userbuf, nbytes, ppos,
1338                                 NULL);
1339 }
1340
1341 static ssize_t mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
1342                                 struct file *file, const char __user *userbuf,
1343                                 size_t nbytes, loff_t *ppos)
1344 {
1345         return res_counter_write(&mem_cgroup_from_cont(cont)->res,
1346                                 cft->private, userbuf, nbytes, ppos,
1347                                 mem_cgroup_write_strategy);
1348 }
1349
1350 static ssize_t mem_force_empty_write(struct cgroup *cont,
1351                                 struct cftype *cft, struct file *file,
1352                                 const char __user *userbuf,
1353                                 size_t nbytes, loff_t *ppos)
1354 {
1355         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
1356 <<<<<<< HEAD:mm/memcontrol.c
1357         int ret;
1358         ret = mem_cgroup_force_empty(mem);
1359 =======
1360         int ret = mem_cgroup_force_empty(mem);
1361 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1362         if (!ret)
1363                 ret = nbytes;
1364         return ret;
1365 }
1366
1367 /*
1368  * Note: This should be removed if cgroup supports write-only file.
1369  */
1370 <<<<<<< HEAD:mm/memcontrol.c
1371
1372 =======
1373 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1374 static ssize_t mem_force_empty_read(struct cgroup *cont,
1375                                 struct cftype *cft,
1376                                 struct file *file, char __user *userbuf,
1377                                 size_t nbytes, loff_t *ppos)
1378 {
1379         return -EINVAL;
1380 }
1381
1382 <<<<<<< HEAD:mm/memcontrol.c
1383
1384 =======
1385 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1386 static const struct mem_cgroup_stat_desc {
1387         const char *msg;
1388         u64 unit;
1389 } mem_cgroup_stat_desc[] = {
1390         [MEM_CGROUP_STAT_CACHE] = { "cache", PAGE_SIZE, },
1391         [MEM_CGROUP_STAT_RSS] = { "rss", PAGE_SIZE, },
1392 };
1393
1394 static int mem_control_stat_show(struct seq_file *m, void *arg)
1395 {
1396         struct cgroup *cont = m->private;
1397         struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
1398         struct mem_cgroup_stat *stat = &mem_cont->stat;
1399         int i;
1400
1401         for (i = 0; i < ARRAY_SIZE(stat->cpustat[0].count); i++) {
1402                 s64 val;
1403
1404                 val = mem_cgroup_read_stat(stat, i);
1405                 val *= mem_cgroup_stat_desc[i].unit;
1406                 seq_printf(m, "%s %lld\n", mem_cgroup_stat_desc[i].msg,
1407                                 (long long)val);
1408         }
1409         /* showing # of active pages */
1410         {
1411                 unsigned long active, inactive;
1412
1413                 inactive = mem_cgroup_get_all_zonestat(mem_cont,
1414                                                 MEM_CGROUP_ZSTAT_INACTIVE);
1415                 active = mem_cgroup_get_all_zonestat(mem_cont,
1416                                                 MEM_CGROUP_ZSTAT_ACTIVE);
1417                 seq_printf(m, "active %ld\n", (active) * PAGE_SIZE);
1418                 seq_printf(m, "inactive %ld\n", (inactive) * PAGE_SIZE);
1419         }
1420         return 0;
1421 }
1422
1423 static const struct file_operations mem_control_stat_file_operations = {
1424         .read = seq_read,
1425         .llseek = seq_lseek,
1426         .release = single_release,
1427 };
1428
1429 static int mem_control_stat_open(struct inode *unused, struct file *file)
1430 {
1431         /* XXX __d_cont */
1432         struct cgroup *cont = file->f_dentry->d_parent->d_fsdata;
1433
1434         file->f_op = &mem_control_stat_file_operations;
1435         return single_open(file, mem_control_stat_show, cont);
1436 }
1437
1438 <<<<<<< HEAD:mm/memcontrol.c
1439
1440
1441 =======
1442 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1443 static struct cftype mem_cgroup_files[] = {
1444         {
1445                 .name = "usage_in_bytes",
1446                 .private = RES_USAGE,
1447                 .read = mem_cgroup_read,
1448         },
1449         {
1450                 .name = "limit_in_bytes",
1451                 .private = RES_LIMIT,
1452                 .write = mem_cgroup_write,
1453                 .read = mem_cgroup_read,
1454         },
1455         {
1456                 .name = "failcnt",
1457                 .private = RES_FAILCNT,
1458                 .read = mem_cgroup_read,
1459         },
1460         {
1461                 .name = "force_empty",
1462                 .write = mem_force_empty_write,
1463                 .read = mem_force_empty_read,
1464         },
1465         {
1466                 .name = "stat",
1467                 .open = mem_control_stat_open,
1468         },
1469 };
1470
1471 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
1472 {
1473         struct mem_cgroup_per_node *pn;
1474         struct mem_cgroup_per_zone *mz;
1475         int zone;
1476         /*
1477          * This routine is called against possible nodes.
1478          * But it's BUG to call kmalloc() against offline node.
1479          *
1480          * TODO: this routine can waste much memory for nodes which will
1481          *       never be onlined. It's better to use memory hotplug callback
1482          *       function.
1483          */
1484         if (node_state(node, N_HIGH_MEMORY))
1485                 pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, node);
1486         else
1487                 pn = kmalloc(sizeof(*pn), GFP_KERNEL);
1488         if (!pn)
1489                 return 1;
1490
1491         mem->info.nodeinfo[node] = pn;
1492         memset(pn, 0, sizeof(*pn));
1493
1494         for (zone = 0; zone < MAX_NR_ZONES; zone++) {
1495                 mz = &pn->zoneinfo[zone];
1496                 INIT_LIST_HEAD(&mz->active_list);
1497                 INIT_LIST_HEAD(&mz->inactive_list);
1498                 spin_lock_init(&mz->lru_lock);
1499         }
1500         return 0;
1501 }
1502
1503 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
1504 {
1505         kfree(mem->info.nodeinfo[node]);
1506 }
1507
1508 <<<<<<< HEAD:mm/memcontrol.c
1509
1510 static struct mem_cgroup init_mem_cgroup;
1511
1512 =======
1513 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1514 static struct cgroup_subsys_state *
1515 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
1516 {
1517         struct mem_cgroup *mem;
1518         int node;
1519
1520         if (unlikely((cont->parent) == NULL)) {
1521                 mem = &init_mem_cgroup;
1522                 init_mm.mem_cgroup = mem;
1523         } else
1524                 mem = kzalloc(sizeof(struct mem_cgroup), GFP_KERNEL);
1525
1526         if (mem == NULL)
1527 <<<<<<< HEAD:mm/memcontrol.c
1528                 return NULL;
1529 =======
1530                 return ERR_PTR(-ENOMEM);
1531 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1532
1533         res_counter_init(&mem->res);
1534
1535         memset(&mem->info, 0, sizeof(mem->info));
1536
1537         for_each_node_state(node, N_POSSIBLE)
1538                 if (alloc_mem_cgroup_per_zone_info(mem, node))
1539                         goto free_out;
1540
1541         return &mem->css;
1542 free_out:
1543         for_each_node_state(node, N_POSSIBLE)
1544                 free_mem_cgroup_per_zone_info(mem, node);
1545         if (cont->parent != NULL)
1546                 kfree(mem);
1547 <<<<<<< HEAD:mm/memcontrol.c
1548         return NULL;
1549 =======
1550         return ERR_PTR(-ENOMEM);
1551 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1552 }
1553
1554 static void mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
1555                                         struct cgroup *cont)
1556 {
1557         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
1558         mem_cgroup_force_empty(mem);
1559 }
1560
1561 static void mem_cgroup_destroy(struct cgroup_subsys *ss,
1562                                 struct cgroup *cont)
1563 {
1564         int node;
1565         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
1566
1567         for_each_node_state(node, N_POSSIBLE)
1568                 free_mem_cgroup_per_zone_info(mem, node);
1569
1570         kfree(mem_cgroup_from_cont(cont));
1571 }
1572
1573 static int mem_cgroup_populate(struct cgroup_subsys *ss,
1574                                 struct cgroup *cont)
1575 {
1576         return cgroup_add_files(cont, ss, mem_cgroup_files,
1577                                         ARRAY_SIZE(mem_cgroup_files));
1578 }
1579
1580 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
1581                                 struct cgroup *cont,
1582                                 struct cgroup *old_cont,
1583                                 struct task_struct *p)
1584 {
1585         struct mm_struct *mm;
1586         struct mem_cgroup *mem, *old_mem;
1587
1588         mm = get_task_mm(p);
1589         if (mm == NULL)
1590                 return;
1591
1592         mem = mem_cgroup_from_cont(cont);
1593         old_mem = mem_cgroup_from_cont(old_cont);
1594
1595         if (mem == old_mem)
1596                 goto out;
1597
1598         /*
1599          * Only thread group leaders are allowed to migrate, the mm_struct is
1600          * in effect owned by the leader
1601          */
1602         if (p->tgid != p->pid)
1603                 goto out;
1604
1605         css_get(&mem->css);
1606         rcu_assign_pointer(mm->mem_cgroup, mem);
1607         css_put(&old_mem->css);
1608
1609 out:
1610         mmput(mm);
1611 <<<<<<< HEAD:mm/memcontrol.c
1612         return;
1613 =======
1614 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:mm/memcontrol.c
1615 }
1616
1617 struct cgroup_subsys mem_cgroup_subsys = {
1618         .name = "memory",
1619         .subsys_id = mem_cgroup_subsys_id,
1620         .create = mem_cgroup_create,
1621         .pre_destroy = mem_cgroup_pre_destroy,
1622         .destroy = mem_cgroup_destroy,
1623         .populate = mem_cgroup_populate,
1624         .attach = mem_cgroup_move_task,
1625         .early_init = 0,
1626 };