mm/swap.c

   1 /*
   2  *  linux/mm/swap.c
   3  *
   4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   5  */
   6
   7 /*
   8  * This file contains the default values for the operation of the
   9  * Linux VM subsystem. Fine-tuning documentation can be found in
  10  * Documentation/sysctl/vm.txt.
  11  * Started 18.12.91
  12  * Swap aging added 23.2.95, Stephen Tweedie.
  13  * Buffermem limits added 12.3.98, Rik van Riel.
  14  */
  15
  16 #include <linux/mm.h>
  17 #include <linux/sched.h>
  18 #include <linux/kernel_stat.h>
  19 #include <linux/swap.h>
  20 #include <linux/mman.h>
  21 #include <linux/pagemap.h>
  22 #include <linux/pagevec.h>
  23 #include <linux/init.h>
  24 #include <linux/export.h>
  25 #include <linux/mm_inline.h>
  26 #include <linux/percpu_counter.h>
  27 #include <linux/percpu.h>
  28 #include <linux/cpu.h>
  29 #include <linux/notifier.h>
  30 #include <linux/backing-dev.h>
  31 #include <linux/memcontrol.h>
  32 #include <linux/gfp.h>
  33
  34 #include "internal.h"
  35
  36 /* How many pages do we try to swap or page in/out together? */
  37 int page_cluster;
  38
  39 static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
  40 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
  41 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
  42
  43 /*
  44  * This path almost never happens for VM activity - pages are normally
  45  * freed via pagevecs.  But it gets used by networking.
  46  */
  47 static void __page_cache_release(struct page *page)
  48 {
  49         if (PageLRU(page)) {
  50                 unsigned long flags;
  51                 struct zone *zone = page_zone(page);
  52
  53                 spin_lock_irqsave(&zone->lru_lock, flags);
  54                 VM_BUG_ON(!PageLRU(page));
  55                 __ClearPageLRU(page);
  56                 del_page_from_lru_list(zone, page, page_off_lru(page));
  57                 spin_unlock_irqrestore(&zone->lru_lock, flags);
  58         }
  59 }
  60
  61 static void __put_single_page(struct page *page)
  62 {
  63         __page_cache_release(page);
  64         free_hot_cold_page(page, 0);
  65 }
  66
  67 static void __put_compound_page(struct page *page)
  68 {
  69         compound_page_dtor *dtor;
  70
  71         __page_cache_release(page);
  72         dtor = get_compound_page_dtor(page);
  73         (*dtor)(page);
  74 }
  75
  76 static void put_compound_page(struct page *page)
  77 {
  78         if (unlikely(PageTail(page))) {
  79                 /* __split_huge_page_refcount can run under us */
  80                 struct page *page_head = compound_trans_head(page);
  81
  82                 if (likely(page != page_head &&
  83                            get_page_unless_zero(page_head))) {
  84                         unsigned long flags;
  85                         /*
  86                          * page_head wasn't a dangling pointer but it
  87                          * may not be a head page anymore by the time
  88                          * we obtain the lock. That is ok as long as it
  89                          * can't be freed from under us.
  90                          */
  91                         flags = compound_lock_irqsave(page_head);
  92                         if (unlikely(!PageTail(page))) {
  93                                 /* __split_huge_page_refcount run before us */
  94                                 compound_unlock_irqrestore(page_head, flags);
  95                                 VM_BUG_ON(PageHead(page_head));
  96                                 if (put_page_testzero(page_head))
  97                                         __put_single_page(page_head);
  98                         out_put_single:
  99                                 if (put_page_testzero(page))
 100                                         __put_single_page(page);
 101                                 return;
 102                         }
 103                         VM_BUG_ON(page_head != page->first_page);
 104                         /*
 105                          * We can release the refcount taken by
 106                          * get_page_unless_zero() now that
 107                          * __split_huge_page_refcount() is blocked on
 108                          * the compound_lock.
 109                          */
 110                         if (put_page_testzero(page_head))
 111                                 VM_BUG_ON(1);
 112                         /* __split_huge_page_refcount will wait now */
 113                         VM_BUG_ON(page_mapcount(page) <= 0);
 114                         atomic_dec(&page->_mapcount);
 115                         VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
 116                         VM_BUG_ON(atomic_read(&page->_count) != 0);
 117                         compound_unlock_irqrestore(page_head, flags);
 118                         if (put_page_testzero(page_head)) {
 119                                 if (PageHead(page_head))
 120                                         __put_compound_page(page_head);
 121                                 else
 122                                         __put_single_page(page_head);
 123                         }
 124                 } else {
 125                         /* page_head is a dangling pointer */
 126                         VM_BUG_ON(PageTail(page));
 127                         goto out_put_single;
 128                 }
 129         } else if (put_page_testzero(page)) {
 130                 if (PageHead(page))
 131                         __put_compound_page(page);
 132                 else
 133                         __put_single_page(page);
 134         }
 135 }
 136
 137 void put_page(struct page *page)
 138 {
 139         if (unlikely(PageCompound(page)))
 140                 put_compound_page(page);
 141         else if (put_page_testzero(page))
 142                 __put_single_page(page);
 143 }
 144 EXPORT_SYMBOL(put_page);
 145
 146 /*
 147  * This function is exported but must not be called by anything other
 148  * than get_page(). It implements the slow path of get_page().
 149  */
 150 bool __get_page_tail(struct page *page)
 151 {
 152         /*
 153          * This takes care of get_page() if run on a tail page
 154          * returned by one of the get_user_pages/follow_page variants.
 155          * get_user_pages/follow_page itself doesn't need the compound
 156          * lock because it runs __get_page_tail_foll() under the
 157          * proper PT lock that already serializes against
 158          * split_huge_page().
 159          */
 160         unsigned long flags;
 161         bool got = false;
 162         struct page *page_head = compound_trans_head(page);
 163
 164         if (likely(page != page_head && get_page_unless_zero(page_head))) {
 165                 /*
 166                  * page_head wasn't a dangling pointer but it
 167                  * may not be a head page anymore by the time
 168                  * we obtain the lock. That is ok as long as it
 169                  * can't be freed from under us.
 170                  */
 171                 flags = compound_lock_irqsave(page_head);
 172                 /* here __split_huge_page_refcount won't run anymore */
 173                 if (likely(PageTail(page))) {
 174                         __get_page_tail_foll(page, false);
 175                         got = true;
 176                 }
 177                 compound_unlock_irqrestore(page_head, flags);
 178                 if (unlikely(!got))
 179                         put_page(page_head);
 180         }
 181         return got;
 182 }
 183 EXPORT_SYMBOL(__get_page_tail);
 184
 185 /**
 186  * put_pages_list() - release a list of pages
 187  * @pages: list of pages threaded on page->lru
 188  *
 189  * Release a list of pages which are strung together on page.lru.  Currently
 190  * used by read_cache_pages() and related error recovery code.
 191  */
 192 void put_pages_list(struct list_head *pages)
 193 {
 194         while (!list_empty(pages)) {
 195                 struct page *victim;
 196
 197                 victim = list_entry(pages->prev, struct page, lru);
 198                 list_del(&victim->lru);
 199                 page_cache_release(victim);
 200         }
 201 }
 202 EXPORT_SYMBOL(put_pages_list);
 203
 204 static void pagevec_lru_move_fn(struct pagevec *pvec,
 205                                 void (*move_fn)(struct page *page, void *arg),
 206                                 void *arg)
 207 {
 208         int i;
 209         struct zone *zone = NULL;
 210         unsigned long flags = 0;
 211
 212         for (i = 0; i < pagevec_count(pvec); i++) {
 213                 struct page *page = pvec->pages[i];
 214                 struct zone *pagezone = page_zone(page);
 215
 216                 if (pagezone != zone) {
 217                         if (zone)
 218                                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 219                         zone = pagezone;
 220                         spin_lock_irqsave(&zone->lru_lock, flags);
 221                 }
 222
 223                 (*move_fn)(page, arg);
 224         }
 225         if (zone)
 226                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 227         release_pages(pvec->pages, pvec->nr, pvec->cold);
 228         pagevec_reinit(pvec);
 229 }
 230
 231 static void pagevec_move_tail_fn(struct page *page, void *arg)
 232 {
 233         int *pgmoved = arg;
 234
 235         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 236                 enum lru_list lru = page_lru_base_type(page);
 237                 struct lruvec *lruvec;
 238
 239                 lruvec = mem_cgroup_lru_move_lists(page_zone(page),
 240                                                    page, lru, lru);
 241                 list_move_tail(&page->lru, &lruvec->lists[lru]);
 242                 (*pgmoved)++;
 243         }
 244 }
 245
 246 /*
 247  * pagevec_move_tail() must be called with IRQ disabled.
 248  * Otherwise this may cause nasty races.
 249  */
 250 static void pagevec_move_tail(struct pagevec *pvec)
 251 {
 252         int pgmoved = 0;
 253
 254         pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved);
 255         __count_vm_events(PGROTATED, pgmoved);
 256 }
 257
 258 /*
 259  * Writeback is about to end against a page which has been marked for immediate
 260  * reclaim.  If it still appears to be reclaimable, move it to the tail of the
 261  * inactive list.
 262  */
 263 void rotate_reclaimable_page(struct page *page)
 264 {
 265         if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) &&
 266             !PageUnevictable(page) && PageLRU(page)) {
 267                 struct pagevec *pvec;
 268                 unsigned long flags;
 269
 270                 page_cache_get(page);
 271                 local_irq_save(flags);
 272                 pvec = &__get_cpu_var(lru_rotate_pvecs);
 273                 if (!pagevec_add(pvec, page))
 274                         pagevec_move_tail(pvec);
 275                 local_irq_restore(flags);
 276         }
 277 }
 278
 279 static void update_page_reclaim_stat(struct zone *zone, struct page *page,
 280                                      int file, int rotated)
 281 {
 282         struct zone_reclaim_stat *reclaim_stat = &zone->reclaim_stat;
 283         struct zone_reclaim_stat *memcg_reclaim_stat;
 284
 285         memcg_reclaim_stat = mem_cgroup_get_reclaim_stat_from_page(page);
 286
 287         reclaim_stat->recent_scanned[file]++;
 288         if (rotated)
 289                 reclaim_stat->recent_rotated[file]++;
 290
 291         if (!memcg_reclaim_stat)
 292                 return;
 293
 294         memcg_reclaim_stat->recent_scanned[file]++;
 295         if (rotated)
 296                 memcg_reclaim_stat->recent_rotated[file]++;
 297 }
 298
 299 static void __activate_page(struct page *page, void *arg)
 300 {
 301         struct zone *zone = page_zone(page);
 302
 303         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 304                 int file = page_is_file_cache(page);
 305                 int lru = page_lru_base_type(page);
 306                 del_page_from_lru_list(zone, page, lru);
 307
 308                 SetPageActive(page);
 309                 lru += LRU_ACTIVE;
 310                 add_page_to_lru_list(zone, page, lru);
 311                 __count_vm_event(PGACTIVATE);
 312
 313                 update_page_reclaim_stat(zone, page, file, 1);
 314         }
 315 }
 316
 317 #ifdef CONFIG_SMP
 318 static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
 319
 320 static void activate_page_drain(int cpu)
 321 {
 322         struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);
 323
 324         if (pagevec_count(pvec))
 325                 pagevec_lru_move_fn(pvec, __activate_page, NULL);
 326 }
 327
 328 void activate_page(struct page *page)
 329 {
 330         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 331                 struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
 332
 333                 page_cache_get(page);
 334                 if (!pagevec_add(pvec, page))
 335                         pagevec_lru_move_fn(pvec, __activate_page, NULL);
 336                 put_cpu_var(activate_page_pvecs);
 337         }
 338 }
 339
 340 #else
 341 static inline void activate_page_drain(int cpu)
 342 {
 343 }
 344
 345 void activate_page(struct page *page)
 346 {
 347         struct zone *zone = page_zone(page);
 348
 349         spin_lock_irq(&zone->lru_lock);
 350         __activate_page(page, NULL);
 351         spin_unlock_irq(&zone->lru_lock);
 352 }
 353 #endif
 354
 355 /*
 356  * Mark a page as having seen activity.
 357  *
 358  * inactive,unreferenced        ->      inactive,referenced
 359  * inactive,referenced          ->      active,unreferenced
 360  * active,unreferenced          ->      active,referenced
 361  */
 362 void mark_page_accessed(struct page *page)
 363 {
 364         if (!PageActive(page) && !PageUnevictable(page) &&
 365                         PageReferenced(page) && PageLRU(page)) {
 366                 activate_page(page);
 367                 ClearPageReferenced(page);
 368         } else if (!PageReferenced(page)) {
 369                 SetPageReferenced(page);
 370         }
 371 }
 372 EXPORT_SYMBOL(mark_page_accessed);
 373
 374 void __lru_cache_add(struct page *page, enum lru_list lru)
 375 {
 376         struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru];
 377
 378         page_cache_get(page);
 379         if (!pagevec_add(pvec, page))
 380                 __pagevec_lru_add(pvec, lru);
 381         put_cpu_var(lru_add_pvecs);
 382 }
 383 EXPORT_SYMBOL(__lru_cache_add);
 384
 385 /**
 386  * lru_cache_add_lru - add a page to a page list
 387  * @page: the page to be added to the LRU.
 388  * @lru: the LRU list to which the page is added.
 389  */
 390 void lru_cache_add_lru(struct page *page, enum lru_list lru)
 391 {
 392         if (PageActive(page)) {
 393                 VM_BUG_ON(PageUnevictable(page));
 394                 ClearPageActive(page);
 395         } else if (PageUnevictable(page)) {
 396                 VM_BUG_ON(PageActive(page));
 397                 ClearPageUnevictable(page);
 398         }
 399
 400         VM_BUG_ON(PageLRU(page) || PageActive(page) || PageUnevictable(page));
 401         __lru_cache_add(page, lru);
 402 }
 403
 404 /**
 405  * add_page_to_unevictable_list - add a page to the unevictable list
 406  * @page:  the page to be added to the unevictable list
 407  *
 408  * Add page directly to its zone's unevictable list.  To avoid races with
 409  * tasks that might be making the page evictable, through eg. munlock,
 410  * munmap or exit, while it's not on the lru, we want to add the page
 411  * while it's locked or otherwise "invisible" to other tasks.  This is
 412  * difficult to do when using the pagevec cache, so bypass that.
 413  */
 414 void add_page_to_unevictable_list(struct page *page)
 415 {
 416         struct zone *zone = page_zone(page);
 417
 418         spin_lock_irq(&zone->lru_lock);
 419         SetPageUnevictable(page);
 420         SetPageLRU(page);
 421         add_page_to_lru_list(zone, page, LRU_UNEVICTABLE);
 422         spin_unlock_irq(&zone->lru_lock);
 423 }
 424
 425 /*
 426  * If the page can not be invalidated, it is moved to the
 427  * inactive list to speed up its reclaim.  It is moved to the
 428  * head of the list, rather than the tail, to give the flusher
 429  * threads some time to write it out, as this is much more
 430  * effective than the single-page writeout from reclaim.
 431  *
 432  * If the page isn't page_mapped and dirty/writeback, the page
 433  * could reclaim asap using PG_reclaim.
 434  *
 435  * 1. active, mapped page -> none
 436  * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
 437  * 3. inactive, mapped page -> none
 438  * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
 439  * 5. inactive, clean -> inactive, tail
 440  * 6. Others -> none
 441  *
 442  * In 4, why it moves inactive's head, the VM expects the page would
 443  * be write it out by flusher threads as this is much more effective
 444  * than the single-page writeout from reclaim.
 445  */
 446 static void lru_deactivate_fn(struct page *page, void *arg)
 447 {
 448         int lru, file;
 449         bool active;
 450         struct zone *zone = page_zone(page);
 451
 452         if (!PageLRU(page))
 453                 return;
 454
 455         if (PageUnevictable(page))
 456                 return;
 457
 458         /* Some processes are using the page */
 459         if (page_mapped(page))
 460                 return;
 461
 462         active = PageActive(page);
 463
 464         file = page_is_file_cache(page);
 465         lru = page_lru_base_type(page);
 466         del_page_from_lru_list(zone, page, lru + active);
 467         ClearPageActive(page);
 468         ClearPageReferenced(page);
 469         add_page_to_lru_list(zone, page, lru);
 470
 471         if (PageWriteback(page) || PageDirty(page)) {
 472                 /*
 473                  * PG_reclaim could be raced with end_page_writeback
 474                  * It can make readahead confusing.  But race window
 475                  * is _really_ small and  it's non-critical problem.
 476                  */
 477                 SetPageReclaim(page);
 478         } else {
 479                 struct lruvec *lruvec;
 480                 /*
 481                  * The page's writeback ends up during pagevec
 482                  * We moves tha page into tail of inactive.
 483                  */
 484                 lruvec = mem_cgroup_lru_move_lists(zone, page, lru, lru);
 485                 list_move_tail(&page->lru, &lruvec->lists[lru]);
 486                 __count_vm_event(PGROTATED);
 487         }
 488
 489         if (active)
 490                 __count_vm_event(PGDEACTIVATE);
 491         update_page_reclaim_stat(zone, page, file, 0);
 492 }
 493
 494 /*
 495  * Drain pages out of the cpu's pagevecs.
 496  * Either "cpu" is the current CPU, and preemption has already been
 497  * disabled; or "cpu" is being hot-unplugged, and is already dead.
 498  */
 499 static void drain_cpu_pagevecs(int cpu)
 500 {
 501         struct pagevec *pvecs = per_cpu(lru_add_pvecs, cpu);
 502         struct pagevec *pvec;
 503         int lru;
 504
 505         for_each_lru(lru) {
 506                 pvec = &pvecs[lru - LRU_BASE];
 507                 if (pagevec_count(pvec))
 508                         __pagevec_lru_add(pvec, lru);
 509         }
 510
 511         pvec = &per_cpu(lru_rotate_pvecs, cpu);
 512         if (pagevec_count(pvec)) {
 513                 unsigned long flags;
 514
 515                 /* No harm done if a racing interrupt already did this */
 516                 local_irq_save(flags);
 517                 pagevec_move_tail(pvec);
 518                 local_irq_restore(flags);
 519         }
 520
 521         pvec = &per_cpu(lru_deactivate_pvecs, cpu);
 522         if (pagevec_count(pvec))
 523                 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
 524
 525         activate_page_drain(cpu);
 526 }
 527
 528 /**
 529  * deactivate_page - forcefully deactivate a page
 530  * @page: page to deactivate
 531  *
 532  * This function hints the VM that @page is a good reclaim candidate,
 533  * for example if its invalidation fails due to the page being dirty
 534  * or under writeback.
 535  */
 536 void deactivate_page(struct page *page)
 537 {
 538         /*
 539          * In a workload with many unevictable page such as mprotect, unevictable
 540          * page deactivation for accelerating reclaim is pointless.
 541          */
 542         if (PageUnevictable(page))
 543                 return;
 544
 545         if (likely(get_page_unless_zero(page))) {
 546                 struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
 547
 548                 if (!pagevec_add(pvec, page))
 549                         pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
 550                 put_cpu_var(lru_deactivate_pvecs);
 551         }
 552 }
 553
 554 void lru_add_drain(void)
 555 {
 556         drain_cpu_pagevecs(get_cpu());
 557         put_cpu();
 558 }
 559
 560 static void lru_add_drain_per_cpu(struct work_struct *dummy)
 561 {
 562         lru_add_drain();
 563 }
 564
 565 /*
 566  * Returns 0 for success
 567  */
 568 int lru_add_drain_all(void)
 569 {
 570         return schedule_on_each_cpu(lru_add_drain_per_cpu);
 571 }
 572
 573 /*
 574  * Batched page_cache_release().  Decrement the reference count on all the
 575  * passed pages.  If it fell to zero then remove the page from the LRU and
 576  * free it.
 577  *
 578  * Avoid taking zone->lru_lock if possible, but if it is taken, retain it
 579  * for the remainder of the operation.
 580  *
 581  * The locking in this function is against shrink_inactive_list(): we recheck
 582  * the page count inside the lock to see whether shrink_inactive_list()
 583  * grabbed the page via the LRU.  If it did, give up: shrink_inactive_list()
 584  * will free it.
 585  */
 586 void release_pages(struct page **pages, int nr, int cold)
 587 {
 588         int i;
 589         LIST_HEAD(pages_to_free);
 590         struct zone *zone = NULL;
 591         unsigned long uninitialized_var(flags);
 592
 593         for (i = 0; i < nr; i++) {
 594                 struct page *page = pages[i];
 595
 596                 if (unlikely(PageCompound(page))) {
 597                         if (zone) {
 598                                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 599                                 zone = NULL;
 600                         }
 601                         put_compound_page(page);
 602                         continue;
 603                 }
 604
 605                 if (!put_page_testzero(page))
 606                         continue;
 607
 608                 if (PageLRU(page)) {
 609                         struct zone *pagezone = page_zone(page);
 610
 611                         if (pagezone != zone) {
 612                                 if (zone)
 613                                         spin_unlock_irqrestore(&zone->lru_lock,
 614                                                                         flags);
 615                                 zone = pagezone;
 616                                 spin_lock_irqsave(&zone->lru_lock, flags);
 617                         }
 618                         VM_BUG_ON(!PageLRU(page));
 619                         __ClearPageLRU(page);
 620                         del_page_from_lru_list(zone, page, page_off_lru(page));
 621                 }
 622
 623                 list_add(&page->lru, &pages_to_free);
 624         }
 625         if (zone)
 626                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 627
 628         free_hot_cold_page_list(&pages_to_free, cold);
 629 }
 630 EXPORT_SYMBOL(release_pages);
 631
 632 /*
 633  * The pages which we're about to release may be in the deferred lru-addition
 634  * queues.  That would prevent them from really being freed right now.  That's
 635  * OK from a correctness point of view but is inefficient - those pages may be
 636  * cache-warm and we want to give them back to the page allocator ASAP.
 637  *
 638  * So __pagevec_release() will drain those queues here.  __pagevec_lru_add()
 639  * and __pagevec_lru_add_active() call release_pages() directly to avoid
 640  * mutual recursion.
 641  */
 642 void __pagevec_release(struct pagevec *pvec)
 643 {
 644         lru_add_drain();
 645         release_pages(pvec->pages, pagevec_count(pvec), pvec->cold);
 646         pagevec_reinit(pvec);
 647 }
 648 EXPORT_SYMBOL(__pagevec_release);
 649
 650 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 651 /* used by __split_huge_page_refcount() */
 652 void lru_add_page_tail(struct zone* zone,
 653                        struct page *page, struct page *page_tail)
 654 {
 655         int active;
 656         enum lru_list lru;
 657         const int file = 0;
 658
 659         VM_BUG_ON(!PageHead(page));
 660         VM_BUG_ON(PageCompound(page_tail));
 661         VM_BUG_ON(PageLRU(page_tail));
 662         VM_BUG_ON(!spin_is_locked(&zone->lru_lock));
 663
 664         SetPageLRU(page_tail);
 665
 666         if (page_evictable(page_tail, NULL)) {
 667                 if (PageActive(page)) {
 668                         SetPageActive(page_tail);
 669                         active = 1;
 670                         lru = LRU_ACTIVE_ANON;
 671                 } else {
 672                         active = 0;
 673                         lru = LRU_INACTIVE_ANON;
 674                 }
 675                 update_page_reclaim_stat(zone, page_tail, file, active);
 676         } else {
 677                 SetPageUnevictable(page_tail);
 678                 lru = LRU_UNEVICTABLE;
 679         }
 680
 681         if (likely(PageLRU(page)))
 682                 list_add_tail(&page_tail->lru, &page->lru);
 683         else {
 684                 struct list_head *list_head;
 685                 /*
 686                  * Head page has not yet been counted, as an hpage,
 687                  * so we must account for each subpage individually.
 688                  *
 689                  * Use the standard add function to put page_tail on the list,
 690                  * but then correct its position so they all end up in order.
 691                  */
 692                 add_page_to_lru_list(zone, page_tail, lru);
 693                 list_head = page_tail->lru.prev;
 694                 list_move_tail(&page_tail->lru, list_head);
 695         }
 696 }
 697 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 698
 699 static void __pagevec_lru_add_fn(struct page *page, void *arg)
 700 {
 701         enum lru_list lru = (enum lru_list)arg;
 702         struct zone *zone = page_zone(page);
 703         int file = is_file_lru(lru);
 704         int active = is_active_lru(lru);
 705
 706         VM_BUG_ON(PageActive(page));
 707         VM_BUG_ON(PageUnevictable(page));
 708         VM_BUG_ON(PageLRU(page));
 709
 710         SetPageLRU(page);
 711         if (active)
 712                 SetPageActive(page);
 713         update_page_reclaim_stat(zone, page, file, active);
 714         add_page_to_lru_list(zone, page, lru);
 715 }
 716
 717 /*
 718  * Add the passed pages to the LRU, then drop the caller's refcount
 719  * on them.  Reinitialises the caller's pagevec.
 720  */
 721 void __pagevec_lru_add(struct pagevec *pvec, enum lru_list lru)
 722 {
 723         VM_BUG_ON(is_unevictable_lru(lru));
 724
 725         pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, (void *)lru);
 726 }
 727 EXPORT_SYMBOL(__pagevec_lru_add);
 728
 729 /**
 730  * pagevec_lookup - gang pagecache lookup
 731  * @pvec:       Where the resulting pages are placed
 732  * @mapping:    The address_space to search
 733  * @start:      The starting page index
 734  * @nr_pages:   The maximum number of pages
 735  *
 736  * pagevec_lookup() will search for and return a group of up to @nr_pages pages
 737  * in the mapping.  The pages are placed in @pvec.  pagevec_lookup() takes a
 738  * reference against the pages in @pvec.
 739  *
 740  * The search returns a group of mapping-contiguous pages with ascending
 741  * indexes.  There may be holes in the indices due to not-present pages.
 742  *
 743  * pagevec_lookup() returns the number of pages which were found.
 744  */
 745 unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
 746                 pgoff_t start, unsigned nr_pages)
 747 {
 748         pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages);
 749         return pagevec_count(pvec);
 750 }
 751 EXPORT_SYMBOL(pagevec_lookup);
 752
 753 unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
 754                 pgoff_t *index, int tag, unsigned nr_pages)
 755 {
 756         pvec->nr = find_get_pages_tag(mapping, index, tag,
 757                                         nr_pages, pvec->pages);
 758         return pagevec_count(pvec);
 759 }
 760 EXPORT_SYMBOL(pagevec_lookup_tag);
 761
 762 /*
 763  * Perform any setup for the swap system
 764  */
 765 void __init swap_setup(void)
 766 {
 767         unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT);
 768
 769 #ifdef CONFIG_SWAP
 770         bdi_init(swapper_space.backing_dev_info);
 771 #endif
 772
 773         /* Use a smaller cluster for small-memory machines */
 774         if (megs < 16)
 775                 page_cluster = 2;
 776         else
 777                 page_cluster = 3;
 778         /*
 779          * Right now other parts of the system means that we
 780          * _really_ don't want to cluster much more
 781          */
 782 }