mm/swap_state.c

   1 /*
   2  *  linux/mm/swap_state.c
   3  *
   4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   5  *  Swap reorganised 29.12.95, Stephen Tweedie
   6  *
   7  *  Rewritten to use page cache, (C) 1998 Stephen Tweedie
   8  */
   9
  10 #include <linux/mm.h>
  11 #include <linux/kernel_stat.h>
  12 #include <linux/swap.h>
  13 #include <linux/init.h>
  14 #include <linux/pagemap.h>
  15 #include <linux/backing-dev.h>
  16
  17 #include <asm/pgtable.h>
  18
  19 /*
  20  * swapper_inode doesn't do anything much.  It is really only here to
  21  * avoid some special-casing in other parts of the kernel.
  22  */
  23 static struct inode swapper_inode = {
  24         .i_mapping      = &swapper_space,
  25 };
  26
  27 static struct backing_dev_info swap_backing_dev_info = {
  28         .ra_pages       = 0,    /* No readahead */
  29         .memory_backed  = 1,    /* Does not contribute to dirty memory */
  30 };
  31
  32 extern struct address_space_operations swap_aops;
  33
  34 struct address_space swapper_space = {
  35         .page_tree      = RADIX_TREE_INIT(GFP_ATOMIC),
  36         .page_lock      = SPIN_LOCK_UNLOCKED,
  37         .clean_pages    = LIST_HEAD_INIT(swapper_space.clean_pages),
  38         .dirty_pages    = LIST_HEAD_INIT(swapper_space.dirty_pages),
  39         .io_pages       = LIST_HEAD_INIT(swapper_space.io_pages),
  40         .locked_pages   = LIST_HEAD_INIT(swapper_space.locked_pages),
  41         .host           = &swapper_inode,
  42         .a_ops          = &swap_aops,
  43         .backing_dev_info = &swap_backing_dev_info,
  44         .i_mmap         = LIST_HEAD_INIT(swapper_space.i_mmap),
  45         .i_mmap_shared  = LIST_HEAD_INIT(swapper_space.i_mmap_shared),
  46         .i_shared_sem   = __MUTEX_INITIALIZER(swapper_space.i_shared_sem),
  47         .private_lock   = SPIN_LOCK_UNLOCKED,
  48         .private_list   = LIST_HEAD_INIT(swapper_space.private_list),
  49 };
  50
  51 #define INC_CACHE_INFO(x)       do { swap_cache_info.x++; } while (0)
  52
  53 static struct {
  54         unsigned long add_total;
  55         unsigned long del_total;
  56         unsigned long find_success;
  57         unsigned long find_total;
  58         unsigned long noent_race;
  59         unsigned long exist_race;
  60 } swap_cache_info;
  61
  62 void show_swap_cache_info(void)
  63 {
  64         printk("Swap cache: add %lu, delete %lu, find %lu/%lu, race %lu+%lu\n",
  65                 swap_cache_info.add_total, swap_cache_info.del_total,
  66                 swap_cache_info.find_success, swap_cache_info.find_total,
  67                 swap_cache_info.noent_race, swap_cache_info.exist_race);
  68 }
  69
  70 static int add_to_swap_cache(struct page *page, swp_entry_t entry)
  71 {
  72         int error;
  73
  74         if (page->mapping)
  75                 BUG();
  76         if (!swap_duplicate(entry)) {
  77                 INC_CACHE_INFO(noent_race);
  78                 return -ENOENT;
  79         }
  80         error = add_to_page_cache(page, &swapper_space, entry.val, GFP_KERNEL);
  81         /*
  82          * Anon pages are already on the LRU, we don't run lru_cache_add here.
  83          */
  84         if (error != 0) {
  85                 swap_free(entry);
  86                 if (error == -EEXIST)
  87                         INC_CACHE_INFO(exist_race);
  88                 return error;
  89         }
  90         if (!PageLocked(page))
  91                 BUG();
  92         if (!PageSwapCache(page))
  93                 BUG();
  94         INC_CACHE_INFO(add_total);
  95         return 0;
  96 }
  97
  98 /*
  99  * This must be called only on pages that have
 100  * been verified to be in the swap cache.
 101  */
 102 void __delete_from_swap_cache(struct page *page)
 103 {
 104         BUG_ON(!PageLocked(page));
 105         BUG_ON(!PageSwapCache(page));
 106         BUG_ON(PageWriteback(page));
 107         __remove_from_page_cache(page);
 108         INC_CACHE_INFO(del_total);
 109 }
 110
 111 /**
 112  * add_to_swap - allocate swap space for a page
 113  * @page: page we want to move to swap
 114  *
 115  * Allocate swap space for the page and add the page to the
 116  * swap cache.  Caller needs to hold the page lock.
 117  */
 118 int add_to_swap(struct page * page)
 119 {
 120         swp_entry_t entry;
 121         int pf_flags;
 122         int err;
 123
 124         if (!PageLocked(page))
 125                 BUG();
 126
 127         for (;;) {
 128                 entry = get_swap_page();
 129                 if (!entry.val)
 130                         return 0;
 131
 132                 /* Radix-tree node allocations are performing
 133                  * GFP_ATOMIC allocations under PF_MEMALLOC.
 134                  * They can completely exhaust the page allocator.
 135                  *
 136                  * So PF_MEMALLOC is dropped here.  This causes the slab
 137                  * allocations to fail earlier, so radix-tree nodes will
 138                  * then be allocated from the mempool reserves.
 139                  *
 140                  * We're still using __GFP_HIGH for radix-tree node
 141                  * allocations, so some of the emergency pools are available,
 142                  * just not all of them.
 143                  */
 144
 145                 pf_flags = current->flags;
 146                 current->flags &= ~PF_MEMALLOC;
 147
 148                 /*
 149                  * Add it to the swap cache and mark it dirty
 150                  */
 151                 err = add_to_page_cache(page, &swapper_space,
 152                                         entry.val, GFP_ATOMIC);
 153
 154                 if (pf_flags & PF_MEMALLOC)
 155                         current->flags |= PF_MEMALLOC;
 156
 157                 switch (err) {
 158                 case 0:                         /* Success */
 159                         SetPageUptodate(page);
 160                         ClearPageDirty(page);
 161                         set_page_dirty(page);
 162                         INC_CACHE_INFO(add_total);
 163                         return 1;
 164                 case -EEXIST:
 165                         /* Raced with "speculative" read_swap_cache_async */
 166                         INC_CACHE_INFO(exist_race);
 167                         swap_free(entry);
 168                         continue;
 169                 default:
 170                         /* -ENOMEM radix-tree allocation failure */
 171                         swap_free(entry);
 172                         return 0;
 173                 }
 174         }
 175 }
 176
 177 /*
 178  * This must be called only on pages that have
 179  * been verified to be in the swap cache and locked.
 180  * It will never put the page into the free list,
 181  * the caller has a reference on the page.
 182  */
 183 void delete_from_swap_cache(struct page *page)
 184 {
 185         swp_entry_t entry;
 186
 187         BUG_ON(!PageLocked(page));
 188         BUG_ON(PageWriteback(page));
 189         BUG_ON(PagePrivate(page));
 190
 191         entry.val = page->index;
 192
 193         spin_lock(&swapper_space.page_lock);
 194         __delete_from_swap_cache(page);
 195         spin_unlock(&swapper_space.page_lock);
 196
 197         swap_free(entry);
 198         page_cache_release(page);
 199 }
 200
 201 int move_to_swap_cache(struct page *page, swp_entry_t entry)
 202 {
 203         struct address_space *mapping = page->mapping;
 204         int err;
 205
 206         spin_lock(&swapper_space.page_lock);
 207         spin_lock(&mapping->page_lock);
 208
 209         err = radix_tree_insert(&swapper_space.page_tree, entry.val, page);
 210         if (!err) {
 211                 __remove_from_page_cache(page);
 212                 ___add_to_page_cache(page, &swapper_space, entry.val);
 213         }
 214
 215         spin_unlock(&mapping->page_lock);
 216         spin_unlock(&swapper_space.page_lock);
 217
 218         if (!err) {
 219                 if (!swap_duplicate(entry))
 220                         BUG();
 221                 /* shift page from clean_pages to dirty_pages list */
 222                 BUG_ON(PageDirty(page));
 223                 set_page_dirty(page);
 224                 INC_CACHE_INFO(add_total);
 225         } else if (err == -EEXIST)
 226                 INC_CACHE_INFO(exist_race);
 227         return err;
 228 }
 229
 230 int move_from_swap_cache(struct page *page, unsigned long index,
 231                 struct address_space *mapping)
 232 {
 233         swp_entry_t entry;
 234         int err;
 235
 236         BUG_ON(!PageLocked(page));
 237         BUG_ON(PageWriteback(page));
 238         BUG_ON(PagePrivate(page));
 239
 240         entry.val = page->index;
 241
 242         spin_lock(&swapper_space.page_lock);
 243         spin_lock(&mapping->page_lock);
 244
 245         err = radix_tree_insert(&mapping->page_tree, index, page);
 246         if (!err) {
 247                 __delete_from_swap_cache(page);
 248                 ___add_to_page_cache(page, mapping, index);
 249         }
 250
 251         spin_unlock(&mapping->page_lock);
 252         spin_unlock(&swapper_space.page_lock);
 253
 254         if (!err) {
 255                 swap_free(entry);
 256                 /* shift page from clean_pages to dirty_pages list */
 257                 ClearPageDirty(page);
 258                 set_page_dirty(page);
 259         }
 260         return err;
 261 }
 262
 263
 264 /*
 265  * If we are the only user, then try to free up the swap cache.
 266  *
 267  * Its ok to check for PageSwapCache without the page lock
 268  * here because we are going to recheck again inside
 269  * exclusive_swap_page() _with_ the lock.
 270  *                                      - Marcelo
 271  */
 272 static inline void free_swap_cache(struct page *page)
 273 {
 274         if (PageSwapCache(page) && !TestSetPageLocked(page)) {
 275                 remove_exclusive_swap_page(page);
 276                 unlock_page(page);
 277         }
 278 }
 279
 280 /*
 281  * Perform a free_page(), also freeing any swap cache associated with
 282  * this page if it is the last user of the page. Can not do a lock_page,
 283  * as we are holding the page_table_lock spinlock.
 284  */
 285 void free_page_and_swap_cache(struct page *page)
 286 {
 287         free_swap_cache(page);
 288         page_cache_release(page);
 289 }
 290
 291 /*
 292  * Passed an array of pages, drop them all from swapcache and then release
 293  * them.  They are removed from the LRU and freed if this is their last use.
 294  */
 295 void free_pages_and_swap_cache(struct page **pages, int nr)
 296 {
 297         int chunk = 16;
 298         struct page **pagep = pages;
 299
 300         lru_add_drain();
 301         while (nr) {
 302                 int todo = min(chunk, nr);
 303                 int i;
 304
 305                 for (i = 0; i < todo; i++)
 306                         free_swap_cache(pagep[i]);
 307                 release_pages(pagep, todo, 0);
 308                 pagep += todo;
 309                 nr -= todo;
 310         }
 311 }
 312
 313 /*
 314  * Lookup a swap entry in the swap cache. A found page will be returned
 315  * unlocked and with its refcount incremented - we rely on the kernel
 316  * lock getting page table operations atomic even if we drop the page
 317  * lock before returning.
 318  */
 319 struct page * lookup_swap_cache(swp_entry_t entry)
 320 {
 321         struct page *found;
 322
 323         found = find_get_page(&swapper_space, entry.val);
 324         /*
 325          * Unsafe to assert PageSwapCache and mapping on page found:
 326          * if SMP nothing prevents swapoff from deleting this page from
 327          * the swap cache at this moment.  find_lock_page would prevent
 328          * that, but no need to change: we _have_ got the right page.
 329          */
 330         INC_CACHE_INFO(find_total);
 331         if (found)
 332                 INC_CACHE_INFO(find_success);
 333         return found;
 334 }
 335
 336 /*
 337  * Locate a page of swap in physical memory, reserving swap cache space
 338  * and reading the disk if it is not already cached.
 339  * A failure return means that either the page allocation failed or that
 340  * the swap entry is no longer in use.
 341  */
 342 struct page * read_swap_cache_async(swp_entry_t entry)
 343 {
 344         struct page *found_page, *new_page = NULL;
 345         int err;
 346
 347         do {
 348                 /*
 349                  * First check the swap cache.  Since this is normally
 350                  * called after lookup_swap_cache() failed, re-calling
 351                  * that would confuse statistics: use find_get_page()
 352                  * directly.
 353                  */
 354                 found_page = find_get_page(&swapper_space, entry.val);
 355                 if (found_page)
 356                         break;
 357
 358                 /*
 359                  * Get a new page to read into from swap.
 360                  */
 361                 if (!new_page) {
 362                         new_page = alloc_page(GFP_HIGHUSER);
 363                         if (!new_page)
 364                                 break;          /* Out of memory */
 365                 }
 366
 367                 /*
 368                  * Associate the page with swap entry in the swap cache.
 369                  * May fail (-ENOENT) if swap entry has been freed since
 370                  * our caller observed it.  May fail (-EEXIST) if there
 371                  * is already a page associated with this entry in the
 372                  * swap cache: added by a racing read_swap_cache_async,
 373                  * or by try_to_swap_out (or shmem_writepage) re-using
 374                  * the just freed swap entry for an existing page.
 375                  * May fail (-ENOMEM) if radix-tree node allocation failed.
 376                  */
 377                 err = add_to_swap_cache(new_page, entry);
 378                 if (!err) {
 379                         /*
 380                          * Initiate read into locked page and return.
 381                          */
 382                         lru_cache_add_active(new_page);
 383                         swap_readpage(NULL, new_page);
 384                         return new_page;
 385                 }
 386         } while (err != -ENOENT && err != -ENOMEM);
 387
 388         if (new_page)
 389                 page_cache_release(new_page);
 390         return found_page;
 391 }