mm/swap_state.c

   1 /*
   2  *  linux/mm/swap_state.c
   3  *
   4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   5  *  Swap reorganised 29.12.95, Stephen Tweedie
   6  *
   7  *  Rewritten to use page cache, (C) 1998 Stephen Tweedie
   8  */
   9
  10 #include <linux/mm.h>
  11 #include <linux/kernel_stat.h>
  12 #include <linux/swap.h>
  13 #include <linux/swapctl.h>
  14 #include <linux/init.h>
  15 #include <linux/pagemap.h>
  16 #include <linux/smp_lock.h>
  17
  18 #include <asm/pgtable.h>
  19
  20 static struct address_space_operations swap_aops = {
  21         sync_page: block_sync_page
  22 };
  23
  24 struct address_space swapper_space = {
  25         {                               /* pages        */
  26                 &swapper_space.pages,   /*        .next */
  27                 &swapper_space.pages    /*        .prev */
  28         },
  29         0,                              /* nrpages      */
  30         &swap_aops,
  31 };
  32
  33 #ifdef SWAP_CACHE_INFO
  34 unsigned long swap_cache_add_total;
  35 unsigned long swap_cache_del_total;
  36 unsigned long swap_cache_find_total;
  37 unsigned long swap_cache_find_success;
  38
  39 void show_swap_cache_info(void)
  40 {
  41         printk("Swap cache: add %ld, delete %ld, find %ld/%ld\n",
  42                 swap_cache_add_total,
  43                 swap_cache_del_total,
  44                 swap_cache_find_success, swap_cache_find_total);
  45 }
  46 #endif
  47
  48 void add_to_swap_cache(struct page *page, swp_entry_t entry)
  49 {
  50         unsigned long flags;
  51
  52 #ifdef SWAP_CACHE_INFO
  53         swap_cache_add_total++;
  54 #endif
  55         if (!PageLocked(page))
  56                 BUG();
  57         if (PageTestandSetSwapCache(page))
  58                 BUG();
  59         if (page->mapping)
  60                 BUG();
  61         flags = page->flags & ~((1 << PG_error) | (1 << PG_dirty) | (1 << PG_referenced));
  62         page->flags = flags | (1 << PG_uptodate);
  63         add_to_page_cache_locked(page, &swapper_space, entry.val);
  64 }
  65
  66 static inline void remove_from_swap_cache(struct page *page)
  67 {
  68         struct address_space *mapping = page->mapping;
  69
  70         if (mapping != &swapper_space)
  71                 BUG();
  72         if (!PageSwapCache(page) || !PageLocked(page))
  73                 PAGE_BUG(page);
  74
  75         PageClearSwapCache(page);
  76         remove_inode_page(page);
  77 }
  78
  79 /*
  80  * This must be called only on pages that have
  81  * been verified to be in the swap cache.
  82  */
  83 void __delete_from_swap_cache(struct page *page)
  84 {
  85         swp_entry_t entry;
  86
  87         entry.val = page->index;
  88
  89 #ifdef SWAP_CACHE_INFO
  90         swap_cache_del_total++;
  91 #endif
  92         remove_from_swap_cache(page);
  93         swap_free(entry);
  94 }
  95
  96 /*
  97  * This will never put the page into the free list, the caller has
  98  * a reference on the page.
  99  */
 100 void delete_from_swap_cache_nolock(struct page *page)
 101 {
 102         if (!PageLocked(page))
 103                 BUG();
 104
 105         if (block_flushpage(page, 0))
 106                 lru_cache_del(page);
 107
 108         __delete_from_swap_cache(page);
 109         page_cache_release(page);
 110 }
 111
 112 /*
 113  * This must be called only on pages that have
 114  * been verified to be in the swap cache and locked.
 115  */
 116 void delete_from_swap_cache(struct page *page)
 117 {
 118         lock_page(page);
 119         delete_from_swap_cache_nolock(page);
 120         UnlockPage(page);
 121 }
 122
 123 /*
 124  * Perform a free_page(), also freeing any swap cache associated with
 125  * this page if it is the last user of the page. Can not do a lock_page,
 126  * as we are holding the page_table_lock spinlock.
 127  */
 128 void free_page_and_swap_cache(struct page *page)
 129 {
 130         /*
 131          * If we are the only user, then try to free up the swap cache.
 132          */
 133         if (PageSwapCache(page) && !TryLockPage(page)) {
 134                 if (!is_page_shared(page)) {
 135                         delete_from_swap_cache_nolock(page);
 136                 }
 137                 UnlockPage(page);
 138         }
 139         page_cache_release(page);
 140 }
 141
 142
 143 /*
 144  * Lookup a swap entry in the swap cache. A found page will be returned
 145  * unlocked and with its refcount incremented - we rely on the kernel
 146  * lock getting page table operations atomic even if we drop the page
 147  * lock before returning.
 148  */
 149
 150 struct page * lookup_swap_cache(swp_entry_t entry)
 151 {
 152         struct page *found;
 153
 154 #ifdef SWAP_CACHE_INFO
 155         swap_cache_find_total++;
 156 #endif
 157         while (1) {
 158                 /*
 159                  * Right now the pagecache is 32-bit only.  But it's a 32 bit index. =)
 160                  */
 161 repeat:
 162                 found = find_lock_page(&swapper_space, entry.val);
 163                 if (!found)
 164                         return 0;
 165                 /*
 166                  * Though the "found" page was in the swap cache an instant
 167                  * earlier, it might have been removed by shrink_mmap etc.
 168                  * Re search ... Since find_lock_page grabs a reference on
 169                  * the page, it can not be reused for anything else, namely
 170                  * it can not be associated with another swaphandle, so it
 171                  * is enough to check whether the page is still in the scache.
 172                  */
 173                 if (!PageSwapCache(found)) {
 174                         UnlockPage(found);
 175                         page_cache_release(found);
 176                         goto repeat;
 177                 }
 178                 if (found->mapping != &swapper_space)
 179                         goto out_bad;
 180 #ifdef SWAP_CACHE_INFO
 181                 swap_cache_find_success++;
 182 #endif
 183                 UnlockPage(found);
 184                 return found;
 185         }
 186
 187 out_bad:
 188         printk (KERN_ERR "VM: Found a non-swapper swap page!\n");
 189         UnlockPage(found);
 190         page_cache_release(found);
 191         return 0;
 192 }
 193
 194 /*
 195  * Locate a page of swap in physical memory, reserving swap cache space
 196  * and reading the disk if it is not already cached.  If wait==0, we are
 197  * only doing readahead, so don't worry if the page is already locked.
 198  *
 199  * A failure return means that either the page allocation failed or that
 200  * the swap entry is no longer in use.
 201  */
 202
 203 struct page * read_swap_cache_async(swp_entry_t entry, int wait)
 204 {
 205         struct page *found_page = 0, *new_page;
 206         unsigned long new_page_addr;
 207
 208         /*
 209          * Make sure the swap entry is still in use.
 210          */
 211         if (!swap_duplicate(entry))     /* Account for the swap cache */
 212                 goto out;
 213         /*
 214          * Look for the page in the swap cache.
 215          */
 216         found_page = lookup_swap_cache(entry);
 217         if (found_page)
 218                 goto out_free_swap;
 219
 220         new_page_addr = __get_free_page(GFP_USER);
 221         if (!new_page_addr)
 222                 goto out_free_swap;     /* Out of memory */
 223         new_page = mem_map + MAP_NR(new_page_addr);
 224
 225         /*
 226          * Check the swap cache again, in case we stalled above.
 227          */
 228         found_page = lookup_swap_cache(entry);
 229         if (found_page)
 230                 goto out_free_page;
 231         /*
 232          * Add it to the swap cache and read its contents.
 233          */
 234         lock_page(new_page);
 235         add_to_swap_cache(new_page, entry);
 236         rw_swap_page(READ, new_page, wait);
 237         return new_page;
 238
 239 out_free_page:
 240         page_cache_release(new_page);
 241 out_free_swap:
 242         swap_free(entry);
 243 out:
 244         return found_page;
 245 }