mm/swap_state.c

   1 /*
   2  *  linux/mm/swap_state.c
   3  *
   4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   5  *  Swap reorganised 29.12.95, Stephen Tweedie
   6  *
   7  *  Rewritten to use page cache, (C) 1998 Stephen Tweedie
   8  */
   9
  10 #include <linux/mm.h>
  11 #include <linux/sched.h>
  12 #include <linux/head.h>
  13 #include <linux/kernel.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/errno.h>
  16 #include <linux/string.h>
  17 #include <linux/stat.h>
  18 #include <linux/swap.h>
  19 #include <linux/fs.h>
  20 #include <linux/swapctl.h>
  21 #include <linux/init.h>
  22 #include <linux/pagemap.h>
  23
  24 #include <asm/bitops.h>
  25 #include <asm/pgtable.h>
  26
  27 /*
  28  * Keep a reserved false inode which we will use to mark pages in the
  29  * page cache are acting as swap cache instead of file cache.
  30  *
  31  * We only need a unique pointer to satisfy the page cache, but we'll
  32  * reserve an entire zeroed inode structure for the purpose just to
  33  * ensure that any mistaken dereferences of this structure cause a
  34  * kernel oops.
  35  */
  36 struct inode swapper_inode;
  37
  38 #ifdef SWAP_CACHE_INFO
  39 unsigned long swap_cache_add_total = 0;
  40 unsigned long swap_cache_add_success = 0;
  41 unsigned long swap_cache_del_total = 0;
  42 unsigned long swap_cache_del_success = 0;
  43 unsigned long swap_cache_find_total = 0;
  44 unsigned long swap_cache_find_success = 0;
  45
  46 void show_swap_cache_info(void)
  47 {
  48         printk("Swap cache: add %ld/%ld, delete %ld/%ld, find %ld/%ld\n",
  49                 swap_cache_add_total, swap_cache_add_success,
  50                 swap_cache_del_total, swap_cache_del_success,
  51                 swap_cache_find_total, swap_cache_find_success);
  52 }
  53 #endif
  54
  55 int add_to_swap_cache(struct page *page, unsigned long entry)
  56 {
  57 #ifdef SWAP_CACHE_INFO
  58         swap_cache_add_total++;
  59 #endif
  60 #ifdef DEBUG_SWAP
  61         printk("DebugVM: add_to_swap_cache(%08lx count %d, entry %08lx)\n",
  62                page_address(page), atomic_read(&page->count), entry);
  63 #endif
  64         if (PageTestandSetSwapCache(page)) {
  65                 printk(KERN_ERR "swap_cache: replacing non-empty entry %08lx "
  66                        "on page %08lx\n",
  67                        page->offset, page_address(page));
  68                 return 0;
  69         }
  70         if (page->inode) {
  71                 printk(KERN_ERR "swap_cache: replacing page-cached entry "
  72                        "on page %08lx\n", page_address(page));
  73                 return 0;
  74         }
  75         atomic_inc(&page->count);
  76         page->inode = &swapper_inode;
  77         page->offset = entry;
  78         add_page_to_hash_queue(page, &swapper_inode, entry);
  79         add_page_to_inode_queue(&swapper_inode, page);
  80 #ifdef SWAP_CACHE_INFO
  81         swap_cache_add_success++;
  82 #endif
  83         return 1;
  84 }
  85
  86 /*
  87  * Verify that a swap entry is valid and increment its swap map count.
  88  *
  89  * Note: if swap_map[] reaches SWAP_MAP_MAX the entries are treated as
  90  * "permanent", but will be reclaimed by the next swapoff.
  91  */
  92 int swap_duplicate(unsigned long entry)
  93 {
  94         struct swap_info_struct * p;
  95         unsigned long offset, type;
  96         int result = 0;
  97
  98         if (!entry)
  99                 goto out;
 100         type = SWP_TYPE(entry);
 101         if (type & SHM_SWP_TYPE)
 102                 goto out;
 103         if (type >= nr_swapfiles)
 104                 goto bad_file;
 105         p = type + swap_info;
 106         offset = SWP_OFFSET(entry);
 107         if (offset >= p->max)
 108                 goto bad_offset;
 109         if (!p->swap_map[offset])
 110                 goto bad_unused;
 111         /*
 112          * Entry is valid, so increment the map count.
 113          */
 114         if (p->swap_map[offset] < SWAP_MAP_MAX)
 115                 p->swap_map[offset]++;
 116         else {
 117                 static int overflow = 0;
 118                 if (overflow++ < 5)
 119                         printk(KERN_WARNING
 120                                 "swap_duplicate: entry %08lx map count=%d\n",
 121                                 entry, p->swap_map[offset]);
 122                 p->swap_map[offset] = SWAP_MAP_MAX;
 123         }
 124         result = 1;
 125 #ifdef DEBUG_SWAP
 126         printk("DebugVM: swap_duplicate(entry %08lx, count now %d)\n",
 127                entry, p->swap_map[offset]);
 128 #endif
 129 out:
 130         return result;
 131
 132 bad_file:
 133         printk(KERN_ERR
 134                 "swap_duplicate: entry %08lx, nonexistent swap file\n", entry);
 135         goto out;
 136 bad_offset:
 137         printk(KERN_ERR
 138                 "swap_duplicate: entry %08lx, offset exceeds max\n", entry);
 139         goto out;
 140 bad_unused:
 141         printk(KERN_ERR
 142                 "swap_duplicate at %8p: entry %08lx, unused page\n",
 143                __builtin_return_address(0), entry);
 144         goto out;
 145 }
 146
 147
 148 static inline void remove_from_swap_cache(struct page *page)
 149 {
 150         if (!page->inode) {
 151                 printk ("VM: Removing swap cache page with zero inode hash "
 152                         "on page %08lx\n", page_address(page));
 153                 return;
 154         }
 155         if (page->inode != &swapper_inode) {
 156                 printk ("VM: Removing swap cache page with wrong inode hash "
 157                         "on page %08lx\n", page_address(page));
 158         }
 159         /*
 160          * This is a legal case, but warn about it.
 161          */
 162         if (atomic_read(&page->count) == 1) {
 163                 printk (KERN_WARNING
 164                         "VM: Removing page cache on unshared page %08lx\n",
 165                         page_address(page));
 166         }
 167
 168 #ifdef DEBUG_SWAP
 169         printk("DebugVM: remove_from_swap_cache(%08lx count %d)\n",
 170                page_address(page), atomic_read(&page->count));
 171 #endif
 172         PageClearSwapCache (page);
 173         remove_inode_page(page);
 174 }
 175
 176
 177 void delete_from_swap_cache(struct page *page)
 178 {
 179 #ifdef SWAP_CACHE_INFO
 180         swap_cache_del_total++;
 181 #endif
 182         if (PageSwapCache (page))  {
 183                 long entry = page->offset;
 184 #ifdef SWAP_CACHE_INFO
 185                 swap_cache_del_success++;
 186 #endif
 187 #ifdef DEBUG_SWAP
 188                 printk("DebugVM: delete_from_swap_cache(%08lx count %d, "
 189                        "entry %08lx)\n",
 190                        page_address(page), atomic_read(&page->count), entry);
 191 #endif
 192                 remove_from_swap_cache (page);
 193                 swap_free (entry);
 194         }
 195 }
 196
 197 /*
 198  * Perform a free_page(), also freeing any swap cache associated with
 199  * this page if it is the last user of the page.
 200  */
 201
 202 void free_page_and_swap_cache(unsigned long addr)
 203 {
 204         struct page *page = mem_map + MAP_NR(addr);
 205         /*
 206          * If we are the only user, then free up the swap cache.
 207          */
 208         if (PageSwapCache(page) && !is_page_shared(page)) {
 209                 delete_from_swap_cache(page);
 210         }
 211
 212         free_page(addr);
 213 }
 214
 215
 216 /*
 217  * Lookup a swap entry in the swap cache.  We need to be careful about
 218  * locked pages.  A found page will be returned with its refcount
 219  * incremented.
 220  */
 221
 222 static struct page * lookup_swap_cache(unsigned long entry)
 223 {
 224         struct page *found;
 225
 226         while (1) {
 227                 found = find_page(&swapper_inode, entry);
 228                 if (!found)
 229                         return 0;
 230                 if (found->inode != &swapper_inode || !PageSwapCache(found))
 231                         goto out_bad;
 232                 if (!PageLocked(found))
 233                         return found;
 234                 __free_page(found);
 235                 __wait_on_page(found);
 236         }
 237
 238 out_bad:
 239         printk (KERN_ERR "VM: Found a non-swapper swap page!\n");
 240         __free_page(found);
 241         return 0;
 242 }
 243
 244 /*
 245  * Locate a page of swap in physical memory, reserving swap cache space
 246  * and reading the disk if it is not already cached.  If wait==0, we are
 247  * only doing readahead, so don't worry if the page is already locked.
 248  *
 249  * A failure return means that either the page allocation failed or that
 250  * the swap entry is no longer in use.
 251  */
 252
 253 struct page * read_swap_cache_async(unsigned long entry, int wait)
 254 {
 255         struct page *found_page, *new_page;
 256         unsigned long new_page_addr;
 257
 258 #ifdef DEBUG_SWAP
 259         printk("DebugVM: read_swap_cache_async entry %08lx%s\n",
 260                entry, wait ? ", wait" : "");
 261 #endif
 262         /*
 263          * Look for the page in the swap cache.
 264          */
 265         found_page = lookup_swap_cache(entry);
 266         if (found_page)
 267                 goto out;
 268
 269         new_page_addr = __get_free_page(GFP_KERNEL);
 270         if (!new_page_addr)
 271                 goto out;       /* Out of memory */
 272         new_page = mem_map + MAP_NR(new_page_addr);
 273
 274         /*
 275          * Check the swap cache again, in case we stalled above.
 276          */
 277         found_page = lookup_swap_cache(entry);
 278         if (found_page)
 279                 goto out_free_page;
 280         /*
 281          * Make sure the swap entry is still in use.
 282          */
 283         if (!swap_duplicate(entry))     /* Account for the swap cache */
 284                 goto out_free_page;
 285         /*
 286          * Add it to the swap cache and read its contents.
 287          */
 288         if (!add_to_swap_cache(new_page, entry))
 289                 goto out_free_page;
 290
 291         set_bit(PG_locked, &new_page->flags);
 292         rw_swap_page(READ, entry, (char *) new_page_addr, wait);
 293 #ifdef DEBUG_SWAP
 294         printk("DebugVM: read_swap_cache_async created "
 295                "entry %08lx at %p\n",
 296                entry, (char *) page_address(new_page));
 297 #endif
 298         return new_page;
 299
 300 out_free_page:
 301         __free_page(new_page);
 302 out:
 303         return found_page;
 304 }