mm/swap_state.c

   1 /*
   2  *  linux/mm/swap_state.c
   3  *
   4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   5  *  Swap reorganised 29.12.95, Stephen Tweedie
   6  *
   7  *  Rewritten to use page cache, (C) 1998 Stephen Tweedie
   8  */
   9
  10 #include <linux/mm.h>
  11 #include <linux/kernel_stat.h>
  12 #include <linux/swap.h>
  13 #include <linux/swapctl.h>
  14 #include <linux/init.h>
  15 #include <linux/pagemap.h>
  16
  17 #include <asm/pgtable.h>
  18
  19 /*
  20  * Keep a reserved false inode which we will use to mark pages in the
  21  * page cache are acting as swap cache instead of file cache.
  22  *
  23  * We only need a unique pointer to satisfy the page cache, but we'll
  24  * reserve an entire zeroed inode structure for the purpose just to
  25  * ensure that any mistaken dereferences of this structure cause a
  26  * kernel oops.
  27  */
  28
  29 static struct inode_operations swapper_inode_operations = {
  30         NULL,                           /* default file operations */
  31         NULL,                           /* create */
  32         NULL,                           /* lookup */
  33         NULL,                           /* link */
  34         NULL,                           /* unlink */
  35         NULL,                           /* symlink */
  36         NULL,                           /* mkdir */
  37         NULL,                           /* rmdir */
  38         NULL,                           /* mknod */
  39         NULL,                           /* rename */
  40         NULL,                           /* readlink */
  41         NULL,                           /* follow_link */
  42         NULL,                           /* get_block */
  43         NULL,                           /* readpage */
  44         NULL,                           /* writepage */
  45         block_flushpage,                /* flushpage */
  46         NULL,                           /* truncate */
  47         NULL,                           /* permission */
  48         NULL,                           /* smap */
  49         NULL                            /* revalidate */
  50 };
  51
  52 struct inode swapper_inode = { i_op: &swapper_inode_operations };
  53
  54 #ifdef SWAP_CACHE_INFO
  55 unsigned long swap_cache_add_total = 0;
  56 unsigned long swap_cache_del_total = 0;
  57 unsigned long swap_cache_find_total = 0;
  58 unsigned long swap_cache_find_success = 0;
  59
  60 void show_swap_cache_info(void)
  61 {
  62         printk("Swap cache: add %ld, delete %ld, find %ld/%ld\n",
  63                 swap_cache_add_total,
  64                 swap_cache_del_total,
  65                 swap_cache_find_success, swap_cache_find_total);
  66 }
  67 #endif
  68
  69 void add_to_swap_cache(struct page *page, unsigned long entry)
  70 {
  71 #ifdef SWAP_CACHE_INFO
  72         swap_cache_add_total++;
  73 #endif
  74 #ifdef DEBUG_SWAP
  75         printk("DebugVM: add_to_swap_cache(%08lx count %d, entry %08lx)\n",
  76                    page_address(page), page_count(page), entry);
  77 #endif
  78         if (PageTestandSetSwapCache(page)) {
  79                 printk(KERN_ERR "swap_cache: replacing non-empty entry %08lx "
  80                            "on page %08lx\n",
  81                            page->offset, page_address(page));
  82         }
  83         if (page->inode) {
  84                 printk(KERN_ERR "swap_cache: replacing page-cached entry "
  85                            "on page %08lx\n", page_address(page));
  86         }
  87         add_to_page_cache(page, &swapper_inode, entry);
  88 }
  89
  90 /*
  91  * Verify that a swap entry is valid and increment its swap map count.
  92  *
  93  * Note: if swap_map[] reaches SWAP_MAP_MAX the entries are treated as
  94  * "permanent", but will be reclaimed by the next swapoff.
  95  */
  96 int swap_duplicate(unsigned long entry)
  97 {
  98         struct swap_info_struct * p;
  99         unsigned long offset, type;
 100         int result = 0;
 101
 102         if (!entry)
 103                 goto out;
 104         type = SWP_TYPE(entry);
 105         if (type & SHM_SWP_TYPE)
 106                 goto out;
 107         if (type >= nr_swapfiles)
 108                 goto bad_file;
 109         p = type + swap_info;
 110         offset = SWP_OFFSET(entry);
 111         if (offset >= p->max)
 112                 goto bad_offset;
 113         if (!p->swap_map[offset])
 114                 goto bad_unused;
 115         /*
 116          * Entry is valid, so increment the map count.
 117          */
 118         if (p->swap_map[offset] < SWAP_MAP_MAX)
 119                 p->swap_map[offset]++;
 120         else {
 121                 static int overflow = 0;
 122                 if (overflow++ < 5)
 123                         printk(KERN_WARNING
 124                                 "swap_duplicate: entry %08lx map count=%d\n",
 125                                 entry, p->swap_map[offset]);
 126                 p->swap_map[offset] = SWAP_MAP_MAX;
 127         }
 128         result = 1;
 129 #ifdef DEBUG_SWAP
 130         printk("DebugVM: swap_duplicate(entry %08lx, count now %d)\n",
 131                    entry, p->swap_map[offset]);
 132 #endif
 133 out:
 134         return result;
 135
 136 bad_file:
 137         printk(KERN_ERR
 138                 "swap_duplicate: entry %08lx, nonexistent swap file\n", entry);
 139         goto out;
 140 bad_offset:
 141         printk(KERN_ERR
 142                 "swap_duplicate: entry %08lx, offset exceeds max\n", entry);
 143         goto out;
 144 bad_unused:
 145         printk(KERN_ERR
 146                 "swap_duplicate at %8p: entry %08lx, unused page\n",
 147                    __builtin_return_address(0), entry);
 148         goto out;
 149 }
 150
 151 int swap_count(unsigned long entry)
 152 {
 153         struct swap_info_struct * p;
 154         unsigned long offset, type;
 155         int retval = 0;
 156
 157         if (!entry)
 158                 goto bad_entry;
 159         type = SWP_TYPE(entry);
 160         if (type & SHM_SWP_TYPE)
 161                 goto out;
 162         if (type >= nr_swapfiles)
 163                 goto bad_file;
 164         p = type + swap_info;
 165         offset = SWP_OFFSET(entry);
 166         if (offset >= p->max)
 167                 goto bad_offset;
 168         if (!p->swap_map[offset])
 169                 goto bad_unused;
 170         retval = p->swap_map[offset];
 171 #ifdef DEBUG_SWAP
 172         printk("DebugVM: swap_count(entry %08lx, count %d)\n",
 173                    entry, retval);
 174 #endif
 175 out:
 176         return retval;
 177
 178 bad_entry:
 179         printk(KERN_ERR "swap_count: null entry!\n");
 180         goto out;
 181 bad_file:
 182         printk(KERN_ERR
 183                    "swap_count: entry %08lx, nonexistent swap file!\n", entry);
 184         goto out;
 185 bad_offset:
 186         printk(KERN_ERR
 187                    "swap_count: entry %08lx, offset exceeds max!\n", entry);
 188         goto out;
 189 bad_unused:
 190         printk(KERN_ERR
 191                    "swap_count at %8p: entry %08lx, unused page!\n",
 192                    __builtin_return_address(0), entry);
 193         goto out;
 194 }
 195
 196 static inline void remove_from_swap_cache(struct page *page)
 197 {
 198         struct inode *inode = page->inode;
 199
 200         if (!inode) {
 201                 printk ("VM: Removing swap cache page with zero inode hash "
 202                         "on page %08lx\n", page_address(page));
 203                 return;
 204         }
 205         if (inode != &swapper_inode) {
 206                 printk ("VM: Removing swap cache page with wrong inode hash "
 207                         "on page %08lx\n", page_address(page));
 208         }
 209         if (!PageSwapCache(page))
 210                 PAGE_BUG(page);
 211
 212 #ifdef DEBUG_SWAP
 213         printk("DebugVM: remove_from_swap_cache(%08lx count %d)\n",
 214                    page_address(page), page_count(page));
 215 #endif
 216         PageClearSwapCache(page);
 217         if (inode->i_op->flushpage)
 218                 inode->i_op->flushpage(inode, page, 0);
 219         remove_inode_page(page);
 220 }
 221
 222 /*
 223  * This must be called only on pages that have
 224  * been verified to be in the swap cache.
 225  */
 226 void __delete_from_swap_cache(struct page *page)
 227 {
 228         long entry = page->offset;
 229
 230 #ifdef SWAP_CACHE_INFO
 231         swap_cache_del_total++;
 232 #endif
 233 #ifdef DEBUG_SWAP
 234         printk("DebugVM: delete_from_swap_cache(%08lx count %d, "
 235                    "entry %08lx)\n",
 236                    page_address(page), page_count(page), entry);
 237 #endif
 238         remove_from_swap_cache (page);
 239         swap_free (entry);
 240 }
 241
 242 /*
 243  * This must be called only on pages that have
 244  * been verified to be in the swap cache.
 245  */
 246 void delete_from_swap_cache(struct page *page)
 247 {
 248         lock_page(page);
 249
 250         __delete_from_swap_cache(page);
 251
 252         UnlockPage(page);
 253         page_cache_release(page);
 254 }
 255
 256 /*
 257  * Perform a free_page(), also freeing any swap cache associated with
 258  * this page if it is the last user of the page.
 259  */
 260
 261 void free_page_and_swap_cache(unsigned long addr)
 262 {
 263         struct page *page = mem_map + MAP_NR(addr);
 264
 265         /*
 266          * If we are the only user, then free up the swap cache.
 267          */
 268         lock_page(page);
 269         if (PageSwapCache(page) && !is_page_shared(page)) {
 270                 long entry = page->offset;
 271                 remove_from_swap_cache(page);
 272                 swap_free(entry);
 273                 page_cache_release(page);
 274         }
 275         UnlockPage(page);
 276
 277         __free_page(page);
 278 }
 279
 280
 281 /*
 282  * Lookup a swap entry in the swap cache. A found page will be returned
 283  * unlocked and with its refcount incremented - we rely on the kernel
 284  * lock getting page table operations atomic even if we drop the page
 285  * lock before returning.
 286  */
 287
 288 struct page * lookup_swap_cache(unsigned long entry)
 289 {
 290         struct page *found;
 291
 292 #ifdef SWAP_CACHE_INFO
 293         swap_cache_find_total++;
 294 #endif
 295         while (1) {
 296                 found = find_lock_page(&swapper_inode, entry);
 297                 if (!found)
 298                         return 0;
 299                 if (found->inode != &swapper_inode || !PageSwapCache(found))
 300                         goto out_bad;
 301 #ifdef SWAP_CACHE_INFO
 302                 swap_cache_find_success++;
 303 #endif
 304                 UnlockPage(found);
 305                 return found;
 306         }
 307
 308 out_bad:
 309         printk (KERN_ERR "VM: Found a non-swapper swap page!\n");
 310         UnlockPage(found);
 311         __free_page(found);
 312         return 0;
 313 }
 314
 315 /*
 316  * Locate a page of swap in physical memory, reserving swap cache space
 317  * and reading the disk if it is not already cached.  If wait==0, we are
 318  * only doing readahead, so don't worry if the page is already locked.
 319  *
 320  * A failure return means that either the page allocation failed or that
 321  * the swap entry is no longer in use.
 322  */
 323
 324 struct page * read_swap_cache_async(unsigned long entry, int wait)
 325 {
 326         struct page *found_page = 0, *new_page;
 327         unsigned long new_page_addr;
 328
 329 #ifdef DEBUG_SWAP
 330         printk("DebugVM: read_swap_cache_async entry %08lx%s\n",
 331                    entry, wait ? ", wait" : "");
 332 #endif
 333         /*
 334          * Make sure the swap entry is still in use.
 335          */
 336         if (!swap_duplicate(entry))     /* Account for the swap cache */
 337                 goto out;
 338         /*
 339          * Look for the page in the swap cache.
 340          */
 341         found_page = lookup_swap_cache(entry);
 342         if (found_page)
 343                 goto out_free_swap;
 344
 345         new_page_addr = __get_free_page(GFP_USER);
 346         if (!new_page_addr)
 347                 goto out_free_swap;     /* Out of memory */
 348         new_page = mem_map + MAP_NR(new_page_addr);
 349
 350         /*
 351          * Check the swap cache again, in case we stalled above.
 352          */
 353         found_page = lookup_swap_cache(entry);
 354         if (found_page)
 355                 goto out_free_page;
 356         /*
 357          * Add it to the swap cache and read its contents.
 358          */
 359         add_to_swap_cache(new_page, entry);
 360         rw_swap_page(READ, new_page, wait);
 361 #ifdef DEBUG_SWAP
 362         printk("DebugVM: read_swap_cache_async created "
 363                    "entry %08lx at %p\n",
 364                         entry, (char *) page_address(new_page));
 365 #endif
 366         return new_page;
 367
 368 out_free_page:
 369         __free_page(new_page);
 370 out_free_swap:
 371         swap_free(entry);
 372 out:
 373         return found_page;
 374 }