sys/vm/vm_page2.h

   1 /*-
   2  * Copyright (c) 1982, 1986, 1993
   3  *      The Regents of the University of California.  All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. Neither the name of the University nor the names of its contributors
  14  *    may be used to endorse or promote products derived from this software
  15  *    without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  *      @(#)vmmeter.h   8.2 (Berkeley) 7/10/94
  30  * $FreeBSD: src/sys/sys/vmmeter.h,v 1.21.2.2 2002/10/10 19:28:21 dillon Exp $
  31  */
  32
  33 #ifndef _VM_VM_PAGE2_H_
  34 #define _VM_VM_PAGE2_H_
  35
  36 #ifdef _KERNEL
  37
  38 #ifndef _SYS_VMMETER_H_
  39 #include <sys/vmmeter.h>
  40 #endif
  41 #ifndef _SYS_QUEUE_H_
  42 #include <sys/queue.h>
  43 #endif
  44 #ifndef _VM_VM_PAGE_H_
  45 #include <vm/vm_page.h>
  46 #endif
  47 #ifndef _SYS_SPINLOCK_H_
  48 #include <sys/spinlock.h>
  49 #endif
  50 #ifndef _SYS_SPINLOCK2_H_
  51 #include <sys/spinlock2.h>
  52 #endif
  53
  54 /*
  55  * SMP NOTE
  56  *
  57  * VM fault rates are highly dependent on SMP locking conflicts and, on
  58  * multi-socket systems, cache mastership changes for globals due to atomic
  59  * ops (even simple atomic_add_*() calls).  Cache mastership changes can
  60  * limit the aggregate fault rate.
  61  *
  62  * For this reason we go through some hoops to access VM statistics for
  63  * low-memory handling, pageout, and other triggers.  Each cpu collects
  64  * adjustments in gd->gd_vmstats_adj.  These get rolled up into the global
  65  * vmstats structure.  The global vmstats structure is then pulled into
  66  * gd->gd_vmstats by each cpu when it needs it.  Critical path checks always
  67  * use the pcpu gd->gd_vmstats structure.
  68  */
  69 /*
  70  * Return TRUE if we are under our severe low-free-pages threshold
  71  *
  72  * This causes user processes to stall to avoid exhausting memory that
  73  * the kernel might need.
  74  *
  75  * reserved < severe < minimum < target < paging_target
  76  */
  77 static __inline
  78 int
  79 vm_page_count_severe(void)
  80 {
  81     globaldata_t gd = mycpu;
  82
  83     return (gd->gd_vmstats.v_free_severe >
  84             gd->gd_vmstats.v_free_count + gd->gd_vmstats.v_cache_count ||
  85             gd->gd_vmstats.v_free_reserved > gd->gd_vmstats.v_free_count);
  86 }
  87
  88 /*
  89  * Return TRUE if we are under our minimum low-free-pages threshold.
  90  * This activates the pageout demon.  The pageout demon tries to
  91  * reach the target but may stop once it satisfies the minimum.
  92  *
  93  * reserved < severe < minimum < target < paging_target
  94  */
  95 static __inline
  96 int
  97 vm_page_count_min(long donotcount)
  98 {
  99     globaldata_t gd = mycpu;
 100
 101     return (gd->gd_vmstats.v_free_min + donotcount >
 102             (gd->gd_vmstats.v_free_count + gd->gd_vmstats.v_cache_count) ||
 103             gd->gd_vmstats.v_free_reserved > gd->gd_vmstats.v_free_count);
 104 }
 105
 106 /*
 107  * Return TRUE if we are under our free page target.  The pageout demon
 108  * tries to reach the target but may stop once it gets past the min.
 109  *
 110  * User threads doing normal allocations might wait based on this
 111  * function but MUST NOT wait in a loop based on this function as the
 112  * VM load may prevent the target from being reached.
 113  */
 114 static __inline
 115 int
 116 vm_page_count_target(void)
 117 {
 118     globaldata_t gd = mycpu;
 119
 120     return (gd->gd_vmstats.v_free_target >
 121             (gd->gd_vmstats.v_free_count + gd->gd_vmstats.v_cache_count) ||
 122             gd->gd_vmstats.v_free_reserved > gd->gd_vmstats.v_free_count);
 123 }
 124
 125 /*
 126  * Return the number of pages the pageout daemon needs to move into the
 127  * cache or free lists.  A negative number means we have sufficient free
 128  * pages.
 129  *
 130  * The target free+cache is greater than vm_page_count_target().  The
 131  * frontend uses vm_page_count_target() while the backend continue freeing
 132  * based on vm_paging_target().
 133  *
 134  * This function DOES NOT return TRUE or FALSE.
 135  */
 136 static __inline
 137 long
 138 vm_paging_target(void)
 139 {
 140     globaldata_t gd = mycpu;
 141
 142     return ((gd->gd_vmstats.v_free_target + gd->gd_vmstats.v_cache_min) -
 143             (gd->gd_vmstats.v_free_count + gd->gd_vmstats.v_cache_count));
 144 }
 145
 146 /*
 147  * Return TRUE if hysteresis dictates we should nominally wakeup the
 148  * pageout daemon to start working on freeing up some memory.  This
 149  * routine should NOT be used to determine when to block on the VM system.
 150  * We want to wakeup the pageout daemon before we might otherwise block.
 151  *
 152  * Paging begins when cache+free drops below cache_min + free_min.
 153  */
 154 static __inline
 155 int
 156 vm_paging_needed(void)
 157 {
 158     globaldata_t gd = mycpu;
 159
 160     if (gd->gd_vmstats.v_free_min + gd->gd_vmstats.v_cache_min >
 161         gd->gd_vmstats.v_free_count + gd->gd_vmstats.v_cache_count) {
 162                 return 1;
 163     }
 164     if (gd->gd_vmstats.v_free_min > gd->gd_vmstats.v_free_count)
 165                 return 1;
 166     return 0;
 167 }
 168
 169 /*
 170  * Clear dirty bits in the VM page but truncate the
 171  * end to a DEV_BSIZE'd boundary.
 172  *
 173  * Used when reading data in, typically via getpages.
 174  * The partial device block at the end of the truncation
 175  * range should not lose its dirty bit.
 176  *
 177  * NOTE: This function does not clear the pmap modified bit.
 178  */
 179 static __inline
 180 void
 181 vm_page_clear_dirty_end_nonincl(vm_page_t m, int base, int size)
 182 {
 183     size = (base + size) & ~DEV_BMASK;
 184     if (base < size)
 185         vm_page_clear_dirty(m, base, size - base);
 186 }
 187
 188 /*
 189  * Clear dirty bits in the VM page but truncate the
 190  * beginning to a DEV_BSIZE'd boundary.
 191  *
 192  * Used when truncating a buffer.  The partial device
 193  * block at the beginning of the truncation range
 194  * should not lose its dirty bit.
 195  *
 196  * NOTE: This function does not clear the pmap modified bit.
 197  */
 198 static __inline
 199 void
 200 vm_page_clear_dirty_beg_nonincl(vm_page_t m, int base, int size)
 201 {
 202     size += base;
 203     base = (base + DEV_BMASK) & ~DEV_BMASK;
 204     if (base < size)
 205         vm_page_clear_dirty(m, base, size - base);
 206 }
 207
 208 static __inline
 209 void
 210 vm_page_spin_lock(vm_page_t m)
 211 {
 212     spin_lock(&m->spin);
 213 }
 214
 215 static __inline
 216 void
 217 vm_page_spin_unlock(vm_page_t m)
 218 {
 219     spin_unlock(&m->spin);
 220 }
 221
 222 /*
 223  * Wire a vm_page that is already wired.  Does not require a busied
 224  * page.
 225  */
 226 static __inline
 227 void
 228 vm_page_wire_quick(vm_page_t m)
 229 {
 230     if (atomic_fetchadd_int(&m->wire_count, 1) == 0)
 231         panic("vm_page_wire_quick: wire_count was 0");
 232 }
 233
 234 /*
 235  * Unwire a vm_page quickly, does not require a busied page.
 236  *
 237  * This routine refuses to drop the wire_count to 0 and will return
 238  * TRUE if it would have had to (instead of decrementing it to 0).
 239  * The caller can then busy the page and deal with it.
 240  */
 241 static __inline
 242 int
 243 vm_page_unwire_quick(vm_page_t m)
 244 {
 245     KKASSERT(m->wire_count > 0);
 246     for (;;) {
 247         u_int wire_count = m->wire_count;
 248
 249         cpu_ccfence();
 250         if (wire_count == 1)
 251                 return TRUE;
 252         if (atomic_cmpset_int(&m->wire_count, wire_count, wire_count - 1))
 253                 return FALSE;
 254     }
 255 }
 256
 257 /*
 258  *      Functions implemented as macros
 259  */
 260
 261 static __inline void
 262 vm_page_flag_set(vm_page_t m, unsigned int bits)
 263 {
 264         atomic_set_int(&(m)->flags, bits);
 265 }
 266
 267 static __inline void
 268 vm_page_flag_clear(vm_page_t m, unsigned int bits)
 269 {
 270         atomic_clear_int(&(m)->flags, bits);
 271 }
 272
 273 /*
 274  * Wakeup anyone waiting for the page after potentially unbusying
 275  * (hard or soft) or doing other work on a page that might make a
 276  * waiter ready.  The setting of PBUSY_WANTED is integrated into the
 277  * related flags and it can't be set once the flags are already
 278  * clear, so there should be no races here.
 279  */
 280 static __inline void
 281 vm_page_flash(vm_page_t m)
 282 {
 283         if (m->busy_count & PBUSY_WANTED) {
 284                 atomic_clear_int(&m->busy_count, PBUSY_WANTED);
 285                 wakeup(m);
 286         }
 287 }
 288
 289 /*
 290  * Adjust the soft-busy count on a page.  The drop code will issue an
 291  * integrated wakeup if busy_count becomes 0.
 292  */
 293 static __inline void
 294 vm_page_sbusy_hold(vm_page_t m)
 295 {
 296         atomic_add_int(&m->busy_count, 1);
 297 }
 298
 299 static __inline void
 300 vm_page_sbusy_drop(vm_page_t m)
 301 {
 302         uint32_t ocount;
 303
 304         ocount = atomic_fetchadd_int(&m->busy_count, -1);
 305         if (ocount - 1 == PBUSY_WANTED) {
 306                 /* WANTED and no longer BUSY or SBUSY */
 307                 atomic_clear_int(&m->busy_count, PBUSY_WANTED);
 308                 wakeup(m);
 309         }
 310 }
 311
 312 /*
 313  * Reduce the protection of a page.  This routine never raises the
 314  * protection and therefore can be safely called if the page is already
 315  * at VM_PROT_NONE (it will be a NOP effectively ).
 316  *
 317  * VM_PROT_NONE will remove all user mappings of a page.  This is often
 318  * necessary when a page changes state (for example, turns into a copy-on-write
 319  * page or needs to be frozen for write I/O) in order to force a fault, or
 320  * to force a page's dirty bits to be synchronized and avoid hardware
 321  * (modified/accessed) bit update races with pmap changes.
 322  *
 323  * Since 'prot' is usually a constant, this inline usually winds up optimizing
 324  * out the primary conditional.
 325  *
 326  * WARNING: VM_PROT_NONE can block, but will loop until all mappings have
 327  * been cleared.  Callers should be aware that other page related elements
 328  * might have changed, however.
 329  */
 330 static __inline void
 331 vm_page_protect(vm_page_t m, int prot)
 332 {
 333         KKASSERT(m->busy_count & PBUSY_LOCKED);
 334         if (prot == VM_PROT_NONE) {
 335                 if (m->flags & (PG_WRITEABLE|PG_MAPPED)) {
 336                         pmap_page_protect(m, VM_PROT_NONE);
 337                         /* PG_WRITEABLE & PG_MAPPED cleared by call */
 338                 }
 339         } else if ((prot == VM_PROT_READ) && (m->flags & PG_WRITEABLE)) {
 340                 pmap_page_protect(m, VM_PROT_READ);
 341                 /* PG_WRITEABLE cleared by call */
 342         }
 343 }
 344
 345 /*
 346  * Zero-fill the specified page.  The entire contents of the page will be
 347  * zero'd out.
 348  */
 349 static __inline boolean_t
 350 vm_page_zero_fill(vm_page_t m)
 351 {
 352         pmap_zero_page(VM_PAGE_TO_PHYS(m));
 353         return (TRUE);
 354 }
 355
 356 /*
 357  * Copy the contents of src_m to dest_m.  The pages must be stable but spl
 358  * and other protections depend on context.
 359  */
 360 static __inline void
 361 vm_page_copy(vm_page_t src_m, vm_page_t dest_m)
 362 {
 363         pmap_copy_page(VM_PAGE_TO_PHYS(src_m), VM_PAGE_TO_PHYS(dest_m));
 364         dest_m->valid = VM_PAGE_BITS_ALL;
 365         dest_m->dirty = VM_PAGE_BITS_ALL;
 366 }
 367
 368 /*
 369  * Free a page.  The page must be marked BUSY.
 370  */
 371 static __inline void
 372 vm_page_free(vm_page_t m)
 373 {
 374         vm_page_free_toq(m);
 375 }
 376
 377 /*
 378  * Free a page to the zerod-pages queue.  The caller must ensure that the
 379  * page has been zerod.
 380  */
 381 static __inline void
 382 vm_page_free_zero(vm_page_t m)
 383 {
 384 #ifdef PMAP_DEBUG
 385 #ifdef PHYS_TO_DMAP
 386         char *p = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 387         int i;
 388
 389         for (i = 0; i < PAGE_SIZE; i++) {
 390                 if (p[i] != 0) {
 391                         panic("non-zero page in vm_page_free_zero()");
 392                 }
 393         }
 394 #endif
 395 #endif
 396         vm_page_free_toq(m);
 397 }
 398
 399 /*
 400  * Set page to not be dirty.  Note: does not clear pmap modify bits .
 401  */
 402 static __inline void
 403 vm_page_undirty(vm_page_t m)
 404 {
 405         m->dirty = 0;
 406 }
 407
 408 #endif  /* _KERNEL */
 409 #endif  /* _VM_VM_PAGE2_H_ */
 410