sys/platform/vkernel64/platform/pmap_inval.c

   1 /*
   2  * Copyright (c) 2003,2004 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/platform/vkernel/platform/pmap_inval.c,v 1.4 2007/07/02 02:22:58 dillon Exp $
  35  */
  36
  37 /*
  38  * pmap invalidation support code.  Certain hardware requirements must
  39  * be dealt with when manipulating page table entries and page directory
  40  * entries within a pmap.  In particular, we cannot safely manipulate
  41  * page tables which are in active use by another cpu (even if it is
  42  * running in userland) for two reasons: First, TLB writebacks will
  43  * race against our own modifications and tests.  Second, even if we
  44  * were to use bus-locked instruction we can still screw up the
  45  * target cpu's instruction pipeline due to Intel cpu errata.
  46  *
  47  * For our virtual page tables, the real kernel will handle SMP interactions
  48  * with pmaps that may be active on other cpus.  Even so, we have to be
  49  * careful about bit setting races particularly when we are trying to clean
  50  * a page and test the modified bit to avoid races where the modified bit
  51  * might get set after our poll but before we clear the field.
  52  */
  53 #include <sys/param.h>
  54 #include <sys/systm.h>
  55 #include <sys/kernel.h>
  56 #include <sys/proc.h>
  57 #include <sys/vmmeter.h>
  58 #include <sys/thread2.h>
  59 #include <sys/cdefs.h>
  60 #include <sys/mman.h>
  61 #include <sys/vmspace.h>
  62 #include <sys/vmm.h>
  63
  64 #include <vm/vm.h>
  65 #include <vm/pmap.h>
  66 #include <vm/vm_object.h>
  67
  68 #include <machine/cputypes.h>
  69 #include <machine/md_var.h>
  70 #include <machine/specialreg.h>
  71 #include <machine/smp.h>
  72 #include <machine/globaldata.h>
  73 #include <machine/pmap.h>
  74 #include <machine/pmap_inval.h>
  75
  76 #include <unistd.h>
  77 #include <pthread.h>
  78
  79 extern int vmm_enabled;
  80
  81 static __inline
  82 void
  83 vmm_cpu_invltlb(void)
  84 {
  85         /* For VMM mode forces vmmexit/resume */
  86         uint64_t rax = -1;
  87         __asm __volatile("syscall;"
  88                         :
  89                         : "a" (rax)
  90                         :);
  91 }
  92
  93 /*
  94  * Invalidate va in the TLB on the current cpu
  95  */
  96 static __inline
  97 void
  98 pmap_inval_cpu(struct pmap *pmap, vm_offset_t va, size_t bytes)
  99 {
 100         if (pmap == &kernel_pmap) {
 101                 madvise((void *)va, bytes, MADV_INVAL);
 102         } else {
 103                 vmspace_mcontrol(pmap, (void *)va, bytes, MADV_INVAL, 0);
 104         }
 105 }
 106
 107 /*
 108  * This is a bit of a mess because we don't know what virtual cpus are
 109  * mapped to real cpus.  Basically try to optimize the degenerate cases
 110  * (primarily related to user processes with only one thread or only one
 111  * running thread), and shunt all the rest to the host cpu.  The host cpu
 112  * will invalidate all real cpu's the vkernel is running on.
 113  *
 114  * This can't optimize situations where a pmap is only mapped to some of
 115  * the virtual cpus, though shunting to the real host will still be faster
 116  * if the virtual kernel processes are running on fewer real-host cpus.
 117  * (And probably will be faster anyway since there's no round-trip signaling
 118  * overhead).
 119  *
 120  * NOTE: The critical section protects against preemption while the pmap
 121  *       is locked, which could otherwise result in a deadlock.
 122  */
 123 static __inline
 124 void
 125 guest_sync_addr(struct pmap *pmap,
 126                 volatile vpte_t *dst_ptep, volatile vpte_t *src_ptep)
 127 {
 128         globaldata_t gd = mycpu;
 129         cpulock_t olock;
 130         cpulock_t nlock;
 131
 132         /*
 133          * Lock the pmap
 134          */
 135         crit_enter();
 136         for (;;) {
 137                 olock = pmap->pm_active_lock;
 138                 cpu_ccfence();
 139                 if ((olock & CPULOCK_EXCL) == 0) {
 140                         nlock = olock | CPULOCK_EXCL;
 141                         if (atomic_cmpset_int(&pmap->pm_active_lock,
 142                                               olock, nlock)) {
 143                                 break;
 144                         }
 145                 }
 146                 cpu_pause();
 147                 lwkt_process_ipiq();
 148                 pthread_yield();
 149         }
 150
 151         /*
 152          * Update the pte and synchronize with other cpus.  If we can update
 153          * it trivially, do so.
 154          */
 155         if (CPUMASK_TESTZERO(pmap->pm_active) ||
 156             CPUMASK_CMPMASKEQ(pmap->pm_active, gd->gd_cpumask)) {
 157                 if (src_ptep)
 158                         *dst_ptep = *src_ptep;
 159                 vmm_cpu_invltlb();
 160         } else {
 161                 vmm_guest_sync_addr(__DEVOLATILE(void *, dst_ptep),
 162                                     __DEVOLATILE(void *, src_ptep));
 163         }
 164
 165         /*
 166          * Unlock the pmap
 167          */
 168         atomic_clear_int(&pmap->pm_active_lock, CPULOCK_EXCL);
 169         crit_exit();
 170 }
 171
 172 /*
 173  * Invalidate a pte in a pmap and synchronize with target cpus
 174  * as required.  Throw away the modified and access bits.  Use
 175  * pmap_clean_pte() to do the same thing but also get an interlocked
 176  * modified/access status.
 177  *
 178  * Clearing the field first (basically clearing VPTE_V) prevents any
 179  * new races from occuring while we invalidate the TLB (i.e. the pmap
 180  * on the real cpu), then clear it again to clean out any race that
 181  * might have occured before the invalidation completed.
 182  */
 183 void
 184 pmap_inval_pte(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va)
 185 {
 186         vpte_t pte;
 187
 188         if (vmm_enabled == 0) {
 189                 *ptep = 0;
 190                 pmap_inval_cpu(pmap, va, PAGE_SIZE);
 191         } else {
 192                 pte = 0;
 193                 guest_sync_addr(pmap, ptep, &pte);
 194         }
 195 }
 196
 197 /*
 198  * Invalidate the tlb for a range of virtual addresses across all cpus
 199  * belonging to the pmap.
 200  */
 201 void
 202 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 203 {
 204         if (vmm_enabled == 0) {
 205                 pmap_inval_cpu(pmap, sva, eva - sva);
 206         } else {
 207                 guest_sync_addr(pmap, NULL, NULL);
 208         }
 209 }
 210
 211 /*
 212  * Same as pmap_inval_pte() but only synchronize with the current
 213  * cpu.  For the moment its the same as the non-quick version.
 214  */
 215 void
 216 pmap_inval_pte_quick(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va)
 217 {
 218         *ptep = 0;
 219         if (vmm_enabled)
 220                 vmm_cpu_invltlb();
 221         else
 222                 pmap_inval_cpu(pmap, va, PAGE_SIZE);
 223 }
 224
 225 /*
 226  * Invalidating page directory entries requires some additional
 227  * sophistication.  The cachemask must be cleared so the kernel
 228  * resynchronizes its temporary page table mappings cache.
 229  */
 230 void
 231 pmap_inval_pde(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va)
 232 {
 233         vpte_t pte;
 234
 235         if (vmm_enabled == 0) {
 236                 *ptep = 0;
 237                 pmap_inval_cpu(pmap, va, SEG_SIZE);
 238         } else if (CPUMASK_TESTMASK(pmap->pm_active,
 239                                     mycpu->gd_other_cpus) == 0) {
 240                 *ptep = 0;
 241                 vmm_cpu_invltlb();
 242         } else {
 243                 pte = 0;
 244                 guest_sync_addr(pmap, ptep, &pte);
 245         }
 246 }
 247
 248 void
 249 pmap_inval_pde_quick(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va)
 250 {
 251         pmap_inval_pde(ptep, pmap, va);
 252 }
 253
 254 /*
 255  * These carefully handle interactions with other cpus and return
 256  * the original vpte.  Clearing VPTE_RW prevents us from racing the
 257  * setting of VPTE_M, allowing us to invalidate the tlb (the real cpu's
 258  * pmap) and get good status for VPTE_M.
 259  *
 260  * When messing with page directory entries we have to clear the cpu
 261  * mask to force a reload of the kernel's page table mapping cache.
 262  *
 263  * clean: clear VPTE_M and VPTE_RW
 264  * setro: clear VPTE_RW
 265  * load&clear: clear entire field
 266  */
 267 #include<stdio.h>
 268 vpte_t
 269 pmap_clean_pte(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va)
 270 {
 271         vpte_t pte;
 272
 273         pte = *ptep;
 274         if (pte & VPTE_V) {
 275                 atomic_clear_long(ptep, VPTE_RW);  /* XXX */
 276                 if (vmm_enabled == 0) {
 277                         pmap_inval_cpu(pmap, va, PAGE_SIZE);
 278                         pte = *ptep;
 279                 } else {
 280                         guest_sync_addr(pmap, &pte, ptep);
 281                 }
 282                 atomic_clear_long(ptep, VPTE_RW|VPTE_M);
 283         }
 284         return(pte);
 285 }
 286
 287 vpte_t
 288 pmap_clean_pde(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va)
 289 {
 290         vpte_t pte;
 291
 292         pte = *ptep;
 293         if (pte & VPTE_V) {
 294                 atomic_clear_long(ptep, VPTE_RW);
 295                 if (vmm_enabled == 0) {
 296                         pmap_inval_cpu(pmap, va, SEG_SIZE);
 297                         pte = *ptep;
 298                 } else {
 299                         guest_sync_addr(pmap, &pte, ptep);
 300                 }
 301                 atomic_clear_long(ptep, VPTE_RW|VPTE_M);
 302         }
 303         return(pte);
 304 }
 305
 306 /*
 307  * This is an odd case and I'm not sure whether it even occurs in normal
 308  * operation.  Turn off write access to the page, clean out the tlb
 309  * (the real cpu's pmap), and deal with any VPTE_M race that may have
 310  * occured.  VPTE_M is not cleared.
 311  */
 312 vpte_t
 313 pmap_setro_pte(volatile vpte_t *ptep, struct pmap *pmap, vm_offset_t va)
 314 {
 315         vpte_t pte;
 316         vpte_t npte;
 317
 318         pte = *ptep;
 319         if (pte & VPTE_V) {
 320                 atomic_clear_long(ptep, VPTE_RW);
 321                 if (vmm_enabled == 0) {
 322                         pmap_inval_cpu(pmap, va, PAGE_SIZE);
 323                         pte |= *ptep & VPTE_M;
 324                 } else {
 325                         guest_sync_addr(pmap, &npte, ptep);
 326                         pte |= npte & VPTE_M;
 327                 }
 328         }
 329         return(pte);
 330 }
 331
 332 /*
 333  * This is a combination of pmap_inval_pte() and pmap_clean_pte().
 334  * Firts prevent races with the 'A' and 'M' bits, then clean out
 335  * the tlb (the real cpu's pmap), then incorporate any races that
 336  * may have occured in the mean time, and finally zero out the pte.
 337  */
 338 vpte_t
 339 pmap_inval_loadandclear(volatile vpte_t *ptep, struct pmap *pmap,
 340                         vm_offset_t va)
 341 {
 342         vpte_t pte;
 343         vpte_t npte;
 344
 345         pte = *ptep;
 346         if (pte & VPTE_V) {
 347                 pte = *ptep;
 348                 atomic_clear_long(ptep, VPTE_RW);
 349                 if (vmm_enabled == 0) {
 350                         pmap_inval_cpu(pmap, va, PAGE_SIZE);
 351                         pte |= *ptep & (VPTE_A | VPTE_M);
 352                 } else {
 353                         guest_sync_addr(pmap, &npte, ptep);
 354                         pte |= npte & (VPTE_A | VPTE_M);
 355                 }
 356         }
 357         *ptep = 0;
 358         return(pte);
 359 }
 360
 361 void
 362 cpu_invlpg(void *addr)
 363 {
 364         if (vmm_enabled)
 365                 vmm_cpu_invltlb(); /* For VMM mode forces vmmexit/resume */
 366         else
 367                 madvise(addr, PAGE_SIZE, MADV_INVAL);
 368 }
 369
 370 void
 371 cpu_invltlb(void)
 372 {
 373         if (vmm_enabled)
 374                 vmm_cpu_invltlb(); /* For VMM mode forces vmmexit/resume */
 375         else
 376                 madvise((void *)KvaStart, KvaEnd - KvaStart, MADV_INVAL);
 377 }
 378
 379 void
 380 smp_invltlb(void)
 381 {
 382         /* XXX must invalidate the tlb on all cpus */
 383         /* at the moment pmap_inval_pte_quick */
 384         /* do nothing */
 385 }