sys/kern/kern_umtx.c

   1 /*
   2  * Copyright (c) 2003,2004,2010,2017 The DragonFly Project.
   3  * All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Matthew Dillon <dillon@backplane.com> and David Xu <davidxu@freebsd.org>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in
  16  *    the documentation and/or other materials provided with the
  17  *    distribution.
  18  * 3. Neither the name of The DragonFly Project nor the names of its
  19  *    contributors may be used to endorse or promote products derived
  20  *    from this software without specific, prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  26  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  27  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  28  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  29  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  30  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  31  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  32  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  33  * SUCH DAMAGE.
  34  */
  35
  36 /*
  37  * This module implements userland mutex helper functions.  umtx_sleep()
  38  * handling blocking and umtx_wakeup() handles wakeups.  The sleep/wakeup
  39  * functions operate on user addresses.
  40  */
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/cdefs.h>
  45 #include <sys/kernel.h>
  46 #include <sys/sysproto.h>
  47 #include <sys/sysunion.h>
  48 #include <sys/sysent.h>
  49 #include <sys/syscall.h>
  50 #include <sys/sysctl.h>
  51 #include <sys/module.h>
  52 #include <sys/thread.h>
  53 #include <sys/proc.h>
  54
  55 #include <cpu/lwbuf.h>
  56
  57 #include <vm/vm.h>
  58 #include <vm/vm_param.h>
  59 #include <sys/lock.h>
  60 #include <vm/pmap.h>
  61 #include <vm/vm_map.h>
  62 #include <vm/vm_object.h>
  63 #include <vm/vm_page.h>
  64 #include <vm/vm_pager.h>
  65 #include <vm/vm_pageout.h>
  66 #include <vm/vm_extern.h>
  67 #include <vm/vm_kern.h>
  68
  69 #include <vm/vm_page2.h>
  70
  71 #include <machine/vmm.h>
  72
  73 /*
  74  * Improve umtx performance by polling for 4000nS before going to sleep.
  75  * This can avoid many IPIs in typical pthreads mutex situations.
  76  */
  77 #ifdef _RDTSC_SUPPORTED_
  78 static int umtx_delay = 4000;           /* nS */
  79 SYSCTL_INT(_kern, OID_AUTO, umtx_delay, CTLFLAG_RW,
  80            &umtx_delay, 0, "");
  81 #endif
  82 static int umtx_timeout_max = 2000000;  /* microseconds */
  83 SYSCTL_INT(_kern, OID_AUTO, umtx_timeout_max, CTLFLAG_RW,
  84            &umtx_timeout_max, 0, "");
  85
  86 /*
  87  * If the contents of the userland-supplied pointer matches the specified
  88  * value enter an interruptable sleep for up to <timeout> microseconds.
  89  * If the contents does not match then return immediately.
  90  *
  91  * Returns 0 if we slept and were woken up, -1 and EWOULDBLOCK if we slept
  92  * and timed out, and EBUSY if the contents of the pointer already does
  93  * not match the specified value.  A timeout of 0 indicates an unlimited sleep.
  94  * EINTR is returned if the call was interrupted by a signal (even if
  95  * the signal specifies that the system call should restart).
  96  *
  97  * This function interlocks against call to umtx_wakeup.  It does NOT interlock
  98  * against changes in *ptr.  However, it does not have to.  The standard use
  99  * of *ptr is to differentiate between an uncontested and a contested mutex
 100  * and call umtx_wakeup when releasing a contested mutex.  Therefore we can
 101  * safely race against changes in *ptr as long as we are properly interlocked
 102  * against the umtx_wakeup() call.
 103  *
 104  * For performance reasons, we do not try to track the underlying page for
 105  * mapping changes.  Instead, the timeout is capped at kern.umtx_timeout_max
 106  * (default 1 second) and the caller is expected to retry.  The kernel
 107  * will wake all umtx_sleep()s if the process fork()s, but not if it vfork()s.
 108  * Other mapping changes must be caught by the timeout.
 109  *
 110  * umtx_sleep { const int *ptr, int value, int timeout }
 111  */
 112 int
 113 sys_umtx_sleep(struct umtx_sleep_args *uap)
 114 {
 115     void *waddr;
 116     void *uptr;
 117     int offset;
 118     int timeout;
 119     int error;
 120     int value;
 121     int fail_counter;
 122
 123     if (uap->timeout < 0)
 124         return (EINVAL);
 125
 126     if (curthread->td_vmm) {
 127         register_t gpa;
 128         vmm_vm_get_gpa(curproc, &gpa, (register_t)uap->ptr);
 129         uap->ptr = (const int *)gpa;
 130     }
 131
 132     uptr = __DEQUALIFY(void *, uap->ptr);
 133     if ((vm_offset_t)uptr & (sizeof(int) - 1))
 134         return EFAULT;
 135
 136     offset = (vm_offset_t)uptr & PAGE_MASK;
 137
 138     /*
 139      * Resolve the physical address.  We allow the case where there are
 140      * sometimes discontinuities (causing a 2 second retry timeout).
 141      */
 142 retry_on_discontinuity:
 143     fail_counter = 10000;
 144     do {
 145         if (--fail_counter == 0) {
 146                 kprintf("umtx_sleep() (X): ERROR Discontinuity %p (%s %d/%d)\n",
 147                         uptr, curthread->td_comm,
 148                         (int)curthread->td_proc->p_pid,
 149                         (int)curthread->td_lwp->lwp_tid);
 150                 return EINVAL;
 151         }
 152         value = fuword32(uptr);
 153         waddr = (void *)(intptr_t)uservtophys((intptr_t)uptr);
 154     } while (waddr == (void *)(intptr_t)-1 && value != -1);
 155
 156     if (value == -1 && waddr == (void *)(intptr_t)-1) {
 157         kprintf("umtx_sleep() (A): WARNING can't translate %p (%s %d/%d)\n",
 158                 uptr, curthread->td_comm,
 159                 (int)curthread->td_proc->p_pid,
 160                 (int)curthread->td_lwp->lwp_tid);
 161         return EINVAL;
 162     }
 163
 164     error = EBUSY;
 165     if (value == uap->value) {
 166 #ifdef _RDTSC_SUPPORTED_
 167         /*
 168          * Poll a little while before sleeping, most mutexes are
 169          * short-lived.
 170          */
 171         if (umtx_delay) {
 172                 int64_t tsc_target;
 173                 int good = 0;
 174
 175                 tsc_target = tsc_get_target(umtx_delay);
 176                 while (tsc_test_target(tsc_target) == 0) {
 177                         cpu_lfence();
 178                         if (fuword32(uptr) != uap->value) {
 179                                 good = 1;
 180                                 break;
 181                         }
 182                         cpu_pause();
 183                 }
 184                 if (good) {
 185                         error = EBUSY;
 186                         goto done;
 187                 }
 188         }
 189 #endif
 190         /*
 191          * Calculate the timeout.  This will be acccurate to within ~2 ticks.
 192          * uap->timeout is in microseconds.
 193          */
 194         timeout = umtx_timeout_max;
 195         if (uap->timeout && uap->timeout < timeout)
 196                 timeout = uap->timeout;
 197         timeout = (timeout / 1000000) * hz +
 198                   ((timeout % 1000000) * hz + 999999) / 1000000;
 199
 200         /*
 201          * Wake us up if the memory location COWs while we are sleeping.
 202          * Use a critical section to tighten up the interlock.  Also,
 203          * tsleep_remove() requires the caller be in a critical section.
 204          */
 205         crit_enter();
 206
 207         /*
 208          * We must interlock just before sleeping.  If we interlock before
 209          * registration the lock operations done by the registration can
 210          * interfere with it.
 211          *
 212          * We cannot leave our interlock hanging on return because this
 213          * will interfere with umtx_wakeup() calls with limited wakeup
 214          * counts.
 215          */
 216         tsleep_interlock(waddr, PCATCH | PDOMAIN_UMTX);
 217         cpu_lfence();
 218         if ((void *)(intptr_t)uservtophys((intptr_t)uptr) != waddr) {
 219                 crit_exit();
 220                 goto retry_on_discontinuity;
 221         }
 222         value = fuword32(uptr);
 223         if (value == uap->value) {
 224                 error = tsleep(waddr, PCATCH | PINTERLOCKED | PDOMAIN_UMTX,
 225                                "umtxsl", timeout);
 226         } else {
 227                 error = EBUSY;
 228         }
 229         crit_exit();
 230         /* Always break out in case of signal, even if restartable */
 231         if (error == ERESTART)
 232                 error = EINTR;
 233     } else {
 234         error = EBUSY;
 235     }
 236 done:
 237     return(error);
 238 }
 239
 240 /*
 241  * umtx_wakeup { const int *ptr, int count }
 242  *
 243  * Wakeup the specified number of processes held in umtx_sleep() on the
 244  * specified user address.  A count of 0 wakes up all waiting processes.
 245  */
 246 int
 247 sys_umtx_wakeup(struct umtx_wakeup_args *uap)
 248 {
 249     int offset;
 250     int error;
 251     int fail_counter;
 252     int32_t value;
 253     void *waddr;
 254     void *uptr;
 255
 256     if (curthread->td_vmm) {
 257         register_t gpa;
 258         vmm_vm_get_gpa(curproc, &gpa, (register_t)uap->ptr);
 259         uap->ptr = (const int *)gpa;
 260     }
 261
 262     /*
 263      * WARNING! We can only use vm_fault_page*() for reading data.  We
 264      *          cannot use it for writing data because there is no pmap
 265      *          interlock to protect against flushes/pageouts.
 266      */
 267     cpu_mfence();
 268     if ((vm_offset_t)uap->ptr & (sizeof(int) - 1))
 269         return EFAULT;
 270
 271     offset = (vm_offset_t)uap->ptr & PAGE_MASK;
 272     uptr = __DEQUALIFY(void *, uap->ptr);
 273
 274     fail_counter = 10000;
 275     do {
 276         if (--fail_counter == 0) {
 277                 kprintf("umtx_wakeup() (X): ERROR Discontinuity "
 278                         "%p (%s %d/%d)\n",
 279                         uptr, curthread->td_comm,
 280                         (int)curthread->td_proc->p_pid,
 281                         (int)curthread->td_lwp->lwp_tid);
 282                 return EINVAL;
 283         }
 284         value = fuword32(uptr);
 285         waddr = (void *)(intptr_t)uservtophys((intptr_t)uptr);
 286     } while (waddr == (void *)(intptr_t)-1 && value != -1);
 287
 288     if (value == -1 && waddr == (void *)(intptr_t)-1) {
 289         kprintf("umtx_wakeup() (A): WARNING can't translate %p (%s %d/%d)\n",
 290                 uptr, curthread->td_comm,
 291                 (int)curthread->td_proc->p_pid,
 292                 (int)curthread->td_lwp->lwp_tid);
 293         return EINVAL;
 294     }
 295
 296     if (uap->count == 1) {
 297         wakeup_domain_one(waddr, PDOMAIN_UMTX);
 298     } else {
 299         /* XXX wakes them all up for now */
 300         wakeup_domain(waddr, PDOMAIN_UMTX);
 301     }
 302     error = 0;
 303
 304     return(error);
 305 }