sys/kern/kern_umtx.c

   1 /*
   2  * Copyright (c) 2003,2004,2010,2017 The DragonFly Project.
   3  * All rights reserved.
   4  *
   5  * This code is derived from software contributed to The DragonFly Project
   6  * by Matthew Dillon <dillon@backplane.com> and David Xu <davidxu@freebsd.org>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in
  16  *    the documentation and/or other materials provided with the
  17  *    distribution.
  18  * 3. Neither the name of The DragonFly Project nor the names of its
  19  *    contributors may be used to endorse or promote products derived
  20  *    from this software without specific, prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  26  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  27  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  28  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  29  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  30  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  31  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  32  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  33  * SUCH DAMAGE.
  34  */
  35
  36 /*
  37  * This module implements userland mutex helper functions.  umtx_sleep()
  38  * handling blocking and umtx_wakeup() handles wakeups.  The sleep/wakeup
  39  * functions operate on user addresses.
  40  */
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/cdefs.h>
  45 #include <sys/kernel.h>
  46 #include <sys/sysmsg.h>
  47 #include <sys/sysent.h>
  48 #include <sys/syscall.h>
  49 #include <sys/sysctl.h>
  50 #include <sys/module.h>
  51 #include <sys/thread.h>
  52 #include <sys/proc.h>
  53
  54 #include <cpu/lwbuf.h>
  55
  56 #include <vm/vm.h>
  57 #include <vm/vm_param.h>
  58 #include <sys/lock.h>
  59 #include <vm/pmap.h>
  60 #include <vm/vm_map.h>
  61 #include <vm/vm_object.h>
  62 #include <vm/vm_page.h>
  63 #include <vm/vm_pager.h>
  64 #include <vm/vm_pageout.h>
  65 #include <vm/vm_extern.h>
  66 #include <vm/vm_kern.h>
  67
  68 #include <vm/vm_page2.h>
  69
  70 /*
  71  * Improve umtx performance by polling for 4000nS before going to sleep.
  72  * This can avoid many IPIs in typical pthreads mutex situations.
  73  */
  74 #ifdef _RDTSC_SUPPORTED_
  75 static int umtx_delay = 4000;           /* nS */
  76 SYSCTL_INT(_kern, OID_AUTO, umtx_delay, CTLFLAG_RW,
  77            &umtx_delay, 0, "");
  78 #endif
  79 static int umtx_timeout_max = 2000000;  /* microseconds */
  80 SYSCTL_INT(_kern, OID_AUTO, umtx_timeout_max, CTLFLAG_RW,
  81            &umtx_timeout_max, 0, "");
  82
  83 /*
  84  * If the contents of the userland-supplied pointer matches the specified
  85  * value enter an interruptable sleep for up to <timeout> microseconds.
  86  * If the contents does not match then return immediately.
  87  *
  88  * Returns 0 if we slept and were woken up, -1 and EWOULDBLOCK if we slept
  89  * and timed out, and EBUSY if the contents of the pointer already does
  90  * not match the specified value.  A timeout of 0 indicates an unlimited sleep.
  91  * EINTR is returned if the call was interrupted by a signal (even if
  92  * the signal specifies that the system call should restart).
  93  *
  94  * This function interlocks against call to umtx_wakeup.  It does NOT interlock
  95  * against changes in *ptr.  However, it does not have to.  The standard use
  96  * of *ptr is to differentiate between an uncontested and a contested mutex
  97  * and call umtx_wakeup when releasing a contested mutex.  Therefore we can
  98  * safely race against changes in *ptr as long as we are properly interlocked
  99  * against the umtx_wakeup() call.
 100  *
 101  * For performance reasons, we do not try to track the underlying page for
 102  * mapping changes.  Instead, the timeout is capped at kern.umtx_timeout_max
 103  * (default 1 second) and the caller is expected to retry.  The kernel
 104  * will wake all umtx_sleep()s if the process fork()s, but not if it vfork()s.
 105  * Other mapping changes must be caught by the timeout.
 106  *
 107  * umtx_sleep { const int *ptr, int value, int timeout }
 108  */
 109 int
 110 sys_umtx_sleep(struct sysmsg *sysmsg, const struct umtx_sleep_args *uap)
 111 {
 112     void *waddr;
 113     void *uptr;
 114     int offset;
 115     int timeout;
 116     int error;
 117     int value;
 118     int fail_counter;
 119     thread_t td;
 120     volatile const int *ptr = uap->ptr;
 121
 122     if (uap->timeout < 0)
 123         return (EINVAL);
 124     td = curthread;
 125
 126     uptr = __DEQUALIFY(void *, ptr);
 127     if ((vm_offset_t)uptr & (sizeof(int) - 1))
 128         return EFAULT;
 129
 130     offset = (vm_offset_t)uptr & PAGE_MASK;
 131
 132     /*
 133      * Resolve the physical address.  We allow the case where there are
 134      * sometimes discontinuities (causing a 2 second retry timeout).
 135      */
 136 retry_on_discontinuity:
 137     fail_counter = 10000;
 138     do {
 139         if (--fail_counter == 0) {
 140                 kprintf("umtx_sleep() (X): ERROR Discontinuity %p (%s %d/%d)\n",
 141                         uptr, td->td_comm,
 142                         (int)td->td_proc->p_pid,
 143                         (int)td->td_lwp->lwp_tid);
 144                 return EINVAL;
 145         }
 146         value = fuwordadd32(uptr, 0);
 147         waddr = (void *)(intptr_t)uservtophys((intptr_t)uptr);
 148     } while (waddr == (void *)(intptr_t)-1 && value != -1);
 149
 150     if (value == -1 && waddr == (void *)(intptr_t)-1) {
 151         kprintf("umtx_sleep() (A): WARNING can't translate %p (%s %d/%d)\n",
 152                 uptr, td->td_comm,
 153                 (int)td->td_proc->p_pid,
 154                 (int)td->td_lwp->lwp_tid);
 155         return EINVAL;
 156     }
 157
 158     error = EBUSY;
 159     if (value == uap->value) {
 160 #ifdef _RDTSC_SUPPORTED_
 161         /*
 162          * Poll a little while before sleeping, most mutexes are
 163          * short-lived.
 164          */
 165         if (umtx_delay) {
 166                 int64_t tsc_target;
 167                 int good = 0;
 168
 169                 tsc_target = tsc_get_target(umtx_delay);
 170                 while (tsc_test_target(tsc_target) == 0) {
 171                         cpu_lfence();
 172                         if (fuwordadd32(uptr, 0) != uap->value) {
 173                                 good = 1;
 174                                 break;
 175                         }
 176                         cpu_pause();
 177                 }
 178                 if (good) {
 179                         error = EBUSY;
 180                         goto done;
 181                 }
 182         }
 183 #endif
 184         /*
 185          * Calculate the timeout.  This will be acccurate to within ~2 ticks.
 186          * uap->timeout is in microseconds.
 187          */
 188         timeout = umtx_timeout_max;
 189         if (uap->timeout && uap->timeout < timeout)
 190                 timeout = uap->timeout;
 191         timeout = (timeout / 1000000) * hz +
 192                   ((timeout % 1000000) * hz + 999999) / 1000000;
 193
 194         /*
 195          * Wake us up if the memory location COWs while we are sleeping.
 196          * Use a critical section to tighten up the interlock.  Also,
 197          * tsleep_remove() requires the caller be in a critical section.
 198          */
 199         crit_enter();
 200
 201         /*
 202          * We must interlock just before sleeping.  If we interlock before
 203          * registration the lock operations done by the registration can
 204          * interfere with it.
 205          *
 206          * We cannot leave our interlock hanging on return because this
 207          * will interfere with umtx_wakeup() calls with limited wakeup
 208          * counts.
 209          */
 210         tsleep_interlock(waddr, PCATCH | PDOMAIN_UMTX);
 211
 212         /*
 213          * Check physical address changed
 214          */
 215         cpu_lfence();
 216         if ((void *)(intptr_t)uservtophys((intptr_t)uptr) != waddr) {
 217                 crit_exit();
 218                 goto retry_on_discontinuity;
 219         }
 220
 221         /*
 222          * Re-read value
 223          */
 224         value = fuwordadd32(uptr, 0);
 225
 226         if (value == uap->value) {
 227                 error = tsleep(waddr, PCATCH | PINTERLOCKED | PDOMAIN_UMTX,
 228                                "umtxsl", timeout);
 229         } else {
 230                 error = EBUSY;
 231         }
 232         crit_exit();
 233         /* Always break out in case of signal, even if restartable */
 234         if (error == ERESTART)
 235                 error = EINTR;
 236     } else {
 237         error = EBUSY;
 238     }
 239 done:
 240     return(error);
 241 }
 242
 243 /*
 244  * umtx_wakeup { const int *ptr, int count }
 245  *
 246  * Wakeup the specified number of processes held in umtx_sleep() on the
 247  * specified user address.  A count of 0 wakes up all waiting processes.
 248  */
 249 int
 250 sys_umtx_wakeup(struct sysmsg *sysmsg, const struct umtx_wakeup_args *uap)
 251 {
 252     int offset;
 253     int error;
 254     int fail_counter;
 255     int32_t value;
 256     void *waddr;
 257     void *uptr;
 258     volatile const int *ptr = uap->ptr;
 259     thread_t td;
 260
 261     td = curthread;
 262
 263     /*
 264      * WARNING! We can only use vm_fault_page*() for reading data.  We
 265      *          cannot use it for writing data because there is no pmap
 266      *          interlock to protect against flushes/pageouts.
 267      */
 268     cpu_mfence();
 269     if ((vm_offset_t)ptr & (sizeof(int) - 1))
 270         return EFAULT;
 271
 272     offset = (vm_offset_t)ptr & PAGE_MASK;
 273     uptr = __DEQUALIFY(void *, ptr);
 274
 275     fail_counter = 10000;
 276     do {
 277         if (--fail_counter == 0) {
 278                 kprintf("umtx_wakeup() (X): ERROR Discontinuity "
 279                         "%p (%s %d/%d)\n",
 280                         uptr, td->td_comm,
 281                         (int)td->td_proc->p_pid,
 282                         (int)td->td_lwp->lwp_tid);
 283                 return EINVAL;
 284         }
 285         value = fuwordadd32(uptr, 0);
 286         waddr = (void *)(intptr_t)uservtophys((intptr_t)uptr);
 287     } while (waddr == (void *)(intptr_t)-1 && value != -1);
 288
 289     if (value == -1 && waddr == (void *)(intptr_t)-1) {
 290         kprintf("umtx_wakeup() (A): WARNING can't translate %p (%s %d/%d)\n",
 291                 uptr, td->td_comm,
 292                 (int)td->td_proc->p_pid,
 293                 (int)td->td_lwp->lwp_tid);
 294         return EINVAL;
 295     }
 296
 297     if (uap->count == 1) {
 298         wakeup_domain_one(waddr, PDOMAIN_UMTX);
 299     } else {
 300         /* XXX wakes them all up for now */
 301         wakeup_domain(waddr, PDOMAIN_UMTX);
 302     }
 303     error = 0;
 304
 305     return(error);
 306 }