2 * Copyright (c) 2003,2004,2010,2017 The DragonFly Project.
5 * This code is derived from software contributed to The DragonFly Project
6 * by Matthew Dillon <dillon@backplane.com> and David Xu <davidxu@freebsd.org>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
18 * 3. Neither the name of The DragonFly Project nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific, prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
28 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
30 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
31 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
32 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * This module implements userland mutex helper functions. umtx_sleep()
38 * handling blocking and umtx_wakeup() handles wakeups. The sleep/wakeup
39 * functions operate on user addresses.
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/cdefs.h>
45 #include <sys/kernel.h>
46 #include <sys/sysproto.h>
47 #include <sys/sysunion.h>
48 #include <sys/sysent.h>
49 #include <sys/syscall.h>
50 #include <sys/sysctl.h>
51 #include <sys/module.h>
52 #include <sys/thread.h>
55 #include <cpu/lwbuf.h>
58 #include <vm/vm_param.h>
61 #include <vm/vm_map.h>
62 #include <vm/vm_object.h>
63 #include <vm/vm_page.h>
64 #include <vm/vm_pager.h>
65 #include <vm/vm_pageout.h>
66 #include <vm/vm_extern.h>
67 #include <vm/vm_kern.h>
69 #include <vm/vm_page2.h>
71 #include <machine/vmm.h>
74 * Improve umtx performance by polling for 4000nS before going to sleep.
75 * This can avoid many IPIs in typical pthreads mutex situations.
77 #ifdef _RDTSC_SUPPORTED_
78 static int umtx_delay
= 4000; /* nS */
79 SYSCTL_INT(_kern
, OID_AUTO
, umtx_delay
, CTLFLAG_RW
,
82 static int umtx_timeout_max
= 2000000; /* microseconds */
83 SYSCTL_INT(_kern
, OID_AUTO
, umtx_timeout_max
, CTLFLAG_RW
,
84 &umtx_timeout_max
, 0, "");
87 * If the contents of the userland-supplied pointer matches the specified
88 * value enter an interruptable sleep for up to <timeout> microseconds.
89 * If the contents does not match then return immediately.
91 * Returns 0 if we slept and were woken up, -1 and EWOULDBLOCK if we slept
92 * and timed out, and EBUSY if the contents of the pointer already does
93 * not match the specified value. A timeout of 0 indicates an unlimited sleep.
94 * EINTR is returned if the call was interrupted by a signal (even if
95 * the signal specifies that the system call should restart).
97 * This function interlocks against call to umtx_wakeup. It does NOT interlock
98 * against changes in *ptr. However, it does not have to. The standard use
99 * of *ptr is to differentiate between an uncontested and a contested mutex
100 * and call umtx_wakeup when releasing a contested mutex. Therefore we can
101 * safely race against changes in *ptr as long as we are properly interlocked
102 * against the umtx_wakeup() call.
104 * For performance reasons, we do not try to track the underlying page for
105 * mapping changes. Instead, the timeout is capped at kern.umtx_timeout_max
106 * (default 1 second) and the caller is expected to retry. The kernel
107 * will wake all umtx_sleep()s if the process fork()s, but not if it vfork()s.
108 * Other mapping changes must be caught by the timeout.
110 * umtx_sleep { const int *ptr, int value, int timeout }
113 sys_umtx_sleep(struct umtx_sleep_args
*uap
)
124 if (uap
->timeout
< 0)
130 vmm_vm_get_gpa(td
->td_proc
, &gpa
, (register_t
)uap
->ptr
);
131 uap
->ptr
= (const int *)gpa
;
134 uptr
= __DEQUALIFY(void *, uap
->ptr
);
135 if ((vm_offset_t
)uptr
& (sizeof(int) - 1))
138 offset
= (vm_offset_t
)uptr
& PAGE_MASK
;
141 * Resolve the physical address. We allow the case where there are
142 * sometimes discontinuities (causing a 2 second retry timeout).
144 retry_on_discontinuity
:
145 fail_counter
= 10000;
147 if (--fail_counter
== 0) {
148 kprintf("umtx_sleep() (X): ERROR Discontinuity %p (%s %d/%d)\n",
150 (int)td
->td_proc
->p_pid
,
151 (int)td
->td_lwp
->lwp_tid
);
154 value
= fuwordadd32(uptr
, 0);
155 waddr
= (void *)(intptr_t)uservtophys((intptr_t)uptr
);
156 } while (waddr
== (void *)(intptr_t)-1 && value
!= -1);
158 if (value
== -1 && waddr
== (void *)(intptr_t)-1) {
159 kprintf("umtx_sleep() (A): WARNING can't translate %p (%s %d/%d)\n",
161 (int)td
->td_proc
->p_pid
,
162 (int)td
->td_lwp
->lwp_tid
);
167 if (value
== uap
->value
) {
168 #ifdef _RDTSC_SUPPORTED_
170 * Poll a little while before sleeping, most mutexes are
177 tsc_target
= tsc_get_target(umtx_delay
);
178 while (tsc_test_target(tsc_target
) == 0) {
180 if (fuwordadd32(uptr
, 0) != uap
->value
) {
193 * Calculate the timeout. This will be acccurate to within ~2 ticks.
194 * uap->timeout is in microseconds.
196 timeout
= umtx_timeout_max
;
197 if (uap
->timeout
&& uap
->timeout
< timeout
)
198 timeout
= uap
->timeout
;
199 timeout
= (timeout
/ 1000000) * hz
+
200 ((timeout
% 1000000) * hz
+ 999999) / 1000000;
203 * Wake us up if the memory location COWs while we are sleeping.
204 * Use a critical section to tighten up the interlock. Also,
205 * tsleep_remove() requires the caller be in a critical section.
210 * We must interlock just before sleeping. If we interlock before
211 * registration the lock operations done by the registration can
214 * We cannot leave our interlock hanging on return because this
215 * will interfere with umtx_wakeup() calls with limited wakeup
218 tsleep_interlock(waddr
, PCATCH
| PDOMAIN_UMTX
);
221 * Check physical address changed
224 if ((void *)(intptr_t)uservtophys((intptr_t)uptr
) != waddr
) {
226 goto retry_on_discontinuity
;
232 value
= fuwordadd32(uptr
, 0);
234 if (value
== uap
->value
) {
235 error
= tsleep(waddr
, PCATCH
| PINTERLOCKED
| PDOMAIN_UMTX
,
241 /* Always break out in case of signal, even if restartable */
242 if (error
== ERESTART
)
252 * umtx_wakeup { const int *ptr, int count }
254 * Wakeup the specified number of processes held in umtx_sleep() on the
255 * specified user address. A count of 0 wakes up all waiting processes.
258 sys_umtx_wakeup(struct umtx_wakeup_args
*uap
)
272 vmm_vm_get_gpa(td
->td_proc
, &gpa
, (register_t
)uap
->ptr
);
273 uap
->ptr
= (const int *)gpa
;
277 * WARNING! We can only use vm_fault_page*() for reading data. We
278 * cannot use it for writing data because there is no pmap
279 * interlock to protect against flushes/pageouts.
282 if ((vm_offset_t
)uap
->ptr
& (sizeof(int) - 1))
285 offset
= (vm_offset_t
)uap
->ptr
& PAGE_MASK
;
286 uptr
= __DEQUALIFY(void *, uap
->ptr
);
288 fail_counter
= 10000;
290 if (--fail_counter
== 0) {
291 kprintf("umtx_wakeup() (X): ERROR Discontinuity "
294 (int)td
->td_proc
->p_pid
,
295 (int)td
->td_lwp
->lwp_tid
);
298 value
= fuwordadd32(uptr
, 0);
299 waddr
= (void *)(intptr_t)uservtophys((intptr_t)uptr
);
300 } while (waddr
== (void *)(intptr_t)-1 && value
!= -1);
302 if (value
== -1 && waddr
== (void *)(intptr_t)-1) {
303 kprintf("umtx_wakeup() (A): WARNING can't translate %p (%s %d/%d)\n",
305 (int)td
->td_proc
->p_pid
,
306 (int)td
->td_lwp
->lwp_tid
);
310 if (uap
->count
== 1) {
311 wakeup_domain_one(waddr
, PDOMAIN_UMTX
);
313 /* XXX wakes them all up for now */
314 wakeup_domain(waddr
, PDOMAIN_UMTX
);