2 * $FreeBSD: src/sys/i386/i386/mplock.s,v 1.29.2.2 2000/05/16 06:58:06 dillon Exp $
3 * $DragonFly: src/sys/platform/pc32/i386/mplock.s,v 1.21 2006/11/07 06:43:24 dillon Exp $
5 * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved.
7 * This code is derived from software contributed to The DragonFly Project
8 * by Matthew Dillon <dillon@backplane.com>
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in
18 * the documentation and/or other materials provided with the
20 * 3. Neither the name of The DragonFly Project nor the names of its
21 * contributors may be used to endorse or promote products derived
22 * from this software without specific, prior written permission.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * DragonFly MPLOCK operation
39 * Each thread has an MP lock count, td_mpcount, and there is a shared
40 * global called mp_lock. mp_lock is the physical MP lock and contains either
41 * -1 or the cpuid of the cpu owning the lock. The count is *NOT* integrated
42 * into mp_lock but instead resides in each thread td_mpcount.
44 * When obtaining or releasing the MP lock the td_mpcount is PREDISPOSED
45 * to the desired count *PRIOR* to operating on the mp_lock itself. MP
46 * lock operations can occur outside a critical section with interrupts
47 * enabled with the provisio (which the routines below handle) that an
48 * interrupt may come along and preempt us, racing our cmpxchgl instruction
49 * to perform the operation we have requested by pre-disposing td_mpcount.
51 * Additionally, the LWKT threading system manages the MP lock and
52 * lwkt_switch(), in particular, may be called after pre-disposing td_mpcount
53 * to handle 'blocking' on the MP lock.
56 * Recoded from the FreeBSD original:
57 * ----------------------------------------------------------------------------
58 * "THE BEER-WARE LICENSE" (Revision 42):
59 * <phk@FreeBSD.org> wrote this file. As long as you retain this notice you
60 * can do whatever you want with this stuff. If we meet some day, and you think
61 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
62 * ----------------------------------------------------------------------------
65 #include <machine/asmacros.h>
66 #include <machine_base/apic/apicreg.h>
71 * YYY Debugging only. Define this to be paranoid about invalidating the
72 * TLB when we get giant.
74 #undef PARANOID_INVLTLB
81 .long -1 /* initialized to not held */
88 * Note on cmpxchgl... exchanges ecx with mem if mem matches eax.
89 * Z=1 (jz) on success. A lock prefix is required for MP.
91 NON_GPROF_ENTRY
(cpu_get_initial_mplock
)
92 movl PCPU
(curthread
),%ecx
93 movl $
1,TD_MPCOUNT
(%ecx
) /* curthread has mpcount of 1 */
94 movl $
0,mp_lock
/* owned by cpu 0 */
98 * cpu_try_mplock() returns non-zero on success, 0 on failure. It
99 * only adjusts mp_lock, it does not touch td_mpcount. Callers
100 * should always increment td_mpcount *before* trying to acquire
101 * the actual lock, predisposing td_mpcount to the desired state of
104 * NOTE! Only call cpu_try_mplock() inside a critical section. If
105 * you don't an interrupt can come along and get and release
106 * the lock before our cmpxchgl instruction, causing us to fail
107 * but resulting in the lock being held by our cpu.
109 NON_GPROF_ENTRY
(cpu_try_mplock
)
110 movl PCPU
(cpuid
),%ecx
112 lock cmpxchgl
%ecx
,mp_lock
/* ecx<->mem if eax matches */
114 #ifdef PARANOID_INVLTLB
115 movl
%cr3
,%eax; movl
%eax
,%cr3
/* YYY check and remove */
124 * get_mplock() Obtains the MP lock and may switch away if it cannot
125 * get it. This routine may be called WITHOUT a critical section
126 * and with cpu interrupts enabled.
128 * To handle races in a sane fashion we predispose TD_MPCOUNT,
129 * which prevents us from losing the lock in a race if we already
130 * have it or happen to get it. It also means that we might get
131 * the lock in an interrupt race before we have a chance to execute
132 * our cmpxchgl instruction, so we have to handle that case.
133 * Fortunately simply calling lwkt_switch() handles the situation
134 * for us and also 'blocks' us until the MP lock can be obtained.
136 NON_GPROF_ENTRY
(get_mplock
)
137 movl PCPU
(cpuid
),%ecx
138 movl PCPU
(curthread
),%edx
139 incl TD_MPCOUNT
(%edx
) /* predispose */
142 NON_GPROF_RET
/* success! */
145 * We don't already own the mp_lock, use cmpxchgl to try to get
150 lock cmpxchgl
%ecx
,mp_lock
152 #ifdef PARANOID_INVLTLB
153 movl
%cr3
,%eax; movl
%eax
,%cr3
/* YYY check and remove */
155 NON_GPROF_RET
/* success */
158 * Failure, but we could end up owning mp_lock anyway due to
159 * an interrupt race. lwkt_switch() will clean up the mess
160 * and 'block' until the mp_lock is obtained.
162 * Create a stack frame for the call so KTR logs the stack
163 * backtrace properly.
168 call lwkt_mp_lock_contested
171 movl PCPU
(cpuid
),%eax
/* failure */
178 cmpl $
0,panicstr
/* don't double panic */
184 * try_mplock() attempts to obtain the MP lock. 1 is returned on
185 * success, 0 on failure. We do not have to be in a critical section
186 * and interrupts are almost certainly enabled.
188 * We must pre-dispose TD_MPCOUNT in order to deal with races in
192 NON_GPROF_ENTRY
(try_mplock
)
193 movl PCPU
(cpuid
),%ecx
194 movl PCPU
(curthread
),%edx
195 incl TD_MPCOUNT
(%edx
) /* pre-dispose for race */
197 je
1f
/* trivial success */
199 lock cmpxchgl
%ecx
,mp_lock
204 #ifdef PARANOID_INVLTLB
205 movl
%cr3
,%eax; movl
%eax
,%cr3
/* YYY check and remove */
208 movl $
1,%eax
/* success (cmpxchgl good!) */
212 * The cmpxchgl failed but we might have raced. Undo the mess by
213 * predispoing TD_MPCOUNT and then checking. If TD_MPCOUNT is
214 * still non-zero we don't care what state the lock is in (since
215 * we obviously didn't own it above), just return failure even if
216 * we won the lock in an interrupt race. If TD_MPCOUNT is zero
217 * make sure we don't own the lock in case we did win it in a race.
220 decl TD_MPCOUNT
(%edx
)
221 cmpl $
0,TD_MPCOUNT
(%edx
)
223 movl PCPU
(cpuid
),%eax
225 lock cmpxchgl
%ecx
,mp_lock
231 * rel_mplock() releases a previously obtained MP lock.
233 * In order to release the MP lock we pre-dispose TD_MPCOUNT for
234 * the release and basically repeat the release portion of try_mplock
237 NON_GPROF_ENTRY
(rel_mplock
)
238 movl PCPU
(curthread
),%edx
239 movl TD_MPCOUNT
(%edx
),%eax
245 movl
%eax
,TD_MPCOUNT
(%edx
)
248 movl PCPU
(cpuid
),%eax
250 lock cmpxchgl
%ecx
,mp_lock
269 .asciz "try/get_mplock(): already have lock! %d %p"
272 .asciz "try/get_mplock(): failed on count or switch %d %p"
275 .asciz "rel_mplock(): mpcount already 0 @ %p %p %p %p %p %p %p %p!"