2 * CPU thread main loop - common bits for user and system mode emulation
4 * Copyright (c) 2003-2005 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/main-loop.h"
22 #include "exec/cpu-common.h"
23 #include "hw/core/cpu.h"
24 #include "sysemu/cpus.h"
26 static QemuMutex qemu_cpu_list_lock
;
27 static QemuCond exclusive_cond
;
28 static QemuCond exclusive_resume
;
29 static QemuCond qemu_work_cond
;
31 /* >= 1 if a thread is inside start_exclusive/end_exclusive. Written
32 * under qemu_cpu_list_lock, read with atomic operations.
34 static int pending_cpus
;
36 void qemu_init_cpu_list(void)
38 /* This is needed because qemu_init_cpu_list is also called by the
39 * child process in a fork. */
42 qemu_mutex_init(&qemu_cpu_list_lock
);
43 qemu_cond_init(&exclusive_cond
);
44 qemu_cond_init(&exclusive_resume
);
45 qemu_cond_init(&qemu_work_cond
);
48 void cpu_list_lock(void)
50 qemu_mutex_lock(&qemu_cpu_list_lock
);
53 void cpu_list_unlock(void)
55 qemu_mutex_unlock(&qemu_cpu_list_lock
);
58 static bool cpu_index_auto_assigned
;
60 static int cpu_get_free_index(void)
65 cpu_index_auto_assigned
= true;
66 CPU_FOREACH(some_cpu
) {
72 void cpu_list_add(CPUState
*cpu
)
74 qemu_mutex_lock(&qemu_cpu_list_lock
);
75 if (cpu
->cpu_index
== UNASSIGNED_CPU_INDEX
) {
76 cpu
->cpu_index
= cpu_get_free_index();
77 assert(cpu
->cpu_index
!= UNASSIGNED_CPU_INDEX
);
79 assert(!cpu_index_auto_assigned
);
81 QTAILQ_INSERT_TAIL_RCU(&cpus
, cpu
, node
);
82 qemu_mutex_unlock(&qemu_cpu_list_lock
);
85 void cpu_list_remove(CPUState
*cpu
)
87 qemu_mutex_lock(&qemu_cpu_list_lock
);
88 if (!QTAILQ_IN_USE(cpu
, node
)) {
89 /* there is nothing to undo since cpu_exec_init() hasn't been called */
90 qemu_mutex_unlock(&qemu_cpu_list_lock
);
94 assert(!(cpu_index_auto_assigned
&& cpu
!= QTAILQ_LAST(&cpus
)));
96 QTAILQ_REMOVE_RCU(&cpus
, cpu
, node
);
97 cpu
->cpu_index
= UNASSIGNED_CPU_INDEX
;
98 qemu_mutex_unlock(&qemu_cpu_list_lock
);
101 struct qemu_work_item
{
102 struct qemu_work_item
*next
;
103 run_on_cpu_func func
;
104 run_on_cpu_data data
;
105 bool free
, exclusive
, done
;
108 static void queue_work_on_cpu(CPUState
*cpu
, struct qemu_work_item
*wi
)
110 qemu_mutex_lock(&cpu
->work_mutex
);
111 if (cpu
->queued_work_first
== NULL
) {
112 cpu
->queued_work_first
= wi
;
114 cpu
->queued_work_last
->next
= wi
;
116 cpu
->queued_work_last
= wi
;
119 qemu_mutex_unlock(&cpu
->work_mutex
);
124 void do_run_on_cpu(CPUState
*cpu
, run_on_cpu_func func
, run_on_cpu_data data
,
127 struct qemu_work_item wi
;
129 if (qemu_cpu_is_self(cpu
)) {
138 wi
.exclusive
= false;
140 queue_work_on_cpu(cpu
, &wi
);
141 while (!atomic_mb_read(&wi
.done
)) {
142 CPUState
*self_cpu
= current_cpu
;
144 qemu_cond_wait(&qemu_work_cond
, mutex
);
145 current_cpu
= self_cpu
;
149 void async_run_on_cpu(CPUState
*cpu
, run_on_cpu_func func
, run_on_cpu_data data
)
151 struct qemu_work_item
*wi
;
153 wi
= g_malloc0(sizeof(struct qemu_work_item
));
158 queue_work_on_cpu(cpu
, wi
);
161 /* Wait for pending exclusive operations to complete. The CPU list lock
163 static inline void exclusive_idle(void)
165 while (pending_cpus
) {
166 qemu_cond_wait(&exclusive_resume
, &qemu_cpu_list_lock
);
170 /* Start an exclusive operation.
171 Must only be called from outside cpu_exec. */
172 void start_exclusive(void)
177 qemu_mutex_lock(&qemu_cpu_list_lock
);
180 /* Make all other cpus stop executing. */
181 atomic_set(&pending_cpus
, 1);
183 /* Write pending_cpus before reading other_cpu->running. */
186 CPU_FOREACH(other_cpu
) {
187 if (atomic_read(&other_cpu
->running
)) {
188 other_cpu
->has_waiter
= true;
190 qemu_cpu_kick(other_cpu
);
194 atomic_set(&pending_cpus
, running_cpus
+ 1);
195 while (pending_cpus
> 1) {
196 qemu_cond_wait(&exclusive_cond
, &qemu_cpu_list_lock
);
199 /* Can release mutex, no one will enter another exclusive
200 * section until end_exclusive resets pending_cpus to 0.
202 qemu_mutex_unlock(&qemu_cpu_list_lock
);
204 current_cpu
->in_exclusive_context
= true;
207 /* Finish an exclusive operation. */
208 void end_exclusive(void)
210 current_cpu
->in_exclusive_context
= false;
212 qemu_mutex_lock(&qemu_cpu_list_lock
);
213 atomic_set(&pending_cpus
, 0);
214 qemu_cond_broadcast(&exclusive_resume
);
215 qemu_mutex_unlock(&qemu_cpu_list_lock
);
218 /* Wait for exclusive ops to finish, and begin cpu execution. */
219 void cpu_exec_start(CPUState
*cpu
)
221 atomic_set(&cpu
->running
, true);
223 /* Write cpu->running before reading pending_cpus. */
226 /* 1. start_exclusive saw cpu->running == true and pending_cpus >= 1.
227 * After taking the lock we'll see cpu->has_waiter == true and run---not
228 * for long because start_exclusive kicked us. cpu_exec_end will
229 * decrement pending_cpus and signal the waiter.
231 * 2. start_exclusive saw cpu->running == false but pending_cpus >= 1.
232 * This includes the case when an exclusive item is running now.
233 * Then we'll see cpu->has_waiter == false and wait for the item to
236 * 3. pending_cpus == 0. Then start_exclusive is definitely going to
237 * see cpu->running == true, and it will kick the CPU.
239 if (unlikely(atomic_read(&pending_cpus
))) {
240 qemu_mutex_lock(&qemu_cpu_list_lock
);
241 if (!cpu
->has_waiter
) {
242 /* Not counted in pending_cpus, let the exclusive item
243 * run. Since we have the lock, just set cpu->running to true
244 * while holding it; no need to check pending_cpus again.
246 atomic_set(&cpu
->running
, false);
248 /* Now pending_cpus is zero. */
249 atomic_set(&cpu
->running
, true);
251 /* Counted in pending_cpus, go ahead and release the
252 * waiter at cpu_exec_end.
255 qemu_mutex_unlock(&qemu_cpu_list_lock
);
259 /* Mark cpu as not executing, and release pending exclusive ops. */
260 void cpu_exec_end(CPUState
*cpu
)
262 atomic_set(&cpu
->running
, false);
264 /* Write cpu->running before reading pending_cpus. */
267 /* 1. start_exclusive saw cpu->running == true. Then it will increment
268 * pending_cpus and wait for exclusive_cond. After taking the lock
269 * we'll see cpu->has_waiter == true.
271 * 2. start_exclusive saw cpu->running == false but here pending_cpus >= 1.
272 * This includes the case when an exclusive item started after setting
273 * cpu->running to false and before we read pending_cpus. Then we'll see
274 * cpu->has_waiter == false and not touch pending_cpus. The next call to
275 * cpu_exec_start will run exclusive_idle if still necessary, thus waiting
276 * for the item to complete.
278 * 3. pending_cpus == 0. Then start_exclusive is definitely going to
279 * see cpu->running == false, and it can ignore this CPU until the
280 * next cpu_exec_start.
282 if (unlikely(atomic_read(&pending_cpus
))) {
283 qemu_mutex_lock(&qemu_cpu_list_lock
);
284 if (cpu
->has_waiter
) {
285 cpu
->has_waiter
= false;
286 atomic_set(&pending_cpus
, pending_cpus
- 1);
287 if (pending_cpus
== 1) {
288 qemu_cond_signal(&exclusive_cond
);
291 qemu_mutex_unlock(&qemu_cpu_list_lock
);
295 void async_safe_run_on_cpu(CPUState
*cpu
, run_on_cpu_func func
,
296 run_on_cpu_data data
)
298 struct qemu_work_item
*wi
;
300 wi
= g_malloc0(sizeof(struct qemu_work_item
));
304 wi
->exclusive
= true;
306 queue_work_on_cpu(cpu
, wi
);
309 void process_queued_cpu_work(CPUState
*cpu
)
311 struct qemu_work_item
*wi
;
313 if (cpu
->queued_work_first
== NULL
) {
317 qemu_mutex_lock(&cpu
->work_mutex
);
318 while (cpu
->queued_work_first
!= NULL
) {
319 wi
= cpu
->queued_work_first
;
320 cpu
->queued_work_first
= wi
->next
;
321 if (!cpu
->queued_work_first
) {
322 cpu
->queued_work_last
= NULL
;
324 qemu_mutex_unlock(&cpu
->work_mutex
);
326 /* Running work items outside the BQL avoids the following deadlock:
327 * 1) start_exclusive() is called with the BQL taken while another
328 * CPU is running; 2) cpu_exec in the other CPU tries to takes the
329 * BQL, so it goes to sleep; start_exclusive() is sleeping too, so
330 * neither CPU can proceed.
332 qemu_mutex_unlock_iothread();
334 wi
->func(cpu
, wi
->data
);
336 qemu_mutex_lock_iothread();
338 wi
->func(cpu
, wi
->data
);
340 qemu_mutex_lock(&cpu
->work_mutex
);
344 atomic_mb_set(&wi
->done
, true);
347 qemu_mutex_unlock(&cpu
->work_mutex
);
348 qemu_cond_broadcast(&qemu_work_cond
);