3 * Copyright (C) Igor Sysoev
4 * Copyright (C) Nginx, Inc.
8 #include <ngx_config.h>
12 * The threads implementation uses the rfork(RFPROC|RFTHREAD|RFMEM) syscall
13 * to create threads. All threads use the stacks of the same size mmap()ed
14 * below the main stack. Thus the current thread id is determined via
15 * the stack pointer value.
17 * The mutex implementation uses the ngx_atomic_cmp_set() operation
18 * to acquire a mutex and the SysV semaphore to wait on a mutex and to wake up
19 * the waiting threads. The light mutex does not use semaphore, so after
20 * spinning in the lock the thread calls sched_yield(). However the light
21 * mutexes are intended to be used with the "trylock" operation only.
22 * The SysV semop() is a cheap syscall, particularly if it has little sembuf's
23 * and does not use SEM_UNDO.
25 * The condition variable implementation uses the signal #64.
26 * The signal handler is SIG_IGN so the kill() is a cheap syscall.
27 * The thread waits a signal in kevent(). The use of the EVFILT_SIGNAL
28 * is safe since FreeBSD 4.10-STABLE.
30 * This threads implementation currently works on i386 (486+) and amd64
35 char *ngx_freebsd_kern_usrstack
;
36 size_t ngx_thread_stack_size
;
39 static size_t rz_size
;
40 static size_t usable_stack_size
;
41 static char *last_stack
;
43 static ngx_uint_t nthreads
;
44 static ngx_uint_t max_threads
;
46 static ngx_uint_t nkeys
;
47 static ngx_tid_t
*tids
; /* the threads tids array */
48 void **ngx_tls
; /* the threads tls's array */
50 /* the thread-safe libc errno */
52 static int errno0
; /* the main thread's errno */
53 static int *errnos
; /* the threads errno's array */
62 return tid
? &errnos
[tid
- 1] : &errno0
;
67 * __isthreaded enables the spinlocks in some libc functions, i.e. in malloc()
68 * and some other places. Nevertheless we protect our malloc()/free() calls
69 * by own mutex that is more efficient than the spinlock.
71 * _spinlock() is a weak referenced stub in src/lib/libc/gen/_spinlock_stub.c
75 extern int __isthreaded
;
78 _spinlock(ngx_atomic_t
*lock
)
87 if (ngx_ncpu
> 1 && tries
++ < 1000) {
95 if (ngx_atomic_cmp_set(lock
, 0, 1)) {
104 * Before FreeBSD 5.1 _spinunlock() is a simple #define in
105 * src/lib/libc/include/spinlock.h that zeroes lock.
107 * Since FreeBSD 5.1 _spinunlock() is a weak referenced stub in
108 * src/lib/libc/gen/_spinlock_stub.c that does nothing.
114 _spinunlock(ngx_atomic_t
*lock
)
123 ngx_create_thread(ngx_tid_t
*tid
, ngx_thread_value_t (*func
)(void *arg
),
124 void *arg
, ngx_log_t
*log
)
128 char *stack
, *stack_top
;
130 if (nthreads
>= max_threads
) {
131 ngx_log_error(NGX_LOG_CRIT
, log
, 0,
132 "no more than %ui threads can be created", max_threads
);
136 last_stack
-= ngx_thread_stack_size
;
138 stack
= mmap(last_stack
, usable_stack_size
, PROT_READ
|PROT_WRITE
,
141 if (stack
== MAP_FAILED
) {
142 ngx_log_error(NGX_LOG_ALERT
, log
, ngx_errno
,
143 "mmap(%p:%uz, MAP_STACK) thread stack failed",
144 last_stack
, usable_stack_size
);
148 if (stack
!= last_stack
) {
149 ngx_log_error(NGX_LOG_ALERT
, log
, 0,
150 "stack %p address was changed to %p", last_stack
, stack
);
154 stack_top
= stack
+ usable_stack_size
;
156 ngx_log_debug2(NGX_LOG_DEBUG_CORE
, log
, 0,
157 "thread stack: %p-%p", stack
, stack_top
);
161 id
= rfork_thread(RFPROC
|RFTHREAD
|RFMEM
, stack_top
,
162 (ngx_rfork_thread_func_pt
) func
, arg
);
167 ngx_log_error(NGX_LOG_ALERT
, log
, err
, "rfork() failed");
171 nthreads
= (ngx_freebsd_kern_usrstack
- stack_top
)
172 / ngx_thread_stack_size
;
175 ngx_log_debug1(NGX_LOG_DEBUG_CORE
, log
, 0, "rfork()ed thread: %P", id
);
183 ngx_init_threads(int n
, size_t size
, ngx_cycle_t
*cycle
)
185 char *red_zone
, *zone
;
192 for (i
= 0; i
< n
; i
++) {
193 ngx_memzero(&sa
, sizeof(struct sigaction
));
194 sa
.sa_handler
= SIG_IGN
;
195 sigemptyset(&sa
.sa_mask
);
196 if (sigaction(NGX_CV_SIGNAL
, &sa
, NULL
) == -1) {
197 ngx_log_error(NGX_LOG_ALERT
, cycle
->log
, ngx_errno
,
198 "sigaction(%d, SIG_IGN) failed", NGX_CV_SIGNAL
);
203 len
= sizeof(ngx_freebsd_kern_usrstack
);
204 if (sysctlbyname("kern.usrstack", &ngx_freebsd_kern_usrstack
, &len
,
207 ngx_log_error(NGX_LOG_ALERT
, cycle
->log
, ngx_errno
,
208 "sysctlbyname(kern.usrstack) failed");
212 /* the main thread stack red zone */
213 rz_size
= ngx_pagesize
;
214 red_zone
= ngx_freebsd_kern_usrstack
- (size
+ rz_size
);
216 ngx_log_debug2(NGX_LOG_DEBUG_CORE
, cycle
->log
, 0,
217 "usrstack: %p red zone: %p",
218 ngx_freebsd_kern_usrstack
, red_zone
);
220 zone
= mmap(red_zone
, rz_size
, PROT_NONE
, MAP_ANON
, -1, 0);
221 if (zone
== MAP_FAILED
) {
222 ngx_log_error(NGX_LOG_ALERT
, cycle
->log
, ngx_errno
,
223 "mmap(%p:%uz, PROT_NONE, MAP_ANON) red zone failed",
228 if (zone
!= red_zone
) {
229 ngx_log_error(NGX_LOG_ALERT
, cycle
->log
, 0,
230 "red zone %p address was changed to %p", red_zone
, zone
);
234 /* create the thread errno' array */
236 errnos
= ngx_calloc(n
* sizeof(int), cycle
->log
);
237 if (errnos
== NULL
) {
241 /* create the thread tids array */
243 tids
= ngx_calloc((n
+ 1) * sizeof(ngx_tid_t
), cycle
->log
);
250 /* create the thread tls' array */
252 ngx_tls
= ngx_calloc(NGX_THREAD_KEYS_MAX
* (n
+ 1) * sizeof(void *),
254 if (ngx_tls
== NULL
) {
260 last_stack
= zone
+ rz_size
;
261 usable_stack_size
= size
;
262 ngx_thread_stack_size
= size
+ rz_size
;
264 /* allow the spinlock in libc malloc() */
289 ngx_thread_key_create(ngx_tls_key_t
*key
)
291 if (nkeys
>= NGX_THREAD_KEYS_MAX
) {
302 ngx_thread_set_tls(ngx_tls_key_t key
, void *value
)
304 if (key
>= NGX_THREAD_KEYS_MAX
) {
308 ngx_tls
[key
* NGX_THREAD_KEYS_MAX
+ ngx_gettid()] = value
;
314 ngx_mutex_init(ngx_log_t
*log
, ngx_uint_t flags
)
319 m
= ngx_alloc(sizeof(ngx_mutex_t
), log
);
327 if (flags
& NGX_MUTEX_LIGHT
) {
332 m
->semid
= semget(IPC_PRIVATE
, 1, SEM_R
|SEM_A
);
333 if (m
->semid
== -1) {
334 ngx_log_error(NGX_LOG_ALERT
, log
, ngx_errno
, "semget() failed");
340 if (semctl(m
->semid
, 0, SETVAL
, op
) == -1) {
341 ngx_log_error(NGX_LOG_ALERT
, log
, ngx_errno
, "semctl(SETVAL) failed");
343 if (semctl(m
->semid
, 0, IPC_RMID
) == -1) {
344 ngx_log_error(NGX_LOG_ALERT
, log
, ngx_errno
,
345 "semctl(IPC_RMID) failed");
356 ngx_mutex_destroy(ngx_mutex_t
*m
)
358 if (semctl(m
->semid
, 0, IPC_RMID
) == -1) {
359 ngx_log_error(NGX_LOG_ALERT
, m
->log
, ngx_errno
,
360 "semctl(IPC_RMID) failed");
363 ngx_free((void *) m
);
368 ngx_mutex_dolock(ngx_mutex_t
*m
, ngx_int_t
try)
380 ngx_log_debug2(NGX_LOG_DEBUG_MUTEX
, m
->log
, 0,
381 "try lock mutex %p lock:%XD", m
, m
->lock
);
383 ngx_log_debug2(NGX_LOG_DEBUG_MUTEX
, m
->log
, 0,
384 "lock mutex %p lock:%XD", m
, m
->lock
);
392 if (old
& NGX_MUTEX_LOCK_BUSY
) {
398 if (ngx_ncpu
> 1 && tries
++ < 1000) {
400 /* the spinlock is used only on the SMP system */
406 if (m
->semid
== -1) {
414 ngx_log_debug2(NGX_LOG_DEBUG_MUTEX
, m
->log
, 0,
415 "mutex %p lock:%XD", m
, m
->lock
);
418 * The mutex is locked so we increase a number
419 * of the threads that are waiting on the mutex
424 if ((lock
& ~NGX_MUTEX_LOCK_BUSY
) > nthreads
) {
425 ngx_log_error(NGX_LOG_ALERT
, m
->log
, ngx_errno
,
426 "%D threads wait for mutex %p, "
427 "while only %ui threads are available",
428 lock
& ~NGX_MUTEX_LOCK_BUSY
, m
, nthreads
);
432 if (ngx_atomic_cmp_set(&m
->lock
, old
, lock
)) {
434 ngx_log_debug2(NGX_LOG_DEBUG_MUTEX
, m
->log
, 0,
435 "wait mutex %p lock:%XD", m
, m
->lock
);
438 * The number of the waiting threads has been increased
439 * and we would wait on the SysV semaphore.
440 * A semaphore should wake up us more efficiently than
441 * a simple sched_yield() or usleep().
448 if (semop(m
->semid
, &op
, 1) == -1) {
449 ngx_log_error(NGX_LOG_ALERT
, m
->log
, ngx_errno
,
450 "semop() failed while waiting on mutex %p", m
);
454 ngx_log_debug2(NGX_LOG_DEBUG_MUTEX
, m
->log
, 0,
455 "mutex waked up %p lock:%XD", m
, m
->lock
);
465 lock
= old
| NGX_MUTEX_LOCK_BUSY
;
467 if (ngx_atomic_cmp_set(&m
->lock
, old
, lock
)) {
469 /* we locked the mutex */
477 if (tries
++ > 1000) {
479 ngx_log_debug1(NGX_LOG_DEBUG_MUTEX
, m
->log
, 0,
480 "mutex %p is contested", m
);
482 /* the mutex is probably contested so we are giving up now */
491 ngx_log_debug2(NGX_LOG_DEBUG_MUTEX
, m
->log
, 0,
492 "mutex %p is locked, lock:%XD", m
, m
->lock
);
499 ngx_mutex_unlock(ngx_mutex_t
*m
)
510 if (!(old
& NGX_MUTEX_LOCK_BUSY
)) {
511 ngx_log_error(NGX_LOG_ALERT
, m
->log
, 0,
512 "trying to unlock the free mutex %p", m
);
519 ngx_log_debug2(NGX_LOG_DEBUG_MUTEX
, m
->log
, 0,
520 "unlock mutex %p lock:%XD", m
, old
);
524 lock
= old
& ~NGX_MUTEX_LOCK_BUSY
;
526 if (ngx_atomic_cmp_set(&m
->lock
, old
, lock
)) {
533 if (m
->semid
== -1) {
534 ngx_log_debug1(NGX_LOG_DEBUG_MUTEX
, m
->log
, 0,
535 "mutex %p is unlocked", m
);
540 /* check whether we need to wake up a waiting thread */
545 if (old
& NGX_MUTEX_LOCK_BUSY
) {
547 /* the mutex is just locked by another thread */
556 /* there are the waiting threads */
560 if (ngx_atomic_cmp_set(&m
->lock
, old
, lock
)) {
562 /* wake up the thread that waits on semaphore */
564 ngx_log_debug1(NGX_LOG_DEBUG_MUTEX
, m
->log
, 0,
565 "wake up mutex %p", m
);
571 if (semop(m
->semid
, &op
, 1) == -1) {
572 ngx_log_error(NGX_LOG_ALERT
, m
->log
, ngx_errno
,
573 "semop() failed while waking up on mutex %p", m
);
583 ngx_log_debug1(NGX_LOG_DEBUG_MUTEX
, m
->log
, 0,
584 "mutex %p is unlocked", m
);
591 ngx_cond_init(ngx_log_t
*log
)
595 cv
= ngx_alloc(sizeof(ngx_cond_t
), log
);
600 cv
->signo
= NGX_CV_SIGNAL
;
610 ngx_cond_destroy(ngx_cond_t
*cv
)
612 if (close(cv
->kq
) == -1) {
613 ngx_log_error(NGX_LOG_ALERT
, cv
->log
, ngx_errno
,
614 "kqueue close() failed");
622 ngx_cond_wait(ngx_cond_t
*cv
, ngx_mutex_t
*m
)
632 * We have to add the EVFILT_SIGNAL filter in the rfork()ed thread.
633 * Otherwise the thread would not get a signal event.
635 * However, we have not to open the kqueue in the thread,
636 * it is simply handy do it together.
641 ngx_log_error(NGX_LOG_ALERT
, cv
->log
, ngx_errno
, "kqueue() failed");
645 ngx_log_debug2(NGX_LOG_DEBUG_CORE
, cv
->log
, 0,
646 "cv kq:%d signo:%d", cv
->kq
, cv
->signo
);
648 kev
.ident
= cv
->signo
;
649 kev
.filter
= EVFILT_SIGNAL
;
658 if (kevent(cv
->kq
, &kev
, 1, NULL
, 0, &ts
) == -1) {
659 ngx_log_error(NGX_LOG_ALERT
, cv
->log
, ngx_errno
, "kevent() failed");
663 cv
->tid
= ngx_thread_self();
668 ngx_log_debug3(NGX_LOG_DEBUG_CORE
, cv
->log
, 0,
669 "cv %p wait, kq:%d, signo:%d", cv
, cv
->kq
, cv
->signo
);
672 n
= kevent(cv
->kq
, NULL
, 0, &kev
, 1, NULL
);
674 ngx_log_debug2(NGX_LOG_DEBUG_CORE
, cv
->log
, 0,
675 "cv %p kevent: %d", cv
, n
);
679 ngx_log_error((err
== NGX_EINTR
) ? NGX_LOG_INFO
: NGX_LOG_ALERT
,
681 "kevent() failed while waiting condition variable %p",
684 if (err
== NGX_EINTR
) {
692 ngx_log_error(NGX_LOG_ALERT
, cv
->log
, 0,
693 "kevent() returned no events "
694 "while waiting condition variable %p",
699 if (kev
.filter
!= EVFILT_SIGNAL
) {
700 ngx_log_error(NGX_LOG_ALERT
, cv
->log
, 0,
701 "kevent() returned unexpected events: %d "
702 "while waiting condition variable %p",
707 if (kev
.ident
!= (uintptr_t) cv
->signo
) {
708 ngx_log_error(NGX_LOG_ALERT
, cv
->log
, 0,
709 "kevent() returned unexpected signal: %d ",
710 "while waiting condition variable %p",
718 ngx_log_debug1(NGX_LOG_DEBUG_CORE
, cv
->log
, 0, "cv %p is waked up", cv
);
727 ngx_cond_signal(ngx_cond_t
*cv
)
731 ngx_log_debug3(NGX_LOG_DEBUG_CORE
, cv
->log
, 0,
732 "cv %p to signal %P %d",
733 cv
, cv
->tid
, cv
->signo
);
739 if (kill(cv
->tid
, cv
->signo
) == -1) {
743 ngx_log_error(NGX_LOG_ALERT
, cv
->log
, err
,
744 "kill() failed while signaling condition variable %p", cv
);
746 if (err
== NGX_ESRCH
) {
753 ngx_log_debug1(NGX_LOG_DEBUG_CORE
, cv
->log
, 0, "cv %p is signaled", cv
);