2 #include "pthread_impl.h"
3 #include "stdio_impl.h"
13 weak_alias(dummy_0
, __acquire_ptc
);
14 weak_alias(dummy_0
, __release_ptc
);
15 weak_alias(dummy_0
, __pthread_tsd_run_dtors
);
16 weak_alias(dummy_0
, __do_orphaned_stdio_locks
);
17 weak_alias(dummy_0
, __dl_thread_cleanup
);
18 weak_alias(dummy_0
, __membarrier_init
);
20 static int tl_lock_count
;
21 static int tl_lock_waiters
;
25 int tid
= __pthread_self()->tid
;
26 int val
= __thread_list_lock
;
31 while ((val
= a_cas(&__thread_list_lock
, 0, tid
)))
32 __wait(&__thread_list_lock
, &tl_lock_waiters
, val
, 0);
35 void __tl_unlock(void)
41 a_store(&__thread_list_lock
, 0);
42 if (tl_lock_waiters
) __wake(&__thread_list_lock
, 1, 0);
45 void __tl_sync(pthread_t td
)
48 int val
= __thread_list_lock
;
50 __wait(&__thread_list_lock
, &tl_lock_waiters
, val
, 0);
51 if (tl_lock_waiters
) __wake(&__thread_list_lock
, 1, 0);
54 _Noreturn
void __pthread_exit(void *result
)
56 pthread_t self
= __pthread_self();
59 self
->canceldisable
= 1;
60 self
->cancelasync
= 0;
61 self
->result
= result
;
63 while (self
->cancelbuf
) {
64 void (*f
)(void *) = self
->cancelbuf
->__f
;
65 void *x
= self
->cancelbuf
->__x
;
66 self
->cancelbuf
= self
->cancelbuf
->__next
;
70 __pthread_tsd_run_dtors();
72 /* Access to target the exiting thread with syscalls that use
73 * its kernel tid is controlled by killlock. For detached threads,
74 * any use past this point would have undefined behavior, but for
75 * joinable threads it's a valid usage that must be handled. */
78 /* The thread list lock must be AS-safe, and thus requires
79 * application signals to be blocked before it can be taken. */
80 __block_app_sigs(&set
);
83 /* If this is the only thread in the list, don't proceed with
84 * termination of the thread, but restore the previous lock and
85 * signal state to prepare for exit to call atexit handlers. */
86 if (self
->next
== self
) {
89 UNLOCK(self
->killlock
);
93 /* At this point we are committed to thread termination. Unlink
94 * the thread from the list. This change will not be visible
95 * until the lock is released, which only happens after SYS_exit
96 * has been called, via the exit futex address pointing at the lock. */
97 libc
.threads_minus_1
--;
98 self
->next
->prev
= self
->prev
;
99 self
->prev
->next
= self
->next
;
100 self
->prev
= self
->next
= self
;
102 /* Process robust list in userspace to handle non-pshared mutexes
103 * and the detached thread case where the robust list head will
104 * be invalid when the kernel would process it. */
106 volatile void *volatile *rp
;
107 while ((rp
=self
->robust_list
.head
) && rp
!= &self
->robust_list
.head
) {
108 pthread_mutex_t
*m
= (void *)((char *)rp
109 - offsetof(pthread_mutex_t
, _m_next
));
110 int waiters
= m
->_m_waiters
;
111 int priv
= (m
->_m_type
& 128) ^ 128;
112 self
->robust_list
.pending
= rp
;
113 self
->robust_list
.head
= *rp
;
114 int cont
= a_swap(&m
->_m_lock
, 0x40000000);
115 self
->robust_list
.pending
= 0;
116 if (cont
< 0 || waiters
)
117 __wake(&m
->_m_lock
, 1, priv
);
121 __do_orphaned_stdio_locks();
122 __dl_thread_cleanup();
124 /* This atomic potentially competes with a concurrent pthread_detach
125 * call; the loser is responsible for freeing thread resources. */
126 int state
= a_cas(&self
->detach_state
, DT_JOINABLE
, DT_EXITING
);
128 if (state
==DT_DETACHED
&& self
->map_base
) {
129 /* Detached threads must block even implementation-internal
130 * signals, since they will not have a stack in their last
131 * moments of existence. */
132 __block_all_sigs(&set
);
134 /* Robust list will no longer be valid, and was already
135 * processed above, so unregister it with the kernel. */
136 if (self
->robust_list
.off
)
137 __syscall(SYS_set_robust_list
, 0, 3*sizeof(long));
139 /* Since __unmapself bypasses the normal munmap code path,
140 * explicitly wait for vmlock holders first. */
143 /* The following call unmaps the thread's stack mapping
144 * and then exits without touching the stack. */
145 __unmapself(self
->map_base
, self
->map_size
);
148 /* Wake any joiner. */
149 __wake(&self
->detach_state
, 1, 1);
151 /* After the kernel thread exits, its tid may be reused. Clear it
152 * to prevent inadvertent use and inform functions that would use
153 * it that it's no longer available. */
155 UNLOCK(self
->killlock
);
157 for (;;) __syscall(SYS_exit
, 0);
160 void __do_cleanup_push(struct __ptcb
*cb
)
162 struct pthread
*self
= __pthread_self();
163 cb
->__next
= self
->cancelbuf
;
164 self
->cancelbuf
= cb
;
167 void __do_cleanup_pop(struct __ptcb
*cb
)
169 __pthread_self()->cancelbuf
= cb
->__next
;
173 void *(*start_func
)(void *);
175 volatile int control
;
176 unsigned long sig_mask
[_NSIG
/8/sizeof(long)];
179 static int start(void *p
)
181 struct start_args
*args
= p
;
182 int state
= args
->control
;
184 if (a_cas(&args
->control
, 1, 2)==1)
185 __wait(&args
->control
, 0, 2, 1);
187 __syscall(SYS_set_tid_address
, &args
->control
);
188 for (;;) __syscall(SYS_exit
, 0);
191 __syscall(SYS_rt_sigprocmask
, SIG_SETMASK
, &args
->sig_mask
, 0, _NSIG
/8);
192 __pthread_exit(args
->start_func(args
->start_arg
));
196 static int start_c11(void *p
)
198 struct start_args
*args
= p
;
199 int (*start
)(void*) = (int(*)(void*)) args
->start_func
;
200 __pthread_exit((void *)(uintptr_t)start(args
->start_arg
));
204 #define ROUND(x) (((x)+PAGE_SIZE-1)&-PAGE_SIZE)
206 /* pthread_key_create.c overrides this */
207 static volatile size_t dummy
= 0;
208 weak_alias(dummy
, __pthread_tsd_size
);
209 static void *dummy_tsd
[1] = { 0 };
210 weak_alias(dummy_tsd
, __pthread_tsd_main
);
212 static FILE *volatile dummy_file
= 0;
213 weak_alias(dummy_file
, __stdin_used
);
214 weak_alias(dummy_file
, __stdout_used
);
215 weak_alias(dummy_file
, __stderr_used
);
217 static void init_file_lock(FILE *f
)
219 if (f
&& f
->lock
<0) f
->lock
= 0;
222 int __pthread_create(pthread_t
*restrict res
, const pthread_attr_t
*restrict attrp
, void *(*entry
)(void *), void *restrict arg
)
224 int ret
, c11
= (attrp
== __ATTRP_C11_THREAD
);
226 struct pthread
*self
, *new;
227 unsigned char *map
= 0, *stack
= 0, *tsd
= 0, *stack_limit
;
228 unsigned flags
= CLONE_VM
| CLONE_FS
| CLONE_FILES
| CLONE_SIGHAND
229 | CLONE_THREAD
| CLONE_SYSVSEM
| CLONE_SETTLS
230 | CLONE_PARENT_SETTID
| CLONE_CHILD_CLEARTID
| CLONE_DETACHED
;
231 pthread_attr_t attr
= { 0 };
234 if (!libc
.can_do_threads
) return ENOSYS
;
235 self
= __pthread_self();
236 if (!libc
.threaded
) {
237 for (FILE *f
=*__ofl_lock(); f
; f
=f
->next
)
240 init_file_lock(__stdin_used
);
241 init_file_lock(__stdout_used
);
242 init_file_lock(__stderr_used
);
243 __syscall(SYS_rt_sigprocmask
, SIG_UNBLOCK
, SIGPT_SET
, 0, _NSIG
/8);
244 self
->tsd
= (void **)__pthread_tsd_main
;
248 if (attrp
&& !c11
) attr
= *attrp
;
252 attr
._a_stacksize
= __default_stacksize
;
253 attr
._a_guardsize
= __default_guardsize
;
256 if (attr
._a_stackaddr
) {
257 size_t need
= libc
.tls_size
+ __pthread_tsd_size
;
258 size
= attr
._a_stacksize
;
259 stack
= (void *)(attr
._a_stackaddr
& -16);
260 stack_limit
= (void *)(attr
._a_stackaddr
- size
);
261 /* Use application-provided stack for TLS only when
262 * it does not take more than ~12% or 2k of the
263 * application's stack space. */
264 if (need
< size
/8 && need
< 2048) {
265 tsd
= stack
- __pthread_tsd_size
;
266 stack
= tsd
- libc
.tls_size
;
267 memset(stack
, 0, need
);
273 guard
= ROUND(attr
._a_guardsize
);
274 size
= guard
+ ROUND(attr
._a_stacksize
275 + libc
.tls_size
+ __pthread_tsd_size
);
280 map
= __mmap(0, size
, PROT_NONE
, MAP_PRIVATE
|MAP_ANON
, -1, 0);
281 if (map
== MAP_FAILED
) goto fail
;
282 if (__mprotect(map
+guard
, size
-guard
, PROT_READ
|PROT_WRITE
)
283 && errno
!= ENOSYS
) {
288 map
= __mmap(0, size
, PROT_READ
|PROT_WRITE
, MAP_PRIVATE
|MAP_ANON
, -1, 0);
289 if (map
== MAP_FAILED
) goto fail
;
291 tsd
= map
+ size
- __pthread_tsd_size
;
293 stack
= tsd
- libc
.tls_size
;
294 stack_limit
= map
+ guard
;
298 new = __copy_tls(tsd
- libc
.tls_size
);
300 new->map_size
= size
;
302 new->stack_size
= stack
- stack_limit
;
303 new->guard_size
= guard
;
305 new->tsd
= (void *)tsd
;
306 new->locale
= &libc
.global_locale
;
307 if (attr
._a_detach
) {
308 new->detach_state
= DT_DETACHED
;
310 new->detach_state
= DT_JOINABLE
;
312 new->robust_list
.head
= &new->robust_list
.head
;
313 new->CANARY
= self
->CANARY
;
314 new->sysinfo
= self
->sysinfo
;
316 /* Setup argument structure for the new thread on its stack.
317 * It's safe to access from the caller only until the thread
318 * list is unlocked. */
319 stack
-= (uintptr_t)stack
% sizeof(uintptr_t);
320 stack
-= sizeof(struct start_args
);
321 struct start_args
*args
= (void *)stack
;
322 args
->start_func
= entry
;
323 args
->start_arg
= arg
;
324 args
->control
= attr
._a_sched
? 1 : 0;
326 /* Application signals (but not the synccall signal) must be
327 * blocked before the thread list lock can be taken, to ensure
328 * that the lock is AS-safe. */
329 __block_app_sigs(&set
);
331 /* Ensure SIGCANCEL is unblocked in new thread. This requires
332 * working with a copy of the set so we can restore the
333 * original mask in the calling thread. */
334 memcpy(&args
->sig_mask
, &set
, sizeof args
->sig_mask
);
335 args
->sig_mask
[(SIGCANCEL
-1)/8/sizeof(long)] &=
336 ~(1UL<<((SIGCANCEL
-1)%(8*sizeof(long))));
339 libc
.threads_minus_1
++;
340 ret
= __clone((c11
? start_c11
: start
), stack
, flags
, args
, &new->tid
, TP_ADJ(new), &__thread_list_lock
);
342 /* All clone failures translate to EAGAIN. If explicit scheduling
343 * was requested, attempt it before unlocking the thread list so
344 * that the failed thread is never exposed and so that we can
345 * clean up all transient resource usage before returning. */
348 } else if (attr
._a_sched
) {
349 ret
= __syscall(SYS_sched_setscheduler
,
350 new->tid
, attr
._a_policy
, &attr
._a_prio
);
351 if (a_swap(&args
->control
, ret
? 3 : 0)==2)
352 __wake(&args
->control
, 1, 1);
354 __wait(&args
->control
, 0, 3, 0);
358 new->next
= self
->next
;
360 new->next
->prev
= new;
361 new->prev
->next
= new;
363 libc
.threads_minus_1
--;
366 __restore_sigs(&set
);
370 if (map
) __munmap(map
, size
);
381 weak_alias(__pthread_exit
, pthread_exit
);
382 weak_alias(__pthread_create
, pthread_create
);