From e19be507c3558537e9f2545ba92e771755eddf5f Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Wed, 1 Nov 2017 17:15:26 -0700 Subject: [PATCH] rtld - Add fork hooks for libthread_xu to install * Add fork hooks for libthread_xu to install. rtld must acquire its locks exclusively during a fork, and then release them after the fork is complete, to prevent the fork() from catching the locks in a bad state. See libthread_xu. --- lib/libc/sysvipc/sem.c | 18 +++++++++++----- lib/libthread_xu/thread/thr_join.c | 3 +++ lib/libthread_xu/thread/thr_umtx.c | 39 ++++++++++++++++++++++----------- libexec/rtld-elf/Symbol.map | 3 +++ libexec/rtld-elf/rtld_lock.c | 44 ++++++++++++++++++++++++++++++++------ libexec/rtld-elf/rtld_lock.h | 6 +++++- 6 files changed, 88 insertions(+), 25 deletions(-) diff --git a/lib/libc/sysvipc/sem.c b/lib/libc/sysvipc/sem.c index d33461195b..65f6e658ae 100644 --- a/lib/libc/sysvipc/sem.c +++ b/lib/libc/sysvipc/sem.c @@ -688,6 +688,14 @@ int sysvipc_semop (int semid, struct sembuf *sops, unsigned nsops) { semptr->semzcnt++; else semptr->semncnt++; + + /* + * Get interlock value before rleeasing sem_mutex. + * + * XXX horrible hack until we get a umtx_sleep16() (and a umtx_sleep64()) + * system call. + */ + val_to_sleep = *(int *)&semptr->semval; #ifdef SYSV_SEMS sysv_mutex_unlock(&semptr->sem_mutex); #endif @@ -698,7 +706,7 @@ int sysvipc_semop (int semid, struct sembuf *sops, unsigned nsops) { for (j = 0; j < i; j++) { xsemptr = &semaptr->ds.sem_base[sops[j].sem_num]; #ifdef SYSV_SEMS - sysv_mutex_lock(&semptr->sem_mutex); + sysv_mutex_lock(&xsemptr->sem_mutex); #endif xsemptr->semval -= sops[j].sem_op; if (xsemptr->semval == 0 && xsemptr->semzcnt > 0) @@ -706,7 +714,7 @@ int sysvipc_semop (int semid, struct sembuf *sops, unsigned nsops) { if (xsemptr->semval <= 0 && xsemptr->semncnt > 0) umtx_wakeup((int *)&xsemptr->semval, 0); //?! #ifdef SYSV_SEMS - sysv_mutex_unlock(&semptr->sem_mutex); + sysv_mutex_unlock(&xsemptr->sem_mutex); #endif } @@ -731,9 +739,7 @@ int sysvipc_semop (int semid, struct sembuf *sops, unsigned nsops) { * race. * */ - sysv_print("semop: good night!\n"); - val_to_sleep = semptr->semval; rwlock_unlock(semid, semaptr); put_shmdata(semid); @@ -790,8 +796,10 @@ int sysvipc_semop (int semid, struct sembuf *sops, unsigned nsops) { * Is it really morning, or was our sleep interrupted? * (Delayed check of tsleep() return code because we * need to decrement sem[nz]cnt either way.) + * + * Always retry on EBUSY */ - if (eval) { + if (eval == EAGAIN) { eval = EINTR; goto done; } diff --git a/lib/libthread_xu/thread/thr_join.c b/lib/libthread_xu/thread/thr_join.c index 5464b576b8..d82d4ded3e 100644 --- a/lib/libthread_xu/thread/thr_join.c +++ b/lib/libthread_xu/thread/thr_join.c @@ -31,6 +31,8 @@ #include "thr_private.h" +#define cpu_ccfence() __asm __volatile("" : : : "memory") + int _pthread_timedjoin_np(pthread_t pthread, void **thread_return, const struct timespec *abstime); @@ -102,6 +104,7 @@ join_common(pthread_t pthread, void **thread_return, oldcancel = _thr_cancel_enter(curthread); while ((state = pthread->state) != PS_DEAD) { + cpu_ccfence(); if (abstime != NULL) { clock_gettime(CLOCK_REALTIME, &ts); TIMESPEC_SUB(&ts2, abstime, &ts); diff --git a/lib/libthread_xu/thread/thr_umtx.c b/lib/libthread_xu/thread/thr_umtx.c index 386b330de8..9ef080afdf 100644 --- a/lib/libthread_xu/thread/thr_umtx.c +++ b/lib/libthread_xu/thread/thr_umtx.c @@ -34,8 +34,13 @@ #include "thr_private.h" +#define cpu_ccfence() __asm __volatile("" : : : "memory") + /* * This function is used to acquire a contested lock. + * + * A *mtx value of 1 indicates locked normally. + * A *mtx value of 2 indicates locked and contested. */ int __thr_umtx_lock(volatile umtx_t *mtx, int timo) @@ -43,14 +48,17 @@ __thr_umtx_lock(volatile umtx_t *mtx, int timo) int v, errval, ret = 0; /* contested */ - do { + for (;;) { v = *mtx; + cpu_ccfence(); + if (v == 0 && atomic_cmpset_acq_int(mtx, 0, 1)) + break; if (v == 2 || atomic_cmpset_acq_int(mtx, 1, 2)) { - if (timo == 0) + if (timo == 0) { _umtx_sleep_err(mtx, 2, timo); - else if ( (errval = _umtx_sleep_err(mtx, 2, timo)) > 0) { + } else if ((errval = _umtx_sleep_err(mtx, 2, timo)) > 0) { if (errval == EAGAIN) { - if (atomic_cmpset_acq_int(mtx, 0, 2)) + if (atomic_cmpset_acq_int(mtx, 0, 1)) ret = 0; else ret = ETIMEDOUT; @@ -58,11 +66,15 @@ __thr_umtx_lock(volatile umtx_t *mtx, int timo) } } } - } while (!atomic_cmpset_acq_int(mtx, 0, 2)); + } return (ret); } +/* + * Release a mutex. A contested mutex has a value + * of 2, an uncontested mutex has a value of 1. + */ void __thr_umtx_unlock(volatile umtx_t *mtx) { @@ -70,11 +82,10 @@ __thr_umtx_unlock(volatile umtx_t *mtx) for (;;) { v = *mtx; - if (atomic_cmpset_acq_int(mtx, v, v-1)) { - if (v != 1) { - *mtx = 0; + cpu_ccfence(); + if (atomic_cmpset_acq_int(mtx, v, 0)) { + if (v != 1) _umtx_wakeup_err(mtx, 1); - } break; } } @@ -128,16 +139,17 @@ _thr_umtx_wait(volatile umtx_t *mtx, int exp, const struct timespec *timeout, struct timespec ts, ts2, ts3; int timo, errval, ret = 0; + cpu_ccfence(); if (*mtx != exp) return (0); if (timeout == NULL) { - while ( (errval = _umtx_sleep_err(mtx, exp, 10000000)) > 0) { + while ((errval = _umtx_sleep_err(mtx, exp, 10000000)) > 0) { if (errval == EBUSY) break; if (errval == EINTR) { ret = EINTR; - break; + break; } #if 0 if (errval == ETIMEDOUT || errval == EWOULDBLOCK) { @@ -172,11 +184,12 @@ _thr_umtx_wait(volatile umtx_t *mtx, int exp, const struct timespec *timeout, timo = 1000000; } - if ( (errval = _umtx_sleep_err(mtx, exp, timo)) > 0) { + if ((errval = _umtx_sleep_err(mtx, exp, timo)) > 0) { if (errval == EBUSY) { ret = 0; break; - } else if (errval == EINTR) { + } + if (errval == EINTR) { ret = EINTR; break; } diff --git a/libexec/rtld-elf/Symbol.map b/libexec/rtld-elf/Symbol.map index b2789d9d50..3490faf9f4 100644 --- a/libexec/rtld-elf/Symbol.map +++ b/libexec/rtld-elf/Symbol.map @@ -23,6 +23,9 @@ DFprivate_1.0 { _rtld_free_tls; _rtld_call_init; _rtld_thread_init; + _rtld_thread_prefork; + _rtld_thread_postfork; + _rtld_thread_childfork; _rtld_addr_phdr; _rtld_get_stack_prot; _r_debug_postinit; diff --git a/libexec/rtld-elf/rtld_lock.c b/libexec/rtld-elf/rtld_lock.c index 1cd30b903d..25bca852b8 100644 --- a/libexec/rtld-elf/rtld_lock.c +++ b/libexec/rtld-elf/rtld_lock.c @@ -176,6 +176,7 @@ thread_mask_clear(int mask) struct rtld_lock { void *handle; int mask; + RtldLockState forkstate; } rtld_locks[RTLD_LOCK_CNT]; rtld_lock_t rtld_bind_lock = &rtld_locks[0]; @@ -201,7 +202,6 @@ rlock_acquire(rtld_lock_t lock, RtldLockState *lockstate) void wlock_acquire(rtld_lock_t lock, RtldLockState *lockstate) { - if (lockstate == NULL) return; @@ -217,7 +217,6 @@ wlock_acquire(rtld_lock_t lock, RtldLockState *lockstate) void lock_release(rtld_lock_t lock, RtldLockState *lockstate) { - if (lockstate == NULL) return; @@ -237,7 +236,6 @@ lock_release(rtld_lock_t lock, RtldLockState *lockstate) void lock_upgrade(rtld_lock_t lock, RtldLockState *lockstate) { - if (lockstate == NULL) return; @@ -248,7 +246,6 @@ lock_upgrade(rtld_lock_t lock, RtldLockState *lockstate) void lock_restart_for_upgrade(RtldLockState *lockstate) { - if (lockstate == NULL) return; @@ -321,13 +318,15 @@ _rtld_thread_init(struct RtldLockInfo *pli) pli = &deflockinfo; - for (i = 0; i < RTLD_LOCK_CNT; i++) + for (i = 0; i < RTLD_LOCK_CNT; i++) { if ((locks[i] = pli->lock_create()) == NULL) break; + } if (i < RTLD_LOCK_CNT) { - while (--i >= 0) + while (--i >= 0) { pli->lock_destroy(locks[i]); + } abort(); } @@ -359,3 +358,36 @@ _rtld_thread_init(struct RtldLockInfo *pli) thread_mask_set(flags); dbg("_rtld_thread_init: done"); } + +void +_rtld_thread_prefork(void) +{ + int i; + + for (i = 0; i < RTLD_LOCK_CNT; i++) { + if (rtld_locks[i].handle == NULL) + continue; + wlock_acquire(&rtld_locks[i], &rtld_locks[i].forkstate); + } +} + +void +_rtld_thread_postfork(void) +{ + int i; + + for (i = 0; i < RTLD_LOCK_CNT; i++) { + if (rtld_locks[i].handle == NULL) + continue; + lock_release(&rtld_locks[i], &rtld_locks[i].forkstate); + } +} + +/* + * pthreads already called _rtld_thread_init(NULL) in the child, so there + * is nothing for us to do here. + */ +void +_rtld_thread_childfork(void) +{ +} diff --git a/libexec/rtld-elf/rtld_lock.h b/libexec/rtld-elf/rtld_lock.h index 6d51423ade..3096f1f377 100644 --- a/libexec/rtld-elf/rtld_lock.h +++ b/libexec/rtld-elf/rtld_lock.h @@ -40,10 +40,14 @@ struct RtldLockInfo void (*lock_release)(void *); int (*thread_set_flag)(int); int (*thread_clr_flag)(int); - void (*at_fork)(void); + void (*at_fork)(void *handle); }; extern void _rtld_thread_init(struct RtldLockInfo *); +extern void _rtld_thread_prefork(void); +extern void _rtld_thread_postfork(void); +extern void _rtld_thread_childfork(void); + #ifdef IN_RTLD -- 2.11.4.GIT