8158 Want named threads API

[unleashed.git] / usr / src / lib / libc / port / threads / thr.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2016 by Delphix. All rights reserved.
 * Copyright (c) 2017 by The MathWorks, Inc. All rights reserved.
 */
/*
 * Copyright 2018 Joyent, Inc.
 */

#include "lint.h"
#include "thr_uberdata.h"
#include <pthread.h>
#include <procfs.h>
#include <sys/uio.h>
#include <ctype.h>
#include "libc.h"

/*
 * These symbols should not be exported from libc, but
 * /lib/libm.so.2 references _thr_main.  libm needs to be fixed.
 * Also, some older versions of the Studio compiler/debugger
 * components reference them.  These need to be fixed, too.
 */
#pragma weak _thr_main = thr_main
#pragma weak _thr_create = thr_create
#pragma weak _thr_join = thr_join
#pragma weak _thr_self = thr_self

#undef errno
extern int errno;

/*
 * Between Solaris 2.5 and Solaris 9, __threaded was used to indicate
 * "we are linked with libthread".  The Sun Workshop 6 update 1 compilation
 * system used it illegally (it is a consolidation private symbol).
 * To accommodate this and possibly other abusers of the symbol,
 * we make it always equal to 1 now that libthread has been folded
 * into libc.  The new __libc_threaded symbol is used to indicate
 * the new meaning, "more than one thread exists".
 */
int __threaded = 1;             /* always equal to 1 */
int __libc_threaded = 0;        /* zero until first thr_create() */

/*
 * thr_concurrency and pthread_concurrency are not used by the library.
 * They exist solely to hold and return the values set by calls to
 * thr_setconcurrency() and pthread_setconcurrency().
 * Because thr_concurrency is affected by the THR_NEW_LWP flag
 * to thr_create(), thr_concurrency is protected by link_lock.
 */
static  int     thr_concurrency = 1;
static  int     pthread_concurrency;

#define HASHTBLSZ       1024    /* must be a power of two */
#define TIDHASH(tid, udp)       (tid & (udp)->hash_mask)

/* initial allocation, just enough for one lwp */
#pragma align 64(init_hash_table)
thr_hash_table_t init_hash_table[1] = {
        { DEFAULTMUTEX, DEFAULTCV, NULL },
};

extern const Lc_interface rtld_funcs[];

/*
 * The weak version is known to libc_db and mdb.
 */
#pragma weak _uberdata = __uberdata
uberdata_t __uberdata = {
        { DEFAULTMUTEX, NULL, 0 },      /* link_lock */
        { RECURSIVEMUTEX, NULL, 0 },    /* ld_lock */
        { RECURSIVEMUTEX, NULL, 0 },    /* fork_lock */
        { RECURSIVEMUTEX, NULL, 0 },    /* atfork_lock */
        { RECURSIVEMUTEX, NULL, 0 },    /* callout_lock */
        { DEFAULTMUTEX, NULL, 0 },      /* tdb_hash_lock */
        { 0, },                         /* tdb_hash_lock_stats */
        { { 0 }, },                     /* siguaction[NSIG] */
        {{ DEFAULTMUTEX, NULL, 0 },             /* bucket[NBUCKETS] */
        { DEFAULTMUTEX, NULL, 0 },
        { DEFAULTMUTEX, NULL, 0 },
        { DEFAULTMUTEX, NULL, 0 },
        { DEFAULTMUTEX, NULL, 0 },
        { DEFAULTMUTEX, NULL, 0 },
        { DEFAULTMUTEX, NULL, 0 },
        { DEFAULTMUTEX, NULL, 0 },
        { DEFAULTMUTEX, NULL, 0 },
        { DEFAULTMUTEX, NULL, 0 }},
        { RECURSIVEMUTEX, NULL, NULL },         /* atexit_root */
        { RECURSIVEMUTEX, NULL },               /* quickexit_root */
        { DEFAULTMUTEX, 0, 0, NULL },           /* tsd_metadata */
        { DEFAULTMUTEX, {0, 0}, {0, 0} },       /* tls_metadata */
        0,                      /* primary_map */
        0,                      /* bucket_init */
        0,                      /* pad[0] */
        0,                      /* pad[1] */
        { 0 },                  /* uberflags */
        NULL,                   /* queue_head */
        init_hash_table,        /* thr_hash_table */
        1,                      /* hash_size: size of the hash table */
        0,                      /* hash_mask: hash_size - 1 */
        NULL,                   /* ulwp_one */
        NULL,                   /* all_lwps */
        NULL,                   /* all_zombies */
        0,                      /* nthreads */
        0,                      /* nzombies */
        0,                      /* ndaemons */
        0,                      /* pid */
        sigacthandler,          /* sigacthandler */
        NULL,                   /* lwp_stacks */
        NULL,                   /* lwp_laststack */
        0,                      /* nfreestack */
        10,                     /* thread_stack_cache */
        NULL,                   /* ulwp_freelist */
        NULL,                   /* ulwp_lastfree */
        NULL,                   /* ulwp_replace_free */
        NULL,                   /* ulwp_replace_last */
        NULL,                   /* atforklist */
        NULL,                   /* robustlocks */
        NULL,                   /* robustlist */
        NULL,                   /* progname */
        NULL,                   /* ub_comm_page */
        NULL,                   /* __tdb_bootstrap */
        {                       /* tdb */
                NULL,           /* tdb_sync_addr_hash */
                0,              /* tdb_register_count */
                0,              /* tdb_hash_alloc_failed */
                NULL,           /* tdb_sync_addr_free */
                NULL,           /* tdb_sync_addr_last */
                0,              /* tdb_sync_alloc */
                { 0, 0 },       /* tdb_ev_global_mask */
                tdb_events,     /* tdb_events array */
        },
};

/*
 * The weak version is known to libc_db and mdb.
 */
#pragma weak _tdb_bootstrap = __tdb_bootstrap
uberdata_t **__tdb_bootstrap = NULL;

int     thread_queue_fifo = 4;
int     thread_queue_dump = 0;
int     thread_cond_wait_defer = 0;
int     thread_error_detection = 0;
int     thread_async_safe = 0;
int     thread_stack_cache = 10;
int     thread_door_noreserve = 0;
int     thread_locks_misaligned = 0;

static  ulwp_t  *ulwp_alloc(void);
static  void    ulwp_free(ulwp_t *);

/*
 * Insert the lwp into the hash table.
 */
void
hash_in_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp)
{
        ulwp->ul_hash = udp->thr_hash_table[ix].hash_bucket;
        udp->thr_hash_table[ix].hash_bucket = ulwp;
        ulwp->ul_ix = ix;
}

void
hash_in(ulwp_t *ulwp, uberdata_t *udp)
{
        int ix = TIDHASH(ulwp->ul_lwpid, udp);
        mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;

        lmutex_lock(mp);
        hash_in_unlocked(ulwp, ix, udp);
        lmutex_unlock(mp);
}

/*
 * Delete the lwp from the hash table.
 */
void
hash_out_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp)
{
        ulwp_t **ulwpp;

        for (ulwpp = &udp->thr_hash_table[ix].hash_bucket;
            ulwp != *ulwpp;
            ulwpp = &(*ulwpp)->ul_hash)
                ;
        *ulwpp = ulwp->ul_hash;
        ulwp->ul_hash = NULL;
        ulwp->ul_ix = -1;
}

void
hash_out(ulwp_t *ulwp, uberdata_t *udp)
{
        int ix;

        if ((ix = ulwp->ul_ix) >= 0) {
                mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;

                lmutex_lock(mp);
                hash_out_unlocked(ulwp, ix, udp);
                lmutex_unlock(mp);
        }
}

/*
 * Retain stack information for thread structures that are being recycled for
 * new threads.  All other members of the thread structure should be zeroed.
 */
static void
ulwp_clean(ulwp_t *ulwp)
{
        caddr_t stk = ulwp->ul_stk;
        size_t mapsiz = ulwp->ul_mapsiz;
        size_t guardsize = ulwp->ul_guardsize;
        uintptr_t stktop = ulwp->ul_stktop;
        size_t stksiz = ulwp->ul_stksiz;

        (void) memset(ulwp, 0, sizeof (*ulwp));

        ulwp->ul_stk = stk;
        ulwp->ul_mapsiz = mapsiz;
        ulwp->ul_guardsize = guardsize;
        ulwp->ul_stktop = stktop;
        ulwp->ul_stksiz = stksiz;
}

static int stackprot;

/*
 * Answer the question, "Is the lwp in question really dead?"
 * We must inquire of the operating system to be really sure
 * because the lwp may have called lwp_exit() but it has not
 * yet completed the exit.
 */
static int
dead_and_buried(ulwp_t *ulwp)
{
        if (ulwp->ul_lwpid == (lwpid_t)(-1))
                return (1);
        if (ulwp->ul_dead && ulwp->ul_detached &&
            _lwp_kill(ulwp->ul_lwpid, 0) == ESRCH) {
                ulwp->ul_lwpid = (lwpid_t)(-1);
                return (1);
        }
        return (0);
}

/*
 * Attempt to keep the stack cache within the specified cache limit.
 */
static void
trim_stack_cache(int cache_limit)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        ulwp_t *prev = NULL;
        ulwp_t **ulwpp = &udp->lwp_stacks;
        ulwp_t *ulwp;

        ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, self));

        while (udp->nfreestack > cache_limit && (ulwp = *ulwpp) != NULL) {
                if (dead_and_buried(ulwp)) {
                        *ulwpp = ulwp->ul_next;
                        if (ulwp == udp->lwp_laststack)
                                udp->lwp_laststack = prev;
                        hash_out(ulwp, udp);
                        udp->nfreestack--;
                        (void) munmap(ulwp->ul_stk, ulwp->ul_mapsiz);
                        /*
                         * Now put the free ulwp on the ulwp freelist.
                         */
                        ulwp->ul_mapsiz = 0;
                        ulwp->ul_next = NULL;
                        if (udp->ulwp_freelist == NULL)
                                udp->ulwp_freelist = udp->ulwp_lastfree = ulwp;
                        else {
                                udp->ulwp_lastfree->ul_next = ulwp;
                                udp->ulwp_lastfree = ulwp;
                        }
                } else {
                        prev = ulwp;
                        ulwpp = &ulwp->ul_next;
                }
        }
}

/*
 * Find an unused stack of the requested size
 * or create a new stack of the requested size.
 * Return a pointer to the ulwp_t structure referring to the stack, or NULL.
 * thr_exit() stores 1 in the ul_dead member.
 * thr_join() stores -1 in the ul_lwpid member.
 */
static ulwp_t *
find_stack(size_t stksize, size_t guardsize)
{
        static size_t pagesize = 0;

        uberdata_t *udp = curthread->ul_uberdata;
        size_t mapsize;
        ulwp_t *prev;
        ulwp_t *ulwp;
        ulwp_t **ulwpp;
        void *stk;

        /*
         * The stack is allocated PROT_READ|PROT_WRITE|PROT_EXEC
         * unless overridden by the system's configuration.
         */
        if (stackprot == 0) {   /* do this once */
                long lprot = _sysconf(_SC_STACK_PROT);
                if (lprot <= 0)
                        lprot = (PROT_READ|PROT_WRITE|PROT_EXEC);
                stackprot = (int)lprot;
        }
        if (pagesize == 0)      /* do this once */
                pagesize = _sysconf(_SC_PAGESIZE);

        /*
         * One megabyte stacks by default, but subtract off
         * two pages for the system-created red zones.
         * Round up a non-zero stack size to a pagesize multiple.
         */
        if (stksize == 0)
                stksize = DEFAULTSTACK - 2 * pagesize;
        else
                stksize = ((stksize + pagesize - 1) & -pagesize);

        /*
         * Round up the mapping size to a multiple of pagesize.
         * Note: mmap() provides at least one page of red zone
         * so we deduct that from the value of guardsize.
         */
        if (guardsize != 0)
                guardsize = ((guardsize + pagesize - 1) & -pagesize) - pagesize;
        mapsize = stksize + guardsize;

        lmutex_lock(&udp->link_lock);
        for (prev = NULL, ulwpp = &udp->lwp_stacks;
            (ulwp = *ulwpp) != NULL;
            prev = ulwp, ulwpp = &ulwp->ul_next) {
                if (ulwp->ul_mapsiz == mapsize &&
                    ulwp->ul_guardsize == guardsize &&
                    dead_and_buried(ulwp)) {
                        /*
                         * The previous lwp is gone; reuse the stack.
                         * Remove the ulwp from the stack list.
                         */
                        *ulwpp = ulwp->ul_next;
                        ulwp->ul_next = NULL;
                        if (ulwp == udp->lwp_laststack)
                                udp->lwp_laststack = prev;
                        hash_out(ulwp, udp);
                        udp->nfreestack--;
                        lmutex_unlock(&udp->link_lock);
                        ulwp_clean(ulwp);
                        return (ulwp);
                }
        }

        /*
         * None of the cached stacks matched our mapping size.
         * Reduce the stack cache to get rid of possibly
         * very old stacks that will never be reused.
         */
        if (udp->nfreestack > udp->thread_stack_cache)
                trim_stack_cache(udp->thread_stack_cache);
        else if (udp->nfreestack > 0)
                trim_stack_cache(udp->nfreestack - 1);
        lmutex_unlock(&udp->link_lock);

        /*
         * Create a new stack.
         */
        if ((stk = mmap(NULL, mapsize, stackprot,
            MAP_PRIVATE|MAP_NORESERVE|MAP_ANON, -1, (off_t)0)) != MAP_FAILED) {
                /*
                 * We have allocated our stack.  Now allocate the ulwp.
                 */
                ulwp = ulwp_alloc();
                if (ulwp == NULL)
                        (void) munmap(stk, mapsize);
                else {
                        ulwp->ul_stk = stk;
                        ulwp->ul_mapsiz = mapsize;
                        ulwp->ul_guardsize = guardsize;
                        ulwp->ul_stktop = (uintptr_t)stk + mapsize;
                        ulwp->ul_stksiz = stksize;
                        if (guardsize)  /* protect the extra red zone */
                                (void) mprotect(stk, guardsize, PROT_NONE);
                }
        }
        return (ulwp);
}

/*
 * Get a ulwp_t structure from the free list or allocate a new one.
 * Such ulwp_t's do not have a stack allocated by the library.
 */
static ulwp_t *
ulwp_alloc(void)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        size_t tls_size;
        ulwp_t *prev;
        ulwp_t *ulwp;
        ulwp_t **ulwpp;
        caddr_t data;

        lmutex_lock(&udp->link_lock);
        for (prev = NULL, ulwpp = &udp->ulwp_freelist;
            (ulwp = *ulwpp) != NULL;
            prev = ulwp, ulwpp = &ulwp->ul_next) {
                if (dead_and_buried(ulwp)) {
                        *ulwpp = ulwp->ul_next;
                        ulwp->ul_next = NULL;
                        if (ulwp == udp->ulwp_lastfree)
                                udp->ulwp_lastfree = prev;
                        hash_out(ulwp, udp);
                        lmutex_unlock(&udp->link_lock);
                        ulwp_clean(ulwp);
                        return (ulwp);
                }
        }
        lmutex_unlock(&udp->link_lock);

        tls_size = roundup64(udp->tls_metadata.static_tls.tls_size);
        data = lmalloc(sizeof (*ulwp) + tls_size);
        if (data != NULL) {
                /* LINTED pointer cast may result in improper alignment */
                ulwp = (ulwp_t *)(data + tls_size);
        }
        return (ulwp);
}

/*
 * Free a ulwp structure.
 * If there is an associated stack, put it on the stack list and
 * munmap() previously freed stacks up to the residual cache limit.
 * Else put it on the ulwp free list and never call lfree() on it.
 */
static void
ulwp_free(ulwp_t *ulwp)
{
        uberdata_t *udp = curthread->ul_uberdata;

        ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, curthread));
        ulwp->ul_next = NULL;
        if (ulwp == udp->ulwp_one)      /* don't reuse the primoridal stack */
                /*EMPTY*/;
        else if (ulwp->ul_mapsiz != 0) {
                if (udp->lwp_stacks == NULL)
                        udp->lwp_stacks = udp->lwp_laststack = ulwp;
                else {
                        udp->lwp_laststack->ul_next = ulwp;
                        udp->lwp_laststack = ulwp;
                }
                if (++udp->nfreestack > udp->thread_stack_cache)
                        trim_stack_cache(udp->thread_stack_cache);
        } else {
                if (udp->ulwp_freelist == NULL)
                        udp->ulwp_freelist = udp->ulwp_lastfree = ulwp;
                else {
                        udp->ulwp_lastfree->ul_next = ulwp;
                        udp->ulwp_lastfree = ulwp;
                }
        }
}

/*
 * Find a named lwp and return a pointer to its hash list location.
 * On success, returns with the hash lock held.
 */
ulwp_t **
find_lwpp(thread_t tid)
{
        uberdata_t *udp = curthread->ul_uberdata;
        int ix = TIDHASH(tid, udp);
        mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
        ulwp_t *ulwp;
        ulwp_t **ulwpp;

        if (tid == 0)
                return (NULL);

        lmutex_lock(mp);
        for (ulwpp = &udp->thr_hash_table[ix].hash_bucket;
            (ulwp = *ulwpp) != NULL;
            ulwpp = &ulwp->ul_hash) {
                if (ulwp->ul_lwpid == tid)
                        return (ulwpp);
        }
        lmutex_unlock(mp);
        return (NULL);
}

/*
 * Wake up all lwps waiting on this lwp for some reason.
 */
void
ulwp_broadcast(ulwp_t *ulwp)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;

        ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self));
        (void) cond_broadcast(ulwp_condvar(ulwp, udp));
}

/*
 * Find a named lwp and return a pointer to it.
 * Returns with the hash lock held.
 */
ulwp_t *
find_lwp(thread_t tid)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        ulwp_t *ulwp = NULL;
        ulwp_t **ulwpp;

        if (self->ul_lwpid == tid) {
                ulwp = self;
                ulwp_lock(ulwp, udp);
        } else if ((ulwpp = find_lwpp(tid)) != NULL) {
                ulwp = *ulwpp;
        }

        if (ulwp && ulwp->ul_dead) {
                ulwp_unlock(ulwp, udp);
                ulwp = NULL;
        }

        return (ulwp);
}

int
_thrp_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
    long flags, thread_t *new_thread, size_t guardsize, const char *name)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        ucontext_t uc;
        uint_t lwp_flags;
        thread_t tid;
        int error;
        ulwp_t *ulwp;

        /*
         * Enforce the restriction of not creating any threads
         * until the primary link map has been initialized.
         * Also, disallow thread creation to a child of vfork().
         */
        if (!self->ul_primarymap || self->ul_vfork)
                return (ENOTSUP);

        if (udp->hash_size == 1)
                finish_init();

        if ((stk || stksize) && stksize < MINSTACK)
                return (EINVAL);

        if (stk == NULL) {
                if ((ulwp = find_stack(stksize, guardsize)) == NULL)
                        return (ENOMEM);
                stksize = ulwp->ul_mapsiz - ulwp->ul_guardsize;
        } else {
                /* initialize the private stack */
                if ((ulwp = ulwp_alloc()) == NULL)
                        return (ENOMEM);
                ulwp->ul_stk = stk;
                ulwp->ul_stktop = (uintptr_t)stk + stksize;
                ulwp->ul_stksiz = stksize;
        }
        /* ulwp is not in the hash table; make sure hash_out() doesn't fail */
        ulwp->ul_ix = -1;
        ulwp->ul_errnop = &ulwp->ul_errno;

        lwp_flags = LWP_SUSPENDED;
        if (flags & (THR_DETACHED|THR_DAEMON)) {
                flags |= THR_DETACHED;
                lwp_flags |= LWP_DETACHED;
        }
        if (flags & THR_DAEMON)
                lwp_flags |= LWP_DAEMON;

        /* creating a thread: enforce mt-correctness in mutex_lock() */
        self->ul_async_safe = 1;

        /* per-thread copies of global variables, for speed */
        ulwp->ul_queue_fifo = self->ul_queue_fifo;
        ulwp->ul_cond_wait_defer = self->ul_cond_wait_defer;
        ulwp->ul_error_detection = self->ul_error_detection;
        ulwp->ul_async_safe = self->ul_async_safe;
        ulwp->ul_max_spinners = self->ul_max_spinners;
        ulwp->ul_adaptive_spin = self->ul_adaptive_spin;
        ulwp->ul_queue_spin = self->ul_queue_spin;
        ulwp->ul_door_noreserve = self->ul_door_noreserve;
        ulwp->ul_misaligned = self->ul_misaligned;

        /* new thread inherits creating thread's scheduling parameters */
        ulwp->ul_policy = self->ul_policy;
        ulwp->ul_pri = (self->ul_epri? self->ul_epri : self->ul_pri);
        ulwp->ul_cid = self->ul_cid;
        ulwp->ul_rtclassid = self->ul_rtclassid;

        ulwp->ul_primarymap = self->ul_primarymap;
        ulwp->ul_self = ulwp;
        ulwp->ul_uberdata = udp;

        /* debugger support */
        ulwp->ul_usropts = flags;

#ifdef __sparc
        /*
         * We cache several instructions in the thread structure for use
         * by the fasttrap DTrace provider. When changing this, read the
         * comment in fasttrap.h for the all the other places that must
         * be changed.
         */
        ulwp->ul_dsave = 0x9de04000;    /* save %g1, %g0, %sp */
        ulwp->ul_drestore = 0x81e80000; /* restore %g0, %g0, %g0 */
        ulwp->ul_dftret = 0x91d0203a;   /* ta 0x3a */
        ulwp->ul_dreturn = 0x81ca0000;  /* return %o0 */
#endif

        ulwp->ul_startpc = func;
        ulwp->ul_startarg = arg;
        _fpinherit(ulwp);
        /*
         * Defer signals on the new thread until its TLS constructors
         * have been called.  _thrp_setup() will call sigon() after
         * it has called tls_setup().
         */
        ulwp->ul_sigdefer = 1;

        error = setup_context(&uc, _thrp_setup, ulwp,
            (caddr_t)ulwp->ul_stk + ulwp->ul_guardsize, stksize);
        if (error != 0 && stk != NULL)  /* inaccessible stack */
                error = EFAULT;

        /*
         * Call enter_critical() to avoid being suspended until we
         * have linked the new thread into the proper lists.
         * This is necessary because forkall() and fork1() must
         * suspend all threads and they must see a complete list.
         */
        enter_critical(self);
        uc.uc_sigmask = ulwp->ul_sigmask = self->ul_sigmask;
        if (error != 0 ||
            (error = __lwp_create(&uc, lwp_flags, &tid)) != 0) {
                exit_critical(self);
                ulwp->ul_lwpid = (lwpid_t)(-1);
                ulwp->ul_dead = 1;
                ulwp->ul_detached = 1;
                lmutex_lock(&udp->link_lock);
                ulwp_free(ulwp);
                lmutex_unlock(&udp->link_lock);
                return (error);
        }
        self->ul_nocancel = 0;  /* cancellation is now possible */
        udp->uberflags.uf_mt = 1;
        if (new_thread)
                *new_thread = tid;
        if (flags & THR_DETACHED)
                ulwp->ul_detached = 1;
        ulwp->ul_lwpid = tid;
        ulwp->ul_stop = TSTP_REGULAR;
        if (flags & THR_SUSPENDED)
                ulwp->ul_created = 1;

        lmutex_lock(&udp->link_lock);
        ulwp->ul_forw = udp->all_lwps;
        ulwp->ul_back = udp->all_lwps->ul_back;
        ulwp->ul_back->ul_forw = ulwp;
        ulwp->ul_forw->ul_back = ulwp;
        hash_in(ulwp, udp);
        udp->nthreads++;
        if (flags & THR_DAEMON)
                udp->ndaemons++;
        if (flags & THR_NEW_LWP)
                thr_concurrency++;
        __libc_threaded = 1;            /* inform stdio */
        lmutex_unlock(&udp->link_lock);

        if (__td_event_report(self, TD_CREATE, udp)) {
                self->ul_td_evbuf.eventnum = TD_CREATE;
                self->ul_td_evbuf.eventdata = (void *)(uintptr_t)tid;
                tdb_event(TD_CREATE, udp);
        }

        exit_critical(self);

        if (name != NULL)
                (void) pthread_setname_np(tid, name);

        if (!(flags & THR_SUSPENDED))
                (void) _thrp_continue(tid, TSTP_REGULAR);

        return (0);
}

int
thr_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
    long flags, thread_t *new_thread)
{
        return (_thrp_create(stk, stksize, func, arg, flags, new_thread, 0,
            NULL));
}

/*
 * A special cancellation cleanup hook for DCE.
 * cleanuphndlr, when it is not NULL, will contain a callback
 * function to be called before a thread is terminated in
 * thr_exit() as a result of being cancelled.
 */
static void (*cleanuphndlr)(void) = NULL;

/*
 * _pthread_setcleanupinit: sets the cleanup hook.
 */
int
_pthread_setcleanupinit(void (*func)(void))
{
        cleanuphndlr = func;
        return (0);
}

void
_thrp_exit()
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        ulwp_t *replace = NULL;

        if (__td_event_report(self, TD_DEATH, udp)) {
                self->ul_td_evbuf.eventnum = TD_DEATH;
                tdb_event(TD_DEATH, udp);
        }

        ASSERT(self->ul_sigdefer != 0);

        lmutex_lock(&udp->link_lock);
        udp->nthreads--;
        if (self->ul_usropts & THR_NEW_LWP)
                thr_concurrency--;
        if (self->ul_usropts & THR_DAEMON)
                udp->ndaemons--;
        else if (udp->nthreads == udp->ndaemons) {
                /*
                 * We are the last non-daemon thread exiting.
                 * Exit the process.  We retain our TSD and TLS so
                 * that atexit() application functions can use them.
                 */
                lmutex_unlock(&udp->link_lock);
                exit(0);
                thr_panic("_thrp_exit(): exit(0) returned");
        }
        lmutex_unlock(&udp->link_lock);

        /*
         * tsd_exit() may call its destructor free(), thus depending on
         * tmem, therefore tmem_exit() needs to be called after tsd_exit()
         * and tls_exit().
         */
        tsd_exit();             /* deallocate thread-specific data */
        tls_exit();             /* deallocate thread-local storage */
        tmem_exit();            /* deallocate tmem allocations */
        heldlock_exit();        /* deal with left-over held locks */

        /* block all signals to finish exiting */
        block_all_signals(self);
        /* also prevent ourself from being suspended */
        enter_critical(self);
        rwl_free(self);
        lmutex_lock(&udp->link_lock);
        ulwp_free(self);
        (void) ulwp_lock(self, udp);

        if (self->ul_mapsiz && !self->ul_detached) {
                /*
                 * We want to free the stack for reuse but must keep
                 * the ulwp_t struct for the benefit of thr_join().
                 * For this purpose we allocate a replacement ulwp_t.
                 */
                if ((replace = udp->ulwp_replace_free) == NULL)
                        replace = lmalloc(REPLACEMENT_SIZE);
                else if ((udp->ulwp_replace_free = replace->ul_next) == NULL)
                        udp->ulwp_replace_last = NULL;
        }

        if (udp->all_lwps == self)
                udp->all_lwps = self->ul_forw;
        if (udp->all_lwps == self)
                udp->all_lwps = NULL;
        else {
                self->ul_forw->ul_back = self->ul_back;
                self->ul_back->ul_forw = self->ul_forw;
        }
        self->ul_forw = self->ul_back = NULL;
#if defined(THREAD_DEBUG)
        /* collect queue lock statistics before marking ourself dead */
        record_spin_locks(self);
#endif
        self->ul_dead = 1;
        self->ul_pleasestop = 0;
        if (replace != NULL) {
                int ix = self->ul_ix;           /* the hash index */
                (void) memcpy(replace, self, REPLACEMENT_SIZE);
                replace->ul_self = replace;
                replace->ul_next = NULL;        /* clone not on stack list */
                replace->ul_mapsiz = 0;         /* allows clone to be freed */
                replace->ul_replace = 1;        /* requires clone to be freed */
                hash_out_unlocked(self, ix, udp);
                hash_in_unlocked(replace, ix, udp);
                ASSERT(!(self->ul_detached));
                self->ul_detached = 1;          /* this frees the stack */
                self->ul_schedctl = NULL;
                self->ul_schedctl_called = &udp->uberflags;
                set_curthread(self = replace);
                /*
                 * Having just changed the address of curthread, we
                 * must reset the ownership of the locks we hold so
                 * that assertions will not fire when we release them.
                 */
                udp->link_lock.mutex_owner = (uintptr_t)self;
                ulwp_mutex(self, udp)->mutex_owner = (uintptr_t)self;
                /*
                 * NOTE:
                 * On i386, %gs still references the original, not the
                 * replacement, ulwp structure.  Fetching the replacement
                 * curthread pointer via %gs:0 works correctly since the
                 * original ulwp structure will not be reallocated until
                 * this lwp has completed its lwp_exit() system call (see
                 * dead_and_buried()), but from here on out, we must make
                 * no references to %gs:<offset> other than %gs:0.
                 */
        }
        /*
         * Put non-detached terminated threads in the all_zombies list.
         */
        if (!self->ul_detached) {
                udp->nzombies++;
                if (udp->all_zombies == NULL) {
                        ASSERT(udp->nzombies == 1);
                        udp->all_zombies = self->ul_forw = self->ul_back = self;
                } else {
                        self->ul_forw = udp->all_zombies;
                        self->ul_back = udp->all_zombies->ul_back;
                        self->ul_back->ul_forw = self;
                        self->ul_forw->ul_back = self;
                }
        }
        /*
         * Notify everyone waiting for this thread.
         */
        ulwp_broadcast(self);
        (void) ulwp_unlock(self, udp);
        /*
         * Prevent any more references to the schedctl data.
         * We are exiting and continue_fork() may not find us.
         * Do this just before dropping link_lock, since fork
         * serializes on link_lock.
         */
        self->ul_schedctl = NULL;
        self->ul_schedctl_called = &udp->uberflags;
        lmutex_unlock(&udp->link_lock);

        ASSERT(self->ul_critical == 1);
        ASSERT(self->ul_preempt == 0);
        _lwp_terminate();       /* never returns */
        thr_panic("_thrp_exit(): _lwp_terminate() returned");
}

#if defined(THREAD_DEBUG)
void
collect_queue_statistics()
{
        uberdata_t *udp = curthread->ul_uberdata;
        ulwp_t *ulwp;

        if (thread_queue_dump) {
                lmutex_lock(&udp->link_lock);
                if ((ulwp = udp->all_lwps) != NULL) {
                        do {
                                record_spin_locks(ulwp);
                        } while ((ulwp = ulwp->ul_forw) != udp->all_lwps);
                }
                lmutex_unlock(&udp->link_lock);
        }
}
#endif

static void __NORETURN
_thrp_exit_common(void *status, int unwind)
{
        ulwp_t *self = curthread;
        int cancelled = (self->ul_cancel_pending && status == PTHREAD_CANCELED);

        ASSERT(self->ul_critical == 0 && self->ul_preempt == 0);

        /*
         * Disable cancellation and call the special DCE cancellation
         * cleanup hook if it is enabled.  Do nothing else before calling
         * the DCE cancellation cleanup hook; it may call longjmp() and
         * never return here.
         */
        self->ul_cancel_disabled = 1;
        self->ul_cancel_async = 0;
        self->ul_save_async = 0;
        self->ul_cancelable = 0;
        self->ul_cancel_pending = 0;
        set_cancel_pending_flag(self, 1);
        if (cancelled && cleanuphndlr != NULL)
                (*cleanuphndlr)();

        /*
         * Block application signals while we are exiting.
         * We call out to C++, TSD, and TLS destructors while exiting
         * and these are application-defined, so we cannot be assured
         * that they won't reset the signal mask.  We use sigoff() to
         * defer any signals that may be received as a result of this
         * bad behavior.  Such signals will be lost to the process
         * when the thread finishes exiting.
         */
        (void) thr_sigsetmask(SIG_SETMASK, &maskset, NULL);
        sigoff(self);

        self->ul_rval = status;

        /*
         * If thr_exit is being called from the places where
         * C++ destructors are to be called such as cancellation
         * points, then set this flag. It is checked in _t_cancel()
         * to decide whether _ex_unwind() is to be called or not.
         */
        if (unwind)
                self->ul_unwind = 1;

        /*
         * _thrp_unwind() will eventually call _thrp_exit().
         * It never returns.
         */
        _thrp_unwind(NULL);
        thr_panic("_thrp_exit_common(): _thrp_unwind() returned");

        for (;;)        /* to shut the compiler up about __NORETURN */
                continue;
}

/*
 * Called when a thread returns from its start function.
 * We are at the top of the stack; no unwinding is necessary.
 */
void
_thrp_terminate(void *status)
{
        _thrp_exit_common(status, 0);
}

#pragma weak pthread_exit = thr_exit
#pragma weak _thr_exit = thr_exit
void
thr_exit(void *status)
{
        _thrp_exit_common(status, 1);
}

int
_thrp_join(thread_t tid, thread_t *departed, void **status, int do_cancel)
{
        uberdata_t *udp = curthread->ul_uberdata;
        mutex_t *mp;
        void *rval;
        thread_t found;
        ulwp_t *ulwp;
        ulwp_t **ulwpp;
        int replace;
        int error;

        if (do_cancel)
                error = lwp_wait(tid, &found);
        else {
                while ((error = __lwp_wait(tid, &found)) == EINTR)
                        ;
        }
        if (error)
                return (error);

        /*
         * We must hold link_lock to avoid a race condition with find_stack().
         */
        lmutex_lock(&udp->link_lock);
        if ((ulwpp = find_lwpp(found)) == NULL) {
                /*
                 * lwp_wait() found an lwp that the library doesn't know
                 * about.  It must have been created with _lwp_create().
                 * Just return its lwpid; we can't know its status.
                 */
                lmutex_unlock(&udp->link_lock);
                rval = NULL;
        } else {
                /*
                 * Remove ulwp from the hash table.
                 */
                ulwp = *ulwpp;
                *ulwpp = ulwp->ul_hash;
                ulwp->ul_hash = NULL;
                /*
                 * Remove ulwp from all_zombies list.
                 */
                ASSERT(udp->nzombies >= 1);
                if (udp->all_zombies == ulwp)
                        udp->all_zombies = ulwp->ul_forw;
                if (udp->all_zombies == ulwp)
                        udp->all_zombies = NULL;
                else {
                        ulwp->ul_forw->ul_back = ulwp->ul_back;
                        ulwp->ul_back->ul_forw = ulwp->ul_forw;
                }
                ulwp->ul_forw = ulwp->ul_back = NULL;
                udp->nzombies--;
                ASSERT(ulwp->ul_dead && !ulwp->ul_detached &&
                    !(ulwp->ul_usropts & (THR_DETACHED|THR_DAEMON)));
                /*
                 * We can't call ulwp_unlock(ulwp) after we set
                 * ulwp->ul_ix = -1 so we have to get a pointer to the
                 * ulwp's hash table mutex now in order to unlock it below.
                 */
                mp = ulwp_mutex(ulwp, udp);
                ulwp->ul_lwpid = (lwpid_t)(-1);
                ulwp->ul_ix = -1;
                rval = ulwp->ul_rval;
                replace = ulwp->ul_replace;
                lmutex_unlock(mp);
                if (replace) {
                        ulwp->ul_next = NULL;
                        if (udp->ulwp_replace_free == NULL)
                                udp->ulwp_replace_free =
                                    udp->ulwp_replace_last = ulwp;
                        else {
                                udp->ulwp_replace_last->ul_next = ulwp;
                                udp->ulwp_replace_last = ulwp;
                        }
                }
                lmutex_unlock(&udp->link_lock);
        }

        if (departed != NULL)
                *departed = found;
        if (status != NULL)
                *status = rval;
        return (0);
}

int
thr_join(thread_t tid, thread_t *departed, void **status)
{
        int error = _thrp_join(tid, departed, status, 1);
        return ((error == EINVAL)? ESRCH : error);
}

/*
 * pthread_join() differs from Solaris thr_join():
 * It does not return the departed thread's id
 * and hence does not have a "departed" argument.
 * It returns EINVAL if tid refers to a detached thread.
 */
#pragma weak _pthread_join = pthread_join
int
pthread_join(pthread_t tid, void **status)
{
        return ((tid == 0)? ESRCH : _thrp_join(tid, NULL, status, 1));
}

int
pthread_detach(pthread_t tid)
{
        uberdata_t *udp = curthread->ul_uberdata;
        ulwp_t *ulwp;
        ulwp_t **ulwpp;
        int error = 0;

        if ((ulwpp = find_lwpp(tid)) == NULL)
                return (ESRCH);
        ulwp = *ulwpp;

        if (ulwp->ul_dead) {
                ulwp_unlock(ulwp, udp);
                error = _thrp_join(tid, NULL, NULL, 0);
        } else {
                error = __lwp_detach(tid);
                ulwp->ul_detached = 1;
                ulwp->ul_usropts |= THR_DETACHED;
                ulwp_unlock(ulwp, udp);
        }
        return (error);
}

static const char *
ematch(const char *ev, const char *match)
{
        int c;

        while ((c = *match++) != '\0') {
                if (*ev++ != c)
                        return (NULL);
        }
        if (*ev++ != '=')
                return (NULL);
        return (ev);
}

static int
envvar(const char *ev, const char *match, int limit)
{
        int val = -1;
        const char *ename;

        if ((ename = ematch(ev, match)) != NULL) {
                int c;
                for (val = 0; (c = *ename) != '\0'; ename++) {
                        if (!isdigit(c)) {
                                val = -1;
                                break;
                        }
                        val = val * 10 + (c - '0');
                        if (val > limit) {
                                val = limit;
                                break;
                        }
                }
        }
        return (val);
}

static void
etest(const char *ev)
{
        int value;

        if ((value = envvar(ev, "QUEUE_SPIN", 1000000)) >= 0)
                thread_queue_spin = value;
        if ((value = envvar(ev, "ADAPTIVE_SPIN", 1000000)) >= 0)
                thread_adaptive_spin = value;
        if ((value = envvar(ev, "MAX_SPINNERS", 255)) >= 0)
                thread_max_spinners = value;
        if ((value = envvar(ev, "QUEUE_FIFO", 8)) >= 0)
                thread_queue_fifo = value;
#if defined(THREAD_DEBUG)
        if ((value = envvar(ev, "QUEUE_VERIFY", 1)) >= 0)
                thread_queue_verify = value;
        if ((value = envvar(ev, "QUEUE_DUMP", 1)) >= 0)
                thread_queue_dump = value;
#endif
        if ((value = envvar(ev, "STACK_CACHE", 10000)) >= 0)
                thread_stack_cache = value;
        if ((value = envvar(ev, "COND_WAIT_DEFER", 1)) >= 0)
                thread_cond_wait_defer = value;
        if ((value = envvar(ev, "ERROR_DETECTION", 2)) >= 0)
                thread_error_detection = value;
        if ((value = envvar(ev, "ASYNC_SAFE", 1)) >= 0)
                thread_async_safe = value;
        if ((value = envvar(ev, "DOOR_NORESERVE", 1)) >= 0)
                thread_door_noreserve = value;
        if ((value = envvar(ev, "LOCKS_MISALIGNED", 1)) >= 0)
                thread_locks_misaligned = value;
}

/*
 * Look for and evaluate environment variables of the form "_THREAD_*".
 * For compatibility with the past, we also look for environment
 * names of the form "LIBTHREAD_*".
 */
static void
set_thread_vars()
{
        extern const char **_environ;
        const char **pev;
        const char *ev;
        char c;

        if ((pev = _environ) == NULL)
                return;
        while ((ev = *pev++) != NULL) {
                c = *ev;
                if (c == '_' && strncmp(ev, "_THREAD_", 8) == 0)
                        etest(ev + 8);
                if (c == 'L' && strncmp(ev, "LIBTHREAD_", 10) == 0)
                        etest(ev + 10);
        }
}

/* PROBE_SUPPORT begin */
#pragma weak __tnf_probe_notify
extern void __tnf_probe_notify(void);
/* PROBE_SUPPORT end */

/* same as atexit() but private to the library */
extern int _atexit(void (*)(void));

/* same as _cleanup() but private to the library */
extern void __cleanup(void);

extern void atfork_init(void);

#ifdef __amd64
extern void __proc64id(void);
#endif

static void
init_auxv_data(uberdata_t *udp)
{
        Dl_argsinfo_t args;

        udp->ub_comm_page = NULL;
        if (dlinfo(RTLD_SELF, RTLD_DI_ARGSINFO, &args) < 0)
                return;

        while (args.dla_auxv->a_type != AT_NULL) {
                if (args.dla_auxv->a_type == AT_SUN_COMMPAGE) {
                        udp->ub_comm_page = args.dla_auxv->a_un.a_ptr;
                }
                args.dla_auxv++;
        }
}

/*
 * libc_init() is called by ld.so.1 for library initialization.
 * We perform minimal initialization; enough to work with the main thread.
 */
void
libc_init(void)
{
        uberdata_t *udp = &__uberdata;
        ulwp_t *oldself = __curthread();
        ucontext_t uc;
        ulwp_t *self;
        struct rlimit rl;
        caddr_t data;
        size_t tls_size;
        int setmask;

        /*
         * For the initial stage of initialization, we must be careful
         * not to call any function that could possibly call _cerror().
         * For this purpose, we call only the raw system call wrappers.
         */

#ifdef __amd64
        /*
         * Gather information about cache layouts for optimized
         * AMD and Intel assembler strfoo() and memfoo() functions.
         */
        __proc64id();
#endif

        /*
         * Every libc, regardless of which link map, must register __cleanup().
         */
        (void) _atexit(__cleanup);

        /*
         * Every libc, regardless of link map, needs to go through and check
         * its aux vectors.  Doing so will indicate whether or not this has
         * been given a comm page (to optimize certain system actions).
         */
        init_auxv_data(udp);

        /*
         * We keep our uberdata on one of (a) the first alternate link map
         * or (b) the primary link map.  We switch to the primary link map
         * and stay there once we see it.  All intermediate link maps are
         * subject to being unloaded at any time.
         */
        if (oldself != NULL && (oldself->ul_primarymap || !primary_link_map)) {
                __tdb_bootstrap = oldself->ul_uberdata->tdb_bootstrap;
                mutex_setup();
                atfork_init();  /* every link map needs atfork() processing */
                init_progname();
                return;
        }

        /*
         * To establish the main stack information, we have to get our context.
         * This is also convenient to use for getting our signal mask.
         */
        uc.uc_flags = UC_ALL;
        (void) __getcontext(&uc);
        ASSERT(uc.uc_link == NULL);

        tls_size = roundup64(udp->tls_metadata.static_tls.tls_size);
        ASSERT(primary_link_map || tls_size == 0);
        data = lmalloc(sizeof (ulwp_t) + tls_size);
        if (data == NULL)
                thr_panic("cannot allocate thread structure for main thread");
        /* LINTED pointer cast may result in improper alignment */
        self = (ulwp_t *)(data + tls_size);
        init_hash_table[0].hash_bucket = self;

        self->ul_sigmask = uc.uc_sigmask;
        delete_reserved_signals(&self->ul_sigmask);
        /*
         * Are the old and new sets different?
         * (This can happen if we are currently blocking SIGCANCEL.)
         * If so, we must explicitly set our signal mask, below.
         */
        setmask =
            ((self->ul_sigmask.__sigbits[0] ^ uc.uc_sigmask.__sigbits[0]) |
            (self->ul_sigmask.__sigbits[1] ^ uc.uc_sigmask.__sigbits[1]) |
            (self->ul_sigmask.__sigbits[2] ^ uc.uc_sigmask.__sigbits[2]) |
            (self->ul_sigmask.__sigbits[3] ^ uc.uc_sigmask.__sigbits[3]));

#ifdef __sparc
        /*
         * We cache several instructions in the thread structure for use
         * by the fasttrap DTrace provider. When changing this, read the
         * comment in fasttrap.h for the all the other places that must
         * be changed.
         */
        self->ul_dsave = 0x9de04000;    /* save %g1, %g0, %sp */
        self->ul_drestore = 0x81e80000; /* restore %g0, %g0, %g0 */
        self->ul_dftret = 0x91d0203a;   /* ta 0x3a */
        self->ul_dreturn = 0x81ca0000;  /* return %o0 */
#endif

        self->ul_stktop = (uintptr_t)uc.uc_stack.ss_sp + uc.uc_stack.ss_size;
        (void) getrlimit(RLIMIT_STACK, &rl);
        self->ul_stksiz = rl.rlim_cur;
        self->ul_stk = (caddr_t)(self->ul_stktop - self->ul_stksiz);

        self->ul_forw = self->ul_back = self;
        self->ul_hash = NULL;
        self->ul_ix = 0;
        self->ul_lwpid = 1; /* _lwp_self() */
        self->ul_main = 1;
        self->ul_self = self;
        self->ul_policy = -1;           /* initialize only when needed */
        self->ul_pri = 0;
        self->ul_cid = 0;
        self->ul_rtclassid = -1;
        self->ul_uberdata = udp;
        if (oldself != NULL) {
                int i;

                ASSERT(primary_link_map);
                ASSERT(oldself->ul_main == 1);
                self->ul_stsd = oldself->ul_stsd;
                for (i = 0; i < TSD_NFAST; i++)
                        self->ul_ftsd[i] = oldself->ul_ftsd[i];
                self->ul_tls = oldself->ul_tls;
                /*
                 * Retrieve all pointers to uberdata allocated
                 * while running on previous link maps.
                 * We would like to do a structure assignment here, but
                 * gcc turns structure assignments into calls to memcpy(),
                 * a function exported from libc.  We can't call any such
                 * external functions until we establish curthread, below,
                 * so we just call our private version of memcpy().
                 */
                (void) memcpy(udp, oldself->ul_uberdata, sizeof (*udp));
                /*
                 * These items point to global data on the primary link map.
                 */
                udp->thr_hash_table = init_hash_table;
                udp->sigacthandler = sigacthandler;
                udp->tdb.tdb_events = tdb_events;
                ASSERT(udp->nthreads == 1 && !udp->uberflags.uf_mt);
                ASSERT(udp->lwp_stacks == NULL);
                ASSERT(udp->ulwp_freelist == NULL);
                ASSERT(udp->ulwp_replace_free == NULL);
                ASSERT(udp->hash_size == 1);
        }
        udp->all_lwps = self;
        udp->ulwp_one = self;
        udp->pid = getpid();
        udp->nthreads = 1;
        /*
         * In every link map, tdb_bootstrap points to the same piece of
         * allocated memory.  When the primary link map is initialized,
         * the allocated memory is assigned a pointer to the one true
         * uberdata.  This allows libc_db to initialize itself regardless
         * of which instance of libc it finds in the address space.
         */
        if (udp->tdb_bootstrap == NULL)
                udp->tdb_bootstrap = lmalloc(sizeof (uberdata_t *));
        __tdb_bootstrap = udp->tdb_bootstrap;
        if (primary_link_map) {
                self->ul_primarymap = 1;
                udp->primary_map = 1;
                *udp->tdb_bootstrap = udp;
        }
        /*
         * Cancellation can't happen until:
         *      pthread_cancel() is called
         * or:
         *      another thread is created
         * For now, as a single-threaded process, set the flag that tells
         * PROLOGUE/EPILOGUE (in scalls.c) that cancellation can't happen.
         */
        self->ul_nocancel = 1;

#if defined(__amd64)
        (void) ___lwp_private(_LWP_SETPRIVATE, _LWP_FSBASE, self);
#elif defined(__i386)
        (void) ___lwp_private(_LWP_SETPRIVATE, _LWP_GSBASE, self);
#endif  /* __i386 || __amd64 */
        set_curthread(self);            /* redundant on i386 */
        /*
         * Now curthread is established and it is safe to call any
         * function in libc except one that uses thread-local storage.
         */
        self->ul_errnop = &errno;
        if (oldself != NULL) {
                /* tls_size was zero when oldself was allocated */
                lfree(oldself, sizeof (ulwp_t));
        }
        mutex_setup();
        atfork_init();
        signal_init();

        /*
         * If the stack is unlimited, we set the size to zero to disable
         * stack checking.
         * XXX: Work harder here.  Get the stack size from /proc/self/rmap
         */
        if (self->ul_stksiz == RLIM_INFINITY) {
                self->ul_ustack.ss_sp = (void *)self->ul_stktop;
                self->ul_ustack.ss_size = 0;
        } else {
                self->ul_ustack.ss_sp = self->ul_stk;
                self->ul_ustack.ss_size = self->ul_stksiz;
        }
        self->ul_ustack.ss_flags = 0;
        (void) setustack(&self->ul_ustack);

        /*
         * Get the variables that affect thread behavior from the environment.
         */
        set_thread_vars();
        udp->uberflags.uf_thread_error_detection = (char)thread_error_detection;
        udp->thread_stack_cache = thread_stack_cache;

        /*
         * Make per-thread copies of global variables, for speed.
         */
        self->ul_queue_fifo = (char)thread_queue_fifo;
        self->ul_cond_wait_defer = (char)thread_cond_wait_defer;
        self->ul_error_detection = (char)thread_error_detection;
        self->ul_async_safe = (char)thread_async_safe;
        self->ul_door_noreserve = (char)thread_door_noreserve;
        self->ul_misaligned = (char)thread_locks_misaligned;
        self->ul_max_spinners = (uint8_t)thread_max_spinners;
        self->ul_adaptive_spin = thread_adaptive_spin;
        self->ul_queue_spin = thread_queue_spin;

#if defined(__sparc) && !defined(_LP64)
        if (self->ul_misaligned) {
                /*
                 * Tell the kernel to fix up ldx/stx instructions that
                 * refer to non-8-byte aligned data instead of giving
                 * the process an alignment trap and generating SIGBUS.
                 *
                 * Programs compiled for 32-bit sparc with the Studio SS12
                 * compiler get this done for them automatically (in _init()).
                 * We do it here for the benefit of programs compiled with
                 * other compilers, like gcc.
                 *
                 * This is necessary for the _THREAD_LOCKS_MISALIGNED=1
                 * environment variable horrible hack to work.
                 */
                extern void _do_fix_align(void);
                _do_fix_align();
        }
#endif

        /*
         * When we have initialized the primary link map, inform
         * the dynamic linker about our interface functions.
         * Set up our pointer to the program name.
         */
        if (self->ul_primarymap)
                _ld_libc((void *)rtld_funcs);
        init_progname();

        /*
         * Defer signals until TLS constructors have been called.
         */
        sigoff(self);
        tls_setup();
        sigon(self);
        if (setmask)
                (void) restore_signals(self);

        /*
         * Make private copies of __xpg4 and __xpg6 so libc can test
         * them after this point without invoking the dynamic linker.
         */
        libc__xpg4 = __xpg4;
        libc__xpg6 = __xpg6;

        /* PROBE_SUPPORT begin */
        if (self->ul_primarymap && __tnf_probe_notify != NULL)
                __tnf_probe_notify();
        /* PROBE_SUPPORT end */

        init_sigev_thread();
        init_aio();

        /*
         * We need to reset __threaded dynamically at runtime, so that
         * __threaded can be bound to __threaded outside libc which may not
         * have initial value of 1 (without a copy relocation in a.out).
         */
        __threaded = 1;
}

#pragma fini(libc_fini)
void
libc_fini()
{
        /*
         * If we are doing fini processing for the instance of libc
         * on the first alternate link map (this happens only when
         * the dynamic linker rejects a bad audit library), then clear
         * __curthread().  We abandon whatever memory was allocated by
         * lmalloc() while running on this alternate link-map but we
         * don't care (and can't find the memory in any case); we just
         * want to protect the application from this bad audit library.
         * No fini processing is done by libc in the normal case.
         */

        uberdata_t *udp = curthread->ul_uberdata;

        if (udp->primary_map == 0 && udp == &__uberdata)
                set_curthread(NULL);
}

/*
 * finish_init is called when we are about to become multi-threaded,
 * that is, on the first call to thr_create().
 */
void
finish_init()
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        thr_hash_table_t *htp;
        void *data;
        int i;

        /*
         * No locks needed here; we are single-threaded on the first call.
         * We can be called only after the primary link map has been set up.
         */
        ASSERT(self->ul_primarymap);
        ASSERT(self == udp->ulwp_one);
        ASSERT(!udp->uberflags.uf_mt);
        ASSERT(udp->hash_size == 1);

        /*
         * Initialize self->ul_policy, self->ul_cid, and self->ul_pri.
         */
        update_sched(self);

        /*
         * Allocate the queue_head array if not already allocated.
         */
        if (udp->queue_head == NULL)
                queue_alloc();

        /*
         * Now allocate the thread hash table.
         */
        if ((data = mmap(NULL, HASHTBLSZ * sizeof (thr_hash_table_t),
            PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0))
            == MAP_FAILED)
                thr_panic("cannot allocate thread hash table");

        udp->thr_hash_table = htp = (thr_hash_table_t *)data;
        udp->hash_size = HASHTBLSZ;
        udp->hash_mask = HASHTBLSZ - 1;

        for (i = 0; i < HASHTBLSZ; i++, htp++) {
                htp->hash_lock.mutex_flag = LOCK_INITED;
                htp->hash_lock.mutex_magic = MUTEX_MAGIC;
                htp->hash_cond.cond_magic = COND_MAGIC;
        }
        hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp);

        /*
         * Set up the SIGCANCEL handler for threads cancellation.
         */
        setup_cancelsig(SIGCANCEL);

        /*
         * Arrange to do special things on exit --
         * - collect queue statistics from all remaining active threads.
         * - dump queue statistics to stderr if _THREAD_QUEUE_DUMP is set.
         * - grab assert_lock to ensure that assertion failures
         *   and a core dump take precedence over _exit().
         * (Functions are called in the reverse order of their registration.)
         */
        (void) _atexit(grab_assert_lock);
#if defined(THREAD_DEBUG)
        (void) _atexit(dump_queue_statistics);
        (void) _atexit(collect_queue_statistics);
#endif
}

/*
 * Used only by postfork1_child(), below.
 */
static void
mark_dead_and_buried(ulwp_t *ulwp)
{
        ulwp->ul_dead = 1;
        ulwp->ul_lwpid = (lwpid_t)(-1);
        ulwp->ul_hash = NULL;
        ulwp->ul_ix = -1;
        ulwp->ul_schedctl = NULL;
        ulwp->ul_schedctl_called = NULL;
}

/*
 * This is called from fork1() in the child.
 * Reset our data structures to reflect one lwp.
 */
void
postfork1_child()
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        queue_head_t *qp;
        ulwp_t *next;
        ulwp_t *ulwp;
        int i;

        /* daemon threads shouldn't call fork1(), but oh well... */
        self->ul_usropts &= ~THR_DAEMON;
        udp->nthreads = 1;
        udp->ndaemons = 0;
        udp->uberflags.uf_mt = 0;
        __libc_threaded = 0;
        for (i = 0; i < udp->hash_size; i++)
                udp->thr_hash_table[i].hash_bucket = NULL;
        self->ul_lwpid = _lwp_self();
        hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp);

        /*
         * Some thread in the parent might have been suspended
         * while holding udp->callout_lock or udp->ld_lock.
         * Reinitialize the child's copies.
         */
        (void) mutex_init(&udp->callout_lock,
            USYNC_THREAD | LOCK_RECURSIVE, NULL);
        (void) mutex_init(&udp->ld_lock,
            USYNC_THREAD | LOCK_RECURSIVE, NULL);

        /* no one in the child is on a sleep queue; reinitialize */
        if ((qp = udp->queue_head) != NULL) {
                (void) memset(qp, 0, 2 * QHASHSIZE * sizeof (queue_head_t));
                for (i = 0; i < 2 * QHASHSIZE; qp++, i++) {
                        qp->qh_type = (i < QHASHSIZE)? MX : CV;
                        qp->qh_lock.mutex_flag = LOCK_INITED;
                        qp->qh_lock.mutex_magic = MUTEX_MAGIC;
                        qp->qh_hlist = &qp->qh_def_root;
#if defined(THREAD_DEBUG)
                        qp->qh_hlen = 1;
                        qp->qh_hmax = 1;
#endif
                }
        }

        /*
         * Do post-fork1 processing for subsystems that need it.
         * We need to do this before unmapping all of the abandoned
         * threads' stacks, below(), because the post-fork1 actions
         * might require access to those stacks.
         */
        postfork1_child_sigev_aio();
        postfork1_child_sigev_mq();
        postfork1_child_sigev_timer();
        postfork1_child_aio();
        /*
         * The above subsystems use thread pools, so this action
         * must be performed after those actions.
         */
        postfork1_child_tpool();

        /*
         * All lwps except ourself are gone.  Mark them so.
         * First mark all of the lwps that have already been freed.
         * Then mark and free all of the active lwps except ourself.
         * Since we are single-threaded, no locks are required here.
         */
        for (ulwp = udp->lwp_stacks; ulwp != NULL; ulwp = ulwp->ul_next)
                mark_dead_and_buried(ulwp);
        for (ulwp = udp->ulwp_freelist; ulwp != NULL; ulwp = ulwp->ul_next)
                mark_dead_and_buried(ulwp);
        for (ulwp = self->ul_forw; ulwp != self; ulwp = next) {
                next = ulwp->ul_forw;
                ulwp->ul_forw = ulwp->ul_back = NULL;
                mark_dead_and_buried(ulwp);
                tsd_free(ulwp);
                tls_free(ulwp);
                rwl_free(ulwp);
                heldlock_free(ulwp);
                ulwp_free(ulwp);
        }
        self->ul_forw = self->ul_back = udp->all_lwps = self;
        if (self != udp->ulwp_one)
                mark_dead_and_buried(udp->ulwp_one);
        if ((ulwp = udp->all_zombies) != NULL) {
                ASSERT(udp->nzombies != 0);
                do {
                        next = ulwp->ul_forw;
                        ulwp->ul_forw = ulwp->ul_back = NULL;
                        mark_dead_and_buried(ulwp);
                        udp->nzombies--;
                        if (ulwp->ul_replace) {
                                ulwp->ul_next = NULL;
                                if (udp->ulwp_replace_free == NULL) {
                                        udp->ulwp_replace_free =
                                            udp->ulwp_replace_last = ulwp;
                                } else {
                                        udp->ulwp_replace_last->ul_next = ulwp;
                                        udp->ulwp_replace_last = ulwp;
                                }
                        }
                } while ((ulwp = next) != udp->all_zombies);
                ASSERT(udp->nzombies == 0);
                udp->all_zombies = NULL;
                udp->nzombies = 0;
        }
        trim_stack_cache(0);
}

lwpid_t
lwp_self(void)
{
        return (curthread->ul_lwpid);
}

#pragma weak _ti_thr_self = thr_self
#pragma weak pthread_self = thr_self
thread_t
thr_self()
{
        return (curthread->ul_lwpid);
}

int
thr_main()
{
        ulwp_t *self = __curthread();

        return ((self == NULL)? -1 : self->ul_main);
}

int
_thrp_cancelled(void)
{
        return (curthread->ul_rval == PTHREAD_CANCELED);
}

int
_thrp_stksegment(ulwp_t *ulwp, stack_t *stk)
{
        stk->ss_sp = (void *)ulwp->ul_stktop;
        stk->ss_size = ulwp->ul_stksiz;
        stk->ss_flags = 0;
        return (0);
}

#pragma weak _thr_stksegment = thr_stksegment
int
thr_stksegment(stack_t *stk)
{
        return (_thrp_stksegment(curthread, stk));
}

void
force_continue(ulwp_t *ulwp)
{
#if defined(THREAD_DEBUG)
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
#endif
        int error;
        timespec_t ts;

        ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
        ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self));

        for (;;) {
                error = _lwp_continue(ulwp->ul_lwpid);
                if (error != 0 && error != EINTR)
                        break;
                error = 0;
                if (ulwp->ul_stopping) {        /* it is stopping itself */
                        ts.tv_sec = 0;          /* give it a chance to run */
                        ts.tv_nsec = 100000;    /* 100 usecs or clock tick */
                        (void) __nanosleep(&ts, NULL);
                }
                if (!ulwp->ul_stopping)         /* it is running now */
                        break;                  /* so we are done */
                /*
                 * It is marked as being in the process of stopping
                 * itself.  Loop around and continue it again.
                 * It may not have been stopped the first time.
                 */
        }
}

/*
 * Suspend an lwp with lwp_suspend(), then move it to a safe point,
 * that is, to a point where ul_critical and ul_rtld are both zero.
 * On return, the ulwp_lock() is dropped as with ulwp_unlock().
 * If 'link_dropped' is non-NULL, then 'link_lock' is held on entry.
 * If we have to drop link_lock, we store 1 through link_dropped.
 * If the lwp exits before it can be suspended, we return ESRCH.
 */
int
safe_suspend(ulwp_t *ulwp, uchar_t whystopped, int *link_dropped)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        cond_t *cvp = ulwp_condvar(ulwp, udp);
        mutex_t *mp = ulwp_mutex(ulwp, udp);
        thread_t tid = ulwp->ul_lwpid;
        int ix = ulwp->ul_ix;
        int error = 0;

        ASSERT(whystopped == TSTP_REGULAR ||
            whystopped == TSTP_MUTATOR ||
            whystopped == TSTP_FORK);
        ASSERT(ulwp != self);
        ASSERT(!ulwp->ul_stop);
        ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
        ASSERT(MUTEX_OWNED(mp, self));

        if (link_dropped != NULL)
                *link_dropped = 0;

        /*
         * We must grab the target's spin lock before suspending it.
         * See the comments below and in _thrp_suspend() for why.
         */
        spin_lock_set(&ulwp->ul_spinlock);
        (void) ___lwp_suspend(tid);
        spin_lock_clear(&ulwp->ul_spinlock);

top:
        if ((ulwp->ul_critical == 0 && ulwp->ul_rtld == 0) ||
            ulwp->ul_stopping) {
                /* thread is already safe */
                ulwp->ul_stop |= whystopped;
        } else {
                /*
                 * Setting ul_pleasestop causes the target thread to stop
                 * itself in _thrp_suspend(), below, after we drop its lock.
                 * We must continue the critical thread before dropping
                 * link_lock because the critical thread may be holding
                 * the queue lock for link_lock.  This is delicate.
                 */
                ulwp->ul_pleasestop |= whystopped;
                force_continue(ulwp);
                if (link_dropped != NULL) {
                        *link_dropped = 1;
                        lmutex_unlock(&udp->link_lock);
                        /* be sure to drop link_lock only once */
                        link_dropped = NULL;
                }

                /*
                 * The thread may disappear by calling thr_exit() so we
                 * cannot rely on the ulwp pointer after dropping the lock.
                 * Instead, we search the hash table to find it again.
                 * When we return, we may find that the thread has been
                 * continued by some other thread.  The suspend/continue
                 * interfaces are prone to such race conditions by design.
                 */
                while (ulwp && !ulwp->ul_dead && !ulwp->ul_stop &&
                    (ulwp->ul_pleasestop & whystopped)) {
                        (void) __cond_wait(cvp, mp);
                        for (ulwp = udp->thr_hash_table[ix].hash_bucket;
                            ulwp != NULL; ulwp = ulwp->ul_hash) {
                                if (ulwp->ul_lwpid == tid)
                                        break;
                        }
                }

                if (ulwp == NULL || ulwp->ul_dead)
                        error = ESRCH;
                else {
                        /*
                         * Do another lwp_suspend() to make sure we don't
                         * return until the target thread is fully stopped
                         * in the kernel.  Don't apply lwp_suspend() until
                         * we know that the target is not holding any
                         * queue locks, that is, that it has completed
                         * ulwp_unlock(self) and has, or at least is
                         * about to, call lwp_suspend() on itself.  We do
                         * this by grabbing the target's spin lock.
                         */
                        ASSERT(ulwp->ul_lwpid == tid);
                        spin_lock_set(&ulwp->ul_spinlock);
                        (void) ___lwp_suspend(tid);
                        spin_lock_clear(&ulwp->ul_spinlock);
                        /*
                         * If some other thread did a thr_continue()
                         * on the target thread we have to start over.
                         */
                        if (!ulwp->ul_stopping || !(ulwp->ul_stop & whystopped))
                                goto top;
                }
        }

        (void) cond_broadcast(cvp);
        lmutex_unlock(mp);
        return (error);
}

int
_thrp_suspend(thread_t tid, uchar_t whystopped)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        ulwp_t *ulwp;
        int error = 0;

        ASSERT((whystopped & (TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) != 0);
        ASSERT((whystopped & ~(TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) == 0);

        /*
         * We can't suspend anyone except ourself while
         * some other thread is performing a fork.
         * This also allows only one suspension at a time.
         */
        if (tid != self->ul_lwpid)
                fork_lock_enter();

        if ((ulwp = find_lwp(tid)) == NULL)
                error = ESRCH;
        else if (whystopped == TSTP_MUTATOR && !ulwp->ul_mutator) {
                ulwp_unlock(ulwp, udp);
                error = EINVAL;
        } else if (ulwp->ul_stop) {     /* already stopped */
                ulwp->ul_stop |= whystopped;
                ulwp_broadcast(ulwp);
                ulwp_unlock(ulwp, udp);
        } else if (ulwp != self) {
                /*
                 * After suspending the other thread, move it out of a
                 * critical section and deal with the schedctl mappings.
                 * safe_suspend() suspends the other thread, calls
                 * ulwp_broadcast(ulwp) and drops the ulwp lock.
                 */
                error = safe_suspend(ulwp, whystopped, NULL);
        } else {
                int schedctl_after_fork = 0;

                /*
                 * We are suspending ourself.  We must not take a signal
                 * until we return from lwp_suspend() and clear ul_stopping.
                 * This is to guard against siglongjmp().
                 */
                enter_critical(self);
                self->ul_sp = stkptr();
                _flush_windows();       /* sparc */
                self->ul_pleasestop = 0;
                self->ul_stop |= whystopped;
                /*
                 * Grab our spin lock before dropping ulwp_mutex(self).
                 * This prevents the suspending thread from applying
                 * lwp_suspend() to us before we emerge from
                 * lmutex_unlock(mp) and have dropped mp's queue lock.
                 */
                spin_lock_set(&self->ul_spinlock);
                self->ul_stopping = 1;
                ulwp_broadcast(self);
                ulwp_unlock(self, udp);
                /*
                 * From this point until we return from lwp_suspend(),
                 * we must not call any function that might invoke the
                 * dynamic linker, that is, we can only call functions
                 * private to the library.
                 *
                 * Also, this is a nasty race condition for a process
                 * that is undergoing a forkall() operation:
                 * Once we clear our spinlock (below), we are vulnerable
                 * to being suspended by the forkall() thread before
                 * we manage to suspend ourself in ___lwp_suspend().
                 * See safe_suspend() and force_continue().
                 *
                 * To avoid a SIGSEGV due to the disappearance
                 * of the schedctl mappings in the child process,
                 * which can happen in spin_lock_clear() if we
                 * are suspended while we are in the middle of
                 * its call to preempt(), we preemptively clear
                 * our own schedctl pointer before dropping our
                 * spinlock.  We reinstate it, in both the parent
                 * and (if this really is a forkall()) the child.
                 */
                if (whystopped & TSTP_FORK) {
                        schedctl_after_fork = 1;
                        self->ul_schedctl = NULL;
                        self->ul_schedctl_called = &udp->uberflags;
                }
                spin_lock_clear(&self->ul_spinlock);
                (void) ___lwp_suspend(tid);
                /*
                 * Somebody else continued us.
                 * We can't grab ulwp_lock(self)
                 * until after clearing ul_stopping.
                 * force_continue() relies on this.
                 */
                self->ul_stopping = 0;
                self->ul_sp = 0;
                if (schedctl_after_fork) {
                        self->ul_schedctl_called = NULL;
                        self->ul_schedctl = NULL;
                        (void) setup_schedctl();
                }
                ulwp_lock(self, udp);
                ulwp_broadcast(self);
                ulwp_unlock(self, udp);
                exit_critical(self);
        }

        if (tid != self->ul_lwpid)
                fork_lock_exit();

        return (error);
}

/*
 * Suspend all lwps other than ourself in preparation for fork.
 */
void
suspend_fork()
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        ulwp_t *ulwp;
        int link_dropped;

        ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
top:
        lmutex_lock(&udp->link_lock);

        for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
                ulwp_lock(ulwp, udp);
                if (ulwp->ul_stop) {    /* already stopped */
                        ulwp->ul_stop |= TSTP_FORK;
                        ulwp_broadcast(ulwp);
                        ulwp_unlock(ulwp, udp);
                } else {
                        /*
                         * Move the stopped lwp out of a critical section.
                         */
                        if (safe_suspend(ulwp, TSTP_FORK, &link_dropped) ||
                            link_dropped)
                                goto top;
                }
        }

        lmutex_unlock(&udp->link_lock);
}

void
continue_fork(int child)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        ulwp_t *ulwp;

        ASSERT(MUTEX_OWNED(&udp->fork_lock, self));

        /*
         * Clear the schedctl pointers in the child of forkall().
         */
        if (child) {
                for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
                        ulwp->ul_schedctl_called =
                            ulwp->ul_dead? &udp->uberflags : NULL;
                        ulwp->ul_schedctl = NULL;
                }
        }

        /*
         * Set all lwps that were stopped for fork() running again.
         */
        lmutex_lock(&udp->link_lock);
        for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
                mutex_t *mp = ulwp_mutex(ulwp, udp);
                lmutex_lock(mp);
                ASSERT(ulwp->ul_stop & TSTP_FORK);
                ulwp->ul_stop &= ~TSTP_FORK;
                ulwp_broadcast(ulwp);
                if (!ulwp->ul_stop)
                        force_continue(ulwp);
                lmutex_unlock(mp);
        }
        lmutex_unlock(&udp->link_lock);
}

int
_thrp_continue(thread_t tid, uchar_t whystopped)
{
        uberdata_t *udp = curthread->ul_uberdata;
        ulwp_t *ulwp;
        mutex_t *mp;
        int error = 0;

        ASSERT(whystopped == TSTP_REGULAR ||
            whystopped == TSTP_MUTATOR);

        /*
         * We single-thread the entire thread suspend/continue mechanism.
         */
        fork_lock_enter();

        if ((ulwp = find_lwp(tid)) == NULL) {
                fork_lock_exit();
                return (ESRCH);
        }

        mp = ulwp_mutex(ulwp, udp);
        if ((whystopped == TSTP_MUTATOR && !ulwp->ul_mutator)) {
                error = EINVAL;
        } else if (ulwp->ul_stop & whystopped) {
                ulwp->ul_stop &= ~whystopped;
                ulwp_broadcast(ulwp);
                if (!ulwp->ul_stop) {
                        if (whystopped == TSTP_REGULAR && ulwp->ul_created) {
                                ulwp->ul_sp = 0;
                                ulwp->ul_created = 0;
                        }
                        force_continue(ulwp);
                }
        }
        lmutex_unlock(mp);

        fork_lock_exit();
        return (error);
}

int
thr_suspend(thread_t tid)
{
        return (_thrp_suspend(tid, TSTP_REGULAR));
}

int
thr_continue(thread_t tid)
{
        return (_thrp_continue(tid, TSTP_REGULAR));
}

void
thr_yield()
{
        yield();
}

#pragma weak pthread_kill = thr_kill
#pragma weak _thr_kill = thr_kill
int
thr_kill(thread_t tid, int sig)
{
        if (sig == SIGCANCEL)
                return (EINVAL);
        return (_lwp_kill(tid, sig));
}

/*
 * Exit a critical section, take deferred actions if necessary.
 * Called from exit_critical() and from sigon().
 */
void
do_exit_critical()
{
        ulwp_t *self = curthread;
        int sig;

        ASSERT(self->ul_critical == 0);

        /*
         * Don't suspend ourself or take a deferred signal while dying
         * or while executing inside the dynamic linker (ld.so.1).
         */
        if (self->ul_dead || self->ul_rtld)
                return;

        while (self->ul_pleasestop ||
            (self->ul_cursig != 0 && self->ul_sigdefer == 0)) {
                /*
                 * Avoid a recursive call to exit_critical() in _thrp_suspend()
                 * by keeping self->ul_critical == 1 here.
                 */
                self->ul_critical++;
                while (self->ul_pleasestop) {
                        /*
                         * Guard against suspending ourself while on a sleep
                         * queue.  See the comments in call_user_handler().
                         */
                        unsleep_self();
                        set_parking_flag(self, 0);
                        (void) _thrp_suspend(self->ul_lwpid,
                            self->ul_pleasestop);
                }
                self->ul_critical--;

                if ((sig = self->ul_cursig) != 0 && self->ul_sigdefer == 0) {
                        /*
                         * Clear ul_cursig before proceeding.
                         * This protects us from the dynamic linker's
                         * calls to bind_guard()/bind_clear() in the
                         * event that it is invoked to resolve a symbol
                         * like take_deferred_signal() below.
                         */
                        self->ul_cursig = 0;
                        take_deferred_signal(sig);
                        ASSERT(self->ul_cursig == 0);
                }
        }
        ASSERT(self->ul_critical == 0);
}

/*
 * _ti_bind_guard() and _ti_bind_clear() are called by the dynamic linker
 * (ld.so.1) when it has do do something, like resolve a symbol to be called
 * by the application or one of its libraries.  _ti_bind_guard() is called
 * on entry to ld.so.1, _ti_bind_clear() on exit from ld.so.1 back to the
 * application.  The dynamic linker gets special dispensation from libc to
 * run in a critical region (all signals deferred and no thread suspension
 * or forking allowed), and to be immune from cancellation for the duration.
 */
int
_ti_bind_guard(int flags)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        int bindflag = (flags & THR_FLG_RTLD);

        if ((self->ul_bindflags & bindflag) == bindflag)
                return (0);
        self->ul_bindflags |= bindflag;
        if ((flags & (THR_FLG_NOLOCK | THR_FLG_REENTER)) == THR_FLG_NOLOCK) {
                sigoff(self);   /* see no signals while holding ld_lock */
                self->ul_rtld++;        /* don't suspend while in ld.so.1 */
                (void) mutex_lock(&udp->ld_lock);
        }
        enter_critical(self);
        self->ul_save_state = self->ul_cancel_disabled;
        self->ul_cancel_disabled = 1;
        set_cancel_pending_flag(self, 0);
        return (1);
}

int
_ti_bind_clear(int flags)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        int bindflag = (flags & THR_FLG_RTLD);

        if ((self->ul_bindflags & bindflag) == 0)
                return (self->ul_bindflags);
        self->ul_bindflags &= ~bindflag;
        self->ul_cancel_disabled = self->ul_save_state;
        set_cancel_pending_flag(self, 0);
        exit_critical(self);
        if ((flags & (THR_FLG_NOLOCK | THR_FLG_REENTER)) == THR_FLG_NOLOCK) {
                if (MUTEX_OWNED(&udp->ld_lock, self)) {
                        (void) mutex_unlock(&udp->ld_lock);
                        self->ul_rtld--;
                        sigon(self);    /* reenable signals */
                }
        }
        return (self->ul_bindflags);
}

/*
 * Tell the dynamic linker (ld.so.1) whether or not it was entered from
 * a critical region in libc.  Return zero if not, else return non-zero.
 */
int
_ti_critical(void)
{
        ulwp_t *self = curthread;
        int level = self->ul_critical;

        if ((self->ul_bindflags & THR_FLG_RTLD) == 0 || level == 0)
                return (level); /* ld.so.1 hasn't (yet) called enter() */
        return (level - 1);
}

/*
 * sigoff() and sigon() enable cond_wait() to behave (optionally) like
 * it does in the old libthread (see the comments in cond_wait_queue()).
 * Also, signals are deferred at thread startup until TLS constructors
 * have all been called, at which time _thrp_setup() calls sigon().
 *
 * _sigoff() and _sigon() are external consolidation-private interfaces to
 * sigoff() and sigon(), respectively, in libc.  These are used in libnsl.
 * Also, _sigoff() and _sigon() are called from dbx's run-time checking
 * (librtc.so) to defer signals during its critical sections (not to be
 * confused with libc critical sections [see exit_critical() above]).
 */
void
_sigoff(void)
{
        ulwp_t *self = curthread;

        sigoff(self);
}

void
_sigon(void)
{
        ulwp_t *self = curthread;

        ASSERT(self->ul_sigdefer > 0);
        sigon(self);
}

int
thr_getconcurrency()
{
        return (thr_concurrency);
}

int
pthread_getconcurrency()
{
        return (pthread_concurrency);
}

int
thr_setconcurrency(int new_level)
{
        uberdata_t *udp = curthread->ul_uberdata;

        if (new_level < 0)
                return (EINVAL);
        if (new_level > 65536)          /* 65536 is totally arbitrary */
                return (EAGAIN);
        lmutex_lock(&udp->link_lock);
        if (new_level > thr_concurrency)
                thr_concurrency = new_level;
        lmutex_unlock(&udp->link_lock);
        return (0);
}

int
pthread_setconcurrency(int new_level)
{
        if (new_level < 0)
                return (EINVAL);
        if (new_level > 65536)          /* 65536 is totally arbitrary */
                return (EAGAIN);
        pthread_concurrency = new_level;
        return (0);
}

size_t
thr_min_stack(void)
{
        return (MINSTACK);
}

int
__nthreads(void)
{
        return (curthread->ul_uberdata->nthreads);
}

/* "/proc/self/lwp/%u/lwpname" w/o stdio */
static void
lwpname_path(pthread_t tid, char *buf, size_t bufsize)
{
        (void) strlcpy(buf, "/proc/self/lwp/", bufsize);
        ultos((uint64_t)tid, 10, buf + strlen(buf));
        (void) strlcat(buf, "/lwpname", bufsize);
}

#pragma weak pthread_setname_np = thr_setname
int
thr_setname(pthread_t tid, const char *name)
{
        extern ssize_t __write(int, const void *, size_t);
        char path[PATH_MAX];
        int saved_errno;
        size_t len;
        ssize_t n;
        int fd;

        if (name == NULL)
                name = "";

        len = strlen(name) + 1;
        if (len > THREAD_NAME_MAX)
                return (ERANGE);

        lwpname_path(tid, path, sizeof (path));

        if ((fd = __open(path, O_WRONLY, 0)) < 0) {
                if (errno == ENOENT)
                        errno = ESRCH;
                return (errno);
        }

        n = __write(fd, name, len);
        saved_errno = errno;
        (void) __close(fd);

        if (n < 0)
                return (saved_errno);
        if (n != len)
                return (EFAULT);
        return (0);
}

#pragma weak pthread_getname_np = thr_getname
int
thr_getname(pthread_t tid, char *buf, size_t bufsize)
{
        extern ssize_t __read(int, void *, size_t);
        char name[THREAD_NAME_MAX];
        char path[PATH_MAX];
        int saved_errno;
        ssize_t n;
        int fd;

        if (buf == NULL)
                return (EINVAL);

        lwpname_path(tid, path, sizeof (path));

        if ((fd = __open(path, O_RDONLY, 0)) < 0) {
                if (errno == ENOENT)
                        errno = ESRCH;
                return (errno);
        }

        n = __read(fd, name, sizeof (name));
        saved_errno = errno;
        (void) __close(fd);

        if (n < 0)
                return (saved_errno);
        if (n != sizeof (name))
                return (EFAULT);
        if (strlcpy(buf, name, bufsize) >= bufsize)
                return (ERANGE);
        return (0);
}

/*
 * XXX
 * The remainder of this file implements the private interfaces to java for
 * garbage collection.  It is no longer used, at least by java 1.2.
 * It can all go away once all old JVMs have disappeared.
 */

int     suspendingallmutators;  /* when non-zero, suspending all mutators. */
int     suspendedallmutators;   /* when non-zero, all mutators suspended. */
int     mutatorsbarrier;        /* when non-zero, mutators barrier imposed. */
mutex_t mutatorslock = DEFAULTMUTEX;    /* used to enforce mutators barrier. */
cond_t  mutatorscv = DEFAULTCV;         /* where non-mutators sleep. */

/*
 * Get the available register state for the target thread.
 * Return non-volatile registers: TRS_NONVOLATILE
 */
#pragma weak _thr_getstate = thr_getstate
int
thr_getstate(thread_t tid, int *flag, lwpid_t *lwp, stack_t *ss, gregset_t rs)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        ulwp_t **ulwpp;
        ulwp_t *ulwp;
        int error = 0;
        int trs_flag = TRS_LWPID;

        if (tid == 0 || self->ul_lwpid == tid) {
                ulwp = self;
                ulwp_lock(ulwp, udp);
        } else if ((ulwpp = find_lwpp(tid)) != NULL) {
                ulwp = *ulwpp;
        } else {
                if (flag)
                        *flag = TRS_INVALID;
                return (ESRCH);
        }

        if (ulwp->ul_dead) {
                trs_flag = TRS_INVALID;
        } else if (!ulwp->ul_stop && !suspendedallmutators) {
                error = EINVAL;
                trs_flag = TRS_INVALID;
        } else if (ulwp->ul_stop) {
                trs_flag = TRS_NONVOLATILE;
                getgregs(ulwp, rs);
        }

        if (flag)
                *flag = trs_flag;
        if (lwp)
                *lwp = tid;
        if (ss != NULL)
                (void) _thrp_stksegment(ulwp, ss);

        ulwp_unlock(ulwp, udp);
        return (error);
}

/*
 * Set the appropriate register state for the target thread.
 * This is not used by java.  It exists solely for the MSTC test suite.
 */
#pragma weak _thr_setstate = thr_setstate
int
thr_setstate(thread_t tid, int flag, gregset_t rs)
{
        uberdata_t *udp = curthread->ul_uberdata;
        ulwp_t *ulwp;
        int error = 0;

        if ((ulwp = find_lwp(tid)) == NULL)
                return (ESRCH);

        if (!ulwp->ul_stop && !suspendedallmutators)
                error = EINVAL;
        else if (rs != NULL) {
                switch (flag) {
                case TRS_NONVOLATILE:
                        /* do /proc stuff here? */
                        if (ulwp->ul_stop)
                                setgregs(ulwp, rs);
                        else
                                error = EINVAL;
                        break;
                case TRS_LWPID:         /* do /proc stuff here? */
                default:
                        error = EINVAL;
                        break;
                }
        }

        ulwp_unlock(ulwp, udp);
        return (error);
}

int
getlwpstatus(thread_t tid, struct lwpstatus *sp)
{
        extern ssize_t __pread(int, void *, size_t, off_t);
        char buf[100];
        int fd;

        /* "/proc/self/lwp/%u/lwpstatus" w/o stdio */
        (void) strcpy(buf, "/proc/self/lwp/");
        ultos((uint64_t)tid, 10, buf + strlen(buf));
        (void) strcat(buf, "/lwpstatus");
        if ((fd = __open(buf, O_RDONLY, 0)) >= 0) {
                while (__pread(fd, sp, sizeof (*sp), 0) == sizeof (*sp)) {
                        if (sp->pr_flags & PR_STOPPED) {
                                (void) __close(fd);
                                return (0);
                        }
                        yield();        /* give it a chance to stop */
                }
                (void) __close(fd);
        }
        return (-1);
}

int
putlwpregs(thread_t tid, prgregset_t prp)
{
        extern ssize_t __writev(int, const struct iovec *, int);
        char buf[100];
        int fd;
        long dstop_sreg[2];
        long run_null[2];
        iovec_t iov[3];

        /* "/proc/self/lwp/%u/lwpctl" w/o stdio */
        (void) strcpy(buf, "/proc/self/lwp/");
        ultos((uint64_t)tid, 10, buf + strlen(buf));
        (void) strcat(buf, "/lwpctl");
        if ((fd = __open(buf, O_WRONLY, 0)) >= 0) {
                dstop_sreg[0] = PCDSTOP;        /* direct it to stop */
                dstop_sreg[1] = PCSREG;         /* set the registers */
                iov[0].iov_base = (caddr_t)dstop_sreg;
                iov[0].iov_len = sizeof (dstop_sreg);
                iov[1].iov_base = (caddr_t)prp; /* from the register set */
                iov[1].iov_len = sizeof (prgregset_t);
                run_null[0] = PCRUN;            /* make it runnable again */
                run_null[1] = 0;
                iov[2].iov_base = (caddr_t)run_null;
                iov[2].iov_len = sizeof (run_null);
                if (__writev(fd, iov, 3) >= 0) {
                        (void) __close(fd);
                        return (0);
                }
                (void) __close(fd);
        }
        return (-1);
}

static ulong_t
gettsp_slow(thread_t tid)
{
        char buf[100];
        struct lwpstatus status;

        if (getlwpstatus(tid, &status) != 0) {
                /* "__gettsp(%u): can't read lwpstatus" w/o stdio */
                (void) strcpy(buf, "__gettsp(");
                ultos((uint64_t)tid, 10, buf + strlen(buf));
                (void) strcat(buf, "): can't read lwpstatus");
                thr_panic(buf);
        }
        return (status.pr_reg[R_SP]);
}

ulong_t
__gettsp(thread_t tid)
{
        uberdata_t *udp = curthread->ul_uberdata;
        ulwp_t *ulwp;
        ulong_t result;

        if ((ulwp = find_lwp(tid)) == NULL)
                return (0);

        if (ulwp->ul_stop && (result = ulwp->ul_sp) != 0) {
                ulwp_unlock(ulwp, udp);
                return (result);
        }

        result = gettsp_slow(tid);
        ulwp_unlock(ulwp, udp);
        return (result);
}

/*
 * This tells java stack walkers how to find the ucontext
 * structure passed to signal handlers.
 */
#pragma weak _thr_sighndlrinfo = thr_sighndlrinfo
void
thr_sighndlrinfo(void (**func)(), int *funcsize)
{
        *func = &__sighndlr;
        *funcsize = (char *)&__sighndlrend - (char *)&__sighndlr;
}

/*
 * Mark a thread a mutator or reset a mutator to being a default,
 * non-mutator thread.
 */
#pragma weak _thr_setmutator = thr_setmutator
int
thr_setmutator(thread_t tid, int enabled)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        ulwp_t *ulwp;
        int error;
        int cancel_state;

        enabled = enabled? 1 : 0;
top:
        if (tid == 0) {
                ulwp = self;
                ulwp_lock(ulwp, udp);
        } else if ((ulwp = find_lwp(tid)) == NULL) {
                return (ESRCH);
        }

        /*
         * The target thread should be the caller itself or a suspended thread.
         * This prevents the target from also changing its ul_mutator field.
         */
        error = 0;
        if (ulwp != self && !ulwp->ul_stop && enabled)
                error = EINVAL;
        else if (ulwp->ul_mutator != enabled) {
                lmutex_lock(&mutatorslock);
                if (mutatorsbarrier) {
                        ulwp_unlock(ulwp, udp);
                        (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
                            &cancel_state);
                        while (mutatorsbarrier)
                                (void) cond_wait(&mutatorscv, &mutatorslock);
                        (void) pthread_setcancelstate(cancel_state, NULL);
                        lmutex_unlock(&mutatorslock);
                        goto top;
                }
                ulwp->ul_mutator = enabled;
                lmutex_unlock(&mutatorslock);
        }

        ulwp_unlock(ulwp, udp);
        return (error);
}

/*
 * Establish a barrier against new mutators.  Any non-mutator trying
 * to become a mutator is suspended until the barrier is removed.
 */
#pragma weak _thr_mutators_barrier = thr_mutators_barrier
void
thr_mutators_barrier(int enabled)
{
        int oldvalue;
        int cancel_state;

        lmutex_lock(&mutatorslock);

        /*
         * Wait if trying to set the barrier while it is already set.
         */
        (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
        while (mutatorsbarrier && enabled)
                (void) cond_wait(&mutatorscv, &mutatorslock);
        (void) pthread_setcancelstate(cancel_state, NULL);

        oldvalue = mutatorsbarrier;
        mutatorsbarrier = enabled;
        /*
         * Wakeup any blocked non-mutators when barrier is removed.
         */
        if (oldvalue && !enabled)
                (void) cond_broadcast(&mutatorscv);
        lmutex_unlock(&mutatorslock);
}

/*
 * Suspend the set of all mutators except for the caller.  The list
 * of actively running threads is searched and only the mutators
 * in this list are suspended.  Actively running non-mutators remain
 * running.  Any other thread is suspended.
 */
#pragma weak _thr_suspend_allmutators = thr_suspend_allmutators
int
thr_suspend_allmutators(void)
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        ulwp_t *ulwp;
        int link_dropped;

        /*
         * We single-thread the entire thread suspend/continue mechanism.
         */
        fork_lock_enter();

top:
        lmutex_lock(&udp->link_lock);

        if (suspendingallmutators || suspendedallmutators) {
                lmutex_unlock(&udp->link_lock);
                fork_lock_exit();
                return (EINVAL);
        }
        suspendingallmutators = 1;

        for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
                ulwp_lock(ulwp, udp);
                if (!ulwp->ul_mutator) {
                        ulwp_unlock(ulwp, udp);
                } else if (ulwp->ul_stop) {     /* already stopped */
                        ulwp->ul_stop |= TSTP_MUTATOR;
                        ulwp_broadcast(ulwp);
                        ulwp_unlock(ulwp, udp);
                } else {
                        /*
                         * Move the stopped lwp out of a critical section.
                         */
                        if (safe_suspend(ulwp, TSTP_MUTATOR, &link_dropped) ||
                            link_dropped) {
                                suspendingallmutators = 0;
                                goto top;
                        }
                }
        }

        suspendedallmutators = 1;
        suspendingallmutators = 0;
        lmutex_unlock(&udp->link_lock);
        fork_lock_exit();
        return (0);
}

/*
 * Suspend the target mutator.  The caller is permitted to suspend
 * itself.  If a mutator barrier is enabled, the caller will suspend
 * itself as though it had been suspended by thr_suspend_allmutators().
 * When the barrier is removed, this thread will be resumed.  Any
 * suspended mutator, whether suspended by thr_suspend_mutator(), or by
 * thr_suspend_allmutators(), can be resumed by thr_continue_mutator().
 */
#pragma weak _thr_suspend_mutator = thr_suspend_mutator
int
thr_suspend_mutator(thread_t tid)
{
        if (tid == 0)
                tid = curthread->ul_lwpid;
        return (_thrp_suspend(tid, TSTP_MUTATOR));
}

/*
 * Resume the set of all suspended mutators.
 */
#pragma weak _thr_continue_allmutators = thr_continue_allmutators
int
thr_continue_allmutators()
{
        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;
        ulwp_t *ulwp;

        /*
         * We single-thread the entire thread suspend/continue mechanism.
         */
        fork_lock_enter();

        lmutex_lock(&udp->link_lock);
        if (!suspendedallmutators) {
                lmutex_unlock(&udp->link_lock);
                fork_lock_exit();
                return (EINVAL);
        }
        suspendedallmutators = 0;

        for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
                mutex_t *mp = ulwp_mutex(ulwp, udp);
                lmutex_lock(mp);
                if (ulwp->ul_stop & TSTP_MUTATOR) {
                        ulwp->ul_stop &= ~TSTP_MUTATOR;
                        ulwp_broadcast(ulwp);
                        if (!ulwp->ul_stop)
                                force_continue(ulwp);
                }
                lmutex_unlock(mp);
        }

        lmutex_unlock(&udp->link_lock);
        fork_lock_exit();
        return (0);
}

/*
 * Resume a suspended mutator.
 */
#pragma weak _thr_continue_mutator = thr_continue_mutator
int
thr_continue_mutator(thread_t tid)
{
        return (_thrp_continue(tid, TSTP_MUTATOR));
}

#pragma weak _thr_wait_mutator = thr_wait_mutator
int
thr_wait_mutator(thread_t tid, int dontwait)
{
        uberdata_t *udp = curthread->ul_uberdata;
        ulwp_t *ulwp;
        int cancel_state;
        int error = 0;

        (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
top:
        if ((ulwp = find_lwp(tid)) == NULL) {
                (void) pthread_setcancelstate(cancel_state, NULL);
                return (ESRCH);
        }

        if (!ulwp->ul_mutator)
                error = EINVAL;
        else if (dontwait) {
                if (!(ulwp->ul_stop & TSTP_MUTATOR))
                        error = EWOULDBLOCK;
        } else if (!(ulwp->ul_stop & TSTP_MUTATOR)) {
                cond_t *cvp = ulwp_condvar(ulwp, udp);
                mutex_t *mp = ulwp_mutex(ulwp, udp);

                (void) cond_wait(cvp, mp);
                (void) lmutex_unlock(mp);
                goto top;
        }

        ulwp_unlock(ulwp, udp);
        (void) pthread_setcancelstate(cancel_state, NULL);
        return (error);
}

/* PROBE_SUPPORT begin */

void
thr_probe_setup(void *data)
{
        curthread->ul_tpdp = data;
}

static void *
_thread_probe_getfunc()
{
        return (curthread->ul_tpdp);
}

void * (*thr_probe_getfunc_addr)(void) = _thread_probe_getfunc;

/* ARGSUSED */
void
_resume(ulwp_t *ulwp, caddr_t sp, int dontsave)
{
        /* never called */
}

/* ARGSUSED */
void
_resume_ret(ulwp_t *oldlwp)
{
        /* never called */
}

/* PROBE_SUPPORT end */