4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
27 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
32 * Copyright 1993 OpenVision Technologies, Inc., All Rights Reserved.
35 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
36 /* All Rights Reserved */
39 * Portions of this source code were derived from Berkeley 4.3 BSD
40 * under license from the Regents of the University of California.
44 * Server-side remote procedure call interface.
46 * Master transport handle (SVCMASTERXPRT).
47 * The master transport handle structure is shared among service
48 * threads processing events on the transport. Some fields in the
49 * master structure are protected by locks
50 * - xp_req_lock protects the request queue:
51 * xp_req_head, xp_req_tail, xp_reqs, xp_size, xp_full, xp_enable
52 * - xp_thread_lock protects the thread (clone) counts
53 * xp_threads, xp_detached_threads, xp_wq
54 * Each master transport is registered to exactly one thread pool.
56 * Clone transport handle (SVCXPRT)
57 * The clone transport handle structure is a per-service-thread handle
58 * to the transport. The structure carries all the fields/buffers used
59 * for request processing. A service thread or, in other words, a clone
60 * structure, can be linked to an arbitrary master structure to process
61 * requests on this transport. The master handle keeps track of reference
62 * counts of threads (clones) linked to it. A service thread can switch
63 * to another transport by unlinking its clone handle from the current
64 * transport and linking to a new one. Switching is relatively inexpensive
65 * but it involves locking (master's xprt->xp_thread_lock).
68 * A pool represents a kernel RPC service (NFS, Lock Manager, etc.).
69 * Transports related to the service are registered to the service pool.
70 * Service threads can switch between different transports in the pool.
71 * Thus, each service has its own pool of service threads. The maximum
72 * number of threads in a pool is pool->p_maxthreads. This limit allows
73 * to restrict resource usage by the service. Some fields are protected
75 * - p_req_lock protects several counts and flags:
76 * p_reqs, p_size, p_walkers, p_asleep, p_drowsy, p_req_cv
77 * - p_thread_lock governs other thread counts:
78 * p_threads, p_detached_threads, p_reserved_threads, p_closing
80 * In addition, each pool contains a doubly-linked list of transports,
81 * an `xprt-ready' queue and a creator thread (see below). Threads in
82 * the pool share some other parameters such as stack size and
85 * Pools are initialized through the svc_pool_create() function called from
86 * the nfssys() system call. However, thread creation must be done by
87 * the userland agent. This is done by using SVCPOOL_WAIT and
88 * SVCPOOL_RUN arguments to nfssys(), which call svc_wait() and
89 * svc_do_run(), respectively. Once the pool has been initialized,
90 * the userland process must set up a 'creator' thread. This thread
91 * should park itself in the kernel by calling svc_wait(). If
92 * svc_wait() returns successfully, it should fork off a new worker
93 * thread, which then calls svc_do_run() in order to get work. When
94 * that thread is complete, svc_do_run() will return, and the user
95 * program should call thr_exit().
97 * When we try to register a new pool and there is an old pool with
98 * the same id in the doubly linked pool list (this happens when we kill
99 * and restart nfsd or lockd), then we unlink the old pool from the list
100 * and mark its state as `closing'. After that the transports can still
101 * process requests but new transports won't be registered. When all the
102 * transports and service threads associated with the pool are gone the
103 * creator thread (see below) will clean up the pool structure and exit.
105 * svc_queuereq() and svc_run().
106 * The kernel RPC server is interrupt driven. The svc_queuereq() interrupt
107 * routine is called to deliver an RPC request. The service threads
108 * loop in svc_run(). The interrupt function queues a request on the
109 * transport's queue and it makes sure that the request is serviced.
110 * It may either wake up one of sleeping threads, or ask for a new thread
111 * to be created, or, if the previous request is just being picked up, do
112 * nothing. In the last case the service thread that is picking up the
113 * previous request will wake up or create the next thread. After a service
114 * thread processes a request and sends a reply it returns to svc_run()
115 * and svc_run() calls svc_poll() to find new input.
118 * In order to avoid unnecessary locking, which causes performance
119 * problems, we always look for a pending request on the current transport.
120 * If there is none we take a hint from the pool's `xprt-ready' queue.
121 * If the queue had an overflow we switch to the `drain' mode checking
122 * each transport in the pool's transport list. Once we find a
123 * master transport handle with a pending request we latch the request
124 * lock on this transport and return to svc_run(). If the request
125 * belongs to a transport different than the one the service thread is
126 * linked to we need to unlink and link again.
128 * A service thread goes asleep when there are no pending
129 * requests on the transports registered on the pool's transports.
130 * All the pool's threads sleep on the same condition variable.
131 * If a thread has been sleeping for too long period of time
132 * (by default 5 seconds) it wakes up and exits. Also when a transport
133 * is closing sleeping threads wake up to unlink from this transport.
135 * The `xprt-ready' queue.
136 * If a service thread finds no request on a transport it is currently linked
137 * to it will find another transport with a pending request. To make
138 * this search more efficient each pool has an `xprt-ready' queue.
139 * The queue is a FIFO. When the interrupt routine queues a request it also
140 * inserts a pointer to the transport into the `xprt-ready' queue. A
141 * thread looking for a transport with a pending request can pop up a
142 * transport and check for a request. The request can be already gone
143 * since it could be taken by a thread linked to that transport. In such a
144 * case we try the next hint. The `xprt-ready' queue has fixed size (by
145 * default 256 nodes). If it overflows svc_poll() has to switch to the
146 * less efficient but safe `drain' mode and walk through the pool's
149 * Both the svc_poll() loop and the `xprt-ready' queue are optimized
150 * for the peak load case that is for the situation when the queue is not
151 * empty, there are all the time few pending requests, and a service
152 * thread which has just processed a request does not go asleep but picks
153 * up immediately the next request.
156 * Each pool has a thread creator associated with it. The creator thread
157 * sleeps on a condition variable and waits for a signal to create a
158 * service thread. The actual thread creation is done in userland by
159 * the method described in "Pools" above.
161 * Signaling threads should turn on the `creator signaled' flag, and
162 * can avoid sending signals when the flag is on. The flag is cleared
163 * when the thread is created.
165 * When the pool is in closing state (ie it has been already unregistered
166 * from the pool list) the last thread on the last transport in the pool
167 * should turn the p_creator_exit flag on. The creator thread will
168 * clean up the pool structure and exit.
170 * Thread reservation; Detaching service threads.
171 * A service thread can detach itself to block for an extended amount
172 * of time. However, to keep the service active we need to guarantee
173 * at least pool->p_redline non-detached threads that can process incoming
174 * requests. This, the maximum number of detached and reserved threads is
175 * p->p_maxthreads - p->p_redline. A service thread should first acquire
176 * a reservation, and if the reservation was granted it can detach itself.
177 * If a reservation was granted but the thread does not detach itself
178 * it should cancel the reservation before it returns to svc_run().
181 #include <sys/param.h>
182 #include <sys/types.h>
183 #include <rpc/types.h>
184 #include <sys/socket.h>
185 #include <sys/time.h>
186 #include <sys/tiuser.h>
187 #include <sys/t_kuser.h>
188 #include <netinet/in.h>
190 #include <rpc/auth.h>
191 #include <rpc/clnt.h>
192 #include <rpc/rpc_msg.h>
194 #include <sys/proc.h>
195 #include <sys/user.h>
196 #include <sys/stream.h>
197 #include <sys/strsubr.h>
198 #include <sys/strsun.h>
199 #include <sys/tihdr.h>
200 #include <sys/debug.h>
201 #include <sys/cmn_err.h>
202 #include <sys/file.h>
203 #include <sys/systm.h>
204 #include <sys/callb.h>
205 #include <sys/vtrace.h>
206 #include <sys/zone.h>
209 #define RQCRED_SIZE 400 /* this size is excessive */
212 * Defines for svc_poll()
214 #define SVC_EXPRTGONE ((SVCMASTERXPRT *)1) /* Transport is closing */
215 #define SVC_ETIMEDOUT ((SVCMASTERXPRT *)2) /* Timeout */
216 #define SVC_EINTR ((SVCMASTERXPRT *)3) /* Interrupted by signal */
219 * Default stack size for service threads.
221 #define DEFAULT_SVC_RUN_STKSIZE (0) /* default kernel stack */
223 int svc_default_stksize
= DEFAULT_SVC_RUN_STKSIZE
;
226 * Default polling timeout for service threads.
227 * Multiplied by hz when used.
229 #define DEFAULT_SVC_POLL_TIMEOUT (5) /* seconds */
231 clock_t svc_default_timeout
= DEFAULT_SVC_POLL_TIMEOUT
;
234 * Size of the `xprt-ready' queue.
236 #define DEFAULT_SVC_QSIZE (256) /* qnodes */
238 size_t svc_default_qsize
= DEFAULT_SVC_QSIZE
;
241 * Default limit for the number of service threads.
243 #define DEFAULT_SVC_MAXTHREADS (INT16_MAX)
245 int svc_default_maxthreads
= DEFAULT_SVC_MAXTHREADS
;
248 * Maximum number of requests from the same transport (in `drain' mode).
250 #define DEFAULT_SVC_MAX_SAME_XPRT (8)
252 int svc_default_max_same_xprt
= DEFAULT_SVC_MAX_SAME_XPRT
;
256 * Default `Redline' of non-detached threads.
257 * Total number of detached and reserved threads in an RPC server
258 * thread pool is limited to pool->p_maxthreads - svc_redline.
260 #define DEFAULT_SVC_REDLINE (1)
262 int svc_default_redline
= DEFAULT_SVC_REDLINE
;
265 * A node for the `xprt-ready' queue.
268 struct __svcxprt_qnode
{
269 __SVCXPRT_QNODE
*q_next
;
270 SVCMASTERXPRT
*q_xprt
;
274 * Global SVC variables (private).
282 * Debug variable to check for rdma based
283 * transport startup and cleanup. Contorlled
284 * through /etc/system. Off by default.
289 * This allows disabling flow control in svc_queuereq().
291 volatile int svc_flowcontrol_disable
= 0;
294 * Authentication parameters list.
296 static caddr_t rqcred_head
;
297 static kmutex_t rqcred_lock
;
300 * Pointers to transport specific `rele' routines in rpcmod (set from rpcmod).
302 void (*rpc_rele
)(queue_t
*, mblk_t
*, bool_t
) = NULL
;
303 void (*mir_rele
)(queue_t
*, mblk_t
*, bool_t
) = NULL
;
307 rpc_rdma_rele(queue_t
*q
, mblk_t
*mp
, bool_t enable
)
310 void (*rdma_rele
)(queue_t
*, mblk_t
*, bool_t
) = rpc_rdma_rele
;
314 * This macro picks which `rele' routine to use, based on the transport type.
316 #define RELE_PROC(xprt) \
317 ((xprt)->xp_type == T_RDMA ? rdma_rele : \
318 (((xprt)->xp_type == T_CLTS) ? rpc_rele : mir_rele))
321 * If true, then keep quiet about version mismatch.
322 * This macro is for broadcast RPC only. We have no broadcast RPC in
323 * kernel now but one may define a flag in the transport structure
324 * and redefine this macro.
326 #define version_keepquiet(xprt) (FALSE)
329 * ZSD key used to retrieve zone-specific svc globals
331 static zone_key_t svc_zone_key
;
333 static void svc_callout_free(SVCMASTERXPRT
*);
334 static void svc_xprt_qinit(SVCPOOL
*, size_t);
335 static void svc_xprt_qdestroy(SVCPOOL
*);
336 static void svc_thread_creator(SVCPOOL
*);
337 static void svc_creator_signal(SVCPOOL
*);
338 static void svc_creator_signalexit(SVCPOOL
*);
339 static void svc_pool_unregister(struct svc_globals
*, SVCPOOL
*);
340 static int svc_run(SVCPOOL
*);
344 svc_zoneinit(zoneid_t zoneid
)
346 struct svc_globals
*svc
;
348 svc
= kmem_alloc(sizeof (*svc
), KM_SLEEP
);
349 mutex_init(&svc
->svc_plock
, NULL
, MUTEX_DEFAULT
, NULL
);
350 svc
->svc_pools
= NULL
;
356 svc_zoneshutdown(zoneid_t zoneid
, void *arg
)
358 struct svc_globals
*svc
= arg
;
361 mutex_enter(&svc
->svc_plock
);
362 while ((pool
= svc
->svc_pools
) != NULL
) {
363 svc_pool_unregister(svc
, pool
);
365 mutex_exit(&svc
->svc_plock
);
370 svc_zonefini(zoneid_t zoneid
, void *arg
)
372 struct svc_globals
*svc
= arg
;
374 ASSERT(svc
->svc_pools
== NULL
);
375 mutex_destroy(&svc
->svc_plock
);
376 kmem_free(svc
, sizeof (*svc
));
380 * Global SVC init routine.
381 * Initialize global generic and transport type specific structures
382 * used by the kernel RPC server side. This routine is called only
383 * once when the module is being loaded.
388 zone_key_create(&svc_zone_key
, svc_zoneinit
, svc_zoneshutdown
,
395 * Destroy the SVCPOOL structure.
398 svc_pool_cleanup(SVCPOOL
*pool
)
400 ASSERT(pool
->p_threads
+ pool
->p_detached_threads
== 0);
401 ASSERT(pool
->p_lcount
== 0);
402 ASSERT(pool
->p_closing
);
405 * Call the user supplied shutdown function. This is done
406 * here so the user of the pool will be able to cleanup
407 * service related resources.
409 if (pool
->p_shutdown
!= NULL
)
410 (pool
->p_shutdown
)();
412 /* Destroy `xprt-ready' queue */
413 svc_xprt_qdestroy(pool
);
415 /* Destroy transport list */
416 rw_destroy(&pool
->p_lrwlock
);
418 /* Destroy locks and condition variables */
419 mutex_destroy(&pool
->p_thread_lock
);
420 mutex_destroy(&pool
->p_req_lock
);
421 cv_destroy(&pool
->p_req_cv
);
423 /* Destroy creator's locks and condition variables */
424 mutex_destroy(&pool
->p_creator_lock
);
425 cv_destroy(&pool
->p_creator_cv
);
426 mutex_destroy(&pool
->p_user_lock
);
427 cv_destroy(&pool
->p_user_cv
);
429 /* Free pool structure */
430 kmem_free(pool
, sizeof (SVCPOOL
));
434 * If all the transports and service threads are already gone
435 * signal the creator thread to clean up and exit.
438 svc_pool_tryexit(SVCPOOL
*pool
)
440 ASSERT(MUTEX_HELD(&pool
->p_thread_lock
));
441 ASSERT(pool
->p_closing
);
443 if (pool
->p_threads
+ pool
->p_detached_threads
== 0) {
444 rw_enter(&pool
->p_lrwlock
, RW_READER
);
445 if (pool
->p_lcount
== 0) {
447 * Release the locks before sending a signal.
449 rw_exit(&pool
->p_lrwlock
);
450 mutex_exit(&pool
->p_thread_lock
);
453 * Notify the creator thread to clean up and exit
455 * NOTICE: No references to the pool beyond this point!
456 * The pool is being destroyed.
458 ASSERT(!MUTEX_HELD(&pool
->p_thread_lock
));
459 svc_creator_signalexit(pool
);
463 rw_exit(&pool
->p_lrwlock
);
466 ASSERT(MUTEX_HELD(&pool
->p_thread_lock
));
471 * Find a pool with a given id.
474 svc_pool_find(struct svc_globals
*svc
, int id
)
478 ASSERT(MUTEX_HELD(&svc
->svc_plock
));
481 * Search the list for a pool with a matching id
482 * and register the transport handle with that pool.
484 for (pool
= svc
->svc_pools
; pool
; pool
= pool
->p_next
)
485 if (pool
->p_id
== id
)
492 * PSARC 2003/523 Contract Private Interface
494 * Changes must be reviewed by Solaris File Sharing
495 * Changes must be communicated to contract-2003-523@sun.com
502 struct svc_globals
*svc
;
504 svc
= zone_getspecific(svc_zone_key
, curproc
->p_zone
);
505 mutex_enter(&svc
->svc_plock
);
507 pool
= svc_pool_find(svc
, id
);
509 mutex_exit(&svc
->svc_plock
);
515 * Increment counter of pool threads now
516 * that a thread has been created.
518 mutex_enter(&pool
->p_thread_lock
);
520 mutex_exit(&pool
->p_thread_lock
);
522 /* Give work to the new thread. */
529 * Unregister a pool from the pool list.
530 * Set the closing state. If all the transports and service threads
531 * are already gone signal the creator thread to clean up and exit.
534 svc_pool_unregister(struct svc_globals
*svc
, SVCPOOL
*pool
)
536 SVCPOOL
*next
= pool
->p_next
;
537 SVCPOOL
*prev
= pool
->p_prev
;
539 ASSERT(MUTEX_HELD(&svc
->svc_plock
));
541 /* Remove from the list */
542 if (pool
== svc
->svc_pools
)
543 svc
->svc_pools
= next
;
548 pool
->p_next
= pool
->p_prev
= NULL
;
551 * Offline the pool. Mark the pool as closing.
552 * If there are no transports in this pool notify
553 * the creator thread to clean it up and exit.
555 mutex_enter(&pool
->p_thread_lock
);
556 if (pool
->p_offline
!= NULL
)
558 pool
->p_closing
= TRUE
;
559 if (svc_pool_tryexit(pool
))
561 mutex_exit(&pool
->p_thread_lock
);
565 * Register a pool with a given id in the global doubly linked pool list.
566 * - if there is a pool with the same id in the list then unregister it
567 * - insert the new pool into the list.
570 svc_pool_register(struct svc_globals
*svc
, SVCPOOL
*pool
, int id
)
575 * If there is a pool with the same id then remove it from
576 * the list and mark the pool as closing.
578 mutex_enter(&svc
->svc_plock
);
580 if (old_pool
= svc_pool_find(svc
, id
))
581 svc_pool_unregister(svc
, old_pool
);
583 /* Insert into the doubly linked list */
585 pool
->p_next
= svc
->svc_pools
;
588 svc
->svc_pools
->p_prev
= pool
;
589 svc
->svc_pools
= pool
;
591 mutex_exit(&svc
->svc_plock
);
595 * Initialize a newly created pool structure
598 svc_pool_init(SVCPOOL
*pool
, uint_t maxthreads
, uint_t redline
,
599 uint_t qsize
, uint_t timeout
, uint_t stksize
, uint_t max_same_xprt
)
601 klwp_t
*lwp
= ttolwp(curthread
);
606 maxthreads
= svc_default_maxthreads
;
608 redline
= svc_default_redline
;
610 qsize
= svc_default_qsize
;
612 timeout
= svc_default_timeout
;
614 stksize
= svc_default_stksize
;
615 if (max_same_xprt
== 0)
616 max_same_xprt
= svc_default_max_same_xprt
;
618 if (maxthreads
< redline
)
621 /* Allocate and initialize the `xprt-ready' queue */
622 svc_xprt_qinit(pool
, qsize
);
624 /* Initialize doubly-linked xprt list */
625 rw_init(&pool
->p_lrwlock
, NULL
, RW_DEFAULT
, NULL
);
628 * Setting lwp_childstksz on the current lwp so that
629 * descendants of this lwp get the modified stacksize, if
630 * it is defined. It is important that either this lwp or
631 * one of its descendants do the actual servicepool thread
632 * creation to maintain the stacksize inheritance.
635 lwp
->lwp_childstksz
= stksize
;
637 /* Initialize thread limits, locks and condition variables */
638 pool
->p_maxthreads
= maxthreads
;
639 pool
->p_redline
= redline
;
640 pool
->p_timeout
= timeout
* hz
;
641 pool
->p_stksize
= stksize
;
642 pool
->p_max_same_xprt
= max_same_xprt
;
643 mutex_init(&pool
->p_thread_lock
, NULL
, MUTEX_DEFAULT
, NULL
);
644 mutex_init(&pool
->p_req_lock
, NULL
, MUTEX_DEFAULT
, NULL
);
645 cv_init(&pool
->p_req_cv
, NULL
, CV_DEFAULT
, NULL
);
647 /* Initialize userland creator */
648 pool
->p_user_exit
= FALSE
;
649 pool
->p_signal_create_thread
= FALSE
;
650 pool
->p_user_waiting
= FALSE
;
651 mutex_init(&pool
->p_user_lock
, NULL
, MUTEX_DEFAULT
, NULL
);
652 cv_init(&pool
->p_user_cv
, NULL
, CV_DEFAULT
, NULL
);
654 /* Initialize the creator and start the creator thread */
655 pool
->p_creator_exit
= FALSE
;
656 mutex_init(&pool
->p_creator_lock
, NULL
, MUTEX_DEFAULT
, NULL
);
657 cv_init(&pool
->p_creator_cv
, NULL
, CV_DEFAULT
, NULL
);
659 (void) zthread_create(NULL
, pool
->p_stksize
, svc_thread_creator
,
660 pool
, 0, minclsyspri
);
666 * PSARC 2003/523 Contract Private Interface
668 * Changes must be reviewed by Solaris File Sharing
669 * Changes must be communicated to contract-2003-523@sun.com
671 * Create an kernel RPC server-side thread/transport pool.
673 * This is public interface for creation of a server RPC thread pool
674 * for a given service provider. Transports registered with the pool's id
675 * will be served by a pool's threads. This function is called from the
676 * nfssys() system call.
679 svc_pool_create(struct svcpool_args
*args
)
683 struct svc_globals
*svc
;
686 * Caller should check credentials in a way appropriate
687 * in the context of the call.
690 svc
= zone_getspecific(svc_zone_key
, curproc
->p_zone
);
691 /* Allocate a new pool */
692 pool
= kmem_zalloc(sizeof (SVCPOOL
), KM_SLEEP
);
695 * Initialize the pool structure and create a creator thread.
697 error
= svc_pool_init(pool
, args
->maxthreads
, args
->redline
,
698 args
->qsize
, args
->timeout
, args
->stksize
, args
->max_same_xprt
);
701 kmem_free(pool
, sizeof (SVCPOOL
));
705 /* Register the pool with the global pool list */
706 svc_pool_register(svc
, pool
, args
->id
);
712 svc_pool_control(int id
, int cmd
, void *arg
)
715 struct svc_globals
*svc
;
717 svc
= zone_getspecific(svc_zone_key
, curproc
->p_zone
);
720 case SVCPSET_SHUTDOWN_PROC
:
722 * Search the list for a pool with a matching id
723 * and register the transport handle with that pool.
725 mutex_enter(&svc
->svc_plock
);
727 if ((pool
= svc_pool_find(svc
, id
)) == NULL
) {
728 mutex_exit(&svc
->svc_plock
);
732 * Grab the transport list lock before releasing the
735 rw_enter(&pool
->p_lrwlock
, RW_WRITER
);
736 mutex_exit(&svc
->svc_plock
);
738 pool
->p_shutdown
= *((void (*)())arg
);
740 rw_exit(&pool
->p_lrwlock
);
743 case SVCPSET_UNREGISTER_PROC
:
745 * Search the list for a pool with a matching id
746 * and register the unregister callback handle with that pool.
748 mutex_enter(&svc
->svc_plock
);
750 if ((pool
= svc_pool_find(svc
, id
)) == NULL
) {
751 mutex_exit(&svc
->svc_plock
);
755 * Grab the transport list lock before releasing the
758 rw_enter(&pool
->p_lrwlock
, RW_WRITER
);
759 mutex_exit(&svc
->svc_plock
);
761 pool
->p_offline
= *((void (*)())arg
);
763 rw_exit(&pool
->p_lrwlock
);
772 * Pool's transport list manipulation routines.
773 * - svc_xprt_register()
774 * - svc_xprt_unregister()
776 * svc_xprt_register() is called from svc_tli_kcreate() to
777 * insert a new master transport handle into the doubly linked
778 * list of server transport handles (one list per pool).
780 * The list is used by svc_poll(), when it operates in `drain'
781 * mode, to search for a next transport with a pending request.
785 svc_xprt_register(SVCMASTERXPRT
*xprt
, int id
)
787 SVCMASTERXPRT
*prev
, *next
;
789 struct svc_globals
*svc
;
791 svc
= zone_getspecific(svc_zone_key
, curproc
->p_zone
);
793 * Search the list for a pool with a matching id
794 * and register the transport handle with that pool.
796 mutex_enter(&svc
->svc_plock
);
798 if ((pool
= svc_pool_find(svc
, id
)) == NULL
) {
799 mutex_exit(&svc
->svc_plock
);
803 /* Grab the transport list lock before releasing the pool list lock */
804 rw_enter(&pool
->p_lrwlock
, RW_WRITER
);
805 mutex_exit(&svc
->svc_plock
);
807 /* Don't register new transports when the pool is in closing state */
808 if (pool
->p_closing
) {
809 rw_exit(&pool
->p_lrwlock
);
814 * Initialize xp_pool to point to the pool.
815 * We don't want to go through the pool list every time.
817 xprt
->xp_pool
= pool
;
820 * Insert a transport handle into the list.
821 * The list head points to the most recently inserted transport.
823 if (pool
->p_lhead
== NULL
)
824 pool
->p_lhead
= xprt
->xp_prev
= xprt
->xp_next
= xprt
;
826 next
= pool
->p_lhead
;
827 prev
= pool
->p_lhead
->xp_prev
;
829 xprt
->xp_next
= next
;
830 xprt
->xp_prev
= prev
;
832 pool
->p_lhead
= prev
->xp_next
= next
->xp_prev
= xprt
;
835 /* Increment the transports count */
838 rw_exit(&pool
->p_lrwlock
);
843 * Called from svc_xprt_cleanup() to remove a master transport handle
844 * from the pool's list of server transports (when a transport is
848 svc_xprt_unregister(SVCMASTERXPRT
*xprt
)
850 SVCPOOL
*pool
= xprt
->xp_pool
;
853 * Unlink xprt from the list.
854 * If the list head points to this xprt then move it
855 * to the next xprt or reset to NULL if this is the last
858 rw_enter(&pool
->p_lrwlock
, RW_WRITER
);
860 if (xprt
== xprt
->xp_next
)
861 pool
->p_lhead
= NULL
;
863 SVCMASTERXPRT
*next
= xprt
->xp_next
;
864 SVCMASTERXPRT
*prev
= xprt
->xp_prev
;
866 next
->xp_prev
= prev
;
867 prev
->xp_next
= next
;
869 if (pool
->p_lhead
== xprt
)
870 pool
->p_lhead
= next
;
873 xprt
->xp_next
= xprt
->xp_prev
= NULL
;
875 /* Decrement list count */
878 rw_exit(&pool
->p_lrwlock
);
882 svc_xprt_qdestroy(SVCPOOL
*pool
)
884 mutex_destroy(&pool
->p_qend_lock
);
885 kmem_free(pool
->p_qbody
, pool
->p_qsize
* sizeof (__SVCXPRT_QNODE
));
889 * Initialize an `xprt-ready' queue for a given pool.
892 svc_xprt_qinit(SVCPOOL
*pool
, size_t qsize
)
896 pool
->p_qsize
= qsize
;
897 pool
->p_qbody
= kmem_zalloc(pool
->p_qsize
* sizeof (__SVCXPRT_QNODE
),
900 for (i
= 0; i
< pool
->p_qsize
- 1; i
++)
901 pool
->p_qbody
[i
].q_next
= &(pool
->p_qbody
[i
+1]);
903 pool
->p_qbody
[pool
->p_qsize
-1].q_next
= &(pool
->p_qbody
[0]);
904 pool
->p_qtop
= &(pool
->p_qbody
[0]);
905 pool
->p_qend
= &(pool
->p_qbody
[0]);
907 mutex_init(&pool
->p_qend_lock
, NULL
, MUTEX_DEFAULT
, NULL
);
911 * Called from the svc_queuereq() interrupt routine to queue
912 * a hint for svc_poll() which transport has a pending request.
913 * - insert a pointer to xprt into the xprt-ready queue (FIFO)
914 * - if the xprt-ready queue is full turn the overflow flag on.
916 * NOTICE: pool->p_qtop is protected by the pool's request lock
917 * and the caller (svc_queuereq()) must hold the lock.
920 svc_xprt_qput(SVCPOOL
*pool
, SVCMASTERXPRT
*xprt
)
922 ASSERT(MUTEX_HELD(&pool
->p_req_lock
));
924 /* If the overflow flag is on there is nothing we can do */
925 if (pool
->p_qoverflow
)
928 /* If the queue is full turn the overflow flag on and exit */
929 if (pool
->p_qtop
->q_next
== pool
->p_qend
) {
930 mutex_enter(&pool
->p_qend_lock
);
931 if (pool
->p_qtop
->q_next
== pool
->p_qend
) {
932 pool
->p_qoverflow
= TRUE
;
933 mutex_exit(&pool
->p_qend_lock
);
936 mutex_exit(&pool
->p_qend_lock
);
939 /* Insert a hint and move pool->p_qtop */
940 pool
->p_qtop
->q_xprt
= xprt
;
941 pool
->p_qtop
= pool
->p_qtop
->q_next
;
945 * Called from svc_poll() to get a hint which transport has a
946 * pending request. Returns a pointer to a transport or NULL if the
947 * `xprt-ready' queue is empty.
949 * Since we do not acquire the pool's request lock while checking if
950 * the queue is empty we may miss a request that is just being delivered.
951 * However this is ok since svc_poll() will retry again until the
952 * count indicates that there are pending requests for this pool.
954 static SVCMASTERXPRT
*
955 svc_xprt_qget(SVCPOOL
*pool
)
959 mutex_enter(&pool
->p_qend_lock
);
962 * If the queue is empty return NULL.
963 * Since we do not acquire the pool's request lock which
964 * protects pool->p_qtop this is not exact check. However,
965 * this is safe - if we miss a request here svc_poll()
968 if (pool
->p_qend
== pool
->p_qtop
) {
969 mutex_exit(&pool
->p_qend_lock
);
973 /* Get a hint and move pool->p_qend */
974 xprt
= pool
->p_qend
->q_xprt
;
975 pool
->p_qend
= pool
->p_qend
->q_next
;
977 /* Skip fields deleted by svc_xprt_qdelete() */
978 } while (xprt
== NULL
);
979 mutex_exit(&pool
->p_qend_lock
);
985 * Delete all the references to a transport handle that
986 * is being destroyed from the xprt-ready queue.
987 * Deleted pointers are replaced with NULLs.
990 svc_xprt_qdelete(SVCPOOL
*pool
, SVCMASTERXPRT
*xprt
)
994 mutex_enter(&pool
->p_req_lock
);
995 for (q
= pool
->p_qend
; q
!= pool
->p_qtop
; q
= q
->q_next
) {
996 if (q
->q_xprt
== xprt
)
999 mutex_exit(&pool
->p_req_lock
);
1003 * Destructor for a master server transport handle.
1004 * - if there are no more non-detached threads linked to this transport
1005 * then, if requested, call xp_closeproc (we don't wait for detached
1006 * threads linked to this transport to complete).
1007 * - if there are no more threads linked to this
1009 * a) remove references to this transport from the xprt-ready queue
1010 * b) remove a reference to this transport from the pool's transport list
1011 * c) call a transport specific `destroy' function
1012 * d) cancel remaining thread reservations.
1014 * NOTICE: Caller must hold the transport's thread lock.
1017 svc_xprt_cleanup(SVCMASTERXPRT
*xprt
, bool_t detached
)
1019 ASSERT(MUTEX_HELD(&xprt
->xp_thread_lock
));
1020 ASSERT(xprt
->xp_wq
== NULL
);
1023 * If called from the last non-detached thread
1024 * it should call the closeproc on this transport.
1026 if (!detached
&& xprt
->xp_threads
== 0 && xprt
->xp_closeproc
) {
1027 (*(xprt
->xp_closeproc
)) (xprt
);
1030 if (xprt
->xp_threads
+ xprt
->xp_detached_threads
> 0)
1031 mutex_exit(&xprt
->xp_thread_lock
);
1033 /* Remove references to xprt from the `xprt-ready' queue */
1034 svc_xprt_qdelete(xprt
->xp_pool
, xprt
);
1036 /* Unregister xprt from the pool's transport list */
1037 svc_xprt_unregister(xprt
);
1038 svc_callout_free(xprt
);
1044 * Find a dispatch routine for a given prog/vers pair.
1045 * This function is called from svc_getreq() to search the callout
1046 * table for an entry with a matching RPC program number `prog'
1047 * and a version range that covers `vers'.
1048 * - if it finds a matching entry it returns pointer to the dispatch routine
1049 * - otherwise it returns NULL and, if `minp' or `maxp' are not NULL,
1050 * fills them with, respectively, lowest version and highest version
1051 * supported for the program `prog'
1053 static SVC_DISPATCH
*
1054 svc_callout_find(SVCXPRT
*xprt
, rpcprog_t prog
, rpcvers_t vers
,
1055 rpcvers_t
*vers_min
, rpcvers_t
*vers_max
)
1057 SVC_CALLOUT_TABLE
*sct
= xprt
->xp_sct
;
1060 *vers_min
= ~(rpcvers_t
)0;
1063 for (i
= 0; i
< sct
->sct_size
; i
++) {
1064 SVC_CALLOUT
*sc
= &sct
->sct_sc
[i
];
1066 if (prog
== sc
->sc_prog
) {
1067 if (vers
>= sc
->sc_versmin
&& vers
<= sc
->sc_versmax
)
1068 return (sc
->sc_dispatch
);
1070 if (*vers_max
< sc
->sc_versmax
)
1071 *vers_max
= sc
->sc_versmax
;
1072 if (*vers_min
> sc
->sc_versmin
)
1073 *vers_min
= sc
->sc_versmin
;
1081 * Optionally free callout table allocated for this transport by
1082 * the service provider.
1085 svc_callout_free(SVCMASTERXPRT
*xprt
)
1087 SVC_CALLOUT_TABLE
*sct
= xprt
->xp_sct
;
1089 if (sct
->sct_free
) {
1090 kmem_free(sct
->sct_sc
, sct
->sct_size
* sizeof (SVC_CALLOUT
));
1091 kmem_free(sct
, sizeof (SVC_CALLOUT_TABLE
));
1096 * Send a reply to an RPC request
1098 * PSARC 2003/523 Contract Private Interface
1100 * Changes must be reviewed by Solaris File Sharing
1101 * Changes must be communicated to contract-2003-523@sun.com
1104 svc_sendreply(const SVCXPRT
*clone_xprt
, const xdrproc_t xdr_results
,
1105 const caddr_t xdr_location
)
1107 struct rpc_msg rply
;
1109 rply
.rm_direction
= REPLY
;
1110 rply
.rm_reply
.rp_stat
= MSG_ACCEPTED
;
1111 rply
.acpted_rply
.ar_verf
= clone_xprt
->xp_verf
;
1112 rply
.acpted_rply
.ar_stat
= SUCCESS
;
1113 rply
.acpted_rply
.ar_results
.where
= xdr_location
;
1114 rply
.acpted_rply
.ar_results
.proc
= xdr_results
;
1116 return (SVC_REPLY((SVCXPRT
*)clone_xprt
, &rply
));
1120 * No procedure error reply
1122 * PSARC 2003/523 Contract Private Interface
1124 * Changes must be reviewed by Solaris File Sharing
1125 * Changes must be communicated to contract-2003-523@sun.com
1128 svcerr_noproc(const SVCXPRT
*clone_xprt
)
1130 struct rpc_msg rply
;
1132 rply
.rm_direction
= REPLY
;
1133 rply
.rm_reply
.rp_stat
= MSG_ACCEPTED
;
1134 rply
.acpted_rply
.ar_verf
= clone_xprt
->xp_verf
;
1135 rply
.acpted_rply
.ar_stat
= PROC_UNAVAIL
;
1136 SVC_FREERES((SVCXPRT
*)clone_xprt
);
1137 SVC_REPLY((SVCXPRT
*)clone_xprt
, &rply
);
1141 * Can't decode arguments error reply
1143 * PSARC 2003/523 Contract Private Interface
1145 * Changes must be reviewed by Solaris File Sharing
1146 * Changes must be communicated to contract-2003-523@sun.com
1149 svcerr_decode(const SVCXPRT
*clone_xprt
)
1151 struct rpc_msg rply
;
1153 rply
.rm_direction
= REPLY
;
1154 rply
.rm_reply
.rp_stat
= MSG_ACCEPTED
;
1155 rply
.acpted_rply
.ar_verf
= clone_xprt
->xp_verf
;
1156 rply
.acpted_rply
.ar_stat
= GARBAGE_ARGS
;
1157 SVC_FREERES((SVCXPRT
*)clone_xprt
);
1158 SVC_REPLY((SVCXPRT
*)clone_xprt
, &rply
);
1165 svcerr_systemerr(const SVCXPRT
*clone_xprt
)
1167 struct rpc_msg rply
;
1169 rply
.rm_direction
= REPLY
;
1170 rply
.rm_reply
.rp_stat
= MSG_ACCEPTED
;
1171 rply
.acpted_rply
.ar_verf
= clone_xprt
->xp_verf
;
1172 rply
.acpted_rply
.ar_stat
= SYSTEM_ERR
;
1173 SVC_FREERES((SVCXPRT
*)clone_xprt
);
1174 SVC_REPLY((SVCXPRT
*)clone_xprt
, &rply
);
1178 * Authentication error reply
1181 svcerr_auth(const SVCXPRT
*clone_xprt
, const enum auth_stat why
)
1183 struct rpc_msg rply
;
1185 rply
.rm_direction
= REPLY
;
1186 rply
.rm_reply
.rp_stat
= MSG_DENIED
;
1187 rply
.rjcted_rply
.rj_stat
= AUTH_ERROR
;
1188 rply
.rjcted_rply
.rj_why
= why
;
1189 SVC_FREERES((SVCXPRT
*)clone_xprt
);
1190 SVC_REPLY((SVCXPRT
*)clone_xprt
, &rply
);
1194 * Authentication too weak error reply
1197 svcerr_weakauth(const SVCXPRT
*clone_xprt
)
1199 svcerr_auth((SVCXPRT
*)clone_xprt
, AUTH_TOOWEAK
);
1203 * Authentication error; bad credentials
1206 svcerr_badcred(const SVCXPRT
*clone_xprt
)
1208 struct rpc_msg rply
;
1210 rply
.rm_direction
= REPLY
;
1211 rply
.rm_reply
.rp_stat
= MSG_DENIED
;
1212 rply
.rjcted_rply
.rj_stat
= AUTH_ERROR
;
1213 rply
.rjcted_rply
.rj_why
= AUTH_BADCRED
;
1214 SVC_FREERES((SVCXPRT
*)clone_xprt
);
1215 SVC_REPLY((SVCXPRT
*)clone_xprt
, &rply
);
1219 * Program unavailable error reply
1221 * PSARC 2003/523 Contract Private Interface
1223 * Changes must be reviewed by Solaris File Sharing
1224 * Changes must be communicated to contract-2003-523@sun.com
1227 svcerr_noprog(const SVCXPRT
*clone_xprt
)
1229 struct rpc_msg rply
;
1231 rply
.rm_direction
= REPLY
;
1232 rply
.rm_reply
.rp_stat
= MSG_ACCEPTED
;
1233 rply
.acpted_rply
.ar_verf
= clone_xprt
->xp_verf
;
1234 rply
.acpted_rply
.ar_stat
= PROG_UNAVAIL
;
1235 SVC_FREERES((SVCXPRT
*)clone_xprt
);
1236 SVC_REPLY((SVCXPRT
*)clone_xprt
, &rply
);
1240 * Program version mismatch error reply
1242 * PSARC 2003/523 Contract Private Interface
1244 * Changes must be reviewed by Solaris File Sharing
1245 * Changes must be communicated to contract-2003-523@sun.com
1248 svcerr_progvers(const SVCXPRT
*clone_xprt
,
1249 const rpcvers_t low_vers
, const rpcvers_t high_vers
)
1251 struct rpc_msg rply
;
1253 rply
.rm_direction
= REPLY
;
1254 rply
.rm_reply
.rp_stat
= MSG_ACCEPTED
;
1255 rply
.acpted_rply
.ar_verf
= clone_xprt
->xp_verf
;
1256 rply
.acpted_rply
.ar_stat
= PROG_MISMATCH
;
1257 rply
.acpted_rply
.ar_vers
.low
= low_vers
;
1258 rply
.acpted_rply
.ar_vers
.high
= high_vers
;
1259 SVC_FREERES((SVCXPRT
*)clone_xprt
);
1260 SVC_REPLY((SVCXPRT
*)clone_xprt
, &rply
);
1264 * Get server side input from some transport.
1266 * Statement of authentication parameters management:
1267 * This function owns and manages all authentication parameters, specifically
1268 * the "raw" parameters (msg.rm_call.cb_cred and msg.rm_call.cb_verf) and
1269 * the "cooked" credentials (rqst->rq_clntcred).
1270 * However, this function does not know the structure of the cooked
1271 * credentials, so it make the following assumptions:
1272 * a) the structure is contiguous (no pointers), and
1273 * b) the cred structure size does not exceed RQCRED_SIZE bytes.
1274 * In all events, all three parameters are freed upon exit from this routine.
1275 * The storage is trivially managed on the call stack in user land, but
1276 * is malloced in kernel land.
1278 * Note: the xprt's xp_svc_lock is not held while the service's dispatch
1279 * routine is running. If we decide to implement svc_unregister(), we'll
1280 * need to decide whether it's okay for a thread to unregister a service
1281 * while a request is being processed. If we decide that this is a
1282 * problem, we can probably use some sort of reference counting scheme to
1283 * keep the callout entry from going away until the request has completed.
1287 SVCXPRT
*clone_xprt
, /* clone transport handle */
1292 char *cred_area
; /* too big to allocate on call stack */
1294 TRACE_0(TR_FAC_KRPC
, TR_SVC_GETREQ_START
,
1295 "svc_getreq_start:");
1297 ASSERT(clone_xprt
->xp_master
!= NULL
);
1299 * Firstly, allocate the authentication parameters' storage
1301 mutex_enter(&rqcred_lock
);
1303 cred_area
= rqcred_head
;
1305 /* LINTED pointer alignment */
1306 rqcred_head
= *(caddr_t
*)rqcred_head
;
1307 mutex_exit(&rqcred_lock
);
1309 mutex_exit(&rqcred_lock
);
1310 cred_area
= kmem_alloc(2 * MAX_AUTH_BYTES
+ RQCRED_SIZE
,
1313 msg
.rm_call
.cb_cred
.oa_base
= cred_area
;
1314 msg
.rm_call
.cb_verf
.oa_base
= &(cred_area
[MAX_AUTH_BYTES
]);
1315 r
.rq_clntcred
= &(cred_area
[2 * MAX_AUTH_BYTES
]);
1318 * Now receive a message from the transport.
1320 if (SVC_RECV(clone_xprt
, mp
, &msg
)) {
1321 void (*dispatchroutine
) (struct svc_req
*, SVCXPRT
*);
1328 * Find the registered program and call its
1331 r
.rq_xprt
= clone_xprt
;
1332 r
.rq_prog
= msg
.rm_call
.cb_prog
;
1333 r
.rq_vers
= msg
.rm_call
.cb_vers
;
1334 r
.rq_proc
= msg
.rm_call
.cb_proc
;
1335 r
.rq_cred
= msg
.rm_call
.cb_cred
;
1338 * First authenticate the message.
1340 TRACE_0(TR_FAC_KRPC
, TR_SVC_GETREQ_AUTH_START
,
1341 "svc_getreq_auth_start:");
1342 if ((why
= sec_svc_msg(&r
, &msg
, &no_dispatch
)) != AUTH_OK
) {
1343 TRACE_1(TR_FAC_KRPC
, TR_SVC_GETREQ_AUTH_END
,
1344 "svc_getreq_auth_end:(%S)", "failed");
1345 svcerr_auth(clone_xprt
, why
);
1347 * Free the arguments.
1349 (void) SVC_FREEARGS(clone_xprt
, NULL
, NULL
);
1350 } else if (no_dispatch
) {
1352 * XXX - when bug id 4053736 is done, remove
1353 * the SVC_FREEARGS() call.
1355 (void) SVC_FREEARGS(clone_xprt
, NULL
, NULL
);
1357 TRACE_1(TR_FAC_KRPC
, TR_SVC_GETREQ_AUTH_END
,
1358 "svc_getreq_auth_end:(%S)", "good");
1360 dispatchroutine
= svc_callout_find(clone_xprt
,
1361 r
.rq_prog
, r
.rq_vers
, &vers_min
, &vers_max
);
1363 if (dispatchroutine
) {
1364 (*dispatchroutine
) (&r
, clone_xprt
);
1367 * If we got here, the program or version
1370 if (vers_max
== 0 ||
1371 version_keepquiet(clone_xprt
))
1372 svcerr_noprog(clone_xprt
);
1374 svcerr_progvers(clone_xprt
, vers_min
,
1378 * Free the arguments. For successful calls
1379 * this is done by the dispatch routine.
1381 (void) SVC_FREEARGS(clone_xprt
, NULL
, NULL
);
1382 /* Fall through to ... */
1385 * Call cleanup procedure for RPCSEC_GSS.
1386 * This is a hack since there is currently no
1387 * op, such as SVC_CLEANAUTH. rpc_gss_cleanup
1388 * should only be called for a non null proc.
1389 * Null procs in RPC GSS are overloaded to
1390 * provide context setup and control. The main
1391 * purpose of rpc_gss_cleanup is to decrement the
1392 * reference count associated with the cached
1393 * GSS security context. We should never get here
1394 * for an RPCSEC_GSS null proc since *no_dispatch
1395 * would have been set to true from sec_svc_msg above.
1397 if (r
.rq_cred
.oa_flavor
== RPCSEC_GSS
)
1398 rpc_gss_cleanup(clone_xprt
);
1403 * Free authentication parameters' storage
1405 mutex_enter(&rqcred_lock
);
1406 /* LINTED pointer alignment */
1407 *(caddr_t
*)cred_area
= rqcred_head
;
1408 rqcred_head
= cred_area
;
1409 mutex_exit(&rqcred_lock
);
1413 * Allocate new clone transport handle.
1416 svc_clone_init(void)
1418 SVCXPRT
*clone_xprt
;
1420 clone_xprt
= kmem_zalloc(sizeof (SVCXPRT
), KM_SLEEP
);
1421 clone_xprt
->xp_cred
= crget();
1422 return (clone_xprt
);
1426 * Free memory allocated by svc_clone_init.
1429 svc_clone_free(SVCXPRT
*clone_xprt
)
1431 /* Fre credentials from crget() */
1432 if (clone_xprt
->xp_cred
)
1433 crfree(clone_xprt
->xp_cred
);
1434 kmem_free(clone_xprt
, sizeof (SVCXPRT
));
1438 * Link a per-thread clone transport handle to a master
1439 * - increment a thread reference count on the master
1440 * - copy some of the master's fields to the clone
1441 * - call a transport specific clone routine.
1444 svc_clone_link(SVCMASTERXPRT
*xprt
, SVCXPRT
*clone_xprt
, SVCXPRT
*clone_xprt2
)
1446 cred_t
*cred
= clone_xprt
->xp_cred
;
1451 * Bump up master's thread count.
1452 * Linking a per-thread clone transport handle to a master
1453 * associates a service thread with the master.
1455 mutex_enter(&xprt
->xp_thread_lock
);
1457 mutex_exit(&xprt
->xp_thread_lock
);
1459 /* Clear everything */
1460 bzero(clone_xprt
, sizeof (SVCXPRT
));
1462 /* Set pointer to the master transport stucture */
1463 clone_xprt
->xp_master
= xprt
;
1465 /* Structure copy of all the common fields */
1466 clone_xprt
->xp_xpc
= xprt
->xp_xpc
;
1468 /* Restore per-thread fields (xp_cred) */
1469 clone_xprt
->xp_cred
= cred
;
1472 SVC_CLONE_XPRT(clone_xprt2
, clone_xprt
);
1476 * Unlink a non-detached clone transport handle from a master
1477 * - decrement a thread reference count on the master
1478 * - if the transport is closing (xp_wq is NULL) call svc_xprt_cleanup();
1479 * if this is the last non-detached/absolute thread on this transport
1480 * then it will close/destroy the transport
1481 * - call transport specific function to destroy the clone handle
1482 * - clear xp_master to avoid recursion.
1485 svc_clone_unlink(SVCXPRT
*clone_xprt
)
1487 SVCMASTERXPRT
*xprt
= clone_xprt
->xp_master
;
1489 /* This cannot be a detached thread */
1490 ASSERT(!clone_xprt
->xp_detached
);
1491 ASSERT(xprt
->xp_threads
> 0);
1493 /* Decrement a reference count on the transport */
1494 mutex_enter(&xprt
->xp_thread_lock
);
1497 /* svc_xprt_cleanup() unlocks xp_thread_lock or destroys xprt */
1499 mutex_exit(&xprt
->xp_thread_lock
);
1501 svc_xprt_cleanup(xprt
, FALSE
);
1503 /* Call a transport specific clone `destroy' function */
1504 SVC_CLONE_DESTROY(clone_xprt
);
1506 /* Clear xp_master */
1507 clone_xprt
->xp_master
= NULL
;
1511 * Unlink a detached clone transport handle from a master
1512 * - decrement the thread count on the master
1513 * - if the transport is closing (xp_wq is NULL) call svc_xprt_cleanup();
1514 * if this is the last thread on this transport then it will destroy
1516 * - call a transport specific function to destroy the clone handle
1517 * - clear xp_master to avoid recursion.
1520 svc_clone_unlinkdetached(SVCXPRT
*clone_xprt
)
1522 SVCMASTERXPRT
*xprt
= clone_xprt
->xp_master
;
1524 /* This must be a detached thread */
1525 ASSERT(clone_xprt
->xp_detached
);
1526 ASSERT(xprt
->xp_detached_threads
> 0);
1527 ASSERT(xprt
->xp_threads
+ xprt
->xp_detached_threads
> 0);
1529 /* Grab xprt->xp_thread_lock and decrement link counts */
1530 mutex_enter(&xprt
->xp_thread_lock
);
1531 xprt
->xp_detached_threads
--;
1533 /* svc_xprt_cleanup() unlocks xp_thread_lock or destroys xprt */
1535 mutex_exit(&xprt
->xp_thread_lock
);
1537 svc_xprt_cleanup(xprt
, TRUE
);
1539 /* Call transport specific clone `destroy' function */
1540 SVC_CLONE_DESTROY(clone_xprt
);
1542 /* Clear xp_master */
1543 clone_xprt
->xp_master
= NULL
;
1547 * Try to exit a non-detached service thread
1548 * - check if there are enough threads left
1549 * - if this thread (ie its clone transport handle) are linked
1550 * to a master transport then unlink it
1551 * - free the clone structure
1552 * - return to userland for thread exit
1554 * If this is the last non-detached or the last thread on this
1555 * transport then the call to svc_clone_unlink() will, respectively,
1556 * close and/or destroy the transport.
1559 svc_thread_exit(SVCPOOL
*pool
, SVCXPRT
*clone_xprt
)
1561 if (clone_xprt
->xp_master
)
1562 svc_clone_unlink(clone_xprt
);
1563 svc_clone_free(clone_xprt
);
1565 mutex_enter(&pool
->p_thread_lock
);
1567 if (pool
->p_closing
&& svc_pool_tryexit(pool
))
1568 /* return - thread exit will be handled at user level */
1570 mutex_exit(&pool
->p_thread_lock
);
1572 /* return - thread exit will be handled at user level */
1576 * Exit a detached service thread that returned to svc_run
1577 * - decrement the `detached thread' count for the pool
1578 * - unlink the detached clone transport handle from the master
1579 * - free the clone structure
1580 * - return to userland for thread exit
1582 * If this is the last thread on this transport then the call
1583 * to svc_clone_unlinkdetached() will destroy the transport.
1586 svc_thread_exitdetached(SVCPOOL
*pool
, SVCXPRT
*clone_xprt
)
1588 /* This must be a detached thread */
1589 ASSERT(clone_xprt
->xp_master
);
1590 ASSERT(clone_xprt
->xp_detached
);
1591 ASSERT(!MUTEX_HELD(&pool
->p_thread_lock
));
1593 svc_clone_unlinkdetached(clone_xprt
);
1594 svc_clone_free(clone_xprt
);
1596 mutex_enter(&pool
->p_thread_lock
);
1598 ASSERT(pool
->p_reserved_threads
>= 0);
1599 ASSERT(pool
->p_detached_threads
> 0);
1601 pool
->p_detached_threads
--;
1602 if (pool
->p_closing
&& svc_pool_tryexit(pool
))
1603 /* return - thread exit will be handled at user level */
1605 mutex_exit(&pool
->p_thread_lock
);
1607 /* return - thread exit will be handled at user level */
1611 * PSARC 2003/523 Contract Private Interface
1613 * Changes must be reviewed by Solaris File Sharing
1614 * Changes must be communicated to contract-2003-523@sun.com
1621 struct svc_globals
*svc
;
1623 svc
= zone_getspecific(svc_zone_key
, curproc
->p_zone
);
1624 mutex_enter(&svc
->svc_plock
);
1625 pool
= svc_pool_find(svc
, id
);
1626 mutex_exit(&svc
->svc_plock
);
1631 mutex_enter(&pool
->p_user_lock
);
1633 /* Check if there's already a user thread waiting on this pool */
1634 if (pool
->p_user_waiting
) {
1635 mutex_exit(&pool
->p_user_lock
);
1639 pool
->p_user_waiting
= TRUE
;
1641 /* Go to sleep, waiting for the signaled flag. */
1642 while (!pool
->p_signal_create_thread
&& !pool
->p_user_exit
) {
1643 if (cv_wait_sig(&pool
->p_user_cv
, &pool
->p_user_lock
) == 0) {
1644 /* Interrupted, return to handle exit or signal */
1645 pool
->p_user_waiting
= FALSE
;
1646 pool
->p_signal_create_thread
= FALSE
;
1647 mutex_exit(&pool
->p_user_lock
);
1650 * Thread has been interrupted and therefore
1651 * the service daemon is leaving as well so
1652 * let's go ahead and remove the service
1653 * pool at this time.
1655 mutex_enter(&svc
->svc_plock
);
1656 svc_pool_unregister(svc
, pool
);
1657 mutex_exit(&svc
->svc_plock
);
1663 pool
->p_signal_create_thread
= FALSE
;
1664 pool
->p_user_waiting
= FALSE
;
1667 * About to exit the service pool. Set return value
1668 * to let the userland code know our intent. Signal
1669 * svc_thread_creator() so that it can clean up the
1672 if (pool
->p_user_exit
) {
1674 cv_signal(&pool
->p_user_cv
);
1677 mutex_exit(&pool
->p_user_lock
);
1679 /* Return to userland with error code, for possible thread creation. */
1684 * `Service threads' creator thread.
1685 * The creator thread waits for a signal to create new thread.
1688 svc_thread_creator(SVCPOOL
*pool
)
1690 callb_cpr_t cpr_info
; /* CPR info for the creator thread */
1692 CALLB_CPR_INIT(&cpr_info
, &pool
->p_creator_lock
, callb_generic_cpr
,
1693 "svc_thread_creator");
1696 mutex_enter(&pool
->p_creator_lock
);
1698 /* Check if someone set the exit flag */
1699 if (pool
->p_creator_exit
)
1702 /* Clear the `signaled' flag and go asleep */
1703 pool
->p_creator_signaled
= FALSE
;
1705 CALLB_CPR_SAFE_BEGIN(&cpr_info
);
1706 cv_wait(&pool
->p_creator_cv
, &pool
->p_creator_lock
);
1707 CALLB_CPR_SAFE_END(&cpr_info
, &pool
->p_creator_lock
);
1709 /* Check if someone signaled to exit */
1710 if (pool
->p_creator_exit
)
1713 mutex_exit(&pool
->p_creator_lock
);
1715 mutex_enter(&pool
->p_thread_lock
);
1718 * When the pool is in closing state and all the transports
1719 * are gone the creator should not create any new threads.
1721 if (pool
->p_closing
) {
1722 rw_enter(&pool
->p_lrwlock
, RW_READER
);
1723 if (pool
->p_lcount
== 0) {
1724 rw_exit(&pool
->p_lrwlock
);
1725 mutex_exit(&pool
->p_thread_lock
);
1728 rw_exit(&pool
->p_lrwlock
);
1732 * Create a new service thread now.
1734 ASSERT(pool
->p_reserved_threads
>= 0);
1735 ASSERT(pool
->p_detached_threads
>= 0);
1737 if (pool
->p_threads
+ pool
->p_detached_threads
<
1738 pool
->p_maxthreads
) {
1740 * Signal the service pool wait thread
1741 * only if it hasn't already been signaled.
1743 mutex_enter(&pool
->p_user_lock
);
1744 if (pool
->p_signal_create_thread
== FALSE
) {
1745 pool
->p_signal_create_thread
= TRUE
;
1746 cv_signal(&pool
->p_user_cv
);
1748 mutex_exit(&pool
->p_user_lock
);
1752 mutex_exit(&pool
->p_thread_lock
);
1756 * Pool is closed. Cleanup and exit.
1759 /* Signal userland creator thread that it can stop now. */
1760 mutex_enter(&pool
->p_user_lock
);
1761 pool
->p_user_exit
= TRUE
;
1762 cv_broadcast(&pool
->p_user_cv
);
1763 mutex_exit(&pool
->p_user_lock
);
1765 /* Wait for svc_wait() to be done with the pool */
1766 mutex_enter(&pool
->p_user_lock
);
1767 while (pool
->p_user_waiting
) {
1768 CALLB_CPR_SAFE_BEGIN(&cpr_info
);
1769 cv_wait(&pool
->p_user_cv
, &pool
->p_user_lock
);
1770 CALLB_CPR_SAFE_END(&cpr_info
, &pool
->p_creator_lock
);
1772 mutex_exit(&pool
->p_user_lock
);
1774 CALLB_CPR_EXIT(&cpr_info
);
1775 svc_pool_cleanup(pool
);
1780 * If the creator thread is idle signal it to create
1781 * a new service thread.
1784 svc_creator_signal(SVCPOOL
*pool
)
1786 mutex_enter(&pool
->p_creator_lock
);
1787 if (pool
->p_creator_signaled
== FALSE
) {
1788 pool
->p_creator_signaled
= TRUE
;
1789 cv_signal(&pool
->p_creator_cv
);
1791 mutex_exit(&pool
->p_creator_lock
);
1795 * Notify the creator thread to clean up and exit.
1798 svc_creator_signalexit(SVCPOOL
*pool
)
1800 mutex_enter(&pool
->p_creator_lock
);
1801 pool
->p_creator_exit
= TRUE
;
1802 cv_signal(&pool
->p_creator_cv
);
1803 mutex_exit(&pool
->p_creator_lock
);
1807 * Polling part of the svc_run().
1808 * - search for a transport with a pending request
1809 * - when one is found then latch the request lock and return to svc_run()
1810 * - if there is no request go asleep and wait for a signal
1811 * - handle two exceptions:
1812 * a) current transport is closing
1813 * b) timeout waiting for a new request
1814 * in both cases return to svc_run()
1816 static SVCMASTERXPRT
*
1817 svc_poll(SVCPOOL
*pool
, SVCMASTERXPRT
*xprt
, SVCXPRT
*clone_xprt
)
1820 * Main loop iterates until
1821 * a) we find a pending request,
1822 * b) detect that the current transport is closing
1823 * c) time out waiting for a new request.
1826 SVCMASTERXPRT
*next
;
1831 * Check if there is a pending request on the current
1832 * transport handle so that we can avoid cloning.
1833 * If so then decrement the `pending-request' count for
1834 * the pool and return to svc_run().
1836 * We need to prevent a potential starvation. When
1837 * a selected transport has all pending requests coming in
1838 * all the time then the service threads will never switch to
1839 * another transport. With a limited number of service
1840 * threads some transports may be never serviced.
1841 * To prevent such a scenario we pick up at most
1842 * pool->p_max_same_xprt requests from the same transport
1843 * and then take a hint from the xprt-ready queue or walk
1844 * the transport list.
1846 if (xprt
&& xprt
->xp_req_head
&& (!pool
->p_qoverflow
||
1847 clone_xprt
->xp_same_xprt
++ < pool
->p_max_same_xprt
)) {
1848 mutex_enter(&xprt
->xp_req_lock
);
1849 if (xprt
->xp_req_head
)
1851 mutex_exit(&xprt
->xp_req_lock
);
1853 clone_xprt
->xp_same_xprt
= 0;
1857 * If there is no request on the current transport try to
1858 * find another transport with a pending request.
1860 mutex_enter(&pool
->p_req_lock
);
1862 mutex_exit(&pool
->p_req_lock
);
1865 * Make sure that transports will not be destroyed just
1866 * while we are checking them.
1868 rw_enter(&pool
->p_lrwlock
, RW_READER
);
1871 SVCMASTERXPRT
*hint
;
1874 * Get the next transport from the xprt-ready queue.
1875 * This is a hint. There is no guarantee that the
1876 * transport still has a pending request since it
1877 * could be picked up by another thread in step 1.
1879 * If the transport has a pending request then keep
1880 * it locked. Decrement the `pending-requests' for
1881 * the pool and `walking-threads' counts, and return
1884 hint
= svc_xprt_qget(pool
);
1886 if (hint
&& hint
->xp_req_head
) {
1887 mutex_enter(&hint
->xp_req_lock
);
1888 if (hint
->xp_req_head
) {
1889 rw_exit(&pool
->p_lrwlock
);
1891 mutex_enter(&pool
->p_req_lock
);
1893 mutex_exit(&pool
->p_req_lock
);
1897 mutex_exit(&hint
->xp_req_lock
);
1901 * If there was no hint in the xprt-ready queue then
1902 * - if there is less pending requests than polling
1904 * - otherwise check if there was an overflow in the
1905 * xprt-ready queue; if so, then we need to break
1909 if (pool
->p_reqs
< pool
->p_walkers
) {
1910 mutex_enter(&pool
->p_req_lock
);
1911 if (pool
->p_reqs
< pool
->p_walkers
)
1913 mutex_exit(&pool
->p_req_lock
);
1915 if (pool
->p_qoverflow
) {
1922 * If there was an overflow in the xprt-ready queue then we
1923 * need to switch to the `drain' mode, i.e. walk through the
1924 * pool's transport list and search for a transport with a
1925 * pending request. If we manage to drain all the pending
1926 * requests then we can clear the overflow flag. This will
1927 * switch svc_poll() back to taking hints from the xprt-ready
1928 * queue (which is generally more efficient).
1930 * If there are no registered transports simply go asleep.
1932 if (xprt
== NULL
&& pool
->p_lhead
== NULL
) {
1933 mutex_enter(&pool
->p_req_lock
);
1938 * `Walk' through the pool's list of master server
1939 * transport handles. Continue to loop until there are less
1940 * looping threads then pending requests.
1942 next
= xprt
? xprt
->xp_next
: pool
->p_lhead
;
1946 * Check if there is a request on this transport.
1948 * Since blocking on a locked mutex is very expensive
1949 * check for a request without a lock first. If we miss
1950 * a request that is just being delivered but this will
1951 * cost at most one full walk through the list.
1953 if (next
->xp_req_head
) {
1955 * Check again, now with a lock.
1957 mutex_enter(&next
->xp_req_lock
);
1958 if (next
->xp_req_head
) {
1959 rw_exit(&pool
->p_lrwlock
);
1961 mutex_enter(&pool
->p_req_lock
);
1963 mutex_exit(&pool
->p_req_lock
);
1967 mutex_exit(&next
->xp_req_lock
);
1971 * Continue to `walk' through the pool's
1972 * transport list until there is less requests
1973 * than walkers. Check this condition without
1974 * a lock first to avoid contention on a mutex.
1976 if (pool
->p_reqs
< pool
->p_walkers
) {
1977 /* Check again, now with the lock. */
1978 mutex_enter(&pool
->p_req_lock
);
1979 if (pool
->p_reqs
< pool
->p_walkers
)
1980 break; /* goto sleep */
1981 mutex_exit(&pool
->p_req_lock
);
1984 next
= next
->xp_next
;
1989 * No work to do. Stop the `walk' and go asleep.
1990 * Decrement the `walking-threads' count for the pool.
1993 rw_exit(&pool
->p_lrwlock
);
1996 * Count us as asleep, mark this thread as safe
1997 * for suspend and wait for a request.
2000 timeleft
= cv_reltimedwait_sig(&pool
->p_req_cv
,
2001 &pool
->p_req_lock
, pool
->p_timeout
, TR_CLOCK_TICK
);
2004 * If the drowsy flag is on this means that
2005 * someone has signaled a wakeup. In such a case
2006 * the `asleep-threads' count has already updated
2007 * so just clear the flag.
2009 * If the drowsy flag is off then we need to update
2010 * the `asleep-threads' count.
2012 if (pool
->p_drowsy
) {
2013 pool
->p_drowsy
= FALSE
;
2015 * If the thread is here because it timedout,
2016 * instead of returning SVC_ETIMEDOUT, it is
2017 * time to do some more work.
2024 mutex_exit(&pool
->p_req_lock
);
2027 * If we received a signal while waiting for a
2028 * request, inform svc_run(), so that we can return
2029 * to user level and exit.
2035 * If the current transport is gone then notify
2036 * svc_run() to unlink from it.
2038 if (xprt
&& xprt
->xp_wq
== NULL
)
2039 return (SVC_EXPRTGONE
);
2042 * If we have timed out waiting for a request inform
2043 * svc_run() that we probably don't need this thread.
2046 return (SVC_ETIMEDOUT
);
2051 * calculate memory space used by message
2054 svc_msgsize(mblk_t
*mp
)
2058 for (; mp
; mp
= mp
->b_cont
)
2059 count
+= MBLKSIZE(mp
);
2065 * svc_flowcontrol() attempts to turn the flow control on or off for the
2068 * On input the xprt->xp_full determines whether the flow control is currently
2069 * off (FALSE) or on (TRUE). If it is off we do tests to see whether we should
2070 * turn it on, and vice versa.
2072 * There are two conditions considered for the flow control. Both conditions
2073 * have the low and the high watermark. Once the high watermark is reached in
2074 * EITHER condition the flow control is turned on. For turning the flow
2075 * control off BOTH conditions must be below the low watermark.
2077 * Condition #1 - Number of requests queued:
2079 * The max number of threads working on the pool is roughly pool->p_maxthreads.
2080 * Every thread could handle up to pool->p_max_same_xprt requests from one
2081 * transport before it moves to another transport. See svc_poll() for details.
2082 * In case all threads in the pool are working on a transport they will handle
2083 * no more than enough_reqs (pool->p_maxthreads * pool->p_max_same_xprt)
2084 * requests in one shot from that transport. We are turning the flow control
2085 * on once the high watermark is reached for a transport so that the underlying
2086 * queue knows the rate of incoming requests is higher than we are able to
2089 * The high watermark: 2 * enough_reqs
2090 * The low watermark: enough_reqs
2092 * Condition #2 - Length of the data payload for the queued messages/requests:
2094 * We want to prevent a particular pool exhausting the memory, so once the
2095 * total length of queued requests for the whole pool reaches the high
2096 * watermark we start to turn on the flow control for significant memory
2097 * consumers (individual transports). To keep the implementation simple
2098 * enough, this condition is not exact, because we count only the data part of
2099 * the queued requests and we ignore the overhead. For our purposes this
2100 * should be enough. We should also consider that up to pool->p_maxthreads
2101 * threads for the pool might work on large requests (this is not counted for
2102 * this condition). We need to leave some space for rest of the system and for
2103 * other big memory consumers (like ZFS). Also, after the flow control is
2104 * turned on (on cots transports) we can start to accumulate a few megabytes in
2105 * queues for each transport.
2107 * Usually, the big memory consumers are NFS WRITE requests, so we do not
2108 * expect to see this condition met for other than NFS pools.
2110 * The high watermark: 1/5 of available memory
2111 * The low watermark: 1/6 of available memory
2113 * Once the high watermark is reached we turn the flow control on only for
2114 * transports exceeding a per-transport memory limit. The per-transport
2115 * fraction of memory is calculated as:
2117 * the high watermark / number of transports
2119 * For transports with less than the per-transport fraction of memory consumed,
2120 * the flow control is not turned on, so they are not blocked by a few "hungry"
2121 * transports. Because of this, the total memory consumption for the
2122 * particular pool might grow up to 2 * the high watermark.
2124 * The individual transports are unblocked once their consumption is below:
2126 * per-transport fraction of memory / 2
2128 * or once the total memory consumption for the whole pool falls below the low
2133 svc_flowcontrol(SVCMASTERXPRT
*xprt
)
2135 SVCPOOL
*pool
= xprt
->xp_pool
;
2136 size_t totalmem
= ptob(physmem
);
2137 int enough_reqs
= pool
->p_maxthreads
* pool
->p_max_same_xprt
;
2139 ASSERT(MUTEX_HELD(&xprt
->xp_req_lock
));
2141 /* Should we turn the flow control on? */
2142 if (xprt
->xp_full
== FALSE
) {
2143 /* Is flow control disabled? */
2144 if (svc_flowcontrol_disable
!= 0)
2147 /* Is there enough requests queued? */
2148 if (xprt
->xp_reqs
>= enough_reqs
* 2) {
2149 xprt
->xp_full
= TRUE
;
2154 * If this pool uses over 20% of memory and this transport is
2155 * significant memory consumer then we are full
2157 if (pool
->p_size
>= totalmem
/ 5 &&
2158 xprt
->xp_size
>= totalmem
/ 5 / pool
->p_lcount
)
2159 xprt
->xp_full
= TRUE
;
2164 /* We might want to turn the flow control off */
2166 /* Do we still have enough requests? */
2167 if (xprt
->xp_reqs
> enough_reqs
)
2171 * If this pool still uses over 16% of memory and this transport is
2172 * still significant memory consumer then we are still full
2174 if (pool
->p_size
>= totalmem
/ 6 &&
2175 xprt
->xp_size
>= totalmem
/ 5 / pool
->p_lcount
/ 2)
2178 /* Turn the flow control off and make sure rpcmod is notified */
2179 xprt
->xp_full
= FALSE
;
2180 xprt
->xp_enable
= TRUE
;
2184 * Main loop of the kernel RPC server
2185 * - wait for input (find a transport with a pending request).
2186 * - dequeue the request
2187 * - call a registered server routine to process the requests
2189 * There can many threads running concurrently in this loop
2190 * on the same or on different transports.
2193 svc_run(SVCPOOL
*pool
)
2195 SVCMASTERXPRT
*xprt
= NULL
; /* master transport handle */
2196 SVCXPRT
*clone_xprt
; /* clone for this thread */
2197 proc_t
*p
= ttoproc(curthread
);
2199 /* Allocate a clone transport handle for this thread */
2200 clone_xprt
= svc_clone_init();
2203 * The loop iterates until the thread becomes
2204 * idle too long or the transport is gone.
2207 SVCMASTERXPRT
*next
;
2212 TRACE_0(TR_FAC_KRPC
, TR_SVC_RUN
, "svc_run");
2215 * If the process is exiting/killed, return
2216 * immediately without processing any more
2219 if (p
->p_flag
& (SEXITING
| SKILLED
)) {
2220 svc_thread_exit(pool
, clone_xprt
);
2224 /* Find a transport with a pending request */
2225 next
= svc_poll(pool
, xprt
, clone_xprt
);
2228 * If svc_poll() finds a transport with a request
2229 * it latches xp_req_lock on it. Therefore we need
2230 * to dequeue the request and release the lock as
2233 ASSERT(next
!= NULL
&&
2234 (next
== SVC_EXPRTGONE
||
2235 next
== SVC_ETIMEDOUT
||
2236 next
== SVC_EINTR
||
2237 MUTEX_HELD(&next
->xp_req_lock
)));
2239 /* Ooops! Current transport is closing. Unlink now */
2240 if (next
== SVC_EXPRTGONE
) {
2241 svc_clone_unlink(clone_xprt
);
2246 /* Ooops! Timeout while waiting for a request. Exit */
2247 if (next
== SVC_ETIMEDOUT
) {
2248 svc_thread_exit(pool
, clone_xprt
);
2253 * Interrupted by a signal while waiting for a
2254 * request. Return to userspace and exit.
2256 if (next
== SVC_EINTR
) {
2257 svc_thread_exit(pool
, clone_xprt
);
2262 * De-queue the request and release the request lock
2263 * on this transport (latched by svc_poll()).
2265 mp
= next
->xp_req_head
;
2266 next
->xp_req_head
= mp
->b_next
;
2267 mp
->b_next
= (mblk_t
*)0;
2268 size
= svc_msgsize(mp
);
2270 mutex_enter(&pool
->p_req_lock
);
2272 if (pool
->p_reqs
== 0)
2273 pool
->p_qoverflow
= FALSE
;
2274 pool
->p_size
-= size
;
2275 mutex_exit(&pool
->p_req_lock
);
2278 next
->xp_size
-= size
;
2281 svc_flowcontrol(next
);
2283 TRACE_2(TR_FAC_KRPC
, TR_NFSFP_QUE_REQ_DEQ
,
2284 "rpc_que_req_deq:pool %p mp %p", pool
, mp
);
2285 mutex_exit(&next
->xp_req_lock
);
2288 * If this is a new request on a current transport then
2289 * the clone structure is already properly initialized.
2290 * Otherwise, if the request is on a different transport,
2291 * unlink from the current master and link to
2292 * the one we got a request on.
2296 svc_clone_unlink(clone_xprt
);
2297 svc_clone_link(next
, clone_xprt
, NULL
);
2302 * If there are more requests and req_cv hasn't
2303 * been signaled yet then wake up one more thread now.
2305 * We avoid signaling req_cv until the most recently
2306 * signaled thread wakes up and gets CPU to clear
2307 * the `drowsy' flag.
2309 if (!(pool
->p_drowsy
|| pool
->p_reqs
<= pool
->p_walkers
||
2310 pool
->p_asleep
== 0)) {
2311 mutex_enter(&pool
->p_req_lock
);
2313 if (pool
->p_drowsy
|| pool
->p_reqs
<= pool
->p_walkers
||
2314 pool
->p_asleep
== 0)
2315 mutex_exit(&pool
->p_req_lock
);
2318 pool
->p_drowsy
= TRUE
;
2320 cv_signal(&pool
->p_req_cv
);
2321 mutex_exit(&pool
->p_req_lock
);
2326 * If there are no asleep/signaled threads, we are
2327 * still below pool->p_maxthreads limit, and no thread is
2328 * currently being created then signal the creator
2329 * for one more service thread.
2331 * The asleep and drowsy checks are not protected
2332 * by a lock since it hurts performance and a wrong
2333 * decision is not essential.
2335 if (pool
->p_asleep
== 0 && !pool
->p_drowsy
&&
2336 pool
->p_threads
+ pool
->p_detached_threads
<
2338 svc_creator_signal(pool
);
2341 * Process the request.
2343 svc_getreq(clone_xprt
, mp
);
2345 /* If thread had a reservation it should have been canceled */
2346 ASSERT(!clone_xprt
->xp_reserved
);
2349 * If the clone is marked detached then exit.
2350 * The rpcmod slot has already been released
2351 * when we detached this thread.
2353 if (clone_xprt
->xp_detached
) {
2354 svc_thread_exitdetached(pool
, clone_xprt
);
2359 * Release our reference on the rpcmod
2360 * slot attached to xp_wq->q_ptr.
2362 mutex_enter(&xprt
->xp_req_lock
);
2363 enable
= xprt
->xp_enable
;
2365 xprt
->xp_enable
= FALSE
;
2366 mutex_exit(&xprt
->xp_req_lock
);
2367 (*RELE_PROC(xprt
)) (clone_xprt
->xp_wq
, NULL
, enable
);
2373 * Flush any pending requests for the queue and
2374 * free the associated mblks.
2377 svc_queueclean(queue_t
*q
)
2379 SVCMASTERXPRT
*xprt
= ((void **) q
->q_ptr
)[0];
2384 * clean up the requests
2386 mutex_enter(&xprt
->xp_req_lock
);
2387 pool
= xprt
->xp_pool
;
2388 while ((mp
= xprt
->xp_req_head
) != NULL
) {
2389 /* remove the request from the list */
2390 xprt
->xp_req_head
= mp
->b_next
;
2391 mp
->b_next
= (mblk_t
*)0;
2392 (*RELE_PROC(xprt
)) (xprt
->xp_wq
, mp
, FALSE
);
2395 mutex_enter(&pool
->p_req_lock
);
2396 pool
->p_reqs
-= xprt
->xp_reqs
;
2397 pool
->p_size
-= xprt
->xp_size
;
2398 mutex_exit(&pool
->p_req_lock
);
2402 xprt
->xp_full
= FALSE
;
2403 xprt
->xp_enable
= FALSE
;
2404 mutex_exit(&xprt
->xp_req_lock
);
2408 * This routine is called by rpcmod to inform kernel RPC that a
2409 * queue is closing. It is called after all the requests have been
2410 * picked up (that is after all the slots on the queue have
2411 * been released by kernel RPC). It is also guaranteed that no more
2412 * request will be delivered on this transport.
2414 * - clear xp_wq to mark the master server transport handle as closing
2415 * - if there are no more threads on this transport close/destroy it
2416 * - otherwise, leave the linked threads to close/destroy the transport
2420 svc_queueclose(queue_t
*q
)
2422 SVCMASTERXPRT
*xprt
= ((void **) q
->q_ptr
)[0];
2426 * If there is no master xprt associated with this stream,
2427 * then there is nothing to do. This happens regularly
2428 * with connection-oriented listening streams created by
2434 mutex_enter(&xprt
->xp_thread_lock
);
2436 ASSERT(xprt
->xp_req_head
== NULL
);
2437 ASSERT(xprt
->xp_wq
!= NULL
);
2441 if (xprt
->xp_threads
== 0) {
2442 SVCPOOL
*pool
= xprt
->xp_pool
;
2445 * svc_xprt_cleanup() destroys the transport
2446 * or releases the transport thread lock
2448 svc_xprt_cleanup(xprt
, FALSE
);
2450 mutex_enter(&pool
->p_thread_lock
);
2453 * If the pool is in closing state and this was
2454 * the last transport in the pool then signal the creator
2455 * thread to clean up and exit.
2457 if (pool
->p_closing
&& svc_pool_tryexit(pool
)) {
2460 mutex_exit(&pool
->p_thread_lock
);
2463 * There are still some threads linked to the transport. They
2464 * are very likely sleeping in svc_poll(). We could wake up
2465 * them by broadcasting on the p_req_cv condition variable, but
2466 * that might give us a performance penalty if there are too
2467 * many sleeping threads.
2469 * Instead, we do nothing here. The linked threads will unlink
2470 * themselves and destroy the transport once they are woken up
2471 * on timeout, or by new request. There is no reason to hurry
2472 * up now with the thread wake up.
2476 * NOTICE: No references to the master transport structure
2477 * beyond this point!
2479 mutex_exit(&xprt
->xp_thread_lock
);
2484 * Interrupt `request delivery' routine called from rpcmod
2485 * - put a request at the tail of the transport request queue
2486 * - insert a hint for svc_poll() into the xprt-ready queue
2487 * - increment the `pending-requests' count for the pool
2488 * - handle flow control
2489 * - wake up a thread sleeping in svc_poll() if necessary
2490 * - if all the threads are running ask the creator for a new one.
2493 svc_queuereq(queue_t
*q
, mblk_t
*mp
, bool_t flowcontrol
)
2495 SVCMASTERXPRT
*xprt
= ((void **) q
->q_ptr
)[0];
2496 SVCPOOL
*pool
= xprt
->xp_pool
;
2499 TRACE_0(TR_FAC_KRPC
, TR_SVC_QUEUEREQ_START
, "svc_queuereq_start");
2503 * Grab the transport's request lock and the
2504 * pool's request lock so that when we put
2505 * the request at the tail of the transport's
2506 * request queue, possibly put the request on
2507 * the xprt ready queue and increment the
2508 * pending request count it looks atomic.
2510 mutex_enter(&xprt
->xp_req_lock
);
2511 if (flowcontrol
&& xprt
->xp_full
) {
2512 mutex_exit(&xprt
->xp_req_lock
);
2516 ASSERT(xprt
->xp_full
== FALSE
);
2517 mutex_enter(&pool
->p_req_lock
);
2518 if (xprt
->xp_req_head
== NULL
)
2519 xprt
->xp_req_head
= mp
;
2521 xprt
->xp_req_tail
->b_next
= mp
;
2522 xprt
->xp_req_tail
= mp
;
2526 * Insert a hint into the xprt-ready queue, increment
2527 * counters, handle flow control, and wake up
2528 * a thread sleeping in svc_poll() if necessary.
2531 /* Insert pointer to this transport into the xprt-ready queue */
2532 svc_xprt_qput(pool
, xprt
);
2534 /* Increment counters */
2538 size
= svc_msgsize(mp
);
2539 xprt
->xp_size
+= size
;
2540 pool
->p_size
+= size
;
2542 /* Handle flow control */
2544 svc_flowcontrol(xprt
);
2546 TRACE_2(TR_FAC_KRPC
, TR_NFSFP_QUE_REQ_ENQ
,
2547 "rpc_que_req_enq:pool %p mp %p", pool
, mp
);
2550 * If there are more requests and req_cv hasn't
2551 * been signaled yet then wake up one more thread now.
2553 * We avoid signaling req_cv until the most recently
2554 * signaled thread wakes up and gets CPU to clear
2555 * the `drowsy' flag.
2557 if (pool
->p_drowsy
|| pool
->p_reqs
<= pool
->p_walkers
||
2558 pool
->p_asleep
== 0) {
2559 mutex_exit(&pool
->p_req_lock
);
2561 pool
->p_drowsy
= TRUE
;
2565 * Signal wakeup and drop the request lock.
2567 cv_signal(&pool
->p_req_cv
);
2568 mutex_exit(&pool
->p_req_lock
);
2570 mutex_exit(&xprt
->xp_req_lock
);
2574 * If there are no asleep/signaled threads, we are
2575 * still below pool->p_maxthreads limit, and no thread is
2576 * currently being created then signal the creator
2577 * for one more service thread.
2579 * The asleep and drowsy checks are not not protected
2580 * by a lock since it hurts performance and a wrong
2581 * decision is not essential.
2583 if (pool
->p_asleep
== 0 && !pool
->p_drowsy
&&
2584 pool
->p_threads
+ pool
->p_detached_threads
< pool
->p_maxthreads
)
2585 svc_creator_signal(pool
);
2587 TRACE_1(TR_FAC_KRPC
, TR_SVC_QUEUEREQ_END
,
2588 "svc_queuereq_end:(%S)", "end");
2594 * Reserve a service thread so that it can be detached later.
2595 * This reservation is required to make sure that when it tries to
2596 * detach itself the total number of detached threads does not exceed
2597 * pool->p_maxthreads - pool->p_redline (i.e. that we can have
2598 * up to pool->p_redline non-detached threads).
2600 * If the thread does not detach itself later, it should cancel the
2601 * reservation before returning to svc_run().
2603 * - check if there is room for more reserved/detached threads
2604 * - if so, then increment the `reserved threads' count for the pool
2605 * - mark the thread as reserved (setting the flag in the clone transport
2606 * handle for this thread
2607 * - returns 1 if the reservation succeeded, 0 if it failed.
2610 svc_reserve_thread(SVCXPRT
*clone_xprt
)
2612 SVCPOOL
*pool
= clone_xprt
->xp_master
->xp_pool
;
2614 /* Recursive reservations are not allowed */
2615 ASSERT(!clone_xprt
->xp_reserved
);
2616 ASSERT(!clone_xprt
->xp_detached
);
2618 /* Check pool counts if there is room for reservation */
2619 mutex_enter(&pool
->p_thread_lock
);
2620 if (pool
->p_reserved_threads
+ pool
->p_detached_threads
>=
2621 pool
->p_maxthreads
- pool
->p_redline
) {
2622 mutex_exit(&pool
->p_thread_lock
);
2625 pool
->p_reserved_threads
++;
2626 mutex_exit(&pool
->p_thread_lock
);
2628 /* Mark the thread (clone handle) as reserved */
2629 clone_xprt
->xp_reserved
= TRUE
;
2635 * Cancel a reservation for a thread.
2636 * - decrement the `reserved threads' count for the pool
2637 * - clear the flag in the clone transport handle for this thread.
2640 svc_unreserve_thread(SVCXPRT
*clone_xprt
)
2642 SVCPOOL
*pool
= clone_xprt
->xp_master
->xp_pool
;
2644 /* Thread must have a reservation */
2645 ASSERT(clone_xprt
->xp_reserved
);
2646 ASSERT(!clone_xprt
->xp_detached
);
2648 /* Decrement global count */
2649 mutex_enter(&pool
->p_thread_lock
);
2650 pool
->p_reserved_threads
--;
2651 mutex_exit(&pool
->p_thread_lock
);
2653 /* Clear reservation flag */
2654 clone_xprt
->xp_reserved
= FALSE
;
2658 * Detach a thread from its transport, so that it can block for an
2659 * extended time. Because the transport can be closed after the thread is
2660 * detached, the thread should have already sent off a reply if it was
2661 * going to send one.
2663 * - decrement `non-detached threads' count and increment `detached threads'
2664 * counts for the transport
2665 * - decrement the `non-detached threads' and `reserved threads'
2666 * counts and increment the `detached threads' count for the pool
2667 * - release the rpcmod slot
2668 * - mark the clone (thread) as detached.
2670 * No need to return a pointer to the thread's CPR information, since
2671 * the thread has a userland identity.
2673 * NOTICE: a thread must not detach itself without making a prior reservation
2674 * through svc_thread_reserve().
2677 svc_detach_thread(SVCXPRT
*clone_xprt
)
2679 SVCMASTERXPRT
*xprt
= clone_xprt
->xp_master
;
2680 SVCPOOL
*pool
= xprt
->xp_pool
;
2683 /* Thread must have a reservation */
2684 ASSERT(clone_xprt
->xp_reserved
);
2685 ASSERT(!clone_xprt
->xp_detached
);
2687 /* Bookkeeping for this transport */
2688 mutex_enter(&xprt
->xp_thread_lock
);
2690 xprt
->xp_detached_threads
++;
2691 mutex_exit(&xprt
->xp_thread_lock
);
2693 /* Bookkeeping for the pool */
2694 mutex_enter(&pool
->p_thread_lock
);
2696 pool
->p_reserved_threads
--;
2697 pool
->p_detached_threads
++;
2698 mutex_exit(&pool
->p_thread_lock
);
2700 /* Release an rpcmod slot for this request */
2701 mutex_enter(&xprt
->xp_req_lock
);
2702 enable
= xprt
->xp_enable
;
2704 xprt
->xp_enable
= FALSE
;
2705 mutex_exit(&xprt
->xp_req_lock
);
2706 (*RELE_PROC(xprt
)) (clone_xprt
->xp_wq
, NULL
, enable
);
2708 /* Mark the clone (thread) as detached */
2709 clone_xprt
->xp_reserved
= FALSE
;
2710 clone_xprt
->xp_detached
= TRUE
;
2716 * This routine is responsible for extracting RDMA plugin master XPRT,
2717 * unregister from the SVCPOOL and initiate plugin specific cleanup.
2718 * It is passed a list/group of rdma transports as records which are
2719 * active in a given registered or unregistered kRPC thread pool. Its shuts
2720 * all active rdma transports in that pool. If the thread active on the trasport
2721 * happens to be last thread for that pool, it will signal the creater thread
2722 * to cleanup the pool and destroy the xprt in svc_queueclose()
2725 rdma_stop(rdma_xprt_group_t
*rdma_xprts
)
2727 SVCMASTERXPRT
*xprt
;
2728 rdma_xprt_record_t
*curr_rec
;
2734 if (rdma_xprts
->rtg_count
== 0)
2737 rtg_count
= rdma_xprts
->rtg_count
;
2739 for (i
= 0; i
< rtg_count
; i
++) {
2740 curr_rec
= rdma_xprts
->rtg_listhead
;
2741 rdma_xprts
->rtg_listhead
= curr_rec
->rtr_next
;
2742 rdma_xprts
->rtg_count
--;
2743 curr_rec
->rtr_next
= NULL
;
2744 xprt
= curr_rec
->rtr_xprt_ptr
;
2746 svc_rdma_kstop(xprt
);
2748 mutex_enter(&xprt
->xp_req_lock
);
2749 pool
= xprt
->xp_pool
;
2750 while ((mp
= xprt
->xp_req_head
) != NULL
) {
2751 rdma_recv_data_t
*rdp
= (rdma_recv_data_t
*)mp
->b_rptr
;
2753 /* remove the request from the list */
2754 xprt
->xp_req_head
= mp
->b_next
;
2755 mp
->b_next
= (mblk_t
*)0;
2757 RDMA_BUF_FREE(rdp
->conn
, &rdp
->rpcmsg
);
2758 RDMA_REL_CONN(rdp
->conn
);
2761 mutex_enter(&pool
->p_req_lock
);
2762 pool
->p_reqs
-= xprt
->xp_reqs
;
2763 pool
->p_size
-= xprt
->xp_size
;
2764 mutex_exit(&pool
->p_req_lock
);
2767 xprt
->xp_full
= FALSE
;
2768 xprt
->xp_enable
= FALSE
;
2769 mutex_exit(&xprt
->xp_req_lock
);
2773 cmn_err(CE_NOTE
, "rdma_stop: Exited svc_queueclose\n");
2776 * Free the rdma transport record for the expunged rdma
2777 * based master transport handle.
2779 kmem_free(curr_rec
, sizeof (rdma_xprt_record_t
));
2780 if (!rdma_xprts
->rtg_listhead
)
2787 * rpc_msg_dup/rpc_msg_free
2788 * Currently only used by svc_rpcsec_gss.c but put in this file as it
2789 * may be useful to others in the future.
2790 * But future consumers should be careful cuz so far
2791 * - only tested/used for call msgs (not reply)
2792 * - only tested/used with call verf oa_length==0
2795 rpc_msg_dup(struct rpc_msg
*src
)
2797 struct rpc_msg
*dst
;
2798 struct opaque_auth oa_src
, oa_dst
;
2800 dst
= kmem_alloc(sizeof (*dst
), KM_SLEEP
);
2802 dst
->rm_xid
= src
->rm_xid
;
2803 dst
->rm_direction
= src
->rm_direction
;
2805 dst
->rm_call
.cb_rpcvers
= src
->rm_call
.cb_rpcvers
;
2806 dst
->rm_call
.cb_prog
= src
->rm_call
.cb_prog
;
2807 dst
->rm_call
.cb_vers
= src
->rm_call
.cb_vers
;
2808 dst
->rm_call
.cb_proc
= src
->rm_call
.cb_proc
;
2810 /* dup opaque auth call body cred */
2811 oa_src
= src
->rm_call
.cb_cred
;
2813 oa_dst
.oa_flavor
= oa_src
.oa_flavor
;
2814 oa_dst
.oa_base
= kmem_alloc(oa_src
.oa_length
, KM_SLEEP
);
2816 bcopy(oa_src
.oa_base
, oa_dst
.oa_base
, oa_src
.oa_length
);
2817 oa_dst
.oa_length
= oa_src
.oa_length
;
2819 dst
->rm_call
.cb_cred
= oa_dst
;
2821 /* dup or just alloc opaque auth call body verifier */
2822 if (src
->rm_call
.cb_verf
.oa_length
> 0) {
2823 oa_src
= src
->rm_call
.cb_verf
;
2825 oa_dst
.oa_flavor
= oa_src
.oa_flavor
;
2826 oa_dst
.oa_base
= kmem_alloc(oa_src
.oa_length
, KM_SLEEP
);
2828 bcopy(oa_src
.oa_base
, oa_dst
.oa_base
, oa_src
.oa_length
);
2829 oa_dst
.oa_length
= oa_src
.oa_length
;
2831 dst
->rm_call
.cb_verf
= oa_dst
;
2833 oa_dst
.oa_flavor
= -1; /* will be set later */
2834 oa_dst
.oa_base
= kmem_alloc(MAX_AUTH_BYTES
, KM_SLEEP
);
2836 oa_dst
.oa_length
= 0; /* will be set later */
2838 dst
->rm_call
.cb_verf
= oa_dst
;
2843 kmem_free(dst
->rm_call
.cb_cred
.oa_base
, dst
->rm_call
.cb_cred
.oa_length
);
2844 kmem_free(dst
, sizeof (*dst
));
2849 rpc_msg_free(struct rpc_msg
**msg
, int cb_verf_oa_length
)
2851 struct rpc_msg
*m
= *msg
;
2853 kmem_free(m
->rm_call
.cb_cred
.oa_base
, m
->rm_call
.cb_cred
.oa_length
);
2854 m
->rm_call
.cb_cred
.oa_base
= NULL
;
2855 m
->rm_call
.cb_cred
.oa_length
= 0;
2857 kmem_free(m
->rm_call
.cb_verf
.oa_base
, cb_verf_oa_length
);
2858 m
->rm_call
.cb_verf
.oa_base
= NULL
;
2859 m
->rm_call
.cb_verf
.oa_length
= 0;
2861 kmem_free(m
, sizeof (*m
));