4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012 by Delphix. All rights reserved.
27 * Architecture-independent CPU control functions.
30 #include <sys/types.h>
31 #include <sys/param.h>
33 #include <sys/thread.h>
34 #include <sys/cpuvar.h>
35 #include <sys/cpu_event.h>
36 #include <sys/kstat.h>
37 #include <sys/uadmin.h>
38 #include <sys/systm.h>
39 #include <sys/errno.h>
40 #include <sys/cmn_err.h>
41 #include <sys/procset.h>
42 #include <sys/processor.h>
43 #include <sys/debug.h>
44 #include <sys/cpupart.h>
49 #include <sys/kmem_impl.h> /* to set per-cpu kmem_cache offset */
50 #include <sys/atomic.h>
51 #include <sys/callb.h>
52 #include <sys/vtrace.h>
53 #include <sys/cyclic.h>
54 #include <sys/bitmap.h>
55 #include <sys/nvpair.h>
56 #include <sys/pool_pset.h>
57 #include <sys/msacct.h>
59 #include <sys/archsystm.h>
61 #if defined(__x86) || defined(__amd64)
62 #include <sys/x86_archext.h>
64 #include <sys/callo.h>
66 extern int mp_cpu_start(cpu_t
*);
67 extern int mp_cpu_stop(cpu_t
*);
68 extern int mp_cpu_poweron(cpu_t
*);
69 extern int mp_cpu_poweroff(cpu_t
*);
70 extern int mp_cpu_configure(int);
71 extern int mp_cpu_unconfigure(int);
72 extern void mp_cpu_faulted_enter(cpu_t
*);
73 extern void mp_cpu_faulted_exit(cpu_t
*);
75 extern int cmp_cpu_to_chip(processorid_t cpuid
);
77 extern char *cpu_fru_fmri(cpu_t
*cp
);
80 static void cpu_add_active_internal(cpu_t
*cp
);
81 static void cpu_remove_active(cpu_t
*cp
);
82 static void cpu_info_kstat_create(cpu_t
*cp
);
83 static void cpu_info_kstat_destroy(cpu_t
*cp
);
84 static void cpu_stats_kstat_create(cpu_t
*cp
);
85 static void cpu_stats_kstat_destroy(cpu_t
*cp
);
87 static int cpu_sys_stats_ks_update(kstat_t
*ksp
, int rw
);
88 static int cpu_vm_stats_ks_update(kstat_t
*ksp
, int rw
);
89 static int cpu_stat_ks_update(kstat_t
*ksp
, int rw
);
90 static int cpu_state_change_hooks(int, cpu_setup_t
, cpu_setup_t
);
93 * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active,
94 * max_cpu_seqid_ever, and dispatch queue reallocations. The lock ordering with
95 * respect to related locks is:
97 * cpu_lock --> thread_free_lock ---> p_lock ---> thread_lock()
99 * Warning: Certain sections of code do not use the cpu_lock when
100 * traversing the cpu_list (e.g. mutex_vector_enter(), clock()). Since
101 * all cpus are paused during modifications to this list, a solution
102 * to protect the list is too either disable kernel preemption while
103 * walking the list, *or* recheck the cpu_next pointer at each
104 * iteration in the loop. Note that in no cases can any cached
105 * copies of the cpu pointers be kept as they may become invalid.
108 cpu_t
*cpu_list
; /* list of all CPUs */
109 cpu_t
*clock_cpu_list
; /* used by clock to walk CPUs */
110 cpu_t
*cpu_active
; /* list of active CPUs */
111 static cpuset_t cpu_available
; /* set of available CPUs */
112 cpuset_t cpu_seqid_inuse
; /* which cpu_seqids are in use */
114 cpu_t
**cpu_seq
; /* ptrs to CPUs, indexed by seq_id */
117 * max_ncpus keeps the max cpus the system can have. Initially
118 * it's NCPU, but since most archs scan the devtree for cpus
119 * fairly early on during boot, the real max can be known before
120 * ncpus is set (useful for early NCPU based allocations).
122 int max_ncpus
= NCPU
;
124 * platforms that set max_ncpus to maxiumum number of cpus that can be
125 * dynamically added will set boot_max_ncpus to the number of cpus found
126 * at device tree scan time during boot.
128 int boot_max_ncpus
= -1;
131 * Maximum possible CPU id. This can never be >= NCPU since NCPU is
132 * used to size arrays that are indexed by CPU id.
134 processorid_t max_cpuid
= NCPU
- 1;
137 * Maximum cpu_seqid was given. This number can only grow and never shrink. It
138 * can be used to optimize NCPU loops to avoid going through CPUs which were
141 processorid_t max_cpu_seqid_ever
= 0;
144 int ncpus_online
= 1;
147 * CPU that we're trying to offline. Protected by cpu_lock.
152 * Can be raised to suppress further weakbinding, which are instead
153 * satisfied by disabling preemption. Must be raised/lowered under cpu_lock,
154 * while individual thread weakbinding synchronization is done under thread
157 int weakbindingbarrier
;
160 * Variables used in pause_cpus().
162 static volatile char safe_list
[NCPU
];
164 static struct _cpu_pause_info
{
165 int cp_spl
; /* spl saved in pause_cpus() */
166 volatile int cp_go
; /* Go signal sent after all ready */
167 int cp_count
; /* # of CPUs to pause */
168 ksema_t cp_sem
; /* synch pause_cpus & cpu_pause */
169 kthread_id_t cp_paused
;
172 static kmutex_t pause_free_mutex
;
173 static kcondvar_t pause_free_cv
;
175 void *(*cpu_pause_func
)(void *) = NULL
;
178 static struct cpu_sys_stats_ks_data
{
179 kstat_named_t cpu_ticks_idle
;
180 kstat_named_t cpu_ticks_user
;
181 kstat_named_t cpu_ticks_kernel
;
182 kstat_named_t cpu_ticks_wait
;
183 kstat_named_t cpu_nsec_idle
;
184 kstat_named_t cpu_nsec_user
;
185 kstat_named_t cpu_nsec_kernel
;
186 kstat_named_t cpu_nsec_dtrace
;
187 kstat_named_t cpu_nsec_intr
;
188 kstat_named_t cpu_load_intr
;
189 kstat_named_t wait_ticks_io
;
190 kstat_named_t dtrace_probes
;
192 kstat_named_t bwrite
;
194 kstat_named_t lwrite
;
195 kstat_named_t phread
;
196 kstat_named_t phwrite
;
197 kstat_named_t pswitch
;
200 kstat_named_t syscall
;
201 kstat_named_t sysread
;
202 kstat_named_t syswrite
;
203 kstat_named_t sysfork
;
204 kstat_named_t sysvfork
;
205 kstat_named_t sysexec
;
206 kstat_named_t readch
;
207 kstat_named_t writech
;
208 kstat_named_t rcvint
;
209 kstat_named_t xmtint
;
210 kstat_named_t mdmint
;
217 kstat_named_t ufsiget
;
218 kstat_named_t ufsdirblk
;
219 kstat_named_t ufsipage
;
220 kstat_named_t ufsinopage
;
221 kstat_named_t procovf
;
222 kstat_named_t intrthread
;
223 kstat_named_t intrblk
;
224 kstat_named_t intrunpin
;
225 kstat_named_t idlethread
;
226 kstat_named_t inv_swtch
;
227 kstat_named_t nthreads
;
228 kstat_named_t cpumigrate
;
229 kstat_named_t xcalls
;
230 kstat_named_t mutex_adenters
;
231 kstat_named_t rw_rdfails
;
232 kstat_named_t rw_wrfails
;
233 kstat_named_t modload
;
234 kstat_named_t modunload
;
235 kstat_named_t bawrite
;
236 kstat_named_t iowait
;
237 } cpu_sys_stats_ks_data_template
= {
238 { "cpu_ticks_idle", KSTAT_DATA_UINT64
},
239 { "cpu_ticks_user", KSTAT_DATA_UINT64
},
240 { "cpu_ticks_kernel", KSTAT_DATA_UINT64
},
241 { "cpu_ticks_wait", KSTAT_DATA_UINT64
},
242 { "cpu_nsec_idle", KSTAT_DATA_UINT64
},
243 { "cpu_nsec_user", KSTAT_DATA_UINT64
},
244 { "cpu_nsec_kernel", KSTAT_DATA_UINT64
},
245 { "cpu_nsec_dtrace", KSTAT_DATA_UINT64
},
246 { "cpu_nsec_intr", KSTAT_DATA_UINT64
},
247 { "cpu_load_intr", KSTAT_DATA_UINT64
},
248 { "wait_ticks_io", KSTAT_DATA_UINT64
},
249 { "dtrace_probes", KSTAT_DATA_UINT64
},
250 { "bread", KSTAT_DATA_UINT64
},
251 { "bwrite", KSTAT_DATA_UINT64
},
252 { "lread", KSTAT_DATA_UINT64
},
253 { "lwrite", KSTAT_DATA_UINT64
},
254 { "phread", KSTAT_DATA_UINT64
},
255 { "phwrite", KSTAT_DATA_UINT64
},
256 { "pswitch", KSTAT_DATA_UINT64
},
257 { "trap", KSTAT_DATA_UINT64
},
258 { "intr", KSTAT_DATA_UINT64
},
259 { "syscall", KSTAT_DATA_UINT64
},
260 { "sysread", KSTAT_DATA_UINT64
},
261 { "syswrite", KSTAT_DATA_UINT64
},
262 { "sysfork", KSTAT_DATA_UINT64
},
263 { "sysvfork", KSTAT_DATA_UINT64
},
264 { "sysexec", KSTAT_DATA_UINT64
},
265 { "readch", KSTAT_DATA_UINT64
},
266 { "writech", KSTAT_DATA_UINT64
},
267 { "rcvint", KSTAT_DATA_UINT64
},
268 { "xmtint", KSTAT_DATA_UINT64
},
269 { "mdmint", KSTAT_DATA_UINT64
},
270 { "rawch", KSTAT_DATA_UINT64
},
271 { "canch", KSTAT_DATA_UINT64
},
272 { "outch", KSTAT_DATA_UINT64
},
273 { "msg", KSTAT_DATA_UINT64
},
274 { "sema", KSTAT_DATA_UINT64
},
275 { "namei", KSTAT_DATA_UINT64
},
276 { "ufsiget", KSTAT_DATA_UINT64
},
277 { "ufsdirblk", KSTAT_DATA_UINT64
},
278 { "ufsipage", KSTAT_DATA_UINT64
},
279 { "ufsinopage", KSTAT_DATA_UINT64
},
280 { "procovf", KSTAT_DATA_UINT64
},
281 { "intrthread", KSTAT_DATA_UINT64
},
282 { "intrblk", KSTAT_DATA_UINT64
},
283 { "intrunpin", KSTAT_DATA_UINT64
},
284 { "idlethread", KSTAT_DATA_UINT64
},
285 { "inv_swtch", KSTAT_DATA_UINT64
},
286 { "nthreads", KSTAT_DATA_UINT64
},
287 { "cpumigrate", KSTAT_DATA_UINT64
},
288 { "xcalls", KSTAT_DATA_UINT64
},
289 { "mutex_adenters", KSTAT_DATA_UINT64
},
290 { "rw_rdfails", KSTAT_DATA_UINT64
},
291 { "rw_wrfails", KSTAT_DATA_UINT64
},
292 { "modload", KSTAT_DATA_UINT64
},
293 { "modunload", KSTAT_DATA_UINT64
},
294 { "bawrite", KSTAT_DATA_UINT64
},
295 { "iowait", KSTAT_DATA_UINT64
},
298 static struct cpu_vm_stats_ks_data
{
300 kstat_named_t pgfrec
;
302 kstat_named_t pgpgin
;
304 kstat_named_t pgpgout
;
305 kstat_named_t swapin
;
306 kstat_named_t pgswapin
;
307 kstat_named_t swapout
;
308 kstat_named_t pgswapout
;
313 kstat_named_t hat_fault
;
314 kstat_named_t as_fault
;
315 kstat_named_t maj_fault
;
316 kstat_named_t cow_fault
;
317 kstat_named_t prot_fault
;
318 kstat_named_t softlock
;
319 kstat_named_t kernel_asflt
;
320 kstat_named_t pgrrun
;
321 kstat_named_t execpgin
;
322 kstat_named_t execpgout
;
323 kstat_named_t execfree
;
324 kstat_named_t anonpgin
;
325 kstat_named_t anonpgout
;
326 kstat_named_t anonfree
;
327 kstat_named_t fspgin
;
328 kstat_named_t fspgout
;
329 kstat_named_t fsfree
;
330 } cpu_vm_stats_ks_data_template
= {
331 { "pgrec", KSTAT_DATA_UINT64
},
332 { "pgfrec", KSTAT_DATA_UINT64
},
333 { "pgin", KSTAT_DATA_UINT64
},
334 { "pgpgin", KSTAT_DATA_UINT64
},
335 { "pgout", KSTAT_DATA_UINT64
},
336 { "pgpgout", KSTAT_DATA_UINT64
},
337 { "swapin", KSTAT_DATA_UINT64
},
338 { "pgswapin", KSTAT_DATA_UINT64
},
339 { "swapout", KSTAT_DATA_UINT64
},
340 { "pgswapout", KSTAT_DATA_UINT64
},
341 { "zfod", KSTAT_DATA_UINT64
},
342 { "dfree", KSTAT_DATA_UINT64
},
343 { "scan", KSTAT_DATA_UINT64
},
344 { "rev", KSTAT_DATA_UINT64
},
345 { "hat_fault", KSTAT_DATA_UINT64
},
346 { "as_fault", KSTAT_DATA_UINT64
},
347 { "maj_fault", KSTAT_DATA_UINT64
},
348 { "cow_fault", KSTAT_DATA_UINT64
},
349 { "prot_fault", KSTAT_DATA_UINT64
},
350 { "softlock", KSTAT_DATA_UINT64
},
351 { "kernel_asflt", KSTAT_DATA_UINT64
},
352 { "pgrrun", KSTAT_DATA_UINT64
},
353 { "execpgin", KSTAT_DATA_UINT64
},
354 { "execpgout", KSTAT_DATA_UINT64
},
355 { "execfree", KSTAT_DATA_UINT64
},
356 { "anonpgin", KSTAT_DATA_UINT64
},
357 { "anonpgout", KSTAT_DATA_UINT64
},
358 { "anonfree", KSTAT_DATA_UINT64
},
359 { "fspgin", KSTAT_DATA_UINT64
},
360 { "fspgout", KSTAT_DATA_UINT64
},
361 { "fsfree", KSTAT_DATA_UINT64
},
365 * Force the specified thread to migrate to the appropriate processor.
366 * Called with thread lock held, returns with it dropped.
369 force_thread_migrate(kthread_id_t tp
)
371 ASSERT(THREAD_LOCK_HELD(tp
));
372 if (tp
== curthread
) {
373 THREAD_TRANSITION(tp
);
375 thread_unlock_nopreempt(tp
);
378 if (tp
->t_state
== TS_ONPROC
) {
380 } else if (tp
->t_state
== TS_RUN
) {
389 * Set affinity for a specified CPU.
390 * A reference count is incremented and the affinity is held until the
391 * reference count is decremented to zero by thread_affinity_clear().
392 * This is so regions of code requiring affinity can be nested.
393 * Caller needs to ensure that cpu_id remains valid, which can be
394 * done by holding cpu_lock across this call, unless the caller
395 * specifies CPU_CURRENT in which case the cpu_lock will be acquired
396 * by thread_affinity_set and CPU->cpu_id will be the target CPU.
399 thread_affinity_set(kthread_id_t t
, int cpu_id
)
404 ASSERT(!(t
== curthread
&& t
->t_weakbound_cpu
!= NULL
));
406 if ((c
= cpu_id
) == CPU_CURRENT
) {
407 mutex_enter(&cpu_lock
);
408 cpu_id
= CPU
->cpu_id
;
411 * We should be asserting that cpu_lock is held here, but
412 * the NCA code doesn't acquire it. The following assert
413 * should be uncommented when the NCA code is fixed.
415 * ASSERT(MUTEX_HELD(&cpu_lock));
417 ASSERT((cpu_id
>= 0) && (cpu_id
< NCPU
));
419 ASSERT(cp
!= NULL
); /* user must provide a good cpu_id */
421 * If there is already a hard affinity requested, and this affinity
422 * conflicts with that, panic.
425 if (t
->t_affinitycnt
> 0 && t
->t_bound_cpu
!= cp
) {
426 panic("affinity_set: setting %p but already bound to %p",
427 (void *)cp
, (void *)t
->t_bound_cpu
);
433 * Make sure we're running on the right CPU.
435 if (cp
!= t
->t_cpu
|| t
!= curthread
) {
436 force_thread_migrate(t
); /* drops thread lock */
441 if (c
== CPU_CURRENT
)
442 mutex_exit(&cpu_lock
);
446 * Wrapper for backward compatibility.
449 affinity_set(int cpu_id
)
451 thread_affinity_set(curthread
, cpu_id
);
455 * Decrement the affinity reservation count and if it becomes zero,
456 * clear the CPU affinity for the current thread, or set it to the user's
457 * software binding request.
460 thread_affinity_clear(kthread_id_t t
)
462 register processorid_t binding
;
465 if (--t
->t_affinitycnt
== 0) {
466 if ((binding
= t
->t_bind_cpu
) == PBIND_NONE
) {
468 * Adjust disp_max_unbound_pri if necessary.
470 disp_adjust_unbound_pri(t
);
471 t
->t_bound_cpu
= NULL
;
472 if (t
->t_cpu
->cpu_part
!= t
->t_cpupart
) {
473 force_thread_migrate(t
);
477 t
->t_bound_cpu
= cpu
[binding
];
479 * Make sure the thread is running on the bound CPU.
481 if (t
->t_cpu
!= t
->t_bound_cpu
) {
482 force_thread_migrate(t
);
483 return; /* already dropped lock */
491 * Wrapper for backward compatibility.
496 thread_affinity_clear(curthread
);
500 * Weak cpu affinity. Bind to the "current" cpu for short periods
501 * of time during which the thread must not block (but may be preempted).
502 * Use this instead of kpreempt_disable() when it is only "no migration"
503 * rather than "no preemption" semantics that are required - disabling
504 * preemption holds higher priority threads off of cpu and if the
505 * operation that is protected is more than momentary this is not good
508 * Weakly bound threads will not prevent a cpu from being offlined -
509 * we'll only run them on the cpu to which they are weakly bound but
510 * (because they do not block) we'll always be able to move them on to
511 * another cpu at offline time if we give them just a short moment to
512 * run during which they will unbind. To give a cpu a chance of offlining,
513 * however, we require a barrier to weak bindings that may be raised for a
514 * given cpu (offline/move code may set this and then wait a short time for
515 * existing weak bindings to drop); the cpu_inmotion pointer is that barrier.
517 * There are few restrictions on the calling context of thread_nomigrate.
518 * The caller must not hold the thread lock. Calls may be nested.
520 * After weakbinding a thread must not perform actions that may block.
521 * In particular it must not call thread_affinity_set; calling that when
522 * already weakbound is nonsensical anyway.
524 * If curthread is prevented from migrating for other reasons
525 * (kernel preemption disabled; high pil; strongly bound; interrupt thread)
526 * then the weak binding will succeed even if this cpu is the target of an
527 * offline/move request.
530 thread_nomigrate(void)
533 kthread_id_t t
= curthread
;
540 * A highlevel interrupt must not modify t_nomigrate or
541 * t_weakbound_cpu of the thread it has interrupted. A lowlevel
542 * interrupt thread cannot migrate and we can avoid the
543 * thread_lock call below by short-circuiting here. In either
544 * case we can just return since no migration is possible and
545 * the condition will persist (ie, when we test for these again
546 * in thread_allowmigrate they can't have changed). Migration
547 * is also impossible if we're at or above DISP_LEVEL pil.
549 if (CPU_ON_INTR(cp
) || t
->t_flag
& T_INTR_THREAD
||
550 getpil() >= DISP_LEVEL
) {
556 * We must be consistent with existing weak bindings. Since we
557 * may be interrupted between the increment of t_nomigrate and
558 * the store to t_weakbound_cpu below we cannot assume that
559 * t_weakbound_cpu will be set if t_nomigrate is. Note that we
560 * cannot assert t_weakbound_cpu == t_bind_cpu since that is not
563 if (t
->t_nomigrate
&& t
->t_weakbound_cpu
&& t
->t_weakbound_cpu
!= cp
) {
565 panic("thread_nomigrate: binding to %p but already "
566 "bound to %p", (void *)cp
,
567 (void *)t
->t_weakbound_cpu
);
571 * At this point we have preemption disabled and we don't yet hold
572 * the thread lock. So it's possible that somebody else could
573 * set t_bind_cpu here and not be able to force us across to the
574 * new cpu (since we have preemption disabled).
576 thread_lock(curthread
);
579 * If further weak bindings are being (temporarily) suppressed then
580 * we'll settle for disabling kernel preemption (which assures
581 * no migration provided the thread does not block which it is
582 * not allowed to if using thread_nomigrate). We must remember
583 * this disposition so we can take appropriate action in
584 * thread_allowmigrate. If this is a nested call and the
585 * thread is already weakbound then fall through as normal.
586 * We remember the decision to settle for kpreempt_disable through
587 * negative nesting counting in t_nomigrate. Once a thread has had one
588 * weakbinding request satisfied in this way any further (nested)
589 * requests will continue to be satisfied in the same way,
590 * even if weak bindings have recommenced.
592 if (t
->t_nomigrate
< 0 || weakbindingbarrier
&& t
->t_nomigrate
== 0) {
594 thread_unlock(curthread
);
595 return; /* with kpreempt_disable still active */
599 * We hold thread_lock so t_bind_cpu cannot change. We could,
600 * however, be running on a different cpu to which we are t_bound_cpu
601 * to (as explained above). If we grant the weak binding request
602 * in that case then the dispatcher must favour our weak binding
603 * over our strong (in which case, just as when preemption is
604 * disabled, we can continue to run on a cpu other than the one to
605 * which we are strongbound; the difference in this case is that
606 * this thread can be preempted and so can appear on the dispatch
607 * queues of a cpu other than the one it is strongbound to).
609 * If the cpu we are running on does not appear to be a current
610 * offline target (we check cpu_inmotion to determine this - since
611 * we don't hold cpu_lock we may not see a recent store to that,
612 * so it's possible that we at times can grant a weak binding to a
613 * cpu that is an offline target, but that one request will not
614 * prevent the offline from succeeding) then we will always grant
615 * the weak binding request. This includes the case above where
616 * we grant a weakbinding not commensurate with our strong binding.
618 * If our cpu does appear to be an offline target then we're inclined
619 * not to grant the weakbinding request just yet - we'd prefer to
620 * migrate to another cpu and grant the request there. The
621 * exceptions are those cases where going through preemption code
622 * will not result in us changing cpu:
624 * . interrupts have already bypassed this case (see above)
625 * . we are already weakbound to this cpu (dispatcher code will
626 * always return us to the weakbound cpu)
627 * . preemption was disabled even before we disabled it above
628 * . we are strongbound to this cpu (if we're strongbound to
629 * another and not yet running there the trip through the
630 * dispatcher will move us to the strongbound cpu and we
631 * will grant the weak binding there)
633 if (cp
!= cpu_inmotion
|| t
->t_nomigrate
> 0 || t
->t_preempt
> 1 ||
634 t
->t_bound_cpu
== cp
) {
636 * Don't be tempted to store to t_weakbound_cpu only on
637 * the first nested bind request - if we're interrupted
638 * after the increment of t_nomigrate and before the
639 * store to t_weakbound_cpu and the interrupt calls
640 * thread_nomigrate then the assertion in thread_allowmigrate
644 t
->t_weakbound_cpu
= cp
;
646 thread_unlock(curthread
);
648 * Now that we have dropped the thread_lock another thread
649 * can set our t_weakbound_cpu, and will try to migrate us
650 * to the strongbound cpu (which will not be prevented by
651 * preemption being disabled since we're about to enable
652 * preemption). We have granted the weakbinding to the current
653 * cpu, so again we are in the position that is is is possible
654 * that our weak and strong bindings differ. Again this
655 * is catered for by dispatcher code which will favour our
661 * Move to another cpu before granting the request by
662 * forcing this thread through preemption code. When we
663 * get to set{front,back}dq called from CL_PREEMPT()
664 * cpu_choose() will be used to select a cpu to queue
665 * us on - that will see cpu_inmotion and take
666 * steps to avoid returning us to this cpu.
668 cp
->cpu_kprunrun
= 1;
669 thread_unlock(curthread
);
670 kpreempt_enable(); /* will call preempt() */
676 thread_allowmigrate(void)
678 kthread_id_t t
= curthread
;
680 ASSERT(t
->t_weakbound_cpu
== CPU
||
681 (t
->t_nomigrate
< 0 && t
->t_preempt
> 0) ||
682 CPU_ON_INTR(CPU
) || t
->t_flag
& T_INTR_THREAD
||
683 getpil() >= DISP_LEVEL
);
685 if (CPU_ON_INTR(CPU
) || (t
->t_flag
& T_INTR_THREAD
) ||
686 getpil() >= DISP_LEVEL
)
689 if (t
->t_nomigrate
< 0) {
691 * This thread was granted "weak binding" in the
692 * stronger form of kernel preemption disabling.
693 * Undo a level of nesting for both t_nomigrate
698 } else if (--t
->t_nomigrate
== 0) {
700 * Time to drop the weak binding. We need to cater
701 * for the case where we're weakbound to a different
702 * cpu than that to which we're strongbound (a very
703 * temporary arrangement that must only persist until
704 * weak binding drops). We don't acquire thread_lock
705 * here so even as this code executes t_bound_cpu
706 * may be changing. So we disable preemption and
707 * a) in the case that t_bound_cpu changes while we
708 * have preemption disabled kprunrun will be set
709 * asynchronously, and b) if before disabling
710 * preemption we were already on a different cpu to
711 * our t_bound_cpu then we set kprunrun ourselves
712 * to force a trip through the dispatcher when
713 * preemption is enabled.
716 if (t
->t_bound_cpu
&&
717 t
->t_weakbound_cpu
!= t
->t_bound_cpu
)
718 CPU
->cpu_kprunrun
= 1;
719 t
->t_weakbound_cpu
= NULL
;
726 * weakbinding_stop can be used to temporarily cause weakbindings made
727 * with thread_nomigrate to be satisfied through the stronger action of
728 * kpreempt_disable. weakbinding_start recommences normal weakbinding.
732 weakbinding_stop(void)
734 ASSERT(MUTEX_HELD(&cpu_lock
));
735 weakbindingbarrier
= 1;
736 membar_producer(); /* make visible before subsequent thread_lock */
740 weakbinding_start(void)
742 ASSERT(MUTEX_HELD(&cpu_lock
));
743 weakbindingbarrier
= 0;
752 * This routine is called to place the CPUs in a safe place so that
753 * one of them can be taken off line or placed on line. What we are
754 * trying to do here is prevent a thread from traversing the list
755 * of active CPUs while we are changing it or from getting placed on
756 * the run queue of a CPU that has just gone off line. We do this by
757 * creating a thread with the highest possible prio for each CPU and
758 * having it call this routine. The advantage of this method is that
759 * we can eliminate all checks for CPU_ACTIVE in the disp routines.
760 * This makes disp faster at the expense of making p_online() slower
761 * which is a good trade off.
767 struct _cpu_pause_info
*cpi
= &cpu_pause_info
;
768 volatile char *safe
= &safe_list
[index
];
771 ASSERT((curthread
->t_bound_cpu
!= NULL
) || (*safe
== PAUSE_DIE
));
773 while (*safe
!= PAUSE_DIE
) {
775 membar_enter(); /* make sure stores are flushed */
776 sema_v(&cpi
->cp_sem
); /* signal requesting thread */
779 * Wait here until all pause threads are running. That
780 * indicates that it's safe to do the spl. Until
781 * cpu_pause_info.cp_go is set, we don't want to spl
782 * because that might block clock interrupts needed
783 * to preempt threads on other CPUs.
785 while (cpi
->cp_go
== 0)
788 * Even though we are at the highest disp prio, we need
789 * to block out all interrupts below LOCK_LEVEL so that
790 * an intr doesn't come in, wake up a thread, and call
791 * setbackdq/setfrontdq.
795 * if cpu_pause_func() has been set then call it using
796 * index as the argument, currently only used by
797 * cpr_suspend_cpus(). This function is used as the
798 * code to execute on the "paused" cpu's when a machine
799 * comes out of a sleep state and CPU's were powered off.
800 * (could also be used for hotplugging CPU's).
802 if (cpu_pause_func
!= NULL
)
803 (*cpu_pause_func
)((void *)lindex
);
805 mach_cpu_pause(safe
);
809 * Waiting is at an end. Switch out of cpu_pause
810 * loop and resume useful work.
815 mutex_enter(&pause_free_mutex
);
817 cv_broadcast(&pause_free_cv
);
818 mutex_exit(&pause_free_mutex
);
822 * Allow the cpus to start running again.
829 ASSERT(MUTEX_HELD(&cpu_lock
));
830 ASSERT(cpu_pause_info
.cp_paused
);
831 cpu_pause_info
.cp_paused
= NULL
;
832 for (i
= 0; i
< NCPU
; i
++)
833 safe_list
[i
] = PAUSE_IDLE
;
834 membar_enter(); /* make sure stores are flushed */
836 splx(cpu_pause_info
.cp_spl
);
841 * Allocate a pause thread for a CPU.
844 cpu_pause_alloc(cpu_t
*cp
)
847 long cpun
= cp
->cpu_id
;
850 * Note, v.v_nglobpris will not change value as long as I hold
853 t
= thread_create(NULL
, 0, cpu_pause
, (void *)cpun
,
854 0, &p0
, TS_STOPPED
, v
.v_nglobpris
- 1);
857 t
->t_disp_queue
= cp
->cpu_disp
;
858 t
->t_affinitycnt
= 1;
861 cp
->cpu_pause_thread
= t
;
863 * Registering a thread in the callback table is usually done
864 * in the initialization code of the thread. In this
865 * case, we do it right after thread creation because the
866 * thread itself may never run, and we need to register the
867 * fact that it is safe for cpr suspend.
869 CALLB_CPR_INIT_SAFE(t
, "cpu_pause");
873 * Free a pause thread for a CPU.
876 cpu_pause_free(cpu_t
*cp
)
879 int cpun
= cp
->cpu_id
;
881 ASSERT(MUTEX_HELD(&cpu_lock
));
883 * We have to get the thread and tell him to die.
885 if ((t
= cp
->cpu_pause_thread
) == NULL
) {
886 ASSERT(safe_list
[cpun
] == PAUSE_IDLE
);
890 t
->t_cpu
= CPU
; /* disp gets upset if last cpu is quiesced. */
891 t
->t_bound_cpu
= NULL
; /* Must un-bind; cpu may not be running. */
892 t
->t_pri
= v
.v_nglobpris
- 1;
893 ASSERT(safe_list
[cpun
] == PAUSE_IDLE
);
894 safe_list
[cpun
] = PAUSE_DIE
;
895 THREAD_TRANSITION(t
);
897 thread_unlock_nopreempt(t
);
900 * If we don't wait for the thread to actually die, it may try to
901 * run on the wrong cpu as part of an actual call to pause_cpus().
903 mutex_enter(&pause_free_mutex
);
904 while (safe_list
[cpun
] != PAUSE_DEAD
) {
905 cv_wait(&pause_free_cv
, &pause_free_mutex
);
907 mutex_exit(&pause_free_mutex
);
908 safe_list
[cpun
] = PAUSE_IDLE
;
910 cp
->cpu_pause_thread
= NULL
;
914 * Initialize basic structures for pausing CPUs.
919 sema_init(&cpu_pause_info
.cp_sem
, 0, NULL
, SEMA_DEFAULT
, NULL
);
921 * Create initial CPU pause thread.
923 cpu_pause_alloc(CPU
);
927 * Start the threads used to pause another CPU.
930 cpu_pause_start(processorid_t cpu_id
)
935 for (i
= 0; i
< NCPU
; i
++) {
940 if (!CPU_IN_SET(cpu_available
, i
) || (i
== cpu_id
)) {
941 safe_list
[i
] = PAUSE_WAIT
;
946 * Skip CPU if it is quiesced or not yet started.
948 if ((cp
->cpu_flags
& (CPU_QUIESCED
| CPU_READY
)) != CPU_READY
) {
949 safe_list
[i
] = PAUSE_WAIT
;
954 * Start this CPU's pause thread.
956 t
= cp
->cpu_pause_thread
;
959 * Reset the priority, since nglobpris may have
960 * changed since the thread was created, if someone
961 * has loaded the RT (or some other) scheduling
964 t
->t_pri
= v
.v_nglobpris
- 1;
965 THREAD_TRANSITION(t
);
967 thread_unlock_nopreempt(t
);
975 * Pause all of the CPUs except the one we are on by creating a high
976 * priority thread bound to those CPUs.
978 * Note that one must be extremely careful regarding code
979 * executed while CPUs are paused. Since a CPU may be paused
980 * while a thread scheduling on that CPU is holding an adaptive
981 * lock, code executed with CPUs paused must not acquire adaptive
982 * (or low-level spin) locks. Also, such code must not block,
983 * since the thread that is supposed to initiate the wakeup may
986 * With a few exceptions, the restrictions on code executed with CPUs
987 * paused match those for code executed at high-level interrupt
991 pause_cpus(cpu_t
*off_cp
)
993 processorid_t cpu_id
;
995 struct _cpu_pause_info
*cpi
= &cpu_pause_info
;
997 ASSERT(MUTEX_HELD(&cpu_lock
));
998 ASSERT(cpi
->cp_paused
== NULL
);
1001 for (i
= 0; i
< NCPU
; i
++)
1002 safe_list
[i
] = PAUSE_IDLE
;
1006 * If running on the cpu that is going offline, get off it.
1007 * This is so that it won't be necessary to rechoose a CPU
1011 cpu_id
= off_cp
->cpu_next_part
->cpu_id
;
1013 cpu_id
= CPU
->cpu_id
;
1014 affinity_set(cpu_id
);
1017 * Start the pause threads and record how many were started
1019 cpi
->cp_count
= cpu_pause_start(cpu_id
);
1022 * Now wait for all CPUs to be running the pause thread.
1024 while (cpi
->cp_count
> 0) {
1026 * Spin reading the count without grabbing the disp
1027 * lock to make sure we don't prevent the pause
1028 * threads from getting the lock.
1030 while (sema_held(&cpi
->cp_sem
))
1032 if (sema_tryp(&cpi
->cp_sem
))
1035 cpi
->cp_go
= 1; /* all have reached cpu_pause */
1038 * Now wait for all CPUs to spl. (Transition from PAUSE_READY
1041 for (i
= 0; i
< NCPU
; i
++) {
1042 while (safe_list
[i
] != PAUSE_WAIT
)
1045 cpi
->cp_spl
= splhigh(); /* block dispatcher on this CPU */
1046 cpi
->cp_paused
= curthread
;
1050 * Check whether the current thread has CPUs paused
1055 if (cpu_pause_info
.cp_paused
!= NULL
) {
1056 ASSERT(cpu_pause_info
.cp_paused
== curthread
);
1063 cpu_get_all(processorid_t cpun
)
1065 ASSERT(MUTEX_HELD(&cpu_lock
));
1067 if (cpun
>= NCPU
|| cpun
< 0 || !CPU_IN_SET(cpu_available
, cpun
))
1073 * Check whether cpun is a valid processor id and whether it should be
1074 * visible from the current zone. If it is, return a pointer to the
1075 * associated CPU structure.
1078 cpu_get(processorid_t cpun
)
1082 ASSERT(MUTEX_HELD(&cpu_lock
));
1083 c
= cpu_get_all(cpun
);
1084 if (c
!= NULL
&& !INGLOBALZONE(curproc
) && pool_pset_enabled() &&
1085 zone_pset_get(curproc
->p_zone
) != cpupart_query_cpu(c
))
1091 * The following functions should be used to check CPU states in the kernel.
1092 * They should be invoked with cpu_lock held. Kernel subsystems interested
1093 * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc
1094 * states. Those are for user-land (and system call) use only.
1098 * Determine whether the CPU is online and handling interrupts.
1101 cpu_is_online(cpu_t
*cpu
)
1103 ASSERT(MUTEX_HELD(&cpu_lock
));
1104 return (cpu_flagged_online(cpu
->cpu_flags
));
1108 * Determine whether the CPU is offline (this includes spare and faulted).
1111 cpu_is_offline(cpu_t
*cpu
)
1113 ASSERT(MUTEX_HELD(&cpu_lock
));
1114 return (cpu_flagged_offline(cpu
->cpu_flags
));
1118 * Determine whether the CPU is powered off.
1121 cpu_is_poweredoff(cpu_t
*cpu
)
1123 ASSERT(MUTEX_HELD(&cpu_lock
));
1124 return (cpu_flagged_poweredoff(cpu
->cpu_flags
));
1128 * Determine whether the CPU is handling interrupts.
1131 cpu_is_nointr(cpu_t
*cpu
)
1133 ASSERT(MUTEX_HELD(&cpu_lock
));
1134 return (cpu_flagged_nointr(cpu
->cpu_flags
));
1138 * Determine whether the CPU is active (scheduling threads).
1141 cpu_is_active(cpu_t
*cpu
)
1143 ASSERT(MUTEX_HELD(&cpu_lock
));
1144 return (cpu_flagged_active(cpu
->cpu_flags
));
1148 * Same as above, but these require cpu_flags instead of cpu_t pointers.
1151 cpu_flagged_online(cpu_flag_t cpu_flags
)
1153 return (cpu_flagged_active(cpu_flags
) &&
1154 (cpu_flags
& CPU_ENABLE
));
1158 cpu_flagged_offline(cpu_flag_t cpu_flags
)
1160 return (((cpu_flags
& CPU_POWEROFF
) == 0) &&
1161 ((cpu_flags
& (CPU_READY
| CPU_OFFLINE
)) != CPU_READY
));
1165 cpu_flagged_poweredoff(cpu_flag_t cpu_flags
)
1167 return ((cpu_flags
& CPU_POWEROFF
) == CPU_POWEROFF
);
1171 cpu_flagged_nointr(cpu_flag_t cpu_flags
)
1173 return (cpu_flagged_active(cpu_flags
) &&
1174 (cpu_flags
& CPU_ENABLE
) == 0);
1178 cpu_flagged_active(cpu_flag_t cpu_flags
)
1180 return (((cpu_flags
& (CPU_POWEROFF
| CPU_FAULTED
| CPU_SPARE
)) == 0) &&
1181 ((cpu_flags
& (CPU_READY
| CPU_OFFLINE
)) == CPU_READY
));
1185 * Bring the indicated CPU online.
1188 cpu_online(cpu_t
*cp
)
1193 * Handle on-line request.
1194 * This code must put the new CPU on the active list before
1195 * starting it because it will not be paused, and will start
1196 * using the active list immediately. The real start occurs
1197 * when the CPU_QUIESCED flag is turned off.
1200 ASSERT(MUTEX_HELD(&cpu_lock
));
1203 * Put all the cpus into a known safe place.
1204 * No mutexes can be entered while CPUs are paused.
1206 error
= mp_cpu_start(cp
); /* arch-dep hook */
1208 pg_cpupart_in(cp
, cp
->cpu_part
);
1210 cpu_add_active_internal(cp
);
1211 if (cp
->cpu_flags
& CPU_FAULTED
) {
1212 cp
->cpu_flags
&= ~CPU_FAULTED
;
1213 mp_cpu_faulted_exit(cp
);
1215 cp
->cpu_flags
&= ~(CPU_QUIESCED
| CPU_OFFLINE
| CPU_FROZEN
|
1217 CPU_NEW_GENERATION(cp
);
1219 cpu_stats_kstat_create(cp
);
1220 cpu_create_intrstat(cp
);
1221 lgrp_kstat_create(cp
);
1222 cpu_state_change_notify(cp
->cpu_id
, CPU_ON
);
1223 cpu_intr_enable(cp
); /* arch-dep hook */
1224 cpu_state_change_notify(cp
->cpu_id
, CPU_INTR_ON
);
1228 * This has to be called only after cyclic_online(). This
1229 * function uses cyclics.
1231 callout_cpu_online(cp
);
1232 poke_cpu(cp
->cpu_id
);
1239 * Take the indicated CPU offline.
1242 cpu_offline(cpu_t
*cp
, int flags
)
1249 int callout_off
= 0;
1252 int (*bound_func
)(struct cpu
*, int);
1257 boolean_t unbind_all_threads
= (flags
& CPU_FORCED
) != 0;
1259 ASSERT(MUTEX_HELD(&cpu_lock
));
1262 * If we're going from faulted or spare to offline, just
1263 * clear these flags and update CPU state.
1265 if (cp
->cpu_flags
& (CPU_FAULTED
| CPU_SPARE
)) {
1266 if (cp
->cpu_flags
& CPU_FAULTED
) {
1267 cp
->cpu_flags
&= ~CPU_FAULTED
;
1268 mp_cpu_faulted_exit(cp
);
1270 cp
->cpu_flags
&= ~CPU_SPARE
;
1276 * Handle off-line request.
1280 * Don't offline last online CPU in partition
1282 if (ncpus_online
<= 1 || pp
->cp_ncpus
<= 1 || cpu_intr_count(cp
) < 2)
1285 * Unbind all soft-bound threads bound to our CPU and hard bound threads
1286 * if we were asked to.
1288 error
= cpu_unbind(cp
->cpu_id
, unbind_all_threads
);
1292 * We shouldn't be bound to this CPU ourselves.
1294 if (curthread
->t_bound_cpu
== cp
)
1298 * Tell interested parties that this CPU is going offline.
1300 CPU_NEW_GENERATION(cp
);
1301 cpu_state_change_notify(cp
->cpu_id
, CPU_OFF
);
1304 * Tell the PG subsystem that the CPU is leaving the partition
1306 pg_cpupart_out(cp
, pp
);
1309 * Take the CPU out of interrupt participation so we won't find
1310 * bound kernel threads. If the architecture cannot completely
1311 * shut off interrupts on the CPU, don't quiesce it, but don't
1312 * run anything but interrupt thread... this is indicated by
1313 * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being
1316 intr_enable
= cp
->cpu_flags
& CPU_ENABLE
;
1318 no_quiesce
= cpu_intr_disable(cp
);
1321 * Record that we are aiming to offline this cpu. This acts as
1322 * a barrier to further weak binding requests in thread_nomigrate
1323 * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to
1324 * lean away from this cpu. Further strong bindings are already
1325 * avoided since we hold cpu_lock. Since threads that are set
1326 * runnable around now and others coming off the target cpu are
1327 * directed away from the target, existing strong and weak bindings
1328 * (especially the latter) to the target cpu stand maximum chance of
1329 * being able to unbind during the short delay loop below (if other
1330 * unbound threads compete they may not see cpu in time to unbind
1331 * even if they would do so immediately.
1337 * Check for kernel threads (strong or weak) bound to that CPU.
1338 * Strongly bound threads may not unbind, and we'll have to return
1339 * EBUSY. Weakly bound threads should always disappear - we've
1340 * stopped more weak binding with cpu_inmotion and existing
1341 * bindings will drain imminently (they may not block). Nonetheless
1342 * we will wait for a fixed period for all bound threads to disappear.
1343 * Inactive interrupt threads are OK (they'll be in TS_FREE
1344 * state). If test finds some bound threads, wait a few ticks
1345 * to give short-lived threads (such as interrupts) chance to
1346 * complete. Note that if no_quiesce is set, i.e. this cpu
1347 * is required to service interrupts, then we take the route
1348 * that permits interrupt threads to be active (or bypassed).
1350 bound_func
= no_quiesce
? disp_bound_threads
: disp_bound_anythreads
;
1352 again
: for (loop_count
= 0; (*bound_func
)(cp
, 0); loop_count
++) {
1353 if (loop_count
>= 5) {
1354 error
= EBUSY
; /* some threads still bound */
1359 * If some threads were assigned, give them
1360 * a chance to complete or move.
1362 * This assumes that the clock_thread is not bound
1363 * to any CPU, because the clock_thread is needed to
1364 * do the delay(hz/100).
1366 * Note: we still hold the cpu_lock while waiting for
1367 * the next clock tick. This is OK since it isn't
1368 * needed for anything else except processor_bind(2),
1369 * and system initialization. If we drop the lock,
1370 * we would risk another p_online disabling the last
1376 if (error
== 0 && callout_off
== 0) {
1377 callout_cpu_offline(cp
);
1381 if (error
== 0 && cyclic_off
== 0) {
1382 if (!cyclic_offline(cp
)) {
1384 * We must have bound cyclics...
1393 * Call mp_cpu_stop() to perform any special operations
1394 * needed for this machine architecture to offline a CPU.
1397 error
= mp_cpu_stop(cp
); /* arch-dep hook */
1400 * If that all worked, take the CPU offline and decrement
1405 * Put all the cpus into a known safe place.
1406 * No mutexes can be entered while CPUs are paused.
1410 * Repeat the operation, if necessary, to make sure that
1411 * all outstanding low-level interrupts run to completion
1412 * before we set the CPU_QUIESCED flag. It's also possible
1413 * that a thread has weak bound to the cpu despite our raising
1414 * cpu_inmotion above since it may have loaded that
1415 * value before the barrier became visible (this would have
1416 * to be the thread that was on the target cpu at the time
1417 * we raised the barrier).
1419 if ((!no_quiesce
&& cp
->cpu_intr_actv
!= 0) ||
1420 (*bound_func
)(cp
, 1)) {
1422 (void) mp_cpu_start(cp
);
1425 ncp
= cp
->cpu_next_part
;
1426 cpu_lpl
= cp
->cpu_lpl
;
1427 ASSERT(cpu_lpl
!= NULL
);
1430 * Remove the CPU from the list of active CPUs.
1432 cpu_remove_active(cp
);
1435 * Walk the active process list and look for threads
1436 * whose home lgroup needs to be updated, or
1437 * the last CPU they run on is the one being offlined now.
1440 ASSERT(curthread
->t_cpu
!= cp
);
1441 for (p
= practive
; p
!= NULL
; p
= p
->p_next
) {
1451 ASSERT(t
->t_lpl
!= NULL
);
1453 * Taking last CPU in lpl offline
1454 * Rehome thread if it is in this lpl
1455 * Otherwise, update the count of how many
1456 * threads are in this CPU's lgroup but have
1460 if (cpu_lpl
->lpl_ncpu
== 0) {
1461 if (t
->t_lpl
== cpu_lpl
)
1465 else if (t
->t_lpl
->lpl_lgrpid
==
1466 cpu_lpl
->lpl_lgrpid
)
1469 ASSERT(t
->t_lpl
->lpl_ncpu
> 0);
1472 * Update CPU last ran on if it was this CPU
1474 if (t
->t_cpu
== cp
&& t
->t_bound_cpu
!= cp
)
1475 t
->t_cpu
= disp_lowpri_cpu(ncp
,
1476 t
->t_lpl
, t
->t_pri
, NULL
);
1477 ASSERT(t
->t_cpu
!= cp
|| t
->t_bound_cpu
== cp
||
1478 t
->t_weakbound_cpu
== cp
);
1481 } while (t
!= p
->p_tlist
);
1484 * Didn't find any threads in the same lgroup as this
1485 * CPU with a different lpl, so remove the lgroup from
1486 * the process lgroup bitmask.
1489 if (lgrp_diff_lpl
== 0)
1490 klgrpset_del(p
->p_lgrpset
, cpu_lpl
->lpl_lgrpid
);
1494 * Walk thread list looking for threads that need to be
1495 * rehomed, since there are some threads that are not in
1496 * their process's p_tlist.
1501 ASSERT(t
!= NULL
&& t
->t_lpl
!= NULL
);
1504 * Rehome threads with same lpl as this CPU when this
1505 * is the last CPU in the lpl.
1508 if ((cpu_lpl
->lpl_ncpu
== 0) && (t
->t_lpl
== cpu_lpl
))
1510 lgrp_choose(t
, t
->t_cpupart
), 1);
1512 ASSERT(t
->t_lpl
->lpl_ncpu
> 0);
1515 * Update CPU last ran on if it was this CPU
1518 if (t
->t_cpu
== cp
&& t
->t_bound_cpu
!= cp
) {
1519 t
->t_cpu
= disp_lowpri_cpu(ncp
,
1520 t
->t_lpl
, t
->t_pri
, NULL
);
1522 ASSERT(t
->t_cpu
!= cp
|| t
->t_bound_cpu
== cp
||
1523 t
->t_weakbound_cpu
== cp
);
1526 } while (t
!= curthread
);
1527 ASSERT((cp
->cpu_flags
& (CPU_FAULTED
| CPU_SPARE
)) == 0);
1528 cp
->cpu_flags
|= CPU_OFFLINE
;
1529 disp_cpu_inactive(cp
);
1531 cp
->cpu_flags
|= CPU_QUIESCED
;
1534 cpu_inmotion
= NULL
;
1536 cpu_stats_kstat_destroy(cp
);
1537 cpu_delete_intrstat(cp
);
1538 lgrp_kstat_destroy(cp
);
1542 cpu_inmotion
= NULL
;
1545 * If we failed, re-enable interrupts.
1546 * Do this even if cpu_intr_disable returned an error, because
1547 * it may have partially disabled interrupts.
1549 if (error
&& intr_enable
)
1550 cpu_intr_enable(cp
);
1553 * If we failed, but managed to offline the cyclic subsystem on this
1554 * CPU, bring it back online.
1556 if (error
&& cyclic_off
)
1560 * If we failed, but managed to offline callouts on this CPU,
1561 * bring it back online.
1563 if (error
&& callout_off
)
1564 callout_cpu_online(cp
);
1567 * If we failed, tell the PG subsystem that the CPU is back
1569 pg_cpupart_in(cp
, pp
);
1572 * If we failed, we need to notify everyone that this CPU is back on.
1575 CPU_NEW_GENERATION(cp
);
1576 cpu_state_change_notify(cp
->cpu_id
, CPU_ON
);
1577 cpu_state_change_notify(cp
->cpu_id
, CPU_INTR_ON
);
1584 * Mark the indicated CPU as faulted, taking it offline.
1587 cpu_faulted(cpu_t
*cp
, int flags
)
1591 ASSERT(MUTEX_HELD(&cpu_lock
));
1592 ASSERT(!cpu_is_poweredoff(cp
));
1594 if (cpu_is_offline(cp
)) {
1595 cp
->cpu_flags
&= ~CPU_SPARE
;
1596 cp
->cpu_flags
|= CPU_FAULTED
;
1597 mp_cpu_faulted_enter(cp
);
1602 if ((error
= cpu_offline(cp
, flags
)) == 0) {
1603 cp
->cpu_flags
|= CPU_FAULTED
;
1604 mp_cpu_faulted_enter(cp
);
1612 * Mark the indicated CPU as a spare, taking it offline.
1615 cpu_spare(cpu_t
*cp
, int flags
)
1619 ASSERT(MUTEX_HELD(&cpu_lock
));
1620 ASSERT(!cpu_is_poweredoff(cp
));
1622 if (cpu_is_offline(cp
)) {
1623 if (cp
->cpu_flags
& CPU_FAULTED
) {
1624 cp
->cpu_flags
&= ~CPU_FAULTED
;
1625 mp_cpu_faulted_exit(cp
);
1627 cp
->cpu_flags
|= CPU_SPARE
;
1632 if ((error
= cpu_offline(cp
, flags
)) == 0) {
1633 cp
->cpu_flags
|= CPU_SPARE
;
1641 * Take the indicated CPU from poweroff to offline.
1644 cpu_poweron(cpu_t
*cp
)
1646 int error
= ENOTSUP
;
1648 ASSERT(MUTEX_HELD(&cpu_lock
));
1649 ASSERT(cpu_is_poweredoff(cp
));
1651 error
= mp_cpu_poweron(cp
); /* arch-dep hook */
1659 * Take the indicated CPU from any inactive state to powered off.
1662 cpu_poweroff(cpu_t
*cp
)
1664 int error
= ENOTSUP
;
1666 ASSERT(MUTEX_HELD(&cpu_lock
));
1667 ASSERT(cpu_is_offline(cp
));
1669 if (!(cp
->cpu_flags
& CPU_QUIESCED
))
1670 return (EBUSY
); /* not completely idle */
1672 error
= mp_cpu_poweroff(cp
); /* arch-dep hook */
1680 * Initialize the Sequential CPU id lookup table
1687 tbl
= kmem_zalloc(sizeof (struct cpu
*) * max_ncpus
, KM_SLEEP
);
1694 * Initialize the CPU lists for the first CPU.
1697 cpu_list_init(cpu_t
*cp
)
1702 clock_cpu_list
= cp
;
1704 cp
->cpu_next_onln
= cp
;
1705 cp
->cpu_prev_onln
= cp
;
1709 CPUSET_ADD(cpu_seqid_inuse
, 0);
1712 * Bootstrap cpu_seq using cpu_list
1713 * The cpu_seq[] table will be dynamically allocated
1714 * when kmem later becomes available (but before going MP)
1716 cpu_seq
= &cpu_list
;
1718 cp
->cpu_cache_offset
= KMEM_CPU_CACHE_OFFSET(cp
->cpu_seqid
);
1719 cp_default
.cp_cpulist
= cp
;
1720 cp_default
.cp_ncpus
= 1;
1721 cp
->cpu_next_part
= cp
;
1722 cp
->cpu_prev_part
= cp
;
1723 cp
->cpu_part
= &cp_default
;
1725 CPUSET_ADD(cpu_available
, cp
->cpu_id
);
1729 * Insert a CPU into the list of available CPUs.
1732 cpu_add_unit(cpu_t
*cp
)
1736 ASSERT(MUTEX_HELD(&cpu_lock
));
1737 ASSERT(cpu_list
!= NULL
); /* list started in cpu_list_init */
1739 lgrp_config(LGRP_CONFIG_CPU_ADD
, (uintptr_t)cp
, 0);
1742 * Note: most users of the cpu_list will grab the
1743 * cpu_lock to insure that it isn't modified. However,
1744 * certain users can't or won't do that. To allow this
1745 * we pause the other cpus. Users who walk the list
1746 * without cpu_lock, must disable kernel preemption
1747 * to insure that the list isn't modified underneath
1748 * them. Also, any cached pointers to cpu structures
1749 * must be revalidated by checking to see if the
1750 * cpu_next pointer points to itself. This check must
1751 * be done with the cpu_lock held or kernel preemption
1752 * disabled. This check relies upon the fact that
1753 * old cpu structures are not free'ed or cleared after
1754 * then are removed from the cpu_list.
1756 * Note that the clock code walks the cpu list dereferencing
1757 * the cpu_part pointer, so we need to initialize it before
1758 * adding the cpu to the list.
1760 cp
->cpu_part
= &cp_default
;
1761 (void) pause_cpus(NULL
);
1762 cp
->cpu_next
= cpu_list
;
1763 cp
->cpu_prev
= cpu_list
->cpu_prev
;
1764 cpu_list
->cpu_prev
->cpu_next
= cp
;
1765 cpu_list
->cpu_prev
= cp
;
1768 for (seqid
= 0; CPU_IN_SET(cpu_seqid_inuse
, seqid
); seqid
++)
1770 CPUSET_ADD(cpu_seqid_inuse
, seqid
);
1771 cp
->cpu_seqid
= seqid
;
1773 if (seqid
> max_cpu_seqid_ever
)
1774 max_cpu_seqid_ever
= seqid
;
1776 ASSERT(ncpus
< max_ncpus
);
1778 cp
->cpu_cache_offset
= KMEM_CPU_CACHE_OFFSET(cp
->cpu_seqid
);
1779 cpu
[cp
->cpu_id
] = cp
;
1780 CPUSET_ADD(cpu_available
, cp
->cpu_id
);
1781 cpu_seq
[cp
->cpu_seqid
] = cp
;
1784 * allocate a pause thread for this CPU.
1786 cpu_pause_alloc(cp
);
1789 * So that new CPUs won't have NULL prev_onln and next_onln pointers,
1790 * link them into a list of just that CPU.
1791 * This is so that disp_lowpri_cpu will work for thread_create in
1792 * pause_cpus() when called from the startup thread in a new CPU.
1794 cp
->cpu_next_onln
= cp
;
1795 cp
->cpu_prev_onln
= cp
;
1796 cpu_info_kstat_create(cp
);
1797 cp
->cpu_next_part
= cp
;
1798 cp
->cpu_prev_part
= cp
;
1800 init_cpu_mstate(cp
, CMS_SYSTEM
);
1802 pool_pset_mod
= gethrtime();
1806 * Do the opposite of cpu_add_unit().
1809 cpu_del_unit(int cpuid
)
1811 struct cpu
*cp
, *cpnext
;
1813 ASSERT(MUTEX_HELD(&cpu_lock
));
1817 ASSERT(cp
->cpu_next_onln
== cp
);
1818 ASSERT(cp
->cpu_prev_onln
== cp
);
1819 ASSERT(cp
->cpu_next_part
== cp
);
1820 ASSERT(cp
->cpu_prev_part
== cp
);
1823 * Tear down the CPU's physical ID cache, and update any
1826 pg_cpu_fini(cp
, NULL
);
1827 pghw_physid_destroy(cp
);
1830 * Destroy kstat stuff.
1832 cpu_info_kstat_destroy(cp
);
1833 term_cpu_mstate(cp
);
1835 * Free up pause thread.
1838 CPUSET_DEL(cpu_available
, cp
->cpu_id
);
1839 cpu
[cp
->cpu_id
] = NULL
;
1840 cpu_seq
[cp
->cpu_seqid
] = NULL
;
1843 * The clock thread and mutex_vector_enter cannot hold the
1844 * cpu_lock while traversing the cpu list, therefore we pause
1845 * all other threads by pausing the other cpus. These, and any
1846 * other routines holding cpu pointers while possibly sleeping
1847 * must be sure to call kpreempt_disable before processing the
1848 * list and be sure to check that the cpu has not been deleted
1849 * after any sleeps (check cp->cpu_next != NULL). We guarantee
1850 * to keep the deleted cpu structure around.
1852 * Note that this MUST be done AFTER cpu_available
1853 * has been updated so that we don't waste time
1854 * trying to pause the cpu we're trying to delete.
1856 (void) pause_cpus(NULL
);
1858 cpnext
= cp
->cpu_next
;
1859 cp
->cpu_prev
->cpu_next
= cp
->cpu_next
;
1860 cp
->cpu_next
->cpu_prev
= cp
->cpu_prev
;
1865 * Signals that the cpu has been deleted (see above).
1867 cp
->cpu_next
= NULL
;
1868 cp
->cpu_prev
= NULL
;
1872 CPUSET_DEL(cpu_seqid_inuse
, cp
->cpu_seqid
);
1874 lgrp_config(LGRP_CONFIG_CPU_DEL
, (uintptr_t)cp
, 0);
1876 pool_pset_mod
= gethrtime();
1880 * Add a CPU to the list of active CPUs.
1881 * This routine must not get any locks, because other CPUs are paused.
1884 cpu_add_active_internal(cpu_t
*cp
)
1886 cpupart_t
*pp
= cp
->cpu_part
;
1888 ASSERT(MUTEX_HELD(&cpu_lock
));
1889 ASSERT(cpu_list
!= NULL
); /* list started in cpu_list_init */
1893 cp
->cpu_next_onln
= cpu_active
;
1894 cp
->cpu_prev_onln
= cpu_active
->cpu_prev_onln
;
1895 cpu_active
->cpu_prev_onln
->cpu_next_onln
= cp
;
1896 cpu_active
->cpu_prev_onln
= cp
;
1898 if (pp
->cp_cpulist
) {
1899 cp
->cpu_next_part
= pp
->cp_cpulist
;
1900 cp
->cpu_prev_part
= pp
->cp_cpulist
->cpu_prev_part
;
1901 pp
->cp_cpulist
->cpu_prev_part
->cpu_next_part
= cp
;
1902 pp
->cp_cpulist
->cpu_prev_part
= cp
;
1904 ASSERT(pp
->cp_ncpus
== 0);
1905 pp
->cp_cpulist
= cp
->cpu_next_part
= cp
->cpu_prev_part
= cp
;
1908 if (pp
->cp_ncpus
== 1) {
1909 cp_numparts_nonempty
++;
1910 ASSERT(cp_numparts_nonempty
!= 0);
1914 lgrp_config(LGRP_CONFIG_CPU_ONLINE
, (uintptr_t)cp
, 0);
1916 bzero(&cp
->cpu_loadavg
, sizeof (cp
->cpu_loadavg
));
1920 * Add a CPU to the list of active CPUs.
1921 * This is called from machine-dependent layers when a new CPU is started.
1924 cpu_add_active(cpu_t
*cp
)
1926 pg_cpupart_in(cp
, cp
->cpu_part
);
1929 cpu_add_active_internal(cp
);
1932 cpu_stats_kstat_create(cp
);
1933 cpu_create_intrstat(cp
);
1934 lgrp_kstat_create(cp
);
1935 cpu_state_change_notify(cp
->cpu_id
, CPU_INIT
);
1940 * Remove a CPU from the list of active CPUs.
1941 * This routine must not get any locks, because other CPUs are paused.
1945 cpu_remove_active(cpu_t
*cp
)
1947 cpupart_t
*pp
= cp
->cpu_part
;
1949 ASSERT(MUTEX_HELD(&cpu_lock
));
1950 ASSERT(cp
->cpu_next_onln
!= cp
); /* not the last one */
1951 ASSERT(cp
->cpu_prev_onln
!= cp
); /* not the last one */
1953 pg_cpu_inactive(cp
);
1955 lgrp_config(LGRP_CONFIG_CPU_OFFLINE
, (uintptr_t)cp
, 0);
1957 if (cp
== clock_cpu_list
)
1958 clock_cpu_list
= cp
->cpu_next_onln
;
1960 cp
->cpu_prev_onln
->cpu_next_onln
= cp
->cpu_next_onln
;
1961 cp
->cpu_next_onln
->cpu_prev_onln
= cp
->cpu_prev_onln
;
1962 if (cpu_active
== cp
) {
1963 cpu_active
= cp
->cpu_next_onln
;
1965 cp
->cpu_next_onln
= cp
;
1966 cp
->cpu_prev_onln
= cp
;
1968 cp
->cpu_prev_part
->cpu_next_part
= cp
->cpu_next_part
;
1969 cp
->cpu_next_part
->cpu_prev_part
= cp
->cpu_prev_part
;
1970 if (pp
->cp_cpulist
== cp
) {
1971 pp
->cp_cpulist
= cp
->cpu_next_part
;
1972 ASSERT(pp
->cp_cpulist
!= cp
);
1974 cp
->cpu_next_part
= cp
;
1975 cp
->cpu_prev_part
= cp
;
1977 if (pp
->cp_ncpus
== 0) {
1978 cp_numparts_nonempty
--;
1979 ASSERT(cp_numparts_nonempty
!= 0);
1984 * Routine used to setup a newly inserted CPU in preparation for starting
1988 cpu_configure(int cpuid
)
1992 ASSERT(MUTEX_HELD(&cpu_lock
));
1995 * Some structures are statically allocated based upon
1996 * the maximum number of cpus the system supports. Do not
1997 * try to add anything beyond this limit.
1999 if (cpuid
< 0 || cpuid
>= NCPU
) {
2003 if ((cpu
[cpuid
] != NULL
) && (cpu
[cpuid
]->cpu_flags
!= 0)) {
2007 if ((retval
= mp_cpu_configure(cpuid
)) != 0) {
2011 cpu
[cpuid
]->cpu_flags
= CPU_QUIESCED
| CPU_OFFLINE
| CPU_POWEROFF
;
2012 cpu_set_state(cpu
[cpuid
]);
2013 retval
= cpu_state_change_hooks(cpuid
, CPU_CONFIG
, CPU_UNCONFIG
);
2015 (void) mp_cpu_unconfigure(cpuid
);
2021 * Routine used to cleanup a CPU that has been powered off. This will
2022 * destroy all per-cpu information related to this cpu.
2025 cpu_unconfigure(int cpuid
)
2029 ASSERT(MUTEX_HELD(&cpu_lock
));
2031 if (cpu
[cpuid
] == NULL
) {
2035 if (cpu
[cpuid
]->cpu_flags
== 0) {
2039 if ((cpu
[cpuid
]->cpu_flags
& CPU_POWEROFF
) == 0) {
2043 if (cpu
[cpuid
]->cpu_props
!= NULL
) {
2044 (void) nvlist_free(cpu
[cpuid
]->cpu_props
);
2045 cpu
[cpuid
]->cpu_props
= NULL
;
2048 error
= cpu_state_change_hooks(cpuid
, CPU_UNCONFIG
, CPU_CONFIG
);
2053 return (mp_cpu_unconfigure(cpuid
));
2057 * Routines for registering and de-registering cpu_setup callback functions.
2060 * These routines must not be called from a driver's attach(9E) or
2061 * detach(9E) entry point.
2063 * NOTE: CPU callbacks should not block. They are called with cpu_lock held.
2067 * Ideally, these would be dynamically allocated and put into a linked
2068 * list; however that is not feasible because the registration routine
2069 * has to be available before the kmem allocator is working (in fact,
2070 * it is called by the kmem allocator init code). In any case, there
2071 * are quite a few extra entries for future users.
2073 #define NCPU_SETUPS 20
2076 cpu_setup_func_t
*func
;
2078 } cpu_setups
[NCPU_SETUPS
];
2081 register_cpu_setup_func(cpu_setup_func_t
*func
, void *arg
)
2085 ASSERT(MUTEX_HELD(&cpu_lock
));
2087 for (i
= 0; i
< NCPU_SETUPS
; i
++)
2088 if (cpu_setups
[i
].func
== NULL
)
2090 if (i
>= NCPU_SETUPS
)
2091 cmn_err(CE_PANIC
, "Ran out of cpu_setup callback entries");
2093 cpu_setups
[i
].func
= func
;
2094 cpu_setups
[i
].arg
= arg
;
2098 unregister_cpu_setup_func(cpu_setup_func_t
*func
, void *arg
)
2102 ASSERT(MUTEX_HELD(&cpu_lock
));
2104 for (i
= 0; i
< NCPU_SETUPS
; i
++)
2105 if ((cpu_setups
[i
].func
== func
) &&
2106 (cpu_setups
[i
].arg
== arg
))
2108 if (i
>= NCPU_SETUPS
)
2109 cmn_err(CE_PANIC
, "Could not find cpu_setup callback to "
2112 cpu_setups
[i
].func
= NULL
;
2113 cpu_setups
[i
].arg
= 0;
2117 * Call any state change hooks for this CPU, ignore any errors.
2120 cpu_state_change_notify(int id
, cpu_setup_t what
)
2124 ASSERT(MUTEX_HELD(&cpu_lock
));
2126 for (i
= 0; i
< NCPU_SETUPS
; i
++) {
2127 if (cpu_setups
[i
].func
!= NULL
) {
2128 cpu_setups
[i
].func(what
, id
, cpu_setups
[i
].arg
);
2134 * Call any state change hooks for this CPU, undo it if error found.
2137 cpu_state_change_hooks(int id
, cpu_setup_t what
, cpu_setup_t undo
)
2142 ASSERT(MUTEX_HELD(&cpu_lock
));
2144 for (i
= 0; i
< NCPU_SETUPS
; i
++) {
2145 if (cpu_setups
[i
].func
!= NULL
) {
2146 retval
= cpu_setups
[i
].func(what
, id
,
2149 for (i
--; i
>= 0; i
--) {
2150 if (cpu_setups
[i
].func
!= NULL
)
2151 cpu_setups
[i
].func(undo
,
2152 id
, cpu_setups
[i
].arg
);
2162 * Export information about this CPU via the kstat mechanism.
2165 kstat_named_t ci_state
;
2166 kstat_named_t ci_state_begin
;
2167 kstat_named_t ci_cpu_type
;
2168 kstat_named_t ci_fpu_type
;
2169 kstat_named_t ci_clock_MHz
;
2170 kstat_named_t ci_chip_id
;
2171 kstat_named_t ci_implementation
;
2172 kstat_named_t ci_brandstr
;
2173 kstat_named_t ci_core_id
;
2174 kstat_named_t ci_curr_clock_Hz
;
2175 kstat_named_t ci_supp_freq_Hz
;
2176 kstat_named_t ci_pg_id
;
2177 #if defined(__sparcv9)
2178 kstat_named_t ci_device_ID
;
2179 kstat_named_t ci_cpu_fru
;
2182 kstat_named_t ci_vendorstr
;
2183 kstat_named_t ci_family
;
2184 kstat_named_t ci_model
;
2185 kstat_named_t ci_step
;
2186 kstat_named_t ci_clogid
;
2187 kstat_named_t ci_pkg_core_id
;
2188 kstat_named_t ci_ncpuperchip
;
2189 kstat_named_t ci_ncoreperchip
;
2190 kstat_named_t ci_max_cstates
;
2191 kstat_named_t ci_curr_cstate
;
2192 kstat_named_t ci_cacheid
;
2193 kstat_named_t ci_sktstr
;
2195 } cpu_info_template
= {
2196 { "state", KSTAT_DATA_CHAR
},
2197 { "state_begin", KSTAT_DATA_LONG
},
2198 { "cpu_type", KSTAT_DATA_CHAR
},
2199 { "fpu_type", KSTAT_DATA_CHAR
},
2200 { "clock_MHz", KSTAT_DATA_LONG
},
2201 { "chip_id", KSTAT_DATA_LONG
},
2202 { "implementation", KSTAT_DATA_STRING
},
2203 { "brand", KSTAT_DATA_STRING
},
2204 { "core_id", KSTAT_DATA_LONG
},
2205 { "current_clock_Hz", KSTAT_DATA_UINT64
},
2206 { "supported_frequencies_Hz", KSTAT_DATA_STRING
},
2207 { "pg_id", KSTAT_DATA_LONG
},
2208 #if defined(__sparcv9)
2209 { "device_ID", KSTAT_DATA_UINT64
},
2210 { "cpu_fru", KSTAT_DATA_STRING
},
2213 { "vendor_id", KSTAT_DATA_STRING
},
2214 { "family", KSTAT_DATA_INT32
},
2215 { "model", KSTAT_DATA_INT32
},
2216 { "stepping", KSTAT_DATA_INT32
},
2217 { "clog_id", KSTAT_DATA_INT32
},
2218 { "pkg_core_id", KSTAT_DATA_LONG
},
2219 { "ncpu_per_chip", KSTAT_DATA_INT32
},
2220 { "ncore_per_chip", KSTAT_DATA_INT32
},
2221 { "supported_max_cstates", KSTAT_DATA_INT32
},
2222 { "current_cstate", KSTAT_DATA_INT32
},
2223 { "cache_id", KSTAT_DATA_INT32
},
2224 { "socket_type", KSTAT_DATA_STRING
},
2228 static kmutex_t cpu_info_template_lock
;
2231 cpu_info_kstat_update(kstat_t
*ksp
, int rw
)
2233 cpu_t
*cp
= ksp
->ks_private
;
2234 const char *pi_state
;
2236 if (rw
== KSTAT_WRITE
)
2240 /* Is the cpu still initialising itself? */
2241 if (cpuid_checkpass(cp
, 1) == 0)
2244 switch (cp
->cpu_type_info
.pi_state
) {
2246 pi_state
= PS_ONLINE
;
2249 pi_state
= PS_POWEROFF
;
2252 pi_state
= PS_NOINTR
;
2255 pi_state
= PS_FAULTED
;
2258 pi_state
= PS_SPARE
;
2261 pi_state
= PS_OFFLINE
;
2264 pi_state
= "unknown";
2266 (void) strcpy(cpu_info_template
.ci_state
.value
.c
, pi_state
);
2267 cpu_info_template
.ci_state_begin
.value
.l
= cp
->cpu_state_begin
;
2268 (void) strncpy(cpu_info_template
.ci_cpu_type
.value
.c
,
2269 cp
->cpu_type_info
.pi_processor_type
, 15);
2270 (void) strncpy(cpu_info_template
.ci_fpu_type
.value
.c
,
2271 cp
->cpu_type_info
.pi_fputypes
, 15);
2272 cpu_info_template
.ci_clock_MHz
.value
.l
= cp
->cpu_type_info
.pi_clock
;
2273 cpu_info_template
.ci_chip_id
.value
.l
=
2274 pg_plat_hw_instance_id(cp
, PGHW_CHIP
);
2275 kstat_named_setstr(&cpu_info_template
.ci_implementation
,
2277 kstat_named_setstr(&cpu_info_template
.ci_brandstr
, cp
->cpu_brandstr
);
2278 cpu_info_template
.ci_core_id
.value
.l
= pg_plat_get_core_id(cp
);
2279 cpu_info_template
.ci_curr_clock_Hz
.value
.ui64
=
2281 cpu_info_template
.ci_pg_id
.value
.l
=
2282 cp
->cpu_pg
&& cp
->cpu_pg
->cmt_lineage
?
2283 cp
->cpu_pg
->cmt_lineage
->pg_id
: -1;
2284 kstat_named_setstr(&cpu_info_template
.ci_supp_freq_Hz
,
2285 cp
->cpu_supp_freqs
);
2286 #if defined(__sparcv9)
2287 cpu_info_template
.ci_device_ID
.value
.ui64
=
2288 cpunodes
[cp
->cpu_id
].device_id
;
2289 kstat_named_setstr(&cpu_info_template
.ci_cpu_fru
, cpu_fru_fmri(cp
));
2292 kstat_named_setstr(&cpu_info_template
.ci_vendorstr
,
2293 cpuid_getvendorstr(cp
));
2294 cpu_info_template
.ci_family
.value
.l
= cpuid_getfamily(cp
);
2295 cpu_info_template
.ci_model
.value
.l
= cpuid_getmodel(cp
);
2296 cpu_info_template
.ci_step
.value
.l
= cpuid_getstep(cp
);
2297 cpu_info_template
.ci_clogid
.value
.l
= cpuid_get_clogid(cp
);
2298 cpu_info_template
.ci_ncpuperchip
.value
.l
= cpuid_get_ncpu_per_chip(cp
);
2299 cpu_info_template
.ci_ncoreperchip
.value
.l
=
2300 cpuid_get_ncore_per_chip(cp
);
2301 cpu_info_template
.ci_pkg_core_id
.value
.l
= cpuid_get_pkgcoreid(cp
);
2302 cpu_info_template
.ci_max_cstates
.value
.l
= cp
->cpu_m
.max_cstates
;
2303 cpu_info_template
.ci_curr_cstate
.value
.l
= cpu_idle_get_cpu_state(cp
);
2304 cpu_info_template
.ci_cacheid
.value
.i32
= cpuid_get_cacheid(cp
);
2305 kstat_named_setstr(&cpu_info_template
.ci_sktstr
,
2306 cpuid_getsocketstr(cp
));
2313 cpu_info_kstat_create(cpu_t
*cp
)
2317 ASSERT(MUTEX_HELD(&cpu_lock
));
2319 if (pool_pset_enabled())
2320 zoneid
= GLOBAL_ZONEID
;
2323 if ((cp
->cpu_info_kstat
= kstat_create_zone("cpu_info", cp
->cpu_id
,
2324 NULL
, "misc", KSTAT_TYPE_NAMED
,
2325 sizeof (cpu_info_template
) / sizeof (kstat_named_t
),
2326 KSTAT_FLAG_VIRTUAL
| KSTAT_FLAG_VAR_SIZE
, zoneid
)) != NULL
) {
2327 cp
->cpu_info_kstat
->ks_data_size
+= 2 * CPU_IDSTRLEN
;
2328 #if defined(__sparcv9)
2329 cp
->cpu_info_kstat
->ks_data_size
+=
2330 strlen(cpu_fru_fmri(cp
)) + 1;
2333 cp
->cpu_info_kstat
->ks_data_size
+= X86_VENDOR_STRLEN
;
2335 if (cp
->cpu_supp_freqs
!= NULL
)
2336 cp
->cpu_info_kstat
->ks_data_size
+=
2337 strlen(cp
->cpu_supp_freqs
) + 1;
2338 cp
->cpu_info_kstat
->ks_lock
= &cpu_info_template_lock
;
2339 cp
->cpu_info_kstat
->ks_data
= &cpu_info_template
;
2340 cp
->cpu_info_kstat
->ks_private
= cp
;
2341 cp
->cpu_info_kstat
->ks_update
= cpu_info_kstat_update
;
2342 kstat_install(cp
->cpu_info_kstat
);
2347 cpu_info_kstat_destroy(cpu_t
*cp
)
2349 ASSERT(MUTEX_HELD(&cpu_lock
));
2351 kstat_delete(cp
->cpu_info_kstat
);
2352 cp
->cpu_info_kstat
= NULL
;
2356 * Create and install kstats for the boot CPU.
2359 cpu_kstat_init(cpu_t
*cp
)
2361 mutex_enter(&cpu_lock
);
2362 cpu_info_kstat_create(cp
);
2363 cpu_stats_kstat_create(cp
);
2364 cpu_create_intrstat(cp
);
2366 mutex_exit(&cpu_lock
);
2370 * Make visible to the zone that subset of the cpu information that would be
2371 * initialized when a cpu is configured (but still offline).
2374 cpu_visibility_configure(cpu_t
*cp
, zone_t
*zone
)
2376 zoneid_t zoneid
= zone
? zone
->zone_id
: ALL_ZONES
;
2378 ASSERT(MUTEX_HELD(&cpu_lock
));
2379 ASSERT(pool_pset_enabled());
2382 if (zoneid
!= ALL_ZONES
&& zoneid
!= GLOBAL_ZONEID
) {
2384 ASSERT(zone
->zone_ncpus
<= ncpus
);
2386 if (cp
->cpu_info_kstat
!= NULL
)
2387 kstat_zone_add(cp
->cpu_info_kstat
, zoneid
);
2391 * Make visible to the zone that subset of the cpu information that would be
2392 * initialized when a previously configured cpu is onlined.
2395 cpu_visibility_online(cpu_t
*cp
, zone_t
*zone
)
2398 char name
[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */
2399 zoneid_t zoneid
= zone
? zone
->zone_id
: ALL_ZONES
;
2402 ASSERT(MUTEX_HELD(&cpu_lock
));
2403 ASSERT(pool_pset_enabled());
2405 ASSERT(cpu_is_active(cp
));
2408 if (zoneid
!= ALL_ZONES
&& zoneid
!= GLOBAL_ZONEID
) {
2409 zone
->zone_ncpus_online
++;
2410 ASSERT(zone
->zone_ncpus_online
<= ncpus_online
);
2412 (void) snprintf(name
, sizeof (name
), "cpu_stat%d", cpun
);
2413 if ((ksp
= kstat_hold_byname("cpu_stat", cpun
, name
, ALL_ZONES
))
2415 kstat_zone_add(ksp
, zoneid
);
2418 if ((ksp
= kstat_hold_byname("cpu", cpun
, "sys", ALL_ZONES
)) != NULL
) {
2419 kstat_zone_add(ksp
, zoneid
);
2422 if ((ksp
= kstat_hold_byname("cpu", cpun
, "vm", ALL_ZONES
)) != NULL
) {
2423 kstat_zone_add(ksp
, zoneid
);
2426 if ((ksp
= kstat_hold_byname("cpu", cpun
, "intrstat", ALL_ZONES
)) !=
2428 kstat_zone_add(ksp
, zoneid
);
2434 * Update relevant kstats such that cpu is now visible to processes
2435 * executing in specified zone.
2438 cpu_visibility_add(cpu_t
*cp
, zone_t
*zone
)
2440 cpu_visibility_configure(cp
, zone
);
2441 if (cpu_is_active(cp
))
2442 cpu_visibility_online(cp
, zone
);
2446 * Make invisible to the zone that subset of the cpu information that would be
2447 * torn down when a previously offlined cpu is unconfigured.
2450 cpu_visibility_unconfigure(cpu_t
*cp
, zone_t
*zone
)
2452 zoneid_t zoneid
= zone
? zone
->zone_id
: ALL_ZONES
;
2454 ASSERT(MUTEX_HELD(&cpu_lock
));
2455 ASSERT(pool_pset_enabled());
2458 if (zoneid
!= ALL_ZONES
&& zoneid
!= GLOBAL_ZONEID
) {
2459 ASSERT(zone
->zone_ncpus
!= 0);
2462 if (cp
->cpu_info_kstat
)
2463 kstat_zone_remove(cp
->cpu_info_kstat
, zoneid
);
2467 * Make invisible to the zone that subset of the cpu information that would be
2468 * torn down when a cpu is offlined (but still configured).
2471 cpu_visibility_offline(cpu_t
*cp
, zone_t
*zone
)
2474 char name
[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */
2475 zoneid_t zoneid
= zone
? zone
->zone_id
: ALL_ZONES
;
2478 ASSERT(MUTEX_HELD(&cpu_lock
));
2479 ASSERT(pool_pset_enabled());
2481 ASSERT(cpu_is_active(cp
));
2484 if (zoneid
!= ALL_ZONES
&& zoneid
!= GLOBAL_ZONEID
) {
2485 ASSERT(zone
->zone_ncpus_online
!= 0);
2486 zone
->zone_ncpus_online
--;
2489 if ((ksp
= kstat_hold_byname("cpu", cpun
, "intrstat", ALL_ZONES
)) !=
2491 kstat_zone_remove(ksp
, zoneid
);
2494 if ((ksp
= kstat_hold_byname("cpu", cpun
, "vm", ALL_ZONES
)) != NULL
) {
2495 kstat_zone_remove(ksp
, zoneid
);
2498 if ((ksp
= kstat_hold_byname("cpu", cpun
, "sys", ALL_ZONES
)) != NULL
) {
2499 kstat_zone_remove(ksp
, zoneid
);
2502 (void) snprintf(name
, sizeof (name
), "cpu_stat%d", cpun
);
2503 if ((ksp
= kstat_hold_byname("cpu_stat", cpun
, name
, ALL_ZONES
))
2505 kstat_zone_remove(ksp
, zoneid
);
2511 * Update relevant kstats such that cpu is no longer visible to processes
2512 * executing in specified zone.
2515 cpu_visibility_remove(cpu_t
*cp
, zone_t
*zone
)
2517 if (cpu_is_active(cp
))
2518 cpu_visibility_offline(cp
, zone
);
2519 cpu_visibility_unconfigure(cp
, zone
);
2523 * Bind a thread to a CPU as requested.
2526 cpu_bind_thread(kthread_id_t tp
, processorid_t bind
, processorid_t
*obind
,
2529 processorid_t binding
;
2532 ASSERT(MUTEX_HELD(&cpu_lock
));
2533 ASSERT(MUTEX_HELD(&ttoproc(tp
)->p_lock
));
2538 * Record old binding, but change the obind, which was initialized
2539 * to PBIND_NONE, only if this thread has a binding. This avoids
2540 * reporting PBIND_NONE for a process when some LWPs are bound.
2542 binding
= tp
->t_bind_cpu
;
2543 if (binding
!= PBIND_NONE
)
2544 *obind
= binding
; /* record old binding */
2548 /* Just return the old binding */
2552 case PBIND_QUERY_TYPE
:
2553 /* Return the binding type */
2554 *obind
= TB_CPU_IS_SOFT(tp
) ? PBIND_SOFT
: PBIND_HARD
;
2560 * Set soft binding for this thread and return the actual
2563 TB_CPU_SOFT_SET(tp
);
2569 * Set hard binding for this thread and return the actual
2572 TB_CPU_HARD_SET(tp
);
2581 * If this thread/LWP cannot be bound because of permission
2582 * problems, just note that and return success so that the
2583 * other threads/LWPs will be bound. This is the way
2584 * processor_bind() is defined to work.
2586 * Binding will get EPERM if the thread is of system class
2587 * or hasprocperm() fails.
2589 if (tp
->t_cid
== 0 || !hasprocperm(tp
->t_cred
, CRED())) {
2596 if (binding
!= PBIND_NONE
) {
2597 cp
= cpu_get((processorid_t
)binding
);
2599 * Make sure binding is valid and is in right partition.
2601 if (cp
== NULL
|| tp
->t_cpupart
!= cp
->cpu_part
) {
2607 tp
->t_bind_cpu
= binding
; /* set new binding */
2610 * If there is no system-set reason for affinity, set
2611 * the t_bound_cpu field to reflect the binding.
2613 if (tp
->t_affinitycnt
== 0) {
2614 if (binding
== PBIND_NONE
) {
2616 * We may need to adjust disp_max_unbound_pri
2617 * since we're becoming unbound.
2619 disp_adjust_unbound_pri(tp
);
2621 tp
->t_bound_cpu
= NULL
; /* set new binding */
2624 * Move thread to lgroup with strongest affinity
2627 if (tp
->t_lgrp_affinity
)
2628 lgrp_move_thread(tp
,
2629 lgrp_choose(tp
, tp
->t_cpupart
), 1);
2631 if (tp
->t_state
== TS_ONPROC
&&
2632 tp
->t_cpu
->cpu_part
!= tp
->t_cpupart
)
2637 tp
->t_bound_cpu
= cp
;
2638 ASSERT(cp
->cpu_lpl
!= NULL
);
2641 * Set home to lgroup with most affinity containing CPU
2642 * that thread is being bound or minimum bounding
2643 * lgroup if no affinities set
2645 if (tp
->t_lgrp_affinity
)
2646 lpl
= lgrp_affinity_best(tp
, tp
->t_cpupart
,
2647 LGRP_NONE
, B_FALSE
);
2651 if (tp
->t_lpl
!= lpl
) {
2652 /* can't grab cpu_lock */
2653 lgrp_move_thread(tp
, lpl
, 1);
2657 * Make the thread switch to the bound CPU.
2658 * If the thread is runnable, we need to
2659 * requeue it even if t_cpu is already set
2660 * to the right CPU, since it may be on a
2661 * kpreempt queue and need to move to a local
2662 * queue. We could check t_disp_queue to
2663 * avoid unnecessary overhead if it's already
2664 * on the right queue, but since this isn't
2665 * a performance-critical operation it doesn't
2666 * seem worth the extra code and complexity.
2668 * If the thread is weakbound to the cpu then it will
2669 * resist the new binding request until the weak
2670 * binding drops. The cpu_surrender or requeueing
2671 * below could be skipped in such cases (since it
2672 * will have no effect), but that would require
2673 * thread_allowmigrate to acquire thread_lock so
2674 * we'll take the very occasional hit here instead.
2676 if (tp
->t_state
== TS_ONPROC
) {
2678 } else if (tp
->t_state
== TS_RUN
) {
2679 cpu_t
*ocp
= tp
->t_cpu
;
2684 * Either on the bound CPU's disp queue now,
2685 * or swapped out or on the swap queue.
2687 ASSERT(tp
->t_disp_queue
== cp
->cpu_disp
||
2688 tp
->t_weakbound_cpu
== ocp
||
2689 (tp
->t_schedflag
& (TS_LOAD
| TS_ON_SWAPQ
))
2696 * Our binding has changed; set TP_CHANGEBIND.
2698 tp
->t_proc_flag
|= TP_CHANGEBIND
;
2706 #if CPUSET_WORDS > 1
2709 * Functions for implementing cpuset operations when a cpuset is more
2710 * than one word. On platforms where a cpuset is a single word these
2711 * are implemented as macros in cpuvar.h.
2715 cpuset_all(cpuset_t
*s
)
2719 for (i
= 0; i
< CPUSET_WORDS
; i
++)
2724 cpuset_all_but(cpuset_t
*s
, uint_t cpu
)
2727 CPUSET_DEL(*s
, cpu
);
2731 cpuset_only(cpuset_t
*s
, uint_t cpu
)
2734 CPUSET_ADD(*s
, cpu
);
2738 cpuset_isnull(cpuset_t
*s
)
2742 for (i
= 0; i
< CPUSET_WORDS
; i
++)
2743 if (s
->cpub
[i
] != 0)
2749 cpuset_cmp(cpuset_t
*s1
, cpuset_t
*s2
)
2753 for (i
= 0; i
< CPUSET_WORDS
; i
++)
2754 if (s1
->cpub
[i
] != s2
->cpub
[i
])
2760 cpuset_find(cpuset_t
*s
)
2764 uint_t cpu
= (uint_t
)-1;
2767 * Find a cpu in the cpuset
2769 for (i
= 0; i
< CPUSET_WORDS
; i
++) {
2770 cpu
= (uint_t
)(lowbit(s
->cpub
[i
]) - 1);
2771 if (cpu
!= (uint_t
)-1) {
2772 cpu
+= i
* BT_NBIPUL
;
2780 cpuset_bounds(cpuset_t
*s
, uint_t
*smallestid
, uint_t
*largestid
)
2786 * First, find the smallest cpu id in the set.
2788 for (i
= 0; i
< CPUSET_WORDS
; i
++) {
2789 if (s
->cpub
[i
] != 0) {
2790 bit
= (uint_t
)(lowbit(s
->cpub
[i
]) - 1);
2791 ASSERT(bit
!= (uint_t
)-1);
2792 *smallestid
= bit
+ (i
* BT_NBIPUL
);
2795 * Now find the largest cpu id in
2796 * the set and return immediately.
2797 * Done in an inner loop to avoid
2798 * having to break out of the first
2801 for (j
= CPUSET_WORDS
- 1; j
>= i
; j
--) {
2802 if (s
->cpub
[j
] != 0) {
2803 bit
= (uint_t
)(highbit(s
->cpub
[j
]) - 1);
2804 ASSERT(bit
!= (uint_t
)-1);
2805 *largestid
= bit
+ (j
* BT_NBIPUL
);
2806 ASSERT(*largestid
>= *smallestid
);
2812 * If this code is reached, a
2813 * smallestid was found, but not a
2814 * largestid. The cpuset must have
2815 * been changed during the course
2816 * of this function call.
2821 *smallestid
= *largestid
= CPUSET_NOTINSET
;
2824 #endif /* CPUSET_WORDS */
2827 * Unbind threads bound to specified CPU.
2829 * If `unbind_all_threads' is true, unbind all user threads bound to a given
2830 * CPU. Otherwise unbind all soft-bound user threads.
2833 cpu_unbind(processorid_t cpu
, boolean_t unbind_all_threads
)
2835 processorid_t obind
;
2841 ASSERT(MUTEX_HELD(&cpu_lock
));
2843 mutex_enter(&pidlock
);
2844 for (pp
= practive
; pp
!= NULL
; pp
= pp
->p_next
) {
2845 mutex_enter(&pp
->p_lock
);
2848 * Skip zombies, kernel processes, and processes in
2849 * other zones, if called from a non-global zone.
2851 if (tp
== NULL
|| (pp
->p_flag
& SSYS
) ||
2852 !HASZONEACCESS(curproc
, pp
->p_zone
->zone_id
)) {
2853 mutex_exit(&pp
->p_lock
);
2857 if (tp
->t_bind_cpu
!= cpu
)
2860 * Skip threads with hard binding when
2861 * `unbind_all_threads' is not specified.
2863 if (!unbind_all_threads
&& TB_CPU_IS_HARD(tp
))
2865 err
= cpu_bind_thread(tp
, PBIND_NONE
, &obind
, &berr
);
2868 } while ((tp
= tp
->t_forw
) != pp
->p_tlist
);
2869 mutex_exit(&pp
->p_lock
);
2871 mutex_exit(&pidlock
);
2879 * Destroy all remaining bound threads on a cpu.
2882 cpu_destroy_bound_threads(cpu_t
*cp
)
2885 register kthread_id_t t
, tlist
, tnext
;
2888 * Destroy all remaining bound threads on the cpu. This
2889 * should include both the interrupt threads and the idle thread.
2890 * This requires some care, since we need to traverse the
2891 * thread list with the pidlock mutex locked, but thread_free
2892 * also locks the pidlock mutex. So, we collect the threads
2893 * we're going to reap in a list headed by "tlist", then we
2894 * unlock the pidlock mutex and traverse the tlist list,
2895 * doing thread_free's on the thread's. Simple, n'est pas?
2896 * Also, this depends on thread_free not mucking with the
2897 * t_next and t_prev links of the thread.
2900 if ((t
= curthread
) != NULL
) {
2903 mutex_enter(&pidlock
);
2906 if (t
->t_bound_cpu
== cp
) {
2909 * We've found a bound thread, carefully unlink
2910 * it out of the thread list, and add it to
2911 * our "tlist". We "know" we don't have to
2912 * worry about unlinking curthread (the thread
2913 * that is executing this code).
2915 t
->t_next
->t_prev
= t
->t_prev
;
2916 t
->t_prev
->t_next
= t
->t_next
;
2919 ASSERT(t
->t_cid
== syscid
);
2920 /* wake up anyone blocked in thread_join */
2921 cv_broadcast(&t
->t_joincv
);
2923 * t_lwp set by interrupt threads and not
2928 * Pause and idle threads always have
2929 * t_state set to TS_ONPROC.
2931 t
->t_state
= TS_FREE
;
2932 t
->t_prev
= NULL
; /* Just in case */
2935 } while ((t
= tnext
) != curthread
);
2937 mutex_exit(&pidlock
);
2940 for (t
= tlist
; t
!= NULL
; t
= tnext
) {
2948 * Update the cpu_supp_freqs of this cpu. This information is returned
2949 * as part of cpu_info kstats. If the cpu_info_kstat exists already, then
2950 * maintain the kstat data size.
2953 cpu_set_supp_freqs(cpu_t
*cp
, const char *freqs
)
2955 char clkstr
[sizeof ("18446744073709551615") + 1]; /* ui64 MAX */
2956 const char *lfreqs
= clkstr
;
2957 boolean_t kstat_exists
= B_FALSE
;
2962 * A NULL pointer means we only support one speed.
2965 (void) snprintf(clkstr
, sizeof (clkstr
), "%"PRIu64
,
2966 cp
->cpu_curr_clock
);
2971 * Make sure the frequency doesn't change while a snapshot is
2972 * going on. Of course, we only need to worry about this if
2975 if ((ksp
= cp
->cpu_info_kstat
) != NULL
) {
2976 mutex_enter(ksp
->ks_lock
);
2977 kstat_exists
= B_TRUE
;
2981 * Free any previously allocated string and if the kstat
2982 * already exists, then update its data size.
2984 if (cp
->cpu_supp_freqs
!= NULL
) {
2985 len
= strlen(cp
->cpu_supp_freqs
) + 1;
2986 kmem_free(cp
->cpu_supp_freqs
, len
);
2988 ksp
->ks_data_size
-= len
;
2992 * Allocate the new string and set the pointer.
2994 len
= strlen(lfreqs
) + 1;
2995 cp
->cpu_supp_freqs
= kmem_alloc(len
, KM_SLEEP
);
2996 (void) strcpy(cp
->cpu_supp_freqs
, lfreqs
);
2999 * If the kstat already exists then update the data size and
3003 ksp
->ks_data_size
+= len
;
3004 mutex_exit(ksp
->ks_lock
);
3009 * Indicate the current CPU's clock freqency (in Hz).
3010 * The calling context must be such that CPU references are safe.
3013 cpu_set_curr_clock(uint64_t new_clk
)
3017 old_clk
= CPU
->cpu_curr_clock
;
3018 CPU
->cpu_curr_clock
= new_clk
;
3021 * The cpu-change-speed DTrace probe exports the frequency in Hz
3023 DTRACE_PROBE3(cpu__change__speed
, processorid_t
, CPU
->cpu_id
,
3024 uint64_t, old_clk
, uint64_t, new_clk
);
3028 * processor_info(2) and p_online(2) status support functions
3029 * The constants returned by the cpu_get_state() and cpu_get_state_str() are
3030 * for use in communicating processor state information to userland. Kernel
3031 * subsystems should only be using the cpu_flags value directly. Subsystems
3032 * modifying cpu_flags should record the state change via a call to the
3037 * Update the pi_state of this CPU. This function provides the CPU status for
3038 * the information returned by processor_info(2).
3041 cpu_set_state(cpu_t
*cpu
)
3043 ASSERT(MUTEX_HELD(&cpu_lock
));
3044 cpu
->cpu_type_info
.pi_state
= cpu_get_state(cpu
);
3045 cpu
->cpu_state_begin
= gethrestime_sec();
3046 pool_cpu_mod
= gethrtime();
3050 * Return offline/online/other status for the indicated CPU. Use only for
3051 * communication with user applications; cpu_flags provides the in-kernel
3055 cpu_get_state(cpu_t
*cpu
)
3057 ASSERT(MUTEX_HELD(&cpu_lock
));
3058 if (cpu
->cpu_flags
& CPU_POWEROFF
)
3059 return (P_POWEROFF
);
3060 else if (cpu
->cpu_flags
& CPU_FAULTED
)
3062 else if (cpu
->cpu_flags
& CPU_SPARE
)
3064 else if ((cpu
->cpu_flags
& (CPU_READY
| CPU_OFFLINE
)) != CPU_READY
)
3066 else if (cpu
->cpu_flags
& CPU_ENABLE
)
3073 * Return processor_info(2) state as a string.
3076 cpu_get_state_str(cpu_t
*cpu
)
3080 switch (cpu_get_state(cpu
)) {
3085 string
= PS_POWEROFF
;
3094 string
= PS_FAULTED
;
3097 string
= PS_OFFLINE
;
3107 * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named
3108 * kstats, respectively. This is done when a CPU is initialized or placed
3109 * online via p_online(2).
3112 cpu_stats_kstat_create(cpu_t
*cp
)
3114 int instance
= cp
->cpu_id
;
3115 char *module
= "cpu";
3116 char *class = "misc";
3120 ASSERT(MUTEX_HELD(&cpu_lock
));
3122 if (pool_pset_enabled())
3123 zoneid
= GLOBAL_ZONEID
;
3127 * Create named kstats
3129 #define CPU_STATS_KS_CREATE(name, tsize, update_func) \
3130 ksp = kstat_create_zone(module, instance, (name), class, \
3131 KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0, \
3133 if (ksp != NULL) { \
3134 ksp->ks_private = cp; \
3135 ksp->ks_update = (update_func); \
3136 kstat_install(ksp); \
3138 cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \
3139 module, instance, (name));
3141 CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template
),
3142 cpu_sys_stats_ks_update
);
3143 CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template
),
3144 cpu_vm_stats_ks_update
);
3147 * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat.
3149 ksp
= kstat_create_zone("cpu_stat", cp
->cpu_id
, NULL
,
3150 "misc", KSTAT_TYPE_RAW
, sizeof (cpu_stat_t
), 0, zoneid
);
3152 ksp
->ks_update
= cpu_stat_ks_update
;
3153 ksp
->ks_private
= cp
;
3159 cpu_stats_kstat_destroy(cpu_t
*cp
)
3161 char ks_name
[KSTAT_STRLEN
];
3163 (void) sprintf(ks_name
, "cpu_stat%d", cp
->cpu_id
);
3164 kstat_delete_byname("cpu_stat", cp
->cpu_id
, ks_name
);
3166 kstat_delete_byname("cpu", cp
->cpu_id
, "sys");
3167 kstat_delete_byname("cpu", cp
->cpu_id
, "vm");
3171 cpu_sys_stats_ks_update(kstat_t
*ksp
, int rw
)
3173 cpu_t
*cp
= (cpu_t
*)ksp
->ks_private
;
3174 struct cpu_sys_stats_ks_data
*csskd
;
3175 cpu_sys_stats_t
*css
;
3176 hrtime_t msnsecs
[NCMSTATES
];
3179 if (rw
== KSTAT_WRITE
)
3182 csskd
= ksp
->ks_data
;
3183 css
= &cp
->cpu_stats
.sys
;
3186 * Read CPU mstate, but compare with the last values we
3187 * received to make sure that the returned kstats never
3191 get_cpu_mstate(cp
, msnsecs
);
3192 if (csskd
->cpu_nsec_idle
.value
.ui64
> msnsecs
[CMS_IDLE
])
3193 msnsecs
[CMS_IDLE
] = csskd
->cpu_nsec_idle
.value
.ui64
;
3194 if (csskd
->cpu_nsec_user
.value
.ui64
> msnsecs
[CMS_USER
])
3195 msnsecs
[CMS_USER
] = csskd
->cpu_nsec_user
.value
.ui64
;
3196 if (csskd
->cpu_nsec_kernel
.value
.ui64
> msnsecs
[CMS_SYSTEM
])
3197 msnsecs
[CMS_SYSTEM
] = csskd
->cpu_nsec_kernel
.value
.ui64
;
3199 bcopy(&cpu_sys_stats_ks_data_template
, ksp
->ks_data
,
3200 sizeof (cpu_sys_stats_ks_data_template
));
3202 csskd
->cpu_ticks_wait
.value
.ui64
= 0;
3203 csskd
->wait_ticks_io
.value
.ui64
= 0;
3205 csskd
->cpu_nsec_idle
.value
.ui64
= msnsecs
[CMS_IDLE
];
3206 csskd
->cpu_nsec_user
.value
.ui64
= msnsecs
[CMS_USER
];
3207 csskd
->cpu_nsec_kernel
.value
.ui64
= msnsecs
[CMS_SYSTEM
];
3208 csskd
->cpu_ticks_idle
.value
.ui64
=
3209 NSEC_TO_TICK(csskd
->cpu_nsec_idle
.value
.ui64
);
3210 csskd
->cpu_ticks_user
.value
.ui64
=
3211 NSEC_TO_TICK(csskd
->cpu_nsec_user
.value
.ui64
);
3212 csskd
->cpu_ticks_kernel
.value
.ui64
=
3213 NSEC_TO_TICK(csskd
->cpu_nsec_kernel
.value
.ui64
);
3214 csskd
->cpu_nsec_dtrace
.value
.ui64
= cp
->cpu_dtrace_nsec
;
3215 csskd
->dtrace_probes
.value
.ui64
= cp
->cpu_dtrace_probes
;
3216 csskd
->cpu_nsec_intr
.value
.ui64
= cp
->cpu_intrlast
;
3217 csskd
->cpu_load_intr
.value
.ui64
= cp
->cpu_intrload
;
3218 csskd
->bread
.value
.ui64
= css
->bread
;
3219 csskd
->bwrite
.value
.ui64
= css
->bwrite
;
3220 csskd
->lread
.value
.ui64
= css
->lread
;
3221 csskd
->lwrite
.value
.ui64
= css
->lwrite
;
3222 csskd
->phread
.value
.ui64
= css
->phread
;
3223 csskd
->phwrite
.value
.ui64
= css
->phwrite
;
3224 csskd
->pswitch
.value
.ui64
= css
->pswitch
;
3225 csskd
->trap
.value
.ui64
= css
->trap
;
3226 csskd
->intr
.value
.ui64
= 0;
3227 for (i
= 0; i
< PIL_MAX
; i
++)
3228 csskd
->intr
.value
.ui64
+= css
->intr
[i
];
3229 csskd
->syscall
.value
.ui64
= css
->syscall
;
3230 csskd
->sysread
.value
.ui64
= css
->sysread
;
3231 csskd
->syswrite
.value
.ui64
= css
->syswrite
;
3232 csskd
->sysfork
.value
.ui64
= css
->sysfork
;
3233 csskd
->sysvfork
.value
.ui64
= css
->sysvfork
;
3234 csskd
->sysexec
.value
.ui64
= css
->sysexec
;
3235 csskd
->readch
.value
.ui64
= css
->readch
;
3236 csskd
->writech
.value
.ui64
= css
->writech
;
3237 csskd
->rcvint
.value
.ui64
= css
->rcvint
;
3238 csskd
->xmtint
.value
.ui64
= css
->xmtint
;
3239 csskd
->mdmint
.value
.ui64
= css
->mdmint
;
3240 csskd
->rawch
.value
.ui64
= css
->rawch
;
3241 csskd
->canch
.value
.ui64
= css
->canch
;
3242 csskd
->outch
.value
.ui64
= css
->outch
;
3243 csskd
->msg
.value
.ui64
= css
->msg
;
3244 csskd
->sema
.value
.ui64
= css
->sema
;
3245 csskd
->namei
.value
.ui64
= css
->namei
;
3246 csskd
->ufsiget
.value
.ui64
= css
->ufsiget
;
3247 csskd
->ufsdirblk
.value
.ui64
= css
->ufsdirblk
;
3248 csskd
->ufsipage
.value
.ui64
= css
->ufsipage
;
3249 csskd
->ufsinopage
.value
.ui64
= css
->ufsinopage
;
3250 csskd
->procovf
.value
.ui64
= css
->procovf
;
3251 csskd
->intrthread
.value
.ui64
= 0;
3252 for (i
= 0; i
< LOCK_LEVEL
- 1; i
++)
3253 csskd
->intrthread
.value
.ui64
+= css
->intr
[i
];
3254 csskd
->intrblk
.value
.ui64
= css
->intrblk
;
3255 csskd
->intrunpin
.value
.ui64
= css
->intrunpin
;
3256 csskd
->idlethread
.value
.ui64
= css
->idlethread
;
3257 csskd
->inv_swtch
.value
.ui64
= css
->inv_swtch
;
3258 csskd
->nthreads
.value
.ui64
= css
->nthreads
;
3259 csskd
->cpumigrate
.value
.ui64
= css
->cpumigrate
;
3260 csskd
->xcalls
.value
.ui64
= css
->xcalls
;
3261 csskd
->mutex_adenters
.value
.ui64
= css
->mutex_adenters
;
3262 csskd
->rw_rdfails
.value
.ui64
= css
->rw_rdfails
;
3263 csskd
->rw_wrfails
.value
.ui64
= css
->rw_wrfails
;
3264 csskd
->modload
.value
.ui64
= css
->modload
;
3265 csskd
->modunload
.value
.ui64
= css
->modunload
;
3266 csskd
->bawrite
.value
.ui64
= css
->bawrite
;
3267 csskd
->iowait
.value
.ui64
= css
->iowait
;
3273 cpu_vm_stats_ks_update(kstat_t
*ksp
, int rw
)
3275 cpu_t
*cp
= (cpu_t
*)ksp
->ks_private
;
3276 struct cpu_vm_stats_ks_data
*cvskd
;
3277 cpu_vm_stats_t
*cvs
;
3279 if (rw
== KSTAT_WRITE
)
3282 cvs
= &cp
->cpu_stats
.vm
;
3283 cvskd
= ksp
->ks_data
;
3285 bcopy(&cpu_vm_stats_ks_data_template
, ksp
->ks_data
,
3286 sizeof (cpu_vm_stats_ks_data_template
));
3287 cvskd
->pgrec
.value
.ui64
= cvs
->pgrec
;
3288 cvskd
->pgfrec
.value
.ui64
= cvs
->pgfrec
;
3289 cvskd
->pgin
.value
.ui64
= cvs
->pgin
;
3290 cvskd
->pgpgin
.value
.ui64
= cvs
->pgpgin
;
3291 cvskd
->pgout
.value
.ui64
= cvs
->pgout
;
3292 cvskd
->pgpgout
.value
.ui64
= cvs
->pgpgout
;
3293 cvskd
->swapin
.value
.ui64
= cvs
->swapin
;
3294 cvskd
->pgswapin
.value
.ui64
= cvs
->pgswapin
;
3295 cvskd
->swapout
.value
.ui64
= cvs
->swapout
;
3296 cvskd
->pgswapout
.value
.ui64
= cvs
->pgswapout
;
3297 cvskd
->zfod
.value
.ui64
= cvs
->zfod
;
3298 cvskd
->dfree
.value
.ui64
= cvs
->dfree
;
3299 cvskd
->scan
.value
.ui64
= cvs
->scan
;
3300 cvskd
->rev
.value
.ui64
= cvs
->rev
;
3301 cvskd
->hat_fault
.value
.ui64
= cvs
->hat_fault
;
3302 cvskd
->as_fault
.value
.ui64
= cvs
->as_fault
;
3303 cvskd
->maj_fault
.value
.ui64
= cvs
->maj_fault
;
3304 cvskd
->cow_fault
.value
.ui64
= cvs
->cow_fault
;
3305 cvskd
->prot_fault
.value
.ui64
= cvs
->prot_fault
;
3306 cvskd
->softlock
.value
.ui64
= cvs
->softlock
;
3307 cvskd
->kernel_asflt
.value
.ui64
= cvs
->kernel_asflt
;
3308 cvskd
->pgrrun
.value
.ui64
= cvs
->pgrrun
;
3309 cvskd
->execpgin
.value
.ui64
= cvs
->execpgin
;
3310 cvskd
->execpgout
.value
.ui64
= cvs
->execpgout
;
3311 cvskd
->execfree
.value
.ui64
= cvs
->execfree
;
3312 cvskd
->anonpgin
.value
.ui64
= cvs
->anonpgin
;
3313 cvskd
->anonpgout
.value
.ui64
= cvs
->anonpgout
;
3314 cvskd
->anonfree
.value
.ui64
= cvs
->anonfree
;
3315 cvskd
->fspgin
.value
.ui64
= cvs
->fspgin
;
3316 cvskd
->fspgout
.value
.ui64
= cvs
->fspgout
;
3317 cvskd
->fsfree
.value
.ui64
= cvs
->fsfree
;
3323 cpu_stat_ks_update(kstat_t
*ksp
, int rw
)
3328 hrtime_t msnsecs
[NCMSTATES
];
3330 cso
= (cpu_stat_t
*)ksp
->ks_data
;
3331 cp
= (cpu_t
*)ksp
->ks_private
;
3333 if (rw
== KSTAT_WRITE
)
3337 * Read CPU mstate, but compare with the last values we
3338 * received to make sure that the returned kstats never
3342 get_cpu_mstate(cp
, msnsecs
);
3343 msnsecs
[CMS_IDLE
] = NSEC_TO_TICK(msnsecs
[CMS_IDLE
]);
3344 msnsecs
[CMS_USER
] = NSEC_TO_TICK(msnsecs
[CMS_USER
]);
3345 msnsecs
[CMS_SYSTEM
] = NSEC_TO_TICK(msnsecs
[CMS_SYSTEM
]);
3346 if (cso
->cpu_sysinfo
.cpu
[CPU_IDLE
] < msnsecs
[CMS_IDLE
])
3347 cso
->cpu_sysinfo
.cpu
[CPU_IDLE
] = msnsecs
[CMS_IDLE
];
3348 if (cso
->cpu_sysinfo
.cpu
[CPU_USER
] < msnsecs
[CMS_USER
])
3349 cso
->cpu_sysinfo
.cpu
[CPU_USER
] = msnsecs
[CMS_USER
];
3350 if (cso
->cpu_sysinfo
.cpu
[CPU_KERNEL
] < msnsecs
[CMS_SYSTEM
])
3351 cso
->cpu_sysinfo
.cpu
[CPU_KERNEL
] = msnsecs
[CMS_SYSTEM
];
3352 cso
->cpu_sysinfo
.cpu
[CPU_WAIT
] = 0;
3353 cso
->cpu_sysinfo
.wait
[W_IO
] = 0;
3354 cso
->cpu_sysinfo
.wait
[W_SWAP
] = 0;
3355 cso
->cpu_sysinfo
.wait
[W_PIO
] = 0;
3356 cso
->cpu_sysinfo
.bread
= CPU_STATS(cp
, sys
.bread
);
3357 cso
->cpu_sysinfo
.bwrite
= CPU_STATS(cp
, sys
.bwrite
);
3358 cso
->cpu_sysinfo
.lread
= CPU_STATS(cp
, sys
.lread
);
3359 cso
->cpu_sysinfo
.lwrite
= CPU_STATS(cp
, sys
.lwrite
);
3360 cso
->cpu_sysinfo
.phread
= CPU_STATS(cp
, sys
.phread
);
3361 cso
->cpu_sysinfo
.phwrite
= CPU_STATS(cp
, sys
.phwrite
);
3362 cso
->cpu_sysinfo
.pswitch
= CPU_STATS(cp
, sys
.pswitch
);
3363 cso
->cpu_sysinfo
.trap
= CPU_STATS(cp
, sys
.trap
);
3364 cso
->cpu_sysinfo
.intr
= 0;
3365 for (i
= 0; i
< PIL_MAX
; i
++)
3366 cso
->cpu_sysinfo
.intr
+= CPU_STATS(cp
, sys
.intr
[i
]);
3367 cso
->cpu_sysinfo
.syscall
= CPU_STATS(cp
, sys
.syscall
);
3368 cso
->cpu_sysinfo
.sysread
= CPU_STATS(cp
, sys
.sysread
);
3369 cso
->cpu_sysinfo
.syswrite
= CPU_STATS(cp
, sys
.syswrite
);
3370 cso
->cpu_sysinfo
.sysfork
= CPU_STATS(cp
, sys
.sysfork
);
3371 cso
->cpu_sysinfo
.sysvfork
= CPU_STATS(cp
, sys
.sysvfork
);
3372 cso
->cpu_sysinfo
.sysexec
= CPU_STATS(cp
, sys
.sysexec
);
3373 cso
->cpu_sysinfo
.readch
= CPU_STATS(cp
, sys
.readch
);
3374 cso
->cpu_sysinfo
.writech
= CPU_STATS(cp
, sys
.writech
);
3375 cso
->cpu_sysinfo
.rcvint
= CPU_STATS(cp
, sys
.rcvint
);
3376 cso
->cpu_sysinfo
.xmtint
= CPU_STATS(cp
, sys
.xmtint
);
3377 cso
->cpu_sysinfo
.mdmint
= CPU_STATS(cp
, sys
.mdmint
);
3378 cso
->cpu_sysinfo
.rawch
= CPU_STATS(cp
, sys
.rawch
);
3379 cso
->cpu_sysinfo
.canch
= CPU_STATS(cp
, sys
.canch
);
3380 cso
->cpu_sysinfo
.outch
= CPU_STATS(cp
, sys
.outch
);
3381 cso
->cpu_sysinfo
.msg
= CPU_STATS(cp
, sys
.msg
);
3382 cso
->cpu_sysinfo
.sema
= CPU_STATS(cp
, sys
.sema
);
3383 cso
->cpu_sysinfo
.namei
= CPU_STATS(cp
, sys
.namei
);
3384 cso
->cpu_sysinfo
.ufsiget
= CPU_STATS(cp
, sys
.ufsiget
);
3385 cso
->cpu_sysinfo
.ufsdirblk
= CPU_STATS(cp
, sys
.ufsdirblk
);
3386 cso
->cpu_sysinfo
.ufsipage
= CPU_STATS(cp
, sys
.ufsipage
);
3387 cso
->cpu_sysinfo
.ufsinopage
= CPU_STATS(cp
, sys
.ufsinopage
);
3388 cso
->cpu_sysinfo
.inodeovf
= 0;
3389 cso
->cpu_sysinfo
.fileovf
= 0;
3390 cso
->cpu_sysinfo
.procovf
= CPU_STATS(cp
, sys
.procovf
);
3391 cso
->cpu_sysinfo
.intrthread
= 0;
3392 for (i
= 0; i
< LOCK_LEVEL
- 1; i
++)
3393 cso
->cpu_sysinfo
.intrthread
+= CPU_STATS(cp
, sys
.intr
[i
]);
3394 cso
->cpu_sysinfo
.intrblk
= CPU_STATS(cp
, sys
.intrblk
);
3395 cso
->cpu_sysinfo
.idlethread
= CPU_STATS(cp
, sys
.idlethread
);
3396 cso
->cpu_sysinfo
.inv_swtch
= CPU_STATS(cp
, sys
.inv_swtch
);
3397 cso
->cpu_sysinfo
.nthreads
= CPU_STATS(cp
, sys
.nthreads
);
3398 cso
->cpu_sysinfo
.cpumigrate
= CPU_STATS(cp
, sys
.cpumigrate
);
3399 cso
->cpu_sysinfo
.xcalls
= CPU_STATS(cp
, sys
.xcalls
);
3400 cso
->cpu_sysinfo
.mutex_adenters
= CPU_STATS(cp
, sys
.mutex_adenters
);
3401 cso
->cpu_sysinfo
.rw_rdfails
= CPU_STATS(cp
, sys
.rw_rdfails
);
3402 cso
->cpu_sysinfo
.rw_wrfails
= CPU_STATS(cp
, sys
.rw_wrfails
);
3403 cso
->cpu_sysinfo
.modload
= CPU_STATS(cp
, sys
.modload
);
3404 cso
->cpu_sysinfo
.modunload
= CPU_STATS(cp
, sys
.modunload
);
3405 cso
->cpu_sysinfo
.bawrite
= CPU_STATS(cp
, sys
.bawrite
);
3406 cso
->cpu_sysinfo
.rw_enters
= 0;
3407 cso
->cpu_sysinfo
.win_uo_cnt
= 0;
3408 cso
->cpu_sysinfo
.win_uu_cnt
= 0;
3409 cso
->cpu_sysinfo
.win_so_cnt
= 0;
3410 cso
->cpu_sysinfo
.win_su_cnt
= 0;
3411 cso
->cpu_sysinfo
.win_suo_cnt
= 0;
3413 cso
->cpu_syswait
.iowait
= CPU_STATS(cp
, sys
.iowait
);
3414 cso
->cpu_syswait
.swap
= 0;
3415 cso
->cpu_syswait
.physio
= 0;
3417 cso
->cpu_vminfo
.pgrec
= CPU_STATS(cp
, vm
.pgrec
);
3418 cso
->cpu_vminfo
.pgfrec
= CPU_STATS(cp
, vm
.pgfrec
);
3419 cso
->cpu_vminfo
.pgin
= CPU_STATS(cp
, vm
.pgin
);
3420 cso
->cpu_vminfo
.pgpgin
= CPU_STATS(cp
, vm
.pgpgin
);
3421 cso
->cpu_vminfo
.pgout
= CPU_STATS(cp
, vm
.pgout
);
3422 cso
->cpu_vminfo
.pgpgout
= CPU_STATS(cp
, vm
.pgpgout
);
3423 cso
->cpu_vminfo
.swapin
= CPU_STATS(cp
, vm
.swapin
);
3424 cso
->cpu_vminfo
.pgswapin
= CPU_STATS(cp
, vm
.pgswapin
);
3425 cso
->cpu_vminfo
.swapout
= CPU_STATS(cp
, vm
.swapout
);
3426 cso
->cpu_vminfo
.pgswapout
= CPU_STATS(cp
, vm
.pgswapout
);
3427 cso
->cpu_vminfo
.zfod
= CPU_STATS(cp
, vm
.zfod
);
3428 cso
->cpu_vminfo
.dfree
= CPU_STATS(cp
, vm
.dfree
);
3429 cso
->cpu_vminfo
.scan
= CPU_STATS(cp
, vm
.scan
);
3430 cso
->cpu_vminfo
.rev
= CPU_STATS(cp
, vm
.rev
);
3431 cso
->cpu_vminfo
.hat_fault
= CPU_STATS(cp
, vm
.hat_fault
);
3432 cso
->cpu_vminfo
.as_fault
= CPU_STATS(cp
, vm
.as_fault
);
3433 cso
->cpu_vminfo
.maj_fault
= CPU_STATS(cp
, vm
.maj_fault
);
3434 cso
->cpu_vminfo
.cow_fault
= CPU_STATS(cp
, vm
.cow_fault
);
3435 cso
->cpu_vminfo
.prot_fault
= CPU_STATS(cp
, vm
.prot_fault
);
3436 cso
->cpu_vminfo
.softlock
= CPU_STATS(cp
, vm
.softlock
);
3437 cso
->cpu_vminfo
.kernel_asflt
= CPU_STATS(cp
, vm
.kernel_asflt
);
3438 cso
->cpu_vminfo
.pgrrun
= CPU_STATS(cp
, vm
.pgrrun
);
3439 cso
->cpu_vminfo
.execpgin
= CPU_STATS(cp
, vm
.execpgin
);
3440 cso
->cpu_vminfo
.execpgout
= CPU_STATS(cp
, vm
.execpgout
);
3441 cso
->cpu_vminfo
.execfree
= CPU_STATS(cp
, vm
.execfree
);
3442 cso
->cpu_vminfo
.anonpgin
= CPU_STATS(cp
, vm
.anonpgin
);
3443 cso
->cpu_vminfo
.anonpgout
= CPU_STATS(cp
, vm
.anonpgout
);
3444 cso
->cpu_vminfo
.anonfree
= CPU_STATS(cp
, vm
.anonfree
);
3445 cso
->cpu_vminfo
.fspgin
= CPU_STATS(cp
, vm
.fspgin
);
3446 cso
->cpu_vminfo
.fspgout
= CPU_STATS(cp
, vm
.fspgout
);
3447 cso
->cpu_vminfo
.fsfree
= CPU_STATS(cp
, vm
.fsfree
);