4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2017, Joyent, Inc.
27 * Copyright (c) 2010, Intel Corporation.
28 * All rights reserved.
31 #include <sys/types.h>
32 #include <sys/t_lock.h>
33 #include <sys/param.h>
34 #include <sys/segments.h>
35 #include <sys/sysmacros.h>
36 #include <sys/signal.h>
37 #include <sys/systm.h>
43 #include <sys/class.h>
49 #include <sys/reboot.h>
50 #include <sys/uadmin.h>
51 #include <sys/callb.h>
54 #include <sys/vnode.h>
57 #include <sys/procfs.h>
63 #include <sys/cmn_err.h>
64 #include <sys/utsname.h>
65 #include <sys/debug.h>
67 #include <sys/dumphdr.h>
68 #include <sys/bootconf.h>
69 #include <sys/varargs.h>
70 #include <sys/promif.h>
71 #include <sys/modctl.h>
73 #include <sys/consdev.h>
74 #include <sys/frame.h>
76 #include <sys/sunddi.h>
77 #include <sys/ddidmareq.h>
79 #include <sys/regset.h>
80 #include <sys/privregs.h>
81 #include <sys/clock.h>
84 #include <sys/stack.h>
92 #include <vm/seg_kmem.h>
93 #include <vm/seg_map.h>
94 #include <vm/seg_vn.h>
95 #include <vm/seg_kp.h>
96 #include <vm/hat_i86.h>
98 #include <sys/thread.h>
99 #include <sys/sysconf.h>
100 #include <sys/vm_machparam.h>
101 #include <sys/archsystm.h>
102 #include <sys/machsystm.h>
103 #include <sys/machlock.h>
104 #include <sys/x_call.h>
105 #include <sys/instance.h>
107 #include <sys/time.h>
108 #include <sys/smp_impldefs.h>
109 #include <sys/psm_types.h>
110 #include <sys/atomic.h>
111 #include <sys/panic.h>
112 #include <sys/cpuvar.h>
113 #include <sys/dtrace.h>
115 #include <sys/nvpair.h>
116 #include <sys/x86_archext.h>
117 #include <sys/pool_pset.h>
118 #include <sys/autoconf.h>
120 #include <sys/dumphdr.h>
121 #include <sys/compress.h>
122 #include <sys/cpu_module.h>
124 #include <sys/hypervisor.h>
125 #include <sys/xpv_panic.h>
128 #include <sys/fastboot.h>
129 #include <sys/machelf.h>
130 #include <sys/kobj.h>
131 #include <sys/multiboot.h>
134 #include <sys/traptrace.h>
135 #endif /* TRAPTRACE */
137 #include <c2/audit.h>
138 #include <sys/clock_impl.h>
140 extern void audit_enterprom(int);
141 extern void audit_exitprom(int);
144 * Tunable to enable apix PSM; if set to 0, pcplusmp PSM will be used.
148 int apic_nvidia_io_max
= 0; /* no. of NVIDIA i/o apics */
151 * Occassionally the kernel knows better whether to power-off or reboot.
153 int force_shutdown_method
= AD_UNKNOWN
;
156 * The panicbuf array is used to record messages and state:
158 char panicbuf
[PANICBUFSIZE
];
161 * Flags to control Dynamic Reconfiguration features.
163 uint64_t plat_dr_options
;
166 * Maximum physical address for memory DR operations.
168 uint64_t plat_dr_physmax
;
171 * maxphys - used during physio
172 * klustsize - used for klustering by swapfs and specfs
174 int maxphys
= 56 * 1024; /* XXX See vm_subr.c - max b_count in physio */
175 int klustsize
= 56 * 1024;
177 caddr_t p0_va
; /* Virtual address for accessing physical page 0 */
180 * defined here, though unused on x86,
181 * to make kstat_fr.c happy.
185 void debug_enter(char *);
187 extern void pm_cfb_check_and_powerup(void);
188 extern void pm_cfb_rele(void);
190 extern fastboot_info_t newkernel
;
193 * Machine dependent code to reboot.
194 * "mdep" is interpreted as a character pointer; if non-null, it is a pointer
195 * to a string to be used as the argument string when rebooting.
197 * "invoke_cb" is a boolean. It is set to true when mdboot() can safely
198 * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when
199 * we are in a normal shutdown sequence (interrupts are not blocked, the
200 * system is not panic'ing or being suspended).
204 mdboot(int cmd
, int fcn
, char *mdep
, boolean_t invoke_cb
)
206 processorid_t bootcpuid
= 0;
207 static int is_first_quiesce
= 1;
208 static int is_first_reset
= 1;
209 int reset_status
= 0;
210 static char fallback_str
[] = "Falling back to regular reboot.\n";
212 if (fcn
== AD_FASTREBOOT
&& !newkernel
.fi_valid
)
217 if (fcn
== AD_FASTREBOOT
) {
218 mutex_enter(&cpu_lock
);
219 if (CPU_ACTIVE(cpu_get(bootcpuid
))) {
220 affinity_set(bootcpuid
);
222 mutex_exit(&cpu_lock
);
224 affinity_set(CPU_CURRENT
);
228 if (force_shutdown_method
!= AD_UNKNOWN
)
229 fcn
= force_shutdown_method
;
232 * XXX - rconsvp is set to NULL to ensure that output messages
233 * are sent to the underlying "hardware" device using the
234 * monitor's printf routine since we are in the process of
235 * either rebooting or halting the machine.
240 * Print the reboot message now, before pausing other cpus.
241 * There is a race condition in the printing support that
242 * can deadlock multiprocessor machines.
244 if (!(fcn
== AD_HALT
|| fcn
== AD_POWEROFF
))
245 prom_printf("rebooting...\n");
251 * We can't bring up the console from above lock level, so do it now
253 pm_cfb_check_and_powerup();
255 /* make sure there are no more changes to the device tree */
259 (void) callb_execute_class(CB_CL_MDBOOT
, NULL
);
262 * Clear any unresolved UEs from memory.
264 page_retire_mdboot();
268 * XXPV Should probably think some more about how we deal
269 * with panicing before it's really safe to panic.
270 * On hypervisors, we reboot very quickly.. Perhaps panic
271 * should only attempt to recover by rebooting if,
272 * say, we were able to mount the root filesystem,
273 * or if we successfully launched init(1m).
275 if (panicstr
&& proc_init
== NULL
)
276 (void) HYPERVISOR_shutdown(SHUTDOWN_poweroff
);
279 * stop other cpus and raise our priority. since there is only
280 * one active cpu after this, and our priority will be too high
281 * for us to be preempted, we're essentially single threaded
286 mutex_enter(&cpu_lock
);
287 pause_cpus(NULL
, NULL
);
288 mutex_exit(&cpu_lock
);
292 * If the system is panicking, the preloaded kernel is valid, and
293 * fastreboot_onpanic has been set, and the system has been up for
294 * longer than fastreboot_onpanic_uptime (default to 10 minutes),
295 * choose Fast Reboot.
297 if (fcn
== AD_BOOT
&& panicstr
&& newkernel
.fi_valid
&&
298 fastreboot_onpanic
&&
299 (panic_lbolt
- lbolt_at_boot
) > fastreboot_onpanic_uptime
) {
304 * Try to quiesce devices.
306 if (is_first_quiesce
) {
308 * Clear is_first_quiesce before calling quiesce_devices()
309 * so that if quiesce_devices() causes panics, it will not
312 is_first_quiesce
= 0;
315 quiesce_devices(ddi_root_node(), &reset_status
);
316 if (reset_status
== -1) {
317 if (fcn
== AD_FASTREBOOT
&& !force_fastreboot
) {
318 prom_printf("Driver(s) not capable of fast "
320 prom_printf(fallback_str
);
321 fastreboot_capable
= 0;
323 } else if (fcn
!= AD_FASTREBOOT
)
324 fastreboot_capable
= 0;
330 * Try to reset devices. reset_leaves() should only be called
331 * a) when there are no other threads that could be accessing devices,
333 * b) on a system that's not capable of fast reboot (fastreboot_capable
334 * being 0), or on a system where quiesce_devices() failed to
335 * complete (quiesce_active being 1).
337 if (is_first_reset
&& (!fastreboot_capable
|| quiesce_active
)) {
339 * Clear is_first_reset before calling reset_devices()
340 * so that if reset_devices() causes panics, it will not
347 /* Verify newkernel checksum */
348 if (fastreboot_capable
&& fcn
== AD_FASTREBOOT
&&
349 fastboot_cksum_verify(&newkernel
) != 0) {
350 fastreboot_capable
= 0;
351 prom_printf("Fast reboot: checksum failed for the new "
353 prom_printf(fallback_str
);
358 if (fastreboot_capable
&& fcn
== AD_FASTREBOOT
) {
360 * psm_shutdown is called within fast_reboot()
364 (*psm_shutdownf
)(cmd
, fcn
);
366 if (fcn
== AD_HALT
|| fcn
== AD_POWEROFF
)
374 /* mdpreboot - may be called prior to mdboot while root fs still mounted */
377 mdpreboot(int cmd
, int fcn
, char *mdep
)
379 if (fcn
== AD_FASTREBOOT
&& !fastreboot_capable
) {
382 cmn_err(CE_WARN
, "Fast reboot is not supported on xVM");
385 "Fast reboot is not supported on this platform%s",
386 fastreboot_nosup_message());
390 if (fcn
== AD_FASTREBOOT
) {
391 fastboot_load_kernel(mdep
);
392 if (!newkernel
.fi_valid
)
396 (*psm_preshutdownf
)(cmd
, fcn
);
400 stop_other_cpus(void)
402 ulong_t s
= clear_int_flag(); /* fast way to keep CPU from changing */
405 CPUSET_ALL_BUT(xcset
, CPU
->cpu_id
);
406 xc_priority(0, 0, 0, CPUSET2BV(xcset
), (xc_func_t
)mach_cpu_halt
);
411 * Machine dependent abort sequence handling
414 abort_sequence_enter(char *msg
)
416 if (abort_enable
== 0) {
417 if (AU_ZONE_AUDITING(GET_KCTX_GZ
))
421 if (AU_ZONE_AUDITING(GET_KCTX_GZ
))
424 if (AU_ZONE_AUDITING(GET_KCTX_GZ
))
429 * Enter debugger. Called when the user types ctrl-alt-d or whenever
430 * code wants to enter the debugger and possibly resume later.
432 * msg: message to print, possibly NULL.
435 debug_enter(char *msg
)
437 if (dtrace_debugger_init
!= NULL
)
438 (*dtrace_debugger_init
)();
440 if (msg
!= NULL
|| (boothowto
& RB_DEBUG
))
444 prom_printf("%s\n", msg
);
446 if (boothowto
& RB_DEBUG
)
449 if (dtrace_debugger_fini
!= NULL
)
450 (*dtrace_debugger_fini
)();
456 extern void acpi_reset_system();
458 ushort_t
*bios_memchk
;
461 * Can't use psm_map_phys or acpi_reset_system before the hat is
465 bios_memchk
= (ushort_t
*)psm_map_phys(0x472,
466 sizeof (ushort_t
), PROT_READ
| PROT_WRITE
);
468 *bios_memchk
= 0x1234; /* bios memory check disable */
470 if (options_dip
!= NULL
&&
471 ddi_prop_exists(DDI_DEV_T_ANY
, ddi_root_node(), 0,
479 * The problem with using stubs is that we can call
480 * acpi_reset_system only after the kernel is up and running.
482 * We should create a global state to keep track of how far
483 * up the kernel is but for the time being we will depend on
484 * bootops. bootops cleared in startup_end().
492 if (IN_XPV_PANIC()) {
493 if (khat_running
&& bootops
== NULL
) {
500 (void) HYPERVISOR_shutdown(SHUTDOWN_reboot
);
501 panic("HYPERVISOR_shutdown() failed");
507 * Halt the machine and return to the monitor
512 stop_other_cpus(); /* send stop signal to other CPUs */
514 prom_printf("(%s) \n", s
);
520 * Initiate interrupt redistribution.
523 i_ddi_intr_redist_all_cpus()
528 * XXX These probably ought to live somewhere else
529 * XXX They are called from mem.c
533 * Convert page frame number to an OBMEM page frame number
534 * (i.e. put in the type bits -- zero for this implementation)
537 impl_obmem_pfnum(pfn_t pf
)
543 int nmi_test
= 0; /* checked in intentry.s during clock int */
545 nmfunc1(int arg
, struct regs
*rp
)
547 printf("nmi called with arg = %x, regs = %x\n", arg
, rp
);
550 printf("ip = %x\n", rp
->r_pc
);
558 #include <sys/bootsvcs.h>
560 /* Hacked up initialization for initial kernel check out is HERE. */
561 /* The basic steps are: */
562 /* kernel bootfuncs definition/initialization for KADB */
563 /* kadb bootfuncs pointer initialization */
564 /* putchar/getchar (interrupts disabled) */
566 /* kadb bootfuncs pointer initialization */
574 if (cons_polledio
== NULL
) {
576 prom_printf("getchar called with no console\n");
581 s
= clear_int_flag();
582 i
= cons_polledio
->cons_polledio_getchar(
583 cons_polledio
->cons_polledio_argument
);
594 * We have no alternative but to drop the output on the floor.
596 if (cons_polledio
== NULL
||
597 cons_polledio
->cons_polledio_putchar
== NULL
)
600 s
= clear_int_flag();
601 cons_polledio
->cons_polledio_putchar(
602 cons_polledio
->cons_polledio_argument
, c
);
612 if (cons_polledio
== NULL
||
613 cons_polledio
->cons_polledio_ischar
== NULL
)
616 s
= clear_int_flag();
617 i
= cons_polledio
->cons_polledio_ischar(
618 cons_polledio
->cons_polledio_argument
);
626 prom_printf("Type any key to continue ");
627 (void) prom_getchar();
632 static struct boot_syscalls kern_sysp
= {
633 sysp_getchar
, /* unchar (*getchar)(); 7 */
634 sysp_putchar
, /* int (*putchar)(); 8 */
635 sysp_ischar
, /* int (*ischar)(); 9 */
639 int using_kern_polledio
;
646 * This routine is now totally misnamed, since it does not in fact
647 * control kadb's I/O; it only controls the kernel's prom_* I/O.
651 using_kern_polledio
= 1;
656 * the interface to the outside world
660 * poll_port -- wait for a register to achieve a
661 * specific state. Arguments are a mask of bits we care about,
662 * and two sub-masks. To return normally, all the bits in the
663 * first sub-mask must be ON, all the bits in the second sub-
664 * mask must be OFF. If about seconds pass without the register
665 * achieving the desired bit configuration, we return 1, else
669 poll_port(ushort_t port
, ushort_t mask
, ushort_t onbits
, ushort_t offbits
)
674 for (i
= 500000; i
; i
--) {
675 maskval
= inb(port
) & mask
;
676 if (((maskval
& onbits
) == onbits
) &&
677 ((maskval
& offbits
) == 0))
685 * set_idle_cpu is called from idle() when a CPU becomes idle.
687 /*LINTED: static unused */
688 static uint_t last_idle_cpu
;
692 set_idle_cpu(int cpun
)
694 last_idle_cpu
= cpun
;
695 (*psm_set_idle_cpuf
)(cpun
);
699 * unset_idle_cpu is called from idle() when a CPU is no longer idle.
703 unset_idle_cpu(int cpun
)
705 (*psm_unset_idle_cpuf
)(cpun
);
709 * This routine is almost correct now, but not quite. It still needs the
710 * equivalent concept of "hres_last_tick", just like on the sparc side.
711 * The idea is to take a snapshot of the hi-res timer while doing the
712 * hrestime_adj updates under hres_lock in locore, so that the small
713 * interval between interrupt assertion and interrupt processing is
714 * accounted for correctly. Once we have this, the code below should
715 * be modified to subtract off hres_last_tick rather than hrtime_base.
717 * I'd have done this myself, but I don't have source to all of the
718 * vendor-specific hi-res timer routines (grrr...). The generic hook I
719 * need is something like "gethrtime_unlocked()", which would be just like
720 * gethrtime() but would assume that you're already holding CLOCK_LOCK().
721 * This is what the GET_HRTIME() macro is for on sparc (although it also
722 * serves the function of making time available without a function call
723 * so you don't take a register window overflow while traps are disabled).
726 pc_gethrestime(timestruc_t
*tp
)
730 int nslt
; /* nsec since last tick */
731 int adj
; /* amount of adjustment to apply */
734 lock_prev
= hres_lock
;
736 nslt
= (int)(gethrtime() - hres_last_tick
);
739 * nslt < 0 means a tick came between sampling
740 * gethrtime() and hres_last_tick; restart the loop
746 if (hrestime_adj
!= 0) {
747 if (hrestime_adj
> 0) {
748 adj
= (nslt
>> ADJ_SHIFT
);
749 if (adj
> hrestime_adj
)
750 adj
= (int)hrestime_adj
;
752 adj
= -(nslt
>> ADJ_SHIFT
);
753 if (adj
< hrestime_adj
)
754 adj
= (int)hrestime_adj
;
758 while ((unsigned long)now
.tv_nsec
>= NANOSEC
) {
761 * We might have a large adjustment or have been in the
762 * debugger for a long time; take care of (at most) four
763 * of those missed seconds (tv_nsec is 32 bits, so
764 * anything >4s will be wrapping around). However,
765 * anything more than 2 seconds out of sync will trigger
766 * timedelta from clock() to go correct the time anyway,
767 * so do what we can, and let the big crowbar do the
768 * rest. A similar correction while loop exists inside
769 * hres_tick(); in all cases we'd like tv_nsec to
770 * satisfy 0 <= tv_nsec < NANOSEC to avoid confusing
771 * user processes, but if tv_sec's a little behind for a
772 * little while, that's OK; time still monotonically
776 now
.tv_nsec
-= NANOSEC
;
779 if ((hres_lock
& ~1) != lock_prev
)
786 gethrestime_lasttick(timespec_t
*tp
)
796 gethrestime_sec(void)
805 * Initialize a kernel thread's stack
809 thread_stk_init(caddr_t stk
)
811 ASSERT(((uintptr_t)stk
& (STACK_ALIGN
- 1)) == 0);
812 return (stk
- SA(MINFRAME
));
816 * Initialize lwp's kernel stack.
821 * There's a tricky interdependency here between use of sysenter and
822 * TRAPTRACE which needs recording to avoid future confusion (this is
823 * about the third time I've re-figured this out ..)
825 * Here's how debugging lcall works with TRAPTRACE.
827 * 1 We're in userland with a breakpoint on the lcall instruction.
828 * 2 We execute the instruction - the instruction pushes the userland
829 * %ss, %esp, %efl, %cs, %eip on the stack and zips into the kernel
831 * 3 The hardware raises a debug trap in kernel mode, the hardware
832 * pushes %efl, %cs, %eip and gets to dbgtrap via the idt.
833 * 4 dbgtrap pushes the error code and trapno and calls cmntrap
834 * 5 cmntrap finishes building a trap frame
835 * 6 The TRACE_REGS macros in cmntrap copy a REGSIZE worth chunk
836 * off the stack into the traptrace buffer.
838 * This means that the traptrace buffer contains the wrong values in
839 * %esp and %ss, but everything else in there is correct.
841 * Here's how debugging sysenter works with TRAPTRACE.
843 * a We're in userland with a breakpoint on the sysenter instruction.
844 * b We execute the instruction - the instruction pushes -nothing-
845 * on the stack, but sets %cs, %eip, %ss, %esp to prearranged
846 * values to take us to sys_sysenter, at the top of the lwp's
850 * At this point, because we got into the kernel without the requisite
851 * five pushes on the stack, if we didn't make extra room, we'd
852 * end up with the TRACE_REGS macro fetching the saved %ss and %esp
853 * values from negative (unmapped) stack addresses -- which really bites.
854 * That's why we do the '-= 8' below.
856 * XXX Note that reading "up" lwp0's stack works because t0 is declared
857 * right next to t0stack in locore.s
862 lwp_stk_init(klwp_t
*lwp
, caddr_t stk
)
865 struct pcb
*pcb
= &lwp
->lwp_pcb
;
868 stk
-= SA(sizeof (struct regs
) + SA(MINFRAME
));
870 stk
-= 2 * sizeof (greg_t
); /* space for phony %ss:%sp (see above) */
872 stk
= (caddr_t
)((uintptr_t)stk
& ~(STACK_ALIGN
- 1ul));
873 bzero(stk
, oldstk
- stk
);
874 lwp
->lwp_regs
= (void *)(stk
+ SA(MINFRAME
));
877 * Arrange that the virtualized %fs and %gs GDT descriptors
878 * have a well-defined initial state (present, ring 3
882 if (lwp_getdatamodel(lwp
) == DATAMODEL_NATIVE
)
883 pcb
->pcb_fsdesc
= pcb
->pcb_gsdesc
= zero_udesc
;
885 pcb
->pcb_fsdesc
= pcb
->pcb_gsdesc
= zero_u32desc
;
886 #elif defined(__i386)
887 pcb
->pcb_fsdesc
= pcb
->pcb_gsdesc
= zero_udesc
;
894 * Use this opportunity to free any dynamically allocated fp storage.
897 lwp_stk_fini(klwp_t
*lwp
)
903 lwp_fp_init(klwp_t
*lwp
)
909 * If we're not the panic CPU, we wait in panic_idle for reboot.
914 splx(ipltospl(CLOCK_LEVEL
));
915 (void) setjmp(&curthread
->t_pcb
);
929 * Stop the other CPUs by cross-calling them and forcing them to enter
930 * the panic_idle() loop above.
934 panic_stopcpus(cpu_t
*cp
, kthread_t
*t
, int spl
)
940 * In the case of a Xen panic, the hypervisor has already stopped
943 if (!IN_XPV_PANIC()) {
946 CPUSET_ALL_BUT(xcset
, cp
->cpu_id
);
947 xc_priority(0, 0, 0, CPUSET2BV(xcset
), (xc_func_t
)panic_idle
);
950 for (i
= 0; i
< NCPU
; i
++) {
951 if (i
!= cp
->cpu_id
&& cpu
[i
] != NULL
&&
952 (cpu
[i
]->cpu_flags
& CPU_EXISTS
))
953 cpu
[i
]->cpu_flags
|= CPU_QUIESCED
;
958 * Platform callback following each entry to panicsys().
962 panic_enter_hw(int spl
)
964 /* Nothing to do here */
968 * Platform-specific code to execute after panicstr is set: we invoke
969 * the PSM entry point to indicate that a panic has occurred.
973 panic_quiesce_hw(panic_data_t
*pdp
)
975 psm_notifyf(PSM_PANIC_ENTER
);
977 cmi_panic_callback();
984 #endif /* TRAPTRACE */
988 * Platform callback prior to writing crash dump.
992 panic_dump_hw(int spl
)
994 /* Nothing to do here */
998 plat_traceback(void *fpreg
)
1002 return (xpv_traceback(fpreg
));
1009 plat_tod_fault(enum tod_fault_type tod_bad
)
1014 blacklist(int cmd
, const char *scheme
, nvlist_t
*fmri
, const char *class)
1020 * The underlying console output routines are protected by raising IPL in case
1021 * we are still calling into the early boot services. Once we start calling
1022 * the kernel console emulator, it will disable interrupts completely during
1023 * character rendering (see sysp_putchar, for example). Refer to the comments
1024 * and code in common/os/console.c for more information on these callbacks.
1028 console_enter(int busy
)
1035 console_exit(int busy
, int spl
)
1041 * Allocate a region of virtual address space, unmapped.
1042 * Stubbed out except on sparc, at least for now.
1046 boot_virt_alloc(void *addr
, size_t size
)
1051 volatile unsigned long tenmicrodata
;
1056 extern int gethrtime_hires
;
1058 if (gethrtime_hires
) {
1059 hrtime_t start
, end
;
1060 start
= end
= gethrtime();
1061 while ((end
- start
) < (10 * (NANOSEC
/ MICROSEC
))) {
1069 newtime
= xpv_gethrtime() + 10000; /* now + 10 us */
1070 while (xpv_gethrtime() < newtime
)
1076 * Artificial loop to induce delay.
1078 for (i
= 0; i
< microdata
; i
++)
1079 tenmicrodata
= microdata
;
1085 * get_cpu_mstate() is passed an array of timestamps, NCMSTATES
1086 * long, and it fills in the array with the time spent on cpu in
1087 * each of the mstates, where time is returned in nsec.
1089 * No guarantee is made that the returned values in times[] will
1090 * monotonically increase on sequential calls, although this will
1091 * be true in the long run. Any such guarantee must be handled by
1092 * the caller, if needed. This can happen if we fail to account
1093 * for elapsed time due to a generation counter conflict, yet we
1094 * did account for it on a prior call (see below).
1096 * The complication is that the cpu in question may be updating
1097 * its microstate at the same time that we are reading it.
1098 * Because the microstate is only updated when the CPU's state
1099 * changes, the values in cpu_intracct[] can be indefinitely out
1100 * of date. To determine true current values, it is necessary to
1101 * compare the current time with cpu_mstate_start, and add the
1102 * difference to times[cpu_mstate].
1104 * This can be a problem if those values are changing out from
1105 * under us. Because the code path in new_cpu_mstate() is
1106 * performance critical, we have not added a lock to it. Instead,
1107 * we have added a generation counter. Before beginning
1108 * modifications, the counter is set to 0. After modifications,
1109 * it is set to the old value plus one.
1111 * get_cpu_mstate() will not consider the values of cpu_mstate
1112 * and cpu_mstate_start to be usable unless the value of
1113 * cpu_mstate_gen is both non-zero and unchanged, both before and
1114 * after reading the mstate information. Note that we must
1115 * protect against out-of-order loads around accesses to the
1116 * generation counter. Also, this is a best effort approach in
1117 * that we do not retry should the counter be found to have
1120 * cpu_intracct[] is used to identify time spent in each CPU
1121 * mstate while handling interrupts. Such time should be reported
1122 * against system time, and so is subtracted out from its
1123 * corresponding cpu_acct[] time and added to
1124 * cpu_acct[CMS_SYSTEM].
1128 get_cpu_mstate(cpu_t
*cpu
, hrtime_t
*times
)
1131 hrtime_t now
, start
;
1134 hrtime_t intracct
[NCMSTATES
];
1137 * Load all volatile state under the protection of membar.
1138 * cpu_acct[cpu_mstate] must be loaded to avoid double counting
1139 * of (now - cpu_mstate_start) by a change in CPU mstate that
1140 * arrives after we make our last check of cpu_mstate_gen.
1143 now
= gethrtime_unscaled();
1144 gen
= cpu
->cpu_mstate_gen
;
1146 membar_consumer(); /* guarantee load ordering */
1147 start
= cpu
->cpu_mstate_start
;
1148 state
= cpu
->cpu_mstate
;
1149 for (i
= 0; i
< NCMSTATES
; i
++) {
1150 intracct
[i
] = cpu
->cpu_intracct
[i
];
1151 times
[i
] = cpu
->cpu_acct
[i
];
1153 membar_consumer(); /* guarantee load ordering */
1155 if (gen
!= 0 && gen
== cpu
->cpu_mstate_gen
&& now
> start
)
1156 times
[state
] += now
- start
;
1158 for (i
= 0; i
< NCMSTATES
; i
++) {
1159 if (i
== CMS_SYSTEM
)
1161 times
[i
] -= intracct
[i
];
1163 intracct
[i
] += times
[i
];
1166 times
[CMS_SYSTEM
] += intracct
[i
];
1167 scalehrtime(×
[i
]);
1169 scalehrtime(×
[CMS_SYSTEM
]);
1173 * This is a version of the rdmsr instruction that allows
1174 * an error code to be returned in the case of failure.
1177 checked_rdmsr(uint_t msr
, uint64_t *value
)
1179 if (!is_x86_feature(x86_featureset
, X86FSET_MSR
))
1181 *value
= rdmsr(msr
);
1186 * This is a version of the wrmsr instruction that allows
1187 * an error code to be returned in the case of failure.
1190 checked_wrmsr(uint_t msr
, uint64_t value
)
1192 if (!is_x86_feature(x86_featureset
, X86FSET_MSR
))
1199 * The mem driver's usual method of using hat_devload() to establish a
1200 * temporary mapping will not work for foreign pages mapped into this
1201 * domain or for the special hypervisor-provided pages. For the foreign
1202 * pages, we often don't know which domain owns them, so we can't ask the
1203 * hypervisor to set up a new mapping. For the other pages, we don't have
1204 * a pfn, so we can't create a new PTE. For these special cases, we do a
1205 * direct uiomove() from the existing kernel virtual address.
1209 plat_mem_do_mmio(struct uio
*uio
, enum uio_rw rw
)
1212 void *va
= (void *)(uintptr_t)uio
->uio_loffset
;
1213 off_t pageoff
= uio
->uio_loffset
& PAGEOFFSET
;
1214 size_t nbytes
= MIN((size_t)(PAGESIZE
- pageoff
),
1215 (size_t)uio
->uio_iov
->iov_len
);
1217 if ((rw
== UIO_READ
&&
1218 (va
== HYPERVISOR_shared_info
|| va
== xen_info
)) ||
1219 (pfn_is_foreign(hat_getpfnum(kas
.a_hat
, va
))))
1220 return (uiomove(va
, nbytes
, rw
, uio
));
1232 if (DOMAIN_IS_INITDOMAIN(xen_info
))
1233 return (xpv_nr_phys_pages());
1236 for (mp
= phys_install
; mp
!= NULL
; mp
= mp
->ml_next
)
1237 npages
+= mp
->ml_size
>> PAGESHIFT
;
1242 /* cpu threshold for compressed dumps */
1244 uint_t dump_plat_mincpu_default
= DUMP_PLAT_X86_64_MINCPU
;
1246 uint_t dump_plat_mincpu_default
= DUMP_PLAT_X86_32_MINCPU
;
1253 pfn_t pfn
= mmu_btop(xen_info
->shared_info
) | PFN_IS_FOREIGN_MFN
;
1254 mem_vtop_t mem_vtop
;
1258 * On the hypervisor, we want to dump the page with shared_info on it.
1260 if (!IN_XPV_PANIC()) {
1261 mem_vtop
.m_as
= &kas
;
1262 mem_vtop
.m_va
= HYPERVISOR_shared_info
;
1263 mem_vtop
.m_pfn
= pfn
;
1264 dumpvp_write(&mem_vtop
, sizeof (mem_vtop_t
));
1267 cnt
= dump_xpv_addr();
1279 pfn_t pfn
= mmu_btop(xen_info
->shared_info
) | PFN_IS_FOREIGN_MFN
;
1281 if (!IN_XPV_PANIC())
1282 dumpvp_write(&pfn
, sizeof (pfn
));
1290 dump_plat_data(void *dump_cbuf
)
1296 if (!IN_XPV_PANIC()) {
1297 csize
= (uint32_t)compress(HYPERVISOR_shared_info
, dump_cbuf
,
1299 dumpvp_write(&csize
, sizeof (uint32_t));
1300 dumpvp_write(dump_cbuf
, csize
);
1303 cnt
= dump_xpv_data(dump_cbuf
);
1312 * Calculates a linear address, given the CS selector and PC values
1313 * by looking up the %cs selector process's LDT or the CPU's GDT.
1314 * proc->p_ldtlock must be held across this call.
1317 linear_pc(struct regs
*rp
, proc_t
*p
, caddr_t
*linearp
)
1319 user_desc_t
*descrp
;
1321 uint16_t idx
= SELTOIDX(rp
->r_cs
);
1323 ASSERT(rp
->r_cs
<= 0xFFFF);
1324 ASSERT(MUTEX_HELD(&p
->p_ldtlock
));
1326 if (SELISLDT(rp
->r_cs
)) {
1328 * Currently 64 bit processes cannot have private LDTs.
1330 ASSERT(p
->p_model
!= DATAMODEL_LP64
);
1332 if (p
->p_ldt
== NULL
)
1335 descrp
= &p
->p_ldt
[idx
];
1336 baseaddr
= (caddr_t
)(uintptr_t)USEGD_GETBASE(descrp
);
1339 * Calculate the linear address (wraparound is not only ok,
1340 * it's expected behavior). The cast to uint32_t is because
1341 * LDT selectors are only allowed in 32-bit processes.
1343 *linearp
= (caddr_t
)(uintptr_t)(uint32_t)((uintptr_t)baseaddr
+
1347 descrp
= &CPU
->cpu_gdt
[idx
];
1348 baseaddr
= (caddr_t
)(uintptr_t)USEGD_GETBASE(descrp
);
1349 /* GDT-based descriptors' base addresses should always be 0 */
1350 ASSERT(baseaddr
== 0);
1352 *linearp
= (caddr_t
)(uintptr_t)rp
->r_pc
;
1359 * The implementation of dtrace_linear_pc is similar to the that of
1360 * linear_pc, above, but here we acquire p_ldtlock before accessing
1361 * p_ldt. This implementation is used by the pid provider; we prefix
1362 * it with "dtrace_" to avoid inducing spurious tracing events.
1365 dtrace_linear_pc(struct regs
*rp
, proc_t
*p
, caddr_t
*linearp
)
1367 user_desc_t
*descrp
;
1369 uint16_t idx
= SELTOIDX(rp
->r_cs
);
1371 ASSERT(rp
->r_cs
<= 0xFFFF);
1373 if (SELISLDT(rp
->r_cs
)) {
1375 * Currently 64 bit processes cannot have private LDTs.
1377 ASSERT(p
->p_model
!= DATAMODEL_LP64
);
1379 mutex_enter(&p
->p_ldtlock
);
1380 if (p
->p_ldt
== NULL
) {
1381 mutex_exit(&p
->p_ldtlock
);
1384 descrp
= &p
->p_ldt
[idx
];
1385 baseaddr
= (caddr_t
)(uintptr_t)USEGD_GETBASE(descrp
);
1386 mutex_exit(&p
->p_ldtlock
);
1389 * Calculate the linear address (wraparound is not only ok,
1390 * it's expected behavior). The cast to uint32_t is because
1391 * LDT selectors are only allowed in 32-bit processes.
1393 *linearp
= (caddr_t
)(uintptr_t)(uint32_t)((uintptr_t)baseaddr
+
1397 descrp
= &CPU
->cpu_gdt
[idx
];
1398 baseaddr
= (caddr_t
)(uintptr_t)USEGD_GETBASE(descrp
);
1399 /* GDT-based descriptors' base addresses should always be 0 */
1400 ASSERT(baseaddr
== 0);
1402 *linearp
= (caddr_t
)(uintptr_t)rp
->r_pc
;
1409 * We need to post a soft interrupt to reprogram the lbolt cyclic when
1410 * switching from event to cyclic driven lbolt. The following code adds
1411 * and posts the softint for x86.
1413 static ddi_softint_hdl_impl_t lbolt_softint_hdl
=
1414 {0, NULL
, NULL
, NULL
, 0, NULL
, NULL
, NULL
};
1417 lbolt_softint_add(void)
1419 (void) add_avsoftintr((void *)&lbolt_softint_hdl
, LOCK_LEVEL
,
1420 (avfunc
)lbolt_ev_to_cyclic
, "lbolt_ev_to_cyclic", NULL
, NULL
);
1424 lbolt_softint_post(void)
1426 (*setsoftint
)(CBE_LOCK_PIL
, lbolt_softint_hdl
.ih_pending
);
1430 plat_dr_check_capability(uint64_t features
)
1432 return ((plat_dr_options
& features
) == features
);
1436 plat_dr_support_cpu(void)
1438 return (plat_dr_options
& PLAT_DR_FEATURE_CPU
);
1442 plat_dr_support_memory(void)
1444 return (plat_dr_options
& PLAT_DR_FEATURE_MEMORY
);
1448 plat_dr_enable_capability(uint64_t features
)
1450 atomic_or_64(&plat_dr_options
, features
);
1454 plat_dr_disable_capability(uint64_t features
)
1456 atomic_and_64(&plat_dr_options
, ~features
);