4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
26 * Copyright 2014 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
29 #include <sys/cpuvar.h>
31 #include <sys/archsystm.h>
33 #include <sys/sunddi.h>
34 #include <sys/ddi_impldefs.h>
35 #include <sys/mach_intr.h>
36 #include <sys/sysmacros.h>
38 #include <sys/x86_archext.h>
39 #include <sys/privregs.h>
40 #include <sys/psm_common.h>
42 /* Function prototypes of local apic and X2APIC */
43 static uint64_t local_apic_read(uint32_t reg
);
44 static void local_apic_write(uint32_t reg
, uint64_t value
);
45 static int get_local_apic_pri(void);
46 static void local_apic_write_task_reg(uint64_t value
);
47 static void local_apic_write_int_cmd(uint32_t cpu_id
, uint32_t cmd1
);
48 static uint64_t local_x2apic_read(uint32_t msr
);
49 static void local_x2apic_write(uint32_t msr
, uint64_t value
);
50 static int get_local_x2apic_pri(void);
51 static void local_x2apic_write_task_reg(uint64_t value
);
52 static void local_x2apic_write_int_cmd(uint32_t cpu_id
, uint32_t cmd1
);
55 * According to the X2APIC specification:
57 * xAPIC global enable X2APIC enable Description
58 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
59 * -----------------------------------------------------------
60 * 0 0 APIC is disabled
62 * 1 0 APIC is enabled in xAPIC mode
63 * 1 1 APIC is enabled in X2APIC mode
64 * -----------------------------------------------------------
66 int x2apic_enable
= 1;
67 apic_mode_t apic_mode
= LOCAL_APIC
; /* Default mode is Local APIC */
69 /* Uses MMIO (Memory Mapped IO) */
70 static apic_reg_ops_t local_apic_regs_ops
= {
74 local_apic_write_task_reg
,
75 local_apic_write_int_cmd
,
79 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
80 static apic_reg_ops_t x2apic_regs_ops
= {
84 local_x2apic_write_task_reg
,
85 local_x2apic_write_int_cmd
,
89 int apic_have_32bit_cr8
= 0;
91 /* The default ops is local APIC (Memory Mapped IO) */
92 apic_reg_ops_t
*apic_reg_ops
= &local_apic_regs_ops
;
95 * APIC register ops related data sturctures and functions.
98 void apic_send_directed_EOI(uint32_t irq
);
100 #define X2APIC_ENABLE_BIT 10
103 * Local APIC Implementation
106 local_apic_read(uint32_t reg
)
108 return ((uint32_t)apicadr
[reg
]);
112 local_apic_write(uint32_t reg
, uint64_t value
)
114 apicadr
[reg
] = (uint32_t)value
;
118 get_local_apic_pri(void)
121 return ((int)getcr8());
123 if (apic_have_32bit_cr8
)
124 return ((int)getcr8());
125 return (apicadr
[APIC_TASK_REG
]);
130 local_apic_write_task_reg(uint64_t value
)
133 setcr8((ulong_t
)(value
>> APIC_IPL_SHIFT
));
135 if (apic_have_32bit_cr8
)
136 setcr8((ulong_t
)(value
>> APIC_IPL_SHIFT
));
138 apicadr
[APIC_TASK_REG
] = (uint32_t)value
;
143 local_apic_write_int_cmd(uint32_t cpu_id
, uint32_t cmd1
)
145 apicadr
[APIC_INT_CMD2
] = cpu_id
<< APIC_ICR_ID_BIT_OFFSET
;
146 apicadr
[APIC_INT_CMD1
] = cmd1
;
150 * X2APIC Implementation.
153 local_x2apic_read(uint32_t msr
)
157 i
= (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR
+ (msr
>> 2)) & 0xffffffff);
162 local_x2apic_write(uint32_t msr
, uint64_t value
)
166 if (msr
!= APIC_EOI_REG
) {
167 tmp
= rdmsr(REG_X2APIC_BASE_MSR
+ (msr
>> 2));
168 tmp
= (tmp
& 0xffffffff00000000) | value
;
173 wrmsr((REG_X2APIC_BASE_MSR
+ (msr
>> 2)), tmp
);
177 get_local_x2apic_pri(void)
179 return (rdmsr(REG_X2APIC_BASE_MSR
+ (APIC_TASK_REG
>> 2)));
183 local_x2apic_write_task_reg(uint64_t value
)
185 X2APIC_WRITE(APIC_TASK_REG
, value
);
189 local_x2apic_write_int_cmd(uint32_t cpu_id
, uint32_t cmd1
)
191 wrmsr((REG_X2APIC_BASE_MSR
+ (APIC_INT_CMD1
>> 2)),
192 (((uint64_t)cpu_id
<< 32) | cmd1
));
197 apic_send_EOI(uint32_t irq
)
199 apic_reg_ops
->apic_write(APIC_EOI_REG
, 0);
203 * Support for Directed EOI capability is available in both the xAPIC
207 apic_send_directed_EOI(uint32_t irq
)
211 apic_irq_t
*apic_irq
;
215 * Following the EOI to the local APIC unit, perform a directed
216 * EOI to the IOxAPIC generating the interrupt by writing to its
219 * A broadcast EOI is not generated.
221 apic_reg_ops
->apic_write(APIC_EOI_REG
, 0);
223 apic_irq
= apic_irq_table
[irq
];
225 intr_index
= apic_irq
->airq_mps_intr_index
;
226 if (intr_index
== ACPI_INDEX
|| intr_index
>= 0) {
227 ioapicindex
= apic_irq
->airq_ioapicindex
;
228 vector
= apic_irq
->airq_vector
;
229 ioapic_write_eoi(ioapicindex
, vector
);
231 apic_irq
= apic_irq
->airq_next
;
236 apic_detect_x2apic(void)
238 if (x2apic_enable
== 0)
241 return (is_x86_feature(x86_featureset
, X86FSET_X2APIC
));
245 apic_enable_x2apic(void)
247 uint64_t apic_base_msr
;
249 if (apic_local_mode() == LOCAL_X2APIC
) {
250 /* BIOS apparently has enabled X2APIC */
251 if (apic_mode
!= LOCAL_X2APIC
)
257 * This is the first time we are enabling X2APIC on this CPU
259 apic_base_msr
= rdmsr(REG_APIC_BASE_MSR
);
260 apic_base_msr
= apic_base_msr
| (0x1 << X2APIC_ENABLE_BIT
);
261 wrmsr(REG_APIC_BASE_MSR
, apic_base_msr
);
263 if (apic_mode
!= LOCAL_X2APIC
)
268 * Determine which mode the current CPU is in. See the table above.
269 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
272 apic_local_mode(void)
274 uint64_t apic_base_msr
;
275 int bit
= ((0x1 << (X2APIC_ENABLE_BIT
+ 1)) |
276 (0x1 << X2APIC_ENABLE_BIT
));
278 apic_base_msr
= rdmsr(REG_APIC_BASE_MSR
);
280 if ((apic_base_msr
& bit
) == bit
)
281 return (LOCAL_X2APIC
);
287 apic_set_directed_EOI_handler()
289 apic_reg_ops
->apic_send_eoi
= apic_send_directed_EOI
;
293 apic_directed_EOI_supported()
297 ver
= apic_reg_ops
->apic_read(APIC_VERS_REG
);
298 if (ver
& APIC_DIRECTED_EOI_BIT
)
305 * Change apic_reg_ops depending upon the apic_mode.
310 if (apic_mode
== LOCAL_APIC
)
311 apic_reg_ops
= &local_apic_regs_ops
;
312 else if (apic_mode
== LOCAL_X2APIC
)
313 apic_reg_ops
= &x2apic_regs_ops
;
317 * Generates an interprocessor interrupt to another CPU when X2APIC mode is
321 x2apic_send_ipi(int cpun
, int ipl
)
326 ASSERT(apic_mode
== LOCAL_X2APIC
);
329 * With X2APIC, Intel relaxed the semantics of the
330 * WRMSR instruction such that references to the X2APIC
331 * MSR registers are no longer serializing instructions.
332 * The code that initiates IPIs assumes that some sort
333 * of memory serialization occurs. The old APIC code
334 * did a write to uncachable memory mapped registers.
335 * Any reference to uncached memory is a serializing
336 * operation. To mimic those semantics here, we do an
337 * atomic operation, which translates to a LOCK OR instruction,
338 * which is serializing.
340 atomic_or_ulong(&flag
, 1);
342 vector
= apic_resv_vector
[ipl
];
347 * According to X2APIC specification in section '2.3.5.1' of
348 * Interrupt Command Register Semantics, the semantics of
349 * programming Interrupt Command Register to dispatch an interrupt
350 * is simplified. A single MSR write to the 64-bit ICR is required
351 * for dispatching an interrupt. Specifically with the 64-bit MSR
352 * interface to ICR, system software is not required to check the
353 * status of the delivery status bit prior to writing to the ICR
354 * to send an IPI. With the removal of the Delivery Status bit,
355 * system software no longer has a reason to read the ICR. It remains
356 * readable only to aid in debugging.
359 APIC_AV_PENDING_SET();
362 if ((cpun
== psm_get_cpu_id())) {
363 X2APIC_WRITE(X2APIC_SELF_IPI
, vector
);
365 apic_reg_ops
->apic_write_int_cmd(
366 apic_cpus
[cpun
].aci_local_id
, vector
);
373 * Generates IPI to another CPU depending on the local APIC mode.
374 * apic_send_ipi() and x2apic_send_ipi() depends on the configured
375 * mode of the local APIC, but that may not match the actual mode
376 * early in CPU startup.
378 * Any changes made to this routine must be accompanied by similar
379 * changes to apic_send_ipi().
382 apic_common_send_ipi(int cpun
, int ipl
)
386 int mode
= apic_local_mode();
388 if (mode
== LOCAL_X2APIC
) {
389 x2apic_send_ipi(cpun
, ipl
);
393 ASSERT(mode
== LOCAL_APIC
);
395 vector
= apic_resv_vector
[ipl
];
396 ASSERT((vector
>= APIC_BASE_VECT
) && (vector
<= APIC_SPUR_INTR
));
398 while (local_apic_regs_ops
.apic_read(APIC_INT_CMD1
) & AV_PENDING
)
400 local_apic_regs_ops
.apic_write_int_cmd(apic_cpus
[cpun
].aci_local_id
,