2 * This file is part of the coreboot project.
4 * Copyright (C) 2013 Google Inc.
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation; version 2 of
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
17 #include <console/console.h>
23 #include <cpu/intel/microcode.h>
24 #include <cpu/x86/cache.h>
25 #include <cpu/x86/gdt.h>
26 #include <cpu/x86/lapic.h>
27 #include <cpu/x86/name.h>
28 #include <cpu/x86/msr.h>
29 #include <cpu/x86/mtrr.h>
30 #include <cpu/x86/smm.h>
31 #include <cpu/x86/mp.h>
33 #include <device/device.h>
34 #include <device/path.h>
36 #include <smp/atomic.h>
37 #include <smp/spinlock.h>
41 #define MAX_APIC_IDS 256
48 * A mp_flight_record details a sequence of calls for the APs to perform
49 * along with the BSP to coordinate sequencing. Each flight record either
50 * provides a barrier for each AP before calling the callback or the APs
51 * are allowed to perform the callback without waiting. Regardless, each
52 * record has the cpus_entered field incremented for each record. When
53 * the BSP observes that the cpus_entered matches the number of APs
54 * the bsp_call is called with bsp_arg and upon returning releases the
55 * barrier allowing the APs to make further progress.
57 * Note that ap_call() and bsp_call() can be NULL. In the NULL case the
58 * callback will just not be called.
60 struct mp_flight_record
{
62 atomic_t cpus_entered
;
63 void (*ap_call
)(void);
64 void (*bsp_call
)(void);
65 } __aligned(CACHELINE_SIZE
);
67 #define _MP_FLIGHT_RECORD(barrier_, ap_func_, bsp_func_) \
69 .barrier = ATOMIC_INIT(barrier_), \
70 .cpus_entered = ATOMIC_INIT(0), \
71 .ap_call = ap_func_, \
72 .bsp_call = bsp_func_, \
75 #define MP_FR_BLOCK_APS(ap_func_, bsp_func_) \
76 _MP_FLIGHT_RECORD(0, ap_func_, bsp_func_)
78 #define MP_FR_NOBLOCK_APS(ap_func_, bsp_func_) \
79 _MP_FLIGHT_RECORD(1, ap_func_, bsp_func_)
81 /* The mp_params structure provides the arguments to the mp subsystem
82 * for bringing up APs. */
84 int num_cpus
; /* Total cpus include BSP */
85 int parallel_microcode_load
;
86 const void *microcode_pointer
;
87 /* Flight plan for APs and BSP. */
88 struct mp_flight_record
*flight_plan
;
92 /* This needs to match the layout in the .module_parametrs section. */
100 uint32_t microcode_lock
; /* 0xffffffff means parallel loading. */
101 uint32_t microcode_ptr
;
102 uint32_t msr_table_ptr
;
108 /* This also needs to match the assembly code for saved MSR encoding. */
116 /* The sipi vector rmodule is included in the ramstage using 'objdump -B'. */
117 extern char _binary_sipi_vector_start
[];
119 /* The SIPI vector is loaded at the SMM_DEFAULT_BASE. The reason is at the
120 * memory range is already reserved so the OS cannot use it. That region is
121 * free to use for AP bringup before SMM is initialized. */
122 static const uint32_t sipi_vector_location
= SMM_DEFAULT_BASE
;
123 static const int sipi_vector_location_size
= SMM_DEFAULT_SIZE
;
125 struct mp_flight_plan
{
127 struct mp_flight_record
*records
;
130 static int global_num_aps
;
131 static struct mp_flight_plan mp_info
;
135 /* Keep track of default apic ids for SMM. */
139 /* Keep track of APIC and device structure for each CPU. */
140 static struct cpu_map cpus
[CONFIG_MAX_CPUS
];
142 static inline void add_cpu_map_entry(const struct cpu_info
*info
)
144 cpus
[info
->index
].dev
= info
->cpu
;
145 cpus
[info
->index
].default_apic_id
= cpuid_ebx(1) >> 24;
148 static inline void barrier_wait(atomic_t
*b
)
150 while (atomic_read(b
) == 0)
155 static inline void release_barrier(atomic_t
*b
)
161 /* Returns 1 if timeout waiting for APs. 0 if target aps found. */
162 static int wait_for_aps(atomic_t
*val
, int target
, int total_delay
,
167 while (atomic_read(val
) != target
) {
169 delayed
+= delay_step
;
170 if (delayed
>= total_delay
) {
179 static void ap_do_flight_plan(void)
183 for (i
= 0; i
< mp_info
.num_records
; i
++) {
184 struct mp_flight_record
*rec
= &mp_info
.records
[i
];
186 atomic_inc(&rec
->cpus_entered
);
187 barrier_wait(&rec
->barrier
);
189 if (rec
->ap_call
!= NULL
)
194 static void park_this_cpu(void)
199 /* By the time APs call ap_init() caching has been setup, and microcode has
201 static void asmlinkage
ap_init(unsigned int cpu
)
203 struct cpu_info
*info
;
205 /* Ensure the local APIC is enabled */
210 info
->cpu
= cpus
[cpu
].dev
;
212 add_cpu_map_entry(info
);
213 thread_init_cpu_info_non_bsp(info
);
215 /* Fix up APIC id with reality. */
216 info
->cpu
->path
.apic
.apic_id
= lapicid();
218 printk(BIOS_INFO
, "AP: slot %d apic_id %x.\n", cpu
,
219 info
->cpu
->path
.apic
.apic_id
);
221 /* Walk the flight plan */
228 static void setup_default_sipi_vector_params(struct sipi_params
*sp
)
230 sp
->gdt
= (uint32_t)&gdt
;
231 sp
->gdtlimit
= (uint32_t)&gdt_end
- (u32
)&gdt
- 1;
232 sp
->idt_ptr
= (uint32_t)&idtarg
;
233 sp
->stack_size
= CONFIG_STACK_SIZE
;
234 sp
->stack_top
= (uint32_t)&_estack
;
235 /* Adjust the stack top to take into account cpu_info. */
236 sp
->stack_top
-= sizeof(struct cpu_info
);
239 #define NUM_FIXED_MTRRS 11
240 static const unsigned int fixed_mtrrs
[NUM_FIXED_MTRRS
] = {
241 MTRR_FIX_64K_00000
, MTRR_FIX_16K_80000
, MTRR_FIX_16K_A0000
,
242 MTRR_FIX_4K_C0000
, MTRR_FIX_4K_C8000
, MTRR_FIX_4K_D0000
,
243 MTRR_FIX_4K_D8000
, MTRR_FIX_4K_E0000
, MTRR_FIX_4K_E8000
,
244 MTRR_FIX_4K_F0000
, MTRR_FIX_4K_F8000
,
247 static inline struct saved_msr
*save_msr(int index
, struct saved_msr
*entry
)
252 entry
->index
= index
;
256 /* Return the next entry. */
261 static int save_bsp_msrs(char *start
, int size
)
265 struct saved_msr
*msr_entry
;
269 /* Determine number of MTRRs need to be saved. */
270 msr
= rdmsr(MTRR_CAP_MSR
);
271 num_var_mtrrs
= msr
.lo
& 0xff;
273 /* 2 * num_var_mtrrs for base and mask. +1 for IA32_MTRR_DEF_TYPE. */
274 msr_count
= 2 * num_var_mtrrs
+ NUM_FIXED_MTRRS
+ 1;
276 if ((msr_count
* sizeof(struct saved_msr
)) > size
) {
277 printk(BIOS_CRIT
, "Cannot mirror all %d msrs.\n", msr_count
);
281 fixed_mtrrs_expose_amd_rwdram();
283 msr_entry
= (void *)start
;
284 for (i
= 0; i
< NUM_FIXED_MTRRS
; i
++)
285 msr_entry
= save_msr(fixed_mtrrs
[i
], msr_entry
);
287 for (i
= 0; i
< num_var_mtrrs
; i
++) {
288 msr_entry
= save_msr(MTRR_PHYS_BASE(i
), msr_entry
);
289 msr_entry
= save_msr(MTRR_PHYS_MASK(i
), msr_entry
);
292 msr_entry
= save_msr(MTRR_DEF_TYPE_MSR
, msr_entry
);
294 fixed_mtrrs_hide_amd_rwdram();
299 static atomic_t
*load_sipi_vector(struct mp_params
*mp_params
)
301 struct rmodule sipi_mod
;
304 struct sipi_params
*sp
;
305 char *mod_loc
= (void *)sipi_vector_location
;
306 const int loc_size
= sipi_vector_location_size
;
307 atomic_t
*ap_count
= NULL
;
309 if (rmodule_parse(&_binary_sipi_vector_start
, &sipi_mod
)) {
310 printk(BIOS_CRIT
, "Unable to parse sipi module.\n");
314 if (rmodule_entry_offset(&sipi_mod
) != 0) {
315 printk(BIOS_CRIT
, "SIPI module entry offset is not 0!\n");
319 if (rmodule_load_alignment(&sipi_mod
) != 4096) {
320 printk(BIOS_CRIT
, "SIPI module load alignment(%d) != 4096.\n",
321 rmodule_load_alignment(&sipi_mod
));
325 module_size
= rmodule_memory_size(&sipi_mod
);
327 /* Align to 4 bytes. */
328 module_size
= ALIGN(module_size
, 4);
330 if (module_size
> loc_size
) {
331 printk(BIOS_CRIT
, "SIPI module size (%d) > region size (%d).\n",
332 module_size
, loc_size
);
336 num_msrs
= save_bsp_msrs(&mod_loc
[module_size
], loc_size
- module_size
);
339 printk(BIOS_CRIT
, "Error mirroring BSP's msrs.\n");
343 if (rmodule_load(mod_loc
, &sipi_mod
)) {
344 printk(BIOS_CRIT
, "Unable to load SIPI module.\n");
348 sp
= rmodule_parameters(&sipi_mod
);
351 printk(BIOS_CRIT
, "SIPI module has no parameters.\n");
355 setup_default_sipi_vector_params(sp
);
356 /* Setup MSR table. */
357 sp
->msr_table_ptr
= (uint32_t)&mod_loc
[module_size
];
358 sp
->msr_count
= num_msrs
;
359 /* Provide pointer to microcode patch. */
360 sp
->microcode_ptr
= (uint32_t)mp_params
->microcode_pointer
;
361 /* Pass on abiility to load microcode in parallel. */
362 if (mp_params
->parallel_microcode_load
)
363 sp
->microcode_lock
= 0;
365 sp
->microcode_lock
= ~0;
366 sp
->c_handler
= (uint32_t)&ap_init
;
367 ap_count
= &sp
->ap_count
;
368 atomic_set(ap_count
, 0);
373 static int allocate_cpu_devices(struct bus
*cpu_bus
, struct mp_params
*p
)
377 struct cpu_info
*info
;
379 max_cpus
= p
->num_cpus
;
380 if (max_cpus
> CONFIG_MAX_CPUS
) {
381 printk(BIOS_CRIT
, "CPU count(%d) exceeds CONFIG_MAX_CPUS(%d)\n",
382 max_cpus
, CONFIG_MAX_CPUS
);
383 max_cpus
= CONFIG_MAX_CPUS
;
387 for (i
= 1; i
< max_cpus
; i
++) {
388 struct device_path cpu_path
;
391 /* Build the CPU device path */
392 cpu_path
.type
= DEVICE_PATH_APIC
;
394 /* Assuming linear APIC space allocation. AP will set its own
395 APIC id in the ap_init() path above. */
396 cpu_path
.apic
.apic_id
= info
->cpu
->path
.apic
.apic_id
+ i
;
398 /* Allocate the new CPU device structure */
399 new = alloc_find_dev(cpu_bus
, &cpu_path
);
401 printk(BIOS_CRIT
, "Could not allocate CPU device\n");
410 /* Returns 1 for timeout. 0 on success. */
411 static int apic_wait_timeout(int total_delay
, int delay_step
)
416 while (lapic_read(LAPIC_ICR
) & LAPIC_ICR_BUSY
) {
419 if (total
>= total_delay
) {
428 static int start_aps(struct bus
*cpu_bus
, int ap_count
, atomic_t
*num_aps
)
431 /* Max location is 4KiB below 1MiB */
432 const int max_vector_loc
= ((1 << 20) - (1 << 12)) >> 12;
437 /* The vector is sent as a 4k aligned address in one byte. */
438 sipi_vector
= sipi_vector_location
>> 12;
440 if (sipi_vector
> max_vector_loc
) {
441 printk(BIOS_CRIT
, "SIPI vector too large! 0x%08x\n",
446 printk(BIOS_DEBUG
, "Attempting to start %d APs\n", ap_count
);
448 if ((lapic_read(LAPIC_ICR
) & LAPIC_ICR_BUSY
)) {
449 printk(BIOS_DEBUG
, "Waiting for ICR not to be busy...");
450 if (apic_wait_timeout(1000 /* 1 ms */, 50)) {
451 printk(BIOS_DEBUG
, "timed out. Aborting.\n");
454 printk(BIOS_DEBUG
, "done.\n");
457 /* Send INIT IPI to all but self. */
458 lapic_write_around(LAPIC_ICR2
, SET_LAPIC_DEST_FIELD(0));
459 lapic_write_around(LAPIC_ICR
, LAPIC_DEST_ALLBUT
| LAPIC_INT_ASSERT
|
461 printk(BIOS_DEBUG
, "Waiting for 10ms after sending INIT.\n");
465 if ((lapic_read(LAPIC_ICR
) & LAPIC_ICR_BUSY
)) {
466 printk(BIOS_DEBUG
, "Waiting for ICR not to be busy...");
467 if (apic_wait_timeout(1000 /* 1 ms */, 50)) {
468 printk(BIOS_DEBUG
, "timed out. Aborting.\n");
471 printk(BIOS_DEBUG
, "done.\n");
474 lapic_write_around(LAPIC_ICR2
, SET_LAPIC_DEST_FIELD(0));
475 lapic_write_around(LAPIC_ICR
, LAPIC_DEST_ALLBUT
| LAPIC_INT_ASSERT
|
476 LAPIC_DM_STARTUP
| sipi_vector
);
477 printk(BIOS_DEBUG
, "Waiting for 1st SIPI to complete...");
478 if (apic_wait_timeout(10000 /* 10 ms */, 50 /* us */)) {
479 printk(BIOS_DEBUG
, "timed out.\n");
482 printk(BIOS_DEBUG
, "done.\n");
484 /* Wait for CPUs to check in up to 200 us. */
485 wait_for_aps(num_aps
, ap_count
, 200 /* us */, 15 /* us */);
488 if ((lapic_read(LAPIC_ICR
) & LAPIC_ICR_BUSY
)) {
489 printk(BIOS_DEBUG
, "Waiting for ICR not to be busy...");
490 if (apic_wait_timeout(1000 /* 1 ms */, 50)) {
491 printk(BIOS_DEBUG
, "timed out. Aborting.\n");
494 printk(BIOS_DEBUG
, "done.\n");
497 lapic_write_around(LAPIC_ICR2
, SET_LAPIC_DEST_FIELD(0));
498 lapic_write_around(LAPIC_ICR
, LAPIC_DEST_ALLBUT
| LAPIC_INT_ASSERT
|
499 LAPIC_DM_STARTUP
| sipi_vector
);
500 printk(BIOS_DEBUG
, "Waiting for 2nd SIPI to complete...");
501 if (apic_wait_timeout(10000 /* 10 ms */, 50 /* us */)) {
502 printk(BIOS_DEBUG
, "timed out.\n");
505 printk(BIOS_DEBUG
, "done.\n");
507 /* Wait for CPUs to check in. */
508 if (wait_for_aps(num_aps
, ap_count
, 10000 /* 10 ms */, 50 /* us */)) {
509 printk(BIOS_DEBUG
, "Not all APs checked in: %d/%d.\n",
510 atomic_read(num_aps
), ap_count
);
517 static int bsp_do_flight_plan(struct mp_params
*mp_params
)
522 * Set time-out to wait for APs to a huge value (=1 second) since it
523 * could take a longer time for APs to check-in as the number of APs
524 * increases (contention for resources like UART also increases).
526 const int timeout_us
= 1000000;
527 const int step_us
= 100;
528 int num_aps
= mp_params
->num_cpus
- 1;
533 for (i
= 0; i
< mp_params
->num_records
; i
++) {
534 struct mp_flight_record
*rec
= &mp_params
->flight_plan
[i
];
536 /* Wait for APs if the record is not released. */
537 if (atomic_read(&rec
->barrier
) == 0) {
538 /* Wait for the APs to check in. */
539 if (wait_for_aps(&rec
->cpus_entered
, num_aps
,
540 timeout_us
, step_us
)) {
541 printk(BIOS_ERR
, "MP record %d timeout.\n", i
);
546 if (rec
->bsp_call
!= NULL
)
549 release_barrier(&rec
->barrier
);
552 printk(BIOS_INFO
, "%s done after %ld msecs.\n", __func__
,
553 stopwatch_duration_msecs(&sw
));
557 static void init_bsp(struct bus
*cpu_bus
)
559 struct device_path cpu_path
;
560 struct cpu_info
*info
;
561 char processor_name
[49];
563 /* Print processor name */
564 fill_processor_name(processor_name
);
565 printk(BIOS_INFO
, "CPU: %s.\n", processor_name
);
567 /* Ensure the local APIC is enabled */
570 /* Set the device path of the boot CPU. */
571 cpu_path
.type
= DEVICE_PATH_APIC
;
572 cpu_path
.apic
.apic_id
= lapicid();
574 /* Find the device structure for the boot CPU. */
576 info
->cpu
= alloc_find_dev(cpu_bus
, &cpu_path
);
578 if (info
->index
!= 0)
579 printk(BIOS_CRIT
, "BSP index(%d) != 0!\n", info
->index
);
581 /* Track BSP in cpu_map structures. */
582 add_cpu_map_entry(info
);
586 * mp_init() will set up the SIPI vector and bring up the APs according to
587 * mp_params. Each flight record will be executed according to the plan. Note
588 * that the MP infrastructure uses SMM default area without saving it. It's
589 * up to the chipset or mainboard to either e820 reserve this area or save this
590 * region prior to calling mp_init() and restoring it after mp_init returns.
592 * At the time mp_init() is called the MTRR MSRs are mirrored into APs then
593 * caching is enabled before running the flight plan.
595 * The MP initialization has the following properties:
596 * 1. APs are brought up in parallel.
597 * 2. The ordering of coreboot CPU number and APIC ids is not deterministic.
598 * Therefore, one cannot rely on this property or the order of devices in
599 * the device tree unless the chipset or mainboard know the APIC ids
602 * mp_init() returns < 0 on error, 0 on success.
604 static int mp_init(struct bus
*cpu_bus
, struct mp_params
*p
)
611 if (p
== NULL
|| p
->flight_plan
== NULL
|| p
->num_records
< 1) {
612 printk(BIOS_CRIT
, "Invalid MP parameters\n");
616 /* Default to currently running CPU. */
617 num_cpus
= allocate_cpu_devices(cpu_bus
, p
);
619 if (num_cpus
< p
->num_cpus
) {
621 "ERROR: More cpus requested (%d) than supported (%d).\n",
622 p
->num_cpus
, num_cpus
);
626 /* Copy needed parameters so that APs have a reference to the plan. */
627 mp_info
.num_records
= p
->num_records
;
628 mp_info
.records
= p
->flight_plan
;
630 /* Load the SIPI vector. */
631 ap_count
= load_sipi_vector(p
);
632 if (ap_count
== NULL
)
635 /* Make sure SIPI data hits RAM so the APs that come up will see
636 * the startup code even if the caches are disabled. */
639 /* Start the APs providing number of APs and the cpus_entered field. */
640 global_num_aps
= p
->num_cpus
- 1;
641 if (start_aps(cpu_bus
, global_num_aps
, ap_count
) < 0) {
643 printk(BIOS_DEBUG
, "%d/%d eventually checked in?\n",
644 atomic_read(ap_count
), global_num_aps
);
648 /* Walk the flight plan for the BSP. */
649 return bsp_do_flight_plan(p
);
652 /* Calls cpu_initialize(info->index) which calls the coreboot CPU drivers. */
653 static void mp_initialize_cpu(void)
655 /* Call back into driver infrastructure for the AP initialization. */
656 struct cpu_info
*info
= cpu_info();
657 cpu_initialize(info
->index
);
660 /* Returns APIC id for coreboot CPU number or < 0 on failure. */
661 static int mp_get_apic_id(int cpu_slot
)
663 if (cpu_slot
>= CONFIG_MAX_CPUS
|| cpu_slot
< 0)
666 return cpus
[cpu_slot
].default_apic_id
;
669 void smm_initiate_relocation_parallel(void)
671 if ((lapic_read(LAPIC_ICR
) & LAPIC_ICR_BUSY
)) {
672 printk(BIOS_DEBUG
, "Waiting for ICR not to be busy...");
673 if (apic_wait_timeout(1000 /* 1 ms */, 50)) {
674 printk(BIOS_DEBUG
, "timed out. Aborting.\n");
677 printk(BIOS_DEBUG
, "done.\n");
680 lapic_write_around(LAPIC_ICR2
, SET_LAPIC_DEST_FIELD(lapicid()));
681 lapic_write_around(LAPIC_ICR
, LAPIC_INT_ASSERT
| LAPIC_DM_SMI
);
682 if (apic_wait_timeout(1000 /* 1 ms */, 100 /* us */))
683 printk(BIOS_DEBUG
, "SMI Relocation timed out.\n");
685 printk(BIOS_DEBUG
, "Relocation complete.\n");
688 DECLARE_SPIN_LOCK(smm_relocation_lock
);
690 /* Send SMI to self with single user serialization. */
691 void smm_initiate_relocation(void)
693 spin_lock(&smm_relocation_lock
);
694 smm_initiate_relocation_parallel();
695 spin_unlock(&smm_relocation_lock
);
701 uintptr_t perm_smbase
;
703 size_t smm_save_state_size
;
707 static int is_smm_enabled(void)
709 return IS_ENABLED(CONFIG_HAVE_SMI_HANDLER
) && mp_state
.do_smm
;
712 static void smm_disable(void)
717 static void smm_enable(void)
719 if (IS_ENABLED(CONFIG_HAVE_SMI_HANDLER
))
723 static void asmlinkage
smm_do_relocation(void *arg
)
725 const struct smm_module_params
*p
;
726 const struct smm_runtime
*runtime
;
728 uintptr_t curr_smbase
;
729 uintptr_t perm_smbase
;
732 runtime
= p
->runtime
;
734 curr_smbase
= runtime
->smbase
;
736 if (cpu
>= CONFIG_MAX_CPUS
) {
738 "Invalid CPU number assigned in SMM stub: %d\n", cpu
);
743 * The permanent handler runs with all cpus concurrently. Precalculate
744 * the location of the new SMBASE. If using SMM modules then this
745 * calculation needs to match that of the module loader.
747 perm_smbase
= mp_state
.perm_smbase
;
748 perm_smbase
-= cpu
* runtime
->save_state_size
;
750 printk(BIOS_DEBUG
, "New SMBASE 0x%08lx\n", perm_smbase
);
752 /* Setup code checks this callback for validity. */
753 mp_state
.ops
.relocation_handler(cpu
, curr_smbase
, perm_smbase
);
756 static void adjust_smm_apic_id_map(struct smm_loader_params
*smm_params
)
759 struct smm_runtime
*runtime
= smm_params
->runtime
;
761 for (i
= 0; i
< CONFIG_MAX_CPUS
; i
++)
762 runtime
->apic_id_to_cpu
[i
] = mp_get_apic_id(i
);
765 static int install_relocation_handler(int num_cpus
, size_t save_state_size
)
767 struct smm_loader_params smm_params
= {
768 .per_cpu_stack_size
= save_state_size
,
769 .num_concurrent_stacks
= num_cpus
,
770 .per_cpu_save_state_size
= save_state_size
,
771 .num_concurrent_save_states
= 1,
772 .handler
= smm_do_relocation
,
775 /* Allow callback to override parameters. */
776 if (mp_state
.ops
.adjust_smm_params
!= NULL
)
777 mp_state
.ops
.adjust_smm_params(&smm_params
, 0);
779 if (smm_setup_relocation_handler(&smm_params
))
782 adjust_smm_apic_id_map(&smm_params
);
787 static int install_permanent_handler(int num_cpus
, uintptr_t smbase
,
788 size_t smsize
, size_t save_state_size
)
790 /* There are num_cpus concurrent stacks and num_cpus concurrent save
791 * state areas. Lastly, set the stack size to 1KiB. */
792 struct smm_loader_params smm_params
= {
793 .per_cpu_stack_size
= 1 * KiB
,
794 .num_concurrent_stacks
= num_cpus
,
795 .per_cpu_save_state_size
= save_state_size
,
796 .num_concurrent_save_states
= num_cpus
,
799 /* Allow callback to override parameters. */
800 if (mp_state
.ops
.adjust_smm_params
!= NULL
)
801 mp_state
.ops
.adjust_smm_params(&smm_params
, 1);
803 printk(BIOS_DEBUG
, "Installing SMM handler to 0x%08lx\n", smbase
);
805 if (smm_load_module((void *)smbase
, smsize
, &smm_params
))
808 adjust_smm_apic_id_map(&smm_params
);
813 /* Load SMM handlers as part of MP flight record. */
814 static void load_smm_handlers(void)
816 size_t smm_save_state_size
= mp_state
.smm_save_state_size
;
818 /* Do nothing if SMM is disabled.*/
819 if (!is_smm_enabled())
822 /* Install handlers. */
823 if (install_relocation_handler(mp_state
.cpu_count
,
824 smm_save_state_size
) < 0) {
825 printk(BIOS_ERR
, "Unable to install SMM relocation handler.\n");
829 if (install_permanent_handler(mp_state
.cpu_count
, mp_state
.perm_smbase
,
830 mp_state
.perm_smsize
, smm_save_state_size
) < 0) {
831 printk(BIOS_ERR
, "Unable to install SMM permanent handler.\n");
835 /* Ensure the SMM handlers hit DRAM before performing first SMI. */
839 * Indicate that the SMM handlers have been loaded and MP
840 * initialization is about to start.
842 if (is_smm_enabled() && mp_state
.ops
.pre_mp_smm_init
!= NULL
)
843 mp_state
.ops
.pre_mp_smm_init();
846 /* Trigger SMM as part of MP flight record. */
847 static void trigger_smm_relocation(void)
849 /* Do nothing if SMM is disabled.*/
850 if (!is_smm_enabled() || mp_state
.ops
.per_cpu_smm_trigger
== NULL
)
852 /* Trigger SMM mode for the currently running processor. */
853 mp_state
.ops
.per_cpu_smm_trigger();
856 static struct mp_callback
*ap_callbacks
[CONFIG_MAX_CPUS
];
858 static struct mp_callback
*read_callback(struct mp_callback
**slot
)
860 struct mp_callback
*ret
;
862 asm volatile ("mov %1, %0\n"
870 static void store_callback(struct mp_callback
**slot
, struct mp_callback
*val
)
872 asm volatile ("mov %1, %0\n"
879 static int run_ap_work(struct mp_callback
*val
, long expire_us
)
884 int cur_cpu
= cpu_index();
886 if (!IS_ENABLED(CONFIG_PARALLEL_MP_AP_WORK
)) {
887 printk(BIOS_ERR
, "APs already parked. PARALLEL_MP_AP_WORK not selected.\n");
891 /* Signal to all the APs to run the func. */
892 for (i
= 0; i
< ARRAY_SIZE(ap_callbacks
); i
++) {
895 store_callback(&ap_callbacks
[i
], val
);
899 /* Wait for all the APs to signal back that call has been accepted. */
901 stopwatch_init_usecs_expire(&sw
, expire_us
);
906 for (i
= 0; i
< ARRAY_SIZE(ap_callbacks
); i
++) {
909 if (read_callback(&ap_callbacks
[i
]) == NULL
)
913 if (cpus_accepted
== global_num_aps
)
915 } while (expire_us
<= 0 || !stopwatch_expired(&sw
));
917 printk(BIOS_ERR
, "AP call expired. %d/%d CPUs accepted.\n",
918 cpus_accepted
, global_num_aps
);
922 static void ap_wait_for_instruction(void)
924 struct mp_callback lcb
;
925 struct mp_callback
**per_cpu_slot
;
927 if (!IS_ENABLED(CONFIG_PARALLEL_MP_AP_WORK
))
930 per_cpu_slot
= &ap_callbacks
[cpu_index()];
933 struct mp_callback
*cb
= read_callback(per_cpu_slot
);
940 /* Copy to local variable before signalling consumption. */
941 memcpy(&lcb
, cb
, sizeof(lcb
));
943 store_callback(per_cpu_slot
, NULL
);
948 int mp_run_on_aps(void (*func
)(void), long expire_us
)
950 struct mp_callback lcb
= { .func
= func
};
951 return run_ap_work(&lcb
, expire_us
);
954 int mp_run_on_all_cpus(void (*func
)(void), long expire_us
)
956 /* Run on BSP first. */
958 return mp_run_on_aps(func
, expire_us
);
961 int mp_park_aps(void)
969 ret
= mp_run_on_aps(park_this_cpu
, 250 * USECS_PER_MSEC
);
971 duration_msecs
= stopwatch_duration_msecs(&sw
);
974 printk(BIOS_DEBUG
, "%s done after %ld msecs.\n", __func__
,
977 printk(BIOS_ERR
, "%s failed after %ld msecs.\n", __func__
,
983 static struct mp_flight_record mp_steps
[] = {
984 /* Once the APs are up load the SMM handlers. */
985 MP_FR_BLOCK_APS(NULL
, load_smm_handlers
),
986 /* Perform SMM relocation. */
987 MP_FR_NOBLOCK_APS(trigger_smm_relocation
, trigger_smm_relocation
),
988 /* Initialize each CPU through the driver framework. */
989 MP_FR_BLOCK_APS(mp_initialize_cpu
, mp_initialize_cpu
),
990 /* Wait for APs to finish then optionally start looking for work. */
991 MP_FR_BLOCK_APS(ap_wait_for_instruction
, NULL
),
994 static void fill_mp_state(struct mp_state
*state
, const struct mp_ops
*ops
)
997 * Make copy of the ops so that defaults can be set in the non-const
998 * structure if needed.
1000 memcpy(&state
->ops
, ops
, sizeof(*ops
));
1002 if (ops
->get_cpu_count
!= NULL
)
1003 state
->cpu_count
= ops
->get_cpu_count();
1005 if (ops
->get_smm_info
!= NULL
)
1006 ops
->get_smm_info(&state
->perm_smbase
, &state
->perm_smsize
,
1007 &state
->smm_save_state_size
);
1010 * Default to smm_initiate_relocation() if trigger callback isn't
1013 if (IS_ENABLED(CONFIG_HAVE_SMI_HANDLER
) &&
1014 ops
->per_cpu_smm_trigger
== NULL
)
1015 mp_state
.ops
.per_cpu_smm_trigger
= smm_initiate_relocation
;
1018 int mp_init_with_smm(struct bus
*cpu_bus
, const struct mp_ops
*mp_ops
)
1021 void *default_smm_area
;
1022 struct mp_params mp_params
;
1024 if (mp_ops
->pre_mp_init
!= NULL
)
1025 mp_ops
->pre_mp_init();
1027 fill_mp_state(&mp_state
, mp_ops
);
1029 memset(&mp_params
, 0, sizeof(mp_params
));
1031 if (mp_state
.cpu_count
<= 0) {
1032 printk(BIOS_ERR
, "Invalid cpu_count: %d\n", mp_state
.cpu_count
);
1036 /* Sanity check SMM state. */
1037 if (mp_state
.perm_smsize
!= 0 && mp_state
.smm_save_state_size
!= 0 &&
1038 mp_state
.ops
.relocation_handler
!= NULL
)
1041 if (is_smm_enabled())
1042 printk(BIOS_INFO
, "Will perform SMM setup.\n");
1044 mp_params
.num_cpus
= mp_state
.cpu_count
;
1045 /* Gather microcode information. */
1046 if (mp_state
.ops
.get_microcode_info
!= NULL
)
1047 mp_state
.ops
.get_microcode_info(&mp_params
.microcode_pointer
,
1048 &mp_params
.parallel_microcode_load
);
1049 mp_params
.flight_plan
= &mp_steps
[0];
1050 mp_params
.num_records
= ARRAY_SIZE(mp_steps
);
1052 /* Perform backup of default SMM area. */
1053 default_smm_area
= backup_default_smm_area();
1055 ret
= mp_init(cpu_bus
, &mp_params
);
1057 restore_default_smm_area(default_smm_area
);
1059 /* Signal callback on success if it's provided. */
1060 if (ret
== 0 && mp_state
.ops
.post_mp_init
!= NULL
)
1061 mp_state
.ops
.post_mp_init();