2 * Intel MP v1.1/v1.4 specification support routines for multi-pentium
5 * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
8 * Supported by Caldera http://www.caldera.com.
9 * Much of the core SMP work is based on previous work by Thomas Radke, to
10 * whom a great many thanks are extended.
12 * Thanks to Intel for making available several different Pentium,
13 * Pentium Pro and Pentium-II/Xeon MP machines.
15 * This code is released under the GNU public license version 2 or
19 * Felix Koop : NR_CPUS used properly
20 * Jose Renau : Handle single CPU case.
21 * Alan Cox : By repeated request 8) - Total BogoMIP report.
22 * Greg Wright : Fix for kernel stacks panic.
23 * Erich Boleyn : MP v1.4 and additional changes.
24 * Matthias Sattler : Changes for 2.1 kernel map.
25 * Michel Lespinasse : Changes for 2.1 kernel map.
26 * Michael Chastain : Change trampoline.S to gnu as.
27 * Alan Cox : Dumb bug: 'B' step PPro's are fine
28 * Ingo Molnar : Added APIC timers, based on code
30 * Alan Cox : Added EBDA scanning
31 * Ingo Molnar : various cleanups and rewrites
32 * Tigran Aivazian : fixed "0.00 in /proc/uptime on SMP" bug.
35 #include <linux/config.h>
37 #include <linux/kernel_stat.h>
38 #include <linux/delay.h>
39 #include <linux/mc146818rtc.h>
40 #include <linux/smp_lock.h>
41 #include <linux/init.h>
47 #define JIFFIE_TIMEOUT 100
49 extern void update_one_process( struct task_struct
*p
,
50 unsigned long ticks
, unsigned long user
,
51 unsigned long system
, int cpu
);
53 * Some notes on processor bugs:
55 * Pentium and Pentium Pro (and all CPUs) have bugs. The Linux issues
56 * for SMP are handled as follows.
59 * Occasional delivery of 'spurious interrupt' as trap #16. This
60 * is very rare. The kernel logs the event and recovers
63 * There is a marginal case where REP MOVS on 100MHz SMP
64 * machines with B stepping processors can fail. XXX should provide
65 * an L1cache=Writethrough or L1cache=off option.
67 * B stepping CPUs may hang. There are hardware work arounds
68 * for this. We warn about it in case your board doesnt have the work
69 * arounds. Basically thats so I can tell anyone with a B stepping
70 * CPU and SMP problems "tough".
72 * Specific items [From Pentium Processor Specification Update]
74 * 1AP. Linux doesn't use remote read
75 * 2AP. Linux doesn't trust APIC errors
76 * 3AP. We work around this
77 * 4AP. Linux never generated 3 interrupts of the same priority
78 * to cause a lost local interrupt.
79 * 5AP. Remote read is never used
80 * 9AP. XXX NEED TO CHECK WE HANDLE THIS XXX
81 * 10AP. XXX NEED TO CHECK WE HANDLE THIS XXX
82 * 11AP. Linux reads the APIC between writes to avoid this, as per
83 * the documentation. Make sure you preserve this as it affects
84 * the C stepping chips too.
86 * If this sounds worrying believe me these bugs are ___RARE___ and
87 * there's about nothing of note with C stepping upwards.
92 spinlock_t kernel_flag
= SPIN_LOCK_UNLOCKED
;
95 * function prototypes:
97 static void cache_APIC_registers (void);
98 static void stop_this_cpu (void);
100 static int smp_b_stepping
= 0; /* Set if we find a B stepping CPU */
102 static int max_cpus
= -1; /* Setup configured maximum number of CPUs to activate */
103 int smp_found_config
=0; /* Have we found an SMP box */
105 unsigned long cpu_present_map
= 0; /* Bitmask of physically existing CPUs */
106 unsigned long cpu_online_map
= 0; /* Bitmask of currently online CPUs */
107 int smp_num_cpus
= 0; /* Total count of live CPUs */
108 int smp_threads_ready
=0; /* Set when the idlers are all forked */
109 volatile int cpu_number_map
[NR_CPUS
]; /* which CPU maps to which logical number */
110 volatile int __cpu_logical_map
[NR_CPUS
]; /* which logical number maps to which CPU */
111 static volatile unsigned long cpu_callin_map
[NR_CPUS
] = {0,}; /* We always use 0 the rest is ready for parallel delivery */
112 static volatile unsigned long cpu_callout_map
[NR_CPUS
] = {0,}; /* We always use 0 the rest is ready for parallel delivery */
113 volatile unsigned long smp_invalidate_needed
; /* Used for the invalidate map that's also checked in the spinlock */
114 volatile unsigned long kstack_ptr
; /* Stack vector for booting CPUs */
115 struct cpuinfo_x86 cpu_data
[NR_CPUS
]; /* Per CPU bogomips and other parameters */
116 static unsigned int num_processors
= 1; /* Internal processor count */
117 unsigned long mp_ioapic_addr
= 0xFEC00000; /* Address of the I/O apic (not yet used) */
118 unsigned char boot_cpu_id
= 0; /* Processor that is doing the boot up */
119 static int smp_activated
= 0; /* Tripped once we need to start cross invalidating */
120 int apic_version
[NR_CPUS
]; /* APIC version number */
121 unsigned long apic_retval
; /* Just debugging the assembler.. */
123 volatile unsigned long kernel_counter
=0; /* Number of times the processor holds the lock */
124 volatile unsigned long syscall_count
=0; /* Number of times the processor holds the syscall lock */
126 volatile unsigned long ipi_count
; /* Number of IPIs delivered */
128 const char lk_lockmsg
[] = "lock from interrupt context at %p\n";
130 int mp_bus_id_to_type
[MAX_MP_BUSSES
] = { -1, };
131 extern int mp_irq_entries
;
132 extern struct mpc_config_intsrc mp_irqs
[MAX_IRQ_SOURCES
];
133 extern int mpc_default_type
;
134 int mp_bus_id_to_pci_bus
[MAX_MP_BUSSES
] = { -1, };
135 int mp_current_pci_id
= 0;
136 unsigned long mp_lapic_addr
= 0;
137 int skip_ioapic_setup
= 0; /* 1 if "noapic" boot option passed */
139 /* #define SMP_DEBUG */
142 #define SMP_PRINTK(x) printk x
144 #define SMP_PRINTK(x)
148 * IA s/w dev Vol 3, Section 7.4
150 #define APIC_DEFAULT_PHYS_BASE 0xfee00000
152 #define CLEAR_TSC wrmsr(0x10, 0x00001000, 0x00001000)
155 * Setup routine for controlling SMP activation
157 * Command-line option of "nosmp" or "maxcpus=0" will disable SMP
158 * activation entirely (the MPS table probe still happens, though).
160 * Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
161 * greater than 0, limits the maximum number of CPUs activated in
165 void __init
smp_setup(char *str
, int *ints
)
167 if (ints
&& ints
[0] > 0)
173 void ack_APIC_irq(void)
178 apic_read(APIC_SPIV
);
180 /* Docs say use 0 for future compatibility */
181 apic_write(APIC_EOI
, 0);
185 * Intel MP BIOS table parsing routines:
188 #ifndef CONFIG_X86_VISWS_APIC
190 * Checksum an MP configuration block.
193 static int mpf_checksum(unsigned char *mp
, int len
)
202 * Processor encoding in an MP configuration block
205 static char *mpc_family(int family
,int model
)
208 static char *model_defs
[]=
211 "80486SX","80486DX/2 or 80487",
212 "80486SL","Intel5X2(tm)",
217 return("Pentium(tm) Pro");
219 return("Pentium(tm)");
220 if (family
==0x0F && model
==0x0F)
221 return("Special controller");
222 if (family
==0x04 && model
<9)
223 return model_defs
[model
];
224 sprintf(n
,"Unknown CPU [%d:%d]",family
, model
);
233 static int __init
smp_read_mpc(struct mp_config_table
*mpc
)
236 int count
=sizeof(*mpc
);
238 unsigned char *mpt
=((unsigned char *)mpc
)+count
;
240 if (memcmp(mpc
->mpc_signature
,MPC_SIGNATURE
,4))
242 panic("SMP mptable: bad signature [%c%c%c%c]!\n",
243 mpc
->mpc_signature
[0],
244 mpc
->mpc_signature
[1],
245 mpc
->mpc_signature
[2],
246 mpc
->mpc_signature
[3]);
249 if (mpf_checksum((unsigned char *)mpc
,mpc
->mpc_length
))
251 panic("SMP mptable: checksum error!\n");
254 if (mpc
->mpc_spec
!=0x01 && mpc
->mpc_spec
!=0x04)
256 printk("Bad Config Table version (%d)!!\n",mpc
->mpc_spec
);
259 memcpy(str
,mpc
->mpc_oem
,8);
261 memcpy(ioapic_OEM_ID
,str
,9);
262 printk("OEM ID: %s ",str
);
264 memcpy(str
,mpc
->mpc_productid
,12);
266 memcpy(ioapic_Product_ID
,str
,13);
267 printk("Product ID: %s ",str
);
269 printk("APIC at: 0x%lX\n",mpc
->mpc_lapic
);
271 /* save the local APIC address, it might be non-default */
272 mp_lapic_addr
= mpc
->mpc_lapic
;
275 * Now process the configuration blocks.
278 while(count
<mpc
->mpc_length
)
284 struct mpc_config_processor
*m
=
285 (struct mpc_config_processor
*)mpt
;
286 if (m
->mpc_cpuflag
&CPU_ENABLED
)
288 printk("Processor #%d %s APIC version %d\n",
290 mpc_family((m
->mpc_cpufeature
&
296 if (m
->mpc_featureflag
&(1<<0))
297 printk(" Floating point unit present.\n");
298 if (m
->mpc_featureflag
&(1<<7))
299 printk(" Machine Exception supported.\n");
300 if (m
->mpc_featureflag
&(1<<8))
301 printk(" 64 bit compare & exchange supported.\n");
302 if (m
->mpc_featureflag
&(1<<9))
303 printk(" Internal APIC present.\n");
305 if (m
->mpc_cpuflag
&CPU_BOOTPROCESSOR
)
307 SMP_PRINTK((" Bootup CPU\n"));
308 boot_cpu_id
=m
->mpc_apicid
;
310 else /* Boot CPU already counted */
313 if (m
->mpc_apicid
>NR_CPUS
)
314 printk("Processor #%d unused. (Max %d processors).\n",m
->mpc_apicid
, NR_CPUS
);
317 int ver
= m
->mpc_apicver
;
319 cpu_present_map
|=(1<<m
->mpc_apicid
);
324 printk("BIOS bug, APIC version is 0 for CPU#%d! fixing up to 0x10. (tell your hw vendor)\n", m
->mpc_apicid
);
327 apic_version
[m
->mpc_apicid
] = ver
;
336 struct mpc_config_bus
*m
=
337 (struct mpc_config_bus
*)mpt
;
338 memcpy(str
,m
->mpc_bustype
,6);
340 SMP_PRINTK(("Bus #%d is %s\n",
343 if (strncmp(m
->mpc_bustype
,"ISA",3) == 0)
344 mp_bus_id_to_type
[m
->mpc_busid
] =
347 if (strncmp(m
->mpc_bustype
,"EISA",4) == 0)
348 mp_bus_id_to_type
[m
->mpc_busid
] =
350 if (strncmp(m
->mpc_bustype
,"PCI",3) == 0) {
351 mp_bus_id_to_type
[m
->mpc_busid
] =
353 mp_bus_id_to_pci_bus
[m
->mpc_busid
] =
363 struct mpc_config_ioapic
*m
=
364 (struct mpc_config_ioapic
*)mpt
;
365 if (m
->mpc_flags
&MPC_APIC_USABLE
)
368 printk("I/O APIC #%d Version %d at 0x%lX.\n",
369 m
->mpc_apicid
,m
->mpc_apicver
,
372 * we use the first one only currently
375 mp_ioapic_addr
= m
->mpc_apicaddr
;
383 struct mpc_config_intsrc
*m
=
384 (struct mpc_config_intsrc
*)mpt
;
386 mp_irqs
[mp_irq_entries
] = *m
;
387 if (++mp_irq_entries
== MAX_IRQ_SOURCES
) {
388 printk("Max irq sources exceeded!!\n");
389 printk("Skipping remaining sources.\n");
399 struct mpc_config_intlocal
*m
=
400 (struct mpc_config_intlocal
*)mpt
;
409 printk("Warning: Multiple IO-APICs not yet supported.\n");
410 printk("Warning: switching to non APIC mode.\n");
413 return num_processors
;
417 * Scan the memory blocks for an SMP configuration block.
420 static int __init
smp_scan_config(unsigned long base
, unsigned long length
)
422 unsigned long *bp
=phys_to_virt(base
);
423 struct intel_mp_floating
*mpf
;
425 SMP_PRINTK(("Scan SMP from %p for %ld bytes.\n",
427 if (sizeof(*mpf
)!=16)
428 printk("Error: MPF size\n");
432 if (*bp
==SMP_MAGIC_IDENT
)
434 mpf
=(struct intel_mp_floating
*)bp
;
435 if (mpf
->mpf_length
==1 &&
436 !mpf_checksum((unsigned char *)bp
,16) &&
437 (mpf
->mpf_specification
== 1
438 || mpf
->mpf_specification
== 4) )
440 printk("Intel MultiProcessor Specification v1.%d\n", mpf
->mpf_specification
);
441 if (mpf
->mpf_feature2
&(1<<7))
442 printk(" IMCR and PIC compatibility mode.\n");
444 printk(" Virtual Wire compatibility mode.\n");
447 * Now see if we need to read further.
449 if (mpf
->mpf_feature1
!=0)
453 /* local APIC has default address */
454 mp_lapic_addr
= APIC_DEFAULT_PHYS_BASE
;
456 * We need to know what the local
457 * APIC id of the boot CPU is!
462 * HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
464 * It's not just a crazy hack. ;-)
467 * Standard page mapping
468 * functions don't work yet.
469 * We know that page 0 is not
470 * used. Steal it for now!
474 pg0
[0] = (mp_lapic_addr
| _PAGE_RW
| _PAGE_PRESENT
);
477 boot_cpu_id
= GET_APIC_ID(*((volatile unsigned long *) APIC_ID
));
488 * END OF HACK END OF HACK END OF HACK END OF HACK END OF HACK
492 * 2 CPUs, numbered 0 & 1.
496 printk("I/O APIC at 0xFEC00000.\n");
499 * Save the default type number, we
500 * need it later to set the IO-APIC
503 mpc_default_type
= mpf
->mpf_feature1
;
505 printk("Bus #0 is ");
507 switch(mpf
->mpf_feature1
)
514 printk("EISA with no IRQ8 chaining\n");
527 printk("???\nUnknown standard configuration %d\n",
531 if (mpf
->mpf_feature1
>4)
533 printk("Bus #1 is PCI\n");
536 * Set local APIC version to
537 * the integrated form.
538 * It's initialized to zero
539 * otherwise, representing
540 * a discrete 82489DX.
542 apic_version
[0] = 0x10;
543 apic_version
[1] = 0x10;
546 * Read the physical hardware table.
547 * Anything here will override the
550 if (mpf
->mpf_physptr
)
551 smp_read_mpc((void *)mpf
->mpf_physptr
);
553 __cpu_logical_map
[0] = boot_cpu_id
;
554 global_irq_holder
= boot_cpu_id
;
555 current
->processor
= boot_cpu_id
;
557 printk("Processors: %d\n", num_processors
);
559 * Only use the first configuration found.
571 void __init
init_intel_smp (void)
574 * FIXME: Linux assumes you have 640K of base ram..
575 * this continues the error...
577 * 1) Scan the bottom 1K for a signature
578 * 2) Scan the top 1K of base RAM
579 * 3) Scan the 64K of bios
581 if (!smp_scan_config(0x0,0x400) &&
582 !smp_scan_config(639*0x400,0x400) &&
583 !smp_scan_config(0xF0000,0x10000)) {
585 * If it is an SMP machine we should know now, unless the
586 * configuration is in an EISA/MCA bus machine with an
587 * extended bios data area.
589 * there is a real-mode segmented pointer pointing to the
590 * 4K EBDA area at 0x40E, calculate and scan it here.
592 * NOTE! There are Linux loaders that will corrupt the EBDA
593 * area, and as such this kind of SMP config may be less
594 * trustworthy, simply because the SMP table may have been
595 * stomped on during early boot. These loaders are buggy and
598 unsigned int address
;
600 address
= *(unsigned short *)phys_to_virt(0x40E);
602 smp_scan_config(address
, 0x1000);
603 if (smp_found_config
)
604 printk(KERN_WARNING
"WARNING: MP table in the EBDA can be UNSAFE, contact linux-smp@vger.rutgers.edu if you experience SMP problems!\n");
611 * The Visual Workstation is Intel MP compliant in the hardware
612 * sense, but it doesnt have a BIOS(-configuration table).
613 * No problem for Linux.
615 void __init
init_visws_smp(void)
617 smp_found_config
= 1;
619 cpu_present_map
|= 2; /* or in id 1 */
620 apic_version
[1] |= 0x10; /* integrated APIC */
621 apic_version
[0] |= 0x10;
623 mp_lapic_addr
= APIC_DEFAULT_PHYS_BASE
;
629 * - Intel MP Configuration Table
630 * - or SGI Visual Workstation configuration
632 void __init
init_smp_config (void)
644 * Trampoline 80x86 program as an array.
647 extern unsigned char trampoline_data
[];
648 extern unsigned char trampoline_end
[];
649 static unsigned char *trampoline_base
;
652 * Currently trivial. Write the real->protected mode
653 * bootstrap into the page concerned. The caller
654 * has made sure it's suitably aligned.
657 static unsigned long __init
setup_trampoline(void)
659 memcpy(trampoline_base
, trampoline_data
, trampoline_end
- trampoline_data
);
660 return virt_to_phys(trampoline_base
);
664 * We are called very early to get the low memory for the
665 * SMP bootup trampoline page.
667 unsigned long __init
smp_alloc_memory(unsigned long mem_base
)
669 if (virt_to_phys((void *)mem_base
) >= 0x9F000)
670 panic("smp_alloc_memory: Insufficient low memory for kernel trampoline 0x%lx.", mem_base
);
671 trampoline_base
= (void *)mem_base
;
672 return mem_base
+ PAGE_SIZE
;
676 * The bootstrap kernel entry code has set these up. Save them for
680 void __init
smp_store_cpu_info(int id
)
682 struct cpuinfo_x86
*c
=&cpu_data
[id
];
687 c
->pgtable_cache_sz
= 0;
690 * Mask B, Pentium, but not Pentium MMX
692 if (c
->x86_vendor
== X86_VENDOR_INTEL
&&
694 c
->x86_mask
>= 1 && c
->x86_mask
<= 4 &&
696 smp_b_stepping
=1; /* Remember we have B step Pentia with bugs */
700 * Architecture specific routine called by the kernel just before init is
701 * fired off. This allows the BP to have everything in order [we hope].
702 * At the end of this all the APs will hit the system scheduling and off
703 * we go. Each AP will load the system gdt's and jump through the kernel
704 * init into idle(). At this point the scheduler will one day take over
705 * and give them jobs to do. smp_callin is a standard routine
706 * we use to track CPUs as they power up.
709 static atomic_t smp_commenced
= ATOMIC_INIT(0);
711 void __init
smp_commence(void)
714 * Lets the callins below out of their loop.
716 SMP_PRINTK(("Setting commenced=1, go go go\n"));
719 atomic_set(&smp_commenced
,1);
722 void __init
enable_local_APIC(void)
726 value
= apic_read(APIC_SPIV
);
727 value
|= (1<<8); /* Enable APIC (bit==1) */
728 value
&= ~(1<<9); /* Enable focus processor (bit==0) */
729 value
|= 0xff; /* Set spurious IRQ vector to 0xff */
730 apic_write(APIC_SPIV
,value
);
733 * Set Task Priority to 'accept all'
735 value
= apic_read(APIC_TASKPRI
);
736 value
&= ~APIC_TPRI_MASK
;
737 apic_write(APIC_TASKPRI
,value
);
740 * Clear the logical destination ID, just to be safe.
741 * also, put the APIC into flat delivery mode.
743 value
= apic_read(APIC_LDR
);
744 value
&= ~APIC_LDR_MASK
;
745 apic_write(APIC_LDR
,value
);
747 value
= apic_read(APIC_DFR
);
748 value
|= SET_APIC_DFR(0xf);
749 apic_write(APIC_DFR
, value
);
751 udelay(100); /* B safe */
754 unsigned long __init
init_smp_mappings(unsigned long memory_start
)
756 unsigned long apic_phys
;
758 memory_start
= PAGE_ALIGN(memory_start
);
759 if (smp_found_config
) {
760 apic_phys
= mp_lapic_addr
;
763 * set up a fake all zeroes page to simulate the
764 * local APIC and another one for the IO-APIC. We
765 * could use the real zero-page, but it's safer
766 * this way if some buggy code writes to this page ...
768 apic_phys
= __pa(memory_start
);
769 memset((void *)memory_start
, 0, PAGE_SIZE
);
770 memory_start
+= PAGE_SIZE
;
772 set_fixmap(FIX_APIC_BASE
,apic_phys
);
773 printk("mapped APIC to %08lx (%08lx)\n", APIC_BASE
, apic_phys
);
775 #ifdef CONFIG_X86_IO_APIC
777 unsigned long ioapic_phys
;
779 if (smp_found_config
) {
780 ioapic_phys
= mp_ioapic_addr
;
782 ioapic_phys
= __pa(memory_start
);
783 memset((void *)memory_start
, 0, PAGE_SIZE
);
784 memory_start
+= PAGE_SIZE
;
786 set_fixmap(FIX_IO_APIC_BASE
,ioapic_phys
);
787 printk("mapped IOAPIC to %08lx (%08lx)\n",
788 fix_to_virt(FIX_IO_APIC_BASE
), ioapic_phys
);
795 extern void calibrate_delay(void);
797 void __init
smp_callin(void)
800 unsigned long timeout
;
803 * (This works even if the APIC is not enabled.)
805 cpuid
= GET_APIC_ID(apic_read(APIC_ID
));
807 SMP_PRINTK(("CPU#%d waiting for CALLOUT\n", cpuid
));
810 * STARTUP IPIs are fragile beasts as they might sometimes
811 * trigger some glue motherboard logic. Complete APIC bus
812 * silence for 1 second, this overestimates the time the
813 * boot CPU is spending to send the up to 2 STARTUP IPIs
814 * by a factor of two. This should be enough.
818 * Waiting 2s total for startup (udelay is not yet working)
820 timeout
= jiffies
+ 2*HZ
;
821 while (time_before(jiffies
,timeout
))
824 * Has the boot CPU finished it's STARTUP sequence?
826 if (test_bit(cpuid
, (unsigned long *)&cpu_callout_map
[0]))
830 while (!time_before(jiffies
,timeout
)) {
831 printk("BUG: CPU%d started up but did not get a callout!\n",
837 * the boot CPU has finished the init stage and is spinning
838 * on callin_map until we finish. We are free to set up this
839 * CPU, first the APIC. (this is probably redundant on most
843 SMP_PRINTK(("CALLIN, before enable_local_APIC().\n"));
847 * Set up our APIC timer.
854 /* Must be done before calibration delay is computed */
855 mtrr_init_secondary_cpu ();
861 SMP_PRINTK(("Stack at about %p\n",&cpuid
));
864 * Save our processor parameters
866 smp_store_cpu_info(cpuid
);
869 * Allow the master to continue.
871 set_bit(cpuid
, (unsigned long *)&cpu_callin_map
[0]);
876 extern int cpu_idle(void * unused
);
879 * Activate a secondary processor.
881 int __init
start_secondary(void *unused
)
884 * Dont put anything before smp_callin(), SMP
885 * booting is too fragile that we want to limit the
886 * things done here to the most necessary things.
890 while (!atomic_read(&smp_commenced
))
892 return cpu_idle(NULL
);
896 * Everything has been set up for the secondary
897 * CPUs - they just need to reload everything
898 * from the task structure
899 * This function must not return.
901 void __init
initialize_secondary(void)
904 * We don't actually need to load the full TSS,
905 * basically just the stack pointer and the eip.
912 :"r" (current
->thread
.esp
),"r" (current
->thread
.eip
));
920 static void __init
do_boot_cpu(int i
)
924 struct task_struct
*idle
;
925 unsigned long send_status
, accept_status
;
926 int timeout
, num_starts
, j
;
927 unsigned long start_eip
;
930 * We need an idle process for each processor.
932 kernel_thread(start_secondary
, NULL
, CLONE_PID
);
936 * We remove it from the pidhash and the runqueue
937 * once we got the process:
939 idle
= init_task
.prev_task
;
941 init_tasks
[cpucount
] = idle
;
943 panic("No idle process for CPU %d", i
);
946 __cpu_logical_map
[cpucount
] = i
;
947 cpu_number_map
[i
] = cpucount
;
948 idle
->has_cpu
= 1; /* we schedule the first task manually */
949 idle
->thread
.eip
= (unsigned long) start_secondary
;
951 del_from_runqueue(idle
);
952 unhash_process(idle
);
954 /* start_eip had better be page-aligned! */
955 start_eip
= setup_trampoline();
957 printk("Booting processor %d eip %lx\n", i
, start_eip
); /* So we see what's up */
958 stack_start
.esp
= (void *) (1024 + PAGE_SIZE
+ (char *)idle
);
961 * This grunge runs the startup process for
962 * the targeted processor.
965 SMP_PRINTK(("Setting warm reset code and vector.\n"));
967 CMOS_WRITE(0xa, 0xf);
969 SMP_PRINTK(("1.\n"));
970 *((volatile unsigned short *) phys_to_virt(0x469)) = start_eip
>> 4;
971 SMP_PRINTK(("2.\n"));
972 *((volatile unsigned short *) phys_to_virt(0x467)) = start_eip
& 0xf;
973 SMP_PRINTK(("3.\n"));
975 maincfg
=swapper_pg_dir
[0];
976 ((unsigned long *)swapper_pg_dir
)[0]=0x102007;
979 * Be paranoid about clearing APIC errors.
982 if ( apic_version
[i
] & 0xF0 )
984 apic_write(APIC_ESR
, 0);
985 accept_status
= (apic_read(APIC_ESR
) & 0xEF);
989 * Status is now clean
996 * Starting actual IPI sequence...
999 SMP_PRINTK(("Asserting INIT.\n"));
1005 cfg
=apic_read(APIC_ICR2
);
1007 apic_write(APIC_ICR2
, cfg
|SET_APIC_DEST_FIELD(i
)); /* Target chip */
1008 cfg
=apic_read(APIC_ICR
);
1009 cfg
&=~0xCDFFF; /* Clear bits */
1010 cfg
|= (APIC_DEST_LEVELTRIG
| APIC_DEST_ASSERT
| APIC_DEST_DM_INIT
);
1011 apic_write(APIC_ICR
, cfg
); /* Send IPI */
1014 SMP_PRINTK(("Deasserting INIT.\n"));
1016 cfg
=apic_read(APIC_ICR2
);
1018 apic_write(APIC_ICR2
, cfg
|SET_APIC_DEST_FIELD(i
)); /* Target chip */
1019 cfg
=apic_read(APIC_ICR
);
1020 cfg
&=~0xCDFFF; /* Clear bits */
1021 cfg
|= (APIC_DEST_LEVELTRIG
| APIC_DEST_DM_INIT
);
1022 apic_write(APIC_ICR
, cfg
); /* Send IPI */
1025 * Should we send STARTUP IPIs ?
1027 * Determine this based on the APIC version.
1028 * If we don't have an integrated APIC, don't
1029 * send the STARTUP IPIs.
1032 if ( apic_version
[i
] & 0xF0 )
1038 * Run STARTUP IPI loop.
1041 for (j
= 1; !(send_status
|| accept_status
)
1042 && (j
<= num_starts
) ; j
++)
1044 SMP_PRINTK(("Sending STARTUP #%d.\n",j
));
1045 apic_write(APIC_ESR
, 0);
1046 SMP_PRINTK(("After apic_write.\n"));
1052 cfg
=apic_read(APIC_ICR2
);
1054 apic_write(APIC_ICR2
, cfg
|SET_APIC_DEST_FIELD(i
)); /* Target chip */
1055 cfg
=apic_read(APIC_ICR
);
1056 cfg
&=~0xCDFFF; /* Clear bits */
1057 cfg
|= (APIC_DEST_DM_STARTUP
| (start_eip
>> 12)); /* Boot on the stack */
1058 SMP_PRINTK(("Before start apic_write.\n"));
1059 apic_write(APIC_ICR
, cfg
); /* Kick the second */
1061 SMP_PRINTK(("Startup point 1.\n"));
1064 SMP_PRINTK(("Waiting for send to finish...\n"));
1068 send_status
= apic_read(APIC_ICR
) & 0x1000;
1069 } while (send_status
&& (timeout
++ < 1000));
1072 * Give the other CPU some time to accept the IPI.
1075 accept_status
= (apic_read(APIC_ESR
) & 0xEF);
1077 SMP_PRINTK(("After Startup.\n"));
1079 if (send_status
) /* APIC never delivered?? */
1080 printk("APIC never delivered???\n");
1081 if (accept_status
) /* Send accept error */
1082 printk("APIC delivery error (%lx).\n", accept_status
);
1084 if ( !(send_status
|| accept_status
) )
1087 * allow APs to start initializing.
1089 SMP_PRINTK(("Before Callout %d.\n", i
));
1090 set_bit(i
, (unsigned long *)&cpu_callout_map
[0]);
1091 SMP_PRINTK(("After Callout %d.\n", i
));
1093 for(timeout
=0;timeout
<50000;timeout
++)
1095 if (cpu_callin_map
[0]&(1<<i
))
1096 break; /* It has booted */
1097 udelay(100); /* Wait 5s total for a response */
1099 if (cpu_callin_map
[0]&(1<<i
))
1101 /* number CPUs logically, starting from 1 (BSP is 0) */
1103 cpu_number_map
[i
] = cpucount
;
1104 __cpu_logical_map
[cpucount
] = i
;
1107 printk("CPU%d: ", i
);
1108 print_cpu_info(&cpu_data
[i
]);
1112 if (*((volatile unsigned char *)phys_to_virt(8192))==0xA5)
1113 printk("Stuck ??\n");
1115 printk("Not responding.\n");
1117 SMP_PRINTK(("CPU has booted.\n"));
1121 __cpu_logical_map
[cpucount
] = -1;
1122 cpu_number_map
[i
] = -1;
1126 swapper_pg_dir
[0]=maincfg
;
1129 /* mark "stuck" area as not stuck */
1130 *((volatile unsigned long *)phys_to_virt(8192)) = 0;
1133 cycles_t cacheflush_time
;
1134 extern unsigned long cpu_hz
;
1136 static void smp_tune_scheduling (void)
1138 unsigned long cachesize
;
1140 * Rough estimation for SMP scheduling, this is the number of
1141 * cycles it takes for a fully memory-limited process to flush
1142 * the SMP-local cache.
1144 * (For a P5 this pretty much means we will choose another idle
1145 * CPU almost always at wakeup time (this is due to the small
1146 * L1 cache), on PIIs it's around 50-100 usecs, depending on
1152 * this basically disables processor-affinity
1153 * scheduling on SMP without a TSC.
1155 cacheflush_time
= 0;
1158 cachesize
= boot_cpu_data
.x86_cache_size
;
1159 if (cachesize
== -1)
1160 cachesize
= 8; /* Pentiums */
1162 cacheflush_time
= cpu_hz
/1024*cachesize
/5000;
1165 printk("per-CPU timeslice cutoff: %ld.%02ld usecs.\n",
1166 (long)cacheflush_time
/(cpu_hz
/1000000),
1167 ((long)cacheflush_time
*100/(cpu_hz
/1000000)) % 100);
1170 unsigned int prof_multiplier
[NR_CPUS
];
1171 unsigned int prof_old_multiplier
[NR_CPUS
];
1172 unsigned int prof_counter
[NR_CPUS
];
1175 * Cycle through the processors sending APIC IPIs to boot each.
1178 void __init
smp_boot_cpus(void)
1183 /* Must be done before other processors booted */
1184 mtrr_init_boot_cpu ();
1187 * Initialize the logical to physical CPU number mapping
1188 * and the per-CPU profiling counter/multiplier
1191 for (i
= 0; i
< NR_CPUS
; i
++) {
1192 cpu_number_map
[i
] = -1;
1193 prof_counter
[i
] = 1;
1194 prof_old_multiplier
[i
] = 1;
1195 prof_multiplier
[i
] = 1;
1199 * Setup boot CPU information
1202 smp_store_cpu_info(boot_cpu_id
); /* Final full version of the data */
1203 smp_tune_scheduling();
1204 printk("CPU%d: ", boot_cpu_id
);
1205 print_cpu_info(&cpu_data
[boot_cpu_id
]);
1208 * not necessary because the MP table should list the boot
1209 * CPU too, but we do it for the sake of robustness anyway.
1210 * (and for the case when a non-SMP board boots an SMP kernel)
1212 cpu_present_map
|= (1 << hard_smp_processor_id());
1214 cpu_number_map
[boot_cpu_id
] = 0;
1219 * If we couldnt find an SMP configuration at boot time,
1220 * get out of here now!
1223 if (!smp_found_config
)
1225 printk(KERN_NOTICE
"SMP motherboard not detected. Using dummy APIC emulation.\n");
1226 #ifndef CONFIG_VISWS
1229 cpu_online_map
= cpu_present_map
;
1234 * If SMP should be disabled, then really disable it!
1239 smp_found_config
= 0;
1240 printk(KERN_INFO
"SMP mode deactivated, forcing use of dummy APIC emulation.\n");
1248 * This is to verify that we're looking at
1249 * a real local APIC. Check these against
1250 * your board if the CPUs aren't getting
1251 * started for no apparent reason.
1254 reg
= apic_read(APIC_VERSION
);
1255 SMP_PRINTK(("Getting VERSION: %x\n", reg
));
1257 apic_write(APIC_VERSION
, 0);
1258 reg
= apic_read(APIC_VERSION
);
1259 SMP_PRINTK(("Getting VERSION: %x\n", reg
));
1262 * The two version reads above should print the same
1263 * NON-ZERO!!! numbers. If the second one is zero,
1264 * there is a problem with the APIC write/read
1267 * The next two are just to see if we have sane values.
1268 * They're only really relevant if we're in Virtual Wire
1269 * compatibility mode, but most boxes are anymore.
1273 reg
= apic_read(APIC_LVT0
);
1274 SMP_PRINTK(("Getting LVT0: %x\n", reg
));
1276 reg
= apic_read(APIC_LVT1
);
1277 SMP_PRINTK(("Getting LVT1: %x\n", reg
));
1281 enable_local_APIC();
1284 * Set up our local APIC timer:
1286 setup_APIC_clock ();
1289 * Now scan the CPU present map and fire up the other CPUs.
1293 * Add all detected CPUs. (later on we can down individual
1294 * CPUs which will change cpu_online_map but not necessarily
1295 * cpu_present_map. We are pretty much ready for hot-swap CPUs.)
1297 cpu_online_map
= cpu_present_map
;
1300 SMP_PRINTK(("CPU map: %lx\n", cpu_present_map
));
1302 for(i
=0;i
<NR_CPUS
;i
++)
1305 * Don't even attempt to start the boot CPU!
1307 if (i
== boot_cpu_id
)
1310 if ((cpu_online_map
& (1 << i
))
1311 && (max_cpus
< 0 || max_cpus
> cpucount
+1))
1317 * Make sure we unmap all failed CPUs
1320 if (cpu_number_map
[i
] == -1 && (cpu_online_map
& (1 << i
))) {
1321 printk("CPU #%d not responding. Removing from cpu_online_map.\n",i
);
1322 cpu_online_map
&= ~(1 << i
);
1327 * Cleanup possible dangling ends...
1330 #ifndef CONFIG_VISWS
1335 * Install writable page 0 entry.
1338 pg0
[0] = _PAGE_RW
| _PAGE_PRESENT
; /* writeable, present, addr 0 */
1342 * Paranoid: Set warm reset code and vector here back
1343 * to default values.
1348 *((volatile long *) phys_to_virt(0x467)) = 0;
1351 * Restore old page 0 entry.
1360 * Allow the user to impress friends.
1363 SMP_PRINTK(("Before bogomips.\n"));
1365 printk(KERN_ERR
"Error: only one processor found.\n");
1366 cpu_online_map
= (1<<hard_smp_processor_id());
1368 unsigned long bogosum
= 0;
1369 for(i
= 0; i
< 32; i
++)
1370 if (cpu_online_map
&(1<<i
))
1371 bogosum
+=cpu_data
[i
].loops_per_sec
;
1372 printk(KERN_INFO
"Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
1374 (bogosum
+2500)/500000,
1375 ((bogosum
+2500)/5000)%100);
1376 SMP_PRINTK(("Before bogocount - setting activated=1.\n"));
1379 smp_num_cpus
= cpucount
+ 1;
1382 printk(KERN_WARNING
"WARNING: SMP operation may be unreliable with B stepping processors.\n");
1383 SMP_PRINTK(("Boot done.\n"));
1386 * now we know the other CPUs have fired off and we know our
1387 * APIC ID, so we can go init the TSS and stuff:
1391 cache_APIC_registers();
1392 #ifndef CONFIG_VISWS
1394 * Here we can be sure that there is an IO-APIC in the system. Let's
1397 if (!skip_ioapic_setup
)
1406 * the following functions deal with sending IPIs between CPUs.
1408 * We use 'broadcast', CPU->CPU IPIs and self-IPIs too.
1413 * Silly serialization to work around CPU bug in P5s.
1414 * We can safely turn it off on a 686.
1416 #ifdef CONFIG_X86_GOOD_APIC
1417 # define FORCE_APIC_SERIALIZATION 0
1419 # define FORCE_APIC_SERIALIZATION 1
1422 static unsigned int cached_APIC_ICR
;
1423 static unsigned int cached_APIC_ICR2
;
1426 * Caches reserved bits, APIC reads are (mildly) expensive
1427 * and force otherwise unnecessary CPU synchronization.
1429 * (We could cache other APIC registers too, but these are the
1430 * main ones used in RL.)
1432 #define slow_ICR (apic_read(APIC_ICR) & ~0xFDFFF)
1433 #define slow_ICR2 (apic_read(APIC_ICR2) & 0x00FFFFFF)
1435 void cache_APIC_registers (void)
1437 cached_APIC_ICR
= slow_ICR
;
1438 cached_APIC_ICR2
= slow_ICR2
;
1442 static inline unsigned int __get_ICR (void)
1444 #if FORCE_APIC_SERIALIZATION
1446 * Wait for the APIC to become ready - this should never occur. It's
1447 * a debugging check really.
1452 while (count
< 1000)
1455 if (!(cfg
&(1<<12))) {
1457 atomic_add(count
, (atomic_t
*)&ipi_count
);
1463 printk("CPU #%d: previous IPI still not cleared after 10mS\n",
1464 smp_processor_id());
1467 return cached_APIC_ICR
;
1471 static inline unsigned int __get_ICR2 (void)
1473 #if FORCE_APIC_SERIALIZATION
1476 return cached_APIC_ICR2
;
1480 static inline int __prepare_ICR (unsigned int shortcut
, int vector
)
1485 cfg
|= APIC_DEST_DM_FIXED
|shortcut
|vector
;
1490 static inline int __prepare_ICR2 (unsigned int dest
)
1495 cfg
|= SET_APIC_DEST_FIELD(dest
);
1500 static inline void __send_IPI_shortcut(unsigned int shortcut
, int vector
)
1504 * Subtle. In the case of the 'never do double writes' workaround we
1505 * have to lock out interrupts to be safe. Otherwise it's just one
1506 * single atomic write to the APIC, no need for cli/sti.
1508 #if FORCE_APIC_SERIALIZATION
1509 unsigned long flags
;
1511 __save_flags(flags
);
1516 * No need to touch the target chip field
1519 cfg
= __prepare_ICR(shortcut
, vector
);
1522 * Send the IPI. The write to APIC_ICR fires this off.
1524 apic_write(APIC_ICR
, cfg
);
1525 #if FORCE_APIC_SERIALIZATION
1526 __restore_flags(flags
);
1530 static inline void send_IPI_allbutself(int vector
)
1532 __send_IPI_shortcut(APIC_DEST_ALLBUT
, vector
);
1535 static inline void send_IPI_all(int vector
)
1537 __send_IPI_shortcut(APIC_DEST_ALLINC
, vector
);
1540 void send_IPI_self(int vector
)
1542 __send_IPI_shortcut(APIC_DEST_SELF
, vector
);
1545 static inline void send_IPI_single(int dest
, int vector
)
1548 #if FORCE_APIC_SERIALIZATION
1549 unsigned long flags
;
1551 __save_flags(flags
);
1556 * prepare target chip field
1559 cfg
= __prepare_ICR2(dest
);
1560 apic_write(APIC_ICR2
, cfg
);
1565 cfg
= __prepare_ICR(0, vector
);
1568 * Send the IPI. The write to APIC_ICR fires this off.
1570 apic_write(APIC_ICR
, cfg
);
1571 #if FORCE_APIC_SERIALIZATION
1572 __restore_flags(flags
);
1577 * This is fraught with deadlocks. Probably the situation is not that
1578 * bad as in the early days of SMP, so we might ease some of the
1582 void smp_flush_tlb(void)
1584 int cpu
= smp_processor_id();
1586 unsigned long flags
;
1589 * it's important that we do not generate any APIC traffic
1590 * until the AP CPUs have booted up!
1592 if (cpu_online_map
) {
1594 * The assignment is safe because it's volatile so the
1595 * compiler cannot reorder it, because the i586 has
1596 * strict memory ordering and because only the kernel
1597 * lock holder may issue a tlb flush. If you break any
1598 * one of those three change this to an atomic bus
1602 smp_invalidate_needed
= cpu_online_map
;
1605 * Processors spinning on some lock with IRQs disabled
1606 * will see this IRQ late. The smp_invalidate_needed
1607 * map will ensure they don't do a spurious flush tlb
1611 __save_flags(flags
);
1614 send_IPI_allbutself(INVALIDATE_TLB_VECTOR
);
1617 * Spin waiting for completion
1621 while (smp_invalidate_needed
) {
1623 * Take care of "crossing" invalidates
1625 if (test_bit(cpu
, &smp_invalidate_needed
))
1626 clear_bit(cpu
, &smp_invalidate_needed
);
1629 printk("stuck on TLB IPI wait (CPU#%d)\n",cpu
);
1633 __restore_flags(flags
);
1637 * Flush the local TLB
1645 * this function sends a 'reschedule' IPI to another CPU.
1646 * it goes straight through and wastes no time serializing
1647 * anything. Worst case is that we lose a reschedule ...
1650 void smp_send_reschedule(int cpu
)
1652 send_IPI_single(cpu
, RESCHEDULE_VECTOR
);
1656 * this function sends a 'stop' IPI to all other CPUs in the system.
1657 * it goes straight through.
1660 void smp_send_stop(void)
1662 send_IPI_allbutself(STOP_CPU_VECTOR
);
1665 /* Structure and data for smp_call_function(). This is designed to minimise
1666 * static memory requirements. It also looks cleaner.
1668 struct smp_call_function_struct
{
1669 void (*func
) (void *info
);
1671 atomic_t unstarted_count
;
1672 atomic_t unfinished_count
;
1675 static volatile struct smp_call_function_struct
*smp_call_function_data
= NULL
;
1678 * this function sends a 'generic call function' IPI to all other CPUs
1682 int smp_call_function (void (*func
) (void *info
), void *info
, int retry
,
1684 /* [SUMMARY] Run a function on all other CPUs.
1685 <func> The function to run. This must be fast and non-blocking.
1686 <info> An arbitrary pointer to pass to the function.
1687 <retry> If true, keep retrying until ready.
1688 <wait> If true, wait until function has completed on other CPUs.
1689 [RETURNS] 0 on success, else a negative status code. Does not return until
1690 remote CPUs are nearly ready to execute <<func>> or are or have executed.
1693 unsigned long timeout
;
1694 struct smp_call_function_struct data
;
1695 static spinlock_t lock
= SPIN_LOCK_UNLOCKED
;
1699 if (smp_call_function_data
) {
1700 schedule (); /* Give a mate a go */
1704 if (smp_call_function_data
) {
1705 spin_unlock (&lock
); /* Bad luck */
1708 /* Mine, all mine! */
1713 if (smp_call_function_data
) return -EBUSY
;
1715 if (smp_call_function_data
) {
1716 spin_unlock (&lock
);
1720 smp_call_function_data
= &data
;
1721 spin_unlock (&lock
);
1724 atomic_set (&data
.unstarted_count
, smp_num_cpus
- 1);
1726 if (wait
) atomic_set (&data
.unfinished_count
, smp_num_cpus
- 1);
1727 /* Send a message to all other CPUs and wait for them to respond */
1728 send_IPI_allbutself (CALL_FUNCTION_VECTOR
);
1729 /* Wait for response */
1730 timeout
= jiffies
+ JIFFIE_TIMEOUT
;
1731 while ( (atomic_read (&data
.unstarted_count
) > 0) &&
1732 time_before (jiffies
, timeout
) )
1734 if (atomic_read (&data
.unstarted_count
) > 0) {
1735 smp_call_function_data
= NULL
;
1739 while (atomic_read (&data
.unfinished_count
) > 0)
1741 smp_call_function_data
= NULL
;
1745 static unsigned int calibration_result
;
1747 void setup_APIC_timer(unsigned int clocks
);
1750 * Local timer interrupt handler. It does both profiling and
1751 * process statistics/rescheduling.
1753 * We do profiling in every local tick, statistics/rescheduling
1754 * happen only every 'profiling multiplier' ticks. The default
1755 * multiplier is 1 and it can be changed by writing the new multiplier
1756 * value into /proc/profile.
1759 void smp_local_timer_interrupt(struct pt_regs
* regs
)
1761 int user
= (user_mode(regs
) != 0);
1762 int cpu
= smp_processor_id();
1765 * The profiling function is SMP safe. (nothing can mess
1766 * around with "current", and the profiling counters are
1767 * updated with atomic operations). This is especially
1768 * useful with a profiling multiplier != 1
1771 x86_do_profile(regs
->eip
);
1773 if (!--prof_counter
[cpu
]) {
1774 int system
= 1 - user
;
1775 struct task_struct
* p
= current
;
1778 * The multiplier may have changed since the last time we got
1779 * to this point as a result of the user writing to
1780 * /proc/profile. In this case we need to adjust the APIC
1781 * timer accordingly.
1783 * Interrupts are already masked off at this point.
1785 prof_counter
[cpu
] = prof_multiplier
[cpu
];
1786 if (prof_counter
[cpu
] != prof_old_multiplier
[cpu
]) {
1787 setup_APIC_timer(calibration_result
/prof_counter
[cpu
]);
1788 prof_old_multiplier
[cpu
] = prof_counter
[cpu
];
1792 * After doing the above, we need to make like
1793 * a normal interrupt - otherwise timer interrupts
1794 * ignore the global interrupt lock, which is the
1795 * WrongThing (tm) to do.
1799 update_one_process(p
, 1, user
, system
, cpu
);
1802 if (p
->counter
<= 0) {
1804 p
->need_resched
= 1;
1806 if (p
->priority
< DEF_PRIORITY
) {
1807 kstat
.cpu_nice
+= user
;
1808 kstat
.per_cpu_nice
[cpu
] += user
;
1810 kstat
.cpu_user
+= user
;
1811 kstat
.per_cpu_user
[cpu
] += user
;
1813 kstat
.cpu_system
+= system
;
1814 kstat
.per_cpu_system
[cpu
] += system
;
1821 * We take the 'long' return path, and there every subsystem
1822 * grabs the apropriate locks (kernel lock/ irq lock).
1824 * we might want to decouple profiling from the 'long path',
1825 * and do the profiling totally in assembly.
1827 * Currently this isn't too much of an issue (performance wise),
1828 * we can take more than 100K local irqs per second on a 100 MHz P5.
1833 * Local APIC timer interrupt. This is the most natural way for doing
1834 * local interrupts, but local timer interrupts can be emulated by
1835 * broadcast interrupts too. [in case the hw doesnt support APIC timers]
1837 * [ if a single-CPU system runs an SMP kernel then we call the local
1838 * interrupt as well. Thus we cannot inline the local irq ... ]
1840 void smp_apic_timer_interrupt(struct pt_regs
* regs
)
1843 * NOTE! We'd better ACK the irq immediately,
1844 * because timer handling can be slow, and we
1845 * want to be able to accept NMI tlb invalidates
1849 smp_local_timer_interrupt(regs
);
1853 * Reschedule call back. Nothing to do,
1854 * all the work is done automatically when
1855 * we return from the interrupt.
1857 asmlinkage
void smp_reschedule_interrupt(void)
1863 * Invalidate call-back
1865 asmlinkage
void smp_invalidate_interrupt(void)
1867 if (test_and_clear_bit(smp_processor_id(), &smp_invalidate_needed
))
1874 static void stop_this_cpu (void)
1879 clear_bit(smp_processor_id(), &cpu_online_map
);
1881 if (cpu_data
[smp_processor_id()].hlt_works_ok
)
1882 for(;;) __asm__("hlt");
1887 * CPU halt call-back
1889 asmlinkage
void smp_stop_cpu_interrupt(void)
1894 asmlinkage
void smp_call_function_interrupt(void)
1896 void (*func
) (void *info
) = smp_call_function_data
->func
;
1897 void *info
= smp_call_function_data
->info
;
1898 int wait
= smp_call_function_data
->wait
;
1901 /* Notify initiating CPU that I've grabbed the data and am about to
1902 execute the function */
1903 atomic_dec (&smp_call_function_data
->unstarted_count
);
1904 /* At this point the structure may be out of scope unless wait==1 */
1906 if (wait
) atomic_dec (&smp_call_function_data
->unfinished_count
);
1910 * This interrupt should _never_ happen with our APIC/SMP architecture
1912 asmlinkage
void smp_spurious_interrupt(void)
1915 /* see sw-dev-man vol 3, chapter 7.4.13.5 */
1916 printk("spurious APIC interrupt on CPU#%d, should never happen.\n",
1917 smp_processor_id());
1921 * This part sets up the APIC 32 bit clock in LVTT1, with HZ interrupts
1922 * per second. We assume that the caller has already set up the local
1925 * The APIC timer is not exactly sync with the external timer chip, it
1926 * closely follows bus clocks.
1930 * The timer chip is already set up at HZ interrupts per second here,
1931 * but we do not accept timer interrupts yet. We only allow the BP
1934 static unsigned int __init
get_8254_timer_count(void)
1939 count
= inb_p(0x40);
1940 count
|= inb_p(0x40) << 8;
1946 * This function sets up the local APIC timer, with a timeout of
1947 * 'clocks' APIC bus clock. During calibration we actually call
1948 * this function twice, once with a bogus timeout value, second
1949 * time for real. The other (noncalibrating) CPUs call this
1950 * function only once, with the real value.
1952 * We are strictly in irqs off mode here, as we do not want to
1953 * get an APIC interrupt go off accidentally.
1955 * We do reads before writes even if unnecessary, to get around the
1956 * APIC double write bug.
1959 #define APIC_DIVISOR 16
1961 void setup_APIC_timer(unsigned int clocks
)
1963 unsigned long lvtt1_value
;
1964 unsigned int tmp_value
;
1967 * Unfortunately the local APIC timer cannot be set up into NMI
1968 * mode. With the IO APIC we can re-route the external timer
1969 * interrupt and broadcast it as an NMI to all CPUs, so no pain.
1971 tmp_value
= apic_read(APIC_LVTT
);
1972 lvtt1_value
= APIC_LVT_TIMER_PERIODIC
| LOCAL_TIMER_VECTOR
;
1973 apic_write(APIC_LVTT
, lvtt1_value
);
1976 * Divide PICLK by 16
1978 tmp_value
= apic_read(APIC_TDCR
);
1979 apic_write(APIC_TDCR
, (tmp_value
& ~APIC_TDR_DIV_1
)
1982 tmp_value
= apic_read(APIC_TMICT
);
1983 apic_write(APIC_TMICT
, clocks
/APIC_DIVISOR
);
1986 void __init
wait_8254_wraparound(void)
1988 unsigned int curr_count
, prev_count
=~0;
1991 curr_count
= get_8254_timer_count();
1994 prev_count
= curr_count
;
1995 curr_count
= get_8254_timer_count();
1996 delta
= curr_count
-prev_count
;
1999 * This limit for delta seems arbitrary, but it isn't, it's
2000 * slightly above the level of error a buggy Mercury/Neptune
2001 * chipset timer can cause.
2004 } while (delta
<300);
2008 * In this function we calibrate APIC bus clocks to the external
2009 * timer. Unfortunately we cannot use jiffies and the timer irq
2010 * to calibrate, since some later bootup code depends on getting
2011 * the first irq? Ugh.
2013 * We want to do the calibration only once since we
2014 * want to have local timer irqs syncron. CPUs connected
2015 * by the same APIC bus have the very same bus frequency.
2016 * And we want to have irqs off anyways, no accidental
2017 * APIC irq that way.
2020 int __init
calibrate_APIC_clock(void)
2022 unsigned long long t1
,t2
;
2024 long calibration_result
;
2027 printk("calibrating APIC timer ... ");
2030 * Put whatever arbitrary (but long enough) timeout
2031 * value into the APIC clock, we just want to get the
2032 * counter running for calibration.
2034 setup_APIC_timer(1000000000);
2037 * The timer chip counts down to zero. Let's wait
2038 * for a wraparound to start exact measurement:
2039 * (the current tick might have been already half done)
2042 wait_8254_wraparound ();
2045 * We wrapped around just now. Let's start:
2048 tt1
=apic_read(APIC_TMCCT
);
2050 #define LOOPS (HZ/10)
2052 * Let's wait LOOPS wraprounds:
2054 for (i
=0; i
<LOOPS
; i
++)
2055 wait_8254_wraparound ();
2057 tt2
=apic_read(APIC_TMCCT
);
2061 * The APIC bus clock counter is 32 bits only, it
2062 * might have overflown, but note that we use signed
2063 * longs, thus no extra care needed.
2065 * underflown to be exact, as the timer counts down ;)
2068 calibration_result
= (tt1
-tt2
)*APIC_DIVISOR
/LOOPS
;
2070 SMP_PRINTK(("\n..... %ld CPU clocks in 1 timer chip tick.",
2071 (unsigned long)(t2
-t1
)/LOOPS
));
2073 SMP_PRINTK(("\n..... %ld APIC bus clocks in 1 timer chip tick.",
2074 calibration_result
));
2077 printk("\n..... CPU clock speed is %ld.%04ld MHz.\n",
2078 ((long)(t2
-t1
)/LOOPS
)/(1000000/HZ
),
2079 ((long)(t2
-t1
)/LOOPS
)%(1000000/HZ
) );
2081 printk("..... system bus clock speed is %ld.%04ld MHz.\n",
2082 calibration_result
/(1000000/HZ
),
2083 calibration_result
%(1000000/HZ
) );
2086 return calibration_result
;
2089 void __init
setup_APIC_clock(void)
2091 unsigned long flags
;
2093 static volatile int calibration_lock
;
2095 __save_flags(flags
);
2098 SMP_PRINTK(("setup_APIC_clock() called.\n"));
2101 * [ setup_APIC_clock() is called from all CPUs, but we want
2102 * to do this part of the setup only once ... and it fits
2105 if (!test_and_set_bit(0,&calibration_lock
)) {
2107 calibration_result
=calibrate_APIC_clock();
2109 * Signal completion to the other CPU[s]:
2111 calibration_lock
= 3;
2115 * Other CPU is calibrating, wait for finish:
2117 SMP_PRINTK(("waiting for other CPU calibrating APIC ... "));
2118 while (calibration_lock
== 1);
2119 SMP_PRINTK(("done, continuing.\n"));
2123 * Now set up the timer for real.
2126 setup_APIC_timer (calibration_result
);
2129 * We ACK the APIC, just in case there is something pending.
2134 __restore_flags(flags
);
2138 * the frequency of the profiling timer can be changed
2139 * by writing a multiplier value into /proc/profile.
2141 int setup_profiling_timer(unsigned int multiplier
)
2146 * Sanity check. [at least 500 APIC cycles should be
2147 * between APIC interrupts as a rule of thumb, to avoid
2150 if ( (!multiplier
) || (calibration_result
/multiplier
< 500))
2154 * Set the new multiplier for each CPU. CPUs don't start using the
2155 * new values until the next timer interrupt in which they do process
2156 * accounting. At that time they also adjust their APIC timers
2159 for (i
= 0; i
< NR_CPUS
; ++i
)
2160 prof_multiplier
[i
] = multiplier
;