2 * Intel MP v1.1/v1.4 specification support routines for multi-pentium
5 * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
8 * Supported by Caldera http://www.caldera.com.
9 * Much of the core SMP work is based on previous work by Thomas Radke, to
10 * whom a great many thanks are extended.
12 * Thanks to Intel for making available several different Pentium,
13 * Pentium Pro and Pentium-II/Xeon MP machines.
15 * This code is released under the GNU public license version 2 or
19 * Felix Koop : NR_CPUS used properly
20 * Jose Renau : Handle single CPU case.
21 * Alan Cox : By repeated request 8) - Total BogoMIP report.
22 * Greg Wright : Fix for kernel stacks panic.
23 * Erich Boleyn : MP v1.4 and additional changes.
24 * Matthias Sattler : Changes for 2.1 kernel map.
25 * Michel Lespinasse : Changes for 2.1 kernel map.
26 * Michael Chastain : Change trampoline.S to gnu as.
27 * Alan Cox : Dumb bug: 'B' step PPro's are fine
28 * Ingo Molnar : Added APIC timers, based on code
30 * Alan Cox : Added EBDA scanning
31 * Ingo Molnar : various cleanups and rewrites
32 * Tigran Aivazian : fixed "0.00 in /proc/uptime on SMP" bug.
35 #include <linux/config.h>
37 #include <linux/kernel_stat.h>
38 #include <linux/delay.h>
39 #include <linux/mc146818rtc.h>
40 #include <linux/smp_lock.h>
41 #include <linux/init.h>
47 #define JIFFIE_TIMEOUT 100
49 extern void update_one_process( struct task_struct
*p
,
50 unsigned long ticks
, unsigned long user
,
51 unsigned long system
, int cpu
);
53 * Some notes on processor bugs:
55 * Pentium and Pentium Pro (and all CPUs) have bugs. The Linux issues
56 * for SMP are handled as follows.
59 * Occasional delivery of 'spurious interrupt' as trap #16. This
60 * is very rare. The kernel logs the event and recovers
63 * There is a marginal case where REP MOVS on 100MHz SMP
64 * machines with B stepping processors can fail. XXX should provide
65 * an L1cache=Writethrough or L1cache=off option.
67 * B stepping CPUs may hang. There are hardware work arounds
68 * for this. We warn about it in case your board doesnt have the work
69 * arounds. Basically thats so I can tell anyone with a B stepping
70 * CPU and SMP problems "tough".
72 * Specific items [From Pentium Processor Specification Update]
74 * 1AP. Linux doesn't use remote read
75 * 2AP. Linux doesn't trust APIC errors
76 * 3AP. We work around this
77 * 4AP. Linux never generated 3 interrupts of the same priority
78 * to cause a lost local interrupt.
79 * 5AP. Remote read is never used
80 * 9AP. XXX NEED TO CHECK WE HANDLE THIS XXX
81 * 10AP. XXX NEED TO CHECK WE HANDLE THIS XXX
82 * 11AP. Linux reads the APIC between writes to avoid this, as per
83 * the documentation. Make sure you preserve this as it affects
84 * the C stepping chips too.
86 * If this sounds worrying believe me these bugs are ___RARE___ and
87 * there's about nothing of note with C stepping upwards.
92 spinlock_t kernel_flag
= SPIN_LOCK_UNLOCKED
;
95 * function prototypes:
97 static void cache_APIC_registers (void);
98 static void stop_this_cpu (void);
100 static int smp_b_stepping
= 0; /* Set if we find a B stepping CPU */
102 static int max_cpus
= -1; /* Setup configured maximum number of CPUs to activate */
103 int smp_found_config
=0; /* Have we found an SMP box */
105 unsigned long cpu_present_map
= 0; /* Bitmask of physically existing CPUs */
106 unsigned long cpu_online_map
= 0; /* Bitmask of currently online CPUs */
107 int smp_num_cpus
= 1; /* Total count of live CPUs */
108 int smp_threads_ready
=0; /* Set when the idlers are all forked */
109 volatile int cpu_number_map
[NR_CPUS
]; /* which CPU maps to which logical number */
110 volatile int __cpu_logical_map
[NR_CPUS
]; /* which logical number maps to which CPU */
111 static volatile unsigned long cpu_callin_map
[NR_CPUS
] = {0,}; /* We always use 0 the rest is ready for parallel delivery */
112 static volatile unsigned long cpu_callout_map
[NR_CPUS
] = {0,}; /* We always use 0 the rest is ready for parallel delivery */
113 volatile unsigned long smp_invalidate_needed
; /* Used for the invalidate map that's also checked in the spinlock */
114 volatile unsigned long kstack_ptr
; /* Stack vector for booting CPUs */
115 struct cpuinfo_x86 cpu_data
[NR_CPUS
]; /* Per CPU bogomips and other parameters */
116 static unsigned int num_processors
= 1; /* Internal processor count */
117 unsigned long mp_ioapic_addr
= 0xFEC00000; /* Address of the I/O apic (not yet used) */
118 unsigned char boot_cpu_id
= 0; /* Processor that is doing the boot up */
119 static int smp_activated
= 0; /* Tripped once we need to start cross invalidating */
120 int apic_version
[NR_CPUS
]; /* APIC version number */
121 unsigned long apic_retval
; /* Just debugging the assembler.. */
123 volatile unsigned long kernel_counter
=0; /* Number of times the processor holds the lock */
124 volatile unsigned long syscall_count
=0; /* Number of times the processor holds the syscall lock */
126 volatile unsigned long ipi_count
; /* Number of IPIs delivered */
128 const char lk_lockmsg
[] = "lock from interrupt context at %p\n";
130 int mp_bus_id_to_type
[MAX_MP_BUSSES
] = { -1, };
131 extern int mp_irq_entries
;
132 extern struct mpc_config_intsrc mp_irqs
[MAX_IRQ_SOURCES
];
133 extern int mpc_default_type
;
134 int mp_bus_id_to_pci_bus
[MAX_MP_BUSSES
] = { -1, };
135 int mp_current_pci_id
= 0;
136 unsigned long mp_lapic_addr
= 0;
137 int skip_ioapic_setup
= 0; /* 1 if "noapic" boot option passed */
139 /* #define SMP_DEBUG */
142 #define SMP_PRINTK(x) printk x
144 #define SMP_PRINTK(x)
148 * IA s/w dev Vol 3, Section 7.4
150 #define APIC_DEFAULT_PHYS_BASE 0xfee00000
152 #define CLEAR_TSC wrmsr(0x10, 0x00001000, 0x00001000)
155 * Setup routine for controlling SMP activation
157 * Command-line option of "nosmp" or "maxcpus=0" will disable SMP
158 * activation entirely (the MPS table probe still happens, though).
160 * Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
161 * greater than 0, limits the maximum number of CPUs activated in
165 void __init
smp_setup(char *str
, int *ints
)
167 if (ints
&& ints
[0] > 0)
173 void ack_APIC_irq(void)
178 apic_read(APIC_SPIV
);
180 /* Docs say use 0 for future compatibility */
181 apic_write(APIC_EOI
, 0);
185 * Intel MP BIOS table parsing routines:
188 #ifndef CONFIG_X86_VISWS_APIC
190 * Checksum an MP configuration block.
193 static int mpf_checksum(unsigned char *mp
, int len
)
202 * Processor encoding in an MP configuration block
205 static char *mpc_family(int family
,int model
)
208 static char *model_defs
[]=
211 "80486SX","80486DX/2 or 80487",
212 "80486SL","Intel5X2(tm)",
217 return("Pentium(tm) Pro");
219 return("Pentium(tm)");
220 if (family
==0x0F && model
==0x0F)
221 return("Special controller");
222 if (family
==0x04 && model
<9)
223 return model_defs
[model
];
224 sprintf(n
,"Unknown CPU [%d:%d]",family
, model
);
232 static int __init
smp_read_mpc(struct mp_config_table
*mpc
)
235 int count
=sizeof(*mpc
);
237 unsigned char *mpt
=((unsigned char *)mpc
)+count
;
239 if (memcmp(mpc
->mpc_signature
,MPC_SIGNATURE
,4))
241 panic("SMP mptable: bad signature [%c%c%c%c]!\n",
242 mpc
->mpc_signature
[0],
243 mpc
->mpc_signature
[1],
244 mpc
->mpc_signature
[2],
245 mpc
->mpc_signature
[3]);
248 if (mpf_checksum((unsigned char *)mpc
,mpc
->mpc_length
))
250 panic("SMP mptable: checksum error!\n");
253 if (mpc
->mpc_spec
!=0x01 && mpc
->mpc_spec
!=0x04)
255 printk("Bad Config Table version (%d)!!\n",mpc
->mpc_spec
);
258 memcpy(str
,mpc
->mpc_oem
,8);
260 memcpy(ioapic_OEM_ID
,str
,9);
261 printk("OEM ID: %s ",str
);
263 memcpy(str
,mpc
->mpc_productid
,12);
265 memcpy(ioapic_Product_ID
,str
,13);
266 printk("Product ID: %s ",str
);
268 printk("APIC at: 0x%lX\n",mpc
->mpc_lapic
);
270 /* save the local APIC address, it might be non-default */
271 mp_lapic_addr
= mpc
->mpc_lapic
;
274 * Now process the configuration blocks.
277 while(count
<mpc
->mpc_length
)
283 struct mpc_config_processor
*m
=
284 (struct mpc_config_processor
*)mpt
;
285 if (m
->mpc_cpuflag
&CPU_ENABLED
)
287 printk("Processor #%d %s APIC version %d\n",
289 mpc_family((m
->mpc_cpufeature
&
295 if (m
->mpc_featureflag
&(1<<0))
296 printk(" Floating point unit present.\n");
297 if (m
->mpc_featureflag
&(1<<7))
298 printk(" Machine Exception supported.\n");
299 if (m
->mpc_featureflag
&(1<<8))
300 printk(" 64 bit compare & exchange supported.\n");
301 if (m
->mpc_featureflag
&(1<<9))
302 printk(" Internal APIC present.\n");
304 if (m
->mpc_cpuflag
&CPU_BOOTPROCESSOR
)
306 SMP_PRINTK((" Bootup CPU\n"));
307 boot_cpu_id
=m
->mpc_apicid
;
309 else /* Boot CPU already counted */
312 if (m
->mpc_apicid
>NR_CPUS
)
313 printk("Processor #%d unused. (Max %d processors).\n",m
->mpc_apicid
, NR_CPUS
);
316 int ver
= m
->mpc_apicver
;
318 cpu_present_map
|=(1<<m
->mpc_apicid
);
323 printk("BIOS bug, APIC version is 0 for CPU#%d! fixing up to 0x10. (tell your hw vendor)\n", m
->mpc_apicid
);
326 apic_version
[m
->mpc_apicid
] = ver
;
335 struct mpc_config_bus
*m
=
336 (struct mpc_config_bus
*)mpt
;
337 memcpy(str
,m
->mpc_bustype
,6);
339 SMP_PRINTK(("Bus #%d is %s\n",
342 if (strncmp(m
->mpc_bustype
,"ISA",3) == 0)
343 mp_bus_id_to_type
[m
->mpc_busid
] =
346 if (strncmp(m
->mpc_bustype
,"EISA",4) == 0)
347 mp_bus_id_to_type
[m
->mpc_busid
] =
349 if (strncmp(m
->mpc_bustype
,"PCI",3) == 0) {
350 mp_bus_id_to_type
[m
->mpc_busid
] =
352 mp_bus_id_to_pci_bus
[m
->mpc_busid
] =
362 struct mpc_config_ioapic
*m
=
363 (struct mpc_config_ioapic
*)mpt
;
364 if (m
->mpc_flags
&MPC_APIC_USABLE
)
367 printk("I/O APIC #%d Version %d at 0x%lX.\n",
368 m
->mpc_apicid
,m
->mpc_apicver
,
371 * we use the first one only currently
374 mp_ioapic_addr
= m
->mpc_apicaddr
;
382 struct mpc_config_intsrc
*m
=
383 (struct mpc_config_intsrc
*)mpt
;
385 mp_irqs
[mp_irq_entries
] = *m
;
386 if (++mp_irq_entries
== MAX_IRQ_SOURCES
) {
387 printk("Max irq sources exceeded!!\n");
388 printk("Skipping remaining sources.\n");
398 struct mpc_config_intlocal
*m
=
399 (struct mpc_config_intlocal
*)mpt
;
408 printk("Warning: Multiple IO-APICs not yet supported.\n");
409 printk("Warning: switching to non APIC mode.\n");
412 return num_processors
;
416 * Scan the memory blocks for an SMP configuration block.
419 static int __init
smp_scan_config(unsigned long base
, unsigned long length
)
421 unsigned long *bp
=phys_to_virt(base
);
422 struct intel_mp_floating
*mpf
;
424 SMP_PRINTK(("Scan SMP from %p for %ld bytes.\n",
426 if (sizeof(*mpf
)!=16)
427 printk("Error: MPF size\n");
431 if (*bp
==SMP_MAGIC_IDENT
)
433 mpf
=(struct intel_mp_floating
*)bp
;
434 if (mpf
->mpf_length
==1 &&
435 !mpf_checksum((unsigned char *)bp
,16) &&
436 (mpf
->mpf_specification
== 1
437 || mpf
->mpf_specification
== 4) )
439 printk("Intel MultiProcessor Specification v1.%d\n", mpf
->mpf_specification
);
440 if (mpf
->mpf_feature2
&(1<<7))
441 printk(" IMCR and PIC compatibility mode.\n");
443 printk(" Virtual Wire compatibility mode.\n");
446 * Now see if we need to read further.
448 if (mpf
->mpf_feature1
!=0)
452 /* local APIC has default address */
453 mp_lapic_addr
= APIC_DEFAULT_PHYS_BASE
;
455 * We need to know what the local
456 * APIC id of the boot CPU is!
461 * HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
463 * It's not just a crazy hack. ;-)
466 * Standard page mapping
467 * functions don't work yet.
468 * We know that page 0 is not
469 * used. Steal it for now!
473 pg0
[0] = (mp_lapic_addr
| _PAGE_RW
| _PAGE_PRESENT
);
476 boot_cpu_id
= GET_APIC_ID(*((volatile unsigned long *) APIC_ID
));
487 * END OF HACK END OF HACK END OF HACK END OF HACK END OF HACK
491 * 2 CPUs, numbered 0 & 1.
495 printk("I/O APIC at 0xFEC00000.\n");
498 * Save the default type number, we
499 * need it later to set the IO-APIC
502 mpc_default_type
= mpf
->mpf_feature1
;
504 printk("Bus #0 is ");
506 switch(mpf
->mpf_feature1
)
513 printk("EISA with no IRQ8 chaining\n");
526 printk("???\nUnknown standard configuration %d\n",
530 if (mpf
->mpf_feature1
>4)
532 printk("Bus #1 is PCI\n");
535 * Set local APIC version to
536 * the integrated form.
537 * It's initialized to zero
538 * otherwise, representing
539 * a discrete 82489DX.
541 apic_version
[0] = 0x10;
542 apic_version
[1] = 0x10;
545 * Read the physical hardware table.
546 * Anything here will override the
549 if (mpf
->mpf_physptr
)
550 smp_read_mpc((void *)mpf
->mpf_physptr
);
552 __cpu_logical_map
[0] = boot_cpu_id
;
553 global_irq_holder
= boot_cpu_id
;
554 current
->processor
= boot_cpu_id
;
556 printk("Processors: %d\n", num_processors
);
558 * Only use the first configuration found.
570 void __init
init_intel_smp (void)
573 * FIXME: Linux assumes you have 640K of base ram..
574 * this continues the error...
576 * 1) Scan the bottom 1K for a signature
577 * 2) Scan the top 1K of base RAM
578 * 3) Scan the 64K of bios
580 if (!smp_scan_config(0x0,0x400) &&
581 !smp_scan_config(639*0x400,0x400) &&
582 !smp_scan_config(0xF0000,0x10000)) {
584 * If it is an SMP machine we should know now, unless the
585 * configuration is in an EISA/MCA bus machine with an
586 * extended bios data area.
588 * there is a real-mode segmented pointer pointing to the
589 * 4K EBDA area at 0x40E, calculate and scan it here.
591 * NOTE! There are Linux loaders that will corrupt the EBDA
592 * area, and as such this kind of SMP config may be less
593 * trustworthy, simply because the SMP table may have been
594 * stomped on during early boot. These loaders are buggy and
597 unsigned int address
;
599 address
= *(unsigned short *)phys_to_virt(0x40E);
601 smp_scan_config(address
, 0x1000);
602 if (smp_found_config
)
603 printk(KERN_WARNING
"WARNING: MP table in the EBDA can be UNSAFE, contact linux-smp@vger.rutgers.edu if you experience SMP problems!\n");
610 * The Visual Workstation is Intel MP compliant in the hardware
611 * sense, but it doesnt have a BIOS(-configuration table).
612 * No problem for Linux.
614 void __init
init_visws_smp(void)
616 smp_found_config
= 1;
618 cpu_present_map
|= 2; /* or in id 1 */
619 apic_version
[1] |= 0x10; /* integrated APIC */
620 apic_version
[0] |= 0x10;
622 mp_lapic_addr
= APIC_DEFAULT_PHYS_BASE
;
628 * - Intel MP Configuration Table
629 * - or SGI Visual Workstation configuration
631 void __init
init_smp_config (void)
641 * Trampoline 80x86 program as an array.
644 extern unsigned char trampoline_data
[];
645 extern unsigned char trampoline_end
[];
646 static unsigned char *trampoline_base
;
649 * Currently trivial. Write the real->protected mode
650 * bootstrap into the page concerned. The caller
651 * has made sure it's suitably aligned.
654 static unsigned long __init
setup_trampoline(void)
656 memcpy(trampoline_base
, trampoline_data
, trampoline_end
- trampoline_data
);
657 return virt_to_phys(trampoline_base
);
661 * We are called very early to get the low memory for the
662 * SMP bootup trampoline page.
664 unsigned long __init
smp_alloc_memory(unsigned long mem_base
)
666 if (virt_to_phys((void *)mem_base
) >= 0x9F000)
667 panic("smp_alloc_memory: Insufficient low memory for kernel trampoline 0x%lx.", mem_base
);
668 trampoline_base
= (void *)mem_base
;
669 return mem_base
+ PAGE_SIZE
;
673 * The bootstrap kernel entry code has set these up. Save them for
677 void __init
smp_store_cpu_info(int id
)
679 struct cpuinfo_x86
*c
=&cpu_data
[id
];
684 c
->pgtable_cache_sz
= 0;
687 * Mask B, Pentium, but not Pentium MMX
689 if (c
->x86_vendor
== X86_VENDOR_INTEL
&&
691 c
->x86_mask
>= 1 && c
->x86_mask
<= 4 &&
693 smp_b_stepping
=1; /* Remember we have B step Pentia with bugs */
697 * Architecture specific routine called by the kernel just before init is
698 * fired off. This allows the BP to have everything in order [we hope].
699 * At the end of this all the APs will hit the system scheduling and off
700 * we go. Each AP will load the system gdt's and jump through the kernel
701 * init into idle(). At this point the scheduler will one day take over
702 * and give them jobs to do. smp_callin is a standard routine
703 * we use to track CPUs as they power up.
706 static atomic_t smp_commenced
= ATOMIC_INIT(0);
708 void __init
smp_commence(void)
711 * Lets the callins below out of their loop.
713 SMP_PRINTK(("Setting commenced=1, go go go\n"));
716 atomic_set(&smp_commenced
,1);
719 void __init
enable_local_APIC(void)
723 value
= apic_read(APIC_SPIV
);
724 value
|= (1<<8); /* Enable APIC (bit==1) */
725 value
&= ~(1<<9); /* Enable focus processor (bit==0) */
726 value
|= 0xff; /* Set spurious IRQ vector to 0xff */
727 apic_write(APIC_SPIV
,value
);
730 * Set Task Priority to 'accept all'
732 value
= apic_read(APIC_TASKPRI
);
733 value
&= ~APIC_TPRI_MASK
;
734 apic_write(APIC_TASKPRI
,value
);
737 * Clear the logical destination ID, just to be safe.
738 * also, put the APIC into flat delivery mode.
740 value
= apic_read(APIC_LDR
);
741 value
&= ~APIC_LDR_MASK
;
742 apic_write(APIC_LDR
,value
);
744 value
= apic_read(APIC_DFR
);
745 value
|= SET_APIC_DFR(0xf);
746 apic_write(APIC_DFR
, value
);
748 udelay(100); /* B safe */
751 unsigned long __init
init_smp_mappings(unsigned long memory_start
)
753 unsigned long apic_phys
;
755 memory_start
= PAGE_ALIGN(memory_start
);
756 if (smp_found_config
) {
757 apic_phys
= mp_lapic_addr
;
760 * set up a fake all zeroes page to simulate the
761 * local APIC and another one for the IO-APIC. We
762 * could use the real zero-page, but it's safer
763 * this way if some buggy code writes to this page ...
765 apic_phys
= __pa(memory_start
);
766 memset((void *)memory_start
, 0, PAGE_SIZE
);
767 memory_start
+= PAGE_SIZE
;
769 set_fixmap(FIX_APIC_BASE
,apic_phys
);
770 printk("mapped APIC to %08lx (%08lx)\n", APIC_BASE
, apic_phys
);
772 #ifdef CONFIG_X86_IO_APIC
774 unsigned long ioapic_phys
;
776 if (smp_found_config
) {
777 ioapic_phys
= mp_ioapic_addr
;
779 ioapic_phys
= __pa(memory_start
);
780 memset((void *)memory_start
, 0, PAGE_SIZE
);
781 memory_start
+= PAGE_SIZE
;
783 set_fixmap(FIX_IO_APIC_BASE
,ioapic_phys
);
784 printk("mapped IOAPIC to %08lx (%08lx)\n",
785 fix_to_virt(FIX_IO_APIC_BASE
), ioapic_phys
);
792 extern void calibrate_delay(void);
794 void __init
smp_callin(void)
797 unsigned long timeout
;
800 * (This works even if the APIC is not enabled.)
802 cpuid
= GET_APIC_ID(apic_read(APIC_ID
));
804 SMP_PRINTK(("CPU#%d waiting for CALLOUT\n", cpuid
));
807 * STARTUP IPIs are fragile beasts as they might sometimes
808 * trigger some glue motherboard logic. Complete APIC bus
809 * silence for 1 second, this overestimates the time the
810 * boot CPU is spending to send the up to 2 STARTUP IPIs
811 * by a factor of two. This should be enough.
815 * Waiting 2s total for startup (udelay is not yet working)
817 timeout
= jiffies
+ 2*HZ
;
818 while (time_before(jiffies
,timeout
))
821 * Has the boot CPU finished it's STARTUP sequence?
823 if (test_bit(cpuid
, (unsigned long *)&cpu_callout_map
[0]))
827 while (!time_before(jiffies
,timeout
)) {
828 printk("BUG: CPU%d started up but did not get a callout!\n",
834 * the boot CPU has finished the init stage and is spinning
835 * on callin_map until we finish. We are free to set up this
836 * CPU, first the APIC. (this is probably redundant on most
840 SMP_PRINTK(("CALLIN, before enable_local_APIC().\n"));
844 * Set up our APIC timer.
851 /* Must be done before calibration delay is computed */
852 mtrr_init_secondary_cpu ();
858 SMP_PRINTK(("Stack at about %p\n",&cpuid
));
861 * Save our processor parameters
863 smp_store_cpu_info(cpuid
);
866 * Allow the master to continue.
868 set_bit(cpuid
, (unsigned long *)&cpu_callin_map
[0]);
873 extern int cpu_idle(void * unused
);
876 * Activate a secondary processor.
878 int __init
start_secondary(void *unused
)
881 * Dont put anything before smp_callin(), SMP
882 * booting is too fragile that we want to limit the
883 * things done here to the most necessary things.
886 while (!atomic_read(&smp_commenced
))
888 return cpu_idle(NULL
);
892 * Everything has been set up for the secondary
893 * CPUs - they just need to reload everything
894 * from the task structure
895 * This function must not return.
897 void __init
initialize_secondary(void)
899 struct thread_struct
* p
= ¤t
->tss
;
902 * Load up the LDT and the task register.
904 asm volatile("lldt %%ax": :"a" (p
->ldt
));
905 asm volatile("ltr %%ax": :"a" (p
->tr
));
909 * We don't actually need to load the full TSS,
910 * basically just the stack pointer and the eip.
917 :"r" (p
->esp
),"r" (p
->eip
));
925 static void __init
do_boot_cpu(int i
)
929 struct task_struct
*idle
;
930 unsigned long send_status
, accept_status
;
931 int timeout
, num_starts
, j
;
932 unsigned long start_eip
;
935 * We need an idle process for each processor.
937 kernel_thread(start_secondary
, NULL
, CLONE_PID
);
940 idle
= task
[cpucount
];
942 panic("No idle process for CPU %d", i
);
945 __cpu_logical_map
[cpucount
] = i
;
946 cpu_number_map
[i
] = cpucount
;
947 idle
->has_cpu
= 1; /* we schedule the first task manually */
948 idle
->tss
.eip
= (unsigned long) start_secondary
;
950 /* start_eip had better be page-aligned! */
951 start_eip
= setup_trampoline();
953 printk("Booting processor %d eip %lx\n", i
, start_eip
); /* So we see what's up */
954 stack_start
.esp
= (void *) (1024 + PAGE_SIZE
+ (char *)idle
);
957 * This grunge runs the startup process for
958 * the targeted processor.
961 SMP_PRINTK(("Setting warm reset code and vector.\n"));
963 CMOS_WRITE(0xa, 0xf);
965 SMP_PRINTK(("1.\n"));
966 *((volatile unsigned short *) phys_to_virt(0x469)) = start_eip
>> 4;
967 SMP_PRINTK(("2.\n"));
968 *((volatile unsigned short *) phys_to_virt(0x467)) = start_eip
& 0xf;
969 SMP_PRINTK(("3.\n"));
971 maincfg
=swapper_pg_dir
[0];
972 ((unsigned long *)swapper_pg_dir
)[0]=0x102007;
975 * Be paranoid about clearing APIC errors.
978 if ( apic_version
[i
] & 0xF0 )
980 apic_write(APIC_ESR
, 0);
981 accept_status
= (apic_read(APIC_ESR
) & 0xEF);
985 * Status is now clean
992 * Starting actual IPI sequence...
995 SMP_PRINTK(("Asserting INIT.\n"));
1001 cfg
=apic_read(APIC_ICR2
);
1003 apic_write(APIC_ICR2
, cfg
|SET_APIC_DEST_FIELD(i
)); /* Target chip */
1004 cfg
=apic_read(APIC_ICR
);
1005 cfg
&=~0xCDFFF; /* Clear bits */
1006 cfg
|= (APIC_DEST_LEVELTRIG
| APIC_DEST_ASSERT
| APIC_DEST_DM_INIT
);
1007 apic_write(APIC_ICR
, cfg
); /* Send IPI */
1010 SMP_PRINTK(("Deasserting INIT.\n"));
1012 cfg
=apic_read(APIC_ICR2
);
1014 apic_write(APIC_ICR2
, cfg
|SET_APIC_DEST_FIELD(i
)); /* Target chip */
1015 cfg
=apic_read(APIC_ICR
);
1016 cfg
&=~0xCDFFF; /* Clear bits */
1017 cfg
|= (APIC_DEST_LEVELTRIG
| APIC_DEST_DM_INIT
);
1018 apic_write(APIC_ICR
, cfg
); /* Send IPI */
1021 * Should we send STARTUP IPIs ?
1023 * Determine this based on the APIC version.
1024 * If we don't have an integrated APIC, don't
1025 * send the STARTUP IPIs.
1028 if ( apic_version
[i
] & 0xF0 )
1034 * Run STARTUP IPI loop.
1037 for (j
= 1; !(send_status
|| accept_status
)
1038 && (j
<= num_starts
) ; j
++)
1040 SMP_PRINTK(("Sending STARTUP #%d.\n",j
));
1041 apic_write(APIC_ESR
, 0);
1042 SMP_PRINTK(("After apic_write.\n"));
1048 cfg
=apic_read(APIC_ICR2
);
1050 apic_write(APIC_ICR2
, cfg
|SET_APIC_DEST_FIELD(i
)); /* Target chip */
1051 cfg
=apic_read(APIC_ICR
);
1052 cfg
&=~0xCDFFF; /* Clear bits */
1053 cfg
|= (APIC_DEST_DM_STARTUP
| (start_eip
>> 12)); /* Boot on the stack */
1054 SMP_PRINTK(("Before start apic_write.\n"));
1055 apic_write(APIC_ICR
, cfg
); /* Kick the second */
1057 SMP_PRINTK(("Startup point 1.\n"));
1060 SMP_PRINTK(("Waiting for send to finish...\n"));
1064 send_status
= apic_read(APIC_ICR
) & 0x1000;
1065 } while (send_status
&& (timeout
++ < 1000));
1068 * Give the other CPU some time to accept the IPI.
1071 accept_status
= (apic_read(APIC_ESR
) & 0xEF);
1073 SMP_PRINTK(("After Startup.\n"));
1075 if (send_status
) /* APIC never delivered?? */
1076 printk("APIC never delivered???\n");
1077 if (accept_status
) /* Send accept error */
1078 printk("APIC delivery error (%lx).\n", accept_status
);
1080 if ( !(send_status
|| accept_status
) )
1083 * allow APs to start initializing.
1085 SMP_PRINTK(("Before Callout %d.\n", i
));
1086 set_bit(i
, (unsigned long *)&cpu_callout_map
[0]);
1087 SMP_PRINTK(("After Callout %d.\n", i
));
1089 for(timeout
=0;timeout
<50000;timeout
++)
1091 if (cpu_callin_map
[0]&(1<<i
))
1092 break; /* It has booted */
1093 udelay(100); /* Wait 5s total for a response */
1095 if (cpu_callin_map
[0]&(1<<i
))
1097 /* number CPUs logically, starting from 1 (BSP is 0) */
1099 cpu_number_map
[i
] = cpucount
;
1100 __cpu_logical_map
[cpucount
] = i
;
1103 printk("CPU%d: ", i
);
1104 print_cpu_info(&cpu_data
[i
]);
1108 if (*((volatile unsigned char *)phys_to_virt(8192))==0xA5)
1109 printk("Stuck ??\n");
1111 printk("Not responding.\n");
1113 SMP_PRINTK(("CPU has booted.\n"));
1117 __cpu_logical_map
[cpucount
] = -1;
1118 cpu_number_map
[i
] = -1;
1122 swapper_pg_dir
[0]=maincfg
;
1125 /* mark "stuck" area as not stuck */
1126 *((volatile unsigned long *)phys_to_virt(8192)) = 0;
1129 cycles_t cacheflush_time
;
1130 extern unsigned long cpu_hz
;
1132 static void smp_tune_scheduling (void)
1134 unsigned long cachesize
;
1136 * Rough estimation for SMP scheduling, this is the number of
1137 * cycles it takes for a fully memory-limited process to flush
1138 * the SMP-local cache.
1140 * (For a P5 this pretty much means we will choose another idle
1141 * CPU almost always at wakeup time (this is due to the small
1142 * L1 cache), on PIIs it's around 50-100 usecs, depending on
1148 * this basically disables processor-affinity
1149 * scheduling on SMP without a TSC.
1151 cacheflush_time
= 0;
1154 cachesize
= boot_cpu_data
.x86_cache_size
;
1155 if (cachesize
== -1)
1156 cachesize
= 8; /* Pentiums */
1158 cacheflush_time
= cpu_hz
/1024*cachesize
/5000;
1161 printk("per-CPU timeslice cutoff: %ld.%02ld usecs.\n",
1162 (long)cacheflush_time
/(cpu_hz
/1000000),
1163 ((long)cacheflush_time
*100/(cpu_hz
/1000000)) % 100);
1166 unsigned int prof_multiplier
[NR_CPUS
];
1167 unsigned int prof_old_multiplier
[NR_CPUS
];
1168 unsigned int prof_counter
[NR_CPUS
];
1171 * Cycle through the processors sending APIC IPIs to boot each.
1174 void __init
smp_boot_cpus(void)
1179 /* Must be done before other processors booted */
1180 mtrr_init_boot_cpu ();
1184 * Initialize the logical to physical CPU number mapping
1185 * and the per-CPU profiling counter/multiplier
1188 for (i
= 0; i
< NR_CPUS
; i
++) {
1189 cpu_number_map
[i
] = -1;
1190 prof_counter
[i
] = 1;
1191 prof_old_multiplier
[i
] = 1;
1192 prof_multiplier
[i
] = 1;
1196 * Setup boot CPU information
1199 smp_store_cpu_info(boot_cpu_id
); /* Final full version of the data */
1200 smp_tune_scheduling();
1201 printk("CPU%d: ", boot_cpu_id
);
1202 print_cpu_info(&cpu_data
[boot_cpu_id
]);
1205 * not necessary because the MP table should list the boot
1206 * CPU too, but we do it for the sake of robustness anyway.
1207 * (and for the case when a non-SMP board boots an SMP kernel)
1209 cpu_present_map
|= (1 << hard_smp_processor_id());
1211 cpu_number_map
[boot_cpu_id
] = 0;
1214 * If we couldnt find an SMP configuration at boot time,
1215 * get out of here now!
1218 if (!smp_found_config
)
1220 printk(KERN_NOTICE
"SMP motherboard not detected. Using dummy APIC emulation.\n");
1221 #ifndef CONFIG_VISWS
1224 cpu_online_map
= cpu_present_map
;
1229 * If SMP should be disabled, then really disable it!
1234 smp_found_config
= 0;
1235 printk(KERN_INFO
"SMP mode deactivated, forcing use of dummy APIC emulation.\n");
1243 * This is to verify that we're looking at
1244 * a real local APIC. Check these against
1245 * your board if the CPUs aren't getting
1246 * started for no apparent reason.
1249 reg
= apic_read(APIC_VERSION
);
1250 SMP_PRINTK(("Getting VERSION: %x\n", reg
));
1252 apic_write(APIC_VERSION
, 0);
1253 reg
= apic_read(APIC_VERSION
);
1254 SMP_PRINTK(("Getting VERSION: %x\n", reg
));
1257 * The two version reads above should print the same
1258 * NON-ZERO!!! numbers. If the second one is zero,
1259 * there is a problem with the APIC write/read
1262 * The next two are just to see if we have sane values.
1263 * They're only really relevant if we're in Virtual Wire
1264 * compatibility mode, but most boxes are anymore.
1268 reg
= apic_read(APIC_LVT0
);
1269 SMP_PRINTK(("Getting LVT0: %x\n", reg
));
1271 reg
= apic_read(APIC_LVT1
);
1272 SMP_PRINTK(("Getting LVT1: %x\n", reg
));
1276 enable_local_APIC();
1279 * Set up our local APIC timer:
1281 setup_APIC_clock ();
1284 * Now scan the CPU present map and fire up the other CPUs.
1288 * Add all detected CPUs. (later on we can down individual
1289 * CPUs which will change cpu_online_map but not necessarily
1290 * cpu_present_map. We are pretty much ready for hot-swap CPUs.)
1292 cpu_online_map
= cpu_present_map
;
1295 SMP_PRINTK(("CPU map: %lx\n", cpu_present_map
));
1297 for(i
=0;i
<NR_CPUS
;i
++)
1300 * Don't even attempt to start the boot CPU!
1302 if (i
== boot_cpu_id
)
1305 if ((cpu_online_map
& (1 << i
))
1306 && (max_cpus
< 0 || max_cpus
> cpucount
+1))
1312 * Make sure we unmap all failed CPUs
1315 if (cpu_number_map
[i
] == -1 && (cpu_online_map
& (1 << i
))) {
1316 printk("CPU #%d not responding. Removing from cpu_online_map.\n",i
);
1317 cpu_online_map
&= ~(1 << i
);
1322 * Cleanup possible dangling ends...
1325 #ifndef CONFIG_VISWS
1330 * Install writable page 0 entry.
1333 pg0
[0] = _PAGE_RW
| _PAGE_PRESENT
; /* writeable, present, addr 0 */
1337 * Paranoid: Set warm reset code and vector here back
1338 * to default values.
1343 *((volatile long *) phys_to_virt(0x467)) = 0;
1346 * Restore old page 0 entry.
1355 * Allow the user to impress friends.
1358 SMP_PRINTK(("Before bogomips.\n"));
1361 printk(KERN_ERR
"Error: only one processor found.\n");
1362 cpu_online_map
= (1<<hard_smp_processor_id());
1366 unsigned long bogosum
=0;
1369 if (cpu_online_map
&(1<<i
))
1370 bogosum
+=cpu_data
[i
].loops_per_sec
;
1372 printk(KERN_INFO
"Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
1374 (bogosum
+2500)/500000,
1375 ((bogosum
+2500)/5000)%100);
1376 SMP_PRINTK(("Before bogocount - setting activated=1.\n"));
1378 smp_num_cpus
=cpucount
+1;
1381 printk(KERN_WARNING
"WARNING: SMP operation may be unreliable with B stepping processors.\n");
1382 SMP_PRINTK(("Boot done.\n"));
1384 cache_APIC_registers();
1385 #ifndef CONFIG_VISWS
1387 * Here we can be sure that there is an IO-APIC in the system. Let's
1390 if (!skip_ioapic_setup
)
1399 * the following functions deal with sending IPIs between CPUs.
1401 * We use 'broadcast', CPU->CPU IPIs and self-IPIs too.
1406 * Silly serialization to work around CPU bug in P5s.
1407 * We can safely turn it off on a 686.
1409 #ifdef CONFIG_X86_GOOD_APIC
1410 # define FORCE_APIC_SERIALIZATION 0
1412 # define FORCE_APIC_SERIALIZATION 1
1415 static unsigned int cached_APIC_ICR
;
1416 static unsigned int cached_APIC_ICR2
;
1419 * Caches reserved bits, APIC reads are (mildly) expensive
1420 * and force otherwise unnecessary CPU synchronization.
1422 * (We could cache other APIC registers too, but these are the
1423 * main ones used in RL.)
1425 #define slow_ICR (apic_read(APIC_ICR) & ~0xFDFFF)
1426 #define slow_ICR2 (apic_read(APIC_ICR2) & 0x00FFFFFF)
1428 void cache_APIC_registers (void)
1430 cached_APIC_ICR
= slow_ICR
;
1431 cached_APIC_ICR2
= slow_ICR2
;
1435 static inline unsigned int __get_ICR (void)
1437 #if FORCE_APIC_SERIALIZATION
1439 * Wait for the APIC to become ready - this should never occur. It's
1440 * a debugging check really.
1445 while (count
< 1000)
1448 if (!(cfg
&(1<<12))) {
1450 atomic_add(count
, (atomic_t
*)&ipi_count
);
1456 printk("CPU #%d: previous IPI still not cleared after 10mS\n",
1457 smp_processor_id());
1460 return cached_APIC_ICR
;
1464 static inline unsigned int __get_ICR2 (void)
1466 #if FORCE_APIC_SERIALIZATION
1469 return cached_APIC_ICR2
;
1473 static inline int __prepare_ICR (unsigned int shortcut
, int vector
)
1478 cfg
|= APIC_DEST_DM_FIXED
|shortcut
|vector
;
1483 static inline int __prepare_ICR2 (unsigned int dest
)
1488 cfg
|= SET_APIC_DEST_FIELD(dest
);
1493 static inline void __send_IPI_shortcut(unsigned int shortcut
, int vector
)
1497 * Subtle. In the case of the 'never do double writes' workaround we
1498 * have to lock out interrupts to be safe. Otherwise it's just one
1499 * single atomic write to the APIC, no need for cli/sti.
1501 #if FORCE_APIC_SERIALIZATION
1502 unsigned long flags
;
1504 __save_flags(flags
);
1509 * No need to touch the target chip field
1512 cfg
= __prepare_ICR(shortcut
, vector
);
1515 * Send the IPI. The write to APIC_ICR fires this off.
1517 apic_write(APIC_ICR
, cfg
);
1518 #if FORCE_APIC_SERIALIZATION
1519 __restore_flags(flags
);
1523 static inline void send_IPI_allbutself(int vector
)
1525 __send_IPI_shortcut(APIC_DEST_ALLBUT
, vector
);
1528 static inline void send_IPI_all(int vector
)
1530 __send_IPI_shortcut(APIC_DEST_ALLINC
, vector
);
1533 void send_IPI_self(int vector
)
1535 __send_IPI_shortcut(APIC_DEST_SELF
, vector
);
1538 static inline void send_IPI_single(int dest
, int vector
)
1541 #if FORCE_APIC_SERIALIZATION
1542 unsigned long flags
;
1544 __save_flags(flags
);
1549 * prepare target chip field
1552 cfg
= __prepare_ICR2(dest
);
1553 apic_write(APIC_ICR2
, cfg
);
1558 cfg
= __prepare_ICR(0, vector
);
1561 * Send the IPI. The write to APIC_ICR fires this off.
1563 apic_write(APIC_ICR
, cfg
);
1564 #if FORCE_APIC_SERIALIZATION
1565 __restore_flags(flags
);
1570 * This is fraught with deadlocks. Probably the situation is not that
1571 * bad as in the early days of SMP, so we might ease some of the
1575 void smp_flush_tlb(void)
1577 int cpu
= smp_processor_id();
1579 unsigned long flags
;
1582 * it's important that we do not generate any APIC traffic
1583 * until the AP CPUs have booted up!
1585 if (cpu_online_map
) {
1587 * The assignment is safe because it's volatile so the
1588 * compiler cannot reorder it, because the i586 has
1589 * strict memory ordering and because only the kernel
1590 * lock holder may issue a tlb flush. If you break any
1591 * one of those three change this to an atomic bus
1595 smp_invalidate_needed
= cpu_online_map
;
1598 * Processors spinning on some lock with IRQs disabled
1599 * will see this IRQ late. The smp_invalidate_needed
1600 * map will ensure they don't do a spurious flush tlb
1604 __save_flags(flags
);
1607 send_IPI_allbutself(INVALIDATE_TLB_VECTOR
);
1610 * Spin waiting for completion
1614 while (smp_invalidate_needed
) {
1616 * Take care of "crossing" invalidates
1618 if (test_bit(cpu
, &smp_invalidate_needed
))
1619 clear_bit(cpu
, &smp_invalidate_needed
);
1622 printk("stuck on TLB IPI wait (CPU#%d)\n",cpu
);
1626 __restore_flags(flags
);
1630 * Flush the local TLB
1638 * this function sends a 'reschedule' IPI to another CPU.
1639 * it goes straight through and wastes no time serializing
1640 * anything. Worst case is that we lose a reschedule ...
1643 void smp_send_reschedule(int cpu
)
1645 send_IPI_single(cpu
, RESCHEDULE_VECTOR
);
1649 * this function sends a 'stop' IPI to all other CPUs in the system.
1650 * it goes straight through.
1653 void smp_send_stop(void)
1655 send_IPI_allbutself(STOP_CPU_VECTOR
);
1658 /* Structure and data for smp_call_function(). This is designed to minimise
1659 * static memory requirements. It also looks cleaner.
1661 struct smp_call_function_struct
{
1662 void (*func
) (void *info
);
1664 atomic_t unstarted_count
;
1665 atomic_t unfinished_count
;
1668 static volatile struct smp_call_function_struct
*smp_call_function_data
= NULL
;
1671 * this function sends a 'generic call function' IPI to all other CPUs
1675 int smp_call_function (void (*func
) (void *info
), void *info
, int retry
,
1677 /* [SUMMARY] Run a function on all other CPUs.
1678 <func> The function to run. This must be fast and non-blocking.
1679 <info> An arbitrary pointer to pass to the function.
1680 <retry> If true, keep retrying until ready.
1681 <wait> If true, wait until function has completed on other CPUs.
1682 [RETURNS] 0 on success, else a negative status code. Does not return until
1683 remote CPUs are nearly ready to execute <<func>> or are or have executed.
1686 unsigned long timeout
;
1687 struct smp_call_function_struct data
;
1688 static spinlock_t lock
= SPIN_LOCK_UNLOCKED
;
1692 if (smp_call_function_data
) {
1693 schedule (); /* Give a mate a go */
1697 if (smp_call_function_data
) {
1698 spin_unlock (&lock
); /* Bad luck */
1701 /* Mine, all mine! */
1706 if (smp_call_function_data
) return -EBUSY
;
1708 if (smp_call_function_data
) {
1709 spin_unlock (&lock
);
1713 smp_call_function_data
= &data
;
1714 spin_unlock (&lock
);
1717 atomic_set (&data
.unstarted_count
, smp_num_cpus
- 1);
1719 if (wait
) atomic_set (&data
.unfinished_count
, smp_num_cpus
- 1);
1720 /* Send a message to all other CPUs and wait for them to respond */
1721 send_IPI_allbutself (CALL_FUNCTION_VECTOR
);
1722 /* Wait for response */
1723 timeout
= jiffies
+ JIFFIE_TIMEOUT
;
1724 while ( (atomic_read (&data
.unstarted_count
) > 0) &&
1725 time_before (jiffies
, timeout
) )
1727 if (atomic_read (&data
.unstarted_count
) > 0) {
1728 smp_call_function_data
= NULL
;
1732 while (atomic_read (&data
.unfinished_count
) > 0)
1734 smp_call_function_data
= NULL
;
1738 static unsigned int calibration_result
;
1740 void setup_APIC_timer(unsigned int clocks
);
1743 * Local timer interrupt handler. It does both profiling and
1744 * process statistics/rescheduling.
1746 * We do profiling in every local tick, statistics/rescheduling
1747 * happen only every 'profiling multiplier' ticks. The default
1748 * multiplier is 1 and it can be changed by writing the new multiplier
1749 * value into /proc/profile.
1752 void smp_local_timer_interrupt(struct pt_regs
* regs
)
1754 int user
= (user_mode(regs
) != 0);
1755 int cpu
= smp_processor_id();
1758 * The profiling function is SMP safe. (nothing can mess
1759 * around with "current", and the profiling counters are
1760 * updated with atomic operations). This is especially
1761 * useful with a profiling multiplier != 1
1764 x86_do_profile(regs
->eip
);
1766 if (!--prof_counter
[cpu
]) {
1767 int system
= 1 - user
;
1768 struct task_struct
* p
= current
;
1771 * The multiplier may have changed since the last time we got
1772 * to this point as a result of the user writing to
1773 * /proc/profile. In this case we need to adjust the APIC
1774 * timer accordingly.
1776 * Interrupts are already masked off at this point.
1778 prof_counter
[cpu
] = prof_multiplier
[cpu
];
1779 if (prof_counter
[cpu
] != prof_old_multiplier
[cpu
]) {
1780 setup_APIC_timer(calibration_result
/prof_counter
[cpu
]);
1781 prof_old_multiplier
[cpu
] = prof_counter
[cpu
];
1785 * After doing the above, we need to make like
1786 * a normal interrupt - otherwise timer interrupts
1787 * ignore the global interrupt lock, which is the
1788 * WrongThing (tm) to do.
1792 update_one_process(p
, 1, user
, system
, cpu
);
1795 if (p
->counter
<= 0) {
1797 p
->need_resched
= 1;
1799 if (p
->priority
< DEF_PRIORITY
) {
1800 kstat
.cpu_nice
+= user
;
1801 kstat
.per_cpu_nice
[cpu
] += user
;
1803 kstat
.cpu_user
+= user
;
1804 kstat
.per_cpu_user
[cpu
] += user
;
1806 kstat
.cpu_system
+= system
;
1807 kstat
.per_cpu_system
[cpu
] += system
;
1814 * We take the 'long' return path, and there every subsystem
1815 * grabs the apropriate locks (kernel lock/ irq lock).
1817 * we might want to decouple profiling from the 'long path',
1818 * and do the profiling totally in assembly.
1820 * Currently this isn't too much of an issue (performance wise),
1821 * we can take more than 100K local irqs per second on a 100 MHz P5.
1826 * Local APIC timer interrupt. This is the most natural way for doing
1827 * local interrupts, but local timer interrupts can be emulated by
1828 * broadcast interrupts too. [in case the hw doesnt support APIC timers]
1830 * [ if a single-CPU system runs an SMP kernel then we call the local
1831 * interrupt as well. Thus we cannot inline the local irq ... ]
1833 void smp_apic_timer_interrupt(struct pt_regs
* regs
)
1836 * NOTE! We'd better ACK the irq immediately,
1837 * because timer handling can be slow, and we
1838 * want to be able to accept NMI tlb invalidates
1842 smp_local_timer_interrupt(regs
);
1846 * Reschedule call back. Nothing to do,
1847 * all the work is done automatically when
1848 * we return from the interrupt.
1850 asmlinkage
void smp_reschedule_interrupt(void)
1856 * Invalidate call-back
1858 asmlinkage
void smp_invalidate_interrupt(void)
1860 if (test_and_clear_bit(smp_processor_id(), &smp_invalidate_needed
))
1867 static void stop_this_cpu (void)
1872 clear_bit(smp_processor_id(), &cpu_online_map
);
1874 if (cpu_data
[smp_processor_id()].hlt_works_ok
)
1875 for(;;) __asm__("hlt");
1880 * CPU halt call-back
1882 asmlinkage
void smp_stop_cpu_interrupt(void)
1887 asmlinkage
void smp_call_function_interrupt(void)
1889 void (*func
) (void *info
) = smp_call_function_data
->func
;
1890 void *info
= smp_call_function_data
->info
;
1891 int wait
= smp_call_function_data
->wait
;
1894 /* Notify initiating CPU that I've grabbed the data and am about to
1895 execute the function */
1896 atomic_dec (&smp_call_function_data
->unstarted_count
);
1897 /* At this point the structure may be out of scope unless wait==1 */
1899 if (wait
) atomic_dec (&smp_call_function_data
->unfinished_count
);
1903 * This interrupt should _never_ happen with our APIC/SMP architecture
1905 asmlinkage
void smp_spurious_interrupt(void)
1908 /* see sw-dev-man vol 3, chapter 7.4.13.5 */
1909 printk("spurious APIC interrupt on CPU#%d, should never happen.\n",
1910 smp_processor_id());
1914 * This part sets up the APIC 32 bit clock in LVTT1, with HZ interrupts
1915 * per second. We assume that the caller has already set up the local
1918 * The APIC timer is not exactly sync with the external timer chip, it
1919 * closely follows bus clocks.
1923 * The timer chip is already set up at HZ interrupts per second here,
1924 * but we do not accept timer interrupts yet. We only allow the BP
1927 static unsigned int __init
get_8254_timer_count(void)
1932 count
= inb_p(0x40);
1933 count
|= inb_p(0x40) << 8;
1939 * This function sets up the local APIC timer, with a timeout of
1940 * 'clocks' APIC bus clock. During calibration we actually call
1941 * this function twice, once with a bogus timeout value, second
1942 * time for real. The other (noncalibrating) CPUs call this
1943 * function only once, with the real value.
1945 * We are strictly in irqs off mode here, as we do not want to
1946 * get an APIC interrupt go off accidentally.
1948 * We do reads before writes even if unnecessary, to get around the
1949 * APIC double write bug.
1952 #define APIC_DIVISOR 16
1954 void setup_APIC_timer(unsigned int clocks
)
1956 unsigned long lvtt1_value
;
1957 unsigned int tmp_value
;
1960 * Unfortunately the local APIC timer cannot be set up into NMI
1961 * mode. With the IO APIC we can re-route the external timer
1962 * interrupt and broadcast it as an NMI to all CPUs, so no pain.
1964 tmp_value
= apic_read(APIC_LVTT
);
1965 lvtt1_value
= APIC_LVT_TIMER_PERIODIC
| LOCAL_TIMER_VECTOR
;
1966 apic_write(APIC_LVTT
, lvtt1_value
);
1969 * Divide PICLK by 16
1971 tmp_value
= apic_read(APIC_TDCR
);
1972 apic_write(APIC_TDCR
, (tmp_value
& ~APIC_TDR_DIV_1
)
1975 tmp_value
= apic_read(APIC_TMICT
);
1976 apic_write(APIC_TMICT
, clocks
/APIC_DIVISOR
);
1979 void __init
wait_8254_wraparound(void)
1981 unsigned int curr_count
, prev_count
=~0;
1984 curr_count
= get_8254_timer_count();
1987 prev_count
= curr_count
;
1988 curr_count
= get_8254_timer_count();
1989 delta
= curr_count
-prev_count
;
1992 * This limit for delta seems arbitrary, but it isn't, it's
1993 * slightly above the level of error a buggy Mercury/Neptune
1994 * chipset timer can cause.
1997 } while (delta
<300);
2001 * In this function we calibrate APIC bus clocks to the external
2002 * timer. Unfortunately we cannot use jiffies and the timer irq
2003 * to calibrate, since some later bootup code depends on getting
2004 * the first irq? Ugh.
2006 * We want to do the calibration only once since we
2007 * want to have local timer irqs syncron. CPUs connected
2008 * by the same APIC bus have the very same bus frequency.
2009 * And we want to have irqs off anyways, no accidental
2010 * APIC irq that way.
2013 int __init
calibrate_APIC_clock(void)
2015 unsigned long long t1
,t2
;
2017 long calibration_result
;
2020 printk("calibrating APIC timer ... ");
2023 * Put whatever arbitrary (but long enough) timeout
2024 * value into the APIC clock, we just want to get the
2025 * counter running for calibration.
2027 setup_APIC_timer(1000000000);
2030 * The timer chip counts down to zero. Let's wait
2031 * for a wraparound to start exact measurement:
2032 * (the current tick might have been already half done)
2035 wait_8254_wraparound ();
2038 * We wrapped around just now. Let's start:
2041 tt1
=apic_read(APIC_TMCCT
);
2043 #define LOOPS (HZ/10)
2045 * Let's wait LOOPS wraprounds:
2047 for (i
=0; i
<LOOPS
; i
++)
2048 wait_8254_wraparound ();
2050 tt2
=apic_read(APIC_TMCCT
);
2054 * The APIC bus clock counter is 32 bits only, it
2055 * might have overflown, but note that we use signed
2056 * longs, thus no extra care needed.
2058 * underflown to be exact, as the timer counts down ;)
2061 calibration_result
= (tt1
-tt2
)*APIC_DIVISOR
/LOOPS
;
2063 SMP_PRINTK(("\n..... %ld CPU clocks in 1 timer chip tick.",
2064 (unsigned long)(t2
-t1
)/LOOPS
));
2066 SMP_PRINTK(("\n..... %ld APIC bus clocks in 1 timer chip tick.",
2067 calibration_result
));
2070 printk("\n..... CPU clock speed is %ld.%04ld MHz.\n",
2071 ((long)(t2
-t1
)/LOOPS
)/(1000000/HZ
),
2072 ((long)(t2
-t1
)/LOOPS
)%(1000000/HZ
) );
2074 printk("..... system bus clock speed is %ld.%04ld MHz.\n",
2075 calibration_result
/(1000000/HZ
),
2076 calibration_result
%(1000000/HZ
) );
2079 return calibration_result
;
2082 void __init
setup_APIC_clock(void)
2084 unsigned long flags
;
2086 static volatile int calibration_lock
;
2088 __save_flags(flags
);
2091 SMP_PRINTK(("setup_APIC_clock() called.\n"));
2094 * [ setup_APIC_clock() is called from all CPUs, but we want
2095 * to do this part of the setup only once ... and it fits
2098 if (!test_and_set_bit(0,&calibration_lock
)) {
2100 calibration_result
=calibrate_APIC_clock();
2102 * Signal completion to the other CPU[s]:
2104 calibration_lock
= 3;
2108 * Other CPU is calibrating, wait for finish:
2110 SMP_PRINTK(("waiting for other CPU calibrating APIC ... "));
2111 while (calibration_lock
== 1);
2112 SMP_PRINTK(("done, continuing.\n"));
2116 * Now set up the timer for real.
2119 setup_APIC_timer (calibration_result
);
2122 * We ACK the APIC, just in case there is something pending.
2127 __restore_flags(flags
);
2131 * the frequency of the profiling timer can be changed
2132 * by writing a multiplier value into /proc/profile.
2134 int setup_profiling_timer(unsigned int multiplier
)
2139 * Sanity check. [at least 500 APIC cycles should be
2140 * between APIC interrupts as a rule of thumb, to avoid
2143 if ( (!multiplier
) || (calibration_result
/multiplier
< 500))
2147 * Set the new multiplier for each CPU. CPUs don't start using the
2148 * new values until the next timer interrupt in which they do process
2149 * accounting. At that time they also adjust their APIC timers
2152 for (i
= 0; i
< NR_CPUS
; ++i
)
2153 prof_multiplier
[i
] = multiplier
;