2.2.0-final

[davej-history.git] / arch / i386 / kernel / smp.c

/*
 *      Intel MP v1.1/v1.4 specification support routines for multi-pentium
 *      hosts.
 *
 *      (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
 *      (c) 1998 Ingo Molnar
 *
 *      Supported by Caldera http://www.caldera.com.
 *      Much of the core SMP work is based on previous work by Thomas Radke, to
 *      whom a great many thanks are extended.
 *
 *      Thanks to Intel for making available several different Pentium,
 *      Pentium Pro and Pentium-II/Xeon MP machines.
 *
 *      This code is released under the GNU public license version 2 or
 *      later.
 *
 *      Fixes
 *              Felix Koop      :       NR_CPUS used properly
 *              Jose Renau      :       Handle single CPU case.
 *              Alan Cox        :       By repeated request 8) - Total BogoMIP report.
 *              Greg Wright     :       Fix for kernel stacks panic.
 *              Erich Boleyn    :       MP v1.4 and additional changes.
 *      Matthias Sattler        :       Changes for 2.1 kernel map.
 *      Michel Lespinasse       :       Changes for 2.1 kernel map.
 *      Michael Chastain        :       Change trampoline.S to gnu as.
 *              Alan Cox        :       Dumb bug: 'B' step PPro's are fine
 *              Ingo Molnar     :       Added APIC timers, based on code
 *                                      from Jose Renau
 *              Alan Cox        :       Added EBDA scanning
 *              Ingo Molnar     :       various cleanups and rewrites
 */

#include <linux/config.h>
#include <linux/mm.h>
#include <linux/kernel_stat.h>
#include <linux/delay.h>
#include <linux/mc146818rtc.h>
#include <linux/smp_lock.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <asm/pgtable.h>
#include <asm/bitops.h>
#include <asm/pgtable.h>
#include <asm/io.h>

#ifdef CONFIG_MTRR
#  include <asm/mtrr.h>
#endif

#define __KERNEL_SYSCALLS__
#include <linux/unistd.h>

#include "irq.h"

extern unsigned long start_kernel, _etext;
extern void update_one_process( struct task_struct *p,
                                unsigned long ticks, unsigned long user,
                                unsigned long system, int cpu);
/*
 *      Some notes on processor bugs:
 *
 *      Pentium and Pentium Pro (and all CPUs) have bugs. The Linux issues
 *      for SMP are handled as follows.
 *
 *      Pentium Pro
 *              Occasional delivery of 'spurious interrupt' as trap #16. This
 *      is very rare. The kernel logs the event and recovers
 *
 *      Pentium
 *              There is a marginal case where REP MOVS on 100MHz SMP
 *      machines with B stepping processors can fail. XXX should provide
 *      an L1cache=Writethrough or L1cache=off option.
 *
 *              B stepping CPUs may hang. There are hardware work arounds
 *      for this. We warn about it in case your board doesnt have the work
 *      arounds. Basically thats so I can tell anyone with a B stepping
 *      CPU and SMP problems "tough".
 *
 *      Specific items [From Pentium Processor Specification Update]
 *
 *      1AP.    Linux doesn't use remote read
 *      2AP.    Linux doesn't trust APIC errors
 *      3AP.    We work around this
 *      4AP.    Linux never generated 3 interrupts of the same priority
 *              to cause a lost local interrupt.
 *      5AP.    Remote read is never used
 *      9AP.    XXX NEED TO CHECK WE HANDLE THIS XXX
 *      10AP.   XXX NEED TO CHECK WE HANDLE THIS XXX
 *      11AP.   Linux reads the APIC between writes to avoid this, as per
 *              the documentation. Make sure you preserve this as it affects
 *              the C stepping chips too.
 *
 *      If this sounds worrying believe me these bugs are ___RARE___ and
 *      there's about nothing of note with C stepping upwards.
 */


/* Kernel spinlock */
spinlock_t kernel_flag = SPIN_LOCK_UNLOCKED;

/*
 *      Why isn't this somewhere standard ??
 *
 * Maybe because this procedure is horribly buggy, and does
 * not deserve to live.  Think about signedness issues for five
 * seconds to see why.          - Linus
 */

extern __inline int max(int a,int b)
{
        if (a>b)
                return a;
        return b;
}

/*
 * function prototypes:
 */
static void cache_APIC_registers (void);
static void stop_this_cpu (void);

static int smp_b_stepping = 0;                          /* Set if we find a B stepping CPU                      */

static int max_cpus = -1;                               /* Setup configured maximum number of CPUs to activate  */
int smp_found_config=0;                                 /* Have we found an SMP box                             */

unsigned long cpu_present_map = 0;                      /* Bitmask of physically existing CPUs                          */
unsigned long cpu_online_map = 0;                       /* Bitmask of currently online CPUs                             */
int smp_num_cpus = 1;                                   /* Total count of live CPUs                             */
int smp_threads_ready=0;                                /* Set when the idlers are all forked                   */
volatile int cpu_number_map[NR_CPUS];                   /* which CPU maps to which logical number               */
volatile int __cpu_logical_map[NR_CPUS];                        /* which logical number maps to which CPU               */
static volatile unsigned long cpu_callin_map[NR_CPUS] = {0,};   /* We always use 0 the rest is ready for parallel delivery */
static volatile unsigned long cpu_callout_map[NR_CPUS] = {0,};  /* We always use 0 the rest is ready for parallel delivery */
volatile unsigned long smp_invalidate_needed;           /* Used for the invalidate map that's also checked in the spinlock */
volatile unsigned long kstack_ptr;                      /* Stack vector for booting CPUs                        */
struct cpuinfo_x86 cpu_data[NR_CPUS];                   /* Per CPU bogomips and other parameters                */
static unsigned int num_processors = 1;                 /* Internal processor count                             */
unsigned long mp_ioapic_addr = 0xFEC00000;              /* Address of the I/O apic (not yet used)               */
unsigned char boot_cpu_id = 0;                          /* Processor that is doing the boot up                  */
static int smp_activated = 0;                           /* Tripped once we need to start cross invalidating     */
int apic_version[NR_CPUS];                              /* APIC version number                                  */
unsigned long apic_retval;                              /* Just debugging the assembler..                       */

volatile unsigned long kernel_counter=0;                /* Number of times the processor holds the lock         */
volatile unsigned long syscall_count=0;                 /* Number of times the processor holds the syscall lock */

volatile unsigned long ipi_count;                       /* Number of IPIs delivered                             */

const char lk_lockmsg[] = "lock from interrupt context at %p\n"; 

int mp_bus_id_to_type [MAX_MP_BUSSES] = { -1, };
extern int mp_irq_entries;
extern struct mpc_config_intsrc mp_irqs [MAX_IRQ_SOURCES];
extern int mpc_default_type;
int mp_bus_id_to_pci_bus [MAX_MP_BUSSES] = { -1, };
int mp_current_pci_id = 0;
unsigned long mp_lapic_addr = 0;
int skip_ioapic_setup = 0;                              /* 1 if "noapic" boot option passed */

/* #define SMP_DEBUG */

#ifdef SMP_DEBUG
#define SMP_PRINTK(x)   printk x
#else
#define SMP_PRINTK(x)
#endif

/*
 *      Setup routine for controlling SMP activation
 *
 *      Command-line option of "nosmp" or "maxcpus=0" will disable SMP
 *      activation entirely (the MPS table probe still happens, though).
 *
 *      Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
 *      greater than 0, limits the maximum number of CPUs activated in
 *      SMP mode to <NUM>.
 */

void __init smp_setup(char *str, int *ints)
{
        if (ints && ints[0] > 0)
                max_cpus = ints[1];
        else
                max_cpus = 0;
}

void ack_APIC_irq(void)
{
        /* Clear the IPI */

        /* Dummy read */
        apic_read(APIC_SPIV);

        /* Docs say use 0 for future compatibility */
        apic_write(APIC_EOI, 0);
}

#ifdef CONFIG_X86_VISWS_APIC
/*
 * hacky!
 */
int __init smp_scan_config(unsigned long base, unsigned long length)
{
        cpu_present_map |= 2; /* or in id 1 */
        apic_version[1] |= 0x10; /* integrated APIC */
        num_processors = 2;

        return 1;
} 
#else
/*
 *      Checksum an MP configuration block.
 */

static int mpf_checksum(unsigned char *mp, int len)
{
        int sum=0;
        while(len--)
                sum+=*mp++;
        return sum&0xFF;
}

/*
 *      Processor encoding in an MP configuration block
 */

static char *mpc_family(int family,int model)
{
        static char n[32];
        static char *model_defs[]=
        {
                "80486DX","80486DX",
                "80486SX","80486DX/2 or 80487",
                "80486SL","Intel5X2(tm)",
                "Unknown","Unknown",
                "80486DX/4"
        };
        if (family==0x6)
                return("Pentium(tm) Pro");
        if (family==0x5)
                return("Pentium(tm)");
        if (family==0x0F && model==0x0F)
                return("Special controller");
        if (family==0x04 && model<9)
                return model_defs[model];
        sprintf(n,"Unknown CPU [%d:%d]",family, model);
        return n;
}

/*
 *      Read the MPC
 */

static int __init smp_read_mpc(struct mp_config_table *mpc)
{
        char str[16];
        int count=sizeof(*mpc);
        int ioapics = 0;
        unsigned char *mpt=((unsigned char *)mpc)+count;

        if (memcmp(mpc->mpc_signature,MPC_SIGNATURE,4))
        {
                panic("SMP mptable: bad signature [%c%c%c%c]!\n",
                        mpc->mpc_signature[0],
                        mpc->mpc_signature[1],
                        mpc->mpc_signature[2],
                        mpc->mpc_signature[3]);
                return 1;
        }
        if (mpf_checksum((unsigned char *)mpc,mpc->mpc_length))
        {
                panic("SMP mptable: checksum error!\n");
                return 1;
        }
        if (mpc->mpc_spec!=0x01 && mpc->mpc_spec!=0x04)
        {
                printk("Bad Config Table version (%d)!!\n",mpc->mpc_spec);
                return 1;
        }
        memcpy(str,mpc->mpc_oem,8);
        str[8]=0;
        memcpy(ioapic_OEM_ID,str,9);
        printk("OEM ID: %s ",str);
        
        memcpy(str,mpc->mpc_productid,12);
        str[12]=0;
        memcpy(ioapic_Product_ID,str,13);
        printk("Product ID: %s ",str);

        printk("APIC at: 0x%lX\n",mpc->mpc_lapic);

        /* save the local APIC address, it might be non-default */
        mp_lapic_addr = mpc->mpc_lapic;

        /*
         *      Now process the configuration blocks.
         */
        
        while(count<mpc->mpc_length)
        {
                switch(*mpt)
                {
                        case MP_PROCESSOR:
                        {
                                struct mpc_config_processor *m=
                                        (struct mpc_config_processor *)mpt;
                                if (m->mpc_cpuflag&CPU_ENABLED)
                                {
                                        printk("Processor #%d %s APIC version %d\n",
                                                m->mpc_apicid,
                                                mpc_family((m->mpc_cpufeature&
                                                        CPU_FAMILY_MASK)>>8,
                                                        (m->mpc_cpufeature&
                                                                CPU_MODEL_MASK)>>4),
                                                m->mpc_apicver);
#ifdef SMP_DEBUG
                                        if (m->mpc_featureflag&(1<<0))
                                                printk("    Floating point unit present.\n");
                                        if (m->mpc_featureflag&(1<<7))
                                                printk("    Machine Exception supported.\n");
                                        if (m->mpc_featureflag&(1<<8))
                                                printk("    64 bit compare & exchange supported.\n");
                                        if (m->mpc_featureflag&(1<<9))
                                                printk("    Internal APIC present.\n");
#endif
                                        if (m->mpc_cpuflag&CPU_BOOTPROCESSOR)
                                        {
                                                SMP_PRINTK(("    Bootup CPU\n"));
                                                boot_cpu_id=m->mpc_apicid;
                                        }
                                        else    /* Boot CPU already counted */
                                                num_processors++;

                                        if (m->mpc_apicid>NR_CPUS)
                                                printk("Processor #%d unused. (Max %d processors).\n",m->mpc_apicid, NR_CPUS);
                                        else
                                        {
                                                cpu_present_map|=(1<<m->mpc_apicid);
                                                apic_version[m->mpc_apicid]=m->mpc_apicver;
                                        }
                                }
                                mpt+=sizeof(*m);
                                count+=sizeof(*m);
                                break;
                        }
                        case MP_BUS:
                        {
                                struct mpc_config_bus *m=
                                        (struct mpc_config_bus *)mpt;
                                memcpy(str,m->mpc_bustype,6);
                                str[6]=0;
                                SMP_PRINTK(("Bus #%d is %s\n",
                                        m->mpc_busid,
                                        str));
                                if ((strncmp(m->mpc_bustype,"ISA",3) == 0) ||
                                        (strncmp(m->mpc_bustype,"EISA",4) == 0))
                                        mp_bus_id_to_type[m->mpc_busid] =
                                                MP_BUS_ISA;
                                else
                                if (strncmp(m->mpc_bustype,"PCI",3) == 0) {
                                        mp_bus_id_to_type[m->mpc_busid] =
                                                MP_BUS_PCI;
                                        mp_bus_id_to_pci_bus[m->mpc_busid] =
                                                mp_current_pci_id;
                                        mp_current_pci_id++;
                                }
                                mpt+=sizeof(*m);
                                count+=sizeof(*m);
                                break;
                        }
                        case MP_IOAPIC:
                        {
                                struct mpc_config_ioapic *m=
                                        (struct mpc_config_ioapic *)mpt;
                                if (m->mpc_flags&MPC_APIC_USABLE)
                                {
                                        ioapics++;
                                        printk("I/O APIC #%d Version %d at 0x%lX.\n",
                                                m->mpc_apicid,m->mpc_apicver,
                                                m->mpc_apicaddr);
                                        /*
                                         * we use the first one only currently
                                         */
                                        if (ioapics == 1)
                                                mp_ioapic_addr = m->mpc_apicaddr;
                                }
                                mpt+=sizeof(*m);
                                count+=sizeof(*m);
                                break;
                        }
                        case MP_INTSRC:
                        {
                                struct mpc_config_intsrc *m=
                                        (struct mpc_config_intsrc *)mpt;

                                mp_irqs [mp_irq_entries] = *m;
                                if (++mp_irq_entries == MAX_IRQ_SOURCES) {
                                        printk("Max irq sources exceeded!!\n");
                                        printk("Skipping remaining sources.\n");
                                        --mp_irq_entries;
                                }

                                mpt+=sizeof(*m);
                                count+=sizeof(*m);
                                break;
                        }
                        case MP_LINTSRC:
                        {
                                struct mpc_config_intlocal *m=
                                        (struct mpc_config_intlocal *)mpt;
                                mpt+=sizeof(*m);
                                count+=sizeof(*m);
                                break;
                        }
                }
        }
        if (ioapics > 1)
        {
                printk("Warning: Multiple IO-APICs not yet supported.\n");
                printk("Warning: switching to non APIC mode.\n");
                skip_ioapic_setup=1;
        }
        return num_processors;
}

/*
 *      Scan the memory blocks for an SMP configuration block.
 */

int __init smp_scan_config(unsigned long base, unsigned long length)
{
        unsigned long *bp=phys_to_virt(base);
        struct intel_mp_floating *mpf;

        SMP_PRINTK(("Scan SMP from %p for %ld bytes.\n",
                bp,length));
        if (sizeof(*mpf)!=16)
                printk("Error: MPF size\n");

        while (length>0)
        {
                if (*bp==SMP_MAGIC_IDENT)
                {
                        mpf=(struct intel_mp_floating *)bp;
                        if (mpf->mpf_length==1 &&
                                !mpf_checksum((unsigned char *)bp,16) &&
                                (mpf->mpf_specification == 1
                                 || mpf->mpf_specification == 4) )
                        {
                                printk("Intel MultiProcessor Specification v1.%d\n", mpf->mpf_specification);
                                if (mpf->mpf_feature2&(1<<7))
                                        printk("    IMCR and PIC compatibility mode.\n");
                                else
                                        printk("    Virtual Wire compatibility mode.\n");
                                smp_found_config=1;
                                /*
                                 *      Now see if we need to read further.
                                 */
                                if (mpf->mpf_feature1!=0)
                                {
                                        unsigned long cfg;

                                        /* local APIC has default address */
                                        mp_lapic_addr = 0xFEE00000;
                                        /*
                                         *      We need to know what the local
                                         *      APIC id of the boot CPU is!
                                         */

/*
 *
 *      HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
 *
 *      It's not just a crazy hack.  ;-)
 */
                                        /*
                                         *      Standard page mapping
                                         *      functions don't work yet.
                                         *      We know that page 0 is not
                                         *      used.  Steal it for now!
                                         */
                        
                                        cfg=pg0[0];
                                        pg0[0] = (mp_lapic_addr | 7);
                                        local_flush_tlb();

                                        boot_cpu_id = GET_APIC_ID(*((volatile unsigned long *) APIC_ID));

                                        /*
                                         *      Give it back
                                         */

                                        pg0[0]= cfg;
                                        local_flush_tlb();

/*
 *
 *      END OF HACK   END OF HACK   END OF HACK   END OF HACK   END OF HACK
 *
 */
                                        /*
                                         *      2 CPUs, numbered 0 & 1.
                                         */
                                        cpu_present_map=3;
                                        num_processors=2;
                                        printk("I/O APIC at 0xFEC00000.\n");

                                        /*
                                         * Save the default type number, we
                                         * need it later to set the IO-APIC
                                         * up properly:
                                         */
                                        mpc_default_type = mpf->mpf_feature1;

                                        printk("Bus #0 is ");
                                }
                                switch(mpf->mpf_feature1)
                                {
                                        case 1:
                                        case 5:
                                                printk("ISA\n");
                                                break;
                                        case 2:
                                                printk("EISA with no IRQ8 chaining\n");
                                                break;
                                        case 6:
                                        case 3:
                                                printk("EISA\n");
                                                break;
                                        case 4:
                                        case 7:
                                                printk("MCA\n");
                                                break;
                                        case 0:
                                                break;
                                        default:
                                                printk("???\nUnknown standard configuration %d\n",
                                                        mpf->mpf_feature1);
                                                return 1;
                                }
                                if (mpf->mpf_feature1>4)
                                {
                                        printk("Bus #1 is PCI\n");

                                        /*
                                         *      Set local APIC version to
                                         *      the integrated form.
                                         *      It's initialized to zero
                                         *      otherwise, representing
                                         *      a discrete 82489DX.
                                         */
                                        apic_version[0] = 0x10;
                                        apic_version[1] = 0x10;
                                }
                                /*
                                 *      Read the physical hardware table.
                                 *      Anything here will override the
                                 *      defaults.
                                 */
                                if (mpf->mpf_physptr)
                                        smp_read_mpc((void *)mpf->mpf_physptr);

                                __cpu_logical_map[0] = boot_cpu_id;
                                global_irq_holder = boot_cpu_id;
                                current->processor = boot_cpu_id;

                                printk("Processors: %d\n", num_processors);
                                /*
                                 *      Only use the first configuration found.
                                 */
                                return 1;
                        }
                }
                bp+=4;
                length-=16;
        }

        return 0;
}
#endif

/*
 *      Trampoline 80x86 program as an array.
 */

extern unsigned char trampoline_data [];
extern unsigned char trampoline_end  [];
static unsigned char *trampoline_base;

/*
 *      Currently trivial. Write the real->protected mode
 *      bootstrap into the page concerned. The caller
 *      has made sure it's suitably aligned.
 */

static unsigned long __init setup_trampoline(void)
{
        memcpy(trampoline_base, trampoline_data, trampoline_end - trampoline_data);
        return virt_to_phys(trampoline_base);
}

/*
 *      We are called very early to get the low memory for the
 *      SMP bootup trampoline page.
 */
unsigned long __init smp_alloc_memory(unsigned long mem_base)
{
        if (virt_to_phys((void *)mem_base) >= 0x9F000)
                panic("smp_alloc_memory: Insufficient low memory for kernel trampoline 0x%lx.", mem_base);
        trampoline_base = (void *)mem_base;
        return mem_base + PAGE_SIZE;
}

/*
 *      The bootstrap kernel entry code has set these up. Save them for
 *      a given CPU
 */

void __init smp_store_cpu_info(int id)
{
        struct cpuinfo_x86 *c=&cpu_data[id];

        *c = boot_cpu_data;
        c->pte_quick = 0;
        c->pgd_quick = 0;
        c->pgtable_cache_sz = 0;
        identify_cpu(c);
        /*
         *      Mask B, Pentium, but not Pentium MMX
         */
        if (c->x86_vendor == X86_VENDOR_INTEL &&
            c->x86 == 5 &&
            c->x86_mask >= 1 && c->x86_mask <= 4 &&
            c->x86_model <= 3)
                smp_b_stepping=1;               /* Remember we have B step Pentia with bugs */
}

/*
 *      Architecture specific routine called by the kernel just before init is
 *      fired off. This allows the BP to have everything in order [we hope].
 *      At the end of this all the APs will hit the system scheduling and off
 *      we go. Each AP will load the system gdt's and jump through the kernel
 *      init into idle(). At this point the scheduler will one day take over
 *      and give them jobs to do. smp_callin is a standard routine
 *      we use to track CPUs as they power up.
 */

static atomic_t smp_commenced = ATOMIC_INIT(0);

void __init smp_commence(void)
{
        /*
         *      Lets the callins below out of their loop.
         */
        SMP_PRINTK(("Setting commenced=1, go go go\n"));

        wmb();
        atomic_set(&smp_commenced,1);
}

void __init enable_local_APIC(void)
{
        unsigned long value;

        value = apic_read(APIC_SPIV);
        value |= (1<<8);                /* Enable APIC (bit==1) */
        value &= ~(1<<9);               /* Enable focus processor (bit==0) */
        value |= 0xff;                  /* Set spurious IRQ vector to 0xff */
        apic_write(APIC_SPIV,value);

        value = apic_read(APIC_TASKPRI);
        value &= ~APIC_TPRI_MASK;       /* Set Task Priority to 'accept all' */
        apic_write(APIC_TASKPRI,value);

        udelay(100);                    /* B safe */
        ack_APIC_irq();
        udelay(100);
}

unsigned long __init init_smp_mappings(unsigned long memory_start)
{
        unsigned long apic_phys, ioapic_phys;

        memory_start = PAGE_ALIGN(memory_start);
        if (smp_found_config) {
                apic_phys = mp_lapic_addr;
#ifdef CONFIG_X86_IO_APIC
                ioapic_phys = mp_ioapic_addr;
#endif
        } else {
                /*
                 * set up a fake all zeroes page to simulate the
                 * local APIC and another one for the IO-APIC. We
                 * could use the real zero-page, but it's safer
                 * this way if some buggy code writes to this page ...
                 */
                apic_phys = __pa(memory_start);
                ioapic_phys = __pa(memory_start+PAGE_SIZE);
                memset((void *)memory_start, 0, 2*PAGE_SIZE);
                memory_start += 2*PAGE_SIZE;
        }

#ifdef CONFIG_X86_IO_APIC
        set_fixmap(FIX_APIC_BASE,apic_phys);
        set_fixmap(FIX_IO_APIC_BASE,ioapic_phys);

        printk("mapped APIC to %08lx (%08lx)\n", APIC_BASE, apic_phys);
        printk("mapped IOAPIC to %08lx (%08lx)\n", fix_to_virt(FIX_IO_APIC_BASE), ioapic_phys);
#endif

        return memory_start;
}

extern void calibrate_delay(void);

void __init smp_callin(void)
{
        int cpuid;
        unsigned long timeout;

        /*
         * (This works even if the APIC is not enabled.)
         */
        cpuid = GET_APIC_ID(apic_read(APIC_ID));

        SMP_PRINTK(("CPU#%d waiting for CALLOUT\n", cpuid));

        /*
         * STARTUP IPIs are fragile beasts as they might sometimes
         * trigger some glue motherboard logic. Complete APIC bus
         * silence for 1 second, this overestimates the time the
         * boot CPU is spending to send the up to 2 STARTUP IPIs
         * by a factor of two. This should be enough.
         */

        /*
         * Waiting 2s total for startup (udelay is not yet working)
         */
        timeout = jiffies + 2*HZ;
        while (time_before(jiffies,timeout))
        {
                /*
                 * Has the boot CPU finished it's STARTUP sequence?
                 */
                if (test_bit(cpuid, (unsigned long *)&cpu_callout_map[0]))
                        break;
        }

        while (!time_before(jiffies,timeout)) {
                printk("BUG: CPU%d started up but did not get a callout!\n",
                        cpuid);
                stop_this_cpu();
        }

        /*
         * the boot CPU has finished the init stage and is spinning
         * on callin_map until we finish. We are free to set up this
         * CPU, first the APIC. (this is probably redundant on most
         * boards)
         */
        
        SMP_PRINTK(("CALLIN, before enable_local_APIC().\n"));
        enable_local_APIC();

        /*
         * Set up our APIC timer.
         */
        setup_APIC_clock();

        __sti();

#ifdef CONFIG_MTRR
        /*  Must be done before calibration delay is computed  */
        mtrr_init_secondary_cpu ();
#endif
        /*
         *      Get our bogomips.
         */
        calibrate_delay();
        SMP_PRINTK(("Stack at about %p\n",&cpuid));

        /*
         *      Save our processor parameters
         */
        smp_store_cpu_info(cpuid);

        /*
         *      Allow the master to continue.
         */
        set_bit(cpuid, (unsigned long *)&cpu_callin_map[0]);
}

int cpucount = 0;

extern int cpu_idle(void * unused);

/*
 *      Activate a secondary processor.
 */
int __init start_secondary(void *unused)
{
        /*
         * Dont put anything before smp_callin(), SMP
         * booting is too fragile that we want to limit the
         * things done here to the most necessary things.
         */
        smp_callin();
        while (!atomic_read(&smp_commenced))
                /* nothing */ ;
        return cpu_idle(NULL);
}

/*
 * Everything has been set up for the secondary
 * CPUs - they just need to reload everything
 * from the task structure
 */
void __init initialize_secondary(void)
{
        struct thread_struct * p = &current->tss;

        /*
         * Load up the LDT and the task register.
         */
        asm volatile("lldt %%ax": :"a" (p->ldt));
        asm volatile("ltr %%ax": :"a" (p->tr));
        stts();

        /*
         * We don't actually need to load the full TSS,
         * basically just the stack pointer and the eip.
         */

        asm volatile(
                "movl %0,%%esp\n\t"
                "jmp *%1"
                :
                :"r" (p->esp),"r" (p->eip));
}

extern struct {
        void * esp;
        unsigned short ss;
} stack_start;

static void __init do_boot_cpu(int i)
{
        unsigned long cfg;
        pgd_t maincfg;
        struct task_struct *idle;
        unsigned long send_status, accept_status;
        int timeout, num_starts, j;
        unsigned long start_eip;

        /*
         *      We need an idle process for each processor.
         */

        kernel_thread(start_secondary, NULL, CLONE_PID);
        cpucount++;

        idle = task[cpucount];
        if (!idle)
                panic("No idle process for CPU %d", i);

        idle->processor = i;
        __cpu_logical_map[cpucount] = i;
        cpu_number_map[i] = cpucount;

        /* start_eip had better be page-aligned! */
        start_eip = setup_trampoline();

        printk("Booting processor %d eip %lx\n", i, start_eip); /* So we see what's up   */
        stack_start.esp = (void *) (1024 + PAGE_SIZE + (char *)idle);

        /*
         *      This grunge runs the startup process for
         *      the targeted processor.
         */

        SMP_PRINTK(("Setting warm reset code and vector.\n"));

        CMOS_WRITE(0xa, 0xf);
        local_flush_tlb();
        SMP_PRINTK(("1.\n"));
        *((volatile unsigned short *) phys_to_virt(0x469)) = start_eip >> 4;
        SMP_PRINTK(("2.\n"));
        *((volatile unsigned short *) phys_to_virt(0x467)) = start_eip & 0xf;
        SMP_PRINTK(("3.\n"));

        maincfg=swapper_pg_dir[0];
        ((unsigned long *)swapper_pg_dir)[0]=0x102007;

        /*
         *      Be paranoid about clearing APIC errors.
         */

        if ( apic_version[i] & 0xF0 )
        {
                apic_write(APIC_ESR, 0);
                accept_status = (apic_read(APIC_ESR) & 0xEF);
        }

        /*
         *      Status is now clean
         */
        
        send_status =   0;
        accept_status = 0;

        /*
         *      Starting actual IPI sequence...
         */

        SMP_PRINTK(("Asserting INIT.\n"));

        /*
         *      Turn INIT on
         */
                        
        cfg=apic_read(APIC_ICR2);
        cfg&=0x00FFFFFF;
        apic_write(APIC_ICR2, cfg|SET_APIC_DEST_FIELD(i));                      /* Target chip          */
        cfg=apic_read(APIC_ICR);
        cfg&=~0xCDFFF;                                                          /* Clear bits           */
        cfg |= (APIC_DEST_FIELD | APIC_DEST_LEVELTRIG
                | APIC_DEST_ASSERT | APIC_DEST_DM_INIT);
        apic_write(APIC_ICR, cfg);                                              /* Send IPI */

        udelay(200);
        SMP_PRINTK(("Deasserting INIT.\n"));

        cfg=apic_read(APIC_ICR2);
        cfg&=0x00FFFFFF;
        apic_write(APIC_ICR2, cfg|SET_APIC_DEST_FIELD(i));                      /* Target chip          */
        cfg=apic_read(APIC_ICR);
        cfg&=~0xCDFFF;                                                          /* Clear bits           */
        cfg |= (APIC_DEST_FIELD | APIC_DEST_LEVELTRIG
                                | APIC_DEST_DM_INIT);
        apic_write(APIC_ICR, cfg);                                              /* Send IPI */

        /*
         *      Should we send STARTUP IPIs ?
         *
         *      Determine this based on the APIC version.
         *      If we don't have an integrated APIC, don't
         *      send the STARTUP IPIs.
         */

        if ( apic_version[i] & 0xF0 )
                num_starts = 2;
        else
                num_starts = 0;

        /*
         *      Run STARTUP IPI loop.
         */

        for (j = 1; !(send_status || accept_status)
                    && (j <= num_starts) ; j++)
        {
                SMP_PRINTK(("Sending STARTUP #%d.\n",j));
                apic_write(APIC_ESR, 0);
                SMP_PRINTK(("After apic_write.\n"));

                /*
                 *      STARTUP IPI
                 */

                cfg=apic_read(APIC_ICR2);
                cfg&=0x00FFFFFF;
                apic_write(APIC_ICR2, cfg|SET_APIC_DEST_FIELD(i));                      /* Target chip          */
                cfg=apic_read(APIC_ICR);
                cfg&=~0xCDFFF;                                                          /* Clear bits           */
                cfg |= (APIC_DEST_FIELD
                        | APIC_DEST_DM_STARTUP
                        | (start_eip >> 12));                                           /* Boot on the stack    */
                SMP_PRINTK(("Before start apic_write.\n"));
                apic_write(APIC_ICR, cfg);                                              /* Kick the second      */

                SMP_PRINTK(("Startup point 1.\n"));

                timeout = 0;
                SMP_PRINTK(("Waiting for send to finish...\n"));
                do {
                        SMP_PRINTK(("+"));
                        udelay(100);
                        send_status = apic_read(APIC_ICR) & 0x1000;
                } while (send_status && (timeout++ < 1000));

                /*
                 * Give the other CPU some time to accept the IPI.
                 */
                udelay(200);
                accept_status = (apic_read(APIC_ESR) & 0xEF);
        }
        SMP_PRINTK(("After Startup.\n"));

        if (send_status)                /* APIC never delivered?? */
                printk("APIC never delivered???\n");
        if (accept_status)              /* Send accept error */
                printk("APIC delivery error (%lx).\n", accept_status);

        if ( !(send_status || accept_status) )
        {
                /*
                 * allow APs to start initializing.
                 */
                SMP_PRINTK(("Before Callout %d.\n", i));
                set_bit(i, (unsigned long *)&cpu_callout_map[0]);
                SMP_PRINTK(("After Callout %d.\n", i));

                for(timeout=0;timeout<50000;timeout++)
                {
                        if (cpu_callin_map[0]&(1<<i))
                                break;                          /* It has booted */
                        udelay(100);                            /* Wait 5s total for a response */
                }
                if (cpu_callin_map[0]&(1<<i))
                {
                        /* number CPUs logically, starting from 1 (BSP is 0) */
#if 0
                        cpu_number_map[i] = cpucount;
                        __cpu_logical_map[cpucount] = i;
#endif
                        printk("OK.\n");
                        printk("CPU%d: ", i);
                        print_cpu_info(&cpu_data[i]);
                }
                else
                {
                        if (*((volatile unsigned char *)phys_to_virt(8192))==0xA5)
                                printk("Stuck ??\n");
                        else
                                printk("Not responding.\n");
                }
        SMP_PRINTK(("CPU has booted.\n"));
        }
        else
        {
                __cpu_logical_map[cpucount] = -1;
                cpu_number_map[i] = -1;
                cpucount--;
        }

        swapper_pg_dir[0]=maincfg;
        local_flush_tlb();

        /* mark "stuck" area as not stuck */
        *((volatile unsigned long *)phys_to_virt(8192)) = 0;
}

cycles_t cacheflush_time;
extern unsigned long cpu_hz;

static void smp_tune_scheduling (void)
{
        unsigned long cachesize;
        /*
         * Rough estimation for SMP scheduling, this is the number of
         * cycles it takes for a fully memory-limited process to flush
         * the SMP-local cache.
         *
         * (For a P5 this pretty much means we will choose another idle
         *  CPU almost always at wakeup time (this is due to the small
         *  L1 cache), on PIIs it's around 50-100 usecs, depending on
         *  the cache size)
         */

        if (!cpu_hz) {
                /*
                 * this basically disables processor-affinity
                 * scheduling on SMP without a TSC.
                 */
                cacheflush_time = 0;
                return;
        } else {
                cachesize = boot_cpu_data.x86_cache_size;
                if (cachesize == -1)
                        cachesize = 8; /* Pentiums */

                cacheflush_time = cpu_hz/1024*cachesize/5000;
        }

        printk("per-CPU timeslice cutoff: %ld.%02ld usecs.\n",
                (long)cacheflush_time/(cpu_hz/1000000),
                ((long)cacheflush_time*100/(cpu_hz/1000000)) % 100);
}

unsigned int prof_multiplier[NR_CPUS];
unsigned int prof_counter[NR_CPUS];

/*
 *      Cycle through the processors sending APIC IPIs to boot each.
 */

void __init smp_boot_cpus(void)
{
        int i;
        unsigned long cfg;

#ifdef CONFIG_MTRR
        /*  Must be done before other processors booted  */
        mtrr_init_boot_cpu ();
#endif
        /*
         *      Initialize the logical to physical CPU number mapping
         *      and the per-CPU profiling counter/multiplier
         */

        for (i = 0; i < NR_CPUS; i++) {
                cpu_number_map[i] = -1;
                prof_counter[i] = 1;
                prof_multiplier[i] = 1;
        }

        /*
         *      Setup boot CPU information
         */

        smp_store_cpu_info(boot_cpu_id);                        /* Final full version of the data */
        smp_tune_scheduling();
        printk("CPU%d: ", boot_cpu_id);
        print_cpu_info(&cpu_data[boot_cpu_id]);

        /*
         * not necessary because the MP table should list the boot
         * CPU too, but we do it for the sake of robustness anyway.
         * (and for the case when a non-SMP board boots an SMP kernel)
         */
        cpu_present_map |= (1 << hard_smp_processor_id());

        cpu_number_map[boot_cpu_id] = 0;

#ifdef CONFIG_X86_IO_APIC
        /*
         *      If we don't conform to the Intel MPS standard, get out
         *      of here now!
         */

        if (!smp_found_config)
        {
                printk(KERN_NOTICE "SMP motherboard not detected. Using dummy APIC emulation.\n");
                io_apic_irqs = 0;
                cpu_online_map = cpu_present_map;
                goto smp_done;
        }
#endif

        /*
         *      If SMP should be disabled, then really disable it!
         */

        if (!max_cpus)
        {
                smp_found_config = 0;
                printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
        }

#ifdef SMP_DEBUG
        {
                int reg;

                /*
                 *      This is to verify that we're looking at
                 *      a real local APIC.  Check these against
                 *      your board if the CPUs aren't getting
                 *      started for no apparent reason.
                 */

                reg = apic_read(APIC_VERSION);
                SMP_PRINTK(("Getting VERSION: %x\n", reg));

                apic_write(APIC_VERSION, 0);
                reg = apic_read(APIC_VERSION);
                SMP_PRINTK(("Getting VERSION: %x\n", reg));

                /*
                 *      The two version reads above should print the same
                 *      NON-ZERO!!! numbers.  If the second one is zero,
                 *      there is a problem with the APIC write/read
                 *      definitions.
                 *
                 *      The next two are just to see if we have sane values.
                 *      They're only really relevant if we're in Virtual Wire
                 *      compatibility mode, but most boxes are anymore.
                 */


                reg = apic_read(APIC_LVT0);
                SMP_PRINTK(("Getting LVT0: %x\n", reg));

                reg = apic_read(APIC_LVT1);
                SMP_PRINTK(("Getting LVT1: %x\n", reg));
        }
#endif

        enable_local_APIC();

        /*
         * Set up our local APIC timer:
         */
        setup_APIC_clock ();

        /*
         *      Now scan the CPU present map and fire up the other CPUs.
         */

        /*
         * Add all detected CPUs. (later on we can down individual
         * CPUs which will change cpu_online_map but not necessarily
         * cpu_present_map. We are pretty much ready for hot-swap CPUs.)
         */
        cpu_online_map = cpu_present_map;
        mb();

        SMP_PRINTK(("CPU map: %lx\n", cpu_present_map));

        for(i=0;i<NR_CPUS;i++)
        {
                /*
                 *      Don't even attempt to start the boot CPU!
                 */
                if (i == boot_cpu_id)
                        continue;

                if ((cpu_online_map & (1 << i))
                    && (max_cpus < 0 || max_cpus > cpucount+1))
                {
                        do_boot_cpu(i);
                }

                /*
                 *      Make sure we unmap all failed CPUs
                 */
                
                if (cpu_number_map[i] == -1 && (cpu_online_map & (1 << i))) {
                        printk("CPU #%d not responding. Removing from cpu_online_map.\n",i);
                        cpu_online_map &= ~(1 << i);
                }
        }

        /*
         *      Cleanup possible dangling ends...
         */

        /*
         *      Install writable page 0 entry.
         */

        cfg = pg0[0];
        pg0[0] = 3;     /* writeable, present, addr 0 */
        local_flush_tlb();

        /*
         *      Paranoid:  Set warm reset code and vector here back
         *      to default values.
         */

        CMOS_WRITE(0, 0xf);

        *((volatile long *) phys_to_virt(0x467)) = 0;

        /*
         *      Restore old page 0 entry.
         */

        pg0[0] = cfg;
        local_flush_tlb();

        /*
         *      Allow the user to impress friends.
         */

        SMP_PRINTK(("Before bogomips.\n"));
        if (cpucount==0)
        {
                printk(KERN_ERR "Error: only one processor found.\n");
                cpu_online_map = (1<<hard_smp_processor_id());
        }
        else
        {
                unsigned long bogosum=0;
                for(i=0;i<32;i++)
                {
                        if (cpu_online_map&(1<<i))
                                bogosum+=cpu_data[i].loops_per_sec;
                }
                printk(KERN_INFO "Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
                        cpucount+1,
                        (bogosum+2500)/500000,
                        ((bogosum+2500)/5000)%100);
                SMP_PRINTK(("Before bogocount - setting activated=1.\n"));
                smp_activated=1;
                smp_num_cpus=cpucount+1;
        }
        if (smp_b_stepping)
                printk(KERN_WARNING "WARNING: SMP operation may be unreliable with B stepping processors.\n");
        SMP_PRINTK(("Boot done.\n"));

        cache_APIC_registers();
#ifdef CONFIG_X86_IO_APIC
        /*
         * Here we can be sure that there is an IO-APIC in the system. Let's
         * go and set it up:
         */
        if (!skip_ioapic_setup) 
                setup_IO_APIC();
smp_done:
#endif
}


/*
 * the following functions deal with sending IPIs between CPUs.
 *
 * We use 'broadcast', CPU->CPU IPIs and self-IPIs too.
 */


/*
 * Silly serialization to work around CPU bug in P5s.
 * We can safely turn it off on a 686.
 */
#ifdef CONFIG_X86_GOOD_APIC
# define FORCE_APIC_SERIALIZATION 0
#else
# define FORCE_APIC_SERIALIZATION 1
#endif

static unsigned int cached_APIC_ICR;
static unsigned int cached_APIC_ICR2;

/*
 * Caches reserved bits, APIC reads are (mildly) expensive
 * and force otherwise unnecessary CPU synchronization.
 *
 * (We could cache other APIC registers too, but these are the
 * main ones used in RL.)
 */
#define slow_ICR (apic_read(APIC_ICR) & ~0xFDFFF)
#define slow_ICR2 (apic_read(APIC_ICR2) & 0x00FFFFFF)

void cache_APIC_registers (void)
{
        cached_APIC_ICR = slow_ICR;
        cached_APIC_ICR2 = slow_ICR2;
        mb();
}

static inline unsigned int __get_ICR (void)
{
#if FORCE_APIC_SERIALIZATION
        /*
         * Wait for the APIC to become ready - this should never occur. It's
         * a debugging check really.
         */
        int count = 0;
        unsigned int cfg;

        while (count < 1000)
        {
                cfg = slow_ICR;
                if (!(cfg&(1<<12))) {
                        if (count)
                                atomic_add(count, (atomic_t*)&ipi_count);
                        return cfg;
                }
                count++;
                udelay(10);
        }
        printk("CPU #%d: previous IPI still not cleared after 10mS\n",
                        smp_processor_id());
        return cfg;
#else
        return cached_APIC_ICR;
#endif
}

static inline unsigned int __get_ICR2 (void)
{
#if FORCE_APIC_SERIALIZATION
        return slow_ICR2;
#else
        return cached_APIC_ICR2;
#endif
}

static inline int __prepare_ICR (unsigned int shortcut, int vector)
{
        unsigned int cfg;

        cfg = __get_ICR();
        cfg |= APIC_DEST_FIELD|APIC_DEST_DM_FIXED|shortcut|vector;

        return cfg;
}

static inline int __prepare_ICR2 (unsigned int dest)
{
        unsigned int cfg;

        cfg = __get_ICR2();
        cfg |= SET_APIC_DEST_FIELD(dest);

        return cfg;
}

static inline void __send_IPI_shortcut(unsigned int shortcut, int vector)
{
        unsigned int cfg;
/*
 * Subtle. In the case of the 'never do double writes' workaround we
 * have to lock out interrupts to be safe. Otherwise it's just one
 * single atomic write to the APIC, no need for cli/sti.
 */
#if FORCE_APIC_SERIALIZATION
        unsigned long flags;

        __save_flags(flags);
        __cli();
#endif

        /*
         * No need to touch the target chip field
         */

        cfg = __prepare_ICR(shortcut, vector);

        /*
         * Send the IPI. The write to APIC_ICR fires this off.
         */
        apic_write(APIC_ICR, cfg);
#if FORCE_APIC_SERIALIZATION
        __restore_flags(flags);
#endif
}

static inline void send_IPI_allbutself(int vector)
{
        __send_IPI_shortcut(APIC_DEST_ALLBUT, vector);
}

static inline void send_IPI_all(int vector)
{
        __send_IPI_shortcut(APIC_DEST_ALLINC, vector);
}

void send_IPI_self(int vector)
{
        __send_IPI_shortcut(APIC_DEST_SELF, vector);
}

static inline void send_IPI_single(int dest, int vector)
{
        unsigned long cfg;
#if FORCE_APIC_SERIALIZATION
        unsigned long flags;

        __save_flags(flags);
        __cli();
#endif

        /*
         * prepare target chip field
         */

        cfg = __prepare_ICR2(dest);
        apic_write(APIC_ICR2, cfg);

        /*
         * program the ICR 
         */
        cfg = __prepare_ICR(0, vector);
        
        /*
         * Send the IPI. The write to APIC_ICR fires this off.
         */
        apic_write(APIC_ICR, cfg);
#if FORCE_APIC_SERIALIZATION
        __restore_flags(flags);
#endif
}

/*
 * This is fraught with deadlocks. Probably the situation is not that
 * bad as in the early days of SMP, so we might ease some of the
 * paranoia here.
 */

void smp_flush_tlb(void)
{
        int cpu = smp_processor_id();
        int stuck;
        unsigned long flags;

        /*
         * it's important that we do not generate any APIC traffic
         * until the AP CPUs have booted up!
         */
        if (cpu_online_map) {
                /*
                 * The assignment is safe because it's volatile so the
                 * compiler cannot reorder it, because the i586 has
                 * strict memory ordering and because only the kernel
                 * lock holder may issue a tlb flush. If you break any
                 * one of those three change this to an atomic bus
                 * locked or.
                 */

                smp_invalidate_needed = cpu_online_map;

                /*
                 * Processors spinning on some lock with IRQs disabled
                 * will see this IRQ late. The smp_invalidate_needed
                 * map will ensure they don't do a spurious flush tlb
                 * or miss one.
                 */
        
                __save_flags(flags);
                __cli();

                send_IPI_allbutself(INVALIDATE_TLB_VECTOR);

                /*
                 * Spin waiting for completion
                 */

                stuck = 50000000;
                while (smp_invalidate_needed) {
                        /*
                         * Take care of "crossing" invalidates
                         */
                        if (test_bit(cpu, &smp_invalidate_needed))
                        clear_bit(cpu, &smp_invalidate_needed);
                        --stuck;
                        if (!stuck) {
                                printk("stuck on TLB IPI wait (CPU#%d)\n",cpu);
                                break;
                        }
                }
                __restore_flags(flags);
        }

        /*
         *      Flush the local TLB
         */
        local_flush_tlb();

}


/*
 * this function sends a 'reschedule' IPI to another CPU.
 * it goes straight through and wastes no time serializing
 * anything. Worst case is that we lose a reschedule ...
 */

void smp_send_reschedule(int cpu)
{
        send_IPI_single(cpu, RESCHEDULE_VECTOR);
}

/*
 * this function sends a 'stop' IPI to all other CPUs in the system.
 * it goes straight through.
 */

void smp_send_stop(void)
{
        send_IPI_allbutself(STOP_CPU_VECTOR);
}

/*
 * this function sends an 'reload MTRR state' IPI to all other CPUs
 * in the system. it goes straight through, completion processing
 * is done on the mttr.c level.
 */

void smp_send_mtrr(void)
{
        send_IPI_allbutself(MTRR_CHANGE_VECTOR);
}

/*
 * Local timer interrupt handler. It does both profiling and
 * process statistics/rescheduling.
 *
 * We do profiling in every local tick, statistics/rescheduling
 * happen only every 'profiling multiplier' ticks. The default
 * multiplier is 1 and it can be changed by writing the new multiplier
 * value into /proc/profile.
 */

void smp_local_timer_interrupt(struct pt_regs * regs)
{
        int cpu = smp_processor_id();

        /*
         * The profiling function is SMP safe. (nothing can mess
         * around with "current", and the profiling counters are
         * updated with atomic operations). This is especially
         * useful with a profiling multiplier != 1
         */
        if (!user_mode(regs))
                x86_do_profile(regs->eip);

        if (!--prof_counter[cpu]) {
                int user=0,system=0;
                struct task_struct * p = current;

                /*
                 * After doing the above, we need to make like
                 * a normal interrupt - otherwise timer interrupts
                 * ignore the global interrupt lock, which is the
                 * WrongThing (tm) to do.
                 */

                if (user_mode(regs))
                        user=1;
                else
                        system=1;

                irq_enter(cpu, 0);
                if (p->pid) {
                        update_one_process(p, 1, user, system, cpu);

                        p->counter -= 1;
                        if (p->counter < 0) {
                                p->counter = 0;
                                p->need_resched = 1;
                        }
                        if (p->priority < DEF_PRIORITY) {
                                kstat.cpu_nice += user;
                                kstat.per_cpu_nice[cpu] += user;
                        } else {
                                kstat.cpu_user += user;
                                kstat.per_cpu_user[cpu] += user;
                        }

                        kstat.cpu_system += system;
                        kstat.per_cpu_system[cpu] += system;

                }
                prof_counter[cpu]=prof_multiplier[cpu];
                irq_exit(cpu, 0);
        }

        /*
         * We take the 'long' return path, and there every subsystem
         * grabs the apropriate locks (kernel lock/ irq lock).
         *
         * we might want to decouple profiling from the 'long path',
         * and do the profiling totally in assembly.
         *
         * Currently this isn't too much of an issue (performance wise),
         * we can take more than 100K local irqs per second on a 100 MHz P5.
         */
}

/*
 * Local APIC timer interrupt. This is the most natural way for doing
 * local interrupts, but local timer interrupts can be emulated by
 * broadcast interrupts too. [in case the hw doesnt support APIC timers]
 *
 * [ if a single-CPU system runs an SMP kernel then we call the local
 *   interrupt as well. Thus we cannot inline the local irq ... ]
 */
void smp_apic_timer_interrupt(struct pt_regs * regs)
{
        /*
         * NOTE! We'd better ACK the irq immediately,
         * because timer handling can be slow, and we
         * want to be able to accept NMI tlb invalidates
         * during this time.
         */
        ack_APIC_irq();
        smp_local_timer_interrupt(regs);
}

/*
 * Reschedule call back. Nothing to do,
 * all the work is done automatically when
 * we return from the interrupt.
 */
asmlinkage void smp_reschedule_interrupt(void)
{
        ack_APIC_irq();
}

/*
 * Invalidate call-back
 */
asmlinkage void smp_invalidate_interrupt(void)
{
        if (test_and_clear_bit(smp_processor_id(), &smp_invalidate_needed))
                local_flush_tlb();

        ack_APIC_irq();
}

static void stop_this_cpu (void)
{
        /*
         * Remove this CPU:
         */
        clear_bit(smp_processor_id(), &cpu_online_map);

        if (cpu_data[smp_processor_id()].hlt_works_ok)
                for(;;) __asm__("hlt");
        for (;;);
}

/*
 *      CPU halt call-back
 */
asmlinkage void smp_stop_cpu_interrupt(void)
{
        stop_this_cpu();
}

void (*mtrr_hook) (void) = NULL;

asmlinkage void smp_mtrr_interrupt(void)
{
        ack_APIC_irq();
        if (mtrr_hook) (*mtrr_hook)();
}

/*
 * This interrupt should _never_ happen with our APIC/SMP architecture
 */
asmlinkage void smp_spurious_interrupt(void)
{
        /* ack_APIC_irq();   see sw-dev-man vol 3, chapter 7.4.13.5 */
        printk("spurious APIC interrupt, ayiee, should never happen.\n");
}

/*
 * This part sets up the APIC 32 bit clock in LVTT1, with HZ interrupts
 * per second. We assume that the caller has already set up the local
 * APIC.
 *
 * The APIC timer is not exactly sync with the external timer chip, it
 * closely follows bus clocks.
 */

#define RDTSC(x)        __asm__ __volatile__ (  "rdtsc" \
                                :"=a" (((unsigned long*)&x)[0]),  \
                                 "=d" (((unsigned long*)&x)[1]))

/*
 * The timer chip is already set up at HZ interrupts per second here,
 * but we do not accept timer interrupts yet. We only allow the BP
 * to calibrate.
 */
static unsigned int __init get_8254_timer_count(void)
{
        unsigned int count;

        outb_p(0x00, 0x43);
        count = inb_p(0x40);
        count |= inb_p(0x40) << 8;

        return count;
}

/*
 * This function sets up the local APIC timer, with a timeout of
 * 'clocks' APIC bus clock. During calibration we actually call
 * this function twice, once with a bogus timeout value, second
 * time for real. The other (noncalibrating) CPUs call this
 * function only once, with the real value.
 *
 * We are strictly in irqs off mode here, as we do not want to
 * get an APIC interrupt go off accidentally.
 *
 * We do reads before writes even if unnecessary, to get around the
 * APIC double write bug.
 */

#define APIC_DIVISOR 16

void setup_APIC_timer(unsigned int clocks)
{
        unsigned long lvtt1_value;
        unsigned int tmp_value;

        /*
         * Unfortunately the local APIC timer cannot be set up into NMI
         * mode. With the IO APIC we can re-route the external timer
         * interrupt and broadcast it as an NMI to all CPUs, so no pain.
         */
        tmp_value = apic_read(APIC_LVTT);
        lvtt1_value = APIC_LVT_TIMER_PERIODIC | LOCAL_TIMER_VECTOR;
        apic_write(APIC_LVTT , lvtt1_value);

        /*
         * Divide PICLK by 16
         */
        tmp_value = apic_read(APIC_TDCR);
        apic_write(APIC_TDCR , (tmp_value & ~APIC_TDR_DIV_1 )
                                 | APIC_TDR_DIV_16);

        tmp_value = apic_read(APIC_TMICT);
        apic_write(APIC_TMICT, clocks/APIC_DIVISOR);
}

void __init wait_8254_wraparound(void)
{
        unsigned int curr_count, prev_count=~0;
        int delta;

        curr_count = get_8254_timer_count();

        do {
                prev_count = curr_count;
                curr_count = get_8254_timer_count();
                delta = curr_count-prev_count;

        /*
         * This limit for delta seems arbitrary, but it isn't, it's
         * slightly above the level of error a buggy Mercury/Neptune
         * chipset timer can cause.
         */

        } while (delta<300);
}

/*
 * In this function we calibrate APIC bus clocks to the external
 * timer. Unfortunately we cannot use jiffies and the timer irq
 * to calibrate, since some later bootup code depends on getting
 * the first irq? Ugh.
 *
 * We want to do the calibration only once since we
 * want to have local timer irqs syncron. CPUs connected
 * by the same APIC bus have the very same bus frequency.
 * And we want to have irqs off anyways, no accidental
 * APIC irq that way.
 */

int __init calibrate_APIC_clock(void)
{
        unsigned long long t1,t2;
        long tt1,tt2;
        long calibration_result;
        int i;

        printk("calibrating APIC timer ... ");

        /*
         * Put whatever arbitrary (but long enough) timeout
         * value into the APIC clock, we just want to get the
         * counter running for calibration.
         */
        setup_APIC_timer(1000000000);

        /*
         * The timer chip counts down to zero. Let's wait
         * for a wraparound to start exact measurement:
         * (the current tick might have been already half done)
         */

        wait_8254_wraparound ();

        /*
         * We wrapped around just now. Let's start:
         */
        RDTSC(t1);
        tt1=apic_read(APIC_TMCCT);

#define LOOPS (HZ/10)
        /*
         * Let's wait LOOPS wraprounds:
         */
        for (i=0; i<LOOPS; i++)
                wait_8254_wraparound ();

        tt2=apic_read(APIC_TMCCT);
        RDTSC(t2);

        /*
         * The APIC bus clock counter is 32 bits only, it
         * might have overflown, but note that we use signed
         * longs, thus no extra care needed.
         *
         * underflown to be exact, as the timer counts down ;)
         */

        calibration_result = (tt1-tt2)*APIC_DIVISOR/LOOPS;

        SMP_PRINTK(("\n..... %ld CPU clocks in 1 timer chip tick.",
                         (unsigned long)(t2-t1)/LOOPS));

        SMP_PRINTK(("\n..... %ld APIC bus clocks in 1 timer chip tick.",
                         calibration_result));


        printk("\n..... CPU clock speed is %ld.%04ld MHz.\n",
                ((long)(t2-t1)/LOOPS)/(1000000/HZ),
                ((long)(t2-t1)/LOOPS)%(1000000/HZ)  );

        printk("..... APIC bus clock speed is %ld.%04ld MHz.\n",
                calibration_result/(1000000/HZ),
                calibration_result%(1000000/HZ)  );
#undef LOOPS

        return calibration_result;
}

static unsigned int calibration_result;

void __init setup_APIC_clock(void)
{
        unsigned long flags;

        static volatile int calibration_lock;

        __save_flags(flags);
        __cli();

        SMP_PRINTK(("setup_APIC_clock() called.\n"));

        /*
         * [ setup_APIC_clock() is called from all CPUs, but we want
         *   to do this part of the setup only once ... and it fits
         *   here best ]
         */
        if (!test_and_set_bit(0,&calibration_lock)) {

                calibration_result=calibrate_APIC_clock();
                /*
                 * Signal completion to the other CPU[s]:
                 */
                calibration_lock = 3;

        } else {
                /*
                 * Other CPU is calibrating, wait for finish:
                 */
                SMP_PRINTK(("waiting for other CPU calibrating APIC ... "));
                while (calibration_lock == 1);
                SMP_PRINTK(("done, continuing.\n"));
        }

/*
 * Now set up the timer for real.
 */

        setup_APIC_timer (calibration_result);

        /*
         * We ACK the APIC, just in case there is something pending.
         */

        ack_APIC_irq ();

        __restore_flags(flags);
}

/*
 * the frequency of the profiling timer can be changed
 * by writing a multiplier value into /proc/profile.
 *
 * usually you want to run this on all CPUs ;)
 */
int setup_profiling_timer(unsigned int multiplier)
{
        int cpu = smp_processor_id();
        unsigned long flags;

        /*
         * Sanity check. [at least 500 APIC cycles should be
         * between APIC interrupts as a rule of thumb, to avoid
         * irqs flooding us]
         */
        if ( (!multiplier) || (calibration_result/multiplier < 500))
                return -EINVAL;

        save_flags(flags);
        cli();
        setup_APIC_timer(calibration_result/multiplier);
        prof_multiplier[cpu]=multiplier;
        restore_flags(flags);

        return 0;
}

#undef APIC_DIVISOR