x86: Streamline the CPU early boot process.

Mainly this involves getting rid of a bunch of cpumasks and replacing
with a single 'cpu_state' enumeration to track progress and allow
master-slave handshaking.

Cleaning this stuff up is a prerequisite for safely handling slave
failure (e.g., out of memory, invalid slave CPU capabilities,
...). This will get fixed up in a future patch.

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>

6 files changed, 35 insertions, 117 deletions

diff --git a/xen/arch/x86/cpu/mtrr/main.c b/xen/arch/x86/cpu/mtrr/main.c
index fcbf1ccda6..1acd03a5a6 100644
--- a/xen/arch/x86/cpu/mtrr/main.c
+++ b/xen/arch/x86/cpu/mtrr/main.c
@@ -236,7 +236,7 @@ static void set_mtrr(unsigned int reg, unsigned long base,
 	data.smp_base = base;
 	data.smp_size = size;
 	data.smp_type = type;
-	atomic_set(&data.count, num_booting_cpus() - 1);
+	atomic_set(&data.count, num_online_cpus() - 1);
 	/* make sure data.count is visible before unleashing other CPUs */
 	smp_wmb();
 	atomic_set(&data.gate,0);
@@ -251,7 +251,7 @@ static void set_mtrr(unsigned int reg, unsigned long base,
 		cpu_relax();
 
 	/* ok, reset count and toggle gate */
-	atomic_set(&data.count, num_booting_cpus() - 1);
+	atomic_set(&data.count, num_online_cpus() - 1);
 	smp_wmb();
 	atomic_set(&data.gate,1);
 
@@ -274,7 +274,7 @@ static void set_mtrr(unsigned int reg, unsigned long base,
 	while(atomic_read(&data.count))
 		cpu_relax();
 
-	atomic_set(&data.count, num_booting_cpus() - 1);
+	atomic_set(&data.count, num_online_cpus() - 1);
 	smp_wmb();
 	atomic_set(&data.gate,0);
 
diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index b442a874ef..311c014e26 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -98,15 +98,10 @@ static void default_dead_idle(void)
 
 static void play_dead(void)
 {
-    /* This must be done before dead CPU ack */
     cpu_exit_clear(smp_processor_id());
-    wbinvd();
     mb();
-    /* Ack it */
-    __get_cpu_var(cpu_state) = CPU_DEAD;
-
-    /* With physical CPU hotplug, we should halt the cpu. */
     local_irq_disable();
+    wbinvd();
     (*dead_idle)();
 }
 
diff --git a/xen/arch/x86/nmi.c b/xen/arch/x86/nmi.c
index 6649afa1cc..d6fd625d01 100644
--- a/xen/arch/x86/nmi.c
+++ b/xen/arch/x86/nmi.c
@@ -105,16 +105,13 @@ int __init check_nmi_watchdog (void)
 
     printk("Testing NMI watchdog --- ");
 
-    for ( cpu = 0; cpu < NR_CPUS; cpu++ ) 
+    for_each_online_cpu ( cpu )
         prev_nmi_count[cpu] = nmi_count(cpu);
     local_irq_enable();
     mdelay((10*1000)/nmi_hz); /* wait 10 ticks */
 
-    for ( cpu = 0; cpu < NR_CPUS; cpu++ ) 
+    for_each_online_cpu ( cpu )
     {
-        if ( !cpu_isset(cpu, cpu_callin_map) && 
-             !cpu_isset(cpu, cpu_online_map) )
-            continue;
         if ( nmi_count(cpu) - prev_nmi_count[cpu] <= 5 )
             printk("CPU#%d stuck. ", cpu);
         else
diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index 69efba68c6..caddfad492 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -65,17 +65,20 @@ DEFINE_PER_CPU_READ_MOSTLY(cpumask_t, cpu_core_map);
 cpumask_t cpu_online_map __read_mostly;
 EXPORT_SYMBOL(cpu_online_map);
 
-cpumask_t cpu_callin_map;
-cpumask_t cpu_callout_map;
-static cpumask_t smp_commenced_mask;
-
 struct cpuinfo_x86 cpu_data[NR_CPUS];
 
 u32 x86_cpu_to_apicid[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = -1U };
 
 static void map_cpu_to_logical_apicid(void);
 
-DEFINE_PER_CPU(int, cpu_state);
+enum cpu_state {
+    CPU_STATE_DEAD = 0, /* slave -> master: I am completely dead */
+    CPU_STATE_INIT,     /* master -> slave: Early bringup phase 1 */
+    CPU_STATE_CALLOUT,  /* master -> slave: Early bringup phase 2 */
+    CPU_STATE_CALLIN,   /* slave -> master: Completed phase 2 */
+    CPU_STATE_ONLINE    /* master -> slave: Go fully online now. */
+} cpu_state;
+#define set_cpu_state(state) do { mb(); cpu_state = (state); } while (0)
 
 void *stack_base[NR_CPUS];
 
@@ -128,75 +131,38 @@ static void smp_store_cpu_info(int id)
     ;
 }
 
-static atomic_t init_deasserted;
-
 void smp_callin(void)
 {
-    int cpuid, phys_id, i;
-
-    /*
-     * If waken up by an INIT in an 82489DX configuration
-     * we may get here before an INIT-deassert IPI reaches
-     * our local APIC.  We have to wait for the IPI or we'll
-     * lock up on an APIC access.
-     */
-    wait_for_init_deassert(&init_deasserted);
-
-    if ( x2apic_enabled )
-        enable_x2apic();
-
-    /*
-     * (This works even if the APIC is not enabled.)
-     */
-    phys_id = get_apic_id();
-    cpuid = smp_processor_id();
-    if ( cpu_isset(cpuid, cpu_callin_map) )
-    {
-        printk("huh, phys CPU#%d, CPU#%d already present??\n",
-               phys_id, cpuid);
-        BUG();
-    }
-    Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
-
-    /*
-     * STARTUP IPIs are fragile beasts as they might sometimes
-     * trigger some glue motherboard logic. Complete APIC bus
-     * silence for 1 second, this overestimates the time the
-     * boot CPU is spending to send the up to 2 STARTUP IPIs
-     * by a factor of two. This should be enough.
-     */
+    int i;
 
     /* Wait 2s total for startup. */
-    for ( i = 0; (i < 200) && !cpu_isset(cpuid, cpu_callout_map); i++ )
+    Dprintk("Waiting for CALLOUT.\n");
+    for ( i = 0; cpu_state != CPU_STATE_CALLOUT; i++ )
     {
+        BUG_ON(i >= 200);
         cpu_relax();
         mdelay(10);
     }
 
-    if ( !cpu_isset(cpuid, cpu_callout_map) )
-    {
-        printk("BUG: CPU%d started up but did not get a callout!\n",
-               cpuid);
-        BUG();
-    }
-
     /*
-     * the boot CPU has finished the init stage and is spinning
-     * on callin_map until we finish. We are free to set up this
-     * CPU, first the APIC. (this is probably redundant on most
-     * boards)
+     * The boot CPU has finished the init stage and is spinning on cpu_state
+     * update until we finish. We are free to set up this CPU: first the APIC.
      */
-
     Dprintk("CALLIN, before setup_local_APIC().\n");
-    smp_callin_clear_local_apic();
+    if ( x2apic_enabled )
+        enable_x2apic();
     setup_local_APIC();
     map_cpu_to_logical_apicid();
 
     /* Save our processor parameters. */
-    smp_store_cpu_info(cpuid);
+    smp_store_cpu_info(smp_processor_id());
 
     /* Allow the master to continue. */
-    cpu_set(cpuid, cpu_callin_map);
+    set_cpu_state(CPU_STATE_CALLIN);
+
+    /* And wait for our final Ack. */
+    while ( cpu_state != CPU_STATE_ONLINE )
+        cpu_relax();
 }
 
 static int booting_cpu;
@@ -316,8 +282,6 @@ void start_secondary(void *unused)
     cpu_init();
 
     smp_callin();
-    while (!cpu_isset(smp_processor_id(), smp_commenced_mask))
-        cpu_relax();
 
     /*
      * At this point, boot CPU has fully initialised the IDT. It is
@@ -348,8 +312,6 @@ void start_secondary(void *unused)
     cpu_set(smp_processor_id(), cpu_online_map);
     unlock_vector_lock();
 
-    per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
-
     init_percpu_time();
 
     /* We can take interrupts now: we're officially "up". */
@@ -469,8 +431,6 @@ static int wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
             send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
     } while ( send_status && (timeout++ < 1000) );
 
-    atomic_set(&init_deasserted, 1);
-
     /*
      * Should we send STARTUP IPIs ?
      *
@@ -570,7 +530,7 @@ static int do_boot_cpu(int apicid, int cpu)
 
     /* This grunge runs the startup process for the targeted processor. */
 
-    atomic_set(&init_deasserted, 0);
+    set_cpu_state(CPU_STATE_INIT);
 
     Dprintk("Setting warm reset code and vector.\n");
 
@@ -582,19 +542,18 @@ static int do_boot_cpu(int apicid, int cpu)
     if ( !boot_error )
     {
         /* Allow AP to start initializing. */
-        Dprintk("Before Callout %d.\n", cpu);
-        cpu_set(cpu, cpu_callout_map);
+        set_cpu_state(CPU_STATE_CALLOUT);
         Dprintk("After Callout %d.\n", cpu);
 
         /* Wait 5s total for a response. */
         for ( timeout = 0; timeout < 50000; timeout++ )
         {
-            if ( cpu_isset(cpu, cpu_callin_map) )
+            if ( cpu_state == CPU_STATE_CALLIN )
                 break; /* It has booted */
             udelay(100);
         }
 
-        if ( cpu_isset(cpu, cpu_callin_map) )
+        if ( cpu_state == CPU_STATE_CALLIN )
         {
             /* number CPUs logically, starting from 1 (BSP is 0) */
             Dprintk("OK.\n");
@@ -630,12 +589,8 @@ static int do_boot_cpu(int apicid, int cpu)
 void cpu_exit_clear(unsigned int cpu)
 {
     cpu_uninit(cpu);
-
-    cpu_clear(cpu, cpu_callout_map);
-    cpu_clear(cpu, cpu_callin_map);
-
-    cpu_clear(cpu, smp_commenced_mask);
     unmap_cpu_to_logical_apicid(cpu);
+    set_cpu_state(CPU_STATE_DEAD);
 }
 
 static void cpu_smpboot_free(unsigned int cpu)
@@ -828,12 +783,8 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
 
 void __init smp_prepare_boot_cpu(void)
 {
-    cpu_set(smp_processor_id(), smp_commenced_mask);
-    cpu_set(smp_processor_id(), cpu_callin_map);
     cpu_set(smp_processor_id(), cpu_online_map);
-    cpu_set(smp_processor_id(), cpu_callout_map);
     cpu_set(smp_processor_id(), cpu_present_map);
-    per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
 }
 
 static void
@@ -888,7 +839,7 @@ void __cpu_die(unsigned int cpu)
     /* We don't do anything here: idle task is faking death itself. */
     unsigned int i = 0;
 
-    while ( per_cpu(cpu_state, cpu) != CPU_DEAD )
+    while ( cpu_state != CPU_STATE_DEAD )
     {
         mdelay(100);
         cpu_relax();
@@ -957,15 +908,13 @@ int __cpu_up(unsigned int cpu)
 {
     int apicid, ret;
 
-    BUG_ON(cpu_isset(cpu, cpu_callin_map));
-
     if ( (apicid = x86_cpu_to_apicid[cpu]) == BAD_APICID )
         return -ENODEV;
 
     if ( (ret = do_boot_cpu(apicid, cpu)) != 0 )
         return ret;
 
-    cpu_set(cpu, smp_commenced_mask);
+    set_cpu_state(CPU_STATE_ONLINE);
     while ( !cpu_isset(cpu, cpu_online_map) )
     {
         cpu_relax();
diff --git a/xen/include/asm-x86/mach-default/mach_wakecpu.h b/xen/include/asm-x86/mach-default/mach_wakecpu.h
index 0e94bc308a..22ce9c0268 100644
--- a/xen/include/asm-x86/mach-default/mach_wakecpu.h
+++ b/xen/include/asm-x86/mach-default/mach_wakecpu.h
@@ -13,17 +13,6 @@
 
 #define boot_cpu_apicid boot_cpu_physical_apicid
 
-static inline void wait_for_init_deassert(atomic_t *deassert)
-{
-	while (!atomic_read(deassert));
-	return;
-}
-
-/* Nothing to do for most platforms, since cleared by the INIT cycle */
-static inline void smp_callin_clear_local_apic(void)
-{
-}
-
 #if APIC_DEBUG
  #define inquire_remote_apic(apicid) __inquire_remote_apic(apicid)
 #else
diff --git a/xen/include/asm-x86/smp.h b/xen/include/asm-x86/smp.h
index 60f516c640..d2a4334ae6 100644
--- a/xen/include/asm-x86/smp.h
+++ b/xen/include/asm-x86/smp.h
@@ -52,9 +52,6 @@ extern u32 cpu_2_logical_apicid[];
 
 #define cpu_physical_id(cpu)	x86_cpu_to_apicid[cpu]
 
-/* State of each CPU. */
-DECLARE_PER_CPU(int, cpu_state);
-
 #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
 extern void cpu_exit_clear(unsigned int cpu);
 extern void cpu_uninit(unsigned int cpu);
@@ -67,15 +64,6 @@ int cpu_add(uint32_t apic_id, uint32_t acpi_id, uint32_t pxm);
  */
 #define raw_smp_processor_id() (get_processor_id())
 
-extern cpumask_t cpu_callout_map;
-extern cpumask_t cpu_callin_map;
-
-/* We don't mark CPUs online until __cpu_up(), so we need another measure */
-static inline int num_booting_cpus(void)
-{
-	return cpus_weight(cpu_callout_map);
-}
-
 #ifdef CONFIG_X86_LOCAL_APIC
 
 static inline int hard_smp_processor_id(void)