Mark CPU present when it is detected

Currently a CPU is marked as present only after it has been kicked off
successfully, i.e. before the CPU is brought up, it is not
present. This patch try to mark CPU as present when it is detected
(either through MPS table or ACPI). If it can't be brought up
successfully, it will be marked as non-present again.  This change is
mainly for CPU hot-plug. As discussed, we'd take two step for physical
CPU hot-add. A CPU is firstly marked as present, and later will bring
as online.

Also, In smp_boot_cpus(), xen need only scan all present CPU, and no
need to loop from 0... NR_CPUS. With this change, the bios_cpu_apicid
is not needed anymore.

Signed-off-by: Jiang, Yunhong <yunhong.jiang@intel.com>

5 files changed, 48 insertions, 26 deletions

diff --git a/xen/arch/x86/mpparse.c b/xen/arch/x86/mpparse.c
index bd94588aa5..7d65c2abaf 100644
--- a/xen/arch/x86/mpparse.c
+++ b/xen/arch/x86/mpparse.c
@@ -71,8 +71,6 @@ static unsigned int __devinitdata num_processors;
 /* Bitmask of physically existing CPUs */
 physid_mask_t phys_cpu_present_map;
 
-u32 bios_cpu_apicid[NR_CPUS] = { [0 ... NR_CPUS-1] = BAD_APICID };
-
 /*
  * Intel MP BIOS table parsing routines:
  */
@@ -101,13 +99,14 @@ static int __init mpf_checksum(unsigned char *mp, int len)
 static int mpc_record; 
 static struct mpc_config_translation *translation_table[MAX_MPC_ENTRY] __initdata;
 
-static void __devinit MP_processor_info (struct mpc_config_processor *m)
+/* Return xen's logical cpu_id of the new added cpu or <0 if error */
+static int __devinit MP_processor_info (struct mpc_config_processor *m)
 {
- 	int ver, apicid;
+ 	int ver, apicid, cpu = 0;
 	physid_mask_t phys_cpu;
  	
 	if (!(m->mpc_cpuflag & CPU_ENABLED))
-		return;
+		return -EINVAL;
 
 	apicid = mpc_apic_id(m, translation_table[mpc_record]);
 
@@ -183,15 +182,27 @@ static void __devinit MP_processor_info (struct mpc_config_processor *m)
 	if (num_processors >= NR_CPUS) {
 		printk(KERN_WARNING "WARNING: NR_CPUS limit of %i reached."
 			"  Processor ignored.\n", NR_CPUS);
-		return;
+		return -ENOSPC;
 	}
 
 	if (num_processors >= maxcpus) {
 		printk(KERN_WARNING "WARNING: maxcpus limit of %i reached."
 			" Processor ignored.\n", maxcpus);
-		return;
+		return -ENOSPC;
 	}
 
+    /* Boot cpu has been marked present in smp_prepare_boot_cpu */
+    if (!(m->mpc_cpuflag & CPU_BOOTPROCESSOR)) {
+        cpu = alloc_cpu_id();
+        if (cpu < 0) {
+            printk(KERN_WARNING "WARNING: Can't alloc cpu_id."
+              " Processor with apicid %i ignored\n", apicid);
+            return cpu;
+        }
+        x86_cpu_to_apicid[cpu] = apicid;
+        cpu_set(cpu, cpu_present_map);
+    }
+
 	cpu_set(num_processors, cpu_possible_map);
 	num_processors++;
 
@@ -202,7 +213,8 @@ static void __devinit MP_processor_info (struct mpc_config_processor *m)
 		 */
 		def_to_bigsmp = 1;
 	}
-	bios_cpu_apicid[num_processors - 1] = m->mpc_apicid;
+
+    return cpu;
 }
 
 static void __init MP_bus_info (struct mpc_config_bus *m)
@@ -826,7 +838,7 @@ void __init mp_register_lapic_address (
 }
 
 
-void __devinit mp_register_lapic (
+int __devinit mp_register_lapic (
 	u8			id, 
 	u8			enabled)
 {
@@ -836,7 +848,7 @@ void __devinit mp_register_lapic (
 	if (MAX_APICS - id <= 0) {
 		printk(KERN_WARNING "Processor #%d invalid (max %d)\n",
 			id, MAX_APICS);
-		return;
+		return -EINVAL;
 	}
 
 	if (id == boot_cpu_physical_apicid)
@@ -853,7 +865,7 @@ void __devinit mp_register_lapic (
 	processor.mpc_reserved[0] = 0;
 	processor.mpc_reserved[1] = 0;
 
-	MP_processor_info(&processor);
+    return MP_processor_info(&processor);
 }
 
 #ifdef	CONFIG_X86_IO_APIC
diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index 045426818f..5d05c1e5cc 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -800,7 +800,10 @@ wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
 #endif	/* WAKE_SECONDARY_VIA_INIT */
 
 extern cpumask_t cpu_initialized;
-static inline int alloc_cpu_id(void)
+/*
+ * Caller should hold cpu_add_remove_lock if not called when booting
+ */
+int alloc_cpu_id(void)
 {
 	cpumask_t	tmp_map;
 	int cpu;
@@ -959,9 +962,13 @@ static int __devinit do_boot_cpu(int apicid, int cpu)
 		cpu_clear(cpu, cpu_callout_map); /* was set here (do_boot_cpu()) */
 		cpu_clear(cpu, cpu_initialized); /* was set by cpu_init() */
 		cpucount--;
+
+        /* Mark the CPU as non-present */
+        spin_lock(&cpu_add_remove_lock);
+		x86_cpu_to_apicid[cpu] = BAD_APICID;
+		cpu_clear(cpu, cpu_present_map);
+        spin_unlock(&cpu_add_remove_lock);
 	} else {
-		x86_cpu_to_apicid[cpu] = apicid;
-		cpu_set(cpu, cpu_present_map);
 	}
 
 	/* mark "stuck" area as not stuck */
@@ -1028,7 +1035,7 @@ EXPORT_SYMBOL(xquad_portio);
 
 static void __init smp_boot_cpus(unsigned int max_cpus)
 {
-	int apicid, cpu, bit, kicked;
+	int apicid, cpu, kicked;
 #ifdef BOGOMIPS
 	unsigned long bogosum = 0;
 #endif
@@ -1112,8 +1119,11 @@ static void __init smp_boot_cpus(unsigned int max_cpus)
 	Dprintk("CPU present map: %lx\n", physids_coerce(phys_cpu_present_map));
 
 	kicked = 1;
-	for (bit = 0; kicked < NR_CPUS && bit < NR_CPUS; bit++) {
-		apicid = cpu_present_to_apicid(bit);
+
+    for_each_present_cpu ( cpu )
+    {
+		apicid = x86_cpu_to_apicid[cpu];
+
 		/*
 		 * Don't even attempt to start the boot CPU!
 		 */
@@ -1121,11 +1131,15 @@ static void __init smp_boot_cpus(unsigned int max_cpus)
 			continue;
 
 		if (!check_apicid_present(apicid))
+        {
+            dprintk(XENLOG_WARNING, "Present CPU has valid apicid\n");
 			continue;
+        }
+
 		if (max_cpus <= cpucount+1)
 			continue;
 
-		if (((cpu = alloc_cpu_id()) <= 0) || do_boot_cpu(apicid, cpu))
+		if ( do_boot_cpu(apicid, cpu))
 			printk("CPU #%d not responding - cannot use it.\n",
 								apicid);
 		else
diff --git a/xen/include/asm-x86/mach-generic/mach_apic.h b/xen/include/asm-x86/mach-generic/mach_apic.h
index a453a08a94..f85bb15f41 100644
--- a/xen/include/asm-x86/mach-generic/mach_apic.h
+++ b/xen/include/asm-x86/mach-generic/mach_apic.h
@@ -25,13 +25,6 @@ static inline void enable_apic_mode(void)
 #define apicid_to_node(apicid) ((int)apicid_to_node[(u32)apicid])
 
 extern u32 bios_cpu_apicid[];
-static inline int cpu_present_to_apicid(int mps_cpu)
-{
-	if (mps_cpu < NR_CPUS)
-		return (int)bios_cpu_apicid[mps_cpu];
-	else
-		return BAD_APICID;
-}
 
 static inline int mpc_apic_id(struct mpc_config_processor *m, 
 			struct mpc_config_translation *translation_record)
diff --git a/xen/include/asm-x86/mpspec.h b/xen/include/asm-x86/mpspec.h
index 72aaf95b0a..58422892b2 100644
--- a/xen/include/asm-x86/mpspec.h
+++ b/xen/include/asm-x86/mpspec.h
@@ -25,7 +25,8 @@ extern int pic_mode;
 extern int using_apic_timer;
 
 #ifdef CONFIG_ACPI
-extern void mp_register_lapic (u8 id, u8 enabled);
+extern int mp_register_lapic (u8 id, u8 enabled);
+extern void mp_unregister_lapic(uint32_t apic_id, uint32_t cpu);
 extern void mp_register_lapic_address (u64 address);
 extern void mp_register_ioapic (u8 id, u32 address, u32 gsi_base);
 extern void mp_override_legacy_irq (u8 bus_irq, u8 polarity, u8 trigger, u32 gsi);
diff --git a/xen/include/xen/smp.h b/xen/include/xen/smp.h
index 3d889ba662..6d77c5d76b 100644
--- a/xen/include/xen/smp.h
+++ b/xen/include/xen/smp.h
@@ -68,4 +68,6 @@ static inline int on_each_cpu(
 #define lock_cpu_hotplug() ((void)0)
 #define unlock_cpu_hotplug() ((void)0)
 
+int alloc_cpu_id(void);
+
 #endif /* __XEN_SMP_H__ */