numa: Correct handling node with CPU populated but no memory populated

In changeset 20599, the node that has no memory populated is marked
parsed, but not online. However, if there are CPU populated in this
node, the corresponding CPU mapping (i.e. the cpu_to_node) is still
setup to the offline node, this will cause trouble for memory
allocation.

This patch changes the init_cpu_to_node() and srant_detect_node(), to
considering the node is offlined situation.

Now the apicid_to_node is only used to keep the mapping between
cpu/node provided by BIOS, and should not be used for memory
allocation anymore.

One thing left is to update the cpu_to_node mapping after memory
populated by memory hot-add.

Signed-off-by: Jiang, Yunhong <yunhong.jiang@intel.com>

This is a reintroduction of 20726:ddb8c5e798f9, which I incorrectly
reverted in 20745:d3215a968db9

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>

4 files changed, 12 insertions, 7 deletions

diff --git a/xen/arch/x86/numa.c b/xen/arch/x86/numa.c
index 676ff37001..6810b484ca 100644
--- a/xen/arch/x86/numa.c
+++ b/xen/arch/x86/numa.c
@@ -35,6 +35,9 @@ u8  memnodemap[NODEMAPSIZE];
 unsigned char cpu_to_node[NR_CPUS] __read_mostly = {
 	[0 ... NR_CPUS-1] = NUMA_NO_NODE
 };
+/*
+ * Keep BIOS's CPU2node information, should not be used for memory allocaion
+ */
 unsigned char apicid_to_node[MAX_LOCAL_APIC] __cpuinitdata = {
  	[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
 };
@@ -288,14 +291,15 @@ static __init int numa_setup(char *opt)
  */
 void __devinit init_cpu_to_node(void)
 {
-	int i;
+	int i, node;
  	for (i = 0; i < NR_CPUS; i++) {
 		u32 apicid = x86_cpu_to_apicid[i];
 		if (apicid == BAD_APICID)
 			continue;
-		if (apicid_to_node[apicid] == NUMA_NO_NODE)
-			continue;
-		numa_set_node(i,apicid_to_node[apicid]);
+		node = apicid_to_node[apicid];
+		if ( node == NUMA_NO_NODE || !node_online(node) )
+			node = 0;
+		numa_set_node(i, node);
 	}
 }
 
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 5956dfcc05..38a0f5cd39 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -20,6 +20,7 @@
 #include <xen/rcupdate.h>
 #include <xen/vga.h>
 #include <xen/dmi.h>
+#include <xen/nodemask.h>
 #include <public/version.h>
 #ifdef CONFIG_COMPAT
 #include <compat/platform.h>
@@ -263,7 +264,7 @@ void __devinit srat_detect_node(int cpu)
     u32 apicid = x86_cpu_to_apicid[cpu];
 
     node = apicid_to_node[apicid];
-    if ( node == NUMA_NO_NODE )
+    if ( node == NUMA_NO_NODE || !node_online(node) )
         node = 0;
     numa_set_node(cpu, node);
 
diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index 5220d5173a..ce18c7f0be 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -913,7 +913,7 @@ static int __devinit do_boot_cpu(int apicid, int cpu)
 	}
 #else
 	if (!per_cpu(compat_arg_xlat, cpu))
-		setup_compat_arg_xlat(cpu, apicid_to_node[apicid]);
+		setup_compat_arg_xlat(cpu, cpu_to_node[cpu]);
 #endif
 
 	if (!idt_tables[cpu]) {
diff --git a/xen/arch/x86/x86_64/mm.c b/xen/arch/x86/x86_64/mm.c
index dfa12ad171..03b61cc34f 100644
--- a/xen/arch/x86/x86_64/mm.c
+++ b/xen/arch/x86/x86_64/mm.c
@@ -997,7 +997,7 @@ void __init subarch_init_memory(void)
     }
 
     if ( setup_compat_arg_xlat(smp_processor_id(),
-                               apicid_to_node[boot_cpu_physical_apicid]) )
+                               cpu_to_node[0]) )
         panic("Could not setup argument translation area");
 }