allocate CPU sibling and core maps dynamically

... thus reducing the per-CPU data area size back to one page even when
building for large NR_CPUS.

At once eliminate the old __cpu{mask,list}_scnprintf() helpers.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
Acked-by: Keir Fraser <keir@xen.org>

10 files changed, 70 insertions, 60 deletions

diff --git a/xen/arch/ia64/linux-xen/setup.c b/xen/arch/ia64/linux-xen/setup.c
index b767a500e3..24169e46a2 100644
--- a/xen/arch/ia64/linux-xen/setup.c
+++ b/xen/arch/ia64/linux-xen/setup.c
@@ -577,8 +577,12 @@ late_setup_arch (char **cmdline_p)
 
 	cpu_physical_id(0) = hard_smp_processor_id();
 
-	cpu_set(0, per_cpu(cpu_sibling_map, 0));
-	cpu_set(0, per_cpu(cpu_core_map, 0));
+	if (!zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, 0)) ||
+            !zalloc_cpumask_var(&per_cpu(cpu_core_mask, 0)))
+		panic("No memory for boot CPU sibling/core maps\n");
+
+	cpumask_set_cpu(0, per_cpu(cpu_sibling_mask, 0));
+	cpumask_set_cpu(0, per_cpu(cpu_core_mask, 0));
 
 	check_for_logical_procs();
 	if (smp_num_cpucores > 1)
diff --git a/xen/arch/ia64/linux-xen/smpboot.c b/xen/arch/ia64/linux-xen/smpboot.c
index 66d656eeba..64c28f7764 100644
--- a/xen/arch/ia64/linux-xen/smpboot.c
+++ b/xen/arch/ia64/linux-xen/smpboot.c
@@ -144,8 +144,8 @@ EXPORT_SYMBOL(cpu_online_map);
 cpumask_t cpu_possible_map;
 EXPORT_SYMBOL(cpu_possible_map);
 
-DEFINE_PER_CPU_READ_MOSTLY(cpumask_t, cpu_core_map);
-DEFINE_PER_CPU_READ_MOSTLY(cpumask_t, cpu_sibling_map);
+DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_mask);
+DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_mask);
 int smp_num_siblings = 1;
 int smp_num_cpucores = 1;
 
@@ -687,13 +687,13 @@ clear_cpu_sibling_map(int cpu)
 {
 	int i;
 
-	for_each_cpu_mask(i, per_cpu(cpu_sibling_map, cpu))
-		cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, i));
-	for_each_cpu_mask(i, per_cpu(cpu_core_map, cpu))
-		cpumask_clear_cpu(cpu, &per_cpu(cpu_core_map, i));
+	for_each_cpu_mask(i, *per_cpu(cpu_sibling_mask, cpu))
+		cpumask_clear_cpu(cpu, per_cpu(cpu_sibling_mask, i));
+	for_each_cpu_mask(i, *per_cpu(cpu_core_mask, cpu))
+		cpumask_clear_cpu(cpu, per_cpu(cpu_core_mask, i));
 
-	cpumask_clear(&per_cpu(cpu_sibling_map, cpu));
-	cpumask_clear(&per_cpu(cpu_core_map, cpu));
+	cpumask_clear(per_cpu(cpu_sibling_mask, cpu));
+	cpumask_clear(per_cpu(cpu_core_mask, cpu));
 }
 
 static void
@@ -703,12 +703,12 @@ remove_siblinginfo(int cpu)
 
 	if (cpu_data(cpu)->threads_per_core == 1 &&
 	    cpu_data(cpu)->cores_per_socket == 1) {
-		cpu_clear(cpu, per_cpu(cpu_core_map, cpu));
-		cpu_clear(cpu, per_cpu(cpu_sibling_map, cpu));
+		cpumask_clear_cpu(cpu, per_cpu(cpu_core_mask, cpu));
+		cpumask_clear_cpu(cpu, per_cpu(cpu_sibling_mask, cpu));
 		return;
 	}
 
-	last = (cpus_weight(per_cpu(cpu_core_map, cpu)) == 1);
+	last = (cpumask_weight(per_cpu(cpu_core_mask, cpu)) == 1);
 
 	/* remove it from all sibling map's */
 	clear_cpu_sibling_map(cpu);
@@ -794,11 +794,11 @@ set_cpu_sibling_map(int cpu)
 
 	for_each_online_cpu(i) {
 		if ((cpu_data(cpu)->socket_id == cpu_data(i)->socket_id)) {
-			cpu_set(i, per_cpu(cpu_core_map, cpu));
-			cpu_set(cpu, per_cpu(cpu_core_map, i));
+			cpumask_set_cpu(i, per_cpu(cpu_core_mask, cpu));
+			cpumask_set_cpu(cpu, per_cpu(cpu_core_mask, i));
 			if (cpu_data(cpu)->core_id == cpu_data(i)->core_id) {
-				cpu_set(i, per_cpu(cpu_sibling_map, cpu));
-				cpu_set(cpu, per_cpu(cpu_sibling_map, i));
+				cpumask_set_cpu(i, per_cpu(cpu_sibling_mask, cpu));
+				cpumask_set_cpu(cpu, per_cpu(cpu_sibling_mask, i));
 			}
 		}
 	}
@@ -821,6 +821,14 @@ __cpu_up (unsigned int cpu)
 	if (cpu_isset(cpu, cpu_callin_map))
 		return -EINVAL;
 
+	if (!per_cpu(cpu_sibling_mask, cpu) &&
+            !zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, cpu)))
+		return -ENOMEM;
+
+	if (!per_cpu(cpu_core_mask, cpu) &&
+            !zalloc_cpumask_var(&per_cpu(cpu_core_mask, cpu)))
+		return -ENOMEM;
+
 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
 	/* Processor goes to start_secondary(), sets online flag */
 	ret = do_boot_cpu(sapicid, cpu);
@@ -829,8 +837,8 @@ __cpu_up (unsigned int cpu)
 
 	if (cpu_data(cpu)->threads_per_core == 1 &&
 	    cpu_data(cpu)->cores_per_socket == 1) {
-		cpu_set(cpu, per_cpu(cpu_sibling_map, cpu));
-		cpu_set(cpu, per_cpu(cpu_core_map, cpu));
+		cpumask_set_cpu(cpu, per_cpu(cpu_sibling_mask, cpu));
+		cpumask_set_cpu(cpu, per_cpu(cpu_core_mask, cpu));
 		return 0;
 	}
 
diff --git a/xen/arch/ia64/xen/dom0_ops.c b/xen/arch/ia64/xen/dom0_ops.c
index 0cfa39f613..65dab551e3 100644
--- a/xen/arch/ia64/xen/dom0_ops.c
+++ b/xen/arch/ia64/xen/dom0_ops.c
@@ -594,9 +594,9 @@ long arch_do_sysctl(xen_sysctl_t *op, XEN_GUEST_HANDLE(xen_sysctl_t) u_sysctl)
         xen_sysctl_physinfo_t *pi = &op->u.physinfo;
 
         memset(pi, 0, sizeof(*pi));
-        pi->threads_per_core = cpus_weight(per_cpu(cpu_sibling_map, 0));
+        pi->threads_per_core = cpumask_weight(per_cpu(cpu_sibling_mask, 0));
         pi->cores_per_socket =
-            cpus_weight(per_cpu(cpu_core_map, 0)) / pi->threads_per_core;
+            cpumask_weight(per_cpu(cpu_core_mask, 0)) / pi->threads_per_core;
         pi->nr_nodes         = (u32)num_online_nodes();
         pi->nr_cpus          = (u32)num_online_cpus();
         pi->total_pages      = total_pages; 
diff --git a/xen/arch/ia64/xen/tlb_track.c b/xen/arch/ia64/xen/tlb_track.c
index ec7ee64ada..55c11885e6 100644
--- a/xen/arch/ia64/xen/tlb_track.c
+++ b/xen/arch/ia64/xen/tlb_track.c
@@ -504,7 +504,7 @@ __tlb_track_entry_printf(const char* func, int line,
     char pcpumask_buf[NR_CPUS + 1];
     char vcpumask_buf[MAX_VIRT_CPUS + 1];
     cpumask_scnprintf(pcpumask_buf, sizeof(pcpumask_buf),
-                      entry->pcpu_dirty_mask);
+                      &entry->pcpu_dirty_mask);
     vcpumask_scnprintf(vcpumask_buf, sizeof(vcpumask_buf),
                        entry->vcpu_dirty_mask);
     printk("%s:%d\n"
diff --git a/xen/arch/x86/cpu/mcheck/mce_intel.c b/xen/arch/x86/cpu/mcheck/mce_intel.c
index 86191ac2d5..f5648754bd 100644
--- a/xen/arch/x86/cpu/mcheck/mce_intel.c
+++ b/xen/arch/x86/cpu/mcheck/mce_intel.c
@@ -867,7 +867,7 @@ static void intel_machine_check(struct cpu_user_regs * regs, long error_code)
         {
             char *ebufp, ebuf[96] = "MCE: Fatal error happened on CPUs ";
             ebufp = ebuf + strlen(ebuf);
-            cpumask_scnprintf(ebufp, 95 - strlen(ebuf), mce_fatal_cpus);
+            cpumask_scnprintf(ebufp, 95 - strlen(ebuf), &mce_fatal_cpus);
             mc_panic(ebuf);
         }
         atomic_set(&found_error, 0);
diff --git a/xen/arch/x86/irq.c b/xen/arch/x86/irq.c
index c4b77a0817..d480b10ad1 100644
--- a/xen/arch/x86/irq.c
+++ b/xen/arch/x86/irq.c
@@ -1998,7 +1998,7 @@ static void dump_irqs(unsigned char key)
         spin_lock_irqsave(&desc->lock, flags);
 
         cpumask_scnprintf(keyhandler_scratch, sizeof(keyhandler_scratch),
-                          desc->affinity);
+                          &desc->affinity);
         printk("   IRQ:%4d affinity:%s vec:%02x type=%-15s"
                " status=%08x ",
                irq, keyhandler_scratch, desc->arch.vector,
diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
index f5231009a4..acc1f343e8 100644
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -3201,7 +3201,7 @@ int do_mmuext_op(
                 cpumask_clear(&mask);
                 for_each_online_cpu(cpu)
                     if ( !cpumask_intersects(&mask,
-                                             &per_cpu(cpu_sibling_map, cpu)) )
+                                             per_cpu(cpu_sibling_mask, cpu)) )
                         cpumask_set_cpu(cpu, &mask);
                 flush_mask(&mask, FLUSH_CACHE);
             }
diff --git a/xen/arch/x86/oprofile/op_model_p4.c b/xen/arch/x86/oprofile/op_model_p4.c
index bb6f3d70fc..fabff9aa0e 100644
--- a/xen/arch/x86/oprofile/op_model_p4.c
+++ b/xen/arch/x86/oprofile/op_model_p4.c
@@ -385,7 +385,7 @@ static unsigned int get_stagger(void)
 {
 #ifdef CONFIG_SMP
 	int cpu = smp_processor_id();
-	return (cpu != first_cpu(per_cpu(cpu_sibling_map, cpu)));
+	return (cpu != cpumask_first(per_cpu(cpu_sibling_mask, cpu)));
 #endif	
 	return 0;
 }
diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index a01698beb2..e9c613cdcc 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -51,9 +51,9 @@
 unsigned long __read_mostly trampoline_phys;
 
 /* representing HT siblings of each logical CPU */
-DEFINE_PER_CPU_READ_MOSTLY(cpumask_t, cpu_sibling_map);
+DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_mask);
 /* representing HT and core siblings of each logical CPU */
-DEFINE_PER_CPU_READ_MOSTLY(cpumask_t, cpu_core_map);
+DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_mask);
 
 cpumask_t cpu_online_map __read_mostly;
 EXPORT_SYMBOL(cpu_online_map);
@@ -233,10 +233,10 @@ static cpumask_t cpu_sibling_setup_map;
 
 static void link_thread_siblings(int cpu1, int cpu2)
 {
-    cpu_set(cpu1, per_cpu(cpu_sibling_map, cpu2));
-    cpu_set(cpu2, per_cpu(cpu_sibling_map, cpu1));
-    cpu_set(cpu1, per_cpu(cpu_core_map, cpu2));
-    cpu_set(cpu2, per_cpu(cpu_core_map, cpu1));
+    cpumask_set_cpu(cpu1, per_cpu(cpu_sibling_mask, cpu2));
+    cpumask_set_cpu(cpu2, per_cpu(cpu_sibling_mask, cpu1));
+    cpumask_set_cpu(cpu1, per_cpu(cpu_core_mask, cpu2));
+    cpumask_set_cpu(cpu2, per_cpu(cpu_core_mask, cpu1));
 }
 
 static void set_cpu_sibling_map(int cpu)
@@ -262,13 +262,13 @@ static void set_cpu_sibling_map(int cpu)
     }
     else
     {
-        cpu_set(cpu, per_cpu(cpu_sibling_map, cpu));
+        cpumask_set_cpu(cpu, per_cpu(cpu_sibling_mask, cpu));
     }
 
     if ( c[cpu].x86_max_cores == 1 )
     {
-        cpumask_copy(&per_cpu(cpu_core_map, cpu),
-                     &per_cpu(cpu_sibling_map, cpu));
+        cpumask_copy(per_cpu(cpu_core_mask, cpu),
+                     per_cpu(cpu_sibling_mask, cpu));
         c[cpu].booted_cores = 1;
         return;
     }
@@ -277,18 +277,18 @@ static void set_cpu_sibling_map(int cpu)
     {
         if ( c[cpu].phys_proc_id == c[i].phys_proc_id )
         {
-            cpu_set(i, per_cpu(cpu_core_map, cpu));
-            cpu_set(cpu, per_cpu(cpu_core_map, i));
+            cpumask_set_cpu(i, per_cpu(cpu_core_mask, cpu));
+            cpumask_set_cpu(cpu, per_cpu(cpu_core_mask, i));
             /*
              *  Does this new cpu bringup a new core?
              */
-            if ( cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1 )
+            if ( cpumask_weight(per_cpu(cpu_sibling_mask, cpu)) == 1 )
             {
                 /*
                  * for each core in package, increment
                  * the booted_cores for this new cpu
                  */
-                if ( first_cpu(per_cpu(cpu_sibling_map, i)) == i )
+                if ( cpumask_first(per_cpu(cpu_sibling_mask, i)) == i )
                     c[cpu].booted_cores++;
                 /*
                  * increment the core count for all
@@ -641,13 +641,14 @@ static void cpu_smpboot_free(unsigned int cpu)
 {
     unsigned int order;
 
+    free_cpumask_var(per_cpu(cpu_sibling_mask, cpu));
+    free_cpumask_var(per_cpu(cpu_core_mask, cpu));
+
     order = get_order_from_pages(NR_RESERVED_GDT_PAGES);
     free_xenheap_pages(per_cpu(gdt_table, cpu), order);
-    per_cpu(gdt_table, cpu) = NULL;
 
 #ifdef __x86_64__
     free_xenheap_pages(per_cpu(compat_gdt_table, cpu), order);
-    per_cpu(compat_gdt_table, cpu) = NULL;
 #endif
 
     order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t));
@@ -696,7 +697,9 @@ static int cpu_smpboot_alloc(unsigned int cpu)
         goto oom;
     memcpy(idt_tables[cpu], idt_table, IDT_ENTRIES * sizeof(idt_entry_t));
 
-    return 0;
+    if ( zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, cpu)) &&
+         zalloc_cpumask_var(&per_cpu(cpu_core_mask, cpu)) )
+        return 0;
 
  oom:
     cpu_smpboot_free(cpu);
@@ -744,6 +747,10 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
 
     stack_base[0] = stack_start.esp;
 
+    if ( !zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, 0)) ||
+         !zalloc_cpumask_var(&per_cpu(cpu_core_mask, 0)) )
+        panic("No memory for boot CPU sibling/core maps\n");
+
     set_cpu_sibling_map(0);
 
     /*
@@ -760,8 +767,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
             printk(KERN_NOTICE "Local APIC not detected."
                    " Using dummy APIC emulation.\n");
         map_cpu_to_logical_apicid();
-        cpu_set(0, per_cpu(cpu_sibling_map, 0));
-        cpu_set(0, per_cpu(cpu_core_map, 0));
         return;
     }
 
@@ -792,13 +797,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
     setup_local_APIC();
     map_cpu_to_logical_apicid();
 
-    /*
-     * construct cpu_sibling_map, so that we can tell sibling CPUs
-     * efficiently.
-     */
-    cpu_set(0, per_cpu(cpu_sibling_map, 0));
-    cpu_set(0, per_cpu(cpu_core_map, 0));
-
     smpboot_setup_io_apic();
 
     setup_boot_APIC_clock();
@@ -816,18 +814,18 @@ remove_siblinginfo(int cpu)
     int sibling;
     struct cpuinfo_x86 *c = cpu_data;
 
-    for_each_cpu_mask ( sibling, per_cpu(cpu_core_map, cpu) )
+    for_each_cpu_mask ( sibling, *per_cpu(cpu_core_mask, cpu) )
     {
-        cpu_clear(cpu, per_cpu(cpu_core_map, sibling));
+        cpumask_clear_cpu(cpu, per_cpu(cpu_core_mask, sibling));
         /* Last thread sibling in this cpu core going down. */
-        if ( cpumask_weight(&per_cpu(cpu_sibling_map, cpu)) == 1 )
+        if ( cpumask_weight(per_cpu(cpu_sibling_mask, cpu)) == 1 )
             c[sibling].booted_cores--;
     }
    
-    for_each_cpu_mask(sibling, per_cpu(cpu_sibling_map, cpu))
-        cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, sibling));
-    cpumask_clear(&per_cpu(cpu_sibling_map, cpu));
-    cpumask_clear(&per_cpu(cpu_core_map, cpu));
+    for_each_cpu_mask(sibling, *per_cpu(cpu_sibling_mask, cpu))
+        cpumask_clear_cpu(cpu, per_cpu(cpu_sibling_mask, sibling));
+    cpumask_clear(per_cpu(cpu_sibling_mask, cpu));
+    cpumask_clear(per_cpu(cpu_core_mask, cpu));
     c[cpu].phys_proc_id = BAD_APICID;
     c[cpu].cpu_core_id = BAD_APICID;
     c[cpu].compute_unit_id = BAD_APICID;
diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c
index cd2f13cf09..723d623ca5 100644
--- a/xen/arch/x86/sysctl.c
+++ b/xen/arch/x86/sysctl.c
@@ -76,9 +76,9 @@ long arch_do_sysctl(
 
         memset(pi, 0, sizeof(*pi));
         pi->threads_per_core =
-            cpus_weight(per_cpu(cpu_sibling_map, 0));
+            cpumask_weight(per_cpu(cpu_sibling_mask, 0));
         pi->cores_per_socket =
-            cpus_weight(per_cpu(cpu_core_map, 0)) / pi->threads_per_core;
+            cpumask_weight(per_cpu(cpu_core_mask, 0)) / pi->threads_per_core;
         pi->nr_cpus = num_online_cpus();
         pi->nr_nodes = num_online_nodes();
         pi->max_node_id = MAX_NUMNODES-1;