xen: arm: rewrite start of day page table and cpu bring up

This is unfortunately a rather large monolithic patch.

Rather than bringing up all CPUs in lockstep as we setup paging and relocate
Xen instead create a simplified set of dedicated boot time pagetables.

This allows secondary CPUs to remain powered down or in the firmware until we
actually want to enable them. The bringup is now done later on in C and can be
driven by DT etc. I have included code for the vexpress platform, but other
platforms will need to be added.

The mechanism for deciding how to bring up a CPU differs between arm32 and
arm64. On arm32 it is essentially a per-platform property, with the exception
of PSCI which can be implemented globally (but isn't here). On arm64 there is a
per-cpu property in the device tree.

Secondary CPUs are brought up directly into the relocated Xen image, instead of
relying on being able to launch on the unrelocated Xen and hoping that it
hasn't been clobbered.

As part of this change drop support for switching from secure mode to NS HYP as
well as the early CPU kick. Xen now requires that it is launched in NS HYP
mode and that firmware configure things such that secondary CPUs can be woken
up by a primarly CPU in HYP mode. This may require fixes to bootloaders or the
use of a boot wrapper.

The changes done here (re)exposed an issue with relocating Xen and the compiler
spilling values to the stack between the copy and the actual switch to the
relocaed copy of Xen in setup_pagetables. Therefore switch to doing the copy
and switch in a single asm function where we can control precisely what gets
spilled to the stack etc.

Since we now have a separate set of boot pagetables it is much easier to build
the real Xen pagetables inplace before relocating rather than the more complex
approach of rewriting the pagetables in the relocated copy before switching.

This will also enable Xen to be loaded above the 4GB boundary on 64-bit.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Tim Deegan <tim@xen.org>
Acked-by: Julien Grall <julien.grall@linaro.org>

1 files changed, 19 insertions, 38 deletions

diff --git a/xen/arch/arm/smpboot.c b/xen/arch/arm/smpboot.c
index 234748e92d..2cb0f36683 100644
--- a/xen/arch/arm/smpboot.c
+++ b/xen/arch/arm/smpboot.c
@@ -56,12 +56,10 @@ struct init_info __initdata init_data =
 };
 
 /* Shared state for coordinating CPU bringup */
-unsigned long smp_up_cpu = 0;
+unsigned long smp_up_cpu = MPIDR_INVALID;
+/* Shared state for coordinating CPU teardown */
 static bool_t cpu_is_dead = 0;
 
-/* Number of non-boot CPUs ready to enter C */
-unsigned long __initdata ready_cpus = 0;
-
 /* ID of the PCPU we're running on */
 DEFINE_PER_CPU(unsigned int, cpu_id);
 /* XXX these seem awfully x86ish... */
@@ -103,7 +101,6 @@ smp_get_max_cpus (void)
     return max_cpus;
 }
 
-
 void __init
 smp_prepare_cpus (unsigned int max_cpus)
 {
@@ -112,32 +109,6 @@ smp_prepare_cpus (unsigned int max_cpus)
     setup_cpu_sibling_map(0);
 }
 
-void __init
-make_cpus_ready(unsigned int max_cpus, unsigned long boot_phys_offset)
-{
-    unsigned long *gate;
-    paddr_t gate_pa;
-    int i;
-
-    printk("Waiting for %i other CPUs to be ready\n", max_cpus - 1);
-    /* We use the unrelocated copy of smp_up_cpu as that's the one the
-     * others can see. */ 
-    gate_pa = ((paddr_t) (unsigned long) &smp_up_cpu) + boot_phys_offset;
-    gate = map_domain_page(gate_pa >> PAGE_SHIFT) + (gate_pa & ~PAGE_MASK); 
-    for ( i = 1; i < max_cpus; i++ )
-    {
-        /* Tell the next CPU to get ready */
-        *gate = cpu_logical_map(i);
-        flush_xen_dcache(*gate);
-        isb();
-        sev();
-        /* And wait for it to respond */
-        while ( ready_cpus < i )
-            smp_rmb();
-    }
-    unmap_domain_page(gate);
-}
-
 /* Boot the current CPU */
 void __cpuinit start_secondary(unsigned long boot_phys_offset,
                                unsigned long fdt_paddr,
@@ -176,6 +147,7 @@ void __cpuinit start_secondary(unsigned long boot_phys_offset,
     wmb();
 
     /* Now report this CPU is up */
+    smp_up_cpu = MPIDR_INVALID;
     cpumask_set_cpu(cpuid, &cpu_online_map);
     wmb();
 
@@ -226,6 +198,8 @@ int __cpu_up(unsigned int cpu)
 {
     int rc;
 
+    printk("Bringing up CPU%d\n", cpu);
+
     rc = init_secondary_pagetables(cpu);
     if ( rc < 0 )
         return rc;
@@ -236,14 +210,22 @@ int __cpu_up(unsigned int cpu)
     /* Tell the remote CPU what is it's logical CPU ID */
     init_data.cpuid = cpu;
 
-    /* Unblock the CPU.  It should be waiting in the loop in head.S
-     * for an event to arrive when smp_up_cpu matches its cpuid. */
+    /* Open the gate for this CPU */
     smp_up_cpu = cpu_logical_map(cpu);
-    /* we need to make sure that the change to smp_up_cpu is visible to
-     * secondary cpus with D-cache off */
     flush_xen_dcache(smp_up_cpu);
-    isb();
-    sev();
+
+    rc = arch_cpu_up(cpu);
+
+    if ( rc < 0 )
+    {
+        printk("Failed to bring up CPU%d\n", cpu);
+        return rc;
+    }
+
+    /* We don't know the GIC ID of the CPU until it has woken up, so just signal
+     * everyone and rely on our own smp_up_cpu gate to ensure only the one we
+     * want gets through. */
+    send_SGI_allbutself(GIC_SGI_EVENT_CHECK);
 
     while ( !cpu_online(cpu) )
     {
@@ -272,7 +254,6 @@ void __cpu_die(unsigned int cpu)
     mb();
 }
 
-
 /*
  * Local variables:
  * mode: C