xen: arm: rewrite start of day page table and cpu bring up

This is unfortunately a rather large monolithic patch.

Rather than bringing up all CPUs in lockstep as we setup paging and relocate
Xen instead create a simplified set of dedicated boot time pagetables.

This allows secondary CPUs to remain powered down or in the firmware until we
actually want to enable them. The bringup is now done later on in C and can be
driven by DT etc. I have included code for the vexpress platform, but other
platforms will need to be added.

The mechanism for deciding how to bring up a CPU differs between arm32 and
arm64. On arm32 it is essentially a per-platform property, with the exception
of PSCI which can be implemented globally (but isn't here). On arm64 there is a
per-cpu property in the device tree.

Secondary CPUs are brought up directly into the relocated Xen image, instead of
relying on being able to launch on the unrelocated Xen and hoping that it
hasn't been clobbered.

As part of this change drop support for switching from secure mode to NS HYP as
well as the early CPU kick. Xen now requires that it is launched in NS HYP
mode and that firmware configure things such that secondary CPUs can be woken
up by a primarly CPU in HYP mode. This may require fixes to bootloaders or the
use of a boot wrapper.

The changes done here (re)exposed an issue with relocating Xen and the compiler
spilling values to the stack between the copy and the actual switch to the
relocaed copy of Xen in setup_pagetables. Therefore switch to doing the copy
and switch in a single asm function where we can control precisely what gets
spilled to the stack etc.

Since we now have a separate set of boot pagetables it is much easier to build
the real Xen pagetables inplace before relocating rather than the more complex
approach of rewriting the pagetables in the relocated copy before switching.

This will also enable Xen to be loaded above the 4GB boundary on 64-bit.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Tim Deegan <tim@xen.org>
Acked-by: Julien Grall <julien.grall@linaro.org>

1 files changed, 23 insertions, 7 deletions

diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index d615b4a401..b2c4101554 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -592,7 +592,7 @@ void __init setup_cache(void)
  * MPIDR values related to logical cpus
  * Code base on Linux arch/arm/kernel/devtree.c
  */
-static void __init init_cpus_maps(void)
+static void __init smp_init_cpus(void)
 {
     register_t mpidr;
     struct dt_device_node *cpus = dt_find_node_by_path("/cpus");
@@ -604,6 +604,14 @@ static void __init init_cpus_maps(void)
         [0 ... NR_CPUS - 1] = MPIDR_INVALID
     };
     bool_t bootcpu_valid = 0;
+    int rc;
+
+    if ( (rc = arch_smp_init()) < 0 )
+    {
+        printk(XENLOG_WARNING "SMP init failed (%d)\n"
+               "Using only 1 CPU\n", rc);
+        return;
+    }
 
     mpidr = boot_cpu_data.mpidr.bits & MPIDR_HWID_MASK;
 
@@ -673,13 +681,20 @@ static void __init init_cpus_maps(void)
 
         if ( cpuidx > NR_CPUS )
         {
-            printk(XENLOG_WARNING "DT /cpu %u node greater than max cores %u, capping them\n",
+            printk(XENLOG_WARNING
+                   "DT /cpu %u node greater than max cores %u, capping them\n",
                    cpuidx, NR_CPUS);
             cpuidx = NR_CPUS;
             break;
         }
 
-        tmp_map[i] = hwid;
+        if ( (rc = arch_cpu_init(i, cpu)) < 0 )
+        {
+            printk("cpu%d init failed (hwid %x): %d\n", i, hwid, rc);
+            tmp_map[i] = MPIDR_INVALID;
+        }
+        else
+            tmp_map[i] = hwid;
     }
 
     if ( !bootcpu_valid )
@@ -691,6 +706,8 @@ static void __init init_cpus_maps(void)
 
     for ( i = 0; i < cpuidx; i++ )
     {
+        if ( tmp_map[i] == MPIDR_INVALID )
+            continue;
         cpumask_set_cpu(i, &cpu_possible_map);
         cpu_logical_map(i) = tmp_map[i];
     }
@@ -732,15 +749,14 @@ void __init start_xen(unsigned long boot_phys_offset,
 
     processor_id();
 
-    init_cpus_maps();
-    cpus = smp_get_max_cpus();
-
     platform_init();
 
+    smp_init_cpus();
+    cpus = smp_get_max_cpus();
+
     init_xen_time();
 
     gic_init();
-    make_cpus_ready(cpus, boot_phys_offset);
 
     set_current((struct vcpu *)0xfffff000); /* debug sanity */
     idle_vcpu[0] = current;