xen: arm: rewrite start of day page table and cpu bring up

This is unfortunately a rather large monolithic patch.

Rather than bringing up all CPUs in lockstep as we setup paging and relocate
Xen instead create a simplified set of dedicated boot time pagetables.

This allows secondary CPUs to remain powered down or in the firmware until we
actually want to enable them. The bringup is now done later on in C and can be
driven by DT etc. I have included code for the vexpress platform, but other
platforms will need to be added.

The mechanism for deciding how to bring up a CPU differs between arm32 and
arm64. On arm32 it is essentially a per-platform property, with the exception
of PSCI which can be implemented globally (but isn't here). On arm64 there is a
per-cpu property in the device tree.

Secondary CPUs are brought up directly into the relocated Xen image, instead of
relying on being able to launch on the unrelocated Xen and hoping that it
hasn't been clobbered.

As part of this change drop support for switching from secure mode to NS HYP as
well as the early CPU kick. Xen now requires that it is launched in NS HYP
mode and that firmware configure things such that secondary CPUs can be woken
up by a primarly CPU in HYP mode. This may require fixes to bootloaders or the
use of a boot wrapper.

The changes done here (re)exposed an issue with relocating Xen and the compiler
spilling values to the stack between the copy and the actual switch to the
relocaed copy of Xen in setup_pagetables. Therefore switch to doing the copy
and switch in a single asm function where we can control precisely what gets
spilled to the stack etc.

Since we now have a separate set of boot pagetables it is much easier to build
the real Xen pagetables inplace before relocating rather than the more complex
approach of rewriting the pagetables in the relocated copy before switching.

This will also enable Xen to be loaded above the 4GB boundary on 64-bit.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Tim Deegan <tim@xen.org>
Acked-by: Julien Grall <julien.grall@linaro.org>

1 files changed, 116 insertions, 81 deletions

diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c
index 89d9f57d11..969a361eb5 100644
--- a/xen/arch/arm/mm.c
+++ b/xen/arch/arm/mm.c
@@ -43,40 +43,70 @@
 
 struct domain *dom_xen, *dom_io, *dom_cow;
 
-/* Static start-of-day pagetables that we use before the
- * allocators are up. These go on to become the boot CPU's real pagetables.
+/* Static start-of-day pagetables that we use before the allocators
+ * are up. These are used by all CPUs during bringup before switching
+ * to the CPUs own pagetables.
+ *
+ * These pagetables have a very simple structure. They include:
+ *  - a 2MB mapping of xen at XEN_VIRT_START, boot_first and
+ *    boot_second are used to populate the trie down to that mapping.
+ *  - a 1:1 mapping of xen at its current physical address. This uses a
+ *    section mapping at whichever of boot_{pgtable,first,second}
+ *    covers that physical address.
+ *
+ * For the boot CPU these mappings point to the address where Xen was
+ * loaded by the bootloader. For secondary CPUs they point to the
+ * relocated copy of Xen for the benefit of secondary CPUs.
+ *
+ * In addition to the above for the boot CPU the device-tree is
+ * initially mapped in the boot misc slot. This mapping is not present
+ * for secondary CPUs.
+ *
+ * Finally, if EARLY_PRINTK is enabled then xen_fixmap will be mapped
+ * by the CPU once it has moved off the 1:1 mapping.
  */
 lpae_t boot_pgtable[LPAE_ENTRIES] __attribute__((__aligned__(4096)));
 #ifdef CONFIG_ARM_64
 lpae_t boot_first[LPAE_ENTRIES] __attribute__((__aligned__(4096)));
-/* The first page of the first level mapping of the xenheap. The
- * subsequent xenheap first level pages are dynamically allocated, but
- * we need this one to bootstrap ourselves. */
-lpae_t xenheap_first_first[LPAE_ENTRIES] __attribute__((__aligned__(4096)));
-/* The zeroeth level slot which uses xenheap_first_first. Used because
- * setup_xenheap_mappings otherwise relies on mfn_to_virt which isn't
- * valid for a non-xenheap mapping. */
-static __initdata int xenheap_first_first_slot = -1;
 #endif
+lpae_t boot_second[LPAE_ENTRIES]  __attribute__((__aligned__(4096)));
+
+/* Main runtime page tables */
 
 /*
- * xen_pgtable and xen_dommap are per-PCPU and are allocated before
- * bringing up each CPU. On 64-bit a first level table is also allocated.
+ * For arm32 xen_pgtable and xen_dommap are per-PCPU and are allocated before
+ * bringing up each CPU. For arm64 xen_pgtable is common to all PCPUs.
  *
- * xen_second, xen_fixmap and xen_xenmap are shared between all PCPUs.
+ * xen_second, xen_fixmap and xen_xenmap are always shared between all
+ * PCPUs.
  */
 
 #ifdef CONFIG_ARM_64
-#define THIS_CPU_PGTABLE boot_pgtable
+lpae_t xen_pgtable[LPAE_ENTRIES] __attribute__((__aligned__(4096)));
+lpae_t xen_first[LPAE_ENTRIES] __attribute__((__aligned__(4096)));
+#define THIS_CPU_PGTABLE xen_pgtable
 #else
 /* Per-CPU pagetable pages */
 /* xen_pgtable == root of the trie (zeroeth level on 64-bit, first on 32-bit) */
 static DEFINE_PER_CPU(lpae_t *, xen_pgtable);
 #define THIS_CPU_PGTABLE this_cpu(xen_pgtable)
 /* xen_dommap == pages used by map_domain_page, these pages contain
- * the second level pagetables which mapp the domheap region
+ * the second level pagetables which map the domheap region
  * DOMHEAP_VIRT_START...DOMHEAP_VIRT_END in 2MB chunks. */
 static DEFINE_PER_CPU(lpae_t *, xen_dommap);
+/* Root of the trie for cpu0 */
+lpae_t cpu0_pgtable[LPAE_ENTRIES] __attribute__((__aligned__(4096)));
+#endif
+
+#ifdef CONFIG_ARM_64
+/* The first page of the first level mapping of the xenheap. The
+ * subsequent xenheap first level pages are dynamically allocated, but
+ * we need this one to bootstrap ourselves. */
+lpae_t xenheap_first_first[LPAE_ENTRIES] __attribute__((__aligned__(4096)));
+/* The zeroeth level slot which uses xenheap_first_first. Used because
+ * setup_xenheap_mappings otherwise relies on mfn_to_virt which isn't
+ * valid for a non-xenheap mapping. */
+static __initdata int xenheap_first_first_slot = -1;
 #endif
 
 /* Common pagetable leaves */
@@ -104,9 +134,8 @@ lpae_t xen_fixmap[LPAE_ENTRIES] __attribute__((__aligned__(4096)));
  * as appropriate. */
 static lpae_t xen_xenmap[LPAE_ENTRIES] __attribute__((__aligned__(4096)));
 
-
 /* Non-boot CPUs use this to find the correct pagetables. */
-uint64_t boot_ttbr;
+uint64_t init_ttbr;
 
 static paddr_t phys_offset;
 
@@ -131,6 +160,12 @@ static inline void check_memory_layout_alignment_constraints(void) {
     BUILD_BUG_ON(BOOT_RELOC_VIRT_START & ~SECOND_MASK);
     /* 1GB aligned regions */
     BUILD_BUG_ON(XENHEAP_VIRT_START & ~FIRST_MASK);
+    /* Page table structure constraints */
+#ifdef CONFIG_ARM_64
+    BUILD_BUG_ON(zeroeth_table_offset(XEN_VIRT_START));
+#endif
+    BUILD_BUG_ON(first_table_offset(XEN_VIRT_START));
+    BUILD_BUG_ON(second_linear_offset(XEN_VIRT_START) >= LPAE_ENTRIES);
 #ifdef CONFIG_DOMAIN_PAGE
     BUILD_BUG_ON(DOMHEAP_VIRT_START & ~FIRST_MASK);
 #endif
@@ -344,16 +379,6 @@ void __cpuinit setup_virt_paging(void)
     WRITE_SYSREG32(0x80002558, VTCR_EL2); isb();
 }
 
-/* This needs to be a macro to stop the compiler spilling to the stack
- * which will change when we change pagetables */
-#define WRITE_TTBR(ttbr)                                                \
-    flush_xen_text_tlb();                                               \
-    WRITE_SYSREG64(ttbr, TTBR0_EL2);                                    \
-    dsb(); /* ensure memory accesses do not cross over the TTBR0 write */ \
-    /* flush_xen_text_tlb contains an initial isb which ensures the     \
-     * write to TTBR0 has completed. */                                 \
-    flush_xen_text_tlb()
-
 static inline lpae_t pte_of_xenaddr(vaddr_t va)
 {
     paddr_t ma = va + phys_offset;
@@ -368,70 +393,77 @@ void __init remove_early_mappings(void)
     flush_xen_data_tlb_range_va(BOOT_FDT_VIRT_START, SECOND_SIZE);
 }
 
+extern void relocate_xen(uint64_t ttbr, void *src, void *dst, size_t len);
+
 /* Boot-time pagetable setup.
  * Changes here may need matching changes in head.S */
 void __init setup_pagetables(unsigned long boot_phys_offset, paddr_t xen_paddr)
 {
+    uint64_t ttbr;
     unsigned long dest_va;
     lpae_t pte, *p;
     int i;
 
-    /* Map the destination in the boot misc area. */
-    dest_va = BOOT_RELOC_VIRT_START;
-    pte = mfn_to_xen_entry(xen_paddr >> PAGE_SHIFT);
-    write_pte(xen_second + second_table_offset(dest_va), pte);
-    flush_xen_data_tlb_range_va(dest_va, SECOND_SIZE);
-
     /* Calculate virt-to-phys offset for the new location */
     phys_offset = xen_paddr - (unsigned long) _start;
 
-    /* Copy */
-    memcpy((void *) dest_va, _start, _end - _start);
-
-    /* Beware!  Any state we modify between now and the PT switch may be
-     * discarded when we switch over to the copy. */
-
-    /* Update the copy of boot_pgtable to use the new paddrs */
-    p = (void *) boot_pgtable + dest_va - (unsigned long) _start;
 #ifdef CONFIG_ARM_64
-    p[0].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT;
-    p = (void *) boot_first + dest_va - (unsigned long) _start;
+    p = (void *) xen_pgtable;
+    p[0] = pte_of_xenaddr((uintptr_t)xen_first);
+    p[0].pt.table = 1;
+    p[0].pt.xn = 0;
+    p = (void *) xen_first;
+#else
+    p = (void *) cpu0_pgtable;
 #endif
-    for ( i = 0; i < 4; i++)
-        p[i].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT;
 
-    p = (void *) xen_second + dest_va - (unsigned long) _start;
-    if ( boot_phys_offset != 0 )
+    /* Initialise first level entries, to point to second level entries */
+    for ( i = 0; i < 4; i++)
     {
-        /* Remove the old identity mapping of the boot paddr */
-        vaddr_t va = (vaddr_t)_start + boot_phys_offset;
-        p[second_linear_offset(va)].bits = 0;
+        p[i] = pte_of_xenaddr((uintptr_t)(xen_second+i*LPAE_ENTRIES));
+        p[i].pt.table = 1;
+        p[i].pt.xn = 0;
     }
-    for ( i = 0; i < 4 * LPAE_ENTRIES; i++)
-        /* The FDT is not relocated */
-        if ( p[i].pt.valid && i != second_linear_offset(BOOT_FDT_VIRT_START) )
-            p[i].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT;
 
-    /* Change pagetables to the copy in the relocated Xen */
-    boot_ttbr = (uintptr_t) boot_pgtable + phys_offset;
-    flush_xen_dcache(boot_ttbr);
-    flush_xen_dcache_va_range((void*)dest_va, _end - _start);
+    /* Initialise xen second level entries ... */
+    /* ... Xen's text etc */
 
-    WRITE_TTBR(boot_ttbr);
+    pte = mfn_to_xen_entry(xen_paddr>>PAGE_SHIFT);
+    pte.pt.xn = 0;/* Contains our text mapping! */
+    xen_second[second_table_offset(XEN_VIRT_START)] = pte;
 
-    /* Undo the temporary map */
-    pte.bits = 0;
-    write_pte(xen_second + second_table_offset(dest_va), pte);
-    flush_xen_text_tlb();
-
-    /* Link in the fixmap pagetable */
+    /* ... Fixmap */
     pte = pte_of_xenaddr((vaddr_t)xen_fixmap);
     pte.pt.table = 1;
-    write_pte(xen_second + second_table_offset(FIXMAP_ADDR(0)), pte);
-    /*
-     * No flush required here. Individual flushes are done in
-     * set_fixmap as entries are used.
-     */
+    xen_second[second_table_offset(FIXMAP_ADDR(0))] = pte;
+
+    /* ... DTB */
+    pte = boot_second[second_table_offset(BOOT_FDT_VIRT_START)];
+    xen_second[second_table_offset(BOOT_FDT_VIRT_START)] = pte;
+
+    /* Map the destination in the boot misc area. */
+    dest_va = BOOT_RELOC_VIRT_START;
+    pte = mfn_to_xen_entry(xen_paddr >> PAGE_SHIFT);
+    write_pte(boot_second + second_table_offset(dest_va), pte);
+    flush_xen_data_tlb_range_va(dest_va, SECOND_SIZE);
+#ifdef CONFIG_ARM_64
+    ttbr = (uintptr_t) xen_pgtable + phys_offset;
+#else
+    ttbr = (uintptr_t) cpu0_pgtable + phys_offset;
+#endif
+
+    relocate_xen(ttbr, _start, (void*)dest_va, _end - _start);
+
+    /* Clear the copy of the boot pagetables. Each secondary CPU
+     * rebuilds these itself (see head.S) */
+    memset(boot_pgtable, 0x0, PAGE_SIZE);
+    flush_xen_dcache(boot_pgtable);
+#ifdef CONFIG_ARM_64
+    memset(boot_pgtable, 0x0, PAGE_SIZE);
+    flush_xen_dcache(boot_first);
+#endif
+    memset(boot_second, 0x0, PAGE_SIZE);
+    flush_xen_dcache(boot_second);
 
     /* Break up the Xen mapping into 4k pages and protect them separately. */
     for ( i = 0; i < LPAE_ENTRIES; i++ )
@@ -452,6 +484,7 @@ void __init setup_pagetables(unsigned long boot_phys_offset, paddr_t xen_paddr)
         write_pte(xen_xenmap + i, pte);
         /* No flush required here as page table is not hooked in yet. */
     }
+
     pte = pte_of_xenaddr((vaddr_t)xen_xenmap);
     pte.pt.table = 1;
     write_pte(xen_second + second_linear_offset(XEN_VIRT_START), pte);
@@ -463,7 +496,7 @@ void __init setup_pagetables(unsigned long boot_phys_offset, paddr_t xen_paddr)
     flush_xen_text_tlb();
 
 #ifdef CONFIG_ARM_32
-    per_cpu(xen_pgtable, 0) = boot_pgtable;
+    per_cpu(xen_pgtable, 0) = cpu0_pgtable;
     per_cpu(xen_dommap, 0) = xen_second +
         second_linear_offset(DOMHEAP_VIRT_START);
 
@@ -474,10 +507,14 @@ void __init setup_pagetables(unsigned long boot_phys_offset, paddr_t xen_paddr)
                               DOMHEAP_SECOND_PAGES*PAGE_SIZE);
 #endif
 }
+
 #ifdef CONFIG_ARM_64
 int init_secondary_pagetables(int cpu)
 {
-    /* All CPUs share a single page table on 64 bit */
+    /* Set init_ttbr for this CPU coming up. All CPus share a single setof
+     * pagetables, but rewrite it each time for consistency with 32 bit. */
+    init_ttbr = (uintptr_t) xen_pgtable + phys_offset;
+    flush_xen_dcache(init_ttbr);
     return 0;
 }
 #else
@@ -498,7 +535,7 @@ int init_secondary_pagetables(int cpu)
     }
 
     /* Initialise root pagetable from root of boot tables */
-    memcpy(first, boot_pgtable, PAGE_SIZE);
+    memcpy(first, cpu0_pgtable, PAGE_SIZE);
 
     /* Ensure the domheap has no stray mappings */
     memset(domheap, 0, DOMHEAP_SECOND_PAGES*PAGE_SIZE);
@@ -518,6 +555,10 @@ int init_secondary_pagetables(int cpu)
     per_cpu(xen_pgtable, cpu) = first;
     per_cpu(xen_dommap, cpu) = domheap;
 
+    /* Set init_ttbr for this CPU coming up */
+    init_ttbr = __pa(first);
+    flush_xen_dcache(init_ttbr);
+
     return 0;
 }
 #endif
@@ -525,12 +566,6 @@ int init_secondary_pagetables(int cpu)
 /* MMU setup for secondary CPUS (which already have paging enabled) */
 void __cpuinit mmu_init_secondary_cpu(void)
 {
-    uint64_t ttbr;
-
-    /* Change to this CPU's pagetables */
-    ttbr = (uintptr_t)virt_to_maddr(THIS_CPU_PGTABLE);
-    WRITE_TTBR(ttbr);
-
     /* From now on, no mapping may be both writable and executable. */
     WRITE_SYSREG32(READ_SYSREG32(SCTLR_EL2) | SCTLR_WXN, SCTLR_EL2);
     flush_xen_text_tlb();
@@ -603,7 +638,7 @@ void __init setup_xenheap_mappings(unsigned long base_mfn,
     while ( base_mfn < end_mfn )
     {
         int slot = zeroeth_table_offset(vaddr);
-        lpae_t *p = &boot_pgtable[slot];
+        lpae_t *p = &xen_pgtable[slot];
 
         if ( p->pt.valid )
         {
@@ -670,7 +705,7 @@ void __init setup_frametable_mappings(paddr_t ps, paddr_t pe)
     {
         pte = mfn_to_xen_entry(second_base + i);
         pte.pt.table = 1;
-        write_pte(&boot_first[first_table_offset(FRAMETABLE_VIRT_START)+i], pte);
+        write_pte(&xen_first[first_table_offset(FRAMETABLE_VIRT_START)+i], pte);
     }
     create_32mb_mappings(second, 0, base_mfn, frametable_size >> PAGE_SHIFT);
 #else