From 271a1c001d03b5f15a123f4714b7f8a5723ac441 Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Thu, 9 Feb 2012 11:33:30 +0000 Subject: arm: mm and p2m Functions to setup pagetables, handle the p2m, map and unmap domain pages, copy data to/from guest addresses. The implementation is based on the LPAE extension for ARMv7 and makes use of the two level transtion mechanism. Signed-off-by: Stefano Stabellini Signed-off-by: Ian Campbell Signed-off-by: Tim Deegan Committed-by: Ian Campbell --- xen/arch/arm/domain.c | 4 + xen/arch/arm/guestcopy.c | 81 +++++++++ xen/arch/arm/mm.c | 321 +++++++++++++++++++++++++++++++++++ xen/arch/arm/p2m.c | 214 +++++++++++++++++++++++ xen/include/asm-arm/domain.h | 2 + xen/include/asm-arm/guest_access.h | 131 +++++++++++++++ xen/include/asm-arm/mm.h | 315 ++++++++++++++++++++++++++++++++++ xen/include/asm-arm/p2m.h | 88 ++++++++++ xen/include/asm-arm/page.h | 335 +++++++++++++++++++++++++++++++++++++ 9 files changed, 1491 insertions(+) create mode 100644 xen/arch/arm/guestcopy.c create mode 100644 xen/arch/arm/mm.c create mode 100644 xen/arch/arm/p2m.c create mode 100644 xen/include/asm-arm/guest_access.h create mode 100644 xen/include/asm-arm/mm.h create mode 100644 xen/include/asm-arm/p2m.h create mode 100644 xen/include/asm-arm/page.h diff --git a/xen/arch/arm/domain.c b/xen/arch/arm/domain.c index 26f2d49e6b..72034f3fdb 100644 --- a/xen/arch/arm/domain.c +++ b/xen/arch/arm/domain.c @@ -224,6 +224,10 @@ int arch_domain_create(struct domain *d, unsigned int domcr_flags) { int rc; + rc = -ENOMEM; + if ( (rc = p2m_init(d)) != 0 ) + goto fail; + d->max_vcpus = 8; rc = 0; diff --git a/xen/arch/arm/guestcopy.c b/xen/arch/arm/guestcopy.c new file mode 100644 index 0000000000..d9eb7ac558 --- /dev/null +++ b/xen/arch/arm/guestcopy.c @@ -0,0 +1,81 @@ +#include +#include +#include + +#include +#include + +unsigned long raw_copy_to_guest(void *to, const void *from, unsigned len) +{ + /* XXX needs to handle faults */ + unsigned offset = ((unsigned long)to & ~PAGE_MASK); + + while ( len ) + { + paddr_t g = gvirt_to_maddr((uint32_t) to); + void *p = map_domain_page(g>>PAGE_SHIFT); + unsigned size = min(len, (unsigned)PAGE_SIZE - offset); + + p += offset; + memcpy(p, from, size); + + unmap_domain_page(p - offset); + len -= size; + from += size; + to += size; + offset = 0; + } + + return 0; +} + +unsigned long raw_clear_guest(void *to, unsigned len) +{ + /* XXX needs to handle faults */ + unsigned offset = ((unsigned long)to & ~PAGE_MASK); + + while ( len ) + { + paddr_t g = gvirt_to_maddr((uint32_t) to); + void *p = map_domain_page(g>>PAGE_SHIFT); + unsigned size = min(len, (unsigned)PAGE_SIZE - offset); + + p += offset; + memset(p, 0x00, size); + + unmap_domain_page(p - offset); + len -= size; + to += size; + offset = 0; + } + + return 0; +} + +unsigned long raw_copy_from_guest(void *to, const void __user *from, unsigned len) +{ + while ( len ) + { + paddr_t g = gvirt_to_maddr((uint32_t) from & PAGE_MASK); + void *p = map_domain_page(g>>PAGE_SHIFT); + unsigned size = min(len, (unsigned)(PAGE_SIZE - ((unsigned)from & (~PAGE_MASK)))); + + p += ((unsigned long)from & (~PAGE_MASK)); + + memcpy(to, p, size); + + unmap_domain_page(p); + len -= size; + from += size; + to += size; + } + return 0; +} +/* + * Local variables: + * mode: C + * c-set-style: "BSD" + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c new file mode 100644 index 0000000000..613d084d92 --- /dev/null +++ b/xen/arch/arm/mm.c @@ -0,0 +1,321 @@ +/* + * xen/arch/arm/mm.c + * + * MMU code for an ARMv7-A with virt extensions. + * + * Tim Deegan + * Copyright (c) 2011 Citrix Systems. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +struct domain *dom_xen, *dom_io; + +/* Static start-of-day pagetables that we use before the allocators are up */ +lpae_t xen_pgtable[LPAE_ENTRIES] __attribute__((__aligned__(4096))); +lpae_t xen_second[LPAE_ENTRIES*4] __attribute__((__aligned__(4096*4))); +static lpae_t xen_fixmap[LPAE_ENTRIES] __attribute__((__aligned__(4096))); +static lpae_t xen_xenmap[LPAE_ENTRIES] __attribute__((__aligned__(4096))); + +/* Limits of the Xen heap */ +unsigned long xenheap_mfn_start, xenheap_mfn_end; +unsigned long xenheap_virt_end; + +unsigned long frametable_virt_end; + +/* Map a 4k page in a fixmap entry */ +void set_fixmap(unsigned map, unsigned long mfn, unsigned attributes) +{ + lpae_t pte = mfn_to_xen_entry(mfn); + pte.pt.table = 1; /* 4k mappings always have this bit set */ + pte.pt.ai = attributes; + write_pte(xen_fixmap + third_table_offset(FIXMAP_ADDR(map)), pte); + flush_xen_data_tlb_va(FIXMAP_ADDR(map)); +} + +/* Remove a mapping from a fixmap entry */ +void clear_fixmap(unsigned map) +{ + lpae_t pte = {0}; + write_pte(xen_fixmap + third_table_offset(FIXMAP_ADDR(map)), pte); + flush_xen_data_tlb_va(FIXMAP_ADDR(map)); +} + +/* Map a page of domheap memory */ +void *map_domain_page(unsigned long mfn) +{ + unsigned long flags; + lpae_t *map = xen_second + second_linear_offset(DOMHEAP_VIRT_START); + unsigned long slot_mfn = mfn & ~LPAE_ENTRY_MASK; + uint32_t va; + lpae_t pte; + int i, slot; + + local_irq_save(flags); + + /* The map is laid out as an open-addressed hash table where each + * entry is a 2MB superpage pte. We use the available bits of each + * PTE as a reference count; when the refcount is zero the slot can + * be reused. */ + for ( slot = (slot_mfn >> LPAE_SHIFT) % DOMHEAP_ENTRIES, i = 0; + i < DOMHEAP_ENTRIES; + slot = (slot + 1) % DOMHEAP_ENTRIES, i++ ) + { + if ( map[slot].pt.avail == 0 ) + { + /* Commandeer this 2MB slot */ + pte = mfn_to_xen_entry(slot_mfn); + pte.pt.avail = 1; + write_pte(map + slot, pte); + break; + } + else if ( map[slot].pt.avail < 0xf && map[slot].pt.base == slot_mfn ) + { + /* This slot already points to the right place; reuse it */ + map[slot].pt.avail++; + break; + } + } + /* If the map fills up, the callers have misbehaved. */ + BUG_ON(i == DOMHEAP_ENTRIES); + +#ifndef NDEBUG + /* Searching the hash could get slow if the map starts filling up. + * Cross that bridge when we come to it */ + { + static int max_tries = 32; + if ( i >= max_tries ) + { + dprintk(XENLOG_WARNING, "Domheap map is filling: %i tries\n", i); + max_tries *= 2; + } + } +#endif + + local_irq_restore(flags); + + va = (DOMHEAP_VIRT_START + + (slot << SECOND_SHIFT) + + ((mfn & LPAE_ENTRY_MASK) << THIRD_SHIFT)); + + /* + * We may not have flushed this specific subpage at map time, + * since we only flush the 4k page not the superpage + */ + flush_xen_data_tlb_va(va); + + return (void *)va; +} + +/* Release a mapping taken with map_domain_page() */ +void unmap_domain_page(const void *va) +{ + unsigned long flags; + lpae_t *map = xen_second + second_linear_offset(DOMHEAP_VIRT_START); + int slot = ((unsigned long) va - DOMHEAP_VIRT_START) >> SECOND_SHIFT; + + local_irq_save(flags); + + ASSERT(slot >= 0 && slot < DOMHEAP_ENTRIES); + ASSERT(map[slot].pt.avail != 0); + + map[slot].pt.avail--; + + local_irq_restore(flags); +} + + +/* Boot-time pagetable setup. + * Changes here may need matching changes in head.S */ +void __init setup_pagetables(unsigned long boot_phys_offset) +{ + paddr_t xen_paddr, phys_offset; + unsigned long dest_va; + lpae_t pte, *p; + int i; + + if ( boot_phys_offset != 0 ) + { + /* Remove the old identity mapping of the boot paddr */ + pte.bits = 0; + dest_va = (unsigned long)_start + boot_phys_offset; + write_pte(xen_second + second_linear_offset(dest_va), pte); + } + + xen_paddr = XEN_PADDR; + + /* Map the destination in the empty L2 above the fixmap */ + dest_va = FIXMAP_ADDR(0) + (1u << SECOND_SHIFT); + pte = mfn_to_xen_entry(xen_paddr >> PAGE_SHIFT); + write_pte(xen_second + second_table_offset(dest_va), pte); + + /* Calculate virt-to-phys offset for the new location */ + phys_offset = xen_paddr - (unsigned long) _start; + + /* Copy */ + memcpy((void *) dest_va, _start, _end - _start); + + /* Beware! Any state we modify between now and the PT switch may be + * discarded when we switch over to the copy. */ + + /* Update the copy of xen_pgtable to use the new paddrs */ + p = (void *) xen_pgtable + dest_va - (unsigned long) _start; + for ( i = 0; i < 4; i++) + p[i].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT; + p = (void *) xen_second + dest_va - (unsigned long) _start; + for ( i = 0; i < 4 * LPAE_ENTRIES; i++) + if ( p[i].pt.valid ) + p[i].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT; + + /* Change pagetables to the copy in the relocated Xen */ + asm volatile ( + STORE_CP64(0, HTTBR) /* Change translation base */ + "dsb;" /* Ensure visibility of HTTBR update */ + STORE_CP32(0, TLBIALLH) /* Flush hypervisor TLB */ + STORE_CP32(0, BPIALL) /* Flush branch predictor */ + "dsb;" /* Ensure completion of TLB+BP flush */ + "isb;" + : : "r" ((unsigned long) xen_pgtable + phys_offset) : "memory"); + + /* Undo the temporary map */ + pte.bits = 0; + write_pte(xen_second + second_table_offset(dest_va), pte); + /* + * Have removed a mapping previously used for .text. Flush everything + * for safety. + */ + asm volatile ( + "dsb;" /* Ensure visibility of PTE write */ + STORE_CP32(0, TLBIALLH) /* Flush hypervisor TLB */ + STORE_CP32(0, BPIALL) /* Flush branch predictor */ + "dsb;" /* Ensure completion of TLB+BP flush */ + "isb;" + : : "r" (i /*dummy*/) : "memory"); + + /* Link in the fixmap pagetable */ + pte = mfn_to_xen_entry((((unsigned long) xen_fixmap) + phys_offset) + >> PAGE_SHIFT); + pte.pt.table = 1; + write_pte(xen_second + second_table_offset(FIXMAP_ADDR(0)), pte); + /* + * No flush required here. Individual flushes are done in + * set_fixmap as entries are used. + */ + + /* Break up the Xen mapping into 4k pages and protect them separately. */ + for ( i = 0; i < LPAE_ENTRIES; i++ ) + { + unsigned long mfn = paddr_to_pfn(xen_paddr) + i; + unsigned long va = XEN_VIRT_START + (i << PAGE_SHIFT); + if ( !is_kernel(va) ) + break; + pte = mfn_to_xen_entry(mfn); + pte.pt.table = 1; /* 4k mappings always have this bit set */ + if ( is_kernel_text(va) || is_kernel_inittext(va) ) + { + pte.pt.xn = 0; + pte.pt.ro = 1; + } + if ( is_kernel_rodata(va) ) + pte.pt.ro = 1; + write_pte(xen_xenmap + i, pte); + /* No flush required here as page table is not hooked in yet. */ + } + pte = mfn_to_xen_entry((((unsigned long) xen_xenmap) + phys_offset) + >> PAGE_SHIFT); + pte.pt.table = 1; + write_pte(xen_second + second_linear_offset(XEN_VIRT_START), pte); + /* Have changed a mapping used for .text. Flush everything for safety. */ + asm volatile ( + "dsb;" /* Ensure visibility of PTE write */ + STORE_CP32(0, TLBIALLH) /* Flush hypervisor TLB */ + STORE_CP32(0, BPIALL) /* Flush branch predictor */ + "dsb;" /* Ensure completion of TLB+BP flush */ + "isb;" + : : "r" (i /*dummy*/) : "memory"); + + /* From now on, no mapping may be both writable and executable. */ + WRITE_CP32(READ_CP32(HSCTLR) | SCTLR_WXN, HSCTLR); +} + +/* Create Xen's mappings of memory. + * Base and virt must be 32MB aligned and size a multiple of 32MB. */ +static void __init create_mappings(unsigned long virt, + unsigned long base_mfn, + unsigned long nr_mfns) +{ + unsigned long i, count; + lpae_t pte, *p; + + ASSERT(!((virt >> PAGE_SHIFT) % (16 * LPAE_ENTRIES))); + ASSERT(!(base_mfn % (16 * LPAE_ENTRIES))); + ASSERT(!(nr_mfns % (16 * LPAE_ENTRIES))); + + count = nr_mfns / LPAE_ENTRIES; + p = xen_second + second_linear_offset(virt); + pte = mfn_to_xen_entry(base_mfn); + pte.pt.hint = 1; /* These maps are in 16-entry contiguous chunks. */ + for ( i = 0; i < count; i++ ) + { + write_pte(p + i, pte); + pte.pt.base += 1 << LPAE_SHIFT; + } + flush_xen_data_tlb(); +} + +/* Set up the xenheap: up to 1GB of contiguous, always-mapped memory. */ +void __init setup_xenheap_mappings(unsigned long base_mfn, + unsigned long nr_mfns) +{ + create_mappings(XENHEAP_VIRT_START, base_mfn, nr_mfns); + + /* Record where the xenheap is, for translation routines. */ + xenheap_virt_end = XENHEAP_VIRT_START + nr_mfns * PAGE_SIZE; + xenheap_mfn_start = base_mfn; + xenheap_mfn_end = base_mfn + nr_mfns; +} + +/* Map a frame table to cover physical addresses ps through pe */ +void __init setup_frametable_mappings(paddr_t ps, paddr_t pe) +{ + unsigned long nr_pages = (pe - ps) >> PAGE_SHIFT; + unsigned long frametable_size = nr_pages * sizeof(struct page_info); + unsigned long base_mfn; + + /* Round up to 32M boundary */ + frametable_size = (frametable_size + 0x1ffffff) & ~0x1ffffff; + base_mfn = alloc_boot_pages(frametable_size >> PAGE_SHIFT, 5); + create_mappings(FRAMETABLE_VIRT_START, base_mfn, frametable_size >> PAGE_SHIFT); + + memset(&frame_table[0], 0, nr_pages * sizeof(struct page_info)); + memset(&frame_table[nr_pages], -1, + frametable_size - (nr_pages * sizeof(struct page_info))); + + frametable_virt_end = FRAMETABLE_VIRT_START + (nr_pages * sizeof(struct page_info)); +} + +/* + * Local variables: + * mode: C + * c-set-style: "BSD" + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/arch/arm/p2m.c b/xen/arch/arm/p2m.c new file mode 100644 index 0000000000..a1d026d025 --- /dev/null +++ b/xen/arch/arm/p2m.c @@ -0,0 +1,214 @@ +#include +#include +#include +#include +#include + +void p2m_load_VTTBR(struct domain *d) +{ + struct p2m_domain *p2m = &d->arch.p2m; + paddr_t maddr = page_to_maddr(p2m->first_level); + uint64_t vttbr = maddr; + + vttbr |= ((uint64_t)p2m->vmid&0xff)<<48; + + printk("VTTBR dom%d = %"PRIx64"\n", d->domain_id, vttbr); + + WRITE_CP64(vttbr, VTTBR); + isb(); /* Ensure update is visible */ +} + +static int p2m_create_entry(struct domain *d, + lpae_t *entry) +{ + struct p2m_domain *p2m = &d->arch.p2m; + struct page_info *page; + void *p; + lpae_t pte; + + BUG_ON(entry->p2m.valid); + + page = alloc_domheap_page(d, 0); + if ( page == NULL ) + return -ENOMEM; + + page_list_add(page, &p2m->pages); + + p = __map_domain_page(page); + clear_page(p); + unmap_domain_page(p); + + pte = mfn_to_p2m_entry(page_to_mfn(page)); + + write_pte(entry, pte); + + return 0; +} + +static int create_p2m_entries(struct domain *d, + int alloc, + paddr_t start_gpaddr, + paddr_t end_gpaddr, + paddr_t maddr) +{ + int rc; + struct p2m_domain *p2m = &d->arch.p2m; + lpae_t *first = NULL, *second = NULL, *third = NULL; + paddr_t addr; + unsigned long cur_first_offset = ~0, cur_second_offset = ~0; + + /* XXX Don't actually handle 40 bit guest physical addresses */ + BUG_ON(start_gpaddr & 0x8000000000ULL); + BUG_ON(end_gpaddr & 0x8000000000ULL); + + first = __map_domain_page(p2m->first_level); + + for(addr = start_gpaddr; addr < end_gpaddr; addr += PAGE_SIZE) + { + if ( !first[first_table_offset(addr)].p2m.valid ) + { + rc = p2m_create_entry(d, &first[first_table_offset(addr)]); + if ( rc < 0 ) { + printk("p2m_populate_ram: L1 failed\n"); + goto out; + } + } + + BUG_ON(!first[first_table_offset(addr)].p2m.valid); + + if ( cur_first_offset != first_table_offset(addr) ) + { + if (second) unmap_domain_page(second); + second = map_domain_page(first[first_table_offset(addr)].p2m.base); + cur_first_offset = first_table_offset(addr); + } + /* else: second already valid */ + + if ( !second[second_table_offset(addr)].p2m.valid ) + { + rc = p2m_create_entry(d, &second[second_table_offset(addr)]); + if ( rc < 0 ) { + printk("p2m_populate_ram: L2 failed\n"); + goto out; + } + } + + BUG_ON(!second[second_table_offset(addr)].p2m.valid); + + if ( cur_second_offset != second_table_offset(addr) ) + { + /* map third level */ + if (third) unmap_domain_page(third); + third = map_domain_page(second[second_table_offset(addr)].p2m.base); + cur_second_offset = second_table_offset(addr); + } + /* else: third already valid */ + + BUG_ON(third[third_table_offset(addr)].p2m.valid); + + /* Allocate a new RAM page and attach */ + if (alloc) + { + struct page_info *page; + lpae_t pte; + + rc = -ENOMEM; + page = alloc_domheap_page(d, 0); + if ( page == NULL ) { + printk("p2m_populate_ram: failed to allocate page\n"); + goto out; + } + + pte = mfn_to_p2m_entry(page_to_mfn(page)); + + write_pte(&third[third_table_offset(addr)], pte); + } else { + lpae_t pte = mfn_to_p2m_entry(maddr >> PAGE_SHIFT); + write_pte(&third[third_table_offset(addr)], pte); + maddr += PAGE_SIZE; + } + } + + rc = 0; + +out: + spin_lock(&p2m->lock); + + if (third) unmap_domain_page(third); + if (second) unmap_domain_page(second); + if (first) unmap_domain_page(first); + + spin_unlock(&p2m->lock); + + return rc; +} + +int p2m_populate_ram(struct domain *d, + paddr_t start, + paddr_t end) +{ + return create_p2m_entries(d, 1, start, end, 0); +} + +int map_mmio_regions(struct domain *d, + paddr_t start_gaddr, + paddr_t end_gaddr, + paddr_t maddr) +{ + return create_p2m_entries(d, 0, start_gaddr, end_gaddr, maddr); +} + +int p2m_alloc_table(struct domain *d) +{ + struct p2m_domain *p2m = &d->arch.p2m; + struct page_info *page; + void *p; + + /* First level P2M is 2 consecutive pages */ + page = alloc_domheap_pages(d, 1, 0); + if ( page == NULL ) + return -ENOMEM; + + spin_lock(&p2m->lock); + + page_list_add(page, &p2m->pages); + + /* Clear both first level pages */ + p = __map_domain_page(page); + clear_page(p); + unmap_domain_page(p); + + p = __map_domain_page(page + 1); + clear_page(p); + unmap_domain_page(p); + + p2m->first_level = page; + + spin_unlock(&p2m->lock); + + return 0; +} + +int p2m_init(struct domain *d) +{ + struct p2m_domain *p2m = &d->arch.p2m; + + spin_lock_init(&p2m->lock); + INIT_PAGE_LIST_HEAD(&p2m->pages); + + /* XXX allocate properly */ + /* Zero is reserved */ + p2m->vmid = d->domain_id + 1; + + p2m->first_level = NULL; + + return 0; +} +/* + * Local variables: + * mode: C + * c-set-style: "BSD" + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/include/asm-arm/domain.h b/xen/include/asm-arm/domain.h index c226bdf58a..2226a24a52 100644 --- a/xen/include/asm-arm/domain.h +++ b/xen/include/asm-arm/domain.h @@ -16,6 +16,8 @@ struct pending_irq struct arch_domain { + struct p2m_domain p2m; + } __cacheline_aligned; struct arch_vcpu diff --git a/xen/include/asm-arm/guest_access.h b/xen/include/asm-arm/guest_access.h new file mode 100644 index 0000000000..0fceae6a01 --- /dev/null +++ b/xen/include/asm-arm/guest_access.h @@ -0,0 +1,131 @@ +#ifndef __ASM_ARM_GUEST_ACCESS_H__ +#define __ASM_ARM_GUEST_ACCESS_H__ + +#include +#include + +/* Guests have their own comlete address space */ +#define access_ok(addr,size) (1) + +#define array_access_ok(addr,count,size) \ + (likely(count < (~0UL/size)) && access_ok(addr,count*size)) + +unsigned long raw_copy_to_guest(void *to, const void *from, unsigned len); +unsigned long raw_copy_from_guest(void *to, const void *from, unsigned len); +unsigned long raw_clear_guest(void *to, unsigned len); + +#define __raw_copy_to_guest raw_copy_to_guest +#define __raw_copy_from_guest raw_copy_from_guest +#define __raw_clear_guest raw_clear_guest + +/* Remainder copied from x86 -- could be common? */ + +/* Is the guest handle a NULL reference? */ +#define guest_handle_is_null(hnd) ((hnd).p == NULL) + +/* Offset the given guest handle into the array it refers to. */ +#define guest_handle_add_offset(hnd, nr) ((hnd).p += (nr)) +#define guest_handle_subtract_offset(hnd, nr) ((hnd).p -= (nr)) + +/* Cast a guest handle to the specified type of handle. */ +#define guest_handle_cast(hnd, type) ({ \ + type *_x = (hnd).p; \ + (XEN_GUEST_HANDLE(type)) { _x }; \ +}) + +#define guest_handle_from_ptr(ptr, type) \ + ((XEN_GUEST_HANDLE(type)) { (type *)ptr }) +#define const_guest_handle_from_ptr(ptr, type) \ + ((XEN_GUEST_HANDLE(const_##type)) { (const type *)ptr }) + +/* + * Copy an array of objects to guest context via a guest handle, + * specifying an offset into the guest array. + */ +#define copy_to_guest_offset(hnd, off, ptr, nr) ({ \ + const typeof(*(ptr)) *_s = (ptr); \ + char (*_d)[sizeof(*_s)] = (void *)(hnd).p; \ + ((void)((hnd).p == (ptr))); \ + raw_copy_to_guest(_d+(off), _s, sizeof(*_s)*(nr)); \ +}) + +/* + * Clear an array of objects in guest context via a guest handle, + * specifying an offset into the guest array. + */ +#define clear_guest_offset(hnd, off, ptr, nr) ({ \ + raw_clear_guest(_d+(off), nr); \ +}) + +/* + * Copy an array of objects from guest context via a guest handle, + * specifying an offset into the guest array. + */ +#define copy_from_guest_offset(ptr, hnd, off, nr) ({ \ + const typeof(*(ptr)) *_s = (hnd).p; \ + typeof(*(ptr)) *_d = (ptr); \ + raw_copy_from_guest(_d, _s+(off), sizeof(*_d)*(nr));\ +}) + +/* Copy sub-field of a structure to guest context via a guest handle. */ +#define copy_field_to_guest(hnd, ptr, field) ({ \ + const typeof(&(ptr)->field) _s = &(ptr)->field; \ + void *_d = &(hnd).p->field; \ + ((void)(&(hnd).p->field == &(ptr)->field)); \ + raw_copy_to_guest(_d, _s, sizeof(*_s)); \ +}) + +/* Copy sub-field of a structure from guest context via a guest handle. */ +#define copy_field_from_guest(ptr, hnd, field) ({ \ + const typeof(&(ptr)->field) _s = &(hnd).p->field; \ + typeof(&(ptr)->field) _d = &(ptr)->field; \ + raw_copy_from_guest(_d, _s, sizeof(*_d)); \ +}) + +/* + * Pre-validate a guest handle. + * Allows use of faster __copy_* functions. + */ +/* All ARM guests are paging mode external and hence safe */ +#define guest_handle_okay(hnd, nr) (1) +#define guest_handle_subrange_okay(hnd, first, last) (1) + +#define __copy_to_guest_offset(hnd, off, ptr, nr) ({ \ + const typeof(*(ptr)) *_s = (ptr); \ + char (*_d)[sizeof(*_s)] = (void *)(hnd).p; \ + ((void)((hnd).p == (ptr))); \ + __raw_copy_to_guest(_d+(off), _s, sizeof(*_s)*(nr));\ +}) + +#define __clear_guest_offset(hnd, off, ptr, nr) ({ \ + __raw_clear_guest(_d+(off), nr); \ +}) + +#define __copy_from_guest_offset(ptr, hnd, off, nr) ({ \ + const typeof(*(ptr)) *_s = (hnd).p; \ + typeof(*(ptr)) *_d = (ptr); \ + __raw_copy_from_guest(_d, _s+(off), sizeof(*_d)*(nr));\ +}) + +#define __copy_field_to_guest(hnd, ptr, field) ({ \ + const typeof(&(ptr)->field) _s = &(ptr)->field; \ + void *_d = &(hnd).p->field; \ + ((void)(&(hnd).p->field == &(ptr)->field)); \ + __raw_copy_to_guest(_d, _s, sizeof(*_s)); \ +}) + +#define __copy_field_from_guest(ptr, hnd, field) ({ \ + const typeof(&(ptr)->field) _s = &(hnd).p->field; \ + typeof(&(ptr)->field) _d = &(ptr)->field; \ + __raw_copy_from_guest(_d, _s, sizeof(*_d)); \ +}) + +#endif /* __ASM_ARM_GUEST_ACCESS_H__ */ +/* + * Local variables: + * mode: C + * c-set-style: "BSD" + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/include/asm-arm/mm.h b/xen/include/asm-arm/mm.h new file mode 100644 index 0000000000..f721c54361 --- /dev/null +++ b/xen/include/asm-arm/mm.h @@ -0,0 +1,315 @@ +#ifndef __ARCH_ARM_MM__ +#define __ARCH_ARM_MM__ + +#include +#include +#include +#include + +/* Find a suitable place at the top of memory for Xen to live */ +/* XXX For now, use the top of the VE's 4GB RAM, at a 40-bit alias */ +#define XEN_PADDR 0x80ffe00000ull + +/* + * Per-page-frame information. + * + * Every architecture must ensure the following: + * 1. 'struct page_info' contains a 'struct page_list_entry list'. + * 2. Provide a PFN_ORDER() macro for accessing the order of a free page. + */ +#define PFN_ORDER(_pfn) ((_pfn)->v.free.order) + +struct page_info +{ + /* Each frame can be threaded onto a doubly-linked list. */ + struct page_list_entry list; + + /* Reference count and various PGC_xxx flags and fields. */ + unsigned long count_info; + + /* Context-dependent fields follow... */ + union { + /* Page is in use: ((count_info & PGC_count_mask) != 0). */ + struct { + /* Type reference count and various PGT_xxx flags and fields. */ + unsigned long type_info; + } inuse; + /* Page is on a free list: ((count_info & PGC_count_mask) == 0). */ + struct { + /* Do TLBs need flushing for safety before next page use? */ + bool_t need_tlbflush; + } free; + + } u; + + union { + /* Page is in use, but not as a shadow. */ + struct { + /* Owner of this page (zero if page is anonymous). */ + struct domain *domain; + } inuse; + + /* Page is on a free list. */ + struct { + /* Order-size of the free chunk this page is the head of. */ + unsigned int order; + } free; + + } v; + + union { + /* + * Timestamp from 'TLB clock', used to avoid extra safety flushes. + * Only valid for: a) free pages, and b) pages with zero type count + */ + u32 tlbflush_timestamp; + }; + u64 pad; +}; + +#define PG_shift(idx) (BITS_PER_LONG - (idx)) +#define PG_mask(x, idx) (x ## UL << PG_shift(idx)) + +#define PGT_none PG_mask(0, 4) /* no special uses of this page */ +#define PGT_writable_page PG_mask(7, 4) /* has writable mappings? */ +#define PGT_shared_page PG_mask(8, 4) /* CoW sharable page */ +#define PGT_type_mask PG_mask(15, 4) /* Bits 28-31 or 60-63. */ + + /* Owning guest has pinned this page to its current type? */ +#define _PGT_pinned PG_shift(5) +#define PGT_pinned PG_mask(1, 5) + + /* Count of uses of this frame as its current type. */ +#define PGT_count_width PG_shift(9) +#define PGT_count_mask ((1UL<count_info&PGC_state) == PGC_state_##st) + +/* Count of references to this frame. */ +#define PGC_count_width PG_shift(9) +#define PGC_count_mask ((1UL<= xenheap_mfn_start && _mfn < xenheap_mfn_end); \ +}) +#define is_xen_fixed_mfn(mfn) is_xen_heap_mfn(mfn) + +#define page_get_owner(_p) (_p)->v.inuse.domain +#define page_set_owner(_p,_d) ((_p)->v.inuse.domain = (_d)) + +#define maddr_get_owner(ma) (page_get_owner(maddr_to_page((ma)))) +#define vaddr_get_owner(va) (page_get_owner(virt_to_page((va)))) + +#define XENSHARE_writable 0 +#define XENSHARE_readonly 1 +extern void share_xen_page_with_guest( + struct page_info *page, struct domain *d, int readonly); +extern void share_xen_page_with_privileged_guests( + struct page_info *page, int readonly); + +#define frame_table ((struct page_info *)FRAMETABLE_VIRT_START) + +extern unsigned long max_page; +extern unsigned long total_pages; + +/* Boot-time pagetable setup */ +extern void setup_pagetables(unsigned long boot_phys_offset); +/* Set up the xenheap: up to 1GB of contiguous, always-mapped memory. + * Base must be 32MB aligned and size a multiple of 32MB. */ +extern void setup_xenheap_mappings(unsigned long base_mfn, unsigned long nr_mfns); +/* Map a frame table to cover physical addresses ps through pe */ +extern void setup_frametable_mappings(paddr_t ps, paddr_t pe); +/* Map a 4k page in a fixmap entry */ +extern void set_fixmap(unsigned map, unsigned long mfn, unsigned attributes); +/* Remove a mapping from a fixmap entry */ +extern void clear_fixmap(unsigned map); + + +#define mfn_valid(mfn) ({ \ + unsigned long __m_f_n = (mfn); \ + likely(__m_f_n < max_page); \ +}) + +#define max_pdx max_page +/* XXX Assume everything in the 40-bit physical alias 0x8000000000 for now */ +#define pfn_to_pdx(pfn) ((pfn) - 0x8000000UL) +#define pdx_to_pfn(pdx) ((pdx) + 0x8000000UL) +#define virt_to_pdx(va) virt_to_mfn(va) +#define pdx_to_virt(pdx) mfn_to_virt(pdx) + +/* Convert between machine frame numbers and page-info structures. */ +#define mfn_to_page(mfn) (frame_table + pfn_to_pdx(mfn)) +#define page_to_mfn(pg) pdx_to_pfn((unsigned long)((pg) - frame_table)) +#define __page_to_mfn(pg) page_to_mfn(pg) +#define __mfn_to_page(mfn) mfn_to_page(mfn) + +/* Convert between machine addresses and page-info structures. */ +#define maddr_to_page(ma) __mfn_to_page((ma) >> PAGE_SHIFT) +#define page_to_maddr(pg) ((paddr_t)__page_to_mfn(pg) << PAGE_SHIFT) + +/* Convert between frame number and address formats. */ +#define pfn_to_paddr(pfn) ((paddr_t)(pfn) << PAGE_SHIFT) +#define paddr_to_pfn(pa) ((unsigned long)((pa) >> PAGE_SHIFT)) +#define paddr_to_pdx(pa) pfn_to_pdx(paddr_to_pfn(pa)) + + +static inline paddr_t virt_to_maddr(void *va) +{ + uint64_t par = va_to_par((uint32_t)va); + return (par & PADDR_MASK & PAGE_MASK) | ((unsigned long) va & ~PAGE_MASK); +} + +static inline void *maddr_to_virt(paddr_t ma) +{ + ASSERT(is_xen_heap_mfn(ma >> PAGE_SHIFT)); + ma -= pfn_to_paddr(xenheap_mfn_start); + return (void *)(unsigned long) ma + XENHEAP_VIRT_START; +} + +static inline paddr_t gvirt_to_maddr(uint32_t va) +{ + uint64_t par = gva_to_par(va); + return (par & PADDR_MASK & PAGE_MASK) | ((unsigned long) va & ~PAGE_MASK); +} + +/* Convert between Xen-heap virtual addresses and machine addresses. */ +#define __pa(x) (virt_to_maddr(x)) +#define __va(x) (maddr_to_virt(x)) + +/* Convert between Xen-heap virtual addresses and machine frame numbers. */ +#define virt_to_mfn(va) (virt_to_maddr(va) >> PAGE_SHIFT) +#define mfn_to_virt(mfn) (maddr_to_virt((paddr_t)(mfn) << PAGE_SHIFT)) + + +static inline int get_order_from_bytes(paddr_t size) +{ + int order; + size = (size-1) >> PAGE_SHIFT; + for ( order = 0; size; order++ ) + size >>= 1; + return order; +} + +static inline int get_order_from_pages(unsigned long nr_pages) +{ + int order; + nr_pages--; + for ( order = 0; nr_pages; order++ ) + nr_pages >>= 1; + return order; +} + + +/* Convert between Xen-heap virtual addresses and page-info structures. */ +static inline struct page_info *virt_to_page(const void *v) +{ + unsigned long va = (unsigned long)v; + ASSERT(va >= XENHEAP_VIRT_START); + ASSERT(va < xenheap_virt_end); + + return frame_table + ((va - XENHEAP_VIRT_START) >> PAGE_SHIFT); +} + +static inline void *page_to_virt(const struct page_info *pg) +{ + ASSERT((unsigned long)pg - FRAMETABLE_VIRT_START < frametable_virt_end); + return (void *)(XENHEAP_VIRT_START + + ((unsigned long)pg - FRAMETABLE_VIRT_START) / + (sizeof(*pg) / (sizeof(*pg) & -sizeof(*pg))) * + (PAGE_SIZE / (sizeof(*pg) & -sizeof(*pg)))); + +} + +struct domain *page_get_owner_and_reference(struct page_info *page); +void put_page(struct page_info *page); +int get_page(struct page_info *page, struct domain *domain); + +/* + * The MPT (machine->physical mapping table) is an array of word-sized + * values, indexed on machine frame number. It is expected that guest OSes + * will use it to store a "physical" frame number to give the appearance of + * contiguous (or near contiguous) physical memory. + */ +#undef machine_to_phys_mapping +#define machine_to_phys_mapping ((unsigned long *)RDWR_MPT_VIRT_START) +#define INVALID_M2P_ENTRY (~0UL) +#define VALID_M2P(_e) (!((_e) & (1UL<<(BITS_PER_LONG-1)))) +#define SHARED_M2P_ENTRY (~0UL - 1UL) +#define SHARED_M2P(_e) ((_e) == SHARED_M2P_ENTRY) + +#define _set_gpfn_from_mfn(mfn, pfn) ({ \ + struct domain *d = page_get_owner(__mfn_to_page(mfn)); \ + if(d && (d == dom_cow)) \ + machine_to_phys_mapping[(mfn)] = SHARED_M2P_ENTRY; \ + else \ + machine_to_phys_mapping[(mfn)] = (pfn); \ + }) + +#define put_gfn(d, g) ((void)0) + +#define INVALID_MFN (~0UL) + +/* Xen always owns P2M on ARM */ +#define set_gpfn_from_mfn(mfn, pfn) do { (void) (mfn), (void)(pfn); } while (0) +#define mfn_to_gmfn(_d, mfn) (mfn) + + +/* Arch-specific portion of memory_op hypercall. */ +long arch_memory_op(int op, XEN_GUEST_HANDLE(void) arg); + +int steal_page( + struct domain *d, struct page_info *page, unsigned int memflags); +int donate_page( + struct domain *d, struct page_info *page, unsigned int memflags); + +#define domain_set_alloc_bitsize(d) ((void)0) +#define domain_clamp_alloc_bitsize(d, b) (b) + +unsigned long domain_get_maximum_gpfn(struct domain *d); + +extern struct domain *dom_xen, *dom_io, *dom_cow; + +#define memguard_init(_s) (_s) +#define memguard_guard_stack(_p) ((void)0) +#define memguard_guard_range(_p,_l) ((void)0) +#define memguard_unguard_range(_p,_l) ((void)0) +int guest_physmap_mark_populate_on_demand(struct domain *d, unsigned long gfn, + unsigned int order); + +extern void put_page_type(struct page_info *page); +static inline void put_page_and_type(struct page_info *page) +{ + put_page_type(page); + put_page(page); +} + +#endif /* __ARCH_ARM_MM__ */ +/* + * Local variables: + * mode: C + * c-set-style: "BSD" + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/include/asm-arm/p2m.h b/xen/include/asm-arm/p2m.h new file mode 100644 index 0000000000..aec52f77a4 --- /dev/null +++ b/xen/include/asm-arm/p2m.h @@ -0,0 +1,88 @@ +#ifndef _XEN_P2M_H +#define _XEN_P2M_H + +#include + +struct domain; + +/* Per-p2m-table state */ +struct p2m_domain { + /* Lock that protects updates to the p2m */ + spinlock_t lock; + + /* Pages used to construct the p2m */ + struct page_list_head pages; + + /* Root of p2m page tables, 2 contiguous pages */ + struct page_info *first_level; + + /* Current VMID in use */ + uint8_t vmid; +}; + +/* Init the datastructures for later use by the p2m code */ +int p2m_init(struct domain *d); + +/* Allocate a new p2m table for a domain. + * + * Returns 0 for success or -errno. + */ +int p2m_alloc_table(struct domain *d); + +/* */ +void p2m_load_VTTBR(struct domain *d); + +/* Setup p2m RAM mapping for domain d from start-end. */ +int p2m_populate_ram(struct domain *d, paddr_t start, paddr_t end); +/* Map MMIO regions in the p2m: start_gaddr and end_gaddr is the range + * in the guest physical address space to map, starting from the machine + * address maddr. */ +int map_mmio_regions(struct domain *d, paddr_t start_gaddr, + paddr_t end_gaddr, paddr_t maddr); + +unsigned long gmfn_to_mfn(struct domain *d, unsigned long gpfn); + +/* + * Populate-on-demand + */ + +/* Call when decreasing memory reservation to handle PoD entries properly. + * Will return '1' if all entries were handled and nothing more need be done.*/ +int +p2m_pod_decrease_reservation(struct domain *d, + xen_pfn_t gpfn, + unsigned int order); + +/* Compatibility function exporting the old untyped interface */ +static inline unsigned long get_gfn_untyped(struct domain *d, unsigned long gpfn) +{ + return gmfn_to_mfn(d, gpfn); +} + +int get_page_type(struct page_info *page, unsigned long type); +int is_iomem_page(unsigned long mfn); +static inline int get_page_and_type(struct page_info *page, + struct domain *domain, + unsigned long type) +{ + int rc = get_page(page, domain); + + if ( likely(rc) && unlikely(!get_page_type(page, type)) ) + { + put_page(page); + rc = 0; + } + + return rc; +} + +#endif /* _XEN_P2M_H */ + +/* + * Local variables: + * mode: C + * c-set-style: "BSD" + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/include/asm-arm/page.h b/xen/include/asm-arm/page.h new file mode 100644 index 0000000000..6dc1659b11 --- /dev/null +++ b/xen/include/asm-arm/page.h @@ -0,0 +1,335 @@ +#ifndef __ARM_PAGE_H__ +#define __ARM_PAGE_H__ + +#include + +#define PADDR_BITS 40 +#define PADDR_MASK ((1ULL << PADDR_BITS)-1) + +#define VADDR_BITS 32 +#define VADDR_MASK (~0UL) + +/* Shareability values for the LPAE entries */ +#define LPAE_SH_NON_SHAREABLE 0x0 +#define LPAE_SH_UNPREDICTALE 0x1 +#define LPAE_SH_OUTER 0x2 +#define LPAE_SH_INNER 0x3 + +/* LPAE Memory region attributes, to match Linux's (non-LPAE) choices. + * Indexed by the AttrIndex bits of a LPAE entry; + * the 8-bit fields are packed little-endian into MAIR0 and MAIR1 + * + * ai encoding + * UNCACHED 000 0000 0000 -- Strongly Ordered + * BUFFERABLE 001 0100 0100 -- Non-Cacheable + * WRITETHROUGH 010 1010 1010 -- Write-through + * WRITEBACK 011 1110 1110 -- Write-back + * DEV_SHARED 100 0000 0100 -- Device + * ?? 101 + * reserved 110 + * WRITEALLOC 111 1111 1111 -- Write-back write-allocate + * + * DEV_NONSHARED 100 (== DEV_SHARED) + * DEV_WC 001 (== BUFFERABLE) + * DEV_CACHED 011 (== WRITEBACK) + */ +#define MAIR0VAL 0xeeaa4400 +#define MAIR1VAL 0xff000004 + +#define UNCACHED 0x0 +#define BUFFERABLE 0x1 +#define WRITETHROUGH 0x2 +#define WRITEBACK 0x3 +#define DEV_SHARED 0x4 +#define WRITEALLOC 0x7 +#define DEV_NONSHARED DEV_SHARED +#define DEV_WC BUFFERABLE +#define DEV_CACHED WRITEBACK + + +#ifndef __ASSEMBLY__ + +#include +#include + +/* WARNING! Unlike the Intel pagetable code, where l1 is the lowest + * level and l4 is the root of the trie, the ARM pagetables follow ARM's + * documentation: the levels are called first, second &c in the order + * that the MMU walks them (i.e. "first" is the root of the trie). */ + +/****************************************************************************** + * ARMv7-A LPAE pagetables: 3-level trie, mapping 40-bit input to + * 40-bit output addresses. Tables at all levels have 512 64-bit entries + * (i.e. are 4Kb long). + * + * The bit-shuffling that has the permission bits in branch nodes in a + * different place from those in leaf nodes seems to be to allow linear + * pagetable tricks. If we're not doing that then the set of permission + * bits that's not in use in a given node type can be used as + * extra software-defined bits. */ + +typedef struct { + /* These are used in all kinds of entry. */ + unsigned long valid:1; /* Valid mapping */ + unsigned long table:1; /* == 1 in 4k map entries too */ + + /* These ten bits are only used in Block entries and are ignored + * in Table entries. */ + unsigned long ai:3; /* Attribute Index */ + unsigned long ns:1; /* Not-Secure */ + unsigned long user:1; /* User-visible */ + unsigned long ro:1; /* Read-Only */ + unsigned long sh:2; /* Shareability */ + unsigned long af:1; /* Access Flag */ + unsigned long ng:1; /* Not-Global */ + + /* The base address must be approprately aligned for Block entries */ + unsigned long base:28; /* Base address of block or next table */ + unsigned long sbz:12; /* Must be zero */ + + /* These seven bits are only used in Block entries and are ignored + * in Table entries. */ + unsigned long hint:1; /* In a block of 16 contiguous entries */ + unsigned long pxn:1; /* Privileged-XN */ + unsigned long xn:1; /* eXecute-Never */ + unsigned long avail:4; /* Ignored by hardware */ + + /* These 5 bits are only used in Table entries and are ignored in + * Block entries */ + unsigned long pxnt:1; /* Privileged-XN */ + unsigned long xnt:1; /* eXecute-Never */ + unsigned long apt:2; /* Access Permissions */ + unsigned long nst:1; /* Not-Secure */ +} __attribute__((__packed__)) lpae_pt_t; + +/* The p2m tables have almost the same layout, but some of the permission + * and cache-control bits are laid out differently (or missing) */ +typedef struct { + /* These are used in all kinds of entry. */ + unsigned long valid:1; /* Valid mapping */ + unsigned long table:1; /* == 1 in 4k map entries too */ + + /* These ten bits are only used in Block entries and are ignored + * in Table entries. */ + unsigned long mattr:4; /* Memory Attributes */ + unsigned long read:1; /* Read access */ + unsigned long write:1; /* Write access */ + unsigned long sh:2; /* Shareability */ + unsigned long af:1; /* Access Flag */ + unsigned long sbz4:1; + + /* The base address must be approprately aligned for Block entries */ + unsigned long base:28; /* Base address of block or next table */ + unsigned long sbz3:12; + + /* These seven bits are only used in Block entries and are ignored + * in Table entries. */ + unsigned long hint:1; /* In a block of 16 contiguous entries */ + unsigned long sbz2:1; + unsigned long xn:1; /* eXecute-Never */ + unsigned long avail:4; /* Ignored by hardware */ + + unsigned long sbz1:5; +} __attribute__((__packed__)) lpae_p2m_t; + +typedef union { + uint64_t bits; + lpae_pt_t pt; + lpae_p2m_t p2m; +} lpae_t; + +/* Standard entry type that we'll use to build Xen's own pagetables. + * We put the same permissions at every level, because they're ignored + * by the walker in non-leaf entries. */ +static inline lpae_t mfn_to_xen_entry(unsigned long mfn) +{ + paddr_t pa = ((paddr_t) mfn) << PAGE_SHIFT; + lpae_t e = (lpae_t) { + .pt = { + .xn = 1, /* No need to execute outside .text */ + .ng = 1, /* Makes TLB flushes easier */ + .af = 1, /* No need for access tracking */ + .sh = LPAE_SH_OUTER, /* Xen mappings are globally coherent */ + .ns = 1, /* Hyp mode is in the non-secure world */ + .user = 1, /* See below */ + .ai = WRITEALLOC, + .table = 0, /* Set to 1 for links and 4k maps */ + .valid = 1, /* Mappings are present */ + }};; + /* Setting the User bit is strange, but the ATS1H[RW] instructions + * don't seem to work otherwise, and since we never run on Xen + * pagetables un User mode it's OK. If this changes, remember + * to update the hard-coded values in head.S too */ + + ASSERT(!(pa & ~PAGE_MASK)); + ASSERT(!(pa & ~PADDR_MASK)); + + // XXX shifts + e.bits |= pa; + return e; +} + +static inline lpae_t mfn_to_p2m_entry(unsigned long mfn) +{ + paddr_t pa = ((paddr_t) mfn) << PAGE_SHIFT; + lpae_t e = (lpae_t) { + .p2m.xn = 0, + .p2m.af = 1, + .p2m.sh = LPAE_SH_OUTER, + .p2m.write = 1, + .p2m.read = 1, + .p2m.mattr = 0xf, + .p2m.table = 1, + .p2m.valid = 1, + }; + + ASSERT(!(pa & ~PAGE_MASK)); + ASSERT(!(pa & ~PADDR_MASK)); + + e.bits |= pa; + + return e; +} + +/* Write a pagetable entry */ +static inline void write_pte(lpae_t *p, lpae_t pte) +{ + asm volatile ( + /* Safely write the entry (STRD is atomic on CPUs that support LPAE) */ + "strd %0, %H0, [%1];" + /* Push this cacheline to the PoC so the rest of the system sees it. */ + STORE_CP32(1, DCCMVAC) + : : "r" (pte.bits), "r" (p) : "memory"); +} + +/* + * Flush all hypervisor mappings from the data TLB. This is not + * sufficient when changing code mappings or for self modifying code. + */ +static inline void flush_xen_data_tlb(void) +{ + register unsigned long r0 asm ("r0"); + asm volatile("dsb;" /* Ensure preceding are visible */ + STORE_CP32(0, TLBIALLH) + "dsb;" /* Ensure completion of the TLB flush */ + "isb;" + : : "r" (r0) /* dummy */: "memory"); +} + +/* + * Flush one VA's hypervisor mappings from the data TLB. This is not + * sufficient when changing code mappings or for self modifying code. + */ +static inline void flush_xen_data_tlb_va(unsigned long va) +{ + asm volatile("dsb;" /* Ensure preceding are visible */ + STORE_CP32(0, TLBIMVAH) + "dsb;" /* Ensure completion of the TLB flush */ + "isb;" + : : "r" (va) : "memory"); +} + +/* Flush all non-hypervisor mappings from the TLB */ +static inline void flush_guest_tlb(void) +{ + register unsigned long r0 asm ("r0"); + WRITE_CP32(r0 /* dummy */, TLBIALLNSNH); +} + +/* Ask the MMU to translate a VA for us */ +static inline uint64_t __va_to_par(uint32_t va) +{ + uint64_t par, tmp; + tmp = READ_CP64(PAR); + WRITE_CP32(va, ATS1HR); + isb(); /* Ensure result is available. */ + par = READ_CP64(PAR); + WRITE_CP64(tmp, PAR); + return par; +} + +static inline uint64_t va_to_par(uint32_t va) +{ + uint64_t par = __va_to_par(va); + /* It is not OK to call this with an invalid VA */ + if ( par & PAR_F ) panic_PAR(par, "Hypervisor"); + return par; +} + +/* Ask the MMU to translate a Guest VA for us */ +static inline uint64_t __gva_to_par(uint32_t va) +{ + uint64_t par, tmp; + tmp = READ_CP64(PAR); + WRITE_CP32(va, ATS12NSOPR); + isb(); /* Ensure result is available. */ + par = READ_CP64(PAR); + WRITE_CP64(tmp, PAR); + return par; +} +static inline uint64_t gva_to_par(uint32_t va) +{ + uint64_t par = __gva_to_par(va); + /* It is not OK to call this with an invalid VA */ + /* XXX harsh for a guest address... */ + if ( par & PAR_F ) panic_PAR(par, "Guest"); + return par; +} +static inline uint64_t __gva_to_ipa(uint32_t va) +{ + uint64_t par, tmp; + tmp = READ_CP64(PAR); + WRITE_CP32(va, ATS1CPR); + isb(); /* Ensure result is available. */ + par = READ_CP64(PAR); + WRITE_CP64(tmp, PAR); + return par; +} +static inline uint64_t gva_to_ipa(uint32_t va) +{ + uint64_t par = __gva_to_ipa(va); + /* It is not OK to call this with an invalid VA */ + /* XXX harsh for a guest address... */ + if ( par & PAR_F ) panic_PAR(par, "Guest"); + return (par & PADDR_MASK & PAGE_MASK) | ((unsigned long) va & ~PAGE_MASK); +} +/* Bits in the PAR returned by va_to_par */ +#define PAR_FAULT 0x1 + +#endif /* __ASSEMBLY__ */ + +/* These numbers add up to a 39-bit input address space. The ARMv7-A + * architecture actually specifies a 40-bit input address space for the p2m, + * with an 8K (1024-entry) top-level table. */ + +#define LPAE_SHIFT 9 +#define LPAE_ENTRIES (1u << LPAE_SHIFT) +#define LPAE_ENTRY_MASK (LPAE_ENTRIES - 1) + +#define THIRD_SHIFT PAGE_SHIFT +#define SECOND_SHIFT (THIRD_SHIFT + LPAE_SHIFT) +#define FIRST_SHIFT (SECOND_SHIFT + LPAE_SHIFT) + +/* Calculate the offsets into the pagetables for a given VA */ +#define first_linear_offset(va) (va >> FIRST_SHIFT) +#define second_linear_offset(va) (va >> SECOND_SHIFT) +#define third_linear_offset(va) (va >> THIRD_SHIFT) +#define first_table_offset(va) (first_linear_offset(va)) +#define second_table_offset(va) (second_linear_offset(va) & LPAE_ENTRY_MASK) +#define third_table_offset(va) (third_linear_offset(va) & LPAE_ENTRY_MASK) + +#define clear_page(page)memset((void *)(page), 0, PAGE_SIZE) + +#define PAGE_ALIGN(x) (((x) + PAGE_SIZE - 1) & PAGE_MASK) + +#endif /* __ARM_PAGE_H__ */ + +/* + * Local variables: + * mode: C + * c-set-style: "BSD" + * c-basic-offset: 4 + * tab-width: 4 + * indent-tabs-mode: nil + * End: + */ -- cgit v1.2.3