bitkeeper revision 1.1159.187.66 (41bf1718JfLUlcF63YjP4sfqtgAPWA)

Some more x86/64 progress...

17 files changed, 626 insertions, 75 deletions

diff --git a/.rootkeys b/.rootkeys
index b8615ff886..2d0c24586c 100644
--- a/.rootkeys
+++ b/.rootkeys
@@ -693,7 +693,9 @@
 40f92331jfOlE7MfKwpdkEb1CEf23g xen/arch/x86/x86_32/seg_fixup.c
 3ddb79bc4nTpGQOe6_-MbyZzkhlhFQ xen/arch/x86/x86_32/usercopy.c
 3ddb79bcOMCu9-5mKpjIh5d0qqBDPg xen/arch/x86/x86_32/xen.lds
+41bf1717Ty3hwN3E9swdu8QfnvGqww xen/arch/x86/x86_64/asm-offsets.c
 40e96d3aLDI-nViMuYneD7VKYlZrVg xen/arch/x86/x86_64/entry.S
+41bf1717XhPz_dNT5OKSjgmbFuWBuA xen/arch/x86/x86_64/mm.c
 40e96d3ahBTZqbTViInnq0lM03vs7A xen/arch/x86/x86_64/usercopy.c
 40e96d3akN3Hu_J5Bk-WXD8OGscrYQ xen/arch/x86/x86_64/xen.lds
 3ddb79bdff-gj-jFGKjOejeHLqL8Lg xen/common/Makefile
@@ -781,6 +783,7 @@
 40715b2dWe0tDhx9LkLXzTQkvD49RA xen/include/asm-x86/acpi.h
 3ddb79c3l4IiQtf6MS2jIzcd-hJS8g xen/include/asm-x86/apic.h
 3ddb79c3QJYWr8LLGdonLbWmNb9pQQ xen/include/asm-x86/apicdef.h
+41bf17171g_hhz2k4B-fN9LQlODDjQ xen/include/asm-x86/asm_defns.h
 3ddb79c3OiG9eTsi9Dy3F_OkuRAzKA xen/include/asm-x86/atomic.h
 3ddb79c3rM-Ote0Xn6Ytg8Y6YqAG-A xen/include/asm-x86/bitops.h
 3ddb79c3KhTI0F_Iw_hRL9QEyOVK-g xen/include/asm-x86/cache.h
@@ -827,6 +830,7 @@
 3ddb79c3mbqEM7QQr3zVq7NiBNhouA xen/include/asm-x86/x86_32/regs.h
 3e7f358aG11EvMI9VJ4_9hD4LUO7rQ xen/include/asm-x86/x86_32/string.h
 3ddb79c3M2n1ROZH6xk3HbyN4CPDqg xen/include/asm-x86/x86_32/uaccess.h
+41bf1717bML6GxpclTWJabiaO5W5vg xen/include/asm-x86/x86_64/asm_defns.h
 404f1b9ceJeGVaPNIENm2FkK0AgEOQ xen/include/asm-x86/x86_64/current.h
 404f1b9fl6AQ_a-T1TDK3fuwTPXmHw xen/include/asm-x86/x86_64/desc.h
 404f1badfXZJZ2sU8sh9PS2EZvd19Q xen/include/asm-x86/x86_64/ldt.h
diff --git a/xen/arch/x86/boot/x86_64.S b/xen/arch/x86/boot/x86_64.S
index b16cde8762..a8253a4ce1 100644
--- a/xen/arch/x86/boot/x86_64.S
+++ b/xen/arch/x86/boot/x86_64.S
@@ -257,29 +257,22 @@ copy_to_user:
 set_intr_gate:
 die:
 machine_to_phys_mapping:
-.globl copy_from_user, show_registers, __set_fixmap, do_iopl, check_descriptor
+.globl copy_from_user, show_registers, do_iopl
 copy_from_user: 
 show_registers: 
-__set_fixmap: 
 do_iopl: 
-check_descriptor:
-.globl set_gdt, idt_table, copy_user_generic, memcmp, idt_tables, new_thread
-set_gdt:
+.globl idt_table, copy_user_generic, memcmp, idt_tables, new_thread
 idt_table:
 copy_user_generic:
 memcmp:
 idt_tables:
 new_thread:
-.globl switch_to, __get_user_1, paging_init, trap_init
+.globl switch_to, __get_user_1, __get_user_4, __get_user_8, trap_init
 switch_to:
 __get_user_1:
-paging_init:
-trap_init: 
-.globl __get_user_8, zap_low_mappings, set_debugreg,synchronise_pagetables
+__get_user_4:
 __get_user_8:
-zap_low_mappings:
+trap_init: 
+.globl set_debugreg
 set_debugreg:
-synchronise_pagetables:
-.globl destroy_gdt
-destroy_gdt:    
         
diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 1881afa3fd..964062cb89 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -427,6 +427,8 @@ long do_iopl(domid_t domain, unsigned int new_io_pl)
     return 0;
 }
 
+#endif
+
 unsigned long hypercall_create_continuation(
     unsigned int op, unsigned int nr_args, ...)
 {
@@ -448,11 +450,15 @@ unsigned long hypercall_create_continuation(
     else
     {
         ec       = get_execution_context();
+#if defined(__i386__)
         ec->eax  = op;
         ec->eip -= 2;  /* re-execute 'int 0x82' */
         
         for ( i = 0, preg = &ec->ebx; i < nr_args; i++, preg++ )
             *preg = va_arg(args, unsigned long);
+#else
+        preg = NULL; /* XXX x86/64 */
+#endif
     }
 
     va_end(args);
@@ -460,9 +466,6 @@ unsigned long hypercall_create_continuation(
     return op;
 }
 
-#endif
-
-
 static void relinquish_list(struct domain *d, struct list_head *list)
 {
     struct list_head *ent;
diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index 842de24582..6837a143d3 100644
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -981,4 +981,10 @@ unsigned long do_get_debugreg(int reg)
     return current->thread.debugreg[reg];
 }
 
+#else
+
+asmlinkage void fatal_trap(int trapnr, struct xen_regs *regs)
+{
+}
+
 #endif /* __i386__ */
diff --git a/xen/arch/x86/x86_32/entry.S b/xen/arch/x86/x86_32/entry.S
index a3ed040538..d3df231fbb 100644
--- a/xen/arch/x86/x86_32/entry.S
+++ b/xen/arch/x86/x86_32/entry.S
@@ -56,7 +56,7 @@
 #include <xen/config.h>
 #include <xen/errno.h>
 #include <xen/softirq.h>
-#include <asm/x86_32/asm_defns.h>
+#include <asm/asm_defns.h>
 #include <public/xen.h>
 
 #define GET_CURRENT(reg)   \
diff --git a/xen/arch/x86/x86_64/asm-offsets.c b/xen/arch/x86/x86_64/asm-offsets.c
new file mode 100644
index 0000000000..2e6c3b396e
--- /dev/null
+++ b/xen/arch/x86/x86_64/asm-offsets.c
@@ -0,0 +1,71 @@
+/*
+ * Generate definitions needed by assembly language modules.
+ * This code generates raw asm output which is post-processed
+ * to extract and format the required data.
+ */
+
+#include <xen/sched.h>
+
+#define DEFINE(_sym, _val) \
+    __asm__ __volatile__ ( "\n->" #_sym " %0 " #_val : : "i" (_val) )
+#define BLANK() \
+    __asm__ __volatile__ ( "\n->" : : )
+#define OFFSET(_sym, _str, _mem) \
+    DEFINE(_sym, offsetof(_str, _mem));
+
+void __dummy__(void)
+{
+    OFFSET(XREGS_r15, struct xen_regs, r15);
+    OFFSET(XREGS_r14, struct xen_regs, r14);
+    OFFSET(XREGS_r13, struct xen_regs, r13);
+    OFFSET(XREGS_r12, struct xen_regs, r12);
+    OFFSET(XREGS_rbp, struct xen_regs, rbp);
+    OFFSET(XREGS_rbx, struct xen_regs, rbx);
+    OFFSET(XREGS_r11, struct xen_regs, r11);
+    OFFSET(XREGS_r10, struct xen_regs, r10);
+    OFFSET(XREGS_r9, struct xen_regs, r9);
+    OFFSET(XREGS_r8, struct xen_regs, r8);
+    OFFSET(XREGS_rax, struct xen_regs, rax);
+    OFFSET(XREGS_rcx, struct xen_regs, rcx);
+    OFFSET(XREGS_rdx, struct xen_regs, rdx);
+    OFFSET(XREGS_rsi, struct xen_regs, rsi);
+    OFFSET(XREGS_rdi, struct xen_regs, rdi);
+    OFFSET(XREGS_orig_rax, struct xen_regs, orig_rax);
+    OFFSET(XREGS_rip, struct xen_regs, rip);
+    OFFSET(XREGS_cs, struct xen_regs, cs);
+    OFFSET(XREGS_eflags, struct xen_regs, eflags);
+    OFFSET(XREGS_rsp, struct xen_regs, rsp);
+    OFFSET(XREGS_ss, struct xen_regs, ss);
+    BLANK();
+
+    OFFSET(DOMAIN_processor, struct domain, processor);
+    OFFSET(DOMAIN_shared_info, struct domain, shared_info);
+    OFFSET(DOMAIN_event_sel, struct domain, thread.event_selector);
+    OFFSET(DOMAIN_event_addr, struct domain, thread.event_address);
+    OFFSET(DOMAIN_failsafe_sel, struct domain, thread.failsafe_selector);
+    OFFSET(DOMAIN_failsafe_addr, struct domain, thread.failsafe_address);
+    OFFSET(DOMAIN_trap_bounce, struct domain, thread.trap_bounce);
+    OFFSET(DOMAIN_thread_flags, struct domain, thread.flags);
+    BLANK();
+
+    OFFSET(SHINFO_upcall_pending, shared_info_t, 
+           vcpu_data[0].evtchn_upcall_pending);
+    OFFSET(SHINFO_upcall_mask, shared_info_t, 
+           vcpu_data[0].evtchn_upcall_mask);
+    BLANK();
+
+    OFFSET(TRAPBOUNCE_error_code, struct trap_bounce, error_code);
+    OFFSET(TRAPBOUNCE_cr2, struct trap_bounce, cr2);
+    OFFSET(TRAPBOUNCE_flags, struct trap_bounce, flags);
+    OFFSET(TRAPBOUNCE_cs, struct trap_bounce, cs);
+    OFFSET(TRAPBOUNCE_eip, struct trap_bounce, eip);
+    BLANK();
+
+    OFFSET(MULTICALL_op, multicall_entry_t, op);
+    OFFSET(MULTICALL_arg0, multicall_entry_t, args[0]);
+    OFFSET(MULTICALL_arg1, multicall_entry_t, args[1]);
+    OFFSET(MULTICALL_arg2, multicall_entry_t, args[2]);
+    OFFSET(MULTICALL_arg3, multicall_entry_t, args[3]);
+    OFFSET(MULTICALL_arg4, multicall_entry_t, args[4]);
+    OFFSET(MULTICALL_result, multicall_entry_t, args[5]);
+}
diff --git a/xen/arch/x86/x86_64/mm.c b/xen/arch/x86/x86_64/mm.c
new file mode 100644
index 0000000000..c7bcb17805
--- /dev/null
+++ b/xen/arch/x86/x86_64/mm.c
@@ -0,0 +1,455 @@
+/******************************************************************************
+ * arch/x86/x86_64/mm.c
+ * 
+ * Modifications to Linux original are copyright (c) 2004, K A Fraser
+ * 
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#include <xen/config.h>
+#include <xen/lib.h>
+#include <xen/init.h>
+#include <xen/mm.h>
+#include <asm/page.h>
+#include <asm/flushtlb.h>
+#include <asm/fixmap.h>
+#include <asm/domain_page.h>
+
+static inline void set_pte_phys(unsigned long vaddr,
+                                l1_pgentry_t entry)
+{
+    l4_pgentry_t *l4ent;
+    l3_pgentry_t *l3ent;
+    l2_pgentry_t *l2ent;
+    l1_pgentry_t *l1ent;
+
+    l4ent = &idle_pg_table[l4_table_offset(vaddr)];
+    l3ent = l4_pgentry_to_l3(*l4ent) + l3_table_offset(vaddr);
+    l2ent = l3_pgentry_to_l2(*l3ent) + l2_table_offset(vaddr);
+    l1ent = l2_pgentry_to_l1(*l2ent) + l1_table_offset(vaddr);
+    *l1ent = entry;
+
+    /* It's enough to flush this one mapping. */
+    __flush_tlb_one(vaddr);
+}
+
+
+void __set_fixmap(enum fixed_addresses idx, 
+                  l1_pgentry_t entry)
+{
+    unsigned long address = fix_to_virt(idx);
+
+    if ( likely(idx < __end_of_fixed_addresses) )
+        set_pte_phys(address, entry);
+    else
+        printk("Invalid __set_fixmap\n");
+}
+
+
+void __init paging_init(void)
+{
+    void *ioremap_pt;
+    int i;
+
+    /* Create page table for ioremap(). */
+    ioremap_pt = (void *)alloc_xenheap_page();
+    clear_page(ioremap_pt);
+    idle_pg_table[IOREMAP_VIRT_START >> L2_PAGETABLE_SHIFT] = 
+        mk_l2_pgentry(__pa(ioremap_pt) | __PAGE_HYPERVISOR);
+
+    /* Create read-only mapping of MPT for guest-OS use. */
+    idle_pg_table[RO_MPT_VIRT_START >> L2_PAGETABLE_SHIFT] =
+        mk_l2_pgentry(l2_pgentry_val(
+            idle_pg_table[RDWR_MPT_VIRT_START >> L2_PAGETABLE_SHIFT]) & 
+                      ~_PAGE_RW);
+
+    /* Set up mapping cache for domain pages. */
+    mapcache = (unsigned long *)alloc_xenheap_page();
+    clear_page(mapcache);
+    idle_pg_table[MAPCACHE_VIRT_START >> L2_PAGETABLE_SHIFT] =
+        mk_l2_pgentry(__pa(mapcache) | __PAGE_HYPERVISOR);
+
+    /* Set up linear page table mapping. */
+    idle_pg_table[LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT] =
+        mk_l2_pgentry(__pa(idle_pg_table) | __PAGE_HYPERVISOR);
+
+}
+
+void __init zap_low_mappings(void)
+{
+    idle_pg_table[0] = 0;
+}
+
+
+/*
+ * Allows shooting down of borrowed page-table use on specific CPUs.
+ * Specifically, we borrow page tables when running the idle domain.
+ */
+static void __synchronise_pagetables(void *mask)
+{
+    struct domain *d = current;
+    if ( ((unsigned long)mask & (1<<d->processor)) && is_idle_task(d) )
+        write_ptbase(&d->mm);
+}
+void synchronise_pagetables(unsigned long cpu_mask)
+{
+    __synchronise_pagetables((void *)cpu_mask);
+    smp_call_function(__synchronise_pagetables, (void *)cpu_mask, 1, 1);
+}
+
+long do_stack_switch(unsigned long ss, unsigned long esp)
+{
+    int nr = smp_processor_id();
+    struct tss_struct *t = &init_tss[nr];
+
+    /* We need to do this check as we load and use SS on guest's behalf. */
+    if ( (ss & 3) == 0 )
+        return -EPERM;
+
+    current->thread.guestos_ss = ss;
+    current->thread.guestos_sp = esp;
+    t->ss1  = ss;
+    t->esp1 = esp;
+
+    return 0;
+}
+
+
+/* Returns TRUE if given descriptor is valid for GDT or LDT. */
+int check_descriptor(unsigned long *d)
+{
+    unsigned long base, limit, a = d[0], b = d[1];
+
+    /* A not-present descriptor will always fault, so is safe. */
+    if ( !(b & _SEGMENT_P) ) 
+        goto good;
+
+    /*
+     * We don't allow a DPL of zero. There is no legitimate reason for 
+     * specifying DPL==0, and it gets rather dangerous if we also accept call 
+     * gates (consider a call gate pointing at another guestos descriptor with 
+     * DPL 0 -- this would get the OS ring-0 privileges).
+     */
+    if ( (b & _SEGMENT_DPL) == 0 )
+        goto bad;
+
+    if ( !(b & _SEGMENT_S) )
+    {
+        /*
+         * System segment:
+         *  1. Don't allow interrupt or trap gates as they belong in the IDT.
+         *  2. Don't allow TSS descriptors or task gates as we don't
+         *     virtualise x86 tasks.
+         *  3. Don't allow LDT descriptors because they're unnecessary and
+         *     I'm uneasy about allowing an LDT page to contain LDT
+         *     descriptors. In any case, Xen automatically creates the
+         *     required descriptor when reloading the LDT register.
+         *  4. We allow call gates but they must not jump to a private segment.
+         */
+
+        /* Disallow everything but call gates. */
+        if ( (b & _SEGMENT_TYPE) != 0xc00 )
+            goto bad;
+
+        /* Can't allow far jump to a Xen-private segment. */
+        if ( !VALID_CODESEL(a>>16) )
+            goto bad;
+
+        /* Reserved bits must be zero. */
+        if ( (b & 0xe0) != 0 )
+            goto bad;
+        
+        /* No base/limit check is needed for a call gate. */
+        goto good;
+    }
+    
+    /* Check that base is at least a page away from Xen-private area. */
+    base  = (b&(0xff<<24)) | ((b&0xff)<<16) | (a>>16);
+    if ( base >= (PAGE_OFFSET - PAGE_SIZE) )
+        goto bad;
+
+    /* Check and truncate the limit if necessary. */
+    limit = (b&0xf0000) | (a&0xffff);
+    limit++; /* We add one because limit is inclusive. */
+    if ( (b & _SEGMENT_G) )
+        limit <<= 12;
+
+    if ( (b & (_SEGMENT_CODE | _SEGMENT_EC)) == _SEGMENT_EC )
+    {
+        /*
+         * Grows-down limit check. 
+         * NB. limit == 0xFFFFF provides no access      (if G=1).
+         *     limit == 0x00000 provides 4GB-4kB access (if G=1).
+         */
+        if ( (base + limit) > base )
+        {
+            limit = -(base & PAGE_MASK);
+            goto truncate;
+        }
+    }
+    else
+    {
+        /*
+         * Grows-up limit check.
+         * NB. limit == 0xFFFFF provides 4GB access (if G=1).
+         *     limit == 0x00000 provides 4kB access (if G=1).
+         */
+        if ( ((base + limit) <= base) || 
+             ((base + limit) > PAGE_OFFSET) )
+        {
+            limit = PAGE_OFFSET - base;
+        truncate:
+            if ( !(b & _SEGMENT_G) )
+                goto bad; /* too dangerous; too hard to work out... */
+            limit = (limit >> 12) - 1;
+            d[0] &= ~0x0ffff; d[0] |= limit & 0x0ffff;
+            d[1] &= ~0xf0000; d[1] |= limit & 0xf0000;
+        }
+    }
+
+ good:
+    return 1;
+ bad:
+    return 0;
+}
+
+
+void destroy_gdt(struct domain *d)
+{
+    int i;
+    unsigned long pfn;
+
+    for ( i = 0; i < 16; i++ )
+    {
+        if ( (pfn = l1_pgentry_to_pagenr(d->mm.perdomain_pt[i])) != 0 )
+            put_page_and_type(&frame_table[pfn]);
+        d->mm.perdomain_pt[i] = mk_l1_pgentry(0);
+    }
+}
+
+
+long set_gdt(struct domain *d, 
+             unsigned long *frames,
+             unsigned int entries)
+{
+    /* NB. There are 512 8-byte entries per GDT page. */
+    int i = 0, nr_pages = (entries + 511) / 512;
+    struct desc_struct *vgdt;
+    unsigned long pfn;
+
+    /* Check the first page in the new GDT. */
+    if ( (pfn = frames[0]) >= max_page )
+        goto fail;
+
+    /* The first page is special because Xen owns a range of entries in it. */
+    if ( !get_page_and_type(&frame_table[pfn], d, PGT_gdt_page) )
+    {
+        /* GDT checks failed: try zapping the Xen reserved entries. */
+        if ( !get_page_and_type(&frame_table[pfn], d, PGT_writable_page) )
+            goto fail;
+        vgdt = map_domain_mem(pfn << PAGE_SHIFT);
+        memset(vgdt + FIRST_RESERVED_GDT_ENTRY, 0,
+               NR_RESERVED_GDT_ENTRIES*8);
+        unmap_domain_mem(vgdt);
+        put_page_and_type(&frame_table[pfn]);
+
+        /* Okay, we zapped the entries. Now try the GDT checks again. */
+        if ( !get_page_and_type(&frame_table[pfn], d, PGT_gdt_page) )
+            goto fail;
+    }
+
+    /* Check the remaining pages in the new GDT. */
+    for ( i = 1; i < nr_pages; i++ )
+        if ( ((pfn = frames[i]) >= max_page) ||
+             !get_page_and_type(&frame_table[pfn], d, PGT_gdt_page) )
+            goto fail;
+
+    /* Copy reserved GDT entries to the new GDT. */
+    vgdt = map_domain_mem(frames[0] << PAGE_SHIFT);
+    memcpy(vgdt + FIRST_RESERVED_GDT_ENTRY, 
+           gdt_table + FIRST_RESERVED_GDT_ENTRY, 
+           NR_RESERVED_GDT_ENTRIES*8);
+    unmap_domain_mem(vgdt);
+
+    /* Tear down the old GDT. */
+    destroy_gdt(d);
+
+    /* Install the new GDT. */
+    for ( i = 0; i < nr_pages; i++ )
+        d->mm.perdomain_pt[i] =
+            mk_l1_pgentry((frames[i] << PAGE_SHIFT) | __PAGE_HYPERVISOR);
+
+    SET_GDT_ADDRESS(d, GDT_VIRT_START);
+    SET_GDT_ENTRIES(d, entries);
+
+    return 0;
+
+ fail:
+    while ( i-- > 0 )
+        put_page_and_type(&frame_table[frames[i]]);
+    return -EINVAL;
+}
+
+
+long do_set_gdt(unsigned long *frame_list, unsigned int entries)
+{
+    int nr_pages = (entries + 511) / 512;
+    unsigned long frames[16];
+    long ret;
+
+    if ( (entries <= LAST_RESERVED_GDT_ENTRY) || (entries > 8192) ) 
+        return -EINVAL;
+    
+    if ( copy_from_user(frames, frame_list, nr_pages * sizeof(unsigned long)) )
+        return -EFAULT;
+
+    if ( (ret = set_gdt(current, frames, entries)) == 0 )
+    {
+        local_flush_tlb();
+        __asm__ __volatile__ ("lgdt %0" : "=m" (*current->mm.gdt));
+    }
+
+    return ret;
+}
+
+
+long do_update_descriptor(
+    unsigned long pa, unsigned long word1, unsigned long word2)
+{
+    unsigned long *gdt_pent, pfn = pa >> PAGE_SHIFT, d[2];
+    struct pfn_info *page;
+    long ret = -EINVAL;
+
+    d[0] = word1;
+    d[1] = word2;
+
+    if ( (pa & 7) || (pfn >= max_page) || !check_descriptor(d) )
+        return -EINVAL;
+
+    page = &frame_table[pfn];
+    if ( unlikely(!get_page(page, current)) )
+        return -EINVAL;
+
+    /* Check if the given frame is in use in an unsafe context. */
+    switch ( page->u.inuse.type_info & PGT_type_mask )
+    {
+    case PGT_gdt_page:
+        /* Disallow updates of Xen-reserved descriptors in the current GDT. */
+        if ( (l1_pgentry_to_pagenr(current->mm.perdomain_pt[0]) == pfn) &&
+             (((pa&(PAGE_SIZE-1))>>3) >= FIRST_RESERVED_GDT_ENTRY) &&
+             (((pa&(PAGE_SIZE-1))>>3) <= LAST_RESERVED_GDT_ENTRY) )
+            goto out;
+        if ( unlikely(!get_page_type(page, PGT_gdt_page)) )
+            goto out;
+        break;
+    case PGT_ldt_page:
+        if ( unlikely(!get_page_type(page, PGT_ldt_page)) )
+            goto out;
+        break;
+    default:
+        if ( unlikely(!get_page_type(page, PGT_writable_page)) )
+            goto out;
+        break;
+    }
+
+    /* All is good so make the update. */
+    gdt_pent = map_domain_mem(pa);
+    memcpy(gdt_pent, d, 8);
+    unmap_domain_mem(gdt_pent);
+
+    put_page_type(page);
+
+    ret = 0; /* success */
+
+ out:
+    put_page(page);
+    return ret;
+}
+
+#ifdef MEMORY_GUARD
+
+void *memguard_init(void *heap_start)
+{
+    l1_pgentry_t *l1;
+    int i, j;
+
+    /* Round the allocation pointer up to a page boundary. */
+    heap_start = (void *)(((unsigned long)heap_start + (PAGE_SIZE-1)) & 
+                          PAGE_MASK);
+
+    /* Memory guarding is incompatible with super pages. */
+    for ( i = 0; i < (xenheap_phys_end >> L2_PAGETABLE_SHIFT); i++ )
+    {
+        l1 = (l1_pgentry_t *)heap_start;
+        heap_start = (void *)((unsigned long)heap_start + PAGE_SIZE);
+        for ( j = 0; j < ENTRIES_PER_L1_PAGETABLE; j++ )
+            l1[j] = mk_l1_pgentry((i << L2_PAGETABLE_SHIFT) |
+                                   (j << L1_PAGETABLE_SHIFT) | 
+                                  __PAGE_HYPERVISOR);
+        idle_pg_table[i] = idle_pg_table[i + l2_table_offset(PAGE_OFFSET)] =
+            mk_l2_pgentry(virt_to_phys(l1) | __PAGE_HYPERVISOR);
+    }
+
+    return heap_start;
+}
+
+static void __memguard_change_range(void *p, unsigned long l, int guard)
+{
+    l1_pgentry_t *l1;
+    l2_pgentry_t *l2;
+    unsigned long _p = (unsigned long)p;
+    unsigned long _l = (unsigned long)l;
+
+    /* Ensure we are dealing with a page-aligned whole number of pages. */
+    ASSERT((_p&PAGE_MASK) != 0);
+    ASSERT((_l&PAGE_MASK) != 0);
+    ASSERT((_p&~PAGE_MASK) == 0);
+    ASSERT((_l&~PAGE_MASK) == 0);
+
+    while ( _l != 0 )
+    {
+        l2  = &idle_pg_table[l2_table_offset(_p)];
+        l1  = l2_pgentry_to_l1(*l2) + l1_table_offset(_p);
+        if ( guard )
+            *l1 = mk_l1_pgentry(l1_pgentry_val(*l1) & ~_PAGE_PRESENT);
+        else
+            *l1 = mk_l1_pgentry(l1_pgentry_val(*l1) | _PAGE_PRESENT);
+        _p += PAGE_SIZE;
+        _l -= PAGE_SIZE;
+    }
+}
+
+void memguard_guard_range(void *p, unsigned long l)
+{
+    __memguard_change_range(p, l, 1);
+    local_flush_tlb();
+}
+
+void memguard_unguard_range(void *p, unsigned long l)
+{
+    __memguard_change_range(p, l, 0);
+}
+
+int memguard_is_guarded(void *p)
+{
+    l1_pgentry_t *l1;
+    l2_pgentry_t *l2;
+    unsigned long _p = (unsigned long)p;
+    l2  = &idle_pg_table[l2_table_offset(_p)];
+    l1  = l2_pgentry_to_l1(*l2) + l1_table_offset(_p);
+    return !(l1_pgentry_val(*l1) & _PAGE_PRESENT);
+}
+
+#endif
diff --git a/xen/arch/x86/x86_64/xen.lds b/xen/arch/x86/x86_64/xen.lds
index 4763fdf55a..8a6b3a27ac 100644
--- a/xen/arch/x86/x86_64/xen.lds
+++ b/xen/arch/x86/x86_64/xen.lds
@@ -28,6 +28,11 @@ SECTIONS
   __ex_table : { *(__ex_table) } :text
   __stop___ex_table = .;
 
+  . = ALIGN(16);                /* Pre-exception table */
+  __start___pre_ex_table = .;
+  __pre_ex_table : { *(__pre_ex_table) } :text
+  __stop___pre_ex_table = .;
+
   __start___ksymtab = .;	/* Kernel symbol table */
   __ksymtab : { *(__ksymtab) } :text
   __stop___ksymtab = .;
diff --git a/xen/include/asm-x86/asm_defns.h b/xen/include/asm-x86/asm_defns.h
new file mode 100644
index 0000000000..0fb3e44727
--- /dev/null
+++ b/xen/include/asm-x86/asm_defns.h
@@ -0,0 +1,18 @@
+
+#ifndef __X86_ASM_DEFNS_H__
+#define __X86_ASM_DEFNS_H__
+
+/* NB. Auto-generated from arch/.../asm-offsets.c */
+#include <asm/asm-offsets.h>
+#include <asm/processor.h>
+
+#define __STR(x) #x
+#define STR(x) __STR(x)
+
+#ifdef __x86_64__
+#include <asm/x86_64/asm_defns.h>
+#else
+#include <asm/x86_32/asm_defns.h>
+#endif
+
+#endif /* __X86_ASM_DEFNS_H__ */
diff --git a/xen/include/asm-x86/irq.h b/xen/include/asm-x86/irq.h
index b07fef7809..2779282659 100644
--- a/xen/include/asm-x86/irq.h
+++ b/xen/include/asm-x86/irq.h
@@ -5,7 +5,7 @@
 
 #include <xen/config.h>
 #include <asm/atomic.h>
-#include <asm/x86_32/asm_defns.h>
+#include <asm/asm_defns.h>
 
 extern void disable_irq(unsigned int);
 extern void disable_irq_nosync(unsigned int);
diff --git a/xen/include/asm-x86/multicall.h b/xen/include/asm-x86/multicall.h
index e1a7770354..d03ac9ffb1 100644
--- a/xen/include/asm-x86/multicall.h
+++ b/xen/include/asm-x86/multicall.h
@@ -5,7 +5,13 @@
 #ifndef __ASM_X86_MULTICALL_H__
 #define __ASM_X86_MULTICALL_H__
 
-#include <asm-x86/x86_32/asm_defns.h>
+#include <asm/asm_defns.h>
+
+#ifdef __x86_64__
+
+#define do_multicall_call(_call) BUG()
+
+#else
 
 #define do_multicall_call(_call)                       \
     do {                                               \
@@ -23,4 +29,6 @@
             : : "b" (_call) : "eax", "ecx", "edx" );   \
     } while ( 0 )
 
+#endif
+
 #endif /* __ASM_X86_MULTICALL_H__ */
diff --git a/xen/include/asm-x86/processor.h b/xen/include/asm-x86/processor.h
index 6d3cf3036a..ae5b13b7d1 100644
--- a/xen/include/asm-x86/processor.h
+++ b/xen/include/asm-x86/processor.h
@@ -254,18 +254,18 @@ static inline unsigned int cpuid_edx(unsigned int op)
 })
 
 #define write_cr0(x) \
-	__asm__("mov"__OS" %0,%%cr0": :"r" (x));
+	__asm__("mov"__OS" %0,%%cr0": :"r" ((unsigned long)x));
 
 #define read_cr4() ({ \
-	unsigned int __dummy; \
+	unsigned long __dummy; \
 	__asm__( \
-		"movl %%cr4,%0\n\t" \
+		"mov"__OS" %%cr4,%0\n\t" \
 		:"=r" (__dummy)); \
 	__dummy; \
 })
 
 #define write_cr4(x) \
-	__asm__("movl %0,%%cr4": :"r" (x));
+	__asm__("mov"__OS" %0,%%cr4": :"r" ((unsigned long)x));
 
 /*
  * Save the cr4 feature set we're using (ie
@@ -290,7 +290,7 @@ static inline void clear_in_cr4 (unsigned long mask)
     mmu_cr4_features &= ~mask;
     __asm__("mov"__OS" %%cr4,%%"__OP"ax\n\t"
             "and"__OS" %0,%%"__OP"ax\n\t"
-            "movl"__OS" %%"__OP"ax,%%cr4\n"
+            "mov"__OS" %%"__OP"ax,%%cr4\n"
             : : "irg" (~mask)
             :"ax");
 }
diff --git a/xen/include/asm-x86/uaccess.h b/xen/include/asm-x86/uaccess.h
index 52a8b25c56..46c02ecef4 100644
--- a/xen/include/asm-x86/uaccess.h
+++ b/xen/include/asm-x86/uaccess.h
@@ -1,6 +1,32 @@
 
+#ifndef __X86_UACCESS_H__
+#define __X86_UACCESS_H__
+
 #ifdef __x86_64__
 #include <asm/x86_64/uaccess.h>
 #else
 #include <asm/x86_32/uaccess.h>
 #endif
+
+/*
+ * The exception table consists of pairs of addresses: the first is the
+ * address of an instruction that is allowed to fault, and the second is
+ * the address at which the program should continue.  No registers are
+ * modified, so it is entirely up to the continuation code to figure out
+ * what to do.
+ *
+ * All the routines below use bits of fixup code that are out of line
+ * with the main instruction path.  This means when everything is well,
+ * we don't even have to jump over them.  Further, they do not intrude
+ * on our cache or tlb entries.
+ */
+
+struct exception_table_entry
+{
+	unsigned long insn, fixup;
+};
+
+extern unsigned long search_exception_table(unsigned long);
+extern void sort_exception_tables(void);
+
+#endif /* __X86_UACCESS_H__ */
diff --git a/xen/include/asm-x86/x86_32/asm_defns.h b/xen/include/asm-x86/x86_32/asm_defns.h
index 8231bb2dec..e11ea34964 100644
--- a/xen/include/asm-x86/x86_32/asm_defns.h
+++ b/xen/include/asm-x86/x86_32/asm_defns.h
@@ -1,12 +1,5 @@
-#ifndef __ASM_DEFNS_H__
-#define __ASM_DEFNS_H__
-
-/* NB. Auto-generated from arch/.../asm-offsets.c */
-#include <asm/asm-offsets.h>
-#include <asm/processor.h>
-
-#define __STR(x) #x
-#define STR(x) __STR(x)
+#ifndef __X86_32_ASM_DEFNS_H__
+#define __X86_32_ASM_DEFNS_H__
 
 /* Maybe auto-generate the following two cases (quoted vs. unquoted). */
 #ifndef __ASSEMBLY__
@@ -85,4 +78,4 @@
 
 #endif
 
-#endif /* __ASM_DEFNS_H__ */
+#endif /* __X86_32_ASM_DEFNS_H__ */
diff --git a/xen/include/asm-x86/x86_32/uaccess.h b/xen/include/asm-x86/x86_32/uaccess.h
index 650492d59e..b202a1a12b 100644
--- a/xen/include/asm-x86/x86_32/uaccess.h
+++ b/xen/include/asm-x86/x86_32/uaccess.h
@@ -69,27 +69,6 @@ extern struct movsl_mask {
 #define array_access_ok(type,addr,count,size) \
     (likely(count < (~0UL/size)) && access_ok(type,addr,count*size))
 
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue.  No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path.  This means when everything is well,
- * we don't even have to jump over them.  Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry
-{
-	unsigned long insn, fixup;
-};
-
-extern unsigned long search_exception_table(unsigned long);
-extern void sort_exception_tables(void);
-
 /**
  * get_user: - Get a simple variable from user space.
  * @x:   Variable to store result.
diff --git a/xen/include/asm-x86/x86_64/asm_defns.h b/xen/include/asm-x86/x86_64/asm_defns.h
new file mode 100644
index 0000000000..fa0b978304
--- /dev/null
+++ b/xen/include/asm-x86/x86_64/asm_defns.h
@@ -0,0 +1,6 @@
+#ifndef __X86_64_ASM_DEFNS_H__
+#define __X86_64_ASM_DEFNS_H__
+
+#define SAVE_ALL(_r) ""
+
+#endif /* __X86_64_ASM_DEFNS_H__ */
diff --git a/xen/include/asm-x86/x86_64/uaccess.h b/xen/include/asm-x86/x86_64/uaccess.h
index be49ff870b..f965c87d32 100644
--- a/xen/include/asm-x86/x86_64/uaccess.h
+++ b/xen/include/asm-x86/x86_64/uaccess.h
@@ -35,31 +35,15 @@
 
 #define access_ok(type, addr, size) (__range_not_ok(addr,size) == 0)
 
+#define array_access_ok(type,addr,count,size)                    \
+    (likely(sizeof(count) <= 4) /* disallow 64-bit counts */ &&  \
+     access_ok(type,addr,count*size))
+
 extern inline int verify_area(int type, const void __user * addr, unsigned long size)
 {
 	return access_ok(type,addr,size) ? 0 : -EFAULT;
 }
 
-
-/*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue.  No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
- *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path.  This means when everything is well,
- * we don't even have to jump over them.  Further, they do not intrude
- * on our cache or tlb entries.
- */
-
-struct exception_table_entry
-{
-	unsigned long insn, fixup;
-};
-
-
 /*
  * These are the main single-value transfer routines.  They automatically
  * use the right size if we just have the right pointer type.