bitkeeper revision 1.1704 (42a885cb3_kkMMOYtYN_PaWP3tupbA)

Merge Scotts's pte_mfn patch with some extra merging between xen/i386
and xen/x86_64. An obvious further step is to fix xen/x86_64's
pci-dma.c and pci-nommu.c to be closer to native x86_64 versions.
Signed-off-by: Scott Parish <srparish@us.ibm.com>
Signed-off-by: Keir Fraser <keir@xensource.com>

18 files changed, 165 insertions, 479 deletions

diff --git a/.rootkeys b/.rootkeys
index 6be475bc10..a7745827e3 100644
--- a/.rootkeys
+++ b/.rootkeys
@@ -291,7 +291,6 @@
 424efaa6xbX9LkKyaXvgbL9s_39Trw linux-2.6.11-xen-sparse/arch/xen/x86_64/kernel/x8664_ksyms.c
 424efaa670zlQTtnOYK_aNgqhmSx-Q linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/Makefile
 424efaa6HUC68-hBHgiWOMDfKZogIA linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/fault.c
-424f001d6S-OiHsk1n9gv1jGwws-9w linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/hypervisor.c
 424efaa65ELRJ3JfgQQKLzW6y0ECYQ linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/init.c
 424efaa60dTbHfv65JBLVhNLcNPcRA linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/ioremap.c
 424efaa6uMX8YJASAVJT8ral74dz9Q linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/pageattr.c
@@ -400,7 +399,6 @@
 424efa20fTFqmaE0stH6lfB_4yN_lA linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/ptrace.h
 424efa21fY4IvK0luYgDJHKV-MD3eQ linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/segment.h
 424efa21KcupuJlHgmPiTk_T214FrA linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/smp.h
-424f001eT-7wjT_7K5AMlNVigIcuHA linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/synch_bitops.h
 424efa210ZRt2U_8WmtyI7g74Nz-4Q linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/system.h
 424f001eBp9fMbZ0Mo2kRJQ84gMgRw linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/timer.h
 424efa21Xk2acvaHYnpyTCLE6nU6hw linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/tlbflush.h
@@ -415,6 +413,7 @@
 3f108af1ylCIm82H052FVTfXACBHrw linux-2.6.11-xen-sparse/include/asm-xen/linux-public/privcmd.h
 3fa8e3f0kBLeE4To2vpdi3cpJbIkbQ linux-2.6.11-xen-sparse/include/asm-xen/linux-public/suspend.h
 4122466356eIBnC9ot44WSVVIFyhQA linux-2.6.11-xen-sparse/include/asm-xen/queues.h
+42a885cawNQ18_b7i5-G7ekMsZ48hw linux-2.6.11-xen-sparse/include/asm-xen/synch_bitops.h
 3f689063BoW-HWV3auUJ-OqXfcGArw linux-2.6.11-xen-sparse/include/asm-xen/xen_proc.h
 419b4e93z2S0gR17XTy8wg09JEwAhg linux-2.6.11-xen-sparse/include/linux/gfp.h
 42305f545Vc5SLCUewZ2-n-P9JJhEQ linux-2.6.11-xen-sparse/include/linux/highmem.h
diff --git a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c
index 0716f0c62b..dc51c7972a 100644
--- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c
+++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c
@@ -24,53 +24,6 @@ struct dma_coherent_mem {
 	unsigned long	*bitmap;
 };
 
-static void
-xen_contig_memory(unsigned long vstart, unsigned int order)
-{
-	/*
-	 * Ensure multi-page extents are contiguous in machine memory.
-	 * This code could be cleaned up some, and the number of
-	 * hypercalls reduced.
-	 */
-	pgd_t         *pgd; 
-	pud_t         *pud; 
-	pmd_t         *pmd;
-	pte_t         *pte;
-	unsigned long  pfn, i, flags;
-
-	scrub_pages(vstart, 1 << order);
-
-        balloon_lock(flags);
-
-	/* 1. Zap current PTEs, giving away the underlying pages. */
-	for (i = 0; i < (1<<order); i++) {
-		pgd = pgd_offset_k(vstart + (i*PAGE_SIZE));
-		pud = pud_offset(pgd, vstart + (i*PAGE_SIZE));
-		pmd = pmd_offset(pud, vstart + (i*PAGE_SIZE));
-		pte = pte_offset_kernel(pmd, vstart + (i*PAGE_SIZE));
-		pfn = pte_val_ma(*pte) >> PAGE_SHIFT;
-		HYPERVISOR_update_va_mapping(vstart + (i*PAGE_SIZE),
-					     __pte_ma(0), 0);
-		phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
-			INVALID_P2M_ENTRY;
-		if (HYPERVISOR_dom_mem_op(MEMOP_decrease_reservation, 
-					  &pfn, 1, 0) != 1) BUG();
-	}
-	/* 2. Get a new contiguous memory extent. */
-	if (HYPERVISOR_dom_mem_op(MEMOP_increase_reservation,
-				  &pfn, 1, order) != 1) BUG();
-	/* 3. Map the new extent in place of old pages. */
-	for (i = 0; i < (1<<order); i++) {
-		HYPERVISOR_update_va_mapping(vstart + (i*PAGE_SIZE),
-			__pte_ma(((pfn+i)<<PAGE_SHIFT)|__PAGE_KERNEL), 0);
-		xen_machphys_update(pfn+i, (__pa(vstart)>>PAGE_SHIFT)+i);
-		phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] = pfn+i;
-	}
-	flush_tlb_all();
-
-	balloon_unlock(flags);
-}
-
 void *dma_alloc_coherent(struct device *dev, size_t size,
 			   dma_addr_t *dma_handle, int gfp)
 {
diff --git a/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c b/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c
index 641bca708d..cf0488175f 100644
--- a/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c
+++ b/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c
@@ -44,8 +44,10 @@
 #define pte_offset_kernel pte_offset
 #define pud_t pgd_t
 #define pud_offset(d, va) d
+#elif defined(CONFIG_X86_64)
+#define pmd_val_ma(v) (v).pmd
 #else
-#define pmd_val_ma(v) (v).pud.pgd.pgd;
+#define pmd_val_ma(v) (v).pud.pgd.pgd
 #endif
 
 #ifndef CONFIG_XEN_SHADOW_MODE
@@ -64,7 +66,25 @@ void xen_l2_entry_update(pmd_t *ptr, pmd_t val)
     u.val = pmd_val_ma(val);
     BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
 }
-#endif
+
+#ifdef CONFIG_X86_64
+void xen_l3_entry_update(pud_t *ptr, pud_t val)
+{
+    mmu_update_t u;
+    u.ptr = virt_to_machine(ptr);
+    u.val = val.pud;
+    BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
+}
+
+void xen_l4_entry_update(pgd_t *ptr, pgd_t val)
+{
+    mmu_update_t u;
+    u.ptr = virt_to_machine(ptr);
+    u.val = val.pgd;
+    BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
+}
+#endif /* CONFIG_X86_64 */
+#endif /* CONFIG_XEN_SHADOW_MODE */
 
 void xen_machphys_update(unsigned long mfn, unsigned long pfn)
 {
@@ -82,6 +102,14 @@ void xen_pt_switch(unsigned long ptr)
     BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 }
 
+void xen_new_user_pt(unsigned long ptr)
+{
+    struct mmuext_op op;
+    op.cmd = MMUEXT_NEW_USER_BASEPTR;
+    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
+    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
+}
+
 void xen_tlb_flush(void)
 {
     struct mmuext_op op;
@@ -141,7 +169,11 @@ void xen_invlpg_mask(cpumask_t *mask, unsigned long ptr)
 void xen_pgd_pin(unsigned long ptr)
 {
     struct mmuext_op op;
+#ifdef CONFIG_X86_64
+    op.cmd = MMUEXT_PIN_L4_TABLE;
+#else
     op.cmd = MMUEXT_PIN_L2_TABLE;
+#endif
     op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
     BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 }
@@ -169,7 +201,41 @@ void xen_pte_unpin(unsigned long ptr)
     op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
     BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 }
-#endif
+
+#ifdef CONFIG_X86_64
+void xen_pud_pin(unsigned long ptr)
+{
+    struct mmuext_op op;
+    op.cmd = MMUEXT_PIN_L3_TABLE;
+    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
+    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
+}
+
+void xen_pud_unpin(unsigned long ptr)
+{
+    struct mmuext_op op;
+    op.cmd = MMUEXT_UNPIN_TABLE;
+    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
+    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
+}
+
+void xen_pmd_pin(unsigned long ptr)
+{
+    struct mmuext_op op;
+    op.cmd = MMUEXT_PIN_L2_TABLE;
+    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
+    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
+}
+
+void xen_pmd_unpin(unsigned long ptr)
+{
+    struct mmuext_op op;
+    op.cmd = MMUEXT_UNPIN_TABLE;
+    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
+    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
+}
+#endif /* CONFIG_X86_64 */
+#endif /* CONFIG_XEN_SHADOW_MODE */
 
 void xen_set_ldt(unsigned long ptr, unsigned long len)
 {
@@ -180,11 +246,60 @@ void xen_set_ldt(unsigned long ptr, unsigned long len)
     BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 }
 
+void xen_contig_memory(unsigned long vstart, unsigned int order)
+{
+    /*
+     * Ensure multi-page extents are contiguous in machine memory. This code 
+     * could be cleaned up some, and the number of hypercalls reduced.
+     */
+    pgd_t         *pgd; 
+    pud_t         *pud; 
+    pmd_t         *pmd;
+    pte_t         *pte;
+    unsigned long  mfn, i, flags;
+
+    scrub_pages(vstart, 1 << order);
+
+    balloon_lock(flags);
+
+    /* 1. Zap current PTEs, giving away the underlying pages. */
+    for (i = 0; i < (1<<order); i++) {
+        pgd = pgd_offset_k(vstart + (i*PAGE_SIZE));
+        pud = pud_offset(pgd, (vstart + (i*PAGE_SIZE)));
+        pmd = pmd_offset(pud, (vstart + (i*PAGE_SIZE)));
+        pte = pte_offset_kernel(pmd, (vstart + (i*PAGE_SIZE)));
+        mfn = pte_mfn(*pte);
+        HYPERVISOR_update_va_mapping(
+            vstart + (i*PAGE_SIZE), __pte_ma(0), 0);
+        phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
+            INVALID_P2M_ENTRY;
+        BUG_ON(HYPERVISOR_dom_mem_op(
+            MEMOP_decrease_reservation, &mfn, 1, 0) != 1);
+    }
+
+    /* 2. Get a new contiguous memory extent. */
+    BUG_ON(HYPERVISOR_dom_mem_op(
+        MEMOP_increase_reservation, &mfn, 1, order) != 1);
+
+    /* 3. Map the new extent in place of old pages. */
+    for (i = 0; i < (1<<order); i++) {
+        HYPERVISOR_update_va_mapping(
+            vstart + (i*PAGE_SIZE),
+            __pte_ma(((mfn+i)<<PAGE_SHIFT)|__PAGE_KERNEL), 0);
+        xen_machphys_update(mfn+i, (__pa(vstart)>>PAGE_SHIFT)+i);
+        phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] = mfn+i;
+    }
+
+    flush_tlb_all();
+
+    balloon_unlock(flags);
+}
+
 #ifdef CONFIG_XEN_PHYSDEV_ACCESS
 
 unsigned long allocate_empty_lowmem_region(unsigned long pages)
 {
-    pgd_t         *pgd; 
+    pgd_t         *pgd;
     pud_t         *pud; 
     pmd_t         *pmd;
     pte_t         *pte;
@@ -209,8 +324,12 @@ unsigned long allocate_empty_lowmem_region(unsigned long pages)
         pud = pud_offset(pgd, (vstart + (i*PAGE_SIZE)));
         pmd = pmd_offset(pud, (vstart + (i*PAGE_SIZE)));
         pte = pte_offset_kernel(pmd, (vstart + (i*PAGE_SIZE))); 
-        pfn_array[i] = pte->pte_low >> PAGE_SHIFT;
+        pfn_array[i] = pte_mfn(*pte);
+#ifdef CONFIG_X86_64
+        xen_l1_entry_update(pte, 0);
+#else
         HYPERVISOR_update_va_mapping(vstart + (i*PAGE_SIZE), __pte_ma(0), 0);
+#endif
         phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
             INVALID_P2M_ENTRY;
     }
diff --git a/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c
index 6d8deeaec2..4e8db3e003 100644
--- a/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c
+++ b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c
@@ -37,7 +37,7 @@
 #include <asm/atomic.h>
 #include <asm/system.h>
 #include <asm/ptrace.h>
-#include <asm/synch_bitops.h>
+#include <asm-xen/synch_bitops.h>
 #include <asm-xen/xen-public/event_channel.h>
 #include <asm-xen/xen-public/physdev.h>
 #include <asm-xen/ctrl_if.h>
diff --git a/linux-2.6.11-xen-sparse/arch/xen/kernel/gnttab.c b/linux-2.6.11-xen-sparse/arch/xen/kernel/gnttab.c
index 0106a6fea0..838b53cd1f 100644
--- a/linux-2.6.11-xen-sparse/arch/xen/kernel/gnttab.c
+++ b/linux-2.6.11-xen-sparse/arch/xen/kernel/gnttab.c
@@ -19,7 +19,7 @@
 #include <asm-xen/xen_proc.h>
 #include <asm-xen/linux-public/privcmd.h>
 #include <asm-xen/gnttab.h>
-#include <asm/synch_bitops.h>
+#include <asm-xen/synch_bitops.h>
 
 #if 1
 #define ASSERT(_p) \
diff --git a/linux-2.6.11-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c b/linux-2.6.11-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c
index 70a2a38b7e..4a66a273f6 100644
--- a/linux-2.6.11-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c
+++ b/linux-2.6.11-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c
@@ -66,59 +66,6 @@ struct dma_coherent_mem {
 	unsigned long	*bitmap;
 };
 
-static void
-xen_contig_memory(unsigned long vstart, unsigned int order)
-{
-	/*
-	 * Ensure multi-page extents are contiguous in machine memory.
-	 * This code could be cleaned up some, and the number of
-	 * hypercalls reduced.
-	 */
-	pgd_t         *pgd; 
-	pud_t         *pud; 
-	pmd_t         *pmd;
-	pte_t         *pte;
-	unsigned long  pfn, i, flags;
-
-	scrub_pages(vstart, 1 << order);
-
-        balloon_lock(flags);
-
-	/* 1. Zap current PTEs, giving away the underlying pages. */
-	for (i = 0; i < (1<<order); i++) {
-                pgd = pgd_offset_k(   (vstart + (i*PAGE_SIZE)));
-		pud = pud_offset(pgd, (vstart + (i*PAGE_SIZE)));
-		pmd = pmd_offset(pud, (vstart + (i*PAGE_SIZE)));
-		pte = pte_offset_kernel(pmd, (vstart + (i*PAGE_SIZE)));
-		pfn = pte->pte >> PAGE_SHIFT;
-		xen_l1_entry_update(pte, 0);
-		phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
-			(u32)INVALID_P2M_ENTRY;
-		if (HYPERVISOR_dom_mem_op(MEMOP_decrease_reservation, 
-					  &pfn, 1, 0) != 1) BUG();
-	}
-	/* 2. Get a new contiguous memory extent. */
-	if (HYPERVISOR_dom_mem_op(MEMOP_increase_reservation,
-				  &pfn, 1, order) != 1) BUG();
-	/* 3. Map the new extent in place of old pages. */
-	for (i = 0; i < (1<<order); i++) {
-		pgd = pgd_offset_k(   (vstart + (i*PAGE_SIZE)));
-		pud = pud_offset(pgd, (vstart + (i*PAGE_SIZE)));
-		pmd = pmd_offset(pud, (vstart + (i*PAGE_SIZE)));
-		pte = pte_offset_kernel(pmd, (vstart + (i*PAGE_SIZE)));
-		xen_l1_entry_update(
-			pte, ((pfn+i)<<PAGE_SHIFT)|__PAGE_KERNEL);
-		xen_machphys_update(
-			pfn+i, (__pa(vstart)>>PAGE_SHIFT)+i);
-		phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
-			pfn+i;
-	}
-	/* Flush updates through and flush the TLB. */
-	xen_tlb_flush();
-
-        balloon_unlock(flags);
-}
-
 void *dma_alloc_coherent(struct device *dev, size_t size,
 			   dma_addr_t *dma_handle, unsigned gfp)
 {
diff --git a/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/Makefile b/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/Makefile
index 9b0223649f..48c16ea11b 100644
--- a/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/Makefile
+++ b/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/Makefile
@@ -6,9 +6,11 @@ XENARCH	:= $(subst ",,$(CONFIG_XENARCH))
 
 CFLAGS	+= -Iarch/$(XENARCH)/mm
 
-obj-y	:= init.o fault.o ioremap.o pageattr.o hypervisor.o
+obj-y	:= init.o fault.o ioremap.o pageattr.o
 c-obj-y	:= extable.o
 
+i386-obj-y := hypervisor.o
+
 #obj-y	 := init.o fault.o ioremap.o extable.o pageattr.o
 #c-obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
 c-obj-$(CONFIG_DISCONTIGMEM) += numa.o
@@ -21,6 +23,9 @@ c-link	:=
 $(patsubst %.o,$(obj)/%.c,$(c-obj-y) $(c-link)):
 	@ln -fsn $(srctree)/arch/x86_64/mm/$(notdir $@) $@
 
-obj-y	+= $(c-obj-y)
+$(patsubst %.o,$(obj)/%.c,$(i386-obj-y)):
+	ln -fsn $(srctree)/arch/xen/i386/mm/$(notdir $@) $@
+
+obj-y	+= $(c-obj-y) $(i386-obj-y)
 
-clean-files += $(patsubst %.o,%.c,$(c-obj-y) $(c-obj-) $(c-link))
+clean-files += $(patsubst %.o,%.c,$(c-obj-y) $(c-obj-) $(c-link) $(i386-obj-y))
diff --git a/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/hypervisor.c b/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/hypervisor.c
deleted file mode 100644
index 3923162df8..0000000000
--- a/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/hypervisor.c
+++ /dev/null
@@ -1,275 +0,0 @@
-/******************************************************************************
- * mm/hypervisor.c
- * 
- * Update page tables via the hypervisor.
- * 
- * Copyright (c) 2002-2004, K A Fraser
- * 
- * This file may be distributed separately from the Linux kernel, or
- * incorporated into other software packages, subject to the following license:
- * 
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this source file (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use, copy, modify,
- * merge, publish, distribute, sublicense, and/or sell copies of the Software,
- * and to permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- * 
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- * 
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-/*
- * Jun Nakajima <jun.nakajima@intel.com>
- *   Added hypercalls for x86-64.
- *
- */
-
-#include <linux/config.h>
-#include <linux/sched.h>
-#include <linux/mm.h>
-#include <linux/vmalloc.h>
-#include <asm/page.h>
-#include <asm/pgtable.h>
-#include <asm/tlbflush.h>
-#include <asm-xen/hypervisor.h>
-#include <asm-xen/balloon.h>
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
-#include <linux/percpu.h>
-#endif
-
-void xen_l1_entry_update(pte_t *ptr, unsigned long val)
-{
-    mmu_update_t u;
-    u.ptr = virt_to_machine(ptr);
-    u.val = val;
-    BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_l2_entry_update(pmd_t *ptr, pmd_t val)
-{
-    mmu_update_t u;
-    u.ptr = virt_to_machine(ptr);
-    u.val = val.pmd;
-    BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_l3_entry_update(pud_t *ptr, pud_t val)
-{
-    mmu_update_t u;
-    u.ptr = virt_to_machine(ptr);
-    u.val = val.pud;
-    BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_l4_entry_update(pgd_t *ptr, pgd_t val)
-{
-    mmu_update_t u;
-    u.ptr = virt_to_machine(ptr);
-    u.val = val.pgd;
-    BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_pt_switch(unsigned long ptr)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_NEW_BASEPTR;
-    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_new_user_pt(unsigned long ptr)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_NEW_USER_BASEPTR;
-    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_tlb_flush(void)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_TLB_FLUSH_LOCAL;
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_invlpg(unsigned long ptr)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_INVLPG_LOCAL;
-    op.linear_addr = ptr & PAGE_MASK;
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-#ifdef CONFIG_SMP
-void xen_tlb_flush_all(void)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_TLB_FLUSH_ALL;
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_tlb_flush_mask(cpumask_t *mask)
-{
-    struct mmuext_op op;
-    if ( cpus_empty(*mask) )
-        return;
-    op.cmd = MMUEXT_TLB_FLUSH_MULTI;
-    op.vcpumask = mask->bits;
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_invlpg_all(unsigned long ptr)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_INVLPG_ALL;
-    op.linear_addr = ptr & PAGE_MASK;
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_invlpg_mask(cpumask_t *mask, unsigned long ptr)
-{
-    struct mmuext_op op;
-    if ( cpus_empty(*mask) )
-        return;
-    op.cmd = MMUEXT_INVLPG_MULTI;
-    op.vcpumask = mask->bits;
-    op.linear_addr = ptr & PAGE_MASK;
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-#endif
-
-void xen_pgd_pin(unsigned long ptr)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_PIN_L4_TABLE;
-    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_pgd_unpin(unsigned long ptr)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_UNPIN_TABLE;
-    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_pud_pin(unsigned long ptr)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_PIN_L3_TABLE;
-    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_pud_unpin(unsigned long ptr)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_UNPIN_TABLE;
-    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_pmd_pin(unsigned long ptr)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_PIN_L2_TABLE;
-    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_pmd_unpin(unsigned long ptr)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_UNPIN_TABLE;
-    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_pte_pin(unsigned long ptr)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_PIN_L1_TABLE;
-    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-
-}
-
-void xen_pte_unpin(unsigned long ptr)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_UNPIN_TABLE;
-    op.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_set_ldt(unsigned long ptr, unsigned long len)
-{
-    struct mmuext_op op;
-    op.cmd = MMUEXT_SET_LDT;
-    op.linear_addr = ptr;
-    op.nr_ents = len;
-    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
-}
-
-void xen_machphys_update(unsigned long mfn, unsigned long pfn)
-{
-    mmu_update_t u;
-    u.ptr = (mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
-    u.val = pfn;
-    BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
-}
-
-#ifdef CONFIG_XEN_PHYSDEV_ACCESS
-unsigned long allocate_empty_lowmem_region(unsigned long pages)
-{
-    pgd_t         *pgd;
-    pud_t         *pud; 
-    pmd_t         *pmd;
-    pte_t         *pte;
-    unsigned long *pfn_array;
-    unsigned long  vstart;
-    unsigned long  i;
-    unsigned int   order = get_order(pages*PAGE_SIZE);
-
-    vstart = __get_free_pages(GFP_KERNEL, order);
-    if ( vstart == 0 )
-        return 0UL;
-
-    scrub_pages(vstart, 1 << order);
-
-    pfn_array = vmalloc((1<<order) * sizeof(*pfn_array));
-    if ( pfn_array == NULL )
-        BUG();
-
-    for ( i = 0; i < (1<<order); i++ )
-    {
-        pgd = pgd_offset_k(   (vstart + (i*PAGE_SIZE)));
-        pud = pud_offset(pgd, (vstart + (i*PAGE_SIZE)));
-        pmd = pmd_offset(pud, (vstart + (i*PAGE_SIZE)));
-        pte = pte_offset_kernel(pmd, (vstart + (i*PAGE_SIZE))); 
-        pfn_array[i] = pte->pte >> PAGE_SHIFT;
-        xen_l1_entry_update(pte, 0);
-        phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
-            (u32)INVALID_P2M_ENTRY;
-    }
-
-    /* Flush updates through and flush the TLB. */
-    flush_tlb_all();
-
-    balloon_put_pages(pfn_array, 1 << order);
-
-    vfree(pfn_array);
-
-    return vstart;
-}
-
-#endif /* CONFIG_XEN_PHYSDEV_ACCESS */
diff --git a/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/init.c b/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/init.c
index 13b7ade9b6..0e85f182b2 100644
--- a/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/init.c
+++ b/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/init.c
@@ -395,7 +395,7 @@ unsigned long get_machine_pfn(unsigned long addr)
         pmd_t* pmd = pmd_offset(pud, addr);
         pte_t *pte = pte_offset_kernel(pmd, addr);
         
-        return (pte->pte >> PAGE_SHIFT);
+        return pte_mfn(*pte);
 } 
 
 #define ALIGN_TO_4K __attribute__((section(".data.page_aligned")))
diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable-2level.h b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable-2level.h
index b25099b6fc..9eddbd8012 100644
--- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable-2level.h
+++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable-2level.h
@@ -46,9 +46,10 @@
  */
 #define INVALID_P2M_ENTRY (~0U)
 #define FOREIGN_FRAME(_m) ((_m) | (1UL<<((sizeof(unsigned long)*8)-1)))
+#define pte_mfn(_pte) ((_pte).pte_low >> PAGE_SHIFT)
 #define pte_pfn(_pte)							\
 ({									\
-	unsigned long mfn = (_pte).pte_low >> PAGE_SHIFT;		\
+	unsigned long mfn = pte_mfn(_pte);				\
 	unsigned long pfn = mfn_to_pfn(mfn);				\
 	if ((pfn >= max_mapnr) || (pfn_to_mfn(pfn) != mfn))		\
 		pfn = max_mapnr; /* special: force !pfn_valid() */	\
diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/synch_bitops.h b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/synch_bitops.h
index 6af3ad2857..11f88c2507 100644
--- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/synch_bitops.h
+++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/synch_bitops.h
@@ -89,6 +89,22 @@ static inline unsigned long __synch_cmpxchg(volatile void *ptr,
 				       "0"(old)
 				     : "memory");
 		return prev;
+#ifdef CONFIG_X86_64
+	case 4:
+		__asm__ __volatile__("lock; cmpxchgl %k1,%2"
+				     : "=a"(prev)
+				     : "q"(new), "m"(*__synch_xg(ptr)),
+				       "0"(old)
+				     : "memory");
+		return prev;
+	case 8:
+		__asm__ __volatile__("lock; cmpxchgq %1,%2"
+				     : "=a"(prev)
+				     : "q"(new), "m"(*__synch_xg(ptr)),
+				       "0"(old)
+				     : "memory");
+		return prev;
+#else
 	case 4:
 		__asm__ __volatile__("lock; cmpxchgl %1,%2"
 				     : "=a"(prev)
@@ -96,6 +112,7 @@ static inline unsigned long __synch_cmpxchg(volatile void *ptr,
 				       "0"(old)
 				     : "memory");
 		return prev;
+#endif
 	}
 	return old;
 }
diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/system.h b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/system.h
index 22e41ce412..021acbd159 100644
--- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/system.h
+++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/system.h
@@ -4,7 +4,7 @@
 #include <linux/config.h>
 #include <linux/kernel.h>
 #include <linux/bitops.h>
-#include <asm/synch_bitops.h>
+#include <asm-xen/synch_bitops.h>
 #include <asm/segment.h>
 #include <asm/cpufeature.h>
 #include <asm-xen/hypervisor.h>
diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/pgtable.h b/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/pgtable.h
index 99e149786c..5b86bd7978 100644
--- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/pgtable.h
+++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/pgtable.h
@@ -275,11 +275,12 @@ static inline unsigned long pud_bad(pud_t pud)
  *      require. In all the cases we care about, the high bit gets shifted out
  *      (e.g., phys_to_machine()) so behaviour there is correct.
  */
-#define INVALID_P2M_ENTRY (~0UL)
+#define INVALID_P2M_ENTRY (~0U)
 #define FOREIGN_FRAME(_m) ((_m) | (1UL<<((sizeof(unsigned long)*8)-1)))
+#define pte_mfn(_pte) (((_pte).pte & PTE_MASK) >> PAGE_SHIFT)
 #define pte_pfn(_pte)							\
 ({									\
-	unsigned long mfn = (_pte).pte >> PAGE_SHIFT;                   \
+	unsigned long mfn = pte_mfn(_pte);                              \
 	unsigned pfn = mfn_to_pfn(mfn);                                 \
 	if ((pfn >= max_mapnr) || (pfn_to_mfn(pfn) != mfn))		\
 		pfn = max_mapnr; /* special: force !pfn_valid() */	\
diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/synch_bitops.h b/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/synch_bitops.h
deleted file mode 100644
index dea3693a67..0000000000
--- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/synch_bitops.h
+++ /dev/null
@@ -1,85 +0,0 @@
-#ifndef __XEN_SYNCH_BITOPS_H__
-#define __XEN_SYNCH_BITOPS_H__
-
-/*
- * Copyright 1992, Linus Torvalds.
- * Heavily modified to provide guaranteed strong synchronisation
- * when communicating with Xen or other guest OSes running on other CPUs.
- * 
- * Copyed from asm-xen/asm-i386
- */
-
-#include <linux/config.h>
-
-#define ADDR (*(volatile long *) addr)
-
-static __inline__ void synch_set_bit(int nr, volatile void * addr)
-{
-    __asm__ __volatile__ ( 
-        "lock btsl %1,%0"
-        : "=m" (ADDR) : "Ir" (nr) : "memory" );
-}
-
-static __inline__ void synch_clear_bit(int nr, volatile void * addr)
-{
-    __asm__ __volatile__ (
-        "lock btrl %1,%0"
-        : "=m" (ADDR) : "Ir" (nr) : "memory" );
-}
-
-static __inline__ void synch_change_bit(int nr, volatile void * addr)
-{
-    __asm__ __volatile__ (
-        "lock btcl %1,%0"
-        : "=m" (ADDR) : "Ir" (nr) : "memory" );
-}
-
-static __inline__ int synch_test_and_set_bit(int nr, volatile void * addr)
-{
-    int oldbit;
-    __asm__ __volatile__ (
-        "lock btsl %2,%1\n\tsbbl %0,%0"
-        : "=r" (oldbit), "=m" (ADDR) : "Ir" (nr) : "memory");
-    return oldbit;
-}
-
-static __inline__ int synch_test_and_clear_bit(int nr, volatile void * addr)
-{
-    int oldbit;
-    __asm__ __volatile__ (
-        "lock btrl %2,%1\n\tsbbl %0,%0"
-        : "=r" (oldbit), "=m" (ADDR) : "Ir" (nr) : "memory");
-    return oldbit;
-}
-
-static __inline__ int synch_test_and_change_bit(int nr, volatile void * addr)
-{
-    int oldbit;
-
-    __asm__ __volatile__ (
-        "lock btcl %2,%1\n\tsbbl %0,%0"
-        : "=r" (oldbit), "=m" (ADDR) : "Ir" (nr) : "memory");
-    return oldbit;
-}
-
-static __inline__ int synch_const_test_bit(int nr, const volatile void * addr)
-{
-    return ((1UL << (nr & 31)) & 
-            (((const volatile unsigned int *) addr)[nr >> 5])) != 0;
-}
-
-static __inline__ int synch_var_test_bit(int nr, volatile void * addr)
-{
-    int oldbit;
-    __asm__ __volatile__ (
-        "btl %2,%1\n\tsbbl %0,%0"
-        : "=r" (oldbit) : "m" (ADDR), "Ir" (nr) );
-    return oldbit;
-}
-
-#define synch_test_bit(nr,addr) \
-(__builtin_constant_p(nr) ? \
- synch_const_test_bit((nr),(addr)) : \
- synch_var_test_bit((nr),(addr)))
-
-#endif /* __XEN_SYNCH_BITOPS_H__ */
diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/system.h b/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/system.h
index e8badb7533..17a8fec17c 100644
--- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/system.h
+++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/system.h
@@ -4,7 +4,7 @@
 #include <linux/config.h>
 #include <linux/kernel.h>
 #include <asm/segment.h>
-#include <asm/synch_bitops.h>
+#include <asm-xen/synch_bitops.h>
 #include <asm-xen/hypervisor.h>
 #include <asm-xen/xen-public/arch-x86_64.h>
 
diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/evtchn.h b/linux-2.6.11-xen-sparse/include/asm-xen/evtchn.h
index 1fa0530c32..a1801c1981 100644
--- a/linux-2.6.11-xen-sparse/include/asm-xen/evtchn.h
+++ b/linux-2.6.11-xen-sparse/include/asm-xen/evtchn.h
@@ -34,7 +34,7 @@
 #include <linux/config.h>
 #include <asm-xen/hypervisor.h>
 #include <asm/ptrace.h>
-#include <asm/synch_bitops.h>
+#include <asm-xen/synch_bitops.h>
 #include <asm-xen/xen-public/event_channel.h>
 #include <linux/smp.h>
 
diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h b/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h
index 5a0da00816..449dba9745 100644
--- a/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h
+++ b/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h
@@ -130,6 +130,8 @@ void xen_invlpg_mask(cpumask_t *mask, unsigned long ptr);
 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
 #endif /* linux < 2.6.0 */
 
+void xen_contig_memory(unsigned long vstart, unsigned int order);
+
 #ifdef CONFIG_XEN_PHYSDEV_ACCESS
 /* Allocate a contiguous empty region of low memory. Return virtual start. */
 unsigned long allocate_empty_lowmem_region(unsigned long pages);
diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/synch_bitops.h b/linux-2.6.11-xen-sparse/include/asm-xen/synch_bitops.h
new file mode 100644
index 0000000000..9b5d468c26
--- /dev/null
+++ b/linux-2.6.11-xen-sparse/include/asm-xen/synch_bitops.h
@@ -0,0 +1,2 @@
+
+#include <asm-i386/synch_bitops.h>