Use a SWIOTLB to use pre-reserved bounce buffers for high memory

and multi-page DMA accesses. This is based on a preliminary patch
from Suresh Siddha at Intel.

Signed-off-by: Keir Fraser <keir@xensource.com>
Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>

24 files changed, 1002 insertions, 604 deletions

diff --git a/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_32 b/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_32
index 7274d10780..5e24b71a97 100644
--- a/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_32
+++ b/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_32
@@ -130,6 +130,7 @@ CONFIG_PREEMPT_BKL=y
 # CONFIG_X86_REBOOTFIXUPS is not set
 CONFIG_MICROCODE=y
 CONFIG_X86_CPUID=y
+CONFIG_SWIOTLB=y
 
 #
 # Firmware Drivers
diff --git a/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_64 b/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_64
index 92818c2477..0ba2b45f0c 100644
--- a/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_64
+++ b/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_64
@@ -123,6 +123,7 @@ CONFIG_X86_XEN_GENAPIC=y
 # CONFIG_X86_MSR is not set
 # CONFIG_GART_IOMMU is not set
 CONFIG_DUMMY_IOMMU=y
+CONFIG_SWIOTLB=y
 # CONFIG_X86_MCE is not set
 
 #
diff --git a/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_32 b/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_32
index 1cd7188c10..51307cfbe0 100644
--- a/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_32
+++ b/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_32
@@ -137,6 +137,7 @@ CONFIG_PREEMPT_BKL=y
 # CONFIG_X86_REBOOTFIXUPS is not set
 CONFIG_MICROCODE=m
 CONFIG_X86_CPUID=m
+CONFIG_SWIOTLB=y
 
 #
 # Firmware Drivers
diff --git a/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_64 b/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_64
index b19f90ec81..f7619599e3 100644
--- a/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_64
+++ b/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_64
@@ -132,6 +132,7 @@ CONFIG_X86_HT=y
 # CONFIG_NUMA_EMU is not set
 # CONFIG_GART_IOMMU is not set
 CONFIG_DUMMY_IOMMU=y
+CONFIG_SWIOTLB=y
 # CONFIG_X86_MCE is not set
 
 #
diff --git a/linux-2.6-xen-sparse/arch/xen/i386/Kconfig b/linux-2.6-xen-sparse/arch/xen/i386/Kconfig
index 066828d224..5629afccc9 100644
--- a/linux-2.6-xen-sparse/arch/xen/i386/Kconfig
+++ b/linux-2.6-xen-sparse/arch/xen/i386/Kconfig
@@ -533,6 +533,11 @@ config X86_CPUID
 	  with major 203 and minors 0 to 31 for /dev/cpu/0/cpuid to
 	  /dev/cpu/31/cpuid.
 
+config SWIOTLB
+       bool
+       depends on PCI
+       default y
+
 source "drivers/firmware/Kconfig"
 
 choice
diff --git a/linux-2.6-xen-sparse/arch/xen/i386/kernel/Makefile b/linux-2.6-xen-sparse/arch/xen/i386/kernel/Makefile
index da948cabc9..29e7c0344a 100644
--- a/linux-2.6-xen-sparse/arch/xen/i386/kernel/Makefile
+++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/Makefile
@@ -44,6 +44,7 @@ c-obj-$(CONFIG_HPET_TIMER) 	+= time_hpet.o
 c-obj-$(CONFIG_EFI) 		+= efi.o efi_stub.o
 c-obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
 c-obj-$(CONFIG_SMP_ALTERNATIVES)+= smpalts.o
+c-obj-$(CONFIG_SWIOTLB)		+= swiotlb.o
 
 EXTRA_AFLAGS   := -traditional
 
diff --git a/linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c b/linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c
index 249507484f..481b61bc27 100644
--- a/linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c
+++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c
@@ -24,6 +24,103 @@ struct dma_coherent_mem {
 	unsigned long	*bitmap;
 };
 
+static void iommu_bug(void)
+{
+	printk(KERN_ALERT "Fatal DMA error! Please use 'swiotlb=force'\n");
+	BUG();
+}
+
+#define IOMMU_BUG_ON(test) do { if (unlikely(test)) iommu_bug(); } while(0)
+
+int
+dma_map_sg(struct device *hwdev, struct scatterlist *sg, int nents,
+	   enum dma_data_direction direction)
+{
+	int i, rc;
+
+	BUG_ON(direction == DMA_NONE);
+
+	if (swiotlb) {
+		rc = swiotlb_map_sg(hwdev, sg, nents, direction);
+	} else {
+		for (i = 0; i < nents; i++ ) {
+			sg[i].dma_address =
+				page_to_phys(sg[i].page) + sg[i].offset;
+			sg[i].dma_length  = sg[i].length;
+			BUG_ON(!sg[i].page);
+			IOMMU_BUG_ON(address_needs_mapping(
+				hwdev, sg[i].dma_address));
+		}
+		rc = nents;
+	}
+
+	flush_write_buffers();
+	return rc;
+}
+EXPORT_SYMBOL(dma_map_sg);
+
+void
+dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents,
+	     enum dma_data_direction direction)
+{
+	BUG_ON(direction == DMA_NONE);
+	if (swiotlb)
+		swiotlb_unmap_sg(hwdev, sg, nents, direction);
+}
+EXPORT_SYMBOL(dma_unmap_sg);
+
+dma_addr_t
+dma_map_page(struct device *dev, struct page *page, unsigned long offset,
+	     size_t size, enum dma_data_direction direction)
+{
+	dma_addr_t dma_addr;
+
+	BUG_ON(direction == DMA_NONE);
+
+	if (swiotlb) {
+		dma_addr = swiotlb_map_page(
+			dev, page, offset, size, direction);
+	} else {
+		dma_addr = page_to_phys(page) + offset;
+		IOMMU_BUG_ON(address_needs_mapping(dev, dma_addr));
+	}
+
+	return dma_addr;
+}
+EXPORT_SYMBOL(dma_map_page);
+
+void
+dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
+	       enum dma_data_direction direction)
+{
+	BUG_ON(direction == DMA_NONE);
+	if (swiotlb)
+		swiotlb_unmap_page(dev, dma_address, size, direction);
+}
+EXPORT_SYMBOL(dma_unmap_page);
+
+int
+dma_mapping_error(dma_addr_t dma_addr)
+{
+	if (swiotlb)
+		return swiotlb_dma_mapping_error(dma_addr);
+	return 0;
+}
+EXPORT_SYMBOL(dma_mapping_error);
+
+int
+dma_supported(struct device *dev, u64 mask)
+{
+	if (swiotlb)
+		return swiotlb_dma_supported(dev, mask);
+	/*
+         * By default we'll BUG when an infeasible DMA is requested, and
+         * request swiotlb=force (see IOMMU_BUG_ON).
+         */
+	return 1;
+}
+EXPORT_SYMBOL(dma_supported);
+
 void *dma_alloc_coherent(struct device *dev, size_t size,
 			   dma_addr_t *dma_handle, unsigned int __nocast gfp)
 {
@@ -54,13 +151,14 @@ void *dma_alloc_coherent(struct device *dev, size_t size,
 	ret = (void *)vstart;
 
 	if (ret != NULL) {
-		xen_contig_memory(vstart, order);
+		xen_create_contiguous_region(vstart, order);
 
 		memset(ret, 0, size);
 		*dma_handle = virt_to_bus(ret);
 	}
 	return ret;
 }
+EXPORT_SYMBOL(dma_alloc_coherent);
 
 void dma_free_coherent(struct device *dev, size_t size,
 			 void *vaddr, dma_addr_t dma_handle)
@@ -72,9 +170,12 @@ void dma_free_coherent(struct device *dev, size_t size,
 		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
 
 		bitmap_release_region(mem->bitmap, page, order);
-	} else
+	} else {
+		xen_destroy_contiguous_region((unsigned long)vaddr, order);
 		free_pages((unsigned long)vaddr, order);
+	}
 }
+EXPORT_SYMBOL(dma_free_coherent);
 
 int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
 				dma_addr_t device_addr, size_t size, int flags)
@@ -153,46 +254,20 @@ void *dma_mark_declared_memory_occupied(struct device *dev,
 }
 EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
 
-static LIST_HEAD(dma_map_head);
-static DEFINE_SPINLOCK(dma_map_lock);
-struct dma_map_entry {
-	struct list_head list;
-	dma_addr_t dma;
-	char *bounce, *host;
-	size_t size;
-};
-#define DMA_MAP_MATCHES(e,d) (((e)->dma<=(d)) && (((e)->dma+(e)->size)>(d)))
-
 dma_addr_t
 dma_map_single(struct device *dev, void *ptr, size_t size,
 	       enum dma_data_direction direction)
 {
-	struct dma_map_entry *ent;
-	void *bnc;
 	dma_addr_t dma;
-	unsigned long flags;
 
 	BUG_ON(direction == DMA_NONE);
 
-	/*
-	 * Even if size is sub-page, the buffer may still straddle a page
-	 * boundary. Take into account buffer start offset. All other calls are
-	 * conservative and always search the dma_map list if it's non-empty.
-	 */
-	if ((((unsigned int)ptr & ~PAGE_MASK) + size) <= PAGE_SIZE) {
-		dma = virt_to_bus(ptr);
+	if (swiotlb) {
+		dma = swiotlb_map_single(dev, ptr, size, direction);
 	} else {
-		BUG_ON((bnc = dma_alloc_coherent(dev, size, &dma, GFP_ATOMIC)) == NULL);
-		BUG_ON((ent = kmalloc(sizeof(*ent), GFP_ATOMIC)) == NULL);
-		if (direction != DMA_FROM_DEVICE)
-			memcpy(bnc, ptr, size);
-		ent->dma    = dma;
-		ent->bounce = bnc;
-		ent->host   = ptr;
-		ent->size   = size;
-		spin_lock_irqsave(&dma_map_lock, flags);
-		list_add(&ent->list, &dma_map_head);
-		spin_unlock_irqrestore(&dma_map_lock, flags);
+		dma = virt_to_bus(ptr);
+		IOMMU_BUG_ON(range_straddles_page_boundary(ptr, size));
+		IOMMU_BUG_ON(address_needs_mapping(dev, dma));
 	}
 
 	flush_write_buffers();
@@ -204,30 +279,9 @@ void
 dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
 		 enum dma_data_direction direction)
 {
-	struct dma_map_entry *ent;
-	unsigned long flags;
-
 	BUG_ON(direction == DMA_NONE);
-
-	/* Fast-path check: are there any multi-page DMA mappings? */
-	if (!list_empty(&dma_map_head)) {
-		spin_lock_irqsave(&dma_map_lock, flags);
-		list_for_each_entry ( ent, &dma_map_head, list ) {
-			if (DMA_MAP_MATCHES(ent, dma_addr)) {
-				list_del(&ent->list);
-				break;
-			}
-		}
-		spin_unlock_irqrestore(&dma_map_lock, flags);
-		if (&ent->list != &dma_map_head) {
-			BUG_ON(dma_addr != ent->dma);
-			BUG_ON(size != ent->size);
-			if (direction != DMA_TO_DEVICE)
-				memcpy(ent->host, ent->bounce, size);
-			dma_free_coherent(dev, size, ent->bounce, ent->dma);
-			kfree(ent);
-		}
-	}
+	if (swiotlb)
+		swiotlb_unmap_single(dev, dma_addr, size, direction);
 }
 EXPORT_SYMBOL(dma_unmap_single);
 
@@ -235,23 +289,8 @@ void
 dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
 			enum dma_data_direction direction)
 {
-	struct dma_map_entry *ent;
-	unsigned long flags, off;
-
-	/* Fast-path check: are there any multi-page DMA mappings? */
-	if (!list_empty(&dma_map_head)) {
-		spin_lock_irqsave(&dma_map_lock, flags);
-		list_for_each_entry ( ent, &dma_map_head, list )
-			if (DMA_MAP_MATCHES(ent, dma_handle))
-				break;
-		spin_unlock_irqrestore(&dma_map_lock, flags);
-		if (&ent->list != &dma_map_head) {
-			off = dma_handle - ent->dma;
-			BUG_ON((off + size) > ent->size);
-			/*if (direction != DMA_TO_DEVICE)*/
-				memcpy(ent->host+off, ent->bounce+off, size);
-		}
-	}
+	if (swiotlb)
+		swiotlb_sync_single_for_cpu(dev, dma_handle, size, direction);
 }
 EXPORT_SYMBOL(dma_sync_single_for_cpu);
 
@@ -259,24 +298,17 @@ void
 dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size,
                            enum dma_data_direction direction)
 {
-	struct dma_map_entry *ent;
-	unsigned long flags, off;
-
-	/* Fast-path check: are there any multi-page DMA mappings? */
-	if (!list_empty(&dma_map_head)) {
-		spin_lock_irqsave(&dma_map_lock, flags);
-		list_for_each_entry ( ent, &dma_map_head, list )
-			if (DMA_MAP_MATCHES(ent, dma_handle))
-				break;
-		spin_unlock_irqrestore(&dma_map_lock, flags);
-		if (&ent->list != &dma_map_head) {
-			off = dma_handle - ent->dma;
-			BUG_ON((off + size) > ent->size);
-			/*if (direction != DMA_FROM_DEVICE)*/
-				memcpy(ent->bounce+off, ent->host+off, size);
-		}
-	}
-
-	flush_write_buffers();
+	if (swiotlb)
+		swiotlb_sync_single_for_device(dev, dma_handle, size, direction);
 }
 EXPORT_SYMBOL(dma_sync_single_for_device);
+
+/*
+ * Local variables:
+ *  c-file-style: "linux"
+ *  indent-tabs-mode: t
+ *  c-indent-level: 8
+ *  c-basic-offset: 8
+ *  tab-width: 8
+ * End:
+ */
diff --git a/linux-2.6-xen-sparse/arch/xen/i386/kernel/swiotlb.c b/linux-2.6-xen-sparse/arch/xen/i386/kernel/swiotlb.c
new file mode 100644
index 0000000000..9f50f4f152
--- /dev/null
+++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/swiotlb.c
@@ -0,0 +1,653 @@
+/*
+ * Dynamic DMA mapping support.
+ *
+ * This implementation is a fallback for platforms that do not support
+ * I/O TLBs (aka DMA address translation hardware).
+ * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
+ * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
+ * Copyright (C) 2000, 2003 Hewlett-Packard Co
+ *	David Mosberger-Tang <davidm@hpl.hp.com>
+ * Copyright (C) 2005 Keir Fraser <keir@xensource.com>
+ */
+
+#include <linux/cache.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/ctype.h>
+#include <linux/init.h>
+#include <linux/bootmem.h>
+#include <linux/highmem.h>
+#include <asm/io.h>
+#include <asm/pci.h>
+#include <asm/dma.h>
+
+#define OFFSET(val,align) ((unsigned long)((val) & ( (align) - 1)))
+
+#define SG_ENT_PHYS_ADDRESS(sg)	(page_to_phys((sg)->page) + (sg)->offset)
+
+/*
+ * Maximum allowable number of contiguous slabs to map,
+ * must be a power of 2.  What is the appropriate value ?
+ * The complexity of {map,unmap}_single is linearly dependent on this value.
+ */
+#define IO_TLB_SEGSIZE	128
+
+/*
+ * log of the size of each IO TLB slab.  The number of slabs is command line
+ * controllable.
+ */
+#define IO_TLB_SHIFT 11
+
+int swiotlb_force;
+
+/*
+ * Used to do a quick range check in swiotlb_unmap_single and
+ * swiotlb_sync_single_*, to see if the memory was in fact allocated by this
+ * API.
+ */
+static char *io_tlb_start, *io_tlb_end;
+
+/*
+ * The number of IO TLB blocks (in groups of 64) betweeen io_tlb_start and
+ * io_tlb_end.  This is command line adjustable via setup_io_tlb_npages.
+ */
+static unsigned long io_tlb_nslabs;
+
+/*
+ * When the IOMMU overflows we return a fallback buffer. This sets the size.
+ */
+static unsigned long io_tlb_overflow = 32*1024;
+
+void *io_tlb_overflow_buffer;
+
+/*
+ * This is a free list describing the number of free entries available from
+ * each index
+ */
+static unsigned int *io_tlb_list;
+static unsigned int io_tlb_index;
+
+/*
+ * We need to save away the original address corresponding to a mapped entry
+ * for the sync operations.
+ */
+static struct phys_addr {
+	struct page *page;
+	unsigned int offset;
+} *io_tlb_orig_addr;
+
+/*
+ * Protect the above data structures in the map and unmap calls
+ */
+static DEFINE_SPINLOCK(io_tlb_lock);
+
+static int __init
+setup_io_tlb_npages(char *str)
+{
+	if (isdigit(*str)) {
+		io_tlb_nslabs = simple_strtoul(str, &str, 0) <<
+			(PAGE_SHIFT - IO_TLB_SHIFT);
+		/* avoid tail segment of size < IO_TLB_SEGSIZE */
+		io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
+	}
+	if (*str == ',')
+		++str;
+	/*
+         * NB. 'force' enables the swiotlb, but doesn't force its use for
+         * every DMA like it does on native Linux.
+         */
+	if (!strcmp(str, "force"))
+		swiotlb_force = 1;
+	return 1;
+}
+__setup("swiotlb=", setup_io_tlb_npages);
+/* make io_tlb_overflow tunable too? */
+
+/*
+ * Statically reserve bounce buffer space and initialize bounce buffer data
+ * structures for the software IO TLB used to implement the PCI DMA API.
+ */
+void
+swiotlb_init_with_default_size (size_t default_size)
+{
+	unsigned long i;
+
+	if (!io_tlb_nslabs) {
+		io_tlb_nslabs = (default_size >> PAGE_SHIFT);
+		io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
+	}
+
+	/*
+	 * Get IO TLB memory from the low pages
+	 */
+	io_tlb_start = alloc_bootmem_low_pages(io_tlb_nslabs *
+					       (1 << IO_TLB_SHIFT));
+	if (!io_tlb_start)
+		panic("Cannot allocate SWIOTLB buffer");
+
+	xen_create_contiguous_region(
+		(unsigned long)io_tlb_start, 
+		get_order(io_tlb_nslabs * (1 << IO_TLB_SHIFT)));
+
+	io_tlb_end = io_tlb_start + io_tlb_nslabs * (1 << IO_TLB_SHIFT);
+
+	/*
+	 * Allocate and initialize the free list array.  This array is used
+	 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
+	 * between io_tlb_start and io_tlb_end.
+	 */
+	io_tlb_list = alloc_bootmem(io_tlb_nslabs * sizeof(int));
+	for (i = 0; i < io_tlb_nslabs; i++)
+ 		io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
+	io_tlb_index = 0;
+	io_tlb_orig_addr = alloc_bootmem(
+		io_tlb_nslabs * sizeof(*io_tlb_orig_addr));
+
+	/*
+	 * Get the overflow emergency buffer
+	 */
+	io_tlb_overflow_buffer = alloc_bootmem_low(io_tlb_overflow);
+	printk(KERN_INFO "Placing software IO TLB between 0x%lx - 0x%lx\n",
+	       virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end-1));
+}
+
+void
+swiotlb_init(void)
+{
+	/* The user can forcibly enable swiotlb. */
+	if (swiotlb_force)
+		swiotlb = 1;
+
+	/*
+         * Otherwise, enable for domain 0 if the machine has 'lots of memory',
+         * which we take to mean more than 2GB.
+         */
+	if (xen_start_info.flags & SIF_INITDOMAIN) {
+		dom0_op_t op;
+		op.cmd = DOM0_PHYSINFO;
+		if ((HYPERVISOR_dom0_op(&op) == 0) &&
+		    (op.u.physinfo.total_pages > 0x7ffff))
+			swiotlb = 1;
+	}
+
+	if (swiotlb)
+		swiotlb_init_with_default_size(64 * (1<<20));
+}
+
+static void
+__sync_single(struct phys_addr buffer, char *dma_addr, size_t size, int dir)
+{
+	if (PageHighMem(buffer.page)) {
+		size_t len, bytes;
+		char *dev, *host, *kmp;
+		len = size;
+		while (len != 0) {
+			if (((bytes = len) + buffer.offset) > PAGE_SIZE)
+				bytes = PAGE_SIZE - buffer.offset;
+			kmp  = kmap_atomic(buffer.page, KM_SWIOTLB);
+			dev  = dma_addr + size - len;
+			host = kmp + buffer.offset;
+			memcpy((dir == DMA_FROM_DEVICE) ? host : dev,
+			       (dir == DMA_FROM_DEVICE) ? dev : host,
+			       bytes);
+			kunmap_atomic(kmp, KM_SWIOTLB);
+			len -= bytes;
+			buffer.page++;
+			buffer.offset = 0;
+		}
+	} else {
+		char *host = (char *)phys_to_virt(
+			page_to_pseudophys(buffer.page)) + buffer.offset;
+		if (dir == DMA_FROM_DEVICE)
+			memcpy(host, dma_addr, size);
+		else if (dir == DMA_TO_DEVICE)
+			memcpy(dma_addr, host, size);
+	}
+}
+
+/*
+ * Allocates bounce buffer and returns its kernel virtual address.
+ */
+static void *
+map_single(struct device *hwdev, struct phys_addr buffer, size_t size, int dir)
+{
+	unsigned long flags;
+	char *dma_addr;
+	unsigned int nslots, stride, index, wrap;
+	int i;
+
+	/*
+	 * For mappings greater than a page, we limit the stride (and
+	 * hence alignment) to a page size.
+	 */
+	nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
+	if (size > PAGE_SIZE)
+		stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT));
+	else
+		stride = 1;
+
+	BUG_ON(!nslots);
+
+	/*
+	 * Find suitable number of IO TLB entries size that will fit this
+	 * request and allocate a buffer from that IO TLB pool.
+	 */
+	spin_lock_irqsave(&io_tlb_lock, flags);
+	{
+		wrap = index = ALIGN(io_tlb_index, stride);
+
+		if (index >= io_tlb_nslabs)
+			wrap = index = 0;
+
+		do {
+			/*
+			 * If we find a slot that indicates we have 'nslots'
+			 * number of contiguous buffers, we allocate the
+			 * buffers from that slot and mark the entries as '0'
+			 * indicating unavailable.
+			 */
+			if (io_tlb_list[index] >= nslots) {
+				int count = 0;
+
+				for (i = index; i < (int)(index + nslots); i++)
+					io_tlb_list[i] = 0;
+				for (i = index - 1;
+				     (OFFSET(i, IO_TLB_SEGSIZE) !=
+				      IO_TLB_SEGSIZE -1) && io_tlb_list[i];
+				     i--)
+					io_tlb_list[i] = ++count;
+				dma_addr = io_tlb_start +
+					(index << IO_TLB_SHIFT);
+
+				/*
+				 * Update the indices to avoid searching in
+				 * the next round.
+				 */
+				io_tlb_index = 
+					((index + nslots) < io_tlb_nslabs
+					 ? (index + nslots) : 0);
+
+				goto found;
+			}
+			index += stride;
+			if (index >= io_tlb_nslabs)
+				index = 0;
+		} while (index != wrap);
+
+		spin_unlock_irqrestore(&io_tlb_lock, flags);
+		return NULL;
+	}
+  found:
+	spin_unlock_irqrestore(&io_tlb_lock, flags);
+
+	/*
+	 * Save away the mapping from the original address to the DMA address.
+	 * This is needed when we sync the memory.  Then we sync the buffer if
+	 * needed.
+	 */
+	io_tlb_orig_addr[index] = buffer;
+	if ((dir == DMA_TO_DEVICE) || (dir == DMA_BIDIRECTIONAL))
+		__sync_single(buffer, dma_addr, size, DMA_TO_DEVICE);
+
+	return dma_addr;
+}
+
+/*
+ * dma_addr is the kernel virtual address of the bounce buffer to unmap.
+ */
+static void
+unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
+{
+	unsigned long flags;
+	int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
+	int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
+	struct phys_addr buffer = io_tlb_orig_addr[index];
+
+	/*
+	 * First, sync the memory before unmapping the entry
+	 */
+	if ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))
+		__sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE);
+
+	/*
+	 * Return the buffer to the free list by setting the corresponding
+	 * entries to indicate the number of contigous entries available.
+	 * While returning the entries to the free list, we merge the entries
+	 * with slots below and above the pool being returned.
+	 */
+	spin_lock_irqsave(&io_tlb_lock, flags);
+	{
+		count = ((index + nslots) < ALIGN(index + 1, IO_TLB_SEGSIZE) ?
+			 io_tlb_list[index + nslots] : 0);
+		/*
+		 * Step 1: return the slots to the free list, merging the
+		 * slots with superceeding slots
+		 */
+		for (i = index + nslots - 1; i >= index; i--)
+			io_tlb_list[i] = ++count;
+		/*
+		 * Step 2: merge the returned slots with the preceding slots,
+		 * if available (non zero)
+		 */
+		for (i = index - 1;
+		     (OFFSET(i, IO_TLB_SEGSIZE) !=
+		      IO_TLB_SEGSIZE -1) && io_tlb_list[i];
+		     i--)
+			io_tlb_list[i] = ++count;
+	}
+	spin_unlock_irqrestore(&io_tlb_lock, flags);
+}
+
+static void
+sync_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
+{
+	int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
+	struct phys_addr buffer = io_tlb_orig_addr[index];
+	BUG_ON((dir != DMA_FROM_DEVICE) && (dir != DMA_TO_DEVICE));
+	__sync_single(buffer, dma_addr, size, dir);
+}
+
+static void
+swiotlb_full(struct device *dev, size_t size, int dir, int do_panic)
+{
+	/*
+	 * Ran out of IOMMU space for this operation. This is very bad.
+	 * Unfortunately the drivers cannot handle this operation properly.
+	 * unless they check for pci_dma_mapping_error (most don't)
+	 * When the mapping is small enough return a static buffer to limit
+	 * the damage, or panic when the transfer is too big.
+	 */
+	printk(KERN_ERR "PCI-DMA: Out of SW-IOMMU space for %lu bytes at "
+	       "device %s\n", (unsigned long)size, dev ? dev->bus_id : "?");
+
+	if (size > io_tlb_overflow && do_panic) {
+		if (dir == PCI_DMA_FROMDEVICE || dir == PCI_DMA_BIDIRECTIONAL)
+			panic("PCI-DMA: Memory would be corrupted\n");
+		if (dir == PCI_DMA_TODEVICE || dir == PCI_DMA_BIDIRECTIONAL)
+			panic("PCI-DMA: Random memory would be DMAed\n");
+	}
+}
+
+/*
+ * Map a single buffer of the indicated size for DMA in streaming mode.  The
+ * PCI address to use is returned.
+ *
+ * Once the device is given the dma address, the device owns this memory until
+ * either swiotlb_unmap_single or swiotlb_dma_sync_single is performed.
+ */
+dma_addr_t
+swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
+{
+	dma_addr_t dev_addr = virt_to_bus(ptr);
+	void *map;
+	struct phys_addr buffer;
+
+	BUG_ON(dir == DMA_NONE);
+
+	/*
+	 * If the pointer passed in happens to be in the device's DMA window,
+	 * we can safely return the device addr and not worry about bounce
+	 * buffering it.
+	 */
+	if (!range_straddles_page_boundary(ptr, size) &&
+	    !address_needs_mapping(hwdev, dev_addr))
+		return dev_addr;
+
+	/*
+	 * Oh well, have to allocate and map a bounce buffer.
+	 */
+	buffer.page   = virt_to_page(ptr);
+	buffer.offset = (unsigned long)ptr & ~PAGE_MASK;
+	map = map_single(hwdev, buffer, size, dir);
+	if (!map) {
+		swiotlb_full(hwdev, size, dir, 1);
+		map = io_tlb_overflow_buffer;
+	}
+
+	dev_addr = virt_to_bus(map);
+
+	/*
+	 * Ensure that the address returned is DMA'ble
+	 */
+	if (address_needs_mapping(hwdev, dev_addr))
+		panic("map_single: bounce buffer is not DMA'ble");
+
+	return dev_addr;
+}
+
+/*
+ * Unmap a single streaming mode DMA translation.  The dma_addr and size must
+ * match what was provided for in a previous swiotlb_map_single call.  All
+ * other usages are undefined.
+ *
+ * After this call, reads by the cpu to the buffer are guaranteed to see
+ * whatever the device wrote there.
+ */
+void
+swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
+		     int dir)
+{
+	char *dma_addr = bus_to_virt(dev_addr);
+
+	BUG_ON(dir == DMA_NONE);
+	if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
+		unmap_single(hwdev, dma_addr, size, dir);
+}
+
+/*
+ * Make physical memory consistent for a single streaming mode DMA translation
+ * after a transfer.
+ *
+ * If you perform a swiotlb_map_single() but wish to interrogate the buffer
+ * using the cpu, yet do not wish to teardown the PCI dma mapping, you must
+ * call this function before doing so.  At the next point you give the PCI dma
+ * address back to the card, you must first perform a
+ * swiotlb_dma_sync_for_device, and then the device again owns the buffer
+ */
+void
+swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
+			    size_t size, int dir)
+{
+	char *dma_addr = bus_to_virt(dev_addr);
+
+	BUG_ON(dir == DMA_NONE);
+	if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
+		sync_single(hwdev, dma_addr, size, dir);
+}
+
+void
+swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
+			       size_t size, int dir)
+{
+	char *dma_addr = bus_to_virt(dev_addr);
+
+	BUG_ON(dir == DMA_NONE);
+	if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
+		sync_single(hwdev, dma_addr, size, dir);
+}
+
+/*
+ * Map a set of buffers described by scatterlist in streaming mode for DMA.
+ * This is the scatter-gather version of the above swiotlb_map_single
+ * interface.  Here the scatter gather list elements are each tagged with the
+ * appropriate dma address and length.  They are obtained via
+ * sg_dma_{address,length}(SG).
+ *
+ * NOTE: An implementation may be able to use a smaller number of
+ *       DMA address/length pairs than there are SG table elements.
+ *       (for example via virtual mapping capabilities)
+ *       The routine returns the number of addr/length pairs actually
+ *       used, at most nents.
+ *
+ * Device ownership issues as mentioned above for swiotlb_map_single are the
+ * same here.
+ */
+int
+swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
+	       int dir)
+{
+	struct phys_addr buffer;
+	dma_addr_t dev_addr;
+	char *map;
+	int i;
+
+	BUG_ON(dir == DMA_NONE);
+
+	for (i = 0; i < nelems; i++, sg++) {
+		dev_addr = SG_ENT_PHYS_ADDRESS(sg);
+		if (address_needs_mapping(hwdev, dev_addr)) {
+			buffer.page   = sg->page;
+			buffer.offset = sg->offset;
+			map = map_single(hwdev, buffer, sg->length, dir);
+			if (!map) {
+				/* Don't panic here, we expect map_sg users
+				   to do proper error handling. */
+				swiotlb_full(hwdev, sg->length, dir, 0);
+				swiotlb_unmap_sg(hwdev, sg - i, i, dir);
+				sg[0].dma_length = 0;
+				return 0;
+			}
+			sg->dma_address = (dma_addr_t)virt_to_bus(map);
+		} else
+			sg->dma_address = dev_addr;
+		sg->dma_length = sg->length;
+	}
+	return nelems;
+}
+
+/*
+ * Unmap a set of streaming mode DMA translations.  Again, cpu read rules
+ * concerning calls here are the same as for swiotlb_unmap_single() above.
+ */
+void
+swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
+		 int dir)
+{
+	int i;
+
+	BUG_ON(dir == DMA_NONE);
+
+	for (i = 0; i < nelems; i++, sg++)
+		if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
+			unmap_single(hwdev, 
+				     (void *)bus_to_virt(sg->dma_address),
+				     sg->dma_length, dir);
+}
+
+/*
+ * Make physical memory consistent for a set of streaming mode DMA translations
+ * after a transfer.
+ *
+ * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
+ * and usage.
+ */
+void
+swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
+			int nelems, int dir)
+{
+	int i;
+
+	BUG_ON(dir == DMA_NONE);
+
+	for (i = 0; i < nelems; i++, sg++)
+		if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
+			sync_single(hwdev,
+				    (void *)bus_to_virt(sg->dma_address),
+				    sg->dma_length, dir);
+}
+
+void
+swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
+			   int nelems, int dir)
+{
+	int i;
+
+	BUG_ON(dir == DMA_NONE);
+
+	for (i = 0; i < nelems; i++, sg++)
+		if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
+			sync_single(hwdev,
+				    (void *)bus_to_virt(sg->dma_address),
+				    sg->dma_length, dir);
+}
+
+dma_addr_t
+swiotlb_map_page(struct device *hwdev, struct page *page,
+		 unsigned long offset, size_t size,
+		 enum dma_data_direction direction)
+{
+	struct phys_addr buffer;
+	dma_addr_t dev_addr;
+	char *map;
+
+	dev_addr = page_to_phys(page) + offset;
+	if (address_needs_mapping(hwdev, dev_addr)) {
+		buffer.page   = page;
+		buffer.offset = offset;
+		map = map_single(hwdev, buffer, size, direction);
+		if (!map) {
+			swiotlb_full(hwdev, size, direction, 1);
+			map = io_tlb_overflow_buffer;
+		}
+		dev_addr = (dma_addr_t)virt_to_bus(map);
+	}
+
+	return dev_addr;
+}
+
+void
+swiotlb_unmap_page(struct device *hwdev, dma_addr_t dma_address,
+		   size_t size, enum dma_data_direction direction)
+{
+	char *dma_addr = bus_to_virt(dma_address);
+
+	BUG_ON(direction == DMA_NONE);
+	if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
+		unmap_single(hwdev, dma_addr, size, direction);
+}
+
+int
+swiotlb_dma_mapping_error(dma_addr_t dma_addr)
+{
+	return (dma_addr == virt_to_bus(io_tlb_overflow_buffer));
+}
+
+/*
+ * Return whether the given PCI device DMA address mask can be supported
+ * properly.  For example, if your device can only drive the low 24-bits
+ * during PCI bus mastering, then you would pass 0x00ffffff as the mask to
+ * this function.
+ */
+int
+swiotlb_dma_supported (struct device *hwdev, u64 mask)
+{
+	return (mask >= 0xffffffffUL);
+}
+
+EXPORT_SYMBOL(swiotlb_init);
+EXPORT_SYMBOL(swiotlb_map_single);
+EXPORT_SYMBOL(swiotlb_unmap_single);
+EXPORT_SYMBOL(swiotlb_map_sg);
+EXPORT_SYMBOL(swiotlb_unmap_sg);
+EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
+EXPORT_SYMBOL(swiotlb_sync_single_for_device);
+EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
+EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
+EXPORT_SYMBOL(swiotlb_map_page);
+EXPORT_SYMBOL(swiotlb_unmap_page);
+EXPORT_SYMBOL(swiotlb_dma_mapping_error);
+EXPORT_SYMBOL(swiotlb_dma_supported);
+
+/*
+ * Local variables:
+ *  c-file-style: "linux"
+ *  indent-tabs-mode: t
+ *  c-indent-level: 8
+ *  c-basic-offset: 8
+ *  tab-width: 8
+ * End:
+ */
diff --git a/linux-2.6-xen-sparse/arch/xen/i386/mm/hypervisor.c b/linux-2.6-xen-sparse/arch/xen/i386/mm/hypervisor.c
index 00b9c6eb73..bd75d0622d 100644
--- a/linux-2.6-xen-sparse/arch/xen/i386/mm/hypervisor.c
+++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/hypervisor.c
@@ -263,12 +263,9 @@ void xen_set_ldt(unsigned long ptr, unsigned long len)
     BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 }
 
-void xen_contig_memory(unsigned long vstart, unsigned int order)
+/* Ensure multi-page extents are contiguous in machine memory. */
+void xen_create_contiguous_region(unsigned long vstart, unsigned int order)
 {
-    /*
-     * Ensure multi-page extents are contiguous in machine memory. This code 
-     * could be cleaned up some, and the number of hypercalls reduced.
-     */
     pgd_t         *pgd; 
     pud_t         *pud; 
     pmd_t         *pmd;
@@ -312,6 +309,49 @@ void xen_contig_memory(unsigned long vstart, unsigned int order)
     balloon_unlock(flags);
 }
 
+void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order)
+{
+    pgd_t         *pgd; 
+    pud_t         *pud; 
+    pmd_t         *pmd;
+    pte_t         *pte;
+    unsigned long  mfn, i, flags;
+
+    scrub_pages(vstart, 1 << order);
+
+    balloon_lock(flags);
+
+    /* 1. Zap current PTEs, giving away the underlying pages. */
+    for (i = 0; i < (1<<order); i++) {
+        pgd = pgd_offset_k(vstart + (i*PAGE_SIZE));
+        pud = pud_offset(pgd, (vstart + (i*PAGE_SIZE)));
+        pmd = pmd_offset(pud, (vstart + (i*PAGE_SIZE)));
+        pte = pte_offset_kernel(pmd, (vstart + (i*PAGE_SIZE)));
+        mfn = pte_mfn(*pte);
+        BUG_ON(HYPERVISOR_update_va_mapping(
+            vstart + (i*PAGE_SIZE), __pte_ma(0), 0));
+        phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
+            INVALID_P2M_ENTRY;
+        BUG_ON(HYPERVISOR_dom_mem_op(
+            MEMOP_decrease_reservation, &mfn, 1, 0) != 1);
+    }
+
+    /* 2. Map new pages in place of old pages. */
+    for (i = 0; i < (1<<order); i++) {
+        BUG_ON(HYPERVISOR_dom_mem_op(
+            MEMOP_increase_reservation, &mfn, 1, 0) != 1);
+        BUG_ON(HYPERVISOR_update_va_mapping(
+            vstart + (i*PAGE_SIZE),
+            __pte_ma((mfn<<PAGE_SHIFT)|__PAGE_KERNEL), 0));
+        xen_machphys_update(mfn, (__pa(vstart)>>PAGE_SHIFT)+i);
+        phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] = mfn;
+    }
+
+    flush_tlb_all();
+
+    balloon_unlock(flags);
+}
+
 #ifdef CONFIG_XEN_PHYSDEV_ACCESS
 
 unsigned long allocate_empty_lowmem_region(unsigned long pages)
diff --git a/linux-2.6-xen-sparse/arch/xen/i386/mm/init.c b/linux-2.6-xen-sparse/arch/xen/i386/mm/init.c
index b8dfade5ec..8e26b9860c 100644
--- a/linux-2.6-xen-sparse/arch/xen/i386/mm/init.c
+++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/init.c
@@ -41,6 +41,12 @@
 #include <asm/sections.h>
 #include <asm-xen/hypervisor.h>
 
+#if defined(CONFIG_SWIOTLB)
+extern void swiotlb_init(void);
+int swiotlb;
+EXPORT_SYMBOL(swiotlb);
+#endif
+
 unsigned int __VMALLOC_RESERVE = 128 << 20;
 
 DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
@@ -631,6 +637,10 @@ void __init mem_init(void)
 	int bad_ppro;
 	unsigned long pfn;
 
+#if defined(CONFIG_SWIOTLB)
+	swiotlb_init();	
+#endif
+
 #ifndef CONFIG_DISCONTIGMEM
 	if (!mem_map)
 		BUG();
diff --git a/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c b/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c
index dda871f2bf..5f50fbfd5f 100644
--- a/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c
+++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c
@@ -277,7 +277,7 @@ void pgd_ctor(void *pgd, kmem_cache_t *cache, unsigned long unused)
 
 #ifdef CONFIG_X86_PAE
 	/* this gives us a page below 4GB */
-	xen_contig_memory((unsigned long)pgd, 0);
+	xen_create_contiguous_region((unsigned long)pgd, 0);
 #endif
 
 	if (!HAVE_SHARED_KERNEL_PMD)
diff --git a/linux-2.6-xen-sparse/arch/xen/x86_64/Kconfig b/linux-2.6-xen-sparse/arch/xen/x86_64/Kconfig
index 771835e371..ad6011633a 100644
--- a/linux-2.6-xen-sparse/arch/xen/x86_64/Kconfig
+++ b/linux-2.6-xen-sparse/arch/xen/x86_64/Kconfig
@@ -329,12 +329,12 @@ config GART_IOMMU
 # need this always enabled with GART_IOMMU for the VIA workaround
 config SWIOTLB
        bool
-       depends on GART_IOMMU
+       depends on PCI
        default y
 
 config DUMMY_IOMMU
 	bool
-	depends on !GART_IOMMU && !SWIOTLB
+	depends on !GART_IOMMU
 	default y
 	help
 	  Don't use IOMMU code. This will cause problems when you have more than 4GB
diff --git a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/Makefile b/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/Makefile
index 66c30436cc..c4f555e0d7 100644
--- a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/Makefile
+++ b/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/Makefile
@@ -36,8 +36,9 @@ c-obj-$(CONFIG_X86_IO_APIC)	+= genapic_cluster.o genapic_flat.o
 #obj-$(CONFIG_CPU_FREQ)		+= cpufreq/
 #obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
 #obj-$(CONFIG_GART_IOMMU)	+= pci-gart.o aperture.o
-obj-$(CONFIG_DUMMY_IOMMU)	+= pci-nommu.o pci-dma.o
-#obj-$(CONFIG_SWIOTLB)		+= swiotlb.o
+obj-$(CONFIG_DUMMY_IOMMU)	+= pci-nommu.o
+i386-obj-$(CONFIG_DUMMY_IOMMU)	+= pci-dma.o
+i386-obj-$(CONFIG_SWIOTLB)	+= swiotlb.o
 obj-$(CONFIG_KPROBES)		+= kprobes.o
 obj-$(CONFIG_X86_PM_TIMER)	+= pmtimer.o
 
@@ -49,7 +50,7 @@ c-obj-y				+= intel_cacheinfo.o
 bootflag-y			+= ../../../i386/kernel/bootflag.o
 cpuid-$(subst m,y,$(CONFIG_X86_CPUID))  += ../../../i386/kernel/cpuid.o
 topology-y                     += ../../../i386/mach-default/topology.o
-swiotlb-$(CONFIG_SWIOTLB)      += ../../../ia64/lib/swiotlb.o
+#swiotlb-$(CONFIG_SWIOTLB)      += ../../../ia64/lib/swiotlb.o
 microcode-$(subst m,y,$(CONFIG_MICROCODE))  += ../../../i386/kernel/microcode.o
 intel_cacheinfo-y		+= ../../../i386/kernel/cpu/intel_cacheinfo.o
 quirks-y			+= ../../i386/kernel/quirks.o
diff --git a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c b/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c
deleted file mode 100644
index 96b2ee64bc..0000000000
--- a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Dynamic DMA mapping support.
- */
-
-#include <linux/types.h>
-#include <linux/mm.h>
-#include <linux/string.h>
-#include <linux/pci.h>
-#include <linux/module.h>
-#include <asm/io.h>
-#include <asm-xen/balloon.h>
-
-/* Map a set of buffers described by scatterlist in streaming
- * mode for DMA.  This is the scatter-gather version of the
- * above pci_map_single interface.  Here the scatter gather list
- * elements are each tagged with the appropriate dma address
- * and length.  They are obtained via sg_dma_{address,length}(SG).
- *
- * NOTE: An implementation may be able to use a smaller number of
- *       DMA address/length pairs than there are SG table elements.
- *       (for example via virtual mapping capabilities)
- *       The routine returns the number of addr/length pairs actually
- *       used, at most nents.
- *
- * Device ownership issues as mentioned above for pci_map_single are
- * the same here.
- */
-int dma_map_sg(struct device *hwdev, struct scatterlist *sg,
-	       int nents, int direction)
-{
-	int i;
-
-	BUG_ON(direction == DMA_NONE);
- 	for (i = 0; i < nents; i++ ) {
-		struct scatterlist *s = &sg[i];
-		BUG_ON(!s->page); 
-		s->dma_address = virt_to_bus(page_address(s->page) +s->offset);
-		s->dma_length = s->length;
-	}
-	return nents;
-}
-
-EXPORT_SYMBOL(dma_map_sg);
-
-/* Unmap a set of streaming mode DMA translations.
- * Again, cpu read rules concerning calls here are the same as for
- * pci_unmap_single() above.
- */
-void dma_unmap_sg(struct device *dev, struct scatterlist *sg,
-		  int nents, int dir)
-{
-	int i;
-	for (i = 0; i < nents; i++) { 
-		struct scatterlist *s = &sg[i];
-		BUG_ON(s->page == NULL); 
-		BUG_ON(s->dma_address == 0); 
-		dma_unmap_single(dev, s->dma_address, s->dma_length, dir);
-	} 
-}
-
-EXPORT_SYMBOL(dma_unmap_sg);
-
-struct dma_coherent_mem {
-	void		*virt_base;
-	u32		device_base;
-	int		size;
-	int		flags;
-	unsigned long	*bitmap;
-};
-
-void *dma_alloc_coherent(struct device *dev, size_t size,
-			   dma_addr_t *dma_handle, unsigned gfp)
-{
-	void *ret;
-	unsigned int order = get_order(size);
-	unsigned long vstart;
-
-	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
-
-	/* ignore region specifiers */
-	gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
-
-	if (mem) {
-		int page = bitmap_find_free_region(mem->bitmap, mem->size,
-						     order);
-		if (page >= 0) {
-			*dma_handle = mem->device_base + (page << PAGE_SHIFT);
-			ret = mem->virt_base + (page << PAGE_SHIFT);
-			memset(ret, 0, size);
-			return ret;
-		}
-		if (mem->flags & DMA_MEMORY_EXCLUSIVE)
-			return NULL;
-	}
-
-	if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
-		gfp |= GFP_DMA;
-
-	vstart = __get_free_pages(gfp, order);
-	ret = (void *)vstart;
-	if (ret == NULL)
-		return ret;
-
-	xen_contig_memory(vstart, order);
-
-	memset(ret, 0, size);
-	*dma_handle = virt_to_bus(ret);
-
-	return ret;
-}
-EXPORT_SYMBOL(dma_alloc_coherent);
-
-void dma_free_coherent(struct device *dev, size_t size,
-			 void *vaddr, dma_addr_t dma_handle)
-{
-	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
-	int order = get_order(size);
-	
-	if (mem && vaddr >= mem->virt_base && vaddr < (mem->virt_base + (mem->size << PAGE_SHIFT))) {
-		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
-
-		bitmap_release_region(mem->bitmap, page, order);
-	} else
-		free_pages((unsigned long)vaddr, order);
-}
-EXPORT_SYMBOL(dma_free_coherent);
-
-#if 0
-int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
-				dma_addr_t device_addr, size_t size, int flags)
-{
-	void __iomem *mem_base;
-	int pages = size >> PAGE_SHIFT;
-	int bitmap_size = (pages + 31)/32;
-
-	if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
-		goto out;
-	if (!size)
-		goto out;
-	if (dev->dma_mem)
-		goto out;
-
-	/* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
-
-	mem_base = ioremap(bus_addr, size);
-	if (!mem_base)
-		goto out;
-
-	dev->dma_mem = kmalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
-	if (!dev->dma_mem)
-		goto out;
-	memset(dev->dma_mem, 0, sizeof(struct dma_coherent_mem));
-	dev->dma_mem->bitmap = kmalloc(bitmap_size, GFP_KERNEL);
-	if (!dev->dma_mem->bitmap)
-		goto free1_out;
-	memset(dev->dma_mem->bitmap, 0, bitmap_size);
-
-	dev->dma_mem->virt_base = mem_base;
-	dev->dma_mem->device_base = device_addr;
-	dev->dma_mem->size = pages;
-	dev->dma_mem->flags = flags;
-
-	if (flags & DMA_MEMORY_MAP)
-		return DMA_MEMORY_MAP;
-
-	return DMA_MEMORY_IO;
-
- free1_out:
-	kfree(dev->dma_mem->bitmap);
- out:
-	return 0;
-}
-EXPORT_SYMBOL(dma_declare_coherent_memory);
-
-void dma_release_declared_memory(struct device *dev)
-{
-	struct dma_coherent_mem *mem = dev->dma_mem;
-	
-	if(!mem)
-		return;
-	dev->dma_mem = NULL;
-	iounmap(mem->virt_base);
-	kfree(mem->bitmap);
-	kfree(mem);
-}
-EXPORT_SYMBOL(dma_release_declared_memory);
-
-void *dma_mark_declared_memory_occupied(struct device *dev,
-					dma_addr_t device_addr, size_t size)
-{
-	struct dma_coherent_mem *mem = dev->dma_mem;
-	int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1) >> PAGE_SHIFT;
-	int pos, err;
-
-	if (!mem)
-		return ERR_PTR(-EINVAL);
-
-	pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
-	err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
-	if (err != 0)
-		return ERR_PTR(err);
-	return mem->virt_base + (pos << PAGE_SHIFT);
-}
-EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
-#endif
-
-static LIST_HEAD(dma_map_head);
-static DEFINE_SPINLOCK(dma_map_lock);
-struct dma_map_entry {
-	struct list_head list;
-	dma_addr_t dma;
-	char *bounce, *host;
-	size_t size;
-};
-#define DMA_MAP_MATCHES(e,d) (((e)->dma<=(d)) && (((e)->dma+(e)->size)>(d)))
-
-dma_addr_t
-dma_map_single(struct device *dev, void *ptr, size_t size,
-	       enum dma_data_direction direction)
-{
-	struct dma_map_entry *ent;
-	void *bnc;
-	dma_addr_t dma;
-	unsigned long flags;
-
-	if (direction == DMA_NONE)
-		out_of_line_bug();
-
-	/*
-	 * Even if size is sub-page, the buffer may still straddle a page
-	 * boundary. Take into account buffer start offset. All other calls are
-	 * conservative and always search the dma_map list if it's non-empty.
-	 */
-	if (((((unsigned long)ptr) & ~PAGE_MASK) + size) <= PAGE_SIZE) {
-		dma = virt_to_bus(ptr);
-	} else {
-		BUG_ON((bnc = dma_alloc_coherent(dev, size, &dma, GFP_ATOMIC)) == NULL);
-		BUG_ON((ent = kmalloc(sizeof(*ent), GFP_ATOMIC)) == NULL);
-		if (direction != DMA_FROM_DEVICE)
-			memcpy(bnc, ptr, size);
-		ent->dma    = dma;
-		ent->bounce = bnc;
-		ent->host   = ptr;
-		ent->size   = size;
-		spin_lock_irqsave(&dma_map_lock, flags);
-		list_add(&ent->list, &dma_map_head);
-		spin_unlock_irqrestore(&dma_map_lock, flags);
-	}
-
-	if ((dma+size) & ~*dev->dma_mask)
-		out_of_line_bug();
-	return dma;
-}
-EXPORT_SYMBOL(dma_map_single);
-
-void
-dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
-		 enum dma_data_direction direction)
-{
-	struct dma_map_entry *ent;
-	unsigned long flags;
-
-	if (direction == DMA_NONE)
-		out_of_line_bug();
-
-	/* Fast-path check: are there any multi-page DMA mappings? */
-	if (!list_empty(&dma_map_head)) {
-		spin_lock_irqsave(&dma_map_lock, flags);
-		list_for_each_entry ( ent, &dma_map_head, list ) {
-			if (DMA_MAP_MATCHES(ent, dma_addr)) {
-				list_del(&ent->list);
-				break;
-			}
-		}
-		spin_unlock_irqrestore(&dma_map_lock, flags);
-		if (&ent->list != &dma_map_head) {
-			BUG_ON(dma_addr != ent->dma);
-			BUG_ON(size != ent->size);
-			if (direction != DMA_TO_DEVICE)
-				memcpy(ent->host, ent->bounce, size);
-			dma_free_coherent(dev, size, ent->bounce, ent->dma);
-			kfree(ent);
-		}
-	}
-}
-EXPORT_SYMBOL(dma_unmap_single);
-
-void
-dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
-			enum dma_data_direction direction)
-{
-	struct dma_map_entry *ent;
-	unsigned long flags, off;
-
-	/* Fast-path check: are there any multi-page DMA mappings? */
-	if (!list_empty(&dma_map_head)) {
-		spin_lock_irqsave(&dma_map_lock, flags);
-		list_for_each_entry ( ent, &dma_map_head, list )
-			if (DMA_MAP_MATCHES(ent, dma_handle))
-				break;
-		spin_unlock_irqrestore(&dma_map_lock, flags);
-		if (&ent->list != &dma_map_head) {
-			off = dma_handle - ent->dma;
-			BUG_ON((off + size) > ent->size);
-			/*if (direction != DMA_TO_DEVICE)*/
-				memcpy(ent->host+off, ent->bounce+off, size);
-		}
-	}
-}
-EXPORT_SYMBOL(dma_sync_single_for_cpu);
-
-void
-dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size,
-                           enum dma_data_direction direction)
-{
-	struct dma_map_entry *ent;
-	unsigned long flags, off;
-
-	/* Fast-path check: are there any multi-page DMA mappings? */
-	if (!list_empty(&dma_map_head)) {
-		spin_lock_irqsave(&dma_map_lock, flags);
-		list_for_each_entry ( ent, &dma_map_head, list )
-			if (DMA_MAP_MATCHES(ent, dma_handle))
-				break;
-		spin_unlock_irqrestore(&dma_map_lock, flags);
-		if (&ent->list != &dma_map_head) {
-			off = dma_handle - ent->dma;
-			BUG_ON((off + size) > ent->size);
-			/*if (direction != DMA_FROM_DEVICE)*/
-				memcpy(ent->bounce+off, ent->host+off, size);
-		}
-	}
-
-	flush_write_buffers();
-}
-EXPORT_SYMBOL(dma_sync_single_for_device);
diff --git a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-nommu.c b/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-nommu.c
index b359ae80bb..3636461a37 100644
--- a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-nommu.c
+++ b/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-nommu.c
@@ -61,6 +61,7 @@ void dma_free_coherent(struct device *hwdev, size_t size,
 EXPORT_SYMBOL(dma_free_coherent);
 #endif
 
+#if 0
 int dma_supported(struct device *hwdev, u64 mask)
 {
         /*
@@ -76,6 +77,7 @@ int dma_supported(struct device *hwdev, u64 mask)
 	return 1;
 } 
 EXPORT_SYMBOL(dma_supported);
+#endif
 
 int dma_get_cache_alignment(void)
 {
diff --git a/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c b/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c
index 26718dde73..bed448639f 100644
--- a/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c
+++ b/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c
@@ -44,10 +44,6 @@
 #define Dprintk(x...)
 #endif
 
-#ifdef CONFIG_GART_IOMMU
-extern int swiotlb;
-#endif
-
 extern char _stext[];
 
 DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
@@ -790,8 +786,6 @@ static inline int page_is_ram (unsigned long pagenr)
         return 1;
 }
 
-extern int swiotlb_force;
-
 static struct kcore_list kcore_mem, kcore_vmalloc, kcore_kernel, kcore_modules,
 			 kcore_vsyscall;
 
@@ -800,14 +794,9 @@ void __init mem_init(void)
 	int codesize, reservedpages, datasize, initsize;
 	int tmp;
 
-#ifdef CONFIG_SWIOTLB
-	if (swiotlb_force)
-		swiotlb = 1;
-	if (!iommu_aperture &&
-	    (end_pfn >= 0xffffffff>>PAGE_SHIFT || force_iommu))
-	       swiotlb = 1;
-	if (swiotlb)
-		swiotlb_init();	
+#if defined(CONFIG_SWIOTLB)
+	extern void swiotlb_init(void);
+	swiotlb_init();	
 #endif
 
 	/* How many end-of-memory variables you have, grandma! */
diff --git a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h
index 2dc373e733..4501817a84 100644
--- a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h
@@ -1,11 +1,33 @@
 #ifndef _ASM_I386_DMA_MAPPING_H
 #define _ASM_I386_DMA_MAPPING_H
 
-#include <linux/mm.h>
+/*
+ * IOMMU interface. See Documentation/DMA-mapping.txt and DMA-API.txt for
+ * documentation.
+ */
 
+#include <linux/config.h>
+#include <linux/mm.h>
 #include <asm/cache.h>
 #include <asm/io.h>
 #include <asm/scatterlist.h>
+#include <asm-i386/swiotlb.h>
+
+static inline int
+address_needs_mapping(struct device *hwdev, dma_addr_t addr)
+{
+	dma_addr_t mask = 0xffffffff;
+	/* If the device has a mask, use it, otherwise default to 32 bits */
+	if (hwdev && hwdev->dma_mask)
+		mask = *hwdev->dma_mask;
+	return (addr & ~mask) != 0;
+}
+
+static inline int
+range_straddles_page_boundary(void *p, size_t size)
+{
+	return ((((unsigned long)p & ~PAGE_MASK) + size) > PAGE_SIZE);
+}
 
 #define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f)
 #define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h)
@@ -24,46 +46,18 @@ extern void
 dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
 		 enum dma_data_direction direction);
 
-static inline int
-dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
-	   enum dma_data_direction direction)
-{
-	int i;
-
-	BUG_ON(direction == DMA_NONE);
-
-	for (i = 0; i < nents; i++ ) {
-		BUG_ON(!sg[i].page);
+extern int dma_map_sg(struct device *hwdev, struct scatterlist *sg,
+		      int nents, enum dma_data_direction direction);
+extern void dma_unmap_sg(struct device *hwdev, struct scatterlist *sg,
+			 int nents, enum dma_data_direction direction);
 
-		sg[i].dma_address = page_to_phys(sg[i].page) + sg[i].offset;
-	}
-
-	flush_write_buffers();
-	return nents;
-}
-
-static inline dma_addr_t
+extern dma_addr_t
 dma_map_page(struct device *dev, struct page *page, unsigned long offset,
-	     size_t size, enum dma_data_direction direction)
-{
-	BUG_ON(direction == DMA_NONE);
-	return page_to_phys(page) + offset;
-}
+	     size_t size, enum dma_data_direction direction);
 
-static inline void
+extern void
 dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
-	       enum dma_data_direction direction)
-{
-	BUG_ON(direction == DMA_NONE);
-}
-
-
-static inline void
-dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries,
-	     enum dma_data_direction direction)
-{
-	BUG_ON(direction == DMA_NONE);
-}
+	       enum dma_data_direction direction);
 
 extern void
 dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
@@ -93,34 +87,25 @@ static inline void
 dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems,
 		    enum dma_data_direction direction)
 {
+	if (swiotlb)
+		swiotlb_sync_sg_for_cpu(dev,sg,nelems,direction);
+	flush_write_buffers();
 }
 
 static inline void
 dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems,
 		    enum dma_data_direction direction)
 {
+	if (swiotlb)
+		swiotlb_sync_sg_for_device(dev,sg,nelems,direction);
 	flush_write_buffers();
 }
 
-static inline int
-dma_mapping_error(dma_addr_t dma_addr)
-{
-	return 0;
-}
+extern int
+dma_mapping_error(dma_addr_t dma_addr);
 
-static inline int
-dma_supported(struct device *dev, u64 mask)
-{
-        /*
-         * we fall back to GFP_DMA when the mask isn't all 1s,
-         * so we can't guarantee allocations that must be
-         * within a tighter range than GFP_DMA..
-         */
-        if(mask < 0x00ffffff)
-                return 0;
-
-	return 1;
-}
+extern int
+dma_supported(struct device *dev, u64 mask);
 
 static inline int
 dma_set_mask(struct device *dev, u64 mask)
@@ -133,6 +118,7 @@ dma_set_mask(struct device *dev, u64 mask)
 	return 0;
 }
 
+#ifdef __i386__
 static inline int
 dma_get_cache_alignment(void)
 {
@@ -140,6 +126,9 @@ dma_get_cache_alignment(void)
 	 * maximum possible, to be safe */
 	return (1 << L1_CACHE_SHIFT_MAX);
 }
+#else
+extern int dma_get_cache_alignment(void);
+#endif
 
 #define dma_is_consistent(d)	(1)
 
diff --git a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/kmap_types.h b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/kmap_types.h
new file mode 100644
index 0000000000..7a660e8e24
--- /dev/null
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/kmap_types.h
@@ -0,0 +1,32 @@
+#ifndef _ASM_KMAP_TYPES_H
+#define _ASM_KMAP_TYPES_H
+
+#include <linux/config.h>
+
+#ifdef CONFIG_DEBUG_HIGHMEM
+# define D(n) __KM_FENCE_##n ,
+#else
+# define D(n)
+#endif
+
+enum km_type {
+D(0)	KM_BOUNCE_READ,
+D(1)	KM_SKB_SUNRPC_DATA,
+D(2)	KM_SKB_DATA_SOFTIRQ,
+D(3)	KM_USER0,
+D(4)	KM_USER1,
+D(5)	KM_BIO_SRC_IRQ,
+D(6)	KM_BIO_DST_IRQ,
+D(7)	KM_PTE0,
+D(8)	KM_PTE1,
+D(9)	KM_IRQ0,
+D(10)	KM_IRQ1,
+D(11)	KM_SOFTIRQ0,
+D(12)	KM_SOFTIRQ1,
+D(13)	KM_SWIOTLB,
+D(14)	KM_TYPE_NR
+};
+
+#undef D
+
+#endif
diff --git a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/pci.h b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/pci.h
index 32d08ada26..707f4da83f 100644
--- a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/pci.h
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/pci.h
@@ -43,11 +43,8 @@ int pcibios_set_irq_routing(struct pci_dev *dev, int pin, int irq);
 
 struct pci_dev;
 
-/* The PCI address space does equal the physical memory
- * address space.  The networking and block device layers use
- * this boolean for bounce buffer decisions.
- */
-#define PCI_DMA_BUS_IS_PHYS	(1)
+/* On Xen we use SWIOTLB instead of blk-specific bounce buffers. */
+#define PCI_DMA_BUS_IS_PHYS	(0)
 
 /* pci_unmap_{page,single} is a nop so... */
 #define DECLARE_PCI_UNMAP_ADDR(ADDR_NAME)
diff --git a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/scatterlist.h b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/scatterlist.h
new file mode 100644
index 0000000000..bec2ff3c8b
--- /dev/null
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/scatterlist.h
@@ -0,0 +1,22 @@
+#ifndef _I386_SCATTERLIST_H
+#define _I386_SCATTERLIST_H
+
+struct scatterlist {
+    struct page		*page;
+    unsigned int	offset;
+    unsigned int	length;
+    dma_addr_t		dma_address;
+    unsigned int	dma_length;
+};
+
+/* These macros should be used after a pci_map_sg call has been done
+ * to get bus addresses of each of the SG entries and their lengths.
+ * You should only work with the number of sg entries pci_map_sg
+ * returns.
+ */
+#define sg_dma_address(sg)	((sg)->dma_address)
+#define sg_dma_len(sg)		((sg)->dma_length)
+
+#define ISA_DMA_THRESHOLD (0x00ffffff)
+
+#endif /* !(_I386_SCATTERLIST_H) */
diff --git a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/swiotlb.h b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/swiotlb.h
new file mode 100644
index 0000000000..b38e56f1f5
--- /dev/null
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/swiotlb.h
@@ -0,0 +1,42 @@
+#ifndef _ASM_SWIOTLB_H
+#define _ASM_SWIOTLB_H 1
+
+#include <linux/config.h>
+
+/* SWIOTLB interface */
+
+extern dma_addr_t swiotlb_map_single(struct device *hwdev, void *ptr, size_t size,
+				      int dir);
+extern void swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr,
+				  size_t size, int dir);
+extern void swiotlb_sync_single_for_cpu(struct device *hwdev,
+					 dma_addr_t dev_addr,
+					 size_t size, int dir);
+extern void swiotlb_sync_single_for_device(struct device *hwdev,
+					    dma_addr_t dev_addr,
+					    size_t size, int dir);
+extern void swiotlb_sync_sg_for_cpu(struct device *hwdev,
+				     struct scatterlist *sg, int nelems,
+				     int dir);
+extern void swiotlb_sync_sg_for_device(struct device *hwdev,
+					struct scatterlist *sg, int nelems,
+					int dir);
+extern int swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg,
+		      int nents, int direction);
+extern void swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg,
+			 int nents, int direction);
+extern int swiotlb_dma_mapping_error(dma_addr_t dma_addr);
+extern dma_addr_t swiotlb_map_page(struct device *hwdev, struct page *page,
+                                   unsigned long offset, size_t size,
+                                   enum dma_data_direction direction);
+extern void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dma_address,
+                               size_t size, enum dma_data_direction direction);
+extern int swiotlb_dma_supported(struct device *hwdev, u64 mask);
+
+#ifdef CONFIG_SWIOTLB
+extern int swiotlb;
+#else
+#define swiotlb 0
+#endif
+
+#endif
diff --git a/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h b/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h
index b280f92ad9..773454240b 100644
--- a/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h
@@ -1,89 +1 @@
-#ifndef _X8664_DMA_MAPPING_H
-#define _X8664_DMA_MAPPING_H 1
-
-/*
- * IOMMU interface. See Documentation/DMA-mapping.txt and DMA-API.txt for
- * documentation.
- */
-
-#include <linux/config.h>
-
-#include <asm/scatterlist.h>
-#include <asm/io.h>
-#include <asm/swiotlb.h>
-
-extern dma_addr_t bad_dma_address;
-#define dma_mapping_error(x) \
-	(swiotlb ? swiotlb_dma_mapping_error(x) : ((x) == bad_dma_address))
-
-void *dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
-			 unsigned gfp);
-void dma_free_coherent(struct device *dev, size_t size, void *vaddr,
-			 dma_addr_t dma_handle);
-
-extern dma_addr_t dma_map_single(struct device *hwdev, void *ptr, size_t size,
-				 enum dma_data_direction direction);
-extern void dma_unmap_single(struct device *dev, dma_addr_t addr,size_t size,
-			     enum dma_data_direction direction);
-
-#define dma_map_page(dev,page,offset,size,dir) \
-	dma_map_single((dev), page_address(page)+(offset), (size), (dir))
-
-extern void
-dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
-			enum dma_data_direction direction);
-
-extern void
-dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size,
-                           enum dma_data_direction direction);
-
-static inline void dma_sync_sg_for_cpu(struct device *hwdev,
-				       struct scatterlist *sg,
-				       int nelems, int direction)
-{
-	if (direction == DMA_NONE)
-		out_of_line_bug();
-
-	if (swiotlb)
-		return swiotlb_sync_sg_for_cpu(hwdev,sg,nelems,direction);
-
-	flush_write_buffers();
-}
-
-static inline void dma_sync_sg_for_device(struct device *hwdev,
-					  struct scatterlist *sg,
-					  int nelems, int direction)
-{
-	if (direction == DMA_NONE)
-		out_of_line_bug();
-
-	if (swiotlb)
-		return swiotlb_sync_sg_for_device(hwdev,sg,nelems,direction);
-
-	flush_write_buffers();
-}
-
-extern int dma_map_sg(struct device *hwdev, struct scatterlist *sg,
-		      int nents, int direction);
-extern void dma_unmap_sg(struct device *hwdev, struct scatterlist *sg,
-			 int nents, int direction);
-
-#define dma_unmap_page dma_unmap_single
-
-extern int dma_supported(struct device *hwdev, u64 mask);
-extern int dma_get_cache_alignment(void);
-#define dma_is_consistent(h) 1
-
-static inline int dma_set_mask(struct device *dev, u64 mask)
-{
-	if (!dev->dma_mask || !dma_supported(dev, mask))
-		return -EIO;
-	*dev->dma_mask = mask;
-	return 0;
-}
-
-static inline void dma_cache_sync(void *vaddr, size_t size, enum dma_data_direction dir)
-{
-	flush_write_buffers();
-}
-#endif
+#include <asm-i386/dma-mapping.h>
diff --git a/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pci.h b/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pci.h
index 039046fdc8..119ccdfb72 100644
--- a/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pci.h
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pci.h
@@ -79,7 +79,9 @@ extern int iommu_sac_force;
 #else
 /* No IOMMU */
 
-#define PCI_DMA_BUS_IS_PHYS	1
+/* On Xen we use SWIOTLB instead of blk-specific bounce buffers. */
+#define PCI_DMA_BUS_IS_PHYS	(0)
+
 #define pci_dac_dma_supported(pci_dev, mask)    1
 
 #define DECLARE_PCI_UNMAP_ADDR(ADDR_NAME)
diff --git a/linux-2.6-xen-sparse/include/asm-xen/hypervisor.h b/linux-2.6-xen-sparse/include/asm-xen/hypervisor.h
index 0787cc3991..af2dacf9ee 100644
--- a/linux-2.6-xen-sparse/include/asm-xen/hypervisor.h
+++ b/linux-2.6-xen-sparse/include/asm-xen/hypervisor.h
@@ -134,7 +134,8 @@ void xen_invlpg_mask(cpumask_t *mask, unsigned long ptr);
 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
 #endif /* linux < 2.6.0 */
 
-void xen_contig_memory(unsigned long vstart, unsigned int order);
+void xen_create_contiguous_region(unsigned long vstart, unsigned int order);
+void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order);
 
 #ifdef CONFIG_XEN_PHYSDEV_ACCESS
 /* Allocate a contiguous empty region of low memory. Return virtual start. */