[LINUX] gnttab: Add basic DMA tracking

This patch adds basic tracking of outstanding DMA requests on
grant table entries marked as PageForeign.

When a PageForeign struct page is about to be mapped for DMA,
we set its map count to 1 (or zero in actual value).  This is
then checked for when we need to free a grant table entry early
to ensure that we don't free an entry that's currently used for
DMA.

So any entry that has been marked for DMA will not be freed early.

If the unmapping API had a struct page (which exists for the sg
case) then we could do this properly.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

4 files changed, 169 insertions, 11 deletions

diff --git a/linux-2.6-xen-sparse/arch/i386/kernel/pci-dma-xen.c b/linux-2.6-xen-sparse/arch/i386/kernel/pci-dma-xen.c
index 70ba380bdb..bafc920884 100644
--- a/linux-2.6-xen-sparse/arch/i386/kernel/pci-dma-xen.c
+++ b/linux-2.6-xen-sparse/arch/i386/kernel/pci-dma-xen.c
@@ -15,6 +15,7 @@
 #include <linux/version.h>
 #include <asm/io.h>
 #include <xen/balloon.h>
+#include <xen/gnttab.h>
 #include <asm/swiotlb.h>
 #include <asm/tlbflush.h>
 #include <asm-i386/mach-xen/asm/swiotlb.h>
@@ -90,7 +91,7 @@ dma_map_sg(struct device *hwdev, struct scatterlist *sg, int nents,
 	} else {
 		for (i = 0; i < nents; i++ ) {
 			sg[i].dma_address =
-				page_to_bus(sg[i].page) + sg[i].offset;
+				gnttab_dma_map_page(sg[i].page) + sg[i].offset;
 			sg[i].dma_length  = sg[i].length;
 			BUG_ON(!sg[i].page);
 			IOMMU_BUG_ON(address_needs_mapping(
@@ -108,9 +109,15 @@ void
 dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents,
 	     enum dma_data_direction direction)
 {
+	int i;
+
 	BUG_ON(direction == DMA_NONE);
 	if (swiotlb)
 		swiotlb_unmap_sg(hwdev, sg, nents, direction);
+	else {
+		for (i = 0; i < nents; i++ )
+			gnttab_dma_unmap_page(sg[i].dma_address);
+	}
 }
 EXPORT_SYMBOL(dma_unmap_sg);
 
@@ -127,7 +134,7 @@ dma_map_page(struct device *dev, struct page *page, unsigned long offset,
 		dma_addr = swiotlb_map_page(
 			dev, page, offset, size, direction);
 	} else {
-		dma_addr = page_to_bus(page) + offset;
+		dma_addr = gnttab_dma_map_page(page) + offset;
 		IOMMU_BUG_ON(address_needs_mapping(dev, dma_addr));
 	}
 
@@ -142,6 +149,8 @@ dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
 	BUG_ON(direction == DMA_NONE);
 	if (swiotlb)
 		swiotlb_unmap_page(dev, dma_address, size, direction);
+	else
+		gnttab_dma_unmap_page(dma_address);
 }
 EXPORT_SYMBOL(dma_unmap_page);
 #endif /* CONFIG_HIGHMEM */
@@ -326,7 +335,8 @@ dma_map_single(struct device *dev, void *ptr, size_t size,
 	if (swiotlb) {
 		dma = swiotlb_map_single(dev, ptr, size, direction);
 	} else {
-		dma = virt_to_bus(ptr);
+		dma = gnttab_dma_map_page(virt_to_page(ptr)) +
+		      offset_in_page(ptr);
 		IOMMU_BUG_ON(range_straddles_page_boundary(ptr, size));
 		IOMMU_BUG_ON(address_needs_mapping(dev, dma));
 	}
@@ -344,6 +354,8 @@ dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
 		BUG();
 	if (swiotlb)
 		swiotlb_unmap_single(dev, dma_addr, size, direction);
+	else
+		gnttab_dma_unmap_page(dma_addr);
 }
 EXPORT_SYMBOL(dma_unmap_single);
 
diff --git a/linux-2.6-xen-sparse/arch/i386/kernel/swiotlb.c b/linux-2.6-xen-sparse/arch/i386/kernel/swiotlb.c
index ca8f42f974..bac31125ce 100644
--- a/linux-2.6-xen-sparse/arch/i386/kernel/swiotlb.c
+++ b/linux-2.6-xen-sparse/arch/i386/kernel/swiotlb.c
@@ -25,6 +25,7 @@
 #include <asm/pci.h>
 #include <asm/dma.h>
 #include <asm/uaccess.h>
+#include <xen/gnttab.h>
 #include <xen/interface/memory.h>
 
 int swiotlb;
@@ -32,8 +33,6 @@ EXPORT_SYMBOL(swiotlb);
 
 #define OFFSET(val,align) ((unsigned long)((val) & ( (align) - 1)))
 
-#define SG_ENT_PHYS_ADDRESS(sg)	(page_to_bus((sg)->page) + (sg)->offset)
-
 /*
  * Maximum allowable number of contiguous slabs to map,
  * must be a power of 2.  What is the appropriate value ?
@@ -468,7 +467,8 @@ swiotlb_full(struct device *dev, size_t size, int dir, int do_panic)
 dma_addr_t
 swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
 {
-	dma_addr_t dev_addr = virt_to_bus(ptr);
+	dma_addr_t dev_addr = gnttab_dma_map_page(virt_to_page(ptr)) +
+			      offset_in_page(ptr);
 	void *map;
 	struct phys_addr buffer;
 
@@ -486,6 +486,7 @@ swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
 	/*
 	 * Oh well, have to allocate and map a bounce buffer.
 	 */
+	gnttab_dma_unmap_page(dev_addr);
 	buffer.page   = virt_to_page(ptr);
 	buffer.offset = (unsigned long)ptr & ~PAGE_MASK;
 	map = map_single(hwdev, buffer, size, dir);
@@ -513,6 +514,8 @@ swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
 	BUG_ON(dir == DMA_NONE);
 	if (in_swiotlb_aperture(dev_addr))
 		unmap_single(hwdev, bus_to_virt(dev_addr), size, dir);
+	else
+		gnttab_dma_unmap_page(dev_addr);
 }
 
 /*
@@ -571,8 +574,10 @@ swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
 	BUG_ON(dir == DMA_NONE);
 
 	for (i = 0; i < nelems; i++, sg++) {
-		dev_addr = SG_ENT_PHYS_ADDRESS(sg);
+		dev_addr = gnttab_dma_map_page(sg->page) + sg->offset;
+
 		if (address_needs_mapping(hwdev, dev_addr)) {
+			gnttab_dma_unmap_page(dev_addr);
 			buffer.page   = sg->page;
 			buffer.offset = sg->offset;
 			map = map_single(hwdev, buffer, sg->length, dir);
@@ -605,10 +610,12 @@ swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
 	BUG_ON(dir == DMA_NONE);
 
 	for (i = 0; i < nelems; i++, sg++)
-		if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
+		if (in_swiotlb_aperture(sg->dma_address))
 			unmap_single(hwdev, 
 				     (void *)bus_to_virt(sg->dma_address),
 				     sg->dma_length, dir);
+		else
+			gnttab_dma_unmap_page(sg->dma_address);
 }
 
 /*
@@ -627,7 +634,7 @@ swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
 	BUG_ON(dir == DMA_NONE);
 
 	for (i = 0; i < nelems; i++, sg++)
-		if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
+		if (in_swiotlb_aperture(sg->dma_address))
 			sync_single(hwdev,
 				    (void *)bus_to_virt(sg->dma_address),
 				    sg->dma_length, dir);
@@ -642,7 +649,7 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
 	BUG_ON(dir == DMA_NONE);
 
 	for (i = 0; i < nelems; i++, sg++)
-		if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
+		if (in_swiotlb_aperture(sg->dma_address))
 			sync_single(hwdev,
 				    (void *)bus_to_virt(sg->dma_address),
 				    sg->dma_length, dir);
@@ -659,8 +666,9 @@ swiotlb_map_page(struct device *hwdev, struct page *page,
 	dma_addr_t dev_addr;
 	char *map;
 
-	dev_addr = page_to_bus(page) + offset;
+	dev_addr = gnttab_dma_map_page(page) + offset;
 	if (address_needs_mapping(hwdev, dev_addr)) {
+		gnttab_dma_unmap_page(dev_addr);
 		buffer.page   = page;
 		buffer.offset = offset;
 		map = map_single(hwdev, buffer, size, direction);
@@ -681,6 +689,8 @@ swiotlb_unmap_page(struct device *hwdev, dma_addr_t dma_address,
 	BUG_ON(direction == DMA_NONE);
 	if (in_swiotlb_aperture(dma_address))
 		unmap_single(hwdev, bus_to_virt(dma_address), size, direction);
+	else
+		gnttab_dma_unmap_page(dma_address);
 }
 
 #endif
diff --git a/linux-2.6-xen-sparse/drivers/xen/core/gnttab.c b/linux-2.6-xen-sparse/drivers/xen/core/gnttab.c
index b63ddecfd0..1e015582f0 100644
--- a/linux-2.6-xen-sparse/drivers/xen/core/gnttab.c
+++ b/linux-2.6-xen-sparse/drivers/xen/core/gnttab.c
@@ -490,6 +490,128 @@ static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 	return 0;
 }
 
+static void gnttab_page_free(struct page *page)
+{
+	if (page->mapping) {
+		put_page((struct page *)page->mapping);
+		page->mapping = NULL;
+	}
+
+	ClearPageForeign(page);
+	gnttab_reset_grant_page(page);
+	put_page(page);
+}
+
+/*
+ * Must not be called with IRQs off.  This should only be used on the
+ * slow path.
+ *
+ * Copy a foreign granted page to local memory.
+ */
+int gnttab_copy_grant_page(grant_ref_t ref, struct page **pagep)
+{
+	struct gnttab_unmap_and_replace unmap;
+	mmu_update_t mmu;
+	struct page *page;
+	struct page *new_page;
+	void *new_addr;
+	void *addr;
+	paddr_t pfn;
+	maddr_t mfn;
+	maddr_t new_mfn;
+	int err;
+
+	page = *pagep;
+	if (!get_page_unless_zero(page))
+		return -ENOENT;
+
+	err = -ENOMEM;
+	new_page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
+	if (!new_page)
+		goto out;
+
+	new_addr = page_address(new_page);
+	addr = page_address(page);
+	memcpy(new_addr, addr, PAGE_SIZE);
+
+	pfn = page_to_pfn(page);
+	mfn = pfn_to_mfn(pfn);
+	new_mfn = virt_to_mfn(new_addr);
+
+	if (!xen_feature(XENFEAT_auto_translated_physmap)) {
+		set_phys_to_machine(pfn, new_mfn);
+		set_phys_to_machine(page_to_pfn(new_page), INVALID_P2M_ENTRY);
+
+		mmu.ptr = (new_mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
+		mmu.val = pfn;
+		err = HYPERVISOR_mmu_update(&mmu, 1, NULL, DOMID_SELF);
+		BUG_ON(err);
+	}
+
+	gnttab_set_replace_op(&unmap, (unsigned long)addr,
+			      (unsigned long)new_addr, ref);
+
+	err = HYPERVISOR_grant_table_op(GNTTABOP_unmap_and_replace,
+					&unmap, 1);
+	BUG_ON(err);
+	BUG_ON(unmap.status);
+
+	new_page->mapping = page->mapping;
+	new_page->index = page->index;
+	set_bit(PG_foreign, &new_page->flags);
+	*pagep = new_page;
+
+	SetPageForeign(page, gnttab_page_free);
+	page->mapping = NULL;
+
+	/*
+	 * Ensure that there is a barrier between setting the p2m entry
+	 * and checking the map count.  See gnttab_dma_map_page.
+	 */
+	smp_mb();
+
+	/* Has the page been DMA-mapped? */
+	if (unlikely(page_mapped(page))) {
+		err = -EBUSY;
+		page->mapping = (void *)new_page;
+	}
+
+out:
+	put_page(page);
+	return err;
+
+}
+EXPORT_SYMBOL(gnttab_copy_grant_page);
+
+/*
+ * Keep track of foreign pages marked as PageForeign so that we don't
+ * return them to the remote domain prematurely.
+ *
+ * PageForeign pages are pinned down by increasing their mapcount.
+ *
+ * All other pages are simply returned as is.
+ */
+maddr_t gnttab_dma_map_page(struct page *page)
+{
+	maddr_t mfn = pfn_to_mfn(page_to_pfn(page)), mfn2;
+
+	if (!PageForeign(page))
+		return mfn << PAGE_SHIFT;
+
+	if (mfn_to_local_pfn(mfn) < max_mapnr)
+		return mfn << PAGE_SHIFT;
+
+	atomic_set(&page->_mapcount, 0);
+
+	/* This barrier corresponds to the one in gnttab_copy_grant_page. */
+	smp_mb();
+
+	/* Has this page been copied in the mean time? */
+	mfn2 = pfn_to_mfn(page_to_pfn(page));
+
+	return mfn2 << PAGE_SHIFT;
+}
+
 int gnttab_resume(void)
 {
 	if (max_nr_grant_frames() < nr_grant_frames)
diff --git a/linux-2.6-xen-sparse/include/xen/gnttab.h b/linux-2.6-xen-sparse/include/xen/gnttab.h
index 9ccd67fe02..c590004277 100644
--- a/linux-2.6-xen-sparse/include/xen/gnttab.h
+++ b/linux-2.6-xen-sparse/include/xen/gnttab.h
@@ -39,6 +39,7 @@
 
 #include <asm/hypervisor.h>
 #include <asm/maddr.h> /* maddr_t */
+#include <linux/mm.h>
 #include <xen/interface/grant_table.h>
 #include <xen/features.h>
 
@@ -101,6 +102,19 @@ void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
 void gnttab_grant_foreign_transfer_ref(grant_ref_t, domid_t domid,
 				       unsigned long pfn);
 
+int gnttab_copy_grant_page(grant_ref_t ref, struct page **pagep);
+maddr_t gnttab_dma_map_page(struct page *page);
+
+static inline void gnttab_dma_unmap_page(maddr_t mfn)
+{
+}
+
+static inline void gnttab_reset_grant_page(struct page *page)
+{
+	init_page_count(page);
+	reset_page_mapcount(page);
+}
+
 int gnttab_suspend(void);
 int gnttab_resume(void);