Add generic_page_range() -- generic page table operation.

Linux has several instances of repeated code to do updates to a range
of PTEs.  Mapping memory between domains in Xen also tends to need to
do this quite frequently, to ensure page tables have been constructed
and to look up PTE addresses when making mapping-related hypercalls.
This patch adds a generic PTE walk-and-fill operation that takes a
function pointer to call on leaf entries.  direct_remap_area_pages()
is updated to use the new call, ass are abuses of
__direct_remap_area_pages.

This patch also introduces two new helper functions for working with
page tables when mapping memory between domains:
create_lookup_pte_addr() returns the machine address of a PTE,
allocating intermediate page tables as necessary.  touch_pte_range()
ensures that page tables exist for a virtual address range.

Many of the existing linux page table operations (e.g. zap/remap/etc)
could be modified to use this interface, which would potentially
shorten up mm/memory.c a bit.

7 files changed, 205 insertions, 101 deletions

diff --git a/linux-2.6-xen-sparse/arch/xen/i386/mm/ioremap.c b/linux-2.6-xen-sparse/arch/xen/i386/mm/ioremap.c
index 86a3672e33..76f4df0115 100644
--- a/linux-2.6-xen-sparse/arch/xen/i386/mm/ioremap.c
+++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/ioremap.c
@@ -298,90 +298,20 @@ void __init bt_iounmap(void *addr, unsigned long size)
 #define direct_mk_pte_phys(physpage, pgprot) \
   __direct_mk_pte((physpage) >> PAGE_SHIFT, pgprot)
 
-static inline void direct_remap_area_pte(pte_t *pte, 
-					 unsigned long address, 
-					 unsigned long size,
-					 mmu_update_t **v)
-{
-	unsigned long end;
-
-	address &= ~PMD_MASK;
-	end = address + size;
-	if (end > PMD_SIZE)
-		end = PMD_SIZE;
-	if (address >= end)
-		BUG();
-
-	do {
-		(*v)->ptr = virt_to_machine(pte);
-		(*v)++;
-		address += PAGE_SIZE;
-		pte++;
-	} while (address && (address < end));
-}
 
-static inline int direct_remap_area_pmd(struct mm_struct *mm,
-					pmd_t *pmd, 
-					unsigned long address, 
-					unsigned long size,
-					mmu_update_t **v)
+static int direct_remap_area_pte_fn(pte_t *pte, 
+                                    struct page *pte_page,
+                                    unsigned long address, 
+                                    void *data)
 {
-	unsigned long end;
-
-	address &= ~PGDIR_MASK;
-	end = address + size;
-	if (end > PGDIR_SIZE)
-		end = PGDIR_SIZE;
-	if (address >= end)
-		BUG();
-	do {
-		pte_t *pte = (mm == &init_mm) ? 
-			pte_alloc_kernel(mm, pmd, address) :
-			pte_alloc_map(mm, pmd, address);
-		if (!pte)
-			return -ENOMEM;
-		direct_remap_area_pte(pte, address, end - address, v);
-		pte_unmap(pte);
-		address = (address + PMD_SIZE) & PMD_MASK;
-		pmd++;
-	} while (address && (address < end));
-	return 0;
-}
- 
-int __direct_remap_area_pages(struct mm_struct *mm,
-			      unsigned long address, 
-			      unsigned long size, 
-			      mmu_update_t *v)
-{
-	pgd_t * dir;
-	unsigned long end = address + size;
-	int error;
-
-	dir = pgd_offset(mm, address);
-	if (address >= end)
-		BUG();
-	spin_lock(&mm->page_table_lock);
-	do {
-		pud_t *pud;
-		pmd_t *pmd;
-
-		error = -ENOMEM;
-		pud = pud_alloc(mm, dir, address);
-		if (!pud)
-			break;
-		pmd = pmd_alloc(mm, pud, address);
-		if (!pmd)
-			break;
-		error = 0;
-		direct_remap_area_pmd(mm, pmd, address, end - address, &v);
-		address = (address + PGDIR_SIZE) & PGDIR_MASK;
-		dir++;
-
-	} while (address && (address < end));
-	spin_unlock(&mm->page_table_lock);
-	return error;
-}
+        mmu_update_t **v = (mmu_update_t **)data;
 
+        (*v)->ptr = (pfn_to_mfn(page_to_pfn(pte_page)) << PAGE_SHIFT)
+                    | ((unsigned long)pte & ~PAGE_MASK);
+        (*v)++;
+
+        return 0;
+}
 
 int direct_remap_area_pages(struct mm_struct *mm,
 			    unsigned long address, 
@@ -393,7 +323,7 @@ int direct_remap_area_pages(struct mm_struct *mm,
 	int i;
 	unsigned long start_address;
 #define MAX_DIRECTMAP_MMU_QUEUE 130
-	mmu_update_t u[MAX_DIRECTMAP_MMU_QUEUE], *v = u;
+	mmu_update_t u[MAX_DIRECTMAP_MMU_QUEUE], *v = u, *w = u;
 
 	start_address = address;
 
@@ -402,10 +332,9 @@ int direct_remap_area_pages(struct mm_struct *mm,
 	for (i = 0; i < size; i += PAGE_SIZE) {
 		if ((v - u) == MAX_DIRECTMAP_MMU_QUEUE) {
 			/* Fill in the PTE pointers. */
-			__direct_remap_area_pages(mm,
-						  start_address, 
-						  address-start_address, 
-						  u);
+                        generic_page_range(mm, start_address, 
+                                           address-start_address,
+                                           direct_remap_area_pte_fn, &w);
  
 			if (HYPERVISOR_mmu_update(u, v - u, NULL, domid) < 0)
 				return -EFAULT;
@@ -426,10 +355,9 @@ int direct_remap_area_pages(struct mm_struct *mm,
 
 	if (v != u) {
 		/* get the ptep's filled in */
-		__direct_remap_area_pages(mm,
-					  start_address, 
-					  address-start_address, 
-					  u);
+                generic_page_range(mm, start_address, 
+                                   address-start_address,
+                                   direct_remap_area_pte_fn, &w);
 		if (unlikely(HYPERVISOR_mmu_update(u, v - u, NULL, domid) < 0))
 			return -EFAULT;
 	}
@@ -440,3 +368,34 @@ int direct_remap_area_pages(struct mm_struct *mm,
 }
 
 EXPORT_SYMBOL(direct_remap_area_pages);
+
+int create_lookup_pte_addr(struct mm_struct *mm, 
+                           unsigned long address,
+                           unsigned long *ptep)
+{
+    int f(pte_t *pte, struct page *pte_page, unsigned long addr, void *data) 
+    {
+        unsigned long *ptep = (unsigned long *)data;
+        if (ptep) *ptep = (pfn_to_mfn(page_to_pfn(pte_page)) << PAGE_SHIFT)
+                       | ((unsigned long)pte & ~PAGE_MASK);
+        return 0;
+    }
+
+    return generic_page_range(mm, address, PAGE_SIZE, f, ptep);
+}
+
+EXPORT_SYMBOL(create_lookup_pte_addr);
+
+int touch_pte_range(struct mm_struct *mm,
+                    unsigned long address,
+                    unsigned long size)
+{
+    int f(pte_t *pte, struct page *pte_page, unsigned long addr, void *data) 
+    {
+        return 0;
+    }
+
+    return generic_page_range(mm, address, size, f, NULL);
+}                 
+
+EXPORT_SYMBOL(touch_pte_range);
diff --git a/linux-2.6-xen-sparse/arch/xen/x86_64/mm/ioremap.c b/linux-2.6-xen-sparse/arch/xen/x86_64/mm/ioremap.c
index 3714880f72..c3d6ee3959 100644
--- a/linux-2.6-xen-sparse/arch/xen/x86_64/mm/ioremap.c
+++ b/linux-2.6-xen-sparse/arch/xen/x86_64/mm/ioremap.c
@@ -464,3 +464,34 @@ int direct_remap_area_pages(struct mm_struct *mm,
 }
 
 EXPORT_SYMBOL(direct_remap_area_pages);
+
+int create_lookup_pte_addr(struct mm_struct *mm, 
+                           unsigned long address,
+                           unsigned long *ptep)
+{
+    int f(pte_t *pte, struct page *pte_page, unsigned long addr, void *data) 
+    {
+        unsigned long *ptep = (unsigned long *)data;
+        if (ptep) *ptep = (pfn_to_mfn(page_to_pfn(pte_page)) << PAGE_SHIFT)
+                       | ((unsigned long)pte & ~PAGE_MASK);
+        return 0;
+    }
+
+    return generic_page_range(mm, address, PAGE_SIZE, f, ptep);
+}
+
+EXPORT_SYMBOL(create_lookup_pte_addr);
+
+int touch_pte_range(struct mm_struct *mm,
+                    unsigned long address,
+                    unsigned long size)
+{
+    int f(pte_t *pte, struct page *pte_page, unsigned long addr, void *data) 
+    {
+        return 0;
+    }
+
+    return generic_page_range(mm, address, size, f, NULL);
+}                 
+
+EXPORT_SYMBOL(touch_pte_range);
diff --git a/linux-2.6-xen-sparse/drivers/xen/privcmd/privcmd.c b/linux-2.6-xen-sparse/drivers/xen/privcmd/privcmd.c
index e7f6ff5fcf..f72b168d14 100644
--- a/linux-2.6-xen-sparse/drivers/xen/privcmd/privcmd.c
+++ b/linux-2.6-xen-sparse/drivers/xen/privcmd/privcmd.c
@@ -139,7 +139,7 @@ static int privcmd_ioctl(struct inode *inode, struct file *file,
         privcmd_mmapbatch_t m;
         struct vm_area_struct *vma = NULL;
         unsigned long *p, addr;
-        unsigned long mfn;
+        unsigned long mfn, ptep;
         int i;
 
         if ( copy_from_user(&m, (void *)data, sizeof(m)) )
@@ -163,12 +163,12 @@ static int privcmd_ioctl(struct inode *inode, struct file *file,
             if ( get_user(mfn, p) )
                 return -EFAULT;
 
-            u.val = (mfn << PAGE_SHIFT) | pgprot_val(vma->vm_page_prot);
+            ret = create_lookup_pte_addr(vma->vm_mm, addr, &ptep);
+            if (ret)
+                goto batch_err;
 
-            __direct_remap_area_pages(vma->vm_mm,
-                                      addr, 
-                                      PAGE_SIZE, 
-                                      &u);
+            u.val = (mfn << PAGE_SHIFT) | pgprot_val(vma->vm_page_prot);
+            u.ptr = ptep;
 
             if ( unlikely(HYPERVISOR_mmu_update(&u, 1, NULL, m.dom) < 0) )
                 put_user(0xF0000000 | mfn, p);
diff --git a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/pgtable.h b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/pgtable.h
index d8066ea6f7..d326f7903a 100644
--- a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/pgtable.h
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/pgtable.h
@@ -466,10 +466,12 @@ int direct_remap_area_pages(struct mm_struct *mm,
                             unsigned long size, 
                             pgprot_t prot,
                             domid_t  domid);
-int __direct_remap_area_pages(struct mm_struct *mm,
-			      unsigned long address, 
-			      unsigned long size, 
-			      mmu_update_t *v);
+int create_lookup_pte_addr(struct mm_struct *mm,
+                           unsigned long address,
+                           unsigned long *ptep);
+int touch_pte_range(struct mm_struct *mm,
+                    unsigned long address,
+                    unsigned long size);
 
 #define io_remap_page_range(vma,from,phys,size,prot) \
 direct_remap_area_pages(vma->vm_mm,from,phys,size,prot,DOMID_IO)
diff --git a/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pgtable.h b/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pgtable.h
index 1884867a8d..20c856360e 100644
--- a/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pgtable.h
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pgtable.h
@@ -538,6 +538,12 @@ int __direct_remap_area_pages(struct mm_struct *mm,
                               unsigned long address,
                               unsigned long size,
                               mmu_update_t *v);
+int create_lookup_pte_addr(struct mm_struct *mm,
+                           unsigned long address,
+                           unsigned long *ptep);
+int touch_pte_range(struct mm_struct *mm,
+                    unsigned long address,
+                    unsigned long size);
 
 #define io_remap_page_range(vma, vaddr, paddr, size, prot)		\
 		direct_remap_area_pages((vma)->vm_mm,vaddr,paddr,size,prot,DOMID_IO)
diff --git a/linux-2.6-xen-sparse/include/linux/mm.h b/linux-2.6-xen-sparse/include/linux/mm.h
index 12763056cb..79b61e8b50 100644
--- a/linux-2.6-xen-sparse/include/linux/mm.h
+++ b/linux-2.6-xen-sparse/include/linux/mm.h
@@ -817,6 +817,12 @@ extern int check_user_page_readable(struct mm_struct *mm, unsigned long address)
 int remap_pfn_range(struct vm_area_struct *, unsigned long,
 		unsigned long, unsigned long, pgprot_t);
 
+typedef int (*pte_fn_t)(pte_t *pte, struct page *pte_page, unsigned long addr, 
+                        void *data);
+extern int generic_page_range(struct mm_struct *mm, unsigned long address, 
+                              unsigned long size, pte_fn_t fn, void *data);
+
+
 #ifdef CONFIG_PROC_FS
 void __vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
 #else
diff --git a/linux-2.6-xen-sparse/mm/memory.c b/linux-2.6-xen-sparse/mm/memory.c
index d62656eb8e..2726818854 100644
--- a/linux-2.6-xen-sparse/mm/memory.c
+++ b/linux-2.6-xen-sparse/mm/memory.c
@@ -954,8 +954,10 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                         i++;
                         start += PAGE_SIZE;
                         len--;
+printk(KERN_ALERT "HIT  0x%lx\n", start);
                         continue;
-                    }
+                    } 
+else printk(KERN_ALERT "MISS 0x%lx\n", start);
                 }
 
 		if (!vma || (vma->vm_flags & VM_IO)
@@ -1213,6 +1215,104 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
 }
 EXPORT_SYMBOL(remap_pfn_range);
 
+static inline int generic_pte_range(struct mm_struct *mm,
+                                    pmd_t *pmd, 
+                                    unsigned long addr, 
+                                    unsigned long end,
+                                    pte_fn_t fn, void *data)
+{
+	pte_t *pte;
+        int err;
+        struct page *pte_page;
+
+        pte = (mm == &init_mm) ? 
+                pte_alloc_kernel(mm, pmd, addr) :
+                pte_alloc_map(mm, pmd, addr);
+        if (!pte)
+                return -ENOMEM;
+
+        pte_page = pmd_page(*pmd);
+
+        do {
+                err = fn(pte, pte_page, addr, data);
+		if (err)
+                        break;
+        } while (pte++, addr += PAGE_SIZE, addr != end);
+
+        if (mm != &init_mm)
+                pte_unmap(pte-1);
+        return err;
+
+}
+
+static inline int generic_pmd_range(struct mm_struct *mm,
+                                    pud_t *pud, 
+                                    unsigned long addr, 
+                                    unsigned long end,
+                                    pte_fn_t fn, void *data)
+{
+	pmd_t *pmd;
+	unsigned long next;
+        int err;
+
+	pmd = pmd_alloc(mm, pud, addr);
+	if (!pmd)
+		return -ENOMEM;
+	do {
+		next = pmd_addr_end(addr, end);
+                err = generic_pte_range(mm, pmd, addr, next, fn, data);
+                if (err)
+                    break;
+	} while (pmd++, addr = next, addr != end);
+	return err;
+}
+
+static inline int generic_pud_range(struct mm_struct *mm, pgd_t *pgd, 
+                                    unsigned long addr,
+                                    unsigned long end,
+                                    pte_fn_t fn, void *data)
+{
+	pud_t *pud;
+	unsigned long next;
+        int err;
+
+	pud = pud_alloc(mm, pgd, addr);
+	if (!pud)
+		return -ENOMEM;
+	do {
+		next = pud_addr_end(addr, end);
+		err = generic_pmd_range(mm, pud, addr, next, fn, data);
+                if (err)
+			break;
+	} while (pud++, addr = next, addr != end);
+	return err;
+}
+
+/*
+ * Scan a region of virtual memory, filling in page tables as necessary
+ * and calling a provided function on each leaf page table.
+ */
+int generic_page_range(struct mm_struct *mm, unsigned long addr, 
+                  unsigned long size, pte_fn_t fn, void *data)
+{
+	pgd_t *pgd;
+	unsigned long next;
+	unsigned long end = addr + size;
+	int err;
+
+	BUG_ON(addr >= end);
+	pgd = pgd_offset(mm, addr);
+	spin_lock(&mm->page_table_lock);
+	do {
+		next = pgd_addr_end(addr, end);
+		err = generic_pud_range(mm, pgd, addr, next, fn, data);
+		if (err)
+			break;
+	} while (pgd++, addr = next, addr != end);
+	spin_unlock(&mm->page_table_lock);
+	return err;
+}
+
 /*
  * Do pte_mkwrite, but only if the vma says VM_WRITE.  We do this when
  * servicing faults for write access.  In the normal case, do always want