Add support for superpages (hugepages) in PV domain

This patch adds the option "superpages" to the domain configuration
file. If it is set, the domain is populated using 2M pages.

This code does not support fallback to small pages.  If the domain can
not be created with 2M pages, the  create will fail.

The patch also includes support for saving and restoring domains with
the superpage flag set.  However, if a domain has freed small pages
within its physical page array and then extended the array, the
restore will fill in those freed pages.  It will then attempt to
allocate more than its memory limit and will fail.  This is
significant because apparently Linux does this during boot, thus a
freshly booted Linux image can not be saved and restored successfully.

Signed-off-by: Dave McCracken <dcm@mccr.org>

15 files changed, 210 insertions, 152 deletions

diff --git a/tools/libxc/xc_dom.h b/tools/libxc/xc_dom.h
index ca03ba5db4..3d6d9164bd 100644
--- a/tools/libxc/xc_dom.h
+++ b/tools/libxc/xc_dom.h
@@ -174,7 +174,7 @@ int xc_dom_build_image(struct xc_dom_image *dom);
 int xc_dom_update_guest_p2m(struct xc_dom_image *dom);
 
 int xc_dom_boot_xen_init(struct xc_dom_image *dom, int xc, domid_t domid);
-int xc_dom_boot_mem_init(struct xc_dom_image *dom);
+int xc_dom_boot_mem_init(struct xc_dom_image *dom, int superpages);
 void *xc_dom_boot_domU_map(struct xc_dom_image *dom, xen_pfn_t pfn,
                            xen_pfn_t count);
 int xc_dom_boot_image(struct xc_dom_image *dom);
@@ -260,7 +260,7 @@ static inline xen_pfn_t xc_dom_p2m_guest(struct xc_dom_image *dom,
 
 /* --- arch bits --------------------------------------------------- */
 
-int arch_setup_meminit(struct xc_dom_image *dom);
+int arch_setup_meminit(struct xc_dom_image *dom, int superpages);
 int arch_setup_bootearly(struct xc_dom_image *dom);
 int arch_setup_bootlate(struct xc_dom_image *dom);
 
diff --git a/tools/libxc/xc_dom_boot.c b/tools/libxc/xc_dom_boot.c
index e767e8cee1..ac192a3dac 100644
--- a/tools/libxc/xc_dom_boot.c
+++ b/tools/libxc/xc_dom_boot.c
@@ -129,13 +129,13 @@ int xc_dom_boot_xen_init(struct xc_dom_image *dom, int xc, domid_t domid)
     return 0;
 }
 
-int xc_dom_boot_mem_init(struct xc_dom_image *dom)
+int xc_dom_boot_mem_init(struct xc_dom_image *dom, int superpages)
 {
     long rc;
 
     xc_dom_printf("%s: called\n", __FUNCTION__);
 
-    rc = arch_setup_meminit(dom);
+    rc = arch_setup_meminit(dom, superpages);
     if ( rc != 0 )
     {
         xc_dom_panic(XC_OUT_OF_MEMORY,
diff --git a/tools/libxc/xc_dom_compat_linux.c b/tools/libxc/xc_dom_compat_linux.c
index 92cdae4e29..41ee3af634 100644
--- a/tools/libxc/xc_dom_compat_linux.c
+++ b/tools/libxc/xc_dom_compat_linux.c
@@ -28,7 +28,8 @@ static int xc_linux_build_internal(struct xc_dom_image *dom,
                                    unsigned int store_evtchn,
                                    unsigned long *store_mfn,
                                    unsigned int console_evtchn,
-                                   unsigned long *console_mfn)
+                                   unsigned long *console_mfn,
+    							   int superpages)
 {
     int rc;
 
@@ -42,7 +43,7 @@ static int xc_linux_build_internal(struct xc_dom_image *dom,
         goto out;
     if ( (rc = xc_dom_mem_init(dom, mem_mb)) != 0 )
         goto out;
-    if ( (rc = xc_dom_boot_mem_init(dom)) != 0 )
+    if ( (rc = xc_dom_boot_mem_init(dom, superpages)) != 0 )
         goto out;
     if ( (rc = xc_dom_build_image(dom)) != 0 )
         goto out;
@@ -67,7 +68,8 @@ int xc_linux_build_mem(int xc_handle, uint32_t domid,
                        unsigned long flags,
                        unsigned int store_evtchn,
                        unsigned long *store_mfn,
-                       unsigned int console_evtchn, unsigned long *console_mfn)
+                       unsigned int console_evtchn, unsigned long *console_mfn,
+                       int superpages)
 {
     struct xc_dom_image *dom;
     int rc;
@@ -82,7 +84,7 @@ int xc_linux_build_mem(int xc_handle, uint32_t domid,
     rc = xc_linux_build_internal(dom, xc_handle, domid,
                                  mem_mb, flags,
                                  store_evtchn, store_mfn,
-                                 console_evtchn, console_mfn);
+                                 console_evtchn, console_mfn, superpages);
 
  out:
     xc_dom_release(dom);
@@ -98,7 +100,8 @@ int xc_linux_build(int xc_handle, uint32_t domid,
                    unsigned long flags,
                    unsigned int store_evtchn,
                    unsigned long *store_mfn,
-                   unsigned int console_evtchn, unsigned long *console_mfn)
+                   unsigned int console_evtchn, unsigned long *console_mfn,
+                   int superpages)
 {
     struct xc_dom_image *dom;
     int rc;
@@ -114,7 +117,7 @@ int xc_linux_build(int xc_handle, uint32_t domid,
     rc = xc_linux_build_internal(dom, xc_handle, domid,
                                  mem_mb, flags,
                                  store_evtchn, store_mfn,
-                                 console_evtchn, console_mfn);
+                                 console_evtchn, console_mfn, superpages);
 
  out:
     xc_dom_release(dom);
@@ -130,7 +133,8 @@ int xc_dom_linux_build(int xc_handle,
                        unsigned long flags,
                        unsigned int store_evtchn,
                        unsigned long *store_mfn,
-                       unsigned int console_evtchn, unsigned long *console_mfn)
+                       unsigned int console_evtchn, unsigned long *console_mfn,
+                       int superpages)
 {
     int rc;
 
@@ -143,7 +147,7 @@ int xc_dom_linux_build(int xc_handle,
     return xc_linux_build_internal(dom, xc_handle, domid,
                                    mem_mb, flags,
                                    store_evtchn, store_mfn,
-                                   console_evtchn, console_mfn);
+                                   console_evtchn, console_mfn, superpages);
 }
 
 /*
diff --git a/tools/libxc/xc_dom_ia64.c b/tools/libxc/xc_dom_ia64.c
index 7610aa1d73..76f845ab1c 100644
--- a/tools/libxc/xc_dom_ia64.c
+++ b/tools/libxc/xc_dom_ia64.c
@@ -149,7 +149,7 @@ static void __init register_arch_hooks(void)
 
 #include "xc_efi.h"
 
-int arch_setup_meminit(struct xc_dom_image *dom)
+int arch_setup_meminit(struct xc_dom_image *dom, int superpages)
 {
     xen_pfn_t pfn;
     int rc;
diff --git a/tools/libxc/xc_dom_x86.c b/tools/libxc/xc_dom_x86.c
index 6ae9487656..a89eb4fef7 100644
--- a/tools/libxc/xc_dom_x86.c
+++ b/tools/libxc/xc_dom_x86.c
@@ -26,6 +26,9 @@
 
 /* ------------------------------------------------------------------------ */
 
+#define SUPERPAGE_PFN_SHIFT  9
+#define SUPERPAGE_NR_PFNS    (1UL << SUPERPAGE_PFN_SHIFT)
+
 #define bits_to_mask(bits)       (((xen_vaddr_t)1 << (bits))-1)
 #define round_down(addr, mask)   ((addr) & ~(mask))
 #define round_up(addr, mask)     ((addr) | (mask))
@@ -691,7 +694,7 @@ static int x86_shadow(int xc, domid_t domid)
     return rc;
 }
 
-int arch_setup_meminit(struct xc_dom_image *dom)
+int arch_setup_meminit(struct xc_dom_image *dom, int superpages)
 {
     int rc;
     xen_pfn_t pfn, allocsz, i;
@@ -707,19 +710,49 @@ int arch_setup_meminit(struct xc_dom_image *dom)
             return rc;
     }
 
-    /* setup initial p2m */
     dom->p2m_host = xc_dom_malloc(dom, sizeof(xen_pfn_t) * dom->total_pages);
-    for ( pfn = 0; pfn < dom->total_pages; pfn++ )
-        dom->p2m_host[pfn] = pfn;
+    if (superpages)
+    {
+        int count = dom->total_pages >> SUPERPAGE_PFN_SHIFT;
+        xen_pfn_t extents[count];
+
+        xc_dom_printf("Populating memory with %d superpages\n", count);
+        for (pfn = 0; pfn < count; pfn++)
+            extents[pfn] = pfn << SUPERPAGE_PFN_SHIFT;
+        rc = xc_domain_memory_populate_physmap(dom->guest_xc, dom->guest_domid,
+                                               count, SUPERPAGE_PFN_SHIFT, 0,
+                                               extents);
+        if (!rc)
+        {
+            int i, j;
+            xen_pfn_t mfn;
 
-    /* allocate guest memory */
-    for ( i = rc = allocsz = 0; (i < dom->total_pages) && !rc; i += allocsz )
+            /* Expand the returned mfn into the p2m array */
+            pfn = 0;
+            for (i = 0; i < count; i++)
+            {
+                mfn = extents[i];
+                for (j = 0; j < SUPERPAGE_NR_PFNS; j++, pfn++)
+                {
+                    dom->p2m_host[pfn] = mfn + j;
+                }
+            }
+        }
+    } else
     {
-        allocsz = dom->total_pages - i;
-        if ( allocsz > 1024*1024 )
-            allocsz = 1024*1024;
-        rc = xc_domain_memory_populate_physmap(
-            dom->guest_xc, dom->guest_domid, allocsz, 0, 0, &dom->p2m_host[i]);
+        /* setup initial p2m */
+        for ( pfn = 0; pfn < dom->total_pages; pfn++ )
+            dom->p2m_host[pfn] = pfn;
+        
+        /* allocate guest memory */
+        for ( i = rc = allocsz = 0; (i < dom->total_pages) && !rc; i += allocsz )
+        {
+            allocsz = dom->total_pages - i;
+            if ( allocsz > 1024*1024 )
+                allocsz = 1024*1024;
+            rc = xc_domain_memory_populate_physmap(
+                dom->guest_xc, dom->guest_domid, allocsz, 0, 0, &dom->p2m_host[i]);
+        }
     }
 
     return rc;
diff --git a/tools/libxc/xc_domain_restore.c b/tools/libxc/xc_domain_restore.c
index 32782e1888..9987d35dfa 100644
--- a/tools/libxc/xc_domain_restore.c
+++ b/tools/libxc/xc_domain_restore.c
@@ -53,13 +53,94 @@ static xen_pfn_t *live_p2m = NULL;
 /* A table mapping each PFN to its new MFN. */
 static xen_pfn_t *p2m = NULL;
 
-/* A table of P2M mappings in the current region */
-static xen_pfn_t *p2m_batch = NULL;
-
 /* Address size of the guest, in bytes */
 unsigned int guest_width;
 
 /*
+**
+**
+*/
+#define SUPERPAGE_PFN_SHIFT  9
+#define SUPERPAGE_NR_PFNS    (1UL << SUPERPAGE_PFN_SHIFT)
+
+static int allocate_mfn(int xc_handle, uint32_t dom, unsigned long pfn, int superpages)
+{
+    unsigned long mfn;
+
+    if (superpages)
+    {
+        unsigned long base_pfn;
+
+        base_pfn = pfn & ~(SUPERPAGE_NR_PFNS-1);
+        mfn = base_pfn;
+
+        if (xc_domain_memory_populate_physmap(xc_handle, dom, 1,
+                                              SUPERPAGE_PFN_SHIFT, 0, &mfn) != 0)
+        {
+            ERROR("Failed to allocate physical memory at pfn 0x%x, base 0x%x.\n", pfn, base_pfn); 
+            errno = ENOMEM;
+            return 1;
+        }
+        for (pfn = base_pfn; pfn < base_pfn + SUPERPAGE_NR_PFNS; pfn++, mfn++)
+        {
+            p2m[pfn] = mfn;
+        }
+    }
+    else
+    {
+        mfn = pfn;
+        if (xc_domain_memory_populate_physmap(xc_handle, dom, 1, 0,
+                                              0, &mfn) != 0)
+        {
+            ERROR("Failed to allocate physical memory.!\n"); 
+            errno = ENOMEM;
+            return 1;
+        }
+        p2m[pfn] = mfn;
+    }
+    return 0;
+}
+
+static int allocate_physmem(int xc_handle, uint32_t dom,
+                            unsigned long *region_pfn_type, int region_size,
+                            unsigned int hvm, xen_pfn_t *region_mfn, int superpages)
+{
+	int i;
+    unsigned long pfn;
+    unsigned long pagetype;
+
+    for (i = 0; i < region_size; i++)
+    {
+        pfn      = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK;
+        pagetype = region_pfn_type[i] &  XEN_DOMCTL_PFINFO_LTAB_MASK;
+
+        if ( pfn > p2m_size )
+        {
+            ERROR("pfn out of range");
+            return 1;
+        }
+        if (pagetype == XEN_DOMCTL_PFINFO_XTAB)
+        {
+            region_mfn[i] = ~0UL;
+        }
+        else 
+        {
+            if (p2m[pfn] == INVALID_P2M_ENTRY)
+            {
+                if (allocate_mfn(xc_handle, dom, pfn, superpages) != 0)
+                    return 1;
+            }
+
+            /* setup region_mfn[] for batch map.
+             * For HVM guests, this interface takes PFNs, not MFNs */
+            region_mfn[i] = hvm ? pfn : p2m[pfn]; 
+        }
+    }
+    return 0;
+}
+
+
+/*
 ** In the state file (or during transfer), all page-table pages are
 ** converted into a 'canonical' form where references to actual mfns
 ** are replaced with references to the corresponding pfns.
@@ -67,74 +148,33 @@ unsigned int guest_width;
 ** the (now known) appropriate mfn values.
 */
 static int uncanonicalize_pagetable(int xc_handle, uint32_t dom, 
-                                    unsigned long type, void *page)
+                                    unsigned long type, void *page, int superpages)
 {
     int i, pte_last;
     unsigned long pfn;
     uint64_t pte;
-    int nr_mfns = 0; 
 
     pte_last = PAGE_SIZE / ((pt_levels == 2)? 4 : 8);
 
-    /* First pass: work out how many (if any) MFNs we need to alloc */
     for ( i = 0; i < pte_last; i++ )
     {
         if ( pt_levels == 2 )
             pte = ((uint32_t *)page)[i];
         else
             pte = ((uint64_t *)page)[i];
-
+        
         /* XXX SMH: below needs fixing for PROT_NONE etc */
         if ( !(pte & _PAGE_PRESENT) )
             continue;
         
         pfn = (pte >> PAGE_SHIFT) & MFN_MASK_X86;
-        
-        if ( pfn >= p2m_size )
-        {
-            /* This "page table page" is probably not one; bail. */
-            ERROR("Frame number in type %lu page table is out of range: "
-                  "i=%d pfn=0x%lx p2m_size=%lu",
-                  type >> 28, i, pfn, p2m_size);
-            return 0;
-        }
-        
+
+        /* Allocate mfn if necessary */
         if ( p2m[pfn] == INVALID_P2M_ENTRY )
         {
-            /* Have a 'valid' PFN without a matching MFN - need to alloc */
-            p2m_batch[nr_mfns++] = pfn; 
-            p2m[pfn]--;
+            if (allocate_mfn(xc_handle, dom, pfn, superpages) != 0)
+                return 0;
         }
-    }
-
-    /* Allocate the requisite number of mfns. */
-    if ( nr_mfns &&
-         (xc_domain_memory_populate_physmap(xc_handle, dom, nr_mfns, 0, 0,
-                                            p2m_batch) != 0) )
-    { 
-        ERROR("Failed to allocate memory for batch.!\n"); 
-        errno = ENOMEM;
-        return 0; 
-    }
-    
-    /* Second pass: uncanonicalize each present PTE */
-    nr_mfns = 0;
-    for ( i = 0; i < pte_last; i++ )
-    {
-        if ( pt_levels == 2 )
-            pte = ((uint32_t *)page)[i];
-        else
-            pte = ((uint64_t *)page)[i];
-        
-        /* XXX SMH: below needs fixing for PROT_NONE etc */
-        if ( !(pte & _PAGE_PRESENT) )
-            continue;
-        
-        pfn = (pte >> PAGE_SHIFT) & MFN_MASK_X86;
-
-        if ( p2m[pfn] == (INVALID_P2M_ENTRY-1) )
-            p2m[pfn] = p2m_batch[nr_mfns++];
-
         pte &= ~MADDR_MASK_X86;
         pte |= (uint64_t)p2m[pfn] << PAGE_SHIFT;
 
@@ -272,7 +312,7 @@ static xen_pfn_t *load_p2m_frame_list(
 int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom,
                       unsigned int store_evtchn, unsigned long *store_mfn,
                       unsigned int console_evtchn, unsigned long *console_mfn,
-                      unsigned int hvm, unsigned int pae)
+                      unsigned int hvm, unsigned int pae, int superpages)
 {
     DECLARE_DOMCTL;
     int rc = 1, frc, i, j, n, m, pae_extended_cr3 = 0, ext_vcpucontext = 0;
@@ -377,11 +417,9 @@ int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom,
 
     region_mfn = xg_memalign(PAGE_SIZE, ROUNDUP(
                               MAX_BATCH_SIZE * sizeof(xen_pfn_t), PAGE_SHIFT));
-    p2m_batch  = xg_memalign(PAGE_SIZE, ROUNDUP(
-                              MAX_BATCH_SIZE * sizeof(xen_pfn_t), PAGE_SHIFT));
 
     if ( (p2m == NULL) || (pfn_type == NULL) ||
-         (region_mfn == NULL) || (p2m_batch == NULL) )
+         (region_mfn == NULL) )
     {
         ERROR("memory alloc failed");
         errno = ENOMEM;
@@ -390,8 +428,6 @@ int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom,
 
     memset(region_mfn, 0,
            ROUNDUP(MAX_BATCH_SIZE * sizeof(xen_pfn_t), PAGE_SHIFT)); 
-    memset(p2m_batch, 0,
-           ROUNDUP(MAX_BATCH_SIZE * sizeof(xen_pfn_t), PAGE_SHIFT)); 
 
     if ( lock_pages(region_mfn, sizeof(xen_pfn_t) * MAX_BATCH_SIZE) )
     {
@@ -399,12 +435,6 @@ int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom,
         goto out;
     }
 
-    if ( lock_pages(p2m_batch, sizeof(xen_pfn_t) * MAX_BATCH_SIZE) )
-    {
-        ERROR("Could not lock p2m_batch");
-        goto out;
-    }
-
     /* Get the domain's shared-info frame. */
     domctl.cmd = XEN_DOMCTL_getdomaininfo;
     domctl.domain = (domid_t)dom;
@@ -437,7 +467,7 @@ int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom,
     n = m = 0;
     for ( ; ; )
     {
-        int j, nr_mfns = 0; 
+        int j; 
 
         this_pc = (n * 100) / p2m_size;
         if ( (this_pc - prev_pc) >= 5 )
@@ -521,57 +551,9 @@ int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom,
             goto out;
         }
 
-        /* First pass for this batch: work out how much memory to alloc */
-        nr_mfns = 0; 
-        for ( i = 0; i < j; i++ )
-        {
-            unsigned long pfn, pagetype;
-            pfn      = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK;
-            pagetype = region_pfn_type[i] &  XEN_DOMCTL_PFINFO_LTAB_MASK;
-
-            if ( (pagetype != XEN_DOMCTL_PFINFO_XTAB) && 
-                 (p2m[pfn] == INVALID_P2M_ENTRY) )
-            {
-                /* Have a live PFN which hasn't had an MFN allocated */
-                p2m_batch[nr_mfns++] = pfn; 
-                p2m[pfn]--;
-            }
-        } 
-
-        /* Now allocate a bunch of mfns for this batch */
-        if ( nr_mfns &&
-             (xc_domain_memory_populate_physmap(xc_handle, dom, nr_mfns, 0,
-                                                0, p2m_batch) != 0) )
-        { 
-            ERROR("Failed to allocate memory for batch.!\n"); 
-            errno = ENOMEM;
+        if (allocate_physmem(xc_handle, dom, region_pfn_type,
+                             j, hvm, region_mfn, superpages) != 0)
             goto out;
-        }
-
-        /* Second pass for this batch: update p2m[] and region_mfn[] */
-        nr_mfns = 0; 
-        for ( i = 0; i < j; i++ )
-        {
-            unsigned long pfn, pagetype;
-            pfn      = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK;
-            pagetype = region_pfn_type[i] &  XEN_DOMCTL_PFINFO_LTAB_MASK;
-
-            if ( pagetype == XEN_DOMCTL_PFINFO_XTAB )
-                region_mfn[i] = ~0UL; /* map will fail but we don't care */
-            else 
-            {
-                if ( p2m[pfn] == (INVALID_P2M_ENTRY-1) )
-                {
-                    /* We just allocated a new mfn above; update p2m */
-                    p2m[pfn] = p2m_batch[nr_mfns++]; 
-                    nr_pfns++; 
-                }
-
-                /* setup region_mfn[] for batch map.
-                 * For HVM guests, this interface takes PFNs, not MFNs */
-                region_mfn[i] = hvm ? pfn : p2m[pfn]; 
-            }
-        } 
 
         /* Map relevant mfns */
         region_base = xc_map_foreign_batch(
@@ -633,7 +615,7 @@ int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom,
                     (pagetype != XEN_DOMCTL_PFINFO_L1TAB)) {
 
                     if (!uncanonicalize_pagetable(xc_handle, dom, 
-                                                  pagetype, page)) {
+                                                  pagetype, page, superpages)) {
                         /*
                         ** Failing to uncanonicalize a page table can be ok
                         ** under live migration since the pages type may have
@@ -875,7 +857,7 @@ int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom,
                 {
                     if ( !uncanonicalize_pagetable(
                         xc_handle, dom, XEN_DOMCTL_PFINFO_L1TAB,
-                        region_base + k*PAGE_SIZE) )
+                        region_base + k*PAGE_SIZE, superpages) )
                     {
                         ERROR("failed uncanonicalize pt!");
                         goto out;
@@ -1223,3 +1205,12 @@ int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom,
     
     return rc;
 }
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libxc/xenguest.h b/tools/libxc/xenguest.h
index d64fc4554b..e8ebeb1ca2 100644
--- a/tools/libxc/xenguest.h
+++ b/tools/libxc/xenguest.h
@@ -40,12 +40,13 @@ int xc_domain_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
  * @parm store_mfn returned with the mfn of the store page
  * @parm hvm non-zero if this is a HVM restore
  * @parm pae non-zero if this HVM domain has PAE support enabled
+ * @parm superpages non-zero to allocate guest memory with superpages
  * @return 0 on success, -1 on failure
  */
 int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom,
                       unsigned int store_evtchn, unsigned long *store_mfn,
                       unsigned int console_evtchn, unsigned long *console_mfn,
-                      unsigned int hvm, unsigned int pae);
+                      unsigned int hvm, unsigned int pae, int superpages);
 
 /**
  * This function will create a domain for a paravirtualized Linux
@@ -62,6 +63,7 @@ int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom,
  * @parm store_mfn returned with the mfn of the store page
  * @parm console_evtchn the console event channel for this domain to use
  * @parm conole_mfn returned with the mfn of the console page
+ * @parm superpages populate memory in guest with superpages
  * @return 0 on success, -1 on failure
  */
 int xc_linux_build(int xc_handle,
@@ -75,7 +77,8 @@ int xc_linux_build(int xc_handle,
                    unsigned int store_evtchn,
                    unsigned long *store_mfn,
                    unsigned int console_evtchn,
-                   unsigned long *console_mfn);
+                   unsigned long *console_mfn,
+	           int superpages);
 
 /** The same interface, but the dom structure is managed by the caller */
 struct xc_dom_image;
@@ -89,7 +92,8 @@ int xc_dom_linux_build(int xc_handle,
 		       unsigned int store_evtchn,
 		       unsigned long *store_mfn,
 		       unsigned int console_evtchn,
-		       unsigned long *console_mfn);
+		       unsigned long *console_mfn,
+                       int superpages);
 
 /**
  * This function will create a domain for a paravirtualized Linux
@@ -108,6 +112,7 @@ int xc_dom_linux_build(int xc_handle,
  * @parm store_mfn returned with the mfn of the store page
  * @parm console_evtchn the console event channel for this domain to use
  * @parm conole_mfn returned with the mfn of the console page
+ * @parm superpages populate memory in guest with superpages
  * @return 0 on success, -1 on failure
  */
 int xc_linux_build_mem(int xc_handle,
@@ -123,7 +128,8 @@ int xc_linux_build_mem(int xc_handle,
                        unsigned int store_evtchn,
                        unsigned long *store_mfn,
                        unsigned int console_evtchn,
-                       unsigned long *console_mfn);
+                       unsigned long *console_mfn,
+                       int superpages);
 
 int xc_hvm_build(int xc_handle,
                  uint32_t domid,
diff --git a/tools/python/xen/lowlevel/xc/xc.c b/tools/python/xen/lowlevel/xc/xc.c
index 2c5096fe69..75a19d3d6b 100644
--- a/tools/python/xen/lowlevel/xc/xc.c
+++ b/tools/python/xen/lowlevel/xc/xc.c
@@ -405,6 +405,7 @@ static PyObject *pyxc_linux_build(XcObject *self,
     int flags = 0;
     int store_evtchn, console_evtchn;
     int vhpt = 0;
+    int superpages = 0;
     unsigned int mem_mb;
     unsigned long store_mfn = 0;
     unsigned long console_mfn = 0;
@@ -417,14 +418,14 @@ static PyObject *pyxc_linux_build(XcObject *self,
                                 "console_evtchn", "image",
                                 /* optional */
                                 "ramdisk", "cmdline", "flags",
-                                "features", "vhpt", NULL };
+                                "features", "vhpt", "superpages", NULL };
 
-    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "iiiis|ssisi", kwd_list,
+    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "iiiis|ssisii", kwd_list,
                                       &domid, &store_evtchn, &mem_mb,
                                       &console_evtchn, &image,
                                       /* optional */
                                       &ramdisk, &cmdline, &flags,
-                                      &features, &vhpt) )
+                                      &features, &vhpt, &superpages) )
         return NULL;
 
     xc_dom_loginit();
@@ -436,7 +437,7 @@ static PyObject *pyxc_linux_build(XcObject *self,
 
     if ( xc_dom_linux_build(self->xc_handle, dom, domid, mem_mb, image,
                             ramdisk, flags, store_evtchn, &store_mfn,
-                            console_evtchn, &console_mfn) != 0 ) {
+                            console_evtchn, &console_mfn, superpages) != 0 ) {
         goto out;
     }
 
diff --git a/tools/python/xen/xend/XendCheckpoint.py b/tools/python/xen/xend/XendCheckpoint.py
index a0ea011665..553cc0f5bc 100644
--- a/tools/python/xen/xend/XendCheckpoint.py
+++ b/tools/python/xen/xend/XendCheckpoint.py
@@ -272,9 +272,11 @@ def restore(xd, fd, dominfo = None, paused = False, relocating = False):
         shadow_cur = xc.shadow_mem_control(dominfo.getDomid(), shadow / 1024)
         dominfo.info['shadow_memory'] = shadow_cur
 
+        superpages = restore_image.superpages
+
         cmd = map(str, [xen.util.auxbin.pathTo(XC_RESTORE),
                         fd, dominfo.getDomid(),
-                        store_port, console_port, int(is_hvm), pae, apic])
+                        store_port, console_port, int(is_hvm), pae, apic, superpages])
         log.debug("[xc_restore]: %s", string.join(cmd))
 
         handler = RestoreInputHandler()
diff --git a/tools/python/xen/xend/XendConfig.py b/tools/python/xen/xend/XendConfig.py
index 02b7d91955..fcbd8b9293 100644
--- a/tools/python/xen/xend/XendConfig.py
+++ b/tools/python/xen/xend/XendConfig.py
@@ -220,6 +220,7 @@ XENAPI_CFG_TYPES = {
     'machine_address_size': int,
     'suppress_spurious_page_faults': bool0,
     's3_integrity' : int,
+    'superpages' : int,
 }
 
 # List of legacy configuration keys that have no equivalent in the
@@ -392,6 +393,7 @@ class XendConfig(dict):
             'other_config': {},
             'platform': {},
             'target': 0,
+            'superpages': 0,
         }
         
         return defaults
@@ -2014,6 +2016,8 @@ class XendConfig(dict):
             image.append(['ramdisk', self['PV_ramdisk']])
         if self.has_key('PV_args') and self['PV_args']:
             image.append(['args', self['PV_args']])
+        if self.has_key('superpages'):
+            image.append(['superpages', self['superpages']])
 
         for key in XENAPI_PLATFORM_CFG_TYPES.keys():
             if key in self['platform']:
@@ -2051,6 +2055,8 @@ class XendConfig(dict):
             self['PV_ramdisk'] = sxp.child_value(image_sxp, 'ramdisk','')
             self['PV_args'] = kernel_args
 
+        self['superpages'] = sxp.child_value(image_sxp, 'superpages',0)
+
         for key in XENAPI_PLATFORM_CFG_TYPES.keys():
             val = sxp.child_value(image_sxp, key, None)
             if val is not None and val != '':
diff --git a/tools/python/xen/xend/image.py b/tools/python/xen/xend/image.py
index 411936d5d6..ccd0858af4 100644
--- a/tools/python/xen/xend/image.py
+++ b/tools/python/xen/xend/image.py
@@ -651,6 +651,7 @@ class LinuxImageHandler(ImageHandler):
         ImageHandler.configure(self, vmConfig)
         self.vramsize = int(vmConfig['platform'].get('videoram',4)) * 1024
         self.is_stubdom = (self.kernel.find('stubdom') >= 0)
+        self.superpages = vmConfig['superpages']
 
     def buildDomain(self):
         store_evtchn = self.vm.getStorePort()
@@ -668,6 +669,7 @@ class LinuxImageHandler(ImageHandler):
         log.debug("vcpus          = %d", self.vm.getVCpuCount())
         log.debug("features       = %s", self.vm.getFeatures())
         log.debug("flags          = %d", self.flags)
+        log.debug("superpages     = %d", self.superpages)
         if arch.type == "ia64":
             log.debug("vhpt          = %d", self.vhpt)
 
@@ -680,7 +682,8 @@ class LinuxImageHandler(ImageHandler):
                               ramdisk        = self.ramdisk,
                               features       = self.vm.getFeatures(),
                               flags          = self.flags,
-                              vhpt           = self.vhpt)
+                              vhpt           = self.vhpt,
+                              superpages     = self.superpages)
 
     def getRequiredAvailableMemory(self, mem_kb):
         if self.is_stubdom :
diff --git a/tools/python/xen/xm/create.dtd b/tools/python/xen/xm/create.dtd
index 693d27e6ef..06e7176b07 100644
--- a/tools/python/xen/xm/create.dtd
+++ b/tools/python/xen/xm/create.dtd
@@ -54,6 +54,7 @@
                  actions_after_reboot   %NORMAL_EXIT; #REQUIRED
                  actions_after_crash    %CRASH_BEHAVIOUR; #REQUIRED
                  PCI_bus                CDATA #REQUIRED
+                 superpages             CDATA #REQUIRED
                  security_label         CDATA #IMPLIED>
 
 <!ELEMENT memory EMPTY> 
diff --git a/tools/python/xen/xm/create.py b/tools/python/xen/xm/create.py
index 509915c830..abe499870f 100644
--- a/tools/python/xen/xm/create.py
+++ b/tools/python/xen/xm/create.py
@@ -660,6 +660,8 @@ def configure_image(vals):
         config_image.append(['videoram', vals.videoram])
     if vals.extra:
         config_image.append(['args', vals.extra])
+    if vals.superpages:
+        config_image.append(['superpages', vals.superpages])
 
     if vals.builder == 'hvm':
         configure_hvm(config_image, vals) 
diff --git a/tools/python/xen/xm/xenapi_create.py b/tools/python/xen/xm/xenapi_create.py
index 0deced5063..2ae56de9c7 100644
--- a/tools/python/xen/xm/xenapi_create.py
+++ b/tools/python/xen/xm/xenapi_create.py
@@ -271,6 +271,8 @@ class xenapi_create:
                 vm.attributes["auto_power_on"].value == 'true',
             "s3_integrity":
                 vm.attributes["s3_integrity"].value,
+            "superpages":
+                vm.attributes["superpages"].value,
             "memory_static_max":
                 get_child_node_attribute(vm, "memory", "static_max"),
             "memory_static_min":
@@ -654,6 +656,8 @@ class sxp2xml:
             = str(get_child_by_name(config, "vcpus", 1))
         vm.attributes["s3_integrity"] \
             = str(get_child_by_name(config, "s3_integrity", 0))
+        vm.attributes["superpages"] \
+            = str(get_child_by_name(config, "superpages", 0))
 
         sec_data = get_child_by_name(config, "security")
         if sec_data:
diff --git a/tools/xcutils/xc_restore.c b/tools/xcutils/xc_restore.c
index 34fa26edd3..da4aa98b78 100644
--- a/tools/xcutils/xc_restore.c
+++ b/tools/xcutils/xc_restore.c
@@ -21,11 +21,12 @@ main(int argc, char **argv)
     unsigned int domid, store_evtchn, console_evtchn;
     unsigned int hvm, pae, apic;
     int xc_fd, io_fd, ret;
+    int superpages;
     unsigned long store_mfn, console_mfn;
 
-    if ( argc != 8 )
+    if ( (argc != 8) && (argc != 9) )
         errx(1, "usage: %s iofd domid store_evtchn "
-             "console_evtchn hvm pae apic", argv[0]);
+             "console_evtchn hvm pae apic [superpages]", argv[0]);
 
     xc_fd = xc_interface_open();
     if ( xc_fd < 0 )
@@ -38,9 +39,13 @@ main(int argc, char **argv)
     hvm  = atoi(argv[5]);
     pae  = atoi(argv[6]);
     apic = atoi(argv[7]);
+    if ( argc == 9 )
+	    superpages = atoi(argv[8]);
+    else
+	    superpages = 0;
 
     ret = xc_domain_restore(xc_fd, io_fd, domid, store_evtchn, &store_mfn,
-                            console_evtchn, &console_mfn, hvm, pae);
+                            console_evtchn, &console_mfn, hvm, pae, superpages);
 
     if ( ret == 0 )
     {