Some save/restore cleanups.

Signed-off-by: Steven Hand <steven@xensource.com>

11 files changed, 132 insertions, 112 deletions

diff --git a/tools/libxc/xc_core.c b/tools/libxc/xc_core.c
index 3c1ff185a6..d755aa8238 100644
--- a/tools/libxc/xc_core.c
+++ b/tools/libxc/xc_core.c
@@ -312,7 +312,7 @@ xc_domain_dumpcore_via_callback(int xc_handle,
 
     int auto_translated_physmap;
     xen_pfn_t *p2m = NULL;
-    unsigned long max_pfn = 0;
+    unsigned long p2m_size = 0;
     struct xen_dumpcore_p2m *p2m_array = NULL;
 
     uint64_t *pfn_array = NULL;
@@ -396,7 +396,7 @@ xc_domain_dumpcore_via_callback(int xc_handle,
         }
 
         sts = xc_core_arch_map_p2m(xc_handle, &info, live_shinfo,
-                                   &p2m, &max_pfn);
+                                   &p2m, &p2m_size);
         if ( sts != 0 )
             goto out;
     }
diff --git a/tools/libxc/xc_core_x86.c b/tools/libxc/xc_core_x86.c
index bd82110713..0f2a19c451 100644
--- a/tools/libxc/xc_core_x86.c
+++ b/tools/libxc/xc_core_x86.c
@@ -38,7 +38,7 @@ xc_core_arch_memory_map_get(int xc_handle, xc_dominfo_t *info,
                             xc_core_memory_map_t **mapp,
                             unsigned int *nr_entries)
 {
-    unsigned long max_pfn = max_gpfn(xc_handle, info->domid);
+    unsigned long p2m_size = max_gpfn(xc_handle, info->domid);
     xc_core_memory_map_t *map;
 
     map = malloc(sizeof(*map));
@@ -49,7 +49,7 @@ xc_core_arch_memory_map_get(int xc_handle, xc_dominfo_t *info,
     }
 
     map->addr = 0;
-    map->size = max_pfn << PAGE_SHIFT;
+    map->size = p2m_size << PAGE_SHIFT;
 
     *mapp = map;
     *nr_entries = 1;
@@ -65,13 +65,13 @@ xc_core_arch_map_p2m(int xc_handle, xc_dominfo_t *info,
     xen_pfn_t *live_p2m_frame_list_list = NULL;
     xen_pfn_t *live_p2m_frame_list = NULL;
     uint32_t dom = info->domid;
-    unsigned long max_pfn = max_gpfn(xc_handle, info->domid);
+    unsigned long p2m_size = max_gpfn(xc_handle, info->domid);
     int ret = -1;
     int err;
 
-    if ( max_pfn < info->nr_pages  )
+    if ( p2m_size < info->nr_pages  )
     {
-        ERROR("max_pfn < nr_pages -1 (%lx < %lx", max_pfn, info->nr_pages - 1);
+        ERROR("p2m_size < nr_pages -1 (%lx < %lx", p2m_size, info->nr_pages - 1);
         goto out;
     }
 
@@ -106,7 +106,7 @@ xc_core_arch_map_p2m(int xc_handle, xc_dominfo_t *info,
         goto out;
     }
 
-    *pfnp = max_pfn;
+    *pfnp = p2m_size;
 
     ret = 0;
 
diff --git a/tools/libxc/xc_hvm_restore.c b/tools/libxc/xc_hvm_restore.c
index 32fa627005..d5e9bf4759 100644
--- a/tools/libxc/xc_hvm_restore.c
+++ b/tools/libxc/xc_hvm_restore.c
@@ -95,7 +95,7 @@ int xc_hvm_restore(int xc_handle, int io_fd,
     unsigned long pfn_array_size = max_pfn + 1;
 
     /* Number of pages of memory the guest has.  *Not* the same as max_pfn. */
-    unsigned long nr_pages = max_pfn + 1;
+    unsigned long nr_pages = max_pfn;
     /* MMIO hole doesn't contain RAM */
     if ( nr_pages >= HVM_BELOW_4G_MMIO_START >> PAGE_SHIFT ) 
         nr_pages -= HVM_BELOW_4G_MMIO_LENGTH >> PAGE_SHIFT; 
@@ -270,7 +270,6 @@ int xc_hvm_restore(int xc_handle, int io_fd,
 
     }/*while 1*/
     
-/*    xc_set_hvm_param(xc_handle, dom, HVM_PARAM_APIC_ENABLED, apic);*/
     xc_set_hvm_param(xc_handle, dom, HVM_PARAM_PAE_ENABLED, pae);
     xc_set_hvm_param(xc_handle, dom, HVM_PARAM_STORE_EVTCHN, store_evtchn);
 
@@ -279,13 +278,22 @@ int xc_hvm_restore(int xc_handle, int io_fd,
     else
         shared_page_nr = (v_end >> PAGE_SHIFT) - 1;
 
+    /* Ensure we clear these pages */
+    if ( xc_clear_domain_page(xc_handle, dom, shared_page_nr) ||
+         xc_clear_domain_page(xc_handle, dom, shared_page_nr-1) ||
+         xc_clear_domain_page(xc_handle, dom, shared_page_nr-2) ) {
+        rc = -1;
+        goto out;
+    }
+
     xc_set_hvm_param(xc_handle, dom, HVM_PARAM_STORE_PFN, shared_page_nr-1);
     xc_set_hvm_param(xc_handle, dom, HVM_PARAM_BUFIOREQ_PFN, shared_page_nr-2);
     xc_set_hvm_param(xc_handle, dom, HVM_PARAM_IOREQ_PFN, shared_page_nr);
 
     /* caculate the store_mfn , wrong val cause hang when introduceDomain */
     *store_mfn = (v_end >> PAGE_SHIFT) - 2;
-    DPRINTF("hvm restore:calculate new store_mfn=0x%lx,v_end=0x%llx..\n", *store_mfn, v_end);
+    DPRINTF("hvm restore: calculate new store_mfn=0x%lx, v_end=0x%llx.\n", 
+            *store_mfn, v_end);
 
     if (!read_exact(io_fd, &nr_vcpus, sizeof(uint32_t))) {
         ERROR("error read nr vcpu !\n");
diff --git a/tools/libxc/xc_hvm_save.c b/tools/libxc/xc_hvm_save.c
index 75a017cf7c..6c05f618c2 100644
--- a/tools/libxc/xc_hvm_save.c
+++ b/tools/libxc/xc_hvm_save.c
@@ -332,10 +332,10 @@ int xc_hvm_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
 
     unsigned long total_sent    = 0;
 
-    DPRINTF("xc_hvm_save:dom=%d, max_iters=%d, max_factor=%d, flags=0x%x, live=%d, debug=%d.\n",
-            dom, max_iters, max_factor, flags,
+    DPRINTF("xc_hvm_save: dom=%d, max_iters=%d, max_factor=%d, flags=0x%x, "
+            "live=%d, debug=%d.\n", dom, max_iters, max_factor, flags,
             live, debug);
-
+    
     /* If no explicit control parameters given, use defaults */
     if(!max_iters)
         max_iters = DEF_MAX_ITERS;
@@ -382,7 +382,6 @@ int xc_hvm_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
         ERROR("HVM: Could not read magic PFN parameters");
         goto out;
     }
-
     DPRINTF("saved hvm domain info:max_memkb=0x%lx, max_mfn=0x%lx, "
             "nr_pages=0x%lx\n", info.max_memkb, max_mfn, info.nr_pages); 
 
diff --git a/tools/libxc/xc_linux_restore.c b/tools/libxc/xc_linux_restore.c
index d3a87262a0..70f6979ba3 100644
--- a/tools/libxc/xc_linux_restore.c
+++ b/tools/libxc/xc_linux_restore.c
@@ -22,8 +22,14 @@ static unsigned long hvirt_start;
 /* #levels of page tables used by the current guest */
 static unsigned int pt_levels;
 
-/* total number of pages used by the current guest */
-static unsigned long max_pfn;
+/* number of pfns this guest has (i.e. number of entries in the P2M) */
+static unsigned long p2m_size;
+
+/* number of 'in use' pfns in the guest (i.e. #P2M entries with a valid mfn) */
+static unsigned long nr_pfns;
+
+/* largest possible value of nr_pfns (i.e. domain's maximum memory size) */
+static unsigned long max_nr_pfns;
 
 /* Live mapping of the table mapping each PFN to its current MFN. */
 static xen_pfn_t *live_p2m = NULL;
@@ -34,7 +40,6 @@ static xen_pfn_t *p2m = NULL;
 /* A table of P2M mappings in the current region */
 static xen_pfn_t *p2m_batch = NULL;
 
-
 static ssize_t
 read_exact(int fd, void *buf, size_t count)
 {
@@ -85,11 +90,11 @@ static int uncanonicalize_pagetable(int xc_handle, uint32_t dom,
         
         pfn = (pte >> PAGE_SHIFT) & MFN_MASK_X86;
         
-        if(pfn >= max_pfn) {
+        if(pfn >= p2m_size) {
             /* This "page table page" is probably not one; bail. */
             ERROR("Frame number in type %lu page table is out of range: "
-                  "i=%d pfn=0x%lx max_pfn=%lu",
-                  type >> 28, i, pfn, max_pfn);
+                  "i=%d pfn=0x%lx p2m_size=%lu",
+                  type >> 28, i, pfn, p2m_size);
             return 0;
         }
         
@@ -138,8 +143,9 @@ static int uncanonicalize_pagetable(int xc_handle, uint32_t dom,
     return 1;
 }
 
-int xc_linux_restore(int xc_handle, int io_fd,
-                     uint32_t dom, unsigned long nr_pfns,
+
+int xc_linux_restore(int xc_handle, int io_fd, uint32_t dom,
+                     unsigned long p2msize, unsigned long maxnrpfns,
                      unsigned int store_evtchn, unsigned long *store_mfn,
                      unsigned int console_evtchn, unsigned long *console_mfn)
 {
@@ -191,9 +197,13 @@ int xc_linux_restore(int xc_handle, int io_fd,
     unsigned int max_vcpu_id = 0;
     int new_ctxt_format = 0;
 
-    max_pfn = nr_pfns;
+    p2m_size    = p2msize;
+    max_nr_pfns = maxnrpfns;
+
+    /* For info only */
+    nr_pfns = 0;
 
-    DPRINTF("xc_linux_restore start: max_pfn = %lx\n", max_pfn);
+    DPRINTF("xc_linux_restore start: p2m_size = %lx\n", p2m_size);
 
     /*
      * XXX For now, 32bit dom0's can only save/restore 32bit domUs
@@ -294,8 +304,8 @@ int xc_linux_restore(int xc_handle, int io_fd,
     }
 
     /* We want zeroed memory so use calloc rather than malloc. */
-    p2m        = calloc(max_pfn, sizeof(xen_pfn_t));
-    pfn_type   = calloc(max_pfn, sizeof(unsigned long));
+    p2m        = calloc(p2m_size, sizeof(xen_pfn_t));
+    pfn_type   = calloc(p2m_size, sizeof(unsigned long));
     region_mfn = calloc(MAX_BATCH_SIZE, sizeof(xen_pfn_t));
     p2m_batch  = calloc(MAX_BATCH_SIZE, sizeof(xen_pfn_t));
 
@@ -325,13 +335,13 @@ int xc_linux_restore(int xc_handle, int io_fd,
     }
     shared_info_frame = domctl.u.getdomaininfo.shared_info_frame;
 
-    if (xc_domain_setmaxmem(xc_handle, dom, PFN_TO_KB(max_pfn)) != 0) {
+    if (xc_domain_setmaxmem(xc_handle, dom, PFN_TO_KB(max_nr_pfns)) != 0) {
         errno = ENOMEM;
         goto out;
     }
 
     /* Mark all PFNs as invalid; we allocate on demand */
-    for ( pfn = 0; pfn < max_pfn; pfn++ )
+    for ( pfn = 0; pfn < p2m_size; pfn++ )
         p2m[pfn] = INVALID_P2M_ENTRY;
 
     if(!(mmu = xc_init_mmu_updates(xc_handle, dom))) {
@@ -352,7 +362,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
 
         int j, nr_mfns = 0; 
 
-        this_pc = (n * 100) / max_pfn;
+        this_pc = (n * 100) / p2m_size;
         if ( (this_pc - prev_pc) >= 5 )
         {
             PPRINTF("\b\b\b\b%3d%%", this_pc);
@@ -436,6 +446,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
                 if (p2m[pfn] == INVALID_P2M_ENTRY) {
                     /* We just allocated a new mfn above; update p2m */
                     p2m[pfn] = p2m_batch[nr_mfns++]; 
+                    nr_pfns++; 
                 }
 
                 /* setup region_mfn[] for batch map */
@@ -465,7 +476,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
                 /* a bogus/unmapped page: skip it */
                 continue;
 
-            if ( pfn > max_pfn )
+            if ( pfn > p2m_size )
             {
                 ERROR("pfn out of range");
                 goto out;
@@ -518,7 +529,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
             else if ( pagetype != XEN_DOMCTL_PFINFO_NOTAB )
             {
                 ERROR("Bogus page type %lx page table is out of range: "
-                    "i=%d max_pfn=%lu", pagetype, i, max_pfn);
+                    "i=%d p2m_size=%lu", pagetype, i, p2m_size);
                 goto out;
 
             }
@@ -598,7 +609,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
         int j, k;
         
         /* First pass: find all L3TABs current in > 4G mfns and get new mfns */
-        for ( i = 0; i < max_pfn; i++ )
+        for ( i = 0; i < p2m_size; i++ )
         {
             if ( ((pfn_type[i] & XEN_DOMCTL_PFINFO_LTABTYPE_MASK) ==
                   XEN_DOMCTL_PFINFO_L3TAB) &&
@@ -646,7 +657,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
         /* Second pass: find all L1TABs and uncanonicalize them */
         j = 0;
 
-        for ( i = 0; i < max_pfn; i++ )
+        for ( i = 0; i < p2m_size; i++ )
         {
             if ( ((pfn_type[i] & XEN_DOMCTL_PFINFO_LTABTYPE_MASK) ==
                   XEN_DOMCTL_PFINFO_L1TAB) )
@@ -655,7 +666,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
                 j++;
             }
 
-            if(i == (max_pfn-1) || j == MAX_BATCH_SIZE) {
+            if(i == (p2m_size-1) || j == MAX_BATCH_SIZE) {
 
                 if (!(region_base = xc_map_foreign_batch(
                           xc_handle, dom, PROT_READ | PROT_WRITE,
@@ -689,7 +700,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
      * will barf when doing the type-checking.
      */
     nr_pins = 0;
-    for ( i = 0; i < max_pfn; i++ )
+    for ( i = 0; i < p2m_size; i++ )
     {
         if ( (pfn_type[i] & XEN_DOMCTL_PFINFO_LPINTAB) == 0 )
             continue;
@@ -736,7 +747,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
     }
 
     DPRINTF("\b\b\b\b100%%\n");
-    DPRINTF("Memory reloaded.\n");
+    DPRINTF("Memory reloaded (%ld pages of max %ld)\n", nr_pfns, max_nr_pfns);
 
     /* Get the list of PFNs that are not in the psuedo-phys map */
     {
@@ -808,7 +819,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
              * resume record.
              */
             pfn = ctxt.user_regs.edx;
-            if ((pfn >= max_pfn) ||
+            if ((pfn >= p2m_size) ||
                 (pfn_type[pfn] != XEN_DOMCTL_PFINFO_NOTAB)) {
                 ERROR("Suspend record frame number is bad");
                 goto out;
@@ -816,7 +827,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
             ctxt.user_regs.edx = mfn = p2m[pfn];
             start_info = xc_map_foreign_range(
                 xc_handle, dom, PAGE_SIZE, PROT_READ | PROT_WRITE, mfn);
-            start_info->nr_pages = max_pfn;
+            start_info->nr_pages = p2m_size;
             start_info->shared_info = shared_info_frame << PAGE_SHIFT;
             start_info->flags = 0;
             *store_mfn = start_info->store_mfn = p2m[start_info->store_mfn];
@@ -835,7 +846,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
 
         for (j = 0; (512*j) < ctxt.gdt_ents; j++) {
             pfn = ctxt.gdt_frames[j];
-            if ((pfn >= max_pfn) ||
+            if ((pfn >= p2m_size) ||
                 (pfn_type[pfn] != XEN_DOMCTL_PFINFO_NOTAB)) {
                 ERROR("GDT frame number is bad");
                 goto out;
@@ -846,16 +857,16 @@ int xc_linux_restore(int xc_handle, int io_fd,
         /* Uncanonicalise the page table base pointer. */
         pfn = xen_cr3_to_pfn(ctxt.ctrlreg[3]);
 
-        if (pfn >= max_pfn) {
-            ERROR("PT base is bad: pfn=%lu max_pfn=%lu type=%08lx",
-                  pfn, max_pfn, pfn_type[pfn]);
+        if (pfn >= p2m_size) {
+            ERROR("PT base is bad: pfn=%lu p2m_size=%lu type=%08lx",
+                  pfn, p2m_size, pfn_type[pfn]);
             goto out;
         }
 
         if ( (pfn_type[pfn] & XEN_DOMCTL_PFINFO_LTABTYPE_MASK) !=
              ((unsigned long)pt_levels<<XEN_DOMCTL_PFINFO_LTAB_SHIFT) ) {
             ERROR("PT base is bad. pfn=%lu nr=%lu type=%08lx %08lx",
-                  pfn, max_pfn, pfn_type[pfn],
+                  pfn, p2m_size, pfn_type[pfn],
                   (unsigned long)pt_levels<<XEN_DOMCTL_PFINFO_LTAB_SHIFT);
             goto out;
         }
@@ -867,16 +878,16 @@ int xc_linux_restore(int xc_handle, int io_fd,
         {
             pfn = xen_cr3_to_pfn(ctxt.ctrlreg[1]);
 
-            if (pfn >= max_pfn) {
-                ERROR("User PT base is bad: pfn=%lu max_pfn=%lu type=%08lx",
-                      pfn, max_pfn, pfn_type[pfn]);
+            if (pfn >= p2m_size) {
+                ERROR("User PT base is bad: pfn=%lu p2m_size=%lu type=%08lx",
+                      pfn, p2m_size, pfn_type[pfn]);
                 goto out;
             }
 
             if ( (pfn_type[pfn] & XEN_DOMCTL_PFINFO_LTABTYPE_MASK) !=
                  ((unsigned long)pt_levels<<XEN_DOMCTL_PFINFO_LTAB_SHIFT) ) {
                 ERROR("User PT base is bad. pfn=%lu nr=%lu type=%08lx %08lx",
-                      pfn, max_pfn, pfn_type[pfn],
+                      pfn, p2m_size, pfn_type[pfn],
                       (unsigned long)pt_levels<<XEN_DOMCTL_PFINFO_LTAB_SHIFT);
                 goto out;
             }
@@ -915,7 +926,7 @@ int xc_linux_restore(int xc_handle, int io_fd,
     /* Uncanonicalise the pfn-to-mfn table frame-number list. */
     for (i = 0; i < P2M_FL_ENTRIES; i++) {
         pfn = p2m_frame_list[i];
-        if ((pfn >= max_pfn) || (pfn_type[pfn] != XEN_DOMCTL_PFINFO_NOTAB)) {
+        if ((pfn >= p2m_size) || (pfn_type[pfn] != XEN_DOMCTL_PFINFO_NOTAB)) {
             ERROR("PFN-to-MFN frame number is bad");
             goto out;
         }
@@ -930,8 +941,8 @@ int xc_linux_restore(int xc_handle, int io_fd,
         goto out;
     }
 
-    memcpy(live_p2m, p2m, P2M_SIZE);
-    munmap(live_p2m, P2M_SIZE);
+    memcpy(live_p2m, p2m, ROUNDUP(p2m_size * sizeof(xen_pfn_t), PAGE_SHIFT));
+    munmap(live_p2m, ROUNDUP(p2m_size * sizeof(xen_pfn_t), PAGE_SHIFT));
 
     DPRINTF("Domain ready to be built.\n");
 
diff --git a/tools/libxc/xc_linux_save.c b/tools/libxc/xc_linux_save.c
index f9800c4d5f..0636eb79ec 100644
--- a/tools/libxc/xc_linux_save.c
+++ b/tools/libxc/xc_linux_save.c
@@ -25,7 +25,7 @@
 **
 */
 #define DEF_MAX_ITERS   29   /* limit us to 30 times round loop   */
-#define DEF_MAX_FACTOR   3   /* never send more than 3x nr_pfns   */
+#define DEF_MAX_FACTOR   3   /* never send more than 3x p2m_size  */
 
 
 /* max mfn of the whole machine */
@@ -37,8 +37,8 @@ static unsigned long hvirt_start;
 /* #levels of page tables used by the current guest */
 static unsigned int pt_levels;
 
-/* total number of pages used by the current guest */
-static unsigned long max_pfn;
+/* number of pfns this guest has (i.e. number of entries in the P2M) */
+static unsigned long p2m_size;
 
 /* Live mapping of the table mapping each PFN to its current MFN. */
 static xen_pfn_t *live_p2m = NULL;
@@ -57,7 +57,7 @@ static unsigned long m2p_mfn0;
  */
 #define MFN_IS_IN_PSEUDOPHYS_MAP(_mfn)          \
 (((_mfn) < (max_mfn)) &&                        \
- ((mfn_to_pfn(_mfn) < (max_pfn)) &&               \
+ ((mfn_to_pfn(_mfn) < (p2m_size)) &&               \
   (live_p2m[mfn_to_pfn(_mfn)] == (_mfn))))
 
 
@@ -79,7 +79,7 @@ static unsigned long m2p_mfn0;
 */
 
 #define BITS_PER_LONG (sizeof(unsigned long) * 8)
-#define BITMAP_SIZE   ((max_pfn + BITS_PER_LONG - 1) / 8)
+#define BITMAP_SIZE   ((p2m_size + BITS_PER_LONG - 1) / 8)
 
 #define BITMAP_ENTRY(_nr,_bmap) \
    ((volatile unsigned long *)(_bmap))[(_nr)/BITS_PER_LONG]
@@ -343,7 +343,7 @@ static int print_stats(int xc_handle, uint32_t domid, int pages_sent,
 }
 
 
-static int analysis_phase(int xc_handle, uint32_t domid, int max_pfn,
+static int analysis_phase(int xc_handle, uint32_t domid, int p2m_size,
                           unsigned long *arr, int runs)
 {
     long long start, now;
@@ -356,7 +356,7 @@ static int analysis_phase(int xc_handle, uint32_t domid, int max_pfn,
         int i;
 
         xc_shadow_control(xc_handle, domid, XEN_DOMCTL_SHADOW_OP_CLEAN,
-                          arr, max_pfn, NULL, 0, NULL);
+                          arr, p2m_size, NULL, 0, NULL);
         DPRINTF("#Flush\n");
         for ( i = 0; i < 40; i++ ) {
             usleep(50000);
@@ -682,7 +682,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
     /* base of the region in which domain memory is mapped */
     unsigned char *region_base = NULL;
 
-    /* power of 2 order of max_pfn */
+    /* power of 2 order of p2m_size */
     int order_nr;
 
     /* bitmap of pages:
@@ -730,7 +730,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
         goto out;
     }
 
-    max_pfn = live_shinfo->arch.max_pfn;
+    p2m_size = live_shinfo->arch.max_pfn;
 
     live_p2m_frame_list_list = map_frame_list_list(xc_handle, dom,
                                                    live_shinfo);
@@ -777,7 +777,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
     memcpy(p2m_frame_list, live_p2m_frame_list, P2M_FL_SIZE);
 
     /* Canonicalise the pfn-to-mfn table frame-number list. */
-    for (i = 0; i < max_pfn; i += fpp) {
+    for (i = 0; i < p2m_size; i += fpp) {
         if (!translate_mfn_to_pfn(&p2m_frame_list[i/fpp])) {
             ERROR("Frame# in pfn-to-mfn frame list is not in pseudophys");
             ERROR("entry %d: p2m_frame_list[%ld] is 0x%"PRIx64, i, i/fpp,
@@ -813,12 +813,12 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
     }
 
     /* pretend we sent all the pages last iteration */
-    sent_last_iter = max_pfn;
+    sent_last_iter = p2m_size;
 
 
-    /* calculate the power of 2 order of max_pfn, e.g.
+    /* calculate the power of 2 order of p2m_size, e.g.
        15->4 16->4 17->5 */
-    for (i = max_pfn-1, order_nr = 0; i ; i >>= 1, order_nr++)
+    for (i = p2m_size-1, order_nr = 0; i ; i >>= 1, order_nr++)
         continue;
 
     /* Setup to_send / to_fix and to_skip bitmaps */
@@ -844,7 +844,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
         return 1;
     }
 
-    analysis_phase(xc_handle, dom, max_pfn, to_skip, 0);
+    analysis_phase(xc_handle, dom, p2m_size, to_skip, 0);
 
     /* We want zeroed memory so use calloc rather than malloc. */
     pfn_type   = calloc(MAX_BATCH_SIZE, sizeof(*pfn_type));
@@ -867,7 +867,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
     {
         int err=0;
         unsigned long mfn;
-        for (i = 0; i < max_pfn; i++) {
+        for (i = 0; i < p2m_size; i++) {
 
             mfn = live_p2m[i];
             if((mfn != INVALID_P2M_ENTRY) && (mfn_to_pfn(mfn) != i)) {
@@ -882,8 +882,8 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
 
     /* Start writing out the saved-domain record. */
 
-    if (!write_exact(io_fd, &max_pfn, sizeof(unsigned long))) {
-        ERROR("write: max_pfn");
+    if (!write_exact(io_fd, &p2m_size, sizeof(unsigned long))) {
+        ERROR("write: p2m_size");
         goto out;
     }
 
@@ -929,9 +929,9 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
 
         DPRINTF("Saving memory pages: iter %d   0%%", iter);
 
-        while( N < max_pfn ){
+        while( N < p2m_size ){
 
-            unsigned int this_pc = (N * 100) / max_pfn;
+            unsigned int this_pc = (N * 100) / p2m_size;
 
             if ((this_pc - prev_pc) >= 5) {
                 DPRINTF("\b\b\b\b%3d%%", this_pc);
@@ -942,7 +942,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
                but this is fast enough for the moment. */
             if (!last_iter && xc_shadow_control(
                     xc_handle, dom, XEN_DOMCTL_SHADOW_OP_PEEK,
-                    to_skip, max_pfn, NULL, 0, NULL) != max_pfn) {
+                    to_skip, p2m_size, NULL, 0, NULL) != p2m_size) {
                 ERROR("Error peeking shadow bitmap");
                 goto out;
             }
@@ -950,9 +950,9 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
 
             /* load pfn_type[] with the mfn of all the pages we're doing in
                this batch. */
-            for (batch = 0; batch < MAX_BATCH_SIZE && N < max_pfn ; N++) {
+            for (batch = 0; batch < MAX_BATCH_SIZE && N < p2m_size ; N++) {
 
-                int n = permute(N, max_pfn, order_nr);
+                int n = permute(N, p2m_size, order_nr);
 
                 if (debug) {
                     DPRINTF("%d pfn= %08lx mfn= %08lx %d  [mfn]= %08lx\n",
@@ -1123,7 +1123,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
             print_stats( xc_handle, dom, sent_this_iter, &stats, 1);
 
             DPRINTF("Total pages sent= %ld (%.2fx)\n",
-                    total_sent, ((float)total_sent)/max_pfn );
+                    total_sent, ((float)total_sent)/p2m_size );
             DPRINTF("(of which %ld were fixups)\n", needed_to_fix  );
         }
 
@@ -1150,7 +1150,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
             if (((sent_this_iter > sent_last_iter) && RATE_IS_MAX()) ||
                 (iter >= max_iters) ||
                 (sent_this_iter+skip_this_iter < 50) ||
-                (total_sent > max_pfn*max_factor)) {
+                (total_sent > p2m_size*max_factor)) {
                 DPRINTF("Start last iteration\n");
                 last_iter = 1;
 
@@ -1168,7 +1168,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
 
             if (xc_shadow_control(xc_handle, dom, 
                                   XEN_DOMCTL_SHADOW_OP_CLEAN, to_send, 
-                                  max_pfn, NULL, 0, &stats) != max_pfn) {
+                                  p2m_size, NULL, 0, &stats) != p2m_size) {
                 ERROR("Error flushing shadow PT");
                 goto out;
             }
@@ -1220,7 +1220,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
         unsigned int i,j;
         unsigned long pfntab[1024];
 
-        for (i = 0, j = 0; i < max_pfn; i++) {
+        for (i = 0, j = 0; i < p2m_size; i++) {
             if (!is_mapped(live_p2m[i]))
                 j++;
         }
@@ -1230,13 +1230,13 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
             goto out;
         }
 
-        for (i = 0, j = 0; i < max_pfn; ) {
+        for (i = 0, j = 0; i < p2m_size; ) {
 
             if (!is_mapped(live_p2m[i]))
                 pfntab[j++] = i;
 
             i++;
-            if (j == 1024 || i == max_pfn) {
+            if (j == 1024 || i == p2m_size) {
                 if(!write_exact(io_fd, &pfntab, sizeof(unsigned long)*j)) {
                     ERROR("Error when writing to state file (6b) (errno %d)",
                           errno);
@@ -1333,7 +1333,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
         munmap(live_p2m_frame_list, P2M_FLL_ENTRIES * PAGE_SIZE);
 
     if (live_p2m)
-        munmap(live_p2m, P2M_SIZE);
+        munmap(live_p2m, ROUNDUP(p2m_size * sizeof(xen_pfn_t), PAGE_SHIFT));
 
     if (live_m2p)
         munmap(live_m2p, M2P_SIZE(max_mfn));
diff --git a/tools/libxc/xc_resume.c b/tools/libxc/xc_resume.c
index 8e509b6fe3..3fb6d13adc 100644
--- a/tools/libxc/xc_resume.c
+++ b/tools/libxc/xc_resume.c
@@ -46,7 +46,7 @@ static int xc_domain_resume_any(int xc_handle, uint32_t domid)
     xc_dominfo_t info;
     int i, rc = -1;
 #if defined(__i386__) || defined(__x86_64__)
-    unsigned long mfn, max_pfn = 0;
+    unsigned long mfn, p2m_size = 0;
     vcpu_guest_context_t ctxt;
     start_info_t *start_info;
     shared_info_t *shinfo = NULL;
@@ -74,7 +74,7 @@ static int xc_domain_resume_any(int xc_handle, uint32_t domid)
         goto out;
     }
 
-    max_pfn = shinfo->arch.max_pfn;
+    p2m_size = shinfo->arch.max_pfn;
 
     p2m_frame_list_list =
         xc_map_foreign_range(xc_handle, domid, PAGE_SIZE, PROT_READ,
diff --git a/tools/libxc/xenguest.h b/tools/libxc/xenguest.h
index 76b7413f15..b1809050b3 100644
--- a/tools/libxc/xenguest.h
+++ b/tools/libxc/xenguest.h
@@ -43,15 +43,16 @@ int xc_hvm_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
  * @parm xc_handle a handle to an open hypervisor interface
  * @parm fd the file descriptor to restore a domain from
  * @parm dom the id of the domain
- * @parm nr_pfns the number of pages
+ * @parm p2m_size number of pages the guest has (i.e. number entries in P2M)
+ * @parm max_nr_pfns domains maximum real memory allocation, in pages
  * @parm store_evtchn the store event channel for this domain to use
  * @parm store_mfn returned with the mfn of the store page
  * @return 0 on success, -1 on failure
  */
 int xc_linux_restore(int xc_handle, int io_fd, uint32_t dom,
-                     unsigned long nr_pfns, unsigned int store_evtchn,
-                     unsigned long *store_mfn, unsigned int console_evtchn,
-                     unsigned long *console_mfn);
+                     unsigned long p2m_size, unsigned long max_nr_pfns,
+                     unsigned int store_evtchn, unsigned long *store_mfn,
+                     unsigned int console_evtchn, unsigned long *console_mfn);
 
 /**
  * This function will restore a saved hvm domain running unmodified guest.
diff --git a/tools/libxc/xg_private.h b/tools/libxc/xg_private.h
index 2b66a29763..5de7083112 100644
--- a/tools/libxc/xg_private.h
+++ b/tools/libxc/xg_private.h
@@ -148,17 +148,16 @@ typedef l4_pgentry_64_t l4_pgentry_t;
 
 #define ROUNDUP(_x,_w) (((unsigned long)(_x)+(1UL<<(_w))-1) & ~((1UL<<(_w))-1))
 
-/* Size in bytes of the P2M (rounded up to the nearest PAGE_SIZE bytes) */
-#define P2M_SIZE        ROUNDUP((max_pfn * sizeof(xen_pfn_t)), PAGE_SHIFT)
-
 /* Number of xen_pfn_t in a page */
 #define fpp             (PAGE_SIZE/sizeof(xen_pfn_t))
 
+/* XXX SMH: following 3 skanky macros rely on variable p2m_size being set */
+
 /* Number of entries in the pfn_to_mfn_frame_list_list */
-#define P2M_FLL_ENTRIES (((max_pfn)+(fpp*fpp)-1)/(fpp*fpp))
+#define P2M_FLL_ENTRIES (((p2m_size)+(fpp*fpp)-1)/(fpp*fpp))
 
 /* Number of entries in the pfn_to_mfn_frame_list */
-#define P2M_FL_ENTRIES  (((max_pfn)+fpp-1)/fpp)
+#define P2M_FL_ENTRIES  (((p2m_size)+fpp-1)/fpp)
 
 /* Size in bytes of the pfn_to_mfn_frame_list     */
 #define P2M_FL_SIZE     ((P2M_FL_ENTRIES)*sizeof(unsigned long))
diff --git a/tools/python/xen/xend/XendCheckpoint.py b/tools/python/xen/xend/XendCheckpoint.py
index 4b22ecc04f..214a59ecd0 100644
--- a/tools/python/xen/xend/XendCheckpoint.py
+++ b/tools/python/xen/xend/XendCheckpoint.py
@@ -187,6 +187,7 @@ def restore(xd, fd, dominfo = None, paused = False):
     assert console_port
 
     nr_pfns = (dominfo.getMemoryTarget() + 3) / 4 
+    max_nr_pfns = (dominfo.getMemoryMaximum() + 3) / 4 
 
     # if hvm, pass mem size to calculate the store_mfn
     image_cfg = dominfo.info.get('image', {})
@@ -203,17 +204,17 @@ def restore(xd, fd, dominfo = None, paused = False):
     try:
         l = read_exact(fd, sizeof_unsigned_long,
                        "not a valid guest state file: pfn count read")
-        max_pfn = unpack("L", l)[0]    # native sizeof long
+        p2m_size = unpack("L", l)[0]    # native sizeof long
 
-        if max_pfn > 16*1024*1024:     # XXX 
+        if p2m_size > 16*1024*1024:     # XXX 
             raise XendError(
                 "not a valid guest state file: pfn count out of range")
 
         shadow = dominfo.info['shadow_memory']
         log.debug("restore:shadow=0x%x, _static_max=0x%x, _static_min=0x%x, "
-                  "nr_pfns=0x%x.", dominfo.info['shadow_memory'],
+                  "p2m_size=0x%x.", dominfo.info['shadow_memory'],
                   dominfo.info['memory_static_max'],
-                  dominfo.info['memory_static_min'], nr_pfns)
+                  dominfo.info['memory_static_min'], p2m_size)
 
         balloon.free(xc.pages_to_kib(nr_pfns) + shadow * 1024)
 
@@ -221,7 +222,7 @@ def restore(xd, fd, dominfo = None, paused = False):
         dominfo.info['shadow_memory'] = shadow_cur
 
         cmd = map(str, [xen.util.auxbin.pathTo(XC_RESTORE),
-                        fd, dominfo.getDomid(), max_pfn,
+                        fd, dominfo.getDomid(), p2m_size, max_nr_pfns, 
                         store_port, console_port, int(is_hvm), pae, apic])
         log.debug("[xc_restore]: %s", string.join(cmd))
 
diff --git a/tools/xcutils/xc_restore.c b/tools/xcutils/xc_restore.c
index a2cf9c115f..4eef2ba8c7 100644
--- a/tools/xcutils/xc_restore.c
+++ b/tools/xcutils/xc_restore.c
@@ -18,15 +18,14 @@
 int
 main(int argc, char **argv)
 {
-    unsigned int xc_fd, io_fd, domid, max_pfn, store_evtchn, console_evtchn;
+    unsigned int xc_fd, io_fd, domid, store_evtchn, console_evtchn;
     unsigned int hvm, pae, apic;
     int ret;
-    unsigned long store_mfn, console_mfn;
+    unsigned long p2m_size, max_nr_pfns, store_mfn, console_mfn;
 
-    if (argc != 9)
-	errx(1,
-	     "usage: %s iofd domid max_pfn store_evtchn console_evtchn hvm pae apic",
-	     argv[0]);
+    if (argc != 10)
+        errx(1, "usage: %s iofd domid p2m_size max_nr_pfns store_evtchn "
+             "console_evtchn hvm pae apic", argv[0]);
 
     xc_fd = xc_interface_open();
     if (xc_fd < 0)
@@ -34,19 +33,21 @@ main(int argc, char **argv)
 
     io_fd = atoi(argv[1]);
     domid = atoi(argv[2]);
-    max_pfn = atoi(argv[3]);
-    store_evtchn = atoi(argv[4]);
-    console_evtchn = atoi(argv[5]);
-    hvm  = atoi(argv[6]);
-    pae  = atoi(argv[7]);
-    apic = atoi(argv[8]);
+    p2m_size = atoi(argv[3]);
+    max_nr_pfns = atoi(argv[4]);
+    store_evtchn = atoi(argv[5]);
+    console_evtchn = atoi(argv[6]);
+    hvm  = atoi(argv[7]);
+    pae  = atoi(argv[8]);
+    apic = atoi(argv[9]);
 
     if (hvm) {
-        ret = xc_hvm_restore(xc_fd, io_fd, domid, max_pfn, store_evtchn,
+        ret = xc_hvm_restore(xc_fd, io_fd, domid, max_nr_pfns, store_evtchn,
                 &store_mfn, pae, apic);
-    } else 
-        ret = xc_linux_restore(xc_fd, io_fd, domid, max_pfn, store_evtchn,
-                &store_mfn, console_evtchn, &console_mfn);
+    } else
+        ret = xc_linux_restore(xc_fd, io_fd, domid, p2m_size,
+                               max_nr_pfns, store_evtchn, &store_mfn,
+                               console_evtchn, &console_mfn);
 
     if (ret == 0) {
 	printf("store-mfn %li\n", store_mfn);