Fix checkpointing - add save/restore support to the xenbus driver.

- Add xenbus driver suspend/resume functions
- Change xenbus irq to not be shared
- Translate store mfn in suspend record to pfn and back
- Make tools re-introduce the domain to the store
Signed-off-by: Rusty Russel <rusty@rustcorp.com.au>
Signed-off-by: Christian Limpach <Christian.Limpach@cl.cam.ac.uk>

5 files changed, 73 insertions, 20 deletions

diff --git a/tools/libxc/xc.h b/tools/libxc/xc.h
index 417f3d3c94..5c82147c11 100644
--- a/tools/libxc/xc.h
+++ b/tools/libxc/xc.h
@@ -279,9 +279,12 @@ int xc_linux_save(int xc_handle, int fd, u32 dom);
  * @parm fd the file descriptor to restore a domain from
  * @parm dom the id of the domain
  * @parm nr_pfns the number of pages
+ * @parm store_evtchn the store event channel for this domain to use
+ * @parm store_mfn returned with the mfn of the store page
  * @return 0 on success, -1 on failure
  */
-int xc_linux_restore(int xc_handle, int io_fd, u32 dom, unsigned long nr_pfns);
+int xc_linux_restore(int xc_handle, int io_fd, u32 dom, unsigned long nr_pfns,
+		     unsigned int store_evtchn, unsigned long *store_mfn);
 
 int xc_linux_build(int xc_handle,
                    u32 domid,
diff --git a/tools/libxc/xc_linux_restore.c b/tools/libxc/xc_linux_restore.c
index 92e57f2c27..c2bce43910 100644
--- a/tools/libxc/xc_linux_restore.c
+++ b/tools/libxc/xc_linux_restore.c
@@ -48,7 +48,8 @@ read_exact(int fd, void *buf, size_t count)
     return r;
 }
 
-int xc_linux_restore(int xc_handle, int io_fd, u32 dom, unsigned long nr_pfns)
+int xc_linux_restore(int xc_handle, int io_fd, u32 dom, unsigned long nr_pfns,
+		     unsigned int store_evtchn, unsigned long *store_mfn)
 {
     dom0_op_t op;
     int rc = 1, i, n, k;
@@ -464,10 +465,13 @@ int xc_linux_restore(int xc_handle, int io_fd, u32 dom, unsigned long nr_pfns)
     }
     ctxt.user_regs.esi = mfn = pfn_to_mfn_table[pfn];
     p_srec = xc_map_foreign_range(
-        xc_handle, dom, PAGE_SIZE, PROT_WRITE, mfn);
+        xc_handle, dom, PAGE_SIZE, PROT_READ | PROT_WRITE, mfn);
     p_srec->resume_info.nr_pages    = nr_pfns;
     p_srec->resume_info.shared_info = shared_info_frame << PAGE_SHIFT;
     p_srec->resume_info.flags       = 0;
+    *store_mfn = p_srec->resume_info.store_mfn   =
+	pfn_to_mfn_table[p_srec->resume_info.store_mfn];
+    p_srec->resume_info.store_evtchn = store_evtchn;
     munmap(p_srec, PAGE_SIZE);
 
     /* Uncanonicalise each GDT frame number. */
diff --git a/tools/libxc/xc_linux_save.c b/tools/libxc/xc_linux_save.c
index 023a59c799..930002f684 100644
--- a/tools/libxc/xc_linux_save.c
+++ b/tools/libxc/xc_linux_save.c
@@ -20,7 +20,7 @@
 #define DEBUG 0
 
 #if 1
-#define ERR(_f, _a...) fprintf ( stderr, _f , ## _a )
+#define ERR(_f, _a...) do { fprintf(stderr, _f , ## _a); fflush(stderr); } while (0)
 #else
 #define ERR(_f, _a...) ((void)0)
 #endif
@@ -643,6 +643,22 @@ int xc_linux_save(int xc_handle, int io_fd, u32 dom)
         goto out;
     }
 
+    /* Map the suspend-record MFN to pin it. The page must be owned by 
+       dom for this to succeed. */
+    p_srec = xc_map_foreign_range(xc_handle, dom,
+                                   sizeof(*p_srec), PROT_READ | PROT_WRITE, 
+                                   ctxt.user_regs.esi);
+    if (!p_srec){
+        ERR("Couldn't map suspend record");
+        goto out;
+    }
+
+    /* Canonicalize store mfn. */
+    if ( !translate_mfn_to_pfn(&p_srec->resume_info.store_mfn) ) {
+	ERR("Store frame is not in range of pseudophys map");
+	goto out;
+    }
+
     print_stats( xc_handle, dom, 0, &stats, 0 );
 
     /* Now write out each data page, canonicalising page tables as we go... */
@@ -983,16 +999,6 @@ int xc_linux_save(int xc_handle, int io_fd, u32 dom)
 	}
     }
 
-    /* Map the suspend-record MFN to pin it. The page must be owned by 
-       dom for this to succeed. */
-    p_srec = xc_map_foreign_range(xc_handle, dom,
-                                   sizeof(*p_srec), PROT_READ, 
-                                   ctxt.user_regs.esi);
-    if (!p_srec){
-        ERR("Couldn't map suspend record");
-        goto out;
-    }
-
     if (nr_pfns != p_srec->nr_pfns )
     {
 	ERR("Suspend record nr_pfns unexpected (%ld != %ld)",
diff --git a/tools/python/xen/xend/XendCheckpoint.py b/tools/python/xen/xend/XendCheckpoint.py
index 705555c510..fa4befeec5 100644
--- a/tools/python/xen/xend/XendCheckpoint.py
+++ b/tools/python/xen/xend/XendCheckpoint.py
@@ -6,6 +6,7 @@
 
 import errno
 import os
+import re
 import select
 import sxp
 from string import join
@@ -64,6 +65,13 @@ def save(xd, fd, dominfo):
                 if l.rstrip() == "suspend":
                     log.info("suspending %d" % dominfo.id)
                     xd.domain_shutdown(dominfo.id, reason='suspend')
+                    if dominfo.store_channel:
+                        try:
+                            dominfo.db.releaseDomain(dominfo.id)
+                        except Exception, ex:
+                            log.warning("error in domain release on xenstore: %s",
+                                        ex)
+                            pass
                     dominfo.state_wait("suspended")
                     log.info("suspend %d done" % dominfo.id)
                     child.tochild.write("done\n")
@@ -76,6 +84,11 @@ def save(xd, fd, dominfo):
     if child.wait() != 0:
         raise XendError("xc_save failed: %s" % lasterr)
 
+    if dominfo.store_channel:
+        dominfo.store_channel.close()
+        dominfo.db['store_channel'].delete()
+        dominfo.db.saveDB(save=True)
+        dominfo.store_channel = None
     xd.domain_destroy(dominfo.id)
     return None
 
@@ -107,8 +120,13 @@ def restore(xd, fd):
         raise XendError(
             "not a valid guest state file: pfn count out of range")
 
+    if dominfo.store_channel:
+        evtchn = dominfo.store_channel.port2
+    else:
+        evtchn = 0
+
     cmd = [PATH_XC_RESTORE, str(xc.handle()), str(fd),
-           str(dominfo.id), str(nr_pfns)]
+           str(dominfo.id), str(nr_pfns), str(evtchn)]
     log.info("[xc_restore] " + join(cmd))
     child = xPopen3(cmd, True, -1, [fd, xc.handle()])
     child.tochild.close()
@@ -128,7 +146,21 @@ def restore(xd, fd):
                 lasterr = l.rstrip()
             if fd == child.fromchild.fileno():
                 l = child.fromchild.readline()
-                log.info(l.rstrip())
+                while l:
+                    m = re.match(r"^(store-mfn) (\d+)\n$", l)
+                    if m:
+                        if dominfo.store_channel:
+                            dominfo.store_mfn = int(m.group(2))
+                            if dominfo.store_mfn >= 0:
+                                dominfo.db.introduceDomain(dominfo.id,
+                                                           dominfo.store_mfn,
+                                                           dominfo.store_channel)
+                            dominfo.exportToDB(save=True, sync=True))
+                    log.info(l.rstrip())
+                    try:
+                        l = child.fromchild.readline()
+                    except:
+                        l = None
         if filter(lambda (fd, event): event & select.POLLHUP, r):
             break
 
diff --git a/tools/xcutils/xc_restore.c b/tools/xcutils/xc_restore.c
index ebba6d698f..2eec44c40c 100644
--- a/tools/xcutils/xc_restore.c
+++ b/tools/xcutils/xc_restore.c
@@ -16,15 +16,23 @@
 int
 main(int argc, char **argv)
 {
-    unsigned int xc_fd, io_fd, domid, nr_pfns;
+    unsigned int xc_fd, io_fd, domid, nr_pfns, evtchn;
+    int ret;
+    unsigned long mfn;
 
-    if (argc != 5)
-	errx(1, "usage: %s xcfd iofd domid nr_pfns", argv[0]);
+    if (argc != 6)
+	errx(1, "usage: %s xcfd iofd domid nr_pfns evtchn", argv[0]);
 
     xc_fd = atoi(argv[1]);
     io_fd = atoi(argv[2]);
     domid = atoi(argv[3]);
     nr_pfns = atoi(argv[4]);
+    evtchn = atoi(argv[5]);
 
-    return xc_linux_restore(xc_fd, io_fd, domid, nr_pfns);
+    ret = xc_linux_restore(xc_fd, io_fd, domid, nr_pfns, evtchn, &mfn);
+    if (ret == 0) {
+	printf("store-mfn %li\n", mfn);
+	fflush(stdout);
+    }
+    return ret;
 }