libxl, minios: stubdom console based save/restore

Add two "special" PV consoles to stubdoms that are going to be used
to send and receive the qemu-xen save files on save/restore.

Use the second PV console to send the qemu-xen save file and the third
PV console to receive the qemu-xen save file on restore.

Fix the console shutdown function free_consfront that is called when the
qemu save file is closed.

Stubdom save/restore is still broken with xend.

Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Committed-by: Ian Jackson <ian.jackson@eu.citrix.com>

6 files changed, 104 insertions, 17 deletions

diff --git a/extras/mini-os/console/xencons_ring.c b/extras/mini-os/console/xencons_ring.c
index bfe5cb663c..22fd6187bc 100644
--- a/extras/mini-os/console/xencons_ring.c
+++ b/extras/mini-os/console/xencons_ring.c
@@ -184,16 +184,43 @@ struct consfront_dev *xencons_ring_init(void)
 
 void free_consfront(struct consfront_dev *dev)
 {
-    mask_evtchn(dev->evtchn);
+    char* err = NULL;
+    XenbusState state;
 
-    free(dev->backend);
+    char path[strlen(dev->backend) + 1 + 5 + 1];
+    char nodename[strlen(dev->nodename) + 1 + 5 + 1];
 
-    gnttab_end_access(dev->ring_ref);
-    free_page(dev->ring);
+    snprintf(path, sizeof(path), "%s/state", dev->backend);
+    snprintf(nodename, sizeof(nodename), "%s/state", dev->nodename);
 
-    unbind_evtchn(dev->evtchn);
+    if ((err = xenbus_switch_state(XBT_NIL, nodename, XenbusStateClosing)) != NULL) {
+        printk("free_consfront: error changing state to %d: %s\n",
+                XenbusStateClosing, err);
+        goto close;
+    }
+    state = xenbus_read_integer(path);
+    while (err == NULL && state < XenbusStateClosing)
+        err = xenbus_wait_for_state_change(path, &state, &dev->events);
+    if (err) free(err);
+
+    if ((err = xenbus_switch_state(XBT_NIL, nodename, XenbusStateClosed)) != NULL) {
+        printk("free_consfront: error changing state to %d: %s\n",
+                XenbusStateClosed, err);
+        goto close;
+    }
 
+close:
+    if (err) free(err);
+    xenbus_unwatch_path_token(XBT_NIL, path, path);
+
+    mask_evtchn(dev->evtchn);
+    unbind_evtchn(dev->evtchn);
+    free(dev->backend);
     free(dev->nodename);
+
+    gnttab_end_access(dev->ring_ref);
+
+    free_page(dev->ring);
     free(dev);
 }
 
@@ -206,7 +233,7 @@ struct consfront_dev *init_consfront(char *_nodename)
     char* msg = NULL;
     char nodename[256];
     char path[256];
-    static int consfrontends = 1;
+    static int consfrontends = 3;
     struct consfront_dev *dev;
     int res;
 
diff --git a/extras/mini-os/include/lib.h b/extras/mini-os/include/lib.h
index 9102d55b81..84e4de2bfe 100644
--- a/extras/mini-os/include/lib.h
+++ b/extras/mini-os/include/lib.h
@@ -146,6 +146,7 @@ enum fd_type {
     FTYPE_KBD,
     FTYPE_FB,
     FTYPE_MEM,
+    FTYPE_SAVEFILE,
 };
 
 #define MAX_EVTCHN_PORTS 16
diff --git a/extras/mini-os/lib/sys.c b/extras/mini-os/lib/sys.c
index 51a801f0ca..7a49886e2d 100644
--- a/extras/mini-os/lib/sys.c
+++ b/extras/mini-os/lib/sys.c
@@ -154,6 +154,9 @@ char *getcwd(char *buf, size_t size)
 }
 
 #define LOG_PATH "/var/log/"
+#define SAVE_PATH "/var/lib/xen"
+#define SAVE_CONSOLE 1
+#define RESTORE_CONSOLE 2
 
 int mkdir(const char *pathname, mode_t mode)
 {
@@ -175,6 +178,21 @@ int posix_openpt(int flags)
     return(dev->fd);
 }
 
+int open_savefile(char *path, int save)
+{
+    struct consfront_dev *dev;
+    char *nodename[64];
+
+    snprintf(nodename, sizeof(nodename), "device/console/%d", save ? SAVE_CONSOLE : RESTORE_CONSOLE);
+
+    dev = init_consfront(nodename);
+    dev->fd = alloc_fd(FTYPE_SAVEFILE);
+    files[dev->fd].cons.dev = dev;
+
+    printk("fd(%d) = open_savefile\n", dev->fd);
+    return(dev->fd);
+}
+
 int open(const char *pathname, int flags, ...)
 {
     int fd;
@@ -191,6 +209,8 @@ int open(const char *pathname, int flags, ...)
     }
     if (!strncmp(pathname, "/dev/ptmx", strlen("/dev/ptmx")))
         return posix_openpt(flags);
+    if (!strncmp(pathname,SAVE_PATH,strlen(SAVE_PATH)))
+        return open_savefile(pathname, flags & O_WRONLY);
     errno = EIO;
     return -1;
 }
@@ -203,6 +223,7 @@ int isatty(int fd)
 int read(int fd, void *buf, size_t nbytes)
 {
     switch (files[fd].type) {
+        case FTYPE_SAVEFILE:
 	case FTYPE_CONSOLE: {
 	    int ret;
             DEFINE_WAIT(w);
@@ -260,6 +281,15 @@ int read(int fd, void *buf, size_t nbytes)
 int write(int fd, const void *buf, size_t nbytes)
 {
     switch (files[fd].type) {
+        case FTYPE_SAVEFILE: {
+                int ret = 0, tot = nbytes;
+                while (nbytes > 0) {
+                    ret = xencons_ring_send(files[fd].cons.dev, (char *)buf, nbytes);
+                    nbytes -= ret;
+                    buf += ret;
+                }
+                return tot - nbytes;
+            }
 	case FTYPE_CONSOLE:
 	    console_print(files[fd].cons.dev, (char *)buf, nbytes);
 	    return nbytes;
@@ -331,6 +361,7 @@ int close(int fd)
             shutdown_fbfront(files[fd].fb.dev);
             files[fd].type = FTYPE_NONE;
             return 0;
+        case FTYPE_SAVEFILE:
         case FTYPE_CONSOLE:
             fini_console(files[fd].cons.dev);
             files[fd].type = FTYPE_NONE;
@@ -364,9 +395,15 @@ int fstat(int fd, struct stat *buf)
 {
     init_stat(buf);
     switch (files[fd].type) {
+	case FTYPE_SAVEFILE:
 	case FTYPE_CONSOLE:
 	case FTYPE_SOCKET: {
-	    buf->st_mode = (files[fd].type == FTYPE_CONSOLE?S_IFCHR:S_IFSOCK) | S_IRUSR|S_IWUSR;
+            if (files[fd].type == FTYPE_CONSOLE)
+                buf->st_mode = S_IFCHR|S_IRUSR|S_IWUSR;
+            else if (files[fd].type == FTYPE_SOCKET)
+                buf->st_mode = S_IFSOCK|S_IRUSR|S_IWUSR;
+            else if (files[fd].type == FTYPE_SAVEFILE)
+                buf->st_mode = S_IFREG|S_IRUSR|S_IWUSR;
 	    buf->st_uid = 0;
 	    buf->st_gid = 0;
 	    buf->st_size = 0;
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 66ed7382cd..8277f53493 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -737,7 +737,8 @@ int libxl_primary_console_exec(libxl_ctx *ctx, uint32_t domid_vm)
 {
     uint32_t stubdomid = libxl_get_stubdom_id(ctx, domid_vm);
     if (stubdomid)
-        return libxl_console_exec(ctx, stubdomid, 1, LIBXL_CONSTYPE_PV);
+        return libxl_console_exec(ctx, stubdomid,
+                STUBDOM_CONSOLE_SERIAL, LIBXL_CONSTYPE_PV);
     else {
         if (libxl__domain_is_hvm(ctx, domid_vm))
             return libxl_console_exec(ctx, domid_vm, 0, LIBXL_CONSTYPE_SERIAL);
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index 878625253f..3cfebbf28a 100644
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -440,7 +440,7 @@ static int libxl_create_stubdom(libxl_ctx *ctx,
                                 libxl__device_model_starting **starting_r)
 {
     libxl__gc gc = LIBXL_INIT_GC(ctx);
-    int i, num_console = 1, ret;
+    int i, num_console = STUBDOM_SPECIAL_CONSOLES, ret;
     libxl_device_console *console;
     libxl_domain_create_info c_info;
     libxl_domain_build_info b_info;
@@ -543,15 +543,31 @@ retry_transaction:
         console[i].devid = i;
         console[i].consback = LIBXL_CONSBACK_IOEMU;
         console[i].domid = domid;
-        if (!i) {
+        /* STUBDOM_CONSOLE_LOGGING (console 0) is for minios logging
+         * STUBDOM_CONSOLE_SAVE (console 1) is for writing the save file
+         * STUBDOM_CONSOLE_RESTORE (console 2) is for reading the save file
+         */
+        switch (i) {
             char *filename;
-            char *name = libxl__sprintf(&gc, "qemu-dm-%s", libxl_domid_to_name(ctx, info->domid));
-            libxl_create_logfile(ctx, name, &filename);
-            console[i].output = libxl__sprintf(&gc, "file:%s", filename);
-            console[i].build_state = &state;
-            free(filename);
-        } else
-            console[i].output = "pty";
+            char *name;
+            case STUBDOM_CONSOLE_LOGGING:
+                name = libxl__sprintf(&gc, "qemu-dm-%s", libxl_domid_to_name(ctx, info->domid));
+                libxl_create_logfile(ctx, name, &filename);
+                console[i].output = libxl__sprintf(&gc, "file:%s", filename);
+                console[i].build_state = &state;
+                free(filename);
+                break;
+            case STUBDOM_CONSOLE_SAVE:
+                console[i].output = libxl__sprintf(&gc, "file:"SAVEFILE".%d", info->domid);
+                break;
+            case STUBDOM_CONSOLE_RESTORE:
+                if (info->saved_state)
+                    console[i].output = libxl__sprintf(&gc, "pipe:%s", info->saved_state);
+                break;
+            default:
+                console[i].output = "pty";
+                break;
+        }
         ret = libxl_device_console_add(ctx, domid, &console[i]);
         if (ret)
             goto out_free;
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 0271fcbad4..2816021ba9 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -45,6 +45,11 @@
 #define LIBXL_HVM_EXTRA_MEMORY 2048
 #define LIBXL_MIN_DOM0_MEM (128*1024)
 #define QEMU_SIGNATURE "DeviceModelRecord0002"
+#define STUBDOM_CONSOLE_LOGGING 0
+#define STUBDOM_CONSOLE_SAVE 1
+#define STUBDOM_CONSOLE_RESTORE 2
+#define STUBDOM_CONSOLE_SERIAL 3
+#define STUBDOM_SPECIAL_CONSOLES 3
 #define SAVEFILE "/var/lib/xen/qemu-save"
 
 #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))