libxl: domain save/restore: run in a separate process

libxenctrl expects to be able to simply run the save or restore
operation synchronously.  This won't work well in a process which is
trying to handle multiple domains.

The options are:

 - Block such a whole process (eg, the whole of libvirt) while
   migration completes (or until it fails).

 - Create a thread to run xc_domain_save and xc_domain_restore on.
   This is quite unpalatable.  Multithreaded programming is error
   prone enough without generating threads in libraries, particularly
   if the thread does some very complex operation.

 - Fork and run the operation in the child without execing.  This is
   no good because we would need to negotiate with the caller about
   fds we would inherit (and we might be a very large process).

 - Fork and exec a helper.

Of these options the latter is the most palatable.

Consequently:

 * A new helper program libxl-save-helper (which does both save and
   restore).  It will be installed in /usr/lib/xen/bin.  It does not
   link against libxl, only libxc, and its error handling does not
   need to be very advanced.  It does contain a plumbing through of
   the logging interface into the callback stream.

 * A small ad-hoc protocol between the helper and libxl which allows
   log messages and the libxc callbacks to be passed up and down.
   Protocol doc comment is in libxl_save_helper.c.

 * To avoid a lot of tedium the marshalling boilerplate (stubs for the
   helper and the callback decoder for libxl) is generated with a
   small perl script.

 * Implement new functionality to spawn the helper, monitor its
   output, provide responses, and check on its exit status.

 * The functions libxl__xc_domain_restore_done and
   libxl__xc_domain_save_done now turn out to want be called in the
   same place.  So make their state argument a void* so that the two
   functions are type compatible.

The domain save path still writes the qemu savefile synchronously.
This will need to be fixed in a subsequent patch.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>

1 files changed, 50 insertions, 6 deletions

diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 1ee5bd2b09..890490482a 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -54,6 +54,7 @@
 
 #include "libxl.h"
 #include "_paths.h"
+#include "_libxl_save_msgs_callout.h"
 
 #if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1)
 #define _hidden __attribute__((visibility("hidden")))
@@ -1773,6 +1774,51 @@ _hidden void libxl__datacopier_kill(libxl__datacopier_state *dc);
 _hidden int libxl__datacopier_start(libxl__datacopier_state *dc);
 
 
+/*----- Save/restore helper (used by creation and suspend) -----*/
+
+typedef struct libxl__srm_save_callbacks {
+    libxl__srm_save_autogen_callbacks a;
+    int (*toolstack_save)(uint32_t domid, uint8_t **buf,
+                          uint32_t *len, void *data);
+} libxl__srm_save_callbacks;
+
+typedef struct libxl__srm_restore_callbacks {
+    libxl__srm_restore_autogen_callbacks a;
+} libxl__srm_restore_callbacks;
+
+/* a pointer to this struct is also passed as "user" to the
+ * save callout helper callback functions */
+typedef struct libxl__save_helper_state {
+    /* public, caller of run_helper initialises */
+    libxl__ao *ao;
+    uint32_t domid;
+    union {
+        libxl__srm_save_callbacks save;
+        libxl__srm_restore_callbacks restore;
+    } callbacks;
+    int (*recv_callback)(const unsigned char *msg, uint32_t len, void *user);
+    void (*completion_callback)(libxl__egc *egc, void *caller_state,
+                                int rc, int retval, int errnoval);
+    void *caller_state;
+    int need_results; /* set to 0 or 1 by caller of run_helper;
+                       * if set to 1 then the ultimate caller's
+                       * results function must set it to 0 */
+    /* private */
+    int rc;
+    int completed; /* retval/errnoval valid iff completed */
+    int retval, errnoval; /* from xc_domain_save / xc_domain_restore */
+    libxl__carefd *pipes[2]; /* 0 = helper's stdin, 1 = helper's stdout */
+    libxl__ev_fd readable;
+    libxl__ev_child child;
+    const char *stdin_what, *stdout_what;
+    FILE *toolstack_data_file;
+
+    libxl__egc *egc; /* valid only for duration of each event callback;
+                      * is here in this struct for the benefit of the
+                      * marshalling and xc callback functions */
+} libxl__save_helper_state;
+
+
 /*----- Domain suspend (save) state structure -----*/
 
 typedef struct libxl__domain_suspend_state libxl__domain_suspend_state;
@@ -1798,7 +1844,7 @@ struct libxl__domain_suspend_state {
     int xcflags;
     int guest_responded;
     int interval; /* checkpoint interval (for Remus) */
-    struct save_callbacks callbacks;
+    libxl__save_helper_state shs;
 };
 
 
@@ -1910,7 +1956,7 @@ struct libxl__domain_create_state {
     libxl__stub_dm_spawn_state dmss;
         /* If we're not doing stubdom, we use only dmss.dm,
          * for the non-stubdom device model. */
-    struct restore_callbacks callbacks;
+    libxl__save_helper_state shs;
 };
 
 /*----- Domain suspend (save) functions -----*/
@@ -1926,8 +1972,7 @@ _hidden void libxl__xc_domain_save(libxl__egc*, libxl__domain_suspend_state*,
 /* If rc==0 then retval is the return value from xc_domain_save
  * and errnoval is the errno value it provided.
  * If rc!=0, retval and errnoval are undefined. */
-_hidden void libxl__xc_domain_save_done(libxl__egc*,
-                                        libxl__domain_suspend_state*,
+_hidden void libxl__xc_domain_save_done(libxl__egc*, void *dss_void,
                                         int rc, int retval, int errnoval);
 
 _hidden int libxl__domain_suspend_common_callback(void *data);
@@ -1945,8 +1990,7 @@ _hidden void libxl__xc_domain_restore(libxl__egc *egc,
 /* If rc==0 then retval is the return value from xc_domain_save
  * and errnoval is the errno value it provided.
  * If rc!=0, retval and errnoval are undefined. */
-_hidden void libxl__xc_domain_restore_done(libxl__egc *egc,
-                                           libxl__domain_create_state *dcs,
+_hidden void libxl__xc_domain_restore_done(libxl__egc *egc, void *dcs_void,
                                            int rc, int retval, int errnoval);