libxl: domain save/restore: run in a separate process

libxenctrl expects to be able to simply run the save or restore
operation synchronously.  This won't work well in a process which is
trying to handle multiple domains.

The options are:

 - Block such a whole process (eg, the whole of libvirt) while
   migration completes (or until it fails).

 - Create a thread to run xc_domain_save and xc_domain_restore on.
   This is quite unpalatable.  Multithreaded programming is error
   prone enough without generating threads in libraries, particularly
   if the thread does some very complex operation.

 - Fork and run the operation in the child without execing.  This is
   no good because we would need to negotiate with the caller about
   fds we would inherit (and we might be a very large process).

 - Fork and exec a helper.

Of these options the latter is the most palatable.

Consequently:

 * A new helper program libxl-save-helper (which does both save and
   restore).  It will be installed in /usr/lib/xen/bin.  It does not
   link against libxl, only libxc, and its error handling does not
   need to be very advanced.  It does contain a plumbing through of
   the logging interface into the callback stream.

 * A small ad-hoc protocol between the helper and libxl which allows
   log messages and the libxc callbacks to be passed up and down.
   Protocol doc comment is in libxl_save_helper.c.

 * To avoid a lot of tedium the marshalling boilerplate (stubs for the
   helper and the callback decoder for libxl) is generated with a
   small perl script.

 * Implement new functionality to spawn the helper, monitor its
   output, provide responses, and check on its exit status.

 * The functions libxl__xc_domain_restore_done and
   libxl__xc_domain_save_done now turn out to want be called in the
   same place.  So make their state argument a void* so that the two
   functions are type compatible.

The domain save path still writes the qemu savefile synchronously.
This will need to be fixed in a subsequent patch.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>

1 files changed, 350 insertions, 11 deletions

diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
index 1b481ab6be..a6abcda123 100644
--- a/tools/libxl/libxl_save_callout.c
+++ b/tools/libxl/libxl_save_callout.c
@@ -16,6 +16,30 @@
 
 #include "libxl_internal.h"
 
+/* stream_fd is as from the caller (eventually, the application).
+ * It may be 0, 1 or 2, in which case we need to dup it elsewhere.
+ * The actual fd value is not included in the supplied argnums; rather
+ * it will be automatically supplied by run_helper as the 2nd argument.
+ *
+ * preserve_fds are fds that the caller is intending to pass to the
+ * helper so which need cloexec clearing.  They may not be 0, 1 or 2.
+ * An entry may be -1 in which case it will be ignored.
+ */
+static void run_helper(libxl__egc *egc, libxl__save_helper_state *shs,
+                       const char *mode_arg,
+                       int stream_fd,
+                       const int *preserve_fds, int num_preserve_fds,
+                       const unsigned long *argnums, int num_argnums);
+
+static void helper_failed(libxl__egc*, libxl__save_helper_state *shs, int rc);
+static void helper_stdout_readable(libxl__egc *egc, libxl__ev_fd *ev,
+                                   int fd, short events, short revents);
+static void helper_exited(libxl__egc *egc, libxl__ev_child *ch,
+                          pid_t pid, int status);
+static void helper_done(libxl__egc *egc, libxl__save_helper_state *shs);
+
+/*----- entrypoints -----*/
+
 void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
                               int hvm, int pae, int superpages,
                               int no_incr_generationid)
@@ -27,22 +51,337 @@ void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
     const int restore_fd = dcs->restore_fd;
     libxl__domain_build_state *const state = &dcs->build_state;
 
-    int r = xc_domain_restore(CTX->xch, restore_fd, domid,
-                              state->store_port, &state->store_mfn,
-                              state->store_domid, state->console_port,
-                              &state->console_mfn, state->console_domid,
-                              hvm, pae, superpages, no_incr_generationid,
-                              &state->vm_generationid_addr, &dcs->callbacks);
-    libxl__xc_domain_restore_done(egc, dcs, 0, r, errno);
+    unsigned cbflags = libxl__srm_callout_enumcallbacks_restore
+        (&dcs->shs.callbacks.restore.a);
+
+    const unsigned long argnums[] = {
+        domid,
+        state->store_port,
+        state->store_domid, state->console_port,
+        state->console_domid,
+        hvm, pae, superpages, no_incr_generationid,
+        cbflags,
+    };
+
+    dcs->shs.ao = ao;
+    dcs->shs.domid = domid;
+    dcs->shs.recv_callback = libxl__srm_callout_received_restore;
+    dcs->shs.completion_callback = libxl__xc_domain_restore_done;
+    dcs->shs.caller_state = dcs;
+    dcs->shs.need_results = 1;
+    dcs->shs.toolstack_data_file = 0;
+
+    run_helper(egc, &dcs->shs, "--restore-domain", restore_fd, 0,0,
+               argnums, ARRAY_SIZE(argnums));
 }
 
 void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss,
                            unsigned long vm_generationid_addr)
 {
     STATE_AO_GC(dss->ao);
-    int r;
+    int r, rc, toolstack_data_fd = -1;
+    uint32_t toolstack_data_len = 0;
+
+    /* Resources we need to free */
+    uint8_t *toolstack_data_buf = 0;
+
+    unsigned cbflags = libxl__srm_callout_enumcallbacks_save
+        (&dss->shs.callbacks.save.a);
+
+    if (dss->shs.callbacks.save.toolstack_save) {
+        r = dss->shs.callbacks.save.toolstack_save
+            (dss->domid, &toolstack_data_buf, &toolstack_data_len, dss);
+        if (r) { rc = ERROR_FAIL; goto out; }
+
+        dss->shs.toolstack_data_file = tmpfile();
+        if (!dss->shs.toolstack_data_file) {
+            LOGE(ERROR, "cannot create toolstack data tmpfile");
+            rc = ERROR_FAIL;
+            goto out;
+        }
+        toolstack_data_fd = fileno(dss->shs.toolstack_data_file);
+
+        r = libxl_write_exactly(CTX, toolstack_data_fd,
+                                toolstack_data_buf, toolstack_data_len,
+                                "toolstack data tmpfile", 0);
+        if (r) { rc = ERROR_FAIL; goto out; }
+    }
+
+    const unsigned long argnums[] = {
+        dss->domid, 0, 0, dss->xcflags, dss->hvm, vm_generationid_addr,
+        toolstack_data_fd, toolstack_data_len,
+        cbflags,
+    };
+
+    dss->shs.ao = ao;
+    dss->shs.domid = dss->domid;
+    dss->shs.recv_callback = libxl__srm_callout_received_save;
+    dss->shs.completion_callback = libxl__xc_domain_save_done;
+    dss->shs.caller_state = dss;
+    dss->shs.need_results = 0;
+
+    free(toolstack_data_buf);
+
+    run_helper(egc, &dss->shs, "--save-domain", dss->fd,
+               &toolstack_data_fd, 1,
+               argnums, ARRAY_SIZE(argnums));
+    return;
+
+ out:
+    free(toolstack_data_buf);
+    if (dss->shs.toolstack_data_file) fclose(dss->shs.toolstack_data_file);
+
+    libxl__xc_domain_save_done(egc, dss, rc, 0, 0);
+}
+
+
+/*----- helper execution -----*/
+
+static void run_helper(libxl__egc *egc, libxl__save_helper_state *shs,
+                       const char *mode_arg, int stream_fd,
+                       const int *preserve_fds, int num_preserve_fds,
+                       const unsigned long *argnums, int num_argnums)
+{
+    STATE_AO_GC(shs->ao);
+    const char *args[4 + num_argnums];
+    const char **arg = args;
+    int i, rc;
+
+    /* Resources we must free */
+    libxl__carefd *childs_pipes[2] = { 0,0 };
+
+    /* Convenience aliases */
+    const uint32_t domid = shs->domid;
+
+    shs->rc = 0;
+    shs->completed = 0;
+    shs->pipes[0] = shs->pipes[1] = 0;
+    libxl__ev_fd_init(&shs->readable);
+    libxl__ev_child_init(&shs->child);
+
+    shs->stdin_what = GCSPRINTF("domain %"PRIu32" save/restore helper"
+                                " stdin pipe", domid);
+    shs->stdout_what = GCSPRINTF("domain %"PRIu32" save/restore helper"
+                                 " stdout pipe", domid);
+
+    *arg++ = getenv("LIBXL_SAVE_HELPER") ?: LIBEXEC "/" "libxl-save-helper";
+    *arg++ = mode_arg;
+    const char **stream_fd_arg = arg++;
+    for (i=0; i<num_argnums; i++)
+        *arg++ = GCSPRINTF("%lu", argnums[i]);
+    *arg++ = 0;
+    assert(arg == args + ARRAY_SIZE(args));
+
+    libxl__carefd_begin();
+    int childfd;
+    for (childfd=0; childfd<2; childfd++) {
+        /* Setting up the pipe for the child's fd childfd */
+        int fds[2];
+        if (libxl_pipe(CTX,fds)) { rc = ERROR_FAIL; goto out; }
+        int childs_end = childfd==0 ? 0 /*read*/  : 1 /*write*/;
+        int our_end    = childfd==0 ? 1 /*write*/ : 0 /*read*/;
+        childs_pipes[childfd] = libxl__carefd_record(CTX, fds[childs_end]);
+        shs->pipes[childfd] =   libxl__carefd_record(CTX, fds[our_end]);
+    }
+    libxl__carefd_unlock();
+
+    pid_t pid = libxl__ev_child_fork(gc, &shs->child, helper_exited);
+    if (!pid) {
+        if (stream_fd <= 2) {
+            stream_fd = dup(stream_fd);
+            if (stream_fd < 0) {
+                LOGE(ERROR,"dup migration stream fd");
+                exit(-1);
+            }
+        }
+        libxl_fd_set_cloexec(CTX, stream_fd, 0);
+        *stream_fd_arg = GCSPRINTF("%d", stream_fd);
+
+        for (i=0; i<num_preserve_fds; i++)
+            if (preserve_fds[i] >= 0) {
+                assert(preserve_fds[i] > 2);
+                libxl_fd_set_cloexec(CTX, preserve_fds[i], 0);
+            }
+
+        libxl__exec(gc,
+                    libxl__carefd_fd(childs_pipes[0]),
+                    libxl__carefd_fd(childs_pipes[1]),
+                    -1,
+                    args[0], (char**)args, 0);
+    }
+
+    libxl__carefd_close(childs_pipes[0]);
+    libxl__carefd_close(childs_pipes[1]);
+
+    rc = libxl__ev_fd_register(gc, &shs->readable, helper_stdout_readable,
+                               libxl__carefd_fd(shs->pipes[1]), POLLIN|POLLPRI);
+    if (rc) goto out;
+    return;
+
+ out:
+    libxl__carefd_close(childs_pipes[0]);
+    libxl__carefd_close(childs_pipes[1]);
+    helper_failed(egc, shs, rc);;
+}
+
+static void helper_failed(libxl__egc *egc, libxl__save_helper_state *shs,
+                          int rc)
+{
+    STATE_AO_GC(shs->ao);
+
+    if (!shs->rc)
+        shs->rc = rc;
+
+    libxl__ev_fd_deregister(gc, &shs->readable);
+
+    if (!libxl__ev_child_inuse(&shs->child)) {
+        helper_done(egc, shs);
+        return;
+    }
+
+    int r = kill(shs->child.pid, SIGKILL);
+    if (r) LOGE(WARN, "failed to kill save/restore helper [%lu]",
+                (unsigned long)shs->child.pid);
+}
+
+static void helper_stdout_readable(libxl__egc *egc, libxl__ev_fd *ev,
+                                   int fd, short events, short revents)
+{
+    libxl__save_helper_state *shs = CONTAINER_OF(ev, *shs, readable);
+    STATE_AO_GC(shs->ao);
+    int rc, errnoval;
+
+    if (revents & (POLLERR|POLLPRI)) {
+        LOG(ERROR, "%s signaled POLLERR|POLLPRI (%#x)",
+            shs->stdout_what, revents);
+        rc = ERROR_FAIL;
+ out:
+        /* this is here because otherwise we bypass the decl of msg[] */
+        helper_failed(egc, shs, rc);
+        return;
+    }
+
+    uint16_t msglen;
+    errnoval = libxl_read_exactly(CTX, fd, &msglen, sizeof(msglen),
+                                  shs->stdout_what, "ipc msg header");
+    if (errnoval) { rc = ERROR_FAIL; goto out; }
+
+    unsigned char msg[msglen];
+    errnoval = libxl_read_exactly(CTX, fd, msg, msglen,
+                                  shs->stdout_what, "ipc msg body");
+    if (errnoval) { rc = ERROR_FAIL; goto out; }
+
+    shs->egc = egc;
+    shs->recv_callback(msg, msglen, shs);
+    shs->egc = 0;
+    return;
+}
+
+static void helper_exited(libxl__egc *egc, libxl__ev_child *ch,
+                          pid_t pid, int status)
+{
+    libxl__save_helper_state *shs = CONTAINER_OF(ch, *shs, child);
+    STATE_AO_GC(shs->ao);
+
+    /* Convenience aliases */
+    const uint32_t domid = shs->domid;
+
+    const char *what =
+        GCSPRINTF("domain %"PRIu32" save/restore helper", domid);
+
+    if (status) {
+        libxl_report_child_exitstatus(CTX, XTL_ERROR, what, pid, status);
+        shs->rc = ERROR_FAIL;
+    }
+
+    if (shs->need_results) {
+        if (!shs->rc)
+            LOG(ERROR,"%s exited without providing results",what);
+        shs->rc = ERROR_FAIL;
+    }
+
+    if (!shs->completed) {
+        if (!shs->rc)
+            LOG(ERROR,"%s exited without signaling completion",what);
+        shs->rc = ERROR_FAIL;
+    }
+
+    helper_done(egc, shs);
+    return;
+}
+
+static void helper_done(libxl__egc *egc, libxl__save_helper_state *shs)
+{
+    STATE_AO_GC(shs->ao);
+
+    libxl__ev_fd_deregister(gc, &shs->readable);
+    libxl__carefd_close(shs->pipes[0]);  shs->pipes[0] = 0;
+    libxl__carefd_close(shs->pipes[1]);  shs->pipes[1] = 0;
+    assert(!libxl__ev_child_inuse(&shs->child));
+    if (shs->toolstack_data_file) fclose(shs->toolstack_data_file);
+
+    shs->egc = egc;
+    shs->completion_callback(egc, shs->caller_state,
+                             shs->rc, shs->retval, shs->errnoval);
+    shs->egc = 0;
+}
+
+/*----- generic helpers for the autogenerated code -----*/
+
+const libxl__srm_save_autogen_callbacks*
+libxl__srm_callout_get_callbacks_save(void *user)
+{
+    libxl__save_helper_state *shs = user;
+    return &shs->callbacks.save.a;
+}
+
+const libxl__srm_restore_autogen_callbacks*
+libxl__srm_callout_get_callbacks_restore(void *user)
+{
+    libxl__save_helper_state *shs = user;
+    return &shs->callbacks.restore.a;
+}
+
+void libxl__srm_callout_sendreply(int r, void *user)
+{
+    libxl__save_helper_state *shs = user;
+    libxl__egc *egc = shs->egc;
+    STATE_AO_GC(shs->ao);
+    int errnoval;
+
+    errnoval = libxl_write_exactly(CTX, libxl__carefd_fd(shs->pipes[0]),
+                                   &r, sizeof(r), shs->stdin_what,
+                                   "callback return value");
+    if (errnoval)
+        helper_failed(egc, shs, ERROR_FAIL);
+}
+
+void libxl__srm_callout_callback_log(uint32_t level, uint32_t errnoval,
+                  const char *context, const char *formatted, void *user)
+{
+    libxl__save_helper_state *shs = user;
+    STATE_AO_GC(shs->ao);
+    xtl_log(CTX->lg, level, errnoval, context, "%s", formatted);
+}
+
+void libxl__srm_callout_callback_progress(const char *context,
+                   const char *doing_what, unsigned long done,
+                   unsigned long total, void *user)
+{
+    libxl__save_helper_state *shs = user;
+    STATE_AO_GC(shs->ao);
+    xtl_progress(CTX->lg, context, doing_what, done, total);
+}
+
+int libxl__srm_callout_callback_complete(int retval, int errnoval,
+                                         void *user)
+{
+    libxl__save_helper_state *shs = user;
+    STATE_AO_GC(shs->ao);
 
-    r = xc_domain_save(CTX->xch, dss->fd, dss->domid, 0, 0, dss->xcflags,
-                       &dss->callbacks, dss->hvm, vm_generationid_addr);
-    libxl__xc_domain_save_done(egc, dss, 0, r, errno);
+    shs->completed = 1;
+    shs->retval = retval;
+    shs->errnoval = errnoval;
+    libxl__ev_fd_deregister(gc, &shs->readable);
+    return 0;
 }