libxl: ao: convert libxl__spawn_*

libxl__spawn_spawn becomes a callback-style asynchronous function.
The implementation is now in terms of libxl__ev_* including
libxl_ev_child.

All the callers need to be updated.  This includes the device model
spawning functions libxl__create_device_model and
libxl__create_stubdom; these are replaced with libxl__spawn_local_dm
and libxl__spawn_stubdom.  libxl__confirm_device_model_startup is
abolished; instead the dm spawner calls back.

(The choice of which kind of device model to create is lifted out of
what used to be libxl__create_device_model, because that function was
indirectly recursive.  Recursive callback-style operations are clumsy
because they require a pointer indirection for the nested states.)

Waiting for proper device model startup it is no longer notionally
optional.  Previously the code appeared to tolerate this by passing
NULL for various libxl__spawner_starting* parameters to device model
spawners.  However, this was not used anywhere.

Conversely, the "for_spawn" parameter to libxl__wait_for_offspring is
no longer supported.  It remains as an unused formal parameter to
avoid updating, in this patch, all the call sites which pass NULL.
libxl__wait_for_offspring is in any case itself an obsolete function,
so this wrinkle will go away when its callers are updated to use the
event system.  Consequently libxl__spawn_check is also abolished.

The "console ready" callback also remains unchanged in this patch.
The API for this needs to be reviewed in the context of the event
series and its reentrancy restrictions documented.

Thus their callers need to be updated.  These are the domain creation
functions libxl_domain_create_new and _restore.  These functions now
take ao_hows, and have a private state structure.

However domain creation remains not completely converted to the event
mechanism; in particular it runs the outward-facing function
libxl_run_bootloader with a NULL ao_how, which is quite wrong.  As it
happens in the current code this is not a bug because none of the rest
of the functionality surrounding the bootloader call will mind if the
event loop is reentered in the middle of its execution.

The file-scope function libxl__set_fd_flag which was used by the
previous spawn arrangements becomes unused and is removed; other
places in libxl can use libxl_fd_set_nonblock and
libxl_fd_set_cloexec, which of course remain.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>

Changes since v8:
 * Make midproc_cb callback with correct pid (that of the grandchild,
   not "middle" which is zero).  Reported by Roger Pau Monne.

Changes since v7:
 * Rename libxl__spawn_stubdom to libxl__spawn_stub_dm (and ..._state);
   rename the state's stubdom_* members to dm_*.
 * Eliminate the union between the two dm creation states in
   libxl__domain_create_state.  Instead, the domain creation code
   simply uses libxl__stub_dm_spawn_state.dm directly, if we're
   taking the local dm path.
 * Remove a spurious "break".
 * In domain creation, move the PV non-qemu case into the switch.
 * Code style fixes.
 * Constify some convenience aliases.
 * Improve comments (including typo fixes).
Committed-by: Ian Jackson <Ian.Jackson@eu.citrix.com>

1 files changed, 162 insertions, 44 deletions

diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index b137288f1f..9ce107a689 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -558,16 +558,40 @@ static int store_libxl_entry(libxl__gc *gc, uint32_t domid,
         libxl_device_model_version_to_string(b_info->device_model_version));
 }
 
-static int do_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
-                            libxl_console_ready cb, void *priv,
-                            uint32_t *domid_out, int restore_fd)
+/*----- main domain creation -----*/
+
+/* We have a linear control flow; only one event callback is
+ * outstanding at any time.  Each initiation and callback function
+ * arranges for the next to be called, as the very last thing it
+ * does.  (If that particular sub-operation is not needed, a
+ * function will call the next event callback directly.)
+ */
+
+/* Event callbacks, in this order: */
+static void domcreate_devmodel_started(libxl__egc *egc,
+                                       libxl__dm_spawn_state *dmss,
+                                       int rc);
+
+/* Our own function to clean up and call the user's callback.
+ * The final call in the sequence. */
+static void domcreate_complete(libxl__egc *egc,
+                               libxl__domain_create_state *dcs,
+                               int rc);
+
+static void initiate_domain_create(libxl__egc *egc,
+                                   libxl__domain_create_state *dcs)
 {
+    STATE_AO_GC(dcs->ao);
     libxl_ctx *ctx = libxl__gc_owner(gc);
-    libxl__spawner_starting *dm_starting = 0;
-    libxl__domain_build_state state[1];
     uint32_t domid;
     int i, ret;
 
+    /* convenience aliases */
+    libxl_domain_config *const d_config = dcs->guest_config;
+    const int restore_fd = dcs->restore_fd;
+    const libxl_console_ready cb = dcs->console_cb;
+    void *const priv = dcs->console_cb_priv;
+
     domid = 0;
 
     ret = libxl__domain_create_info_setdefault(gc, &d_config->c_info);
@@ -580,9 +604,12 @@ static int do_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
         goto error_out;
     }
 
+    dcs->guest_domid = domid;
+    dcs->dmss.dm.guest_domid = 0; /* means we haven't spawned */
+
     if ( d_config->c_info.type == LIBXL_DOMAIN_TYPE_PV && cb ) {
-        if ( (*cb)(ctx, domid, priv) )
-            goto error_out;
+        ret = (*cb)(ctx, domid, priv);
+        if (ret) goto error_out;
     }
 
     ret = libxl__domain_build_info_setdefault(gc, &d_config->b_info);
@@ -606,7 +633,17 @@ static int do_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
         }
     }
 
-    memset(state, 0, sizeof(*state));
+    memset(&dcs->build_state, 0, sizeof(dcs->build_state));
+    libxl__domain_build_state *state = &dcs->build_state;
+
+    /* We might be going to call libxl__spawn_local_dm, or _spawn_stub_dm.
+     * Fill in any field required by either, including both relevant
+     * callbacks (_spawn_stub_dm will overwrite our trespass if needed). */
+    dcs->dmss.dm.spawn.ao = ao;
+    dcs->dmss.dm.guest_config = dcs->guest_config;
+    dcs->dmss.dm.build_state = &dcs->build_state;
+    dcs->dmss.dm.callback = domcreate_devmodel_started;
+    dcs->dmss.callback = domcreate_devmodel_started;
 
     if ( restore_fd >= 0 ) {
         ret = domain_restore(gc, &d_config->b_info, domid, restore_fd, state);
@@ -656,14 +693,12 @@ static int do_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
         libxl_device_vkb_add(ctx, domid, &vkb);
         libxl_device_vkb_dispose(&vkb);
 
-        ret = libxl__create_device_model(gc, domid, d_config,
-                                         state, &dm_starting);
-        if (ret < 0) {
-            LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
-                       "failed to create device model: %d", ret);
-            goto error_out;
-        }
-        break;
+        dcs->dmss.dm.guest_domid = domid;
+        if (libxl_defbool_val(d_config->b_info.device_model_stubdomain))
+            libxl__spawn_stub_dm(egc, &dcs->dmss);
+        else
+            libxl__spawn_local_dm(egc, &dcs->dmss.dm);
+        return;
     }
     case LIBXL_DOMAIN_TYPE_PV:
     {
@@ -687,26 +722,52 @@ static int do_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
         libxl__device_console_dispose(&console);
 
         if (need_qemu) {
-            libxl__create_xenpv_qemu(gc, domid, d_config, state, &dm_starting);
+            dcs->dmss.dm.guest_domid = domid;
+            libxl__spawn_local_dm(egc, &dcs->dmss.dm);
+            return;
+        } else {
+            assert(!dcs->dmss.dm.guest_domid);
+            domcreate_devmodel_started(egc, &dcs->dmss.dm, 0);
+            return;
         }
-        break;
     }
     default:
         ret = ERROR_INVAL;
         goto error_out;
     }
+    abort(); /* not reached */
+
+ error_out:
+    assert(ret);
+    domcreate_complete(egc, dcs, ret);
+}
+
+static void domcreate_devmodel_started(libxl__egc *egc,
+                                       libxl__dm_spawn_state *dmss,
+                                       int ret)
+{
+    libxl__domain_create_state *dcs = CONTAINER_OF(dmss, *dcs, dmss.dm);
+    STATE_AO_GC(dmss->spawn.ao);
+    int i;
+    libxl_ctx *ctx = CTX;
+    int domid = dcs->guest_domid;
+
+    /* convenience aliases */
+    libxl_domain_config *const d_config = dcs->guest_config;
+    const libxl_console_ready cb = dcs->console_cb;
+    void *const priv = dcs->console_cb_priv;
+
+    if (ret) {
+        LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
+                   "device model did not start: %d", ret);
+        goto error_out;
+    }
 
-    if (dm_starting) {
+    if (dcs->dmss.dm.guest_domid) {
         if (d_config->b_info.device_model_version
             == LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN) {
             libxl__qmp_initializations(gc, domid, d_config);
         }
-        ret = libxl__confirm_device_model_startup(gc, state, dm_starting);
-        if (ret < 0) {
-            LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
-                       "device model did not start: %d", ret);
-            goto error_out;
-        }
     }
 
     for (i = 0; i < d_config->num_pcidevs; i++)
@@ -734,38 +795,95 @@ static int do_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
     if ( cb && (d_config->c_info.type == LIBXL_DOMAIN_TYPE_HVM ||
                 (d_config->c_info.type == LIBXL_DOMAIN_TYPE_PV &&
                  d_config->b_info.u.pv.bootloader ))) {
-        if ( (*cb)(ctx, domid, priv) )
-            goto error_out;
+        ret = (*cb)(ctx, domid, priv);
+        if (ret) goto error_out;
     }
 
-    *domid_out = domid;
-    return 0;
+    domcreate_complete(egc, dcs, 0);
+    return;
 
 error_out:
-    if (domid)
-        libxl_domain_destroy(ctx, domid);
+    assert(ret);
+    domcreate_complete(egc, dcs, ret);
+}
 
-    return ret;
+static void domcreate_complete(libxl__egc *egc,
+                               libxl__domain_create_state *dcs,
+                               int rc)
+{
+    STATE_AO_GC(dcs->ao);
+
+    if (rc) {
+        if (dcs->guest_domid) {
+            int rc2 = libxl_domain_destroy(CTX, dcs->guest_domid);
+            if (rc2)
+                LOG(ERROR, "unable to destroy domain %d following"
+                    " failed creation", dcs->guest_domid);
+        }
+        dcs->guest_domid = -1;
+    }
+    dcs->callback(egc, dcs, rc, dcs->guest_domid);
 }
 
+/*----- application-facing domain creation interface -----*/
+
+typedef struct {
+    libxl__domain_create_state dcs;
+    uint32_t *domid_out;
+} libxl__app_domain_create_state;
+
+static void domain_create_cb(libxl__egc *egc,
+                             libxl__domain_create_state *dcs,
+                             int rc, uint32_t domid);
+
+static int do_domain_create(libxl_ctx *ctx, libxl_domain_config *d_config,
+                            libxl_console_ready cb, void *priv, uint32_t *domid,
+                            int restore_fd, const libxl_asyncop_how *ao_how)
+{
+    AO_CREATE(ctx, 0, ao_how);
+    libxl__app_domain_create_state *cdcs;
+
+    GCNEW(cdcs);
+    cdcs->dcs.ao = ao;
+    cdcs->dcs.guest_config = d_config;
+    cdcs->dcs.restore_fd = restore_fd;
+    cdcs->dcs.console_cb = cb;
+    cdcs->dcs.console_cb_priv = priv;
+    cdcs->dcs.callback = domain_create_cb;
+    cdcs->domid_out = domid;
+
+    initiate_domain_create(egc, &cdcs->dcs);
+
+    return AO_INPROGRESS;
+}
+
+static void domain_create_cb(libxl__egc *egc,
+                             libxl__domain_create_state *dcs,
+                             int rc, uint32_t domid)
+{
+    libxl__app_domain_create_state *cdcs = CONTAINER_OF(dcs, *cdcs, dcs);
+    STATE_AO_GC(cdcs->dcs.ao);
+
+    if (!rc)
+        *cdcs->domid_out = domid;
+
+    libxl__ao_complete(egc, ao, rc);
+}
+    
 int libxl_domain_create_new(libxl_ctx *ctx, libxl_domain_config *d_config,
-                            libxl_console_ready cb, void *priv, uint32_t *domid)
+                            libxl_console_ready cb, void *priv,
+                            uint32_t *domid,
+                            const libxl_asyncop_how *ao_how)
 {
-    GC_INIT(ctx);
-    int rc;
-    rc = do_domain_create(gc, d_config, cb, priv, domid, -1);
-    GC_FREE;
-    return rc;
+    return do_domain_create(ctx, d_config, cb, priv, domid, -1, ao_how);
 }
 
 int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
-                                libxl_console_ready cb, void *priv, uint32_t *domid, int restore_fd)
+                                libxl_console_ready cb, void *priv,
+                                uint32_t *domid, int restore_fd,
+                                const libxl_asyncop_how *ao_how)
 {
-    GC_INIT(ctx);
-    int rc;
-    rc = do_domain_create(gc, d_config, cb, priv, domid, restore_fd);
-    GC_FREE;
-    return rc;
+    return do_domain_create(ctx, d_config, cb, priv, domid, restore_fd, ao_how);
 }
 
 /*