diff options
| -rw-r--r-- | tools/libxl/libxl.c | 17 | ||||
| -rw-r--r-- | tools/libxl/libxl.h | 1 | ||||
| -rw-r--r-- | tools/libxl/libxl_event.c | 53 | ||||
| -rw-r--r-- | tools/libxl/libxl_event.h | 147 | ||||
| -rw-r--r-- | tools/libxl/libxl_fork.c | 255 | ||||
| -rw-r--r-- | tools/libxl/libxl_internal.h | 62 |
6 files changed, 515 insertions, 20 deletions
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index c420e19b69..939cc029e1 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c @@ -39,7 +39,7 @@ int libxl_ctx_alloc(libxl_ctx **pctx, int version, memset(ctx, 0, sizeof(libxl_ctx)); ctx->lg = lg; - /* First initialise pointers (cannot fail) */ + /* First initialise pointers etc. (cannot fail) */ LIBXL_TAILQ_INIT(&ctx->occurred); @@ -58,6 +58,11 @@ int libxl_ctx_alloc(libxl_ctx **pctx, int version, LIBXL_TAILQ_INIT(&ctx->death_list); libxl__ev_xswatch_init(&ctx->death_watch); + ctx->childproc_hooks = &libxl__childproc_default_hooks; + ctx->childproc_user = 0; + + ctx->sigchld_selfpipe[0] = -1; + /* The mutex is special because we can't idempotently destroy it */ if (libxl__init_recursive_mutex(ctx, &ctx->lock) < 0) { @@ -160,6 +165,16 @@ int libxl_ctx_free(libxl_ctx *ctx) discard_events(&ctx->occurred); + /* If we have outstanding children, then the application inherits + * them; we wish the application good luck with understanding + * this if and when it reaps them. */ + libxl__sigchld_removehandler(ctx); + + if (ctx->sigchld_selfpipe[0] >= 0) { + close(ctx->sigchld_selfpipe[0]); + close(ctx->sigchld_selfpipe[1]); + } + pthread_mutex_destroy(&ctx->lock); GC_FREE; diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h index d59f0ee943..fb90aedb20 100644 --- a/tools/libxl/libxl.h +++ b/tools/libxl/libxl.h @@ -380,6 +380,7 @@ enum { ERROR_NOT_READY = -11, ERROR_OSEVENT_REG_FAIL = -12, ERROR_BUFFERFULL = -13, + ERROR_UNKNOWN_CHILD = -14, }; diff --git a/tools/libxl/libxl_event.c b/tools/libxl/libxl_event.c index 93c1d1f969..3e784f03f6 100644 --- a/tools/libxl/libxl_event.c +++ b/tools/libxl/libxl_event.c @@ -627,6 +627,10 @@ static int beforepoll_internal(libxl__gc *gc, libxl__poller *poller, \ REQUIRE_FD(poller->wakeup_pipe[0], POLLIN, BODY); \ \ + int selfpipe = libxl__fork_selfpipe_active(CTX); \ + if (selfpipe >= 0) \ + REQUIRE_FD(selfpipe, POLLIN, BODY); \ + \ }while(0) #define REQUIRE_FD(req_fd_, req_events_, BODY) do{ \ @@ -766,10 +770,11 @@ static void afterpoll_internal(libxl__egc *egc, libxl__poller *poller, int nfds, const struct pollfd *fds, struct timeval now) { + /* May make callbacks into the application for child processes. + * ctx must be locked exactly once */ EGC_GC; libxl__ev_fd *efd; - LIBXL_LIST_FOREACH(efd, &CTX->efds, entry) { if (!efd->events) continue; @@ -780,11 +785,16 @@ static void afterpoll_internal(libxl__egc *egc, libxl__poller *poller, } if (afterpoll_check_fd(poller,fds,nfds, poller->wakeup_pipe[0],POLLIN)) { - char buf[256]; - int r = read(poller->wakeup_pipe[0], buf, sizeof(buf)); - if (r < 0) - if (errno != EINTR && errno != EWOULDBLOCK) - LIBXL__EVENT_DISASTER(egc, "read wakeup", errno, 0); + int e = libxl__self_pipe_eatall(poller->wakeup_pipe[0]); + if (e) LIBXL__EVENT_DISASTER(egc, "read wakeup", e, 0); + } + + int selfpipe = libxl__fork_selfpipe_active(CTX); + if (selfpipe >= 0 && + afterpoll_check_fd(poller,fds,nfds, selfpipe, POLLIN)) { + int e = libxl__self_pipe_eatall(selfpipe); + if (e) LIBXL__EVENT_DISASTER(egc, "read sigchld pipe", e, 0); + libxl__fork_selfpipe_woken(egc); } for (;;) { @@ -1082,16 +1092,37 @@ void libxl__poller_put(libxl_ctx *ctx, libxl__poller *p) void libxl__poller_wakeup(libxl__egc *egc, libxl__poller *p) { + int e = libxl__self_pipe_wakeup(p->wakeup_pipe[1]); + if (e) LIBXL__EVENT_DISASTER(egc, "cannot poke watch pipe", e, 0); +} + +int libxl__self_pipe_wakeup(int fd) +{ static const char buf[1] = ""; for (;;) { - int r = write(p->wakeup_pipe[1], buf, 1); - if (r==1) return; + int r = write(fd, buf, 1); + if (r==1) return 0; assert(r==-1); if (errno == EINTR) continue; - if (errno == EWOULDBLOCK) return; - LIBXL__EVENT_DISASTER(egc, "cannot poke watch pipe", errno, 0); - return; + if (errno == EWOULDBLOCK) return 0; + assert(errno); + return errno; + } +} + +int libxl__self_pipe_eatall(int fd) +{ + char buf[256]; + for (;;) { + int r = read(fd, buf, sizeof(buf)); + if (r == sizeof(buf)) continue; + if (r >= 0) return 0; + assert(r == -1); + if (errno == EINTR) continue; + if (errno == EWOULDBLOCK) return 0; + assert(errno); + return errno; } } diff --git a/tools/libxl/libxl_event.h b/tools/libxl/libxl_event.h index 2d2196f34a..713d96d1ed 100644 --- a/tools/libxl/libxl_event.h +++ b/tools/libxl/libxl_event.h @@ -163,11 +163,6 @@ void libxl_event_register_callbacks(libxl_ctx *ctx, * After libxl_ctx_free, all corresponding evgen handles become * invalid and must no longer be passed to evdisable. * - * Events enabled with evenable prior to a fork and libxl_ctx_postfork - * are no longer generated after the fork/postfork; however the evgen - * structures are still valid and must be passed to evdisable if the - * memory they use should not be leaked. - * * Applications should ensure that they eventually retrieve every * event using libxl_event_check or libxl_event_wait, since events * which occur but are not retreived by the application will be queued @@ -372,6 +367,148 @@ void libxl_osevent_occurred_fd(libxl_ctx *ctx, void *for_libxl, void libxl_osevent_occurred_timeout(libxl_ctx *ctx, void *for_libxl); +/*======================================================================*/ + +/* + * Subprocess handling. + * + * Unfortunately the POSIX interface makes this very awkward. + * + * There are two possible arrangements for collecting statuses from + * wait/waitpid. + * + * For naive programs: + * + * libxl will keep a SIGCHLD handler installed whenever it has an + * active (unreaped) child. It will reap all children with + * wait(); any children it does not recognise will be passed to + * the application via an optional callback (and will result in + * logged warnings if no callback is provided or the callback + * denies responsibility for the child). + * + * libxl may have children whenever: + * + * - libxl is performing an operation which can be made + * asynchronous; ie one taking a libxl_asyncop_how, even + * if NULL is passed indicating that the operation is + * synchronous; or + * + * - events of any kind are being generated, as requested + * by libxl_evenable_.... + * + * A multithreaded application which is naive in this sense may + * block SIGCHLD on some of its threads, but there must be at + * least one thread that has SIGCHLD unblocked. libxl will not + * modify the blocking flag for SIGCHLD (except that it may create + * internal service threads with all signals blocked). + * + * A naive program must only have at any one time only + * one libxl context which might have children. + * + * For programs which run their own children alongside libxl's: + * + * A program which does this must call libxl_childproc_setmode. + * There are two options: + * + * libxl_sigchld_owner_mainloop: + * The application must install a SIGCHLD handler and reap (at + * least) all of libxl's children and pass their exit status + * to libxl by calling libxl_childproc_exited. + * + * libxl_sigchld_owner_libxl_always: + * The application expects libxl to reap all of its children, + * and provides a callback to be notified of their exit + * statues. + * + * An application which fails to call setmode, or which passes 0 for + * hooks, while it uses any libxl operation which might + * create or use child processes (see above): + * - Must not have any child processes running. + * - Must not install a SIGCHLD handler. + * - Must not reap any children. + */ + + +typedef enum { + /* libxl owns SIGCHLD whenever it has a child. */ + libxl_sigchld_owner_libxl, + + /* Application promises to call libxl_childproc_exited but NOT + * from within a signal handler. libxl will not itself arrange to + * (un)block or catch SIGCHLD. */ + libxl_sigchld_owner_mainloop, + + /* libxl owns SIGCHLD all the time, and the application is + * relying on libxl's event loop for reaping its own children. */ + libxl_sigchld_owner_libxl_always, +} libxl_sigchld_owner; + +typedef struct { + libxl_sigchld_owner chldowner; + + /* All of these are optional: */ + + /* Called by libxl instead of fork. Should behave exactly like + * fork, including setting errno etc. May NOT reenter into libxl. + * Application may use this to discover pids of libxl's children, + * for example. + */ + pid_t (*fork_replacement)(void *user); + + /* With libxl_sigchld_owner_libxl, called by libxl when it has + * reaped a pid. (Not permitted with _owner_mainloop.) + * + * Should return 0 if the child was recognised by the application + * (or if the application does not keep those kind of records), + * ERROR_UNKNOWN_CHILD if the application knows that the child is not + * the application's; if it returns another error code it is a + * disaster as described for libxl_event_register_callbacks. + * (libxl will report unexpected children to its error log.) + * + * If not supplied, the application is assumed not to start + * any children of its own. + * + * This function is NOT called from within the signal handler. + * Rather it will be called from inside a libxl's event handling + * code and thus only when libxl is running, for example from + * within libxl_event_wait. (libxl uses the self-pipe trick + * to implement this.) + * + * childproc_exited_callback may call back into libxl, but it + * is best to avoid making long-running libxl calls as that might + * stall the calling event loop while the nested operation + * completes. + */ + int (*reaped_callback)(pid_t, int status, void *user); +} libxl_childproc_hooks; + +/* hooks may be 0 in which is equivalent to &{ libxl_sigchld_owner_libxl, 0, 0 } + * + * May not be called when libxl might have any child processes, or the + * behaviour is undefined. So it is best to call this at + * initialisation. + */ +void libxl_childproc_setmode(libxl_ctx *ctx, const libxl_childproc_hooks *hooks, + void *user); + +/* + * This function is for an application which owns SIGCHLD and which + * therefore reaps all of the process's children. + * + * May be called only by an application which has called setmode with + * chldowner == libxl_sigchld_owner_mainloop. If pid was a process started + * by this instance of libxl, returns 0 after doing whatever + * processing is appropriate. Otherwise silently returns + * ERROR_UNKNOWN_CHILD. No other error returns are possible. + * + * May NOT be called from within a signal handler which might + * interrupt any libxl operation. The application will almost + * certainly need to use the self-pipe trick (or a working pselect or + * ppoll) to implement this. + */ +int libxl_childproc_reaped(libxl_ctx *ctx, pid_t, int status); + + /* * An application which initialises a libxl_ctx in a parent process * and then forks a child which does not quickly exec, must diff --git a/tools/libxl/libxl_fork.c b/tools/libxl/libxl_fork.c index dce88ad54b..35c8bdd7f3 100644 --- a/tools/libxl/libxl_fork.c +++ b/tools/libxl/libxl_fork.c @@ -46,6 +46,12 @@ static int atfork_registered; static LIBXL_LIST_HEAD(, libxl__carefd) carefds = LIBXL_LIST_HEAD_INITIALIZER(carefds); +/* non-null iff installed, protected by no_forking */ +static libxl_ctx *sigchld_owner; +static struct sigaction sigchld_saved_action; + +static void sigchld_removehandler_core(void); + static void atfork_lock(void) { int r = pthread_mutex_lock(&no_forking); @@ -107,6 +113,7 @@ void libxl_postfork_child_noexec(libxl_ctx *ctx) int r; atfork_lock(); + LIBXL_LIST_FOREACH_SAFE(cf, &carefds, entry, cf_tmp) { if (cf->fd >= 0) { r = close(cf->fd); @@ -118,6 +125,10 @@ void libxl_postfork_child_noexec(libxl_ctx *ctx) free(cf); } LIBXL_LIST_INIT(&carefds); + + if (sigchld_owner) + sigchld_removehandler_core(); + atfork_unlock(); } @@ -141,6 +152,250 @@ int libxl__carefd_fd(const libxl__carefd *cf) } /* + * Actual child process handling + */ + +static void sigchld_handler(int signo) +{ + int e = libxl__self_pipe_wakeup(sigchld_owner->sigchld_selfpipe[1]); + assert(!e); /* errors are probably EBADF, very bad */ +} + +static void sigchld_removehandler_core(void) +{ + struct sigaction was; + int r; + + r = sigaction(SIGCHLD, &sigchld_saved_action, &was); + assert(!r); + assert(!(was.sa_flags & SA_SIGINFO)); + assert(was.sa_handler == sigchld_handler); + sigchld_owner = 0; +} + +void libxl__sigchld_removehandler(libxl_ctx *ctx) /* non-reentrant */ +{ + atfork_lock(); + if (sigchld_owner == ctx) + sigchld_removehandler_core(); + atfork_unlock(); +} + +int libxl__sigchld_installhandler(libxl_ctx *ctx) /* non-reentrant */ +{ + int r, rc; + + if (ctx->sigchld_selfpipe[0] < 0) { + r = pipe(ctx->sigchld_selfpipe); + if (r) { + ctx->sigchld_selfpipe[0] = -1; + LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, + "failed to create sigchld pipe"); + rc = ERROR_FAIL; + goto out; + } + } + + atfork_lock(); + if (sigchld_owner != ctx) { + struct sigaction ours; + + assert(!sigchld_owner); + sigchld_owner = ctx; + + memset(&ours,0,sizeof(ours)); + ours.sa_handler = sigchld_handler; + sigemptyset(&ours.sa_mask); + ours.sa_flags = SA_NOCLDSTOP | SA_RESTART; + r = sigaction(SIGCHLD, &ours, &sigchld_saved_action); + assert(!r); + + assert(((void)"application must negotiate with libxl about SIGCHLD", + !(sigchld_saved_action.sa_flags & SA_SIGINFO) && + (sigchld_saved_action.sa_handler == SIG_DFL || + sigchld_saved_action.sa_handler == SIG_IGN))); + } + atfork_unlock(); + + rc = 0; + out: + return rc; +} + +static int chldmode_ours(libxl_ctx *ctx) +{ + return ctx->childproc_hooks->chldowner == libxl_sigchld_owner_libxl; +} + +int libxl__fork_selfpipe_active(libxl_ctx *ctx) +{ + /* Returns the fd to read, or -1 */ + if (!chldmode_ours(ctx)) + return -1; + + if (LIBXL_LIST_EMPTY(&ctx->children)) + return -1; + + return ctx->sigchld_selfpipe[0]; +} + +static void perhaps_removehandler(libxl_ctx *ctx) +{ + if (LIBXL_LIST_EMPTY(&ctx->children) && + ctx->childproc_hooks->chldowner != libxl_sigchld_owner_libxl_always) + libxl__sigchld_removehandler(ctx); +} + +static int childproc_reaped(libxl__egc *egc, pid_t pid, int status) +{ + EGC_GC; + libxl__ev_child *ch; + + LIBXL_LIST_FOREACH(ch, &CTX->children, entry) + if (ch->pid == pid) + goto found; + + /* not found */ + return ERROR_UNKNOWN_CHILD; + + found: + LIBXL_LIST_REMOVE(ch, entry); + ch->pid = -1; + ch->callback(egc, ch, pid, status); + + perhaps_removehandler(CTX); + + return 0; +} + +int libxl_childproc_reaped(libxl_ctx *ctx, pid_t pid, int status) +{ + EGC_INIT(ctx); + CTX_LOCK; + int rc = childproc_reaped(egc, pid, status); + CTX_UNLOCK; + EGC_FREE; + return rc; +} + +void libxl__fork_selfpipe_woken(libxl__egc *egc) +{ + /* May make callbacks into the application for child processes. + * ctx must be locked EXACTLY ONCE */ + EGC_GC; + + while (chldmode_ours(CTX) /* in case the app changes the mode */) { + int status; + pid_t pid = waitpid(-1, &status, WNOHANG); + + if (pid == 0) return; + + if (pid == -1) { + if (errno == ECHILD) return; + if (errno == EINTR) continue; + LIBXL__EVENT_DISASTER(egc, "waitpid() failed", errno, 0); + return; + } + + int rc = childproc_reaped(egc, pid, status); + + if (rc) { + if (CTX->childproc_hooks->reaped_callback) { + CTX_UNLOCK; + rc = CTX->childproc_hooks->reaped_callback + (pid, status, CTX->childproc_user); + CTX_LOCK; + if (rc != 0 && rc != ERROR_UNKNOWN_CHILD) { + char disasterbuf[200]; + snprintf(disasterbuf, sizeof(disasterbuf), " reported by" + " libxl_childproc_hooks->reaped_callback" + " (for pid=%lu, status=%d; error code %d)", + (unsigned long)pid, status, rc); + LIBXL__EVENT_DISASTER(egc, disasterbuf, 0, 0); + return; + } + } else { + rc = ERROR_UNKNOWN_CHILD; + } + if (rc) + libxl_report_child_exitstatus(CTX, XTL_WARN, + "unknown child", (long)pid, status); + } + } +} + +pid_t libxl__ev_child_fork(libxl__gc *gc, libxl__ev_child *ch, + libxl__ev_child_callback *death) +{ + CTX_LOCK; + int rc; + + if (chldmode_ours(CTX)) { + rc = libxl__sigchld_installhandler(CTX); + if (rc) goto out; + } + + pid_t pid = + CTX->childproc_hooks->fork_replacement + ? CTX->childproc_hooks->fork_replacement(CTX->childproc_user) + : fork(); + if (pid == -1) { + LOGE(ERROR, "fork failed"); + rc = ERROR_FAIL; + goto out; + } + + if (!pid) { + /* woohoo! */ + return 0; /* Yes, CTX is left locked in the child. */ + } + + ch->pid = pid; + ch->callback = death; + LIBXL_LIST_INSERT_HEAD(&CTX->children, ch, entry); + rc = pid; + + out: + perhaps_removehandler(CTX); + CTX_UNLOCK; + return rc; +} + +void libxl_childproc_setmode(libxl_ctx *ctx, const libxl_childproc_hooks *hooks, + void *user) +{ + GC_INIT(ctx); + CTX_LOCK; + + assert(LIBXL_LIST_EMPTY(&CTX->children)); + + if (!hooks) + hooks = &libxl__childproc_default_hooks; + + ctx->childproc_hooks = hooks; + ctx->childproc_user = user; + + switch (ctx->childproc_hooks->chldowner) { + case libxl_sigchld_owner_mainloop: + case libxl_sigchld_owner_libxl: + libxl__sigchld_removehandler(ctx); + break; + case libxl_sigchld_owner_libxl_always: + libxl__sigchld_installhandler(ctx); + break; + default: + abort(); + } + + CTX_UNLOCK; + GC_FREE; +} + +const libxl_childproc_hooks libxl__childproc_default_hooks = { + libxl_sigchld_owner_libxl, 0, 0 +}; + +/* * Local variables: * mode: C * c-basic-offset: 4 diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index 01c9ba9188..d04e856821 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -198,6 +198,19 @@ _hidden libxl__ev_xswatch *libxl__watch_slot_contents(libxl__gc *gc, int slotnum); +typedef struct libxl__ev_child libxl__ev_child; +typedef void libxl__ev_child_callback(libxl__egc *egc, libxl__ev_child*, + pid_t pid, int status); +struct libxl__ev_child { + /* caller should include this in their own struct */ + /* read-only for caller: */ + pid_t pid; /* -1 means unused ("unregistered", ie Idle) */ + libxl__ev_child_callback *callback; + /* remainder is private for libxl__ev_... */ + LIBXL_LIST_ENTRY(struct libxl__ev_child) entry; +}; + + /* * evgen structures, which are the state we use for generating * events for the caller. @@ -306,10 +319,14 @@ struct libxl__ctx { LIBXL_LIST_HEAD(, libxl_evgen_disk_eject) disk_eject_evgens; - /* for callers who reap children willy-nilly; caller must only - * set this after libxl_init and before any other call - or - * may leave them untouched */ + const libxl_childproc_hooks *childproc_hooks; + void *childproc_user; + int sigchld_selfpipe[2]; /* [0]==-1 means handler not installed */ + LIBXL_LIST_HEAD(, libxl__ev_child) children; + + /* This is obsolete and must be removed: */ int (*waitpid_instead)(pid_t pid, int *status, int flags); + libxl_version_info version_info; }; @@ -557,6 +574,36 @@ static inline int libxl__ev_xswatch_isregistered(const libxl__ev_xswatch *xw) /* + * For making subprocesses. This is the only permitted mechanism for + * code in libxl to do so. + * + * In the parent, returns the pid, filling in childw_out. + * In the child, returns 0. + * If it fails, returns a libxl error (all of which are -ve). + * + * The child should go on to exec (or exit) soon. The child may not + * make any further calls to libxl infrastructure, except for memory + * allocation and logging. If the child needs to use xenstore it + * must open its own xs handle and use it directly, rather than via + * the libxl event machinery. + * + * The parent may signal the child but it must not reap it. That will + * be done by the event machinery. death may be NULL in which case + * the child is still reaped but its death is ignored. + * + * It is not possible to "deregister" the child death event source. + * It will generate exactly one event callback; until then the childw + * is Active and may not be reused. + */ +_hidden pid_t libxl__ev_child_fork(libxl__gc *gc, libxl__ev_child *childw_out, + libxl__ev_child_callback *death); +static inline void libxl__ev_child_init(libxl__ev_child *childw_out) + { childw_out->pid = -1; } +static inline int libxl__ev_child_inuse(libxl__ev_child *childw_out) + { return childw_out->pid >= 0; } + + +/* * Other event-handling support provided by the libxl event core to * the rest of libxl. */ @@ -610,6 +657,15 @@ _hidden void libxl__poller_put(libxl_ctx *ctx, libxl__poller *p); * ctx must be locked. */ _hidden void libxl__poller_wakeup(libxl__egc *egc, libxl__poller *p); +/* Internal to fork and child reaping machinery */ +extern const libxl_childproc_hooks libxl__childproc_default_hooks; +int libxl__sigchld_installhandler(libxl_ctx *ctx); /* non-reentrant;logs errs */ +void libxl__sigchld_removehandler(libxl_ctx *ctx); /* non-reentrant */ +int libxl__fork_selfpipe_active(libxl_ctx *ctx); /* returns read fd or -1 */ +void libxl__fork_selfpipe_woken(libxl__egc *egc); +int libxl__self_pipe_wakeup(int fd); /* returns 0 or -1 setting errno */ +int libxl__self_pipe_eatall(int fd); /* returns 0 or -1 setting errno */ + _hidden int libxl__atfork_init(libxl_ctx *ctx); |