blktap2: Sort out tapdisk AIO init.

Move event callbacks registration into tapdisk-queue. This should also
obsoletes the dummy pollfd pipe in the synchronous I/O case.

Signed-off-by: Daniel Stodden <daniel.stodden@citrix.com>

6 files changed, 64 insertions, 63 deletions

diff --git a/tools/blktap2/drivers/io-optimize.c b/tools/blktap2/drivers/io-optimize.c
index 5d397652e5..86fd7e61fd 100644
--- a/tools/blktap2/drivers/io-optimize.c
+++ b/tools/blktap2/drivers/io-optimize.c
@@ -51,9 +51,16 @@ void
 opio_free(struct opioctx *ctx)
 {
 	free(ctx->opios);
+	ctx->opios = NULL;
+
 	free(ctx->free_opios);
+	ctx->free_opios = NULL;
+
 	free(ctx->iocb_queue);
+	ctx->iocb_queue = NULL;
+
 	free(ctx->event_queue);
+	ctx->event_queue = NULL;
 }
 
 int
diff --git a/tools/blktap2/drivers/qcow2raw.c b/tools/blktap2/drivers/qcow2raw.c
index 689e7f5cd1..45f20ade19 100644
--- a/tools/blktap2/drivers/qcow2raw.c
+++ b/tools/blktap2/drivers/qcow2raw.c
@@ -101,12 +101,6 @@ static void print_bytes(void *ptr, int length)
     return;
 }
 
-void
-queue_event(event_id_t id, char mode, void *private)
-{
-  tapdisk_complete_tiocbs(&server.aio_queue);
-}
-
 static void debug_output(uint64_t progress, uint64_t size)
 {
         //Output progress every PROGRESS_QUANT 
diff --git a/tools/blktap2/drivers/tapdisk-queue.c b/tools/blktap2/drivers/tapdisk-queue.c
index bc344f67d3..2940ffd2be 100644
--- a/tools/blktap2/drivers/tapdisk-queue.c
+++ b/tools/blktap2/drivers/tapdisk-queue.c
@@ -35,6 +35,7 @@
 #include "tapdisk-log.h"
 #include "tapdisk-queue.h"
 #include "tapdisk-filter.h"
+#include "tapdisk-server.h"
 #include "atomicio.h"
 
 #define WARN(_f, _a...) tlog_write(TLOG_WARN, _f, ##_a)
@@ -46,7 +47,7 @@
  * so that we can concurrently poll on synchronous and async descriptors.
  * This is signalled by passing 1 as the io context to io_setup.
  */
-#define REQUEST_ASYNC_FD 1
+#define REQUEST_ASYNC_FD ((io_context_t)1)
 
 static inline void
 queue_tiocb(struct tqueue *queue, struct tiocb *tiocb)
@@ -220,6 +221,8 @@ io_synchronous_rw(struct tqueue *queue)
 	return split;
 }
 
+static void tapdisk_tiocb_event(event_id_t id, char mode, void *private);
+
 int
 tapdisk_init_queue(struct tqueue *queue, int size,
 		   int sync, struct tfilter *filter)
@@ -232,18 +235,18 @@ tapdisk_init_queue(struct tqueue *queue, int size,
 	queue->sync   = sync;
 	queue->filter = filter;
 
-	if (sync) {
-		/* set up a pipe so we can return
-		 * a poll fd that won't fire. */
-		if (pipe(queue->dummy_pipe))
-			return -errno;
-		queue->poll_fd = queue->dummy_pipe[0];
-	} else {
-		queue->aio_ctx = (io_context_t)REQUEST_ASYNC_FD;
-		queue->poll_fd = io_setup(size, &queue->aio_ctx);
+	queue->event   = -1;
+	queue->aio_ctx = NULL;
 
-		if (queue->poll_fd < 0) {
-			if (queue->poll_fd == -EAGAIN)
+	if (!size)
+		return 0;
+
+	if (!sync) {
+		queue->aio_ctx = REQUEST_ASYNC_FD;
+		queue->poll_fd = io_setup(size, &queue->aio_ctx);
+		err = queue->poll_fd;
+		if (err < 0) {
+			if (err == -EAGAIN)
 				DPRINTF("Couldn't setup AIO context.  If you "
 					"are trying to concurrently use a "
 					"large number of blktap-based disks, "
@@ -256,8 +259,19 @@ tapdisk_init_queue(struct tqueue *queue, int size,
 					"support.  This is probably because "
 					"your kernel does not have the "
 					"aio-poll patch applied.\n");
-			return queue->poll_fd;
+			queue->aio_ctx = NULL;
+			goto fail;
 		}
+
+		queue->event =
+			tapdisk_server_register_event(SCHEDULER_POLL_READ_FD,
+						      queue->poll_fd, 0,
+						      tapdisk_tiocb_event,
+						      queue);
+		err = queue->event;
+		if (err < 0)
+			goto fail;
+
 	}
 
 	err               = -ENOMEM;
@@ -280,14 +294,22 @@ tapdisk_init_queue(struct tqueue *queue, int size,
 void
 tapdisk_free_queue(struct tqueue *queue)
 {
-	if (queue->sync) {
-		close(queue->dummy_pipe[0]);
-		close(queue->dummy_pipe[1]);
-	} else
+	if (queue->event >= 0) {
+		tapdisk_server_unregister_event(queue->event);
+		queue->event = -1;
+	}
+
+	if (queue->aio_ctx) {
 		io_destroy(queue->aio_ctx);
+		queue->aio_ctx = NULL;
+	}
 
 	free(queue->iocbs);
+	queue->iocbs = NULL;
+
 	free(queue->aio_events);
+	queue->aio_events = NULL;
+
 	opio_free(&queue->opioctx);
 }
 
@@ -390,7 +412,7 @@ tapdisk_submit_all_tiocbs(struct tqueue *queue)
 	return submitted;
 }
 
-int
+static void
 tapdisk_complete_tiocbs(struct tqueue *queue)
 {
 	int i, ret, split;
@@ -415,8 +437,13 @@ tapdisk_complete_tiocbs(struct tqueue *queue)
 	}
 
 	queue_deferred_tiocbs(queue);
+}
 
-	return split;
+static void
+tapdisk_tiocb_event(event_id_t id, char mode, void *private)
+{
+	struct tqueue *queue = private;
+	tapdisk_complete_tiocbs(queue);
 }
 
 /*
diff --git a/tools/blktap2/drivers/tapdisk-queue.h b/tools/blktap2/drivers/tapdisk-queue.h
index 40ff88669c..dd12c65498 100644
--- a/tools/blktap2/drivers/tapdisk-queue.h
+++ b/tools/blktap2/drivers/tapdisk-queue.h
@@ -32,6 +32,7 @@
 #include <libaio.h>
 
 #include "io-optimize.h"
+#include "scheduler.h"
 
 struct tiocb;
 struct tfilter;
@@ -57,9 +58,9 @@ struct tqueue {
 	int                   sync;
 
 	int                   poll_fd;
+	event_id_t	      event;
 	io_context_t          aio_ctx;
 	struct opioctx        opioctx;
-	int                   dummy_pipe[2];
 
 	int                   queued;
 	struct iocb         **iocbs;
@@ -104,7 +105,6 @@ void tapdisk_debug_queue(struct tqueue *);
 void tapdisk_queue_tiocb(struct tqueue *, struct tiocb *);
 int tapdisk_submit_tiocbs(struct tqueue *);
 int tapdisk_submit_all_tiocbs(struct tqueue *);
-int tapdisk_complete_tiocbs(struct tqueue *);
 int tapdisk_cancel_tiocbs(struct tqueue *);
 int tapdisk_cancel_all_tiocbs(struct tqueue *);
 void tapdisk_prep_tiocb(struct tiocb *, int, int, char *, size_t,
diff --git a/tools/blktap2/drivers/tapdisk-server.c b/tools/blktap2/drivers/tapdisk-server.c
index 4e74e30a48..c26980dcc4 100644
--- a/tools/blktap2/drivers/tapdisk-server.c
+++ b/tools/blktap2/drivers/tapdisk-server.c
@@ -221,19 +221,6 @@ tapdisk_server_send_error(const char *message)
 		tapdisk_ipc_write_error(&vbd->ipc, message);
 }
 
-static void
-tapdisk_server_aio_queue_event(event_id_t id, char mode, void *private)
-{
-	tapdisk_complete_tiocbs(&server.aio_queue);
-}
-
-static void
-tapdisk_server_free_aio_queue(void)
-{
-	tapdisk_server_unregister_event(server.aio_queue_event_id);
-	tapdisk_free_queue(&server.aio_queue);
-}
-
 static int
 tapdisk_server_init_ipc(const char *read, const char *write)
 {
@@ -247,34 +234,21 @@ tapdisk_server_close_ipc(void)
 }
 
 static int
-tapdisk_server_initialize_aio_queue(void)
+tapdisk_server_init_aio(void)
 {
-	int err;
-	event_id_t id;
-
-	err = tapdisk_init_queue(&server.aio_queue,
-				 TAPDISK_TIOCBS, 0, NULL);
-	if (err)
-		return err;
-
-	id = tapdisk_server_register_event(SCHEDULER_POLL_READ_FD,
-					   server.aio_queue.poll_fd, 0,
-					   tapdisk_server_aio_queue_event,
-					   NULL);
-	if (id < 0) {
-		tapdisk_free_queue(&server.aio_queue);
-		return id;
-	}
-
-	server.aio_queue_event_id = id;
+	return tapdisk_init_queue(&server.aio_queue, TAPDISK_TIOCBS, 0, NULL);
+}
 
-	return 0;
+static void
+tapdisk_server_close_aio(void)
+{
+	tapdisk_free_queue(&server.aio_queue);
 }
 
 static void
 tapdisk_server_close(void)
 {
-	tapdisk_server_free_aio_queue();
+	tapdisk_server_close_aio();
 	tapdisk_server_close_ipc();
 }
 
@@ -341,7 +315,7 @@ tapdisk_server_initialize(const char *read, const char *write)
 	if (err)
 		goto fail;
 
-	err = tapdisk_server_initialize_aio_queue();
+	err = tapdisk_server_init_aio();
 	if (err)
 		goto fail;
 
diff --git a/tools/blktap2/drivers/tapdisk-server.h b/tools/blktap2/drivers/tapdisk-server.h
index ffdc579183..53a9f82858 100644
--- a/tools/blktap2/drivers/tapdisk-server.h
+++ b/tools/blktap2/drivers/tapdisk-server.h
@@ -58,7 +58,6 @@ typedef struct tapdisk_server {
 	struct list_head             vbds;
 	scheduler_t                  scheduler;
 	struct tqueue                aio_queue;
-	event_id_t                   aio_queue_event_id;
 } tapdisk_server_t;
 
 #endif