xenstored: Fairly round-robin schedule work across all connections.

Avoids total starvation under some workloads.
Signed-off-by: Keir Fraser <keir@xensource.com>

2 files changed, 60 insertions, 44 deletions

diff --git a/tools/xenstore/talloc.c b/tools/xenstore/talloc.c
index ba189199d7..8bff92b3bc 100644
--- a/tools/xenstore/talloc.c
+++ b/tools/xenstore/talloc.c
@@ -97,6 +97,7 @@ struct talloc_chunk {
 	struct talloc_chunk *next, *prev;
 	struct talloc_chunk *parent, *child;
 	struct talloc_reference_handle *refs;
+	unsigned int null_refs; /* references from null_context */
 	talloc_destructor_t destructor;
 	const char *name;
 	size_t size;
@@ -189,6 +190,7 @@ void *_talloc(const void *context, size_t size)
 	tc->child = NULL;
 	tc->name = NULL;
 	tc->refs = NULL;
+	tc->null_refs = 0;
 
 	if (context) {
 		struct talloc_chunk *parent = talloc_chunk_from_ptr(context);
@@ -225,7 +227,11 @@ void talloc_set_destructor(const void *ptr, int (*destructor)(void *))
 */
 void talloc_increase_ref_count(const void *ptr)
 {
-	talloc_reference(null_context, ptr);
+	struct talloc_chunk *tc;
+	if (ptr == NULL) return;
+
+	tc = talloc_chunk_from_ptr(ptr);
+	tc->null_refs++;
 }
 
 /*
@@ -287,6 +293,11 @@ static int talloc_unreference(const void *context, const void *ptr)
 		context = null_context;
 	}
 
+	if ((context == null_context) && tc->null_refs) {
+		tc->null_refs--;
+		return 0;
+	}
+
 	for (h=tc->refs;h;h=h->next) {
 		struct talloc_chunk *p = talloc_parent_chunk(h);
 		if (p == NULL) {
@@ -539,6 +550,11 @@ int talloc_free(void *ptr)
 
 	tc = talloc_chunk_from_ptr(ptr);
 
+	if (tc->null_refs) {
+		tc->null_refs--;
+		return -1;
+	}
+
 	if (tc->refs) {
 		talloc_reference_destructor(tc->refs);
 		return -1;
diff --git a/tools/xenstore/xenstored_core.c b/tools/xenstore/xenstored_core.c
index f576c1c83f..f62de82b77 100644
--- a/tools/xenstore/xenstored_core.c
+++ b/tools/xenstore/xenstored_core.c
@@ -299,11 +299,15 @@ static void set_fd(int fd, fd_set *set, int *max)
 }
 
 
-static int initialize_set(fd_set *inset, fd_set *outset, int sock, int ro_sock)
+static int initialize_set(fd_set *inset, fd_set *outset, int sock, int ro_sock,
+			  struct timeval **ptimeout)
 {
-	struct connection *i;
+	static struct timeval zero_timeout = { 0 };
+	struct connection *conn;
 	int max = -1;
 
+	*ptimeout = NULL;
+
 	FD_ZERO(inset);
 	FD_ZERO(outset);
 
@@ -314,13 +318,19 @@ static int initialize_set(fd_set *inset, fd_set *outset, int sock, int ro_sock)
 	if (xce_handle != -1)
 		set_fd(xc_evtchn_fd(xce_handle), inset, &max);
 
-	list_for_each_entry(i, &connections, list) {
-		if (i->domain)
-			continue;
-		set_fd(i->fd, inset, &max);
-		if (!list_empty(&i->out_list))
-			FD_SET(i->fd, outset);
+	list_for_each_entry(conn, &connections, list) {
+		if (conn->domain) {
+			if (domain_can_read(conn) ||
+			    (domain_can_write(conn) &&
+			     !list_empty(&conn->out_list)))
+				*ptimeout = &zero_timeout;
+		} else {
+			set_fd(conn->fd, inset, &max);
+			if (!list_empty(&conn->out_list))
+				FD_SET(conn->fd, outset);
+		}
 	}
+
 	return max;
 }
 
@@ -1709,6 +1719,7 @@ int main(int argc, char *argv[])
 	bool no_domain_init = false;
 	const char *pidfile = NULL;
 	int evtchn_fd = -1;
+	struct timeval *timeout;
 
 	while ((opt = getopt_long(argc, argv, "DE:F:HNPS:t:T:RLVW:", options,
 				  NULL)) != -1) {
@@ -1850,17 +1861,16 @@ int main(int argc, char *argv[])
 		evtchn_fd = xc_evtchn_fd(xce_handle);
 
 	/* Get ready to listen to the tools. */
-	max = initialize_set(&inset, &outset, *sock, *ro_sock);
+	max = initialize_set(&inset, &outset, *sock, *ro_sock, &timeout);
 
 	/* Tell the kernel we're up and running. */
 	xenbus_notify_running();
 
 	/* Main loop. */
-	/* FIXME: Rewrite so noone can starve. */
 	for (;;) {
-		struct connection *i;
+		struct connection *conn, *old_conn;
 
-		if (select(max+1, &inset, &outset, NULL, NULL) < 0) {
+		if (select(max+1, &inset, &outset, NULL, timeout) < 0) {
 			if (errno == EINTR)
 				continue;
 			barf_perror("Select failed");
@@ -1882,41 +1892,31 @@ int main(int argc, char *argv[])
 		if (evtchn_fd != -1 && FD_ISSET(evtchn_fd, &inset))
 			handle_event();
 
-		list_for_each_entry(i, &connections, list) {
-			if (i->domain)
-				continue;
-
-			/* Operations can delete themselves or others
-			 * (xs_release): list is not safe after input,
-			 * so break. */
-			if (FD_ISSET(i->fd, &inset)) {
-				handle_input(i);
-				break;
+		conn = list_entry(connections.next, typeof(*conn), list);
+		while (&conn->list != &connections) {
+			talloc_increase_ref_count(conn);
+
+			if (conn->domain) {
+				if (domain_can_read(conn))
+					handle_input(conn);
+				if (domain_can_write(conn) &&
+				    !list_empty(&conn->out_list))
+					handle_output(conn);
+			} else {
+				if (FD_ISSET(conn->fd, &inset))
+					handle_input(conn);
+				if (FD_ISSET(conn->fd, &outset))
+					handle_output(conn);
 			}
-			if (FD_ISSET(i->fd, &outset)) {
-				handle_output(i);
-				break;
-			}
-		}
-
-		/* Handle all possible I/O for domain connections. */
-	more:
-		list_for_each_entry(i, &connections, list) {
-			if (!i->domain)
-				continue;
 
-			if (domain_can_read(i)) {
-				handle_input(i);
-				goto more;
-			}
-
-			if (domain_can_write(i) && !list_empty(&i->out_list)) {
-				handle_output(i);
-				goto more;
-			}
+			old_conn = conn;
+			conn = list_entry(old_conn->list.next,
+					  typeof(*conn), list);
+			talloc_free(old_conn);
 		}
 
-		max = initialize_set(&inset, &outset, *sock, *ro_sock);
+		max = initialize_set(&inset, &outset, *sock, *ro_sock,
+				     &timeout);
 	}
 }