FSIF: extend protocol to multi-grant requests and multi-page ring

To improve throughput and inline the stat response.

Signed-off-by: Grzegorz Milos <gm281@cam.ac.uk>
Signed-off-by: Samuel Thibault <samuel.thibault@eu.citrix.com>

4 files changed, 72 insertions, 48 deletions

diff --git a/tools/fs-back/fs-backend.c b/tools/fs-back/fs-backend.c
index 14c48ba967..f0d2758627 100644
--- a/tools/fs-back/fs-backend.c
+++ b/tools/fs-back/fs-backend.c
@@ -147,7 +147,8 @@ moretodo:
             int i;
             struct fs_op *op;
 
-            printf("Got a request at %d\n", cons);
+            printf("Got a request at %d (of %d)\n", 
+                    cons, RING_SIZE(&mount->ring));
             req = RING_GET_REQUEST(&mount->ring, cons);
             printf("Request type=%d\n", req->type); 
             for(i=0;;i++)
@@ -198,6 +199,7 @@ static void handle_connection(int frontend_dom_id, int export_id, char *frontend
     int evt_port;
     pthread_t handling_thread;
     struct fsif_sring *sring;
+    uint32_t dom_ids[MAX_RING_SIZE];
     int i;
 
     printf("Handling connection from dom=%d, for export=%d\n", 
@@ -222,7 +224,7 @@ static void handle_connection(int frontend_dom_id, int export_id, char *frontend
     mount->mount_id = mount_id++;
     xenbus_read_mount_request(mount, frontend);
     printf("Frontend found at: %s (gref=%d, evtchn=%d)\n", 
-            mount->frontend, mount->gref, mount->remote_evtchn);
+            mount->frontend, mount->grefs[0], mount->remote_evtchn);
     xenbus_write_backend_node(mount);
     mount->evth = -1;
     mount->evth = xc_evtchn_open(); 
@@ -235,11 +237,15 @@ static void handle_connection(int frontend_dom_id, int export_id, char *frontend
     mount->gnth = -1;
     mount->gnth = xc_gnttab_open(); 
     assert(mount->gnth != -1);
-    sring = xc_gnttab_map_grant_ref(mount->gnth,
-                                    mount->dom_id,
-                                    mount->gref,
-                                    PROT_READ | PROT_WRITE);
-    BACK_RING_INIT(&mount->ring, sring, XC_PAGE_SIZE);
+    for(i=0; i<mount->shared_ring_size; i++)
+        dom_ids[i] = mount->dom_id;
+    sring = xc_gnttab_map_grant_refs(mount->gnth,
+                                     mount->shared_ring_size,
+                                     dom_ids,
+                                     mount->grefs,
+                                     PROT_READ | PROT_WRITE);
+
+    BACK_RING_INIT(&mount->ring, sring, mount->shared_ring_size * XC_PAGE_SIZE);
     mount->nr_entries = mount->ring.nr_ents; 
     for (i = 0; i < MAX_FDS; i++)
         mount->fds[i] = -1;
diff --git a/tools/fs-back/fs-backend.h b/tools/fs-back/fs-backend.h
index 2999a60f32..b2a6be6f4a 100644
--- a/tools/fs-back/fs-backend.h
+++ b/tools/fs-back/fs-backend.h
@@ -13,6 +13,7 @@
 #define EXPORTS_NODE        ROOT_NODE"/"EXPORTS_SUBNODE
 #define WATCH_NODE          EXPORTS_NODE"/requests"
 #define MAX_FDS             16
+#define MAX_RING_SIZE       16
 
 struct fs_export
 {
@@ -26,6 +27,7 @@ struct fs_request
 {
     int active;
     void *page;                         /* Pointer to mapped grant */
+    int count;
     struct fsif_request req_shadow;
     struct aiocb aiocb; 
 };
@@ -37,11 +39,12 @@ struct fs_mount
     int dom_id;
     char *frontend;
     int mount_id;                     /* = backend id */
-    grant_ref_t gref;
+    grant_ref_t grefs[MAX_RING_SIZE];
     evtchn_port_t remote_evtchn;
     int evth;                         /* Handle to the event channel */
     evtchn_port_t local_evtchn;
     int gnth;
+    int shared_ring_size;             /* in pages */
     struct fsif_back_ring ring;
     int nr_entries;
     struct fs_request *requests;
diff --git a/tools/fs-back/fs-ops.c b/tools/fs-back/fs-ops.c
index de6c4a6456..f8d1f9908f 100644
--- a/tools/fs-back/fs-ops.c
+++ b/tools/fs-back/fs-ops.c
@@ -123,19 +123,24 @@ static void dispatch_file_close(struct fs_mount *mount, struct fsif_request *req
     rsp->ret_val = (uint64_t)ret;
 }
 
+#define MAX_GNTS 16
 static void dispatch_file_read(struct fs_mount *mount, struct fsif_request *req)
 {
     void *buf;
-    int fd;
+    int fd, i, count;
     uint16_t req_id;
     unsigned short priv_id;
     struct fs_request *priv_req;
 
     /* Read the request */
-    buf = xc_gnttab_map_grant_ref(mount->gnth,
-                                  mount->dom_id,
-                                  req->u.fread.gref,
-                                  PROT_WRITE);
+    assert(req->u.fread.len > 0); 
+    count = (req->u.fread.len - 1) / XC_PAGE_SIZE + 1;
+    assert(count <= FSIF_NR_READ_GNTS);
+    buf = xc_gnttab_map_domain_grant_refs(mount->gnth,
+                                          count,
+                                          mount->dom_id,
+                                          req->u.fread.grefs,
+                                          PROT_WRITE);
    
     req_id = req->id;
     printf("File read issued for FD=%d (len=%"PRIu64", offest=%"PRIu64")\n", 
@@ -150,6 +155,7 @@ static void dispatch_file_read(struct fs_mount *mount, struct fsif_request *req)
     printf("Private id is: %d\n", priv_id);
     priv_req = &mount->requests[priv_id];
     priv_req->page = buf;
+    priv_req->count = count;
 
     /* Dispatch AIO read request */
     bzero(&priv_req->aiocb, sizeof(struct aiocb));
@@ -172,7 +178,9 @@ static void end_file_read(struct fs_mount *mount, struct fs_request *priv_req)
     uint16_t req_id;
 
     /* Release the grant */
-    assert(xc_gnttab_munmap(mount->gnth, priv_req->page, 1) == 0);
+    assert(xc_gnttab_munmap(mount->gnth, 
+                            priv_req->page, 
+                            priv_req->count) == 0);
 
     /* Get a response from the ring */
     rsp_idx = mount->ring.rsp_prod_pvt++;
@@ -186,16 +194,20 @@ static void end_file_read(struct fs_mount *mount, struct fs_request *priv_req)
 static void dispatch_file_write(struct fs_mount *mount, struct fsif_request *req)
 {
     void *buf;
-    int fd;
+    int fd, count, i;
     uint16_t req_id;
     unsigned short priv_id;
     struct fs_request *priv_req;
 
     /* Read the request */
-    buf = xc_gnttab_map_grant_ref(mount->gnth,
-                                  mount->dom_id,
-                                  req->u.fwrite.gref,
-                                  PROT_READ);
+    assert(req->u.fwrite.len > 0); 
+    count = (req->u.fwrite.len - 1) / XC_PAGE_SIZE + 1;
+    assert(count <= FSIF_NR_WRITE_GNTS);
+    buf = xc_gnttab_map_domain_grant_refs(mount->gnth,
+                                          count,
+                                          mount->dom_id,
+                                          req->u.fwrite.grefs,
+                                          PROT_READ);
    
     req_id = req->id;
     printf("File write issued for FD=%d (len=%"PRIu64", offest=%"PRIu64")\n", 
@@ -210,6 +222,7 @@ static void dispatch_file_write(struct fs_mount *mount, struct fsif_request *req
     printf("Private id is: %d\n", priv_id);
     priv_req = &mount->requests[priv_id];
     priv_req->page = buf;
+    priv_req->count = count;
 
     /* Dispatch AIO write request */
     bzero(&priv_req->aiocb, sizeof(struct aiocb));
@@ -232,7 +245,9 @@ static void end_file_write(struct fs_mount *mount, struct fs_request *priv_req)
     uint16_t req_id;
 
     /* Release the grant */
-    assert(xc_gnttab_munmap(mount->gnth, priv_req->page, 1) == 0);
+    assert(xc_gnttab_munmap(mount->gnth, 
+                            priv_req->page, 
+                            priv_req->count) == 0);
     
     /* Get a response from the ring */
     rsp_idx = mount->ring.rsp_prod_pvt++;
@@ -252,12 +267,6 @@ static void dispatch_stat(struct fs_mount *mount, struct fsif_request *req)
     RING_IDX rsp_idx;
     fsif_response_t *rsp;
 
-    /* Read the request */
-    buf = xc_gnttab_map_grant_ref(mount->gnth,
-                                  mount->dom_id,
-                                  req->u.fstat.gref,
-                                  PROT_WRITE);
-   
     req_id = req->id;
     if (req->u.fstat.fd < MAX_FDS)
         fd = mount->fds[req->u.fstat.fd];
@@ -273,34 +282,31 @@ static void dispatch_stat(struct fs_mount *mount, struct fsif_request *req)
     /* Stat, and create the response */ 
     ret = fstat(fd, &stat);
     printf("Mode=%o, uid=%d, a_time=%ld\n",
-            stat.st_mode, stat.st_uid, (long)stat.st_atime);
-    buf->stat_mode  = stat.st_mode;
-    buf->stat_uid   = stat.st_uid;
-    buf->stat_gid   = stat.st_gid;
+            stat.st_mode, stat.st_uid, stat.st_atime);
+    
+    /* Get a response from the ring */
+    rsp_idx = mount->ring.rsp_prod_pvt++;
+    printf("Writing response at: idx=%d, id=%d\n", rsp_idx, req_id);
+    rsp = RING_GET_RESPONSE(&mount->ring, rsp_idx);
+    rsp->id = req_id; 
+    rsp->fstat.stat_ret = (uint32_t)ret;
+    rsp->fstat.stat_mode  = stat.st_mode;
+    rsp->fstat.stat_uid   = stat.st_uid;
+    rsp->fstat.stat_gid   = stat.st_gid;
 #ifdef BLKGETSIZE
     if (S_ISBLK(stat.st_mode)) {
 	unsigned long sectors;
 	if (ioctl(fd, BLKGETSIZE, &sectors)) {
 	    perror("getting device size\n");
-	    buf->stat_size = 0;
+	    rsp->fstat.stat_size = 0;
 	} else
-	    buf->stat_size = sectors << 9;
+	    rsp->fstat.stat_size = sectors << 9;
     } else
 #endif
-	buf->stat_size  = stat.st_size;
-    buf->stat_atime = stat.st_atime;
-    buf->stat_mtime = stat.st_mtime;
-    buf->stat_ctime = stat.st_ctime;
-
-    /* Release the grant */
-    assert(xc_gnttab_munmap(mount->gnth, buf, 1) == 0);
-    
-    /* Get a response from the ring */
-    rsp_idx = mount->ring.rsp_prod_pvt++;
-    printf("Writing response at: idx=%d, id=%d\n", rsp_idx, req_id);
-    rsp = RING_GET_RESPONSE(&mount->ring, rsp_idx);
-    rsp->id = req_id; 
-    rsp->ret_val = (uint64_t)ret;
+	rsp->fstat.stat_size  = stat.st_size;
+    rsp->fstat.stat_atime = stat.st_atime;
+    rsp->fstat.stat_mtime = stat.st_mtime;
+    rsp->fstat.stat_ctime = stat.st_ctime;
 }
 
 
diff --git a/tools/fs-back/fs-xenbus.c b/tools/fs-back/fs-xenbus.c
index 552ebf5ca6..6a86e245e0 100644
--- a/tools/fs-back/fs-xenbus.c
+++ b/tools/fs-back/fs-xenbus.c
@@ -113,6 +113,7 @@ void xenbus_read_mount_request(struct fs_mount *mount, char *frontend)
 {
     char node[1024];
     char *s;
+    int i;
 
     assert(xsh != NULL);
 #if 0
@@ -125,10 +126,18 @@ void xenbus_read_mount_request(struct fs_mount *mount, char *frontend)
     s = xs_read(xsh, XBT_NULL, node, NULL);
     assert(strcmp(s, STATE_READY) == 0);
     free(s);
-    snprintf(node, sizeof(node), "%s/ring-ref", frontend);
+    snprintf(node, sizeof(node), "%s/ring-size", frontend);
     s = xs_read(xsh, XBT_NULL, node, NULL);
-    mount->gref = atoi(s);
+    mount->shared_ring_size = atoi(s);
+    assert(mount->shared_ring_size <= MAX_RING_SIZE);
     free(s);
+    for(i=0; i<mount->shared_ring_size; i++)
+    {
+        snprintf(node, sizeof(node), "%s/ring-ref-%d", frontend, i);
+        s = xs_read(xsh, XBT_NULL, node, NULL);
+        mount->grefs[i] = atoi(s);
+        free(s);
+    }
     snprintf(node, sizeof(node), "%s/event-channel", frontend);
     s = xs_read(xsh, XBT_NULL, node, NULL);
     mount->remote_evtchn = atoi(s);