Use memops for mem paging, sharing, and access, instead of domctls

Per page operations in the paging, sharing, and access tracking subsystems are
all implemented with domctls (e.g. a domctl to evict one page, or to share one
page).

Under heavy load, the domctl path reveals a lack of scalability. The domctl
lock serializes dom0's vcpus in the hypervisor. When performing thousands of
per-page operations on dozens of domains, these vcpus will spin in the
hypervisor. Beyond the aggressive locking, an added inefficiency of blocking
vcpus in the domctl lock is that dom0 is prevented from re-scheduling any of
its other work-starved processes.

We retain the domctl interface for setting up and tearing down
paging/sharing/mem access for a domain. But we migrate all the per page
operations to use the memory_op hypercalls (e.g XENMEM_*).

Signed-off-by: Andres Lagar-Cavilla <andres@lagarcavilla>
Signed-off-by: Adin Scannell <adin@scannell.ca>
Acked-by: Tim Deegan <tim@xen.org>
Acked-by: Ian Jackson <ian.jackson@eu.citrix.com>
Committed-by: Tim Deegan <tim@xen.org>

7 files changed, 138 insertions, 137 deletions

diff --git a/tools/libxc/xc_mem_access.c b/tools/libxc/xc_mem_access.c
index 6606878567..3dca355dc1 100644
--- a/tools/libxc/xc_mem_access.c
+++ b/tools/libxc/xc_mem_access.c
@@ -30,7 +30,7 @@ int xc_mem_access_enable(xc_interface *xch, domid_t domain_id,
     return xc_mem_event_control(xch, domain_id,
                                 XEN_DOMCTL_MEM_EVENT_OP_ACCESS_ENABLE,
                                 XEN_DOMCTL_MEM_EVENT_OP_ACCESS,
-                                shared_page, ring_page, INVALID_MFN);
+                                shared_page, ring_page);
 }
 
 int xc_mem_access_disable(xc_interface *xch, domid_t domain_id)
@@ -38,15 +38,15 @@ int xc_mem_access_disable(xc_interface *xch, domid_t domain_id)
     return xc_mem_event_control(xch, domain_id,
                                 XEN_DOMCTL_MEM_EVENT_OP_ACCESS_DISABLE,
                                 XEN_DOMCTL_MEM_EVENT_OP_ACCESS,
-                                NULL, NULL, INVALID_MFN);
+                                NULL, NULL);
 }
 
 int xc_mem_access_resume(xc_interface *xch, domid_t domain_id, unsigned long gfn)
 {
-    return xc_mem_event_control(xch, domain_id,
-                                XEN_DOMCTL_MEM_EVENT_OP_ACCESS_RESUME,
-                                XEN_DOMCTL_MEM_EVENT_OP_ACCESS,
-                                NULL, NULL, gfn);
+    return xc_mem_event_memop(xch, domain_id,
+                                XENMEM_access_op_resume,
+                                XENMEM_access_op,
+                                gfn, NULL);
 }
 
 /*
diff --git a/tools/libxc/xc_mem_event.c b/tools/libxc/xc_mem_event.c
index 80259071b1..a568a52e77 100644
--- a/tools/libxc/xc_mem_event.c
+++ b/tools/libxc/xc_mem_event.c
@@ -24,8 +24,7 @@
 #include "xc_private.h"
 
 int xc_mem_event_control(xc_interface *xch, domid_t domain_id, unsigned int op,
-                         unsigned int mode, void *page,
-                         void *ring_page, unsigned long gfn)
+                         unsigned int mode, void *page, void *ring_page)
 {
     DECLARE_DOMCTL;
 
@@ -34,11 +33,25 @@ int xc_mem_event_control(xc_interface *xch, domid_t domain_id, unsigned int op,
     domctl.u.mem_event_op.op = op;
     domctl.u.mem_event_op.mode = mode;
 
-    domctl.u.mem_event_op.u.shared_addr = (unsigned long)page;
+    domctl.u.mem_event_op.shared_addr = (unsigned long)page;
     domctl.u.mem_event_op.ring_addr = (unsigned long)ring_page;
-
-    domctl.u.mem_event_op.gfn = gfn;
     
     return do_domctl(xch, &domctl);
 }
 
+int xc_mem_event_memop(xc_interface *xch, domid_t domain_id, 
+                        unsigned int op, unsigned int mode,
+                        uint64_t gfn, void *buffer)
+{
+    xen_mem_event_op_t meo;
+
+    memset(&meo, 0, sizeof(meo));
+
+    meo.op      = op;
+    meo.domain  = domain_id;
+    meo.gfn     = gfn;
+    meo.buffer  = (unsigned long) buffer;
+
+    return do_memory_op(xch, mode, &meo, sizeof(meo));
+}
+
diff --git a/tools/libxc/xc_mem_paging.c b/tools/libxc/xc_mem_paging.c
index a108a5c4cf..eaf713388f 100644
--- a/tools/libxc/xc_mem_paging.c
+++ b/tools/libxc/xc_mem_paging.c
@@ -30,7 +30,7 @@ int xc_mem_paging_enable(xc_interface *xch, domid_t domain_id,
     return xc_mem_event_control(xch, domain_id,
                                 XEN_DOMCTL_MEM_EVENT_OP_PAGING_ENABLE,
                                 XEN_DOMCTL_MEM_EVENT_OP_PAGING,
-                                shared_page, ring_page, INVALID_MFN);
+                                shared_page, ring_page);
 }
 
 int xc_mem_paging_disable(xc_interface *xch, domid_t domain_id)
@@ -38,31 +38,31 @@ int xc_mem_paging_disable(xc_interface *xch, domid_t domain_id)
     return xc_mem_event_control(xch, domain_id,
                                 XEN_DOMCTL_MEM_EVENT_OP_PAGING_DISABLE,
                                 XEN_DOMCTL_MEM_EVENT_OP_PAGING,
-                                NULL, NULL, INVALID_MFN);
+                                NULL, NULL);
 }
 
 int xc_mem_paging_nominate(xc_interface *xch, domid_t domain_id, unsigned long gfn)
 {
-    return xc_mem_event_control(xch, domain_id,
-                                XEN_DOMCTL_MEM_EVENT_OP_PAGING_NOMINATE,
-                                XEN_DOMCTL_MEM_EVENT_OP_PAGING,
-                                NULL, NULL, gfn);
+    return xc_mem_event_memop(xch, domain_id,
+                                XENMEM_paging_op_nominate,
+                                XENMEM_paging_op,
+                                gfn, NULL);
 }
 
 int xc_mem_paging_evict(xc_interface *xch, domid_t domain_id, unsigned long gfn)
 {
-    return xc_mem_event_control(xch, domain_id,
-                                XEN_DOMCTL_MEM_EVENT_OP_PAGING_EVICT,
-                                XEN_DOMCTL_MEM_EVENT_OP_PAGING,
-                                NULL, NULL, gfn);
+    return xc_mem_event_memop(xch, domain_id,
+                                XENMEM_paging_op_evict,
+                                XENMEM_paging_op,
+                                gfn, NULL);
 }
 
 int xc_mem_paging_prep(xc_interface *xch, domid_t domain_id, unsigned long gfn)
 {
-    return xc_mem_event_control(xch, domain_id,
-                                XEN_DOMCTL_MEM_EVENT_OP_PAGING_PREP,
-                                XEN_DOMCTL_MEM_EVENT_OP_PAGING,
-                                NULL, NULL, gfn);
+    return xc_mem_event_memop(xch, domain_id,
+                                XENMEM_paging_op_prep,
+                                XENMEM_paging_op,
+                                gfn, NULL);
 }
 
 int xc_mem_paging_load(xc_interface *xch, domid_t domain_id, 
@@ -81,10 +81,10 @@ int xc_mem_paging_load(xc_interface *xch, domid_t domain_id,
     if ( mlock(buffer, XC_PAGE_SIZE) )
         return -1;
         
-    rc = xc_mem_event_control(xch, domain_id,
-                                XEN_DOMCTL_MEM_EVENT_OP_PAGING_PREP,
-                                XEN_DOMCTL_MEM_EVENT_OP_PAGING,
-                                buffer, NULL, gfn);
+    rc = xc_mem_event_memop(xch, domain_id,
+                                XENMEM_paging_op_prep,
+                                XENMEM_paging_op,
+                                gfn, buffer);
 
     old_errno = errno;
     munlock(buffer, XC_PAGE_SIZE);
@@ -95,10 +95,10 @@ int xc_mem_paging_load(xc_interface *xch, domid_t domain_id,
 
 int xc_mem_paging_resume(xc_interface *xch, domid_t domain_id, unsigned long gfn)
 {
-    return xc_mem_event_control(xch, domain_id,
-                                XEN_DOMCTL_MEM_EVENT_OP_PAGING_RESUME,
-                                XEN_DOMCTL_MEM_EVENT_OP_PAGING,
-                                NULL, NULL, gfn);
+    return xc_mem_event_memop(xch, domain_id,
+                                XENMEM_paging_op_resume,
+                                XENMEM_paging_op,
+                                gfn, NULL);
 }
 
 
diff --git a/tools/libxc/xc_memshr.c b/tools/libxc/xc_memshr.c
index 810200c5e4..5866960f38 100644
--- a/tools/libxc/xc_memshr.c
+++ b/tools/libxc/xc_memshr.c
@@ -36,32 +36,38 @@ int xc_memshr_control(xc_interface *xch,
     domctl.interface_version = XEN_DOMCTL_INTERFACE_VERSION;
     domctl.domain = domid;
     op = &(domctl.u.mem_sharing_op);
-    op->op = XEN_DOMCTL_MEM_EVENT_OP_SHARING_CONTROL;
+    op->op = XEN_DOMCTL_MEM_SHARING_CONTROL;
     op->u.enable = enable;
 
     return do_domctl(xch, &domctl);
 }
 
+static int xc_memshr_memop(xc_interface *xch, domid_t domid, 
+                            xen_mem_sharing_op_t *mso)
+{
+    mso->domain = domid;
+
+    return do_memory_op(xch, XENMEM_sharing_op, mso, sizeof(*mso));
+}
+
 int xc_memshr_nominate_gfn(xc_interface *xch,
                            domid_t domid,
                            unsigned long gfn,
                            uint64_t *handle)
 {
-    DECLARE_DOMCTL;
-    struct xen_domctl_mem_sharing_op *op;
-    int ret;
+    int rc;
+    xen_mem_sharing_op_t mso;
 
-    domctl.cmd = XEN_DOMCTL_mem_sharing_op;
-    domctl.interface_version = XEN_DOMCTL_INTERFACE_VERSION;
-    domctl.domain = domid;
-    op = &(domctl.u.mem_sharing_op);
-    op->op = XEN_DOMCTL_MEM_EVENT_OP_SHARING_NOMINATE_GFN;
-    op->u.nominate.u.gfn = gfn;
+    memset(&mso, 0, sizeof(mso));
 
-    ret = do_domctl(xch, &domctl);
-    if(!ret) *handle = op->u.nominate.handle; 
+    mso.op = XENMEM_sharing_op_nominate_gfn;
+    mso.u.nominate.u.gfn = gfn; 
 
-    return ret;
+    rc = xc_memshr_memop(xch, domid, &mso);
+
+    if (!rc) *handle = mso.u.nominate.handle; 
+
+    return rc;
 }
 
 int xc_memshr_nominate_gref(xc_interface *xch,
@@ -69,21 +75,19 @@ int xc_memshr_nominate_gref(xc_interface *xch,
                             grant_ref_t gref,
                             uint64_t *handle)
 {
-    DECLARE_DOMCTL;
-    struct xen_domctl_mem_sharing_op *op;
-    int ret;
+    int rc;
+    xen_mem_sharing_op_t mso;
 
-    domctl.cmd = XEN_DOMCTL_mem_sharing_op;
-    domctl.interface_version = XEN_DOMCTL_INTERFACE_VERSION;
-    domctl.domain = domid;
-    op = &(domctl.u.mem_sharing_op);
-    op->op = XEN_DOMCTL_MEM_EVENT_OP_SHARING_NOMINATE_GREF;
-    op->u.nominate.u.grant_ref = gref;
+    memset(&mso, 0, sizeof(mso));
+
+    mso.op = XENMEM_sharing_op_nominate_gref;
+    mso.u.nominate.u.grant_ref = gref; 
 
-    ret = do_domctl(xch, &domctl);
-    if(!ret) *handle = op->u.nominate.handle; 
+    rc = xc_memshr_memop(xch, domid, &mso);
 
-    return ret;
+    if (!rc) *handle = mso.u.nominate.handle; 
+
+    return rc;
 }
 
 int xc_memshr_share_gfns(xc_interface *xch,
@@ -94,21 +98,19 @@ int xc_memshr_share_gfns(xc_interface *xch,
                          unsigned long client_gfn,
                          uint64_t client_handle)
 {
-    DECLARE_DOMCTL;
-    struct xen_domctl_mem_sharing_op *op;
+    xen_mem_sharing_op_t mso;
 
-    domctl.cmd = XEN_DOMCTL_mem_sharing_op;
-    domctl.interface_version = XEN_DOMCTL_INTERFACE_VERSION;
-    domctl.domain = source_domain;
-    op = &(domctl.u.mem_sharing_op);
-    op->op = XEN_DOMCTL_MEM_EVENT_OP_SHARING_SHARE;
-    op->u.share.source_handle = source_handle;
-    op->u.share.source_gfn    = source_gfn;
-    op->u.share.client_domain = client_domain;
-    op->u.share.client_gfn    = client_gfn;
-    op->u.share.client_handle = client_handle;
+    memset(&mso, 0, sizeof(mso));
 
-    return do_domctl(xch, &domctl);
+    mso.op = XENMEM_sharing_op_share;
+
+    mso.u.share.source_handle = source_handle;
+    mso.u.share.source_gfn    = source_gfn;
+    mso.u.share.client_domain = client_domain;
+    mso.u.share.client_gfn    = client_gfn;
+    mso.u.share.client_handle = client_handle;
+
+    return xc_memshr_memop(xch, source_domain, &mso);
 }
 
 int xc_memshr_share_grefs(xc_interface *xch,
@@ -119,21 +121,19 @@ int xc_memshr_share_grefs(xc_interface *xch,
                           grant_ref_t client_gref,
                           uint64_t client_handle)
 {
-    DECLARE_DOMCTL;
-    struct xen_domctl_mem_sharing_op *op;
+    xen_mem_sharing_op_t mso;
 
-    domctl.cmd = XEN_DOMCTL_mem_sharing_op;
-    domctl.interface_version = XEN_DOMCTL_INTERFACE_VERSION;
-    domctl.domain = source_domain;
-    op = &(domctl.u.mem_sharing_op);
-    op->op = XEN_DOMCTL_MEM_EVENT_OP_SHARING_SHARE;
-    op->u.share.source_handle = source_handle;
-    XEN_DOMCTL_MEM_SHARING_FIELD_MAKE_GREF(op->u.share.source_gfn, source_gref);
-    op->u.share.client_domain = client_domain;
-    XEN_DOMCTL_MEM_SHARING_FIELD_MAKE_GREF(op->u.share.client_gfn, client_gref);
-    op->u.share.client_handle = client_handle;
+    memset(&mso, 0, sizeof(mso));
 
-    return do_domctl(xch, &domctl);
+    mso.op = XENMEM_sharing_op_share;
+
+    mso.u.share.source_handle = source_handle;
+    XENMEM_SHARING_OP_FIELD_MAKE_GREF(mso.u.share.source_gfn, source_gref);
+    mso.u.share.client_domain = client_domain;
+    XENMEM_SHARING_OP_FIELD_MAKE_GREF(mso.u.share.client_gfn, client_gref);
+    mso.u.share.client_handle = client_handle;
+
+    return xc_memshr_memop(xch, source_domain, &mso);
 }
 
 int xc_memshr_add_to_physmap(xc_interface *xch,
@@ -143,86 +143,72 @@ int xc_memshr_add_to_physmap(xc_interface *xch,
                     domid_t client_domain,
                     unsigned long client_gfn)
 {
-    DECLARE_DOMCTL;
-    struct xen_domctl_mem_sharing_op *op;
+    xen_mem_sharing_op_t mso;
 
-    domctl.cmd                  = XEN_DOMCTL_mem_sharing_op;
-    domctl.interface_version    = XEN_DOMCTL_INTERFACE_VERSION;
-    domctl.domain               = source_domain;
-    op = &(domctl.u.mem_sharing_op);
-    op->op = XEN_DOMCTL_MEM_EVENT_OP_SHARING_ADD_PHYSMAP;
-    op->u.share.source_gfn      = source_gfn;
-    op->u.share.source_handle   = source_handle;
-    op->u.share.client_gfn      = client_gfn;
-    op->u.share.client_domain   = client_domain;
+    memset(&mso, 0, sizeof(mso));
 
-    return do_domctl(xch, &domctl);
+    mso.op = XENMEM_sharing_op_add_physmap;
+
+    mso.u.share.source_handle = source_handle;
+    mso.u.share.source_gfn    = source_gfn;
+    mso.u.share.client_domain = client_domain;
+    mso.u.share.client_gfn    = client_gfn;
+
+    return xc_memshr_memop(xch, source_domain, &mso);
 }
 
 int xc_memshr_domain_resume(xc_interface *xch,
                             domid_t domid)
 {
-    DECLARE_DOMCTL;
-    struct xen_domctl_mem_sharing_op *op;
+    xen_mem_sharing_op_t mso;
 
-    domctl.cmd = XEN_DOMCTL_mem_sharing_op;
-    domctl.interface_version = XEN_DOMCTL_INTERFACE_VERSION;
-    domctl.domain = domid;
-    op = &(domctl.u.mem_sharing_op);
-    op->op = XEN_DOMCTL_MEM_EVENT_OP_SHARING_RESUME;
+    memset(&mso, 0, sizeof(mso));
 
-    return do_domctl(xch, &domctl);
+    mso.op = XENMEM_sharing_op_resume;
+
+    return xc_memshr_memop(xch, domid, &mso);
 }
 
 int xc_memshr_debug_gfn(xc_interface *xch,
                         domid_t domid,
                         unsigned long gfn)
 {
-    DECLARE_DOMCTL;
-    struct xen_domctl_mem_sharing_op *op;
+    xen_mem_sharing_op_t mso;
 
-    domctl.cmd = XEN_DOMCTL_mem_sharing_op;
-    domctl.interface_version = XEN_DOMCTL_INTERFACE_VERSION;
-    domctl.domain = domid;
-    op = &(domctl.u.mem_sharing_op);
-    op->op = XEN_DOMCTL_MEM_EVENT_OP_SHARING_DEBUG_GFN;
-    op->u.debug.u.gfn = gfn;
+    memset(&mso, 0, sizeof(mso));
 
-    return do_domctl(xch, &domctl);
+    mso.op = XENMEM_sharing_op_debug_gfn;
+    mso.u.debug.u.gfn = gfn; 
+
+    return xc_memshr_memop(xch, domid, &mso);
 }
 
 int xc_memshr_debug_mfn(xc_interface *xch,
                         domid_t domid,
                         unsigned long mfn)
 {
-    DECLARE_DOMCTL;
-    struct xen_domctl_mem_sharing_op *op;
+    xen_mem_sharing_op_t mso;
 
-    domctl.cmd = XEN_DOMCTL_mem_sharing_op;
-    domctl.interface_version = XEN_DOMCTL_INTERFACE_VERSION;
-    domctl.domain = domid;
-    op = &(domctl.u.mem_sharing_op);
-    op->op = XEN_DOMCTL_MEM_EVENT_OP_SHARING_DEBUG_MFN;
-    op->u.debug.u.mfn = mfn;
+    memset(&mso, 0, sizeof(mso));
 
-    return do_domctl(xch, &domctl);
+    mso.op = XENMEM_sharing_op_debug_mfn;
+    mso.u.debug.u.mfn = mfn; 
+
+    return xc_memshr_memop(xch, domid, &mso);
 }
 
 int xc_memshr_debug_gref(xc_interface *xch,
                          domid_t domid,
                          grant_ref_t gref)
 {
-    DECLARE_DOMCTL;
-    struct xen_domctl_mem_sharing_op *op;
+    xen_mem_sharing_op_t mso;
 
-    domctl.cmd = XEN_DOMCTL_mem_sharing_op;
-    domctl.interface_version = XEN_DOMCTL_INTERFACE_VERSION;
-    domctl.domain = domid;
-    op = &(domctl.u.mem_sharing_op);
-    op->op = XEN_DOMCTL_MEM_EVENT_OP_SHARING_DEBUG_GREF;
-    op->u.debug.u.gref = gref;
+    memset(&mso, 0, sizeof(mso));
 
-    return do_domctl(xch, &domctl);
+    mso.op = XENMEM_sharing_op_debug_gref;
+    mso.u.debug.u.gref = gref; 
+
+    return xc_memshr_memop(xch, domid, &mso);
 }
 
 long xc_sharing_freed_pages(xc_interface *xch)
diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h
index e6fd488af8..371a97f5e8 100644
--- a/tools/libxc/xenctrl.h
+++ b/tools/libxc/xenctrl.h
@@ -1883,8 +1883,10 @@ int xc_tmem_restore_extra(xc_interface *xch, int dom, int fd);
  * mem_event operations
  */
 int xc_mem_event_control(xc_interface *xch, domid_t domain_id, unsigned int op,
-                         unsigned int mode, void *shared_page,
-                          void *ring_page, unsigned long gfn);
+                         unsigned int mode, void *shared_page, void *ring_page);
+int xc_mem_event_memop(xc_interface *xch, domid_t domain_id, 
+                        unsigned int op, unsigned int mode,
+                        uint64_t gfn, void *buffer);
 
 int xc_mem_paging_enable(xc_interface *xch, domid_t domain_id,
                         void *shared_page, void *ring_page);
diff --git a/tools/memshr/interface.c b/tools/memshr/interface.c
index b4d6d4e190..1c39dfa77d 100644
--- a/tools/memshr/interface.c
+++ b/tools/memshr/interface.c
@@ -186,12 +186,12 @@ int memshr_vbd_issue_ro_request(char *buf,
            remove the relevant ones from the map */
         switch(ret)
         {
-            case XEN_DOMCTL_MEM_SHARING_S_HANDLE_INVALID:
+            case XENMEM_SHARING_OP_S_HANDLE_INVALID:
                 ret = blockshr_shrhnd_remove(memshr.blks, source_st, NULL);
                 if(ret) DPRINTF("Could not rm invl s_hnd: %u %"PRId64" %"PRId64"\n", 
                                     source_st.domain, source_st.frame, source_st.handle);
                 break;
-            case XEN_DOMCTL_MEM_SHARING_C_HANDLE_INVALID:
+            case XENMEM_SHARING_OP_C_HANDLE_INVALID:
                 ret = blockshr_shrhnd_remove(memshr.blks, client_st, NULL);
                 if(ret) DPRINTF("Could not rm invl c_hnd: %u %"PRId64" %"PRId64"\n", 
                                     client_st.domain, client_st.frame, client_st.handle);
diff --git a/tools/tests/mem-sharing/memshrtool.c b/tools/tests/mem-sharing/memshrtool.c
index b876619ac9..a5dfc24f94 100644
--- a/tools/tests/mem-sharing/memshrtool.c
+++ b/tools/tests/mem-sharing/memshrtool.c
@@ -34,9 +34,9 @@ static int usage(const char* prog)
     int rc = f; \
     if ( rc < 0 ) { \
         printf("error executing %s: %s\n", #f, \
-                ((errno * -1) == XEN_DOMCTL_MEM_SHARING_C_HANDLE_INVALID) ? \
+                ((errno * -1) == XENMEM_SHARING_OP_S_HANDLE_INVALID) ? \
                 "problem with client handle" :\
-                ((errno * -1) == XEN_DOMCTL_MEM_SHARING_S_HANDLE_INVALID) ? \
+                ((errno * -1) == XENMEM_SHARING_OP_C_HANDLE_INVALID) ? \
                 "problem with source handle" : strerror(errno)); \
         return rc; \
     } \