diff options
author | Keir Fraser <keir.fraser@citrix.com> | 2010-01-22 10:59:03 +0000 |
---|---|---|
committer | Keir Fraser <keir.fraser@citrix.com> | 2010-01-22 10:59:03 +0000 |
commit | d6aaa9ee0f8ba5d2d8ff1187b05ed9becee0b40c (patch) | |
tree | 3b69d194167117b66f08f86321756f58703a44e7 | |
parent | 379e63ed3da8f6d874d9bc5d6fa05a85afb60238 (diff) | |
download | xen-d6aaa9ee0f8ba5d2d8ff1187b05ed9becee0b40c.tar.gz xen-d6aaa9ee0f8ba5d2d8ff1187b05ed9becee0b40c.tar.bz2 xen-d6aaa9ee0f8ba5d2d8ff1187b05ed9becee0b40c.zip |
libxc: New hcall_buf_{prep,release} pre-mlock interface
Allow certain performance-critical hypercall wrappers to register data
buffers via a new interface which allows them to be 'bounced' into a
pre-mlock'ed page-sized per-thread data area. This saves the cost of
mlock/munlock on every such hypercall, which can be very expensive on
modern kernels.
Signed-off-by: Keir Fraser <keir.fraser@citrix.com>
-rw-r--r-- | tools/libxc/xc_domain_restore.c | 2 | ||||
-rw-r--r-- | tools/libxc/xc_domain_save.c | 6 | ||||
-rw-r--r-- | tools/libxc/xc_misc.c | 50 | ||||
-rw-r--r-- | tools/libxc/xc_physdev.c | 6 | ||||
-rw-r--r-- | tools/libxc/xc_private.c | 135 | ||||
-rw-r--r-- | tools/libxc/xc_private.h | 46 | ||||
-rw-r--r-- | tools/libxc/xg_private.c | 16 | ||||
-rw-r--r-- | tools/libxc/xg_private.h | 2 |
8 files changed, 181 insertions, 82 deletions
diff --git a/tools/libxc/xc_domain_restore.c b/tools/libxc/xc_domain_restore.c index 23a73730c3..4c04bf8a53 100644 --- a/tools/libxc/xc_domain_restore.c +++ b/tools/libxc/xc_domain_restore.c @@ -1424,7 +1424,7 @@ int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom, ctx->p2m = calloc(dinfo->p2m_size, sizeof(xen_pfn_t)); pfn_type = calloc(dinfo->p2m_size, sizeof(unsigned long)); - region_mfn = xg_memalign(PAGE_SIZE, ROUNDUP( + region_mfn = xc_memalign(PAGE_SIZE, ROUNDUP( MAX_BATCH_SIZE * sizeof(xen_pfn_t), PAGE_SHIFT)); if ( (ctx->p2m == NULL) || (pfn_type == NULL) || diff --git a/tools/libxc/xc_domain_save.c b/tools/libxc/xc_domain_save.c index c26330dc9b..aa9748a7e5 100644 --- a/tools/libxc/xc_domain_save.c +++ b/tools/libxc/xc_domain_save.c @@ -1012,9 +1012,9 @@ int xc_domain_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters, sent_last_iter = dinfo->p2m_size; /* Setup to_send / to_fix and to_skip bitmaps */ - to_send = xg_memalign(PAGE_SIZE, ROUNDUP(BITMAP_SIZE, PAGE_SHIFT)); + to_send = xc_memalign(PAGE_SIZE, ROUNDUP(BITMAP_SIZE, PAGE_SHIFT)); to_fix = calloc(1, BITMAP_SIZE); - to_skip = xg_memalign(PAGE_SIZE, ROUNDUP(BITMAP_SIZE, PAGE_SHIFT)); + to_skip = xc_memalign(PAGE_SIZE, ROUNDUP(BITMAP_SIZE, PAGE_SHIFT)); if ( !to_send || !to_fix || !to_skip ) { @@ -1056,7 +1056,7 @@ int xc_domain_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters, analysis_phase(xc_handle, dom, ctx, to_skip, 0); - pfn_type = xg_memalign(PAGE_SIZE, ROUNDUP( + pfn_type = xc_memalign(PAGE_SIZE, ROUNDUP( MAX_BATCH_SIZE * sizeof(*pfn_type), PAGE_SHIFT)); pfn_batch = calloc(MAX_BATCH_SIZE, sizeof(*pfn_batch)); if ( (pfn_type == NULL) || (pfn_batch == NULL) ) diff --git a/tools/libxc/xc_misc.c b/tools/libxc/xc_misc.c index e237b7862d..44039b03be 100644 --- a/tools/libxc/xc_misc.c +++ b/tools/libxc/xc_misc.c @@ -175,29 +175,29 @@ int xc_hvm_set_pci_intx_level( unsigned int level) { DECLARE_HYPERCALL; - struct xen_hvm_set_pci_intx_level arg; + struct xen_hvm_set_pci_intx_level _arg, *arg = &_arg; int rc; - hypercall.op = __HYPERVISOR_hvm_op; - hypercall.arg[0] = HVMOP_set_pci_intx_level; - hypercall.arg[1] = (unsigned long)&arg; - - arg.domid = dom; - arg.domain = domain; - arg.bus = bus; - arg.device = device; - arg.intx = intx; - arg.level = level; - - if ( (rc = lock_pages(&arg, sizeof(arg))) != 0 ) + if ( (rc = hcall_buf_prep((void **)&arg, sizeof(*arg))) != 0 ) { PERROR("Could not lock memory"); return rc; } + hypercall.op = __HYPERVISOR_hvm_op; + hypercall.arg[0] = HVMOP_set_pci_intx_level; + hypercall.arg[1] = (unsigned long)arg; + + arg->domid = dom; + arg->domain = domain; + arg->bus = bus; + arg->device = device; + arg->intx = intx; + arg->level = level; + rc = do_xen_hypercall(xc_handle, &hypercall); - unlock_pages(&arg, sizeof(arg)); + hcall_buf_release((void **)&arg, sizeof(*arg)); return rc; } @@ -208,26 +208,26 @@ int xc_hvm_set_isa_irq_level( unsigned int level) { DECLARE_HYPERCALL; - struct xen_hvm_set_isa_irq_level arg; + struct xen_hvm_set_isa_irq_level _arg, *arg = &_arg; int rc; - hypercall.op = __HYPERVISOR_hvm_op; - hypercall.arg[0] = HVMOP_set_isa_irq_level; - hypercall.arg[1] = (unsigned long)&arg; - - arg.domid = dom; - arg.isa_irq = isa_irq; - arg.level = level; - - if ( (rc = lock_pages(&arg, sizeof(arg))) != 0 ) + if ( (rc = hcall_buf_prep((void **)&arg, sizeof(*arg))) != 0 ) { PERROR("Could not lock memory"); return rc; } + hypercall.op = __HYPERVISOR_hvm_op; + hypercall.arg[0] = HVMOP_set_isa_irq_level; + hypercall.arg[1] = (unsigned long)arg; + + arg->domid = dom; + arg->isa_irq = isa_irq; + arg->level = level; + rc = do_xen_hypercall(xc_handle, &hypercall); - unlock_pages(&arg, sizeof(arg)); + hcall_buf_release((void **)&arg, sizeof(*arg)); return rc; } diff --git a/tools/libxc/xc_physdev.c b/tools/libxc/xc_physdev.c index ec0ebbeb76..00ac829afa 100644 --- a/tools/libxc/xc_physdev.c +++ b/tools/libxc/xc_physdev.c @@ -36,7 +36,7 @@ int xc_physdev_map_pirq(int xc_handle, map.index = index; map.pirq = *pirq; - rc = do_physdev_op(xc_handle, PHYSDEVOP_map_pirq, &map); + rc = do_physdev_op(xc_handle, PHYSDEVOP_map_pirq, &map, sizeof(map)); if ( !rc ) *pirq = map.pirq; @@ -68,7 +68,7 @@ int xc_physdev_map_pirq_msi(int xc_handle, map.entry_nr = entry_nr; map.table_base = table_base; - rc = do_physdev_op(xc_handle, PHYSDEVOP_map_pirq, &map); + rc = do_physdev_op(xc_handle, PHYSDEVOP_map_pirq, &map, sizeof(map)); if ( !rc ) *pirq = map.pirq; @@ -86,7 +86,7 @@ int xc_physdev_unmap_pirq(int xc_handle, unmap.domid = domid; unmap.pirq = pirq; - rc = do_physdev_op(xc_handle, PHYSDEVOP_unmap_pirq, &unmap); + rc = do_physdev_op(xc_handle, PHYSDEVOP_unmap_pirq, &unmap, sizeof(unmap)); return rc; } diff --git a/tools/libxc/xc_private.c b/tools/libxc/xc_private.c index 7bde4bbe04..aaa1a39d87 100644 --- a/tools/libxc/xc_private.c +++ b/tools/libxc/xc_private.c @@ -8,6 +8,9 @@ #include "xc_private.h" #include "xg_private.h" #include <stdarg.h> +#include <stdlib.h> +#include <malloc.h> +#include <unistd.h> #include <pthread.h> static pthread_key_t last_error_pkey; @@ -126,28 +129,120 @@ void xc_set_error(int code, const char *fmt, ...) } } +#ifdef __sun__ + +int lock_pages(void *addr, size_t len) { return 0; } +void unlock_pages(void *addr, size_t len) { } + +int hcall_buf_prep(void **addr, size_t len) { return 0; } +void hcall_buf_release(void **addr, size_t len) { } + +#else /* !__sun__ */ + int lock_pages(void *addr, size_t len) { - int e = 0; -#ifndef __sun__ + int e; void *laddr = (void *)((unsigned long)addr & PAGE_MASK); size_t llen = (len + ((unsigned long)addr - (unsigned long)laddr) + PAGE_SIZE - 1) & PAGE_MASK; e = mlock(laddr, llen); -#endif return e; } void unlock_pages(void *addr, size_t len) { -#ifndef __sun__ void *laddr = (void *)((unsigned long)addr & PAGE_MASK); size_t llen = (len + ((unsigned long)addr - (unsigned long)laddr) + PAGE_SIZE - 1) & PAGE_MASK; safe_munlock(laddr, llen); -#endif } +static pthread_key_t hcall_buf_pkey; +static pthread_once_t hcall_buf_pkey_once = PTHREAD_ONCE_INIT; +struct hcall_buf { + void *buf; + void *oldbuf; +}; + +static void _xc_clean_hcall_buf(void *m) +{ + struct hcall_buf *hcall_buf = m; + + if ( hcall_buf ) + { + if ( hcall_buf->buf ) + { + unlock_pages(hcall_buf->buf, PAGE_SIZE); + free(hcall_buf->buf); + } + + free(hcall_buf); + } + + pthread_setspecific(hcall_buf_pkey, NULL); +} + +static void _xc_init_hcall_buf(void) +{ + pthread_key_create(&hcall_buf_pkey, _xc_clean_hcall_buf); +} + +int hcall_buf_prep(void **addr, size_t len) +{ + struct hcall_buf *hcall_buf; + + pthread_once(&hcall_buf_pkey_once, _xc_init_hcall_buf); + + hcall_buf = pthread_getspecific(hcall_buf_pkey); + if ( !hcall_buf ) + { + hcall_buf = calloc(1, sizeof(*hcall_buf)); + if ( !hcall_buf ) + goto out; + pthread_setspecific(hcall_buf_pkey, hcall_buf); + } + + if ( !hcall_buf->buf ) + { + hcall_buf->buf = xc_memalign(PAGE_SIZE, PAGE_SIZE); + if ( !hcall_buf->buf || lock_pages(hcall_buf->buf, PAGE_SIZE) ) + { + free(hcall_buf->buf); + hcall_buf->buf = NULL; + goto out; + } + } + + if ( (len < PAGE_SIZE) && !hcall_buf->oldbuf ) + { + memcpy(hcall_buf->buf, *addr, len); + hcall_buf->oldbuf = *addr; + *addr = hcall_buf->buf; + return 0; + } + + out: + return lock_pages(*addr, len); +} + +void hcall_buf_release(void **addr, size_t len) +{ + struct hcall_buf *hcall_buf = pthread_getspecific(hcall_buf_pkey); + + if ( hcall_buf && (hcall_buf->buf == *addr) ) + { + memcpy(hcall_buf->oldbuf, *addr, len); + *addr = hcall_buf->oldbuf; + hcall_buf->oldbuf = NULL; + } + else + { + unlock_pages(*addr, len); + } +} + +#endif + /* NB: arr must be locked */ int xc_get_pfn_type_batch(int xc_handle, uint32_t dom, unsigned int num, xen_pfn_t *arr) @@ -169,21 +264,21 @@ int xc_mmuext_op( DECLARE_HYPERCALL; long ret = -EINVAL; + if ( hcall_buf_prep((void **)&op, nr_ops*sizeof(*op)) != 0 ) + { + PERROR("Could not lock memory for Xen hypercall"); + goto out1; + } + hypercall.op = __HYPERVISOR_mmuext_op; hypercall.arg[0] = (unsigned long)op; hypercall.arg[1] = (unsigned long)nr_ops; hypercall.arg[2] = (unsigned long)0; hypercall.arg[3] = (unsigned long)dom; - if ( lock_pages(op, nr_ops*sizeof(*op)) != 0 ) - { - PERROR("Could not lock memory for Xen hypercall"); - goto out1; - } - ret = do_xen_hypercall(xc_handle, &hypercall); - unlock_pages(op, nr_ops*sizeof(*op)); + hcall_buf_release((void **)&op, nr_ops*sizeof(*op)); out1: return ret; @@ -656,6 +751,22 @@ int xc_ffs64(uint64_t x) return l ? xc_ffs32(l) : h ? xc_ffs32(h) + 32 : 0; } +void *xc_memalign(size_t alignment, size_t size) +{ +#if defined(_POSIX_C_SOURCE) && !defined(__sun__) + int ret; + void *ptr; + ret = posix_memalign(&ptr, alignment, size); + if (ret != 0) + return NULL; + return ptr; +#elif defined(__NetBSD__) || defined(__OpenBSD__) + return valloc(size); +#else + return memalign(alignment, size); +#endif +} + /* * Local variables: * mode: C diff --git a/tools/libxc/xc_private.h b/tools/libxc/xc_private.h index dc88add9c8..fba384cc73 100644 --- a/tools/libxc/xc_private.h +++ b/tools/libxc/xc_private.h @@ -78,9 +78,14 @@ void xc_set_error(int code, const char *fmt, ...); #define PERROR(_m, _a...) xc_set_error(XC_INTERNAL_ERROR, _m " (%d = %s)", \ ## _a , errno, safe_strerror(errno)) +void *xc_memalign(size_t alignment, size_t size); + int lock_pages(void *addr, size_t len); void unlock_pages(void *addr, size_t len); +int hcall_buf_prep(void **addr, size_t len); +void hcall_buf_release(void **addr, size_t len); + static inline void safe_munlock(const void *addr, size_t len) { int saved_errno = errno; @@ -101,21 +106,22 @@ static inline int do_xen_version(int xc_handle, int cmd, void *dest) return do_xen_hypercall(xc_handle, &hypercall); } -static inline int do_physdev_op(int xc_handle, int cmd, void *op) +static inline int do_physdev_op(int xc_handle, int cmd, void *op, size_t len) { int ret = -1; DECLARE_HYPERCALL; - hypercall.op = __HYPERVISOR_physdev_op; - hypercall.arg[0] = (unsigned long) cmd; - hypercall.arg[1] = (unsigned long) op; - if ( lock_pages(op, sizeof(*op)) != 0 ) + if ( hcall_buf_prep(&op, len) != 0 ) { PERROR("Could not lock memory for Xen hypercall"); goto out1; } + hypercall.op = __HYPERVISOR_physdev_op; + hypercall.arg[0] = (unsigned long) cmd; + hypercall.arg[1] = (unsigned long) op; + if ( (ret = do_xen_hypercall(xc_handle, &hypercall)) < 0 ) { if ( errno == EACCES ) @@ -123,7 +129,7 @@ static inline int do_physdev_op(int xc_handle, int cmd, void *op) " rebuild the user-space tool set?\n"); } - unlock_pages(op, sizeof(*op)); + hcall_buf_release(&op, len); out1: return ret; @@ -134,17 +140,17 @@ static inline int do_domctl(int xc_handle, struct xen_domctl *domctl) int ret = -1; DECLARE_HYPERCALL; - domctl->interface_version = XEN_DOMCTL_INTERFACE_VERSION; - - hypercall.op = __HYPERVISOR_domctl; - hypercall.arg[0] = (unsigned long)domctl; - - if ( lock_pages(domctl, sizeof(*domctl)) != 0 ) + if ( hcall_buf_prep((void **)&domctl, sizeof(*domctl)) != 0 ) { PERROR("Could not lock memory for Xen hypercall"); goto out1; } + domctl->interface_version = XEN_DOMCTL_INTERFACE_VERSION; + + hypercall.op = __HYPERVISOR_domctl; + hypercall.arg[0] = (unsigned long)domctl; + if ( (ret = do_xen_hypercall(xc_handle, &hypercall)) < 0 ) { if ( errno == EACCES ) @@ -152,7 +158,7 @@ static inline int do_domctl(int xc_handle, struct xen_domctl *domctl) " rebuild the user-space tool set?\n"); } - unlock_pages(domctl, sizeof(*domctl)); + hcall_buf_release((void **)&domctl, sizeof(*domctl)); out1: return ret; @@ -163,17 +169,17 @@ static inline int do_sysctl(int xc_handle, struct xen_sysctl *sysctl) int ret = -1; DECLARE_HYPERCALL; - sysctl->interface_version = XEN_SYSCTL_INTERFACE_VERSION; - - hypercall.op = __HYPERVISOR_sysctl; - hypercall.arg[0] = (unsigned long)sysctl; - - if ( lock_pages(sysctl, sizeof(*sysctl)) != 0 ) + if ( hcall_buf_prep((void **)&sysctl, sizeof(*sysctl)) != 0 ) { PERROR("Could not lock memory for Xen hypercall"); goto out1; } + sysctl->interface_version = XEN_SYSCTL_INTERFACE_VERSION; + + hypercall.op = __HYPERVISOR_sysctl; + hypercall.arg[0] = (unsigned long)sysctl; + if ( (ret = do_xen_hypercall(xc_handle, &hypercall)) < 0 ) { if ( errno == EACCES ) @@ -181,7 +187,7 @@ static inline int do_sysctl(int xc_handle, struct xen_sysctl *sysctl) " rebuild the user-space tool set?\n"); } - unlock_pages(sysctl, sizeof(*sysctl)); + hcall_buf_release((void **)&sysctl, sizeof(*sysctl)); out1: return ret; diff --git a/tools/libxc/xg_private.c b/tools/libxc/xg_private.c index 4dc3a5c66f..457001ce25 100644 --- a/tools/libxc/xg_private.c +++ b/tools/libxc/xg_private.c @@ -183,22 +183,6 @@ __attribute__((weak)) return -1; } -void *xg_memalign(size_t alignment, size_t size) -{ -#if defined(_POSIX_C_SOURCE) && !defined(__sun__) - int ret; - void *ptr; - ret = posix_memalign(&ptr, alignment, size); - if (ret != 0) - return NULL; - return ptr; -#elif defined(__NetBSD__) || defined(__OpenBSD__) - return valloc(size); -#else - return memalign(alignment, size); -#endif -} - /* * Local variables: * mode: C diff --git a/tools/libxc/xg_private.h b/tools/libxc/xg_private.h index aef011dd2a..a15dd91825 100644 --- a/tools/libxc/xg_private.h +++ b/tools/libxc/xg_private.h @@ -177,6 +177,4 @@ struct domain_info_context { int pin_table(int xc_handle, unsigned int type, unsigned long mfn, domid_t dom); -void *xg_memalign(size_t alignment, size_t size); - #endif /* XG_PRIVATE_H */ |