From 88910061ec615b2d05e721a82c37139e05df0712 Mon Sep 17 00:00:00 2001 From: David Vrabel Date: Mon, 14 Oct 2013 10:22:07 +0200 Subject: evtchn: add FIFO-based event channel hypercalls and port ops Add the implementation for the FIFO-based event channel ABI. The new hypercall sub-ops (EVTCHNOP_init_control, EVTCHNOP_expand_array) and the required evtchn_ops (set_pending, unmask, etc.). Signed-off-by: David Vrabel Reviewed-by: Jan Beulich Acked-by: Keir Fraser --- xen/common/Makefile | 1 + xen/common/event_channel.c | 21 ++ xen/common/event_fifo.c | 445 +++++++++++++++++++++++++++++++++++++++++++ xen/include/xen/event_fifo.h | 47 +++++ xen/include/xen/sched.h | 6 +- 5 files changed, 519 insertions(+), 1 deletion(-) create mode 100644 xen/common/event_fifo.c create mode 100644 xen/include/xen/event_fifo.h diff --git a/xen/common/Makefile b/xen/common/Makefile index f6c473a44a..686f7a1130 100644 --- a/xen/common/Makefile +++ b/xen/common/Makefile @@ -7,6 +7,7 @@ obj-y += domctl.o obj-y += domain.o obj-y += event_2l.o obj-y += event_channel.o +obj-y += event_fifo.o obj-y += grant_table.o obj-y += irq.o obj-y += kernel.o diff --git a/xen/common/event_channel.c b/xen/common/event_channel.c index 340bf32583..0c0bbe494f 100644 --- a/xen/common/event_channel.c +++ b/xen/common/event_channel.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -1085,6 +1086,24 @@ long do_event_channel_op(int cmd, XEN_GUEST_HANDLE_PARAM(void) arg) break; } + case EVTCHNOP_init_control: { + struct evtchn_init_control init_control; + if ( copy_from_guest(&init_control, arg, 1) != 0 ) + return -EFAULT; + rc = evtchn_fifo_init_control(&init_control); + if ( !rc && __copy_to_guest(arg, &init_control, 1) ) + rc = -EFAULT; + break; + } + + case EVTCHNOP_expand_array: { + struct evtchn_expand_array expand_array; + if ( copy_from_guest(&expand_array, arg, 1) != 0 ) + return -EFAULT; + rc = evtchn_fifo_expand_array(&expand_array); + break; + } + case EVTCHNOP_set_priority: { struct evtchn_set_priority set_priority; if ( copy_from_guest(&set_priority, arg, 1) != 0 ) @@ -1269,6 +1288,8 @@ void evtchn_destroy(struct domain *d) spin_unlock(&d->event_lock); clear_global_virq_handlers(d); + + evtchn_fifo_destroy(d); } diff --git a/xen/common/event_fifo.c b/xen/common/event_fifo.c new file mode 100644 index 0000000000..eb0b97c9a5 --- /dev/null +++ b/xen/common/event_fifo.c @@ -0,0 +1,445 @@ +/* + * FIFO event channel management. + * + * Copyright (C) 2013 Citrix Systems R&D Ltd. + * + * This source code is licensed under the GNU General Public License, + * Version 2 or later. See the file COPYING for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static inline event_word_t *evtchn_fifo_word_from_port(struct domain *d, + unsigned int port) +{ + unsigned int p, w; + + if ( unlikely(port >= d->evtchn_fifo->num_evtchns) ) + return NULL; + + p = port / EVTCHN_FIFO_EVENT_WORDS_PER_PAGE; + w = port % EVTCHN_FIFO_EVENT_WORDS_PER_PAGE; + + return d->evtchn_fifo->event_array[p] + w; +} + +static bool_t evtchn_fifo_set_link(event_word_t *word, uint32_t link) +{ + event_word_t n, o, w; + + w = *word; + + do { + if ( !(w & (1 << EVTCHN_FIFO_LINKED)) ) + return 0; + o = w; + n = (w & ~EVTCHN_FIFO_LINK_MASK) | link; + } while ( (w = cmpxchg(word, o, n)) != o ); + + return 1; +} + +static void evtchn_fifo_set_pending(struct vcpu *v, struct evtchn *evtchn) +{ + struct domain *d = v->domain; + unsigned int port; + event_word_t *word; + struct evtchn_fifo_queue *q; + unsigned long flags; + bool_t was_pending; + + port = evtchn->port; + word = evtchn_fifo_word_from_port(d, port); + if ( unlikely(!word) ) + return; + + /* + * No locking around getting the queue. This may race with + * changing the priority but we are allowed to signal the event + * once on the old priority. + */ + q = &v->evtchn_fifo->queue[evtchn->priority]; + + was_pending = test_and_set_bit(EVTCHN_FIFO_PENDING, word); + + /* + * Link the event if it unmasked and not already linked. + */ + if ( !test_bit(EVTCHN_FIFO_MASKED, word) + && !test_and_set_bit(EVTCHN_FIFO_LINKED, word) ) + { + event_word_t *tail_word; + bool_t linked = 0; + + spin_lock_irqsave(&q->lock, flags); + + /* + * Atomically link the tail to port iff the tail is linked. + * If the tail is unlinked the queue is empty. + * + * If port is the same as tail, the queue is empty but q->tail + * will appear linked as we just set LINKED above. + * + * If the queue is empty (i.e., we haven't linked to the new + * event), head must be updated. + */ + if ( port != q->tail ) + { + tail_word = evtchn_fifo_word_from_port(d, q->tail); + linked = evtchn_fifo_set_link(tail_word, port); + } + if ( !linked ) + write_atomic(q->head, port); + q->tail = port; + + spin_unlock_irqrestore(&q->lock, flags); + + if ( !test_and_set_bit(q->priority, + &v->evtchn_fifo->control_block->ready) ) + vcpu_mark_events_pending(v); + } + + if ( !was_pending ) + evtchn_check_pollers(d, port); +} + +static void evtchn_fifo_clear_pending(struct domain *d, struct evtchn *evtchn) +{ + event_word_t *word; + + word = evtchn_fifo_word_from_port(d, evtchn->port); + if ( unlikely(!word) ) + return; + + /* + * Just clear the P bit. + * + * No need to unlink as the guest will unlink and ignore + * non-pending events. + */ + clear_bit(EVTCHN_FIFO_PENDING, word); +} + +static void evtchn_fifo_unmask(struct domain *d, struct evtchn *evtchn) +{ + struct vcpu *v = d->vcpu[evtchn->notify_vcpu_id]; + event_word_t *word; + + word = evtchn_fifo_word_from_port(d, evtchn->port); + if ( unlikely(!word) ) + return; + + clear_bit(EVTCHN_FIFO_MASKED, word); + + /* Relink if pending. */ + if ( test_bit(EVTCHN_FIFO_PENDING, word) ) + evtchn_fifo_set_pending(v, evtchn); +} + +static bool_t evtchn_fifo_is_pending(struct domain *d, + const struct evtchn *evtchn) +{ + event_word_t *word; + + word = evtchn_fifo_word_from_port(d, evtchn->port); + if ( unlikely(!word) ) + return 0; + + return test_bit(EVTCHN_FIFO_PENDING, word); +} + +static bool_t evtchn_fifo_is_masked(struct domain *d, + const struct evtchn *evtchn) +{ + event_word_t *word; + + word = evtchn_fifo_word_from_port(d, evtchn->port); + if ( unlikely(!word) ) + return 1; + + return test_bit(EVTCHN_FIFO_MASKED, word); +} + +static int evtchn_fifo_set_priority(struct domain *d, struct evtchn *evtchn, + unsigned int priority) +{ + if ( priority > EVTCHN_FIFO_PRIORITY_MIN ) + return -EINVAL; + + /* + * Only need to switch to the new queue for future events. If the + * event is already pending or in the process of being linked it + * will be on the old queue -- this is fine. + */ + evtchn->priority = priority; + + return 0; +} + +static void evtchn_fifo_print_state(struct domain *d, + const struct evtchn *evtchn) +{ + event_word_t *word; + + word = evtchn_fifo_word_from_port(d, evtchn->port); + if ( !word ) + printk("? "); + else if ( test_bit(EVTCHN_FIFO_LINKED, word) ) + printk("%-4u", *word & EVTCHN_FIFO_LINK_MASK); + else + printk("- "); +} + +static const struct evtchn_port_ops evtchn_port_ops_fifo = +{ + .set_pending = evtchn_fifo_set_pending, + .clear_pending = evtchn_fifo_clear_pending, + .unmask = evtchn_fifo_unmask, + .is_pending = evtchn_fifo_is_pending, + .is_masked = evtchn_fifo_is_masked, + .set_priority = evtchn_fifo_set_priority, + .print_state = evtchn_fifo_print_state, +}; + +static int map_guest_page(struct domain *d, uint64_t gfn, void **virt) +{ + struct page_info *p; + + p = get_page_from_gfn(d, gfn, NULL, P2M_ALLOC); + if ( !p ) + return -EINVAL; + + if ( !get_page_type(p, PGT_writable_page) ) + { + put_page(p); + return -EINVAL; + } + + *virt = map_domain_page_global(gfn); + if ( !*virt ) + { + put_page_and_type(p); + return -ENOMEM; + } + return 0; +} + +static void unmap_guest_page(void *virt) +{ + struct page_info *page; + + if ( !virt ) + return; + + virt = (void *)((unsigned long)virt & PAGE_MASK); + page = mfn_to_page(domain_page_map_to_mfn(virt)); + + unmap_domain_page_global(virt); + put_page_and_type(page); +} + +static void init_queue(struct vcpu *v, struct evtchn_fifo_queue *q, + unsigned int i) +{ + spin_lock_init(&q->lock); + q->priority = i; + q->head = &v->evtchn_fifo->control_block->head[i]; +} + +static int setup_control_block(struct vcpu *v, uint64_t gfn, uint32_t offset) +{ + struct domain *d = v->domain; + struct evtchn_fifo_vcpu *efv; + void *virt; + unsigned int i; + int rc; + + if ( v->evtchn_fifo ) + return -EINVAL; + + efv = xzalloc(struct evtchn_fifo_vcpu); + if ( !efv ) + return -ENOMEM; + + rc = map_guest_page(d, gfn, &virt); + if ( rc < 0 ) + { + xfree(efv); + return rc; + } + + v->evtchn_fifo = efv; + v->evtchn_fifo->control_block = virt + offset; + + for ( i = 0; i <= EVTCHN_FIFO_PRIORITY_MIN; i++ ) + init_queue(v, &v->evtchn_fifo->queue[i], i); + + return 0; +} + +static void cleanup_control_block(struct vcpu *v) +{ + if ( !v->evtchn_fifo ) + return; + + unmap_guest_page(v->evtchn_fifo->control_block); + xfree(v->evtchn_fifo); + v->evtchn_fifo = NULL; +} + +/* + * Setup an event array with no pages. + */ +static int setup_event_array(struct domain *d) +{ + d->evtchn_fifo = xzalloc(struct evtchn_fifo_domain); + if ( !d->evtchn_fifo ) + return -ENOMEM; + + return 0; +} + +static void cleanup_event_array(struct domain *d) +{ + unsigned int i; + + if ( !d->evtchn_fifo ) + return; + + for ( i = 0; i < EVTCHN_FIFO_MAX_EVENT_ARRAY_PAGES; i++ ) + unmap_guest_page(d->evtchn_fifo->event_array[i]); + xfree(d->evtchn_fifo); +} + +static void set_priority_all(struct domain *d, unsigned int priority) +{ + unsigned int port; + + for ( port = 1; port < d->max_evtchns; port++ ) + { + if ( !port_is_valid(d, port) ) + break; + + evtchn_port_set_priority(d, evtchn_from_port(d, port), priority); + } +} + +int evtchn_fifo_init_control(struct evtchn_init_control *init_control) +{ + struct domain *d = current->domain; + uint32_t vcpu_id; + uint64_t gfn; + uint32_t offset; + struct vcpu *v; + int rc; + + init_control->link_bits = EVTCHN_FIFO_LINK_BITS; + + vcpu_id = init_control->vcpu; + gfn = init_control->control_gfn; + offset = init_control->offset; + + if ( vcpu_id >= d->max_vcpus || !d->vcpu[vcpu_id] ) + return -ENOENT; + v = d->vcpu[vcpu_id]; + + /* Must not cross page boundary. */ + if ( offset > (PAGE_SIZE - sizeof(evtchn_fifo_control_block_t)) ) + return -EINVAL; + + /* Must be 8-bytes aligned. */ + if ( offset & (8 - 1) ) + return -EINVAL; + + spin_lock(&d->event_lock); + + rc = setup_control_block(v, gfn, offset); + + /* + * If this is the first control block, setup an empty event array + * and switch to the fifo port ops. + * + * Any ports currently bound will have their priority set to the + * default. + */ + if ( rc == 0 && !d->evtchn_fifo ) + { + rc = setup_event_array(d); + if ( rc < 0 ) + cleanup_control_block(v); + else + { + d->evtchn_port_ops = &evtchn_port_ops_fifo; + d->max_evtchns = EVTCHN_FIFO_NR_CHANNELS; + set_priority_all(d, EVTCHN_FIFO_PRIORITY_DEFAULT); + } + } + + spin_unlock(&d->event_lock); + + return rc; +} + +static int add_page_to_event_array(struct domain *d, unsigned long gfn) +{ + void *virt; + unsigned int slot; + int rc; + + slot = d->evtchn_fifo->num_evtchns / EVTCHN_FIFO_EVENT_WORDS_PER_PAGE; + if ( slot >= EVTCHN_FIFO_MAX_EVENT_ARRAY_PAGES ) + return -ENOSPC; + + rc = map_guest_page(d, gfn, &virt); + if ( rc < 0 ) + return rc; + + d->evtchn_fifo->event_array[slot] = virt; + d->evtchn_fifo->num_evtchns += EVTCHN_FIFO_EVENT_WORDS_PER_PAGE; + + return 0; +} + +int evtchn_fifo_expand_array(const struct evtchn_expand_array *expand_array) +{ + struct domain *d = current->domain; + int rc; + + if ( !d->evtchn_fifo ) + return -ENOSYS; + + spin_lock(&d->event_lock); + rc = add_page_to_event_array(d, expand_array->array_gfn); + spin_unlock(&d->event_lock); + + return rc; +} + +void evtchn_fifo_destroy(struct domain *d) +{ + struct vcpu *v; + + for_each_vcpu( d, v ) + cleanup_control_block(v); + cleanup_event_array(d); +} + +/* + * Local variables: + * mode: C + * c-file-style: "BSD" + * c-basic-offset: 4 + * tab-width: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/include/xen/event_fifo.h b/xen/include/xen/event_fifo.h new file mode 100644 index 0000000000..4115f6f527 --- /dev/null +++ b/xen/include/xen/event_fifo.h @@ -0,0 +1,47 @@ +/* + * FIFO-based event channel ABI. + * + * Copyright (C) 2013 Citrix Systems R&D Ltd. + * + * This source code is licensed under the GNU General Public License, + * Version 2 or later. See the file COPYING for more details. + */ +#ifndef __XEN_EVENT_FIFO_H__ +#define __XEN_EVENT_FIFO_H__ + +struct evtchn_fifo_queue { + uint32_t *head; /* points into control block */ + uint32_t tail; + uint8_t priority; + spinlock_t lock; +}; + +struct evtchn_fifo_vcpu { + struct evtchn_fifo_control_block *control_block; + struct evtchn_fifo_queue queue[EVTCHN_FIFO_MAX_QUEUES]; +}; + +#define EVTCHN_FIFO_EVENT_WORDS_PER_PAGE (PAGE_SIZE / sizeof(event_word_t)) +#define EVTCHN_FIFO_MAX_EVENT_ARRAY_PAGES \ + (EVTCHN_FIFO_NR_CHANNELS / EVTCHN_FIFO_EVENT_WORDS_PER_PAGE) + +struct evtchn_fifo_domain { + event_word_t *event_array[EVTCHN_FIFO_MAX_EVENT_ARRAY_PAGES]; + unsigned int num_evtchns; +}; + +int evtchn_fifo_init_control(struct evtchn_init_control *init_control); +int evtchn_fifo_expand_array(const struct evtchn_expand_array *expand_array); +void evtchn_fifo_destroy(struct domain *domain); + +#endif /* __XEN_EVENT_FIFO_H__ */ + +/* + * Local variables: + * mode: C + * c-file-style: "BSD" + * c-basic-offset: 4 + * tab-width: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h index 1bc6475402..ca7e41712d 100644 --- a/xen/include/xen/sched.h +++ b/xen/include/xen/sched.h @@ -61,7 +61,7 @@ extern struct domain *dom0; #define next_power_of_2(x) (__RDU32((x)-1) + 1) /* Maximum number of event channels for any ABI. */ -#define MAX_NR_EVTCHNS EVTCHN_2L_NR_CHANNELS +#define MAX_NR_EVTCHNS MAX(EVTCHN_2L_NR_CHANNELS, EVTCHN_FIFO_NR_CHANNELS) #define EVTCHNS_PER_BUCKET (PAGE_SIZE / next_power_of_2(sizeof(struct evtchn))) #define EVTCHNS_PER_GROUP (BUCKETS_PER_GROUP * EVTCHNS_PER_BUCKET) @@ -95,6 +95,7 @@ struct evtchn } pirq; /* state == ECS_PIRQ */ u16 virq; /* state == ECS_VIRQ */ } u; + u8 priority; #ifdef FLASK_ENABLE void *ssid; #endif @@ -209,6 +210,8 @@ struct vcpu /* Guest-specified relocation of vcpu_info. */ unsigned long vcpu_info_mfn; + struct evtchn_fifo_vcpu *evtchn_fifo; + struct arch_vcpu arch; }; @@ -290,6 +293,7 @@ struct domain unsigned int max_evtchns; spinlock_t event_lock; const struct evtchn_port_ops *evtchn_port_ops; + struct evtchn_fifo_domain *evtchn_fifo; struct grant_table *grant_table; -- cgit v1.2.3