diff options
author | Keir Fraser <keir@xen.org> | 2012-10-29 09:01:52 +0100 |
---|---|---|
committer | Keir Fraser <keir@xen.org> | 2012-10-29 09:01:52 +0100 |
commit | 08a5c2cf7dd908db57be8268aa8dd7a80ae17de7 (patch) | |
tree | 8d16dad88fb3dcccf627e868d01e7d507b3ec5da | |
parent | 62881b57c2dc5884c09ac98c70a2ae6c748bb42d (diff) | |
download | xen-08a5c2cf7dd908db57be8268aa8dd7a80ae17de7.tar.gz xen-08a5c2cf7dd908db57be8268aa8dd7a80ae17de7.tar.bz2 xen-08a5c2cf7dd908db57be8268aa8dd7a80ae17de7.zip |
More efficient TLB-flush filtering in alloc_heap_pages().
Rather than per-cpu filtering for every page in a super-page
allocation, simply remember the most recent TLB timestamp across all
allocated pages, and filter on that, just once, at the end of the
function.
For large-CPU systems, doing 2MB allocations during domain creation,
this cuts down the domain creation time *massively*.
TODO: It may make sense to move the filtering out into some callers,
such as memory.c:populate_physmap() and
memory.c:increase_reservation(), so that the filtering can be moved
outside their loops, too.
Signed-off-by: Keir Fraser <keir@xen.org>
xen-unstable changeset: 26056:177fdda0be56
xen-unstable date: Mon Oct 15 15:38:11 UTC 2012
-rw-r--r-- | xen/common/page_alloc.c | 27 |
1 files changed, 16 insertions, 11 deletions
diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c index 0d53cef79f..311b5ee2a4 100644 --- a/xen/common/page_alloc.c +++ b/xen/common/page_alloc.c @@ -303,9 +303,10 @@ static struct page_info *alloc_heap_pages( unsigned int first_node, i, j, zone = 0, nodemask_retry = 0; unsigned int node = (uint8_t)((memflags >> _MEMF_node) - 1); unsigned long request = 1UL << order; - cpumask_t extra_cpus_mask, mask; struct page_info *pg; nodemask_t nodemask = (d != NULL ) ? d->node_affinity : node_online_map; + bool_t need_tlbflush = 0; + uint32_t tlbflush_timestamp = 0; if ( node == NUMA_NO_NODE ) { @@ -417,20 +418,19 @@ static struct page_info *alloc_heap_pages( if ( d != NULL ) d->last_alloc_node = node; - cpus_clear(mask); - for ( i = 0; i < (1 << order); i++ ) { /* Reference count must continuously be zero for free pages. */ BUG_ON(pg[i].count_info != PGC_state_free); pg[i].count_info = PGC_state_inuse; - if ( pg[i].u.free.need_tlbflush ) + if ( pg[i].u.free.need_tlbflush && + (pg[i].tlbflush_timestamp <= tlbflush_current_time()) && + (!need_tlbflush || + (pg[i].tlbflush_timestamp > tlbflush_timestamp)) ) { - /* Add in extra CPUs that need flushing because of this page. */ - cpus_andnot(extra_cpus_mask, cpu_online_map, mask); - tlbflush_filter(extra_cpus_mask, pg[i].tlbflush_timestamp); - cpus_or(mask, mask, extra_cpus_mask); + need_tlbflush = 1; + tlbflush_timestamp = pg[i].tlbflush_timestamp; } /* Initialise fields which have other uses for free pages. */ @@ -440,10 +440,15 @@ static struct page_info *alloc_heap_pages( spin_unlock(&heap_lock); - if ( unlikely(!cpus_empty(mask)) ) + if ( need_tlbflush ) { - perfc_incr(need_flush_tlb_flush); - flush_tlb_mask(&mask); + cpumask_t mask = cpu_online_map; + tlbflush_filter(mask, tlbflush_timestamp); + if ( !cpus_empty(mask) ) + { + perfc_incr(need_flush_tlb_flush); + flush_tlb_mask(&mask); + } } return pg; |