From 17caf7b1a9321131dbdc27e570607a8aa7f5b2b0 Mon Sep 17 00:00:00 2001 From: Tim Deegan Date: Fri, 1 Jun 2007 14:50:52 +0100 Subject: [XEN] Shadow: cache gva->gfn+rights translations between guest TLB flushes to make lookups faster for emulation and hvm_copy. Signed-off-by: Tim Deegan --- xen/arch/x86/mm/shadow/common.c | 30 +++++++++++++++++ xen/arch/x86/mm/shadow/multi.c | 66 ++++++++++++++++++++++++++++++++++---- xen/arch/x86/mm/shadow/private.h | 69 +++++++++++++++++++++++++++++++++++++++- xen/arch/x86/mm/shadow/types.h | 3 ++ xen/include/asm-x86/domain.h | 3 ++ 5 files changed, 163 insertions(+), 8 deletions(-) diff --git a/xen/arch/x86/mm/shadow/common.c b/xen/arch/x86/mm/shadow/common.c index 2690acbeb2..312bbe2848 100644 --- a/xen/arch/x86/mm/shadow/common.c +++ b/xen/arch/x86/mm/shadow/common.c @@ -2206,6 +2206,24 @@ static void sh_update_paging_modes(struct vcpu *v) ASSERT(shadow_locked_by_me(d)); +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) + /* Make sure this vcpu has a virtual TLB array allocated */ + if ( unlikely(!v->arch.paging.vtlb) ) + { + v->arch.paging.vtlb = xmalloc_array(struct shadow_vtlb, VTLB_ENTRIES); + if ( unlikely(!v->arch.paging.vtlb) ) + { + SHADOW_ERROR("Could not allocate vTLB space for dom %u vcpu %u\n", + d->domain_id, v->vcpu_id); + domain_crash(v->domain); + return; + } + memset(v->arch.paging.vtlb, 0, + VTLB_ENTRIES * sizeof (struct shadow_vtlb)); + spin_lock_init(&v->arch.paging.vtlb_lock); + } +#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */ + // Valid transitions handled by this function: // - For PV guests: // - after a shadow mode has been changed @@ -2514,6 +2532,18 @@ void shadow_teardown(struct domain *d) } } +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) + /* Free the virtual-TLB array attached to each vcpu */ + for_each_vcpu(d, v) + { + if ( v->arch.paging.vtlb ) + { + xfree(v->arch.paging.vtlb); + v->arch.paging.vtlb = NULL; + } + } +#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */ + list_for_each_safe(entry, n, &d->arch.paging.shadow.p2m_freelist) { list_del(entry); diff --git a/xen/arch/x86/mm/shadow/multi.c b/xen/arch/x86/mm/shadow/multi.c index c5fd34a3ca..5e27b86bc6 100644 --- a/xen/arch/x86/mm/shadow/multi.c +++ b/xen/arch/x86/mm/shadow/multi.c @@ -2997,6 +2997,11 @@ sh_invlpg(struct vcpu *v, unsigned long va) perfc_incr(shadow_invlpg); +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) + /* No longer safe to use cached gva->gfn translations */ + vtlb_flush(v); +#endif + /* First check that we can safely read the shadow l2e. SMP/PAE linux can * run as high as 6% of invlpg calls where we haven't shadowed the l2 * yet. */ @@ -3057,6 +3062,7 @@ sh_invlpg(struct vcpu *v, unsigned long va) return 1; } + static unsigned long sh_gva_to_gfn(struct vcpu *v, unsigned long va) /* Called to translate a guest virtual address to what the *guest* @@ -3064,11 +3070,24 @@ sh_gva_to_gfn(struct vcpu *v, unsigned long va) { walk_t gw; gfn_t gfn; + +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) + struct shadow_vtlb t = {0}; + if ( vtlb_lookup(v, va, &t) ) + return t.frame_number; +#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */ guest_walk_tables(v, va, &gw, 0); gfn = guest_walk_to_gfn(&gw); - unmap_walk(v, &gw); +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) + t.page_number = va >> PAGE_SHIFT; + t.frame_number = gfn_x(gfn); + t.flags = accumulate_guest_flags(v, &gw); + vtlb_insert(v, t); +#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */ + + unmap_walk(v, &gw); return gfn_x(gfn); } @@ -3694,6 +3713,11 @@ sh_update_cr3(struct vcpu *v, int do_locking) /* Fix up the linear pagetable mappings */ sh_update_linear_entries(v); +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) + /* No longer safe to use cached gva->gfn translations */ + vtlb_flush(v); +#endif + /* Release the lock, if we took it (otherwise it's the caller's problem) */ if ( do_locking ) shadow_unlock(v->domain); } @@ -3918,13 +3942,41 @@ static inline void * emulate_map_dest(struct vcpu *v, if ( ring_3(sh_ctxt->ctxt.regs) ) return NULL; - /* Walk the guest pagetables */ - guest_walk_tables(v, vaddr, &gw, 1); - flags = accumulate_guest_flags(v, &gw); - gfn = guest_l1e_get_gfn(gw.eff_l1e); +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) + /* Try the virtual TLB first */ + { + struct shadow_vtlb t = {0}; + if ( vtlb_lookup(v, vaddr, &t) + && ((t.flags & (_PAGE_PRESENT|_PAGE_RW)) + == (_PAGE_PRESENT|_PAGE_RW)) ) + { + flags = t.flags; + gfn = _gfn(t.frame_number); + } + else + { + /* Need to do the full lookup, just in case permissions + * have increased since we cached this entry */ + +#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */ + + /* Walk the guest pagetables */ + guest_walk_tables(v, vaddr, &gw, 1); + flags = accumulate_guest_flags(v, &gw); + gfn = guest_l1e_get_gfn(gw.eff_l1e); + sh_audit_gw(v, &gw); + unmap_walk(v, &gw); + +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) + /* Remember this translation for next time */ + t.page_number = vaddr >> PAGE_SHIFT; + t.frame_number = gfn_x(gfn); + t.flags = flags; + vtlb_insert(v, t); + } + } +#endif mfn = vcpu_gfn_to_mfn(v, gfn); - sh_audit_gw(v, &gw); - unmap_walk(v, &gw); errcode = PFEC_write_access; if ( !(flags & _PAGE_PRESENT) ) diff --git a/xen/arch/x86/mm/shadow/private.h b/xen/arch/x86/mm/shadow/private.h index 52ea6d5d16..8a6b93f4d8 100644 --- a/xen/arch/x86/mm/shadow/private.h +++ b/xen/arch/x86/mm/shadow/private.h @@ -61,8 +61,9 @@ extern int shadow_audit_enable; #define SHOPT_PREFETCH 0x08 /* Shadow multiple entries per fault */ #define SHOPT_LINUX_L3_TOPLEVEL 0x10 /* Pin l3es on early 64bit linux */ #define SHOPT_SKIP_VERIFY 0x20 /* Skip PTE v'fy when safe to do so */ +#define SHOPT_VIRTUAL_TLB 0x40 /* Cache guest v->p translations */ -#define SHADOW_OPTIMIZATIONS 0x3f +#define SHADOW_OPTIMIZATIONS 0x7f /****************************************************************************** @@ -649,6 +650,72 @@ struct x86_emulate_ops *shadow_init_emulation( void shadow_continue_emulation( struct sh_emulate_ctxt *sh_ctxt, struct cpu_user_regs *regs); + +#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) +/**************************************************************************/ +/* Virtual TLB entries + * + * We keep a cache of virtual-to-physical translations that we have seen + * since the last TLB flush. This is safe to use for frame translations, + * but callers that use the rights need to re-check the actual guest tables + * before triggering a fault. + * + * Lookups and updates are protected by a per-vTLB (and hence per-vcpu) + * lock. This lock is held *only* while reading or writing the table, + * so it is safe to take in any non-interrupt context. Most lookups + * happen with v==current, so we expect contention to be low. + */ + +#define VTLB_ENTRIES 13 + +struct shadow_vtlb { + unsigned long page_number; /* Guest virtual address >> PAGE_SHIFT */ + unsigned long frame_number; /* Guest physical address >> PAGE_SHIFT */ + u32 flags; /* Accumulated guest pte flags, or 0 for an empty slot. */ +}; + +/* Call whenever the guest flushes hit actual TLB */ +static inline void vtlb_flush(struct vcpu *v) +{ + spin_lock(&v->arch.paging.vtlb_lock); + memset(v->arch.paging.vtlb, 0, VTLB_ENTRIES * sizeof (struct shadow_vtlb)); + spin_unlock(&v->arch.paging.vtlb_lock); +} + +static inline int vtlb_hash(unsigned long page_number) +{ + return page_number % VTLB_ENTRIES; +} + +/* Put a translation into the vTLB, potentially clobbering an old one */ +static inline void vtlb_insert(struct vcpu *v, struct shadow_vtlb entry) +{ + spin_lock(&v->arch.paging.vtlb_lock); + v->arch.paging.vtlb[vtlb_hash(entry.page_number)] = entry; + spin_unlock(&v->arch.paging.vtlb_lock); +} + +/* Look a translation up in the vTLB. Returns 0 if not found. */ +static inline int vtlb_lookup(struct vcpu *v, unsigned long va, + struct shadow_vtlb *result) +{ + unsigned long page_number = va >> PAGE_SHIFT; + int rv = 0; + int i = vtlb_hash(page_number); + + spin_lock(&v->arch.paging.vtlb_lock); + if ( v->arch.paging.vtlb[i].flags != 0 + && v->arch.paging.vtlb[i].page_number == page_number ) + { + rv = 1; + result[0] = v->arch.paging.vtlb[i]; + } + spin_unlock(&v->arch.paging.vtlb_lock); + return rv; +} +#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */ + + #endif /* _XEN_SHADOW_PRIVATE_H */ /* diff --git a/xen/arch/x86/mm/shadow/types.h b/xen/arch/x86/mm/shadow/types.h index 8397e7ae2c..1595a25a5b 100644 --- a/xen/arch/x86/mm/shadow/types.h +++ b/xen/arch/x86/mm/shadow/types.h @@ -553,6 +553,9 @@ accumulate_guest_flags(struct vcpu *v, walk_t *gw) { u32 accumulated_flags; + if ( unlikely(!(guest_l1e_get_flags(gw->eff_l1e) & _PAGE_PRESENT)) ) + return 0; + // We accumulate the permission flags with bitwise ANDing. // This works for the PRESENT bit, RW bit, and USER bit. // For the NX bit, however, the polarity is wrong, so we accumulate the diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h index 30c7601d1a..b3c910cb0b 100644 --- a/xen/include/asm-x86/domain.h +++ b/xen/include/asm-x86/domain.h @@ -173,6 +173,9 @@ struct paging_vcpu { unsigned int translate_enabled:1; /* HVM guest: last emulate was to a pagetable */ unsigned int last_write_was_pt:1; + /* Translated guest: virtual TLB */ + struct shadow_vtlb *vtlb; + spinlock_t vtlb_lock; /* paging support extension */ struct shadow_vcpu shadow; -- cgit v1.2.3