From 998e4cfacfe9217a1d96e9300fbce22cf7e8b6bb Mon Sep 17 00:00:00 2001 From: "kaf24@scramble.cl.cam.ac.uk" Date: Sat, 15 Mar 2003 00:16:44 +0000 Subject: bitkeeper revision 1.131 (3e7270ecL24hQl_PjDBYoS8hhB8GTA) multicall.h: new file Many files: Multicall now available, so that Xen syscalls can be batched to amortise cost of trap to ring 0. Used by xenolinux to reduce the cost of a context switch. --- .../arch/xeno/kernel/hypervisor.c | 3 ++ .../arch/xeno/kernel/process.c | 26 +++++++++++-- .../arch/xeno/kernel/setup.c | 2 +- .../arch/xeno/mm/hypervisor.c | 23 +++++++++++ .../include/asm-xeno/hypervisor.h | 37 ++++++++++++++---- .../include/asm-xeno/mmu_context.h | 10 ++++- .../include/asm-xeno/multicall.h | 45 ++++++++++++++++++++++ 7 files changed, 132 insertions(+), 14 deletions(-) create mode 100644 xenolinux-2.4.21-pre4-sparse/include/asm-xeno/multicall.h (limited to 'xenolinux-2.4.21-pre4-sparse') diff --git a/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/hypervisor.c b/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/hypervisor.c index c49087173f..68670daa8c 100644 --- a/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/hypervisor.c +++ b/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/hypervisor.c @@ -13,6 +13,9 @@ #include #include +multicall_entry_t multicall_list[8]; +int nr_multicall_ents = 0; + static unsigned long event_mask = 0; void frobb(void) {} diff --git a/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/process.c b/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/process.c index a4e4cd2497..32ce1a66ab 100644 --- a/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/process.c +++ b/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/process.c @@ -43,6 +43,7 @@ #include #include #include +#include #include @@ -85,7 +86,7 @@ void cpu_idle (void) while (1) { while (!current->need_resched) - HYPERVISOR_do_sched_op(NULL); + HYPERVISOR_yield(); schedule(); check_pgt_cache(); } @@ -334,9 +335,28 @@ void __switch_to(struct task_struct *prev_p, struct task_struct *next_p) struct thread_struct *prev = &prev_p->thread, *next = &next_p->thread; - unlazy_fpu(prev_p); + /* + * This is basically 'unlazy_fpu', except that we queue a multicall to + * indicate FPU task switch, rather than synchronously trapping to Xen. + */ + if ( prev_p->flags & PF_USEDFPU ) + { + if ( cpu_has_fxsr ) + asm volatile( "fxsave %0 ; fnclex" + : "=m" (prev_p->thread.i387.fxsave) ); + else + asm volatile( "fnsave %0 ; fwait" + : "=m" (prev_p->thread.i387.fsave) ); + prev_p->flags &= ~PF_USEDFPU; + queue_multicall0(__HYPERVISOR_fpu_taskswitch); + } + + if ( next->esp0 != 0 ) + queue_multicall2(__HYPERVISOR_stack_switch, __KERNEL_DS, next->esp0); - HYPERVISOR_stack_and_ldt_switch(__KERNEL_DS, next->esp0, 0); + /* EXECUTE ALL TASK SWITCH XEN SYSCALLS AT THIS POINT. */ + execute_multicall_list(); + sti(); /* matches 'cli' in switch_mm() */ /* * Save away %fs and %gs. No need to save %es and %ds, as diff --git a/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/setup.c b/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/setup.c index 1650d7028f..00c68a836f 100644 --- a/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/setup.c +++ b/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/setup.c @@ -967,7 +967,7 @@ void __init cpu_init (void) BUG(); enter_lazy_tlb(&init_mm, current, nr); - HYPERVISOR_stack_and_ldt_switch(__KERNEL_DS, current->thread.esp0, 0); + HYPERVISOR_stack_switch(__KERNEL_DS, current->thread.esp0); /* Force FPU initialization. */ current->flags &= ~PF_USEDFPU; diff --git a/xenolinux-2.4.21-pre4-sparse/arch/xeno/mm/hypervisor.c b/xenolinux-2.4.21-pre4-sparse/arch/xeno/mm/hypervisor.c index 135ed5c066..93554c3420 100644 --- a/xenolinux-2.4.21-pre4-sparse/arch/xeno/mm/hypervisor.c +++ b/xenolinux-2.4.21-pre4-sparse/arch/xeno/mm/hypervisor.c @@ -11,6 +11,7 @@ #include #include #include +#include /* * This suffices to protect us if we ever move to SMP domains. @@ -85,6 +86,28 @@ static void DEBUG_disallow_pt_read(unsigned long pa) */ unsigned long pt_baseptr; +/* + * MULTICALL_flush_page_update_queue: + * This is a version of the flush which queues as part of a multicall. + */ +void MULTICALL_flush_page_update_queue(void) +{ + unsigned long flags; + spin_lock_irqsave(&update_lock, flags); + if ( idx != 0 ) + { +#if PT_UPDATE_DEBUG > 1 + printk("Flushing %d entries from pt update queue\n", idx); +#endif +#if PT_UPDATE_DEBUG > 0 + DEBUG_allow_pt_reads(); +#endif + queue_multicall2(__HYPERVISOR_pt_update, update_queue, idx); + idx = 0; + } + spin_unlock_irqrestore(&update_lock, flags); +} + static inline void __flush_page_update_queue(void) { #if PT_UPDATE_DEBUG > 1 diff --git a/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/hypervisor.h b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/hypervisor.h index 16f37cfe65..4b9591102c 100644 --- a/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/hypervisor.h +++ b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/hypervisor.h @@ -129,6 +129,7 @@ static inline int flush_page_update_queue(void) return idx; } #define XENO_flush_page_update_queue() (_flush_page_update_queue()) +void MULTICALL_flush_page_update_queue(void); /* @@ -183,14 +184,24 @@ static inline int HYPERVISOR_set_gdt(unsigned long *frame_list, int entries) return ret; } -static inline int HYPERVISOR_stack_and_ldt_switch( - unsigned long ss, unsigned long esp, unsigned long ldts) +static inline int HYPERVISOR_stack_switch(unsigned long ss, unsigned long esp) { int ret; __asm__ __volatile__ ( TRAP_INSTR - : "=a" (ret) : "0" (__HYPERVISOR_stack_and_ldt_switch), - "b" (ss), "c" (esp), "d" (ldts) ); + : "=a" (ret) : "0" (__HYPERVISOR_stack_switch), + "b" (ss), "c" (esp) : "memory" ); + + return ret; +} + +static inline int HYPERVISOR_ldt_switch(unsigned long ldts) +{ + int ret; + __asm__ __volatile__ ( + TRAP_INSTR + : "=a" (ret) : "0" (__HYPERVISOR_ldt_switch), + "b" (ldts) : "memory" ); return ret; } @@ -215,13 +226,12 @@ static inline int HYPERVISOR_fpu_taskswitch(void) return ret; } -static inline int HYPERVISOR_do_sched_op(void *sched_op) +static inline int HYPERVISOR_yield(void) { int ret; __asm__ __volatile__ ( TRAP_INSTR - : "=a" (ret) : "0" (__HYPERVISOR_sched_op), - "b" (sched_op) ); + : "=a" (ret) : "0" (__HYPERVISOR_yield) ); return ret; } @@ -296,7 +306,7 @@ static inline int HYPERVISOR_update_descriptor( int ret; __asm__ __volatile__ ( TRAP_INSTR - : "=a" (ret) : "0" (__HYPERVISOR_set_gdt), + : "=a" (ret) : "0" (__HYPERVISOR_update_descriptor), "b" (pa), "c" (word1), "d" (word2) ); return ret; @@ -324,4 +334,15 @@ static inline int HYPERVISOR_dom_mem_op(void *dom_mem_op) return ret; } +static inline int HYPERVISOR_multicall(void *call_list, int nr_calls) +{ + int ret; + __asm__ __volatile__ ( + TRAP_INSTR + : "=a" (ret) : "0" (__HYPERVISOR_multicall), + "b" (call_list), "c" (nr_calls) : "memory" ); + + return ret; +} + #endif /* __HYPERVISOR_H__ */ diff --git a/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/mmu_context.h b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/mmu_context.h index c5b3cdcd61..ad07796f95 100644 --- a/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/mmu_context.h +++ b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/mmu_context.h @@ -5,6 +5,7 @@ #include #include #include +#include /* * possibly do the LDT unload here? @@ -33,6 +34,7 @@ extern pgd_t *cur_pgd; static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk, unsigned cpu) { + cli(); /* protect flush_update_queue multicall */ if (prev != next) { /* stop flush ipis for the previous mm */ clear_bit(cpu, &prev->cpu_vm_mask); @@ -50,7 +52,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, str /* Re-load page tables */ cur_pgd = next->pgd; queue_pt_switch(__pa(cur_pgd)); - XENO_flush_page_update_queue(); + MULTICALL_flush_page_update_queue(); } #ifdef CONFIG_SMP else { @@ -70,6 +72,10 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, str } #define activate_mm(prev, next) \ - switch_mm((prev),(next),NULL,smp_processor_id()) +do { \ + switch_mm((prev),(next),NULL,smp_processor_id()); \ + execute_multicall_list(); \ + sti(); /* matches 'cli' in switch_mm() */ \ +} while ( 0 ) #endif diff --git a/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/multicall.h b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/multicall.h new file mode 100644 index 0000000000..27126dcaac --- /dev/null +++ b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/multicall.h @@ -0,0 +1,45 @@ +/****************************************************************************** + * multicall.h + */ + +#ifndef __MULTICALL_H__ +#define __MULTICALL_H__ + +#include + +extern multicall_entry_t multicall_list[]; +extern int nr_multicall_ents; + +static inline void queue_multicall0(unsigned long op) +{ + int i = nr_multicall_ents; + multicall_list[i].op = op; + nr_multicall_ents = i+1; +} + +static inline void queue_multicall1(unsigned long op, unsigned long arg1) +{ + int i = nr_multicall_ents; + multicall_list[i].op = op; + multicall_list[i].args[0] = arg1; + nr_multicall_ents = i+1; +} + +static inline void queue_multicall2( + unsigned long op, unsigned long arg1, unsigned long arg2) +{ + int i = nr_multicall_ents; + multicall_list[i].op = op; + multicall_list[i].args[0] = arg1; + multicall_list[i].args[1] = arg2; + nr_multicall_ents = i+1; +} + +static inline void execute_multicall_list(void) +{ + if ( unlikely(nr_multicall_ents == 0) ) return; + (void)HYPERVISOR_multicall(multicall_list, nr_multicall_ents); + nr_multicall_ents = 0; +} + +#endif /* __MULTICALL_H__ */ -- cgit v1.2.3