bitkeeper revision 1.131 (3e7270ecL24hQl_PjDBYoS8hhB8GTA)

multicall.h:
  new file
Many files:
  Multicall now available, so that Xen syscalls can be batched to amortise cost of trap to ring 0. Used by xenolinux to reduce the cost of a context switch.

7 files changed, 132 insertions, 14 deletions

diff --git a/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/hypervisor.c b/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/hypervisor.c
index c49087173f..68670daa8c 100644
--- a/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/hypervisor.c
+++ b/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/hypervisor.c
@@ -13,6 +13,9 @@
 #include <asm/system.h>
 #include <asm/ptrace.h>
 
+multicall_entry_t multicall_list[8];
+int nr_multicall_ents = 0;
+
 static unsigned long event_mask = 0;
 
 void frobb(void) {}
diff --git a/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/process.c b/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/process.c
index a4e4cd2497..32ce1a66ab 100644
--- a/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/process.c
+++ b/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/process.c
@@ -43,6 +43,7 @@
 #include <asm/i387.h>
 #include <asm/desc.h>
 #include <asm/mmu_context.h>
+#include <asm/multicall.h>
 
 #include <linux/irq.h>
 
@@ -85,7 +86,7 @@ void cpu_idle (void)
 
     while (1) {
         while (!current->need_resched)
-            HYPERVISOR_do_sched_op(NULL);
+            HYPERVISOR_yield();
         schedule();
         check_pgt_cache();
     }
@@ -334,9 +335,28 @@ void __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
     struct thread_struct *prev = &prev_p->thread,
         *next = &next_p->thread;
 
-    unlazy_fpu(prev_p);
+    /*
+     * This is basically 'unlazy_fpu', except that we queue a multicall to 
+     * indicate FPU task switch, rather than synchronously trapping to Xen.
+     */
+    if ( prev_p->flags & PF_USEDFPU )
+    {
+	if ( cpu_has_fxsr )
+            asm volatile( "fxsave %0 ; fnclex"
+                          : "=m" (prev_p->thread.i387.fxsave) );
+	else
+            asm volatile( "fnsave %0 ; fwait"
+                          : "=m" (prev_p->thread.i387.fsave) );
+	prev_p->flags &= ~PF_USEDFPU;
+        queue_multicall0(__HYPERVISOR_fpu_taskswitch);
+    }
+
+    if ( next->esp0 != 0 )
+        queue_multicall2(__HYPERVISOR_stack_switch, __KERNEL_DS, next->esp0);
 
-    HYPERVISOR_stack_and_ldt_switch(__KERNEL_DS, next->esp0, 0);
+    /* EXECUTE ALL TASK SWITCH XEN SYSCALLS AT THIS POINT. */
+    execute_multicall_list();
+    sti(); /* matches 'cli' in switch_mm() */
 
     /*
      * Save away %fs and %gs. No need to save %es and %ds, as
diff --git a/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/setup.c b/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/setup.c
index 1650d7028f..00c68a836f 100644
--- a/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/setup.c
+++ b/xenolinux-2.4.21-pre4-sparse/arch/xeno/kernel/setup.c
@@ -967,7 +967,7 @@ void __init cpu_init (void)
         BUG();
     enter_lazy_tlb(&init_mm, current, nr);
 
-    HYPERVISOR_stack_and_ldt_switch(__KERNEL_DS, current->thread.esp0, 0);
+    HYPERVISOR_stack_switch(__KERNEL_DS, current->thread.esp0);
 
     /* Force FPU initialization. */
     current->flags &= ~PF_USEDFPU;
diff --git a/xenolinux-2.4.21-pre4-sparse/arch/xeno/mm/hypervisor.c b/xenolinux-2.4.21-pre4-sparse/arch/xeno/mm/hypervisor.c
index 135ed5c066..93554c3420 100644
--- a/xenolinux-2.4.21-pre4-sparse/arch/xeno/mm/hypervisor.c
+++ b/xenolinux-2.4.21-pre4-sparse/arch/xeno/mm/hypervisor.c
@@ -11,6 +11,7 @@
 #include <asm/hypervisor.h>
 #include <asm/page.h>
 #include <asm/pgtable.h>
+#include <asm/multicall.h>
 
 /*
  * This suffices to protect us if we ever move to SMP domains.
@@ -85,6 +86,28 @@ static void DEBUG_disallow_pt_read(unsigned long pa)
  */
 unsigned long pt_baseptr;
 
+/*
+ * MULTICALL_flush_page_update_queue:
+ *   This is a version of the flush which queues as part of a multicall.
+ */
+void MULTICALL_flush_page_update_queue(void)
+{
+    unsigned long flags;
+    spin_lock_irqsave(&update_lock, flags);
+    if ( idx != 0 ) 
+    {
+#if PT_UPDATE_DEBUG > 1
+        printk("Flushing %d entries from pt update queue\n", idx);
+#endif
+#if PT_UPDATE_DEBUG > 0
+        DEBUG_allow_pt_reads();
+#endif
+        queue_multicall2(__HYPERVISOR_pt_update, update_queue, idx);
+        idx = 0;
+    }
+    spin_unlock_irqrestore(&update_lock, flags);
+}
+
 static inline void __flush_page_update_queue(void)
 {
 #if PT_UPDATE_DEBUG > 1
diff --git a/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/hypervisor.h b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/hypervisor.h
index 16f37cfe65..4b9591102c 100644
--- a/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/hypervisor.h
+++ b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/hypervisor.h
@@ -129,6 +129,7 @@ static inline int flush_page_update_queue(void)
     return idx;
 }
 #define XENO_flush_page_update_queue() (_flush_page_update_queue())
+void MULTICALL_flush_page_update_queue(void);
 
 
 /*
@@ -183,14 +184,24 @@ static inline int HYPERVISOR_set_gdt(unsigned long *frame_list, int entries)
     return ret;
 }
 
-static inline int HYPERVISOR_stack_and_ldt_switch(
-    unsigned long ss, unsigned long esp, unsigned long ldts)
+static inline int HYPERVISOR_stack_switch(unsigned long ss, unsigned long esp)
 {
     int ret;
     __asm__ __volatile__ (
         TRAP_INSTR
-        : "=a" (ret) : "0" (__HYPERVISOR_stack_and_ldt_switch),
-        "b" (ss), "c" (esp), "d" (ldts) );
+        : "=a" (ret) : "0" (__HYPERVISOR_stack_switch),
+        "b" (ss), "c" (esp) : "memory" );
+
+    return ret;
+}
+
+static inline int HYPERVISOR_ldt_switch(unsigned long ldts)
+{
+    int ret;
+    __asm__ __volatile__ (
+        TRAP_INSTR
+        : "=a" (ret) : "0" (__HYPERVISOR_ldt_switch),
+        "b" (ldts) : "memory" );
 
     return ret;
 }
@@ -215,13 +226,12 @@ static inline int HYPERVISOR_fpu_taskswitch(void)
     return ret;
 }
 
-static inline int HYPERVISOR_do_sched_op(void *sched_op)
+static inline int HYPERVISOR_yield(void)
 {
     int ret;
     __asm__ __volatile__ (
         TRAP_INSTR
-        : "=a" (ret) : "0" (__HYPERVISOR_sched_op),
-        "b" (sched_op) );
+        : "=a" (ret) : "0" (__HYPERVISOR_yield) );
 
     return ret;
 }
@@ -296,7 +306,7 @@ static inline int HYPERVISOR_update_descriptor(
     int ret;
     __asm__ __volatile__ (
         TRAP_INSTR
-        : "=a" (ret) : "0" (__HYPERVISOR_set_gdt), 
+        : "=a" (ret) : "0" (__HYPERVISOR_update_descriptor), 
         "b" (pa), "c" (word1), "d" (word2) );
 
     return ret;
@@ -324,4 +334,15 @@ static inline int HYPERVISOR_dom_mem_op(void *dom_mem_op)
     return ret;
 }
 
+static inline int HYPERVISOR_multicall(void *call_list, int nr_calls)
+{
+    int ret;
+    __asm__ __volatile__ (
+        TRAP_INSTR
+        : "=a" (ret) : "0" (__HYPERVISOR_multicall),
+        "b" (call_list), "c" (nr_calls) : "memory" );
+
+    return ret;
+}
+
 #endif /* __HYPERVISOR_H__ */
diff --git a/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/mmu_context.h b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/mmu_context.h
index c5b3cdcd61..ad07796f95 100644
--- a/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/mmu_context.h
+++ b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/mmu_context.h
@@ -5,6 +5,7 @@
 #include <asm/desc.h>
 #include <asm/atomic.h>
 #include <asm/pgalloc.h>
+#include <asm/multicall.h>
 
 /*
  * possibly do the LDT unload here?
@@ -33,6 +34,7 @@ extern pgd_t *cur_pgd;
 
 static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk, unsigned cpu)
 {
+	cli(); /* protect flush_update_queue multicall */
 	if (prev != next) {
 		/* stop flush ipis for the previous mm */
 		clear_bit(cpu, &prev->cpu_vm_mask);
@@ -50,7 +52,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, str
 		/* Re-load page tables */
 		cur_pgd = next->pgd;
 		queue_pt_switch(__pa(cur_pgd));
-		XENO_flush_page_update_queue();
+		MULTICALL_flush_page_update_queue();
 	}
 #ifdef CONFIG_SMP
 	else {
@@ -70,6 +72,10 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, str
 }
 
 #define activate_mm(prev, next) \
-	switch_mm((prev),(next),NULL,smp_processor_id())
+do { \
+	switch_mm((prev),(next),NULL,smp_processor_id()); \
+	execute_multicall_list(); \
+	sti(); /* matches 'cli' in switch_mm() */ \
+} while ( 0 )
 
 #endif
diff --git a/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/multicall.h b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/multicall.h
new file mode 100644
index 0000000000..27126dcaac
--- /dev/null
+++ b/xenolinux-2.4.21-pre4-sparse/include/asm-xeno/multicall.h
@@ -0,0 +1,45 @@
+/******************************************************************************
+ * multicall.h
+ */
+
+#ifndef __MULTICALL_H__
+#define __MULTICALL_H__
+
+#include <asm/hypervisor.h>
+
+extern multicall_entry_t multicall_list[];
+extern int nr_multicall_ents;
+
+static inline void queue_multicall0(unsigned long op)
+{
+    int i = nr_multicall_ents;
+    multicall_list[i].op      = op;
+    nr_multicall_ents = i+1;
+}
+
+static inline void queue_multicall1(unsigned long op, unsigned long arg1)
+{
+    int i = nr_multicall_ents;
+    multicall_list[i].op      = op;
+    multicall_list[i].args[0] = arg1;
+    nr_multicall_ents = i+1;
+}
+
+static inline void queue_multicall2(
+    unsigned long op, unsigned long arg1, unsigned long arg2)
+{
+    int i = nr_multicall_ents;
+    multicall_list[i].op      = op;
+    multicall_list[i].args[0] = arg1;
+    multicall_list[i].args[1] = arg2;
+    nr_multicall_ents = i+1;
+}
+
+static inline void execute_multicall_list(void)
+{
+    if ( unlikely(nr_multicall_ents == 0) ) return;
+    (void)HYPERVISOR_multicall(multicall_list, nr_multicall_ents);
+    nr_multicall_ents = 0;
+}
+
+#endif /* __MULTICALL_H__ */