Move tasklet implementation into its own source files.

This is preparation for implementing tasklets in vcpu context rather
than softirq context. There is no change to the implementation of
tasklets in this patch.

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>

20 files changed, 221 insertions, 167 deletions

diff --git a/xen/arch/ia64/xen/xensetup.c b/xen/arch/ia64/xen/xensetup.c
index da8de417c2..82b758b788 100644
--- a/xen/arch/ia64/xen/xensetup.c
+++ b/xen/arch/ia64/xen/xensetup.c
@@ -562,6 +562,7 @@ skip_move:
     end_boot_allocator();
 
     softirq_init();
+    tasklet_subsys_init();
 
     late_setup_arch(&cmdline);
 
diff --git a/xen/arch/x86/acpi/cpu_idle.c b/xen/arch/x86/acpi/cpu_idle.c
index bd4d68a14d..7c75f04f3f 100644
--- a/xen/arch/x86/acpi/cpu_idle.c
+++ b/xen/arch/x86/acpi/cpu_idle.c
@@ -47,6 +47,7 @@
 #include <asm/hpet.h>
 #include <asm/processor.h>
 #include <xen/pmstat.h>
+#include <xen/softirq.h>
 #include <public/platform.h>
 #include <public/sysctl.h>
 #include <acpi/cpufreq/cpufreq.h>
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 68be297eec..855cb4ad54 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -972,6 +972,7 @@ void __init __start_xen(unsigned long mbi_p)
 #endif
 
     softirq_init();
+    tasklet_subsys_init();
 
     early_cpu_init();
 
diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index 5f8a96100b..4b2f64d5ff 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -42,6 +42,7 @@
 #include <xen/irq.h>
 #include <xen/delay.h>
 #include <xen/softirq.h>
+#include <xen/tasklet.h>
 #include <xen/serial.h>
 #include <xen/numa.h>
 #include <xen/event.h>
diff --git a/xen/common/Makefile b/xen/common/Makefile
index baa6b5872b..c36d0f351d 100644
--- a/xen/common/Makefile
+++ b/xen/common/Makefile
@@ -23,6 +23,7 @@ obj-y += stop_machine.o
 obj-y += string.o
 obj-y += symbols.o
 obj-y += sysctl.o
+obj-y += tasklet.o
 obj-y += time.o
 obj-y += timer.o
 obj-y += trace.o
diff --git a/xen/common/domain.c b/xen/common/domain.c
index 808d250930..f67445c77e 100644
--- a/xen/common/domain.c
+++ b/xen/common/domain.c
@@ -17,6 +17,7 @@
 #include <xen/time.h>
 #include <xen/console.h>
 #include <xen/softirq.h>
+#include <xen/tasklet.h>
 #include <xen/domain_page.h>
 #include <xen/rangeset.h>
 #include <xen/guest_access.h>
diff --git a/xen/common/keyhandler.c b/xen/common/keyhandler.c
index e3e64a1953..611724a645 100644
--- a/xen/common/keyhandler.c
+++ b/xen/common/keyhandler.c
@@ -9,7 +9,7 @@
 #include <xen/console.h>
 #include <xen/serial.h>
 #include <xen/sched.h>
-#include <xen/softirq.h>
+#include <xen/tasklet.h>
 #include <xen/domain.h>
 #include <xen/rangeset.h>
 #include <xen/compat.h>
diff --git a/xen/common/softirq.c b/xen/common/softirq.c
index e90b71ed2b..fac59818ac 100644
--- a/xen/common/softirq.c
+++ b/xen/common/softirq.c
@@ -88,146 +88,8 @@ void raise_softirq(unsigned int nr)
     set_bit(nr, &softirq_pending(smp_processor_id()));
 }
 
-static bool_t tasklets_initialised;
-static DEFINE_PER_CPU(struct list_head, tasklet_list);
-static DEFINE_SPINLOCK(tasklet_lock);
-
-void tasklet_schedule_on_cpu(struct tasklet *t, unsigned int cpu)
-{
-    unsigned long flags;
-
-    spin_lock_irqsave(&tasklet_lock, flags);
-
-    if ( tasklets_initialised && !t->is_dead )
-    {
-        t->scheduled_on = cpu;
-        if ( !t->is_running )
-        {
-            list_del(&t->list);
-            list_add_tail(&t->list, &per_cpu(tasklet_list, cpu));
-            cpu_raise_softirq(cpu, TASKLET_SOFTIRQ);
-        }
-    }
-
-    spin_unlock_irqrestore(&tasklet_lock, flags);
-}
-
-void tasklet_schedule(struct tasklet *t)
-{
-    tasklet_schedule_on_cpu(t, smp_processor_id());
-}
-
-static void tasklet_action(void)
-{
-    unsigned int cpu = smp_processor_id();
-    struct list_head *list = &per_cpu(tasklet_list, cpu);
-    struct tasklet *t;
-
-    spin_lock_irq(&tasklet_lock);
-
-    if ( list_empty(list) )
-    {
-        spin_unlock_irq(&tasklet_lock);
-        return;
-    }
-
-    t = list_entry(list->next, struct tasklet, list);
-    list_del_init(&t->list);
-
-    BUG_ON(t->is_dead || t->is_running || (t->scheduled_on != cpu));
-    t->scheduled_on = -1;
-    t->is_running = 1;
-
-    spin_unlock_irq(&tasklet_lock);
-    t->func(t->data);
-    spin_lock_irq(&tasklet_lock);
-
-    t->is_running = 0;
-
-    if ( t->scheduled_on >= 0 )
-    {
-        BUG_ON(t->is_dead || !list_empty(&t->list));
-        list_add_tail(&t->list, &per_cpu(tasklet_list, t->scheduled_on));
-        if ( t->scheduled_on != cpu )
-            cpu_raise_softirq(t->scheduled_on, TASKLET_SOFTIRQ);
-    }
-
-    /*
-     * If there is more work to do then reschedule. We don't grab more work
-     * immediately as we want to allow other softirq work to happen first.
-     */
-    if ( !list_empty(list) )
-        raise_softirq(TASKLET_SOFTIRQ);
-
-    spin_unlock_irq(&tasklet_lock);
-}
-
-void tasklet_kill(struct tasklet *t)
-{
-    unsigned long flags;
-
-    spin_lock_irqsave(&tasklet_lock, flags);
-
-    if ( !list_empty(&t->list) )
-    {
-        BUG_ON(t->is_dead || t->is_running || (t->scheduled_on < 0));
-        list_del_init(&t->list);
-    }
-    t->scheduled_on = -1;
-    t->is_dead = 1;
-
-    while ( t->is_running )
-    {
-        spin_unlock_irqrestore(&tasklet_lock, flags);
-        cpu_relax();
-        spin_lock_irqsave(&tasklet_lock, flags);
-    }
-
-    spin_unlock_irqrestore(&tasklet_lock, flags);
-}
-
-void migrate_tasklets_from_cpu(unsigned int cpu)
-{
-    struct list_head *list = &per_cpu(tasklet_list, cpu);
-    unsigned long flags;
-    struct tasklet *t;
-
-    spin_lock_irqsave(&tasklet_lock, flags);
-
-    while ( !list_empty(list) )
-    {
-        t = list_entry(list->next, struct tasklet, list);
-        BUG_ON(t->scheduled_on != cpu);
-        t->scheduled_on = smp_processor_id();
-        list_del(&t->list);
-        list_add_tail(&t->list, &this_cpu(tasklet_list));
-    }
-
-    raise_softirq(TASKLET_SOFTIRQ);
-
-    spin_unlock_irqrestore(&tasklet_lock, flags);
-}
-
-void tasklet_init(
-    struct tasklet *t, void (*func)(unsigned long), unsigned long data)
-{
-    memset(t, 0, sizeof(*t));
-    INIT_LIST_HEAD(&t->list);
-    t->scheduled_on = -1;
-    t->func = func;
-    t->data = data;
-}
-
 void __init softirq_init(void)
 {
-    unsigned int cpu;
-
-    for_each_possible_cpu ( cpu )
-        INIT_LIST_HEAD(&per_cpu(tasklet_list, cpu));
-
-    open_softirq(TASKLET_SOFTIRQ, tasklet_action);
-
-    tasklets_initialised = 1;
 }
 
 /*
diff --git a/xen/common/tasklet.c b/xen/common/tasklet.c
new file mode 100644
index 0000000000..c7d692bfbc
--- /dev/null
+++ b/xen/common/tasklet.c
@@ -0,0 +1,170 @@
+/******************************************************************************
+ * tasklet.c
+ * 
+ * Dynamically-allocatable tasks run in softirq context on at most one CPU at
+ * a time.
+ * 
+ * Copyright (c) 2010, Citrix Systems, Inc.
+ * Copyright (c) 1992, Linus Torvalds
+ * 
+ * Authors:
+ *    Keir Fraser <keir.fraser@citrix.com>
+ */
+
+#include <xen/config.h>
+#include <xen/init.h>
+#include <xen/sched.h>
+#include <xen/softirq.h>
+#include <xen/tasklet.h>
+
+static bool_t tasklets_initialised;
+static DEFINE_PER_CPU(struct list_head, tasklet_list);
+static DEFINE_SPINLOCK(tasklet_lock);
+
+void tasklet_schedule_on_cpu(struct tasklet *t, unsigned int cpu)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&tasklet_lock, flags);
+
+    if ( tasklets_initialised && !t->is_dead )
+    {
+        t->scheduled_on = cpu;
+        if ( !t->is_running )
+        {
+            list_del(&t->list);
+            list_add_tail(&t->list, &per_cpu(tasklet_list, cpu));
+            cpu_raise_softirq(cpu, TASKLET_SOFTIRQ);
+        }
+    }
+
+    spin_unlock_irqrestore(&tasklet_lock, flags);
+}
+
+void tasklet_schedule(struct tasklet *t)
+{
+    tasklet_schedule_on_cpu(t, smp_processor_id());
+}
+
+static void tasklet_action(void)
+{
+    unsigned int cpu = smp_processor_id();
+    struct list_head *list = &per_cpu(tasklet_list, cpu);
+    struct tasklet *t;
+
+    spin_lock_irq(&tasklet_lock);
+
+    if ( list_empty(list) )
+    {
+        spin_unlock_irq(&tasklet_lock);
+        return;
+    }
+
+    t = list_entry(list->next, struct tasklet, list);
+    list_del_init(&t->list);
+
+    BUG_ON(t->is_dead || t->is_running || (t->scheduled_on != cpu));
+    t->scheduled_on = -1;
+    t->is_running = 1;
+
+    spin_unlock_irq(&tasklet_lock);
+    t->func(t->data);
+    spin_lock_irq(&tasklet_lock);
+
+    t->is_running = 0;
+
+    if ( t->scheduled_on >= 0 )
+    {
+        BUG_ON(t->is_dead || !list_empty(&t->list));
+        list_add_tail(&t->list, &per_cpu(tasklet_list, t->scheduled_on));
+        if ( t->scheduled_on != cpu )
+            cpu_raise_softirq(t->scheduled_on, TASKLET_SOFTIRQ);
+    }
+
+    /*
+     * If there is more work to do then reschedule. We don't grab more work
+     * immediately as we want to allow other softirq work to happen first.
+     */
+    if ( !list_empty(list) )
+        raise_softirq(TASKLET_SOFTIRQ);
+
+    spin_unlock_irq(&tasklet_lock);
+}
+
+void tasklet_kill(struct tasklet *t)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&tasklet_lock, flags);
+
+    if ( !list_empty(&t->list) )
+    {
+        BUG_ON(t->is_dead || t->is_running || (t->scheduled_on < 0));
+        list_del_init(&t->list);
+    }
+    t->scheduled_on = -1;
+    t->is_dead = 1;
+
+    while ( t->is_running )
+    {
+        spin_unlock_irqrestore(&tasklet_lock, flags);
+        cpu_relax();
+        spin_lock_irqsave(&tasklet_lock, flags);
+    }
+
+    spin_unlock_irqrestore(&tasklet_lock, flags);
+}
+
+void migrate_tasklets_from_cpu(unsigned int cpu)
+{
+    struct list_head *list = &per_cpu(tasklet_list, cpu);
+    unsigned long flags;
+    struct tasklet *t;
+
+    spin_lock_irqsave(&tasklet_lock, flags);
+
+    while ( !list_empty(list) )
+    {
+        t = list_entry(list->next, struct tasklet, list);
+        BUG_ON(t->scheduled_on != cpu);
+        t->scheduled_on = smp_processor_id();
+        list_del(&t->list);
+        list_add_tail(&t->list, &this_cpu(tasklet_list));
+    }
+
+    raise_softirq(TASKLET_SOFTIRQ);
+
+    spin_unlock_irqrestore(&tasklet_lock, flags);
+}
+
+void tasklet_init(
+    struct tasklet *t, void (*func)(unsigned long), unsigned long data)
+{
+    memset(t, 0, sizeof(*t));
+    INIT_LIST_HEAD(&t->list);
+    t->scheduled_on = -1;
+    t->func = func;
+    t->data = data;
+}
+
+void __init tasklet_subsys_init(void)
+{
+    unsigned int cpu;
+
+    for_each_possible_cpu ( cpu )
+        INIT_LIST_HEAD(&per_cpu(tasklet_list, cpu));
+
+    open_softirq(TASKLET_SOFTIRQ, tasklet_action);
+
+    tasklets_initialised = 1;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/xen/common/trace.c b/xen/common/trace.c
index cd1f0fb7f6..110f3b8c7d 100644
--- a/xen/common/trace.c
+++ b/xen/common/trace.c
@@ -25,7 +25,7 @@
 #include <xen/trace.h>
 #include <xen/errno.h>
 #include <xen/event.h>
-#include <xen/softirq.h>
+#include <xen/tasklet.h>
 #include <xen/init.h>
 #include <xen/mm.h>
 #include <xen/percpu.h>
diff --git a/xen/drivers/char/console.c b/xen/drivers/char/console.c
index 56500ad681..354ff86b73 100644
--- a/xen/drivers/char/console.c
+++ b/xen/drivers/char/console.c
@@ -21,6 +21,7 @@
 #include <xen/console.h>
 #include <xen/serial.h>
 #include <xen/softirq.h>
+#include <xen/tasklet.h>
 #include <xen/keyhandler.h>
 #include <xen/mm.h>
 #include <xen/delay.h>
diff --git a/xen/drivers/passthrough/io.c b/xen/drivers/passthrough/io.c
index cac0045054..8e14af75a2 100644
--- a/xen/drivers/passthrough/io.c
+++ b/xen/drivers/passthrough/io.c
@@ -24,6 +24,7 @@
 #include <asm/hvm/iommu.h>
 #include <asm/hvm/support.h>
 #include <xen/hvm/irq.h>
+#include <xen/tasklet.h>
 
 static void hvm_dirq_assist(unsigned long _d);
 
diff --git a/xen/drivers/passthrough/pci.c b/xen/drivers/passthrough/pci.c
index bc6974c122..23dc95ca4e 100644
--- a/xen/drivers/passthrough/pci.c
+++ b/xen/drivers/passthrough/pci.c
@@ -25,7 +25,7 @@
 #include <asm/hvm/irq.h>
 #include <xen/delay.h>
 #include <xen/keyhandler.h>
-
+#include <xen/tasklet.h>
 
 LIST_HEAD(alldevs_list);
 spinlock_t pcidevs_lock = SPIN_LOCK_UNLOCKED;
diff --git a/xen/drivers/passthrough/vtd/iommu.c b/xen/drivers/passthrough/vtd/iommu.c
index 1c47d5c6c8..f217dfc978 100644
--- a/xen/drivers/passthrough/vtd/iommu.c
+++ b/xen/drivers/passthrough/vtd/iommu.c
@@ -26,6 +26,7 @@
 #include <xen/iommu.h>
 #include <asm/hvm/iommu.h>
 #include <xen/numa.h>
+#include <xen/softirq.h>
 #include <xen/time.h>
 #include <xen/pci.h>
 #include <xen/pci_regs.h>
diff --git a/xen/include/asm-x86/hvm/vcpu.h b/xen/include/asm-x86/hvm/vcpu.h
index 840625040f..ed6b250ff9 100644
--- a/xen/include/asm-x86/hvm/vcpu.h
+++ b/xen/include/asm-x86/hvm/vcpu.h
@@ -20,6 +20,7 @@
 #ifndef __ASM_X86_HVM_VCPU_H__
 #define __ASM_X86_HVM_VCPU_H__
 
+#include <xen/tasklet.h>
 #include <asm/hvm/io.h>
 #include <asm/hvm/vlapic.h>
 #include <asm/hvm/vmx/vmcs.h>
diff --git a/xen/include/asm-x86/hvm/vlapic.h b/xen/include/asm-x86/hvm/vlapic.h
index 880754d2ab..05bd719da1 100644
--- a/xen/include/asm-x86/hvm/vlapic.h
+++ b/xen/include/asm-x86/hvm/vlapic.h
@@ -21,7 +21,7 @@
 #ifndef __ASM_X86_HVM_VLAPIC_H__
 #define __ASM_X86_HVM_VLAPIC_H__
 
-#include <xen/softirq.h>
+#include <xen/tasklet.h>
 #include <asm/msr.h>
 #include <public/hvm/ioreq.h>
 #include <asm/hvm/vpt.h>
diff --git a/xen/include/xen/hvm/irq.h b/xen/include/xen/hvm/irq.h
index c1747ed73c..f21b02ceeb 100644
--- a/xen/include/xen/hvm/irq.h
+++ b/xen/include/xen/hvm/irq.h
@@ -24,7 +24,7 @@
 
 #include <xen/types.h>
 #include <xen/spinlock.h>
-#include <xen/softirq.h>
+#include <xen/tasklet.h>
 #include <asm/irq.h>
 #include <public/hvm/save.h>
 
diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
index a1baa78e69..f0adc55284 100644
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -20,6 +20,7 @@
 #include <xen/rcupdate.h>
 #include <xen/irq.h>
 #include <xen/mm.h>
+#include <xen/tasklet.h>
 #include <public/mem_event.h>
 
 #ifdef CONFIG_COMPAT
diff --git a/xen/include/xen/softirq.h b/xen/include/xen/softirq.h
index 0e3c4ea5f2..cadf15a010 100644
--- a/xen/include/xen/softirq.h
+++ b/xen/include/xen/softirq.h
@@ -40,28 +40,4 @@ void raise_softirq(unsigned int nr);
  */
 void process_pending_softirqs(void);
 
-/*
- * TASKLETS -- dynamically-allocatable tasks run in softirq context
- * on at most one CPU at a time.
- */
-struct tasklet
-{
-    struct list_head list;
-    int scheduled_on;
-    bool_t is_running;
-    bool_t is_dead;
-    void (*func)(unsigned long);
-    unsigned long data;
-};
-
-#define DECLARE_TASKLET(name, func, data) \
-    struct tasklet name = { LIST_HEAD_INIT(name.list), -1, 0, 0, func, data }
-
-void tasklet_schedule_on_cpu(struct tasklet *t, unsigned int cpu);
-void tasklet_schedule(struct tasklet *t);
-void tasklet_kill(struct tasklet *t);
-void migrate_tasklets_from_cpu(unsigned int cpu);
-void tasklet_init(
-    struct tasklet *t, void (*func)(unsigned long), unsigned long data);
-
 #endif /* __XEN_SOFTIRQ_H__ */
diff --git a/xen/include/xen/tasklet.h b/xen/include/xen/tasklet.h
new file mode 100644
index 0000000000..9187462a09
--- /dev/null
+++ b/xen/include/xen/tasklet.h
@@ -0,0 +1,35 @@
+/******************************************************************************
+ * tasklet.h
+ * 
+ * Dynamically-allocatable tasks run in softirq context on at most one CPU at
+ * a time.
+ */
+
+#ifndef __XEN_TASKLET_H__
+#define __XEN_TASKLET_H__
+
+#include <xen/types.h>
+#include <xen/list.h>
+
+struct tasklet
+{
+    struct list_head list;
+    int scheduled_on;
+    bool_t is_running;
+    bool_t is_dead;
+    void (*func)(unsigned long);
+    unsigned long data;
+};
+
+#define DECLARE_TASKLET(name, func, data) \
+    struct tasklet name = { LIST_HEAD_INIT(name.list), -1, 0, 0, func, data }
+
+void tasklet_schedule_on_cpu(struct tasklet *t, unsigned int cpu);
+void tasklet_schedule(struct tasklet *t);
+void tasklet_kill(struct tasklet *t);
+void migrate_tasklets_from_cpu(unsigned int cpu);
+void tasklet_init(
+    struct tasklet *t, void (*func)(unsigned long), unsigned long data);
+void tasklet_subsys_init(void);
+
+#endif /* __XEN_TASKLET_H__ */