x2APIC: improve enabling logic

This patch masks PIC and IOAPIC RTE's before x2APIC enabling, unmask
and restore them after x2APIC enabling. It also really enables
interrupt remapping before x2APIC enabling instead of just checking
interrupt remapping setting. This patch also handles all x2APIC
configuration including BIOS settings and command line
settings. Especially, it handles that BIOS hands over in x2APIC mode
(when there is apic id > 255). It checks if x2APIC is already enabled
by BIOS. If already enabled, it will disable interrupt remapping and
queued invalidation first, then enable them again.

Signed-off-by: Weidong Han <weidong.han@intel.com>
xen-unstable changeset:   21718:34f612ed4184
xen-unstable date:        Mon Jul 05 08:31:29 2010 +0100

11 files changed, 431 insertions, 25 deletions

diff --git a/xen/arch/x86/apic.c b/xen/arch/x86/apic.c
index 51ba81dce9..c02d50ebbf 100644
--- a/xen/arch/x86/apic.c
+++ b/xen/arch/x86/apic.c
@@ -70,6 +70,9 @@ int apic_verbosity;
 int x2apic_enabled __read_mostly = 0;
 int directed_eoi_enabled __read_mostly = 0;
 
+/* x2APIC is enabled in BIOS */
+static int x2apic_preenabled;
+
 /*
  * The following vectors are part of the Linux architecture, there
  * is no hardware IRQ pin equivalent for them, they are triggered
@@ -490,6 +493,47 @@ static void apic_pm_activate(void)
     apic_pm_state.active = 1;
 }
 
+static void resume_x2apic(void)
+{
+    uint64_t msr_content;
+    struct IO_APIC_route_entry **ioapic_entries = NULL;
+
+    ASSERT(x2apic_enabled);
+
+    ioapic_entries = alloc_ioapic_entries();
+    if ( !ioapic_entries )
+    {
+        printk("Allocate ioapic_entries failed\n");
+        goto out;
+    }
+
+    if ( save_IO_APIC_setup(ioapic_entries) )
+    {
+        printk("Saving IO-APIC state failed\n");
+        goto out;
+    }
+
+    mask_8259A();
+    mask_IO_APIC_setup(ioapic_entries);
+
+    iommu_enable_IR();
+
+    rdmsrl(MSR_IA32_APICBASE, msr_content);
+    if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
+    {
+        msr_content |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
+        msr_content = (uint32_t)msr_content;
+        wrmsrl(MSR_IA32_APICBASE, msr_content);
+    }
+
+    restore_IO_APIC_setup(ioapic_entries);
+    unmask_8259A();
+
+out:
+    if ( ioapic_entries )
+        free_ioapic_entries(ioapic_entries);
+}
+
 void __devinit setup_local_APIC(void)
 {
     unsigned long oldvalue, value, ver, maxlvt;
@@ -728,7 +772,7 @@ int lapic_resume(void)
         wrmsr(MSR_IA32_APICBASE, l, h);
     }
     else
-        enable_x2apic();
+        resume_x2apic();
 
     apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
     apic_write(APIC_ID, apic_pm_state.apic_id);
@@ -892,45 +936,152 @@ no_apic:
     return -1;
 }
 
-void enable_x2apic(void)
+void check_x2apic_preenabled(void)
 {
     u32 lo, hi;
 
-    if ( smp_processor_id() == 0 )
+    if ( !x2apic_is_available() )
+        return;
+
+    rdmsr(MSR_IA32_APICBASE, lo, hi);
+    if ( lo & MSR_IA32_APICBASE_EXTD )
     {
-        if ( !iommu_supports_eim() )
+        printk("x2APIC mode is already enabled by BIOS.\n");
+        x2apic_preenabled = 1;
+        x2apic_enabled = 1;
+    }
+}
+
+static void enable_bsp_x2apic(void)
+{
+    struct IO_APIC_route_entry **ioapic_entries = NULL;
+    const struct genapic *x2apic_genapic = NULL;
+
+    ASSERT(smp_processor_id() == 0);
+
+    if ( x2apic_preenabled )
+    {
+        /*
+         * Interrupt remapping should be also enabled by BIOS when
+         * x2APIC is already enabled by BIOS, otherwise it's a BIOS
+         * bug
+         */
+        if ( !intremap_enabled() )
+            panic("Interrupt remapping is not enabled by BIOS while "
+                  "x2APIC is already enabled by BIOS!\n");
+    }
+
+    x2apic_genapic = apic_x2apic_probe();
+    if ( x2apic_genapic )
+        genapic = x2apic_genapic;
+    else
+    {
+        if ( x2apic_cmdline_disable() )
         {
-            printk("x2APIC would not be enabled without EIM.\n");
-            return;
+            if ( x2apic_preenabled )
+            {
+                /* Ignore x2apic=0, and set default x2apic mode */
+                genapic = &apic_x2apic_cluster;
+                printk("x2APIC: already enabled by BIOS, ignore x2apic=0.\n");
+            }
+            else
+            {
+                printk("Not enable x2APIC due to x2apic=0 is set.\n");
+                return;
+            }
         }
-
-        if ( apic_x2apic_phys.probe() )
-            genapic = &apic_x2apic_phys;
-        else if ( apic_x2apic_cluster.probe() )
-            genapic = &apic_x2apic_cluster;
         else
         {
-            printk("x2APIC would not be enabled due to x2apic=off.\n");
-            return;
+            if ( !iommu_enabled || !iommu_intremap || !iommu_qinval )
+                panic("Cannot enable x2APIC due to iommu or interrupt "
+                      "remapping or queued invalidation is disabled "
+                      "by command line!\n");
+            else
+            {
+                if ( x2apic_preenabled )
+                    panic("x2APIC: already enabled by BIOS, but "
+                          "iommu_supports_eim fails\n");
+                else
+                {
+                    printk("Not enable x2APIC due to "
+                           "iommu_supports_eim fails!\n");
+                    return;
+                }
+            }
         }
+    }
 
-        x2apic_enabled = 1;
-        printk("Switched to APIC driver %s.\n", genapic->name);
+    ioapic_entries = alloc_ioapic_entries();
+    if ( !ioapic_entries )
+    {
+        printk("Allocate ioapic_entries failed\n");
+        goto out;
     }
-    else
+
+    if ( save_IO_APIC_setup(ioapic_entries) )
     {
-        BUG_ON(!x2apic_enabled); /* APs only enable x2apic when BSP did so. */
+        printk("Saving IO-APIC state failed\n");
+        goto out;
     }
 
+    mask_8259A();
+    mask_IO_APIC_setup(ioapic_entries);
+
+    if ( iommu_enable_IR() )
+    {
+        printk("Would not enable x2APIC due to interrupt remapping "
+               "cannot be enabled.\n");
+        goto restore_out;
+    }
+
+    x2apic_enabled = 1;
+    printk("Switched to APIC driver %s.\n", genapic->name);
+
+    if ( !x2apic_preenabled )
+    {
+        u32 lo, hi;
+
+        rdmsr(MSR_IA32_APICBASE, lo, hi);
+        if ( !(lo & MSR_IA32_APICBASE_EXTD) )
+        {
+            lo |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
+            wrmsr(MSR_IA32_APICBASE, lo, 0);
+            printk("x2APIC mode enabled.\n");
+        }
+    }
+
+restore_out:
+    restore_IO_APIC_setup(ioapic_entries);
+    unmask_8259A();
+
+out:
+    if ( ioapic_entries )
+        free_ioapic_entries(ioapic_entries);
+}
+
+static void enable_ap_x2apic(void)
+{
+    u32 lo, hi;
+
+    ASSERT(smp_processor_id() != 0);
+
+    /* APs only enable x2apic when BSP did so. */
+    BUG_ON(!x2apic_enabled);
+
     rdmsr(MSR_IA32_APICBASE, lo, hi);
     if ( !(lo & MSR_IA32_APICBASE_EXTD) )
     {
         lo |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
         wrmsr(MSR_IA32_APICBASE, lo, 0);
-        printk("x2APIC mode enabled.\n");
     }
+}
+
+void enable_x2apic(void)
+{
+    if ( smp_processor_id() == 0 )
+        enable_bsp_x2apic();
     else
-        printk("x2APIC mode enabled by BIOS.\n");
+        enable_ap_x2apic();
 }
 
 void __init init_apic_mappings(void)
diff --git a/xen/arch/x86/genapic/x2apic.c b/xen/arch/x86/genapic/x2apic.c
index 2cb7835060..882caea8bd 100644
--- a/xen/arch/x86/genapic/x2apic.c
+++ b/xen/arch/x86/genapic/x2apic.c
@@ -29,6 +29,11 @@ boolean_param("x2apic", x2apic);
 static int  x2apic_phys; /* By default we use logical cluster mode. */
 boolean_param("x2apic_phys", x2apic_phys);
 
+int x2apic_cmdline_disable(void)
+{
+    return (x2apic == 0);
+}
+
 static int probe_x2apic_phys(void)
 {
     return x2apic && x2apic_phys && x2apic_is_available() &&
@@ -51,6 +56,20 @@ const struct genapic apic_x2apic_cluster = {
     GENAPIC_X2APIC_CLUSTER
 };
 
+const struct genapic *apic_x2apic_probe(void)
+{
+    if ( !x2apic || !x2apic_is_available() )
+        return NULL;
+
+    if ( !iommu_supports_eim() )
+        return NULL;
+
+    if ( x2apic_phys )
+        return &apic_x2apic_phys;
+    else
+        return &apic_x2apic_cluster;
+}
+
 void init_apic_ldr_x2apic_phys(void)
 {
     return;
diff --git a/xen/arch/x86/i8259.c b/xen/arch/x86/i8259.c
index 2474f7864d..bd9c956ef2 100644
--- a/xen/arch/x86/i8259.c
+++ b/xen/arch/x86/i8259.c
@@ -175,6 +175,26 @@ int i8259A_irq_pending(unsigned int irq)
     return ret;
 }
 
+void mask_8259A(void)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&i8259A_lock, flags);
+    outb(0xff, 0xA1);
+    outb(0xff, 0x21);
+    spin_unlock_irqrestore(&i8259A_lock, flags);
+}
+
+void unmask_8259A(void)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&i8259A_lock, flags);
+    outb(cached_A1, 0xA1);
+    outb(cached_21, 0x21);
+    spin_unlock_irqrestore(&i8259A_lock, flags);
+}
+
 /*
  * This function assumes to be called rarely. Switching between
  * 8259A registers is slow.
diff --git a/xen/arch/x86/io_apic.c b/xen/arch/x86/io_apic.c
index f081cf8754..4fc461a954 100644
--- a/xen/arch/x86/io_apic.c
+++ b/xen/arch/x86/io_apic.c
@@ -136,6 +136,126 @@ static void __init replace_pin_at_irq(unsigned int irq,
     }
 }
 
+struct IO_APIC_route_entry **alloc_ioapic_entries(void)
+{
+    int apic;
+    struct IO_APIC_route_entry **ioapic_entries;
+
+    ioapic_entries = xmalloc_array(struct IO_APIC_route_entry *, nr_ioapics);
+    if (!ioapic_entries)
+        return 0;
+
+    for (apic = 0; apic < nr_ioapics; apic++) {
+        ioapic_entries[apic] =
+            xmalloc_array(struct IO_APIC_route_entry,
+                          nr_ioapic_registers[apic]);
+        if (!ioapic_entries[apic])
+            goto nomem;
+    }
+
+    return ioapic_entries;
+
+nomem:
+    while (--apic >= 0)
+        xfree(ioapic_entries[apic]);
+    xfree(ioapic_entries);
+
+    return 0;
+}
+
+/*
+ * Saves all the IO-APIC RTE's
+ */
+int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
+{
+    int apic, pin;
+
+    if (!ioapic_entries)
+        return -ENOMEM;
+
+    for (apic = 0; apic < nr_ioapics; apic++) {
+        if (!ioapic_entries[apic])
+            return -ENOMEM;
+
+        for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
+            *(((int *)&ioapic_entries[apic][pin])+0) =
+                __io_apic_read(apic, 0x10+pin*2);
+            *(((int *)&ioapic_entries[apic][pin])+1) =
+                __io_apic_read(apic, 0x11+pin*2);
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Mask all IO APIC entries.
+ */
+void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
+{
+    int apic, pin;
+
+    if (!ioapic_entries)
+        return;
+
+    for (apic = 0; apic < nr_ioapics; apic++) {
+        if (!ioapic_entries[apic])
+            break;
+
+        for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
+            struct IO_APIC_route_entry entry;
+            unsigned long flags;
+
+            entry = ioapic_entries[apic][pin];
+            if (!entry.mask) {
+                entry.mask = 1;
+
+                spin_lock_irqsave(&ioapic_lock, flags);
+                __io_apic_write(apic, 0x11+2*pin, *(((int *)&entry)+1));
+                __io_apic_write(apic, 0x10+2*pin, *(((int *)&entry)+0));
+                spin_unlock_irqrestore(&ioapic_lock, flags);
+            }
+        }
+    }
+}
+
+/*
+ * Restore IO APIC entries which was saved in ioapic_entries.
+ */
+int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
+{
+    int apic, pin;
+    unsigned long flags;
+    struct IO_APIC_route_entry entry;
+
+    if (!ioapic_entries)
+        return -ENOMEM;
+
+    for (apic = 0; apic < nr_ioapics; apic++) {
+        if (!ioapic_entries[apic])
+            return -ENOMEM;
+
+        for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
+            entry = ioapic_entries[apic][pin];
+            spin_lock_irqsave(&ioapic_lock, flags);
+            __io_apic_write(apic, 0x11+2*pin, *(((int *)&entry)+1));
+            __io_apic_write(apic, 0x10+2*pin, *(((int *)&entry)+0));
+            spin_unlock_irqrestore(&ioapic_lock, flags);
+    }
+
+    return 0;
+}
+
+void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
+{
+    int apic;
+
+    for (apic = 0; apic < nr_ioapics; apic++)
+        xfree(ioapic_entries[apic]);
+
+    xfree(ioapic_entries);
+}
+
 static void __modify_IO_APIC_irq (unsigned int irq, unsigned long enable, unsigned long disable)
 {
     struct irq_pin_list *entry = irq_2_pin + irq;
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 68be297eec..f1557fc68a 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -979,6 +979,9 @@ void __init __start_xen(unsigned long mbi_p)
 
     tboot_probe();
 
+    /* Check if x2APIC is already enabled in BIOS */
+    check_x2apic_preenabled();
+
     /* Unmap the first page of CPU0's stack. */
     memguard_guard_stack(cpu0_stack);
 
@@ -997,9 +1000,6 @@ void __init __start_xen(unsigned long mbi_p)
 
     acpi_boot_init();
 
-    if ( x2apic_is_available() )
-        enable_x2apic();
-
     init_cpu_to_node();
 
     if ( smp_found_config )
@@ -1012,6 +1012,9 @@ void __init __start_xen(unsigned long mbi_p)
 
     init_apic_mappings();
 
+    if ( x2apic_is_available() )
+        enable_x2apic();
+
     percpu_free_unused_areas();
 
     init_IRQ();
diff --git a/xen/drivers/passthrough/vtd/intremap.c b/xen/drivers/passthrough/vtd/intremap.c
index d3bb96a8a9..1c2ff1bf67 100644
--- a/xen/drivers/passthrough/vtd/intremap.c
+++ b/xen/drivers/passthrough/vtd/intremap.c
@@ -132,6 +132,12 @@ int iommu_supports_eim(void)
     if ( !iommu_enabled || !iommu_qinval || !iommu_intremap )
         return 0;
 
+    if ( list_empty(&acpi_drhd_units) )
+    {
+        dprintk(XENLOG_WARNING VTDPREFIX, "VT-d is not supported\n");
+        return 0;
+    }
+
     /* We MUST have a DRHD unit for each IOAPIC. */
     for ( apic = 0; apic < nr_ioapics; apic++ )
         if ( !ioapic_to_drhd(IO_APIC_ID(apic)) )
@@ -142,9 +148,6 @@ int iommu_supports_eim(void)
             return 0;
     }
 
-    if ( list_empty(&acpi_drhd_units) )
-        return 0;
-
     for_each_drhd_unit ( drhd )
         if ( !ecap_queued_inval(drhd->iommu->ecap) ||
              !ecap_intr_remap(drhd->iommu->ecap) ||
@@ -802,3 +805,80 @@ void disable_intremap(struct iommu *iommu)
 out:
     spin_unlock_irqrestore(&iommu->register_lock, flags);
 }
+
+/*
+ * This function is used to enable Interrutp remapping when
+ * enable x2apic
+ */
+int iommu_enable_IR(void)
+{
+    struct acpi_drhd_unit *drhd;
+    struct iommu *iommu;
+
+    if ( !iommu_supports_eim() )
+        return -1;
+
+    for_each_drhd_unit ( drhd )
+    {
+        struct qi_ctrl *qi_ctrl = NULL;
+
+        iommu = drhd->iommu;
+        qi_ctrl = iommu_qi_ctrl(iommu);
+
+        /* Clear previous faults */
+        clear_fault_bits(iommu);
+
+        /*
+         * Disable interrupt remapping and queued invalidation if
+         * already enabled by BIOS
+         */
+        disable_intremap(iommu);
+        disable_qinval(iommu);
+    }
+
+    /* Enable queue invalidation */
+    for_each_drhd_unit ( drhd )
+    {
+        iommu = drhd->iommu;
+        if ( enable_qinval(iommu) != 0 )
+        {
+            dprintk(XENLOG_INFO VTDPREFIX,
+                    "Failed to enable Queued Invalidation!\n");
+            return -1;
+        }
+    }
+
+    /* Enable interrupt remapping */
+    for_each_drhd_unit ( drhd )
+    {
+        iommu = drhd->iommu;
+        if ( enable_intremap(iommu, 1) )
+        {
+            dprintk(XENLOG_INFO VTDPREFIX,
+                    "Failed to enable Interrupt Remapping!\n");
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Check if interrupt remapping is enabled or not
+ * return 1: enabled
+ * return 0: not enabled
+ */
+int intremap_enabled(void)
+{
+    struct acpi_drhd_unit *drhd;
+    u32 sts;
+
+    for_each_drhd_unit ( drhd )
+    {
+        sts = dmar_readl(drhd->iommu->reg, DMAR_GSTS_REG);
+        if ( !(sts & DMA_GSTS_IRES) )
+            return 0;
+    }
+
+    return 1;
+}
diff --git a/xen/include/asm-x86/apic.h b/xen/include/asm-x86/apic.h
index 4bf4c70d43..9e10c89d4a 100644
--- a/xen/include/asm-x86/apic.h
+++ b/xen/include/asm-x86/apic.h
@@ -25,6 +25,8 @@ extern int apic_verbosity;
 extern int x2apic_enabled;
 extern int directed_eoi_enabled;
 
+extern void check_x2apic_preenabled(void);
+extern int x2apic_cmdline_disable(void);
 extern void enable_x2apic(void);
 
 static __inline int x2apic_is_available(void)
diff --git a/xen/include/asm-x86/genapic.h b/xen/include/asm-x86/genapic.h
index ef3c35fd51..bcd03aed84 100644
--- a/xen/include/asm-x86/genapic.h
+++ b/xen/include/asm-x86/genapic.h
@@ -70,6 +70,7 @@ cpumask_t vector_allocation_domain_flat(int cpu);
 	.send_IPI_mask = send_IPI_mask_flat, \
 	.send_IPI_self = send_IPI_self_flat
 
+const struct genapic *apic_x2apic_probe(void);
 void init_apic_ldr_x2apic_phys(void);
 void init_apic_ldr_x2apic_cluster(void);
 void clustered_apic_check_x2apic(void);
diff --git a/xen/include/asm-x86/io_apic.h b/xen/include/asm-x86/io_apic.h
index d546380435..7d577f39d6 100644
--- a/xen/include/asm-x86/io_apic.h
+++ b/xen/include/asm-x86/io_apic.h
@@ -199,6 +199,12 @@ extern int (*ioapic_renumber_irq)(int ioapic, int irq);
 extern void ioapic_suspend(void);
 extern void ioapic_resume(void);
 
+extern struct IO_APIC_route_entry **alloc_ioapic_entries(void);
+extern void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries);
+extern int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+extern void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+extern int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+
 #else  /* !CONFIG_X86_IO_APIC */
 static inline void init_ioapic_mappings(void) {}
 static inline void ioapic_suspend(void) {}
diff --git a/xen/include/asm-x86/irq.h b/xen/include/asm-x86/irq.h
index 66337ae363..de1e728171 100644
--- a/xen/include/asm-x86/irq.h
+++ b/xen/include/asm-x86/irq.h
@@ -91,6 +91,8 @@ asmlinkage void do_IRQ(struct cpu_user_regs *regs);
 void disable_8259A_irq(unsigned int irq);
 void enable_8259A_irq(unsigned int irq);
 int i8259A_irq_pending(unsigned int irq);
+void mask_8259A(void);
+void unmask_8259A(void);
 void init_8259A(int aeoi);
 int i8259A_suspend(void);
 int i8259A_resume(void);
diff --git a/xen/include/xen/iommu.h b/xen/include/xen/iommu.h
index 5181ffc5be..592a6b1b9e 100644
--- a/xen/include/xen/iommu.h
+++ b/xen/include/xen/iommu.h
@@ -58,6 +58,8 @@ struct iommu {
 
 int iommu_setup(void);
 int iommu_supports_eim(void);
+int iommu_enable_IR(void);
+int intremap_enabled(void);
 
 int iommu_add_device(struct pci_dev *pdev);
 int iommu_remove_device(struct pci_dev *pdev);