x86: Support x2APIC mode.

Signed-off-by: Dexuan Cui <dexuan.cui@intel.com>

21 files changed, 330 insertions, 68 deletions

diff --git a/xen/arch/x86/apic.c b/xen/arch/x86/apic.c
index c3ee1bd980..55a25bce39 100644
--- a/xen/arch/x86/apic.c
+++ b/xen/arch/x86/apic.c
@@ -47,6 +47,8 @@ int enable_local_apic __initdata = 0; /* -1=force-disable, +1=force-enable */
  */
 int apic_verbosity;
 
+int x2apic_enabled __read_mostly = 0;
+
 
 static void apic_pm_activate(void);
 
@@ -306,7 +308,10 @@ int __init verify_local_APIC(void)
      */
     reg0 = apic_read(APIC_LVR);
     apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg0);
-    apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK);
+
+    /* We don't try writing LVR in x2APIC mode since that incurs #GP. */
+    if ( !x2apic_enabled )
+        apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK);
     reg1 = apic_read(APIC_LVR);
     apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg1);
 
@@ -610,7 +615,8 @@ int lapic_suspend(void)
     apic_pm_state.apic_id = apic_read(APIC_ID);
     apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
     apic_pm_state.apic_ldr = apic_read(APIC_LDR);
-    apic_pm_state.apic_dfr = apic_read(APIC_DFR);
+    if ( !x2apic_enabled )
+        apic_pm_state.apic_dfr = apic_read(APIC_DFR);
     apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
     apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
     apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
@@ -643,14 +649,20 @@ int lapic_resume(void)
      * FIXME! This will be wrong if we ever support suspend on
      * SMP! We'll need to do this as part of the CPU restore!
      */
-    rdmsr(MSR_IA32_APICBASE, l, h);
-    l &= ~MSR_IA32_APICBASE_BASE;
-    l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
-    wrmsr(MSR_IA32_APICBASE, l, h);
+    if ( !x2apic_enabled )
+    {
+        rdmsr(MSR_IA32_APICBASE, l, h);
+        l &= ~MSR_IA32_APICBASE_BASE;
+        l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
+        wrmsr(MSR_IA32_APICBASE, l, h);
+    }
+    else
+        enable_x2apic();
 
     apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
     apic_write(APIC_ID, apic_pm_state.apic_id);
-    apic_write(APIC_DFR, apic_pm_state.apic_dfr);
+    if ( !x2apic_enabled )
+        apic_write(APIC_DFR, apic_pm_state.apic_dfr);
     apic_write(APIC_LDR, apic_pm_state.apic_ldr);
     apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
     apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
@@ -809,10 +821,29 @@ no_apic:
     return -1;
 }
 
+void enable_x2apic(void)
+{
+    u32 lo, hi;
+
+    rdmsr(MSR_IA32_APICBASE, lo, hi);
+    if ( !(lo & MSR_IA32_APICBASE_EXTD) )
+    {
+        lo |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
+        wrmsr(MSR_IA32_APICBASE, lo, 0);
+        printk("x2APIC mode enabled.\n");
+    }
+    else
+        printk("x2APIC mode enabled by BIOS.\n");
+
+    x2apic_enabled = 1;
+}
+
 void __init init_apic_mappings(void)
 {
     unsigned long apic_phys;
 
+    if ( x2apic_enabled )
+        goto __next;
     /*
      * If no local APIC can be found then set up a fake all
      * zeroes page to simulate the local APIC and another
@@ -828,12 +859,13 @@ void __init init_apic_mappings(void)
     apic_printk(APIC_VERBOSE, "mapped APIC to %08lx (%08lx)\n", APIC_BASE,
                 apic_phys);
 
+__next:
     /*
      * Fetch the APIC ID of the BSP in case we have a
      * default configuration (or the MP table is broken).
      */
     if (boot_cpu_physical_apicid == -1U)
-        boot_cpu_physical_apicid = GET_APIC_ID(apic_read(APIC_ID));
+        boot_cpu_physical_apicid = get_apic_id();
 
 #ifdef CONFIG_X86_IO_APIC
     {
@@ -1271,7 +1303,7 @@ int __init APIC_init_uniprocessor (void)
      * might be zero if read from MP tables. Get it from LAPIC.
      */
 #ifdef CONFIG_CRASH_DUMP
-    boot_cpu_physical_apicid = GET_APIC_ID(apic_read(APIC_ID));
+    boot_cpu_physical_apicid = get_apic_id();
 #endif
     phys_cpu_present_map = physid_mask_of_physid(boot_cpu_physical_apicid);
 
diff --git a/xen/arch/x86/genapic/Makefile b/xen/arch/x86/genapic/Makefile
index 83e406d4f7..d6a8717d87 100644
--- a/xen/arch/x86/genapic/Makefile
+++ b/xen/arch/x86/genapic/Makefile
@@ -1,4 +1,5 @@
 obj-y += bigsmp.o
+obj-y += x2apic.o
 obj-y += default.o
 obj-y += delivery.o
 obj-y += probe.o
diff --git a/xen/arch/x86/genapic/delivery.c b/xen/arch/x86/genapic/delivery.c
index e343220f73..6327f1c029 100644
--- a/xen/arch/x86/genapic/delivery.c
+++ b/xen/arch/x86/genapic/delivery.c
@@ -17,7 +17,7 @@ void init_apic_ldr_flat(void)
 
 	apic_write_around(APIC_DFR, APIC_DFR_FLAT);
 	val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
-	val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
+	val |= SET_xAPIC_LOGICAL_ID(1UL << smp_processor_id());
 	apic_write_around(APIC_LDR, val);
 }
 
diff --git a/xen/arch/x86/genapic/probe.c b/xen/arch/x86/genapic/probe.c
index eaf1df24c7..295037e8cf 100644
--- a/xen/arch/x86/genapic/probe.c
+++ b/xen/arch/x86/genapic/probe.c
@@ -14,6 +14,7 @@
 #include <asm/apicdef.h>
 #include <asm/genapic.h>
 
+extern struct genapic apic_x2apic;
 extern struct genapic apic_summit;
 extern struct genapic apic_bigsmp;
 extern struct genapic apic_default;
@@ -21,6 +22,7 @@ extern struct genapic apic_default;
 struct genapic *genapic;
 
 struct genapic *apic_probe[] __initdata = { 
+	&apic_x2apic, 
 	&apic_summit,
 	&apic_bigsmp, 
 	&apic_default,	/* must be last */
diff --git a/xen/arch/x86/genapic/x2apic.c b/xen/arch/x86/genapic/x2apic.c
new file mode 100644
index 0000000000..bfd3a286e3
--- /dev/null
+++ b/xen/arch/x86/genapic/x2apic.c
@@ -0,0 +1,79 @@
+/*
+ * x2APIC driver.
+ *
+ * Copyright (c) 2008, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ */
+
+#include <xen/cpumask.h>
+#include <asm/apicdef.h>
+#include <asm/genapic.h>
+#include <xen/smp.h>
+#include <asm/mach-default/mach_mpparse.h>
+
+__init int probe_x2apic(void)
+{
+    return x2apic_is_available();
+}
+
+struct genapic apic_x2apic= {
+    APIC_INIT("x2apic", probe_x2apic),
+    GENAPIC_X2APIC
+};
+
+void init_apic_ldr_x2apic(void)
+{
+    /* We only use physical delivery mode. */
+    return;
+}
+
+void clustered_apic_check_x2apic(void)
+{
+    /* We only use physical delivery mode. */
+    return;
+}
+
+cpumask_t target_cpus_x2apic(void)
+{
+    /* Deliver interrupts only to CPU0 for now */
+    return cpumask_of_cpu(0);
+}
+
+unsigned int cpu_mask_to_apicid_x2apic(cpumask_t cpumask)
+{
+    return cpu_physical_id(first_cpu(cpumask));
+}
+
+void send_IPI_mask_x2apic(cpumask_t cpumask, int vector)
+{
+    unsigned int query_cpu;
+    u32 cfg, dest;
+    unsigned long flags;
+
+    ASSERT(cpus_subset(cpumask, cpu_online_map));
+    ASSERT(!cpus_empty(cpumask));
+
+    local_irq_save(flags);
+
+    cfg = APIC_DM_FIXED | 0 /* no shorthand */ | APIC_DEST_PHYSICAL | vector;
+    for_each_cpu_mask(query_cpu, cpumask)
+    {
+        dest =  cpu_physical_id(query_cpu);
+        apic_icr_write(cfg, dest);
+    }
+
+    local_irq_restore(flags);
+}
+
diff --git a/xen/arch/x86/hvm/vlapic.c b/xen/arch/x86/hvm/vlapic.c
index 905de5a865..c31ff16327 100644
--- a/xen/arch/x86/hvm/vlapic.c
+++ b/xen/arch/x86/hvm/vlapic.c
@@ -171,7 +171,7 @@ int vlapic_match_logical_addr(struct vlapic *vlapic, uint8_t mda)
     int result = 0;
     uint8_t logical_id;
 
-    logical_id = GET_APIC_LOGICAL_ID(vlapic_get_reg(vlapic, APIC_LDR));
+    logical_id = GET_xAPIC_LOGICAL_ID(vlapic_get_reg(vlapic, APIC_LDR));
 
     switch ( vlapic_get_reg(vlapic, APIC_DFR) )
     {
@@ -484,7 +484,7 @@ void vlapic_EOI_set(struct vlapic *vlapic)
 static int vlapic_ipi(
     struct vlapic *vlapic, uint32_t icr_low, uint32_t icr_high)
 {
-    unsigned int dest =         GET_APIC_DEST_FIELD(icr_high);
+    unsigned int dest =         GET_xAPIC_DEST_FIELD(icr_high);
     unsigned int short_hand =   icr_low & APIC_SHORT_MASK;
     unsigned int trig_mode =    icr_low & APIC_INT_LEVELTRIG;
     unsigned int level =        icr_low & APIC_INT_ASSERT;
diff --git a/xen/arch/x86/io_apic.c b/xen/arch/x86/io_apic.c
index c8795517b6..c661320663 100644
--- a/xen/arch/x86/io_apic.c
+++ b/xen/arch/x86/io_apic.c
@@ -1125,7 +1125,7 @@ void disable_IO_APIC(void)
         entry.delivery_mode   = dest_ExtINT; /* ExtInt */
         entry.vector          = 0;
         entry.dest.physical.physical_dest =
-            GET_APIC_ID(apic_read(APIC_ID));
+            get_apic_id();
 
         /*
          * Add it to the IO-APIC irq-routing table:
diff --git a/xen/arch/x86/mpparse.c b/xen/arch/x86/mpparse.c
index 0d14f35c77..5f2be943e2 100644
--- a/xen/arch/x86/mpparse.c
+++ b/xen/arch/x86/mpparse.c
@@ -814,12 +814,15 @@ void __init find_smp_config (void)
 void __init mp_register_lapic_address (
 	u64			address)
 {
-	mp_lapic_addr = (unsigned long) address;
+    if ( !x2apic_enabled )
+    {
+    	mp_lapic_addr = (unsigned long) address;
 
-	set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr);
+	    set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr);
+    }
 
 	if (boot_cpu_physical_apicid == -1U)
-		boot_cpu_physical_apicid = GET_APIC_ID(apic_read(APIC_ID));
+		boot_cpu_physical_apicid = get_apic_id();
 
 	Dprintk("Boot CPU = %d\n", boot_cpu_physical_apicid);
 }
diff --git a/xen/arch/x86/nmi.c b/xen/arch/x86/nmi.c
index ad83f8beb3..7b69e71264 100644
--- a/xen/arch/x86/nmi.c
+++ b/xen/arch/x86/nmi.c
@@ -434,14 +434,13 @@ void nmi_watchdog_tick(struct cpu_user_regs * regs)
  */
 static void do_nmi_trigger(unsigned char key)
 {
-    u32 id = GET_APIC_ID(apic_read(APIC_ID));
+    u32 id = get_apic_id();
 
     printk("Triggering NMI on APIC ID %x\n", id);
 
     local_irq_disable();
     apic_wait_icr_idle();
-    apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(id));
-    apic_write_around(APIC_ICR, APIC_DM_NMI | APIC_DEST_PHYSICAL);
+    apic_icr_write(APIC_DM_NMI | APIC_DEST_PHYSICAL, id);
     local_irq_enable();
 }
 
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 362cfb69e7..c71cfd045f 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -890,6 +890,9 @@ void __init __start_xen(unsigned long mbi_p)
 
     generic_apic_probe();
 
+    if ( x2apic_is_available() )
+        enable_x2apic();
+
     acpi_boot_init();
 
     init_cpu_to_node();
diff --git a/xen/arch/x86/shutdown.c b/xen/arch/x86/shutdown.c
index 760b260bc2..74f7075e5f 100644
--- a/xen/arch/x86/shutdown.c
+++ b/xen/arch/x86/shutdown.c
@@ -209,7 +209,7 @@ void machine_restart(void)
     local_irq_enable();
 
     /* Ensure we are the boot CPU. */
-    if ( GET_APIC_ID(apic_read(APIC_ID)) != boot_cpu_physical_apicid )
+    if ( get_apic_id() != boot_cpu_physical_apicid )
     {
         /* Send IPI to the boot CPU (logical cpu 0). */
         on_selected_cpus(cpumask_of_cpu(0), (void *)machine_restart,
diff --git a/xen/arch/x86/smp.c b/xen/arch/x86/smp.c
index c000c26b64..497468bf8b 100644
--- a/xen/arch/x86/smp.c
+++ b/xen/arch/x86/smp.c
@@ -72,11 +72,14 @@ static inline int __prepare_ICR (unsigned int shortcut, int vector)
 
 static inline int __prepare_ICR2 (unsigned int mask)
 {
-    return SET_APIC_DEST_FIELD(mask);
+    return SET_xAPIC_DEST_FIELD(mask);
 }
 
 void apic_wait_icr_idle(void)
 {
+    if ( x2apic_enabled )
+        return;
+
     while ( apic_read( APIC_ICR ) & APIC_ICR_BUSY )
         cpu_relax();
 }
diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index f5a705335f..1d7c81a85d 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -325,10 +325,13 @@ void __devinit smp_callin(void)
 	 */
 	wait_for_init_deassert(&init_deasserted);
 
+	if ( x2apic_is_available() )
+		enable_x2apic();
+
 	/*
 	 * (This works even if the APIC is not enabled.)
 	 */
-	phys_id = GET_APIC_ID(apic_read(APIC_ID));
+	phys_id = get_apic_id();
 	cpuid = smp_processor_id();
 	if (cpu_isset(cpuid, cpu_callin_map)) {
 		printk("huh, phys CPU#%d, CPU#%d already present??\n",
@@ -548,7 +551,7 @@ u32 cpu_2_logical_apicid[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = BAD_APIC
 static void map_cpu_to_logical_apicid(void)
 {
 	int cpu = smp_processor_id();
-	int apicid = hard_smp_processor_id();
+	int apicid = logical_smp_processor_id();
 
 	cpu_2_logical_apicid[cpu] = apicid;
 }
@@ -575,8 +578,7 @@ static inline void __inquire_remote_apic(int apicid)
 		 */
 		apic_wait_icr_idle();
 
-		apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
-		apic_write_around(APIC_ICR, APIC_DM_REMRD | regs[i]);
+		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
 
 		timeout = 0;
 		do {
@@ -597,6 +599,21 @@ static inline void __inquire_remote_apic(int apicid)
 #endif
 
 #ifdef WAKE_SECONDARY_VIA_NMI
+
+static int logical_apicid_to_cpu(int logical_apicid)
+{
+	int i;
+
+	for ( i = 0; i < sizeof(cpu_2_logical_apicid); i++ )
+		if ( cpu_2_logical_apicid[i] == logical_apicid )
+			break;
+
+	if ( i == sizeof(cpu_2_logical_apicid) );
+		i = -1; /* not found */
+
+	return i;
+}
+
 /* 
  * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
  * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
@@ -607,20 +624,26 @@ wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
 {
 	unsigned long send_status = 0, accept_status = 0;
 	int timeout, maxlvt;
+	int dest_cpu;
+	u32 dest;
 
-	/* Target chip */
-	apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(logical_apicid));
+	dest_cpu = logical_apicid_to_cpu(logical_apicid);
+	BUG_ON(dest_cpu == -1);
+
+	dest = cpu_physical_id(dest_cpu);
 
 	/* Boot on the stack */
-	/* Kick the second */
-	apic_write_around(APIC_ICR, APIC_DM_NMI | APIC_DEST_LOGICAL);
+	apic_icr_write(APIC_DM_NMI | APIC_DEST_PHYSICAL, dest_cpu);
 
 	Dprintk("Waiting for send to finish...\n");
 	timeout = 0;
 	do {
 		Dprintk("+");
 		udelay(100);
-		send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
+		if ( !x2apic_enabled )
+			send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
+		else
+			send_status = 0; /* We go out of the loop directly. */
 	} while (send_status && (timeout++ < 1000));
 
 	/*
@@ -666,40 +689,37 @@ wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
 	Dprintk("Asserting INIT.\n");
 
 	/*
-	 * Turn INIT on target chip
+	 * Turn INIT on target chip via IPI
 	 */
-	apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
-
-	/*
-	 * Send IPI
-	 */
-	apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
-				| APIC_DM_INIT);
+	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
+			        phys_apicid);
 
 	Dprintk("Waiting for send to finish...\n");
 	timeout = 0;
 	do {
 		Dprintk("+");
 		udelay(100);
-		send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
+		if ( !x2apic_enabled )
+			send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
+		else
+			send_status = 0; /* We go out of the loop dirctly. */
 	} while (send_status && (timeout++ < 1000));
 
 	mdelay(10);
 
 	Dprintk("Deasserting INIT.\n");
 
-	/* Target chip */
-	apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
-
-	/* Send IPI */
-	apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);
+	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
 
 	Dprintk("Waiting for send to finish...\n");
 	timeout = 0;
 	do {
 		Dprintk("+");
 		udelay(100);
-		send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
+		if ( !x2apic_enabled )
+			send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
+		else
+			send_status = 0; /* We go out of the loop dirctly. */
 	} while (send_status && (timeout++ < 1000));
 
 	atomic_set(&init_deasserted, 1);
@@ -731,15 +751,9 @@ wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
 
 		/*
 		 * STARTUP IPI
+		 * Boot on the stack
 		 */
-
-		/* Target chip */
-		apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
-
-		/* Boot on the stack */
-		/* Kick the second */
-		apic_write_around(APIC_ICR, APIC_DM_STARTUP
-					| (start_eip >> 12));
+		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12), phys_apicid);
 
 		/*
 		 * Give the other CPU some time to accept the IPI.
@@ -753,7 +767,10 @@ wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
 		do {
 			Dprintk("+");
 			udelay(100);
-			send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
+			if ( !x2apic_enabled )
+				send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
+			else
+			    send_status = 0; /* We go out of the loop dirctly. */
 		} while (send_status && (timeout++ < 1000));
 
 		/*
@@ -988,7 +1005,7 @@ static void __init smp_boot_cpus(unsigned int max_cpus)
 	printk("CPU%d: ", 0);
 	print_cpu_info(&cpu_data[0]);
 
-	boot_cpu_physical_apicid = GET_APIC_ID(apic_read(APIC_ID));
+	boot_cpu_physical_apicid = get_apic_id();
 	x86_cpu_to_apicid[0] = boot_cpu_physical_apicid;
 
 	stack_base[0] = stack_start.esp;
diff --git a/xen/include/asm-x86/apic.h b/xen/include/asm-x86/apic.h
index af85c9f18d..b079e69fda 100644
--- a/xen/include/asm-x86/apic.h
+++ b/xen/include/asm-x86/apic.h
@@ -16,6 +16,21 @@
 #define APIC_DEBUG   2
 
 extern int apic_verbosity;
+extern int x2apic_enabled;
+
+extern void enable_x2apic(void);
+
+static __inline int x2apic_is_available(void)
+{
+    unsigned int op = 1, eax, ecx;
+
+    asm ( "cpuid"
+          : "=a" (eax), "=c" (ecx)
+          : "0" (op)
+          : "bx", "dx" );
+
+    return (ecx & (1U << 21));
+}
 
 /*
  * Define the default level of output to be very little
@@ -35,21 +50,105 @@ extern int apic_verbosity;
  * Basic functions accessing APICs.
  */
 
-static __inline void apic_write(unsigned long reg, u32 v)
+static __inline void apic_mem_write(unsigned long reg, u32 v)
 {
 	*((volatile u32 *)(APIC_BASE+reg)) = v;
 }
 
-static __inline void apic_write_atomic(unsigned long reg, u32 v)
+static __inline void apic_mem_write_atomic(unsigned long reg, u32 v)
 {
 	(void)xchg((volatile u32 *)(APIC_BASE+reg), v);
 }
 
-static __inline u32 apic_read(unsigned long reg)
+static __inline u32 apic_mem_read(unsigned long reg)
 {
 	return *((volatile u32 *)(APIC_BASE+reg));
 }
 
+/* NOTE: in x2APIC mode, we should use apic_icr_write()/apic_icr_read() to
+ * access the 64-bit ICR register.
+ */
+
+static __inline void apic_wrmsr(unsigned long reg, u32 low, u32 high)
+{
+    __asm__ __volatile__("wrmsr"
+            : /* no outputs */
+            : "c" (APIC_MSR_BASE + (reg >> 4)), "a" (low), "d" (high));
+}
+
+static __inline void apic_rdmsr(unsigned long reg, u32 *low, u32 *high)
+{
+    __asm__ __volatile__("rdmsr"
+            : "=a" (*low), "=d" (*high)
+            : "c" (APIC_MSR_BASE + (reg >> 4)));
+}
+
+static __inline void apic_write(unsigned long reg, u32 v)
+{
+
+    if ( x2apic_enabled )
+        apic_wrmsr(reg, v, 0);
+    else
+        apic_mem_write(reg, v);
+}
+
+static __inline void apic_write_atomic(unsigned long reg, u32 v)
+{
+    if ( x2apic_enabled )
+        apic_wrmsr(reg, v, 0);
+    else
+        apic_mem_write_atomic(reg, v);
+}
+
+static __inline u32 apic_read(unsigned long reg)
+{
+    u32 lo, hi;
+
+    if ( x2apic_enabled )
+        apic_rdmsr(reg, &lo, &hi);
+    else
+        lo = apic_mem_read(reg);
+    return lo;
+}
+
+static __inline u64 apic_icr_read(void)
+{
+    u32 lo, hi;
+
+    if ( x2apic_enabled )
+        apic_rdmsr(APIC_ICR, &lo, &hi);
+    else
+    {
+        lo = apic_mem_read(APIC_ICR);
+        hi = apic_mem_read(APIC_ICR2);
+    }
+    
+    return ((u64)lo) | (((u64)hi) << 32);
+}
+
+static __inline void apic_icr_write(u32 low, u32 dest)
+{
+    if ( x2apic_enabled )
+        apic_wrmsr(APIC_ICR, low, dest);
+    else
+    {
+        apic_mem_write(APIC_ICR2, dest << 24);
+        apic_mem_write(APIC_ICR, low);
+    }
+}
+
+static __inline u32 get_apic_id(void) /* Get the physical APIC id */
+{
+    u32 id = apic_read(APIC_ID);
+    return x2apic_enabled ? id : GET_xAPIC_ID(id);
+}
+
+static __inline u32 get_logical_apic_id(void)
+{
+    u32 logical_id = apic_read(APIC_LDR);
+    return x2apic_enabled ? logical_id : GET_xAPIC_LOGICAL_ID(logical_id);
+}
+
 void apic_wait_icr_idle(void);
 
 int get_physical_broadcast(void);
diff --git a/xen/include/asm-x86/apicdef.h b/xen/include/asm-x86/apicdef.h
index c8a6e161c1..134ac890e4 100644
--- a/xen/include/asm-x86/apicdef.h
+++ b/xen/include/asm-x86/apicdef.h
@@ -12,8 +12,8 @@
  
 #define		APIC_ID		0x20
 #define			APIC_ID_MASK		(0xFFu<<24)
-#define			GET_APIC_ID(x)		(((x)>>24)&0xFFu)
-#define			SET_APIC_ID(x)		(((x)<<24))
+#define			GET_xAPIC_ID(x)		(((x)>>24)&0xFFu)
+#define			SET_xAPIC_ID(x)		(((x)<<24))
 #define		APIC_LVR	0x30
 #define			APIC_LVR_MASK		0xFF00FF
 #define			GET_APIC_VERSION(x)	((x)&0xFF)
@@ -30,8 +30,8 @@
 #define		APIC_RRR	0xC0
 #define		APIC_LDR	0xD0
 #define			APIC_LDR_MASK		(0xFF<<24)
-#define			GET_APIC_LOGICAL_ID(x)	(((x)>>24)&0xFF)
-#define			SET_APIC_LOGICAL_ID(x)	(((x)<<24))
+#define			GET_xAPIC_LOGICAL_ID(x)	(((x)>>24)&0xFF)
+#define			SET_xAPIC_LOGICAL_ID(x)	(((x)<<24))
 #define			APIC_ALL_CPUS		0xFF
 #define		APIC_DFR	0xE0
 #define			APIC_DFR_CLUSTER		0x0FFFFFFFul
@@ -74,8 +74,8 @@
 #define			APIC_DM_EXTINT		0x00700
 #define			APIC_VECTOR_MASK	0x000FF
 #define		APIC_ICR2	0x310
-#define			GET_APIC_DEST_FIELD(x)	(((x)>>24)&0xFF)
-#define			SET_APIC_DEST_FIELD(x)	((x)<<24)
+#define			GET_xAPIC_DEST_FIELD(x)	(((x)>>24)&0xFF)
+#define			SET_xAPIC_DEST_FIELD(x)	((x)<<24)
 #define		APIC_LVTT	0x320
 #define		APIC_LVTTHMR	0x330
 #define		APIC_LVTPC	0x340
@@ -103,6 +103,10 @@
 #define		APIC_TMICT	0x380
 #define		APIC_TMCCT	0x390
 #define		APIC_TDCR	0x3E0
+
+/* Only available in x2APIC mode */
+#define		APIC_SELF_IPI	0x400
+
 #define			APIC_TDR_DIV_TMBASE	(1<<2)
 #define			APIC_TDR_DIV_1		0xB
 #define			APIC_TDR_DIV_2		0x0
@@ -115,6 +119,9 @@
 
 #define APIC_BASE (fix_to_virt(FIX_APIC_BASE))
 
+/* It's only used in x2APIC mode of an x2APIC unit. */
+#define APIC_MSR_BASE 0x800
+
 #ifdef __i386__
  #define MAX_IO_APICS 64
 #else
diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h
index 2a164844a2..18f6aff015 100644
--- a/xen/include/asm-x86/cpufeature.h
+++ b/xen/include/asm-x86/cpufeature.h
@@ -92,6 +92,7 @@
 #define X86_FEATURE_DCA		(4*32+18) /* Direct Cache Access */
 #define X86_FEATURE_SSE4_1	(4*32+19) /* Streaming SIMD Extensions 4.1 */
 #define X86_FEATURE_SSE4_2	(4*32+20) /* Streaming SIMD Extensions 4.2 */
+#define X86_FEATURE_X2APIC	(4*32+21) /* Extended xAPIC */
 #define X86_FEATURE_POPCNT	(4*32+23) /* POPCNT instruction */
 
 /* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */
@@ -182,6 +183,7 @@
 #define cpu_has_ffxsr           ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) \
                                  && boot_cpu_has(X86_FEATURE_FFXSR))
 
+#define cpu_has_x2apic          boot_cpu_has(X86_FEATURE_X2APIC)
 #endif /* __ASM_I386_CPUFEATURE_H */
 
 /* 
diff --git a/xen/include/asm-x86/genapic.h b/xen/include/asm-x86/genapic.h
index d3cbef26b1..0653335be9 100644
--- a/xen/include/asm-x86/genapic.h
+++ b/xen/include/asm-x86/genapic.h
@@ -62,6 +62,20 @@ void send_IPI_mask_flat(cpumask_t mask, int vector);
 	.cpu_mask_to_apicid = cpu_mask_to_apicid_flat, \
 	.send_IPI_mask = send_IPI_mask_flat
 
+void init_apic_ldr_x2apic(void);
+void clustered_apic_check_x2apic(void);
+cpumask_t target_cpus_x2apic(void);
+unsigned int cpu_mask_to_apicid_x2apic(cpumask_t cpumask);
+void send_IPI_mask_x2apic(cpumask_t mask, int vector);
+#define GENAPIC_X2APIC \
+	.int_delivery_mode = dest_Fixed, \
+	.int_dest_mode = 0 /* physical delivery */, \
+	.init_apic_ldr = init_apic_ldr_x2apic, \
+	.clustered_apic_check = clustered_apic_check_x2apic, \
+	.target_cpus = target_cpus_x2apic, \
+	.cpu_mask_to_apicid = cpu_mask_to_apicid_x2apic, \
+	.send_IPI_mask = send_IPI_mask_x2apic
+
 void init_apic_ldr_phys(void);
 void clustered_apic_check_phys(void);
 cpumask_t target_cpus_phys(void);
diff --git a/xen/include/asm-x86/hvm/vlapic.h b/xen/include/asm-x86/hvm/vlapic.h
index cc8a70608d..e0ef14205a 100644
--- a/xen/include/asm-x86/hvm/vlapic.h
+++ b/xen/include/asm-x86/hvm/vlapic.h
@@ -34,7 +34,7 @@
 #define vlapic_domain(vpic) (vlapic_vcpu(vlapic)->domain)
 
 #define VLAPIC_ID(vlapic)   \
-    (GET_APIC_ID(vlapic_get_reg((vlapic), APIC_ID)))
+    (GET_xAPIC_ID(vlapic_get_reg((vlapic), APIC_ID)))
 
 /*
  * APIC can be disabled in two ways:
diff --git a/xen/include/asm-x86/mach-generic/mach_apic.h b/xen/include/asm-x86/mach-generic/mach_apic.h
index 02746d47e6..15f4c39010 100644
--- a/xen/include/asm-x86/mach-generic/mach_apic.h
+++ b/xen/include/asm-x86/mach-generic/mach_apic.h
@@ -62,7 +62,7 @@ extern void generic_bigsmp_probe(void);
  */
 static inline int apic_id_registered(void)
 {
-	return physid_isset(GET_APIC_ID(apic_read(APIC_ID)),
+	return physid_isset(get_apic_id(),
 			    phys_cpu_present_map);
 }
 
diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h
index 455075f196..71f803fe36 100644
--- a/xen/include/asm-x86/msr-index.h
+++ b/xen/include/asm-x86/msr-index.h
@@ -236,6 +236,7 @@
 
 #define MSR_IA32_APICBASE		0x0000001b
 #define MSR_IA32_APICBASE_BSP		(1<<8)
+#define MSR_IA32_APICBASE_EXTD		(1<<10)
 #define MSR_IA32_APICBASE_ENABLE	(1<<11)
 #define MSR_IA32_APICBASE_BASE		(0xfffff<<12)
 
diff --git a/xen/include/asm-x86/smp.h b/xen/include/asm-x86/smp.h
index 67dd5c6c69..2078d441ec 100644
--- a/xen/include/asm-x86/smp.h
+++ b/xen/include/asm-x86/smp.h
@@ -90,13 +90,13 @@ static inline int num_booting_cpus(void)
 static inline int hard_smp_processor_id(void)
 {
 	/* we don't want to mark this access volatile - bad code generation */
-	return GET_APIC_ID(*(unsigned int *)(APIC_BASE+APIC_ID));
+	return get_apic_id();
 }
 
 static __inline int logical_smp_processor_id(void)
 {
 	/* we don't want to mark this access volatile - bad code generation */
-	return GET_APIC_LOGICAL_ID(*(unsigned int *)(APIC_BASE+APIC_LDR));
+	return get_logical_apic_id();
 }
 
 #endif