hvmloader: Dynamically size the PCI hole.

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>

5 files changed, 68 insertions, 35 deletions

diff --git a/tools/firmware/hvmloader/acpi/build.c b/tools/firmware/hvmloader/acpi/build.c
index 2bd1d3eb74..7fc006aa4a 100644
--- a/tools/firmware/hvmloader/acpi/build.c
+++ b/tools/firmware/hvmloader/acpi/build.c
@@ -85,8 +85,8 @@ static int construct_bios_info_table(uint8_t *buf)
 
     bios_info->hpet_present = hpet_exists(ACPI_HPET_ADDRESS);
 
-    bios_info->pci_min = PCI_MEMBASE;
-    bios_info->pci_len = PCI_MEMSIZE;
+    bios_info->pci_min = pci_mem_start;
+    bios_info->pci_len = pci_mem_end - pci_mem_start;
     bios_info->xen_pfiob = 0xdead;
 
     return align16(sizeof(*bios_info));
diff --git a/tools/firmware/hvmloader/cacheattr.c b/tools/firmware/hvmloader/cacheattr.c
index 5dc2c7f807..fb6e9f4480 100644
--- a/tools/firmware/hvmloader/cacheattr.c
+++ b/tools/firmware/hvmloader/cacheattr.c
@@ -89,9 +89,11 @@ void cacheattr_init(void)
     if ( nr_var_ranges != 0 )
     {
         /* A single UC range covering PCI space. */
-        wrmsr(MSR_MTRRphysBase(0), PCI_MEMBASE);
+        /* pci_mem_start must be of the binary form 1....10....0 */
+        BUG_ON(~(pci_mem_start | (pci_mem_start - 1)));
+        wrmsr(MSR_MTRRphysBase(0), pci_mem_start);
         wrmsr(MSR_MTRRphysMask(0),
-              ((uint64_t)(int32_t)PCI_MEMBASE & addr_mask) | (1u << 11));
+              ((uint64_t)(int32_t)pci_mem_start & addr_mask) | (1u << 11));
         printf("var MTRRs ... ");
     }
 
diff --git a/tools/firmware/hvmloader/config.h b/tools/firmware/hvmloader/config.h
index 088779fe90..e5b4fc4a80 100644
--- a/tools/firmware/hvmloader/config.h
+++ b/tools/firmware/hvmloader/config.h
@@ -14,8 +14,10 @@
 #define PCI_ISA_DEVFN       0x08    /* dev 1, fn 0 */
 #define PCI_ISA_IRQ_MASK    0x0c20U /* ISA IRQs 5,10,11 are PCI connected */
 
-#define PCI_MEMBASE         0xf0000000
-#define PCI_MEMSIZE         0x0c000000
+/* MMIO hole: Hardcoded defaults, which can be dynamically expanded. */
+#define PCI_MEM_START       0xf0000000
+#define PCI_MEM_END         0xfc000000
+extern unsigned long pci_mem_start, pci_mem_end;
 
 /* We reserve 16MB at the top of the 4GB memory hole. */
 #define RESERVED_MEMBASE    0xff000000
@@ -28,6 +30,7 @@
 #define ROMBIOS_END            (ROMBIOS_BEGIN + ROMBIOS_SIZE)
 
 /* Memory map. */
+#define SCRATCH_PHYSICAL_ADDRESS      0x00010000
 #define HYPERCALL_PHYSICAL_ADDRESS    0x00080000
 #define VGABIOS_PHYSICAL_ADDRESS      0x000C0000
 #define OPTIONROM_PHYSICAL_ADDRESS    0x000C8000
@@ -37,7 +40,6 @@
 #define SMBIOS_PHYSICAL_ADDRESS       0x000EB000
 #define SMBIOS_MAXIMUM_SIZE           0x00005000
 #define ROMBIOS_PHYSICAL_ADDRESS      0x000F0000
-#define SCRATCH_PHYSICAL_ADDRESS      0x00010000
 
 /* Offsets from E820_PHYSICAL_ADDRESS. */
 #define E820_NR_OFFSET                0x0
diff --git a/tools/firmware/hvmloader/hvmloader.c b/tools/firmware/hvmloader/hvmloader.c
index 69a90fc5fb..19e35bc5e4 100644
--- a/tools/firmware/hvmloader/hvmloader.c
+++ b/tools/firmware/hvmloader/hvmloader.c
@@ -31,6 +31,7 @@
 #include "option_rom.h"
 #include <xen/version.h>
 #include <xen/hvm/params.h>
+#include <xen/memory.h>
 
 asm (
     "    .text                       \n"
@@ -99,6 +100,9 @@ asm (
     "    .text                       \n"
     );
 
+unsigned long pci_mem_start = PCI_MEM_START;
+unsigned long pci_mem_end = PCI_MEM_END;
+
 static enum { VGA_none, VGA_std, VGA_cirrus } virtual_vga = VGA_none;
 
 static void init_hypercalls(void)
@@ -148,16 +152,14 @@ static void apic_setup(void)
 
 static void pci_setup(void)
 {
-    uint32_t base, devfn, bar_reg, bar_data, bar_sz, cmd;
+    uint32_t base, devfn, bar_reg, bar_data, bar_sz, cmd, mmio_total = 0;
     uint16_t class, vendor_id, device_id;
     unsigned int bar, pin, link, isa_irq;
 
     /* Resources assignable to PCI devices via BARs. */
     struct resource {
         uint32_t base, max;
-    } *resource;
-    struct resource mem_resource = { PCI_MEMBASE, PCI_MEMBASE + PCI_MEMSIZE };
-    struct resource io_resource  = { 0xc000, 0x10000 };
+    } *resource, mem_resource, io_resource;
 
     /* Create a list of device BARs in descending order of size. */
     struct bars {
@@ -248,6 +250,10 @@ static void pci_setup(void)
             bars[i].bar_reg = bar_reg;
             bars[i].bar_sz  = bar_sz;
 
+            if ( (bar_data & PCI_BASE_ADDRESS_SPACE) ==
+                 PCI_BASE_ADDRESS_SPACE_MEMORY )
+                mmio_total += bar_sz;
+
             nr_bars++;
 
             /* Skip the upper-half of the address for a 64-bit BAR. */
@@ -276,6 +282,28 @@ static void pci_setup(void)
         pci_writew(devfn, PCI_COMMAND, cmd);
     }
 
+    while ( (mmio_total > (pci_mem_end - pci_mem_start)) &&
+            ((pci_mem_start << 1) != 0) )
+        pci_mem_start <<= 1;
+
+    while ( (pci_mem_start >> PAGE_SHIFT) < hvm_info->low_mem_pgend )
+    {
+        struct xen_add_to_physmap xatp;
+        if ( hvm_info->high_mem_pgend == 0 )
+            hvm_info->high_mem_pgend = 1ull << (32 - PAGE_SHIFT);
+        xatp.domid = DOMID_SELF;
+        xatp.space = XENMAPSPACE_gmfn;
+        xatp.idx   = --hvm_info->low_mem_pgend;
+        xatp.gpfn  = hvm_info->high_mem_pgend++;
+        if ( hypercall_memory_op(XENMEM_add_to_physmap, &xatp) != 0 )
+            BUG();
+    }
+
+    mem_resource.base = pci_mem_start;
+    mem_resource.max = pci_mem_end;
+    io_resource.base = 0xc000;
+    io_resource.max = 0x10000;
+
     /* Assign iomem and ioport resources in descending order of size. */
     for ( i = 0; i < nr_bars; i++ )
     {
@@ -625,6 +653,9 @@ int main(void)
 
     printf("CPU speed is %u MHz\n", get_cpu_mhz());
 
+    apic_setup();
+    pci_setup();
+
     smp_initialise();
 
     perform_tests();
@@ -639,9 +670,6 @@ int main(void)
     memcpy((void *)ROMBIOS_PHYSICAL_ADDRESS, rombios, rombios_sz);
     highbios_setup();
 
-    apic_setup();
-    pci_setup();
-
     if ( (hvm_info->nr_vcpus > 1) || hvm_info->apic_mode )
         create_mp_tables();
 
diff --git a/tools/firmware/hvmloader/util.c b/tools/firmware/hvmloader/util.c
index 50d23d8453..bd247fc65b 100644
--- a/tools/firmware/hvmloader/util.c
+++ b/tools/firmware/hvmloader/util.c
@@ -307,6 +307,7 @@ void *mem_alloc(uint32_t size, uint32_t align)
 {
     static uint32_t reserve = RESERVED_MEMBASE - 1;
     static int over_allocated;
+    struct xen_add_to_physmap xatp;
     struct xen_memory_reservation xmr;
     xen_pfn_t mfn;
     uint32_t s, e;
@@ -323,36 +324,36 @@ void *mem_alloc(uint32_t size, uint32_t align)
     while ( (reserve >> PAGE_SHIFT) != (e >> PAGE_SHIFT) )
     {
         reserve += PAGE_SIZE;
-
-        /* Try to allocate another page in the reserved area. */
-        xmr.domid = DOMID_SELF;
-        xmr.mem_flags = 0;
-        xmr.extent_order = 0;
-        xmr.nr_extents = 1;
-        set_xen_guest_handle(xmr.extent_start, &mfn);
         mfn = reserve >> PAGE_SHIFT;
-        if ( !over_allocated &&
-             (hypercall_memory_op(XENMEM_populate_physmap, &xmr) == 1) )
-            continue;
 
-        /* If we fail, steal a page from the ordinary RAM map. */
-        over_allocated = 1;
+        /* Try to allocate a brand new page in the reserved area. */
+        if ( !over_allocated )
+        {
+            xmr.domid = DOMID_SELF;
+            xmr.mem_flags = 0;
+            xmr.extent_order = 0;
+            xmr.nr_extents = 1;
+            set_xen_guest_handle(xmr.extent_start, &mfn);
+            if ( hypercall_memory_op(XENMEM_populate_physmap, &xmr) == 1 )
+                continue;
+            over_allocated = 1;
+        }
+
+        /* Otherwise, relocate a page from the ordinary RAM map. */
         if ( hvm_info->high_mem_pgend )
         {
-            mfn = --hvm_info->high_mem_pgend;
-            if ( mfn == (1ull << (32 - PAGE_SHIFT)) )
+            xatp.idx = --hvm_info->high_mem_pgend;
+            if ( xatp.idx == (1ull << (32 - PAGE_SHIFT)) )
                 hvm_info->high_mem_pgend = 0;
         }
         else
         {
-            mfn = --hvm_info->low_mem_pgend;
+            xatp.idx = --hvm_info->low_mem_pgend;
         }
-        if ( hypercall_memory_op(XENMEM_decrease_reservation, &xmr) != 1 )
-            BUG();
-
-        /* Now try the allocation again. Must not fail. */
-        mfn = reserve >> PAGE_SHIFT;
-        if ( hypercall_memory_op(XENMEM_populate_physmap, &xmr) != 1 )
+        xatp.domid = DOMID_SELF;
+        xatp.space = XENMAPSPACE_gmfn;
+        xatp.gpfn  = mfn;
+        if ( hypercall_memory_op(XENMEM_add_to_physmap, &xatp) != 0 )
             BUG();
     }