x86: Fix the dom0 booting hang when VT-d is enabled.

Dom0 C/S 593 hooks the pci bus probe and remove function. When
probing a function at booting time, it will first add the device into
Dom0's list through a hypercall, then execute the driver's probe
function. If the probe function fails, another hypercall is called to
remove the device from Dom0's list. But for some RMRR devices, for
example, USB devices, they may still be in use for some operation by
BIOS at booting time. So when removing those kind of devices, we
should still keep the RMRR information.

Also add a small fix for "for_each_rmrr_device".

Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
Signed-off-by: Weidong Han <weidong.han@intel.com>
Signed-off-by: Yunhong Jiang <yunhong.jiang@intel.com>

2 files changed, 37 insertions, 14 deletions

diff --git a/xen/drivers/passthrough/vtd/dmar.h b/xen/drivers/passthrough/vtd/dmar.h
index 747b68677e..bcbb88bf34 100644
--- a/xen/drivers/passthrough/vtd/dmar.h
+++ b/xen/drivers/passthrough/vtd/dmar.h
@@ -76,7 +76,7 @@ struct acpi_atsr_unit {
 #define for_each_rmrr_device(rmrr, bdf, idx)            \
     list_for_each_entry(rmrr, &acpi_rmrr_units, list)   \
         /* assume there never is a bdf == 0 */          \
-        for (idx = 0; (bdf = rmrr->scope.devices[i]) && \
+        for (idx = 0; (bdf = rmrr->scope.devices[idx]) && \
                  idx < rmrr->scope.devices_cnt; idx++)
 
 struct acpi_drhd_unit * acpi_find_matched_drhd_unit(u8 bus, u8 devfn);
diff --git a/xen/drivers/passthrough/vtd/iommu.c b/xen/drivers/passthrough/vtd/iommu.c
index 8155121188..3a11f5f007 100644
--- a/xen/drivers/passthrough/vtd/iommu.c
+++ b/xen/drivers/passthrough/vtd/iommu.c
@@ -1294,11 +1294,18 @@ static int domain_context_mapping(struct domain *domain, u8 bus, u8 devfn)
     return ret;
 }
 
-static int domain_context_unmap_one(struct iommu *iommu, u8 bus, u8 devfn)
+static int domain_context_unmap_one(
+    struct domain *domain,
+    struct iommu *iommu,
+    u8 bus, u8 devfn)
 {
     struct context_entry *context, *context_entries;
     unsigned long flags;
     u64 maddr;
+    struct acpi_rmrr_unit *rmrr;
+    u16 bdf;
+    int i;
+    unsigned int is_rmrr_device = 0;
 
     maddr = bus_to_context_maddr(iommu, bus);
     context_entries = (struct context_entry *)map_vtd_domain_page(maddr);
@@ -1311,18 +1318,32 @@ static int domain_context_unmap_one(struct iommu *iommu, u8 bus, u8 devfn)
     }
 
     spin_lock_irqsave(&iommu->lock, flags);
-    context_clear_present(*context);
-    context_clear_entry(*context);
-    iommu_flush_cache_entry(context);
-    iommu_flush_context_global(iommu, 0);
-    iommu_flush_iotlb_global(iommu, 0);
+    if ( domain->domain_id == 0 )
+    {
+        for_each_rmrr_device ( rmrr, bdf, i )
+        {
+            if ( PCI_BUS(bdf) == bus && PCI_DEVFN2(bdf) == devfn )
+            {
+                is_rmrr_device = 1;
+                break;
+            }
+        }
+    }
+    if ( !is_rmrr_device )
+    {
+        context_clear_present(*context);
+        context_clear_entry(*context);
+        iommu_flush_cache_entry(context);
+        iommu_flush_context_domain(iommu, domain_iommu_domid(domain), 0);
+        iommu_flush_iotlb_dsi(iommu, domain_iommu_domid(domain), 0);
+    }
     unmap_vtd_domain_page(context_entries);
     spin_unlock_irqrestore(&iommu->lock, flags);
 
     return 0;
 }
 
-static int domain_context_unmap(u8 bus, u8 devfn)
+static int domain_context_unmap(struct domain *domain, u8 bus, u8 devfn)
 {
     struct acpi_drhd_unit *drhd;
     u16 sec_bus, sub_bus;
@@ -1345,18 +1366,18 @@ static int domain_context_unmap(u8 bus, u8 devfn)
                                  PCI_SUBORDINATE_BUS);
         /*dmar_scope_remove_buses(&drhd->scope, sec_bus, sub_bus);*/
         if ( DEV_TYPE_PCI_BRIDGE )
-            ret = domain_context_unmap_one(drhd->iommu, bus, devfn);
+            ret = domain_context_unmap_one(domain, drhd->iommu, bus, devfn);
         break;
 
     case DEV_TYPE_PCIe_ENDPOINT:
-        ret = domain_context_unmap_one(drhd->iommu, bus, devfn);
+        ret = domain_context_unmap_one(domain, drhd->iommu, bus, devfn);
         break;
 
     case DEV_TYPE_PCI:
         if ( find_pcie_endpoint(&bus, &devfn, &secbus) )
-            ret = domain_context_unmap_one(drhd->iommu, bus, devfn);
+            ret = domain_context_unmap_one(domain, drhd->iommu, bus, devfn);
         if ( bus != secbus )
-            domain_context_unmap_one(drhd->iommu, secbus, 0);
+            domain_context_unmap_one(domain, drhd->iommu, secbus, 0);
         break;
 
     default:
@@ -1386,7 +1407,7 @@ static int reassign_device_ownership(
 
     drhd = acpi_find_matched_drhd_unit(bus, devfn);
     pdev_iommu = drhd->iommu;
-    domain_context_unmap(bus, devfn);
+    domain_context_unmap(source, bus, devfn);
 
     write_lock(&pcidevs_lock);
     list_move(&pdev->domain_list, &target->arch.pdev_list);
@@ -1584,7 +1605,9 @@ static int intel_iommu_add_device(struct pci_dev *pdev)
 
 static int intel_iommu_remove_device(struct pci_dev *pdev)
 {
-    return domain_context_unmap(pdev->bus, pdev->devfn);
+    if ( !pdev->domain )
+        return -EINVAL;
+    return domain_context_unmap(pdev->domain, pdev->bus, pdev->devfn);
 }
 
 static void setup_dom0_devices(struct domain *d)