vtd hotplug: check if a device can be hot-plugged.

When we statically assign a pci device (the pci=3D['xx:xx.x'] string
in guest config file) to guest, we make many checkings (for instance,
if the device is specified in 'pciback.hide', if it has
non-page-aligned MMIO BARs, if it has a proper FLR capability, if the
related devices should be co-assigned). However, with respect to the
guest hotplug, we only check if the device exists and not assigned yet
-- this is not enough, for instance, now xend allows us to assign an
in-use device (being used by Dom0) to an HVM guest (because
xc.test_assigned() returns OK) -- this will cause disaster... The
patch adds some necessary checkings.

Signed-off-by: Dexuan Cui <dexuan.cui@intel.com>

3 files changed, 97 insertions, 11 deletions

diff --git a/tools/python/xen/util/pci.py b/tools/python/xen/util/pci.py
index aeb25d8566..bea58c7a63 100644
--- a/tools/python/xen/util/pci.py
+++ b/tools/python/xen/util/pci.py
@@ -276,7 +276,7 @@ def check_FLR_capability(dev_list):
                     coassigned_pci_list = dev.find_all_the_multi_functions()
                     need_transform = True
                 elif dev.dev_type == DEV_TYPE_PCI and not dev.pci_af_flr:
-                    coassigned_pci_list = dev.find_coassigned_devices(True)
+                    coassigned_pci_list = dev.find_coassigned_pci_devices(True)
                     del coassigned_pci_list[0]
                     need_transform = True
 
@@ -434,7 +434,7 @@ class PciDevice:
                 list = list + [dev.name]
         return list
         
-    def find_coassigned_devices(self, ignore_bridge = True):
+    def find_coassigned_pci_devices(self, ignore_bridge = True):
         ''' Here'self' is a PCI device, we need find the uppermost PCI/PCI-X
             bridge, and all devices behind it must be co-assigned to the same
             guest.
@@ -532,6 +532,16 @@ class PciDevice:
         funcs = re.findall(p, pci_names)
         return funcs
 
+    def find_coassigned_devices(self):
+        if self.dev_type == DEV_TYPE_PCIe_ENDPOINT and not self.pcie_flr:
+            return self.find_all_the_multi_functions()
+        elif self.dev_type == DEV_TYPE_PCI and not self.pci_af_flr:
+            coassigned_pci_list = self.find_coassigned_pci_devices(True)
+            del coassigned_pci_list[0]
+            return coassigned_pci_list
+        else:
+            return [self.name]
+
     def find_cap_offset(self, cap):
         path = find_sysfs_mnt()+SYSFS_PCI_DEVS_PATH+'/'+ \
                self.name+SYSFS_PCI_DEV_CONFIG_PATH
@@ -718,7 +728,7 @@ class PciDevice:
                 if self.bus == 0:
                     self.do_FLR_for_integrated_device()
                 else:
-                    devs = self.find_coassigned_devices(False)
+                    devs = self.find_coassigned_pci_devices(False)
                     # Remove the element 0 which is a bridge
                     target_bus = devs[0]
                     del devs[0]
diff --git a/tools/python/xen/xend/XendDomainInfo.py b/tools/python/xen/xend/XendDomainInfo.py
index 3cc054dba1..72f718398d 100644
--- a/tools/python/xen/xend/XendDomainInfo.py
+++ b/tools/python/xen/xend/XendDomainInfo.py
@@ -290,19 +290,21 @@ def dom_get(dom):
         log.trace("domain_getinfo(%d) failed, ignoring: %s", dom, str(err))
     return None
 
-def do_FLR(domid):
-    from xen.xend.server.pciif import parse_pci_name, PciDevice
+def get_assigned_pci_devices(domid):
+    dev_str_list = []
     path = '/local/domain/0/backend/pci/%u/0/' % domid
     num_devs = xstransact.Read(path + 'num_devs');
     if num_devs is None or num_devs == "":
-        return;
-
-    num_devs = int(xstransact.Read(path + 'num_devs'));
-
-    dev_str_list = []
+        return dev_str_list
+    num_devs = int(num_devs);
     for i in range(num_devs):
         dev_str = xstransact.Read(path + 'dev-%i' % i)
         dev_str_list = dev_str_list + [dev_str]
+    return dev_str_list 
+
+def do_FLR(domid):
+    from xen.xend.server.pciif import parse_pci_name, PciDevice
+    dev_str_list = get_assigned_pci_devices(domid)
 
     for dev_str in dev_str_list:
         (dom, b, d, f) = parse_pci_name(dev_str)
@@ -645,6 +647,55 @@ class XendDomainInfo:
                           " already been assigned to other domain, or maybe"
                           " it doesn't exist." % (bus, dev, func))
 
+        # Here, we duplicate some checkings (in some cases, we mustn't allow
+        # a device to be hot-plugged into an HVM guest) that are also done in
+        # pci_device_configure()'s self.device_create(dev_sxp) or
+        # dev_control.reconfigureDevice(devid, dev_config).
+        # We must make the checkings before sending the command 'pci-ins' to
+        # ioemu.
+
+        # Test whether the device is owned by pciback. For instance, we can't
+        # hotplug a device being used by Dom0 itself to an HVM guest.
+        from xen.xend.server.pciif import PciDevice, parse_pci_name
+        domain = int(new_dev['domain'],16)
+        bus    = int(new_dev['bus'],16)
+        dev    = int(new_dev['slot'],16)
+        func   = int(new_dev['func'],16)
+        try:
+            pci_device = PciDevice(domain, bus, dev, func)
+        except Exception, e:
+            raise VmError("pci: failed to locate device and "+
+                    "parse it's resources - "+str(e))
+        if pci_device.driver!='pciback':
+            raise VmError(("pci: PCI Backend does not own device "+ \
+                    "%s\n"+ \
+                    "See the pciback.hide kernel "+ \
+                    "command-line parameter or\n"+ \
+                    "bind your slot/device to the PCI backend using sysfs" \
+                    )%(pci_device.name))
+
+        # Check non-page-aligned MMIO BAR.
+        if pci_device.has_non_page_aligned_bar and arch.type != "ia64":
+            raise VmError("pci: %s: non-page-aligned MMIO BAR found." % \
+                pci_device.name)
+
+        # Check the co-assignment.
+        # To pci-attach a device D to domN, we should ensure each of D's
+        # co-assignment devices hasn't been assigned, or has been assigned to
+        # domN.
+        coassignment_list = pci_device.find_coassigned_devices()
+        assigned_pci_device_str_list = get_assigned_pci_devices(self.domid)
+        for pci_str in coassignment_list:
+            (domain, bus, dev, func) = parse_pci_name(pci_str) 
+            dev_str =  '0x%x,0x%x,0x%x,0x%x' % (domain, bus, dev, func)
+            if xc.test_assign_device(self.domid, dev_str) == 0:
+                continue
+            if not pci_str in assigned_pci_device_str_list:
+                raise VmError(('pci: failed to pci-attach %s to dom%d" + \
+                    " because one of its co-assignment device %s has been" + \
+                    " assigned to other domain.' \
+                    )% (pci_device.name, self.domid, pci_str))
+
         bdf_str = "%s:%s:%s.%s@%s" % (new_dev['domain'],
                 new_dev['bus'],
                 new_dev['slot'],
@@ -935,6 +986,31 @@ class XendDomainInfo:
         if vslot == 0:
             raise VmError("Device @ vslot 0x%x do not support hotplug." % (vslot))
 
+        # Check the co-assignment.
+        # To pci-detach a device D from domN, we should ensure: for each DD in the
+        # list of D's co-assignment devices, DD is not assigned (to domN).
+        # 
+        from xen.xend.server.pciif import PciDevice
+        domain = int(x['domain'],16)
+        bus    = int(x['bus'],16)
+        dev    = int(x['slot'],16)
+        func   = int(x['func'],16)
+        try:
+            pci_device = PciDevice(domain, bus, dev, func)
+        except Exception, e:
+            raise VmError("pci: failed to locate device and "+
+                    "parse it's resources - "+str(e))
+        coassignment_list = pci_device.find_coassigned_devices()
+        coassignment_list.remove(pci_device.name)
+        assigned_pci_device_str_list = get_assigned_pci_devices(self.domid)
+        for pci_str in coassignment_list:
+            if pci_str in assigned_pci_device_str_list:
+                raise VmError(('pci: failed to pci-detach %s from dom%d" + \
+                    " because one of its co-assignment device %s is still " + \
+                    " assigned to the domain.' \
+                    )% (pci_device.name, self.domid, pci_str))
+
+
         bdf_str = "%s:%s:%s.%s" % (x['domain'], x['bus'], x['slot'], x['func'])
         log.info("hvm_destroyPCIDevice:%s:%s!", x, bdf_str)
 
diff --git a/tools/python/xen/xend/server/pciif.py b/tools/python/xen/xend/server/pciif.py
index 1051450a08..94036b8430 100644
--- a/tools/python/xen/xend/server/pciif.py
+++ b/tools/python/xen/xend/server/pciif.py
@@ -417,7 +417,7 @@ class PciController(DevController):
                 else:
                     # All devices behind the uppermost PCI/PCI-X bridge must be\
                     # co-assigned to the same guest.
-                    devs_str = dev.find_coassigned_devices(True)
+                    devs_str = dev.find_coassigned_pci_devices(True)
                     # Remove the element 0 which is a bridge
                     del devs_str[0]