bitkeeper revision 1.940 (40c5a0a5VY_RJcApOo3Gmxz36JQSrQ)

Manual merge.

6 files changed, 232 insertions, 66 deletions

diff --git a/tools/examples/xc_dom_create.py b/tools/examples/xc_dom_create.py
index 8f1cfdb053..1b6fbd0c8f 100755
--- a/tools/examples/xc_dom_create.py
+++ b/tools/examples/xc_dom_create.py
@@ -274,66 +274,78 @@ def make_domain():
     # set the expertise level appropriately
     xenctl.utils.VBD_EXPERT_MODE = vbd_expert
 
-    if new_io_world:
-        cmsg = 'new_block_interface(dom='+str(id)+')'
-        xend_response = xenctl.utils.xend_control_message(cmsg)
-        if not xend_response['success']:
-            print "Error creating block interface"
-            print "Error type: " + xend_response['error_type']
-            if xend_response['error_type'] == 'exception':
-                print "Exception type: " + xend_response['exception_type']
-                print "Exception val:  " + xend_response['exception_value']
-            xc.domain_destroy ( dom=id )
-            sys.exit()
-    
-    for ( uname, virt_name, rw ) in vbd_list:
-	virt_dev = xenctl.utils.blkdev_name_to_number( virt_name )
-
-	segments = xenctl.utils.lookup_disk_uname( uname )
-	if not segments:
-	    print "Error looking up %s\n" % uname
-	    xc.domain_destroy ( dom=id )
-	    sys.exit()
-
+    if not (flags & 4): # It's not a block backend (or it's old IO world)
         if new_io_world:
-            if len(segments) > 1:
-                print "New I/O world cannot deal with multi-extent vdisks"
-                xc.domain_destroy ( dom=id )
-                sys.exit()
-            seg = segments[0]
-            cmsg = 'new_block_device(dom=' + str(id) + \
-                   ',handle=0,vdev=' + str(virt_dev) + \
-                   ',pdev=' + str(seg['device']) + \
-                   ',start_sect=' + str(seg['start_sector']) + \
-                   ',nr_sect=' + str(seg['nr_sectors']) + \
-                   ',readonly=' + str(not re.match('w',rw)) + ')'
+            cmsg = 'new_block_interface(dom='+str(id)+')'
             xend_response = xenctl.utils.xend_control_message(cmsg)
             if not xend_response['success']:
-                print "Error creating virtual block device"
+                print "Error creating block interface"
                 print "Error type: " + xend_response['error_type']
                 if xend_response['error_type'] == 'exception':
                     print "Exception type: " + xend_response['exception_type']
                     print "Exception val:  " + xend_response['exception_value']
                 xc.domain_destroy ( dom=id )
                 sys.exit()
-        else:
-            # check that setting up this VBD won't violate the sharing
-            # allowed by the current VBD expertise level
-            if xenctl.utils.vd_extents_validate(segments,
-                                                rw=='w' or rw=='rw') < 0:
-                xc.domain_destroy( dom = id )
-                sys.exit()
-            
-            if xc.vbd_create( dom=id, vbd=virt_dev,
-                              writeable= rw=='w' or rw=='rw' ):
-                print "Error creating VBD %d (writeable=%d)\n" % (virt_dev,rw)
+
+        for ( uname, virt_name, rw ) in vbd_list:
+            virt_dev = xenctl.utils.blkdev_name_to_number( virt_name )
+
+            segments = xenctl.utils.lookup_disk_uname( uname )
+            if not segments:
+                print "Error looking up %s\n" % uname
                 xc.domain_destroy ( dom=id )
                 sys.exit()
+
+            if new_io_world:
+                if len(segments) > 1:
+                    print "New I/O world cannot deal with multi-extent vdisks"
+                    xc.domain_destroy ( dom=id )
+                    sys.exit()
+                seg = segments[0]
+                cmsg = 'new_block_device(dom=' + str(id) + \
+                       ',handle=0,vdev=' + str(virt_dev) + \
+                       ',pdev=' + str(seg['device']) + \
+                       ',start_sect=' + str(seg['start_sector']) + \
+                       ',nr_sect=' + str(seg['nr_sectors']) + \
+                       ',readonly=' + str(not re.match('w',rw)) + ')'
+                xend_response = xenctl.utils.xend_control_message(cmsg)
+                if not xend_response['success']:
+                    print "Error creating virtual block device"
+                    print "Error type: " + xend_response['error_type']
+                    if xend_response['error_type'] == 'exception':
+                        print "Exception type: " + xend_response['exception_type']
+                        print "Exception val:  " + xend_response['exception_value']
+                    xc.domain_destroy ( dom=id )
+                    sys.exit()
+            else:
+                # check that setting up this VBD won't violate the sharing
+                # allowed by the current VBD expertise level
+                if xenctl.utils.vd_extents_validate(segments,
+                                                    rw=='w' or rw=='rw') < 0:
+                    xc.domain_destroy( dom = id )
+                    sys.exit()
+            
+                if xc.vbd_create( dom=id, vbd=virt_dev,
+                                  writeable= rw=='w' or rw=='rw' ):
+                    print "Error creating VBD %d (writeable=%d)\n" % (virt_dev,rw)
+                    xc.domain_destroy ( dom=id )
+                    sys.exit()
 	
-            if xc.vbd_setextents( dom=id,
-                                  vbd=virt_dev,
-                                  extents=segments):
-                print "Error populating VBD vbd=%d\n" % virt_dev
+                if xc.vbd_setextents( dom=id,
+                                      vbd=virt_dev,
+                                      extents=segments):
+                    print "Error populating VBD vbd=%d\n" % virt_dev
+                    xc.domain_destroy ( dom=id )
+                    sys.exit()
+    else: # It's a block backend - notify Xend.
+        cmsg = 'set_block_backend(dom='+str(id)+')'
+        xend_response = xenctl.utils.xend_control_message(cmsg)
+        if not xend_response['success']:
+            print "Error registering network backend"
+            print "Error type: " + xend_response['error_type']
+            if xend_response['error_type'] == 'exception':
+                print "Exception type: " + xend_response['exception_type']
+                print "Exception val:  " + xend_response['exception_value']
                 xc.domain_destroy ( dom=id )
                 sys.exit()
 
diff --git a/tools/xend/lib/blkif.py b/tools/xend/lib/blkif.py
index a35d1bcea1..3dd05a0a95 100644
--- a/tools/xend/lib/blkif.py
+++ b/tools/xend/lib/blkif.py
@@ -11,6 +11,7 @@ CMSG_BLKIF_BE = 1
 CMSG_BLKIF_FE = 2
 CMSG_BLKIF_FE_INTERFACE_STATUS_CHANGED =  0
 CMSG_BLKIF_FE_DRIVER_STATUS_CHANGED    = 32
+CMSG_BLKIF_BE_DRIVER_STATUS_CHANGED    = 32
 CMSG_BLKIF_FE_INTERFACE_CONNECT        = 33
 CMSG_BLKIF_FE_INTERFACE_DISCONNECT     = 34
 CMSG_BLKIF_BE_CREATE      = 0
@@ -22,11 +23,19 @@ CMSG_BLKIF_BE_VBD_DESTROY = 5
 CMSG_BLKIF_BE_VBD_GROW    = 6
 CMSG_BLKIF_BE_VBD_SHRINK  = 7
 
+BLKIF_DRIVER_STATUS_DOWN  = 0
+BLKIF_DRIVER_STATUS_UP    = 1
+
 pendmsg = None
 pendaddr = None
 
+recovery = False # Is a recovery in progress? (if so we'll need to notify guests)
+be_port  = None  # Port object for backend domain
+
 def backend_tx_req(msg):
-    port = xend.main.dom0_port
+    port = xend.blkif.be_port
+    if not port:
+        print "BUG: attempt to transmit request to non-existant blkif driver"
     if port.space_to_write_request():
         port.write_request(msg)
         port.notify()
@@ -35,6 +44,26 @@ def backend_tx_req(msg):
 
 def backend_rx_req(port, msg):
     port.write_response(msg)
+    subtype = (msg.get_header())['subtype']
+    print "Received blkif-be request, subtype %d" % subtype
+    if subtype == CMSG_BLKIF_BE_DRIVER_STATUS_CHANGED:
+        (status, dummy) = struct.unpack("II", msg.get_payload())
+        if status == BLKIF_DRIVER_STATUS_UP:
+            if xend.blkif.recovery:
+                # Nasty hack: we count the number of VBDs we reattach so that
+                # we'll know when to notify the guests.  Must make this better!
+                interface.rebuilt_so_far = 0
+                interface.nr_to_rebuild  = 0
+                print "New blkif backend now UP, rebuilding VBDs:"
+                for blkif_key in interface.list.keys():
+                    blkif = interface.list[blkif_key]
+                    blkif.create()
+                    for vdev in blkif.devices.keys():
+                        blkif.reattach_device(vdev)
+                        interface.nr_to_rebuild += 1
+        else:
+            print "Unexpected block backend driver status: %d" % status
+
 
 def backend_rx_rsp(port, msg):
     subtype = (msg.get_header())['subtype']
@@ -68,8 +97,23 @@ def backend_rx_rsp(port, msg):
                                    'extent.device' : pdev })
         backend_tx_req(msg)
     elif subtype == CMSG_BLKIF_BE_VBD_GROW:
-        rsp = { 'success': True }
-        xend.main.send_management_response(rsp, xend.blkif.pendaddr)
+       if not xend.blkif.recovery:
+           rsp = { 'success': True }
+           xend.main.send_management_response(rsp, xend.blkif.pendaddr)
+       else:
+           interface.rebuilt_so_far += 1
+           if interface.rebuilt_so_far == interface.nr_to_rebuild:
+               print "Rebuilt VBDs, notifying guests:"
+               for blkif_key in interface.list.keys():
+                   blkif = interface.list[blkif_key]
+                   print "  Notifying %d" % blkif.dom
+                   msg = xend.utils.message(CMSG_BLKIF_FE,                   \
+                                            CMSG_BLKIF_FE_INTERFACE_STATUS_CHANGED, 0)
+                   msg.append_payload(struct.pack("III", 0,1,0))
+                   blkif.ctrlif_tx_req(xend.main.port_from_dom(blkif.dom),msg)
+               xend.blkif.recovery = False
+               print "Done notifying guests"
+
 
 def backend_do_work(port):
     global pendmsg
@@ -85,7 +129,6 @@ class interface:
     # Dictionary of all block-device interfaces.
     list = {}
 
-
     # NB. 'key' is an opaque value that has no meaning in this class.
     def __init__(self, dom, key):
         self.dom     = dom
@@ -93,8 +136,11 @@ class interface:
         self.devices = {}
         self.pendmsg = None
         interface.list[key] = self
+        self.create()
+
+    def create(self):
         msg = xend.utils.message(CMSG_BLKIF_BE, CMSG_BLKIF_BE_CREATE, 0,
-                                 { 'domid' : dom, 'blkif_handle' : 0 })
+                                 { 'domid' : self.dom, 'blkif_handle' : 0 })
         xend.blkif.pendaddr = xend.main.mgmt_req_addr
         backend_tx_req(msg)
 
@@ -110,6 +156,12 @@ class interface:
         backend_tx_req(msg)
         return True
 
+    def reattach_device(self, vdev):
+        (pdev, start_sect, nr_sect, readonly) = self.devices[vdev]
+        msg = xend.utils.message(CMSG_BLKIF_BE, CMSG_BLKIF_BE_VBD_CREATE, 0)
+        msg.append_payload(struct.pack("IIHII",self.dom,0,vdev,readonly,0))
+        xend.blkif.pendaddr = xend.main.mgmt_req_addr
+        backend_tx_req(msg)
 
     # Completely destroy this interface.
     def destroy(self):
@@ -148,7 +200,9 @@ class interface:
             pl = msg.get_payload()
             (hnd, frame) = (pl['handle'], pl['shmem_frame'])
             xc = Xc.new()
-            self.evtchn = xc.evtchn_bind_interdomain(dom1=0,dom2=self.dom)
+            self.evtchn = xc.evtchn_bind_interdomain( \
+                dom1=xend.blkif.be_port.remote_dom,   \
+                dom2=self.dom)
             msg = xend.utils.message(CMSG_BLKIF_BE,
                                      CMSG_BLKIF_BE_CONNECT, 0,
                                      { 'domid' : self.dom, 'blkif_handle' : 0,
diff --git a/tools/xend/lib/main.py b/tools/xend/lib/main.py
index ce7c9ec44b..08a51a9cb1 100755
--- a/tools/xend/lib/main.py
+++ b/tools/xend/lib/main.py
@@ -72,6 +72,8 @@ def daemon_loop():
     # Note that console messages don't come our way (actually, only driver
     # back-ends should use the DOM0 control interface).
     dom0_port = xend.utils.port(0)
+    xend.netif.be_port = dom0_port
+    xend.blkif_be_port = dom0_port
     notifier.bind(dom0_port.local_port)
     port_list[dom0_port.local_port] = dom0_port
 
@@ -197,7 +199,7 @@ def daemon_loop():
                     con_if.ctrlif_rx_req(port, msg)
                 elif type == CMSG_BLKIF_FE and blk_if:
                     blk_if.ctrlif_rx_req(port, msg)
-                elif type == CMSG_BLKIF_BE and port == dom0_port:
+                elif type == CMSG_BLKIF_BE and port == xend.blkif.be_port:
                     xend.blkif.backend_rx_req(port, msg)
                 elif type == CMSG_NETIF_FE and net_if:
                     net_if.ctrlif_rx_req(port, msg)
@@ -211,7 +213,7 @@ def daemon_loop():
                 msg = port.read_response()
                 work_done = True
                 type = (msg.get_header())['type']
-                if type == CMSG_BLKIF_BE and port == dom0_port:
+                if type == CMSG_BLKIF_BE and port == xend.blkif.be_port:
                     xend.blkif.backend_rx_rsp(port, msg)
                 elif type == CMSG_NETIF_BE and port == xend.netif.be_port:
                     xend.netif.backend_rx_rsp(port, msg)
diff --git a/tools/xend/lib/manager.py b/tools/xend/lib/manager.py
index 3975aa7232..49517583b5 100644
--- a/tools/xend/lib/manager.py
+++ b/tools/xend/lib/manager.py
@@ -162,3 +162,13 @@ def set_network_backend(dom):
         xend.netif.recovery = True
     xend.netif.be_port = xend.main.port_from_dom(dom)
     return { 'success' : True }
+
+##
+## set_block_backend
+##  Authorise a domain to act as the block backend (assumes we only have one
+##  backend driver for now).  After this call, back end "up" notifications
+##  for the network will only be accepted from this domain.
+def set_block_backend(dom):
+    if xend.blkif.be_port: xend.blkif.recovery = True
+    xend.blkif.be_port = xend.main.port_from_dom(dom)
+    return { 'success' : True }
diff --git a/xenolinux-2.4.26-sparse/arch/xen/drivers/blkif/backend/main.c b/xenolinux-2.4.26-sparse/arch/xen/drivers/blkif/backend/main.c
index b5d406ba5e..803af976d2 100644
--- a/xenolinux-2.4.26-sparse/arch/xen/drivers/blkif/backend/main.c
+++ b/xenolinux-2.4.26-sparse/arch/xen/drivers/blkif/backend/main.c
@@ -487,7 +487,8 @@ static int __init init_module(void)
 {
     int i;
 
-    if ( !(start_info.flags & SIF_INITDOMAIN) )
+    if ( !(start_info.flags & SIF_INITDOMAIN)
+	 && !(start_info.flags & SIF_BLK_BE_DOMAIN) )
         return 0;
 
     blkif_interface_init();
diff --git a/xenolinux-2.4.26-sparse/arch/xen/drivers/blkif/frontend/main.c b/xenolinux-2.4.26-sparse/arch/xen/drivers/blkif/frontend/main.c
index 63f1aeea26..4e5dbca093 100644
--- a/xenolinux-2.4.26-sparse/arch/xen/drivers/blkif/frontend/main.c
+++ b/xenolinux-2.4.26-sparse/arch/xen/drivers/blkif/frontend/main.c
@@ -16,6 +16,8 @@
 #include <scsi/scsi.h>
 #include <asm/ctrl_if.h>
 
+
+
 typedef unsigned char byte; /* from linux/ide.h */
 
 #define BLKIF_STATE_CLOSED       0
@@ -31,6 +33,15 @@ static blkif_ring_t *blk_ring;
 static BLK_RING_IDX resp_cons; /* Response consumer for comms ring. */
 static BLK_RING_IDX req_prod;  /* Private request producer.         */
 
+
+static blkif_ring_t *blk_ring_rec; /* Private copy of requests, used for
+                                    * recovery.  Responses not stored here. */
+static BLK_RING_IDX resp_cons_rec; /* Copy of response consumer, used for
+                                    * recovery */
+static int recovery = 0;           /* "Recovery in progress" flag.  Protected
+                                    * by the io_request_lock */
+
+
 /* We plug the I/O ring if the driver is suspended or if the ring is full. */
 #define RING_PLUGGED (((req_prod - resp_cons) == BLK_RING_SIZE) || \
                       (blkif_state != BLKIF_STATE_CONNECTED))
@@ -352,6 +363,11 @@ static int blkif_queue_request(unsigned long   id,
                 sg_next_sect += nr_sectors;
             else
                 DISABLE_SCATTERGATHER();
+
+            /* Update the copy of the request in the recovery ring. */
+            blk_ring_rec->ring[MASK_BLK_IDX(blk_ring_rec->req_prod - 1)].req
+                = *req;
+
             return 0;
         }
         else if ( RING_PLUGGED )
@@ -380,6 +396,10 @@ static int blkif_queue_request(unsigned long   id,
     req->frame_and_sects[0] = buffer_ma | (fsect<<3) | lsect;
     req_prod++;
 
+    /* Keep a private copy so we can reissue requests when recovering. */
+    blk_ring_rec->ring[MASK_BLK_IDX(blk_ring_rec->req_prod)].req = *req;
+    blk_ring_rec->req_prod++;
+
     return 0;
 }
 
@@ -485,11 +505,18 @@ static void blkif_int(int irq, void *dev_id, struct pt_regs *ptregs)
     unsigned long flags; 
     struct buffer_head *bh, *next_bh;
     
-    if ( unlikely(blkif_state == BLKIF_STATE_CLOSED) )
-        return;
-    
+//    printk(KERN_ALERT "blkif_int\n");
+
     spin_lock_irqsave(&io_request_lock, flags);     
 
+    if ( unlikely(blkif_state == BLKIF_STATE_CLOSED || recovery) )
+    {
+        printk("Bailed out\n");
+        
+        spin_unlock_irqrestore(&io_request_lock, flags);
+        return;
+    }
+
     for ( i = resp_cons; i != blk_ring->resp_prod; i++ )
     {
         blkif_response_t *bret = &blk_ring->ring[MASK_BLK_IDX(i)].resp;
@@ -519,6 +546,7 @@ static void blkif_int(int irq, void *dev_id, struct pt_regs *ptregs)
     }
     
     resp_cons = i;
+    resp_cons_rec = i;
 
     kick_pending_request_queues();
 
@@ -546,6 +574,8 @@ void blkif_control_send(blkif_request_t *req, blkif_response_t *rsp)
 
     DISABLE_SCATTERGATHER();
     memcpy(&blk_ring->ring[MASK_BLK_IDX(req_prod)].req, req, sizeof(*req));
+    memcpy(&blk_ring_rec->ring[MASK_BLK_IDX(blk_ring_rec->req_prod++)].req,
+           req, sizeof(*req));
     req_prod++;
     flush_requests();
 
@@ -586,7 +616,19 @@ static void blkif_status_change(blkif_fe_interface_status_changed_t *status)
         {
             printk(KERN_WARNING "Unexpected blkif-DISCONNECTED message"
                    " in state %d\n", blkif_state);
-            break;
+
+            printk(KERN_INFO "VBD driver recovery in progress\n");
+            
+            /* Prevent new requests being issued until we've fixed things up. */
+            spin_lock_irq(&io_request_lock);
+            recovery = 1;
+            blkif_state = BLKIF_STATE_DISCONNECTED;
+            spin_unlock_irq(&io_request_lock);
+
+            /* Free resources associated with old device channel. */
+            free_page((unsigned long)blk_ring);
+            free_irq(blkif_irq, NULL);
+            unbind_evtchn_from_irq(blkif_evtchn);
         }
 
         /* Move from CLOSED to DISCONNECTED state. */
@@ -617,16 +659,55 @@ static void blkif_status_change(blkif_fe_interface_status_changed_t *status)
         blkif_evtchn = status->evtchn;
         blkif_irq = bind_evtchn_to_irq(blkif_evtchn);
         (void)request_irq(blkif_irq, blkif_int, 0, "blkif", NULL);
-        
-        /* Probe for discs that are attached to the interface. */
-        xlvbd_init();
-        
+
+        if ( recovery )
+        {
+            int i;
+
+	    /* Shouldn't need the io_request_lock here - the device is
+	     * plugged and the recovery flag prevents the interrupt handler
+	     * changing anything. */
+
+            /* Reissue requests from the private block ring. */
+            for ( i = 0;
+		  resp_cons_rec < blk_ring_rec->req_prod;
+                  resp_cons_rec++, i++ )
+            {
+                blk_ring->ring[i].req
+                    = blk_ring_rec->ring[MASK_BLK_IDX(resp_cons_rec)].req;
+            }
+
+            /* Reset the private block ring to match the new ring. */
+            memcpy(blk_ring_rec, blk_ring, sizeof(*blk_ring));
+            resp_cons_rec = 0;
+
+            /* blk_ring->req_prod will be set when we flush_requests().*/
+            blk_ring_rec->req_prod = req_prod = i;
+
+            wmb();
+
+            /* Switch off recovery mode, using a memory barrier to ensure that
+             * it's seen before we flush requests - we don't want to miss any
+             * interrupts. */
+            recovery = 0;
+            wmb();
+
+            /* Kicks things back into life. */
+            flush_requests();
+        }
+        else
+        {
+            /* Probe for discs that are attached to the interface. */
+            xlvbd_init();
+        }
+
         blkif_state = BLKIF_STATE_CONNECTED;
         
         /* Kick pending requests. */
         spin_lock_irq(&io_request_lock);
         kick_pending_request_queues();
         spin_unlock_irq(&io_request_lock);
+
         break;
 
     default:
@@ -671,9 +752,15 @@ int __init xlblk_init(void)
     ctrl_msg_t                       cmsg;
     blkif_fe_driver_status_changed_t st;
 
-    if ( start_info.flags & SIF_INITDOMAIN )
+    if ( (start_info.flags & SIF_INITDOMAIN) 
+        || (start_info.flags & SIF_BLK_BE_DOMAIN) )
         return 0;
 
+    printk(KERN_INFO "Initialising Xen virtual block device\n");
+
+    blk_ring_rec = (blkif_ring_t *)__get_free_page(GFP_KERNEL);
+    memset(blk_ring_rec, 0, sizeof(*blk_ring_rec));
+
     (void)ctrl_if_register_receiver(CMSG_BLKIF_FE, blkif_ctrlif_rx,
                                     CALLBACK_IN_BLOCKING_CONTEXT);