bitkeeper revision 1.906 (40a5e91cnvIS_3gLwfnD2G3HV3odHA)

manual merge

2 files changed, 404 insertions, 162 deletions

diff --git a/xenolinux-2.4.26-sparse/arch/xen/drivers/netif/backend/main.c b/xenolinux-2.4.26-sparse/arch/xen/drivers/netif/backend/main.c
index 7a32149404..5b563f41d9 100644
--- a/xenolinux-2.4.26-sparse/arch/xen/drivers/netif/backend/main.c
+++ b/xenolinux-2.4.26-sparse/arch/xen/drivers/netif/backend/main.c
@@ -13,19 +13,33 @@
 #include "common.h"
 #include <asm/hypervisor-ifs/dom_mem_ops.h>
 
-static void net_tx_action(unsigned long unused);
 static void netif_page_release(struct page *page);
 static void make_tx_response(netif_t *netif, 
                              u16      id,
                              s8       st);
-static void make_rx_response(netif_t     *netif, 
+static int  make_rx_response(netif_t     *netif, 
                              u16          id, 
                              s8           st,
                              netif_addr_t addr,
                              u16          size);
 
+static void net_tx_action(unsigned long unused);
 static DECLARE_TASKLET(net_tx_tasklet, net_tx_action, 0);
 
+static void net_rx_action(unsigned long unused);
+static DECLARE_TASKLET(net_rx_tasklet, net_rx_action, 0);
+
+typedef struct {
+    u16 id;
+    unsigned long old_mach_ptr;
+    unsigned long new_mach_pfn;
+    netif_t *netif;
+} rx_info_t;
+static struct sk_buff_head rx_queue;
+static multicall_entry_t rx_mcl[NETIF_RX_RING_SIZE*2];
+static mmu_update_t rx_mmu[NETIF_RX_RING_SIZE*4];
+static unsigned char rx_notify[NR_EVENT_CHANNELS];
+
 /* Don't currently gate addition of an interface to the tx scheduling list. */
 #define tx_work_exists(_if) (1)
 
@@ -38,12 +52,24 @@ static unsigned long mmap_vstart;
 static u16 pending_id[MAX_PENDING_REQS];
 static netif_t *pending_netif[MAX_PENDING_REQS];
 static u16 pending_ring[MAX_PENDING_REQS];
-static spinlock_t pend_prod_lock = SPIN_LOCK_UNLOCKED;
 typedef unsigned int PEND_RING_IDX;
 #define MASK_PEND_IDX(_i) ((_i)&(MAX_PENDING_REQS-1))
 static PEND_RING_IDX pending_prod, pending_cons;
 #define NR_PENDING_REQS (MAX_PENDING_REQS - pending_prod + pending_cons)
 
+/* Freed TX SKBs get batched on this ring before return to pending_ring. */
+static u16 dealloc_ring[MAX_PENDING_REQS];
+static spinlock_t dealloc_lock = SPIN_LOCK_UNLOCKED;
+static PEND_RING_IDX dealloc_prod, dealloc_cons;
+
+typedef struct {
+    u16 idx;
+    netif_tx_request_t req;
+    netif_t *netif;
+} tx_info_t;
+static struct sk_buff_head tx_queue;
+static multicall_entry_t tx_mcl[MAX_PENDING_REQS];
+
 static struct list_head net_schedule_list;
 static spinlock_t net_schedule_list_lock;
 
@@ -98,22 +124,12 @@ static inline void maybe_schedule_tx_action(void)
 int netif_be_start_xmit(struct sk_buff *skb, struct net_device *dev)
 {
     netif_t *netif = (netif_t *)dev->priv;
-    s8 status = NETIF_RSP_OKAY;
-    u16 size=0, id;
-    mmu_update_t mmu[4];
-    multicall_entry_t mcl[2];
-    unsigned long vdata, mdata=0, new_mfn;
 
     /* Drop the packet if the target domain has no receive buffers. */
     if ( (netif->rx_req_cons == netif->rx->req_prod) ||
          ((netif->rx_req_cons-netif->rx_resp_prod) == NETIF_RX_RING_SIZE) )
-    {
-        dev_kfree_skb(skb);
-        return 0;
-    }
+        goto drop;
 
-    id = netif->rx->ring[MASK_NETIF_RX_IDX(netif->rx_req_cons++)].req.id;
- 
     /*
      * We do not copy the packet unless:
      *  1. The data is shared; or
@@ -130,11 +146,7 @@ int netif_be_start_xmit(struct sk_buff *skb, struct net_device *dev)
         struct sk_buff *nskb = alloc_skb(PAGE_SIZE-1024, GFP_ATOMIC);
         int hlen = skb->data - skb->head;
         if ( unlikely(nskb == NULL) )
-        {
-            DPRINTK("DOM%llu couldn't get memory for skb.\n", netif->domid);
-            status = NETIF_RSP_ERROR;
-            goto out;
-        }
+            goto drop;
         skb_reserve(nskb, hlen);
         __skb_put(nskb, skb->len);
         (void)skb_copy_bits(skb, -hlen, nskb->head, hlen + skb->len);
@@ -142,63 +154,164 @@ int netif_be_start_xmit(struct sk_buff *skb, struct net_device *dev)
         skb = nskb;
     }
 
-    vdata = (unsigned long)skb->data;
-    mdata = virt_to_machine(vdata);
-    size  = skb->tail - skb->data;
-
-    new_mfn = get_new_mfn();
-
-    mmu[0].ptr  = (new_mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
-    mmu[0].val  = __pa(vdata) >> PAGE_SHIFT;
-
-    mmu[1].val  = (unsigned long)(netif->domid<<16) & ~0xFFFFUL;
-    mmu[1].ptr  = (unsigned long)(netif->domid<< 0) & ~0xFFFFUL;
-    mmu[2].val  = (unsigned long)(netif->domid>>16) & ~0xFFFFUL;
-    mmu[2].ptr  = (unsigned long)(netif->domid>>32) & ~0xFFFFUL;
-    mmu[1].ptr |= MMU_EXTENDED_COMMAND;
-    mmu[1].val |= MMUEXT_SET_SUBJECTDOM_L;
-    mmu[2].ptr |= MMU_EXTENDED_COMMAND;
-    mmu[2].val |= MMUEXT_SET_SUBJECTDOM_H;
-
-    mmu[3].ptr  = (mdata & PAGE_MASK) | MMU_EXTENDED_COMMAND;
-    mmu[3].val  = MMUEXT_REASSIGN_PAGE;
-
-    mcl[0].op = __HYPERVISOR_mmu_update;
-    mcl[0].args[0] = (unsigned long)mmu;
-    mcl[0].args[1] = 4;
-    mcl[1].op = __HYPERVISOR_update_va_mapping;
-    mcl[1].args[0] = vdata >> PAGE_SHIFT;
-    mcl[1].args[1] = (new_mfn << PAGE_SHIFT) | __PAGE_KERNEL;
-    mcl[1].args[2] = UVMF_INVLPG;
-
-    (void)HYPERVISOR_multicall(mcl, 2);
-    if ( mcl[0].args[5] != 0 )
+    ((rx_info_t *)&skb->cb[0])->id    =
+        netif->rx->ring[MASK_NETIF_RX_IDX(netif->rx_req_cons++)].req.id;
+    ((rx_info_t *)&skb->cb[0])->netif = netif;
+        
+    __skb_queue_tail(&rx_queue, skb);
+    tasklet_schedule(&net_rx_tasklet);
+
+    return 0;
+
+ drop:
+    netif->stats.rx_dropped++;
+    dev_kfree_skb(skb);
+    return 0;
+}
+
+#if 0
+static void xen_network_done_notify(void)
+{
+    static struct net_device *eth0_dev = NULL;
+    if ( unlikely(eth0_dev == NULL) )
+        eth0_dev = __dev_get_by_name("eth0");
+    netif_rx_schedule(eth0_dev);
+}
+/* 
+ * Add following to poll() function in NAPI driver (Tigon3 is example):
+ *  if ( xen_network_done() )
+ *      tge_3nable_ints(tp); 
+ */
+int xen_network_done(void)
+{
+    return skb_queue_empty(&rx_queue);
+}
+#endif
+
+static void net_rx_action(unsigned long unused)
+{
+    netif_t *netif;
+    s8 status;
+    u16 size, id, evtchn;
+    mmu_update_t *mmu = rx_mmu;
+    multicall_entry_t *mcl;
+    unsigned long vdata, mdata, new_mfn;
+    struct sk_buff_head rxq;
+    struct sk_buff *skb;
+    u16 notify_list[NETIF_RX_RING_SIZE];
+    int notify_nr = 0;
+
+    skb_queue_head_init(&rxq);
+
+    mcl = rx_mcl;
+    while ( (skb = __skb_dequeue(&rx_queue)) != NULL )
     {
-        DPRINTK("Failed MMU update transferring to DOM%llu\n", netif->domid);
-        (void)HYPERVISOR_update_va_mapping(
-            vdata >> PAGE_SHIFT,
-            (pte_t) { (mdata & PAGE_MASK) | __PAGE_KERNEL },
-            UVMF_INVLPG);
-        dealloc_mfn(new_mfn);
-        status = NETIF_RSP_ERROR;
-        goto out;
+        netif   = ((rx_info_t *)&skb->cb[0])->netif;
+        vdata   = (unsigned long)skb->data;
+        mdata   = virt_to_machine(vdata);
+        new_mfn = get_new_mfn();
+        
+        mmu[0].ptr  = (new_mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
+        mmu[0].val  = __pa(vdata) >> PAGE_SHIFT;        
+        mmu[1].val  = (unsigned long)(netif->domid<<16) & ~0xFFFFUL;
+        mmu[1].ptr  = (unsigned long)(netif->domid<< 0) & ~0xFFFFUL;
+        mmu[2].val  = (unsigned long)(netif->domid>>16) & ~0xFFFFUL;
+        mmu[2].ptr  = (unsigned long)(netif->domid>>32) & ~0xFFFFUL;
+        mmu[1].ptr |= MMU_EXTENDED_COMMAND;
+        mmu[1].val |= MMUEXT_SET_SUBJECTDOM_L;
+        mmu[2].ptr |= MMU_EXTENDED_COMMAND;
+        mmu[2].val |= MMUEXT_SET_SUBJECTDOM_H;
+        mmu[3].ptr  = (mdata & PAGE_MASK) | MMU_EXTENDED_COMMAND;
+        mmu[3].val  = MMUEXT_REASSIGN_PAGE;
+
+        mcl[0].op = __HYPERVISOR_update_va_mapping;
+        mcl[0].args[0] = vdata >> PAGE_SHIFT;
+        mcl[0].args[1] = (new_mfn << PAGE_SHIFT) | __PAGE_KERNEL;
+        mcl[0].args[2] = 0;
+        mcl[1].op = __HYPERVISOR_mmu_update;
+        mcl[1].args[0] = (unsigned long)mmu;
+        mcl[1].args[1] = 4;
+
+        mmu += 4;
+        mcl += 2;
+
+        ((rx_info_t *)&skb->cb[0])->old_mach_ptr = mdata;
+        ((rx_info_t *)&skb->cb[0])->new_mach_pfn = new_mfn;
+        __skb_queue_tail(&rxq, skb);
+
+        /* Filled the batch queue? */
+        if ( (mcl - rx_mcl) == ARRAY_SIZE(rx_mcl) )
+            break;
     }
 
-    phys_to_machine_mapping[__pa(vdata) >> PAGE_SHIFT] = new_mfn;
+    if ( mcl == rx_mcl )
+        return;
+
+    mcl[-2].args[2] = UVMF_FLUSH_TLB;
+    (void)HYPERVISOR_multicall(rx_mcl, mcl - rx_mcl);
 
-    atomic_set(&(skb_shinfo(skb)->dataref), 1);
-    skb_shinfo(skb)->nr_frags = 0;
-    skb_shinfo(skb)->frag_list = NULL;
+    mcl = rx_mcl;
+    while ( (skb = __skb_dequeue(&rxq)) != NULL )
+    {
+        netif   = ((rx_info_t *)&skb->cb[0])->netif;
+        size    = skb->tail - skb->data;
+        id      = ((rx_info_t *)&skb->cb[0])->id;
+        new_mfn = ((rx_info_t *)&skb->cb[0])->new_mach_pfn;
+        mdata   = ((rx_info_t *)&skb->cb[0])->old_mach_ptr;
+
+        /* Check the reassignment error code. */
+        if ( unlikely(mcl[1].args[5] != 0) )
+        {
+            DPRINTK("Failed MMU update transferring to DOM%llu\n",
+                    netif->domid);
+            (void)HYPERVISOR_update_va_mapping(
+                (unsigned long)skb->head >> PAGE_SHIFT,
+                (pte_t) { (mdata & PAGE_MASK) | __PAGE_KERNEL },
+                UVMF_INVLPG);
+            dealloc_mfn(new_mfn);
+            status = NETIF_RSP_ERROR;
+        }
+        else
+        {
+            phys_to_machine_mapping[__pa(skb->data) >> PAGE_SHIFT] = new_mfn;
 
-    netif->stats.rx_bytes += size;
-    netif->stats.rx_packets++;
+            atomic_set(&(skb_shinfo(skb)->dataref), 1);
+            skb_shinfo(skb)->nr_frags = 0;
+            skb_shinfo(skb)->frag_list = NULL;
 
- out:
-    spin_lock(&netif->rx_lock);
-    make_rx_response(netif, id, status, mdata, size);
-    spin_unlock(&netif->rx_lock);    
-    dev_kfree_skb(skb);
-    return 0;
+            netif->stats.rx_bytes += size;
+            netif->stats.rx_packets++;
+
+            status = NETIF_RSP_OKAY;
+        }
+
+        evtchn = netif->evtchn;
+        if ( make_rx_response(netif, id, status, mdata, size) &&
+             (rx_notify[evtchn] == 0) )
+        {
+            rx_notify[evtchn] = 1;
+            notify_list[notify_nr++] = evtchn;
+        }
+
+        dev_kfree_skb(skb);
+
+        mcl += 2;
+    }
+
+    while ( notify_nr != 0 )
+    {
+        evtchn = notify_list[--notify_nr];
+        rx_notify[evtchn] = 0;
+        notify_via_evtchn(evtchn);
+    }
+
+    /* More work to do? */
+    if ( !skb_queue_empty(&rx_queue) )
+        tasklet_schedule(&net_rx_tasklet);
+#if 0
+    else
+        xen_network_done_notify();
+#endif
 }
 
 struct net_device_stats *netif_be_get_stats(struct net_device *dev)
@@ -215,10 +328,12 @@ static int __on_net_schedule_list(netif_t *netif)
 static void remove_from_net_schedule_list(netif_t *netif)
 {
     spin_lock(&net_schedule_list_lock);
-    ASSERT(__on_net_schedule_list(netif));
-    list_del(&netif->list);
-    netif->list.next = NULL;
-    netif_put(netif);
+    if ( likely(__on_net_schedule_list(netif)) )
+    {
+        list_del(&netif->list);
+        netif->list.next = NULL;
+        netif_put(netif);
+    }
     spin_unlock(&net_schedule_list_lock);
 }
 
@@ -269,7 +384,51 @@ static void net_tx_action(unsigned long unused)
     u16 pending_idx;
     NETIF_RING_IDX i;
     struct page *page;
+    multicall_entry_t *mcl;
+
+    if ( (i = dealloc_cons) == dealloc_prod )
+        goto skip_dealloc;
+
+    mcl = tx_mcl;
+    while ( i != dealloc_prod )
+    {
+        pending_idx = dealloc_ring[MASK_PEND_IDX(i++)];
+        mcl[0].op = __HYPERVISOR_update_va_mapping;
+        mcl[0].args[0] = MMAP_VADDR(pending_idx) >> PAGE_SHIFT;
+        mcl[0].args[1] = 0;
+        mcl[0].args[2] = 0;
+        mcl++;        
+    }
+
+    mcl[-1].args[2] = UVMF_FLUSH_TLB;
+    (void)HYPERVISOR_multicall(tx_mcl, mcl - tx_mcl);
+
+    while ( dealloc_cons != dealloc_prod )
+    {
+        pending_idx = dealloc_ring[MASK_PEND_IDX(dealloc_cons++)];
+
+        netif = pending_netif[pending_idx];
+
+        spin_lock(&netif->tx_lock);
+        make_tx_response(netif, pending_id[pending_idx], NETIF_RSP_OKAY);
+        spin_unlock(&netif->tx_lock);
+        
+        pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
+        
+        /*
+         * Scheduling checks must happen after the above response is posted.
+         * This avoids a possible race with a guest OS on another CPU.
+         */
+        mb();
+        if ( (netif->tx_req_cons != netif->tx->req_prod) &&
+             ((netif->tx_req_cons-netif->tx_resp_prod) != NETIF_TX_RING_SIZE) )
+            add_to_net_schedule_list_tail(netif);
+        
+        netif_put(netif);
+    }
 
+ skip_dealloc:
+    mcl = tx_mcl;
     while ( (NR_PENDING_REQS < MAX_PENDING_REQS) &&
             !list_empty(&net_schedule_list) )
     {
@@ -340,29 +499,61 @@ static void net_tx_action(unsigned long unused)
 
         pending_idx = pending_ring[MASK_PEND_IDX(pending_cons)];
 
-        if ( HYPERVISOR_update_va_mapping_otherdomain(
-            MMAP_VADDR(pending_idx) >> PAGE_SHIFT,
-            (pte_t) { (txreq.addr & PAGE_MASK) | __PAGE_KERNEL },
-            0, netif->domid) != 0 )
+        if ( unlikely((skb = alloc_skb(PKT_PROT_LEN, GFP_ATOMIC)) == NULL) )
         {
-            DPRINTK("Bad page frame\n");
+            DPRINTK("Can't allocate a skb in start_xmit.\n");
             make_tx_response(netif, txreq.id, NETIF_RSP_ERROR);
             netif_put(netif);
-            continue;
+            break;
         }
-        phys_to_machine_mapping[__pa(MMAP_VADDR(pending_idx)) >> PAGE_SHIFT] =
-            txreq.addr >> PAGE_SHIFT;
 
-        if ( unlikely((skb = alloc_skb(PKT_PROT_LEN, GFP_ATOMIC)) == NULL) )
+        mcl[0].op = __HYPERVISOR_update_va_mapping_otherdomain;
+        mcl[0].args[0] = MMAP_VADDR(pending_idx) >> PAGE_SHIFT;
+        mcl[0].args[1] = (txreq.addr & PAGE_MASK) | __PAGE_KERNEL;
+        mcl[0].args[2] = 0;
+        mcl[0].args[3] = (unsigned long)netif->domid;
+        mcl[0].args[4] = (unsigned long)(netif->domid>>32);
+        mcl++;
+        
+        ((tx_info_t *)&skb->cb[0])->idx = pending_idx;
+        ((tx_info_t *)&skb->cb[0])->netif = netif;
+        memcpy(&((tx_info_t *)&skb->cb[0])->req, &txreq, sizeof(txreq));
+        __skb_queue_tail(&tx_queue, skb);
+
+        pending_cons++;
+
+        /* Filled the batch queue? */
+        if ( (mcl - tx_mcl) == ARRAY_SIZE(tx_mcl) )
+            break;
+    }
+
+    if ( mcl == tx_mcl )
+        return;
+
+    (void)HYPERVISOR_multicall(tx_mcl, mcl - tx_mcl);
+
+    mcl = tx_mcl;
+    while ( (skb = __skb_dequeue(&tx_queue)) != NULL )
+    {
+        pending_idx = ((tx_info_t *)&skb->cb[0])->idx;
+        netif       = ((tx_info_t *)&skb->cb[0])->netif;
+        memcpy(&txreq, &((tx_info_t *)&skb->cb[0])->req, sizeof(txreq));
+
+        /* Check the remap error code. */
+        if ( unlikely(mcl[0].args[5] != 0) )
         {
-            DPRINTK("Can't allocate a skb in start_xmit.\n");
+            DPRINTK("Bad page frame\n");
             make_tx_response(netif, txreq.id, NETIF_RSP_ERROR);
             netif_put(netif);
-            HYPERVISOR_update_va_mapping(MMAP_VADDR(pending_idx) >> PAGE_SHIFT,
-                                         (pte_t) { 0 }, UVMF_INVLPG);
-            break;
+            kfree_skb(skb);
+            mcl++;
+            pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
+            continue;
         }
-        
+
+        phys_to_machine_mapping[__pa(MMAP_VADDR(pending_idx)) >> PAGE_SHIFT] =
+            txreq.addr >> PAGE_SHIFT;
+
         __skb_put(skb, PKT_PROT_LEN);
         memcpy(skb->data, 
                (void *)(MMAP_VADDR(pending_idx)|(txreq.addr&~PAGE_MASK)),
@@ -391,42 +582,23 @@ static void net_tx_action(unsigned long unused)
         netif->stats.tx_bytes += txreq.size;
         netif->stats.tx_packets++;
 
-        pending_cons++;
-
         netif_rx(skb);
         netif->dev->last_rx = jiffies;
+
+        mcl++;
     }
 }
 
 static void netif_page_release(struct page *page)
 {
     unsigned long flags;
-    netif_t *netif;
-    u16 pending_idx;
-
-    pending_idx = page - virt_to_page(mmap_vstart);
+    u16 pending_idx = page - virt_to_page(mmap_vstart);
 
-    netif = pending_netif[pending_idx];
-
-    HYPERVISOR_update_va_mapping(MMAP_VADDR(pending_idx) >> PAGE_SHIFT,
-                                 (pte_t) { 0 }, UVMF_INVLPG);
-        
-    spin_lock(&netif->tx_lock);
-    make_tx_response(netif, pending_id[pending_idx], NETIF_RSP_OKAY);
-    spin_unlock(&netif->tx_lock);
+    spin_lock_irqsave(&dealloc_lock, flags);
+    dealloc_ring[MASK_PEND_IDX(dealloc_prod++)] = pending_idx;
+    spin_unlock_irqrestore(&dealloc_lock, flags);
 
-    /*
-     * Scheduling checks must happen after the above response is posted.
-     * This avoids a possible race with a guest OS on another CPU.
-     */
-    mb();
-    netif_schedule_work(netif);
-
-    netif_put(netif);
- 
-    spin_lock_irqsave(&pend_prod_lock, flags);
-    pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
-    spin_unlock_irqrestore(&pend_prod_lock, flags);
+    tasklet_schedule(&net_tx_tasklet);
 }
 
 #if 0
@@ -497,11 +669,11 @@ static void make_tx_response(netif_t *netif,
         notify_via_evtchn(netif->evtchn);
 }
 
-static void make_rx_response(netif_t     *netif, 
-                             u16          id, 
-                             s8           st,
-                             netif_addr_t addr,
-                             u16          size)
+static int make_rx_response(netif_t     *netif, 
+                            u16          id, 
+                            s8           st,
+                            netif_addr_t addr,
+                            u16          size)
 {
     NET_RING_IDX i = netif->rx_resp_prod;
     netif_rx_response_t *resp;
@@ -516,8 +688,7 @@ static void make_rx_response(netif_t     *netif,
     netif->rx->resp_prod = netif->rx_resp_prod = ++i;
 
     mb(); /* Update producer before checking event threshold. */
-    if ( i == netif->rx->event )
-        notify_via_evtchn(netif->evtchn);
+    return (i == netif->rx->event);
 }
 
 static int __init init_module(void)
@@ -527,6 +698,9 @@ static int __init init_module(void)
     if ( !(start_info.flags & SIF_INITDOMAIN) )
         return 0;
 
+    skb_queue_head_init(&rx_queue);
+    skb_queue_head_init(&tx_queue);
+
     netif_interface_init();
 
     if ( (mmap_vstart = allocate_empty_lowmem_region(MAX_PENDING_REQS)) == 0 )
diff --git a/xenolinux-2.4.26-sparse/arch/xen/drivers/netif/frontend/main.c b/xenolinux-2.4.26-sparse/arch/xen/drivers/netif/frontend/main.c
index ff73952e88..85f8375f8e 100644
--- a/xenolinux-2.4.26-sparse/arch/xen/drivers/netif/frontend/main.c
+++ b/xenolinux-2.4.26-sparse/arch/xen/drivers/netif/frontend/main.c
@@ -37,6 +37,10 @@ static void network_tx_buf_gc(struct net_device *dev);
 static void network_alloc_rx_buffers(struct net_device *dev);
 static void cleanup_module(void);
 
+static unsigned long rx_pfn_array[NETIF_RX_RING_SIZE];
+static multicall_entry_t rx_mcl[NETIF_RX_RING_SIZE+1];
+static mmu_update_t rx_mmu[NETIF_RX_RING_SIZE];
+
 static struct list_head dev_list;
 
 struct net_private
@@ -178,8 +182,7 @@ static void network_alloc_rx_buffers(struct net_device *dev)
     struct sk_buff *skb;
     NETIF_RING_IDX i = np->rx->req_prod;
     dom_mem_op_t op;
-    unsigned long pfn_array[NETIF_RX_RING_SIZE];
-    int ret, nr_pfns = 0;
+    int nr_pfns = 0;
 
     /* Make sure the batch is large enough to be worthwhile (1/2 ring). */
     if ( unlikely((i - np->rx_resp_cons) > (NETIF_RX_RING_SIZE/2)) || 
@@ -201,9 +204,14 @@ static void network_alloc_rx_buffers(struct net_device *dev)
 
         np->rx->ring[MASK_NET_RX_IDX(i)].req.id = id;
         
-        pfn_array[nr_pfns++] = virt_to_machine(skb->head) >> PAGE_SHIFT;
-        HYPERVISOR_update_va_mapping((unsigned long)skb->head >> PAGE_SHIFT,
-                                     (pte_t) { 0 }, UVMF_INVLPG);
+        rx_pfn_array[nr_pfns] = virt_to_machine(skb->head) >> PAGE_SHIFT;
+
+        rx_mcl[nr_pfns].op = __HYPERVISOR_update_va_mapping;
+        rx_mcl[nr_pfns].args[0] = (unsigned long)skb->head >> PAGE_SHIFT;
+        rx_mcl[nr_pfns].args[1] = 0;
+        rx_mcl[nr_pfns].args[2] = 0;
+
+        nr_pfns++;
     }
     while ( (++i - np->rx_resp_cons) != NETIF_RX_RING_SIZE );
 
@@ -213,14 +221,22 @@ static void network_alloc_rx_buffers(struct net_device *dev)
      */
     flush_page_update_queue();
 
+    /* After all PTEs have been zapped we blow away stale TLB entries. */
+    rx_mcl[nr_pfns-1].args[2] = UVMF_FLUSH_TLB;
+
+    /* Give away a batch of pages. */
     op.op = MEMOP_RESERVATION_DECREASE;
     op.u.decrease.size  = nr_pfns;
-    op.u.decrease.pages = pfn_array;
-    if ( (ret = HYPERVISOR_dom_mem_op(&op)) != nr_pfns )
-    {
-        printk(KERN_WARNING "Unable to reduce memory reservation (%d)\n", ret);
-        BUG();
-    }
+    op.u.decrease.pages = rx_pfn_array;
+    rx_mcl[nr_pfns].op = __HYPERVISOR_dom_mem_op;
+    rx_mcl[nr_pfns].args[0] = (unsigned long)&op;
+
+    /* Zap PTEs and give away pages in one big multicall. */
+    (void)HYPERVISOR_multicall(rx_mcl, nr_pfns+1);
+
+    /* Check return status of HYPERVISOR_dom_mem_op(). */
+    if ( rx_mcl[nr_pfns].args[5] != nr_pfns )
+        panic("Unable to reduce memory reservation\n");
 
     np->rx->req_prod = i;
 }
@@ -295,17 +311,36 @@ static void netif_int(int irq, void *dev_id, struct pt_regs *ptregs)
     struct net_device *dev = dev_id;
     struct net_private *np = dev->priv;
     unsigned long flags;
-    struct sk_buff *skb;
-    netif_rx_response_t *rx;
-    NETIF_RING_IDX i;
-    mmu_update_t mmu;
 
     spin_lock_irqsave(&np->tx_lock, flags);
     network_tx_buf_gc(dev);
     spin_unlock_irqrestore(&np->tx_lock, flags);
 
- again:
-    for ( i = np->rx_resp_cons; i != np->rx->resp_prod; i++ )
+    if ( np->rx_resp_cons != np->rx->resp_prod )
+        netif_rx_schedule(dev);
+}
+
+
+static int netif_poll(struct net_device *dev, int *pbudget)
+{
+    struct net_private *np = dev->priv;
+    struct sk_buff *skb;
+    netif_rx_response_t *rx;
+    NETIF_RING_IDX i;
+    mmu_update_t *mmu = rx_mmu;
+    multicall_entry_t *mcl = rx_mcl;
+    int work_done, budget, more_to_do = 1;
+    struct sk_buff_head rxq;
+    unsigned long flags;
+
+    skb_queue_head_init(&rxq);
+
+    if ( (budget = *pbudget) > dev->quota )
+        budget = dev->quota;
+
+    for ( i = np->rx_resp_cons, work_done = 0; 
+          (i != np->rx->resp_prod) && (work_done < budget); 
+          i++, work_done++ )
     {
         rx = &np->rx->ring[MASK_NET_RX_IDX(i)].resp;
 
@@ -317,38 +352,53 @@ static void netif_int(int irq, void *dev_id, struct pt_regs *ptregs)
             /* Gate this error. We get a (valid) slew of them on suspend. */
             if ( np->state == NETIF_STATE_ACTIVE )
                 printk(KERN_ALERT "bad buffer on RX ring!(%d)\n", rx->status);
-            dev_kfree_skb_any(skb);
+            dev_kfree_skb(skb);
             continue;
         }
 
+        skb->data = skb->tail = skb->head + (rx->addr & ~PAGE_MASK);
+        skb_put(skb, rx->status);
+
+        np->stats.rx_packets++;
+        np->stats.rx_bytes += rx->status;
+
         /* Remap the page. */
-        mmu.ptr  = (rx->addr & PAGE_MASK) | MMU_MACHPHYS_UPDATE;
-        mmu.val  = __pa(skb->head) >> PAGE_SHIFT;
-        if ( HYPERVISOR_mmu_update(&mmu, 1) != 0 )
-            BUG();
-        HYPERVISOR_update_va_mapping((unsigned long)skb->head >> PAGE_SHIFT,
-                                     (pte_t) { (rx->addr & PAGE_MASK) | 
-                                                   __PAGE_KERNEL },
-                                     0);
+        mmu->ptr  = (rx->addr & PAGE_MASK) | MMU_MACHPHYS_UPDATE;
+        mmu->val  = __pa(skb->head) >> PAGE_SHIFT;
+        mmu++;
+        mcl->op = __HYPERVISOR_update_va_mapping;
+        mcl->args[0] = (unsigned long)skb->head >> PAGE_SHIFT;
+        mcl->args[1] = (rx->addr & PAGE_MASK) | __PAGE_KERNEL;
+        mcl->args[2] = 0;
+        mcl++;
+
         phys_to_machine_mapping[__pa(skb->head) >> PAGE_SHIFT] = 
             rx->addr >> PAGE_SHIFT;
 
-        /*
-         * Set up shinfo -- from alloc_skb This was particularily nasty:  the
-         * shared info is hidden at the back of the data area (presumably so it
-         * can be shared), but on page flip it gets very spunked.
-         */
+        __skb_queue_tail(&rxq, skb);
+    }
+
+    /* Do all the remapping work, and M->P updates, in one big hypercall. */
+    if ( likely((mcl - rx_mcl) != 0) )
+    {
+        mcl->op = __HYPERVISOR_mmu_update;
+        mcl->args[0] = (unsigned long)rx_mmu;
+        mcl->args[1] = mmu - rx_mmu;
+        mcl++;
+        (void)HYPERVISOR_multicall(rx_mcl, mcl - rx_mcl);
+    }
+
+    while ( (skb = __skb_dequeue(&rxq)) != NULL )
+    {
+        /* Set the shared-info area, which is hidden behind the real data. */
         atomic_set(&(skb_shinfo(skb)->dataref), 1);
         skb_shinfo(skb)->nr_frags = 0;
         skb_shinfo(skb)->frag_list = NULL;
 
-        skb->data = skb->tail = skb->head + (rx->addr & ~PAGE_MASK);
-        skb_put(skb, rx->status);
+        /* Ethernet-specific work. Delayed to here as it peeks the header. */
         skb->protocol = eth_type_trans(skb, dev);
 
-        np->stats.rx_packets++;
-
-        np->stats.rx_bytes += rx->status;
+        /* Pass it up. */
         netif_rx(skb);
         dev->last_rx = jiffies;
     }
@@ -356,12 +406,28 @@ static void netif_int(int irq, void *dev_id, struct pt_regs *ptregs)
     np->rx_resp_cons = i;
 
     network_alloc_rx_buffers(dev);
-    np->rx->event = np->rx_resp_cons + 1;
+
+    *pbudget   -= work_done;
+    dev->quota -= work_done;
+
+    if ( work_done < budget )
+    {
+        local_irq_save(flags);
+
+        np->rx->event = i + 1;
     
-    /* Deal with hypervisor racing our resetting of rx_event. */
-    mb();
-    if ( np->rx->resp_prod != i )
-        goto again;
+        /* Deal with hypervisor racing our resetting of rx_event. */
+        mb();
+        if ( np->rx->resp_prod == i )
+        {
+            __netif_rx_complete(dev);
+            more_to_do = 0;
+        }
+
+        local_irq_restore(flags);
+    }
+
+    return more_to_do;
 }
 
 
@@ -524,6 +590,8 @@ static int __init init_module(void)
     dev->hard_start_xmit = network_start_xmit;
     dev->stop            = network_close;
     dev->get_stats       = network_get_stats;
+    dev->poll            = netif_poll;
+    dev->weight          = 64;
     
     if ( (err = register_netdev(dev)) != 0 )
     {