bitkeeper revision 1.670 (3ffaf723Sn5FVTMBSOvb9binR3cthQ)

network.c, dev.c, pgalloc.h, flushtlb.h, domain.c, smp.c:
  Fixed heinous TLB-flush and network bugs.

6 files changed, 55 insertions, 21 deletions

diff --git a/xen/arch/i386/smp.c b/xen/arch/i386/smp.c
index 4ec5176194..f60a42dd0c 100644
--- a/xen/arch/i386/smp.c
+++ b/xen/arch/i386/smp.c
@@ -269,8 +269,34 @@ asmlinkage void smp_invalidate_interrupt(void)
     local_flush_tlb();
 }
 
+int try_flush_tlb_mask(unsigned long mask)
+{
+    if ( mask & (1 << smp_processor_id()) )
+    {
+        local_flush_tlb();
+        mask &= ~(1 << smp_processor_id());
+    }
+
+    if ( mask != 0 )
+    {
+        if ( unlikely(!spin_trylock(&tlbstate_lock)) )
+            return 0;
+        flush_cpumask = mask;
+        send_IPI_mask(mask, INVALIDATE_TLB_VECTOR);
+        while ( flush_cpumask != 0 )
+        {
+            rep_nop();
+            barrier();
+        }
+        spin_unlock(&tlbstate_lock);
+    }
+
+    return 1;
+}
+
 void flush_tlb_mask(unsigned long mask)
 {
+    /* WARNING: Only try_flush_tlb_mask() is safe in IRQ context. */
     if ( unlikely(in_irq()) )
         BUG();
     
diff --git a/xen/common/domain.c b/xen/common/domain.c
index 9cb4437d70..147d9c0f97 100644
--- a/xen/common/domain.c
+++ b/xen/common/domain.c
@@ -17,7 +17,6 @@
 #include <xeno/blkdev.h>
 #include <xeno/console.h>
 #include <xeno/vbd.h>
-
 #include <asm/i387.h>
 
 /*
@@ -237,12 +236,12 @@ struct pfn_info *alloc_domain_page(struct task_struct *p)
     if ( unlikely(page == NULL) )
         return NULL;
 
-    if ( unlikely((mask = page->u.cpu_mask) != 0) )
+    if ( (mask = page->u.cpu_mask) != 0 )
     {
         pfn_stamp = page->tlbflush_timestamp;
         for ( i = 0; (mask != 0) && (i < NR_CPUS); i++ )
         {
-            if ( unlikely(mask & (1<<i)) )
+            if ( mask & (1<<i) )
             {
                 cpu_stamp = tlbflush_time[i];
                 if ( !NEED_FLUSH(cpu_stamp, pfn_stamp) )
@@ -252,13 +251,12 @@ struct pfn_info *alloc_domain_page(struct task_struct *p)
 
         if ( unlikely(mask != 0) )
         {
-            if ( unlikely(in_irq()) )
-            {
-                DPRINTK("Returning NULL from alloc_domain_page: in_irq\n");
+            /* In IRQ ctxt, flushing is best-effort only, to avoid deadlock. */
+            if ( likely(!in_irq()) )
+                flush_tlb_mask(mask);
+            else if ( unlikely(!try_flush_tlb_mask(mask)) )
                 goto free_and_exit;
-            }
             perfc_incrc(need_flush_tlb_flush);
-            flush_tlb_mask(mask);
         }
     }
 
diff --git a/xen/include/asm-i386/flushtlb.h b/xen/include/asm-i386/flushtlb.h
index e6f61cb521..49760ef70c 100644
--- a/xen/include/asm-i386/flushtlb.h
+++ b/xen/include/asm-i386/flushtlb.h
@@ -24,8 +24,8 @@
  * used for a purpose that may have caused the CPU's TLB to become tainted.
  */
 #define NEED_FLUSH(_cpu_stamp, _lastuse_stamp) \
- (((_cpu_stamp) > (_lastuse_stamp)) ||         \
-  (((_lastuse_stamp) - (_cpu_stamp)) > (2*GLOBAL_FLUSH_PERIOD)))
+ (((_cpu_stamp) <= (_lastuse_stamp)) &&        \
+  (((_lastuse_stamp) - (_cpu_stamp)) <= (2*GLOBAL_FLUSH_PERIOD)))
 
 extern unsigned long tlbflush_mask;
 extern unsigned long tlbflush_clock;
diff --git a/xen/include/asm-i386/pgalloc.h b/xen/include/asm-i386/pgalloc.h
index 88e9064641..6f01b44441 100644
--- a/xen/include/asm-i386/pgalloc.h
+++ b/xen/include/asm-i386/pgalloc.h
@@ -47,17 +47,19 @@
 
 #ifndef CONFIG_SMP
 
-#define flush_tlb()           __flush_tlb()
-#define flush_tlb_all()       __flush_tlb()
-#define flush_tlb_all_pge()   __flush_tlb_pge()
-#define local_flush_tlb()     __flush_tlb()
-#define flush_tlb_cpu(_cpu)   __flush_tlb()
-#define flush_tlb_mask(_mask) __flush_tlb()
+#define flush_tlb()               __flush_tlb()
+#define flush_tlb_all()           __flush_tlb()
+#define flush_tlb_all_pge()       __flush_tlb_pge()
+#define local_flush_tlb()         __flush_tlb()
+#define flush_tlb_cpu(_cpu)       __flush_tlb()
+#define flush_tlb_mask(_mask)     __flush_tlb()
+#define try_flush_tlb_mask(_mask) __flush_tlb()
 
 #else
 
 #include <xeno/smp.h>
 
+extern int try_flush_tlb_mask(unsigned long mask);
 extern void flush_tlb_mask(unsigned long mask);
 extern void flush_tlb_all_pge(void);
 
diff --git a/xen/net/dev.c b/xen/net/dev.c
index a20a0ee6cd..936d40f04c 100644
--- a/xen/net/dev.c
+++ b/xen/net/dev.c
@@ -535,6 +535,7 @@ void deliver_packet(struct sk_buff *skb, net_vif_t *vif)
                           (pte & ~PAGE_MASK) | _PAGE_RW | _PAGE_PRESENT |
                           ((new_page - frame_table) << PAGE_SHIFT))) != pte )
     {
+        DPRINTK("PTE was modified or reused! %08lx %08lx\n", pte, *ptep);
         unmap_domain_mem(ptep);
         /* At some point maybe should have 'new_page' in error response. */
         put_page_and_type(new_page);
@@ -2112,6 +2113,9 @@ static void get_rx_bufs(net_vif_t *vif)
             goto rx_unmap_and_continue;
         }
             
+        buf_page->tlbflush_timestamp = tlbflush_clock;
+        buf_page->u.cpu_mask = 1 << p->processor;
+
         /* Remove from the domain's allocation list. */
         spin_lock(&p->page_list_lock);
         list_del(&buf_page->list);
@@ -2167,7 +2171,7 @@ long flush_bufs_for_vif(net_vif_t *vif)
     for ( i = vif->rx_req_cons; 
           (i != shared_idxs->rx_req_prod) &&
               ((i-vif->rx_resp_prod) != RX_RING_SIZE);
-          i++ );
+          i++ )
     {
         make_rx_response(vif, shared_rings->rx_ring[MASK_NET_RX_IDX(i)].req.id,
                          0, RING_STATUS_DROPPED, 0);
@@ -2179,6 +2183,7 @@ long flush_bufs_for_vif(net_vif_t *vif)
 
         /* Give the buffer page back to the domain. */
         page = &frame_table[rx->buf_pfn];
+        page->u.domain = p;
         spin_lock(&p->page_list_lock);
         list_add(&page->list, &p->page_list);
         page->count_and_flags = PGC_allocated | 2;
@@ -2194,7 +2199,10 @@ long flush_bufs_for_vif(net_vif_t *vif)
              unlikely(cmpxchg(ptep, pte, (rx->buf_pfn<<PAGE_SHIFT) | 
                               (pte & ~PAGE_MASK) | _PAGE_RW | _PAGE_PRESENT)
                       != pte) )
+        {
+            DPRINTK("PTE was modified or reused! %08lx %08lx\n", pte, *ptep);
             put_page_and_type(page);
+        }
         unmap_domain_mem(ptep);
 
         put_page_and_type(&frame_table[rx->pte_ptr >> PAGE_SHIFT]);
diff --git a/xenolinux-2.4.23-sparse/arch/xeno/drivers/network/network.c b/xenolinux-2.4.23-sparse/arch/xeno/drivers/network/network.c
index 492a2cc881..ac557a3c11 100644
--- a/xenolinux-2.4.23-sparse/arch/xeno/drivers/network/network.c
+++ b/xenolinux-2.4.23-sparse/arch/xeno/drivers/network/network.c
@@ -60,8 +60,8 @@ struct net_private
      * {tx,rx}_skbs store outstanding skbuffs. The first entry in each
      * array is an index into a chain of free entries.
      */
-    struct sk_buff *tx_skbs[TX_RING_SIZE];
-    struct sk_buff *rx_skbs[RX_RING_SIZE];
+    struct sk_buff *tx_skbs[TX_RING_SIZE+1];
+    struct sk_buff *rx_skbs[RX_RING_SIZE+1];
 };
 
 /* Access macros for acquiring freeing slots in {tx,rx}_skbs[]. */
@@ -145,9 +145,9 @@ static int network_open(struct net_device *dev)
     memset(np->net_idx, 0, sizeof(*np->net_idx));
 
     /* Initialise {tx,rx}_skbs to be a free chain containing every entry. */
-    for ( i = 0; i < TX_RING_SIZE; i++ )
+    for ( i = 0; i <= TX_RING_SIZE; i++ )
         np->tx_skbs[i] = (void *)(i+1);
-    for ( i = 0; i < RX_RING_SIZE; i++ )
+    for ( i = 0; i <= RX_RING_SIZE; i++ )
         np->rx_skbs[i] = (void *)(i+1);
 
     wmb();