diff options
Diffstat (limited to 'xen-2.4.16/net/skbuff.c')
| -rw-r--r-- | xen-2.4.16/net/skbuff.c | 1355 |
1 files changed, 280 insertions, 1075 deletions
diff --git a/xen-2.4.16/net/skbuff.c b/xen-2.4.16/net/skbuff.c index 0a4f2639e8..695a6f6b63 100644 --- a/xen-2.4.16/net/skbuff.c +++ b/xen-2.4.16/net/skbuff.c @@ -32,10 +32,6 @@ * 2 of the License, or (at your option) any later version. */ -/* - * The functions in this file will not compile correctly with gcc 2.4.x - */ - #include <linux/config.h> #include <linux/lib.h> #include <linux/errno.h> @@ -55,16 +51,13 @@ #define BUG_TRAP ASSERT -#define put_page(_p) ((void)0) /* XXXX KAF */ -#define get_page(_p) ((void)0) - int sysctl_hot_list_len = 128; static kmem_cache_t *skbuff_head_cache; static union { - struct sk_buff_head list; - char pad[SMP_CACHE_BYTES]; + struct sk_buff_head list; + char pad[SMP_CACHE_BYTES]; } skb_head_pool[NR_CPUS]; /* @@ -84,9 +77,9 @@ static union { void skb_over_panic(struct sk_buff *skb, int sz, void *here) { - printk("skput:over: %p:%d put:%d dev:%s", - here, skb->len, sz, skb->dev ? skb->dev->name : "<NULL>"); - BUG(); + printk("skput:over: %p:%d put:%d dev:%s", + here, skb->len, sz, skb->dev ? skb->dev->name : "<NULL>"); + BUG(); } /** @@ -101,148 +94,90 @@ void skb_over_panic(struct sk_buff *skb, int sz, void *here) void skb_under_panic(struct sk_buff *skb, int sz, void *here) { - printk("skput:under: %p:%d put:%d dev:%s", - here, skb->len, sz, skb->dev ? skb->dev->name : "<NULL>"); - BUG(); + printk("skput:under: %p:%d put:%d dev:%s", + here, skb->len, sz, skb->dev ? skb->dev->name : "<NULL>"); + BUG(); } static __inline__ struct sk_buff *skb_head_from_pool(void) { - struct sk_buff_head *list = &skb_head_pool[smp_processor_id()].list; - - if (skb_queue_len(list)) { - struct sk_buff *skb; - unsigned long flags; - - local_irq_save(flags); - skb = __skb_dequeue(list); - local_irq_restore(flags); - return skb; - } - return NULL; + struct sk_buff_head *list = &skb_head_pool[smp_processor_id()].list; + + if (skb_queue_len(list)) { + struct sk_buff *skb; + unsigned long flags; + + local_irq_save(flags); + skb = __skb_dequeue(list); + local_irq_restore(flags); + return skb; + } + return NULL; } static __inline__ void skb_head_to_pool(struct sk_buff *skb) { - struct sk_buff_head *list = &skb_head_pool[smp_processor_id()].list; + struct sk_buff_head *list = &skb_head_pool[smp_processor_id()].list; - if (skb_queue_len(list) < sysctl_hot_list_len) { - unsigned long flags; + if (skb_queue_len(list) < sysctl_hot_list_len) { + unsigned long flags; - local_irq_save(flags); - __skb_queue_head(list, skb); - local_irq_restore(flags); + local_irq_save(flags); + __skb_queue_head(list, skb); + local_irq_restore(flags); - return; - } - kmem_cache_free(skbuff_head_cache, skb); + return; + } + kmem_cache_free(skbuff_head_cache, skb); } static inline u8 *alloc_skb_data_page(struct sk_buff *skb) { - struct list_head *list_ptr; - struct pfn_info *pf; - unsigned long flags; + struct list_head *list_ptr; + struct pfn_info *pf; + unsigned long flags; - spin_lock_irqsave(&free_list_lock, flags); + spin_lock_irqsave(&free_list_lock, flags); - if (!free_pfns) return NULL; + if (!free_pfns) return NULL; - list_ptr = free_list.next; - pf = list_entry(list_ptr, struct pfn_info, list); - pf->flags = 0; /* owned by dom0 */ - list_del(&pf->list); - free_pfns--; + list_ptr = free_list.next; + pf = list_entry(list_ptr, struct pfn_info, list); + pf->flags = 0; /* owned by dom0 */ + list_del(&pf->list); + free_pfns--; - spin_unlock_irqrestore(&free_list_lock, flags); + spin_unlock_irqrestore(&free_list_lock, flags); - skb->pf = pf; - return (u8 *)((pf - frame_table) << PAGE_SHIFT); + skb->pf = pf; + return (u8 *)((pf - frame_table) << PAGE_SHIFT); } static inline void dealloc_skb_data_page(struct sk_buff *skb) { - struct pfn_info *pf; - unsigned long flags; + struct pfn_info *pf; + unsigned long flags; - pf = skb->pf; + pf = skb->pf; - spin_lock_irqsave(&free_list_lock, flags); + spin_lock_irqsave(&free_list_lock, flags); - list_add(&pf->list, &free_list); - free_pfns++; + list_add(&pf->list, &free_list); + free_pfns++; - spin_unlock_irqrestore(&free_list_lock, flags); + spin_unlock_irqrestore(&free_list_lock, flags); } -struct sk_buff *alloc_zc_skb(unsigned int size,int gfp_mask) +static inline void INTERRUPT_CHECK(int gfp_mask) { - struct sk_buff *skb; - u8 *data; - - if (in_interrupt() && (gfp_mask & __GFP_WAIT)) { - static int count = 0; - if (++count < 5) { - printk(KERN_ERR "alloc_skb called nonatomically " - "from interrupt %p\n", NET_CALLER(size)); - BUG(); - } - gfp_mask &= ~__GFP_WAIT; - } - - /* Get the HEAD */ - skb = skb_head_from_pool(); - if (skb == NULL) { - skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA); - if (skb == NULL) - goto nohead; - } - - /* Get the DATA. Size must match skb_add_mtu(). */ - size = SKB_DATA_ALIGN(size); - data = alloc_skb_data_page(skb); - - if (data == NULL) - goto nodata; - - /* A FAKE virtual address, so that pci_map_xxx dor the right thing. */ - data = phys_to_virt((unsigned long)data); - - /* Load the data pointers. */ - skb->head = data; - skb->data = data; - skb->tail = data; - skb->end = data + size; - - /* Set up other state */ - skb->len = 0; - skb->cloned = 0; - skb->data_len = 0; - skb->src_vif = VIF_UNKNOWN_INTERFACE; - skb->dst_vif = VIF_UNKNOWN_INTERFACE; - skb->skb_type = SKB_ZERO_COPY; - - atomic_set(&skb->users, 1); - atomic_set(&(skb_shinfo(skb)->dataref), 1); - skb_shinfo(skb)->nr_frags = 0; - skb_shinfo(skb)->frag_list = NULL; - - return skb; - -nodata: - skb_head_to_pool(skb); -nohead: - return NULL; + if (in_interrupt() && (gfp_mask & __GFP_WAIT)) { + printk(KERN_ERR "alloc_skb called nonatomically\n"); + BUG(); + } } -/* Allocate a new skbuff. We do this ourselves so we can fill in a few - * 'private' fields and also do memory statistics to find all the - * [BEEP] leaks. - * - */ - /** * alloc_skb - allocate a network buffer * @size: size to allocate @@ -258,136 +193,150 @@ nohead: struct sk_buff *alloc_skb(unsigned int size,int gfp_mask) { - struct sk_buff *skb; - u8 *data; - - if (in_interrupt() && (gfp_mask & __GFP_WAIT)) { - static int count = 0; - if (++count < 5) { - printk(KERN_ERR "alloc_skb called nonatomically " - "from interrupt %p\n", NET_CALLER(size)); - BUG(); - } - gfp_mask &= ~__GFP_WAIT; - } - - /* Get the HEAD */ - skb = skb_head_from_pool(); - if (skb == NULL) { - skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA); - if (skb == NULL) - goto nohead; - } - - /* Get the DATA. Size must match skb_add_mtu(). */ - size = SKB_DATA_ALIGN(size); - data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask); - if (data == NULL) - goto nodata; - - /* Load the data pointers. */ - skb->head = data; - skb->data = data; - skb->tail = data; - skb->end = data + size; - - /* Set up other state */ - skb->len = 0; - skb->cloned = 0; - skb->data_len = 0; - skb->src_vif = VIF_UNKNOWN_INTERFACE; - skb->dst_vif = VIF_UNKNOWN_INTERFACE; - skb->skb_type = SKB_NORMAL; - - atomic_set(&skb->users, 1); - atomic_set(&(skb_shinfo(skb)->dataref), 1); - skb_shinfo(skb)->nr_frags = 0; - skb_shinfo(skb)->frag_list = NULL; - return skb; - -nodata: - skb_head_to_pool(skb); -nohead: - return NULL; + struct sk_buff *skb; + u8 *data; + + INTERRUPT_CHECK(gfp_mask); + + /* Get the HEAD */ + skb = skb_head_from_pool(); + if (skb == NULL) { + skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA); + if (skb == NULL) + goto nohead; + } + + /* Get the DATA. Size must match skb_add_mtu(). */ + size = SKB_DATA_ALIGN(size); + data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask); + if (data == NULL) + goto nodata; + + /* Load the data pointers. */ + skb->head = data; + skb->data = data; + skb->tail = data; + skb->end = data + size; + + /* Set up other state */ + skb->len = 0; + skb->data_len = 0; + skb->src_vif = VIF_UNKNOWN_INTERFACE; + skb->dst_vif = VIF_UNKNOWN_INTERFACE; + skb->skb_type = SKB_NORMAL; + + skb_shinfo(skb)->nr_frags = 0; + return skb; + + nodata: + skb_head_to_pool(skb); + nohead: + return NULL; } -/* - * Slab constructor for a skb head. - */ -static inline void skb_headerinit(void *p, kmem_cache_t *cache, - unsigned long flags) +struct sk_buff *alloc_zc_skb(unsigned int size,int gfp_mask) { - struct sk_buff *skb = p; - - skb->next = NULL; - skb->prev = NULL; - skb->list = NULL; - skb->dev = NULL; - skb->pkt_type = PACKET_HOST; /* Default type */ - skb->ip_summed = 0; - skb->destructor = NULL; - -#ifdef CONFIG_NETFILTER - skb->nfmark = skb->nfcache = 0; - skb->nfct = NULL; -#ifdef CONFIG_NETFILTER_DEBUG - skb->nf_debug = 0; -#endif -#endif -#ifdef CONFIG_NET_SCHED - skb->tc_index = 0; -#endif + struct sk_buff *skb; + u8 *data; + + INTERRUPT_CHECK(gfp_mask); + + /* Get the HEAD */ + skb = skb_head_from_pool(); + if (skb == NULL) { + skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA); + if (skb == NULL) + goto nohead; + } + + /* Get the DATA. Size must match skb_add_mtu(). */ + size = SKB_DATA_ALIGN(size); + data = alloc_skb_data_page(skb); + + if (data == NULL) + goto nodata; + + /* A FAKE virtual address, so that pci_map_xxx dor the right thing. */ + data = phys_to_virt((unsigned long)data); + + /* Load the data pointers. */ + skb->head = data; + skb->data = data; + skb->tail = data; + skb->end = data + size; + + /* Set up other state */ + skb->len = 0; + skb->data_len = 0; + skb->src_vif = VIF_UNKNOWN_INTERFACE; + skb->dst_vif = VIF_UNKNOWN_INTERFACE; + skb->skb_type = SKB_ZERO_COPY; + + skb_shinfo(skb)->nr_frags = 0; + + return skb; + + nodata: + skb_head_to_pool(skb); + nohead: + return NULL; } -static void skb_drop_fraglist(struct sk_buff *skb) + +struct sk_buff *alloc_skb_nodata(int gfp_mask) { - struct sk_buff *list = skb_shinfo(skb)->frag_list; + struct sk_buff *skb; - skb_shinfo(skb)->frag_list = NULL; + INTERRUPT_CHECK(gfp_mask); - do { - struct sk_buff *this = list; - list = list->next; - kfree_skb(this); - } while (list); + /* Get the HEAD */ + skb = skb_head_from_pool(); + if (skb == NULL) { + skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA); + if (skb == NULL) + return NULL; + } + + skb->skb_type = SKB_NODATA; + return skb; } -static void skb_clone_fraglist(struct sk_buff *skb) -{ - struct sk_buff *list; - for (list = skb_shinfo(skb)->frag_list; list; list=list->next) - skb_get(list); +/* + * Slab constructor for a skb head. + */ +static inline void skb_headerinit(void *p, kmem_cache_t *cache, + unsigned long flags) +{ + struct sk_buff *skb = p; + + skb->next = NULL; + skb->prev = NULL; + skb->list = NULL; + skb->dev = NULL; + skb->pkt_type = PACKET_HOST; /* Default type */ + skb->ip_summed = 0; + skb->destructor = NULL; } static void skb_release_data(struct sk_buff *skb) { - - if (!skb->cloned || - atomic_dec_and_test(&(skb_shinfo(skb)->dataref))) { - if (skb_shinfo(skb)->nr_frags) { - int i; - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) - put_page(skb_shinfo(skb)->frags[i].page); - } - - if (skb_shinfo(skb)->frag_list) - skb_drop_fraglist(skb); - - if (skb->skb_type == SKB_NORMAL) - { - kfree(skb->head); - } - else if (skb->skb_type == SKB_ZERO_COPY) - { - dealloc_skb_data_page(skb); - } - else - { - BUG(); - } - } + if (skb_shinfo(skb)->nr_frags) BUG(); + + switch ( skb->skb_type ) + { + case SKB_NORMAL: + kfree(skb->head); + break; + case SKB_ZERO_COPY: + dealloc_skb_data_page(skb); + break; + case SKB_NODATA: + break; + default: + BUG(); + } } /* @@ -395,8 +344,8 @@ static void skb_release_data(struct sk_buff *skb) */ void kfree_skbmem(struct sk_buff *skb) { - skb_release_data(skb); - skb_head_to_pool(skb); + skb_release_data(skb); + skb_head_to_pool(skb); } /** @@ -410,124 +359,32 @@ void kfree_skbmem(struct sk_buff *skb) void __kfree_skb(struct sk_buff *skb) { - if (skb->list) { - printk(KERN_WARNING "Warning: kfree_skb passed an skb still " - "on a list (from %p).\n", NET_CALLER(skb)); - BUG(); - } - - if(skb->destructor) { - if (in_irq()) { - printk(KERN_WARNING "Warning: kfree_skb on hard IRQ %p\n", - NET_CALLER(skb)); - } - skb->destructor(skb); - } - -#ifdef CONFIG_NETFILTER - nf_conntrack_put(skb->nfct); -#endif - skb_headerinit(skb, NULL, 0); /* clean state */ - kfree_skbmem(skb); -} + if ( skb->list ) + panic(KERN_WARNING "Warning: kfree_skb passed an skb still " + "on a list (from %p).\n", NET_CALLER(skb)); -/** - * skb_clone - duplicate an sk_buff - * @skb: buffer to clone - * @gfp_mask: allocation priority - * - * Duplicate an &sk_buff. The new one is not owned by a socket. Both - * copies share the same packet data but not structure. The new - * buffer has a reference count of 1. If the allocation fails the - * function returns %NULL otherwise the new buffer is returned. - * - * If this function is called from an interrupt gfp_mask() must be - * %GFP_ATOMIC. - */ + if ( skb->destructor ) + skb->destructor(skb); -struct sk_buff *skb_clone(struct sk_buff *skb, int gfp_mask) -{ - struct sk_buff *n; - - n = skb_head_from_pool(); - if (!n) { - n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); - if (!n) - return NULL; - } - -#define C(x) n->x = skb->x - - n->next = n->prev = NULL; - n->list = NULL; - C(dev); - C(h); - C(nh); - C(mac); - C(len); - C(data_len); - C(csum); - n->cloned = 1; - C(pkt_type); - C(ip_summed); - atomic_set(&n->users, 1); - C(protocol); - C(head); - C(data); - C(tail); - C(end); - n->destructor = NULL; -#ifdef CONFIG_NETFILTER - C(nfmark); - C(nfcache); - C(nfct); -#ifdef CONFIG_NETFILTER_DEBUG - C(nf_debug); -#endif -#endif /*CONFIG_NETFILTER*/ -#if defined(CONFIG_HIPPI) - C(private); -#endif -#ifdef CONFIG_NET_SCHED - C(tc_index); -#endif - - atomic_inc(&(skb_shinfo(skb)->dataref)); - skb->cloned = 1; -#ifdef CONFIG_NETFILTER - nf_conntrack_get(skb->nfct); -#endif - return n; + skb_headerinit(skb, NULL, 0); /* clean state */ + kfree_skbmem(skb); } static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old) { - /* - * Shift between the two data areas in bytes - */ - unsigned long offset = new->data - old->data; - - new->list=NULL; - new->dev=old->dev; - new->protocol=old->protocol; - new->h.raw=old->h.raw+offset; - new->nh.raw=old->nh.raw+offset; - new->mac.raw=old->mac.raw+offset; - atomic_set(&new->users, 1); - new->pkt_type=old->pkt_type; - new->destructor = NULL; -#ifdef CONFIG_NETFILTER - new->nfmark=old->nfmark; - new->nfcache=old->nfcache; - new->nfct=old->nfct; - nf_conntrack_get(new->nfct); -#ifdef CONFIG_NETFILTER_DEBUG - new->nf_debug=old->nf_debug; -#endif -#endif -#ifdef CONFIG_NET_SCHED - new->tc_index = old->tc_index; -#endif + /* + * Shift between the two data areas in bytes + */ + unsigned long offset = new->data - old->data; + + new->list=NULL; + new->dev=old->dev; + new->protocol=old->protocol; + new->h.raw=old->h.raw+offset; + new->nh.raw=old->nh.raw+offset; + new->mac.raw=old->mac.raw+offset; + new->pkt_type=old->pkt_type; + new->destructor = NULL; } /** @@ -549,748 +406,96 @@ static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old) struct sk_buff *skb_copy(const struct sk_buff *skb, int gfp_mask) { - struct sk_buff *n; - int headerlen = skb->data-skb->head; - - /* - * Allocate the copy buffer - */ - n=alloc_skb(skb->end - skb->head + skb->data_len, gfp_mask); - if(n==NULL) - return NULL; - - /* Set the data pointer */ - skb_reserve(n,headerlen); - /* Set the tail pointer and length */ - skb_put(n,skb->len); - n->csum = skb->csum; - n->ip_summed = skb->ip_summed; - - if (skb_copy_bits(skb, -headerlen, n->head, headerlen+skb->len)) - BUG(); - - copy_skb_header(n, skb); - - return n; -} - -/* Keep head the same: replace data */ -int skb_linearize(struct sk_buff *skb, int gfp_mask) -{ - unsigned int size; - u8 *data; - long offset; - int headerlen = skb->data - skb->head; - int expand = (skb->tail+skb->data_len) - skb->end; - - if (skb_shared(skb)) - BUG(); - - if (expand <= 0) - expand = 0; - - size = (skb->end - skb->head + expand); - size = SKB_DATA_ALIGN(size); - data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask); - if (data == NULL) - return -ENOMEM; - - /* Copy entire thing */ - if (skb_copy_bits(skb, -headerlen, data, headerlen+skb->len)) - BUG(); - - /* Offset between the two in bytes */ - offset = data - skb->head; - - /* Free old data. */ - skb_release_data(skb); - - skb->head = data; - skb->end = data + size; - - /* Set up new pointers */ - skb->h.raw += offset; - skb->nh.raw += offset; - skb->mac.raw += offset; - skb->tail += offset; - skb->data += offset; - - /* Set up shinfo */ - atomic_set(&(skb_shinfo(skb)->dataref), 1); - skb_shinfo(skb)->nr_frags = 0; - skb_shinfo(skb)->frag_list = NULL; - - /* We are no longer a clone, even if we were. */ - skb->cloned = 0; - - skb->tail += skb->data_len; - skb->data_len = 0; - return 0; -} - - -/** - * pskb_copy - create copy of an sk_buff with private head. - * @skb: buffer to copy - * @gfp_mask: allocation priority - * - * Make a copy of both an &sk_buff and part of its data, located - * in header. Fragmented data remain shared. This is used when - * the caller wishes to modify only header of &sk_buff and needs - * private copy of the header to alter. Returns %NULL on failure - * or the pointer to the buffer on success. - * The returned buffer has a reference count of 1. - */ - -struct sk_buff *pskb_copy(struct sk_buff *skb, int gfp_mask) -{ - struct sk_buff *n; - - /* - * Allocate the copy buffer - */ - n=alloc_skb(skb->end - skb->head, gfp_mask); - if(n==NULL) - return NULL; - - /* Set the data pointer */ - skb_reserve(n,skb->data-skb->head); - /* Set the tail pointer and length */ - skb_put(n,skb_headlen(skb)); - /* Copy the bytes */ - memcpy(n->data, skb->data, n->len); - n->csum = skb->csum; - n->ip_summed = skb->ip_summed; - - n->data_len = skb->data_len; - n->len = skb->len; - - if (skb_shinfo(skb)->nr_frags) { - int i; - - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { - skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; - get_page(skb_shinfo(n)->frags[i].page); - } - skb_shinfo(n)->nr_frags = i; - } - - if (skb_shinfo(skb)->frag_list) { - skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; - skb_clone_fraglist(n); - } - - copy_skb_header(n, skb); - - return n; -} - -/** - * pskb_expand_head - reallocate header of &sk_buff - * @skb: buffer to reallocate - * @nhead: room to add at head - * @ntail: room to add at tail - * @gfp_mask: allocation priority - * - * Expands (or creates identical copy, if &nhead and &ntail are zero) - * header of skb. &sk_buff itself is not changed. &sk_buff MUST have - * reference count of 1. Returns zero in the case of success or error, - * if expansion failed. In the last case, &sk_buff is not changed. - * - * All the pointers pointing into skb header may change and must be - * reloaded after call to this function. - */ - -int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, int gfp_mask) -{ - int i; - u8 *data; - int size = nhead + (skb->end - skb->head) + ntail; - long off; - - if (skb_shared(skb)) - BUG(); - - size = SKB_DATA_ALIGN(size); - - data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask); - if (data == NULL) - goto nodata; - - /* Copy only real data... and, alas, header. This should be - * optimized for the cases when header is void. */ - memcpy(data+nhead, skb->head, skb->tail-skb->head); - memcpy(data+size, skb->end, sizeof(struct skb_shared_info)); - - for (i=0; i<skb_shinfo(skb)->nr_frags; i++) - get_page(skb_shinfo(skb)->frags[i].page); - - if (skb_shinfo(skb)->frag_list) - skb_clone_fraglist(skb); - - skb_release_data(skb); - - off = (data+nhead) - skb->head; - - skb->head = data; - skb->end = data+size; - - skb->data += off; - skb->tail += off; - skb->mac.raw += off; - skb->h.raw += off; - skb->nh.raw += off; - skb->cloned = 0; - atomic_set(&skb_shinfo(skb)->dataref, 1); - return 0; - -nodata: - return -ENOMEM; -} - -/* Make private copy of skb with writable head and some headroom */ - -struct sk_buff * -skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) -{ - struct sk_buff *skb2; - int delta = headroom - skb_headroom(skb); - - if (delta <= 0) - return pskb_copy(skb, GFP_ATOMIC); - - skb2 = skb_clone(skb, GFP_ATOMIC); - if (skb2 == NULL || - !pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, GFP_ATOMIC)) - return skb2; - - kfree_skb(skb2); - return NULL; -} - - -/** - * skb_copy_expand - copy and expand sk_buff - * @skb: buffer to copy - * @newheadroom: new free bytes at head - * @newtailroom: new free bytes at tail - * @gfp_mask: allocation priority - * - * Make a copy of both an &sk_buff and its data and while doing so - * allocate additional space. - * - * This is used when the caller wishes to modify the data and needs a - * private copy of the data to alter as well as more space for new fields. - * Returns %NULL on failure or the pointer to the buffer - * on success. The returned buffer has a reference count of 1. - * - * You must pass %GFP_ATOMIC as the allocation priority if this function - * is called from an interrupt. - */ - - -struct sk_buff *skb_copy_expand(const struct sk_buff *skb, - int newheadroom, - int newtailroom, - int gfp_mask) -{ - struct sk_buff *n; - - /* - * Allocate the copy buffer - */ - - n=alloc_skb(newheadroom + skb->len + newtailroom, - gfp_mask); - if(n==NULL) - return NULL; - - skb_reserve(n,newheadroom); - - /* Set the tail pointer and length */ - skb_put(n,skb->len); - - /* Copy the data only. */ - if (skb_copy_bits(skb, 0, n->data, skb->len)) - BUG(); - - copy_skb_header(n, skb); - return n; -} + struct sk_buff *n; + int headerlen = skb->data-skb->head; + + /* + * Allocate the copy buffer + */ + n=alloc_skb(skb->end - skb->head + skb->data_len, gfp_mask); + if(n==NULL) + return NULL; -/* Trims skb to length len. It can change skb pointers, if "realloc" is 1. - * If realloc==0 and trimming is impossible without change of data, - * it is BUG(). - */ + /* Set the data pointer */ + skb_reserve(n,headerlen); + /* Set the tail pointer and length */ + skb_put(n,skb->len); + n->csum = skb->csum; + n->ip_summed = skb->ip_summed; -int ___pskb_trim(struct sk_buff *skb, unsigned int len, int realloc) -{ - int offset = skb_headlen(skb); - int nfrags = skb_shinfo(skb)->nr_frags; - int i; - - for (i=0; i<nfrags; i++) { - int end = offset + skb_shinfo(skb)->frags[i].size; - if (end > len) { - if (skb_cloned(skb)) { - if (!realloc) - BUG(); - if (!pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) - return -ENOMEM; - } - if (len <= offset) { - put_page(skb_shinfo(skb)->frags[i].page); - skb_shinfo(skb)->nr_frags--; - } else { - skb_shinfo(skb)->frags[i].size = len-offset; - } - } - offset = end; - } - - if (offset < len) { - skb->data_len -= skb->len - len; - skb->len = len; - } else { - if (len <= skb_headlen(skb)) { - skb->len = len; - skb->data_len = 0; - skb->tail = skb->data + len; - if (skb_shinfo(skb)->frag_list && !skb_cloned(skb)) - skb_drop_fraglist(skb); - } else { - skb->data_len -= skb->len - len; - skb->len = len; - } - } - - return 0; -} + if (skb_copy_bits(skb, -headerlen, n->head, headerlen+skb->len)) + BUG(); -/** - * __pskb_pull_tail - advance tail of skb header - * @skb: buffer to reallocate - * @delta: number of bytes to advance tail - * - * The function makes a sense only on a fragmented &sk_buff, - * it expands header moving its tail forward and copying necessary - * data from fragmented part. - * - * &sk_buff MUST have reference count of 1. - * - * Returns %NULL (and &sk_buff does not change) if pull failed - * or value of new tail of skb in the case of success. - * - * All the pointers pointing into skb header may change and must be - * reloaded after call to this function. - */ + copy_skb_header(n, skb); -/* Moves tail of skb head forward, copying data from fragmented part, - * when it is necessary. - * 1. It may fail due to malloc failure. - * 2. It may change skb pointers. - * - * It is pretty complicated. Luckily, it is called only in exceptional cases. - */ -unsigned char * __pskb_pull_tail(struct sk_buff *skb, int delta) -{ - int i, k, eat; - - /* If skb has not enough free space at tail, get new one - * plus 128 bytes for future expansions. If we have enough - * room at tail, reallocate without expansion only if skb is cloned. - */ - eat = (skb->tail+delta) - skb->end; - - if (eat > 0 || skb_cloned(skb)) { - if (pskb_expand_head(skb, 0, eat>0 ? eat+128 : 0, GFP_ATOMIC)) - return NULL; - } - - if (skb_copy_bits(skb, skb_headlen(skb), skb->tail, delta)) - BUG(); - - /* Optimization: no fragments, no reasons to preestimate - * size of pulled pages. Superb. - */ - if (skb_shinfo(skb)->frag_list == NULL) - goto pull_pages; - - /* Estimate size of pulled pages. */ - eat = delta; - for (i=0; i<skb_shinfo(skb)->nr_frags; i++) { - if (skb_shinfo(skb)->frags[i].size >= eat) - goto pull_pages; - eat -= skb_shinfo(skb)->frags[i].size; - } - - /* If we need update frag list, we are in troubles. - * Certainly, it possible to add an offset to skb data, - * but taking into account that pulling is expected to - * be very rare operation, it is worth to fight against - * further bloating skb head and crucify ourselves here instead. - * Pure masohism, indeed. 8)8) - */ - if (eat) { - struct sk_buff *list = skb_shinfo(skb)->frag_list; - struct sk_buff *clone = NULL; - struct sk_buff *insp = NULL; - - do { - if (list == NULL) - BUG(); - - if (list->len <= eat) { - /* Eaten as whole. */ - eat -= list->len; - list = list->next; - insp = list; - } else { - /* Eaten partially. */ - - if (skb_shared(list)) { - /* Sucks! We need to fork list. :-( */ - clone = skb_clone(list, GFP_ATOMIC); - if (clone == NULL) - return NULL; - insp = list->next; - list = clone; - } else { - /* This may be pulled without - * problems. */ - insp = list; - } - if (pskb_pull(list, eat) == NULL) { - if (clone) - kfree_skb(clone); - return NULL; - } - break; - } - } while (eat); - - /* Free pulled out fragments. */ - while ((list = skb_shinfo(skb)->frag_list) != insp) { - skb_shinfo(skb)->frag_list = list->next; - kfree_skb(list); - } - /* And insert new clone at head. */ - if (clone) { - clone->next = list; - skb_shinfo(skb)->frag_list = clone; - } - } - /* Success! Now we may commit changes to skb data. */ - -pull_pages: - eat = delta; - k = 0; - for (i=0; i<skb_shinfo(skb)->nr_frags; i++) { - if (skb_shinfo(skb)->frags[i].size <= eat) { - put_page(skb_shinfo(skb)->frags[i].page); - eat -= skb_shinfo(skb)->frags[i].size; - } else { - skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; - if (eat) { - skb_shinfo(skb)->frags[k].page_offset += eat; - skb_shinfo(skb)->frags[k].size -= eat; - eat = 0; - } - k++; - } - } - skb_shinfo(skb)->nr_frags = k; - - skb->tail += delta; - skb->data_len -= delta; - - return skb->tail; + return n; } /* Copy some data bits from skb to kernel buffer. */ int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) { - int i, copy; - int start = skb->len - skb->data_len; - - if (offset > (int)skb->len-len) - goto fault; - - /* Copy header. */ - if ((copy = start-offset) > 0) { - if (copy > len) - copy = len; - memcpy(to, skb->data + offset, copy); - if ((len -= copy) == 0) - return 0; - offset += copy; - to += copy; - } - - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { - int end; - - BUG_TRAP(start <= offset+len); - - end = start + skb_shinfo(skb)->frags[i].size; - if ((copy = end-offset) > 0) { - u8 *vaddr; - - if (copy > len) - copy = len; - - vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]); - memcpy(to, vaddr+skb_shinfo(skb)->frags[i].page_offset+ - offset-start, copy); - kunmap_skb_frag(vaddr); - - if ((len -= copy) == 0) - return 0; - offset += copy; - to += copy; - } - start = end; - } - - if (skb_shinfo(skb)->frag_list) { - struct sk_buff *list; - - for (list = skb_shinfo(skb)->frag_list; list; list=list->next) { - int end; - - BUG_TRAP(start <= offset+len); - - end = start + list->len; - if ((copy = end-offset) > 0) { - if (copy > len) - copy = len; - if (skb_copy_bits(list, offset-start, to, copy)) - goto fault; - if ((len -= copy) == 0) - return 0; - offset += copy; - to += copy; - } - start = end; - } - } - if (len == 0) - return 0; - -fault: - return -EFAULT; -} - -/* Checksum skb data. */ - -#if 0 - -unsigned int skb_checksum(const struct sk_buff *skb, int offset, int len, unsigned int csum) -{ - int i, copy; - int start = skb->len - skb->data_len; - int pos = 0; - - /* Checksum header. */ - if ((copy = start-offset) > 0) { - if (copy > len) - copy = len; - csum = csum_partial(skb->data+offset, copy, csum); - if ((len -= copy) == 0) - return csum; - offset += copy; - pos = copy; - } - - for (i=0; i<skb_shinfo(skb)->nr_frags; i++) { - int end; - - BUG_TRAP(start <= offset+len); - - end = start + skb_shinfo(skb)->frags[i].size; - if ((copy = end-offset) > 0) { - unsigned int csum2; - u8 *vaddr; - skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; - - if (copy > len) - copy = len; - vaddr = kmap_skb_frag(frag); - csum2 = csum_partial(vaddr + frag->page_offset + - offset-start, copy, 0); - kunmap_skb_frag(vaddr); - csum = csum_block_add(csum, csum2, pos); - if (!(len -= copy)) - return csum; - offset += copy; - pos += copy; - } - start = end; - } - - if (skb_shinfo(skb)->frag_list) { - struct sk_buff *list; - - for (list = skb_shinfo(skb)->frag_list; list; list=list->next) { - int end; - - BUG_TRAP(start <= offset+len); - - end = start + list->len; - if ((copy = end-offset) > 0) { - unsigned int csum2; - if (copy > len) - copy = len; - csum2 = skb_checksum(list, offset-start, copy, 0); - csum = csum_block_add(csum, csum2, pos); - if ((len -= copy) == 0) - return csum; - offset += copy; - pos += copy; - } - start = end; - } - } - if (len == 0) - return csum; - - BUG(); - return csum; -} - -/* Both of above in one bottle. */ - -unsigned int skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to, int len, unsigned int csum) -{ - int i, copy; - int start = skb->len - skb->data_len; - int pos = 0; - - /* Copy header. */ - if ((copy = start-offset) > 0) { - if (copy > len) - copy = len; - csum = csum_partial_copy_nocheck(skb->data+offset, to, copy, csum); - if ((len -= copy) == 0) - return csum; - offset += copy; - to += copy; - pos = copy; - } - - for (i=0; i<skb_shinfo(skb)->nr_frags; i++) { - int end; - - BUG_TRAP(start <= offset+len); - - end = start + skb_shinfo(skb)->frags[i].size; - if ((copy = end-offset) > 0) { - unsigned int csum2; - u8 *vaddr; - skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; - - if (copy > len) - copy = len; - vaddr = kmap_skb_frag(frag); - csum2 = csum_partial_copy_nocheck(vaddr + frag->page_offset + - offset-start, to, copy, 0); - kunmap_skb_frag(vaddr); - csum = csum_block_add(csum, csum2, pos); - if (!(len -= copy)) - return csum; - offset += copy; - to += copy; - pos += copy; - } - start = end; - } - - if (skb_shinfo(skb)->frag_list) { - struct sk_buff *list; - - for (list = skb_shinfo(skb)->frag_list; list; list=list->next) { - unsigned int csum2; - int end; - - BUG_TRAP(start <= offset+len); - - end = start + list->len; - if ((copy = end-offset) > 0) { - if (copy > len) - copy = len; - csum2 = skb_copy_and_csum_bits(list, offset-start, to, copy, 0); - csum = csum_block_add(csum, csum2, pos); - if ((len -= copy) == 0) - return csum; - offset += copy; - to += copy; - pos += copy; - } - start = end; - } - } - if (len == 0) - return csum; - - BUG(); - return csum; -} - -void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) -{ - unsigned int csum; - long csstart; - - if (skb->ip_summed == CHECKSUM_HW) - csstart = skb->h.raw - skb->data; - else - csstart = skb->len - skb->data_len; - - if (csstart > skb->len - skb->data_len) - BUG(); - - memcpy(to, skb->data, csstart); - - csum = 0; - if (csstart != skb->len) - csum = skb_copy_and_csum_bits(skb, csstart, to+csstart, - skb->len-csstart, 0); - - if (skb->ip_summed == CHECKSUM_HW) { - long csstuff = csstart + skb->csum; - - *((unsigned short *)(to + csstuff)) = csum_fold(csum); - } -} - -#endif + int i, copy; + int start = skb->len - skb->data_len; + + if (offset > (int)skb->len-len) + goto fault; + + /* Copy header. */ + if ((copy = start-offset) > 0) { + if (copy > len) + copy = len; + memcpy(to, skb->data + offset, copy); + if ((len -= copy) == 0) + return 0; + offset += copy; + to += copy; + } + + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + int end; + + BUG_TRAP(start <= offset+len); + + end = start + skb_shinfo(skb)->frags[i].size; + if ((copy = end-offset) > 0) { + u8 *vaddr; + + if (copy > len) + copy = len; + + vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]); + memcpy(to, vaddr+skb_shinfo(skb)->frags[i].page_offset+ + offset-start, copy); + kunmap_skb_frag(vaddr); + + if ((len -= copy) == 0) + return 0; + offset += copy; + to += copy; + } + start = end; + } -#if 0 -/* - * Tune the memory allocator for a new MTU size. - */ -void skb_add_mtu(int mtu) -{ - /* Must match allocation in alloc_skb */ - mtu = SKB_DATA_ALIGN(mtu) + sizeof(struct skb_shared_info); + if (len == 0) + return 0; - kmem_add_cache_size(mtu); + fault: + return -EFAULT; } -#endif void __init skb_init(void) { - int i; - - skbuff_head_cache = kmem_cache_create("skbuff_head_cache", - sizeof(struct sk_buff), - 0, - SLAB_HWCACHE_ALIGN, - skb_headerinit, NULL); - if (!skbuff_head_cache) - panic("cannot create skbuff cache"); - - for (i=0; i<NR_CPUS; i++) - skb_queue_head_init(&skb_head_pool[i].list); + int i; + + skbuff_head_cache = kmem_cache_create("skbuff_head_cache", + sizeof(struct sk_buff), + 0, + SLAB_HWCACHE_ALIGN, + skb_headerinit, NULL); + if (!skbuff_head_cache) + panic("cannot create skbuff cache"); + + for (i=0; i<NR_CPUS; i++) + skb_queue_head_init(&skb_head_pool[i].list); } |