diff options
Diffstat (limited to 'include/linux/skbuff.h')
| -rw-r--r-- | include/linux/skbuff.h | 220 |
1 files changed, 168 insertions, 52 deletions
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 111f26b6e28b..7632c87da2c9 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -117,11 +117,11 @@ struct nf_conntrack { #ifdef CONFIG_BRIDGE_NETFILTER struct nf_bridge_info { - atomic_t use; - struct net_device *physindev; - struct net_device *physoutdev; - unsigned int mask; - unsigned long data[32 / sizeof(unsigned long)]; + atomic_t use; + unsigned int mask; + struct net_device *physindev; + struct net_device *physoutdev; + unsigned long data[32 / sizeof(unsigned long)]; }; #endif @@ -225,14 +225,11 @@ enum { /* device driver is going to provide hardware time stamp */ SKBTX_IN_PROGRESS = 1 << 2, - /* ensure the originating sk reference is available on driver level */ - SKBTX_DRV_NEEDS_SK_REF = 1 << 3, - /* device driver supports TX zero-copy buffers */ - SKBTX_DEV_ZEROCOPY = 1 << 4, + SKBTX_DEV_ZEROCOPY = 1 << 3, /* generate wifi status information (where possible) */ - SKBTX_WIFI_STATUS = 1 << 5, + SKBTX_WIFI_STATUS = 1 << 4, }; /* @@ -465,12 +462,14 @@ struct sk_buff { #ifdef CONFIG_IPV6_NDISC_NODETYPE __u8 ndisc_nodetype:2; #endif + __u8 pfmemalloc:1; __u8 ooo_okay:1; __u8 l4_rxhash:1; __u8 wifi_acked_valid:1; __u8 wifi_acked:1; __u8 no_fcs:1; - /* 9/11 bit hole (depending on ndisc_nodetype presence) */ + __u8 head_frag:1; + /* 8/10 bit hole (depending on ndisc_nodetype presence) */ kmemcheck_bitfield_end(flags2); #ifdef CONFIG_NET_DMA @@ -504,6 +503,15 @@ struct sk_buff { #include <linux/slab.h> +#define SKB_ALLOC_FCLONE 0x01 +#define SKB_ALLOC_RX 0x02 + +/* Returns true if the skb was allocated from PFMEMALLOC reserves */ +static inline bool skb_pfmemalloc(const struct sk_buff *skb) +{ + return unlikely(skb->pfmemalloc); +} + /* * skb might have a dst pointer attached, refcounted or not. * _skb_refdst low order bit is set if refcount was _not_ taken @@ -560,9 +568,15 @@ static inline struct rtable *skb_rtable(const struct sk_buff *skb) extern void kfree_skb(struct sk_buff *skb); extern void consume_skb(struct sk_buff *skb); extern void __kfree_skb(struct sk_buff *skb); +extern struct kmem_cache *skbuff_head_cache; + +extern void kfree_skb_partial(struct sk_buff *skb, bool head_stolen); +extern bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, + bool *fragstolen, int *delta_truesize); + extern struct sk_buff *__alloc_skb(unsigned int size, - gfp_t priority, int fclone, int node); -extern struct sk_buff *build_skb(void *data); + gfp_t priority, int flags, int node); +extern struct sk_buff *build_skb(void *data, unsigned int frag_size); static inline struct sk_buff *alloc_skb(unsigned int size, gfp_t priority) { @@ -572,7 +586,7 @@ static inline struct sk_buff *alloc_skb(unsigned int size, static inline struct sk_buff *alloc_skb_fclone(unsigned int size, gfp_t priority) { - return __alloc_skb(size, priority, 1, NUMA_NO_NODE); + return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE); } extern void skb_recycle(struct sk_buff *skb); @@ -643,11 +657,21 @@ static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) { return skb->head + skb->end; } + +static inline unsigned int skb_end_offset(const struct sk_buff *skb) +{ + return skb->end; +} #else static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) { return skb->end; } + +static inline unsigned int skb_end_offset(const struct sk_buff *skb) +{ + return skb->end - skb->head; +} #endif /* Internal */ @@ -881,10 +905,11 @@ static inline struct sk_buff *skb_unshare(struct sk_buff *skb, */ static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_) { - struct sk_buff *list = ((const struct sk_buff *)list_)->next; - if (list == (struct sk_buff *)list_) - list = NULL; - return list; + struct sk_buff *skb = list_->next; + + if (skb == (struct sk_buff *)list_) + skb = NULL; + return skb; } /** @@ -900,6 +925,7 @@ static inline struct sk_buff *skb_peek_next(struct sk_buff *skb, const struct sk_buff_head *list_) { struct sk_buff *next = skb->next; + if (next == (struct sk_buff *)list_) next = NULL; return next; @@ -920,10 +946,12 @@ static inline struct sk_buff *skb_peek_next(struct sk_buff *skb, */ static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_) { - struct sk_buff *list = ((const struct sk_buff *)list_)->prev; - if (list == (struct sk_buff *)list_) - list = NULL; - return list; + struct sk_buff *skb = list_->prev; + + if (skb == (struct sk_buff *)list_) + skb = NULL; + return skb; + } /** @@ -1219,6 +1247,17 @@ static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, { skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + /* + * Propagate page->pfmemalloc to the skb if we can. The problem is + * that not all callers have unique ownership of the page. If + * pfmemalloc is set, we check the mapping as a mapping implies + * page->index is set (index and pfmemalloc share space). + * If it's a valid mapping, we cannot use page->pfmemalloc but we + * do not lose pfmemalloc information as the pages would not be + * allocated using __GFP_MEMALLOC. + */ + if (page->pfmemalloc && !page->mapping) + skb->pfmemalloc = true; frag->page.p = page; frag->page_offset = off; skb_frag_size_set(frag, size); @@ -1649,6 +1688,22 @@ static inline void skb_orphan(struct sk_buff *skb) } /** + * skb_orphan_frags - orphan the frags contained in a buffer + * @skb: buffer to orphan frags from + * @gfp_mask: allocation mask for replacement pages + * + * For each frag in the SKB which needs a destructor (i.e. has an + * owner) create a copy of that frag and release the original + * page by calling the destructor. + */ +static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask) +{ + if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY))) + return 0; + return skb_copy_ubufs(skb, gfp_mask); +} + +/** * __skb_queue_purge - empty a list * @list: list to empty * @@ -1664,31 +1719,11 @@ static inline void __skb_queue_purge(struct sk_buff_head *list) kfree_skb(skb); } -/** - * __dev_alloc_skb - allocate an skbuff for receiving - * @length: length to allocate - * @gfp_mask: get_free_pages mask, passed to alloc_skb - * - * Allocate a new &sk_buff and assign it a usage count of one. The - * buffer has unspecified headroom built in. Users should allocate - * the headroom they think they need without accounting for the - * built in space. The built in space is used for optimisations. - * - * %NULL is returned if there is no free memory. - */ -static inline struct sk_buff *__dev_alloc_skb(unsigned int length, - gfp_t gfp_mask) -{ - struct sk_buff *skb = alloc_skb(length + NET_SKB_PAD, gfp_mask); - if (likely(skb)) - skb_reserve(skb, NET_SKB_PAD); - return skb; -} - -extern struct sk_buff *dev_alloc_skb(unsigned int length); +extern void *netdev_alloc_frag(unsigned int fragsz); extern struct sk_buff *__netdev_alloc_skb(struct net_device *dev, - unsigned int length, gfp_t gfp_mask); + unsigned int length, + gfp_t gfp_mask); /** * netdev_alloc_skb - allocate an skbuff for rx on a specific device @@ -1704,11 +1739,25 @@ extern struct sk_buff *__netdev_alloc_skb(struct net_device *dev, * allocates memory it can be called from an interrupt. */ static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev, - unsigned int length) + unsigned int length) { return __netdev_alloc_skb(dev, length, GFP_ATOMIC); } +/* legacy helper around __netdev_alloc_skb() */ +static inline struct sk_buff *__dev_alloc_skb(unsigned int length, + gfp_t gfp_mask) +{ + return __netdev_alloc_skb(NULL, length, gfp_mask); +} + +/* legacy helper around netdev_alloc_skb() */ +static inline struct sk_buff *dev_alloc_skb(unsigned int length) +{ + return netdev_alloc_skb(NULL, length); +} + + static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev, unsigned int length, gfp_t gfp) { @@ -1725,6 +1774,61 @@ static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev, return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC); } +/* + * __skb_alloc_page - allocate pages for ps-rx on a skb and preserve pfmemalloc data + * @gfp_mask: alloc_pages_node mask. Set __GFP_NOMEMALLOC if not for network packet RX + * @skb: skb to set pfmemalloc on if __GFP_MEMALLOC is used + * @order: size of the allocation + * + * Allocate a new page. + * + * %NULL is returned if there is no free memory. +*/ +static inline struct page *__skb_alloc_pages(gfp_t gfp_mask, + struct sk_buff *skb, + unsigned int order) +{ + struct page *page; + + gfp_mask |= __GFP_COLD; + + if (!(gfp_mask & __GFP_NOMEMALLOC)) + gfp_mask |= __GFP_MEMALLOC; + + page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, order); + if (skb && page && page->pfmemalloc) + skb->pfmemalloc = true; + + return page; +} + +/** + * __skb_alloc_page - allocate a page for ps-rx for a given skb and preserve pfmemalloc data + * @gfp_mask: alloc_pages_node mask. Set __GFP_NOMEMALLOC if not for network packet RX + * @skb: skb to set pfmemalloc on if __GFP_MEMALLOC is used + * + * Allocate a new page. + * + * %NULL is returned if there is no free memory. + */ +static inline struct page *__skb_alloc_page(gfp_t gfp_mask, + struct sk_buff *skb) +{ + return __skb_alloc_pages(gfp_mask, skb, 0); +} + +/** + * skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page + * @page: The page that was allocated from skb_alloc_page + * @skb: The skb that may need pfmemalloc set + */ +static inline void skb_propagate_pfmemalloc(struct page *page, + struct sk_buff *skb) +{ + if (page && page->pfmemalloc) + skb->pfmemalloc = true; +} + /** * skb_frag_page - retrieve the page refered to by a paged fragment * @frag: the paged fragment @@ -1881,8 +1985,6 @@ static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom, { int delta = 0; - if (headroom < NET_SKB_PAD) - headroom = NET_SKB_PAD; if (headroom > skb_headroom(skb)) delta = headroom - skb_headroom(skb); @@ -1963,8 +2065,8 @@ static inline int skb_add_data(struct sk_buff *skb, return -EFAULT; } -static inline int skb_can_coalesce(struct sk_buff *skb, int i, - const struct page *page, int off) +static inline bool skb_can_coalesce(struct sk_buff *skb, int i, + const struct page *page, int off) { if (i) { const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1]; @@ -1972,7 +2074,7 @@ static inline int skb_can_coalesce(struct sk_buff *skb, int i, return page == skb_frag_page(frag) && off == frag->page_offset + skb_frag_size(frag); } - return 0; + return false; } static inline int __skb_linearize(struct sk_buff *skb) @@ -2552,7 +2654,7 @@ static inline bool skb_is_recycleable(const struct sk_buff *skb, int skb_size) return false; skb_size = SKB_DATA_ALIGN(skb_size + NET_SKB_PAD); - if (skb_end_pointer(skb) - skb->head < skb_size) + if (skb_end_offset(skb) < skb_size) return false; if (skb_shared(skb) || skb_cloned(skb)) @@ -2560,5 +2662,19 @@ static inline bool skb_is_recycleable(const struct sk_buff *skb, int skb_size) return true; } + +/** + * skb_head_is_locked - Determine if the skb->head is locked down + * @skb: skb to check + * + * The head on skbs build around a head frag can be removed if they are + * not cloned. This function returns true if the skb head is locked down + * due to either being allocated via kmalloc, or by being a clone with + * multiple references to the head. + */ +static inline bool skb_head_is_locked(const struct sk_buff *skb) +{ + return !skb->head_frag || skb_cloned(skb); +} #endif /* __KERNEL__ */ #endif /* _LINUX_SKBUFF_H */ |