From 22781a5b9cd8aa58908eec8e269643559335f455 Mon Sep 17 00:00:00 2001 From: Duan Jiong Date: Fri, 6 Dec 2013 13:29:36 +0800 Subject: packet: use macro GET_PBDQC_FROM_RB to simplify the codes Signed-off-by: Duan Jiong Signed-off-by: David S. Miller --- net/packet/af_packet.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index ba2548bd85bf..c235da940019 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -437,7 +437,8 @@ static void prb_shutdown_retire_blk_timer(struct packet_sock *po, { struct tpacket_kbdq_core *pkc; - pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc; + pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) : + GET_PBDQC_FROM_RB(&po->rx_ring); spin_lock_bh(&rb_queue->lock); pkc->delete_blk_timer = 1; @@ -463,7 +464,8 @@ static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring) if (tx_ring) BUG(); - pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc; + pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) : + GET_PBDQC_FROM_RB(&po->rx_ring); prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired); } @@ -521,7 +523,7 @@ static void init_prb_bdqc(struct packet_sock *po, struct pgv *pg_vec, union tpacket_req_u *req_u, int tx_ring) { - struct tpacket_kbdq_core *p1 = &rb->prb_bdqc; + struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb); struct tpacket_block_desc *pbd; memset(p1, 0x0, sizeof(*p1)); @@ -585,7 +587,7 @@ static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc) static void prb_retire_rx_blk_timer_expired(unsigned long data) { struct packet_sock *po = (struct packet_sock *)data; - struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc; + struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring); unsigned int frozen; struct tpacket_block_desc *pbd; -- cgit v1.2.3 From d346a3fae3ff1d99f5d0c819bf86edf9094a26a1 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 6 Dec 2013 11:36:17 +0100 Subject: packet: introduce PACKET_QDISC_BYPASS socket option This patch introduces a PACKET_QDISC_BYPASS socket option, that allows for using a similar xmit() function as in pktgen instead of taking the dev_queue_xmit() path. This can be very useful when PF_PACKET applications are required to be used in a similar scenario as pktgen, but with full, flexible packet payload that needs to be provided, for example. On default, nothing changes in behaviour for normal PF_PACKET TX users, so everything stays as is for applications. New users, however, can now set PACKET_QDISC_BYPASS if needed to prevent own packets from i) reentering packet_rcv() and ii) to directly push the frame to the driver. In doing so we can increase pps (here 64 byte packets) for PF_PACKET a bit: # CPUs -- QDISC_BYPASS -- qdisc path -- qdisc path[**] 1 CPU == 1,509,628 pps -- 1,208,708 -- 1,247,436 2 CPUs == 3,198,659 pps -- 2,536,012 -- 1,605,779 3 CPUs == 4,787,992 pps -- 3,788,740 -- 1,735,610 4 CPUs == 6,173,956 pps -- 4,907,799 -- 1,909,114 5 CPUs == 7,495,676 pps -- 5,956,499 -- 2,014,422 6 CPUs == 9,001,496 pps -- 7,145,064 -- 2,155,261 7 CPUs == 10,229,776 pps -- 8,190,596 -- 2,220,619 8 CPUs == 11,040,732 pps -- 9,188,544 -- 2,241,879 9 CPUs == 12,009,076 pps -- 10,275,936 -- 2,068,447 10 CPUs == 11,380,052 pps -- 11,265,337 -- 1,578,689 11 CPUs == 11,672,676 pps -- 11,845,344 -- 1,297,412 [...] 20 CPUs == 11,363,192 pps -- 11,014,933 -- 1,245,081 [**]: qdisc path with packet_rcv(), how probably most people seem to use it (hopefully not anymore if not needed) The test was done using a modified trafgen, sending a simple static 64 bytes packet, on all CPUs. The trick in the fast "qdisc path" case, is to avoid reentering packet_rcv() by setting the RAW socket protocol to zero, like: socket(PF_PACKET, SOCK_RAW, 0); Tradeoffs are documented as well in this patch, clearly, if queues are busy, we will drop more packets, tc disciplines are ignored, and these packets are not visible to taps anymore. For a pktgen like scenario, we argue that this is acceptable. The pointer to the xmit function has been placed in packet socket structure hole between cached_dev and prot_hook that is hot anyway as we're working on cached_dev in each send path. Done in joint work together with Jesper Dangaard Brouer. Signed-off-by: Daniel Borkmann Signed-off-by: Jesper Dangaard Brouer Signed-off-by: David S. Miller --- Documentation/networking/packet_mmap.txt | 21 ++++++++ include/uapi/linux/if_packet.h | 1 + net/packet/af_packet.c | 91 +++++++++++++++++++++++++++----- net/packet/internal.h | 1 + 4 files changed, 102 insertions(+), 12 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/Documentation/networking/packet_mmap.txt b/Documentation/networking/packet_mmap.txt index 8e48e3b14227..4288ffafba9f 100644 --- a/Documentation/networking/packet_mmap.txt +++ b/Documentation/networking/packet_mmap.txt @@ -952,6 +952,27 @@ int main(int argc, char **argp) return 0; } +------------------------------------------------------------------------------- ++ PACKET_QDISC_BYPASS +------------------------------------------------------------------------------- + +If there is a requirement to load the network with many packets in a similar +fashion as pktgen does, you might set the following option after socket +creation: + + int one = 1; + setsockopt(fd, SOL_PACKET, PACKET_QDISC_BYPASS, &one, sizeof(one)); + +This has the side-effect, that packets sent through PF_PACKET will bypass the +kernel's qdisc layer and are forcedly pushed to the driver directly. Meaning, +packet are not buffered, tc disciplines are ignored, increased loss can occur +and such packets are also not visible to other PF_PACKET sockets anymore. So, +you have been warned; generally, this can be useful for stress testing various +components of a system. + +On default, PACKET_QDISC_BYPASS is disabled and needs to be explicitly enabled +on PF_PACKET sockets. + ------------------------------------------------------------------------------- + PACKET_TIMESTAMP ------------------------------------------------------------------------------- diff --git a/include/uapi/linux/if_packet.h b/include/uapi/linux/if_packet.h index dbf06667394b..1e24aa701cbd 100644 --- a/include/uapi/linux/if_packet.h +++ b/include/uapi/linux/if_packet.h @@ -51,6 +51,7 @@ struct sockaddr_ll { #define PACKET_TIMESTAMP 17 #define PACKET_FANOUT 18 #define PACKET_TX_HAS_OFF 19 +#define PACKET_QDISC_BYPASS 20 #define PACKET_FANOUT_HASH 0 #define PACKET_FANOUT_LB 1 diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index e4171dd98590..9d70f1349926 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -237,6 +237,48 @@ struct packet_skb_cb { static void __fanout_unlink(struct sock *sk, struct packet_sock *po); static void __fanout_link(struct sock *sk, struct packet_sock *po); +static int packet_direct_xmit(struct sk_buff *skb) +{ + struct net_device *dev = skb->dev; + const struct net_device_ops *ops = dev->netdev_ops; + netdev_features_t features; + struct netdev_queue *txq; + u16 queue_map; + int ret; + + if (unlikely(!netif_running(dev) || + !netif_carrier_ok(dev))) { + kfree_skb(skb); + return NET_XMIT_DROP; + } + + features = netif_skb_features(skb); + if (skb_needs_linearize(skb, features) && + __skb_linearize(skb)) { + kfree_skb(skb); + return NET_XMIT_DROP; + } + + queue_map = skb_get_queue_mapping(skb); + txq = netdev_get_tx_queue(dev, queue_map); + + __netif_tx_lock_bh(txq); + if (unlikely(netif_xmit_frozen_or_stopped(txq))) { + ret = NETDEV_TX_BUSY; + kfree_skb(skb); + goto out; + } + + ret = ops->ndo_start_xmit(skb, dev); + if (likely(dev_xmit_complete(ret))) + txq_trans_update(txq); + else + kfree_skb(skb); +out: + __netif_tx_unlock_bh(txq); + return ret; +} + static struct net_device *packet_cached_dev_get(struct packet_sock *po) { struct net_device *dev; @@ -261,6 +303,16 @@ static void packet_cached_dev_reset(struct packet_sock *po) RCU_INIT_POINTER(po->cached_dev, NULL); } +static bool packet_use_direct_xmit(const struct packet_sock *po) +{ + return po->xmit == packet_direct_xmit; +} + +static u16 packet_pick_tx_queue(struct net_device *dev) +{ + return (u16) smp_processor_id() % dev->real_num_tx_queues; +} + /* register_prot_hook must be invoked with the po->bind_lock held, * or from a context in which asynchronous accesses to the packet * socket is not possible (packet_create()). @@ -1994,9 +2046,10 @@ static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, skb_reserve(skb, hlen); skb_reset_network_header(skb); - skb_probe_transport_header(skb, 0); - if (po->tp_tx_has_off) { + if (!packet_use_direct_xmit(po)) + skb_probe_transport_header(skb, 0); + if (unlikely(po->tp_tx_has_off)) { int off_min, off_max, off; off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll); off_max = po->tx_ring.frame_size - tp_len; @@ -2166,12 +2219,13 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) } } + skb_set_queue_mapping(skb, packet_pick_tx_queue(dev)); skb->destructor = tpacket_destruct_skb; __packet_set_status(po, ph, TP_STATUS_SENDING); atomic_inc(&po->tx_ring.pending); status = TP_STATUS_SEND_REQUEST; - err = dev_queue_xmit(skb); + err = po->xmit(skb); if (unlikely(err > 0)) { err = net_xmit_errno(err); if (err && __packet_get_status(po, ph) == @@ -2230,8 +2284,7 @@ static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad, return skb; } -static int packet_snd(struct socket *sock, - struct msghdr *msg, size_t len) +static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) { struct sock *sk = sock->sk; struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name; @@ -2376,6 +2429,7 @@ static int packet_snd(struct socket *sock, skb->dev = dev; skb->priority = sk->sk_priority; skb->mark = sk->sk_mark; + skb_set_queue_mapping(skb, packet_pick_tx_queue(dev)); if (po->has_vnet_hdr) { if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { @@ -2396,16 +2450,12 @@ static int packet_snd(struct socket *sock, len += vnet_hdr_len; } - skb_probe_transport_header(skb, reserve); - + if (!packet_use_direct_xmit(po)) + skb_probe_transport_header(skb, reserve); if (unlikely(extra_len == 4)) skb->no_fcs = 1; - /* - * Now send it - */ - - err = dev_queue_xmit(skb); + err = po->xmit(skb); if (err > 0 && (err = net_xmit_errno(err)) != 0) goto out_unlock; @@ -2427,6 +2477,7 @@ static int packet_sendmsg(struct kiocb *iocb, struct socket *sock, { struct sock *sk = sock->sk; struct packet_sock *po = pkt_sk(sk); + if (po->tx_ring.pg_vec) return tpacket_snd(po, msg); else @@ -2641,6 +2692,7 @@ static int packet_create(struct net *net, struct socket *sock, int protocol, po = pkt_sk(sk); sk->sk_family = PF_PACKET; po->num = proto; + po->xmit = dev_queue_xmit; packet_cached_dev_reset(po); @@ -3220,6 +3272,18 @@ packet_setsockopt(struct socket *sock, int level, int optname, char __user *optv po->tp_tx_has_off = !!val; return 0; } + case PACKET_QDISC_BYPASS: + { + int val; + + if (optlen != sizeof(val)) + return -EINVAL; + if (copy_from_user(&val, optval, sizeof(val))) + return -EFAULT; + + po->xmit = val ? packet_direct_xmit : dev_queue_xmit; + return 0; + } default: return -ENOPROTOOPT; } @@ -3312,6 +3376,9 @@ static int packet_getsockopt(struct socket *sock, int level, int optname, case PACKET_TX_HAS_OFF: val = po->tp_tx_has_off; break; + case PACKET_QDISC_BYPASS: + val = packet_use_direct_xmit(po); + break; default: return -ENOPROTOOPT; } diff --git a/net/packet/internal.h b/net/packet/internal.h index 1035fa2d909c..0a87d7b36c9e 100644 --- a/net/packet/internal.h +++ b/net/packet/internal.h @@ -114,6 +114,7 @@ struct packet_sock { unsigned int tp_tx_has_off:1; unsigned int tp_tstamp; struct net_device __rcu *cached_dev; + int (*xmit)(struct sk_buff *skb); struct packet_type prot_hook ____cacheline_aligned_in_smp; }; -- cgit v1.2.3 From 1cbac010529b393edbeffd0fb7e7e710e14084ac Mon Sep 17 00:00:00 2001 From: Li Zhong Date: Thu, 12 Dec 2013 22:39:55 +0100 Subject: packet: fix using smp_processor_id() in preemptible code This patches fixes the following warning by replacing smp_processor_id() with raw_smp_processor_id(): [ 11.120893] BUG: using smp_processor_id() in preemptible [00000000] code: arping/3510 [ 11.120913] caller is .packet_sendmsg+0xc14/0xe68 [ 11.120920] CPU: 13 PID: 3510 Comm: arping Not tainted 3.13.0-rc3-next-20131211-dirty #1 [ 11.120926] Call Trace: [ 11.120932] [c0000001f803f6f0] [c0000000000138dc] .show_stack+0x110/0x25c (unreliable) [ 11.120942] [c0000001f803f7e0] [c00000000083dd24] .dump_stack+0xa0/0x37c [ 11.120951] [c0000001f803f870] [c000000000493fd4] .debug_smp_processor_id+0xfc/0x12c [ 11.120959] [c0000001f803f900] [c0000000007eba78] .packet_sendmsg+0xc14/0xe68 [ 11.120968] [c0000001f803fa80] [c000000000700968] .sock_sendmsg+0xa0/0xe0 [ 11.120975] [c0000001f803fbf0] [c0000000007014d8] .SyS_sendto+0x100/0x148 [ 11.120983] [c0000001f803fd60] [c0000000006fff10] .SyS_socketcall+0x1c4/0x2e8 [ 11.120990] [c0000001f803fe30] [c00000000000a1e4] syscall_exit+0x0/0x9c Signed-off-by: Li Zhong Acked-by: Jesper Dangaard Brouer Signed-off-by: Daniel Borkmann Signed-off-by: David S. Miller --- net/packet/af_packet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'net/packet/af_packet.c') diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 9d70f1349926..cc803c63059a 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -310,7 +310,7 @@ static bool packet_use_direct_xmit(const struct packet_sock *po) static u16 packet_pick_tx_queue(struct net_device *dev) { - return (u16) smp_processor_id() % dev->real_num_tx_queues; + return (u16) raw_smp_processor_id() % dev->real_num_tx_queues; } /* register_prot_hook must be invoked with the po->bind_lock held, -- cgit v1.2.3 From 3958afa1b272eb07109fd31549e69193b4d7c364 Mon Sep 17 00:00:00 2001 From: Tom Herbert Date: Sun, 15 Dec 2013 22:12:06 -0800 Subject: net: Change skb_get_rxhash to skb_get_hash Changing name of function as part of making the hash in skbuff to be generic property, not just for receive path. Signed-off-by: Tom Herbert Signed-off-by: David S. Miller --- drivers/net/macvtap.c | 2 +- drivers/net/tun.c | 4 ++-- drivers/net/vxlan.c | 2 +- include/linux/skbuff.h | 6 +++--- net/core/dev.c | 4 ++-- net/core/flow_dissector.c | 6 +++--- net/packet/af_packet.c | 4 ++-- net/sched/cls_flow.c | 2 +- net/sched/em_meta.c | 2 +- net/sched/sch_fq.c | 2 +- 10 files changed, 17 insertions(+), 17 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c index 64409af0da31..431f58cb2ceb 100644 --- a/drivers/net/macvtap.c +++ b/drivers/net/macvtap.c @@ -224,7 +224,7 @@ static struct macvtap_queue *macvtap_get_queue(struct net_device *dev, goto out; /* Check if we can use flow to select a queue */ - rxq = skb_get_rxhash(skb); + rxq = skb_get_hash(skb); if (rxq) { tap = rcu_dereference(vlan->taps[rxq % numvtaps]); goto out; diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 3c5a8d8cde50..8569da248336 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -358,7 +358,7 @@ static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb) rcu_read_lock(); numqueues = ACCESS_ONCE(tun->numqueues); - txq = skb_get_rxhash(skb); + txq = skb_get_hash(skb); if (txq) { e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq); if (e) @@ -1146,7 +1146,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, skb_reset_network_header(skb); skb_probe_transport_header(skb, 0); - rxhash = skb_get_rxhash(skb); + rxhash = skb_get_hash(skb); netif_rx_ni(skb); tun->dev->stats.rx_packets++; diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c index 58f6a0c02b17..b247a7a29366 100644 --- a/drivers/net/vxlan.c +++ b/drivers/net/vxlan.c @@ -1405,7 +1405,7 @@ __be16 vxlan_src_port(__u16 port_min, __u16 port_max, struct sk_buff *skb) unsigned int range = (port_max - port_min) + 1; u32 hash; - hash = skb_get_rxhash(skb); + hash = skb_get_hash(skb); if (!hash) hash = jhash(skb->data, 2 * ETH_ALEN, (__force u32) skb->protocol); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 77c7aae1c6b2..4725b953e00d 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -703,11 +703,11 @@ unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, unsigned int to, struct ts_config *config, struct ts_state *state); -void __skb_get_rxhash(struct sk_buff *skb); -static inline __u32 skb_get_rxhash(struct sk_buff *skb) +void __skb_get_hash(struct sk_buff *skb); +static inline __u32 skb_get_hash(struct sk_buff *skb) { if (!skb->l4_rxhash) - __skb_get_rxhash(skb); + __skb_get_hash(skb); return skb->rxhash; } diff --git a/net/core/dev.c b/net/core/dev.c index 9d4369ece679..c482fe8abf87 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3006,7 +3006,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, } skb_reset_network_header(skb); - if (!skb_get_rxhash(skb)) + if (!skb_get_hash(skb)) goto done; flow_table = rcu_dereference(rxqueue->rps_flow_table); @@ -3151,7 +3151,7 @@ static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) rcu_read_lock(); fl = rcu_dereference(sd->flow_limit); if (fl) { - new_flow = skb_get_rxhash(skb) & (fl->num_buckets - 1); + new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); old_flow = fl->history[fl->history_head]; fl->history[fl->history_head] = new_flow; diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index d6ef17322500..b324bfa3485c 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -202,12 +202,12 @@ static __always_inline u32 __flow_hash_1word(u32 a) } /* - * __skb_get_rxhash: calculate a flow hash based on src/dst addresses + * __skb_get_hash: calculate a flow hash based on src/dst addresses * and src/dst port numbers. Sets rxhash in skb to non-zero hash value * on success, zero indicates no valid hash. Also, sets l4_rxhash in skb * if hash is a canonical 4-tuple hash over transport ports. */ -void __skb_get_rxhash(struct sk_buff *skb) +void __skb_get_hash(struct sk_buff *skb) { struct flow_keys keys; u32 hash; @@ -234,7 +234,7 @@ void __skb_get_rxhash(struct sk_buff *skb) skb->rxhash = hash; } -EXPORT_SYMBOL(__skb_get_rxhash); +EXPORT_SYMBOL(__skb_get_hash); /* * Returns a Tx hash based on the given packet descriptor a Tx queues' number diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index cc803c63059a..24675f06f4fd 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -963,7 +963,7 @@ static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb) static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc, struct tpacket3_hdr *ppd) { - ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb); + ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb); } static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc, @@ -1295,7 +1295,7 @@ static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev, if (!skb) return 0; } - skb_get_rxhash(skb); + skb_get_hash(skb); idx = fanout_demux_hash(f, skb, num); break; case PACKET_FANOUT_LB: diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c index 7881e2fccbc2..83a6322b7751 100644 --- a/net/sched/cls_flow.c +++ b/net/sched/cls_flow.c @@ -220,7 +220,7 @@ static u32 flow_get_vlan_tag(const struct sk_buff *skb) static u32 flow_get_rxhash(struct sk_buff *skb) { - return skb_get_rxhash(skb); + return skb_get_hash(skb); } static u32 flow_key_get(struct sk_buff *skb, int key, struct flow_keys *flow) diff --git a/net/sched/em_meta.c b/net/sched/em_meta.c index 382519a5d7f9..9b8c0b0e60d7 100644 --- a/net/sched/em_meta.c +++ b/net/sched/em_meta.c @@ -222,7 +222,7 @@ META_COLLECTOR(int_maclen) META_COLLECTOR(int_rxhash) { - dst->value = skb_get_rxhash(skb); + dst->value = skb_get_hash(skb); } /************************************************************************** diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index f2fb92dd970d..08ef7a42c0e4 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -226,7 +226,7 @@ static struct fq_flow *fq_classify(struct sk_buff *skb, struct fq_sched_data *q) /* By forcing low order bit to 1, we make sure to not * collide with a local flow (socket pointers are word aligned) */ - sk = (struct sock *)(skb_get_rxhash(skb) | 1L); + sk = (struct sock *)(skb_get_hash(skb) | 1L); } root = &q->fq_root[hash_32((u32)(long)sk, q->fq_trees_log)]; -- cgit v1.2.3 From 51846355bc239b7d229ed7a1ea7b0333cbd23f01 Mon Sep 17 00:00:00 2001 From: Atzm Watanabe Date: Tue, 17 Dec 2013 22:53:32 +0900 Subject: packet: make aligned size of struct tpacket{2,3}_hdr clear struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT. We may add members to them until current aligned size without forcing userspace to call getsockopt(..., PACKET_HDRLEN, ...). Signed-off-by: Atzm Watanabe Acked-by: Daniel Borkmann Signed-off-by: David S. Miller --- net/packet/af_packet.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'net/packet/af_packet.c') diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 24675f06f4fd..4fd52181b6ce 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1812,6 +1812,13 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, struct timespec ts; __u32 ts_status; + /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT. + * We may add members to them until current aligned size without forcing + * userspace to call getsockopt(..., PACKET_HDRLEN, ...). + */ + BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32); + BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48); + if (skb->pkt_type == PACKET_LOOPBACK) goto drop; -- cgit v1.2.3 From e4d26f4b080f55e9577b45e6b51a04971eb459e9 Mon Sep 17 00:00:00 2001 From: Atzm Watanabe Date: Tue, 17 Dec 2013 22:53:36 +0900 Subject: packet: fill the gap of TPACKET_ALIGNMENT with zeros struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT. Explicitly defining and zeroing the gap of this makes additional changes easier. Signed-off-by: Atzm Watanabe Acked-by: Daniel Borkmann Signed-off-by: David S. Miller --- include/uapi/linux/if_packet.h | 3 ++- net/packet/af_packet.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/include/uapi/linux/if_packet.h b/include/uapi/linux/if_packet.h index 1e24aa701cbd..9185dc9a4485 100644 --- a/include/uapi/linux/if_packet.h +++ b/include/uapi/linux/if_packet.h @@ -133,7 +133,7 @@ struct tpacket2_hdr { __u32 tp_sec; __u32 tp_nsec; __u16 tp_vlan_tci; - __u16 tp_padding; + __u8 tp_padding[6]; }; struct tpacket_hdr_variant1 { @@ -154,6 +154,7 @@ struct tpacket3_hdr { union { struct tpacket_hdr_variant1 hv1; }; + __u8 tp_padding[12]; }; struct tpacket_bd_ts { diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 4fd52181b6ce..f281999ba92e 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1929,7 +1929,7 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, } else { h.h2->tp_vlan_tci = 0; } - h.h2->tp_padding = 0; + memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding)); hdrlen = sizeof(*h.h2); break; case TPACKET_V3: @@ -1943,6 +1943,7 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, h.h3->tp_net = netoff; h.h3->tp_sec = ts.tv_sec; h.h3->tp_nsec = ts.tv_nsec; + memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding)); hdrlen = sizeof(*h.h3); break; default: -- cgit v1.2.3 From a0cdfcf39362410d5ea983f4daf67b38de129408 Mon Sep 17 00:00:00 2001 From: Atzm Watanabe Date: Tue, 17 Dec 2013 22:53:40 +0900 Subject: packet: deliver VLAN TPID to userspace This enables userspace to get VLAN TPID as well as the VLAN TCI. Signed-off-by: Atzm Watanabe Acked-by: Daniel Borkmann Signed-off-by: David S. Miller --- include/uapi/linux/if_packet.h | 24 ++++++++++++++---------- net/packet/af_packet.c | 14 ++++++++++---- 2 files changed, 24 insertions(+), 14 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/include/uapi/linux/if_packet.h b/include/uapi/linux/if_packet.h index 9185dc9a4485..e9d844c80c11 100644 --- a/include/uapi/linux/if_packet.h +++ b/include/uapi/linux/if_packet.h @@ -84,17 +84,18 @@ struct tpacket_auxdata { __u16 tp_mac; __u16 tp_net; __u16 tp_vlan_tci; - __u16 tp_padding; + __u16 tp_vlan_tpid; }; /* Rx ring - header status */ -#define TP_STATUS_KERNEL 0 -#define TP_STATUS_USER (1 << 0) -#define TP_STATUS_COPY (1 << 1) -#define TP_STATUS_LOSING (1 << 2) -#define TP_STATUS_CSUMNOTREADY (1 << 3) -#define TP_STATUS_VLAN_VALID (1 << 4) /* auxdata has valid tp_vlan_tci */ -#define TP_STATUS_BLK_TMO (1 << 5) +#define TP_STATUS_KERNEL 0 +#define TP_STATUS_USER (1 << 0) +#define TP_STATUS_COPY (1 << 1) +#define TP_STATUS_LOSING (1 << 2) +#define TP_STATUS_CSUMNOTREADY (1 << 3) +#define TP_STATUS_VLAN_VALID (1 << 4) /* auxdata has valid tp_vlan_tci */ +#define TP_STATUS_BLK_TMO (1 << 5) +#define TP_STATUS_VLAN_TPID_VALID (1 << 6) /* auxdata has valid tp_vlan_tpid */ /* Tx ring - header status */ #define TP_STATUS_AVAILABLE 0 @@ -133,12 +134,15 @@ struct tpacket2_hdr { __u32 tp_sec; __u32 tp_nsec; __u16 tp_vlan_tci; - __u8 tp_padding[6]; + __u16 tp_vlan_tpid; + __u8 tp_padding[4]; }; struct tpacket_hdr_variant1 { __u32 tp_rxhash; __u32 tp_vlan_tci; + __u16 tp_vlan_tpid; + __u16 tp_padding; }; struct tpacket3_hdr { @@ -154,7 +158,7 @@ struct tpacket3_hdr { union { struct tpacket_hdr_variant1 hv1; }; - __u8 tp_padding[12]; + __u8 tp_padding[8]; }; struct tpacket_bd_ts { diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index f281999ba92e..dd3840846ce2 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -977,9 +977,11 @@ static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc, { if (vlan_tx_tag_present(pkc->skb)) { ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb); - ppd->tp_status = TP_STATUS_VLAN_VALID; + ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto); + ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID; } else { ppd->hv1.tp_vlan_tci = 0; + ppd->hv1.tp_vlan_tpid = 0; ppd->tp_status = TP_STATUS_AVAILABLE; } } @@ -987,6 +989,7 @@ static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc, static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc, struct tpacket3_hdr *ppd) { + ppd->hv1.tp_padding = 0; prb_fill_vlan_info(pkc, ppd); if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH) @@ -1925,9 +1928,11 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, h.h2->tp_nsec = ts.tv_nsec; if (vlan_tx_tag_present(skb)) { h.h2->tp_vlan_tci = vlan_tx_tag_get(skb); - status |= TP_STATUS_VLAN_VALID; + h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto); + status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID; } else { h.h2->tp_vlan_tci = 0; + h.h2->tp_vlan_tpid = 0; } memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding)); hdrlen = sizeof(*h.h2); @@ -2875,11 +2880,12 @@ static int packet_recvmsg(struct kiocb *iocb, struct socket *sock, aux.tp_net = skb_network_offset(skb); if (vlan_tx_tag_present(skb)) { aux.tp_vlan_tci = vlan_tx_tag_get(skb); - aux.tp_status |= TP_STATUS_VLAN_VALID; + aux.tp_vlan_tpid = ntohs(skb->vlan_proto); + aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID; } else { aux.tp_vlan_tci = 0; + aux.tp_vlan_tpid = 0; } - aux.tp_padding = 0; put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux); } -- cgit v1.2.3 From d4dd8aeefda96225abbb6ab21c80bb1cd9b684db Mon Sep 17 00:00:00 2001 From: Weilong Chen Date: Mon, 23 Dec 2013 11:31:38 +0800 Subject: packet: fix "foo * bar" and "(foo*)" problems Cleanup checkpatch errors.Specially,the second changed line is exactly 80 columns long. Signed-off-by: Weilong Chen Signed-off-by: David S. Miller --- net/packet/af_packet.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index dd3840846ce2..279467b74eb7 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1356,9 +1356,9 @@ static void __fanout_unlink(struct sock *sk, struct packet_sock *po) spin_unlock(&f->lock); } -static bool match_fanout_group(struct packet_type *ptype, struct sock * sk) +static bool match_fanout_group(struct packet_type *ptype, struct sock *sk) { - if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout) + if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout) return true; return false; -- cgit v1.2.3 From 902fefb82ef72a50c78cb4a20cc954b037a98d1c Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 15 Jan 2014 16:25:34 +0100 Subject: packet: improve socket create/bind latency in some cases Most people acquire PF_PACKET sockets with a protocol argument in the socket call, e.g. libpcap does so with htons(ETH_P_ALL) for all its sockets. Most likely, at some point in time a subsequent bind() call will follow, e.g. in libpcap with ... memset(&sll, 0, sizeof(sll)); sll.sll_family = AF_PACKET; sll.sll_ifindex = ifindex; sll.sll_protocol = htons(ETH_P_ALL); ... as arguments. What happens in the kernel is that already in socket() syscall, we install a proto hook via register_prot_hook() if our protocol argument is != 0. Yet, in bind() we're almost doing the same work by doing a unregister_prot_hook() with an expensive synchronize_net() call in case during socket() the proto was != 0, plus follow-up register_prot_hook() with a bound device to it this time, in order to limit traffic we get. In the case when the protocol and user supplied device index (== 0) does not change from socket() to bind(), we can spare us doing the same work twice. Similarly for re-binding to the same device and protocol. For these scenarios, we can decrease create/bind latency from ~7447us (sock-bind-2 case) to ~89us (sock-bind-1 case) with this patch. Alternatively, for the first case, if people care, they should simply create their sockets with proto == 0 argument and define the protocol during bind() as this saves a call to synchronize_net() as well (sock-bind-3 case). In all other cases, we're tied to user space behaviour we must not change, also since a bind() is not strictly required. Thus, we need the synchronize_net() to make sure no asynchronous packet processing paths still refer to the previous elements of po->prot_hook. In case of mmap()ed sockets, the workflow that includes bind() is socket() -> setsockopt() -> bind(). In that case, a pair of {__unregister, register}_prot_hook is being called from setsockopt() in order to install the new protocol receive handler. Thus, when we call bind and can skip a re-hook, we have already previously installed the new handler. For fanout, this is handled different entirely, so we should be good. Timings on an i7-3520M machine: * sock-bind-1: 89 us * sock-bind-2: 7447 us * sock-bind-3: 75 us sock-bind-1: socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP)) = 3 bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=all(0), pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0 sock-bind-2: socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP)) = 3 bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=lo(1), pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0 sock-bind-3: socket(PF_PACKET, SOCK_RAW, 0) = 3 bind(3, {sa_family=AF_PACKET, proto=htons(ETH_P_IP), if=lo(1), pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0 Signed-off-by: Daniel Borkmann Signed-off-by: David S. Miller --- net/packet/af_packet.c | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 279467b74eb7..85bb38cb56fd 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2567,9 +2567,12 @@ static int packet_release(struct socket *sock) * Attach a packet hook. */ -static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol) +static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto) { struct packet_sock *po = pkt_sk(sk); + const struct net_device *dev_curr; + __be16 proto_curr; + bool need_rehook; if (po->fanout) { if (dev) @@ -2579,21 +2582,29 @@ static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protoc } lock_sock(sk); - spin_lock(&po->bind_lock); - unregister_prot_hook(sk, true); - po->num = protocol; - po->prot_hook.type = protocol; - if (po->prot_hook.dev) - dev_put(po->prot_hook.dev); + proto_curr = po->prot_hook.type; + dev_curr = po->prot_hook.dev; + + need_rehook = proto_curr != proto || dev_curr != dev; + + if (need_rehook) { + unregister_prot_hook(sk, true); - po->prot_hook.dev = dev; - po->ifindex = dev ? dev->ifindex : 0; + po->num = proto; + po->prot_hook.type = proto; + + if (po->prot_hook.dev) + dev_put(po->prot_hook.dev); - packet_cached_dev_assign(po, dev); + po->prot_hook.dev = dev; + + po->ifindex = dev ? dev->ifindex : 0; + packet_cached_dev_assign(po, dev); + } - if (protocol == 0) + if (proto == 0 || !need_rehook) goto out_unlock; if (!dev || (dev->flags & IFF_UP)) { -- cgit v1.2.3 From 87a2fd286adf35a87cf6cb30fa80a0726eb74f76 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 15 Jan 2014 16:25:35 +0100 Subject: packet: don't unconditionally schedule() in case of MSG_DONTWAIT In tpacket_snd(), when we've discovered a first frame that is not in status TP_STATUS_SEND_REQUEST, and return a NULL buffer, we exit the send routine in case of MSG_DONTWAIT, since we've finished traversing the mmaped send ring buffer and don't care about pending frames. While doing so, we still unconditionally call an expensive schedule() in the packet_current_frame() "error" path, which is unnecessary in this case since it's enough to just quit the function. Also, in case MSG_DONTWAIT is not set, we should rather test for need_resched() first and do schedule() only if necessary since meanwhile pending frames could already have finished processing and called skb destructor. Signed-off-by: Daniel Borkmann Signed-off-by: David S. Miller --- net/packet/af_packet.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 85bb38cb56fd..d5495d87f399 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2156,6 +2156,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) int err, reserve = 0; void *ph; struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name; + bool need_wait = !(msg->msg_flags & MSG_DONTWAIT); int tp_len, size_max; unsigned char *addr; int len_sum = 0; @@ -2198,10 +2199,10 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) do { ph = packet_current_frame(po, &po->tx_ring, - TP_STATUS_SEND_REQUEST); - + TP_STATUS_SEND_REQUEST); if (unlikely(ph == NULL)) { - schedule(); + if (need_wait && need_resched()) + schedule(); continue; } @@ -2255,10 +2256,8 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) } packet_increment_head(&po->tx_ring); len_sum += tp_len; - } while (likely((ph != NULL) || - ((!(msg->msg_flags & MSG_DONTWAIT)) && - (atomic_read(&po->tx_ring.pending)))) - ); + } while (likely((ph != NULL) || (need_wait && + atomic_read(&po->tx_ring.pending)))); err = len_sum; goto out_put; -- cgit v1.2.3 From b013840810c221f2b0cf641d01531526052dc1fb Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 15 Jan 2014 16:25:36 +0100 Subject: packet: use percpu mmap tx frame pending refcount In PF_PACKET's packet mmap(), we can avoid using one atomic_inc() and one atomic_dec() call in skb destructor and use a percpu reference count instead in order to determine if packets are still pending to be sent out. Micro-benchmark with [1] that has been slightly modified (that is, protcol = 0 in socket(2) and bind(2)), example on a rather crappy testing machine; I expect it to scale and have even better results on bigger machines: ./packet_mm_tx -s7000 -m7200 -z700000 em1, avg over 2500 runs: With patch: 4,022,015 cyc Without patch: 4,812,994 cyc time ./packet_mm_tx -s64 -c10000000 em1 > /dev/null, stable: With patch: real 1m32.241s user 0m0.287s sys 1m29.316s Without patch: real 1m38.386s user 0m0.265s sys 1m35.572s In function tpacket_snd(), it is okay to use packet_read_pending() since in fast-path we short-circuit the condition already with ph != NULL, since we have next frames to process. In case we have MSG_DONTWAIT, we also do not execute this path as need_wait is false here anyway, and in case of _no_ MSG_DONTWAIT flag, it is okay to call a packet_read_pending(), because when we ever reach that path, we're done processing outgoing frames anyway and only look if there are skbs still outstanding to be orphaned. We can stay lockless in this percpu counter since it's acceptable when we reach this path for the sum to be imprecise first, but we'll level out at 0 after all pending frames have reached the skb destructor eventually through tx reclaim. When people pin a tx process to particular CPUs, we expect overflows to happen in the reference counter as on one CPU we expect heavy increase; and distributed through ksoftirqd on all CPUs a decrease, for example. As David Laight points out, since the C language doesn't define the result of signed int overflow (i.e. rather than wrap, it is allowed to saturate as a possible outcome), we have to use unsigned int as reference count. The sum over all CPUs when tx is complete will result in 0 again. The BUG_ON() in tpacket_destruct_skb() we can remove as well. It can _only_ be set from inside tpacket_snd() path and we made sure to increase tx_ring.pending in any case before we called po->xmit(skb). So testing for tx_ring.pending == 0 is not too useful. Instead, it would rather have been useful to test if lower layers didn't orphan the skb so that we're missing ring slots being put back to TP_STATUS_AVAILABLE. But such a bug will be caught in user space already as we end up realizing that we do not have any TP_STATUS_AVAILABLE slots left anymore. Therefore, we're all set. Btw, in case of RX_RING path, we do not make use of the pending member, therefore we also don't need to use up any percpu memory here. Also note that __alloc_percpu() already returns a zero-filled percpu area, so initialization is done already. [1] http://wiki.ipxwarzone.com/index.php5?title=Linux_packet_mmap Signed-off-by: Daniel Borkmann Signed-off-by: David S. Miller --- net/packet/af_packet.c | 66 +++++++++++++++++++++++++++++++++++++++++++++----- net/packet/diag.c | 1 + net/packet/internal.h | 2 +- 3 files changed, 62 insertions(+), 7 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index d5495d87f399..12f2f725a945 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -89,6 +89,7 @@ #include #include #include +#include #ifdef CONFIG_INET #include #endif @@ -1168,6 +1169,47 @@ static void packet_increment_head(struct packet_ring_buffer *buff) buff->head = buff->head != buff->frame_max ? buff->head+1 : 0; } +static void packet_inc_pending(struct packet_ring_buffer *rb) +{ + this_cpu_inc(*rb->pending_refcnt); +} + +static void packet_dec_pending(struct packet_ring_buffer *rb) +{ + this_cpu_dec(*rb->pending_refcnt); +} + +static unsigned int packet_read_pending(const struct packet_ring_buffer *rb) +{ + unsigned int refcnt = 0; + int cpu; + + /* We don't use pending refcount in rx_ring. */ + if (rb->pending_refcnt == NULL) + return 0; + + for_each_possible_cpu(cpu) + refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu); + + return refcnt; +} + +static int packet_alloc_pending(struct packet_sock *po) +{ + po->rx_ring.pending_refcnt = NULL; + + po->tx_ring.pending_refcnt = alloc_percpu(unsigned int); + if (unlikely(po->tx_ring.pending_refcnt == NULL)) + return -ENOBUFS; + + return 0; +} + +static void packet_free_pending(struct packet_sock *po) +{ + free_percpu(po->tx_ring.pending_refcnt); +} + static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb) { struct sock *sk = &po->sk; @@ -2014,8 +2056,7 @@ static void tpacket_destruct_skb(struct sk_buff *skb) __u32 ts; ph = skb_shinfo(skb)->destructor_arg; - BUG_ON(atomic_read(&po->tx_ring.pending) == 0); - atomic_dec(&po->tx_ring.pending); + packet_dec_pending(&po->tx_ring); ts = __packet_set_timestamp(po, ph, skb); __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); @@ -2236,7 +2277,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) skb_set_queue_mapping(skb, packet_pick_tx_queue(dev)); skb->destructor = tpacket_destruct_skb; __packet_set_status(po, ph, TP_STATUS_SENDING); - atomic_inc(&po->tx_ring.pending); + packet_inc_pending(&po->tx_ring); status = TP_STATUS_SEND_REQUEST; err = po->xmit(skb); @@ -2256,8 +2297,14 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) } packet_increment_head(&po->tx_ring); len_sum += tp_len; - } while (likely((ph != NULL) || (need_wait && - atomic_read(&po->tx_ring.pending)))); + } while (likely((ph != NULL) || + /* Note: packet_read_pending() might be slow if we have + * to call it as it's per_cpu variable, but in fast-path + * we already short-circuit the loop with the first + * condition, and luckily don't have to go that path + * anyway. + */ + (need_wait && packet_read_pending(&po->tx_ring)))); err = len_sum; goto out_put; @@ -2556,6 +2603,7 @@ static int packet_release(struct socket *sock) /* Purge queues */ skb_queue_purge(&sk->sk_receive_queue); + packet_free_pending(po); sk_refcnt_debug_release(sk); sock_put(sk); @@ -2717,6 +2765,10 @@ static int packet_create(struct net *net, struct socket *sock, int protocol, po->num = proto; po->xmit = dev_queue_xmit; + err = packet_alloc_pending(po); + if (err) + goto out2; + packet_cached_dev_reset(po); sk->sk_destruct = packet_sock_destruct; @@ -2749,6 +2801,8 @@ static int packet_create(struct net *net, struct socket *sock, int protocol, preempt_enable(); return 0; +out2: + sk_free(sk); out: return err; } @@ -3676,7 +3730,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, if (!closing) { if (atomic_read(&po->mapped)) goto out; - if (atomic_read(&rb->pending)) + if (packet_read_pending(rb)) goto out; } diff --git a/net/packet/diag.c b/net/packet/diag.c index a9584a2f6d69..533ce4ff108a 100644 --- a/net/packet/diag.c +++ b/net/packet/diag.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/net/packet/internal.h b/net/packet/internal.h index 0a87d7b36c9e..eb9580a6b25f 100644 --- a/net/packet/internal.h +++ b/net/packet/internal.h @@ -64,7 +64,7 @@ struct packet_ring_buffer { unsigned int pg_vec_pages; unsigned int pg_vec_len; - atomic_t pending; + unsigned int __percpu *pending_refcnt; struct tpacket_kbdq_core prb_bdqc; }; -- cgit v1.2.3 From 342dfc306fb32155314dad277f3c3686b83fb9f1 Mon Sep 17 00:00:00 2001 From: Steffen Hurrle Date: Fri, 17 Jan 2014 22:53:15 +0100 Subject: net: add build-time checks for msg->msg_name size This is a follow-up patch to f3d3342602f8bc ("net: rework recvmsg handler msg_name and msg_namelen logic"). DECLARE_SOCKADDR validates that the structure we use for writing the name information to is not larger than the buffer which is reserved for msg->msg_name (which is 128 bytes). Also use DECLARE_SOCKADDR consistently in sendmsg code paths. Signed-off-by: Steffen Hurrle Suggested-by: Hannes Frederic Sowa Acked-by: Hannes Frederic Sowa Signed-off-by: David S. Miller --- drivers/isdn/mISDN/socket.c | 5 ++--- net/appletalk/ddp.c | 4 ++-- net/ax25/af_ax25.c | 4 ++-- net/bluetooth/l2cap_sock.c | 2 +- net/can/bcm.c | 4 ++-- net/can/raw.c | 4 ++-- net/decnet/af_decnet.c | 3 ++- net/ieee802154/dgram.c | 4 +--- net/ipv4/ip_sockglue.c | 3 +-- net/ipv4/ping.c | 7 +++---- net/ipv4/raw.c | 4 ++-- net/ipv4/udp.c | 4 ++-- net/ipv6/datagram.c | 6 ++---- net/ipv6/ping.c | 2 +- net/ipv6/raw.c | 4 ++-- net/ipv6/udp.c | 6 ++---- net/ipx/af_ipx.c | 4 ++-- net/irda/af_irda.c | 2 +- net/l2tp/l2tp_ip.c | 4 ++-- net/l2tp/l2tp_ip6.c | 5 ++--- net/llc/af_llc.c | 4 ++-- net/netlink/af_netlink.c | 4 ++-- net/netrom/af_netrom.c | 4 ++-- net/nfc/llcp_sock.c | 8 ++++---- net/packet/af_packet.c | 7 ++++--- net/phonet/datagram.c | 6 +++--- net/rds/recv.c | 3 +-- net/rds/send.c | 2 +- net/rose/af_rose.c | 5 +++-- net/rxrpc/ar-output.c | 4 ++-- net/tipc/socket.c | 6 +++--- net/unix/af_unix.c | 4 ++-- net/vmw_vsock/vmci_transport.c | 4 +--- net/x25/af_x25.c | 4 ++-- 34 files changed, 68 insertions(+), 78 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/drivers/isdn/mISDN/socket.c b/drivers/isdn/mISDN/socket.c index 5cefb479c707..1be82284cf9d 100644 --- a/drivers/isdn/mISDN/socket.c +++ b/drivers/isdn/mISDN/socket.c @@ -135,7 +135,7 @@ mISDN_sock_recvmsg(struct kiocb *iocb, struct socket *sock, return err; if (msg->msg_name) { - struct sockaddr_mISDN *maddr = msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_mISDN *, maddr, msg->msg_name); maddr->family = AF_ISDN; maddr->dev = _pms(sk)->dev->id; @@ -179,7 +179,6 @@ mISDN_sock_sendmsg(struct kiocb *iocb, struct socket *sock, struct sock *sk = sock->sk; struct sk_buff *skb; int err = -ENOMEM; - struct sockaddr_mISDN *maddr; if (*debug & DEBUG_SOCKET) printk(KERN_DEBUG "%s: len %d flags %x ch %d proto %x\n", @@ -214,7 +213,7 @@ mISDN_sock_sendmsg(struct kiocb *iocb, struct socket *sock, if (msg->msg_namelen >= sizeof(struct sockaddr_mISDN)) { /* if we have a address, we use it */ - maddr = (struct sockaddr_mISDN *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_mISDN *, maddr, msg->msg_name); mISDN_HEAD_ID(skb) = maddr->channel; } else { /* use default for L2 messages */ if ((sk->sk_protocol == ISDN_P_LAPD_TE) || diff --git a/net/appletalk/ddp.c b/net/appletalk/ddp.c index 7d424ac6e760..02806c6b2ff3 100644 --- a/net/appletalk/ddp.c +++ b/net/appletalk/ddp.c @@ -1566,7 +1566,7 @@ static int atalk_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr { struct sock *sk = sock->sk; struct atalk_sock *at = at_sk(sk); - struct sockaddr_at *usat = (struct sockaddr_at *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_at *, usat, msg->msg_name); int flags = msg->msg_flags; int loopback = 0; struct sockaddr_at local_satalk, gsat; @@ -1764,7 +1764,7 @@ static int atalk_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr err = skb_copy_datagram_iovec(skb, offset, msg->msg_iov, copied); if (!err && msg->msg_name) { - struct sockaddr_at *sat = msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_at *, sat, msg->msg_name); sat->sat_family = AF_APPLETALK; sat->sat_port = ddp->deh_sport; sat->sat_addr.s_node = ddp->deh_snode; diff --git a/net/ax25/af_ax25.c b/net/ax25/af_ax25.c index 7bb1605bdfd9..c35c3f48fc0f 100644 --- a/net/ax25/af_ax25.c +++ b/net/ax25/af_ax25.c @@ -1435,7 +1435,7 @@ out: static int ax25_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg, size_t len) { - struct sockaddr_ax25 *usax = (struct sockaddr_ax25 *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_ax25 *, usax, msg->msg_name); struct sock *sk = sock->sk; struct sockaddr_ax25 sax; struct sk_buff *skb; @@ -1640,7 +1640,7 @@ static int ax25_recvmsg(struct kiocb *iocb, struct socket *sock, ax25_digi digi; ax25_address src; const unsigned char *mac = skb_mac_header(skb); - struct sockaddr_ax25 *sax = msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_ax25 *, sax, msg->msg_name); memset(sax, 0, sizeof(struct full_sockaddr_ax25)); ax25_addr_parse(mac + 1, skb->data - mac - 1, &src, NULL, diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 20ef748b2906..d58f76bcebd1 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -1413,7 +1413,7 @@ static void l2cap_sock_destruct(struct sock *sk) static void l2cap_skb_msg_name(struct sk_buff *skb, void *msg_name, int *msg_namelen) { - struct sockaddr_l2 *la = (struct sockaddr_l2 *) msg_name; + DECLARE_SOCKADDR(struct sockaddr_l2 *, la, msg_name); memset(la, 0, sizeof(struct sockaddr_l2)); la->l2_family = AF_BLUETOOTH; diff --git a/net/can/bcm.c b/net/can/bcm.c index 46f20bfafc0e..3fc737b214c7 100644 --- a/net/can/bcm.c +++ b/net/can/bcm.c @@ -1256,8 +1256,7 @@ static int bcm_sendmsg(struct kiocb *iocb, struct socket *sock, if (!ifindex && msg->msg_name) { /* no bound device as default => check msg_name */ - struct sockaddr_can *addr = - (struct sockaddr_can *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_can *, addr, msg->msg_name); if (msg->msg_namelen < sizeof(*addr)) return -EINVAL; @@ -1568,6 +1567,7 @@ static int bcm_recvmsg(struct kiocb *iocb, struct socket *sock, sock_recv_ts_and_drops(msg, sk, skb); if (msg->msg_name) { + __sockaddr_check_size(sizeof(struct sockaddr_can)); msg->msg_namelen = sizeof(struct sockaddr_can); memcpy(msg->msg_name, skb->cb, msg->msg_namelen); } diff --git a/net/can/raw.c b/net/can/raw.c index 641e1c895123..07d72d852324 100644 --- a/net/can/raw.c +++ b/net/can/raw.c @@ -675,8 +675,7 @@ static int raw_sendmsg(struct kiocb *iocb, struct socket *sock, int err; if (msg->msg_name) { - struct sockaddr_can *addr = - (struct sockaddr_can *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_can *, addr, msg->msg_name); if (msg->msg_namelen < sizeof(*addr)) return -EINVAL; @@ -775,6 +774,7 @@ static int raw_recvmsg(struct kiocb *iocb, struct socket *sock, sock_recv_ts_and_drops(msg, sk, skb); if (msg->msg_name) { + __sockaddr_check_size(sizeof(struct sockaddr_can)); msg->msg_namelen = sizeof(struct sockaddr_can); memcpy(msg->msg_name, skb->cb, msg->msg_namelen); } diff --git a/net/decnet/af_decnet.c b/net/decnet/af_decnet.c index dd4d506ef923..2954dcbca832 100644 --- a/net/decnet/af_decnet.c +++ b/net/decnet/af_decnet.c @@ -1808,6 +1808,7 @@ out: rv = (flags & MSG_PEEK) ? -sk->sk_err : sock_error(sk); if ((rv >= 0) && msg->msg_name) { + __sockaddr_check_size(sizeof(struct sockaddr_dn)); memcpy(msg->msg_name, &scp->peer, sizeof(struct sockaddr_dn)); msg->msg_namelen = sizeof(struct sockaddr_dn); } @@ -1914,7 +1915,7 @@ static int dn_sendmsg(struct kiocb *iocb, struct socket *sock, int err = 0; size_t sent = 0; int addr_len = msg->msg_namelen; - struct sockaddr_dn *addr = (struct sockaddr_dn *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_dn *, addr, msg->msg_name); struct sk_buff *skb = NULL; struct dn_skb_cb *cb; size_t len; diff --git a/net/ieee802154/dgram.c b/net/ieee802154/dgram.c index 1865fdf5a5a5..1846c1fe0d06 100644 --- a/net/ieee802154/dgram.c +++ b/net/ieee802154/dgram.c @@ -291,9 +291,7 @@ static int dgram_recvmsg(struct kiocb *iocb, struct sock *sk, size_t copied = 0; int err = -EOPNOTSUPP; struct sk_buff *skb; - struct sockaddr_ieee802154 *saddr; - - saddr = (struct sockaddr_ieee802154 *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_ieee802154 *, saddr, msg->msg_name); skb = skb_recv_datagram(sk, flags, noblock, &err); if (!skb) diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c index a9fc435dc89f..22f15eb1c260 100644 --- a/net/ipv4/ip_sockglue.c +++ b/net/ipv4/ip_sockglue.c @@ -390,7 +390,7 @@ int ip_recv_error(struct sock *sk, struct msghdr *msg, int len, int *addr_len) { struct sock_exterr_skb *serr; struct sk_buff *skb, *skb2; - struct sockaddr_in *sin; + DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name); struct { struct sock_extended_err ee; struct sockaddr_in offender; @@ -416,7 +416,6 @@ int ip_recv_error(struct sock *sk, struct msghdr *msg, int len, int *addr_len) serr = SKB_EXT_ERR(skb); - sin = (struct sockaddr_in *)msg->msg_name; if (sin) { sin->sin_family = AF_INET; sin->sin_addr.s_addr = *(__be32 *)(skb_network_header(skb) + diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index cae5262a337c..e09e8839d622 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -700,7 +700,7 @@ static int ping_v4_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *m */ if (msg->msg_name) { - struct sockaddr_in *usin = (struct sockaddr_in *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name); if (msg->msg_namelen < sizeof(*usin)) return -EINVAL; if (usin->sin_family != AF_INET) @@ -873,7 +873,7 @@ int ping_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, /* Copy the address and add cmsg data. */ if (family == AF_INET) { - struct sockaddr_in *sin = (struct sockaddr_in *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name); if (sin) { sin->sin_family = AF_INET; @@ -890,8 +890,7 @@ int ping_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, } else if (family == AF_INET6) { struct ipv6_pinfo *np = inet6_sk(sk); struct ipv6hdr *ip6 = ipv6_hdr(skb); - struct sockaddr_in6 *sin6 = - (struct sockaddr_in6 *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name); if (sin6) { sin6->sin6_family = AF_INET6; diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c index 81e6cfd5a365..c04518f4850a 100644 --- a/net/ipv4/raw.c +++ b/net/ipv4/raw.c @@ -493,7 +493,7 @@ static int raw_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, */ if (msg->msg_namelen) { - struct sockaddr_in *usin = (struct sockaddr_in *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name); err = -EINVAL; if (msg->msg_namelen < sizeof(*usin)) goto out; @@ -690,7 +690,7 @@ static int raw_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, struct inet_sock *inet = inet_sk(sk); size_t copied = 0; int err = -EOPNOTSUPP; - struct sockaddr_in *sin = (struct sockaddr_in *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name); struct sk_buff *skb; if (flags & MSG_OOB) diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 3d3141fd0580..77bd16fa9f34 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -902,7 +902,7 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, * Get and verify the address. */ if (msg->msg_name) { - struct sockaddr_in *usin = (struct sockaddr_in *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name); if (msg->msg_namelen < sizeof(*usin)) return -EINVAL; if (usin->sin_family != AF_INET) { @@ -1226,7 +1226,7 @@ int udp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, size_t len, int noblock, int flags, int *addr_len) { struct inet_sock *inet = inet_sk(sk); - struct sockaddr_in *sin = (struct sockaddr_in *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name); struct sk_buff *skb; unsigned int ulen, copied; int peeked, off = 0; diff --git a/net/ipv6/datagram.c b/net/ipv6/datagram.c index 6983058942ea..bce73cbd203a 100644 --- a/net/ipv6/datagram.c +++ b/net/ipv6/datagram.c @@ -322,7 +322,7 @@ int ipv6_recv_error(struct sock *sk, struct msghdr *msg, int len, int *addr_len) struct ipv6_pinfo *np = inet6_sk(sk); struct sock_exterr_skb *serr; struct sk_buff *skb, *skb2; - struct sockaddr_in6 *sin; + DECLARE_SOCKADDR(struct sockaddr_in6 *, sin, msg->msg_name); struct { struct sock_extended_err ee; struct sockaddr_in6 offender; @@ -348,7 +348,6 @@ int ipv6_recv_error(struct sock *sk, struct msghdr *msg, int len, int *addr_len) serr = SKB_EXT_ERR(skb); - sin = (struct sockaddr_in6 *)msg->msg_name; if (sin) { const unsigned char *nh = skb_network_header(skb); sin->sin6_family = AF_INET6; @@ -429,8 +428,8 @@ int ipv6_recv_rxpmtu(struct sock *sk, struct msghdr *msg, int len, { struct ipv6_pinfo *np = inet6_sk(sk); struct sk_buff *skb; - struct sockaddr_in6 *sin; struct ip6_mtuinfo mtu_info; + DECLARE_SOCKADDR(struct sockaddr_in6 *, sin, msg->msg_name); int err; int copied; @@ -452,7 +451,6 @@ int ipv6_recv_rxpmtu(struct sock *sk, struct msghdr *msg, int len, memcpy(&mtu_info, IP6CBMTU(skb), sizeof(mtu_info)); - sin = (struct sockaddr_in6 *)msg->msg_name; if (sin) { sin->sin6_family = AF_INET6; sin->sin6_flowinfo = 0; diff --git a/net/ipv6/ping.c b/net/ipv6/ping.c index 15d23b8c2129..cd71f3a540be 100644 --- a/net/ipv6/ping.c +++ b/net/ipv6/ping.c @@ -103,7 +103,7 @@ int ping_v6_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, return err; if (msg->msg_name) { - struct sockaddr_in6 *u = (struct sockaddr_in6 *) msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in6 *, u, msg->msg_name); if (msg->msg_namelen < sizeof(struct sockaddr_in6) || u->sin6_family != AF_INET6) { return -EINVAL; diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c index 5f10b7ea7ccc..b4523117aeae 100644 --- a/net/ipv6/raw.c +++ b/net/ipv6/raw.c @@ -457,7 +457,7 @@ static int rawv6_recvmsg(struct kiocb *iocb, struct sock *sk, int noblock, int flags, int *addr_len) { struct ipv6_pinfo *np = inet6_sk(sk); - struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name); struct sk_buff *skb; size_t copied; int err; @@ -734,7 +734,7 @@ static int rawv6_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, size_t len) { struct ipv6_txoptions opt_space; - struct sockaddr_in6 * sin6 = (struct sockaddr_in6 *) msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name); struct in6_addr *daddr, *final_p, final; struct inet_sock *inet = inet_sk(sk); struct ipv6_pinfo *np = inet6_sk(sk); diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index fa9d988f4012..233c3ab6aee4 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -460,9 +460,7 @@ try_again: /* Copy the address. */ if (msg->msg_name) { - struct sockaddr_in6 *sin6; - - sin6 = (struct sockaddr_in6 *) msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name); sin6->sin6_family = AF_INET6; sin6->sin6_port = udp_hdr(skb)->source; sin6->sin6_flowinfo = 0; @@ -1041,7 +1039,7 @@ int udpv6_sendmsg(struct kiocb *iocb, struct sock *sk, struct udp_sock *up = udp_sk(sk); struct inet_sock *inet = inet_sk(sk); struct ipv6_pinfo *np = inet6_sk(sk); - struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name); struct in6_addr *daddr, *final_p, final; struct ipv6_txoptions *opt = NULL; struct ip6_flowlabel *flowlabel = NULL; diff --git a/net/ipx/af_ipx.c b/net/ipx/af_ipx.c index e096025b477f..994e28bfb32e 100644 --- a/net/ipx/af_ipx.c +++ b/net/ipx/af_ipx.c @@ -1707,7 +1707,7 @@ static int ipx_sendmsg(struct kiocb *iocb, struct socket *sock, { struct sock *sk = sock->sk; struct ipx_sock *ipxs = ipx_sk(sk); - struct sockaddr_ipx *usipx = (struct sockaddr_ipx *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_ipx *, usipx, msg->msg_name); struct sockaddr_ipx local_sipx; int rc = -EINVAL; int flags = msg->msg_flags; @@ -1774,7 +1774,7 @@ static int ipx_recvmsg(struct kiocb *iocb, struct socket *sock, { struct sock *sk = sock->sk; struct ipx_sock *ipxs = ipx_sk(sk); - struct sockaddr_ipx *sipx = (struct sockaddr_ipx *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_ipx *, sipx, msg->msg_name); struct ipxhdr *ipx = NULL; struct sk_buff *skb; int copied, rc; diff --git a/net/irda/af_irda.c b/net/irda/af_irda.c index 73baf9b346b6..54747c25c86c 100644 --- a/net/irda/af_irda.c +++ b/net/irda/af_irda.c @@ -1652,7 +1652,7 @@ static int irda_sendmsg_ultra(struct kiocb *iocb, struct socket *sock, /* Check if an address was specified with sendto. Jean II */ if (msg->msg_name) { - struct sockaddr_irda *addr = (struct sockaddr_irda *) msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_irda *, addr, msg->msg_name); err = -EINVAL; /* Check address, extract pid. Jean II */ if (msg->msg_namelen < sizeof(*addr)) diff --git a/net/l2tp/l2tp_ip.c b/net/l2tp/l2tp_ip.c index da1a1cee1a08..0b44d855269c 100644 --- a/net/l2tp/l2tp_ip.c +++ b/net/l2tp/l2tp_ip.c @@ -403,7 +403,7 @@ static int l2tp_ip_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *m /* Get and verify the address. */ if (msg->msg_name) { - struct sockaddr_l2tpip *lip = (struct sockaddr_l2tpip *) msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_l2tpip *, lip, msg->msg_name); rc = -EINVAL; if (msg->msg_namelen < sizeof(*lip)) goto out; @@ -512,7 +512,7 @@ static int l2tp_ip_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *m struct inet_sock *inet = inet_sk(sk); size_t copied = 0; int err = -EOPNOTSUPP; - struct sockaddr_in *sin = (struct sockaddr_in *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name); struct sk_buff *skb; if (flags & MSG_OOB) diff --git a/net/l2tp/l2tp_ip6.c b/net/l2tp/l2tp_ip6.c index 29487a8f7fa0..aab5f74e91e6 100644 --- a/net/l2tp/l2tp_ip6.c +++ b/net/l2tp/l2tp_ip6.c @@ -481,8 +481,7 @@ static int l2tp_ip6_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, size_t len) { struct ipv6_txoptions opt_space; - struct sockaddr_l2tpip6 *lsa = - (struct sockaddr_l2tpip6 *) msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_l2tpip6 *, lsa, msg->msg_name); struct in6_addr *daddr, *final_p, final; struct ipv6_pinfo *np = inet6_sk(sk); struct ipv6_txoptions *opt = NULL; @@ -652,7 +651,7 @@ static int l2tp_ip6_recvmsg(struct kiocb *iocb, struct sock *sk, int flags, int *addr_len) { struct ipv6_pinfo *np = inet6_sk(sk); - struct sockaddr_l2tpip6 *lsa = (struct sockaddr_l2tpip6 *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_l2tpip6 *, lsa, msg->msg_name); size_t copied = 0; int err = -EOPNOTSUPP; struct sk_buff *skb; diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c index c71b699eb555..0080d2b0a8ae 100644 --- a/net/llc/af_llc.c +++ b/net/llc/af_llc.c @@ -707,7 +707,7 @@ out: static int llc_ui_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg, size_t len, int flags) { - struct sockaddr_llc *uaddr = (struct sockaddr_llc *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_llc *, uaddr, msg->msg_name); const int nonblock = flags & MSG_DONTWAIT; struct sk_buff *skb = NULL; struct sock *sk = sock->sk; @@ -884,7 +884,7 @@ static int llc_ui_sendmsg(struct kiocb *iocb, struct socket *sock, { struct sock *sk = sock->sk; struct llc_sock *llc = llc_sk(sk); - struct sockaddr_llc *addr = (struct sockaddr_llc *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_llc *, addr, msg->msg_name); int flags = msg->msg_flags; int noblock = flags & MSG_DONTWAIT; struct sk_buff *skb; diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 34a656d90175..fdf51353cf78 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -2222,7 +2222,7 @@ static int netlink_sendmsg(struct kiocb *kiocb, struct socket *sock, struct sock_iocb *siocb = kiocb_to_siocb(kiocb); struct sock *sk = sock->sk; struct netlink_sock *nlk = nlk_sk(sk); - struct sockaddr_nl *addr = msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_nl *, addr, msg->msg_name); u32 dst_portid; u32 dst_group; struct sk_buff *skb; @@ -2353,7 +2353,7 @@ static int netlink_recvmsg(struct kiocb *kiocb, struct socket *sock, err = skb_copy_datagram_iovec(data_skb, 0, msg->msg_iov, copied); if (msg->msg_name) { - struct sockaddr_nl *addr = (struct sockaddr_nl *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_nl *, addr, msg->msg_name); addr->nl_family = AF_NETLINK; addr->nl_pad = 0; addr->nl_pid = NETLINK_CB(skb).portid; diff --git a/net/netrom/af_netrom.c b/net/netrom/af_netrom.c index 53c19a35fc6d..b74aa0755521 100644 --- a/net/netrom/af_netrom.c +++ b/net/netrom/af_netrom.c @@ -1028,7 +1028,7 @@ static int nr_sendmsg(struct kiocb *iocb, struct socket *sock, { struct sock *sk = sock->sk; struct nr_sock *nr = nr_sk(sk); - struct sockaddr_ax25 *usax = (struct sockaddr_ax25 *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_ax25 *, usax, msg->msg_name); int err; struct sockaddr_ax25 sax; struct sk_buff *skb; @@ -1137,7 +1137,7 @@ static int nr_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg, size_t size, int flags) { struct sock *sk = sock->sk; - struct sockaddr_ax25 *sax = (struct sockaddr_ax25 *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_ax25 *, sax, msg->msg_name); size_t copied; struct sk_buff *skb; int er; diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c index 4a53bb58a463..51f077a92fa9 100644 --- a/net/nfc/llcp_sock.c +++ b/net/nfc/llcp_sock.c @@ -769,8 +769,8 @@ static int llcp_sock_sendmsg(struct kiocb *iocb, struct socket *sock, lock_sock(sk); if (sk->sk_type == SOCK_DGRAM) { - struct sockaddr_nfc_llcp *addr = - (struct sockaddr_nfc_llcp *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_nfc_llcp *, addr, + msg->msg_name); if (msg->msg_namelen < sizeof(*addr)) { release_sock(sk); @@ -842,8 +842,8 @@ static int llcp_sock_recvmsg(struct kiocb *iocb, struct socket *sock, if (sk->sk_type == SOCK_DGRAM && msg->msg_name) { struct nfc_llcp_ui_cb *ui_cb = nfc_llcp_ui_skb_cb(skb); - struct sockaddr_nfc_llcp *sockaddr = - (struct sockaddr_nfc_llcp *) msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_nfc_llcp *, sockaddr, + msg->msg_name); msg->msg_namelen = sizeof(struct sockaddr_nfc_llcp); diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 12f2f725a945..d711ecbbb9d3 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1584,7 +1584,7 @@ static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock, struct msghdr *msg, size_t len) { struct sock *sk = sock->sk; - struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name); struct sk_buff *skb = NULL; struct net_device *dev; __be16 proto = 0; @@ -2196,7 +2196,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) __be16 proto; int err, reserve = 0; void *ph; - struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name); bool need_wait = !(msg->msg_flags & MSG_DONTWAIT); int tp_len, size_max; unsigned char *addr; @@ -2346,7 +2346,7 @@ static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad, static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) { struct sock *sk = sock->sk; - struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name); struct sk_buff *skb; struct net_device *dev; __be16 proto; @@ -2922,6 +2922,7 @@ static int packet_recvmsg(struct kiocb *iocb, struct socket *sock, * in, we fill it in now. */ if (sock->type == SOCK_PACKET) { + __sockaddr_check_size(sizeof(struct sockaddr_pkt)); msg->msg_namelen = sizeof(struct sockaddr_pkt); } else { struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll; diff --git a/net/phonet/datagram.c b/net/phonet/datagram.c index 38946b26e471..290352c0e6b4 100644 --- a/net/phonet/datagram.c +++ b/net/phonet/datagram.c @@ -86,7 +86,7 @@ static int pn_init(struct sock *sk) static int pn_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, size_t len) { - struct sockaddr_pn *target; + DECLARE_SOCKADDR(struct sockaddr_pn *, target, msg->msg_name); struct sk_buff *skb; int err; @@ -94,13 +94,12 @@ static int pn_sendmsg(struct kiocb *iocb, struct sock *sk, MSG_CMSG_COMPAT)) return -EOPNOTSUPP; - if (msg->msg_name == NULL) + if (target == NULL) return -EDESTADDRREQ; if (msg->msg_namelen < sizeof(struct sockaddr_pn)) return -EINVAL; - target = (struct sockaddr_pn *)msg->msg_name; if (target->spn_family != AF_PHONET) return -EAFNOSUPPORT; @@ -160,6 +159,7 @@ static int pn_recvmsg(struct kiocb *iocb, struct sock *sk, rval = (flags & MSG_TRUNC) ? skb->len : copylen; if (msg->msg_name != NULL) { + __sockaddr_check_size(sizeof(sa)); memcpy(msg->msg_name, &sa, sizeof(sa)); *addr_len = sizeof(sa); } diff --git a/net/rds/recv.c b/net/rds/recv.c index de339b24ca14..bd82522534fc 100644 --- a/net/rds/recv.c +++ b/net/rds/recv.c @@ -402,7 +402,7 @@ int rds_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg, struct rds_sock *rs = rds_sk_to_rs(sk); long timeo; int ret = 0, nonblock = msg_flags & MSG_DONTWAIT; - struct sockaddr_in *sin; + DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name); struct rds_incoming *inc = NULL; /* udp_recvmsg()->sock_recvtimeo() gets away without locking too.. */ @@ -479,7 +479,6 @@ int rds_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg, rds_stats_inc(s_recv_delivered); - sin = (struct sockaddr_in *)msg->msg_name; if (sin) { sin->sin_family = AF_INET; sin->sin_port = inc->i_hdr.h_sport; diff --git a/net/rds/send.c b/net/rds/send.c index 88eace57dd6b..a82fb660ec00 100644 --- a/net/rds/send.c +++ b/net/rds/send.c @@ -922,7 +922,7 @@ int rds_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg, { struct sock *sk = sock->sk; struct rds_sock *rs = rds_sk_to_rs(sk); - struct sockaddr_in *usin = (struct sockaddr_in *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name); __be32 daddr; __be16 dport; struct rds_message *rm = NULL; diff --git a/net/rose/af_rose.c b/net/rose/af_rose.c index d080eb4b0d29..c2cca2ee6aef 100644 --- a/net/rose/af_rose.c +++ b/net/rose/af_rose.c @@ -1051,7 +1051,7 @@ static int rose_sendmsg(struct kiocb *iocb, struct socket *sock, { struct sock *sk = sock->sk; struct rose_sock *rose = rose_sk(sk); - struct sockaddr_rose *usrose = (struct sockaddr_rose *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_rose *, usrose, msg->msg_name); int err; struct full_sockaddr_rose srose; struct sk_buff *skb; @@ -1253,7 +1253,8 @@ static int rose_recvmsg(struct kiocb *iocb, struct socket *sock, if (msg->msg_name) { struct sockaddr_rose *srose; - struct full_sockaddr_rose *full_srose = msg->msg_name; + DECLARE_SOCKADDR(struct full_sockaddr_rose *, full_srose, + msg->msg_name); memset(msg->msg_name, 0, sizeof(struct full_sockaddr_rose)); srose = msg->msg_name; diff --git a/net/rxrpc/ar-output.c b/net/rxrpc/ar-output.c index e1ac183d50bb..d0e8f1c1898a 100644 --- a/net/rxrpc/ar-output.c +++ b/net/rxrpc/ar-output.c @@ -152,8 +152,8 @@ int rxrpc_client_sendmsg(struct kiocb *iocb, struct rxrpc_sock *rx, if (trans) { service_id = rx->service_id; if (msg->msg_name) { - struct sockaddr_rxrpc *srx = - (struct sockaddr_rxrpc *) msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx, + msg->msg_name); service_id = htons(srx->srx_service); } key = rx->key; diff --git a/net/tipc/socket.c b/net/tipc/socket.c index eab17eb9ca1d..aab4948f0aff 100644 --- a/net/tipc/socket.c +++ b/net/tipc/socket.c @@ -608,7 +608,7 @@ static int send_msg(struct kiocb *iocb, struct socket *sock, { struct sock *sk = sock->sk; struct tipc_port *tport = tipc_sk_port(sk); - struct sockaddr_tipc *dest = (struct sockaddr_tipc *)m->msg_name; + DECLARE_SOCKADDR(struct sockaddr_tipc *, dest, m->msg_name); int needs_conn; long timeo; int res = -EINVAL; @@ -736,7 +736,7 @@ static int send_packet(struct kiocb *iocb, struct socket *sock, { struct sock *sk = sock->sk; struct tipc_port *tport = tipc_sk_port(sk); - struct sockaddr_tipc *dest = (struct sockaddr_tipc *)m->msg_name; + DECLARE_SOCKADDR(struct sockaddr_tipc *, dest, m->msg_name); int res = -EINVAL; long timeo; @@ -906,7 +906,7 @@ static int auto_connect(struct socket *sock, struct tipc_msg *msg) */ static void set_orig_addr(struct msghdr *m, struct tipc_msg *msg) { - struct sockaddr_tipc *addr = (struct sockaddr_tipc *)m->msg_name; + DECLARE_SOCKADDR(struct sockaddr_tipc *, addr, m->msg_name); if (addr) { addr->family = AF_TIPC; diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 800ca61758ff..29fc8bee9702 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -1450,7 +1450,7 @@ static int unix_dgram_sendmsg(struct kiocb *kiocb, struct socket *sock, struct sock *sk = sock->sk; struct net *net = sock_net(sk); struct unix_sock *u = unix_sk(sk); - struct sockaddr_un *sunaddr = msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_un *, sunaddr, msg->msg_name); struct sock *other = NULL; int namelen = 0; /* fake GCC */ int err; @@ -1912,7 +1912,7 @@ static int unix_stream_recvmsg(struct kiocb *iocb, struct socket *sock, struct scm_cookie tmp_scm; struct sock *sk = sock->sk; struct unix_sock *u = unix_sk(sk); - struct sockaddr_un *sunaddr = msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_un *, sunaddr, msg->msg_name); int copied = 0; int check_creds = 0; int target; diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c index 687360da62d9..9bb63ffec4f2 100644 --- a/net/vmw_vsock/vmci_transport.c +++ b/net/vmw_vsock/vmci_transport.c @@ -1779,10 +1779,8 @@ static int vmci_transport_dgram_dequeue(struct kiocb *kiocb, goto out; if (msg->msg_name) { - struct sockaddr_vm *vm_addr; - /* Provide the address of the sender. */ - vm_addr = (struct sockaddr_vm *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_vm *, vm_addr, msg->msg_name); vsock_addr_init(vm_addr, dg->src.context, dg->src.resource); msg->msg_namelen = sizeof(*vm_addr); } diff --git a/net/x25/af_x25.c b/net/x25/af_x25.c index c8a8297cd4b8..6177479c7de9 100644 --- a/net/x25/af_x25.c +++ b/net/x25/af_x25.c @@ -1082,7 +1082,7 @@ static int x25_sendmsg(struct kiocb *iocb, struct socket *sock, { struct sock *sk = sock->sk; struct x25_sock *x25 = x25_sk(sk); - struct sockaddr_x25 *usx25 = (struct sockaddr_x25 *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_x25 *, usx25, msg->msg_name); struct sockaddr_x25 sx25; struct sk_buff *skb; unsigned char *asmptr; @@ -1258,7 +1258,7 @@ static int x25_recvmsg(struct kiocb *iocb, struct socket *sock, { struct sock *sk = sock->sk; struct x25_sock *x25 = x25_sk(sk); - struct sockaddr_x25 *sx25 = (struct sockaddr_x25 *)msg->msg_name; + DECLARE_SOCKADDR(struct sockaddr_x25 *, sx25, msg->msg_name); size_t copied; int qbit, header_len; struct sk_buff *skb; -- cgit v1.2.3 From f0d4eb29d18f732ed80cea4bf0f6bbee9c03ed62 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Sun, 19 Jan 2014 11:46:53 +0100 Subject: packet: fix a couple of cppcheck warnings Doesn't bring much, but also doesn't hurt us to fix 'em: 1) In tpacket_rcv() flush dcache page we can restirct the scope for start and end and remove one layer of indent. 2) In tpacket_destruct_skb() we can restirct the scope for ph. 3) In alloc_one_pg_vec_page() we can remove the NULL assignment and change spacing a bit. Signed-off-by: Daniel Borkmann Signed-off-by: David S. Miller --- net/packet/af_packet.c | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index d711ecbbb9d3..59fb3db2e713 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2009,19 +2009,20 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, sll->sll_ifindex = dev->ifindex; smp_mb(); + #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1 - { + if (po->tp_version <= TPACKET_V2) { u8 *start, *end; - if (po->tp_version <= TPACKET_V2) { - end = (u8 *)PAGE_ALIGN((unsigned long)h.raw - + macoff + snaplen); - for (start = h.raw; start < end; start += PAGE_SIZE) - flush_dcache_page(pgv_to_page(start)); - } - smp_wmb(); + end = (u8 *) PAGE_ALIGN((unsigned long) h.raw + + macoff + snaplen); + + for (start = h.raw; start < end; start += PAGE_SIZE) + flush_dcache_page(pgv_to_page(start)); } + smp_wmb(); #endif + if (po->tp_version <= TPACKET_V2) __packet_set_status(po, h.raw, status); else @@ -2050,9 +2051,9 @@ ring_is_full: static void tpacket_destruct_skb(struct sk_buff *skb) { struct packet_sock *po = pkt_sk(skb->sk); - void *ph; if (likely(po->tx_ring.pg_vec)) { + void *ph; __u32 ts; ph = skb_shinfo(skb)->destructor_arg; @@ -3649,34 +3650,26 @@ static void free_pg_vec(struct pgv *pg_vec, unsigned int order, static char *alloc_one_pg_vec_page(unsigned long order) { - char *buffer = NULL; + char *buffer; gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY; buffer = (char *) __get_free_pages(gfp_flags, order); - if (buffer) return buffer; - /* - * __get_free_pages failed, fall back to vmalloc - */ + /* __get_free_pages failed, fall back to vmalloc */ buffer = vzalloc((1 << order) * PAGE_SIZE); - if (buffer) return buffer; - /* - * vmalloc failed, lets dig into swap here - */ + /* vmalloc failed, lets dig into swap here */ gfp_flags &= ~__GFP_NORETRY; - buffer = (char *)__get_free_pages(gfp_flags, order); + buffer = (char *) __get_free_pages(gfp_flags, order); if (buffer) return buffer; - /* - * complete and utter failure - */ + /* complete and utter failure */ return NULL; } -- cgit v1.2.3 From f337db64af059c9a94278a8b0ab97d87259ff62f Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 22 Jan 2014 02:29:39 +0100 Subject: random32: add prandom_u32_max and convert open coded users Many functions have open coded a function that returns a random number in range [0,N-1]. Under the assumption that we have a PRNG such as taus113 with being well distributed in [0, ~0U] space, we can implement such a function as uword t = (n*m')>>32, where m' is a random number obtained from PRNG, n the right open interval border and t our resulting random number, with n,m',t in u32 universe. Lets go with Joe and simply call it prandom_u32_max(), although technically we have an right open interval endpoint, but that we have documented. Other users can further be migrated to the new prandom_u32_max() function later on; for now, we need to make sure to migrate reciprocal_divide() users for the reciprocal_divide() follow-up fixup since their function signatures are going to change. Joint work with Hannes Frederic Sowa. Cc: Jakub Zawadzki Cc: Eric Dumazet Cc: linux-kernel@vger.kernel.org Signed-off-by: Hannes Frederic Sowa Signed-off-by: Daniel Borkmann Signed-off-by: David S. Miller --- drivers/net/team/team_mode_random.c | 8 +------- include/linux/random.h | 18 +++++++++++++++++- net/packet/af_packet.c | 2 +- net/sched/sch_choke.c | 9 +-------- 4 files changed, 20 insertions(+), 17 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/drivers/net/team/team_mode_random.c b/drivers/net/team/team_mode_random.c index 7f032e211343..cd2f692b8074 100644 --- a/drivers/net/team/team_mode_random.c +++ b/drivers/net/team/team_mode_random.c @@ -13,20 +13,14 @@ #include #include #include -#include #include -static u32 random_N(unsigned int N) -{ - return reciprocal_divide(prandom_u32(), N); -} - static bool rnd_transmit(struct team *team, struct sk_buff *skb) { struct team_port *port; int port_index; - port_index = random_N(team->en_port_count); + port_index = prandom_u32_max(team->en_port_count); port = team_get_port_by_index_rcu(team, port_index); if (unlikely(!port)) goto drop; diff --git a/include/linux/random.h b/include/linux/random.h index 4002b3df4c85..1cfce0e24dbd 100644 --- a/include/linux/random.h +++ b/include/linux/random.h @@ -8,7 +8,6 @@ #include - extern void add_device_randomness(const void *, unsigned int); extern void add_input_randomness(unsigned int type, unsigned int code, unsigned int value); @@ -38,6 +37,23 @@ struct rnd_state { u32 prandom_u32_state(struct rnd_state *state); void prandom_bytes_state(struct rnd_state *state, void *buf, int nbytes); +/** + * prandom_u32_max - returns a pseudo-random number in interval [0, ep_ro) + * @ep_ro: right open interval endpoint + * + * Returns a pseudo-random number that is in interval [0, ep_ro). Note + * that the result depends on PRNG being well distributed in [0, ~0U] + * u32 space. Here we use maximally equidistributed combined Tausworthe + * generator, that is, prandom_u32(). This is useful when requesting a + * random index of an array containing ep_ro elements, for example. + * + * Returns: pseudo-random number in interval [0, ep_ro) + */ +static inline u32 prandom_u32_max(u32 ep_ro) +{ + return (u32)(((u64) prandom_u32() * ep_ro) >> 32); +} + /* * Handle minimum values for seeds */ diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 59fb3db2e713..df3cbdd585c7 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1289,7 +1289,7 @@ static unsigned int fanout_demux_rnd(struct packet_fanout *f, struct sk_buff *skb, unsigned int num) { - return reciprocal_divide(prandom_u32(), num); + return prandom_u32_max(num); } static unsigned int fanout_demux_rollover(struct packet_fanout *f, diff --git a/net/sched/sch_choke.c b/net/sched/sch_choke.c index ddd73cb2d7ba..2aee02802c27 100644 --- a/net/sched/sch_choke.c +++ b/net/sched/sch_choke.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -77,12 +76,6 @@ struct choke_sched_data { struct sk_buff **tab; }; -/* deliver a random number between 0 and N - 1 */ -static u32 random_N(unsigned int N) -{ - return reciprocal_divide(prandom_u32(), N); -} - /* number of elements in queue including holes */ static unsigned int choke_len(const struct choke_sched_data *q) { @@ -233,7 +226,7 @@ static struct sk_buff *choke_peek_random(const struct choke_sched_data *q, int retrys = 3; do { - *pidx = (q->head + random_N(choke_len(q))) & q->tab_mask; + *pidx = (q->head + prandom_u32_max(choke_len(q))) & q->tab_mask; skb = q->tab[*pidx]; if (skb) return skb; -- cgit v1.2.3 From 89770b0a69ee0e0e5e99c722192d535115f73778 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 22 Jan 2014 02:29:40 +0100 Subject: net: introduce reciprocal_scale helper and convert users As David Laight suggests, we shouldn't necessarily call this reciprocal_divide() when users didn't requested a reciprocal_value(); lets keep the basic idea and call it reciprocal_scale(). More background information on this topic can be found in [1]. Joint work with Hannes Frederic Sowa. [1] http://homepage.cs.uiowa.edu/~jones/bcd/divide.html Suggested-by: David Laight Cc: Jakub Zawadzki Cc: Eric Dumazet Cc: linux-kernel@vger.kernel.org Signed-off-by: Hannes Frederic Sowa Signed-off-by: Daniel Borkmann Signed-off-by: David S. Miller --- include/linux/kernel.h | 19 +++++++++++++++++++ include/net/codel.h | 4 +--- net/packet/af_packet.c | 3 +-- 3 files changed, 21 insertions(+), 5 deletions(-) (limited to 'net/packet/af_packet.c') diff --git a/include/linux/kernel.h b/include/linux/kernel.h index ecb87544cc5d..03d8a6b0e2e8 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -193,6 +193,25 @@ extern int _cond_resched(void); (__x < 0) ? -__x : __x; \ }) +/** + * reciprocal_scale - "scale" a value into range [0, ep_ro) + * @val: value + * @ep_ro: right open interval endpoint + * + * Perform a "reciprocal multiplication" in order to "scale" a value into + * range [0, ep_ro), where the upper interval endpoint is right-open. + * This is useful, e.g. for accessing a index of an array containing + * ep_ro elements, for example. Think of it as sort of modulus, only that + * the result isn't that of modulo. ;) Note that if initial input is a + * small value, then result will return 0. + * + * Return: a result based on val in interval [0, ep_ro). + */ +static inline u32 reciprocal_scale(u32 val, u32 ep_ro) +{ + return (u32)(((u64) val * ep_ro) >> 32); +} + #if defined(CONFIG_MMU) && \ (defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)) void might_fault(void); diff --git a/include/net/codel.h b/include/net/codel.h index 3b04ff5f6f8d..fe0eab32ce76 100644 --- a/include/net/codel.h +++ b/include/net/codel.h @@ -46,7 +46,6 @@ #include #include #include -#include /* Controlling Queue Delay (CoDel) algorithm * ========================================= @@ -211,10 +210,9 @@ static codel_time_t codel_control_law(codel_time_t t, codel_time_t interval, u32 rec_inv_sqrt) { - return t + reciprocal_divide(interval, rec_inv_sqrt << REC_INV_SQRT_SHIFT); + return t + reciprocal_scale(interval, rec_inv_sqrt << REC_INV_SQRT_SHIFT); } - static bool codel_should_drop(const struct sk_buff *skb, struct Qdisc *sch, struct codel_vars *vars, diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index df3cbdd585c7..97346162803d 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -88,7 +88,6 @@ #include #include #include -#include #include #ifdef CONFIG_INET #include @@ -1262,7 +1261,7 @@ static unsigned int fanout_demux_hash(struct packet_fanout *f, struct sk_buff *skb, unsigned int num) { - return reciprocal_divide(skb->rxhash, num); + return reciprocal_scale(skb->rxhash, num); } static unsigned int fanout_demux_lb(struct packet_fanout *f, -- cgit v1.2.3 From 2d36097d26b5991d71a2cf4a20c1a158f0f1bfcd Mon Sep 17 00:00:00 2001 From: Neil Horman Date: Wed, 22 Jan 2014 16:01:44 -0500 Subject: af_packet: Add Queue mapping mode to af_packet fanout operation This patch adds a queue mapping mode to the fanout operation of af_packet sockets. This allows user space af_packet users to better filter on flows ingressing and egressing via a specific hardware queue, and avoids the potential packet reordering that can occur when FANOUT_CPU is being used and irq affinity varies. Tested successfully by myself. applies to net-next Signed-off-by: Neil Horman CC: "David S. Miller" Signed-off-by: David S. Miller --- include/uapi/linux/if_packet.h | 1 + net/packet/af_packet.c | 11 +++++++++++ 2 files changed, 12 insertions(+) (limited to 'net/packet/af_packet.c') diff --git a/include/uapi/linux/if_packet.h b/include/uapi/linux/if_packet.h index 1988a02842cc..bac27fa05f5b 100644 --- a/include/uapi/linux/if_packet.h +++ b/include/uapi/linux/if_packet.h @@ -60,6 +60,7 @@ struct sockaddr_ll { #define PACKET_FANOUT_CPU 2 #define PACKET_FANOUT_ROLLOVER 3 #define PACKET_FANOUT_RND 4 +#define PACKET_FANOUT_QM 5 #define PACKET_FANOUT_FLAG_ROLLOVER 0x1000 #define PACKET_FANOUT_FLAG_DEFRAG 0x8000 diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 97346162803d..6a2bb37506c5 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1312,6 +1312,13 @@ static unsigned int fanout_demux_rollover(struct packet_fanout *f, return idx; } +static unsigned int fanout_demux_qm(struct packet_fanout *f, + struct sk_buff *skb, + unsigned int num) +{ + return skb_get_queue_mapping(skb) % num; +} + static bool fanout_has_flag(struct packet_fanout *f, u16 flag) { return f->flags & (flag >> 8); @@ -1351,6 +1358,9 @@ static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev, case PACKET_FANOUT_RND: idx = fanout_demux_rnd(f, skb, num); break; + case PACKET_FANOUT_QM: + idx = fanout_demux_qm(f, skb, num); + break; case PACKET_FANOUT_ROLLOVER: idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num); break; @@ -1421,6 +1431,7 @@ static int fanout_add(struct sock *sk, u16 id, u16 type_flags) case PACKET_FANOUT_LB: case PACKET_FANOUT_CPU: case PACKET_FANOUT_RND: + case PACKET_FANOUT_QM: break; default: return -EINVAL; -- cgit v1.2.3