From b530256d2e0f1a75fab31f9821129fff1bb49faa Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 4 Jan 2009 16:13:19 -0800 Subject: gro: Use gso_size to store MSS In order to allow GRO packets without frag_list at all, we need to store the MSS in the packet itself. The obvious place is gso_size. The only thing to watch out for is if the packet ends up not being GRO then we need to clear gso_size before pushing the packet into the stack. Signed-off-by: Herbert Xu Signed-off-by: David S. Miller --- net/ipv4/tcp.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'net/ipv4/tcp.c') diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index f28acf11fc67..4d655e945413 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -2519,9 +2519,7 @@ found: flush |= memcmp(th + 1, th2 + 1, thlen - sizeof(*th)); total = p->len; - mss = total; - if (skb_shinfo(p)->frag_list) - mss = skb_shinfo(p)->frag_list->len; + mss = skb_shinfo(p)->gso_size; flush |= skb->len > mss || skb->len <= 0; flush |= ntohl(th2->seq) + total != ntohl(th->seq); @@ -2557,7 +2555,6 @@ int tcp_gro_complete(struct sk_buff *skb) skb->csum_offset = offsetof(struct tcphdr, check); skb->ip_summed = CHECKSUM_PARTIAL; - skb_shinfo(skb)->gso_size = skb_shinfo(skb)->frag_list->len; skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count; if (th->cwr) -- cgit v1.2.3 From 4f7d54f59bc470f0aaa932f747a95232d7ebf8b1 Mon Sep 17 00:00:00 2001 From: Lennert Buytenhek Date: Mon, 5 Jan 2009 00:00:12 -0800 Subject: tcp: don't mask EOF and socket errors on nonblocking splice receive Currently, setting SPLICE_F_NONBLOCK on splice from a TCP socket results in masking of EOF (RDHUP) and error conditions on the socket by an -EAGAIN return. Move the NONBLOCK check in tcp_splice_read() to be after the EOF and error checks to fix this. Signed-off-by: Lennert Buytenhek Signed-off-by: David S. Miller --- net/ipv4/tcp.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'net/ipv4/tcp.c') diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 4d655e945413..bce1b068f2a7 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -580,10 +580,6 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos, else if (!ret) { if (spliced) break; - if (flags & SPLICE_F_NONBLOCK) { - ret = -EAGAIN; - break; - } if (sock_flag(sk, SOCK_DONE)) break; if (sk->sk_err) { @@ -601,6 +597,10 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos, ret = -ENOTCONN; break; } + if (flags & SPLICE_F_NONBLOCK) { + ret = -EAGAIN; + break; + } if (!timeo) { ret = -EAGAIN; break; -- cgit v1.2.3 From 7945cc6464a4db0caf6dfacdfe05806051c4cb7b Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 5 Jan 2009 00:59:00 -0800 Subject: tcp: Kill extraneous SPLICE_F_NONBLOCK checks. In splice TCP receive, the SPLICE_F_NONBLOCK flag is used to compute the "timeo" value. So checking it again inside of the main receive loop to trigger -EAGAIN processing is entirely unnecessary. Noticed by Jarek P. and Lennert Buytenhek. Signed-off-by: David S. Miller --- net/ipv4/tcp.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'net/ipv4/tcp.c') diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index bce1b068f2a7..35bcddf8a932 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -597,10 +597,6 @@ ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos, ret = -ENOTCONN; break; } - if (flags & SPLICE_F_NONBLOCK) { - ret = -EAGAIN; - break; - } if (!timeo) { ret = -EAGAIN; break; -- cgit v1.2.3 From 6f49a57aa5a0c6d4e4e27c85f7af6c83325a12d1 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 6 Jan 2009 11:38:14 -0700 Subject: dmaengine: up-level reference counting to the module level Simply, if a client wants any dmaengine channel then prevent all dmaengine modules from being removed. Once the clients are done re-enable module removal. Why?, beyond reducing complication: 1/ Tracking reference counts per-transaction in an efficient manner, as is currently done, requires a complicated scheme to avoid cache-line bouncing effects. 2/ Per-transaction ref-counting gives the false impression that a dma-driver can be gracefully removed ahead of its user (net, md, or dma-slave) 3/ None of the in-tree dma-drivers talk to hot pluggable hardware, but if such an engine were built one day we still would not need to notify clients of remove events. The driver can simply return NULL to a ->prep() request, something that is much easier for a client to handle. Reviewed-by: Andrew Morton Acked-by: Maciej Sosnowski Signed-off-by: Dan Williams --- crypto/async_tx/async_tx.c | 4 - drivers/dma/dmaengine.c | 205 +++++++++++++++++++++++++++---------------- drivers/dma/dmatest.c | 2 - drivers/dma/dw_dmac.c | 2 +- drivers/mmc/host/atmel-mci.c | 4 +- include/linux/dmaengine.h | 21 ----- include/net/netdma.h | 4 +- net/ipv4/tcp.c | 1 - 8 files changed, 132 insertions(+), 111 deletions(-) (limited to 'net/ipv4/tcp.c') diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c index 8cfac182165d..43fe4cbe71e6 100644 --- a/crypto/async_tx/async_tx.c +++ b/crypto/async_tx/async_tx.c @@ -198,8 +198,6 @@ dma_channel_add_remove(struct dma_client *client, /* add the channel to the generic management list */ master_ref = kmalloc(sizeof(*master_ref), GFP_KERNEL); if (master_ref) { - /* keep a reference until async_tx is unloaded */ - dma_chan_get(chan); init_dma_chan_ref(master_ref, chan); spin_lock_irqsave(&async_tx_lock, flags); list_add_tail_rcu(&master_ref->node, @@ -221,8 +219,6 @@ dma_channel_add_remove(struct dma_client *client, spin_lock_irqsave(&async_tx_lock, flags); list_for_each_entry(ref, &async_tx_master_list, node) if (ref->chan == chan) { - /* permit backing devices to go away */ - dma_chan_put(ref->chan); list_del_rcu(&ref->node); call_rcu(&ref->rcu, free_dma_chan_ref); found = 1; diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index b9008932a8f3..d4d925912c47 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -74,6 +74,7 @@ static DEFINE_MUTEX(dma_list_mutex); static LIST_HEAD(dma_device_list); static LIST_HEAD(dma_client_list); +static long dmaengine_ref_count; /* --- sysfs implementation --- */ @@ -105,19 +106,8 @@ static ssize_t show_bytes_transferred(struct device *dev, struct device_attribut static ssize_t show_in_use(struct device *dev, struct device_attribute *attr, char *buf) { struct dma_chan *chan = to_dma_chan(dev); - int in_use = 0; - - if (unlikely(chan->slow_ref) && - atomic_read(&chan->refcount.refcount) > 1) - in_use = 1; - else { - if (local_read(&(per_cpu_ptr(chan->local, - get_cpu())->refcount)) > 0) - in_use = 1; - put_cpu(); - } - return sprintf(buf, "%d\n", in_use); + return sprintf(buf, "%d\n", chan->client_count); } static struct device_attribute dma_attrs[] = { @@ -155,6 +145,78 @@ __dma_chan_satisfies_mask(struct dma_chan *chan, dma_cap_mask_t *want) return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END); } +static struct module *dma_chan_to_owner(struct dma_chan *chan) +{ + return chan->device->dev->driver->owner; +} + +/** + * balance_ref_count - catch up the channel reference count + * @chan - channel to balance ->client_count versus dmaengine_ref_count + * + * balance_ref_count must be called under dma_list_mutex + */ +static void balance_ref_count(struct dma_chan *chan) +{ + struct module *owner = dma_chan_to_owner(chan); + + while (chan->client_count < dmaengine_ref_count) { + __module_get(owner); + chan->client_count++; + } +} + +/** + * dma_chan_get - try to grab a dma channel's parent driver module + * @chan - channel to grab + * + * Must be called under dma_list_mutex + */ +static int dma_chan_get(struct dma_chan *chan) +{ + int err = -ENODEV; + struct module *owner = dma_chan_to_owner(chan); + + if (chan->client_count) { + __module_get(owner); + err = 0; + } else if (try_module_get(owner)) + err = 0; + + if (err == 0) + chan->client_count++; + + /* allocate upon first client reference */ + if (chan->client_count == 1 && err == 0) { + int desc_cnt = chan->device->device_alloc_chan_resources(chan, NULL); + + if (desc_cnt < 0) { + err = desc_cnt; + chan->client_count = 0; + module_put(owner); + } else + balance_ref_count(chan); + } + + return err; +} + +/** + * dma_chan_put - drop a reference to a dma channel's parent driver module + * @chan - channel to release + * + * Must be called under dma_list_mutex + */ +static void dma_chan_put(struct dma_chan *chan) +{ + if (!chan->client_count) + return; /* this channel failed alloc_chan_resources */ + chan->client_count--; + module_put(dma_chan_to_owner(chan)); + if (chan->client_count == 0) + chan->device->device_free_chan_resources(chan); +} + /** * dma_client_chan_alloc - try to allocate channels to a client * @client: &dma_client @@ -165,7 +227,6 @@ static void dma_client_chan_alloc(struct dma_client *client) { struct dma_device *device; struct dma_chan *chan; - int desc; /* allocated descriptor count */ enum dma_state_client ack; /* Find a channel */ @@ -178,23 +239,16 @@ static void dma_client_chan_alloc(struct dma_client *client) list_for_each_entry(chan, &device->channels, device_node) { if (!dma_chan_satisfies_mask(chan, client->cap_mask)) continue; + if (!chan->client_count) + continue; + ack = client->event_callback(client, chan, + DMA_RESOURCE_AVAILABLE); - desc = chan->device->device_alloc_chan_resources( - chan, client); - if (desc >= 0) { - ack = client->event_callback(client, - chan, - DMA_RESOURCE_AVAILABLE); - - /* we are done once this client rejects - * an available resource - */ - if (ack == DMA_ACK) { - dma_chan_get(chan); - chan->client_count++; - } else if (ack == DMA_NAK) - return; - } + /* we are done once this client rejects + * an available resource + */ + if (ack == DMA_NAK) + return; } } } @@ -224,7 +278,6 @@ EXPORT_SYMBOL(dma_sync_wait); void dma_chan_cleanup(struct kref *kref) { struct dma_chan *chan = container_of(kref, struct dma_chan, refcount); - chan->device->device_free_chan_resources(chan); kref_put(&chan->device->refcount, dma_async_device_cleanup); } EXPORT_SYMBOL(dma_chan_cleanup); @@ -232,18 +285,12 @@ EXPORT_SYMBOL(dma_chan_cleanup); static void dma_chan_free_rcu(struct rcu_head *rcu) { struct dma_chan *chan = container_of(rcu, struct dma_chan, rcu); - int bias = 0x7FFFFFFF; - int i; - for_each_possible_cpu(i) - bias -= local_read(&per_cpu_ptr(chan->local, i)->refcount); - atomic_sub(bias, &chan->refcount.refcount); + kref_put(&chan->refcount, dma_chan_cleanup); } static void dma_chan_release(struct dma_chan *chan) { - atomic_add(0x7FFFFFFF, &chan->refcount.refcount); - chan->slow_ref = 1; call_rcu(&chan->rcu, dma_chan_free_rcu); } @@ -262,44 +309,37 @@ static void dma_clients_notify_available(void) mutex_unlock(&dma_list_mutex); } -/** - * dma_chans_notify_available - tell the clients that a channel is going away - * @chan: channel on its way out - */ -static void dma_clients_notify_removed(struct dma_chan *chan) -{ - struct dma_client *client; - enum dma_state_client ack; - - mutex_lock(&dma_list_mutex); - - list_for_each_entry(client, &dma_client_list, global_node) { - ack = client->event_callback(client, chan, - DMA_RESOURCE_REMOVED); - - /* client was holding resources for this channel so - * free it - */ - if (ack == DMA_ACK) { - dma_chan_put(chan); - chan->client_count--; - } - } - - mutex_unlock(&dma_list_mutex); -} - /** * dma_async_client_register - register a &dma_client * @client: ptr to a client structure with valid 'event_callback' and 'cap_mask' */ void dma_async_client_register(struct dma_client *client) { + struct dma_device *device, *_d; + struct dma_chan *chan; + int err; + /* validate client data */ BUG_ON(dma_has_cap(DMA_SLAVE, client->cap_mask) && !client->slave); mutex_lock(&dma_list_mutex); + dmaengine_ref_count++; + + /* try to grab channels */ + list_for_each_entry_safe(device, _d, &dma_device_list, global_node) + list_for_each_entry(chan, &device->channels, device_node) { + err = dma_chan_get(chan); + if (err == -ENODEV) { + /* module removed before we could use it */ + list_del_init(&device->global_node); + break; + } else if (err) + pr_err("dmaengine: failed to get %s: (%d)\n", + dev_name(&chan->dev), err); + } + + list_add_tail(&client->global_node, &dma_client_list); mutex_unlock(&dma_list_mutex); } @@ -315,23 +355,17 @@ void dma_async_client_unregister(struct dma_client *client) { struct dma_device *device; struct dma_chan *chan; - enum dma_state_client ack; if (!client) return; mutex_lock(&dma_list_mutex); - /* free all channels the client is holding */ + dmaengine_ref_count--; + BUG_ON(dmaengine_ref_count < 0); + /* drop channel references */ list_for_each_entry(device, &dma_device_list, global_node) - list_for_each_entry(chan, &device->channels, device_node) { - ack = client->event_callback(client, chan, - DMA_RESOURCE_REMOVED); - - if (ack == DMA_ACK) { - dma_chan_put(chan); - chan->client_count--; - } - } + list_for_each_entry(chan, &device->channels, device_node) + dma_chan_put(chan); list_del(&client->global_node); mutex_unlock(&dma_list_mutex); @@ -423,6 +457,21 @@ int dma_async_device_register(struct dma_device *device) } mutex_lock(&dma_list_mutex); + if (dmaengine_ref_count) + list_for_each_entry(chan, &device->channels, device_node) { + /* if clients are already waiting for channels we need + * to take references on their behalf + */ + if (dma_chan_get(chan) == -ENODEV) { + /* note we can only get here for the first + * channel as the remaining channels are + * guaranteed to get a reference + */ + rc = -ENODEV; + mutex_unlock(&dma_list_mutex); + goto err_out; + } + } list_add_tail(&device->global_node, &dma_device_list); mutex_unlock(&dma_list_mutex); @@ -456,7 +505,7 @@ static void dma_async_device_cleanup(struct kref *kref) } /** - * dma_async_device_unregister - unregisters DMA devices + * dma_async_device_unregister - unregister a DMA device * @device: &dma_device */ void dma_async_device_unregister(struct dma_device *device) @@ -468,7 +517,9 @@ void dma_async_device_unregister(struct dma_device *device) mutex_unlock(&dma_list_mutex); list_for_each_entry(chan, &device->channels, device_node) { - dma_clients_notify_removed(chan); + WARN_ONCE(chan->client_count, + "%s called while %d clients hold a reference\n", + __func__, chan->client_count); device_unregister(&chan->dev); dma_chan_release(chan); } diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index ed9636bfb54a..db4050884713 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -215,7 +215,6 @@ static int dmatest_func(void *data) smp_rmb(); chan = thread->chan; - dma_chan_get(chan); while (!kthread_should_stop()) { total_tests++; @@ -293,7 +292,6 @@ static int dmatest_func(void *data) } ret = 0; - dma_chan_put(chan); kfree(thread->dstbuf); err_dstbuf: kfree(thread->srcbuf); diff --git a/drivers/dma/dw_dmac.c b/drivers/dma/dw_dmac.c index 0778d99aea7c..377dafa37a20 100644 --- a/drivers/dma/dw_dmac.c +++ b/drivers/dma/dw_dmac.c @@ -773,7 +773,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan, dev_vdbg(&chan->dev, "alloc_chan_resources\n"); /* Channels doing slave DMA can only handle one client. */ - if (dwc->dws || client->slave) { + if (dwc->dws || (client && client->slave)) { if (chan->client_count) return -EBUSY; } diff --git a/drivers/mmc/host/atmel-mci.c b/drivers/mmc/host/atmel-mci.c index 7a3f2436b011..6c11f4d4c4e9 100644 --- a/drivers/mmc/host/atmel-mci.c +++ b/drivers/mmc/host/atmel-mci.c @@ -593,10 +593,8 @@ atmci_submit_data_dma(struct atmel_mci *host, struct mmc_data *data) /* If we don't have a channel, we can't do DMA */ chan = host->dma.chan; - if (chan) { - dma_chan_get(chan); + if (chan) host->data_chan = chan; - } if (!chan) return -ENODEV; diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index e4ec7e7b8056..d18d37d1015d 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -165,7 +165,6 @@ struct dma_slave { */ struct dma_chan_percpu { - local_t refcount; /* stats */ unsigned long memcpy_count; unsigned long bytes_transferred; @@ -205,26 +204,6 @@ struct dma_chan { void dma_chan_cleanup(struct kref *kref); -static inline void dma_chan_get(struct dma_chan *chan) -{ - if (unlikely(chan->slow_ref)) - kref_get(&chan->refcount); - else { - local_inc(&(per_cpu_ptr(chan->local, get_cpu())->refcount)); - put_cpu(); - } -} - -static inline void dma_chan_put(struct dma_chan *chan) -{ - if (unlikely(chan->slow_ref)) - kref_put(&chan->refcount, dma_chan_cleanup); - else { - local_dec(&(per_cpu_ptr(chan->local, get_cpu())->refcount)); - put_cpu(); - } -} - /* * typedef dma_event_callback - function pointer to a DMA event callback * For each channel added to the system this routine is called for each client. diff --git a/include/net/netdma.h b/include/net/netdma.h index f28c6e064e8f..cbe2737f4a61 100644 --- a/include/net/netdma.h +++ b/include/net/netdma.h @@ -27,11 +27,11 @@ static inline struct dma_chan *get_softnet_dma(void) { struct dma_chan *chan; + rcu_read_lock(); chan = rcu_dereference(__get_cpu_var(softnet_data).net_dma); - if (chan) - dma_chan_get(chan); rcu_read_unlock(); + return chan; } diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index f28acf11fc67..75e0e0a2d8db 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1632,7 +1632,6 @@ skip_copy: /* Safe to free early-copied skbs now */ __skb_queue_purge(&sk->sk_async_wait_queue); - dma_chan_put(tp->ucopy.dma_chan); tp->ucopy.dma_chan = NULL; } if (tp->ucopy.pinned_list) { -- cgit v1.2.3 From f67b45999205164958de4ec0658d51fa4bee066d Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 6 Jan 2009 11:38:15 -0700 Subject: net_dma: convert to dma_find_channel Use the general-purpose channel allocation provided by dmaengine. Reviewed-by: Andrew Morton Signed-off-by: Dan Williams --- include/linux/netdevice.h | 3 --- include/net/netdma.h | 11 ----------- net/core/dev.c | 40 ---------------------------------------- net/ipv4/tcp.c | 4 ++-- net/ipv4/tcp_input.c | 2 +- net/ipv4/tcp_ipv4.c | 2 +- net/ipv6/tcp_ipv6.c | 2 +- 7 files changed, 5 insertions(+), 59 deletions(-) (limited to 'net/ipv4/tcp.c') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 41e1224651cf..bac2c458d9b8 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1113,9 +1113,6 @@ struct softnet_data struct sk_buff *completion_queue; struct napi_struct backlog; -#ifdef CONFIG_NET_DMA - struct dma_chan *net_dma; -#endif }; DECLARE_PER_CPU(struct softnet_data,softnet_data); diff --git a/include/net/netdma.h b/include/net/netdma.h index cbe2737f4a61..8ba8ce284eeb 100644 --- a/include/net/netdma.h +++ b/include/net/netdma.h @@ -24,17 +24,6 @@ #include #include -static inline struct dma_chan *get_softnet_dma(void) -{ - struct dma_chan *chan; - - rcu_read_lock(); - chan = rcu_dereference(__get_cpu_var(softnet_data).net_dma); - rcu_read_unlock(); - - return chan; -} - int dma_skb_copy_datagram_iovec(struct dma_chan* chan, struct sk_buff *skb, int offset, struct iovec *to, size_t len, struct dma_pinned_list *pinned_list); diff --git a/net/core/dev.c b/net/core/dev.c index e40b0d57f8ff..bbb07dbe1740 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4827,44 +4827,6 @@ static int dev_cpu_callback(struct notifier_block *nfb, } #ifdef CONFIG_NET_DMA -/** - * net_dma_rebalance - try to maintain one DMA channel per CPU - * @net_dma: DMA client and associated data (lock, channels, channel_mask) - * - * This is called when the number of channels allocated to the net_dma client - * changes. The net_dma client tries to have one DMA channel per CPU. - */ - -static void net_dma_rebalance(struct net_dma *net_dma) -{ - unsigned int cpu, i, n, chan_idx; - struct dma_chan *chan; - - if (cpus_empty(net_dma->channel_mask)) { - for_each_online_cpu(cpu) - rcu_assign_pointer(per_cpu(softnet_data, cpu).net_dma, NULL); - return; - } - - i = 0; - cpu = first_cpu(cpu_online_map); - - for_each_cpu_mask_nr(chan_idx, net_dma->channel_mask) { - chan = net_dma->channels[chan_idx]; - - n = ((num_online_cpus() / cpus_weight(net_dma->channel_mask)) - + (i < (num_online_cpus() % - cpus_weight(net_dma->channel_mask)) ? 1 : 0)); - - while(n) { - per_cpu(softnet_data, cpu).net_dma = chan; - cpu = next_cpu(cpu, cpu_online_map); - n--; - } - i++; - } -} - /** * netdev_dma_event - event callback for the net_dma_client * @client: should always be net_dma_client @@ -4894,7 +4856,6 @@ netdev_dma_event(struct dma_client *client, struct dma_chan *chan, ack = DMA_ACK; net_dma->channels[pos] = chan; cpu_set(pos, net_dma->channel_mask); - net_dma_rebalance(net_dma); } break; case DMA_RESOURCE_REMOVED: @@ -4909,7 +4870,6 @@ netdev_dma_event(struct dma_client *client, struct dma_chan *chan, ack = DMA_ACK; cpu_clear(pos, net_dma->channel_mask); net_dma->channels[i] = NULL; - net_dma_rebalance(net_dma); } break; default: diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 75e0e0a2d8db..9b275abc8eb9 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1317,7 +1317,7 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, if ((available < target) && (len > sysctl_tcp_dma_copybreak) && !(flags & MSG_PEEK) && !sysctl_tcp_low_latency && - __get_cpu_var(softnet_data).net_dma) { + dma_find_channel(DMA_MEMCPY)) { preempt_enable_no_resched(); tp->ucopy.pinned_list = dma_pin_iovec_pages(msg->msg_iov, len); @@ -1527,7 +1527,7 @@ do_prequeue: if (!(flags & MSG_TRUNC)) { #ifdef CONFIG_NET_DMA if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list) - tp->ucopy.dma_chan = get_softnet_dma(); + tp->ucopy.dma_chan = dma_find_channel(DMA_MEMCPY); if (tp->ucopy.dma_chan) { tp->ucopy.dma_cookie = dma_skb_copy_datagram_iovec( diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 99b7ecbe8893..a6961d75c7ea 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -5005,7 +5005,7 @@ static int tcp_dma_try_early_copy(struct sock *sk, struct sk_buff *skb, return 0; if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list) - tp->ucopy.dma_chan = get_softnet_dma(); + tp->ucopy.dma_chan = dma_find_channel(DMA_MEMCPY); if (tp->ucopy.dma_chan && skb_csum_unnecessary(skb)) { diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 9d839fa9331e..19d7b429a262 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1594,7 +1594,7 @@ process: #ifdef CONFIG_NET_DMA struct tcp_sock *tp = tcp_sk(sk); if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list) - tp->ucopy.dma_chan = get_softnet_dma(); + tp->ucopy.dma_chan = dma_find_channel(DMA_MEMCPY); if (tp->ucopy.dma_chan) ret = tcp_v4_do_rcv(sk, skb); else diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index e8b8337a8310..71cd70951d7d 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -1640,7 +1640,7 @@ process: #ifdef CONFIG_NET_DMA struct tcp_sock *tp = tcp_sk(sk); if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list) - tp->ucopy.dma_chan = get_softnet_dma(); + tp->ucopy.dma_chan = dma_find_channel(DMA_MEMCPY); if (tp->ucopy.dma_chan) ret = tcp_v6_do_rcv(sk, skb); else -- cgit v1.2.3 From 684f2176015b313ab59cecf574117969cf638f28 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 8 Jan 2009 10:41:23 -0800 Subject: tcp6: Add GRO support This patch adds GRO support for TCP over IPv6. The code is exactly the same as the IPv4 version except for the pseudo-header checksum computation. Note that I've removed the unused tcphdr argument from tcp_v6_check rather than invent a bogus value for GRO. Signed-off-by: Herbert Xu Signed-off-by: David S. Miller --- net/ipv4/tcp.c | 2 ++ net/ipv6/tcp_ipv6.c | 45 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 4 deletions(-) (limited to 'net/ipv4/tcp.c') diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 35bcddf8a932..bd6ff907d9e4 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -2542,6 +2542,7 @@ out: return pp; } +EXPORT_SYMBOL(tcp_gro_receive); int tcp_gro_complete(struct sk_buff *skb) { @@ -2558,6 +2559,7 @@ int tcp_gro_complete(struct sk_buff *skb) return 0; } +EXPORT_SYMBOL(tcp_gro_complete); #ifdef CONFIG_TCP_MD5SIG static unsigned long tcp_md5sig_users; diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index e8b8337a8310..1297306d729c 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -101,7 +101,7 @@ static void tcp_v6_hash(struct sock *sk) } } -static __inline__ __sum16 tcp_v6_check(struct tcphdr *th, int len, +static __inline__ __sum16 tcp_v6_check(int len, struct in6_addr *saddr, struct in6_addr *daddr, __wsum base) @@ -501,7 +501,7 @@ static int tcp_v6_send_synack(struct sock *sk, struct request_sock *req) if (skb) { struct tcphdr *th = tcp_hdr(skb); - th->check = tcp_v6_check(th, skb->len, + th->check = tcp_v6_check(skb->len, &treq->loc_addr, &treq->rmt_addr, csum_partial(th, skb->len, skb->csum)); @@ -942,6 +942,41 @@ static int tcp_v6_gso_send_check(struct sk_buff *skb) return 0; } +struct sk_buff **tcp6_gro_receive(struct sk_buff **head, struct sk_buff *skb) +{ + struct ipv6hdr *iph = ipv6_hdr(skb); + + switch (skb->ip_summed) { + case CHECKSUM_COMPLETE: + if (!tcp_v6_check(skb->len, &iph->saddr, &iph->daddr, + skb->csum)) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + break; + } + + /* fall through */ + case CHECKSUM_NONE: + NAPI_GRO_CB(skb)->flush = 1; + return NULL; + } + + return tcp_gro_receive(head, skb); +} +EXPORT_SYMBOL(tcp6_gro_receive); + +int tcp6_gro_complete(struct sk_buff *skb) +{ + struct ipv6hdr *iph = ipv6_hdr(skb); + struct tcphdr *th = tcp_hdr(skb); + + th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb), + &iph->saddr, &iph->daddr, 0); + skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; + + return tcp_gro_complete(skb); +} +EXPORT_SYMBOL(tcp6_gro_complete); + static void tcp_v6_send_response(struct sk_buff *skb, u32 seq, u32 ack, u32 win, u32 ts, struct tcp_md5sig_key *key, int rst) { @@ -1429,14 +1464,14 @@ out: static __sum16 tcp_v6_checksum_init(struct sk_buff *skb) { if (skb->ip_summed == CHECKSUM_COMPLETE) { - if (!tcp_v6_check(tcp_hdr(skb), skb->len, &ipv6_hdr(skb)->saddr, + if (!tcp_v6_check(skb->len, &ipv6_hdr(skb)->saddr, &ipv6_hdr(skb)->daddr, skb->csum)) { skb->ip_summed = CHECKSUM_UNNECESSARY; return 0; } } - skb->csum = ~csum_unfold(tcp_v6_check(tcp_hdr(skb), skb->len, + skb->csum = ~csum_unfold(tcp_v6_check(skb->len, &ipv6_hdr(skb)->saddr, &ipv6_hdr(skb)->daddr, 0)); @@ -2062,6 +2097,8 @@ static struct inet6_protocol tcpv6_protocol = { .err_handler = tcp_v6_err, .gso_send_check = tcp_v6_gso_send_check, .gso_segment = tcp_tso_segment, + .gro_receive = tcp6_gro_receive, + .gro_complete = tcp6_gro_complete, .flags = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL, }; -- cgit v1.2.3