summaryrefslogtreecommitdiff
path: root/drivers/net/ethernet/sfc
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/ethernet/sfc')
-rw-r--r--drivers/net/ethernet/sfc/ef10.c14
-rw-r--r--drivers/net/ethernet/sfc/efx.c67
-rw-r--r--drivers/net/ethernet/sfc/efx.h5
-rw-r--r--drivers/net/ethernet/sfc/ethtool.c59
-rw-r--r--drivers/net/ethernet/sfc/falcon.c9
-rw-r--r--drivers/net/ethernet/sfc/mcdi_port.c11
-rw-r--r--drivers/net/ethernet/sfc/net_driver.h154
-rw-r--r--drivers/net/ethernet/sfc/nic.h13
-rw-r--r--drivers/net/ethernet/sfc/rx.c12
-rw-r--r--drivers/net/ethernet/sfc/selftest.c5
-rw-r--r--drivers/net/ethernet/sfc/siena.c9
-rw-r--r--drivers/net/ethernet/sfc/tx.c28
12 files changed, 350 insertions, 36 deletions
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
index b5ed30a39144..002d4cdc319f 100644
--- a/drivers/net/ethernet/sfc/ef10.c
+++ b/drivers/net/ethernet/sfc/ef10.c
@@ -755,6 +755,8 @@ static int efx_ef10_reset(struct efx_nic *efx, enum reset_type reset_type)
{ NULL, 64, 8 * MC_CMD_MAC_ ## mcdi_name }
#define EF10_OTHER_STAT(ext_name) \
[EF10_STAT_ ## ext_name] = { #ext_name, 0, 0 }
+#define GENERIC_SW_STAT(ext_name) \
+ [GENERIC_STAT_ ## ext_name] = { #ext_name, 0, 0 }
static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = {
EF10_DMA_STAT(tx_bytes, TX_BYTES),
@@ -798,6 +800,8 @@ static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = {
EF10_DMA_STAT(rx_align_error, RX_ALIGN_ERROR_PKTS),
EF10_DMA_STAT(rx_length_error, RX_LENGTH_ERROR_PKTS),
EF10_DMA_STAT(rx_nodesc_drops, RX_NODESC_DROPS),
+ GENERIC_SW_STAT(rx_nodesc_trunc),
+ GENERIC_SW_STAT(rx_noskb_drops),
EF10_DMA_STAT(rx_pm_trunc_bb_overflow, PM_TRUNC_BB_OVERFLOW),
EF10_DMA_STAT(rx_pm_discard_bb_overflow, PM_DISCARD_BB_OVERFLOW),
EF10_DMA_STAT(rx_pm_trunc_vfifo_full, PM_TRUNC_VFIFO_FULL),
@@ -841,7 +845,9 @@ static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = {
(1ULL << EF10_STAT_rx_gtjumbo) | \
(1ULL << EF10_STAT_rx_bad_gtjumbo) | \
(1ULL << EF10_STAT_rx_overflow) | \
- (1ULL << EF10_STAT_rx_nodesc_drops))
+ (1ULL << EF10_STAT_rx_nodesc_drops) | \
+ (1ULL << GENERIC_STAT_rx_nodesc_trunc) | \
+ (1ULL << GENERIC_STAT_rx_noskb_drops))
/* These statistics are only provided by the 10G MAC. For a 10G/40G
* switchable port we do not expose these because they might not
@@ -951,7 +957,7 @@ static int efx_ef10_try_update_nic_stats(struct efx_nic *efx)
stats[EF10_STAT_rx_bytes_minus_good_bytes];
efx_update_diff_stat(&stats[EF10_STAT_rx_bad_bytes],
stats[EF10_STAT_rx_bytes_minus_good_bytes]);
-
+ efx_update_sw_stats(efx, stats);
return 0;
}
@@ -990,7 +996,9 @@ static size_t efx_ef10_update_stats(struct efx_nic *efx, u64 *full_stats,
core_stats->tx_packets = stats[EF10_STAT_tx_packets];
core_stats->rx_bytes = stats[EF10_STAT_rx_bytes];
core_stats->tx_bytes = stats[EF10_STAT_tx_bytes];
- core_stats->rx_dropped = stats[EF10_STAT_rx_nodesc_drops];
+ core_stats->rx_dropped = stats[EF10_STAT_rx_nodesc_drops] +
+ stats[GENERIC_STAT_rx_nodesc_trunc] +
+ stats[GENERIC_STAT_rx_noskb_drops];
core_stats->multicast = stats[EF10_STAT_rx_multicast];
core_stats->rx_length_errors =
stats[EF10_STAT_rx_gtjumbo] +
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index 1e274045970f..4cebe9d37816 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -272,6 +272,9 @@ static int efx_poll(struct napi_struct *napi, int budget)
struct efx_nic *efx = channel->efx;
int spent;
+ if (!efx_channel_lock_napi(channel))
+ return budget;
+
netif_vdbg(efx, intr, efx->net_dev,
"channel %d NAPI poll executing on CPU %d\n",
channel->channel, raw_smp_processor_id());
@@ -311,6 +314,7 @@ static int efx_poll(struct napi_struct *napi, int budget)
efx_nic_eventq_read_ack(channel);
}
+ efx_channel_unlock_napi(channel);
return spent;
}
@@ -357,7 +361,7 @@ static int efx_init_eventq(struct efx_channel *channel)
}
/* Enable event queue processing and NAPI */
-static void efx_start_eventq(struct efx_channel *channel)
+void efx_start_eventq(struct efx_channel *channel)
{
netif_dbg(channel->efx, ifup, channel->efx->net_dev,
"chan %d start event queue\n", channel->channel);
@@ -366,17 +370,20 @@ static void efx_start_eventq(struct efx_channel *channel)
channel->enabled = true;
smp_wmb();
+ efx_channel_enable(channel);
napi_enable(&channel->napi_str);
efx_nic_eventq_read_ack(channel);
}
/* Disable event queue processing and NAPI */
-static void efx_stop_eventq(struct efx_channel *channel)
+void efx_stop_eventq(struct efx_channel *channel)
{
if (!channel->enabled)
return;
napi_disable(&channel->napi_str);
+ while (!efx_channel_disable(channel))
+ usleep_range(1000, 20000);
channel->enabled = false;
}
@@ -1960,6 +1967,8 @@ static void efx_init_napi_channel(struct efx_channel *channel)
channel->napi_dev = efx->net_dev;
netif_napi_add(channel->napi_dev, &channel->napi_str,
efx_poll, napi_weight);
+ napi_hash_add(&channel->napi_str);
+ efx_channel_init_lock(channel);
}
static void efx_init_napi(struct efx_nic *efx)
@@ -1972,8 +1981,10 @@ static void efx_init_napi(struct efx_nic *efx)
static void efx_fini_napi_channel(struct efx_channel *channel)
{
- if (channel->napi_dev)
+ if (channel->napi_dev) {
netif_napi_del(&channel->napi_str);
+ napi_hash_del(&channel->napi_str);
+ }
channel->napi_dev = NULL;
}
@@ -2008,6 +2019,37 @@ static void efx_netpoll(struct net_device *net_dev)
#endif
+#ifdef CONFIG_NET_RX_BUSY_POLL
+static int efx_busy_poll(struct napi_struct *napi)
+{
+ struct efx_channel *channel =
+ container_of(napi, struct efx_channel, napi_str);
+ struct efx_nic *efx = channel->efx;
+ int budget = 4;
+ int old_rx_packets, rx_packets;
+
+ if (!netif_running(efx->net_dev))
+ return LL_FLUSH_FAILED;
+
+ if (!efx_channel_lock_poll(channel))
+ return LL_FLUSH_BUSY;
+
+ old_rx_packets = channel->rx_queue.rx_packets;
+ efx_process_channel(channel, budget);
+
+ rx_packets = channel->rx_queue.rx_packets - old_rx_packets;
+
+ /* There is no race condition with NAPI here.
+ * NAPI will automatically be rescheduled if it yielded during busy
+ * polling, because it was not able to take the lock and thus returned
+ * the full budget.
+ */
+ efx_channel_unlock_poll(channel);
+
+ return rx_packets;
+}
+#endif
+
/**************************************************************************
*
* Kernel net device interface
@@ -2177,6 +2219,9 @@ static const struct net_device_ops efx_farch_netdev_ops = {
.ndo_poll_controller = efx_netpoll,
#endif
.ndo_setup_tc = efx_setup_tc,
+#ifdef CONFIG_NET_RX_BUSY_POLL
+ .ndo_busy_poll = efx_busy_poll,
+#endif
#ifdef CONFIG_RFS_ACCEL
.ndo_rx_flow_steer = efx_filter_rfs,
#endif
@@ -2197,6 +2242,9 @@ static const struct net_device_ops efx_ef10_netdev_ops = {
#ifdef CONFIG_NET_POLL_CONTROLLER
.ndo_poll_controller = efx_netpoll,
#endif
+#ifdef CONFIG_NET_RX_BUSY_POLL
+ .ndo_busy_poll = efx_busy_poll,
+#endif
#ifdef CONFIG_RFS_ACCEL
.ndo_rx_flow_steer = efx_filter_rfs,
#endif
@@ -2607,6 +2655,8 @@ static DEFINE_PCI_DEVICE_TABLE(efx_pci_table) = {
.driver_data = (unsigned long) &siena_a0_nic_type},
{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */
.driver_data = (unsigned long) &efx_hunt_a0_nic_type},
+ {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */
+ .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
{0} /* end of list */
};
@@ -2722,6 +2772,17 @@ static void efx_fini_struct(struct efx_nic *efx)
}
}
+void efx_update_sw_stats(struct efx_nic *efx, u64 *stats)
+{
+ u64 n_rx_nodesc_trunc = 0;
+ struct efx_channel *channel;
+
+ efx_for_each_channel(channel, efx)
+ n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc;
+ stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc;
+ stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops);
+}
+
/**************************************************************************
*
* PCI interface
diff --git a/drivers/net/ethernet/sfc/efx.h b/drivers/net/ethernet/sfc/efx.h
index 99032581336f..2587c582a821 100644
--- a/drivers/net/ethernet/sfc/efx.h
+++ b/drivers/net/ethernet/sfc/efx.h
@@ -194,11 +194,16 @@ int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
bool rx_may_override_tx);
void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs,
unsigned int *rx_usecs, bool *rx_adaptive);
+void efx_stop_eventq(struct efx_channel *channel);
+void efx_start_eventq(struct efx_channel *channel);
/* Dummy PHY ops for PHY drivers */
int efx_port_dummy_op_int(struct efx_nic *efx);
void efx_port_dummy_op_void(struct efx_nic *efx);
+/* Update the generic software stats in the passed stats array */
+void efx_update_sw_stats(struct efx_nic *efx, u64 *stats);
+
/* MTD */
#ifdef CONFIG_SFC_MTD
int efx_mtd_add(struct efx_nic *efx, struct efx_mtd_partition *parts,
diff --git a/drivers/net/ethernet/sfc/ethtool.c b/drivers/net/ethernet/sfc/ethtool.c
index 74739c4b9997..cad258a78708 100644
--- a/drivers/net/ethernet/sfc/ethtool.c
+++ b/drivers/net/ethernet/sfc/ethtool.c
@@ -77,7 +77,6 @@ static const struct efx_sw_stat_desc efx_sw_stat_desc[] = {
EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_tcp_udp_chksum_err),
EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_mcast_mismatch),
EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_frm_trunc),
- EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_nodesc_trunc),
EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_merge_events),
EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_merge_packets),
};
@@ -360,6 +359,37 @@ static int efx_ethtool_fill_self_tests(struct efx_nic *efx,
return n;
}
+static size_t efx_describe_per_queue_stats(struct efx_nic *efx, u8 *strings)
+{
+ size_t n_stats = 0;
+ struct efx_channel *channel;
+
+ efx_for_each_channel(channel, efx) {
+ if (efx_channel_has_tx_queues(channel)) {
+ n_stats++;
+ if (strings != NULL) {
+ snprintf(strings, ETH_GSTRING_LEN,
+ "tx-%u.tx_packets",
+ channel->tx_queue[0].queue /
+ EFX_TXQ_TYPES);
+
+ strings += ETH_GSTRING_LEN;
+ }
+ }
+ }
+ efx_for_each_channel(channel, efx) {
+ if (efx_channel_has_rx_queue(channel)) {
+ n_stats++;
+ if (strings != NULL) {
+ snprintf(strings, ETH_GSTRING_LEN,
+ "rx-%d.rx_packets", channel->channel);
+ strings += ETH_GSTRING_LEN;
+ }
+ }
+ }
+ return n_stats;
+}
+
static int efx_ethtool_get_sset_count(struct net_device *net_dev,
int string_set)
{
@@ -368,8 +398,9 @@ static int efx_ethtool_get_sset_count(struct net_device *net_dev,
switch (string_set) {
case ETH_SS_STATS:
return efx->type->describe_stats(efx, NULL) +
- EFX_ETHTOOL_SW_STAT_COUNT +
- efx_ptp_describe_stats(efx, NULL);
+ EFX_ETHTOOL_SW_STAT_COUNT +
+ efx_describe_per_queue_stats(efx, NULL) +
+ efx_ptp_describe_stats(efx, NULL);
case ETH_SS_TEST:
return efx_ethtool_fill_self_tests(efx, NULL, NULL, NULL);
default:
@@ -391,6 +422,8 @@ static void efx_ethtool_get_strings(struct net_device *net_dev,
strlcpy(strings + i * ETH_GSTRING_LEN,
efx_sw_stat_desc[i].name, ETH_GSTRING_LEN);
strings += EFX_ETHTOOL_SW_STAT_COUNT * ETH_GSTRING_LEN;
+ strings += (efx_describe_per_queue_stats(efx, strings) *
+ ETH_GSTRING_LEN);
efx_ptp_describe_stats(efx, strings);
break;
case ETH_SS_TEST:
@@ -410,6 +443,7 @@ static void efx_ethtool_get_stats(struct net_device *net_dev,
const struct efx_sw_stat_desc *stat;
struct efx_channel *channel;
struct efx_tx_queue *tx_queue;
+ struct efx_rx_queue *rx_queue;
int i;
spin_lock_bh(&efx->stats_lock);
@@ -445,6 +479,25 @@ static void efx_ethtool_get_stats(struct net_device *net_dev,
spin_unlock_bh(&efx->stats_lock);
+ efx_for_each_channel(channel, efx) {
+ if (efx_channel_has_tx_queues(channel)) {
+ *data = 0;
+ efx_for_each_channel_tx_queue(tx_queue, channel) {
+ *data += tx_queue->tx_packets;
+ }
+ data++;
+ }
+ }
+ efx_for_each_channel(channel, efx) {
+ if (efx_channel_has_rx_queue(channel)) {
+ *data = 0;
+ efx_for_each_channel_rx_queue(rx_queue, channel) {
+ *data += rx_queue->rx_packets;
+ }
+ data++;
+ }
+ }
+
efx_ptp_update_stats(efx, data);
}
diff --git a/drivers/net/ethernet/sfc/falcon.c b/drivers/net/ethernet/sfc/falcon.c
index fae25a418647..157037546d30 100644
--- a/drivers/net/ethernet/sfc/falcon.c
+++ b/drivers/net/ethernet/sfc/falcon.c
@@ -142,6 +142,8 @@
hw_name ## _ ## offset }
#define FALCON_OTHER_STAT(ext_name) \
[FALCON_STAT_ ## ext_name] = { #ext_name, 0, 0 }
+#define GENERIC_SW_STAT(ext_name) \
+ [GENERIC_STAT_ ## ext_name] = { #ext_name, 0, 0 }
static const struct efx_hw_stat_desc falcon_stat_desc[FALCON_STAT_COUNT] = {
FALCON_DMA_STAT(tx_bytes, XgTxOctets),
@@ -191,6 +193,8 @@ static const struct efx_hw_stat_desc falcon_stat_desc[FALCON_STAT_COUNT] = {
FALCON_DMA_STAT(rx_length_error, XgRxLengthError),
FALCON_DMA_STAT(rx_internal_error, XgRxInternalMACError),
FALCON_OTHER_STAT(rx_nodesc_drop_cnt),
+ GENERIC_SW_STAT(rx_nodesc_trunc),
+ GENERIC_SW_STAT(rx_noskb_drops),
};
static const unsigned long falcon_stat_mask[] = {
[0 ... BITS_TO_LONGS(FALCON_STAT_COUNT) - 1] = ~0UL,
@@ -2574,6 +2578,7 @@ static size_t falcon_update_nic_stats(struct efx_nic *efx, u64 *full_stats,
stats[FALCON_STAT_rx_bytes] -
stats[FALCON_STAT_rx_good_bytes] -
stats[FALCON_STAT_rx_control] * 64);
+ efx_update_sw_stats(efx, stats);
}
if (full_stats)
@@ -2584,7 +2589,9 @@ static size_t falcon_update_nic_stats(struct efx_nic *efx, u64 *full_stats,
core_stats->tx_packets = stats[FALCON_STAT_tx_packets];
core_stats->rx_bytes = stats[FALCON_STAT_rx_bytes];
core_stats->tx_bytes = stats[FALCON_STAT_tx_bytes];
- core_stats->rx_dropped = stats[FALCON_STAT_rx_nodesc_drop_cnt];
+ core_stats->rx_dropped = stats[FALCON_STAT_rx_nodesc_drop_cnt] +
+ stats[GENERIC_STAT_rx_nodesc_trunc] +
+ stats[GENERIC_STAT_rx_noskb_drops];
core_stats->multicast = stats[FALCON_STAT_rx_multicast];
core_stats->rx_length_errors =
stats[FALCON_STAT_rx_gtjumbo] +
diff --git a/drivers/net/ethernet/sfc/mcdi_port.c b/drivers/net/ethernet/sfc/mcdi_port.c
index e5fc4e1574b5..fb19b70eac01 100644
--- a/drivers/net/ethernet/sfc/mcdi_port.c
+++ b/drivers/net/ethernet/sfc/mcdi_port.c
@@ -183,6 +183,8 @@ static u32 mcdi_to_ethtool_cap(u32 media, u32 cap)
result |= SUPPORTED_1000baseKX_Full;
if (cap & (1 << MC_CMD_PHY_CAP_10000FDX_LBN))
result |= SUPPORTED_10000baseKX4_Full;
+ if (cap & (1 << MC_CMD_PHY_CAP_40000FDX_LBN))
+ result |= SUPPORTED_40000baseKR4_Full;
break;
case MC_CMD_MEDIA_XFP:
@@ -190,6 +192,12 @@ static u32 mcdi_to_ethtool_cap(u32 media, u32 cap)
result |= SUPPORTED_FIBRE;
break;
+ case MC_CMD_MEDIA_QSFP_PLUS:
+ result |= SUPPORTED_FIBRE;
+ if (cap & (1 << MC_CMD_PHY_CAP_40000FDX_LBN))
+ result |= SUPPORTED_40000baseCR4_Full;
+ break;
+
case MC_CMD_MEDIA_BASE_T:
result |= SUPPORTED_TP;
if (cap & (1 << MC_CMD_PHY_CAP_10HDX_LBN))
@@ -237,6 +245,8 @@ static u32 ethtool_to_mcdi_cap(u32 cap)
result |= (1 << MC_CMD_PHY_CAP_1000FDX_LBN);
if (cap & (SUPPORTED_10000baseT_Full | SUPPORTED_10000baseKX4_Full))
result |= (1 << MC_CMD_PHY_CAP_10000FDX_LBN);
+ if (cap & (SUPPORTED_40000baseCR4_Full | SUPPORTED_40000baseKR4_Full))
+ result |= (1 << MC_CMD_PHY_CAP_40000FDX_LBN);
if (cap & SUPPORTED_Pause)
result |= (1 << MC_CMD_PHY_CAP_PAUSE_LBN);
if (cap & SUPPORTED_Asym_Pause)
@@ -285,6 +295,7 @@ static u32 mcdi_to_ethtool_media(u32 media)
case MC_CMD_MEDIA_XFP:
case MC_CMD_MEDIA_SFP_PLUS:
+ case MC_CMD_MEDIA_QSFP_PLUS:
return PORT_FIBRE;
case MC_CMD_MEDIA_BASE_T:
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index 5bdae8ed7c57..9ede32064685 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -28,6 +28,7 @@
#include <linux/vmalloc.h>
#include <linux/i2c.h>
#include <linux/mtd/mtd.h>
+#include <net/busy_poll.h>
#include "enum.h"
#include "bitfield.h"
@@ -249,6 +250,8 @@ struct efx_tx_queue {
unsigned int tso_packets;
unsigned int pushes;
unsigned int pio_packets;
+ /* Statistics to supplement MAC stats */
+ unsigned long tx_packets;
/* Members shared between paths and sometimes updated */
unsigned int empty_read_count ____cacheline_aligned_in_smp;
@@ -358,6 +361,8 @@ struct efx_rx_queue {
unsigned int recycle_count;
struct timer_list slow_fill;
unsigned int slow_fill_count;
+ /* Statistics to supplement MAC stats */
+ unsigned long rx_packets;
};
enum efx_sync_events_state {
@@ -383,6 +388,8 @@ enum efx_sync_events_state {
* @irq_moderation: IRQ moderation value (in hardware ticks)
* @napi_dev: Net device used with NAPI
* @napi_str: NAPI control structure
+ * @state: state for NAPI vs busy polling
+ * @state_lock: lock protecting @state
* @eventq: Event queue buffer
* @eventq_mask: Event queue pointer mask
* @eventq_read_ptr: Event queue read pointer
@@ -420,6 +427,22 @@ struct efx_channel {
unsigned int irq_moderation;
struct net_device *napi_dev;
struct napi_struct napi_str;
+#ifdef CONFIG_NET_RX_BUSY_POLL
+ unsigned int state;
+ spinlock_t state_lock;
+#define EFX_CHANNEL_STATE_IDLE 0
+#define EFX_CHANNEL_STATE_NAPI (1 << 0) /* NAPI owns this channel */
+#define EFX_CHANNEL_STATE_POLL (1 << 1) /* poll owns this channel */
+#define EFX_CHANNEL_STATE_DISABLED (1 << 2) /* channel is disabled */
+#define EFX_CHANNEL_STATE_NAPI_YIELD (1 << 3) /* NAPI yielded this channel */
+#define EFX_CHANNEL_STATE_POLL_YIELD (1 << 4) /* poll yielded this channel */
+#define EFX_CHANNEL_OWNED \
+ (EFX_CHANNEL_STATE_NAPI | EFX_CHANNEL_STATE_POLL)
+#define EFX_CHANNEL_LOCKED \
+ (EFX_CHANNEL_OWNED | EFX_CHANNEL_STATE_DISABLED)
+#define EFX_CHANNEL_USER_PEND \
+ (EFX_CHANNEL_STATE_POLL | EFX_CHANNEL_STATE_POLL_YIELD)
+#endif /* CONFIG_NET_RX_BUSY_POLL */
struct efx_special_buffer eventq;
unsigned int eventq_mask;
unsigned int eventq_read_ptr;
@@ -453,6 +476,135 @@ struct efx_channel {
u32 sync_timestamp_minor;
};
+#ifdef CONFIG_NET_RX_BUSY_POLL
+static inline void efx_channel_init_lock(struct efx_channel *channel)
+{
+ spin_lock_init(&channel->state_lock);
+}
+
+/* Called from the device poll routine to get ownership of a channel. */
+static inline bool efx_channel_lock_napi(struct efx_channel *channel)
+{
+ bool rc = true;
+
+ spin_lock_bh(&channel->state_lock);
+ if (channel->state & EFX_CHANNEL_LOCKED) {
+ WARN_ON(channel->state & EFX_CHANNEL_STATE_NAPI);
+ channel->state |= EFX_CHANNEL_STATE_NAPI_YIELD;
+ rc = false;
+ } else {
+ /* we don't care if someone yielded */
+ channel->state = EFX_CHANNEL_STATE_NAPI;
+ }
+ spin_unlock_bh(&channel->state_lock);
+ return rc;
+}
+
+static inline void efx_channel_unlock_napi(struct efx_channel *channel)
+{
+ spin_lock_bh(&channel->state_lock);
+ WARN_ON(channel->state &
+ (EFX_CHANNEL_STATE_POLL | EFX_CHANNEL_STATE_NAPI_YIELD));
+
+ channel->state &= EFX_CHANNEL_STATE_DISABLED;
+ spin_unlock_bh(&channel->state_lock);
+}
+
+/* Called from efx_busy_poll(). */
+static inline bool efx_channel_lock_poll(struct efx_channel *channel)
+{
+ bool rc = true;
+
+ spin_lock_bh(&channel->state_lock);
+ if ((channel->state & EFX_CHANNEL_LOCKED)) {
+ channel->state |= EFX_CHANNEL_STATE_POLL_YIELD;
+ rc = false;
+ } else {
+ /* preserve yield marks */
+ channel->state |= EFX_CHANNEL_STATE_POLL;
+ }
+ spin_unlock_bh(&channel->state_lock);
+ return rc;
+}
+
+/* Returns true if NAPI tried to get the channel while it was locked. */
+static inline void efx_channel_unlock_poll(struct efx_channel *channel)
+{
+ spin_lock_bh(&channel->state_lock);
+ WARN_ON(channel->state & EFX_CHANNEL_STATE_NAPI);
+
+ /* will reset state to idle, unless channel is disabled */
+ channel->state &= EFX_CHANNEL_STATE_DISABLED;
+ spin_unlock_bh(&channel->state_lock);
+}
+
+/* True if a socket is polling, even if it did not get the lock. */
+static inline bool efx_channel_busy_polling(struct efx_channel *channel)
+{
+ WARN_ON(!(channel->state & EFX_CHANNEL_OWNED));
+ return channel->state & EFX_CHANNEL_USER_PEND;
+}
+
+static inline void efx_channel_enable(struct efx_channel *channel)
+{
+ spin_lock_bh(&channel->state_lock);
+ channel->state = EFX_CHANNEL_STATE_IDLE;
+ spin_unlock_bh(&channel->state_lock);
+}
+
+/* False if the channel is currently owned. */
+static inline bool efx_channel_disable(struct efx_channel *channel)
+{
+ bool rc = true;
+
+ spin_lock_bh(&channel->state_lock);
+ if (channel->state & EFX_CHANNEL_OWNED)
+ rc = false;
+ channel->state |= EFX_CHANNEL_STATE_DISABLED;
+ spin_unlock_bh(&channel->state_lock);
+
+ return rc;
+}
+
+#else /* CONFIG_NET_RX_BUSY_POLL */
+
+static inline void efx_channel_init_lock(struct efx_channel *channel)
+{
+}
+
+static inline bool efx_channel_lock_napi(struct efx_channel *channel)
+{
+ return true;
+}
+
+static inline void efx_channel_unlock_napi(struct efx_channel *channel)
+{
+}
+
+static inline bool efx_channel_lock_poll(struct efx_channel *channel)
+{
+ return false;
+}
+
+static inline void efx_channel_unlock_poll(struct efx_channel *channel)
+{
+}
+
+static inline bool efx_channel_busy_polling(struct efx_channel *channel)
+{
+ return false;
+}
+
+static inline void efx_channel_enable(struct efx_channel *channel)
+{
+}
+
+static inline bool efx_channel_disable(struct efx_channel *channel)
+{
+ return true;
+}
+#endif /* CONFIG_NET_RX_BUSY_POLL */
+
/**
* struct efx_msi_context - Context for each MSI
* @efx: The associated NIC
@@ -777,6 +929,7 @@ struct vfdi_status;
* interrupt has occurred.
* @stats_lock: Statistics update lock. Must be held when calling
* efx_nic_type::{update,start,stop}_stats.
+ * @n_rx_noskb_drops: Count of RX packets dropped due to failure to allocate an skb
*
* This is stored in the private area of the &struct net_device.
*/
@@ -930,6 +1083,7 @@ struct efx_nic {
spinlock_t biu_lock;
int last_irq_cpu;
spinlock_t stats_lock;
+ atomic_t n_rx_noskb_drops;
};
static inline int efx_dev_registered(struct efx_nic *efx)
diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
index d3ad8ed8d901..60f85149fc4c 100644
--- a/drivers/net/ethernet/sfc/nic.h
+++ b/drivers/net/ethernet/sfc/nic.h
@@ -135,6 +135,13 @@ enum {
/* Size and alignment of buffer table entries (same) */
#define EFX_BUF_SIZE EFX_PAGE_SIZE
+/* NIC-generic software stats */
+enum {
+ GENERIC_STAT_rx_noskb_drops,
+ GENERIC_STAT_rx_nodesc_trunc,
+ GENERIC_STAT_COUNT
+};
+
/**
* struct falcon_board_type - board operations and type information
* @id: Board type id, as found in NVRAM
@@ -205,7 +212,7 @@ static inline bool falcon_spi_present(const struct falcon_spi_device *spi)
}
enum {
- FALCON_STAT_tx_bytes,
+ FALCON_STAT_tx_bytes = GENERIC_STAT_COUNT,
FALCON_STAT_tx_packets,
FALCON_STAT_tx_pause,
FALCON_STAT_tx_control,
@@ -290,7 +297,7 @@ static inline struct falcon_board *falcon_board(struct efx_nic *efx)
}
enum {
- SIENA_STAT_tx_bytes,
+ SIENA_STAT_tx_bytes = GENERIC_STAT_COUNT,
SIENA_STAT_tx_good_bytes,
SIENA_STAT_tx_bad_bytes,
SIENA_STAT_tx_packets,
@@ -361,7 +368,7 @@ struct siena_nic_data {
};
enum {
- EF10_STAT_tx_bytes,
+ EF10_STAT_tx_bytes = GENERIC_STAT_COUNT,
EF10_STAT_tx_packets,
EF10_STAT_tx_pause,
EF10_STAT_tx_control,
diff --git a/drivers/net/ethernet/sfc/rx.c b/drivers/net/ethernet/sfc/rx.c
index 48588ddf81b0..c0ad95d2f63d 100644
--- a/drivers/net/ethernet/sfc/rx.c
+++ b/drivers/net/ethernet/sfc/rx.c
@@ -462,6 +462,7 @@ efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf,
skb_record_rx_queue(skb, channel->rx_queue.core_index);
+ skb_mark_napi_id(skb, &channel->napi_str);
gro_result = napi_gro_frags(napi);
if (gro_result != GRO_DROP)
channel->irq_mod_score += 2;
@@ -480,8 +481,10 @@ static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel,
skb = netdev_alloc_skb(efx->net_dev,
efx->rx_ip_align + efx->rx_prefix_size +
hdr_len);
- if (unlikely(skb == NULL))
+ if (unlikely(skb == NULL)) {
+ atomic_inc(&efx->n_rx_noskb_drops);
return NULL;
+ }
EFX_BUG_ON_PARANOID(rx_buf->len < hdr_len);
@@ -518,6 +521,8 @@ static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel,
/* Move past the ethernet header */
skb->protocol = eth_type_trans(skb, efx->net_dev);
+ skb_mark_napi_id(skb, &channel->napi_str);
+
return skb;
}
@@ -528,6 +533,8 @@ void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index,
struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
struct efx_rx_buffer *rx_buf;
+ rx_queue->rx_packets++;
+
rx_buf = efx_rx_buffer(rx_queue, index);
rx_buf->flags |= flags;
@@ -662,7 +669,8 @@ void __efx_rx_packet(struct efx_channel *channel)
if (unlikely(!(efx->net_dev->features & NETIF_F_RXCSUM)))
rx_buf->flags &= ~EFX_RX_PKT_CSUMMED;
- if ((rx_buf->flags & EFX_RX_PKT_TCP) && !channel->type->receive_skb)
+ if ((rx_buf->flags & EFX_RX_PKT_TCP) && !channel->type->receive_skb &&
+ !efx_channel_busy_polling(channel))
efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh);
else
efx_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags);
diff --git a/drivers/net/ethernet/sfc/selftest.c b/drivers/net/ethernet/sfc/selftest.c
index 0fc5baef45b1..10b6173d557d 100644
--- a/drivers/net/ethernet/sfc/selftest.c
+++ b/drivers/net/ethernet/sfc/selftest.c
@@ -188,7 +188,7 @@ static int efx_test_eventq_irq(struct efx_nic *efx,
schedule_timeout_uninterruptible(wait);
efx_for_each_channel(channel, efx) {
- napi_disable(&channel->napi_str);
+ efx_stop_eventq(channel);
if (channel->eventq_read_ptr !=
read_ptr[channel->channel]) {
set_bit(channel->channel, &napi_ran);
@@ -200,8 +200,7 @@ static int efx_test_eventq_irq(struct efx_nic *efx,
if (efx_nic_event_test_irq_cpu(channel) >= 0)
clear_bit(channel->channel, &int_pend);
}
- napi_enable(&channel->napi_str);
- efx_nic_eventq_read_ack(channel);
+ efx_start_eventq(channel);
}
wait *= 2;
diff --git a/drivers/net/ethernet/sfc/siena.c b/drivers/net/ethernet/sfc/siena.c
index 50ffefed492c..ae696855f21a 100644
--- a/drivers/net/ethernet/sfc/siena.c
+++ b/drivers/net/ethernet/sfc/siena.c
@@ -424,6 +424,8 @@ static void siena_remove_nic(struct efx_nic *efx)
{ #ext_name, 64, 8 * MC_CMD_MAC_ ## mcdi_name }
#define SIENA_OTHER_STAT(ext_name) \
[SIENA_STAT_ ## ext_name] = { #ext_name, 0, 0 }
+#define GENERIC_SW_STAT(ext_name) \
+ [GENERIC_STAT_ ## ext_name] = { #ext_name, 0, 0 }
static const struct efx_hw_stat_desc siena_stat_desc[SIENA_STAT_COUNT] = {
SIENA_DMA_STAT(tx_bytes, TX_BYTES),
@@ -483,6 +485,8 @@ static const struct efx_hw_stat_desc siena_stat_desc[SIENA_STAT_COUNT] = {
SIENA_DMA_STAT(rx_length_error, RX_LENGTH_ERROR_PKTS),
SIENA_DMA_STAT(rx_internal_error, RX_INTERNAL_ERROR_PKTS),
SIENA_DMA_STAT(rx_nodesc_drop_cnt, RX_NODESC_DROPS),
+ GENERIC_SW_STAT(rx_nodesc_trunc),
+ GENERIC_SW_STAT(rx_noskb_drops),
};
static const unsigned long siena_stat_mask[] = {
[0 ... BITS_TO_LONGS(SIENA_STAT_COUNT) - 1] = ~0UL,
@@ -528,6 +532,7 @@ static int siena_try_update_nic_stats(struct efx_nic *efx)
efx_update_diff_stat(&stats[SIENA_STAT_rx_good_bytes],
stats[SIENA_STAT_rx_bytes] -
stats[SIENA_STAT_rx_bad_bytes]);
+ efx_update_sw_stats(efx, stats);
return 0;
}
@@ -554,7 +559,9 @@ static size_t siena_update_nic_stats(struct efx_nic *efx, u64 *full_stats,
core_stats->tx_packets = stats[SIENA_STAT_tx_packets];
core_stats->rx_bytes = stats[SIENA_STAT_rx_bytes];
core_stats->tx_bytes = stats[SIENA_STAT_tx_bytes];
- core_stats->rx_dropped = stats[SIENA_STAT_rx_nodesc_drop_cnt];
+ core_stats->rx_dropped = stats[SIENA_STAT_rx_nodesc_drop_cnt] +
+ stats[GENERIC_STAT_rx_nodesc_trunc] +
+ stats[GENERIC_STAT_rx_noskb_drops];
core_stats->multicast = stats[SIENA_STAT_rx_multicast];
core_stats->collisions = stats[SIENA_STAT_tx_collision];
core_stats->rx_length_errors =
diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c
index ede8dcca0ff3..65c220f8661d 100644
--- a/drivers/net/ethernet/sfc/tx.c
+++ b/drivers/net/ethernet/sfc/tx.c
@@ -189,18 +189,6 @@ struct efx_short_copy_buffer {
u8 buf[L1_CACHE_BYTES];
};
-/* Copy in explicit 64-bit writes. */
-static void efx_memcpy_64(void __iomem *dest, void *src, size_t len)
-{
- u64 *src64 = src;
- u64 __iomem *dest64 = dest;
- size_t l64 = len / 8;
- size_t i;
-
- for (i = 0; i < l64; i++)
- writeq(src64[i], &dest64[i]);
-}
-
/* Copy to PIO, respecting that writes to PIO buffers must be dword aligned.
* Advances piobuf pointer. Leaves additional data in the copy buffer.
*/
@@ -210,7 +198,7 @@ static void efx_memcpy_toio_aligned(struct efx_nic *efx, u8 __iomem **piobuf,
{
int block_len = len & ~(sizeof(copy_buf->buf) - 1);
- efx_memcpy_64(*piobuf, data, block_len);
+ __iowrite64_copy(*piobuf, data, block_len >> 3);
*piobuf += block_len;
len -= block_len;
@@ -242,7 +230,8 @@ static void efx_memcpy_toio_aligned_cb(struct efx_nic *efx, u8 __iomem **piobuf,
if (copy_buf->used < sizeof(copy_buf->buf))
return;
- efx_memcpy_64(*piobuf, copy_buf->buf, sizeof(copy_buf->buf));
+ __iowrite64_copy(*piobuf, copy_buf->buf,
+ sizeof(copy_buf->buf) >> 3);
*piobuf += sizeof(copy_buf->buf);
data += copy_to_buf;
len -= copy_to_buf;
@@ -257,7 +246,8 @@ static void efx_flush_copy_buffer(struct efx_nic *efx, u8 __iomem *piobuf,
{
/* if there's anything in it, write the whole buffer, including junk */
if (copy_buf->used)
- efx_memcpy_64(piobuf, copy_buf->buf, sizeof(copy_buf->buf));
+ __iowrite64_copy(piobuf, copy_buf->buf,
+ sizeof(copy_buf->buf) >> 3);
}
/* Traverse skb structure and copy fragments in to PIO buffer.
@@ -316,8 +306,8 @@ efx_enqueue_skb_pio(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
*/
BUILD_BUG_ON(L1_CACHE_BYTES >
SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
- efx_memcpy_64(tx_queue->piobuf, skb->data,
- ALIGN(skb->len, L1_CACHE_BYTES));
+ __iowrite64_copy(tx_queue->piobuf, skb->data,
+ ALIGN(skb->len, L1_CACHE_BYTES) >> 3);
}
EFX_POPULATE_QWORD_5(buffer->option,
@@ -452,6 +442,8 @@ finish_packet:
/* Pass off to hardware */
efx_nic_push_buffers(tx_queue);
+ tx_queue->tx_packets++;
+
efx_tx_maybe_stop_queue(tx_queue);
return NETDEV_TX_OK;
@@ -1245,6 +1237,8 @@ static int tso_start_new_packet(struct efx_tx_queue *tx_queue,
++tx_queue->tso_packets;
+ ++tx_queue->tx_packets;
+
return 0;
}
generated by cgit v1.2.3 (git 2.39.0) at 2025-01-08 07:33:21 +0300