summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/net/ethernet/sfc/Makefile2
-rw-r--r--drivers/net/ethernet/sfc/ef100_rx.c23
-rw-r--r--drivers/net/ethernet/sfc/efx_channels.c9
-rw-r--r--drivers/net/ethernet/sfc/mae.c170
-rw-r--r--drivers/net/ethernet/sfc/mae.h7
-rw-r--r--drivers/net/ethernet/sfc/mae_counter_format.h73
-rw-r--r--drivers/net/ethernet/sfc/mcdi.h5
-rw-r--r--drivers/net/ethernet/sfc/net_driver.h17
-rw-r--r--drivers/net/ethernet/sfc/rx_common.c3
-rw-r--r--drivers/net/ethernet/sfc/tc.c122
-rw-r--r--drivers/net/ethernet/sfc/tc.h16
-rw-r--r--drivers/net/ethernet/sfc/tc_counters.c501
-rw-r--r--drivers/net/ethernet/sfc/tc_counters.h59
13 files changed, 998 insertions, 9 deletions
diff --git a/drivers/net/ethernet/sfc/Makefile b/drivers/net/ethernet/sfc/Makefile
index b5e45fc6337e..712a48d00069 100644
--- a/drivers/net/ethernet/sfc/Makefile
+++ b/drivers/net/ethernet/sfc/Makefile
@@ -9,7 +9,7 @@ sfc-y += efx.o efx_common.o efx_channels.o nic.o \
ef100_ethtool.o ef100_rx.o ef100_tx.o
sfc-$(CONFIG_SFC_MTD) += mtd.o
sfc-$(CONFIG_SFC_SRIOV) += sriov.o ef10_sriov.o ef100_sriov.o ef100_rep.o \
- mae.o tc.o tc_bindings.o
+ mae.o tc.o tc_bindings.o tc_counters.o
obj-$(CONFIG_SFC) += sfc.o
diff --git a/drivers/net/ethernet/sfc/ef100_rx.c b/drivers/net/ethernet/sfc/ef100_rx.c
index 65bbe37753e6..83d9db71d7d7 100644
--- a/drivers/net/ethernet/sfc/ef100_rx.c
+++ b/drivers/net/ethernet/sfc/ef100_rx.c
@@ -21,7 +21,7 @@
/* Get the value of a field in the RX prefix */
#define PREFIX_OFFSET_W(_f) (ESF_GZ_RX_PREFIX_ ## _f ## _LBN / 32)
#define PREFIX_OFFSET_B(_f) (ESF_GZ_RX_PREFIX_ ## _f ## _LBN % 32)
-#define PREFIX_WIDTH_MASK(_f) ((1UL << ESF_GZ_RX_PREFIX_ ## _f ## _WIDTH) - 1)
+#define PREFIX_WIDTH_MASK(_f) ((1ULL << ESF_GZ_RX_PREFIX_ ## _f ## _WIDTH) - 1)
#define PREFIX_WORD(_p, _f) le32_to_cpu((__force __le32)(_p)[PREFIX_OFFSET_W(_f)])
#define PREFIX_FIELD(_p, _f) ((PREFIX_WORD(_p, _f) >> PREFIX_OFFSET_B(_f)) & \
PREFIX_WIDTH_MASK(_f))
@@ -67,6 +67,13 @@ void __ef100_rx_packet(struct efx_channel *channel)
prefix = (u32 *)(eh - ESE_GZ_RX_PKT_PREFIX_LEN);
+ if (channel->type->receive_raw) {
+ u32 mark = PREFIX_FIELD(prefix, USER_MARK);
+
+ if (channel->type->receive_raw(rx_queue, mark))
+ return; /* packet was consumed */
+ }
+
if (ef100_has_fcs_error(channel, prefix) &&
unlikely(!(efx->net_dev->features & NETIF_F_RXALL)))
goto out;
@@ -183,24 +190,32 @@ void efx_ef100_ev_rx(struct efx_channel *channel, const efx_qword_t *p_event)
void ef100_rx_write(struct efx_rx_queue *rx_queue)
{
+ unsigned int notified_count = rx_queue->notified_count;
struct efx_rx_buffer *rx_buf;
unsigned int idx;
efx_qword_t *rxd;
efx_dword_t rxdb;
- while (rx_queue->notified_count != rx_queue->added_count) {
- idx = rx_queue->notified_count & rx_queue->ptr_mask;
+ while (notified_count != rx_queue->added_count) {
+ idx = notified_count & rx_queue->ptr_mask;
rx_buf = efx_rx_buffer(rx_queue, idx);
rxd = efx_rx_desc(rx_queue, idx);
EFX_POPULATE_QWORD_1(*rxd, ESF_GZ_RX_BUF_ADDR, rx_buf->dma_addr);
- ++rx_queue->notified_count;
+ ++notified_count;
}
+ if (notified_count == rx_queue->notified_count)
+ return;
wmb();
EFX_POPULATE_DWORD_1(rxdb, ERF_GZ_RX_RING_PIDX,
rx_queue->added_count & rx_queue->ptr_mask);
efx_writed_page(rx_queue->efx, &rxdb,
ER_GZ_RX_RING_DOORBELL, efx_rx_queue_index(rx_queue));
+ if (rx_queue->grant_credits)
+ wmb();
+ rx_queue->notified_count = notified_count;
+ if (rx_queue->grant_credits)
+ schedule_work(&rx_queue->grant_work);
}
diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c
index aaa381743bca..fcea3ea809d7 100644
--- a/drivers/net/ethernet/sfc/efx_channels.c
+++ b/drivers/net/ethernet/sfc/efx_channels.c
@@ -1119,6 +1119,8 @@ void efx_start_channels(struct efx_nic *efx)
struct efx_channel *channel;
efx_for_each_channel_rev(channel, efx) {
+ if (channel->type->start)
+ channel->type->start(channel);
efx_for_each_channel_tx_queue(tx_queue, channel) {
efx_init_tx_queue(tx_queue);
atomic_inc(&efx->active_queues);
@@ -1143,8 +1145,13 @@ void efx_stop_channels(struct efx_nic *efx)
struct efx_channel *channel;
int rc = 0;
- /* Stop RX refill */
+ /* Stop special channels and RX refill.
+ * The channel's stop has to be called first, since it might wait
+ * for a sentinel RX to indicate the channel has fully drained.
+ */
efx_for_each_channel(channel, efx) {
+ if (channel->type->stop)
+ channel->type->stop(channel);
efx_for_each_channel_rx_queue(rx_queue, channel)
rx_queue->refill_enabled = false;
}
diff --git a/drivers/net/ethernet/sfc/mae.c b/drivers/net/ethernet/sfc/mae.c
index 1e605e2a08c5..583baf69981c 100644
--- a/drivers/net/ethernet/sfc/mae.c
+++ b/drivers/net/ethernet/sfc/mae.c
@@ -112,6 +112,117 @@ int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id)
return 0;
}
+int efx_mae_start_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue)
+{
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_LEN);
+ MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTERS_STREAM_START_OUT_LEN);
+ u32 out_flags;
+ size_t outlen;
+ int rc;
+
+ MCDI_SET_WORD(inbuf, MAE_COUNTERS_STREAM_START_V2_IN_QID,
+ efx_rx_queue_index(rx_queue));
+ MCDI_SET_WORD(inbuf, MAE_COUNTERS_STREAM_START_V2_IN_PACKET_SIZE,
+ efx->net_dev->mtu);
+ MCDI_SET_DWORD(inbuf, MAE_COUNTERS_STREAM_START_V2_IN_COUNTER_TYPES_MASK,
+ BIT(MAE_COUNTER_TYPE_AR) | BIT(MAE_COUNTER_TYPE_CT) |
+ BIT(MAE_COUNTER_TYPE_OR));
+ rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTERS_STREAM_START,
+ inbuf, sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);
+ if (rc)
+ return rc;
+ if (outlen < sizeof(outbuf))
+ return -EIO;
+ out_flags = MCDI_DWORD(outbuf, MAE_COUNTERS_STREAM_START_OUT_FLAGS);
+ if (out_flags & BIT(MC_CMD_MAE_COUNTERS_STREAM_START_OUT_USES_CREDITS_OFST)) {
+ netif_dbg(efx, drv, efx->net_dev,
+ "MAE counter stream uses credits\n");
+ rx_queue->grant_credits = true;
+ out_flags &= ~BIT(MC_CMD_MAE_COUNTERS_STREAM_START_OUT_USES_CREDITS_OFST);
+ }
+ if (out_flags) {
+ netif_err(efx, drv, efx->net_dev,
+ "MAE counter stream start: unrecognised flags %x\n",
+ out_flags);
+ goto out_stop;
+ }
+ return 0;
+out_stop:
+ efx_mae_stop_counters(efx, rx_queue);
+ return -EOPNOTSUPP;
+}
+
+static bool efx_mae_counters_flushed(u32 *flush_gen, u32 *seen_gen)
+{
+ int i;
+
+ for (i = 0; i < EFX_TC_COUNTER_TYPE_MAX; i++)
+ if ((s32)(flush_gen[i] - seen_gen[i]) > 0)
+ return false;
+ return true;
+}
+
+int efx_mae_stop_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue)
+{
+ MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_LENMAX);
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTERS_STREAM_STOP_IN_LEN);
+ size_t outlen;
+ int rc, i;
+
+ MCDI_SET_WORD(inbuf, MAE_COUNTERS_STREAM_STOP_IN_QID,
+ efx_rx_queue_index(rx_queue));
+ rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTERS_STREAM_STOP,
+ inbuf, sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);
+
+ if (rc)
+ return rc;
+
+ netif_dbg(efx, drv, efx->net_dev, "Draining counters:\n");
+ /* Only process received generation counts */
+ for (i = 0; (i < (outlen / 4)) && (i < EFX_TC_COUNTER_TYPE_MAX); i++) {
+ efx->tc->flush_gen[i] = MCDI_ARRAY_DWORD(outbuf,
+ MAE_COUNTERS_STREAM_STOP_V2_OUT_GENERATION_COUNT,
+ i);
+ netif_dbg(efx, drv, efx->net_dev,
+ "\ttype %u, awaiting gen %u\n", i,
+ efx->tc->flush_gen[i]);
+ }
+
+ efx->tc->flush_counters = true;
+
+ /* Drain can take up to 2 seconds owing to FWRIVERHD-2884; whatever
+ * timeout we use, that delay is added to unload on nonresponsive
+ * hardware, so 2500ms seems like a reasonable compromise.
+ */
+ if (!wait_event_timeout(efx->tc->flush_wq,
+ efx_mae_counters_flushed(efx->tc->flush_gen,
+ efx->tc->seen_gen),
+ msecs_to_jiffies(2500)))
+ netif_warn(efx, drv, efx->net_dev,
+ "Failed to drain counters RXQ, FW may be unhappy\n");
+
+ efx->tc->flush_counters = false;
+
+ return rc;
+}
+
+void efx_mae_counters_grant_credits(struct work_struct *work)
+{
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_LEN);
+ struct efx_rx_queue *rx_queue = container_of(work, struct efx_rx_queue,
+ grant_work);
+ struct efx_nic *efx = rx_queue->efx;
+ unsigned int credits;
+
+ BUILD_BUG_ON(MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_OUT_LEN);
+ credits = READ_ONCE(rx_queue->notified_count) - rx_queue->granted_count;
+ MCDI_SET_DWORD(inbuf, MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_NUM_CREDITS,
+ credits);
+ if (!efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS,
+ inbuf, sizeof(inbuf), NULL, 0, NULL))
+ rx_queue->granted_count += credits;
+}
+
static int efx_mae_get_basic_caps(struct efx_nic *efx, struct mae_caps *caps)
{
MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_GET_CAPS_OUT_LEN);
@@ -323,6 +434,57 @@ int efx_mae_match_check_caps(struct efx_nic *efx,
#undef CHECK_BIT
#undef CHECK
+int efx_mae_allocate_counter(struct efx_nic *efx, struct efx_tc_counter *cnt)
+{
+ MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTER_ALLOC_OUT_LEN(1));
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTER_ALLOC_V2_IN_LEN);
+ size_t outlen;
+ int rc;
+
+ if (!cnt)
+ return -EINVAL;
+
+ MCDI_SET_DWORD(inbuf, MAE_COUNTER_ALLOC_V2_IN_REQUESTED_COUNT, 1);
+ MCDI_SET_DWORD(inbuf, MAE_COUNTER_ALLOC_V2_IN_COUNTER_TYPE, cnt->type);
+ rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTER_ALLOC, inbuf, sizeof(inbuf),
+ outbuf, sizeof(outbuf), &outlen);
+ if (rc)
+ return rc;
+ /* pcol says this can't happen, since count is 1 */
+ if (outlen < sizeof(outbuf))
+ return -EIO;
+ cnt->fw_id = MCDI_DWORD(outbuf, MAE_COUNTER_ALLOC_OUT_COUNTER_ID);
+ cnt->gen = MCDI_DWORD(outbuf, MAE_COUNTER_ALLOC_OUT_GENERATION_COUNT);
+ return 0;
+}
+
+int efx_mae_free_counter(struct efx_nic *efx, struct efx_tc_counter *cnt)
+{
+ MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTER_FREE_OUT_LEN(1));
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTER_FREE_V2_IN_LEN);
+ size_t outlen;
+ int rc;
+
+ MCDI_SET_DWORD(inbuf, MAE_COUNTER_FREE_V2_IN_COUNTER_ID_COUNT, 1);
+ MCDI_SET_DWORD(inbuf, MAE_COUNTER_FREE_V2_IN_FREE_COUNTER_ID, cnt->fw_id);
+ MCDI_SET_DWORD(inbuf, MAE_COUNTER_FREE_V2_IN_COUNTER_TYPE, cnt->type);
+ rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTER_FREE, inbuf, sizeof(inbuf),
+ outbuf, sizeof(outbuf), &outlen);
+ if (rc)
+ return rc;
+ /* pcol says this can't happen, since count is 1 */
+ if (outlen < sizeof(outbuf))
+ return -EIO;
+ /* FW freed a different ID than we asked for, should also never happen.
+ * Warn because it means we've now got a different idea to the FW of
+ * what counters exist, which could cause mayhem later.
+ */
+ if (WARN_ON(MCDI_DWORD(outbuf, MAE_COUNTER_FREE_OUT_FREED_COUNTER_ID) !=
+ cnt->fw_id))
+ return -EIO;
+ return 0;
+}
+
static bool efx_mae_asl_id(u32 id)
{
return !!(id & BIT(31));
@@ -339,8 +501,12 @@ int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act)
MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_NULL);
MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_DST_MAC_ID,
MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_NULL);
- MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID,
- MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NULL);
+ if (act->count && !WARN_ON(!act->count->cnt))
+ MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID,
+ act->count->cnt->fw_id);
+ else
+ MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID,
+ MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NULL);
MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_LIST_ID,
MC_CMD_MAE_COUNTER_LIST_ALLOC_OUT_COUNTER_LIST_ID_NULL);
MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_ENCAP_HEADER_ID,
diff --git a/drivers/net/ethernet/sfc/mae.h b/drivers/net/ethernet/sfc/mae.h
index 3e0cd238d523..72343e90e222 100644
--- a/drivers/net/ethernet/sfc/mae.h
+++ b/drivers/net/ethernet/sfc/mae.h
@@ -27,6 +27,10 @@ void efx_mae_mport_mport(struct efx_nic *efx, u32 mport_id, u32 *out);
int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id);
+int efx_mae_start_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue);
+int efx_mae_stop_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue);
+void efx_mae_counters_grant_credits(struct work_struct *work);
+
#define MAE_NUM_FIELDS (MAE_FIELD_ENC_VNET_ID + 1)
struct mae_caps {
@@ -41,6 +45,9 @@ int efx_mae_match_check_caps(struct efx_nic *efx,
const struct efx_tc_match_fields *mask,
struct netlink_ext_ack *extack);
+int efx_mae_allocate_counter(struct efx_nic *efx, struct efx_tc_counter *cnt);
+int efx_mae_free_counter(struct efx_nic *efx, struct efx_tc_counter *cnt);
+
int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act);
int efx_mae_free_action_set(struct efx_nic *efx, u32 fw_id);
diff --git a/drivers/net/ethernet/sfc/mae_counter_format.h b/drivers/net/ethernet/sfc/mae_counter_format.h
new file mode 100644
index 000000000000..7e252e393fbe
--- /dev/null
+++ b/drivers/net/ethernet/sfc/mae_counter_format.h
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/****************************************************************************
+ * Driver for Solarflare network controllers and boards
+ * Copyright 2020 Xilinx, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+/* Format of counter packets (version 2) from the ef100 Match-Action Engine */
+
+#ifndef EFX_MAE_COUNTER_FORMAT_H
+#define EFX_MAE_COUNTER_FORMAT_H
+
+
+/*------------------------------------------------------------*/
+/*
+ * ER_RX_SL_PACKETISER_HEADER_WORD(160bit):
+ *
+ */
+#define ER_RX_SL_PACKETISER_HEADER_WORD_SIZE 20
+#define ER_RX_SL_PACKETISER_HEADER_WORD_WIDTH 160
+
+#define ERF_SC_PACKETISER_HEADER_VERSION_LBN 0
+#define ERF_SC_PACKETISER_HEADER_VERSION_WIDTH 8
+#define ERF_SC_PACKETISER_HEADER_VERSION_VALUE 2
+#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_LBN 8
+#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_WIDTH 8
+#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_AR 0
+#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_CT 1
+#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_OR 2
+#define ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_LBN 16
+#define ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_WIDTH 8
+#define ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_DEFAULT 0x4
+#define ERF_SC_PACKETISER_HEADER_PAYLOAD_OFFSET_LBN 24
+#define ERF_SC_PACKETISER_HEADER_PAYLOAD_OFFSET_WIDTH 8
+#define ERF_SC_PACKETISER_HEADER_PAYLOAD_OFFSET_DEFAULT 0x14
+#define ERF_SC_PACKETISER_HEADER_INDEX_LBN 32
+#define ERF_SC_PACKETISER_HEADER_INDEX_WIDTH 16
+#define ERF_SC_PACKETISER_HEADER_COUNT_LBN 48
+#define ERF_SC_PACKETISER_HEADER_COUNT_WIDTH 16
+#define ERF_SC_PACKETISER_HEADER_RESERVED_0_LBN 64
+#define ERF_SC_PACKETISER_HEADER_RESERVED_0_WIDTH 32
+#define ERF_SC_PACKETISER_HEADER_RESERVED_1_LBN 96
+#define ERF_SC_PACKETISER_HEADER_RESERVED_1_WIDTH 32
+#define ERF_SC_PACKETISER_HEADER_RESERVED_2_LBN 128
+#define ERF_SC_PACKETISER_HEADER_RESERVED_2_WIDTH 32
+
+
+/*------------------------------------------------------------*/
+/*
+ * ER_RX_SL_PACKETISER_PAYLOAD_WORD(128bit):
+ *
+ */
+#define ER_RX_SL_PACKETISER_PAYLOAD_WORD_SIZE 16
+#define ER_RX_SL_PACKETISER_PAYLOAD_WORD_WIDTH 128
+
+#define ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_LBN 0
+#define ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_WIDTH 24
+#define ERF_SC_PACKETISER_PAYLOAD_RESERVED_LBN 24
+#define ERF_SC_PACKETISER_PAYLOAD_RESERVED_WIDTH 8
+#define ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_OFST 4
+#define ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_SIZE 6
+#define ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_LBN 32
+#define ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_WIDTH 48
+#define ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_OFST 10
+#define ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_SIZE 6
+#define ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_LBN 80
+#define ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_WIDTH 48
+
+
+#endif /* EFX_MAE_COUNTER_FORMAT_H */
diff --git a/drivers/net/ethernet/sfc/mcdi.h b/drivers/net/ethernet/sfc/mcdi.h
index fbeb58104936..7e35fec9da35 100644
--- a/drivers/net/ethernet/sfc/mcdi.h
+++ b/drivers/net/ethernet/sfc/mcdi.h
@@ -221,6 +221,11 @@ void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev);
#define MCDI_BYTE(_buf, _field) \
((void)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 1), \
*MCDI_PTR(_buf, _field))
+#define MCDI_SET_WORD(_buf, _field, _value) do { \
+ BUILD_BUG_ON(MC_CMD_ ## _field ## _LEN != 2); \
+ BUILD_BUG_ON(MC_CMD_ ## _field ## _OFST & 1); \
+ *(__force __le16 *)MCDI_PTR(_buf, _field) = cpu_to_le16(_value);\
+ } while (0)
#define MCDI_WORD(_buf, _field) \
((u16)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 2) + \
le16_to_cpu(*(__force const __le16 *)MCDI_PTR(_buf, _field)))
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index 7ef823d7a89a..3b49e216768b 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -56,7 +56,8 @@
#define EFX_MAX_RX_QUEUES EFX_MAX_CHANNELS
#define EFX_EXTRA_CHANNEL_IOV 0
#define EFX_EXTRA_CHANNEL_PTP 1
-#define EFX_MAX_EXTRA_CHANNELS 2U
+#define EFX_EXTRA_CHANNEL_TC 2
+#define EFX_MAX_EXTRA_CHANNELS 3U
/* Checksum generation is a per-queue option in hardware, so each
* queue visible to the networking core is backed by two hardware TX
@@ -363,8 +364,12 @@ struct efx_rx_page_state {
* @refill_enabled: Enable refill whenever fill level is low
* @flush_pending: Set when a RX flush is pending. Has the same lifetime as
* @rxq_flush_pending.
+ * @grant_credits: Posted RX descriptors need to be granted to the MAE with
+ * %MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS. For %EFX_EXTRA_CHANNEL_TC,
+ * and only supported on EF100.
* @added_count: Number of buffers added to the receive queue.
* @notified_count: Number of buffers given to NIC (<= @added_count).
+ * @granted_count: Number of buffers granted to the MAE (<= @notified_count).
* @removed_count: Number of buffers removed from the receive queue.
* @scatter_n: Used by NIC specific receive code.
* @scatter_len: Used by NIC specific receive code.
@@ -385,6 +390,7 @@ struct efx_rx_page_state {
* refill was triggered.
* @recycle_count: RX buffer recycle counter.
* @slow_fill: Timer used to defer efx_nic_generate_fill_event().
+ * @grant_work: workitem used to grant credits to the MAE if @grant_credits
* @xdp_rxq_info: XDP specific RX queue information.
* @xdp_rxq_info_valid: Is xdp_rxq_info valid data?.
*/
@@ -396,9 +402,11 @@ struct efx_rx_queue {
unsigned int ptr_mask;
bool refill_enabled;
bool flush_pending;
+ bool grant_credits;
unsigned int added_count;
unsigned int notified_count;
+ unsigned int granted_count;
unsigned int removed_count;
unsigned int scatter_n;
unsigned int scatter_len;
@@ -416,6 +424,7 @@ struct efx_rx_queue {
unsigned int recycle_count;
struct timer_list slow_fill;
unsigned int slow_fill_count;
+ struct work_struct grant_work;
/* Statistics to supplement MAC stats */
unsigned long rx_packets;
struct xdp_rxq_info xdp_rxq_info;
@@ -577,12 +586,15 @@ struct efx_msi_context {
* struct efx_channel_type - distinguishes traffic and extra channels
* @handle_no_channel: Handle failure to allocate an extra channel
* @pre_probe: Set up extra state prior to initialisation
+ * @start: called early in efx_start_channels()
+ * @stop: called early in efx_stop_channels()
* @post_remove: Tear down extra state after finalisation, if allocated.
* May be called on channels that have not been probed.
* @get_name: Generate the channel's name (used for its IRQ handler)
* @copy: Copy the channel state prior to reallocation. May be %NULL if
* reallocation is not supported.
* @receive_skb: Handle an skb ready to be passed to netif_receive_skb()
+ * @receive_raw: Handle an RX buffer ready to be passed to __efx_rx_packet()
* @want_txqs: Determine whether this channel should have TX queues
* created. If %NULL, TX queues are not created.
* @keep_eventq: Flag for whether event queue should be kept initialised
@@ -593,10 +605,13 @@ struct efx_msi_context {
struct efx_channel_type {
void (*handle_no_channel)(struct efx_nic *);
int (*pre_probe)(struct efx_channel *);
+ int (*start)(struct efx_channel *);
+ void (*stop)(struct efx_channel *);
void (*post_remove)(struct efx_channel *);
void (*get_name)(struct efx_channel *, char *buf, size_t len);
struct efx_channel *(*copy)(const struct efx_channel *);
bool (*receive_skb)(struct efx_channel *, struct sk_buff *);
+ bool (*receive_raw)(struct efx_rx_queue *, u32);
bool (*want_txqs)(struct efx_channel *);
bool keep_eventq;
bool want_pio;
diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c
index 9220afeddee8..d2f35ee15eff 100644
--- a/drivers/net/ethernet/sfc/rx_common.c
+++ b/drivers/net/ethernet/sfc/rx_common.c
@@ -229,6 +229,7 @@ void efx_init_rx_queue(struct efx_rx_queue *rx_queue)
/* Initialise ptr fields */
rx_queue->added_count = 0;
rx_queue->notified_count = 0;
+ rx_queue->granted_count = 0;
rx_queue->removed_count = 0;
rx_queue->min_fill = -1U;
efx_init_rx_recycle_ring(rx_queue);
@@ -281,6 +282,8 @@ void efx_fini_rx_queue(struct efx_rx_queue *rx_queue)
"shutting down RX queue %d\n", efx_rx_queue_index(rx_queue));
del_timer_sync(&rx_queue->slow_fill);
+ if (rx_queue->grant_credits)
+ flush_work(&rx_queue->grant_work);
/* Release RX buffers from the current read ptr to the write ptr */
if (rx_queue->buffer) {
diff --git a/drivers/net/ethernet/sfc/tc.c b/drivers/net/ethernet/sfc/tc.c
index 17e1a3447554..deeaab9ee761 100644
--- a/drivers/net/ethernet/sfc/tc.c
+++ b/drivers/net/ethernet/sfc/tc.c
@@ -77,6 +77,8 @@ static void efx_tc_free_action_set(struct efx_nic *efx,
*/
list_del(&act->list);
}
+ if (act->count)
+ efx_tc_flower_put_counter_index(efx, act->count);
kfree(act);
}
@@ -282,6 +284,29 @@ static int efx_tc_flower_parse_match(struct efx_nic *efx,
return 0;
}
+/* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */
+enum efx_tc_action_order {
+ EFX_TC_AO_COUNT,
+ EFX_TC_AO_DELIVER
+};
+/* Determine whether we can add @new action without violating order */
+static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act,
+ enum efx_tc_action_order new)
+{
+ switch (new) {
+ case EFX_TC_AO_COUNT:
+ if (act->count)
+ return false;
+ fallthrough;
+ case EFX_TC_AO_DELIVER:
+ return !act->deliver;
+ default:
+ /* Bad caller. Whatever they wanted to do, say they can't. */
+ WARN_ON_ONCE(1);
+ return false;
+ }
+}
+
static int efx_tc_flower_replace(struct efx_nic *efx,
struct net_device *net_dev,
struct flow_cls_offload *tc,
@@ -376,6 +401,47 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
goto release;
}
+ if ((fa->id == FLOW_ACTION_REDIRECT ||
+ fa->id == FLOW_ACTION_MIRRED ||
+ fa->id == FLOW_ACTION_DROP) && fa->hw_stats) {
+ struct efx_tc_counter_index *ctr;
+
+ /* Currently the only actions that want stats are
+ * mirred and gact (ok, shot, trap, goto-chain), which
+ * means we want stats just before delivery. Also,
+ * note that tunnel_key set shouldn't change the length
+ * — it's only the subsequent mirred that does that,
+ * and the stats are taken _before_ the mirred action
+ * happens.
+ */
+ if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {
+ /* All supported actions that count either steal
+ * (gact shot, mirred redirect) or clone act
+ * (mirred mirror), so we should never get two
+ * count actions on one action_set.
+ */
+ NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)");
+ rc = -EOPNOTSUPP;
+ goto release;
+ }
+
+ if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
+ NL_SET_ERR_MSG_FMT_MOD(extack, "hw_stats_type %u not supported (only 'delayed')",
+ fa->hw_stats);
+ rc = -EOPNOTSUPP;
+ goto release;
+ }
+
+ ctr = efx_tc_flower_get_counter_index(efx, tc->cookie,
+ EFX_TC_COUNTER_TYPE_AR);
+ if (IS_ERR(ctr)) {
+ rc = PTR_ERR(ctr);
+ NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
+ goto release;
+ }
+ act->count = ctr;
+ }
+
switch (fa->id) {
case FLOW_ACTION_DROP:
rc = efx_mae_alloc_action_set(efx, act);
@@ -389,6 +455,14 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
case FLOW_ACTION_REDIRECT:
case FLOW_ACTION_MIRRED:
save = *act;
+
+ if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {
+ /* can't happen */
+ rc = -EOPNOTSUPP;
+ NL_SET_ERR_MSG_MOD(extack, "Deliver action violates action order (can't happen)");
+ goto release;
+ }
+
to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
if (IS_ERR(to_efv)) {
NL_SET_ERR_MSG_MOD(extack, "Mirred egress device not on switch");
@@ -412,6 +486,7 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
if (fa->id == FLOW_ACTION_REDIRECT)
break; /* end of the line */
/* Mirror, so continue on with saved act */
+ save.count = NULL;
act = kzalloc(sizeof(*act), GFP_USER);
if (!act) {
rc = -ENOMEM;
@@ -520,6 +595,42 @@ static int efx_tc_flower_destroy(struct efx_nic *efx,
return 0;
}
+static int efx_tc_flower_stats(struct efx_nic *efx, struct net_device *net_dev,
+ struct flow_cls_offload *tc)
+{
+ struct netlink_ext_ack *extack = tc->common.extack;
+ struct efx_tc_counter_index *ctr;
+ struct efx_tc_counter *cnt;
+ u64 packets, bytes;
+
+ ctr = efx_tc_flower_find_counter_index(efx, tc->cookie);
+ if (!ctr) {
+ /* See comment in efx_tc_flower_destroy() */
+ if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))
+ if (net_ratelimit())
+ netif_warn(efx, drv, efx->net_dev,
+ "Filter %lx not found for stats\n",
+ tc->cookie);
+ NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
+ return -ENOENT;
+ }
+ if (WARN_ON(!ctr->cnt)) /* can't happen */
+ return -EIO;
+ cnt = ctr->cnt;
+
+ spin_lock_bh(&cnt->lock);
+ /* Report only new pkts/bytes since last time TC asked */
+ packets = cnt->packets;
+ bytes = cnt->bytes;
+ flow_stats_update(&tc->stats, bytes - cnt->old_bytes,
+ packets - cnt->old_packets, 0, cnt->touched,
+ FLOW_ACTION_HW_STATS_DELAYED);
+ cnt->old_packets = packets;
+ cnt->old_bytes = bytes;
+ spin_unlock_bh(&cnt->lock);
+ return 0;
+}
+
int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev,
struct flow_cls_offload *tc, struct efx_rep *efv)
{
@@ -536,6 +647,9 @@ int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev,
case FLOW_CLS_DESTROY:
rc = efx_tc_flower_destroy(efx, net_dev, tc);
break;
+ case FLOW_CLS_STATS:
+ rc = efx_tc_flower_stats(efx, net_dev, tc);
+ break;
default:
rc = -EOPNOTSUPP;
break;
@@ -751,6 +865,10 @@ int efx_init_struct_tc(struct efx_nic *efx)
INIT_LIST_HEAD(&efx->tc->block_list);
mutex_init(&efx->tc->mutex);
+ init_waitqueue_head(&efx->tc->flush_wq);
+ rc = efx_tc_init_counters(efx);
+ if (rc < 0)
+ goto fail_counters;
rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params);
if (rc < 0)
goto fail_match_action_ht;
@@ -760,8 +878,11 @@ int efx_init_struct_tc(struct efx_nic *efx)
efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list);
efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
+ efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type;
return 0;
fail_match_action_ht:
+ efx_tc_destroy_counters(efx);
+fail_counters:
mutex_destroy(&efx->tc->mutex);
kfree(efx->tc->caps);
fail_alloc_caps:
@@ -782,6 +903,7 @@ void efx_fini_struct_tc(struct efx_nic *efx)
MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free,
efx);
+ efx_tc_fini_counters(efx);
mutex_unlock(&efx->tc->mutex);
mutex_destroy(&efx->tc->mutex);
kfree(efx->tc->caps);
diff --git a/drivers/net/ethernet/sfc/tc.h b/drivers/net/ethernet/sfc/tc.h
index 4240c375a8e6..418ce8c13a06 100644
--- a/drivers/net/ethernet/sfc/tc.h
+++ b/drivers/net/ethernet/sfc/tc.h
@@ -14,11 +14,13 @@
#include <net/flow_offload.h>
#include <linux/rhashtable.h>
#include "net_driver.h"
+#include "tc_counters.h"
#define IS_ALL_ONES(v) (!(typeof (v))~(v))
struct efx_tc_action_set {
u16 deliver:1;
+ struct efx_tc_counter_index *count;
u32 dest_mport;
u32 fw_id; /* index of this entry in firmware actions table */
struct list_head list;
@@ -74,11 +76,19 @@ enum efx_tc_rule_prios {
* @caps: MAE capabilities reported by MCDI
* @block_list: List of &struct efx_tc_block_binding
* @mutex: Used to serialise operations on TC hashtables
+ * @counter_ht: Hashtable of TC counters (FW IDs and counter values)
+ * @counter_id_ht: Hashtable mapping TC counter cookies to counters
* @match_action_ht: Hashtable of TC match-action rules
* @reps_mport_id: MAE port allocated for representor RX
* @reps_filter_uc: VNIC filter for representor unicast RX (promisc)
* @reps_filter_mc: VNIC filter for representor multicast RX (allmulti)
* @reps_mport_vport_id: vport_id for representor RX filters
+ * @flush_counters: counters have been stopped, waiting for drain
+ * @flush_gen: final generation count per type array as reported by
+ * MC_CMD_MAE_COUNTERS_STREAM_STOP
+ * @seen_gen: most recent generation count per type as seen by efx_tc_rx()
+ * @flush_wq: wait queue used by efx_mae_stop_counters() to wait for
+ * MAE counters RXQ to finish draining
* @dflt: Match-action rules for default switching; at priority
* %EFX_TC_PRIO_DFLT. Named by *ingress* port
* @dflt.pf: rule for traffic ingressing from PF (egresses to wire)
@@ -89,9 +99,15 @@ struct efx_tc_state {
struct mae_caps *caps;
struct list_head block_list;
struct mutex mutex;
+ struct rhashtable counter_ht;
+ struct rhashtable counter_id_ht;
struct rhashtable match_action_ht;
u32 reps_mport_id, reps_mport_vport_id;
s32 reps_filter_uc, reps_filter_mc;
+ bool flush_counters;
+ u32 flush_gen[EFX_TC_COUNTER_TYPE_MAX];
+ u32 seen_gen[EFX_TC_COUNTER_TYPE_MAX];
+ wait_queue_head_t flush_wq;
struct {
struct efx_tc_flow_rule pf;
struct efx_tc_flow_rule wire;
diff --git a/drivers/net/ethernet/sfc/tc_counters.c b/drivers/net/ethernet/sfc/tc_counters.c
new file mode 100644
index 000000000000..2bba5d3a2fdb
--- /dev/null
+++ b/drivers/net/ethernet/sfc/tc_counters.c
@@ -0,0 +1,501 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/****************************************************************************
+ * Driver for Solarflare network controllers and boards
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#include "tc_counters.h"
+#include "mae_counter_format.h"
+#include "mae.h"
+#include "rx_common.h"
+
+/* Counter-management hashtables */
+
+static const struct rhashtable_params efx_tc_counter_id_ht_params = {
+ .key_len = offsetof(struct efx_tc_counter_index, linkage),
+ .key_offset = 0,
+ .head_offset = offsetof(struct efx_tc_counter_index, linkage),
+};
+
+static const struct rhashtable_params efx_tc_counter_ht_params = {
+ .key_len = offsetof(struct efx_tc_counter, linkage),
+ .key_offset = 0,
+ .head_offset = offsetof(struct efx_tc_counter, linkage),
+};
+
+static void efx_tc_counter_free(void *ptr, void *__unused)
+{
+ struct efx_tc_counter *cnt = ptr;
+
+ kfree(cnt);
+}
+
+static void efx_tc_counter_id_free(void *ptr, void *__unused)
+{
+ struct efx_tc_counter_index *ctr = ptr;
+
+ WARN_ON(refcount_read(&ctr->ref));
+ kfree(ctr);
+}
+
+int efx_tc_init_counters(struct efx_nic *efx)
+{
+ int rc;
+
+ rc = rhashtable_init(&efx->tc->counter_id_ht, &efx_tc_counter_id_ht_params);
+ if (rc < 0)
+ goto fail_counter_id_ht;
+ rc = rhashtable_init(&efx->tc->counter_ht, &efx_tc_counter_ht_params);
+ if (rc < 0)
+ goto fail_counter_ht;
+ return 0;
+fail_counter_ht:
+ rhashtable_destroy(&efx->tc->counter_id_ht);
+fail_counter_id_ht:
+ return rc;
+}
+
+/* Only call this in init failure teardown.
+ * Normal exit should fini instead as there may be entries in the table.
+ */
+void efx_tc_destroy_counters(struct efx_nic *efx)
+{
+ rhashtable_destroy(&efx->tc->counter_ht);
+ rhashtable_destroy(&efx->tc->counter_id_ht);
+}
+
+void efx_tc_fini_counters(struct efx_nic *efx)
+{
+ rhashtable_free_and_destroy(&efx->tc->counter_id_ht, efx_tc_counter_id_free, NULL);
+ rhashtable_free_and_destroy(&efx->tc->counter_ht, efx_tc_counter_free, NULL);
+}
+
+/* Counter allocation */
+
+static struct efx_tc_counter *efx_tc_flower_allocate_counter(struct efx_nic *efx,
+ int type)
+{
+ struct efx_tc_counter *cnt;
+ int rc, rc2;
+
+ cnt = kzalloc(sizeof(*cnt), GFP_USER);
+ if (!cnt)
+ return ERR_PTR(-ENOMEM);
+
+ spin_lock_init(&cnt->lock);
+ cnt->touched = jiffies;
+ cnt->type = type;
+
+ rc = efx_mae_allocate_counter(efx, cnt);
+ if (rc)
+ goto fail1;
+ rc = rhashtable_insert_fast(&efx->tc->counter_ht, &cnt->linkage,
+ efx_tc_counter_ht_params);
+ if (rc)
+ goto fail2;
+ return cnt;
+fail2:
+ /* If we get here, it implies that we couldn't insert into the table,
+ * which in turn probably means that the fw_id was already taken.
+ * In that case, it's unclear whether we really 'own' the fw_id; but
+ * the firmware seemed to think we did, so it's proper to free it.
+ */
+ rc2 = efx_mae_free_counter(efx, cnt);
+ if (rc2)
+ netif_warn(efx, hw, efx->net_dev,
+ "Failed to free MAE counter %u, rc %d\n",
+ cnt->fw_id, rc2);
+fail1:
+ kfree(cnt);
+ return ERR_PTR(rc > 0 ? -EIO : rc);
+}
+
+static void efx_tc_flower_release_counter(struct efx_nic *efx,
+ struct efx_tc_counter *cnt)
+{
+ int rc;
+
+ rhashtable_remove_fast(&efx->tc->counter_ht, &cnt->linkage,
+ efx_tc_counter_ht_params);
+ rc = efx_mae_free_counter(efx, cnt);
+ if (rc)
+ netif_warn(efx, hw, efx->net_dev,
+ "Failed to free MAE counter %u, rc %d\n",
+ cnt->fw_id, rc);
+ /* This doesn't protect counter updates coming in arbitrarily long
+ * after we deleted the counter. The RCU just ensures that we won't
+ * free the counter while another thread has a pointer to it.
+ * Ensuring we don't update the wrong counter if the ID gets re-used
+ * is handled by the generation count.
+ */
+ synchronize_rcu();
+ EFX_WARN_ON_PARANOID(spin_is_locked(&cnt->lock));
+ kfree(cnt);
+}
+
+static struct efx_tc_counter *efx_tc_flower_find_counter_by_fw_id(
+ struct efx_nic *efx, int type, u32 fw_id)
+{
+ struct efx_tc_counter key = {};
+
+ key.fw_id = fw_id;
+ key.type = type;
+
+ return rhashtable_lookup_fast(&efx->tc->counter_ht, &key,
+ efx_tc_counter_ht_params);
+}
+
+/* TC cookie to counter mapping */
+
+void efx_tc_flower_put_counter_index(struct efx_nic *efx,
+ struct efx_tc_counter_index *ctr)
+{
+ if (!refcount_dec_and_test(&ctr->ref))
+ return; /* still in use */
+ rhashtable_remove_fast(&efx->tc->counter_id_ht, &ctr->linkage,
+ efx_tc_counter_id_ht_params);
+ efx_tc_flower_release_counter(efx, ctr->cnt);
+ kfree(ctr);
+}
+
+struct efx_tc_counter_index *efx_tc_flower_get_counter_index(
+ struct efx_nic *efx, unsigned long cookie,
+ enum efx_tc_counter_type type)
+{
+ struct efx_tc_counter_index *ctr, *old;
+ struct efx_tc_counter *cnt;
+
+ ctr = kzalloc(sizeof(*ctr), GFP_USER);
+ if (!ctr)
+ return ERR_PTR(-ENOMEM);
+ ctr->cookie = cookie;
+ old = rhashtable_lookup_get_insert_fast(&efx->tc->counter_id_ht,
+ &ctr->linkage,
+ efx_tc_counter_id_ht_params);
+ if (old) {
+ /* don't need our new entry */
+ kfree(ctr);
+ if (!refcount_inc_not_zero(&old->ref))
+ return ERR_PTR(-EAGAIN);
+ /* existing entry found */
+ ctr = old;
+ } else {
+ cnt = efx_tc_flower_allocate_counter(efx, type);
+ if (IS_ERR(cnt)) {
+ rhashtable_remove_fast(&efx->tc->counter_id_ht,
+ &ctr->linkage,
+ efx_tc_counter_id_ht_params);
+ kfree(ctr);
+ return (void *)cnt; /* it's an ERR_PTR */
+ }
+ ctr->cnt = cnt;
+ refcount_set(&ctr->ref, 1);
+ }
+ return ctr;
+}
+
+struct efx_tc_counter_index *efx_tc_flower_find_counter_index(
+ struct efx_nic *efx, unsigned long cookie)
+{
+ struct efx_tc_counter_index key = {};
+
+ key.cookie = cookie;
+ return rhashtable_lookup_fast(&efx->tc->counter_id_ht, &key,
+ efx_tc_counter_id_ht_params);
+}
+
+/* TC Channel. Counter updates are delivered on this channel's RXQ. */
+
+static void efx_tc_handle_no_channel(struct efx_nic *efx)
+{
+ netif_warn(efx, drv, efx->net_dev,
+ "MAE counters require MSI-X and 1 additional interrupt vector.\n");
+}
+
+static int efx_tc_probe_channel(struct efx_channel *channel)
+{
+ struct efx_rx_queue *rx_queue = &channel->rx_queue;
+
+ channel->irq_moderation_us = 0;
+ rx_queue->core_index = 0;
+
+ INIT_WORK(&rx_queue->grant_work, efx_mae_counters_grant_credits);
+
+ return 0;
+}
+
+static int efx_tc_start_channel(struct efx_channel *channel)
+{
+ struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
+ struct efx_nic *efx = channel->efx;
+
+ return efx_mae_start_counters(efx, rx_queue);
+}
+
+static void efx_tc_stop_channel(struct efx_channel *channel)
+{
+ struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
+ struct efx_nic *efx = channel->efx;
+ int rc;
+
+ rc = efx_mae_stop_counters(efx, rx_queue);
+ if (rc)
+ netif_warn(efx, drv, efx->net_dev,
+ "Failed to stop MAE counters streaming, rc=%d.\n",
+ rc);
+ rx_queue->grant_credits = false;
+ flush_work(&rx_queue->grant_work);
+}
+
+static void efx_tc_remove_channel(struct efx_channel *channel)
+{
+}
+
+static void efx_tc_get_channel_name(struct efx_channel *channel,
+ char *buf, size_t len)
+{
+ snprintf(buf, len, "%s-mae", channel->efx->name);
+}
+
+static void efx_tc_counter_update(struct efx_nic *efx,
+ enum efx_tc_counter_type counter_type,
+ u32 counter_idx, u64 packets, u64 bytes,
+ u32 mark)
+{
+ struct efx_tc_counter *cnt;
+
+ rcu_read_lock(); /* Protect against deletion of 'cnt' */
+ cnt = efx_tc_flower_find_counter_by_fw_id(efx, counter_type, counter_idx);
+ if (!cnt) {
+ /* This can legitimately happen when a counter is removed,
+ * with updates for the counter still in-flight; however this
+ * should be an infrequent occurrence.
+ */
+ if (net_ratelimit())
+ netif_dbg(efx, drv, efx->net_dev,
+ "Got update for unwanted MAE counter %u type %u\n",
+ counter_idx, counter_type);
+ goto out;
+ }
+
+ spin_lock_bh(&cnt->lock);
+ if ((s32)mark - (s32)cnt->gen < 0) {
+ /* This counter update packet is from before the counter was
+ * allocated; thus it must be for a previous counter with
+ * the same ID that has since been freed, and it should be
+ * ignored.
+ */
+ } else {
+ /* Update latest seen generation count. This ensures that
+ * even a long-lived counter won't start getting ignored if
+ * the generation count wraps around, unless it somehow
+ * manages to go 1<<31 generations without an update.
+ */
+ cnt->gen = mark;
+ /* update counter values */
+ cnt->packets += packets;
+ cnt->bytes += bytes;
+ cnt->touched = jiffies;
+ }
+ spin_unlock_bh(&cnt->lock);
+out:
+ rcu_read_unlock();
+}
+
+static void efx_tc_rx_version_1(struct efx_nic *efx, const u8 *data, u32 mark)
+{
+ u16 n_counters, i;
+
+ /* Header format:
+ * + | 0 | 1 | 2 | 3 |
+ * 0 |version | reserved |
+ * 4 | seq_index | n_counters |
+ */
+
+ n_counters = le16_to_cpu(*(const __le16 *)(data + 6));
+
+ /* Counter update entry format:
+ * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | a | b | c | d | e | f |
+ * | counter_idx | packet_count | byte_count |
+ */
+ for (i = 0; i < n_counters; i++) {
+ const void *entry = data + 8 + 16 * i;
+ u64 packet_count, byte_count;
+ u32 counter_idx;
+
+ counter_idx = le32_to_cpu(*(const __le32 *)entry);
+ packet_count = le32_to_cpu(*(const __le32 *)(entry + 4)) |
+ ((u64)le16_to_cpu(*(const __le16 *)(entry + 8)) << 32);
+ byte_count = le16_to_cpu(*(const __le16 *)(entry + 10)) |
+ ((u64)le32_to_cpu(*(const __le32 *)(entry + 12)) << 16);
+ efx_tc_counter_update(efx, EFX_TC_COUNTER_TYPE_AR, counter_idx,
+ packet_count, byte_count, mark);
+ }
+}
+
+#define TCV2_HDR_PTR(pkt, field) \
+ ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_LBN & 7), \
+ (pkt) + ERF_SC_PACKETISER_HEADER_##field##_LBN / 8)
+#define TCV2_HDR_BYTE(pkt, field) \
+ ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_WIDTH != 8),\
+ *TCV2_HDR_PTR(pkt, field))
+#define TCV2_HDR_WORD(pkt, field) \
+ ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_WIDTH != 16),\
+ (void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_LBN & 15), \
+ *(__force const __le16 *)TCV2_HDR_PTR(pkt, field))
+#define TCV2_PKT_PTR(pkt, poff, i, field) \
+ ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_PAYLOAD_##field##_LBN & 7), \
+ (pkt) + ERF_SC_PACKETISER_PAYLOAD_##field##_LBN/8 + poff + \
+ i * ER_RX_SL_PACKETISER_PAYLOAD_WORD_SIZE)
+
+/* Read a little-endian 48-bit field with 16-bit alignment */
+static u64 efx_tc_read48(const __le16 *field)
+{
+ u64 out = 0;
+ int i;
+
+ for (i = 0; i < 3; i++)
+ out |= (u64)le16_to_cpu(field[i]) << (i * 16);
+ return out;
+}
+
+static enum efx_tc_counter_type efx_tc_rx_version_2(struct efx_nic *efx,
+ const u8 *data, u32 mark)
+{
+ u8 payload_offset, header_offset, ident;
+ enum efx_tc_counter_type type;
+ u16 n_counters, i;
+
+ ident = TCV2_HDR_BYTE(data, IDENTIFIER);
+ switch (ident) {
+ case ERF_SC_PACKETISER_HEADER_IDENTIFIER_AR:
+ type = EFX_TC_COUNTER_TYPE_AR;
+ break;
+ case ERF_SC_PACKETISER_HEADER_IDENTIFIER_CT:
+ type = EFX_TC_COUNTER_TYPE_CT;
+ break;
+ case ERF_SC_PACKETISER_HEADER_IDENTIFIER_OR:
+ type = EFX_TC_COUNTER_TYPE_OR;
+ break;
+ default:
+ if (net_ratelimit())
+ netif_err(efx, drv, efx->net_dev,
+ "ignored v2 MAE counter packet (bad identifier %u"
+ "), counters may be inaccurate\n", ident);
+ return EFX_TC_COUNTER_TYPE_MAX;
+ }
+ header_offset = TCV2_HDR_BYTE(data, HEADER_OFFSET);
+ /* mae_counter_format.h implies that this offset is fixed, since it
+ * carries on with SOP-based LBNs for the fields in this header
+ */
+ if (header_offset != ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_DEFAULT) {
+ if (net_ratelimit())
+ netif_err(efx, drv, efx->net_dev,
+ "choked on v2 MAE counter packet (bad header_offset %u"
+ "), counters may be inaccurate\n", header_offset);
+ return EFX_TC_COUNTER_TYPE_MAX;
+ }
+ payload_offset = TCV2_HDR_BYTE(data, PAYLOAD_OFFSET);
+ n_counters = le16_to_cpu(TCV2_HDR_WORD(data, COUNT));
+
+ for (i = 0; i < n_counters; i++) {
+ const void *counter_idx_p, *packet_count_p, *byte_count_p;
+ u64 packet_count, byte_count;
+ u32 counter_idx;
+
+ /* 24-bit field with 32-bit alignment */
+ counter_idx_p = TCV2_PKT_PTR(data, payload_offset, i, COUNTER_INDEX);
+ BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_WIDTH != 24);
+ BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_LBN & 31);
+ counter_idx = le32_to_cpu(*(const __le32 *)counter_idx_p) & 0xffffff;
+ /* 48-bit field with 16-bit alignment */
+ packet_count_p = TCV2_PKT_PTR(data, payload_offset, i, PACKET_COUNT);
+ BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_WIDTH != 48);
+ BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_LBN & 15);
+ packet_count = efx_tc_read48((const __le16 *)packet_count_p);
+ /* 48-bit field with 16-bit alignment */
+ byte_count_p = TCV2_PKT_PTR(data, payload_offset, i, BYTE_COUNT);
+ BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_WIDTH != 48);
+ BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_LBN & 15);
+ byte_count = efx_tc_read48((const __le16 *)byte_count_p);
+
+ if (type == EFX_TC_COUNTER_TYPE_CT) {
+ /* CT counters are 1-bit saturating counters to update
+ * the lastuse time in CT stats. A received CT counter
+ * should have packet counter to 0 and only LSB bit on
+ * in byte counter.
+ */
+ if (packet_count || byte_count != 1)
+ netdev_warn_once(efx->net_dev,
+ "CT counter with inconsistent state (%llu, %llu)\n",
+ packet_count, byte_count);
+ /* Do not increment the driver's byte counter */
+ byte_count = 0;
+ }
+
+ efx_tc_counter_update(efx, type, counter_idx, packet_count,
+ byte_count, mark);
+ }
+ return type;
+}
+
+/* We always swallow the packet, whether successful or not, since it's not
+ * a network packet and shouldn't ever be forwarded to the stack.
+ * @mark is the generation count for counter allocations.
+ */
+static bool efx_tc_rx(struct efx_rx_queue *rx_queue, u32 mark)
+{
+ struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
+ struct efx_rx_buffer *rx_buf = efx_rx_buffer(rx_queue,
+ channel->rx_pkt_index);
+ const u8 *data = efx_rx_buf_va(rx_buf);
+ struct efx_nic *efx = rx_queue->efx;
+ enum efx_tc_counter_type type;
+ u8 version;
+
+ /* version is always first byte of packet */
+ version = *data;
+ switch (version) {
+ case 1:
+ type = EFX_TC_COUNTER_TYPE_AR;
+ efx_tc_rx_version_1(efx, data, mark);
+ break;
+ case ERF_SC_PACKETISER_HEADER_VERSION_VALUE: // 2
+ type = efx_tc_rx_version_2(efx, data, mark);
+ break;
+ default:
+ if (net_ratelimit())
+ netif_err(efx, drv, efx->net_dev,
+ "choked on MAE counter packet (bad version %u"
+ "); counters may be inaccurate\n",
+ version);
+ goto out;
+ }
+
+ /* Update seen_gen unconditionally, to avoid a missed wakeup if
+ * we race with efx_mae_stop_counters().
+ */
+ efx->tc->seen_gen[type] = mark;
+ if (efx->tc->flush_counters &&
+ (s32)(efx->tc->flush_gen[type] - mark) <= 0)
+ wake_up(&efx->tc->flush_wq);
+out:
+ efx_free_rx_buffers(rx_queue, rx_buf, 1);
+ channel->rx_pkt_n_frags = 0;
+ return true;
+}
+
+const struct efx_channel_type efx_tc_channel_type = {
+ .handle_no_channel = efx_tc_handle_no_channel,
+ .pre_probe = efx_tc_probe_channel,
+ .start = efx_tc_start_channel,
+ .stop = efx_tc_stop_channel,
+ .post_remove = efx_tc_remove_channel,
+ .get_name = efx_tc_get_channel_name,
+ .receive_raw = efx_tc_rx,
+ .keep_eventq = true,
+};
diff --git a/drivers/net/ethernet/sfc/tc_counters.h b/drivers/net/ethernet/sfc/tc_counters.h
new file mode 100644
index 000000000000..8fc7c4bbb29c
--- /dev/null
+++ b/drivers/net/ethernet/sfc/tc_counters.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/****************************************************************************
+ * Driver for Solarflare network controllers and boards
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#ifndef EFX_TC_COUNTERS_H
+#define EFX_TC_COUNTERS_H
+#include <linux/refcount.h>
+#include "net_driver.h"
+
+#include "mcdi_pcol.h" /* for MAE_COUNTER_TYPE_* */
+
+enum efx_tc_counter_type {
+ EFX_TC_COUNTER_TYPE_AR = MAE_COUNTER_TYPE_AR,
+ EFX_TC_COUNTER_TYPE_CT = MAE_COUNTER_TYPE_CT,
+ EFX_TC_COUNTER_TYPE_OR = MAE_COUNTER_TYPE_OR,
+ EFX_TC_COUNTER_TYPE_MAX
+};
+
+struct efx_tc_counter {
+ u32 fw_id; /* index in firmware counter table */
+ enum efx_tc_counter_type type;
+ struct rhash_head linkage; /* efx->tc->counter_ht */
+ spinlock_t lock; /* Serialises updates to counter values */
+ u32 gen; /* Generation count at which this counter is current */
+ u64 packets, bytes;
+ u64 old_packets, old_bytes; /* Values last time passed to userspace */
+ /* jiffies of the last time we saw packets increase */
+ unsigned long touched;
+};
+
+struct efx_tc_counter_index {
+ unsigned long cookie;
+ struct rhash_head linkage; /* efx->tc->counter_id_ht */
+ refcount_t ref;
+ struct efx_tc_counter *cnt;
+};
+
+/* create/uncreate/teardown hashtables */
+int efx_tc_init_counters(struct efx_nic *efx);
+void efx_tc_destroy_counters(struct efx_nic *efx);
+void efx_tc_fini_counters(struct efx_nic *efx);
+
+struct efx_tc_counter_index *efx_tc_flower_get_counter_index(
+ struct efx_nic *efx, unsigned long cookie,
+ enum efx_tc_counter_type type);
+void efx_tc_flower_put_counter_index(struct efx_nic *efx,
+ struct efx_tc_counter_index *ctr);
+struct efx_tc_counter_index *efx_tc_flower_find_counter_index(
+ struct efx_nic *efx, unsigned long cookie);
+
+extern const struct efx_channel_type efx_tc_channel_type;
+
+#endif /* EFX_TC_COUNTERS_H */
generated by cgit v1.2.3 (git 2.39.0) at 2025-02-16 01:48:02 +0300