diff options
Diffstat (limited to 'drivers/net/ethernet/sfc')
22 files changed, 1319 insertions, 118 deletions
diff --git a/drivers/net/ethernet/sfc/Makefile b/drivers/net/ethernet/sfc/Makefile index b5e45fc6337e..712a48d00069 100644 --- a/drivers/net/ethernet/sfc/Makefile +++ b/drivers/net/ethernet/sfc/Makefile @@ -9,7 +9,7 @@ sfc-y += efx.o efx_common.o efx_channels.o nic.o \ ef100_ethtool.o ef100_rx.o ef100_tx.o sfc-$(CONFIG_SFC_MTD) += mtd.o sfc-$(CONFIG_SFC_SRIOV) += sriov.o ef10_sriov.o ef100_sriov.o ef100_rep.o \ - mae.o tc.o tc_bindings.o + mae.o tc.o tc_bindings.o tc_counters.o obj-$(CONFIG_SFC) += sfc.o diff --git a/drivers/net/ethernet/sfc/ef100_ethtool.c b/drivers/net/ethernet/sfc/ef100_ethtool.c index 135ece2f1375..702abbe59b76 100644 --- a/drivers/net/ethernet/sfc/ef100_ethtool.c +++ b/drivers/net/ethernet/sfc/ef100_ethtool.c @@ -43,8 +43,6 @@ const struct ethtool_ops ef100_ethtool_ops = { .get_pauseparam = efx_ethtool_get_pauseparam, .set_pauseparam = efx_ethtool_set_pauseparam, .get_sset_count = efx_ethtool_get_sset_count, - .get_priv_flags = efx_ethtool_get_priv_flags, - .set_priv_flags = efx_ethtool_set_priv_flags, .self_test = efx_ethtool_self_test, .get_strings = efx_ethtool_get_strings, .get_link_ksettings = efx_ethtool_get_link_ksettings, diff --git a/drivers/net/ethernet/sfc/ef100_rx.c b/drivers/net/ethernet/sfc/ef100_rx.c index 65bbe37753e6..83d9db71d7d7 100644 --- a/drivers/net/ethernet/sfc/ef100_rx.c +++ b/drivers/net/ethernet/sfc/ef100_rx.c @@ -21,7 +21,7 @@ /* Get the value of a field in the RX prefix */ #define PREFIX_OFFSET_W(_f) (ESF_GZ_RX_PREFIX_ ## _f ## _LBN / 32) #define PREFIX_OFFSET_B(_f) (ESF_GZ_RX_PREFIX_ ## _f ## _LBN % 32) -#define PREFIX_WIDTH_MASK(_f) ((1UL << ESF_GZ_RX_PREFIX_ ## _f ## _WIDTH) - 1) +#define PREFIX_WIDTH_MASK(_f) ((1ULL << ESF_GZ_RX_PREFIX_ ## _f ## _WIDTH) - 1) #define PREFIX_WORD(_p, _f) le32_to_cpu((__force __le32)(_p)[PREFIX_OFFSET_W(_f)]) #define PREFIX_FIELD(_p, _f) ((PREFIX_WORD(_p, _f) >> PREFIX_OFFSET_B(_f)) & \ PREFIX_WIDTH_MASK(_f)) @@ -67,6 +67,13 @@ void __ef100_rx_packet(struct efx_channel *channel) prefix = (u32 *)(eh - ESE_GZ_RX_PKT_PREFIX_LEN); + if (channel->type->receive_raw) { + u32 mark = PREFIX_FIELD(prefix, USER_MARK); + + if (channel->type->receive_raw(rx_queue, mark)) + return; /* packet was consumed */ + } + if (ef100_has_fcs_error(channel, prefix) && unlikely(!(efx->net_dev->features & NETIF_F_RXALL))) goto out; @@ -183,24 +190,32 @@ void efx_ef100_ev_rx(struct efx_channel *channel, const efx_qword_t *p_event) void ef100_rx_write(struct efx_rx_queue *rx_queue) { + unsigned int notified_count = rx_queue->notified_count; struct efx_rx_buffer *rx_buf; unsigned int idx; efx_qword_t *rxd; efx_dword_t rxdb; - while (rx_queue->notified_count != rx_queue->added_count) { - idx = rx_queue->notified_count & rx_queue->ptr_mask; + while (notified_count != rx_queue->added_count) { + idx = notified_count & rx_queue->ptr_mask; rx_buf = efx_rx_buffer(rx_queue, idx); rxd = efx_rx_desc(rx_queue, idx); EFX_POPULATE_QWORD_1(*rxd, ESF_GZ_RX_BUF_ADDR, rx_buf->dma_addr); - ++rx_queue->notified_count; + ++notified_count; } + if (notified_count == rx_queue->notified_count) + return; wmb(); EFX_POPULATE_DWORD_1(rxdb, ERF_GZ_RX_RING_PIDX, rx_queue->added_count & rx_queue->ptr_mask); efx_writed_page(rx_queue->efx, &rxdb, ER_GZ_RX_RING_DOORBELL, efx_rx_queue_index(rx_queue)); + if (rx_queue->grant_credits) + wmb(); + rx_queue->notified_count = notified_count; + if (rx_queue->grant_credits) + schedule_work(&rx_queue->grant_work); } diff --git a/drivers/net/ethernet/sfc/ef100_tx.c b/drivers/net/ethernet/sfc/ef100_tx.c index 102ddc7e206a..29ffaf35559d 100644 --- a/drivers/net/ethernet/sfc/ef100_tx.c +++ b/drivers/net/ethernet/sfc/ef100_tx.c @@ -367,7 +367,8 @@ void ef100_ev_tx(struct efx_channel *channel, const efx_qword_t *p_event) * Returns 0 on success, error code otherwise. In case of an error this * function will free the SKB. */ -int ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) +netdev_tx_t ef100_enqueue_skb(struct efx_tx_queue *tx_queue, + struct sk_buff *skb) { return __ef100_enqueue_skb(tx_queue, skb, NULL); } diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c index aaa381743bca..fcea3ea809d7 100644 --- a/drivers/net/ethernet/sfc/efx_channels.c +++ b/drivers/net/ethernet/sfc/efx_channels.c @@ -1119,6 +1119,8 @@ void efx_start_channels(struct efx_nic *efx) struct efx_channel *channel; efx_for_each_channel_rev(channel, efx) { + if (channel->type->start) + channel->type->start(channel); efx_for_each_channel_tx_queue(tx_queue, channel) { efx_init_tx_queue(tx_queue); atomic_inc(&efx->active_queues); @@ -1143,8 +1145,13 @@ void efx_stop_channels(struct efx_nic *efx) struct efx_channel *channel; int rc = 0; - /* Stop RX refill */ + /* Stop special channels and RX refill. + * The channel's stop has to be called first, since it might wait + * for a sentinel RX to indicate the channel has fully drained. + */ efx_for_each_channel(channel, efx) { + if (channel->type->stop) + channel->type->stop(channel); efx_for_each_channel_rx_queue(rx_queue, channel) rx_queue->refill_enabled = false; } diff --git a/drivers/net/ethernet/sfc/efx_common.c b/drivers/net/ethernet/sfc/efx_common.c index c2224e41a694..cc30524c2fe4 100644 --- a/drivers/net/ethernet/sfc/efx_common.c +++ b/drivers/net/ethernet/sfc/efx_common.c @@ -1164,7 +1164,7 @@ static ssize_t mcdi_logging_show(struct device *dev, struct efx_nic *efx = dev_get_drvdata(dev); struct efx_mcdi_iface *mcdi = efx_mcdi(efx); - return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); + return sysfs_emit(buf, "%d\n", mcdi->logging_enabled); } static ssize_t mcdi_logging_store(struct device *dev, diff --git a/drivers/net/ethernet/sfc/ethtool_common.c b/drivers/net/ethernet/sfc/ethtool_common.c index 6649a2327d03..a8cbceeb301b 100644 --- a/drivers/net/ethernet/sfc/ethtool_common.c +++ b/drivers/net/ethernet/sfc/ethtool_common.c @@ -101,14 +101,6 @@ static const struct efx_sw_stat_desc efx_sw_stat_desc[] = { #define EFX_ETHTOOL_SW_STAT_COUNT ARRAY_SIZE(efx_sw_stat_desc) -static const char efx_ethtool_priv_flags_strings[][ETH_GSTRING_LEN] = { - "log-tc-errors", -}; - -#define EFX_ETHTOOL_PRIV_FLAGS_LOG_TC_ERRS BIT(0) - -#define EFX_ETHTOOL_PRIV_FLAGS_COUNT ARRAY_SIZE(efx_ethtool_priv_flags_strings) - void efx_ethtool_get_drvinfo(struct net_device *net_dev, struct ethtool_drvinfo *info) { @@ -460,8 +452,6 @@ int efx_ethtool_get_sset_count(struct net_device *net_dev, int string_set) efx_ptp_describe_stats(efx, NULL); case ETH_SS_TEST: return efx_ethtool_fill_self_tests(efx, NULL, NULL, NULL); - case ETH_SS_PRIV_FLAGS: - return EFX_ETHTOOL_PRIV_FLAGS_COUNT; default: return -EINVAL; } @@ -488,39 +478,12 @@ void efx_ethtool_get_strings(struct net_device *net_dev, case ETH_SS_TEST: efx_ethtool_fill_self_tests(efx, NULL, strings, NULL); break; - case ETH_SS_PRIV_FLAGS: - for (i = 0; i < EFX_ETHTOOL_PRIV_FLAGS_COUNT; i++) - strscpy(strings + i * ETH_GSTRING_LEN, - efx_ethtool_priv_flags_strings[i], - ETH_GSTRING_LEN); - break; default: /* No other string sets */ break; } } -u32 efx_ethtool_get_priv_flags(struct net_device *net_dev) -{ - struct efx_nic *efx = efx_netdev_priv(net_dev); - u32 ret_flags = 0; - - if (efx->log_tc_errs) - ret_flags |= EFX_ETHTOOL_PRIV_FLAGS_LOG_TC_ERRS; - - return ret_flags; -} - -int efx_ethtool_set_priv_flags(struct net_device *net_dev, u32 flags) -{ - struct efx_nic *efx = efx_netdev_priv(net_dev); - - efx->log_tc_errs = - !!(flags & EFX_ETHTOOL_PRIV_FLAGS_LOG_TC_ERRS); - - return 0; -} - void efx_ethtool_get_stats(struct net_device *net_dev, struct ethtool_stats *stats, u64 *data) diff --git a/drivers/net/ethernet/sfc/ethtool_common.h b/drivers/net/ethernet/sfc/ethtool_common.h index 0afc74021a5e..659491932101 100644 --- a/drivers/net/ethernet/sfc/ethtool_common.h +++ b/drivers/net/ethernet/sfc/ethtool_common.h @@ -27,8 +27,6 @@ int efx_ethtool_fill_self_tests(struct efx_nic *efx, int efx_ethtool_get_sset_count(struct net_device *net_dev, int string_set); void efx_ethtool_get_strings(struct net_device *net_dev, u32 string_set, u8 *strings); -u32 efx_ethtool_get_priv_flags(struct net_device *net_dev); -int efx_ethtool_set_priv_flags(struct net_device *net_dev, u32 flags); void efx_ethtool_get_stats(struct net_device *net_dev, struct ethtool_stats *stats __attribute__ ((unused)), u64 *data); diff --git a/drivers/net/ethernet/sfc/mae.c b/drivers/net/ethernet/sfc/mae.c index 874c765b2465..583baf69981c 100644 --- a/drivers/net/ethernet/sfc/mae.c +++ b/drivers/net/ethernet/sfc/mae.c @@ -112,6 +112,117 @@ int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id) return 0; } +int efx_mae_start_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue) +{ + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_LEN); + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTERS_STREAM_START_OUT_LEN); + u32 out_flags; + size_t outlen; + int rc; + + MCDI_SET_WORD(inbuf, MAE_COUNTERS_STREAM_START_V2_IN_QID, + efx_rx_queue_index(rx_queue)); + MCDI_SET_WORD(inbuf, MAE_COUNTERS_STREAM_START_V2_IN_PACKET_SIZE, + efx->net_dev->mtu); + MCDI_SET_DWORD(inbuf, MAE_COUNTERS_STREAM_START_V2_IN_COUNTER_TYPES_MASK, + BIT(MAE_COUNTER_TYPE_AR) | BIT(MAE_COUNTER_TYPE_CT) | + BIT(MAE_COUNTER_TYPE_OR)); + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTERS_STREAM_START, + inbuf, sizeof(inbuf), outbuf, sizeof(outbuf), &outlen); + if (rc) + return rc; + if (outlen < sizeof(outbuf)) + return -EIO; + out_flags = MCDI_DWORD(outbuf, MAE_COUNTERS_STREAM_START_OUT_FLAGS); + if (out_flags & BIT(MC_CMD_MAE_COUNTERS_STREAM_START_OUT_USES_CREDITS_OFST)) { + netif_dbg(efx, drv, efx->net_dev, + "MAE counter stream uses credits\n"); + rx_queue->grant_credits = true; + out_flags &= ~BIT(MC_CMD_MAE_COUNTERS_STREAM_START_OUT_USES_CREDITS_OFST); + } + if (out_flags) { + netif_err(efx, drv, efx->net_dev, + "MAE counter stream start: unrecognised flags %x\n", + out_flags); + goto out_stop; + } + return 0; +out_stop: + efx_mae_stop_counters(efx, rx_queue); + return -EOPNOTSUPP; +} + +static bool efx_mae_counters_flushed(u32 *flush_gen, u32 *seen_gen) +{ + int i; + + for (i = 0; i < EFX_TC_COUNTER_TYPE_MAX; i++) + if ((s32)(flush_gen[i] - seen_gen[i]) > 0) + return false; + return true; +} + +int efx_mae_stop_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue) +{ + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_LENMAX); + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTERS_STREAM_STOP_IN_LEN); + size_t outlen; + int rc, i; + + MCDI_SET_WORD(inbuf, MAE_COUNTERS_STREAM_STOP_IN_QID, + efx_rx_queue_index(rx_queue)); + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTERS_STREAM_STOP, + inbuf, sizeof(inbuf), outbuf, sizeof(outbuf), &outlen); + + if (rc) + return rc; + + netif_dbg(efx, drv, efx->net_dev, "Draining counters:\n"); + /* Only process received generation counts */ + for (i = 0; (i < (outlen / 4)) && (i < EFX_TC_COUNTER_TYPE_MAX); i++) { + efx->tc->flush_gen[i] = MCDI_ARRAY_DWORD(outbuf, + MAE_COUNTERS_STREAM_STOP_V2_OUT_GENERATION_COUNT, + i); + netif_dbg(efx, drv, efx->net_dev, + "\ttype %u, awaiting gen %u\n", i, + efx->tc->flush_gen[i]); + } + + efx->tc->flush_counters = true; + + /* Drain can take up to 2 seconds owing to FWRIVERHD-2884; whatever + * timeout we use, that delay is added to unload on nonresponsive + * hardware, so 2500ms seems like a reasonable compromise. + */ + if (!wait_event_timeout(efx->tc->flush_wq, + efx_mae_counters_flushed(efx->tc->flush_gen, + efx->tc->seen_gen), + msecs_to_jiffies(2500))) + netif_warn(efx, drv, efx->net_dev, + "Failed to drain counters RXQ, FW may be unhappy\n"); + + efx->tc->flush_counters = false; + + return rc; +} + +void efx_mae_counters_grant_credits(struct work_struct *work) +{ + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_LEN); + struct efx_rx_queue *rx_queue = container_of(work, struct efx_rx_queue, + grant_work); + struct efx_nic *efx = rx_queue->efx; + unsigned int credits; + + BUILD_BUG_ON(MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_OUT_LEN); + credits = READ_ONCE(rx_queue->notified_count) - rx_queue->granted_count; + MCDI_SET_DWORD(inbuf, MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_NUM_CREDITS, + credits); + if (!efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS, + inbuf, sizeof(inbuf), NULL, 0, NULL)) + rx_queue->granted_count += credits; +} + static int efx_mae_get_basic_caps(struct efx_nic *efx, struct mae_caps *caps) { MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_GET_CAPS_OUT_LEN); @@ -250,6 +361,32 @@ static int efx_mae_match_check_cap_typ(u8 support, enum mask_type typ) } } +/* Validate field mask against hardware capabilities. Captures caller's 'rc' */ +#define CHECK(_mcdi, _field) ({ \ + enum mask_type typ = classify_mask((const u8 *)&mask->_field, \ + sizeof(mask->_field)); \ + \ + rc = efx_mae_match_check_cap_typ(supported_fields[MAE_FIELD_ ## _mcdi],\ + typ); \ + if (rc) \ + NL_SET_ERR_MSG_FMT_MOD(extack, \ + "No support for %s mask in field %s", \ + mask_type_name(typ), #_field); \ + rc; \ +}) +/* Booleans need special handling */ +#define CHECK_BIT(_mcdi, _field) ({ \ + enum mask_type typ = mask->_field ? MASK_ONES : MASK_ZEROES; \ + \ + rc = efx_mae_match_check_cap_typ(supported_fields[MAE_FIELD_ ## _mcdi],\ + typ); \ + if (rc) \ + NL_SET_ERR_MSG_FMT_MOD(extack, \ + "No support for %s mask in field %s", \ + mask_type_name(typ), #_field); \ + rc; \ +}) + int efx_mae_match_check_caps(struct efx_nic *efx, const struct efx_tc_match_fields *mask, struct netlink_ext_ack *extack) @@ -265,11 +402,86 @@ int efx_mae_match_check_caps(struct efx_nic *efx, rc = efx_mae_match_check_cap_typ(supported_fields[MAE_FIELD_INGRESS_PORT], ingress_port_mask_type); if (rc) { - efx_tc_err(efx, "No support for %s mask in field ingress_port\n", - mask_type_name(ingress_port_mask_type)); - NL_SET_ERR_MSG_MOD(extack, "Unsupported mask type for ingress_port"); + NL_SET_ERR_MSG_FMT_MOD(extack, "No support for %s mask in field ingress_port", + mask_type_name(ingress_port_mask_type)); return rc; } + if (CHECK(ETHER_TYPE, eth_proto) || + CHECK(VLAN0_TCI, vlan_tci[0]) || + CHECK(VLAN0_PROTO, vlan_proto[0]) || + CHECK(VLAN1_TCI, vlan_tci[1]) || + CHECK(VLAN1_PROTO, vlan_proto[1]) || + CHECK(ETH_SADDR, eth_saddr) || + CHECK(ETH_DADDR, eth_daddr) || + CHECK(IP_PROTO, ip_proto) || + CHECK(IP_TOS, ip_tos) || + CHECK(IP_TTL, ip_ttl) || + CHECK(SRC_IP4, src_ip) || + CHECK(DST_IP4, dst_ip) || +#ifdef CONFIG_IPV6 + CHECK(SRC_IP6, src_ip6) || + CHECK(DST_IP6, dst_ip6) || +#endif + CHECK(L4_SPORT, l4_sport) || + CHECK(L4_DPORT, l4_dport) || + CHECK(TCP_FLAGS, tcp_flags) || + CHECK_BIT(IS_IP_FRAG, ip_frag) || + CHECK_BIT(IP_FIRST_FRAG, ip_firstfrag) || + CHECK(RECIRC_ID, recirc_id)) + return rc; + return 0; +} +#undef CHECK_BIT +#undef CHECK + +int efx_mae_allocate_counter(struct efx_nic *efx, struct efx_tc_counter *cnt) +{ + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTER_ALLOC_OUT_LEN(1)); + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTER_ALLOC_V2_IN_LEN); + size_t outlen; + int rc; + + if (!cnt) + return -EINVAL; + + MCDI_SET_DWORD(inbuf, MAE_COUNTER_ALLOC_V2_IN_REQUESTED_COUNT, 1); + MCDI_SET_DWORD(inbuf, MAE_COUNTER_ALLOC_V2_IN_COUNTER_TYPE, cnt->type); + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTER_ALLOC, inbuf, sizeof(inbuf), + outbuf, sizeof(outbuf), &outlen); + if (rc) + return rc; + /* pcol says this can't happen, since count is 1 */ + if (outlen < sizeof(outbuf)) + return -EIO; + cnt->fw_id = MCDI_DWORD(outbuf, MAE_COUNTER_ALLOC_OUT_COUNTER_ID); + cnt->gen = MCDI_DWORD(outbuf, MAE_COUNTER_ALLOC_OUT_GENERATION_COUNT); + return 0; +} + +int efx_mae_free_counter(struct efx_nic *efx, struct efx_tc_counter *cnt) +{ + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTER_FREE_OUT_LEN(1)); + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTER_FREE_V2_IN_LEN); + size_t outlen; + int rc; + + MCDI_SET_DWORD(inbuf, MAE_COUNTER_FREE_V2_IN_COUNTER_ID_COUNT, 1); + MCDI_SET_DWORD(inbuf, MAE_COUNTER_FREE_V2_IN_FREE_COUNTER_ID, cnt->fw_id); + MCDI_SET_DWORD(inbuf, MAE_COUNTER_FREE_V2_IN_COUNTER_TYPE, cnt->type); + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTER_FREE, inbuf, sizeof(inbuf), + outbuf, sizeof(outbuf), &outlen); + if (rc) + return rc; + /* pcol says this can't happen, since count is 1 */ + if (outlen < sizeof(outbuf)) + return -EIO; + /* FW freed a different ID than we asked for, should also never happen. + * Warn because it means we've now got a different idea to the FW of + * what counters exist, which could cause mayhem later. + */ + if (WARN_ON(MCDI_DWORD(outbuf, MAE_COUNTER_FREE_OUT_FREED_COUNTER_ID) != + cnt->fw_id)) + return -EIO; return 0; } @@ -289,8 +501,12 @@ int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act) MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_NULL); MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_DST_MAC_ID, MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_NULL); - MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID, - MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NULL); + if (act->count && !WARN_ON(!act->count->cnt)) + MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID, + act->count->cnt->fw_id); + else + MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID, + MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NULL); MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_LIST_ID, MC_CMD_MAE_COUNTER_LIST_ALLOC_OUT_COUNTER_LIST_ID_NULL); MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_ENCAP_HEADER_ID, @@ -440,10 +656,90 @@ static int efx_mae_populate_match_criteria(MCDI_DECLARE_STRUCT_PTR(match_crit), } MCDI_STRUCT_SET_DWORD(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_MASK, match->mask.ingress_port); + EFX_POPULATE_DWORD_2(*_MCDI_STRUCT_DWORD(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS), + MAE_FIELD_MASK_VALUE_PAIRS_V2_IS_IP_FRAG, + match->value.ip_frag, + MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FIRST_FRAG, + match->value.ip_firstfrag); + EFX_POPULATE_DWORD_2(*_MCDI_STRUCT_DWORD(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS_MASK), + MAE_FIELD_MASK_VALUE_PAIRS_V2_IS_IP_FRAG, + match->mask.ip_frag, + MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FIRST_FRAG, + match->mask.ip_firstfrag); MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID, match->value.recirc_id); MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID_MASK, match->mask.recirc_id); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ETHER_TYPE_BE, + match->value.eth_proto); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ETHER_TYPE_BE_MASK, + match->mask.eth_proto); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_TCI_BE, + match->value.vlan_tci[0]); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_TCI_BE_MASK, + match->mask.vlan_tci[0]); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_PROTO_BE, + match->value.vlan_proto[0]); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_PROTO_BE_MASK, + match->mask.vlan_proto[0]); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_TCI_BE, + match->value.vlan_tci[1]); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_TCI_BE_MASK, + match->mask.vlan_tci[1]); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_PROTO_BE, + match->value.vlan_proto[1]); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_PROTO_BE_MASK, + match->mask.vlan_proto[1]); + memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_SADDR_BE), + match->value.eth_saddr, ETH_ALEN); + memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_SADDR_BE_MASK), + match->mask.eth_saddr, ETH_ALEN); + memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_DADDR_BE), + match->value.eth_daddr, ETH_ALEN); + memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_DADDR_BE_MASK), + match->mask.eth_daddr, ETH_ALEN); + MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_PROTO, + match->value.ip_proto); + MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_PROTO_MASK, + match->mask.ip_proto); + MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TOS, + match->value.ip_tos); + MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TOS_MASK, + match->mask.ip_tos); + MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TTL, + match->value.ip_ttl); + MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TTL_MASK, + match->mask.ip_ttl); + MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP4_BE, + match->value.src_ip); + MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP4_BE_MASK, + match->mask.src_ip); + MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP4_BE, + match->value.dst_ip); + MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP4_BE_MASK, + match->mask.dst_ip); +#ifdef CONFIG_IPV6 + memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP6_BE), + &match->value.src_ip6, sizeof(struct in6_addr)); + memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP6_BE_MASK), + &match->mask.src_ip6, sizeof(struct in6_addr)); + memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP6_BE), + &match->value.dst_ip6, sizeof(struct in6_addr)); + memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP6_BE_MASK), + &match->mask.dst_ip6, sizeof(struct in6_addr)); +#endif + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_SPORT_BE, + match->value.l4_sport); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_SPORT_BE_MASK, + match->mask.l4_sport); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_DPORT_BE, + match->value.l4_dport); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_DPORT_BE_MASK, + match->mask.l4_dport); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE, + match->value.tcp_flags); + MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE_MASK, + match->mask.tcp_flags); return 0; } diff --git a/drivers/net/ethernet/sfc/mae.h b/drivers/net/ethernet/sfc/mae.h index 3e0cd238d523..72343e90e222 100644 --- a/drivers/net/ethernet/sfc/mae.h +++ b/drivers/net/ethernet/sfc/mae.h @@ -27,6 +27,10 @@ void efx_mae_mport_mport(struct efx_nic *efx, u32 mport_id, u32 *out); int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id); +int efx_mae_start_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue); +int efx_mae_stop_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue); +void efx_mae_counters_grant_credits(struct work_struct *work); + #define MAE_NUM_FIELDS (MAE_FIELD_ENC_VNET_ID + 1) struct mae_caps { @@ -41,6 +45,9 @@ int efx_mae_match_check_caps(struct efx_nic *efx, const struct efx_tc_match_fields *mask, struct netlink_ext_ack *extack); +int efx_mae_allocate_counter(struct efx_nic *efx, struct efx_tc_counter *cnt); +int efx_mae_free_counter(struct efx_nic *efx, struct efx_tc_counter *cnt); + int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act); int efx_mae_free_action_set(struct efx_nic *efx, u32 fw_id); diff --git a/drivers/net/ethernet/sfc/mae_counter_format.h b/drivers/net/ethernet/sfc/mae_counter_format.h new file mode 100644 index 000000000000..7e252e393fbe --- /dev/null +++ b/drivers/net/ethernet/sfc/mae_counter_format.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2020 Xilinx, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +/* Format of counter packets (version 2) from the ef100 Match-Action Engine */ + +#ifndef EFX_MAE_COUNTER_FORMAT_H +#define EFX_MAE_COUNTER_FORMAT_H + + +/*------------------------------------------------------------*/ +/* + * ER_RX_SL_PACKETISER_HEADER_WORD(160bit): + * + */ +#define ER_RX_SL_PACKETISER_HEADER_WORD_SIZE 20 +#define ER_RX_SL_PACKETISER_HEADER_WORD_WIDTH 160 + +#define ERF_SC_PACKETISER_HEADER_VERSION_LBN 0 +#define ERF_SC_PACKETISER_HEADER_VERSION_WIDTH 8 +#define ERF_SC_PACKETISER_HEADER_VERSION_VALUE 2 +#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_LBN 8 +#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_WIDTH 8 +#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_AR 0 +#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_CT 1 +#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_OR 2 +#define ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_LBN 16 +#define ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_WIDTH 8 +#define ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_DEFAULT 0x4 +#define ERF_SC_PACKETISER_HEADER_PAYLOAD_OFFSET_LBN 24 +#define ERF_SC_PACKETISER_HEADER_PAYLOAD_OFFSET_WIDTH 8 +#define ERF_SC_PACKETISER_HEADER_PAYLOAD_OFFSET_DEFAULT 0x14 +#define ERF_SC_PACKETISER_HEADER_INDEX_LBN 32 +#define ERF_SC_PACKETISER_HEADER_INDEX_WIDTH 16 +#define ERF_SC_PACKETISER_HEADER_COUNT_LBN 48 +#define ERF_SC_PACKETISER_HEADER_COUNT_WIDTH 16 +#define ERF_SC_PACKETISER_HEADER_RESERVED_0_LBN 64 +#define ERF_SC_PACKETISER_HEADER_RESERVED_0_WIDTH 32 +#define ERF_SC_PACKETISER_HEADER_RESERVED_1_LBN 96 +#define ERF_SC_PACKETISER_HEADER_RESERVED_1_WIDTH 32 +#define ERF_SC_PACKETISER_HEADER_RESERVED_2_LBN 128 +#define ERF_SC_PACKETISER_HEADER_RESERVED_2_WIDTH 32 + + +/*------------------------------------------------------------*/ +/* + * ER_RX_SL_PACKETISER_PAYLOAD_WORD(128bit): + * + */ +#define ER_RX_SL_PACKETISER_PAYLOAD_WORD_SIZE 16 +#define ER_RX_SL_PACKETISER_PAYLOAD_WORD_WIDTH 128 + +#define ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_LBN 0 +#define ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_WIDTH 24 +#define ERF_SC_PACKETISER_PAYLOAD_RESERVED_LBN 24 +#define ERF_SC_PACKETISER_PAYLOAD_RESERVED_WIDTH 8 +#define ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_OFST 4 +#define ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_SIZE 6 +#define ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_LBN 32 +#define ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_WIDTH 48 +#define ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_OFST 10 +#define ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_SIZE 6 +#define ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_LBN 80 +#define ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_WIDTH 48 + + +#endif /* EFX_MAE_COUNTER_FORMAT_H */ diff --git a/drivers/net/ethernet/sfc/mcdi.h b/drivers/net/ethernet/sfc/mcdi.h index 1f18e9dc62e8..7e35fec9da35 100644 --- a/drivers/net/ethernet/sfc/mcdi.h +++ b/drivers/net/ethernet/sfc/mcdi.h @@ -221,15 +221,32 @@ void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev); #define MCDI_BYTE(_buf, _field) \ ((void)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 1), \ *MCDI_PTR(_buf, _field)) +#define MCDI_SET_WORD(_buf, _field, _value) do { \ + BUILD_BUG_ON(MC_CMD_ ## _field ## _LEN != 2); \ + BUILD_BUG_ON(MC_CMD_ ## _field ## _OFST & 1); \ + *(__force __le16 *)MCDI_PTR(_buf, _field) = cpu_to_le16(_value);\ + } while (0) #define MCDI_WORD(_buf, _field) \ ((u16)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 2) + \ le16_to_cpu(*(__force const __le16 *)MCDI_PTR(_buf, _field))) +/* Write a 16-bit field defined in the protocol as being big-endian. */ +#define MCDI_STRUCT_SET_WORD_BE(_buf, _field, _value) do { \ + BUILD_BUG_ON(_field ## _LEN != 2); \ + BUILD_BUG_ON(_field ## _OFST & 1); \ + *(__force __be16 *)MCDI_STRUCT_PTR(_buf, _field) = (_value); \ + } while (0) #define MCDI_SET_DWORD(_buf, _field, _value) \ EFX_POPULATE_DWORD_1(*_MCDI_DWORD(_buf, _field), EFX_DWORD_0, _value) #define MCDI_STRUCT_SET_DWORD(_buf, _field, _value) \ EFX_POPULATE_DWORD_1(*_MCDI_STRUCT_DWORD(_buf, _field), EFX_DWORD_0, _value) #define MCDI_DWORD(_buf, _field) \ EFX_DWORD_FIELD(*_MCDI_DWORD(_buf, _field), EFX_DWORD_0) +/* Write a 32-bit field defined in the protocol as being big-endian. */ +#define MCDI_STRUCT_SET_DWORD_BE(_buf, _field, _value) do { \ + BUILD_BUG_ON(_field ## _LEN != 4); \ + BUILD_BUG_ON(_field ## _OFST & 3); \ + *(__force __be32 *)MCDI_STRUCT_PTR(_buf, _field) = (_value); \ + } while (0) #define MCDI_POPULATE_DWORD_1(_buf, _field, _name1, _value1) \ EFX_POPULATE_DWORD_1(*_MCDI_DWORD(_buf, _field), \ MC_CMD_ ## _name1, _value1) diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index 2e9ba0cfe848..3b49e216768b 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -56,7 +56,8 @@ #define EFX_MAX_RX_QUEUES EFX_MAX_CHANNELS #define EFX_EXTRA_CHANNEL_IOV 0 #define EFX_EXTRA_CHANNEL_PTP 1 -#define EFX_MAX_EXTRA_CHANNELS 2U +#define EFX_EXTRA_CHANNEL_TC 2 +#define EFX_MAX_EXTRA_CHANNELS 3U /* Checksum generation is a per-queue option in hardware, so each * queue visible to the networking core is backed by two hardware TX @@ -363,8 +364,12 @@ struct efx_rx_page_state { * @refill_enabled: Enable refill whenever fill level is low * @flush_pending: Set when a RX flush is pending. Has the same lifetime as * @rxq_flush_pending. + * @grant_credits: Posted RX descriptors need to be granted to the MAE with + * %MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS. For %EFX_EXTRA_CHANNEL_TC, + * and only supported on EF100. * @added_count: Number of buffers added to the receive queue. * @notified_count: Number of buffers given to NIC (<= @added_count). + * @granted_count: Number of buffers granted to the MAE (<= @notified_count). * @removed_count: Number of buffers removed from the receive queue. * @scatter_n: Used by NIC specific receive code. * @scatter_len: Used by NIC specific receive code. @@ -385,6 +390,7 @@ struct efx_rx_page_state { * refill was triggered. * @recycle_count: RX buffer recycle counter. * @slow_fill: Timer used to defer efx_nic_generate_fill_event(). + * @grant_work: workitem used to grant credits to the MAE if @grant_credits * @xdp_rxq_info: XDP specific RX queue information. * @xdp_rxq_info_valid: Is xdp_rxq_info valid data?. */ @@ -396,9 +402,11 @@ struct efx_rx_queue { unsigned int ptr_mask; bool refill_enabled; bool flush_pending; + bool grant_credits; unsigned int added_count; unsigned int notified_count; + unsigned int granted_count; unsigned int removed_count; unsigned int scatter_n; unsigned int scatter_len; @@ -416,6 +424,7 @@ struct efx_rx_queue { unsigned int recycle_count; struct timer_list slow_fill; unsigned int slow_fill_count; + struct work_struct grant_work; /* Statistics to supplement MAC stats */ unsigned long rx_packets; struct xdp_rxq_info xdp_rxq_info; @@ -577,12 +586,15 @@ struct efx_msi_context { * struct efx_channel_type - distinguishes traffic and extra channels * @handle_no_channel: Handle failure to allocate an extra channel * @pre_probe: Set up extra state prior to initialisation + * @start: called early in efx_start_channels() + * @stop: called early in efx_stop_channels() * @post_remove: Tear down extra state after finalisation, if allocated. * May be called on channels that have not been probed. * @get_name: Generate the channel's name (used for its IRQ handler) * @copy: Copy the channel state prior to reallocation. May be %NULL if * reallocation is not supported. * @receive_skb: Handle an skb ready to be passed to netif_receive_skb() + * @receive_raw: Handle an RX buffer ready to be passed to __efx_rx_packet() * @want_txqs: Determine whether this channel should have TX queues * created. If %NULL, TX queues are not created. * @keep_eventq: Flag for whether event queue should be kept initialised @@ -593,10 +605,13 @@ struct efx_msi_context { struct efx_channel_type { void (*handle_no_channel)(struct efx_nic *); int (*pre_probe)(struct efx_channel *); + int (*start)(struct efx_channel *); + void (*stop)(struct efx_channel *); void (*post_remove)(struct efx_channel *); void (*get_name)(struct efx_channel *, char *buf, size_t len); struct efx_channel *(*copy)(const struct efx_channel *); bool (*receive_skb)(struct efx_channel *, struct sk_buff *); + bool (*receive_raw)(struct efx_rx_queue *, u32); bool (*want_txqs)(struct efx_channel *); bool keep_eventq; bool want_pio; @@ -855,7 +870,6 @@ enum efx_xdp_tx_queues_mode { * @timer_max_ns: Interrupt timer maximum value, in nanoseconds * @irq_rx_adaptive: Adaptive IRQ moderation enabled for RX event queues * @irqs_hooked: Channel interrupts are hooked - * @log_tc_errs: Error logging for TC filter insertion is enabled * @irq_rx_mod_step_us: Step size for IRQ moderation for RX event queues * @irq_rx_moderation_us: IRQ moderation time for RX event queues * @msg_enable: Log message enable flags @@ -1018,7 +1032,6 @@ struct efx_nic { unsigned int timer_max_ns; bool irq_rx_adaptive; bool irqs_hooked; - bool log_tc_errs; unsigned int irq_mod_step_us; unsigned int irq_rx_moderation_us; u32 msg_enable; diff --git a/drivers/net/ethernet/sfc/ptp.c b/drivers/net/ethernet/sfc/ptp.c index eaef4a15008a..9f07e1ba7780 100644 --- a/drivers/net/ethernet/sfc/ptp.c +++ b/drivers/net/ethernet/sfc/ptp.c @@ -351,7 +351,7 @@ struct efx_ptp_data { void (*xmit_skb)(struct efx_nic *efx, struct sk_buff *skb); }; -static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta); +static int efx_phc_adjfine(struct ptp_clock_info *ptp, long scaled_ppm); static int efx_phc_adjtime(struct ptp_clock_info *ptp, s64 delta); static int efx_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts); static int efx_phc_settime(struct ptp_clock_info *ptp, @@ -1508,7 +1508,7 @@ static const struct ptp_clock_info efx_phc_clock_info = { .n_per_out = 0, .n_pins = 0, .pps = 1, - .adjfreq = efx_phc_adjfreq, + .adjfine = efx_phc_adjfine, .adjtime = efx_phc_adjtime, .gettime64 = efx_phc_gettime, .settime64 = efx_phc_settime, @@ -2137,11 +2137,12 @@ void __efx_rx_skb_attach_timestamp(struct efx_channel *channel, ptp->ts_corrections.general_rx); } -static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta) +static int efx_phc_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) { struct efx_ptp_data *ptp_data = container_of(ptp, struct efx_ptp_data, phc_clock_info); + s32 delta = scaled_ppm_to_ppb(scaled_ppm); struct efx_nic *efx = ptp_data->efx; MCDI_DECLARE_BUF(inadj, MC_CMD_PTP_IN_ADJUST_LEN); s64 adjustment_ns; diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c index 9220afeddee8..d2f35ee15eff 100644 --- a/drivers/net/ethernet/sfc/rx_common.c +++ b/drivers/net/ethernet/sfc/rx_common.c @@ -229,6 +229,7 @@ void efx_init_rx_queue(struct efx_rx_queue *rx_queue) /* Initialise ptr fields */ rx_queue->added_count = 0; rx_queue->notified_count = 0; + rx_queue->granted_count = 0; rx_queue->removed_count = 0; rx_queue->min_fill = -1U; efx_init_rx_recycle_ring(rx_queue); @@ -281,6 +282,8 @@ void efx_fini_rx_queue(struct efx_rx_queue *rx_queue) "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue)); del_timer_sync(&rx_queue->slow_fill); + if (rx_queue->grant_credits) + flush_work(&rx_queue->grant_work); /* Release RX buffers from the current read ptr to the write ptr */ if (rx_queue->buffer) { diff --git a/drivers/net/ethernet/sfc/siena/efx_common.c b/drivers/net/ethernet/sfc/siena/efx_common.c index 1fd396b00bfb..e4b294b8e9ac 100644 --- a/drivers/net/ethernet/sfc/siena/efx_common.c +++ b/drivers/net/ethernet/sfc/siena/efx_common.c @@ -1178,7 +1178,7 @@ static ssize_t mcdi_logging_show(struct device *dev, struct efx_nic *efx = dev_get_drvdata(dev); struct efx_mcdi_iface *mcdi = efx_mcdi(efx); - return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); + return sysfs_emit(buf, "%d\n", mcdi->logging_enabled); } static ssize_t mcdi_logging_store(struct device *dev, diff --git a/drivers/net/ethernet/sfc/siena/ptp.c b/drivers/net/ethernet/sfc/siena/ptp.c index 7c46752e6eae..38e666561bcd 100644 --- a/drivers/net/ethernet/sfc/siena/ptp.c +++ b/drivers/net/ethernet/sfc/siena/ptp.c @@ -347,7 +347,7 @@ struct efx_ptp_data { void (*xmit_skb)(struct efx_nic *efx, struct sk_buff *skb); }; -static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta); +static int efx_phc_adjfine(struct ptp_clock_info *ptp, long scaled_ppm); static int efx_phc_adjtime(struct ptp_clock_info *ptp, s64 delta); static int efx_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts); static int efx_phc_settime(struct ptp_clock_info *ptp, @@ -1429,7 +1429,7 @@ static const struct ptp_clock_info efx_phc_clock_info = { .n_per_out = 0, .n_pins = 0, .pps = 1, - .adjfreq = efx_phc_adjfreq, + .adjfine = efx_phc_adjfine, .adjtime = efx_phc_adjtime, .gettime64 = efx_phc_gettime, .settime64 = efx_phc_settime, @@ -2044,11 +2044,12 @@ void __efx_siena_rx_skb_attach_timestamp(struct efx_channel *channel, ptp->ts_corrections.general_rx); } -static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta) +static int efx_phc_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) { struct efx_ptp_data *ptp_data = container_of(ptp, struct efx_ptp_data, phc_clock_info); + s32 delta = scaled_ppm_to_ppb(scaled_ppm); struct efx_nic *efx = ptp_data->efx; MCDI_DECLARE_BUF(inadj, MC_CMD_PTP_IN_ADJUST_LEN); s64 adjustment_ns; diff --git a/drivers/net/ethernet/sfc/tc.c b/drivers/net/ethernet/sfc/tc.c index 3478860d4023..deeaab9ee761 100644 --- a/drivers/net/ethernet/sfc/tc.c +++ b/drivers/net/ethernet/sfc/tc.c @@ -77,6 +77,8 @@ static void efx_tc_free_action_set(struct efx_nic *efx, */ list_del(&act->list); } + if (act->count) + efx_tc_flower_put_counter_index(efx, act->count); kfree(act); } @@ -124,50 +126,187 @@ static void efx_tc_flow_free(void *ptr, void *arg) kfree(rule); } +/* Boilerplate for the simple 'copy a field' cases */ +#define _MAP_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \ +if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_##_name)) { \ + struct flow_match_##_type fm; \ + \ + flow_rule_match_##_tcget(rule, &fm); \ + match->value._field = fm.key->_tcfield; \ + match->mask._field = fm.mask->_tcfield; \ +} +#define MAP_KEY_AND_MASK(_name, _type, _tcfield, _field) \ + _MAP_KEY_AND_MASK(_name, _type, _type, _tcfield, _field) +#define MAP_ENC_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \ + _MAP_KEY_AND_MASK(ENC_##_name, _type, _tcget, _tcfield, _field) + static int efx_tc_flower_parse_match(struct efx_nic *efx, struct flow_rule *rule, struct efx_tc_match *match, struct netlink_ext_ack *extack) { struct flow_dissector *dissector = rule->match.dissector; + unsigned char ipv = 0; + /* Owing to internal TC infelicities, the IPV6_ADDRS key might be set + * even on IPv4 filters; so rather than relying on dissector->used_keys + * we check the addr_type in the CONTROL key. If we don't find it (or + * it's masked, which should never happen), we treat both IPV4_ADDRS + * and IPV6_ADDRS as absent. + */ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { struct flow_match_control fm; flow_rule_match_control(rule, &fm); + if (IS_ALL_ONES(fm.mask->addr_type)) + switch (fm.key->addr_type) { + case FLOW_DISSECTOR_KEY_IPV4_ADDRS: + ipv = 4; + break; + case FLOW_DISSECTOR_KEY_IPV6_ADDRS: + ipv = 6; + break; + default: + break; + } - if (fm.mask->flags) { - efx_tc_err(efx, "Unsupported match on control.flags %#x\n", - fm.mask->flags); - NL_SET_ERR_MSG_MOD(extack, "Unsupported match on control.flags"); + if (fm.mask->flags & FLOW_DIS_IS_FRAGMENT) { + match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT; + match->mask.ip_frag = true; + } + if (fm.mask->flags & FLOW_DIS_FIRST_FRAG) { + match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG; + match->mask.ip_firstfrag = true; + } + if (fm.mask->flags & ~(FLOW_DIS_IS_FRAGMENT | FLOW_DIS_FIRST_FRAG)) { + NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on control.flags %#x", + fm.mask->flags); return -EOPNOTSUPP; } } if (dissector->used_keys & ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | - BIT(FLOW_DISSECTOR_KEY_BASIC))) { - efx_tc_err(efx, "Unsupported flower keys %#x\n", dissector->used_keys); - NL_SET_ERR_MSG_MOD(extack, "Unsupported flower keys encountered"); + BIT(FLOW_DISSECTOR_KEY_BASIC) | + BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT(FLOW_DISSECTOR_KEY_VLAN) | + BIT(FLOW_DISSECTOR_KEY_CVLAN) | + BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT(FLOW_DISSECTOR_KEY_PORTS) | + BIT(FLOW_DISSECTOR_KEY_TCP) | + BIT(FLOW_DISSECTOR_KEY_IP))) { + NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#x", + dissector->used_keys); return -EOPNOTSUPP; } - if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { - struct flow_match_basic fm; + MAP_KEY_AND_MASK(BASIC, basic, n_proto, eth_proto); + /* Make sure we're IP if any L3/L4 keys used. */ + if (!IS_ALL_ONES(match->mask.eth_proto) || + !(match->value.eth_proto == htons(ETH_P_IP) || + match->value.eth_proto == htons(ETH_P_IPV6))) + if (dissector->used_keys & + (BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT(FLOW_DISSECTOR_KEY_PORTS) | + BIT(FLOW_DISSECTOR_KEY_IP) | + BIT(FLOW_DISSECTOR_KEY_TCP))) { + NL_SET_ERR_MSG_FMT_MOD(extack, "L3/L4 flower keys %#x require protocol ipv[46]", + dissector->used_keys); + return -EINVAL; + } - flow_rule_match_basic(rule, &fm); - if (fm.mask->n_proto) { - EFX_TC_ERR_MSG(efx, extack, "Unsupported eth_proto match\n"); - return -EOPNOTSUPP; + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { + struct flow_match_vlan fm; + + flow_rule_match_vlan(rule, &fm); + if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) { + match->value.vlan_proto[0] = fm.key->vlan_tpid; + match->mask.vlan_proto[0] = fm.mask->vlan_tpid; + match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 | + fm.key->vlan_id); + match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 | + fm.mask->vlan_id); } - if (fm.mask->ip_proto) { - EFX_TC_ERR_MSG(efx, extack, "Unsupported ip_proto match\n"); - return -EOPNOTSUPP; + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) { + struct flow_match_vlan fm; + + flow_rule_match_cvlan(rule, &fm); + if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) { + match->value.vlan_proto[1] = fm.key->vlan_tpid; + match->mask.vlan_proto[1] = fm.mask->vlan_tpid; + match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 | + fm.key->vlan_id); + match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 | + fm.mask->vlan_id); } } + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { + struct flow_match_eth_addrs fm; + + flow_rule_match_eth_addrs(rule, &fm); + ether_addr_copy(match->value.eth_saddr, fm.key->src); + ether_addr_copy(match->value.eth_daddr, fm.key->dst); + ether_addr_copy(match->mask.eth_saddr, fm.mask->src); + ether_addr_copy(match->mask.eth_daddr, fm.mask->dst); + } + + MAP_KEY_AND_MASK(BASIC, basic, ip_proto, ip_proto); + /* Make sure we're TCP/UDP if any L4 keys used. */ + if ((match->value.ip_proto != IPPROTO_UDP && + match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto)) + if (dissector->used_keys & + (BIT(FLOW_DISSECTOR_KEY_PORTS) | + BIT(FLOW_DISSECTOR_KEY_TCP))) { + NL_SET_ERR_MSG_FMT_MOD(extack, "L4 flower keys %#x require ipproto udp or tcp", + dissector->used_keys); + return -EINVAL; + } + MAP_KEY_AND_MASK(IP, ip, tos, ip_tos); + MAP_KEY_AND_MASK(IP, ip, ttl, ip_ttl); + if (ipv == 4) { + MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, src, src_ip); + MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, dst, dst_ip); + } +#ifdef CONFIG_IPV6 + else if (ipv == 6) { + MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, src, src_ip6); + MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, dst, dst_ip6); + } +#endif + MAP_KEY_AND_MASK(PORTS, ports, src, l4_sport); + MAP_KEY_AND_MASK(PORTS, ports, dst, l4_dport); + MAP_KEY_AND_MASK(TCP, tcp, flags, tcp_flags); + return 0; } +/* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */ +enum efx_tc_action_order { + EFX_TC_AO_COUNT, + EFX_TC_AO_DELIVER +}; +/* Determine whether we can add @new action without violating order */ +static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act, + enum efx_tc_action_order new) +{ + switch (new) { + case EFX_TC_AO_COUNT: + if (act->count) + return false; + fallthrough; + case EFX_TC_AO_DELIVER: + return !act->deliver; + default: + /* Bad caller. Whatever they wanted to do, say they can't. */ + WARN_ON_ONCE(1); + return false; + } +} + static int efx_tc_flower_replace(struct efx_nic *efx, struct net_device *net_dev, struct flow_cls_offload *tc, @@ -200,13 +339,9 @@ static int efx_tc_flower_replace(struct efx_nic *efx, if (efv != from_efv) { /* can't happen */ - efx_tc_err(efx, "for %s efv is %snull but from_efv is %snull\n", - netdev_name(net_dev), efv ? "non-" : "", - from_efv ? "non-" : ""); - if (efv) - NL_SET_ERR_MSG_MOD(extack, "vfrep filter has PF net_dev (can't happen)"); - else - NL_SET_ERR_MSG_MOD(extack, "PF filter has vfrep net_dev (can't happen)"); + NL_SET_ERR_MSG_FMT_MOD(extack, "for %s efv is %snull but from_efv is %snull (can't happen)", + netdev_name(net_dev), efv ? "non-" : "", + from_efv ? "non-" : ""); return -EINVAL; } @@ -214,7 +349,7 @@ static int efx_tc_flower_replace(struct efx_nic *efx, memset(&match, 0, sizeof(match)); rc = efx_tc_flower_external_mport(efx, from_efv); if (rc < 0) { - EFX_TC_ERR_MSG(efx, extack, "Failed to identify ingress m-port"); + NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port"); return rc; } match.value.ingress_port = rc; @@ -224,7 +359,7 @@ static int efx_tc_flower_replace(struct efx_nic *efx, return rc; if (tc->common.chain_index) { - EFX_TC_ERR_MSG(efx, extack, "No support for nonzero chain_index"); + NL_SET_ERR_MSG_MOD(extack, "No support for nonzero chain_index"); return -EOPNOTSUPP; } match.mask.recirc_id = 0xff; @@ -261,16 +396,57 @@ static int efx_tc_flower_replace(struct efx_nic *efx, if (!act) { /* more actions after a non-pipe action */ - EFX_TC_ERR_MSG(efx, extack, "Action follows non-pipe action"); + NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action"); rc = -EINVAL; goto release; } + if ((fa->id == FLOW_ACTION_REDIRECT || + fa->id == FLOW_ACTION_MIRRED || + fa->id == FLOW_ACTION_DROP) && fa->hw_stats) { + struct efx_tc_counter_index *ctr; + + /* Currently the only actions that want stats are + * mirred and gact (ok, shot, trap, goto-chain), which + * means we want stats just before delivery. Also, + * note that tunnel_key set shouldn't change the length + * — it's only the subsequent mirred that does that, + * and the stats are taken _before_ the mirred action + * happens. + */ + if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) { + /* All supported actions that count either steal + * (gact shot, mirred redirect) or clone act + * (mirred mirror), so we should never get two + * count actions on one action_set. + */ + NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)"); + rc = -EOPNOTSUPP; + goto release; + } + + if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { + NL_SET_ERR_MSG_FMT_MOD(extack, "hw_stats_type %u not supported (only 'delayed')", + fa->hw_stats); + rc = -EOPNOTSUPP; + goto release; + } + + ctr = efx_tc_flower_get_counter_index(efx, tc->cookie, + EFX_TC_COUNTER_TYPE_AR); + if (IS_ERR(ctr)) { + rc = PTR_ERR(ctr); + NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter"); + goto release; + } + act->count = ctr; + } + switch (fa->id) { case FLOW_ACTION_DROP: rc = efx_mae_alloc_action_set(efx, act); if (rc) { - EFX_TC_ERR_MSG(efx, extack, "Failed to write action set to hw (drop)"); + NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (drop)"); goto release; } list_add_tail(&act->list, &rule->acts.list); @@ -279,22 +455,30 @@ static int efx_tc_flower_replace(struct efx_nic *efx, case FLOW_ACTION_REDIRECT: case FLOW_ACTION_MIRRED: save = *act; + + if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) { + /* can't happen */ + rc = -EOPNOTSUPP; + NL_SET_ERR_MSG_MOD(extack, "Deliver action violates action order (can't happen)"); + goto release; + } + to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); if (IS_ERR(to_efv)) { - EFX_TC_ERR_MSG(efx, extack, "Mirred egress device not on switch"); + NL_SET_ERR_MSG_MOD(extack, "Mirred egress device not on switch"); rc = PTR_ERR(to_efv); goto release; } rc = efx_tc_flower_external_mport(efx, to_efv); if (rc < 0) { - EFX_TC_ERR_MSG(efx, extack, "Failed to identify egress m-port"); + NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port"); goto release; } act->dest_mport = rc; act->deliver = 1; rc = efx_mae_alloc_action_set(efx, act); if (rc) { - EFX_TC_ERR_MSG(efx, extack, "Failed to write action set to hw (mirred)"); + NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (mirred)"); goto release; } list_add_tail(&act->list, &rule->acts.list); @@ -302,6 +486,7 @@ static int efx_tc_flower_replace(struct efx_nic *efx, if (fa->id == FLOW_ACTION_REDIRECT) break; /* end of the line */ /* Mirror, so continue on with saved act */ + save.count = NULL; act = kzalloc(sizeof(*act), GFP_USER); if (!act) { rc = -ENOMEM; @@ -310,9 +495,9 @@ static int efx_tc_flower_replace(struct efx_nic *efx, *act = save; break; default: - efx_tc_err(efx, "Unhandled action %u\n", fa->id); + NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u", + fa->id); rc = -EOPNOTSUPP; - NL_SET_ERR_MSG_MOD(extack, "Unsupported action"); goto release; } } @@ -334,7 +519,7 @@ static int efx_tc_flower_replace(struct efx_nic *efx, act->deliver = 1; rc = efx_mae_alloc_action_set(efx, act); if (rc) { - EFX_TC_ERR_MSG(efx, extack, "Failed to write action set to hw (deliver)"); + NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)"); goto release; } list_add_tail(&act->list, &rule->acts.list); @@ -349,13 +534,13 @@ static int efx_tc_flower_replace(struct efx_nic *efx, rc = efx_mae_alloc_action_set_list(efx, &rule->acts); if (rc) { - EFX_TC_ERR_MSG(efx, extack, "Failed to write action set list to hw"); + NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw"); goto release; } rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC, rule->acts.fw_id, &rule->fw_id); if (rc) { - EFX_TC_ERR_MSG(efx, extack, "Failed to insert rule in hw"); + NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw"); goto release_acts; } return 0; @@ -410,6 +595,42 @@ static int efx_tc_flower_destroy(struct efx_nic *efx, return 0; } +static int efx_tc_flower_stats(struct efx_nic *efx, struct net_device *net_dev, + struct flow_cls_offload *tc) +{ + struct netlink_ext_ack *extack = tc->common.extack; + struct efx_tc_counter_index *ctr; + struct efx_tc_counter *cnt; + u64 packets, bytes; + + ctr = efx_tc_flower_find_counter_index(efx, tc->cookie); + if (!ctr) { + /* See comment in efx_tc_flower_destroy() */ + if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev))) + if (net_ratelimit()) + netif_warn(efx, drv, efx->net_dev, + "Filter %lx not found for stats\n", + tc->cookie); + NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules"); + return -ENOENT; + } + if (WARN_ON(!ctr->cnt)) /* can't happen */ + return -EIO; + cnt = ctr->cnt; + + spin_lock_bh(&cnt->lock); + /* Report only new pkts/bytes since last time TC asked */ + packets = cnt->packets; + bytes = cnt->bytes; + flow_stats_update(&tc->stats, bytes - cnt->old_bytes, + packets - cnt->old_packets, 0, cnt->touched, + FLOW_ACTION_HW_STATS_DELAYED); + cnt->old_packets = packets; + cnt->old_bytes = bytes; + spin_unlock_bh(&cnt->lock); + return 0; +} + int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev, struct flow_cls_offload *tc, struct efx_rep *efv) { @@ -426,6 +647,9 @@ int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev, case FLOW_CLS_DESTROY: rc = efx_tc_flower_destroy(efx, net_dev, tc); break; + case FLOW_CLS_STATS: + rc = efx_tc_flower_stats(efx, net_dev, tc); + break; default: rc = -EOPNOTSUPP; break; @@ -641,6 +865,10 @@ int efx_init_struct_tc(struct efx_nic *efx) INIT_LIST_HEAD(&efx->tc->block_list); mutex_init(&efx->tc->mutex); + init_waitqueue_head(&efx->tc->flush_wq); + rc = efx_tc_init_counters(efx); + if (rc < 0) + goto fail_counters; rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params); if (rc < 0) goto fail_match_action_ht; @@ -650,8 +878,11 @@ int efx_init_struct_tc(struct efx_nic *efx) efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list); efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; + efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type; return 0; fail_match_action_ht: + efx_tc_destroy_counters(efx); +fail_counters: mutex_destroy(&efx->tc->mutex); kfree(efx->tc->caps); fail_alloc_caps: @@ -672,6 +903,7 @@ void efx_fini_struct_tc(struct efx_nic *efx) MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL); rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free, efx); + efx_tc_fini_counters(efx); mutex_unlock(&efx->tc->mutex); mutex_destroy(&efx->tc->mutex); kfree(efx->tc->caps); diff --git a/drivers/net/ethernet/sfc/tc.h b/drivers/net/ethernet/sfc/tc.h index 196fd74ed973..418ce8c13a06 100644 --- a/drivers/net/ethernet/sfc/tc.h +++ b/drivers/net/ethernet/sfc/tc.h @@ -14,27 +14,13 @@ #include <net/flow_offload.h> #include <linux/rhashtable.h> #include "net_driver.h" +#include "tc_counters.h" -/* Error reporting: convenience macros. For indicating why a given filter - * insertion is not supported; errors in internal operation or in the - * hardware should be netif_err()s instead. - */ -/* Used when error message is constant. */ -#define EFX_TC_ERR_MSG(efx, extack, message) do { \ - NL_SET_ERR_MSG_MOD(extack, message); \ - if (efx->log_tc_errs) \ - netif_info(efx, drv, efx->net_dev, "%s\n", message); \ -} while (0) -/* Used when error message is not constant; caller should also supply a - * constant extack message with NL_SET_ERR_MSG_MOD(). - */ -#define efx_tc_err(efx, fmt, args...) do { \ -if (efx->log_tc_errs) \ - netif_info(efx, drv, efx->net_dev, fmt, ##args);\ -} while (0) +#define IS_ALL_ONES(v) (!(typeof (v))~(v)) struct efx_tc_action_set { u16 deliver:1; + struct efx_tc_counter_index *count; u32 dest_mport; u32 fw_id; /* index of this entry in firmware actions table */ struct list_head list; @@ -44,6 +30,20 @@ struct efx_tc_match_fields { /* L1 */ u32 ingress_port; u8 recirc_id; + /* L2 (inner when encap) */ + __be16 eth_proto; + __be16 vlan_tci[2], vlan_proto[2]; + u8 eth_saddr[ETH_ALEN], eth_daddr[ETH_ALEN]; + /* L3 (when IP) */ + u8 ip_proto, ip_tos, ip_ttl; + __be32 src_ip, dst_ip; +#ifdef CONFIG_IPV6 + struct in6_addr src_ip6, dst_ip6; +#endif + bool ip_frag, ip_firstfrag; + /* L4 */ + __be16 l4_sport, l4_dport; /* Ports (UDP, TCP) */ + __be16 tcp_flags; }; struct efx_tc_match { @@ -76,11 +76,19 @@ enum efx_tc_rule_prios { * @caps: MAE capabilities reported by MCDI * @block_list: List of &struct efx_tc_block_binding * @mutex: Used to serialise operations on TC hashtables + * @counter_ht: Hashtable of TC counters (FW IDs and counter values) + * @counter_id_ht: Hashtable mapping TC counter cookies to counters * @match_action_ht: Hashtable of TC match-action rules * @reps_mport_id: MAE port allocated for representor RX * @reps_filter_uc: VNIC filter for representor unicast RX (promisc) * @reps_filter_mc: VNIC filter for representor multicast RX (allmulti) * @reps_mport_vport_id: vport_id for representor RX filters + * @flush_counters: counters have been stopped, waiting for drain + * @flush_gen: final generation count per type array as reported by + * MC_CMD_MAE_COUNTERS_STREAM_STOP + * @seen_gen: most recent generation count per type as seen by efx_tc_rx() + * @flush_wq: wait queue used by efx_mae_stop_counters() to wait for + * MAE counters RXQ to finish draining * @dflt: Match-action rules for default switching; at priority * %EFX_TC_PRIO_DFLT. Named by *ingress* port * @dflt.pf: rule for traffic ingressing from PF (egresses to wire) @@ -91,9 +99,15 @@ struct efx_tc_state { struct mae_caps *caps; struct list_head block_list; struct mutex mutex; + struct rhashtable counter_ht; + struct rhashtable counter_id_ht; struct rhashtable match_action_ht; u32 reps_mport_id, reps_mport_vport_id; s32 reps_filter_uc, reps_filter_mc; + bool flush_counters; + u32 flush_gen[EFX_TC_COUNTER_TYPE_MAX]; + u32 seen_gen[EFX_TC_COUNTER_TYPE_MAX]; + wait_queue_head_t flush_wq; struct { struct efx_tc_flow_rule pf; struct efx_tc_flow_rule wire; diff --git a/drivers/net/ethernet/sfc/tc_counters.c b/drivers/net/ethernet/sfc/tc_counters.c new file mode 100644 index 000000000000..d1a91d54c6bb --- /dev/null +++ b/drivers/net/ethernet/sfc/tc_counters.c @@ -0,0 +1,503 @@ +// SPDX-License-Identifier: GPL-2.0-only +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2022 Advanced Micro Devices, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +#include "tc_counters.h" +#include "mae_counter_format.h" +#include "mae.h" +#include "rx_common.h" + +/* Counter-management hashtables */ + +static const struct rhashtable_params efx_tc_counter_id_ht_params = { + .key_len = offsetof(struct efx_tc_counter_index, linkage), + .key_offset = 0, + .head_offset = offsetof(struct efx_tc_counter_index, linkage), +}; + +static const struct rhashtable_params efx_tc_counter_ht_params = { + .key_len = offsetof(struct efx_tc_counter, linkage), + .key_offset = 0, + .head_offset = offsetof(struct efx_tc_counter, linkage), +}; + +static void efx_tc_counter_free(void *ptr, void *__unused) +{ + struct efx_tc_counter *cnt = ptr; + + kfree(cnt); +} + +static void efx_tc_counter_id_free(void *ptr, void *__unused) +{ + struct efx_tc_counter_index *ctr = ptr; + + WARN_ON(refcount_read(&ctr->ref)); + kfree(ctr); +} + +int efx_tc_init_counters(struct efx_nic *efx) +{ + int rc; + + rc = rhashtable_init(&efx->tc->counter_id_ht, &efx_tc_counter_id_ht_params); + if (rc < 0) + goto fail_counter_id_ht; + rc = rhashtable_init(&efx->tc->counter_ht, &efx_tc_counter_ht_params); + if (rc < 0) + goto fail_counter_ht; + return 0; +fail_counter_ht: + rhashtable_destroy(&efx->tc->counter_id_ht); +fail_counter_id_ht: + return rc; +} + +/* Only call this in init failure teardown. + * Normal exit should fini instead as there may be entries in the table. + */ +void efx_tc_destroy_counters(struct efx_nic *efx) +{ + rhashtable_destroy(&efx->tc->counter_ht); + rhashtable_destroy(&efx->tc->counter_id_ht); +} + +void efx_tc_fini_counters(struct efx_nic *efx) +{ + rhashtable_free_and_destroy(&efx->tc->counter_id_ht, efx_tc_counter_id_free, NULL); + rhashtable_free_and_destroy(&efx->tc->counter_ht, efx_tc_counter_free, NULL); +} + +/* Counter allocation */ + +static struct efx_tc_counter *efx_tc_flower_allocate_counter(struct efx_nic *efx, + int type) +{ + struct efx_tc_counter *cnt; + int rc, rc2; + + cnt = kzalloc(sizeof(*cnt), GFP_USER); + if (!cnt) + return ERR_PTR(-ENOMEM); + + spin_lock_init(&cnt->lock); + cnt->touched = jiffies; + cnt->type = type; + + rc = efx_mae_allocate_counter(efx, cnt); + if (rc) + goto fail1; + rc = rhashtable_insert_fast(&efx->tc->counter_ht, &cnt->linkage, + efx_tc_counter_ht_params); + if (rc) + goto fail2; + return cnt; +fail2: + /* If we get here, it implies that we couldn't insert into the table, + * which in turn probably means that the fw_id was already taken. + * In that case, it's unclear whether we really 'own' the fw_id; but + * the firmware seemed to think we did, so it's proper to free it. + */ + rc2 = efx_mae_free_counter(efx, cnt); + if (rc2) + netif_warn(efx, hw, efx->net_dev, + "Failed to free MAE counter %u, rc %d\n", + cnt->fw_id, rc2); +fail1: + kfree(cnt); + return ERR_PTR(rc > 0 ? -EIO : rc); +} + +static void efx_tc_flower_release_counter(struct efx_nic *efx, + struct efx_tc_counter *cnt) +{ + int rc; + + rhashtable_remove_fast(&efx->tc->counter_ht, &cnt->linkage, + efx_tc_counter_ht_params); + rc = efx_mae_free_counter(efx, cnt); + if (rc) + netif_warn(efx, hw, efx->net_dev, + "Failed to free MAE counter %u, rc %d\n", + cnt->fw_id, rc); + /* This doesn't protect counter updates coming in arbitrarily long + * after we deleted the counter. The RCU just ensures that we won't + * free the counter while another thread has a pointer to it. + * Ensuring we don't update the wrong counter if the ID gets re-used + * is handled by the generation count. + */ + synchronize_rcu(); + EFX_WARN_ON_PARANOID(spin_is_locked(&cnt->lock)); + kfree(cnt); +} + +static struct efx_tc_counter *efx_tc_flower_find_counter_by_fw_id( + struct efx_nic *efx, int type, u32 fw_id) +{ + struct efx_tc_counter key = {}; + + key.fw_id = fw_id; + key.type = type; + + return rhashtable_lookup_fast(&efx->tc->counter_ht, &key, + efx_tc_counter_ht_params); +} + +/* TC cookie to counter mapping */ + +void efx_tc_flower_put_counter_index(struct efx_nic *efx, + struct efx_tc_counter_index *ctr) +{ + if (!refcount_dec_and_test(&ctr->ref)) + return; /* still in use */ + rhashtable_remove_fast(&efx->tc->counter_id_ht, &ctr->linkage, + efx_tc_counter_id_ht_params); + efx_tc_flower_release_counter(efx, ctr->cnt); + kfree(ctr); +} + +struct efx_tc_counter_index *efx_tc_flower_get_counter_index( + struct efx_nic *efx, unsigned long cookie, + enum efx_tc_counter_type type) +{ + struct efx_tc_counter_index *ctr, *old; + struct efx_tc_counter *cnt; + + ctr = kzalloc(sizeof(*ctr), GFP_USER); + if (!ctr) + return ERR_PTR(-ENOMEM); + ctr->cookie = cookie; + old = rhashtable_lookup_get_insert_fast(&efx->tc->counter_id_ht, + &ctr->linkage, + efx_tc_counter_id_ht_params); + if (old) { + /* don't need our new entry */ + kfree(ctr); + if (!refcount_inc_not_zero(&old->ref)) + return ERR_PTR(-EAGAIN); + /* existing entry found */ + ctr = old; + } else { + cnt = efx_tc_flower_allocate_counter(efx, type); + if (IS_ERR(cnt)) { + rhashtable_remove_fast(&efx->tc->counter_id_ht, + &ctr->linkage, + efx_tc_counter_id_ht_params); + kfree(ctr); + return (void *)cnt; /* it's an ERR_PTR */ + } + ctr->cnt = cnt; + refcount_set(&ctr->ref, 1); + } + return ctr; +} + +struct efx_tc_counter_index *efx_tc_flower_find_counter_index( + struct efx_nic *efx, unsigned long cookie) +{ + struct efx_tc_counter_index key = {}; + + key.cookie = cookie; + return rhashtable_lookup_fast(&efx->tc->counter_id_ht, &key, + efx_tc_counter_id_ht_params); +} + +/* TC Channel. Counter updates are delivered on this channel's RXQ. */ + +static void efx_tc_handle_no_channel(struct efx_nic *efx) +{ + netif_warn(efx, drv, efx->net_dev, + "MAE counters require MSI-X and 1 additional interrupt vector.\n"); +} + +static int efx_tc_probe_channel(struct efx_channel *channel) +{ + struct efx_rx_queue *rx_queue = &channel->rx_queue; + + channel->irq_moderation_us = 0; + rx_queue->core_index = 0; + + INIT_WORK(&rx_queue->grant_work, efx_mae_counters_grant_credits); + + return 0; +} + +static int efx_tc_start_channel(struct efx_channel *channel) +{ + struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); + struct efx_nic *efx = channel->efx; + + return efx_mae_start_counters(efx, rx_queue); +} + +static void efx_tc_stop_channel(struct efx_channel *channel) +{ + struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); + struct efx_nic *efx = channel->efx; + int rc; + + rc = efx_mae_stop_counters(efx, rx_queue); + if (rc) + netif_warn(efx, drv, efx->net_dev, + "Failed to stop MAE counters streaming, rc=%d.\n", + rc); + rx_queue->grant_credits = false; + flush_work(&rx_queue->grant_work); +} + +static void efx_tc_remove_channel(struct efx_channel *channel) +{ +} + +static void efx_tc_get_channel_name(struct efx_channel *channel, + char *buf, size_t len) +{ + snprintf(buf, len, "%s-mae", channel->efx->name); +} + +static void efx_tc_counter_update(struct efx_nic *efx, + enum efx_tc_counter_type counter_type, + u32 counter_idx, u64 packets, u64 bytes, + u32 mark) +{ + struct efx_tc_counter *cnt; + + rcu_read_lock(); /* Protect against deletion of 'cnt' */ + cnt = efx_tc_flower_find_counter_by_fw_id(efx, counter_type, counter_idx); + if (!cnt) { + /* This can legitimately happen when a counter is removed, + * with updates for the counter still in-flight; however this + * should be an infrequent occurrence. + */ + if (net_ratelimit()) + netif_dbg(efx, drv, efx->net_dev, + "Got update for unwanted MAE counter %u type %u\n", + counter_idx, counter_type); + goto out; + } + + spin_lock_bh(&cnt->lock); + if ((s32)mark - (s32)cnt->gen < 0) { + /* This counter update packet is from before the counter was + * allocated; thus it must be for a previous counter with + * the same ID that has since been freed, and it should be + * ignored. + */ + } else { + /* Update latest seen generation count. This ensures that + * even a long-lived counter won't start getting ignored if + * the generation count wraps around, unless it somehow + * manages to go 1<<31 generations without an update. + */ + cnt->gen = mark; + /* update counter values */ + cnt->packets += packets; + cnt->bytes += bytes; + cnt->touched = jiffies; + } + spin_unlock_bh(&cnt->lock); +out: + rcu_read_unlock(); +} + +static void efx_tc_rx_version_1(struct efx_nic *efx, const u8 *data, u32 mark) +{ + u16 n_counters, i; + + /* Header format: + * + | 0 | 1 | 2 | 3 | + * 0 |version | reserved | + * 4 | seq_index | n_counters | + */ + + n_counters = le16_to_cpu(*(const __le16 *)(data + 6)); + + /* Counter update entry format: + * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | a | b | c | d | e | f | + * | counter_idx | packet_count | byte_count | + */ + for (i = 0; i < n_counters; i++) { + const void *entry = data + 8 + 16 * i; + u64 packet_count, byte_count; + u32 counter_idx; + + counter_idx = le32_to_cpu(*(const __le32 *)entry); + packet_count = le32_to_cpu(*(const __le32 *)(entry + 4)) | + ((u64)le16_to_cpu(*(const __le16 *)(entry + 8)) << 32); + byte_count = le16_to_cpu(*(const __le16 *)(entry + 10)) | + ((u64)le32_to_cpu(*(const __le32 *)(entry + 12)) << 16); + efx_tc_counter_update(efx, EFX_TC_COUNTER_TYPE_AR, counter_idx, + packet_count, byte_count, mark); + } +} + +#define TCV2_HDR_PTR(pkt, field) \ + ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_LBN & 7), \ + (pkt) + ERF_SC_PACKETISER_HEADER_##field##_LBN / 8) +#define TCV2_HDR_BYTE(pkt, field) \ + ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_WIDTH != 8),\ + *TCV2_HDR_PTR(pkt, field)) +#define TCV2_HDR_WORD(pkt, field) \ + ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_WIDTH != 16),\ + (void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_LBN & 15), \ + *(__force const __le16 *)TCV2_HDR_PTR(pkt, field)) +#define TCV2_PKT_PTR(pkt, poff, i, field) \ + ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_PAYLOAD_##field##_LBN & 7), \ + (pkt) + ERF_SC_PACKETISER_PAYLOAD_##field##_LBN/8 + poff + \ + i * ER_RX_SL_PACKETISER_PAYLOAD_WORD_SIZE) + +/* Read a little-endian 48-bit field with 16-bit alignment */ +static u64 efx_tc_read48(const __le16 *field) +{ + u64 out = 0; + int i; + + for (i = 0; i < 3; i++) + out |= (u64)le16_to_cpu(field[i]) << (i * 16); + return out; +} + +static enum efx_tc_counter_type efx_tc_rx_version_2(struct efx_nic *efx, + const u8 *data, u32 mark) +{ + u8 payload_offset, header_offset, ident; + enum efx_tc_counter_type type; + u16 n_counters, i; + + ident = TCV2_HDR_BYTE(data, IDENTIFIER); + switch (ident) { + case ERF_SC_PACKETISER_HEADER_IDENTIFIER_AR: + type = EFX_TC_COUNTER_TYPE_AR; + break; + case ERF_SC_PACKETISER_HEADER_IDENTIFIER_CT: + type = EFX_TC_COUNTER_TYPE_CT; + break; + case ERF_SC_PACKETISER_HEADER_IDENTIFIER_OR: + type = EFX_TC_COUNTER_TYPE_OR; + break; + default: + if (net_ratelimit()) + netif_err(efx, drv, efx->net_dev, + "ignored v2 MAE counter packet (bad identifier %u" + "), counters may be inaccurate\n", ident); + return EFX_TC_COUNTER_TYPE_MAX; + } + header_offset = TCV2_HDR_BYTE(data, HEADER_OFFSET); + /* mae_counter_format.h implies that this offset is fixed, since it + * carries on with SOP-based LBNs for the fields in this header + */ + if (header_offset != ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_DEFAULT) { + if (net_ratelimit()) + netif_err(efx, drv, efx->net_dev, + "choked on v2 MAE counter packet (bad header_offset %u" + "), counters may be inaccurate\n", header_offset); + return EFX_TC_COUNTER_TYPE_MAX; + } + payload_offset = TCV2_HDR_BYTE(data, PAYLOAD_OFFSET); + n_counters = le16_to_cpu(TCV2_HDR_WORD(data, COUNT)); + + for (i = 0; i < n_counters; i++) { + const void *counter_idx_p, *packet_count_p, *byte_count_p; + u64 packet_count, byte_count; + u32 counter_idx; + + /* 24-bit field with 32-bit alignment */ + counter_idx_p = TCV2_PKT_PTR(data, payload_offset, i, COUNTER_INDEX); + BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_WIDTH != 24); + BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_LBN & 31); + counter_idx = le32_to_cpu(*(const __le32 *)counter_idx_p) & 0xffffff; + /* 48-bit field with 16-bit alignment */ + packet_count_p = TCV2_PKT_PTR(data, payload_offset, i, PACKET_COUNT); + BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_WIDTH != 48); + BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_LBN & 15); + packet_count = efx_tc_read48((const __le16 *)packet_count_p); + /* 48-bit field with 16-bit alignment */ + byte_count_p = TCV2_PKT_PTR(data, payload_offset, i, BYTE_COUNT); + BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_WIDTH != 48); + BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_LBN & 15); + byte_count = efx_tc_read48((const __le16 *)byte_count_p); + + if (type == EFX_TC_COUNTER_TYPE_CT) { + /* CT counters are 1-bit saturating counters to update + * the lastuse time in CT stats. A received CT counter + * should have packet counter to 0 and only LSB bit on + * in byte counter. + */ + if (packet_count || byte_count != 1) + netdev_warn_once(efx->net_dev, + "CT counter with inconsistent state (%llu, %llu)\n", + packet_count, byte_count); + /* Do not increment the driver's byte counter */ + byte_count = 0; + } + + efx_tc_counter_update(efx, type, counter_idx, packet_count, + byte_count, mark); + } + return type; +} + +/* We always swallow the packet, whether successful or not, since it's not + * a network packet and shouldn't ever be forwarded to the stack. + * @mark is the generation count for counter allocations. + */ +static bool efx_tc_rx(struct efx_rx_queue *rx_queue, u32 mark) +{ + struct efx_channel *channel = efx_rx_queue_channel(rx_queue); + struct efx_rx_buffer *rx_buf = efx_rx_buffer(rx_queue, + channel->rx_pkt_index); + const u8 *data = efx_rx_buf_va(rx_buf); + struct efx_nic *efx = rx_queue->efx; + enum efx_tc_counter_type type; + u8 version; + + /* version is always first byte of packet */ + version = *data; + switch (version) { + case 1: + type = EFX_TC_COUNTER_TYPE_AR; + efx_tc_rx_version_1(efx, data, mark); + break; + case ERF_SC_PACKETISER_HEADER_VERSION_VALUE: // 2 + type = efx_tc_rx_version_2(efx, data, mark); + break; + default: + if (net_ratelimit()) + netif_err(efx, drv, efx->net_dev, + "choked on MAE counter packet (bad version %u" + "); counters may be inaccurate\n", + version); + goto out; + } + + if (type < EFX_TC_COUNTER_TYPE_MAX) { + /* Update seen_gen unconditionally, to avoid a missed wakeup if + * we race with efx_mae_stop_counters(). + */ + efx->tc->seen_gen[type] = mark; + if (efx->tc->flush_counters && + (s32)(efx->tc->flush_gen[type] - mark) <= 0) + wake_up(&efx->tc->flush_wq); + } +out: + efx_free_rx_buffers(rx_queue, rx_buf, 1); + channel->rx_pkt_n_frags = 0; + return true; +} + +const struct efx_channel_type efx_tc_channel_type = { + .handle_no_channel = efx_tc_handle_no_channel, + .pre_probe = efx_tc_probe_channel, + .start = efx_tc_start_channel, + .stop = efx_tc_stop_channel, + .post_remove = efx_tc_remove_channel, + .get_name = efx_tc_get_channel_name, + .receive_raw = efx_tc_rx, + .keep_eventq = true, +}; diff --git a/drivers/net/ethernet/sfc/tc_counters.h b/drivers/net/ethernet/sfc/tc_counters.h new file mode 100644 index 000000000000..8fc7c4bbb29c --- /dev/null +++ b/drivers/net/ethernet/sfc/tc_counters.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2022 Advanced Micro Devices, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +#ifndef EFX_TC_COUNTERS_H +#define EFX_TC_COUNTERS_H +#include <linux/refcount.h> +#include "net_driver.h" + +#include "mcdi_pcol.h" /* for MAE_COUNTER_TYPE_* */ + +enum efx_tc_counter_type { + EFX_TC_COUNTER_TYPE_AR = MAE_COUNTER_TYPE_AR, + EFX_TC_COUNTER_TYPE_CT = MAE_COUNTER_TYPE_CT, + EFX_TC_COUNTER_TYPE_OR = MAE_COUNTER_TYPE_OR, + EFX_TC_COUNTER_TYPE_MAX +}; + +struct efx_tc_counter { + u32 fw_id; /* index in firmware counter table */ + enum efx_tc_counter_type type; + struct rhash_head linkage; /* efx->tc->counter_ht */ + spinlock_t lock; /* Serialises updates to counter values */ + u32 gen; /* Generation count at which this counter is current */ + u64 packets, bytes; + u64 old_packets, old_bytes; /* Values last time passed to userspace */ + /* jiffies of the last time we saw packets increase */ + unsigned long touched; +}; + +struct efx_tc_counter_index { + unsigned long cookie; + struct rhash_head linkage; /* efx->tc->counter_id_ht */ + refcount_t ref; + struct efx_tc_counter *cnt; +}; + +/* create/uncreate/teardown hashtables */ +int efx_tc_init_counters(struct efx_nic *efx); +void efx_tc_destroy_counters(struct efx_nic *efx); +void efx_tc_fini_counters(struct efx_nic *efx); + +struct efx_tc_counter_index *efx_tc_flower_get_counter_index( + struct efx_nic *efx, unsigned long cookie, + enum efx_tc_counter_type type); +void efx_tc_flower_put_counter_index(struct efx_nic *efx, + struct efx_tc_counter_index *ctr); +struct efx_tc_counter_index *efx_tc_flower_find_counter_index( + struct efx_nic *efx, unsigned long cookie); + +extern const struct efx_channel_type efx_tc_channel_type; + +#endif /* EFX_TC_COUNTERS_H */ diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c index c5f88f7a7a04..4ed4082836a9 100644 --- a/drivers/net/ethernet/sfc/tx.c +++ b/drivers/net/ethernet/sfc/tx.c @@ -207,11 +207,11 @@ static void efx_skb_copy_bits_to_pio(struct efx_nic *efx, struct sk_buff *skb, skb_frag_t *f = &skb_shinfo(skb)->frags[i]; u8 *vaddr; - vaddr = kmap_atomic(skb_frag_page(f)); + vaddr = kmap_local_page(skb_frag_page(f)); efx_memcpy_toio_aligned_cb(efx, piobuf, vaddr + skb_frag_off(f), skb_frag_size(f), copy_buf); - kunmap_atomic(vaddr); + kunmap_local(vaddr); } EFX_WARN_ON_ONCE_PARANOID(skb_shinfo(skb)->frag_list); |