diff options
Diffstat (limited to 'drivers/net/ethernet/meta')
22 files changed, 3323 insertions, 590 deletions
diff --git a/drivers/net/ethernet/meta/fbnic/Makefile b/drivers/net/ethernet/meta/fbnic/Makefile index 0dbc634adb4b..15e8ff649615 100644 --- a/drivers/net/ethernet/meta/fbnic/Makefile +++ b/drivers/net/ethernet/meta/fbnic/Makefile @@ -12,6 +12,7 @@ fbnic-y := fbnic_csr.o \ fbnic_devlink.o \ fbnic_ethtool.o \ fbnic_fw.o \ + fbnic_fw_log.o \ fbnic_hw_stats.o \ fbnic_hwmon.o \ fbnic_irq.o \ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic.h b/drivers/net/ethernet/meta/fbnic/fbnic.h index 65815d4f379e..b03e5a3d5144 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic.h @@ -12,6 +12,7 @@ #include "fbnic_csr.h" #include "fbnic_fw.h" +#include "fbnic_fw_log.h" #include "fbnic_hw_stats.h" #include "fbnic_mac.h" #include "fbnic_rpc.h" @@ -26,6 +27,8 @@ struct fbnic_dev { struct net_device *netdev; struct dentry *dbg_fbd; struct device *hwmon; + struct devlink_health_reporter *fw_reporter; + struct devlink_health_reporter *otp_reporter; u32 __iomem *uc_addr0; u32 __iomem *uc_addr4; @@ -83,8 +86,11 @@ struct fbnic_dev { /* Local copy of hardware statistics */ struct fbnic_hw_stats hw_stats; - /* Lock protecting access to hw_stats */ - spinlock_t hw_stats_lock; + /* Firmware time since boot in milliseconds */ + u64 firmware_time; + u64 prev_firmware_time; + + struct fbnic_fw_log fw_log; }; /* Reserve entry 0 in the MSI-X "others" array until we have filled all @@ -155,8 +161,13 @@ extern char fbnic_driver_name[]; void fbnic_devlink_free(struct fbnic_dev *fbd); struct fbnic_dev *fbnic_devlink_alloc(struct pci_dev *pdev); +int fbnic_devlink_health_create(struct fbnic_dev *fbd); +void fbnic_devlink_health_destroy(struct fbnic_dev *fbd); void fbnic_devlink_register(struct fbnic_dev *fbd); void fbnic_devlink_unregister(struct fbnic_dev *fbd); +void __printf(2, 3) +fbnic_devlink_fw_report(struct fbnic_dev *fbd, const char *format, ...); +void fbnic_devlink_otp_check(struct fbnic_dev *fbd, const char *msg); int fbnic_fw_request_mbx(struct fbnic_dev *fbd); void fbnic_fw_free_mbx(struct fbnic_dev *fbd); @@ -187,6 +198,8 @@ void fbnic_dbg_fbd_exit(struct fbnic_dev *fbd); void fbnic_dbg_init(void); void fbnic_dbg_exit(void); +void fbnic_rpc_reset_valid_entries(struct fbnic_dev *fbd); + void fbnic_csr_get_regs(struct fbnic_dev *fbd, u32 *data, u32 *regs_version); int fbnic_csr_regs_len(struct fbnic_dev *fbd); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h index 36393a17d92d..d3a7ad921f18 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h @@ -12,13 +12,28 @@ #define DESC_BIT(nr) BIT_ULL(nr) #define DESC_GENMASK(h, l) GENMASK_ULL(h, l) +#define FW_VER_CODE(_major, _minor, _patch, _build) ( \ + FIELD_PREP(FBNIC_FW_CAP_RESP_VERSION_MAJOR, _major) | \ + FIELD_PREP(FBNIC_FW_CAP_RESP_VERSION_MINOR, _minor) | \ + FIELD_PREP(FBNIC_FW_CAP_RESP_VERSION_PATCH, _patch) | \ + FIELD_PREP(FBNIC_FW_CAP_RESP_VERSION_BUILD, _build)) + /* Defines the minimum firmware version required by the driver */ -#define MIN_FW_MAJOR_VERSION 0 -#define MIN_FW_MINOR_VERSION 10 -#define MIN_FW_BUILD_VERSION 6 -#define MIN_FW_VERSION_CODE (MIN_FW_MAJOR_VERSION * (1u << 24) + \ - MIN_FW_MINOR_VERSION * (1u << 16) + \ - MIN_FW_BUILD_VERSION) +#define MIN_FW_VER_CODE FW_VER_CODE(0, 10, 6, 0) + +/* Defines the minimum firmware version required for firmware logs */ +#define MIN_FW_VER_CODE_LOG FW_VER_CODE(0, 12, 9, 0) + +/* Driver can request that firmware sends all cached logs in bulk. This + * feature was enabled on older firmware however firmware has a bug + * which attempted to send 30 messages per mbx message which caused an + * overflow flooding the mailbox. This results in a kernel warning + * related to corrupt mailbox messages. + * + * If firmware is new enough only request sending historical logs when + * the log buffer is empty to prevent duplicate logs. + */ +#define MIN_FW_VER_CODE_HIST FW_VER_CODE(25, 5, 7, 0) #define PCI_DEVICE_ID_META_FBNIC_ASIC 0x0013 @@ -446,6 +461,26 @@ enum { #define FBNIC_TMI_ILLEGAL_PTP_REQS 0x04409 /* 0x11024 */ #define FBNIC_TMI_GOOD_PTP_TS 0x0440a /* 0x11028 */ #define FBNIC_TMI_BAD_PTP_TS 0x0440b /* 0x1102c */ +#define FBNIC_TMI_STAT_TX_PACKET_1519_2047_BYTES_L \ + 0x04433 /* 0x110cc */ +#define FBNIC_TMI_STAT_TX_PACKET_1519_2047_BYTES_H \ + 0x04434 /* 0x110d0 */ +#define FBNIC_TMI_STAT_TX_PACKET_2048_4095_BYTES_L \ + 0x04435 /* 0x110d4 */ +#define FBNIC_TMI_STAT_TX_PACKET_2048_4095_BYTES_H \ + 0x04436 /* 0x110d8 */ +#define FBNIC_TMI_STAT_TX_PACKET_4096_8191_BYTES_L \ + 0x04437 /* 0x110dc */ +#define FBNIC_TMI_STAT_TX_PACKET_4096_8191_BYTES_H \ + 0x04438 /* 0x110e0 */ +#define FBNIC_TMI_STAT_TX_PACKET_8192_9216_BYTES_L \ + 0x04439 /* 0x110e4 */ +#define FBNIC_TMI_STAT_TX_PACKET_8192_9216_BYTES_H \ + 0x0443a /* 0x110e8 */ +#define FBNIC_TMI_STAT_TX_PACKET_9217_MAX_BYTES_L \ + 0x0443b /* 0x110ec */ +#define FBNIC_TMI_STAT_TX_PACKET_9217_MAX_BYTES_H \ + 0x0443c /* 0x110f0 */ #define FBNIC_CSR_END_TMI 0x0443f /* CSR section delimiter */ /* Precision Time Protocol Registers */ @@ -473,7 +508,7 @@ enum { #define FBNIC_PTP_ADD_VAL_NS 0x04806 /* 0x12018 */ #define FBNIC_PTP_ADD_VAL_NS_MASK CSR_GENMASK(15, 0) -#define FBNIC_PTP_ADD_VAL_SUBNS 0x04807 /* 0x1201c */ +#define FBNIC_PTP_ADD_VAL_SUBNS 0x04807 /* 0x1201c */ #define FBNIC_PTP_CTR_VAL_HI 0x04808 /* 0x12020 */ #define FBNIC_PTP_CTR_VAL_LO 0x04809 /* 0x12024 */ @@ -674,6 +709,26 @@ enum { #define FBNIC_RPC_CNTR_OVR_SIZE_ERR 0x084a6 /* 0x21298 */ #define FBNIC_RPC_TCAM_MACDA_VALIDATE 0x0852d /* 0x214b4 */ +#define FBNIC_RPC_STAT_RX_PACKET_1519_2047_BYTES_L \ + 0x0855f /* 0x2157c */ +#define FBNIC_RPC_STAT_RX_PACKET_1519_2047_BYTES_H \ + 0x08560 /* 0x21580 */ +#define FBNIC_RPC_STAT_RX_PACKET_2048_4095_BYTES_L \ + 0x08561 /* 0x21584 */ +#define FBNIC_RPC_STAT_RX_PACKET_2048_4095_BYTES_H \ + 0x08562 /* 0x21588 */ +#define FBNIC_RPC_STAT_RX_PACKET_4096_8191_BYTES_L \ + 0x08563 /* 0x2158c */ +#define FBNIC_RPC_STAT_RX_PACKET_4096_8191_BYTES_H \ + 0x08564 /* 0x21590 */ +#define FBNIC_RPC_STAT_RX_PACKET_8192_9216_BYTES_L \ + 0x08565 /* 0x21594 */ +#define FBNIC_RPC_STAT_RX_PACKET_8192_9216_BYTES_H \ + 0x08566 /* 0x21598 */ +#define FBNIC_RPC_STAT_RX_PACKET_9217_MAX_BYTES_L \ + 0x08567 /* 0x2159c */ +#define FBNIC_RPC_STAT_RX_PACKET_9217_MAX_BYTES_H \ + 0x08568 /* 0x215a0 */ #define FBNIC_CSR_END_RPC 0x0856b /* CSR section delimiter */ /* RPC RAM Registers */ @@ -735,6 +790,21 @@ enum { #define FBNIC_CSR_END_PCS 0x10668 /* CSR section delimiter */ #define FBNIC_CSR_START_RSFEC 0x10800 /* CSR section delimiter */ + +/* We have 4 RSFEC engines present in our part, however we are only using 1. + * As such only CCW(0) and NCCW(0) will never be non-zero and the other + * registers can be ignored. + */ +#define FBNIC_RSFEC_CCW_LO(n) (0x10802 + 8 * (n)) /* 0x42008 + 32*n */ +#define FBNIC_RSFEC_CCW_HI(n) (0x10803 + 8 * (n)) /* 0x4200c + 32*n */ +#define FBNIC_RSFEC_NCCW_LO(n) (0x10804 + 8 * (n)) /* 0x42010 + 32*n */ +#define FBNIC_RSFEC_NCCW_HI(n) (0x10805 + 8 * (n)) /* 0x42014 + 32*n */ + +#define FBNIC_PCS_MAX_LANES 4 +#define FBNIC_PCS_SYMBLERR_LO(n) \ + (0x10880 + 2 * (n)) /* 0x42200 + 8*n */ +#define FBNIC_PCS_SYMBLERR_HI(n) \ + (0x10881 + 2 * (n)) /* 0x42204 + 8*n */ #define FBNIC_CSR_END_RSFEC 0x108c8 /* CSR section delimiter */ /* MAC MAC registers (ASIC only) */ @@ -774,18 +844,18 @@ enum { #define FBNIC_CSR_END_SIG 0x1184e /* CSR section delimiter */ #define FBNIC_CSR_START_MAC_STAT 0x11a00 +#define FBNIC_MAC_STAT_RX_XOFF_STB_L 0x11a00 /* 0x46800 */ +#define FBNIC_MAC_STAT_RX_XOFF_STB_H 0x11a01 /* 0x46804 */ +#define FBNIC_MAC_STAT_TX_XOFF_STB_L 0x11a04 /* 0x46810 */ +#define FBNIC_MAC_STAT_TX_XOFF_STB_H 0x11a05 /* 0x46814 */ #define FBNIC_MAC_STAT_RX_BYTE_COUNT_L 0x11a08 /* 0x46820 */ #define FBNIC_MAC_STAT_RX_BYTE_COUNT_H 0x11a09 /* 0x46824 */ -#define FBNIC_MAC_STAT_RX_ALIGN_ERROR_L \ - 0x11a0a /* 0x46828 */ -#define FBNIC_MAC_STAT_RX_ALIGN_ERROR_H \ - 0x11a0b /* 0x4682c */ +#define FBNIC_MAC_STAT_RX_ALIGN_ERROR_L 0x11a0a /* 0x46828 */ +#define FBNIC_MAC_STAT_RX_ALIGN_ERROR_H 0x11a0b /* 0x4682c */ #define FBNIC_MAC_STAT_RX_TOOLONG_L 0x11a0e /* 0x46838 */ #define FBNIC_MAC_STAT_RX_TOOLONG_H 0x11a0f /* 0x4683c */ -#define FBNIC_MAC_STAT_RX_RECEIVED_OK_L \ - 0x11a12 /* 0x46848 */ -#define FBNIC_MAC_STAT_RX_RECEIVED_OK_H \ - 0x11a13 /* 0x4684c */ +#define FBNIC_MAC_STAT_RX_RECEIVED_OK_L 0x11a12 /* 0x46848 */ +#define FBNIC_MAC_STAT_RX_RECEIVED_OK_H 0x11a13 /* 0x4684c */ #define FBNIC_MAC_STAT_RX_PACKET_BAD_FCS_L \ 0x11a14 /* 0x46850 */ #define FBNIC_MAC_STAT_RX_PACKET_BAD_FCS_H \ @@ -796,20 +866,90 @@ enum { #define FBNIC_MAC_STAT_RX_MULTICAST_H 0x11a1d /* 0x46874 */ #define FBNIC_MAC_STAT_RX_BROADCAST_L 0x11a1e /* 0x46878 */ #define FBNIC_MAC_STAT_RX_BROADCAST_H 0x11a1f /* 0x4687c */ +#define FBNIC_MAC_STAT_RX_UNDERSIZE_L 0x11a24 /* 0x46890 */ +#define FBNIC_MAC_STAT_RX_UNDERSIZE_H 0x11a25 /* 0x46894 */ +#define FBNIC_MAC_STAT_RX_PACKET_64_BYTES_L \ + 0x11a26 /* 0x46898 */ +#define FBNIC_MAC_STAT_RX_PACKET_64_BYTES_H \ + 0x11a27 /* 0x4689c */ +#define FBNIC_MAC_STAT_RX_PACKET_65_127_BYTES_L \ + 0x11a28 /* 0x468a0 */ +#define FBNIC_MAC_STAT_RX_PACKET_65_127_BYTES_H \ + 0x11a29 /* 0x468a4 */ +#define FBNIC_MAC_STAT_RX_PACKET_128_255_BYTES_L \ + 0x11a2a /* 0x468a8 */ +#define FBNIC_MAC_STAT_RX_PACKET_128_255_BYTES_H \ + 0x11a2b /* 0x468ac */ +#define FBNIC_MAC_STAT_RX_PACKET_256_511_BYTES_L \ + 0x11a2c /* 0x468b0 */ +#define FBNIC_MAC_STAT_RX_PACKET_256_511_BYTES_H \ + 0x11a2d /* 0x468b4 */ +#define FBNIC_MAC_STAT_RX_PACKET_512_1023_BYTES_L \ + 0x11a2e /* 0x468b8 */ +#define FBNIC_MAC_STAT_RX_PACKET_512_1023_BYTES_H \ + 0x11a2f /* 0x468bc */ +#define FBNIC_MAC_STAT_RX_PACKET_1024_1518_BYTES_L \ + 0x11a30 /* 0x468c0 */ +#define FBNIC_MAC_STAT_RX_PACKET_1024_1518_BYTES_H \ + 0x11a31 /* 0x468c4 */ +#define FBNIC_MAC_STAT_RX_PACKET_1519_MAX_BYTES_L \ + 0x11a32 /* 0x468c8 */ +#define FBNIC_MAC_STAT_RX_PACKET_1519_MAX_BYTES_H \ + 0x11a33 /* 0x468cc */ +#define FBNIC_MAC_STAT_RX_OVERSIZE_L 0x11a34 /* 0x468d0 */ +#define FBNIC_MAC_STAT_RX_OVERSSIZE_H 0x11a35 /* 0x468d4 */ +#define FBNIC_MAC_STAT_RX_JABBER_L 0x11a36 /* 0x468d8 */ +#define FBNIC_MAC_STAT_RX_JABBER_H 0x11a37 /* 0x468dc */ +#define FBNIC_MAC_STAT_RX_FRAGMENT_L 0x11a38 /* 0x468e0 */ +#define FBNIC_MAC_STAT_RX_FRAGMENT_H 0x11a39 /* 0x468e4 */ +#define FBNIC_MAC_STAT_RX_CONTROL_FRAMES_L \ + 0x11a3c /* 0x468f0 */ +#define FBNIC_MAC_STAT_RX_CONTROL_FRAMES_H \ + 0x11a3d /* 0x468f4 */ #define FBNIC_MAC_STAT_TX_BYTE_COUNT_L 0x11a3e /* 0x468f8 */ #define FBNIC_MAC_STAT_TX_BYTE_COUNT_H 0x11a3f /* 0x468fc */ #define FBNIC_MAC_STAT_TX_TRANSMITTED_OK_L \ 0x11a42 /* 0x46908 */ #define FBNIC_MAC_STAT_TX_TRANSMITTED_OK_H \ 0x11a43 /* 0x4690c */ -#define FBNIC_MAC_STAT_TX_IFOUTERRORS_L \ - 0x11a46 /* 0x46918 */ -#define FBNIC_MAC_STAT_TX_IFOUTERRORS_H \ - 0x11a47 /* 0x4691c */ +#define FBNIC_MAC_STAT_TX_IFOUTERRORS_L 0x11a46 /* 0x46918 */ +#define FBNIC_MAC_STAT_TX_IFOUTERRORS_H 0x11a47 /* 0x4691c */ #define FBNIC_MAC_STAT_TX_MULTICAST_L 0x11a4a /* 0x46928 */ #define FBNIC_MAC_STAT_TX_MULTICAST_H 0x11a4b /* 0x4692c */ #define FBNIC_MAC_STAT_TX_BROADCAST_L 0x11a4c /* 0x46930 */ #define FBNIC_MAC_STAT_TX_BROADCAST_H 0x11a4d /* 0x46934 */ +#define FBNIC_MAC_STAT_TX_PACKET_64_BYTES_L \ + 0x11a4e /* 0x46938 */ +#define FBNIC_MAC_STAT_TX_PACKET_64_BYTES_H \ + 0x11a4f /* 0x4693c */ +#define FBNIC_MAC_STAT_TX_PACKET_65_127_BYTES_L \ + 0x11a50 /* 0x46940 */ +#define FBNIC_MAC_STAT_TX_PACKET_65_127_BYTES_H \ + 0x11a51 /* 0x46944 */ +#define FBNIC_MAC_STAT_TX_PACKET_128_255_BYTES_L \ + 0x11a52 /* 0x46948 */ +#define FBNIC_MAC_STAT_TX_PACKET_128_255_BYTES_H \ + 0x11a53 /* 0x4694c */ +#define FBNIC_MAC_STAT_TX_PACKET_256_511_BYTES_L \ + 0x11a54 /* 0x46950 */ +#define FBNIC_MAC_STAT_TX_PACKET_256_511_BYTES_H \ + 0x11a55 /* 0x46954 */ +#define FBNIC_MAC_STAT_TX_PACKET_512_1023_BYTES_L \ + 0x11a56 /* 0x46958 */ +#define FBNIC_MAC_STAT_TX_PACKET_512_1023_BYTES_H \ + 0x11a57 /* 0x4695c */ +#define FBNIC_MAC_STAT_TX_PACKET_1024_1518_BYTES_L \ + 0x11a58 /* 0x46960 */ +#define FBNIC_MAC_STAT_TX_PACKET_1024_1518_BYTES_H \ + 0x11a59 /* 0x46964 */ +#define FBNIC_MAC_STAT_TX_PACKET_1519_MAX_BYTES_L \ + 0x11a5a /* 0x46968 */ +#define FBNIC_MAC_STAT_TX_PACKET_1519_MAX_BYTES_H \ + 0x11a5b /* 0x4696c */ +#define FBNIC_MAC_STAT_TX_CONTROL_FRAMES_L \ + 0x11a5e /* 0x46978 */ +#define FBNIC_MAC_STAT_TX_CONTROL_FRAMES_H \ + 0x11a5f /* 0x4697c */ /* PCIE Comphy Registers */ #define FBNIC_CSR_START_PCIE_SS_COMPHY 0x2442e /* CSR section delimiter */ @@ -857,7 +997,7 @@ enum { 0x3107e /* 0xc41f8 */ #define FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_63_32 \ 0x3107f /* 0xc41fc */ -#define FBNIC_CSR_END_PUL_USER 0x310ea /* CSR section delimiter */ +#define FBNIC_CSR_END_PUL_USER 0x310ea /* CSR section delimiter */ /* Queue Registers * @@ -1038,4 +1178,22 @@ enum { #define FBNIC_IPC_MBX_DESC_FW_CMPL DESC_BIT(1) #define FBNIC_IPC_MBX_DESC_HOST_CMPL DESC_BIT(0) +/* OTP Registers + * These registers are accessible via bar4 offset and are written by CMRT + * on boot. For the write status, the register is broken up in half with OTP + * Write Data Status occupying the top 16 bits and the ECC status occupying the + * bottom 16 bits. + */ +#define FBNIC_NS_OTP_STATUS 0x0021d +#define FBNIC_NS_OTP_WRITE_STATUS 0x0021e + +#define FBNIC_NS_OTP_WRITE_DATA_STATUS_MASK CSR_GENMASK(31, 16) +#define FBNIC_NS_OTP_WRITE_ECC_STATUS_MASK CSR_GENMASK(15, 0) + +#define FBNIC_REGS_VERSION CSR_GENMASK(31, 16) +#define FBNIC_REGS_HW_TYPE CSR_GENMASK(15, 8) +enum{ + FBNIC_CSR_VERSION_V1_0_ASIC = 1, +}; + #endif /* _FBNIC_CSR_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c index e8f2d7f2d962..b7238dd967fe 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c @@ -170,6 +170,33 @@ static int fbnic_dbg_ipo_dst_show(struct seq_file *s, void *v) } DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_ipo_dst); +static int fbnic_dbg_fw_log_show(struct seq_file *s, void *v) +{ + struct fbnic_dev *fbd = s->private; + struct fbnic_fw_log_entry *entry; + unsigned long flags; + + if (!fbnic_fw_log_ready(fbd)) + return -ENXIO; + + spin_lock_irqsave(&fbd->fw_log.lock, flags); + + list_for_each_entry_reverse(entry, &fbd->fw_log.entries, list) { + seq_printf(s, FBNIC_FW_LOG_FMT, entry->index, + (entry->timestamp / (MSEC_PER_SEC * 60 * 60 * 24)), + (entry->timestamp / (MSEC_PER_SEC * 60 * 60)) % 24, + ((entry->timestamp / (MSEC_PER_SEC * 60) % 60)), + ((entry->timestamp / MSEC_PER_SEC) % 60), + (entry->timestamp % MSEC_PER_SEC), + entry->msg); + } + + spin_unlock_irqrestore(&fbd->fw_log.lock, flags); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_fw_log); + static int fbnic_dbg_pcie_stats_show(struct seq_file *s, void *v) { struct fbnic_dev *fbd = s->private; @@ -222,6 +249,8 @@ void fbnic_dbg_fbd_init(struct fbnic_dev *fbd) &fbnic_dbg_ipo_src_fops); debugfs_create_file("ipo_dst", 0400, fbd->dbg_fbd, fbd, &fbnic_dbg_ipo_dst_fops); + debugfs_create_file("fw_log", 0400, fbd->dbg_fbd, fbd, + &fbnic_dbg_fw_log_fops); } void fbnic_dbg_fbd_exit(struct fbnic_dev *fbd) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c b/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c index 4c4938eedd7b..b62b1d5b1453 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c @@ -8,6 +8,7 @@ #include <net/devlink.h> #include "fbnic.h" +#include "fbnic_fw.h" #include "fbnic_tlv.h" #define FBNIC_SN_STR_LEN 24 @@ -182,7 +183,7 @@ fbnic_flash_start(struct fbnic_dev *fbd, struct pldmfw_component *component) else err = -ETIMEDOUT; - fbnic_fw_clear_cmpl(fbd, cmpl); + fbnic_mbx_clear_cmpl(fbd, cmpl); cmpl_free: fbnic_fw_put_cmpl(cmpl); @@ -300,7 +301,7 @@ err_no_msg: component_name, 0, 0); } - fbnic_fw_clear_cmpl(fbd, cmpl); + fbnic_mbx_clear_cmpl(fbd, cmpl); cmpl_free: fbnic_fw_put_cmpl(cmpl); @@ -369,6 +370,254 @@ static const struct devlink_ops fbnic_devlink_ops = { .flash_update = fbnic_devlink_flash_update, }; +static int fbnic_fw_reporter_dump(struct devlink_health_reporter *reporter, + struct devlink_fmsg *fmsg, void *priv_ctx, + struct netlink_ext_ack *extack) +{ + struct fbnic_dev *fbd = devlink_health_reporter_priv(reporter); + u32 offset, index, index_count, length, size; + struct fbnic_fw_completion *fw_cmpl; + u8 *dump_data, **data; + int err; + + fw_cmpl = fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_COREDUMP_GET_INFO_RESP); + if (!fw_cmpl) + return -ENOMEM; + + err = fbnic_fw_xmit_coredump_info_msg(fbd, fw_cmpl, true); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to transmit core dump info msg"); + goto cmpl_free; + } + if (!wait_for_completion_timeout(&fw_cmpl->done, 2 * HZ)) { + NL_SET_ERR_MSG_MOD(extack, + "Timed out waiting on core dump info"); + err = -ETIMEDOUT; + goto cmpl_cleanup; + } + + size = fw_cmpl->u.coredump_info.size; + err = fw_cmpl->result; + + fbnic_mbx_clear_cmpl(fbd, fw_cmpl); + fbnic_fw_put_cmpl(fw_cmpl); + + /* Handle error returned by firmware */ + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware core dump returned error"); + return err; + } + if (!size) { + NL_SET_ERR_MSG_MOD(extack, + "Firmware core dump returned size 0"); + return -EIO; + } + + /* Read the dump, we can only transfer TLV_MAX_DATA at a time */ + index_count = DIV_ROUND_UP(size, TLV_MAX_DATA); + + fw_cmpl = __fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_COREDUMP_READ_RESP, + sizeof(void *) * index_count + size); + if (!fw_cmpl) + return -ENOMEM; + + /* Populate pointer table w/ pointer offsets */ + dump_data = (void *)&fw_cmpl->u.coredump.data[index_count]; + data = fw_cmpl->u.coredump.data; + fw_cmpl->u.coredump.size = size; + fw_cmpl->u.coredump.stride = TLV_MAX_DATA; + + for (index = 0; index < index_count; index++) { + /* First iteration installs completion */ + struct fbnic_fw_completion *cmpl_arg = index ? NULL : fw_cmpl; + + offset = index * TLV_MAX_DATA; + length = min(size - offset, TLV_MAX_DATA); + + data[index] = dump_data + offset; + err = fbnic_fw_xmit_coredump_read_msg(fbd, cmpl_arg, + offset, length); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to transmit core dump msg"); + if (cmpl_arg) + goto cmpl_free; + else + goto cmpl_cleanup; + } + + if (wait_for_completion_timeout(&fw_cmpl->done, 2 * HZ)) { + reinit_completion(&fw_cmpl->done); + } else { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Timed out waiting on core dump (%d/%d)", + index + 1, index_count); + err = -ETIMEDOUT; + goto cmpl_cleanup; + } + + /* If we didn't see the reply record as incomplete */ + if (fw_cmpl->u.coredump.data[index]) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "No data for core dump chunk (%d/%d)", + index + 1, index_count); + err = -EIO; + goto cmpl_cleanup; + } + } + + devlink_fmsg_binary_pair_nest_start(fmsg, "FW coredump"); + + for (offset = 0; offset < size; offset += length) { + length = min_t(u32, size - offset, TLV_MAX_DATA); + + devlink_fmsg_binary_put(fmsg, dump_data + offset, length); + } + + devlink_fmsg_binary_pair_nest_end(fmsg); + +cmpl_cleanup: + fbnic_mbx_clear_cmpl(fbd, fw_cmpl); +cmpl_free: + fbnic_fw_put_cmpl(fw_cmpl); + + return err; +} + +static int +fbnic_fw_reporter_diagnose(struct devlink_health_reporter *reporter, + struct devlink_fmsg *fmsg, + struct netlink_ext_ack *extack) +{ + struct fbnic_dev *fbd = devlink_health_reporter_priv(reporter); + u32 sec, msec; + + /* Device is most likely down, we're not exchanging heartbeats */ + if (!fbd->prev_firmware_time) + return 0; + + sec = div_u64_rem(fbd->firmware_time, MSEC_PER_SEC, &msec); + + devlink_fmsg_pair_nest_start(fmsg, "last_heartbeat"); + devlink_fmsg_obj_nest_start(fmsg); + devlink_fmsg_pair_nest_start(fmsg, "fw_uptime"); + devlink_fmsg_obj_nest_start(fmsg); + devlink_fmsg_u32_pair_put(fmsg, "sec", sec); + devlink_fmsg_u32_pair_put(fmsg, "msec", msec); + devlink_fmsg_obj_nest_end(fmsg); + devlink_fmsg_pair_nest_end(fmsg); + devlink_fmsg_obj_nest_end(fmsg); + devlink_fmsg_pair_nest_end(fmsg); + + return 0; +} + +void __printf(2, 3) +fbnic_devlink_fw_report(struct fbnic_dev *fbd, const char *format, ...) +{ + char msg[FBNIC_FW_LOG_MAX_SIZE]; + va_list args; + + va_start(args, format); + vsnprintf(msg, FBNIC_FW_LOG_MAX_SIZE, format, args); + va_end(args); + + devlink_health_report(fbd->fw_reporter, msg, fbd); + if (fbnic_fw_log_ready(fbd)) + fbnic_fw_log_write(fbd, 0, fbd->firmware_time, msg); +} + +static const struct devlink_health_reporter_ops fbnic_fw_ops = { + .name = "fw", + .dump = fbnic_fw_reporter_dump, + .diagnose = fbnic_fw_reporter_diagnose, +}; + +static u32 fbnic_read_otp_status(struct fbnic_dev *fbd) +{ + return fbnic_fw_rd32(fbd, FBNIC_NS_OTP_STATUS); +} + +static int +fbnic_otp_reporter_dump(struct devlink_health_reporter *reporter, + struct devlink_fmsg *fmsg, void *priv_ctx, + struct netlink_ext_ack *extack) +{ + struct fbnic_dev *fbd = devlink_health_reporter_priv(reporter); + u32 otp_status, otp_write_status, m; + + otp_status = fbnic_read_otp_status(fbd); + otp_write_status = fbnic_fw_rd32(fbd, FBNIC_NS_OTP_WRITE_STATUS); + + /* Dump OTP status */ + devlink_fmsg_pair_nest_start(fmsg, "OTP"); + devlink_fmsg_obj_nest_start(fmsg); + + devlink_fmsg_u32_pair_put(fmsg, "Status", otp_status); + + /* Extract OTP Write Data status */ + m = FBNIC_NS_OTP_WRITE_DATA_STATUS_MASK; + devlink_fmsg_u32_pair_put(fmsg, "Data", + FIELD_GET(m, otp_write_status)); + + /* Extract OTP Write ECC status */ + m = FBNIC_NS_OTP_WRITE_ECC_STATUS_MASK; + devlink_fmsg_u32_pair_put(fmsg, "ECC", + FIELD_GET(m, otp_write_status)); + + devlink_fmsg_obj_nest_end(fmsg); + devlink_fmsg_pair_nest_end(fmsg); + + return 0; +} + +void fbnic_devlink_otp_check(struct fbnic_dev *fbd, const char *msg) +{ + /* Check if there is anything to report */ + if (!fbnic_read_otp_status(fbd)) + return; + + devlink_health_report(fbd->otp_reporter, msg, fbd); + if (fbnic_fw_log_ready(fbd)) + fbnic_fw_log_write(fbd, 0, fbd->firmware_time, msg); +} + +static const struct devlink_health_reporter_ops fbnic_otp_ops = { + .name = "otp", + .dump = fbnic_otp_reporter_dump, +}; + +int fbnic_devlink_health_create(struct fbnic_dev *fbd) +{ + fbd->fw_reporter = devlink_health_reporter_create(priv_to_devlink(fbd), + &fbnic_fw_ops, fbd); + if (IS_ERR(fbd->fw_reporter)) { + dev_warn(fbd->dev, + "Failed to create FW fault reporter: %pe\n", + fbd->fw_reporter); + return PTR_ERR(fbd->fw_reporter); + } + + fbd->otp_reporter = devlink_health_reporter_create(priv_to_devlink(fbd), + &fbnic_otp_ops, fbd); + if (IS_ERR(fbd->otp_reporter)) { + devlink_health_reporter_destroy(fbd->fw_reporter); + dev_warn(fbd->dev, + "Failed to create OTP fault reporter: %pe\n", + fbd->otp_reporter); + return PTR_ERR(fbd->otp_reporter); + } + + return 0; +} + +void fbnic_devlink_health_destroy(struct fbnic_dev *fbd) +{ + devlink_health_reporter_destroy(fbd->otp_reporter); + devlink_health_reporter_destroy(fbd->fw_reporter); +} + void fbnic_devlink_free(struct fbnic_dev *fbd) { struct devlink *devlink = priv_to_devlink(fbd); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 5c7556c8c4c5..a1c2db69b198 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -2,6 +2,7 @@ /* Copyright (c) Meta Platforms, Inc. and affiliates. */ #include <linux/ethtool.h> +#include <linux/ethtool_netlink.h> #include <linux/netdevice.h> #include <linux/pci.h> #include <net/ipv6.h> @@ -111,6 +112,20 @@ static const struct fbnic_stat fbnic_gstrings_hw_q_stats[] = { FBNIC_HW_RXB_DEQUEUE_STATS_LEN * FBNIC_RXB_DEQUEUE_INDICES + \ FBNIC_HW_Q_STATS_LEN * FBNIC_MAX_QUEUES) +#define FBNIC_QUEUE_STAT(name, stat) \ + FBNIC_STAT_FIELDS(fbnic_ring, name, stat) + +static const struct fbnic_stat fbnic_gstrings_xdp_stats[] = { + FBNIC_QUEUE_STAT("xdp_tx_queue_%u_packets", stats.packets), + FBNIC_QUEUE_STAT("xdp_tx_queue_%u_bytes", stats.bytes), + FBNIC_QUEUE_STAT("xdp_tx_queue_%u_dropped", stats.dropped), +}; + +#define FBNIC_XDP_STATS_LEN ARRAY_SIZE(fbnic_gstrings_xdp_stats) + +#define FBNIC_STATS_LEN \ + (FBNIC_HW_STATS_LEN + FBNIC_XDP_STATS_LEN * FBNIC_MAX_XDPQS) + static void fbnic_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) { @@ -160,6 +175,7 @@ static void fbnic_clone_swap_cfg(struct fbnic_net *orig, swap(clone->num_rx_queues, orig->num_rx_queues); swap(clone->num_tx_queues, orig->num_tx_queues); swap(clone->num_napi, orig->num_napi); + swap(clone->hds_thresh, orig->hds_thresh); } static void fbnic_aggregate_vector_counters(struct fbnic_net *fbn, @@ -277,15 +293,21 @@ fbnic_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, ring->rx_mini_pending = fbn->hpq_size; ring->rx_jumbo_pending = fbn->ppq_size; ring->tx_pending = fbn->txq_size; + + kernel_ring->tcp_data_split = ETHTOOL_TCP_DATA_SPLIT_ENABLED; + kernel_ring->hds_thresh_max = FBNIC_HDS_THRESH_MAX; + kernel_ring->hds_thresh = fbn->hds_thresh; } static void fbnic_set_rings(struct fbnic_net *fbn, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring) { fbn->rcq_size = ring->rx_pending; fbn->hpq_size = ring->rx_mini_pending; fbn->ppq_size = ring->rx_jumbo_pending; fbn->txq_size = ring->tx_pending; + fbn->hds_thresh = kernel_ring->hds_thresh; } static int @@ -316,8 +338,24 @@ fbnic_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, return -EINVAL; } + if (kernel_ring->tcp_data_split == ETHTOOL_TCP_DATA_SPLIT_DISABLED) { + NL_SET_ERR_MSG_MOD(extack, "Cannot disable TCP data split"); + return -EINVAL; + } + + /* If an XDP program is attached, we should check for potential frame + * splitting. If the new HDS threshold can cause splitting, we should + * only allow if the attached XDP program can handle frags. + */ + if (fbnic_check_split_frames(fbn->xdp_prog, netdev->mtu, + kernel_ring->hds_thresh)) { + NL_SET_ERR_MSG_MOD(extack, + "Use higher HDS threshold or multi-buf capable program"); + return -EINVAL; + } + if (!netif_running(netdev)) { - fbnic_set_rings(fbn, ring); + fbnic_set_rings(fbn, ring, kernel_ring); return 0; } @@ -325,7 +363,7 @@ fbnic_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, if (!clone) return -ENOMEM; - fbnic_set_rings(clone, ring); + fbnic_set_rings(clone, ring, kernel_ring); err = fbnic_alloc_napi_vectors(clone); if (err) @@ -398,6 +436,16 @@ static void fbnic_get_rxb_dequeue_strings(u8 **data, unsigned int idx) ethtool_sprintf(data, stat->string, idx); } +static void fbnic_get_xdp_queue_strings(u8 **data, unsigned int idx) +{ + const struct fbnic_stat *stat; + int i; + + stat = fbnic_gstrings_xdp_stats; + for (i = 0; i < FBNIC_XDP_STATS_LEN; i++, stat++) + ethtool_sprintf(data, stat->string, idx); +} + static void fbnic_get_strings(struct net_device *dev, u32 sset, u8 *data) { const struct fbnic_stat *stat; @@ -423,6 +471,9 @@ static void fbnic_get_strings(struct net_device *dev, u32 sset, u8 *data) for (i = 0; i < FBNIC_HW_Q_STATS_LEN; i++, stat++) ethtool_sprintf(&data, stat->string, idx); } + + for (i = 0; i < FBNIC_MAX_XDPQS; i++) + fbnic_get_xdp_queue_strings(&data, i); break; } } @@ -440,6 +491,24 @@ static void fbnic_report_hw_stats(const struct fbnic_stat *stat, } } +static void fbnic_get_xdp_queue_stats(struct fbnic_ring *ring, u64 **data) +{ + const struct fbnic_stat *stat; + int i; + + if (!ring) { + *data += FBNIC_XDP_STATS_LEN; + return; + } + + stat = fbnic_gstrings_xdp_stats; + for (i = 0; i < FBNIC_XDP_STATS_LEN; i++, stat++, (*data)++) { + u8 *p = (u8 *)ring + stat->offset; + + **data = *(u64 *)p; + } +} + static void fbnic_get_ethtool_stats(struct net_device *dev, struct ethtool_stats *stats, u64 *data) { @@ -449,7 +518,7 @@ static void fbnic_get_ethtool_stats(struct net_device *dev, fbnic_get_hw_stats(fbn->fbd); - spin_lock(&fbd->hw_stats_lock); + spin_lock(&fbd->hw_stats.lock); fbnic_report_hw_stats(fbnic_gstrings_hw_stats, &fbd->hw_stats, FBNIC_HW_FIXED_STATS_LEN, &data); @@ -486,14 +555,17 @@ static void fbnic_get_ethtool_stats(struct net_device *dev, fbnic_report_hw_stats(fbnic_gstrings_hw_q_stats, hw_q, FBNIC_HW_Q_STATS_LEN, &data); } - spin_unlock(&fbd->hw_stats_lock); + spin_unlock(&fbd->hw_stats.lock); + + for (i = 0; i < FBNIC_MAX_XDPQS; i++) + fbnic_get_xdp_queue_stats(fbn->tx[i + FBNIC_MAX_TXQS], &data); } static int fbnic_get_sset_count(struct net_device *dev, int sset) { switch (sset) { case ETH_SS_STATS: - return FBNIC_HW_STATS_LEN; + return FBNIC_STATS_LEN; default: return -EOPNOTSUPP; } @@ -531,20 +603,6 @@ static int fbnic_get_rss_hash_idx(u32 flow_type) return -1; } -static int -fbnic_get_rss_hash_opts(struct fbnic_net *fbn, struct ethtool_rxnfc *cmd) -{ - int hash_opt_idx = fbnic_get_rss_hash_idx(cmd->flow_type); - - if (hash_opt_idx < 0) - return -EINVAL; - - /* Report options from rss_en table in fbn */ - cmd->data = fbn->rss_flow_hash[hash_opt_idx]; - - return 0; -} - static int fbnic_get_cls_rule_all(struct fbnic_net *fbn, struct ethtool_rxnfc *cmd, u32 *rule_locs) @@ -779,9 +837,6 @@ static int fbnic_get_rxnfc(struct net_device *netdev, cmd->data = fbn->num_rx_queues; ret = 0; break; - case ETHTOOL_GRXFH: - ret = fbnic_get_rss_hash_opts(fbn, cmd); - break; case ETHTOOL_GRXCLSRULE: ret = fbnic_get_cls_rule(fbn, cmd); break; @@ -803,41 +858,6 @@ static int fbnic_get_rxnfc(struct net_device *netdev, return ret; } -#define FBNIC_L2_HASH_OPTIONS \ - (RXH_L2DA | RXH_DISCARD) -#define FBNIC_L3_HASH_OPTIONS \ - (FBNIC_L2_HASH_OPTIONS | RXH_IP_SRC | RXH_IP_DST) -#define FBNIC_L4_HASH_OPTIONS \ - (FBNIC_L3_HASH_OPTIONS | RXH_L4_B_0_1 | RXH_L4_B_2_3) - -static int -fbnic_set_rss_hash_opts(struct fbnic_net *fbn, const struct ethtool_rxnfc *cmd) -{ - int hash_opt_idx; - - /* Verify the type requested is correct */ - hash_opt_idx = fbnic_get_rss_hash_idx(cmd->flow_type); - if (hash_opt_idx < 0) - return -EINVAL; - - /* Verify the fields asked for can actually be assigned based on type */ - if (cmd->data & ~FBNIC_L4_HASH_OPTIONS || - (hash_opt_idx > FBNIC_L4_HASH_OPT && - cmd->data & ~FBNIC_L3_HASH_OPTIONS) || - (hash_opt_idx > FBNIC_IP_HASH_OPT && - cmd->data & ~FBNIC_L2_HASH_OPTIONS)) - return -EINVAL; - - fbn->rss_flow_hash[hash_opt_idx] = cmd->data; - - if (netif_running(fbn->netdev)) { - fbnic_rss_reinit(fbn->fbd, fbn); - fbnic_write_rules(fbn->fbd); - } - - return 0; -} - static int fbnic_cls_rule_any_loc(struct fbnic_dev *fbd) { int i; @@ -1244,9 +1264,6 @@ static int fbnic_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) int ret = -EOPNOTSUPP; switch (cmd->cmd) { - case ETHTOOL_SRXFH: - ret = fbnic_set_rss_hash_opts(fbn, cmd); - break; case ETHTOOL_SRXCLSRLINS: ret = fbnic_set_cls_rule_ins(fbn, cmd); break; @@ -1347,6 +1364,60 @@ fbnic_set_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh, } static int +fbnic_get_rss_hash_opts(struct net_device *netdev, + struct ethtool_rxfh_fields *cmd) +{ + int hash_opt_idx = fbnic_get_rss_hash_idx(cmd->flow_type); + struct fbnic_net *fbn = netdev_priv(netdev); + + if (hash_opt_idx < 0) + return -EINVAL; + + /* Report options from rss_en table in fbn */ + cmd->data = fbn->rss_flow_hash[hash_opt_idx]; + + return 0; +} + +#define FBNIC_L2_HASH_OPTIONS \ + (RXH_L2DA | RXH_DISCARD) +#define FBNIC_L3_HASH_OPTIONS \ + (FBNIC_L2_HASH_OPTIONS | RXH_IP_SRC | RXH_IP_DST | RXH_IP6_FL) +#define FBNIC_L4_HASH_OPTIONS \ + (FBNIC_L3_HASH_OPTIONS | RXH_L4_B_0_1 | RXH_L4_B_2_3) + +static int +fbnic_set_rss_hash_opts(struct net_device *netdev, + const struct ethtool_rxfh_fields *cmd, + struct netlink_ext_ack *extack) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int hash_opt_idx; + + /* Verify the type requested is correct */ + hash_opt_idx = fbnic_get_rss_hash_idx(cmd->flow_type); + if (hash_opt_idx < 0) + return -EINVAL; + + /* Verify the fields asked for can actually be assigned based on type */ + if (cmd->data & ~FBNIC_L4_HASH_OPTIONS || + (hash_opt_idx > FBNIC_L4_HASH_OPT && + cmd->data & ~FBNIC_L3_HASH_OPTIONS) || + (hash_opt_idx > FBNIC_IP_HASH_OPT && + cmd->data & ~FBNIC_L2_HASH_OPTIONS)) + return -EINVAL; + + fbn->rss_flow_hash[hash_opt_idx] = cmd->data; + + if (netif_running(fbn->netdev)) { + fbnic_rss_reinit(fbn->fbd, fbn); + fbnic_write_rules(fbn->fbd); + } + + return 0; +} + +static int fbnic_modify_rxfh_context(struct net_device *netdev, struct ethtool_rxfh_context *ctx, const struct ethtool_rxfh_param *rxfh, @@ -1446,7 +1517,7 @@ static int fbnic_set_channels(struct net_device *netdev, standalone = ch->rx_count + ch->tx_count; /* Limits for standalone queues: - * - each queue has it's own NAPI (num_napi >= rx + tx + combined) + * - each queue has its own NAPI (num_napi >= rx + tx + combined) * - combining queues (combined not 0, rx or tx must be 0) */ if ((ch->rx_count && ch->tx_count && ch->combined_count) || @@ -1564,6 +1635,65 @@ static void fbnic_get_ts_stats(struct net_device *netdev, } } +static int +fbnic_get_module_eeprom_by_page(struct net_device *netdev, + const struct ethtool_module_eeprom *page_data, + struct netlink_ext_ack *extack) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_fw_completion *fw_cmpl; + struct fbnic_dev *fbd = fbn->fbd; + int err; + + if (page_data->i2c_address != 0x50) { + NL_SET_ERR_MSG_MOD(extack, + "Invalid i2c address. Only 0x50 is supported"); + return -EINVAL; + } + + fw_cmpl = __fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_QSFP_READ_RESP, + page_data->length); + if (!fw_cmpl) + return -ENOMEM; + + /* Initialize completion and queue it for FW to process */ + fw_cmpl->u.qsfp.length = page_data->length; + fw_cmpl->u.qsfp.offset = page_data->offset; + fw_cmpl->u.qsfp.page = page_data->page; + fw_cmpl->u.qsfp.bank = page_data->bank; + + err = fbnic_fw_xmit_qsfp_read_msg(fbd, fw_cmpl, page_data->page, + page_data->bank, page_data->offset, + page_data->length); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to transmit EEPROM read request"); + goto exit_free; + } + + if (!wait_for_completion_timeout(&fw_cmpl->done, 2 * HZ)) { + err = -ETIMEDOUT; + NL_SET_ERR_MSG_MOD(extack, + "Timed out waiting for firmware response"); + goto exit_cleanup; + } + + if (fw_cmpl->result) { + err = fw_cmpl->result; + NL_SET_ERR_MSG_MOD(extack, "Failed to read EEPROM"); + goto exit_cleanup; + } + + memcpy(page_data->data, fw_cmpl->u.qsfp.data, page_data->length); + +exit_cleanup: + fbnic_mbx_clear_cmpl(fbd, fw_cmpl); +exit_free: + fbnic_fw_put_cmpl(fw_cmpl); + + return err ? : page_data->length; +} + static void fbnic_set_counter(u64 *stat, struct fbnic_stat_counter *counter) { if (counter->reported) @@ -1571,6 +1701,63 @@ static void fbnic_set_counter(u64 *stat, struct fbnic_stat_counter *counter) } static void +fbnic_get_pause_stats(struct net_device *netdev, + struct ethtool_pause_stats *pause_stats) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_mac_stats *mac_stats; + struct fbnic_dev *fbd = fbn->fbd; + + mac_stats = &fbd->hw_stats.mac; + + fbd->mac->get_pause_stats(fbd, false, &mac_stats->pause); + + pause_stats->tx_pause_frames = mac_stats->pause.tx_pause_frames.value; + pause_stats->rx_pause_frames = mac_stats->pause.rx_pause_frames.value; +} + +static void +fbnic_get_fec_stats(struct net_device *netdev, + struct ethtool_fec_stats *fec_stats, + struct ethtool_fec_hist *hist) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_phy_stats *phy_stats; + struct fbnic_dev *fbd = fbn->fbd; + + fbnic_get_hw_stats32(fbd); + phy_stats = &fbd->hw_stats.phy; + + spin_lock(&fbd->hw_stats.lock); + fec_stats->corrected_blocks.total = + phy_stats->fec.corrected_blocks.value; + fec_stats->uncorrectable_blocks.total = + phy_stats->fec.uncorrectable_blocks.value; + spin_unlock(&fbd->hw_stats.lock); +} + +static void +fbnic_get_eth_phy_stats(struct net_device *netdev, + struct ethtool_eth_phy_stats *eth_phy_stats) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_phy_stats *phy_stats; + struct fbnic_dev *fbd = fbn->fbd; + u64 total = 0; + int i; + + fbnic_get_hw_stats32(fbd); + phy_stats = &fbd->hw_stats.phy; + + spin_lock(&fbd->hw_stats.lock); + for (i = 0; i < FBNIC_PCS_MAX_LANES; i++) + total += phy_stats->pcs.SymbolErrorDuringCarrier.lanes[i].value; + + eth_phy_stats->SymbolErrorDuringCarrier = total; + spin_unlock(&fbd->hw_stats.lock); +} + +static void fbnic_get_eth_mac_stats(struct net_device *netdev, struct ethtool_eth_mac_stats *eth_mac_stats) { @@ -1612,35 +1799,114 @@ fbnic_get_eth_mac_stats(struct net_device *netdev, &mac_stats->eth_mac.FrameTooLongErrors); } +static void +fbnic_get_eth_ctrl_stats(struct net_device *netdev, + struct ethtool_eth_ctrl_stats *eth_ctrl_stats) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_mac_stats *mac_stats; + struct fbnic_dev *fbd = fbn->fbd; + + mac_stats = &fbd->hw_stats.mac; + + fbd->mac->get_eth_ctrl_stats(fbd, false, &mac_stats->eth_ctrl); + + eth_ctrl_stats->MACControlFramesReceived = + mac_stats->eth_ctrl.MACControlFramesReceived.value; + eth_ctrl_stats->MACControlFramesTransmitted = + mac_stats->eth_ctrl.MACControlFramesTransmitted.value; +} + +static const struct ethtool_rmon_hist_range fbnic_rmon_ranges[] = { + { 0, 64 }, + { 65, 127 }, + { 128, 255 }, + { 256, 511 }, + { 512, 1023 }, + { 1024, 1518 }, + { 1519, 2047 }, + { 2048, 4095 }, + { 4096, 8191 }, + { 8192, 9216 }, + { 9217, FBNIC_MAX_JUMBO_FRAME_SIZE }, + {} +}; + +static void +fbnic_get_rmon_stats(struct net_device *netdev, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_mac_stats *mac_stats; + struct fbnic_dev *fbd = fbn->fbd; + int i; + + mac_stats = &fbd->hw_stats.mac; + + fbd->mac->get_rmon_stats(fbd, false, &mac_stats->rmon); + + rmon_stats->undersize_pkts = + mac_stats->rmon.undersize_pkts.value; + rmon_stats->oversize_pkts = + mac_stats->rmon.oversize_pkts.value; + rmon_stats->fragments = + mac_stats->rmon.fragments.value; + rmon_stats->jabbers = + mac_stats->rmon.jabbers.value; + + for (i = 0; fbnic_rmon_ranges[i].high; i++) { + rmon_stats->hist[i] = mac_stats->rmon.hist[i].value; + rmon_stats->hist_tx[i] = mac_stats->rmon.hist_tx[i].value; + } + + *ranges = fbnic_rmon_ranges; +} + static const struct ethtool_ops fbnic_ethtool_ops = { - .supported_coalesce_params = - ETHTOOL_COALESCE_USECS | - ETHTOOL_COALESCE_RX_MAX_FRAMES, - .rxfh_max_num_contexts = FBNIC_RPC_RSS_TBL_COUNT, - .get_drvinfo = fbnic_get_drvinfo, - .get_regs_len = fbnic_get_regs_len, - .get_regs = fbnic_get_regs, - .get_coalesce = fbnic_get_coalesce, - .set_coalesce = fbnic_set_coalesce, - .get_ringparam = fbnic_get_ringparam, - .set_ringparam = fbnic_set_ringparam, - .get_strings = fbnic_get_strings, - .get_ethtool_stats = fbnic_get_ethtool_stats, - .get_sset_count = fbnic_get_sset_count, - .get_rxnfc = fbnic_get_rxnfc, - .set_rxnfc = fbnic_set_rxnfc, - .get_rxfh_key_size = fbnic_get_rxfh_key_size, - .get_rxfh_indir_size = fbnic_get_rxfh_indir_size, - .get_rxfh = fbnic_get_rxfh, - .set_rxfh = fbnic_set_rxfh, - .create_rxfh_context = fbnic_create_rxfh_context, - .modify_rxfh_context = fbnic_modify_rxfh_context, - .remove_rxfh_context = fbnic_remove_rxfh_context, - .get_channels = fbnic_get_channels, - .set_channels = fbnic_set_channels, - .get_ts_info = fbnic_get_ts_info, - .get_ts_stats = fbnic_get_ts_stats, - .get_eth_mac_stats = fbnic_get_eth_mac_stats, + .cap_link_lanes_supported = true, + .supported_coalesce_params = ETHTOOL_COALESCE_USECS | + ETHTOOL_COALESCE_RX_MAX_FRAMES, + .supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT | + ETHTOOL_RING_USE_HDS_THRS, + .rxfh_max_num_contexts = FBNIC_RPC_RSS_TBL_COUNT, + .get_drvinfo = fbnic_get_drvinfo, + .get_regs_len = fbnic_get_regs_len, + .get_regs = fbnic_get_regs, + .get_link = ethtool_op_get_link, + .get_coalesce = fbnic_get_coalesce, + .set_coalesce = fbnic_set_coalesce, + .get_ringparam = fbnic_get_ringparam, + .set_ringparam = fbnic_set_ringparam, + .get_pause_stats = fbnic_get_pause_stats, + .get_pauseparam = fbnic_phylink_get_pauseparam, + .set_pauseparam = fbnic_phylink_set_pauseparam, + .get_strings = fbnic_get_strings, + .get_ethtool_stats = fbnic_get_ethtool_stats, + .get_sset_count = fbnic_get_sset_count, + .get_rxnfc = fbnic_get_rxnfc, + .set_rxnfc = fbnic_set_rxnfc, + .get_rxfh_key_size = fbnic_get_rxfh_key_size, + .get_rxfh_indir_size = fbnic_get_rxfh_indir_size, + .get_rxfh = fbnic_get_rxfh, + .set_rxfh = fbnic_set_rxfh, + .get_rxfh_fields = fbnic_get_rss_hash_opts, + .set_rxfh_fields = fbnic_set_rss_hash_opts, + .create_rxfh_context = fbnic_create_rxfh_context, + .modify_rxfh_context = fbnic_modify_rxfh_context, + .remove_rxfh_context = fbnic_remove_rxfh_context, + .get_channels = fbnic_get_channels, + .set_channels = fbnic_set_channels, + .get_ts_info = fbnic_get_ts_info, + .get_ts_stats = fbnic_get_ts_stats, + .get_link_ksettings = fbnic_phylink_ethtool_ksettings_get, + .get_fec_stats = fbnic_get_fec_stats, + .get_fecparam = fbnic_phylink_get_fecparam, + .get_module_eeprom_by_page = fbnic_get_module_eeprom_by_page, + .get_eth_phy_stats = fbnic_get_eth_phy_stats, + .get_eth_mac_stats = fbnic_get_eth_mac_stats, + .get_eth_ctrl_stats = fbnic_get_eth_ctrl_stats, + .get_rmon_stats = fbnic_get_rmon_stats, }; void fbnic_set_ethtool_ops(struct net_device *dev) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c index 4521d0483d18..c87cb9ed09e7 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c @@ -95,6 +95,9 @@ void fbnic_mbx_init(struct fbnic_dev *fbd) /* Initialize lock to protect Tx ring */ spin_lock_init(&fbd->fw_tx_lock); + /* Reset FW Capabilities */ + memset(&fbd->fw_cap, 0, sizeof(fbd->fw_cap)); + /* Reinitialize mailbox memory */ for (i = 0; i < FBNIC_IPC_MBX_INDICES; i++) memset(&fbd->mbx[i], 0, sizeof(struct fbnic_fw_mbx)); @@ -335,6 +338,16 @@ unlock_mbx: return err; } +void fbnic_mbx_clear_cmpl(struct fbnic_dev *fbd, + struct fbnic_fw_completion *fw_cmpl) +{ + unsigned long flags; + + spin_lock_irqsave(&fbd->fw_tx_lock, flags); + fbnic_mbx_clear_cmpl_slot(fbd, fw_cmpl); + spin_unlock_irqrestore(&fbd->fw_tx_lock, flags); +} + static void fbnic_fw_release_cmpl_data(struct kref *kref) { struct fbnic_fw_completion *cmpl_data; @@ -373,11 +386,11 @@ fbnic_fw_get_cmpl_by_type(struct fbnic_dev *fbd, u32 msg_type) * * Return: * One the following values: - * -EOPNOTSUPP: Is not ASIC so mailbox is not supported - * -ENODEV: Device I/O error - * -ENOMEM: Failed to allocate message - * -EBUSY: No space in mailbox - * -ENOSPC: DMA mapping failed + * -EOPNOTSUPP: Is not ASIC so mailbox is not supported + * -ENODEV: Device I/O error + * -ENOMEM: Failed to allocate message + * -EBUSY: No space in mailbox + * -ENOSPC: DMA mapping failed * * This function sends a single TLV header indicating the host wants to take * some action. However there are no other side effects which means that any @@ -482,6 +495,11 @@ int fbnic_fw_xmit_ownership_msg(struct fbnic_dev *fbd, bool take_ownership) fbd->last_heartbeat_request = req_time; + /* Set prev_firmware_time to 0 to avoid triggering firmware crash + * detection until we receive the second uptime in a heartbeat resp. + */ + fbd->prev_firmware_time = 0; + /* Set heartbeat detection based on if we are taking ownership */ fbd->fw_heartbeat_enabled = take_ownership; @@ -560,16 +578,15 @@ static int fbnic_fw_parse_cap_resp(void *opaque, struct fbnic_tlv_msg **results) if (!fbd->fw_cap.running.mgmt.version) return -EINVAL; - if (fbd->fw_cap.running.mgmt.version < MIN_FW_VERSION_CODE) { + if (fbd->fw_cap.running.mgmt.version < MIN_FW_VER_CODE) { + char required_ver[FBNIC_FW_VER_MAX_SIZE]; char running_ver[FBNIC_FW_VER_MAX_SIZE]; fbnic_mk_fw_ver_str(fbd->fw_cap.running.mgmt.version, running_ver); - dev_err(fbd->dev, "Device firmware version(%s) is older than minimum required version(%02d.%02d.%02d)\n", - running_ver, - MIN_FW_MAJOR_VERSION, - MIN_FW_MINOR_VERSION, - MIN_FW_BUILD_VERSION); + fbnic_mk_fw_ver_str(MIN_FW_VER_CODE, required_ver); + dev_err(fbd->dev, "Device firmware version(%s) is older than minimum required version(%s)\n", + running_ver, required_ver); /* Disable TX mailbox to prevent card use until firmware is * updated. */ @@ -641,10 +658,14 @@ static int fbnic_fw_parse_cap_resp(void *opaque, struct fbnic_tlv_msg **results) fbd->fw_cap.anti_rollback_version = fta_get_uint(results, FBNIC_FW_CAP_RESP_ANTI_ROLLBACK_VERSION); + /* Always assume we need a BMC reinit */ + fbd->fw_cap.need_bmc_tcam_reinit = true; + return 0; } static const struct fbnic_tlv_index fbnic_ownership_resp_index[] = { + FBNIC_TLV_ATTR_U64(FBNIC_FW_OWNERSHIP_TIME), FBNIC_TLV_ATTR_LAST }; @@ -656,10 +677,14 @@ static int fbnic_fw_parse_ownership_resp(void *opaque, /* Count the ownership response as a heartbeat reply */ fbd->last_heartbeat_response = jiffies; + /* Capture firmware time for logging and firmware crash check */ + fbd->firmware_time = fta_get_uint(results, FBNIC_FW_OWNERSHIP_TIME); + return 0; } static const struct fbnic_tlv_index fbnic_heartbeat_resp_index[] = { + FBNIC_TLV_ATTR_U64(FBNIC_FW_HEARTBEAT_UPTIME), FBNIC_TLV_ATTR_LAST }; @@ -670,6 +695,9 @@ static int fbnic_fw_parse_heartbeat_resp(void *opaque, fbd->last_heartbeat_response = jiffies; + /* Capture firmware time for logging and firmware crash check */ + fbd->firmware_time = fta_get_uint(results, FBNIC_FW_HEARTBEAT_UPTIME); + return 0; } @@ -691,6 +719,7 @@ static int fbnic_fw_xmit_heartbeat_message(struct fbnic_dev *fbd) goto free_message; fbd->last_heartbeat_request = req_time; + fbd->prev_firmware_time = fbd->firmware_time; return err; @@ -751,7 +780,8 @@ void fbnic_fw_check_heartbeat(struct fbnic_dev *fbd) return; /* Was the last heartbeat response long time ago? */ - if (!fbnic_fw_heartbeat_current(fbd)) { + if (!fbnic_fw_heartbeat_current(fbd) || + fbd->firmware_time < fbd->prev_firmware_time) { dev_warn(fbd->dev, "Firmware did not respond to heartbeat message\n"); fbd->fw_heartbeat_enabled = false; @@ -763,6 +793,215 @@ void fbnic_fw_check_heartbeat(struct fbnic_dev *fbd) dev_warn(fbd->dev, "Failed to send heartbeat message\n"); } +/** + * fbnic_fw_xmit_coredump_info_msg - Create and transmit a coredump info message + * @fbd: FBNIC device structure + * @cmpl_data: Structure to store info in + * @force: Force coredump event if one hasn't already occurred + * + * Return: zero on success, negative errno on failure + * + * Asks the FW for info related to coredump. If a coredump doesn't exist it + * can optionally force one if force is true. + */ +int fbnic_fw_xmit_coredump_info_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + bool force) +{ + struct fbnic_tlv_msg *msg; + int err = 0; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_COREDUMP_GET_INFO_REQ); + if (!msg) + return -ENOMEM; + + if (force) { + err = fbnic_tlv_attr_put_flag(msg, FBNIC_FW_COREDUMP_REQ_INFO_CREATE); + if (err) + goto free_msg; + } + + err = fbnic_mbx_map_req_w_cmpl(fbd, msg, cmpl_data); + if (err) + goto free_msg; + + return 0; + +free_msg: + free_page((unsigned long)msg); + return err; +} + +static const struct fbnic_tlv_index fbnic_coredump_info_resp_index[] = { + FBNIC_TLV_ATTR_FLAG(FBNIC_FW_COREDUMP_INFO_AVAILABLE), + FBNIC_TLV_ATTR_U32(FBNIC_FW_COREDUMP_INFO_SIZE), + FBNIC_TLV_ATTR_S32(FBNIC_FW_COREDUMP_INFO_ERROR), + FBNIC_TLV_ATTR_LAST +}; + +static int +fbnic_fw_parse_coredump_info_resp(void *opaque, struct fbnic_tlv_msg **results) +{ + struct fbnic_fw_completion *cmpl_data; + struct fbnic_dev *fbd = opaque; + u32 msg_type; + s32 err; + + /* Verify we have a completion pointer to provide with data */ + msg_type = FBNIC_TLV_MSG_ID_COREDUMP_GET_INFO_RESP; + cmpl_data = fbnic_fw_get_cmpl_by_type(fbd, msg_type); + if (!cmpl_data) + return -ENOSPC; + + err = fta_get_sint(results, FBNIC_FW_COREDUMP_INFO_ERROR); + if (err) + goto msg_err; + + if (!results[FBNIC_FW_COREDUMP_INFO_AVAILABLE]) { + err = -ENOENT; + goto msg_err; + } + + cmpl_data->u.coredump_info.size = + fta_get_uint(results, FBNIC_FW_COREDUMP_INFO_SIZE); + +msg_err: + cmpl_data->result = err; + complete(&cmpl_data->done); + fbnic_fw_put_cmpl(cmpl_data); + + return err; +} + +/** + * fbnic_fw_xmit_coredump_read_msg - Create and transmit a coredump read request + * @fbd: FBNIC device structure + * @cmpl_data: Completion struct to store coredump + * @offset: Offset into coredump requested + * @length: Length of section of cordeump to fetch + * + * Return: zero on success, negative errno on failure + * + * Asks the firmware to provide a section of the cordeump back in a message. + * The response will have an offset and size matching the values provided. + */ +int fbnic_fw_xmit_coredump_read_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + u32 offset, u32 length) +{ + struct fbnic_tlv_msg *msg; + int err = 0; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_COREDUMP_READ_REQ); + if (!msg) + return -ENOMEM; + + if (offset) { + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_COREDUMP_READ_OFFSET, + offset); + if (err) + goto free_message; + } + + if (length) { + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_COREDUMP_READ_LENGTH, + length); + if (err) + goto free_message; + } + + err = fbnic_mbx_map_req_w_cmpl(fbd, msg, cmpl_data); + if (err) + goto free_message; + + return 0; + +free_message: + free_page((unsigned long)msg); + return err; +} + +static const struct fbnic_tlv_index fbnic_coredump_resp_index[] = { + FBNIC_TLV_ATTR_U32(FBNIC_FW_COREDUMP_READ_OFFSET), + FBNIC_TLV_ATTR_U32(FBNIC_FW_COREDUMP_READ_LENGTH), + FBNIC_TLV_ATTR_RAW_DATA(FBNIC_FW_COREDUMP_READ_DATA), + FBNIC_TLV_ATTR_S32(FBNIC_FW_COREDUMP_READ_ERROR), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_parse_coredump_resp(void *opaque, + struct fbnic_tlv_msg **results) +{ + struct fbnic_fw_completion *cmpl_data; + u32 index, last_offset, last_length; + struct fbnic_dev *fbd = opaque; + struct fbnic_tlv_msg *data_hdr; + u32 length, offset; + u32 msg_type; + s32 err; + + /* Verify we have a completion pointer to provide with data */ + msg_type = FBNIC_TLV_MSG_ID_COREDUMP_READ_RESP; + cmpl_data = fbnic_fw_get_cmpl_by_type(fbd, msg_type); + if (!cmpl_data) + return -ENOSPC; + + err = fta_get_sint(results, FBNIC_FW_COREDUMP_READ_ERROR); + if (err) + goto msg_err; + + data_hdr = results[FBNIC_FW_COREDUMP_READ_DATA]; + if (!data_hdr) { + err = -ENODATA; + goto msg_err; + } + + offset = fta_get_uint(results, FBNIC_FW_COREDUMP_READ_OFFSET); + length = fta_get_uint(results, FBNIC_FW_COREDUMP_READ_LENGTH); + + if (length > le16_to_cpu(data_hdr->hdr.len) - sizeof(u32)) { + dev_err(fbd->dev, "length greater than size of message\n"); + err = -EINVAL; + goto msg_err; + } + + /* Only the last offset can have a length != stride */ + last_length = + (cmpl_data->u.coredump.size % cmpl_data->u.coredump.stride) ? : + cmpl_data->u.coredump.stride; + last_offset = cmpl_data->u.coredump.size - last_length; + + /* Verify offset and length */ + if (offset % cmpl_data->u.coredump.stride || offset > last_offset) { + dev_err(fbd->dev, "offset %d out of range\n", offset); + err = -EINVAL; + } else if (length != ((offset == last_offset) ? + last_length : cmpl_data->u.coredump.stride)) { + dev_err(fbd->dev, "length %d out of range for offset %d\n", + length, offset); + err = -EINVAL; + } + if (err) + goto msg_err; + + /* If data pointer is NULL it is already filled, just skip the copy */ + index = offset / cmpl_data->u.coredump.stride; + if (!cmpl_data->u.coredump.data[index]) + goto msg_err; + + /* Copy data and mark index filled by setting pointer to NULL */ + memcpy(cmpl_data->u.coredump.data[index], + fbnic_tlv_attr_get_value_ptr(data_hdr), length); + cmpl_data->u.coredump.data[index] = NULL; + +msg_err: + cmpl_data->result = err; + complete(&cmpl_data->done); + fbnic_fw_put_cmpl(cmpl_data); + + return err; +} + int fbnic_fw_xmit_fw_start_upgrade(struct fbnic_dev *fbd, struct fbnic_fw_completion *cmpl_data, unsigned int id, unsigned int len) @@ -946,6 +1185,138 @@ static int fbnic_fw_parse_fw_finish_upgrade_req(void *opaque, } /** + * fbnic_fw_xmit_qsfp_read_msg - Transmit a QSFP read request + * @fbd: FBNIC device structure + * @cmpl_data: Structure to store EEPROM response in + * @page: Refers to page number on page enabled QSFP modules + * @bank: Refers to a collection of pages + * @offset: Offset into QSFP EEPROM requested + * @length: Length of section of QSFP EEPROM to fetch + * + * Return: zero on success, negative value on failure + * + * Asks the firmware to provide a section of the QSFP EEPROM back in a + * message. The response will have an offset and size matching the values + * provided. + */ +int fbnic_fw_xmit_qsfp_read_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + u32 page, u32 bank, u32 offset, u32 length) +{ + struct fbnic_tlv_msg *msg; + int err = 0; + + if (!length || length > TLV_MAX_DATA) + return -EINVAL; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_QSFP_READ_REQ); + if (!msg) + return -ENOMEM; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_QSFP_BANK, bank); + if (err) + goto free_message; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_QSFP_PAGE, page); + if (err) + goto free_message; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_QSFP_OFFSET, offset); + if (err) + goto free_message; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_QSFP_LENGTH, length); + if (err) + goto free_message; + + err = fbnic_mbx_map_req_w_cmpl(fbd, msg, cmpl_data); + if (err) + goto free_message; + + return 0; + +free_message: + free_page((unsigned long)msg); + return err; +} + +static const struct fbnic_tlv_index fbnic_qsfp_read_resp_index[] = { + FBNIC_TLV_ATTR_U32(FBNIC_FW_QSFP_BANK), + FBNIC_TLV_ATTR_U32(FBNIC_FW_QSFP_PAGE), + FBNIC_TLV_ATTR_U32(FBNIC_FW_QSFP_OFFSET), + FBNIC_TLV_ATTR_U32(FBNIC_FW_QSFP_LENGTH), + FBNIC_TLV_ATTR_RAW_DATA(FBNIC_FW_QSFP_DATA), + FBNIC_TLV_ATTR_S32(FBNIC_FW_QSFP_ERROR), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_parse_qsfp_read_resp(void *opaque, + struct fbnic_tlv_msg **results) +{ + struct fbnic_fw_completion *cmpl_data; + struct fbnic_dev *fbd = opaque; + struct fbnic_tlv_msg *data_hdr; + u32 length, offset, page, bank; + u8 *data; + s32 err; + + /* Verify we have a completion pointer to provide with data */ + cmpl_data = fbnic_fw_get_cmpl_by_type(fbd, + FBNIC_TLV_MSG_ID_QSFP_READ_RESP); + if (!cmpl_data) + return -ENOSPC; + + bank = fta_get_uint(results, FBNIC_FW_QSFP_BANK); + if (bank != cmpl_data->u.qsfp.bank) { + dev_warn(fbd->dev, "bank not equal to bank requested: %d vs %d\n", + bank, cmpl_data->u.qsfp.bank); + err = -EINVAL; + goto msg_err; + } + + page = fta_get_uint(results, FBNIC_FW_QSFP_PAGE); + if (page != cmpl_data->u.qsfp.page) { + dev_warn(fbd->dev, "page not equal to page requested: %d vs %d\n", + page, cmpl_data->u.qsfp.page); + err = -EINVAL; + goto msg_err; + } + + offset = fta_get_uint(results, FBNIC_FW_QSFP_OFFSET); + length = fta_get_uint(results, FBNIC_FW_QSFP_LENGTH); + + if (length != cmpl_data->u.qsfp.length || + offset != cmpl_data->u.qsfp.offset) { + dev_warn(fbd->dev, + "offset/length not equal to size requested: %d/%d vs %d/%d\n", + offset, length, + cmpl_data->u.qsfp.offset, cmpl_data->u.qsfp.length); + err = -EINVAL; + goto msg_err; + } + + err = fta_get_sint(results, FBNIC_FW_QSFP_ERROR); + if (err) + goto msg_err; + + data_hdr = results[FBNIC_FW_QSFP_DATA]; + if (!data_hdr) { + err = -ENODATA; + goto msg_err; + } + + /* Copy data */ + data = fbnic_tlv_attr_get_value_ptr(data_hdr); + memcpy(cmpl_data->u.qsfp.data, data, length); +msg_err: + cmpl_data->result = err; + complete(&cmpl_data->done); + fbnic_fw_put_cmpl(cmpl_data); + + return err; +} + +/** * fbnic_fw_xmit_tsene_read_msg - Create and transmit a sensor read request * @fbd: FBNIC device structure * @cmpl_data: Completion data structure to store sensor response @@ -1022,6 +1393,169 @@ msg_err: return err; } +static const struct fbnic_tlv_index fbnic_fw_log_req_index[] = { + FBNIC_TLV_ATTR_U32(FBNIC_FW_LOG_MSEC), + FBNIC_TLV_ATTR_U64(FBNIC_FW_LOG_INDEX), + FBNIC_TLV_ATTR_STRING(FBNIC_FW_LOG_MSG, FBNIC_FW_LOG_MAX_SIZE), + FBNIC_TLV_ATTR_U32(FBNIC_FW_LOG_LENGTH), + FBNIC_TLV_ATTR_ARRAY(FBNIC_FW_LOG_MSEC_ARRAY), + FBNIC_TLV_ATTR_ARRAY(FBNIC_FW_LOG_INDEX_ARRAY), + FBNIC_TLV_ATTR_ARRAY(FBNIC_FW_LOG_MSG_ARRAY), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_process_log_array(struct fbnic_tlv_msg **results, + u16 length, u16 arr_type_idx, + u16 attr_type_idx, + struct fbnic_tlv_msg **tlv_array_out) +{ + struct fbnic_tlv_msg *attr; + int attr_len; + int err; + + if (!results[attr_type_idx]) + return -EINVAL; + + tlv_array_out[0] = results[attr_type_idx]; + + if (!length) + return 0; + + if (!results[arr_type_idx]) + return -EINVAL; + + attr = results[arr_type_idx]; + attr_len = le16_to_cpu(attr->hdr.len) / sizeof(u32) - 1; + err = fbnic_tlv_attr_parse_array(&attr[1], attr_len, &tlv_array_out[1], + fbnic_fw_log_req_index, + attr_type_idx, + length); + if (err) + return err; + + return 0; +} + +static int fbnic_fw_parse_logs(struct fbnic_dev *fbd, + struct fbnic_tlv_msg **msec_tlv, + struct fbnic_tlv_msg **index_tlv, + struct fbnic_tlv_msg **log_tlv, + int count) +{ + int i; + + for (i = 0; i < count; i++) { + char log[FBNIC_FW_LOG_MAX_SIZE]; + ssize_t len; + u64 index; + u32 msec; + int err; + + if (!msec_tlv[i] || !index_tlv[i] || !log_tlv[i]) { + dev_warn(fbd->dev, "Received log message with missing attributes!\n"); + return -EINVAL; + } + + index = fbnic_tlv_attr_get_signed(index_tlv[i], 0); + msec = fbnic_tlv_attr_get_signed(msec_tlv[i], 0); + len = fbnic_tlv_attr_get_string(log_tlv[i], log, + FBNIC_FW_LOG_MAX_SIZE); + if (len < 0) + return len; + + err = fbnic_fw_log_write(fbd, index, msec, log); + if (err) + return err; + } + + return 0; +} + +static int fbnic_fw_parse_log_req(void *opaque, + struct fbnic_tlv_msg **results) +{ + struct fbnic_tlv_msg *index_tlv[FBNIC_FW_MAX_LOG_HISTORY]; + struct fbnic_tlv_msg *msec_tlv[FBNIC_FW_MAX_LOG_HISTORY]; + struct fbnic_tlv_msg *log_tlv[FBNIC_FW_MAX_LOG_HISTORY]; + struct fbnic_dev *fbd = opaque; + u16 length; + int err; + + length = fta_get_uint(results, FBNIC_FW_LOG_LENGTH); + if (length >= FBNIC_FW_MAX_LOG_HISTORY) + return -E2BIG; + + err = fbnic_fw_process_log_array(results, length, + FBNIC_FW_LOG_MSEC_ARRAY, + FBNIC_FW_LOG_MSEC, msec_tlv); + if (err) + return err; + + err = fbnic_fw_process_log_array(results, length, + FBNIC_FW_LOG_INDEX_ARRAY, + FBNIC_FW_LOG_INDEX, index_tlv); + if (err) + return err; + + err = fbnic_fw_process_log_array(results, length, + FBNIC_FW_LOG_MSG_ARRAY, + FBNIC_FW_LOG_MSG, log_tlv); + if (err) + return err; + + err = fbnic_fw_parse_logs(fbd, msec_tlv, index_tlv, log_tlv, + length + 1); + if (err) + return err; + + return 0; +} + +int fbnic_fw_xmit_send_logs(struct fbnic_dev *fbd, bool enable, + bool send_log_history) +{ + struct fbnic_tlv_msg *msg; + int err; + + if (fbd->fw_cap.running.mgmt.version < MIN_FW_VER_CODE_LOG) { + dev_warn(fbd->dev, "Firmware version is too old to support firmware logs!\n"); + return -EOPNOTSUPP; + } + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_LOG_SEND_LOGS_REQ); + if (!msg) + return -ENOMEM; + + if (enable) { + err = fbnic_tlv_attr_put_flag(msg, FBNIC_SEND_LOGS); + if (err) + goto free_message; + + /* Report request for version 1 of logs */ + err = fbnic_tlv_attr_put_int(msg, FBNIC_SEND_LOGS_VERSION, + FBNIC_FW_LOG_VERSION); + if (err) + goto free_message; + + if (send_log_history) { + err = fbnic_tlv_attr_put_flag(msg, + FBNIC_SEND_LOGS_HISTORY); + if (err) + goto free_message; + } + } + + err = fbnic_mbx_map_tlv_msg(fbd, msg); + if (err) + goto free_message; + + return 0; + +free_message: + free_page((unsigned long)msg); + return err; +} + static const struct fbnic_tlv_parser fbnic_fw_tlv_parser[] = { FBNIC_TLV_PARSER(FW_CAP_RESP, fbnic_fw_cap_resp_index, fbnic_fw_parse_cap_resp), @@ -1029,6 +1563,11 @@ static const struct fbnic_tlv_parser fbnic_fw_tlv_parser[] = { fbnic_fw_parse_ownership_resp), FBNIC_TLV_PARSER(HEARTBEAT_RESP, fbnic_heartbeat_resp_index, fbnic_fw_parse_heartbeat_resp), + FBNIC_TLV_PARSER(COREDUMP_GET_INFO_RESP, + fbnic_coredump_info_resp_index, + fbnic_fw_parse_coredump_info_resp), + FBNIC_TLV_PARSER(COREDUMP_READ_RESP, fbnic_coredump_resp_index, + fbnic_fw_parse_coredump_resp), FBNIC_TLV_PARSER(FW_START_UPGRADE_RESP, fbnic_fw_start_upgrade_resp_index, fbnic_fw_parse_fw_start_upgrade_resp), @@ -1038,9 +1577,15 @@ static const struct fbnic_tlv_parser fbnic_fw_tlv_parser[] = { FBNIC_TLV_PARSER(FW_FINISH_UPGRADE_REQ, fbnic_fw_finish_upgrade_req_index, fbnic_fw_parse_fw_finish_upgrade_req), + FBNIC_TLV_PARSER(QSFP_READ_RESP, + fbnic_qsfp_read_resp_index, + fbnic_fw_parse_qsfp_read_resp), FBNIC_TLV_PARSER(TSENE_READ_RESP, fbnic_tsene_read_resp_index, fbnic_fw_parse_tsene_read_resp), + FBNIC_TLV_PARSER(LOG_MSG_REQ, + fbnic_fw_log_req_index, + fbnic_fw_parse_log_req), FBNIC_TLV_MSG_ERROR }; @@ -1117,6 +1662,7 @@ void fbnic_mbx_poll(struct fbnic_dev *fbd) int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd) { + struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; unsigned long timeout = jiffies + 10 * HZ + 1; int err, i; @@ -1149,8 +1695,23 @@ int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd) if (err) goto clean_mbx; - /* Use "1" to indicate we entered the state waiting for a response */ - fbd->fw_cap.running.mgmt.version = 1; + /* Poll until we get a current management firmware version, use "1" + * to indicate we entered the polling state waiting for a response + */ + for (fbd->fw_cap.running.mgmt.version = 1; + fbd->fw_cap.running.mgmt.version < MIN_FW_VER_CODE;) { + if (!tx_mbx->ready) + err = -ENODEV; + if (err) + goto clean_mbx; + + msleep(20); + fbnic_mbx_poll(fbd); + + /* set err, but wait till mgmt.version check to report it */ + if (!time_is_after_jiffies(timeout)) + err = -ETIMEDOUT; + } return 0; clean_mbx: @@ -1216,6 +1777,109 @@ void fbnic_mbx_flush_tx(struct fbnic_dev *fbd) } while (time_is_after_jiffies(timeout)); } +int fbnic_fw_xmit_rpc_macda_sync(struct fbnic_dev *fbd) +{ + struct fbnic_tlv_msg *mac_array; + int i, addr_count = 0, err; + struct fbnic_tlv_msg *msg; + u32 rx_flags = 0; + + /* Nothing to do if there is no FW to sync with */ + if (!fbd->mbx[FBNIC_IPC_MBX_TX_IDX].ready) + return 0; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_RPC_MAC_SYNC_REQ); + if (!msg) + return -ENOMEM; + + mac_array = fbnic_tlv_attr_nest_start(msg, + FBNIC_FW_RPC_MAC_SYNC_UC_ARRAY); + if (!mac_array) + goto free_message_nospc; + + /* Populate the unicast MAC addrs and capture PROMISC/ALLMULTI flags */ + for (addr_count = 0, i = FBNIC_RPC_TCAM_MACDA_PROMISC_IDX; + i >= fbd->mac_addr_boundary; i--) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; + + if (mac_addr->state != FBNIC_TCAM_S_VALID) + continue; + if (test_bit(FBNIC_MAC_ADDR_T_ALLMULTI, mac_addr->act_tcam)) + rx_flags |= FW_RPC_MAC_SYNC_RX_FLAGS_ALLMULTI; + if (test_bit(FBNIC_MAC_ADDR_T_PROMISC, mac_addr->act_tcam)) + rx_flags |= FW_RPC_MAC_SYNC_RX_FLAGS_PROMISC; + if (!test_bit(FBNIC_MAC_ADDR_T_UNICAST, mac_addr->act_tcam)) + continue; + if (addr_count == FW_RPC_MAC_SYNC_UC_ARRAY_SIZE) { + rx_flags |= FW_RPC_MAC_SYNC_RX_FLAGS_PROMISC; + continue; + } + + err = fbnic_tlv_attr_put_value(mac_array, + FBNIC_FW_RPC_MAC_SYNC_MAC_ADDR, + mac_addr->value.addr8, + ETH_ALEN); + if (err) + goto free_message; + addr_count++; + } + + /* Close array */ + fbnic_tlv_attr_nest_stop(msg); + + mac_array = fbnic_tlv_attr_nest_start(msg, + FBNIC_FW_RPC_MAC_SYNC_MC_ARRAY); + if (!mac_array) + goto free_message_nospc; + + /* Repeat for multicast addrs, record BROADCAST/ALLMULTI flags */ + for (addr_count = 0, i = FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX; + i < fbd->mac_addr_boundary; i++) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; + + if (mac_addr->state != FBNIC_TCAM_S_VALID) + continue; + if (test_bit(FBNIC_MAC_ADDR_T_BROADCAST, mac_addr->act_tcam)) + rx_flags |= FW_RPC_MAC_SYNC_RX_FLAGS_BROADCAST; + if (test_bit(FBNIC_MAC_ADDR_T_ALLMULTI, mac_addr->act_tcam)) + rx_flags |= FW_RPC_MAC_SYNC_RX_FLAGS_ALLMULTI; + if (!test_bit(FBNIC_MAC_ADDR_T_MULTICAST, mac_addr->act_tcam)) + continue; + if (addr_count == FW_RPC_MAC_SYNC_MC_ARRAY_SIZE) { + rx_flags |= FW_RPC_MAC_SYNC_RX_FLAGS_ALLMULTI; + continue; + } + + err = fbnic_tlv_attr_put_value(mac_array, + FBNIC_FW_RPC_MAC_SYNC_MAC_ADDR, + mac_addr->value.addr8, + ETH_ALEN); + if (err) + goto free_message; + addr_count++; + } + + /* Close array */ + fbnic_tlv_attr_nest_stop(msg); + + /* Report flags at end of list */ + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_RPC_MAC_SYNC_RX_FLAGS, + rx_flags); + if (err) + goto free_message; + + /* Send message of to FW notifying it of current RPC config */ + err = fbnic_mbx_map_tlv_msg(fbd, msg); + if (err) + goto free_message; + return 0; +free_message_nospc: + err = -ENOSPC; +free_message: + free_page((unsigned long)msg); + return err; +} + void fbnic_get_fw_ver_commit_str(struct fbnic_dev *fbd, char *fw_version, const size_t str_sz) { @@ -1229,11 +1893,12 @@ void fbnic_get_fw_ver_commit_str(struct fbnic_dev *fbd, char *fw_version, fw_version, str_sz); } -struct fbnic_fw_completion *fbnic_fw_alloc_cmpl(u32 msg_type) +struct fbnic_fw_completion *__fbnic_fw_alloc_cmpl(u32 msg_type, + size_t priv_size) { struct fbnic_fw_completion *cmpl; - cmpl = kzalloc(sizeof(*cmpl), GFP_KERNEL); + cmpl = kzalloc(sizeof(*cmpl) + priv_size, GFP_KERNEL); if (!cmpl) return NULL; @@ -1244,14 +1909,9 @@ struct fbnic_fw_completion *fbnic_fw_alloc_cmpl(u32 msg_type) return cmpl; } -void fbnic_fw_clear_cmpl(struct fbnic_dev *fbd, - struct fbnic_fw_completion *fw_cmpl) +struct fbnic_fw_completion *fbnic_fw_alloc_cmpl(u32 msg_type) { - unsigned long flags; - - spin_lock_irqsave(&fbd->fw_tx_lock, flags); - fbnic_mbx_clear_cmpl_slot(fbd, fw_cmpl); - spin_unlock_irqrestore(&fbd->fw_tx_lock, flags); + return __fbnic_fw_alloc_cmpl(msg_type, 0); } void fbnic_fw_put_cmpl(struct fbnic_fw_completion *fw_cmpl) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h index 08bc4b918de7..1ecd777aaada 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h @@ -19,10 +19,23 @@ struct fbnic_fw_mbx { }; // FW_VER_MAX_SIZE must match ETHTOOL_FWVERS_LEN -#define FBNIC_FW_VER_MAX_SIZE 32 +#define FBNIC_FW_VER_MAX_SIZE 32 // Formatted version is in the format XX.YY.ZZ_RRR_COMMIT #define FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE (FBNIC_FW_VER_MAX_SIZE - 13) -#define FBNIC_FW_LOG_MAX_SIZE 256 +#define FBNIC_FW_LOG_VERSION 1 +#define FBNIC_FW_LOG_MAX_SIZE 256 +/* + * The max amount of logs which can fit in a single mailbox message. Firmware + * assumes each mailbox message is 4096B. The amount of messages supported is + * calculated as 4096 minus headers for message, arrays, and length minus the + * size of length divided by headers for each array plus the maximum LOG size, + * and the size of MSEC and INDEX. Put another way: + * + * MAX_LOG_HISTORY = ((4096 - TLV_HDR_SZ * 5 - LENGTH_SZ) + * / (FBNIC_FW_LOG_MAX_SIZE + TLV_HDR_SZ * 3 + MSEC_SZ + * + INDEX_SZ)) + */ +#define FBNIC_FW_MAX_LOG_HISTORY 14 struct fbnic_fw_ver { u32 version; @@ -38,8 +51,10 @@ struct fbnic_fw_cap { } stored; u8 active_slot; u8 bmc_mac_addr[4][ETH_ALEN]; - u8 bmc_present : 1; - u8 all_multi : 1; + u8 bmc_present : 1; + u8 need_bmc_tcam_reinit : 1; + u8 need_bmc_macda_sync : 1; + u8 all_multi : 1; u8 link_speed; u8 link_fec; u32 anti_rollback_version; @@ -52,10 +67,25 @@ struct fbnic_fw_completion { int result; union { struct { + u32 size; + } coredump_info; + struct { + u32 size; + u16 stride; + u8 *data[]; + } coredump; + struct { u32 offset; u32 length; } fw_update; struct { + u16 length; + u8 offset; + u8 page; + u8 bank; + u8 data[] __aligned(sizeof(u32)) __counted_by(length); + } qsfp; + struct { s32 millivolts; s32 millidegrees; } tsene; @@ -66,23 +96,37 @@ void fbnic_mbx_init(struct fbnic_dev *fbd); void fbnic_mbx_clean(struct fbnic_dev *fbd); int fbnic_mbx_set_cmpl(struct fbnic_dev *fbd, struct fbnic_fw_completion *cmpl_data); +void fbnic_mbx_clear_cmpl(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data); void fbnic_mbx_poll(struct fbnic_dev *fbd); int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd); void fbnic_mbx_flush_tx(struct fbnic_dev *fbd); int fbnic_fw_xmit_ownership_msg(struct fbnic_dev *fbd, bool take_ownership); int fbnic_fw_init_heartbeat(struct fbnic_dev *fbd, bool poll); void fbnic_fw_check_heartbeat(struct fbnic_dev *fbd); +int fbnic_fw_xmit_coredump_info_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + bool force); +int fbnic_fw_xmit_coredump_read_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + u32 offset, u32 length); int fbnic_fw_xmit_fw_start_upgrade(struct fbnic_dev *fbd, struct fbnic_fw_completion *cmpl_data, unsigned int id, unsigned int len); int fbnic_fw_xmit_fw_write_chunk(struct fbnic_dev *fbd, const u8 *data, u32 offset, u16 length, int cancel_error); +int fbnic_fw_xmit_qsfp_read_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + u32 page, u32 bank, u32 offset, u32 length); int fbnic_fw_xmit_tsene_read_msg(struct fbnic_dev *fbd, struct fbnic_fw_completion *cmpl_data); +int fbnic_fw_xmit_send_logs(struct fbnic_dev *fbd, bool enable, + bool send_log_history); +int fbnic_fw_xmit_rpc_macda_sync(struct fbnic_dev *fbd); +struct fbnic_fw_completion *__fbnic_fw_alloc_cmpl(u32 msg_type, + size_t priv_size); struct fbnic_fw_completion *fbnic_fw_alloc_cmpl(u32 msg_type); -void fbnic_fw_clear_cmpl(struct fbnic_dev *fbd, - struct fbnic_fw_completion *cmpl_data); void fbnic_fw_put_cmpl(struct fbnic_fw_completion *cmpl_data); #define fbnic_mk_full_fw_ver_str(_rev_id, _delim, _commit, _str, _str_sz) \ @@ -117,14 +161,24 @@ enum { FBNIC_TLV_MSG_ID_OWNERSHIP_RESP = 0x13, FBNIC_TLV_MSG_ID_HEARTBEAT_REQ = 0x14, FBNIC_TLV_MSG_ID_HEARTBEAT_RESP = 0x15, + FBNIC_TLV_MSG_ID_COREDUMP_GET_INFO_REQ = 0x18, + FBNIC_TLV_MSG_ID_COREDUMP_GET_INFO_RESP = 0x19, + FBNIC_TLV_MSG_ID_COREDUMP_READ_REQ = 0x20, + FBNIC_TLV_MSG_ID_COREDUMP_READ_RESP = 0x21, FBNIC_TLV_MSG_ID_FW_START_UPGRADE_REQ = 0x22, FBNIC_TLV_MSG_ID_FW_START_UPGRADE_RESP = 0x23, FBNIC_TLV_MSG_ID_FW_WRITE_CHUNK_REQ = 0x24, FBNIC_TLV_MSG_ID_FW_WRITE_CHUNK_RESP = 0x25, FBNIC_TLV_MSG_ID_FW_FINISH_UPGRADE_REQ = 0x28, FBNIC_TLV_MSG_ID_FW_FINISH_UPGRADE_RESP = 0x29, + FBNIC_TLV_MSG_ID_QSFP_READ_REQ = 0x38, + FBNIC_TLV_MSG_ID_QSFP_READ_RESP = 0x39, FBNIC_TLV_MSG_ID_TSENE_READ_REQ = 0x3C, FBNIC_TLV_MSG_ID_TSENE_READ_RESP = 0x3D, + FBNIC_TLV_MSG_ID_LOG_SEND_LOGS_REQ = 0x43, + FBNIC_TLV_MSG_ID_LOG_MSG_REQ = 0x44, + FBNIC_TLV_MSG_ID_LOG_MSG_RESP = 0x45, + FBNIC_TLV_MSG_ID_RPC_MAC_SYNC_REQ = 0x46, }; #define FBNIC_FW_CAP_RESP_VERSION_MAJOR CSR_GENMASK(31, 24) @@ -155,10 +209,10 @@ enum { }; enum { - FBNIC_FW_LINK_SPEED_25R1 = 1, - FBNIC_FW_LINK_SPEED_50R2 = 2, - FBNIC_FW_LINK_SPEED_50R1 = 3, - FBNIC_FW_LINK_SPEED_100R2 = 4, + FBNIC_FW_LINK_MODE_25CR = 1, + FBNIC_FW_LINK_MODE_50CR2 = 2, + FBNIC_FW_LINK_MODE_50CR = 3, + FBNIC_FW_LINK_MODE_100CR2 = 4, }; enum { @@ -168,6 +222,16 @@ enum { }; enum { + FBNIC_FW_QSFP_BANK = 0x0, + FBNIC_FW_QSFP_PAGE = 0x1, + FBNIC_FW_QSFP_OFFSET = 0x2, + FBNIC_FW_QSFP_LENGTH = 0x3, + FBNIC_FW_QSFP_ERROR = 0x4, + FBNIC_FW_QSFP_DATA = 0x5, + FBNIC_FW_QSFP_MSG_MAX +}; + +enum { FBNIC_FW_TSENE_THERM = 0x0, FBNIC_FW_TSENE_VOLT = 0x1, FBNIC_FW_TSENE_ERROR = 0x2, @@ -176,10 +240,37 @@ enum { enum { FBNIC_FW_OWNERSHIP_FLAG = 0x0, + FBNIC_FW_OWNERSHIP_TIME = 0x1, FBNIC_FW_OWNERSHIP_MSG_MAX }; enum { + FBNIC_FW_HEARTBEAT_UPTIME = 0x0, + FBNIC_FW_HEARTBEAT_NUMBER_OF_MESSAGES = 0x1, + FBNIC_FW_HEARTBEAT_MSG_MAX +}; + +enum { + FBNIC_FW_COREDUMP_REQ_INFO_CREATE = 0x0, + FBNIC_FW_COREDUMP_REQ_INFO_MSG_MAX +}; + +enum { + FBNIC_FW_COREDUMP_INFO_AVAILABLE = 0x0, + FBNIC_FW_COREDUMP_INFO_SIZE = 0x1, + FBNIC_FW_COREDUMP_INFO_ERROR = 0x2, + FBNIC_FW_COREDUMP_INFO_MSG_MAX +}; + +enum { + FBNIC_FW_COREDUMP_READ_OFFSET = 0x0, + FBNIC_FW_COREDUMP_READ_LENGTH = 0x1, + FBNIC_FW_COREDUMP_READ_DATA = 0x2, + FBNIC_FW_COREDUMP_READ_ERROR = 0x3, + FBNIC_FW_COREDUMP_READ_MSG_MAX +}; + +enum { FBNIC_FW_START_UPGRADE_ERROR = 0x0, FBNIC_FW_START_UPGRADE_SECTION = 0x1, FBNIC_FW_START_UPGRADE_IMAGE_LENGTH = 0x2, @@ -199,4 +290,37 @@ enum { FBNIC_FW_FINISH_UPGRADE_MSG_MAX }; +enum { + FBNIC_SEND_LOGS = 0x0, + FBNIC_SEND_LOGS_VERSION = 0x1, + FBNIC_SEND_LOGS_HISTORY = 0x2, + FBNIC_SEND_LOGS_MSG_MAX +}; + +enum { + FBNIC_FW_LOG_MSEC = 0x0, + FBNIC_FW_LOG_INDEX = 0x1, + FBNIC_FW_LOG_MSG = 0x2, + FBNIC_FW_LOG_LENGTH = 0x3, + FBNIC_FW_LOG_MSEC_ARRAY = 0x4, + FBNIC_FW_LOG_INDEX_ARRAY = 0x5, + FBNIC_FW_LOG_MSG_ARRAY = 0x6, + FBNIC_FW_LOG_MSG_MAX +}; + +enum { + FBNIC_FW_RPC_MAC_SYNC_RX_FLAGS = 0x0, + FBNIC_FW_RPC_MAC_SYNC_UC_ARRAY = 0x1, + FBNIC_FW_RPC_MAC_SYNC_MC_ARRAY = 0x2, + FBNIC_FW_RPC_MAC_SYNC_MAC_ADDR = 0x3, + FBNIC_FW_RPC_MAC_SYNC_MSG_MAX +}; + +#define FW_RPC_MAC_SYNC_RX_FLAGS_PROMISC 1 +#define FW_RPC_MAC_SYNC_RX_FLAGS_ALLMULTI 2 +#define FW_RPC_MAC_SYNC_RX_FLAGS_BROADCAST 4 + +#define FW_RPC_MAC_SYNC_UC_ARRAY_SIZE 8 +#define FW_RPC_MAC_SYNC_MC_ARRAY_SIZE 8 + #endif /* _FBNIC_FW_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c new file mode 100644 index 000000000000..85a883dba385 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/spinlock.h> +#include <linux/vmalloc.h> + +#include "fbnic.h" +#include "fbnic_fw.h" +#include "fbnic_fw_log.h" + +void fbnic_fw_log_enable(struct fbnic_dev *fbd, bool send_hist) +{ + int err; + + if (!fbnic_fw_log_ready(fbd)) + return; + + if (fbd->fw_cap.running.mgmt.version < MIN_FW_VER_CODE_HIST) + send_hist = false; + + err = fbnic_fw_xmit_send_logs(fbd, true, send_hist); + if (err && err != -EOPNOTSUPP) + dev_warn(fbd->dev, "Unable to enable firmware logs: %d\n", err); +} + +void fbnic_fw_log_disable(struct fbnic_dev *fbd) +{ + int err; + + err = fbnic_fw_xmit_send_logs(fbd, false, false); + if (err && err != -EOPNOTSUPP) + dev_warn(fbd->dev, "Unable to disable firmware logs: %d\n", + err); +} + +int fbnic_fw_log_init(struct fbnic_dev *fbd) +{ + struct fbnic_fw_log *log = &fbd->fw_log; + void *data; + + if (WARN_ON_ONCE(fbnic_fw_log_ready(fbd))) + return -EEXIST; + + data = vmalloc(FBNIC_FW_LOG_SIZE); + if (!data) + return -ENOMEM; + + spin_lock_init(&fbd->fw_log.lock); + INIT_LIST_HEAD(&log->entries); + log->size = FBNIC_FW_LOG_SIZE; + log->data_start = data; + log->data_end = data + FBNIC_FW_LOG_SIZE; + + fbnic_fw_log_enable(fbd, true); + + return 0; +} + +void fbnic_fw_log_free(struct fbnic_dev *fbd) +{ + struct fbnic_fw_log *log = &fbd->fw_log; + + if (!fbnic_fw_log_ready(fbd)) + return; + + fbnic_fw_log_disable(fbd); + INIT_LIST_HEAD(&log->entries); + log->size = 0; + vfree(log->data_start); + log->data_start = NULL; + log->data_end = NULL; +} + +int fbnic_fw_log_write(struct fbnic_dev *fbd, u64 index, u32 timestamp, + const char *msg) +{ + struct fbnic_fw_log_entry *entry, *head, *tail, *next; + struct fbnic_fw_log *log = &fbd->fw_log; + size_t msg_len = strlen(msg) + 1; + unsigned long flags; + void *entry_end; + + if (!fbnic_fw_log_ready(fbd)) { + dev_err(fbd->dev, "Firmware sent log entry without being requested!\n"); + return -ENOSPC; + } + + spin_lock_irqsave(&log->lock, flags); + + if (list_empty(&log->entries)) { + entry = log->data_start; + } else { + head = list_first_entry(&log->entries, typeof(*head), list); + entry_end = head->msg + head->len + 1; + entry = PTR_ALIGN(entry_end, 8); + } + + entry_end = entry->msg + msg_len + 1; + + /* We've reached the end of the buffer, wrap around */ + if (entry_end > log->data_end) { + entry = log->data_start; + entry_end = entry->msg + msg_len + 1; + } + + /* Make room for entry by removing from tail. */ + list_for_each_entry_safe_reverse(tail, next, &log->entries, list) { + if (entry <= tail && entry_end > (void *)tail) + list_del(&tail->list); + else + break; + } + + entry->index = index; + entry->timestamp = timestamp; + entry->len = msg_len; + strscpy(entry->msg, msg, entry->len); + list_add(&entry->list, &log->entries); + + spin_unlock_irqrestore(&log->lock, flags); + + return 0; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.h b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.h new file mode 100644 index 000000000000..50ec79003108 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _FBNIC_FW_LOG_H_ +#define _FBNIC_FW_LOG_H_ + +#include <linux/spinlock.h> +#include <linux/types.h> + +/* A 512K log buffer was chosen fairly arbitrarily */ +#define FBNIC_FW_LOG_SIZE (512 * 1024) /* bytes */ + +/* Firmware log output is prepended with log index followed by a timestamp. + * The timestamp is similar to Zephyr's format DD:HH:MM:SS.MMM + */ +#define FBNIC_FW_LOG_FMT "[%5lld] [%02ld:%02ld:%02ld:%02ld.%03ld] %s\n" + +struct fbnic_dev; + +struct fbnic_fw_log_entry { + struct list_head list; + u64 index; + u32 timestamp; + u16 len; + char msg[] __counted_by(len); +}; + +struct fbnic_fw_log { + void *data_start; + void *data_end; + size_t size; + struct list_head entries; + /* Spin lock for accessing or modifying entries */ + spinlock_t lock; +}; + +#define fbnic_fw_log_ready(_fbd) (!!(_fbd)->fw_log.data_start) + +void fbnic_fw_log_enable(struct fbnic_dev *fbd, bool send_hist); +void fbnic_fw_log_disable(struct fbnic_dev *fbd); +int fbnic_fw_log_init(struct fbnic_dev *fbd); +void fbnic_fw_log_free(struct fbnic_dev *fbd); +int fbnic_fw_log_write(struct fbnic_dev *fbd, u64 index, u32 timestamp, + const char *msg); +#endif /* _FBNIC_FW_LOG_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.c b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.c index 4223d8100e64..8b9b2076beec 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ +#include <linux/rtnetlink.h> + #include "fbnic.h" static void fbnic_hw_stat_rst32(struct fbnic_dev *fbd, u32 reg, @@ -421,9 +423,9 @@ static void fbnic_get_hw_rxq_stats32(struct fbnic_dev *fbd, void fbnic_get_hw_q_stats(struct fbnic_dev *fbd, struct fbnic_hw_q_stats *hw_q) { - spin_lock(&fbd->hw_stats_lock); + spin_lock(&fbd->hw_stats.lock); fbnic_get_hw_rxq_stats32(fbd, hw_q); - spin_unlock(&fbd->hw_stats_lock); + spin_unlock(&fbd->hw_stats.lock); } static void fbnic_reset_pcie_stats_asic(struct fbnic_dev *fbd, @@ -510,20 +512,68 @@ static void fbnic_get_pcie_stats_asic64(struct fbnic_dev *fbd, &pcie->ob_rd_no_np_cred); } +static void fbnic_reset_phy_stats(struct fbnic_dev *fbd, + struct fbnic_phy_stats *phy_stats) +{ + const struct fbnic_mac *mac = fbd->mac; + + mac->get_fec_stats(fbd, true, &phy_stats->fec); + mac->get_pcs_stats(fbd, true, &phy_stats->pcs); +} + +static void fbnic_get_phy_stats32(struct fbnic_dev *fbd, + struct fbnic_phy_stats *phy_stats) +{ + const struct fbnic_mac *mac = fbd->mac; + + mac->get_fec_stats(fbd, false, &phy_stats->fec); + mac->get_pcs_stats(fbd, false, &phy_stats->pcs); +} + +static void fbnic_reset_hw_mac_stats(struct fbnic_dev *fbd, + struct fbnic_mac_stats *mac_stats) +{ + const struct fbnic_mac *mac = fbd->mac; + + mac->get_eth_mac_stats(fbd, true, &mac_stats->eth_mac); + mac->get_pause_stats(fbd, true, &mac_stats->pause); + mac->get_eth_ctrl_stats(fbd, true, &mac_stats->eth_ctrl); + mac->get_rmon_stats(fbd, true, &mac_stats->rmon); +} + void fbnic_reset_hw_stats(struct fbnic_dev *fbd) { - spin_lock(&fbd->hw_stats_lock); + spin_lock(&fbd->hw_stats.lock); + fbnic_reset_phy_stats(fbd, &fbd->hw_stats.phy); fbnic_reset_tmi_stats(fbd, &fbd->hw_stats.tmi); fbnic_reset_tti_stats(fbd, &fbd->hw_stats.tti); fbnic_reset_rpc_stats(fbd, &fbd->hw_stats.rpc); fbnic_reset_rxb_stats(fbd, &fbd->hw_stats.rxb); fbnic_reset_hw_rxq_stats(fbd, fbd->hw_stats.hw_q); fbnic_reset_pcie_stats_asic(fbd, &fbd->hw_stats.pcie); - spin_unlock(&fbd->hw_stats_lock); + spin_unlock(&fbd->hw_stats.lock); + + /* Once registered, the only other access to MAC stats is via the + * ethtool API which is protected by the rtnl_lock. The call to + * fbnic_reset_hw_stats() during PCI recovery is also protected + * by the rtnl_lock hence, we don't need the spinlock to access + * the MAC stats. + */ + if (fbd->netdev) + ASSERT_RTNL(); + fbnic_reset_hw_mac_stats(fbd, &fbd->hw_stats.mac); +} + +void fbnic_init_hw_stats(struct fbnic_dev *fbd) +{ + spin_lock_init(&fbd->hw_stats.lock); + + fbnic_reset_hw_stats(fbd); } static void __fbnic_get_hw_stats32(struct fbnic_dev *fbd) { + fbnic_get_phy_stats32(fbd, &fbd->hw_stats.phy); fbnic_get_tmi_stats32(fbd, &fbd->hw_stats.tmi); fbnic_get_tti_stats32(fbd, &fbd->hw_stats.tti); fbnic_get_rpc_stats32(fbd, &fbd->hw_stats.rpc); @@ -533,19 +583,19 @@ static void __fbnic_get_hw_stats32(struct fbnic_dev *fbd) void fbnic_get_hw_stats32(struct fbnic_dev *fbd) { - spin_lock(&fbd->hw_stats_lock); + spin_lock(&fbd->hw_stats.lock); __fbnic_get_hw_stats32(fbd); - spin_unlock(&fbd->hw_stats_lock); + spin_unlock(&fbd->hw_stats.lock); } void fbnic_get_hw_stats(struct fbnic_dev *fbd) { - spin_lock(&fbd->hw_stats_lock); + spin_lock(&fbd->hw_stats.lock); __fbnic_get_hw_stats32(fbd); fbnic_get_tmi_stats(fbd, &fbd->hw_stats.tmi); fbnic_get_tti_stats(fbd, &fbd->hw_stats.tti); fbnic_get_rxb_stats(fbd, &fbd->hw_stats.rxb); fbnic_get_pcie_stats_asic64(fbd, &fbd->hw_stats.pcie); - spin_unlock(&fbd->hw_stats_lock); + spin_unlock(&fbd->hw_stats.lock); } diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h index 07e54bb75bf3..aa3f429a9aed 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h @@ -5,6 +5,7 @@ #define _FBNIC_HW_STATS_H_ #include <linux/ethtool.h> +#include <linux/spinlock.h> #include "fbnic_csr.h" @@ -22,6 +23,39 @@ struct fbnic_hw_stat { struct fbnic_stat_counter bytes; }; +struct fbnic_fec_stats { + struct fbnic_stat_counter corrected_blocks, uncorrectable_blocks; +}; + +struct fbnic_pcs_stats { + struct { + struct fbnic_stat_counter lanes[FBNIC_PCS_MAX_LANES]; + } SymbolErrorDuringCarrier; +}; + +/* Note: not updated by fbnic_get_hw_stats() */ +struct fbnic_eth_ctrl_stats { + struct fbnic_stat_counter MACControlFramesTransmitted; + struct fbnic_stat_counter MACControlFramesReceived; +}; + +/* Note: not updated by fbnic_get_hw_stats() */ +struct fbnic_rmon_stats { + struct fbnic_stat_counter undersize_pkts; + struct fbnic_stat_counter oversize_pkts; + struct fbnic_stat_counter fragments; + struct fbnic_stat_counter jabbers; + + struct fbnic_stat_counter hist[ETHTOOL_RMON_HIST_MAX]; + struct fbnic_stat_counter hist_tx[ETHTOOL_RMON_HIST_MAX]; +}; + +/* Note: not updated by fbnic_get_hw_stats() */ +struct fbnic_pause_stats { + struct fbnic_stat_counter tx_pause_frames; + struct fbnic_stat_counter rx_pause_frames; +}; + struct fbnic_eth_mac_stats { struct fbnic_stat_counter FramesTransmittedOK; struct fbnic_stat_counter FramesReceivedOK; @@ -38,8 +72,16 @@ struct fbnic_eth_mac_stats { struct fbnic_stat_counter FrameTooLongErrors; }; +struct fbnic_phy_stats { + struct fbnic_fec_stats fec; + struct fbnic_pcs_stats pcs; +}; + struct fbnic_mac_stats { struct fbnic_eth_mac_stats eth_mac; + struct fbnic_pause_stats pause; + struct fbnic_eth_ctrl_stats eth_ctrl; + struct fbnic_rmon_stats rmon; }; struct fbnic_tmi_stats { @@ -96,6 +138,7 @@ struct fbnic_pcie_stats { }; struct fbnic_hw_stats { + struct fbnic_phy_stats phy; struct fbnic_mac_stats mac; struct fbnic_tmi_stats tmi; struct fbnic_tti_stats tti; @@ -103,11 +146,15 @@ struct fbnic_hw_stats { struct fbnic_rxb_stats rxb; struct fbnic_hw_q_stats hw_q[FBNIC_MAX_QUEUES]; struct fbnic_pcie_stats pcie; + + /* Lock protecting the access to hw stats */ + spinlock_t lock; }; u64 fbnic_stat_rd64(struct fbnic_dev *fbd, u32 reg, u32 offset); void fbnic_reset_hw_stats(struct fbnic_dev *fbd); +void fbnic_init_hw_stats(struct fbnic_dev *fbd); void fbnic_get_hw_q_stats(struct fbnic_dev *fbd, struct fbnic_hw_q_stats *hw_q); void fbnic_get_hw_stats32(struct fbnic_dev *fbd); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c index 10e108c1fcd0..8f998d26b9a3 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c @@ -452,7 +452,7 @@ static u32 __fbnic_mac_cmd_config_asic(struct fbnic_dev *fbd, command_config |= FBNIC_MAC_COMMAND_CONFIG_RX_PAUSE_DIS; /* Disable fault handling if no FEC is requested */ - if ((fbn->fec & FBNIC_FEC_MODE_MASK) == FBNIC_FEC_OFF) + if (fbn->fec == FBNIC_FEC_OFF) command_config |= FBNIC_MAC_COMMAND_CONFIG_FLT_HDL_DIS; return command_config; @@ -468,15 +468,15 @@ static bool fbnic_mac_get_pcs_link_status(struct fbnic_dev *fbd) return false; /* Define the expected lane mask for the status bits we need to check */ - switch (fbn->link_mode & FBNIC_LINK_MODE_MASK) { - case FBNIC_LINK_100R2: + switch (fbn->aui) { + case FBNIC_AUI_100GAUI2: lane_mask = 0xf; break; - case FBNIC_LINK_50R1: + case FBNIC_AUI_50GAUI1: lane_mask = 3; break; - case FBNIC_LINK_50R2: - switch (fbn->fec & FBNIC_FEC_MODE_MASK) { + case FBNIC_AUI_LAUI2: + switch (fbn->fec) { case FBNIC_FEC_OFF: lane_mask = 0x63; break; @@ -488,13 +488,13 @@ static bool fbnic_mac_get_pcs_link_status(struct fbnic_dev *fbd) break; } break; - case FBNIC_LINK_25R1: + case FBNIC_AUI_25GAUI: lane_mask = 1; break; } /* Use an XOR to remove the bits we expect to see set */ - switch (fbn->fec & FBNIC_FEC_MODE_MASK) { + switch (fbn->fec) { case FBNIC_FEC_OFF: lane_mask ^= FIELD_GET(FBNIC_SIG_PCS_OUT0_BLOCK_LOCK, pcs_status); @@ -540,53 +540,41 @@ static bool fbnic_pcs_get_link_asic(struct fbnic_dev *fbd) return link; } -static void fbnic_pcs_get_fw_settings(struct fbnic_dev *fbd) +void fbnic_mac_get_fw_settings(struct fbnic_dev *fbd, u8 *aui, u8 *fec) { - struct fbnic_net *fbn = netdev_priv(fbd->netdev); - u8 link_mode = fbn->link_mode; - u8 fec = fbn->fec; - - /* Update FEC first to reflect FW current mode */ - if (fbn->fec & FBNIC_FEC_AUTO) { - switch (fbd->fw_cap.link_fec) { - case FBNIC_FW_LINK_FEC_NONE: - fec = FBNIC_FEC_OFF; - break; - case FBNIC_FW_LINK_FEC_RS: - fec = FBNIC_FEC_RS; - break; - case FBNIC_FW_LINK_FEC_BASER: - fec = FBNIC_FEC_BASER; - break; - default: - return; - } - - fbn->fec = fec; + /* Retrieve default speed from FW */ + switch (fbd->fw_cap.link_speed) { + case FBNIC_FW_LINK_MODE_25CR: + *aui = FBNIC_AUI_25GAUI; + break; + case FBNIC_FW_LINK_MODE_50CR2: + *aui = FBNIC_AUI_LAUI2; + break; + case FBNIC_FW_LINK_MODE_50CR: + *aui = FBNIC_AUI_50GAUI1; + *fec = FBNIC_FEC_RS; + return; + case FBNIC_FW_LINK_MODE_100CR2: + *aui = FBNIC_AUI_100GAUI2; + *fec = FBNIC_FEC_RS; + return; + default: + *aui = FBNIC_AUI_UNKNOWN; + return; } - /* Do nothing if AUTO mode is not engaged */ - if (fbn->link_mode & FBNIC_LINK_AUTO) { - switch (fbd->fw_cap.link_speed) { - case FBNIC_FW_LINK_SPEED_25R1: - link_mode = FBNIC_LINK_25R1; - break; - case FBNIC_FW_LINK_SPEED_50R2: - link_mode = FBNIC_LINK_50R2; - break; - case FBNIC_FW_LINK_SPEED_50R1: - link_mode = FBNIC_LINK_50R1; - fec = FBNIC_FEC_RS; - break; - case FBNIC_FW_LINK_SPEED_100R2: - link_mode = FBNIC_LINK_100R2; - fec = FBNIC_FEC_RS; - break; - default: - return; - } - - fbn->link_mode = link_mode; + /* Update FEC first to reflect FW current mode */ + switch (fbd->fw_cap.link_fec) { + case FBNIC_FW_LINK_FEC_NONE: + *fec = FBNIC_FEC_OFF; + break; + case FBNIC_FW_LINK_FEC_RS: + default: + *fec = FBNIC_FEC_RS; + break; + case FBNIC_FW_LINK_FEC_BASER: + *fec = FBNIC_FEC_BASER; + break; } } @@ -596,9 +584,6 @@ static int fbnic_pcs_enable_asic(struct fbnic_dev *fbd) wr32(fbd, FBNIC_SIG_PCS_INTR_MASK, ~0); wr32(fbd, FBNIC_SIG_PCS_INTR_STS, ~0); - /* Pull in settings from FW */ - fbnic_pcs_get_fw_settings(fbd); - return 0; } @@ -647,6 +632,50 @@ static void fbnic_mac_link_up_asic(struct fbnic_dev *fbd, } static void +fbnic_pcs_rsfec_stat_rd32(struct fbnic_dev *fbd, u32 reg, bool reset, + struct fbnic_stat_counter *stat) +{ + u32 pcs_rsfec_stat; + + /* The PCS/RFSEC registers are only 16b wide each. So what we will + * have after the 64b read is 0x0000xxxx0000xxxx. To make it usable + * as a full stat we will shift the upper bits into the lower set of + * 0s and then mask off the math at 32b. + * + * Read ordering must be lower reg followed by upper reg. + */ + pcs_rsfec_stat = rd32(fbd, reg) & 0xffff; + pcs_rsfec_stat |= rd32(fbd, reg + 1) << 16; + + /* RFSEC registers clear themselves upon being read so there is no + * need to store the old_reg_value. + */ + if (!reset) + stat->value += pcs_rsfec_stat; +} + +static void +fbnic_mac_get_fec_stats(struct fbnic_dev *fbd, bool reset, + struct fbnic_fec_stats *s) +{ + fbnic_pcs_rsfec_stat_rd32(fbd, FBNIC_RSFEC_CCW_LO(0), reset, + &s->corrected_blocks); + fbnic_pcs_rsfec_stat_rd32(fbd, FBNIC_RSFEC_NCCW_LO(0), reset, + &s->uncorrectable_blocks); +} + +static void +fbnic_mac_get_pcs_stats(struct fbnic_dev *fbd, bool reset, + struct fbnic_pcs_stats *s) +{ + int i; + + for (i = 0; i < FBNIC_PCS_MAX_LANES; i++) + fbnic_pcs_rsfec_stat_rd32(fbd, FBNIC_PCS_SYMBLERR_LO(i), reset, + &s->SymbolErrorDuringCarrier.lanes[i]); +} + +static void fbnic_mac_get_eth_mac_stats(struct fbnic_dev *fbd, bool reset, struct fbnic_eth_mac_stats *mac_stats) { @@ -680,6 +709,86 @@ fbnic_mac_get_eth_mac_stats(struct fbnic_dev *fbd, bool reset, MAC_STAT_TX_BROADCAST); } +static void +fbnic_mac_get_pause_stats(struct fbnic_dev *fbd, bool reset, + struct fbnic_pause_stats *pause_stats) +{ + fbnic_mac_stat_rd64(fbd, reset, pause_stats->tx_pause_frames, + MAC_STAT_TX_XOFF_STB); + fbnic_mac_stat_rd64(fbd, reset, pause_stats->rx_pause_frames, + MAC_STAT_RX_XOFF_STB); +} + +static void +fbnic_mac_get_eth_ctrl_stats(struct fbnic_dev *fbd, bool reset, + struct fbnic_eth_ctrl_stats *ctrl_stats) +{ + fbnic_mac_stat_rd64(fbd, reset, ctrl_stats->MACControlFramesReceived, + MAC_STAT_RX_CONTROL_FRAMES); + fbnic_mac_stat_rd64(fbd, reset, ctrl_stats->MACControlFramesTransmitted, + MAC_STAT_TX_CONTROL_FRAMES); +} + +static void +fbnic_mac_get_rmon_stats(struct fbnic_dev *fbd, bool reset, + struct fbnic_rmon_stats *rmon_stats) +{ + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->undersize_pkts, + MAC_STAT_RX_UNDERSIZE); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->oversize_pkts, + MAC_STAT_RX_OVERSIZE); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->fragments, + MAC_STAT_RX_FRAGMENT); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->jabbers, + MAC_STAT_RX_JABBER); + + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[0], + MAC_STAT_RX_PACKET_64_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[1], + MAC_STAT_RX_PACKET_65_127_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[2], + MAC_STAT_RX_PACKET_128_255_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[3], + MAC_STAT_RX_PACKET_256_511_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[4], + MAC_STAT_RX_PACKET_512_1023_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[5], + MAC_STAT_RX_PACKET_1024_1518_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[6], + RPC_STAT_RX_PACKET_1519_2047_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[7], + RPC_STAT_RX_PACKET_2048_4095_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[8], + RPC_STAT_RX_PACKET_4096_8191_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[9], + RPC_STAT_RX_PACKET_8192_9216_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[10], + RPC_STAT_RX_PACKET_9217_MAX_BYTES); + + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[0], + MAC_STAT_TX_PACKET_64_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[1], + MAC_STAT_TX_PACKET_65_127_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[2], + MAC_STAT_TX_PACKET_128_255_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[3], + MAC_STAT_TX_PACKET_256_511_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[4], + MAC_STAT_TX_PACKET_512_1023_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[5], + MAC_STAT_TX_PACKET_1024_1518_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[6], + TMI_STAT_TX_PACKET_1519_2047_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[7], + TMI_STAT_TX_PACKET_2048_4095_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[8], + TMI_STAT_TX_PACKET_4096_8191_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[9], + TMI_STAT_TX_PACKET_8192_9216_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[10], + TMI_STAT_TX_PACKET_9217_MAX_BYTES); +} + static int fbnic_mac_get_sensor_asic(struct fbnic_dev *fbd, int id, long *val) { @@ -741,7 +850,7 @@ static int fbnic_mac_get_sensor_asic(struct fbnic_dev *fbd, int id, *val = *sensor; exit_cleanup: - fbnic_fw_clear_cmpl(fbd, fw_cmpl); + fbnic_mbx_clear_cmpl(fbd, fw_cmpl); exit_free: fbnic_fw_put_cmpl(fw_cmpl); @@ -754,7 +863,12 @@ static const struct fbnic_mac fbnic_mac_asic = { .pcs_disable = fbnic_pcs_disable_asic, .pcs_get_link = fbnic_pcs_get_link_asic, .pcs_get_link_event = fbnic_pcs_get_link_event_asic, + .get_fec_stats = fbnic_mac_get_fec_stats, + .get_pcs_stats = fbnic_mac_get_pcs_stats, .get_eth_mac_stats = fbnic_mac_get_eth_mac_stats, + .get_pause_stats = fbnic_mac_get_pause_stats, + .get_eth_ctrl_stats = fbnic_mac_get_eth_ctrl_stats, + .get_rmon_stats = fbnic_mac_get_rmon_stats, .link_down = fbnic_mac_link_down_asic, .link_up = fbnic_mac_link_up_asic, .get_sensor = fbnic_mac_get_sensor_asic, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.h b/drivers/net/ethernet/meta/fbnic/fbnic_mac.h index 05a591653e09..ede5ff0dae22 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.h @@ -25,27 +25,23 @@ enum { FBNIC_FEC_OFF = 0, FBNIC_FEC_RS = 1, FBNIC_FEC_BASER = 2, - FBNIC_FEC_AUTO = 4, }; -#define FBNIC_FEC_MODE_MASK (FBNIC_FEC_AUTO - 1) - -/* Treat the link modes as a set of modulation/lanes bitmask: +/* Treat the AUI modes as a modulation/lanes bitmask: * Bit 0: Lane Count, 0 = R1, 1 = R2 * Bit 1: Modulation, 0 = NRZ, 1 = PAM4 - * Bit 2: Retrieve link mode from FW + * Bit 2: Unknown Modulation/Lane Configuration */ enum { - FBNIC_LINK_25R1 = 0, - FBNIC_LINK_50R2 = 1, - FBNIC_LINK_50R1 = 2, - FBNIC_LINK_100R2 = 3, - FBNIC_LINK_AUTO = 4, + FBNIC_AUI_25GAUI = 0, /* 25.7812GBd 25.78125 * 1 */ + FBNIC_AUI_LAUI2 = 1, /* 51.5625GBd 25.78128 * 2 */ + FBNIC_AUI_50GAUI1 = 2, /* 53.125GBd 53.125 * 1 */ + FBNIC_AUI_100GAUI2 = 3, /* 106.25GBd 53.125 * 2 */ + FBNIC_AUI_UNKNOWN = 4, }; -#define FBNIC_LINK_MODE_R2 (FBNIC_LINK_50R2) -#define FBNIC_LINK_MODE_PAM4 (FBNIC_LINK_50R1) -#define FBNIC_LINK_MODE_MASK (FBNIC_LINK_AUTO - 1) +#define FBNIC_AUI_MODE_R2 (FBNIC_AUI_LAUI2) +#define FBNIC_AUI_MODE_PAM4 (FBNIC_AUI_50GAUI1) enum fbnic_sensor_id { FBNIC_SENSOR_TEMP, /* Temp in millidegrees Centigrade */ @@ -83,8 +79,18 @@ struct fbnic_mac { bool (*pcs_get_link)(struct fbnic_dev *fbd); int (*pcs_get_link_event)(struct fbnic_dev *fbd); + void (*get_fec_stats)(struct fbnic_dev *fbd, bool reset, + struct fbnic_fec_stats *fec_stats); + void (*get_pcs_stats)(struct fbnic_dev *fbd, bool reset, + struct fbnic_pcs_stats *pcs_stats); void (*get_eth_mac_stats)(struct fbnic_dev *fbd, bool reset, struct fbnic_eth_mac_stats *mac_stats); + void (*get_pause_stats)(struct fbnic_dev *fbd, bool reset, + struct fbnic_pause_stats *pause_stats); + void (*get_eth_ctrl_stats)(struct fbnic_dev *fbd, bool reset, + struct fbnic_eth_ctrl_stats *ctrl_stats); + void (*get_rmon_stats)(struct fbnic_dev *fbd, bool reset, + struct fbnic_rmon_stats *rmon_stats); void (*link_down)(struct fbnic_dev *fbd); void (*link_up)(struct fbnic_dev *fbd, bool tx_pause, bool rx_pause); @@ -93,4 +99,5 @@ struct fbnic_mac { }; int fbnic_mac_init(struct fbnic_dev *fbd); +void fbnic_mac_get_fw_settings(struct fbnic_dev *fbd, u8 *aui, u8 *fec); #endif /* _FBNIC_MAC_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c index aa812c63d5af..d12b4cad84a5 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c @@ -33,7 +33,7 @@ int __fbnic_open(struct fbnic_net *fbn) dev_warn(fbd->dev, "Error %d sending host ownership message to the firmware\n", err); - goto free_resources; + goto err_reset_queues; } err = fbnic_time_start(fbn); @@ -52,11 +52,15 @@ int __fbnic_open(struct fbnic_net *fbn) fbnic_bmc_rpc_init(fbd); fbnic_rss_reinit(fbd, fbn); + phylink_resume(fbn->phylink); + return 0; time_stop: fbnic_time_stop(fbn); release_ownership: fbnic_fw_xmit_ownership_msg(fbn->fbd, false); +err_reset_queues: + fbnic_reset_netif_queues(fbn); free_resources: fbnic_free_resources(fbn); free_napi_vectors: @@ -82,6 +86,8 @@ static int fbnic_stop(struct net_device *netdev) { struct fbnic_net *fbn = netdev_priv(netdev); + phylink_suspend(fbn->phylink, fbnic_bmc_present(fbn->fbd)); + fbnic_down(fbn); fbnic_pcs_free_irq(fbn->fbd); @@ -177,11 +183,10 @@ static int fbnic_mc_unsync(struct net_device *netdev, const unsigned char *addr) return ret; } -void __fbnic_set_rx_mode(struct net_device *netdev) +void __fbnic_set_rx_mode(struct fbnic_dev *fbd) { - struct fbnic_net *fbn = netdev_priv(netdev); bool uc_promisc = false, mc_promisc = false; - struct fbnic_dev *fbd = fbn->fbd; + struct net_device *netdev = fbd->netdev; struct fbnic_mac_addr *mac_addr; int err; @@ -218,49 +223,8 @@ void __fbnic_set_rx_mode(struct net_device *netdev) uc_promisc |= !!(netdev->flags & IFF_PROMISC); mc_promisc |= !!(netdev->flags & IFF_ALLMULTI) || uc_promisc; - /* Populate last TCAM entry with promiscuous entry and 0/1 bit mask */ - mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_PROMISC_IDX]; - if (uc_promisc) { - if (!is_zero_ether_addr(mac_addr->value.addr8) || - mac_addr->state != FBNIC_TCAM_S_VALID) { - eth_zero_addr(mac_addr->value.addr8); - eth_broadcast_addr(mac_addr->mask.addr8); - clear_bit(FBNIC_MAC_ADDR_T_ALLMULTI, - mac_addr->act_tcam); - set_bit(FBNIC_MAC_ADDR_T_PROMISC, - mac_addr->act_tcam); - mac_addr->state = FBNIC_TCAM_S_ADD; - } - } else if (mc_promisc && - (!fbnic_bmc_present(fbd) || !fbd->fw_cap.all_multi)) { - /* We have to add a special handler for multicast as the - * BMC may have an all-multi rule already in place. As such - * adding a rule ourselves won't do any good so we will have - * to modify the rules for the ALL MULTI below if the BMC - * already has the rule in place. - */ - if (!is_multicast_ether_addr(mac_addr->value.addr8) || - mac_addr->state != FBNIC_TCAM_S_VALID) { - eth_zero_addr(mac_addr->value.addr8); - eth_broadcast_addr(mac_addr->mask.addr8); - mac_addr->value.addr8[0] ^= 1; - mac_addr->mask.addr8[0] ^= 1; - set_bit(FBNIC_MAC_ADDR_T_ALLMULTI, - mac_addr->act_tcam); - clear_bit(FBNIC_MAC_ADDR_T_PROMISC, - mac_addr->act_tcam); - mac_addr->state = FBNIC_TCAM_S_ADD; - } - } else if (mac_addr->state == FBNIC_TCAM_S_VALID) { - if (test_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam)) { - clear_bit(FBNIC_MAC_ADDR_T_ALLMULTI, - mac_addr->act_tcam); - clear_bit(FBNIC_MAC_ADDR_T_PROMISC, - mac_addr->act_tcam); - } else { - mac_addr->state = FBNIC_TCAM_S_DELETE; - } - } + /* Update the promiscuous rules */ + fbnic_promisc_sync(fbd, uc_promisc, mc_promisc); /* Add rules for BMC all multicast if it is enabled */ fbnic_bmc_rpc_all_multi_config(fbd, mc_promisc); @@ -276,9 +240,12 @@ void __fbnic_set_rx_mode(struct net_device *netdev) static void fbnic_set_rx_mode(struct net_device *netdev) { + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + /* No need to update the hardware if we are not running */ if (netif_running(netdev)) - __fbnic_set_rx_mode(netdev); + __fbnic_set_rx_mode(fbd); } static int fbnic_set_mac(struct net_device *netdev, void *p) @@ -295,10 +262,9 @@ static int fbnic_set_mac(struct net_device *netdev, void *p) return 0; } -void fbnic_clear_rx_mode(struct net_device *netdev) +void fbnic_clear_rx_mode(struct fbnic_dev *fbd) { - struct fbnic_net *fbn = netdev_priv(netdev); - struct fbnic_dev *fbd = fbn->fbd; + struct net_device *netdev = fbd->netdev; int idx; for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) { @@ -405,11 +371,12 @@ static void fbnic_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats64) { u64 rx_bytes, rx_packets, rx_dropped = 0, rx_errors = 0; + u64 rx_over = 0, rx_missed = 0, rx_length = 0; u64 tx_bytes, tx_packets, tx_dropped = 0; struct fbnic_net *fbn = netdev_priv(dev); struct fbnic_dev *fbd = fbn->fbd; struct fbnic_queue_stats *stats; - u64 rx_over = 0, rx_missed = 0; + unsigned int start, i; fbnic_get_hw_stats(fbd); @@ -420,15 +387,17 @@ static void fbnic_get_stats64(struct net_device *dev, tx_packets = stats->packets; tx_dropped = stats->dropped; - stats64->tx_bytes = tx_bytes; - stats64->tx_packets = tx_packets; - stats64->tx_dropped = tx_dropped; - /* Record drops from Tx HW Datapath */ + spin_lock(&fbd->hw_stats.lock); tx_dropped += fbd->hw_stats.tmi.drop.frames.value + fbd->hw_stats.tti.cm_drop.frames.value + fbd->hw_stats.tti.frame_drop.frames.value + fbd->hw_stats.tti.tbi_drop.frames.value; + spin_unlock(&fbd->hw_stats.lock); + + stats64->tx_bytes = tx_bytes; + stats64->tx_packets = tx_packets; + stats64->tx_dropped = tx_dropped; for (i = 0; i < fbn->num_tx_queues; i++) { struct fbnic_ring *txr = fbn->tx[i]; @@ -455,7 +424,7 @@ static void fbnic_get_stats64(struct net_device *dev, rx_packets = stats->packets; rx_dropped = stats->dropped; - spin_lock(&fbd->hw_stats_lock); + spin_lock(&fbd->hw_stats.lock); /* Record drops for the host FIFOs. * 4: network to Host, 6: BMC to Host * Exclude the BMC and MC FIFOs as those stats may contain drops @@ -475,7 +444,7 @@ static void fbnic_get_stats64(struct net_device *dev, /* Report packets with errors */ rx_errors += fbd->hw_stats.hw_q[i].rde_pkt_err.value; } - spin_unlock(&fbd->hw_stats_lock); + spin_unlock(&fbd->hw_stats.lock); stats64->rx_bytes = rx_bytes; stats64->rx_packets = rx_packets; @@ -485,6 +454,7 @@ static void fbnic_get_stats64(struct net_device *dev, stats64->rx_missed_errors = rx_missed; for (i = 0; i < fbn->num_rx_queues; i++) { + struct fbnic_ring *xdpr = fbn->tx[FBNIC_MAX_TXQS + i]; struct fbnic_ring *rxr = fbn->rx[i]; if (!rxr) @@ -496,14 +466,66 @@ static void fbnic_get_stats64(struct net_device *dev, rx_bytes = stats->bytes; rx_packets = stats->packets; rx_dropped = stats->dropped; + rx_length = stats->rx.length_errors; } while (u64_stats_fetch_retry(&stats->syncp, start)); stats64->rx_bytes += rx_bytes; stats64->rx_packets += rx_packets; stats64->rx_dropped += rx_dropped; + stats64->rx_errors += rx_length; + stats64->rx_length_errors += rx_length; + + if (!xdpr) + continue; + + stats = &xdpr->stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + tx_bytes = stats->bytes; + tx_packets = stats->packets; + tx_dropped = stats->dropped; + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + stats64->tx_bytes += tx_bytes; + stats64->tx_packets += tx_packets; + stats64->tx_dropped += tx_dropped; } } +bool fbnic_check_split_frames(struct bpf_prog *prog, unsigned int mtu, + u32 hds_thresh) +{ + if (!prog) + return false; + + if (prog->aux->xdp_has_frags) + return false; + + return mtu + ETH_HLEN > hds_thresh; +} + +static int fbnic_bpf(struct net_device *netdev, struct netdev_bpf *bpf) +{ + struct bpf_prog *prog = bpf->prog, *prev_prog; + struct fbnic_net *fbn = netdev_priv(netdev); + + if (bpf->command != XDP_SETUP_PROG) + return -EINVAL; + + if (fbnic_check_split_frames(prog, netdev->mtu, + fbn->hds_thresh)) { + NL_SET_ERR_MSG_MOD(bpf->extack, + "MTU too high, or HDS threshold is too low for single buffer XDP"); + return -EOPNOTSUPP; + } + + prev_prog = xchg(&fbn->xdp_prog, prog); + if (prev_prog) + bpf_prog_put(prev_prog); + + return 0; +} + static const struct net_device_ops fbnic_netdev_ops = { .ndo_open = fbnic_open, .ndo_stop = fbnic_stop, @@ -513,6 +535,7 @@ static const struct net_device_ops fbnic_netdev_ops = { .ndo_set_mac_address = fbnic_set_mac, .ndo_set_rx_mode = fbnic_set_rx_mode, .ndo_get_stats64 = fbnic_get_stats64, + .ndo_bpf = fbnic_bpf, .ndo_hwtstamp_get = fbnic_hwtstamp_get, .ndo_hwtstamp_set = fbnic_hwtstamp_set, }; @@ -549,12 +572,12 @@ static void fbnic_get_queue_stats_rx(struct net_device *dev, int idx, fbnic_get_hw_q_stats(fbd, fbd->hw_stats.hw_q); - spin_lock(&fbd->hw_stats_lock); + spin_lock(&fbd->hw_stats.lock); rx->hw_drop_overruns = fbd->hw_stats.hw_q[idx].rde_pkt_cq_drop.value + fbd->hw_stats.hw_q[idx].rde_pkt_bdq_drop.value; rx->hw_drops = fbd->hw_stats.hw_q[idx].rde_pkt_err.value + rx->hw_drop_overruns; - spin_unlock(&fbd->hw_stats_lock); + spin_unlock(&fbd->hw_stats.lock); } static void fbnic_get_queue_stats_tx(struct net_device *dev, int idx, @@ -564,6 +587,7 @@ static void fbnic_get_queue_stats_tx(struct net_device *dev, int idx, struct fbnic_ring *txr = fbn->tx[idx]; struct fbnic_queue_stats *stats; u64 stop, wake, csum, lso; + struct fbnic_ring *xdpr; unsigned int start; u64 bytes, packets; @@ -587,6 +611,19 @@ static void fbnic_get_queue_stats_tx(struct net_device *dev, int idx, tx->hw_gso_wire_packets = lso; tx->stop = stop; tx->wake = wake; + + xdpr = fbn->tx[FBNIC_MAX_TXQS + idx]; + if (xdpr) { + stats = &xdpr->stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + bytes = stats->bytes; + packets = stats->packets; + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + tx->bytes += bytes; + tx->packets += packets; + } } static void fbnic_get_base_stats(struct net_device *dev, @@ -674,6 +711,8 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) netdev->netdev_ops = &fbnic_netdev_ops; netdev->stat_ops = &fbnic_stat_ops; + netdev->queue_mgmt_ops = &fbnic_queue_mgmt_ops; + netdev->netmem_tx = true; fbnic_set_ethtool_ops(netdev); @@ -691,6 +730,10 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) fbn->rx_usecs = FBNIC_RX_USECS_DEFAULT; fbn->rx_max_frames = FBNIC_RX_FRAMES_DEFAULT; + /* Initialize the hds_thresh */ + netdev->cfg->hds_thresh = FBNIC_HDS_THRESH_DEFAULT; + fbn->hds_thresh = FBNIC_HDS_THRESH_DEFAULT; + default_queues = netif_get_num_default_rss_queues(); if (default_queues > fbd->max_num_queues) default_queues = fbd->max_num_queues; @@ -736,8 +779,6 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) */ netdev->ethtool->wol_enabled = true; - fbn->fec = FBNIC_FEC_AUTO | FBNIC_FEC_RS; - fbn->link_mode = FBNIC_LINK_AUTO | FBNIC_LINK_50R2; netif_carrier_off(netdev); netif_tx_stop_all_queues(netdev); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h index 561837e80ec8..e84e0527c3a9 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h @@ -4,8 +4,8 @@ #ifndef _FBNIC_NETDEV_H_ #define _FBNIC_NETDEV_H_ -#include <linux/types.h> #include <linux/phylink.h> +#include <linux/types.h> #include "fbnic_csr.h" #include "fbnic_rpc.h" @@ -18,7 +18,9 @@ #define FBNIC_TUN_GSO_FEATURES NETIF_F_GSO_IPXIP6 struct fbnic_net { - struct fbnic_ring *tx[FBNIC_MAX_TXQS]; + struct bpf_prog *xdp_prog; + + struct fbnic_ring *tx[FBNIC_MAX_TXQS + FBNIC_MAX_XDPQS]; struct fbnic_ring *rx[FBNIC_MAX_RXQS]; struct fbnic_napi_vector *napi[FBNIC_MAX_NAPI_VECTORS]; @@ -31,6 +33,8 @@ struct fbnic_net { u32 ppq_size; u32 rcq_size; + u32 hds_thresh; + u16 rx_usecs; u16 tx_usecs; @@ -42,9 +46,8 @@ struct fbnic_net { struct phylink_config phylink_config; struct phylink_pcs phylink_pcs; - /* TBD: Remove these when phylink supports FEC and lane config */ + u8 aui; u8 fec; - u8 link_mode; /* Cached top bits of the HW time counter for 40b -> 64b conversion */ u32 time_high; @@ -67,7 +70,7 @@ struct fbnic_net { struct fbnic_queue_stats rx_stats; u64 link_down_events; - /* Time stampinn filter config */ + /* Time stamping filter config */ struct kernel_hwtstamp_config hwtstamp_config; }; @@ -82,6 +85,7 @@ int fbnic_netdev_register(struct net_device *netdev); void fbnic_netdev_unregister(struct net_device *netdev); void fbnic_reset_queues(struct fbnic_net *fbn, unsigned int tx, unsigned int rx); + void fbnic_set_ethtool_ops(struct net_device *dev); int fbnic_ptp_setup(struct fbnic_dev *fbd); @@ -90,8 +94,19 @@ void fbnic_time_init(struct fbnic_net *fbn); int fbnic_time_start(struct fbnic_net *fbn); void fbnic_time_stop(struct fbnic_net *fbn); -void __fbnic_set_rx_mode(struct net_device *netdev); -void fbnic_clear_rx_mode(struct net_device *netdev); - +void __fbnic_set_rx_mode(struct fbnic_dev *fbd); +void fbnic_clear_rx_mode(struct fbnic_dev *fbd); + +void fbnic_phylink_get_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause); +int fbnic_phylink_set_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause); +int fbnic_phylink_ethtool_ksettings_get(struct net_device *netdev, + struct ethtool_link_ksettings *cmd); +int fbnic_phylink_get_fecparam(struct net_device *netdev, + struct ethtool_fecparam *fecparam); int fbnic_phylink_init(struct net_device *netdev); + +bool fbnic_check_split_frames(struct bpf_prog *prog, + unsigned int mtu, u32 hds_threshold); #endif /* _FBNIC_NETDEV_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index 249d3ef862d5..a7a6b4db8016 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -118,14 +118,12 @@ static void fbnic_service_task_start(struct fbnic_net *fbn) struct fbnic_dev *fbd = fbn->fbd; schedule_delayed_work(&fbd->service_task, HZ); - phylink_resume(fbn->phylink); } static void fbnic_service_task_stop(struct fbnic_net *fbn) { struct fbnic_dev *fbd = fbn->fbd; - phylink_suspend(fbn->phylink, fbnic_bmc_present(fbd)); cancel_delayed_work(&fbd->service_task); } @@ -137,7 +135,7 @@ void fbnic_up(struct fbnic_net *fbn) fbnic_rss_reinit_hw(fbn->fbd, fbn); - __fbnic_set_rx_mode(fbn->netdev); + __fbnic_set_rx_mode(fbn->fbd); /* Enable Tx/Rx processing */ fbnic_napi_enable(fbn); @@ -154,7 +152,7 @@ void fbnic_down_noidle(struct fbnic_net *fbn) fbnic_napi_disable(fbn); netif_tx_disable(fbn->netdev); - fbnic_clear_rx_mode(fbn->netdev); + fbnic_clear_rx_mode(fbn->fbd); fbnic_clear_rules(fbn->fbd); fbnic_rss_disable_hw(fbn->fbd); fbnic_disable(fbn); @@ -169,6 +167,20 @@ void fbnic_down(struct fbnic_net *fbn) fbnic_flush(fbn); } +static int fbnic_fw_config_after_crash(struct fbnic_dev *fbd) +{ + if (fbnic_fw_xmit_ownership_msg(fbd, true)) { + dev_err(fbd->dev, "NIC failed to take ownership\n"); + + return -1; + } + + fbnic_rpc_reset_valid_entries(fbd); + __fbnic_set_rx_mode(fbd); + + return 0; +} + static void fbnic_health_check(struct fbnic_dev *fbd) { struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; @@ -184,13 +196,11 @@ static void fbnic_health_check(struct fbnic_dev *fbd) if (tx_mbx->head != tx_mbx->tail) return; - /* TBD: Need to add a more thorough recovery here. - * Specifically I need to verify what all the firmware will have - * changed since we had setup and it rebooted. May just need to - * perform a down/up. For now we will just reclaim ownership so - * the heartbeat can catch the next fault. - */ - fbnic_fw_xmit_ownership_msg(fbd, true); + fbnic_devlink_fw_report(fbd, "Firmware crashed detected!"); + fbnic_devlink_otp_check(fbd, "error detected after firmware recovery"); + + if (fbnic_fw_config_after_crash(fbd)) + dev_err(fbd->dev, "Firmware recovery failed after crash\n"); } static void fbnic_service_task(struct work_struct *work) @@ -206,8 +216,13 @@ static void fbnic_service_task(struct work_struct *work) fbnic_health_check(fbd); - if (netif_carrier_ok(fbd->netdev)) + fbnic_bmc_rpc_check(fbd); + + if (netif_carrier_ok(fbd->netdev)) { + netdev_lock(fbd->netdev); fbnic_napi_depletion_check(fbd->netdev); + netdev_unlock(fbd->netdev); + } if (netif_running(fbd->netdev)) schedule_delayed_work(&fbd->service_task, HZ); @@ -266,6 +281,10 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENOMEM; } + err = fbnic_devlink_health_create(fbd); + if (err) + goto free_fbd; + /* Populate driver with hardware-specific info and handlers */ fbd->max_num_queues = info->max_num_queues; @@ -276,7 +295,7 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err = fbnic_alloc_irqs(fbd); if (err) - goto free_fbd; + goto err_destroy_health; err = fbnic_mac_init(fbd); if (err) { @@ -291,12 +310,23 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto free_irqs; } + /* Send the request to enable the FW logging to host. Note if this + * fails we ignore the error and just display a message as it is + * possible the FW is just too old to support the logging and needs + * to be updated. + */ + err = fbnic_fw_log_init(fbd); + if (err) + dev_warn(fbd->dev, + "Unable to initialize firmware log buffer: %d\n", + err); + fbnic_devlink_register(fbd); + fbnic_devlink_otp_check(fbd, "error detected during probe"); fbnic_dbg_fbd_init(fbd); - spin_lock_init(&fbd->hw_stats_lock); /* Capture snapshot of hardware stats so netdev can calculate delta */ - fbnic_reset_hw_stats(fbd); + fbnic_init_hw_stats(fbd); fbnic_hwmon_register(fbd); @@ -335,6 +365,8 @@ init_failure_mode: return 0; free_irqs: fbnic_free_irqs(fbd); +err_destroy_health: + fbnic_devlink_health_destroy(fbd); free_fbd: fbnic_devlink_free(fbd); @@ -365,9 +397,11 @@ static void fbnic_remove(struct pci_dev *pdev) fbnic_hwmon_unregister(fbd); fbnic_dbg_fbd_exit(fbd); fbnic_devlink_unregister(fbd); + fbnic_fw_log_free(fbd); fbnic_fw_free_mbx(fbd); fbnic_free_irqs(fbd); + fbnic_devlink_health_destroy(fbd); fbnic_devlink_free(fbd); } @@ -380,15 +414,19 @@ static int fbnic_pm_suspend(struct device *dev) goto null_uc_addr; rtnl_lock(); + netdev_lock(netdev); netif_device_detach(netdev); if (netif_running(netdev)) netdev->netdev_ops->ndo_stop(netdev); + netdev_unlock(netdev); rtnl_unlock(); null_uc_addr: + fbnic_fw_log_disable(fbd); + devl_lock(priv_to_devlink(fbd)); fbnic_fw_free_mbx(fbd); @@ -429,10 +467,17 @@ static int __fbnic_pm_resume(struct device *dev) /* Re-enable mailbox */ err = fbnic_fw_request_mbx(fbd); + devl_unlock(priv_to_devlink(fbd)); if (err) goto err_free_irqs; - devl_unlock(priv_to_devlink(fbd)); + /* Only send log history if log buffer is empty to prevent duplicate + * log entries. + */ + fbnic_fw_log_enable(fbd, list_empty(&fbd->fw_log.entries)); + + /* Since the FW should be up, check if it reported OTP errors */ + fbnic_devlink_otp_check(fbd, "error detected after PM resume"); /* No netdev means there isn't a network interface to bring up */ if (fbnic_init_failure(fbd)) @@ -444,19 +489,23 @@ static int __fbnic_pm_resume(struct device *dev) fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues); rtnl_lock(); + netdev_lock(netdev); - if (netif_running(netdev)) { + if (netif_running(netdev)) err = __fbnic_open(fbn); - if (err) - goto err_free_mbx; - } + netdev_unlock(netdev); rtnl_unlock(); + if (err) + goto err_free_mbx; return 0; err_free_mbx: - rtnl_unlock(); + fbnic_fw_log_disable(fbd); + + devl_lock(priv_to_devlink(fbd)); fbnic_fw_free_mbx(fbd); + devl_unlock(priv_to_devlink(fbd)); err_free_irqs: fbnic_free_irqs(fbd); err_invalidate_uc_addr: @@ -471,6 +520,10 @@ static void __fbnic_pm_attach(struct device *dev) struct net_device *netdev = fbd->netdev; struct fbnic_net *fbn; + rtnl_lock(); + fbnic_reset_hw_stats(fbd); + rtnl_unlock(); + if (fbnic_init_failure(fbd)) return; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c b/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c index 860b02b22c15..7ce3fdd25282 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c @@ -8,6 +8,99 @@ #include "fbnic_mac.h" #include "fbnic_netdev.h" +static phy_interface_t fbnic_phylink_select_interface(u8 aui) +{ + switch (aui) { + case FBNIC_AUI_100GAUI2: + return PHY_INTERFACE_MODE_100GBASEP; + case FBNIC_AUI_50GAUI1: + return PHY_INTERFACE_MODE_50GBASER; + case FBNIC_AUI_LAUI2: + return PHY_INTERFACE_MODE_LAUI; + case FBNIC_AUI_25GAUI: + return PHY_INTERFACE_MODE_25GBASER; + } + + return PHY_INTERFACE_MODE_NA; +} + +void fbnic_phylink_get_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + phylink_ethtool_get_pauseparam(fbn->phylink, pause); +} + +int fbnic_phylink_set_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + return phylink_ethtool_set_pauseparam(fbn->phylink, pause); +} + +static void +fbnic_phylink_get_supported_fec_modes(unsigned long *supported) +{ + /* The NIC can support up to 8 possible combinations. + * Either 50G-CR, or 100G-CR2 + * This is with RS FEC mode only + * Either 25G-CR, or 50G-CR2 + * This is with No FEC, RS, or Base-R + */ + if (phylink_test(supported, 100000baseCR2_Full) || + phylink_test(supported, 50000baseCR_Full)) + phylink_set(supported, FEC_RS); + if (phylink_test(supported, 50000baseCR2_Full) || + phylink_test(supported, 25000baseCR_Full)) { + phylink_set(supported, FEC_BASER); + phylink_set(supported, FEC_NONE); + phylink_set(supported, FEC_RS); + } +} + +int fbnic_phylink_ethtool_ksettings_get(struct net_device *netdev, + struct ethtool_link_ksettings *cmd) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int err; + + err = phylink_ethtool_ksettings_get(fbn->phylink, cmd); + if (!err) { + unsigned long *supp = cmd->link_modes.supported; + + cmd->base.port = PORT_DA; + cmd->lanes = (fbn->aui & FBNIC_AUI_MODE_R2) ? 2 : 1; + + fbnic_phylink_get_supported_fec_modes(supp); + } + + return err; +} + +int fbnic_phylink_get_fecparam(struct net_device *netdev, + struct ethtool_fecparam *fecparam) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + if (fbn->fec & FBNIC_FEC_RS) { + fecparam->active_fec = ETHTOOL_FEC_RS; + fecparam->fec = ETHTOOL_FEC_RS; + } else if (fbn->fec & FBNIC_FEC_BASER) { + fecparam->active_fec = ETHTOOL_FEC_BASER; + fecparam->fec = ETHTOOL_FEC_BASER; + } else { + fecparam->active_fec = ETHTOOL_FEC_OFF; + fecparam->fec = ETHTOOL_FEC_OFF; + } + + if (fbn->aui & FBNIC_AUI_MODE_PAM4) + fecparam->fec |= ETHTOOL_FEC_AUTO; + + return 0; +} + static struct fbnic_net * fbnic_pcs_to_net(struct phylink_pcs *pcs) { @@ -21,23 +114,20 @@ fbnic_phylink_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode, struct fbnic_net *fbn = fbnic_pcs_to_net(pcs); struct fbnic_dev *fbd = fbn->fbd; - /* For now we use hard-coded defaults and FW config to determine - * the current values. In future patches we will add support for - * reconfiguring these values and changing link settings. - */ - switch (fbd->fw_cap.link_speed) { - case FBNIC_FW_LINK_SPEED_25R1: + switch (fbn->aui) { + case FBNIC_AUI_25GAUI: state->speed = SPEED_25000; break; - case FBNIC_FW_LINK_SPEED_50R2: + case FBNIC_AUI_LAUI2: + case FBNIC_AUI_50GAUI1: state->speed = SPEED_50000; break; - case FBNIC_FW_LINK_SPEED_100R2: + case FBNIC_AUI_100GAUI2: state->speed = SPEED_100000; break; default: - state->speed = SPEED_UNKNOWN; - break; + state->link = 0; + return; } state->duplex = DUPLEX_FULL; @@ -131,6 +221,7 @@ static const struct phylink_mac_ops fbnic_phylink_mac_ops = { int fbnic_phylink_init(struct net_device *netdev) { struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; struct phylink *phylink; fbn->phylink_pcs.ops = &fbnic_phylink_pcs_ops; @@ -138,18 +229,23 @@ int fbnic_phylink_init(struct net_device *netdev) fbn->phylink_config.dev = &netdev->dev; fbn->phylink_config.type = PHYLINK_NETDEV; fbn->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_ASYM_PAUSE | - MAC_10000FD | MAC_25000FD | - MAC_40000FD | MAC_50000FD | + MAC_25000FD | MAC_50000FD | MAC_100000FD; fbn->phylink_config.default_an_inband = true; - __set_bit(PHY_INTERFACE_MODE_XGMII, + __set_bit(PHY_INTERFACE_MODE_100GBASEP, + fbn->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_50GBASER, fbn->phylink_config.supported_interfaces); - __set_bit(PHY_INTERFACE_MODE_XLGMII, + __set_bit(PHY_INTERFACE_MODE_LAUI, fbn->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_25GBASER, + fbn->phylink_config.supported_interfaces); + + fbnic_mac_get_fw_settings(fbd, &fbn->aui, &fbn->fec); phylink = phylink_create(&fbn->phylink_config, NULL, - PHY_INTERFACE_MODE_XLGMII, + fbnic_phylink_select_interface(fbn->aui), &fbnic_phylink_mac_ops); if (IS_ERR(phylink)) return PTR_ERR(phylink); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c index 8ff07b5562e3..7f31e890031c 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c @@ -6,6 +6,7 @@ #include <net/ipv6.h> #include "fbnic.h" +#include "fbnic_fw.h" #include "fbnic_netdev.h" #include "fbnic_rpc.h" @@ -71,6 +72,8 @@ u16 fbnic_flow_hash_2_rss_en_mask(struct fbnic_net *fbn, int flow_type) rss_en_mask |= FBNIC_FH_2_RSSEM_BIT(IP_DST, IP_DST, flow_hash); rss_en_mask |= FBNIC_FH_2_RSSEM_BIT(L4_B_0_1, L4_SRC, flow_hash); rss_en_mask |= FBNIC_FH_2_RSSEM_BIT(L4_B_2_3, L4_DST, flow_hash); + rss_en_mask |= FBNIC_FH_2_RSSEM_BIT(IP6_FL, OV6_FL_LBL, flow_hash); + rss_en_mask |= FBNIC_FH_2_RSSEM_BIT(IP6_FL, IV6_FL_LBL, flow_hash); return rss_en_mask; } @@ -129,12 +132,9 @@ void fbnic_bmc_rpc_all_multi_config(struct fbnic_dev *fbd, else clear_bit(FBNIC_MAC_ADDR_T_ALLMULTI, mac_addr->act_tcam); - } else if (!test_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam) && - !is_zero_ether_addr(mac_addr->mask.addr8) && - mac_addr->state == FBNIC_TCAM_S_VALID) { - clear_bit(FBNIC_MAC_ADDR_T_ALLMULTI, mac_addr->act_tcam); - clear_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam); - mac_addr->state = FBNIC_TCAM_S_DELETE; + } else { + __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_BMC); + __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_ALLMULTI); } /* We have to add a special handler for multicast as the @@ -236,8 +236,25 @@ void fbnic_bmc_rpc_init(struct fbnic_dev *fbd) act_tcam->mask.tcam[j] = 0xffff; act_tcam->state = FBNIC_TCAM_S_UPDATE; +} + +void fbnic_bmc_rpc_check(struct fbnic_dev *fbd) +{ + int err; + + if (fbd->fw_cap.need_bmc_tcam_reinit) { + fbnic_bmc_rpc_init(fbd); + __fbnic_set_rx_mode(fbd); + fbd->fw_cap.need_bmc_tcam_reinit = false; + } - fbnic_bmc_rpc_all_multi_config(fbd, false); + if (fbd->fw_cap.need_bmc_macda_sync) { + err = fbnic_fw_xmit_rpc_macda_sync(fbd); + if (err) + dev_warn(fbd->dev, + "Writing MACDA table to FW failed, err: %d\n", err); + fbd->fw_cap.need_bmc_macda_sync = false; + } } #define FBNIC_ACT1_INIT(_l4, _udp, _ip, _v6) \ @@ -452,6 +469,50 @@ int __fbnic_xc_unsync(struct fbnic_mac_addr *mac_addr, unsigned int tcam_idx) return 0; } +void fbnic_promisc_sync(struct fbnic_dev *fbd, + bool uc_promisc, bool mc_promisc) +{ + struct fbnic_mac_addr *mac_addr; + + /* Populate last TCAM entry with promiscuous entry and 0/1 bit mask */ + mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_PROMISC_IDX]; + if (uc_promisc) { + if (!is_zero_ether_addr(mac_addr->value.addr8) || + mac_addr->state != FBNIC_TCAM_S_VALID) { + eth_zero_addr(mac_addr->value.addr8); + eth_broadcast_addr(mac_addr->mask.addr8); + clear_bit(FBNIC_MAC_ADDR_T_ALLMULTI, + mac_addr->act_tcam); + set_bit(FBNIC_MAC_ADDR_T_PROMISC, + mac_addr->act_tcam); + mac_addr->state = FBNIC_TCAM_S_ADD; + } + } else if (mc_promisc && + (!fbnic_bmc_present(fbd) || !fbd->fw_cap.all_multi)) { + /* We have to add a special handler for multicast as the + * BMC may have an all-multi rule already in place. As such + * adding a rule ourselves won't do any good so we will have + * to modify the rules for the ALL MULTI below if the BMC + * already has the rule in place. + */ + if (!is_multicast_ether_addr(mac_addr->value.addr8) || + mac_addr->state != FBNIC_TCAM_S_VALID) { + eth_zero_addr(mac_addr->value.addr8); + eth_broadcast_addr(mac_addr->mask.addr8); + mac_addr->value.addr8[0] ^= 1; + mac_addr->mask.addr8[0] ^= 1; + set_bit(FBNIC_MAC_ADDR_T_ALLMULTI, + mac_addr->act_tcam); + clear_bit(FBNIC_MAC_ADDR_T_PROMISC, + mac_addr->act_tcam); + mac_addr->state = FBNIC_TCAM_S_ADD; + } + } else if (mac_addr->state == FBNIC_TCAM_S_VALID) { + __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_ALLMULTI); + __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_PROMISC); + } +} + void fbnic_sift_macda(struct fbnic_dev *fbd) { int dest, src; @@ -535,6 +596,21 @@ static void fbnic_clear_macda(struct fbnic_dev *fbd) } } +static void fbnic_clear_valid_macda(struct fbnic_dev *fbd) +{ + int idx; + + for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx]; + + if (mac_addr->state == FBNIC_TCAM_S_VALID) { + fbnic_clear_macda_entry(fbd, idx); + + mac_addr->state = FBNIC_TCAM_S_UPDATE; + } + } +} + static void fbnic_write_macda_entry(struct fbnic_dev *fbd, unsigned int idx, struct fbnic_mac_addr *mac_addr) { @@ -556,7 +632,7 @@ static void fbnic_write_macda_entry(struct fbnic_dev *fbd, unsigned int idx, void fbnic_write_macda(struct fbnic_dev *fbd) { - int idx; + int idx, updates = 0; for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) { struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx]; @@ -565,6 +641,9 @@ void fbnic_write_macda(struct fbnic_dev *fbd) if (!(mac_addr->state & FBNIC_TCAM_S_UPDATE)) continue; + /* Record update count */ + updates++; + /* Clear by writing 0s. */ if (mac_addr->state == FBNIC_TCAM_S_DELETE) { /* Invalidate entry and clear addr state info */ @@ -578,6 +657,14 @@ void fbnic_write_macda(struct fbnic_dev *fbd) mac_addr->state = FBNIC_TCAM_S_VALID; } + + /* If reinitializing the BMC TCAM we are doing an initial update */ + if (fbd->fw_cap.need_bmc_tcam_reinit) + updates++; + + /* If needed notify firmware of changes to MACDA TCAM */ + if (updates != 0 && fbnic_bmc_present(fbd)) + fbd->fw_cap.need_bmc_macda_sync = true; } static void fbnic_clear_act_tcam(struct fbnic_dev *fbd, unsigned int idx) @@ -1052,13 +1139,25 @@ void fbnic_write_ip_addr(struct fbnic_dev *fbd) } } -void fbnic_clear_rules(struct fbnic_dev *fbd) +static void fbnic_clear_valid_act_tcam(struct fbnic_dev *fbd) { - u32 dest = FIELD_PREP(FBNIC_RPC_ACT_TBL0_DEST_MASK, - FBNIC_RPC_ACT_TBL0_DEST_BMC); int i = FBNIC_RPC_TCAM_ACT_NUM_ENTRIES - 1; struct fbnic_act_tcam *act_tcam; + /* Work from the bottom up deleting all other rules from hardware */ + do { + act_tcam = &fbd->act_tcam[i]; + + if (act_tcam->state != FBNIC_TCAM_S_VALID) + continue; + + fbnic_clear_act_tcam(fbd, i); + act_tcam->state = FBNIC_TCAM_S_UPDATE; + } while (i--); +} + +void fbnic_clear_rules(struct fbnic_dev *fbd) +{ /* Clear MAC rules */ fbnic_clear_macda(fbd); @@ -1073,6 +1172,11 @@ void fbnic_clear_rules(struct fbnic_dev *fbd) * the interface back up. */ if (fbnic_bmc_present(fbd)) { + u32 dest = FIELD_PREP(FBNIC_RPC_ACT_TBL0_DEST_MASK, + FBNIC_RPC_ACT_TBL0_DEST_BMC); + int i = FBNIC_RPC_TCAM_ACT_NUM_ENTRIES - 1; + struct fbnic_act_tcam *act_tcam; + act_tcam = &fbd->act_tcam[i]; if (act_tcam->state == FBNIC_TCAM_S_VALID && @@ -1081,21 +1185,10 @@ void fbnic_clear_rules(struct fbnic_dev *fbd) wr32(fbd, FBNIC_RPC_ACT_TBL1(i), 0); act_tcam->state = FBNIC_TCAM_S_UPDATE; - - i--; } } - /* Work from the bottom up deleting all other rules from hardware */ - do { - act_tcam = &fbd->act_tcam[i]; - - if (act_tcam->state != FBNIC_TCAM_S_VALID) - continue; - - fbnic_clear_act_tcam(fbd, i); - act_tcam->state = FBNIC_TCAM_S_UPDATE; - } while (i--); + fbnic_clear_valid_act_tcam(fbd); } static void fbnic_delete_act_tcam(struct fbnic_dev *fbd, unsigned int idx) @@ -1145,3 +1238,9 @@ void fbnic_write_rules(struct fbnic_dev *fbd) fbnic_update_act_tcam(fbd, i); } } + +void fbnic_rpc_reset_valid_entries(struct fbnic_dev *fbd) +{ + fbnic_clear_valid_act_tcam(fbd); + fbnic_clear_valid_macda(fbd); +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.h b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.h index 6892414195c3..3d4925b2ac75 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.h @@ -184,6 +184,7 @@ struct fbnic_net; void fbnic_bmc_rpc_init(struct fbnic_dev *fbd); void fbnic_bmc_rpc_all_multi_config(struct fbnic_dev *fbd, bool enable_host); +void fbnic_bmc_rpc_check(struct fbnic_dev *fbd); void fbnic_reset_indir_tbl(struct fbnic_net *fbn); void fbnic_rss_key_fill(u32 *buffer); @@ -201,6 +202,9 @@ struct fbnic_mac_addr *__fbnic_mc_sync(struct fbnic_dev *fbd, void fbnic_sift_macda(struct fbnic_dev *fbd); void fbnic_write_macda(struct fbnic_dev *fbd); +void fbnic_promisc_sync(struct fbnic_dev *fbd, + bool uc_promisc, bool mc_promisc); + struct fbnic_ip_addr *__fbnic_ip4_sync(struct fbnic_dev *fbd, struct fbnic_ip_addr *ip_addr, const struct in_addr *addr, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index ac11389a764c..cf773cc78e40 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -2,11 +2,14 @@ /* Copyright (c) Meta Platforms, Inc. and affiliates. */ #include <linux/bitfield.h> +#include <linux/bpf.h> +#include <linux/bpf_trace.h> #include <linux/iopoll.h> #include <linux/pci.h> #include <net/netdev_queues.h> #include <net/page_pool/helpers.h> #include <net/tcp.h> +#include <net/xdp.h> #include "fbnic.h" #include "fbnic_csr.h" @@ -14,6 +17,13 @@ #include "fbnic_txrx.h" enum { + FBNIC_XDP_PASS = 0, + FBNIC_XDP_CONSUME, + FBNIC_XDP_TX, + FBNIC_XDP_LEN_ERR, +}; + +enum { FBNIC_XMIT_CB_TS = 0x01, }; @@ -27,6 +37,8 @@ struct fbnic_xmit_cb { #define FBNIC_XMIT_CB(__skb) ((struct fbnic_xmit_cb *)((__skb)->cb)) +#define FBNIC_XMIT_NOUNMAP ((void *)1) + static u32 __iomem *fbnic_ring_csr_base(const struct fbnic_ring *ring) { unsigned long csr_base = (unsigned long)ring->doorbell; @@ -305,6 +317,7 @@ fbnic_tx_map(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) unsigned int tail = ring->tail, first; unsigned int size, data_len; skb_frag_t *frag; + bool is_net_iov; dma_addr_t dma; __le64 *twd; @@ -320,6 +333,7 @@ fbnic_tx_map(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) if (size > FIELD_MAX(FBNIC_TWD_LEN_MASK)) goto dma_error; + is_net_iov = false; dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); for (frag = &skb_shinfo(skb)->frags[0];; frag++) { @@ -332,6 +346,8 @@ fbnic_tx_map(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) FIELD_PREP(FBNIC_TWD_LEN_MASK, size) | FIELD_PREP(FBNIC_TWD_TYPE_MASK, FBNIC_TWD_TYPE_AL)); + if (is_net_iov) + ring->tx_buf[tail] = FBNIC_XMIT_NOUNMAP; tail++; tail &= ring->size_mask; @@ -345,6 +361,7 @@ fbnic_tx_map(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) if (size > FIELD_MAX(FBNIC_TWD_LEN_MASK)) goto dma_error; + is_net_iov = skb_frag_is_net_iov(frag); dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); } @@ -380,6 +397,8 @@ dma_error: twd = &ring->desc[tail]; if (tail == first) fbnic_unmap_single_twd(dev, twd); + else if (ring->tx_buf[tail] == FBNIC_XMIT_NOUNMAP) + ring->tx_buf[tail] = NULL; else fbnic_unmap_page_twd(dev, twd); } @@ -564,7 +583,11 @@ static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget, desc_cnt--; while (desc_cnt--) { - fbnic_unmap_page_twd(nv->dev, &ring->desc[head]); + if (ring->tx_buf[head] != FBNIC_XMIT_NOUNMAP) + fbnic_unmap_page_twd(nv->dev, + &ring->desc[head]); + else + ring->tx_buf[head] = NULL; head++; head &= ring->size_mask; } @@ -606,6 +629,54 @@ static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget, } } +static void fbnic_clean_twq1(struct fbnic_napi_vector *nv, bool pp_allow_direct, + struct fbnic_ring *ring, bool discard, + unsigned int hw_head) +{ + u64 total_bytes = 0, total_packets = 0; + unsigned int head = ring->head; + + while (hw_head != head) { + struct page *page; + u64 twd; + + if (unlikely(!(ring->desc[head] & FBNIC_TWD_TYPE(AL)))) + goto next_desc; + + twd = le64_to_cpu(ring->desc[head]); + page = ring->tx_buf[head]; + + /* TYPE_AL is 2, TYPE_LAST_AL is 3. So this trick gives + * us one increment per packet, with no branches. + */ + total_packets += FIELD_GET(FBNIC_TWD_TYPE_MASK, twd) - + FBNIC_TWD_TYPE_AL; + total_bytes += FIELD_GET(FBNIC_TWD_LEN_MASK, twd); + + page_pool_put_page(page->pp, page, -1, pp_allow_direct); +next_desc: + head++; + head &= ring->size_mask; + } + + if (!total_bytes) + return; + + ring->head = head; + + if (discard) { + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.dropped += total_packets; + u64_stats_update_end(&ring->stats.syncp); + return; + } + + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.bytes += total_bytes; + ring->stats.packets += total_packets; + u64_stats_update_end(&ring->stats.syncp); +} + static void fbnic_clean_tsq(struct fbnic_napi_vector *nv, struct fbnic_ring *ring, u64 tcd, int *ts_head, int *head0) @@ -657,44 +728,65 @@ static void fbnic_clean_tsq(struct fbnic_napi_vector *nv, } static void fbnic_page_pool_init(struct fbnic_ring *ring, unsigned int idx, - struct page *page) + netmem_ref netmem) { struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; - page_pool_fragment_page(page, PAGECNT_BIAS_MAX); - rx_buf->pagecnt_bias = PAGECNT_BIAS_MAX; - rx_buf->page = page; + page_pool_fragment_netmem(netmem, FBNIC_PAGECNT_BIAS_MAX); + rx_buf->pagecnt_bias = FBNIC_PAGECNT_BIAS_MAX; + rx_buf->netmem = netmem; } -static struct page *fbnic_page_pool_get(struct fbnic_ring *ring, - unsigned int idx) +static struct page * +fbnic_page_pool_get_head(struct fbnic_q_triad *qt, unsigned int idx) { - struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; + struct fbnic_rx_buf *rx_buf = &qt->sub0.rx_buf[idx]; rx_buf->pagecnt_bias--; - return rx_buf->page; + /* sub0 is always fed system pages, from the NAPI-level page_pool */ + return netmem_to_page(rx_buf->netmem); +} + +static netmem_ref +fbnic_page_pool_get_data(struct fbnic_q_triad *qt, unsigned int idx) +{ + struct fbnic_rx_buf *rx_buf = &qt->sub1.rx_buf[idx]; + + rx_buf->pagecnt_bias--; + + return rx_buf->netmem; } static void fbnic_page_pool_drain(struct fbnic_ring *ring, unsigned int idx, - struct fbnic_napi_vector *nv, int budget) + int budget) { struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; - struct page *page = rx_buf->page; + netmem_ref netmem = rx_buf->netmem; - if (!page_pool_unref_page(page, rx_buf->pagecnt_bias)) - page_pool_put_unrefed_page(nv->page_pool, page, -1, !!budget); + if (!page_pool_unref_netmem(netmem, rx_buf->pagecnt_bias)) + page_pool_put_unrefed_netmem(ring->page_pool, netmem, -1, + !!budget); - rx_buf->page = NULL; + rx_buf->netmem = 0; } static void fbnic_clean_twq(struct fbnic_napi_vector *nv, int napi_budget, - struct fbnic_q_triad *qt, s32 ts_head, s32 head0) + struct fbnic_q_triad *qt, s32 ts_head, s32 head0, + s32 head1) { if (head0 >= 0) fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, head0); else if (ts_head >= 0) fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, ts_head); + + if (head1 >= 0) { + qt->cmpl.deferred_head = -1; + if (napi_budget) + fbnic_clean_twq1(nv, true, &qt->sub1, false, head1); + else + qt->cmpl.deferred_head = head1; + } } static void @@ -702,6 +794,7 @@ fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt, int napi_budget) { struct fbnic_ring *cmpl = &qt->cmpl; + s32 head1 = cmpl->deferred_head; s32 head0 = -1, ts_head = -1; __le64 *raw_tcd, done; u32 head = cmpl->head; @@ -719,7 +812,10 @@ fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt, switch (FIELD_GET(FBNIC_TCD_TYPE_MASK, tcd)) { case FBNIC_TCD_TYPE_0: - if (!(tcd & FBNIC_TCD_TWQ1)) + if (tcd & FBNIC_TCD_TWQ1) + head1 = FIELD_GET(FBNIC_TCD_TYPE0_HEAD1_MASK, + tcd); + else head0 = FIELD_GET(FBNIC_TCD_TYPE0_HEAD0_MASK, tcd); /* Currently all err status bits are related to @@ -752,11 +848,11 @@ fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt, } /* Unmap and free processed buffers */ - fbnic_clean_twq(nv, napi_budget, qt, ts_head, head0); + fbnic_clean_twq(nv, napi_budget, qt, ts_head, head0, head1); } -static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget, - struct fbnic_ring *ring, unsigned int hw_head) +static void fbnic_clean_bdq(struct fbnic_ring *ring, unsigned int hw_head, + int napi_budget) { unsigned int head = ring->head; @@ -764,7 +860,7 @@ static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget, return; do { - fbnic_page_pool_drain(ring, head, nv, napi_budget); + fbnic_page_pool_drain(ring, head, napi_budget); head++; head &= ring->size_mask; @@ -773,10 +869,10 @@ static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget, ring->head = head; } -static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, struct page *page) +static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, netmem_ref netmem) { __le64 *bdq_desc = &bdq->desc[id * FBNIC_BD_FRAG_COUNT]; - dma_addr_t dma = page_pool_get_dma_addr(page); + dma_addr_t dma = page_pool_get_dma_addr_netmem(netmem); u64 bd, i = FBNIC_BD_FRAG_COUNT; bd = (FBNIC_BD_PAGE_ADDR_MASK & dma) | @@ -794,7 +890,7 @@ static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, struct page *page) } while (--i); } -static void fbnic_fill_bdq(struct fbnic_napi_vector *nv, struct fbnic_ring *bdq) +static void fbnic_fill_bdq(struct fbnic_ring *bdq) { unsigned int count = fbnic_desc_unused(bdq); unsigned int i = bdq->tail; @@ -803,10 +899,10 @@ static void fbnic_fill_bdq(struct fbnic_napi_vector *nv, struct fbnic_ring *bdq) return; do { - struct page *page; + netmem_ref netmem; - page = page_pool_dev_alloc_pages(nv->page_pool); - if (!page) { + netmem = page_pool_dev_alloc_netmems(bdq->page_pool); + if (!netmem) { u64_stats_update_begin(&bdq->stats.syncp); bdq->stats.rx.alloc_failed++; u64_stats_update_end(&bdq->stats.syncp); @@ -814,8 +910,8 @@ static void fbnic_fill_bdq(struct fbnic_napi_vector *nv, struct fbnic_ring *bdq) break; } - fbnic_page_pool_init(bdq, i, page); - fbnic_bd_prep(bdq, i, page); + fbnic_page_pool_init(bdq, i, netmem); + fbnic_bd_prep(bdq, i, netmem); i++; i &= bdq->size_mask; @@ -862,7 +958,7 @@ static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd, { unsigned int hdr_pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); unsigned int hdr_pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); - struct page *page = fbnic_page_pool_get(&qt->sub0, hdr_pg_idx); + struct page *page = fbnic_page_pool_get_head(qt, hdr_pg_idx); unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); unsigned int frame_sz, hdr_pg_start, hdr_pg_end, headroom; unsigned char *hdr_start; @@ -877,7 +973,7 @@ static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd, headroom = hdr_pg_off - hdr_pg_start + FBNIC_RX_PAD; frame_sz = hdr_pg_end - hdr_pg_start; - xdp_init_buff(&pkt->buff, frame_sz, NULL); + xdp_init_buff(&pkt->buff, frame_sz, &qt->xdp_rxq); hdr_pg_start += (FBNIC_RCD_AL_BUFF_FRAG_MASK & rcd) * FBNIC_BD_FRAG_SIZE; @@ -888,13 +984,12 @@ static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd, /* Build frame around buffer */ hdr_start = page_address(page) + hdr_pg_start; - + net_prefetch(pkt->buff.data); xdp_prepare_buff(&pkt->buff, hdr_start, headroom, len - FBNIC_RX_PAD, true); - pkt->data_truesize = 0; - pkt->data_len = 0; - pkt->nr_frags = 0; + pkt->hwtstamp = 0; + pkt->add_frag_failed = false; } static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd, @@ -904,9 +999,9 @@ static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd, unsigned int pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); unsigned int pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); - struct page *page = fbnic_page_pool_get(&qt->sub1, pg_idx); - struct skb_shared_info *shinfo; + netmem_ref netmem = fbnic_page_pool_get_data(qt, pg_idx); unsigned int truesize; + bool added; truesize = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? FBNIC_BD_FRAG_SIZE - pg_off : ALIGN(len, 128); @@ -915,88 +1010,171 @@ static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd, FBNIC_BD_FRAG_SIZE; /* Sync DMA buffer */ - dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page), - pg_off, truesize, DMA_BIDIRECTIONAL); - - /* Add page to xdp shared info */ - shinfo = xdp_get_shared_info_from_buff(&pkt->buff); - - /* We use gso_segs to store truesize */ - pkt->data_truesize += truesize; - - __skb_fill_page_desc_noacc(shinfo, pkt->nr_frags++, page, pg_off, len); - - /* Store data_len in gso_size */ - pkt->data_len += len; + page_pool_dma_sync_netmem_for_cpu(qt->sub1.page_pool, netmem, + pg_off, truesize); + + added = xdp_buff_add_frag(&pkt->buff, netmem, pg_off, len, truesize); + if (unlikely(!added)) { + pkt->add_frag_failed = true; + netdev_err_once(nv->napi.dev, + "Failed to add fragment to xdp_buff\n"); + } } -static void fbnic_put_pkt_buff(struct fbnic_napi_vector *nv, +static void fbnic_put_pkt_buff(struct fbnic_q_triad *qt, struct fbnic_pkt_buff *pkt, int budget) { - struct skb_shared_info *shinfo; struct page *page; - int nr_frags; if (!pkt->buff.data_hard_start) return; - shinfo = xdp_get_shared_info_from_buff(&pkt->buff); - nr_frags = pkt->nr_frags; + if (xdp_buff_has_frags(&pkt->buff)) { + struct skb_shared_info *shinfo; + netmem_ref netmem; + int nr_frags; + + shinfo = xdp_get_shared_info_from_buff(&pkt->buff); + nr_frags = shinfo->nr_frags; - while (nr_frags--) { - page = skb_frag_page(&shinfo->frags[nr_frags]); - page_pool_put_full_page(nv->page_pool, page, !!budget); + while (nr_frags--) { + netmem = skb_frag_netmem(&shinfo->frags[nr_frags]); + page_pool_put_full_netmem(qt->sub1.page_pool, netmem, + !!budget); + } } page = virt_to_page(pkt->buff.data_hard_start); - page_pool_put_full_page(nv->page_pool, page, !!budget); + page_pool_put_full_page(qt->sub0.page_pool, page, !!budget); } static struct sk_buff *fbnic_build_skb(struct fbnic_napi_vector *nv, struct fbnic_pkt_buff *pkt) { - unsigned int nr_frags = pkt->nr_frags; - struct skb_shared_info *shinfo; - unsigned int truesize; struct sk_buff *skb; - truesize = xdp_data_hard_end(&pkt->buff) + FBNIC_RX_TROOM - - pkt->buff.data_hard_start; - - /* Build frame around buffer */ - skb = napi_build_skb(pkt->buff.data_hard_start, truesize); - if (unlikely(!skb)) + skb = xdp_build_skb_from_buff(&pkt->buff); + if (!skb) return NULL; - /* Push data pointer to start of data, put tail to end of data */ - skb_reserve(skb, pkt->buff.data - pkt->buff.data_hard_start); - __skb_put(skb, pkt->buff.data_end - pkt->buff.data); + /* Add timestamp if present */ + if (pkt->hwtstamp) + skb_hwtstamps(skb)->hwtstamp = pkt->hwtstamp; + + return skb; +} - /* Add tracking for metadata at the start of the frame */ - skb_metadata_set(skb, pkt->buff.data - pkt->buff.data_meta); +static long fbnic_pkt_tx(struct fbnic_napi_vector *nv, + struct fbnic_pkt_buff *pkt) +{ + struct fbnic_ring *ring = &nv->qt[0].sub1; + int size, offset, nsegs = 1, data_len = 0; + unsigned int tail = ring->tail; + struct skb_shared_info *shinfo; + skb_frag_t *frag = NULL; + struct page *page; + dma_addr_t dma; + __le64 *twd; - /* Add Rx frags */ - if (nr_frags) { - /* Verify that shared info didn't move */ + if (unlikely(xdp_buff_has_frags(&pkt->buff))) { shinfo = xdp_get_shared_info_from_buff(&pkt->buff); - WARN_ON(skb_shinfo(skb) != shinfo); + nsegs += shinfo->nr_frags; + data_len = shinfo->xdp_frags_size; + frag = &shinfo->frags[0]; + } - skb->truesize += pkt->data_truesize; - skb->data_len += pkt->data_len; - shinfo->nr_frags = nr_frags; - skb->len += pkt->data_len; + if (fbnic_desc_unused(ring) < nsegs) { + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.dropped++; + u64_stats_update_end(&ring->stats.syncp); + return -FBNIC_XDP_CONSUME; } - skb_mark_for_recycle(skb); + page = virt_to_page(pkt->buff.data_hard_start); + offset = offset_in_page(pkt->buff.data); + dma = page_pool_get_dma_addr(page); - /* Set MAC header specific fields */ - skb->protocol = eth_type_trans(skb, nv->napi.dev); + size = pkt->buff.data_end - pkt->buff.data; - /* Add timestamp if present */ - if (pkt->hwtstamp) - skb_hwtstamps(skb)->hwtstamp = pkt->hwtstamp; + while (nsegs--) { + dma_sync_single_range_for_device(nv->dev, dma, offset, size, + DMA_BIDIRECTIONAL); + dma += offset; - return skb; + ring->tx_buf[tail] = page; + + twd = &ring->desc[tail]; + *twd = cpu_to_le64(FIELD_PREP(FBNIC_TWD_ADDR_MASK, dma) | + FIELD_PREP(FBNIC_TWD_LEN_MASK, size) | + FIELD_PREP(FBNIC_TWD_TYPE_MASK, + FBNIC_TWD_TYPE_AL)); + + tail++; + tail &= ring->size_mask; + + if (!data_len) + break; + + offset = skb_frag_off(frag); + page = skb_frag_page(frag); + dma = page_pool_get_dma_addr(page); + + size = skb_frag_size(frag); + data_len -= size; + frag++; + } + + *twd |= FBNIC_TWD_TYPE(LAST_AL); + + ring->tail = tail; + + return -FBNIC_XDP_TX; +} + +static void fbnic_pkt_commit_tail(struct fbnic_napi_vector *nv, + unsigned int pkt_tail) +{ + struct fbnic_ring *ring = &nv->qt[0].sub1; + + /* Force DMA writes to flush before writing to tail */ + dma_wmb(); + + writel(pkt_tail, ring->doorbell); +} + +static struct sk_buff *fbnic_run_xdp(struct fbnic_napi_vector *nv, + struct fbnic_pkt_buff *pkt) +{ + struct fbnic_net *fbn = netdev_priv(nv->napi.dev); + struct bpf_prog *xdp_prog; + int act; + + xdp_prog = READ_ONCE(fbn->xdp_prog); + if (!xdp_prog) + goto xdp_pass; + + /* Should never happen, config paths enforce HDS threshold > MTU */ + if (xdp_buff_has_frags(&pkt->buff) && !xdp_prog->aux->xdp_has_frags) + return ERR_PTR(-FBNIC_XDP_LEN_ERR); + + act = bpf_prog_run_xdp(xdp_prog, &pkt->buff); + switch (act) { + case XDP_PASS: +xdp_pass: + return fbnic_build_skb(nv, pkt); + case XDP_TX: + return ERR_PTR(fbnic_pkt_tx(nv, pkt)); + default: + bpf_warn_invalid_xdp_action(nv->napi.dev, xdp_prog, act); + fallthrough; + case XDP_ABORTED: + trace_xdp_exception(nv->napi.dev, xdp_prog, act); + fallthrough; + case XDP_DROP: + break; + } + + return ERR_PTR(-FBNIC_XDP_CONSUME); } static enum pkt_hash_types fbnic_skb_hash_type(u64 rcd) @@ -1050,10 +1228,10 @@ static int fbnic_clean_rcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt, int budget) { unsigned int packets = 0, bytes = 0, dropped = 0, alloc_failed = 0; - u64 csum_complete = 0, csum_none = 0; + u64 csum_complete = 0, csum_none = 0, length_errors = 0; + s32 head0 = -1, head1 = -1, pkt_tail = -1; struct fbnic_ring *rcq = &qt->cmpl; struct fbnic_pkt_buff *pkt; - s32 head0 = -1, head1 = -1; __le64 *raw_rcd, done; u32 head = rcq->head; @@ -1094,8 +1272,10 @@ static int fbnic_clean_rcq(struct fbnic_napi_vector *nv, /* We currently ignore the action table index */ break; case FBNIC_RCD_TYPE_META: - if (likely(!fbnic_rcd_metadata_err(rcd))) - skb = fbnic_build_skb(nv, pkt); + if (unlikely(pkt->add_frag_failed)) + skb = NULL; + else if (likely(!fbnic_rcd_metadata_err(rcd))) + skb = fbnic_run_xdp(nv, pkt); /* Populate skb and invalidate XDP */ if (!IS_ERR_OR_NULL(skb)) { @@ -1107,15 +1287,20 @@ static int fbnic_clean_rcq(struct fbnic_napi_vector *nv, bytes += skb->len; napi_gro_receive(&nv->napi, skb); + } else if (skb == ERR_PTR(-FBNIC_XDP_TX)) { + pkt_tail = nv->qt[0].sub1.tail; + bytes += xdp_get_buff_len(&pkt->buff); } else { if (!skb) { alloc_failed++; dropped++; + } else if (skb == ERR_PTR(-FBNIC_XDP_LEN_ERR)) { + length_errors++; } else { dropped++; } - fbnic_put_pkt_buff(nv, pkt, 1); + fbnic_put_pkt_buff(qt, pkt, 1); } pkt->buff.data_hard_start = NULL; @@ -1140,16 +1325,20 @@ static int fbnic_clean_rcq(struct fbnic_napi_vector *nv, rcq->stats.rx.alloc_failed += alloc_failed; rcq->stats.rx.csum_complete += csum_complete; rcq->stats.rx.csum_none += csum_none; + rcq->stats.rx.length_errors += length_errors; u64_stats_update_end(&rcq->stats.syncp); + if (pkt_tail >= 0) + fbnic_pkt_commit_tail(nv, pkt_tail); + /* Unmap and free processed buffers */ if (head0 >= 0) - fbnic_clean_bdq(nv, budget, &qt->sub0, head0); - fbnic_fill_bdq(nv, &qt->sub0); + fbnic_clean_bdq(&qt->sub0, head0, budget); + fbnic_fill_bdq(&qt->sub0); if (head1 >= 0) - fbnic_clean_bdq(nv, budget, &qt->sub1, head1); - fbnic_fill_bdq(nv, &qt->sub1); + fbnic_clean_bdq(&qt->sub1, head1, budget); + fbnic_fill_bdq(&qt->sub1); /* Record the current head/tail of the queue */ if (rcq->head != head) { @@ -1220,8 +1409,9 @@ void fbnic_aggregate_ring_rx_counters(struct fbnic_net *fbn, fbn->rx_stats.rx.alloc_failed += stats->rx.alloc_failed; fbn->rx_stats.rx.csum_complete += stats->rx.csum_complete; fbn->rx_stats.rx.csum_none += stats->rx.csum_none; + fbn->rx_stats.rx.length_errors += stats->rx.length_errors; /* Remember to add new stats here */ - BUILD_BUG_ON(sizeof(fbn->rx_stats.rx) / 8 != 3); + BUILD_BUG_ON(sizeof(fbn->rx_stats.rx) / 8 != 4); } void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn, @@ -1243,6 +1433,22 @@ void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn, BUILD_BUG_ON(sizeof(fbn->tx_stats.twq) / 8 != 6); } +static void fbnic_aggregate_ring_xdp_counters(struct fbnic_net *fbn, + struct fbnic_ring *xdpr) +{ + struct fbnic_queue_stats *stats = &xdpr->stats; + + if (!(xdpr->flags & FBNIC_RING_F_STATS)) + return; + + /* Capture stats from queues before dissasociating them */ + fbn->rx_stats.bytes += stats->bytes; + fbn->rx_stats.packets += stats->packets; + fbn->rx_stats.dropped += stats->dropped; + fbn->tx_stats.bytes += stats->bytes; + fbn->tx_stats.packets += stats->packets; +} + static void fbnic_remove_tx_ring(struct fbnic_net *fbn, struct fbnic_ring *txr) { @@ -1256,6 +1462,19 @@ static void fbnic_remove_tx_ring(struct fbnic_net *fbn, fbn->tx[txr->q_idx] = NULL; } +static void fbnic_remove_xdp_ring(struct fbnic_net *fbn, + struct fbnic_ring *xdpr) +{ + if (!(xdpr->flags & FBNIC_RING_F_STATS)) + return; + + fbnic_aggregate_ring_xdp_counters(fbn, xdpr); + + /* Remove pointer to the Tx ring */ + WARN_ON(fbn->tx[xdpr->q_idx] && fbn->tx[xdpr->q_idx] != xdpr); + fbn->tx[xdpr->q_idx] = NULL; +} + static void fbnic_remove_rx_ring(struct fbnic_net *fbn, struct fbnic_ring *rxr) { @@ -1269,6 +1488,12 @@ static void fbnic_remove_rx_ring(struct fbnic_net *fbn, fbn->rx[rxr->q_idx] = NULL; } +static void fbnic_free_qt_page_pools(struct fbnic_q_triad *qt) +{ + page_pool_destroy(qt->sub0.page_pool); + page_pool_destroy(qt->sub1.page_pool); +} + static void fbnic_free_napi_vector(struct fbnic_net *fbn, struct fbnic_napi_vector *nv) { @@ -1277,6 +1502,7 @@ static void fbnic_free_napi_vector(struct fbnic_net *fbn, for (i = 0; i < nv->txt_count; i++) { fbnic_remove_tx_ring(fbn, &nv->qt[i].sub0); + fbnic_remove_xdp_ring(fbn, &nv->qt[i].sub1); fbnic_remove_tx_ring(fbn, &nv->qt[i].cmpl); } @@ -1287,8 +1513,7 @@ static void fbnic_free_napi_vector(struct fbnic_net *fbn, } fbnic_napi_free_irq(fbd, nv); - page_pool_destroy(nv->page_pool); - netif_napi_del(&nv->napi); + netif_napi_del_locked(&nv->napi); fbn->napi[fbnic_napi_idx(nv)] = NULL; kfree(nv); } @@ -1302,23 +1527,22 @@ void fbnic_free_napi_vectors(struct fbnic_net *fbn) fbnic_free_napi_vector(fbn, fbn->napi[i]); } -#define FBNIC_PAGE_POOL_FLAGS \ - (PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV) - -static int fbnic_alloc_nv_page_pool(struct fbnic_net *fbn, - struct fbnic_napi_vector *nv) +static int +fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt, + unsigned int rxq_idx) { struct page_pool_params pp_params = { .order = 0, - .flags = FBNIC_PAGE_POOL_FLAGS, - .pool_size = (fbn->hpq_size + fbn->ppq_size) * nv->rxt_count, + .flags = PP_FLAG_DMA_MAP | + PP_FLAG_DMA_SYNC_DEV, + .pool_size = fbn->hpq_size + fbn->ppq_size, .nid = NUMA_NO_NODE, - .dev = nv->dev, + .dev = fbn->netdev->dev.parent, .dma_dir = DMA_BIDIRECTIONAL, .offset = 0, .max_len = PAGE_SIZE, - .napi = &nv->napi, .netdev = fbn->netdev, + .queue_idx = rxq_idx, }; struct page_pool *pp; @@ -1338,9 +1562,24 @@ static int fbnic_alloc_nv_page_pool(struct fbnic_net *fbn, if (IS_ERR(pp)) return PTR_ERR(pp); - nv->page_pool = pp; + qt->sub0.page_pool = pp; + if (netif_rxq_has_unreadable_mp(fbn->netdev, rxq_idx)) { + pp_params.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM; + pp_params.dma_dir = DMA_FROM_DEVICE; + + pp = page_pool_create(&pp_params); + if (IS_ERR(pp)) + goto err_destroy_sub0; + } else { + page_pool_get(pp); + } + qt->sub1.page_pool = pp; return 0; + +err_destroy_sub0: + page_pool_destroy(pp); + return PTR_ERR(pp); } static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell, @@ -1350,6 +1589,7 @@ static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell, ring->doorbell = doorbell; ring->q_idx = q_idx; ring->flags = flags; + ring->deferred_head = -1; } static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn, @@ -1359,11 +1599,18 @@ static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn, { int txt_count = txq_count, rxt_count = rxq_count; u32 __iomem *uc_addr = fbd->uc_addr0; + int xdp_count = 0, qt_count, err; struct fbnic_napi_vector *nv; struct fbnic_q_triad *qt; - int qt_count, err; u32 __iomem *db; + /* We need to reserve at least one Tx Queue Triad for an XDP ring */ + if (rxq_count) { + xdp_count = 1; + if (!txt_count) + txt_count = 1; + } + qt_count = txt_count + rxq_count; if (!qt_count) return -EINVAL; @@ -1387,37 +1634,33 @@ static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn, /* Tie napi to netdev */ fbn->napi[fbnic_napi_idx(nv)] = nv; - netif_napi_add(fbn->netdev, &nv->napi, fbnic_poll); + netif_napi_add_config_locked(fbn->netdev, &nv->napi, fbnic_poll, + fbnic_napi_idx(nv)); /* Record IRQ to NAPI struct */ - netif_napi_set_irq(&nv->napi, - pci_irq_vector(to_pci_dev(fbd->dev), nv->v_idx)); + netif_napi_set_irq_locked(&nv->napi, + pci_irq_vector(to_pci_dev(fbd->dev), + nv->v_idx)); /* Tie nv back to PCIe dev */ nv->dev = fbd->dev; - /* Allocate page pool */ - if (rxq_count) { - err = fbnic_alloc_nv_page_pool(fbn, nv); - if (err) - goto napi_del; - } - /* Request the IRQ for napi vector */ err = fbnic_napi_request_irq(fbd, nv); if (err) - goto pp_destroy; + goto napi_del; /* Initialize queue triads */ qt = nv->qt; while (txt_count) { + u8 flags = FBNIC_RING_F_CTX | FBNIC_RING_F_STATS; + /* Configure Tx queue */ db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TWQ0_TAIL]; /* Assign Tx queue to netdev if applicable */ if (txq_count > 0) { - u8 flags = FBNIC_RING_F_CTX | FBNIC_RING_F_STATS; fbnic_ring_init(&qt->sub0, db, txq_idx, flags); fbn->tx[txq_idx] = &qt->sub0; @@ -1427,6 +1670,28 @@ static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn, FBNIC_RING_F_DISABLED); } + /* Configure XDP queue */ + db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TWQ1_TAIL]; + + /* Assign XDP queue to netdev if applicable + * + * The setup for this is in itself a bit different. + * 1. We only need one XDP Tx queue per NAPI vector. + * 2. We associate it to the first Rx queue index. + * 3. The hardware side is associated based on the Tx Queue. + * 4. The netdev queue is offset by FBNIC_MAX_TXQs. + */ + if (xdp_count > 0) { + unsigned int xdp_idx = FBNIC_MAX_TXQS + rxq_idx; + + fbnic_ring_init(&qt->sub1, db, xdp_idx, flags); + fbn->tx[xdp_idx] = &qt->sub1; + xdp_count--; + } else { + fbnic_ring_init(&qt->sub1, db, 0, + FBNIC_RING_F_DISABLED); + } + /* Configure Tx completion queue */ db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TCQ_HEAD]; fbnic_ring_init(&qt->cmpl, db, 0, 0); @@ -1463,10 +1728,8 @@ static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn, return 0; -pp_destroy: - page_pool_destroy(nv->page_pool); napi_del: - netif_napi_del(&nv->napi); + netif_napi_del_locked(&nv->napi); fbn->napi[fbnic_napi_idx(nv)] = NULL; kfree(nv); return err; @@ -1680,6 +1943,12 @@ static void fbnic_free_qt_resources(struct fbnic_net *fbn, fbnic_free_ring_resources(dev, &qt->cmpl); fbnic_free_ring_resources(dev, &qt->sub1); fbnic_free_ring_resources(dev, &qt->sub0); + + if (xdp_rxq_info_is_reg(&qt->xdp_rxq)) { + xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq); + xdp_rxq_info_unreg(&qt->xdp_rxq); + fbnic_free_qt_page_pools(qt); + } } static int fbnic_alloc_tx_qt_resources(struct fbnic_net *fbn, @@ -1692,6 +1961,10 @@ static int fbnic_alloc_tx_qt_resources(struct fbnic_net *fbn, if (err) return err; + err = fbnic_alloc_tx_ring_resources(fbn, &qt->sub1); + if (err) + goto free_sub0; + err = fbnic_alloc_tx_ring_resources(fbn, &qt->cmpl); if (err) goto free_sub1; @@ -1699,20 +1972,37 @@ static int fbnic_alloc_tx_qt_resources(struct fbnic_net *fbn, return 0; free_sub1: + fbnic_free_ring_resources(dev, &qt->sub1); +free_sub0: fbnic_free_ring_resources(dev, &qt->sub0); return err; } static int fbnic_alloc_rx_qt_resources(struct fbnic_net *fbn, + struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt) { struct device *dev = fbn->netdev->dev.parent; int err; - err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub0); + err = fbnic_alloc_qt_page_pools(fbn, qt, qt->cmpl.q_idx); if (err) return err; + err = xdp_rxq_info_reg(&qt->xdp_rxq, fbn->netdev, qt->sub0.q_idx, + nv->napi.napi_id); + if (err) + goto free_page_pools; + + err = xdp_rxq_info_reg_mem_model(&qt->xdp_rxq, MEM_TYPE_PAGE_POOL, + qt->sub0.page_pool); + if (err) + goto unreg_rxq; + + err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub0); + if (err) + goto unreg_mm; + err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub1); if (err) goto free_sub0; @@ -1727,19 +2017,21 @@ free_sub1: fbnic_free_ring_resources(dev, &qt->sub1); free_sub0: fbnic_free_ring_resources(dev, &qt->sub0); +unreg_mm: + xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq); +unreg_rxq: + xdp_rxq_info_unreg(&qt->xdp_rxq); +free_page_pools: + fbnic_free_qt_page_pools(qt); return err; } static void fbnic_free_nv_resources(struct fbnic_net *fbn, struct fbnic_napi_vector *nv) { - int i, j; - - /* Free Tx Resources */ - for (i = 0; i < nv->txt_count; i++) - fbnic_free_qt_resources(fbn, &nv->qt[i]); + int i; - for (j = 0; j < nv->rxt_count; j++, i++) + for (i = 0; i < nv->txt_count + nv->rxt_count; i++) fbnic_free_qt_resources(fbn, &nv->qt[i]); } @@ -1752,19 +2044,19 @@ static int fbnic_alloc_nv_resources(struct fbnic_net *fbn, for (i = 0; i < nv->txt_count; i++) { err = fbnic_alloc_tx_qt_resources(fbn, &nv->qt[i]); if (err) - goto free_resources; + goto free_qt_resources; } /* Allocate Rx Resources */ for (j = 0; j < nv->rxt_count; j++, i++) { - err = fbnic_alloc_rx_qt_resources(fbn, &nv->qt[i]); + err = fbnic_alloc_rx_qt_resources(fbn, nv, &nv->qt[i]); if (err) - goto free_resources; + goto free_qt_resources; } return 0; -free_resources: +free_qt_resources: while (i--) fbnic_free_qt_resources(fbn, &nv->qt[i]); return err; @@ -1871,6 +2163,15 @@ static void fbnic_disable_twq0(struct fbnic_ring *txr) fbnic_ring_wr32(txr, FBNIC_QUEUE_TWQ0_CTL, twq_ctl); } +static void fbnic_disable_twq1(struct fbnic_ring *txr) +{ + u32 twq_ctl = fbnic_ring_rd32(txr, FBNIC_QUEUE_TWQ1_CTL); + + twq_ctl &= ~FBNIC_QUEUE_TWQ_CTL_ENABLE; + + fbnic_ring_wr32(txr, FBNIC_QUEUE_TWQ1_CTL, twq_ctl); +} + static void fbnic_disable_tcq(struct fbnic_ring *txr) { fbnic_ring_wr32(txr, FBNIC_QUEUE_TCQ_CTL, 0); @@ -1897,36 +2198,48 @@ void fbnic_napi_disable(struct fbnic_net *fbn) int i; for (i = 0; i < fbn->num_napi; i++) { - napi_disable(&fbn->napi[i]->napi); + napi_disable_locked(&fbn->napi[i]->napi); fbnic_nv_irq_disable(fbn->napi[i]); } } -void fbnic_disable(struct fbnic_net *fbn) +static void __fbnic_nv_disable(struct fbnic_napi_vector *nv) { - struct fbnic_dev *fbd = fbn->fbd; - int i, j, t; - - for (i = 0; i < fbn->num_napi; i++) { - struct fbnic_napi_vector *nv = fbn->napi[i]; + int i, t; - /* Disable Tx queue triads */ - for (t = 0; t < nv->txt_count; t++) { - struct fbnic_q_triad *qt = &nv->qt[t]; + /* Disable Tx queue triads */ + for (t = 0; t < nv->txt_count; t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; - fbnic_disable_twq0(&qt->sub0); - fbnic_disable_tcq(&qt->cmpl); - } + fbnic_disable_twq0(&qt->sub0); + fbnic_disable_twq1(&qt->sub1); + fbnic_disable_tcq(&qt->cmpl); + } - /* Disable Rx queue triads */ - for (j = 0; j < nv->rxt_count; j++, t++) { - struct fbnic_q_triad *qt = &nv->qt[t]; + /* Disable Rx queue triads */ + for (i = 0; i < nv->rxt_count; i++, t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; - fbnic_disable_bdq(&qt->sub0, &qt->sub1); - fbnic_disable_rcq(&qt->cmpl); - } + fbnic_disable_bdq(&qt->sub0, &qt->sub1); + fbnic_disable_rcq(&qt->cmpl); } +} + +static void +fbnic_nv_disable(struct fbnic_net *fbn, struct fbnic_napi_vector *nv) +{ + __fbnic_nv_disable(nv); + fbnic_wrfl(fbn->fbd); +} + +void fbnic_disable(struct fbnic_net *fbn) +{ + struct fbnic_dev *fbd = fbn->fbd; + int i; + + for (i = 0; i < fbn->num_napi; i++) + __fbnic_nv_disable(fbn->napi[i]); fbnic_wrfl(fbd); } @@ -2015,73 +2328,119 @@ int fbnic_wait_all_queues_idle(struct fbnic_dev *fbd, bool may_fail) return err; } -void fbnic_flush(struct fbnic_net *fbn) +static int +fbnic_wait_queue_idle(struct fbnic_net *fbn, bool rx, unsigned int idx) { - int i; + static const unsigned int tx_regs[] = { + FBNIC_QM_TWQ_IDLE(0), FBNIC_QM_TQS_IDLE(0), + FBNIC_QM_TDE_IDLE(0), FBNIC_QM_TCQ_IDLE(0), + }, rx_regs[] = { + FBNIC_QM_HPQ_IDLE(0), FBNIC_QM_PPQ_IDLE(0), + FBNIC_QM_RCQ_IDLE(0), + }; + struct fbnic_dev *fbd = fbn->fbd; + unsigned int val, mask, off; + const unsigned int *regs; + unsigned int reg_cnt; + int i, err; - for (i = 0; i < fbn->num_napi; i++) { - struct fbnic_napi_vector *nv = fbn->napi[i]; - int j, t; + regs = rx ? rx_regs : tx_regs; + reg_cnt = rx ? ARRAY_SIZE(rx_regs) : ARRAY_SIZE(tx_regs); - /* Flush any processed Tx Queue Triads and drop the rest */ - for (t = 0; t < nv->txt_count; t++) { - struct fbnic_q_triad *qt = &nv->qt[t]; - struct netdev_queue *tx_queue; + off = idx / 32; + mask = BIT(idx % 32); - /* Clean the work queues of unprocessed work */ - fbnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail); + for (i = 0; i < reg_cnt; i++) { + err = read_poll_timeout_atomic(fbnic_rd32, val, val & mask, + 2, 500000, false, + fbd, regs[i] + off); + if (err) { + netdev_err(fbd->netdev, + "wait for queue %s%d idle failed 0x%04x(%d): %08x (mask: %08x)\n", + rx ? "Rx" : "Tx", idx, regs[i] + off, i, + val, mask); + return err; + } + } - /* Reset completion queue descriptor ring */ - memset(qt->cmpl.desc, 0, qt->cmpl.size); + return 0; +} - /* Nothing else to do if Tx queue is disabled */ - if (qt->sub0.flags & FBNIC_RING_F_DISABLED) - continue; +static void fbnic_nv_flush(struct fbnic_napi_vector *nv) +{ + int j, t; - /* Reset BQL associated with Tx queue */ - tx_queue = netdev_get_tx_queue(nv->napi.dev, - qt->sub0.q_idx); - netdev_tx_reset_queue(tx_queue); - } + /* Flush any processed Tx Queue Triads and drop the rest */ + for (t = 0; t < nv->txt_count; t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; + struct netdev_queue *tx_queue; - /* Flush any processed Rx Queue Triads and drop the rest */ - for (j = 0; j < nv->rxt_count; j++, t++) { - struct fbnic_q_triad *qt = &nv->qt[t]; + /* Clean the work queues of unprocessed work */ + fbnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail); + fbnic_clean_twq1(nv, false, &qt->sub1, true, + qt->sub1.tail); - /* Clean the work queues of unprocessed work */ - fbnic_clean_bdq(nv, 0, &qt->sub0, qt->sub0.tail); - fbnic_clean_bdq(nv, 0, &qt->sub1, qt->sub1.tail); + /* Reset completion queue descriptor ring */ + memset(qt->cmpl.desc, 0, qt->cmpl.size); - /* Reset completion queue descriptor ring */ - memset(qt->cmpl.desc, 0, qt->cmpl.size); + /* Nothing else to do if Tx queue is disabled */ + if (qt->sub0.flags & FBNIC_RING_F_DISABLED) + continue; - fbnic_put_pkt_buff(nv, qt->cmpl.pkt, 0); - qt->cmpl.pkt->buff.data_hard_start = NULL; - } + /* Reset BQL associated with Tx queue */ + tx_queue = netdev_get_tx_queue(nv->napi.dev, + qt->sub0.q_idx); + netdev_tx_reset_queue(tx_queue); + } + + /* Flush any processed Rx Queue Triads and drop the rest */ + for (j = 0; j < nv->rxt_count; j++, t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; + + /* Clean the work queues of unprocessed work */ + fbnic_clean_bdq(&qt->sub0, qt->sub0.tail, 0); + fbnic_clean_bdq(&qt->sub1, qt->sub1.tail, 0); + + /* Reset completion queue descriptor ring */ + memset(qt->cmpl.desc, 0, qt->cmpl.size); + + fbnic_put_pkt_buff(qt, qt->cmpl.pkt, 0); + memset(qt->cmpl.pkt, 0, sizeof(struct fbnic_pkt_buff)); } } -void fbnic_fill(struct fbnic_net *fbn) +void fbnic_flush(struct fbnic_net *fbn) { int i; - for (i = 0; i < fbn->num_napi; i++) { - struct fbnic_napi_vector *nv = fbn->napi[i]; - int j, t; + for (i = 0; i < fbn->num_napi; i++) + fbnic_nv_flush(fbn->napi[i]); +} - /* Configure NAPI mapping and populate pages - * in the BDQ rings to use for Rx - */ - for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) { - struct fbnic_q_triad *qt = &nv->qt[t]; +static void fbnic_nv_fill(struct fbnic_napi_vector *nv) +{ + int j, t; - /* Populate the header and payload BDQs */ - fbnic_fill_bdq(nv, &qt->sub0); - fbnic_fill_bdq(nv, &qt->sub1); - } + /* Configure NAPI mapping and populate pages + * in the BDQ rings to use for Rx + */ + for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; + + /* Populate the header and payload BDQs */ + fbnic_fill_bdq(&qt->sub0); + fbnic_fill_bdq(&qt->sub1); } } +void fbnic_fill(struct fbnic_net *fbn) +{ + int i; + + for (i = 0; i < fbn->num_napi; i++) + fbnic_nv_fill(fbn->napi[i]); +} + static void fbnic_enable_twq0(struct fbnic_ring *twq) { u32 log_size = fls(twq->size_mask); @@ -2104,6 +2463,28 @@ static void fbnic_enable_twq0(struct fbnic_ring *twq) fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_ENABLE); } +static void fbnic_enable_twq1(struct fbnic_ring *twq) +{ + u32 log_size = fls(twq->size_mask); + + if (!twq->size_mask) + return; + + /* Reset head/tail */ + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_CTL, FBNIC_QUEUE_TWQ_CTL_RESET); + twq->tail = 0; + twq->head = 0; + + /* Store descriptor ring address and size */ + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_BAL, lower_32_bits(twq->dma)); + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_BAH, upper_32_bits(twq->dma)); + + /* Write lower 4 bits of log size as 64K ring size is 0 */ + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_SIZE, log_size & 0xf); + + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_CTL, FBNIC_QUEUE_TWQ_CTL_ENABLE); +} + static void fbnic_enable_tcq(struct fbnic_napi_vector *nv, struct fbnic_ring *tcq) { @@ -2232,13 +2613,22 @@ static void fbnic_enable_rcq(struct fbnic_napi_vector *nv, { struct fbnic_net *fbn = netdev_priv(nv->napi.dev); u32 log_size = fls(rcq->size_mask); - u32 rcq_ctl; + u32 hds_thresh = fbn->hds_thresh; + u32 rcq_ctl = 0; fbnic_config_drop_mode_rcq(nv, rcq); - rcq_ctl = FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PADLEN_MASK, FBNIC_RX_PAD) | - FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_MAX_HDR_MASK, - FBNIC_RX_MAX_HDR) | + /* Force lower bound on MAX_HEADER_BYTES. Below this, all frames should + * be split at L4. It would also result in the frames being split at + * L2/L3 depending on the frame size. + */ + if (fbn->hds_thresh < FBNIC_HDR_BYTES_MIN) { + rcq_ctl = FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT; + hds_thresh = FBNIC_HDR_BYTES_MIN; + } + + rcq_ctl |= FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PADLEN_MASK, FBNIC_RX_PAD) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_MAX_HDR_MASK, hds_thresh) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_OFF_MASK, FBNIC_RX_PAYLD_OFFSET) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_PG_CL_MASK, @@ -2266,32 +2656,47 @@ static void fbnic_enable_rcq(struct fbnic_napi_vector *nv, fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_ENABLE); } -void fbnic_enable(struct fbnic_net *fbn) +static void __fbnic_nv_enable(struct fbnic_napi_vector *nv) { - struct fbnic_dev *fbd = fbn->fbd; - int i; + int j, t; - for (i = 0; i < fbn->num_napi; i++) { - struct fbnic_napi_vector *nv = fbn->napi[i]; - int j, t; + /* Setup Tx Queue Triads */ + for (t = 0; t < nv->txt_count; t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; - /* Setup Tx Queue Triads */ - for (t = 0; t < nv->txt_count; t++) { - struct fbnic_q_triad *qt = &nv->qt[t]; + fbnic_enable_twq0(&qt->sub0); + fbnic_enable_twq1(&qt->sub1); + fbnic_enable_tcq(nv, &qt->cmpl); + } - fbnic_enable_twq0(&qt->sub0); - fbnic_enable_tcq(nv, &qt->cmpl); - } + /* Setup Rx Queue Triads */ + for (j = 0; j < nv->rxt_count; j++, t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; - /* Setup Rx Queue Triads */ - for (j = 0; j < nv->rxt_count; j++, t++) { - struct fbnic_q_triad *qt = &nv->qt[t]; + page_pool_enable_direct_recycling(qt->sub0.page_pool, + &nv->napi); + page_pool_enable_direct_recycling(qt->sub1.page_pool, + &nv->napi); - fbnic_enable_bdq(&qt->sub0, &qt->sub1); - fbnic_config_drop_mode_rcq(nv, &qt->cmpl); - fbnic_enable_rcq(nv, &qt->cmpl); - } + fbnic_enable_bdq(&qt->sub0, &qt->sub1); + fbnic_config_drop_mode_rcq(nv, &qt->cmpl); + fbnic_enable_rcq(nv, &qt->cmpl); } +} + +static void fbnic_nv_enable(struct fbnic_net *fbn, struct fbnic_napi_vector *nv) +{ + __fbnic_nv_enable(nv); + fbnic_wrfl(fbn->fbd); +} + +void fbnic_enable(struct fbnic_net *fbn) +{ + struct fbnic_dev *fbd = fbn->fbd; + int i; + + for (i = 0; i < fbn->num_napi; i++) + __fbnic_nv_enable(fbn->napi[i]); fbnic_wrfl(fbd); } @@ -2310,7 +2715,7 @@ void fbnic_napi_enable(struct fbnic_net *fbn) for (i = 0; i < fbn->num_napi; i++) { struct fbnic_napi_vector *nv = fbn->napi[i]; - napi_enable(&nv->napi); + napi_enable_locked(&nv->napi); fbnic_nv_irq_enable(nv); @@ -2363,3 +2768,123 @@ void fbnic_napi_depletion_check(struct net_device *netdev) fbnic_wrfl(fbd); } + +static int fbnic_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) +{ + struct fbnic_net *fbn = netdev_priv(dev); + const struct fbnic_q_triad *real; + struct fbnic_q_triad *qt = qmem; + struct fbnic_napi_vector *nv; + + if (!netif_running(dev)) + return fbnic_alloc_qt_page_pools(fbn, qt, idx); + + real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); + nv = fbn->napi[idx % fbn->num_napi]; + + fbnic_ring_init(&qt->sub0, real->sub0.doorbell, real->sub0.q_idx, + real->sub0.flags); + fbnic_ring_init(&qt->sub1, real->sub1.doorbell, real->sub1.q_idx, + real->sub1.flags); + fbnic_ring_init(&qt->cmpl, real->cmpl.doorbell, real->cmpl.q_idx, + real->cmpl.flags); + + return fbnic_alloc_rx_qt_resources(fbn, nv, qt); +} + +static void fbnic_queue_mem_free(struct net_device *dev, void *qmem) +{ + struct fbnic_net *fbn = netdev_priv(dev); + struct fbnic_q_triad *qt = qmem; + + if (!netif_running(dev)) + fbnic_free_qt_page_pools(qt); + else + fbnic_free_qt_resources(fbn, qt); +} + +static void __fbnic_nv_restart(struct fbnic_net *fbn, + struct fbnic_napi_vector *nv) +{ + struct fbnic_dev *fbd = fbn->fbd; + int i; + + fbnic_nv_enable(fbn, nv); + fbnic_nv_fill(nv); + + napi_enable_locked(&nv->napi); + fbnic_nv_irq_enable(nv); + fbnic_wr32(fbd, FBNIC_INTR_SET(nv->v_idx / 32), BIT(nv->v_idx % 32)); + fbnic_wrfl(fbd); + + for (i = 0; i < nv->txt_count; i++) + netif_wake_subqueue(fbn->netdev, nv->qt[i].sub0.q_idx); +} + +static int fbnic_queue_start(struct net_device *dev, void *qmem, int idx) +{ + struct fbnic_net *fbn = netdev_priv(dev); + struct fbnic_napi_vector *nv; + struct fbnic_q_triad *real; + + real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); + nv = fbn->napi[idx % fbn->num_napi]; + + fbnic_aggregate_ring_rx_counters(fbn, &real->sub0); + fbnic_aggregate_ring_rx_counters(fbn, &real->sub1); + fbnic_aggregate_ring_rx_counters(fbn, &real->cmpl); + + memcpy(real, qmem, sizeof(*real)); + + __fbnic_nv_restart(fbn, nv); + + return 0; +} + +static int fbnic_queue_stop(struct net_device *dev, void *qmem, int idx) +{ + struct fbnic_net *fbn = netdev_priv(dev); + const struct fbnic_q_triad *real; + struct fbnic_napi_vector *nv; + int i, t; + int err; + + real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); + nv = fbn->napi[idx % fbn->num_napi]; + + napi_disable_locked(&nv->napi); + fbnic_nv_irq_disable(nv); + + for (i = 0; i < nv->txt_count; i++) + netif_stop_subqueue(dev, nv->qt[i].sub0.q_idx); + fbnic_nv_disable(fbn, nv); + + for (t = 0; t < nv->txt_count + nv->rxt_count; t++) { + err = fbnic_wait_queue_idle(fbn, t >= nv->txt_count, + nv->qt[t].sub0.q_idx); + if (err) + goto err_restart; + } + + fbnic_synchronize_irq(fbn->fbd, nv->v_idx); + fbnic_nv_flush(nv); + + page_pool_disable_direct_recycling(real->sub0.page_pool); + page_pool_disable_direct_recycling(real->sub1.page_pool); + + memcpy(qmem, real, sizeof(*real)); + + return 0; + +err_restart: + __fbnic_nv_restart(fbn, nv); + return err; +} + +const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops = { + .ndo_queue_mem_size = sizeof(struct fbnic_q_triad), + .ndo_queue_mem_alloc = fbnic_queue_mem_alloc, + .ndo_queue_mem_free = fbnic_queue_mem_free, + .ndo_queue_start = fbnic_queue_start, + .ndo_queue_stop = fbnic_queue_stop, +}; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h index f46616af41ea..31fac0ba0902 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h @@ -35,6 +35,7 @@ struct fbnic_net; #define FBNIC_MAX_TXQS 128u #define FBNIC_MAX_RXQS 128u +#define FBNIC_MAX_XDPQS 128u /* These apply to TWQs, TCQ, RCQ */ #define FBNIC_QUEUE_SIZE_MIN 16u @@ -50,10 +51,10 @@ struct fbnic_net; #define FBNIC_RX_TROOM \ SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +#define FBNIC_RX_HROOM_PAD 128 #define FBNIC_RX_HROOM \ - (ALIGN(FBNIC_RX_TROOM + NET_SKB_PAD, 128) - FBNIC_RX_TROOM) + (ALIGN(FBNIC_RX_TROOM + FBNIC_RX_HROOM_PAD, 128) - FBNIC_RX_TROOM) #define FBNIC_RX_PAD 0 -#define FBNIC_RX_MAX_HDR (1536 - FBNIC_RX_PAD) #define FBNIC_RX_PAYLD_OFFSET 0 #define FBNIC_RX_PAYLD_PG_CL 0 @@ -61,12 +62,16 @@ struct fbnic_net; #define FBNIC_RING_F_CTX BIT(1) #define FBNIC_RING_F_STATS BIT(2) /* Ring's stats may be used */ +#define FBNIC_HDS_THRESH_MAX \ + (4096 - FBNIC_RX_HROOM - FBNIC_RX_TROOM - FBNIC_RX_PAD) +#define FBNIC_HDS_THRESH_DEFAULT \ + (1536 - FBNIC_RX_PAD) +#define FBNIC_HDR_BYTES_MIN 128 + struct fbnic_pkt_buff { struct xdp_buff buff; ktime_t hwtstamp; - u32 data_truesize; - u16 data_len; - u16 nr_frags; + bool add_frag_failed; }; struct fbnic_queue_stats { @@ -85,18 +90,17 @@ struct fbnic_queue_stats { u64 alloc_failed; u64 csum_complete; u64 csum_none; + u64 length_errors; } rx; }; u64 dropped; struct u64_stats_sync syncp; }; -/* Pagecnt bias is long max to reserve the last bit to catch overflow - * cases where if we overcharge the bias it will flip over to be negative. - */ -#define PAGECNT_BIAS_MAX LONG_MAX +#define FBNIC_PAGECNT_BIAS_MAX PAGE_SIZE + struct fbnic_rx_buf { - struct page *page; + netmem_ref netmem; long pagecnt_bias; }; @@ -117,6 +121,17 @@ struct fbnic_ring { u32 head, tail; /* Head/Tail of ring */ + union { + /* Rx BDQs only */ + struct page_pool *page_pool; + + /* Deferred_head is used to cache the head for TWQ1 if + * an attempt is made to clean TWQ1 with zero napi_budget. + * We do not use it for any other ring. + */ + s32 deferred_head; + }; + struct fbnic_queue_stats stats; /* Slow path fields follow */ @@ -126,12 +141,12 @@ struct fbnic_ring { struct fbnic_q_triad { struct fbnic_ring sub0, sub1, cmpl; + struct xdp_rxq_info xdp_rxq; }; struct fbnic_napi_vector { struct napi_struct napi; struct device *dev; /* Device for DMA unmapping */ - struct page_pool *page_pool; struct fbnic_dev *fbd; u16 v_idx; @@ -141,8 +156,7 @@ struct fbnic_napi_vector { struct fbnic_q_triad qt[]; }; -#define FBNIC_MAX_TXQS 128u -#define FBNIC_MAX_RXQS 128u +extern const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops; netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev); netdev_features_t |